Mark official release

Release notes

.github/workflows/ci.yml

@@ -64,7 +64,7 @@ jobs:
     - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
     - uses: cachix/install-nix-action@v25
       with:
-        install_url: https://releases.nixos.org/nix/nix-2.13.3/install
+        install_url: https://releases.nixos.org/nix/nix-2.20.3/install
     - uses: cachix/cachix-action@v14
       with:
         name: '${{ env.CACHIX_NAME }}'
@@ -116,7 +116,7 @@ jobs:
         fetch-depth: 0
     - uses: cachix/install-nix-action@v25
       with:
-        install_url: https://releases.nixos.org/nix/nix-2.13.3/install
+        install_url: https://releases.nixos.org/nix/nix-2.20.3/install
     - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
     - run: echo NIX_VERSION="$(nix --experimental-features 'nix-command flakes' eval .\#default.version | tr -d \")" >> $GITHUB_ENV
     - uses: cachix/cachix-action@v14
@@ -153,6 +153,9 @@ jobs:
         IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]')
 
         docker tag nix:$NIX_VERSION $IMAGE_ID:$NIX_VERSION
-        docker tag nix:$NIX_VERSION $IMAGE_ID:master
+        docker tag nix:$NIX_VERSION $IMAGE_ID:latest
         docker push $IMAGE_ID:$NIX_VERSION
+        docker push $IMAGE_ID:latest
+        # deprecated 2024-02-24
+        docker tag nix:$NIX_VERSION $IMAGE_ID:master
         docker push $IMAGE_ID:master

.gitignore

@@ -10,6 +10,7 @@ perl/Makefile.config
 /stamp-h1
 /svn-revision
 /libtool
+/config/config.*
 
 # /doc/manual/
 /doc/manual/*.1
@@ -45,13 +46,16 @@ perl/Makefile.config
 /src/libexpr/parser-tab.hh
 /src/libexpr/parser-tab.output
 /src/libexpr/nix.tbl
+/src/libexpr/tests
 /tests/unit/libexpr/libnixexpr-tests
 
 # /src/libstore/
 *.gen.*
+/src/libstore/tests
 /tests/unit/libstore/libnixstore-tests
 
 # /src/libutil/
+/src/libutil/tests
 /tests/unit/libutil/libnixutil-tests
 
 /src/nix/nix

CONTRIBUTING.md

@@ -67,7 +67,7 @@ Check out the [security policy](https://github.com/NixOS/nix/security/policy).
    - [ ] API documentation in header files
    - [ ] Code and comments are self-explanatory
    - [ ] Commit message explains **why** the change was made
-   - [ ] New feature or incompatible change: updated [release notes](./doc/manual/src/release-notes/rl-next.md)
+   - [ ] New feature or incompatible change: [add a release note](https://nixos.org/manual/nix/stable/contributing/hacking#add-a-release-note)
 
 7. If you need additional feedback or help to getting pull request into shape, ask other contributors using [@mentions](https://docs.github.com/en/get-started/writing-on-github/getting-started-with-writing-and-formatting-on-github/basic-writing-and-formatting-syntax#mentioning-people-and-teams).
 

Makefile

@@ -42,6 +42,7 @@ ifeq ($(ENABLE_FUNCTIONAL_TESTS), yes)
 makefiles += \
   tests/functional/local.mk \
   tests/functional/ca/local.mk \
+  tests/functional/git-hashing/local.mk \
   tests/functional/dyn-drv/local.mk \
   tests/functional/test-libstoreconsumer/local.mk \
   tests/functional/plugins/local.mk
@@ -67,6 +68,7 @@ ifeq ($(OPTIMIZE), 1)
   GLOBAL_LDFLAGS += $(CXXLTO)
 else
   GLOBAL_CXXFLAGS += -O0 -U_FORTIFY_SOURCE
+  unexport NIX_HARDENING_ENABLE
 endif
 
 include mk/platform.mk
@@ -81,7 +83,7 @@ ifdef HOST_WINDOWS
   GLOBAL_LDFLAGS += -Wl,--export-all-symbols
 endif
 
-GLOBAL_CXXFLAGS += -g -Wall -include $(buildprefix)config.h -std=c++2a -I src
+GLOBAL_CXXFLAGS += -g -Wall -Wimplicit-fallthrough -include $(buildprefix)config.h -std=c++2a -I src
 
 # Include the main lib, causing rules to be defined
 

doc/manual/rl-next/better-errors-in-nix-repl.md

@@ -1,40 +0,0 @@
----
-synopsis: Concise error printing in `nix repl`
-prs: 9928
----
-
-Previously, if an element of a list or attribute set threw an error while
-evaluating, `nix repl` would print the entire error (including source location
-information) inline. This output was clumsy and difficult to parse:
-
-```
-nix-repl> { err = builtins.throw "uh oh!"; }
-{ err = «error:
-       … while calling the 'throw' builtin
-         at «string»:1:9:
-            1| { err = builtins.throw "uh oh!"; }
-             |         ^
-
-       error: uh oh!»; }
-```
-
-Now, only the error message is displayed, making the output much more readable.
-```
-nix-repl> { err = builtins.throw "uh oh!"; }
-{ err = «error: uh oh!»; }
-```
-
-However, if the whole expression being evaluated throws an error, source
-locations and (if applicable) a stack trace are printed, just like you'd expect:
-
-```
-nix-repl> builtins.throw "uh oh!"
-error:
-       … while calling the 'throw' builtin
-         at «string»:1:1:
-            1| builtins.throw "uh oh!"
-             | ^
-
-       error: uh oh!
-```
-

doc/manual/rl-next/debugger-locals-for-let-expressions.md

@@ -1,9 +0,0 @@
----
-synopsis: "`--debugger` can now access bindings from `let` expressions"
-prs: 9918
-issues: 8827.
----
-
-Breakpoints and errors in the bindings of a `let` expression can now access
-those bindings in the debugger. Previously, only the body of `let` expressions
-could access those bindings.

doc/manual/rl-next/debugger-on-trace.md

@@ -1,9 +0,0 @@
----
-synopsis: Enter the `--debugger` when `builtins.trace` is called if `debugger-on-trace` is set
-prs: 9914
----
-
-If the `debugger-on-trace` option is set and `--debugger` is given,
-`builtins.trace` calls will behave similarly to `builtins.break` and will enter
-the debug REPL. This is useful for determining where warnings are being emitted
-from.

doc/manual/rl-next/debugger-positions.md

@@ -1,25 +0,0 @@
----
-synopsis: Debugger prints source position information
-prs: 9913
----
-
-The `--debugger` now prints source location information, instead of the
-pointers of source location information. Before:
-
-```
-nix-repl> :bt
-0: while evaluating the attribute 'python311.pythonForBuild.pkgs'
-0x600001522598
-```
-
-After:
-
-```
-0: while evaluating the attribute 'python311.pythonForBuild.pkgs'
-/nix/store/hg65h51xnp74ikahns9hyf3py5mlbbqq-source/overrides/default.nix:132:27
-
-   131|
-   132|       bootstrappingBase = pkgs.${self.python.pythonAttr}.pythonForBuild.pkgs;
-      |                           ^
-   133|     in
-```

doc/manual/rl-next/enter-debugger-more-reliably-in-let-and-calls.md

@@ -1,25 +0,0 @@
----
-synopsis: The `--debugger` will start more reliably in `let` expressions and function calls
-prs: 9917
-issues: 6649
----
-
-Previously, if you attempted to evaluate this file with the debugger:
-
-```nix
-let
-  a = builtins.trace "before inner break" (
-    builtins.break "hello"
-  );
-  b = builtins.trace "before outer break" (
-    builtins.break a
-  );
-in
-  b
-```
-
-Nix would correctly enter the debugger at `builtins.break a`, but if you asked
-it to `:continue`, it would skip over the `builtins.break "hello"` expression
-entirely.
-
-Now, Nix will correctly enter the debugger at both breakpoints.

doc/manual/rl-next/lambda-printing.md

@@ -1,50 +0,0 @@
----
-synopsis: Functions are printed with more detail
-prs: 9606
-issues: 7145
----
-
-Functions and `builtins` are printed with more detail in `nix repl`, `nix
-eval`, `builtins.trace`, and most other places values are printed.
-
-Before:
-
-```
-$ nix repl nixpkgs
-nix-repl> builtins.map
-«primop»
-
-nix-repl> builtins.map lib.id
-«primop-app»
-
-nix-repl> builtins.trace lib.id "my-value"
-trace: <LAMBDA>
-"my-value"
-
-$ nix eval --file functions.nix
-{ id = <LAMBDA>; primop = <PRIMOP>; primop-app = <PRIMOP-APP>; }
-```
-
-After:
-
-```
-$ nix repl nixpkgs
-nix-repl> builtins.map
-«primop map»
-
-nix-repl> builtins.map lib.id
-«partially applied primop map»
-
-nix-repl> builtins.trace lib.id "my-value"
-trace: «lambda id @ /nix/store/8rrzq23h2zq7sv5l2vhw44kls5w0f654-source/lib/trivial.nix:26:5»
-"my-value"
-
-$ nix eval --file functions.nix
-{ id = «lambda id @ /Users/wiggles/nix/functions.nix:2:8»; primop = «primop map»; primop-app = «partially applied primop map»; }
-```
-
-This was actually released in Nix 2.20, but wasn't added to the release notes
-so we're announcing it here. The historical release notes have been updated as well.
-
-[type-error]: https://github.com/NixOS/nix/pull/9753
-[coercion-error]: https://github.com/NixOS/nix/pull/9754

doc/manual/rl-next/leading-period.md

@@ -1,10 +0,0 @@
----
-synopsis: Store paths are allowed to start with `.`
-issues: 912
-prs: 9867 9091 9095 9120 9121 9122 9130 9219 9224
----
-
-Leading periods were allowed by accident in Nix 2.4. The Nix team has considered this to be a bug, but this behavior has since been relied on by users, leading to unnecessary difficulties.
-From now on, leading periods are officially, definitively supported. The names `.` and `..` are disallowed, as well as those starting with `.-` or `..-`.
-
-Nix versions that denied leading periods are documented [in the issue](https://github.com/NixOS/nix/issues/912#issuecomment-1919583286).

doc/manual/rl-next/more-commands-respect-ctrl-c.md

@@ -1,13 +0,0 @@
----
-synopsis: Nix commands respect Ctrl-C
-prs: 9687 6995
-issues: 7245
----
-
-Previously, many Nix commands would hang indefinitely if Ctrl-C was pressed
-while performing various operations (including `nix develop`, `nix flake
-update`, and so on). With several fixes to Nix's signal handlers, Nix commands
-will now exit quickly after Ctrl-C is pressed.
-
-This was actually released in Nix 2.20, but wasn't added to the release notes
-so we're announcing it here. The historical release notes have been updated as well.

doc/manual/rl-next/pretty-print-in-nix-repl.md

@@ -1,24 +0,0 @@
----
-synopsis: "`nix repl` pretty-prints values"
-prs: 9931
----
-
-`nix repl` will now pretty-print values:
-
-```
-{
-  attrs = {
-    a = {
-      b = {
-        c = { };
-      };
-    };
-  };
-  list = [ 1 ];
-  list' = [
-    1
-    2
-    3
-  ];
-}
-```

doc/manual/rl-next/reduce-debugger-clutter.md

@@ -1,37 +0,0 @@
----
-synopsis: "Visual clutter in `--debugger` is reduced"
-prs: 9919
----
-
-Before:
-```
-info: breakpoint reached
-
-
-Starting REPL to allow you to inspect the current state of the evaluator.
-
-Welcome to Nix 2.20.0pre20231222_dirty. Type :? for help.
-
-nix-repl> :continue
-error: uh oh
-
-
-Starting REPL to allow you to inspect the current state of the evaluator.
-
-Welcome to Nix 2.20.0pre20231222_dirty. Type :? for help.
-
-nix-repl>
-```
-
-After:
-
-```
-info: breakpoint reached
-
-Nix 2.20.0pre20231222_dirty debugger
-Type :? for help.
-nix-repl> :continue
-error: uh oh
-
-nix-repl>
-```

doc/manual/rl-next/repl-ctrl-c-while-printing.md

@@ -1,8 +0,0 @@
----
-synopsis: "`nix repl` now respects Ctrl-C while printing values"
-prs: 9927
----
-
-`nix repl` will now halt immediately when Ctrl-C is pressed while it's printing
-a value. This is useful if you got curious about what would happen if you
-printed all of Nixpkgs.

doc/manual/rl-next/repl-cycle-detection.md

@@ -1,22 +0,0 @@
----
-synopsis: Cycle detection in `nix repl` is simpler and more reliable
-prs: 9926
-issues: 8672
----
-
-The cycle detection in `nix repl`, `nix eval`, `builtins.trace`, and everywhere
-else values are printed is now simpler and matches the cycle detection in
-`nix-instantiate --eval` output.
-
-Before:
-
-```
-nix eval --expr 'let self = { inherit self; }; in self'
-{ self = { self = «repeated»; }; }
-```
-
-After:
-
-```
-{ self = «repeated»; }
-```

doc/manual/rl-next/stack-size-macos.md

@@ -1,9 +0,0 @@
----
-synopsis: Stack size is increased on macOS
-prs: 9860
----
-
-Previously, Nix would set the stack size to 64MiB on Linux, but would leave the
-stack size set to the default (approximately 8KiB) on macOS. Now, the stack
-size is correctly set to 64MiB on macOS as well, which should reduce stack
-overflow segfaults in deeply-recursive Nix expressions.

doc/manual/src/SUMMARY.md.in

@@ -121,6 +121,7 @@
   - [C++ style guide](contributing/cxx.md)
 - [Release Notes](release-notes/index.md)
 {{#include ./SUMMARY-rl-next.md}}
+  - [Release 2.21 (2024-03-11)](release-notes/rl-2.21.md)
   - [Release 2.20 (2024-01-29)](release-notes/rl-2.20.md)
   - [Release 2.19 (2023-11-17)](release-notes/rl-2.19.md)
   - [Release 2.18 (2023-09-20)](release-notes/rl-2.18.md)

doc/manual/src/glossary.md

@@ -86,7 +86,7 @@
 
   [store path]: #gloss-store-path
 
-- [file system object]{#gloss-store-object}
+- [file system object]{#gloss-file-system-object}
 
   The Nix data model for representing simplified file system data.
 

doc/manual/src/language/index.md

@@ -432,6 +432,32 @@ This is an incomplete overview of language features, by example.
 
   </td>
  </tr>
+ <tr>
+  <td>
+
+   `inherit pkgs src;`
+
+  </td>
+  <td>
+
+   Adds the variables to the current scope (attribute set or `let` binding).
+   Desugars to `pkgs = pkgs; src = src;`
+
+  </td>
+ </tr>
+ <tr>
+  <td>
+
+   `inherit (pkgs) lib stdenv;`
+
+  </td>
+  <td>
+
+   Adds the attributes, from the attribute set in parentheses, to the current scope (attribute set or `let` binding).
+   Desugars to `lib = pkgs.lib; stdenv = pkgs.stdenv;`
+
+  </td>
+ </tr>
  <tr>
   <td>
 

doc/manual/src/protocols/store-path.md

@@ -89,15 +89,20 @@ where
 
       - `rec` = one of:
 
+        - ```ebnf
+          | ""
+          ```
+          (empty string) for hashes of the flat (single file) serialization
+
         - ```ebnf
           | "r:"
           ```
           hashes of the for [Nix Archive (NAR)] (arbitrary file system object) serialization
 
         - ```ebnf
-          | ""
+          | "git:"
           ```
-          (empty string) for hashes of the flat (single file) serialization
+          hashes of the [Git blob/tree](https://git-scm.com/book/en/v2/Git-Internals-Git-Objects) [Merkel tree](https://en.wikipedia.org/wiki/Merkle_tree) format
 
       - ```ebnf
         algo = "md5" | "sha1" | "sha256"

doc/manual/src/quick-start.md

@@ -34,7 +34,7 @@ For more in-depth information you are kindly referred to subsequent chapters.
    lolcat: command not found
    ```
 
-1. Search for more packages on <search.nixos.org> to try them out.
+1. Search for more packages on [search.nixos.org](https://search.nixos.org/) to try them out.
 
 1. Free up storage space:
 

doc/manual/src/release-notes/rl-2.21.md

@@ -0,0 +1,302 @@
+# Release 2.21.0 (2024-03-11)
+
+- Fix a fixed-output derivation sandbox escape (CVE-2024-27297)
+
+  Cooperating Nix derivations could send file descriptors to files in the Nix
+  store to each other via Unix domain sockets in the abstract namespace. This
+  allowed one derivation to modify the output of the other derivation, after Nix
+  has registered the path as "valid" and immutable in the Nix database.
+  In particular, this allowed the output of fixed-output derivations to be
+  modified from their expected content.
+
+  This isn't the case any more.
+
+- CLI options `--arg-from-file` and `--arg-from-stdin` [#10122](https://github.com/NixOS/nix/pull/10122)
+
+  The new CLI option `--arg-from-file` *name* *path* passes the contents
+  of file *path* as a string value via the function argument *name* to a
+  Nix expression. Similarly, the new option `--arg-from-stdin` *name*
+  reads the contents of the string from standard input.
+
+- Concise error printing in `nix repl` [#9928](https://github.com/NixOS/nix/pull/9928)
+
+  Previously, if an element of a list or attribute set threw an error while
+  evaluating, `nix repl` would print the entire error (including source location
+  information) inline. This output was clumsy and difficult to parse:
+
+  ```
+  nix-repl> { err = builtins.throw "uh oh!"; }
+  { err = «error:
+         … while calling the 'throw' builtin
+           at «string»:1:9:
+              1| { err = builtins.throw "uh oh!"; }
+               |         ^
+
+         error: uh oh!»; }
+  ```
+
+  Now, only the error message is displayed, making the output much more readable.
+  ```
+  nix-repl> { err = builtins.throw "uh oh!"; }
+  { err = «error: uh oh!»; }
+  ```
+
+  However, if the whole expression being evaluated throws an error, source
+  locations and (if applicable) a stack trace are printed, just like you'd expect:
+
+  ```
+  nix-repl> builtins.throw "uh oh!"
+  error:
+         … while calling the 'throw' builtin
+           at «string»:1:1:
+              1| builtins.throw "uh oh!"
+               | ^
+
+         error: uh oh!
+  ```
+
+- `--debugger` can now access bindings from `let` expressions [#8827](https://github.com/NixOS/nix/issues/8827) [#9918](https://github.com/NixOS/nix/pull/9918)
+
+  Breakpoints and errors in the bindings of a `let` expression can now access
+  those bindings in the debugger. Previously, only the body of `let` expressions
+  could access those bindings.
+
+- Enter the `--debugger` when `builtins.trace` is called if `debugger-on-trace` is set [#9914](https://github.com/NixOS/nix/pull/9914)
+
+  If the `debugger-on-trace` option is set and `--debugger` is given,
+  `builtins.trace` calls will behave similarly to `builtins.break` and will enter
+  the debug REPL. This is useful for determining where warnings are being emitted
+  from.
+
+- Debugger prints source position information [#9913](https://github.com/NixOS/nix/pull/9913)
+
+  The `--debugger` now prints source location information, instead of the
+  pointers of source location information. Before:
+
+  ```
+  nix-repl> :bt
+  0: while evaluating the attribute 'python311.pythonForBuild.pkgs'
+  0x600001522598
+  ```
+
+  After:
+
+  ```
+  0: while evaluating the attribute 'python311.pythonForBuild.pkgs'
+  /nix/store/hg65h51xnp74ikahns9hyf3py5mlbbqq-source/overrides/default.nix:132:27
+
+     131|
+     132|       bootstrappingBase = pkgs.${self.python.pythonAttr}.pythonForBuild.pkgs;
+        |                           ^
+     133|     in
+  ```
+
+- The `--debugger` will start more reliably in `let` expressions and function calls [#6649](https://github.com/NixOS/nix/issues/6649) [#9917](https://github.com/NixOS/nix/pull/9917)
+
+  Previously, if you attempted to evaluate this file with the debugger:
+
+  ```nix
+  let
+    a = builtins.trace "before inner break" (
+      builtins.break "hello"
+    );
+    b = builtins.trace "before outer break" (
+      builtins.break a
+    );
+  in
+    b
+  ```
+
+  Nix would correctly enter the debugger at `builtins.break a`, but if you asked
+  it to `:continue`, it would skip over the `builtins.break "hello"` expression
+  entirely.
+
+  Now, Nix will correctly enter the debugger at both breakpoints.
+
+- Nested debuggers are no longer supported [#9920](https://github.com/NixOS/nix/pull/9920)
+
+  Previously, evaluating an expression that throws an error in the debugger would
+  enter a second, nested debugger:
+
+  ```
+  nix-repl> builtins.throw "what"
+  error: what
+
+
+  Starting REPL to allow you to inspect the current state of the evaluator.
+
+  Welcome to Nix 2.18.1. Type :? for help.
+
+  nix-repl>
+  ```
+
+  Now, it just prints the error message like `nix repl`:
+
+  ```
+  nix-repl> builtins.throw "what"
+  error:
+         … while calling the 'throw' builtin
+           at «string»:1:1:
+              1| builtins.throw "what"
+               | ^
+
+         error: what
+  ```
+
+- Consistent order of function arguments in printed expressions [#9874](https://github.com/NixOS/nix/pull/9874)
+
+  Function arguments are now printed in lexicographic order rather than the internal, creation-time based symbol order.
+
+- Fix duplicate attribute error positions for `inherit` [#9874](https://github.com/NixOS/nix/pull/9874)
+
+  When an `inherit` caused a duplicate attribute error the position of the error was not reported correctly, placing the error with the inherit itself or at the start of the bindings block instead of the offending attribute name.
+
+- `inherit (x) ...` evaluates `x` only once [#9847](https://github.com/NixOS/nix/pull/9847)
+
+  `inherit (x) a b ...` now evaluates the expression `x` only once for all inherited attributes rather than once for each inherited attribute.
+  This does not usually have a measurable impact, but side-effects (such as `builtins.trace`) would be duplicated and expensive expressions (such as derivations) could cause a measurable slowdown.
+
+- Store paths are allowed to start with `.` [#912](https://github.com/NixOS/nix/issues/912) [#9091](https://github.com/NixOS/nix/pull/9091) [#9095](https://github.com/NixOS/nix/pull/9095) [#9120](https://github.com/NixOS/nix/pull/9120) [#9121](https://github.com/NixOS/nix/pull/9121) [#9122](https://github.com/NixOS/nix/pull/9122) [#9130](https://github.com/NixOS/nix/pull/9130) [#9219](https://github.com/NixOS/nix/pull/9219) [#9224](https://github.com/NixOS/nix/pull/9224) [#9867](https://github.com/NixOS/nix/pull/9867)
+
+  Leading periods were allowed by accident in Nix 2.4. The Nix team has considered this to be a bug, but this behavior has since been relied on by users, leading to unnecessary difficulties.
+  From now on, leading periods are supported. The names `.` and `..` are disallowed, as well as those starting with `.-` or `..-`.
+
+  Nix versions that denied leading periods are documented [in the issue](https://github.com/NixOS/nix/issues/912#issuecomment-1919583286).
+
+- `nix repl` pretty-prints values [#9931](https://github.com/NixOS/nix/pull/9931)
+
+  `nix repl` will now pretty-print values:
+
+  ```
+  {
+    attrs = {
+      a = {
+        b = {
+          c = { };
+        };
+      };
+    };
+    list = [ 1 ];
+    list' = [
+      1
+      2
+      3
+    ];
+  }
+  ```
+
+- Introduction of `--regex` and `--all` in `nix profile remove` and `nix profile upgrade` [#10166](https://github.com/NixOS/nix/pull/10166)
+
+  Previously the command-line arguments for `nix profile remove` and `nix profile upgrade` matched the package entries using regular expression.
+  For instance:
+
+  ```
+  nix profile remove '.*vim.*'
+  ```
+
+  This would remove all packages that contain `vim` in their name.
+
+  In most cases, only singular package names were used to remove and upgrade packages. Mixing this with regular expressions sometimes lead to unintended behavior. For instance, `python3.1` could match `python311`.
+
+  To avoid unintended behavior, the arguments are now only matching exact names.
+
+  Matching using regular expressions is still possible by using the new `--regex` flag:
+
+  ```
+  nix profile remove --regex '.*vim.*'
+  ```
+
+  One of the most useful cases for using regular expressions was to upgrade all packages. This was previously accomplished by:
+
+  ```
+  nix profile upgrade '.*'
+  ```
+
+  With the introduction of the `--all` flag, this now becomes more straightforward:
+
+  ```
+  nix profile upgrade --all
+  ```
+
+- Visual clutter in `--debugger` is reduced [#9919](https://github.com/NixOS/nix/pull/9919)
+
+  Before:
+  ```
+  info: breakpoint reached
+
+
+  Starting REPL to allow you to inspect the current state of the evaluator.
+
+  Welcome to Nix 2.20.0pre20231222_dirty. Type :? for help.
+
+  nix-repl> :continue
+  error: uh oh
+
+
+  Starting REPL to allow you to inspect the current state of the evaluator.
+
+  Welcome to Nix 2.20.0pre20231222_dirty. Type :? for help.
+
+  nix-repl>
+  ```
+
+  After:
+
+  ```
+  info: breakpoint reached
+
+  Nix 2.20.0pre20231222_dirty debugger
+  Type :? for help.
+  nix-repl> :continue
+  error: uh oh
+
+  nix-repl>
+  ```
+
+- Cycle detection in `nix repl` is simpler and more reliable [#8672](https://github.com/NixOS/nix/issues/8672) [#9926](https://github.com/NixOS/nix/pull/9926)
+
+  The cycle detection in `nix repl`, `nix eval`, `builtins.trace`, and everywhere
+  else values are printed is now simpler and matches the cycle detection in
+  `nix-instantiate --eval` output.
+
+  Before:
+
+  ```
+  nix eval --expr 'let self = { inherit self; }; in self'
+  { self = { self = «repeated»; }; }
+  ```
+
+  After:
+
+  ```
+  { self = «repeated»; }
+  ```
+
+- In the debugger, `while evaluating the attribute` errors now include position information [#9915](https://github.com/NixOS/nix/pull/9915)
+
+  Before:
+
+  ```
+  0: while evaluating the attribute 'python311.pythonForBuild.pkgs'
+  0x600001522598
+  ```
+
+  After:
+
+  ```
+  0: while evaluating the attribute 'python311.pythonForBuild.pkgs'
+  /nix/store/hg65h51xnp74ikahns9hyf3py5mlbbqq-source/overrides/default.nix:132:27
+
+     131|
+     132|       bootstrappingBase = pkgs.${self.python.pythonAttr}.pythonForBuild.pkgs;
+        |                           ^
+     133|     in
+  ```
+
+- Stack size is increased on macOS [#9860](https://github.com/NixOS/nix/pull/9860)
+
+  Previously, Nix would set the stack size to 64MiB on Linux, but would leave the
+  stack size set to the default (approximately 8KiB) on macOS. Now, the stack
+  size is correctly set to 64MiB on macOS as well, which should reduce stack
+  overflow segfaults in deeply-recursive Nix expressions.
+

flake.lock

@@ -34,16 +34,16 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1705033721,
-        "narHash": "sha256-K5eJHmL1/kev6WuqyqqbS1cdNnSidIZ3jeqJ7GbrYnQ=",
+        "lastModified": 1709083642,
+        "narHash": "sha256-7kkJQd4rZ+vFrzWu8sTRtta5D1kBG0LSRYAfhtmMlSo=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "a1982c92d8980a0114372973cbdfe0a307f1bdea",
+        "rev": "b550fe4b4776908ac2a861124307045f8e717c8e",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "nixos-23.05-small",
+        "ref": "release-23.11",
         "repo": "nixpkgs",
         "type": "github"
       }

flake.nix

@@ -1,7 +1,9 @@
 {
   description = "The purely functional package manager";
 
-  inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05-small";
+  # TODO switch to nixos-23.11-small
+  #      https://nixpk.gs/pr-tracker.html?pr=291954
+  inputs.nixpkgs.url = "github:NixOS/nixpkgs/release-23.11";
   inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2";
   inputs.flake-compat = { url = "github:edolstra/flake-compat"; flake = false; };
   inputs.libgit2 = { url = "github:libgit2/libgit2"; flake = false; };
@@ -10,19 +12,9 @@
 
     let
       inherit (nixpkgs) lib;
+      inherit (lib) fileset;
 
-      # Experimental fileset library: https://github.com/NixOS/nixpkgs/pull/222981
-      # Not an "idiomatic" flake input because:
-      #  - Propagation to dependent locks: https://github.com/NixOS/nix/issues/7730
-      #  - Subflake would download redundant and huge parent flake
-      #  - No git tree hash support: https://github.com/NixOS/nix/issues/6044
-      inherit (import (builtins.fetchTarball { url = "https://github.com/NixOS/nix/archive/1bdcd7fc8a6a40b2e805bad759b36e64e911036b.tar.gz"; sha256 = "sha256:14ljlpdsp4x7h1fkhbmc4bd3vsqnx8zdql4h3037wh09ad6a0893"; }))
-        fileset;
-
-      officialRelease = false;
-
-      # Set to true to build the release notes for the next release.
-      buildUnreleasedNotes = false;
+      officialRelease = true;
 
       version = lib.fileContents ./.version + versionSuffix;
       versionSuffix =
@@ -39,7 +31,6 @@
       crossSystems = [
         "armv6l-unknown-linux-gnueabihf"
         "armv7l-unknown-linux-gnueabihf"
-        "x86_64-unknown-freebsd13"
         "x86_64-unknown-netbsd"
       ];
 
@@ -174,7 +165,7 @@
 
           nix =
             let
-              officialRelease = false;
+              officialRelease = true;
               versionSuffix =
                 if officialRelease
                 then ""
@@ -186,7 +177,7 @@
                 stdenv
                 versionSuffix
                 ;
-              officialRelease = false;
+              officialRelease = true;
               boehmgc = final.boehmgc-nix;
               libgit2 = final.libgit2-nix;
               busybox-sandbox-shell = final.busybox-sandbox-shell or final.default-busybox-sandbox-shell;
@@ -307,8 +298,11 @@
               ''
                 type -p nix-env
                 # Note: we're filtering out nixos-install-tools because https://github.com/NixOS/nixpkgs/pull/153594#issuecomment-1020530593.
-                time nix-env --store dummy:// -f ${nixpkgs-regression} -qaP --drv-path | sort | grep -v nixos-install-tools > packages
-                [[ $(sha1sum < packages | cut -c1-40) = ff451c521e61e4fe72bdbe2d0ca5d1809affa733 ]]
+                (
+                  set -x
+                  time nix-env --store dummy:// -f ${nixpkgs-regression} -qaP --drv-path | sort | grep -v nixos-install-tools > packages
+                  [[ $(sha1sum < packages | cut -c1-40) = e01b031fc9785a572a38be6bc473957e3b6faad7 ]]
+                )
                 mkdir $out
               '';
 
@@ -349,7 +343,6 @@
 
       checks = forAllSystems (system: {
         binaryTarball = self.hydraJobs.binaryTarball.${system};
-        perlBindings = self.hydraJobs.perlBindings.${system};
         installTests = self.hydraJobs.installTests.${system};
         nixpkgsLibTests = self.hydraJobs.tests.nixpkgsLibTests.${system};
         rl-next =
@@ -359,6 +352,11 @@
         '';
       } // (lib.optionalAttrs (builtins.elem system linux64BitSystems)) {
         dockerImage = self.hydraJobs.dockerImage.${system};
+      } // (lib.optionalAttrs (!(builtins.elem system linux32BitSystems))) {
+        # Some perl dependencies are broken on i686-linux.
+        # Since the support is only best-effort there, disable the perl
+        # bindings
+        perlBindings = self.hydraJobs.perlBindings.${system};
       });
 
       packages = forAllSystems (system: rec {
@@ -404,8 +402,11 @@
             # Make bash completion work.
             XDG_DATA_DIRS+=:$out/share
           '';
+
           nativeBuildInputs = attrs.nativeBuildInputs or []
-            ++ lib.optional stdenv.cc.isClang pkgs.buildPackages.bear
+            # TODO: Remove the darwin check once
+            # https://github.com/NixOS/nixpkgs/pull/291814 is available
+            ++ lib.optional (stdenv.cc.isClang && !stdenv.buildPlatform.isDarwin) pkgs.buildPackages.bear
             ++ lib.optional (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform) pkgs.buildPackages.clang-tools;
         });
         in
@@ -417,8 +418,9 @@
               (forAllStdenvs (stdenvName: makeShell pkgs pkgs.${stdenvName}));
           in
             (makeShells "native" nixpkgsFor.${system}.native) //
-            (makeShells "static" nixpkgsFor.${system}.static) //
-            (lib.genAttrs shellCrossSystems (crossSystem: let pkgs = nixpkgsFor.${system}.cross.${crossSystem}; in makeShell pkgs pkgs.stdenv)) //
+            (lib.optionalAttrs (!nixpkgsFor.${system}.native.stdenv.isDarwin)
+              (makeShells "static" nixpkgsFor.${system}.static)) //
+              (lib.genAttrs shellCrossSystems (crossSystem: let pkgs = nixpkgsFor.${system}.cross.${crossSystem}; in makeShell pkgs pkgs.stdenv)) //
             {
               default = self.devShells.${system}.native-stdenvPackages;
             }

maintainers/upload-release.pl

@@ -11,6 +11,8 @@ use JSON::PP;
 use LWP::UserAgent;
 use Net::Amazon::S3;
 
+delete $ENV{'shell'}; # shut up a LWP::UserAgent.pm warning
+
 my $evalId = $ARGV[0] or die "Usage: $0 EVAL-ID\n";
 
 my $releasesBucketName = "nix-releases";
@@ -36,9 +38,9 @@ sub fetch {
 my $evalUrl = "https://hydra.nixos.org/eval/$evalId";
 my $evalInfo = decode_json(fetch($evalUrl, 'application/json'));
 #print Dumper($evalInfo);
-my $flakeUrl = $evalInfo->{flake} or die;
-my $flakeInfo = decode_json(`nix flake metadata --json "$flakeUrl"` or die);
-my $nixRev = $flakeInfo->{revision} or die;
+my $flakeUrl = $evalInfo->{flake};
+my $flakeInfo = decode_json(`nix flake metadata --json "$flakeUrl"` or die) if $flakeUrl;
+my $nixRev = ($flakeInfo ? $flakeInfo->{revision} : $evalInfo->{jobsetevalinputs}->{nix}->{revision}) or die;
 
 my $buildInfo = decode_json(fetch("$evalUrl/job/build.x86_64-linux", 'application/json'));
 #print Dumper($buildInfo);
@@ -83,12 +85,19 @@ my $channelsBucket = $s3_us->bucket($channelsBucketName) or die;
 sub getStorePath {
     my ($jobName, $output) = @_;
     my $buildInfo = decode_json(fetch("$evalUrl/job/$jobName", 'application/json'));
-    return $buildInfo->{buildoutputs}->{$output or "out"}->{path} or die "cannot get store path for '$jobName'";
+    return $buildInfo->{buildoutputs}->{$output or "out"}->{path} // die "cannot get store path for '$jobName'";
 }
 
 sub copyManual {
-    my $manual = getStorePath("build.x86_64-linux", "doc");
-    print "$manual\n";
+    my $manual;
+    eval {
+        $manual = getStorePath("build.x86_64-linux", "doc");
+    };
+    if ($@) {
+        warn "$@";
+        return;
+    }
+    print "Manual: $manual\n";
 
     my $manualNar = "$tmpDir/$releaseName-manual.nar.xz";
     print "$manualNar\n";
@@ -154,19 +163,33 @@ downloadFile("binaryTarball.x86_64-linux", "1");
 downloadFile("binaryTarball.aarch64-linux", "1");
 downloadFile("binaryTarball.x86_64-darwin", "1");
 downloadFile("binaryTarball.aarch64-darwin", "1");
-downloadFile("binaryTarballCross.x86_64-linux.armv6l-unknown-linux-gnueabihf", "1");
-downloadFile("binaryTarballCross.x86_64-linux.armv7l-unknown-linux-gnueabihf", "1");
+eval {
+    downloadFile("binaryTarballCross.x86_64-linux.armv6l-unknown-linux-gnueabihf", "1");
+};
+warn "$@" if $@;
+eval {
+    downloadFile("binaryTarballCross.x86_64-linux.armv7l-unknown-linux-gnueabihf", "1");
+};
+warn "$@" if $@;
 downloadFile("installerScript", "1");
 
 # Upload docker images to dockerhub.
 my $dockerManifest = "";
 my $dockerManifestLatest = "";
+my $haveDocker = 0;
 
 for my $platforms (["x86_64-linux", "amd64"], ["aarch64-linux", "arm64"]) {
     my $system = $platforms->[0];
     my $dockerPlatform = $platforms->[1];
     my $fn = "nix-$version-docker-image-$dockerPlatform.tar.gz";
-    downloadFile("dockerImage.$system", "1", $fn);
+    eval {
+        downloadFile("dockerImage.$system", "1", $fn);
+    };
+    if ($@) {
+        warn "$@" if $@;
+        next;
+    }
+    $haveDocker = 1;
 
     print STDERR "loading docker image for $dockerPlatform...\n";
     system("docker load -i $tmpDir/$fn") == 0 or die;
@@ -194,21 +217,23 @@ for my $platforms (["x86_64-linux", "amd64"], ["aarch64-linux", "arm64"]) {
     $dockerManifestLatest .= " --amend $latestTag"
 }
 
-print STDERR "creating multi-platform docker manifest...\n";
-system("docker manifest rm nixos/nix:$version");
-system("docker manifest create nixos/nix:$version $dockerManifest") == 0 or die;
-if ($isLatest) {
-    print STDERR "creating latest multi-platform docker manifest...\n";
-    system("docker manifest rm nixos/nix:latest");
-    system("docker manifest create nixos/nix:latest $dockerManifestLatest") == 0 or die;
-}
+if ($haveDocker) {
+    print STDERR "creating multi-platform docker manifest...\n";
+    system("docker manifest rm nixos/nix:$version");
+    system("docker manifest create nixos/nix:$version $dockerManifest") == 0 or die;
+    if ($isLatest) {
+        print STDERR "creating latest multi-platform docker manifest...\n";
+        system("docker manifest rm nixos/nix:latest");
+        system("docker manifest create nixos/nix:latest $dockerManifestLatest") == 0 or die;
+    }
 
-print STDERR "pushing multi-platform docker manifest...\n";
-system("docker manifest push nixos/nix:$version") == 0 or die;
+    print STDERR "pushing multi-platform docker manifest...\n";
+    system("docker manifest push nixos/nix:$version") == 0 or die;
 
-if ($isLatest) {
-    print STDERR "pushing latest multi-platform docker manifest...\n";
-    system("docker manifest push nixos/nix:latest") == 0 or die;
+    if ($isLatest) {
+        print STDERR "pushing latest multi-platform docker manifest...\n";
+        system("docker manifest push nixos/nix:latest") == 0 or die;
+    }
 }
 
 # Upload nix-fallback-paths.nix.

mk/cxx-big-literal.mk

@@ -1,5 +1,5 @@
 %.gen.hh: %
-	@echo 'R"foo(' >> $@.tmp
+	@echo 'R"__NIX_STR(' >> $@.tmp
 	$(trace-gen) cat $< >> $@.tmp
-	@echo ')foo"' >> $@.tmp
+	@echo ')__NIX_STR"' >> $@.tmp
 	@mv $@.tmp $@

package.nix

@@ -24,6 +24,7 @@
 , libgit2
 , libseccomp
 , libsodium
+, man
 , lowdown
 , mdbook
 , mdbook-linkcheck
@@ -154,7 +155,7 @@ in {
     in
       fileset.toSource {
         root = ./.;
-        fileset = fileset.intersect baseFiles (fileset.unions ([
+        fileset = fileset.intersection baseFiles (fileset.unions ([
           # For configure
           ./.version
           ./configure.ac
@@ -209,6 +210,11 @@ in {
     (lib.getBin lowdown)
     mdbook
     mdbook-linkcheck
+  ] ++ lib.optionals doInstallCheck [
+    git
+    mercurial
+    openssh
+    man # for testing `nix-* --help`
   ] ++ lib.optionals (doInstallCheck || enableManual) [
     jq # Also for custom mdBook preprocessor.
   ] ++ lib.optional stdenv.hostPlatform.isLinux util-linux
@@ -249,12 +255,6 @@ in {
   dontBuild = !attrs.doBuild;
   doCheck = attrs.doCheck;
 
-  nativeCheckInputs = [
-    git
-    mercurial
-    openssh
-  ];
-
   disallowedReferences = [ boost ];
 
   preConfigure = lib.optionalString (doBuild && ! stdenv.hostPlatform.isStatic) (
@@ -343,15 +343,22 @@ in {
 
   # Work around weird bug where it doesn't think there is a Makefile.
   installCheckPhase = if (!doBuild && doInstallCheck) then ''
+    runHook preInstallCheck
     mkdir -p src/nix-channel
     make installcheck -j$NIX_BUILD_CORES -l$NIX_BUILD_CORES
   '' else null;
 
   # Needed for tests if we are not doing a build, but testing existing
   # built Nix.
-  preInstallCheck = lib.optionalString (! doBuild) ''
-    mkdir -p src/nix-channel
-  '';
+  preInstallCheck =
+    lib.optionalString (! doBuild) ''
+      mkdir -p src/nix-channel
+    ''
+    # See https://github.com/NixOS/nix/issues/2523
+    # Occurs often in tests since https://github.com/NixOS/nix/pull/9900
+    + lib.optionalString stdenv.hostPlatform.isDarwin ''
+      export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES
+    '';
 
   separateDebugInfo = !stdenv.hostPlatform.isStatic;
 

perl/lib/Nix/Store.xs

@@ -259,7 +259,7 @@ hashPath(char * algo, int base32, char * path)
             auto [accessor, canonPath] = PosixSourceAccessor::createAtRoot(path);
             Hash h = hashPath(
                 accessor, canonPath,
-                FileIngestionMethod::Recursive, parseHashAlgo(algo)).first;
+                FileIngestionMethod::Recursive, parseHashAlgo(algo));
             auto s = h.to_string(base32 ? HashFormat::Nix32 : HashFormat::Base16, false);
             XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
         } catch (Error & e) {

src/libcmd/common-eval-args.cc

@@ -20,7 +20,7 @@ MixEvalArgs::MixEvalArgs()
         .description = "Pass the value *expr* as the argument *name* to Nix functions.",
         .category = category,
         .labels = {"name", "expr"},
-        .handler = {[&](std::string name, std::string expr) { autoArgs[name] = 'E' + expr; }}
+        .handler = {[&](std::string name, std::string expr) { autoArgs.insert_or_assign(name, AutoArg{AutoArgExpr(expr)}); }}
     });
 
     addFlag({
@@ -28,7 +28,24 @@ MixEvalArgs::MixEvalArgs()
         .description = "Pass the string *string* as the argument *name* to Nix functions.",
         .category = category,
         .labels = {"name", "string"},
-        .handler = {[&](std::string name, std::string s) { autoArgs[name] = 'S' + s; }},
+        .handler = {[&](std::string name, std::string s) { autoArgs.insert_or_assign(name, AutoArg{AutoArgString(s)}); }},
+    });
+
+    addFlag({
+        .longName = "arg-from-file",
+        .description = "Pass the contents of file *path* as the argument *name* to Nix functions.",
+        .category = category,
+        .labels = {"name", "path"},
+        .handler = {[&](std::string name, std::string path) { autoArgs.insert_or_assign(name, AutoArg{AutoArgFile(path)}); }},
+        .completer = completePath
+    });
+
+    addFlag({
+        .longName = "arg-from-stdin",
+        .description = "Pass the contents of stdin as the argument *name* to Nix functions.",
+        .category = category,
+        .labels = {"name"},
+        .handler = {[&](std::string name) { autoArgs.insert_or_assign(name, AutoArg{AutoArgStdin{}}); }},
     });
 
     addFlag({
@@ -154,13 +171,23 @@ MixEvalArgs::MixEvalArgs()
 Bindings * MixEvalArgs::getAutoArgs(EvalState & state)
 {
     auto res = state.buildBindings(autoArgs.size());
-    for (auto & i : autoArgs) {
+    for (auto & [name, arg] : autoArgs) {
         auto v = state.allocValue();
-        if (i.second[0] == 'E')
-            state.mkThunk_(*v, state.parseExprFromString(i.second.substr(1), state.rootPath(".")));
-        else
-            v->mkString(((std::string_view) i.second).substr(1));
-        res.insert(state.symbols.create(i.first), v);
+        std::visit(overloaded {
+            [&](const AutoArgExpr & arg) {
+                state.mkThunk_(*v, state.parseExprFromString(arg.expr, state.rootPath(".")));
+            },
+            [&](const AutoArgString & arg) {
+                v->mkString(arg.s);
+            },
+            [&](const AutoArgFile & arg) {
+                v->mkString(readFile(arg.path));
+            },
+            [&](const AutoArgStdin & arg) {
+                v->mkString(readFile(STDIN_FILENO));
+            }
+        }, arg);
+        res.insert(state.symbols.create(name), v);
     }
     return res.finish();
 }

src/libcmd/common-eval-args.hh

@@ -6,6 +6,8 @@
 #include "common-args.hh"
 #include "search-path.hh"
 
+#include <filesystem>
+
 namespace nix {
 
 class Store;
@@ -26,7 +28,14 @@ struct MixEvalArgs : virtual Args, virtual MixRepair
     std::optional<std::string> evalStoreUrl;
 
 private:
-    std::map<std::string, std::string> autoArgs;
+    struct AutoArgExpr { std::string expr; };
+    struct AutoArgString { std::string s; };
+    struct AutoArgFile { std::filesystem::path path; };
+    struct AutoArgStdin { };
+
+    using AutoArg = std::variant<AutoArgExpr, AutoArgString, AutoArgFile, AutoArgStdin>;
+
+    std::map<std::string, AutoArg> autoArgs;
 };
 
 SourcePath lookupFileArg(EvalState & state, std::string_view s, const Path * baseDir = nullptr);

src/libcmd/installables.cc

@@ -21,6 +21,7 @@
 #include "url.hh"
 #include "registry.hh"
 #include "build-result.hh"
+#include "fs-input-accessor.hh"
 
 #include <regex>
 #include <queue>
@@ -146,7 +147,7 @@ MixFlakeOptions::MixFlakeOptions()
         .category = category,
         .labels = {"flake-lock-path"},
         .handler = {[&](std::string lockFilePath) {
-            lockFlags.referenceLockFilePath = lockFilePath;
+            lockFlags.referenceLockFilePath = getUnfilteredRootPath(CanonPath(absPath(lockFilePath)));
         }},
         .completer = completePath
     });
@@ -442,10 +443,10 @@ ref<eval_cache::EvalCache> openEvalCache(
     EvalState & state,
     std::shared_ptr<flake::LockedFlake> lockedFlake)
 {
-    auto fingerprint = lockedFlake->getFingerprint();
+    auto fingerprint = lockedFlake->getFingerprint(state.store);
     return make_ref<nix::eval_cache::EvalCache>(
         evalSettings.useEvalCache && evalSettings.pureEval
-            ? std::optional { std::cref(fingerprint) }
+            ? fingerprint
             : std::nullopt,
         state,
         [&state, lockedFlake]()

src/libcmd/repl.cc

@@ -336,13 +336,7 @@ ReplExitStatus NixRepl::mainLoop()
               printMsg(lvlError, e.msg());
             }
         } catch (EvalError & e) {
-            // in debugger mode, an EvalError should trigger another repl session.
-            // when that session returns the exception will land here.  No need to show it again;
-            // show the error for this repl session instead.
-            if (state->debugRepl && !state->debugTraces.empty())
-                showDebugTrace(std::cout, state->positions, state->debugTraces.front());
-            else
-                printMsg(lvlError, e.msg());
+            printMsg(lvlError, e.msg());
         } catch (Error & e) {
             printMsg(lvlError, e.msg());
         } catch (Interrupted & e) {
@@ -383,7 +377,6 @@ bool NixRepl::getLine(std::string & input, const std::string & prompt)
     };
 
     setupSignals();
-    Finally resetTerminal([&]() { rl_deprep_terminal(); });
     char * s = readline(prompt.c_str());
     Finally doFree([&]() { free(s); });
     restoreSignals();
@@ -549,6 +542,7 @@ ProcessLineResult NixRepl::processLine(std::string line)
              << "  :l, :load <path>             Load Nix expression and add it to scope\n"
              << "  :lf, :load-flake <ref>       Load Nix flake and add it to scope\n"
              << "  :p, :print <expr>            Evaluate and print expression recursively\n"
+             << "                               Strings are printed directly, without escaping.\n"
              << "  :q, :quit                    Exit nix-repl\n"
              << "  :r, :reload                  Reload all files\n"
              << "  :sh <expr>                   Build dependencies of derivation, then start\n"
@@ -756,7 +750,11 @@ ProcessLineResult NixRepl::processLine(std::string line)
     else if (command == ":p" || command == ":print") {
         Value v;
         evalString(arg, v);
-        printValue(std::cout, v);
+        if (v.type() == nString) {
+            std::cout << v.string_view();
+        } else {
+            printValue(std::cout, v);
+        }
         std::cout << std::endl;
     }
 

src/libexpr/eval-error.cc

@@ -28,15 +28,7 @@ template<class T>
 EvalErrorBuilder<T> & EvalErrorBuilder<T>::withTrace(PosIdx pos, const std::string_view text)
 {
     error.err.traces.push_front(
-        Trace{.pos = error.state.positions[pos], .hint = HintFmt(std::string(text)), .frame = false});
-    return *this;
-}
-
-template<class T>
-EvalErrorBuilder<T> & EvalErrorBuilder<T>::withFrameTrace(PosIdx pos, const std::string_view text)
-{
-    error.err.traces.push_front(
-        Trace{.pos = error.state.positions[pos], .hint = HintFmt(std::string(text)), .frame = true});
+        Trace{.pos = error.state.positions[pos], .hint = HintFmt(std::string(text))});
     return *this;
 }
 
@@ -63,9 +55,9 @@ EvalErrorBuilder<T> & EvalErrorBuilder<T>::withFrame(const Env & env, const Expr
 }
 
 template<class T>
-EvalErrorBuilder<T> & EvalErrorBuilder<T>::addTrace(PosIdx pos, HintFmt hint, bool frame)
+EvalErrorBuilder<T> & EvalErrorBuilder<T>::addTrace(PosIdx pos, HintFmt hint)
 {
-    error.addTrace(error.state.positions[pos], hint, frame);
+    error.addTrace(error.state.positions[pos], hint);
     return *this;
 }
 

src/libexpr/eval-error.hh

@@ -89,7 +89,7 @@ public:
 
     [[nodiscard, gnu::noinline]] EvalErrorBuilder<T> & withFrame(const Env & e, const Expr & ex);
 
-    [[nodiscard, gnu::noinline]] EvalErrorBuilder<T> & addTrace(PosIdx pos, HintFmt hint, bool frame = false);
+    [[nodiscard, gnu::noinline]] EvalErrorBuilder<T> & addTrace(PosIdx pos, HintFmt hint);
 
     template<typename... Args>
     [[nodiscard, gnu::noinline]] EvalErrorBuilder<T> &

src/libexpr/eval-settings.hh

@@ -21,11 +21,24 @@ struct EvalSettings : Config
     Setting<Strings> nixPath{
         this, getDefaultNixPath(), "nix-path",
         R"(
-          List of directories to be searched for `<...>` file references
+          List of search paths to use for [lookup path](@docroot@/language/constructs/lookup-path.md) resolution.
+          This setting determines the value of
+          [`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtins-nixPath) and can be used with [`builtins.findFile`](@docroot@/language/builtin-constants.md#builtins-findFile).
 
-          In particular, outside of [pure evaluation mode](#conf-pure-eval), this determines the value of
-          [`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtins-nixPath).
-        )"};
+          The default value is
+
+          ```
+          $HOME/.nix-defexpr/channels
+          nixpkgs=$NIX_STATE_DIR/profiles/per-user/root/channels/nixpkgs
+          $NIX_STATE_DIR/profiles/per-user/root/channels
+          ```
+
+          It can be overridden with the [`NIX_PATH` environment variable](@docroot@/command-ref/env-common.md#env-NIX_PATH) or the [`-I` command line option](@docroot@/command-ref/opt-common.md#opt-I).
+
+          > **Note**
+          >
+          > If [pure evaluation](#conf-pure-eval) is enabled, `nixPath` evaluates to the empty list `[ ]`.
+        )", {}, false};
 
     Setting<std::string> currentSystem{
         this, "", "eval-system",
@@ -55,8 +68,6 @@ struct EvalSettings : Config
           [`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtins-nixPath),
           or to URIs outside of
           [`allowed-uris`](@docroot@/command-ref/conf-file.md#conf-allowed-uris).
-
-          Also the default value for [`nix-path`](#conf-nix-path) is ignored, such that only explicitly set search path entries are taken into account.
         )"};
 
     Setting<bool> pureEval{this, false, "pure-eval",

src/libexpr/eval.cc

@@ -762,10 +762,24 @@ std::unique_ptr<ValMap> mapStaticEnvBindings(const SymbolTable & st, const Stati
     return vm;
 }
 
+/**
+ * Sets `inDebugger` to true on construction and false on destruction.
+ */
+class DebuggerGuard {
+    bool & inDebugger;
+public:
+    DebuggerGuard(bool & inDebugger) : inDebugger(inDebugger) {
+        inDebugger = true;
+    }
+    ~DebuggerGuard() {
+        inDebugger = false;
+    }
+};
+
 void EvalState::runDebugRepl(const Error * error, const Env & env, const Expr & expr)
 {
-    // double check we've got the debugRepl function pointer.
-    if (!debugRepl)
+    // Make sure we have a debugger to run and we're not already in a debugger.
+    if (!debugRepl || inDebugger)
         return;
 
     auto dts =
@@ -792,6 +806,7 @@ void EvalState::runDebugRepl(const Error * error, const Env & env, const Expr &
     auto se = getStaticEnv(expr);
     if (se) {
         auto vm = mapStaticEnvBindings(symbols, *se.get(), env);
+        DebuggerGuard _guard(inDebugger);
         auto exitStatus = (debugRepl)(ref<EvalState>(shared_from_this()), *vm);
         switch (exitStatus) {
             case ReplExitStatus::QuitAll:
@@ -806,14 +821,16 @@ void EvalState::runDebugRepl(const Error * error, const Env & env, const Expr &
     }
 }
 
-void EvalState::addErrorTrace(Error & e, const char * s, const std::string & s2) const
+template<typename... Args>
+void EvalState::addErrorTrace(Error & e, const Args & ... formatArgs) const
 {
-    e.addTrace(nullptr, s, s2);
+    e.addTrace(nullptr, HintFmt(formatArgs...));
 }
 
-void EvalState::addErrorTrace(Error & e, const PosIdx pos, const char * s, const std::string & s2, bool frame) const
+template<typename... Args>
+void EvalState::addErrorTrace(Error & e, const PosIdx pos, const Args & ... formatArgs) const
 {
-    e.addTrace(positions[pos], HintFmt(s, s2), frame);
+    e.addTrace(positions[pos], HintFmt(formatArgs...));
 }
 
 template<typename... Args>
@@ -932,12 +949,11 @@ void EvalState::mkThunk_(Value & v, Expr * expr)
 
 void EvalState::mkPos(Value & v, PosIdx p)
 {
-    auto pos = positions[p];
-    if (auto path = std::get_if<SourcePath>(&pos.origin)) {
+    auto origin = positions.originOf(p);
+    if (auto path = std::get_if<SourcePath>(&origin)) {
         auto attrs = buildBindings(3);
         attrs.alloc(sFile).mkString(path->path.abs());
-        attrs.alloc(sLine).mkInt(pos.line);
-        attrs.alloc(sColumn).mkInt(pos.column);
+        makePositionThunks(*this, p, attrs.alloc(sLine), attrs.alloc(sColumn));
         v.mkAttrs(attrs);
     } else
         v.mkNull();
@@ -1196,6 +1212,18 @@ void ExprPath::eval(EvalState & state, Env & env, Value & v)
 }
 
 
+Env * ExprAttrs::buildInheritFromEnv(EvalState & state, Env & up)
+{
+    Env & inheritEnv = state.allocEnv(inheritFromExprs->size());
+    inheritEnv.up = &up;
+
+    Displacement displ = 0;
+    for (auto from : *inheritFromExprs)
+        inheritEnv.values[displ++] = from->maybeThunk(state, up);
+
+    return &inheritEnv;
+}
+
 void ExprAttrs::eval(EvalState & state, Env & env, Value & v)
 {
     v.mkAttrs(state.buildBindings(attrs.size() + dynamicAttrs.size()).finish());
@@ -1207,6 +1235,7 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v)
         Env & env2(state.allocEnv(attrs.size()));
         env2.up = &env;
         dynamicEnv = &env2;
+        Env * inheritEnv = inheritFromExprs ? buildInheritFromEnv(state, env2) : nullptr;
 
         AttrDefs::iterator overrides = attrs.find(state.sOverrides);
         bool hasOverrides = overrides != attrs.end();
@@ -1217,11 +1246,11 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v)
         Displacement displ = 0;
         for (auto & i : attrs) {
             Value * vAttr;
-            if (hasOverrides && !i.second.inherited) {
+            if (hasOverrides && i.second.kind != AttrDef::Kind::Inherited) {
                 vAttr = state.allocValue();
-                mkThunk(*vAttr, env2, i.second.e);
+                mkThunk(*vAttr, *i.second.chooseByKind(&env2, &env, inheritEnv), i.second.e);
             } else
-                vAttr = i.second.e->maybeThunk(state, i.second.inherited ? env : env2);
+                vAttr = i.second.e->maybeThunk(state, *i.second.chooseByKind(&env2, &env, inheritEnv));
             env2.values[displ++] = vAttr;
             v.attrs->push_back(Attr(i.first, vAttr, i.second.pos));
         }
@@ -1253,9 +1282,15 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v)
         }
     }
 
-    else
-        for (auto & i : attrs)
-            v.attrs->push_back(Attr(i.first, i.second.e->maybeThunk(state, env), i.second.pos));
+    else {
+        Env * inheritEnv = inheritFromExprs ? buildInheritFromEnv(state, env) : nullptr;
+        for (auto & i : attrs) {
+            v.attrs->push_back(Attr(
+                    i.first,
+                    i.second.e->maybeThunk(state, *i.second.chooseByKind(&env, &env, inheritEnv)),
+                    i.second.pos));
+        }
+    }
 
     /* Dynamic attrs apply *after* rec and __overrides. */
     for (auto & i : dynamicAttrs) {
@@ -1287,12 +1322,17 @@ void ExprLet::eval(EvalState & state, Env & env, Value & v)
     Env & env2(state.allocEnv(attrs->attrs.size()));
     env2.up = &env;
 
+    Env * inheritEnv = attrs->inheritFromExprs ? attrs->buildInheritFromEnv(state, env2) : nullptr;
+
     /* The recursive attributes are evaluated in the new environment,
        while the inherited attributes are evaluated in the original
        environment. */
     Displacement displ = 0;
-    for (auto & i : attrs->attrs)
-        env2.values[displ++] = i.second.e->maybeThunk(state, i.second.inherited ? env : env2);
+    for (auto & i : attrs->attrs) {
+        env2.values[displ++] = i.second.e->maybeThunk(
+            state,
+            *i.second.chooseByKind(&env2, &env, inheritEnv));
+    }
 
     auto dts = state.debugRepl
         ? makeDebugTraceStacker(
@@ -1369,7 +1409,7 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v)
                 state,
                 *this,
                 env,
-                state.positions[pos2],
+                state.positions[getPos()],
                 "while evaluating the attribute '%1%'",
                 showAttrPath(state, env, attrPath))
             : nullptr;
@@ -1587,9 +1627,8 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value &
                         "while calling %s",
                         lambda.name
                         ? concatStrings("'", symbols[lambda.name], "'")
-                        : "anonymous lambda",
-                        true);
-                    if (pos) addErrorTrace(e, pos, "from call site%s", "", true);
+                        : "anonymous lambda");
+                    if (pos) addErrorTrace(e, pos, "from call site");
                 }
                 throw;
             }
@@ -2737,9 +2776,12 @@ Expr * EvalState::parseExprFromFile(const SourcePath & path, std::shared_ptr<Sta
 
 Expr * EvalState::parseExprFromString(std::string s_, const SourcePath & basePath, std::shared_ptr<StaticEnv> & staticEnv)
 {
-    auto s = make_ref<std::string>(std::move(s_));
-    s->append("\0\0", 2);
-    return parse(s->data(), s->size(), Pos::String{.source = s}, basePath, staticEnv);
+    // NOTE this method (and parseStdin) must take care to *fully copy* their input
+    // into their respective Pos::Origin until the parser stops overwriting its input
+    // data.
+    auto s = make_ref<std::string>(s_);
+    s_.append("\0\0", 2);
+    return parse(s_.data(), s_.size(), Pos::String{.source = s}, basePath, staticEnv);
 }
 
 
@@ -2751,12 +2793,15 @@ Expr * EvalState::parseExprFromString(std::string s, const SourcePath & basePath
 
 Expr * EvalState::parseStdin()
 {
+    // NOTE this method (and parseExprFromString) must take care to *fully copy* their
+    // input into their respective Pos::Origin until the parser stops overwriting its
+    // input data.
     //Activity act(*logger, lvlTalkative, "parsing standard input");
     auto buffer = drainFD(0);
     // drainFD should have left some extra space for terminators
     buffer.append("\0\0", 2);
-    auto s = make_ref<std::string>(std::move(buffer));
-    return parse(s->data(), s->size(), Pos::Stdin{.source = s}, rootPath("."), staticBaseEnv);
+    auto s = make_ref<std::string>(buffer);
+    return parse(buffer.data(), buffer.size(), Pos::Stdin{.source = s}, rootPath("."), staticBaseEnv);
 }
 
 

src/libexpr/eval.hh

@@ -153,6 +153,7 @@ struct DebugTrace {
     bool isError;
 };
 
+
 class EvalState : public std::enable_shared_from_this<EvalState>
 {
 public:
@@ -222,6 +223,7 @@ public:
      */
     ReplExitStatus (* debugRepl)(ref<EvalState> es, const ValMap & extraEnv);
     bool debugStop;
+    bool inDebugger = false;
     int trylevel;
     std::list<DebugTrace> debugTraces;
     std::map<const Expr*, const std::shared_ptr<const StaticEnv>> exprEnvs;
@@ -432,10 +434,12 @@ public:
     std::string_view forceString(Value & v, NixStringContext & context, const PosIdx pos, std::string_view errorCtx);
     std::string_view forceStringNoCtx(Value & v, const PosIdx pos, std::string_view errorCtx);
 
+    template<typename... Args>
     [[gnu::noinline]]
-    void addErrorTrace(Error & e, const char * s, const std::string & s2) const;
+    void addErrorTrace(Error & e, const Args & ... formatArgs) const;
+    template<typename... Args>
     [[gnu::noinline]]
-    void addErrorTrace(Error & e, const PosIdx pos, const char * s, const std::string & s2, bool frame = false) const;
+    void addErrorTrace(Error & e, const PosIdx pos, const Args & ... formatArgs) const;
 
 public:
     /**

src/libexpr/flake/flake.cc

@@ -139,7 +139,7 @@ static FlakeInput parseFlakeInput(EvalState & state,
                         attrs.emplace(state.symbols[attr.name], Explicit<bool> { attr.value->boolean });
                         break;
                     case nInt:
-                        attrs.emplace(state.symbols[attr.name], (long unsigned int)attr.value->integer);
+                        attrs.emplace(state.symbols[attr.name], (long unsigned int) attr.value->integer);
                         break;
                     default:
                         if (attr.name == state.symbols.create("publicKeys")) {
@@ -202,43 +202,27 @@ static std::map<FlakeId, FlakeInput> parseFlakeInputs(
     return inputs;
 }
 
-static Flake getFlake(
+static Flake readFlake(
     EvalState & state,
     const FlakeRef & originalRef,
-    bool allowLookup,
-    FlakeCache & flakeCache,
-    InputPath lockRootPath)
+    const FlakeRef & resolvedRef,
+    const FlakeRef & lockedRef,
+    const SourcePath & rootDir,
+    const InputPath & lockRootPath)
 {
-    auto [storePath, resolvedRef, lockedRef] = fetchOrSubstituteTree(
-        state, originalRef, allowLookup, flakeCache);
+    auto flakePath = rootDir / CanonPath(resolvedRef.subdir) / "flake.nix";
 
-    // We need to guard against symlink attacks, but before we start doing
-    // filesystem operations we should make sure there's a flake.nix in the
-    // first place.
-    auto unsafeFlakeDir = state.store->toRealPath(storePath) + "/" + lockedRef.subdir;
-    auto unsafeFlakeFile = unsafeFlakeDir + "/flake.nix";
-    if (!pathExists(unsafeFlakeFile))
-        throw Error("source tree referenced by '%s' does not contain a '%s/flake.nix' file", lockedRef, lockedRef.subdir);
-
-    // Guard against symlink attacks.
-    auto flakeDir = canonPath(unsafeFlakeDir, true);
-    auto flakeFile = canonPath(flakeDir + "/flake.nix", true);
-    if (!isInDir(flakeFile, state.store->toRealPath(storePath)))
-        throw Error("'flake.nix' file of flake '%s' escapes from '%s'",
-            lockedRef, state.store->printStorePath(storePath));
+    // NOTE evalFile forces vInfo to be an attrset because mustBeTrivial is true.
+    Value vInfo;
+    state.evalFile(flakePath, vInfo, true);
 
     Flake flake {
         .originalRef = originalRef,
         .resolvedRef = resolvedRef,
         .lockedRef = lockedRef,
-        .storePath = storePath,
+        .path = flakePath,
     };
 
-    Value vInfo;
-    state.evalFile(state.rootPath(CanonPath(flakeFile)), vInfo, true); // FIXME: symlink attack
-
-    expectType(state, nAttrs, vInfo, state.positions.add({state.rootPath(CanonPath(flakeFile))}, 1, 1));
-
     if (auto description = vInfo.attrs->get(state.sDescription)) {
         expectType(state, nString, *description->value, description->pos);
         flake.description = description->value->c_str();
@@ -247,7 +231,7 @@ static Flake getFlake(
     auto sInputs = state.symbols.create("inputs");
 
     if (auto inputs = vInfo.attrs->get(sInputs))
-        flake.inputs = parseFlakeInputs(state, inputs->value, inputs->pos, flakeDir, lockRootPath);
+        flake.inputs = parseFlakeInputs(state, inputs->value, inputs->pos, flakePath.parent().path.abs(), lockRootPath); // FIXME
 
     auto sOutputs = state.symbols.create("outputs");
 
@@ -264,7 +248,7 @@ static Flake getFlake(
         }
 
     } else
-        throw Error("flake '%s' lacks attribute 'outputs'", lockedRef);
+        throw Error("flake '%s' lacks attribute 'outputs'", resolvedRef);
 
     auto sNixConfig = state.symbols.create("nixConfig");
 
@@ -281,7 +265,7 @@ static Flake getFlake(
                 NixStringContext emptyContext = {};
                 flake.config.settings.emplace(
                     state.symbols[setting.name],
-                    state.coerceToString(setting.pos, *setting.value, emptyContext, "", false, true, true) .toOwned());
+                    state.coerceToString(setting.pos, *setting.value, emptyContext, "", false, true, true).toOwned());
             }
             else if (setting.value->type() == nInt)
                 flake.config.settings.emplace(
@@ -313,12 +297,25 @@ static Flake getFlake(
             attr.name != sOutputs &&
             attr.name != sNixConfig)
             throw Error("flake '%s' has an unsupported attribute '%s', at %s",
-                lockedRef, state.symbols[attr.name], state.positions[attr.pos]);
+                resolvedRef, state.symbols[attr.name], state.positions[attr.pos]);
     }
 
     return flake;
 }
 
+static Flake getFlake(
+    EvalState & state,
+    const FlakeRef & originalRef,
+    bool allowLookup,
+    FlakeCache & flakeCache,
+    InputPath lockRootPath)
+{
+    auto [storePath, resolvedRef, lockedRef] = fetchOrSubstituteTree(
+        state, originalRef, allowLookup, flakeCache);
+
+    return readFlake(state, originalRef, resolvedRef, lockedRef, state.rootPath(state.store->toRealPath(storePath)), lockRootPath);
+}
+
 Flake getFlake(EvalState & state, const FlakeRef & originalRef, bool allowLookup, FlakeCache & flakeCache)
 {
     return getFlake(state, originalRef, allowLookup, flakeCache, {});
@@ -330,6 +327,13 @@ Flake getFlake(EvalState & state, const FlakeRef & originalRef, bool allowLookup
     return getFlake(state, originalRef, allowLookup, flakeCache);
 }
 
+static LockFile readLockFile(const SourcePath & lockFilePath)
+{
+    return lockFilePath.pathExists()
+        ? LockFile(lockFilePath.readFile(), fmt("%s", lockFilePath))
+        : LockFile();
+}
+
 /* Compute an in-memory lock file for the specified top-level flake,
    and optionally write it to file, if the flake is writable. */
 LockedFlake lockFlake(
@@ -355,17 +359,16 @@ LockedFlake lockFlake(
             throw Error("reference lock file was provided, but the `allow-dirty` setting is set to false");
         }
 
-        // FIXME: symlink attack
-        auto oldLockFile = LockFile::read(
+        auto oldLockFile = readLockFile(
             lockFlags.referenceLockFilePath.value_or(
-                state.store->toRealPath(flake.storePath) + "/" + flake.lockedRef.subdir + "/flake.lock"));
+                flake.lockFilePath()));
 
         debug("old lock file: %s", oldLockFile);
 
         std::map<InputPath, FlakeInput> overrides;
         std::set<InputPath> explicitCliOverrides;
         std::set<InputPath> overridesUsed, updatesUsed;
-        std::map<ref<Node>, StorePath> nodePaths;
+        std::map<ref<Node>, SourcePath> nodePaths;
 
         for (auto & i : lockFlags.inputOverrides) {
             overrides.insert_or_assign(i.first, FlakeInput { .ref = i.second });
@@ -538,7 +541,7 @@ LockedFlake lockFlake(
 
                         if (mustRefetch) {
                             auto inputFlake = getFlake(state, oldLock->lockedRef, false, flakeCache, inputPath);
-                            nodePaths.emplace(childNode, inputFlake.storePath);
+                            nodePaths.emplace(childNode, inputFlake.path.parent());
                             computeLocks(inputFlake.inputs, childNode, inputPath, oldLock, lockRootPath, parentPath, false);
                         } else {
                             computeLocks(fakeInputs, childNode, inputPath, oldLock, lockRootPath, parentPath, true);
@@ -587,13 +590,12 @@ LockedFlake lockFlake(
                                flake. Also, unless we already have this flake
                                in the top-level lock file, use this flake's
                                own lock file. */
-                            nodePaths.emplace(childNode, inputFlake.storePath);
+                            nodePaths.emplace(childNode, inputFlake.path.parent());
                             computeLocks(
                                 inputFlake.inputs, childNode, inputPath,
                                 oldLock
                                 ? std::dynamic_pointer_cast<const Node>(oldLock)
-                                : LockFile::read(
-                                    state.store->toRealPath(inputFlake.storePath) + "/" + inputFlake.lockedRef.subdir + "/flake.lock").root.get_ptr(),
+                                : readLockFile(inputFlake.lockFilePath()).root.get_ptr(),
                                 oldLock ? lockRootPath : inputPath,
                                 localPath,
                                 false);
@@ -605,7 +607,7 @@ LockedFlake lockFlake(
 
                             auto childNode = make_ref<LockedNode>(lockedRef, ref, false);
 
-                            nodePaths.emplace(childNode, storePath);
+                            nodePaths.emplace(childNode, state.rootPath(state.store->toRealPath(storePath)));
 
                             node->inputs.insert_or_assign(id, childNode);
                         }
@@ -619,9 +621,9 @@ LockedFlake lockFlake(
         };
 
         // Bring in the current ref for relative path resolution if we have it
-        auto parentPath = canonPath(state.store->toRealPath(flake.storePath) + "/" + flake.lockedRef.subdir, true);
+        auto parentPath = flake.path.parent().path.abs();
 
-        nodePaths.emplace(newLockFile.root, flake.storePath);
+        nodePaths.emplace(newLockFile.root, flake.path.parent());
 
         computeLocks(
             flake.inputs,
@@ -746,13 +748,15 @@ void callFlake(EvalState & state,
 
     auto overrides = state.buildBindings(lockedFlake.nodePaths.size());
 
-    for (auto & [node, storePath] : lockedFlake.nodePaths) {
+    for (auto & [node, sourcePath] : lockedFlake.nodePaths) {
         auto override = state.buildBindings(2);
 
         auto & vSourceInfo = override.alloc(state.symbols.create("sourceInfo"));
 
         auto lockedNode = node.dynamic_pointer_cast<const LockedNode>();
 
+        auto [storePath, subdir] = state.store->toStorePath(sourcePath.path.abs());
+
         emitTreeAttrs(
             state,
             storePath,
@@ -766,7 +770,7 @@ void callFlake(EvalState & state,
 
         override
             .alloc(state.symbols.create("dir"))
-            .mkString(lockedNode ? lockedNode->lockedRef.subdir : lockedFlake.flake.lockedRef.subdir);
+            .mkString(CanonPath(subdir).rel());
 
         overrides.alloc(state.symbols.create(key->second)).mkAttrs(override);
     }
@@ -921,18 +925,17 @@ static RegisterPrimOp r4({
 
 }
 
-Fingerprint LockedFlake::getFingerprint() const
+std::optional<Fingerprint> LockedFlake::getFingerprint(ref<Store> store) const
 {
+    if (lockFile.isUnlocked()) return std::nullopt;
+
+    auto fingerprint = flake.lockedRef.input.getFingerprint(store);
+    if (!fingerprint) return std::nullopt;
+
     // FIXME: as an optimization, if the flake contains a lock file
     // and we haven't changed it, then it's sufficient to use
     // flake.sourceInfo.storePath for the fingerprint.
-    return hashString(HashAlgorithm::SHA256,
-        fmt("%s;%s;%d;%d;%s",
-            flake.storePath.to_string(),
-            flake.lockedRef.subdir,
-            flake.lockedRef.input.getRevCount().value_or(0),
-            flake.lockedRef.input.getLastModified().value_or(0),
-            lockFile));
+    return hashString(HashAlgorithm::SHA256, fmt("%s;%s;%s", *fingerprint, flake.lockedRef.subdir, lockFile));
 }
 
 Flake::~Flake() { }

src/libexpr/flake/flake.hh

@@ -77,18 +77,27 @@ struct Flake
      * the specific local store result of invoking the fetcher
      */
     FlakeRef lockedRef;
+    /**
+     * The path of `flake.nix`.
+     */
+    SourcePath path;
     /**
      * pretend that 'lockedRef' is dirty
      */
     bool forceDirty = false;
     std::optional<std::string> description;
-    StorePath storePath;
     FlakeInputs inputs;
     /**
      * 'nixConfig' attribute
      */
     ConfigFile config;
+
     ~Flake();
+
+    SourcePath lockFilePath()
+    {
+        return path.parent() / "flake.lock";
+    }
 };
 
 Flake getFlake(EvalState & state, const FlakeRef & flakeRef, bool allowLookup);
@@ -104,13 +113,13 @@ struct LockedFlake
     LockFile lockFile;
 
     /**
-     * Store paths of nodes that have been fetched in
+     * Source tree accessors for nodes that have been fetched in
      * lockFlake(); in particular, the root node and the overriden
      * inputs.
      */
-    std::map<ref<Node>, StorePath> nodePaths;
+    std::map<ref<Node>, SourcePath> nodePaths;
 
-    Fingerprint getFingerprint() const;
+    std::optional<Fingerprint> getFingerprint(ref<Store> store) const;
 };
 
 struct LockFlags
@@ -165,7 +174,7 @@ struct LockFlags
     /**
      * The path to a lock file to read instead of the `flake.lock` file in the top-level flake
      */
-    std::optional<std::string> referenceLockFilePath;
+    std::optional<SourcePath> referenceLockFilePath;
 
     /**
      * The path to a lock file to write to instead of the `flake.lock` file in the top-level flake

src/libexpr/flake/flakeref.cc

@@ -102,6 +102,19 @@ std::pair<FlakeRef, std::string> parsePathFlakeRefWithFragment(
 
         if (isFlake) {
 
+            if (!S_ISDIR(lstat(path).st_mode)) {
+                if (baseNameOf(path) == "flake.nix") {
+                    // Be gentle with people who accidentally write `/foo/bar/flake.nix` instead of `/foo/bar`
+                    warn(
+                        "Path '%s' should point at the directory containing the 'flake.nix' file, not the file itself. "
+                        "Pretending that you meant '%s'"
+                        , path, dirOf(path));
+                    path = dirOf(path);
+                } else {
+                    throw BadURL("path '%s' is not a flake (because it's not a directory)", path);
+                }
+            }
+
             if (!allowMissing && !pathExists(path + "/flake.nix")){
                 notice("path '%s' does not contain a 'flake.nix', searching up",path);
 
@@ -124,9 +137,6 @@ std::pair<FlakeRef, std::string> parsePathFlakeRefWithFragment(
                     throw BadURL("could not find a flake.nix file");
             }
 
-            if (!S_ISDIR(lstat(path).st_mode))
-                throw BadURL("path '%s' is not a flake (because it's not a directory)", path);
-
             if (!allowMissing && !pathExists(path + "/flake.nix"))
                 throw BadURL("path '%s' is not a flake (because it doesn't contain a 'flake.nix' file)", path);
 
@@ -274,7 +284,7 @@ FlakeRef FlakeRef::fromAttrs(const fetchers::Attrs & attrs)
 
 std::pair<StorePath, FlakeRef> FlakeRef::fetchTree(ref<Store> store) const
 {
-    auto [storePath, lockedInput] = input.fetch(store);
+    auto [storePath, lockedInput] = input.fetchToStore(store);
     return {std::move(storePath), FlakeRef(std::move(lockedInput), subdir)};
 }
 

src/libexpr/flake/lockfile.cc

@@ -84,8 +84,10 @@ std::shared_ptr<Node> LockFile::findInput(const InputPath & path)
     return doFind(root, path, visited);
 }
 
-LockFile::LockFile(const nlohmann::json & json, const Path & path)
+LockFile::LockFile(std::string_view contents, std::string_view path)
 {
+    auto json = nlohmann::json::parse(contents);
+
     auto version = json.value("version", 0);
     if (version < 5 || version > 7)
         throw Error("lock file '%s' has unsupported version %d", path, version);
@@ -203,12 +205,6 @@ std::pair<std::string, LockFile::KeyMap> LockFile::to_string() const
     return {json.dump(2), std::move(nodeKeys)};
 }
 
-LockFile LockFile::read(const Path & path)
-{
-    if (!pathExists(path)) return LockFile();
-    return LockFile(nlohmann::json::parse(readFile(path)), path);
-}
-
 std::ostream & operator <<(std::ostream & stream, const LockFile & lockFile)
 {
     stream << lockFile.toJSON().first.dump(2);

src/libexpr/flake/lockfile.hh

@@ -55,7 +55,7 @@ struct LockFile
     ref<Node> root = make_ref<Node>();
 
     LockFile() {};
-    LockFile(const nlohmann::json & json, const Path & path);
+    LockFile(std::string_view contents, std::string_view path);
 
     typedef std::map<ref<const Node>, std::string> KeyMap;
 
@@ -63,8 +63,6 @@ struct LockFile
 
     std::pair<std::string, KeyMap> to_string() const;
 
-    static LockFile read(const Path & path);
-
     /**
      * Check whether this lock file has any unlocked inputs. If so,
      * return one.

src/libexpr/flake/url-name.cc

@@ -5,13 +5,12 @@
 namespace nix {
 
 static const std::string attributeNamePattern("[a-zA-Z0-9_-]+");
-static const std::regex lastAttributeRegex("(?:" + attributeNamePattern + "\\.)*(?!default)(" + attributeNamePattern +")(\\^.*)?");
+static const std::regex lastAttributeRegex("^((?:" + attributeNamePattern + "\\.)*)(" + attributeNamePattern +")(\\^.*)?$");
 static const std::string pathSegmentPattern("[a-zA-Z0-9_-]+");
 static const std::regex lastPathSegmentRegex(".*/(" + pathSegmentPattern +")");
 static const std::regex secondPathSegmentRegex("(?:" + pathSegmentPattern + ")/(" + pathSegmentPattern +")(?:/.*)?");
 static const std::regex gitProviderRegex("github|gitlab|sourcehut");
 static const std::regex gitSchemeRegex("git($|\\+.*)");
-static const std::regex defaultOutputRegex(".*\\.default($|\\^.*)");
 
 std::optional<std::string> getNameFromURL(const ParsedURL & url)
 {
@@ -22,8 +21,11 @@ std::optional<std::string> getNameFromURL(const ParsedURL & url)
         return url.query.at("dir");
 
     /* If the fragment isn't a "default" and contains two attribute elements, use the last one */
-    if (std::regex_match(url.fragment, match, lastAttributeRegex))
-        return match.str(1);
+    if (std::regex_match(url.fragment, match, lastAttributeRegex)
+        && match.str(1) != "defaultPackage."
+        && match.str(2) != "default") {
+        return match.str(2);
+    }
 
     /* If this is a github/gitlab/sourcehut flake, use the repo name */
     if (std::regex_match(url.scheme, gitProviderRegex) && std::regex_match(url.path, match, secondPathSegmentRegex))
@@ -33,10 +35,6 @@ std::optional<std::string> getNameFromURL(const ParsedURL & url)
     if (std::regex_match(url.scheme, gitSchemeRegex) && std::regex_match(url.path, match, lastPathSegmentRegex))
         return match.str(1);
 
-    /* If everything failed but there is a non-default fragment, use it in full */
-    if (!url.fragment.empty() && !std::regex_match(url.fragment, defaultOutputRegex))
-        return url.fragment;
-
     /* If there is no fragment, take the last element of the path */
     if (std::regex_match(url.path, match, lastPathSegmentRegex))
         return match.str(1);

src/libexpr/lexer.l

@@ -33,33 +33,16 @@ namespace nix {
 
 static void initLoc(YYLTYPE * loc)
 {
-    loc->first_line = loc->last_line = 1;
-    loc->first_column = loc->last_column = 1;
+    loc->first_line = loc->last_line = 0;
+    loc->first_column = loc->last_column = 0;
 }
 
 static void adjustLoc(YYLTYPE * loc, const char * s, size_t len)
 {
     loc->stash();
 
-    loc->first_line = loc->last_line;
     loc->first_column = loc->last_column;
-
-    for (size_t i = 0; i < len; i++) {
-       switch (*s++) {
-       case '\r':
-           if (*s == '\n') { /* cr/lf */
-               i++;
-               s++;
-           }
-           /* fall through */
-       case '\n':
-           ++loc->last_line;
-           loc->last_column = 1;
-           break;
-       default:
-           ++loc->last_column;
-       }
-    }
+    loc->last_column += len;
 }
 
 
@@ -94,6 +77,9 @@ static StringToken unescapeStr(SymbolTable & symbols, char * s, size_t length)
 
 }
 
+// yacc generates code that uses unannotated fallthrough.
+#pragma GCC diagnostic ignored "-Wimplicit-fallthrough"
+
 #define YY_USER_INIT initLoc(yylloc)
 #define YY_USER_ACTION adjustLoc(yylloc, yytext, yyleng);
 

src/libexpr/nixexpr.cc

@@ -70,10 +70,8 @@ void ExprOpHasAttr::show(const SymbolTable & symbols, std::ostream & str) const
     str << ") ? " << showAttrPath(symbols, attrPath) << ")";
 }
 
-void ExprAttrs::show(const SymbolTable & symbols, std::ostream & str) const
+void ExprAttrs::showBindings(const SymbolTable & symbols, std::ostream & str) const
 {
-    if (recursive) str << "rec ";
-    str << "{ ";
     typedef const decltype(attrs)::value_type * Attr;
     std::vector<Attr> sorted;
     for (auto & i : attrs) sorted.push_back(&i);
@@ -81,10 +79,37 @@ void ExprAttrs::show(const SymbolTable & symbols, std::ostream & str) const
         std::string_view sa = symbols[a->first], sb = symbols[b->first];
         return sa < sb;
     });
+    std::vector<Symbol> inherits;
+    std::map<ExprInheritFrom *, std::vector<Symbol>> inheritsFrom;
     for (auto & i : sorted) {
-        if (i->second.inherited)
-            str << "inherit " << symbols[i->first] << " " << "; ";
-        else {
+        switch (i->second.kind) {
+        case AttrDef::Kind::Plain:
+            break;
+        case AttrDef::Kind::Inherited:
+            inherits.push_back(i->first);
+            break;
+        case AttrDef::Kind::InheritedFrom: {
+            auto & select = dynamic_cast<ExprSelect &>(*i->second.e);
+            auto & from = dynamic_cast<ExprInheritFrom &>(*select.e);
+            inheritsFrom[&from].push_back(i->first);
+            break;
+        }
+        }
+    }
+    if (!inherits.empty()) {
+        str << "inherit";
+        for (auto sym : inherits) str << " " << symbols[sym];
+        str << "; ";
+    }
+    for (const auto & [from, syms] : inheritsFrom) {
+        str << "inherit (";
+        (*inheritFromExprs)[from->displ]->show(symbols, str);
+        str << ")";
+        for (auto sym : syms) str << " " << symbols[sym];
+        str << "; ";
+    }
+    for (auto & i : sorted) {
+        if (i->second.kind == AttrDef::Kind::Plain) {
             str << symbols[i->first] << " = ";
             i->second.e->show(symbols, str);
             str << "; ";
@@ -97,6 +122,13 @@ void ExprAttrs::show(const SymbolTable & symbols, std::ostream & str) const
         i.valueExpr->show(symbols, str);
         str << "; ";
     }
+}
+
+void ExprAttrs::show(const SymbolTable & symbols, std::ostream & str) const
+{
+    if (recursive) str << "rec ";
+    str << "{ ";
+    showBindings(symbols, str);
     str << "}";
 }
 
@@ -117,7 +149,10 @@ void ExprLambda::show(const SymbolTable & symbols, std::ostream & str) const
     if (hasFormals()) {
         str << "{ ";
         bool first = true;
-        for (auto & i : formals->formals) {
+        // the natural Symbol ordering is by creation time, which can lead to the
+        // same expression being printed in two different ways depending on its
+        // context. always use lexicographic ordering to avoid this.
+        for (auto & i : formals->lexicographicOrder(symbols)) {
             if (first) first = false; else str << ", ";
             str << symbols[i.name];
             if (i.def) {
@@ -152,15 +187,7 @@ void ExprCall::show(const SymbolTable & symbols, std::ostream & str) const
 void ExprLet::show(const SymbolTable & symbols, std::ostream & str) const
 {
     str << "(let ";
-    for (auto & i : attrs->attrs)
-        if (i.second.inherited) {
-            str << "inherit " << symbols[i.first] << "; ";
-        }
-        else {
-            str << symbols[i.first] << " = ";
-            i.second.e->show(symbols, str);
-            str << "; ";
-        }
+    attrs->showBindings(symbols, str);
     str << "in ";
     body->show(symbols, str);
     str << ")";
@@ -305,6 +332,12 @@ void ExprVar::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> &
     this->level = withLevel;
 }
 
+void ExprInheritFrom::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> & env)
+{
+    if (es.debugRepl)
+        es.exprEnvs.insert(std::make_pair(this, env));
+}
+
 void ExprSelect::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> & env)
 {
     if (es.debugRepl)
@@ -328,22 +361,47 @@ void ExprOpHasAttr::bindVars(EvalState & es, const std::shared_ptr<const StaticE
             i.expr->bindVars(es, env);
 }
 
+std::shared_ptr<const StaticEnv> ExprAttrs::bindInheritSources(
+    EvalState & es, const std::shared_ptr<const StaticEnv> & env)
+{
+    if (!inheritFromExprs)
+        return nullptr;
+
+    // the inherit (from) source values are inserted into an env of its own, which
+    // does not introduce any variable names.
+    // analysis must see an empty env, or an env that contains only entries with
+    // otherwise unused names to not interfere with regular names. the parser
+    // has already filled all exprs that access this env with appropriate level
+    // and displacement, and nothing else is allowed to access it. ideally we'd
+    // not even *have* an expr that grabs anything from this env since it's fully
+    // invisible, but the evaluator does not allow for this yet.
+    auto inner = std::make_shared<StaticEnv>(nullptr, env.get(), 0);
+    for (auto from : *inheritFromExprs)
+        from->bindVars(es, env);
+
+    return inner;
+}
+
 void ExprAttrs::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> & env)
 {
     if (es.debugRepl)
         es.exprEnvs.insert(std::make_pair(this, env));
 
     if (recursive) {
-        auto newEnv = std::make_shared<StaticEnv>(nullptr, env.get(), recursive ? attrs.size() : 0);
+        auto newEnv = [&] () -> std::shared_ptr<const StaticEnv> {
+            auto newEnv = std::make_shared<StaticEnv>(nullptr, env.get(), attrs.size());
 
-        Displacement displ = 0;
-        for (auto & i : attrs)
-            newEnv->vars.emplace_back(i.first, i.second.displ = displ++);
+            Displacement displ = 0;
+            for (auto & i : attrs)
+                newEnv->vars.emplace_back(i.first, i.second.displ = displ++);
+            return newEnv;
+        }();
 
         // No need to sort newEnv since attrs is in sorted order.
 
+        auto inheritFromEnv = bindInheritSources(es, newEnv);
         for (auto & i : attrs)
-            i.second.e->bindVars(es, i.second.inherited ? env : newEnv);
+            i.second.e->bindVars(es, i.second.chooseByKind(newEnv, env, inheritFromEnv));
 
         for (auto & i : dynamicAttrs) {
             i.nameExpr->bindVars(es, newEnv);
@@ -351,8 +409,10 @@ void ExprAttrs::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv>
         }
     }
     else {
+        auto inheritFromEnv = bindInheritSources(es, env);
+
         for (auto & i : attrs)
-            i.second.e->bindVars(es, env);
+            i.second.e->bindVars(es, i.second.chooseByKind(env, env, inheritFromEnv));
 
         for (auto & i : dynamicAttrs) {
             i.nameExpr->bindVars(es, env);
@@ -409,16 +469,20 @@ void ExprCall::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> &
 
 void ExprLet::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> & env)
 {
-    auto newEnv = std::make_shared<StaticEnv>(nullptr, env.get(), attrs->attrs.size());
+    auto newEnv = [&] () -> std::shared_ptr<const StaticEnv> {
+        auto newEnv = std::make_shared<StaticEnv>(nullptr, env.get(), attrs->attrs.size());
 
-    Displacement displ = 0;
-    for (auto & i : attrs->attrs)
-        newEnv->vars.emplace_back(i.first, i.second.displ = displ++);
+        Displacement displ = 0;
+        for (auto & i : attrs->attrs)
+            newEnv->vars.emplace_back(i.first, i.second.displ = displ++);
+        return newEnv;
+    }();
 
     // No need to sort newEnv since attrs->attrs is in sorted order.
 
+    auto inheritFromEnv = attrs->bindInheritSources(es, newEnv);
     for (auto & i : attrs->attrs)
-        i.second.e->bindVars(es, i.second.inherited ? env : newEnv);
+        i.second.e->bindVars(es, i.second.chooseByKind(newEnv, env, inheritFromEnv));
 
     if (es.debugRepl)
         es.exprEnvs.insert(std::make_pair(this, newEnv));
@@ -519,6 +583,39 @@ std::string ExprLambda::showNamePos(const EvalState & state) const
 
 
 
+/* Position table. */
+
+Pos PosTable::operator[](PosIdx p) const
+{
+    auto origin = resolve(p);
+    if (!origin)
+        return {};
+
+    const auto offset = origin->offsetOf(p);
+
+    Pos result{0, 0, origin->origin};
+    auto lines = this->lines.lock();
+    auto linesForInput = (*lines)[origin->offset];
+
+    if (linesForInput.empty()) {
+        auto source = result.getSource().value_or("");
+        const char * begin = source.data();
+        for (Pos::LinesIterator it(source), end; it != end; it++)
+            linesForInput.push_back(it->data() - begin);
+        if (linesForInput.empty())
+            linesForInput.push_back(0);
+    }
+    // as above: the first line starts at byte 0 and is always present
+    auto lineStartOffset = std::prev(
+        std::upper_bound(linesForInput.begin(), linesForInput.end(), offset));
+
+    result.line = 1 + (lineStartOffset - linesForInput.begin());
+    result.column = 1 + (offset - *lineStartOffset);
+    return result;
+}
+
+
+
 /* Symbol table. */
 
 size_t SymbolTable::totalSize() const

src/libexpr/nixexpr.hh

@@ -7,7 +7,6 @@
 #include "value.hh"
 #include "symbol-table.hh"
 #include "error.hh"
-#include "chunked-vector.hh"
 #include "position.hh"
 #include "eval-error.hh"
 #include "pos-idx.hh"
@@ -135,6 +134,23 @@ struct ExprVar : Expr
     COMMON_METHODS
 };
 
+/**
+ * A pseudo-expression for the purpose of evaluating the `from` expression in `inherit (from)` syntax.
+ * Unlike normal variable references, the displacement is set during parsing, and always refers to
+ * `ExprAttrs::inheritFromExprs` (by itself or in `ExprLet`), whose values are put into their own `Env`.
+ */
+struct ExprInheritFrom : ExprVar
+{
+    ExprInheritFrom(PosIdx pos, Displacement displ): ExprVar(pos, {})
+    {
+        this->level = 0;
+        this->displ = displ;
+        this->fromWith = nullptr;
+    }
+
+    void bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> & env);
+};
+
 struct ExprSelect : Expr
 {
     PosIdx pos;
@@ -160,16 +176,40 @@ struct ExprAttrs : Expr
     bool recursive;
     PosIdx pos;
     struct AttrDef {
-        bool inherited;
+        enum class Kind {
+            /** `attr = expr;` */
+            Plain,
+            /** `inherit attr1 attrn;` */
+            Inherited,
+            /** `inherit (expr) attr1 attrn;` */
+            InheritedFrom,
+        };
+
+        Kind kind;
         Expr * e;
         PosIdx pos;
         Displacement displ; // displacement
-        AttrDef(Expr * e, const PosIdx & pos, bool inherited=false)
-            : inherited(inherited), e(e), pos(pos) { };
+        AttrDef(Expr * e, const PosIdx & pos, Kind kind = Kind::Plain)
+            : kind(kind), e(e), pos(pos) { };
         AttrDef() { };
+
+        template<typename T>
+        const T & chooseByKind(const T & plain, const T & inherited, const T & inheritedFrom) const
+        {
+            switch (kind) {
+            case Kind::Plain:
+                return plain;
+            case Kind::Inherited:
+                return inherited;
+            default:
+            case Kind::InheritedFrom:
+                return inheritedFrom;
+            }
+        }
     };
     typedef std::map<Symbol, AttrDef> AttrDefs;
     AttrDefs attrs;
+    std::unique_ptr<std::vector<Expr *>> inheritFromExprs;
     struct DynamicAttrDef {
         Expr * nameExpr, * valueExpr;
         PosIdx pos;
@@ -182,6 +222,11 @@ struct ExprAttrs : Expr
     ExprAttrs() : recursive(false) { };
     PosIdx getPos() const override { return pos; }
     COMMON_METHODS
+
+    std::shared_ptr<const StaticEnv> bindInheritSources(
+        EvalState & es, const std::shared_ptr<const StaticEnv> & env);
+    Env * buildInheritFromEnv(EvalState & state, Env & up);
+    void showBindings(const SymbolTable & symbols, std::ostream & str) const;
 };
 
 struct ExprList : Expr

src/libexpr/parser-state.hh

@@ -24,20 +24,15 @@ struct ParserLocation
     int last_line, last_column;
 
     // backup to recover from yyless(0)
-    int stashed_first_line, stashed_first_column;
-    int stashed_last_line, stashed_last_column;
+    int stashed_first_column, stashed_last_column;
 
     void stash() {
-        stashed_first_line = first_line;
         stashed_first_column = first_column;
-        stashed_last_line = last_line;
         stashed_last_column = last_column;
     }
 
     void unstash() {
-        first_line = stashed_first_line;
         first_column = stashed_first_column;
-        last_line = stashed_last_line;
         last_column = stashed_last_column;
     }
 };
@@ -89,7 +84,7 @@ inline void ParserState::addAttr(ExprAttrs * attrs, AttrPath && attrPath, Expr *
         if (i->symbol) {
             ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(i->symbol);
             if (j != attrs->attrs.end()) {
-                if (!j->second.inherited) {
+                if (j->second.kind != ExprAttrs::AttrDef::Kind::Inherited) {
                     ExprAttrs * attrs2 = dynamic_cast<ExprAttrs *>(j->second.e);
                     if (!attrs2) dupAttr(attrPath, pos, j->second.pos);
                     attrs = attrs2;
@@ -118,13 +113,24 @@ inline void ParserState::addAttr(ExprAttrs * attrs, AttrPath && attrPath, Expr *
             auto ae = dynamic_cast<ExprAttrs *>(e);
             auto jAttrs = dynamic_cast<ExprAttrs *>(j->second.e);
             if (jAttrs && ae) {
+                if (ae->inheritFromExprs && !jAttrs->inheritFromExprs)
+                    jAttrs->inheritFromExprs = std::make_unique<std::vector<Expr *>>();
                 for (auto & ad : ae->attrs) {
                     auto j2 = jAttrs->attrs.find(ad.first);
                     if (j2 != jAttrs->attrs.end()) // Attr already defined in iAttrs, error.
                         dupAttr(ad.first, j2->second.pos, ad.second.pos);
                     jAttrs->attrs.emplace(ad.first, ad.second);
+                    if (ad.second.kind == ExprAttrs::AttrDef::Kind::InheritedFrom) {
+                        auto & sel = dynamic_cast<ExprSelect &>(*ad.second.e);
+                        auto & from = dynamic_cast<ExprInheritFrom &>(*sel.e);
+                        from.displ += jAttrs->inheritFromExprs->size();
+                    }
                 }
                 jAttrs->dynamicAttrs.insert(jAttrs->dynamicAttrs.end(), ae->dynamicAttrs.begin(), ae->dynamicAttrs.end());
+                if (ae->inheritFromExprs) {
+                    jAttrs->inheritFromExprs->insert(jAttrs->inheritFromExprs->end(),
+                        ae->inheritFromExprs->begin(), ae->inheritFromExprs->end());
+                }
             } else {
                 dupAttr(attrPath, pos, j->second.pos);
             }
@@ -265,7 +271,7 @@ inline Expr * ParserState::stripIndentation(const PosIdx pos,
 
 inline PosIdx ParserState::at(const ParserLocation & loc)
 {
-    return positions.add(origin, loc.first_line, loc.first_column);
+    return positions.add(origin, loc.first_column);
 }
 
 }

src/libexpr/parser.y

@@ -64,6 +64,10 @@ using namespace nix;
 
 void yyerror(YYLTYPE * loc, yyscan_t scanner, ParserState * state, const char * error)
 {
+    if (std::string_view(error).starts_with("syntax error, unexpected end of file")) {
+        loc->first_column = loc->last_column;
+        loc->first_line = loc->last_line;
+    }
     throw ParseError({
         .msg = HintFmt(error),
         .pos = state->positions[state->at(*loc)]
@@ -87,6 +91,7 @@ void yyerror(YYLTYPE * loc, yyscan_t scanner, ParserState * state, const char *
   nix::StringToken uri;
   nix::StringToken str;
   std::vector<nix::AttrName> * attrNames;
+  std::vector<std::pair<nix::AttrName, nix::PosIdx>> * inheritAttrs;
   std::vector<std::pair<nix::PosIdx, nix::Expr *>> * string_parts;
   std::vector<std::pair<nix::PosIdx, std::variant<nix::Expr *, nix::StringToken>>> * ind_string_parts;
 }
@@ -97,7 +102,8 @@ void yyerror(YYLTYPE * loc, yyscan_t scanner, ParserState * state, const char *
 %type <attrs> binds
 %type <formals> formals
 %type <formal> formal
-%type <attrNames> attrs attrpath
+%type <attrNames> attrpath
+%type <inheritAttrs> attrs
 %type <string_parts> string_parts_interpolated
 %type <ind_string_parts> ind_string_parts
 %type <e> path_start string_parts string_attr
@@ -309,21 +315,30 @@ binds
   : binds attrpath '=' expr ';' { $$ = $1; state->addAttr($$, std::move(*$2), $4, state->at(@2)); delete $2; }
   | binds INHERIT attrs ';'
     { $$ = $1;
-      for (auto & i : *$3) {
+      for (auto & [i, iPos] : *$3) {
           if ($$->attrs.find(i.symbol) != $$->attrs.end())
-              state->dupAttr(i.symbol, state->at(@3), $$->attrs[i.symbol].pos);
-          auto pos = state->at(@3);
-          $$->attrs.emplace(i.symbol, ExprAttrs::AttrDef(new ExprVar(CUR_POS, i.symbol), pos, true));
+              state->dupAttr(i.symbol, iPos, $$->attrs[i.symbol].pos);
+          $$->attrs.emplace(
+              i.symbol,
+              ExprAttrs::AttrDef(new ExprVar(iPos, i.symbol), iPos, ExprAttrs::AttrDef::Kind::Inherited));
       }
       delete $3;
     }
   | binds INHERIT '(' expr ')' attrs ';'
     { $$ = $1;
-      /* !!! Should ensure sharing of the expression in $4. */
-      for (auto & i : *$6) {
+      if (!$$->inheritFromExprs)
+          $$->inheritFromExprs = std::make_unique<std::vector<Expr *>>();
+      $$->inheritFromExprs->push_back($4);
+      auto from = new nix::ExprInheritFrom(state->at(@4), $$->inheritFromExprs->size() - 1);
+      for (auto & [i, iPos] : *$6) {
           if ($$->attrs.find(i.symbol) != $$->attrs.end())
-              state->dupAttr(i.symbol, state->at(@6), $$->attrs[i.symbol].pos);
-          $$->attrs.emplace(i.symbol, ExprAttrs::AttrDef(new ExprSelect(CUR_POS, $4, i.symbol), state->at(@6)));
+              state->dupAttr(i.symbol, iPos, $$->attrs[i.symbol].pos);
+          $$->attrs.emplace(
+              i.symbol,
+              ExprAttrs::AttrDef(
+                  new ExprSelect(iPos, from, i.symbol),
+                  iPos,
+                  ExprAttrs::AttrDef::Kind::InheritedFrom));
       }
       delete $6;
     }
@@ -331,12 +346,12 @@ binds
   ;
 
 attrs
-  : attrs attr { $$ = $1; $1->push_back(AttrName(state->symbols.create($2))); }
+  : attrs attr { $$ = $1; $1->emplace_back(AttrName(state->symbols.create($2)), state->at(@2)); }
   | attrs string_attr
     { $$ = $1;
       ExprString * str = dynamic_cast<ExprString *>($2);
       if (str) {
-          $$->push_back(AttrName(state->symbols.create(str->s)));
+          $$->emplace_back(AttrName(state->symbols.create(str->s)), state->at(@2));
           delete str;
       } else
           throw ParseError({
@@ -344,7 +359,7 @@ attrs
               .pos = state->positions[state->at(@2)]
           });
     }
-  | { $$ = new AttrPath; }
+  | { $$ = new std::vector<std::pair<AttrName, PosIdx>>; }
   ;
 
 attrpath
@@ -423,7 +438,7 @@ Expr * parseExprFromBuf(
         .symbols = symbols,
         .positions = positions,
         .basePath = basePath,
-        .origin = {origin},
+        .origin = positions.addOrigin(origin, length),
         .rootFS = rootFS,
         .s = astSymbols,
     };

src/libexpr/pos-idx.hh

@@ -6,6 +6,7 @@ namespace nix {
 
 class PosIdx
 {
+    friend struct LazyPosAcessors;
     friend class PosTable;
 
 private:

src/libexpr/pos-table.hh

@@ -7,6 +7,7 @@
 #include "chunked-vector.hh"
 #include "pos-idx.hh"
 #include "position.hh"
+#include "sync.hh"
 
 namespace nix {
 
@@ -17,66 +18,69 @@ public:
     {
         friend PosTable;
     private:
-        // must always be invalid by default, add() replaces this with the actual value.
-        // subsequent add() calls use this index as a token to quickly check whether the
-        // current origins.back() can be reused or not.
-        mutable uint32_t idx = std::numeric_limits<uint32_t>::max();
+        uint32_t offset;
 
-        // Used for searching in PosTable::[].
-        explicit Origin(uint32_t idx)
-            : idx(idx)
-            , origin{std::monostate()}
-        {
-        }
+        Origin(Pos::Origin origin, uint32_t offset, size_t size):
+            offset(offset), origin(origin), size(size)
+        {}
 
     public:
         const Pos::Origin origin;
+        const size_t size;
 
-        Origin(Pos::Origin origin)
-            : origin(origin)
+        uint32_t offsetOf(PosIdx p) const
         {
+            return p.id - 1 - offset;
         }
     };
 
-    struct Offset
-    {
-        uint32_t line, column;
-    };
-
 private:
-    std::vector<Origin> origins;
-    ChunkedVector<Offset, 8192> offsets;
+    using Lines = std::vector<uint32_t>;
 
-public:
-    PosTable()
-        : offsets(1024)
-    {
-        origins.reserve(1024);
-    }
+    std::map<uint32_t, Origin> origins;
+    mutable Sync<std::map<uint32_t, Lines>> lines;
 
-    PosIdx add(const Origin & origin, uint32_t line, uint32_t column)
+    const Origin * resolve(PosIdx p) const
     {
-        const auto idx = offsets.add({line, column}).second;
-        if (origins.empty() || origins.back().idx != origin.idx) {
-            origin.idx = idx;
-            origins.push_back(origin);
-        }
-        return PosIdx(idx + 1);
-    }
+        if (p.id == 0)
+            return nullptr;
 
-    Pos operator[](PosIdx p) const
-    {
-        if (p.id == 0 || p.id > offsets.size())
-            return {};
         const auto idx = p.id - 1;
         /* we want the last key <= idx, so we'll take prev(first key > idx).
-           this is guaranteed to never rewind origin.begin because the first
-           key is always 0. */
-        const auto pastOrigin = std::upper_bound(
-            origins.begin(), origins.end(), Origin(idx), [](const auto & a, const auto & b) { return a.idx < b.idx; });
-        const auto origin = *std::prev(pastOrigin);
-        const auto offset = offsets[idx];
-        return {offset.line, offset.column, origin.origin};
+            this is guaranteed to never rewind origin.begin because the first
+            key is always 0. */
+        const auto pastOrigin = origins.upper_bound(idx);
+        return &std::prev(pastOrigin)->second;
+    }
+
+public:
+    Origin addOrigin(Pos::Origin origin, size_t size)
+    {
+        uint32_t offset = 0;
+        if (auto it = origins.rbegin(); it != origins.rend())
+            offset = it->first + it->second.size;
+        // +1 because all PosIdx are offset by 1 to begin with, and
+        // another +1 to ensure that all origins can point to EOF, eg
+        // on (invalid) empty inputs.
+        if (2 + offset + size < offset)
+            return Origin{origin, offset, 0};
+        return origins.emplace(offset, Origin{origin, offset, size}).first->second;
+    }
+
+    PosIdx add(const Origin & origin, size_t offset)
+    {
+        if (offset > origin.size)
+            return PosIdx();
+        return PosIdx(1 + origin.offset + offset);
+    }
+
+    Pos operator[](PosIdx p) const;
+
+    Pos::Origin originOf(PosIdx p) const
+    {
+        if (auto o = resolve(p))
+            return o->origin;
+        return std::monostate{};
     }
 };
 

src/libexpr/primops.cc

@@ -705,38 +705,53 @@ static RegisterPrimOp primop_genericClosure(PrimOp {
     .args = {"attrset"},
     .arity = 1,
     .doc = R"(
-      Take an *attrset* with values named `startSet` and `operator` in order to
-      return a *list of attrsets* by starting with the `startSet` and recursively
-      applying the `operator` function to each `item`. The *attrsets* in the
-      `startSet` and the *attrsets* produced by `operator` must contain a value
-      named `key` which is comparable. The result is produced by calling `operator`
-      for each `item` with a value for `key` that has not been called yet including
-      newly produced `item`s. The function terminates when no new `item`s are
-      produced. The resulting *list of attrsets* contains only *attrsets* with a
-      unique key. For example,
+      `builtins.genericClosure` iteratively computes the transitive closure over an arbitrary relation defined by a function.
 
-      ```
-      builtins.genericClosure {
-        startSet = [ {key = 5;} ];
-        operator = item: [{
-          key = if (item.key / 2 ) * 2 == item.key
-               then item.key / 2
-               else 3 * item.key + 1;
-        }];
-      }
-      ```
-      evaluates to
-      ```
-      [ { key = 5; } { key = 16; } { key = 8; } { key = 4; } { key = 2; } { key = 1; } ]
-      ```
+      It takes *attrset* with two attributes named `startSet` and `operator`, and returns a list of attrbute sets:
+
+      - `startSet`:
+        The initial list of attribute sets.
+
+      - `operator`:
+        A function that takes an attribute set and returns a list of attribute sets.
+        It defines how each item in the current set is processed and expanded into more items.
+
+      Each attribute set in the list `startSet` and the list returned by `operator` must have an attribute `key`, which must support equality comparison.
+      The value of `key` can be one of the following types:
 
-      `key` can be one of the following types:
       - [Number](@docroot@/language/values.md#type-number)
       - [Boolean](@docroot@/language/values.md#type-boolean)
       - [String](@docroot@/language/values.md#type-string)
       - [Path](@docroot@/language/values.md#type-path)
       - [List](@docroot@/language/values.md#list)
 
+      The result is produced by calling the `operator` on each `item` that has not been called yet, including newly added items, until no new items are added.
+      Items are compared by their `key` attribute.
+
+      Common usages are:
+
+      - Generating unique collections of items, such as dependency graphs.
+      - Traversing through structures that may contain cycles or loops.
+      - Processing data structures with complex internal relationships.
+
+      > **Example**
+      >
+      > ```nix
+      > builtins.genericClosure {
+      >   startSet = [ {key = 5;} ];
+      >   operator = item: [{
+      >     key = if (item.key / 2 ) * 2 == item.key
+      >          then item.key / 2
+      >          else 3 * item.key + 1;
+      >   }];
+      > }
+      > ```
+      >
+      > evaluates to
+      >
+      > ```nix
+      > [ { key = 5; } { key = 16; } { key = 8; } { key = 4; } { key = 2; } { key = 1; } ]
+      > ```
       )",
     .fun = prim_genericClosure,
 });
@@ -811,7 +826,7 @@ static void prim_addErrorContext(EvalState & state, const PosIdx pos, Value * *
         auto message = state.coerceToString(pos, *args[0], context,
                 "while evaluating the error message passed to builtins.addErrorContext",
                 false, false).toOwned();
-        e.addTrace(nullptr, HintFmt(message), true);
+        e.addTrace(nullptr, HintFmt(message));
         throw;
     }
 }
@@ -1075,7 +1090,7 @@ static void prim_derivationStrict(EvalState & state, const PosIdx pos, Value * *
         e.addTrace(nullptr, HintFmt(
                 "while evaluating derivation '%s'\n"
                 "  whose name attribute is located at %s",
-                drvName, pos), true);
+                drvName, pos));
         throw;
     }
 }
@@ -1123,7 +1138,10 @@ drvName, Bindings * attrs, Value & v)
         auto handleHashMode = [&](const std::string_view s) {
             if (s == "recursive") ingestionMethod = FileIngestionMethod::Recursive;
             else if (s == "flat") ingestionMethod = FileIngestionMethod::Flat;
-            else if (s == "text") {
+            else if (s == "git") {
+                experimentalFeatureSettings.require(Xp::GitHashing);
+                ingestionMethod = FileIngestionMethod::Git;
+            } else if (s == "text") {
                 experimentalFeatureSettings.require(Xp::DynamicDerivations);
                 ingestionMethod = TextIngestionMethod {};
             } else
@@ -1233,8 +1251,7 @@ drvName, Bindings * attrs, Value & v)
 
         } catch (Error & e) {
             e.addTrace(state.positions[i->pos],
-                HintFmt("while evaluating attribute '%1%' of derivation '%2%'", key, drvName),
-                true);
+                HintFmt("while evaluating attribute '%1%' of derivation '%2%'", key, drvName));
             throw;
         }
     }
@@ -1719,7 +1736,7 @@ static RegisterPrimOp primop_findFile(PrimOp {
       - If the suffix is found inside that directory, then the entry is a match.
         The combined absolute path of the directory (now downloaded if need be) and the suffix is returned.
 
-      [Lookup path](@docroot@/language/constructs/lookup-path.md) expressions can be [desugared](https://en.wikipedia.org/wiki/Syntactic_sugar) using this and [`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtins-nixPath):
+      [Lookup path](@docroot@/language/constructs/lookup-path.md) expressions are [desugared](https://en.wikipedia.org/wiki/Syntactic_sugar) using this and [`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtins-nixPath):
 
       ```nix
       <nixpkgs>
@@ -2075,7 +2092,7 @@ static void prim_toFile(EvalState & state, const PosIdx pos, Value * * args, Val
         })
         : ({
             StringSource s { contents };
-            state.store->addToStoreFromDump(s, name, TextIngestionMethod {}, HashAlgorithm::SHA256, refs, state.repair);
+            state.store->addToStoreFromDump(s, name, FileSerialisationMethod::Flat, TextIngestionMethod {}, HashAlgorithm::SHA256, refs, state.repair);
         });
 
     /* Note: we don't need to add `context' to the context of the
@@ -2507,6 +2524,54 @@ static RegisterPrimOp primop_unsafeGetAttrPos(PrimOp {
     .fun = prim_unsafeGetAttrPos,
 });
 
+// access to exact position information (ie, line and colum numbers) is deferred
+// due to the cost associated with calculating that information and how rarely
+// it is used in practice. this is achieved by creating thunks to otherwise
+// inaccessible primops that are not exposed as __op or under builtins to turn
+// the internal PosIdx back into a line and column number, respectively. exposing
+// these primops in any way would at best be not useful and at worst create wildly
+// indeterministic eval results depending on parse order of files.
+//
+// in a simpler world this would instead be implemented as another kind of thunk,
+// but each type of thunk has an associated runtime cost in the current evaluator.
+// as with black holes this cost is too high to justify another thunk type to check
+// for in the very hot path that is forceValue.
+static struct LazyPosAcessors {
+    PrimOp primop_lineOfPos{
+        .arity = 1,
+        .fun = [] (EvalState & state, PosIdx pos, Value * * args, Value & v) {
+            v.mkInt(state.positions[PosIdx(args[0]->integer)].line);
+        }
+    };
+    PrimOp primop_columnOfPos{
+        .arity = 1,
+        .fun = [] (EvalState & state, PosIdx pos, Value * * args, Value & v) {
+            v.mkInt(state.positions[PosIdx(args[0]->integer)].column);
+        }
+    };
+
+    Value lineOfPos, columnOfPos;
+
+    LazyPosAcessors()
+    {
+        lineOfPos.mkPrimOp(&primop_lineOfPos);
+        columnOfPos.mkPrimOp(&primop_columnOfPos);
+    }
+
+    void operator()(EvalState & state, const PosIdx pos, Value & line, Value & column)
+    {
+        Value * posV = state.allocValue();
+        posV->mkInt(pos.id);
+        line.mkApp(&lineOfPos, posV);
+        column.mkApp(&columnOfPos, posV);
+    }
+} makeLazyPosAccessors;
+
+void makePositionThunks(EvalState & state, const PosIdx pos, Value & line, Value & column)
+{
+    makeLazyPosAccessors(state, pos, line, column);
+}
+
 /* Dynamic version of the `?' operator. */
 static void prim_hasAttr(EvalState & state, const PosIdx pos, Value * * args, Value & v)
 {
@@ -4505,11 +4570,9 @@ void EvalState::createBaseEnv()
     addConstant("__nixPath", v, {
         .type = nList,
         .doc = R"(
-          List of search path entries used to resolve [lookup paths](@docroot@/language/constructs/lookup-path.md).
+          The value of the [`nix-path` configuration setting](@docroot@/command-ref/conf-file.md#conf-nix-path): a list of search path entries used to resolve [lookup paths](@docroot@/language/constructs/lookup-path.md).
 
-          Lookup path expressions can be
-          [desugared](https://en.wikipedia.org/wiki/Syntactic_sugar)
-          using this and
+          Lookup path expressions are [desugared](https://en.wikipedia.org/wiki/Syntactic_sugar) using this and
           [`builtins.findFile`](./builtins.html#builtins-findFile):
 
           ```nix

src/libexpr/primops.hh

@@ -51,4 +51,6 @@ void prim_importNative(EvalState & state, const PosIdx pos, Value * * args, Valu
  */
 void prim_exec(EvalState & state, const PosIdx pos, Value * * args, Value & v);
 
+void makePositionThunks(EvalState & state, const PosIdx pos, Value & line, Value & column);
+
 }

src/libexpr/primops/fetchMercurial.cc

@@ -64,8 +64,7 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value * * a
     if (rev) attrs.insert_or_assign("rev", rev->gitRev());
     auto input = fetchers::Input::fromAttrs(std::move(attrs));
 
-    // FIXME: use name
-    auto [storePath, input2] = input.fetch(state.store);
+    auto [storePath, input2] = input.fetchToStore(state.store);
 
     auto attrs2 = state.buildBindings(8);
     state.mkStorePathString(storePath, attrs2.alloc(state.sOutPath));

src/libexpr/primops/fetchTree.cc

@@ -182,7 +182,7 @@ static void fetchTree(
 
     state.checkURI(input.toURLString());
 
-    auto [storePath, input2] = input.fetch(state.store);
+    auto [storePath, input2] = input.fetchToStore(state.store);
 
     state.allowPath(storePath);
 

src/libfetchers/fetchers.cc

@@ -161,7 +161,7 @@ bool Input::contains(const Input & other) const
     return false;
 }
 
-std::pair<StorePath, Input> Input::fetch(ref<Store> store) const
+std::pair<StorePath, Input> Input::fetchToStore(ref<Store> store) const
 {
     if (!scheme)
         throw Error("cannot fetch unsupported input '%s'", attrsToJSON(toAttrs()));
@@ -186,56 +186,85 @@ std::pair<StorePath, Input> Input::fetch(ref<Store> store) const
 
     auto [storePath, input] = [&]() -> std::pair<StorePath, Input> {
         try {
-            return scheme->fetch(store, *this);
+            auto [accessor, final] = getAccessorUnchecked(store);
+
+            auto storePath = nix::fetchToStore(*store, SourcePath(accessor), FetchMode::Copy, final.getName());
+
+            auto narHash = store->queryPathInfo(storePath)->narHash;
+            final.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true));
+
+            scheme->checkLocks(*this, final);
+
+            return {storePath, final};
         } catch (Error & e) {
             e.addTrace({}, "while fetching the input '%s'", to_string());
             throw;
         }
     }();
 
-    auto narHash = store->queryPathInfo(storePath)->narHash;
-    input.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true));
-
-    if (auto prevNarHash = getNarHash()) {
-        if (narHash != *prevNarHash)
-            throw Error((unsigned int) 102, "NAR hash mismatch in input '%s' (%s), expected '%s', got '%s'",
-                to_string(),
-                store->printStorePath(storePath),
-                prevNarHash->to_string(HashFormat::SRI, true),
-                narHash.to_string(HashFormat::SRI, true));
-    }
-
-    if (auto prevLastModified = getLastModified()) {
-        if (input.getLastModified() != prevLastModified)
-            throw Error("'lastModified' attribute mismatch in input '%s', expected %d",
-                input.to_string(), *prevLastModified);
-    }
-
-    if (auto prevRev = getRev()) {
-        if (input.getRev() != prevRev)
-            throw Error("'rev' attribute mismatch in input '%s', expected %s",
-                input.to_string(), prevRev->gitRev());
-    }
-
-    if (auto prevRevCount = getRevCount()) {
-        if (input.getRevCount() != prevRevCount)
-            throw Error("'revCount' attribute mismatch in input '%s', expected %d",
-                input.to_string(), *prevRevCount);
-    }
-
     return {std::move(storePath), input};
 }
 
+void InputScheme::checkLocks(const Input & specified, const Input & final) const
+{
+    if (auto prevNarHash = specified.getNarHash()) {
+        if (final.getNarHash() != prevNarHash) {
+            if (final.getNarHash())
+                throw Error((unsigned int) 102, "NAR hash mismatch in input '%s', expected '%s' but got '%s'",
+                    specified.to_string(), prevNarHash->to_string(HashFormat::SRI, true), final.getNarHash()->to_string(HashFormat::SRI, true));
+            else
+                throw Error((unsigned int) 102, "NAR hash mismatch in input '%s', expected '%s' but got none",
+                    specified.to_string(), prevNarHash->to_string(HashFormat::SRI, true));
+        }
+    }
+
+    if (auto prevLastModified = specified.getLastModified()) {
+        if (final.getLastModified() != prevLastModified)
+            throw Error("'lastModified' attribute mismatch in input '%s', expected %d",
+                final.to_string(), *prevLastModified);
+    }
+
+    if (auto prevRev = specified.getRev()) {
+        if (final.getRev() != prevRev)
+            throw Error("'rev' attribute mismatch in input '%s', expected %s",
+                final.to_string(), prevRev->gitRev());
+    }
+
+    if (auto prevRevCount = specified.getRevCount()) {
+        if (final.getRevCount() != prevRevCount)
+            throw Error("'revCount' attribute mismatch in input '%s', expected %d",
+                final.to_string(), *prevRevCount);
+    }
+}
+
 std::pair<ref<InputAccessor>, Input> Input::getAccessor(ref<Store> store) const
 {
     try {
-        return scheme->getAccessor(store, *this);
+        auto [accessor, final] = getAccessorUnchecked(store);
+
+        scheme->checkLocks(*this, final);
+
+        return {accessor, std::move(final)};
     } catch (Error & e) {
         e.addTrace({}, "while fetching the input '%s'", to_string());
         throw;
     }
 }
 
+std::pair<ref<InputAccessor>, Input> Input::getAccessorUnchecked(ref<Store> store) const
+{
+    // FIXME: cache the accessor
+
+    if (!scheme)
+        throw Error("cannot fetch unsupported input '%s'", attrsToJSON(toAttrs()));
+
+    auto [accessor, final] = scheme->getAccessor(store, *this);
+
+    accessor->fingerprint = scheme->getFingerprint(store, final);
+
+    return {accessor, std::move(final)};
+}
+
 Input Input::applyOverrides(
     std::optional<std::string> ref,
     std::optional<Hash> rev) const
@@ -372,18 +401,6 @@ void InputScheme::clone(const Input & input, const Path & destDir) const
     throw Error("do not know how to clone input '%s'", input.to_string());
 }
 
-std::pair<StorePath, Input> InputScheme::fetch(ref<Store> store, const Input & input)
-{
-    auto [accessor, input2] = getAccessor(store, input);
-    auto storePath = fetchToStore(*store, SourcePath(accessor), FetchMode::Copy, input2.getName());
-    return {storePath, input2};
-}
-
-std::pair<ref<InputAccessor>, Input> InputScheme::getAccessor(ref<Store> store, const Input & input) const
-{
-    throw UnimplementedError("InputScheme must implement fetch() or getAccessor()");
-}
-
 std::optional<ExperimentalFeature> InputScheme::experimentalFeature() const
 {
     return {};

src/libfetchers/fetchers.hh

@@ -80,10 +80,21 @@ public:
      * Fetch the entire input into the Nix store, returning the
      * location in the Nix store and the locked input.
      */
-    std::pair<StorePath, Input> fetch(ref<Store> store) const;
+    std::pair<StorePath, Input> fetchToStore(ref<Store> store) const;
 
+    /**
+     * Return an InputAccessor that allows access to files in the
+     * input without copying it to the store. Also return a possibly
+     * unlocked input.
+     */
     std::pair<ref<InputAccessor>, Input> getAccessor(ref<Store> store) const;
 
+private:
+
+    std::pair<ref<InputAccessor>, Input> getAccessorUnchecked(ref<Store> store) const;
+
+public:
+
     Input applyOverrides(
         std::optional<std::string> ref,
         std::optional<Hash> rev) const;
@@ -173,9 +184,7 @@ struct InputScheme
         std::string_view contents,
         std::optional<std::string> commitMsg) const;
 
-    virtual std::pair<StorePath, Input> fetch(ref<Store> store, const Input & input);
-
-    virtual std::pair<ref<InputAccessor>, Input> getAccessor(ref<Store> store, const Input & input) const;
+    virtual std::pair<ref<InputAccessor>, Input> getAccessor(ref<Store> store, const Input & input) const = 0;
 
     /**
      * Is this `InputScheme` part of an experimental feature?
@@ -202,6 +211,14 @@ struct InputScheme
      */
     virtual bool isLocked(const Input & input) const
     { return false; }
+
+    /**
+     * Check the locking attributes in `final` against
+     * `specified`. E.g. if `specified` has a `rev` attribute, then
+     * `final` must have the same `rev` attribute. Throw an exception
+     * if there is a mismatch.
+     */
+    virtual void checkLocks(const Input & specified, const Input & final) const;
 };
 
 void registerInputScheme(std::shared_ptr<InputScheme> && fetcher);

src/libfetchers/git.cc

@@ -761,8 +761,6 @@ struct GitInputScheme : InputScheme
             ? getAccessorFromCommit(store, repoInfo, std::move(input))
             : getAccessorFromWorkdir(store, repoInfo, std::move(input));
 
-        accessor->fingerprint = final.getFingerprint(store);
-
         return {accessor, std::move(final)};
     }
 

src/libfetchers/github.cc

@@ -98,6 +98,10 @@ struct GitArchiveInputScheme : InputScheme
         if (ref) input.attrs.insert_or_assign("ref", *ref);
         if (host_url) input.attrs.insert_or_assign("host", *host_url);
 
+        auto narHash = url.query.find("narHash");
+        if (narHash != url.query.end())
+            input.attrs.insert_or_assign("narHash", narHash->second);
+
         return input;
     }
 
@@ -111,6 +115,7 @@ struct GitArchiveInputScheme : InputScheme
             "narHash",
             "lastModified",
             "host",
+            "treeHash",
         };
     }
 
@@ -134,10 +139,13 @@ struct GitArchiveInputScheme : InputScheme
         assert(!(ref && rev));
         if (ref) path += "/" + *ref;
         if (rev) path += "/" + rev->to_string(HashFormat::Base16, false);
-        return ParsedURL {
+        auto url = ParsedURL {
             .scheme = std::string { schemeName() },
             .path = path,
         };
+        if (auto narHash = input.getNarHash())
+            url.query.insert_or_assign("narHash", narHash->to_string(HashFormat::SRI, true));
+        return url;
     }
 
     Input applyOverrides(
@@ -268,15 +276,15 @@ struct GitArchiveInputScheme : InputScheme
     {
         auto [input, tarballInfo] = downloadArchive(store, _input);
 
+        #if 0
         input.attrs.insert_or_assign("treeHash", tarballInfo.treeHash.gitRev());
+        #endif
         input.attrs.insert_or_assign("lastModified", uint64_t(tarballInfo.lastModified));
 
         auto accessor = getTarballCache()->getAccessor(tarballInfo.treeHash, false);
 
         accessor->setPathDisplay("«" + input.to_string() + "»");
 
-        accessor->fingerprint = input.getFingerprint(store);
-
         return {accessor, input};
     }
 

src/libfetchers/indirect.cc

@@ -97,7 +97,7 @@ struct IndirectInputScheme : InputScheme
         return input;
     }
 
-    std::pair<StorePath, Input> fetch(ref<Store> store, const Input & input) override
+    std::pair<ref<InputAccessor>, Input> getAccessor(ref<Store> store, const Input & input) const override
     {
         throw Error("indirect input '%s' cannot be fetched directly", input.to_string());
     }

src/libfetchers/mercurial.cc

@@ -6,8 +6,8 @@
 #include "tarfile.hh"
 #include "store-api.hh"
 #include "url-parts.hh"
+#include "fs-input-accessor.hh"
 #include "posix-source-accessor.hh"
-
 #include "fetch-settings.hh"
 
 #include <sys/time.h>
@@ -161,9 +161,9 @@ struct MercurialInputScheme : InputScheme
         return {isLocal, isLocal ? url.path : url.base};
     }
 
-    std::pair<StorePath, Input> fetch(ref<Store> store, const Input & _input) override
+    StorePath fetchToStore(ref<Store> store, Input & input) const
     {
-        Input input(_input);
+        auto origRev = input.getRev();
 
         auto name = input.getName();
 
@@ -218,7 +218,7 @@ struct MercurialInputScheme : InputScheme
                     FileIngestionMethod::Recursive, HashAlgorithm::SHA256, {},
                     filter);
 
-                return {std::move(storePath), input};
+                return storePath;
             }
         }
 
@@ -242,13 +242,12 @@ struct MercurialInputScheme : InputScheme
             });
         };
 
-        auto makeResult = [&](const Attrs & infoAttrs, StorePath && storePath)
-            -> std::pair<StorePath, Input>
+        auto makeResult = [&](const Attrs & infoAttrs, const StorePath & storePath) -> StorePath
         {
             assert(input.getRev());
-            assert(!_input.getRev() || _input.getRev() == input.getRev());
+            assert(!origRev || origRev == input.getRev());
             input.attrs.insert_or_assign("revCount", getIntAttr(infoAttrs, "revCount"));
-            return {std::move(storePath), input};
+            return storePath;
         };
 
         if (input.getRev()) {
@@ -329,7 +328,7 @@ struct MercurialInputScheme : InputScheme
             {"revCount", (uint64_t) revCount},
         });
 
-        if (!_input.getRev())
+        if (!origRev)
             getCache()->add(
                 *store,
                 unlockedAttrs,
@@ -347,6 +346,15 @@ struct MercurialInputScheme : InputScheme
         return makeResult(infoAttrs, std::move(storePath));
     }
 
+    std::pair<ref<InputAccessor>, Input> getAccessor(ref<Store> store, const Input & _input) const override
+    {
+        Input input(_input);
+
+        auto storePath = fetchToStore(store, input);
+
+        return {makeStorePathAccessor(store, storePath), input};
+    }
+
     bool isLocked(const Input & input) const override
     {
         return (bool) input.getRev();

src/libfetchers/path.cc

@@ -1,6 +1,8 @@
 #include "fetchers.hh"
 #include "store-api.hh"
 #include "archive.hh"
+#include "fs-input-accessor.hh"
+#include "posix-source-accessor.hh"
 
 namespace nix::fetchers {
 
@@ -87,6 +89,15 @@ struct PathInputScheme : InputScheme
         writeFile((CanonPath(getAbsPath(input)) / path).abs(), contents);
     }
 
+    std::optional<std::string> isRelative(const Input & input) const
+    {
+        auto path = getStrAttr(input.attrs, "path");
+        if (hasPrefix(path, "/"))
+            return std::nullopt;
+        else
+            return path;
+    }
+
     bool isLocked(const Input & input) const override
     {
         return (bool) input.getNarHash();
@@ -102,7 +113,7 @@ struct PathInputScheme : InputScheme
         throw Error("cannot fetch input '%s' because it uses a relative path", input.to_string());
     }
 
-    std::pair<StorePath, Input> fetch(ref<Store> store, const Input & _input) override
+    std::pair<ref<InputAccessor>, Input> getAccessor(ref<Store> store, const Input & _input) const override
     {
         Input input(_input);
         std::string absPath;
@@ -144,7 +155,24 @@ struct PathInputScheme : InputScheme
         }
         input.attrs.insert_or_assign("lastModified", uint64_t(mtime));
 
-        return {std::move(*storePath), input};
+        return {makeStorePathAccessor(store, *storePath), std::move(input)};
+    }
+
+    std::optional<std::string> getFingerprint(ref<Store> store, const Input & input) const override
+    {
+        if (isRelative(input))
+            return std::nullopt;
+
+        /* If this path is in the Nix store, use the hash of the
+           store object and the subpath. */
+        auto path = getAbsPath(input);
+        try {
+            auto [storePath, subPath] = store->toStorePath(path.abs());
+            auto info = store->queryPathInfo(storePath);
+            return fmt("path:%s:%s", info->narHash.to_string(HashFormat::Base16, false), subPath);
+        } catch (Error &) {
+            return std::nullopt;
+        }
     }
 
     std::optional<ExperimentalFeature> experimentalFeature() const override

src/libstore/binary-cache-store.cc

@@ -305,7 +305,8 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
 StorePath BinaryCacheStore::addToStoreFromDump(
     Source & dump,
     std::string_view name,
-    ContentAddressMethod method,
+    FileSerialisationMethod dumpMethod,
+    ContentAddressMethod hashMethod,
     HashAlgorithm hashAlgo,
     const StorePathSet & references,
     RepairFlag repair)
@@ -313,17 +314,27 @@ StorePath BinaryCacheStore::addToStoreFromDump(
     std::optional<Hash> caHash;
     std::string nar;
 
+    // Calculating Git hash from NAR stream not yet implemented. May not
+    // be possible to implement in single-pass if the NAR is in an
+    // inconvenient order. Could fetch after uploading, however.
+    if (hashMethod.getFileIngestionMethod() == FileIngestionMethod::Git)
+        unsupported("addToStoreFromDump");
+
     if (auto * dump2p = dynamic_cast<StringSource *>(&dump)) {
         auto & dump2 = *dump2p;
         // Hack, this gives us a "replayable" source so we can compute
         // multiple hashes more easily.
-        caHash = hashString(HashAlgorithm::SHA256, dump2.s);
-        switch (method.getFileIngestionMethod()) {
-        case FileIngestionMethod::Recursive:
+        //
+        // Only calculate if the dump is in the right format, however.
+        if (static_cast<FileIngestionMethod>(dumpMethod) == hashMethod.getFileIngestionMethod())
+            caHash = hashString(HashAlgorithm::SHA256, dump2.s);
+        switch (dumpMethod) {
+        case FileSerialisationMethod::Recursive:
             // The dump is already NAR in this case, just use it.
             nar = dump2.s;
             break;
-        case FileIngestionMethod::Flat:
+        case FileSerialisationMethod::Flat:
+        {
             // The dump is Flat, so we need to convert it to NAR with a
             // single file.
             StringSink s;
@@ -331,10 +342,11 @@ StorePath BinaryCacheStore::addToStoreFromDump(
             nar = std::move(s.s);
             break;
         }
+        }
     } else {
         // Otherwise, we have to do th same hashing as NAR so our single
         // hash will suffice for both purposes.
-        if (method != FileIngestionMethod::Recursive || hashAlgo != HashAlgorithm::SHA256)
+        if (dumpMethod != FileSerialisationMethod::Recursive || hashAlgo != HashAlgorithm::SHA256)
             unsupported("addToStoreFromDump");
     }
     StringSource narDump { nar };
@@ -349,7 +361,7 @@ StorePath BinaryCacheStore::addToStoreFromDump(
             *this,
             name,
             ContentAddressWithReferences::fromParts(
-                method,
+                hashMethod,
                 caHash ? *caHash : nar.first,
                 {
                     .others = references,
@@ -450,7 +462,7 @@ StorePath BinaryCacheStore::addToStore(
        non-recursive+sha256 so we can just use the default
        implementation of this method in terms of addToStoreFromDump. */
 
-    auto h = hashPath(accessor, path, method.getFileIngestionMethod(), hashAlgo, filter).first;
+    auto h = hashPath(accessor, path, method.getFileIngestionMethod(), hashAlgo, filter);
 
     auto source = sinkToSource([&](Sink & sink) {
         accessor.dumpPath(path, sink, filter);

src/libstore/binary-cache-store.hh

@@ -125,7 +125,8 @@ public:
     StorePath addToStoreFromDump(
         Source & dump,
         std::string_view name,
-        ContentAddressMethod method,
+        FileSerialisationMethod dumpMethod,
+        ContentAddressMethod hashMethod,
         HashAlgorithm hashAlgo,
         const StorePathSet & references,
         RepairFlag repair) override;
@@ -147,7 +148,7 @@ public:
 
     void narFromPath(const StorePath & path, Sink & sink) override;
 
-    ref<SourceAccessor> getFSAccessor(bool requireValidPath) override;
+    ref<SourceAccessor> getFSAccessor(bool requireValidPath = true) override;
 
     void addSignatures(const StorePath & storePath, const StringSet & sigs) override;
 

src/libstore/build-result.hh

@@ -123,6 +123,11 @@ struct KeyedBuildResult : BuildResult
      * The derivation we built or the store path we substituted.
      */
     DerivedPath path;
+
+    // Hack to work around a gcc "may be used uninitialized" warning.
+    KeyedBuildResult(BuildResult res, DerivedPath path)
+        : BuildResult(std::move(res)), path(std::move(path))
+    { }
 };
 
 }

src/libstore/build/local-derivation-goal.cc

@@ -8,6 +8,7 @@
 #include "finally.hh"
 #include "util.hh"
 #include "archive.hh"
+#include "git.hh"
 #include "compression.hh"
 #include "daemon.hh"
 #include "topo-sort.hh"
@@ -1311,12 +1312,13 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual In
     StorePath addToStoreFromDump(
         Source & dump,
         std::string_view name,
-        ContentAddressMethod method,
+        FileSerialisationMethod dumpMethod,
+        ContentAddressMethod hashMethod,
         HashAlgorithm hashAlgo,
         const StorePathSet & references,
         RepairFlag repair) override
     {
-        auto path = next->addToStoreFromDump(dump, name, method, hashAlgo, references, repair);
+        auto path = next->addToStoreFromDump(dump, name, dumpMethod, hashMethod, hashAlgo, references, repair);
         goal.addDependency(path);
         return path;
     }
@@ -2457,15 +2459,29 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
             rewriteOutput(outputRewrites);
             /* FIXME optimize and deduplicate with addToStore */
             std::string oldHashPart { scratchPath->hashPart() };
-            auto got = ({
-                HashModuloSink caSink { outputHash.hashAlgo, oldHashPart };
+            auto got = [&]{
                 PosixSourceAccessor accessor;
-                dumpPath(
-                    accessor, CanonPath { actualPath },
-                    caSink,
-                    outputHash.method.getFileIngestionMethod());
-                caSink.finish().first;
-            });
+                auto fim = outputHash.method.getFileIngestionMethod();
+                switch (fim) {
+                case FileIngestionMethod::Flat:
+                case FileIngestionMethod::Recursive:
+                {
+                    HashModuloSink caSink { outputHash.hashAlgo, oldHashPart };
+                    auto fim = outputHash.method.getFileIngestionMethod();
+                    dumpPath(
+                        accessor, CanonPath { actualPath },
+                        caSink,
+                        (FileSerialisationMethod) fim);
+                    return caSink.finish().first;
+                }
+                case FileIngestionMethod::Git: {
+                    return git::dumpHash(
+                        outputHash.hashAlgo, accessor,
+                        CanonPath { tmpDir + "/tmp" }).hash;
+                }
+                }
+                assert(false);
+            }();
 
             ValidPathInfo newInfo0 {
                 worker.store,
@@ -2491,7 +2507,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
                 PosixSourceAccessor accessor;
                 HashResult narHashAndSize = hashPath(
                     accessor, CanonPath { actualPath },
-                    FileIngestionMethod::Recursive, HashAlgorithm::SHA256);
+                    FileSerialisationMethod::Recursive, HashAlgorithm::SHA256);
                 newInfo0.narHash = narHashAndSize.first;
                 newInfo0.narSize = narHashAndSize.second;
             }
@@ -2515,7 +2531,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
                 PosixSourceAccessor accessor;
                 HashResult narHashAndSize = hashPath(
                     accessor, CanonPath { actualPath },
-                    FileIngestionMethod::Recursive, HashAlgorithm::SHA256);
+                    FileSerialisationMethod::Recursive, HashAlgorithm::SHA256);
                 ValidPathInfo newInfo0 { requiredFinalPath, narHashAndSize.first };
                 newInfo0.narSize = narHashAndSize.second;
                 auto refs = rewriteRefs();
@@ -2528,6 +2544,12 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
             [&](const DerivationOutput::CAFixed & dof) {
                 auto & wanted = dof.ca.hash;
 
+                // Replace the output by a fresh copy of itself to make sure
+                // that there's no stale file descriptor pointing to it
+                Path tmpOutput = actualPath + ".tmp";
+                copyFile(actualPath, tmpOutput, true);
+                renameFile(tmpOutput, actualPath);
+
                 auto newInfo0 = newInfoFromCA(DerivationOutput::CAFloating {
                     .method = dof.ca.method,
                     .hashAlgo = wanted.algo,

src/libstore/build/sandbox-defaults.sb

@@ -45,7 +45,7 @@ R""(
 ; allow it if the package explicitly asks for it.
 (if (param "_ALLOW_LOCAL_NETWORKING")
     (begin
-      (allow network* (local ip) (local tcp) (local udp))
+      (allow network* (remote ip "localhost:*"))
 
       ; Allow access to /etc/resolv.conf (which is a symlink to
       ; /private/var/run/resolv.conf).

src/libstore/build/worker.cc

@@ -529,11 +529,11 @@ bool Worker::pathContentsGood(const StorePath & path)
     if (!pathExists(store.printStorePath(path)))
         res = false;
     else {
-        HashResult current = hashPath(
+        Hash current = hashPath(
             *store.getFSAccessor(), CanonPath { store.printStorePath(path) },
             FileIngestionMethod::Recursive, info->narHash.algo);
         Hash nullHash(HashAlgorithm::SHA256);
-        res = info->narHash == nullHash || info->narHash == current.first;
+        res = info->narHash == nullHash || info->narHash == current;
     }
     pathContentsGoodCache.insert_or_assign(path, res);
     if (!res)

src/libstore/content-address.cc

@@ -11,6 +11,9 @@ std::string_view makeFileIngestionPrefix(FileIngestionMethod m)
         return "";
     case FileIngestionMethod::Recursive:
         return "r:";
+    case FileIngestionMethod::Git:
+        experimentalFeatureSettings.require(Xp::GitHashing);
+        return "git:";
     default:
         throw Error("impossible, caught both cases");
     }
@@ -51,6 +54,10 @@ ContentAddressMethod ContentAddressMethod::parsePrefix(std::string_view & m)
     if (splitPrefix(m, "r:")) {
         return FileIngestionMethod::Recursive;
     }
+    else if (splitPrefix(m, "git:")) {
+        experimentalFeatureSettings.require(Xp::GitHashing);
+        return FileIngestionMethod::Git;
+    }
     else if (splitPrefix(m, "text:")) {
         return TextIngestionMethod {};
     }
@@ -111,10 +118,10 @@ static std::pair<ContentAddressMethod, HashAlgorithm> parseContentAddressMethodP
     }
 
     auto parseHashAlgorithm_ = [&](){
-        auto hashTypeRaw = splitPrefixTo(rest, ':');
-        if (!hashTypeRaw)
+        auto hashAlgoRaw = splitPrefixTo(rest, ':');
+        if (!hashAlgoRaw)
             throw UsageError("content address hash must be in form '<algo>:<hash>', but found: %s", wholeInput);
-        HashAlgorithm hashAlgo = parseHashAlgo(*hashTypeRaw);
+        HashAlgorithm hashAlgo = parseHashAlgo(*hashAlgoRaw);
         return hashAlgo;
     };
 
@@ -131,6 +138,10 @@ static std::pair<ContentAddressMethod, HashAlgorithm> parseContentAddressMethodP
         auto method = FileIngestionMethod::Flat;
         if (splitPrefix(rest, "r:"))
             method = FileIngestionMethod::Recursive;
+        else if (splitPrefix(rest, "git:")) {
+            experimentalFeatureSettings.require(Xp::GitHashing);
+            method = FileIngestionMethod::Git;
+        }
         HashAlgorithm hashAlgo = parseHashAlgorithm_();
         return {
             std::move(method),

src/libstore/content-address.hh

@@ -91,17 +91,17 @@ struct ContentAddressMethod
     std::string_view renderPrefix() const;
 
     /**
-     * Parse a content addressing method and hash type.
+     * Parse a content addressing method and hash algorithm.
      */
     static std::pair<ContentAddressMethod, HashAlgorithm> parseWithAlgo(std::string_view rawCaMethod);
 
     /**
-     * Render a content addressing method and hash type in a
+     * Render a content addressing method and hash algorithm in a
      * nicer way, prefixing both cases.
      *
      * The rough inverse of `parse()`.
      */
-    std::string renderWithAlgo(HashAlgorithm ht) const;
+    std::string renderWithAlgo(HashAlgorithm ha) const;
 
     /**
      * Get the underlying way to content-address file system objects.
@@ -127,7 +127,7 @@ struct ContentAddressMethod
  *   ‘text:sha256:<sha256 hash of file contents>’
  *
  * - `FixedIngestionMethod`:
- *   ‘fixed:<r?>:<hash type>:<hash of file contents>’
+ *   ‘fixed:<r?>:<hash algorithm>:<hash of file contents>’
  */
 struct ContentAddress
 {

src/libstore/daemon.cc

@@ -13,6 +13,7 @@
 #include "archive.hh"
 #include "derivations.hh"
 #include "args.hh"
+#include "git.hh"
 
 namespace nix::daemon {
 
@@ -400,11 +401,25 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
             logger->startWork();
             auto pathInfo = [&]() {
                 // NB: FramedSource must be out of scope before logger->stopWork();
-                auto [contentAddressMethod, hashAlgo_] = ContentAddressMethod::parseWithAlgo(camStr);
-                auto hashAlgo = hashAlgo_; // work around clang bug
+                auto [contentAddressMethod, hashAlgo] = ContentAddressMethod::parseWithAlgo(camStr);
                 FramedSource source(from);
+                FileSerialisationMethod dumpMethod;
+                switch (contentAddressMethod.getFileIngestionMethod()) {
+                case FileIngestionMethod::Flat:
+                    dumpMethod = FileSerialisationMethod::Flat;
+                    break;
+                case FileIngestionMethod::Recursive:
+                    dumpMethod = FileSerialisationMethod::Recursive;
+                    break;
+                case FileIngestionMethod::Git:
+                    // Use NAR; Git is not a serialization method
+                    dumpMethod = FileSerialisationMethod::Recursive;
+                    break;
+                default:
+                    assert(false);
+                }
                 // TODO these two steps are essentially RemoteStore::addCAToStore. Move it up to Store.
-                auto path = store->addToStoreFromDump(source, name, contentAddressMethod, hashAlgo, refs, repair);
+                auto path = store->addToStoreFromDump(source, name, dumpMethod, contentAddressMethod, hashAlgo, refs, repair);
                 return store->queryPathInfo(path);
             }();
             logger->stopWork();
@@ -430,30 +445,23 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
                 hashAlgo = parseHashAlgo(hashAlgoRaw);
             }
 
+            // Old protocol always sends NAR, regardless of hashing method
             auto dumpSource = sinkToSource([&](Sink & saved) {
-                if (method == FileIngestionMethod::Recursive) {
-                    /* We parse the NAR dump through into `saved` unmodified,
-                       so why all this extra work? We still parse the NAR so
-                       that we aren't sending arbitrary data to `saved`
-                       unwittingly`, and we know when the NAR ends so we don't
-                       consume the rest of `from` and can't parse another
-                       command. (We don't trust `addToStoreFromDump` to not
-                       eagerly consume the entire stream it's given, past the
-                       length of the Nar. */
-                    TeeSource savedNARSource(from, saved);
-                    NullFileSystemObjectSink sink; /* just parse the NAR */
-                    parseDump(sink, savedNARSource);
-                } else {
-                    /* Incrementally parse the NAR file, stripping the
-                       metadata, and streaming the sole file we expect into
-                       `saved`. */
-                    RegularFileSink savedRegular { saved };
-                    parseDump(savedRegular, from);
-                    if (!savedRegular.regular) throw Error("regular file expected");
-                }
+                /* We parse the NAR dump through into `saved` unmodified,
+                   so why all this extra work? We still parse the NAR so
+                   that we aren't sending arbitrary data to `saved`
+                   unwittingly`, and we know when the NAR ends so we don't
+                   consume the rest of `from` and can't parse another
+                   command. (We don't trust `addToStoreFromDump` to not
+                   eagerly consume the entire stream it's given, past the
+                   length of the Nar. */
+                TeeSource savedNARSource(from, saved);
+                NullFileSystemObjectSink sink; /* just parse the NAR */
+                parseDump(sink, savedNARSource);
             });
             logger->startWork();
-            auto path = store->addToStoreFromDump(*dumpSource, baseName, method, hashAlgo);
+            auto path = store->addToStoreFromDump(
+                *dumpSource, baseName, FileSerialisationMethod::Recursive, method, hashAlgo);
             logger->stopWork();
 
             to << store->printStorePath(path);
@@ -485,7 +493,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         logger->startWork();
         auto path = ({
             StringSource source { s };
-            store->addToStoreFromDump(source, suffix, TextIngestionMethod {}, HashAlgorithm::SHA256, refs, NoRepair);
+            store->addToStoreFromDump(source, suffix, FileSerialisationMethod::Flat, TextIngestionMethod {}, HashAlgorithm::SHA256, refs, NoRepair);
         });
         logger->stopWork();
         to << store->printStorePath(path);

src/libstore/derivations.cc

@@ -150,7 +150,7 @@ StorePath writeDerivation(Store & store,
         })
         : ({
             StringSource s { contents };
-            store.addToStoreFromDump(s, suffix, TextIngestionMethod {}, HashAlgorithm::SHA256, references, repair);
+            store.addToStoreFromDump(s, suffix, FileSerialisationMethod::Flat, TextIngestionMethod {}, HashAlgorithm::SHA256, references, repair);
         });
 }
 
@@ -701,7 +701,7 @@ DerivationType BasicDerivation::type() const
                     floatingHashAlgo = dof.hashAlgo;
                 } else {
                     if (*floatingHashAlgo != dof.hashAlgo)
-                        throw Error("all floating outputs must use the same hash type");
+                        throw Error("all floating outputs must use the same hash algorithm");
                 }
             },
             [&](const DerivationOutput::Deferred &) {

src/libstore/dummy-store.cc

@@ -61,7 +61,8 @@ struct DummyStore : public virtual DummyStoreConfig, public virtual Store
     virtual StorePath addToStoreFromDump(
         Source & dump,
         std::string_view name,
-        ContentAddressMethod method = FileIngestionMethod::Recursive,
+        FileSerialisationMethod dumpMethod = FileSerialisationMethod::Recursive,
+        ContentAddressMethod hashMethod = FileIngestionMethod::Recursive,
         HashAlgorithm hashAlgo = HashAlgorithm::SHA256,
         const StorePathSet & references = StorePathSet(),
         RepairFlag repair = NoRepair) override

src/libstore/globals.hh

@@ -1094,8 +1094,8 @@ public:
         this, {}, "hashed-mirrors",
         R"(
           A list of web servers used by `builtins.fetchurl` to obtain files by
-          hash. Given a hash type *ht* and a base-16 hash *h*, Nix will try to
-          download the file from *hashed-mirror*/*ht*/*h*. This allows files to
+          hash. Given a hash algorithm *ha* and a base-16 hash *h*, Nix will try to
+          download the file from *hashed-mirror*/*ha*/*h*. This allows files to
           be downloaded even if they have disappeared from their original URI.
           For example, given an example mirror `http://tarballs.nixos.org/`,
           when building the derivation

src/libstore/legacy-ssh-store.hh

@@ -72,7 +72,8 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
     virtual StorePath addToStoreFromDump(
         Source & dump,
         std::string_view name,
-        ContentAddressMethod method = FileIngestionMethod::Recursive,
+        FileSerialisationMethod dumpMethod = FileSerialisationMethod::Recursive,
+        ContentAddressMethod hashMethod = FileIngestionMethod::Recursive,
         HashAlgorithm hashAlgo = HashAlgorithm::SHA256,
         const StorePathSet & references = StorePathSet(),
         RepairFlag repair = NoRepair) override

src/libstore/local-fs-store.hh

@@ -43,7 +43,7 @@ public:
     LocalFSStore(const Params & params);
 
     void narFromPath(const StorePath & path, Sink & sink) override;
-    ref<SourceAccessor> getFSAccessor(bool requireValidPath) override;
+    ref<SourceAccessor> getFSAccessor(bool requireValidPath = true) override;
 
     /**
      * Creates symlink from the `gcRoot` to the `storePath` and

src/libstore/local-store.cc

@@ -1,5 +1,6 @@
 #include "local-store.hh"
 #include "globals.hh"
+#include "git.hh"
 #include "archive.hh"
 #include "pathlocks.hh"
 #include "worker-protocol.hh"
@@ -1097,19 +1098,29 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
             if (info.ca) {
                 auto & specified = *info.ca;
                 auto actualHash = ({
-                    HashModuloSink caSink {
-                        specified.hash.algo,
-                        std::string { info.path.hashPart() },
-                    };
-                    PosixSourceAccessor accessor;
-                    dumpPath(
-                        *getFSAccessor(false),
-                        CanonPath { printStorePath(info.path) },
-                        caSink,
-                        specified.method.getFileIngestionMethod());
+                    auto accessor = getFSAccessor(false);
+                    CanonPath path { printStorePath(info.path) };
+                    Hash h { HashAlgorithm::SHA256 }; // throwaway def to appease C++
+                    auto fim = specified.method.getFileIngestionMethod();
+                    switch (fim) {
+                    case FileIngestionMethod::Flat:
+                    case FileIngestionMethod::Recursive:
+                    {
+                        HashModuloSink caSink {
+                            specified.hash.algo,
+                            std::string { info.path.hashPart() },
+                        };
+                        dumpPath(*accessor, path, caSink, (FileSerialisationMethod) fim);
+                        h = caSink.finish().first;
+                        break;
+                    }
+                    case FileIngestionMethod::Git:
+                        h = git::dumpHash(specified.hash.algo, *accessor, path).hash;
+                        break;
+                    }
                     ContentAddress {
                         .method = specified.method,
-                        .hash = caSink.finish().first,
+                        .hash = std::move(h),
                     };
                 });
                 if (specified.hash != actualHash.hash) {
@@ -1137,7 +1148,8 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
 StorePath LocalStore::addToStoreFromDump(
     Source & source0,
     std::string_view name,
-    ContentAddressMethod method,
+    FileSerialisationMethod dumpMethod,
+    ContentAddressMethod hashMethod,
     HashAlgorithm hashAlgo,
     const StorePathSet & references,
     RepairFlag repair)
@@ -1190,7 +1202,13 @@ StorePath LocalStore::addToStoreFromDump(
     Path tempDir;
     AutoCloseFD tempDirFd;
 
-    if (!inMemory) {
+    bool methodsMatch = ContentAddressMethod(FileIngestionMethod(dumpMethod)) == hashMethod;
+
+    /* If the methods don't match, our streaming hash of the dump is the
+       wrong sort, and we need to rehash. */
+    bool inMemoryAndDontNeedRestore = inMemory && methodsMatch;
+
+    if (!inMemoryAndDontNeedRestore) {
         /* Drain what we pulled so far, and then keep on pulling */
         StringSource dumpSource { dump };
         ChainSource bothSource { dumpSource, source };
@@ -1199,17 +1217,23 @@ StorePath LocalStore::addToStoreFromDump(
         delTempDir = std::make_unique<AutoDelete>(tempDir);
         tempPath = tempDir + "/x";
 
-        restorePath(tempPath, bothSource, method.getFileIngestionMethod());
+        restorePath(tempPath, bothSource, dumpMethod);
 
         dumpBuffer.reset();
         dump = {};
     }
 
-    auto [hash, size] = hashSink->finish();
+    auto [dumpHash, size] = hashSink->finish();
+
+    PosixSourceAccessor accessor;
 
     auto desc = ContentAddressWithReferences::fromParts(
-        method,
-        hash,
+        hashMethod,
+        methodsMatch
+            ? dumpHash
+            : hashPath(
+                accessor, CanonPath { tempPath },
+                hashMethod.getFileIngestionMethod(), hashAlgo),
         {
             .others = references,
             // caller is not capable of creating a self-reference, because this is content-addressed without modulus
@@ -1235,10 +1259,20 @@ StorePath LocalStore::addToStoreFromDump(
 
             autoGC();
 
-            if (inMemory) {
+            if (inMemoryAndDontNeedRestore) {
                 StringSource dumpSource { dump };
                 /* Restore from the buffer in memory. */
-                restorePath(realPath, dumpSource, method.getFileIngestionMethod());
+                auto fim = hashMethod.getFileIngestionMethod();
+                switch (fim) {
+                case FileIngestionMethod::Flat:
+                case FileIngestionMethod::Recursive:
+                    restorePath(realPath, dumpSource, (FileSerialisationMethod) fim);
+                    break;
+                case FileIngestionMethod::Git:
+                    // doesn't correspond to serialization method, so
+                    // this should be unreachable
+                    assert(false);
+                }
             } else {
                 /* Move the temporary path we restored above. */
                 moveFile(tempPath, realPath);
@@ -1246,8 +1280,8 @@ StorePath LocalStore::addToStoreFromDump(
 
             /* For computing the nar hash. In recursive SHA-256 mode, this
                is the same as the store hash, so no need to do it again. */
-            auto narHash = std::pair { hash, size };
-            if (method != FileIngestionMethod::Recursive || hashAlgo != HashAlgorithm::SHA256) {
+            auto narHash = std::pair { dumpHash, size };
+            if (dumpMethod != FileSerialisationMethod::Recursive || hashAlgo != HashAlgorithm::SHA256) {
                 HashSink narSink { HashAlgorithm::SHA256 };
                 dumpPath(realPath, narSink);
                 narHash = narSink.finish();
@@ -1367,7 +1401,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
             PosixSourceAccessor accessor;
             std::string hash = hashPath(
                 accessor, CanonPath { linkPath },
-                FileIngestionMethod::Recursive, HashAlgorithm::SHA256).first.to_string(HashFormat::Nix32, false);
+                FileIngestionMethod::Recursive, HashAlgorithm::SHA256).to_string(HashFormat::Nix32, false);
             if (hash != link.name) {
                 printError("link '%s' was modified! expected hash '%s', got '%s'",
                     linkPath, link.name, hash);

src/libstore/local-store.hh

@@ -180,7 +180,8 @@ public:
     StorePath addToStoreFromDump(
         Source & dump,
         std::string_view name,
-        ContentAddressMethod method,
+        FileSerialisationMethod dumpMethod,
+        ContentAddressMethod hashMethod,
         HashAlgorithm hashAlgo,
         const StorePathSet & references,
         RepairFlag repair) override;

src/libstore/optimise-store.cc

@@ -151,7 +151,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
         PosixSourceAccessor accessor;
         hashPath(
             accessor, CanonPath { path },
-            FileIngestionMethod::Recursive, HashAlgorithm::SHA256).first;
+            FileSerialisationMethod::Recursive, HashAlgorithm::SHA256).first;
     });
     debug("'%1%' has hash '%2%'", path, hash.to_string(HashFormat::Nix32, true));
 
@@ -166,7 +166,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
                 PosixSourceAccessor accessor;
                 hashPath(
                     accessor, CanonPath { linkPath },
-                    FileIngestionMethod::Recursive, HashAlgorithm::SHA256).first;
+                    FileSerialisationMethod::Recursive, HashAlgorithm::SHA256).first;
            })))
         {
             // XXX: Consider overwriting linkPath with our valid version.

src/libstore/remote-store.cc

@@ -13,6 +13,7 @@
 #include "derivations.hh"
 #include "pool.hh"
 #include "finally.hh"
+#include "git.hh"
 #include "logging.hh"
 #include "callback.hh"
 #include "filetransfer.hh"
@@ -508,12 +509,30 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore(
 StorePath RemoteStore::addToStoreFromDump(
     Source & dump,
     std::string_view name,
-    ContentAddressMethod method,
+    FileSerialisationMethod dumpMethod,
+    ContentAddressMethod hashMethod,
     HashAlgorithm hashAlgo,
     const StorePathSet & references,
     RepairFlag repair)
 {
-    return addCAToStore(dump, name, method, hashAlgo, references, repair)->path;
+    FileSerialisationMethod fsm;
+    switch (hashMethod.getFileIngestionMethod()) {
+    case FileIngestionMethod::Flat:
+        fsm = FileSerialisationMethod::Flat;
+        break;
+    case FileIngestionMethod::Recursive:
+        fsm = FileSerialisationMethod::Recursive;
+        break;
+    case FileIngestionMethod::Git:
+        // Use NAR; Git is not a serialization method
+        fsm = FileSerialisationMethod::Recursive;
+        break;
+    default:
+        assert(false);
+    }
+    if (fsm != dumpMethod)
+        unsupported("RemoteStore::addToStoreFromDump doesn't support this `dumpMethod` `hashMethod` combination");
+    return addCAToStore(dump, name, hashMethod, hashAlgo, references, repair)->path;
 }
 
 

src/libstore/remote-store.hh

@@ -87,7 +87,8 @@ public:
     StorePath addToStoreFromDump(
         Source & dump,
         std::string_view name,
-        ContentAddressMethod method = FileIngestionMethod::Recursive,
+        FileSerialisationMethod dumpMethod = FileSerialisationMethod::Recursive,
+        ContentAddressMethod hashMethod = FileIngestionMethod::Recursive,
         HashAlgorithm hashAlgo = HashAlgorithm::SHA256,
         const StorePathSet & references = StorePathSet(),
         RepairFlag repair = NoRepair) override;
@@ -184,7 +185,7 @@ protected:
 
     friend struct ConnectionHandle;
 
-    virtual ref<SourceAccessor> getFSAccessor(bool requireValidPath) override;
+    virtual ref<SourceAccessor> getFSAccessor(bool requireValidPath = true) override;
 
     virtual void narFromPath(const StorePath & path, Sink & sink) override;
 

src/libstore/store-api.cc

@@ -12,7 +12,9 @@
 #include "references.hh"
 #include "archive.hh"
 #include "callback.hh"
+#include "git.hh"
 #include "remote-store.hh"
+#include "posix-source-accessor.hh"
 // FIXME this should not be here, see TODO below on
 // `addMultipleToStore`.
 #include "worker-protocol.hh"
@@ -119,6 +121,9 @@ static std::string makeType(
 
 StorePath StoreDirConfig::makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const
 {
+    if (info.method == FileIngestionMethod::Git && info.hash.algo != HashAlgorithm::SHA1)
+        throw Error("Git file ingestion must use SHA-1 hash");
+
     if (info.hash.algo == HashAlgorithm::SHA256 && info.method == FileIngestionMethod::Recursive) {
         return makeStorePath(makeType(*this, "source", info.references), info.hash, name);
     } else {
@@ -166,7 +171,7 @@ std::pair<StorePath, Hash> StoreDirConfig::computeStorePath(
     const StorePathSet & references,
     PathFilter & filter) const
 {
-    auto h = hashPath(accessor, path, method.getFileIngestionMethod(), hashAlgo, filter).first;
+    auto h = hashPath(accessor, path, method.getFileIngestionMethod(), hashAlgo, filter);
     return {
         makeFixedOutputPathFromCA(
             name,
@@ -192,10 +197,23 @@ StorePath Store::addToStore(
     PathFilter & filter,
     RepairFlag repair)
 {
+    FileSerialisationMethod fsm;
+    switch (method.getFileIngestionMethod()) {
+    case FileIngestionMethod::Flat:
+        fsm = FileSerialisationMethod::Flat;
+        break;
+    case FileIngestionMethod::Recursive:
+        fsm = FileSerialisationMethod::Recursive;
+        break;
+    case FileIngestionMethod::Git:
+        // Use NAR; Git is not a serialization method
+        fsm = FileSerialisationMethod::Recursive;
+        break;
+    }
     auto source = sinkToSource([&](Sink & sink) {
-        dumpPath(accessor, path, sink, method.getFileIngestionMethod(), filter);
+        dumpPath(accessor, path, sink, fsm, filter);
     });
-    return addToStoreFromDump(*source, name, method, hashAlgo, references, repair);
+    return addToStoreFromDump(*source, name, fsm, method, hashAlgo, references, repair);
 }
 
 void Store::addMultipleToStore(
@@ -355,9 +373,7 @@ ValidPathInfo Store::addToStoreSlow(
     NullFileSystemObjectSink blank;
     auto & parseSink = method.getFileIngestionMethod() == FileIngestionMethod::Flat
         ? (FileSystemObjectSink &) fileSink
-        : method.getFileIngestionMethod() == FileIngestionMethod::Recursive
-        ? (FileSystemObjectSink &) blank
-        : (abort(), (FileSystemObjectSink &)*(FileSystemObjectSink *)nullptr); // handled both cases
+        : (FileSystemObjectSink &) blank; // for recursive or git we do recursive
 
     /* The information that flows from tapped (besides being replicated in
        narSink), is now put in parseSink. */
@@ -369,6 +385,8 @@ ValidPathInfo Store::addToStoreSlow(
 
     auto hash = method == FileIngestionMethod::Recursive && hashAlgo == HashAlgorithm::SHA256
         ? narHash
+        : method == FileIngestionMethod::Git
+        ? git::dumpHash(hashAlgo, accessor, srcPath).hash
         : caHashSink.finish().first;
 
     if (expectedCAHash && expectedCAHash != hash)

src/libstore/store-api.hh

@@ -466,14 +466,23 @@ public:
      * in `dump`, which is either a NAR serialisation (if recursive ==
      * true) or simply the contents of a regular file (if recursive ==
      * false).
-     * `dump` may be drained
      *
-     * \todo remove?
+     * `dump` may be drained.
+     *
+     * @param dumpMethod What serialisation format is `dump`, i.e. how
+     * to deserialize it. Must either match hashMethod or be
+     * `FileSerialisationMethod::Recursive`.
+     *
+     * @param hashMethod How content addressing? Need not match be the
+     * same as `dumpMethod`.
+     *
+     * @todo remove?
      */
     virtual StorePath addToStoreFromDump(
         Source & dump,
         std::string_view name,
-        ContentAddressMethod method = FileIngestionMethod::Recursive,
+        FileSerialisationMethod dumpMethod = FileSerialisationMethod::Recursive,
+        ContentAddressMethod hashMethod = FileIngestionMethod::Recursive,
         HashAlgorithm hashAlgo = HashAlgorithm::SHA256,
         const StorePathSet & references = StorePathSet(),
         RepairFlag repair = NoRepair) = 0;
@@ -772,7 +781,7 @@ protected:
      * Helper for methods that are not unsupported: this is used for
      * default definitions for virtual methods that are meant to be overriden.
      *
-     * \todo Using this should be a last resort. It is better to make
+     * @todo Using this should be a last resort. It is better to make
      * the method "virtual pure" and/or move it to a subclass.
      */
     [[noreturn]] void unsupported(const std::string & op)

src/libstore/uds-remote-store.hh

@@ -35,7 +35,7 @@ public:
     static std::set<std::string> uriSchemes()
     { return {"unix"}; }
 
-    ref<SourceAccessor> getFSAccessor(bool requireValidPath) override
+    ref<SourceAccessor> getFSAccessor(bool requireValidPath = true) override
     { return LocalFSStore::getFSAccessor(requireValidPath); }
 
     void narFromPath(const StorePath & path, Sink & sink) override

src/libutil/canon-path.cc

@@ -8,7 +8,7 @@ CanonPath CanonPath::root = CanonPath("/");
 
 static std::string absPathPure(std::string_view path)
 {
-    return canonPathInner(path, [](auto &, auto &){});
+    return canonPathInner<UnixPathTrait>(path, [](auto &, auto &){});
 }
 
 CanonPath::CanonPath(std::string_view raw)

src/libutil/current-process.cc

@@ -38,6 +38,11 @@ unsigned int getMaxCPU()
 
         auto cpuMax = readFile(cpuFile);
         auto cpuMaxParts = tokenizeString<std::vector<std::string>>(cpuMax, " \n");
+
+        if (cpuMaxParts.size() != 2) {
+            return 0;
+        }
+
         auto quota = cpuMaxParts[0];
         auto period = cpuMaxParts[1];
         if (quota != "max")

src/libutil/error.cc

@@ -11,9 +11,9 @@
 
 namespace nix {
 
-void BaseError::addTrace(std::shared_ptr<Pos> && e, HintFmt hint, bool frame)
+void BaseError::addTrace(std::shared_ptr<Pos> && e, HintFmt hint)
 {
-    err.traces.push_front(Trace { .pos = std::move(e), .hint = hint, .frame = frame });
+    err.traces.push_front(Trace { .pos = std::move(e), .hint = hint });
 }
 
 void throwExceptionSelfCheck(){
@@ -61,8 +61,7 @@ inline bool operator<(const Trace& lhs, const Trace& rhs)
     // This formats a freshly formatted hint string and then throws it away, which
     // shouldn't be much of a problem because it only runs when pos is equal, and this function is
     // used for trace printing, which is infrequent.
-    return std::forward_as_tuple(lhs.hint.str(), lhs.frame)
-        < std::forward_as_tuple(rhs.hint.str(), rhs.frame);
+    return lhs.hint.str() < rhs.hint.str();
 }
 inline bool operator> (const Trace& lhs, const Trace& rhs) { return rhs < lhs; }
 inline bool operator<=(const Trace& lhs, const Trace& rhs) { return !(lhs > rhs); }
@@ -373,7 +372,6 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s
     // prepended to each element of the trace
     auto ellipsisIndent = "  ";
 
-    bool frameOnly = false;
     if (!einfo.traces.empty()) {
         // Stack traces seen since we last printed a chunk of `duplicate frames
         // omitted`.
@@ -384,7 +382,6 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s
 
         for (const auto & trace : einfo.traces) {
             if (trace.hint.str().empty()) continue;
-            if (frameOnly && !trace.frame) continue;
 
             if (!showTrace && count > 3) {
                 oss << "\n" << ANSI_WARNING "(stack trace truncated; use '--show-trace' to show the full trace)" ANSI_NORMAL << "\n";
@@ -400,7 +397,6 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s
             printSkippedTracesMaybe(oss, ellipsisIndent, count, skippedTraces, tracesSeen);
 
             count++;
-            frameOnly = trace.frame;
 
             printTrace(oss, ellipsisIndent, count, trace);
         }

src/libutil/error.hh

@@ -64,7 +64,6 @@ void printCodeLines(std::ostream & out,
 struct Trace {
     std::shared_ptr<Pos> pos;
     HintFmt hint;
-    bool frame;
 };
 
 inline bool operator<(const Trace& lhs, const Trace& rhs);
@@ -162,7 +161,7 @@ public:
         addTrace(std::move(e), HintFmt(std::string(fs), args...));
     }
 
-    void addTrace(std::shared_ptr<Pos> && e, HintFmt hint, bool frame = false);
+    void addTrace(std::shared_ptr<Pos> && e, HintFmt hint);
 
     bool hasTrace() const { return !err.traces.empty(); }
 

src/libutil/experimental-features.cc

@@ -1,4 +1,5 @@
 #include "experimental-features.hh"
+#include "fmt.hh"
 #include "util.hh"
 
 #include "nlohmann/json.hpp"
@@ -10,6 +11,7 @@ struct ExperimentalFeatureDetails
     ExperimentalFeature tag;
     std::string_view name;
     std::string_view description;
+    std::string_view trackingUrl;
 };
 
 /**
@@ -35,6 +37,7 @@ constexpr std::array<ExperimentalFeatureDetails, numXpFeatures> xpFeatureDetails
             [__contentAddressed](@docroot@/language/advanced-attributes.md#adv-attr-__contentAddressed)
             for details.
         )",
+        .trackingUrl = "https://github.com/NixOS/nix/milestone/35",
     },
     {
         .tag = Xp::ImpureDerivations,
@@ -65,6 +68,7 @@ constexpr std::array<ExperimentalFeatureDetails, numXpFeatures> xpFeatureDetails
 
             This is a more explicit alternative to using [`builtins.currentTime`](@docroot@/language/builtin-constants.md#builtins-currentTime).
         )",
+        .trackingUrl = "https://github.com/NixOS/nix/milestone/42",
     },
     {
         .tag = Xp::Flakes,
@@ -73,6 +77,7 @@ constexpr std::array<ExperimentalFeatureDetails, numXpFeatures> xpFeatureDetails
             Enable flakes. See the manual entry for [`nix
             flake`](@docroot@/command-ref/new-cli/nix3-flake.md) for details.
         )",
+        .trackingUrl = "https://github.com/NixOS/nix/milestone/27",
     },
     {
         .tag = Xp::FetchTree,
@@ -86,6 +91,7 @@ constexpr std::array<ExperimentalFeatureDetails, numXpFeatures> xpFeatureDetails
 
             Enabling just this feature serves as a "release candidate", allowing users to try it out in isolation.
         )",
+        .trackingUrl = "https://github.com/NixOS/nix/milestone/31",
     },
     {
         .tag = Xp::NixCommand,
@@ -94,6 +100,7 @@ constexpr std::array<ExperimentalFeatureDetails, numXpFeatures> xpFeatureDetails
             Enable the new `nix` subcommands. See the manual on
             [`nix`](@docroot@/command-ref/new-cli/nix.md) for details.
         )",
+        .trackingUrl = "https://github.com/NixOS/nix/milestone/28",
     },
     {
         .tag = Xp::GitHashing,
@@ -102,6 +109,7 @@ constexpr std::array<ExperimentalFeatureDetails, numXpFeatures> xpFeatureDetails
             Allow creating (content-addressed) store objects which are hashed via Git's hashing algorithm.
             These store objects will not be understandable by older versions of Nix.
         )",
+        .trackingUrl = "https://github.com/NixOS/nix/milestone/41",
     },
     {
         .tag = Xp::RecursiveNix,
@@ -143,6 +151,7 @@ constexpr std::array<ExperimentalFeatureDetails, numXpFeatures> xpFeatureDetails
             already in the build inputs or built by a previous recursive Nix
             call.
         )",
+        .trackingUrl = "https://github.com/NixOS/nix/milestone/47",
     },
     {
         .tag = Xp::NoUrlLiterals,
@@ -184,6 +193,7 @@ constexpr std::array<ExperimentalFeatureDetails, numXpFeatures> xpFeatureDetails
             containing parameters have to be quoted anyway, and unquoted URLs
             may confuse external tooling.
         )",
+        .trackingUrl = "https://github.com/NixOS/nix/milestone/44",
     },
     {
         .tag = Xp::FetchClosure,
@@ -191,6 +201,7 @@ constexpr std::array<ExperimentalFeatureDetails, numXpFeatures> xpFeatureDetails
         .description = R"(
             Enable the use of the [`fetchClosure`](@docroot@/language/builtins.md#builtins-fetchClosure) built-in function in the Nix language.
         )",
+        .trackingUrl = "https://github.com/NixOS/nix/milestone/40",
     },
     {
         .tag = Xp::ReplFlake,
@@ -200,6 +211,7 @@ constexpr std::array<ExperimentalFeatureDetails, numXpFeatures> xpFeatureDetails
 
             Allow passing [installables](@docroot@/command-ref/new-cli/nix.md#installables) to `nix repl`, making its interface consistent with the other experimental commands.
         )",
+        .trackingUrl = "https://github.com/NixOS/nix/milestone/32",
     },
     {
         .tag = Xp::AutoAllocateUids,
@@ -208,6 +220,7 @@ constexpr std::array<ExperimentalFeatureDetails, numXpFeatures> xpFeatureDetails
             Allows Nix to automatically pick UIDs for builds, rather than creating
             `nixbld*` user accounts. See the [`auto-allocate-uids`](@docroot@/command-ref/conf-file.md#conf-auto-allocate-uids) setting for details.
         )",
+        .trackingUrl = "https://github.com/NixOS/nix/milestone/34",
     },
     {
         .tag = Xp::Cgroups,
@@ -216,6 +229,7 @@ constexpr std::array<ExperimentalFeatureDetails, numXpFeatures> xpFeatureDetails
             Allows Nix to execute builds inside cgroups. See
             the [`use-cgroups`](@docroot@/command-ref/conf-file.md#conf-use-cgroups) setting for details.
         )",
+        .trackingUrl = "https://github.com/NixOS/nix/milestone/36",
     },
     {
         .tag = Xp::DaemonTrustOverride,
@@ -226,6 +240,7 @@ constexpr std::array<ExperimentalFeatureDetails, numXpFeatures> xpFeatureDetails
             useful for various experiments with `nix-daemon --stdio`
             networking.
         )",
+        .trackingUrl = "https://github.com/NixOS/nix/milestone/38",
     },
     {
         .tag = Xp::DynamicDerivations,
@@ -239,6 +254,7 @@ constexpr std::array<ExperimentalFeatureDetails, numXpFeatures> xpFeatureDetails
               - dependencies in derivations on the outputs of
                 derivations that are themselves derivations outputs.
         )",
+        .trackingUrl = "https://github.com/NixOS/nix/milestone/39",
     },
     {
         .tag = Xp::ParseTomlTimestamps,
@@ -246,6 +262,7 @@ constexpr std::array<ExperimentalFeatureDetails, numXpFeatures> xpFeatureDetails
         .description = R"(
             Allow parsing of timestamps in builtins.fromTOML.
         )",
+        .trackingUrl = "https://github.com/NixOS/nix/milestone/45",
     },
     {
         .tag = Xp::ReadOnlyLocalStore,
@@ -253,6 +270,7 @@ constexpr std::array<ExperimentalFeatureDetails, numXpFeatures> xpFeatureDetails
         .description = R"(
             Allow the use of the `read-only` parameter in [local store](@docroot@/store/types/local-store.md) URIs.
         )",
+        .trackingUrl = "https://github.com/NixOS/nix/milestone/46",
     },
     {
         .tag = Xp::ConfigurableImpureEnv,
@@ -260,6 +278,7 @@ constexpr std::array<ExperimentalFeatureDetails, numXpFeatures> xpFeatureDetails
         .description = R"(
             Allow the use of the [impure-env](@docroot@/command-ref/conf-file.md#conf-impure-env) setting.
         )",
+        .trackingUrl = "https://github.com/NixOS/nix/milestone/37",
     },
     {
         .tag = Xp::MountedSSHStore,
@@ -267,6 +286,7 @@ constexpr std::array<ExperimentalFeatureDetails, numXpFeatures> xpFeatureDetails
         .description = R"(
             Allow the use of the [`mounted SSH store`](@docroot@/command-ref/new-cli/nix3-help-stores.html#experimental-ssh-store-with-filesytem-mounted).
         )",
+        .trackingUrl = "https://github.com/NixOS/nix/milestone/43",
     },
     {
         .tag = Xp::VerifiedFetches,
@@ -274,6 +294,7 @@ constexpr std::array<ExperimentalFeatureDetails, numXpFeatures> xpFeatureDetails
         .description = R"(
             Enables verification of git commit signatures through the [`fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit) built-in.
         )",
+        .trackingUrl = "https://github.com/NixOS/nix/milestone/48",
     },
 }};
 
@@ -312,9 +333,12 @@ std::string_view showExperimentalFeature(const ExperimentalFeature tag)
 nlohmann::json documentExperimentalFeatures()
 {
     StringMap res;
-    for (auto & xpFeature : xpFeatureDetails)
-        res[std::string { xpFeature.name }] =
-            trim(stripIndentation(xpFeature.description));
+    for (auto & xpFeature : xpFeatureDetails) {
+        std::stringstream docOss;
+        docOss << stripIndentation(xpFeature.description);
+        docOss << fmt("\nRefer to [%1% tracking issue](%2%) for feature tracking.", xpFeature.name, xpFeature.trackingUrl);
+        res[std::string{xpFeature.name}] = trim(docOss.str());
+    }
     return (nlohmann::json) res;
 }
 

src/libutil/file-content-address.cc

@@ -1,16 +1,53 @@
 #include "file-content-address.hh"
 #include "archive.hh"
+#include "git.hh"
 
 namespace nix {
 
-FileIngestionMethod parseFileIngestionMethod(std::string_view input)
+static std::optional<FileSerialisationMethod> parseFileSerialisationMethodOpt(std::string_view input)
 {
     if (input == "flat") {
-        return FileIngestionMethod::Flat;
+        return FileSerialisationMethod::Flat;
     } else if (input == "nar") {
-        return FileIngestionMethod::Recursive;
+        return FileSerialisationMethod::Recursive;
     } else {
-        throw UsageError("Unknown file ingestion method '%s', expect `flat` or `nar`");
+        return std::nullopt;
+    }
+}
+
+FileSerialisationMethod parseFileSerialisationMethod(std::string_view input)
+{
+    auto ret = parseFileSerialisationMethodOpt(input);
+    if (ret)
+        return *ret;
+    else
+        throw UsageError("Unknown file serialiation method '%s', expect `flat` or `nar`");
+}
+
+
+FileIngestionMethod parseFileIngestionMethod(std::string_view input)
+{
+    if (input == "git") {
+        return FileIngestionMethod::Git;
+    } else {
+        auto ret = parseFileSerialisationMethodOpt(input);
+        if (ret)
+            return static_cast<FileIngestionMethod>(*ret);
+        else
+            throw UsageError("Unknown file ingestion method '%s', expect `flat`, `nar`, or `git`");
+    }
+}
+
+
+std::string_view renderFileSerialisationMethod(FileSerialisationMethod method)
+{
+    switch (method) {
+    case FileSerialisationMethod::Flat:
+        return "flat";
+    case FileSerialisationMethod::Recursive:
+        return "nar";
+    default:
+        assert(false);
     }
 }
 
@@ -19,9 +56,11 @@ std::string_view renderFileIngestionMethod(FileIngestionMethod method)
 {
     switch (method) {
     case FileIngestionMethod::Flat:
-        return "flat";
     case FileIngestionMethod::Recursive:
-        return "nar";
+        return renderFileSerialisationMethod(
+            static_cast<FileSerialisationMethod>(method));
+    case FileIngestionMethod::Git:
+        return "git";
     default:
         abort();
     }
@@ -31,14 +70,14 @@ std::string_view renderFileIngestionMethod(FileIngestionMethod method)
 void dumpPath(
     SourceAccessor & accessor, const CanonPath & path,
     Sink & sink,
-    FileIngestionMethod method,
+    FileSerialisationMethod method,
     PathFilter & filter)
 {
     switch (method) {
-    case FileIngestionMethod::Flat:
+    case FileSerialisationMethod::Flat:
         accessor.readFile(path, sink);
         break;
-    case FileIngestionMethod::Recursive:
+    case FileSerialisationMethod::Recursive:
         accessor.dumpPath(path, sink, filter);
         break;
     }
@@ -48,13 +87,13 @@ void dumpPath(
 void restorePath(
     const Path & path,
     Source & source,
-    FileIngestionMethod method)
+    FileSerialisationMethod method)
 {
     switch (method) {
-    case FileIngestionMethod::Flat:
+    case FileSerialisationMethod::Flat:
         writeFile(path, source);
         break;
-    case FileIngestionMethod::Recursive:
+    case FileSerialisationMethod::Recursive:
         restorePath(path, source);
         break;
     }
@@ -63,12 +102,28 @@ void restorePath(
 
 HashResult hashPath(
     SourceAccessor & accessor, const CanonPath & path,
-    FileIngestionMethod method, HashAlgorithm ht,
+    FileSerialisationMethod method, HashAlgorithm ha,
     PathFilter & filter)
 {
-    HashSink sink { ht };
+    HashSink sink { ha };
     dumpPath(accessor, path, sink, method, filter);
     return sink.finish();
 }
 
+
+Hash hashPath(
+    SourceAccessor & accessor, const CanonPath & path,
+    FileIngestionMethod method, HashAlgorithm ht,
+    PathFilter & filter)
+{
+    switch (method) {
+    case FileIngestionMethod::Flat:
+    case FileIngestionMethod::Recursive:
+        return hashPath(accessor, path, (FileSerialisationMethod) method, ht, filter).first;
+    case FileIngestionMethod::Git:
+        return git::dumpHash(ht, accessor, path, filter).hash;
+    }
+    assert(false);
+}
+
 }

src/libutil/file-content-address.hh

@@ -8,37 +8,38 @@
 namespace nix {
 
 /**
- * An enumeration of the main ways we can serialize file system
+ * An enumeration of the ways we can serialize file system
  * objects.
  */
-enum struct FileIngestionMethod : uint8_t {
+enum struct FileSerialisationMethod : uint8_t {
     /**
-     * Flat-file hashing. Directly ingest the contents of a single file
+     * Flat-file. The contents of a single file exactly.
      */
-    Flat = 0,
+    Flat,
+
     /**
-     * Recursive (or NAR) hashing. Serializes the file-system object in
-     * Nix Archive format and ingest that.
+     * Nix Archive. Serializes the file-system object in
+     * Nix Archive format.
      */
-    Recursive = 1,
+    Recursive,
 };
 
 /**
- * Parse a `FileIngestionMethod` by name. Choice of:
+ * Parse a `FileSerialisationMethod` by name. Choice of:
  *
- *  - `flat`: `FileIngestionMethod::Flat`
- *  - `nar`: `FileIngestionMethod::Recursive`
+ *  - `flat`: `FileSerialisationMethod::Flat`
+ *  - `nar`: `FileSerialisationMethod::Recursive`
  *
- * Oppostite of `renderFileIngestionMethod`.
+ * Opposite of `renderFileSerialisationMethod`.
  */
-FileIngestionMethod parseFileIngestionMethod(std::string_view input);
+FileSerialisationMethod parseFileSerialisationMethod(std::string_view input);
 
 /**
- * Render a `FileIngestionMethod` by name.
+ * Render a `FileSerialisationMethod` by name.
  *
- * Oppostite of `parseFileIngestionMethod`.
+ * Opposite of `parseFileSerialisationMethod`.
  */
-std::string_view renderFileIngestionMethod(FileIngestionMethod method);
+std::string_view renderFileSerialisationMethod(FileSerialisationMethod method);
 
 /**
  * Dump a serialization of the given file system object.
@@ -46,28 +47,99 @@ std::string_view renderFileIngestionMethod(FileIngestionMethod method);
 void dumpPath(
     SourceAccessor & accessor, const CanonPath & path,
     Sink & sink,
-    FileIngestionMethod method,
+    FileSerialisationMethod method,
     PathFilter & filter = defaultPathFilter);
 
 /**
- * Restore a serialization of the given file system object.
+ * Restore a serialisation of the given file system object.
  *
  * @TODO use an arbitrary `FileSystemObjectSink`.
  */
 void restorePath(
     const Path & path,
     Source & source,
-    FileIngestionMethod method);
+    FileSerialisationMethod method);
+
 
 /**
  * Compute the hash of the given file system object according to the
  * given method.
  *
- * The hash is defined as (essentially) hashString(ht, dumpPath(path)).
+ * the hash is defined as (in pseudocode):
+ *
+ * ```
+ * hashString(ha, dumpPath(...))
+ * ```
  */
 HashResult hashPath(
     SourceAccessor & accessor, const CanonPath & path,
-    FileIngestionMethod method, HashAlgorithm ht,
+    FileSerialisationMethod method, HashAlgorithm ha,
+    PathFilter & filter = defaultPathFilter);
+
+/**
+ * An enumeration of the ways we can ingest file system
+ * objects, producing a hash or digest.
+ */
+enum struct FileIngestionMethod : uint8_t {
+    /**
+     * Hash `FileSerialisationMethod::Flat` serialisation.
+     */
+    Flat,
+
+    /**
+     * Hash `FileSerialisationMethod::Git` serialisation.
+     */
+    Recursive,
+
+    /**
+     * Git hashing. In particular files are hashed as git "blobs", and
+     * directories are hashed as git "trees".
+     *
+     * Unlike `Flat` and `Recursive`, this is not a hash of a single
+     * serialisation but a [Merkle
+     * DAG](https://en.wikipedia.org/wiki/Merkle_tree) of multiple
+     * rounds of serialisation and hashing.
+     *
+     * @note Git's data model is slightly different, in that a plain
+     * file doesn't have an executable bit, directory entries do
+     * instead. We decide treat a bare file as non-executable by fiat,
+     * as we do with `FileIngestionMethod::Flat` which also lacks this
+     * information. Thus, Git can encode some but all of Nix's "File
+     * System Objects", and this sort of hashing is likewise partial.
+     */
+    Git,
+};
+
+/**
+ * Parse a `FileIngestionMethod` by name. Choice of:
+ *
+ *  - `flat`: `FileIngestionMethod::Flat`
+ *  - `nar`: `FileIngestionMethod::Recursive`
+ *  - `git`: `FileIngestionMethod::Git`
+ *
+ * Opposite of `renderFileIngestionMethod`.
+ */
+FileIngestionMethod parseFileIngestionMethod(std::string_view input);
+
+/**
+ * Render a `FileIngestionMethod` by name.
+ *
+ * Opposite of `parseFileIngestionMethod`.
+ */
+std::string_view renderFileIngestionMethod(FileIngestionMethod method);
+
+/**
+ * Compute the hash of the given file system object according to the
+ * given method.
+ *
+ * Unlike the other `hashPath`, this works on an arbitrary
+ * `FileIngestionMethod` instead of `FileSerialisationMethod`, but
+ * doesn't return the size as this is this is not a both simple and
+ * useful defined for a merkle format.
+ */
+Hash hashPath(
+    SourceAccessor & accessor, const CanonPath & path,
+    FileIngestionMethod method, HashAlgorithm ha,
     PathFilter & filter = defaultPathFilter);
 
 }

src/libutil/file-path-impl.hh

@@ -10,6 +10,100 @@
 
 namespace nix {
 
+/**
+ * Unix-style path primives.
+ *
+ * Nix'result own "logical" paths are always Unix-style. So this is always
+ * used for that, and additionally used for native paths on Unix.
+ */
+struct UnixPathTrait
+{
+    using CharT = char;
+
+    using String = std::string;
+
+    using StringView = std::string_view;
+
+    constexpr static char preferredSep = '/';
+
+    static inline bool isPathSep(char c)
+    {
+        return c == '/';
+    }
+
+    static inline size_t findPathSep(StringView path, size_t from = 0)
+    {
+        return path.find('/', from);
+    }
+
+    static inline size_t rfindPathSep(StringView path, size_t from = StringView::npos)
+    {
+        return path.rfind('/', from);
+    }
+};
+
+
+/**
+ * Windows-style path primitives.
+ *
+ * The character type is a parameter because while windows paths rightly
+ * work over UTF-16 (*) using `wchar_t`, at the current time we are
+ * often manipulating them converted to UTF-8 (*) using `char`.
+ *
+ * (Actually neither are guaranteed to be valid unicode; both are
+ * arbitrary non-0 8- or 16-bit bytes. But for charcters with specifical
+ * meaning like '/', '\\', ':', etc., we refer to an encoding scheme,
+ * and also for sake of UIs that display paths a text.)
+ */
+template<class CharT0>
+struct WindowsPathTrait
+{
+    using CharT = CharT0;
+
+    using String = std::basic_string<CharT>;
+
+    using StringView = std::basic_string_view<CharT>;
+
+    constexpr static CharT preferredSep = '\\';
+
+    static inline bool isPathSep(CharT c)
+    {
+        return c == '/' || c == preferredSep;
+    }
+
+    static size_t findPathSep(StringView path, size_t from = 0)
+    {
+        size_t p1 = path.find('/', from);
+        size_t p2 = path.find(preferredSep, from);
+        return p1 == String::npos ? p2 :
+               p2 == String::npos ? p1 :
+               std::min(p1, p2);
+    }
+
+    static size_t rfindPathSep(StringView path, size_t from = String::npos)
+    {
+        size_t p1 = path.rfind('/', from);
+        size_t p2 = path.rfind(preferredSep, from);
+        return p1 == String::npos ? p2 :
+               p2 == String::npos ? p1 :
+               std::max(p1, p2);
+    }
+};
+
+
+/**
+ * @todo Revisit choice of `char` or `wchar_t` for `WindowsPathTrait`
+ * argument.
+ */
+using NativePathTrait =
+#ifdef _WIN32
+    WindowsPathTrait<char>
+#else
+    UnixPathTrait
+#endif
+    ;
+
+
 /**
  * Core pure path canonicalization algorithm.
  *
@@ -24,25 +118,26 @@ namespace nix {
  *   This is a chance to modify those two paths in arbitrary way, e.g. if
  *   "result" points to a symlink.
  */
-typename std::string canonPathInner(
-    std::string_view remaining,
+template<class PathDict>
+typename PathDict::String canonPathInner(
+    typename PathDict::StringView remaining,
     auto && hookComponent)
 {
     assert(remaining != "");
 
-    std::string result;
+    typename PathDict::String result;
     result.reserve(256);
 
     while (true) {
 
         /* Skip slashes. */
-        while (!remaining.empty() && remaining[0] == '/')
+        while (!remaining.empty() && PathDict::isPathSep(remaining[0]))
             remaining.remove_prefix(1);
 
         if (remaining.empty()) break;
 
         auto nextComp = ({
-            auto nextPathSep = remaining.find('/');
+            auto nextPathSep = PathDict::findPathSep(remaining);
             nextPathSep == remaining.npos ? remaining : remaining.substr(0, nextPathSep);
         });
 
@@ -53,14 +148,14 @@ typename std::string canonPathInner(
         /* If `..', delete the last component. */
         else if (nextComp == "..")
         {
-            if (!result.empty()) result.erase(result.rfind('/'));
+            if (!result.empty()) result.erase(PathDict::rfindPathSep(result));
             remaining.remove_prefix(2);
         }
 
         /* Normal component; copy it. */
         else {
-            result += '/';
-            if (const auto slash = remaining.find('/'); slash == result.npos) {
+            result += PathDict::preferredSep;
+            if (const auto slash = PathDict::findPathSep(remaining); slash == result.npos) {
                 result += remaining;
                 remaining = {};
             } else {
@@ -73,7 +168,7 @@ typename std::string canonPathInner(
     }
 
     if (result.empty())
-        result = "/";
+        result = typename PathDict::String { PathDict::preferredSep };
 
     return result;
 }

src/libutil/file-system.cc

@@ -22,10 +22,14 @@ namespace fs = std::filesystem;
 
 namespace nix {
 
-/** Treat the string as possibly an absolute path, by inspecting the start of it. Return whether it was probably intended to be absolute. */
+/**
+ * Treat the string as possibly an absolute path, by inspecting the
+ * start of it. Return whether it was probably intended to be
+ * absolute.
+ */
 static bool isAbsolute(PathView path)
 {
-    return !path.empty() && path[0] == '/';
+    return fs::path { path }.is_absolute();
 }
 
 
@@ -69,6 +73,9 @@ Path canonPath(PathView path, bool resolveSymlinks)
     if (!isAbsolute(path))
         throw Error("not an absolute path: '%1%'", path);
 
+    // For Windows
+    auto rootName = fs::path { path }.root_name();
+
     /* This just exists because we cannot set the target of `remaining`
        (the callback parameter) directly to a newly-constructed string,
        since it is `std::string_view`. */
@@ -78,7 +85,7 @@ Path canonPath(PathView path, bool resolveSymlinks)
        arbitrary (but high) limit to prevent infinite loops. */
     unsigned int followCount = 0, maxFollow = 1024;
 
-    return canonPathInner(
+    auto ret = canonPathInner<NativePathTrait>(
         path,
         [&followCount, &temp, maxFollow, resolveSymlinks]
         (std::string & result, std::string_view & remaining) {
@@ -99,6 +106,10 @@ Path canonPath(PathView path, bool resolveSymlinks)
                 }
             }
         });
+
+    if (!rootName.empty())
+        ret = rootName.string() + std::move(ret);
+    return ret;
 }
 
 
@@ -606,6 +617,11 @@ void copy(const fs::directory_entry & from, const fs::path & to, bool andDelete)
     }
 }
 
+void copyFile(const Path & oldPath, const Path & newPath, bool andDelete)
+{
+    return copy(fs::directory_entry(fs::path(oldPath)), fs::path(newPath), andDelete);
+}
+
 void renameFile(const Path & oldName, const Path & newName)
 {
     fs::rename(oldName, newName);

src/libutil/file-system.hh

@@ -186,6 +186,13 @@ void renameFile(const Path & src, const Path & dst);
  */
 void moveFile(const Path & src, const Path & dst);
 
+/**
+ * Recursively copy the content of `oldPath` to `newPath`. If `andDelete` is
+ * `true`, then also remove `oldPath` (making this equivalent to `moveFile`, but
+ * with the guaranty that the destination will be “fresh”, with no stale inode
+ * or file descriptor pointing to it).
+ */
+void copyFile(const Path & oldPath, const Path & newPath, bool andDelete);
 
 /**
  * Automatic cleanup of resources.

src/libutil/fmt.hh

@@ -8,7 +8,6 @@
 
 namespace nix {
 
-namespace {
 /**
  * A helper for writing `boost::format` expressions.
  *
@@ -35,14 +34,13 @@ inline void formatHelper(F & f, const T & x, const Args & ... args)
 /**
  * Set the correct exceptions for `fmt`.
  */
-void setExceptions(boost::format & fmt)
+inline void setExceptions(boost::format & fmt)
 {
     fmt.exceptions(
         boost::io::all_error_bits ^
         boost::io::too_many_args_bit ^
         boost::io::too_few_args_bit);
 }
-}
 
 /**
  * A helper for writing a `boost::format` expression to a string.

src/libutil/fs-sink.cc

@@ -15,6 +15,7 @@ void copyRecursive(
     case SourceAccessor::tSymlink:
     {
         sink.createSymlink(to, accessor.readLink(from));
+        break;
     }
 
     case SourceAccessor::tRegular:
@@ -38,6 +39,7 @@ void copyRecursive(
                 sink, to + "/" + name);
             break;
         }
+        break;
     }
 
     case SourceAccessor::tMisc:

src/libutil/git.cc

@@ -56,31 +56,63 @@ void parseBlob(
     FileSystemObjectSink & sink,
     const Path & sinkPath,
     Source & source,
-    bool executable,
+    BlobMode blobMode,
     const ExperimentalFeatureSettings & xpSettings)
 {
     xpSettings.require(Xp::GitHashing);
 
-    sink.createRegularFile(sinkPath, [&](auto & crf) {
-        if (executable)
-            crf.isExecutable();
+    unsigned long long size = std::stoi(getStringUntil(source, 0));
 
-        unsigned long long size = std::stoi(getStringUntil(source, 0));
+    auto doRegularFile = [&](bool executable) {
+        sink.createRegularFile(sinkPath, [&](auto & crf) {
+            if (executable)
+                crf.isExecutable();
 
-        crf.preallocateContents(size);
+            crf.preallocateContents(size);
 
-        unsigned long long left = size;
-        std::string buf;
-        buf.reserve(65536);
+            unsigned long long left = size;
+            std::string buf;
+            buf.reserve(65536);
 
-        while (left) {
+            while (left) {
+                checkInterrupt();
+                buf.resize(std::min((unsigned long long)buf.capacity(), left));
+                source(buf);
+                crf(buf);
+                left -= buf.size();
+            }
+        });
+    };
+
+    switch (blobMode) {
+
+    case BlobMode::Regular:
+        doRegularFile(false);
+        break;
+
+    case BlobMode::Executable:
+        doRegularFile(true);
+        break;
+
+    case BlobMode::Symlink:
+    {
+        std::string target;
+        target.resize(size, '0');
+        target.reserve(size);
+        for (size_t n = 0; n < target.size();) {
             checkInterrupt();
-            buf.resize(std::min((unsigned long long)buf.capacity(), left));
-            source(buf);
-            crf(buf);
-            left -= buf.size();
+            n += source.read(
+                const_cast<char *>(target.c_str()) + n,
+                target.size() - n);
         }
-    });
+
+        sink.createSymlink(sinkPath, target);
+        break;
+    }
+
+    default:
+        assert(false);
+    }
 }
 
 void parseTree(
@@ -142,7 +174,7 @@ void parse(
     FileSystemObjectSink & sink,
     const Path & sinkPath,
     Source & source,
-    bool executable,
+    BlobMode rootModeIfBlob,
     std::function<SinkHook> hook,
     const ExperimentalFeatureSettings & xpSettings)
 {
@@ -152,7 +184,7 @@ void parse(
 
     switch (type) {
     case ObjectType::Blob:
-        parseBlob(sink, sinkPath, source, executable, xpSettings);
+        parseBlob(sink, sinkPath, source, rootModeIfBlob, xpSettings);
         break;
     case ObjectType::Tree:
         parseTree(sink, sinkPath, source, hook, xpSettings);
@@ -177,7 +209,7 @@ std::optional<Mode> convertMode(SourceAccessor::Type type)
 
 void restore(FileSystemObjectSink & sink, Source & source, std::function<RestoreHook> hook)
 {
-    parse(sink, "", source, false, [&](Path name, TreeEntry entry) {
+    parse(sink, "", source, BlobMode::Regular, [&](Path name, TreeEntry entry) {
         auto [accessor, from] = hook(entry.hash);
         auto stat = accessor->lstat(from);
         auto gotOpt = convertMode(stat.type);
@@ -275,6 +307,13 @@ Mode dump(
     }
 
     case SourceAccessor::tSymlink:
+    {
+        auto target = accessor.readLink(path);
+        dumpBlobPrefix(target.size(), sink, xpSettings);
+        sink(target);
+        return Mode::Symlink;
+    }
+
     case SourceAccessor::tMisc:
     default:
         throw Error("file '%1%' has an unsupported type", path);

src/libutil/git.hh

@@ -75,10 +75,23 @@ ObjectType parseObjectType(
     Source & source,
     const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
 
+/**
+ * These 3 modes are represented by blob objects.
+ *
+ * Sometimes we need this information to disambiguate how a blob is
+ * being used to better match our own "file system object" data model.
+ */
+enum struct BlobMode : RawMode
+{
+    Regular = static_cast<RawMode>(Mode::Regular),
+    Executable = static_cast<RawMode>(Mode::Executable),
+    Symlink = static_cast<RawMode>(Mode::Symlink),
+};
+
 void parseBlob(
     FileSystemObjectSink & sink, const Path & sinkPath,
     Source & source,
-    bool executable,
+    BlobMode blobMode,
     const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
 
 void parseTree(
@@ -89,11 +102,15 @@ void parseTree(
 
 /**
  * Helper putting the previous three `parse*` functions together.
+ *
+ * @rootModeIfBlob How to interpret a root blob, for which there is no
+ * disambiguating dir entry to answer that questino. If the root it not
+ * a blob, this is ignored.
  */
 void parse(
     FileSystemObjectSink & sink, const Path & sinkPath,
     Source & source,
-    bool executable,
+    BlobMode rootModeIfBlob,
     std::function<SinkHook> hook,
     const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);