Merge pull request #11045 from NixOS/backport-11031-to-2.23-maintenance

[Backport 2.23-maintenance] libstore: fix sandboxed builds on macOS

.github/CODEOWNERS

@@ -11,7 +11,16 @@
 .github/CODEOWNERS @edolstra
 
 # Documentation of built-in functions
-src/libexpr/primops.cc @roberth
+src/libexpr/primops.cc @roberth @fricklerhandwerk
+
+# Documentation of settings
+src/libexpr/eval-settings.hh @fricklerhandwerk
+src/libstore/globals.hh @fricklerhandwerk
+
+# Documentation
+doc/manual @fricklerhandwerk
+maintainers/*.md @fricklerhandwerk
+src/**/*.md @fricklerhandwerk
 
 # Libstore layer
 /src/libstore @thufschmitt @ericson2314

.github/workflows/backport.yml

@@ -21,7 +21,7 @@ jobs:
           fetch-depth: 0
       - name: Create backport PRs
         # should be kept in sync with `version`
-        uses: zeebe-io/backport-action@v2.5.0
+        uses: zeebe-io/backport-action@v3.0.2
         with:
           # Config README: https://github.com/zeebe-io/backport-action#backport-action
           github_token: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -92,7 +92,7 @@ perl/Makefile.config
 
 # /tests/functional/
 /tests/functional/test-tmp
-/tests/functional/common/vars-and-functions.sh
+/tests/functional/common/subst-vars.sh
 /tests/functional/result*
 /tests/functional/restricted-innocent
 /tests/functional/shell

.shellcheckrc

@@ -0,0 +1,2 @@
+external-sources=true
+source-path=SCRIPTDIR

.version

@@ -1 +1 @@
-2.23.0
+2.23.2

build/hydra.nix

@@ -0,0 +1,186 @@
+{ inputs
+, binaryTarball
+, forAllCrossSystems
+, forAllSystems
+, lib
+, linux64BitSystems
+, nixpkgsFor
+, self
+}:
+let
+  inherit (inputs) nixpkgs nixpkgs-regression;
+  inherit (lib) fileset;
+
+  installScriptFor = tarballs:
+    nixpkgsFor.x86_64-linux.native.callPackage ../scripts/installer.nix {
+      inherit tarballs;
+    };
+
+  testNixVersions = pkgs: client: daemon:
+    pkgs.callPackage ../package.nix {
+      pname =
+        "nix-tests"
+        + lib.optionalString
+          (lib.versionAtLeast daemon.version "2.4pre20211005" &&
+           lib.versionAtLeast client.version "2.4pre20211005")
+          "-${client.version}-against-${daemon.version}";
+
+      inherit fileset;
+
+      test-client = client;
+      test-daemon = daemon;
+
+      doBuild = false;
+    };
+in
+{
+  # Binary package for various platforms.
+  build = forAllSystems (system: self.packages.${system}.nix);
+
+  shellInputs = forAllSystems (system: self.devShells.${system}.default.inputDerivation);
+
+  buildStatic = lib.genAttrs linux64BitSystems (system: self.packages.${system}.nix-static);
+
+  buildCross = forAllCrossSystems (crossSystem:
+    lib.genAttrs [ "x86_64-linux" ] (system: self.packages.${system}."nix-${crossSystem}"));
+
+  buildNoGc = forAllSystems (system:
+    self.packages.${system}.nix.override { enableGC = false; }
+  );
+
+  buildNoTests = forAllSystems (system:
+    self.packages.${system}.nix.override {
+      doCheck = false;
+      doInstallCheck = false;
+      installUnitTests = false;
+    }
+  );
+
+  # Toggles some settings for better coverage. Windows needs these
+  # library combinations, and Debian build Nix with GNU readline too.
+  buildReadlineNoMarkdown = forAllSystems (system:
+    self.packages.${system}.nix.override {
+      enableMarkdown = false;
+      readlineFlavor = "readline";
+    }
+  );
+
+  # Perl bindings for various platforms.
+  perlBindings = forAllSystems (system: nixpkgsFor.${system}.native.nix.perl-bindings);
+
+  # Binary tarball for various platforms, containing a Nix store
+  # with the closure of 'nix' package, and the second half of
+  # the installation script.
+  binaryTarball = forAllSystems (system: binaryTarball nixpkgsFor.${system}.native.nix nixpkgsFor.${system}.native);
+
+  binaryTarballCross = lib.genAttrs [ "x86_64-linux" ] (system:
+    forAllCrossSystems (crossSystem:
+      binaryTarball
+        self.packages.${system}."nix-${crossSystem}"
+        nixpkgsFor.${system}.cross.${crossSystem}));
+
+  # The first half of the installation script. This is uploaded
+  # to https://nixos.org/nix/install. It downloads the binary
+  # tarball for the user's system and calls the second half of the
+  # installation script.
+  installerScript = installScriptFor [
+    # Native
+    self.hydraJobs.binaryTarball."x86_64-linux"
+    self.hydraJobs.binaryTarball."i686-linux"
+    self.hydraJobs.binaryTarball."aarch64-linux"
+    self.hydraJobs.binaryTarball."x86_64-darwin"
+    self.hydraJobs.binaryTarball."aarch64-darwin"
+    # Cross
+    self.hydraJobs.binaryTarballCross."x86_64-linux"."armv6l-unknown-linux-gnueabihf"
+    self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-unknown-linux-gnueabihf"
+    self.hydraJobs.binaryTarballCross."x86_64-linux"."riscv64-unknown-linux-gnu"
+  ];
+  installerScriptForGHA = installScriptFor [
+    # Native
+    self.hydraJobs.binaryTarball."x86_64-linux"
+    self.hydraJobs.binaryTarball."aarch64-darwin"
+    # Cross
+    self.hydraJobs.binaryTarballCross."x86_64-linux"."armv6l-unknown-linux-gnueabihf"
+    self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-unknown-linux-gnueabihf"
+    self.hydraJobs.binaryTarballCross."x86_64-linux"."riscv64-unknown-linux-gnu"
+  ];
+
+  # docker image with Nix inside
+  dockerImage = lib.genAttrs linux64BitSystems (system: self.packages.${system}.dockerImage);
+
+  # Line coverage analysis.
+  coverage = nixpkgsFor.x86_64-linux.native.nix.override {
+    pname = "nix-coverage";
+    withCoverageChecks = true;
+  };
+
+  # API docs for Nix's unstable internal C++ interfaces.
+  internal-api-docs = nixpkgsFor.x86_64-linux.native.callPackage ../package.nix {
+    inherit fileset;
+    doBuild = false;
+    enableInternalAPIDocs = true;
+  };
+
+  # API docs for Nix's C bindings.
+  external-api-docs = nixpkgsFor.x86_64-linux.native.callPackage ../package.nix {
+    inherit fileset;
+    doBuild = false;
+    enableExternalAPIDocs = true;
+  };
+
+  # System tests.
+  tests = import ../tests/nixos { inherit lib nixpkgs nixpkgsFor; } // {
+
+    # Make sure that nix-env still produces the exact same result
+    # on a particular version of Nixpkgs.
+    evalNixpkgs =
+      let
+        inherit (nixpkgsFor.x86_64-linux.native) runCommand nix;
+      in
+      runCommand "eval-nixos" { buildInputs = [ nix ]; }
+        ''
+          type -p nix-env
+          # Note: we're filtering out nixos-install-tools because https://github.com/NixOS/nixpkgs/pull/153594#issuecomment-1020530593.
+          (
+            set -x
+            time nix-env --store dummy:// -f ${nixpkgs-regression} -qaP --drv-path | sort | grep -v nixos-install-tools > packages
+            [[ $(sha1sum < packages | cut -c1-40) = e01b031fc9785a572a38be6bc473957e3b6faad7 ]]
+          )
+          mkdir $out
+        '';
+
+    nixpkgsLibTests =
+      forAllSystems (system:
+        import (nixpkgs + "/lib/tests/release.nix")
+          {
+            pkgs = nixpkgsFor.${system}.native;
+            nixVersions = [ self.packages.${system}.nix ];
+          }
+      );
+  };
+
+  metrics.nixpkgs = import "${nixpkgs-regression}/pkgs/top-level/metrics.nix" {
+    pkgs = nixpkgsFor.x86_64-linux.native;
+    nixpkgs = nixpkgs-regression;
+  };
+
+  installTests = forAllSystems (system:
+    let pkgs = nixpkgsFor.${system}.native; in
+    pkgs.runCommand "install-tests"
+      {
+        againstSelf = testNixVersions pkgs pkgs.nix pkgs.pkgs.nix;
+        againstCurrentUnstable =
+          # FIXME: temporarily disable this on macOS because of #3605.
+          if system == "x86_64-linux"
+          then testNixVersions pkgs pkgs.nix pkgs.nixUnstable
+          else null;
+        # Disabled because the latest stable version doesn't handle
+        # `NIX_DAEMON_SOCKET_PATH` which is required for the tests to work
+        # againstLatestStable = testNixVersions pkgs pkgs.nix pkgs.nixStable;
+      } "touch $out");
+
+  installerTests = import ../tests/installer {
+    binaryTarballs = self.hydraJobs.binaryTarball;
+    inherit nixpkgsFor;
+  };
+}

doc/manual/rl-next/consistent-nix-build.md

@@ -1,6 +0,0 @@
----
-synopsis: Show all FOD errors with `nix build --keep-going`
----
-
-`nix build --keep-going` now behaves consistently with `nix-build --keep-going`. This means
-that if e.g. multiple FODs fail to build, all hash mismatches are displayed.

doc/manual/rl-next/derivation-json-change.md

@@ -1,12 +0,0 @@
----
-synopsis: Modify `nix derivation {add,show}` JSON format
-issues: 9866
-prs: 10722
----
-
-The JSON format for derivations has been slightly revised to better conform to our [JSON guidelines](@docroot@contributing/cli-guideline#returning-future-proof-json).
-In particular, the hash algorithm and content addressing method of content-addresed derivation outputs is now separated into two fields `hashAlgo` and `method`,
-rather than one field with an arcane `:`-separated format.
-
-This JSON format is only used by the experimental `nix derivation` family of commands, at this time.
-Future revisions are expected as the JSON format is still not entirely in compliance even after these changes.

doc/manual/rl-next/fix-silent-unknown-options

@@ -1,28 +0,0 @@
----
-synopsis: Warn on unknown settings anywhere in the command line
-prs: 10701
----
-
-All `nix` commands will now properly warn when an unknown option is specified anywhere in the command line.
-
-Before:
-
-```console
-$ nix-instantiate --option foobar baz --expr '{}'
-warning: unknown setting 'foobar'
-$ nix-instantiate '{}' --option foobar baz --expr
-$ nix eval --expr '{}' --option foobar baz
-{ }
-```
-
-After:
-
-```console
-$ nix-instantiate --option foobar baz --expr '{}'
-warning: unknown setting 'foobar'
-$ nix-instantiate '{}' --option foobar baz --expr
-warning: unknown setting 'foobar'
-$ nix eval --expr '{}' --option foobar baz
-warning: unknown setting 'foobar'
-{ }
-```

doc/manual/rl-next/harden-user-sandboxing.md

@@ -0,0 +1,6 @@
+---
+synopsis: Harden the user sandboxing
+significance: significant
+---
+
+The build directory has been hardened against interference with the outside world by nesting it inside another directory owned by (and only readable by) the daemon user.

doc/manual/src/SUMMARY.md.in

@@ -121,9 +121,11 @@
   - [Documentation](contributing/documentation.md)
   - [Experimental Features](contributing/experimental-features.md)
   - [CLI guideline](contributing/cli-guideline.md)
+  - [JSON guideline](contributing/json-guideline.md)
   - [C++ style guide](contributing/cxx.md)
 - [Releases](release-notes/index.md)
 {{#include ./SUMMARY-rl-next.md}}
+  - [Release 2.23 (2024-06-03)](release-notes/rl-2.23.md)
   - [Release 2.22 (2024-04-23)](release-notes/rl-2.22.md)
   - [Release 2.21 (2024-03-11)](release-notes/rl-2.21.md)
   - [Release 2.20 (2024-01-29)](release-notes/rl-2.20.md)

doc/manual/src/command-ref/nix-shell.md

@@ -202,14 +202,14 @@ For example, here is a Python script that depends on Python and the
 
 ```python
 #! /usr/bin/env nix-shell
-#! nix-shell -i python --packages python pythonPackages.prettytable
+#! nix-shell -i python3 --packages python3 python3Packages.prettytable
 
 import prettytable
 
 # Print a simple table.
 t = prettytable.PrettyTable(["N", "N^2"])
 for n in range(1, 10): t.add_row([n, n * n])
-print t
+print(t)
 ```
 
 Similarly, the following is a Perl script that specifies that it

doc/manual/src/contributing/cli-guideline.md

@@ -389,88 +389,6 @@ colors, no emojis and using ASCII instead of Unicode symbols). The same should
 happen when TTY is not detected on STDERR. We should not display progress /
 status section, but only print warnings and errors.
 
-## Returning future proof JSON
-
-The schema of JSON output should allow for backwards compatible extension. This section explains how to achieve this.
-
-Two definitions are helpful here, because while JSON only defines one "key-value"
-object type, we use it to cover two use cases:
-
- - **dictionary**: a map from names to value that all have the same type. In
-   C++ this would be a `std::map` with string keys.
- - **record**: a fixed set of attributes each with their own type. In C++, this
-   would be represented by a `struct`.
-
-It is best not to mix these use cases, as that may lead to incompatibilities when the schema changes. For example, adding a record field to a dictionary breaks consumers that assume all JSON object fields to have the same meaning and type.
-
-This leads to the following guidelines:
-
- - The top-level (root) value must be a record.
-
-   Otherwise, one can not change the structure of a command's output.
-
- - The value of a dictionary item must be a record.
-
-   Otherwise, the item type can not be extended.
-
- - List items should be records.
-
-   Otherwise, one can not change the structure of the list items.
-
-   If the order of the items does not matter, and each item has a unique key that is a string, consider representing the list as a dictionary instead. If the order of the items needs to be preserved, return a list of records.
-
- - Streaming JSON should return records.
-
-   An example of a streaming JSON format is [JSON lines](https://jsonlines.org/), where each line represents a JSON value. These JSON values can be considered top-level values or list items, and they must be records.
-
-### Examples
-
-
-This is bad, because all keys must be assumed to be store types:
-
-```json
-{
-  "local": { ... },
-  "remote": { ... },
-  "http": { ... }
-}
-```
-
-This is good, because the it is extensible at the root, and is somewhat self-documenting:
-
-```json
-{
-  "storeTypes": { "local": { ... }, ... },
-  "pluginSupport": true
-}
-```
-
-While the dictionary of store types seems like a very complete response at first, a use case may arise that warrants returning additional information.
-For example, the presence of plugin support may be crucial information for a client to proceed when their desired store type is missing.
-
-
-
-The following representation is bad because it is not extensible:
-
-```json
-{ "outputs": [ "out" "bin" ] }
-```
-
-However, simply converting everything to records is not enough, because the order of outputs must be preserved:
-
-```json
-{ "outputs": { "bin": {}, "out": {} } }
-```
-
-The first item is the default output. Deriving this information from the outputs ordering is not great, but this is how Nix currently happens to work.
-While it is possible for a JSON parser to preserve the order of fields, we can not rely on this capability to be present in all JSON libraries.
-
-This representation is extensible and preserves the ordering:
-
-```json
-{ "outputs": [ { "outputName": "out" }, { "outputName": "bin" } ] }
-```
-
 ## Dialog with the user
 
 CLIs don't always make it clear when an action has taken place. For every

doc/manual/src/contributing/json-guideline.md

@@ -0,0 +1,128 @@
+# JSON guideline
+
+Nix consumes and produces JSON in a variety of contexts.
+These guidelines ensure consistent practices for all our JSON interfaces, for ease of use, and so that experience in one part carries over to another.
+
+## Extensibility
+
+The schema of JSON input and output should allow for backwards compatible extension.
+This section explains how to achieve this.
+
+Two definitions are helpful here, because while JSON only defines one "key-value" object type, we use it to cover two use cases:
+
+ - **dictionary**: a map from names to value that all have the same type.
+   In C++ this would be a `std::map` with string keys.
+
+ - **record**: a fixed set of attributes each with their own type.
+   In C++, this would be represented by a `struct`.
+
+It is best not to mix these use cases, as that may lead to incompatibilities when the schema changes.
+For example, adding a record field to a dictionary breaks consumers that assume all JSON object fields to have the same meaning and type, and dictionary items with a colliding name can not be represented anymore.
+
+This leads to the following guidelines:
+
+ - The top-level (root) value must be a record.
+
+   Otherwise, one can not change the structure of a command's output.
+
+ - The value of a dictionary item must be a record.
+
+   Otherwise, the item type can not be extended.
+
+ - List items should be records.
+
+   Otherwise, one can not change the structure of the list items.
+
+   If the order of the items does not matter, and each item has a unique key that is a string, consider representing the list as a dictionary instead.
+   If the order of the items needs to be preserved, return a list of records.
+
+ - Streaming JSON should return records.
+
+   An example of a streaming JSON format is [JSON lines](https://jsonlines.org/), where each line represents a JSON value.
+   These JSON values can be considered top-level values or list items, and they must be records.
+
+### Examples
+
+This is bad, because all keys must be assumed to be store types:
+
+```json
+{
+  "local": { ... },
+  "remote": { ... },
+  "http": { ... }
+}
+```
+
+This is good, because the it is extensible at the root, and is somewhat self-documenting:
+
+```json
+{
+  "storeTypes": { "local": { ... }, ... },
+  "pluginSupport": true
+}
+```
+
+While the dictionary of store types seems like a very complete response at first, a use case may arise that warrants returning additional information.
+For example, the presence of plugin support may be crucial information for a client to proceed when their desired store type is missing.
+
+
+
+The following representation is bad because it is not extensible:
+
+```json
+{ "outputs": [ "out" "bin" ] }
+```
+
+However, simply converting everything to records is not enough, because the order of outputs must be preserved:
+
+```json
+{ "outputs": { "bin": {}, "out": {} } }
+```
+
+The first item is the default output. Deriving this information from the outputs ordering is not great, but this is how Nix currently happens to work.
+While it is possible for a JSON parser to preserve the order of fields, we can not rely on this capability to be present in all JSON libraries.
+
+This representation is extensible and preserves the ordering:
+
+```json
+{ "outputs": [ { "outputName": "out" }, { "outputName": "bin" } ] }
+```
+
+## Self-describing values
+
+As described in the previous section, it's crucial that schemas can be extended with with new fields without breaking compatibility.
+However, that should *not* mean we use the presence/absence of fields to indicate optional information *within* a version of the schema.
+Instead, always include the field, and use `null` to indicate the "nothing" case.
+
+### Examples
+
+Here are two JSON objects:
+
+```json
+{
+  "foo": {}
+}
+```
+```json
+{
+  "foo": {},
+  "bar": {}
+}
+```
+
+Since they differ in which fields they contain, they should *not* both be valid values of the same schema.
+At most, they can match two different schemas where the second (with `foo` and `bar`) is considered a newer version of the first (with just `foo`).
+Within each version, all fields are mandatory (always `foo`, and always `foo` and `bar`).
+Only *between* each version, `bar` gets added as a new mandatory field.
+
+Here are another two JSON objects:
+
+```json
+{ "foo": null }
+```
+```json
+{ "foo": { "bar": 1 } }
+```
+
+Since they both contain a `foo` field, they could be valid values of the same schema.
+The schema would have `foo` has an optional field, which is either `null` or an object where `bar` is an integer.

doc/manual/src/glossary.md

@@ -12,7 +12,7 @@
   For how Nix uses content addresses, see:
 
     - [Content-Addressing File System Objects](@docroot@/store/file-system-object/content-address.md)
-    - [content-addressed store object](#gloss-content-addressed-store-object)
+    - [Content-Addressing Store Objects](@docroot@/store/store-object/content-address.md)
     - [content-addressed derivation](#gloss-content-addressed-derivation)
 
   Software Heritage's writing on [*Intrinsic and Extrinsic identifiers*](https://www.softwareheritage.org/2020/07/09/intrinsic-vs-extrinsic-identifiers) is also a good introduction to the value of content-addressing over other referencing schemes.
@@ -137,9 +137,12 @@
 
 - [content-addressed store object]{#gloss-content-addressed-store-object}
 
-  A [store object] whose [store path] is determined by its contents.
+  A [store object] which is [content-addressed](#gloss-content-address),
+  i.e. whose [store path] is determined by its contents.
   This includes derivations, the outputs of [content-addressed derivations](#gloss-content-addressed-derivation), and the outputs of [fixed-output derivations](#gloss-fixed-output-derivation).
 
+  See [Content-Addressing Store Objects](@docroot@/store/store-object/content-address.md) for details.
+
 - [substitute]{#gloss-substitute}
 
   A substitute is a command invocation stored in the [Nix database] that

doc/manual/src/language/operators.md

@@ -141,7 +141,7 @@ The result is a string.
 
 Update [attribute set] *attrset1* with names and values from *attrset2*.
 
-The returned attribute set will have of all the attributes in *attrset1* and *attrset2*.
+The returned attribute set will have all of the attributes in *attrset1* and *attrset2*.
 If an attribute name is present in both, the attribute value from the latter is taken.
 
 [Update]: #update

doc/manual/src/protocols/json/store-object-info.md

@@ -24,9 +24,11 @@ Info about a [store object].
 
   An array of [store paths][store path], possibly including this one.
 
-* `ca` (optional):
+* `ca`:
 
-  Content address of this store object's file system object, used to compute its store path.
+  If the store object is [content-addressed],
+  this is the content address of this store object's file system object, used to compute its store path.
+  Otherwise (i.e. if it is [input-addressed]), this is `null`.
 
 [store path]: @docroot@/store/store-path.md
 [file system object]: @docroot@/store/file-system-object.md
@@ -37,28 +39,30 @@ Info about a [store object].
 These are not intrinsic properties of the store object.
 In other words, the same store object residing in different store could have different values for these properties.
 
-* `deriver` (optional):
+* `deriver`:
 
-  The path to the [derivation] from which this store object is produced.
+  If known, the path to the [derivation] from which this store object was produced.
+  Otherwise `null`.
 
   [derivation]: @docroot@/glossary.md#gloss-store-derivation
 
 * `registrationTime` (optional):
 
-  When this derivation was added to the store.
+  If known, when this derivation was added to the store.
+  Otherwise `null`.
 
-* `ultimate` (optional):
+* `ultimate`:
 
   Whether this store object is trusted because we built it ourselves, rather than substituted a build product from elsewhere.
 
-* `signatures` (optional):
+* `signatures`:
 
   Signatures claiming that this store object is what it claims to be.
   Not relevant for [content-addressed] store objects,
   but useful for [input-addressed] store objects.
 
-  [content-addressed]: @docroot@/glossary.md#gloss-content-addressed-store-object
-  [input-addressed]: @docroot@/glossary.md#gloss-input-addressed-store-object
+[content-addressed]: @docroot@/store/store-object/content-address.md
+[input-addressed]: @docroot@/glossary.md#gloss-input-addressed-store-object
 
 ### `.narinfo` extra fields
 

doc/manual/src/release-notes/rl-2.23.md

@@ -0,0 +1,102 @@
+# Release 2.23.0 (2024-06-03)
+
+- New builtin: `builtins.warn` [#306026](https://github.com/NixOS/nix/issues/306026) [#10592](https://github.com/NixOS/nix/pull/10592)
+
+  `builtins.warn` behaves like `builtins.trace "warning: ${msg}"`, has an accurate log level, and is controlled by the options
+  [`debugger-on-trace`](@docroot@/command-ref/conf-file.md#conf-debugger-on-trace),
+  [`debugger-on-warn`](@docroot@/command-ref/conf-file.md#conf-debugger-on-warn) and
+  [`abort-on-warn`](@docroot@/command-ref/conf-file.md#conf-abort-on-warn).
+
+- Make `nix build --keep-going` consistent with `nix-build --keep-going`
+
+  This means that if e.g. multiple fixed-output derivations fail to
+  build, all hash mismatches are displayed.
+
+- Modify `nix derivation {add,show}` JSON format [#9866](https://github.com/NixOS/nix/issues/9866) [#10722](https://github.com/NixOS/nix/pull/10722)
+
+  The JSON format for derivations has been slightly revised to better conform to our [JSON guidelines](@docroot@/contributing/cli-guideline.md#returning-future-proof-json).
+  In particular, the hash algorithm and content addressing method of content-addresed derivation outputs are now separated into two fields `hashAlgo` and `method`,
+  rather than one field with an arcane `:`-separated format.
+
+  This JSON format is only used by the experimental `nix derivation` family of commands, at this time.
+  Future revisions are expected as the JSON format is still not entirely in compliance even after these changes.
+
+- Warn on unknown settings anywhere in the command line [#10701](https://github.com/NixOS/nix/pull/10701)
+
+  All `nix` commands will now properly warn when an unknown option is specified anywhere in the command line.
+
+  Before:
+
+  ```console
+  $ nix-instantiate --option foobar baz --expr '{}'
+  warning: unknown setting 'foobar'
+  $ nix-instantiate '{}' --option foobar baz --expr
+  $ nix eval --expr '{}' --option foobar baz
+  { }
+  ```
+
+  After:
+
+  ```console
+  $ nix-instantiate --option foobar baz --expr '{}'
+  warning: unknown setting 'foobar'
+  $ nix-instantiate '{}' --option foobar baz --expr
+  warning: unknown setting 'foobar'
+  $ nix eval --expr '{}' --option foobar baz
+  warning: unknown setting 'foobar'
+  { }
+  ```
+
+- `nix env shell` is the new `nix shell`, and `nix shell` remains an accepted alias [#10504](https://github.com/NixOS/nix/issues/10504) [#10807](https://github.com/NixOS/nix/pull/10807)
+
+  This is part of an effort to bring more structure to the CLI subcommands.
+
+  `nix env` will be about the process environment.
+  Future commands may include `nix env run` and `nix env print-env`.
+
+  It is also somewhat analogous to the [planned](https://github.com/NixOS/nix/issues/10504) `nix dev shell` (currently `nix develop`), which is less about environment variables, and more about running a development shell, which is a more powerful command, but also requires more setup.
+
+- Flake operations that expect derivations now print the failing value and its type [#10778](https://github.com/NixOS/nix/pull/10778)
+
+  In errors like `flake output attribute 'nixosConfigurations.yuki.config' is not a derivation or path`, the message now includes the failing value and type.
+
+  Before:
+
+  ```
+  error: flake output attribute 'nixosConfigurations.yuki.config' is not a derivation or path
+  ````
+
+  After:
+
+  ```
+  error: expected flake output attribute 'nixosConfigurations.yuki.config' to be a derivation or path but found a set: { appstream = «thunk»; assertions = «thunk»; boot = { bcache = «thunk»; binfmt = «thunk»; binfmtMiscRegistrations = «thunk»; blacklistedKernelModules = «thunk»; bootMount = «thunk»; bootspec = «thunk»; cleanTmpDir = «thunk»; consoleLogLevel = «thunk»; «43 attributes elided» }; «48 attributes elided» }
+  ```
+
+- `fetchTree` now fetches Git repositories shallowly by default [#10028](https://github.com/NixOS/nix/pull/10028)
+
+  `builtins.fetchTree` now clones Git repositories shallowly by default, which reduces network traffic and disk usage significantly in many cases.
+
+  Previously, the default behavior was to clone the full history of a specific tag or branch (e.g. `ref`) and only afterwards extract the files of one specific revision.
+
+  From now on, the `ref` and `allRefs` arguments will be ignored, except if shallow cloning is disabled by setting `shallow = false`.
+
+  The defaults for `builtins.fetchGit` remain unchanged. Here, shallow cloning has to be enabled manually by passing `shallow = true`.
+
+- Store object info JSON format now uses `null` rather than omitting fields [#9995](https://github.com/NixOS/nix/pull/9995)
+
+  The [store object info JSON format](@docroot@/protocols/json/store-object-info.md), used for e.g. `nix path-info`, no longer omits fields to indicate absent information, but instead includes the fields with a `null` value.
+  For example, `"ca": null` is used to to indicate a store object that isn't content-addressed rather than omitting the `ca` field entirely.
+  This makes records of this sort more self-describing, and easier to consume programmatically.
+
+  We will follow this design principle going forward;
+  the [JSON guidelines](@docroot@/contributing/json-guideline.md) in the contributing section have been updated accordingly.
+
+- Large path warnings [#10661](https://github.com/NixOS/nix/pull/10661)
+
+  Nix can now warn when evaluation of a Nix expression causes a large
+  path to be copied to the Nix store. The threshold for this warning can
+  be configured using [the `warn-large-path-threshold`
+  setting](@docroot@/command-ref/conf-file.md#warn-large-path-threshold),
+  e.g. `--warn-large-path-threshold 100M` will warn about paths larger
+  than 100 MiB.
+

flake.nix

@@ -26,7 +26,7 @@
       inherit (nixpkgs) lib;
       inherit (lib) fileset;
 
-      officialRelease = false;
+      officialRelease = true;
 
       version = lib.fileContents ./.version + versionSuffix;
       versionSuffix =
@@ -104,28 +104,6 @@
           cross = forAllCrossSystems (crossSystem: make-pkgs crossSystem "stdenv");
         });
 
-      installScriptFor = tarballs:
-        nixpkgsFor.x86_64-linux.native.callPackage ./scripts/installer.nix {
-          inherit tarballs;
-        };
-
-      testNixVersions = pkgs: client: daemon:
-        pkgs.callPackage ./package.nix {
-          pname =
-            "nix-tests"
-            + lib.optionalString
-              (lib.versionAtLeast daemon.version "2.4pre20211005" &&
-               lib.versionAtLeast client.version "2.4pre20211005")
-              "-${client.version}-against-${daemon.version}";
-
-          inherit fileset;
-
-          test-client = client;
-          test-daemon = daemon;
-
-          doBuild = false;
-        };
-
       binaryTarball = nix: pkgs: pkgs.callPackage ./scripts/binary-tarball.nix {
         inherit nix;
       };
@@ -191,7 +169,7 @@
 
           nix =
             let
-              officialRelease = false;
+              officialRelease = true;
               versionSuffix =
                 if officialRelease
                 then ""
@@ -203,7 +181,7 @@
                 stdenv
                 versionSuffix
                 ;
-              officialRelease = false;
+              officialRelease = true;
               boehmgc = final.boehmgc-nix;
               libgit2 = final.libgit2-nix;
               libseccomp = final.libseccomp-nix;
@@ -232,156 +210,17 @@
       # 'nix.perl-bindings' packages.
       overlays.default = overlayFor (p: p.stdenv);
 
-      hydraJobs = {
-
-        # Binary package for various platforms.
-        build = forAllSystems (system: self.packages.${system}.nix);
-
-        shellInputs = forAllSystems (system: self.devShells.${system}.default.inputDerivation);
-
-        buildStatic = lib.genAttrs linux64BitSystems (system: self.packages.${system}.nix-static);
-
-        buildCross = forAllCrossSystems (crossSystem:
-          lib.genAttrs ["x86_64-linux"] (system: self.packages.${system}."nix-${crossSystem}"));
-
-        buildNoGc = forAllSystems (system:
-          self.packages.${system}.nix.override { enableGC = false; }
-        );
-
-        buildNoTests = forAllSystems (system:
-          self.packages.${system}.nix.override {
-            doCheck = false;
-            doInstallCheck = false;
-            installUnitTests = false;
-          }
-        );
-
-        # Toggles some settings for better coverage. Windows needs these
-        # library combinations, and Debian build Nix with GNU readline too.
-        buildReadlineNoMarkdown = forAllSystems (system:
-          self.packages.${system}.nix.override {
-            enableMarkdown = false;
-            readlineFlavor = "readline";
-          }
-        );
-
-        # Perl bindings for various platforms.
-        perlBindings = forAllSystems (system: nixpkgsFor.${system}.native.nix.perl-bindings);
-
-        # Binary tarball for various platforms, containing a Nix store
-        # with the closure of 'nix' package, and the second half of
-        # the installation script.
-        binaryTarball = forAllSystems (system: binaryTarball nixpkgsFor.${system}.native.nix nixpkgsFor.${system}.native);
-
-        binaryTarballCross = lib.genAttrs ["x86_64-linux"] (system:
-          forAllCrossSystems (crossSystem:
-            binaryTarball
-              self.packages.${system}."nix-${crossSystem}"
-              nixpkgsFor.${system}.cross.${crossSystem}));
-
-        # The first half of the installation script. This is uploaded
-        # to https://nixos.org/nix/install. It downloads the binary
-        # tarball for the user's system and calls the second half of the
-        # installation script.
-        installerScript = installScriptFor [
-          # Native
-          self.hydraJobs.binaryTarball."x86_64-linux"
-          self.hydraJobs.binaryTarball."i686-linux"
-          self.hydraJobs.binaryTarball."aarch64-linux"
-          self.hydraJobs.binaryTarball."x86_64-darwin"
-          self.hydraJobs.binaryTarball."aarch64-darwin"
-          # Cross
-          self.hydraJobs.binaryTarballCross."x86_64-linux"."armv6l-unknown-linux-gnueabihf"
-          self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-unknown-linux-gnueabihf"
-          self.hydraJobs.binaryTarballCross."x86_64-linux"."riscv64-unknown-linux-gnu"
-        ];
-        installerScriptForGHA = installScriptFor [
-          # Native
-          self.hydraJobs.binaryTarball."x86_64-linux"
-          self.hydraJobs.binaryTarball."x86_64-darwin"
-          # Cross
-          self.hydraJobs.binaryTarballCross."x86_64-linux"."armv6l-unknown-linux-gnueabihf"
-          self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-unknown-linux-gnueabihf"
-          self.hydraJobs.binaryTarballCross."x86_64-linux"."riscv64-unknown-linux-gnu"
-        ];
-
-        # docker image with Nix inside
-        dockerImage = lib.genAttrs linux64BitSystems (system: self.packages.${system}.dockerImage);
-
-        # Line coverage analysis.
-        coverage = nixpkgsFor.x86_64-linux.native.nix.override {
-          pname = "nix-coverage";
-          withCoverageChecks = true;
-        };
-
-        # API docs for Nix's unstable internal C++ interfaces.
-        internal-api-docs = nixpkgsFor.x86_64-linux.native.callPackage ./package.nix {
-          inherit fileset;
-          doBuild = false;
-          enableInternalAPIDocs = true;
-        };
-
-        # API docs for Nix's C bindings.
-        external-api-docs = nixpkgsFor.x86_64-linux.native.callPackage ./package.nix {
-          inherit fileset;
-          doBuild = false;
-          enableExternalAPIDocs = true;
-        };
-
-        # System tests.
-        tests = import ./tests/nixos { inherit lib nixpkgs nixpkgsFor; } // {
-
-          # Make sure that nix-env still produces the exact same result
-          # on a particular version of Nixpkgs.
-          evalNixpkgs =
-            let
-              inherit (nixpkgsFor.x86_64-linux.native) runCommand nix;
-            in
-            runCommand "eval-nixos" { buildInputs = [ nix ]; }
-              ''
-                type -p nix-env
-                # Note: we're filtering out nixos-install-tools because https://github.com/NixOS/nixpkgs/pull/153594#issuecomment-1020530593.
-                (
-                  set -x
-                  time nix-env --store dummy:// -f ${nixpkgs-regression} -qaP --drv-path | sort | grep -v nixos-install-tools > packages
-                  [[ $(sha1sum < packages | cut -c1-40) = e01b031fc9785a572a38be6bc473957e3b6faad7 ]]
-                )
-                mkdir $out
-              '';
-
-          nixpkgsLibTests =
-            forAllSystems (system:
-              import (nixpkgs + "/lib/tests/release.nix")
-                { pkgs = nixpkgsFor.${system}.native;
-                  nixVersions = [ self.packages.${system}.nix ];
-                }
-            );
-        };
-
-        metrics.nixpkgs = import "${nixpkgs-regression}/pkgs/top-level/metrics.nix" {
-          pkgs = nixpkgsFor.x86_64-linux.native;
-          nixpkgs = nixpkgs-regression;
-        };
-
-        installTests = forAllSystems (system:
-          let pkgs = nixpkgsFor.${system}.native; in
-          pkgs.runCommand "install-tests" {
-            againstSelf = testNixVersions pkgs pkgs.nix pkgs.pkgs.nix;
-            againstCurrentUnstable =
-              # FIXME: temporarily disable this on macOS because of #3605.
-              if system == "x86_64-linux"
-              then testNixVersions pkgs pkgs.nix pkgs.nixUnstable
-              else null;
-            # Disabled because the latest stable version doesn't handle
-            # `NIX_DAEMON_SOCKET_PATH` which is required for the tests to work
-            # againstLatestStable = testNixVersions pkgs pkgs.nix pkgs.nixStable;
-          } "touch $out");
-
-        installerTests = import ./tests/installer {
-          binaryTarballs = self.hydraJobs.binaryTarball;
-          inherit nixpkgsFor;
-        };
-
+      hydraJobs = import ./build/hydra.nix {
+        inherit
+          inputs
+          binaryTarball
+          forAllCrossSystems
+          forAllSystems
+          lib
+          linux64BitSystems
+          nixpkgsFor
+          self
+          ;
       };
 
       checks = forAllSystems (system: {
@@ -393,6 +232,7 @@
           in pkgs.buildPackages.runCommand "test-rl-next-release-notes" { } ''
           LANG=C.UTF-8 ${pkgs.changelog-d-nix}/bin/changelog-d ${./doc/manual/rl-next} >$out
         '';
+        repl-completion = nixpkgsFor.${system}.native.callPackage ./tests/repl-completion.nix { };
       } // (lib.optionalAttrs (builtins.elem system linux64BitSystems)) {
         dockerImage = self.hydraJobs.dockerImage.${system};
       } // (lib.optionalAttrs (!(builtins.elem system linux32BitSystems))) {

maintainers/upload-release.pl

@@ -243,6 +243,7 @@ write_file("$tmpDir/fallback-paths.nix",
     "  x86_64-linux = \"" . getStorePath("build.x86_64-linux") . "\";\n" .
     "  i686-linux = \"" . getStorePath("build.i686-linux") . "\";\n" .
     "  aarch64-linux = \"" . getStorePath("build.aarch64-linux") . "\";\n" .
+    "  riscv64-linux = \"" . getStorePath("buildCross.riscv64-unknown-linux-gnu.x86_64-linux") . "\";\n" .
     "  x86_64-darwin = \"" . getStorePath("build.x86_64-darwin") . "\";\n" .
     "  aarch64-darwin = \"" . getStorePath("build.aarch64-darwin") . "\";\n" .
     "}\n");

mk/common-test.sh

@@ -1,19 +1,23 @@
+# shellcheck shell=bash
+
 # Remove overall test dir (at most one of the two should match) and
 # remove file extension.
-test_name=$(echo -n "$test" | sed \
+
+test_name=$(echo -n "${test?must be defined by caller (test runner)}" | sed \
     -e "s|^tests/unit/[^/]*/data/||" \
     -e "s|^tests/functional/||" \
     -e "s|\.sh$||" \
     )
 
+# shellcheck disable=SC2016
 TESTS_ENVIRONMENT=(
     "TEST_NAME=$test_name"
     'NIX_REMOTE='
     'PS4=+(${BASH_SOURCE[0]-$0}:$LINENO) '
 )
 
-: ${BASH:=/usr/bin/env bash}
+read -r -a bash <<< "${BASH:-/usr/bin/env bash}"
 
 run () {
-   cd "$(dirname $1)" && env "${TESTS_ENVIRONMENT[@]}" $BASH -x -e -u -o pipefail $(basename $1)
+   cd "$(dirname "$1")" && env "${TESTS_ENVIRONMENT[@]}" "${bash[@]}" -x -e -u -o pipefail "$(basename "$1")"
 }

mk/run-test.sh

@@ -26,12 +26,13 @@ run_test () {
 
 run_test
 
-if [ $status -eq 0 ]; then
+if [[ "$status" = 0 ]]; then
   echo "$post_run_msg [${green}PASS$normal]"
-elif [ $status -eq 99 ]; then
+elif [[ "$status" = 99 ]]; then
   echo "$post_run_msg [${yellow}SKIP$normal]"
 else
   echo "$post_run_msg [${red}FAIL$normal]"
+  # shellcheck disable=SC2001
   echo "$log" | sed 's/^/    /'
   exit "$status"
 fi

perl/lib/Nix/Store.xs

@@ -258,7 +258,7 @@ hashPath(char * algo, int base32, char * path)
         try {
             Hash h = hashPath(
                 PosixSourceAccessor::createAtRoot(path),
-                FileIngestionMethod::Recursive, parseHashAlgo(algo));
+                FileIngestionMethod::Recursive, parseHashAlgo(algo)).first;
             auto s = h.to_string(base32 ? HashFormat::Nix32 : HashFormat::Base16, false);
             XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
         } catch (Error & e) {

scripts/bigsur-nixbld-user-migration.sh

@@ -3,7 +3,7 @@
 ((NEW_NIX_FIRST_BUILD_UID=301))
 
 id_available(){
-	dscl . list /Users UniqueID | grep -E '\b'$1'\b' >/dev/null
+	dscl . list /Users UniqueID | grep -E '\b'"$1"'\b' >/dev/null
 }
 
 change_nixbld_names_and_ids(){
@@ -26,18 +26,18 @@ change_nixbld_names_and_ids(){
 			fi
 		done
 
-		if [[ $name == _* ]]; then
+		if [[ "$name" == _* ]]; then
 			echo "      It looks like $name has already been renamed--skipping."
 		else
 			# first 3 are cleanup, it's OK if they aren't here
-			sudo dscl . delete /Users/$name dsAttrTypeNative:_writers_passwd &>/dev/null || true
-			sudo dscl . change /Users/$name NFSHomeDirectory "/private/var/empty 1" "/var/empty" &>/dev/null || true
+			sudo dscl . delete "/Users/$name" dsAttrTypeNative:_writers_passwd &>/dev/null || true
+			sudo dscl . change "/Users/$name" NFSHomeDirectory "/private/var/empty 1" "/var/empty" &>/dev/null || true
 			# remove existing user from group
-			sudo dseditgroup -o edit -t user -d $name nixbld || true
-			sudo dscl . change /Users/$name UniqueID $uid $next_id
-			sudo dscl . change /Users/$name RecordName $name _$name
+			sudo dseditgroup -o edit -t user -d "$name" nixbld || true
+			sudo dscl . change "/Users/$name" UniqueID "$uid" "$next_id"
+			sudo dscl . change "/Users/$name" RecordName "$name" "_$name"
 			# add renamed user to group
-			sudo dseditgroup -o edit -t user -a _$name nixbld
+			sudo dseditgroup -o edit -t user -a "_$name" nixbld
 			echo "      $name migrated to _$name (uid: $next_id)"
 		fi
 	done < <(dscl . list /Users UniqueID | grep nixbld | sort -n -k2)

scripts/install.in

@@ -50,6 +50,11 @@ case "$(uname -s).$(uname -m)" in
         path=@tarballPath_armv7l-linux@
         system=armv7l-linux
         ;;
+    Linux.riscv64)
+        hash=@tarballHash_riscv64-linux@
+        path=@tarballPath_riscv64-linux@
+        system=riscv64-linux
+        ;;
     Darwin.x86_64)
         hash=@tarballHash_x86_64-darwin@
         path=@tarballPath_x86_64-darwin@

scripts/nix-profile-daemon.sh.in

@@ -1,4 +1,5 @@
 # Only execute this file once per shell.
+# This file is tested by tests/installer/default.nix.
 if [ -n "${__ETC_PROFILE_NIX_SOURCED:-}" ]; then return; fi
 __ETC_PROFILE_NIX_SOURCED=1
 
@@ -9,11 +10,9 @@ else
     NIX_LINK_NEW=$HOME/.local/state/nix/profile
 fi
 if [ -e "$NIX_LINK_NEW" ]; then
-    NIX_LINK="$NIX_LINK_NEW"
-else
-    if [ -t 2 ] && [ -e "$NIX_LINK_NEW" ]; then
+    if [ -t 2 ] && [ -e "$NIX_LINK" ]; then
         warning="\033[1;35mwarning:\033[0m"
-        printf "$warning Both %s and legacy %s exist; using the latter.\n" "$NIX_LINK_NEW" "$NIX_LINK" 1>&2
+        printf "$warning Both %s and legacy %s exist; using the former.\n" "$NIX_LINK_NEW" "$NIX_LINK" 1>&2
         if [ "$(realpath "$NIX_LINK")" = "$(realpath "$NIX_LINK_NEW")" ]; then
             printf "         Since the profiles match, you can safely delete either of them.\n" 1>&2
         else
@@ -26,6 +25,7 @@ else
             printf "$warning Profiles do not match. You should manually migrate from %s to %s.\n" "$NIX_LINK" "$NIX_LINK_NEW" 1>&2
         fi
     fi
+    NIX_LINK="$NIX_LINK_NEW"
 fi
 
 export NIX_PROFILES="@localstatedir@/nix/profiles/default $NIX_LINK"

scripts/nix-profile.sh.in

@@ -1,3 +1,4 @@
+# This file is tested by tests/installer/default.nix.
 if [ -n "$HOME" ] && [ -n "$USER" ]; then
 
     # Set up the per-user profile.
@@ -9,11 +10,9 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then
         NIX_LINK_NEW="$HOME/.local/state/nix/profile"
     fi
     if [ -e "$NIX_LINK_NEW" ]; then
-        NIX_LINK="$NIX_LINK_NEW"
-    else
-        if [ -t 2 ] && [ -e "$NIX_LINK_NEW" ]; then
+        if [ -t 2 ] && [ -e "$NIX_LINK" ]; then
             warning="\033[1;35mwarning:\033[0m"
-            printf "$warning Both %s and legacy %s exist; using the latter.\n" "$NIX_LINK_NEW" "$NIX_LINK" 1>&2
+            printf "$warning Both %s and legacy %s exist; using the former.\n" "$NIX_LINK_NEW" "$NIX_LINK" 1>&2
             if [ "$(realpath "$NIX_LINK")" = "$(realpath "$NIX_LINK_NEW")" ]; then
                 printf "         Since the profiles match, you can safely delete either of them.\n" 1>&2
             else
@@ -26,6 +25,7 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then
                 printf "$warning Profiles do not match. You should manually migrate from %s to %s.\n" "$NIX_LINK" "$NIX_LINK_NEW" 1>&2
             fi
         fi
+        NIX_LINK="$NIX_LINK_NEW"
     fi
 
     # Set up environment.

src/libcmd/common-eval-args.cc

@@ -20,7 +20,7 @@ MixEvalArgs::MixEvalArgs()
         .description = "Pass the value *expr* as the argument *name* to Nix functions.",
         .category = category,
         .labels = {"name", "expr"},
-        .handler = {[&](std::string name, std::string expr) { autoArgs.insert_or_assign(name, AutoArg{AutoArgExpr(expr)}); }}
+        .handler = {[&](std::string name, std::string expr) { autoArgs.insert_or_assign(name, AutoArg{AutoArgExpr{expr}}); }}
     });
 
     addFlag({
@@ -28,7 +28,7 @@ MixEvalArgs::MixEvalArgs()
         .description = "Pass the string *string* as the argument *name* to Nix functions.",
         .category = category,
         .labels = {"name", "string"},
-        .handler = {[&](std::string name, std::string s) { autoArgs.insert_or_assign(name, AutoArg{AutoArgString(s)}); }},
+        .handler = {[&](std::string name, std::string s) { autoArgs.insert_or_assign(name, AutoArg{AutoArgString{s}}); }},
     });
 
     addFlag({
@@ -36,7 +36,7 @@ MixEvalArgs::MixEvalArgs()
         .description = "Pass the contents of file *path* as the argument *name* to Nix functions.",
         .category = category,
         .labels = {"name", "path"},
-        .handler = {[&](std::string name, std::string path) { autoArgs.insert_or_assign(name, AutoArg{AutoArgFile(path)}); }},
+        .handler = {[&](std::string name, std::string path) { autoArgs.insert_or_assign(name, AutoArg{AutoArgFile{path}}); }},
         .completer = completePath
     });
 

src/libcmd/installable-attr-path.cc

@@ -75,6 +75,8 @@ DerivedPathsWithInfo InstallableAttrPath::toDerivedPaths()
                 std::set<std::string> outputsToInstall;
                 for (auto & output : packageInfo.queryOutputs(false, true))
                     outputsToInstall.insert(output.first);
+                if (outputsToInstall.empty())
+                    outputsToInstall.insert("out");
                 return OutputsSpec::Names { std::move(outputsToInstall) };
             },
             [&](const ExtendedOutputsSpec::Explicit & e) -> OutputsSpec {

src/libcmd/installable-flake.cc

@@ -106,9 +106,14 @@ DerivedPathsWithInfo InstallableFlake::toDerivedPaths()
             fmt("while evaluating the flake output attribute '%s'", attrPath)))
         {
             return { *derivedPathWithInfo };
+        } else {
+            throw Error(
+                "expected flake output attribute '%s' to be a derivation or path but found %s: %s",
+                attrPath,
+                showType(v),
+                ValuePrinter(*this->state, v, errorPrintOptions)
+            );
         }
-        else
-            throw Error("flake output attribute '%s' is not a derivation or path", attrPath);
     }
 
     auto drvPath = attr->forceDerivation();

src/libcmd/repl.cc

@@ -137,12 +137,13 @@ void runNix(Path program, const Strings & args,
 {
     auto subprocessEnv = getEnv();
     subprocessEnv["NIX_CONFIG"] = globalConfig.toKeyValue();
-
+    //isInteractive avoid grabling interactive commands
     runProgram2(RunOptions {
         .program = settings.nixBinDir+ "/" + program,
         .args = args,
         .environment = subprocessEnv,
         .input = input,
+        .isInteractive = true,
     });
 
     return;
@@ -304,6 +305,8 @@ StringSet NixRepl::completePrefix(const std::string & prefix)
             // Quietly ignore evaluation errors.
         } catch (BadURL & e) {
             // Quietly ignore BadURL flake-related errors.
+        } catch (FileNotFound & e) {
+            // Quietly ignore non-existent file beeing `import`-ed.
         }
     }
 
@@ -508,13 +511,9 @@ ProcessLineResult NixRepl::processLine(std::string line)
         auto editor = args.front();
         args.pop_front();
 
-        // avoid garbling the editor with the progress bar
-        logger->pause();
-        Finally resume([&]() { logger->resume(); });
-
         // runProgram redirects stdout to a StringSink,
         // using runProgram2 to allow editors to display their UI
-        runProgram2(RunOptions { .program = editor, .lookupPath = true, .args = args });
+        runProgram2(RunOptions { .program = editor, .lookupPath = true, .args = args , .isInteractive = true });
 
         // Reload right after exiting the editor
         state->resetFileCache();

src/libexpr-c/nix_api_expr.cc

@@ -65,6 +65,17 @@ nix_err nix_value_call(nix_c_context * context, EvalState * state, Value * fn, V
     NIXC_CATCH_ERRS
 }
 
+nix_err nix_value_call_multi(nix_c_context * context, EvalState * state, Value * fn, size_t nargs, Value ** args, Value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        state->state.callFunction(*(nix::Value *) fn, nargs, (nix::Value * *)args, *(nix::Value *) value, nix::noPos);
+        state->state.forceValue(*(nix::Value *) value, nix::noPos);
+    }
+    NIXC_CATCH_ERRS
+}
+
 nix_err nix_value_force(nix_c_context * context, EvalState * state, Value * value)
 {
     if (context)

src/libexpr-c/nix_api_expr.h

@@ -12,6 +12,7 @@
 
 #include "nix_api_store.h"
 #include "nix_api_util.h"
+#include <stddef.h>
 
 #ifdef __cplusplus
 extern "C" {
@@ -80,6 +81,46 @@ nix_err nix_expr_eval_from_string(
  */
 nix_err nix_value_call(nix_c_context * context, EvalState * state, Value * fn, Value * arg, Value * value);
 
+/**
+ * @brief Calls a Nix function with multiple arguments.
+ *
+ * Technically these are functions that return functions. It is common for Nix
+ * functions to be curried, so this function is useful for calling them.
+ *
+ * @param[out] context Optional, stores error information
+ * @param[in] state The state of the evaluation.
+ * @param[in] fn The Nix function to call.
+ * @param[in] nargs The number of arguments.
+ * @param[in] args The arguments to pass to the function.
+ * @param[out] value The result of the function call.
+ *
+ * @see nix_value_call     For the single argument primitive.
+ * @see NIX_VALUE_CALL           For a macro that wraps this function for convenience.
+ */
+nix_err nix_value_call_multi(
+    nix_c_context * context, EvalState * state, Value * fn, size_t nargs, Value ** args, Value * value);
+
+/**
+ * @brief Calls a Nix function with multiple arguments.
+ *
+ * Technically these are functions that return functions. It is common for Nix
+ * functions to be curried, so this function is useful for calling them.
+ *
+ * @param[out] context Optional, stores error information
+ * @param[in] state The state of the evaluation.
+ * @param[out] value The result of the function call.
+ * @param[in] fn The Nix function to call.
+ * @param[in] args The arguments to pass to the function.
+ *
+ * @see nix_value_call_multi
+ */
+#define NIX_VALUE_CALL(context, state, value, fn, ...)                  \
+  do {                                                                  \
+    Value * args_array[] = {__VA_ARGS__};                               \
+    size_t nargs = sizeof(args_array) / sizeof(args_array[0]);          \
+    nix_value_call_multi(context, state, fn, nargs, args_array, value); \
+  } while (0)
+
 /**
  * @brief Forces the evaluation of a Nix value.
  *
@@ -88,10 +129,7 @@ nix_err nix_value_call(nix_c_context * context, EvalState * state, Value * fn, V
  *
  * This function converts these Values into their final type.
  *
- * @note You don't need this function for basic API usage, since all functions
- * that return a value call it for you. The only place you will see a
- * NIX_TYPE_THUNK is in the arguments that are passed to a PrimOp function
- * you supplied to nix_alloc_primop.
+ * @note This function is mainly needed before calling @ref getters, but not for API calls that return a `Value`.
  *
  * @param[out] context Optional, stores error information
  * @param[in] state The state of the evaluation.

src/libexpr-c/nix_api_value.cc

@@ -73,10 +73,43 @@ static void nix_c_primop_wrapper(
     PrimOpFun f, void * userdata, nix::EvalState & state, const nix::PosIdx pos, nix::Value ** args, nix::Value & v)
 {
     nix_c_context ctx;
-    f(userdata, &ctx, (EvalState *) &state, (Value **) args, (Value *) &v);
-    /* TODO: In the future, this should throw different errors depending on the error code */
-    if (ctx.last_err_code != NIX_OK)
-        state.error<nix::EvalError>("Error from builtin function: %s", *ctx.last_err).atPos(pos).debugThrow();
+
+    // v currently has a thunk, but the C API initializers require an uninitialized value.
+    //
+    // We can't destroy the thunk, because that makes it impossible to retry,
+    // which is needed for tryEval and for evaluation drivers that evaluate more
+    // than one value (e.g. an attrset with two derivations, both of which
+    // reference v).
+    //
+    // Instead we create a temporary value, and then assign the result to v.
+    // This does not give the primop definition access to the thunk, but that's
+    // ok because we don't see a need for this yet (e.g. inspecting thunks,
+    // or maybe something to make blackholes work better; we don't know).
+    nix::Value vTmp;
+
+    f(userdata, &ctx, (EvalState *) &state, (Value **) args, (Value *) &vTmp);
+
+    if (ctx.last_err_code != NIX_OK) {
+        /* TODO: Throw different errors depending on the error code */
+        state.error<nix::EvalError>("Error from custom function: %s", *ctx.last_err).atPos(pos).debugThrow();
+    }
+
+    if (!vTmp.isValid()) {
+        state.error<nix::EvalError>("Implementation error in custom function: return value was not initialized")
+            .atPos(pos)
+            .debugThrow();
+    }
+
+    if (vTmp.type() == nix::nThunk) {
+        // We might allow this in the future if it makes sense for the evaluator
+        // e.g. implementing tail recursion by returning a thunk to the next
+        // "iteration". Until then, this is most likely a mistake or misunderstanding.
+        state.error<nix::EvalError>("Implementation error in custom function: return value must not be a thunk")
+            .atPos(pos)
+            .debugThrow();
+    }
+
+    v = vTmp;
 }
 
 PrimOp * nix_alloc_primop(

src/libexpr-c/nix_api_value.h

@@ -79,6 +79,7 @@ typedef struct nix_realised_string nix_realised_string;
  * @{
  */
 /** @brief Function pointer for primops
+ *
  * When you want to return an error, call nix_set_err_msg(context, NIX_ERR_UNKNOWN, "your error message here").
  *
  * @param[in] user_data Arbitrary data that was initially supplied to nix_alloc_primop
@@ -147,7 +148,8 @@ Value * nix_alloc_value(nix_c_context * context, EvalState * state);
  * @brief Functions to inspect and change Nix language values, represented by Value.
  * @{
  */
-/** @name Getters
+/** @anchor getters
+ * @name Getters
  */
 /**@{*/
 /** @brief Get value type

src/libexpr/eval-cache.cc

@@ -7,6 +7,25 @@
 
 namespace nix::eval_cache {
 
+CachedEvalError::CachedEvalError(ref<AttrCursor> cursor, Symbol attr)
+    : EvalError(cursor->root->state, "cached failure of attribute '%s'", cursor->getAttrPathStr(attr))
+    , cursor(cursor), attr(attr)
+{ }
+
+void CachedEvalError::force()
+{
+    auto & v = cursor->forceValue();
+
+    if (v.type() == nAttrs) {
+        auto a = v.attrs()->get(this->attr);
+
+        state.forceValue(*a->value, a->pos);
+    }
+
+    // Shouldn't happen.
+    throw EvalError(state, "evaluation of cached failed attribute '%s' unexpectedly succeeded", cursor->getAttrPathStr(attr));
+}
+
 static const char * schema = R"sql(
 create table if not exists Attributes (
     parent      integer not null,
@@ -470,7 +489,7 @@ Suggestions AttrCursor::getSuggestionsForAttr(Symbol name)
     return Suggestions::bestMatches(strAttrNames, root->state.symbols[name]);
 }
 
-std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(Symbol name, bool forceErrors)
+std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(Symbol name)
 {
     if (root->db) {
         if (!cachedValue)
@@ -487,12 +506,9 @@ std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(Symbol name, bool forceErro
                 if (attr) {
                     if (std::get_if<missing_t>(&attr->second))
                         return nullptr;
-                    else if (std::get_if<failed_t>(&attr->second)) {
-                        if (forceErrors)
-                            debug("reevaluating failed cached attribute '%s'", getAttrPathStr(name));
-                        else
-                            throw CachedEvalError(root->state, "cached failure of attribute '%s'", getAttrPathStr(name));
-                    } else
+                    else if (std::get_if<failed_t>(&attr->second))
+                        throw CachedEvalError(ref(shared_from_this()), name);
+                    else
                         return std::make_shared<AttrCursor>(root,
                             std::make_pair(shared_from_this(), name), nullptr, std::move(attr));
                 }
@@ -537,9 +553,9 @@ std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(std::string_view name)
     return maybeGetAttr(root->state.symbols.create(name));
 }
 
-ref<AttrCursor> AttrCursor::getAttr(Symbol name, bool forceErrors)
+ref<AttrCursor> AttrCursor::getAttr(Symbol name)
 {
-    auto p = maybeGetAttr(name, forceErrors);
+    auto p = maybeGetAttr(name);
     if (!p)
         throw Error("attribute '%s' does not exist", getAttrPathStr(name));
     return ref(p);
@@ -550,11 +566,11 @@ ref<AttrCursor> AttrCursor::getAttr(std::string_view name)
     return getAttr(root->state.symbols.create(name));
 }
 
-OrSuggestions<ref<AttrCursor>> AttrCursor::findAlongAttrPath(const std::vector<Symbol> & attrPath, bool force)
+OrSuggestions<ref<AttrCursor>> AttrCursor::findAlongAttrPath(const std::vector<Symbol> & attrPath)
 {
     auto res = shared_from_this();
     for (auto & attr : attrPath) {
-        auto child = res->maybeGetAttr(attr, force);
+        auto child = res->maybeGetAttr(attr);
         if (!child) {
             auto suggestions = res->getSuggestionsForAttr(attr);
             return OrSuggestions<ref<AttrCursor>>::failed(suggestions);
@@ -751,8 +767,9 @@ bool AttrCursor::isDerivation()
 
 StorePath AttrCursor::forceDerivation()
 {
-    auto aDrvPath = getAttr(root->state.sDrvPath, true);
+    auto aDrvPath = getAttr(root->state.sDrvPath);
     auto drvPath = root->state.store->parseStorePath(aDrvPath->getString());
+    drvPath.requireDerivation();
     if (!root->state.store->isValidPath(drvPath) && !settings.readOnlyMode) {
         /* The eval cache contains 'drvPath', but the actual path has
            been garbage-collected. So force it to be regenerated. */

src/libexpr/eval-cache.hh

@@ -10,14 +10,28 @@
 
 namespace nix::eval_cache {
 
-MakeError(CachedEvalError, EvalError);
-
 struct AttrDb;
 class AttrCursor;
 
+struct CachedEvalError : EvalError
+{
+    const ref<AttrCursor> cursor;
+    const Symbol attr;
+
+    CachedEvalError(ref<AttrCursor> cursor, Symbol attr);
+
+    /**
+     * Evaluate this attribute, which should result in a regular
+     * `EvalError` exception being thrown.
+     */
+    [[noreturn]]
+    void force();
+};
+
 class EvalCache : public std::enable_shared_from_this<EvalCache>
 {
     friend class AttrCursor;
+    friend class CachedEvalError;
 
     std::shared_ptr<AttrDb> db;
     EvalState & state;
@@ -73,6 +87,7 @@ typedef std::variant<
 class AttrCursor : public std::enable_shared_from_this<AttrCursor>
 {
     friend class EvalCache;
+    friend class CachedEvalError;
 
     ref<EvalCache> root;
     typedef std::optional<std::pair<std::shared_ptr<AttrCursor>, Symbol>> Parent;
@@ -102,11 +117,11 @@ public:
 
     Suggestions getSuggestionsForAttr(Symbol name);
 
-    std::shared_ptr<AttrCursor> maybeGetAttr(Symbol name, bool forceErrors = false);
+    std::shared_ptr<AttrCursor> maybeGetAttr(Symbol name);
 
     std::shared_ptr<AttrCursor> maybeGetAttr(std::string_view name);
 
-    ref<AttrCursor> getAttr(Symbol name, bool forceErrors = false);
+    ref<AttrCursor> getAttr(Symbol name);
 
     ref<AttrCursor> getAttr(std::string_view name);
 
@@ -114,7 +129,7 @@ public:
      * Get an attribute along a chain of attrsets. Note that this does
      * not auto-call functors or functions.
      */
-    OrSuggestions<ref<AttrCursor>> findAlongAttrPath(const std::vector<Symbol> & attrPath, bool force = false);
+    OrSuggestions<ref<AttrCursor>> findAlongAttrPath(const std::vector<Symbol> & attrPath);
 
     std::string getString();
 

src/libexpr/eval-error.cc

@@ -27,8 +27,7 @@ EvalErrorBuilder<T> & EvalErrorBuilder<T>::atPos(Value & value, PosIdx fallback)
 template<class T>
 EvalErrorBuilder<T> & EvalErrorBuilder<T>::withTrace(PosIdx pos, const std::string_view text)
 {
-    error.err.traces.push_front(
-        Trace{.pos = error.state.positions[pos], .hint = HintFmt(std::string(text))});
+    error.addTrace(error.state.positions[pos], text);
     return *this;
 }
 
@@ -71,15 +70,17 @@ EvalErrorBuilder<T>::addTrace(PosIdx pos, std::string_view formatString, const A
     return *this;
 }
 
+template<class T>
+EvalErrorBuilder<T> & EvalErrorBuilder<T>::setIsFromExpr()
+{
+    error.err.isFromExpr = true;
+    return *this;
+}
+
 template<class T>
 void EvalErrorBuilder<T>::debugThrow()
 {
-    if (error.state.debugRepl && !error.state.debugTraces.empty()) {
-        const DebugTrace & last = error.state.debugTraces.front();
-        const Env * env = &last.env;
-        const Expr * expr = &last.expr;
-        error.state.runDebugRepl(&error, *env, *expr);
-    }
+    error.state.runDebugRepl(&error);
 
     // `EvalState` is the only class that can construct an `EvalErrorBuilder`,
     // and it does so in dynamic storage. This is the final method called on
@@ -91,6 +92,7 @@ void EvalErrorBuilder<T>::debugThrow()
     throw error;
 }
 
+template class EvalErrorBuilder<EvalBaseError>;
 template class EvalErrorBuilder<EvalError>;
 template class EvalErrorBuilder<AssertionError>;
 template class EvalErrorBuilder<ThrownError>;
@@ -99,7 +101,6 @@ template class EvalErrorBuilder<TypeError>;
 template class EvalErrorBuilder<UndefinedVarError>;
 template class EvalErrorBuilder<MissingArgumentError>;
 template class EvalErrorBuilder<InfiniteRecursionError>;
-template class EvalErrorBuilder<CachedEvalError>;
 template class EvalErrorBuilder<InvalidPathError>;
 
 }

src/libexpr/eval-error.hh

@@ -15,27 +15,39 @@ class EvalState;
 template<class T>
 class EvalErrorBuilder;
 
-class EvalError : public Error
+/**
+ * Base class for all errors that occur during evaluation.
+ *
+ * Most subclasses should inherit from `EvalError` instead of this class.
+ */
+class EvalBaseError : public Error
 {
     template<class T>
     friend class EvalErrorBuilder;
 public:
     EvalState & state;
 
-    EvalError(EvalState & state, ErrorInfo && errorInfo)
+    EvalBaseError(EvalState & state, ErrorInfo && errorInfo)
         : Error(errorInfo)
         , state(state)
     {
     }
 
     template<typename... Args>
-    explicit EvalError(EvalState & state, const std::string & formatString, const Args &... formatArgs)
+    explicit EvalBaseError(EvalState & state, const std::string & formatString, const Args &... formatArgs)
         : Error(formatString, formatArgs...)
         , state(state)
     {
     }
 };
 
+/**
+ * `EvalError` is the base class for almost all errors that occur during evaluation.
+ *
+ * All instances of `EvalError` should show a degree of purity that allows them to be
+ * cached in pure mode. This means that they should not depend on the configuration or the overall environment.
+ */
+MakeError(EvalError, EvalBaseError);
 MakeError(ParseError, Error);
 MakeError(AssertionError, EvalError);
 MakeError(ThrownError, AssertionError);
@@ -43,7 +55,6 @@ MakeError(Abort, EvalError);
 MakeError(TypeError, EvalError);
 MakeError(UndefinedVarError, EvalError);
 MakeError(MissingArgumentError, EvalError);
-MakeError(CachedEvalError, EvalError);
 MakeError(InfiniteRecursionError, EvalError);
 
 struct InvalidPathError : public EvalError
@@ -91,6 +102,8 @@ public:
 
     [[nodiscard, gnu::noinline]] EvalErrorBuilder<T> & addTrace(PosIdx pos, HintFmt hint);
 
+    [[nodiscard, gnu::noinline]] EvalErrorBuilder<T> & setIsFromExpr();
+
     template<typename... Args>
     [[nodiscard, gnu::noinline]] EvalErrorBuilder<T> &
     addTrace(PosIdx pos, std::string_view formatString, const Args &... formatArgs);

src/libexpr/eval-settings.cc

@@ -48,6 +48,10 @@ EvalSettings::EvalSettings()
 {
     auto var = getEnv("NIX_PATH");
     if (var) nixPath = parseNixPath(*var);
+
+    var = getEnv("NIX_ABORT_ON_WARN");
+    if (var && (var == "1" || var == "yes" || var == "true"))
+        builtinsAbortOnWarn = true;
 }
 
 Strings EvalSettings::getDefaultNixPath()

src/libexpr/eval-settings.hh

@@ -15,8 +15,24 @@ struct EvalSettings : Config
 
     static std::string resolvePseudoUrl(std::string_view url);
 
-    Setting<bool> enableNativeCode{this, false, "allow-unsafe-native-code-during-evaluation",
-        "Whether builtin functions that allow executing native code should be enabled."};
+    Setting<bool> enableNativeCode{this, false, "allow-unsafe-native-code-during-evaluation", R"(
+        Enable built-in functions that allow executing native code.
+
+        In particular, this adds:
+        - `builtins.importNative` *path*
+        
+          Load a dynamic shared object (DSO) at *path* which exposes a function pointer to a procedure that initialises a Nix language value, and return that value.
+          The procedure must have the following signature:
+          ```cpp
+          extern "C" typedef void (*ValueInitialiser) (EvalState & state, Value & v);
+          ``` 
+          
+          The [Nix C++ API documentation](@docroot@/contributing/documentation.md#api-documentation) has more details on evaluator internals.
+          
+        - `builtins.exec` *arguments*
+        
+          Execute a program, where *arguments* are specified as a list of strings, and parse its output as a Nix expression.
+    )"};
 
     Setting<Strings> nixPath{
         this, getDefaultNixPath(), "nix-path",
@@ -142,13 +158,39 @@ struct EvalSettings : Config
 
     Setting<bool> builtinsTraceDebugger{this, false, "debugger-on-trace",
         R"(
-          If set to true and the `--debugger` flag is given,
-          [`builtins.trace`](@docroot@/language/builtins.md#builtins-trace) will
-          enter the debugger like
-          [`builtins.break`](@docroot@/language/builtins.md#builtins-break).
+          If set to true and the `--debugger` flag is given, the following functions
+          will enter the debugger like [`builtins.break`](@docroot@/language/builtins.md#builtins-break).
+
+          * [`builtins.trace`](@docroot@/language/builtins.md#builtins-trace)
+          * [`builtins.traceVerbose`](@docroot@/language/builtins.md#builtins-traceVerbose)
+            if [`trace-verbose`](#conf-trace-verbose) is set to true.
+          * [`builtins.warn`](@docroot@/language/builtins.md#builtins-warn)
 
           This is useful for debugging warnings in third-party Nix code.
         )"};
+
+    Setting<bool> builtinsDebuggerOnWarn{this, false, "debugger-on-warn",
+        R"(
+          If set to true and the `--debugger` flag is given, [`builtins.warn`](@docroot@/language/builtins.md#builtins-warn)
+          will enter the debugger like [`builtins.break`](@docroot@/language/builtins.md#builtins-break).
+
+          This is useful for debugging warnings in third-party Nix code.
+
+          Use [`debugger-on-trace`](#conf-debugger-on-trace) to also enter the debugger on legacy warnings that are logged with [`builtins.trace`](@docroot@/language/builtins.md#builtins-trace).
+        )"};
+
+    Setting<bool> builtinsAbortOnWarn{this, false, "abort-on-warn",
+        R"(
+          If set to true, [`builtins.warn`](@docroot@/language/builtins.md#builtins-warn) will throw an error when logging a warning.
+
+          This will give you a stack trace that leads to the location of the warning.
+
+          This is useful for finding information about warnings in third-party Nix code when you can not start the interactive debugger, such as when Nix is called from a non-interactive script. See [`debugger-on-warn`](#conf-debugger-on-warn).
+
+          Currently, a stack trace can only be produced when the debugger is enabled, or when evaluation is aborted.
+
+          This option can be enabled by setting `NIX_ABORT_ON_WARN=1` in the environment.
+        )"};
 };
 
 extern EvalSettings evalSettings;

src/libexpr/eval.cc

@@ -28,13 +28,13 @@
 #include <algorithm>
 #include <chrono>
 #include <iostream>
+#include <sstream>
 #include <cstring>
 #include <optional>
 #include <unistd.h>
 #include <sys/time.h>
 #include <fstream>
 #include <functional>
-#include <iostream>
 
 #include <nlohmann/json.hpp>
 #include <boost/container/small_vector.hpp>
@@ -785,6 +785,24 @@ public:
     }
 };
 
+bool EvalState::canDebug()
+{
+    return debugRepl && !debugTraces.empty();
+}
+
+void EvalState::runDebugRepl(const Error * error)
+{
+    if (!canDebug())
+        return;
+
+    assert(!debugTraces.empty());
+    const DebugTrace & last = debugTraces.front();
+    const Env & env = last.env;
+    const Expr & expr = last.expr;
+
+    runDebugRepl(error, env, expr);
+}
+
 void EvalState::runDebugRepl(const Error * error, const Env & env, const Expr & expr)
 {
     // Make sure we have a debugger to run and we're not already in a debugger.

src/libexpr/eval.hh

@@ -276,6 +276,18 @@ public:
             return std::shared_ptr<const StaticEnv>();;
     }
 
+    /** Whether a debug repl can be started. If `false`, `runDebugRepl(error)` will return without starting a repl. */
+    bool canDebug();
+
+    /** Use front of `debugTraces`; see `runDebugRepl(error,env,expr)` */
+    void runDebugRepl(const Error * error);
+
+    /**
+     * Run a debug repl with the given error, environment and expression.
+     * @param error The error to debug, may be nullptr.
+     * @param env The environment to debug, matching the expression.
+     * @param expr The expression to debug, matching the environment.
+     */
     void runDebugRepl(const Error * error, const Env & env, const Expr & expr);
 
     template<class T, typename... Args>

src/libexpr/flake/flake.cc

@@ -949,10 +949,20 @@ std::optional<Fingerprint> LockedFlake::getFingerprint(ref<Store> store) const
     auto fingerprint = flake.lockedRef.input.getFingerprint(store);
     if (!fingerprint) return std::nullopt;
 
+    *fingerprint += fmt(";%s;%s", flake.lockedRef.subdir, lockFile);
+
+    /* Include revCount and lastModified because they're not
+       necessarily implied by the content fingerprint (e.g. for
+       tarball flakes) but can influence the evaluation result. */
+    if (auto revCount = flake.lockedRef.input.getRevCount())
+        *fingerprint += fmt(";revCount=%d", *revCount);
+    if (auto lastModified = flake.lockedRef.input.getLastModified())
+        *fingerprint += fmt(";lastModified=%d", *lastModified);
+
     // FIXME: as an optimization, if the flake contains a lock file
     // and we haven't changed it, then it's sufficient to use
     // flake.sourceInfo.storePath for the fingerprint.
-    return hashString(HashAlgorithm::SHA256, fmt("%s;%s;%s", *fingerprint, flake.lockedRef.subdir, lockFile));
+    return hashString(HashAlgorithm::SHA256, *fingerprint);
 }
 
 Flake::~Flake() { }

src/libexpr/get-drvs.cc

@@ -69,10 +69,17 @@ std::string PackageInfo::querySystem() const
 std::optional<StorePath> PackageInfo::queryDrvPath() const
 {
     if (!drvPath && attrs) {
-        NixStringContext context;
-        if (auto i = attrs->get(state->sDrvPath))
-            drvPath = {state->coerceToStorePath(i->pos, *i->value, context, "while evaluating the 'drvPath' attribute of a derivation")};
-        else
+        if (auto i = attrs->get(state->sDrvPath)) {
+            NixStringContext context;
+            auto found = state->coerceToStorePath(i->pos, *i->value, context, "while evaluating the 'drvPath' attribute of a derivation");
+            try {
+                found.requireDerivation();
+            } catch (Error & e) {
+                e.addTrace(state->positions[i->pos], "while evaluating the 'drvPath' attribute of a derivation");
+                throw;
+            }
+            drvPath = {std::move(found)};
+        } else
             drvPath = {std::nullopt};
     }
     return drvPath.value_or(std::nullopt);

src/libexpr/lexer.l

@@ -20,8 +20,6 @@
 #pragma clang diagnostic ignored "-Wunneeded-internal-declaration"
 #endif
 
-#include <boost/lexical_cast.hpp>
-
 #include "nixexpr.hh"
 #include "parser-tab.hh"
 
@@ -129,9 +127,10 @@ or          { return OR_KW; }
 
 {ID}        { yylval->id = {yytext, (size_t) yyleng}; return ID; }
 {INT}       { errno = 0;
-              try {
-                  yylval->n = boost::lexical_cast<int64_t>(yytext);
-              } catch (const boost::bad_lexical_cast &) {
+              std::optional<int64_t> numMay = string2Int<int64_t>(yytext);
+              if (numMay.has_value()) {
+                  yylval->n = *numMay;
+              } else {
                   throw ParseError(ErrorInfo{
                       .msg = HintFmt("invalid integer '%1%'", yytext),
                       .pos = state->positions[CUR_POS],

src/libexpr/nixexpr.cc

@@ -6,6 +6,7 @@
 #include "print.hh"
 
 #include <cstdlib>
+#include <sstream>
 
 namespace nix {
 

src/libexpr/primops.cc

@@ -26,6 +26,7 @@
 
 #include <algorithm>
 #include <cstring>
+#include <sstream>
 #include <regex>
 
 #ifndef _WIN32
@@ -78,7 +79,7 @@ StringMap EvalState::realiseContext(const NixStringContext & context, StorePathS
     if (drvs.empty()) return {};
 
     if (isIFD && !evalSettings.enableImportFromDerivation)
-        error<EvalError>(
+        error<EvalBaseError>(
             "cannot build '%1%' during evaluation because the option 'allow-import-from-derivation' is disabled",
             drvs.begin()->to_string(*store)
         ).debugThrow();
@@ -779,15 +780,14 @@ static RegisterPrimOp primop_break({
     )",
     .fun = [](EvalState & state, const PosIdx pos, Value * * args, Value & v)
     {
-        if (state.debugRepl && !state.debugTraces.empty()) {
+        if (state.canDebug()) {
             auto error = Error(ErrorInfo {
                 .level = lvlInfo,
                 .msg = HintFmt("breakpoint reached"),
                 .pos = state.positions[pos],
             });
 
-            auto & dt = state.debugTraces.front();
-            state.runDebugRepl(&error, dt.env, dt.expr);
+            state.runDebugRepl(&error);
         }
 
         // Return the value we were passed.
@@ -806,7 +806,7 @@ static RegisterPrimOp primop_abort({
         NixStringContext context;
         auto s = state.coerceToString(pos, *args[0], context,
                 "while evaluating the error message passed to builtins.abort").toOwned();
-        state.error<Abort>("evaluation aborted with the following error message: '%1%'", s).debugThrow();
+        state.error<Abort>("evaluation aborted with the following error message: '%1%'", s).setIsFromExpr().debugThrow();
     }
 });
 
@@ -825,7 +825,7 @@ static RegisterPrimOp primop_throw({
       NixStringContext context;
       auto s = state.coerceToString(pos, *args[0], context,
               "while evaluating the error message passed to builtin.throw").toOwned();
-      state.error<ThrownError>(s).debugThrow();
+      state.error<ThrownError>(s).setIsFromExpr().debugThrow();
     }
 });
 
@@ -1017,9 +1017,8 @@ static void prim_trace(EvalState & state, const PosIdx pos, Value * * args, Valu
         printError("trace: %1%", args[0]->string_view());
     else
         printError("trace: %1%", ValuePrinter(state, *args[0]));
-    if (evalSettings.builtinsTraceDebugger && state.debugRepl && !state.debugTraces.empty()) {
-        const DebugTrace & last = state.debugTraces.front();
-        state.runDebugRepl(nullptr, last.env, last.expr);
+    if (evalSettings.builtinsTraceDebugger) {
+        state.runDebugRepl(nullptr);
     }
     state.forceValue(*args[1], pos);
     v = *args[1];
@@ -1042,6 +1041,55 @@ static RegisterPrimOp primop_trace({
     .fun = prim_trace,
 });
 
+static void prim_warn(EvalState & state, const PosIdx pos, Value * * args, Value & v)
+{
+    // We only accept a string argument for now. The use case for pretty printing a value is covered by `trace`.
+    // By rejecting non-strings we allow future versions to add more features without breaking existing code.
+    auto msgStr = state.forceString(*args[0], pos, "while evaluating the first argument; the message passed to builtins.warn");
+
+    {
+        BaseError msg(std::string{msgStr});
+        msg.atPos(state.positions[pos]);
+        auto info = msg.info();
+        info.level = lvlWarn;
+        info.isFromExpr = true;
+        logWarning(info);
+    }
+
+    if (evalSettings.builtinsAbortOnWarn) {
+        // Not an EvalError or subclass, which would cause the error to be stored in the eval cache.
+        state.error<EvalBaseError>("aborting to reveal stack trace of warning, as abort-on-warn is set").setIsFromExpr().debugThrow();
+    }
+    if (evalSettings.builtinsTraceDebugger || evalSettings.builtinsDebuggerOnWarn) {
+        state.runDebugRepl(nullptr);
+    }
+    state.forceValue(*args[1], pos);
+    v = *args[1];
+}
+
+static RegisterPrimOp primop_warn({
+    .name = "__warn",
+    .args = {"e1", "e2"},
+    .doc = R"(
+      Evaluate *e1*, which must be a string and print iton standard error as a warning.
+      Then return *e2*.
+      This function is useful for non-critical situations where attention is advisable.
+
+      If the
+      [`debugger-on-trace`](@docroot@/command-ref/conf-file.md#conf-debugger-on-trace)
+      or [`debugger-on-warn`](@docroot@/command-ref/conf-file.md#conf-debugger-on-warn)
+      option is set to `true` and the `--debugger` flag is given, the
+      interactive debugger will be started when `warn` is called (like
+      [`break`](@docroot@/language/builtins.md#builtins-break)).
+
+      If the
+      [`abort-on-warn`](@docroot@/command-ref/conf-file.md#conf-abort-on-warn)
+      option is set, the evaluation will be aborted after the warning is printed.
+      This is useful to reveal the stack trace of the warning, when the context is non-interactive and a debugger can not be launched.
+    )",
+    .fun = prim_warn,
+});
+
 
 /* Takes two arguments and evaluates to the second one. Used as the
  * builtins.traceVerbose implementation when --trace-verbose is not enabled

src/libexpr/primops/fetchTree.cc

@@ -137,6 +137,11 @@ static void fetchTree(
             attrs.emplace("exportIgnore", Explicit<bool>{true});
         }
 
+        // fetchTree should fetch git repos with shallow = true by default
+        if (type == "git" && !params.isFetchGit && !attrs.contains("shallow")) {
+            attrs.emplace("shallow", Explicit<bool>{true});
+        }
+
         if (!params.allowNameArgument)
             if (auto nameIter = attrs.find("name"); nameIter != attrs.end())
                 state.error<EvalError>(
@@ -320,6 +325,8 @@ static RegisterPrimOp primop_fetchTree({
 
         - `ref` (String, optional)
 
+          By default, this has no effect. This becomes relevant only once `shallow` cloning is disabled.
+
           A [Git reference](https://git-scm.com/book/en/v2/Git-Internals-Git-References), such as a branch or tag name.
 
           Default: `"HEAD"`
@@ -333,8 +340,9 @@ static RegisterPrimOp primop_fetchTree({
         - `shallow` (Bool, optional)
 
           Make a shallow clone when fetching the Git tree.
+          When this is enabled, the options `ref` and `allRefs` have no effect anymore.
 
-          Default: `false`
+          Default: `true`
 
         - `submodules` (Bool, optional)
 
@@ -344,8 +352,11 @@ static RegisterPrimOp primop_fetchTree({
 
         - `allRefs` (Bool, optional)
 
-          If set to `true`, always fetch the entire repository, even if the latest commit is still in the cache.
-          Otherwise, only the latest commit is fetched if it is not already cached.
+          By default, this has no effect. This becomes relevant only once `shallow` cloning is disabled.
+
+          Whether to fetch all references (eg. branches and tags) of the repository.
+          With this argument being true, it's possible to load a `rev` from *any* `ref`.
+          (Without setting this option, only `rev`s from the specified `ref` are supported).
 
           Default: `false`
 
@@ -599,6 +610,8 @@ static RegisterPrimOp primop_fetchGit({
 
         [Git reference]: https://git-scm.com/book/en/v2/Git-Internals-Git-References
 
+        This option has no effect once `shallow` cloning is enabled.
+
         By default, the `ref` value is prefixed with `refs/heads/`.
         As of 2.3.0, Nix will not prefix `refs/heads/` if `ref` starts with `refs/`.
 
@@ -616,13 +629,15 @@ static RegisterPrimOp primop_fetchGit({
       - `shallow` (default: `false`)
 
         Make a shallow clone when fetching the Git tree.
-
+        When this is enabled, the options `ref` and `allRefs` have no effect anymore.
       - `allRefs`
 
-        Whether to fetch all references of the repository.
-        With this argument being true, it's possible to load a `rev` from *any* `ref`
+        Whether to fetch all references (eg. branches and tags) of the repository.
+        With this argument being true, it's possible to load a `rev` from *any* `ref`.
         (by default only `rev`s from the specified `ref` are supported).
 
+        This option has no effect once `shallow` cloning is enabled.
+
       - `verifyCommit` (default: `true` if `publicKey` or `publicKeys` are provided, otherwise `false`)
 
         Whether to check `rev` for a signature matching `publicKey` or `publicKeys`.

src/libexpr/print.cc

@@ -1,5 +1,6 @@
 #include <limits>
 #include <unordered_set>
+#include <sstream>
 
 #include "print.hh"
 #include "ansicolor.hh"
@@ -271,16 +272,27 @@ private:
 
     void printDerivation(Value & v)
     {
-        NixStringContext context;
-        std::string storePath;
-        if (auto i = v.attrs()->get(state.sDrvPath))
-            storePath = state.store->printStorePath(state.coerceToStorePath(i->pos, *i->value, context, "while evaluating the drvPath of a derivation"));
+        std::optional<StorePath> storePath;
+        if (auto i = v.attrs()->get(state.sDrvPath)) {
+            NixStringContext context;
+            storePath = state.coerceToStorePath(i->pos, *i->value, context, "while evaluating the drvPath of a derivation");
+        }
+
+        /* This unfortunately breaks printing nested values because of
+           how the pretty printer is used (when pretting printing and warning
+           to same terminal / std stream). */
+#if 0
+        if (storePath && !storePath->isDerivation())
+            warn(
+                "drvPath attribute '%s' is not a valid store path to a derivation, this value not work properly",
+                state.store->printStorePath(*storePath));
+#endif
 
         if (options.ansiColors)
             output << ANSI_GREEN;
         output << "«derivation";
-        if (!storePath.empty()) {
-            output << " " << storePath;
+        if (storePath) {
+            output << " " << state.store->printStorePath(*storePath);
         }
         output << "»";
         if (options.ansiColors)

src/libfetchers/fetchers.cc

@@ -260,6 +260,7 @@ std::pair<ref<SourceAccessor>, Input> Input::getAccessorUnchecked(ref<Store> sto
 
     auto [accessor, final] = scheme->getAccessor(store, *this);
 
+    assert(!accessor->fingerprint);
     accessor->fingerprint = scheme->getFingerprint(store, final);
 
     return {accessor, std::move(final)};
@@ -418,7 +419,7 @@ namespace nlohmann {
 using namespace nix;
 
 fetchers::PublicKey adl_serializer<fetchers::PublicKey>::from_json(const json & json) {
-    fetchers::PublicKey res = {  };
+    fetchers::PublicKey res = { };
     if (auto type = optionalValueAt(json, "type"))
         res.type = getString(*type);
 

src/libfetchers/filtering-source-accessor.cc

@@ -2,6 +2,12 @@
 
 namespace nix {
 
+std::optional<std::filesystem::path> FilteringSourceAccessor::getPhysicalPath(const CanonPath & path)
+{
+    checkAccess(path);
+    return next->getPhysicalPath(prefix / path);
+}
+
 std::string FilteringSourceAccessor::readFile(const CanonPath & path)
 {
     checkAccess(path);

src/libfetchers/filtering-source-accessor.hh

@@ -30,6 +30,8 @@ struct FilteringSourceAccessor : SourceAccessor
         displayPrefix.clear();
     }
 
+    std::optional<std::filesystem::path> getPhysicalPath(const CanonPath & path) override;
+
     std::string readFile(const CanonPath & path) override;
 
     bool pathExists(const CanonPath & path) override;

src/libfetchers/git.cc

@@ -19,7 +19,10 @@
 #include <regex>
 #include <string.h>
 #include <sys/time.h>
-#include <sys/wait.h>
+
+#ifndef _WIN32
+#  include <sys/wait.h>
+#endif
 
 using namespace std::string_literals;
 
@@ -40,6 +43,7 @@ bool isCacheFileWithinTtl(time_t now, const struct stat & st)
 
 bool touchCacheFile(const Path & path, time_t touch_time)
 {
+#ifndef _WIN32 // TODO implement
     struct timeval times[2];
     times[0].tv_sec = touch_time;
     times[0].tv_usec = 0;
@@ -47,6 +51,9 @@ bool touchCacheFile(const Path & path, time_t touch_time)
     times[1].tv_usec = 0;
 
     return lutimes(path.c_str(), times) == 0;
+#else
+    return false;
+#endif
 }
 
 Path getCachePath(std::string_view key, bool shallow)
@@ -98,7 +105,15 @@ bool storeCachedHead(const std::string & actualUrl, const std::string & headRef)
     try {
         runProgram("git", true, { "-C", cacheDir, "--git-dir", ".", "symbolic-ref", "--", "HEAD", headRef });
     } catch (ExecError &e) {
-        if (!WIFEXITED(e.status)) throw;
+        if (
+#ifndef WIN32 // TODO abstract over exit status handling on Windows
+            !WIFEXITED(e.status)
+#else
+            e.status != 0
+#endif
+            )
+            throw;
+
         return false;
     }
     /* No need to touch refs/HEAD, because `git symbolic-ref` updates the mtime. */
@@ -329,7 +344,13 @@ struct GitInputScheme : InputScheme
             .program = "git",
             .args = {"-C", repoInfo.url, "--git-dir", repoInfo.gitDir, "check-ignore", "--quiet", std::string(path.rel())},
         });
-        auto exitCode = WEXITSTATUS(result.first);
+        auto exitCode =
+#ifndef WIN32 // TODO abstract over exit status handling on Windows
+            WEXITSTATUS(result.first)
+#else
+            result.first
+#endif
+            ;
 
         if (exitCode != 0) {
             // The path is not `.gitignore`d, we can add the file.

src/libfetchers/github.cc

@@ -433,9 +433,15 @@ struct GitLabInputScheme : GitArchiveInputScheme
                 store->toRealPath(
                     downloadFile(store, url, "source", headers).storePath)));
 
-        return RefInfo {
-            .rev = Hash::parseAny(std::string(json[0]["id"]), HashAlgorithm::SHA1)
-        };
+        if (json.is_array() && json.size() >= 1 && json[0]["id"] != nullptr) {
+          return RefInfo {
+              .rev = Hash::parseAny(std::string(json[0]["id"]), HashAlgorithm::SHA1)
+          };
+        } if (json.is_array() && json.size() == 0) {
+            throw Error("No commits returned by GitLab API -- does the git ref really exist?");
+        } else {
+            throw Error("Unexpected response received from GitLab: %s", json);
+        }
     }
 
     DownloadUrl getDownloadUrl(const Input & input) const override

src/libfetchers/local.mk

@@ -5,16 +5,10 @@ libfetchers_NAME = libnixfetchers
 libfetchers_DIR := $(d)
 
 libfetchers_SOURCES := $(wildcard $(d)/*.cc)
-ifdef HOST_UNIX
-  libfetchers_SOURCES += $(wildcard $(d)/unix/*.cc)
-endif
 
 # Not just for this library itself, but also for downstream libraries using this library
 
 INCLUDE_libfetchers := -I $(d)
-ifdef HOST_UNIX
-  INCLUDE_libfetchers += -I $(d)/unix
-endif
 
 libfetchers_CXXFLAGS += $(INCLUDE_libutil) $(INCLUDE_libstore) $(INCLUDE_libfetchers)
 

src/libfetchers/tarball.cc

@@ -365,6 +365,16 @@ struct TarballInputScheme : CurlInputScheme
 
         return {result.accessor, input};
     }
+
+    std::optional<std::string> getFingerprint(ref<Store> store, const Input & input) const override
+    {
+        if (auto narHash = input.getNarHash())
+            return narHash->to_string(HashFormat::SRI, true);
+        else if (auto rev = input.getRev())
+            return rev->gitRev();
+        else
+            return std::nullopt;
+    }
 };
 
 static auto rTarballInputScheme = OnStartup([] { registerInputScheme(std::make_unique<TarballInputScheme>()); });

src/libmain/progress-bar.cc

@@ -7,6 +7,7 @@
 #include <atomic>
 #include <map>
 #include <thread>
+#include <sstream>
 #include <iostream>
 #include <chrono>
 

src/libstore-c/nix_api_store.h

@@ -54,8 +54,10 @@ nix_err nix_libstore_init_no_load_config(nix_c_context * context);
 nix_err nix_init_plugins(nix_c_context * context);
 
 /**
- * @brief Open a nix store
+ * @brief Open a nix store.
+ *
  * Store instances may share state and resources behind the scenes.
+ *
  * @param[out] context Optional, stores error information
  * @param[in] uri URI of the Nix store, copied. See [*Store URL format* in the Nix Reference
  * Manual](https://nixos.org/manual/nix/stable/store/types/#store-url-format).
@@ -157,7 +159,9 @@ nix_err nix_store_realise(
 
 /**
  * @brief get the version of a nix store.
+ *
  * If the store doesn't have a version (like the dummy store), returns an empty string.
+ *
  * @param[out] context Optional, stores error information
  * @param[in] store nix store reference
  * @param[in] callback Called with the version.

src/libstore/binary-cache-store.cc

@@ -18,6 +18,7 @@
 #include <future>
 #include <regex>
 #include <fstream>
+#include <sstream>
 
 #include <nlohmann/json.hpp>
 
@@ -453,7 +454,7 @@ StorePath BinaryCacheStore::addToStore(
        non-recursive+sha256 so we can just use the default
        implementation of this method in terms of addToStoreFromDump. */
 
-    auto h = hashPath(path, method.getFileIngestionMethod(), hashAlgo, filter);
+    auto h = hashPath(path, method.getFileIngestionMethod(), hashAlgo, filter).first;
 
     auto source = sinkToSource([&](Sink & sink) {
         path.dumpPath(sink, filter);

src/libstore/build/derivation-goal.cc

@@ -1,5 +1,8 @@
 #include "derivation-goal.hh"
-#include "hook-instance.hh"
+#ifndef _WIN32 // TODO enable build hook on Windows
+#  include "hook-instance.hh"
+#endif
+#include "processes.hh"
 #include "worker.hh"
 #include "builtins.hh"
 #include "builtins/buildenv.hh"
@@ -19,19 +22,8 @@
 
 #include <fstream>
 #include <sys/types.h>
-#include <sys/socket.h>
-#include <sys/un.h>
-#include <sys/wait.h>
-#include <netdb.h>
 #include <fcntl.h>
-#include <termios.h>
 #include <unistd.h>
-#include <sys/mman.h>
-#include <sys/utsname.h>
-#include <sys/resource.h>
-
-#include <pwd.h>
-#include <grp.h>
 
 #include <nlohmann/json.hpp>
 
@@ -101,7 +93,9 @@ std::string DerivationGoal::key()
 
 void DerivationGoal::killChild()
 {
+#ifndef _WIN32 // TODO enable build hook on Windows
     hook.reset();
+#endif
 }
 
 
@@ -641,9 +635,17 @@ void DerivationGoal::started()
         buildMode == bmCheck ? "checking outputs of '%s'" :
         "building '%s'", worker.store.printStorePath(drvPath));
     fmt("building '%s'", worker.store.printStorePath(drvPath));
+#ifndef _WIN32 // TODO enable build hook on Windows
     if (hook) msg += fmt(" on '%s'", machineName);
+#endif
     act = std::make_unique<Activity>(*logger, lvlInfo, actBuild, msg,
-        Logger::Fields{worker.store.printStorePath(drvPath), hook ? machineName : "", 1, 1});
+        Logger::Fields{worker.store.printStorePath(drvPath),
+#ifndef _WIN32 // TODO enable build hook on Windows
+        hook ? machineName :
+#endif
+            "",
+        1,
+        1});
     mcRunningBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.runningBuilds);
     worker.updateProgress();
 }
@@ -778,7 +780,13 @@ static void movePath(const Path & src, const Path & dst)
 {
     auto st = lstat(src);
 
-    bool changePerm = (geteuid() && S_ISDIR(st.st_mode) && !(st.st_mode & S_IWUSR));
+    bool changePerm = (
+#ifndef _WIN32
+        geteuid()
+#else
+        !isRootUser()
+#endif
+        && S_ISDIR(st.st_mode) && !(st.st_mode & S_IWUSR));
 
     if (changePerm)
         chmod_(src, st.st_mode | S_IWUSR);
@@ -796,7 +804,7 @@ void replaceValidPath(const Path & storePath, const Path & tmpPath)
        tmpPath (the replacement), so we have to move it out of the
        way first.  We'd better not be interrupted here, because if
        we're repairing (say) Glibc, we end up with a broken system. */
-    Path oldPath = fmt("%1%.old-%2%-%3%", storePath, getpid(), random());
+    Path oldPath = fmt("%1%.old-%2%-%3%", storePath, getpid(), rand());
     if (pathExists(storePath))
         movePath(storePath, oldPath);
 
@@ -818,14 +826,20 @@ void replaceValidPath(const Path & storePath, const Path & tmpPath)
 
 int DerivationGoal::getChildStatus()
 {
+#ifndef _WIN32 // TODO enable build hook on Windows
     return hook->pid.kill();
+#else
+    return 0;
+#endif
 }
 
 
 void DerivationGoal::closeReadPipes()
 {
-    hook->builderOut.readSide = -1;
-    hook->fromHook.readSide = -1;
+#ifndef _WIN32 // TODO enable build hook on Windows
+    hook->builderOut.readSide.close();
+    hook->fromHook.readSide.close();
+#endif
 }
 
 
@@ -1019,13 +1033,16 @@ void DerivationGoal::buildDone()
 
         BuildResult::Status st = BuildResult::MiscFailure;
 
+#ifndef _WIN32
         if (hook && WIFEXITED(status) && WEXITSTATUS(status) == 101)
             st = BuildResult::TimedOut;
 
         else if (hook && (!WIFEXITED(status) || WEXITSTATUS(status) != 100)) {
         }
 
-        else {
+        else
+#endif
+        {
             assert(derivationType);
             st =
                 dynamic_cast<NotDeterministic*>(&e) ? BuildResult::NotDeterministic :
@@ -1112,6 +1129,9 @@ void DerivationGoal::resolvedFinished()
 
 HookReply DerivationGoal::tryBuildHook()
 {
+#ifdef _WIN32 // TODO enable build hook on Windows
+    return rpDecline;
+#else
     if (settings.buildHook.get().empty() || !worker.tryBuildHook || !useDerivation) return rpDecline;
 
     if (!worker.hook)
@@ -1205,17 +1225,18 @@ HookReply DerivationGoal::tryBuildHook()
     }
 
     hook->sink = FdSink();
-    hook->toHook.writeSide = -1;
+    hook->toHook.writeSide.close();
 
     /* Create the log file and pipe. */
     Path logFile = openLogFile();
 
-    std::set<int> fds;
+    std::set<MuxablePipePollState::CommChannel> fds;
     fds.insert(hook->fromHook.readSide.get());
     fds.insert(hook->builderOut.readSide.get());
     worker.childStarted(shared_from_this(), fds, false, false);
 
     return rpAccept;
+#endif
 }
 
 
@@ -1251,7 +1272,11 @@ Path DerivationGoal::openLogFile()
     Path logFileName = fmt("%s/%s%s", dir, baseName.substr(2),
         settings.compressLog ? ".bz2" : "");
 
-    fdLogFile = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC | O_CLOEXEC, 0666);
+    fdLogFile = toDescriptor(open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC
+#ifndef _WIN32
+        | O_CLOEXEC
+#endif
+        , 0666));
     if (!fdLogFile) throw SysError("creating log file '%1%'", logFileName);
 
     logFileSink = std::make_shared<FdSink>(fdLogFile.get());
@@ -1271,16 +1296,20 @@ void DerivationGoal::closeLogFile()
     if (logSink2) logSink2->finish();
     if (logFileSink) logFileSink->flush();
     logSink = logFileSink = 0;
-    fdLogFile = -1;
+    fdLogFile.close();
 }
 
 
-bool DerivationGoal::isReadDesc(int fd)
+bool DerivationGoal::isReadDesc(Descriptor fd)
 {
+#ifdef _WIN32 // TODO enable build hook on Windows
+    return false;
+#else
     return fd == hook->builderOut.readSide.get();
+#endif
 }
 
-void DerivationGoal::handleChildOutput(int fd, std::string_view data)
+void DerivationGoal::handleChildOutput(Descriptor fd, std::string_view data)
 {
     // local & `ssh://`-builds are dealt with here.
     auto isWrittenToLog = isReadDesc(fd);
@@ -1310,6 +1339,7 @@ void DerivationGoal::handleChildOutput(int fd, std::string_view data)
         if (logSink) (*logSink)(data);
     }
 
+#ifndef _WIN32 // TODO enable build hook on Windows
     if (hook && fd == hook->fromHook.readSide.get()) {
         for (auto c : data)
             if (c == '\n') {
@@ -1344,10 +1374,11 @@ void DerivationGoal::handleChildOutput(int fd, std::string_view data)
             } else
                 currentHookLine += c;
     }
+#endif
 }
 
 
-void DerivationGoal::handleEOF(int fd)
+void DerivationGoal::handleEOF(Descriptor fd)
 {
     if (!currentLogLine.empty()) flushLine();
     worker.wakeUp(shared_from_this());

src/libstore/build/derivation-goal.hh

@@ -2,7 +2,9 @@
 ///@file
 
 #include "parsed-derivations.hh"
-#include "lock.hh"
+#ifndef _WIN32
+#  include "user-lock.hh"
+#endif
 #include "outputs-spec.hh"
 #include "store-api.hh"
 #include "pathlocks.hh"
@@ -12,7 +14,9 @@ namespace nix {
 
 using std::map;
 
+#ifndef _WIN32 // TODO enable build hook on Windows
 struct HookInstance;
+#endif
 
 typedef enum {rpAccept, rpDecline, rpPostpone} HookReply;
 
@@ -178,10 +182,12 @@ struct DerivationGoal : public Goal
 
     std::string currentHookLine;
 
+#ifndef _WIN32 // TODO enable build hook on Windows
     /**
      * The build hook.
      */
     std::unique_ptr<HookInstance> hook;
+#endif
 
     /**
      * The sort of derivation we are building.
@@ -287,13 +293,13 @@ struct DerivationGoal : public Goal
     virtual void cleanupPostOutputsRegisteredModeCheck();
     virtual void cleanupPostOutputsRegisteredModeNonCheck();
 
-    virtual bool isReadDesc(int fd);
+    virtual bool isReadDesc(Descriptor fd);
 
     /**
      * Callback used by the worker to write to the log.
      */
-    void handleChildOutput(int fd, std::string_view data) override;
-    void handleEOF(int fd) override;
+    void handleChildOutput(Descriptor fd, std::string_view data) override;
+    void handleEOF(Descriptor fd) override;
     void flushLine();
 
     /**

src/libstore/build/drv-output-substitution-goal.cc

@@ -66,7 +66,11 @@ void DrvOutputSubstitutionGoal::tryNext()
        some other error occurs), so it must not touch `this`. So put
        the shared state in a separate refcounted object. */
     downloadState = std::make_shared<DownloadState>();
+#ifndef _WIN32
     downloadState->outPipe.create();
+#else
+    downloadState->outPipe.createAsyncPipe(worker.ioport.get());
+#endif
 
     sub->queryRealisation(
         id,
@@ -79,7 +83,13 @@ void DrvOutputSubstitutionGoal::tryNext()
             }
         } });
 
-    worker.childStarted(shared_from_this(), {downloadState->outPipe.readSide.get()}, true, false);
+    worker.childStarted(shared_from_this(), {
+#ifndef _WIN32
+        downloadState->outPipe.readSide.get()
+#else
+        &downloadState->outPipe
+#endif
+    }, true, false);
 
     state = &DrvOutputSubstitutionGoal::realisationFetched;
 }
@@ -158,7 +168,7 @@ void DrvOutputSubstitutionGoal::work()
     (this->*state)();
 }
 
-void DrvOutputSubstitutionGoal::handleEOF(int fd)
+void DrvOutputSubstitutionGoal::handleEOF(Descriptor fd)
 {
     if (fd == downloadState->outPipe.readSide.get()) worker.wakeUp(shared_from_this());
 }

src/libstore/build/drv-output-substitution-goal.hh

@@ -1,11 +1,13 @@
 #pragma once
 ///@file
 
+#include <thread>
+#include <future>
+
 #include "store-api.hh"
 #include "goal.hh"
 #include "realisation.hh"
-#include <thread>
-#include <future>
+#include "muxable-pipe.hh"
 
 namespace nix {
 
@@ -43,7 +45,7 @@ class DrvOutputSubstitutionGoal : public Goal {
 
     struct DownloadState
     {
-        Pipe outPipe;
+        MuxablePipe outPipe;
         std::promise<std::shared_ptr<const Realisation>> promise;
     };
 
@@ -71,7 +73,7 @@ public:
     std::string key() override;
 
     void work() override;
-    void handleEOF(int fd) override;
+    void handleEOF(Descriptor fd) override;
 
     JobCategory jobCategory() const override {
         return JobCategory::Substitution;

src/libstore/build/entry-points.cc

@@ -1,6 +1,8 @@
 #include "worker.hh"
 #include "substitution-goal.hh"
-#include "derivation-goal.hh"
+#ifndef _WIN32 // TODO Enable building on Windows
+#  include "derivation-goal.hh"
+#endif
 #include "local-store.hh"
 
 namespace nix {
@@ -25,9 +27,12 @@ void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMod
                 ex = std::move(i->ex);
         }
         if (i->exitCode != Goal::ecSuccess) {
+#ifndef _WIN32 // TODO Enable building on Windows
             if (auto i2 = dynamic_cast<DerivationGoal *>(i.get()))
                 failed.insert(printStorePath(i2->drvPath));
-            else if (auto i2 = dynamic_cast<PathSubstitutionGoal *>(i.get()))
+            else
+#endif
+            if (auto i2 = dynamic_cast<PathSubstitutionGoal *>(i.get()))
                 failed.insert(printStorePath(i2->storePath));
         }
     }
@@ -74,7 +79,12 @@ BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivat
     BuildMode buildMode)
 {
     Worker worker(*this, *this);
+#ifndef _WIN32 // TODO Enable building on Windows
     auto goal = worker.makeBasicDerivationGoal(drvPath, drv, OutputsSpec::All {}, buildMode);
+#else
+    std::shared_ptr<Goal> goal;
+    throw UnimplementedError("Building derivations not yet implemented on windows.");
+#endif
 
     try {
         worker.run(Goals{goal});

src/libstore/build/goal.hh

@@ -138,12 +138,12 @@ public:
 
     virtual void waiteeDone(GoalPtr waitee, ExitCode result);
 
-    virtual void handleChildOutput(int fd, std::string_view data)
+    virtual void handleChildOutput(Descriptor fd, std::string_view data)
     {
         abort();
     }
 
-    virtual void handleEOF(int fd)
+    virtual void handleEOF(Descriptor fd)
     {
         abort();
     }

src/libstore/build/substitution-goal.cc

@@ -212,7 +212,11 @@ void PathSubstitutionGoal::tryToRun()
     maintainRunningSubstitutions = std::make_unique<MaintainCount<uint64_t>>(worker.runningSubstitutions);
     worker.updateProgress();
 
+#ifndef _WIN32
     outPipe.create();
+#else
+    outPipe.createAsyncPipe(worker.ioport.get());
+#endif
 
     promise = std::promise<void>();
 
@@ -235,7 +239,13 @@ void PathSubstitutionGoal::tryToRun()
         }
     });
 
-    worker.childStarted(shared_from_this(), {outPipe.readSide.get()}, true, false);
+    worker.childStarted(shared_from_this(), {
+#ifndef _WIN32
+        outPipe.readSide.get()
+#else
+        &outPipe
+#endif
+    }, true, false);
 
     state = &PathSubstitutionGoal::finished;
 }
@@ -294,12 +304,12 @@ void PathSubstitutionGoal::finished()
 }
 
 
-void PathSubstitutionGoal::handleChildOutput(int fd, std::string_view data)
+void PathSubstitutionGoal::handleChildOutput(Descriptor fd, std::string_view data)
 {
 }
 
 
-void PathSubstitutionGoal::handleEOF(int fd)
+void PathSubstitutionGoal::handleEOF(Descriptor fd)
 {
     if (fd == outPipe.readSide.get()) worker.wakeUp(shared_from_this());
 }

src/libstore/build/substitution-goal.hh

@@ -1,9 +1,9 @@
 #pragma once
 ///@file
 
-#include "lock.hh"
 #include "store-api.hh"
 #include "goal.hh"
+#include "muxable-pipe.hh"
 
 namespace nix {
 
@@ -45,7 +45,7 @@ struct PathSubstitutionGoal : public Goal
     /**
      * Pipe for the substituter's standard output.
      */
-    Pipe outPipe;
+    MuxablePipe outPipe;
 
     /**
      * The substituter thread.
@@ -111,8 +111,8 @@ public:
     /**
      * Callback used by the worker to write to the log.
      */
-    void handleChildOutput(int fd, std::string_view data) override;
-    void handleEOF(int fd) override;
+    void handleChildOutput(Descriptor fd, std::string_view data) override;
+    void handleEOF(Descriptor fd) override;
 
     /* Called by destructor, can't be overridden */
     void cleanup() override final;

src/libstore/build/worker.cc

@@ -1,13 +1,15 @@
+#include "local-store.hh"
 #include "machines.hh"
 #include "worker.hh"
 #include "substitution-goal.hh"
 #include "drv-output-substitution-goal.hh"
-#include "local-derivation-goal.hh"
-#include "hook-instance.hh"
+#include "derivation-goal.hh"
+#ifndef _WIN32 // TODO Enable building on Windows
+#  include "local-derivation-goal.hh"
+#  include "hook-instance.hh"
+#endif
 #include "signals.hh"
 
-#include <poll.h>
-
 namespace nix {
 
 Worker::Worker(Store & store, Store & evalStore)
@@ -64,20 +66,27 @@ std::shared_ptr<DerivationGoal> Worker::makeDerivationGoal(const StorePath & drv
     const OutputsSpec & wantedOutputs, BuildMode buildMode)
 {
     return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() -> std::shared_ptr<DerivationGoal> {
-        return !dynamic_cast<LocalStore *>(&store)
-            ? std::make_shared</* */DerivationGoal>(drvPath, wantedOutputs, *this, buildMode)
-            : std::make_shared<LocalDerivationGoal>(drvPath, wantedOutputs, *this, buildMode);
+        return
+#ifndef _WIN32 // TODO Enable building on Windows
+            dynamic_cast<LocalStore *>(&store)
+            ? std::make_shared<LocalDerivationGoal>(drvPath, wantedOutputs, *this, buildMode)
+            :
+#endif
+            std::make_shared</* */DerivationGoal>(drvPath, wantedOutputs, *this, buildMode);
     });
 }
 
-
 std::shared_ptr<DerivationGoal> Worker::makeBasicDerivationGoal(const StorePath & drvPath,
     const BasicDerivation & drv, const OutputsSpec & wantedOutputs, BuildMode buildMode)
 {
     return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() -> std::shared_ptr<DerivationGoal> {
-        return !dynamic_cast<LocalStore *>(&store)
-            ? std::make_shared</* */DerivationGoal>(drvPath, drv, wantedOutputs, *this, buildMode)
-            : std::make_shared<LocalDerivationGoal>(drvPath, drv, wantedOutputs, *this, buildMode);
+        return
+#ifndef _WIN32 // TODO Enable building on Windows
+            dynamic_cast<LocalStore *>(&store)
+            ? std::make_shared<LocalDerivationGoal>(drvPath, drv, wantedOutputs, *this, buildMode)
+            :
+#endif
+            std::make_shared</* */DerivationGoal>(drvPath, drv, wantedOutputs, *this, buildMode);
     });
 }
 
@@ -143,7 +152,8 @@ void Worker::removeGoal(GoalPtr goal)
 {
     if (auto drvGoal = std::dynamic_pointer_cast<DerivationGoal>(goal))
         nix::removeGoal(drvGoal, derivationGoals);
-    else if (auto subGoal = std::dynamic_pointer_cast<PathSubstitutionGoal>(goal))
+    else
+    if (auto subGoal = std::dynamic_pointer_cast<PathSubstitutionGoal>(goal))
         nix::removeGoal(subGoal, substitutionGoals);
     else if (auto subGoal = std::dynamic_pointer_cast<DrvOutputSubstitutionGoal>(goal))
         nix::removeGoal(subGoal, drvOutputSubstitutionGoals);
@@ -187,13 +197,13 @@ unsigned Worker::getNrSubstitutions()
 }
 
 
-void Worker::childStarted(GoalPtr goal, const std::set<int> & fds,
+void Worker::childStarted(GoalPtr goal, const std::set<MuxablePipePollState::CommChannel> & channels,
     bool inBuildSlot, bool respectTimeouts)
 {
     Child child;
     child.goal = goal;
     child.goal2 = goal.get();
-    child.fds = fds;
+    child.channels = channels;
     child.timeStarted = child.lastOutput = steady_time_point::clock::now();
     child.inBuildSlot = inBuildSlot;
     child.respectTimeouts = respectTimeouts;
@@ -286,7 +296,8 @@ void Worker::run(const Goals & _topGoals)
                 .drvPath = makeConstantStorePathRef(goal->drvPath),
                 .outputs = goal->wantedOutputs,
             });
-        } else if (auto goal = dynamic_cast<PathSubstitutionGoal *>(i.get())) {
+        } else
+        if (auto goal = dynamic_cast<PathSubstitutionGoal *>(i.get())) {
             topPaths.push_back(DerivedPath::Opaque{goal->storePath});
         }
     }
@@ -408,23 +419,25 @@ void Worker::waitForInput()
     if (useTimeout)
         vomit("sleeping %d seconds", timeout);
 
+    MuxablePipePollState state;
+
+#ifndef _WIN32
     /* Use select() to wait for the input side of any logger pipe to
        become `available'.  Note that `available' (i.e., non-blocking)
        includes EOF. */
-    std::vector<struct pollfd> pollStatus;
-    std::map<int, size_t> fdToPollStatus;
     for (auto & i : children) {
-        for (auto & j : i.fds) {
-            pollStatus.push_back((struct pollfd) { .fd = j, .events = POLLIN });
-            fdToPollStatus[j] = pollStatus.size() - 1;
+        for (auto & j : i.channels) {
+            state.pollStatus.push_back((struct pollfd) { .fd = j, .events = POLLIN });
+            state.fdToPollStatus[j] = state.pollStatus.size() - 1;
         }
     }
+#endif
 
-    if (poll(pollStatus.data(), pollStatus.size(),
-            useTimeout ? timeout * 1000 : -1) == -1) {
-        if (errno == EINTR) return;
-        throw SysError("waiting for input");
-    }
+    state.poll(
+#ifdef _WIN32
+        ioport.get(),
+#endif
+        useTimeout ? (std::optional { timeout * 1000 }) : std::nullopt);
 
     auto after = steady_time_point::clock::now();
 
@@ -439,32 +452,18 @@ void Worker::waitForInput()
         GoalPtr goal = j->goal.lock();
         assert(goal);
 
-        std::set<int> fds2(j->fds);
-        std::vector<unsigned char> buffer(4096);
-        for (auto & k : fds2) {
-            const auto fdPollStatusId = get(fdToPollStatus, k);
-            assert(fdPollStatusId);
-            assert(*fdPollStatusId < pollStatus.size());
-            if (pollStatus.at(*fdPollStatusId).revents) {
-                ssize_t rd = ::read(k, buffer.data(), buffer.size());
-                // FIXME: is there a cleaner way to handle pt close
-                // than EIO? Is this even standard?
-                if (rd == 0 || (rd == -1 && errno == EIO)) {
-                    debug("%1%: got EOF", goal->getName());
-                    goal->handleEOF(k);
-                    j->fds.erase(k);
-                } else if (rd == -1) {
-                    if (errno != EINTR)
-                        throw SysError("%s: read failed", goal->getName());
-                } else {
-                    printMsg(lvlVomit, "%1%: read %2% bytes",
-                        goal->getName(), rd);
-                    std::string_view data((char *) buffer.data(), rd);
-                    j->lastOutput = after;
-                    goal->handleChildOutput(k, data);
-                }
-            }
-        }
+        state.iterate(
+            j->channels,
+            [&](Descriptor k, std::string_view data) {
+                printMsg(lvlVomit, "%1%: read %2% bytes",
+                    goal->getName(), data.size());
+                j->lastOutput = after;
+                goal->handleChildOutput(k, data);
+            },
+            [&](Descriptor k) {
+                debug("%1%: got EOF", goal->getName());
+                goal->handleEOF(k);
+            });
 
         if (goal->exitCode == Goal::ecBusy &&
             0 != settings.maxSilentTime &&
@@ -529,9 +528,9 @@ bool Worker::pathContentsGood(const StorePath & path)
     if (!pathExists(store.printStorePath(path)))
         res = false;
     else {
-        Hash current = hashPath(
+        auto current = hashPath(
             {store.getFSAccessor(), CanonPath(store.printStorePath(path))},
-            FileIngestionMethod::Recursive, info->narHash.algo);
+            FileIngestionMethod::Recursive, info->narHash.algo).first;
         Hash nullHash(HashAlgorithm::SHA256);
         res = info->narHash == nullHash || info->narHash == current;
     }

src/libstore/build/worker.hh

@@ -2,10 +2,10 @@
 ///@file
 
 #include "types.hh"
-#include "lock.hh"
 #include "store-api.hh"
 #include "goal.hh"
 #include "realisation.hh"
+#include "muxable-pipe.hh"
 
 #include <future>
 #include <thread>
@@ -36,14 +36,14 @@ typedef std::chrono::time_point<std::chrono::steady_clock> steady_time_point;
 
 /**
  * A mapping used to remember for each child process to what goal it
- * belongs, and file descriptors for receiving log data and output
+ * belongs, and comm channels for receiving log data and output
  * path creation commands.
  */
 struct Child
 {
     WeakGoalPtr goal;
     Goal * goal2; // ugly hackery
-    std::set<int> fds;
+    std::set<MuxablePipePollState::CommChannel> channels;
     bool respectTimeouts;
     bool inBuildSlot;
     /**
@@ -53,8 +53,10 @@ struct Child
     steady_time_point timeStarted;
 };
 
+#ifndef _WIN32 // TODO Enable building on Windows
 /* Forward definition. */
 struct HookInstance;
+#endif
 
 /**
  * The worker class.
@@ -152,10 +154,16 @@ public:
      */
     bool checkMismatch;
 
+#ifdef _WIN32
+    AutoCloseFD ioport;
+#endif
+
     Store & store;
     Store & evalStore;
 
+#ifndef _WIN32 // TODO Enable building on Windows
     std::unique_ptr<HookInstance> hook;
+#endif
 
     uint64_t expectedBuilds = 0;
     uint64_t doneBuilds = 0;
@@ -238,7 +246,7 @@ public:
      * Registers a running child process.  `inBuildSlot` means that
      * the process counts towards the jobs limit.
      */
-    void childStarted(GoalPtr goal, const std::set<int> & fds,
+    void childStarted(GoalPtr goal, const std::set<MuxablePipePollState::CommChannel> & channels,
         bool inBuildSlot, bool respectTimeouts);
 
     /**

src/libstore/daemon.cc

@@ -1,6 +1,7 @@
 #include "daemon.hh"
 #include "signals.hh"
 #include "worker-protocol.hh"
+#include "worker-protocol-connection.hh"
 #include "worker-protocol-impl.hh"
 #include "build-result.hh"
 #include "store-api.hh"
@@ -19,6 +20,8 @@
 # include "monitor-fd.hh"
 #endif
 
+#include <sstream>
+
 namespace nix::daemon {
 
 Sink & operator << (Sink & sink, const Logger::Fields & fields)
@@ -531,7 +534,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         auto drvs = WorkerProto::Serialise<DerivedPaths>::read(*store, rconn);
         BuildMode mode = bmNormal;
         if (GET_PROTOCOL_MINOR(clientVersion) >= 15) {
-            mode = (BuildMode) readInt(from);
+            mode = WorkerProto::Serialise<BuildMode>::read(*store, rconn);
 
             /* Repairing is not atomic, so disallowed for "untrusted"
                clients.
@@ -555,7 +558,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     case WorkerProto::Op::BuildPathsWithResults: {
         auto drvs = WorkerProto::Serialise<DerivedPaths>::read(*store, rconn);
         BuildMode mode = bmNormal;
-        mode = (BuildMode) readInt(from);
+        mode = WorkerProto::Serialise<BuildMode>::read(*store, rconn);
 
         /* Repairing is not atomic, so disallowed for "untrusted"
            clients.
@@ -586,7 +589,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
          * correctly.
          */
         readDerivation(from, *store, drv, Derivation::nameFromPath(drvPath));
-        BuildMode buildMode = (BuildMode) readInt(from);
+        auto buildMode = WorkerProto::Serialise<BuildMode>::read(*store, rconn);
         logger->startWork();
 
         auto drvType = drv.type();
@@ -1026,11 +1029,9 @@ void processConnection(
 #endif
 
     /* Exchange the greeting. */
-    unsigned int magic = readInt(from);
-    if (magic != WORKER_MAGIC_1) throw Error("protocol mismatch");
-    to << WORKER_MAGIC_2 << PROTOCOL_VERSION;
-    to.flush();
-    WorkerProto::Version clientVersion = readInt(from);
+    WorkerProto::Version clientVersion =
+        WorkerProto::BasicServerConnection::handshake(
+            to, from, PROTOCOL_VERSION);
 
     if (clientVersion < 0x10a)
         throw Error("the Nix client version is too old");
@@ -1048,29 +1049,20 @@ void processConnection(
         printMsgUsing(prevLogger, lvlDebug, "%d operations", opCount);
     });
 
-    if (GET_PROTOCOL_MINOR(clientVersion) >= 14 && readInt(from)) {
-        // Obsolete CPU affinity.
-        readInt(from);
-    }
+    WorkerProto::BasicServerConnection conn {
+        .to = to,
+        .from = from,
+        .clientVersion = clientVersion,
+    };
 
-    if (GET_PROTOCOL_MINOR(clientVersion) >= 11)
-        readInt(from); // obsolete reserveSpace
-
-    if (GET_PROTOCOL_MINOR(clientVersion) >= 33)
-        to << nixVersion;
-
-    if (GET_PROTOCOL_MINOR(clientVersion) >= 35) {
+    conn.postHandshake(*store, {
+        .daemonNixVersion = nixVersion,
         // We and the underlying store both need to trust the client for
         // it to be trusted.
-        auto temp = trusted
+        .remoteTrustsUs = trusted
             ? store->isTrustedClient()
-            : std::optional { NotTrusted };
-        WorkerProto::WriteConn wconn {
-            .to = to,
-            .version = clientVersion,
-        };
-        WorkerProto::write(*store, wconn, temp);
-    }
+            : std::optional { NotTrusted },
+    });
 
     /* Send startup error messages to the client. */
     tunnelLogger->startWork();

src/libstore/derivations.cc

@@ -930,10 +930,9 @@ DerivationOutputsAndOptPaths BasicDerivation::outputsAndOptPaths(const StoreDirC
 
 std::string_view BasicDerivation::nameFromPath(const StorePath & drvPath)
 {
+    drvPath.requireDerivation();
     auto nameWithSuffix = drvPath.name();
-    constexpr std::string_view extension = ".drv";
-    assert(hasSuffix(nameWithSuffix, extension));
-    nameWithSuffix.remove_suffix(extension.size());
+    nameWithSuffix.remove_suffix(drvExtension.size());
     return nameWithSuffix;
 }
 

src/libstore/globals.hh

@@ -1262,6 +1262,16 @@ public:
           store paths of the latest Nix release.
         )"
     };
+
+    Setting<uint64_t> warnLargePathThreshold{
+        this,
+        std::numeric_limits<uint64_t>::max(),
+        "warn-large-path-threshold",
+        R"(
+          Warn when copying a path larger than this number of bytes to the Nix store
+          (as determined by its NAR serialisation).
+        )"
+    };
 };
 
 

src/libstore/legacy-ssh-store.cc

@@ -4,6 +4,7 @@
 #include "pool.hh"
 #include "remote-store.hh"
 #include "serve-protocol.hh"
+#include "serve-protocol-connection.hh"
 #include "serve-protocol-impl.hh"
 #include "build-result.hh"
 #include "store-api.hh"
@@ -32,32 +33,19 @@ LegacySSHStore::LegacySSHStore(
     std::string_view scheme,
     std::string_view host,
     const Params & params)
-    : LegacySSHStore{scheme, LegacySSHStoreConfig::extractConnStr(scheme, host), params}
-{
-}
-
-LegacySSHStore::LegacySSHStore(
-    std::string_view scheme,
-    std::string host,
-    const Params & params)
     : StoreConfig(params)
-    , CommonSSHStoreConfig(params)
+    , CommonSSHStoreConfig(scheme, host, params)
     , LegacySSHStoreConfig(params)
     , Store(params)
-    , host(host)
     , connections(make_ref<Pool<Connection>>(
         std::max(1, (int) maxConnections),
         [this]() { return openConnection(); },
         [](const ref<Connection> & r) { return r->good; }
         ))
-    , master(
-        host,
-        sshKey.get(),
-        sshPublicHostKey.get(),
+    , master(createSSHMaster(
         // Use SSH master only if using more than 1 connection.
         connections->capacity() > 1,
-        compress,
-        logFD)
+        logFD))
 {
 }
 
@@ -86,7 +74,7 @@ ref<LegacySSHStore::Connection> LegacySSHStore::openConnection()
         conn->sshConn->in.close();
         {
             NullSink nullSink;
-            conn->from.drainInto(nullSink);
+            tee.drainInto(nullSink);
         }
         throw Error("'nix-store --serve' protocol mismatch from '%s', got '%s'",
             host, chomp(saved.s));
@@ -168,41 +156,38 @@ void LegacySSHStore::addToStore(const ValidPathInfo & info, Source & source,
         }
         conn->to.flush();
 
+        if (readInt(conn->from) != 1)
+            throw Error("failed to add path '%s' to remote host '%s'", printStorePath(info.path), host);
+
     } else {
 
-        conn->to
-            << ServeProto::Command::ImportPaths
-            << 1;
-        try {
-            copyNAR(source, conn->to);
-        } catch (...) {
-            conn->good = false;
-            throw;
-        }
-        conn->to
-            << exportMagic
-            << printStorePath(info.path);
-        ServeProto::write(*this, *conn, info.references);
-        conn->to
-            << (info.deriver ? printStorePath(*info.deriver) : "")
-            << 0
-            << 0;
-        conn->to.flush();
+        conn->importPaths(*this, [&](Sink & sink) {
+            try {
+                copyNAR(source, sink);
+            } catch (...) {
+                conn->good = false;
+                throw;
+            }
+            sink
+                << exportMagic
+                << printStorePath(info.path);
+            ServeProto::write(*this, *conn, info.references);
+            sink
+                << (info.deriver ? printStorePath(*info.deriver) : "")
+                << 0
+                << 0;
+        });
 
     }
-
-    if (readInt(conn->from) != 1)
-        throw Error("failed to add path '%s' to remote host '%s'", printStorePath(info.path), host);
 }
 
 
 void LegacySSHStore::narFromPath(const StorePath & path, Sink & sink)
 {
     auto conn(connections->get());
-
-    conn->to << ServeProto::Command::DumpStorePath << printStorePath(path);
-    conn->to.flush();
-    copyNAR(conn->from, sink);
+    conn->narFromPath(*this, path, [&](auto & source) {
+        copyNAR(source, sink);
+    });
 }
 
 
@@ -226,7 +211,7 @@ BuildResult LegacySSHStore::buildDerivation(const StorePath & drvPath, const Bas
 
     conn->putBuildDerivationRequest(*this, drvPath, drv, buildSettings());
 
-    return ServeProto::Serialise<BuildResult>::read(*this, *conn);
+    return conn->getBuildDerivationResponse(*this);
 }
 
 

src/libstore/legacy-ssh-store.hh

@@ -26,15 +26,17 @@ struct LegacySSHStoreConfig : virtual CommonSSHStoreConfig
 
 struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Store
 {
+#ifndef _WIN32
     // Hack for getting remote build log output.
     // Intentionally not in `LegacySSHStoreConfig` so that it doesn't appear in
     // the documentation
-    const Setting<int> logFD{this, -1, "log-fd", "file descriptor to which SSH's stderr is connected"};
+    const Setting<int> logFD{this, INVALID_DESCRIPTOR, "log-fd", "file descriptor to which SSH's stderr is connected"};
+#else
+    Descriptor logFD = INVALID_DESCRIPTOR;
+#endif
 
     struct Connection;
 
-    std::string host;
-
     ref<Pool<Connection>> connections;
 
     SSHMaster master;
@@ -46,13 +48,6 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
         std::string_view host,
         const Params & params);
 
-private:
-    LegacySSHStore(
-        std::string_view scheme,
-        std::string host,
-        const Params & params);
-public:
-
     ref<Connection> openConnection();
 
     std::string getUri() override;

src/libstore/local-store.cc

@@ -1282,7 +1282,7 @@ StorePath LocalStore::addToStoreFromDump(
             ? dumpHash
             : hashPath(
                 PosixSourceAccessor::createAtRoot(tempPath),
-                hashMethod.getFileIngestionMethod(), hashAlgo),
+                hashMethod.getFileIngestionMethod(), hashAlgo).first,
         {
             .others = references,
             // caller is not capable of creating a self-reference, because this is content-addressed without modulus
@@ -1422,7 +1422,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
             PosixSourceAccessor accessor;
             std::string hash = hashPath(
                 PosixSourceAccessor::createAtRoot(link.path()),
-                FileIngestionMethod::Recursive, HashAlgorithm::SHA256).to_string(HashFormat::Nix32, false);
+                FileIngestionMethod::Recursive, HashAlgorithm::SHA256).first.to_string(HashFormat::Nix32, false);
             if (hash != name.string()) {
                 printError("link '%s' was modified! expected hash '%s', got '%s'",
                     link.path(), name, hash);

src/libstore/local.mk

@@ -4,9 +4,9 @@ libstore_NAME = libnixstore
 
 libstore_DIR := $(d)
 
-libstore_SOURCES := $(wildcard $(d)/*.cc $(d)/builtins/*.cc)
+libstore_SOURCES := $(wildcard $(d)/*.cc $(d)/builtins/*.cc $(d)/build/*.cc)
 ifdef HOST_UNIX
-  libstore_SOURCES += $(wildcard $(d)/unix/*.cc $(d)/unix/builtins/*.cc $(d)/unix/build/*.cc)
+  libstore_SOURCES += $(wildcard $(d)/unix/*.cc $(d)/unix/build/*.cc)
 endif
 ifdef HOST_LINUX
   libstore_SOURCES += $(wildcard $(d)/linux/*.cc)
@@ -43,7 +43,7 @@ endif
 
 INCLUDE_libstore := -I $(d) -I $(d)/build
 ifdef HOST_UNIX
-  INCLUDE_libstore += -I $(d)/unix
+  INCLUDE_libstore += -I $(d)/unix -I $(d)/unix/build
 endif
 ifdef HOST_LINUX
   INCLUDE_libstore += -I $(d)/linux

src/libstore/machines.cc

@@ -131,7 +131,7 @@ static std::vector<std::string> expandBuilderLines(const std::string & builders)
     return result;
 }
 
-static Machine parseBuilderLine(const std::string & line)
+static Machine parseBuilderLine(const std::set<std::string> & defaultSystems, const std::string & line)
 {
     const auto tokens = tokenizeString<std::vector<std::string>>(line);
 
@@ -148,7 +148,7 @@ static Machine parseBuilderLine(const std::string & line)
     };
 
     auto parseFloatField = [&](size_t fieldIndex) {
-        const auto result = string2Int<float>(tokens[fieldIndex]);
+        const auto result = string2Float<float>(tokens[fieldIndex]);
         if (!result) {
             throw FormatError("bad machine specification: failed to convert column #%lu in a row: '%s' to 'float'", fieldIndex, line);
         }
@@ -168,29 +168,46 @@ static Machine parseBuilderLine(const std::string & line)
     if (!isSet(0))
         throw FormatError("bad machine specification: store URL was not found at the first column of a row: '%s'", line);
 
+    // TODO use designated initializers, once C++ supports those with
+    // custom constructors.
     return {
+        // `storeUri`
         tokens[0],
-        isSet(1) ? tokenizeString<std::set<std::string>>(tokens[1], ",") : std::set<std::string>{settings.thisSystem},
+        // `systemTypes`
+        isSet(1) ? tokenizeString<std::set<std::string>>(tokens[1], ",") : defaultSystems,
+        // `sshKey`
         isSet(2) ? tokens[2] : "",
+        // `maxJobs`
         isSet(3) ? parseUnsignedIntField(3) : 1U,
+        // `speedFactor`
         isSet(4) ? parseFloatField(4) : 1.0f,
+        // `supportedFeatures`
         isSet(5) ? tokenizeString<std::set<std::string>>(tokens[5], ",") : std::set<std::string>{},
+        // `mandatoryFeatures`
         isSet(6) ? tokenizeString<std::set<std::string>>(tokens[6], ",") : std::set<std::string>{},
+        // `sshPublicHostKey`
         isSet(7) ? ensureBase64(7) : ""
     };
 }
 
-static Machines parseBuilderLines(const std::vector<std::string> & builders)
+static Machines parseBuilderLines(const std::set<std::string> & defaultSystems, const std::vector<std::string> & builders)
 {
     Machines result;
-    std::transform(builders.begin(), builders.end(), std::back_inserter(result), parseBuilderLine);
+    std::transform(
+        builders.begin(), builders.end(), std::back_inserter(result),
+        [&](auto && line) { return parseBuilderLine(defaultSystems, line); });
     return result;
 }
 
+Machines Machine::parseConfig(const std::set<std::string> & defaultSystems, const std::string & s)
+{
+    const auto builderLines = expandBuilderLines(s);
+    return parseBuilderLines(defaultSystems, builderLines);
+}
+
 Machines getMachines()
 {
-    const auto builderLines = expandBuilderLines(settings.builders);
-    return parseBuilderLines(builderLines);
+    return Machine::parseConfig({settings.thisSystem}, settings.builders);
 }
 
 }

src/libstore/machines.hh

@@ -8,6 +8,10 @@ namespace nix {
 
 class Store;
 
+struct Machine;
+
+typedef std::vector<Machine> Machines;
+
 struct Machine {
 
     const StoreReference storeUri;
@@ -63,12 +67,22 @@ struct Machine {
      * ```
      */
     ref<Store> openStore() const;
+
+    /**
+     * Parse a machine configuration.
+     *
+     * Every machine is specified on its own line, and lines beginning
+     * with `@` are interpreted as paths to other configuration files in
+     * the same format.
+     */
+    static Machines parseConfig(const std::set<std::string> & defaultSystems, const std::string & config);
 };
 
-typedef std::vector<Machine> Machines;
-
-void parseMachines(const std::string & s, Machines & machines);
-
+/**
+ * Parse machines from the global config
+ *
+ * @todo Remove, globals are bad.
+ */
 Machines getMachines();
 
 }

src/libstore/nix-store.pc.in

@@ -5,5 +5,6 @@ includedir=@includedir@
 Name: Nix
 Description: Nix Package Manager
 Version: @PACKAGE_VERSION@
-Libs: -L${libdir} -lnixstore -lnixutil
+Requires: nix-util
+Libs: -L${libdir} -lnixstore
 Cflags: -I${includedir}/nix -std=c++2a

src/libstore/parsed-derivations.cc

@@ -135,18 +135,37 @@ static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*");
 /**
  * Write a JSON representation of store object metadata, such as the
  * hash and the references.
+ *
+ * @note Do *not* use `ValidPathInfo::toJSON` because this function is
+ * subject to stronger stability requirements since it is used to
+ * prepare build environments. Perhaps someday we'll have a versionining
+ * mechanism to allow this to evolve again and get back in sync, but for
+ * now we must not change - not even extend - the behavior.
  */
 static nlohmann::json pathInfoToJSON(
     Store & store,
     const StorePathSet & storePaths)
 {
-    nlohmann::json::array_t jsonList = nlohmann::json::array();
+    using nlohmann::json;
+
+    nlohmann::json::array_t jsonList = json::array();
 
     for (auto & storePath : storePaths) {
         auto info = store.queryPathInfo(storePath);
 
-        auto & jsonPath = jsonList.emplace_back(
-            info->toJSON(store, false, HashFormat::Nix32));
+        auto & jsonPath = jsonList.emplace_back(json::object());
+
+        jsonPath["narHash"] = info->narHash.to_string(HashFormat::Nix32, true);
+        jsonPath["narSize"] = info->narSize;
+
+        {
+            auto & jsonRefs = jsonPath["references"] = json::array();
+            for (auto & ref : info->references)
+                jsonRefs.emplace_back(store.printStorePath(ref));
+        }
+
+        if (info->ca)
+            jsonPath["ca"] = renderContentAddress(info->ca);
 
         // Add the path to the object whose metadata we are including.
         jsonPath["path"] = store.printStorePath(storePath);

src/libstore/path-info.cc

@@ -161,28 +161,23 @@ nlohmann::json UnkeyedValidPathInfo::toJSON(
     jsonObject["narSize"] = narSize;
 
     {
-        auto& jsonRefs = (jsonObject["references"] = json::array());
+        auto & jsonRefs = jsonObject["references"] = json::array();
         for (auto & ref : references)
             jsonRefs.emplace_back(store.printStorePath(ref));
     }
 
-    if (ca)
-        jsonObject["ca"] = renderContentAddress(ca);
+    jsonObject["ca"] = ca ? (std::optional { renderContentAddress(*ca) }) : std::nullopt;
 
     if (includeImpureInfo) {
-        if (deriver)
-            jsonObject["deriver"] = store.printStorePath(*deriver);
+        jsonObject["deriver"] = deriver ? (std::optional { store.printStorePath(*deriver) }) : std::nullopt;
 
-        if (registrationTime)
-            jsonObject["registrationTime"] = registrationTime;
+        jsonObject["registrationTime"] = registrationTime ? (std::optional { registrationTime }) : std::nullopt;
 
-        if (ultimate)
-            jsonObject["ultimate"] = ultimate;
+        jsonObject["ultimate"] = ultimate;
 
-        if (!sigs.empty()) {
-            for (auto & sig : sigs)
-                jsonObject["signatures"].push_back(sig);
-        }
+        auto & sigsObj = jsonObject["signatures"] = json::array();
+        for (auto & sig : sigs)
+            sigsObj.push_back(sig);
     }
 
     return jsonObject;
@@ -210,20 +205,25 @@ UnkeyedValidPathInfo UnkeyedValidPathInfo::fromJSON(
         throw;
     }
 
+    // New format as this as nullable but mandatory field; handling
+    // missing is for back-compat.
     if (json.contains("ca"))
-        res.ca = ContentAddress::parse(getString(valueAt(json, "ca")));
+        if (auto * rawCa = getNullable(valueAt(json, "ca")))
+            res.ca = ContentAddress::parse(getString(*rawCa));
 
     if (json.contains("deriver"))
-        res.deriver = store.parseStorePath(getString(valueAt(json, "deriver")));
+        if (auto * rawDeriver = getNullable(valueAt(json, "deriver")))
+            res.deriver = store.parseStorePath(getString(*rawDeriver));
 
     if (json.contains("registrationTime"))
-        res.registrationTime = getInteger(valueAt(json, "registrationTime"));
+        if (auto * rawRegistrationTime = getNullable(valueAt(json, "registrationTime")))
+            res.registrationTime = getInteger(*rawRegistrationTime);
 
     if (json.contains("ultimate"))
         res.ultimate = getBoolean(valueAt(json, "ultimate"));
 
     if (json.contains("signatures"))
-        res.sigs = valueAt(json, "signatures");
+        res.sigs = getStringSet(valueAt(json, "signatures"));
 
     return res;
 }

src/libstore/path.cc

@@ -49,11 +49,17 @@ StorePath::StorePath(const Hash & hash, std::string_view _name)
     checkName(baseName, name());
 }
 
-bool StorePath::isDerivation() const
+bool StorePath::isDerivation() const noexcept
 {
     return hasSuffix(name(), drvExtension);
 }
 
+void StorePath::requireDerivation() const
+{
+    if (!isDerivation())
+        throw FormatError("store path '%s' is not a valid derivation path", to_string());
+}
+
 StorePath StorePath::dummy("ffffffffffffffffffffffffffffffff-x");
 
 StorePath StorePath::random(std::string_view name)

src/libstore/path.hh

@@ -35,30 +35,23 @@ public:
 
     StorePath(const Hash & hash, std::string_view name);
 
-    std::string_view to_string() const
+    std::string_view to_string() const noexcept
     {
         return baseName;
     }
 
-    bool operator < (const StorePath & other) const
-    {
-        return baseName < other.baseName;
-    }
-
-    bool operator == (const StorePath & other) const
-    {
-        return baseName == other.baseName;
-    }
-
-    bool operator != (const StorePath & other) const
-    {
-        return baseName != other.baseName;
-    }
+    bool operator == (const StorePath & other) const noexcept = default;
+    auto operator <=> (const StorePath & other) const noexcept = default;
 
     /**
      * Check whether a file name ends with the extension for derivations.
      */
-    bool isDerivation() const;
+    bool isDerivation() const noexcept;
+
+    /**
+     * Throw an exception if `isDerivation` is false.
+     */
+    void requireDerivation() const;
 
     std::string_view name() const
     {
@@ -82,7 +75,7 @@ typedef std::vector<StorePath> StorePaths;
  * The file extension of \ref Derivation derivations when serialized
  * into store objects.
  */
-const std::string drvExtension = ".drv";
+constexpr std::string_view drvExtension = ".drv";
 
 }
 

src/libstore/remote-store-connection.hh

@@ -3,6 +3,7 @@
 
 #include "remote-store.hh"
 #include "worker-protocol.hh"
+#include "worker-protocol-connection.hh"
 #include "pool.hh"
 
 namespace nix {
@@ -14,90 +15,13 @@ namespace nix {
  * Contains `Source` and `Sink` for actual communication, along with
  * other information learned when negotiating the connection.
  */
-struct RemoteStore::Connection
+struct RemoteStore::Connection : WorkerProto::BasicClientConnection,
+  WorkerProto::ClientHandshakeInfo
 {
-    /**
-     * Send with this.
-     */
-    FdSink to;
-
-    /**
-     * Receive with this.
-     */
-    FdSource from;
-
-    /**
-     * Worker protocol version used for the connection.
-     *
-     * Despite its name, I think it is actually the maximum version both
-     * sides support. (If the maximum doesn't exist, we would fail to
-     * establish a connection and produce a value of this type.)
-     */
-    WorkerProto::Version daemonVersion;
-
-    /**
-     * Whether the remote side trusts us or not.
-     *
-     * 3 values: "yes", "no", or `std::nullopt` for "unknown".
-     *
-     * Note that the "remote side" might not be just the end daemon, but
-     * also an intermediary forwarder that can make its own trusting
-     * decisions. This would be the intersection of all their trust
-     * decisions, since it takes only one link in the chain to start
-     * denying operations.
-     */
-    std::optional<TrustedFlag> remoteTrustsUs;
-
-    /**
-     * The version of the Nix daemon that is processing our requests.
-     *
-     * Do note, it may or may not communicating with another daemon,
-     * rather than being an "end" `LocalStore` or similar.
-     */
-    std::optional<std::string> daemonNixVersion;
-
     /**
      * Time this connection was established.
      */
     std::chrono::time_point<std::chrono::steady_clock> startTime;
-
-    /**
-     * Coercion to `WorkerProto::ReadConn`. This makes it easy to use the
-     * factored out worker protocol searlizers with a
-     * `RemoteStore::Connection`.
-     *
-     * The worker protocol connection types are unidirectional, unlike
-     * this type.
-     */
-    operator WorkerProto::ReadConn ()
-    {
-        return WorkerProto::ReadConn {
-            .from = from,
-            .version = daemonVersion,
-        };
-    }
-
-    /**
-     * Coercion to `WorkerProto::WriteConn`. This makes it easy to use the
-     * factored out worker protocol searlizers with a
-     * `RemoteStore::Connection`.
-     *
-     * The worker protocol connection types are unidirectional, unlike
-     * this type.
-     */
-    operator WorkerProto::WriteConn ()
-    {
-        return WorkerProto::WriteConn {
-            .to = to,
-            .version = daemonVersion,
-        };
-    }
-
-    virtual ~Connection();
-
-    virtual void closeWrite() = 0;
-
-    std::exception_ptr processStderr(Sink * sink = 0, Source * source = 0, bool flush = true);
 };
 
 /**

src/libstore/remote-store.cc

@@ -69,50 +69,26 @@ void RemoteStore::initConnection(Connection & conn)
     /* Send the magic greeting, check for the reply. */
     try {
         conn.from.endOfFileError = "Nix daemon disconnected unexpectedly (maybe it crashed?)";
-        conn.to << WORKER_MAGIC_1;
-        conn.to.flush();
+
         StringSink saved;
+        TeeSource tee(conn.from, saved);
         try {
-            TeeSource tee(conn.from, saved);
-            unsigned int magic = readInt(tee);
-            if (magic != WORKER_MAGIC_2)
-                throw Error("protocol mismatch");
+            conn.daemonVersion = WorkerProto::BasicClientConnection::handshake(
+                conn.to, tee, PROTOCOL_VERSION);
         } catch (SerialisationError & e) {
             /* In case the other side is waiting for our input, close
                it. */
             conn.closeWrite();
-            auto msg = conn.from.drain();
-            throw Error("protocol mismatch, got '%s'", chomp(saved.s + msg));
+            {
+                NullSink nullSink;
+                tee.drainInto(nullSink);
+            }
+            throw Error("protocol mismatch, got '%s'", chomp(saved.s));
         }
 
-        conn.from >> conn.daemonVersion;
-        if (GET_PROTOCOL_MAJOR(conn.daemonVersion) != GET_PROTOCOL_MAJOR(PROTOCOL_VERSION))
-            throw Error("Nix daemon protocol version not supported");
-        if (GET_PROTOCOL_MINOR(conn.daemonVersion) < 10)
-            throw Error("the Nix daemon version is too old");
-        conn.to << PROTOCOL_VERSION;
+        static_cast<WorkerProto::ClientHandshakeInfo &>(conn) = conn.postHandshake(*this);
 
-        if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 14) {
-            // Obsolete CPU affinity.
-            conn.to << 0;
-        }
-
-        if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 11)
-            conn.to << false; // obsolete reserveSpace
-
-        if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 33) {
-            conn.to.flush();
-            conn.daemonNixVersion = readString(conn.from);
-        }
-
-        if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 35) {
-            conn.remoteTrustsUs = WorkerProto::Serialise<std::optional<TrustedFlag>>::read(*this, conn);
-        } else {
-            // We don't know the answer; protocol to old.
-            conn.remoteTrustsUs = std::nullopt;
-        }
-
-        auto ex = conn.processStderr();
+        auto ex = conn.processStderrReturn();
         if (ex) std::rethrow_exception(ex);
     }
     catch (Error & e) {
@@ -158,7 +134,7 @@ void RemoteStore::setOptions(Connection & conn)
             conn.to << i.first << i.second.value;
     }
 
-    auto ex = conn.processStderr();
+    auto ex = conn.processStderrReturn();
     if (ex) std::rethrow_exception(ex);
 }
 
@@ -173,28 +149,7 @@ RemoteStore::ConnectionHandle::~ConnectionHandle()
 
 void RemoteStore::ConnectionHandle::processStderr(Sink * sink, Source * source, bool flush)
 {
-    auto ex = handle->processStderr(sink, source, flush);
-    if (ex) {
-        daemonException = true;
-        try {
-            std::rethrow_exception(ex);
-        } catch (const Error & e) {
-            // Nix versions before #4628 did not have an adequate behavior for reporting that the derivation format was upgraded.
-            // To avoid having to add compatibility logic in many places, we expect to catch almost all occurrences of the
-            // old incomprehensible error here, so that we can explain to users what's going on when their daemon is
-            // older than #4628 (2023).
-            if (experimentalFeatureSettings.isEnabled(Xp::DynamicDerivations) &&
-                GET_PROTOCOL_MINOR(handle->daemonVersion) <= 35)
-            {
-                auto m = e.msg();
-                if (m.find("parsing derivation") != std::string::npos &&
-                    m.find("expected string") != std::string::npos &&
-                    m.find("Derive([") != std::string::npos)
-                    throw Error("%s, this might be because the daemon is too old to understand dependencies on dynamic derivations. Check to see if the raw derivation is in the form '%s'", std::move(m), "DrvWithVersion(..)");
-            }
-            throw;
-        }
-    }
+    handle->processStderr(&daemonException, sink, source, flush);
 }
 
 
@@ -226,13 +181,7 @@ StorePathSet RemoteStore::queryValidPaths(const StorePathSet & paths, Substitute
             if (isValidPath(i)) res.insert(i);
         return res;
     } else {
-        conn->to << WorkerProto::Op::QueryValidPaths;
-        WorkerProto::write(*this, *conn, paths);
-        if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 27) {
-            conn->to << maybeSubstitute;
-        }
-        conn.processStderr();
-        return WorkerProto::Serialise<StorePathSet>::read(*this, *conn);
+        return conn->queryValidPaths(*this, &conn.daemonException, paths, maybeSubstitute);
     }
 }
 
@@ -322,22 +271,10 @@ void RemoteStore::queryPathInfoUncached(const StorePath & path,
         std::shared_ptr<const ValidPathInfo> info;
         {
             auto conn(getConnection());
-            conn->to << WorkerProto::Op::QueryPathInfo << printStorePath(path);
-            try {
-                conn.processStderr();
-            } catch (Error & e) {
-                // Ugly backwards compatibility hack.
-                if (e.msg().find("is not valid") != std::string::npos)
-                    throw InvalidPath(std::move(e.info()));
-                throw;
-            }
-            if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 17) {
-                bool valid; conn->from >> valid;
-                if (!valid) throw InvalidPath("path '%s' is not valid", printStorePath(path));
-            }
             info = std::make_shared<ValidPathInfo>(
                 StorePath{path},
-                WorkerProto::Serialise<UnkeyedValidPathInfo>::read(*this, *conn));
+                conn->queryPathInfo(*this, &conn.daemonException, path));
+
         }
         callback(std::move(info));
     } catch (...) { callback.rethrow(); }
@@ -542,8 +479,6 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
     auto conn(getConnection());
 
     if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 18) {
-        conn->to << WorkerProto::Op::ImportPaths;
-
         auto source2 = sinkToSource([&](Sink & sink) {
             sink << 1 // == path follows
                 ;
@@ -558,11 +493,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
                 << 0 // == no path follows
                 ;
         });
-
-        conn.processStderr(0, source2.get());
-
-        auto importedPaths = WorkerProto::Serialise<StorePathSet>::read(*this, *conn);
-        assert(importedPaths.size() <= 1);
+        conn->importPaths(*this, &conn.daemonException, *source2);
     }
 
     else {
@@ -807,9 +738,7 @@ BuildResult RemoteStore::buildDerivation(const StorePath & drvPath, const BasicD
     BuildMode buildMode)
 {
     auto conn(getConnection());
-    conn->to << WorkerProto::Op::BuildDerivation << printStorePath(drvPath);
-    writeDerivation(conn->to, *this, drv);
-    conn->to << buildMode;
+    conn->putBuildDerivationRequest(*this, &conn.daemonException, drvPath, drv, buildMode);
     conn.processStderr();
     return WorkerProto::Serialise<BuildResult>::read(*this, *conn);
 }
@@ -827,9 +756,7 @@ void RemoteStore::ensurePath(const StorePath & path)
 void RemoteStore::addTempRoot(const StorePath & path)
 {
     auto conn(getConnection());
-    conn->to << WorkerProto::Op::AddTempRoot << printStorePath(path);
-    conn.processStderr();
-    readInt(conn->from);
+    conn->addTempRoot(*this, &conn.daemonException, path);
 }
 
 
@@ -969,22 +896,12 @@ void RemoteStore::flushBadConnections()
     connections->flushBad();
 }
 
-
-RemoteStore::Connection::~Connection()
-{
-    try {
-        to.flush();
-    } catch (...) {
-        ignoreException();
-    }
-}
-
 void RemoteStore::narFromPath(const StorePath & path, Sink & sink)
 {
-    auto conn(connections->get());
-    conn->to << WorkerProto::Op::NarFromPath << printStorePath(path);
-    conn->processStderr();
-    copyNAR(conn->from, sink);
+    auto conn(getConnection());
+    conn->narFromPath(*this, &conn.daemonException, path, [&](Source & source) {
+        copyNAR(conn->from, sink);
+    });
 }
 
 ref<SourceAccessor> RemoteStore::getFSAccessor(bool requireValidPath)
@@ -992,91 +909,6 @@ ref<SourceAccessor> RemoteStore::getFSAccessor(bool requireValidPath)
     return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this()));
 }
 
-static Logger::Fields readFields(Source & from)
-{
-    Logger::Fields fields;
-    size_t size = readInt(from);
-    for (size_t n = 0; n < size; n++) {
-        auto type = (decltype(Logger::Field::type)) readInt(from);
-        if (type == Logger::Field::tInt)
-            fields.push_back(readNum<uint64_t>(from));
-        else if (type == Logger::Field::tString)
-            fields.push_back(readString(from));
-        else
-            throw Error("got unsupported field type %x from Nix daemon", (int) type);
-    }
-    return fields;
-}
-
-
-std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source * source, bool flush)
-{
-    if (flush)
-        to.flush();
-
-    while (true) {
-
-        auto msg = readNum<uint64_t>(from);
-
-        if (msg == STDERR_WRITE) {
-            auto s = readString(from);
-            if (!sink) throw Error("no sink");
-            (*sink)(s);
-        }
-
-        else if (msg == STDERR_READ) {
-            if (!source) throw Error("no source");
-            size_t len = readNum<size_t>(from);
-            auto buf = std::make_unique<char[]>(len);
-            writeString({(const char *) buf.get(), source->read(buf.get(), len)}, to);
-            to.flush();
-        }
-
-        else if (msg == STDERR_ERROR) {
-            if (GET_PROTOCOL_MINOR(daemonVersion) >= 26) {
-                return std::make_exception_ptr(readError(from));
-            } else {
-                auto error = readString(from);
-                unsigned int status = readInt(from);
-                return std::make_exception_ptr(Error(status, error));
-            }
-        }
-
-        else if (msg == STDERR_NEXT)
-            printError(chomp(readString(from)));
-
-        else if (msg == STDERR_START_ACTIVITY) {
-            auto act = readNum<ActivityId>(from);
-            auto lvl = (Verbosity) readInt(from);
-            auto type = (ActivityType) readInt(from);
-            auto s = readString(from);
-            auto fields = readFields(from);
-            auto parent = readNum<ActivityId>(from);
-            logger->startActivity(act, lvl, type, s, fields, parent);
-        }
-
-        else if (msg == STDERR_STOP_ACTIVITY) {
-            auto act = readNum<ActivityId>(from);
-            logger->stopActivity(act);
-        }
-
-        else if (msg == STDERR_RESULT) {
-            auto act = readNum<ActivityId>(from);
-            auto type = (ResultType) readInt(from);
-            auto fields = readFields(from);
-            logger->result(act, type, fields);
-        }
-
-        else if (msg == STDERR_LAST)
-            break;
-
-        else
-            throw Error("got unknown message type %x from Nix daemon", msg);
-    }
-
-    return nullptr;
-}
-
 void RemoteStore::ConnectionHandle::withFramedSink(std::function<void(Sink & sink)> fun)
 {
     (*this)->to.flush();

src/libstore/serve-protocol-connection.cc

@@ -1,3 +1,4 @@
+#include "serve-protocol-connection.hh"
 #include "serve-protocol-impl.hh"
 #include "build-result.hh"
 #include "derivations.hh"
@@ -5,10 +6,7 @@
 namespace nix {
 
 ServeProto::Version ServeProto::BasicClientConnection::handshake(
-    BufferedSink & to,
-    Source & from,
-    ServeProto::Version localVersion,
-    std::string_view host)
+    BufferedSink & to, Source & from, ServeProto::Version localVersion, std::string_view host)
 {
     to << SERVE_MAGIC_1 << localVersion;
     to.flush();
@@ -22,39 +20,30 @@ ServeProto::Version ServeProto::BasicClientConnection::handshake(
     return std::min(remoteVersion, localVersion);
 }
 
-ServeProto::Version ServeProto::BasicServerConnection::handshake(
-    BufferedSink & to,
-    Source & from,
-    ServeProto::Version localVersion)
+ServeProto::Version
+ServeProto::BasicServerConnection::handshake(BufferedSink & to, Source & from, ServeProto::Version localVersion)
 {
     unsigned int magic = readInt(from);
-    if (magic != SERVE_MAGIC_1) throw Error("protocol mismatch");
+    if (magic != SERVE_MAGIC_1)
+        throw Error("protocol mismatch");
     to << SERVE_MAGIC_2 << localVersion;
     to.flush();
     auto remoteVersion = readInt(from);
     return std::min(remoteVersion, localVersion);
 }
 
-
 StorePathSet ServeProto::BasicClientConnection::queryValidPaths(
-    const Store & store,
-    bool lock, const StorePathSet & paths,
-    SubstituteFlag maybeSubstitute)
+    const StoreDirConfig & store, bool lock, const StorePathSet & paths, SubstituteFlag maybeSubstitute)
 {
-    to
-        << ServeProto::Command::QueryValidPaths
-        << lock
-        << maybeSubstitute;
+    to << ServeProto::Command::QueryValidPaths << lock << maybeSubstitute;
     write(store, *this, paths);
     to.flush();
 
     return Serialise<StorePathSet>::read(store, *this);
 }
 
-
-std::map<StorePath, UnkeyedValidPathInfo> ServeProto::BasicClientConnection::queryPathInfos(
-    const Store & store,
-    const StorePathSet & paths)
+std::map<StorePath, UnkeyedValidPathInfo>
+ServeProto::BasicClientConnection::queryPathInfos(const StoreDirConfig & store, const StorePathSet & paths)
 {
     std::map<StorePath, UnkeyedValidPathInfo> infos;
 
@@ -64,7 +53,8 @@ std::map<StorePath, UnkeyedValidPathInfo> ServeProto::BasicClientConnection::que
 
     while (true) {
         auto storePathS = readString(from);
-        if (storePathS == "") break;
+        if (storePathS == "")
+            break;
 
         auto storePath = store.parseStorePath(storePathS);
         assert(paths.count(storePath) == 1);
@@ -75,15 +65,13 @@ std::map<StorePath, UnkeyedValidPathInfo> ServeProto::BasicClientConnection::que
     return infos;
 }
 
-
 void ServeProto::BasicClientConnection::putBuildDerivationRequest(
-    const Store & store,
-    const StorePath & drvPath, const BasicDerivation & drv,
+    const StoreDirConfig & store,
+    const StorePath & drvPath,
+    const BasicDerivation & drv,
     const ServeProto::BuildOptions & options)
 {
-    to
-        << ServeProto::Command::BuildDerivation
-        << store.printStorePath(drvPath);
+    to << ServeProto::Command::BuildDerivation << store.printStorePath(drvPath);
     writeDerivation(to, store, drv);
 
     ServeProto::write(store, *this, options);
@@ -91,4 +79,28 @@ void ServeProto::BasicClientConnection::putBuildDerivationRequest(
     to.flush();
 }
 
+BuildResult ServeProto::BasicClientConnection::getBuildDerivationResponse(const StoreDirConfig & store)
+{
+    return ServeProto::Serialise<BuildResult>::read(store, *this);
+}
+
+void ServeProto::BasicClientConnection::narFromPath(
+    const StoreDirConfig & store, const StorePath & path, std::function<void(Source &)> fun)
+{
+    to << ServeProto::Command::DumpStorePath << store.printStorePath(path);
+    to.flush();
+
+    fun(from);
+}
+
+void ServeProto::BasicClientConnection::importPaths(const StoreDirConfig & store, std::function<void(Sink &)> fun)
+{
+    to << ServeProto::Command::ImportPaths;
+    fun(to);
+    to.flush();
+
+    if (readInt(from) != 1)
+        throw Error("remote machine failed to import closure");
+}
+
 }

src/libstore/serve-protocol-connection.hh

@@ -0,0 +1,108 @@
+#pragma once
+///@file
+
+#include "serve-protocol.hh"
+#include "store-api.hh"
+
+namespace nix {
+
+struct ServeProto::BasicClientConnection
+{
+    FdSink to;
+    FdSource from;
+    ServeProto::Version remoteVersion;
+
+    /**
+     * Establishes connection, negotiating version.
+     *
+     * @return the version provided by the other side of the
+     * connection.
+     *
+     * @param to Taken by reference to allow for various error handling
+     * mechanisms.
+     *
+     * @param from Taken by reference to allow for various error
+     * handling mechanisms.
+     *
+     * @param localVersion Our version which is sent over
+     *
+     * @param host Just used to add context to thrown exceptions.
+     */
+    static ServeProto::Version
+    handshake(BufferedSink & to, Source & from, ServeProto::Version localVersion, std::string_view host);
+
+    /**
+     * Coercion to `ServeProto::ReadConn`. This makes it easy to use the
+     * factored out serve protocol serializers with a
+     * `LegacySSHStore::Connection`.
+     *
+     * The serve protocol connection types are unidirectional, unlike
+     * this type.
+     */
+    operator ServeProto::ReadConn()
+    {
+        return ServeProto::ReadConn{
+            .from = from,
+            .version = remoteVersion,
+        };
+    }
+
+    /**
+     * Coercion to `ServeProto::WriteConn`. This makes it easy to use the
+     * factored out serve protocol serializers with a
+     * `LegacySSHStore::Connection`.
+     *
+     * The serve protocol connection types are unidirectional, unlike
+     * this type.
+     */
+    operator ServeProto::WriteConn()
+    {
+        return ServeProto::WriteConn{
+            .to = to,
+            .version = remoteVersion,
+        };
+    }
+
+    StorePathSet queryValidPaths(
+        const StoreDirConfig & remoteStore, bool lock, const StorePathSet & paths, SubstituteFlag maybeSubstitute);
+
+    std::map<StorePath, UnkeyedValidPathInfo> queryPathInfos(const StoreDirConfig & store, const StorePathSet & paths);
+    ;
+
+    void putBuildDerivationRequest(
+        const StoreDirConfig & store,
+        const StorePath & drvPath,
+        const BasicDerivation & drv,
+        const ServeProto::BuildOptions & options);
+
+    /**
+     * Get the response, must be paired with
+     * `putBuildDerivationRequest`.
+     */
+    BuildResult getBuildDerivationResponse(const StoreDirConfig & store);
+
+    void narFromPath(const StoreDirConfig & store, const StorePath & path, std::function<void(Source &)> fun);
+
+    void importPaths(const StoreDirConfig & store, std::function<void(Sink &)> fun);
+};
+
+struct ServeProto::BasicServerConnection
+{
+    /**
+     * Establishes connection, negotiating version.
+     *
+     * @return the version provided by the other side of the
+     * connection.
+     *
+     * @param to Taken by reference to allow for various error handling
+     * mechanisms.
+     *
+     * @param from Taken by reference to allow for various error
+     * handling mechanisms.
+     *
+     * @param localVersion Our version which is sent over
+     */
+    static ServeProto::Version handshake(BufferedSink & to, Source & from, ServeProto::Version localVersion);
+};
+
+}

src/libstore/serve-protocol-impl.hh

@@ -57,105 +57,4 @@ struct ServeProto::Serialise
 
 /* protocol-specific templates */
 
-struct ServeProto::BasicClientConnection
-{
-    FdSink to;
-    FdSource from;
-    ServeProto::Version remoteVersion;
-
-    /**
-     * Establishes connection, negotiating version.
-     *
-     * @return the version provided by the other side of the
-     * connection.
-     *
-     * @param to Taken by reference to allow for various error handling
-     * mechanisms.
-     *
-     * @param from Taken by reference to allow for various error
-     * handling mechanisms.
-     *
-     * @param localVersion Our version which is sent over
-     *
-     * @param host Just used to add context to thrown exceptions.
-     */
-    static ServeProto::Version handshake(
-        BufferedSink & to,
-        Source & from,
-        ServeProto::Version localVersion,
-        std::string_view host);
-
-    /**
-     * Coercion to `ServeProto::ReadConn`. This makes it easy to use the
-     * factored out serve protocol serializers with a
-     * `LegacySSHStore::Connection`.
-     *
-     * The serve protocol connection types are unidirectional, unlike
-     * this type.
-     */
-    operator ServeProto::ReadConn ()
-    {
-        return ServeProto::ReadConn {
-            .from = from,
-            .version = remoteVersion,
-        };
-    }
-
-    /**
-     * Coercion to `ServeProto::WriteConn`. This makes it easy to use the
-     * factored out serve protocol serializers with a
-     * `LegacySSHStore::Connection`.
-     *
-     * The serve protocol connection types are unidirectional, unlike
-     * this type.
-     */
-    operator ServeProto::WriteConn ()
-    {
-        return ServeProto::WriteConn {
-            .to = to,
-            .version = remoteVersion,
-        };
-    }
-
-    StorePathSet queryValidPaths(
-        const Store & remoteStore,
-        bool lock, const StorePathSet & paths,
-        SubstituteFlag maybeSubstitute);
-
-    std::map<StorePath, UnkeyedValidPathInfo> queryPathInfos(
-        const Store & store,
-        const StorePathSet & paths);
-
-    /**
-     * Just the request half, because Hydra may do other things between
-     * issuing the request and reading the `BuildResult` response.
-     */
-    void putBuildDerivationRequest(
-        const Store & store,
-        const StorePath & drvPath, const BasicDerivation & drv,
-        const ServeProto::BuildOptions & options);
-};
-
-struct ServeProto::BasicServerConnection
-{
-    /**
-     * Establishes connection, negotiating version.
-     *
-     * @return the version provided by the other side of the
-     * connection.
-     *
-     * @param to Taken by reference to allow for various error handling
-     * mechanisms.
-     *
-     * @param from Taken by reference to allow for various error
-     * handling mechanisms.
-     *
-     * @param localVersion Our version which is sent over
-     */
-    static ServeProto::Version handshake(
-        BufferedSink & to,
-        Source & from,
-        ServeProto::Version localVersion);
-};
-
 }

src/libstore/ssh-store-config.cc

@@ -1,10 +1,11 @@
 #include <regex>
 
 #include "ssh-store-config.hh"
+#include "ssh.hh"
 
 namespace nix {
 
-std::string CommonSSHStoreConfig::extractConnStr(std::string_view scheme, std::string_view _connStr)
+static std::string extractConnStr(std::string_view scheme, std::string_view _connStr)
 {
     if (_connStr.empty())
         throw UsageError("`%s` store requires a valid SSH host as the authority part in Store URI", scheme);
@@ -21,4 +22,22 @@ std::string CommonSSHStoreConfig::extractConnStr(std::string_view scheme, std::s
     return connStr;
 }
 
+CommonSSHStoreConfig::CommonSSHStoreConfig(std::string_view scheme, std::string_view host, const Params & params)
+    : StoreConfig(params)
+    , host(extractConnStr(scheme, host))
+{
+}
+
+SSHMaster CommonSSHStoreConfig::createSSHMaster(bool useMaster, Descriptor logFD)
+{
+    return {
+        host,
+        sshKey.get(),
+        sshPublicHostKey.get(),
+        useMaster,
+        compress,
+        logFD,
+    };
+}
+
 }

src/libstore/ssh-store-config.hh

@@ -5,10 +5,14 @@
 
 namespace nix {
 
+class SSHMaster;
+
 struct CommonSSHStoreConfig : virtual StoreConfig
 {
     using StoreConfig::StoreConfig;
 
+    CommonSSHStoreConfig(std::string_view scheme, std::string_view host, const Params & params);
+
     const Setting<Path> sshKey{this, "", "ssh-key",
         "Path to the SSH private key used to authenticate to the remote machine."};
 
@@ -30,9 +34,7 @@ struct CommonSSHStoreConfig : virtual StoreConfig
      * RFC2732, but also pure addresses. The latter one is needed here to
      * connect to a remote store via SSH (it's possible to do e.g. `ssh root@::1`).
      *
-     * This function now ensures that a usable connection string is available:
-     *
-     * - If the store to be opened is not an SSH store, nothing will be done.
+     * When initialized, the following adjustments are made:
      *
      * - If the URL looks like `root@[::1]` (which is allowed by the URL parser and probably
      *   needed to pass further flags), it
@@ -44,9 +46,17 @@ struct CommonSSHStoreConfig : virtual StoreConfig
      *
      * Will throw an error if `connStr` is empty too.
      */
-    static std::string extractConnStr(
-        std::string_view scheme,
-        std::string_view connStr);
+    std::string host;
+
+    /**
+     * Small wrapper around `SSHMaster::SSHMaster` that gets most
+     * arguments from this configuration.
+     *
+     * See that constructor for details on the remaining two arguments.
+     */
+    SSHMaster createSSHMaster(
+        bool useMaster,
+        Descriptor logFD = INVALID_DESCRIPTOR);
 };
 
 }

src/libstore/ssh-store.cc

@@ -32,34 +32,21 @@ struct SSHStoreConfig : virtual RemoteStoreConfig, virtual CommonSSHStoreConfig
 
 class SSHStore : public virtual SSHStoreConfig, public virtual RemoteStore
 {
-    SSHStore(
-        std::string_view scheme,
-        std::string host,
-        const Params & params)
-        : StoreConfig(params)
-        , RemoteStoreConfig(params)
-        , CommonSSHStoreConfig(params)
-        , SSHStoreConfig(params)
-        , Store(params)
-        , RemoteStore(params)
-        , host(host)
-        , master(
-            host,
-            sshKey.get(),
-            sshPublicHostKey.get(),
-            // Use SSH master only if using more than 1 connection.
-            connections->capacity() > 1,
-            compress)
-    {
-    }
-
 public:
 
     SSHStore(
         std::string_view scheme,
         std::string_view host,
         const Params & params)
-        : SSHStore{scheme, SSHStoreConfig::extractConnStr(scheme, host), params}
+        : StoreConfig(params)
+        , RemoteStoreConfig(params)
+        , CommonSSHStoreConfig(scheme, host, params)
+        , SSHStoreConfig(params)
+        , Store(params)
+        , RemoteStore(params)
+        , master(createSSHMaster(
+            // Use SSH master only if using more than 1 connection.
+            connections->capacity() > 1))
     {
     }
 
@@ -119,6 +106,15 @@ struct MountedSSHStoreConfig : virtual SSHStoreConfig, virtual LocalFSStoreConfi
     {
     }
 
+    MountedSSHStoreConfig(std::string_view scheme, std::string_view host, StringMap params)
+        : StoreConfig(params)
+        , RemoteStoreConfig(params)
+        , CommonSSHStoreConfig(scheme, host, params)
+        , SSHStoreConfig(params)
+        , LocalFSStoreConfig(params)
+    {
+    }
+
     const std::string name() override { return "Experimental SSH Store with filesystem mounted"; }
 
     std::string doc() override
@@ -158,7 +154,7 @@ public:
         const Params & params)
         : StoreConfig(params)
         , RemoteStoreConfig(params)
-        , CommonSSHStoreConfig(params)
+        , CommonSSHStoreConfig(scheme, host, params)
         , SSHStoreConfig(params)
         , LocalFSStoreConfig(params)
         , MountedSSHStoreConfig(params)