Merge remote-tracking branch 'cve/fod-cves-2.28' into 2.28-maintenance

Make the repl test more robust (backport #13348)

.version

@@ -1 +1 @@
-2.28.1
+2.28.4

doc/manual/generate-deps.py

@@ -14,7 +14,7 @@ import sys
 # literally. since the rules for these aren't even the same for
 # all three we will just fail when we encounter any of them (if
 # asserts are off for some reason the depfile will likely point
-# to nonexistant paths, making everything phony and thus fine.)
+# to nonexistent paths, making everything phony and thus fine.)
 for path in glob.glob(sys.argv[1] + '/**', recursive=True):
     assert '\\' not in path
     assert ' ' not in path

doc/manual/meson.build

@@ -67,7 +67,7 @@ subdir('source/release-notes')
 subdir('source')
 
 # Hacky way to figure out if `nix` is an `ExternalProgram` or
-# `Exectuable`. Only the latter can occur in custom target input lists.
+# `Executable`. Only the latter can occur in custom target input lists.
 if nix.full_path().startswith(meson.build_root())
   nix_input = nix
 else

doc/manual/source/advanced-topics/distributed-builds.md

@@ -27,7 +27,7 @@ nix store info --store ssh://username@mac
 ```
 
 To specify an SSH identity file as part of the remote store URI add a
-query paramater, e.g.
+query parameter, e.g.
 
 ```console
 nix store info --store ssh://username@mac?ssh-key=/home/alice/my-key

doc/manual/source/architecture/architecture.md

@@ -22,9 +22,9 @@ The following [concept map] shows its main components (rectangles), the objects
            |                   |
 +----------|-------------------|--------------------------------+
 | Nix      |                   V                                |
-|          |      +-------------------------+                   |
-|          |      | commmand line interface |------.            |
-|          |      +-------------------------+      |            |
+|          |       +------------------------+                   |
+|          |       | command line interface |------.            |
+|          |       +------------------------+      |            |
 |          |                   |                   |            |
 |    evaluated by            calls              manages         |
 |          |                   |                   |            |

doc/manual/source/command-ref/nix-env/delete-generations.md

@@ -27,7 +27,7 @@ This operation deletes the specified generations of the current profile.
   >
   > Older *and newer* generations will be deleted by this operation.
   >
-  > One might expect this to just delete older generations than the curent one, but that is only true if the current generation is also the latest.
+  > One might expect this to just delete older generations than the current one, but that is only true if the current generation is also the latest.
   > Because one can roll back to a previous generation, it is possible to have generations newer than the current one.
   > They will also be deleted.
 

doc/manual/source/development/building.md

@@ -195,19 +195,25 @@ Nix uses a string with the following format to identify the *system type* or *pl
 <cpu>-<os>[-<abi>]
 ```
 
-It is set when Nix is compiled for the given system, and based on the output of [`config.guess`](https://github.com/nixos/nix/blob/master/config/config.guess) ([upstream](https://git.savannah.gnu.org/cgit/config.git/tree/config.guess)):
+It is set when Nix is compiled for the given system, and based on the output of Meson's [`host_machine` information](https://mesonbuild.com/Reference-manual_builtin_host_machine.html)>
 
 ```
 <cpu>-<vendor>-<os>[<version>][-<abi>]
 ```
 
-When Nix is built such that `./configure` is passed any of the `--host`, `--build`, `--target` options, the value is based on the output of [`config.sub`](https://github.com/nixos/nix/blob/master/config/config.sub) ([upstream](https://git.savannah.gnu.org/cgit/config.git/tree/config.sub)):
+When cross-compiling Nix with Meson for local development, you need to specify a [cross-file](https://mesonbuild.com/Cross-compilation.html) using the `--cross-file` option. Cross-files define the target architecture and toolchain. When cross-compiling Nix with Nix, Nixpkgs takes care of this for you.
+
+In the nix flake we also have some cross-compilation targets available:
 
 ```
-<cpu>-<vendor>[-<kernel>]-<os>
+nix build .#nix-everything-riscv64-unknown-linux-gnu
+nix build .#nix-everything-armv7l-unknown-linux-gnueabihf
+nix build .#nix-everything-armv7l-unknown-linux-gnueabihf
+nix build .#nix-everything-x86_64-unknown-freebsd
+nix build .#nix-everything-x86_64-w64-mingw32
 ```
 
-For historic reasons and backward-compatibility, some CPU and OS identifiers are translated from the GNU Autotools naming convention in [`configure.ac`](https://github.com/nixos/nix/blob/master/configure.ac) as follows:
+For historic reasons and backward-compatibility, some CPU and OS identifiers are translated as follows:
 
 | `config.guess`             | Nix                 |
 |----------------------------|---------------------|
@@ -230,18 +236,18 @@ Nix can be compiled using multiple environments:
 To build with one of those environments, you can use
 
 ```console
-$ nix build .#nix-ccacheStdenv
+$ nix build .#nix-cli-ccacheStdenv
 ```
 
 for flake-enabled Nix, or
 
 ```console
-$ nix-build --attr nix-ccacheStdenv
+$ nix-build --attr nix-cli-ccacheStdenv
 ```
 
 for classic Nix.
 
-You can use any of the other supported environments in place of `nix-ccacheStdenv`.
+You can use any of the other supported environments in place of `nix-cli-ccacheStdenv`.
 
 ## Editor integration
 

doc/manual/source/development/cli-guideline.md

@@ -170,9 +170,9 @@ sensitive.
 
 
 ```shell
-$ nix init --template=template#pyton
+$ nix init --template=template#python
 ------------------------------------------------------------------------
-  Error! Template `template#pyton` not found.
+  Error! Template `template#python` not found.
 ------------------------------------------------------------------------
 Initializing Nix project at `/path/to/here`.
       Select a template for you new project:

doc/manual/source/glossary.md

@@ -320,7 +320,7 @@
 
   See [Nix Archive](store/file-system-object/content-address.html#serial-nix-archive) for details.
 
-- [`∅`]{#gloss-emtpy-set}
+- [`∅`]{#gloss-empty-set}
 
   The empty set symbol. In the context of profile history, this denotes a package is not present in a particular version of the profile.
 

doc/manual/source/language/advanced-attributes.md

@@ -73,7 +73,7 @@ Derivations can declare some infrequently used optional attributes.
 
     > **Warning**
     >
-    > If set to `true`, other advanced attributes such as [`allowedReferences`](#adv-attr-allowedReferences), [`allowedReferences`](#adv-attr-allowedReferences), [`allowedRequisites`](#adv-attr-allowedRequisites),
+    > If set to `true`, other advanced attributes such as [`allowedReferences`](#adv-attr-allowedReferences), [`allowedRequisites`](#adv-attr-allowedRequisites),
     [`disallowedReferences`](#adv-attr-disallowedReferences) and [`disallowedRequisites`](#adv-attr-disallowedRequisites), maxSize, and maxClosureSize.
     will have no effect.
 
@@ -280,7 +280,7 @@ All other combinations are invalid.
 
   <!--
 
-  `__contentAddressed` is ignored, becaused fixed-output derivations always content-address their outputs, by definition.
+  `__contentAddressed` is ignored, because fixed-output derivations always content-address their outputs, by definition.
 
   **TODO CHECK**
 
@@ -333,17 +333,17 @@ Here is more information on the `output*` attributes, and what values they may b
 
   - [`outputHashAlgo`]{#adv-attr-outputHashAlgo}
 
-    This specifies the hash alorithm used to digest the [file system object] data of a content-addressing derivation output.
+    This specifies the hash algorithm used to digest the [file system object] data of a content-addressing derivation output.
 
     This works in conjunction with [`outputHashMode`](#adv-attr-outputHashAlgo).
-    Specifying one without the other is an error (unless [`outputHash` is also specified and includes its own hash algorithm as described below).
+    Specifying one without the other is an error (unless `outputHash` is also specified and includes its own hash algorithm as described below).
 
     The `outputHashAlgo` attribute specifies the hash algorithm used to compute the hash.
-    It can currently be `"blake3"`, "sha1"`, `"sha256"`, `"sha512"`, or `null`.
+    It can currently be `"blake3"`, `"sha1"`, `"sha256"`, `"sha512"`, or `null`.
 
     `outputHashAlgo` can only be `null` when `outputHash` follows the SRI format, because in that case the choice of hash algorithm is determined by `outputHash`.
 
-  - [`outputHash`]{#adv-attr-outputHashAlgo}; [`outputHash`]{#adv-attr-outputHashMode}\
+  - [`outputHash`]{#adv-attr-outputHashAlgo}; [`outputHash`]{#adv-attr-outputHashMode}
 
     This will specify the output hash of the single output of a [fixed-output derivation].
 

doc/manual/source/language/string-context.md

@@ -115,7 +115,7 @@ It creates an [attribute set] representing the string context, which can be insp
 
 ## Clearing string contexts
 
-[`buitins.unsafeDiscardStringContext`](./builtins.md#builtins-unsafeDiscardStringContext) will make a copy of a string, but with an empty string context.
+[`builtins.unsafeDiscardStringContext`](./builtins.md#builtins-unsafeDiscardStringContext) will make a copy of a string, but with an empty string context.
 The returned string can be used in more ways, e.g. by operators that require the string context to be empty.
 The requirement to explicitly discard the string context in such use cases helps ensure that string context elements are not lost by mistake.
 The "unsafe" marker is only there to remind that Nix normally guarantees that dependencies are tracked, whereas the returned string has lost them.

doc/manual/source/protocols/json/derivation.md

@@ -24,7 +24,7 @@ is a JSON object with the following fields:
 
 
   * `method`:
-    For an output which will be [content addresed], a string representing the [method](@docroot@/store/store-object/content-address.md) of content addressing that is chosen.
+    For an output which will be [content addressed], a string representing the [method](@docroot@/store/store-object/content-address.md) of content addressing that is chosen.
     Valid method strings are:
 
     - [`flat`](@docroot@/store/store-object/content-address.md#method-flat)
@@ -35,7 +35,7 @@ is a JSON object with the following fields:
     Otherwise, `null`.
 
   * `hashAlgo`:
-    For an output which will be [content addresed], the name of the hash algorithm used.
+    For an output which will be [content addressed], the name of the hash algorithm used.
     Valid algorithm strings are:
 
     - `blake3`

doc/manual/source/protocols/tarball-fetcher.md

@@ -46,7 +46,7 @@ defined as the timestamp of the newest file inside the tarball.
 This protocol is supported by Gitea since v1.22.1 and by Forgejo since v7.0.4/v8.0.0 and can be used with the following flake URL schema:
 
 ```
-https://<domain name>/<owner>/<repo>/archive/<reference or revison>.tar.gz
+https://<domain name>/<owner>/<repo>/archive/<reference or revision>.tar.gz
 ```
 
 > **Example**

doc/manual/source/release-notes/rl-2.19.md

@@ -31,7 +31,7 @@
     - To operate on a flake outside the current directory, you must now pass `--flake path/to/flake`.
 
   - The flake-specific flags `--recreate-lock-file` and `--update-input` have been removed from all commands operating on installables.
-    They are superceded by `nix flake update`.
+    They are superseded by `nix flake update`.
 
 - Commit signature verification for the [`builtins.fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit) is added as the new [`verified-fetches` experimental feature](@docroot@/development/experimental-features.md#xp-feature-verified-fetches).
 

doc/manual/source/release-notes/rl-2.23.md

@@ -15,7 +15,7 @@
 - Modify `nix derivation {add,show}` JSON format [#9866](https://github.com/NixOS/nix/issues/9866) [#10722](https://github.com/NixOS/nix/pull/10722)
 
   The JSON format for derivations has been slightly revised to better conform to our [JSON guidelines](@docroot@/development/cli-guideline.md#returning-future-proof-json).
-  In particular, the hash algorithm and content addressing method of content-addresed derivation outputs are now separated into two fields `hashAlgo` and `method`,
+  In particular, the hash algorithm and content addressing method of content-addressed derivation outputs are now separated into two fields `hashAlgo` and `method`,
   rather than one field with an arcane `:`-separated format.
 
   This JSON format is only used by the experimental `nix derivation` family of commands, at this time.

doc/manual/source/release-notes/rl-2.24.md

@@ -173,7 +173,7 @@
   **Deprecation**: Use `nix32` instead of `base32` as `toHashFormat`
 
   For the builtin `convertHash`, the `toHashFormat` parameter now accepts the same hash formats as the `--to`/`--from`
-  parameters of the `nix hash conert` command: `"base16"`, `"nix32"`, `"base64"`, and `"sri"`. The former `"base32"` value
+  parameters of the `nix hash convert` command: `"base16"`, `"nix32"`, `"base64"`, and `"sri"`. The former `"base32"` value
   remains as a deprecated alias for `"nix32"`. Please convert your code from:
 
   ```nix

doc/manual/source/release-notes/rl-2.27.md

@@ -38,6 +38,15 @@
 
   Curl created sockets without setting `FD_CLOEXEC`/`SOCK_CLOEXEC`. This could previously cause connections to remain open forever when using commands like `nix shell`. This change sets the `FD_CLOEXEC` flag using a `CURLOPT_SOCKOPTFUNCTION` callback.
 
+- Add BLAKE3 hash algorithm [#12379](https://github.com/NixOS/nix/pull/12379)
+
+  Nix now supports the BLAKE3 hash algorithm as an experimental feature (`blake3-hashes`):
+
+  ```console
+  # nix hash file ./file --type blake3 --extra-experimental-features blake3-hashes
+  blake3-34P4p+iZXcbbyB1i4uoF7eWCGcZHjmaRn6Y7QdynLwU=
+  ```
+
 # Contributors
 
 This release was made possible by the following 21 contributors:

doc/manual/source/release-notes/rl-2.6.md

@@ -13,7 +13,7 @@
 * New command `nix store copy-log` to copy build logs from one store
   to another.
 * The `commit-lockfile-summary` option can be set to a non-empty
-  string to override the commit summary used when commiting an updated
+  string to override the commit summary used when committing an updated
   lockfile.  This may be used in conjunction with the `nixConfig`
   attribute in `flake.nix` to better conform to repository
   conventions.

doc/manual/source/store/derivation/outputs/content-address.md

@@ -23,7 +23,7 @@ The output spec for an output with a fixed content addresses additionally contai
 > **Design note**
 >
 > In principle, the output spec could also specify the references the store object should have, since the references and file system objects are equally parts of a content-addressed store object proper that contribute to its content-addressed.
-> However, at this time, the references are not not done because all fixed content-addressed outputs are required to have no references (including no self-reference).
+> However, at this time, the references are not done because all fixed content-addressed outputs are required to have no references (including no self-reference).
 >
 > Also in principle, rather than specifying the references and file system object data with separate hashes, a single hash that constraints both could be used.
 > This could be done with the final store path's digest, or better yet, the hash that will become the store path's digest before it is truncated.
@@ -110,18 +110,18 @@ Because the derivation output is not fixed (just like with [input addressing]),
 >
 > Strictly speaking, the extent to which sandboxing and deprivilaging is possible varies with the environment Nix is running in.
 > Nix's configuration settings indicate what level of sandboxing is required or enabled.
-> Builds of derivations will fail if they request an absense of sandboxing which is not allowed.
-> Builds of derivations will also fail if the level of sandboxing specified in the configure exceeds what is possible in teh given environment.
+> Builds of derivations will fail if they request an absence of sandboxing which is not allowed.
+> Builds of derivations will also fail if the level of sandboxing specified in the configure exceeds what is possible in the given environment.
 >
-> (The "environment", in this case, consists of attributes such as the Operating System Nix runs atop, along with the operating-system-specific privilages that Nix has been granted.
-> Because of how conventional operating systems like macos, Linux, etc. work, granting builders *fewer* privilages may ironically require that Nix be run with *more* privilages.)
+> (The "environment", in this case, consists of attributes such as the Operating System Nix runs atop, along with the operating-system-specific privileges that Nix has been granted.
+> Because of how conventional operating systems like macos, Linux, etc. work, granting builders *fewer* privileges may ironically require that Nix be run with *more* privileges.)
 
-That said, derivations producing floating content-addressed outputs may declare their builders as impure (like the builders of derivations producing producing fixed outputs).
+That said, derivations producing floating content-addressed outputs may declare their builders as impure (like the builders of derivations producing fixed outputs).
 This is provisionally supported as part of the [`impure-derivations`][xp-feature-impure-derivations] experimental feature.
 
 ### Compatibility negotiation
 
-Any derivation producing a floating content-addresssed output implicitly requires the `ca-derivations` [system feature](@docroot@/command-ref/conf-file.md#conf-system-features).
+Any derivation producing a floating content-addressed output implicitly requires the `ca-derivations` [system feature](@docroot@/command-ref/conf-file.md#conf-system-features).
 This prevents scheduling the building of the derivation on a machine without the experimental feature enabled.
 Even once the experimental feature is stabilized, this is still useful in order to be allow using remote builder running odler versions of Nix, or alternative implementations that do not support floating content addressing.
 
@@ -132,7 +132,7 @@ For store objects produced by manually inserting into the store to create a stor
 But for store objects produced by derivation, the "method is quite formal" --- the whole point of derivations is to be a formal notion of building, after all.
 In this case, we can elevate this informal property to a formal one.
 
-A *determinstic* content-addressing derivation should produce outputs with the same content addresses:
+A *deterministic* content-addressing derivation should produce outputs with the same content addresses:
 
 1. Every time the builder is run
 
@@ -144,7 +144,7 @@ A *determinstic* content-addressing derivation should produce outputs with the s
   The choice of provisional store path can be thought of as an impurity, since it is an arbitrary choice.
 
   If provisional outputs paths are deterministically chosen, we are in the first branch of part (1).
-  The builder the data it produces based on it in arbitrary ways, but this gets us closer to to [input addressing].
+  The builder the data it produces based on it in arbitrary ways, but this gets us closer to [input addressing].
   Deterministically choosing the provisional path may be considered "complete sandboxing" by removing an impurity, but this is unsatisfactory
 
   <!--

doc/manual/source/store/derivation/outputs/index.md

@@ -2,7 +2,7 @@
 
 As stated on the [main pages on derivations](../index.md#store-derivation),
 a derivation produces [store objects](@docroot@/store/store-object.md), which are known as the *outputs* of the derivation.
-Indeed, the entire point of derivations is to produce these outputs, and to reliably and reproducably produce these derivations each time the derivation is run.
+Indeed, the entire point of derivations is to produce these outputs, and to reliably and reproducibly produce these derivations each time the derivation is run.
 
 One of the parts of a derivation is its *outputs specification*, which specifies certain information about the outputs the derivation produces when run.
 The outputs specification is a map, from names to specifications for individual outputs.
@@ -83,11 +83,11 @@ The rules for this are fairly concise:
 
   - A content-addressing derivation may be pure or impure
 
-   - If it is impure, it may be be fixed (typical), or it may be floating if the additional [`impure-derivations`][xp-feature-impure-derivations] experimental feature is enabled.
+   - If it is impure, it may be fixed (typical), or it may be floating if the additional [`impure-derivations`][xp-feature-impure-derivations] experimental feature is enabled.
 
    - If it is pure, it must be floating.
 
-   - Pure, fixed content-addressing derivations are not suppported
+   - Pure, fixed content-addressing derivations are not supported
 
      > There is no use for this forth combination.
      > The sole purpose of an output's store path being fixed is to support the derivation being impure.

doc/manual/source/store/store-object/content-address.md

@@ -45,7 +45,7 @@ Self-references however cannot be referred to by their path, because we are in t
 > As far as we know, this is equivalent to finding a hash collision.
 
 Instead we have a "has self-reference" boolean, which ends up affecting the digest:
-In all currently-supported store object content-addressing methods, when hashing the file system object data, any occurence of store object's own store path in the digested data is replaced with a [sentinel value](https://en.wikipedia.org/wiki/Sentinel_value).
+In all currently-supported store object content-addressing methods, when hashing the file system object data, any occurrence of store object's own store path in the digested data is replaced with a [sentinel value](https://en.wikipedia.org/wiki/Sentinel_value).
 The hashes of these modified input streams are used instead.
 
 When validating the content address of a store object after the fact, the above process works as written.

flake.nix

@@ -143,14 +143,15 @@
           # without "polluting" the top level "`pkgs`" attrset.
           # This also has the benefit of providing us with a distinct set of packages
           # we can iterate over.
-          nixComponents =
+          # The `2` suffix is here because otherwise it interferes with `nixVersions.latest`, which is used in daemon compat tests.
+          nixComponents2 =
             lib.makeScopeWithSplicing'
               {
                 inherit (final) splicePackages;
-                inherit (final.nixDependencies) newScope;
+                inherit (final.nixDependencies2) newScope;
               }
               {
-                otherSplices = final.generateSplicesForMkScope "nixComponents";
+                otherSplices = final.generateSplicesForMkScope "nixComponents2";
                 f = import ./packaging/components.nix {
                   inherit (final) lib;
                   inherit officialRelease;
@@ -161,22 +162,23 @@
               };
 
           # The dependencies are in their own scope, so that they don't have to be
-          # in Nixpkgs top level `pkgs` or `nixComponents`.
-          nixDependencies =
+          # in Nixpkgs top level `pkgs` or `nixComponents2`.
+          # The `2` suffix is here because otherwise it interferes with `nixVersions.latest`, which is used in daemon compat tests.
+          nixDependencies2 =
             lib.makeScopeWithSplicing'
               {
                 inherit (final) splicePackages;
-                inherit (final) newScope; # layered directly on pkgs, unlike nixComponents above
+                inherit (final) newScope; # layered directly on pkgs, unlike nixComponents2 above
               }
               {
-                otherSplices = final.generateSplicesForMkScope "nixDependencies";
+                otherSplices = final.generateSplicesForMkScope "nixDependencies2";
                 f = import ./packaging/dependencies.nix {
                   inherit inputs stdenv;
                   pkgs = final;
                 };
               };
 
-          nix = final.nixComponents.nix-cli;
+          nix = final.nixComponents2.nix-cli;
 
           # See https://github.com/NixOS/nixpkgs/pull/214409
           # Remove when fixed in this flake's nixpkgs
@@ -277,7 +279,7 @@
                 # memory leaks with detect_leaks=0.
                 "" = rec {
                   nixpkgs = nixpkgsFor.${system}.native;
-                  nixComponents = nixpkgs.nixComponents.overrideScope (
+                  nixComponents = nixpkgs.nixComponents2.overrideScope (
                     nixCompFinal: nixCompPrev: {
                       mesonComponentOverrides = _finalAttrs: prevAttrs: {
                         mesonFlags =
@@ -305,7 +307,7 @@
               nixpkgsPrefix:
               {
                 nixpkgs,
-                nixComponents ? nixpkgs.nixComponents,
+                nixComponents ? nixpkgs.nixComponents2,
               }:
               flatMapAttrs nixComponents (
                 pkgName: pkg:
@@ -335,9 +337,9 @@
           binaryTarball = self.hydraJobs.binaryTarball.${system};
           # TODO probably should be `nix-cli`
           nix = self.packages.${system}.nix-everything;
-          nix-manual = nixpkgsFor.${system}.native.nixComponents.nix-manual;
-          nix-internal-api-docs = nixpkgsFor.${system}.native.nixComponents.nix-internal-api-docs;
-          nix-external-api-docs = nixpkgsFor.${system}.native.nixComponents.nix-external-api-docs;
+          nix-manual = nixpkgsFor.${system}.native.nixComponents2.nix-manual;
+          nix-internal-api-docs = nixpkgsFor.${system}.native.nixComponents2.nix-internal-api-docs;
+          nix-external-api-docs = nixpkgsFor.${system}.native.nixComponents2.nix-external-api-docs;
         }
         # We need to flatten recursive attribute sets of derivations to pass `flake check`.
         //
@@ -389,9 +391,9 @@
               }:
               {
                 # These attributes go right into `packages.<system>`.
-                "${pkgName}" = nixpkgsFor.${system}.native.nixComponents.${pkgName};
-                "${pkgName}-static" = nixpkgsFor.${system}.native.pkgsStatic.nixComponents.${pkgName};
-                "${pkgName}-llvm" = nixpkgsFor.${system}.native.pkgsLLVM.nixComponents.${pkgName};
+                "${pkgName}" = nixpkgsFor.${system}.native.nixComponents2.${pkgName};
+                "${pkgName}-static" = nixpkgsFor.${system}.native.pkgsStatic.nixComponents2.${pkgName};
+                "${pkgName}-llvm" = nixpkgsFor.${system}.native.pkgsLLVM.nixComponents2.${pkgName};
               }
               // lib.optionalAttrs supportsCross (
                 flatMapAttrs (lib.genAttrs crossSystems (_: { })) (
@@ -399,7 +401,7 @@
                   { }:
                   {
                     # These attributes go right into `packages.<system>`.
-                    "${pkgName}-${crossSystem}" = nixpkgsFor.${system}.cross.${crossSystem}.nixComponents.${pkgName};
+                    "${pkgName}-${crossSystem}" = nixpkgsFor.${system}.cross.${crossSystem}.nixComponents2.${pkgName};
                   }
                 )
               )
@@ -409,7 +411,7 @@
                 {
                   # These attributes go right into `packages.<system>`.
                   "${pkgName}-${stdenvName}" =
-                    nixpkgsFor.${system}.nativeForStdenv.${stdenvName}.nixComponents.${pkgName};
+                    nixpkgsFor.${system}.nativeForStdenv.${stdenvName}.nixComponents2.${pkgName};
                 }
               )
             )

maintainers/flake-module.nix

@@ -172,14 +172,14 @@
               ''^src/libfetchers/include/nix/fetchers/tarball\.hh$''
               ''^src/libfetchers/git\.cc$''
               ''^src/libfetchers/mercurial\.cc$''
-              ''^src/libflake/flake/config\.cc$''
-              ''^src/libflake/flake/flake\.cc$''
+              ''^src/libflake/config\.cc$''
+              ''^src/libflake/flake\.cc$''
               ''^src/libflake/include/nix/flake/flake\.hh$''
-              ''^src/libflake/flake/flakeref\.cc$''
+              ''^src/libflake/flakeref\.cc$''
               ''^src/libflake/include/nix/flake/flakeref\.hh$''
-              ''^src/libflake/flake/lockfile\.cc$''
+              ''^src/libflake/lockfile\.cc$''
               ''^src/libflake/include/nix/flake/lockfile\.hh$''
-              ''^src/libflake/flake/url-name\.cc$''
+              ''^src/libflake/url-name\.cc$''
               ''^src/libmain/common-args\.cc$''
               ''^src/libmain/include/nix/main/common-args\.hh$''
               ''^src/libmain/loggers\.cc$''

maintainers/release-notes

@@ -2,6 +2,8 @@
 # vim: set filetype=bash:
 #!nix shell .#changelog-d --command bash
 
+set -euo pipefail
+
 # --- CONFIGURATION ---
 
 # This does double duty for

packaging/dev-shell.nix

@@ -5,11 +5,11 @@
 
 { pkgs }:
 
-pkgs.nixComponents.nix-util.overrideAttrs (
+pkgs.nixComponents2.nix-util.overrideAttrs (
   attrs:
 
   let
-    stdenv = pkgs.nixDependencies.stdenv;
+    stdenv = pkgs.nixDependencies2.stdenv;
     buildCanExecuteHost = stdenv.buildPlatform.canExecute stdenv.hostPlatform;
     modular = devFlake.getSystem stdenv.buildPlatform.system;
     transformFlag =
@@ -79,26 +79,26 @@ pkgs.nixComponents.nix-util.overrideAttrs (
     };
 
     mesonFlags =
-      map (transformFlag "libutil") (ignoreCrossFile pkgs.nixComponents.nix-util.mesonFlags)
-      ++ map (transformFlag "libstore") (ignoreCrossFile pkgs.nixComponents.nix-store.mesonFlags)
-      ++ map (transformFlag "libfetchers") (ignoreCrossFile pkgs.nixComponents.nix-fetchers.mesonFlags)
+      map (transformFlag "libutil") (ignoreCrossFile pkgs.nixComponents2.nix-util.mesonFlags)
+      ++ map (transformFlag "libstore") (ignoreCrossFile pkgs.nixComponents2.nix-store.mesonFlags)
+      ++ map (transformFlag "libfetchers") (ignoreCrossFile pkgs.nixComponents2.nix-fetchers.mesonFlags)
       ++ lib.optionals havePerl (
-        map (transformFlag "perl") (ignoreCrossFile pkgs.nixComponents.nix-perl-bindings.mesonFlags)
+        map (transformFlag "perl") (ignoreCrossFile pkgs.nixComponents2.nix-perl-bindings.mesonFlags)
       )
-      ++ map (transformFlag "libexpr") (ignoreCrossFile pkgs.nixComponents.nix-expr.mesonFlags)
-      ++ map (transformFlag "libcmd") (ignoreCrossFile pkgs.nixComponents.nix-cmd.mesonFlags);
+      ++ map (transformFlag "libexpr") (ignoreCrossFile pkgs.nixComponents2.nix-expr.mesonFlags)
+      ++ map (transformFlag "libcmd") (ignoreCrossFile pkgs.nixComponents2.nix-cmd.mesonFlags);
 
     nativeBuildInputs =
       attrs.nativeBuildInputs or [ ]
-      ++ pkgs.nixComponents.nix-util.nativeBuildInputs
-      ++ pkgs.nixComponents.nix-store.nativeBuildInputs
-      ++ pkgs.nixComponents.nix-fetchers.nativeBuildInputs
-      ++ pkgs.nixComponents.nix-expr.nativeBuildInputs
-      ++ lib.optionals havePerl pkgs.nixComponents.nix-perl-bindings.nativeBuildInputs
-      ++ lib.optionals buildCanExecuteHost pkgs.nixComponents.nix-manual.externalNativeBuildInputs
-      ++ pkgs.nixComponents.nix-internal-api-docs.nativeBuildInputs
-      ++ pkgs.nixComponents.nix-external-api-docs.nativeBuildInputs
-      ++ pkgs.nixComponents.nix-functional-tests.externalNativeBuildInputs
+      ++ pkgs.nixComponents2.nix-util.nativeBuildInputs
+      ++ pkgs.nixComponents2.nix-store.nativeBuildInputs
+      ++ pkgs.nixComponents2.nix-fetchers.nativeBuildInputs
+      ++ pkgs.nixComponents2.nix-expr.nativeBuildInputs
+      ++ lib.optionals havePerl pkgs.nixComponents2.nix-perl-bindings.nativeBuildInputs
+      ++ lib.optionals buildCanExecuteHost pkgs.nixComponents2.nix-manual.externalNativeBuildInputs
+      ++ pkgs.nixComponents2.nix-internal-api-docs.nativeBuildInputs
+      ++ pkgs.nixComponents2.nix-external-api-docs.nativeBuildInputs
+      ++ pkgs.nixComponents2.nix-functional-tests.externalNativeBuildInputs
       ++ lib.optional (
         !buildCanExecuteHost
         # Hack around https://github.com/nixos/nixpkgs/commit/bf7ad8cfbfa102a90463433e2c5027573b462479
@@ -114,23 +114,20 @@ pkgs.nixComponents.nix-util.overrideAttrs (
         (pkgs.writeScriptBin "pre-commit-hooks-install" modular.pre-commit.settings.installationScript)
         pkgs.buildPackages.nixfmt-rfc-style
       ]
-      # TODO: Remove the darwin check once
-      # https://github.com/NixOS/nixpkgs/pull/291814 is available
-      ++ lib.optional (stdenv.cc.isClang && !stdenv.buildPlatform.isDarwin) pkgs.buildPackages.bear
       ++ lib.optional (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform) (
         lib.hiPrio pkgs.buildPackages.clang-tools
       );
 
     buildInputs =
       attrs.buildInputs or [ ]
-      ++ pkgs.nixComponents.nix-util.buildInputs
-      ++ pkgs.nixComponents.nix-store.buildInputs
-      ++ pkgs.nixComponents.nix-store-tests.externalBuildInputs
-      ++ pkgs.nixComponents.nix-fetchers.buildInputs
-      ++ pkgs.nixComponents.nix-expr.buildInputs
-      ++ pkgs.nixComponents.nix-expr.externalPropagatedBuildInputs
-      ++ pkgs.nixComponents.nix-cmd.buildInputs
-      ++ lib.optionals havePerl pkgs.nixComponents.nix-perl-bindings.externalBuildInputs
+      ++ pkgs.nixComponents2.nix-util.buildInputs
+      ++ pkgs.nixComponents2.nix-store.buildInputs
+      ++ pkgs.nixComponents2.nix-store-tests.externalBuildInputs
+      ++ pkgs.nixComponents2.nix-fetchers.buildInputs
+      ++ pkgs.nixComponents2.nix-expr.buildInputs
+      ++ pkgs.nixComponents2.nix-expr.externalPropagatedBuildInputs
+      ++ pkgs.nixComponents2.nix-cmd.buildInputs
+      ++ lib.optionals havePerl pkgs.nixComponents2.nix-perl-bindings.externalBuildInputs
       ++ lib.optional havePerl pkgs.perl;
   }
 )

packaging/hydra.nix

@@ -19,14 +19,14 @@ let
 
   testNixVersions =
     pkgs: daemon:
-    pkgs.nixComponents.nix-functional-tests.override {
+    pkgs.nixComponents2.nix-functional-tests.override {
       pname = "nix-daemon-compat-tests";
       version = "${pkgs.nix.version}-with-daemon-${daemon.version}";
 
       test-daemon = daemon;
     };
 
-  # Technically we could just return `pkgs.nixComponents`, but for Hydra it's
+  # Technically we could just return `pkgs.nixComponents2`, but for Hydra it's
   # convention to transpose it, and to transpose it efficiently, we need to
   # enumerate them manually, so that we don't evaluate unnecessary package sets.
   # See listingIsComplete below.
@@ -85,7 +85,7 @@ in
       } (_: null);
       actualPkgs = lib.concatMapAttrs (
         k: v: if lib.strings.hasPrefix "nix-" k then { ${k} = null; } else { }
-      ) nixpkgsFor.${arbitrarySystem}.native.nixComponents;
+      ) nixpkgsFor.${arbitrarySystem}.native.nixComponents2;
       diff = lib.concatStringsSep "\n" (
         lib.concatLists (
           lib.mapAttrsToList (
@@ -111,7 +111,7 @@ in
 
   # Binary package for various platforms.
   build = forAllPackages (
-    pkgName: forAllSystems (system: nixpkgsFor.${system}.native.nixComponents.${pkgName})
+    pkgName: forAllSystems (system: nixpkgsFor.${system}.native.nixComponents2.${pkgName})
   );
 
   shellInputs = removeAttrs (forAllSystems (
@@ -121,7 +121,7 @@ in
   buildStatic = forAllPackages (
     pkgName:
     lib.genAttrs linux64BitSystems (
-      system: nixpkgsFor.${system}.native.pkgsStatic.nixComponents.${pkgName}
+      system: nixpkgsFor.${system}.native.pkgsStatic.nixComponents2.${pkgName}
     )
   );
 
@@ -138,7 +138,7 @@ in
         forAllCrossSystems (
           crossSystem:
           lib.genAttrs [ "x86_64-linux" ] (
-            system: nixpkgsFor.${system}.cross.${crossSystem}.nixComponents.${pkgName}
+            system: nixpkgsFor.${system}.cross.${crossSystem}.nixComponents2.${pkgName}
           )
         )
       )
@@ -148,7 +148,7 @@ in
     let
       components = forAllSystems (
         system:
-        nixpkgsFor.${system}.native.nixComponents.overrideScope (
+        nixpkgsFor.${system}.native.nixComponents2.overrideScope (
           self: super: {
             nix-expr = super.nix-expr.override { enableGC = false; };
           }
@@ -157,7 +157,7 @@ in
     in
     forAllPackages (pkgName: forAllSystems (system: components.${system}.${pkgName}));
 
-  buildNoTests = forAllSystems (system: nixpkgsFor.${system}.native.nixComponents.nix-cli);
+  buildNoTests = forAllSystems (system: nixpkgsFor.${system}.native.nixComponents2.nix-cli);
 
   # Toggles some settings for better coverage. Windows needs these
   # library combinations, and Debian build Nix with GNU readline too.
@@ -165,7 +165,7 @@ in
     let
       components = forAllSystems (
         system:
-        nixpkgsFor.${system}.native.nixComponents.overrideScope (
+        nixpkgsFor.${system}.native.nixComponents2.overrideScope (
           self: super: {
             nix-cmd = super.nix-cmd.override {
               enableMarkdown = false;
@@ -178,7 +178,7 @@ in
     forAllPackages (pkgName: forAllSystems (system: components.${system}.${pkgName}));
 
   # Perl bindings for various platforms.
-  perlBindings = forAllSystems (system: nixpkgsFor.${system}.native.nixComponents.nix-perl-bindings);
+  perlBindings = forAllSystems (system: nixpkgsFor.${system}.native.nixComponents2.nix-perl-bindings);
 
   # Binary tarball for various platforms, containing a Nix store
   # with the closure of 'nix' package, and the second half of
@@ -228,13 +228,13 @@ in
   # };
 
   # Nix's manual
-  manual = nixpkgsFor.x86_64-linux.native.nixComponents.nix-manual;
+  manual = nixpkgsFor.x86_64-linux.native.nixComponents2.nix-manual;
 
   # API docs for Nix's unstable internal C++ interfaces.
-  internal-api-docs = nixpkgsFor.x86_64-linux.native.nixComponents.nix-internal-api-docs;
+  internal-api-docs = nixpkgsFor.x86_64-linux.native.nixComponents2.nix-internal-api-docs;
 
   # API docs for Nix's C bindings.
-  external-api-docs = nixpkgsFor.x86_64-linux.native.nixComponents.nix-external-api-docs;
+  external-api-docs = nixpkgsFor.x86_64-linux.native.nixComponents2.nix-external-api-docs;
 
   # System tests.
   tests =

src/libcmd/command.cc

@@ -237,12 +237,13 @@ void StorePathCommand::run(ref<Store> store, StorePaths && storePaths)
 
 MixProfile::MixProfile()
 {
-    addFlag(
-        {.longName = "profile",
-         .description = "The profile to operate on.",
-         .labels = {"path"},
-         .handler = {&profile},
-         .completer = completePath});
+    addFlag({
+        .longName = "profile",
+        .description = "The profile to operate on.",
+        .labels = {"path"},
+        .handler = {&profile},
+        .completer = completePath,
+    });
 }
 
 void MixProfile::updateProfile(const StorePath & storePath)

src/libcmd/common-eval-args.cc

@@ -63,7 +63,7 @@ MixEvalArgs::MixEvalArgs()
         .description = "Pass the value *expr* as the argument *name* to Nix functions.",
         .category = category,
         .labels = {"name", "expr"},
-        .handler = {[&](std::string name, std::string expr) { autoArgs.insert_or_assign(name, AutoArg{AutoArgExpr{expr}}); }}
+        .handler = {[&](std::string name, std::string expr) { autoArgs.insert_or_assign(name, AutoArg{AutoArgExpr{expr}}); }},
     });
 
     addFlag({
@@ -80,7 +80,7 @@ MixEvalArgs::MixEvalArgs()
         .category = category,
         .labels = {"name", "path"},
         .handler = {[&](std::string name, std::string path) { autoArgs.insert_or_assign(name, AutoArg{AutoArgFile{path}}); }},
-        .completer = completePath
+        .completer = completePath,
     });
 
     addFlag({
@@ -105,7 +105,7 @@ MixEvalArgs::MixEvalArgs()
         .labels = {"path"},
         .handler = {[&](std::string s) {
             lookupPath.elements.emplace_back(LookupPath::Elem::parse(s));
-        }}
+        }},
     });
 
     addFlag({
@@ -131,7 +131,7 @@ MixEvalArgs::MixEvalArgs()
         }},
         .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) {
             completeFlakeRef(completions, openStore(), prefix);
-        }}
+        }},
     });
 
     addFlag({

src/libcmd/installables.cc

@@ -40,7 +40,7 @@ void completeFlakeInputAttrPath(
     std::string_view prefix)
 {
     for (auto & flakeRef : flakeRefs) {
-        auto flake = flake::getFlake(*evalState, flakeRef, true);
+        auto flake = flake::getFlake(*evalState, flakeRef, fetchers::UseRegistries::All);
         for (auto & input : flake.inputs)
             if (hasPrefix(input.first, prefix))
                 completions.add(input.first);
@@ -64,21 +64,21 @@ MixFlakeOptions::MixFlakeOptions()
         .handler = {[&]() {
             lockFlags.recreateLockFile = true;
             warn("'--recreate-lock-file' is deprecated and will be removed in a future version; use 'nix flake update' instead.");
-        }}
+        }},
     });
 
     addFlag({
         .longName = "no-update-lock-file",
         .description = "Do not allow any updates to the flake's lock file.",
         .category = category,
-        .handler = {&lockFlags.updateLockFile, false}
+        .handler = {&lockFlags.updateLockFile, false},
     });
 
     addFlag({
         .longName = "no-write-lock-file",
         .description = "Do not write the flake's newly generated lock file.",
         .category = category,
-        .handler = {&lockFlags.writeLockFile, false}
+        .handler = {&lockFlags.writeLockFile, false},
     });
 
     addFlag({
@@ -94,14 +94,14 @@ MixFlakeOptions::MixFlakeOptions()
         .handler = {[&]() {
             lockFlags.useRegistries = false;
             warn("'--no-registries' is deprecated; use '--no-use-registries'");
-        }}
+        }},
     });
 
     addFlag({
         .longName = "commit-lock-file",
         .description = "Commit changes to the flake's lock file.",
         .category = category,
-        .handler = {&lockFlags.commitLockFile, true}
+        .handler = {&lockFlags.commitLockFile, true},
     });
 
     addFlag({
@@ -121,7 +121,7 @@ MixFlakeOptions::MixFlakeOptions()
         }},
         .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) {
             completeFlakeInputAttrPath(completions, getEvalState(), getFlakeRefsForCompletion(), prefix);
-        }}
+        }},
     });
 
     addFlag({
@@ -141,7 +141,7 @@ MixFlakeOptions::MixFlakeOptions()
             } else if (n == 1) {
                 completeFlakeRef(completions, getEvalState()->store, prefix);
             }
-        }}
+        }},
     });
 
     addFlag({
@@ -152,7 +152,7 @@ MixFlakeOptions::MixFlakeOptions()
         .handler = {[&](std::string lockFilePath) {
             lockFlags.referenceLockFilePath = {getFSSourceAccessor(), CanonPath(absPath(lockFilePath))};
         }},
-        .completer = completePath
+        .completer = completePath,
     });
 
     addFlag({
@@ -163,7 +163,7 @@ MixFlakeOptions::MixFlakeOptions()
         .handler = {[&](std::string lockFilePath) {
             lockFlags.outputLockFilePath = lockFilePath;
         }},
-        .completer = completePath
+        .completer = completePath,
     });
 
     addFlag({
@@ -190,7 +190,7 @@ MixFlakeOptions::MixFlakeOptions()
         }},
         .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) {
             completeFlakeRef(completions, getEvalState()->store, prefix);
-        }}
+        }},
     });
 }
 
@@ -206,7 +206,7 @@ SourceExprCommand::SourceExprCommand()
         .category = installablesCategory,
         .labels = {"file"},
         .handler = {&file},
-        .completer = completePath
+        .completer = completePath,
     });
 
     addFlag({
@@ -214,7 +214,7 @@ SourceExprCommand::SourceExprCommand()
         .description = "Interpret [*installables*](@docroot@/command-ref/new-cli/nix.md#installables) as attribute paths relative to the Nix expression *expr*.",
         .category = installablesCategory,
         .labels = {"expr"},
-        .handler = {&expr}
+        .handler = {&expr},
     });
 }
 
@@ -834,7 +834,7 @@ RawInstallablesCommand::RawInstallablesCommand()
     addFlag({
         .longName = "stdin",
         .description = "Read installables from the standard input. No default installable applied.",
-        .handler = {&readFromStdIn, true}
+        .handler = {&readFromStdIn, true},
     });
 
     expectArgs({

src/libcmd/repl.cc

@@ -61,7 +61,10 @@ struct NixRepl
 {
     size_t debugTraceIndex;
 
+    // Arguments passed to :load, saved so they can be reloaded with :reload
     Strings loadedFiles;
+    // Arguments passed to :load-flake, saved so they can be reloaded with :reload
+    Strings loadedFlakes;
     std::function<AnnotatedValues()> getValues;
 
     const static int envSize = 32768;
@@ -90,7 +93,8 @@ struct NixRepl
     void loadFile(const Path & path);
     void loadFlake(const std::string & flakeRef);
     void loadFiles();
-    void reloadFiles();
+    void loadFlakes();
+    void reloadFilesAndFlakes();
     void addAttrsToScope(Value & attrs);
     void addVarToScope(const Symbol name, Value & v);
     Expr * parseString(std::string s);
@@ -244,14 +248,13 @@ StringSet NixRepl::completePrefix(const std::string & prefix)
         try {
             auto dir = std::string(cur, 0, slash);
             auto prefix2 = std::string(cur, slash + 1);
-            for (auto & entry : std::filesystem::directory_iterator{dir == "" ? "/" : dir}) {
+            for (auto & entry : DirectoryIterator{dir == "" ? "/" : dir}) {
                 checkInterrupt();
                 auto name = entry.path().filename().string();
                 if (name[0] != '.' && hasPrefix(name, prefix2))
                     completions.insert(prev + entry.path().string());
             }
         } catch (Error &) {
-        } catch (std::filesystem::filesystem_error &) {
         }
     } else if ((dot = cur.rfind('.')) == std::string::npos) {
         /* This is a variable name; look it up in the current scope. */
@@ -467,7 +470,7 @@ ProcessLineResult NixRepl::processLine(std::string line)
 
     else if (command == ":r" || command == ":reload") {
         state->resetFileCache();
-        reloadFiles();
+        reloadFilesAndFlakes();
     }
 
     else if (command == ":e" || command == ":edit") {
@@ -502,7 +505,7 @@ ProcessLineResult NixRepl::processLine(std::string line)
 
         // Reload right after exiting the editor
         state->resetFileCache();
-        reloadFiles();
+        reloadFilesAndFlakes();
     }
 
     else if (command == ":t") {
@@ -717,6 +720,9 @@ void NixRepl::loadFlake(const std::string & flakeRefS)
     if (flakeRefS.empty())
         throw Error("cannot use ':load-flake' without a path specified. (Use '.' for the current working directory.)");
 
+    loadedFlakes.remove(flakeRefS);
+    loadedFlakes.push_back(flakeRefS);
+
     std::filesystem::path cwd;
     try {
         cwd = std::filesystem::current_path();
@@ -755,11 +761,12 @@ void NixRepl::initEnv()
 }
 
 
-void NixRepl::reloadFiles()
+void NixRepl::reloadFilesAndFlakes()
 {
     initEnv();
 
     loadFiles();
+    loadFlakes();
 }
 
 
@@ -780,6 +787,18 @@ void NixRepl::loadFiles()
 }
 
 
+void NixRepl::loadFlakes()
+{
+    Strings old = loadedFlakes;
+    loadedFlakes.clear();
+
+    for (auto & i : old) {
+        notice("Loading flake '%1%'...", i);
+        loadFlake(i);
+    }
+}
+
+
 void NixRepl::addAttrsToScope(Value & attrs)
 {
     state->forceAttrs(attrs, [&]() { return attrs.determinePos(noPos); }, "while evaluating an attribute set to be merged in the global scope");

src/libexpr/include/nix/expr/meson.build

@@ -20,7 +20,6 @@ headers = [config_pub_h] + files(
   'gc-small-vector.hh',
   'get-drvs.hh',
   'json-to-value.hh',
-  # internal: 'lexer-helpers.hh',
   'nixexpr.hh',
   'parser-state.hh',
   'primops.hh',

src/libexpr/include/nix/expr/nixexpr.hh

@@ -65,7 +65,7 @@ struct DocComment {
 struct AttrName
 {
     Symbol symbol;
-    Expr * expr;
+    Expr * expr = nullptr;
     AttrName(Symbol s) : symbol(s) {};
     AttrName(Expr * e) : expr(e) {};
 };
@@ -159,7 +159,7 @@ struct ExprVar : Expr
 
        `nullptr`: Not from a `with`.
        Valid pointer: the nearest, innermost `with` expression to query first. */
-    ExprWith * fromWith;
+    ExprWith * fromWith = nullptr;
 
     /* In the former case, the value is obtained by going `level`
        levels up from the current environment and getting the
@@ -167,7 +167,7 @@ struct ExprVar : Expr
        value is obtained by getting the attribute named `name` from
        the set stored in the environment that is `level` levels up
        from the current one.*/
-    Level level;
+    Level level = 0;
     Displacement displ = 0;
 
     ExprVar(Symbol name) : name(name) { };

src/libexpr/lexer-helpers.cc

@@ -1,7 +1,4 @@
-#include "lexer-tab.hh"
-#include "parser-tab.hh"
-
-#include "nix/expr/lexer-helpers.hh"
+#include "lexer-helpers.hh"
 
 void nix::lexer::internal::initLoc(YYLTYPE * loc)
 {

src/libexpr/lexer.l

@@ -25,8 +25,7 @@
 #endif
 
 #include "nix/expr/nixexpr.hh"
-#include "parser-tab.hh"
-#include "nix/expr/lexer-helpers.hh"
+#include "lexer-helpers.hh"
 
 namespace nix {
   struct LexerState;

src/libexpr/primops/fetchTree.cc

@@ -181,7 +181,7 @@ static void fetchTree(
     }
 
     if (!state.settings.pureEval && !input.isDirect() && experimentalFeatureSettings.isEnabled(Xp::Flakes))
-        input = lookupInRegistries(state.store, input).first;
+        input = lookupInRegistries(state.store, input, fetchers::UseRegistries::Limited).first;
 
     if (state.settings.pureEval && !input.isLocked()) {
         if (input.getNarHash())

src/libfetchers/include/nix/fetchers/registry.hh

@@ -65,7 +65,11 @@ void overrideRegistry(
     const Input & to,
     const Attrs & extraAttrs);
 
-using RegistryFilter = std::function<bool(Registry::RegistryType)>;
+enum class UseRegistries : int {
+    No,
+    All,
+    Limited, // global and flag registry only
+};
 
 /**
  * Rewrite a flakeref using the registries. If `filter` is set, only
@@ -74,6 +78,6 @@ using RegistryFilter = std::function<bool(Registry::RegistryType)>;
 std::pair<Input, Attrs> lookupInRegistries(
     ref<Store> store,
     const Input & input,
-    const RegistryFilter & filter = {});
+    UseRegistries useRegistries);
 
 }

src/libfetchers/registry.cc

@@ -14,6 +14,8 @@ std::shared_ptr<Registry> Registry::read(
     const Settings & settings,
     const Path & path, RegistryType type)
 {
+    debug("reading registry '%s'", path);
+
     auto registry = std::make_shared<Registry>(settings, type);
 
     if (!pathExists(path))
@@ -179,29 +181,36 @@ Registries getRegistries(const Settings & settings, ref<Store> store)
 std::pair<Input, Attrs> lookupInRegistries(
     ref<Store> store,
     const Input & _input,
-    const RegistryFilter & filter)
+    UseRegistries useRegistries)
 {
     Attrs extraAttrs;
     int n = 0;
     Input input(_input);
 
+    if (useRegistries == UseRegistries::No)
+        return {input, extraAttrs};
+
  restart:
 
     n++;
     if (n > 100) throw Error("cycle detected in flake registry for '%s'", input.to_string());
 
     for (auto & registry : getRegistries(*input.settings, store)) {
-        if (filter && !filter(registry->type)) continue;
+        if (useRegistries == UseRegistries::Limited
+            && !(registry->type == fetchers::Registry::Flag || registry->type == fetchers::Registry::Global))
+            continue;
         // FIXME: O(n)
         for (auto & entry : registry->entries) {
             if (entry.exact) {
                 if (entry.from == input) {
+                    debug("resolved flakeref '%s' against registry %d exactly", input.to_string(), registry->type);
                     input = entry.to;
                     extraAttrs = entry.extraAttrs;
                     goto restart;
                 }
             } else {
                 if (entry.from.contains(input)) {
+                    debug("resolved flakeref '%s' against registry %d", input.to_string(), registry->type);
                     input = entry.to.applyOverrides(
                         !entry.from.getRef() && input.getRef() ? input.getRef() : std::optional<std::string>(),
                         !entry.from.getRev() && input.getRev() ? input.getRev() : std::optional<Hash>());

src/libflake/call-flake.nix

@@ -14,6 +14,7 @@ overrides:
 fetchTreeFinal:
 
 let
+  inherit (builtins) mapAttrs;
 
   lockFile = builtins.fromJSON lockFileStr;
 
@@ -35,19 +36,26 @@ let
         (resolveInput lockFile.nodes.${nodeName}.inputs.${builtins.head path})
         (builtins.tail path);
 
-  allNodes = builtins.mapAttrs (
+  allNodes = mapAttrs (
     key: node:
     let
 
       parentNode = allNodes.${getInputByPath lockFile.root node.parent};
 
+      flakeDir =
+        let
+          dir = overrides.${key}.dir or node.locked.path or "";
+          parentDir = parentNode.flakeDir;
+        in
+        if node ? parent then parentDir + ("/" + dir) else dir;
+
       sourceInfo =
         if overrides ? ${key} then
           overrides.${key}.sourceInfo
         else if node.locked.type == "path" && builtins.substring 0 1 node.locked.path != "/" then
           parentNode.sourceInfo
           // {
-            outPath = parentNode.outPath + ("/" + node.locked.path);
+            outPath = parentNode.sourceInfo.outPath + ("/" + flakeDir);
           }
         else
           # FIXME: remove obsolete node.info.
@@ -60,7 +68,7 @@ let
 
       flake = import (outPath + "/flake.nix");
 
-      inputs = builtins.mapAttrs (inputName: inputSpec: allNodes.${resolveInput inputSpec}) (
+      inputs = mapAttrs (inputName: inputSpec: allNodes.${resolveInput inputSpec}.result) (
         node.inputs or { }
       );
 
@@ -85,12 +93,17 @@ let
         };
 
     in
-    if node.flake or true then
-      assert builtins.isFunction flake.outputs;
-      result
-    else
-      sourceInfo
+    {
+      result =
+        if node.flake or true then
+          assert builtins.isFunction flake.outputs;
+          result
+        else
+          sourceInfo;
+
+      inherit flakeDir sourceInfo;
+    }
   ) lockFile.nodes;
 
 in
-allNodes.${lockFile.root}
+allNodes.${lockFile.root}.result

src/libflake/flake.cc

@@ -45,7 +45,7 @@ static std::optional<FetchedFlake> lookupInFlakeCache(
 static std::tuple<ref<SourceAccessor>, FlakeRef, FlakeRef> fetchOrSubstituteTree(
     EvalState & state,
     const FlakeRef & originalRef,
-    bool useRegistries,
+    fetchers::UseRegistries useRegistries,
     FlakeCache & flakeCache)
 {
     auto fetched = lookupInFlakeCache(flakeCache, originalRef);
@@ -56,14 +56,8 @@ static std::tuple<ref<SourceAccessor>, FlakeRef, FlakeRef> fetchOrSubstituteTree
             auto [accessor, lockedRef] = originalRef.lazyFetch(state.store);
             fetched.emplace(FetchedFlake{.lockedRef = lockedRef, .accessor = accessor});
         } else {
-            if (useRegistries) {
-                resolvedRef = originalRef.resolve(
-                    state.store,
-                    [](fetchers::Registry::RegistryType type) {
-                        /* Only use the global registry and CLI flags
-                           to resolve indirect flakerefs. */
-                        return type == fetchers::Registry::Flag || type == fetchers::Registry::Global;
-                    });
+            if (useRegistries != fetchers::UseRegistries::No) {
+                resolvedRef = originalRef.resolve(state.store, useRegistries);
                 fetched = lookupInFlakeCache(flakeCache, originalRef);
                 if (!fetched) {
                     auto [accessor, lockedRef] = resolvedRef.lazyFetch(state.store);
@@ -396,7 +390,7 @@ static FlakeRef applySelfAttrs(
 static Flake getFlake(
     EvalState & state,
     const FlakeRef & originalRef,
-    bool useRegistries,
+    fetchers::UseRegistries useRegistries,
     FlakeCache & flakeCache,
     const InputAttrPath & lockRootAttrPath)
 {
@@ -415,7 +409,7 @@ static Flake getFlake(
         // FIXME: need to remove attrs that are invalidated by the changed input attrs, such as 'narHash'.
         newLockedRef.input.attrs.erase("narHash");
         auto [accessor2, resolvedRef2, lockedRef2] = fetchOrSubstituteTree(
-            state, newLockedRef, false, flakeCache);
+            state, newLockedRef, fetchers::UseRegistries::No, flakeCache);
         accessor = accessor2;
         lockedRef = lockedRef2;
     }
@@ -427,7 +421,7 @@ static Flake getFlake(
     return readFlake(state, originalRef, resolvedRef, lockedRef, state.storePath(storePath), lockRootAttrPath);
 }
 
-Flake getFlake(EvalState & state, const FlakeRef & originalRef, bool useRegistries)
+Flake getFlake(EvalState & state, const FlakeRef & originalRef, fetchers::UseRegistries useRegistries)
 {
     FlakeCache flakeCache;
     return getFlake(state, originalRef, useRegistries, flakeCache, {});
@@ -455,8 +449,15 @@ LockedFlake lockFlake(
     FlakeCache flakeCache;
 
     auto useRegistries = lockFlags.useRegistries.value_or(settings.useRegistries);
+    auto useRegistriesTop = useRegistries ? fetchers::UseRegistries::All : fetchers::UseRegistries::No;
+    auto useRegistriesInputs = useRegistries ? fetchers::UseRegistries::Limited : fetchers::UseRegistries::No;
 
-    auto flake = getFlake(state, topRef, useRegistries, flakeCache, {});
+    auto flake = getFlake(
+        state,
+        topRef,
+        useRegistriesTop,
+        flakeCache,
+        {});
 
     if (lockFlags.applyNixConfig) {
         flake.config.apply(settings);
@@ -626,12 +627,17 @@ LockedFlake lockFlake(
 
                     /* Get the input flake, resolve 'path:./...'
                        flakerefs relative to the parent flake. */
-                    auto getInputFlake = [&](const FlakeRef & ref)
+                    auto getInputFlake = [&](const FlakeRef & ref, const fetchers::UseRegistries useRegistries)
                     {
                         if (auto resolvedPath = resolveRelativePath()) {
                             return readFlake(state, ref, ref, ref, *resolvedPath, inputAttrPath);
                         } else {
-                            return getFlake(state, ref, useRegistries, flakeCache, inputAttrPath);
+                            return getFlake(
+                                state,
+                                ref,
+                                useRegistries,
+                                flakeCache,
+                                inputAttrPath);
                         }
                     };
 
@@ -647,7 +653,7 @@ LockedFlake lockFlake(
                                 oldLock = *oldLock3;
 
                     if (oldLock
-                        && oldLock->originalRef == *input.ref
+                        && oldLock->originalRef.canonicalize() == input.ref->canonicalize()
                         && oldLock->parentInputAttrPath == overridenParentPath
                         && !hasCliOverride)
                     {
@@ -712,7 +718,7 @@ LockedFlake lockFlake(
                         }
 
                         if (mustRefetch) {
-                            auto inputFlake = getInputFlake(oldLock->lockedRef);
+                            auto inputFlake = getInputFlake(oldLock->lockedRef, useRegistriesInputs);
                             nodePaths.emplace(childNode, inputFlake.path.parent());
                             computeLocks(inputFlake.inputs, childNode, inputAttrPath, oldLock, followsPrefix,
                                 inputFlake.path, false);
@@ -737,10 +743,11 @@ LockedFlake lockFlake(
                             nuked the next time we update the lock
                             file. That is, overrides are sticky unless you
                             use --no-write-lock-file. */
-                        auto ref = (input2.ref && explicitCliOverrides.contains(inputAttrPath)) ? *input2.ref : *input.ref;
+                        auto inputIsOverride = explicitCliOverrides.contains(inputAttrPath);
+                        auto ref = (input2.ref && inputIsOverride) ? *input2.ref : *input.ref;
 
                         if (input.isFlake) {
-                            auto inputFlake = getInputFlake(*input.ref);
+                            auto inputFlake = getInputFlake(*input.ref, inputIsOverride ? fetchers::UseRegistries::All : useRegistriesInputs);
 
                             auto childNode = make_ref<LockedNode>(
                                 inputFlake.lockedRef,
@@ -780,7 +787,7 @@ LockedFlake lockFlake(
                                     return {*resolvedPath, *input.ref};
                                 } else {
                                     auto [accessor, resolvedRef, lockedRef] = fetchOrSubstituteTree(
-                                        state, *input.ref, useRegistries, flakeCache);
+                                        state, *input.ref, useRegistriesInputs, flakeCache);
 
                                     // FIXME: allow input to be lazy.
                                     auto storePath = copyInputToStore(state, lockedRef.input, input.ref->input, accessor);
@@ -895,7 +902,10 @@ LockedFlake lockFlake(
                            repo, so we should re-read it. FIXME: we could
                            also just clear the 'rev' field... */
                         auto prevLockedRef = flake.lockedRef;
-                        flake = getFlake(state, topRef, useRegistries);
+                        flake = getFlake(
+                            state,
+                            topRef,
+                            useRegistriesTop);
 
                         if (lockFlags.commitLockFile &&
                             flake.lockedRef.input.getRev() &&

src/libflake/flakeref.cc

@@ -37,9 +37,9 @@ std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef)
 
 FlakeRef FlakeRef::resolve(
     ref<Store> store,
-    const fetchers::RegistryFilter & filter) const
+    fetchers::UseRegistries useRegistries) const
 {
-    auto [input2, extraAttrs] = lookupInRegistries(store, input);
+    auto [input2, extraAttrs] = lookupInRegistries(store, input, useRegistries);
     return FlakeRef(std::move(input2), fetchers::maybeGetStrAttr(extraAttrs, "dir").value_or(subdir));
 }
 
@@ -289,6 +289,55 @@ std::pair<ref<SourceAccessor>, FlakeRef> FlakeRef::lazyFetch(ref<Store> store) c
     return {accessor, FlakeRef(std::move(lockedInput), subdir)};
 }
 
+FlakeRef FlakeRef::canonicalize() const
+{
+    auto flakeRef(*this);
+
+    /* Backward compatibility hack: In old versions of Nix, if you had
+       a flake input like
+
+         inputs.foo.url = "git+https://foo/bar?dir=subdir";
+
+       it would result in a lock file entry like
+
+         "original": {
+           "dir": "subdir",
+           "type": "git",
+           "url": "https://foo/bar?dir=subdir"
+         }
+
+       New versions of Nix remove `?dir=subdir` from the `url` field,
+       since the subdirectory is intended for `FlakeRef`, not the
+       fetcher (and specifically the remote server), that is, the
+       flakeref is parsed into
+
+         "original": {
+           "dir": "subdir",
+           "type": "git",
+           "url": "https://foo/bar"
+         }
+
+       However, this causes new versions of Nix to consider the lock
+       file entry to be stale since the `original` ref no longer
+       matches exactly.
+
+       For this reason, we canonicalise the `original` ref by
+       filtering the `dir` query parameter from the URL. */
+    if (auto url = fetchers::maybeGetStrAttr(flakeRef.input.attrs, "url")) {
+        try {
+            auto parsed = parseURL(*url);
+            if (auto dir2 = get(parsed.query, "dir")) {
+                if (flakeRef.subdir != "" && flakeRef.subdir == *dir2)
+                    parsed.query.erase("dir");
+            }
+            flakeRef.input.attrs.insert_or_assign("url", parsed.to_string());
+        } catch (BadURL &) {
+        }
+    }
+
+    return flakeRef;
+}
+
 std::tuple<FlakeRef, std::string, ExtendedOutputsSpec> parseFlakeRefWithFragmentAndExtendedOutputsSpec(
     const fetchers::Settings & fetchSettings,
     const std::string & url,

src/libflake/include/nix/flake/flake.hh

@@ -115,7 +115,7 @@ struct Flake
     }
 };
 
-Flake getFlake(EvalState & state, const FlakeRef & flakeRef, bool useRegistries);
+Flake getFlake(EvalState & state, const FlakeRef & flakeRef, fetchers::UseRegistries useRegistries);
 
 /**
  * Fingerprint of a locked flake; used as a cache key.

src/libflake/include/nix/flake/flakeref.hh

@@ -65,13 +65,19 @@ struct FlakeRef
 
     FlakeRef resolve(
         ref<Store> store,
-        const fetchers::RegistryFilter & filter = {}) const;
+        fetchers::UseRegistries useRegistries = fetchers::UseRegistries::All) const;
 
     static FlakeRef fromAttrs(
         const fetchers::Settings & fetchSettings,
         const fetchers::Attrs & attrs);
 
     std::pair<ref<SourceAccessor>, FlakeRef> lazyFetch(ref<Store> store) const;
+
+    /**
+     * Canonicalize a flakeref for the purpose of comparing "old" and
+     * "new" `original` fields in lock files.
+     */
+    FlakeRef canonicalize() const;
 };
 
 std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef);

src/libflake/meson.build

@@ -39,13 +39,13 @@ foreach header : [
 endforeach
 
 sources = files(
-  'flake/config.cc',
-  'flake/flake.cc',
-  'flake/flakeref.cc',
-  'flake/lockfile.cc',
-  'flake/flake-primops.cc',
-  'flake/settings.cc',
-  'flake/url-name.cc',
+  'config.cc',
+  'flake.cc',
+  'flakeref.cc',
+  'lockfile.cc',
+  'flake-primops.cc',
+  'settings.cc',
+  'url-name.cc',
 )
 
 subdir('include/nix/flake')

src/libmain/common-args.cc

@@ -57,7 +57,7 @@ MixCommonArgs::MixCommonArgs(const std::string & programName)
                     if (hasPrefix(s.first, prefix))
                         completions.add(s.first, fmt("Set the `%s` setting.", s.first));
             }
-        }
+        },
     });
 
     addFlag({
@@ -75,7 +75,7 @@ MixCommonArgs::MixCommonArgs(const std::string & programName)
         .labels = Strings{"jobs"},
         .handler = {[=](std::string s) {
             settings.set("max-jobs", s);
-        }}
+        }},
     });
 
     std::string cat = "Options to override configuration settings";

src/libmain/plugin.cc

@@ -6,6 +6,7 @@
 
 #include "nix/util/config-global.hh"
 #include "nix/util/signals.hh"
+#include "nix/util/file-system.hh"
 
 namespace nix {
 
@@ -77,13 +78,13 @@ void initPlugins()
     for (const auto & pluginFile : pluginSettings.pluginFiles.get()) {
         std::vector<std::filesystem::path> pluginFiles;
         try {
-            auto ents = std::filesystem::directory_iterator{pluginFile};
+            auto ents = DirectoryIterator{pluginFile};
             for (const auto & ent : ents) {
                 checkInterrupt();
                 pluginFiles.emplace_back(ent.path());
             }
-        } catch (std::filesystem::filesystem_error & e) {
-            if (e.code() != std::errc::not_a_directory)
+        } catch (SysError & e) {
+            if (e.errNo != ENOTDIR)
                 throw;
             pluginFiles.emplace_back(pluginFile);
         }

src/libmain/shared.cc

@@ -231,7 +231,7 @@ LegacyArgs::LegacyArgs(const std::string & programName,
             .handler = {[=](std::string s) {
                 auto n = string2IntWithUnitPrefix<uint64_t>(s);
                 settings.set(dest, std::to_string(n));
-            }}
+            }},
         });
     };
 

src/libstore-tests/data/derivation/advanced-attributes-defaults.drv

@@ -1 +0,0 @@
-../../../../tests/functional/derivation/advanced-attributes-defaults.drv

src/libstore-tests/data/derivation/advanced-attributes-structured-attrs-defaults.drv

@@ -1 +0,0 @@
-../../../../tests/functional/derivation/advanced-attributes-structured-attrs-defaults.drv

src/libstore-tests/data/derivation/advanced-attributes-structured-attrs.drv

@@ -1 +0,0 @@
-../../../../tests/functional/derivation/advanced-attributes-structured-attrs.drv

src/libstore-tests/data/derivation/advanced-attributes-structured-attrs.json

@@ -1,41 +0,0 @@
-{
-  "args": [
-    "-c",
-    "echo hello > $out"
-  ],
-  "builder": "/bin/bash",
-  "env": {
-    "__json": "{\"__darwinAllowLocalNetworking\":true,\"__impureHostDeps\":[\"/usr/bin/ditto\"],\"__noChroot\":true,\"__sandboxProfile\":\"sandcastle\",\"allowSubstitutes\":false,\"builder\":\"/bin/bash\",\"impureEnvVars\":[\"UNICORN\"],\"name\":\"advanced-attributes-structured-attrs\",\"outputChecks\":{\"bin\":{\"disallowedReferences\":[\"/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar\"],\"disallowedRequisites\":[\"/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar\"]},\"dev\":{\"maxClosureSize\":5909,\"maxSize\":789},\"out\":{\"allowedReferences\":[\"/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo\"],\"allowedRequisites\":[\"/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo\"]}},\"outputs\":[\"out\",\"bin\",\"dev\"],\"preferLocalBuild\":true,\"requiredSystemFeatures\":[\"rainbow\",\"uid-range\"],\"system\":\"my-system\"}",
-    "bin": "/nix/store/pbzb48v0ycf80jgligcp4n8z0rblna4n-advanced-attributes-structured-attrs-bin",
-    "dev": "/nix/store/7xapi8jv7flcz1qq8jhw55ar8ag8hldh-advanced-attributes-structured-attrs-dev",
-    "out": "/nix/store/mpq3l1l1qc2yr50q520g08kprprwv79f-advanced-attributes-structured-attrs"
-  },
-  "inputDrvs": {
-    "/nix/store/4xm4wccqsvagz9gjksn24s7rip2fdy7v-foo.drv": {
-      "dynamicOutputs": {},
-      "outputs": [
-        "out"
-      ]
-    },
-    "/nix/store/plsq5jbr5nhgqwcgb2qxw7jchc09dnl8-bar.drv": {
-      "dynamicOutputs": {},
-      "outputs": [
-        "out"
-      ]
-    }
-  },
-  "inputSrcs": [],
-  "name": "advanced-attributes-structured-attrs",
-  "outputs": {
-    "bin": {
-      "path": "/nix/store/pbzb48v0ycf80jgligcp4n8z0rblna4n-advanced-attributes-structured-attrs-bin"
-    },
-    "dev": {
-      "path": "/nix/store/7xapi8jv7flcz1qq8jhw55ar8ag8hldh-advanced-attributes-structured-attrs-dev"
-    },
-    "out": {
-      "path": "/nix/store/mpq3l1l1qc2yr50q520g08kprprwv79f-advanced-attributes-structured-attrs"
-    }
-  },
-  "system": "my-system"
-}

src/libstore-tests/data/derivation/advanced-attributes.drv

@@ -1 +0,0 @@
-../../../../tests/functional/derivation/advanced-attributes.drv

src/libstore-tests/data/derivation/ca/advanced-attributes-defaults.drv

@@ -0,0 +1 @@
+../../../../../tests/functional/derivation/ca/advanced-attributes-defaults.drv

src/libstore-tests/data/derivation/ca/advanced-attributes-defaults.json

@@ -0,0 +1,25 @@
+{
+  "args": [
+    "-c",
+    "echo hello > $out"
+  ],
+  "builder": "/bin/bash",
+  "env": {
+    "builder": "/bin/bash",
+    "name": "advanced-attributes-defaults",
+    "out": "/1rz4g4znpzjwh1xymhjpm42vipw92pr73vdgl6xs1hycac8kf2n9",
+    "outputHashAlgo": "sha256",
+    "outputHashMode": "recursive",
+    "system": "my-system"
+  },
+  "inputDrvs": {},
+  "inputSrcs": [],
+  "name": "advanced-attributes-defaults",
+  "outputs": {
+    "out": {
+      "hashAlgo": "sha256",
+      "method": "nar"
+    }
+  },
+  "system": "my-system"
+}

src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs-defaults.drv

@@ -0,0 +1 @@
+../../../../../tests/functional/derivation/ca/advanced-attributes-structured-attrs-defaults.drv

src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs-defaults.json

@@ -0,0 +1,26 @@
+{
+  "args": [
+    "-c",
+    "echo hello > $out"
+  ],
+  "builder": "/bin/bash",
+  "env": {
+    "__json": "{\"builder\":\"/bin/bash\",\"name\":\"advanced-attributes-structured-attrs-defaults\",\"outputHashAlgo\":\"sha256\",\"outputHashMode\":\"recursive\",\"outputs\":[\"out\",\"dev\"],\"system\":\"my-system\"}",
+    "dev": "/02qcpld1y6xhs5gz9bchpxaw0xdhmsp5dv88lh25r2ss44kh8dxz",
+    "out": "/1rz4g4znpzjwh1xymhjpm42vipw92pr73vdgl6xs1hycac8kf2n9"
+  },
+  "inputDrvs": {},
+  "inputSrcs": [],
+  "name": "advanced-attributes-structured-attrs-defaults",
+  "outputs": {
+    "dev": {
+      "hashAlgo": "sha256",
+      "method": "nar"
+    },
+    "out": {
+      "hashAlgo": "sha256",
+      "method": "nar"
+    }
+  },
+  "system": "my-system"
+}

src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs.drv

@@ -0,0 +1 @@
+../../../../../tests/functional/derivation/ca/advanced-attributes-structured-attrs.drv

src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs.json

@@ -0,0 +1,46 @@
+{
+  "args": [
+    "-c",
+    "echo hello > $out"
+  ],
+  "builder": "/bin/bash",
+  "env": {
+    "__json": "{\"__darwinAllowLocalNetworking\":true,\"__impureHostDeps\":[\"/usr/bin/ditto\"],\"__noChroot\":true,\"__sandboxProfile\":\"sandcastle\",\"allowSubstitutes\":false,\"builder\":\"/bin/bash\",\"impureEnvVars\":[\"UNICORN\"],\"name\":\"advanced-attributes-structured-attrs\",\"outputChecks\":{\"bin\":{\"disallowedReferences\":[\"/0nyw57wm2iicnm9rglvjmbci3ikmcp823czdqdzdcgsnnwqps71g\"],\"disallowedRequisites\":[\"/07f301yqyz8c6wf6bbbavb2q39j4n8kmcly1s09xadyhgy6x2wr8\"]},\"dev\":{\"maxClosureSize\":5909,\"maxSize\":789},\"out\":{\"allowedReferences\":[\"/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9\"],\"allowedRequisites\":[\"/0nr45p69vn6izw9446wsh9bng9nndhvn19kpsm4n96a5mycw0s4z\"]}},\"outputHashAlgo\":\"sha256\",\"outputHashMode\":\"recursive\",\"outputs\":[\"out\",\"bin\",\"dev\"],\"preferLocalBuild\":true,\"requiredSystemFeatures\":[\"rainbow\",\"uid-range\"],\"system\":\"my-system\"}",
+    "bin": "/04f3da1kmbr67m3gzxikmsl4vjz5zf777sv6m14ahv22r65aac9m",
+    "dev": "/02qcpld1y6xhs5gz9bchpxaw0xdhmsp5dv88lh25r2ss44kh8dxz",
+    "out": "/1rz4g4znpzjwh1xymhjpm42vipw92pr73vdgl6xs1hycac8kf2n9"
+  },
+  "inputDrvs": {
+    "/nix/store/j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv": {
+      "dynamicOutputs": {},
+      "outputs": [
+        "dev",
+        "out"
+      ]
+    },
+    "/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv": {
+      "dynamicOutputs": {},
+      "outputs": [
+        "dev",
+        "out"
+      ]
+    }
+  },
+  "inputSrcs": [],
+  "name": "advanced-attributes-structured-attrs",
+  "outputs": {
+    "bin": {
+      "hashAlgo": "sha256",
+      "method": "nar"
+    },
+    "dev": {
+      "hashAlgo": "sha256",
+      "method": "nar"
+    },
+    "out": {
+      "hashAlgo": "sha256",
+      "method": "nar"
+    }
+  },
+  "system": "my-system"
+}

src/libstore-tests/data/derivation/ca/advanced-attributes.drv

@@ -0,0 +1 @@
+../../../../../tests/functional/derivation/ca/advanced-attributes.drv

src/libstore-tests/data/derivation/ca/advanced-attributes.json

@@ -0,0 +1,52 @@
+{
+  "args": [
+    "-c",
+    "echo hello > $out"
+  ],
+  "builder": "/bin/bash",
+  "env": {
+    "__darwinAllowLocalNetworking": "1",
+    "__impureHostDeps": "/usr/bin/ditto",
+    "__noChroot": "1",
+    "__sandboxProfile": "sandcastle",
+    "allowSubstitutes": "",
+    "allowedReferences": "/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9",
+    "allowedRequisites": "/0nr45p69vn6izw9446wsh9bng9nndhvn19kpsm4n96a5mycw0s4z",
+    "builder": "/bin/bash",
+    "disallowedReferences": "/0nyw57wm2iicnm9rglvjmbci3ikmcp823czdqdzdcgsnnwqps71g",
+    "disallowedRequisites": "/07f301yqyz8c6wf6bbbavb2q39j4n8kmcly1s09xadyhgy6x2wr8",
+    "impureEnvVars": "UNICORN",
+    "name": "advanced-attributes",
+    "out": "/1rz4g4znpzjwh1xymhjpm42vipw92pr73vdgl6xs1hycac8kf2n9",
+    "outputHashAlgo": "sha256",
+    "outputHashMode": "recursive",
+    "preferLocalBuild": "1",
+    "requiredSystemFeatures": "rainbow uid-range",
+    "system": "my-system"
+  },
+  "inputDrvs": {
+    "/nix/store/j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv": {
+      "dynamicOutputs": {},
+      "outputs": [
+        "dev",
+        "out"
+      ]
+    },
+    "/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv": {
+      "dynamicOutputs": {},
+      "outputs": [
+        "dev",
+        "out"
+      ]
+    }
+  },
+  "inputSrcs": [],
+  "name": "advanced-attributes",
+  "outputs": {
+    "out": {
+      "hashAlgo": "sha256",
+      "method": "nar"
+    }
+  },
+  "system": "my-system"
+}

src/libstore-tests/data/derivation/ia/advanced-attributes-defaults.drv

@@ -0,0 +1 @@
+../../../../../tests/functional/derivation/ia/advanced-attributes-defaults.drv

src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs-defaults.drv

@@ -0,0 +1 @@
+../../../../../tests/functional/derivation/ia/advanced-attributes-structured-attrs-defaults.drv

src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs.drv

@@ -0,0 +1 @@
+../../../../../tests/functional/derivation/ia/advanced-attributes-structured-attrs.drv

src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs.json

@@ -0,0 +1,43 @@
+{
+  "args": [
+    "-c",
+    "echo hello > $out"
+  ],
+  "builder": "/bin/bash",
+  "env": {
+    "__json": "{\"__darwinAllowLocalNetworking\":true,\"__impureHostDeps\":[\"/usr/bin/ditto\"],\"__noChroot\":true,\"__sandboxProfile\":\"sandcastle\",\"allowSubstitutes\":false,\"builder\":\"/bin/bash\",\"impureEnvVars\":[\"UNICORN\"],\"name\":\"advanced-attributes-structured-attrs\",\"outputChecks\":{\"bin\":{\"disallowedReferences\":[\"/nix/store/r5cff30838majxk5mp3ip2diffi8vpaj-bar\"],\"disallowedRequisites\":[\"/nix/store/9b61w26b4avv870dw0ymb6rw4r1hzpws-bar-dev\"]},\"dev\":{\"maxClosureSize\":5909,\"maxSize\":789},\"out\":{\"allowedReferences\":[\"/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo\"],\"allowedRequisites\":[\"/nix/store/z0rjzy29v9k5qa4nqpykrbzirj7sd43v-foo-dev\"]}},\"outputs\":[\"out\",\"bin\",\"dev\"],\"preferLocalBuild\":true,\"requiredSystemFeatures\":[\"rainbow\",\"uid-range\"],\"system\":\"my-system\"}",
+    "bin": "/nix/store/qjjj3zrlimpjbkk686m052b3ks9iz2sl-advanced-attributes-structured-attrs-bin",
+    "dev": "/nix/store/lpz5grl48v93pdadavyg5is1rqvfdipf-advanced-attributes-structured-attrs-dev",
+    "out": "/nix/store/nzvz1bmh1g89a5dkpqcqan0av7q3hgv3-advanced-attributes-structured-attrs"
+  },
+  "inputDrvs": {
+    "/nix/store/afc3vbjbzql750v2lp8gxgaxsajphzih-foo.drv": {
+      "dynamicOutputs": {},
+      "outputs": [
+        "dev",
+        "out"
+      ]
+    },
+    "/nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv": {
+      "dynamicOutputs": {},
+      "outputs": [
+        "dev",
+        "out"
+      ]
+    }
+  },
+  "inputSrcs": [],
+  "name": "advanced-attributes-structured-attrs",
+  "outputs": {
+    "bin": {
+      "path": "/nix/store/qjjj3zrlimpjbkk686m052b3ks9iz2sl-advanced-attributes-structured-attrs-bin"
+    },
+    "dev": {
+      "path": "/nix/store/lpz5grl48v93pdadavyg5is1rqvfdipf-advanced-attributes-structured-attrs-dev"
+    },
+    "out": {
+      "path": "/nix/store/nzvz1bmh1g89a5dkpqcqan0av7q3hgv3-advanced-attributes-structured-attrs"
+    }
+  },
+  "system": "my-system"
+}

src/libstore-tests/data/derivation/ia/advanced-attributes.drv

@@ -0,0 +1 @@
+../../../../../tests/functional/derivation/ia/advanced-attributes.drv

src/libstore-tests/data/derivation/ia/advanced-attributes.json

@@ -0,0 +1,49 @@
+{
+  "args": [
+    "-c",
+    "echo hello > $out"
+  ],
+  "builder": "/bin/bash",
+  "env": {
+    "__darwinAllowLocalNetworking": "1",
+    "__impureHostDeps": "/usr/bin/ditto",
+    "__noChroot": "1",
+    "__sandboxProfile": "sandcastle",
+    "allowSubstitutes": "",
+    "allowedReferences": "/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo",
+    "allowedRequisites": "/nix/store/z0rjzy29v9k5qa4nqpykrbzirj7sd43v-foo-dev",
+    "builder": "/bin/bash",
+    "disallowedReferences": "/nix/store/r5cff30838majxk5mp3ip2diffi8vpaj-bar",
+    "disallowedRequisites": "/nix/store/9b61w26b4avv870dw0ymb6rw4r1hzpws-bar-dev",
+    "impureEnvVars": "UNICORN",
+    "name": "advanced-attributes",
+    "out": "/nix/store/swkj0mrq0cq3dfli95v4am0427mi2hxf-advanced-attributes",
+    "preferLocalBuild": "1",
+    "requiredSystemFeatures": "rainbow uid-range",
+    "system": "my-system"
+  },
+  "inputDrvs": {
+    "/nix/store/afc3vbjbzql750v2lp8gxgaxsajphzih-foo.drv": {
+      "dynamicOutputs": {},
+      "outputs": [
+        "dev",
+        "out"
+      ]
+    },
+    "/nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv": {
+      "dynamicOutputs": {},
+      "outputs": [
+        "dev",
+        "out"
+      ]
+    }
+  },
+  "inputSrcs": [],
+  "name": "advanced-attributes",
+  "outputs": {
+    "out": {
+      "path": "/nix/store/swkj0mrq0cq3dfli95v4am0427mi2hxf-advanced-attributes"
+    }
+  },
+  "system": "my-system"
+}

src/libstore-tests/derivation-advanced-attrs.cc

@@ -18,68 +18,93 @@ using nlohmann::json;
 
 class DerivationAdvancedAttrsTest : public CharacterizationTest, public LibStoreTest
 {
-    std::filesystem::path unitTestData = getUnitTestData() / "derivation";
+protected:
+    std::filesystem::path unitTestData = getUnitTestData() / "derivation" / "ia";
 
 public:
     std::filesystem::path goldenMaster(std::string_view testStem) const override
     {
         return unitTestData / testStem;
     }
+
+    /**
+     * We set these in tests rather than the regular globals so we don't have
+     * to worry about race conditions if the tests run concurrently.
+     */
+    ExperimentalFeatureSettings mockXpSettings;
 };
 
-#define TEST_ATERM_JSON(STEM, NAME)                                                                   \
-    TEST_F(DerivationAdvancedAttrsTest, Derivation_##STEM##_from_json)                                \
-    {                                                                                                 \
-        readTest(NAME ".json", [&](const auto & encoded_) {                                           \
-            auto encoded = json::parse(encoded_);                                                     \
-            /* Use DRV file instead of C++ literal as source of truth. */                             \
-            auto aterm = readFile(goldenMaster(NAME ".drv"));                                         \
-            auto expected = parseDerivation(*store, std::move(aterm), NAME);                          \
-            Derivation got = Derivation::fromJSON(*store, encoded);                                   \
-            EXPECT_EQ(got, expected);                                                                 \
-        });                                                                                           \
-    }                                                                                                 \
-                                                                                                      \
-    TEST_F(DerivationAdvancedAttrsTest, Derivation_##STEM##_to_json)                                  \
-    {                                                                                                 \
-        writeTest(                                                                                    \
-            NAME ".json",                                                                             \
-            [&]() -> json {                                                                           \
-                /* Use DRV file instead of C++ literal as source of truth. */                         \
-                auto aterm = readFile(goldenMaster(NAME ".drv"));                                     \
-                return parseDerivation(*store, std::move(aterm), NAME).toJSON(*store);                \
-            },                                                                                        \
-            [](const auto & file) { return json::parse(readFile(file)); },                            \
-            [](const auto & file, const auto & got) { return writeFile(file, got.dump(2) + "\n"); }); \
-    }                                                                                                 \
-                                                                                                      \
-    TEST_F(DerivationAdvancedAttrsTest, Derivation_##STEM##_from_aterm)                               \
-    {                                                                                                 \
-        readTest(NAME ".drv", [&](auto encoded) {                                                     \
-            /* Use JSON file instead of C++ literal as source of truth. */                            \
-            auto json = json::parse(readFile(goldenMaster(NAME ".json")));                            \
-            auto expected = Derivation::fromJSON(*store, json);                                       \
-            auto got = parseDerivation(*store, std::move(encoded), NAME);                             \
-            EXPECT_EQ(got.toJSON(*store), expected.toJSON(*store));                                   \
-            EXPECT_EQ(got, expected);                                                                 \
-        });                                                                                           \
-    }                                                                                                 \
-                                                                                                      \
+class CaDerivationAdvancedAttrsTest : public DerivationAdvancedAttrsTest
+{
+    void SetUp() override
+    {
+        unitTestData = getUnitTestData() / "derivation" / "ca";
+        mockXpSettings.set("experimental-features", "ca-derivations");
+    }
+};
+
+template<class Fixture>
+class DerivationAdvancedAttrsBothTest : public Fixture
+{};
+
+using BothFixtures = ::testing::Types<DerivationAdvancedAttrsTest, CaDerivationAdvancedAttrsTest>;
+
+TYPED_TEST_SUITE(DerivationAdvancedAttrsBothTest, BothFixtures);
+
+#define TEST_ATERM_JSON(STEM, NAME)                                                                      \
+    TYPED_TEST(DerivationAdvancedAttrsBothTest, Derivation_##STEM##_from_json)                           \
+    {                                                                                                    \
+        this->readTest(NAME ".json", [&](const auto & encoded_) {                                        \
+            auto encoded = json::parse(encoded_);                                                        \
+            /* Use DRV file instead of C++ literal as source of truth. */                                \
+            auto aterm = readFile(this->goldenMaster(NAME ".drv"));                                      \
+            auto expected = parseDerivation(*this->store, std::move(aterm), NAME, this->mockXpSettings); \
+            Derivation got = Derivation::fromJSON(*this->store, encoded, this->mockXpSettings);          \
+            EXPECT_EQ(got, expected);                                                                    \
+        });                                                                                              \
+    }                                                                                                    \
+                                                                                                         \
+    TYPED_TEST(DerivationAdvancedAttrsBothTest, Derivation_##STEM##_to_json)                             \
+    {                                                                                                    \
+        this->writeTest(                                                                                 \
+            NAME ".json",                                                                                \
+            [&]() -> json {                                                                              \
+                /* Use DRV file instead of C++ literal as source of truth. */                            \
+                auto aterm = readFile(this->goldenMaster(NAME ".drv"));                                  \
+                return parseDerivation(*this->store, std::move(aterm), NAME, this->mockXpSettings)       \
+                    .toJSON(*this->store);                                                               \
+            },                                                                                           \
+            [](const auto & file) { return json::parse(readFile(file)); },                               \
+            [](const auto & file, const auto & got) { return writeFile(file, got.dump(2) + "\n"); });    \
+    }                                                                                                    \
+                                                                                                         \
+    TYPED_TEST(DerivationAdvancedAttrsBothTest, Derivation_##STEM##_from_aterm)                          \
+    {                                                                                                    \
+        this->readTest(NAME ".drv", [&](auto encoded) {                                                  \
+            /* Use JSON file instead of C++ literal as source of truth. */                               \
+            auto json = json::parse(readFile(this->goldenMaster(NAME ".json")));                         \
+            auto expected = Derivation::fromJSON(*this->store, json, this->mockXpSettings);              \
+            auto got = parseDerivation(*this->store, std::move(encoded), NAME, this->mockXpSettings);    \
+            EXPECT_EQ(got.toJSON(*this->store), expected.toJSON(*this->store));                          \
+            EXPECT_EQ(got, expected);                                                                    \
+        });                                                                                              \
+    }                                                                                                    \
+                                                                                                         \
     /* No corresponding write test, because we need to read the drv to write the json file */
 
-TEST_ATERM_JSON(advancedAttributes_defaults, "advanced-attributes-defaults");
 TEST_ATERM_JSON(advancedAttributes, "advanced-attributes-defaults");
-TEST_ATERM_JSON(advancedAttributes_structuredAttrs_defaults, "advanced-attributes-structured-attrs");
+TEST_ATERM_JSON(advancedAttributes_defaults, "advanced-attributes");
 TEST_ATERM_JSON(advancedAttributes_structuredAttrs, "advanced-attributes-structured-attrs-defaults");
+TEST_ATERM_JSON(advancedAttributes_structuredAttrs_defaults, "advanced-attributes-structured-attrs");
 
 #undef TEST_ATERM_JSON
 
-TEST_F(DerivationAdvancedAttrsTest, Derivation_advancedAttributes_defaults)
+TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_defaults)
 {
-    readTest("advanced-attributes-defaults.drv", [&](auto encoded) {
-        auto got = parseDerivation(*store, std::move(encoded), "foo");
+    this->readTest("advanced-attributes-defaults.drv", [&](auto encoded) {
+        auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings);
 
-        auto drvPath = writeDerivation(*store, got, NoRepair, true);
+        auto drvPath = writeDerivation(*this->store, got, NoRepair, true);
 
         ParsedDerivation parsedDrv(drvPath, got);
         DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv);
@@ -101,25 +126,50 @@ TEST_F(DerivationAdvancedAttrsTest, Derivation_advancedAttributes_defaults)
             EXPECT_EQ(checksForAllOutputs.disallowedReferences, StringSet{});
             EXPECT_EQ(checksForAllOutputs.disallowedRequisites, StringSet{});
         }
-        EXPECT_EQ(options.getRequiredSystemFeatures(got), StringSet());
-        EXPECT_EQ(options.canBuildLocally(*store, got), false);
-        EXPECT_EQ(options.willBuildLocally(*store, got), false);
+        EXPECT_EQ(options.canBuildLocally(*this->store, got), false);
+        EXPECT_EQ(options.willBuildLocally(*this->store, got), false);
         EXPECT_EQ(options.substitutesAllowed(), true);
         EXPECT_EQ(options.useUidRange(got), false);
     });
 };
 
-TEST_F(DerivationAdvancedAttrsTest, Derivation_advancedAttributes)
+TEST_F(DerivationAdvancedAttrsTest, advancedAttributes_defaults)
 {
-    readTest("advanced-attributes.drv", [&](auto encoded) {
-        auto got = parseDerivation(*store, std::move(encoded), "foo");
+    this->readTest("advanced-attributes-defaults.drv", [&](auto encoded) {
+        auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings);
 
-        auto drvPath = writeDerivation(*store, got, NoRepair, true);
+        auto drvPath = writeDerivation(*this->store, got, NoRepair, true);
 
         ParsedDerivation parsedDrv(drvPath, got);
         DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv);
 
-        StringSet systemFeatures{"rainbow", "uid-range"};
+        EXPECT_EQ(options.getRequiredSystemFeatures(got), StringSet{});
+    });
+};
+
+TEST_F(CaDerivationAdvancedAttrsTest, advancedAttributes_defaults)
+{
+    this->readTest("advanced-attributes-defaults.drv", [&](auto encoded) {
+        auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings);
+
+        auto drvPath = writeDerivation(*this->store, got, NoRepair, true);
+
+        ParsedDerivation parsedDrv(drvPath, got);
+        DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv);
+
+        EXPECT_EQ(options.getRequiredSystemFeatures(got), StringSet{"ca-derivations"});
+    });
+};
+
+TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes)
+{
+    this->readTest("advanced-attributes.drv", [&](auto encoded) {
+        auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings);
+
+        auto drvPath = writeDerivation(*this->store, got, NoRepair, true);
+
+        ParsedDerivation parsedDrv(drvPath, got);
+        DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv);
 
         EXPECT_TRUE(!parsedDrv.hasStructuredAttrs());
 
@@ -128,34 +178,88 @@ TEST_F(DerivationAdvancedAttrsTest, Derivation_advancedAttributes)
         EXPECT_EQ(options.impureHostDeps, StringSet{"/usr/bin/ditto"});
         EXPECT_EQ(options.impureEnvVars, StringSet{"UNICORN"});
         EXPECT_EQ(options.allowLocalNetworking, true);
+        EXPECT_EQ(options.canBuildLocally(*this->store, got), false);
+        EXPECT_EQ(options.willBuildLocally(*this->store, got), false);
+        EXPECT_EQ(options.substitutesAllowed(), false);
+        EXPECT_EQ(options.useUidRange(got), true);
+    });
+};
+
+TEST_F(DerivationAdvancedAttrsTest, advancedAttributes)
+{
+    this->readTest("advanced-attributes.drv", [&](auto encoded) {
+        auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings);
+
+        auto drvPath = writeDerivation(*this->store, got, NoRepair, true);
+
+        ParsedDerivation parsedDrv(drvPath, got);
+        DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv);
+
         {
             auto * checksForAllOutputs_ = std::get_if<0>(&options.outputChecks);
             ASSERT_TRUE(checksForAllOutputs_ != nullptr);
             auto & checksForAllOutputs = *checksForAllOutputs_;
 
             EXPECT_EQ(
-                checksForAllOutputs.allowedReferences, StringSet{"/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo"});
+                checksForAllOutputs.allowedReferences, StringSet{"/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo"});
             EXPECT_EQ(
-                checksForAllOutputs.allowedRequisites, StringSet{"/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo"});
+                checksForAllOutputs.allowedRequisites,
+                StringSet{"/nix/store/z0rjzy29v9k5qa4nqpykrbzirj7sd43v-foo-dev"});
             EXPECT_EQ(
-                checksForAllOutputs.disallowedReferences, StringSet{"/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar"});
+                checksForAllOutputs.disallowedReferences, StringSet{"/nix/store/r5cff30838majxk5mp3ip2diffi8vpaj-bar"});
             EXPECT_EQ(
-                checksForAllOutputs.disallowedRequisites, StringSet{"/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar"});
+                checksForAllOutputs.disallowedRequisites,
+                StringSet{"/nix/store/9b61w26b4avv870dw0ymb6rw4r1hzpws-bar-dev"});
         }
+
+        StringSet systemFeatures{"rainbow", "uid-range"};
+
         EXPECT_EQ(options.getRequiredSystemFeatures(got), systemFeatures);
-        EXPECT_EQ(options.canBuildLocally(*store, got), false);
-        EXPECT_EQ(options.willBuildLocally(*store, got), false);
-        EXPECT_EQ(options.substitutesAllowed(), false);
-        EXPECT_EQ(options.useUidRange(got), true);
     });
 };
 
-TEST_F(DerivationAdvancedAttrsTest, Derivation_advancedAttributes_structuredAttrs_defaults)
+TEST_F(CaDerivationAdvancedAttrsTest, advancedAttributes)
 {
-    readTest("advanced-attributes-structured-attrs-defaults.drv", [&](auto encoded) {
-        auto got = parseDerivation(*store, std::move(encoded), "foo");
+    this->readTest("advanced-attributes.drv", [&](auto encoded) {
+        auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings);
 
-        auto drvPath = writeDerivation(*store, got, NoRepair, true);
+        auto drvPath = writeDerivation(*this->store, got, NoRepair, true);
+
+        ParsedDerivation parsedDrv(drvPath, got);
+        DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv);
+
+        {
+            auto * checksForAllOutputs_ = std::get_if<0>(&options.outputChecks);
+            ASSERT_TRUE(checksForAllOutputs_ != nullptr);
+            auto & checksForAllOutputs = *checksForAllOutputs_;
+
+            EXPECT_EQ(
+                checksForAllOutputs.allowedReferences,
+                StringSet{"/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9"});
+            EXPECT_EQ(
+                checksForAllOutputs.allowedRequisites,
+                StringSet{"/0nr45p69vn6izw9446wsh9bng9nndhvn19kpsm4n96a5mycw0s4z"});
+            EXPECT_EQ(
+                checksForAllOutputs.disallowedReferences,
+                StringSet{"/0nyw57wm2iicnm9rglvjmbci3ikmcp823czdqdzdcgsnnwqps71g"});
+            EXPECT_EQ(
+                checksForAllOutputs.disallowedRequisites,
+                StringSet{"/07f301yqyz8c6wf6bbbavb2q39j4n8kmcly1s09xadyhgy6x2wr8"});
+        }
+
+        StringSet systemFeatures{"rainbow", "uid-range"};
+        systemFeatures.insert("ca-derivations");
+
+        EXPECT_EQ(options.getRequiredSystemFeatures(got), systemFeatures);
+    });
+};
+
+TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_structuredAttrs_defaults)
+{
+    this->readTest("advanced-attributes-structured-attrs-defaults.drv", [&](auto encoded) {
+        auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings);
+
+        auto drvPath = writeDerivation(*this->store, got, NoRepair, true);
 
         ParsedDerivation parsedDrv(drvPath, got);
         DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv);
@@ -176,25 +280,50 @@ TEST_F(DerivationAdvancedAttrsTest, Derivation_advancedAttributes_structuredAttr
             EXPECT_EQ(checksPerOutput.size(), 0);
         }
 
-        EXPECT_EQ(options.getRequiredSystemFeatures(got), StringSet());
-        EXPECT_EQ(options.canBuildLocally(*store, got), false);
-        EXPECT_EQ(options.willBuildLocally(*store, got), false);
+        EXPECT_EQ(options.canBuildLocally(*this->store, got), false);
+        EXPECT_EQ(options.willBuildLocally(*this->store, got), false);
         EXPECT_EQ(options.substitutesAllowed(), true);
         EXPECT_EQ(options.useUidRange(got), false);
     });
 };
 
-TEST_F(DerivationAdvancedAttrsTest, Derivation_advancedAttributes_structuredAttrs)
+TEST_F(DerivationAdvancedAttrsTest, advancedAttributes_structuredAttrs_defaults)
 {
-    readTest("advanced-attributes-structured-attrs.drv", [&](auto encoded) {
-        auto got = parseDerivation(*store, std::move(encoded), "foo");
+    this->readTest("advanced-attributes-structured-attrs-defaults.drv", [&](auto encoded) {
+        auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings);
 
-        auto drvPath = writeDerivation(*store, got, NoRepair, true);
+        auto drvPath = writeDerivation(*this->store, got, NoRepair, true);
 
         ParsedDerivation parsedDrv(drvPath, got);
         DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv);
 
-        StringSet systemFeatures{"rainbow", "uid-range"};
+        EXPECT_EQ(options.getRequiredSystemFeatures(got), StringSet{});
+    });
+};
+
+TEST_F(CaDerivationAdvancedAttrsTest, advancedAttributes_structuredAttrs_defaults)
+{
+    this->readTest("advanced-attributes-structured-attrs-defaults.drv", [&](auto encoded) {
+        auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings);
+
+        auto drvPath = writeDerivation(*this->store, got, NoRepair, true);
+
+        ParsedDerivation parsedDrv(drvPath, got);
+        DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv);
+
+        EXPECT_EQ(options.getRequiredSystemFeatures(got), StringSet{"ca-derivations"});
+    });
+};
+
+TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_structuredAttrs)
+{
+    this->readTest("advanced-attributes-structured-attrs.drv", [&](auto encoded) {
+        auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings);
+
+        auto drvPath = writeDerivation(*this->store, got, NoRepair, true);
+
+        ParsedDerivation parsedDrv(drvPath, got);
+        DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv);
 
         EXPECT_TRUE(parsedDrv.hasStructuredAttrs());
 
@@ -204,14 +333,40 @@ TEST_F(DerivationAdvancedAttrsTest, Derivation_advancedAttributes_structuredAttr
         EXPECT_EQ(options.impureEnvVars, StringSet{"UNICORN"});
         EXPECT_EQ(options.allowLocalNetworking, true);
 
+        {
+            auto output_ = get(std::get<1>(options.outputChecks), "dev");
+            ASSERT_TRUE(output_);
+            auto & output = *output_;
+
+            EXPECT_EQ(output.maxSize, 789);
+            EXPECT_EQ(output.maxClosureSize, 5909);
+        }
+
+        EXPECT_EQ(options.canBuildLocally(*this->store, got), false);
+        EXPECT_EQ(options.willBuildLocally(*this->store, got), false);
+        EXPECT_EQ(options.substitutesAllowed(), false);
+        EXPECT_EQ(options.useUidRange(got), true);
+    });
+};
+
+TEST_F(DerivationAdvancedAttrsTest, advancedAttributes_structuredAttrs)
+{
+    this->readTest("advanced-attributes-structured-attrs.drv", [&](auto encoded) {
+        auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings);
+
+        auto drvPath = writeDerivation(*this->store, got, NoRepair, true);
+
+        ParsedDerivation parsedDrv(drvPath, got);
+        DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv);
+
         {
             {
                 auto output_ = get(std::get<1>(options.outputChecks), "out");
                 ASSERT_TRUE(output_);
                 auto & output = *output_;
 
-                EXPECT_EQ(output.allowedReferences, StringSet{"/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo"});
-                EXPECT_EQ(output.allowedRequisites, StringSet{"/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo"});
+                EXPECT_EQ(output.allowedReferences, StringSet{"/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo"});
+                EXPECT_EQ(output.allowedRequisites, StringSet{"/nix/store/z0rjzy29v9k5qa4nqpykrbzirj7sd43v-foo-dev"});
             }
 
             {
@@ -219,25 +374,54 @@ TEST_F(DerivationAdvancedAttrsTest, Derivation_advancedAttributes_structuredAttr
                 ASSERT_TRUE(output_);
                 auto & output = *output_;
 
-                EXPECT_EQ(output.disallowedReferences, StringSet{"/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar"});
-                EXPECT_EQ(output.disallowedRequisites, StringSet{"/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar"});
-            }
-
-            {
-                auto output_ = get(std::get<1>(options.outputChecks), "dev");
-                ASSERT_TRUE(output_);
-                auto & output = *output_;
-
-                EXPECT_EQ(output.maxSize, 789);
-                EXPECT_EQ(output.maxClosureSize, 5909);
+                EXPECT_EQ(output.disallowedReferences, StringSet{"/nix/store/r5cff30838majxk5mp3ip2diffi8vpaj-bar"});
+                EXPECT_EQ(
+                    output.disallowedRequisites, StringSet{"/nix/store/9b61w26b4avv870dw0ymb6rw4r1hzpws-bar-dev"});
             }
         }
 
+        StringSet systemFeatures{"rainbow", "uid-range"};
+
+        EXPECT_EQ(options.getRequiredSystemFeatures(got), systemFeatures);
+    });
+};
+
+TEST_F(CaDerivationAdvancedAttrsTest, advancedAttributes_structuredAttrs)
+{
+    this->readTest("advanced-attributes-structured-attrs.drv", [&](auto encoded) {
+        auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings);
+
+        auto drvPath = writeDerivation(*this->store, got, NoRepair, true);
+
+        ParsedDerivation parsedDrv(drvPath, got);
+        DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv);
+
+        {
+            {
+                auto output_ = get(std::get<1>(options.outputChecks), "out");
+                ASSERT_TRUE(output_);
+                auto & output = *output_;
+
+                EXPECT_EQ(output.allowedReferences, StringSet{"/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9"});
+                EXPECT_EQ(output.allowedRequisites, StringSet{"/0nr45p69vn6izw9446wsh9bng9nndhvn19kpsm4n96a5mycw0s4z"});
+            }
+
+            {
+                auto output_ = get(std::get<1>(options.outputChecks), "bin");
+                ASSERT_TRUE(output_);
+                auto & output = *output_;
+
+                EXPECT_EQ(
+                    output.disallowedReferences, StringSet{"/0nyw57wm2iicnm9rglvjmbci3ikmcp823czdqdzdcgsnnwqps71g"});
+                EXPECT_EQ(
+                    output.disallowedRequisites, StringSet{"/07f301yqyz8c6wf6bbbavb2q39j4n8kmcly1s09xadyhgy6x2wr8"});
+            }
+        }
+
+        StringSet systemFeatures{"rainbow", "uid-range"};
+        systemFeatures.insert("ca-derivations");
+
         EXPECT_EQ(options.getRequiredSystemFeatures(got), systemFeatures);
-        EXPECT_EQ(options.canBuildLocally(*store, got), false);
-        EXPECT_EQ(options.willBuildLocally(*store, got), false);
-        EXPECT_EQ(options.substitutesAllowed(), false);
-        EXPECT_EQ(options.useUidRange(got), true);
     });
 };
 

src/libstore-tests/machines.cc

@@ -73,6 +73,32 @@ TEST(machines, getMachinesWithSemicolonSeparator) {
     EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, AuthorityMatches("nix@itchy.labs.cs.uu.nl"))));
 }
 
+TEST(machines, getMachinesWithCommentsAndSemicolonSeparator) {
+    auto actual = Machine::parseConfig({},
+        "# This is a comment ; this is still that comment\n"
+        "nix@scratchy.labs.cs.uu.nl ; nix@itchy.labs.cs.uu.nl\n"
+        "# This is also a comment ; this also is still that comment\n"
+        "nix@scabby.labs.cs.uu.nl\n");
+    EXPECT_THAT(actual, SizeIs(3));
+    EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, AuthorityMatches("nix@scratchy.labs.cs.uu.nl"))));
+    EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, AuthorityMatches("nix@itchy.labs.cs.uu.nl"))));
+    EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, AuthorityMatches("nix@scabby.labs.cs.uu.nl"))));
+}
+
+TEST(machines, getMachinesWithFunnyWhitespace) {
+    auto actual = Machine::parseConfig({},
+        "        # commment ; comment\n"
+        "   nix@scratchy.labs.cs.uu.nl ; nix@itchy.labs.cs.uu.nl   \n"
+        "\n    \n"
+        "\n ;;; \n"
+        "\n ; ; \n"
+        "nix@scabby.labs.cs.uu.nl\n\n");
+    EXPECT_THAT(actual, SizeIs(3));
+    EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, AuthorityMatches("nix@scratchy.labs.cs.uu.nl"))));
+    EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, AuthorityMatches("nix@itchy.labs.cs.uu.nl"))));
+    EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, AuthorityMatches("nix@scabby.labs.cs.uu.nl"))));
+}
+
 TEST(machines, getMachinesWithCorrectCompleteSingleBuilder) {
     auto actual = Machine::parseConfig({},
         "nix@scratchy.labs.cs.uu.nl     i686-linux      "

src/libstore/builtins/buildenv.cc

@@ -18,12 +18,12 @@ struct State
 /* For each activated package, create symlinks */
 static void createLinks(State & state, const Path & srcDir, const Path & dstDir, int priority)
 {
-    std::filesystem::directory_iterator srcFiles;
+    DirectoryIterator srcFiles;
 
     try {
-        srcFiles = std::filesystem::directory_iterator{srcDir};
-    } catch (std::filesystem::filesystem_error & e) {
-        if (e.code() == std::errc::not_a_directory) {
+        srcFiles = DirectoryIterator{srcDir};
+    } catch (SysError & e) {
+        if (e.errNo == ENOTDIR) {
             warn("not including '%s' in the user environment because it's not a directory", srcDir);
             return;
         }

src/libstore/builtins/unpack-channel.cc

@@ -29,13 +29,9 @@ void builtinUnpackChannel(
 
     size_t fileCount;
     std::string fileName;
-    try {
-        auto entries = fs::directory_iterator{out};
-        fileName = entries->path().string();
-        fileCount = std::distance(fs::begin(entries), fs::end(entries));
-    } catch (fs::filesystem_error &) {
-        throw SysError("failed to read directory %1%", out.string());
-    }
+    auto entries = DirectoryIterator{out};
+    fileName = entries->path().string();
+    fileCount = std::distance(entries.begin(), entries.end());
 
     if (fileCount != 1)
         throw Error("channel tarball '%s' contains more than one file", src);

src/libstore/derivation-options.cc

@@ -68,7 +68,6 @@ DerivationOptions DerivationOptions::fromParsedDerivation(const ParsedDerivation
                                         throw Error("attribute '%s' must be a list of strings", name);
                                     res.insert(j->get<std::string>());
                                 }
-                                checks.disallowedRequisites = res;
                                 return res;
                             }
                             return {};

src/libstore/derivations.cc

@@ -1368,7 +1368,7 @@ Derivation Derivation::fromJSON(
         for (auto & [outputName, output] : getObject(valueAt(json, "outputs"))) {
             res.outputs.insert_or_assign(
                 outputName,
-                DerivationOutput::fromJSON(store, res.name, outputName, output));
+                DerivationOutput::fromJSON(store, res.name, outputName, output, xpSettings));
         }
     } catch (Error & e) {
         e.addTrace({}, "while reading key 'outputs'");

src/libstore/filetransfer.cc

@@ -22,10 +22,8 @@
 
 #include <curl/curl.h>
 
-#include <algorithm>
 #include <cmath>
 #include <cstring>
-#include <iostream>
 #include <queue>
 #include <random>
 #include <thread>
@@ -95,7 +93,7 @@ struct curlFileTransfer : public FileTransfer
             : fileTransfer(fileTransfer)
             , request(request)
             , act(*logger, lvlTalkative, actFileTransfer,
-                request.post ? "" : fmt(request.data ?  "uploading '%s'" : "downloading '%s'", request.uri),
+                fmt("%sing '%s'", request.verb(), request.uri),
                 {request.uri}, request.parentAct)
             , callback(std::move(callback))
             , finalSink([this](std::string_view data) {
@@ -272,19 +270,11 @@ struct curlFileTransfer : public FileTransfer
             return getInterrupted();
         }
 
-        int silentProgressCallback(curl_off_t dltotal, curl_off_t dlnow)
-        {
-            return getInterrupted();
-        }
-
         static int progressCallbackWrapper(void * userp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow)
         {
-            return ((TransferItem *) userp)->progressCallback(dltotal, dlnow);
-        }
-
-        static int silentProgressCallbackWrapper(void * userp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow)
-        {
-            return ((TransferItem *) userp)->silentProgressCallback(dltotal, dlnow);
+            auto & item = *static_cast<TransferItem *>(userp);
+            auto isUpload = bool(item.request.data);
+            return item.progressCallback(isUpload ? ultotal : dltotal, isUpload ? ulnow : dlnow);
         }
 
         static int debugCallback(CURL * handle, curl_infotype type, char * data, size_t size, void * userptr)
@@ -319,6 +309,23 @@ struct curlFileTransfer : public FileTransfer
         }
         #endif
 
+        size_t seekCallback(curl_off_t offset, int origin)
+        {
+            if (origin == SEEK_SET) {
+                readOffset = offset;
+            } else if (origin == SEEK_CUR) {
+                readOffset += offset;
+            } else if (origin == SEEK_END) {
+                readOffset = request.data->length() + offset;
+            }
+            return CURL_SEEKFUNC_OK;
+        }
+
+        static size_t seekCallbackWrapper(void *clientp, curl_off_t offset, int origin)
+        {
+            return ((TransferItem *) clientp)->seekCallback(offset, origin);
+        }
+
         void init()
         {
             if (!req) req = curl_easy_init();
@@ -351,10 +358,7 @@ struct curlFileTransfer : public FileTransfer
             curl_easy_setopt(req, CURLOPT_HEADERFUNCTION, TransferItem::headerCallbackWrapper);
             curl_easy_setopt(req, CURLOPT_HEADERDATA, this);
 
-            if (request.post)
-                curl_easy_setopt(req, CURLOPT_XFERINFOFUNCTION, silentProgressCallbackWrapper);
-            else
-                curl_easy_setopt(req, CURLOPT_XFERINFOFUNCTION, progressCallbackWrapper);
+            curl_easy_setopt(req, CURLOPT_XFERINFOFUNCTION, progressCallbackWrapper);
             curl_easy_setopt(req, CURLOPT_XFERINFODATA, this);
             curl_easy_setopt(req, CURLOPT_NOPROGRESS, 0);
 
@@ -374,6 +378,8 @@ struct curlFileTransfer : public FileTransfer
                 curl_easy_setopt(req, CURLOPT_READFUNCTION, readCallbackWrapper);
                 curl_easy_setopt(req, CURLOPT_READDATA, this);
                 curl_easy_setopt(req, CURLOPT_INFILESIZE_LARGE, (curl_off_t) request.data->length());
+                curl_easy_setopt(req, CURLOPT_SEEKFUNCTION, seekCallbackWrapper);
+                curl_easy_setopt(req, CURLOPT_SEEKDATA, this);
             }
 
             if (request.verifyTLS) {
@@ -447,8 +453,7 @@ struct curlFileTransfer : public FileTransfer
                 if (httpStatus == 304 && result.etag == "")
                     result.etag = request.expectedETag;
 
-                if (!request.post)
-                    act.progress(result.bodySize, result.bodySize);
+                act.progress(result.bodySize, result.bodySize);
                 done = true;
                 callback(std::move(result));
             }
@@ -537,6 +542,8 @@ struct curlFileTransfer : public FileTransfer
                         warn("%s; retrying from offset %d in %d ms", exc.what(), writtenToSink, ms);
                     else
                         warn("%s; retrying in %d ms", exc.what(), ms);
+                    decompressionSink.reset();
+                    errorSink.reset();
                     embargo = std::chrono::steady_clock::now() + std::chrono::milliseconds(ms);
                     fileTransfer.enqueueItem(shared_from_this());
                 }
@@ -789,10 +796,6 @@ struct curlFileTransfer : public FileTransfer
 
                 S3Helper s3Helper(profile, region, scheme, endpoint);
 
-                Activity act(*logger, lvlTalkative, actFileTransfer,
-                    fmt("downloading '%s'", request.uri),
-                    {request.uri}, request.parentAct);
-
                 // FIXME: implement ETag
                 auto s3Res = s3Helper.getObject(bucketName, key);
                 FileTransferResult res;

src/libstore/gc.cc

@@ -164,7 +164,7 @@ void LocalStore::findTempRoots(Roots & tempRoots, bool censor)
 {
     /* Read the `temproots' directory for per-process temporary root
        files. */
-    for (auto & i : std::filesystem::directory_iterator{tempRootsDir}) {
+    for (auto & i : DirectoryIterator{tempRootsDir}) {
         checkInterrupt();
         auto name = i.path().filename().string();
         if (name[0] == '.') {
@@ -232,7 +232,7 @@ void LocalStore::findRoots(const Path & path, std::filesystem::file_type type, R
             type = std::filesystem::symlink_status(path).type();
 
         if (type == std::filesystem::file_type::directory) {
-            for (auto & i : std::filesystem::directory_iterator{path}) {
+            for (auto & i : DirectoryIterator{path}) {
                 checkInterrupt();
                 findRoots(i.path().string(), i.symlink_status().type(), roots);
             }

src/libstore/globals.cc

@@ -278,21 +278,21 @@ template<> void BaseSetting<SandboxMode>::convertToArg(Args & args, const std::s
         .aliases = aliases,
         .description = "Enable sandboxing.",
         .category = category,
-        .handler = {[this]() { override(smEnabled); }}
+        .handler = {[this]() { override(smEnabled); }},
     });
     args.addFlag({
         .longName = "no-" + name,
         .aliases = aliases,
         .description = "Disable sandboxing.",
         .category = category,
-        .handler = {[this]() { override(smDisabled); }}
+        .handler = {[this]() { override(smDisabled); }},
     });
     args.addFlag({
         .longName = "relaxed-" + name,
         .aliases = aliases,
         .description = "Enable sandboxing, but allow builds to disable it.",
         .category = category,
-        .handler = {[this]() { override(smRelaxed); }}
+        .handler = {[this]() { override(smRelaxed); }},
     });
 }
 

src/libstore/include/nix/store/filetransfer.hh

@@ -77,7 +77,7 @@ struct FileTransferRequest
     FileTransferRequest(std::string_view uri)
         : uri(uri), parentAct(getCurActivity()) { }
 
-    std::string verb()
+    std::string verb() const
     {
         return data ? "upload" : "download";
     }

src/libstore/linux/include/nix/store/meson.build

@@ -1,6 +1,5 @@
 include_dirs += include_directories('../..')
 
 headers += files(
-  'fchmodat2-compat.hh',
   'personality.hh',
 )

src/libstore/local-binary-cache-store.cc

@@ -84,7 +84,7 @@ protected:
     {
         StorePathSet paths;
 
-        for (auto & entry : std::filesystem::directory_iterator{binaryCacheDir}) {
+        for (auto & entry : DirectoryIterator{binaryCacheDir}) {
             checkInterrupt();
             auto name = entry.path().filename().string();
             if (name.size() != 40 ||

src/libstore/local-overlay-store.md

@@ -85,7 +85,7 @@ The parts of a local overlay store are as follows:
     > The location of the database instead depends on the [`state`](#store-experimental-local-overlay-store-state) setting.
     > It is always `${state}/db`.
 
-    This contains the metadata of all of the upper layer [store objects][store object] (everything beyond their file system objects), and also duplicate copies of some lower layer store object's metadta.
+    This contains the metadata of all of the upper layer [store objects][store object] (everything beyond their file system objects), and also duplicate copies of some lower layer store object's metadata.
     The duplication is so the metadata for the [closure](@docroot@/glossary.md#gloss-closure) of upper layer [store objects][store object] can be found entirely within the upper layer.
     (This allows us to use the same SQL Schema as the [local store]'s SQLite database, as foreign keys in that schema enforce closure metadata to be self-contained in this way.)
 

src/libstore/local-store.cc

@@ -249,7 +249,7 @@ LocalStore::LocalStore(
     else if (curSchema == 0) { /* new store */
         curSchema = nixSchemaVersion;
         openDB(*state, true);
-        writeFile(schemaPath, fmt("%1%", curSchema), 0666, true);
+        writeFile(schemaPath, fmt("%1%", curSchema), 0666, FsSync::Yes);
     }
 
     else if (curSchema < nixSchemaVersion) {
@@ -300,7 +300,7 @@ LocalStore::LocalStore(
             txn.commit();
         }
 
-        writeFile(schemaPath, fmt("%1%", nixSchemaVersion), 0666, true);
+        writeFile(schemaPath, fmt("%1%", nixSchemaVersion), 0666, FsSync::Yes);
 
         lockFile(globalLock.get(), ltRead, true);
     }
@@ -1386,7 +1386,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
 
         printInfo("checking link hashes...");
 
-        for (auto & link : std::filesystem::directory_iterator{linksDir}) {
+        for (auto & link : DirectoryIterator{linksDir}) {
             checkInterrupt();
             auto name = link.path().filename();
             printMsg(lvlTalkative, "checking contents of '%s'", name);
@@ -1479,7 +1479,7 @@ LocalStore::VerificationResult LocalStore::verifyAllValidPaths(RepairFlag repair
        database and the filesystem) in the loop below, in order to catch
        invalid states.
      */
-    for (auto & i : std::filesystem::directory_iterator{realStoreDir.to_string()}) {
+    for (auto & i : DirectoryIterator{realStoreDir.to_string()}) {
         checkInterrupt();
         try {
             storePathsInStoreDir.insert({i.path().filename().string()});

src/libstore/machines.cc

@@ -105,28 +105,31 @@ ref<Store> Machine::openStore() const
 static std::vector<std::string> expandBuilderLines(const std::string & builders)
 {
     std::vector<std::string> result;
-    for (auto line : tokenizeString<std::vector<std::string>>(builders, "\n;")) {
-        trim(line);
+    for (auto line : tokenizeString<std::vector<std::string>>(builders, "\n")) {
         line.erase(std::find(line.begin(), line.end(), '#'), line.end());
-        if (line.empty()) continue;
+        for (auto entry : tokenizeString<std::vector<std::string>>(line, ";")) {
+            entry = trim(entry);
 
-        if (line[0] == '@') {
-            const std::string path = trim(std::string(line, 1));
-            std::string text;
-            try {
-                text = readFile(path);
-            } catch (const SysError & e) {
-                if (e.errNo != ENOENT)
-                    throw;
-                debug("cannot find machines file '%s'", path);
+            if (entry.empty()) {
+                // skip blank entries
+            } else if (entry[0] == '@') {
+                const std::string path = trim(std::string_view{entry}.substr(1));
+                std::string text;
+                try {
+                    text = readFile(path);
+                } catch (const SysError & e) {
+                    if (e.errNo != ENOENT)
+                        throw;
+                    debug("cannot find machines file '%s'", path);
+                    continue;
+                }
+
+                const auto entrys = expandBuilderLines(text);
+                result.insert(end(result), begin(entrys), end(entrys));
+            } else {
+                result.emplace_back(entry);
             }
-
-            const auto lines = expandBuilderLines(text);
-            result.insert(end(result), begin(lines), end(lines));
-            continue;
         }
-
-        result.emplace_back(line);
     }
     return result;
 }

src/libstore/nar-info.cc

@@ -176,7 +176,7 @@ NarInfo NarInfo::fromJSON(
             std::nullopt);
 
     if (json.contains("downloadSize"))
-        res.fileSize = getInteger(valueAt(json, "downloadSize"));
+        res.fileSize = getUnsigned(valueAt(json, "downloadSize"));
 
     return res;
 }

src/libstore/path-info.cc

@@ -192,7 +192,7 @@ UnkeyedValidPathInfo UnkeyedValidPathInfo::fromJSON(
 
     auto & json = getObject(_json);
     res.narHash = Hash::parseAny(getString(valueAt(json, "narHash")), std::nullopt);
-    res.narSize = getInteger(valueAt(json, "narSize"));
+    res.narSize = getUnsigned(valueAt(json, "narSize"));
 
     try {
         auto references = getStringList(valueAt(json, "references"));
@@ -216,7 +216,7 @@ UnkeyedValidPathInfo UnkeyedValidPathInfo::fromJSON(
 
     if (json.contains("registrationTime"))
         if (auto * rawRegistrationTime = getNullable(valueAt(json, "registrationTime")))
-            res.registrationTime = getInteger(*rawRegistrationTime);
+            res.registrationTime = getInteger<time_t>(*rawRegistrationTime);
 
     if (json.contains("ultimate"))
         res.ultimate = getBoolean(valueAt(json, "ultimate"));

src/libstore/posix-fs-canonicalise.cc

@@ -136,7 +136,7 @@ static void canonicalisePathMetaData_(
 #endif
 
     if (S_ISDIR(st.st_mode)) {
-        for (auto & i : std::filesystem::directory_iterator{path}) {
+        for (auto & i : DirectoryIterator{path}) {
             checkInterrupt();
             canonicalisePathMetaData_(
                 i.path().string(),

src/libstore/profiles.cc

@@ -38,7 +38,7 @@ std::pair<Generations, std::optional<GenerationNumber>> findGenerations(Path pro
     std::filesystem::path profileDir = dirOf(profile);
     auto profileName = std::string(baseNameOf(profile));
 
-    for (auto & i : std::filesystem::directory_iterator{profileDir}) {
+    for (auto & i : DirectoryIterator{profileDir}) {
         checkInterrupt();
         if (auto n = parseName(profileName, i.path().filename().string())) {
             auto path = i.path().string();

src/libstore/s3-binary-cache-store.cc

@@ -160,7 +160,10 @@ ref<Aws::Client::ClientConfiguration> S3Helper::makeConfig(
 S3Helper::FileTransferResult S3Helper::getObject(
     const std::string & bucketName, const std::string & key)
 {
-    debug("fetching 's3://%s/%s'...", bucketName, key);
+    std::string uri = "s3://" + bucketName + "/" + key;
+    Activity act(*logger, lvlTalkative, actFileTransfer,
+        fmt("downloading '%s'", uri),
+        Logger::Fields{uri}, getCurActivity());
 
     auto request =
         Aws::S3::Model::GetObjectRequest()
@@ -171,6 +174,22 @@ S3Helper::FileTransferResult S3Helper::getObject(
         return Aws::New<std::stringstream>("STRINGSTREAM");
     });
 
+    size_t bytesDone = 0;
+    size_t bytesExpected = 0;
+    request.SetDataReceivedEventHandler([&](const Aws::Http::HttpRequest * req, Aws::Http::HttpResponse * resp, long long l) {
+        if (!bytesExpected && resp->HasHeader("Content-Length")) {
+            if (auto length = string2Int<size_t>(resp->GetHeader("Content-Length"))) {
+                bytesExpected = *length;
+            }
+        }
+        bytesDone += l;
+        act.progress(bytesDone, bytesExpected);
+    });
+
+    request.SetContinueRequestHandler([](const Aws::Http::HttpRequest*) {
+        return !isInterrupted();
+    });
+
     FileTransferResult res;
 
     auto now1 = std::chrono::steady_clock::now();
@@ -180,6 +199,8 @@ S3Helper::FileTransferResult S3Helper::getObject(
         auto result = checkAws(fmt("AWS error fetching '%s'", key),
             client->GetObject(request));
 
+        act.progress(result.GetContentLength(), result.GetContentLength());
+
         res.data = decompress(result.GetContentEncoding(),
             dynamic_cast<std::stringstream &>(result.GetBody()).str());
 
@@ -307,11 +328,35 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual
     std::shared_ptr<TransferManager> transferManager;
     std::once_flag transferManagerCreated;
 
+    struct AsyncContext : public Aws::Client::AsyncCallerContext
+    {
+        mutable std::mutex mutex;
+        mutable std::condition_variable cv;
+        const Activity & act;
+
+        void notify() const
+        {
+            cv.notify_one();
+        }
+
+        void wait() const
+        {
+            std::unique_lock<std::mutex> lk(mutex);
+            cv.wait(lk);
+        }
+
+        AsyncContext(const Activity & act) : act(act) {}
+    };
+
     void uploadFile(const std::string & path,
         std::shared_ptr<std::basic_iostream<char>> istream,
         const std::string & mimeType,
         const std::string & contentEncoding)
     {
+        std::string uri = "s3://" + bucketName + "/" + path;
+        Activity act(*logger, lvlTalkative, actFileTransfer,
+            fmt("uploading '%s'", uri),
+            Logger::Fields{uri}, getCurActivity());
         istream->seekg(0, istream->end);
         auto size = istream->tellg();
         istream->seekg(0, istream->beg);
@@ -330,16 +375,25 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual
                 transferConfig.bufferSize = bufferSize;
 
                 transferConfig.uploadProgressCallback =
-                    [](const TransferManager *transferManager,
-                        const std::shared_ptr<const TransferHandle>
-                        &transferHandle)
+                    [](const TransferManager * transferManager,
+                        const std::shared_ptr<const TransferHandle> & transferHandle)
                     {
-                        //FIXME: find a way to properly abort the multipart upload.
-                        //checkInterrupt();
-                        debug("upload progress ('%s'): '%d' of '%d' bytes",
-                            transferHandle->GetKey(),
-                            transferHandle->GetBytesTransferred(),
-                            transferHandle->GetBytesTotalSize());
+                        auto context = std::dynamic_pointer_cast<const AsyncContext>(transferHandle->GetContext());
+                        size_t bytesDone = transferHandle->GetBytesTransferred();
+                        size_t bytesTotal = transferHandle->GetBytesTotalSize();
+                        try {
+                            checkInterrupt();
+                            context->act.progress(bytesDone, bytesTotal);
+                        } catch (...) {
+                            context->notify();
+                        }
+                    };
+                transferConfig.transferStatusUpdatedCallback =
+                    [](const TransferManager * transferManager,
+                        const std::shared_ptr<const TransferHandle> & transferHandle)
+                    {
+                        auto context = std::dynamic_pointer_cast<const AsyncContext>(transferHandle->GetContext());
+                        context->notify();
                     };
 
                 transferManager = TransferManager::Create(transferConfig);
@@ -353,29 +407,51 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual
             if (contentEncoding != "")
                 throw Error("setting a content encoding is not supported with S3 multi-part uploads");
 
+            auto context = std::make_shared<AsyncContext>(act);
             std::shared_ptr<TransferHandle> transferHandle =
                 transferManager->UploadFile(
                     istream, bucketName, path, mimeType,
                     Aws::Map<Aws::String, Aws::String>(),
-                    nullptr /*, contentEncoding */);
+                    context /*, contentEncoding */);
 
-            transferHandle->WaitUntilFinished();
+            TransferStatus status = transferHandle->GetStatus();
+            while (status == TransferStatus::IN_PROGRESS || status == TransferStatus::NOT_STARTED) {
+                if (!isInterrupted()) {
+                    context->wait();
+                } else {
+                    transferHandle->Cancel();
+                    transferHandle->WaitUntilFinished();
+                }
+                status = transferHandle->GetStatus();
+            }
+            act.progress(transferHandle->GetBytesTransferred(), transferHandle->GetBytesTotalSize());
 
-            if (transferHandle->GetStatus() == TransferStatus::FAILED)
+            if (status == TransferStatus::FAILED)
                 throw Error("AWS error: failed to upload 's3://%s/%s': %s",
                     bucketName, path, transferHandle->GetLastError().GetMessage());
 
-            if (transferHandle->GetStatus() != TransferStatus::COMPLETED)
+            if (status != TransferStatus::COMPLETED)
                 throw Error("AWS error: transfer status of 's3://%s/%s' in unexpected state",
                     bucketName, path);
 
         } else {
+            act.progress(0, size);
 
             auto request =
                 Aws::S3::Model::PutObjectRequest()
                 .WithBucket(bucketName)
                 .WithKey(path);
 
+            size_t bytesSent = 0;
+            request.SetDataSentEventHandler([&](const Aws::Http::HttpRequest * req, long long l) {
+                bytesSent += l;
+                act.progress(bytesSent, size);
+            });
+
+            request.SetContinueRequestHandler([](const Aws::Http::HttpRequest*) {
+                return !isInterrupted();
+            });
+
             request.SetContentType(mimeType);
 
             if (contentEncoding != "")
@@ -385,6 +461,8 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual
 
             auto result = checkAws(fmt("AWS error uploading '%s'", path),
                 s3Helper.client->PutObject(request));
+
+            act.progress(size, size);
         }
 
         auto now2 = std::chrono::steady_clock::now();

src/libstore/unix/build/local-derivation-goal.cc

@@ -42,7 +42,7 @@
 
 /* Includes required for chroot support. */
 #ifdef __linux__
-# include "nix/store/fchmodat2-compat.hh"
+# include "linux/fchmodat2-compat.hh"
 # include <sys/ioctl.h>
 # include <net/if.h>
 # include <netinet/ip.h>
@@ -567,7 +567,14 @@ void LocalDerivationGoal::startBuilder()
     } else {
         tmpDir = topTmpDir;
     }
-    chownToBuilder(tmpDir);
+
+    /* The TOCTOU between the previous mkdir call and this open call is unavoidable due to
+       POSIX semantics.*/
+    tmpDirFd = AutoCloseFD{open(tmpDir.c_str(), O_RDONLY | O_NOFOLLOW | O_DIRECTORY)};
+    if (!tmpDirFd)
+        throw SysError("failed to open the build temporary directory descriptor '%1%'", tmpDir);
+
+    chownToBuilder(tmpDirFd.get(), tmpDir);
 
     for (auto & [outputName, status] : initialOutputs) {
         /* Set scratch path we'll actually use during the build.
@@ -1159,9 +1166,7 @@ void LocalDerivationGoal::initTmpDir()
             } else {
                 auto hash = hashString(HashAlgorithm::SHA256, i.first);
                 std::string fn = ".attr-" + hash.to_string(HashFormat::Nix32, false);
-                Path p = tmpDir + "/" + fn;
-                writeFile(p, rewriteStrings(i.second, inputRewrites));
-                chownToBuilder(p);
+                writeBuilderFile(fn, rewriteStrings(i.second, inputRewrites));
                 env[i.first + "Path"] = tmpDirInSandbox + "/" + fn;
             }
         }
@@ -1266,11 +1271,9 @@ void LocalDerivationGoal::writeStructuredAttrs()
 
         auto jsonSh = writeStructuredAttrsShell(json);
 
-        writeFile(tmpDir + "/.attrs.sh", rewriteStrings(jsonSh, inputRewrites));
-        chownToBuilder(tmpDir + "/.attrs.sh");
+        writeBuilderFile(".attrs.sh", rewriteStrings(jsonSh, inputRewrites));
         env["NIX_ATTRS_SH_FILE"] = tmpDirInSandbox + "/.attrs.sh";
-        writeFile(tmpDir + "/.attrs.json", rewriteStrings(json.dump(), inputRewrites));
-        chownToBuilder(tmpDir + "/.attrs.json");
+        writeBuilderFile(".attrs.json", rewriteStrings(json.dump(), inputRewrites));
         env["NIX_ATTRS_JSON_FILE"] = tmpDirInSandbox + "/.attrs.json";
     }
 }
@@ -1781,6 +1784,24 @@ void setupSeccomp()
 #endif
 }
 
+void LocalDerivationGoal::chownToBuilder(int fd, const Path & path)
+{
+    if (!buildUser) return;
+    if (fchown(fd, buildUser->getUID(), buildUser->getGID()) == -1)
+        throw SysError("cannot change ownership of file '%1%'", path);
+}
+
+void LocalDerivationGoal::writeBuilderFile(
+    const std::string & name,
+    std::string_view contents)
+{
+    auto path = std::filesystem::path(tmpDir) / name;
+    AutoCloseFD fd{openat(tmpDirFd.get(), name.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC | O_EXCL | O_NOFOLLOW, 0666)};
+    if (!fd)
+        throw SysError("creating file %s", path);
+    writeFile(fd, path, contents);
+    chownToBuilder(fd.get(), path);
+}
 
 void LocalDerivationGoal::runChild()
 {
@@ -3000,6 +3021,15 @@ void LocalDerivationGoal::checkOutputs(const std::map<std::string, ValidPathInfo
 void LocalDerivationGoal::deleteTmpDir(bool force)
 {
     if (topTmpDir != "") {
+        /* As an extra precaution, even in the event of `deletePath` failing to
+         * clean up, the `tmpDir` will be chowned as if we were to move
+         * it inside the Nix store.
+         *
+         * This hardens against an attack which smuggles a file descriptor
+         * to make use of the temporary directory.
+         */
+        chmod(topTmpDir.c_str(), 0000);
+
         /* Don't keep temporary directories for builtins because they
            might have privileged stuff (like a copy of netrc). */
         if (settings.keepFailed && !force && !drv->isBuiltin()) {

src/libstore/unix/include/nix/store/build/local-derivation-goal.hh

@@ -37,6 +37,11 @@ struct LocalDerivationGoal : public DerivationGoal
      */
     Path topTmpDir;
 
+    /**
+     * The file descriptor of the temporary directory.
+     */
+    AutoCloseFD tmpDirFd;
+
     /**
      * The path of the temporary directory in the sandbox.
      */
@@ -239,9 +244,24 @@ struct LocalDerivationGoal : public DerivationGoal
 
     /**
      * Make a file owned by the builder.
+     *
+     * SAFETY: this function is prone to TOCTOU as it receives a path and not a descriptor.
+     * It's only safe to call in a child of a directory only visible to the owner.
      */
     void chownToBuilder(const Path & path);
 
+    /**
+     * Make a file owned by the builder addressed by its file descriptor.
+     */
+    void chownToBuilder(int fd, const Path & path);
+
+    /**
+     * Create a file in `tmpDir` owned by the builder.
+     */
+    void writeBuilderFile(
+        const std::string & name,
+        std::string_view contents);
+
     int getChildStatus() override;
 
     /**

src/libutil-test-support/include/nix/util/tests/tracing-file-system-object-sink.hh

@@ -1,41 +0,0 @@
-#pragma once
-#include "nix/util/fs-sink.hh"
-
-namespace nix::test {
-
-/**
- * A `FileSystemObjectSink` that traces calls, writing to stderr.
- */
-class TracingFileSystemObjectSink : public virtual FileSystemObjectSink
-{
-    FileSystemObjectSink & sink;
-public:
-    TracingFileSystemObjectSink(FileSystemObjectSink & sink)
-        : sink(sink)
-    {
-    }
-
-    void createDirectory(const CanonPath & path) override;
-
-    void createRegularFile(const CanonPath & path, std::function<void(CreateRegularFileSink &)> fn) override;
-
-    void createSymlink(const CanonPath & path, const std::string & target) override;
-};
-
-/**
- * A `ExtendedFileSystemObjectSink` that traces calls, writing to stderr.
- */
-class TracingExtendedFileSystemObjectSink : public TracingFileSystemObjectSink, public ExtendedFileSystemObjectSink
-{
-    ExtendedFileSystemObjectSink & sink;
-public:
-    TracingExtendedFileSystemObjectSink(ExtendedFileSystemObjectSink & sink)
-        : TracingFileSystemObjectSink(sink)
-        , sink(sink)
-    {
-    }
-
-    void createHardlink(const CanonPath & path, const CanonPath & target) override;
-};
-
-}

src/libutil-test-support/tracing-file-system-object-sink.cc

@@ -1,34 +0,0 @@
-#include <iostream>
-#include "nix/tracing-file-system-object-sink.hh"
-
-namespace nix::test {
-
-void TracingFileSystemObjectSink::createDirectory(const CanonPath & path)
-{
-    std::cerr << "createDirectory(" << path << ")\n";
-    sink.createDirectory(path);
-}
-
-void TracingFileSystemObjectSink::createRegularFile(
-    const CanonPath & path, std::function<void(CreateRegularFileSink &)> fn)
-{
-    std::cerr << "createRegularFile(" << path << ")\n";
-    sink.createRegularFile(path, [&](CreateRegularFileSink & crf) {
-        // We could wrap this and trace about the chunks of data and such
-        fn(crf);
-    });
-}
-
-void TracingFileSystemObjectSink::createSymlink(const CanonPath & path, const std::string & target)
-{
-    std::cerr << "createSymlink(" << path << ", target: " << target << ")\n";
-    sink.createSymlink(path, target);
-}
-
-void TracingExtendedFileSystemObjectSink::createHardlink(const CanonPath & path, const CanonPath & target)
-{
-    std::cerr << "createHardlink(" << path << ", target: " << target << ")\n";
-    sink.createHardlink(path, target);
-}
-
-} // namespace nix::test

src/libutil-tests/file-content-address.cc

@@ -1,3 +1,4 @@
+#include <gmock/gmock.h>
 #include <gtest/gtest.h>
 
 #include "nix/util/file-content-address.hh"
@@ -26,8 +27,11 @@ TEST(FileSerialisationMethod, testRoundTripPrintParse_2) {
     }
 }
 
-TEST(FileSerialisationMethod, testParseFileSerialisationMethodOptException) {
-    EXPECT_THROW(parseFileSerialisationMethod("narwhal"), UsageError);
+TEST(FileSerialisationMethod, testParseFileSerialisationMethodOptException)
+{
+    EXPECT_THAT(
+        []() { parseFileSerialisationMethod("narwhal"); },
+        testing::ThrowsMessage<UsageError>(testing::HasSubstr("narwhal")));
 }
 
 /* ----------------------------------------------------------------------------
@@ -54,8 +58,11 @@ TEST(FileIngestionMethod, testRoundTripPrintParse_2) {
     }
 }
 
-TEST(FileIngestionMethod, testParseFileIngestionMethodOptException) {
-    EXPECT_THROW(parseFileIngestionMethod("narwhal"), UsageError);
+TEST(FileIngestionMethod, testParseFileIngestionMethodOptException)
+{
+    EXPECT_THAT(
+        []() { parseFileIngestionMethod("narwhal"); },
+        testing::ThrowsMessage<UsageError>(testing::HasSubstr("narwhal")));
 }
 
 }

src/libutil-tests/file-system.cc

@@ -275,4 +275,26 @@ TEST(makeParentCanonical, root)
 {
     ASSERT_EQ(makeParentCanonical("/"), "/");
 }
+
+/* ----------------------------------------------------------------------------
+ * DirectoryIterator
+ * --------------------------------------------------------------------------*/
+
+TEST(DirectoryIterator, works)
+{
+    auto tmpDir = nix::createTempDir();
+    nix::AutoDelete delTmpDir(tmpDir, true);
+
+    nix::writeFile(tmpDir + "/somefile", "");
+
+    for (auto path : DirectoryIterator(tmpDir)) {
+        ASSERT_EQ(path.path().string(), tmpDir + "/somefile");
+    }
+}
+
+TEST(DirectoryIterator, nonexistent)
+{
+    ASSERT_THROW(DirectoryIterator("/schnitzel/darmstadt/pommes"), SysError);
+}
+
 }