Add release note

This allows doing `builtins.getFlake ./subflake` instead of ugly hacks.

.github/actions/install-nix-action/action.yaml

@@ -24,8 +24,8 @@ inputs:
     description: "Github token"
     required: true
   use_cache:
-    description: "Whether to setup magic-nix-cache"
-    default: true
+    description: "Whether to setup github actions cache (not implemented currently)"
+    default: false
     required: false
 runs:
   using: "composite"
@@ -122,10 +122,3 @@ runs:
         source-url: ${{ inputs.experimental-installer-version != 'latest' && 'https://artifacts.nixos.org/experimental-installer/tag/${{ inputs.experimental-installer-version }}/${{ env.EXPERIMENTAL_INSTALLER_ARTIFACT }}' || '' }}
         nix-package-url: ${{ inputs.dogfood == 'true' && steps.download-nix-installer.outputs.tarball-path || (inputs.tarball_url || '') }}
         extra-conf: ${{ inputs.extra_nix_config }}
-    - uses: DeterminateSystems/magic-nix-cache-action@565684385bcd71bad329742eefe8d12f2e765b39 # v13
-      if: ${{ inputs.use_cache == 'true' }}
-      with:
-        diagnostic-endpoint: ''
-        use-flakehub: false
-        use-gha-cache: true
-        source-revision: 92d9581367be2233c2d5714a2640e1339f4087d8 # main

.github/workflows/upload-release.yml

@@ -39,13 +39,24 @@ jobs:
           role-to-assume: "arn:aws:iam::080433136561:role/nix-release"
           role-session-name: nix-release-oidc-${{ github.run_id }}
           aws-region: eu-west-1
+      - name: Disable containerd image store
+        run: |
+          # Docker 28+ defaults to the containerd image store, which
+          # pushes layers uncompressed instead of gzip. OCI clients
+          # that only support gzip (e.g. go-containerregistry) fail
+          # with "gzip: invalid header". Disabling the containerd
+          # snapshotter restores the classic storage driver, which
+          # preserves gzip-compressed layers through the
+          # `docker load` / `docker push` pipeline.
+          echo '{"features":{"containerd-snapshotter":false}}' | sudo tee /etc/docker/daemon.json > /dev/null
+          sudo systemctl restart docker
       - name: Login to Docker Hub
-        uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
+        uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
       - name: Login to GitHub Container Registry
-        uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
+        uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0
         with:
           registry: ghcr.io
           username: ${{ github.actor }}

.gitignore

@@ -1,5 +1,7 @@
 # Default meson build dir
 /build
+# Meson creates this file too
+src/.wraplock
 
 # /tests/functional/
 /tests/functional/common/subst-vars.sh

doc/manual/rl-next/beta-installer

@@ -0,0 +1,29 @@
+---
+synopsis: "Rust nix-installer in beta"
+prs: []
+---
+
+The Rust-based rewrite of the Nix installer is now in beta.
+We'd love help testing it out!
+
+To test out the new installer, run:
+```
+curl -sSfL https://artifacts.nixos.org/nix-installer | sh -s -- install
+```
+
+This installer can be run even when you have an existing, script-based Nix installation without any adjustments.
+
+This new installer also comes with the ability to uninstall your Nix installation; run:
+```
+/nix/nix-installer uninstall
+```
+
+This will get rid of your entire Nix installation (even if you installed over an existing, script-based installation).
+
+This installer is a modified version of the [Determinate Nix Installer](https://github.com/DeterminateSystems/nix-installer) by Determinate Systems.
+Thanks to Determinate Systems for all the investment they've put into the installer.
+
+Source for the installer is in https://github.com/NixOS/nix-installer.
+Report any issues in that repo.
+
+For CI usage, a GitHub Action to install Nix using this installer is available at https://github.com/NixOS/nix-installer-action.

doc/manual/rl-next/c-api-new-store-methods.md

@@ -0,0 +1,9 @@
+---
+synopsis: "C API: New store API methods"
+prs: [14766]
+---
+
+The C API now includes additional methods:
+
+- `nix_store_query_path_from_hash_part()` - Get the full store path given its hash part
+- `nix_store_copy_path()` - Copy a single store path between two stores, allows repairs and configuring signature checking

doc/manual/rl-next/getflake-path.md

@@ -0,0 +1,6 @@
+---
+synopsis: "`builtins.getFlake` now supports path values"
+prs: [15290]
+---
+
+`builtins.getFlake` now accepts path values in addition to flakerefs, allowing you to write `builtins.getFlake ./subflake` instead of having to use ugly workarounds to construct a pure flakeref.

doc/manual/rl-next/ignore-gc-delete-failure.md

@@ -0,0 +1,10 @@
+---
+synopsis: "New setting `ignore-gc-delete-failure` for local stores"
+prs: [15054]
+---
+
+A new local store setting [`ignore-gc-delete-failure`](@docroot@/store/types/local-store.md#store-local-store-ignore-gc-delete-failure) has been added.
+When enabled, garbage collection will log warnings instead of failing when it cannot delete store paths.
+This is useful when running Nix as an unprivileged user that may not have write access to all paths in the store.
+
+This setting is experimental and requires the [`local-overlay-store`](@docroot@/development/experimental-features.md#xp-feature-local-overlay-store) experimental feature.

doc/manual/rl-next/mtls-substituter.md

@@ -0,0 +1,15 @@
+---
+synopsis: Support HTTPS binary caches using mTLS (client certificate) authentication
+issues: [13002]
+prs: [13030]
+---
+
+Added support for `tls-certificate` and `tls-private-key` options in substituter URLs.
+
+Example:
+
+```
+https://substituter.invalid?tls-certificate=/path/to/cert.pem&tls-private-key=/path/to/key.pem
+```
+
+When these options are configured, Nix will use this certificate/private key pair to authenticate to the server.

doc/manual/rl-next/roots-daemon.md

@@ -0,0 +1,11 @@
+---
+synopsis: New command `nix store roots-daemon` for serving GC roots
+prs: [15143]
+---
+
+New command [`nix store roots-daemon`](@docroot@/command-ref/new-cli/nix3-store-roots-daemon.md) runs a daemon that serves garbage collector roots over a Unix domain socket.
+It enables the garbage collector to discover runtime roots when the main Nix daemon doesn't have `CAP_SYS_PTRACE` capability and therefore cannot scan `/proc`.
+
+The garbage collector can be configured to use this daemon via the [`use-roots-daemon`](@docroot@/store/types/local-store.md#store-experimental-option-use-roots-daemon) store setting.
+
+This feature requires the [`local-overlay-store` experimental feature](@docroot@/development/experimental-features.md#xp-feature-local-overlay-store).

doc/manual/rl-next/s3-virtual-hosted-style.md

@@ -0,0 +1,32 @@
+---
+synopsis: S3 binary caches now use virtual-hosted-style addressing by default
+issues: [15208]
+---
+
+S3 binary caches now use virtual-hosted-style URLs
+(`https://bucket.s3.region.amazonaws.com/key`) instead of path-style URLs
+(`https://s3.region.amazonaws.com/bucket/key`) when connecting to standard AWS
+S3 endpoints. This enables HTTP/2 multiplexing and fixes TCP connection
+exhaustion (TIME_WAIT socket accumulation) under high-concurrency workloads.
+
+A new `addressing-style` store option controls this behavior:
+
+- `auto` (default): virtual-hosted-style for standard AWS endpoints, path-style
+  for custom endpoints.
+- `path`: forces path-style addressing (deprecated by AWS).
+- `virtual`: forces virtual-hosted-style addressing (bucket names must not
+  contain dots).
+
+Bucket names containing dots (e.g., `my.bucket.name`) automatically fall back
+to path-style addressing in `auto` mode, because dotted names create
+multi-level subdomains that break TLS wildcard certificate validation.
+
+Example using path-style for backwards compatibility:
+
+```
+s3://my-bucket/key?region=us-east-1&addressing-style=path
+```
+
+Additionally, TCP keep-alive is now enabled on all HTTP connections, preventing
+idle connections from being silently dropped by intermediate network devices
+(NATs, firewalls, load balancers).

doc/manual/source/SUMMARY.md.in

@@ -135,7 +135,9 @@
   - [Serving Tarball Flakes](protocols/tarball-fetcher.md)
   - [Store Path Specification](protocols/store-path.md)
   - [Nix Archive (NAR) Format](protocols/nix-archive/index.md)
+  - [Nix Cache Info Format](protocols/nix-cache-info.md)
   - [Derivation "ATerm" file format](protocols/derivation-aterm.md)
+  - [Nix32 Encoding](protocols/nix32.md)
 - [C API](c-api.md)
 - [Glossary](glossary.md)
 - [Development](development/index.md)

doc/manual/source/command-ref/env-common.md

@@ -57,11 +57,6 @@ Most Nix commands interpret the following environment variables:
 
   Overrides the location of the Nix store (default `prefix/store`).
 
-- <span id="env-NIX_DATA_DIR">[`NIX_DATA_DIR`](#env-NIX_DATA_DIR)</span>
-
-  Overrides the location of the Nix static data directory (default
-  `prefix/share`).
-
 - <span id="env-NIX_LOG_DIR">[`NIX_LOG_DIR`](#env-NIX_LOG_DIR)</span>
 
   Overrides the location of the Nix log directory (default

doc/manual/source/installation/building-source.md

@@ -6,14 +6,23 @@ It is broken up into multiple Meson packages, which are optionally combined in a
 There are no mandatory extra steps to the building process:
 generic Meson installation instructions like [this](https://mesonbuild.com/Quick-guide.html#using-meson-as-a-distro-packager) should work.
 
-The installation path can be specified by passing the `-Dprefix=prefix`
-to `configure`. The default installation directory is `/usr/local`. You
+```bash
+git clone https://github.com/NixOS/nix.git
+cd nix
+meson setup build
+cd build
+ninja
+(sudo) ninja install
+```
+
+The installation path can be specified by passing `-Dprefix=prefix`
+to `meson setup build`. The default installation directory is `/usr/local`. You
 can change this to any location you like. You must have write permission
 to the *prefix* path.
 
 Nix keeps its *store* (the place where packages are stored) in
 `/nix/store` by default. This can be changed using
-`-Dstore-dir=path`.
+`-Dlibstore:store-dir=path`.
 
 > **Warning**
 >

doc/manual/source/language/advanced-attributes.md

@@ -338,7 +338,7 @@ Here is more information on the `output*` attributes, and what values they may b
     This will specify the output hash of the single output of a [fixed-output derivation].
 
     The `outputHash` attribute must be a string containing the hash in either hexadecimal or "nix32" encoding, or following the format for integrity metadata as defined by [SRI](https://www.w3.org/TR/SRI/).
-    The "nix32" encoding is an adaptation of base-32 encoding.
+    The ["nix32" encoding](@docroot@/protocols/nix32.md) is Nix's variant of base-32 encoding.
 
     > **Note**
     >

doc/manual/source/package-management/binary-cache-substituter.md

@@ -19,17 +19,16 @@ whatever port you like:
 $ nix-serve -p 8080
 ```
 
-To check whether it works, try the following on the client:
+To check whether it works, try fetching the [`nix-cache-info`](@docroot@/protocols/nix-cache-info.md) file on the client:
 
 ```console
 $ curl http://avalon:8080/nix-cache-info
+StoreDir: /nix/store
+WantMassQuery: 1
+Priority: 30
 ```
 
-which should print something like:
-
-    StoreDir: /nix/store
-    WantMassQuery: 1
-    Priority: 30
+When writing to a binary cache (e.g., with [`nix copy`](@docroot@/command-ref/new-cli/nix3-copy.md)), Nix creates [`nix-cache-info`](@docroot@/protocols/nix-cache-info.md) automatically if it doesn't exist.
 
 On the client side, you can tell Nix to use your binary cache using
 `--substituters`, e.g.:

doc/manual/source/protocols/json/schema/store-path-v1.yaml

@@ -24,7 +24,7 @@ description: |
 
   The format follows this pattern: `${digest}-${name}`
 
-  - **hash**: Digest rendered in a custom variant of [Base32](https://en.wikipedia.org/wiki/Base32) (20 arbitrary bytes become 32 ASCII characters)
+  - **hash**: Digest rendered in [Nix32](@docroot@/protocols/nix32.md), a variant of base-32 (20 hash bytes become 32 ASCII characters)
   - **name**: The package name and optional version/suffix information
 
 type: string

doc/manual/source/protocols/nix-cache-info.md

@@ -0,0 +1,55 @@
+# Nix Cache Info Format
+
+The `nix-cache-info` file is a metadata file at the root of a [binary cache](@docroot@/package-management/binary-cache-substituter.md) (e.g., `https://cache.example.com/nix-cache-info`).
+
+MIME type: `text/x-nix-cache-info`
+
+## Format
+
+Line-based key-value format:
+
+```
+Key: value
+```
+
+Leading and trailing whitespace is trimmed from values.
+Lines without a colon are ignored.
+Unknown keys are silently ignored.
+
+## Fields
+
+### `StoreDir`
+
+The Nix store directory path that this cache was built for (e.g., `/nix/store`).
+
+If present, Nix verifies that this matches the client's store directory:
+
+```
+error: binary cache 'https://example.com' is for Nix stores with prefix '/nix/store', not '/home/user/nix/store'
+```
+
+### `WantMassQuery`
+
+`1` or `0`. Sets the default for [`want-mass-query`](@docroot@/store/types/http-binary-cache-store.md#store-http-binary-cache-store-want-mass-query).
+
+### `Priority`
+
+Integer. Sets the default for [`priority`](@docroot@/store/types/http-binary-cache-store.md#store-http-binary-cache-store-priority).
+
+## Example
+
+```
+StoreDir: /nix/store
+WantMassQuery: 1
+Priority: 30
+```
+
+## Caching Behavior
+
+Nix caches `nix-cache-info` in the [cache directory](@docroot@/command-ref/env-common.md#env-NIX_CACHE_HOME) with a 7-day TTL.
+
+## See Also
+
+- [HTTP Binary Cache Store](@docroot@/store/types/http-binary-cache-store.md)
+- [Serving a Nix store via HTTP](@docroot@/package-management/binary-cache-substituter.md)
+- [`substituters`](@docroot@/command-ref/conf-file.md#conf-substituters)

doc/manual/source/protocols/nix32.md

@@ -0,0 +1,19 @@
+# Nix32 Encoding
+
+Nix32 is Nix's variant of base-32 encoding, used for [store path digests](@docroot@/protocols/store-path.md), hash output via [`nix hash`](@docroot@/command-ref/new-cli/nix3-hash.md), and the [`outputHash`](@docroot@/language/advanced-attributes.md#adv-attr-outputHash) derivation attribute.
+
+## Alphabet
+
+The Nix32 alphabet consists of these 32 characters:
+
+```
+0 1 2 3 4 5 6 7 8 9 a b c d f g h i j k l m n p q r s v w x y z
+```
+
+The letters `e`, `o`, `u`, and `t` are omitted.
+
+## Byte Order
+
+Nix32 encoding processes the hash bytes from the end (last byte first), while base-16 encoding processes from the beginning (first byte first).
+
+Consequently, the string sort order is determined primarily by the first bytes for base-16, and by the last bytes for Nix32.

doc/manual/source/protocols/store-path.md

@@ -20,12 +20,11 @@ where
 
 - `store-dir` = the [store directory](@docroot@/store/store-path.md#store-directory)
 
-- `digest` = base-32 representation of the compressed to 160 bits [SHA-256] hash of `fingerprint`
+- `digest` = base-32 representation of the compressed to 160 bits [SHA-256] hash of `fingerprint`.
 
-For the definition of the hash compression algorithm, please refer to the section 5.1 of
-the [Nix thesis](https://edolstra.github.io/pubs/phd-thesis.pdf), which also defines the
-specifics of base-32 encoding. Note that base-32 encoding processes the hash bytestring from
-the end, while base-16 processes in from the beginning.
+  Nix uses a custom base-32 encoding called [Nix32](@docroot@/protocols/nix32.md).
+
+  For the definition of the hash compression algorithm, please refer to section 5.1 of the [Nix thesis](https://edolstra.github.io/pubs/phd-thesis.pdf).
 
 ## Fingerprint
 

doc/manual/source/store/store-path.md

@@ -31,7 +31,7 @@ A store path is rendered to a file system path as the concatenation of
 
 - [Store directory](#store-directory) (typically `/nix/store`)
 - Path separator (`/`)
-- Digest rendered in a custom variant of [Base32](https://en.wikipedia.org/wiki/Base32) (20 arbitrary bytes become 32 ASCII characters)
+- Digest rendered in [Nix32](@docroot@/protocols/nix32.md), a variant of base-32 (20 hash bytes become 32 ASCII characters)
 - Hyphen (`-`)
 - Name
 

flake.lock

@@ -63,15 +63,15 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1763948260,
-        "narHash": "sha256-zZk7fn2ARAqmLwaYTpxBJmj81KIdz11NiWt7ydHHD/M=",
-        "rev": "1c8ba8d3f7634acac4a2094eef7c32ad9106532c",
+        "lastModified": 1771043024,
+        "narHash": "sha256-WoiezqWJQ3OHILah+p6rzNXdJceEAmAhyDFZFZ6pZzY=",
+        "rev": "3aadb7ca9eac2891d52a9dec199d9580a6e2bf44",
         "type": "tarball",
-        "url": "https://releases.nixos.org/nixos/25.05/nixos-25.05.813095.1c8ba8d3f763/nixexprs.tar.xz"
+        "url": "https://releases.nixos.org/nixos/25.11/nixos-25.11.5960.3aadb7ca9eac/nixexprs.tar.xz"
       },
       "original": {
         "type": "tarball",
-        "url": "https://channels.nixos.org/nixos-25.05/nixexprs.tar.xz"
+        "url": "https://channels.nixos.org/nixos-25.11/nixexprs.tar.xz"
       }
     },
     "nixpkgs-23-11": {

flake.nix

@@ -1,7 +1,7 @@
 {
   description = "The purely functional package manager";
 
-  inputs.nixpkgs.url = "https://channels.nixos.org/nixos-25.05/nixexprs.tar.xz";
+  inputs.nixpkgs.url = "https://channels.nixos.org/nixos-25.11/nixexprs.tar.xz";
 
   inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2";
   inputs.nixpkgs-23-11.url = "github:NixOS/nixpkgs/a62e6edd6d5e1fa0329b8653c801147986f8d446";
@@ -115,6 +115,9 @@
                     }
                     // lib.optionalAttrs (crossSystem == "x86_64-unknown-freebsd13") {
                       useLLVM = true;
+                    }
+                    // lib.optionalAttrs (crossSystem == "x86_64-w64-mingw32") {
+                      emulator = pkgs: "${pkgs.buildPackages.wineWow64Packages.stable_11}/bin/wine";
                     };
                 overlays = [
                   (overlayFor (pkgs: pkgs.${stdenv}))
@@ -406,6 +409,8 @@
 
               "nix-cmd" = { };
 
+              "nix-nswrapper" = { };
+
               "nix-cli" = { };
 
               "nix-everything" = { };

maintainers/flake-module.nix

@@ -88,16 +88,23 @@
               ''^tests/functional/lang/eval-fail-path-slash\.nix$''
               ''^tests/functional/lang/eval-fail-toJSON-non-utf-8\.nix$''
               ''^tests/functional/lang/eval-fail-set\.nix$''
+
+              # Language tests, don't churn the formatting of strings
+              ''^tests/functional/lang/eval-fail-fromTOML-overflow\.nix$''
+              ''^tests/functional/lang/eval-fail-fromTOML-underflow\.nix$''
+              ''^tests/functional/lang/eval-fail-bad-string-interpolation-3\.nix$''
+              ''^tests/functional/lang/eval-fail-bad-string-interpolation-4\.nix$''
+              ''^tests/functional/lang/eval-okay-regex-match2\.nix$''
             ];
           };
           clang-format = {
             enable = true;
             # https://github.com/cachix/git-hooks.nix/pull/532
-            package = pkgs.llvmPackages_latest.clang-tools;
+            package = pkgs.llvmPackages_21.clang-tools;
             excludes = [
               # We don't want to format test data
               # ''tests/(?!nixos/).*\.nix''
-              ''^src/[^/]*-tests/data/.*$''
+              "^src/[^/]*-tests/data/.*$"
 
               # Don't format vendored code
               ''^doc/manual/redirects\.js$''

meson.build

@@ -24,6 +24,10 @@ subproject('libcmd')
 # Executables
 subproject('nix')
 
+if host_machine.system() == 'linux'
+  subproject('nswrapper')
+endif
+
 # Docs
 if get_option('doc-gen')
   subproject('internal-api-docs')

misc/systemd/nix-daemon.service.in

@@ -11,7 +11,7 @@ ExecStart=@@bindir@/nix-daemon nix-daemon --daemon
 KillMode=process
 LimitNOFILE=1048576
 TasksMax=1048576
-Delegate=yes
+Delegate=
 
 [Install]
 WantedBy=multi-user.target

nix-meson-build-support/common/asan-options/meson.build

@@ -1,7 +1,7 @@
 # Clang gets grumpy about missing libasan symbols if -shared-libasan is not
 # passed when building shared libs, at least on Linux
 if cxx.get_id() == 'clang' and ('address' in get_option('b_sanitize') or 'undefined' in get_option(
-  'b_sanitize',
+    'b_sanitize',
 ))
   add_project_link_arguments('-shared-libasan', language : 'cpp')
 endif

nix-meson-build-support/common/meson.build

@@ -22,6 +22,8 @@ add_project_arguments(
   '-Werror=undef',
   '-Werror=unused-result',
   '-Werror=sign-compare',
+  '-Werror=return-type',
+  '-Werror=non-virtual-dtor',
   '-Wignored-qualifiers',
   '-Wimplicit-fallthrough',
   '-Wno-deprecated-declarations',
@@ -31,6 +33,13 @@ add_project_arguments(
 # GCC doesn't benefit much from precompiled headers.
 do_pch = cxx.get_id() == 'clang'
 
+if cxx.get_id() == 'gcc'
+  add_project_arguments(
+    '-Wno-interference-size', # Used for C++ ABI only. We don't provide any guarantees about different march tunings.
+    language : 'cpp',
+  )
+endif
+
 # This is a clang-only option for improving build times.
 # It forces the instantiation of templates in the PCH itself and
 # not every translation unit it's included in.
@@ -40,6 +49,11 @@ do_pch = cxx.get_id() == 'clang'
 # instantiations in libutil and libstore.
 if cxx.get_id() == 'clang'
   add_project_arguments('-fpch-instantiate-templates', language : 'cpp')
+  # Catch brace elision bugs: when WorkerProto::Version changed from `unsigned int`
+  # to `struct { unsigned int major; uint8_t minor; }`, `.version = 16` silently
+  # became `.version = {16, 0}` instead of failing, breaking protocol compatibility
+  # in a subtle way
+  add_project_arguments('-Werror=c99-designator', language : 'cpp')
 endif
 
 # Detect if we're using libstdc++ (GCC's standard library)

packaging/binary-tarball.nix

@@ -4,6 +4,7 @@
   buildPackages,
   cacert,
   nix,
+  nixComponents2,
 }:
 
 let
@@ -11,6 +12,7 @@ let
   installerClosureInfo = buildPackages.closureInfo {
     rootPaths = [
       nix
+      nixComponents2.nix-manual.man
       cacert
     ];
   };
@@ -42,6 +44,7 @@ runCommand "nix-binary-tarball-${version}" env ''
     --subst-var-by cacert ${cacert}
   substitute ${../scripts/install-multi-user.sh} $TMPDIR/install-multi-user \
     --subst-var-by nix ${nix} \
+    --subst-var-by nix-manual ${nixComponents2.nix-manual.man} \
     --subst-var-by cacert ${cacert}
 
   if type -p shellcheck; then

packaging/components.nix

@@ -155,12 +155,14 @@ let
     ];
   };
 
-  mesonBuildLayer = finalAttrs: prevAttrs: {
+  mesonBuildLayer = finalAttrs: prevAttrs: rec {
     nativeBuildInputs = prevAttrs.nativeBuildInputs or [ ] ++ [
       pkg-config
     ];
     separateDebugInfo = !stdenv.hostPlatform.isStatic;
-    hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie";
+    # needed by separateDebugInfo
+    # SEE: https://github.com/NixOS/nixpkgs/pull/394674/commits/a4d355342976e9e9823fb94f133bc43ebec9da5b
+    __structuredAttrs = separateDebugInfo;
   };
 
   mesonLibraryLayer = finalAttrs: prevAttrs: {
@@ -416,6 +418,8 @@ in
 
   nix-cmd = callPackage ../src/libcmd/package.nix { };
 
+  nix-nswrapper = callPackage ../src/nswrapper/package.nix { };
+
   /**
     The Nix command line interface. Note that this does not include its tests, whereas `nix-everything` does.
   */

packaging/dependencies.nix

@@ -30,33 +30,13 @@ scope: {
         NIX_CFLAGS_COMPILE = "-DINITIAL_MARK_STACK_SIZE=1048576";
       });
 
-  lowdown = pkgs.lowdown.overrideAttrs (prevAttrs: rec {
-    version = "2.0.2";
-    src = pkgs.fetchurl {
-      url = "https://kristaps.bsd.lv/lowdown/snapshots/lowdown-${version}.tar.gz";
-      hash = "sha512-cfzhuF4EnGmLJf5EGSIbWqJItY3npbRSALm+GarZ7SMU7Hr1xw0gtBFMpOdi5PBar4TgtvbnG4oRPh+COINGlA==";
-    };
-    nativeBuildInputs = prevAttrs.nativeBuildInputs ++ [ pkgs.buildPackages.bmake ];
-    postInstall =
-      lib.replaceStrings [ "lowdown.so.1" "lowdown.1.dylib" ] [ "lowdown.so.2" "lowdown.2.dylib" ]
-        (prevAttrs.postInstall or "");
-    patches = [ ];
-  });
+  curl = pkgs.curl.override {
+    http3Support = !pkgs.stdenv.hostPlatform.isWindows;
+  };
 
-  # TODO: Remove this when https://github.com/NixOS/nixpkgs/pull/442682 is included in a stable release
-  toml11 =
-    if lib.versionAtLeast pkgs.toml11.version "4.4.0" then
-      pkgs.toml11
-    else
-      pkgs.toml11.overrideAttrs rec {
-        version = "4.4.0";
-        src = pkgs.fetchFromGitHub {
-          owner = "ToruNiina";
-          repo = "toml11";
-          tag = "v${version}";
-          hash = "sha256-sgWKYxNT22nw376ttGsTdg0AMzOwp8QH3E8mx0BZJTQ=";
-        };
-      };
+  libblake3 = pkgs.libblake3.override {
+    useTBB = !(stdenv.hostPlatform.isWindows || stdenv.hostPlatform.isStatic);
+  };
 
   # TODO Hack until https://github.com/NixOS/nixpkgs/issues/45462 is fixed.
   boost =

packaging/dev-shell.nix

@@ -131,7 +131,7 @@ pkgs.nixComponents2.nix-util.overrideAttrs (
     ignoreCrossFile = flags: builtins.filter (flag: !(lib.strings.hasInfix "cross-file" flag)) flags;
 
     availableComponents = lib.filterAttrs (
-      k: v: lib.meta.availableOn pkgs.hostPlatform v
+      k: v: lib.meta.availableOn pkgs.stdenv.hostPlatform v
     ) allComponents;
 
     activeComponents = buildInputsClosureCond isInternal (
@@ -300,7 +300,7 @@ pkgs.nixComponents2.nix-util.overrideAttrs (
             lib.filter (x: !isInternal x) (
               lib.lists.concatMap (
                 # Nix manual has a build-time dependency on nix, but we
-                # don't want to do a native build just to enter the ross
+                # don't want to do a native build just to enter the cross
                 # dev shell.
                 #
                 # TODO: think of a more principled fix for this.
@@ -323,7 +323,7 @@ pkgs.nixComponents2.nix-util.overrideAttrs (
             pkgs.buildPackages.shellcheck
             pkgs.buildPackages.include-what-you-use
           ]
-          ++ lib.optional pkgs.hostPlatform.isUnix pkgs.buildPackages.gdb
+          ++ lib.optional stdenv.hostPlatform.isUnix pkgs.buildPackages.gdb
           ++ lib.optional (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform) (
             lib.hiPrio pkgs.buildPackages.clang-tools
           )
@@ -341,7 +341,7 @@ pkgs.nixComponents2.nix-util.overrideAttrs (
 
     buildInputs =
       # TODO change Nixpkgs to mark gbenchmark as building on Windows
-      lib.optional pkgs.hostPlatform.isUnix pkgs.gbenchmark
+      lib.optional stdenv.hostPlatform.isUnix pkgs.gbenchmark
       ++ dedupByString (v: "${v}") (
         lib.filter (x: !isInternal x) (lib.lists.concatMap (c: c.buildInputs) activeComponents)
       )

packaging/everything.nix

@@ -31,6 +31,8 @@
 
   nix-cmd,
 
+  nix-nswrapper,
+
   nix-cli,
 
   nix-functional-tests,
@@ -171,6 +173,9 @@ stdenv.mkDerivation (finalAttrs: {
       # Forwarded outputs
       ln -sT ${nix-manual} $doc
       ln -sT ${nix-manual.man} $man
+    ''
+    + lib.optionalString stdenv.isLinux ''
+      lndir ${nix-nswrapper} $out
     '';
 
   passthru = {

packaging/hydra.nix

@@ -57,6 +57,7 @@ let
         "nix-flake"
         "nix-flake-c"
         "nix-flake-tests"
+        "nix-nswrapper"
         "nix-main"
         "nix-main-c"
         "nix-cmd"

scripts/install-multi-user.sh

@@ -52,6 +52,7 @@ readonly PROFILE_FISH_PREFIXES=(
 readonly PROFILE_NIX_FILE_FISH="$NIX_ROOT/var/nix/profiles/default/etc/profile.d/nix-daemon.fish"
 
 readonly NIX_INSTALLED_NIX="@nix@"
+readonly NIX_INSTALLED_NIX_MAN="@nix-manual@"
 readonly NIX_INSTALLED_CACERT="@cacert@"
 #readonly NIX_INSTALLED_NIX="/nix/store/byi37zv50wnfrpp4d81z3spswd5zva37-nix-2.3.6"
 #readonly NIX_INSTALLED_CACERT="/nix/store/7pi45g541xa8ahwgpbpy7ggsl0xj1jj6-nss-cacert-3.49.2"
@@ -969,6 +970,8 @@ setup_default_profile() {
     task "Setting up the default profile"
     _sudo "to install a bootstrapping Nix in to the default profile" \
           HOME="$ROOT_HOME" "$NIX_INSTALLED_NIX/bin/nix-env" -i "$NIX_INSTALLED_NIX"
+    _sudo "to install Nix man pages in to the default profile" \
+          HOME="$ROOT_HOME" "$NIX_INSTALLED_NIX/bin/nix-env" -i "$NIX_INSTALLED_NIX_MAN"
 
     if [ -z "${NIX_SSL_CERT_FILE:-}" ] || ! [ -f "${NIX_SSL_CERT_FILE:-}" ] || cert_in_store; then
         _sudo "to install a bootstrapping SSL certificate just for Nix in to the default profile" \

scripts/install-systemd-multi-user.sh

@@ -38,6 +38,7 @@ escape_systemd_env() {
 create_systemd_proxy_env() {
     vars="http_proxy https_proxy ftp_proxy all_proxy no_proxy HTTP_PROXY HTTPS_PROXY FTP_PROXY ALL_PROXY NO_PROXY"
     for v in $vars; do
+        # shellcheck disable=SC2268
         if [ "x${!v:-}" != "x" ]; then
             echo "Environment=${v}=$(escape_systemd_env "${!v}")"
         fi

src/libcmd/command.cc

@@ -4,6 +4,7 @@
 #include "nix/cmd/command.hh"
 #include "nix/cmd/legacy.hh"
 #include "nix/cmd/markdown.hh"
+#include "nix/store/globals.hh"
 #include "nix/store/store-open.hh"
 #include "nix/store/local-fs-store.hh"
 #include "nix/store/derivations.hh"
@@ -63,6 +64,25 @@ void NixMultiCommand::run()
     command->second->run();
 }
 
+StoreConfigCommand::StoreConfigCommand() {}
+
+ref<StoreConfig> StoreConfigCommand::getStoreConfig()
+{
+    if (!_storeConfig)
+        _storeConfig = createStoreConfig();
+    return ref<StoreConfig>(_storeConfig);
+}
+
+ref<StoreConfig> StoreConfigCommand::createStoreConfig()
+{
+    return resolveStoreConfig(StoreReference{settings.storeUri.get()});
+}
+
+void StoreConfigCommand::run()
+{
+    run(getStoreConfig());
+}
+
 StoreCommand::StoreCommand() {}
 
 ref<Store> StoreCommand::getStore()
@@ -74,12 +94,20 @@ ref<Store> StoreCommand::getStore()
 
 ref<Store> StoreCommand::createStore()
 {
-    return openStore();
+    auto store = getStoreConfig()->openStore();
+    store->init();
+    return store;
 }
 
-void StoreCommand::run()
+void StoreCommand::run(ref<StoreConfig> storeConfig)
 {
-    run(getStore());
+    // We can either efficiently implement getStore/createStore with memoization,
+    // or use the StoreConfig passed in run.
+    // It's more efficient to memoize, especially since there are some direct users
+    // of getStore. The StoreConfig in both cases should be the same, though.
+    auto store = getStore();
+    assert(&*storeConfig == &store->config);
+    run(std::move(store));
 }
 
 CopyCommand::CopyCommand()
@@ -88,28 +116,28 @@ CopyCommand::CopyCommand()
         .longName = "from",
         .description = "URL of the source Nix store.",
         .labels = {"store-uri"},
-        .handler = {&srcUri},
+        .handler = {[this](std::string s) { srcUri = StoreReference::parse(s); }},
     });
 
     addFlag({
         .longName = "to",
         .description = "URL of the destination Nix store.",
         .labels = {"store-uri"},
-        .handler = {&dstUri},
+        .handler = {[this](std::string s) { dstUri = StoreReference::parse(s); }},
     });
 }
 
-ref<Store> CopyCommand::createStore()
+ref<StoreConfig> CopyCommand::createStoreConfig()
 {
-    return srcUri.empty() ? StoreCommand::createStore() : openStore(srcUri);
+    return !srcUri ? StoreCommand::createStoreConfig() : resolveStoreConfig(StoreReference{*srcUri});
 }
 
 ref<Store> CopyCommand::getDstStore()
 {
-    if (srcUri.empty() && dstUri.empty())
+    if (!srcUri && !dstUri)
         throw UsageError("you must pass '--from' and/or '--to'");
 
-    return dstUri.empty() ? openStore() : openStore(dstUri);
+    return !dstUri ? openStore() : openStore(StoreReference{*dstUri});
 }
 
 EvalCommand::EvalCommand()
@@ -131,7 +159,7 @@ EvalCommand::~EvalCommand()
 ref<Store> EvalCommand::getEvalStore()
 {
     if (!evalStore)
-        evalStore = evalStoreUrl ? openStore(*evalStoreUrl) : getStore();
+        evalStore = evalStoreUrl ? openStore(StoreReference{*evalStoreUrl}) : getStore();
     return ref<Store>(evalStore);
 }
 
@@ -257,18 +285,18 @@ MixProfile::MixProfile()
     });
 }
 
-void MixProfile::updateProfile(const StorePath & storePath)
+void MixProfile::updateProfile(Store & store_, const StorePath & storePath)
 {
     if (!profile)
         return;
-    auto store = getDstStore().dynamic_pointer_cast<LocalFSStore>();
+    auto * store = dynamic_cast<LocalFSStore *>(&store_);
     if (!store)
         throw Error("'--profile' is not supported for this Nix store");
     auto profile2 = absPath(*profile);
     switchLink(profile2, createGeneration(*store, profile2, storePath));
 }
 
-void MixProfile::updateProfile(const BuiltPaths & buildables)
+void MixProfile::updateProfile(Store & store, const BuiltPaths & buildables)
 {
     if (!profile)
         return;
@@ -292,14 +320,16 @@ void MixProfile::updateProfile(const BuiltPaths & buildables)
         throw UsageError(
             "'--profile' requires that the arguments produce a single store path, but there are %d", result.size());
 
-    updateProfile(result[0]);
+    updateProfile(store, result[0]);
 }
 
 MixDefaultProfile::MixDefaultProfile()
 {
-    profile = getDefaultProfile().string();
+    profile = getDefaultProfile(settings.getProfileDirsOptions()).string();
 }
 
+static constexpr auto environmentVariablesCategory = "Options that change environment variables";
+
 MixEnvironment::MixEnvironment()
     : ignoreEnvironment(false)
 {

src/libcmd/common-eval-args.cc

@@ -148,7 +148,7 @@ MixEvalArgs::MixEvalArgs()
           )",
         .category = category,
         .labels = {"store-url"},
-        .handler = {&evalStoreUrl},
+        .handler = {[this](std::string s) { evalStoreUrl = StoreReference::parse(s); }},
     });
 }
 

src/libcmd/get-build-log.cc

@@ -0,0 +1,31 @@
+#include "nix/cmd/get-build-log.hh"
+#include "nix/store/log-store.hh"
+#include "nix/store/store-open.hh"
+
+namespace nix {
+
+std::string fetchBuildLog(ref<Store> store, const StorePath & path, std::string_view what)
+{
+    auto subs = getDefaultSubstituters();
+
+    subs.push_front(store);
+
+    for (auto & sub : subs) {
+        auto * logSubP = dynamic_cast<LogStore *>(&*sub);
+        if (!logSubP) {
+            printInfo("Skipped '%s' which does not support retrieving build logs", sub->config.getHumanReadableURI());
+            continue;
+        }
+        auto & logSub = *logSubP;
+
+        auto log = logSub.getBuildLog(path);
+        if (!log)
+            continue;
+        printInfo("got build log for '%s' from '%s'", what, logSub.config.getHumanReadableURI());
+        return *log;
+    }
+
+    throw Error("build log of '%s' is not available", what);
+}
+
+} // namespace nix

src/libcmd/include/nix/cmd/command.hh

@@ -5,6 +5,7 @@
 #include "nix/util/args.hh"
 #include "nix/cmd/common-eval-args.hh"
 #include "nix/store/path.hh"
+#include "nix/store/store-reference.hh"
 #include "nix/flake/lockfile.hh"
 
 #include <optional>
@@ -40,28 +41,43 @@ struct NixMultiCommand : MultiCommand, virtual Command
 // For the overloaded run methods
 #pragma GCC diagnostic ignored "-Woverloaded-virtual"
 
+/**
+ * A command that requires a \ref StoreConfig store configuration.
+ */
+struct StoreConfigCommand : virtual Command
+{
+    StoreConfigCommand();
+    void run() override;
+
+    /**
+     * Return the default Nix store configuration.
+     */
+    ref<StoreConfig> getStoreConfig();
+
+    virtual ref<StoreConfig> createStoreConfig();
+    /**
+     * Main entry point, with a `StoreConfig` provided
+     */
+    virtual void run(ref<StoreConfig>) = 0;
+
+private:
+    std::shared_ptr<StoreConfig> _storeConfig;
+};
+
 /**
  * A command that requires a \ref Store "Nix store".
  */
-struct StoreCommand : virtual Command
+struct StoreCommand : virtual StoreConfigCommand
 {
     StoreCommand();
-    void run() override;
+    void run(ref<StoreConfig>) override;
 
     /**
      * Return the default Nix store.
      */
     ref<Store> getStore();
 
-    /**
-     * Return the destination Nix store.
-     */
-    virtual ref<Store> getDstStore()
-    {
-        return getStore();
-    }
-
-    virtual ref<Store> createStore();
+    ref<Store> createStore();
     /**
      * Main entry point, with a `Store` provided
      */
@@ -77,13 +93,13 @@ private:
  */
 struct CopyCommand : virtual StoreCommand
 {
-    std::string srcUri, dstUri;
+    std::optional<StoreReference> srcUri, dstUri;
 
     CopyCommand();
 
-    ref<Store> createStore() override;
+    ref<StoreConfig> createStoreConfig() override;
 
-    ref<Store> getDstStore() override;
+    ref<Store> getDstStore();
 };
 
 /**
@@ -315,11 +331,11 @@ struct MixProfile : virtual StoreCommand
     MixProfile();
 
     /* If 'profile' is set, make it point at 'storePath'. */
-    void updateProfile(const StorePath & storePath);
+    void updateProfile(Store & store, const StorePath & storePath);
 
     /* If 'profile' is set, make it point at the store path produced
        by 'buildables'. */
-    void updateProfile(const BuiltPaths & buildables);
+    void updateProfile(Store & store, const BuiltPaths & buildables);
 };
 
 struct MixDefaultProfile : MixProfile

src/libcmd/include/nix/cmd/common-eval-args.hh

@@ -6,6 +6,7 @@
 #include "nix/main/common-args.hh"
 #include "nix/expr/search-path.hh"
 #include "nix/expr/eval-settings.hh"
+#include "nix/store/store-reference.hh"
 
 #include <filesystem>
 
@@ -55,7 +56,7 @@ struct MixEvalArgs : virtual Args, virtual MixRepair
 
     LookupPath lookupPath;
 
-    std::optional<std::string> evalStoreUrl;
+    std::optional<StoreReference> evalStoreUrl;
 
 private:
     struct AutoArgExpr

src/libcmd/include/nix/cmd/get-build-log.hh

@@ -0,0 +1,23 @@
+#pragma once
+///@file
+
+#include "nix/store/store-api.hh"
+
+#include <string>
+#include <string_view>
+
+namespace nix {
+
+/**
+ * Fetch the build log for a store path, searching the store and its
+ * substituters.
+ *
+ * @param store The store to search (and its substituters).
+ * @param path The store path to get the build log for.
+ * @param what A description of what we're fetching the log for (used in messages).
+ * @return The build log content.
+ * @throws Error if the build log is not available.
+ */
+std::string fetchBuildLog(ref<Store> store, const StorePath & path, std::string_view what);
+
+} // namespace nix

src/libcmd/include/nix/cmd/meson.build

@@ -9,6 +9,7 @@ headers = files(
   'common-eval-args.hh',
   'compatibility-settings.hh',
   'editor-for.hh',
+  'get-build-log.hh',
   'installable-attr-path.hh',
   'installable-derived-path.hh',
   'installable-flake.hh',
@@ -20,4 +21,5 @@ headers = files(
   'network-proxy.hh',
   'repl-interacter.hh',
   'repl.hh',
+  'unix-socket-server.hh',
 )

src/libcmd/include/nix/cmd/repl-interacter.hh

@@ -14,6 +14,7 @@ namespace detail {
 struct ReplCompleterMixin
 {
     virtual StringSet completePrefix(const std::string & prefix) = 0;
+    virtual ~ReplCompleterMixin() = default;
 };
 }; // namespace detail
 

src/libcmd/include/nix/cmd/repl.hh

@@ -25,7 +25,7 @@ struct AbstractNixRepl
      * @todo this is a layer violation
      *
      * @param programName Name of the command, e.g. `nix` or `nix-env`.
-     * @param args aguments to the command.
+     * @param args arguments to the command.
      */
     using RunNix =
         void(const std::string & programName, const Strings & args, const std::optional<std::string> & input);
@@ -37,7 +37,6 @@ struct AbstractNixRepl
      */
     static std::unique_ptr<AbstractNixRepl> create(
         const LookupPath & lookupPath,
-        nix::ref<Store> store,
         ref<EvalState> state,
         std::function<AnnotatedValues()> getValues,
         RunNix * runNix = nullptr);

src/libcmd/include/nix/cmd/unix-socket-server.hh

@@ -0,0 +1,79 @@
+#pragma once
+///@file
+
+#include "nix/util/file-descriptor.hh"
+
+#include <filesystem>
+#include <functional>
+#include <optional>
+#include <sys/types.h>
+
+namespace nix::unix {
+
+/**
+ * Information about the identity of the peer on a Unix domain socket connection.
+ */
+struct PeerInfo
+{
+    std::optional<pid_t> pid;
+    std::optional<uid_t> uid;
+    std::optional<gid_t> gid;
+};
+
+/**
+ * Get the identity of the caller, if possible.
+ */
+PeerInfo getPeerInfo(Descriptor remote);
+
+/**
+ * Callback type for handling new connections.
+ *
+ * The callback receives ownership of the connection and is responsible
+ * for handling it (e.g., forking a child process, spawning a thread, etc.).
+ *
+ * @param socket The accepted connection file descriptor.
+ * @param closeListeners A callback to close the listening sockets.
+ *   Useful in forked child processes to release the bound sockets.
+ */
+using UnixSocketHandler = std::function<void(AutoCloseFD socket, std::function<void()> closeListeners)>;
+
+/**
+ * Options for the serve loop.
+ *
+ * Only used if no systemd socket activation is detected.
+ */
+struct ServeUnixSocketOptions
+{
+    /**
+     * The Unix domain socket path to create and listen on.
+     */
+    std::filesystem::path socketPath;
+
+    /**
+     * Mode for the created socket file.
+     */
+    mode_t socketMode = 0666;
+};
+
+/**
+ * Run a server loop that accepts connections and calls the handler for each.
+ *
+ * This function handles:
+ * - systemd socket activation (via LISTEN_FDS environment variable)
+ * - Creating and binding a Unix domain socket if no activation is detected
+ * - Polling for incoming connections
+ * - Accepting connections
+ *
+ * For each accepted connection, the handler is called with the connection
+ * file descriptor. The handler takes ownership of the file descriptor and
+ * is responsible for closing it when done.
+ *
+ * This function never returns normally. It runs until interrupted
+ * (e.g., via SIGINT), at which point it throws `Interrupted`.
+ *
+ * @param options Configuration for the server.
+ * @param handler Callback invoked for each accepted connection.
+ */
+[[noreturn]] void serveUnixSocket(const ServeUnixSocketOptions & options, UnixSocketHandler handler);
+
+} // namespace nix::unix

src/libcmd/installable-flake.cc

@@ -203,8 +203,10 @@ FlakeRef InstallableFlake::nixpkgsFlakeRef() const
 
     if (auto nixpkgsInput = lockedFlake->lockFile.findInput({"nixpkgs"})) {
         if (auto lockedNode = std::dynamic_pointer_cast<const flake::LockedNode>(nixpkgsInput)) {
-            debug("using nixpkgs flake '%s'", lockedNode->lockedRef);
-            return std::move(lockedNode->lockedRef);
+            if (lockedNode->isFlake) {
+                debug("using nixpkgs flake '%s'", lockedNode->lockedRef);
+                return std::move(lockedNode->lockedRef);
+            }
         }
     }
 

src/libcmd/installables.cc

@@ -583,16 +583,12 @@ static void throwBuildErrors(std::vector<KeyedBuildResult> & buildResults, const
     auto failedResult = failed.begin();
     if (failedResult != failed.end()) {
         if (failed.size() == 1) {
-            failedResult->second->rethrow();
+            throw *failedResult->second;
         } else {
             StringSet failedPaths;
             for (; failedResult != failed.end(); failedResult++) {
-                if (!failedResult->second->errorMsg.empty()) {
-                    logError(
-                        ErrorInfo{
-                            .level = lvlError,
-                            .msg = failedResult->second->errorMsg,
-                        });
+                if (!failedResult->second->message().empty()) {
+                    logError(failedResult->second->info());
                 }
                 failedPaths.insert(failedResult->first->path.to_string(store));
             }

src/libcmd/meson.build

@@ -74,6 +74,7 @@ sources = files(
   'command.cc',
   'common-eval-args.cc',
   'editor-for.cc',
+  'get-build-log.cc',
   'installable-attr-path.cc',
   'installable-derived-path.cc',
   'installable-flake.cc',
@@ -86,6 +87,12 @@ sources = files(
   'repl.cc',
 )
 
+if host_machine.system() != 'windows'
+  sources += files(
+    'unix/unix-socket-server.cc',
+  )
+endif
+
 subdir('include/nix/cmd')
 
 subdir('nix-meson-build-support/export-all-symbols')

src/libcmd/repl-interacter.cc

@@ -40,8 +40,8 @@ void sigintHandler(int signo)
 static detail::ReplCompleterMixin * curRepl; // ugly
 
 #if !USE_READLINE
-static char * completionCallback(char * s, int * match)
-{
+static char * completionCallback(char * s, int * match) noexcept
+try {
     auto possible = curRepl->completePrefix(s);
     if (possible.size() == 1) {
         *match = 1;
@@ -73,10 +73,12 @@ static char * completionCallback(char * s, int * match)
 
     *match = 0;
     return nullptr;
+} catch (...) {
+    return nullptr;
 }
 
-static int listPossibleCallback(char * s, char *** avp)
-{
+static int listPossibleCallback(char * s, char *** avp) noexcept
+try {
     auto possible = curRepl->completePrefix(s);
 
     if (possible.size() > (std::numeric_limits<int>::max() / sizeof(char *)))
@@ -105,6 +107,9 @@ static int listPossibleCallback(char * s, char *** avp)
     *avp = vp;
 
     return ac;
+} catch (...) {
+    *avp = nullptr;
+    return 0;
 }
 #endif
 

src/libcmd/repl.cc

@@ -12,9 +12,8 @@
 #include "nix/expr/eval-settings.hh"
 #include "nix/expr/attr-path.hh"
 #include "nix/util/signals.hh"
-#include "nix/store/store-open.hh"
-#include "nix/store/log-store.hh"
 #include "nix/cmd/common-eval-args.hh"
+#include "nix/cmd/get-build-log.hh"
 #include "nix/expr/get-drvs.hh"
 #include "nix/store/derivations.hh"
 #include "nix/store/globals.hh"
@@ -65,7 +64,7 @@ struct NixRepl : AbstractNixRepl, detail::ReplCompleterMixin, gc
 
     const static int envSize = 32768;
     std::shared_ptr<StaticEnv> staticEnv;
-    Value lastLoaded;
+    std::optional<Value> lastLoaded;
     Env * env;
     int displ;
     StringSet varNames;
@@ -78,7 +77,6 @@ struct NixRepl : AbstractNixRepl, detail::ReplCompleterMixin, gc
 
     NixRepl(
         const LookupPath & lookupPath,
-        nix::ref<Store> store,
         ref<EvalState> state,
         std::function<AnnotatedValues()> getValues,
         RunNix * runNix);
@@ -133,7 +131,6 @@ std::string removeWhitespace(std::string s)
 
 NixRepl::NixRepl(
     const LookupPath & lookupPath,
-    nix::ref<Store> store,
     ref<EvalState> state,
     std::function<NixRepl::AnnotatedValues()> getValues,
     RunNix * runNix)
@@ -566,31 +563,9 @@ ProcessLineResult NixRepl::processLine(std::string line)
         } else if (command == ":log") {
             settings.readOnlyMode = true;
             Finally roModeReset([&]() { settings.readOnlyMode = false; });
-            auto subs = getDefaultSubstituters();
-
-            subs.push_front(state->store);
-
-            bool foundLog = false;
             RunPager pager;
-            for (auto & sub : subs) {
-                auto * logSubP = dynamic_cast<LogStore *>(&*sub);
-                if (!logSubP) {
-                    printInfo(
-                        "Skipped '%s' which does not support retrieving build logs", sub->config.getHumanReadableURI());
-                    continue;
-                }
-                auto & logSub = *logSubP;
-
-                auto log = logSub.getBuildLog(drvPath);
-                if (log) {
-                    printInfo("got build log for '%s' from '%s'", drvPathRaw, logSub.config.getHumanReadableURI());
-                    logger->writeToStdout(*log);
-                    foundLog = true;
-                    break;
-                }
-            }
-            if (!foundLog)
-                throw Error("build log of '%s' is not available", drvPathRaw);
+            auto log = fetchBuildLog(state->store, drvPath, drvPathRaw);
+            logger->writeToStdout(log);
         } else {
             runNix("nix-shell", {drvPathRaw});
         }
@@ -735,7 +710,7 @@ void NixRepl::loadFlake(const std::string & flakeRefS)
     try {
         cwd = std::filesystem::current_path();
     } catch (std::filesystem::filesystem_error & e) {
-        throw SysError("cannot determine current working directory");
+        throw SystemError(e.code(), "cannot determine current working directory");
     }
 
     auto flakeRef = parseFlakeRef(fetchSettings, flakeRefS, cwd.string(), true);
@@ -773,11 +748,19 @@ void NixRepl::initEnv()
 
 void NixRepl::showLastLoaded()
 {
-    RunPager pager;
+    if (!lastLoaded)
+        throw Error("nothing has been loaded yet");
 
-    for (auto & i : *lastLoaded.attrs()) {
-        std::string_view name = state->symbols[i.name];
-        logger->cout(name);
+    RunPager pager;
+    try {
+        for (auto & i : *lastLoaded->attrs()) {
+            std::string_view name = state->symbols[i.name];
+            logger->cout(name);
+        }
+    } catch (SystemError & e) {
+        /* Ignore broken pipes when the pager gets interrupted. */
+        if (!e.is(std::errc::broken_pipe))
+            throw;
     }
 }
 
@@ -795,7 +778,7 @@ void NixRepl::loadFiles()
     loadedFiles.clear();
 
     for (auto & i : old) {
-        notice("Loading '%1%'...", i);
+        notice("Loading %1%...", PathFmt(i));
         loadFile(i);
     }
 
@@ -900,13 +883,9 @@ void NixRepl::runNix(const std::string & program, const Strings & args, const st
 }
 
 std::unique_ptr<AbstractNixRepl> AbstractNixRepl::create(
-    const LookupPath & lookupPath,
-    nix::ref<Store> store,
-    ref<EvalState> state,
-    std::function<AnnotatedValues()> getValues,
-    RunNix * runNix)
+    const LookupPath & lookupPath, ref<EvalState> state, std::function<AnnotatedValues()> getValues, RunNix * runNix)
 {
-    return std::make_unique<NixRepl>(lookupPath, std::move(store), state, getValues, runNix);
+    return std::make_unique<NixRepl>(lookupPath, state, getValues, runNix);
 }
 
 ReplExitStatus AbstractNixRepl::runSimple(ref<EvalState> evalState, const ValMap & extraEnv)
@@ -919,7 +898,6 @@ ReplExitStatus AbstractNixRepl::runSimple(ref<EvalState> evalState, const ValMap
     // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDelete)
     auto repl = std::make_unique<NixRepl>(
         lookupPath,
-        openStore(),
         evalState,
         getValues,
         /*runNix=*/nullptr);

src/libcmd/unix/unix-socket-server.cc

@@ -0,0 +1,126 @@
+///@file
+
+#include "nix/cmd/unix-socket-server.hh"
+#include "nix/util/environment-variables.hh"
+#include "nix/util/file-system.hh"
+#include "nix/util/logging.hh"
+#include "nix/util/signals.hh"
+#include "nix/util/strings.hh"
+#include "nix/util/unix-domain-socket.hh"
+#include "nix/util/util.hh"
+
+#include <sys/socket.h>
+#include <sys/un.h>
+#include <poll.h>
+
+#if defined(__APPLE__) || defined(__FreeBSD__)
+#  include <sys/ucred.h>
+#endif
+
+namespace nix::unix {
+
+PeerInfo getPeerInfo(Descriptor remote)
+{
+    PeerInfo peer;
+
+#if defined(SO_PEERCRED)
+
+#  if defined(__OpenBSD__)
+    struct sockpeercred cred;
+#  else
+    ucred cred;
+#  endif
+    socklen_t credLen = sizeof(cred);
+    if (getsockopt(remote, SOL_SOCKET, SO_PEERCRED, &cred, &credLen) == 0) {
+        peer.pid = cred.pid;
+        peer.uid = cred.uid;
+        peer.gid = cred.gid;
+    }
+
+#elif defined(LOCAL_PEERCRED)
+
+#  if !defined(SOL_LOCAL)
+#    define SOL_LOCAL 0
+#  endif
+
+    xucred cred;
+    socklen_t credLen = sizeof(cred);
+    if (getsockopt(remote, SOL_LOCAL, LOCAL_PEERCRED, &cred, &credLen) == 0)
+        peer.uid = cred.cr_uid;
+
+#endif
+
+    return peer;
+}
+
+[[noreturn]] void serveUnixSocket(const ServeUnixSocketOptions & options, UnixSocketHandler handler)
+{
+    std::vector<AutoCloseFD> listeningSockets;
+
+    static constexpr int SD_LISTEN_FDS_START = 3;
+
+    //  Handle socket-based activation by systemd.
+    auto listenFds = getEnv("LISTEN_FDS");
+    if (listenFds) {
+        if (getEnv("LISTEN_PID") != std::to_string(getpid()))
+            throw Error("unexpected systemd environment variables");
+        auto count = string2Int<unsigned int>(*listenFds);
+        assert(count);
+        for (unsigned int i = 0; i < count; ++i) {
+            AutoCloseFD fdSocket(SD_LISTEN_FDS_START + i);
+            closeOnExec(fdSocket.get());
+            listeningSockets.push_back(std::move(fdSocket));
+        }
+    }
+
+    //  Otherwise, create and bind to a Unix domain socket.
+    else {
+        createDirs(options.socketPath.parent_path());
+        listeningSockets.push_back(createUnixDomainSocket(options.socketPath.string(), options.socketMode));
+    }
+
+    std::vector<struct pollfd> fds;
+    for (auto & i : listeningSockets)
+        fds.push_back({.fd = i.get(), .events = POLLIN});
+
+    //  Loop accepting connections.
+    while (1) {
+        try {
+            checkInterrupt();
+
+            auto count = poll(fds.data(), fds.size(), -1);
+            if (count == -1) {
+                if (errno == EINTR)
+                    continue;
+                throw SysError("polling for incoming connections");
+            }
+
+            for (auto & fd : fds) {
+                if (!fd.revents)
+                    continue;
+
+                // Accept a connection.
+                struct sockaddr_un remoteAddr;
+                socklen_t remoteAddrLen = sizeof(remoteAddr);
+
+                AutoCloseFD remote = accept(fd.fd, (struct sockaddr *) &remoteAddr, &remoteAddrLen);
+                checkInterrupt();
+                if (!remote) {
+                    if (errno == EINTR)
+                        continue;
+                    throw SysError("accepting connection");
+                }
+
+                handler(std::move(remote), [&]() { listeningSockets.clear(); });
+            }
+
+        } catch (Error & error) {
+            auto ei = error.info();
+            // FIXME: add to trace?
+            ei.msg = HintFmt("while processing connection: %1%", ei.msg.str());
+            logError(ei);
+        }
+    }
+}
+
+} // namespace nix::unix

src/libexpr-tests/primops.cc

@@ -756,7 +756,7 @@ TEST_F(PrimOpTest, langVersion)
 TEST_F(PrimOpTest, storeDir)
 {
     auto v = eval("builtins.storeDir");
-    ASSERT_THAT(v, IsStringEq(settings.nixStore));
+    ASSERT_THAT(v, IsStringEq(state.store->storeDir));
 }
 
 TEST_F(PrimOpTest, nixVersion)

src/libexpr-tests/value/value.cc

@@ -13,8 +13,7 @@ TEST_F(ValueTest, unsetValue)
 {
     Value unsetValue;
     ASSERT_EQ(false, unsetValue.isValid());
-    ASSERT_EQ(nThunk, unsetValue.type(true));
-    ASSERT_DEATH(unsetValue.type(), "");
+    ASSERT_EQ(nThunk, unsetValue.type</*invalidIsThunk=*/true>());
 }
 
 TEST_F(ValueTest, vInt)

src/libexpr/eval-cache.cc

@@ -75,7 +75,7 @@ struct AttrDb
 
         auto dbPath = cacheDir / (fingerprint.to_string(HashFormat::Base16, false) + ".sqlite");
 
-        state->db = SQLite(dbPath);
+        state->db = SQLite(dbPath, {.useWAL = settings.useSQLiteWAL});
         state->db.isCache();
         state->db.exec(schema);
 

src/libexpr/eval-profiler.cc

@@ -133,13 +133,19 @@ class SampleStack : public EvalProfiler
     FrameInfo getPrimOpFrameInfo(const PrimOp & primOp, std::span<Value *> args, PosIdx pos);
 
 public:
-    SampleStack(EvalState & state, std::filesystem::path profileFile, std::chrono::nanoseconds period)
+    SampleStack(EvalState & state, const std::filesystem::path & profileFile, std::chrono::nanoseconds period)
         : state(state)
         , sampleInterval(period)
         , profileFd([&]() {
-            AutoCloseFD fd = toDescriptor(open(profileFile.string().c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0660));
+            AutoCloseFD fd = openNewFileForWrite(
+                profileFile,
+                0660,
+                {
+                    .truncateExisting = true,
+                    .followSymlinksOnTruncate = true, /* FIXME: Probably shouldn't follow symlinks. */
+                });
             if (!fd)
-                throw SysError("opening file %s", profileFile);
+                throw SysError("opening file %s", PathFmt(profileFile));
             return fd;
         }())
         , posCache(state)

src/libexpr/eval-settings.cc

@@ -71,8 +71,9 @@ Strings EvalSettings::getDefaultNixPath()
     };
 
     add(std::filesystem::path{getNixDefExpr()} / "channels");
-    add(rootChannelsDir() / "nixpkgs", "nixpkgs");
-    add(rootChannelsDir());
+    auto profilesDirOpts = settings.getProfileDirsOptions();
+    add(rootChannelsDir(profilesDirOpts) / "nixpkgs", "nixpkgs");
+    add(rootChannelsDir(profilesDirOpts));
 
     return res;
 }

src/libexpr/eval.cc

@@ -318,7 +318,6 @@ EvalState::EvalState(
     countCalls = getEnv("NIX_COUNT_CALLS").value_or("0") != "0";
 
     static_assert(sizeof(Env) <= 16, "environment must be <= 16 bytes");
-    static_assert(sizeof(Counter) == 64, "counters must be 64 bytes");
 
     /* Construct the Nix expression search path. */
     assert(lookupPath.elements.empty());
@@ -467,7 +466,7 @@ void EvalState::addConstant(const std::string & name, Value * v, Constant info)
 
            We might know the type of a thunk in advance, so be allowed
            to just write it down in that case. */
-        if (auto gotType = v->type(true); gotType != nThunk)
+        if (auto gotType = v->type</*invalidIsThunk=*/true>(); gotType != nThunk)
             assert(info.type == gotType);
 
         /* Install value the base environment. */
@@ -886,7 +885,7 @@ void Value::mkPath(const SourcePath & path, EvalMemory & mem)
     mkPath(&*path.accessor, StringData::make(mem, path.path.abs()));
 }
 
-inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval)
+[[gnu::always_inline]] inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval)
 {
     for (auto l = var.level; l; --l, env = env->up)
         ;
@@ -904,11 +903,11 @@ inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval)
     while (1) {
         forceAttrs(*env->values[0], fromWith->pos, "while evaluating the first subexpression of a with expression");
         if (auto j = env->values[0]->attrs()->get(var.name)) {
-            if (countCalls)
+            if (countCalls) [[unlikely]]
                 attrSelects[j->pos]++;
             return j->value;
         }
-        if (!fromWith->parentWith)
+        if (!fromWith->parentWith) [[unlikely]]
             error<UndefinedVarError>("undefined variable '%1%'", symbols[var.name])
                 .atPos(var.pos)
                 .withFrame(*env, var)

src/libexpr/include/nix/expr/counter.hh

@@ -11,7 +11,7 @@ namespace nix {
  * variable is set. This is to prevent contention on these counters
  * when multi-threaded evaluation is enabled.
  */
-struct alignas(64) Counter
+struct alignas(std::hardware_destructive_interference_size) Counter
 {
     using value_type = uint64_t;
 

src/libexpr/include/nix/expr/eval.hh

@@ -1002,6 +1002,12 @@ public:
     [[nodiscard]] StringMap
     realiseContext(const NixStringContext & context, StorePathSet * maybePaths = nullptr, bool isIFD = true);
 
+    /**
+     * Coerce `v` to a path and realise it, i.e. build anything in the value's string context using `realiseContext()`.
+     */
+    SourcePath realisePath(
+        const PosIdx pos, Value & v, std::optional<SymlinkResolution> resolveSymlinks = SymlinkResolution::Full);
+
     /**
      * Realise the given string with context, and return the string with outputs instead of downstream output
      * placeholders.

src/libexpr/include/nix/expr/value.hh

@@ -35,7 +35,7 @@ class BindingsBuilder;
  * about how this is mapped into the alignment bits to save significant memory.
  * This also restricts the number of internal types represented with distinct memory layouts.
  */
-typedef enum {
+enum InternalType {
     tUninitialized = 0,
     /* layout: Single/zero field payload */
     tInt = 1,
@@ -46,16 +46,20 @@ typedef enum {
     tPrimOp,
     tAttrs,
     /* layout: Pair of pointers payload */
-    tListSmall,
+    tFirstPairOfPointers,
+    tListSmall = tFirstPairOfPointers,
     tPrimOpApp,
     tApp,
     tThunk,
     tLambda,
+    tLastPairOfPointers = tLambda,
     /* layout: Single untaggable field */
-    tListN,
+    tFirstSingleUntaggable,
+    tListN = tFirstSingleUntaggable,
     tString,
     tPath,
-} InternalType;
+    tNumberOfInternalTypes, // Must be last
+};
 
 /**
  * This type abstracts over all actual value types in the language,
@@ -134,7 +138,7 @@ public:
     virtual bool operator==(const ExternalValueBase & b) const noexcept;
 
     /**
-     * Print the value as JSON. Defaults to unconvertable, i.e. throws an error
+     * Print the value as JSON. Defaults to unconvertible, i.e. throws an error
      */
     virtual nlohmann::json
     printValueAsJSON(EvalState & state, bool strict, NixStringContext & context, bool copyToStore = true) const;
@@ -533,8 +537,8 @@ inline constexpr bool useBitPackedValueStorage = (ptrSize == 8) && (__STDCPP_DEF
  * Packs discriminator bits into the pointer alignment niches.
  */
 template<std::size_t ptrSize>
-class alignas(16) ValueStorage<ptrSize, std::enable_if_t<detail::useBitPackedValueStorage<ptrSize>>>
-    : public detail::ValueBase
+class alignas(16)
+    ValueStorage<ptrSize, std::enable_if_t<detail::useBitPackedValueStorage<ptrSize>>> : public detail::ValueBase
 {
     /* Needs a dependent type name in order for member functions (and
      * potentially ill-formed bit casts) to be SFINAE'd out.
@@ -587,7 +591,7 @@ class alignas(16) ValueStorage<ptrSize, std::enable_if_t<detail::useBitPackedVal
     enum PrimaryDiscriminator : int {
         pdUninitialized = 0,
         pdSingleDWord, //< layout: Single/zero field payload
-        /* The order of these enumations must be the same as in InternalType. */
+        /* The order of these enumerations must be the same as in InternalType. */
         pdListN, //< layout: Single untaggable field.
         pdString,
         pdPath,
@@ -633,7 +637,7 @@ class alignas(16) ValueStorage<ptrSize, std::enable_if_t<detail::useBitPackedVal
     template<InternalType type, typename T, typename U>
     void setPairOfPointersPayload(T * firstPtrField, U * secondPtrField) noexcept
     {
-        static_assert(type >= tListSmall && type <= tLambda);
+        static_assert(type >= tFirstPairOfPointers && type <= tLastPairOfPointers);
         {
             auto firstFieldPayload = std::bit_cast<PackedPointer>(firstPtrField);
             assertAligned(firstFieldPayload);
@@ -642,7 +646,7 @@ class alignas(16) ValueStorage<ptrSize, std::enable_if_t<detail::useBitPackedVal
         {
             auto secondFieldPayload = std::bit_cast<PackedPointer>(secondPtrField);
             assertAligned(secondFieldPayload);
-            payload[1] = (type - tListSmall) | secondFieldPayload;
+            payload[1] = (type - tFirstPairOfPointers) | secondFieldPayload;
         }
     }
 
@@ -670,11 +674,11 @@ protected:
         case pdListN:
         case pdString:
         case pdPath:
-            return static_cast<InternalType>(tListN + (pd - pdListN));
+            return static_cast<InternalType>(tFirstSingleUntaggable + (pd - pdListN));
         case pdPairOfPointers:
-            return static_cast<InternalType>(tListSmall + (payload[1] & discriminatorMask));
+            return static_cast<InternalType>(tFirstPairOfPointers + (payload[1] & discriminatorMask));
         [[unlikely]] default:
-            unreachable();
+            nixUnreachableWhenHardened();
         }
     }
 
@@ -1027,7 +1031,7 @@ private:
     T getStorage() const noexcept
     {
         if (getInternalType() != detail::payloadTypeToInternalType<T>) [[unlikely]]
-            unreachable();
+            nixUnreachableWhenHardened();
         T out;
         ValueStorage::getStorage(out);
         return out;
@@ -1079,45 +1083,44 @@ public:
      * Returns the normal type of a Value. This only returns nThunk if
      * the Value hasn't been forceValue'd
      *
-     * @param invalidIsThunk Instead of aborting an an invalid (probably
+     * @param invalidIsThunk Instead of UB an an invalid (probably
      * 0, so uninitialized) internal type, return `nThunk`.
      */
-    inline ValueType type(bool invalidIsThunk = false) const
+    template<bool invalidIsThunk = false>
+    inline ValueType type() const
     {
-        switch (getInternalType()) {
-        case tUninitialized:
-            break;
-        case tInt:
-            return nInt;
-        case tBool:
-            return nBool;
-        case tString:
-            return nString;
-        case tPath:
-            return nPath;
-        case tNull:
-            return nNull;
-        case tAttrs:
-            return nAttrs;
-        case tListSmall:
-        case tListN:
-            return nList;
-        case tLambda:
-        case tPrimOp:
-        case tPrimOpApp:
-            return nFunction;
-        case tExternal:
-            return nExternal;
-        case tFloat:
-            return nFloat;
-        case tThunk:
-        case tApp:
-            return nThunk;
+        /* Explicit lookup table. switch() might compile down (and it does at least with GCC 14)
+           to a jump table. Let's help the compiler a bit here. */
+        static constexpr auto table = [] {
+            std::array<ValueType, tNumberOfInternalTypes> t{};
+            t[tUninitialized] = nThunk;
+            t[tInt] = nInt;
+            t[tBool] = nBool;
+            t[tNull] = nNull;
+            t[tFloat] = nFloat;
+            t[tExternal] = nExternal;
+            t[tAttrs] = nAttrs;
+            t[tPrimOp] = nFunction;
+            t[tLambda] = nFunction;
+            t[tPrimOpApp] = nFunction;
+            t[tApp] = nThunk;
+            t[tThunk] = nThunk;
+            t[tListSmall] = nList;
+            t[tListN] = nList;
+            t[tString] = nString;
+            t[tPath] = nPath;
+            return t;
+        }();
+
+        auto it = getInternalType();
+        if (it == tUninitialized || it >= tNumberOfInternalTypes) [[unlikely]] {
+            if constexpr (invalidIsThunk)
+                return nThunk;
+            else
+                nixUnreachableWhenHardened();
         }
-        if (invalidIsThunk)
-            return nThunk;
-        else
-            unreachable();
+
+        return table[it];
     }
 
     /**

src/libexpr/meson.build

@@ -28,7 +28,7 @@ deps_public_maybe_subproject = [
 subdir('nix-meson-build-support/subprojects')
 subdir('nix-meson-build-support/big-objs')
 
-# Check for each of these functions, and create a define like `#define HAVE_LCHOWN 1`.
+# Check for each of these functions, and create a define like `#define HAVE_SYSCONF 1`.
 check_funcs = [
   'sysconf',
 ]

src/libexpr/parser.y

@@ -3,7 +3,7 @@
 %define api.namespace { ::nix::parser }
 %define api.parser.class { BisonParser }
 %locations
-%define parse.error verbose
+%define parse.error detailed
 %defines
 /* %no-lines */
 %parse-param { void * scanner }
@@ -140,18 +140,41 @@ static Expr * makeCall(Exprs & exprs, PosIdx pos, Expr * fn, Expr * arg) {
 %type <Expr *> path_start
 %type <ToBeStringyExpr> string_parts string_attr
 %type <StringToken> attr
-%token <StringToken> ID
-%token <StringToken> STR IND_STR
-%token <NixInt> INT_LIT
-%token <NixFloat> FLOAT_LIT
-%token <StringToken> PATH HPATH SPATH PATH_END
-%token <StringToken> URI
-%token IF THEN ELSE ASSERT WITH LET IN_KW REC INHERIT EQ NEQ AND OR IMPL OR_KW
-%token PIPE_FROM PIPE_INTO /* <| and |> */
-%token DOLLAR_CURLY /* == ${ */
-%token IND_STRING_OPEN IND_STRING_CLOSE
-%token ELLIPSIS
-
+%token <StringToken> ID "identifier"
+%token <StringToken> STR "string"
+%token <StringToken> IND_STR "indented string"
+%token <NixInt> INT_LIT "integer"
+%token <NixFloat> FLOAT_LIT "floating-point literal"
+%token <StringToken> PATH "path"
+%token <StringToken> HPATH "'~/…' path"
+%token <StringToken> SPATH "'<…>' path"
+%token <StringToken> PATH_END "end of path"
+%token <StringToken> URI "URI"
+%token IF "'if'"
+%token THEN "'then'"
+%token ELSE "'else'"
+%token ASSERT "'assert'"
+%token WITH "'with'"
+%token LET "'let'"
+%token IN_KW "'in'"
+%token REC "'rec'"
+%token INHERIT "'inherit'"
+%token EQ "'=='"
+%token NEQ "'!='"
+%token LEQ "'<='"
+%token GEQ "'>='"
+%token UPDATE "'//'"
+%token CONCAT "'++'"
+%token AND "'&&'"
+%token OR "'||'"
+%token IMPL "'->'"
+%token OR_KW "'or'"
+%token PIPE_FROM "'<|'"
+%token PIPE_INTO "'|>'"
+%token DOLLAR_CURLY "'${'"
+%token IND_STRING_OPEN "start of an indented string"
+%token IND_STRING_CLOSE "end of an indented string"
+%token ELLIPSIS "'...'"
 
 %right IMPL
 %left OR

src/libexpr/paths.cc

@@ -23,13 +23,12 @@ SourcePath EvalState::storePath(const StorePath & path)
 StorePath
 EvalState::mountInput(fetchers::Input & input, const fetchers::Input & originalInput, ref<SourceAccessor> accessor)
 {
-    auto storePath = fetchToStore(fetchSettings, *store, accessor, FetchMode::Copy, input.getName());
+    auto [storePath, narHash] = fetchToStore2(fetchSettings, *store, accessor, FetchMode::Copy, input.getName());
 
     allowPath(storePath); // FIXME: should just whitelist the entire virtual store
 
     storeFS->mount(CanonPath(store->printStorePath(storePath)), accessor);
 
-    auto narHash = store->queryPathInfo(storePath)->narHash;
     input.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true));
 
     if (originalInput.getNarHash() && narHash != *originalInput.getNarHash())

src/libexpr/primops.cc

@@ -157,24 +157,20 @@ StringMap EvalState::realiseContext(const NixStringContext & context, StorePathS
     return res;
 }
 
-static SourcePath realisePath(
-    EvalState & state,
-    const PosIdx pos,
-    Value & v,
-    std::optional<SymlinkResolution> resolveSymlinks = SymlinkResolution::Full)
+SourcePath EvalState::realisePath(const PosIdx pos, Value & v, std::optional<SymlinkResolution> resolveSymlinks)
 {
     NixStringContext context;
 
-    auto path = state.coerceToPath(noPos, v, context, "while realising the context of a path");
+    auto path = coerceToPath(noPos, v, context, "while realising the context of a path");
 
     try {
-        if (!context.empty() && path.accessor == state.rootFS) {
-            auto rewrites = state.realiseContext(context);
+        if (!context.empty() && path.accessor == rootFS) {
+            auto rewrites = realiseContext(context);
             path = {path.accessor, CanonPath(rewriteStrings(path.path.abs(), rewrites))};
         }
         return resolveSymlinks ? path.resolveSymlinks(*resolveSymlinks) : path;
     } catch (Error & e) {
-        e.addTrace(state.positions[pos], "while realising the context of path '%s'", path);
+        e.addTrace(positions[pos], "while realising the context of path '%s'", path);
         throw;
     }
 }
@@ -294,7 +290,7 @@ static void scopedImport(EvalState & state, const PosIdx pos, SourcePath & path,
    argument. */
 static void import(EvalState & state, const PosIdx pos, Value & vPath, Value * vScope, Value & v)
 {
-    auto path = realisePath(state, pos, vPath, std::nullopt);
+    auto path = state.realisePath(pos, vPath, std::nullopt);
     auto path2 = path.path.abs();
 
     // FIXME
@@ -447,7 +443,7 @@ extern "C" typedef void (*ValueInitializer)(EvalState & state, Value & v);
 /* Load a ValueInitializer from a DSO and return whatever it initializes */
 void prim_importNative(EvalState & state, const PosIdx pos, Value ** args, Value & v)
 {
-    auto path = realisePath(state, pos, *args[0]);
+    auto path = state.realisePath(pos, *args[0]);
 
     std::string sym(
         state.forceStringNoCtx(*args[1], pos, "while evaluating the second argument passed to builtins.importNative"));
@@ -1818,8 +1814,13 @@ static void derivationStrictInternal(EvalState & state, std::string_view drvName
         drv.fillInOutputPaths(*state.store);
     }
 
-    /* Write the resulting term into the Nix store directory. */
-    auto drvPath = writeDerivation(*state.store, drv, state.repair);
+    /* Write the resulting term into the Nix store directory.
+
+       Unless we are in read-only mode, that is, in which case we do not
+       write anything. Users commonly do this to speed up evaluation in
+       contexts where they don't actually want to build anything. */
+    auto drvPath =
+        settings.readOnlyMode ? computeStorePath(*state.store, drv) : state.store->writeDerivation(drv, state.repair);
     auto drvPathS = state.store->printStorePath(drvPath);
 
     printMsg(lvlChatty, "instantiated '%1%' -> '%2%'", drvName, drvPathS);
@@ -1971,7 +1972,7 @@ static void prim_pathExists(EvalState & state, const PosIdx pos, Value ** args,
             arg.type() == nString && (arg.string_view().ends_with("/") || arg.string_view().ends_with("/."));
 
         auto symlinkResolution = mustBeDir ? SymlinkResolution::Full : SymlinkResolution::Ancestors;
-        auto path = realisePath(state, pos, arg, symlinkResolution);
+        auto path = state.realisePath(pos, arg, symlinkResolution);
 
         auto st = path.maybeLstat();
         auto exists = st && (!mustBeDir || st->type == SourceAccessor::tDirectory);
@@ -2078,7 +2079,7 @@ static RegisterPrimOp primop_dirOf({
 /* Return the contents of a file as a string. */
 static void prim_readFile(EvalState & state, const PosIdx pos, Value ** args, Value & v)
 {
-    auto path = realisePath(state, pos, *args[0]);
+    auto path = state.realisePath(pos, *args[0]);
     auto s = path.readFile();
     if (s.find((char) 0) != std::string::npos)
         state.error<EvalError>("the contents of the file '%1%' cannot be represented as a Nix string", path)
@@ -2313,7 +2314,7 @@ static void prim_hashFile(EvalState & state, const PosIdx pos, Value ** args, Va
     if (!ha)
         state.error<EvalError>("unknown hash algorithm '%1%'", algo).atPos(pos).debugThrow();
 
-    auto path = realisePath(state, pos, *args[1]);
+    auto path = state.realisePath(pos, *args[1]);
 
     v.mkString(hashString(*ha, path.readFile()).to_string(HashFormat::Base16, false), state.mem);
 }
@@ -2365,7 +2366,7 @@ static const Value & fileTypeToString(EvalState & state, SourceAccessor::Type ty
 
 static void prim_readFileType(EvalState & state, const PosIdx pos, Value ** args, Value & v)
 {
-    auto path = realisePath(state, pos, *args[0], std::nullopt);
+    auto path = state.realisePath(pos, *args[0], std::nullopt);
     /* Retrieve the directory entry type and stringize it. */
     v = fileTypeToString(state, path.lstat().type);
 }
@@ -2383,7 +2384,7 @@ static RegisterPrimOp primop_readFileType({
 /* Read a directory (without . or ..) */
 static void prim_readDir(EvalState & state, const PosIdx pos, Value ** args, Value & v)
 {
-    auto path = realisePath(state, pos, *args[0]);
+    auto path = state.realisePath(pos, *args[0]);
 
     // Retrieve directory entries for all nodes in a directory.
     // This is similar to `getFileType` but is optimized to reduce system calls

src/libexpr/primops/fetchClosure.cc

@@ -191,19 +191,25 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value ** args
             {.msg = HintFmt("attribute '%s' is missing in call to 'fetchClosure'", "fromStore"),
              .pos = state.positions[pos]});
 
-    auto parsedURL = parseURL(*fromStoreUrl, /*lenient=*/true);
+    auto storeRef = StoreReference::parse(*fromStoreUrl);
 
-    if (parsedURL.scheme != "http" && parsedURL.scheme != "https"
-        && !(getEnv("_NIX_IN_TEST").has_value() && parsedURL.scheme == "file"))
-        throw Error(
-            {.msg = HintFmt("'fetchClosure' only supports http:// and https:// stores"), .pos = state.positions[pos]});
+    if ([&] {
+            auto * specified = std::get_if<StoreReference::Specified>(&storeRef.variant);
+            return !specified
+                   || (specified->scheme != "http" && specified->scheme != "https"
+                       && !(getEnv("_NIX_IN_TEST").has_value() && specified->scheme == "file"));
+        }())
+        throw Error({
+            .msg = HintFmt("'fetchClosure' only supports http:// and https:// stores"),
+            .pos = state.positions[pos],
+        });
 
-    if (!parsedURL.query.empty())
+    if (!storeRef.params.empty())
         throw Error(
             {.msg = HintFmt("'fetchClosure' does not support URL query parameters (in '%s')", *fromStoreUrl),
              .pos = state.positions[pos]});
 
-    auto fromStore = openStore(parsedURL.to_string());
+    auto fromStore = openStore(std::move(storeRef));
 
     if (toPath)
         runFetchClosureWithRewrite(state, pos, *fromStore, *fromPath, *toPath, v);

src/libfetchers/cache.cc

@@ -34,14 +34,20 @@ struct CacheImpl : Cache
 
     Sync<State> _state;
 
-    CacheImpl()
+    /**
+     * This is a back-reference to the `Settings` that owns us.
+     */
+    const Settings & settings;
+
+    CacheImpl(const Settings & _settings)
+        : settings(_settings)
     {
         auto state(_state.lock());
 
         auto dbPath = (getCacheDir() / "fetcher-cache-v4.sqlite").string();
         createDirs(dirOf(dbPath));
 
-        state->db = SQLite(dbPath);
+        state->db = SQLite(dbPath, {.useWAL = nix::settings.useSQLiteWAL});
         state->db.isCache();
         state->db.exec(schema);
 
@@ -154,7 +160,7 @@ ref<Cache> Settings::getCache() const
 {
     auto cache(_cache.lock());
     if (!*cache)
-        *cache = std::make_shared<CacheImpl>();
+        *cache = std::make_shared<CacheImpl>(*this);
     return ref<Cache>(*cache);
 }
 

src/libfetchers/fetch-to-store.cc

@@ -5,12 +5,12 @@
 
 namespace nix {
 
-fetchers::Cache::Key makeFetchToStoreCacheKey(
-    const std::string & name, const std::string & fingerprint, ContentAddressMethod method, const std::string & path)
+fetchers::Cache::Key
+makeSourcePathToHashCacheKey(std::string_view fingerprint, ContentAddressMethod method, const CanonPath & path)
 {
     return fetchers::Cache::Key{
-        "fetchToStore",
-        {{"name", name}, {"fingerprint", fingerprint}, {"method", std::string{method.render()}}, {"path", path}}};
+        "sourcePathToHash",
+        {{"fingerprint", std::string(fingerprint)}, {"method", std::string{method.render()}}, {"path", path.abs()}}};
 }
 
 StorePath fetchToStore(
@@ -23,19 +23,45 @@ StorePath fetchToStore(
     PathFilter * filter,
     RepairFlag repair)
 {
-    // FIXME: add an optimisation for the case where the accessor is
-    // a `PosixSourceAccessor` pointing to a store path.
+    return fetchToStore2(settings, store, path, mode, name, method, filter, repair).first;
+}
 
+std::pair<StorePath, Hash> fetchToStore2(
+    const fetchers::Settings & settings,
+    Store & store,
+    const SourcePath & path,
+    FetchMode mode,
+    std::string_view name,
+    ContentAddressMethod method,
+    PathFilter * filter,
+    RepairFlag repair)
+{
     std::optional<fetchers::Cache::Key> cacheKey;
 
     auto [subpath, fingerprint] = filter ? std::pair<CanonPath, std::optional<std::string>>{path.path, std::nullopt}
                                          : path.accessor->getFingerprint(path.path);
 
     if (fingerprint) {
-        cacheKey = makeFetchToStoreCacheKey(std::string{name}, *fingerprint, method, subpath.abs());
-        if (auto res = settings.getCache()->lookupStorePath(*cacheKey, store)) {
-            debug("store path cache hit for '%s'", path);
-            return res->storePath;
+        cacheKey = makeSourcePathToHashCacheKey(*fingerprint, method, subpath);
+        if (auto res = settings.getCache()->lookup(*cacheKey)) {
+            auto hash = Hash::parseSRI(fetchers::getStrAttr(*res, "hash"));
+            auto storePath =
+                store.makeFixedOutputPathFromCA(name, ContentAddressWithReferences::fromParts(method, hash, {}));
+
+            /* Add a temproot before the call to isValidPath to prevent accidental GC in case the
+               input is cached. Note that this must be done before to avoid races. */
+            if (mode != FetchMode::DryRun)
+                store.addTempRoot(storePath);
+
+            if (mode == FetchMode::DryRun || store.isValidPath(storePath)) {
+                debug(
+                    "source path '%s' cache hit in '%s' (hash '%s')",
+                    path,
+                    store.printStorePath(storePath),
+                    hash.to_string(HashFormat::SRI, true));
+                return {storePath, hash};
+            }
+            debug("source path '%s' not in store", path);
         }
     } else {
         static auto barf = getEnv("_NIX_TEST_BARF_ON_UNCACHEABLE").value_or("") == "1";
@@ -53,16 +79,41 @@ StorePath fetchToStore(
 
     auto filter2 = filter ? *filter : defaultPathFilter;
 
-    auto storePath = mode == FetchMode::DryRun
-                         ? store.computeStorePath(name, path, method, HashAlgorithm::SHA256, {}, filter2).first
-                         : store.addToStore(name, path, method, HashAlgorithm::SHA256, {}, filter2, repair);
+    auto [storePath, hash] =
+        mode == FetchMode::DryRun
+            ? [&]() {
+                  auto [storePath, hash] =
+                      store.computeStorePath(name, path, method, HashAlgorithm::SHA256, {}, filter2);
+                  debug(
+                      "hashed '%s' to '%s' (hash '%s')",
+                      path,
+                      store.printStorePath(storePath),
+                      hash.to_string(HashFormat::SRI, true));
+                  return std::make_pair(storePath, hash);
+              }()
+            : [&]() {
+                  // FIXME: ideally addToStore() would return the hash
+                  // right away (like computeStorePath()).
+                  auto storePath = store.addToStore(name, path, method, HashAlgorithm::SHA256, {}, filter2, repair);
+                  auto info = store.queryPathInfo(storePath);
+                  assert(info->references.empty());
+                  auto hash = method == ContentAddressMethod::Raw::NixArchive ? info->narHash : ({
+                      if (!info->ca || info->ca->method != method)
+                          throw Error("path '%s' lacks a CA field", store.printStorePath(storePath));
+                      info->ca->hash;
+                  });
+                  debug(
+                      "copied '%s' to '%s' (hash '%s')",
+                      path,
+                      store.printStorePath(storePath),
+                      hash.to_string(HashFormat::SRI, true));
+                  return std::make_pair(storePath, hash);
+              }();
 
-    debug(mode == FetchMode::DryRun ? "hashed '%s'" : "copied '%s' to '%s'", path, store.printStorePath(storePath));
+    if (cacheKey)
+        settings.getCache()->upsert(*cacheKey, {{"hash", hash.to_string(HashFormat::SRI, true)}});
 
-    if (cacheKey && mode == FetchMode::Copy)
-        settings.getCache()->upsert(*cacheKey, store, {}, storePath);
-
-    return storePath;
+    return {storePath, hash};
 }
 
 } // namespace nix

src/libfetchers/fetchers.cc

@@ -339,9 +339,10 @@ std::pair<ref<SourceAccessor>, Input> Input::getAccessorUnchecked(const Settings
             // can reuse the existing nar instead of copying the unpacked
             // input back into the store on every evaluation.
             if (accessor->fingerprint) {
-                ContentAddressMethod method = ContentAddressMethod::Raw::NixArchive;
-                auto cacheKey = makeFetchToStoreCacheKey(getName(), *accessor->fingerprint, method, "/");
-                settings.getCache()->upsert(cacheKey, store, {}, storePath);
+                settings.getCache()->upsert(
+                    makeSourcePathToHashCacheKey(
+                        *accessor->fingerprint, ContentAddressMethod::Raw::NixArchive, CanonPath::root),
+                    {{"hash", store.queryPathInfo(storePath)->narHash.to_string(HashFormat::SRI, true)}});
             }
 
             accessor->setPathDisplay("«" + to_string() + "»");
@@ -489,11 +490,11 @@ void InputScheme::clone(
     const Settings & settings, Store & store, const Input & input, const std::filesystem::path & destDir) const
 {
     if (std::filesystem::exists(destDir))
-        throw Error("cannot clone into existing path %s", destDir);
+        throw Error("cannot clone into existing path %s", PathFmt(destDir));
 
     auto [accessor, input2] = getAccessor(settings, store, input);
 
-    Activity act(*logger, lvlTalkative, actUnknown, fmt("copying '%s' to %s...", input2.to_string(), destDir));
+    Activity act(*logger, lvlTalkative, actUnknown, fmt("copying '%s' to %s...", input2.to_string(), PathFmt(destDir)));
 
     RestoreSink sink(/*startFsync=*/false);
     sink.dstPath = destDir;

src/libfetchers/filtering-source-accessor.cc

@@ -10,12 +10,6 @@ std::optional<std::filesystem::path> FilteringSourceAccessor::getPhysicalPath(co
     return next->getPhysicalPath(prefix / path);
 }
 
-std::string FilteringSourceAccessor::readFile(const CanonPath & path)
-{
-    checkAccess(path);
-    return next->readFile(prefix / path);
-}
-
 void FilteringSourceAccessor::readFile(const CanonPath & path, Sink & sink, std::function<void(uint64_t)> sizeCallback)
 {
     checkAccess(path);
@@ -67,6 +61,11 @@ std::pair<CanonPath, std::optional<std::string>> FilteringSourceAccessor::getFin
     return next->getFingerprint(prefix / path);
 }
 
+void FilteringSourceAccessor::invalidateCache(const CanonPath & path)
+{
+    next->invalidateCache(prefix / path);
+}
+
 void FilteringSourceAccessor::checkAccess(const CanonPath & path)
 {
     if (!isAllowed(path))

src/libfetchers/git-lfs-fetch.cc

@@ -273,7 +273,7 @@ void Fetch::fetch(
                       + "/" + pointer->oid;
     std::filesystem::path cachePath = cacheDir / key;
     if (pathExists(cachePath)) {
-        debug("using cache entry %s -> %s", key, cachePath);
+        debug("using cache entry %s -> %s", key, PathFmt(cachePath));
         sink(readFile(cachePath));
         return;
     }
@@ -301,7 +301,7 @@ void Fetch::fetch(
         sizeCallback(size);
         downloadToSink(ourl, authHeader, sink, sha256, size);
 
-        debug("creating cache entry %s -> %s", key, cachePath);
+        debug("creating cache entry %s -> %s", key, PathFmt(cachePath));
         if (!pathExists(cachePath.parent_path()))
             createDirs(cachePath.parent_path());
         writeFile(cachePath, sink.s);

src/libfetchers/git-utils.cc

@@ -74,6 +74,29 @@ namespace nix {
 
 struct GitSourceAccessor;
 
+struct GitError : public Error
+{
+    template<typename... Ts>
+    GitError(const git_error & error, Ts &&... args)
+        : Error("")
+    {
+        auto hf = HintFmt(std::forward<Ts>(args)...);
+        err.msg = HintFmt("%1%: %2% (libgit2 error code = %3%)", Uncolored(hf.str()), error.message, error.klass);
+    }
+
+    template<typename... Ts>
+    GitError(Ts &&... args)
+        : GitError(
+              []() -> const git_error & {
+                  const git_error * p = git_error_last();
+                  assert(p && "git_error_last() is unexpectedly null");
+                  return *p;
+              }(),
+              std::forward<Ts>(args)...)
+    {
+    }
+};
+
 typedef std::unique_ptr<git_repository, Deleter<git_repository_free>> Repository;
 typedef std::unique_ptr<git_tree_entry, Deleter<git_tree_entry_free>> TreeEntry;
 typedef std::unique_ptr<git_tree, Deleter<git_tree_free>> Tree;
@@ -106,7 +129,7 @@ static void initLibGit2()
     static std::once_flag initialized;
     std::call_once(initialized, []() {
         if (git_libgit2_init() < 0)
-            throw Error("initialising libgit2: %s", git_error_last()->message);
+            throw GitError("initialising libgit2");
     });
 }
 
@@ -114,7 +137,7 @@ static git_oid hashToOID(const Hash & hash)
 {
     git_oid oid;
     if (git_oid_fromstr(&oid, hash.gitRev().c_str()))
-        throw Error("cannot convert '%s' to a Git OID", hash.gitRev());
+        throw GitError("cannot convert '%s' to a Git OID", hash.gitRev());
     return oid;
 }
 
@@ -122,8 +145,7 @@ static Object lookupObject(git_repository * repo, const git_oid & oid, git_objec
 {
     Object obj;
     if (git_object_lookup(Setter(obj), repo, &oid, type)) {
-        auto err = git_error_last();
-        throw Error("getting Git object '%s': %s", oid, err->message);
+        throw GitError("getting Git object '%s'", oid);
     }
     return obj;
 }
@@ -133,8 +155,7 @@ static T peelObject(git_object * obj, git_object_t type)
 {
     T obj2;
     if (git_object_peel((git_object **) (typename T::pointer *) Setter(obj2), obj, type)) {
-        auto err = git_error_last();
-        throw Error("peeling Git object '%s': %s", *git_object_id(obj), err->message);
+        throw Error("peeling Git object '%s'", *git_object_id(obj));
     }
     return obj2;
 }
@@ -144,7 +165,7 @@ static T dupObject(typename T::pointer obj)
 {
     T obj2;
     if (git_object_dup((git_object **) (typename T::pointer *) Setter(obj2), (git_object *) obj))
-        throw Error("duplicating object '%s': %s", *git_object_id((git_object *) obj), git_error_last()->message);
+        throw GitError("duplicating object '%s'", *git_object_id((git_object *) obj));
     return obj2;
 }
 
@@ -209,14 +230,14 @@ static void initRepoAtomically(std::filesystem::path & path, GitRepo::Options op
         return;
 
     if (!options.create)
-        throw Error("Git repository %s does not exist.", path);
+        throw Error("Git repository %s does not exist.", PathFmt(path));
 
     std::filesystem::path tmpDir = createTempDir(path.parent_path());
     AutoDelete delTmpDir(tmpDir, true);
     Repository tmpRepo;
 
     if (git_repository_init(Setter(tmpRepo), tmpDir.string().c_str(), options.bare))
-        throw Error("creating Git repository %s: %s", path, git_error_last()->message);
+        throw GitError("creating Git repository %s", PathFmt(path));
     try {
         std::filesystem::rename(tmpDir, path);
     } catch (std::filesystem::filesystem_error & e) {
@@ -226,7 +247,8 @@ static void initRepoAtomically(std::filesystem::path & path, GitRepo::Options op
             || e.code() == std::errc::directory_not_empty) {
             return;
         } else
-            throw SysError("moving temporary git repository from %s to %s", tmpDir, path);
+            throw SystemError(
+                e.code(), "moving temporary git repository from %s to %s", PathFmt(tmpDir), PathFmt(path));
     }
     // we successfully moved the repository, so the temporary directory no longer exists.
     delTmpDir.cancel();
@@ -266,7 +288,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
 
         initRepoAtomically(path, options);
         if (git_repository_open(Setter(repo), path.string().c_str()))
-            throw Error("opening Git repository %s: %s", path, git_error_last()->message);
+            throw GitError("opening Git repository %s", PathFmt(path));
 
         ObjectDb odb;
         if (options.packfilesOnly) {
@@ -279,28 +301,28 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
                */
 
             if (git_odb_new(Setter(odb)))
-                throw Error("creating Git object database: %s", git_error_last()->message);
+                throw GitError("creating Git object database");
 
             if (git_odb_backend_pack(&packBackend, (path / "objects").string().c_str()))
-                throw Error("creating pack backend: %s", git_error_last()->message);
+                throw GitError("creating pack backend");
 
             if (git_odb_add_backend(odb.get(), packBackend, 1))
-                throw Error("adding pack backend to Git object database: %s", git_error_last()->message);
+                throw GitError("adding pack backend to Git object database");
         } else {
             if (git_repository_odb(Setter(odb), repo.get()))
-                throw Error("getting Git object database: %s", git_error_last()->message);
+                throw GitError("getting Git object database");
         }
 
         // mempack_backend will be owned by the repository, so we are not expected to free it ourselves.
         if (git_mempack_new(&mempackBackend))
-            throw Error("creating mempack backend: %s", git_error_last()->message);
+            throw GitError("creating mempack backend");
 
         if (git_odb_add_backend(odb.get(), mempackBackend, 999))
-            throw Error("adding mempack backend to Git object database: %s", git_error_last()->message);
+            throw GitError("adding mempack backend to Git object database");
 
         if (options.packfilesOnly) {
             if (git_repository_set_odb(repo.get(), odb.get()))
-                throw Error("setting Git object database: %s", git_error_last()->message);
+                throw GitError("setting Git object database");
         }
     }
 
@@ -340,7 +362,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
         Indexer indexer;
         git_indexer_progress stats;
         if (git_indexer_new(Setter(indexer), pack_dir_path.c_str(), 0, nullptr, nullptr))
-            throw Error("creating git packfile indexer: %s", git_error_last()->message);
+            throw GitError("creating git packfile indexer");
 
         // TODO: provide index callback for checkInterrupt() termination
         //       though this is about an order of magnitude faster than the packbuilder
@@ -348,15 +370,15 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
         constexpr size_t chunkSize = 128 * 1024;
         for (size_t offset = 0; offset < buf.size; offset += chunkSize) {
             if (git_indexer_append(indexer.get(), buf.ptr + offset, std::min(chunkSize, buf.size - offset), &stats))
-                throw Error("appending to git packfile index: %s", git_error_last()->message);
+                throw GitError("appending to git packfile index");
             checkInterrupt();
         }
 
         if (git_indexer_commit(indexer.get(), &stats))
-            throw Error("committing git packfile index: %s", git_error_last()->message);
+            throw GitError("committing git packfile index");
 
         if (git_mempack_reset(mempackBackend))
-            throw Error("resetting git mempack backend: %s", git_error_last()->message);
+            throw GitError("resetting git mempack backend");
 
         checkInterrupt();
     }
@@ -449,7 +471,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
     void setRemote(const std::string & name, const std::string & url) override
     {
         if (git_remote_set_url(*this, name.c_str(), url.c_str()))
-            throw Error("setting remote '%s' URL to '%s': %s", name, url, git_error_last()->message);
+            throw GitError("setting remote '%s' URL to '%s'", name, url);
     }
 
     Hash resolveRef(std::string ref) override
@@ -462,7 +484,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
         // an object_id.
         std::string peeledRef = ref + "^{commit}";
         if (git_revparse_single(Setter(object), *this, peeledRef.c_str()))
-            throw Error("resolving Git reference '%s': %s", ref, git_error_last()->message);
+            throw GitError("resolving Git reference '%s'", ref);
         auto oid = git_object_id(object.get());
         return toHash(*oid);
     }
@@ -471,11 +493,11 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
     {
         GitConfig config;
         if (git_config_open_ondisk(Setter(config), configFile.string().c_str()))
-            throw Error("parsing .gitmodules file: %s", git_error_last()->message);
+            throw GitError("parsing .gitmodules file");
 
         ConfigIterator it;
         if (git_config_iterator_glob_new(Setter(it), config.get(), "^submodule\\..*\\.(path|url|branch)$"))
-            throw Error("iterating over .gitmodules: %s", git_error_last()->message);
+            throw GitError("iterating over .gitmodules");
 
         StringMap entries;
 
@@ -484,7 +506,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
             if (auto err = git_config_next(&entry, it.get())) {
                 if (err == GIT_ITEROVER)
                     break;
-                throw Error("iterating over .gitmodules: %s", git_error_last()->message);
+                throw GitError("iterating over .gitmodules");
             }
             entries.emplace(entry->name + 10, entry->value);
         }
@@ -521,7 +543,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
         git_oid headRev;
         if (auto err = git_reference_name_to_id(&headRev, *this, "HEAD")) {
             if (err != GIT_ENOTFOUND)
-                throw Error("resolving HEAD: %s", git_error_last()->message);
+                throw GitError("resolving HEAD");
         } else
             info.headRev = toHash(headRev);
 
@@ -544,7 +566,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
         options.flags |= GIT_STATUS_OPT_INCLUDE_UNMODIFIED;
         options.flags |= GIT_STATUS_OPT_EXCLUDE_SUBMODULES;
         if (git_status_foreach_ext(*this, &options, &statusCallbackTrampoline, &statusCallback))
-            throw Error("getting working directory status: %s", git_error_last()->message);
+            throw GitError("getting working directory status");
 
         /* Get submodule info. */
         auto modulesFile = path / ".gitmodules";
@@ -587,8 +609,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
         if (auto errCode = git_object_lookup(Setter(obj), *this, &oid, GIT_OBJECT_ANY)) {
             if (errCode == GIT_ENOTFOUND)
                 return false;
-            auto err = git_error_last();
-            throw Error("getting Git object '%s': %s", oid, err->message);
+            throw GitError("getting Git object '%s'", oid);
         }
 
         return true;
@@ -766,7 +787,7 @@ struct GitSourceAccessor : SourceAccessor
     {
     }
 
-    std::string readBlob(const CanonPath & path, bool symlink)
+    void readBlob(const CanonPath & path, bool symlink, Sink & sink, std::function<void(uint64_t)> sizeCallback)
     {
         auto state(state_.lock());
 
@@ -785,16 +806,22 @@ struct GitSourceAccessor : SourceAccessor
                     e.addTrace({}, "while smudging git-lfs file '%s'", path);
                     throw;
                 }
-                return s.s;
+                sizeCallback(s.s.size());
+                StringSource source{s.s};
+                source.drainInto(sink);
+                return;
             }
         }
 
-        return std::string((const char *) git_blob_rawcontent(blob.get()), git_blob_rawsize(blob.get()));
+        auto view = std::string_view((const char *) git_blob_rawcontent(blob.get()), git_blob_rawsize(blob.get()));
+        sizeCallback(view.size());
+        StringSource source{view};
+        source.drainInto(sink);
     }
 
-    std::string readFile(const CanonPath & path) override
+    void readFile(const CanonPath & path, Sink & sink, std::function<void(uint64_t)> sizeCallback) override
     {
-        return readBlob(path, false);
+        return readBlob(path, false, sink, sizeCallback);
     }
 
     bool pathExists(const CanonPath & path) override
@@ -861,7 +888,9 @@ struct GitSourceAccessor : SourceAccessor
 
     std::string readLink(const CanonPath & path) override
     {
-        return readBlob(path, true);
+        StringSink s;
+        readBlob(path, true, s, [&](uint64_t size) { s.s.reserve(size); });
+        return std::move(s.s);
     }
 
     /**
@@ -910,7 +939,7 @@ struct GitSourceAccessor : SourceAccessor
 
             TreeEntry copy;
             if (git_tree_entry_dup(Setter(copy), entry))
-                throw Error("dupping tree entry: %s", git_error_last()->message);
+                throw GitError("dupping tree entry");
 
             auto entryName = std::string_view(git_tree_entry_name(entry));
 
@@ -940,7 +969,7 @@ struct GitSourceAccessor : SourceAccessor
 
         Tree tree;
         if (git_tree_entry_to_object((git_object **) (git_tree **) Setter(tree), *state.repo, entry))
-            throw Error("looking up directory '%s': %s", showPath(path), git_error_last()->message);
+            throw GitError("looking up directory '%s'", showPath(path));
 
         return tree;
     }
@@ -975,7 +1004,7 @@ struct GitSourceAccessor : SourceAccessor
 
         Tree tree;
         if (git_tree_entry_to_object((git_object **) (git_tree **) Setter(tree), *state.repo, entry))
-            throw Error("looking up directory '%s': %s", showPath(path), git_error_last()->message);
+            throw GitError("looking up directory '%s'", showPath(path));
 
         return tree;
     }
@@ -1008,7 +1037,7 @@ struct GitSourceAccessor : SourceAccessor
 
         Blob blob;
         if (git_tree_entry_to_object((git_object **) (git_blob **) Setter(blob), *state.repo, entry))
-            throw Error("looking up file '%s': %s", showPath(path), git_error_last()->message);
+            throw GitError("looking up file '%s'", showPath(path));
 
         return blob;
     }
@@ -1060,7 +1089,7 @@ struct GitExportIgnoreSourceAccessor : CachingFilteringSourceAccessor
             if (git_error_last()->klass == GIT_ENOTFOUND)
                 return false;
             else
-                throw Error("looking up '%s': %s", showPath(path), git_error_last()->message);
+                throw GitError("looking up '%s'", showPath(path));
         } else {
             // Official git will silently reject export-ignore lines that have
             // values. We do the same.
@@ -1216,17 +1245,17 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink
                         repo.emplace(parent.repoPool.get());
 
                         if (git_blob_create_from_stream(Setter(stream), **repo, nullptr))
-                            throw Error("creating a blob stream object: %s", git_error_last()->message);
+                            throw GitError("creating a blob stream object");
 
                         if (stream->write(stream.get(), contents.data(), contents.size()))
-                            throw Error("writing a blob for tarball member '%s': %s", path, git_error_last()->message);
+                            throw GitError("writing a blob for tarball member '%s'", path);
 
                         parent.totalBufSize -= contents.size();
                         contents.clear();
                     }
                 } else {
                     if (stream->write(stream.get(), data.data(), data.size()))
-                        throw Error("writing a blob for tarball member '%s': %s", path, git_error_last()->message);
+                        throw GitError("writing a blob for tarball member '%s'", path);
                 }
             }
 
@@ -1248,7 +1277,7 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink
                acquires ownership and frees the stream. */
             git_oid oid;
             if (git_blob_create_from_stream_commit(&oid, crf->stream.release()))
-                throw Error("creating a blob object for '%s': %s", path, git_error_last()->message);
+                throw GitError("creating a blob object for '%s'", path);
             addNode(
                 *_state.lock(),
                 crf->path,
@@ -1262,8 +1291,7 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink
 
             git_oid oid;
             if (git_blob_create_from_buffer(&oid, *repo, crf->contents.data(), crf->contents.size()))
-                throw Error(
-                    "creating a blob object for '%s' from in-memory buffer: %s", crf->path, git_error_last()->message);
+                throw GitError("creating a blob object for '%s' from in-memory buffer", crf->path);
 
             addNode(
                 *_state.lock(),
@@ -1287,8 +1315,7 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink
 
             git_oid oid;
             if (git_blob_create_from_buffer(&oid, *repo, target.c_str(), target.size()))
-                throw Error(
-                    "creating a blob object for tarball symlink member '%s': %s", path, git_error_last()->message);
+                throw GitError("creating a blob object for tarball symlink member '%s'", path);
 
             auto state(_state.lock());
             addNode(*state, path, Child{GIT_FILEMODE_LINK, oid});
@@ -1349,19 +1376,19 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink
             // Write this directory.
             git_treebuilder * b;
             if (git_treebuilder_new(&b, *repo, nullptr))
-                throw Error("creating a tree builder: %s", git_error_last()->message);
+                throw GitError("creating a tree builder");
             TreeBuilder builder(b);
 
             for (auto & [name, child] : node.children) {
                 auto oid_p = std::get_if<git_oid>(&child.file);
                 auto oid = oid_p ? *oid_p : std::get<Directory>(child.file).oid.value();
                 if (git_treebuilder_insert(nullptr, builder.get(), name.c_str(), &oid, child.mode))
-                    throw Error("adding a file to a tree builder: %s", git_error_last()->message);
+                    throw GitError("adding a file to a tree builder");
             }
 
             git_oid oid;
             if (git_treebuilder_write(&oid, builder.get()))
-                throw Error("creating a tree object: %s", git_error_last()->message);
+                throw GitError("creating a tree object");
             node.oid = oid;
         }(_state.lock()->root);
 
@@ -1425,7 +1452,7 @@ std::vector<std::tuple<GitRepoImpl::Submodule, Hash>> GitRepoImpl::getSubmodules
     auto [fdTemp, pathTemp] = createTempFile("nix-git-submodules");
     try {
         writeFull(fdTemp.get(), configS);
-    } catch (SysError & e) {
+    } catch (SystemError & e) {
         e.addTrace({}, "while writing .gitmodules file to temporary file");
         throw;
     }

src/libfetchers/git.cc

@@ -37,7 +37,7 @@ namespace {
 // old version of git, which will ignore unrecognized `-c` options.
 const std::string gitInitialBranch = "__nix_dummy_branch";
 
-bool isCacheFileWithinTtl(time_t now, const struct stat & st)
+static bool isCacheFileWithinTtl(const Settings & settings, time_t now, const PosixStat & st)
 {
     return st.st_mtime + static_cast<time_t>(settings.tarballTtl) > now;
 }
@@ -72,10 +72,10 @@ std::optional<std::string> readHead(const std::filesystem::path & path)
     if (const auto parseResult = git::parseLsRemoteLine(line); parseResult && parseResult->reference == "HEAD") {
         switch (parseResult->kind) {
         case git::LsRemoteRefLine::Kind::Symbolic:
-            debug("resolved HEAD ref '%s' for repo '%s'", parseResult->target, path);
+            debug("resolved HEAD ref '%s' for repo %s", parseResult->target, PathFmt(path));
             break;
         case git::LsRemoteRefLine::Kind::Object:
-            debug("resolved HEAD rev '%s' for repo '%s'", parseResult->target, path);
+            debug("resolved HEAD rev '%s' for repo %s", parseResult->target, PathFmt(path));
             break;
         }
         return parseResult->target;
@@ -105,7 +105,7 @@ bool storeCachedHead(const std::string & actualUrl, bool shallow, const std::str
     return true;
 }
 
-std::optional<std::string> readHeadCached(const std::string & actualUrl, bool shallow)
+static std::optional<std::string> readHeadCached(const Settings & settings, const std::string & actualUrl, bool shallow)
 {
     // Create a cache path to store the branch of the HEAD ref. Append something
     // in front of the URL to prevent collision with the repository itself.
@@ -113,11 +113,11 @@ std::optional<std::string> readHeadCached(const std::string & actualUrl, bool sh
     std::filesystem::path headRefFile = cacheDir / "HEAD";
 
     time_t now = time(0);
-    struct stat st;
+    auto st = maybeStat(headRefFile);
     std::optional<std::string> cachedRef;
-    if (stat(headRefFile.string().c_str(), &st) == 0) {
+    if (st) {
         cachedRef = readHead(cacheDir);
-        if (cachedRef != std::nullopt && *cachedRef != gitInitialBranch && isCacheFileWithinTtl(now, st)) {
+        if (cachedRef != std::nullopt && *cachedRef != gitInitialBranch && isCacheFileWithinTtl(settings, now, *st)) {
             debug("using cached HEAD ref '%s' for repo '%s'", *cachedRef, actualUrl);
             return cachedRef;
         }
@@ -728,12 +728,12 @@ struct GitInputScheme : InputScheme
         return revCount;
     }
 
-    std::string getDefaultRef(const RepoInfo & repoInfo, bool shallow) const
+    std::string getDefaultRef(const Settings & settings, const RepoInfo & repoInfo, bool shallow) const
     {
         auto head = std::visit(
             overloaded{
                 [&](const std::filesystem::path & path) { return GitRepo::openRepo(path, {})->getWorkdirRef(); },
-                [&](const ParsedURL & url) { return readHeadCached(url.to_string(), shallow); }},
+                [&](const ParsedURL & url) { return readHeadCached(settings, url.to_string(), shallow); }},
             repoInfo.location);
         if (!head) {
             warn("could not read HEAD ref from repo at '%s', using 'master'", repoInfo.locationToArg());
@@ -753,9 +753,10 @@ struct GitInputScheme : InputScheme
                     "\n"
                     "git -C %2% add \"%1%\"",
                     path.rel(),
-                    repoPath);
+                    PathFmt(repoPath));
             else
-                return RestrictedPathError("Path '%s' does not exist in Git repository %s.", path.rel(), repoPath);
+                return RestrictedPathError(
+                    "Path '%s' does not exist in Git repository %s.", path.rel(), PathFmt(repoPath));
         };
     }
 
@@ -782,7 +783,7 @@ struct GitInputScheme : InputScheme
 
         auto originalRef = input.getRef();
         bool shallow = getShallowAttr(input);
-        auto ref = originalRef ? *originalRef : getDefaultRef(repoInfo, shallow);
+        auto ref = originalRef ? *originalRef : getDefaultRef(settings, repoInfo, shallow);
         input.attrs.insert_or_assign("ref", ref);
 
         std::filesystem::path repoDir;
@@ -818,10 +819,10 @@ struct GitInputScheme : InputScheme
                 if (getAllRefsAttr(input)) {
                     doFetch = true;
                 } else {
-                    /* If the local ref is older than ‘tarball-ttl’ seconds, do a
+                    /* If the local ref is older than 'tarball-ttl' seconds, do a
                        git fetch to update the local ref to the remote ref. */
-                    struct stat st;
-                    doFetch = stat(localRefFile.string().c_str(), &st) != 0 || !isCacheFileWithinTtl(now, st);
+                    auto st = maybeStat(localRefFile);
+                    doFetch = !st || !isCacheFileWithinTtl(settings, now, *st);
                 }
             }
 
@@ -848,7 +849,7 @@ struct GitInputScheme : InputScheme
                     if (!input.getRev())
                         setWriteTime(localRefFile, now, now);
                 } catch (Error & e) {
-                    warn("could not update mtime for file %s: %s", localRefFile, e.info().msg);
+                    warn("could not update mtime for file %s: %s", PathFmt(localRefFile), e.info().msg);
                 }
                 if (!originalRef && !storeCachedHead(repoUrl.to_string(), shallow, ref))
                     warn("could not update cached head '%s' for '%s'", ref, repoInfo.locationToArg());

src/libfetchers/include/nix/fetchers/fetch-settings.hh

@@ -129,6 +129,25 @@ struct Settings : public Config
         true,
         Xp::Flakes};
 
+    Setting<unsigned int> tarballTtl{
+        this,
+        60 * 60,
+        "tarball-ttl",
+        R"(
+          The number of seconds a downloaded tarball is considered fresh. If
+          the cached tarball is stale, Nix checks whether it is still up
+          to date using the ETag header. Nix downloads a new version if
+          the ETag header is unsupported, or the cached ETag doesn't match.
+
+          Setting the TTL to `0` forces Nix to always check if the tarball is
+          up to date.
+
+          Nix caches tarballs in `$XDG_CACHE_HOME/nix/tarballs`.
+
+          Files fetched via `NIX_PATH`, `fetchGit`, `fetchMercurial`,
+          `fetchTarball`, and `fetchurl` respect this TTL.
+        )"};
+
     ref<Cache> getCache() const;
 
     ref<GitRepo> getTarballCache() const;

src/libfetchers/include/nix/fetchers/fetch-to-store.hh

@@ -24,7 +24,17 @@ StorePath fetchToStore(
     PathFilter * filter = nullptr,
     RepairFlag repair = NoRepair);
 
-fetchers::Cache::Key makeFetchToStoreCacheKey(
-    const std::string & name, const std::string & fingerprint, ContentAddressMethod method, const std::string & path);
+std::pair<StorePath, Hash> fetchToStore2(
+    const fetchers::Settings & settings,
+    Store & store,
+    const SourcePath & path,
+    FetchMode mode,
+    std::string_view name = "source",
+    ContentAddressMethod method = ContentAddressMethod::Raw::NixArchive,
+    PathFilter * filter = nullptr,
+    RepairFlag repair = NoRepair);
+
+fetchers::Cache::Key
+makeSourcePathToHashCacheKey(std::string_view fingerprint, ContentAddressMethod method, const CanonPath & path);
 
 } // namespace nix

src/libfetchers/include/nix/fetchers/filtering-source-accessor.hh

@@ -34,7 +34,7 @@ struct FilteringSourceAccessor : SourceAccessor
 
     std::optional<std::filesystem::path> getPhysicalPath(const CanonPath & path) override;
 
-    std::string readFile(const CanonPath & path) override;
+    using SourceAccessor::readFile;
 
     void readFile(const CanonPath & path, Sink & sink, std::function<void(uint64_t)> sizeCallback) override;
 
@@ -52,6 +52,8 @@ struct FilteringSourceAccessor : SourceAccessor
 
     std::pair<CanonPath, std::optional<std::string>> getFingerprint(const CanonPath & path) override;
 
+    void invalidateCache(const CanonPath & path) override;
+
     /**
      * Call `makeNotAllowedError` to throw a `RestrictedPathError`
      * exception if `isAllowed()` returns `false` for `path`.

src/libfetchers/include/nix/fetchers/input-cache.hh

@@ -1,3 +1,5 @@
+#pragma once
+
 #include "nix/fetchers/fetchers.hh"
 
 namespace nix::fetchers {

src/libfetchers/mercurial.cc

@@ -1,5 +1,6 @@
 #include "nix/fetchers/fetchers.hh"
 #include "nix/util/processes.hh"
+#include "nix/util/environment-variables.hh"
 #include "nix/util/users.hh"
 #include "nix/fetchers/cache.hh"
 #include "nix/store/globals.hh"
@@ -16,9 +17,9 @@ namespace nix::fetchers {
 
 static RunOptions hgOptions(const Strings & args)
 {
-    auto env = getEnv();
+    auto env = getEnvOs();
     // Set HGPLAIN: this means we get consistent output from hg and avoids leakage from a user or system .hgrc.
-    env["HGPLAIN"] = "";
+    env[OS_STR("HGPLAIN")] = OS_STR("");
 
     return {.program = "hg", .lookupPath = true, .args = args, .environment = env};
 }

src/libfetchers/path.cc

@@ -154,7 +154,7 @@ struct PathInputScheme : InputScheme
 
         time_t mtime = 0;
         if (!storePath || storePath->name() != "source" || !store.isValidPath(*storePath)) {
-            Activity act(*logger, lvlTalkative, actUnknown, fmt("copying %s to the store", absPath));
+            Activity act(*logger, lvlTalkative, actUnknown, fmt("copying %s to the store", PathFmt(absPath)));
             // FIXME: try to substitute storePath.
             auto src = sinkToSource(
                 [&](Sink & sink) { mtime = dumpPathAndGetMtime(absPath.string(), sink, defaultPathFilter); });
@@ -165,14 +165,12 @@ struct PathInputScheme : InputScheme
 
         // To prevent `fetchToStore()` copying the path again to Nix
         // store, pre-create an entry in the fetcher cache.
-        accessor->fingerprint =
-            fmt("path:%s", store.queryPathInfo(*storePath)->narHash.to_string(HashFormat::SRI, true));
+        auto narHash = store.queryPathInfo(*storePath)->narHash.to_string(HashFormat::SRI, true);
+        accessor->fingerprint = fmt("path:%s", narHash);
         settings.getCache()->upsert(
-            makeFetchToStoreCacheKey(
-                input.getName(), *accessor->fingerprint, ContentAddressMethod::Raw::NixArchive, "/"),
-            store,
-            {},
-            *storePath);
+            makeSourcePathToHashCacheKey(
+                *accessor->fingerprint, ContentAddressMethod::Raw::NixArchive, CanonPath::root),
+            {{"hash", narHash}});
 
         /* Trust the lastModified value supplied by the user, if
            any. It's not a "secure" attribute so we don't care. */

src/libfetchers/registry.cc

@@ -92,7 +92,7 @@ void Registry::remove(const Input & input)
 
 static std::filesystem::path getSystemRegistryPath()
 {
-    return settings.nixConfDir / "registry.json";
+    return nixConfDir() / "registry.json";
 }
 
 static std::shared_ptr<Registry> getSystemRegistry(const Settings & settings)

src/libfetchers/tarball.cc

@@ -121,14 +121,15 @@ static DownloadTarballResult downloadTarball_(
                 url);
         }
         if (!exists(localPath)) {
-            throw Error("tarball '%s' does not exist.", localPath);
+            throw Error("tarball %s does not exist.", PathFmt(localPath));
         }
         if (is_directory(localPath)) {
             if (exists(localPath / ".git")) {
                 throw Error(
-                    "tarball '%s' is a git repository, not a tarball. Please use `git+file` as the scheme.", localPath);
+                    "tarball %s is a git repository, not a tarball. Please use `git+file` as the scheme.",
+                    PathFmt(localPath));
             }
-            throw Error("tarball '%s' is a directory, not a file.", localPath);
+            throw Error("tarball %s is a directory, not a file.", PathFmt(localPath));
         }
     }
 
@@ -176,7 +177,8 @@ static DownloadTarballResult downloadTarball_(
            the entire file to disk so libarchive can access it
            in random-access mode. */
         auto [fdTemp, path] = createTempFile("nix-zipfile");
-        cleanupTemp.reset(path);
+        cleanupTemp.cancel();
+        cleanupTemp = {path};
         debug("downloading '%s' into '%s'...", url, path);
         {
             FdSink sink(fdTemp.get());

src/libflake-tests/flakeref.cc

@@ -283,4 +283,18 @@ TEST(to_string, doesntReencodeUrl)
     ASSERT_EQ(unparsed, expected);
 }
 
+TEST(parseFlakeRef, malformedGithubUrlDoesNotCrash)
+{
+    experimentalFeatureSettings.experimentalFeatures.get().insert(Xp::Flakes);
+
+    fetchers::Settings fetchSettings;
+
+    // Using ref= instead of rev= with a github: URL should produce an
+    // error, not an assertion failure in renderAuthorityAndPath
+    // (https://github.com/NixOS/nix/issues/15196).
+    EXPECT_THROW(
+        parseFlakeRef(fetchSettings, "github:nixos/nixpkgs/nixpkgs.git?ref=aead170c1a49253ebfa5027010dfd89a77b73ca4"),
+        Error);
+}
+
 } // namespace nix

src/libflake/flake-primops.cc

@@ -35,35 +35,39 @@ namespace nix::flake::primops {
 PrimOp getFlake(const Settings & settings)
 {
     auto prim_getFlake = [&settings](EvalState & state, const PosIdx pos, Value ** args, Value & v) {
-        std::string flakeRefS(
-            state.forceStringNoCtx(*args[0], pos, "while evaluating the argument passed to builtins.getFlake"));
-        auto flakeRef = nix::parseFlakeRef(state.fetchSettings, flakeRefS, {}, true);
-        if (state.settings.pureEval && !flakeRef.input.isLocked(state.fetchSettings))
-            throw Error(
-                "cannot call 'getFlake' on unlocked flake reference '%s', at %s (use --impure to override)",
-                flakeRefS,
-                state.positions[pos]);
+        state.forceValue(*args[0], pos);
 
-        callFlake(
-            state,
-            lockFlake(
-                settings,
-                state,
-                flakeRef,
-                LockFlags{
-                    .updateLockFile = false,
-                    .writeLockFile = false,
-                    .useRegistries = !state.settings.pureEval && settings.useRegistries,
-                    .allowUnlocked = !state.settings.pureEval,
-                }),
-            v);
+        LockFlags lockFlags{
+            .updateLockFile = false,
+            .writeLockFile = false,
+            .useRegistries = !state.settings.pureEval && settings.useRegistries,
+            .allowUnlocked = !state.settings.pureEval,
+        };
+
+        if (args[0]->type() == nPath) {
+            auto path = state.realisePath(pos, *args[0]);
+            callFlake(state, lockFlake(settings, state, path, lockFlags), v);
+        } else {
+            NixStringContext context;
+            std::string flakeRefS(
+                state.forceStringNoCtx(*args[0], pos, "while evaluating the argument passed to builtins.getFlake"));
+
+            auto flakeRef = nix::parseFlakeRef(state.fetchSettings, flakeRefS, {}, true);
+            if (state.settings.pureEval && !flakeRef.input.isLocked(state.fetchSettings))
+                throw Error(
+                    "cannot call 'getFlake' on unlocked flake reference '%s', at %s (use --impure to override)",
+                    flakeRefS,
+                    state.positions[pos]);
+
+            callFlake(state, lockFlake(settings, state, flakeRef, lockFlags), v);
+        }
     };
 
     return PrimOp{
         .name = "__getFlake",
         .args = {"args"},
         .doc = R"(
-          Fetch a flake from a flake reference, and return its output attributes and some metadata. For example:
+          Fetch a flake from a flake reference or a path, and return its output attributes and some metadata. For example:
 
           ```nix
           (builtins.getFlake "nix/55bc52401966fbffa525c574c14f67b00bc4fb3a").packages.x86_64-linux.nix
@@ -128,6 +132,7 @@ static void prim_flakeRefToString(EvalState & state, const PosIdx pos, Value **
     state.forceAttrs(*args[0], noPos, "while evaluating the argument passed to builtins.flakeRefToString");
     fetchers::Attrs attrs;
     for (const auto & attr : *args[0]->attrs()) {
+        state.forceValue(*attr.value, attr.pos);
         auto t = attr.value->type();
         if (t == nInt) {
             auto intValue = attr.value->integer().value;

src/libflake/flake.cc

@@ -416,10 +416,8 @@ static LockFile readLockFile(const fetchers::Settings & fetchSettings, const Sou
                                      : LockFile();
 }
 
-/* Compute an in-memory lock file for the specified top-level flake,
-   and optionally write it to file, if the flake is writable. */
-LockedFlake
-lockFlake(const Settings & settings, EvalState & state, const FlakeRef & topRef, const LockFlags & lockFlags)
+LockedFlake lockFlake(
+    const Settings & settings, EvalState & state, const FlakeRef & topRef, const LockFlags & lockFlags, Flake flake)
 {
     experimentalFeatureSettings.require(Xp::Flakes);
 
@@ -427,8 +425,6 @@ lockFlake(const Settings & settings, EvalState & state, const FlakeRef & topRef,
     auto useRegistriesTop = useRegistries ? fetchers::UseRegistries::All : fetchers::UseRegistries::No;
     auto useRegistriesInputs = useRegistries ? fetchers::UseRegistries::Limited : fetchers::UseRegistries::No;
 
-    auto flake = getFlake(state, topRef, useRegistriesTop, {});
-
     if (lockFlags.applyNixConfig) {
         flake.config.apply(settings);
         state.store->setOptions();
@@ -850,11 +846,11 @@ lockFlake(const Settings & settings, EvalState & state, const FlakeRef & topRef,
                             auto s = chomp(diff);
                             if (lockFileExists) {
                                 if (s.empty())
-                                    warn("updating lock file %s", outputLockFilePath);
+                                    warn("updating lock file %s", PathFmt(outputLockFilePath));
                                 else
-                                    warn("updating lock file %s:\n%s", outputLockFilePath, s);
+                                    warn("updating lock file %s:\n%s", PathFmt(outputLockFilePath), s);
                             } else
-                                warn("creating lock file %s: \n%s", outputLockFilePath, s);
+                                warn("creating lock file %s: \n%s", PathFmt(outputLockFilePath), s);
 
                             std::optional<std::string> commitMessage = std::nullopt;
 
@@ -876,6 +872,8 @@ lockFlake(const Settings & settings, EvalState & state, const FlakeRef & topRef,
                                 CanonPath((topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock"),
                                 newLockFileS,
                                 commitMessage);
+
+                            flake.lockFilePath().invalidateCache();
                         }
 
                         /* Rewriting the lockfile changed the top-level
@@ -906,6 +904,22 @@ lockFlake(const Settings & settings, EvalState & state, const FlakeRef & topRef,
     }
 }
 
+LockedFlake
+lockFlake(const Settings & settings, EvalState & state, const FlakeRef & topRef, const LockFlags & lockFlags)
+{
+    auto useRegistries = lockFlags.useRegistries.value_or(settings.useRegistries);
+    auto useRegistriesTop = useRegistries ? fetchers::UseRegistries::All : fetchers::UseRegistries::No;
+    return lockFlake(settings, state, topRef, lockFlags, getFlake(state, topRef, useRegistriesTop, {}));
+}
+
+LockedFlake
+lockFlake(const Settings & settings, EvalState & state, const SourcePath & flakeDir, const LockFlags & lockFlags)
+{
+    /* We need a fake flakeref to put in the `Flake` struct, but it's not used for anything. */
+    auto fakeRef = parseFlakeRef(state.fetchSettings, "flake:get-flake");
+    return lockFlake(settings, state, fakeRef, lockFlags, readFlake(state, fakeRef, fakeRef, fakeRef, flakeDir, {}));
+}
+
 static ref<SourceAccessor> makeInternalFS()
 {
     auto internalFS = make_ref<MemorySourceAccessor>(MemorySourceAccessor{});

src/libflake/flakeref.cc

@@ -205,7 +205,7 @@ std::pair<FlakeRef, std::string> parsePathFlakeRefWithFragment(
         fetchSettings,
         {
             .scheme = "path",
-            .authority = ParsedURL::Authority{},
+            .authority = isAbsolute(path) ? std::optional{ParsedURL::Authority{}} : std::nullopt,
             .path = splitString<std::vector<std::string>>(path, "/"),
             .query = query,
             .fragment = fragment,

src/libflake/include/nix/flake/flake.hh

@@ -214,9 +214,16 @@ struct LockFlags
     std::set<NonEmptyInputAttrPath> inputUpdates;
 };
 
+/*
+ * Compute an in-memory lock file for the specified top-level flake, and optionally write it to file, if the flake is
+ * writable.
+ */
 LockedFlake
 lockFlake(const Settings & settings, EvalState & state, const FlakeRef & flakeRef, const LockFlags & lockFlags);
 
+LockedFlake
+lockFlake(const Settings & settings, EvalState & state, const SourcePath & flakeDir, const LockFlags & lockFlags);
+
 void callFlake(EvalState & state, const LockedFlake & lockedFlake, Value & v);
 
 /**

src/libmain/include/nix/main/shared.hh

@@ -13,8 +13,6 @@
 
 namespace nix {
 
-int handleExceptions(const std::string & programName, std::function<void()> fun);
-
 /**
  * Don't forget to call initPlugins() after settings are initialized!
  * @param loadConfig Whether to load configuration from `nix.conf`, `NIX_CONFIG`, etc. May be disabled for unit tests.
@@ -91,19 +89,7 @@ extern volatile ::sig_atomic_t blockInt;
 
 struct GCResults;
 
-struct PrintFreed
-{
-    bool show;
-    const GCResults & results;
-
-    PrintFreed(bool show, const GCResults & results)
-        : show(show)
-        , results(results)
-    {
-    }
-
-    ~PrintFreed();
-};
+void printFreed(bool dryRun, const GCResults & results);
 
 #ifndef _WIN32
 /**

src/libmain/plugin.cc

@@ -83,8 +83,8 @@ void initPlugins()
                 checkInterrupt();
                 pluginFiles.emplace_back(ent.path());
             }
-        } catch (SysError & e) {
-            if (e.errNo != ENOTDIR)
+        } catch (SystemError & e) {
+            if (!e.is(std::errc::not_a_directory))
                 throw;
             pluginFiles.emplace_back(pluginFile);
         }
@@ -95,7 +95,7 @@ void initPlugins()
 #ifndef _WIN32 // TODO implement via DLL loading on Windows
             void * handle = dlopen(file.c_str(), RTLD_LAZY | RTLD_LOCAL);
             if (!handle)
-                throw Error("could not dynamically open plugin file '%s': %s", file, dlerror());
+                throw Error("could not dynamically open plugin file %s: %s", PathFmt(file), dlerror());
 
             /* Older plugins use a statically initialized object to run their code.
                Newer plugins can also export nix_plugin_entry() */
@@ -103,7 +103,7 @@ void initPlugins()
             if (nix_plugin_entry)
                 nix_plugin_entry();
 #else
-            throw Error("could not dynamically open plugin file '%s'", file);
+            throw Error("could not dynamically open plugin file %s", PathFmt(file));
 #endif
         }
     }

src/libmain/shared.cc

@@ -1,7 +1,9 @@
 #include "nix/store/globals.hh"
 #include "nix/util/current-process.hh"
+#include "nix/util/executable-path.hh"
 #include "nix/main/shared.hh"
 #include "nix/store/store-api.hh"
+#include "nix/store/store-open.hh"
 #include "nix/store/gc-store.hh"
 #include "nix/main/loggers.hh"
 #include "nix/main/progress-bar.hh"
@@ -17,6 +19,10 @@
 #include <sys/stat.h>
 #include <unistd.h>
 #include <signal.h>
+
+#ifndef _WIN32
+#  include <sys/resource.h>
+#endif
 #ifdef __linux__
 #  include <features.h>
 #endif
@@ -116,6 +122,26 @@ std::string getArg(const std::string & opt, Strings::iterator & i, const Strings
 static void sigHandler(int signo) {}
 #endif
 
+/**
+ * Increase the open file soft limit to the hard limit. On some
+ * platforms (macOS), the default soft limit is very low, but the hard
+ * limit is high. So let's just raise it the maximum permitted.
+ */
+void bumpFileLimit()
+{
+#ifndef _WIN32
+    struct rlimit limit;
+    if (getrlimit(RLIMIT_NOFILE, &limit) != 0)
+        return;
+
+    if (limit.rlim_cur < limit.rlim_max) {
+        limit.rlim_cur = limit.rlim_max;
+        // Ignore errors, this is best effort.
+        setrlimit(RLIMIT_NOFILE, &limit);
+    }
+#endif
+}
+
 void initNix(bool loadConfig)
 {
     /* Turn on buffering for cerr. */
@@ -183,6 +209,8 @@ void initNix(bool loadConfig)
        now.  In particular, store objects should be readable by
        everybody. */
     umask(0022);
+
+    bumpFileLimit();
 }
 
 LegacyArgs::LegacyArgs(
@@ -209,13 +237,13 @@ LegacyArgs::LegacyArgs(
         .longName = "keep-going",
         .shortName = 'k',
         .description = "Keep going after a build fails.",
-        .handler = {&(bool &) settings.keepGoing, true},
+        .handler = {&(bool &) settings.getWorkerSettings().keepGoing, true},
     });
 
     addFlag({
         .longName = "fallback",
         .description = "Build from source if substitution fails.",
-        .handler = {&(bool &) settings.tryFallback, true},
+        .handler = {&(bool &) settings.getWorkerSettings().tryFallback, true},
     });
 
     auto intSettingAlias =
@@ -252,7 +280,7 @@ LegacyArgs::LegacyArgs(
         .longName = "store",
         .description = "The URL of the Nix store to use.",
         .labels = {"store-uri"},
-        .handler = {&(std::string &) settings.storeUri},
+        .handler = {[](std::string s) { settings.storeUri = StoreReference::parse(s); }},
     });
 }
 
@@ -304,44 +332,16 @@ void printVersion(const std::string & programName)
         std::cout << "System type: " << settings.thisSystem << "\n";
         std::cout << "Additional system types: " << concatStringsSep(", ", settings.extraPlatforms.get()) << "\n";
         std::cout << "Features: " << concatStringsSep(", ", cfg) << "\n";
-        std::cout << "System configuration file: " << (settings.nixConfDir / "nix.conf") << "\n";
-        std::cout << "User configuration files: " << concatStringsSep(":", settings.nixUserConfFiles) << "\n";
-        std::cout << "Store directory: " << settings.nixStore << "\n";
+        std::cout << "System configuration file: " << nixConfFile() << "\n";
+        std::cout << "User configuration files: "
+                  << os_string_to_string(ExecutablePath{.directories = nixUserConfFiles()}.render()) << "\n";
+        std::cout << "Store directory: " << resolveStoreConfig(StoreReference{settings.storeUri.get()})->storeDir
+                  << "\n";
         std::cout << "State directory: " << settings.nixStateDir << "\n";
-        std::cout << "Data directory: " << settings.nixDataDir << "\n";
     }
     throw Exit();
 }
 
-int handleExceptions(const std::string & programName, std::function<void()> fun)
-{
-    ReceiveInterrupts receiveInterrupts; // FIXME: need better place for this
-
-    ErrorInfo::programName = baseNameOf(programName);
-
-    std::string error = ANSI_RED "error:" ANSI_NORMAL " ";
-    try {
-        fun();
-    } catch (Exit & e) {
-        return e.status;
-    } catch (UsageError & e) {
-        logError(e.info());
-        printError("Try '%1% --help' for more information.", programName);
-        return 1;
-    } catch (BaseError & e) {
-        logError(e.info());
-        return e.info().status;
-    } catch (std::bad_alloc & e) {
-        printError(error + "out of memory");
-        return 1;
-    } catch (std::exception & e) {
-        printError(error + e.what());
-        return 1;
-    }
-
-    return 0;
-}
-
 RunPager::RunPager()
 {
     if (!isatty(STDOUT_FILENO))
@@ -396,9 +396,12 @@ RunPager::~RunPager()
     }
 }
 
-PrintFreed::~PrintFreed()
+void printFreed(bool dryRun, const GCResults & results)
 {
-    if (show)
+    /* bytesFreed cannot be reliably computed without actually deleting store paths because of hardlinking. */
+    if (dryRun)
+        std::cout << fmt("%d store paths would be deleted\n", results.paths.size());
+    else
         std::cout << fmt("%d store paths deleted, %s freed\n", results.paths.size(), renderSize(results.bytesFreed));
 }
 

src/libstore-c/nix_api_store.cc

@@ -9,6 +9,7 @@
 #include "nix/store/path.hh"
 #include "nix/store/store-api.hh"
 #include "nix/store/store-open.hh"
+#include "nix/store/store-reference.hh"
 #include "nix/store/build-result.hh"
 #include "nix/store/local-fs-store.hh"
 #include "nix/util/base-nix-32.hh"
@@ -47,14 +48,14 @@ Store * nix_store_open(nix_c_context * context, const char * uri, const char ***
         if (uri_str.empty())
             return new Store{nix::openStore()};
 
-        if (!params)
-            return new Store{nix::openStore(uri_str)};
+        auto storeRef = nix::StoreReference::parse(uri_str);
 
-        nix::Store::Config::Params params_map;
-        for (size_t i = 0; params[i] != nullptr; i++) {
-            params_map[params[i][0]] = params[i][1];
+        if (params) {
+            for (size_t i = 0; params[i] != nullptr; i++) {
+                storeRef.params[params[i][0]] = params[i][1];
+            }
         }
-        return new Store{nix::openStore(uri_str, params_map)};
+        return new Store{nix::openStore(std::move(storeRef))};
     }
     NIXC_CATCH_ERRS_NULL
 }
@@ -182,10 +183,8 @@ nix_err nix_store_realise(
         assert(results.size() == 1);
 
         // Check if any builds failed
-        for (auto & result : results) {
-            if (auto * failureP = result.tryGetFailure())
-                failureP->rethrow();
-        }
+        for (auto & result : results)
+            result.tryThrowBuildError();
 
         if (callback) {
             for (const auto & result : results) {
@@ -309,7 +308,12 @@ StorePath * nix_add_derivation(nix_c_context * context, Store * store, nix_deriv
     if (context)
         context->last_err_code = NIX_OK;
     try {
-        auto ret = nix::writeDerivation(*store->ptr, derivation->drv, nix::NoRepair);
+        /* Quite dubious that users would want this to silently suceed
+           without actually writing the derivation if this setting is
+           set, but it was that way already, so we are doing this for
+           back-compat for now. */
+        auto ret = nix::settings.readOnlyMode ? nix::computeStorePath(*store->ptr, derivation->drv)
+                                              : store->ptr->writeDerivation(derivation->drv, nix::NoRepair);
 
         return new StorePath{ret};
     }
@@ -338,4 +342,42 @@ nix_derivation * nix_store_drv_from_store_path(nix_c_context * context, Store *
     NIXC_CATCH_ERRS_NULL
 }
 
+StorePath * nix_store_query_path_from_hash_part(nix_c_context * context, Store * store, const char * hash)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        std::optional<nix::StorePath> s = store->ptr->queryPathFromHashPart(hash);
+
+        if (!s.has_value()) {
+            return nullptr;
+        }
+
+        return new StorePath{std::move(s.value())};
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+nix_err nix_store_copy_path(
+    nix_c_context * context, Store * srcStore, Store * dstStore, const StorePath * path, bool repair, bool checkSigs)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        if (srcStore == nullptr)
+            return nix_set_err_msg(context, NIX_ERR_UNKNOWN, "Source store is null");
+
+        if (dstStore == nullptr)
+            return nix_set_err_msg(context, NIX_ERR_UNKNOWN, "Destination store is null");
+
+        if (path == nullptr)
+            return nix_set_err_msg(context, NIX_ERR_UNKNOWN, "Store path is null");
+
+        auto repairFlag = repair ? nix::RepairFlag::Repair : nix::RepairFlag::NoRepair;
+        auto checkSigsFlag = checkSigs ? nix::CheckSigsFlag::CheckSigs : nix::CheckSigsFlag::NoCheckSigs;
+        nix::copyStorePath(*srcStore->ptr, *dstStore->ptr, path->path, repairFlag, checkSigsFlag);
+    }
+    NIXC_CATCH_ERRS
+}
+
 } // extern "C"

src/libstore-c/nix_api_store.h

@@ -259,6 +259,30 @@ nix_err nix_store_get_fs_closure(
  */
 nix_derivation * nix_store_drv_from_store_path(nix_c_context * context, Store * store, const StorePath * path);
 
+/**
+ * @brief Query the full store path given the hash part of a valid store
+ * path, or empty if no matching path is found.
+ *
+ * @param[out] context Optional, stores error information
+ * @param[in] store nix store reference
+ * @param[in] hash Hash part of path as a string
+ * @return Store path reference, NULL if no matching path is found.
+ */
+StorePath * nix_store_query_path_from_hash_part(nix_c_context * context, Store * store, const char * hash);
+
+/**
+ * @brief Copy a path from one store to another.
+ *
+ * @param[out] context Optional, stores error information
+ * @param[in] srcStore nix source store reference
+ * @param[in] dstStore nix destination store reference
+ * @param[in] path The path to copy
+ * @param[in] repair Whether to repair the path
+ * @param[in] checkSigs Whether to check path signatures are trusted before copying
+ */
+nix_err nix_store_copy_path(
+    nix_c_context * context, Store * srcStore, Store * dstStore, const StorePath * path, bool repair, bool checkSigs);
+
 // cffi end
 #ifdef __cplusplus
 }

src/libstore-test-support/https-store.cc

@@ -0,0 +1,144 @@
+#include "nix/store/tests/https-store.hh"
+
+#include <thread>
+
+namespace nix::testing {
+
+void TestHttpBinaryCacheStore::init()
+{
+    BinaryCacheStore::init();
+}
+
+ref<TestHttpBinaryCacheStore> TestHttpBinaryCacheStoreConfig::openTestStore(ref<FileTransfer> fileTransfer) const
+{
+    auto store = make_ref<TestHttpBinaryCacheStore>(
+        ref{// FIXME we shouldn't actually need a mutable config
+            std::const_pointer_cast<HttpBinaryCacheStore::Config>(shared_from_this())},
+        fileTransfer);
+    store->init();
+    return store;
+}
+
+void HttpsBinaryCacheStoreTest::openssl(Strings args)
+{
+    runProgram("openssl", /*lookupPath=*/true, args);
+}
+
+void HttpsBinaryCacheStoreTest::SetUp()
+{
+    LibStoreNetworkTest::SetUp();
+
+#ifdef _WIN32
+    GTEST_SKIP() << "HTTPS store tests are not supported on Windows";
+#endif
+
+    tmpDir = createTempDir();
+    cacheDir = tmpDir / "cache";
+    delTmpDir = std::make_unique<AutoDelete>(tmpDir);
+
+    localCacheStore =
+        make_ref<LocalBinaryCacheStoreConfig>("file", cacheDir.string(), LocalBinaryCacheStoreConfig::Params{})
+            ->openStore();
+
+    caCert = tmpDir / "ca.crt";
+    caKey = tmpDir / "ca.key";
+    serverCert = tmpDir / "server.crt";
+    serverKey = tmpDir / "server.key";
+    clientCert = tmpDir / "client.crt";
+    clientKey = tmpDir / "client.key";
+
+    // clang-format off
+    openssl({"ecparam", "-genkey", "-name", "prime256v1", "-out", caKey.string()});
+    openssl({"req", "-new", "-x509", "-days", "1", "-key", caKey.string(), "-out", caCert.string(), "-subj", "/CN=TestCA"});
+    auto serverExtFile = tmpDir / "server.ext";
+    writeFile(serverExtFile, "subjectAltName=DNS:localhost,IP:127.0.0.1");
+    openssl({"ecparam", "-genkey", "-name", "prime256v1", "-out", serverKey.string()});
+    openssl({"req", "-new", "-key", serverKey.string(), "-out", (tmpDir / "server.csr").string(), "-subj", "/CN=localhost", "-addext", "subjectAltName=DNS:localhost,IP:127.0.0.1"});
+    openssl({"x509", "-req", "-in", (tmpDir / "server.csr").string(), "-CA", caCert.string(), "-CAkey", caKey.string(), "-CAcreateserial", "-out", serverCert.string(), "-days", "1", "-extfile", serverExtFile.string()});
+    openssl({"ecparam", "-genkey", "-name", "prime256v1", "-out", clientKey.string()});
+    openssl({"req", "-new", "-key", clientKey.string(), "-out", (tmpDir / "client.csr").string(), "-subj", "/CN=TestClient"});
+    openssl({"x509", "-req", "-in", (tmpDir / "client.csr").string(), "-CA", caCert.string(), "-CAkey", caKey.string(), "-CAcreateserial", "-out", clientCert.string(), "-days", "1"});
+    // clang-format on
+
+#ifndef _WIN32 /* FIXME: Can't yet start processes on windows */
+    auto args = serverArgs();
+    serverPid = startProcess(
+        [&] {
+            if (chdir(cacheDir.c_str()) == -1)
+                _exit(1);
+            std::vector<char *> argv;
+            argv.push_back(const_cast<char *>("openssl"));
+            for (auto & a : args)
+                argv.push_back(const_cast<char *>(a.c_str()));
+            argv.push_back(nullptr);
+            execvp("openssl", argv.data());
+            _exit(1);
+        },
+        {.dieWithParent = true});
+#endif
+
+    /* As an optimization, sleep for a bit to allow the server to come up to avoid retrying when connecting.
+       This won't make the tests fail, but does make them run faster. We don't need to overcomplicate by waiting
+       for the port explicitly - this is enough. */
+    std::this_thread::sleep_for(std::chrono::milliseconds(50));
+
+    /* Create custom FileTransferSettings with our test CA certificate.
+       This avoids mutating global settings. */
+    testFileTransferSettings = std::make_unique<FileTransferSettings>();
+    testFileTransferSettings->caFile = caCert;
+    testFileTransfer = makeFileTransfer(*testFileTransferSettings);
+}
+
+void HttpsBinaryCacheStoreTest::TearDown()
+{
+    serverPid.kill();
+    delTmpDir.reset();
+    testFileTransferSettings.reset();
+}
+
+std::vector<std::string> HttpsBinaryCacheStoreTest::serverArgs()
+{
+    return {
+        "s_server",
+        "-accept",
+        std::to_string(port),
+        "-cert",
+        serverCert.string(),
+        "-key",
+        serverKey.string(),
+        "-WWW", /* Serve from current directory. */
+        "-quiet",
+    };
+}
+
+std::vector<std::string> HttpsBinaryCacheStoreMtlsTest::serverArgs()
+{
+    auto args = HttpsBinaryCacheStoreTest::serverArgs();
+    /* With the -Verify option the client must supply a certificate or an error occurs, which is not the
+       case with -verify. */
+    args.insert(args.end(), {"-CAfile", caCert.string(), "-Verify", "1", "-verify_return_error"});
+    return args;
+}
+
+ref<TestHttpBinaryCacheStoreConfig> HttpsBinaryCacheStoreTest::makeConfig()
+{
+    auto res = make_ref<TestHttpBinaryCacheStoreConfig>(
+        ParsedURL{
+            .scheme = "https",
+            .authority =
+                ParsedURL::Authority{
+                    .host = "localhost",
+                    .port = port,
+                },
+        },
+        TestHttpBinaryCacheStoreConfig::Params{});
+    res->pathInfoCacheSize = 0; /* We don't want any caching in tests. */
+    return res;
+}
+
+ref<TestHttpBinaryCacheStore> HttpsBinaryCacheStoreTest::openStore(ref<TestHttpBinaryCacheStoreConfig> config)
+{
+    return config->openTestStore(ref<FileTransfer>{testFileTransfer});
+}
+
+} // namespace nix::testing

src/libstore-test-support/include/nix/store/tests/https-store.hh

@@ -0,0 +1,99 @@
+#pragma once
+///@file
+
+#include <gtest/gtest.h>
+#include <gmock/gmock.h>
+
+#include "nix/store/tests/libstore-network.hh"
+#include "nix/store/http-binary-cache-store.hh"
+#include "nix/store/store-api.hh"
+#include "nix/store/globals.hh"
+#include "nix/store/local-binary-cache-store.hh"
+#include "nix/util/file-system.hh"
+#include "nix/util/processes.hh"
+
+namespace nix::testing {
+
+class TestHttpBinaryCacheStoreConfig;
+
+/**
+ * Test shim for testing. We don't want to use the on-disk narinfo cache in unit
+ * tests.
+ */
+class TestHttpBinaryCacheStore : public HttpBinaryCacheStore
+{
+public:
+    TestHttpBinaryCacheStore(const TestHttpBinaryCacheStore &) = delete;
+    TestHttpBinaryCacheStore(TestHttpBinaryCacheStore &&) = delete;
+    TestHttpBinaryCacheStore & operator=(const TestHttpBinaryCacheStore &) = delete;
+    TestHttpBinaryCacheStore & operator=(TestHttpBinaryCacheStore &&) = delete;
+
+    TestHttpBinaryCacheStore(ref<HttpBinaryCacheStoreConfig> config, ref<FileTransfer> fileTransfer)
+        : Store{*config}
+        , BinaryCacheStore{*config}
+        , HttpBinaryCacheStore(config, fileTransfer)
+    {
+        diskCache = nullptr; /* Disable caching, we'll be creating a new binary cache for each test. */
+    }
+
+    void init() override;
+};
+
+class TestHttpBinaryCacheStoreConfig : public HttpBinaryCacheStoreConfig
+{
+public:
+    TestHttpBinaryCacheStoreConfig(ParsedURL url, const Store::Config::Params & params)
+        : StoreConfig(params)
+        , HttpBinaryCacheStoreConfig(url, params)
+    {
+    }
+
+    ref<TestHttpBinaryCacheStore> openTestStore(ref<FileTransfer> fileTransfer) const;
+};
+
+class HttpsBinaryCacheStoreTest : public virtual LibStoreNetworkTest
+{
+    std::unique_ptr<AutoDelete> delTmpDir;
+
+public:
+    static void SetUpTestSuite()
+    {
+        initLibStore(/*loadConfig=*/false);
+    }
+
+protected:
+    std::filesystem::path tmpDir, cacheDir;
+    std::filesystem::path caCert, caKey, serverCert, serverKey;
+    std::filesystem::path clientCert, clientKey;
+    Pid serverPid;
+    uint16_t port = 8443;
+    std::shared_ptr<Store> localCacheStore;
+
+    /**
+     * Custom FileTransferSettings with the test CA certificate.
+     * This is used instead of modifying global settings.
+     */
+    std::unique_ptr<FileTransferSettings> testFileTransferSettings;
+
+    /**
+     * FileTransfer instance using our test settings.
+     * Initialized in SetUp().
+     */
+    std::shared_ptr<FileTransfer> testFileTransfer;
+
+    static void openssl(Strings args);
+    void SetUp() override;
+    void TearDown() override;
+
+    virtual std::vector<std::string> serverArgs();
+    ref<TestHttpBinaryCacheStoreConfig> makeConfig();
+    ref<TestHttpBinaryCacheStore> openStore(ref<TestHttpBinaryCacheStoreConfig> config);
+};
+
+class HttpsBinaryCacheStoreMtlsTest : public HttpsBinaryCacheStoreTest
+{
+protected:
+    std::vector<std::string> serverArgs() override;
+};
+
+} // namespace nix::testing

src/libstore-test-support/include/nix/store/tests/libstore-network.hh

@@ -0,0 +1,39 @@
+#pragma once
+/// @file
+
+#include <gtest/gtest.h>
+
+namespace nix::testing {
+
+/**
+ * Whether to run network tests. This is global so that the test harness can
+ * enable this by default if we can run tests in isolation.
+ */
+extern bool networkTestsAvailable;
+
+/**
+ * Set up network tests and, if on linux, create a new network namespace for
+ * tests with a loopback interface. This is to avoid binding to ports in the
+ * host's namespace.
+ */
+void setupNetworkTests();
+
+class LibStoreNetworkTest : public virtual ::testing::Test
+{
+protected:
+    void SetUp() override
+    {
+        if (networkTestsAvailable)
+            return;
+        static bool warned = false;
+        if (!warned) {
+            warned = true;
+            GTEST_SKIP()
+                << "Network tests not enabled by default without user namespaces, use NIX_TEST_FORCE_NETWORK_TESTS=1 to override";
+        } else {
+            GTEST_SKIP();
+        }
+    }
+};
+
+} // namespace nix::testing

src/libstore-test-support/include/nix/store/tests/meson.build

@@ -4,6 +4,8 @@ include_dirs = [ include_directories('../../..') ]
 
 headers = files(
   'derived-path.hh',
+  'https-store.hh',
+  'libstore-network.hh',
   'libstore.hh',
   'nix_api_store.hh',
   'outputs-spec.hh',

src/libstore-test-support/include/nix/store/tests/protocol.hh

@@ -100,26 +100,26 @@ public:
         writeProtoTest(STEM, VERSION, VALUE);                                              \
     }
 
-#define VERSIONED_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE)  \
-    VERSIONED_READ_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE) \
-    VERSIONED_WRITE_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE)
+#define VERSIONED_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE)    \
+    VERSIONED_READ_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, (VERSION), VALUE) \
+    VERSIONED_WRITE_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, (VERSION), VALUE)
 
-#define VERSIONED_READ_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE)     \
-    VERSIONED_READ_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE) \
-    TEST_F(FIXTURE, NAME##_json_read)                                                 \
-    {                                                                                 \
-        readJsonTest(STEM, VALUE);                                                    \
+#define VERSIONED_READ_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE)       \
+    VERSIONED_READ_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, (VERSION), VALUE) \
+    TEST_F(FIXTURE, NAME##_json_read)                                                   \
+    {                                                                                   \
+        readJsonTest(STEM, VALUE);                                                      \
     }
 
-#define VERSIONED_WRITE_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE)     \
-    VERSIONED_WRITE_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE) \
-    TEST_F(FIXTURE, NAME##_json_write)                                                 \
-    {                                                                                  \
-        writeJsonTest(STEM, VALUE);                                                    \
+#define VERSIONED_WRITE_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE)       \
+    VERSIONED_WRITE_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, (VERSION), VALUE) \
+    TEST_F(FIXTURE, NAME##_json_write)                                                   \
+    {                                                                                    \
+        writeJsonTest(STEM, VALUE);                                                      \
     }
 
-#define VERSIONED_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE)  \
-    VERSIONED_READ_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE) \
-    VERSIONED_WRITE_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE)
+#define VERSIONED_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE)    \
+    VERSIONED_READ_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, (VERSION), VALUE) \
+    VERSIONED_WRITE_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, (VERSION), VALUE)
 
 } // namespace nix

src/libstore-test-support/libstore-network.cc

@@ -0,0 +1,60 @@
+#include "nix/store/tests/libstore-network.hh"
+#include "nix/util/error.hh"
+#include "nix/util/environment-variables.hh"
+
+#ifdef __linux__
+#  include "nix/util/file-system.hh"
+#  include "nix/util/linux-namespaces.hh"
+#  include <sched.h>
+#  include <sys/ioctl.h>
+#  include <net/if.h>
+#  include <netinet/in.h>
+#endif
+
+namespace nix::testing {
+
+bool networkTestsAvailable = false;
+
+#ifdef __linux__
+
+static void enterNetworkNamespace()
+{
+    auto uid = ::getuid();
+    auto gid = ::getgid();
+
+    if (::unshare(CLONE_NEWUSER | CLONE_NEWNET) == -1)
+        throw SysError("setting up a private network namespace for tests");
+
+    std::filesystem::path procSelf = "/proc/self";
+    writeFile(procSelf / "setgroups", "deny");
+    writeFile(procSelf / "uid_map", fmt("%d %d 1", uid, uid));
+    writeFile(procSelf / "gid_map", fmt("%d %d 1", gid, gid));
+
+    AutoCloseFD fd(::socket(PF_INET, SOCK_DGRAM, IPPROTO_IP));
+    if (!fd)
+        throw SysError("cannot open IP socket for loopback interface");
+
+    struct ::ifreq ifr = {};
+    strcpy(ifr.ifr_name, "lo");
+    ifr.ifr_flags = IFF_UP | IFF_LOOPBACK | IFF_RUNNING;
+    if (::ioctl(fd.get(), SIOCSIFFLAGS, &ifr) == -1)
+        throw SysError("cannot set loopback interface flags");
+}
+
+#endif
+
+void setupNetworkTests()
+try {
+    networkTestsAvailable = getEnvOs(OS_STR("NIX_TEST_FORCE_NETWORK_TESTS")).has_value();
+
+#ifdef __linux__
+    if (!networkTestsAvailable && userNamespacesSupported()) {
+        enterNetworkNamespace();
+        networkTestsAvailable = true;
+    }
+#endif
+} catch (SystemError & e) {
+    /* Ignore any set up errors. */
+}
+
+} // namespace nix::testing

src/libstore-test-support/meson.build

@@ -28,10 +28,15 @@ subdir('nix-meson-build-support/subprojects')
 rapidcheck = dependency('rapidcheck')
 deps_public += rapidcheck
 
+gtest = dependency('gtest')
+deps_public += gtest
+
 subdir('nix-meson-build-support/common')
 
 sources = files(
   'derived-path.cc',
+  'https-store.cc',
+  'libstore-network.cc',
   'outputs-spec.cc',
   'path.cc',
   'test-main.cc',

src/libstore-test-support/package.nix

@@ -7,6 +7,7 @@
   nix-store-c,
 
   rapidcheck,
+  gtest,
 
   # Configuration Options
 
@@ -39,6 +40,7 @@ mkMesonLibrary (finalAttrs: {
     nix-store
     nix-store-c
     rapidcheck
+    gtest
   ];
 
   mesonFlags = [

src/libstore-test-support/test-main.cc

@@ -15,10 +15,10 @@ int testMainForBuidingPre(int argc, char ** argv)
     }
 
     // Disable build hook. We won't be testing remote builds in these unit tests. If we do, fix the above build hook.
-    settings.buildHook = {};
+    settings.getWorkerSettings().buildHook = {};
 
     // No substituters, unless a test specifically requests.
-    settings.substituters = {};
+    settings.getWorkerSettings().substituters = {};
 
 #ifdef __linux__ // should match the conditional around sandboxBuildDir declaration.
 
@@ -31,13 +31,13 @@ int testMainForBuidingPre(int argc, char ** argv)
     //   sandboxBuildDir = /build
     // However, we have a rule that the store dir must not be inside the storeDir, so we need to pick a different
     // sandboxBuildDir.
-    settings.sandboxBuildDir = "/test-build-dir-instead-of-usual-build-dir";
+    settings.getLocalSettings().sandboxBuildDir = "/test-build-dir-instead-of-usual-build-dir";
 #endif
 
 #ifdef __APPLE__
     // Avoid this error, when already running in a sandbox:
     // sandbox-exec: sandbox_apply: Operation not permitted
-    settings.sandboxMode = smDisabled;
+    settings.getLocalSettings().sandboxMode = smDisabled;
     setEnv("_NIX_TEST_NO_SANDBOX", "1");
 #endif
 

src/libstore-tests/build-result.cc

@@ -1,6 +1,7 @@
 #include <gtest/gtest.h>
 
 #include "nix/store/build-result.hh"
+#include "nix/util/tests/characterization.hh"
 #include "nix/util/tests/json-characterization.hh"
 
 namespace nix {
@@ -44,22 +45,22 @@ INSTANTIATE_TEST_SUITE_P(
         std::pair{
             "not-deterministic",
             BuildResult{
-                .inner{BuildResult::Failure{
+                .inner{BuildResult::Failure{{
                     .status = BuildResult::Failure::NotDeterministic,
-                    .errorMsg = "no idea why",
+                    .msg = HintFmt("no idea why"),
                     .isNonDeterministic = false, // Note: This field is separate from the status
-                }},
+                }}},
                 .timesBuilt = 1,
             },
         },
         std::pair{
             "output-rejected",
             BuildResult{
-                .inner{BuildResult::Failure{
+                .inner{BuildResult::Failure{{
                     .status = BuildResult::Failure::OutputRejected,
-                    .errorMsg = "no idea why",
+                    .msg = HintFmt("no idea why"),
                     .isNonDeterministic = false,
-                }},
+                }}},
                 .timesBuilt = 3,
                 .startTime = 30,
                 .stopTime = 50,