Merge pull request #14354 from NixOS/backport-14343-to-2.32-maintenance

[Backport 2.32-maintenance] Revert "libmain: Catch logger exceptions in `handleExceptions`"

.github/PULL_REQUEST_TEMPLATE.md

@@ -15,6 +15,10 @@ so you understand the process and the expectations.
 - volunteering contributions effectively
 - how to get help and our review process.
 
+PR stuck in review? We have two Nix team meetings per week online that are open for everyone in a jitsi conference:
+
+- https://calendar.google.com/calendar/u/0/embed?src=b9o52fobqjak8oq8lfkhg3t0qg@group.calendar.google.com
+
 -->
 
 ## Motivation

.github/workflows/ci.yml

@@ -29,7 +29,32 @@ jobs:
         github_token: ${{ secrets.GITHUB_TOKEN }}
     - run: nix flake show --all-systems --json
 
+  pre-commit-checks:
+    name: pre-commit checks
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@v5
+      - uses: ./.github/actions/install-nix-action
+        with:
+          dogfood: ${{ github.event_name == 'workflow_dispatch' && inputs.dogfood || github.event_name != 'workflow_dispatch' }}
+          extra_nix_config: experimental-features = nix-command flakes
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+      - uses: DeterminateSystems/magic-nix-cache-action@main
+      - run: ./ci/gha/tests/pre-commit-checks
+
+  basic-checks:
+    name: aggregate basic checks
+    if: ${{ always() }}
+    runs-on: ubuntu-24.04
+    needs: [pre-commit-checks, eval]
+    steps:
+      - name: Exit with any errors
+        if: ${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }}
+        run: |
+          exit 1
+
   tests:
+    needs: basic-checks
     strategy:
       fail-fast: false
       matrix:
@@ -214,6 +239,7 @@ jobs:
         docker push $IMAGE_ID:master
 
   vm_tests:
+    needs: basic-checks
     runs-on: ubuntu-24.04
     steps:
       - uses: actions/checkout@v5

.version

@@ -1 +1 @@
-2.32.0
+2.32.2

COPYING

@@ -1,8 +1,8 @@
-		  GNU LESSER GENERAL PUBLIC LICENSE
-		       Version 2.1, February 1999
+                  GNU LESSER GENERAL PUBLIC LICENSE
+                       Version 2.1, February 1999
 
  Copyright (C) 1991, 1999 Free Software Foundation, Inc.
- 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ <https://fsf.org/>
  Everyone is permitted to copy and distribute verbatim copies
  of this license document, but changing it is not allowed.
 
@@ -10,7 +10,7 @@
  as the successor of the GNU Library Public License, version 2, hence
  the version number 2.1.]
 
-			    Preamble
+                            Preamble
 
   The licenses for most software are designed to take away your
 freedom to share and change it.  By contrast, the GNU General Public
@@ -112,7 +112,7 @@ modification follow.  Pay close attention to the difference between a
 former contains code derived from the library, whereas the latter must
 be combined with the library in order to run.
 
-		  GNU LESSER GENERAL PUBLIC LICENSE
+                  GNU LESSER GENERAL PUBLIC LICENSE
    TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
 
   0. This License Agreement applies to any software library or other
@@ -146,7 +146,7 @@ such a program is covered only if its contents constitute a work based
 on the Library (independent of the use of the Library in a tool for
 writing it).  Whether that is true depends on what the Library does
 and what the program that uses the Library does.
-  
+
   1. You may copy and distribute verbatim copies of the Library's
 complete source code as you receive it, in any medium, provided that
 you conspicuously and appropriately publish on each copy an
@@ -432,7 +432,7 @@ decision will be guided by the two goals of preserving the free status
 of all derivatives of our free software and of promoting the sharing
 and reuse of software generally.
 
-			    NO WARRANTY
+                            NO WARRANTY
 
   15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO
 WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW.
@@ -455,7 +455,7 @@ FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF
 SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH
 DAMAGES.
 
-		     END OF TERMS AND CONDITIONS
+                     END OF TERMS AND CONDITIONS
 
            How to Apply These Terms to Your New Libraries
 
@@ -484,8 +484,7 @@ convey the exclusion of warranty; and each file should have at least the
     Lesser General Public License for more details.
 
     You should have received a copy of the GNU Lesser General Public
-    License along with this library; if not, write to the Free Software
-    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+    License along with this library; if not, see <https://www.gnu.org/licenses/>.
 
 Also add information on how to contact you by electronic and paper mail.
 
@@ -496,9 +495,7 @@ necessary.  Here is a sample; alter the names:
   Yoyodyne, Inc., hereby disclaims all copyright interest in the
   library `Frob' (a library for tweaking knobs) written by James Random Hacker.
 
-  <signature of Ty Coon>, 1 April 1990
-  Ty Coon, President of Vice
+  <signature of Moe Ghoul>, 1 April 1990
+  Moe Ghoul, President of Vice
 
 That's all there is to it!
-
-

ci/gha/tests/default.nix

@@ -24,16 +24,7 @@ let
   enableSanitizersLayer = finalAttrs: prevAttrs: {
     mesonFlags =
       (prevAttrs.mesonFlags or [ ])
-      ++ [
-        # Run all tests with UBSAN enabled. Running both with ubsan and
-        # without doesn't seem to have much immediate benefit for doubling
-        # the GHA CI workaround.
-        #
-        # TODO: Work toward enabling "address,undefined" if it seems feasible.
-        # This would maybe require dropping Boost coroutines and ignoring intentional
-        # memory leaks with detect_leaks=0.
-        (lib.mesonOption "b_sanitize" "undefined")
-      ]
+      ++ [ (lib.mesonOption "b_sanitize" "address,undefined") ]
       ++ (lib.optionals stdenv.cc.isClang [
         # https://www.github.com/mesonbuild/meson/issues/764
         (lib.mesonBool "b_lundef" false)
@@ -71,8 +62,12 @@ rec {
   nixComponentsInstrumented = nixComponents.overrideScope (
     final: prev: {
       nix-store-tests = prev.nix-store-tests.override { withBenchmarks = true; };
+      # Boehm is incompatible with ASAN.
+      nix-expr = prev.nix-expr.override { enableGC = !withSanitizers; };
 
       mesonComponentOverrides = lib.composeManyExtensions componentOverrides;
+      # Unclear how to make Perl bindings work with a dynamically linked ASAN.
+      nix-perl-bindings = if withSanitizers then null else prev.nix-perl-bindings;
     }
   );
 

ci/gha/tests/pre-commit-checks

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+system=$(nix eval --raw --impure --expr builtins.currentSystem)
+
+echo "::group::Running pre-commit checks"
+
+if nix build ".#checks.$system.pre-commit" -L; then
+    echo "::endgroup::"
+    exit 0
+fi
+
+echo "::error ::Changes do not pass pre-commit checks"
+
+cat <<EOF
+The code isn't formatted or doesn't pass lints. You can run pre-commit locally with:
+
+    nix develop -c ./maintainers/format.sh
+EOF
+
+echo "::endgroup::"
+
+exit 1

doc/manual/meson.build

@@ -15,6 +15,7 @@ pymod = import('python')
 python = pymod.find_installation('python3')
 
 nix_env_for_docs = {
+  'ASAN_OPTIONS' : 'abort_on_error=1:print_summary=1:detect_leaks=0',
   'HOME' : '/dummy',
   'NIX_CONF_DIR' : '/dummy',
   'NIX_SSL_CERT_FILE' : '/dummy/no-ca-bundle.crt',

doc/manual/rl-next/c-api-recoverable-errors.md

@@ -1,23 +0,0 @@
----
-synopsis: "C API: Errors returned from your primops are not treated as recoverable by default"
-prs: [13930]
----
-
-Nix 2.32 by default remembers the error in the thunk that triggered it.
-
-Previously the following sequence of events worked:
-
-1. Have a thunk that invokes a primop that's defined through the C API
-2. The primop returns an error
-3. Force the thunk again
-4. The primop returns a value
-5. The thunk evaluated successfully
-
-**Resolution**
-
-C API consumers that rely on this must change their recoverable error calls:
-
-```diff
--nix_set_err_msg(context, NIX_ERR_*, msg);
-+nix_set_err_msg(context, NIX_ERR_RECOVERABLE, msg);
-```

doc/manual/rl-next/dropped-compat.md

@@ -1,6 +0,0 @@
----
-synopsis: "Removed support for daemons and clients older than Nix 2.0"
-prs: [13951]
----
-
-We have dropped support in the daemon worker protocol for daemons and clients that don't speak at least version 18 of the protocol. This first Nix release that supports this version is Nix 2.0, released in February 2018.

doc/manual/rl-next/shorter-build-dir-names.md

@@ -1,6 +0,0 @@
----
-synopsis: "Temporary build directories no longer include derivation names"
-prs: [13839]
----
-
-Temporary build directories created during derivation builds no longer include the derivation name in their path to avoid build failures when the derivation name is too long. This change ensures predictable prefix lengths for build directories under `/nix/var/nix/builds`.

doc/manual/source/SUMMARY.md.in

@@ -138,6 +138,7 @@
   - [Contributing](development/contributing.md)
 - [Releases](release-notes/index.md)
 {{#include ./SUMMARY-rl-next.md}}
+  - [Release 2.32 (2025-10-06)](release-notes/rl-2.32.md)
   - [Release 2.31 (2025-08-21)](release-notes/rl-2.31.md)
   - [Release 2.30 (2025-07-07)](release-notes/rl-2.30.md)
   - [Release 2.29 (2025-05-14)](release-notes/rl-2.29.md)

doc/manual/source/command-ref/meson.build

@@ -2,6 +2,7 @@ xp_features_json = custom_target(
   command : [ nix, '__dump-xp-features' ],
   capture : true,
   output : 'xp-features.json',
+  env : nix_env_for_docs,
 )
 
 experimental_features_shortlist_md = custom_target(

doc/manual/source/development/building.md

@@ -23,7 +23,7 @@ $ nix-shell
 To get a shell with one of the other [supported compilation environments](#compilation-environments):
 
 ```console
-$ nix-shell --attr devShells.x86_64-linux.native-clangStdenvPackages
+$ nix-shell --attr devShells.x86_64-linux.native-clangStdenv
 ```
 
 > **Note**

doc/manual/source/development/debugging.md

@@ -24,6 +24,19 @@ It is also possible to build without debugging for faster build:
 
 (The first line is needed because `fortify` hardening requires at least some optimization.)
 
+## Building Nix with sanitizers
+
+Nix can be built with [Address](https://clang.llvm.org/docs/AddressSanitizer.html) and
+[UB](https://clang.llvm.org/docs/UndefinedBehaviorSanitizer.html) sanitizers using LLVM
+or GCC. This is useful when debugging memory corruption issues.
+
+```console
+[nix-shell]$ export mesonBuildType=debugoptimized
+[nix-shell]$ appendToVar mesonFlags "-Dlibexpr:gc=disabled" # Disable Boehm
+[nix-shell]$ appendToVar mesonFlags "-Dbindings=false" # Disable nix-perl
+[nix-shell]$ appendToVar mesonFlags "-Db_sanitize=address,undefined"
+```
+
 ## Debugging the Nix Binary
 
 Obtain your preferred debugger within the development shell:

doc/manual/source/development/meson.build

@@ -7,5 +7,6 @@ experimental_feature_descriptions_md = custom_target(
     xp_features_json,
   ],
   capture : true,
+  env : nix_env_for_docs,
   output : 'experimental-feature-descriptions.md',
 )

doc/manual/source/language/builtins-prefix.md

@@ -5,12 +5,28 @@ All built-ins are available through the global [`builtins`](#builtins-builtins)
 
 Some built-ins are also exposed directly in the global scope:
 
-<!-- TODO(@rhendric, #10970): this list is incomplete -->
-
 - [`derivation`](#builtins-derivation)
-- [`import`](#builtins-import)
+- `derivationStrict`
 - [`abort`](#builtins-abort)
+- [`baseNameOf`](#builtins-baseNameOf)
+- [`break`](#builtins-break)
+- [`dirOf`](#builtins-dirOf)
+- [`false`](#builtins-false)
+- [`fetchGit`](#builtins-fetchGit)
+- `fetchMercurial`
+- [`fetchTarball`](#builtins-fetchTarball)
+- [`fetchTree`](#builtins-fetchTree)
+- [`fromTOML`](#builtins-fromTOML)
+- [`import`](#builtins-import)
+- [`isNull`](#builtins-isNull)
+- [`map`](#builtins-map)
+- [`null`](#builtins-null)
+- [`placeholder`](#builtins-placeholder)
+- [`removeAttrs`](#builtins-removeAttrs)
+- `scopedImport`
 - [`throw`](#builtins-throw)
+- [`toString`](#builtins-toString)
+- [`true`](#builtins-true)
 
 <dl>
   <dt id="builtins-derivation"><a href="#builtins-derivation"><code>derivation <var>attrs</var></code></a></dt>

doc/manual/source/protocols/json/derivation.md

@@ -14,6 +14,21 @@ is a JSON object with the following fields:
   The name of the derivation.
   This is used when calculating the store paths of the derivation's outputs.
 
+* `version`:
+  Must be `3`.
+  This is a guard that allows us to continue evolving this format.
+  The choice of `3` is fairly arbitrary, but corresponds to this informal version:
+
+  - Version 0: A-Term format
+
+  - Version 1: Original JSON format, with ugly `"r:sha256"` inherited from A-Term format.
+
+  - Version 2: Separate `method` and `hashAlgo` fields in output specs
+
+  - Verison 3: Drop store dir from store paths, just include base name.
+
+  Note that while this format is experimental, the maintenance of versions is best-effort, and not promised to identify every change.
+
 * `outputs`:
   Information about the output paths of the derivation.
   This is a JSON object with one member per output, where the key is the output name and the value is a JSON object with these fields:
@@ -52,7 +67,6 @@ is a JSON object with the following fields:
   > ```json
   > "outputs": {
   >   "out": {
-  >     "path": "/nix/store/2543j7c6jn75blc3drf4g5vhb1rhdq29-source",
   >     "method": "nar",
   >     "hashAlgo": "sha256",
   >     "hash": "6fc80dcc62179dbc12fc0b5881275898f93444833d21b89dfe5f7fbcbb1d0d62"
@@ -63,6 +77,15 @@ is a JSON object with the following fields:
 * `inputSrcs`:
   A list of store paths on which this derivation depends.
 
+  > **Example**
+  >
+  > ```json
+  > "inputSrcs": [
+  >   "47y241wqdhac3jm5l7nv0x4975mb1975-separate-debug-info.sh",
+  >   "56d0w71pjj9bdr363ym3wj1zkwyqq97j-fix-pop-var-context-error.patch"
+  > ]
+  > ```
+
 * `inputDrvs`:
   A JSON object specifying the derivations on which this derivation depends, and what outputs of those derivations.
 
@@ -70,8 +93,8 @@ is a JSON object with the following fields:
   >
   > ```json
   > "inputDrvs": {
-  >   "/nix/store/6lkh5yi7nlb7l6dr8fljlli5zfd9hq58-curl-7.73.0.drv": ["dev"],
-  >   "/nix/store/fn3kgnfzl5dzym26j8g907gq3kbm8bfh-unzip-6.0.drv": ["out"]
+  >   "6lkh5yi7nlb7l6dr8fljlli5zfd9hq58-curl-7.73.0.drv": ["dev"],
+  >   "fn3kgnfzl5dzym26j8g907gq3kbm8bfh-unzip-6.0.drv": ["out"]
   > }
   > ```
 

doc/manual/source/release-notes/rl-2.32.md

@@ -0,0 +1,130 @@
+# Release 2.32.0 (2025-10-06)
+
+## Incompatible changes
+
+- Removed support for daemons and clients older than Nix 2.0 [#13951](https://github.com/NixOS/nix/pull/13951)
+
+  We have dropped support in the daemon worker protocol for daemons and clients that don't speak at least version 18 of the protocol. This first Nix release that supports this version is Nix 2.0, released in February 2018.
+
+- Derivation JSON format now uses store path basenames only [#13570](https://github.com/NixOS/nix/issues/13570) [#13980](https://github.com/NixOS/nix/pull/13980)
+
+  Experience with many JSON frameworks (e.g. nlohmann/json in C++, Serde in Rust, and Aeson in Haskell) has shown that the use of the store directory in JSON formats is an impediment to systematic JSON formats, because it requires the serializer/deserializer to take an extra paramater (the store directory).
+
+  We ultimately want to rectify this issue with all JSON formats to the extent allowed by our stability promises. To start with, we are changing the JSON format for derivations because the `nix derivation` commands are — in addition to being formally unstable — less widely used than other unstable commands.
+
+  See the documentation on the [JSON format for derivations](@docroot@/protocols/json/derivation.md) for further details.
+
+- C API: `nix_get_attr_name_byidx`, `nix_get_attr_byidx` take a `nix_value *` instead of `const nix_value *` [#13987](https://github.com/NixOS/nix/pull/13987)
+
+  In order to accommodate a more optimized internal representation of attribute set merges these functions require
+  a mutable `nix_value *` that might be modified on access. This does *not* break the ABI of these functions.
+
+## New features
+
+- C API: Add lazy attribute and list item accessors [#14030](https://github.com/NixOS/nix/pull/14030)
+
+  The C API now includes lazy accessor functions for retrieving values from lists and attribute sets without forcing evaluation:
+
+  - `nix_get_list_byidx_lazy()` - Get a list element without forcing its evaluation
+  - `nix_get_attr_byname_lazy()` - Get an attribute value by name without forcing evaluation
+  - `nix_get_attr_byidx_lazy()` - Get an attribute by index without forcing evaluation
+
+  These functions are useful when forwarding unevaluated sub-values to other lists, attribute sets, or function calls. They allow more efficient handling of Nix values by deferring evaluation until actually needed.
+
+  Additionally, bounds checking has been improved for all `_byidx` functions to properly validate indices before access, preventing potential out-of-bounds errors.
+
+  The documentation for `NIX_ERR_KEY` error handling has also been clarified to specify when this error code is returned.
+
+- HTTP binary caches now support transparent compression for metadata
+
+  HTTP binary cache stores can now compress `.narinfo`, `.ls`, and build log files before uploading them,
+  reducing bandwidth usage and storage requirements. The compression is applied transparently using the
+  `Content-Encoding` header, allowing compatible clients to automatically decompress the files.
+
+  Three new configuration options control this behavior:
+  - `narinfo-compression`: Compression method for `.narinfo` files
+  - `ls-compression`: Compression method for `.ls` files
+  - `log-compression`: Compression method for build logs in `log/` directory
+
+  Example usage:
+  ```
+  nix copy --to 'http://cache.example.com?narinfo-compression=gzip&ls-compression=gzip' /nix/store/...
+  nix store copy-log --to 'http://cache.example.com?log-compression=br' /nix/store/...
+  ```
+
+- Temporary build directories no longer include derivation names [#13839](https://github.com/NixOS/nix/pull/13839)
+
+  Temporary build directories created during derivation builds no longer include the derivation name in their path to avoid build failures when the derivation name is too long. This change ensures predictable prefix lengths for build directories under `/nix/var/nix/builds`.
+
+- External derivation builders [#14145](https://github.com/NixOS/nix/pull/14145)
+
+  These are helper programs that Nix calls to perform derivations for specified system types, e.g. by using QEMU to emulate a different type of platform. For more information, see the [`external-builders` setting](../command-ref/conf-file.md#conf-external-builders).
+
+  This is currently an experimental feature.
+
+## Performance improvements
+
+- Optimize memory usage of attribute set merges [#13987](https://github.com/NixOS/nix/pull/13987)
+
+  [Attribute set update operations](@docroot@/language/operators.md#update) have been optimized to
+  reduce reallocations in cases when the second operand is small.
+
+  For typical evaluations of nixpkgs this optimization leads to ~20% less memory allocated in total
+  without significantly affecting evaluation performance.
+
+  See [eval-attrset-update-layer-rhs-threshold](@docroot@/command-ref/conf-file.md#conf-eval-attrset-update-layer-rhs-threshold)
+
+- Substituted flake inputs are no longer re-copied to the store [#14041](https://github.com/NixOS/nix/pull/14041)
+
+  Since 2.25, Nix would fail to store a cache entry for substituted flake inputs, which in turn would cause them to be re-copied to the store on initial evaluation. Caching these inputs results in a near doubling of performance in some cases — especially on I/O-bound machines and when using commands that fetch many inputs, like `nix flake [archive|prefetch-inputs]`.
+
+- `nix flake check` now skips derivations that can be substituted [#13574](https://github.com/NixOS/nix/pull/13574)
+
+  Previously, `nix flake check` would evaluate and build/substitute all
+  derivations. Now, it will skip downloading derivations that can be substituted.
+  This can drastically decrease the time invocations take in environments where
+  checks may already be cached (like in CI).
+
+- `fetchTarball` and `fetchurl` now correctly substitute (#14138)
+
+  At some point we stopped substituting calls to `fetchTarball` and `fetchurl` with a set `narHash` to avoid incorrectly substituting things in `fetchTree`, even though it would be safe to substitute when calling the legacy `fetch{Tarball,url}`. This fixes that regression where it is safe.
+- Started moving AST allocations into a bump allocator [#14088](https://github.com/NixOS/nix/issues/14088)
+
+  This leaves smaller, immutable structures in the AST. So far this saves about 2% memory on a NixOS config evaluation.
+## Contributors
+
+This release was made possible by the following 32 contributors:
+
+- Farid Zakaria [**(@fzakaria)**](https://github.com/fzakaria)
+- dram [**(@dramforever)**](https://github.com/dramforever)
+- Ephraim Siegfried [**(@EphraimSiegfried)**](https://github.com/EphraimSiegfried)
+- Robert Hensing [**(@roberth)**](https://github.com/roberth)
+- Taeer Bar-Yam [**(@Radvendii)**](https://github.com/Radvendii)
+- Emily [**(@emilazy)**](https://github.com/emilazy)
+- Jens Petersen [**(@juhp)**](https://github.com/juhp)
+- Bernardo Meurer [**(@lovesegfault)**](https://github.com/lovesegfault)
+- Jörg Thalheim [**(@Mic92)**](https://github.com/Mic92)
+- Leandro Emmanuel Reina Kiperman [**(@kip93)**](https://github.com/kip93)
+- Marie [**(@NyCodeGHG)**](https://github.com/NyCodeGHG)
+- Ethan Evans [**(@ethanavatar)**](https://github.com/ethanavatar)
+- Yaroslav Bolyukin [**(@CertainLach)**](https://github.com/CertainLach)
+- Matej Urbas [**(@urbas)**](https://github.com/urbas)
+- Jami Kettunen [**(@JamiKettunen)**](https://github.com/JamiKettunen)
+- Clayton [**(@netadr)**](https://github.com/netadr)
+- Grégory Marti [**(@gmarti)**](https://github.com/gmarti)
+- Eelco Dolstra [**(@edolstra)**](https://github.com/edolstra)
+- rszyma [**(@rszyma)**](https://github.com/rszyma)
+- Philip Wilk [**(@philipwilk)**](https://github.com/philipwilk)
+- John Ericson [**(@Ericson2314)**](https://github.com/Ericson2314)
+- Tom Westerhout [**(@twesterhout)**](https://github.com/twesterhout)
+- Tristan Ross [**(@RossComputerGuy)**](https://github.com/RossComputerGuy)
+- Sergei Zimmerman [**(@xokdvium)**](https://github.com/xokdvium)
+- Jean-François Roche [**(@jfroche)**](https://github.com/jfroche)
+- Seth Flynn [**(@getchoo)**](https://github.com/getchoo)
+- éclairevoyant [**(@eclairevoyant)**](https://github.com/eclairevoyant)
+- Glen Huang [**(@hgl)**](https://github.com/hgl)
+- osman - オスマン [**(@osbm)**](https://github.com/osbm)
+- David McFarland [**(@corngood)**](https://github.com/corngood)
+- Cole Helbling [**(@cole-h)**](https://github.com/cole-h)
+- Sinan Mohd [**(@sinanmohd)**](https://github.com/sinanmohd)
+- Philipp Otterbein

doc/manual/source/store/store-object.md

@@ -20,7 +20,8 @@ The graph of references excluding self-references thus forms a [directed acyclic
 
 [directed acyclic graph]: @docroot@/glossary.md#gloss-directed-acyclic-graph
 
-We can take the [transitive closure] of the references graph, which any pair of store objects have an edge not if there is a single reference from the first to the second, but a path of one or more references from the first to the second.
+We can take the [transitive closure] of the references graph, in which any pair of store objects have an edge if a *path* of one or more references exists from the first to the second object.
+(A single reference always forms a path which is one reference long, but longer paths may connect objects which have no direct reference between them.)
 The *requisites* of a store object are all store objects reachable by paths of references which start with given store object's references.
 
 [transitive closure]: https://en.wikipedia.org/wiki/Transitive_closure

flake.nix

@@ -32,7 +32,7 @@
     let
       inherit (nixpkgs) lib;
 
-      officialRelease = false;
+      officialRelease = true;
 
       linux32BitSystems = [ "i686-linux" ];
       linux64BitSystems = [

maintainers/data/release-credits-email-to-handle.json

@@ -203,5 +203,26 @@
     "ConnorBaker01@Gmail.com": "ConnorBaker",
     "jsoo1@asu.edu": "jsoo1",
     "hsngrmpf+github@gmail.com": "DavHau",
-    "matthew@floxdev.com": "mkenigs"
+    "matthew@floxdev.com": "mkenigs",
+    "taeer@bar-yam.me": "Radvendii",
+    "beme@anthropic.com": "lovesegfault",
+    "osbm@osbm.dev": "osbm",
+    "jami.kettunen@protonmail.com": "JamiKettunen",
+    "ephraim.siegfried@hotmail.com": "EphraimSiegfried",
+    "rszyma.dev@gmail.com": "rszyma",
+    "tristan.ross@determinate.systems": "RossComputerGuy",
+    "corngood@gmail.com": "corngood",
+    "jfroche@pyxel.be": "jfroche",
+    "848000+eclairevoyant@users.noreply.github.com": "eclairevoyant",
+    "petersen@redhat.com": "juhp",
+    "dramforever@live.com": "dramforever",
+    "me@glenhuang.com": "hgl",
+    "philip.wilk@fivium.co.uk": "philipwilk",
+    "me@nycode.dev": "NyCodeGHG",
+    "14264576+twesterhout@users.noreply.github.com": "twesterhout",
+    "sinan@sinanmohd.com": "sinanmohd",
+    "42688647+netadr@users.noreply.github.com": "netadr",
+    "matej.urbas@gmail.com": "urbas",
+    "ethanalexevans@gmail.com": "ethanavatar",
+    "greg.marti@gmail.com": "gmarti"
 }

maintainers/data/release-credits-handle-to-name.json

@@ -177,5 +177,24 @@
     "avnik": "Alexander V. Nikolaev",
     "DavHau": null,
     "aln730": "AGawas",
-    "vog": "Volker Diels-Grabsch"
+    "vog": "Volker Diels-Grabsch",
+    "corngood": "David McFarland",
+    "twesterhout": "Tom Westerhout",
+    "JamiKettunen": "Jami Kettunen",
+    "dramforever": "dram",
+    "philipwilk": "Philip Wilk",
+    "netadr": "Clayton",
+    "NyCodeGHG": "Marie",
+    "jfroche": "Jean-Fran\u00e7ois Roche",
+    "urbas": "Matej Urbas",
+    "osbm": "osman - \u30aa\u30b9\u30de\u30f3",
+    "rszyma": null,
+    "eclairevoyant": "\u00e9clairevoyant",
+    "Radvendii": "Taeer Bar-Yam",
+    "sinanmohd": "Sinan Mohd",
+    "ethanavatar": "Ethan Evans",
+    "gmarti": "Gr\u00e9gory Marti",
+    "lovesegfault": "Bernardo Meurer",
+    "EphraimSiegfried": "Ephraim Siegfried",
+    "hgl": "Glen Huang"
 }

maintainers/flake-module.nix

@@ -104,151 +104,6 @@
           };
           shellcheck = {
             enable = true;
-            excludes = [
-              # We haven't linted these files yet
-              ''^config/install-sh$''
-              ''^misc/bash/completion\.sh$''
-              ''^misc/fish/completion\.fish$''
-              ''^misc/zsh/completion\.zsh$''
-              ''^scripts/create-darwin-volume\.sh$''
-              ''^scripts/install-darwin-multi-user\.sh$''
-              ''^scripts/install-multi-user\.sh$''
-              ''^scripts/install-systemd-multi-user\.sh$''
-              ''^src/nix/get-env\.sh$''
-              ''^tests/functional/ca/build-dry\.sh$''
-              ''^tests/functional/ca/build-with-garbage-path\.sh$''
-              ''^tests/functional/ca/common\.sh$''
-              ''^tests/functional/ca/concurrent-builds\.sh$''
-              ''^tests/functional/ca/eval-store\.sh$''
-              ''^tests/functional/ca/gc\.sh$''
-              ''^tests/functional/ca/import-from-derivation\.sh$''
-              ''^tests/functional/ca/new-build-cmd\.sh$''
-              ''^tests/functional/ca/nix-shell\.sh$''
-              ''^tests/functional/ca/post-hook\.sh$''
-              ''^tests/functional/ca/recursive\.sh$''
-              ''^tests/functional/ca/repl\.sh$''
-              ''^tests/functional/ca/selfref-gc\.sh$''
-              ''^tests/functional/ca/why-depends\.sh$''
-              ''^tests/functional/characterisation-test-infra\.sh$''
-              ''^tests/functional/common/vars-and-functions\.sh$''
-              ''^tests/functional/completions\.sh$''
-              ''^tests/functional/compute-levels\.sh$''
-              ''^tests/functional/config\.sh$''
-              ''^tests/functional/db-migration\.sh$''
-              ''^tests/functional/debugger\.sh$''
-              ''^tests/functional/dependencies\.builder0\.sh$''
-              ''^tests/functional/dependencies\.sh$''
-              ''^tests/functional/dump-db\.sh$''
-              ''^tests/functional/dyn-drv/build-built-drv\.sh$''
-              ''^tests/functional/dyn-drv/common\.sh$''
-              ''^tests/functional/dyn-drv/dep-built-drv\.sh$''
-              ''^tests/functional/dyn-drv/eval-outputOf\.sh$''
-              ''^tests/functional/dyn-drv/old-daemon-error-hack\.sh$''
-              ''^tests/functional/dyn-drv/recursive-mod-json\.sh$''
-              ''^tests/functional/eval-store\.sh$''
-              ''^tests/functional/export-graph\.sh$''
-              ''^tests/functional/export\.sh$''
-              ''^tests/functional/extra-sandbox-profile\.sh$''
-              ''^tests/functional/fetchClosure\.sh$''
-              ''^tests/functional/fetchGit\.sh$''
-              ''^tests/functional/fetchGitRefs\.sh$''
-              ''^tests/functional/fetchGitSubmodules\.sh$''
-              ''^tests/functional/fetchGitVerification\.sh$''
-              ''^tests/functional/fetchMercurial\.sh$''
-              ''^tests/functional/fixed\.builder1\.sh$''
-              ''^tests/functional/fixed\.builder2\.sh$''
-              ''^tests/functional/fixed\.sh$''
-              ''^tests/functional/flakes/absolute-paths\.sh$''
-              ''^tests/functional/flakes/check\.sh$''
-              ''^tests/functional/flakes/config\.sh$''
-              ''^tests/functional/flakes/flakes\.sh$''
-              ''^tests/functional/flakes/follow-paths\.sh$''
-              ''^tests/functional/flakes/prefetch\.sh$''
-              ''^tests/functional/flakes/run\.sh$''
-              ''^tests/functional/flakes/show\.sh$''
-              ''^tests/functional/formatter\.sh$''
-              ''^tests/functional/formatter\.simple\.sh$''
-              ''^tests/functional/gc-auto\.sh$''
-              ''^tests/functional/gc-concurrent\.builder\.sh$''
-              ''^tests/functional/gc-concurrent\.sh$''
-              ''^tests/functional/gc-concurrent2\.builder\.sh$''
-              ''^tests/functional/gc-non-blocking\.sh$''
-              ''^tests/functional/hash-convert\.sh$''
-              ''^tests/functional/impure-derivations\.sh$''
-              ''^tests/functional/impure-eval\.sh$''
-              ''^tests/functional/install-darwin\.sh$''
-              ''^tests/functional/legacy-ssh-store\.sh$''
-              ''^tests/functional/linux-sandbox\.sh$''
-              ''^tests/functional/local-overlay-store/add-lower-inner\.sh$''
-              ''^tests/functional/local-overlay-store/add-lower\.sh$''
-              ''^tests/functional/local-overlay-store/bad-uris\.sh$''
-              ''^tests/functional/local-overlay-store/build-inner\.sh$''
-              ''^tests/functional/local-overlay-store/build\.sh$''
-              ''^tests/functional/local-overlay-store/check-post-init-inner\.sh$''
-              ''^tests/functional/local-overlay-store/check-post-init\.sh$''
-              ''^tests/functional/local-overlay-store/common\.sh$''
-              ''^tests/functional/local-overlay-store/delete-duplicate-inner\.sh$''
-              ''^tests/functional/local-overlay-store/delete-duplicate\.sh$''
-              ''^tests/functional/local-overlay-store/delete-refs-inner\.sh$''
-              ''^tests/functional/local-overlay-store/delete-refs\.sh$''
-              ''^tests/functional/local-overlay-store/gc-inner\.sh$''
-              ''^tests/functional/local-overlay-store/gc\.sh$''
-              ''^tests/functional/local-overlay-store/optimise-inner\.sh$''
-              ''^tests/functional/local-overlay-store/optimise\.sh$''
-              ''^tests/functional/local-overlay-store/redundant-add-inner\.sh$''
-              ''^tests/functional/local-overlay-store/redundant-add\.sh$''
-              ''^tests/functional/local-overlay-store/remount\.sh$''
-              ''^tests/functional/local-overlay-store/stale-file-handle-inner\.sh$''
-              ''^tests/functional/local-overlay-store/stale-file-handle\.sh$''
-              ''^tests/functional/local-overlay-store/verify-inner\.sh$''
-              ''^tests/functional/local-overlay-store/verify\.sh$''
-              ''^tests/functional/logging\.sh$''
-              ''^tests/functional/misc\.sh$''
-              ''^tests/functional/multiple-outputs\.sh$''
-              ''^tests/functional/nested-sandboxing\.sh$''
-              ''^tests/functional/nested-sandboxing/command\.sh$''
-              ''^tests/functional/nix-build\.sh$''
-              ''^tests/functional/nix-channel\.sh$''
-              ''^tests/functional/nix-collect-garbage-d\.sh$''
-              ''^tests/functional/nix-copy-ssh-common\.sh$''
-              ''^tests/functional/nix-copy-ssh-ng\.sh$''
-              ''^tests/functional/nix-copy-ssh\.sh$''
-              ''^tests/functional/nix-daemon-untrusting\.sh$''
-              ''^tests/functional/nix-profile\.sh$''
-              ''^tests/functional/nix-shell\.sh$''
-              ''^tests/functional/nix_path\.sh$''
-              ''^tests/functional/optimise-store\.sh$''
-              ''^tests/functional/output-normalization\.sh$''
-              ''^tests/functional/parallel\.builder\.sh$''
-              ''^tests/functional/parallel\.sh$''
-              ''^tests/functional/pass-as-file\.sh$''
-              ''^tests/functional/path-from-hash-part\.sh$''
-              ''^tests/functional/path-info\.sh$''
-              ''^tests/functional/placeholders\.sh$''
-              ''^tests/functional/post-hook\.sh$''
-              ''^tests/functional/pure-eval\.sh$''
-              ''^tests/functional/push-to-store-old\.sh$''
-              ''^tests/functional/push-to-store\.sh$''
-              ''^tests/functional/read-only-store\.sh$''
-              ''^tests/functional/readfile-context\.sh$''
-              ''^tests/functional/recursive\.sh$''
-              ''^tests/functional/referrers\.sh$''
-              ''^tests/functional/remote-store\.sh$''
-              ''^tests/functional/repair\.sh$''
-              ''^tests/functional/restricted\.sh$''
-              ''^tests/functional/search\.sh$''
-              ''^tests/functional/secure-drv-outputs\.sh$''
-              ''^tests/functional/selfref-gc\.sh$''
-              ''^tests/functional/shell\.shebang\.sh$''
-              ''^tests/functional/simple\.builder\.sh$''
-              ''^tests/functional/supplementary-groups\.sh$''
-              ''^tests/functional/toString-path\.sh$''
-              ''^tests/functional/user-envs-migration\.sh$''
-              ''^tests/functional/user-envs-test-case\.sh$''
-              ''^tests/functional/user-envs\.builder\.sh$''
-              ''^tests/functional/user-envs\.sh$''
-              ''^tests/functional/why-depends\.sh$''
-            ];
           };
         };
       };

meson.build

@@ -41,8 +41,10 @@ subproject('libexpr-c')
 subproject('libflake-c')
 subproject('libmain-c')
 
+asan_enabled = 'address' in get_option('b_sanitize')
+
 # Language Bindings
-if get_option('bindings') and not meson.is_cross_build()
+if get_option('bindings') and not meson.is_cross_build() and not asan_enabled
   subproject('perl')
 endif
 

misc/bash/completion.sh

@@ -1,3 +1,4 @@
+# shellcheck shell=bash
 function _complete_nix {
     local -a words
     local cword cur

misc/fish/completion.fish

@@ -1,3 +1,4 @@
+# shellcheck disable=all
 function _nix_complete
   # Get the current command up to a cursor.
   # - Behaves correctly even with pipes and nested in commands like env.

misc/zsh/completion.zsh

@@ -1,3 +1,4 @@
+# shellcheck disable=all
 #compdef nix
 
 function _nix() {

nix-meson-build-support/asan-options/meson.build

@@ -0,0 +1,12 @@
+asan_test_options_env = {
+  'ASAN_OPTIONS' : 'abort_on_error=1:print_summary=1:detect_leaks=0',
+}
+
+# Clang gets grumpy about missing libasan symbols if -shared-libasan is not
+# passed when building shared libs, at least on Linux
+if cxx.get_id() == 'clang' and ('address' in get_option('b_sanitize') or 'undefined' in get_option(
+  'b_sanitize',
+))
+  add_project_link_arguments('-shared-libasan', language : 'cpp')
+endif
+

nix-meson-build-support/common/assert-fail/meson.build

@@ -0,0 +1,32 @@
+can_wrap_assert_fail_test_code = '''
+#include <cstdlib>
+#include <cassert>
+
+int main()
+{
+    assert(0);
+}
+
+extern "C" void * __real___assert_fail(const char *, const char *, unsigned int, const char *);
+
+extern "C" void *
+__wrap___assert_fail(const char *, const char *, unsigned int, const char *)
+{
+    return __real___assert_fail(nullptr, nullptr, 0, nullptr);
+}
+'''
+
+wrap_assert_fail_args = [ '-Wl,--wrap=__assert_fail' ]
+
+can_wrap_assert_fail = cxx.links(
+  can_wrap_assert_fail_test_code,
+  args : wrap_assert_fail_args,
+  name : 'linker can wrap __assert_fail',
+)
+
+if can_wrap_assert_fail
+  deps_other += declare_dependency(
+    sources : 'wrap-assert-fail.cc',
+    link_args : wrap_assert_fail_args,
+  )
+endif

nix-meson-build-support/common/assert-fail/wrap-assert-fail.cc

@@ -0,0 +1,17 @@
+#include "nix/util/error.hh"
+
+#include <cstdio>
+#include <cstdlib>
+#include <cinttypes>
+#include <string_view>
+
+extern "C" [[noreturn]] void __attribute__((weak))
+__wrap___assert_fail(const char * assertion, const char * file, unsigned int line, const char * function)
+{
+    char buf[512];
+    int n =
+        snprintf(buf, sizeof(buf), "Assertion '%s' failed in %s at %s:%" PRIuLEAST32, assertion, function, file, line);
+    if (n < 0)
+        nix::panic("Assertion failed and could not format error message");
+    nix::panic(std::string_view(buf, std::min(static_cast<int>(sizeof(buf)), n)));
+}

nix-meson-build-support/common/meson.build

@@ -5,6 +5,15 @@ if not (host_machine.system() == 'windows' and cxx.get_id() == 'gcc')
   deps_private += dependency('threads')
 endif
 
+if host_machine.system() == 'cygwin'
+  # -std=gnu on cygwin defines 'unix', which conflicts with the namespace
+  add_project_arguments(
+    '-D_POSIX_C_SOURCE=200809L',
+    '-D_GNU_SOURCE',
+    language : 'cpp',
+  )
+endif
+
 add_project_arguments(
   '-Wdeprecated-copy',
   '-Werror=suggest-override',
@@ -33,10 +42,7 @@ if cxx.get_id() == 'clang'
   add_project_arguments('-fpch-instantiate-templates', language : 'cpp')
 endif
 
-# Clang gets grumpy about missing libasan symbols if -shared-libasan is not
-# passed when building shared libs, at least on Linux
-if cxx.get_id() == 'clang' and ('address' in get_option('b_sanitize') or 'undefined' in get_option(
-  'b_sanitize',
-))
-  add_project_link_arguments('-shared-libasan', language : 'cpp')
-endif
+# Darwin ld doesn't like "X.Y.Zpre"
+nix_soversion = meson.project_version().split('pre')[0]
+
+subdir('assert-fail')

nix-meson-build-support/libatomic/meson.build

@@ -3,6 +3,6 @@
 # This is needed for std::atomic on some platforms
 # We did not manage to test this reliably on all platforms, so we hardcode
 # it for now.
-if host_machine.cpu_family() == 'arm'
+if host_machine.cpu_family() in [ 'arm', 'ppc' ]
   deps_other += cxx.find_library('atomic')
 endif

packaging/components.nix

@@ -164,6 +164,24 @@ let
   };
 
   mesonLibraryLayer = finalAttrs: prevAttrs: {
+    preConfigure =
+      let
+        interpositionFlags = [
+          "-fno-semantic-interposition"
+          "-Wl,-Bsymbolic-functions"
+        ];
+      in
+      # NOTE: By default GCC disables interprocedular optimizations (in particular inlining) for
+      # position-independent code and thus shared libraries.
+      # Since LD_PRELOAD tricks aren't worth losing out on optimizations, we disable it for good.
+      # This is not the case for Clang, where inlining is done by default even without -fno-semantic-interposition.
+      # https://reviews.llvm.org/D102453
+      # https://fedoraproject.org/wiki/Changes/PythonNoSemanticInterpositionSpeedup
+      prevAttrs.preConfigure or ""
+      + lib.optionalString stdenv.cc.isGNU ''
+        export CFLAGS="''${CFLAGS:-} ${toString interpositionFlags}"
+        export CXXFLAGS="''${CXXFLAGS:-} ${toString interpositionFlags}"
+      '';
     outputs = prevAttrs.outputs or [ "out" ] ++ [ "dev" ];
   };
 

packaging/dependencies.nix

@@ -10,27 +10,8 @@
   stdenv,
 }:
 
-let
-  prevStdenv = stdenv;
-in
-
 let
   inherit (pkgs) lib;
-
-  stdenv = if prevStdenv.isDarwin && prevStdenv.isx86_64 then darwinStdenv else prevStdenv;
-
-  # Fix the following error with the default x86_64-darwin SDK:
-  #
-  #     error: aligned allocation function of type 'void *(std::size_t, std::align_val_t)' is only available on macOS 10.13 or newer
-  #
-  # Despite the use of the 10.13 deployment target here, the aligned
-  # allocation function Clang uses with this setting actually works
-  # all the way back to 10.6.
-  # NOTE: this is not just a version constraint, but a request to make Darwin
-  #       provide this version level of support. Removing this minimum version
-  #       request will regress the above error.
-  darwinStdenv = pkgs.overrideSDK prevStdenv { darwinMinVersion = "10.13"; };
-
 in
 scope: {
   inherit stdenv;
@@ -76,15 +57,20 @@ scope: {
         prevAttrs.postInstall;
   });
 
-  toml11 = pkgs.toml11.overrideAttrs rec {
-    version = "4.4.0";
-    src = pkgs.fetchFromGitHub {
-      owner = "ToruNiina";
-      repo = "toml11";
-      tag = "v${version}";
-      hash = "sha256-sgWKYxNT22nw376ttGsTdg0AMzOwp8QH3E8mx0BZJTQ=";
-    };
-  };
+  # TODO: Remove this when https://github.com/NixOS/nixpkgs/pull/442682 is included in a stable release
+  toml11 =
+    if lib.versionAtLeast pkgs.toml11.version "4.4.0" then
+      pkgs.toml11
+    else
+      pkgs.toml11.overrideAttrs rec {
+        version = "4.4.0";
+        src = pkgs.fetchFromGitHub {
+          owner = "ToruNiina";
+          repo = "toml11";
+          tag = "v${version}";
+          hash = "sha256-sgWKYxNT22nw376ttGsTdg0AMzOwp8QH3E8mx0BZJTQ=";
+        };
+      };
 
   # TODO Hack until https://github.com/NixOS/nixpkgs/issues/45462 is fixed.
   boost =

packaging/dev-shell.nix

@@ -118,6 +118,7 @@ pkgs.nixComponents2.nix-util.overrideAttrs (
         modular.pre-commit.settings.package
         (pkgs.writeScriptBin "pre-commit-hooks-install" modular.pre-commit.settings.installationScript)
         pkgs.buildPackages.nixfmt-rfc-style
+        pkgs.buildPackages.shellcheck
         pkgs.buildPackages.gdb
       ]
       ++ lib.optional (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform) (

scripts/install-multi-user.sh

@@ -55,18 +55,22 @@ readonly NIX_INSTALLED_NIX="@nix@"
 readonly NIX_INSTALLED_CACERT="@cacert@"
 #readonly NIX_INSTALLED_NIX="/nix/store/j8dbv5w6jl34caywh2ygdy88knx1mdf7-nix-2.3.6"
 #readonly NIX_INSTALLED_CACERT="/nix/store/7dxhzymvy330i28ii676fl1pqwcahv2f-nss-cacert-3.49.2"
-readonly EXTRACTED_NIX_PATH="$(dirname "$0")"
+EXTRACTED_NIX_PATH="$(dirname "$0")"
+readonly EXTRACTED_NIX_PATH
 
 # allow to override identity change command
-readonly NIX_BECOME=${NIX_BECOME:-sudo}
+NIX_BECOME=${NIX_BECOME:-sudo}
+readonly NIX_BECOME
 
-readonly ROOT_HOME=~root
+ROOT_HOME=~root
+readonly ROOT_HOME
 
 if [ -t 0 ] && [ -z "${NIX_INSTALLER_YES:-}" ]; then
-    readonly IS_HEADLESS='no'
+    IS_HEADLESS='no'
 else
-    readonly IS_HEADLESS='yes'
+    IS_HEADLESS='yes'
 fi
+readonly IS_HEADLESS
 
 headless() {
     if [ "$IS_HEADLESS" = "yes" ]; then
@@ -156,6 +160,7 @@ EOF
 }
 
 nix_user_for_core() {
+    # shellcheck disable=SC2059
     printf "$NIX_BUILD_USER_NAME_TEMPLATE" "$1"
 }
 
@@ -381,10 +386,12 @@ _sudo() {
 
 # Ensure that $TMPDIR exists if defined.
 if [[ -n "${TMPDIR:-}" ]] && [[ ! -d "${TMPDIR:-}" ]]; then
+    # shellcheck disable=SC2174
     mkdir -m 0700 -p "${TMPDIR:-}"
 fi
 
-readonly SCRATCH=$(mktemp -d)
+SCRATCH=$(mktemp -d)
+readonly SCRATCH
 finish_cleanup() {
     rm -rf "$SCRATCH"
 }
@@ -677,7 +684,8 @@ create_directories() {
         # hiding behind || true, and the general state
         # should be one the user can repair once they
         # figure out where chown is...
-        local get_chr_own="$(PATH="$(getconf PATH 2>/dev/null)" command -vp chown)"
+        local get_chr_own
+        get_chr_own="$(PATH="$(getconf PATH 2>/dev/null)" command -vp chown)"
         if [[ -z "$get_chr_own" ]]; then
             get_chr_own="$(command -v chown)"
         fi
@@ -915,9 +923,11 @@ configure_shell_profile() {
         fi
 
         if [ -e "$profile_target" ]; then
-            shell_source_lines \
-                | _sudo "extend your $profile_target with nix-daemon settings" \
-                        tee -a "$profile_target"
+            {
+                shell_source_lines
+                cat "$profile_target"
+            } | _sudo "extend your $profile_target with nix-daemon settings" \
+                      tee "$profile_target"
         fi
     done
 
@@ -1013,6 +1023,7 @@ main() {
 
     # Set profile targets after OS-specific scripts are loaded
     if command -v poly_configure_default_profile_targets > /dev/null 2>&1; then
+        # shellcheck disable=SC2207
         PROFILE_TARGETS=($(poly_configure_default_profile_targets))
     else
         PROFILE_TARGETS=("/etc/bashrc" "/etc/profile.d/nix.sh" "/etc/zshrc" "/etc/bash.bashrc" "/etc/zsh/zshrc")

scripts/install-systemd-multi-user.sh

@@ -39,7 +39,7 @@ create_systemd_proxy_env() {
     vars="http_proxy https_proxy ftp_proxy all_proxy no_proxy HTTP_PROXY HTTPS_PROXY FTP_PROXY ALL_PROXY NO_PROXY"
     for v in $vars; do
         if [ "x${!v:-}" != "x" ]; then
-            echo "Environment=${v}=$(escape_systemd_env ${!v})"
+            echo "Environment=${v}=$(escape_systemd_env "${!v}")"
         fi
     done
 }

src/libcmd/built-path.cc

@@ -83,12 +83,22 @@ nlohmann::json SingleBuiltPath::Built::toJSON(const StoreDirConfig & store) cons
 
 nlohmann::json SingleBuiltPath::toJSON(const StoreDirConfig & store) const
 {
-    return std::visit([&](const auto & buildable) { return buildable.toJSON(store); }, raw());
+    return std::visit(
+        overloaded{
+            [&](const SingleBuiltPath::Opaque & o) -> nlohmann::json { return store.printStorePath(o.path); },
+            [&](const SingleBuiltPath::Built & b) { return b.toJSON(store); },
+        },
+        raw());
 }
 
 nlohmann::json BuiltPath::toJSON(const StoreDirConfig & store) const
 {
-    return std::visit([&](const auto & buildable) { return buildable.toJSON(store); }, raw());
+    return std::visit(
+        overloaded{
+            [&](const BuiltPath::Opaque & o) -> nlohmann::json { return store.printStorePath(o.path); },
+            [&](const BuiltPath::Built & b) { return b.toJSON(store); },
+        },
+        raw());
 }
 
 RealisedPath::Set BuiltPath::toRealisedPaths(Store & store) const

src/libcmd/installables.cc

@@ -604,28 +604,28 @@ std::vector<BuiltPathWithResult> Installable::build(
 
 static void throwBuildErrors(std::vector<KeyedBuildResult> & buildResults, const Store & store)
 {
-    std::vector<KeyedBuildResult> failed;
+    std::vector<std::pair<const KeyedBuildResult *, const KeyedBuildResult::Failure *>> failed;
     for (auto & buildResult : buildResults) {
-        if (!buildResult.success()) {
-            failed.push_back(buildResult);
+        if (auto * failure = buildResult.tryGetFailure()) {
+            failed.push_back({&buildResult, failure});
         }
     }
 
     auto failedResult = failed.begin();
     if (failedResult != failed.end()) {
         if (failed.size() == 1) {
-            failedResult->rethrow();
+            failedResult->second->rethrow();
         } else {
             StringSet failedPaths;
             for (; failedResult != failed.end(); failedResult++) {
-                if (!failedResult->errorMsg.empty()) {
+                if (!failedResult->second->errorMsg.empty()) {
                     logError(
                         ErrorInfo{
                             .level = lvlError,
-                            .msg = failedResult->errorMsg,
+                            .msg = failedResult->second->errorMsg,
                         });
                 }
-                failedPaths.insert(failedResult->path.to_string(store));
+                failedPaths.insert(failedResult->first->path.to_string(store));
             }
             throw Error("build of %s failed", concatStringsSep(", ", quoteStrings(failedPaths)));
         }
@@ -695,12 +695,14 @@ std::vector<std::pair<ref<Installable>, BuiltPathWithResult>> Installable::build
         auto buildResults = store->buildPathsWithResults(pathsToBuild, bMode, evalStore);
         throwBuildErrors(buildResults, *store);
         for (auto & buildResult : buildResults) {
+            // If we didn't throw, they must all be sucesses
+            auto & success = std::get<nix::BuildResult::Success>(buildResult.inner);
             for (auto & aux : backmap[buildResult.path]) {
                 std::visit(
                     overloaded{
                         [&](const DerivedPath::Built & bfd) {
                             std::map<std::string, StorePath> outputs;
-                            for (auto & [outputName, realisation] : buildResult.builtOutputs)
+                            for (auto & [outputName, realisation] : success.builtOutputs)
                                 outputs.emplace(outputName, realisation.outPath);
                             res.push_back(
                                 {aux.installable,

src/libcmd/meson.build

@@ -67,6 +67,7 @@ config_priv_h = configure_file(
 )
 
 subdir('nix-meson-build-support/common')
+subdir('nix-meson-build-support/asan-options')
 
 sources = files(
   'built-path.cc',
@@ -95,6 +96,7 @@ this_library = library(
   'nixcmd',
   sources,
   config_priv_h,
+  soversion : nix_soversion,
   dependencies : deps_public + deps_private + deps_other,
   include_directories : include_dirs,
   link_args : linker_export_flags,

src/libcmd/repl.cc

@@ -669,7 +669,7 @@ ProcessLineResult NixRepl::processLine(std::string line)
                 ss << "No documentation found.\n\n";
             }
 
-            auto markdown = toView(ss);
+            auto markdown = ss.view();
             logger->cout(trim(renderMarkdownToTerminal(markdown)));
 
         } else
@@ -760,7 +760,7 @@ void NixRepl::loadFlake(const std::string & flakeRefS)
 
 void NixRepl::initEnv()
 {
-    env = &state->allocEnv(envSize);
+    env = &state->mem.allocEnv(envSize);
     env->up = &state->baseEnv;
     displ = 0;
     staticEnv->vars.clear();
@@ -869,14 +869,8 @@ void NixRepl::addVarToScope(const Symbol name, Value & v)
 
 Expr * NixRepl::parseString(std::string s)
 {
-    return state->parseExprFromString(std::move(s), state->rootPath("."), staticEnv);
-}
-
-void NixRepl::evalString(std::string s, Value & v)
-{
-    Expr * e;
     try {
-        e = parseString(s);
+        return state->parseExprFromString(std::move(s), state->rootPath("."), staticEnv);
     } catch (ParseError & e) {
         if (e.msg().find("unexpected end of file") != std::string::npos)
             // For parse errors on incomplete input, we continue waiting for the next line of
@@ -885,6 +879,11 @@ void NixRepl::evalString(std::string s, Value & v)
         else
             throw;
     }
+}
+
+void NixRepl::evalString(std::string s, Value & v)
+{
+    Expr * e = parseString(s);
     e->eval(*state, *env, v);
     state->forceValue(v, v.determinePos(noPos));
 }

src/libexpr-c/meson.build

@@ -28,6 +28,7 @@ deps_public_maybe_subproject = [
 subdir('nix-meson-build-support/subprojects')
 
 subdir('nix-meson-build-support/common')
+subdir('nix-meson-build-support/asan-options')
 
 sources = files(
   'nix_api_expr.cc',
@@ -50,6 +51,7 @@ subdir('nix-meson-build-support/windows-version')
 this_library = library(
   'nixexprc',
   sources,
+  soversion : nix_soversion,
   dependencies : deps_public + deps_private + deps_other,
   include_directories : include_dirs,
   link_args : linker_export_flags,

src/libexpr-c/nix_api_expr.cc

@@ -16,7 +16,7 @@
 #include "nix_api_util_internal.h"
 
 #if NIX_USE_BOEHMGC
-#  include <mutex>
+#  include <boost/unordered/concurrent_flat_map.hpp>
 #endif
 
 /**
@@ -137,7 +137,7 @@ nix_eval_state_builder * nix_eval_state_builder_new(nix_c_context * context, Sto
 
 void nix_eval_state_builder_free(nix_eval_state_builder * builder)
 {
-    delete builder;
+    operator delete(builder, static_cast<std::align_val_t>(alignof(nix_eval_state_builder)));
 }
 
 nix_err nix_eval_state_builder_load(nix_c_context * context, nix_eval_state_builder * builder)
@@ -203,32 +203,24 @@ EvalState * nix_state_create(nix_c_context * context, const char ** lookupPath_c
 
 void nix_state_free(EvalState * state)
 {
-    delete state;
+    operator delete(state, static_cast<std::align_val_t>(alignof(EvalState)));
 }
 
 #if NIX_USE_BOEHMGC
-std::unordered_map<
+boost::concurrent_flat_map<
     const void *,
     unsigned int,
     std::hash<const void *>,
     std::equal_to<const void *>,
     traceable_allocator<std::pair<const void * const, unsigned int>>>
-    nix_refcounts;
-
-std::mutex nix_refcount_lock;
+    nix_refcounts{};
 
 nix_err nix_gc_incref(nix_c_context * context, const void * p)
 {
     if (context)
         context->last_err_code = NIX_OK;
     try {
-        std::scoped_lock lock(nix_refcount_lock);
-        auto f = nix_refcounts.find(p);
-        if (f != nix_refcounts.end()) {
-            f->second++;
-        } else {
-            nix_refcounts[p] = 1;
-        }
+        nix_refcounts.insert_or_visit({p, 1}, [](auto & kv) { kv.second++; });
     }
     NIXC_CATCH_ERRS
 }
@@ -239,12 +231,12 @@ nix_err nix_gc_decref(nix_c_context * context, const void * p)
     if (context)
         context->last_err_code = NIX_OK;
     try {
-        std::scoped_lock lock(nix_refcount_lock);
-        auto f = nix_refcounts.find(p);
-        if (f != nix_refcounts.end()) {
-            if (--f->second == 0)
-                nix_refcounts.erase(f);
-        } else
+        bool fail = true;
+        nix_refcounts.erase_if(p, [&](auto & kv) {
+            fail = false;
+            return !--kv.second;
+        });
+        if (fail)
             throw std::runtime_error("nix_gc_decref: object was not referenced");
     }
     NIXC_CATCH_ERRS

src/libexpr-c/nix_api_value.cc

@@ -1,5 +1,4 @@
 #include "nix/expr/attr-set.hh"
-#include "nix/expr/eval-error.hh"
 #include "nix/util/configuration.hh"
 #include "nix/expr/eval.hh"
 #include "nix/store/globals.hh"
@@ -90,13 +89,8 @@ static void nix_c_primop_wrapper(
     f(userdata, &ctx, (EvalState *) &state, (nix_value **) args, (nix_value *) &vTmp);
 
     if (ctx.last_err_code != NIX_OK) {
-        if (ctx.last_err_code == NIX_ERR_RECOVERABLE) {
-            state.error<nix::RecoverableEvalError>("Recoverable error from custom function: %s", *ctx.last_err)
-                .atPos(pos)
-                .debugThrow();
-        } else {
-            state.error<nix::EvalError>("Error from custom function: %s", *ctx.last_err).atPos(pos).debugThrow();
-        }
+        /* TODO: Throw different errors depending on the error code */
+        state.error<nix::EvalError>("Error from custom function: %s", *ctx.last_err).atPos(pos).debugThrow();
     }
 
     if (!vTmp.isValid()) {
@@ -183,8 +177,6 @@ ValueType nix_get_type(nix_c_context * context, const nix_value * value)
         switch (v.type()) {
         case nThunk:
             return NIX_TYPE_THUNK;
-        case nFailed:
-            return NIX_TYPE_FAILED;
         case nInt:
             return NIX_TYPE_INT;
         case nFloat:
@@ -334,6 +326,10 @@ nix_value * nix_get_list_byidx(nix_c_context * context, const nix_value * value,
     try {
         auto & v = check_value_in(value);
         assert(v.type() == nix::nList);
+        if (ix >= v.listSize()) {
+            nix_set_err_msg(context, NIX_ERR_KEY, "list index out of bounds");
+            return nullptr;
+        }
         auto * p = v.listView()[ix];
         nix_gc_incref(nullptr, p);
         if (p != nullptr)
@@ -343,6 +339,26 @@ nix_value * nix_get_list_byidx(nix_c_context * context, const nix_value * value,
     NIXC_CATCH_ERRS_NULL
 }
 
+nix_value *
+nix_get_list_byidx_lazy(nix_c_context * context, const nix_value * value, EvalState * state, unsigned int ix)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_in(value);
+        assert(v.type() == nix::nList);
+        if (ix >= v.listSize()) {
+            nix_set_err_msg(context, NIX_ERR_KEY, "list index out of bounds");
+            return nullptr;
+        }
+        auto * p = v.listView()[ix];
+        nix_gc_incref(nullptr, p);
+        // Note: intentionally NOT calling forceValue() to keep the element lazy
+        return as_nix_value_ptr(p);
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
 nix_value * nix_get_attr_byname(nix_c_context * context, const nix_value * value, EvalState * state, const char * name)
 {
     if (context)
@@ -363,6 +379,27 @@ nix_value * nix_get_attr_byname(nix_c_context * context, const nix_value * value
     NIXC_CATCH_ERRS_NULL
 }
 
+nix_value *
+nix_get_attr_byname_lazy(nix_c_context * context, const nix_value * value, EvalState * state, const char * name)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_in(value);
+        assert(v.type() == nix::nAttrs);
+        nix::Symbol s = state->state.symbols.create(name);
+        auto attr = v.attrs()->get(s);
+        if (attr) {
+            nix_gc_incref(nullptr, attr->value);
+            // Note: intentionally NOT calling forceValue() to keep the attribute lazy
+            return as_nix_value_ptr(attr->value);
+        }
+        nix_set_err_msg(context, NIX_ERR_KEY, "missing attribute");
+        return nullptr;
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
 bool nix_has_attr_byname(nix_c_context * context, const nix_value * value, EvalState * state, const char * name)
 {
     if (context)
@@ -379,13 +416,28 @@ bool nix_has_attr_byname(nix_c_context * context, const nix_value * value, EvalS
     NIXC_CATCH_ERRS_RES(false);
 }
 
-nix_value * nix_get_attr_byidx(
-    nix_c_context * context, const nix_value * value, EvalState * state, unsigned int i, const char ** name)
+static void collapse_attrset_layer_chain_if_needed(nix::Value & v, EvalState * state)
+{
+    auto & attrs = *v.attrs();
+    if (attrs.isLayered()) {
+        auto bindings = state->state.buildBindings(attrs.size());
+        std::ranges::copy(attrs, std::back_inserter(bindings));
+        v.mkAttrs(bindings);
+    }
+}
+
+nix_value *
+nix_get_attr_byidx(nix_c_context * context, nix_value * value, EvalState * state, unsigned int i, const char ** name)
 {
     if (context)
         context->last_err_code = NIX_OK;
     try {
         auto & v = check_value_in(value);
+        collapse_attrset_layer_chain_if_needed(v, state);
+        if (i >= v.attrs()->size()) {
+            nix_set_err_msg(context, NIX_ERR_KEY, "attribute index out of bounds");
+            return nullptr;
+        }
         const nix::Attr & a = (*v.attrs())[i];
         *name = state->state.symbols[a.name].c_str();
         nix_gc_incref(nullptr, a.value);
@@ -395,13 +447,38 @@ nix_value * nix_get_attr_byidx(
     NIXC_CATCH_ERRS_NULL
 }
 
-const char *
-nix_get_attr_name_byidx(nix_c_context * context, const nix_value * value, EvalState * state, unsigned int i)
+nix_value * nix_get_attr_byidx_lazy(
+    nix_c_context * context, nix_value * value, EvalState * state, unsigned int i, const char ** name)
 {
     if (context)
         context->last_err_code = NIX_OK;
     try {
         auto & v = check_value_in(value);
+        collapse_attrset_layer_chain_if_needed(v, state);
+        if (i >= v.attrs()->size()) {
+            nix_set_err_msg(context, NIX_ERR_KEY, "attribute index out of bounds (Nix C API contract violation)");
+            return nullptr;
+        }
+        const nix::Attr & a = (*v.attrs())[i];
+        *name = state->state.symbols[a.name].c_str();
+        nix_gc_incref(nullptr, a.value);
+        // Note: intentionally NOT calling forceValue() to keep the attribute lazy
+        return as_nix_value_ptr(a.value);
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+const char * nix_get_attr_name_byidx(nix_c_context * context, nix_value * value, EvalState * state, unsigned int i)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_in(value);
+        collapse_attrset_layer_chain_if_needed(v, state);
+        if (i >= v.attrs()->size()) {
+            nix_set_err_msg(context, NIX_ERR_KEY, "attribute index out of bounds (Nix C API contract violation)");
+            return nullptr;
+        }
         const nix::Attr & a = (*v.attrs())[i];
         return state->state.symbols[a.name].c_str();
     }
@@ -602,7 +679,7 @@ nix_err nix_bindings_builder_insert(nix_c_context * context, BindingsBuilder * b
         context->last_err_code = NIX_OK;
     try {
         auto & v = check_value_not_null(value);
-        nix::Symbol s = bb->builder.state.get().symbols.create(name);
+        nix::Symbol s = bb->builder.symbols.get().create(name);
         bb->builder.insert(s, &v);
     }
     NIXC_CATCH_ERRS

src/libexpr-c/nix_api_value.h

@@ -32,8 +32,7 @@ typedef enum {
     NIX_TYPE_ATTRS,
     NIX_TYPE_LIST,
     NIX_TYPE_FUNCTION,
-    NIX_TYPE_EXTERNAL,
-    NIX_TYPE_FAILED,
+    NIX_TYPE_EXTERNAL
 } ValueType;
 
 // forward declarations
@@ -266,10 +265,25 @@ ExternalValue * nix_get_external(nix_c_context * context, nix_value * value);
  */
 nix_value * nix_get_list_byidx(nix_c_context * context, const nix_value * value, EvalState * state, unsigned int ix);
 
-/** @brief Get an attr by name
+/** @brief Get the ix'th element of a list without forcing evaluation of the element
+ *
+ * Returns the list element without forcing its evaluation, allowing access to lazy values.
+ * The list value itself must already be evaluated.
  *
  * Owned by the GC. Use nix_gc_decref when you're done with the pointer
  * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect (must be an evaluated list)
+ * @param[in] state nix evaluator state
+ * @param[in] ix list element to get
+ * @return value, NULL in case of errors
+ */
+nix_value *
+nix_get_list_byidx_lazy(nix_c_context * context, const nix_value * value, EvalState * state, unsigned int ix);
+
+/** @brief Get an attr by name
+ *
+ * Use nix_gc_decref when you're done with the pointer
+ * @param[out] context Optional, stores error information
  * @param[in] value Nix value to inspect
  * @param[in] state nix evaluator state
  * @param[in] name attribute name
@@ -277,6 +291,21 @@ nix_value * nix_get_list_byidx(nix_c_context * context, const nix_value * value,
  */
 nix_value * nix_get_attr_byname(nix_c_context * context, const nix_value * value, EvalState * state, const char * name);
 
+/** @brief Get an attribute value by attribute name, without forcing evaluation of the attribute's value
+ *
+ * Returns the attribute value without forcing its evaluation, allowing access to lazy values.
+ * The attribute set value itself must already be evaluated.
+ *
+ * Use nix_gc_decref when you're done with the pointer
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect (must be an evaluated attribute set)
+ * @param[in] state nix evaluator state
+ * @param[in] name attribute name
+ * @return value, NULL in case of errors
+ */
+nix_value *
+nix_get_attr_byname_lazy(nix_c_context * context, const nix_value * value, EvalState * state, const char * name);
+
 /** @brief Check if an attribute name exists on a value
  * @param[out] context Optional, stores error information
  * @param[in] value Nix value to inspect
@@ -286,11 +315,21 @@ nix_value * nix_get_attr_byname(nix_c_context * context, const nix_value * value
  */
 bool nix_has_attr_byname(nix_c_context * context, const nix_value * value, EvalState * state, const char * name);
 
-/** @brief Get an attribute by index in the sorted bindings
+/** @brief Get an attribute by index
  *
  * Also gives you the name.
  *
- * Owned by the GC. Use nix_gc_decref when you're done with the pointer
+ * Attributes are returned in an unspecified order which is NOT suitable for
+ * reproducible operations. In Nix's domain, reproducibility is paramount. The caller
+ * is responsible for sorting the attributes or storing them in an ordered map to
+ * ensure deterministic behavior in your application.
+ *
+ * @note When Nix does sort attributes, which it does for virtually all intermediate
+ * operations and outputs, it uses byte-wise lexicographic order (equivalent to
+ * lexicographic order by Unicode scalar value for valid UTF-8). We recommend
+ * applying this same ordering for consistency.
+ *
+ * Use nix_gc_decref when you're done with the pointer
  * @param[out] context Optional, stores error information
  * @param[in] value Nix value to inspect
  * @param[in] state nix evaluator state
@@ -298,12 +337,50 @@ bool nix_has_attr_byname(nix_c_context * context, const nix_value * value, EvalS
  * @param[out] name will store a pointer to the attribute name
  * @return value, NULL in case of errors
  */
-nix_value * nix_get_attr_byidx(
-    nix_c_context * context, const nix_value * value, EvalState * state, unsigned int i, const char ** name);
+nix_value *
+nix_get_attr_byidx(nix_c_context * context, nix_value * value, EvalState * state, unsigned int i, const char ** name);
 
-/** @brief Get an attribute name by index in the sorted bindings
+/** @brief Get an attribute by index, without forcing evaluation of the attribute's value
  *
- * Useful when you want the name but want to avoid evaluation.
+ * Also gives you the name.
+ *
+ * Returns the attribute value without forcing its evaluation, allowing access to lazy values.
+ * The attribute set value itself must already have been evaluated.
+ *
+ * Attributes are returned in an unspecified order which is NOT suitable for
+ * reproducible operations. In Nix's domain, reproducibility is paramount. The caller
+ * is responsible for sorting the attributes or storing them in an ordered map to
+ * ensure deterministic behavior in your application.
+ *
+ * @note When Nix does sort attributes, which it does for virtually all intermediate
+ * operations and outputs, it uses byte-wise lexicographic order (equivalent to
+ * lexicographic order by Unicode scalar value for valid UTF-8). We recommend
+ * applying this same ordering for consistency.
+ *
+ * Use nix_gc_decref when you're done with the pointer
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect (must be an evaluated attribute set)
+ * @param[in] state nix evaluator state
+ * @param[in] i attribute index
+ * @param[out] name will store a pointer to the attribute name
+ * @return value, NULL in case of errors
+ */
+nix_value * nix_get_attr_byidx_lazy(
+    nix_c_context * context, nix_value * value, EvalState * state, unsigned int i, const char ** name);
+
+/** @brief Get an attribute name by index
+ *
+ * Returns the attribute name without forcing evaluation of the attribute's value.
+ *
+ * Attributes are returned in an unspecified order which is NOT suitable for
+ * reproducible operations. In Nix's domain, reproducibility is paramount. The caller
+ * is responsible for sorting the attributes or storing them in an ordered map to
+ * ensure deterministic behavior in your application.
+ *
+ * @note When Nix does sort attributes, which it does for virtually all intermediate
+ * operations and outputs, it uses byte-wise lexicographic order (equivalent to
+ * lexicographic order by Unicode scalar value for valid UTF-8). We recommend
+ * applying this same ordering for consistency.
  *
  * Owned by the nix EvalState
  * @param[out] context Optional, stores error information
@@ -312,8 +389,7 @@ nix_value * nix_get_attr_byidx(
  * @param[in] i attribute index
  * @return name, NULL in case of errors
  */
-const char *
-nix_get_attr_name_byidx(nix_c_context * context, const nix_value * value, EvalState * state, unsigned int i);
+const char * nix_get_attr_name_byidx(nix_c_context * context, nix_value * value, EvalState * state, unsigned int i);
 
 /**@}*/
 /** @name Initializers

src/libexpr-test-support/include/nix/expr/tests/libexpr.hh

@@ -26,11 +26,20 @@ public:
     }
 
 protected:
-    LibExprTest()
+    LibExprTest(ref<Store> store, auto && makeEvalSettings)
         : LibStoreTest()
+        , evalSettings(makeEvalSettings(readOnlyMode))
         , state({}, store, fetchSettings, evalSettings, nullptr)
     {
-        evalSettings.nixPath = {};
+    }
+
+    LibExprTest()
+        : LibExprTest(openStore("dummy://"), [](bool & readOnlyMode) {
+            EvalSettings settings{readOnlyMode};
+            settings.nixPath = {};
+            return settings;
+        })
+    {
     }
 
     Value eval(std::string input, bool forceValue = true)

src/libexpr-test-support/meson.build

@@ -31,6 +31,7 @@ rapidcheck = dependency('rapidcheck')
 deps_public += rapidcheck
 
 subdir('nix-meson-build-support/common')
+subdir('nix-meson-build-support/asan-options')
 
 sources = files(
   'tests/value/context.cc',
@@ -44,6 +45,7 @@ subdir('nix-meson-build-support/windows-version')
 this_library = library(
   'nix-expr-test-support',
   sources,
+  soversion : nix_soversion,
   dependencies : deps_public + deps_private + deps_other,
   include_directories : include_dirs,
   # TODO: Remove `-lrapidcheck` when https://github.com/emil-e/rapidcheck/pull/326

src/libexpr-tests/eval.cc

@@ -3,6 +3,7 @@
 
 #include "nix/expr/eval.hh"
 #include "nix/expr/tests/libexpr.hh"
+#include "nix/util/memory-source-accessor.hh"
 
 namespace nix {
 
@@ -174,4 +175,41 @@ TEST_F(EvalStateTest, getBuiltin_fail)
     ASSERT_THROW(state.getBuiltin("nonexistent"), EvalError);
 }
 
+class PureEvalTest : public LibExprTest
+{
+public:
+    PureEvalTest()
+        : LibExprTest(openStore("dummy://", {{"read-only", "false"}}), [](bool & readOnlyMode) {
+            EvalSettings settings{readOnlyMode};
+            settings.pureEval = true;
+            settings.restrictEval = true;
+            return settings;
+        })
+    {
+    }
+};
+
+TEST_F(PureEvalTest, pathExists)
+{
+    ASSERT_THAT(eval("builtins.pathExists /."), IsFalse());
+    ASSERT_THAT(eval("builtins.pathExists /nix"), IsFalse());
+    ASSERT_THAT(eval("builtins.pathExists /nix/store"), IsFalse());
+
+    {
+        std::string contents = "Lorem ipsum";
+
+        StringSource s{contents};
+        auto path = state.store->addToStoreFromDump(
+            s, "source", FileSerialisationMethod::Flat, ContentAddressMethod::Raw::Text, HashAlgorithm::SHA256);
+        auto printed = store->printStorePath(path);
+
+        ASSERT_THROW(eval(fmt("builtins.readFile %s", printed)), RestrictedPathError);
+        ASSERT_THAT(eval(fmt("builtins.pathExists %s", printed)), IsFalse());
+
+        ASSERT_THROW(eval("builtins.readDir /."), RestrictedPathError);
+        state.allowPath(path); // FIXME: This shouldn't behave this way.
+        ASSERT_THAT(eval("builtins.readDir /."), IsAttrsOfSize(0));
+    }
+}
+
 } // namespace nix

src/libexpr-tests/main.cc

@@ -1,40 +1,15 @@
 #include <gtest/gtest.h>
-#include <cstdlib>
-#include "nix/store/globals.hh"
-#include "nix/util/logging.hh"
+
+#include "nix/store/tests/test-main.hh"
+#include "nix/util/config-global.hh"
 
 using namespace nix;
 
 int main(int argc, char ** argv)
 {
-    if (argc > 1 && std::string_view(argv[1]) == "__build-remote") {
-        printError("test-build-remote: not supported in libexpr unit tests");
-        return 1;
-    }
-
-    // Disable build hook. We won't be testing remote builds in these unit tests. If we do, fix the above build hook.
-    settings.buildHook = {};
-
-#ifdef __linux__ // should match the conditional around sandboxBuildDir declaration.
-
-    // When building and testing nix within the host's Nix sandbox, our store dir will be located in the host's
-    // sandboxBuildDir, e.g.: Host
-    //   storeDir = /nix/store
-    //   sandboxBuildDir = /build
-    // This process
-    //   storeDir = /build/foo/bar/store
-    //   sandboxBuildDir = /build
-    // However, we have a rule that the store dir must not be inside the storeDir, so we need to pick a different
-    // sandboxBuildDir.
-    settings.sandboxBuildDir = "/test-build-dir-instead-of-usual-build-dir";
-#endif
-
-#ifdef __APPLE__
-    // Avoid this error, when already running in a sandbox:
-    // sandbox-exec: sandbox_apply: Operation not permitted
-    settings.sandboxMode = smDisabled;
-    setEnv("_NIX_TEST_NO_SANDBOX", "1");
-#endif
+    auto res = testMainForBuidingPre(argc, argv);
+    if (res)
+        return res;
 
     // For pipe operator tests in trivial.cc
     experimentalFeatureSettings.set("experimental-features", "pipe-operators");

src/libexpr-tests/meson.build

@@ -45,6 +45,7 @@ config_priv_h = configure_file(
 )
 
 subdir('nix-meson-build-support/common')
+subdir('nix-meson-build-support/asan-options')
 
 sources = files(
   'derived-path.cc',
@@ -82,7 +83,7 @@ this_exe = executable(
 test(
   meson.project_name(),
   this_exe,
-  env : {
+  env : asan_test_options_env + {
     '_NIX_TEST_UNIT_DATA' : meson.current_source_dir() / 'data',
   },
   protocol : 'gtest',

src/libexpr-tests/nix_api_expr.cc

@@ -423,6 +423,55 @@ TEST_F(nix_api_expr_test, nix_expr_primop_bad_return_thunk)
     ASSERT_THAT(nix_err_msg(nullptr, ctx, nullptr), testing::HasSubstr("badReturnThunk"));
 }
 
+static void primop_with_nix_err_key(
+    void * user_data, nix_c_context * context, EvalState * state, nix_value ** args, nix_value * ret)
+{
+    nix_set_err_msg(context, NIX_ERR_KEY, "Test error from primop");
+}
+
+TEST_F(nix_api_expr_test, nix_expr_primop_nix_err_key_conversion)
+{
+    // Test that NIX_ERR_KEY from a custom primop gets converted to a generic EvalError
+    //
+    // RATIONALE: NIX_ERR_KEY must not be propagated from custom primops because it would
+    // create semantic confusion. NIX_ERR_KEY indicates missing keys/indices in C API functions
+    // (like nix_get_attr_byname, nix_get_list_byidx). If custom primops could return NIX_ERR_KEY,
+    // an evaluation error would be indistinguishable from an actual missing attribute.
+    //
+    // For example, if nix_get_attr_byname returned NIX_ERR_KEY when the attribute is present
+    // but the value evaluation fails, callers expecting NIX_ERR_KEY to mean "missing attribute"
+    // would incorrectly handle evaluation failures as missing attributes. In places where
+    // missing attributes are tolerated (like optional attributes), this would cause the
+    // program to continue after swallowing the error, leading to silent failures.
+    PrimOp * primop = nix_alloc_primop(
+        ctx, primop_with_nix_err_key, 1, "testErrorPrimop", nullptr, "a test primop that sets NIX_ERR_KEY", nullptr);
+    assert_ctx_ok();
+    nix_value * primopValue = nix_alloc_value(ctx, state);
+    assert_ctx_ok();
+    nix_init_primop(ctx, primopValue, primop);
+    assert_ctx_ok();
+
+    nix_value * arg = nix_alloc_value(ctx, state);
+    assert_ctx_ok();
+    nix_init_int(ctx, arg, 42);
+    assert_ctx_ok();
+
+    nix_value * result = nix_alloc_value(ctx, state);
+    assert_ctx_ok();
+    nix_value_call(ctx, state, primopValue, arg, result);
+
+    // Verify that NIX_ERR_KEY gets converted to NIX_ERR_NIX_ERROR (generic evaluation error)
+    ASSERT_EQ(nix_err_code(ctx), NIX_ERR_NIX_ERROR);
+    ASSERT_THAT(nix_err_msg(nullptr, ctx, nullptr), testing::HasSubstr("Error from custom function"));
+    ASSERT_THAT(nix_err_msg(nullptr, ctx, nullptr), testing::HasSubstr("Test error from primop"));
+    ASSERT_THAT(nix_err_msg(nullptr, ctx, nullptr), testing::HasSubstr("testErrorPrimop"));
+
+    // Clean up
+    nix_gc_decref(ctx, primopValue);
+    nix_gc_decref(ctx, arg);
+    nix_gc_decref(ctx, result);
+}
+
 TEST_F(nix_api_expr_test, nix_value_call_multi_no_args)
 {
     nix_value * n = nix_alloc_value(ctx, state);
@@ -438,104 +487,30 @@ TEST_F(nix_api_expr_test, nix_value_call_multi_no_args)
     ASSERT_EQ(3, rInt);
 }
 
-// The following is a test case for retryable thunks.
-// This is a requirement for the current way in which NixOps4 evaluates its deployment expressions.
-// An alternative strategy could be implemented, but unwinding the stack may be a more efficient way to deal with many
-// suspensions/resumptions, compared to e.g. using a thread or coroutine stack for each suspended dependency. This test
-// models the essential bits of a deployment tool that uses such a strategy.
-
-// State for the retryable primop - simulates deployment resource availability
-struct DeploymentResourceState
+TEST_F(nix_api_expr_test, nix_expr_attrset_update)
 {
-    bool vm_created = false;
-};
+    nix_expr_eval_from_string(ctx, state, "{ a = 0; b = 2; } // { a = 1; b = 3; } // { a = 2; }", ".", value);
+    assert_ctx_ok();
 
-static void primop_load_resource_input(
-    void * user_data, nix_c_context * context, EvalState * state, nix_value ** args, nix_value * ret)
-{
-    assert(context);
-    assert(state);
-    auto * resource_state = static_cast<DeploymentResourceState *>(user_data);
-
-    // Get the resource input name argument
-    std::string input_name;
-    if (nix_get_string(context, args[0], OBSERVE_STRING(input_name)) != NIX_OK)
-        return;
-
-    // Only handle "vm_id" input - throw for anything else
-    if (input_name != "vm_id") {
-        std::string error_msg = "unknown resource input: " + input_name;
-        nix_set_err_msg(context, NIX_ERR_NIX_ERROR, error_msg.c_str());
-        return;
+    ASSERT_EQ(nix_get_attrs_size(ctx, value), 2);
+    assert_ctx_ok();
+    std::array<std::pair<std::string_view, nix_value *>, 2> values;
+    for (unsigned int i = 0; i < 2; ++i) {
+        const char * name;
+        values[i].second = nix_get_attr_byidx(ctx, value, state, i, &name);
+        assert_ctx_ok();
+        values[i].first = name;
     }
+    std::sort(values.begin(), values.end(), [](const auto & lhs, const auto & rhs) { return lhs.first < rhs.first; });
 
-    if (resource_state->vm_created) {
-        // VM has been created, return the ID
-        nix_init_string(context, ret, "vm-12345");
-    } else {
-        // VM not created yet, fail with dependency error
-        nix_set_err_msg(context, NIX_ERR_RECOVERABLE, "VM not yet created");
-    }
-}
-
-TEST_F(nix_api_expr_test, nix_expr_thunk_re_evaluation_after_deployment)
-{
-    // This test demonstrates NixOps4's requirement: a thunk calling a primop should be
-    // re-evaluable when deployment resources become available that were not available initially.
-
-    DeploymentResourceState resource_state;
-
-    PrimOp * primop = nix_alloc_primop(
-        ctx,
-        primop_load_resource_input,
-        1,
-        "loadResourceInput",
-        nullptr,
-        "load a deployment resource input",
-        &resource_state);
+    nix_value * a = values[0].second;
+    ASSERT_EQ("a", values[0].first);
+    ASSERT_EQ(nix_get_int(ctx, a), 2);
     assert_ctx_ok();
-
-    nix_value * primopValue = nix_alloc_value(ctx, state);
+    nix_value * b = values[1].second;
+    ASSERT_EQ("b", values[1].first);
+    ASSERT_EQ(nix_get_int(ctx, b), 3);
     assert_ctx_ok();
-    nix_init_primop(ctx, primopValue, primop);
-    assert_ctx_ok();
-
-    nix_value * inputName = nix_alloc_value(ctx, state);
-    assert_ctx_ok();
-    nix_init_string(ctx, inputName, "vm_id");
-    assert_ctx_ok();
-
-    // Create a single thunk by using nix_init_apply instead of nix_value_call
-    // This creates a lazy application that can be forced multiple times
-    nix_value * thunk = nix_alloc_value(ctx, state);
-    assert_ctx_ok();
-    nix_init_apply(ctx, thunk, primopValue, inputName);
-    assert_ctx_ok();
-
-    // First force: VM not created yet, should fail
-    nix_value_force(ctx, state, thunk);
-    ASSERT_EQ(NIX_ERR_NIX_ERROR, nix_err_code(ctx));
-    ASSERT_THAT(nix_err_msg(nullptr, ctx, nullptr), testing::HasSubstr("VM not yet created"));
-
-    // Clear the error context for the next attempt
-    nix_c_context_free(ctx);
-    ctx = nix_c_context_create();
-
-    // Simulate deployment process: VM gets created
-    resource_state.vm_created = true;
-
-    // Second force of the SAME thunk: this is where the "failed" value issue appears
-    // With failed value caching, this should fail because the thunk is marked as permanently failed
-    // Without failed value caching (or with retryable failures), this should succeed
-    nix_value_force(ctx, state, thunk);
-
-    // If we get here without error, the thunk was successfully re-evaluated
-    assert_ctx_ok();
-
-    std::string result;
-    nix_get_string(ctx, thunk, OBSERVE_STRING(result));
-    assert_ctx_ok();
-    ASSERT_STREQ("vm-12345", result.c_str());
 }
 
 } // namespace nixC

src/libexpr-tests/nix_api_value.cc

@@ -162,6 +162,114 @@ TEST_F(nix_api_expr_test, nix_build_and_init_list)
     nix_gc_decref(ctx, intValue);
 }
 
+TEST_F(nix_api_expr_test, nix_get_list_byidx_large_indices)
+{
+    // Create a small list to test extremely large out-of-bounds access
+    ListBuilder * builder = nix_make_list_builder(ctx, state, 2);
+    nix_value * intValue = nix_alloc_value(ctx, state);
+    nix_init_int(ctx, intValue, 42);
+    nix_list_builder_insert(ctx, builder, 0, intValue);
+    nix_list_builder_insert(ctx, builder, 1, intValue);
+    nix_make_list(ctx, builder, value);
+    nix_list_builder_free(builder);
+
+    // Test extremely large indices that would definitely crash without bounds checking
+    ASSERT_EQ(nullptr, nix_get_list_byidx(ctx, value, state, 1000000));
+    ASSERT_EQ(NIX_ERR_KEY, nix_err_code(ctx));
+    ASSERT_EQ(nullptr, nix_get_list_byidx(ctx, value, state, UINT_MAX / 2));
+    ASSERT_EQ(NIX_ERR_KEY, nix_err_code(ctx));
+    ASSERT_EQ(nullptr, nix_get_list_byidx(ctx, value, state, UINT_MAX / 2 + 1000000));
+    ASSERT_EQ(NIX_ERR_KEY, nix_err_code(ctx));
+
+    // Clean up
+    nix_gc_decref(ctx, intValue);
+}
+
+TEST_F(nix_api_expr_test, nix_get_list_byidx_lazy)
+{
+    // Create a list with a throwing lazy element, an already-evaluated int, and a lazy function call
+
+    // 1. Throwing lazy element - create a function application thunk that will throw when forced
+    nix_value * throwingFn = nix_alloc_value(ctx, state);
+    nix_value * throwingValue = nix_alloc_value(ctx, state);
+
+    nix_expr_eval_from_string(
+        ctx,
+        state,
+        R"(
+        _: throw "This should not be evaluated by the lazy accessor"
+    )",
+        "<test>",
+        throwingFn);
+    assert_ctx_ok();
+
+    nix_init_apply(ctx, throwingValue, throwingFn, throwingFn);
+    assert_ctx_ok();
+
+    // 2. Already evaluated int (not lazy)
+    nix_value * intValue = nix_alloc_value(ctx, state);
+    nix_init_int(ctx, intValue, 42);
+    assert_ctx_ok();
+
+    // 3. Lazy function application that would compute increment 5 = 6
+    nix_value * lazyApply = nix_alloc_value(ctx, state);
+    nix_value * incrementFn = nix_alloc_value(ctx, state);
+    nix_value * argFive = nix_alloc_value(ctx, state);
+
+    nix_expr_eval_from_string(ctx, state, "x: x + 1", "<test>", incrementFn);
+    assert_ctx_ok();
+    nix_init_int(ctx, argFive, 5);
+
+    // Create a lazy application: (x: x + 1) 5
+    nix_init_apply(ctx, lazyApply, incrementFn, argFive);
+    assert_ctx_ok();
+
+    ListBuilder * builder = nix_make_list_builder(ctx, state, 3);
+    nix_list_builder_insert(ctx, builder, 0, throwingValue);
+    nix_list_builder_insert(ctx, builder, 1, intValue);
+    nix_list_builder_insert(ctx, builder, 2, lazyApply);
+    nix_make_list(ctx, builder, value);
+    nix_list_builder_free(builder);
+
+    // Test 1: Lazy accessor should return the throwing element without forcing evaluation
+    nix_value * lazyThrowingElement = nix_get_list_byidx_lazy(ctx, value, state, 0);
+    assert_ctx_ok();
+    ASSERT_NE(nullptr, lazyThrowingElement);
+
+    // Verify the element is still lazy by checking that forcing it throws
+    nix_value_force(ctx, state, lazyThrowingElement);
+    assert_ctx_err();
+    ASSERT_THAT(
+        nix_err_msg(nullptr, ctx, nullptr), testing::HasSubstr("This should not be evaluated by the lazy accessor"));
+
+    // Test 2: Lazy accessor should return the already-evaluated int
+    nix_value * intElement = nix_get_list_byidx_lazy(ctx, value, state, 1);
+    assert_ctx_ok();
+    ASSERT_NE(nullptr, intElement);
+    ASSERT_EQ(42, nix_get_int(ctx, intElement));
+
+    // Test 3: Lazy accessor should return the lazy function application without forcing
+    nix_value * lazyFunctionElement = nix_get_list_byidx_lazy(ctx, value, state, 2);
+    assert_ctx_ok();
+    ASSERT_NE(nullptr, lazyFunctionElement);
+
+    // Force the lazy function application - should compute 5 + 1 = 6
+    nix_value_force(ctx, state, lazyFunctionElement);
+    assert_ctx_ok();
+    ASSERT_EQ(6, nix_get_int(ctx, lazyFunctionElement));
+
+    // Clean up
+    nix_gc_decref(ctx, throwingFn);
+    nix_gc_decref(ctx, throwingValue);
+    nix_gc_decref(ctx, intValue);
+    nix_gc_decref(ctx, lazyApply);
+    nix_gc_decref(ctx, incrementFn);
+    nix_gc_decref(ctx, argFive);
+    nix_gc_decref(ctx, lazyThrowingElement);
+    nix_gc_decref(ctx, intElement);
+    nix_gc_decref(ctx, lazyFunctionElement);
+}
+
 TEST_F(nix_api_expr_test, nix_build_and_init_attr_invalid)
 {
     ASSERT_EQ(nullptr, nix_get_attr_byname(ctx, nullptr, state, 0));
@@ -244,6 +352,225 @@ TEST_F(nix_api_expr_test, nix_build_and_init_attr)
     free(out_name);
 }
 
+TEST_F(nix_api_expr_test, nix_get_attr_byidx_large_indices)
+{
+    // Create a small attribute set to test extremely large out-of-bounds access
+    const char ** out_name = (const char **) malloc(sizeof(char *));
+    BindingsBuilder * builder = nix_make_bindings_builder(ctx, state, 2);
+    nix_value * intValue = nix_alloc_value(ctx, state);
+    nix_init_int(ctx, intValue, 42);
+    nix_bindings_builder_insert(ctx, builder, "test", intValue);
+    nix_make_attrs(ctx, value, builder);
+    nix_bindings_builder_free(builder);
+
+    // Test extremely large indices that would definitely crash without bounds checking
+    ASSERT_EQ(nullptr, nix_get_attr_byidx(ctx, value, state, 1000000, out_name));
+    ASSERT_EQ(NIX_ERR_KEY, nix_err_code(ctx));
+    ASSERT_EQ(nullptr, nix_get_attr_byidx(ctx, value, state, UINT_MAX / 2, out_name));
+    ASSERT_EQ(NIX_ERR_KEY, nix_err_code(ctx));
+    ASSERT_EQ(nullptr, nix_get_attr_byidx(ctx, value, state, UINT_MAX / 2 + 1000000, out_name));
+    ASSERT_EQ(NIX_ERR_KEY, nix_err_code(ctx));
+
+    // Test nix_get_attr_name_byidx with large indices too
+    ASSERT_EQ(nullptr, nix_get_attr_name_byidx(ctx, value, state, 1000000));
+    ASSERT_EQ(NIX_ERR_KEY, nix_err_code(ctx));
+    ASSERT_EQ(nullptr, nix_get_attr_name_byidx(ctx, value, state, UINT_MAX / 2));
+    ASSERT_EQ(NIX_ERR_KEY, nix_err_code(ctx));
+    ASSERT_EQ(nullptr, nix_get_attr_name_byidx(ctx, value, state, UINT_MAX / 2 + 1000000));
+    ASSERT_EQ(NIX_ERR_KEY, nix_err_code(ctx));
+
+    // Clean up
+    nix_gc_decref(ctx, intValue);
+    free(out_name);
+}
+
+TEST_F(nix_api_expr_test, nix_get_attr_byname_lazy)
+{
+    // Create an attribute set with a throwing lazy attribute, an already-evaluated int, and a lazy function call
+
+    // 1. Throwing lazy element - create a function application thunk that will throw when forced
+    nix_value * throwingFn = nix_alloc_value(ctx, state);
+    nix_value * throwingValue = nix_alloc_value(ctx, state);
+
+    nix_expr_eval_from_string(
+        ctx,
+        state,
+        R"(
+        _: throw "This should not be evaluated by the lazy accessor"
+    )",
+        "<test>",
+        throwingFn);
+    assert_ctx_ok();
+
+    nix_init_apply(ctx, throwingValue, throwingFn, throwingFn);
+    assert_ctx_ok();
+
+    // 2. Already evaluated int (not lazy)
+    nix_value * intValue = nix_alloc_value(ctx, state);
+    nix_init_int(ctx, intValue, 42);
+    assert_ctx_ok();
+
+    // 3. Lazy function application that would compute increment 7 = 8
+    nix_value * lazyApply = nix_alloc_value(ctx, state);
+    nix_value * incrementFn = nix_alloc_value(ctx, state);
+    nix_value * argSeven = nix_alloc_value(ctx, state);
+
+    nix_expr_eval_from_string(ctx, state, "x: x + 1", "<test>", incrementFn);
+    assert_ctx_ok();
+    nix_init_int(ctx, argSeven, 7);
+
+    // Create a lazy application: (x: x + 1) 7
+    nix_init_apply(ctx, lazyApply, incrementFn, argSeven);
+    assert_ctx_ok();
+
+    BindingsBuilder * builder = nix_make_bindings_builder(ctx, state, 3);
+    nix_bindings_builder_insert(ctx, builder, "throwing", throwingValue);
+    nix_bindings_builder_insert(ctx, builder, "normal", intValue);
+    nix_bindings_builder_insert(ctx, builder, "lazy", lazyApply);
+    nix_make_attrs(ctx, value, builder);
+    nix_bindings_builder_free(builder);
+
+    // Test 1: Lazy accessor should return the throwing attribute without forcing evaluation
+    nix_value * lazyThrowingAttr = nix_get_attr_byname_lazy(ctx, value, state, "throwing");
+    assert_ctx_ok();
+    ASSERT_NE(nullptr, lazyThrowingAttr);
+
+    // Verify the attribute is still lazy by checking that forcing it throws
+    nix_value_force(ctx, state, lazyThrowingAttr);
+    assert_ctx_err();
+    ASSERT_THAT(
+        nix_err_msg(nullptr, ctx, nullptr), testing::HasSubstr("This should not be evaluated by the lazy accessor"));
+
+    // Test 2: Lazy accessor should return the already-evaluated int
+    nix_value * intAttr = nix_get_attr_byname_lazy(ctx, value, state, "normal");
+    assert_ctx_ok();
+    ASSERT_NE(nullptr, intAttr);
+    ASSERT_EQ(42, nix_get_int(ctx, intAttr));
+
+    // Test 3: Lazy accessor should return the lazy function application without forcing
+    nix_value * lazyFunctionAttr = nix_get_attr_byname_lazy(ctx, value, state, "lazy");
+    assert_ctx_ok();
+    ASSERT_NE(nullptr, lazyFunctionAttr);
+
+    // Force the lazy function application - should compute 7 + 1 = 8
+    nix_value_force(ctx, state, lazyFunctionAttr);
+    assert_ctx_ok();
+    ASSERT_EQ(8, nix_get_int(ctx, lazyFunctionAttr));
+
+    // Test 4: Missing attribute should return NULL with NIX_ERR_KEY
+    nix_value * missingAttr = nix_get_attr_byname_lazy(ctx, value, state, "nonexistent");
+    ASSERT_EQ(nullptr, missingAttr);
+    ASSERT_EQ(NIX_ERR_KEY, nix_err_code(ctx));
+
+    // Clean up
+    nix_gc_decref(ctx, throwingFn);
+    nix_gc_decref(ctx, throwingValue);
+    nix_gc_decref(ctx, intValue);
+    nix_gc_decref(ctx, lazyApply);
+    nix_gc_decref(ctx, incrementFn);
+    nix_gc_decref(ctx, argSeven);
+    nix_gc_decref(ctx, lazyThrowingAttr);
+    nix_gc_decref(ctx, intAttr);
+    nix_gc_decref(ctx, lazyFunctionAttr);
+}
+
+TEST_F(nix_api_expr_test, nix_get_attr_byidx_lazy)
+{
+    // Create an attribute set with a throwing lazy attribute, an already-evaluated int, and a lazy function call
+
+    // 1. Throwing lazy element - create a function application thunk that will throw when forced
+    nix_value * throwingFn = nix_alloc_value(ctx, state);
+    nix_value * throwingValue = nix_alloc_value(ctx, state);
+
+    nix_expr_eval_from_string(
+        ctx,
+        state,
+        R"(
+        _: throw "This should not be evaluated by the lazy accessor"
+    )",
+        "<test>",
+        throwingFn);
+    assert_ctx_ok();
+
+    nix_init_apply(ctx, throwingValue, throwingFn, throwingFn);
+    assert_ctx_ok();
+
+    // 2. Already evaluated int (not lazy)
+    nix_value * intValue = nix_alloc_value(ctx, state);
+    nix_init_int(ctx, intValue, 99);
+    assert_ctx_ok();
+
+    // 3. Lazy function application that would compute increment 10 = 11
+    nix_value * lazyApply = nix_alloc_value(ctx, state);
+    nix_value * incrementFn = nix_alloc_value(ctx, state);
+    nix_value * argTen = nix_alloc_value(ctx, state);
+
+    nix_expr_eval_from_string(ctx, state, "x: x + 1", "<test>", incrementFn);
+    assert_ctx_ok();
+    nix_init_int(ctx, argTen, 10);
+
+    // Create a lazy application: (x: x + 1) 10
+    nix_init_apply(ctx, lazyApply, incrementFn, argTen);
+    assert_ctx_ok();
+
+    BindingsBuilder * builder = nix_make_bindings_builder(ctx, state, 3);
+    nix_bindings_builder_insert(ctx, builder, "a_throwing", throwingValue);
+    nix_bindings_builder_insert(ctx, builder, "b_normal", intValue);
+    nix_bindings_builder_insert(ctx, builder, "c_lazy", lazyApply);
+    nix_make_attrs(ctx, value, builder);
+    nix_bindings_builder_free(builder);
+
+    // Proper usage: first get the size and gather all attributes into a map
+    unsigned int attrCount = nix_get_attrs_size(ctx, value);
+    assert_ctx_ok();
+    ASSERT_EQ(3u, attrCount);
+
+    // Gather all attributes into a map (proper contract usage)
+    std::map<std::string, nix_value *> attrMap;
+    const char * name;
+
+    for (unsigned int i = 0; i < attrCount; i++) {
+        nix_value * attr = nix_get_attr_byidx_lazy(ctx, value, state, i, &name);
+        assert_ctx_ok();
+        ASSERT_NE(nullptr, attr);
+        attrMap[std::string(name)] = attr;
+    }
+
+    // Now test the gathered attributes
+    ASSERT_EQ(3u, attrMap.size());
+    ASSERT_TRUE(attrMap.count("a_throwing"));
+    ASSERT_TRUE(attrMap.count("b_normal"));
+    ASSERT_TRUE(attrMap.count("c_lazy"));
+
+    // Test 1: Throwing attribute should be lazy
+    nix_value * throwingAttr = attrMap["a_throwing"];
+    nix_value_force(ctx, state, throwingAttr);
+    assert_ctx_err();
+    ASSERT_THAT(
+        nix_err_msg(nullptr, ctx, nullptr), testing::HasSubstr("This should not be evaluated by the lazy accessor"));
+
+    // Test 2: Normal attribute should be already evaluated
+    nix_value * normalAttr = attrMap["b_normal"];
+    ASSERT_EQ(99, nix_get_int(ctx, normalAttr));
+
+    // Test 3: Lazy function should compute when forced
+    nix_value * lazyAttr = attrMap["c_lazy"];
+    nix_value_force(ctx, state, lazyAttr);
+    assert_ctx_ok();
+    ASSERT_EQ(11, nix_get_int(ctx, lazyAttr));
+
+    // Clean up
+    nix_gc_decref(ctx, throwingFn);
+    nix_gc_decref(ctx, throwingValue);
+    nix_gc_decref(ctx, intValue);
+    nix_gc_decref(ctx, lazyApply);
+    nix_gc_decref(ctx, incrementFn);
+    nix_gc_decref(ctx, argTen);
+    for (auto & pair : attrMap) {
+        nix_gc_decref(ctx, pair.second);
+    }
+}
+
 TEST_F(nix_api_expr_test, nix_value_init)
 {
     // Setup

src/libexpr-tests/package.nix

@@ -62,6 +62,7 @@ mkMesonExecutable (finalAttrs: {
               mkdir -p "$HOME"
             ''
             + ''
+              export ASAN_OPTIONS=abort_on_error=1:print_summary=1:detect_leaks=0
               export _NIX_TEST_UNIT_DATA=${resolvePath ./data}
               ${stdenv.hostPlatform.emulator buildPackages} ${lib.getExe finalAttrs.finalPackage}
               touch $out

src/libexpr-tests/primops.cc

@@ -195,18 +195,18 @@ TEST_F(PrimOpTest, unsafeGetAttrPos)
     auto v = eval(expr);
     ASSERT_THAT(v, IsAttrsOfSize(3));
 
-    auto file = v.attrs()->find(createSymbol("file"));
+    auto file = v.attrs()->get(createSymbol("file"));
     ASSERT_NE(file, nullptr);
     ASSERT_THAT(*file->value, IsString());
     auto s = baseNameOf(file->value->string_view());
     ASSERT_EQ(s, "foo.nix");
 
-    auto line = v.attrs()->find(createSymbol("line"));
+    auto line = v.attrs()->get(createSymbol("line"));
     ASSERT_NE(line, nullptr);
     state.forceValue(*line->value, noPos);
     ASSERT_THAT(*line->value, IsIntEq(4));
 
-    auto column = v.attrs()->find(createSymbol("column"));
+    auto column = v.attrs()->get(createSymbol("column"));
     ASSERT_NE(column, nullptr);
     state.forceValue(*column->value, noPos);
     ASSERT_THAT(*column->value, IsIntEq(3));
@@ -246,7 +246,7 @@ TEST_F(PrimOpTest, removeAttrsRetains)
 {
     auto v = eval("builtins.removeAttrs { x = 1; y = 2; } [\"x\"]");
     ASSERT_THAT(v, IsAttrsOfSize(1));
-    ASSERT_NE(v.attrs()->find(createSymbol("y")), nullptr);
+    ASSERT_NE(v.attrs()->get(createSymbol("y")), nullptr);
 }
 
 TEST_F(PrimOpTest, listToAttrsEmptyList)
@@ -266,7 +266,7 @@ TEST_F(PrimOpTest, listToAttrs)
 {
     auto v = eval("builtins.listToAttrs [ { name = \"key\"; value = 123; } ]");
     ASSERT_THAT(v, IsAttrsOfSize(1));
-    auto key = v.attrs()->find(createSymbol("key"));
+    auto key = v.attrs()->get(createSymbol("key"));
     ASSERT_NE(key, nullptr);
     ASSERT_THAT(*key->value, IsIntEq(123));
 }
@@ -275,7 +275,7 @@ TEST_F(PrimOpTest, intersectAttrs)
 {
     auto v = eval("builtins.intersectAttrs { a = 1; b = 2; } { b = 3; c = 4; }");
     ASSERT_THAT(v, IsAttrsOfSize(1));
-    auto b = v.attrs()->find(createSymbol("b"));
+    auto b = v.attrs()->get(createSymbol("b"));
     ASSERT_NE(b, nullptr);
     ASSERT_THAT(*b->value, IsIntEq(3));
 }
@@ -293,11 +293,11 @@ TEST_F(PrimOpTest, functionArgs)
     auto v = eval("builtins.functionArgs ({ x, y ? 123}: 1)");
     ASSERT_THAT(v, IsAttrsOfSize(2));
 
-    auto x = v.attrs()->find(createSymbol("x"));
+    auto x = v.attrs()->get(createSymbol("x"));
     ASSERT_NE(x, nullptr);
     ASSERT_THAT(*x->value, IsFalse());
 
-    auto y = v.attrs()->find(createSymbol("y"));
+    auto y = v.attrs()->get(createSymbol("y"));
     ASSERT_NE(y, nullptr);
     ASSERT_THAT(*y->value, IsTrue());
 }
@@ -307,13 +307,13 @@ TEST_F(PrimOpTest, mapAttrs)
     auto v = eval("builtins.mapAttrs (name: value: value * 10) { a = 1; b = 2; }");
     ASSERT_THAT(v, IsAttrsOfSize(2));
 
-    auto a = v.attrs()->find(createSymbol("a"));
+    auto a = v.attrs()->get(createSymbol("a"));
     ASSERT_NE(a, nullptr);
     ASSERT_THAT(*a->value, IsThunk());
     state.forceValue(*a->value, noPos);
     ASSERT_THAT(*a->value, IsIntEq(10));
 
-    auto b = v.attrs()->find(createSymbol("b"));
+    auto b = v.attrs()->get(createSymbol("b"));
     ASSERT_NE(b, nullptr);
     ASSERT_THAT(*b->value, IsThunk());
     state.forceValue(*b->value, noPos);
@@ -642,7 +642,7 @@ class ToStringPrimOpTest : public PrimOpTest,
 
 TEST_P(ToStringPrimOpTest, toString)
 {
-    const auto [input, output] = GetParam();
+    const auto & [input, output] = GetParam();
     auto v = eval(input);
     ASSERT_THAT(v, IsStringEq(output));
 }
@@ -798,7 +798,7 @@ class CompareVersionsPrimOpTest : public PrimOpTest,
 
 TEST_P(CompareVersionsPrimOpTest, compareVersions)
 {
-    auto [expression, expectation] = GetParam();
+    const auto & [expression, expectation] = GetParam();
     auto v = eval(expression);
     ASSERT_THAT(v, IsIntEq(expectation));
 }
@@ -834,16 +834,16 @@ class ParseDrvNamePrimOpTest
 
 TEST_P(ParseDrvNamePrimOpTest, parseDrvName)
 {
-    auto [input, expectedName, expectedVersion] = GetParam();
+    const auto & [input, expectedName, expectedVersion] = GetParam();
     const auto expr = fmt("builtins.parseDrvName \"%1%\"", input);
     auto v = eval(expr);
     ASSERT_THAT(v, IsAttrsOfSize(2));
 
-    auto name = v.attrs()->find(createSymbol("name"));
+    auto name = v.attrs()->get(createSymbol("name"));
     ASSERT_TRUE(name);
     ASSERT_THAT(*name->value, IsStringEq(expectedName));
 
-    auto version = v.attrs()->find(createSymbol("version"));
+    auto version = v.attrs()->get(createSymbol("version"));
     ASSERT_TRUE(version);
     ASSERT_THAT(*version->value, IsStringEq(expectedVersion));
 }

src/libexpr-tests/trivial.cc

@@ -75,11 +75,11 @@ TEST_F(TrivialExpressionTest, updateAttrs)
 {
     auto v = eval("{ a = 1; } // { b = 2; a = 3; }");
     ASSERT_THAT(v, IsAttrsOfSize(2));
-    auto a = v.attrs()->find(createSymbol("a"));
+    auto a = v.attrs()->get(createSymbol("a"));
     ASSERT_NE(a, nullptr);
     ASSERT_THAT(*a->value, IsIntEq(3));
 
-    auto b = v.attrs()->find(createSymbol("b"));
+    auto b = v.attrs()->get(createSymbol("b"));
     ASSERT_NE(b, nullptr);
     ASSERT_THAT(*b->value, IsIntEq(2));
 }
@@ -176,7 +176,7 @@ TEST_P(AttrSetMergeTrvialExpressionTest, attrsetMergeLazy)
     auto v = eval(expr);
     ASSERT_THAT(v, IsAttrsOfSize(1));
 
-    auto a = v.attrs()->find(createSymbol("a"));
+    auto a = v.attrs()->get(createSymbol("a"));
     ASSERT_NE(a, nullptr);
 
     ASSERT_THAT(*a->value, IsThunk());
@@ -184,11 +184,11 @@ TEST_P(AttrSetMergeTrvialExpressionTest, attrsetMergeLazy)
 
     ASSERT_THAT(*a->value, IsAttrsOfSize(2));
 
-    auto b = a->value->attrs()->find(createSymbol("b"));
+    auto b = a->value->attrs()->get(createSymbol("b"));
     ASSERT_NE(b, nullptr);
     ASSERT_THAT(*b->value, IsIntEq(1));
 
-    auto c = a->value->attrs()->find(createSymbol("c"));
+    auto c = a->value->attrs()->get(createSymbol("c"));
     ASSERT_NE(c, nullptr);
     ASSERT_THAT(*c->value, IsIntEq(2));
 }
@@ -330,7 +330,7 @@ TEST_F(TrivialExpressionTest, bindOr)
 {
     auto v = eval("{ or = 1; }");
     ASSERT_THAT(v, IsAttrsOfSize(1));
-    auto b = v.attrs()->find(createSymbol("or"));
+    auto b = v.attrs()->get(createSymbol("or"));
     ASSERT_NE(b, nullptr);
     ASSERT_THAT(*b->value, IsIntEq(1));
 }

src/libexpr/attr-path.cc

@@ -110,8 +110,8 @@ std::pair<SourcePath, uint32_t> findPackageFilename(EvalState & state, Value & v
 {
     Value * v2;
     try {
-        auto dummyArgs = state.allocBindings(0);
-        v2 = findAlongAttrPath(state, "meta.position", *dummyArgs, v).first;
+        auto & dummyArgs = Bindings::emptyBindings;
+        v2 = findAlongAttrPath(state, "meta.position", dummyArgs, v).first;
     } catch (Error &) {
         throw NoPositionInfo("package '%s' has no source location information", what);
     }

src/libexpr/attr-set.cc

@@ -5,36 +5,37 @@
 
 namespace nix {
 
+Bindings Bindings::emptyBindings;
+
 /* Allocate a new array of attributes for an attribute set with a specific
    capacity. The space is implicitly reserved after the Bindings
    structure. */
-Bindings * EvalState::allocBindings(size_t capacity)
+Bindings * EvalMemory::allocBindings(size_t capacity)
 {
     if (capacity == 0)
-        return &emptyBindings;
-    if (capacity > std::numeric_limits<Bindings::size_t>::max())
+        return &Bindings::emptyBindings;
+    if (capacity > std::numeric_limits<Bindings::size_type>::max())
         throw Error("attribute set of size %d is too big", capacity);
-    nrAttrsets++;
-    nrAttrsInAttrsets += capacity;
+    stats.nrAttrsets++;
+    stats.nrAttrsInAttrsets += capacity;
     return new (allocBytes(sizeof(Bindings) + sizeof(Attr) * capacity)) Bindings();
 }
 
 Value & BindingsBuilder::alloc(Symbol name, PosIdx pos)
 {
-    auto value = state.get().allocValue();
+    auto value = mem.get().allocValue();
     bindings->push_back(Attr(name, value, pos));
     return *value;
 }
 
 Value & BindingsBuilder::alloc(std::string_view name, PosIdx pos)
 {
-    return alloc(state.get().symbols.create(name), pos);
+    return alloc(symbols.get().create(name), pos);
 }
 
 void Bindings::sort()
 {
-    if (size_)
-        std::sort(begin(), end());
+    std::sort(attrs, attrs + numAttrs);
 }
 
 Value & Value::mkAttrs(BindingsBuilder & bindings)

src/libexpr/eval-error.cc

@@ -113,6 +113,5 @@ template class EvalErrorBuilder<MissingArgumentError>;
 template class EvalErrorBuilder<InfiniteRecursionError>;
 template class EvalErrorBuilder<InvalidPathError>;
 template class EvalErrorBuilder<IFDError>;
-template class EvalErrorBuilder<RecoverableEvalError>;
 
 } // namespace nix

src/libexpr/eval-gc.cc

@@ -24,6 +24,10 @@
 
 #endif
 
+namespace nix {
+
+#if NIX_USE_BOEHMGC
+
 /*
  * Ensure that Boehm satisfies our alignment requirements. This is the default configuration [^]
  * and this assertion should never break for any platform. Let's assert it just in case.
@@ -35,9 +39,6 @@
  */
 static_assert(sizeof(void *) * 2 == GC_GRANULE_BYTES, "Boehm GC must use GC_GRANULE_WORDS = 2");
 
-namespace nix {
-
-#if NIX_USE_BOEHMGC
 /* Called when the Boehm GC runs out of memory. */
 static void * oomHandler(size_t requested)
 {

src/libexpr/eval-profiler.cc

@@ -324,7 +324,7 @@ void SampleStack::saveProfile()
             std::visit([&](auto && info) { info.symbolize(state, os, posCache); }, pos);
         }
         os << " " << count;
-        writeLine(profileFd.get(), std::move(os).str());
+        writeLine(profileFd.get(), os.str());
         /* Clear ostringstream. */
         os.str("");
         os.clear();

src/libexpr/eval.cc

@@ -1,5 +1,4 @@
 #include "nix/expr/eval.hh"
-#include "nix/expr/eval-error.hh"
 #include "nix/expr/eval-settings.hh"
 #include "nix/expr/primops.hh"
 #include "nix/expr/print-options.hh"
@@ -18,6 +17,7 @@
 #include "nix/expr/print.hh"
 #include "nix/fetchers/filtering-source-accessor.hh"
 #include "nix/util/memory-source-accessor.hh"
+#include "nix/util/mounted-source-accessor.hh"
 #include "nix/expr/gc-small-vector.hh"
 #include "nix/util/url.hh"
 #include "nix/fetchers/fetch-to-store.hh"
@@ -28,7 +28,6 @@
 #include "parser-tab.hh"
 
 #include <algorithm>
-#include <exception>
 #include <iostream>
 #include <sstream>
 #include <cstring>
@@ -40,6 +39,7 @@
 
 #include <nlohmann/json.hpp>
 #include <boost/container/small_vector.hpp>
+#include <boost/unordered/concurrent_flat_map.hpp>
 
 #include "nix/util/strings-inline.hh"
 
@@ -126,8 +126,6 @@ std::string_view showType(ValueType type, bool withArticle)
         return WA("a", "float");
     case nThunk:
         return WA("a", "thunk");
-    case nFailed:
-        return WA("a", "failure");
     }
     unreachable();
 }
@@ -196,6 +194,15 @@ static Symbol getName(const AttrName & name, EvalState & state, Env & env)
 
 static constexpr size_t BASE_ENV_SIZE = 128;
 
+EvalMemory::EvalMemory()
+#if NIX_USE_BOEHMGC
+    : valueAllocCache(std::allocate_shared<void *>(traceable_allocator<void *>(), nullptr))
+    , env1AllocCache(std::allocate_shared<void *>(traceable_allocator<void *>(), nullptr))
+#endif
+{
+    assertGCInitialized();
+}
+
 EvalState::EvalState(
     const LookupPath & lookupPathFromArguments,
     ref<Store> store,
@@ -206,7 +213,6 @@ EvalState::EvalState(
     , settings{settings}
     , symbols(StaticEvalSymbols::staticSymbolTable())
     , repair(NoRepair)
-    , emptyBindings(Bindings())
     , storeFS(makeMountedSourceAccessor({
           {CanonPath::root, makeEmptySourceAccessor()},
           /* In the pure eval case, we can simply require
@@ -229,22 +235,18 @@ EvalState::EvalState(
            */
           {CanonPath(store->storeDir), store->getFSAccessor(settings.pureEval)},
       }))
-    , rootFS(({
+    , rootFS([&] {
         /* In pure eval mode, we provide a filesystem that only
            contains the Nix store.
 
-           If we have a chroot store and pure eval is not enabled,
-           use a union accessor to make the chroot store available
-           at its logical location while still having the
+           Otherwise, use a union accessor to make the augmented store
+           available at its logical location while still having the
            underlying directory available. This is necessary for
            instance if we're evaluating a file from the physical
-           /nix/store while using a chroot store. */
-        auto accessor = getFSSourceAccessor();
-
-        auto realStoreDir = dirOf(store->toRealPath(StorePath::dummy));
-        if (settings.pureEval || store->storeDir != realStoreDir) {
-            accessor = settings.pureEval ? storeFS : makeUnionSourceAccessor({accessor, storeFS});
-        }
+           /nix/store while using a chroot store, and also for lazy
+           mounted fetchTree. */
+        auto accessor = settings.pureEval ? storeFS.cast<SourceAccessor>()
+                                          : makeUnionSourceAccessor({getFSSourceAccessor(), storeFS});
 
         /* Apply access control if needed. */
         if (settings.restrictEval || settings.pureEval)
@@ -255,11 +257,11 @@ EvalState::EvalState(
                     throw RestrictedPathError("access to absolute path '%1%' is forbidden %2%", path, modeInformation);
                 });
 
-        accessor;
-    }))
+        return accessor;
+    }())
     , corepkgsFS(make_ref<MemorySourceAccessor>())
     , internalFS(make_ref<MemorySourceAccessor>())
-    , derivationInternal{corepkgsFS->addFile(
+    , derivationInternal{internalFS->addFile(
           CanonPath("derivation-internal.nix"),
 #include "primops/derivation.nix.gen.hh"
           )}
@@ -269,14 +271,15 @@ EvalState::EvalState(
     , debugRepl(nullptr)
     , debugStop(false)
     , trylevel(0)
+    , srcToStore(make_ref<decltype(srcToStore)::element_type>())
+    , importResolutionCache(make_ref<decltype(importResolutionCache)::element_type>())
+    , fileEvalCache(make_ref<decltype(fileEvalCache)::element_type>())
     , regexCache(makeRegexCache())
 #if NIX_USE_BOEHMGC
-    , valueAllocCache(std::allocate_shared<void *>(traceable_allocator<void *>(), nullptr))
-    , env1AllocCache(std::allocate_shared<void *>(traceable_allocator<void *>(), nullptr))
-    , baseEnvP(std::allocate_shared<Env *>(traceable_allocator<Env *>(), &allocEnv(BASE_ENV_SIZE)))
+    , baseEnvP(std::allocate_shared<Env *>(traceable_allocator<Env *>(), &mem.allocEnv(BASE_ENV_SIZE)))
     , baseEnv(**baseEnvP)
 #else
-    , baseEnv(allocEnv(BASE_ENV_SIZE))
+    , baseEnv(mem.allocEnv(BASE_ENV_SIZE))
 #endif
     , staticBaseEnv{std::make_shared<StaticEnv>(nullptr, nullptr)}
 {
@@ -285,18 +288,8 @@ EvalState::EvalState(
 
     countCalls = getEnv("NIX_COUNT_CALLS").value_or("0") != "0";
 
-    assertGCInitialized();
-
     static_assert(sizeof(Env) <= 16, "environment must be <= 16 bytes");
-
-    vEmptyList.mkList(buildList(0));
-    vNull.mkNull();
-    vTrue.mkBool(true);
-    vFalse.mkBool(false);
-    vStringRegular.mkStringNoCopy("regular");
-    vStringDirectory.mkStringNoCopy("directory");
-    vStringSymlink.mkStringNoCopy("symlink");
-    vStringUnknown.mkStringNoCopy("unknown");
+    static_assert(sizeof(Counter) == 64, "counters must be 64 bytes");
 
     /* Construct the Nix expression search path. */
     assert(lookupPath.elements.empty());
@@ -343,7 +336,7 @@ EvalState::EvalState(
 
 EvalState::~EvalState() {}
 
-void EvalState::allowPath(const Path & path)
+void EvalState::allowPathLegacy(const Path & path)
 {
     if (auto rootFS2 = rootFS.dynamic_pointer_cast<AllowListSourceAccessor>())
         rootFS2->allowPrefix(CanonPath(path));
@@ -591,12 +584,12 @@ std::optional<EvalState::Doc> EvalState::getDoc(Value & v)
             .name = name,
             .arity = 0, // FIXME: figure out how deep by syntax only? It's not semantically useful though...
             .args = {},
-            .doc = makeImmutableString(toView(s)), // NOTE: memory leak when compiled without GC
+            .doc = makeImmutableString(s.view()), // NOTE: memory leak when compiled without GC
         };
     }
     if (isFunctor(v)) {
         try {
-            Value & functor = *v.attrs()->find(s.functor)->value;
+            Value & functor = *v.attrs()->get(s.functor)->value;
             Value * vp[] = {&v};
             Value partiallyApplied;
             // The first parameter is not user-provided, and may be
@@ -890,19 +883,18 @@ inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval)
     }
 }
 
-ListBuilder::ListBuilder(EvalState & state, size_t size)
+ListBuilder::ListBuilder(size_t size)
     : size(size)
     , elems(size <= 2 ? inlineElems : (Value **) allocBytes(size * sizeof(Value *)))
 {
-    state.nrListElems += size;
 }
 
 Value * EvalState::getBool(bool b)
 {
-    return b ? &vTrue : &vFalse;
+    return b ? &Value::vTrue : &Value::vFalse;
 }
 
-unsigned long nrThunks = 0;
+static Counter nrThunks;
 
 static inline void mkThunk(Value & v, Env & env, Expr * expr)
 {
@@ -993,10 +985,6 @@ void EvalState::mkSingleDerivedPathString(const SingleDerivedPath & p, Value & v
         });
 }
 
-/* Create a thunk for the delayed computation of the given expression
-   in the given environment.  But if the expression is a variable,
-   then look it up right away.  This significantly reduces the number
-   of thunks allocated. */
 Value * Expr::maybeThunk(EvalState & state, Env & env)
 {
     Value * v = state.allocValue();
@@ -1040,61 +1028,90 @@ Value * ExprPath::maybeThunk(EvalState & state, Env & env)
     return &v;
 }
 
+/**
+ * A helper `Expr` class to lets us parse and evaluate Nix expressions
+ * from a thunk, ensuring that every file is parsed/evaluated only
+ * once (via the thunk stored in `EvalState::fileEvalCache`).
+ */
+struct ExprParseFile : Expr, gc
+{
+    // FIXME: make this a reference (see below).
+    SourcePath path;
+    bool mustBeTrivial;
+
+    ExprParseFile(SourcePath & path, bool mustBeTrivial)
+        : path(path)
+        , mustBeTrivial(mustBeTrivial)
+    {
+    }
+
+    void eval(EvalState & state, Env & env, Value & v) override
+    {
+        printTalkative("evaluating file '%s'", path);
+
+        auto e = state.parseExprFromFile(path);
+
+        try {
+            auto dts =
+                state.debugRepl
+                    ? makeDebugTraceStacker(
+                          state, *e, state.baseEnv, e->getPos(), "while evaluating the file '%s':", path.to_string())
+                    : nullptr;
+
+            // Enforce that 'flake.nix' is a direct attrset, not a
+            // computation.
+            if (mustBeTrivial && !(dynamic_cast<ExprAttrs *>(e)))
+                state.error<EvalError>("file '%s' must be an attribute set", path).debugThrow();
+
+            state.eval(e, v);
+        } catch (Error & e) {
+            state.addErrorTrace(e, "while evaluating the file '%s':", path.to_string());
+            throw;
+        }
+    }
+};
+
 void EvalState::evalFile(const SourcePath & path, Value & v, bool mustBeTrivial)
 {
-    FileEvalCache::iterator i;
-    if ((i = fileEvalCache.find(path)) != fileEvalCache.end()) {
-        v = i->second;
+    auto resolvedPath = getConcurrent(*importResolutionCache, path);
+
+    if (!resolvedPath) {
+        resolvedPath = resolveExprPath(path);
+        importResolutionCache->emplace(path, *resolvedPath);
+    }
+
+    if (auto v2 = getConcurrent(*fileEvalCache, *resolvedPath)) {
+        forceValue(**v2, noPos);
+        v = **v2;
         return;
     }
 
-    auto resolvedPath = resolveExprPath(path);
-    if ((i = fileEvalCache.find(resolvedPath)) != fileEvalCache.end()) {
-        v = i->second;
-        return;
-    }
+    Value * vExpr;
+    // FIXME: put ExprParseFile on the stack instead of the heap once
+    // https://github.com/NixOS/nix/pull/13930 is merged. That will ensure
+    // the post-condition that `expr` is unreachable after
+    // `forceValue()` returns.
+    auto expr = new ExprParseFile{*resolvedPath, mustBeTrivial};
 
-    printTalkative("evaluating file '%1%'", resolvedPath);
-    Expr * e = nullptr;
+    fileEvalCache->try_emplace_and_cvisit(
+        *resolvedPath,
+        nullptr,
+        [&](auto & i) {
+            vExpr = allocValue();
+            vExpr->mkThunk(&baseEnv, expr);
+            i.second = vExpr;
+        },
+        [&](auto & i) { vExpr = i.second; });
 
-    auto j = fileParseCache.find(resolvedPath);
-    if (j != fileParseCache.end())
-        e = j->second;
+    forceValue(*vExpr, noPos);
 
-    if (!e)
-        e = parseExprFromFile(resolvedPath);
-
-    fileParseCache.emplace(resolvedPath, e);
-
-    try {
-        auto dts = debugRepl ? makeDebugTraceStacker(
-                                   *this,
-                                   *e,
-                                   this->baseEnv,
-                                   e->getPos(),
-                                   "while evaluating the file '%1%':",
-                                   resolvedPath.to_string())
-                             : nullptr;
-
-        // Enforce that 'flake.nix' is a direct attrset, not a
-        // computation.
-        if (mustBeTrivial && !(dynamic_cast<ExprAttrs *>(e)))
-            error<EvalError>("file '%s' must be an attribute set", path).debugThrow();
-        eval(e, v);
-    } catch (Error & e) {
-        addErrorTrace(e, "while evaluating the file '%1%':", resolvedPath.to_string());
-        throw;
-    }
-
-    fileEvalCache.emplace(resolvedPath, v);
-    if (path != resolvedPath)
-        fileEvalCache.emplace(path, v);
+    v = *vExpr;
 }
 
 void EvalState::resetFileCache()
 {
-    fileEvalCache.clear();
-    fileParseCache.clear();
+    importResolutionCache->clear();
+    fileEvalCache->clear();
     inputCache->clear();
 }
 
@@ -1163,7 +1180,7 @@ void ExprPath::eval(EvalState & state, Env & env, Value & v)
 
 Env * ExprAttrs::buildInheritFromEnv(EvalState & state, Env & up)
 {
-    Env & inheritEnv = state.allocEnv(inheritFromExprs->size());
+    Env & inheritEnv = state.mem.allocEnv(inheritFromExprs->size());
     inheritEnv.up = &up;
 
     Displacement displ = 0;
@@ -1182,7 +1199,7 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v)
     if (recursive) {
         /* Create a new environment that contains the attributes in
            this `rec'. */
-        Env & env2(state.allocEnv(attrs.size()));
+        Env & env2(state.mem.allocEnv(attrs.size()));
         env2.up = &env;
         dynamicEnv = &env2;
         Env * inheritEnv = inheritFromExprs ? buildInheritFromEnv(state, env2) : nullptr;
@@ -1274,7 +1291,7 @@ void ExprLet::eval(EvalState & state, Env & env, Value & v)
 {
     /* Create a new environment that contains the attributes in this
        `let'. */
-    Env & env2(state.allocEnv(attrs->attrs.size()));
+    Env & env2(state.mem.allocEnv(attrs->attrs.size()));
     env2.up = &env;
 
     Env * inheritEnv = attrs->inheritFromExprs ? attrs->buildInheritFromEnv(state, env2) : nullptr;
@@ -1305,7 +1322,7 @@ void ExprList::eval(EvalState & state, Env & env, Value & v)
 Value * ExprList::maybeThunk(EvalState & state, Env & env)
 {
     if (elems.empty()) {
-        return &state.vEmptyList;
+        return &Value::vEmptyList;
     }
     return Expr::maybeThunk(state, env);
 }
@@ -1317,7 +1334,7 @@ void ExprVar::eval(EvalState & state, Env & env, Value & v)
     v = *v2;
 }
 
-static std::string showAttrPath(EvalState & state, Env & env, const AttrPath & attrPath)
+static std::string showAttrPath(EvalState & state, Env & env, std::span<const AttrName> attrPath)
 {
     std::ostringstream out;
     bool first = true;
@@ -1353,10 +1370,10 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v)
                                          env,
                                          getPos(),
                                          "while evaluating the attribute '%1%'",
-                                         showAttrPath(state, env, attrPath))
+                                         showAttrPath(state, env, getAttrPath()))
                                    : nullptr;
 
-        for (auto & i : attrPath) {
+        for (auto & i : getAttrPath()) {
             state.nrLookups++;
             const Attr * j;
             auto name = getName(i, state, env);
@@ -1394,7 +1411,7 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v)
             auto origin = std::get_if<SourcePath>(&pos2r.origin);
             if (!(origin && *origin == state.derivationInternal))
                 state.addErrorTrace(
-                    e, pos2, "while evaluating the attribute '%1%'", showAttrPath(state, env, attrPath));
+                    e, pos2, "while evaluating the attribute '%1%'", showAttrPath(state, env, getAttrPath()));
         }
         throw;
     }
@@ -1405,13 +1422,13 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v)
 Symbol ExprSelect::evalExceptFinalSelect(EvalState & state, Env & env, Value & attrs)
 {
     Value vTmp;
-    Symbol name = getName(attrPath[attrPath.size() - 1], state, env);
+    Symbol name = getName(attrPathStart[nAttrPath - 1], state, env);
 
-    if (attrPath.size() == 1) {
+    if (nAttrPath == 1) {
         e->eval(state, env, vTmp);
     } else {
         ExprSelect init(*this);
-        init.attrPath.pop_back();
+        init.nAttrPath--;
         init.eval(state, env, vTmp);
     }
     attrs = vTmp;
@@ -1480,7 +1497,7 @@ void EvalState::callFunction(Value & fun, std::span<Value *> args, Value & vRes,
             ExprLambda & lambda(*vCur.lambda().fun);
 
             auto size = (!lambda.arg ? 0 : 1) + (lambda.hasFormals() ? lambda.formals->formals.size() : 0);
-            Env & env2(allocEnv(size));
+            Env & env2(mem.allocEnv(size));
             env2.up = vCur.lambda().env;
 
             Displacement displ = 0;
@@ -1721,8 +1738,8 @@ void EvalState::autoCallFunction(const Bindings & args, Value & fun, Value & res
     forceValue(fun, pos);
 
     if (fun.type() == nAttrs) {
-        auto found = fun.attrs()->find(s.functor);
-        if (found != fun.attrs()->end()) {
+        auto found = fun.attrs()->get(s.functor);
+        if (found) {
             Value * v = allocValue();
             callFunction(*found->value, fun, *v, pos);
             forceValue(*v, pos);
@@ -1769,7 +1786,7 @@ https://nix.dev/manual/nix/stable/language/syntax.html#functions.)",
 
 void ExprWith::eval(EvalState & state, Env & env, Value & v)
 {
-    Env & env2(state.allocEnv(1));
+    Env & env2(state.mem.allocEnv(1));
     env2.up = &env;
     env2.values[0] = attrs->maybeThunk(state, env);
 
@@ -1787,7 +1804,7 @@ void ExprAssert::eval(EvalState & state, Env & env, Value & v)
     if (!state.evalBool(env, cond, pos, "in the condition of the assert statement")) {
         std::ostringstream out;
         cond->show(state.symbols, out);
-        auto exprStr = toView(out);
+        auto exprStr = out.view();
 
         if (auto eq = dynamic_cast<ExprOpEq *>(cond)) {
             try {
@@ -1851,51 +1868,113 @@ void ExprOpImpl::eval(EvalState & state, Env & env, Value & v)
         || state.evalBool(env, e2, pos, "in the right operand of the IMPL (->) operator"));
 }
 
-void ExprOpUpdate::eval(EvalState & state, Env & env, Value & v)
+void ExprOpUpdate::eval(EvalState & state, Value & v, Value & v1, Value & v2)
 {
-    Value v1, v2;
-    state.evalAttrs(env, e1, v1, pos, "in the left operand of the update (//) operator");
-    state.evalAttrs(env, e2, v2, pos, "in the right operand of the update (//) operator");
-
     state.nrOpUpdates++;
 
-    if (v1.attrs()->size() == 0) {
+    const Bindings & bindings1 = *v1.attrs();
+    if (bindings1.empty()) {
         v = v2;
         return;
     }
-    if (v2.attrs()->size() == 0) {
+
+    const Bindings & bindings2 = *v2.attrs();
+    if (bindings2.empty()) {
         v = v1;
         return;
     }
 
-    auto attrs = state.buildBindings(v1.attrs()->size() + v2.attrs()->size());
+    /* Simple heuristic for determining whether attrs2 should be "layered" on top of
+       attrs1 instead of copying to a new Bindings. */
+    bool shouldLayer = [&]() -> bool {
+        if (bindings1.isLayerListFull())
+            return false;
+
+        if (bindings2.size() > state.settings.bindingsUpdateLayerRhsSizeThreshold)
+            return false;
+
+        return true;
+    }();
+
+    if (shouldLayer) {
+        auto attrs = state.buildBindings(bindings2.size());
+        attrs.layerOnTopOf(bindings1);
+
+        std::ranges::copy(bindings2, std::back_inserter(attrs));
+        v.mkAttrs(attrs.alreadySorted());
+
+        state.nrOpUpdateValuesCopied += bindings2.size();
+        return;
+    }
+
+    auto attrs = state.buildBindings(bindings1.size() + bindings2.size());
 
     /* Merge the sets, preferring values from the second set.  Make
        sure to keep the resulting vector in sorted order. */
-    auto i = v1.attrs()->begin();
-    auto j = v2.attrs()->begin();
+    auto i = bindings1.begin();
+    auto j = bindings2.begin();
 
-    while (i != v1.attrs()->end() && j != v2.attrs()->end()) {
+    while (i != bindings1.end() && j != bindings2.end()) {
         if (i->name == j->name) {
             attrs.insert(*j);
             ++i;
             ++j;
-        } else if (i->name < j->name)
-            attrs.insert(*i++);
-        else
-            attrs.insert(*j++);
+        } else if (i->name < j->name) {
+            attrs.insert(*i);
+            ++i;
+        } else {
+            attrs.insert(*j);
+            ++j;
+        }
     }
 
-    while (i != v1.attrs()->end())
-        attrs.insert(*i++);
-    while (j != v2.attrs()->end())
-        attrs.insert(*j++);
+    while (i != bindings1.end()) {
+        attrs.insert(*i);
+        ++i;
+    }
+
+    while (j != bindings2.end()) {
+        attrs.insert(*j);
+        ++j;
+    }
 
     v.mkAttrs(attrs.alreadySorted());
 
     state.nrOpUpdateValuesCopied += v.attrs()->size();
 }
 
+void ExprOpUpdate::eval(EvalState & state, Env & env, Value & v)
+{
+    UpdateQueue q;
+    evalForUpdate(state, env, q);
+
+    v.mkAttrs(&Bindings::emptyBindings);
+    for (auto & rhs : std::views::reverse(q)) {
+        /* Remember that queue is sorted rightmost attrset first. */
+        eval(state, /*v=*/v, /*v1=*/v, /*v2=*/rhs);
+    }
+}
+
+void Expr::evalForUpdate(EvalState & state, Env & env, UpdateQueue & q, std::string_view errorCtx)
+{
+    Value v;
+    state.evalAttrs(env, this, v, getPos(), errorCtx);
+    q.push_back(v);
+}
+
+void ExprOpUpdate::evalForUpdate(EvalState & state, Env & env, UpdateQueue & q)
+{
+    /* Output rightmost attrset first to the merge queue as the one
+       with the most priority. */
+    e2->evalForUpdate(state, env, q, "in the right operand of the update (//) operator");
+    e1->evalForUpdate(state, env, q, "in the left operand of the update (//) operator");
+}
+
+void ExprOpUpdate::evalForUpdate(EvalState & state, Env & env, UpdateQueue & q, std::string_view errorCtx)
+{
+    evalForUpdate(state, env, q);
+}
+
 void ExprOpConcatLists::eval(EvalState & state, Env & env, Value & v)
 {
     Value v1;
@@ -2064,54 +2143,6 @@ void ExprBlackHole::eval(EvalState & state, [[maybe_unused]] Env & env, Value &
 // always force this to be separate, otherwise forceValue may inline it and take
 // a massive perf hit
 [[gnu::noinline]]
-void EvalState::handleEvalExceptionForThunk(Env * env, Expr * expr, Value & v, const PosIdx pos)
-{
-    v.mkThunk(env, expr);
-    tryFixupBlackHolePos(v, pos);
-
-    auto e = std::current_exception();
-    Value * recovery = nullptr;
-    try {
-        std::rethrow_exception(e);
-    } catch (const RecoverableEvalError & e) {
-        recovery = allocValue();
-    } catch (...) {
-    }
-    if (recovery) {
-        recovery->mkThunk(env, expr);
-    }
-    v.mkFailed(e, recovery);
-}
-
-[[gnu::noinline]]
-void EvalState::handleEvalExceptionForApp(Value & v)
-{
-    auto e = std::current_exception();
-    Value * recovery = nullptr;
-    try {
-        std::rethrow_exception(e);
-    } catch (const RecoverableEvalError & e) {
-        recovery = allocValue();
-    } catch (...) {
-    }
-    if (recovery) {
-        *recovery = v;
-    }
-    v.mkFailed(e, recovery);
-}
-
-[[gnu::noinline]]
-void EvalState::handleEvalFailed(Value & v, const PosIdx pos)
-{
-    assert(v.isFailed());
-    if (auto recoveryValue = v.failed()->recoveryValue) {
-        v = *recoveryValue;
-        forceValue(v, pos);
-    } else {
-        std::rethrow_exception(v.failed()->ex);
-    }
-}
-
 void EvalState::tryFixupBlackHolePos(Value & v, PosIdx pos)
 {
     if (!v.isBlackhole())
@@ -2120,8 +2151,7 @@ void EvalState::tryFixupBlackHolePos(Value & v, PosIdx pos)
     try {
         std::rethrow_exception(e);
     } catch (InfiniteRecursionError & e) {
-        if (!e.hasPos())
-            e.atPos(positions[pos]);
+        e.atPos(positions[pos]);
     } catch (...) {
     }
 }
@@ -2221,10 +2251,10 @@ bool EvalState::forceBool(Value & v, const PosIdx pos, std::string_view errorCtx
     return v.boolean();
 }
 
-Bindings::const_iterator EvalState::getAttr(Symbol attrSym, const Bindings * attrSet, std::string_view errorCtx)
+const Attr * EvalState::getAttr(Symbol attrSym, const Bindings * attrSet, std::string_view errorCtx)
 {
-    auto value = attrSet->find(attrSym);
-    if (value == attrSet->end()) {
+    auto value = attrSet->get(attrSym);
+    if (!value) {
         error<TypeError>("attribute '%s' missing", symbols[attrSym]).withTrace(noPos, errorCtx).debugThrow();
     }
     return value;
@@ -2232,7 +2262,7 @@ Bindings::const_iterator EvalState::getAttr(Symbol attrSym, const Bindings * att
 
 bool EvalState::isFunctor(const Value & fun) const
 {
-    return fun.type() == nAttrs && fun.attrs()->find(s.functor) != fun.attrs()->end();
+    return fun.type() == nAttrs && fun.attrs()->get(s.functor);
 }
 
 void EvalState::forceFunction(Value & v, const PosIdx pos, std::string_view errorCtx)
@@ -2313,8 +2343,8 @@ bool EvalState::isDerivation(Value & v)
 std::optional<std::string>
 EvalState::tryAttrsToString(const PosIdx pos, Value & v, NixStringContext & context, bool coerceMore, bool copyToStore)
 {
-    auto i = v.attrs()->find(s.toString);
-    if (i != v.attrs()->end()) {
+    auto i = v.attrs()->get(s.toString);
+    if (i) {
         Value v1;
         callFunction(*i->value, v, v1, pos);
         return coerceToString(
@@ -2359,8 +2389,8 @@ BackedStringView EvalState::coerceToString(
         auto maybeString = tryAttrsToString(pos, v, context, coerceMore, copyToStore);
         if (maybeString)
             return std::move(*maybeString);
-        auto i = v.attrs()->find(s.outPath);
-        if (i == v.attrs()->end()) {
+        auto i = v.attrs()->get(s.outPath);
+        if (!i) {
             error<TypeError>(
                 "cannot coerce %1% to a string: %2%", showType(v), ValuePrinter(*this, v, errorPrintOptions))
                 .withTrace(pos, errorCtx)
@@ -2428,7 +2458,7 @@ StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePat
     if (nix::isDerivation(path.path.abs()))
         error<EvalError>("file names are not allowed to end in '%1%'", drvExtension).debugThrow();
 
-    auto dstPathCached = get(*srcToStore.lock(), path);
+    auto dstPathCached = getConcurrent(*srcToStore, path);
 
     auto dstPath = dstPathCached ? *dstPathCached : [&]() {
         auto dstPath = fetchToStore(
@@ -2441,7 +2471,7 @@ StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePat
             nullptr,
             repair);
         allowPath(dstPath);
-        srcToStore.lock()->try_emplace(path, dstPath);
+        srcToStore->try_emplace(path, dstPath);
         printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, store->printStorePath(dstPath));
         return dstPath;
     }();
@@ -2466,8 +2496,8 @@ SourcePath EvalState::coerceToPath(const PosIdx pos, Value & v, NixStringContext
     /* Similarly, handle __toString where the result may be a path
        value. */
     if (v.type() == nAttrs) {
-        auto i = v.attrs()->find(s.toString);
-        if (i != v.attrs()->end()) {
+        auto i = v.attrs()->get(s.toString);
+        if (i) {
             Value v1;
             callFunction(*i->value, v, v1, pos);
             return coerceToPath(pos, v1, context, errorCtx);
@@ -2746,11 +2776,8 @@ void EvalState::assertEqValues(Value & v1, Value & v2, const PosIdx pos, std::st
         }
         return;
 
-    // Cannot be returned by forceValue().
-    case nThunk:
-    case nFailed:
-        unreachable();
-
+    case nThunk: // Must not be left by forceValue
+        assert(false);
     default: // Note that we pass compiler flags that should make `default:` unreachable.
         // Also note that this probably ran after `eqValues`, which implements
         // the same logic more efficiently (without having to unwind stacks),
@@ -2842,11 +2869,8 @@ bool EvalState::eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_v
         // !!!
         return v1.fpoint() == v2.fpoint();
 
-    // Cannot be returned by forceValue().
-    case nThunk:
-    case nFailed:
-        unreachable();
-
+    case nThunk: // Must not be left by forceValue
+        assert(false);
     default: // Note that we pass compiler flags that should make `default:` unreachable.
         error<EvalError>("eqValues: cannot compare %1% with %2%", showType(v1), showType(v2))
             .withTrace(pos, errorCtx)
@@ -2869,11 +2893,11 @@ bool EvalState::fullGC()
 #endif
 }
 
+bool Counter::enabled = getEnv("NIX_SHOW_STATS").value_or("0") != "0";
+
 void EvalState::maybePrintStats()
 {
-    bool showStats = getEnv("NIX_SHOW_STATS").value_or("0") != "0";
-
-    if (showStats) {
+    if (Counter::enabled) {
         // Make the final heap size more deterministic.
 #if NIX_USE_BOEHMGC
         if (!fullGC()) {
@@ -2889,10 +2913,12 @@ void EvalState::printStatistics()
     std::chrono::microseconds cpuTimeDuration = getCpuUserTime();
     float cpuTime = std::chrono::duration_cast<std::chrono::duration<float>>(cpuTimeDuration).count();
 
-    uint64_t bEnvs = nrEnvs * sizeof(Env) + nrValuesInEnvs * sizeof(Value *);
-    uint64_t bLists = nrListElems * sizeof(Value *);
-    uint64_t bValues = nrValues * sizeof(Value);
-    uint64_t bAttrsets = nrAttrsets * sizeof(Bindings) + nrAttrsInAttrsets * sizeof(Attr);
+    auto & memstats = mem.getStats();
+
+    uint64_t bEnvs = memstats.nrEnvs * sizeof(Env) + memstats.nrValuesInEnvs * sizeof(Value *);
+    uint64_t bLists = memstats.nrListElems * sizeof(Value *);
+    uint64_t bValues = memstats.nrValues * sizeof(Value);
+    uint64_t bAttrsets = memstats.nrAttrsets * sizeof(Bindings) + memstats.nrAttrsInAttrsets * sizeof(Attr);
 
 #if NIX_USE_BOEHMGC
     GC_word heapSize, totalBytes;
@@ -2918,18 +2944,18 @@ void EvalState::printStatistics()
 #endif
     };
     topObj["envs"] = {
-        {"number", nrEnvs},
-        {"elements", nrValuesInEnvs},
+        {"number", memstats.nrEnvs.load()},
+        {"elements", memstats.nrValuesInEnvs.load()},
         {"bytes", bEnvs},
     };
-    topObj["nrExprs"] = Expr::nrExprs;
+    topObj["nrExprs"] = Expr::nrExprs.load();
     topObj["list"] = {
-        {"elements", nrListElems},
+        {"elements", memstats.nrListElems.load()},
         {"bytes", bLists},
-        {"concats", nrListConcats},
+        {"concats", nrListConcats.load()},
     };
     topObj["values"] = {
-        {"number", nrValues},
+        {"number", memstats.nrValues.load()},
         {"bytes", bValues},
     };
     topObj["symbols"] = {
@@ -2937,9 +2963,9 @@ void EvalState::printStatistics()
         {"bytes", symbols.totalSize()},
     };
     topObj["sets"] = {
-        {"number", nrAttrsets},
+        {"number", memstats.nrAttrsets.load()},
         {"bytes", bAttrsets},
-        {"elements", nrAttrsInAttrsets},
+        {"elements", memstats.nrAttrsInAttrsets.load()},
     };
     topObj["sizes"] = {
         {"Env", sizeof(Env)},
@@ -2947,13 +2973,13 @@ void EvalState::printStatistics()
         {"Bindings", sizeof(Bindings)},
         {"Attr", sizeof(Attr)},
     };
-    topObj["nrOpUpdates"] = nrOpUpdates;
-    topObj["nrOpUpdateValuesCopied"] = nrOpUpdateValuesCopied;
-    topObj["nrThunks"] = nrThunks;
-    topObj["nrAvoided"] = nrAvoided;
-    topObj["nrLookups"] = nrLookups;
-    topObj["nrPrimOpCalls"] = nrPrimOpCalls;
-    topObj["nrFunctionCalls"] = nrFunctionCalls;
+    topObj["nrOpUpdates"] = nrOpUpdates.load();
+    topObj["nrOpUpdateValuesCopied"] = nrOpUpdateValuesCopied.load();
+    topObj["nrThunks"] = nrThunks.load();
+    topObj["nrAvoided"] = nrAvoided.load();
+    topObj["nrLookups"] = nrLookups.load();
+    topObj["nrPrimOpCalls"] = nrPrimOpCalls.load();
+    topObj["nrFunctionCalls"] = nrFunctionCalls.load();
 #if NIX_USE_BOEHMGC
     topObj["gc"] = {
         {"heapSize", heapSize},
@@ -3100,6 +3126,11 @@ SourcePath EvalState::findFile(const LookupPath & lookupPath, const std::string_
         auto res = (r / CanonPath(suffix)).resolveSymlinks();
         if (res.pathExists())
             return res;
+
+        // Backward compatibility hack: throw an exception if access
+        // to this path is not allowed.
+        if (auto accessor = res.accessor.dynamic_pointer_cast<FilteringSourceAccessor>())
+            accessor->checkAccess(res.path);
     }
 
     if (hasPrefix(path, "nix/"))
@@ -3154,7 +3185,7 @@ std::optional<SourcePath> EvalState::resolveLookupPathPath(const LookupPath::Pat
 
         /* Allow access to paths in the search path. */
         if (initAccessControl) {
-            allowPath(path.path.abs());
+            allowPathLegacy(path.path.abs());
             if (store->isInStore(path.path.abs())) {
                 try {
                     allowClosure(store->toStorePath(path.path.abs()).first);
@@ -3166,6 +3197,11 @@ std::optional<SourcePath> EvalState::resolveLookupPathPath(const LookupPath::Pat
         if (path.resolveSymlinks().pathExists())
             return finish(std::move(path));
         else {
+            // Backward compatibility hack: throw an exception if access
+            // to this path is not allowed.
+            if (auto accessor = path.accessor.dynamic_pointer_cast<FilteringSourceAccessor>())
+                accessor->checkAccess(path.path);
+
             logWarning({.msg = HintFmt("Nix search path entry '%1%' does not exist, ignoring", value)});
         }
     }
@@ -3184,7 +3220,8 @@ Expr * EvalState::parse(
         docComments = &it->second;
     }
 
-    auto result = parseExprFromBuf(text, length, origin, basePath, symbols, settings, positions, *docComments, rootFS);
+    auto result = parseExprFromBuf(
+        text, length, origin, basePath, mem.exprs.alloc, symbols, settings, positions, *docComments, rootFS);
 
     result->bindVars(*this, staticEnv);
 

src/libexpr/get-drvs.cc

@@ -45,8 +45,8 @@ PackageInfo::PackageInfo(EvalState & state, ref<Store> store, const std::string
 std::string PackageInfo::queryName() const
 {
     if (name == "" && attrs) {
-        auto i = attrs->find(state->s.name);
-        if (i == attrs->end())
+        auto i = attrs->get(state->s.name);
+        if (!i)
             state->error<TypeError>("derivation name missing").debugThrow();
         name = state->forceStringNoCtx(*i->value, noPos, "while evaluating the 'name' attribute of a derivation");
     }
@@ -56,11 +56,10 @@ std::string PackageInfo::queryName() const
 std::string PackageInfo::querySystem() const
 {
     if (system == "" && attrs) {
-        auto i = attrs->find(state->s.system);
+        auto i = attrs->get(state->s.system);
         system =
-            i == attrs->end()
-                ? "unknown"
-                : state->forceStringNoCtx(*i->value, i->pos, "while evaluating the 'system' attribute of a derivation");
+            !i ? "unknown"
+               : state->forceStringNoCtx(*i->value, i->pos, "while evaluating the 'system' attribute of a derivation");
     }
     return system;
 }
@@ -95,9 +94,9 @@ StorePath PackageInfo::requireDrvPath() const
 StorePath PackageInfo::queryOutPath() const
 {
     if (!outPath && attrs) {
-        auto i = attrs->find(state->s.outPath);
+        auto i = attrs->get(state->s.outPath);
         NixStringContext context;
-        if (i != attrs->end())
+        if (i)
             outPath = state->coerceToStorePath(
                 i->pos, *i->value, context, "while evaluating the output path of a derivation");
     }

src/libexpr/include/nix/expr/attr-set.hh

@@ -4,12 +4,16 @@
 #include "nix/expr/nixexpr.hh"
 #include "nix/expr/symbol-table.hh"
 
+#include <boost/container/static_vector.hpp>
+
 #include <algorithm>
 #include <functional>
+#include <ranges>
+#include <optional>
 
 namespace nix {
 
-class EvalState;
+class EvalMemory;
 struct Value;
 
 /**
@@ -47,15 +51,53 @@ static_assert(
  * by its size and its capacity, the capacity being the number of Attr
  * elements allocated after this structure, while the size corresponds to
  * the number of elements already inserted in this structure.
+ *
+ * Bindings can be efficiently `//`-composed into an intrusive linked list of "layers"
+ * that saves on copies and allocations. Each lookup (@see Bindings::get) traverses
+ * this linked list until a matching attribute is found (thus overlays earlier in
+ * the list take precedence). For iteration over the whole Bindings, an on-the-fly
+ * k-way merge is performed by Bindings::iterator class.
  */
 class Bindings
 {
 public:
-    typedef uint32_t size_t;
+    using size_type = uint32_t;
+
     PosIdx pos;
 
+    /**
+     * An instance of bindings objects with 0 attributes.
+     * This object must never be modified.
+     */
+    static Bindings emptyBindings;
+
 private:
-    size_t size_ = 0;
+    /**
+     * Number of attributes in the attrs FAM (Flexible Array Member).
+     */
+    size_type numAttrs = 0;
+
+    /**
+     * Number of attributes with unique names in the layer chain.
+     *
+     * This is the *real* user-facing size of bindings, whereas @ref numAttrs is
+     * an implementation detail of the data structure.
+     */
+    size_type numAttrsInChain = 0;
+
+    /**
+     * Length of the layers list.
+     */
+    uint32_t numLayers = 1;
+
+    /**
+     * Bindings that this attrset is "layered" on top of.
+     */
+    const Bindings * baseLayer = nullptr;
+
+    /**
+     * Flexible array member of attributes.
+     */
     Attr attrs[0];
 
     Bindings() = default;
@@ -64,71 +106,306 @@ private:
     Bindings & operator=(const Bindings &) = delete;
     Bindings & operator=(Bindings &&) = delete;
 
+    friend class BindingsBuilder;
+
+    /**
+     * Maximum length of the Bindings layer chains.
+     */
+    static constexpr unsigned maxLayers = 8;
+
 public:
-    size_t size() const
+    size_type size() const
     {
-        return size_;
+        return numAttrsInChain;
     }
 
     bool empty() const
     {
-        return !size_;
+        return size() == 0;
     }
 
-    typedef Attr * iterator;
+    class iterator
+    {
+    public:
+        using value_type = Attr;
+        using pointer = const value_type *;
+        using reference = const value_type &;
+        using difference_type = std::ptrdiff_t;
+        using iterator_category = std::forward_iterator_tag;
 
-    typedef const Attr * const_iterator;
+        friend class Bindings;
+
+    private:
+        struct BindingsCursor
+        {
+            /**
+             * Attr that the cursor currently points to.
+             */
+            pointer current;
+
+            /**
+             * One past the end pointer to the contiguous buffer of Attrs.
+             */
+            pointer end;
+
+            /**
+             * Priority of the value. Lesser values have more priority (i.e. they override
+             * attributes that appear later in the linked list of Bindings).
+             */
+            uint32_t priority;
+
+            pointer operator->() const noexcept
+            {
+                return current;
+            }
+
+            reference get() const noexcept
+            {
+                return *current;
+            }
+
+            bool empty() const noexcept
+            {
+                return current == end;
+            }
+
+            void increment() noexcept
+            {
+                ++current;
+            }
+
+            void consume(Symbol name) noexcept
+            {
+                while (!empty() && current->name <= name)
+                    ++current;
+            }
+
+            GENERATE_CMP(BindingsCursor, me->current->name, me->priority)
+        };
+
+        using QueueStorageType = boost::container::static_vector<BindingsCursor, maxLayers>;
+
+        /**
+         * Comparator implementing the override priority / name ordering
+         * for BindingsCursor.
+         */
+        static constexpr auto comp = std::greater<BindingsCursor>();
+
+        /**
+         * A priority queue used to implement an on-the-fly k-way merge.
+         */
+        QueueStorageType cursorHeap;
+
+        /**
+         * The attribute the iterator currently points to.
+         */
+        pointer current = nullptr;
+
+        /**
+         * Whether iterating over a single attribute and not a merge chain.
+         */
+        bool doMerge = true;
+
+        void push(BindingsCursor cursor) noexcept
+        {
+            cursorHeap.push_back(cursor);
+            std::ranges::make_heap(cursorHeap, comp);
+        }
+
+        [[nodiscard]] BindingsCursor pop() noexcept
+        {
+            std::ranges::pop_heap(cursorHeap, comp);
+            auto cursor = cursorHeap.back();
+            cursorHeap.pop_back();
+            return cursor;
+        }
+
+        iterator & finished() noexcept
+        {
+            current = nullptr;
+            return *this;
+        }
+
+        void next(BindingsCursor cursor) noexcept
+        {
+            current = &cursor.get();
+            cursor.increment();
+
+            if (!cursor.empty())
+                push(cursor);
+        }
+
+        std::optional<BindingsCursor> consumeAllUntilCurrentName() noexcept
+        {
+            auto cursor = pop();
+            Symbol lastHandledName = current->name;
+
+            while (cursor->name <= lastHandledName) {
+                cursor.consume(lastHandledName);
+                if (!cursor.empty())
+                    push(cursor);
+
+                if (cursorHeap.empty())
+                    return std::nullopt;
+
+                cursor = pop();
+            }
+
+            return cursor;
+        }
+
+        explicit iterator(const Bindings & attrs) noexcept
+            : doMerge(attrs.baseLayer)
+        {
+            auto pushBindings = [this, priority = unsigned{0}](const Bindings & layer) mutable {
+                auto first = layer.attrs;
+                push(
+                    BindingsCursor{
+                        .current = first,
+                        .end = first + layer.numAttrs,
+                        .priority = priority++,
+                    });
+            };
+
+            if (!doMerge) {
+                if (attrs.empty())
+                    return;
+
+                current = attrs.attrs;
+                pushBindings(attrs);
+
+                return;
+            }
+
+            const Bindings * layer = &attrs;
+            while (layer) {
+                if (layer->numAttrs != 0)
+                    pushBindings(*layer);
+                layer = layer->baseLayer;
+            }
+
+            if (cursorHeap.empty())
+                return;
+
+            next(pop());
+        }
+
+    public:
+        iterator() = default;
+
+        reference operator*() const noexcept
+        {
+            return *current;
+        }
+
+        pointer operator->() const noexcept
+        {
+            return current;
+        }
+
+        iterator & operator++() noexcept
+        {
+            if (!doMerge) {
+                ++current;
+                if (current == cursorHeap.front().end)
+                    return finished();
+                return *this;
+            }
+
+            if (cursorHeap.empty())
+                return finished();
+
+            auto cursor = consumeAllUntilCurrentName();
+            if (!cursor)
+                return finished();
+
+            next(*cursor);
+            return *this;
+        }
+
+        iterator operator++(int) noexcept
+        {
+            iterator tmp = *this;
+            ++*this;
+            return tmp;
+        }
+
+        bool operator==(const iterator & rhs) const noexcept
+        {
+            return current == rhs.current;
+        }
+    };
+
+    using const_iterator = iterator;
 
     void push_back(const Attr & attr)
     {
-        attrs[size_++] = attr;
+        attrs[numAttrs++] = attr;
+        numAttrsInChain = numAttrs;
     }
 
-    const_iterator find(Symbol name) const
+    /**
+     * Get attribute by name or nullptr if no such attribute exists.
+     */
+    const Attr * get(Symbol name) const noexcept
     {
-        Attr key(name, 0);
-        const_iterator i = std::lower_bound(begin(), end(), key);
-        if (i != end() && i->name == name)
-            return i;
-        return end();
-    }
+        auto getInChunk = [key = Attr{name, nullptr}](const Bindings & chunk) -> const Attr * {
+            auto first = chunk.attrs;
+            auto last = first + chunk.numAttrs;
+            const Attr * i = std::lower_bound(first, last, key);
+            if (i != last && i->name == key.name)
+                return i;
+            return nullptr;
+        };
+
+        const Bindings * currentChunk = this;
+        while (currentChunk) {
+            const Attr * maybeAttr = getInChunk(*currentChunk);
+            if (maybeAttr)
+                return maybeAttr;
+            currentChunk = currentChunk->baseLayer;
+        }
 
-    const Attr * get(Symbol name) const
-    {
-        Attr key(name, 0);
-        const_iterator i = std::lower_bound(begin(), end(), key);
-        if (i != end() && i->name == name)
-            return &*i;
         return nullptr;
     }
 
-    iterator begin()
+    /**
+     * Check if the layer chain is full.
+     */
+    bool isLayerListFull() const noexcept
     {
-        return &attrs[0];
+        return numLayers == Bindings::maxLayers;
     }
 
-    iterator end()
+    /**
+     * Test if the length of the linked list of layers is greater than 1.
+     */
+    bool isLayered() const noexcept
     {
-        return &attrs[size_];
+        return numLayers > 1;
     }
 
     const_iterator begin() const
     {
-        return &attrs[0];
+        return const_iterator(*this);
     }
 
     const_iterator end() const
     {
-        return &attrs[size_];
+        return const_iterator();
     }
 
-    Attr & operator[](size_t pos)
+    Attr & operator[](size_type pos)
     {
+        if (isLayered()) [[unlikely]]
+            unreachable();
         return attrs[pos];
     }
 
-    const Attr & operator[](size_t pos) const
+    const Attr & operator[](size_type pos) const
     {
+        if (isLayered()) [[unlikely]]
+            unreachable();
         return attrs[pos];
     }
 
@@ -140,19 +417,21 @@ public:
     std::vector<const Attr *> lexicographicOrder(const SymbolTable & symbols) const
     {
         std::vector<const Attr *> res;
-        res.reserve(size_);
-        for (size_t n = 0; n < size_; n++)
-            res.emplace_back(&attrs[n]);
-        std::sort(res.begin(), res.end(), [&](const Attr * a, const Attr * b) {
+        res.reserve(size());
+        std::ranges::transform(*this, std::back_inserter(res), [](const Attr & a) { return &a; });
+        std::ranges::sort(res, [&](const Attr * a, const Attr * b) {
             std::string_view sa = symbols[a->name], sb = symbols[b->name];
             return sa < sb;
         });
         return res;
     }
 
-    friend class EvalState;
+    friend class EvalMemory;
 };
 
+static_assert(std::forward_iterator<Bindings::iterator>);
+static_assert(std::ranges::forward_range<Bindings>);
+
 /**
  * A wrapper around Bindings that ensures that its always in sorted
  * order at the end. The only way to consume a BindingsBuilder is to
@@ -163,23 +442,38 @@ class BindingsBuilder final
 public:
     // needed by std::back_inserter
     using value_type = Attr;
-    using size_type = Bindings::size_t;
+    using size_type = Bindings::size_type;
 
 private:
     Bindings * bindings;
-    Bindings::size_t capacity_;
+    Bindings::size_type capacity_;
 
-    friend class EvalState;
+    friend class EvalMemory;
 
-    BindingsBuilder(EvalState & state, Bindings * bindings, size_type capacity)
+    BindingsBuilder(EvalMemory & mem, SymbolTable & symbols, Bindings * bindings, size_type capacity)
         : bindings(bindings)
         , capacity_(capacity)
-        , state(state)
+        , mem(mem)
+        , symbols(symbols)
     {
     }
 
+    bool hasBaseLayer() const noexcept
+    {
+        return bindings->baseLayer;
+    }
+
+    void finishSizeIfNecessary()
+    {
+        if (hasBaseLayer())
+            /* NOTE: Do not use std::ranges::distance, since Bindings is a sized
+               range, but we are calculating this size here. */
+            bindings->numAttrsInChain = std::distance(bindings->begin(), bindings->end());
+    }
+
 public:
-    std::reference_wrapper<EvalState> state;
+    std::reference_wrapper<EvalMemory> mem;
+    std::reference_wrapper<SymbolTable> symbols;
 
     void insert(Symbol name, Value * value, PosIdx pos = noPos)
     {
@@ -193,10 +487,26 @@ public:
 
     void push_back(const Attr & attr)
     {
-        assert(bindings->size() < capacity_);
+        assert(bindings->numAttrs < capacity_);
         bindings->push_back(attr);
     }
 
+    /**
+     * "Layer" the newly constructured Bindings on top of another attribute set.
+     *
+     * This effectively performs an attribute set merge, while giving preference
+     * to attributes from the newly constructed Bindings in case of duplicate attribute
+     * names.
+     *
+     * This operation amortizes the need to copy over all attributes and allows
+     * for efficient implementation of attribute set merges (ExprOpUpdate::eval).
+     */
+    void layerOnTopOf(const Bindings & base) noexcept
+    {
+        bindings->baseLayer = &base;
+        bindings->numLayers = base.numLayers + 1;
+    }
+
     Value & alloc(Symbol name, PosIdx pos = noPos);
 
     Value & alloc(std::string_view name, PosIdx pos = noPos);
@@ -204,11 +514,13 @@ public:
     Bindings * finish()
     {
         bindings->sort();
+        finishSizeIfNecessary();
         return bindings;
     }
 
     Bindings * alreadySorted()
     {
+        finishSizeIfNecessary();
         return bindings;
     }
 

src/libexpr/include/nix/expr/counter.hh

@@ -0,0 +1,70 @@
+#pragma once
+
+#include <atomic>
+#include <cstdint>
+
+namespace nix {
+
+/**
+ * An atomic counter aligned on a cache line to prevent false sharing.
+ * The counter is only enabled when the `NIX_SHOW_STATS` environment
+ * variable is set. This is to prevent contention on these counters
+ * when multi-threaded evaluation is enabled.
+ */
+struct alignas(64) Counter
+{
+    using value_type = uint64_t;
+
+    std::atomic<value_type> inner{0};
+
+    static bool enabled;
+
+    Counter() {}
+
+    operator value_type() const noexcept
+    {
+        return inner;
+    }
+
+    void operator=(value_type n) noexcept
+    {
+        inner = n;
+    }
+
+    value_type load() const noexcept
+    {
+        return inner;
+    }
+
+    value_type operator++() noexcept
+    {
+        return enabled ? ++inner : 0;
+    }
+
+    value_type operator++(int) noexcept
+    {
+        return enabled ? inner++ : 0;
+    }
+
+    value_type operator--() noexcept
+    {
+        return enabled ? --inner : 0;
+    }
+
+    value_type operator--(int) noexcept
+    {
+        return enabled ? inner-- : 0;
+    }
+
+    value_type operator+=(value_type n) noexcept
+    {
+        return enabled ? inner += n : 0;
+    }
+
+    value_type operator-=(value_type n) noexcept
+    {
+        return enabled ? inner -= n : 0;
+    }
+};
+
+} // namespace nix

src/libexpr/include/nix/expr/eval-error.hh

@@ -56,14 +56,6 @@ MakeError(MissingArgumentError, EvalError);
 MakeError(InfiniteRecursionError, EvalError);
 MakeError(IFDError, EvalBaseError);
 
-/**
- * An evaluation error which should be retried instead of rethrown
- *
- * A RecoverableEvalError is not an EvalError, because we shouldn't cache it in the eval cache, as it should be retried
- * anyway.
- */
-MakeError(RecoverableEvalError, EvalBaseError);
-
 struct InvalidPathError : public EvalError
 {
 public:

src/libexpr/include/nix/expr/eval-inline.hh

@@ -5,7 +5,6 @@
 #include "nix/expr/eval.hh"
 #include "nix/expr/eval-error.hh"
 #include "nix/expr/eval-settings.hh"
-#include <exception>
 
 namespace nix {
 
@@ -27,7 +26,7 @@ inline void * allocBytes(size_t n)
 }
 
 [[gnu::always_inline]]
-Value * EvalState::allocValue()
+Value * EvalMemory::allocValue()
 {
 #if NIX_USE_BOEHMGC
     /* We use the boehm batch allocator to speed up allocations of Values (of which there are many).
@@ -49,15 +48,15 @@ Value * EvalState::allocValue()
     void * p = allocBytes(sizeof(Value));
 #endif
 
-    nrValues++;
+    stats.nrValues++;
     return (Value *) p;
 }
 
 [[gnu::always_inline]]
-Env & EvalState::allocEnv(size_t size)
+Env & EvalMemory::allocEnv(size_t size)
 {
-    nrEnvs++;
-    nrValuesInEnvs += size;
+    stats.nrEnvs++;
+    stats.nrValuesInEnvs += size;
 
     Env * env;
 
@@ -98,19 +97,12 @@ void EvalState::forceValue(Value & v, const PosIdx pos)
             else
                 ExprBlackHole::throwInfiniteRecursionError(*this, v);
         } catch (...) {
-            handleEvalExceptionForThunk(env, expr, v, pos);
+            v.mkThunk(env, expr);
+            tryFixupBlackHolePos(v, pos);
             throw;
         }
-    } else if (v.isApp()) {
-        try {
-            callFunction(*v.app().left, *v.app().right, v, pos);
-        } catch (...) {
-            handleEvalExceptionForApp(v);
-            throw;
-        }
-    } else if (v.isFailed()) {
-        handleEvalFailed(v, pos);
-    }
+    } else if (v.isApp())
+        callFunction(*v.app().left, *v.app().right, v, pos);
 }
 
 [[gnu::always_inline]]

src/libexpr/include/nix/expr/eval-settings.hh

@@ -342,6 +342,25 @@ struct EvalSettings : Config
           This is useful for improving code readability and making path literals
           more explicit.
     )"};
+
+    Setting<unsigned> bindingsUpdateLayerRhsSizeThreshold{
+        this,
+        sizeof(void *) == 4 ? 8192 : 16,
+        "eval-attrset-update-layer-rhs-threshold",
+        R"(
+          Tunes the maximum size of an attribute set that, when used
+          as a right operand in an [attribute set update expression](@docroot@/language/operators.md#update),
+          uses a more space-efficient linked-list representation of attribute sets.
+
+          Setting this to larger values generally leads to less memory allocations,
+          but may lead to worse evaluation performance.
+
+          A value of `0` disables this optimization completely.
+
+          This is an advanced performance tuning option and typically should not be changed.
+          The default value is chosen to balance performance and memory usage. On 32 bit systems
+          where memory is scarce, the default is a large value to reduce the amount of allocations.
+    )"};
 };
 
 /**

src/libexpr/include/nix/expr/eval.hh

@@ -16,10 +16,14 @@
 #include "nix/expr/search-path.hh"
 #include "nix/expr/repl-exit-status.hh"
 #include "nix/util/ref.hh"
+#include "nix/expr/counter.hh"
 
 // For `NIX_USE_BOEHMGC`, and if that's set, `GC_THREADS`
 #include "nix/expr/config.hh"
 
+#include <boost/unordered/unordered_flat_map.hpp>
+#include <boost/unordered/concurrent_flat_map_fwd.hpp>
+
 #include <map>
 #include <optional>
 #include <functional>
@@ -38,6 +42,7 @@ class Store;
 namespace fetchers {
 struct Settings;
 struct InputCache;
+struct Input;
 } // namespace fetchers
 struct EvalSettings;
 class EvalState;
@@ -45,6 +50,7 @@ class StorePath;
 struct SingleDerivedPath;
 enum RepairFlag : bool;
 struct MemorySourceAccessor;
+struct MountedSourceAccessor;
 
 namespace eval_cache {
 class EvalCache;
@@ -162,7 +168,7 @@ typedef std::
     map<std::string, Value *, std::less<std::string>, traceable_allocator<std::pair<const std::string, Value *>>>
         ValMap;
 
-typedef std::unordered_map<PosIdx, DocComment> DocCommentMap;
+typedef boost::unordered_flat_map<PosIdx, DocComment, std::hash<PosIdx>> DocCommentMap;
 
 struct Env
 {
@@ -297,6 +303,68 @@ struct StaticEvalSymbols
     }
 };
 
+class EvalMemory
+{
+#if NIX_USE_BOEHMGC
+    /**
+     * Allocation cache for GC'd Value objects.
+     */
+    std::shared_ptr<void *> valueAllocCache;
+
+    /**
+     * Allocation cache for size-1 Env objects.
+     */
+    std::shared_ptr<void *> env1AllocCache;
+#endif
+
+public:
+    struct Statistics
+    {
+        Counter nrEnvs;
+        Counter nrValuesInEnvs;
+        Counter nrValues;
+        Counter nrAttrsets;
+        Counter nrAttrsInAttrsets;
+        Counter nrListElems;
+    };
+
+    EvalMemory();
+
+    EvalMemory(const EvalMemory &) = delete;
+    EvalMemory(EvalMemory &&) = delete;
+    EvalMemory & operator=(const EvalMemory &) = delete;
+    EvalMemory & operator=(EvalMemory &&) = delete;
+
+    inline Value * allocValue();
+    inline Env & allocEnv(size_t size);
+
+    Bindings * allocBindings(size_t capacity);
+
+    BindingsBuilder buildBindings(SymbolTable & symbols, size_t capacity)
+    {
+        return BindingsBuilder(*this, symbols, allocBindings(capacity), capacity);
+    }
+
+    ListBuilder buildList(size_t size)
+    {
+        stats.nrListElems += size;
+        return ListBuilder(size);
+    }
+
+    const Statistics & getStats() const &
+    {
+        return stats;
+    }
+
+    /**
+     * Storage for the AST nodes
+     */
+    Exprs exprs;
+
+private:
+    Statistics stats;
+};
+
 class EvalState : public std::enable_shared_from_this<EvalState>
 {
 public:
@@ -307,53 +375,18 @@ public:
     SymbolTable symbols;
     PosTable positions;
 
+    EvalMemory mem;
+
     /**
      * If set, force copying files to the Nix store even if they
      * already exist there.
      */
     RepairFlag repair;
 
-    Bindings emptyBindings;
-
-    /**
-     * Empty list constant.
-     */
-    Value vEmptyList;
-
-    /**
-     * `null` constant.
-     *
-     * This is _not_ a singleton. Pointer equality is _not_ sufficient.
-     */
-    Value vNull;
-
-    /**
-     * `true` constant.
-     *
-     * This is _not_ a singleton. Pointer equality is _not_ sufficient.
-     */
-    Value vTrue;
-
-    /**
-     * `true` constant.
-     *
-     * This is _not_ a singleton. Pointer equality is _not_ sufficient.
-     */
-    Value vFalse;
-
-    /** `"regular"` */
-    Value vStringRegular;
-    /** `"directory"` */
-    Value vStringDirectory;
-    /** `"symlink"` */
-    Value vStringSymlink;
-    /** `"unknown"` */
-    Value vStringUnknown;
-
     /**
      * The accessor corresponding to `store`.
      */
-    const ref<SourceAccessor> storeFS;
+    const ref<MountedSourceAccessor> storeFS;
 
     /**
      * The accessor for the root filesystem.
@@ -395,7 +428,7 @@ public:
     bool inDebugger = false;
     int trylevel;
     std::list<DebugTrace> debugTraces;
-    std::map<const Expr *, const std::shared_ptr<const StaticEnv>> exprEnvs;
+    boost::unordered_flat_map<const Expr *, const std::shared_ptr<const StaticEnv>> exprEnvs;
 
     const std::shared_ptr<const StaticEnv> getStaticEnv(const Expr & expr) const
     {
@@ -438,59 +471,41 @@ private:
 
     /* Cache for calls to addToStore(); maps source paths to the store
        paths. */
-    Sync<std::unordered_map<SourcePath, StorePath>> srcToStore;
+    ref<boost::concurrent_flat_map<SourcePath, StorePath>> srcToStore;
 
     /**
-     * A cache from path names to parse trees.
+     * A cache that maps paths to "resolved" paths for importing Nix
+     * expressions, i.e. `/foo` to `/foo/default.nix`.
      */
-    typedef std::unordered_map<
-        SourcePath,
-        Expr *,
-        std::hash<SourcePath>,
-        std::equal_to<SourcePath>,
-        traceable_allocator<std::pair<const SourcePath, Expr *>>>
-        FileParseCache;
-    FileParseCache fileParseCache;
+    ref<boost::concurrent_flat_map<SourcePath, SourcePath>> importResolutionCache;
 
     /**
-     * A cache from path names to values.
+     * A cache from resolved paths to values.
      */
-    typedef std::unordered_map<
+    ref<boost::concurrent_flat_map<
         SourcePath,
-        Value,
+        Value *,
         std::hash<SourcePath>,
         std::equal_to<SourcePath>,
-        traceable_allocator<std::pair<const SourcePath, Value>>>
-        FileEvalCache;
-    FileEvalCache fileEvalCache;
+        traceable_allocator<std::pair<const SourcePath, Value *>>>>
+        fileEvalCache;
 
     /**
      * Associate source positions of certain AST nodes with their preceding doc comment, if they have one.
      * Grouped by file.
      */
-    std::unordered_map<SourcePath, DocCommentMap> positionToDocComment;
+    boost::unordered_flat_map<SourcePath, DocCommentMap> positionToDocComment;
 
     LookupPath lookupPath;
 
-    std::map<std::string, std::optional<SourcePath>> lookupPathResolved;
+    boost::unordered_flat_map<std::string, std::optional<SourcePath>, StringViewHash, std::equal_to<>>
+        lookupPathResolved;
 
     /**
      * Cache used by prim_match().
      */
     std::shared_ptr<RegexCache> regexCache;
 
-#if NIX_USE_BOEHMGC
-    /**
-     * Allocation cache for GC'd Value objects.
-     */
-    std::shared_ptr<void *> valueAllocCache;
-
-    /**
-     * Allocation cache for size-1 Env objects.
-     */
-    std::shared_ptr<void *> env1AllocCache;
-#endif
-
 public:
 
     EvalState(
@@ -501,6 +516,15 @@ public:
         std::shared_ptr<Store> buildStore = nullptr);
     ~EvalState();
 
+    /**
+     * A wrapper around EvalMemory::allocValue() to avoid code churn when it
+     * was introduced.
+     */
+    inline Value * allocValue()
+    {
+        return mem.allocValue();
+    }
+
     LookupPath getLookupPath()
     {
         return lookupPath;
@@ -528,8 +552,11 @@ public:
 
     /**
      * Allow access to a path.
+     *
+     * Only for restrict eval: pure eval just whitelist store paths,
+     * never arbitrary paths.
      */
-    void allowPath(const Path & path);
+    void allowPathLegacy(const Path & path);
 
     /**
      * Allow access to a store path. Note that this gets remapped to
@@ -549,6 +576,11 @@ public:
 
     void checkURI(const std::string & uri);
 
+    /**
+     * Mount an input on the Nix store.
+     */
+    StorePath mountInput(fetchers::Input & input, const fetchers::Input & originalInput, ref<SourceAccessor> accessor);
+
     /**
      * Parse a Nix expression from the specified file.
      */
@@ -610,28 +642,8 @@ public:
      */
     inline void forceValue(Value & v, const PosIdx pos);
 
-private:
-
-    /**
-     * Internal support function for forceValue
-     *
-     * This code is factored out so that it's not in the heavily inlined hot path.
-     */
-    void handleEvalExceptionForThunk(Env * env, Expr * expr, Value & v, const PosIdx pos);
-
-    /**
-     * Internal support function for forceValue
-     *
-     * This code is factored out so that it's not in the heavily inlined hot path.
-     */
-    void handleEvalExceptionForApp(Value & v);
-
-    void handleEvalFailed(Value & v, PosIdx pos);
-
     void tryFixupBlackHolePos(Value & v, PosIdx pos);
 
-public:
-
     /**
      * Force a value, then recursively force list elements and
      * attributes.
@@ -667,7 +679,7 @@ public:
     /**
      * Get attribute from an attribute set and throw an error if it doesn't exist.
      */
-    Bindings::const_iterator getAttr(Symbol attrSym, const Bindings * attrSet, std::string_view errorCtx);
+    const Attr * getAttr(Symbol attrSym, const Bindings * attrSet, std::string_view errorCtx);
 
     template<typename... Args>
     [[gnu::noinline]]
@@ -766,11 +778,11 @@ public:
     /**
      * Internal primops not exposed to the user.
      */
-    std::unordered_map<
+    boost::unordered_flat_map<
         std::string,
         Value *,
-        std::hash<std::string>,
-        std::equal_to<std::string>,
+        StringViewHash,
+        std::equal_to<>,
         traceable_allocator<std::pair<const std::string, Value *>>>
         internalPrimOps;
 
@@ -889,22 +901,14 @@ public:
      */
     void autoCallFunction(const Bindings & args, Value & fun, Value & res);
 
-    /**
-     * Allocation primitives.
-     */
-    inline Value * allocValue();
-    inline Env & allocEnv(size_t size);
-
-    Bindings * allocBindings(size_t capacity);
-
     BindingsBuilder buildBindings(size_t capacity)
     {
-        return BindingsBuilder(*this, allocBindings(capacity), capacity);
+        return mem.buildBindings(symbols, capacity);
     }
 
     ListBuilder buildList(size_t size)
     {
-        return ListBuilder(*this, size);
+        return mem.buildList(size);
     }
 
     /**
@@ -1021,26 +1025,20 @@ private:
      */
     std::string mkSingleDerivedPathStringRaw(const SingleDerivedPath & p);
 
-    unsigned long nrEnvs = 0;
-    unsigned long nrValuesInEnvs = 0;
-    unsigned long nrValues = 0;
-    unsigned long nrListElems = 0;
-    unsigned long nrLookups = 0;
-    unsigned long nrAttrsets = 0;
-    unsigned long nrAttrsInAttrsets = 0;
-    unsigned long nrAvoided = 0;
-    unsigned long nrOpUpdates = 0;
-    unsigned long nrOpUpdateValuesCopied = 0;
-    unsigned long nrListConcats = 0;
-    unsigned long nrPrimOpCalls = 0;
-    unsigned long nrFunctionCalls = 0;
+    Counter nrLookups;
+    Counter nrAvoided;
+    Counter nrOpUpdates;
+    Counter nrOpUpdateValuesCopied;
+    Counter nrListConcats;
+    Counter nrPrimOpCalls;
+    Counter nrFunctionCalls;
 
     bool countCalls;
 
-    typedef std::map<std::string, size_t> PrimOpCalls;
+    typedef boost::unordered_flat_map<std::string, size_t, StringViewHash, std::equal_to<>> PrimOpCalls;
     PrimOpCalls primOpCalls;
 
-    typedef std::map<ExprLambda *, size_t> FunctionCalls;
+    typedef boost::unordered_flat_map<ExprLambda *, size_t> FunctionCalls;
     FunctionCalls functionCalls;
 
     /** Evaluation/call profiler. */
@@ -1048,7 +1046,7 @@ private:
 
     void incrFunctionCall(ExprLambda * fun);
 
-    typedef std::map<PosIdx, size_t> AttrSelects;
+    typedef boost::unordered_flat_map<PosIdx, size_t, std::hash<PosIdx>> AttrSelects;
     AttrSelects attrSelects;
 
     friend struct ExprOpUpdate;

src/libexpr/include/nix/expr/gc-small-vector.hh

@@ -26,4 +26,20 @@ using SmallValueVector = SmallVector<Value *, nItems>;
 template<size_t nItems>
 using SmallTemporaryValueVector = SmallVector<Value, nItems>;
 
+/**
+ * For functions where we do not expect deep recursion, we can use a sizable
+ * part of the stack a free allocation space.
+ *
+ * Note: this is expected to be multiplied by sizeof(Value), or about 24 bytes.
+ */
+constexpr size_t nonRecursiveStackReservation = 128;
+
+/**
+ * Functions that maybe applied to self-similar inputs, such as concatMap on a
+ * tree, should reserve a smaller part of the stack for allocation.
+ *
+ * Note: this is expected to be multiplied by sizeof(Value), or about 24 bytes.
+ */
+constexpr size_t conservativeStackReservation = 16;
+
 } // namespace nix

src/libexpr/include/nix/expr/meson.build

@@ -10,6 +10,7 @@ config_pub_h = configure_file(
 headers = [ config_pub_h ] + files(
   'attr-path.hh',
   'attr-set.hh',
+  'counter.hh',
   'eval-cache.hh',
   'eval-error.hh',
   'eval-gc.hh',

src/libexpr/include/nix/expr/nixexpr.hh

@@ -2,12 +2,17 @@
 ///@file
 
 #include <map>
+#include <span>
 #include <vector>
+#include <memory_resource>
+#include <algorithm>
 
+#include "nix/expr/gc-small-vector.hh"
 #include "nix/expr/value.hh"
 #include "nix/expr/symbol-table.hh"
 #include "nix/expr/eval-error.hh"
 #include "nix/util/pos-idx.hh"
+#include "nix/expr/counter.hh"
 
 namespace nix {
 
@@ -76,9 +81,20 @@ struct AttrName
         : expr(e) {};
 };
 
+static_assert(std::is_trivially_copy_constructible_v<AttrName>);
+
 typedef std::vector<AttrName> AttrPath;
 
-std::string showAttrPath(const SymbolTable & symbols, const AttrPath & attrPath);
+std::string showAttrPath(const SymbolTable & symbols, std::span<const AttrName> attrPath);
+
+using UpdateQueue = SmallTemporaryValueVector<conservativeStackReservation>;
+
+class Exprs
+{
+    std::pmr::monotonic_buffer_resource buffer;
+public:
+    std::pmr::polymorphic_allocator<char> alloc{&buffer};
+};
 
 /* Abstract syntax of Nix expressions. */
 
@@ -89,7 +105,7 @@ struct Expr
         Symbol sub, lessThan, mul, div, or_, findFile, nixPath, body;
     };
 
-    static unsigned long nrExprs;
+    static Counter nrExprs;
 
     Expr()
     {
@@ -99,8 +115,25 @@ struct Expr
     virtual ~Expr() {};
     virtual void show(const SymbolTable & symbols, std::ostream & str) const;
     virtual void bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> & env);
+
+    /** Normal evaluation, implemented directly by all subclasses. */
     virtual void eval(EvalState & state, Env & env, Value & v);
+
+    /**
+     * Create a thunk for the delayed computation of the given expression
+     * in the given environment. But if the expression is a variable,
+     * then look it up right away. This significantly reduces the number
+     * of thunks allocated.
+     */
     virtual Value * maybeThunk(EvalState & state, Env & env);
+
+    /**
+     * Only called when performing an attrset update: `//` or similar.
+     * Instead of writing to a Value &, this function writes to an UpdateQueue.
+     * This allows the expression to perform multiple updates in a delayed manner, gathering up all the updates before
+     * applying them.
+     */
+    virtual void evalForUpdate(EvalState & state, Env & env, UpdateQueue & q, std::string_view errorCtx);
     virtual void setName(Symbol name);
     virtual void setDocComment(DocComment docComment) {};
 
@@ -152,13 +185,28 @@ struct ExprFloat : Expr
 
 struct ExprString : Expr
 {
-    std::string s;
     Value v;
 
-    ExprString(std::string && s)
-        : s(std::move(s))
+    /**
+     * This is only for strings already allocated in our polymorphic allocator,
+     * or that live at least that long (e.g. c++ string literals)
+     */
+    ExprString(const char * s)
     {
-        v.mkStringNoCopy(this->s.data());
+        v.mkStringNoCopy(s);
+    };
+
+    ExprString(std::pmr::polymorphic_allocator<char> & alloc, std::string_view sv)
+    {
+        auto len = sv.length();
+        if (len == 0) {
+            v.mkStringNoCopy("");
+            return;
+        }
+        char * s = alloc.allocate(len + 1);
+        sv.copy(s, len);
+        s[len] = '\0';
+        v.mkStringNoCopy(s);
     };
 
     Value * maybeThunk(EvalState & state, Env & env) override;
@@ -168,14 +216,16 @@ struct ExprString : Expr
 struct ExprPath : Expr
 {
     ref<SourceAccessor> accessor;
-    std::string s;
     Value v;
 
-    ExprPath(ref<SourceAccessor> accessor, std::string s)
+    ExprPath(std::pmr::polymorphic_allocator<char> & alloc, ref<SourceAccessor> accessor, std::string_view sv)
         : accessor(accessor)
-        , s(std::move(s))
     {
-        v.mkPath(&*accessor, this->s.c_str());
+        auto len = sv.length();
+        char * s = alloc.allocate(len + 1);
+        sv.copy(s, len);
+        s[len] = '\0';
+        v.mkPath(&*accessor, s);
     }
 
     Value * maybeThunk(EvalState & state, Env & env) override;
@@ -242,20 +292,33 @@ struct ExprInheritFrom : ExprVar
 struct ExprSelect : Expr
 {
     PosIdx pos;
+    uint32_t nAttrPath;
     Expr *e, *def;
-    AttrPath attrPath;
-    ExprSelect(const PosIdx & pos, Expr * e, AttrPath attrPath, Expr * def)
+    AttrName * attrPathStart;
+
+    ExprSelect(
+        std::pmr::polymorphic_allocator<char> & alloc,
+        const PosIdx & pos,
+        Expr * e,
+        std::span<const AttrName> attrPath,
+        Expr * def)
         : pos(pos)
+        , nAttrPath(attrPath.size())
         , e(e)
         , def(def)
-        , attrPath(std::move(attrPath)) {};
+        , attrPathStart(alloc.allocate_object<AttrName>(nAttrPath))
+    {
+        std::ranges::copy(attrPath, attrPathStart);
+    };
 
-    ExprSelect(const PosIdx & pos, Expr * e, Symbol name)
+    ExprSelect(std::pmr::polymorphic_allocator<char> & alloc, const PosIdx & pos, Expr * e, Symbol name)
         : pos(pos)
+        , nAttrPath(1)
         , e(e)
         , def(0)
+        , attrPathStart((alloc.allocate_object<AttrName>()))
     {
-        attrPath.push_back(AttrName(name));
+        *attrPathStart = AttrName(name);
     };
 
     PosIdx getPos() const override
@@ -263,6 +326,11 @@ struct ExprSelect : Expr
         return pos;
     }
 
+    std::span<const AttrName> getAttrPath() const
+    {
+        return {attrPathStart, nAttrPath};
+    }
+
     /**
      * Evaluate the `a.b.c` part of `a.b.c.d`. This exists mostly for the purpose of :doc in the repl.
      *
@@ -280,10 +348,14 @@ struct ExprSelect : Expr
 struct ExprOpHasAttr : Expr
 {
     Expr * e;
-    AttrPath attrPath;
-    ExprOpHasAttr(Expr * e, AttrPath attrPath)
+    std::span<AttrName> attrPath;
+
+    ExprOpHasAttr(std::pmr::polymorphic_allocator<char> & alloc, Expr * e, std::vector<AttrName> attrPath)
         : e(e)
-        , attrPath(std::move(attrPath)) {};
+        , attrPath({alloc.allocate_object<AttrName>(attrPath.size()), attrPath.size()})
+    {
+        std::ranges::copy(attrPath, this->attrPath.begin());
+    };
 
     PosIdx getPos() const override
     {
@@ -565,36 +637,39 @@ struct ExprOpNot : Expr
     COMMON_METHODS
 };
 
-#define MakeBinOp(name, s)                                                                   \
-    struct name : Expr                                                                       \
-    {                                                                                        \
-        PosIdx pos;                                                                          \
-        Expr *e1, *e2;                                                                       \
-        name(Expr * e1, Expr * e2)                                                           \
-            : e1(e1)                                                                         \
-            , e2(e2) {};                                                                     \
-        name(const PosIdx & pos, Expr * e1, Expr * e2)                                       \
-            : pos(pos)                                                                       \
-            , e1(e1)                                                                         \
-            , e2(e2) {};                                                                     \
-        void show(const SymbolTable & symbols, std::ostream & str) const override            \
-        {                                                                                    \
-            str << "(";                                                                      \
-            e1->show(symbols, str);                                                          \
-            str << " " s " ";                                                                \
-            e2->show(symbols, str);                                                          \
-            str << ")";                                                                      \
-        }                                                                                    \
-        void bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> & env) override \
-        {                                                                                    \
-            e1->bindVars(es, env);                                                           \
-            e2->bindVars(es, env);                                                           \
-        }                                                                                    \
-        void eval(EvalState & state, Env & env, Value & v) override;                         \
-        PosIdx getPos() const override                                                       \
-        {                                                                                    \
-            return pos;                                                                      \
-        }                                                                                    \
+#define MakeBinOpMembers(name, s)                                                        \
+    PosIdx pos;                                                                          \
+    Expr *e1, *e2;                                                                       \
+    name(Expr * e1, Expr * e2)                                                           \
+        : e1(e1)                                                                         \
+        , e2(e2){};                                                                      \
+    name(const PosIdx & pos, Expr * e1, Expr * e2)                                       \
+        : pos(pos)                                                                       \
+        , e1(e1)                                                                         \
+        , e2(e2){};                                                                      \
+    void show(const SymbolTable & symbols, std::ostream & str) const override            \
+    {                                                                                    \
+        str << "(";                                                                      \
+        e1->show(symbols, str);                                                          \
+        str << " " s " ";                                                                \
+        e2->show(symbols, str);                                                          \
+        str << ")";                                                                      \
+    }                                                                                    \
+    void bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> & env) override \
+    {                                                                                    \
+        e1->bindVars(es, env);                                                           \
+        e2->bindVars(es, env);                                                           \
+    }                                                                                    \
+    void eval(EvalState & state, Env & env, Value & v) override;                         \
+    PosIdx getPos() const override                                                       \
+    {                                                                                    \
+        return pos;                                                                      \
+    }
+
+#define MakeBinOp(name, s)        \
+    struct name : Expr            \
+    {                             \
+        MakeBinOpMembers(name, s) \
     }
 
 MakeBinOp(ExprOpEq, "==");
@@ -602,9 +677,20 @@ MakeBinOp(ExprOpNEq, "!=");
 MakeBinOp(ExprOpAnd, "&&");
 MakeBinOp(ExprOpOr, "||");
 MakeBinOp(ExprOpImpl, "->");
-MakeBinOp(ExprOpUpdate, "//");
 MakeBinOp(ExprOpConcatLists, "++");
 
+struct ExprOpUpdate : Expr
+{
+private:
+    /** Special case for merging of two attrsets. */
+    void eval(EvalState & state, Value & v, Value & v1, Value & v2);
+    void evalForUpdate(EvalState & state, Env & env, UpdateQueue & q);
+
+public:
+    MakeBinOpMembers(ExprOpUpdate, "//");
+    virtual void evalForUpdate(EvalState & state, Env & env, UpdateQueue & q, std::string_view errorCtx) override;
+};
+
 struct ExprConcatStrings : Expr
 {
     PosIdx pos;

src/libexpr/include/nix/expr/parser-state.hh

@@ -24,7 +24,6 @@ struct StringToken
     }
 };
 
-// This type must be trivially copyable; see YYLTYPE_IS_TRIVIAL in parser.y.
 struct ParserLocation
 {
     int beginOffset;
@@ -44,9 +43,6 @@ struct ParserLocation
         beginOffset = stashedBeginOffset;
         endOffset = stashedEndOffset;
     }
-
-    /** Latest doc comment position, or 0. */
-    int doc_comment_first_column, doc_comment_last_column;
 };
 
 struct LexerState
@@ -71,7 +67,7 @@ struct LexerState
     /**
      * @brief Maps some positions to a DocComment, where the comment is relevant to the location.
      */
-    std::unordered_map<PosIdx, DocComment> & positionToDocComment;
+    DocCommentMap & positionToDocComment;
 
     PosTable & positions;
     PosTable::Origin origin;
@@ -82,6 +78,7 @@ struct LexerState
 struct ParserState
 {
     const LexerState & lexerState;
+    std::pmr::polymorphic_allocator<char> & alloc;
     SymbolTable & symbols;
     PosTable & positions;
     Expr * result;
@@ -327,7 +324,7 @@ ParserState::stripIndentation(const PosIdx pos, std::vector<std::pair<PosIdx, st
 
         // Ignore empty strings for a minor optimisation and AST simplification
         if (s2 != "") {
-            es2->emplace_back(i->first, new ExprString(std::move(s2)));
+            es2->emplace_back(i->first, new ExprString(alloc, s2));
         }
     };
     for (; i != es.end(); ++i, --n) {

src/libexpr/include/nix/expr/primops.hh

@@ -8,22 +8,6 @@
 
 namespace nix {
 
-/**
- * For functions where we do not expect deep recursion, we can use a sizable
- * part of the stack a free allocation space.
- *
- * Note: this is expected to be multiplied by sizeof(Value), or about 24 bytes.
- */
-constexpr size_t nonRecursiveStackReservation = 128;
-
-/**
- * Functions that maybe applied to self-similar inputs, such as concatMap on a
- * tree, should reserve a smaller part of the stack for allocation.
- *
- * Note: this is expected to be multiplied by sizeof(Value), or about 24 bytes.
- */
-constexpr size_t conservativeStackReservation = 16;
-
 struct RegisterPrimOp
 {
     typedef std::vector<PrimOp> PrimOps;

src/libexpr/include/nix/expr/value.hh

@@ -2,7 +2,6 @@
 ///@file
 
 #include <cassert>
-#include <exception>
 #include <span>
 #include <type_traits>
 #include <concepts>
@@ -13,6 +12,7 @@
 #include "nix/expr/print-options.hh"
 #include "nix/util/checked-arithmetic.hh"
 
+#include <boost/unordered/unordered_flat_map_fwd.hpp>
 #include <nlohmann/json_fwd.hpp>
 
 namespace nix {
@@ -36,7 +36,6 @@ typedef enum {
     tBool,
     tNull,
     tFloat,
-    tFailed,
     tExternal,
     tPrimOp,
     tAttrs,
@@ -59,7 +58,6 @@ typedef enum {
  */
 typedef enum {
     nThunk,
-    nFailed,
     nInt,
     nFloat,
     nBool,
@@ -157,7 +155,7 @@ class ListBuilder
     Value * inlineElems[2] = {nullptr, nullptr};
 public:
     Value ** elems;
-    ListBuilder(EvalState & state, size_t size);
+    ListBuilder(size_t size);
 
     // NOTE: Can be noexcept because we are just copying integral values and
     // raw pointers.
@@ -268,30 +266,6 @@ struct ValueBase
         size_t size;
         Value * const * elems;
     };
-
-    /**
-      Representation of an evaluation that previously failed.
-
-      `Value` references `Failed` by packed pointer, and its `new` is GC managed.
-
-      @see gc_cleanup std::exception_ptr
-     */
-    class Failed : public gc_cleanup
-    {
-    public:
-        std::exception_ptr ex;
-        /**
-         * Optional value for recovering `RecoverableEvalError`
-         * Must be set iff `ex` is an instance of `RecoverableEvalError`.
-         */
-        Value * recoveryValue;
-
-        Failed(std::exception_ptr ex, Value * recoveryValue)
-            : ex(ex)
-            , recoveryValue(recoveryValue)
-        {
-        }
-    };
 };
 
 template<typename T>
@@ -318,7 +292,6 @@ struct PayloadTypeToInternalType
     MACRO(PrimOp *, primOp, tPrimOp)                                \
     MACRO(ValueBase::PrimOpApplicationThunk, primOpApp, tPrimOpApp) \
     MACRO(ExternalValueBase *, external, tExternal)                 \
-    MACRO(ValueBase::Failed *, failed, tFailed)                     \
     MACRO(NixFloat, fpoint, tFloat)
 
 #define NIX_VALUE_PAYLOAD_TYPE(T, FIELD_NAME, DISCRIMINATOR) \
@@ -623,11 +596,6 @@ protected:
         path.path = std::bit_cast<const char *>(payload[1]);
     }
 
-    void getStorage(Failed *& failed) const noexcept
-    {
-        failed = std::bit_cast<Failed *>(payload[1]);
-    }
-
     void setStorage(NixInt integer) noexcept
     {
         setSingleDWordPayload<tInt>(integer.value);
@@ -677,11 +645,6 @@ protected:
     {
         setUntaggablePayload<pdPath>(path.accessor, path.path);
     }
-
-    void setStorage(Failed * failed) noexcept
-    {
-        setSingleDWordPayload<tFailed>(std::bit_cast<PackedPointer>(failed));
-    }
 };
 
 /**
@@ -871,6 +834,35 @@ struct Value : public ValueStorage<sizeof(void *)>
 {
     friend std::string showType(const Value & v);
 
+    /**
+     * Empty list constant.
+     *
+     * This is _not_ a singleton. Pointer equality is _not_ sufficient.
+     */
+    static Value vEmptyList;
+
+    /**
+     * `null` constant.
+     *
+     * This is _not_ a singleton. Pointer equality is _not_ sufficient.
+     */
+    static Value vNull;
+
+    /**
+     * `true` constant.
+     *
+     * This is _not_ a singleton. Pointer equality is _not_ sufficient.
+     */
+    static Value vTrue;
+
+    /**
+     * `true` constant.
+     *
+     * This is _not_ a singleton. Pointer equality is _not_ sufficient.
+     */
+    static Value vFalse;
+
+private:
     template<InternalType... discriminator>
     bool isa() const noexcept
     {
@@ -904,12 +896,12 @@ public:
     inline bool isThunk() const
     {
         return isa<tThunk>();
-    }
+    };
 
     inline bool isApp() const
     {
         return isa<tApp>();
-    }
+    };
 
     inline bool isBlackhole() const;
 
@@ -917,22 +909,17 @@ public:
     inline bool isLambda() const
     {
         return isa<tLambda>();
-    }
+    };
 
     inline bool isPrimOp() const
     {
         return isa<tPrimOp>();
-    }
+    };
 
     inline bool isPrimOpApp() const
     {
         return isa<tPrimOpApp>();
-    }
-
-    inline bool isFailed() const
-    {
-        return isa<tFailed>();
-    }
+    };
 
     /**
      * Returns the normal type of a Value. This only returns nThunk if
@@ -969,8 +956,6 @@ public:
             return nExternal;
         case tFloat:
             return nFloat;
-        case tFailed:
-            return nFailed;
         case tThunk:
         case tApp:
             return nThunk;
@@ -1093,11 +1078,6 @@ public:
         setStorage(n);
     }
 
-    inline void mkFailed(std::exception_ptr e, Value * recovery) noexcept
-    {
-        setStorage(new Value::Failed(e, recovery));
-    }
-
     bool isList() const noexcept
     {
         return isa<tListSmall, tListN>();
@@ -1201,11 +1181,6 @@ public:
     {
         return getStorage<Path>().accessor;
     }
-
-    Failed * failed() const noexcept
-    {
-        return getStorage<Failed *>();
-    }
 };
 
 extern ExprBlackHole eBlackHole;
@@ -1221,7 +1196,7 @@ void Value::mkBlackhole()
 }
 
 typedef std::vector<Value *, traceable_allocator<Value *>> ValueVector;
-typedef std::unordered_map<
+typedef boost::unordered_flat_map<
     Symbol,
     Value *,
     std::hash<Symbol>,

src/libexpr/lexer-helpers.cc

@@ -1,11 +1,11 @@
 #include "lexer-helpers.hh"
 
-void nix::lexer::internal::initLoc(YYLTYPE * loc)
+void nix::lexer::internal::initLoc(Parser::location_type * loc)
 {
     loc->beginOffset = loc->endOffset = 0;
 }
 
-void nix::lexer::internal::adjustLoc(yyscan_t yyscanner, YYLTYPE * loc, const char * s, size_t len)
+void nix::lexer::internal::adjustLoc(yyscan_t yyscanner, Parser::location_type * loc, const char * s, size_t len)
 {
     loc->stash();
 

src/libexpr/lexer-helpers.hh

@@ -2,16 +2,12 @@
 
 #include <cstddef>
 
-// including the generated headers twice leads to errors
-#ifndef BISON_HEADER
-#  include "lexer-tab.hh"
-#  include "parser-tab.hh"
-#endif
+#include "parser-scanner-decls.hh"
 
 namespace nix::lexer::internal {
 
-void initLoc(YYLTYPE * loc);
+void initLoc(Parser::location_type * loc);
 
-void adjustLoc(yyscan_t yyscanner, YYLTYPE * loc, const char * s, size_t len);
+void adjustLoc(yyscan_t yyscanner, Parser::location_type * loc, const char * s, size_t len);
 
 } // namespace nix::lexer::internal

src/libexpr/lexer.l

@@ -82,6 +82,10 @@ static void requireExperimentalFeature(const ExperimentalFeature & feature, cons
 
 }
 
+using enum nix::Parser::token::token_kind_type;
+using YYSTYPE = nix::Parser::value_type;
+using YYLTYPE = nix::Parser::location_type;
+
 // yacc generates code that uses unannotated fallthrough.
 #pragma GCC diagnostic ignored "-Wimplicit-fallthrough"
 

src/libexpr/meson.build

@@ -53,7 +53,12 @@ deps_other += boost
 nlohmann_json = dependency('nlohmann_json', version : '>= 3.9')
 deps_public += nlohmann_json
 
-bdw_gc = dependency('bdw-gc', required : get_option('gc'))
+bdw_gc_required = get_option('gc').disable_if(
+  'address' in get_option('b_sanitize'),
+  error_message : 'Building with Boehm GC and ASAN is not supported',
+)
+
+bdw_gc = dependency('bdw-gc', required : bdw_gc_required)
 if bdw_gc.found()
   deps_public += bdw_gc
   foreach funcspec : [
@@ -64,6 +69,10 @@ if bdw_gc.found()
     define_value = cxx.has_function(funcspec).to_int()
     configdata_priv.set(define_name, define_value)
   endforeach
+  if host_machine.system() == 'cygwin'
+    # undefined reference to `__wrap__Znwm'
+    configdata_pub.set('GC_NO_INLINE_STD_NEW', 1)
+  endif
 endif
 # Used in public header. Affects ABI!
 configdata_pub.set('NIX_USE_BOEHMGC', bdw_gc.found().to_int())
@@ -88,6 +97,7 @@ config_priv_h = configure_file(
 )
 
 subdir('nix-meson-build-support/common')
+subdir('nix-meson-build-support/asan-options')
 
 parser_tab = custom_target(
   input : 'parser.y',
@@ -163,6 +173,7 @@ sources = files(
   'search-path.cc',
   'value-to-json.cc',
   'value-to-xml.cc',
+  'value.cc',
   'value/context.cc',
 )
 
@@ -180,6 +191,7 @@ this_library = library(
   parser_tab,
   lexer_tab,
   generated_headers,
+  soversion : nix_soversion,
   dependencies : deps_public + deps_private + deps_other,
   include_directories : include_dirs,
   link_args : linker_export_flags,

src/libexpr/nixexpr.cc

@@ -11,7 +11,7 @@
 
 namespace nix {
 
-unsigned long Expr::nrExprs = 0;
+Counter Expr::nrExprs;
 
 ExprBlackHole eBlackHole;
 
@@ -40,12 +40,12 @@ void ExprFloat::show(const SymbolTable & symbols, std::ostream & str) const
 
 void ExprString::show(const SymbolTable & symbols, std::ostream & str) const
 {
-    printLiteralString(str, s);
+    printLiteralString(str, v.string_view());
 }
 
 void ExprPath::show(const SymbolTable & symbols, std::ostream & str) const
 {
-    str << s;
+    str << v.pathStr();
 }
 
 void ExprVar::show(const SymbolTable & symbols, std::ostream & str) const
@@ -57,7 +57,7 @@ void ExprSelect::show(const SymbolTable & symbols, std::ostream & str) const
 {
     str << "(";
     e->show(symbols, str);
-    str << ")." << showAttrPath(symbols, attrPath);
+    str << ")." << showAttrPath(symbols, getAttrPath());
     if (def) {
         str << " or (";
         def->show(symbols, str);
@@ -261,7 +261,7 @@ void ExprPos::show(const SymbolTable & symbols, std::ostream & str) const
     str << "__curPos";
 }
 
-std::string showAttrPath(const SymbolTable & symbols, const AttrPath & attrPath)
+std::string showAttrPath(const SymbolTable & symbols, std::span<const AttrName> attrPath)
 {
     std::ostringstream out;
     bool first = true;
@@ -362,7 +362,7 @@ void ExprSelect::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv>
     e->bindVars(es, env);
     if (def)
         def->bindVars(es, env);
-    for (auto & i : attrPath)
+    for (auto & i : getAttrPath())
         if (!i.symbol)
             i.expr->bindVars(es, env);
 }

src/libexpr/parser-scanner-decls.hh

@@ -0,0 +1,17 @@
+#pragma once
+
+#ifndef BISON_HEADER
+#  include "parser-tab.hh"
+using YYSTYPE = nix::parser::BisonParser::value_type;
+using YYLTYPE = nix::parser::BisonParser::location_type;
+#  include "lexer-tab.hh" // IWYU pragma: export
+#endif
+
+namespace nix {
+
+class Parser : public parser::BisonParser
+{
+    using BisonParser::BisonParser;
+};
+
+} // namespace nix

src/libexpr/parser.y

@@ -1,5 +1,7 @@
+%skeleton "lalr1.cc"
 %define api.location.type { ::nix::ParserLocation }
-%define api.pure
+%define api.namespace { ::nix::parser }
+%define api.parser.class { BisonParser }
 %locations
 %define parse.error verbose
 %defines
@@ -26,19 +28,12 @@
 #include "nix/expr/eval-settings.hh"
 #include "nix/expr/parser-state.hh"
 
-// Bison seems to have difficulty growing the parser stack when using C++ with
-// a custom location type. This undocumented macro tells Bison that our
-// location type is "trivially copyable" in C++-ese, so it is safe to use the
-// same memcpy macro it uses to grow the stack that it uses with its own
-// default location type. Without this, we get "error: memory exhausted" when
-// parsing some large Nix files. Our other options are to increase the initial
-// stack size (200 by default) to be as large as we ever want to support (so
-// that growing the stack is unnecessary), or redefine the stack-relocation
-// macro ourselves (which is also undocumented).
-#define YYLTYPE_IS_TRIVIAL 1
-
-#define YY_DECL int yylex \
-    (YYSTYPE * yylval_param, YYLTYPE * yylloc_param, yyscan_t yyscanner, nix::ParserState * state)
+#define YY_DECL                                    \
+    int yylex(                                     \
+        nix::Parser::value_type * yylval_param,    \
+        nix::Parser::location_type * yylloc_param, \
+        yyscan_t yyscanner,                        \
+        nix::ParserState * state)
 
 // For efficiency, we only track offsets; not line,column coordinates
 # define YYLLOC_DEFAULT(Current, Rhs, N)                                \
@@ -57,13 +52,14 @@
 
 namespace nix {
 
-typedef std::unordered_map<PosIdx, DocComment> DocCommentMap;
+typedef boost::unordered_flat_map<PosIdx, DocComment, std::hash<PosIdx>> DocCommentMap;
 
 Expr * parseExprFromBuf(
     char * text,
     size_t length,
     Pos::Origin origin,
     const SourcePath & basePath,
+    std::pmr::polymorphic_allocator<char> & alloc,
     SymbolTable & symbols,
     const EvalSettings & settings,
     PosTable & positions,
@@ -78,24 +74,30 @@ Expr * parseExprFromBuf(
 
 %{
 
-#include "parser-tab.hh"
-#include "lexer-tab.hh"
+/* The parser is very performance sensitive and loses out on a lot
+   of performance even with basic stdlib assertions. Since those don't
+   affect ABI we can disable those just for this file. */
+#if defined(_GLIBCXX_ASSERTIONS) && !defined(_GLIBCXX_DEBUG)
+#undef _GLIBCXX_ASSERTIONS
+#endif
+
+#include "parser-scanner-decls.hh"
 
 YY_DECL;
 
 using namespace nix;
 
-#define CUR_POS state->at(yyloc)
+#define CUR_POS state->at(yylhs.location)
 
-
-void yyerror(YYLTYPE * loc, yyscan_t scanner, ParserState * state, const char * error)
+void parser::BisonParser::error(const location_type &loc_, const std::string &error)
 {
+    auto loc = loc_;
     if (std::string_view(error).starts_with("syntax error, unexpected end of file")) {
-        loc->beginOffset = loc->endOffset;
+        loc.beginOffset = loc.endOffset;
     }
     throw ParseError({
         .msg = HintFmt(error),
-        .pos = state->positions[state->at(*loc)]
+        .pos = state->positions[state->at(loc)]
     });
 }
 
@@ -134,6 +136,7 @@ static Expr * makeCall(PosIdx pos, Expr * fn, Expr * arg) {
   std::vector<nix::AttrName> * attrNames;
   std::vector<std::pair<nix::AttrName, nix::PosIdx>> * inheritAttrs;
   std::vector<std::pair<nix::PosIdx, nix::Expr *>> * string_parts;
+  std::variant<nix::Expr *, std::string_view> * to_be_string;
   std::vector<std::pair<nix::PosIdx, std::variant<nix::Expr *, nix::StringToken>>> * ind_string_parts;
 }
 
@@ -148,7 +151,8 @@ static Expr * makeCall(PosIdx pos, Expr * fn, Expr * arg) {
 %type <inheritAttrs> attrs
 %type <string_parts> string_parts_interpolated
 %type <ind_string_parts> ind_string_parts
-%type <e> path_start string_parts string_attr
+%type <e> path_start
+%type <to_be_string> string_parts string_attr
 %type <id> attr
 %token <id> ID
 %token <str> STR IND_STR
@@ -182,7 +186,7 @@ start: expr {
   state->result = $1;
 
   // This parser does not use yynerrs; suppress the warning.
-  (void) yynerrs;
+  (void) yynerrs_;
 };
 
 expr: expr_function;
@@ -257,7 +261,7 @@ expr_op
   | expr_op OR expr_op { $$ = new ExprOpOr(state->at(@2), $1, $3); }
   | expr_op IMPL expr_op { $$ = new ExprOpImpl(state->at(@2), $1, $3); }
   | expr_op UPDATE expr_op { $$ = new ExprOpUpdate(state->at(@2), $1, $3); }
-  | expr_op '?' attrpath { $$ = new ExprOpHasAttr($1, std::move(*$3)); delete $3; }
+  | expr_op '?' attrpath { $$ = new ExprOpHasAttr(state->alloc, $1, std::move(*$3)); delete $3; }
   | expr_op '+' expr_op
     { $$ = new ExprConcatStrings(state->at(@2), false, new std::vector<std::pair<PosIdx, Expr *> >({{state->at(@1), $1}, {state->at(@3), $3}})); }
   | expr_op '-' expr_op { $$ = new ExprCall(state->at(@2), new ExprVar(state->s.sub), {$1, $3}); }
@@ -278,9 +282,9 @@ expr_app
 
 expr_select
   : expr_simple '.' attrpath
-    { $$ = new ExprSelect(CUR_POS, $1, std::move(*$3), nullptr); delete $3; }
+    { $$ = new ExprSelect(state->alloc, CUR_POS, $1, std::move(*$3), nullptr); delete $3; }
   | expr_simple '.' attrpath OR_KW expr_select
-    { $$ = new ExprSelect(CUR_POS, $1, std::move(*$3), $5); delete $3; $5->warnIfCursedOr(state->symbols, state->positions); }
+    { $$ = new ExprSelect(state->alloc, CUR_POS, $1, std::move(*$3), $5); delete $3; $5->warnIfCursedOr(state->symbols, state->positions); }
   | /* Backwards compatibility: because Nixpkgs has a function named ‘or’,
        allow stuff like ‘map or [...]’. This production is problematic (see
        https://github.com/NixOS/nix/issues/11118) and will be refactored in the
@@ -303,7 +307,13 @@ expr_simple
   }
   | INT_LIT { $$ = new ExprInt($1); }
   | FLOAT_LIT { $$ = new ExprFloat($1); }
-  | '"' string_parts '"' { $$ = $2; }
+  | '"' string_parts '"' {
+      std::visit(overloaded{
+          [&](std::string_view str) { $$ = new ExprString(state->alloc, str); },
+          [&](Expr * expr) { $$ = expr; }},
+      *$2);
+      delete $2;
+  }
   | IND_STRING_OPEN ind_string_parts IND_STRING_CLOSE {
       $$ = state->stripIndentation(CUR_POS, std::move(*$2));
       delete $2;
@@ -314,11 +324,11 @@ expr_simple
       $$ = new ExprConcatStrings(CUR_POS, false, $2);
   }
   | SPATH {
-      std::string path($1.p + 1, $1.l - 2);
+      std::string_view path($1.p + 1, $1.l - 2);
       $$ = new ExprCall(CUR_POS,
           new ExprVar(state->s.findFile),
           {new ExprVar(state->s.nixPath),
-           new ExprString(std::move(path))});
+           new ExprString(state->alloc, path)});
   }
   | URI {
       static bool noURLLiterals = experimentalFeatureSettings.isEnabled(Xp::NoUrlLiterals);
@@ -327,13 +337,13 @@ expr_simple
               .msg = HintFmt("URL literals are disabled"),
               .pos = state->positions[CUR_POS]
           });
-      $$ = new ExprString(std::string($1));
+      $$ = new ExprString(state->alloc, $1);
   }
   | '(' expr ')' { $$ = $2; }
   /* Let expressions `let {..., body = ...}' are just desugared
      into `(rec {..., body = ...}).body'. */
   | LET '{' binds '}'
-    { $3->recursive = true; $3->pos = CUR_POS; $$ = new ExprSelect(noPos, $3, state->s.body); }
+    { $3->recursive = true; $3->pos = CUR_POS; $$ = new ExprSelect(state->alloc, noPos, $3, state->s.body); }
   | REC '{' binds '}'
     { $3->recursive = true; $3->pos = CUR_POS; $$ = $3; }
   | '{' binds1 '}'
@@ -344,19 +354,19 @@ expr_simple
   ;
 
 string_parts
-  : STR { $$ = new ExprString(std::string($1)); }
-  | string_parts_interpolated { $$ = new ExprConcatStrings(CUR_POS, true, $1); }
-  | { $$ = new ExprString(""); }
+  : STR { $$ = new std::variant<Expr *, std::string_view>($1); }
+  | string_parts_interpolated { $$ = new std::variant<Expr *, std::string_view>(new ExprConcatStrings(CUR_POS, true, $1)); }
+  | { $$ = new std::variant<Expr *, std::string_view>(std::string_view()); }
   ;
 
 string_parts_interpolated
   : string_parts_interpolated STR
-  { $$ = $1; $1->emplace_back(state->at(@2), new ExprString(std::string($2))); }
+  { $$ = $1; $1->emplace_back(state->at(@2), new ExprString(state->alloc, $2)); }
   | string_parts_interpolated DOLLAR_CURLY expr '}' { $$ = $1; $1->emplace_back(state->at(@2), $3); }
   | DOLLAR_CURLY expr '}' { $$ = new std::vector<std::pair<PosIdx, Expr *>>; $$->emplace_back(state->at(@1), $2); }
   | STR DOLLAR_CURLY expr '}' {
       $$ = new std::vector<std::pair<PosIdx, Expr *>>;
-      $$->emplace_back(state->at(@1), new ExprString(std::string($1)));
+      $$->emplace_back(state->at(@1), new ExprString(state->alloc, $1));
       $$->emplace_back(state->at(@2), $3);
     }
   ;
@@ -382,8 +392,8 @@ path_start
            root filesystem accessor, rather than the accessor of the
            current Nix expression. */
         literal.front() == '/'
-        ? new ExprPath(state->rootFS, std::move(path))
-        : new ExprPath(state->basePath.accessor, std::move(path));
+        ? new ExprPath(state->alloc, state->rootFS, path)
+        : new ExprPath(state->alloc, state->basePath.accessor, path);
   }
   | HPATH {
     if (state->settings.pureEval) {
@@ -393,7 +403,7 @@ path_start
         );
     }
     Path path(getHome() + std::string($1.p + 1, $1.l - 1));
-    $$ = new ExprPath(ref<SourceAccessor>(state->rootFS), std::move(path));
+    $$ = new ExprPath(state->alloc, ref<SourceAccessor>(state->rootFS), path);
   }
   ;
 
@@ -437,7 +447,7 @@ binds1
           $accum->attrs.emplace(
               i.symbol,
               ExprAttrs::AttrDef(
-                  new ExprSelect(iPos, from, i.symbol),
+                  new ExprSelect(state->alloc, iPos, from, i.symbol),
                   iPos,
                   ExprAttrs::AttrDef::Kind::InheritedFrom));
       }
@@ -454,15 +464,16 @@ attrs
   : attrs attr { $$ = $1; $1->emplace_back(AttrName(state->symbols.create($2)), state->at(@2)); }
   | attrs string_attr
     { $$ = $1;
-      ExprString * str = dynamic_cast<ExprString *>($2);
-      if (str) {
-          $$->emplace_back(AttrName(state->symbols.create(str->s)), state->at(@2));
-          delete str;
-      } else
-          throw ParseError({
-              .msg = HintFmt("dynamic attributes not allowed in inherit"),
-              .pos = state->positions[state->at(@2)]
-          });
+      std::visit(overloaded {
+          [&](std::string_view str) { $$->emplace_back(AttrName(state->symbols.create(str)), state->at(@2)); },
+          [&](Expr * expr) {
+              throw ParseError({
+                  .msg = HintFmt("dynamic attributes not allowed in inherit"),
+                  .pos = state->positions[state->at(@2)]
+              });
+          }
+      }, *$2);
+      delete $2;
     }
   | { $$ = new std::vector<std::pair<AttrName, PosIdx>>; }
   ;
@@ -471,22 +482,20 @@ attrpath
   : attrpath '.' attr { $$ = $1; $1->push_back(AttrName(state->symbols.create($3))); }
   | attrpath '.' string_attr
     { $$ = $1;
-      ExprString * str = dynamic_cast<ExprString *>($3);
-      if (str) {
-          $$->push_back(AttrName(state->symbols.create(str->s)));
-          delete str;
-      } else
-          $$->push_back(AttrName($3));
+      std::visit(overloaded {
+          [&](std::string_view str) { $$->push_back(AttrName(state->symbols.create(str))); },
+          [&](Expr * expr) { $$->push_back(AttrName(expr)); }
+      }, *$3);
+      delete $3;
     }
   | attr { $$ = new std::vector<AttrName>; $$->push_back(AttrName(state->symbols.create($1))); }
   | string_attr
     { $$ = new std::vector<AttrName>;
-      ExprString *str = dynamic_cast<ExprString *>($1);
-      if (str) {
-          $$->push_back(AttrName(state->symbols.create(str->s)));
-          delete str;
-      } else
-          $$->push_back(AttrName($1));
+      std::visit(overloaded {
+          [&](std::string_view str) { $$->push_back(AttrName(state->symbols.create(str))); },
+          [&](Expr * expr) { $$->push_back(AttrName(expr)); }
+      }, *$1);
+      delete $1;
     }
   ;
 
@@ -497,7 +506,7 @@ attr
 
 string_attr
   : '"' string_parts '"' { $$ = $2; }
-  | DOLLAR_CURLY expr '}' { $$ = $2; }
+  | DOLLAR_CURLY expr '}' { $$ = new std::variant<Expr *, std::string_view>($2); }
   ;
 
 expr_list
@@ -537,6 +546,7 @@ Expr * parseExprFromBuf(
     size_t length,
     Pos::Origin origin,
     const SourcePath & basePath,
+    std::pmr::polymorphic_allocator<char> & alloc,
     SymbolTable & symbols,
     const EvalSettings & settings,
     PosTable & positions,
@@ -551,6 +561,7 @@ Expr * parseExprFromBuf(
     };
     ParserState state {
         .lexerState = lexerState,
+        .alloc = alloc,
         .symbols = symbols,
         .positions = positions,
         .basePath = basePath,
@@ -563,7 +574,8 @@ Expr * parseExprFromBuf(
     Finally _destroy([&] { yylex_destroy(scanner); });
 
     yy_scan_buffer(text, length, scanner);
-    yyparse(scanner, &state);
+    Parser parser(scanner, &state);
+    parser.parse();
 
     return state.result;
 }

src/libexpr/paths.cc

@@ -1,5 +1,7 @@
 #include "nix/store/store-api.hh"
 #include "nix/expr/eval.hh"
+#include "nix/util/mounted-source-accessor.hh"
+#include "nix/fetchers/fetch-to-store.hh"
 
 namespace nix {
 
@@ -18,4 +20,27 @@ SourcePath EvalState::storePath(const StorePath & path)
     return {rootFS, CanonPath{store->printStorePath(path)}};
 }
 
+StorePath
+EvalState::mountInput(fetchers::Input & input, const fetchers::Input & originalInput, ref<SourceAccessor> accessor)
+{
+    auto storePath = fetchToStore(fetchSettings, *store, accessor, FetchMode::Copy, input.getName());
+
+    allowPath(storePath); // FIXME: should just whitelist the entire virtual store
+
+    storeFS->mount(CanonPath(store->printStorePath(storePath)), accessor);
+
+    auto narHash = store->queryPathInfo(storePath)->narHash;
+    input.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true));
+
+    if (originalInput.getNarHash() && narHash != *originalInput.getNarHash())
+        throw Error(
+            (unsigned int) 102,
+            "NAR hash mismatch in input '%s', expected '%s' but got '%s'",
+            originalInput.to_string(),
+            narHash.to_string(HashFormat::SRI, true),
+            originalInput.getNarHash()->to_string(HashFormat::SRI, true));
+
+    return storePath;
+}
+
 } // namespace nix

src/libexpr/primops.cc

@@ -18,6 +18,8 @@
 #include "nix/util/sort.hh"
 
 #include <boost/container/small_vector.hpp>
+#include <boost/unordered/concurrent_flat_map.hpp>
+#include <boost/unordered/unordered_flat_map.hpp>
 #include <nlohmann/json.hpp>
 
 #include <sys/types.h>
@@ -260,7 +262,7 @@ static void scopedImport(EvalState & state, const PosIdx pos, SourcePath & path,
 {
     state.forceAttrs(*vScope, pos, "while evaluating the first argument passed to builtins.scopedImport");
 
-    Env * env = &state.allocEnv(vScope->attrs()->size());
+    Env * env = &state.mem.allocEnv(vScope->attrs()->size());
     env->up = &state.baseEnv;
 
     auto staticEnv = std::make_shared<StaticEnv>(nullptr, state.staticBaseEnv, vScope->attrs()->size());
@@ -515,7 +517,6 @@ static void prim_typeOf(EvalState & state, const PosIdx pos, Value ** args, Valu
         v.mkStringNoCopy("float");
         break;
     case nThunk:
-    case nFailed:
         unreachable();
     }
 }
@@ -1076,11 +1077,11 @@ static void prim_tryEval(EvalState & state, const PosIdx pos, Value ** args, Val
     try {
         state.forceValue(*args[0], pos);
         attrs.insert(state.s.value, args[0]);
-        attrs.insert(state.symbols.create("success"), &state.vTrue);
+        attrs.insert(state.symbols.create("success"), &Value::vTrue);
     } catch (AssertionError & e) {
         // `value = false;` is unfortunate but removing it is a breaking change.
-        attrs.insert(state.s.value, &state.vFalse);
-        attrs.insert(state.symbols.create("success"), &state.vFalse);
+        attrs.insert(state.s.value, &Value::vFalse);
+        attrs.insert(state.symbols.create("success"), &Value::vFalse);
     }
 
     // restore the debugRepl pointer if we saved it earlier.
@@ -1366,8 +1367,8 @@ static void derivationStrictInternal(EvalState & state, std::string_view drvName
     using nlohmann::json;
     std::optional<StructuredAttrs> jsonObject;
     auto pos = v.determinePos(noPos);
-    auto attr = attrs->find(state.s.structuredAttrs);
-    if (attr != attrs->end()
+    auto attr = attrs->get(state.s.structuredAttrs);
+    if (attr
         && state.forceBool(
             *attr->value,
             pos,
@@ -1377,8 +1378,8 @@ static void derivationStrictInternal(EvalState & state, std::string_view drvName
 
     /* Check whether null attributes should be ignored. */
     bool ignoreNulls = false;
-    attr = attrs->find(state.s.ignoreNulls);
-    if (attr != attrs->end())
+    attr = attrs->get(state.s.ignoreNulls);
+    if (attr)
         ignoreNulls = state.forceBool(
             *attr->value,
             pos,
@@ -1751,7 +1752,7 @@ static void derivationStrictInternal(EvalState & state, std::string_view drvName
        read them later. */
     {
         auto h = hashDerivationModulo(*state.store, drv, false);
-        drvHashes.lock()->insert_or_assign(drvPath, h);
+        drvHashes.insert_or_assign(drvPath, std::move(h));
     }
 
     auto result = state.buildBindings(1 + drv.outputs.size());
@@ -2039,8 +2040,8 @@ static void prim_findFile(EvalState & state, const PosIdx pos, Value ** args, Va
         state.forceAttrs(*v2, pos, "while evaluating an element of the list passed to builtins.findFile");
 
         std::string prefix;
-        auto i = v2->attrs()->find(state.s.prefix);
-        if (i != v2->attrs()->end())
+        auto i = v2->attrs()->get(state.s.prefix);
+        if (i)
             prefix = state.forceStringNoCtx(
                 *i->value,
                 pos,
@@ -2242,19 +2243,45 @@ static RegisterPrimOp primop_hashFile({
     .fun = prim_hashFile,
 });
 
-static Value * fileTypeToString(EvalState & state, SourceAccessor::Type type)
+static const Value & fileTypeToString(EvalState & state, SourceAccessor::Type type)
 {
-    return type == SourceAccessor::Type::tRegular     ? &state.vStringRegular
-           : type == SourceAccessor::Type::tDirectory ? &state.vStringDirectory
-           : type == SourceAccessor::Type::tSymlink   ? &state.vStringSymlink
-                                                      : &state.vStringUnknown;
+    struct Constants
+    {
+        Value regular;
+        Value directory;
+        Value symlink;
+        Value unknown;
+    };
+
+    static const Constants stringValues = []() {
+        Constants res;
+        res.regular.mkStringNoCopy("regular");
+        res.directory.mkStringNoCopy("directory");
+        res.symlink.mkStringNoCopy("symlink");
+        res.unknown.mkStringNoCopy("unknown");
+        return res;
+    }();
+
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wswitch-enum"
+    using enum SourceAccessor::Type;
+    switch (type) {
+    case tRegular:
+        return stringValues.regular;
+    case tDirectory:
+        return stringValues.directory;
+    case tSymlink:
+        return stringValues.symlink;
+    default:
+        return stringValues.unknown;
+    }
 }
 
 static void prim_readFileType(EvalState & state, const PosIdx pos, Value ** args, Value & v)
 {
     auto path = realisePath(state, pos, *args[0], std::nullopt);
     /* Retrieve the directory entry type and stringize it. */
-    v = *fileTypeToString(state, path.lstat().type);
+    v = fileTypeToString(state, path.lstat().type);
 }
 
 static RegisterPrimOp primop_readFileType({
@@ -2298,7 +2325,9 @@ static void prim_readDir(EvalState & state, const PosIdx pos, Value ** args, Val
         } else {
             // This branch of the conditional is much more likely.
             // Here we just stringize the directory entry type.
-            attrs.insert(state.symbols.create(name), fileTypeToString(state, *type));
+            // N.B. const_cast here is ok, because these values will never be modified, since
+            // only thunks are mutable - other types do not change once constructed.
+            attrs.insert(state.symbols.create(name), const_cast<Value *>(&fileTypeToString(state, *type)));
         }
     }
 
@@ -2383,7 +2412,7 @@ static void prim_toXML(EvalState & state, const PosIdx pos, Value ** args, Value
     std::ostringstream out;
     NixStringContext context;
     printValueAsXML(state, true, false, *args[0], out, context, pos);
-    v.mkString(toView(out), context);
+    v.mkString(out.view(), context);
 }
 
 static RegisterPrimOp primop_toXML({
@@ -2491,7 +2520,7 @@ static void prim_toJSON(EvalState & state, const PosIdx pos, Value ** args, Valu
     std::ostringstream out;
     NixStringContext context;
     printValueAsJSON(state, true, *args[0], pos, out, context);
-    v.mkString(toView(out), context);
+    v.mkString(out.view(), context);
 }
 
 static RegisterPrimOp primop_toJSON({
@@ -2673,7 +2702,7 @@ bool EvalState::callPathFilter(Value * filterFun, const SourcePath & path, PosId
     arg1.mkString(path.path.abs());
 
     // assert that type is not "unknown"
-    Value * args[]{&arg1, fileTypeToString(*this, st.type)};
+    Value * args[]{&arg1, const_cast<Value *>(&fileTypeToString(*this, st.type))};
     Value res;
     callFunction(*filterFun, args, res, pos);
 
@@ -2979,8 +3008,8 @@ static void prim_unsafeGetAttrPos(EvalState & state, const PosIdx pos, Value **
     auto attr = state.forceStringNoCtx(
         *args[0], pos, "while evaluating the first argument passed to builtins.unsafeGetAttrPos");
     state.forceAttrs(*args[1], pos, "while evaluating the second argument passed to builtins.unsafeGetAttrPos");
-    auto i = args[1]->attrs()->find(state.symbols.create(attr));
-    if (i == args[1]->attrs()->end())
+    auto i = args[1]->attrs()->get(state.symbols.create(attr));
+    if (!i)
         v.mkNull();
     else
         state.mkPos(v, i->pos);
@@ -3047,7 +3076,7 @@ static void prim_hasAttr(EvalState & state, const PosIdx pos, Value ** args, Val
 {
     auto attr = state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.hasAttr");
     state.forceAttrs(*args[1], pos, "while evaluating the second argument passed to builtins.hasAttr");
-    v.mkBool(args[1]->attrs()->find(state.symbols.create(attr)) != args[1]->attrs()->end());
+    v.mkBool(args[1]->attrs()->get(state.symbols.create(attr)));
 }
 
 static RegisterPrimOp primop_hasAttr({
@@ -3132,7 +3161,7 @@ static void prim_listToAttrs(EvalState & state, const PosIdx pos, Value ** args,
     // Step 1. Sort the name-value attrsets in place using the memory we allocate for the result
     auto listView = args[0]->listView();
     size_t listSize = listView.size();
-    auto & bindings = *state.allocBindings(listSize);
+    auto & bindings = *state.mem.allocBindings(listSize);
     using ElemPtr = decltype(&bindings[0].value);
 
     for (const auto & [n, v2] : enumerate(listView)) {
@@ -3257,14 +3286,14 @@ static void prim_intersectAttrs(EvalState & state, const PosIdx pos, Value ** ar
 
     if (left.size() < right.size()) {
         for (auto & l : left) {
-            auto r = right.find(l.name);
-            if (r != right.end())
+            auto r = right.get(l.name);
+            if (r)
                 attrs.insert(*r);
         }
     } else {
         for (auto & r : right) {
-            auto l = left.find(r.name);
-            if (l != left.end())
+            auto l = left.get(r.name);
+            if (l)
                 attrs.insert(r);
         }
     }
@@ -3327,14 +3356,14 @@ static void prim_functionArgs(EvalState & state, const PosIdx pos, Value ** args
 {
     state.forceValue(*args[0], pos);
     if (args[0]->isPrimOpApp() || args[0]->isPrimOp()) {
-        v.mkAttrs(&state.emptyBindings);
+        v.mkAttrs(&Bindings::emptyBindings);
         return;
     }
     if (!args[0]->isLambda())
         state.error<TypeError>("'functionArgs' requires a function").atPos(pos).debugThrow();
 
     if (!args[0]->lambda().fun->hasFormals()) {
-        v.mkAttrs(&state.emptyBindings);
+        v.mkAttrs(&Bindings::emptyBindings);
         return;
     }
 
@@ -4028,7 +4057,7 @@ static void prim_groupBy(EvalState & state, const PosIdx pos, Value ** args, Val
         auto name = state.forceStringNoCtx(
             res, pos, "while evaluating the return value of the grouping function passed to builtins.groupBy");
         auto sym = state.symbols.create(name);
-        auto vector = attrs.try_emplace(sym, ValueVector()).first;
+        auto vector = attrs.try_emplace<ValueVector>(sym, {}).first;
         vector->second.push_back(vElem);
     }
 
@@ -4563,27 +4592,21 @@ static RegisterPrimOp primop_convertHash({
 
 struct RegexCache
 {
-    struct State
-    {
-        std::unordered_map<std::string, std::regex, StringViewHash, std::equal_to<>> cache;
-    };
-
-    Sync<State> state_;
+    boost::concurrent_flat_map<std::string, std::regex, StringViewHash, std::equal_to<>> cache;
 
     std::regex get(std::string_view re)
     {
-        auto state(state_.lock());
-        auto it = state->cache.find(re);
-        if (it != state->cache.end())
-            return it->second;
+        std::regex regex;
         /* No std::regex constructor overload from std::string_view, but can be constructed
            from a pointer + size or an iterator range. */
-        return state->cache
-            .emplace(
-                std::piecewise_construct,
-                std::forward_as_tuple(re),
-                std::forward_as_tuple(/*s=*/re.data(), /*count=*/re.size(), std::regex::extended))
-            .first->second;
+        cache.try_emplace_and_cvisit(
+            re,
+            /*s=*/re.data(),
+            /*count=*/re.size(),
+            std::regex::extended,
+            [&regex](const auto & kv) { regex = kv.second; },
+            [&regex](const auto & kv) { regex = kv.second; });
+        return regex;
     }
 };
 
@@ -4614,7 +4637,7 @@ void prim_match(EvalState & state, const PosIdx pos, Value ** args, Value & v)
         auto list = state.buildList(match.size() - 1);
         for (const auto & [i, v2] : enumerate(list))
             if (!match[i + 1].matched)
-                v2 = &state.vNull;
+                v2 = &Value::vNull;
             else
                 v2 = mkString(state, match[i + 1]);
         v.mkList(list);
@@ -4706,7 +4729,7 @@ void prim_split(EvalState & state, const PosIdx pos, Value ** args, Value & v)
             auto list2 = state.buildList(slen);
             for (const auto & [si, v2] : enumerate(list2)) {
                 if (!match[si + 1].matched)
-                    v2 = &state.vNull;
+                    v2 = &Value::vNull;
                 else
                     v2 = mkString(state, match[si + 1]);
             }
@@ -4827,7 +4850,7 @@ static void prim_replaceStrings(EvalState & state, const PosIdx pos, Value ** ar
         from.emplace_back(state.forceString(
             *elem, pos, "while evaluating one of the strings to replace passed to builtins.replaceStrings"));
 
-    std::unordered_map<size_t, std::string_view> cache;
+    boost::unordered_flat_map<size_t, std::string_view> cache;
     auto to = args[1]->listView();
 
     NixStringContext context;
@@ -5060,7 +5083,7 @@ void EvalState::createBaseEnv(const EvalSettings & evalSettings)
 
     addConstant(
         "null",
-        &vNull,
+        &Value::vNull,
         {
             .type = nNull,
             .doc = R"(

src/libexpr/primops/fetchTree.cc

@@ -10,6 +10,7 @@
 #include "nix/util/url.hh"
 #include "nix/expr/value-to-json.hh"
 #include "nix/fetchers/fetch-to-store.hh"
+#include "nix/fetchers/input-cache.hh"
 
 #include <nlohmann/json.hpp>
 
@@ -218,11 +219,11 @@ static void fetchTree(
             throw Error("input '%s' is not allowed to use the '__final' attribute", input.to_string());
     }
 
-    auto [storePath, input2] = input.fetchToStore(state.store);
+    auto cachedInput = state.inputCache->getAccessor(state.store, input, fetchers::UseRegistries::No);
 
-    state.allowPath(storePath);
+    auto storePath = state.mountInput(cachedInput.lockedInput, input, cachedInput.accessor);
 
-    emitTreeAttrs(state, storePath, input2, v, params.emptyRevFallback, false);
+    emitTreeAttrs(state, storePath, cachedInput.lockedInput, v, params.emptyRevFallback, false);
 }
 
 static void prim_fetchTree(EvalState & state, const PosIdx pos, Value ** args, Value & v)
@@ -561,14 +562,22 @@ static void fetch(
                 .hash = *expectedHash,
                 .references = {}});
 
-        if (state.store->isValidPath(expectedPath)) {
+        // Try to get the path from the local store or substituters
+        try {
+            state.store->ensurePath(expectedPath);
+            debug("using substituted/cached path '%s' for '%s'", state.store->printStorePath(expectedPath), *url);
             state.allowAndSetStorePathString(expectedPath, v);
             return;
+        } catch (Error & e) {
+            debug(
+                "substitution of '%s' failed, will try to download: %s",
+                state.store->printStorePath(expectedPath),
+                e.what());
+            // Fall through to download
         }
     }
 
-    // TODO: fetching may fail, yet the path may be substitutable.
-    //       https://github.com/NixOS/nix/issues/4313
+    // Download the file/tarball if substitution failed or no hash was provided
     auto storePath = unpack ? fetchToStore(
                                   state.fetchSettings,
                                   *state.store,

src/libexpr/primops/fromTOML.cc

@@ -139,7 +139,7 @@ static void prim_fromTOML(EvalState & state, const PosIdx pos, Value ** args, Va
                 attrs.alloc("_type").mkStringNoCopy("timestamp");
                 std::ostringstream s;
                 s << t;
-                auto str = toView(s);
+                auto str = s.view();
                 forceNoNullByte(str);
                 attrs.alloc("value").mkString(str);
                 v.mkAttrs(attrs);

src/libexpr/print-ambiguous.cc

@@ -75,9 +75,6 @@ void printAmbiguous(
             str << "«potential infinite recursion»";
         }
         break;
-    case nFailed:
-        str << "«failed»";
-        break;
     case nFunction:
         if (v.isLambda()) {
             str << "<LAMBDA>";

src/libexpr/print.cc

@@ -1,5 +1,4 @@
 #include <limits>
-#include <unordered_set>
 #include <sstream>
 
 #include "nix/expr/print.hh"
@@ -10,6 +9,8 @@
 #include "nix/util/english.hh"
 #include "nix/expr/eval.hh"
 
+#include <boost/unordered/unordered_flat_set.hpp>
+
 namespace nix {
 
 void printElided(
@@ -81,7 +82,7 @@ std::ostream & printLiteralBool(std::ostream & str, bool boolean)
 // For example `or' doesn't need to be quoted.
 bool isReservedKeyword(const std::string_view str)
 {
-    static const std::unordered_set<std::string_view> reservedKeywords = {
+    static const boost::unordered_flat_set<std::string_view> reservedKeywords = {
         "if", "then", "else", "assert", "with", "let", "in", "rec", "inherit"};
     return reservedKeywords.contains(str);
 }
@@ -460,7 +461,7 @@ private:
 
                 std::ostringstream s;
                 s << state.positions[v.lambda().fun->pos];
-                output << " @ " << filterANSIEscapes(toView(s));
+                output << " @ " << filterANSIEscapes(s.view());
             }
         } else if (v.isPrimOp()) {
             if (v.primOp())
@@ -508,11 +509,6 @@ private:
         }
     }
 
-    void printFailed(Value & v)
-    {
-        output << "«failed»";
-    }
-
     void printExternal(Value & v)
     {
         v.external()->print(output);
@@ -588,10 +584,6 @@ private:
                 printThunk(v);
                 break;
 
-            case nFailed:
-                printFailed(v);
-                break;
-
             case nExternal:
                 printExternal(v);
                 break;

src/libexpr/value-to-json.cc

@@ -96,7 +96,6 @@ json printValueAsJSON(
         break;
 
     case nThunk:
-    case nFailed:
     case nFunction:
         state.error<TypeError>("cannot convert %1% to JSON", showType(v)).atPos(v.determinePos(pos)).debugThrow();
     }

src/libexpr/value-to-xml.cc

@@ -170,11 +170,6 @@ static void printValueAsXML(
 
     case nThunk:
         doc.writeEmptyElement("unevaluated");
-        break;
-
-    case nFailed:
-        doc.writeEmptyElement("failed");
-        break;
     }
 }
 

src/libexpr/value.cc

@@ -0,0 +1,29 @@
+#include "nix/expr/value.hh"
+
+namespace nix {
+
+Value Value::vEmptyList = []() {
+    Value res;
+    res.setStorage(List{.size = 0, .elems = nullptr});
+    return res;
+}();
+
+Value Value::vNull = []() {
+    Value res;
+    res.mkNull();
+    return res;
+}();
+
+Value Value::vTrue = []() {
+    Value res;
+    res.mkBool(true);
+    return res;
+}();
+
+Value Value::vFalse = []() {
+    Value res;
+    res.mkBool(false);
+    return res;
+}();
+
+} // namespace nix

src/libfetchers-c/meson.build

@@ -32,6 +32,7 @@ add_project_arguments(
 )
 
 subdir('nix-meson-build-support/common')
+subdir('nix-meson-build-support/asan-options')
 
 sources = files(
   'nix_api_fetchers.cc',
@@ -53,6 +54,7 @@ subdir('nix-meson-build-support/windows-version')
 this_library = library(
   'nixfetchersc',
   sources,
+  soversion : nix_soversion,
   dependencies : deps_public + deps_private + deps_other,
   include_directories : include_dirs,
   link_args : linker_export_flags,

src/libfetchers-tests/git-utils.cc

@@ -175,6 +175,12 @@ TEST_F(GitUtilsTest, peel_reference)
 
 TEST(GitUtils, isLegalRefName)
 {
+    ASSERT_TRUE(isLegalRefName("A/b"));
+    ASSERT_TRUE(isLegalRefName("AaA/b"));
+    ASSERT_TRUE(isLegalRefName("FOO/BAR/BAZ"));
+    ASSERT_TRUE(isLegalRefName("HEAD"));
+    ASSERT_TRUE(isLegalRefName("refs/tags/1.2.3"));
+    ASSERT_TRUE(isLegalRefName("refs/heads/master"));
     ASSERT_TRUE(isLegalRefName("foox"));
     ASSERT_TRUE(isLegalRefName("1337"));
     ASSERT_TRUE(isLegalRefName("foo.baz"));

src/libfetchers-tests/git.cc

@@ -1,5 +1,6 @@
 #include "nix/store/store-open.hh"
 #include "nix/store/globals.hh"
+#include "nix/store/dummy-store.hh"
 #include "nix/fetchers/fetch-settings.hh"
 #include "nix/fetchers/fetchers.hh"
 #include "nix/fetchers/git-utils.hh"
@@ -179,10 +180,11 @@ TEST_F(GitTest, submodulePeriodSupport)
     // 6) Commit the addition in super
     commitAll(super.get(), "Add submodule with branch='.'");
 
-    // TODO: Use dummy:// store with MemorySourceAccessor.
-    Path storeTmpDir = createTempDir();
-    auto storeTmpDirAutoDelete = AutoDelete(storeTmpDir, true);
-    ref<Store> store = openStore(storeTmpDir);
+    auto store = [] {
+        auto cfg = make_ref<DummyStoreConfig>(StoreReference::Params{});
+        cfg->readOnly = false;
+        return cfg->openStore();
+    }();
 
     auto settings = fetchers::Settings{};
     auto input = fetchers::Input::fromAttrs(

src/libfetchers-tests/meson.build

@@ -37,6 +37,7 @@ libgit2 = dependency('libgit2')
 deps_private += libgit2
 
 subdir('nix-meson-build-support/common')
+subdir('nix-meson-build-support/asan-options')
 
 sources = files(
   'access-tokens.cc',
@@ -63,7 +64,7 @@ this_exe = executable(
 test(
   meson.project_name(),
   this_exe,
-  env : {
+  env : asan_test_options_env + {
     '_NIX_TEST_UNIT_DATA' : meson.current_source_dir() / 'data',
   },
   protocol : 'gtest',

src/libfetchers-tests/package.nix

@@ -61,6 +61,7 @@ mkMesonExecutable (finalAttrs: {
             buildInputs = [ writableTmpDirAsHomeHook ];
           }
           ''
+            export ASAN_OPTIONS=abort_on_error=1:print_summary=1:detect_leaks=0
             export _NIX_TEST_UNIT_DATA=${resolvePath ./data}
             ${stdenv.hostPlatform.emulator buildPackages} ${lib.getExe finalAttrs.finalPackage}
             touch $out

src/libfetchers-tests/public-key.cc

@@ -1,14 +1,14 @@
 #include <gtest/gtest.h>
 #include "nix/fetchers/fetchers.hh"
 #include "nix/util/json-utils.hh"
-#include <nlohmann/json.hpp>
-#include "nix/util/tests/characterization.hh"
+#include "nix/util/tests/json-characterization.hh"
 
 namespace nix {
 
 using nlohmann::json;
 
-class PublicKeyTest : public CharacterizationTest
+class PublicKeyTest : public JsonCharacterizationTest<fetchers::PublicKey>,
+                      public ::testing::WithParamInterface<std::pair<std::string_view, fetchers::PublicKey>>
 {
     std::filesystem::path unitTestData = getUnitTestData() / "public-key";
 
@@ -19,30 +19,35 @@ public:
     }
 };
 
-#define TEST_JSON(FIXTURE, NAME, VAL)                                                                 \
-    TEST_F(FIXTURE, PublicKey_##NAME##_from_json)                                                     \
-    {                                                                                                 \
-        readTest(#NAME ".json", [&](const auto & encoded_) {                                          \
-            fetchers::PublicKey expected{VAL};                                                        \
-            fetchers::PublicKey got = nlohmann::json::parse(encoded_);                                \
-            ASSERT_EQ(got, expected);                                                                 \
-        });                                                                                           \
-    }                                                                                                 \
-                                                                                                      \
-    TEST_F(FIXTURE, PublicKey_##NAME##_to_json)                                                       \
-    {                                                                                                 \
-        writeTest(                                                                                    \
-            #NAME ".json",                                                                            \
-            [&]() -> json { return nlohmann::json(fetchers::PublicKey{VAL}); },                       \
-            [](const auto & file) { return json::parse(readFile(file)); },                            \
-            [](const auto & file, const auto & got) { return writeFile(file, got.dump(2) + "\n"); }); \
-    }
+TEST_P(PublicKeyTest, from_json)
+{
+    const auto & [name, expected] = GetParam();
+    readJsonTest(name, expected);
+}
 
-TEST_JSON(PublicKeyTest, simple, (fetchers::PublicKey{.type = "ssh-rsa", .key = "ABCDE"}))
+TEST_P(PublicKeyTest, to_json)
+{
+    const auto & [name, value] = GetParam();
+    writeJsonTest(name, value);
+}
 
-TEST_JSON(PublicKeyTest, defaultType, fetchers::PublicKey{.key = "ABCDE"})
-
-#undef TEST_JSON
+INSTANTIATE_TEST_SUITE_P(
+    PublicKeyJSON,
+    PublicKeyTest,
+    ::testing::Values(
+        std::pair{
+            "simple",
+            fetchers::PublicKey{
+                .type = "ssh-rsa",
+                .key = "ABCDE",
+            },
+        },
+        std::pair{
+            "defaultType",
+            fetchers::PublicKey{
+                .key = "ABCDE",
+            },
+        }));
 
 TEST_F(PublicKeyTest, PublicKey_noRoundTrip_from_json)
 {

src/libfetchers/fetch-to-store.cc

@@ -1,6 +1,7 @@
 #include "nix/fetchers/fetch-to-store.hh"
 #include "nix/fetchers/fetchers.hh"
 #include "nix/fetchers/fetch-settings.hh"
+#include "nix/util/environment-variables.hh"
 
 namespace nix {
 
@@ -27,14 +28,22 @@ StorePath fetchToStore(
 
     std::optional<fetchers::Cache::Key> cacheKey;
 
-    if (!filter && path.accessor->fingerprint) {
-        cacheKey = makeFetchToStoreCacheKey(std::string{name}, *path.accessor->fingerprint, method, path.path.abs());
+    auto [subpath, fingerprint] = filter ? std::pair<CanonPath, std::optional<std::string>>{path.path, std::nullopt}
+                                         : path.accessor->getFingerprint(path.path);
+
+    if (fingerprint) {
+        cacheKey = makeFetchToStoreCacheKey(std::string{name}, *fingerprint, method, subpath.abs());
         if (auto res = settings.getCache()->lookupStorePath(*cacheKey, store)) {
             debug("store path cache hit for '%s'", path);
             return res->storePath;
         }
-    } else
+    } else {
+        static auto barf = getEnv("_NIX_TEST_BARF_ON_UNCACHEABLE").value_or("") == "1";
+        if (barf && !filter)
+            throw Error("source path '%s' is uncacheable (filter=%d)", path, (bool) filter);
+        // FIXME: could still provide in-memory caching keyed on `SourcePath`.
         debug("source path '%s' is uncacheable", path);
+    }
 
     Activity act(
         *logger,

src/libfetchers/fetchers.cc

@@ -3,8 +3,8 @@
 #include "nix/util/source-path.hh"
 #include "nix/fetchers/fetch-to-store.hh"
 #include "nix/util/json-utils.hh"
-#include "nix/fetchers/store-path-accessor.hh"
 #include "nix/fetchers/fetch-settings.hh"
+#include "nix/fetchers/fetch-to-store.hh"
 
 #include <nlohmann/json.hpp>
 
@@ -332,10 +332,20 @@ std::pair<ref<SourceAccessor>, Input> Input::getAccessorUnchecked(ref<Store> sto
 
             debug("using substituted/cached input '%s' in '%s'", to_string(), store->printStorePath(storePath));
 
-            auto accessor = makeStorePathAccessor(store, storePath);
+            // We just ensured the store object was there
+            auto accessor = ref{store->getFSAccessor(storePath)};
 
             accessor->fingerprint = getFingerprint(store);
 
+            // Store a cache entry for the substituted tree so later fetches
+            // can reuse the existing nar instead of copying the unpacked
+            // input back into the store on every evaluation.
+            if (accessor->fingerprint) {
+                ContentAddressMethod method = ContentAddressMethod::Raw::NixArchive;
+                auto cacheKey = makeFetchToStoreCacheKey(getName(), *accessor->fingerprint, method, "/");
+                settings->getCache()->upsert(cacheKey, *store, {}, storePath);
+            }
+
             accessor->setPathDisplay("«" + to_string() + "»");
 
             return {accessor, *this};
@@ -346,8 +356,10 @@ std::pair<ref<SourceAccessor>, Input> Input::getAccessorUnchecked(ref<Store> sto
 
     auto [accessor, result] = scheme->getAccessor(store, *this);
 
-    assert(!accessor->fingerprint);
-    accessor->fingerprint = result.getFingerprint(store);
+    if (!accessor->fingerprint)
+        accessor->fingerprint = result.getFingerprint(store);
+    else
+        result.cachedFingerprint = accessor->fingerprint;
 
     return {accessor, std::move(result)};
 }
@@ -509,7 +521,7 @@ fetchers::PublicKey adl_serializer<fetchers::PublicKey>::from_json(const json &
     return res;
 }
 
-void adl_serializer<fetchers::PublicKey>::to_json(json & json, fetchers::PublicKey p)
+void adl_serializer<fetchers::PublicKey>::to_json(json & json, const fetchers::PublicKey & p)
 {
     json["type"] = p.type;
     json["key"] = p.key;

src/libfetchers/filtering-source-accessor.cc

@@ -1,5 +1,7 @@
 #include "nix/fetchers/filtering-source-accessor.hh"
 
+#include <boost/unordered/unordered_flat_set.hpp>
+
 namespace nix {
 
 std::optional<std::filesystem::path> FilteringSourceAccessor::getPhysicalPath(const CanonPath & path)
@@ -14,15 +16,26 @@ std::string FilteringSourceAccessor::readFile(const CanonPath & path)
     return next->readFile(prefix / path);
 }
 
+void FilteringSourceAccessor::readFile(const CanonPath & path, Sink & sink, std::function<void(uint64_t)> sizeCallback)
+{
+    checkAccess(path);
+    return next->readFile(prefix / path, sink, sizeCallback);
+}
+
 bool FilteringSourceAccessor::pathExists(const CanonPath & path)
 {
     return isAllowed(path) && next->pathExists(prefix / path);
 }
 
 std::optional<SourceAccessor::Stat> FilteringSourceAccessor::maybeLstat(const CanonPath & path)
+{
+    return isAllowed(path) ? next->maybeLstat(prefix / path) : std::nullopt;
+}
+
+SourceAccessor::Stat FilteringSourceAccessor::lstat(const CanonPath & path)
 {
     checkAccess(path);
-    return next->maybeLstat(prefix / path);
+    return next->lstat(prefix / path);
 }
 
 SourceAccessor::DirEntries FilteringSourceAccessor::readDirectory(const CanonPath & path)
@@ -47,6 +60,13 @@ std::string FilteringSourceAccessor::showPath(const CanonPath & path)
     return displayPrefix + next->showPath(prefix / path) + displaySuffix;
 }
 
+std::pair<CanonPath, std::optional<std::string>> FilteringSourceAccessor::getFingerprint(const CanonPath & path)
+{
+    if (fingerprint)
+        return {path, fingerprint};
+    return next->getFingerprint(prefix / path);
+}
+
 void FilteringSourceAccessor::checkAccess(const CanonPath & path)
 {
     if (!isAllowed(path))
@@ -57,12 +77,12 @@ void FilteringSourceAccessor::checkAccess(const CanonPath & path)
 struct AllowListSourceAccessorImpl : AllowListSourceAccessor
 {
     std::set<CanonPath> allowedPrefixes;
-    std::unordered_set<CanonPath> allowedPaths;
+    boost::unordered_flat_set<CanonPath> allowedPaths;
 
     AllowListSourceAccessorImpl(
         ref<SourceAccessor> next,
         std::set<CanonPath> && allowedPrefixes,
-        std::unordered_set<CanonPath> && allowedPaths,
+        boost::unordered_flat_set<CanonPath> && allowedPaths,
         MakeNotAllowedError && makeNotAllowedError)
         : AllowListSourceAccessor(SourcePath(next), std::move(makeNotAllowedError))
         , allowedPrefixes(std::move(allowedPrefixes))
@@ -84,7 +104,7 @@ struct AllowListSourceAccessorImpl : AllowListSourceAccessor
 ref<AllowListSourceAccessor> AllowListSourceAccessor::create(
     ref<SourceAccessor> next,
     std::set<CanonPath> && allowedPrefixes,
-    std::unordered_set<CanonPath> && allowedPaths,
+    boost::unordered_flat_set<CanonPath> && allowedPaths,
     MakeNotAllowedError && makeNotAllowedError)
 {
     return make_ref<AllowListSourceAccessorImpl>(

src/libfetchers/git-utils.cc

@@ -12,6 +12,7 @@
 
 #include <git2/attr.h>
 #include <git2/blob.h>
+#include <git2/branch.h>
 #include <git2/commit.h>
 #include <git2/config.h>
 #include <git2/describe.h>
@@ -28,10 +29,12 @@
 #include <git2/submodule.h>
 #include <git2/sys/odb_backend.h>
 #include <git2/sys/mempack.h>
+#include <git2/tag.h>
 #include <git2/tree.h>
 
+#include <boost/unordered/unordered_flat_map.hpp>
+#include <boost/unordered/unordered_flat_set.hpp>
 #include <iostream>
-#include <unordered_set>
 #include <queue>
 #include <regex>
 #include <span>
@@ -315,7 +318,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
 
     uint64_t getRevCount(const Hash & rev) override
     {
-        std::unordered_set<git_oid> done;
+        boost::unordered_flat_set<git_oid, std::hash<git_oid>> done;
         std::queue<Commit> todo;
 
         todo.push(peelObject<Commit>(lookupObject(*this, hashToOID(rev)).get(), GIT_OBJECT_COMMIT));
@@ -569,7 +572,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
     void verifyCommit(const Hash & rev, const std::vector<fetchers::PublicKey> & publicKeys) override
     {
         // Map of SSH key types to their internal OpenSSH representations
-        static const std::unordered_map<std::string_view, std::string_view> keyTypeMap = {
+        static const boost::unordered_flat_map<std::string_view, std::string_view> keyTypeMap = {
             {"ssh-dsa", "ssh-dsa"},
             {"ssh-ecdsa", "ssh-ecdsa"},
             {"ssh-ecdsa-sk", "sk-ecdsa-sha2-nistp256@openssh.com"},
@@ -816,7 +819,7 @@ struct GitSourceAccessor : SourceAccessor
         return toHash(*git_tree_entry_id(entry));
     }
 
-    std::unordered_map<CanonPath, TreeEntry> lookupCache;
+    boost::unordered_flat_map<CanonPath, TreeEntry> lookupCache;
 
     /* Recursively look up 'path' relative to the root. */
     git_tree_entry * lookup(State & state, const CanonPath & path)
@@ -1253,7 +1256,7 @@ GitRepoImpl::getAccessor(const WorkdirInfo & wd, bool exportIgnore, MakeNotAllow
                                            makeFSSourceAccessor(path),
                                            std::set<CanonPath>{wd.files},
                                            // Always allow access to the root, but not its children.
-                                           std::unordered_set<CanonPath>{CanonPath::root},
+                                           boost::unordered_flat_set<CanonPath>{CanonPath::root},
                                            std::move(makeNotAllowedError))
                                            .cast<SourceAccessor>();
     if (exportIgnore)
@@ -1322,63 +1325,33 @@ GitRepo::WorkdirInfo GitRepo::getCachedWorkdirInfo(const std::filesystem::path &
     return workdirInfo;
 }
 
-/**
- * Checks that the git reference is valid and normalizes slash '/' sequences.
- *
- * Accepts shorthand references (one-level refnames are allowed).
- */
-bool isValidRefNameAllowNormalizations(const std::string & refName)
-{
-    /* Unfortunately libgit2 doesn't expose the limit in headers, but its internal
-       limit is also 1024. */
-    std::array<char, 1024> normalizedRefBuffer;
-
-    /* It would be nice to have a better API like git_reference_name_is_valid, but
-     * with GIT_REFERENCE_FORMAT_REFSPEC_SHORTHAND flag. libgit2 uses it internally
-     * but doesn't expose it in public headers [1].
-     * [1]:
-     * https://github.com/libgit2/libgit2/blob/9d5f1bacc23594c2ba324c8f0d41b88bf0e9ef04/src/libgit2/refs.c#L1362-L1365
-     */
-
-    auto res = git_reference_normalize_name(
-        normalizedRefBuffer.data(),
-        normalizedRefBuffer.size(),
-        refName.c_str(),
-        GIT_REFERENCE_FORMAT_ALLOW_ONELEVEL | GIT_REFERENCE_FORMAT_REFSPEC_SHORTHAND);
-
-    return res == 0;
-}
-
 bool isLegalRefName(const std::string & refName)
 {
     initLibGit2();
 
-    /* Since `git_reference_normalize_name` is the best API libgit2 has for verifying
-     * reference names with shorthands (see comment in normalizeRefName), we need to
-     * ensure that exceptions to the validity checks imposed by normalization [1] are checked
-     * explicitly.
-     * [1]: https://git-scm.com/docs/git-check-ref-format#Documentation/git-check-ref-format.txt---normalize
-     */
-
     /* Check for cases that don't get rejected by libgit2.
      * FIXME: libgit2 should reject this. */
     if (refName == "@")
         return false;
 
-    /* Leading slashes and consecutive slashes are stripped during normalizatiton. */
-    if (refName.starts_with('/') || refName.find("//") != refName.npos)
-        return false;
-
-    /* Refer to libgit2. */
-    if (!isValidRefNameAllowNormalizations(refName))
-        return false;
-
     /* libgit2 doesn't barf on DEL symbol.
      * FIXME: libgit2 should reject this. */
     if (refName.find('\177') != refName.npos)
         return false;
 
-    return true;
+    for (auto * func : {
+             git_reference_name_is_valid,
+             git_branch_name_is_valid,
+             git_tag_name_is_valid,
+         }) {
+        int valid = 0;
+        if (func(&valid, refName.c_str()))
+            throw Error("checking git reference '%s': %s", refName, git_error_last()->message);
+        if (valid)
+            return true;
+    }
+
+    return false;
 }
 
 } // namespace nix