Merge pull request from GHSA-q82p-44mg-mgh5

Fix sandbox escape 2.22

.clang-format

@@ -28,3 +28,5 @@ EmptyLineBeforeAccessModifier: Leave
 #PackConstructorInitializers: BinPack
 BreakBeforeBinaryOperators: NonAssignment
 AlwaysBreakBeforeMultilineStrings: true
+IndentPPDirectives: AfterHash
+PPIndentWidth: 2

.github/CODEOWNERS

@@ -14,4 +14,4 @@
 src/libexpr/primops.cc @roberth
 
 # Libstore layer
-/src/libstore @thufschmitt
+/src/libstore @thufschmitt @ericson2314

.github/labeler.yml

@@ -1,3 +1,16 @@
+"c api":
+  - changed-files:
+    - any-glob-to-any-file: "src/lib*-c/**/*"
+    - any-glob-to-any-file: "test/unit/**/nix_api_*"
+    - any-glob-to-any-file: "doc/external-api/**/*"
+
+"contributor-experience":
+  - changed-files:
+    - any-glob-to-any-file: "CONTRIBUTING.md"
+    - any-glob-to-any-file: ".github/ISSUE_TEMPLATE/*"
+    - any-glob-to-any-file: ".github/PULL_REQUEST_TEMPLATE.md"
+    - any-glob-to-any-file: "doc/manual/src/contributing/**"
+
 "documentation":
   - changed-files:
     - any-glob-to-any-file: "doc/manual/*"

.github/workflows/backport.yml

@@ -21,7 +21,7 @@ jobs:
           fetch-depth: 0
       - name: Create backport PRs
         # should be kept in sync with `version`
-        uses: zeebe-io/backport-action@v2.4.1
+        uses: zeebe-io/backport-action@v2.5.0
         with:
           # Config README: https://github.com/zeebe-io/backport-action#backport-action
           github_token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/ci.yml

@@ -20,7 +20,7 @@ jobs:
     - uses: actions/checkout@v4
       with:
         fetch-depth: 0
-    - uses: cachix/install-nix-action@v25
+    - uses: cachix/install-nix-action@v26
       with:
         # The sandbox would otherwise be disabled by default on Darwin
         extra_nix_config: "sandbox = true"
@@ -62,7 +62,7 @@ jobs:
       with:
         fetch-depth: 0
     - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
-    - uses: cachix/install-nix-action@v25
+    - uses: cachix/install-nix-action@v26
       with:
         install_url: https://releases.nixos.org/nix/nix-2.20.3/install
     - uses: cachix/cachix-action@v14
@@ -84,7 +84,7 @@ jobs:
     steps:
     - uses: actions/checkout@v4
     - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
-    - uses: cachix/install-nix-action@v25
+    - uses: cachix/install-nix-action@v26
       with:
         install_url: '${{needs.installer.outputs.installerURL}}'
         install_options: "--tarball-url-prefix https://${{ env.CACHIX_NAME }}.cachix.org/serve"
@@ -114,7 +114,7 @@ jobs:
     - uses: actions/checkout@v4
       with:
         fetch-depth: 0
-    - uses: cachix/install-nix-action@v25
+    - uses: cachix/install-nix-action@v26
       with:
         install_url: https://releases.nixos.org/nix/nix-2.20.3/install
     - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
@@ -159,3 +159,11 @@ jobs:
         # deprecated 2024-02-24
         docker tag nix:$NIX_VERSION $IMAGE_ID:master
         docker push $IMAGE_ID:master
+
+  vm_tests:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v4
+      - uses: DeterminateSystems/nix-installer-action@main
+      - uses: DeterminateSystems/magic-nix-cache-action@main
+      - run: nix build -L .#hydraJobs.tests.githubFlakes .#hydraJobs.tests.tarballFlakes

.gitignore

@@ -49,6 +49,9 @@ perl/Makefile.config
 /src/libexpr/tests
 /tests/unit/libexpr/libnixexpr-tests
 
+# /src/libfetchers
+/tests/unit/libfetchers/libnixfetchers-tests
+
 # /src/libstore/
 *.gen.*
 /src/libstore/tests
@@ -115,8 +118,6 @@ perl/Makefile.config
 /misc/systemd/nix-daemon.conf
 /misc/upstart/nix-daemon.conf
 
-/src/resolve-system-dependencies/resolve-system-dependencies
-
 outputs/
 
 *.a
@@ -142,6 +143,7 @@ GTAGS
 
 # auto-generated compilation database
 compile_commands.json
+*.compile_commands.json
 
 nix-rust/target
 
@@ -152,6 +154,8 @@ result-*
 .vscode/
 .idea/
 
+.pre-commit-config.yaml
+
 # clangd and possibly more
 .cache/
 

.version

@@ -1 +1 @@
-2.21.0
+2.22.2

Makefile

@@ -7,6 +7,8 @@ clean-files += $(buildprefix)Makefile.config
 
 # List makefiles
 
+include mk/platform.mk
+
 ifeq ($(ENABLE_BUILD), yes)
 makefiles = \
   mk/precompiled-headers.mk \
@@ -18,8 +20,14 @@ makefiles = \
   src/libexpr/local.mk \
   src/libcmd/local.mk \
   src/nix/local.mk \
-  src/resolve-system-dependencies/local.mk \
+  src/libutil-c/local.mk \
+  src/libstore-c/local.mk \
+  src/libexpr-c/local.mk
+
+ifdef HOST_UNIX
+makefiles += \
   scripts/local.mk \
+  maintainers/local.mk \
   misc/bash/local.mk \
   misc/fish/local.mk \
   misc/zsh/local.mk \
@@ -27,6 +35,7 @@ makefiles = \
   misc/launchd/local.mk \
   misc/upstart/local.mk
 endif
+endif
 
 ifeq ($(ENABLE_UNIT_TESTS), yes)
 makefiles += \
@@ -34,19 +43,23 @@ makefiles += \
   tests/unit/libutil-support/local.mk \
   tests/unit/libstore/local.mk \
   tests/unit/libstore-support/local.mk \
+  tests/unit/libfetchers/local.mk \
   tests/unit/libexpr/local.mk \
   tests/unit/libexpr-support/local.mk
 endif
 
 ifeq ($(ENABLE_FUNCTIONAL_TESTS), yes)
+ifdef HOST_UNIX
 makefiles += \
   tests/functional/local.mk \
   tests/functional/ca/local.mk \
   tests/functional/git-hashing/local.mk \
   tests/functional/dyn-drv/local.mk \
+  tests/functional/local-overlay-store/local.mk \
   tests/functional/test-libstoreconsumer/local.mk \
   tests/functional/plugins/local.mk
 endif
+endif
 
 # Some makefiles require access to built programs and must be included late.
 makefiles-late =
@@ -59,6 +72,10 @@ ifeq ($(ENABLE_INTERNAL_API_DOCS), yes)
 makefiles-late += doc/internal-api/local.mk
 endif
 
+ifeq ($(ENABLE_EXTERNAL_API_DOCS), yes)
+makefiles-late += doc/external-api/local.mk
+endif
+
 # Miscellaneous global Flags
 
 OPTIMIZE = 1
@@ -71,8 +88,6 @@ else
   unexport NIX_HARDENING_ENABLE
 endif
 
-include mk/platform.mk
-
 ifdef HOST_WINDOWS
   # Windows DLLs are stricter about symbol visibility than Unix shared
   # objects --- see https://gcc.gnu.org/wiki/Visibility for details.
@@ -123,3 +138,10 @@ internal-api-html:
 	@echo "Internal API docs are disabled. Configure with '--enable-internal-api-docs', or avoid calling 'make internal-api-html'."
 	@exit 1
 endif
+
+ifneq ($(ENABLE_EXTERNAL_API_DOCS), yes)
+.PHONY: external-api-html
+external-api-html:
+	@echo "External API docs are disabled. Configure with '--enable-external-api-docs', or avoid calling 'make external-api-html'."
+	@exit 1
+endif

Makefile.config.in

@@ -12,6 +12,7 @@ ENABLE_BUILD = @ENABLE_BUILD@
 ENABLE_DOC_GEN = @ENABLE_DOC_GEN@
 ENABLE_FUNCTIONAL_TESTS = @ENABLE_FUNCTIONAL_TESTS@
 ENABLE_INTERNAL_API_DOCS = @ENABLE_INTERNAL_API_DOCS@
+ENABLE_EXTERNAL_API_DOCS = @ENABLE_EXTERNAL_API_DOCS@
 ENABLE_S3 = @ENABLE_S3@
 ENABLE_UNIT_TESTS = @ENABLE_UNIT_TESTS@
 GTEST_LIBS = @GTEST_LIBS@

configure.ac

@@ -150,6 +150,11 @@ AC_ARG_ENABLE(unit-tests, AS_HELP_STRING([--disable-unit-tests],[Do not build th
   ENABLE_UNIT_TESTS=$enableval, ENABLE_UNIT_TESTS=$ENABLE_BUILD)
 AC_SUBST(ENABLE_UNIT_TESTS)
 
+# Build external API docs by default
+AC_ARG_ENABLE(external_api_docs, AS_HELP_STRING([--enable-external-api-docs],[Build API docs for Nix's C interface]),
+  external_api_docs=$enableval, external_api_docs=yes)
+AC_SUBST(external_api_docs)
+
 AS_IF(
   [test "$ENABLE_BUILD" == "no" && test "$ENABLE_UNIT_TESTS" == "yes"],
   [AC_MSG_ERROR([Cannot enable unit tests when building overall is disabled. Please do not pass '--enable-unit-tests' or do not pass '--disable-build'.])])
@@ -172,6 +177,10 @@ AC_ARG_ENABLE(internal-api-docs, AS_HELP_STRING([--enable-internal-api-docs],[Bu
   ENABLE_INTERNAL_API_DOCS=$enableval, ENABLE_INTERNAL_API_DOCS=no)
 AC_SUBST(ENABLE_INTERNAL_API_DOCS)
 
+AC_ARG_ENABLE(external-api-docs, AS_HELP_STRING([--enable-external-api-docs],[Build API docs for Nix's external unstable C interfaces]),
+  ENABLE_EXTERNAL_API_DOCS=$enableval, ENABLE_EXTERNAL_API_DOCS=no)
+AC_SUBST(ENABLE_EXTERNAL_API_DOCS)
+
 AS_IF(
   [test "$ENABLE_FUNCTIONAL_TESTS" == "yes" || test "$ENABLE_DOC_GEN" == "yes"],
   [NEED_PROG(jq, jq)])
@@ -308,6 +317,17 @@ case "$host_os" in
                         [CXXFLAGS="$LIBSECCOMP_CFLAGS $CXXFLAGS"])
       have_seccomp=1
       AC_DEFINE([HAVE_SECCOMP], [1], [Whether seccomp is available and should be used for sandboxing.])
+      AC_COMPILE_IFELSE([
+        AC_LANG_SOURCE([[
+          #include <seccomp.h>
+          #ifndef __SNR_fchmodat2
+          # error "Missing support for fchmodat2"
+          #endif
+        ]])
+      ], [], [
+        echo "libseccomp is missing __SNR_fchmodat2. Please provide libseccomp 2.5.5 or later"
+        exit 1
+      ])
     else
       have_seccomp=
     fi

doc/external-api/.gitignore

@@ -0,0 +1,3 @@
+/doxygen.cfg
+/html
+/latex

doc/external-api/README.md

@@ -0,0 +1,121 @@
+# Getting started
+
+> **Warning** These bindings are **experimental**, which means they can change
+> at any time or be removed outright; nevertheless the plan is to provide a
+> stable external C API to the Nix language and the Nix store.
+
+The language library allows evaluating Nix expressions and interacting with Nix
+language values. The Nix store API is still rudimentary, and only allows
+initialising and connecting to a store for the Nix language evaluator to
+interact with.
+
+Currently there are two ways to interface with the Nix language evaluator
+programmatically:
+
+1. Embedding the evaluator
+2. Writing language plug-ins
+
+Embedding means you link the Nix C libraries in your program and use them from
+there. Adding a plug-in means you make a library that gets loaded by the Nix
+language evaluator, specified through a configuration option.
+
+Many of the components and mechanisms involved are not yet documented, therefore
+please refer to the [Nix source code](https://github.com/NixOS/nix/) for
+details. Additions to in-code documentation and the reference manual are highly
+appreciated.
+
+The following examples, for simplicity, don't include error handling. See the
+[Handling errors](@ref errors) section for more information.
+
+# Embedding the Nix Evaluator
+
+In this example we programmatically start the Nix language evaluator with a
+dummy store (that has no store paths and cannot be written to), and evaluate the
+Nix expression `builtins.nixVersion`.
+
+**main.c:**
+
+```C
+#include <nix_api_util.h>
+#include <nix_api_expr.h>
+#include <nix_api_value.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+// NOTE: This example lacks all error handling. Production code must check for
+// errors, as some return values will be undefined.
+
+void my_get_string_cb(const char * start, unsigned int n, char ** user_data)
+{
+    *user_data = strdup(start);
+}
+
+int main()
+{
+    nix_libexpr_init(NULL);
+
+    Store * store = nix_store_open(NULL, "dummy://", NULL);
+    EvalState * state = nix_state_create(NULL, NULL, store); // empty search path (NIX_PATH)
+    Value * value = nix_alloc_value(NULL, state);
+
+    nix_expr_eval_from_string(NULL, state, "builtins.nixVersion", ".", value);
+    nix_value_force(NULL, state, value);
+
+    char * version;
+    nix_get_string(NULL, value, my_get_string_cb, version);
+    printf("Nix version: %s\n", version);
+
+    free(version);
+    nix_gc_decref(NULL, value);
+    nix_state_free(state);
+    nix_store_free(store);
+    return 0;
+}
+```
+
+**Usage:**
+
+```ShellSession
+$ gcc main.c $(pkg-config nix-expr-c --libs --cflags) -o main
+$ ./main
+Nix version: 2.17
+```
+
+# Writing a Nix language plug-in
+
+In this example we add a custom primitive operation (_primop_) to `builtins`. It
+will increment the argument if it is an integer and throw an error otherwise.
+
+**plugin.c:**
+
+```C
+#include <nix_api_util.h>
+#include <nix_api_expr.h>
+#include <nix_api_value.h>
+
+void increment(void* user_data, nix_c_context* ctx, EvalState* state, Value** args, Value* v) {
+    nix_value_force(NULL, state, args[0]);
+    if (nix_get_type(NULL, args[0]) == NIX_TYPE_INT) {
+      nix_init_int(NULL, v, nix_get_int(NULL, args[0]) + 1);
+    } else {
+      nix_set_err_msg(ctx, NIX_ERR_UNKNOWN, "First argument should be an integer.");
+    }
+}
+
+void nix_plugin_entry() {
+  const char* args[] = {"n", NULL};
+  PrimOp *p = nix_alloc_primop(NULL, increment, 1, "increment", args, "Example custom built-in function: increments an integer", NULL);
+  nix_register_primop(NULL, p);
+  nix_gc_decref(NULL, p);
+}
+```
+
+**Usage:**
+
+```ShellSession
+$ gcc plugin.c $(pkg-config nix-expr-c --libs --cflags) -shared -o plugin.so
+$ nix --plugin-files ./plugin.so repl
+nix-repl> builtins.increment 1
+2
+```

doc/external-api/doxygen.cfg.in

@@ -0,0 +1,57 @@
+# Doxyfile 1.9.5
+
+# The PROJECT_NAME tag is a single word (or a sequence of words surrounded by
+# double-quotes, unless you are using Doxywizard) that should identify the
+# project for which the documentation is generated. This name is used in the
+# title of most generated pages and in a few other places.
+# The default value is: My Project.
+
+PROJECT_NAME           = "Nix"
+
+# The PROJECT_NUMBER tag can be used to enter a project or revision number. This
+# could be handy for archiving the generated documentation or if some version
+# control system is used.
+
+PROJECT_NUMBER         = @PACKAGE_VERSION@
+
+# Using the PROJECT_BRIEF tag one can provide an optional one line description
+# for a project that appears at the top of each page and should give viewer a
+# quick idea about the purpose of the project. Keep the description short.
+
+PROJECT_BRIEF          = "Nix, the purely functional package manager: C API (experimental)"
+
+# If the GENERATE_LATEX tag is set to YES, doxygen will generate LaTeX output.
+# The default value is: YES.
+
+GENERATE_LATEX         = NO
+
+# The INPUT tag is used to specify the files and/or directories that contain
+# documented source files. You may enter file names like myfile.cpp or
+# directories like /usr/src/myproject. Separate the files or directories with
+# spaces. See also FILE_PATTERNS and EXTENSION_MAPPING
+# Note: If this tag is empty the current directory is searched.
+
+# FIXME Make this list more maintainable somehow. We could maybe generate this
+# in the Makefile, but we would need to change how `.in` files are preprocessed
+# so they can expand variables despite configure variables.
+
+INPUT                  = \
+  src/libutil-c \
+  src/libexpr-c \
+  src/libstore-c \
+  doc/external-api/README.md
+
+FILE_PATTERNS = nix_api_*.h *.md
+
+# The INCLUDE_PATH tag can be used to specify one or more directories that
+# contain include files that are not input files but should be processed by the
+# preprocessor. Note that the INCLUDE_PATH is not recursive, so the setting of
+# RECURSIVE has no effect here.
+# This tag requires that the tag SEARCH_INCLUDES is set to YES.
+
+INCLUDE_PATH           = @RAPIDCHECK_HEADERS@
+EXCLUDE_PATTERNS = *_internal.h
+GENERATE_TREEVIEW = YES
+OPTIMIZE_OUTPUT_FOR_C  = YES
+
+USE_MDFILE_AS_MAINPAGE = doc/external-api/README.md

doc/external-api/local.mk

@@ -0,0 +1,7 @@
+$(docdir)/external-api/html/index.html $(docdir)/external-api/latex: $(d)/doxygen.cfg
+	mkdir -p $(docdir)/external-api
+	{ cat $< ; echo "OUTPUT_DIRECTORY=$(docdir)/external-api" ; } | doxygen -
+
+# Generate the HTML API docs for Nix's unstable C bindings
+.PHONY: external-api-html
+external-api-html: $(docdir)/external-api/html/index.html

doc/manual/book.toml

@@ -6,8 +6,6 @@ additional-css = ["custom.css"]
 additional-js = ["redirects.js"]
 edit-url-template = "https://github.com/NixOS/nix/tree/master/doc/manual/{path}"
 git-repository-url = "https://github.com/NixOS/nix"
-fold.enable = true
-fold.level = 1
 
 [preprocessor.anchors]
 renderers = ["html"]

doc/manual/custom.css

@@ -1,3 +1,25 @@
+:root {
+    --sidebar-width: 23em;
+}
+
+h1.menu-title::before {
+  content: "";
+  background-image: url("./favicon.svg");
+  padding: 1.25em;
+  background-position: center center;
+  background-size: 2em;
+  background-repeat: no-repeat;
+}
+
+
+h1.menu-title {
+  padding: 0.5em;
+}
+
+.sidebar .sidebar-scrollbox {
+  padding: 1em;
+}
+
 h1:not(:first-of-type) {
     margin-top: 1.3em;
 }

doc/manual/local.mk

@@ -175,6 +175,16 @@ $(d)/src/SUMMARY-rl-next.md: $(d)/src/release-notes/rl-next.md
 # Generate the HTML manual.
 .PHONY: manual-html
 manual-html: $(docdir)/manual/index.html
+
+# Open the built HTML manual in the default browser.
+manual-html-open: $(docdir)/manual/index.html
+	@echo "  OPEN  " $<; \
+	  xdg-open $< \
+		  || open $< \
+			|| { \
+		echo "Could not open the manual in a browser. Please open '$<'" >&2; \
+		false; \
+		}
 install: $(docdir)/manual/index.html
 
 # Generate 'nix' manpages.
@@ -207,7 +217,7 @@ doc/manual/generated/man1/nix3-manpages: $(d)/src/command-ref/new-cli
 # `@docroot@` is to be preserved for documenting the mechanism
 # FIXME: maybe contributing guides should live right next to the code
 # instead of in the manual
-$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/store/types $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md $(d)/src/language/builtin-constants.md $(d)/src/release-notes/rl-next.md
+$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/store/types $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md $(d)/src/language/builtin-constants.md $(d)/src/release-notes/rl-next.md $(d)/src/figures $(d)/src/favicon.png $(d)/src/favicon.svg
 	$(trace-gen) \
 		tmp="$$(mktemp -d)"; \
 		cp -r doc/manual "$$tmp"; \

doc/manual/redirects.js

@@ -14,7 +14,7 @@
 
 const redirects = {
  "index.html": {
-    "part-advanced-topics": "advanced-topics/advanced-topics.html",
+    "part-advanced-topics": "advanced-topics/index.html",
     "chap-tuning-cores-and-jobs": "advanced-topics/cores-vs-jobs.html",
     "chap-diff-hook": "advanced-topics/diff-hook.html",
     "check-dirs-are-unregistered": "advanced-topics/diff-hook.html#check-dirs-are-unregistered",
@@ -22,7 +22,7 @@ const redirects = {
     "chap-post-build-hook": "advanced-topics/post-build-hook.html",
     "chap-post-build-hook-caveats": "advanced-topics/post-build-hook.html#implementation-caveats",
     "chap-writing-nix-expressions": "language/index.html",
-    "part-command-ref": "command-ref/command-ref.html",
+    "part-command-ref": "command-ref/index.html",
     "conf-allow-import-from-derivation": "command-ref/conf-file.html#conf-allow-import-from-derivation",
     "conf-allow-new-privileges": "command-ref/conf-file.html#conf-allow-new-privileges",
     "conf-allowed-uris": "command-ref/conf-file.html#conf-allowed-uris",
@@ -261,7 +261,7 @@ const redirects = {
     "sec-installer-proxy-settings": "installation/env-variables.html#proxy-environment-variables",
     "sec-nix-ssl-cert-file": "installation/env-variables.html#nix_ssl_cert_file",
     "sec-nix-ssl-cert-file-with-nix-daemon-and-macos": "installation/env-variables.html#nix_ssl_cert_file-with-macos-and-the-nix-daemon",
-    "chap-installation": "installation/installation.html",
+    "chap-installation": "installation/index.html",
     "ch-installing-binary": "installation/installing-binary.html",
     "sect-macos-installation": "installation/installing-binary.html#macos-installation",
     "sect-macos-installation-change-store-prefix": "installation/installing-binary.html#macos-installation",
@@ -288,16 +288,16 @@ const redirects = {
     "ssec-copy-closure": "package-management/copy-closure.html",
     "sec-garbage-collection": "package-management/garbage-collection.html",
     "ssec-gc-roots": "package-management/garbage-collector-roots.html",
-    "chap-package-management": "package-management/package-management.html",
+    "chap-package-management": "package-management/index.html",
     "sec-profiles": "package-management/profiles.html",
-    "ssec-s3-substituter": "package-management/s3-substituter.html",
-    "ssec-s3-substituter-anonymous-reads": "package-management/s3-substituter.html#anonymous-reads-to-your-s3-compatible-binary-cache",
-    "ssec-s3-substituter-authenticated-reads": "package-management/s3-substituter.html#authenticated-reads-to-your-s3-binary-cache",
-    "ssec-s3-substituter-authenticated-writes": "package-management/s3-substituter.html#authenticated-writes-to-your-s3-compatible-binary-cache",
+    "ssec-s3-substituter": "store/types/s3-substituter.html",
+    "ssec-s3-substituter-anonymous-reads": "store/types/s3-substituter.html#anonymous-reads-to-your-s3-compatible-binary-cache",
+    "ssec-s3-substituter-authenticated-reads": "store/types/s3-substituter.html#authenticated-reads-to-your-s3-binary-cache",
+    "ssec-s3-substituter-authenticated-writes": "store/types/s3-substituter.html#authenticated-writes-to-your-s3-compatible-binary-cache",
     "sec-sharing-packages": "package-management/sharing-packages.html",
     "ssec-ssh-substituter": "package-management/ssh-substituter.html",
     "chap-quick-start": "quick-start.html",
-    "sec-relnotes": "release-notes/release-notes.html",
+    "sec-relnotes": "release-notes/index.html",
     "ch-relnotes-0.10.1": "release-notes/rl-0.10.1.html",
     "ch-relnotes-0.10": "release-notes/rl-0.10.html",
     "ssec-relnotes-0.11": "release-notes/rl-0.11.html",

doc/manual/rl-next/harden-user-sandboxing.md

@@ -0,0 +1,8 @@
+---
+synopsis: Harden the user sandboxing
+significance: significant
+issues:
+prs: <only provided once merged>
+---
+
+The build directory has been hardened against interference with the outside world by nesting it inside another directory owned by (and only readable by) the daemon user.

doc/manual/src/SUMMARY.md.in

@@ -42,7 +42,6 @@
     - [Serving a Nix store via HTTP](package-management/binary-cache-substituter.md)
     - [Copying Closures via SSH](package-management/copy-closure.md)
     - [Serving a Nix store via SSH](package-management/ssh-substituter.md)
-    - [Serving a Nix store via S3](package-management/s3-substituter.md)
   - [Remote Builds](advanced-topics/distributed-builds.md)
   - [Tuning Cores and Jobs](advanced-topics/cores-vs-jobs.md)
   - [Verifying Build Reproducibility](advanced-topics/diff-hook.md)
@@ -110,6 +109,7 @@
     - [Derivation](protocols/json/derivation.md)
   - [Serving Tarball Flakes](protocols/tarball-fetcher.md)
   - [Store Path Specification](protocols/store-path.md)
+  - [Nix Archive (NAR) Format](protocols/nix-archive.md)
   - [Derivation "ATerm" file format](protocols/derivation-aterm.md)
 - [Glossary](glossary.md)
 - [Contributing](contributing/index.md)
@@ -119,8 +119,9 @@
   - [Experimental Features](contributing/experimental-features.md)
   - [CLI guideline](contributing/cli-guideline.md)
   - [C++ style guide](contributing/cxx.md)
-- [Release Notes](release-notes/index.md)
+- [Releases](release-notes/index.md)
 {{#include ./SUMMARY-rl-next.md}}
+  - [Release 2.22 (2024-04-23)](release-notes/rl-2.22.md)
   - [Release 2.21 (2024-03-11)](release-notes/rl-2.21.md)
   - [Release 2.20 (2024-01-29)](release-notes/rl-2.20.md)
   - [Release 2.19 (2023-11-17)](release-notes/rl-2.19.md)

doc/manual/src/architecture/architecture.md

@@ -69,7 +69,7 @@ It can also execute build plans to produce new data, which are made available to
 A build plan itself is a series of *build tasks*, together with their build inputs.
 
 > **Important**
-> A build task in Nix is called [derivation](../glossary.md#gloss-derivation).
+> A build task in Nix is called [derivation](@docroot@/glossary.md#gloss-derivation).
 
 Each build task has a special build input executed as *build instructions* in order to perform the build.
 The result of a build task can be input to another build task.

doc/manual/src/command-ref/nix-build.md

@@ -41,7 +41,7 @@ expression to a low-level [store derivation]) and [`nix-store
 --realise`](@docroot@/command-ref/nix-store/realise.md) (to build the store
 derivation).
 
-[store derivation]: ../glossary.md#gloss-store-derivation
+[store derivation]: @docroot@/glossary.md#gloss-store-derivation
 
 > **Warning**
 >

doc/manual/src/command-ref/nix-copy-closure.md

@@ -49,7 +49,7 @@ authentication, you can avoid typing the passphrase with `ssh-agent`.
   - `--include-outputs`\
     Also copy the outputs of [store derivation]s included in the closure.
 
-    [store derivation]: ../glossary.md#gloss-store-derivation
+    [store derivation]: @docroot@/glossary.md#gloss-store-derivation
 
   - `--use-substitutes` / `-s`\
     Attempt to download missing paths on the target machine using Nix’s

doc/manual/src/command-ref/nix-instantiate.md

@@ -23,7 +23,7 @@ It evaluates the Nix expressions in each of *files* (which defaults to
 derivation, a list of derivations, or a set of derivations. The paths
 of the resulting store derivations are printed on standard output.
 
-[store derivation]: ../glossary.md#gloss-store-derivation
+[store derivation]: @docroot@/glossary.md#gloss-store-derivation
 
 If *files* is the character `-`, then a Nix expression will be read from
 standard input.

doc/manual/src/command-ref/nix-store/query.md

@@ -40,12 +40,12 @@ symlink.
     derivations *paths*. These are the paths that will be produced when
     the derivation is built.
 
-    [output paths]: ../../glossary.md#gloss-output-path
+    [output paths]: @docroot@/glossary.md#gloss-output-path
 
   - `--requisites`; `-R`\
     Prints out the [closure] of the store path *paths*.
 
-    [closure]: ../../glossary.md#gloss-closure
+    [closure]: @docroot@/glossary.md#gloss-closure
 
     This query has one option:
 
@@ -66,7 +66,7 @@ symlink.
     *paths*, that is, their immediate dependencies. (For *all*
     dependencies, use `--requisites`.)
 
-    [references]: ../../glossary.md#gloss-reference
+    [references]: @docroot@/glossary.md#gloss-reference
 
   - `--referrers`\
     Prints the set of *referrers* of the store paths *paths*, that is,
@@ -90,7 +90,7 @@ symlink.
     example when *paths* were substituted from a binary cache.
     Use `--valid-derivers` instead to obtain valid paths only.
 
-    [deriver]: ../../glossary.md#gloss-deriver
+    [deriver]: @docroot@/glossary.md#gloss-deriver
 
   - `--valid-derivers`\
     Prints a set of derivation files (`.drv`) which are supposed produce

doc/manual/src/contributing/documentation.md

@@ -27,11 +27,9 @@ and open `./result-doc/share/doc/nix/manual/index.html`.
 To build the manual incrementally, [enter the development shell](./hacking.md) and run:
 
 ```console
-make manual-html -j $NIX_BUILD_CORES
+make manual-html-open -j $NIX_BUILD_CORES
 ```
 
-and open `./outputs/out/share/doc/nix/manual/language/index.html`.
-
 In order to reflect changes to the [Makefile for the manual], clear all generated files before re-building:
 
 [Makefile for the manual]: https://github.com/NixOS/nix/blob/master/doc/manual/local.mk
@@ -208,3 +206,22 @@ or inside `nix-shell` or `nix develop`:
 # make internal-api-html
 # xdg-open ./outputs/doc/share/doc/nix/internal-api/html/index.html
 ```
+
+## C API documentation (experimental)
+
+[C API documentation] is available online.
+You can also build and view it yourself:
+
+[C API documentation]: https://hydra.nixos.org/job/nix/master/external-api-docs/latest/download-by-type/doc/external-api-docs
+
+```console
+# nix build .#hydraJobs.external-api-docs
+# xdg-open ./result/share/doc/nix/external-api/html/index.html
+```
+
+or inside `nix-shell` or `nix develop`:
+
+```
+# make external-api-html
+# xdg-open ./outputs/doc/share/doc/nix/external-api/html/index.html
+```

doc/manual/src/contributing/hacking.md

@@ -144,6 +144,7 @@ Nix can be built for various platforms, as specified in [`flake.nix`]:
 - `aarch64-darwin`
 - `armv6l-linux`
 - `armv7l-linux`
+- `riscv64-linux`
 
 In order to build Nix for a different platform than the one you're currently
 on, you need a way for your current Nix installation to build code for that
@@ -166,7 +167,10 @@ or for Nix with the [`flakes`] and [`nix-command`] experimental features enabled
 $ nix build .#packages.aarch64-linux.default
 ```
 
-Cross-compiled builds are available for ARMv6 (`armv6l-linux`) and ARMv7 (`armv7l-linux`).
+Cross-compiled builds are available for:
+- `armv6l-linux`
+- `armv7l-linux`
+- `riscv64-linux`
 Add more [system types](#system-type) to `crossSystems` in `flake.nix` to bootstrap Nix on unsupported platforms.
 
 ### Building for multiple platforms at once
@@ -196,7 +200,7 @@ In order to facilitate this, Nix has some support for being built out of tree
 
 ## System type
 
-Nix uses a string with he following format to identify the *system type* or *platform* it runs on:
+Nix uses a string with the following format to identify the *system type* or *platform* it runs on:
 
 ```
 <cpu>-<os>[-<abi>]
@@ -258,10 +262,10 @@ See [supported compilation environments](#compilation-environments) and instruct
 To use the LSP with your editor, you first need to [set up `clangd`](https://clangd.llvm.org/installation#project-setup) by running:
 
 ```console
-make clean && bear -- make -j$NIX_BUILD_CORES default check install
+make compile_commands.json
 ```
 
-Configure your editor to use the `clangd` from the shell, either by running it inside the development shell, or by using [nix-direnv](https://github.com/nix-community/nix-direnv) and [the appropriate editor plugin](https://github.com/direnv/direnv/wiki#editor-integration).
+Configure your editor to use the `clangd` from the `.#native-clangStdenvPackages` shell. You can do that either by running it inside the development shell, or by using [nix-direnv](https://github.com/nix-community/nix-direnv) and [the appropriate editor plugin](https://github.com/direnv/direnv/wiki#editor-integration).
 
 > **Note**
 >
@@ -269,6 +273,29 @@ Configure your editor to use the `clangd` from the shell, either by running it i
 > Some other editors (e.g. Emacs, Vim) need a plugin to support LSP servers in general (e.g. [lsp-mode](https://github.com/emacs-lsp/lsp-mode) for Emacs and [vim-lsp](https://github.com/prabirshrestha/vim-lsp) for vim).
 > Editor-specific setup is typically opinionated, so we will not cover it here in more detail.
 
+## Formatting and pre-commit hooks
+
+You may run the formatters as a one-off using:
+
+```console
+make format
+```
+
+If you'd like to run the formatters before every commit, install the hooks:
+
+```
+pre-commit-hooks-install
+```
+
+This installs [pre-commit](https://pre-commit.com) using [cachix/git-hooks.nix](https://github.com/cachix/git-hooks.nix).
+
+When making a commit, pay attention to the console output.
+If it fails, run `git add --patch` to approve the suggestions _and commit again_.
+
+To refresh pre-commit hook's config file, do the following:
+1. Exit the development shell and start it again by running `nix develop`.
+2. If you also use the pre-commit hook, also run `pre-commit-hooks-install` again.
+
 ## Add a release note
 
 `doc/manual/rl-next` contains release notes entries for all unreleased changes.

doc/manual/src/favicon.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="587.11" height="516.604" viewBox="0 0 550.416 484.317"><defs><linearGradient id="a"><stop offset="0" style="stop-color:#699ad7;stop-opacity:1"/><stop offset=".243" style="stop-color:#7eb1dd;stop-opacity:1"/><stop offset="1" style="stop-color:#7ebae4;stop-opacity:1"/></linearGradient><linearGradient id="b"><stop offset="0" style="stop-color:#415e9a;stop-opacity:1"/><stop offset=".232" style="stop-color:#4a6baf;stop-opacity:1"/><stop offset="1" style="stop-color:#5277c3;stop-opacity:1"/></linearGradient><linearGradient xlink:href="#a" id="c" x1="200.597" x2="290.087" y1="351.411" y2="506.188" gradientTransform="translate(70.65 -1055.151)" gradientUnits="userSpaceOnUse"/><linearGradient xlink:href="#b" id="e" x1="-584.199" x2="-496.297" y1="782.336" y2="937.714" gradientTransform="translate(864.696 -1491.34)" gradientUnits="userSpaceOnUse"/></defs><g style="display:inline;opacity:1" transform="translate(-132.651 958.04)"><path id="d" d="m309.549-710.388 122.197 211.675-56.157.527-32.624-56.87-32.856 56.566-27.903-.011-14.29-24.69 46.81-80.49-33.23-57.826z" style="opacity:1;fill:url(#c);fill-opacity:1;fill-rule:evenodd;stroke:none;stroke-width:3;stroke-linecap:butt;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"/><use xlink:href="#d" width="100%" height="100%" transform="rotate(60 407.112 -715.787)"/><use xlink:href="#d" width="100%" height="100%" transform="rotate(-60 407.312 -715.7)"/><use xlink:href="#d" width="100%" height="100%" transform="rotate(180 407.419 -715.756)"/><path id="f" d="m309.549-710.388 122.197 211.675-56.157.527-32.624-56.87-32.856 56.566-27.903-.011-14.29-24.69 46.81-80.49-33.23-57.826z" style="color:#000;clip-rule:nonzero;display:inline;overflow:visible;visibility:visible;opacity:1;isolation:auto;mix-blend-mode:normal;color-interpolation:sRGB;color-interpolation-filters:linearRGB;solid-color:#000;solid-opacity:1;fill:url(#e);fill-opacity:1;fill-rule:evenodd;stroke:none;stroke-width:3;stroke-linecap:butt;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1;color-rendering:auto;image-rendering:auto;shape-rendering:auto;text-rendering:auto;enable-background:accumulate"/><use xlink:href="#f" width="100%" height="100%" style="display:inline" transform="rotate(120 407.34 -716.084)"/><use xlink:href="#f" width="100%" height="100%" style="display:inline" transform="rotate(-120 407.288 -715.87)"/></g></svg>

doc/manual/src/glossary.md

@@ -215,6 +215,9 @@
 
   [output path]: #gloss-output-path
 
+- [output closure]{#gloss-output-closure}\
+  The [closure] of an [output path]. It only contains what is [reachable] from the output.
+
 - [deriver]{#gloss-deriver}
 
   The [store derivation] that produced an [output path].
@@ -292,6 +295,25 @@
   [path]: ./language/values.md#type-path
   [attribute name]: ./language/values.md#attribute-set
 
+- [base directory]{#gloss-base-directory}
+
+  The location from which relative paths are resolved.
+
+  - For expressions in a file, the base directory is the directory containing that file.
+    This is analogous to the directory of a [base URL](https://datatracker.ietf.org/doc/html/rfc1808#section-3.3).
+    <!-- which is sufficient for resolving non-empty URLs -->
+
+  <!--
+    The wording here may look awkward, but it's for these reasons:
+      * "with --expr": it's a flag, and not an option with an accompanying value
+      * "written in": the expression itself must be written as an argument,
+        whereas the more natural "passed as an argument" allows an interpretation
+        where the expression could be passed by file name.
+    -->
+  - For expressions written in command line arguments with [`--expr`](@docroot@/command-ref/opt-common.html#opt-expr), the base directory is the current working directory.
+
+  [base directory]: #gloss-base-directory
+
 - [experimental feature]{#gloss-experimental-feature}
 
   Not yet stabilized functionality guarded by named experimental feature flags.

doc/manual/src/installation/installing-binary.md

@@ -50,7 +50,7 @@ Supported systems:
 To explicitly instruct the installer to perform a multi-user installation on your system:
 
 ```console
-$ curl -L https://nixos.org/nix/install | sh -s -- --daemon
+$ bash <(curl -L https://nixos.org/nix/install) --daemon
 ```
 
 You can run this under your usual user account or `root`.
@@ -61,7 +61,7 @@ The script will invoke `sudo` as needed.
 To explicitly select a single-user installation on your system:
 
 ```console
-$ curl -L https://nixos.org/nix/install | sh -s -- --no-daemon
+$ bash <(curl -L https://nixos.org/nix/install) --no-daemon
 ```
 
 In a single-user installation, `/nix` is owned by the invoking user.

doc/manual/src/installation/uninstall.md

@@ -5,8 +5,9 @@
 If you have a [single-user installation](./installing-binary.md#single-user-installation) of Nix, uninstall it by running:
 
 ```console
-$ rm -rf /nix
+$ rm -rf /nix ~/.nix-channels ~/.nix-defexpr ~/.nix-profile
 ```
+You might also want to manually remove references to Nix from your `~/.profile`.
 
 ## Multi User
 

doc/manual/src/installation/upgrading.md

@@ -28,7 +28,7 @@ $ sudo su
 ## macOS multi-user
 
 ```console
-$ sudo nix-env --install --file '<nixpkgs>' --attr nix -I nixpkgs=channel:nixpkgs-unstable
+$ sudo nix-env --install --file '<nixpkgs>' --attr nix cacert -I nixpkgs=channel:nixpkgs-unstable
 $ sudo launchctl remove org.nixos.nix-daemon
 $ sudo launchctl load /Library/LaunchDaemons/org.nixos.nix-daemon.plist
 ```

doc/manual/src/language/advanced-attributes.md

@@ -188,9 +188,13 @@ Derivations can declare some infrequently used optional attributes.
     }
     ```
 
-    The `outputHashAlgo` attribute specifies the hash algorithm used to
-    compute the hash. It can currently be `"sha1"`, `"sha256"` or
-    `"sha512"`.
+    The `outputHash` attribute must be a string containing the hash in either hexadecimal or "nix32" encoding, or following the format for integrity metadata as defined by [SRI](https://www.w3.org/TR/SRI/).
+    The "nix32" encoding is an adaptation of base-32 encoding.
+    The [`convertHash`](@docroot@/language/builtins.md#builtins-convertHash) function shows how to convert between different encodings, and the [`nix-hash` command](../command-ref/nix-hash.md) has information about obtaining the hash for some contents, as well as converting to and from encodings.
+
+    The `outputHashAlgo` attribute specifies the hash algorithm used to compute the hash.
+    It can currently be `"sha1"`, `"sha256"`, `"sha512"`, or `null`.
+    `outputHashAlgo` can only be `null` when `outputHash` follows the SRI format.
 
     The `outputHashMode` attribute determines how the hash is computed.
     It must be one of the following two values:
@@ -203,16 +207,16 @@ Derivations can declare some infrequently used optional attributes.
 
         This is the default.
 
-      - `"recursive"`\
-        The hash is computed over the NAR archive dump of the output
+      - `"recursive"` or `"nar"`\
+        The hash is computed over the [NAR archive](@docroot@/glossary.md#gloss-nar) dump of the output
         (i.e., the result of [`nix-store --dump`](@docroot@/command-ref/nix-store/dump.md)). In
         this case, the output can be anything, including a directory
         tree.
 
-    The `outputHash` attribute, finally, must be a string containing
-    the hash in either hexadecimal or base-32 notation. (See the
-    [`nix-hash` command](../command-ref/nix-hash.md) for information
-    about converting to and from base-32 notation.)
+    `"recursive"` is the traditional way of indicating this,
+    and is supported since 2005 (virtually the entire history of Nix).
+    `"nar"` is more clear, and consistent with other parts of Nix (such as the CLI),
+    however support for it is only added in Nix version 2.21.
 
   - [`__contentAddressed`]{#adv-attr-__contentAddressed}
     > **Warning**
@@ -299,7 +303,7 @@ Derivations can declare some infrequently used optional attributes.
     [`disallowedReferences`](#adv-attr-disallowedReferences) and [`disallowedRequisites`](#adv-attr-disallowedRequisites),
     the following attributes are available:
 
-    - `maxSize` defines the maximum size of the resulting [store object](../glossary.md#gloss-store-object).
+    - `maxSize` defines the maximum size of the resulting [store object](@docroot@/glossary.md#gloss-store-object).
     - `maxClosureSize` defines the maximum size of the output's closure.
     - `ignoreSelfRefs` controls whether self-references should be considered when
       checking for allowed references/requisites.

doc/manual/src/language/index.md

@@ -1,7 +1,13 @@
 # Nix Language
 
 The Nix language is designed for conveniently creating and composing *derivations* – precise descriptions of how contents of existing files are used to derive new files.
-It is:
+
+> **Tip**
+>
+> These pages are written as a reference.
+> If you are learning Nix, nix.dev has a good [introduction to the Nix language](https://nix.dev/tutorials/nix-language).
+
+The language is:
 
 - *domain-specific*
 

doc/manual/src/language/operators.md

@@ -128,8 +128,8 @@ The result is a string.
 > The file or directory at *path* must exist and is copied to the [store].
 > The path appears in the result as the corresponding [store path].
 
-[store path]: ../glossary.md#gloss-store-path
-[store]: ../glossary.md#gloss-store
+[store path]: @docroot@/glossary.md#gloss-store-path
+[store]: @docroot@/glossary.md#gloss-store
 
 [String and path concatenation]: #string-and-path-concatenation
 

doc/manual/src/language/string-interpolation.md

@@ -20,7 +20,7 @@ Rather than writing
 
 (where `freetype` is a [derivation]), you can instead write
 
-[derivation]: ../glossary.md#gloss-derivation
+[derivation]: @docroot@/glossary.md#gloss-derivation
 
 ```nix
 "--with-freetype2-library=${freetype}/lib"
@@ -107,9 +107,9 @@ An expression that is interpolated must evaluate to one of the following:
 
 A string interpolates to itself.
 
-A path in an interpolated expression is first copied into the Nix store, and the resulting string is the [store path] of the newly created [store object](../glossary.md#gloss-store-object).
+A path in an interpolated expression is first copied into the Nix store, and the resulting string is the [store path] of the newly created [store object](@docroot@/glossary.md#gloss-store-object).
 
-[store path]: ../glossary.md#gloss-store-path
+[store path]: @docroot@/glossary.md#gloss-store-path
 
 > **Example**
 >

doc/manual/src/language/values.md

@@ -97,8 +97,8 @@
   is not a path: it's parsed as an expression that selects the
   attribute `sh` from the variable `builder`. If the file name is
   relative, i.e., if it does not begin with a slash, it is made
-  absolute at parse time relative to the directory of the Nix
-  expression that contained it. For instance, if a Nix expression in
+  absolute at parse time relative to the [base directory](@docroot@/glossary.md#gloss-base-directory).
+  For instance, if a Nix expression in
   `/foo/bar/bla.nix` refers to `../xyzzy/fnord.nix`, the absolute path
   is `/foo/xyzzy/fnord.nix`.
 
@@ -107,13 +107,13 @@
   e.g. `~/foo` would be equivalent to `/home/edolstra/foo` for a user
   whose home directory is `/home/edolstra`.
 
-  For instance, evaluating `"${./foo.txt}"` will cause `foo.txt` in the current directory to be copied into the Nix store and result in the string `"/nix/store/<hash>-foo.txt"`.
+  For instance, evaluating `"${./foo.txt}"` will cause `foo.txt` in the base directory to be copied into the Nix store and result in the string `"/nix/store/<hash>-foo.txt"`.
 
   Note that the Nix language assumes that all input files will remain _unchanged_ while  evaluating a Nix expression.
   For example, assume you used a file path in an interpolated string during a `nix repl` session.
   Later in the same session, after having changed the file contents, evaluating the interpolated string with the file path again might not return a new [store path], since Nix might not re-read the file contents.
 
-  [store path]: ../glossary.md#gloss-store-path
+  [store path]: @docroot@/glossary.md#gloss-store-path
 
   Paths can include [string interpolation] and can themselves be [interpolated in other expressions].
 

doc/manual/src/protocols/json/store-object-info.md

@@ -83,7 +83,7 @@ This information is not intrinsic to the store object, but about how it is store
 
 ## Computed closure fields
 
-These fields are not stored at all, but computed by traverising the other other fields across all the store objects in a [closure].
+These fields are not stored at all, but computed by traversing the other fields across all the store objects in a [closure].
 
 * `closureSize`:
 

doc/manual/src/protocols/nix-archive.md

@@ -0,0 +1,42 @@
+# Nix Archive (NAR) format
+
+This is the complete specification of the Nix Archive format.
+The Nix Archive format closely follows the abstract specification of a [file system object] tree,
+because it is designed to serialize exactly that data structure.
+
+[file system object]: @docroot@/store/file-system-object.md
+
+The format of this specification is close to [Extended Backus–Naur form](https://en.wikipedia.org/wiki/Extended_Backus%E2%80%93Naur_form), with the exception of the `str(..)` function / parameterized rule, which length-prefixes and pads strings.
+This makes the resulting binary format easier to parse.
+
+Regular users do *not* need to know this information.
+But for those interested in exactly how Nix works, e.g. if they are reimplementing it, this information can be useful.
+
+```ebnf
+nar = str("nix-archive-1"), nar-obj;
+
+nar-obj = str("("), nar-obj-inner, str(")");
+
+nar-obj-inner
+  = str("type"), str("regular") regular
+  | str("type"), str("symlink") symlink
+  | str("type"), str("directory") directory
+  ;
+
+regular = [ str("executable"), str("") ], str("contents"), str(contents);
+
+symlink = str("target"), str(target);
+
+(* side condition: directory entries must be ordered by their names *)
+directory = str("type"), str("directory") { directory-entry };
+
+directory-entry = str("entry"), str("("), str("name"), str(name), str("node"), nar-obj, str(")");
+```
+
+The `str` function / parameterized rule is defined as follows:
+
+- `str(s)` = `int(|s|), pad(s);`
+
+- `int(n)` = the 64-bit little endian representation of the number `n`
+
+- `pad(s)` = the byte sequence `s`, padded with 0s to a multiple of 8 byte

doc/manual/src/release-notes/index.md

@@ -1,12 +1,13 @@
 # Nix Release Notes
 
+The Nix release cycle is calendar-based as follows:
+
 Nix has a release cycle of roughly 6 weeks.
 Notable changes and additions are announced in the release notes for each version.
 
-Bugfixes can be backported on request to previous Nix releases.
-We typically backport only as far back as the Nix version used in the latest NixOS release, which is announced in the [NixOS release notes](https://nixos.org/manual/nixos/stable/release-notes.html#ch-release-notes).
-
-Backports never skip releases.
-If a feature is backported to version `x.y`, it must also be available in version `x.(y+1)`.
-This ensures that upgrading from an older version with backports is still safe and no backported functionality will go missing.
+The supported Nix versions are:
+- The latest release
+- The version used in the stable NixOS release, which is announced in the [NixOS release notes](https://nixos.org/manual/nixos/stable/release-notes.html#ch-release-notes).
 
+Bugfixes and security issues are backported to every supported version.
+Patch releases are published as needed.

doc/manual/src/release-notes/rl-2.15.md

@@ -11,7 +11,7 @@
   As the choice of hash formats is no longer binary, the `--base16` flag is also added
   to explicitly specify the Base16 format, which is still the default.
 
-* The special handling of an [installable](../command-ref/new-cli/nix.md#installables) with `.drv` suffix being interpreted as all of the given [store derivation](../glossary.md#gloss-store-derivation)'s output paths is removed, and instead taken as the literal store path that it represents.
+* The special handling of an [installable](../command-ref/new-cli/nix.md#installables) with `.drv` suffix being interpreted as all of the given [store derivation](@docroot@/glossary.md#gloss-store-derivation)'s output paths is removed, and instead taken as the literal store path that it represents.
 
   The new `^` syntax for store paths introduced in Nix 2.13 allows explicitly referencing output paths of a derivation.
   Using this is better and more clear than relying on the now-removed `.drv` special handling.

doc/manual/src/release-notes/rl-2.20.md

@@ -200,3 +200,9 @@
   while performing various operations (including `nix develop`, `nix flake
   update`, and so on). With several fixes to Nix's signal handlers, Nix
   commands will now exit quickly after Ctrl-C is pressed.
+
+- `nix copy` to a `ssh-ng` store now needs `--substitute-on-destination` (a.k.a. `-s`)
+  in order to substitute paths on the remote store instead of copying them.
+  The behavior is consistent with `nix copy` to a different kind of remote store.
+  Previously this behavior was controlled by the
+  `builders-use-substitutes` setting and `--substitute-on-destination` was ignored.

doc/manual/src/release-notes/rl-2.22.md

@@ -0,0 +1,21 @@
+# Release 2.22.0 (2024-04-23)
+
+### Significant changes
+
+- Remove experimental repl-flake [#10103](https://github.com/NixOS/nix/issues/10103) [#10299](https://github.com/NixOS/nix/pull/10299)
+
+  The `repl-flake` experimental feature has been removed. The `nix repl` command now works like the rest of the new CLI in that `nix repl {path}` now tries to load a flake at `{path}` (or fails if the `flakes` experimental feature isn't enabled).
+
+### Other changes
+
+- `nix eval` prints derivations as `.drv` paths [#10200](https://github.com/NixOS/nix/pull/10200)
+
+  `nix eval` will now print derivations as their `.drv` paths, rather than as
+  attribute sets. This makes commands like `nix eval nixpkgs#bash` terminate
+  instead of infinitely looping into recursive self-referential attributes:
+
+  ```ShellSession
+  $ nix eval nixpkgs#bash
+  «derivation /nix/store/m32cbgbd598f4w299g0hwyv7gbw6rqcg-bash-5.2p26.drv»
+  ```
+

doc/manual/src/store/store-path.md

@@ -46,7 +46,7 @@ But if the store has a file system representation, the store directory contains
 
 [file system objects]: ./file-system-object.md
 
-This means a store path is not just derived from the referenced store object itself, but depends on the store the store object is in.
+This means a store path is not just derived from the referenced store object itself, but depends on the store that the store object is in.
 
 > **Note**
 >

flake.lock

@@ -16,6 +16,41 @@
         "type": "github"
       }
     },
+    "flake-parts": {
+      "inputs": {
+        "nixpkgs-lib": [
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1712014858,
+        "narHash": "sha256-sB4SWl2lX95bExY2gMFG5HIzvva5AVMJd4Igm+GpZNw=",
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "rev": "9126214d0a59633752a136528f5f3b9aa8565b7d",
+        "type": "github"
+      },
+      "original": {
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "type": "github"
+      }
+    },
+    "flake-utils": {
+      "locked": {
+        "lastModified": 1667395993,
+        "narHash": "sha256-nuEHfE/LcWyuSWnS8t12N1wc105Qtau+/OdUAjtQ0rA=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "5aed5285a952e0b949eb3ba02c12fa4fcfef535f",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
     "libgit2": {
       "flake": false,
       "locked": {
@@ -64,12 +99,40 @@
         "type": "github"
       }
     },
+    "pre-commit-hooks": {
+      "inputs": {
+        "flake-compat": [],
+        "flake-utils": "flake-utils",
+        "gitignore": [],
+        "nixpkgs": [
+          "nixpkgs"
+        ],
+        "nixpkgs-stable": [
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1712897695,
+        "narHash": "sha256-nMirxrGteNAl9sWiOhoN5tIHyjBbVi5e2tgZUgZlK3Y=",
+        "owner": "cachix",
+        "repo": "pre-commit-hooks.nix",
+        "rev": "40e6053ecb65fcbf12863338a6dcefb3f55f1bf8",
+        "type": "github"
+      },
+      "original": {
+        "owner": "cachix",
+        "repo": "pre-commit-hooks.nix",
+        "type": "github"
+      }
+    },
     "root": {
       "inputs": {
         "flake-compat": "flake-compat",
+        "flake-parts": "flake-parts",
         "libgit2": "libgit2",
         "nixpkgs": "nixpkgs",
-        "nixpkgs-regression": "nixpkgs-regression"
+        "nixpkgs-regression": "nixpkgs-regression",
+        "pre-commit-hooks": "pre-commit-hooks"
       }
     }
   },

flake.nix

@@ -8,13 +8,25 @@
   inputs.flake-compat = { url = "github:edolstra/flake-compat"; flake = false; };
   inputs.libgit2 = { url = "github:libgit2/libgit2"; flake = false; };
 
-  outputs = { self, nixpkgs, nixpkgs-regression, libgit2, ... }:
+  # dev tooling
+  inputs.flake-parts.url = "github:hercules-ci/flake-parts";
+  inputs.pre-commit-hooks.url = "github:cachix/pre-commit-hooks.nix";
+  # work around https://github.com/NixOS/nix/issues/7730
+  inputs.flake-parts.inputs.nixpkgs-lib.follows = "nixpkgs";
+  inputs.pre-commit-hooks.inputs.nixpkgs.follows = "nixpkgs";
+  inputs.pre-commit-hooks.inputs.nixpkgs-stable.follows = "nixpkgs";
+  # work around 7730 and https://github.com/NixOS/nix/issues/7807
+  inputs.pre-commit-hooks.inputs.flake-compat.follows = "";
+  inputs.pre-commit-hooks.inputs.gitignore.follows = "";
+
+  outputs = inputs@{ self, nixpkgs, nixpkgs-regression, libgit2, ... }:
+
 
     let
       inherit (nixpkgs) lib;
       inherit (lib) fileset;
 
-      officialRelease = false;
+      officialRelease = true;
 
       version = lib.fileContents ./.version + versionSuffix;
       versionSuffix =
@@ -31,13 +43,8 @@
       crossSystems = [
         "armv6l-unknown-linux-gnueabihf"
         "armv7l-unknown-linux-gnueabihf"
+        "riscv64-unknown-linux-gnu"
         "x86_64-unknown-netbsd"
-      ];
-
-      # Nix doesn't yet build on this platform, so we put it in a
-      # separate list. We just use this for `devShells` and
-      # `nixpkgsFor`, which this depends on.
-      shellCrossSystems = crossSystems ++ [
         "x86_64-w64-mingw32"
       ];
 
@@ -62,6 +69,17 @@
             })
             stdenvs);
 
+
+      # We don't apply flake-parts to the whole flake so that non-development attributes
+      # load without fetching any development inputs.
+      devFlake = inputs.flake-parts.lib.mkFlake { inherit inputs; } {
+        imports = [ ./maintainers/flake-module.nix ];
+        systems = lib.subtractLists crossSystems systems;
+        perSystem = { system, ... }: {
+          _module.args.pkgs = nixpkgsFor.${system}.native;
+        };
+      };
+
       # Memoize nixpkgs for different platforms for efficiency.
       nixpkgsFor = forAllSystems
         (system: let
@@ -83,7 +101,7 @@
         in {
           inherit stdenvs native;
           static = native.pkgsStatic;
-          cross = lib.genAttrs shellCrossSystems (crossSystem: make-pkgs crossSystem "stdenv");
+          cross = forAllCrossSystems (crossSystem: make-pkgs crossSystem "stdenv");
         });
 
       installScriptFor = tarballs:
@@ -165,7 +183,7 @@
 
           nix =
             let
-              officialRelease = false;
+              officialRelease = true;
               versionSuffix =
                 if officialRelease
                 then ""
@@ -177,7 +195,7 @@
                 stdenv
                 versionSuffix
                 ;
-              officialRelease = false;
+              officialRelease = true;
               boehmgc = final.boehmgc-nix;
               libgit2 = final.libgit2-nix;
               busybox-sandbox-shell = final.busybox-sandbox-shell or final.default-busybox-sandbox-shell;
@@ -191,6 +209,13 @@
             inherit fileset stdenv;
           };
 
+          # See https://github.com/NixOS/nixpkgs/pull/214409
+          # Remove when fixed in this flake's nixpkgs
+          pre-commit =
+            if prev.stdenv.hostPlatform.system == "i686-linux"
+            then (prev.pre-commit.override (o: { dotnet-sdk = ""; })).overridePythonAttrs (o: { doCheck = false; })
+            else prev.pre-commit;
+
         };
 
     in {
@@ -259,6 +284,7 @@
           # Cross
           self.hydraJobs.binaryTarballCross."x86_64-linux"."armv6l-unknown-linux-gnueabihf"
           self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-unknown-linux-gnueabihf"
+          self.hydraJobs.binaryTarballCross."x86_64-linux"."riscv64-unknown-linux-gnu"
         ];
         installerScriptForGHA = installScriptFor [
           # Native
@@ -267,6 +293,7 @@
           # Cross
           self.hydraJobs.binaryTarballCross."x86_64-linux"."armv6l-unknown-linux-gnueabihf"
           self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-unknown-linux-gnueabihf"
+          self.hydraJobs.binaryTarballCross."x86_64-linux"."riscv64-unknown-linux-gnu"
         ];
 
         # docker image with Nix inside
@@ -285,6 +312,13 @@
           enableInternalAPIDocs = true;
         };
 
+        # API docs for Nix's C bindings.
+        external-api-docs = nixpkgsFor.x86_64-linux.native.callPackage ./package.nix {
+          inherit fileset;
+          doBuild = false;
+          enableExternalAPIDocs = true;
+        };
+
         # System tests.
         tests = import ./tests/nixos { inherit lib nixpkgs nixpkgsFor; } // {
 
@@ -357,7 +391,8 @@
         # Since the support is only best-effort there, disable the perl
         # bindings
         perlBindings = self.hydraJobs.perlBindings.${system};
-      });
+      } // devFlake.checks.${system} or {}
+      );
 
       packages = forAllSystems (system: rec {
         inherit (nixpkgsFor.${system}.native) nix changelog-d-nix;
@@ -392,7 +427,11 @@
           stdenvs)));
 
       devShells = let
-        makeShell = pkgs: stdenv: (pkgs.nix.override { inherit stdenv; forDevShell = true; }).overrideAttrs (attrs: {
+        makeShell = pkgs: stdenv: (pkgs.nix.override { inherit stdenv; forDevShell = true; }).overrideAttrs (attrs:
+        let
+          modular = devFlake.getSystem stdenv.buildPlatform.system;
+        in {
+          pname = "shell-for-" + attrs.pname;
           installFlags = "sysconfdir=$(out)/etc";
           shellHook = ''
             PATH=$prefix/bin:$PATH
@@ -403,7 +442,21 @@
             XDG_DATA_DIRS+=:$out/share
           '';
 
+          # We use this shell with the local checkout, not unpackPhase.
+          src = null;
+
+          env = {
+            # For `make format`, to work without installing pre-commit
+            _NIX_PRE_COMMIT_HOOKS_CONFIG =
+              "${(pkgs.formats.yaml { }).generate "pre-commit-config.yaml" modular.pre-commit.settings.rawConfig}";
+          };
+
           nativeBuildInputs = attrs.nativeBuildInputs or []
+            ++ [
+              modular.pre-commit.settings.package
+              (pkgs.writeScriptBin "pre-commit-hooks-install"
+                modular.pre-commit.settings.installationScript)
+            ]
             # TODO: Remove the darwin check once
             # https://github.com/NixOS/nixpkgs/pull/291814 is available
             ++ lib.optional (stdenv.cc.isClang && !stdenv.buildPlatform.isDarwin) pkgs.buildPackages.bear
@@ -419,8 +472,8 @@
           in
             (makeShells "native" nixpkgsFor.${system}.native) //
             (lib.optionalAttrs (!nixpkgsFor.${system}.native.stdenv.isDarwin)
-              (makeShells "static" nixpkgsFor.${system}.static)) //
-              (lib.genAttrs shellCrossSystems (crossSystem: let pkgs = nixpkgsFor.${system}.cross.${crossSystem}; in makeShell pkgs pkgs.stdenv)) //
+              (makeShells "static" nixpkgsFor.${system}.static) //
+              (forAllCrossSystems (crossSystem: let pkgs = nixpkgsFor.${system}.cross.${crossSystem}; in makeShell pkgs pkgs.stdenv))) //
             {
               default = self.devShells.${system}.native-stdenvPackages;
             }

local.mk

@@ -2,9 +2,14 @@ GLOBAL_CXXFLAGS += -Wno-deprecated-declarations -Werror=switch
 # Allow switch-enum to be overridden for files that do not support it, usually because of dependency headers.
 ERROR_SWITCH_ENUM = -Werror=switch-enum
 
-$(foreach i, config.h $(wildcard src/lib*/*.hh), \
+$(foreach i, config.h $(wildcard src/lib*/*.hh) $(filter-out %_internal.h, $(wildcard src/lib*c/*.h)), \
   $(eval $(call install-file-in, $(i), $(includedir)/nix, 0644)))
 
+ifdef HOST_UNIX
+  $(foreach i, $(wildcard src/lib*/unix/*.hh), \
+    $(eval $(call install-file-in, $(i), $(includedir)/nix, 0644)))
+endif
+
 $(GCH): src/libutil/util.hh config.h
 
-GCH_CXXFLAGS = -I src/libutil
+GCH_CXXFLAGS = $(INCLUDE_libutil)

m4/gcc_bug_80431.m4

@@ -46,11 +46,13 @@ AC_DEFUN([ENSURE_NO_GCC_BUG_80431],
       ]])],
     [status_80431=0],
     [status_80431=$?],
-    [
-      # Assume we're bug-free when cross-compiling
-    ])
+    [status_80431=''])
   AC_LANG_POP(C++)
   AS_CASE([$status_80431],
+    [''],[
+      AC_MSG_RESULT(cannot check because cross compiling)
+      AC_MSG_NOTICE(assume we are bug free)
+    ],
     [0],[
       AC_MSG_RESULT(yes)
     ],

maintainers/flake-module.nix

@@ -0,0 +1,436 @@
+{ lib, getSystem, inputs, ... }:
+
+{
+  imports = [
+    inputs.pre-commit-hooks.flakeModule
+  ];
+
+  perSystem = { config, pkgs, ... }: {
+
+    # https://flake.parts/options/pre-commit-hooks-nix.html#options
+    pre-commit.settings = {
+      hooks = {
+        clang-format.enable = true;
+        # TODO: nixfmt, https://github.com/NixOS/nixfmt/issues/153
+      };
+
+      excludes = [
+        # We don't want to format test data
+        # ''tests/(?!nixos/).*\.nix''
+        ''^tests/.*''
+
+        # Don't format vendored code
+        ''^src/toml11/.*''
+        ''^doc/manual/redirects\.js$''
+        ''^doc/manual/theme/highlight\.js$''
+
+        # We haven't applied formatting to these files yet
+        ''^doc/manual/redirects\.js$''
+        ''^doc/manual/theme/highlight\.js$''
+        ''^precompiled-headers\.h$''
+        ''^src/build-remote/build-remote\.cc$''
+        ''^src/libcmd/built-path\.cc$''
+        ''^src/libcmd/built-path\.hh$''
+        ''^src/libcmd/command\.cc$''
+        ''^src/libcmd/command\.hh$''
+        ''^src/libcmd/common-eval-args\.cc$''
+        ''^src/libcmd/common-eval-args\.hh$''
+        ''^src/libcmd/editor-for\.cc$''
+        ''^src/libcmd/installable-attr-path\.cc$''
+        ''^src/libcmd/installable-attr-path\.hh$''
+        ''^src/libcmd/installable-derived-path\.cc$''
+        ''^src/libcmd/installable-derived-path\.hh$''
+        ''^src/libcmd/installable-flake\.cc$''
+        ''^src/libcmd/installable-flake\.hh$''
+        ''^src/libcmd/installable-value\.cc$''
+        ''^src/libcmd/installable-value\.hh$''
+        ''^src/libcmd/installables\.cc$''
+        ''^src/libcmd/installables\.hh$''
+        ''^src/libcmd/legacy\.hh$''
+        ''^src/libcmd/markdown\.cc$''
+        ''^src/libcmd/misc-store-flags\.cc$''
+        ''^src/libcmd/repl-interacter\.cc$''
+        ''^src/libcmd/repl-interacter\.hh$''
+        ''^src/libcmd/repl\.cc$''
+        ''^src/libcmd/repl\.hh$''
+        ''^src/libexpr-c/nix_api_expr\.cc$''
+        ''^src/libexpr-c/nix_api_external\.cc$''
+        ''^src/libexpr/attr-path\.cc$''
+        ''^src/libexpr/attr-path\.hh$''
+        ''^src/libexpr/attr-set\.cc$''
+        ''^src/libexpr/attr-set\.hh$''
+        ''^src/libexpr/eval-cache\.cc$''
+        ''^src/libexpr/eval-cache\.hh$''
+        ''^src/libexpr/eval-error\.cc$''
+        ''^src/libexpr/eval-inline\.hh$''
+        ''^src/libexpr/eval-settings\.cc$''
+        ''^src/libexpr/eval-settings\.hh$''
+        ''^src/libexpr/eval\.cc$''
+        ''^src/libexpr/eval\.hh$''
+        ''^src/libexpr/flake/config\.cc$''
+        ''^src/libexpr/flake/flake\.cc$''
+        ''^src/libexpr/flake/flake\.hh$''
+        ''^src/libexpr/flake/flakeref\.cc$''
+        ''^src/libexpr/flake/flakeref\.hh$''
+        ''^src/libexpr/flake/lockfile\.cc$''
+        ''^src/libexpr/flake/lockfile\.hh$''
+        ''^src/libexpr/flake/url-name\.cc$''
+        ''^src/libexpr/function-trace\.cc$''
+        ''^src/libexpr/gc-small-vector\.hh$''
+        ''^src/libexpr/get-drvs\.cc$''
+        ''^src/libexpr/get-drvs\.hh$''
+        ''^src/libexpr/json-to-value\.cc$''
+        ''^src/libexpr/nixexpr\.cc$''
+        ''^src/libexpr/nixexpr\.hh$''
+        ''^src/libexpr/parser-state\.hh$''
+        ''^src/libexpr/pos-table\.hh$''
+        ''^src/libexpr/primops\.cc$''
+        ''^src/libexpr/primops\.hh$''
+        ''^src/libexpr/primops/context\.cc$''
+        ''^src/libexpr/primops/fetchClosure\.cc$''
+        ''^src/libexpr/primops/fetchMercurial\.cc$''
+        ''^src/libexpr/primops/fetchTree\.cc$''
+        ''^src/libexpr/primops/fromTOML\.cc$''
+        ''^src/libexpr/print-ambiguous\.cc$''
+        ''^src/libexpr/print-ambiguous\.hh$''
+        ''^src/libexpr/print-options\.hh$''
+        ''^src/libexpr/print\.cc$''
+        ''^src/libexpr/print\.hh$''
+        ''^src/libexpr/search-path\.cc$''
+        ''^src/libexpr/symbol-table\.hh$''
+        ''^src/libexpr/value-to-json\.cc$''
+        ''^src/libexpr/value-to-json\.hh$''
+        ''^src/libexpr/value-to-xml\.cc$''
+        ''^src/libexpr/value-to-xml\.hh$''
+        ''^src/libexpr/value\.hh$''
+        ''^src/libexpr/value/context\.cc$''
+        ''^src/libexpr/value/context\.hh$''
+        ''^src/libfetchers/attrs\.cc$''
+        ''^src/libfetchers/cache\.cc$''
+        ''^src/libfetchers/cache\.hh$''
+        ''^src/libfetchers/fetch-settings\.cc$''
+        ''^src/libfetchers/fetch-settings\.hh$''
+        ''^src/libfetchers/fetch-to-store\.cc$''
+        ''^src/libfetchers/fetchers\.cc$''
+        ''^src/libfetchers/fetchers\.hh$''
+        ''^src/libfetchers/filtering-input-accessor\.cc$''
+        ''^src/libfetchers/filtering-input-accessor\.hh$''
+        ''^src/libfetchers/fs-input-accessor\.cc$''
+        ''^src/libfetchers/fs-input-accessor\.hh$''
+        ''^src/libfetchers/git-utils\.cc$''
+        ''^src/libfetchers/git-utils\.hh$''
+        ''^src/libfetchers/github\.cc$''
+        ''^src/libfetchers/indirect\.cc$''
+        ''^src/libfetchers/memory-input-accessor\.cc$''
+        ''^src/libfetchers/path\.cc$''
+        ''^src/libfetchers/registry\.cc$''
+        ''^src/libfetchers/registry\.hh$''
+        ''^src/libfetchers/tarball\.cc$''
+        ''^src/libfetchers/tarball\.hh$''
+        ''^src/libfetchers/unix/git\.cc$''
+        ''^src/libfetchers/unix/mercurial\.cc$''
+        ''^src/libmain/common-args\.cc$''
+        ''^src/libmain/common-args\.hh$''
+        ''^src/libmain/loggers\.cc$''
+        ''^src/libmain/loggers\.hh$''
+        ''^src/libmain/progress-bar\.cc$''
+        ''^src/libmain/shared\.cc$''
+        ''^src/libmain/shared\.hh$''
+        ''^src/libmain/unix/stack\.cc$''
+        ''^src/libstore/binary-cache-store\.cc$''
+        ''^src/libstore/binary-cache-store\.hh$''
+        ''^src/libstore/build-result\.hh$''
+        ''^src/libstore/builtins\.hh$''
+        ''^src/libstore/builtins/buildenv\.cc$''
+        ''^src/libstore/builtins/buildenv\.hh$''
+        ''^src/libstore/common-protocol-impl\.hh$''
+        ''^src/libstore/common-protocol\.cc$''
+        ''^src/libstore/common-protocol\.hh$''
+        ''^src/libstore/content-address\.cc$''
+        ''^src/libstore/content-address\.hh$''
+        ''^src/libstore/daemon\.cc$''
+        ''^src/libstore/daemon\.hh$''
+        ''^src/libstore/derivations\.cc$''
+        ''^src/libstore/derivations\.hh$''
+        ''^src/libstore/derived-path-map\.cc$''
+        ''^src/libstore/derived-path-map\.hh$''
+        ''^src/libstore/derived-path\.cc$''
+        ''^src/libstore/derived-path\.hh$''
+        ''^src/libstore/downstream-placeholder\.cc$''
+        ''^src/libstore/downstream-placeholder\.hh$''
+        ''^src/libstore/dummy-store\.cc$''
+        ''^src/libstore/export-import\.cc$''
+        ''^src/libstore/filetransfer\.cc$''
+        ''^src/libstore/filetransfer\.hh$''
+        ''^src/libstore/gc-store\.hh$''
+        ''^src/libstore/globals\.cc$''
+        ''^src/libstore/globals\.hh$''
+        ''^src/libstore/http-binary-cache-store\.cc$''
+        ''^src/libstore/legacy-ssh-store\.cc$''
+        ''^src/libstore/legacy-ssh-store\.hh$''
+        ''^src/libstore/length-prefixed-protocol-helper\.hh$''
+        ''^src/libstore/linux/personality\.cc$''
+        ''^src/libstore/linux/personality\.hh$''
+        ''^src/libstore/local-binary-cache-store\.cc$''
+        ''^src/libstore/local-fs-store\.cc$''
+        ''^src/libstore/local-fs-store\.hh$''
+        ''^src/libstore/log-store\.cc$''
+        ''^src/libstore/log-store\.hh$''
+        ''^src/libstore/machines\.cc$''
+        ''^src/libstore/machines\.hh$''
+        ''^src/libstore/make-content-addressed\.cc$''
+        ''^src/libstore/make-content-addressed\.hh$''
+        ''^src/libstore/misc\.cc$''
+        ''^src/libstore/names\.cc$''
+        ''^src/libstore/names\.hh$''
+        ''^src/libstore/nar-accessor\.cc$''
+        ''^src/libstore/nar-accessor\.hh$''
+        ''^src/libstore/nar-info-disk-cache\.cc$''
+        ''^src/libstore/nar-info-disk-cache\.hh$''
+        ''^src/libstore/nar-info\.cc$''
+        ''^src/libstore/nar-info\.hh$''
+        ''^src/libstore/outputs-spec\.cc$''
+        ''^src/libstore/outputs-spec\.hh$''
+        ''^src/libstore/parsed-derivations\.cc$''
+        ''^src/libstore/path-info\.cc$''
+        ''^src/libstore/path-info\.hh$''
+        ''^src/libstore/path-references\.cc$''
+        ''^src/libstore/path-regex\.hh$''
+        ''^src/libstore/path-with-outputs\.cc$''
+        ''^src/libstore/path\.cc$''
+        ''^src/libstore/path\.hh$''
+        ''^src/libstore/pathlocks\.cc$''
+        ''^src/libstore/pathlocks\.hh$''
+        ''^src/libstore/profiles\.cc$''
+        ''^src/libstore/profiles\.hh$''
+        ''^src/libstore/realisation\.cc$''
+        ''^src/libstore/realisation\.hh$''
+        ''^src/libstore/remote-fs-accessor\.cc$''
+        ''^src/libstore/remote-fs-accessor\.hh$''
+        ''^src/libstore/remote-store-connection\.hh$''
+        ''^src/libstore/remote-store\.cc$''
+        ''^src/libstore/remote-store\.hh$''
+        ''^src/libstore/s3-binary-cache-store\.cc$''
+        ''^src/libstore/s3\.hh$''
+        ''^src/libstore/serve-protocol-impl\.cc$''
+        ''^src/libstore/serve-protocol-impl\.hh$''
+        ''^src/libstore/serve-protocol\.cc$''
+        ''^src/libstore/serve-protocol\.hh$''
+        ''^src/libstore/sqlite\.cc$''
+        ''^src/libstore/sqlite\.hh$''
+        ''^src/libstore/ssh-store-config\.hh$''
+        ''^src/libstore/ssh-store\.cc$''
+        ''^src/libstore/ssh\.cc$''
+        ''^src/libstore/ssh\.hh$''
+        ''^src/libstore/store-api\.cc$''
+        ''^src/libstore/store-api\.hh$''
+        ''^src/libstore/store-dir-config\.hh$''
+        ''^src/libstore/unix/build/derivation-goal\.cc$''
+        ''^src/libstore/unix/build/derivation-goal\.hh$''
+        ''^src/libstore/unix/build/drv-output-substitution-goal\.cc$''
+        ''^src/libstore/unix/build/drv-output-substitution-goal\.hh$''
+        ''^src/libstore/unix/build/entry-points\.cc$''
+        ''^src/libstore/unix/build/goal\.cc$''
+        ''^src/libstore/unix/build/goal\.hh$''
+        ''^src/libstore/unix/build/hook-instance\.cc$''
+        ''^src/libstore/unix/build/local-derivation-goal\.cc$''
+        ''^src/libstore/unix/build/local-derivation-goal\.hh$''
+        ''^src/libstore/unix/build/substitution-goal\.cc$''
+        ''^src/libstore/unix/build/substitution-goal\.hh$''
+        ''^src/libstore/unix/build/worker\.cc$''
+        ''^src/libstore/unix/build/worker\.hh$''
+        ''^src/libstore/unix/builtins/fetchurl\.cc$''
+        ''^src/libstore/unix/builtins/unpack-channel\.cc$''
+        ''^src/libstore/unix/gc\.cc$''
+        ''^src/libstore/unix/local-overlay-store\.cc$''
+        ''^src/libstore/unix/local-overlay-store\.hh$''
+        ''^src/libstore/unix/local-store\.cc$''
+        ''^src/libstore/unix/local-store\.hh$''
+        ''^src/libstore/unix/lock\.cc$''
+        ''^src/libstore/unix/lock\.hh$''
+        ''^src/libstore/unix/optimise-store\.cc$''
+        ''^src/libstore/unix/pathlocks\.cc$''
+        ''^src/libstore/unix/posix-fs-canonicalise\.cc$''
+        ''^src/libstore/unix/posix-fs-canonicalise\.hh$''
+        ''^src/libstore/unix/uds-remote-store\.cc$''
+        ''^src/libstore/unix/uds-remote-store\.hh$''
+        ''^src/libstore/windows/build\.cc$''
+        ''^src/libstore/worker-protocol-impl\.hh$''
+        ''^src/libstore/worker-protocol\.cc$''
+        ''^src/libstore/worker-protocol\.hh$''
+        ''^src/libutil-c/nix_api_util_internal\.h$''
+        ''^src/libutil/archive\.cc$''
+        ''^src/libutil/archive\.hh$''
+        ''^src/libutil/args\.cc$''
+        ''^src/libutil/args\.hh$''
+        ''^src/libutil/args/root\.hh$''
+        ''^src/libutil/callback\.hh$''
+        ''^src/libutil/canon-path\.cc$''
+        ''^src/libutil/canon-path\.hh$''
+        ''^src/libutil/chunked-vector\.hh$''
+        ''^src/libutil/closure\.hh$''
+        ''^src/libutil/comparator\.hh$''
+        ''^src/libutil/compute-levels\.cc$''
+        ''^src/libutil/config-impl\.hh$''
+        ''^src/libutil/config\.cc$''
+        ''^src/libutil/config\.hh$''
+        ''^src/libutil/current-process\.cc$''
+        ''^src/libutil/current-process\.hh$''
+        ''^src/libutil/english\.cc$''
+        ''^src/libutil/english\.hh$''
+        ''^src/libutil/environment-variables\.cc$''
+        ''^src/libutil/error\.cc$''
+        ''^src/libutil/error\.hh$''
+        ''^src/libutil/exit\.hh$''
+        ''^src/libutil/experimental-features\.cc$''
+        ''^src/libutil/experimental-features\.hh$''
+        ''^src/libutil/file-content-address\.cc$''
+        ''^src/libutil/file-content-address\.hh$''
+        ''^src/libutil/file-descriptor\.cc$''
+        ''^src/libutil/file-descriptor\.hh$''
+        ''^src/libutil/file-path-impl\.hh$''
+        ''^src/libutil/file-path\.hh$''
+        ''^src/libutil/file-system\.cc$''
+        ''^src/libutil/file-system\.hh$''
+        ''^src/libutil/finally\.hh$''
+        ''^src/libutil/fmt\.hh$''
+        ''^src/libutil/fs-sink\.cc$''
+        ''^src/libutil/fs-sink\.hh$''
+        ''^src/libutil/git\.cc$''
+        ''^src/libutil/git\.hh$''
+        ''^src/libutil/hash\.cc$''
+        ''^src/libutil/hash\.hh$''
+        ''^src/libutil/hilite\.cc$''
+        ''^src/libutil/hilite\.hh$''
+        ''^src/libutil/input-accessor\.hh$''
+        ''^src/libutil/json-impls\.hh$''
+        ''^src/libutil/json-utils\.cc$''
+        ''^src/libutil/json-utils\.hh$''
+        ''^src/libutil/linux/cgroup\.cc$''
+        ''^src/libutil/linux/namespaces\.cc$''
+        ''^src/libutil/logging\.cc$''
+        ''^src/libutil/logging\.hh$''
+        ''^src/libutil/lru-cache\.hh$''
+        ''^src/libutil/memory-source-accessor\.cc$''
+        ''^src/libutil/memory-source-accessor\.hh$''
+        ''^src/libutil/pool\.hh$''
+        ''^src/libutil/position\.cc$''
+        ''^src/libutil/position\.hh$''
+        ''^src/libutil/posix-source-accessor\.cc$''
+        ''^src/libutil/posix-source-accessor\.hh$''
+        ''^src/libutil/processes\.hh$''
+        ''^src/libutil/ref\.hh$''
+        ''^src/libutil/references\.cc$''
+        ''^src/libutil/references\.hh$''
+        ''^src/libutil/regex-combinators\.hh$''
+        ''^src/libutil/serialise\.cc$''
+        ''^src/libutil/serialise\.hh$''
+        ''^src/libutil/signals\.hh$''
+        ''^src/libutil/signature/local-keys\.cc$''
+        ''^src/libutil/signature/local-keys\.hh$''
+        ''^src/libutil/signature/signer\.cc$''
+        ''^src/libutil/signature/signer\.hh$''
+        ''^src/libutil/source-accessor\.cc$''
+        ''^src/libutil/source-accessor\.hh$''
+        ''^src/libutil/source-path\.cc$''
+        ''^src/libutil/source-path\.hh$''
+        ''^src/libutil/split\.hh$''
+        ''^src/libutil/suggestions\.cc$''
+        ''^src/libutil/suggestions\.hh$''
+        ''^src/libutil/sync\.hh$''
+        ''^src/libutil/terminal\.cc$''
+        ''^src/libutil/terminal\.hh$''
+        ''^src/libutil/thread-pool\.cc$''
+        ''^src/libutil/thread-pool\.hh$''
+        ''^src/libutil/topo-sort\.hh$''
+        ''^src/libutil/types\.hh$''
+        ''^src/libutil/unix/file-descriptor\.cc$''
+        ''^src/libutil/unix/file-path\.cc$''
+        ''^src/libutil/unix/monitor-fd\.hh$''
+        ''^src/libutil/unix/processes\.cc$''
+        ''^src/libutil/unix/signals-impl\.hh$''
+        ''^src/libutil/unix/signals\.cc$''
+        ''^src/libutil/unix/unix-domain-socket\.cc$''
+        ''^src/libutil/unix/users\.cc$''
+        ''^src/libutil/url-parts\.hh$''
+        ''^src/libutil/url\.cc$''
+        ''^src/libutil/url\.hh$''
+        ''^src/libutil/users\.cc$''
+        ''^src/libutil/users\.hh$''
+        ''^src/libutil/util\.cc$''
+        ''^src/libutil/util\.hh$''
+        ''^src/libutil/variant-wrapper\.hh$''
+        ''^src/libutil/windows/environment-variables\.cc$''
+        ''^src/libutil/windows/file-descriptor\.cc$''
+        ''^src/libutil/windows/file-path\.cc$''
+        ''^src/libutil/windows/processes\.cc$''
+        ''^src/libutil/windows/users\.cc$''
+        ''^src/libutil/windows/windows-error\.cc$''
+        ''^src/libutil/windows/windows-error\.hh$''
+        ''^src/libutil/xml-writer\.cc$''
+        ''^src/libutil/xml-writer\.hh$''
+        ''^src/nix-build/nix-build\.cc$''
+        ''^src/nix-channel/nix-channel\.cc$''
+        ''^src/nix-collect-garbage/nix-collect-garbage\.cc$''
+        ''^src/nix-env/buildenv.nix$''
+        ''^src/nix-env/nix-env\.cc$''
+        ''^src/nix-env/user-env\.cc$''
+        ''^src/nix-env/user-env\.hh$''
+        ''^src/nix-instantiate/nix-instantiate\.cc$''
+        ''^src/nix-store/dotgraph\.cc$''
+        ''^src/nix-store/graphml\.cc$''
+        ''^src/nix-store/nix-store\.cc$''
+        ''^src/nix/add-to-store\.cc$''
+        ''^src/nix/app\.cc$''
+        ''^src/nix/build\.cc$''
+        ''^src/nix/bundle\.cc$''
+        ''^src/nix/cat\.cc$''
+        ''^src/nix/config-check\.cc$''
+        ''^src/nix/config\.cc$''
+        ''^src/nix/copy\.cc$''
+        ''^src/nix/derivation-add\.cc$''
+        ''^src/nix/derivation-show\.cc$''
+        ''^src/nix/derivation\.cc$''
+        ''^src/nix/develop\.cc$''
+        ''^src/nix/diff-closures\.cc$''
+        ''^src/nix/dump-path\.cc$''
+        ''^src/nix/edit\.cc$''
+        ''^src/nix/eval\.cc$''
+        ''^src/nix/flake\.cc$''
+        ''^src/nix/fmt\.cc$''
+        ''^src/nix/hash\.cc$''
+        ''^src/nix/log\.cc$''
+        ''^src/nix/ls\.cc$''
+        ''^src/nix/main\.cc$''
+        ''^src/nix/make-content-addressed\.cc$''
+        ''^src/nix/nar\.cc$''
+        ''^src/nix/optimise-store\.cc$''
+        ''^src/nix/path-from-hash-part\.cc$''
+        ''^src/nix/path-info\.cc$''
+        ''^src/nix/prefetch\.cc$''
+        ''^src/nix/profile\.cc$''
+        ''^src/nix/realisation\.cc$''
+        ''^src/nix/registry\.cc$''
+        ''^src/nix/repl\.cc$''
+        ''^src/nix/run\.cc$''
+        ''^src/nix/run\.hh$''
+        ''^src/nix/search\.cc$''
+        ''^src/nix/sigs\.cc$''
+        ''^src/nix/store-copy-log\.cc$''
+        ''^src/nix/store-delete\.cc$''
+        ''^src/nix/store-gc\.cc$''
+        ''^src/nix/store-info\.cc$''
+        ''^src/nix/store-repair\.cc$''
+        ''^src/nix/store\.cc$''
+        ''^src/nix/unix/daemon\.cc$''
+        ''^src/nix/upgrade-nix\.cc$''
+        ''^src/nix/verify\.cc$''
+        ''^src/nix/why-depends\.cc$''
+      ];
+    };
+
+  };
+
+  # We'll be pulling from this in the main flake
+  flake.getSystem = getSystem;
+}

maintainers/local.mk

@@ -0,0 +1,15 @@
+
+.PHONY: format
+print-top-help += echo '  format: Format source code'
+
+# This uses the cached .pre-commit-hooks.yaml file
+format:
+	@if ! type -p pre-commit &>/dev/null; then \
+	  echo "make format: pre-commit not found. Please use \`nix develop\`."; \
+	  exit 1; \
+	fi; \
+	if test -z "$$_NIX_PRE_COMMIT_HOOKS_CONFIG"; then \
+	  echo "make format: _NIX_PRE_COMMIT_HOOKS_CONFIG not set. Please use \`nix develop\`."; \
+	  exit 1; \
+	fi; \
+	pre-commit run --config $$_NIX_PRE_COMMIT_HOOKS_CONFIG --all-files

maintainers/upload-release.pl

@@ -171,6 +171,10 @@ eval {
     downloadFile("binaryTarballCross.x86_64-linux.armv7l-unknown-linux-gnueabihf", "1");
 };
 warn "$@" if $@;
+eval {
+    downloadFile("binaryTarballCross.x86_64-linux.riscv64-unknown-linux-gnu", "1");
+};
+warn "$@" if $@;
 downloadFile("installerScript", "1");
 
 # Upload docker images to dockerhub.

mk/compilation-database.mk

@@ -0,0 +1,11 @@
+compile-commands-json-files :=
+
+define write-compile-commands
+  _srcs := $$(sort $$(foreach src, $$($(1)_SOURCES), $$(src)))
+
+  $(1)_COMPILE_COMMANDS_JSON := $$(addprefix $(buildprefix), $$(addsuffix .compile_commands.json, $$(basename $$(_srcs))))
+
+  compile-commands-json-files += $$($(1)_COMPILE_COMMANDS_JSON)
+
+  clean-files += $$($(1)_COMPILE_COMMANDS_JSON)
+endef

mk/lib.mk

@@ -68,6 +68,7 @@ include mk/patterns.mk
 include mk/templates.mk
 include mk/cxx-big-literal.mk
 include mk/tests.mk
+include mk/compilation-database.mk
 
 
 # Include all sub-Makefiles.
@@ -97,6 +98,13 @@ $(foreach test-group, $(install-tests-groups), \
     $(eval $(call run-test,$(test),$(install_test_init))) \
     $(eval $(test-group).test-group: $(test).test)))
 
+# Compilation database.
+$(foreach lib, $(libraries), $(eval $(call write-compile-commands,$(lib))))
+$(foreach prog, $(programs), $(eval $(call write-compile-commands,$(prog))))
+
+compile_commands.json: $(compile-commands-json-files)
+	@jq --slurp '.' $^ >$@
+
 # Include makefiles requiring built programs.
 $(foreach mf, $(makefiles-late), $(eval $(call include-sub-makefile,$(mf))))
 

mk/patterns.mk

@@ -1,11 +1,41 @@
+
+# These are the complete command lines we use to compile C and C++ files.
+# - $< is the source file.
+# - $1 is the object file to create.
+CC_CMD=$(CC) -o $1 -c $< $(CPPFLAGS) $(GLOBAL_CFLAGS) $(CFLAGS) $($1_CFLAGS) -MMD -MF $(call filename-to-dep,$1) -MP
+CXX_CMD=$(CXX) -o $1 -c $< $(CPPFLAGS) $(GLOBAL_CXXFLAGS_PCH) $(GLOBAL_CXXFLAGS) $(CXXFLAGS) $($1_CXXFLAGS) $(ERROR_SWITCH_ENUM) -MMD -MF $(call filename-to-dep,$1) -MP
+
+# We use COMPILE_COMMANDS_JSON_CMD to turn a compilation command (like CC_CMD
+# or CXX_CMD above) into a comple_commands.json file. We rely on bash native
+# word splitting to define the positional arguments.
+# - $< is the source file being compiled.
+COMPILE_COMMANDS_JSON_CMD=jq --null-input '{ directory: $$ENV.PWD, file: "$<", arguments: $$ARGS.positional }' --args --
+
+
 $(buildprefix)%.o: %.cc
 	@mkdir -p "$(dir $@)"
-	$(trace-cxx) $(CXX) -o $@ -c $< $(CPPFLAGS) $(GLOBAL_CXXFLAGS_PCH) $(GLOBAL_CXXFLAGS) $(CXXFLAGS) $($@_CXXFLAGS) $(ERROR_SWITCH_ENUM) -MMD -MF $(call filename-to-dep, $@) -MP
+	$(trace-cxx) $(call CXX_CMD,$@)
 
 $(buildprefix)%.o: %.cpp
 	@mkdir -p "$(dir $@)"
-	$(trace-cxx) $(CXX) -o $@ -c $< $(CPPFLAGS) $(GLOBAL_CXXFLAGS_PCH) $(GLOBAL_CXXFLAGS) $(CXXFLAGS) $($@_CXXFLAGS) $(ERROR_SWITCH_ENUM) -MMD -MF $(call filename-to-dep, $@) -MP
+	$(trace-cxx) $(call CXX_CMD,$@)
 
 $(buildprefix)%.o: %.c
 	@mkdir -p "$(dir $@)"
-	$(trace-cc) $(CC) -o $@ -c $< $(CPPFLAGS) $(GLOBAL_CFLAGS) $(CFLAGS) $($@_CFLAGS) -MMD -MF $(call filename-to-dep, $@) -MP
+	$(trace-cc) $(call CC_CMD,$@)
+
+# In the following we need to replace the .compile_commands.json extension in $@ with .o
+# to make the object file. This is needed because CC_CMD and CXX_CMD do further expansions
+# based on the object file name (i.e. *_CXXFLAGS and filename-to-dep).
+
+$(buildprefix)%.compile_commands.json: %.cc
+	@mkdir -p "$(dir $@)"
+	$(trace-jq) $(COMPILE_COMMANDS_JSON_CMD) $(call CXX_CMD,$(@:.compile_commands.json=.o)) > $@
+
+$(buildprefix)%.compile_commands.json: %.cpp
+	@mkdir -p "$(dir $@)"
+	$(trace-jq) $(COMPILE_COMMANDS_JSON_CMD) $(call CXX_CMD,$(@:.compile_commands.json=.o)) > $@
+
+$(buildprefix)%.compile_commands.json: %.c
+	@mkdir -p "$(dir $@)"
+	$(trace-jq) $(COMPILE_COMMANDS_JSON_CMD) $(call CC_CMD,$(@:.compile_commands.json=.o)) > $@

mk/tracing.mk

@@ -10,6 +10,8 @@ ifeq ($(V), 0)
   trace-install = @echo "  INST  " $@;
   trace-mkdir   = @echo "  MKDIR " $@;
   trace-test    = @echo "  TEST  " $@;
+  trace-sh      = @echo "  SH    " $@;
+  trace-jq      = @echo "  JQ    " $@;
 
   suppress  = @
 

package.nix

@@ -1,10 +1,12 @@
 { lib
+, fetchurl
 , stdenv
 , releaseTools
 , autoconf-archive
 , autoreconfHook
 , aws-sdk-cpp
 , boehmgc
+, buildPackages
 , nlohmann_json
 , bison
 , boost
@@ -75,7 +77,10 @@
 # sounds so long as evaluation just takes places within short-lived
 # processes. (When the process exits, the memory is reclaimed; it is
 # only leaked *within* the process.)
-, enableGC ? true
+#
+# Temporarily disabled on Windows because the `GC_throw_bad_alloc`
+# symbol is missing during linking.
+, enableGC ? !stdenv.hostPlatform.isWindows
 
 # Whether to enable Markdown rendering in the Nix binary.
 , enableMarkdown ? !stdenv.hostPlatform.isWindows
@@ -88,9 +93,10 @@
 # - readline
 , readlineFlavor ? if stdenv.hostPlatform.isWindows then "readline" else "editline"
 
-# Whether to build the internal API docs, can be done separately from
+# Whether to build the internal/external API docs, can be done separately from
 # everything else.
-, enableInternalAPIDocs ? false
+, enableInternalAPIDocs ? forDevShell
+, enableExternalAPIDocs ? forDevShell
 
 # Whether to install unit tests. This is useful when cross compiling
 # since we cannot run them natively during the build, but can do so
@@ -162,6 +168,8 @@ in {
           ./m4
           # TODO: do we really need README.md? It doesn't seem used in the build.
           ./README.md
+          # This could be put behind a conditional
+          ./maintainers/local.mk
           # For make, regardless of what we are building
           ./local.mk
           ./Makefile
@@ -179,6 +187,9 @@ in {
           ./doc/manual
         ] ++ lib.optionals enableInternalAPIDocs [
           ./doc/internal-api
+        ] ++ lib.optionals enableExternalAPIDocs [
+          ./doc/external-api
+        ] ++ lib.optionals (enableInternalAPIDocs || enableExternalAPIDocs) [
           # Source might not be compiled, but still must be available
           # for Doxygen to gather comments.
           ./src
@@ -196,7 +207,7 @@ in {
     ++ lib.optional doBuild "dev"
     # If we are doing just build or just docs, the one thing will use
     # "out". We only need additional outputs if we are doing both.
-    ++ lib.optional (doBuild && (enableManual || enableInternalAPIDocs)) "doc"
+    ++ lib.optional (doBuild && (enableManual || enableInternalAPIDocs || enableExternalAPIDocs)) "doc"
     ++ lib.optional installUnitTests "check";
 
   nativeBuildInputs = [
@@ -218,7 +229,7 @@ in {
   ] ++ lib.optionals (doInstallCheck || enableManual) [
     jq # Also for custom mdBook preprocessor.
   ] ++ lib.optional stdenv.hostPlatform.isLinux util-linux
-    ++ lib.optional enableInternalAPIDocs doxygen
+    ++ lib.optional (enableInternalAPIDocs || enableExternalAPIDocs) doxygen
   ;
 
   buildInputs = lib.optionals doBuild [
@@ -238,7 +249,13 @@ in {
   ] ++ lib.optionals buildUnitTests [
     gtest
     rapidcheck
-  ] ++ lib.optional stdenv.isLinux libseccomp
+  ] ++ lib.optional stdenv.isLinux (libseccomp.overrideAttrs (_: rec {
+    version = "2.5.5";
+    src = fetchurl {
+      url = "https://github.com/seccomp/libseccomp/releases/download/v${version}/libseccomp-${version}.tar.gz";
+      hash = "sha256-JIosik2bmFiqa69ScSw0r+/PnJ6Ut23OAsHJqiX7M3U=";
+    };
+  }))
     ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid
     # There have been issues building these dependencies
     ++ lib.optional (stdenv.hostPlatform == stdenv.buildPlatform && (stdenv.isLinux || stdenv.isDarwin))
@@ -282,6 +299,7 @@ in {
     (lib.enableFeature buildUnitTests "unit-tests")
     (lib.enableFeature doInstallCheck "functional-tests")
     (lib.enableFeature enableInternalAPIDocs "internal-api-docs")
+    (lib.enableFeature enableExternalAPIDocs "external-api-docs")
     (lib.enableFeature enableManual "doc-gen")
     (lib.enableFeature enableGC "gc")
     (lib.enableFeature enableMarkdown "markdown")
@@ -306,7 +324,8 @@ in {
   makeFlags = "profiledir=$(out)/etc/profile.d PRECOMPILE_HEADERS=1";
 
   installTargets = lib.optional doBuild "install"
-    ++ lib.optional enableInternalAPIDocs "internal-api-html";
+    ++ lib.optional enableInternalAPIDocs "internal-api-html"
+    ++ lib.optional enableExternalAPIDocs "external-api-html";
 
   installFlags = "sysconfdir=$(out)/etc";
 
@@ -333,6 +352,16 @@ in {
   '' + lib.optionalString enableInternalAPIDocs ''
     mkdir -p ''${!outputDoc}/nix-support
     echo "doc internal-api-docs $out/share/doc/nix/internal-api/html" >> ''${!outputDoc}/nix-support/hydra-build-products
+  ''
+    + lib.optionalString enableExternalAPIDocs ''
+    mkdir -p ''${!outputDoc}/nix-support
+    echo "doc external-api-docs $out/share/doc/nix/external-api/html" >> ''${!outputDoc}/nix-support/hydra-build-products
+  '';
+
+  # So the check output gets links for DLLs in the out output.
+  preFixup = lib.optionalString (stdenv.hostPlatform.isWindows && builtins.elem "check" finalAttrs.outputs) ''
+    ln -s "$check/lib/"*.dll "$check/bin"
+    ln -s "$out/bin/"*.dll "$check/bin"
   '';
 
   doInstallCheck = attrs.doInstallCheck;

precompiled-headers.h

@@ -42,19 +42,22 @@
 #include <dirent.h>
 #include <errno.h>
 #include <fcntl.h>
-#include <grp.h>
-#include <netdb.h>
-#include <pwd.h>
 #include <signal.h>
-#include <sys/resource.h>
-#include <sys/select.h>
-#include <sys/socket.h>
 #include <sys/stat.h>
 #include <sys/time.h>
 #include <sys/types.h>
-#include <sys/utsname.h>
-#include <sys/wait.h>
-#include <termios.h>
 #include <unistd.h>
 
+#ifndef _WIN32
+# include <grp.h>
+# include <netdb.h>
+# include <pwd.h>
+# include <sys/resource.h>
+# include <sys/select.h>
+# include <sys/socket.h>
+# include <sys/utsname.h>
+# include <sys/wait.h>
+# include <termios.h>
+#endif
+
 #include <nlohmann/json.hpp>

scripts/nix-profile-daemon.fish.in

@@ -28,7 +28,7 @@ else
 end
 
 # Set $NIX_SSL_CERT_FILE so that Nixpkgs applications like curl work.
-if test -n "$NIX_SSH_CERT_FILE"
+if test -n "$NIX_SSL_CERT_FILE"
   : # Allow users to override the NIX_SSL_CERT_FILE
 else if test -e /etc/ssl/certs/ca-certificates.crt # NixOS, Ubuntu, Debian, Gentoo, Arch
   set --export NIX_SSL_CERT_FILE /etc/ssl/certs/ca-certificates.crt
@@ -44,7 +44,7 @@ else if test -e "$NIX_LINK/etc/ca-bundle.crt" # old cacert in Nix profile
   set --export NIX_SSL_CERT_FILE "$NIX_LINK/etc/ca-bundle.crt"
 else
   # Fall back to what is in the nix profiles, favouring whatever is defined last.
-  for i in $NIX_PROFILES
+  for i in (string split ' ' $NIX_PROFILES)
     if test -e "$i/etc/ssl/certs/ca-bundle.crt"
       set --export NIX_SSL_CERT_FILE "$i/etc/ssl/certs/ca-bundle.crt"
     end

scripts/nix-profile-daemon.sh.in

@@ -69,4 +69,4 @@ else
 fi
 
 export PATH="$NIX_LINK/bin:@localstatedir@/nix/profiles/default/bin:$PATH"
-unset NIX_LINK
+unset NIX_LINK NIX_LINK_NEW

src/build-remote/build-remote.cc

@@ -202,7 +202,7 @@ static int main_build_remote(int argc, char * * argv)
                         else
                             drvstr = "<unknown>";
 
-                        auto error = HintFmt(errorText);
+                        auto error = HintFmt::fromFormatString(errorText);
                         error
                             % drvstr
                             % neededSystem

src/libcmd/command.cc

@@ -128,10 +128,10 @@ ref<EvalState> EvalCommand::getEvalState()
         evalState =
             #if HAVE_BOEHMGC
             std::allocate_shared<EvalState>(traceable_allocator<EvalState>(),
-                searchPath, getEvalStore(), getStore())
+                lookupPath, getEvalStore(), getStore())
             #else
             std::make_shared<EvalState>(
-                searchPath, getEvalStore(), getStore())
+                lookupPath, getEvalStore(), getStore())
             #endif
             ;
 
@@ -148,7 +148,7 @@ MixOperateOnOptions::MixOperateOnOptions()
 {
     addFlag({
         .longName = "derivation",
-        .description = "Operate on the [store derivation](../../glossary.md#gloss-store-derivation) rather than its outputs.",
+        .description = "Operate on the [store derivation](@docroot@/glossary.md#gloss-store-derivation) rather than its outputs.",
         .category = installablesCategory,
         .handler = {&operateOn, OperateOn::Derivation},
     });

src/libcmd/common-eval-args.cc

@@ -125,7 +125,7 @@ MixEvalArgs::MixEvalArgs()
         .category = category,
         .labels = {"path"},
         .handler = {[&](std::string s) {
-            searchPath.elements.emplace_back(SearchPath::Elem::parse(s));
+            lookupPath.elements.emplace_back(LookupPath::Elem::parse(s));
         }}
     });
 
@@ -181,7 +181,7 @@ Bindings * MixEvalArgs::getAutoArgs(EvalState & state)
                 v->mkString(arg.s);
             },
             [&](const AutoArgFile & arg) {
-                v->mkString(readFile(arg.path));
+                v->mkString(readFile(arg.path.string()));
             },
             [&](const AutoArgStdin & arg) {
                 v->mkString(readFile(STDIN_FILENO));

src/libcmd/common-eval-args.hh

@@ -23,7 +23,7 @@ struct MixEvalArgs : virtual Args, virtual MixRepair
 
     Bindings * getAutoArgs(EvalState & state);
 
-    SearchPath searchPath;
+    LookupPath lookupPath;
 
     std::optional<std::string> evalStoreUrl;
 
@@ -38,6 +38,9 @@ private:
     std::map<std::string, AutoArg> autoArgs;
 };
 
+/**
+ * @param baseDir Optional [base directory](https://nixos.org/manual/nix/unstable/glossary#gloss-base-directory)
+ */
 SourcePath lookupFileArg(EvalState & state, std::string_view s, const Path * baseDir = nullptr);
 
 }

src/libcmd/installable-flake.cc

@@ -49,7 +49,7 @@ Value * InstallableFlake::getFlakeOutputs(EvalState & state, const flake::Locked
 
     callFlake(state, lockedFlake, *vFlake);
 
-    auto aOutputs = vFlake->attrs->get(state.symbols.create("outputs"));
+    auto aOutputs = vFlake->attrs()->get(state.symbols.create("outputs"));
     assert(aOutputs);
 
     state.forceValue(*aOutputs->value, aOutputs->value->determinePos(noPos));

src/libcmd/installables.cc

@@ -289,7 +289,7 @@ void SourceExprCommand::completeInstallable(AddCompletions & completions, std::s
             state->autoCallFunction(*autoArgs, v1, v2);
 
             if (v2.type() == nAttrs) {
-                for (auto & i : *v2.attrs) {
+                for (auto & i : *v2.attrs()) {
                     std::string name = state->symbols[i.name];
                     if (name.find(searchWord) == 0) {
                         if (prefix_ == "")
@@ -461,7 +461,7 @@ ref<eval_cache::EvalCache> openEvalCache(
 
             state.forceAttrs(*vFlake, noPos, "while parsing cached flake data");
 
-            auto aOutputs = vFlake->attrs->get(state.symbols.create("outputs"));
+            auto aOutputs = vFlake->attrs()->get(state.symbols.create("outputs"));
             assert(aOutputs);
 
             return aOutputs->value;

src/libcmd/local.mk

@@ -6,7 +6,7 @@ libcmd_DIR := $(d)
 
 libcmd_SOURCES := $(wildcard $(d)/*.cc)
 
-libcmd_CXXFLAGS += -I src/libutil -I src/libstore -I src/libexpr -I src/libmain -I src/libfetchers
+libcmd_CXXFLAGS += $(INCLUDE_libutil) $(INCLUDE_libstore) $(INCLUDE_libfetchers) $(INCLUDE_libexpr) $(INCLUDE_libmain)
 
 libcmd_LDFLAGS = $(EDITLINE_LIBS) $(LOWDOWN_LIBS) $(THREAD_LDFLAGS)
 

src/libcmd/markdown.cc

@@ -3,9 +3,9 @@
 #include "finally.hh"
 #include "terminal.hh"
 
-#include <sys/queue.h>
 #if HAVE_LOWDOWN
-#include <lowdown.h>
+# include <sys/queue.h>
+# include <lowdown.h>
 #endif
 
 namespace nix {
@@ -50,7 +50,7 @@ std::string renderMarkdownToTerminal(std::string_view markdown)
     if (!rndr_res)
         throw Error("allocation error while rendering Markdown");
 
-    return filterANSIEscapes(std::string(buf->data, buf->size), !shouldANSI());
+    return filterANSIEscapes(std::string(buf->data, buf->size), !isTTY());
 #else
     return std::string(markdown);
 #endif

src/libcmd/repl-interacter.cc

@@ -0,0 +1,190 @@
+#include <cstdio>
+
+#ifdef USE_READLINE
+#include <readline/history.h>
+#include <readline/readline.h>
+#else
+// editline < 1.15.2 don't wrap their API for C++ usage
+// (added in https://github.com/troglobit/editline/commit/91398ceb3427b730995357e9d120539fb9bb7461).
+// This results in linker errors due to to name-mangling of editline C symbols.
+// For compatibility with these versions, we wrap the API here
+// (wrapping multiple times on newer versions is no problem).
+extern "C" {
+#include <editline.h>
+}
+#endif
+
+#include "signals.hh"
+#include "finally.hh"
+#include "repl-interacter.hh"
+#include "file-system.hh"
+#include "libcmd/repl.hh"
+
+namespace nix {
+
+namespace {
+// Used to communicate to NixRepl::getLine whether a signal occurred in ::readline.
+volatile sig_atomic_t g_signal_received = 0;
+
+void sigintHandler(int signo)
+{
+    g_signal_received = signo;
+}
+};
+
+static detail::ReplCompleterMixin * curRepl; // ugly
+
+static char * completionCallback(char * s, int * match)
+{
+    auto possible = curRepl->completePrefix(s);
+    if (possible.size() == 1) {
+        *match = 1;
+        auto * res = strdup(possible.begin()->c_str() + strlen(s));
+        if (!res)
+            throw Error("allocation failure");
+        return res;
+    } else if (possible.size() > 1) {
+        auto checkAllHaveSameAt = [&](size_t pos) {
+            auto & first = *possible.begin();
+            for (auto & p : possible) {
+                if (p.size() <= pos || p[pos] != first[pos])
+                    return false;
+            }
+            return true;
+        };
+        size_t start = strlen(s);
+        size_t len = 0;
+        while (checkAllHaveSameAt(start + len))
+            ++len;
+        if (len > 0) {
+            *match = 1;
+            auto * res = strdup(std::string(*possible.begin(), start, len).c_str());
+            if (!res)
+                throw Error("allocation failure");
+            return res;
+        }
+    }
+
+    *match = 0;
+    return nullptr;
+}
+
+static int listPossibleCallback(char * s, char *** avp)
+{
+    auto possible = curRepl->completePrefix(s);
+
+    if (possible.size() > (INT_MAX / sizeof(char *)))
+        throw Error("too many completions");
+
+    int ac = 0;
+    char ** vp = nullptr;
+
+    auto check = [&](auto * p) {
+        if (!p) {
+            if (vp) {
+                while (--ac >= 0)
+                    free(vp[ac]);
+                free(vp);
+            }
+            throw Error("allocation failure");
+        }
+        return p;
+    };
+
+    vp = check((char **) malloc(possible.size() * sizeof(char *)));
+
+    for (auto & p : possible)
+        vp[ac++] = check(strdup(p.c_str()));
+
+    *avp = vp;
+
+    return ac;
+}
+
+ReadlineLikeInteracter::Guard ReadlineLikeInteracter::init(detail::ReplCompleterMixin * repl)
+{
+    // Allow nix-repl specific settings in .inputrc
+    rl_readline_name = "nix-repl";
+    try {
+        createDirs(dirOf(historyFile));
+    } catch (SystemError & e) {
+        logWarning(e.info());
+    }
+#ifndef USE_READLINE
+    el_hist_size = 1000;
+#endif
+    read_history(historyFile.c_str());
+    auto oldRepl = curRepl;
+    curRepl = repl;
+    Guard restoreRepl([oldRepl] { curRepl = oldRepl; });
+#ifndef USE_READLINE
+    rl_set_complete_func(completionCallback);
+    rl_set_list_possib_func(listPossibleCallback);
+#endif
+    return restoreRepl;
+}
+
+static constexpr const char * promptForType(ReplPromptType promptType)
+{
+    switch (promptType) {
+    case ReplPromptType::ReplPrompt:
+        return "nix-repl> ";
+    case ReplPromptType::ContinuationPrompt:
+        return "          ";
+    }
+    assert(false);
+}
+
+bool ReadlineLikeInteracter::getLine(std::string & input, ReplPromptType promptType)
+{
+#ifndef _WIN32 // TODO use more signals.hh for this
+    struct sigaction act, old;
+    sigset_t savedSignalMask, set;
+
+    auto setupSignals = [&]() {
+        act.sa_handler = sigintHandler;
+        sigfillset(&act.sa_mask);
+        act.sa_flags = 0;
+        if (sigaction(SIGINT, &act, &old))
+            throw SysError("installing handler for SIGINT");
+
+        sigemptyset(&set);
+        sigaddset(&set, SIGINT);
+        if (sigprocmask(SIG_UNBLOCK, &set, &savedSignalMask))
+            throw SysError("unblocking SIGINT");
+    };
+    auto restoreSignals = [&]() {
+        if (sigprocmask(SIG_SETMASK, &savedSignalMask, nullptr))
+            throw SysError("restoring signals");
+
+        if (sigaction(SIGINT, &old, 0))
+            throw SysError("restoring handler for SIGINT");
+    };
+
+    setupSignals();
+#endif
+    char * s = readline(promptForType(promptType));
+    Finally doFree([&]() { free(s); });
+#ifndef _WIN32 // TODO use more signals.hh for this
+    restoreSignals();
+#endif
+
+    if (g_signal_received) {
+        g_signal_received = 0;
+        input.clear();
+        return true;
+    }
+
+    if (!s)
+        return false;
+    input += s;
+    input += '\n';
+    return true;
+}
+
+ReadlineLikeInteracter::~ReadlineLikeInteracter()
+{
+    write_history(historyFile.c_str());
+}
+
+};

src/libcmd/repl-interacter.hh

@@ -0,0 +1,48 @@
+#pragma once
+/// @file
+
+#include "finally.hh"
+#include "types.hh"
+#include <functional>
+#include <string>
+
+namespace nix {
+
+namespace detail {
+/** Provides the completion hooks for the repl, without exposing its complete
+ * internals. */
+struct ReplCompleterMixin {
+    virtual StringSet completePrefix(const std::string & prefix) = 0;
+};
+};
+
+enum class ReplPromptType {
+    ReplPrompt,
+    ContinuationPrompt,
+};
+
+class ReplInteracter
+{
+public:
+    using Guard = Finally<std::function<void()>>;
+
+    virtual Guard init(detail::ReplCompleterMixin * repl) = 0;
+    /** Returns a boolean of whether the interacter got EOF */
+    virtual bool getLine(std::string & input, ReplPromptType promptType) = 0;
+    virtual ~ReplInteracter(){};
+};
+
+class ReadlineLikeInteracter : public virtual ReplInteracter
+{
+    std::string historyFile;
+public:
+    ReadlineLikeInteracter(std::string historyFile)
+        : historyFile(historyFile)
+    {
+    }
+    virtual Guard init(detail::ReplCompleterMixin * repl) override;
+    virtual bool getLine(std::string & input, ReplPromptType promptType) override;
+    virtual ~ReadlineLikeInteracter() override;
+};
+
+};

src/libcmd/repl.cc

@@ -3,32 +3,17 @@
 #include <cstring>
 #include <climits>
 
-#include <setjmp.h>
-
-#ifdef USE_READLINE
-#include <readline/history.h>
-#include <readline/readline.h>
-#else
-// editline < 1.15.2 don't wrap their API for C++ usage
-// (added in https://github.com/troglobit/editline/commit/91398ceb3427b730995357e9d120539fb9bb7461).
-// This results in linker errors due to to name-mangling of editline C symbols.
-// For compatibility with these versions, we wrap the API here
-// (wrapping multiple times on newer versions is no problem).
-extern "C" {
-#include <editline.h>
-}
-#endif
-
+#include "libcmd/repl-interacter.hh"
 #include "repl.hh"
 
 #include "ansicolor.hh"
-#include "signals.hh"
 #include "shared.hh"
 #include "eval.hh"
 #include "eval-cache.hh"
 #include "eval-inline.hh"
 #include "eval-settings.hh"
 #include "attr-path.hh"
+#include "signals.hh"
 #include "store-api.hh"
 #include "log-store.hh"
 #include "common-eval-args.hh"
@@ -38,7 +23,6 @@ extern "C" {
 #include "flake/flake.hh"
 #include "flake/lockfile.hh"
 #include "users.hh"
-#include "terminal.hh"
 #include "editor-for.hh"
 #include "finally.hh"
 #include "markdown.hh"
@@ -75,6 +59,7 @@ enum class ProcessLineResult {
 
 struct NixRepl
     : AbstractNixRepl
+    , detail::ReplCompleterMixin
     #if HAVE_BOEHMGC
     , gc
     #endif
@@ -90,17 +75,16 @@ struct NixRepl
     int displ;
     StringSet varNames;
 
-    const Path historyFile;
+    std::unique_ptr<ReplInteracter> interacter;
 
-    NixRepl(const SearchPath & searchPath, nix::ref<Store> store,ref<EvalState> state,
+    NixRepl(const LookupPath & lookupPath, nix::ref<Store> store,ref<EvalState> state,
             std::function<AnnotatedValues()> getValues);
-    virtual ~NixRepl();
+    virtual ~NixRepl() = default;
 
     ReplExitStatus mainLoop() override;
     void initEnv() override;
 
-    StringSet completePrefix(const std::string & prefix);
-    bool getLine(std::string & input, const std::string & prompt);
+    virtual StringSet completePrefix(const std::string & prefix) override;
     StorePath getDerivationPath(Value & v);
     ProcessLineResult processLine(std::string line);
 
@@ -123,7 +107,8 @@ struct NixRepl
             .force = true,
             .derivationPaths = true,
             .maxDepth = maxDepth,
-            .prettyIndent = 2
+            .prettyIndent = 2,
+            .errors = ErrorPrintBehavior::ThrowTopLevel,
         });
     }
 };
@@ -137,111 +122,33 @@ std::string removeWhitespace(std::string s)
 }
 
 
-NixRepl::NixRepl(const SearchPath & searchPath, nix::ref<Store> store, ref<EvalState> state,
+NixRepl::NixRepl(const LookupPath & lookupPath, nix::ref<Store> store, ref<EvalState> state,
             std::function<NixRepl::AnnotatedValues()> getValues)
     : AbstractNixRepl(state)
     , debugTraceIndex(0)
     , getValues(getValues)
     , staticEnv(new StaticEnv(nullptr, state->staticBaseEnv.get()))
-    , historyFile(getDataDir() + "/nix/repl-history")
+    , interacter(make_unique<ReadlineLikeInteracter>(getDataDir() + "/nix/repl-history"))
 {
 }
 
-
-NixRepl::~NixRepl()
-{
-    write_history(historyFile.c_str());
-}
-
 void runNix(Path program, const Strings & args,
     const std::optional<std::string> & input = {})
 {
     auto subprocessEnv = getEnv();
     subprocessEnv["NIX_CONFIG"] = globalConfig.toKeyValue();
-
+    //isInteractive avoid grabling interactive commands
     runProgram2(RunOptions {
         .program = settings.nixBinDir+ "/" + program,
         .args = args,
         .environment = subprocessEnv,
         .input = input,
+        .isInteractive = true,
     });
 
     return;
 }
 
-static NixRepl * curRepl; // ugly
-
-static char * completionCallback(char * s, int *match) {
-  auto possible = curRepl->completePrefix(s);
-  if (possible.size() == 1) {
-    *match = 1;
-    auto *res = strdup(possible.begin()->c_str() + strlen(s));
-    if (!res) throw Error("allocation failure");
-    return res;
-  } else if (possible.size() > 1) {
-    auto checkAllHaveSameAt = [&](size_t pos) {
-      auto &first = *possible.begin();
-      for (auto &p : possible) {
-        if (p.size() <= pos || p[pos] != first[pos])
-          return false;
-      }
-      return true;
-    };
-    size_t start = strlen(s);
-    size_t len = 0;
-    while (checkAllHaveSameAt(start + len)) ++len;
-    if (len > 0) {
-      *match = 1;
-      auto *res = strdup(std::string(*possible.begin(), start, len).c_str());
-      if (!res) throw Error("allocation failure");
-      return res;
-    }
-  }
-
-  *match = 0;
-  return nullptr;
-}
-
-static int listPossibleCallback(char *s, char ***avp) {
-  auto possible = curRepl->completePrefix(s);
-
-  if (possible.size() > (INT_MAX / sizeof(char*)))
-    throw Error("too many completions");
-
-  int ac = 0;
-  char **vp = nullptr;
-
-  auto check = [&](auto *p) {
-    if (!p) {
-      if (vp) {
-        while (--ac >= 0)
-          free(vp[ac]);
-        free(vp);
-      }
-      throw Error("allocation failure");
-    }
-    return p;
-  };
-
-  vp = check((char **)malloc(possible.size() * sizeof(char*)));
-
-  for (auto & p : possible)
-    vp[ac++] = check(strdup(p.c_str()));
-
-  *avp = vp;
-
-  return ac;
-}
-
-namespace {
-    // Used to communicate to NixRepl::getLine whether a signal occurred in ::readline.
-    volatile sig_atomic_t g_signal_received = 0;
-
-    void sigintHandler(int signo) {
-        g_signal_received = signo;
-    }
-}
-
 static std::ostream & showDebugTrace(std::ostream & out, const PosTable & positions, const DebugTrace & dt)
 {
     if (dt.isError)
@@ -281,24 +188,7 @@ ReplExitStatus NixRepl::mainLoop()
 
     loadFiles();
 
-    // Allow nix-repl specific settings in .inputrc
-    rl_readline_name = "nix-repl";
-    try {
-        createDirs(dirOf(historyFile));
-    } catch (SystemError & e) {
-        logWarning(e.info());
-    }
-#ifndef USE_READLINE
-    el_hist_size = 1000;
-#endif
-    read_history(historyFile.c_str());
-    auto oldRepl = curRepl;
-    curRepl = this;
-    Finally restoreRepl([&] { curRepl = oldRepl; });
-#ifndef USE_READLINE
-    rl_set_complete_func(completionCallback);
-    rl_set_list_possib_func(listPossibleCallback);
-#endif
+    auto _guard = interacter->init(static_cast<detail::ReplCompleterMixin *>(this));
 
     std::string input;
 
@@ -307,7 +197,7 @@ ReplExitStatus NixRepl::mainLoop()
         logger->pause();
         // When continuing input from previous lines, don't print a prompt, just align to the same
         // number of chars as the prompt.
-        if (!getLine(input, input.empty() ? "nix-repl> " : "          ")) {
+        if (!interacter->getLine(input, input.empty() ? ReplPromptType::ReplPrompt : ReplPromptType::ContinuationPrompt)) {
             // Ctrl-D should exit the debugger.
             state->debugStop = false;
             logger->cout("");
@@ -350,51 +240,6 @@ ReplExitStatus NixRepl::mainLoop()
     }
 }
 
-
-bool NixRepl::getLine(std::string & input, const std::string & prompt)
-{
-    struct sigaction act, old;
-    sigset_t savedSignalMask, set;
-
-    auto setupSignals = [&]() {
-        act.sa_handler = sigintHandler;
-        sigfillset(&act.sa_mask);
-        act.sa_flags = 0;
-        if (sigaction(SIGINT, &act, &old))
-            throw SysError("installing handler for SIGINT");
-
-        sigemptyset(&set);
-        sigaddset(&set, SIGINT);
-        if (sigprocmask(SIG_UNBLOCK, &set, &savedSignalMask))
-            throw SysError("unblocking SIGINT");
-    };
-    auto restoreSignals = [&]() {
-        if (sigprocmask(SIG_SETMASK, &savedSignalMask, nullptr))
-            throw SysError("restoring signals");
-
-        if (sigaction(SIGINT, &old, 0))
-            throw SysError("restoring handler for SIGINT");
-    };
-
-    setupSignals();
-    char * s = readline(prompt.c_str());
-    Finally doFree([&]() { free(s); });
-    restoreSignals();
-
-    if (g_signal_received) {
-        g_signal_received = 0;
-        input.clear();
-        return true;
-    }
-
-    if (!s)
-      return false;
-    input += s;
-    input += '\n';
-    return true;
-}
-
-
 StringSet NixRepl::completePrefix(const std::string & prefix)
 {
     StringSet completions;
@@ -446,7 +291,7 @@ StringSet NixRepl::completePrefix(const std::string & prefix)
             e->eval(*state, *env, v);
             state->forceAttrs(v, noPos, "while evaluating an attrset for the purpose of completion (this error should not be displayed; file an issue?)");
 
-            for (auto & i : *v.attrs) {
+            for (auto & i : *v.attrs()) {
                 std::string_view name = state->symbols[i.name];
                 if (name.substr(0, cur2.size()) != cur2) continue;
                 completions.insert(concatStrings(prev, expr, ".", name));
@@ -513,7 +358,7 @@ ProcessLineResult NixRepl::processLine(std::string line)
     if (line.empty())
         return ProcessLineResult::PromptAgain;
 
-    _isInterrupted = false;
+    setInterrupted(false);
 
     std::string command, arg;
 
@@ -646,7 +491,7 @@ ProcessLineResult NixRepl::processLine(std::string line)
                 auto path = state->coerceToPath(noPos, v, context, "while evaluating the filename to edit");
                 return {path, 0};
             } else if (v.isLambda()) {
-                auto pos = state->positions[v.lambda.fun->pos];
+                auto pos = state->positions[v.payload.lambda.fun->pos];
                 if (auto path = std::get_if<SourcePath>(&pos.origin))
                     return {*path, pos.line};
                 else
@@ -664,7 +509,7 @@ ProcessLineResult NixRepl::processLine(std::string line)
 
         // runProgram redirects stdout to a StringSink,
         // using runProgram2 to allow editors to display their UI
-        runProgram2(RunOptions { .program = editor, .searchPath = true, .args = args });
+        runProgram2(RunOptions { .program = editor, .lookupPath = true, .args = args , .isInteractive = true });
 
         // Reload right after exiting the editor
         state->resetFileCache();
@@ -898,17 +743,17 @@ void NixRepl::loadFiles()
 void NixRepl::addAttrsToScope(Value & attrs)
 {
     state->forceAttrs(attrs, [&]() { return attrs.determinePos(noPos); }, "while evaluating an attribute set to be merged in the global scope");
-    if (displ + attrs.attrs->size() >= envSize)
+    if (displ + attrs.attrs()->size() >= envSize)
         throw Error("environment full; cannot add more variables");
 
-    for (auto & i : *attrs.attrs) {
+    for (auto & i : *attrs.attrs()) {
         staticEnv->vars.emplace_back(i.name, displ);
         env->values[displ++] = i.value;
         varNames.emplace(state->symbols[i.name]);
     }
     staticEnv->sort();
     staticEnv->deduplicate();
-    notice("Added %1% variables.", attrs.attrs->size());
+    notice("Added %1% variables.", attrs.attrs()->size());
 }
 
 
@@ -940,11 +785,11 @@ void NixRepl::evalString(std::string s, Value & v)
 
 
 std::unique_ptr<AbstractNixRepl> AbstractNixRepl::create(
-   const SearchPath & searchPath, nix::ref<Store> store, ref<EvalState> state,
+   const LookupPath & lookupPath, nix::ref<Store> store, ref<EvalState> state,
    std::function<AnnotatedValues()> getValues)
 {
     return std::make_unique<NixRepl>(
-        searchPath,
+        lookupPath,
         openStore(),
         state,
         getValues
@@ -960,9 +805,9 @@ ReplExitStatus AbstractNixRepl::runSimple(
         NixRepl::AnnotatedValues values;
         return values;
     };
-    SearchPath searchPath = {};
+    LookupPath lookupPath = {};
     auto repl = std::make_unique<NixRepl>(
-            searchPath,
+            lookupPath,
             openStore(),
             evalState,
             getValues

src/libcmd/repl.hh

@@ -3,11 +3,6 @@
 
 #include "eval.hh"
 
-#if HAVE_BOEHMGC
-#define GC_INCLUDE_NEW
-#include <gc/gc_cpp.h>
-#endif
-
 namespace nix {
 
 struct AbstractNixRepl
@@ -25,7 +20,7 @@ struct AbstractNixRepl
     typedef std::vector<std::pair<Value*,std::string>> AnnotatedValues;
 
     static std::unique_ptr<AbstractNixRepl> create(
-        const SearchPath & searchPath, nix::ref<Store> store, ref<EvalState> state,
+        const LookupPath & lookupPath, nix::ref<Store> store, ref<EvalState> state,
         std::function<AnnotatedValues()> getValues);
 
     static ReplExitStatus runSimple(

src/libexpr-c/local.mk

@@ -0,0 +1,25 @@
+libraries += libexprc
+
+libexprc_NAME = libnixexprc
+
+libexprc_DIR := $(d)
+
+libexprc_SOURCES := \
+  $(wildcard $(d)/*.cc) \
+
+# Not just for this library itself, but also for downstream libraries using this library
+
+INCLUDE_libexprc := -I $(d)
+libexprc_CXXFLAGS += $(INCLUDE_libutil) $(INCLUDE_libutilc) \
+                     $(INCLUDE_libfetchers) \
+                     $(INCLUDE_libstore) $(INCLUDE_libstorec) \
+                     $(INCLUDE_libexpr) $(INCLUDE_libexprc)
+
+libexprc_LIBS = libutil libutilc libstore libstorec libexpr
+
+libexprc_LDFLAGS += $(THREAD_LDFLAGS)
+
+$(eval $(call install-file-in, $(d)/nix-expr-c.pc, $(libdir)/pkgconfig, 0644))
+
+libexprc_FORCE_INSTALL := 1
+

src/libexpr-c/nix-expr-c.pc.in

@@ -0,0 +1,10 @@
+prefix=@prefix@
+libdir=@libdir@
+includedir=@includedir@
+
+Name: Nix
+Description: Nix Language Evaluator - C API
+Version: @PACKAGE_VERSION@
+Requires: nix-store-c
+Libs: -L${libdir} -lnixexprc
+Cflags: -I${includedir}/nix

src/libexpr-c/nix_api_expr.cc

@@ -0,0 +1,178 @@
+#include <cstring>
+#include <iostream>
+#include <stdexcept>
+#include <string>
+
+#include "config.hh"
+#include "eval.hh"
+#include "globals.hh"
+#include "util.hh"
+
+#include "nix_api_expr.h"
+#include "nix_api_expr_internal.h"
+#include "nix_api_store.h"
+#include "nix_api_store_internal.h"
+#include "nix_api_util.h"
+#include "nix_api_util_internal.h"
+
+#ifdef HAVE_BOEHMGC
+#include <mutex>
+#define GC_INCLUDE_NEW 1
+#include "gc_cpp.h"
+#endif
+
+nix_err nix_libexpr_init(nix_c_context * context)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    {
+        auto ret = nix_libutil_init(context);
+        if (ret != NIX_OK)
+            return ret;
+    }
+    {
+        auto ret = nix_libstore_init(context);
+        if (ret != NIX_OK)
+            return ret;
+    }
+    try {
+        nix::initGC();
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_expr_eval_from_string(
+    nix_c_context * context, EvalState * state, const char * expr, const char * path, Value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        nix::Expr * parsedExpr = state->state.parseExprFromString(expr, state->state.rootPath(nix::CanonPath(path)));
+        state->state.eval(parsedExpr, *(nix::Value *) value);
+        state->state.forceValue(*(nix::Value *) value, nix::noPos);
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_value_call(nix_c_context * context, EvalState * state, Value * fn, Value * arg, Value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        state->state.callFunction(*(nix::Value *) fn, *(nix::Value *) arg, *(nix::Value *) value, nix::noPos);
+        state->state.forceValue(*(nix::Value *) value, nix::noPos);
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_value_force(nix_c_context * context, EvalState * state, Value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        state->state.forceValue(*(nix::Value *) value, nix::noPos);
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_value_force_deep(nix_c_context * context, EvalState * state, Value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        state->state.forceValueDeep(*(nix::Value *) value);
+    }
+    NIXC_CATCH_ERRS
+}
+
+EvalState * nix_state_create(nix_c_context * context, const char ** lookupPath_c, Store * store)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        nix::Strings lookupPath;
+        if (lookupPath_c != nullptr)
+            for (size_t i = 0; lookupPath_c[i] != nullptr; i++)
+                lookupPath.push_back(lookupPath_c[i]);
+
+        return new EvalState{nix::EvalState(nix::LookupPath::parse(lookupPath), store->ptr)};
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+void nix_state_free(EvalState * state)
+{
+    delete state;
+}
+
+#ifdef HAVE_BOEHMGC
+std::unordered_map<
+    const void *,
+    unsigned int,
+    std::hash<const void *>,
+    std::equal_to<const void *>,
+    traceable_allocator<std::pair<const void * const, unsigned int>>>
+    nix_refcounts;
+
+std::mutex nix_refcount_lock;
+
+nix_err nix_gc_incref(nix_c_context * context, const void * p)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        std::scoped_lock lock(nix_refcount_lock);
+        auto f = nix_refcounts.find(p);
+        if (f != nix_refcounts.end()) {
+            f->second++;
+        } else {
+            nix_refcounts[p] = 1;
+        }
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_gc_decref(nix_c_context * context, const void * p)
+{
+
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        std::scoped_lock lock(nix_refcount_lock);
+        auto f = nix_refcounts.find(p);
+        if (f != nix_refcounts.end()) {
+            if (--f->second == 0)
+                nix_refcounts.erase(f);
+        } else
+            throw std::runtime_error("nix_gc_decref: object was not referenced");
+    }
+    NIXC_CATCH_ERRS
+}
+
+void nix_gc_now()
+{
+    GC_gcollect();
+}
+
+#else
+nix_err nix_gc_incref(nix_c_context * context, const void *)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    return NIX_OK;
+}
+nix_err nix_gc_decref(nix_c_context * context, const void *)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    return NIX_OK;
+}
+void nix_gc_now() {}
+#endif
+
+void nix_gc_register_finalizer(void * obj, void * cd, void (*finalizer)(void * obj, void * cd))
+{
+#ifdef HAVE_BOEHMGC
+    GC_REGISTER_FINALIZER(obj, finalizer, cd, 0, 0);
+#endif
+}

src/libexpr-c/nix_api_expr.h

@@ -0,0 +1,215 @@
+#ifndef NIX_API_EXPR_H
+#define NIX_API_EXPR_H
+/** @defgroup libexpr libexpr
+ * @brief Bindings to the Nix language evaluator
+ *
+ * Example (without error handling):
+ * @code{.c}
+ * int main() {
+ *    nix_libexpr_init(NULL);
+ *
+ *    Store* store = nix_store_open(NULL, "dummy", NULL);
+ *    EvalState* state = nix_state_create(NULL, NULL, store); // empty nix path
+ *    Value *value = nix_alloc_value(NULL, state);
+ *
+ *    nix_expr_eval_from_string(NULL, state, "builtins.nixVersion", ".", value);
+ *    nix_value_force(NULL, state, value);
+ *    printf("nix version: %s\n", nix_get_string(NULL, value));
+ *
+ *    nix_gc_decref(NULL, value);
+ *    nix_state_free(state);
+ *    nix_store_free(store);
+ *    return 0;
+ *    }
+ * @endcode
+ * @{
+ */
+/** @file
+ * @brief Main entry for the libexpr C bindings
+ */
+
+#include "nix_api_store.h"
+#include "nix_api_util.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+// cffi start
+
+// Type definitions
+/**
+ * @brief Represents a state of the Nix language evaluator.
+ *
+ * Multiple states can be created for multi-threaded
+ * operation.
+ * @struct EvalState
+ * @see nix_state_create
+ */
+typedef struct EvalState EvalState; // nix::EvalState
+/**
+ * @brief Represents a value in the Nix language.
+ *
+ * Owned by the garbage collector.
+ * @struct Value
+ * @see value_manip
+ */
+typedef void Value; // nix::Value
+
+// Function prototypes
+/**
+ * @brief Initialize the Nix language evaluator.
+ *
+ * This function must be called at least once,
+ * at some point before constructing a EvalState for the first time.
+ * This function can be called multiple times, and is idempotent.
+ *
+ * @param[out] context Optional, stores error information
+ * @return NIX_OK if the initialization was successful, an error code otherwise.
+ */
+nix_err nix_libexpr_init(nix_c_context * context);
+
+/**
+ * @brief Parses and evaluates a Nix expression from a string.
+ *
+ * @param[out] context Optional, stores error information
+ * @param[in] state The state of the evaluation.
+ * @param[in] expr The Nix expression to parse.
+ * @param[in] path The file path to associate with the expression.
+ * This is required for expressions that contain relative paths (such as `./.`) that are resolved relative to the given
+ * directory.
+ * @param[out] value The result of the evaluation. You must allocate this
+ * yourself.
+ * @return NIX_OK if the evaluation was successful, an error code otherwise.
+ */
+nix_err nix_expr_eval_from_string(
+    nix_c_context * context, EvalState * state, const char * expr, const char * path, Value * value);
+
+/**
+ * @brief Calls a Nix function with an argument.
+ *
+ * @param[out] context Optional, stores error information
+ * @param[in] state The state of the evaluation.
+ * @param[in] fn The Nix function to call.
+ * @param[in] arg The argument to pass to the function.
+ * @param[out] value The result of the function call.
+ * @return NIX_OK if the function call was successful, an error code otherwise.
+ * @see nix_init_apply() for a similar function that does not performs the call immediately, but stores it as a thunk.
+ *      Note the different argument order.
+ */
+nix_err nix_value_call(nix_c_context * context, EvalState * state, Value * fn, Value * arg, Value * value);
+
+/**
+ * @brief Forces the evaluation of a Nix value.
+ *
+ * The Nix interpreter is lazy, and not-yet-evaluated Values can be
+ * of type NIX_TYPE_THUNK instead of their actual value.
+ *
+ * This function converts these Values into their final type.
+ *
+ * @note You don't need this function for basic API usage, since all functions
+ * that return a value call it for you. The only place you will see a
+ * NIX_TYPE_THUNK is in the arguments that are passed to a PrimOp function
+ * you supplied to nix_alloc_primop.
+ *
+ * @param[out] context Optional, stores error information
+ * @param[in] state The state of the evaluation.
+ * @param[in,out] value The Nix value to force.
+ * @post value is not of type NIX_TYPE_THUNK
+ * @return NIX_OK if the force operation was successful, an error code
+ * otherwise.
+ */
+nix_err nix_value_force(nix_c_context * context, EvalState * state, Value * value);
+
+/**
+ * @brief Forces the deep evaluation of a Nix value.
+ *
+ * Recursively calls nix_value_force
+ *
+ * @see nix_value_force
+ * @warning Calling this function on a recursive data structure will cause a
+ * stack overflow.
+ * @param[out] context Optional, stores error information
+ * @param[in] state The state of the evaluation.
+ * @param[in,out] value The Nix value to force.
+ * @return NIX_OK if the deep force operation was successful, an error code
+ * otherwise.
+ */
+nix_err nix_value_force_deep(nix_c_context * context, EvalState * state, Value * value);
+
+/**
+ * @brief Create a new Nix language evaluator state.
+ *
+ * @param[out] context Optional, stores error information
+ * @param[in] lookupPath Array of strings corresponding to entries in NIX_PATH.
+ * @param[in] store The Nix store to use.
+ * @return A new Nix state or NULL on failure.
+ */
+EvalState * nix_state_create(nix_c_context * context, const char ** lookupPath, Store * store);
+
+/**
+ * @brief Frees a Nix state.
+ *
+ * Does not fail.
+ *
+ * @param[in] state The state to free.
+ */
+void nix_state_free(EvalState * state);
+
+/** @addtogroup GC
+ * @brief Reference counting and garbage collector operations
+ *
+ * The Nix language evaluator uses a garbage collector. To ease C interop, we implement
+ * a reference counting scheme, where objects will be deallocated
+ * when there are no references from the Nix side, and the reference count kept
+ * by the C API reaches `0`.
+ *
+ * Functions returning a garbage-collected object will automatically increase
+ * the refcount for you. You should make sure to call `nix_gc_decref` when
+ * you're done with a value returned by the evaluator.
+ * @{
+ */
+/**
+ * @brief Increment the garbage collector reference counter for the given object.
+ *
+ * The Nix language evaluator C API keeps track of alive objects by reference counting.
+ * When you're done with a refcounted pointer, call nix_gc_decref().
+ *
+ * @param[out] context Optional, stores error information
+ * @param[in] object The object to keep alive
+ */
+nix_err nix_gc_incref(nix_c_context * context, const void * object);
+/**
+ * @brief Decrement the garbage collector reference counter for the given object
+ *
+ * @param[out] context Optional, stores error information
+ * @param[in] object The object to stop referencing
+ */
+nix_err nix_gc_decref(nix_c_context * context, const void * object);
+
+/**
+ * @brief Trigger the garbage collector manually
+ *
+ * You should not need to do this, but it can be useful for debugging.
+ */
+void nix_gc_now();
+
+/**
+ * @brief Register a callback that gets called when the object is garbage
+ * collected.
+ * @note Objects can only have a single finalizer. This function overwrites existing values
+ * silently.
+ * @param[in] obj the object to watch
+ * @param[in] cd the data to pass to the finalizer
+ * @param[in] finalizer the callback function, called with obj and cd
+ */
+void nix_gc_register_finalizer(void * obj, void * cd, void (*finalizer)(void * obj, void * cd));
+
+/** @} */
+// cffi end
+#ifdef __cplusplus
+}
+#endif
+
+/** @} */
+
+#endif // NIX_API_EXPR_H

src/libexpr-c/nix_api_expr_internal.h

@@ -0,0 +1,44 @@
+#ifndef NIX_API_EXPR_INTERNAL_H
+#define NIX_API_EXPR_INTERNAL_H
+
+#include "eval.hh"
+#include "attr-set.hh"
+#include "nix_api_value.h"
+
+struct EvalState
+{
+    nix::EvalState state;
+};
+
+struct BindingsBuilder
+{
+    nix::BindingsBuilder builder;
+};
+
+struct ListBuilder
+{
+    nix::ListBuilder builder;
+};
+
+struct nix_string_return
+{
+    std::string str;
+};
+
+struct nix_printer
+{
+    std::ostream & s;
+};
+
+struct nix_string_context
+{
+    nix::NixStringContext & ctx;
+};
+
+struct nix_realised_string
+{
+    std::string str;
+    std::vector<StorePath> storePaths;
+};
+
+#endif // NIX_API_EXPR_INTERNAL_H

src/libexpr-c/nix_api_external.cc

@@ -0,0 +1,198 @@
+#include "attr-set.hh"
+#include "config.hh"
+#include "eval.hh"
+#include "globals.hh"
+#include "value.hh"
+
+#include "nix_api_expr.h"
+#include "nix_api_expr_internal.h"
+#include "nix_api_external.h"
+#include "nix_api_util.h"
+#include "nix_api_util_internal.h"
+#include "nix_api_value.h"
+#include "value/context.hh"
+
+#include <nlohmann/json.hpp>
+
+#ifdef HAVE_BOEHMGC
+# include "gc/gc.h"
+# define GC_INCLUDE_NEW 1
+# include "gc_cpp.h"
+#endif
+
+void nix_set_string_return(nix_string_return * str, const char * c)
+{
+    str->str = c;
+}
+
+nix_err nix_external_print(nix_c_context * context, nix_printer * printer, const char * c)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        printer->s << c;
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_external_add_string_context(nix_c_context * context, nix_string_context * ctx, const char * c)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto r = nix::NixStringContextElem::parse(c);
+        ctx->ctx.insert(r);
+    }
+    NIXC_CATCH_ERRS
+}
+
+class NixCExternalValue : public nix::ExternalValueBase
+{
+    NixCExternalValueDesc & desc;
+    void * v;
+
+public:
+    NixCExternalValue(NixCExternalValueDesc & desc, void * v)
+        : desc(desc)
+        , v(v){};
+    void * get_ptr()
+    {
+        return v;
+    }
+    /**
+     * Print out the value
+     */
+    virtual std::ostream & print(std::ostream & str) const override
+    {
+        nix_printer p{str};
+        desc.print(v, &p);
+        return str;
+    }
+
+    /**
+     * Return a simple string describing the type
+     */
+    virtual std::string showType() const override
+    {
+        nix_string_return res;
+        desc.showType(v, &res);
+        return std::move(res.str);
+    }
+
+    /**
+     * Return a string to be used in builtins.typeOf
+     */
+    virtual std::string typeOf() const override
+    {
+        nix_string_return res;
+        desc.typeOf(v, &res);
+        return std::move(res.str);
+    }
+
+    /**
+     * Coerce the value to a string.
+     */
+    virtual std::string coerceToString(
+        nix::EvalState & state,
+        const nix::PosIdx & pos,
+        nix::NixStringContext & context,
+        bool copyMore,
+        bool copyToStore) const override
+    {
+        if (!desc.coerceToString) {
+            return nix::ExternalValueBase::coerceToString(state, pos, context, copyMore, copyToStore);
+        }
+        nix_string_context ctx{context};
+        nix_string_return res{""};
+        // todo: pos, errors
+        desc.coerceToString(v, &ctx, copyMore, copyToStore, &res);
+        if (res.str.empty()) {
+            return nix::ExternalValueBase::coerceToString(state, pos, context, copyMore, copyToStore);
+        }
+        return std::move(res.str);
+    }
+
+    /**
+     * Compare to another value of the same type.
+     */
+    virtual bool operator==(const ExternalValueBase & b) const override
+    {
+        if (!desc.equal) {
+            return false;
+        }
+        auto r = dynamic_cast<const NixCExternalValue *>(&b);
+        if (!r)
+            return false;
+        return desc.equal(v, r->v);
+    }
+
+    /**
+     * Print the value as JSON.
+     */
+    virtual nlohmann::json printValueAsJSON(
+        nix::EvalState & state, bool strict, nix::NixStringContext & context, bool copyToStore = true) const override
+    {
+        if (!desc.printValueAsJSON) {
+            return nix::ExternalValueBase::printValueAsJSON(state, strict, context, copyToStore);
+        }
+        nix_string_context ctx{context};
+        nix_string_return res{""};
+        desc.printValueAsJSON(v, (EvalState *) &state, strict, &ctx, copyToStore, &res);
+        if (res.str.empty()) {
+            return nix::ExternalValueBase::printValueAsJSON(state, strict, context, copyToStore);
+        }
+        return nlohmann::json::parse(res.str);
+    }
+
+    /**
+     * Print the value as XML.
+     */
+    virtual void printValueAsXML(
+        nix::EvalState & state,
+        bool strict,
+        bool location,
+        nix::XMLWriter & doc,
+        nix::NixStringContext & context,
+        nix::PathSet & drvsSeen,
+        const nix::PosIdx pos) const override
+    {
+        if (!desc.printValueAsXML) {
+            return nix::ExternalValueBase::printValueAsXML(state, strict, location, doc, context, drvsSeen, pos);
+        }
+        nix_string_context ctx{context};
+        desc.printValueAsXML(
+            v, (EvalState *) &state, strict, location, &doc, &ctx, &drvsSeen,
+            *reinterpret_cast<const uint32_t *>(&pos));
+    }
+
+    virtual ~NixCExternalValue() override{};
+};
+
+ExternalValue * nix_create_external_value(nix_c_context * context, NixCExternalValueDesc * desc, void * v)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto ret = new
+#ifdef HAVE_BOEHMGC
+            (GC)
+#endif
+                NixCExternalValue(*desc, v);
+        nix_gc_incref(nullptr, ret);
+        return (ExternalValue *) ret;
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+void * nix_get_external_value_content(nix_c_context * context, ExternalValue * b)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto r = dynamic_cast<NixCExternalValue *>((nix::ExternalValueBase *) b);
+        if (r)
+            return r->get_ptr();
+        return nullptr;
+    }
+    NIXC_CATCH_ERRS_NULL
+}

src/libexpr-c/nix_api_external.h

@@ -0,0 +1,196 @@
+#ifndef NIX_API_EXTERNAL_H
+#define NIX_API_EXTERNAL_H
+/** @ingroup libexpr
+ * @addtogroup Externals
+ * @brief Deal with external values
+ * @{
+ */
+/** @file
+ * @brief libexpr C bindings dealing with external values
+ */
+
+#include "nix_api_expr.h"
+#include "nix_api_util.h"
+#include "nix_api_value.h"
+#include "stdbool.h"
+#include "stddef.h"
+#include "stdint.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+// cffi start
+
+/**
+ * @brief Represents a string owned by the Nix language evaluator.
+ * @see nix_set_owned_string
+ */
+typedef struct nix_string_return nix_string_return;
+/**
+ * @brief Wraps a stream that can output multiple string pieces.
+ */
+typedef struct nix_printer nix_printer;
+/**
+ * @brief A list of string context items
+ */
+typedef struct nix_string_context nix_string_context;
+
+/**
+ * @brief Sets the contents of a nix_string_return
+ *
+ * Copies the passed string.
+ * @param[out] str the nix_string_return to write to
+ * @param[in]  c   The string to copy
+ */
+void nix_set_string_return(nix_string_return * str, const char * c);
+
+/**
+ * Print to the nix_printer
+ *
+ * @param[out] context Optional, stores error information
+ * @param printer The nix_printer to print to
+ * @param[in] str The string to print
+ * @returns NIX_OK if everything worked
+ */
+nix_err nix_external_print(nix_c_context * context, nix_printer * printer, const char * str);
+
+/**
+ * Add string context to the nix_string_context object
+ * @param[out] context Optional, stores error information
+ * @param[out] string_context The nix_string_context to add to
+ * @param[in] c The context string to add
+ * @returns NIX_OK if everything worked
+ */
+nix_err nix_external_add_string_context(nix_c_context * context, nix_string_context * string_context, const char * c);
+
+/**
+ * @brief Definition for a class of external values
+ *
+ * Create and implement one of these, then pass it to nix_create_external_value
+ * Make sure to keep it alive while the external value lives.
+ *
+ * Optional functions can be set to NULL
+ *
+ * @see nix_create_external_value
+ */
+typedef struct NixCExternalValueDesc
+{
+    /**
+     * @brief Called when printing the external value
+     *
+     * @param[in] self the void* passed to nix_create_external_value
+     * @param[out] printer The printer to print to, pass to nix_external_print
+     */
+    void (*print)(void * self, nix_printer * printer);
+    /**
+     * @brief Called on :t
+     * @param[in] self the void* passed to nix_create_external_value
+     * @param[out] res the return value
+     */
+    void (*showType)(void * self, nix_string_return * res);
+    /**
+     * @brief Called on `builtins.typeOf`
+     * @param self the void* passed to nix_create_external_value
+     * @param[out] res the return value
+     */
+    void (*typeOf)(void * self, nix_string_return * res);
+    /**
+     * @brief Called on "${str}" and builtins.toString.
+     *
+     * The latter with coerceMore=true
+     * Optional, the default is to throw an error.
+     * @param[in] self the void* passed to nix_create_external_value
+     * @param[out] c writable string context for the resulting string
+     * @param[in] coerceMore boolean, try to coerce to strings in more cases
+     * instead of throwing an error
+     * @param[in] copyToStore boolean, whether to copy referenced paths to store
+     * or keep them as-is
+     * @param[out] res the return value. Not touching this, or setting it to the
+     * empty string, will make the conversion throw an error.
+     */
+    void (*coerceToString)(
+        void * self, nix_string_context * c, int coerceMore, int copyToStore, nix_string_return * res);
+    /**
+     * @brief Try to compare two external values
+     *
+     * Optional, the default is always false.
+     * If the other object was not a Nix C external value, this comparison will
+     * also return false
+     * @param[in] self the void* passed to nix_create_external_value
+     * @param[in] other the void* passed to the other object's
+     * nix_create_external_value
+     * @returns true if the objects are deemed to be equal
+     */
+    int (*equal)(void * self, void * other);
+    /**
+     * @brief Convert the external value to json
+     *
+     * Optional, the default is to throw an error
+     * @param[in] self the void* passed to nix_create_external_value
+     * @param[in] state The evaluator state
+     * @param[in] strict boolean Whether to force the value before printing
+     * @param[out] c writable string context for the resulting string
+     * @param[in] copyToStore whether to copy referenced paths to store or keep
+     * them as-is
+     * @param[out] res the return value. Gets parsed as JSON. Not touching this,
+     * or setting it to the empty string, will make the conversion throw an error.
+     */
+    void (*printValueAsJSON)(
+        void * self, EvalState *, bool strict, nix_string_context * c, bool copyToStore, nix_string_return * res);
+    /**
+     * @brief Convert the external value to XML
+     *
+     * Optional, the default is to throw an error
+     * @todo The mechanisms for this call are incomplete. There are no C
+     *       bindings to work with XML, pathsets and positions.
+     * @param[in] self the void* passed to nix_create_external_value
+     * @param[in] state The evaluator state
+     * @param[in] strict boolean Whether to force the value before printing
+     * @param[in] location boolean Whether to include position information in the
+     * xml
+     * @param[out] doc XML document to output to
+     * @param[out] c writable string context for the resulting string
+     * @param[in,out] drvsSeen a path set to avoid duplicating derivations
+     * @param[in] pos The position of the call.
+     */
+    void (*printValueAsXML)(
+        void * self,
+        EvalState *,
+        int strict,
+        int location,
+        void * doc,
+        nix_string_context * c,
+        void * drvsSeen,
+        int pos);
+} NixCExternalValueDesc;
+
+/**
+ * @brief Create an external value, that can be given to nix_init_external
+ *
+ * Owned by the GC. Use nix_gc_decref when you're done with the pointer.
+ *
+ * @param[out] context Optional, stores error information
+ * @param[in] desc a NixCExternalValueDesc, you should keep this alive as long
+ * as the ExternalValue lives
+ * @param[in] v the value to store
+ * @returns external value, owned by the garbage collector
+ * @see nix_init_external
+ */
+ExternalValue * nix_create_external_value(nix_c_context * context, NixCExternalValueDesc * desc, void * v);
+
+/**
+ * @brief Extract the pointer from a nix c external value.
+ * @param[out] context Optional, stores error information
+ * @param[in] b The external value
+ * @returns The pointer, or null if the external value was not from nix c.
+ * @see nix_get_external
+ */
+void * nix_get_external_value_content(nix_c_context * context, ExternalValue * b);
+
+// cffi end
+#ifdef __cplusplus
+}
+#endif
+/** @} */
+
+#endif // NIX_API_EXTERNAL_H

src/libexpr-c/nix_api_value.cc

@@ -0,0 +1,595 @@
+#include "attr-set.hh"
+#include "config.hh"
+#include "eval.hh"
+#include "globals.hh"
+#include "path.hh"
+#include "primops.hh"
+#include "value.hh"
+
+#include "nix_api_expr.h"
+#include "nix_api_expr_internal.h"
+#include "nix_api_util.h"
+#include "nix_api_util_internal.h"
+#include "nix_api_store_internal.h"
+#include "nix_api_value.h"
+#include "value/context.hh"
+
+#ifdef HAVE_BOEHMGC
+#  include "gc/gc.h"
+#  define GC_INCLUDE_NEW 1
+#  include "gc_cpp.h"
+#endif
+
+// Helper function to throw an exception if value is null
+static const nix::Value & check_value_not_null(const Value * value)
+{
+    if (!value) {
+        throw std::runtime_error("Value is null");
+    }
+    return *((const nix::Value *) value);
+}
+
+static nix::Value & check_value_not_null(Value * value)
+{
+    if (!value) {
+        throw std::runtime_error("Value is null");
+    }
+    return *((nix::Value *) value);
+}
+
+/**
+ * Helper function to convert calls from nix into C API.
+ *
+ * Deals with errors and converts arguments from C++ into C types.
+ */
+static void nix_c_primop_wrapper(
+    PrimOpFun f, void * userdata, nix::EvalState & state, const nix::PosIdx pos, nix::Value ** args, nix::Value & v)
+{
+    nix_c_context ctx;
+    f(userdata, &ctx, (EvalState *) &state, (Value **) args, (Value *) &v);
+    /* TODO: In the future, this should throw different errors depending on the error code */
+    if (ctx.last_err_code != NIX_OK)
+        state.error<nix::EvalError>("Error from builtin function: %s", *ctx.last_err).atPos(pos).debugThrow();
+}
+
+PrimOp * nix_alloc_primop(
+    nix_c_context * context,
+    PrimOpFun fun,
+    int arity,
+    const char * name,
+    const char ** args,
+    const char * doc,
+    void * user_data)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        using namespace std::placeholders;
+        auto p = new
+#ifdef HAVE_BOEHMGC
+            (GC)
+#endif
+                nix::PrimOp{
+                    .name = name,
+                    .args = {},
+                    .arity = (size_t) arity,
+                    .doc = doc,
+                    .fun = std::bind(nix_c_primop_wrapper, fun, user_data, _1, _2, _3, _4)};
+        if (args)
+            for (size_t i = 0; args[i]; i++)
+                p->args.emplace_back(*args);
+        nix_gc_incref(nullptr, p);
+        return (PrimOp *) p;
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+nix_err nix_register_primop(nix_c_context * context, PrimOp * primOp)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        nix::RegisterPrimOp r(std::move(*((nix::PrimOp *) primOp)));
+    }
+    NIXC_CATCH_ERRS
+}
+
+Value * nix_alloc_value(nix_c_context * context, EvalState * state)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        Value * res = state->state.allocValue();
+        nix_gc_incref(nullptr, res);
+        return res;
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+ValueType nix_get_type(nix_c_context * context, const Value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        using namespace nix;
+        switch (v.type()) {
+        case nThunk:
+            return NIX_TYPE_THUNK;
+        case nInt:
+            return NIX_TYPE_INT;
+        case nFloat:
+            return NIX_TYPE_FLOAT;
+        case nBool:
+            return NIX_TYPE_BOOL;
+        case nString:
+            return NIX_TYPE_STRING;
+        case nPath:
+            return NIX_TYPE_PATH;
+        case nNull:
+            return NIX_TYPE_NULL;
+        case nAttrs:
+            return NIX_TYPE_ATTRS;
+        case nList:
+            return NIX_TYPE_LIST;
+        case nFunction:
+            return NIX_TYPE_FUNCTION;
+        case nExternal:
+            return NIX_TYPE_EXTERNAL;
+        }
+        return NIX_TYPE_NULL;
+    }
+    NIXC_CATCH_ERRS_RES(NIX_TYPE_NULL);
+}
+
+const char * nix_get_typename(nix_c_context * context, const Value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        auto s = nix::showType(v);
+        return strdup(s.c_str());
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+bool nix_get_bool(nix_c_context * context, const Value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        assert(v.type() == nix::nBool);
+        return v.boolean();
+    }
+    NIXC_CATCH_ERRS_RES(false);
+}
+
+nix_err nix_get_string(nix_c_context * context, const Value * value, nix_get_string_callback callback, void * user_data)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        assert(v.type() == nix::nString);
+        call_nix_get_string_callback(v.c_str(), callback, user_data);
+    }
+    NIXC_CATCH_ERRS
+}
+
+const char * nix_get_path_string(nix_c_context * context, const Value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        assert(v.type() == nix::nPath);
+        // NOTE (from @yorickvP)
+        // v._path.path should work but may not be how Eelco intended it.
+        // Long-term this function should be rewritten to copy some data into a
+        // user-allocated string.
+        // We could use v.path().to_string().c_str(), but I'm concerned this
+        // crashes. Looks like .path() allocates a CanonPath with a copy of the
+        // string, then it gets the underlying data from that.
+        return v.payload.path.path;
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+unsigned int nix_get_list_size(nix_c_context * context, const Value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        assert(v.type() == nix::nList);
+        return v.listSize();
+    }
+    NIXC_CATCH_ERRS_RES(0);
+}
+
+unsigned int nix_get_attrs_size(nix_c_context * context, const Value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        assert(v.type() == nix::nAttrs);
+        return v.attrs()->size();
+    }
+    NIXC_CATCH_ERRS_RES(0);
+}
+
+double nix_get_float(nix_c_context * context, const Value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        assert(v.type() == nix::nFloat);
+        return v.fpoint();
+    }
+    NIXC_CATCH_ERRS_RES(0.0);
+}
+
+int64_t nix_get_int(nix_c_context * context, const Value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        assert(v.type() == nix::nInt);
+        return v.integer();
+    }
+    NIXC_CATCH_ERRS_RES(0);
+}
+
+ExternalValue * nix_get_external(nix_c_context * context, Value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        assert(v.type() == nix::nExternal);
+        return (ExternalValue *) v.external();
+    }
+    NIXC_CATCH_ERRS_NULL;
+}
+
+Value * nix_get_list_byidx(nix_c_context * context, const Value * value, EvalState * state, unsigned int ix)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        assert(v.type() == nix::nList);
+        auto * p = v.listElems()[ix];
+        nix_gc_incref(nullptr, p);
+        if (p != nullptr)
+            state->state.forceValue(*p, nix::noPos);
+        return (Value *) p;
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+Value * nix_get_attr_byname(nix_c_context * context, const Value * value, EvalState * state, const char * name)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        assert(v.type() == nix::nAttrs);
+        nix::Symbol s = state->state.symbols.create(name);
+        auto attr = v.attrs()->get(s);
+        if (attr) {
+            nix_gc_incref(nullptr, attr->value);
+            state->state.forceValue(*attr->value, nix::noPos);
+            return attr->value;
+        }
+        nix_set_err_msg(context, NIX_ERR_KEY, "missing attribute");
+        return nullptr;
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+bool nix_has_attr_byname(nix_c_context * context, const Value * value, EvalState * state, const char * name)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        assert(v.type() == nix::nAttrs);
+        nix::Symbol s = state->state.symbols.create(name);
+        auto attr = v.attrs()->get(s);
+        if (attr)
+            return true;
+        return false;
+    }
+    NIXC_CATCH_ERRS_RES(false);
+}
+
+Value *
+nix_get_attr_byidx(nix_c_context * context, const Value * value, EvalState * state, unsigned int i, const char ** name)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        const nix::Attr & a = (*v.attrs())[i];
+        *name = ((const std::string &) (state->state.symbols[a.name])).c_str();
+        nix_gc_incref(nullptr, a.value);
+        state->state.forceValue(*a.value, nix::noPos);
+        return a.value;
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+const char * nix_get_attr_name_byidx(nix_c_context * context, const Value * value, EvalState * state, unsigned int i)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        const nix::Attr & a = (*v.attrs())[i];
+        return ((const std::string &) (state->state.symbols[a.name])).c_str();
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+nix_err nix_init_bool(nix_c_context * context, Value * value, bool b)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        v.mkBool(b);
+    }
+    NIXC_CATCH_ERRS
+}
+
+// todo string context
+nix_err nix_init_string(nix_c_context * context, Value * value, const char * str)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        v.mkString(std::string_view(str));
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_init_path_string(nix_c_context * context, EvalState * s, Value * value, const char * str)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        v.mkPath(s->state.rootPath(nix::CanonPath(str)));
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_init_float(nix_c_context * context, Value * value, double d)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        v.mkFloat(d);
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_init_int(nix_c_context * context, Value * value, int64_t i)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        v.mkInt(i);
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_init_null(nix_c_context * context, Value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        v.mkNull();
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_init_apply(nix_c_context * context, Value * value, Value * fn, Value * arg)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        auto & f = check_value_not_null(fn);
+        auto & a = check_value_not_null(arg);
+        v.mkApp(&f, &a);
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_init_external(nix_c_context * context, Value * value, ExternalValue * val)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        auto r = (nix::ExternalValueBase *) val;
+        v.mkExternal(r);
+    }
+    NIXC_CATCH_ERRS
+}
+
+ListBuilder * nix_make_list_builder(nix_c_context * context, EvalState * state, size_t capacity)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto builder = state->state.buildList(capacity);
+        return new
+#if HAVE_BOEHMGC
+            (NoGC)
+#endif
+                ListBuilder{std::move(builder)};
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+nix_err nix_list_builder_insert(nix_c_context * context, ListBuilder * list_builder, unsigned int index, Value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & e = check_value_not_null(value);
+        list_builder->builder[index] = &e;
+    }
+    NIXC_CATCH_ERRS
+}
+
+void nix_list_builder_free(ListBuilder * list_builder)
+{
+#if HAVE_BOEHMGC
+    GC_FREE(list_builder);
+#else
+    delete list_builder;
+#endif
+}
+
+nix_err nix_make_list(nix_c_context * context, ListBuilder * list_builder, Value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        v.mkList(list_builder->builder);
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_init_primop(nix_c_context * context, Value * value, PrimOp * p)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        v.mkPrimOp((nix::PrimOp *) p);
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_copy_value(nix_c_context * context, Value * value, Value * source)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        auto & s = check_value_not_null(source);
+        v = s;
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_make_attrs(nix_c_context * context, Value * value, BindingsBuilder * b)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        v.mkAttrs(b->builder);
+    }
+    NIXC_CATCH_ERRS
+}
+
+BindingsBuilder * nix_make_bindings_builder(nix_c_context * context, EvalState * state, size_t capacity)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto bb = state->state.buildBindings(capacity);
+        return new
+#if HAVE_BOEHMGC
+            (NoGC)
+#endif
+                BindingsBuilder{std::move(bb)};
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+nix_err nix_bindings_builder_insert(nix_c_context * context, BindingsBuilder * bb, const char * name, Value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        nix::Symbol s = bb->builder.state.symbols.create(name);
+        bb->builder.insert(s, &v);
+    }
+    NIXC_CATCH_ERRS
+}
+
+void nix_bindings_builder_free(BindingsBuilder * bb)
+{
+#if HAVE_BOEHMGC
+    GC_FREE((nix::BindingsBuilder *) bb);
+#else
+    delete (nix::BindingsBuilder *) bb;
+#endif
+}
+
+nix_realised_string * nix_string_realise(nix_c_context * context, EvalState * state, Value * value, bool isIFD)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        nix::NixStringContext stringContext;
+        auto rawStr = state->state.coerceToString(nix::noPos, v, stringContext, "while realising a string").toOwned();
+        nix::StorePathSet storePaths;
+        auto rewrites = state->state.realiseContext(stringContext, &storePaths);
+
+        auto s = nix::rewriteStrings(rawStr, rewrites);
+
+        // Convert to the C API StorePath type and convert to vector for index-based access
+        std::vector<StorePath> vec;
+        for (auto & sp : storePaths) {
+            vec.push_back(StorePath{sp});
+        }
+
+        return new nix_realised_string{.str = s, .storePaths = vec};
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+void nix_realised_string_free(nix_realised_string * s)
+{
+    delete s;
+}
+
+size_t nix_realised_string_get_buffer_size(nix_realised_string * s)
+{
+    return s->str.size();
+}
+
+const char * nix_realised_string_get_buffer_start(nix_realised_string * s)
+{
+    return s->str.data();
+}
+
+size_t nix_realised_string_get_store_path_count(nix_realised_string * s)
+{
+    return s->storePaths.size();
+}
+
+const StorePath * nix_realised_string_get_store_path(nix_realised_string * s, size_t i)
+{
+    return &s->storePaths[i];
+}

src/libexpr-c/nix_api_value.h

@@ -0,0 +1,510 @@
+#ifndef NIX_API_VALUE_H
+#define NIX_API_VALUE_H
+
+/** @addtogroup libexpr
+ * @{
+ */
+/** @file
+ * @brief libexpr C bindings dealing with values
+ */
+
+#include "nix_api_util.h"
+#include "nix_api_store.h"
+#include "stdbool.h"
+#include "stddef.h"
+#include "stdint.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+// cffi start
+
+// Type definitions
+typedef enum {
+    NIX_TYPE_THUNK,
+    NIX_TYPE_INT,
+    NIX_TYPE_FLOAT,
+    NIX_TYPE_BOOL,
+    NIX_TYPE_STRING,
+    NIX_TYPE_PATH,
+    NIX_TYPE_NULL,
+    NIX_TYPE_ATTRS,
+    NIX_TYPE_LIST,
+    NIX_TYPE_FUNCTION,
+    NIX_TYPE_EXTERNAL
+} ValueType;
+
+// forward declarations
+typedef void Value;
+typedef struct EvalState EvalState;
+// type defs
+/** @brief Stores an under-construction set of bindings
+ * @ingroup value_manip
+ *
+ * Do not reuse.
+ * @see nix_make_bindings_builder, nix_bindings_builder_free, nix_make_attrs
+ * @see nix_bindings_builder_insert
+ */
+typedef struct BindingsBuilder BindingsBuilder;
+
+/** @brief Stores an under-construction list
+ * @ingroup value_manip
+ *
+ * Do not reuse.
+ * @see nix_make_list_builder, nix_list_builder_free, nix_make_list
+ * @see nix_list_builder_insert
+ */
+typedef struct ListBuilder ListBuilder;
+
+/** @brief PrimOp function
+ * @ingroup primops
+ *
+ * Owned by the GC
+ * @see nix_alloc_primop, nix_init_primop
+ */
+typedef struct PrimOp PrimOp;
+/** @brief External Value
+ * @ingroup Externals
+ *
+ * Owned by the GC
+ */
+typedef struct ExternalValue ExternalValue;
+
+/** @brief String without placeholders, and realised store paths
+ */
+typedef struct nix_realised_string nix_realised_string;
+
+/** @defgroup primops
+ * @brief Create your own primops
+ * @{
+ */
+/** @brief Function pointer for primops
+ * When you want to return an error, call nix_set_err_msg(context, NIX_ERR_UNKNOWN, "your error message here").
+ *
+ * @param[in] user_data Arbitrary data that was initially supplied to nix_alloc_primop
+ * @param[out] context Stores error information.
+ * @param[in] state Evaluator state
+ * @param[in] args list of arguments. Note that these can be thunks and should be forced using nix_value_force before
+ * use.
+ * @param[out] ret return value
+ * @see nix_alloc_primop, nix_init_primop
+ */
+typedef void (*PrimOpFun)(void * user_data, nix_c_context * context, EvalState * state, Value ** args, Value * ret);
+
+/** @brief Allocate a PrimOp
+ *
+ * Owned by the garbage collector.
+ * Use nix_gc_decref() when you're done with the returned PrimOp.
+ *
+ * @param[out] context Optional, stores error information
+ * @param[in] fun callback
+ * @param[in] arity expected number of function arguments
+ * @param[in] name function name
+ * @param[in] args array of argument names, NULL-terminated
+ * @param[in] doc optional, documentation for this primop
+ * @param[in] user_data optional, arbitrary data, passed to the callback when it's called
+ * @return primop, or null in case of errors
+ * @see nix_init_primop
+ */
+PrimOp * nix_alloc_primop(
+    nix_c_context * context,
+    PrimOpFun fun,
+    int arity,
+    const char * name,
+    const char ** args,
+    const char * doc,
+    void * user_data);
+
+/** @brief add a primop to the `builtins` attribute set
+ *
+ * Only applies to States created after this call.
+ *
+ * Moves your PrimOp content into the global evaluator
+ * registry, meaning your input PrimOp pointer is no longer usable.
+ * You are free to remove your references to it,
+ * after which it will be garbage collected.
+ *
+ * @param[out] context Optional, stores error information
+ * @return primop, or null in case of errors
+ *
+ */
+nix_err nix_register_primop(nix_c_context * context, PrimOp * primOp);
+/** @} */
+
+// Function prototypes
+
+/** @brief Allocate a Nix value
+ *
+ * Owned by the GC. Use nix_gc_decref() when you're done with the pointer
+ * @param[out] context Optional, stores error information
+ * @param[in] state nix evaluator state
+ * @return value, or null in case of errors
+ *
+ */
+Value * nix_alloc_value(nix_c_context * context, EvalState * state);
+
+/** @addtogroup value_manip Manipulating values
+ * @brief Functions to inspect and change Nix language values, represented by Value.
+ * @{
+ */
+/** @name Getters
+ */
+/**@{*/
+/** @brief Get value type
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect
+ * @return type of nix value
+ */
+ValueType nix_get_type(nix_c_context * context, const Value * value);
+
+/** @brief Get type name of value as defined in the evaluator
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect
+ * @return type name, owned string
+ * @todo way to free the result
+ */
+const char * nix_get_typename(nix_c_context * context, const Value * value);
+
+/** @brief Get boolean value
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect
+ * @return true or false, error info via context
+ */
+bool nix_get_bool(nix_c_context * context, const Value * value);
+
+/** @brief Get the raw string
+ *
+ * This may contain placeholders.
+ *
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect
+ * @param[in] callback Called with the string value.
+ * @param[in] user_data optional, arbitrary data, passed to the callback when it's called.
+ * @return string
+ * @return error code, NIX_OK on success.
+ */
+nix_err
+nix_get_string(nix_c_context * context, const Value * value, nix_get_string_callback callback, void * user_data);
+
+/** @brief Get path as string
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect
+ * @return string
+ * @return NULL in case of error.
+ */
+const char * nix_get_path_string(nix_c_context * context, const Value * value);
+
+/** @brief Get the length of a list
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect
+ * @return length of list, error info via context
+ */
+unsigned int nix_get_list_size(nix_c_context * context, const Value * value);
+
+/** @brief Get the element count of an attrset
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect
+ * @return attrset element count, error info via context
+ */
+unsigned int nix_get_attrs_size(nix_c_context * context, const Value * value);
+
+/** @brief Get float value in 64 bits
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect
+ * @return float contents, error info via context
+ */
+double nix_get_float(nix_c_context * context, const Value * value);
+
+/** @brief Get int value
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect
+ * @return int contents, error info via context
+ */
+int64_t nix_get_int(nix_c_context * context, const Value * value);
+
+/** @brief Get external reference
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect
+ * @return reference to external, NULL in case of error
+ */
+ExternalValue * nix_get_external(nix_c_context * context, Value *);
+
+/** @brief Get the ix'th element of a list
+ *
+ * Owned by the GC. Use nix_gc_decref when you're done with the pointer
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect
+ * @param[in] state nix evaluator state
+ * @param[in] ix list element to get
+ * @return value, NULL in case of errors
+ */
+Value * nix_get_list_byidx(nix_c_context * context, const Value * value, EvalState * state, unsigned int ix);
+
+/** @brief Get an attr by name
+ *
+ * Owned by the GC. Use nix_gc_decref when you're done with the pointer
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect
+ * @param[in] state nix evaluator state
+ * @param[in] name attribute name
+ * @return value, NULL in case of errors
+ */
+Value * nix_get_attr_byname(nix_c_context * context, const Value * value, EvalState * state, const char * name);
+
+/** @brief Check if an attribute name exists on a value
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect
+ * @param[in] state nix evaluator state
+ * @param[in] name attribute name
+ * @return value, error info via context
+ */
+bool nix_has_attr_byname(nix_c_context * context, const Value * value, EvalState * state, const char * name);
+
+/** @brief Get an attribute by index in the sorted bindings
+ *
+ * Also gives you the name.
+ *
+ * Owned by the GC. Use nix_gc_decref when you're done with the pointer
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect
+ * @param[in] state nix evaluator state
+ * @param[in] i attribute index
+ * @param[out] name will store a pointer to the attribute name
+ * @return value, NULL in case of errors
+ */
+Value *
+nix_get_attr_byidx(nix_c_context * context, const Value * value, EvalState * state, unsigned int i, const char ** name);
+
+/** @brief Get an attribute name by index in the sorted bindings
+ *
+ * Useful when you want the name but want to avoid evaluation.
+ *
+ * Owned by the nix EvalState
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect
+ * @param[in] state nix evaluator state
+ * @param[in] i attribute index
+ * @return name, NULL in case of errors
+ */
+const char * nix_get_attr_name_byidx(nix_c_context * context, const Value * value, EvalState * state, unsigned int i);
+
+/**@}*/
+/** @name Initializers
+ *
+ * Values are typically "returned" by initializing already allocated memory that serves as the return value.
+ * For this reason, the construction of values is not tied their allocation.
+ * Nix is a language with immutable values. Respect this property by only initializing Values once; and only initialize
+ * Values that are meant to be initialized by you. Failing to adhere to these rules may lead to undefined behavior.
+ */
+/**@{*/
+/** @brief Set boolean value
+ * @param[out] context Optional, stores error information
+ * @param[out] value Nix value to modify
+ * @param[in] b the boolean value
+ * @return error code, NIX_OK on success.
+ */
+nix_err nix_init_bool(nix_c_context * context, Value * value, bool b);
+
+/** @brief Set a string
+ * @param[out] context Optional, stores error information
+ * @param[out] value Nix value to modify
+ * @param[in] str the string, copied
+ * @return error code, NIX_OK on success.
+ */
+nix_err nix_init_string(nix_c_context * context, Value * value, const char * str);
+
+/** @brief Set a path
+ * @param[out] context Optional, stores error information
+ * @param[out] value Nix value to modify
+ * @param[in] str the path string, copied
+ * @return error code, NIX_OK on success.
+ */
+nix_err nix_init_path_string(nix_c_context * context, EvalState * s, Value * value, const char * str);
+
+/** @brief Set a float
+ * @param[out] context Optional, stores error information
+ * @param[out] value Nix value to modify
+ * @param[in] d the float, 64-bits
+ * @return error code, NIX_OK on success.
+ */
+nix_err nix_init_float(nix_c_context * context, Value * value, double d);
+
+/** @brief Set an int
+ * @param[out] context Optional, stores error information
+ * @param[out] value Nix value to modify
+ * @param[in] i the int
+ * @return error code, NIX_OK on success.
+ */
+
+nix_err nix_init_int(nix_c_context * context, Value * value, int64_t i);
+/** @brief Set null
+ * @param[out] context Optional, stores error information
+ * @param[out] value Nix value to modify
+ * @return error code, NIX_OK on success.
+ */
+nix_err nix_init_null(nix_c_context * context, Value * value);
+
+/** @brief Set the value to a thunk that will perform a function application when needed.
+ *
+ * Thunks may be put into attribute sets and lists to perform some computation lazily; on demand.
+ * However, note that in some places, a thunk must not be returned, such as in the return value of a PrimOp.
+ * In such cases, you may use nix_value_call() instead (but note the different argument order).
+ *
+ * @param[out] context Optional, stores error information
+ * @param[out] value Nix value to modify
+ * @param[in] fn function to call
+ * @param[in] arg argument to pass
+ * @return error code, NIX_OK on successful initialization.
+ * @see nix_value_call() for a similar function that performs the call immediately and only stores the return value.
+ *      Note the different argument order.
+ */
+nix_err nix_init_apply(nix_c_context * context, Value * value, Value * fn, Value * arg);
+
+/** @brief Set an external value
+ * @param[out] context Optional, stores error information
+ * @param[out] value Nix value to modify
+ * @param[in] val the external value to set. Will be GC-referenced by the value.
+ * @return error code, NIX_OK on success.
+ */
+nix_err nix_init_external(nix_c_context * context, Value * value, ExternalValue * val);
+
+/** @brief Create a list from a list builder
+ * @param[out] context Optional, stores error information
+ * @param[in] list_builder list builder to use. Make sure to unref this afterwards.
+ * @param[out] value Nix value to modify
+ * @return error code, NIX_OK on success.
+ */
+nix_err nix_make_list(nix_c_context * context, ListBuilder * list_builder, Value * value);
+
+/** @brief Create a list builder
+ * @param[out] context Optional, stores error information
+ * @param[in] state nix evaluator state
+ * @param[in] capacity how many bindings you'll add. Don't exceed.
+ * @return owned reference to a list builder. Make sure to unref when you're done.
+ */
+ListBuilder * nix_make_list_builder(nix_c_context * context, EvalState * state, size_t capacity);
+
+/** @brief Insert bindings into a builder
+ * @param[out] context Optional, stores error information
+ * @param[in] list_builder ListBuilder to insert into
+ * @param[in] index index to manipulate
+ * @param[in] value value to insert
+ * @return error code, NIX_OK on success.
+ */
+nix_err nix_list_builder_insert(nix_c_context * context, ListBuilder * list_builder, unsigned int index, Value * value);
+
+/** @brief Free a list builder
+ *
+ * Does not fail.
+ * @param[in] builder the builder to free
+ */
+void nix_list_builder_free(ListBuilder * list_builder);
+
+/** @brief Create an attribute set from a bindings builder
+ * @param[out] context Optional, stores error information
+ * @param[out] value Nix value to modify
+ * @param[in] b bindings builder to use. Make sure to unref this afterwards.
+ * @return error code, NIX_OK on success.
+ */
+nix_err nix_make_attrs(nix_c_context * context, Value * value, BindingsBuilder * b);
+
+/** @brief Set primop
+ * @param[out] context Optional, stores error information
+ * @param[out] value Nix value to modify
+ * @param[in] op primop, will be gc-referenced by the value
+ * @see nix_alloc_primop
+ * @return error code, NIX_OK on success.
+ */
+nix_err nix_init_primop(nix_c_context * context, Value * value, PrimOp * op);
+/** @brief Copy from another value
+ * @param[out] context Optional, stores error information
+ * @param[out] value Nix value to modify
+ * @param[in] source value to copy from
+ * @return error code, NIX_OK on success.
+ */
+nix_err nix_copy_value(nix_c_context * context, Value * value, Value * source);
+/**@}*/
+
+/** @brief Create a bindings builder
+* @param[out] context Optional, stores error information
+* @param[in] state nix evaluator state
+* @param[in] capacity how many bindings you'll add. Don't exceed.
+* @return owned reference to a bindings builder. Make sure to unref when you're
+done.
+*/
+BindingsBuilder * nix_make_bindings_builder(nix_c_context * context, EvalState * state, size_t capacity);
+
+/** @brief Insert bindings into a builder
+ * @param[out] context Optional, stores error information
+ * @param[in] builder BindingsBuilder to insert into
+ * @param[in] name attribute name, only used for the duration of the call.
+ * @param[in] value value to give the binding
+ * @return error code, NIX_OK on success.
+ */
+nix_err
+nix_bindings_builder_insert(nix_c_context * context, BindingsBuilder * builder, const char * name, Value * value);
+
+/** @brief Free a bindings builder
+ *
+ * Does not fail.
+ * @param[in] builder the builder to free
+ */
+void nix_bindings_builder_free(BindingsBuilder * builder);
+/**@}*/
+
+/** @brief Realise a string context.
+ *
+ * This will
+ *  - realise the store paths referenced by the string's context, and
+ *  - perform the replacement of placeholders.
+ *  - create temporary garbage collection roots for the store paths, for
+ *    the lifetime of the current process.
+ *  - log to stderr
+ *
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value, which must be a string
+ * @param[in] state Nix evaluator state
+ * @param[in] isIFD If true, disallow derivation outputs if setting `allow-import-from-derivation` is false.
+                    You should set this to true when this call is part of a primop.
+                    You should set this to false when building for your application's purpose.
+ * @return NULL if failed, are a new nix_realised_string, which must be freed with nix_realised_string_free
+ */
+nix_realised_string * nix_string_realise(nix_c_context * context, EvalState * state, Value * value, bool isIFD);
+
+/** @brief Start of the string
+ * @param[in] realised_string
+ * @return pointer to the start of the string. It may not be null-terminated.
+ */
+const char * nix_realised_string_get_buffer_start(nix_realised_string * realised_string);
+
+/** @brief Length of the string
+ * @param[in] realised_string
+ * @return length of the string in bytes
+ */
+size_t nix_realised_string_get_buffer_size(nix_realised_string * realised_string);
+
+/** @brief Number of realised store paths
+ * @param[in] realised_string
+ * @return number of realised store paths that were referenced by the string via its context
+ */
+size_t nix_realised_string_get_store_path_count(nix_realised_string * realised_string);
+
+/** @brief Get a store path. The store paths are stored in an arbitrary order.
+ * @param[in] realised_string
+ * @param[in] index index of the store path, must be less than the count
+ * @return store path
+ */
+const StorePath * nix_realised_string_get_store_path(nix_realised_string * realised_string, size_t index);
+
+/** @brief Free a realised string
+ * @param[in] realised_string
+ */
+void nix_realised_string_free(nix_realised_string * realised_string);
+
+// cffi end
+#ifdef __cplusplus
+}
+#endif
+
+/** @} */
+#endif // NIX_API_VALUE_H

src/libexpr/attr-path.cc

@@ -72,10 +72,10 @@ std::pair<Value *, PosIdx> findAlongAttrPath(EvalState & state, const std::strin
             if (attr.empty())
                 throw Error("empty attribute name in selection path '%1%'", attrPath);
 
-            Bindings::iterator a = v->attrs->find(state.symbols.create(attr));
-            if (a == v->attrs->end()) {
+            auto a = v->attrs()->get(state.symbols.create(attr));
+            if (!a) {
                 std::set<std::string> attrNames;
-                for (auto & attr : *v->attrs)
+                for (auto & attr : *v->attrs())
                     attrNames.insert(state.symbols[attr.name]);
 
                 auto suggestions = Suggestions::bestMatches(attrNames, attr);

src/libexpr/attr-set.cc

@@ -23,23 +23,6 @@ Bindings * EvalState::allocBindings(size_t capacity)
 }
 
 
-/* Create a new attribute named 'name' on an existing attribute set stored
-   in 'vAttrs' and return the newly allocated Value which is associated with
-   this attribute. */
-Value * EvalState::allocAttr(Value & vAttrs, Symbol name)
-{
-    Value * v = allocValue();
-    vAttrs.attrs->push_back(Attr(name, v));
-    return v;
-}
-
-
-Value * EvalState::allocAttr(Value & vAttrs, std::string_view name)
-{
-    return allocAttr(vAttrs, symbols.create(name));
-}
-
-
 Value & BindingsBuilder::alloc(Symbol name, PosIdx pos)
 {
     auto value = state.allocValue();

src/libexpr/attr-set.hh

@@ -65,24 +65,26 @@ public:
 
     typedef Attr * iterator;
 
+    typedef const Attr * const_iterator;
+
     void push_back(const Attr & attr)
     {
         assert(size_ < capacity_);
         attrs[size_++] = attr;
     }
 
-    iterator find(Symbol name)
+    const_iterator find(Symbol name) const
     {
         Attr key(name, 0);
-        iterator i = std::lower_bound(begin(), end(), key);
+        const_iterator i = std::lower_bound(begin(), end(), key);
         if (i != end() && i->name == name) return i;
         return end();
     }
 
-    Attr * get(Symbol name)
+    const Attr * get(Symbol name) const
     {
         Attr key(name, 0);
-        iterator i = std::lower_bound(begin(), end(), key);
+        const_iterator i = std::lower_bound(begin(), end(), key);
         if (i != end() && i->name == name) return &*i;
         return nullptr;
     }
@@ -90,14 +92,22 @@ public:
     iterator begin() { return &attrs[0]; }
     iterator end() { return &attrs[size_]; }
 
+    const_iterator begin() const { return &attrs[0]; }
+    const_iterator end() const { return &attrs[size_]; }
+
     Attr & operator[](size_t pos)
     {
         return attrs[pos];
     }
 
+    const Attr & operator[](size_t pos) const
+    {
+        return attrs[pos];
+    }
+
     void sort();
 
-    size_t capacity() { return capacity_; }
+    size_t capacity() const { return capacity_; }
 
     /**
      * Returns the attributes in lexicographically sorted order.
@@ -166,6 +176,20 @@ public:
     {
         return bindings;
     }
+
+    size_t capacity()
+    {
+        return bindings->capacity();
+    }
+
+    void grow(Bindings * newBindings)
+    {
+        for (auto & i : *bindings)
+            newBindings->push_back(i);
+        bindings = newBindings;
+    }
+
+    friend struct ExprAttrs;
 };
 
 }

src/libexpr/eval-cache.cc

@@ -387,7 +387,7 @@ Value & AttrCursor::getValue()
         if (parent) {
             auto & vParent = parent->first->getValue();
             root->state.forceAttrs(vParent, noPos, "while searching for an attribute");
-            auto attr = vParent.attrs->get(parent->second);
+            auto attr = vParent.attrs()->get(parent->second);
             if (!attr)
                 throw Error("attribute '%s' is unexpectedly missing", getAttrPathStr());
             _value = allocRootValue(attr->value);
@@ -448,9 +448,9 @@ Value & AttrCursor::forceValue()
             cachedValue = {root->db->setString(getKey(), path.abs()), string_t{path.abs(), {}}};
         }
         else if (v.type() == nBool)
-            cachedValue = {root->db->setBool(getKey(), v.boolean), v.boolean};
+            cachedValue = {root->db->setBool(getKey(), v.boolean()), v.boolean()};
         else if (v.type() == nInt)
-            cachedValue = {root->db->setInt(getKey(), v.integer), int_t{v.integer}};
+            cachedValue = {root->db->setInt(getKey(), v.integer()), int_t{v.integer()}};
         else if (v.type() == nAttrs)
             ; // FIXME: do something?
         else
@@ -510,7 +510,7 @@ std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(Symbol name, bool forceErro
         return nullptr;
         //error<TypeError>("'%s' is not an attribute set", getAttrPathStr()).debugThrow();
 
-    auto attr = v.attrs->get(name);
+    auto attr = v.attrs()->get(name);
 
     if (!attr) {
         if (root->db) {
@@ -581,7 +581,7 @@ std::string AttrCursor::getString()
     auto & v = forceValue();
 
     if (v.type() != nString && v.type() != nPath)
-        root->state.error<TypeError>("'%s' is not a string but %s", getAttrPathStr()).debugThrow();
+        root->state.error<TypeError>("'%s' is not a string but %s", getAttrPathStr(), showType(v)).debugThrow();
 
     return v.type() == nString ? v.c_str() : v.path().to_string();
 }
@@ -630,7 +630,7 @@ string_t AttrCursor::getStringWithContext()
     else if (v.type() == nPath)
         return {v.path().to_string(), {}};
     else
-        root->state.error<TypeError>("'%s' is not a string but %s", getAttrPathStr()).debugThrow();
+        root->state.error<TypeError>("'%s' is not a string but %s", getAttrPathStr(), showType(v)).debugThrow();
 }
 
 bool AttrCursor::getBool()
@@ -652,7 +652,7 @@ bool AttrCursor::getBool()
     if (v.type() != nBool)
         root->state.error<TypeError>("'%s' is not a Boolean", getAttrPathStr()).debugThrow();
 
-    return v.boolean;
+    return v.boolean();
 }
 
 NixInt AttrCursor::getInt()
@@ -674,7 +674,7 @@ NixInt AttrCursor::getInt()
     if (v.type() != nInt)
         root->state.error<TypeError>("'%s' is not an integer", getAttrPathStr()).debugThrow();
 
-    return v.integer;
+    return v.integer();
 }
 
 std::vector<std::string> AttrCursor::getListOfStrings()
@@ -730,7 +730,7 @@ std::vector<Symbol> AttrCursor::getAttrs()
         root->state.error<TypeError>("'%s' is not an attribute set", getAttrPathStr()).debugThrow();
 
     std::vector<Symbol> attrs;
-    for (auto & attr : *getValue().attrs)
+    for (auto & attr : *getValue().attrs())
         attrs.push_back(attr.name);
     std::sort(attrs.begin(), attrs.end(), [&](Symbol a, Symbol b) {
         std::string_view sa = root->state.symbols[a], sb = root->state.symbols[b];

src/libexpr/eval-inline.hh

@@ -85,8 +85,8 @@ Env & EvalState::allocEnv(size_t size)
 void EvalState::forceValue(Value & v, const PosIdx pos)
 {
     if (v.isThunk()) {
-        Env * env = v.thunk.env;
-        Expr * expr = v.thunk.expr;
+        Env * env = v.payload.thunk.env;
+        Expr * expr = v.payload.thunk.expr;
         try {
             v.mkBlackhole();
             //checkInterrupt();
@@ -98,7 +98,7 @@ void EvalState::forceValue(Value & v, const PosIdx pos)
         }
     }
     else if (v.isApp())
-        callFunction(*v.app.left, *v.app.right, v, pos);
+        callFunction(*v.payload.app.left, *v.payload.app.right, v, pos);
 }
 
 

src/libexpr/eval-settings.hh

@@ -76,9 +76,10 @@ struct EvalSettings : Config
 
           - Restrict file system and network access to files specified by cryptographic hash
           - Disable impure constants:
-            - [`bultins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem)
+            - [`builtins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem)
             - [`builtins.currentTime`](@docroot@/language/builtin-constants.md#builtins-currentTime)
             - [`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtins-nixPath)
+            - [`builtins.storePath`](@docroot@/language/builtin-constants.md#builtins-storePath)
         )"
         };
 

src/libexpr/eval.cc

@@ -33,15 +33,17 @@
 #include <optional>
 #include <unistd.h>
 #include <sys/time.h>
-#include <sys/resource.h>
 #include <fstream>
 #include <functional>
 #include <iostream>
 
-#include <sys/resource.h>
 #include <nlohmann/json.hpp>
 #include <boost/container/small_vector.hpp>
 
+#ifndef _WIN32 // TODO use portable implementation
+# include <sys/resource.h>
+#endif
+
 #if HAVE_BOEHMGC
 
 #define GC_INCLUDE_NEW
@@ -131,7 +133,7 @@ void Value::print(EvalState & state, std::ostream & str, PrintOptions options)
 const Value * getPrimOp(const Value &v) {
     const Value * primOp = &v;
     while (primOp->isPrimOpApp()) {
-        primOp = primOp->primOpApp.left;
+        primOp = primOp->payload.primOpApp.left;
     }
     assert(primOp->isPrimOp());
     return primOp;
@@ -163,12 +165,12 @@ std::string showType(const Value & v)
     #pragma GCC diagnostic push
     #pragma GCC diagnostic ignored "-Wswitch-enum"
     switch (v.internalType) {
-        case tString: return v.string.context ? "a string with context" : "a string";
+        case tString: return v.payload.string.context ? "a string with context" : "a string";
         case tPrimOp:
-            return fmt("the built-in function '%s'", std::string(v.primOp->name));
+            return fmt("the built-in function '%s'", std::string(v.payload.primOp->name));
         case tPrimOpApp:
-            return fmt("the partially applied built-in function '%s'", std::string(getPrimOp(v)->primOp->name));
-        case tExternal: return v.external->showType();
+            return fmt("the partially applied built-in function '%s'", std::string(getPrimOp(v)->payload.primOp->name));
+        case tExternal: return v.external()->showType();
         case tThunk: return v.isBlackhole() ? "a black hole" : "a thunk";
         case tApp: return "a function application";
     default:
@@ -183,9 +185,9 @@ PosIdx Value::determinePos(const PosIdx pos) const
     #pragma GCC diagnostic push
     #pragma GCC diagnostic ignored "-Wswitch-enum"
     switch (internalType) {
-        case tAttrs: return attrs->pos;
-        case tLambda: return lambda.fun->pos;
-        case tApp: return app.left->determinePos(pos);
+        case tAttrs: return attrs()->pos;
+        case tLambda: return payload.lambda.fun->pos;
+        case tApp: return payload.app.left->determinePos(pos);
         default: return pos;
     }
     #pragma GCC diagnostic pop
@@ -197,10 +199,10 @@ bool Value::isTrivial() const
         internalType != tApp
         && internalType != tPrimOpApp
         && (internalType != tThunk
-            || (dynamic_cast<ExprAttrs *>(thunk.expr)
-                && ((ExprAttrs *) thunk.expr)->dynamicAttrs.empty())
-            || dynamic_cast<ExprLambda *>(thunk.expr)
-            || dynamic_cast<ExprList *>(thunk.expr));
+            || (dynamic_cast<ExprAttrs *>(payload.thunk.expr)
+                && ((ExprAttrs *) payload.thunk.expr)->dynamicAttrs.empty())
+            || dynamic_cast<ExprLambda *>(payload.thunk.expr)
+            || dynamic_cast<ExprList *>(payload.thunk.expr));
 }
 
 
@@ -341,7 +343,7 @@ void initGC()
 }
 
 EvalState::EvalState(
-    const SearchPath & _searchPath,
+    const LookupPath & _lookupPath,
     ref<Store> store,
     std::shared_ptr<Store> buildStore)
     : sWith(symbols.create("<with>"))
@@ -435,20 +437,27 @@ EvalState::EvalState(
 
     static_assert(sizeof(Env) <= 16, "environment must be <= 16 bytes");
 
-    vEmptyList.mkList(0);
+    vEmptyList.mkList(buildList(0));
+    vNull.mkNull();
+    vTrue.mkBool(true);
+    vFalse.mkBool(false);
+    vStringRegular.mkString("regular");
+    vStringDirectory.mkString("directory");
+    vStringSymlink.mkString("symlink");
+    vStringUnknown.mkString("unknown");
 
     /* Initialise the Nix expression search path. */
     if (!evalSettings.pureEval) {
-        for (auto & i : _searchPath.elements)
-            searchPath.elements.emplace_back(SearchPath::Elem {i});
+        for (auto & i : _lookupPath.elements)
+            lookupPath.elements.emplace_back(LookupPath::Elem {i});
         for (auto & i : evalSettings.nixPath.get())
-            searchPath.elements.emplace_back(SearchPath::Elem::parse(i));
+            lookupPath.elements.emplace_back(LookupPath::Elem::parse(i));
     }
 
     /* Allow access to all paths in the search path. */
     if (rootFS.dynamic_pointer_cast<AllowListInputAccessor>())
-        for (auto & i : searchPath.elements)
-            resolveSearchPathPath(i.path, true);
+        for (auto & i : lookupPath.elements)
+            resolveLookupPathPath(i.path, true);
 
     corepkgsFS->addFile(
         CanonPath("fetchurl.nix"),
@@ -577,7 +586,7 @@ void EvalState::addConstant(const std::string & name, Value * v, Constant info)
         /* Install value the base environment. */
         staticBaseEnv->vars.emplace_back(symbols.create(name), baseEnvDispl);
         baseEnv.values[baseEnvDispl++] = v;
-        baseEnv.values[0]->attrs->push_back(Attr(symbols.create(name2), v));
+        baseEnv.values[0]->payload.attrs->push_back(Attr(symbols.create(name2), v));
     }
 }
 
@@ -590,32 +599,30 @@ void PrimOp::check()
 }
 
 
-std::ostream & operator<<(std::ostream & output, PrimOp & primOp)
+std::ostream & operator<<(std::ostream & output, const PrimOp & primOp)
 {
     output << "primop " << primOp.name;
     return output;
 }
 
 
-PrimOp * Value::primOpAppPrimOp() const
+const PrimOp * Value::primOpAppPrimOp() const
 {
-    Value * left = primOpApp.left;
+    Value * left = payload.primOpApp.left;
     while (left && !left->isPrimOp()) {
-        left = left->primOpApp.left;
+        left = left->payload.primOpApp.left;
     }
 
     if (!left)
         return nullptr;
-    return left->primOp;
+    return left->primOp();
 }
 
 
 void Value::mkPrimOp(PrimOp * p)
 {
     p->check();
-    clearValue();
-    internalType = tPrimOp;
-    primOp = p;
+    finishValue(tPrimOp, { .primOp = p });
 }
 
 
@@ -643,14 +650,14 @@ Value * EvalState::addPrimOp(PrimOp && primOp)
     v->mkPrimOp(new PrimOp(primOp));
     staticBaseEnv->vars.emplace_back(envName, baseEnvDispl);
     baseEnv.values[baseEnvDispl++] = v;
-    baseEnv.values[0]->attrs->push_back(Attr(symbols.create(primOp.name), v));
+    baseEnv.values[0]->payload.attrs->push_back(Attr(symbols.create(primOp.name), v));
     return v;
 }
 
 
 Value & EvalState::getBuiltin(const std::string & name)
 {
-    return *baseEnv.values[0]->attrs->find(symbols.create(name))->value;
+    return *baseEnv.values[0]->attrs()->find(symbols.create(name))->value;
 }
 
 
@@ -658,12 +665,12 @@ std::optional<EvalState::Doc> EvalState::getDoc(Value & v)
 {
     if (v.isPrimOp()) {
         auto v2 = &v;
-        if (auto * doc = v2->primOp->doc)
+        if (auto * doc = v2->primOp()->doc)
             return Doc {
                 .pos = {},
-                .name = v2->primOp->name,
-                .arity = v2->primOp->arity,
-                .args = v2->primOp->args,
+                .name = v2->primOp()->name,
+                .arity = v2->primOp()->arity,
+                .args = v2->primOp()->args,
                 .doc = doc,
             };
     }
@@ -687,8 +694,8 @@ void printWithBindings(const SymbolTable & st, const Env & env)
     if (!env.values[0]->isThunk()) {
         std::cout << "with: ";
         std::cout << ANSI_MAGENTA;
-        Bindings::iterator j = env.values[0]->attrs->begin();
-        while (j != env.values[0]->attrs->end()) {
+        auto j = env.values[0]->attrs()->begin();
+        while (j != env.values[0]->attrs()->end()) {
             std::cout << st[j->name] << " ";
             ++j;
         }
@@ -742,11 +749,8 @@ void mapStaticEnvBindings(const SymbolTable & st, const StaticEnv & se, const En
 
         if (se.isWith && !env.values[0]->isThunk()) {
             // add 'with' bindings.
-            Bindings::iterator j = env.values[0]->attrs->begin();
-            while (j != env.values[0]->attrs->end()) {
-                vm[st[j->name]] = j->value;
-                ++j;
-            }
+            for (auto & j : *env.values[0]->attrs())
+                vm[st[j.name]] = j.value;
         } else {
             // iterate through staticenv bindings and add them.
             for (auto & i : se.vars)
@@ -866,31 +870,30 @@ void Value::mkString(std::string_view s)
 }
 
 
-static void copyContextToValue(Value & v, const NixStringContext & context)
+static const char * * encodeContext(const NixStringContext & context)
 {
     if (!context.empty()) {
         size_t n = 0;
-        v.string.context = (const char * *)
+        auto ctx = (const char * *)
             allocBytes((context.size() + 1) * sizeof(char *));
         for (auto & i : context)
-            v.string.context[n++] = dupString(i.to_string().c_str());
-        v.string.context[n] = 0;
-    }
+            ctx[n++] = dupString(i.to_string().c_str());
+        ctx[n] = 0;
+        return ctx;
+    } else
+        return nullptr;
 }
 
 void Value::mkString(std::string_view s, const NixStringContext & context)
 {
-    mkString(s);
-    copyContextToValue(*this, context);
+    mkString(makeImmutableString(s), encodeContext(context));
 }
 
 void Value::mkStringMove(const char * s, const NixStringContext & context)
 {
-    mkString(s);
-    copyContextToValue(*this, context);
+    mkString(s, encodeContext(context));
 }
 
-
 void Value::mkPath(const SourcePath & path)
 {
     mkPath(&*path.accessor, makeImmutableString(path.path.abs()));
@@ -911,8 +914,7 @@ inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval)
     auto * fromWith = var.fromWith;
     while (1) {
         forceAttrs(*env->values[0], fromWith->pos, "while evaluating the first subexpression of a with expression");
-        Bindings::iterator j = env->values[0]->attrs->find(var.name);
-        if (j != env->values[0]->attrs->end()) {
+        if (auto j = env->values[0]->attrs()->get(var.name)) {
             if (countCalls) attrSelects[j->pos]++;
             return j->value;
         }
@@ -923,14 +925,16 @@ inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval)
     }
 }
 
-void EvalState::mkList(Value & v, size_t size)
+ListBuilder::ListBuilder(EvalState & state, size_t size)
+    : size(size)
+    , elems(size <= 2 ? inlineElems : (Value * *) allocBytes(size * sizeof(Value *)))
 {
-    v.mkList(size);
-    if (size > 2)
-        v.bigList.elems = (Value * *) allocBytes(size * sizeof(Value *));
-    nrListElems += size;
+    state.nrListElems += size;
 }
 
+Value * EvalState::getBool(bool b) {
+    return b ? &vTrue : &vFalse;
+}
 
 unsigned long nrThunks = 0;
 
@@ -1158,7 +1162,7 @@ inline bool EvalState::evalBool(Env & env, Expr * e, const PosIdx pos, std::stri
                  showType(v),
                  ValuePrinter(*this, v, errorPrintOptions)
              ).atPos(pos).withFrame(env, *e).debugThrow();
-        return v.boolean;
+        return v.boolean();
     } catch (Error & e) {
         e.addTrace(positions[pos], errorCtx);
         throw;
@@ -1226,8 +1230,9 @@ Env * ExprAttrs::buildInheritFromEnv(EvalState & state, Env & up)
 
 void ExprAttrs::eval(EvalState & state, Env & env, Value & v)
 {
-    v.mkAttrs(state.buildBindings(attrs.size() + dynamicAttrs.size()).finish());
+    auto bindings = state.buildBindings(attrs.size() + dynamicAttrs.size());
     auto dynamicEnv = &env;
+    bool sort = false;
 
     if (recursive) {
         /* Create a new environment that contains the attributes in
@@ -1252,7 +1257,7 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v)
             } else
                 vAttr = i.second.e->maybeThunk(state, *i.second.chooseByKind(&env2, &env, inheritEnv));
             env2.values[displ++] = vAttr;
-            v.attrs->push_back(Attr(i.first, vAttr, i.second.pos));
+            bindings.insert(i.first, vAttr, i.second.pos);
         }
 
         /* If the rec contains an attribute called `__overrides', then
@@ -1264,32 +1269,28 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v)
            been substituted into the bodies of the other attributes.
            Hence we need __overrides.) */
         if (hasOverrides) {
-            Value * vOverrides = (*v.attrs)[overrides->second.displ].value;
+            Value * vOverrides = (*bindings.bindings)[overrides->second.displ].value;
             state.forceAttrs(*vOverrides, [&]() { return vOverrides->determinePos(noPos); }, "while evaluating the `__overrides` attribute");
-            Bindings * newBnds = state.allocBindings(v.attrs->capacity() + vOverrides->attrs->size());
-            for (auto & i : *v.attrs)
-                newBnds->push_back(i);
-            for (auto & i : *vOverrides->attrs) {
+            bindings.grow(state.allocBindings(bindings.capacity() + vOverrides->attrs()->size()));
+            for (auto & i : *vOverrides->attrs()) {
                 AttrDefs::iterator j = attrs.find(i.name);
                 if (j != attrs.end()) {
-                    (*newBnds)[j->second.displ] = i;
+                    (*bindings.bindings)[j->second.displ] = i;
                     env2.values[j->second.displ] = i.value;
                 } else
-                    newBnds->push_back(i);
+                    bindings.push_back(i);
             }
-            newBnds->sort();
-            v.attrs = newBnds;
+            sort = true;
         }
     }
 
     else {
         Env * inheritEnv = inheritFromExprs ? buildInheritFromEnv(state, env) : nullptr;
-        for (auto & i : attrs) {
-            v.attrs->push_back(Attr(
-                    i.first,
-                    i.second.e->maybeThunk(state, *i.second.chooseByKind(&env, &env, inheritEnv)),
-                    i.second.pos));
-        }
+        for (auto & i : attrs)
+            bindings.insert(
+                i.first,
+                i.second.e->maybeThunk(state, *i.second.chooseByKind(&env, &env, inheritEnv)),
+                i.second.pos);
     }
 
     /* Dynamic attrs apply *after* rec and __overrides. */
@@ -1301,17 +1302,21 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v)
             continue;
         state.forceStringNoCtx(nameVal, i.pos, "while evaluating the name of a dynamic attribute");
         auto nameSym = state.symbols.create(nameVal.string_view());
-        Bindings::iterator j = v.attrs->find(nameSym);
-        if (j != v.attrs->end())
+        if (sort)
+            // FIXME: inefficient
+            bindings.bindings->sort();
+        if (auto j = bindings.bindings->get(nameSym))
             state.error<EvalError>("dynamic attribute '%1%' already defined at %2%", state.symbols[nameSym], state.positions[j->pos]).atPos(i.pos).withFrame(env, *this).debugThrow();
 
         i.valueExpr->setName(nameSym);
         /* Keep sorted order so find can catch duplicates */
-        v.attrs->push_back(Attr(nameSym, i.valueExpr->maybeThunk(state, *dynamicEnv), i.pos));
-        v.attrs->sort(); // FIXME: inefficient
+        bindings.insert(nameSym, i.valueExpr->maybeThunk(state, *dynamicEnv), i.pos);
+        sort = true;
     }
 
-    v.attrs->pos = pos;
+    bindings.bindings->pos = pos;
+
+    v.mkAttrs(sort ? bindings.finish() : bindings.alreadySorted());
 }
 
 
@@ -1353,9 +1358,10 @@ void ExprLet::eval(EvalState & state, Env & env, Value & v)
 
 void ExprList::eval(EvalState & state, Env & env, Value & v)
 {
-    state.mkList(v, elems.size());
-    for (auto [n, v2] : enumerate(v.listItems()))
-        const_cast<Value * &>(v2) = elems[n]->maybeThunk(state, env);
+    auto list = state.buildList(elems.size());
+    for (const auto & [n, v2] : enumerate(list))
+        v2 = elems[n]->maybeThunk(state, env);
+    v.mkList(list);
 }
 
 
@@ -1416,21 +1422,21 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v)
 
         for (auto & i : attrPath) {
             state.nrLookups++;
-            Bindings::iterator j;
+            const Attr * j;
             auto name = getName(i, state, env);
             if (def) {
                 state.forceValue(*vAttrs, pos);
                 if (vAttrs->type() != nAttrs ||
-                    (j = vAttrs->attrs->find(name)) == vAttrs->attrs->end())
+                    !(j = vAttrs->attrs()->get(name)))
                 {
                     def->eval(state, env, v);
                     return;
                 }
             } else {
                 state.forceAttrs(*vAttrs, pos, "while selecting an attribute");
-                if ((j = vAttrs->attrs->find(name)) == vAttrs->attrs->end()) {
+                if (!(j = vAttrs->attrs()->get(name))) {
                     std::set<std::string> allAttrNames;
-                    for (auto & attr : *vAttrs->attrs)
+                    for (auto & attr : *vAttrs->attrs())
                         allAttrNames.insert(state.symbols[attr.name]);
                     auto suggestions = Suggestions::bestMatches(allAttrNames, state.symbols[name]);
                     state.error<EvalError>("attribute '%1%' missing", state.symbols[name])
@@ -1468,15 +1474,15 @@ void ExprOpHasAttr::eval(EvalState & state, Env & env, Value & v)
 
     for (auto & i : attrPath) {
         state.forceValue(*vAttrs, getPos());
-        Bindings::iterator j;
+        const Attr * j;
         auto name = getName(i, state, env);
-        if (vAttrs->type() != nAttrs ||
-            (j = vAttrs->attrs->find(name)) == vAttrs->attrs->end())
+        if (vAttrs->type() == nAttrs &&
+            (j = vAttrs->attrs()->get(name)))
         {
+            vAttrs = j->value;
+        } else {
             v.mkBool(false);
             return;
-        } else {
-            vAttrs = j->value;
         }
     }
 
@@ -1528,19 +1534,19 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value &
         }
     };
 
-    Attr * functor;
+    const Attr * functor;
 
     while (nrArgs > 0) {
 
         if (vCur.isLambda()) {
 
-            ExprLambda & lambda(*vCur.lambda.fun);
+            ExprLambda & lambda(*vCur.payload.lambda.fun);
 
             auto size =
                 (!lambda.arg ? 0 : 1) +
                 (lambda.hasFormals() ? lambda.formals->formals.size() : 0);
             Env & env2(allocEnv(size));
-            env2.up = vCur.lambda.env;
+            env2.up = vCur.payload.lambda.env;
 
             Displacement displ = 0;
 
@@ -1562,7 +1568,7 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value &
                    argument has a default, use the default. */
                 size_t attrsUsed = 0;
                 for (auto & i : lambda.formals->formals) {
-                    auto j = args[0]->attrs->get(i.name);
+                    auto j = args[0]->attrs()->get(i.name);
                     if (!j) {
                         if (!i.def) {
                             error<TypeError>("function '%1%' called without required argument '%2%'",
@@ -1570,7 +1576,7 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value &
                                              symbols[i.name])
                                     .atPos(lambda.pos)
                                     .withTrace(pos, "from call site")
-                                    .withFrame(*fun.lambda.env, lambda)
+                                    .withFrame(*fun.payload.lambda.env, lambda)
                                     .debugThrow();
                         }
                         env2.values[displ++] = i.def->maybeThunk(*this, env2);
@@ -1582,10 +1588,10 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value &
 
                 /* Check that each actual argument is listed as a formal
                    argument (unless the attribute match specifies a `...'). */
-                if (!lambda.formals->ellipsis && attrsUsed != args[0]->attrs->size()) {
+                if (!lambda.formals->ellipsis && attrsUsed != args[0]->attrs()->size()) {
                     /* Nope, so show the first unexpected argument to the
                        user. */
-                    for (auto & i : *args[0]->attrs)
+                    for (auto & i : *args[0]->attrs())
                         if (!lambda.formals->has(i.name)) {
                             std::set<std::string> formalNames;
                             for (auto & formal : lambda.formals->formals)
@@ -1597,7 +1603,7 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value &
                                 .atPos(lambda.pos)
                                 .withTrace(pos, "from call site")
                                 .withSuggestions(suggestions)
-                                .withFrame(*fun.lambda.env, lambda)
+                                .withFrame(*fun.payload.lambda.env, lambda)
                                 .debugThrow();
                         }
                     abort(); // can't happen
@@ -1639,7 +1645,7 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value &
 
         else if (vCur.isPrimOp()) {
 
-            size_t argsLeft = vCur.primOp->arity;
+            size_t argsLeft = vCur.primOp()->arity;
 
             if (nrArgs < argsLeft) {
                 /* We don't have enough arguments, so create a tPrimOpApp chain. */
@@ -1647,7 +1653,7 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value &
                 return;
             } else {
                 /* We have all the arguments, so call the primop. */
-                auto * fn = vCur.primOp;
+                auto * fn = vCur.primOp();
 
                 nrPrimOpCalls++;
                 if (countCalls) primOpCalls[fn->name]++;
@@ -1655,7 +1661,8 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value &
                 try {
                     fn->fun(*this, vCur.determinePos(noPos), args, vCur);
                 } catch (Error & e) {
-                    addErrorTrace(e, pos, "while calling the '%1%' builtin", fn->name);
+                    if (fn->addTrace)
+                        addErrorTrace(e, pos, "while calling the '%1%' builtin", fn->name);
                     throw;
                 }
 
@@ -1670,10 +1677,10 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value &
             Value * primOp = &vCur;
             while (primOp->isPrimOpApp()) {
                 argsDone++;
-                primOp = primOp->primOpApp.left;
+                primOp = primOp->payload.primOpApp.left;
             }
             assert(primOp->isPrimOp());
-            auto arity = primOp->primOp->arity;
+            auto arity = primOp->primOp()->arity;
             auto argsLeft = arity - argsDone;
 
             if (nrArgs < argsLeft) {
@@ -1686,13 +1693,13 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value &
 
                 Value * vArgs[maxPrimOpArity];
                 auto n = argsDone;
-                for (Value * arg = &vCur; arg->isPrimOpApp(); arg = arg->primOpApp.left)
-                    vArgs[--n] = arg->primOpApp.right;
+                for (Value * arg = &vCur; arg->isPrimOpApp(); arg = arg->payload.primOpApp.left)
+                    vArgs[--n] = arg->payload.primOpApp.right;
 
                 for (size_t i = 0; i < argsLeft; ++i)
                     vArgs[argsDone + i] = args[i];
 
-                auto fn = primOp->primOp;
+                auto fn = primOp->primOp();
                 nrPrimOpCalls++;
                 if (countCalls) primOpCalls[fn->name]++;
 
@@ -1703,7 +1710,8 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value &
                     //    so the debugger allows to inspect the wrong parameters passed to the builtin.
                     fn->fun(*this, vCur.determinePos(noPos), vArgs, vCur);
                 } catch (Error & e) {
-                    addErrorTrace(e, pos, "while calling the '%1%' builtin", fn->name);
+                    if (fn->addTrace)
+                        addErrorTrace(e, pos, "while calling the '%1%' builtin", fn->name);
                     throw;
                 }
 
@@ -1712,7 +1720,7 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value &
             }
         }
 
-        else if (vCur.type() == nAttrs && (functor = vCur.attrs->get(sFunctor))) {
+        else if (vCur.type() == nAttrs && (functor = vCur.attrs()->get(sFunctor))) {
             /* 'vCur' may be allocated on the stack of the calling
                function, but for functors we may keep a reference, so
                heap-allocate a copy and use that instead. */
@@ -1787,8 +1795,8 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res)
     forceValue(fun, pos);
 
     if (fun.type() == nAttrs) {
-        auto found = fun.attrs->find(sFunctor);
-        if (found != fun.attrs->end()) {
+        auto found = fun.attrs()->find(sFunctor);
+        if (found != fun.attrs()->end()) {
             Value * v = allocValue();
             callFunction(*found->value, fun, *v, pos);
             forceValue(*v, pos);
@@ -1796,14 +1804,14 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res)
         }
     }
 
-    if (!fun.isLambda() || !fun.lambda.fun->hasFormals()) {
+    if (!fun.isLambda() || !fun.payload.lambda.fun->hasFormals()) {
         res = fun;
         return;
     }
 
-    auto attrs = buildBindings(std::max(static_cast<uint32_t>(fun.lambda.fun->formals->formals.size()), args.size()));
+    auto attrs = buildBindings(std::max(static_cast<uint32_t>(fun.payload.lambda.fun->formals->formals.size()), args.size()));
 
-    if (fun.lambda.fun->formals->ellipsis) {
+    if (fun.payload.lambda.fun->formals->ellipsis) {
         // If the formals have an ellipsis (eg the function accepts extra args) pass
         // all available automatic arguments (which includes arguments specified on
         // the command line via --arg/--argstr)
@@ -1811,9 +1819,9 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res)
             attrs.insert(v);
     } else {
         // Otherwise, only pass the arguments that the function accepts
-        for (auto & i : fun.lambda.fun->formals->formals) {
-            Bindings::iterator j = args.find(i.name);
-            if (j != args.end()) {
+        for (auto & i : fun.payload.lambda.fun->formals->formals) {
+            auto j = args.get(i.name);
+            if (j) {
                 attrs.insert(*j);
             } else if (!i.def) {
                 error<MissingArgumentError>(R"(cannot evaluate a function that has an argument without a value ('%1%')
@@ -1821,7 +1829,7 @@ Nix attempted to evaluate a function as a top level expression; in
 this case it must have its arguments supplied either by default
 values, or passed explicitly with '--arg' or '--argstr'. See
 https://nixos.org/manual/nix/stable/language/constructs.html#functions.)", symbols[i.name])
-                    .atPos(i.pos).withFrame(*fun.lambda.env, *fun.lambda.fun).debugThrow();
+                    .atPos(i.pos).withFrame(*fun.payload.lambda.env, *fun.payload.lambda.fun).debugThrow();
             }
         }
     }
@@ -1906,17 +1914,17 @@ void ExprOpUpdate::eval(EvalState & state, Env & env, Value & v)
 
     state.nrOpUpdates++;
 
-    if (v1.attrs->size() == 0) { v = v2; return; }
-    if (v2.attrs->size() == 0) { v = v1; return; }
+    if (v1.attrs()->size() == 0) { v = v2; return; }
+    if (v2.attrs()->size() == 0) { v = v1; return; }
 
-    auto attrs = state.buildBindings(v1.attrs->size() + v2.attrs->size());
+    auto attrs = state.buildBindings(v1.attrs()->size() + v2.attrs()->size());
 
     /* Merge the sets, preferring values from the second set.  Make
        sure to keep the resulting vector in sorted order. */
-    Bindings::iterator i = v1.attrs->begin();
-    Bindings::iterator j = v2.attrs->begin();
+    auto i = v1.attrs()->begin();
+    auto j = v2.attrs()->begin();
 
-    while (i != v1.attrs->end() && j != v2.attrs->end()) {
+    while (i != v1.attrs()->end() && j != v2.attrs()->end()) {
         if (i->name == j->name) {
             attrs.insert(*j);
             ++i; ++j;
@@ -1927,12 +1935,12 @@ void ExprOpUpdate::eval(EvalState & state, Env & env, Value & v)
             attrs.insert(*j++);
     }
 
-    while (i != v1.attrs->end()) attrs.insert(*i++);
-    while (j != v2.attrs->end()) attrs.insert(*j++);
+    while (i != v1.attrs()->end()) attrs.insert(*i++);
+    while (j != v2.attrs()->end()) attrs.insert(*j++);
 
     v.mkAttrs(attrs.alreadySorted());
 
-    state.nrOpUpdateValuesCopied += v.attrs->size();
+    state.nrOpUpdateValuesCopied += v.attrs()->size();
 }
 
 
@@ -1945,7 +1953,7 @@ void ExprOpConcatLists::eval(EvalState & state, Env & env, Value & v)
 }
 
 
-void EvalState::concatLists(Value & v, size_t nrLists, Value * * lists, const PosIdx pos, std::string_view errorCtx)
+void EvalState::concatLists(Value & v, size_t nrLists, Value * const * lists, const PosIdx pos, std::string_view errorCtx)
 {
     nrListConcats++;
 
@@ -1963,14 +1971,15 @@ void EvalState::concatLists(Value & v, size_t nrLists, Value * * lists, const Po
         return;
     }
 
-    mkList(v, len);
-    auto out = v.listElems();
+    auto list = buildList(len);
+    auto out = list.elems;
     for (size_t n = 0, pos = 0; n < nrLists; ++n) {
         auto l = lists[n]->listSize();
         if (l)
             memcpy(out + pos, lists[n]->listElems(), l * sizeof(Value *));
         pos += l;
     }
+    v.mkList(list);
 }
 
 
@@ -2023,19 +2032,19 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
 
         if (firstType == nInt) {
             if (vTmp.type() == nInt) {
-                n += vTmp.integer;
+                n += vTmp.integer();
             } else if (vTmp.type() == nFloat) {
                 // Upgrade the type from int to float;
                 firstType = nFloat;
                 nf = n;
-                nf += vTmp.fpoint;
+                nf += vTmp.fpoint();
             } else
                 state.error<EvalError>("cannot add %1% to an integer", showType(vTmp)).atPos(i_pos).withFrame(env, *this).debugThrow();
         } else if (firstType == nFloat) {
             if (vTmp.type() == nInt) {
-                nf += vTmp.integer;
+                nf += vTmp.integer();
             } else if (vTmp.type() == nFloat) {
-                nf += vTmp.fpoint;
+                nf += vTmp.fpoint();
             } else
                 state.error<EvalError>("cannot add %1% to a float", showType(vTmp)).atPos(i_pos).withFrame(env, *this).debugThrow();
         } else {
@@ -2108,11 +2117,11 @@ void EvalState::forceValueDeep(Value & v)
         forceValue(v, v.determinePos(noPos));
 
         if (v.type() == nAttrs) {
-            for (auto & i : *v.attrs)
+            for (auto & i : *v.attrs())
                 try {
                     // If the value is a thunk, we're evaling. Otherwise no trace necessary.
                     auto dts = debugRepl && i.value->isThunk()
-                        ? makeDebugTraceStacker(*this, *i.value->thunk.expr, *i.value->thunk.env, positions[i.pos],
+                        ? makeDebugTraceStacker(*this, *i.value->payload.thunk.expr, *i.value->payload.thunk.env, positions[i.pos],
                             "while evaluating the attribute '%1%'", symbols[i.name])
                         : nullptr;
 
@@ -2143,13 +2152,13 @@ NixInt EvalState::forceInt(Value & v, const PosIdx pos, std::string_view errorCt
                 showType(v),
                 ValuePrinter(*this, v, errorPrintOptions)
             ).atPos(pos).debugThrow();
-        return v.integer;
+        return v.integer();
     } catch (Error & e) {
         e.addTrace(positions[pos], errorCtx);
         throw;
     }
 
-    return v.integer;
+    return v.integer();
 }
 
 
@@ -2158,14 +2167,14 @@ NixFloat EvalState::forceFloat(Value & v, const PosIdx pos, std::string_view err
     try {
         forceValue(v, pos);
         if (v.type() == nInt)
-            return v.integer;
+            return v.integer();
         else if (v.type() != nFloat)
             error<TypeError>(
                 "expected a float but found %1%: %2%",
                 showType(v),
                 ValuePrinter(*this, v, errorPrintOptions)
             ).atPos(pos).debugThrow();
-        return v.fpoint;
+        return v.fpoint();
     } catch (Error & e) {
         e.addTrace(positions[pos], errorCtx);
         throw;
@@ -2183,19 +2192,19 @@ bool EvalState::forceBool(Value & v, const PosIdx pos, std::string_view errorCtx
                 showType(v),
                 ValuePrinter(*this, v, errorPrintOptions)
             ).atPos(pos).debugThrow();
-        return v.boolean;
+        return v.boolean();
     } catch (Error & e) {
         e.addTrace(positions[pos], errorCtx);
         throw;
     }
 
-    return v.boolean;
+    return v.boolean();
 }
 
 
 bool EvalState::isFunctor(Value & fun)
 {
-    return fun.type() == nAttrs && fun.attrs->find(sFunctor) != fun.attrs->end();
+    return fun.type() == nAttrs && fun.attrs()->find(sFunctor) != fun.attrs()->end();
 }
 
 
@@ -2236,8 +2245,8 @@ std::string_view EvalState::forceString(Value & v, const PosIdx pos, std::string
 
 void copyContext(const Value & v, NixStringContext & context)
 {
-    if (v.string.context)
-        for (const char * * p = v.string.context; *p; ++p)
+    if (v.payload.string.context)
+        for (const char * * p = v.payload.string.context; *p; ++p)
             context.insert(NixStringContextElem::parse(*p));
 }
 
@@ -2263,8 +2272,8 @@ std::string_view EvalState::forceStringNoCtx(Value & v, const PosIdx pos, std::s
 bool EvalState::isDerivation(Value & v)
 {
     if (v.type() != nAttrs) return false;
-    Bindings::iterator i = v.attrs->find(sType);
-    if (i == v.attrs->end()) return false;
+    auto i = v.attrs()->get(sType);
+    if (!i) return false;
     forceValue(*i->value, i->pos);
     if (i->value->type() != nString) return false;
     return i->value->string_view().compare("derivation") == 0;
@@ -2274,8 +2283,8 @@ bool EvalState::isDerivation(Value & v)
 std::optional<std::string> EvalState::tryAttrsToString(const PosIdx pos, Value & v,
     NixStringContext & context, bool coerceMore, bool copyToStore)
 {
-    auto i = v.attrs->find(sToString);
-    if (i != v.attrs->end()) {
+    auto i = v.attrs()->find(sToString);
+    if (i != v.attrs()->end()) {
         Value v1;
         callFunction(*i->value, v, v1, pos);
         return coerceToString(pos, v1, context,
@@ -2307,7 +2316,7 @@ BackedStringView EvalState::coerceToString(
             !canonicalizePath && !copyToStore
             ? // FIXME: hack to preserve path literals that end in a
               // slash, as in /foo/${x}.
-              v._path.path
+              v.payload.path.path
             : copyToStore
             ? store->printStorePath(copyPathToStore(context, v.path()))
             : std::string(v.path().path.abs());
@@ -2317,8 +2326,8 @@ BackedStringView EvalState::coerceToString(
         auto maybeString = tryAttrsToString(pos, v, context, coerceMore, copyToStore);
         if (maybeString)
             return std::move(*maybeString);
-        auto i = v.attrs->find(sOutPath);
-        if (i == v.attrs->end()) {
+        auto i = v.attrs()->find(sOutPath);
+        if (i == v.attrs()->end()) {
             error<TypeError>(
                 "cannot coerce %1% to a string: %2%",
                 showType(v),
@@ -2333,7 +2342,7 @@ BackedStringView EvalState::coerceToString(
 
     if (v.type() == nExternal) {
         try {
-            return v.external->coerceToString(*this, pos, context, coerceMore, copyToStore);
+            return v.external()->coerceToString(*this, pos, context, coerceMore, copyToStore);
         } catch (Error & e) {
             e.addTrace(nullptr, errorCtx);
             throw;
@@ -2343,10 +2352,10 @@ BackedStringView EvalState::coerceToString(
     if (coerceMore) {
         /* Note that `false' is represented as an empty string for
            shell scripting convenience, just like `null'. */
-        if (v.type() == nBool && v.boolean) return "1";
-        if (v.type() == nBool && !v.boolean) return "";
-        if (v.type() == nInt) return std::to_string(v.integer);
-        if (v.type() == nFloat) return std::to_string(v.fpoint);
+        if (v.type() == nBool && v.boolean()) return "1";
+        if (v.type() == nBool && !v.boolean()) return "";
+        if (v.type() == nInt) return std::to_string(v.integer());
+        if (v.type() == nFloat) return std::to_string(v.fpoint());
         if (v.type() == nNull) return "";
 
         if (v.isList()) {
@@ -2425,8 +2434,8 @@ SourcePath EvalState::coerceToPath(const PosIdx pos, Value & v, NixStringContext
     /* Similarly, handle __toString where the result may be a path
        value. */
     if (v.type() == nAttrs) {
-        auto i = v.attrs->find(sToString);
-        if (i != v.attrs->end()) {
+        auto i = v.attrs()->find(sToString);
+        if (i != v.attrs()->end()) {
             Value v1;
             callFunction(*i->value, v, v1, pos);
             return coerceToPath(pos, v1, context, errorCtx);
@@ -2520,19 +2529,19 @@ bool EvalState::eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_v
 
     // Special case type-compatibility between float and int
     if (v1.type() == nInt && v2.type() == nFloat)
-        return v1.integer == v2.fpoint;
+        return v1.integer() == v2.fpoint();
     if (v1.type() == nFloat && v2.type() == nInt)
-        return v1.fpoint == v2.integer;
+        return v1.fpoint() == v2.integer();
 
     // All other types are not compatible with each other.
     if (v1.type() != v2.type()) return false;
 
     switch (v1.type()) {
         case nInt:
-            return v1.integer == v2.integer;
+            return v1.integer() == v2.integer();
 
         case nBool:
-            return v1.boolean == v2.boolean;
+            return v1.boolean() == v2.boolean();
 
         case nString:
             return strcmp(v1.c_str(), v2.c_str()) == 0;
@@ -2540,8 +2549,8 @@ bool EvalState::eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_v
         case nPath:
             return
                 // FIXME: compare accessors by their fingerprint.
-                v1._path.accessor == v2._path.accessor
-                && strcmp(v1._path.path, v2._path.path) == 0;
+                v1.payload.path.accessor == v2.payload.path.accessor
+                && strcmp(v1.payload.path.path, v2.payload.path.path) == 0;
 
         case nNull:
             return true;
@@ -2556,17 +2565,17 @@ bool EvalState::eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_v
             /* If both sets denote a derivation (type = "derivation"),
                then compare their outPaths. */
             if (isDerivation(v1) && isDerivation(v2)) {
-                Bindings::iterator i = v1.attrs->find(sOutPath);
-                Bindings::iterator j = v2.attrs->find(sOutPath);
-                if (i != v1.attrs->end() && j != v2.attrs->end())
+                auto i = v1.attrs()->get(sOutPath);
+                auto j = v2.attrs()->get(sOutPath);
+                if (i && j)
                     return eqValues(*i->value, *j->value, pos, errorCtx);
             }
 
-            if (v1.attrs->size() != v2.attrs->size()) return false;
+            if (v1.attrs()->size() != v2.attrs()->size()) return false;
 
             /* Otherwise, compare the attributes one by one. */
-            Bindings::iterator i, j;
-            for (i = v1.attrs->begin(), j = v2.attrs->begin(); i != v1.attrs->end(); ++i, ++j)
+            Bindings::const_iterator i, j;
+            for (i = v1.attrs()->begin(), j = v2.attrs()->begin(); i != v1.attrs()->end(); ++i, ++j)
                 if (i->name != j->name || !eqValues(*i->value, *j->value, pos, errorCtx))
                     return false;
 
@@ -2578,10 +2587,10 @@ bool EvalState::eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_v
             return false;
 
         case nExternal:
-            return *v1.external == *v2.external;
+            return *v1.external() == *v2.external();
 
         case nFloat:
-            return v1.fpoint == v2.fpoint;
+            return v1.fpoint() == v2.fpoint();
 
         case nThunk: // Must not be left by forceValue
         default:
@@ -2620,9 +2629,11 @@ void EvalState::maybePrintStats()
 
 void EvalState::printStatistics()
 {
+#ifndef _WIN32 // TODO use portable implementation
     struct rusage buf;
     getrusage(RUSAGE_SELF, &buf);
     float cpuTime = buf.ru_utime.tv_sec + ((float) buf.ru_utime.tv_usec / 1000000);
+#endif
 
     uint64_t bEnvs = nrEnvs * sizeof(Env) + nrValuesInEnvs * sizeof(Value *);
     uint64_t bLists = nrListElems * sizeof(Value *);
@@ -2639,7 +2650,9 @@ void EvalState::printStatistics()
     if (outPath != "-")
         fs.open(outPath, std::fstream::out);
     json topObj = json::object();
+#ifndef _WIN32 // TODO implement
     topObj["cpuTime"] = cpuTime;
+#endif
     topObj["envs"] = {
         {"number", nrEnvs},
         {"elements", nrValuesInEnvs},
@@ -2807,19 +2820,19 @@ Expr * EvalState::parseStdin()
 
 SourcePath EvalState::findFile(const std::string_view path)
 {
-    return findFile(searchPath, path);
+    return findFile(lookupPath, path);
 }
 
 
-SourcePath EvalState::findFile(const SearchPath & searchPath, const std::string_view path, const PosIdx pos)
+SourcePath EvalState::findFile(const LookupPath & lookupPath, const std::string_view path, const PosIdx pos)
 {
-    for (auto & i : searchPath.elements) {
+    for (auto & i : lookupPath.elements) {
         auto suffixOpt = i.prefix.suffixIfPotentialMatch(path);
 
         if (!suffixOpt) continue;
         auto suffix = *suffixOpt;
 
-        auto rOpt = resolveSearchPathPath(i.path);
+        auto rOpt = resolveLookupPathPath(i.path);
         if (!rOpt) continue;
         auto r = *rOpt;
 
@@ -2839,11 +2852,11 @@ SourcePath EvalState::findFile(const SearchPath & searchPath, const std::string_
 }
 
 
-std::optional<std::string> EvalState::resolveSearchPathPath(const SearchPath::Path & value0, bool initAccessControl)
+std::optional<std::string> EvalState::resolveLookupPathPath(const LookupPath::Path & value0, bool initAccessControl)
 {
     auto & value = value0.s;
-    auto i = searchPathResolved.find(value);
-    if (i != searchPathResolved.end()) return i->second;
+    auto i = lookupPathResolved.find(value);
+    if (i != lookupPathResolved.end()) return i->second;
 
     std::optional<std::string> res;
 
@@ -2899,7 +2912,7 @@ std::optional<std::string> EvalState::resolveSearchPathPath(const SearchPath::Pa
     else
         debug("failed to resolve search path element '%s'", value);
 
-    searchPathResolved.emplace(value, res);
+    lookupPathResolved.emplace(value, res);
     return res;
 }
 

src/libexpr/eval.hh

@@ -17,6 +17,7 @@
 #include <optional>
 #include <unordered_map>
 #include <mutex>
+#include <functional>
 
 namespace nix {
 
@@ -69,10 +70,17 @@ struct PrimOp
      */
     const char * doc = nullptr;
 
+    /**
+     * Add a trace item, `while calling the '<name>' builtin`
+     *
+     * This is used to remove the redundant item for `builtins.addErrorContext`.
+     */
+    bool addTrace = true;
+
     /**
      * Implementation of the primop.
      */
-    PrimOpFun fun;
+    std::function<std::remove_pointer<PrimOpFun>::type> fun;
 
     /**
      * Optional experimental for this to be gated on.
@@ -86,7 +94,7 @@ struct PrimOp
     void check();
 };
 
-std::ostream & operator<<(std::ostream & output, PrimOp & primOp);
+std::ostream & operator<<(std::ostream & output, const PrimOp & primOp);
 
 /**
  * Info about a constant
@@ -153,7 +161,6 @@ struct DebugTrace {
     bool isError;
 };
 
-
 class EvalState : public std::enable_shared_from_this<EvalState>
 {
 public:
@@ -186,6 +193,36 @@ public:
      */
     Value vEmptyList;
 
+    /**
+     * `null` constant.
+     *
+     * This is _not_ a singleton. Pointer equality is _not_ sufficient.
+     */
+    Value vNull;
+
+    /**
+     * `true` constant.
+     *
+     * This is _not_ a singleton. Pointer equality is _not_ sufficient.
+     */
+    Value vTrue;
+
+    /**
+     * `true` constant.
+     *
+     * This is _not_ a singleton. Pointer equality is _not_ sufficient.
+     */
+    Value vFalse;
+
+    /** `"regular"` */
+    Value vStringRegular;
+    /** `"directory"` */
+    Value vStringDirectory;
+    /** `"symlink"` */
+    Value vStringSymlink;
+    /** `"unknown"` */
+    Value vStringUnknown;
+
     /**
      * The accessor for the root filesystem.
      */
@@ -271,9 +308,9 @@ private:
 #endif
     FileEvalCache fileEvalCache;
 
-    SearchPath searchPath;
+    LookupPath lookupPath;
 
-    std::map<std::string, std::optional<std::string>> searchPathResolved;
+    std::map<std::string, std::optional<std::string>> lookupPathResolved;
 
     /**
      * Cache used by prim_match().
@@ -295,12 +332,12 @@ private:
 public:
 
     EvalState(
-        const SearchPath & _searchPath,
+        const LookupPath & _lookupPath,
         ref<Store> store,
         std::shared_ptr<Store> buildStore = nullptr);
     ~EvalState();
 
-    SearchPath getSearchPath() { return searchPath; }
+    LookupPath getLookupPath() { return lookupPath; }
 
     /**
      * Return a `SourcePath` that refers to `path` in the root
@@ -369,7 +406,7 @@ public:
      * Look up a file in the search path.
      */
     SourcePath findFile(const std::string_view path);
-    SourcePath findFile(const SearchPath & searchPath, const std::string_view path, const PosIdx pos = noPos);
+    SourcePath findFile(const LookupPath & lookupPath, const std::string_view path, const PosIdx pos = noPos);
 
     /**
      * Try to resolve a search path value (not the optional key part).
@@ -378,8 +415,8 @@ public:
      *
      * If it is not found, return `std::nullopt`
      */
-    std::optional<std::string> resolveSearchPathPath(
-        const SearchPath::Path & elem,
+    std::optional<std::string> resolveLookupPathPath(
+        const LookupPath::Path & elem,
         bool initAccessControl = false);
 
     /**
@@ -605,9 +642,6 @@ public:
     inline Value * allocValue();
     inline Env & allocEnv(size_t size);
 
-    Value * allocAttr(Value & vAttrs, Symbol name);
-    Value * allocAttr(Value & vAttrs, std::string_view name);
-
     Bindings * allocBindings(size_t capacity);
 
     BindingsBuilder buildBindings(size_t capacity)
@@ -615,7 +649,16 @@ public:
         return BindingsBuilder(*this, allocBindings(capacity));
     }
 
-    void mkList(Value & v, size_t length);
+    ListBuilder buildList(size_t size)
+    {
+        return ListBuilder(*this, size);
+    }
+
+    /**
+     * Return a boolean `Value *` without allocating.
+     */
+    Value *getBool(bool b);
+
     void mkThunk_(Value & v, Expr * expr);
     void mkPos(Value & v, PosIdx pos);
 
@@ -662,7 +705,7 @@ public:
         const SingleDerivedPath & p,
         Value & v);
 
-    void concatLists(Value & v, size_t nrLists, Value * * lists, const PosIdx pos, std::string_view errorCtx);
+    void concatLists(Value & v, size_t nrLists, Value * const * lists, const PosIdx pos, std::string_view errorCtx);
 
     /**
      * Print statistics, if enabled.
@@ -686,10 +729,12 @@ public:
     bool fullGC();
 
     /**
-     * Realise the given context, and return a mapping from the placeholders
-     * used to construct the associated value to their final store path
+     * Realise the given context
+     * @param[in] context the context to realise
+     * @param[out] maybePaths if not nullptr, all built or referenced store paths will be added to this set
+     * @return a mapping from the placeholders used to construct the associated value to their final store path.
      */
-    [[nodiscard]] StringMap realiseContext(const NixStringContext & context);
+    [[nodiscard]] StringMap realiseContext(const NixStringContext & context, StorePathSet * maybePaths = nullptr, bool isIFD = true);
 
     /* Call the binary path filter predicate used builtins.path etc. */
     bool callPathFilter(
@@ -756,6 +801,7 @@ private:
     friend void prim_split(EvalState & state, const PosIdx pos, Value * * args, Value & v);
 
     friend struct Value;
+    friend class ListBuilder;
 };
 
 struct DebugTraceStacker {

src/libexpr/flake/flake.cc

@@ -10,6 +10,7 @@
 #include "finally.hh"
 #include "fetch-settings.hh"
 #include "value-to-json.hh"
+#include "local-fs-store.hh"
 
 namespace nix {
 
@@ -111,7 +112,7 @@ static FlakeInput parseFlakeInput(EvalState & state,
     fetchers::Attrs attrs;
     std::optional<std::string> url;
 
-    for (nix::Attr attr : *(value->attrs)) {
+    for (auto & attr : *value->attrs()) {
         try {
             if (attr.name == sUrl) {
                 expectType(state, nString, *attr.value, attr.pos);
@@ -119,7 +120,7 @@ static FlakeInput parseFlakeInput(EvalState & state,
                 attrs.emplace("url", *url);
             } else if (attr.name == sFlake) {
                 expectType(state, nBool, *attr.value, attr.pos);
-                input.isFlake = attr.value->boolean;
+                input.isFlake = attr.value->boolean();
             } else if (attr.name == sInputs) {
                 input.overrides = parseFlakeInputs(state, attr.value, attr.pos, baseDir, lockRootPath);
             } else if (attr.name == sFollows) {
@@ -136,10 +137,10 @@ static FlakeInput parseFlakeInput(EvalState & state,
                         attrs.emplace(state.symbols[attr.name], attr.value->c_str());
                         break;
                     case nBool:
-                        attrs.emplace(state.symbols[attr.name], Explicit<bool> { attr.value->boolean });
+                        attrs.emplace(state.symbols[attr.name], Explicit<bool> { attr.value->boolean() });
                         break;
                     case nInt:
-                        attrs.emplace(state.symbols[attr.name], (long unsigned int) attr.value->integer);
+                        attrs.emplace(state.symbols[attr.name], (long unsigned int) attr.value->integer());
                         break;
                     default:
                         if (attr.name == state.symbols.create("publicKeys")) {
@@ -189,7 +190,7 @@ static std::map<FlakeId, FlakeInput> parseFlakeInputs(
 
     expectType(state, nAttrs, *value, pos);
 
-    for (nix::Attr & inputAttr : *(*value).attrs) {
+    for (auto & inputAttr : *value->attrs()) {
         inputs.emplace(state.symbols[inputAttr.name],
             parseFlakeInput(state,
                 state.symbols[inputAttr.name],
@@ -223,23 +224,23 @@ static Flake readFlake(
         .path = flakePath,
     };
 
-    if (auto description = vInfo.attrs->get(state.sDescription)) {
+    if (auto description = vInfo.attrs()->get(state.sDescription)) {
         expectType(state, nString, *description->value, description->pos);
         flake.description = description->value->c_str();
     }
 
     auto sInputs = state.symbols.create("inputs");
 
-    if (auto inputs = vInfo.attrs->get(sInputs))
+    if (auto inputs = vInfo.attrs()->get(sInputs))
         flake.inputs = parseFlakeInputs(state, inputs->value, inputs->pos, flakePath.parent().path.abs(), lockRootPath); // FIXME
 
     auto sOutputs = state.symbols.create("outputs");
 
-    if (auto outputs = vInfo.attrs->get(sOutputs)) {
+    if (auto outputs = vInfo.attrs()->get(sOutputs)) {
         expectType(state, nFunction, *outputs->value, outputs->pos);
 
-        if (outputs->value->isLambda() && outputs->value->lambda.fun->hasFormals()) {
-            for (auto & formal : outputs->value->lambda.fun->formals->formals) {
+        if (outputs->value->isLambda() && outputs->value->payload.lambda.fun->hasFormals()) {
+            for (auto & formal : outputs->value->payload.lambda.fun->formals->formals) {
                 if (formal.name != state.sSelf)
                     flake.inputs.emplace(state.symbols[formal.name], FlakeInput {
                         .ref = parseFlakeRef(state.symbols[formal.name])
@@ -252,10 +253,10 @@ static Flake readFlake(
 
     auto sNixConfig = state.symbols.create("nixConfig");
 
-    if (auto nixConfig = vInfo.attrs->get(sNixConfig)) {
+    if (auto nixConfig = vInfo.attrs()->get(sNixConfig)) {
         expectType(state, nAttrs, *nixConfig->value, nixConfig->pos);
 
-        for (auto & setting : *nixConfig->value->attrs) {
+        for (auto & setting : *nixConfig->value->attrs()) {
             forceTrivialValue(state, *setting.value, setting.pos);
             if (setting.value->type() == nString)
                 flake.config.settings.emplace(
@@ -291,7 +292,7 @@ static Flake readFlake(
         }
     }
 
-    for (auto & attr : *vInfo.attrs) {
+    for (auto & attr : *vInfo.attrs()) {
         if (attr.name != state.sDescription &&
             attr.name != sInputs &&
             attr.name != sOutputs &&
@@ -755,7 +756,17 @@ void callFlake(EvalState & state,
 
         auto lockedNode = node.dynamic_pointer_cast<const LockedNode>();
 
-        auto [storePath, subdir] = state.store->toStorePath(sourcePath.path.abs());
+        // FIXME: This is a hack to support chroot stores. Remove this
+        // once we can pass a sourcePath rather than a storePath to
+        // call-flake.nix.
+        auto path = sourcePath.path.abs();
+        if (auto store = state.store.dynamic_pointer_cast<LocalFSStore>()) {
+            auto realStoreDir = store->getRealStoreDir();
+            if (isInDir(path, realStoreDir))
+                path = store->storeDir + path.substr(realStoreDir.size());
+        }
+
+        auto [storePath, subdir] = state.store->toStorePath(path);
 
         emitTreeAttrs(
             state,
@@ -857,10 +868,13 @@ static RegisterPrimOp r3({
       Parse a flake reference, and return its exploded form.
 
       For example:
+
       ```nix
       builtins.parseFlakeRef "github:NixOS/nixpkgs/23.05?dir=lib"
       ```
+
       evaluates to:
+
       ```nix
       { dir = "lib"; owner = "NixOS"; ref = "23.05"; repo = "nixpkgs"; type = "github"; }
       ```
@@ -879,17 +893,17 @@ static void prim_flakeRefToString(
     state.forceAttrs(*args[0], noPos,
         "while evaluating the argument passed to builtins.flakeRefToString");
     fetchers::Attrs attrs;
-    for (const auto & attr : *args[0]->attrs) {
+    for (const auto & attr : *args[0]->attrs()) {
         auto t = attr.value->type();
         if (t == nInt) {
             attrs.emplace(state.symbols[attr.name],
-                          (uint64_t) attr.value->integer);
+                (uint64_t) attr.value->integer());
         } else if (t == nBool) {
             attrs.emplace(state.symbols[attr.name],
-                          Explicit<bool> { attr.value->boolean });
+                Explicit<bool> { attr.value->boolean() });
         } else if (t == nString) {
             attrs.emplace(state.symbols[attr.name],
-                          std::string(attr.value->string_view()));
+                std::string(attr.value->string_view()));
         } else {
             state.error<EvalError>(
                 "flake reference attribute sets may only contain integers, Booleans, "
@@ -909,12 +923,15 @@ static RegisterPrimOp r4({
       Convert a flake reference from attribute set format to URL format.
 
       For example:
+
       ```nix
       builtins.flakeRefToString {
         dir = "lib"; owner = "NixOS"; ref = "23.05"; repo = "nixpkgs"; type = "github";
       }
       ```
+
       evaluates to
+
       ```nix
       "github:NixOS/nixpkgs/23.05?dir=lib"
       ```

src/libexpr/flake/flakeref.hh

@@ -68,24 +68,39 @@ struct FlakeRef
 
 std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef);
 
+/**
+ * @param baseDir Optional [base directory](https://nixos.org/manual/nix/unstable/glossary#gloss-base-directory)
+ */
 FlakeRef parseFlakeRef(
     const std::string & url,
     const std::optional<Path> & baseDir = {},
     bool allowMissing = false,
     bool isFlake = true);
 
+/**
+ * @param baseDir Optional [base directory](https://nixos.org/manual/nix/unstable/glossary#gloss-base-directory)
+ */
 std::optional<FlakeRef> maybeParseFlake(
     const std::string & url, const std::optional<Path> & baseDir = {});
 
+/**
+ * @param baseDir Optional [base directory](https://nixos.org/manual/nix/unstable/glossary#gloss-base-directory)
+ */
 std::pair<FlakeRef, std::string> parseFlakeRefWithFragment(
     const std::string & url,
     const std::optional<Path> & baseDir = {},
     bool allowMissing = false,
     bool isFlake = true);
 
+/**
+ * @param baseDir Optional [base directory](https://nixos.org/manual/nix/unstable/glossary#gloss-base-directory)
+ */
 std::optional<std::pair<FlakeRef, std::string>> maybeParseFlakeRefWithFragment(
     const std::string & url, const std::optional<Path> & baseDir = {});
 
+/**
+ * @param baseDir Optional [base directory](https://nixos.org/manual/nix/unstable/glossary#gloss-base-directory)
+ */
 std::tuple<FlakeRef, std::string, ExtendedOutputsSpec> parseFlakeRefWithFragmentAndExtendedOutputsSpec(
     const std::string & url,
     const std::optional<Path> & baseDir = {},

src/libexpr/get-drvs.cc

@@ -11,7 +11,7 @@
 namespace nix {
 
 
-PackageInfo::PackageInfo(EvalState & state, std::string attrPath, Bindings * attrs)
+PackageInfo::PackageInfo(EvalState & state, std::string attrPath, const Bindings * attrs)
     : state(&state), attrs(attrs), attrPath(std::move(attrPath))
 {
 }
@@ -69,12 +69,11 @@ std::string PackageInfo::querySystem() const
 std::optional<StorePath> PackageInfo::queryDrvPath() const
 {
     if (!drvPath && attrs) {
-        Bindings::iterator i = attrs->find(state->sDrvPath);
         NixStringContext context;
-        if (i == attrs->end())
-            drvPath = {std::nullopt};
-        else
+        if (auto i = attrs->get(state->sDrvPath))
             drvPath = {state->coerceToStorePath(i->pos, *i->value, context, "while evaluating the 'drvPath' attribute of a derivation")};
+        else
+            drvPath = {std::nullopt};
     }
     return drvPath.value_or(std::nullopt);
 }
@@ -91,7 +90,7 @@ StorePath PackageInfo::requireDrvPath() const
 StorePath PackageInfo::queryOutPath() const
 {
     if (!outPath && attrs) {
-        Bindings::iterator i = attrs->find(state->sOutPath);
+        auto i = attrs->find(state->sOutPath);
         NixStringContext context;
         if (i != attrs->end())
             outPath = state->coerceToStorePath(i->pos, *i->value, context, "while evaluating the output path of a derivation");
@@ -106,8 +105,8 @@ PackageInfo::Outputs PackageInfo::queryOutputs(bool withPaths, bool onlyOutputsT
 {
     if (outputs.empty()) {
         /* Get the ‘outputs’ list. */
-        Bindings::iterator i;
-        if (attrs && (i = attrs->find(state->sOutputs)) != attrs->end()) {
+        const Attr * i;
+        if (attrs && (i = attrs->get(state->sOutputs))) {
             state->forceList(*i->value, i->pos, "while evaluating the 'outputs' attribute of a derivation");
 
             /* For each output... */
@@ -116,13 +115,13 @@ PackageInfo::Outputs PackageInfo::queryOutputs(bool withPaths, bool onlyOutputsT
 
                 if (withPaths) {
                     /* Evaluate the corresponding set. */
-                    Bindings::iterator out = attrs->find(state->symbols.create(output));
-                    if (out == attrs->end()) continue; // FIXME: throw error?
+                    auto out = attrs->get(state->symbols.create(output));
+                    if (!out) continue; // FIXME: throw error?
                     state->forceAttrs(*out->value, i->pos, "while evaluating an output of a derivation");
 
                     /* And evaluate its ‘outPath’ attribute. */
-                    Bindings::iterator outPath = out->value->attrs->find(state->sOutPath);
-                    if (outPath == out->value->attrs->end()) continue; // FIXME: throw error?
+                    auto outPath = out->value->attrs()->get(state->sOutPath);
+                    if (!outPath) continue; // FIXME: throw error?
                     NixStringContext context;
                     outputs.emplace(output, state->coerceToStorePath(outPath->pos, *outPath->value, context, "while evaluating an output path of a derivation"));
                 } else
@@ -135,8 +134,8 @@ PackageInfo::Outputs PackageInfo::queryOutputs(bool withPaths, bool onlyOutputsT
     if (!onlyOutputsToInstall || !attrs)
         return outputs;
 
-    Bindings::iterator i;
-    if (attrs && (i = attrs->find(state->sOutputSpecified)) != attrs->end() && state->forceBool(*i->value, i->pos, "while evaluating the 'outputSpecified' attribute of a derivation")) {
+    const Attr * i;
+    if (attrs && (i = attrs->get(state->sOutputSpecified)) && state->forceBool(*i->value, i->pos, "while evaluating the 'outputSpecified' attribute of a derivation")) {
         Outputs result;
         auto out = outputs.find(queryOutputName());
         if (out == outputs.end())
@@ -167,21 +166,21 @@ PackageInfo::Outputs PackageInfo::queryOutputs(bool withPaths, bool onlyOutputsT
 std::string PackageInfo::queryOutputName() const
 {
     if (outputName == "" && attrs) {
-        Bindings::iterator i = attrs->find(state->sOutputName);
-        outputName = i != attrs->end() ? state->forceStringNoCtx(*i->value, noPos, "while evaluating the output name of a derivation") : "";
+        auto i = attrs->get(state->sOutputName);
+        outputName = i ? state->forceStringNoCtx(*i->value, noPos, "while evaluating the output name of a derivation") : "";
     }
     return outputName;
 }
 
 
-Bindings * PackageInfo::getMeta()
+const Bindings * PackageInfo::getMeta()
 {
     if (meta) return meta;
     if (!attrs) return 0;
-    Bindings::iterator a = attrs->find(state->sMeta);
-    if (a == attrs->end()) return 0;
+    auto a = attrs->get(state->sMeta);
+    if (!a) return 0;
     state->forceAttrs(*a->value, a->pos, "while evaluating the 'meta' attribute of a derivation");
-    meta = a->value->attrs;
+    meta = a->value->attrs();
     return meta;
 }
 
@@ -205,9 +204,8 @@ bool PackageInfo::checkMeta(Value & v)
         return true;
     }
     else if (v.type() == nAttrs) {
-        Bindings::iterator i = v.attrs->find(state->sOutPath);
-        if (i != v.attrs->end()) return false;
-        for (auto & i : *v.attrs)
+        if (v.attrs()->get(state->sOutPath)) return false;
+        for (auto & i : *v.attrs())
             if (!checkMeta(*i.value)) return false;
         return true;
     }
@@ -219,8 +217,8 @@ bool PackageInfo::checkMeta(Value & v)
 Value * PackageInfo::queryMeta(const std::string & name)
 {
     if (!getMeta()) return 0;
-    Bindings::iterator a = meta->find(state->symbols.create(name));
-    if (a == meta->end() || !checkMeta(*a->value)) return 0;
+    auto a = meta->get(state->symbols.create(name));
+    if (!a || !checkMeta(*a->value)) return 0;
     return a->value;
 }
 
@@ -237,7 +235,7 @@ NixInt PackageInfo::queryMetaInt(const std::string & name, NixInt def)
 {
     Value * v = queryMeta(name);
     if (!v) return def;
-    if (v->type() == nInt) return v->integer;
+    if (v->type() == nInt) return v->integer();
     if (v->type() == nString) {
         /* Backwards compatibility with before we had support for
            integer meta fields. */
@@ -251,7 +249,7 @@ NixFloat PackageInfo::queryMetaFloat(const std::string & name, NixFloat def)
 {
     Value * v = queryMeta(name);
     if (!v) return def;
-    if (v->type() == nFloat) return v->fpoint;
+    if (v->type() == nFloat) return v->fpoint();
     if (v->type() == nString) {
         /* Backwards compatibility with before we had support for
            float meta fields. */
@@ -266,7 +264,7 @@ bool PackageInfo::queryMetaBool(const std::string & name, bool def)
 {
     Value * v = queryMeta(name);
     if (!v) return def;
-    if (v->type() == nBool) return v->boolean;
+    if (v->type() == nBool) return v->boolean();
     if (v->type() == nString) {
         /* Backwards compatibility with before we had support for
            Boolean meta fields. */
@@ -292,7 +290,7 @@ void PackageInfo::setMeta(const std::string & name, Value * v)
 
 
 /* Cache for already considered attrsets. */
-typedef std::set<Bindings *> Done;
+typedef std::set<const Bindings *> Done;
 
 
 /* Evaluate value `v'.  If it evaluates to a set of type `derivation',
@@ -309,9 +307,9 @@ static bool getDerivation(EvalState & state, Value & v,
 
         /* Remove spurious duplicates (e.g., a set like `rec { x =
            derivation {...}; y = x;}'. */
-        if (!done.insert(v.attrs).second) return false;
+        if (!done.insert(v.attrs()).second) return false;
 
-        PackageInfo drv(state, attrPath, v.attrs);
+        PackageInfo drv(state, attrPath, v.attrs());
 
         drv.queryName();
 
@@ -361,14 +359,14 @@ static void getDerivations(EvalState & state, Value & vIn,
 
         /* !!! undocumented hackery to support combining channels in
            nix-env.cc. */
-        bool combineChannels = v.attrs->find(state.symbols.create("_combineChannels")) != v.attrs->end();
+        bool combineChannels = v.attrs()->get(state.symbols.create("_combineChannels"));
 
         /* Consider the attributes in sorted order to get more
            deterministic behaviour in nix-env operations (e.g. when
            there are names clashes between derivations, the derivation
            bound to the attribute with the "lower" name should take
            precedence). */
-        for (auto & i : v.attrs->lexicographicOrder(state.symbols)) {
+        for (auto & i : v.attrs()->lexicographicOrder(state.symbols)) {
             debug("evaluating attribute '%1%'", state.symbols[i->name]);
             if (!std::regex_match(std::string(state.symbols[i->name]), attrRegex))
                 continue;
@@ -380,8 +378,8 @@ static void getDerivations(EvalState & state, Value & vIn,
                    should we recurse into it?  => Only if it has a
                    `recurseForDerivations = true' attribute. */
                 if (i->value->type() == nAttrs) {
-                    Bindings::iterator j = i->value->attrs->find(state.sRecurseForDerivations);
-                    if (j != i->value->attrs->end() && state.forceBool(*j->value, j->pos, "while evaluating the attribute `recurseForDerivations`"))
+                    auto j = i->value->attrs()->get(state.sRecurseForDerivations);
+                    if (j && state.forceBool(*j->value, j->pos, "while evaluating the attribute `recurseForDerivations`"))
                         getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures);
                 }
             }

src/libexpr/get-drvs.hh

@@ -33,9 +33,9 @@ private:
      */
     bool failed = false;
 
-    Bindings * attrs = nullptr, * meta = nullptr;
+    const Bindings * attrs = nullptr, * meta = nullptr;
 
-    Bindings * getMeta();
+    const Bindings * getMeta();
 
     bool checkMeta(Value & v);
 
@@ -46,7 +46,7 @@ public:
     std::string attrPath;
 
     PackageInfo(EvalState & state) : state(&state) { };
-    PackageInfo(EvalState & state, std::string attrPath, Bindings * attrs);
+    PackageInfo(EvalState & state, std::string attrPath, const Bindings * attrs);
     PackageInfo(EvalState & state, ref<Store> store, const std::string & drvPathWithOutputs);
 
     std::string queryName() const;

src/libexpr/json-to-value.cc

@@ -57,11 +57,10 @@ class JSONSax : nlohmann::json_sax<json> {
         ValueVector values;
         std::unique_ptr<JSONState> resolve(EvalState & state) override
         {
-            Value & v = parent->value(state);
-            state.mkList(v, values.size());
-            for (size_t n = 0; n < values.size(); ++n) {
-                v.listElems()[n] = values[n];
-            }
+            auto list = state.buildList(values.size());
+            for (const auto & [n, v2] : enumerate(list))
+                v2 = values[n];
+            parent->value(state).mkList(list);
             return std::move(parent);
         }
         void add() override {

src/libexpr/local.mk

@@ -11,8 +11,11 @@ libexpr_SOURCES := \
   $(wildcard $(d)/flake/*.cc) \
   $(d)/lexer-tab.cc \
   $(d)/parser-tab.cc
+# Not just for this library itself, but also for downstream libraries using this library
 
-libexpr_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/libmain -I src/libexpr
+INCLUDE_libexpr := -I $(d)
+
+libexpr_CXXFLAGS += $(INCLUDE_libutil) $(INCLUDE_libstore) $(INCLUDE_libfetchers) $(INCLUDE_libmain) $(INCLUDE_libexpr)
 
 libexpr_LIBS = libutil libstore libfetchers
 

src/libexpr/nixexpr.cc

@@ -28,12 +28,12 @@ void Expr::show(const SymbolTable & symbols, std::ostream & str) const
 
 void ExprInt::show(const SymbolTable & symbols, std::ostream & str) const
 {
-    str << v.integer;
+    str << v.integer();
 }
 
 void ExprFloat::show(const SymbolTable & symbols, std::ostream & str) const
 {
-    str << v.fpoint;
+    str << v.fpoint();
 }
 
 void ExprString::show(const SymbolTable & symbols, std::ostream & str) const

src/libexpr/primops/context.cc

@@ -137,14 +137,14 @@ static RegisterPrimOp primop_addDrvOutputDependencies({
     .name = "__addDrvOutputDependencies",
     .args = {"s"},
     .doc = R"(
-      Create a copy of the given string where a single consant string context element is turned into a "derivation deep" string context element.
+      Create a copy of the given string where a single constant string context element is turned into a "derivation deep" string context element.
 
       The store path that is the constant string context element should point to a valid derivation, and end in `.drv`.
 
       The original string context element must not be empty or have multiple elements, and it must not have any other type of element other than a constant or derivation deep element.
       The latter is supported so this function is idempotent.
 
-      This is the opposite of [`builtins.unsafeDiscardOutputDependency`](#builtins-addDrvOutputDependencies).
+      This is the opposite of [`builtins.unsafeDiscardOutputDependency`](#builtins-unsafeDiscardOutputDependency).
     )",
     .fun = prim_addDrvOutputDependencies
 });
@@ -207,10 +207,10 @@ static void prim_getContext(EvalState & state, const PosIdx pos, Value * * args,
         if (info.second.allOutputs)
             infoAttrs.alloc(sAllOutputs).mkBool(true);
         if (!info.second.outputs.empty()) {
-            auto & outputsVal = infoAttrs.alloc(state.sOutputs);
-            state.mkList(outputsVal, info.second.outputs.size());
+            auto list = state.buildList(info.second.outputs.size());
             for (const auto & [i, output] : enumerate(info.second.outputs))
-                (outputsVal.listElems()[i] = state.allocValue())->mkString(output);
+                (list[i] = state.allocValue())->mkString(output);
+            infoAttrs.alloc(state.sOutputs).mkList(list);
         }
         attrs.alloc(state.store->printStorePath(info.first)).mkAttrs(infoAttrs);
     }
@@ -246,7 +246,7 @@ static RegisterPrimOp primop_getContext({
 
 /* Append the given context to a given string.
 
-   See the commentary above unsafeGetContext for details of the
+   See the commentary above getContext for details of the
    context representation.
 */
 static void prim_appendContext(EvalState & state, const PosIdx pos, Value * * args, Value & v)
@@ -258,7 +258,7 @@ static void prim_appendContext(EvalState & state, const PosIdx pos, Value * * ar
 
     auto sPath = state.symbols.create("path");
     auto sAllOutputs = state.symbols.create("allOutputs");
-    for (auto & i : *args[1]->attrs) {
+    for (auto & i : *args[1]->attrs()) {
         const auto & name = state.symbols[i.name];
         if (!state.store->isStorePath(name))
             state.error<EvalError>(
@@ -269,17 +269,16 @@ static void prim_appendContext(EvalState & state, const PosIdx pos, Value * * ar
         if (!settings.readOnlyMode)
             state.store->ensurePath(namePath);
         state.forceAttrs(*i.value, i.pos, "while evaluating the value of a string context");
-        auto iter = i.value->attrs->find(sPath);
-        if (iter != i.value->attrs->end()) {
-            if (state.forceBool(*iter->value, iter->pos, "while evaluating the `path` attribute of a string context"))
+
+        if (auto attr = i.value->attrs()->get(sPath)) {
+            if (state.forceBool(*attr->value, attr->pos, "while evaluating the `path` attribute of a string context"))
                 context.emplace(NixStringContextElem::Opaque {
                     .path = namePath,
                 });
         }
 
-        iter = i.value->attrs->find(sAllOutputs);
-        if (iter != i.value->attrs->end()) {
-            if (state.forceBool(*iter->value, iter->pos, "while evaluating the `allOutputs` attribute of a string context")) {
+        if (auto attr = i.value->attrs()->get(sAllOutputs)) {
+            if (state.forceBool(*attr->value, attr->pos, "while evaluating the `allOutputs` attribute of a string context")) {
                 if (!isDerivation(name)) {
                     state.error<EvalError>(
                         "tried to add all-outputs context of %s, which is not a derivation, to a string",
@@ -292,17 +291,16 @@ static void prim_appendContext(EvalState & state, const PosIdx pos, Value * * ar
             }
         }
 
-        iter = i.value->attrs->find(state.sOutputs);
-        if (iter != i.value->attrs->end()) {
-            state.forceList(*iter->value, iter->pos, "while evaluating the `outputs` attribute of a string context");
-            if (iter->value->listSize() && !isDerivation(name)) {
+        if (auto attr = i.value->attrs()->get(state.sOutputs)) {
+            state.forceList(*attr->value, attr->pos, "while evaluating the `outputs` attribute of a string context");
+            if (attr->value->listSize() && !isDerivation(name)) {
                 state.error<EvalError>(
                     "tried to add derivation output context of %s, which is not a derivation, to a string",
                     name
                 ).atPos(i.pos).debugThrow();
             }
-            for (auto elem : iter->value->listItems()) {
-                auto outputName = state.forceStringNoCtx(*elem, iter->pos, "while evaluating an output name within a string context");
+            for (auto elem : attr->value->listItems()) {
+                auto outputName = state.forceStringNoCtx(*elem, attr->pos, "while evaluating an output name within a string context");
                 context.emplace(NixStringContextElem::Built {
                     .drvPath = makeConstantStorePathRef(namePath),
                     .output = std::string { outputName },

src/libexpr/primops/fetchClosure.cc

@@ -121,7 +121,7 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * arg
     std::optional<StorePathOrGap> toPath;
     std::optional<bool> inputAddressedMaybe;
 
-    for (auto & attr : *args[0]->attrs) {
+    for (auto & attr : *args[0]->attrs()) {
         const auto & attrName = state.symbols[attr.name];
         auto attrHint = [&]() -> std::string {
             return "while evaluating the '" + attrName + "' attribute passed to builtins.fetchClosure";

src/libexpr/primops/fetchMercurial.cc

@@ -20,7 +20,7 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value * * a
 
     if (args[0]->type() == nAttrs) {
 
-        for (auto & attr : *args[0]->attrs) {
+        for (auto & attr : *args[0]->attrs()) {
             std::string_view n(state.symbols[attr.name]);
             if (n == "url")
                 url = state.coerceToString(attr.pos, *attr.value, context,