Merge pull request #10669 from NixOS/backport-10588-to-2.22-maintenance

[Backport 2.22-maintenance] Fix fetchGit/fetchTree for nested submodules

.clang-format

@@ -28,3 +28,5 @@ EmptyLineBeforeAccessModifier: Leave
 #PackConstructorInitializers: BinPack
 BreakBeforeBinaryOperators: NonAssignment
 AlwaysBreakBeforeMultilineStrings: true
+IndentPPDirectives: AfterHash
+PPIndentWidth: 2

.github/CODEOWNERS

@@ -14,4 +14,4 @@
 src/libexpr/primops.cc @roberth
 
 # Libstore layer
-/src/libstore @thufschmitt
+/src/libstore @thufschmitt @ericson2314

.github/labeler.yml

@@ -1,3 +1,16 @@
+"c api":
+  - changed-files:
+    - any-glob-to-any-file: "src/lib*-c/**/*"
+    - any-glob-to-any-file: "test/unit/**/nix_api_*"
+    - any-glob-to-any-file: "doc/external-api/**/*"
+
+"contributor-experience":
+  - changed-files:
+    - any-glob-to-any-file: "CONTRIBUTING.md"
+    - any-glob-to-any-file: ".github/ISSUE_TEMPLATE/*"
+    - any-glob-to-any-file: ".github/PULL_REQUEST_TEMPLATE.md"
+    - any-glob-to-any-file: "doc/manual/src/contributing/**"
+
 "documentation":
   - changed-files:
     - any-glob-to-any-file: "doc/manual/*"

.github/workflows/backport.yml

@@ -21,7 +21,7 @@ jobs:
           fetch-depth: 0
       - name: Create backport PRs
         # should be kept in sync with `version`
-        uses: zeebe-io/backport-action@v2.4.1
+        uses: zeebe-io/backport-action@v2.5.0
         with:
           # Config README: https://github.com/zeebe-io/backport-action#backport-action
           github_token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/ci.yml

@@ -20,7 +20,7 @@ jobs:
     - uses: actions/checkout@v4
       with:
         fetch-depth: 0
-    - uses: cachix/install-nix-action@v25
+    - uses: cachix/install-nix-action@v26
       with:
         # The sandbox would otherwise be disabled by default on Darwin
         extra_nix_config: "sandbox = true"
@@ -62,9 +62,9 @@ jobs:
       with:
         fetch-depth: 0
     - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
-    - uses: cachix/install-nix-action@v25
+    - uses: cachix/install-nix-action@v26
       with:
-        install_url: https://releases.nixos.org/nix/nix-2.13.3/install
+        install_url: https://releases.nixos.org/nix/nix-2.20.3/install
     - uses: cachix/cachix-action@v14
       with:
         name: '${{ env.CACHIX_NAME }}'
@@ -84,7 +84,7 @@ jobs:
     steps:
     - uses: actions/checkout@v4
     - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
-    - uses: cachix/install-nix-action@v25
+    - uses: cachix/install-nix-action@v26
       with:
         install_url: '${{needs.installer.outputs.installerURL}}'
         install_options: "--tarball-url-prefix https://${{ env.CACHIX_NAME }}.cachix.org/serve"
@@ -114,9 +114,9 @@ jobs:
     - uses: actions/checkout@v4
       with:
         fetch-depth: 0
-    - uses: cachix/install-nix-action@v25
+    - uses: cachix/install-nix-action@v26
       with:
-        install_url: https://releases.nixos.org/nix/nix-2.13.3/install
+        install_url: https://releases.nixos.org/nix/nix-2.20.3/install
     - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
     - run: echo NIX_VERSION="$(nix --experimental-features 'nix-command flakes' eval .\#default.version | tr -d \")" >> $GITHUB_ENV
     - uses: cachix/cachix-action@v14
@@ -153,6 +153,17 @@ jobs:
         IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]')
 
         docker tag nix:$NIX_VERSION $IMAGE_ID:$NIX_VERSION
-        docker tag nix:$NIX_VERSION $IMAGE_ID:master
+        docker tag nix:$NIX_VERSION $IMAGE_ID:latest
         docker push $IMAGE_ID:$NIX_VERSION
+        docker push $IMAGE_ID:latest
+        # deprecated 2024-02-24
+        docker tag nix:$NIX_VERSION $IMAGE_ID:master
         docker push $IMAGE_ID:master
+
+  vm_tests:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v4
+      - uses: DeterminateSystems/nix-installer-action@main
+      - uses: DeterminateSystems/magic-nix-cache-action@main
+      - run: nix build -L .#hydraJobs.tests.githubFlakes .#hydraJobs.tests.tarballFlakes

.gitignore

@@ -10,6 +10,7 @@ perl/Makefile.config
 /stamp-h1
 /svn-revision
 /libtool
+/config/config.*
 
 # /doc/manual/
 /doc/manual/*.1
@@ -45,13 +46,19 @@ perl/Makefile.config
 /src/libexpr/parser-tab.hh
 /src/libexpr/parser-tab.output
 /src/libexpr/nix.tbl
+/src/libexpr/tests
 /tests/unit/libexpr/libnixexpr-tests
 
+# /src/libfetchers
+/tests/unit/libfetchers/libnixfetchers-tests
+
 # /src/libstore/
 *.gen.*
+/src/libstore/tests
 /tests/unit/libstore/libnixstore-tests
 
 # /src/libutil/
+/src/libutil/tests
 /tests/unit/libutil/libnixutil-tests
 
 /src/nix/nix
@@ -111,8 +118,6 @@ perl/Makefile.config
 /misc/systemd/nix-daemon.conf
 /misc/upstart/nix-daemon.conf
 
-/src/resolve-system-dependencies/resolve-system-dependencies
-
 outputs/
 
 *.a
@@ -138,6 +143,7 @@ GTAGS
 
 # auto-generated compilation database
 compile_commands.json
+*.compile_commands.json
 
 nix-rust/target
 
@@ -148,6 +154,8 @@ result-*
 .vscode/
 .idea/
 
+.pre-commit-config.yaml
+
 # clangd and possibly more
 .cache/
 

.version

@@ -1 +1 @@
-2.21.0
+2.22.1

CONTRIBUTING.md

@@ -67,7 +67,7 @@ Check out the [security policy](https://github.com/NixOS/nix/security/policy).
    - [ ] API documentation in header files
    - [ ] Code and comments are self-explanatory
    - [ ] Commit message explains **why** the change was made
-   - [ ] New feature or incompatible change: updated [release notes](./doc/manual/src/release-notes/rl-next.md)
+   - [ ] New feature or incompatible change: [add a release note](https://nixos.org/manual/nix/stable/contributing/hacking#add-a-release-note)
 
 7. If you need additional feedback or help to getting pull request into shape, ask other contributors using [@mentions](https://docs.github.com/en/get-started/writing-on-github/getting-started-with-writing-and-formatting-on-github/basic-writing-and-formatting-syntax#mentioning-people-and-teams).
 

Makefile

@@ -7,6 +7,8 @@ clean-files += $(buildprefix)Makefile.config
 
 # List makefiles
 
+include mk/platform.mk
+
 ifeq ($(ENABLE_BUILD), yes)
 makefiles = \
   mk/precompiled-headers.mk \
@@ -18,8 +20,14 @@ makefiles = \
   src/libexpr/local.mk \
   src/libcmd/local.mk \
   src/nix/local.mk \
-  src/resolve-system-dependencies/local.mk \
+  src/libutil-c/local.mk \
+  src/libstore-c/local.mk \
+  src/libexpr-c/local.mk
+
+ifdef HOST_UNIX
+makefiles += \
   scripts/local.mk \
+  maintainers/local.mk \
   misc/bash/local.mk \
   misc/fish/local.mk \
   misc/zsh/local.mk \
@@ -27,6 +35,7 @@ makefiles = \
   misc/launchd/local.mk \
   misc/upstart/local.mk
 endif
+endif
 
 ifeq ($(ENABLE_UNIT_TESTS), yes)
 makefiles += \
@@ -34,18 +43,23 @@ makefiles += \
   tests/unit/libutil-support/local.mk \
   tests/unit/libstore/local.mk \
   tests/unit/libstore-support/local.mk \
+  tests/unit/libfetchers/local.mk \
   tests/unit/libexpr/local.mk \
   tests/unit/libexpr-support/local.mk
 endif
 
 ifeq ($(ENABLE_FUNCTIONAL_TESTS), yes)
+ifdef HOST_UNIX
 makefiles += \
   tests/functional/local.mk \
   tests/functional/ca/local.mk \
+  tests/functional/git-hashing/local.mk \
   tests/functional/dyn-drv/local.mk \
+  tests/functional/local-overlay-store/local.mk \
   tests/functional/test-libstoreconsumer/local.mk \
   tests/functional/plugins/local.mk
 endif
+endif
 
 # Some makefiles require access to built programs and must be included late.
 makefiles-late =
@@ -58,6 +72,10 @@ ifeq ($(ENABLE_INTERNAL_API_DOCS), yes)
 makefiles-late += doc/internal-api/local.mk
 endif
 
+ifeq ($(ENABLE_EXTERNAL_API_DOCS), yes)
+makefiles-late += doc/external-api/local.mk
+endif
+
 # Miscellaneous global Flags
 
 OPTIMIZE = 1
@@ -67,10 +85,9 @@ ifeq ($(OPTIMIZE), 1)
   GLOBAL_LDFLAGS += $(CXXLTO)
 else
   GLOBAL_CXXFLAGS += -O0 -U_FORTIFY_SOURCE
+  unexport NIX_HARDENING_ENABLE
 endif
 
-include mk/platform.mk
-
 ifdef HOST_WINDOWS
   # Windows DLLs are stricter about symbol visibility than Unix shared
   # objects --- see https://gcc.gnu.org/wiki/Visibility for details.
@@ -81,7 +98,7 @@ ifdef HOST_WINDOWS
   GLOBAL_LDFLAGS += -Wl,--export-all-symbols
 endif
 
-GLOBAL_CXXFLAGS += -g -Wall -include $(buildprefix)config.h -std=c++2a -I src
+GLOBAL_CXXFLAGS += -g -Wall -Wimplicit-fallthrough -include $(buildprefix)config.h -std=c++2a -I src
 
 # Include the main lib, causing rules to be defined
 
@@ -121,3 +138,10 @@ internal-api-html:
 	@echo "Internal API docs are disabled. Configure with '--enable-internal-api-docs', or avoid calling 'make internal-api-html'."
 	@exit 1
 endif
+
+ifneq ($(ENABLE_EXTERNAL_API_DOCS), yes)
+.PHONY: external-api-html
+external-api-html:
+	@echo "External API docs are disabled. Configure with '--enable-external-api-docs', or avoid calling 'make external-api-html'."
+	@exit 1
+endif

Makefile.config.in

@@ -12,6 +12,7 @@ ENABLE_BUILD = @ENABLE_BUILD@
 ENABLE_DOC_GEN = @ENABLE_DOC_GEN@
 ENABLE_FUNCTIONAL_TESTS = @ENABLE_FUNCTIONAL_TESTS@
 ENABLE_INTERNAL_API_DOCS = @ENABLE_INTERNAL_API_DOCS@
+ENABLE_EXTERNAL_API_DOCS = @ENABLE_EXTERNAL_API_DOCS@
 ENABLE_S3 = @ENABLE_S3@
 ENABLE_UNIT_TESTS = @ENABLE_UNIT_TESTS@
 GTEST_LIBS = @GTEST_LIBS@

configure.ac

@@ -150,6 +150,11 @@ AC_ARG_ENABLE(unit-tests, AS_HELP_STRING([--disable-unit-tests],[Do not build th
   ENABLE_UNIT_TESTS=$enableval, ENABLE_UNIT_TESTS=$ENABLE_BUILD)
 AC_SUBST(ENABLE_UNIT_TESTS)
 
+# Build external API docs by default
+AC_ARG_ENABLE(external_api_docs, AS_HELP_STRING([--enable-external-api-docs],[Build API docs for Nix's C interface]),
+  external_api_docs=$enableval, external_api_docs=yes)
+AC_SUBST(external_api_docs)
+
 AS_IF(
   [test "$ENABLE_BUILD" == "no" && test "$ENABLE_UNIT_TESTS" == "yes"],
   [AC_MSG_ERROR([Cannot enable unit tests when building overall is disabled. Please do not pass '--enable-unit-tests' or do not pass '--disable-build'.])])
@@ -172,6 +177,10 @@ AC_ARG_ENABLE(internal-api-docs, AS_HELP_STRING([--enable-internal-api-docs],[Bu
   ENABLE_INTERNAL_API_DOCS=$enableval, ENABLE_INTERNAL_API_DOCS=no)
 AC_SUBST(ENABLE_INTERNAL_API_DOCS)
 
+AC_ARG_ENABLE(external-api-docs, AS_HELP_STRING([--enable-external-api-docs],[Build API docs for Nix's external unstable C interfaces]),
+  ENABLE_EXTERNAL_API_DOCS=$enableval, ENABLE_EXTERNAL_API_DOCS=no)
+AC_SUBST(ENABLE_EXTERNAL_API_DOCS)
+
 AS_IF(
   [test "$ENABLE_FUNCTIONAL_TESTS" == "yes" || test "$ENABLE_DOC_GEN" == "yes"],
   [NEED_PROG(jq, jq)])
@@ -308,6 +317,17 @@ case "$host_os" in
                         [CXXFLAGS="$LIBSECCOMP_CFLAGS $CXXFLAGS"])
       have_seccomp=1
       AC_DEFINE([HAVE_SECCOMP], [1], [Whether seccomp is available and should be used for sandboxing.])
+      AC_COMPILE_IFELSE([
+        AC_LANG_SOURCE([[
+          #include <seccomp.h>
+          #ifndef __SNR_fchmodat2
+          # error "Missing support for fchmodat2"
+          #endif
+        ]])
+      ], [], [
+        echo "libseccomp is missing __SNR_fchmodat2. Please provide libseccomp 2.5.5 or later"
+        exit 1
+      ])
     else
       have_seccomp=
     fi

doc/external-api/.gitignore

@@ -0,0 +1,3 @@
+/doxygen.cfg
+/html
+/latex

doc/external-api/README.md

@@ -0,0 +1,121 @@
+# Getting started
+
+> **Warning** These bindings are **experimental**, which means they can change
+> at any time or be removed outright; nevertheless the plan is to provide a
+> stable external C API to the Nix language and the Nix store.
+
+The language library allows evaluating Nix expressions and interacting with Nix
+language values. The Nix store API is still rudimentary, and only allows
+initialising and connecting to a store for the Nix language evaluator to
+interact with.
+
+Currently there are two ways to interface with the Nix language evaluator
+programmatically:
+
+1. Embedding the evaluator
+2. Writing language plug-ins
+
+Embedding means you link the Nix C libraries in your program and use them from
+there. Adding a plug-in means you make a library that gets loaded by the Nix
+language evaluator, specified through a configuration option.
+
+Many of the components and mechanisms involved are not yet documented, therefore
+please refer to the [Nix source code](https://github.com/NixOS/nix/) for
+details. Additions to in-code documentation and the reference manual are highly
+appreciated.
+
+The following examples, for simplicity, don't include error handling. See the
+[Handling errors](@ref errors) section for more information.
+
+# Embedding the Nix Evaluator
+
+In this example we programmatically start the Nix language evaluator with a
+dummy store (that has no store paths and cannot be written to), and evaluate the
+Nix expression `builtins.nixVersion`.
+
+**main.c:**
+
+```C
+#include <nix_api_util.h>
+#include <nix_api_expr.h>
+#include <nix_api_value.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+// NOTE: This example lacks all error handling. Production code must check for
+// errors, as some return values will be undefined.
+
+void my_get_string_cb(const char * start, unsigned int n, char ** user_data)
+{
+    *user_data = strdup(start);
+}
+
+int main()
+{
+    nix_libexpr_init(NULL);
+
+    Store * store = nix_store_open(NULL, "dummy://", NULL);
+    EvalState * state = nix_state_create(NULL, NULL, store); // empty search path (NIX_PATH)
+    Value * value = nix_alloc_value(NULL, state);
+
+    nix_expr_eval_from_string(NULL, state, "builtins.nixVersion", ".", value);
+    nix_value_force(NULL, state, value);
+
+    char * version;
+    nix_get_string(NULL, value, my_get_string_cb, version);
+    printf("Nix version: %s\n", version);
+
+    free(version);
+    nix_gc_decref(NULL, value);
+    nix_state_free(state);
+    nix_store_free(store);
+    return 0;
+}
+```
+
+**Usage:**
+
+```ShellSession
+$ gcc main.c $(pkg-config nix-expr-c --libs --cflags) -o main
+$ ./main
+Nix version: 2.17
+```
+
+# Writing a Nix language plug-in
+
+In this example we add a custom primitive operation (_primop_) to `builtins`. It
+will increment the argument if it is an integer and throw an error otherwise.
+
+**plugin.c:**
+
+```C
+#include <nix_api_util.h>
+#include <nix_api_expr.h>
+#include <nix_api_value.h>
+
+void increment(void* user_data, nix_c_context* ctx, EvalState* state, Value** args, Value* v) {
+    nix_value_force(NULL, state, args[0]);
+    if (nix_get_type(NULL, args[0]) == NIX_TYPE_INT) {
+      nix_init_int(NULL, v, nix_get_int(NULL, args[0]) + 1);
+    } else {
+      nix_set_err_msg(ctx, NIX_ERR_UNKNOWN, "First argument should be an integer.");
+    }
+}
+
+void nix_plugin_entry() {
+  const char* args[] = {"n", NULL};
+  PrimOp *p = nix_alloc_primop(NULL, increment, 1, "increment", args, "Example custom built-in function: increments an integer", NULL);
+  nix_register_primop(NULL, p);
+  nix_gc_decref(NULL, p);
+}
+```
+
+**Usage:**
+
+```ShellSession
+$ gcc plugin.c $(pkg-config nix-expr-c --libs --cflags) -shared -o plugin.so
+$ nix --plugin-files ./plugin.so repl
+nix-repl> builtins.increment 1
+2
+```

doc/external-api/doxygen.cfg.in

@@ -0,0 +1,57 @@
+# Doxyfile 1.9.5
+
+# The PROJECT_NAME tag is a single word (or a sequence of words surrounded by
+# double-quotes, unless you are using Doxywizard) that should identify the
+# project for which the documentation is generated. This name is used in the
+# title of most generated pages and in a few other places.
+# The default value is: My Project.
+
+PROJECT_NAME           = "Nix"
+
+# The PROJECT_NUMBER tag can be used to enter a project or revision number. This
+# could be handy for archiving the generated documentation or if some version
+# control system is used.
+
+PROJECT_NUMBER         = @PACKAGE_VERSION@
+
+# Using the PROJECT_BRIEF tag one can provide an optional one line description
+# for a project that appears at the top of each page and should give viewer a
+# quick idea about the purpose of the project. Keep the description short.
+
+PROJECT_BRIEF          = "Nix, the purely functional package manager: C API (experimental)"
+
+# If the GENERATE_LATEX tag is set to YES, doxygen will generate LaTeX output.
+# The default value is: YES.
+
+GENERATE_LATEX         = NO
+
+# The INPUT tag is used to specify the files and/or directories that contain
+# documented source files. You may enter file names like myfile.cpp or
+# directories like /usr/src/myproject. Separate the files or directories with
+# spaces. See also FILE_PATTERNS and EXTENSION_MAPPING
+# Note: If this tag is empty the current directory is searched.
+
+# FIXME Make this list more maintainable somehow. We could maybe generate this
+# in the Makefile, but we would need to change how `.in` files are preprocessed
+# so they can expand variables despite configure variables.
+
+INPUT                  = \
+  src/libutil-c \
+  src/libexpr-c \
+  src/libstore-c \
+  doc/external-api/README.md
+
+FILE_PATTERNS = nix_api_*.h *.md
+
+# The INCLUDE_PATH tag can be used to specify one or more directories that
+# contain include files that are not input files but should be processed by the
+# preprocessor. Note that the INCLUDE_PATH is not recursive, so the setting of
+# RECURSIVE has no effect here.
+# This tag requires that the tag SEARCH_INCLUDES is set to YES.
+
+INCLUDE_PATH           = @RAPIDCHECK_HEADERS@
+EXCLUDE_PATTERNS = *_internal.h
+GENERATE_TREEVIEW = YES
+OPTIMIZE_OUTPUT_FOR_C  = YES
+
+USE_MDFILE_AS_MAINPAGE = doc/external-api/README.md

doc/external-api/local.mk

@@ -0,0 +1,7 @@
+$(docdir)/external-api/html/index.html $(docdir)/external-api/latex: $(d)/doxygen.cfg
+	mkdir -p $(docdir)/external-api
+	{ cat $< ; echo "OUTPUT_DIRECTORY=$(docdir)/external-api" ; } | doxygen -
+
+# Generate the HTML API docs for Nix's unstable C bindings
+.PHONY: external-api-html
+external-api-html: $(docdir)/external-api/html/index.html

doc/manual/custom.css

@@ -1,3 +1,25 @@
+:root {
+    --sidebar-width: 23em;
+}
+
+h1.menu-title::before {
+  content: "";
+  background-image: url("./favicon.svg");
+  padding: 1.25em;
+  background-position: center center;
+  background-size: 2em;
+  background-repeat: no-repeat;
+}
+
+
+h1.menu-title {
+  padding: 0.5em;
+}
+
+.sidebar .sidebar-scrollbox {
+  padding: 1em;
+}
+
 h1:not(:first-of-type) {
     margin-top: 1.3em;
 }

doc/manual/local.mk

@@ -175,6 +175,16 @@ $(d)/src/SUMMARY-rl-next.md: $(d)/src/release-notes/rl-next.md
 # Generate the HTML manual.
 .PHONY: manual-html
 manual-html: $(docdir)/manual/index.html
+
+# Open the built HTML manual in the default browser.
+manual-html-open: $(docdir)/manual/index.html
+	@echo "  OPEN  " $<; \
+	  xdg-open $< \
+		  || open $< \
+			|| { \
+		echo "Could not open the manual in a browser. Please open '$<'" >&2; \
+		false; \
+		}
 install: $(docdir)/manual/index.html
 
 # Generate 'nix' manpages.
@@ -207,7 +217,7 @@ doc/manual/generated/man1/nix3-manpages: $(d)/src/command-ref/new-cli
 # `@docroot@` is to be preserved for documenting the mechanism
 # FIXME: maybe contributing guides should live right next to the code
 # instead of in the manual
-$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/store/types $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md $(d)/src/language/builtin-constants.md $(d)/src/release-notes/rl-next.md
+$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/store/types $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md $(d)/src/language/builtin-constants.md $(d)/src/release-notes/rl-next.md $(d)/src/figures $(d)/src/favicon.png $(d)/src/favicon.svg
 	$(trace-gen) \
 		tmp="$$(mktemp -d)"; \
 		cp -r doc/manual "$$tmp"; \

doc/manual/redirects.js

@@ -14,7 +14,7 @@
 
 const redirects = {
  "index.html": {
-    "part-advanced-topics": "advanced-topics/advanced-topics.html",
+    "part-advanced-topics": "advanced-topics/index.html",
     "chap-tuning-cores-and-jobs": "advanced-topics/cores-vs-jobs.html",
     "chap-diff-hook": "advanced-topics/diff-hook.html",
     "check-dirs-are-unregistered": "advanced-topics/diff-hook.html#check-dirs-are-unregistered",
@@ -22,7 +22,7 @@ const redirects = {
     "chap-post-build-hook": "advanced-topics/post-build-hook.html",
     "chap-post-build-hook-caveats": "advanced-topics/post-build-hook.html#implementation-caveats",
     "chap-writing-nix-expressions": "language/index.html",
-    "part-command-ref": "command-ref/command-ref.html",
+    "part-command-ref": "command-ref/index.html",
     "conf-allow-import-from-derivation": "command-ref/conf-file.html#conf-allow-import-from-derivation",
     "conf-allow-new-privileges": "command-ref/conf-file.html#conf-allow-new-privileges",
     "conf-allowed-uris": "command-ref/conf-file.html#conf-allowed-uris",
@@ -261,7 +261,7 @@ const redirects = {
     "sec-installer-proxy-settings": "installation/env-variables.html#proxy-environment-variables",
     "sec-nix-ssl-cert-file": "installation/env-variables.html#nix_ssl_cert_file",
     "sec-nix-ssl-cert-file-with-nix-daemon-and-macos": "installation/env-variables.html#nix_ssl_cert_file-with-macos-and-the-nix-daemon",
-    "chap-installation": "installation/installation.html",
+    "chap-installation": "installation/index.html",
     "ch-installing-binary": "installation/installing-binary.html",
     "sect-macos-installation": "installation/installing-binary.html#macos-installation",
     "sect-macos-installation-change-store-prefix": "installation/installing-binary.html#macos-installation",
@@ -288,7 +288,7 @@ const redirects = {
     "ssec-copy-closure": "package-management/copy-closure.html",
     "sec-garbage-collection": "package-management/garbage-collection.html",
     "ssec-gc-roots": "package-management/garbage-collector-roots.html",
-    "chap-package-management": "package-management/package-management.html",
+    "chap-package-management": "package-management/index.html",
     "sec-profiles": "package-management/profiles.html",
     "ssec-s3-substituter": "package-management/s3-substituter.html",
     "ssec-s3-substituter-anonymous-reads": "package-management/s3-substituter.html#anonymous-reads-to-your-s3-compatible-binary-cache",
@@ -297,7 +297,7 @@ const redirects = {
     "sec-sharing-packages": "package-management/sharing-packages.html",
     "ssec-ssh-substituter": "package-management/ssh-substituter.html",
     "chap-quick-start": "quick-start.html",
-    "sec-relnotes": "release-notes/release-notes.html",
+    "sec-relnotes": "release-notes/index.html",
     "ch-relnotes-0.10.1": "release-notes/rl-0.10.1.html",
     "ch-relnotes-0.10": "release-notes/rl-0.10.html",
     "ssec-relnotes-0.11": "release-notes/rl-0.11.html",

doc/manual/rl-next/better-errors-in-nix-repl.md

@@ -1,40 +0,0 @@
----
-synopsis: Concise error printing in `nix repl`
-prs: 9928
----
-
-Previously, if an element of a list or attribute set threw an error while
-evaluating, `nix repl` would print the entire error (including source location
-information) inline. This output was clumsy and difficult to parse:
-
-```
-nix-repl> { err = builtins.throw "uh oh!"; }
-{ err = «error:
-       … while calling the 'throw' builtin
-         at «string»:1:9:
-            1| { err = builtins.throw "uh oh!"; }
-             |         ^
-
-       error: uh oh!»; }
-```
-
-Now, only the error message is displayed, making the output much more readable.
-```
-nix-repl> { err = builtins.throw "uh oh!"; }
-{ err = «error: uh oh!»; }
-```
-
-However, if the whole expression being evaluated throws an error, source
-locations and (if applicable) a stack trace are printed, just like you'd expect:
-
-```
-nix-repl> builtins.throw "uh oh!"
-error:
-       … while calling the 'throw' builtin
-         at «string»:1:1:
-            1| builtins.throw "uh oh!"
-             | ^
-
-       error: uh oh!
-```
-

doc/manual/rl-next/debugger-locals-for-let-expressions.md

@@ -1,9 +0,0 @@
----
-synopsis: "`--debugger` can now access bindings from `let` expressions"
-prs: 9918
-issues: 8827.
----
-
-Breakpoints and errors in the bindings of a `let` expression can now access
-those bindings in the debugger. Previously, only the body of `let` expressions
-could access those bindings.

doc/manual/rl-next/debugger-on-trace.md

@@ -1,9 +0,0 @@
----
-synopsis: Enter the `--debugger` when `builtins.trace` is called if `debugger-on-trace` is set
-prs: 9914
----
-
-If the `debugger-on-trace` option is set and `--debugger` is given,
-`builtins.trace` calls will behave similarly to `builtins.break` and will enter
-the debug REPL. This is useful for determining where warnings are being emitted
-from.

doc/manual/rl-next/debugger-positions.md

@@ -1,25 +0,0 @@
----
-synopsis: Debugger prints source position information
-prs: 9913
----
-
-The `--debugger` now prints source location information, instead of the
-pointers of source location information. Before:
-
-```
-nix-repl> :bt
-0: while evaluating the attribute 'python311.pythonForBuild.pkgs'
-0x600001522598
-```
-
-After:
-
-```
-0: while evaluating the attribute 'python311.pythonForBuild.pkgs'
-/nix/store/hg65h51xnp74ikahns9hyf3py5mlbbqq-source/overrides/default.nix:132:27
-
-   131|
-   132|       bootstrappingBase = pkgs.${self.python.pythonAttr}.pythonForBuild.pkgs;
-      |                           ^
-   133|     in
-```

doc/manual/rl-next/enter-debugger-more-reliably-in-let-and-calls.md

@@ -1,25 +0,0 @@
----
-synopsis: The `--debugger` will start more reliably in `let` expressions and function calls
-prs: 9917
-issues: 6649
----
-
-Previously, if you attempted to evaluate this file with the debugger:
-
-```nix
-let
-  a = builtins.trace "before inner break" (
-    builtins.break "hello"
-  );
-  b = builtins.trace "before outer break" (
-    builtins.break a
-  );
-in
-  b
-```
-
-Nix would correctly enter the debugger at `builtins.break a`, but if you asked
-it to `:continue`, it would skip over the `builtins.break "hello"` expression
-entirely.
-
-Now, Nix will correctly enter the debugger at both breakpoints.

doc/manual/rl-next/lambda-printing.md

@@ -1,50 +0,0 @@
----
-synopsis: Functions are printed with more detail
-prs: 9606
-issues: 7145
----
-
-Functions and `builtins` are printed with more detail in `nix repl`, `nix
-eval`, `builtins.trace`, and most other places values are printed.
-
-Before:
-
-```
-$ nix repl nixpkgs
-nix-repl> builtins.map
-«primop»
-
-nix-repl> builtins.map lib.id
-«primop-app»
-
-nix-repl> builtins.trace lib.id "my-value"
-trace: <LAMBDA>
-"my-value"
-
-$ nix eval --file functions.nix
-{ id = <LAMBDA>; primop = <PRIMOP>; primop-app = <PRIMOP-APP>; }
-```
-
-After:
-
-```
-$ nix repl nixpkgs
-nix-repl> builtins.map
-«primop map»
-
-nix-repl> builtins.map lib.id
-«partially applied primop map»
-
-nix-repl> builtins.trace lib.id "my-value"
-trace: «lambda id @ /nix/store/8rrzq23h2zq7sv5l2vhw44kls5w0f654-source/lib/trivial.nix:26:5»
-"my-value"
-
-$ nix eval --file functions.nix
-{ id = «lambda id @ /Users/wiggles/nix/functions.nix:2:8»; primop = «primop map»; primop-app = «partially applied primop map»; }
-```
-
-This was actually released in Nix 2.20, but wasn't added to the release notes
-so we're announcing it here. The historical release notes have been updated as well.
-
-[type-error]: https://github.com/NixOS/nix/pull/9753
-[coercion-error]: https://github.com/NixOS/nix/pull/9754

doc/manual/rl-next/leading-period.md

@@ -1,10 +0,0 @@
----
-synopsis: Store paths are allowed to start with `.`
-issues: 912
-prs: 9867 9091 9095 9120 9121 9122 9130 9219 9224
----
-
-Leading periods were allowed by accident in Nix 2.4. The Nix team has considered this to be a bug, but this behavior has since been relied on by users, leading to unnecessary difficulties.
-From now on, leading periods are officially, definitively supported. The names `.` and `..` are disallowed, as well as those starting with `.-` or `..-`.
-
-Nix versions that denied leading periods are documented [in the issue](https://github.com/NixOS/nix/issues/912#issuecomment-1919583286).

doc/manual/rl-next/more-commands-respect-ctrl-c.md

@@ -1,13 +0,0 @@
----
-synopsis: Nix commands respect Ctrl-C
-prs: 9687 6995
-issues: 7245
----
-
-Previously, many Nix commands would hang indefinitely if Ctrl-C was pressed
-while performing various operations (including `nix develop`, `nix flake
-update`, and so on). With several fixes to Nix's signal handlers, Nix commands
-will now exit quickly after Ctrl-C is pressed.
-
-This was actually released in Nix 2.20, but wasn't added to the release notes
-so we're announcing it here. The historical release notes have been updated as well.

doc/manual/rl-next/pretty-print-in-nix-repl.md

@@ -1,24 +0,0 @@
----
-synopsis: "`nix repl` pretty-prints values"
-prs: 9931
----
-
-`nix repl` will now pretty-print values:
-
-```
-{
-  attrs = {
-    a = {
-      b = {
-        c = { };
-      };
-    };
-  };
-  list = [ 1 ];
-  list' = [
-    1
-    2
-    3
-  ];
-}
-```

doc/manual/rl-next/reduce-debugger-clutter.md

@@ -1,37 +0,0 @@
----
-synopsis: "Visual clutter in `--debugger` is reduced"
-prs: 9919
----
-
-Before:
-```
-info: breakpoint reached
-
-
-Starting REPL to allow you to inspect the current state of the evaluator.
-
-Welcome to Nix 2.20.0pre20231222_dirty. Type :? for help.
-
-nix-repl> :continue
-error: uh oh
-
-
-Starting REPL to allow you to inspect the current state of the evaluator.
-
-Welcome to Nix 2.20.0pre20231222_dirty. Type :? for help.
-
-nix-repl>
-```
-
-After:
-
-```
-info: breakpoint reached
-
-Nix 2.20.0pre20231222_dirty debugger
-Type :? for help.
-nix-repl> :continue
-error: uh oh
-
-nix-repl>
-```

doc/manual/rl-next/repl-ctrl-c-while-printing.md

@@ -1,8 +0,0 @@
----
-synopsis: "`nix repl` now respects Ctrl-C while printing values"
-prs: 9927
----
-
-`nix repl` will now halt immediately when Ctrl-C is pressed while it's printing
-a value. This is useful if you got curious about what would happen if you
-printed all of Nixpkgs.

doc/manual/rl-next/repl-cycle-detection.md

@@ -1,22 +0,0 @@
----
-synopsis: Cycle detection in `nix repl` is simpler and more reliable
-prs: 9926
-issues: 8672
----
-
-The cycle detection in `nix repl`, `nix eval`, `builtins.trace`, and everywhere
-else values are printed is now simpler and matches the cycle detection in
-`nix-instantiate --eval` output.
-
-Before:
-
-```
-nix eval --expr 'let self = { inherit self; }; in self'
-{ self = { self = «repeated»; }; }
-```
-
-After:
-
-```
-{ self = «repeated»; }
-```

doc/manual/rl-next/stack-size-macos.md

@@ -1,9 +0,0 @@
----
-synopsis: Stack size is increased on macOS
-prs: 9860
----
-
-Previously, Nix would set the stack size to 64MiB on Linux, but would leave the
-stack size set to the default (approximately 8KiB) on macOS. Now, the stack
-size is correctly set to 64MiB on macOS as well, which should reduce stack
-overflow segfaults in deeply-recursive Nix expressions.

doc/manual/src/SUMMARY.md.in

@@ -110,6 +110,7 @@
     - [Derivation](protocols/json/derivation.md)
   - [Serving Tarball Flakes](protocols/tarball-fetcher.md)
   - [Store Path Specification](protocols/store-path.md)
+  - [Nix Archive (NAR) Format](protocols/nix-archive.md)
   - [Derivation "ATerm" file format](protocols/derivation-aterm.md)
 - [Glossary](glossary.md)
 - [Contributing](contributing/index.md)
@@ -119,8 +120,10 @@
   - [Experimental Features](contributing/experimental-features.md)
   - [CLI guideline](contributing/cli-guideline.md)
   - [C++ style guide](contributing/cxx.md)
-- [Release Notes](release-notes/index.md)
+- [Releases](release-notes/index.md)
 {{#include ./SUMMARY-rl-next.md}}
+  - [Release 2.22 (2024-04-23)](release-notes/rl-2.22.md)
+  - [Release 2.21 (2024-03-11)](release-notes/rl-2.21.md)
   - [Release 2.20 (2024-01-29)](release-notes/rl-2.20.md)
   - [Release 2.19 (2023-11-17)](release-notes/rl-2.19.md)
   - [Release 2.18 (2023-09-20)](release-notes/rl-2.18.md)

doc/manual/src/architecture/architecture.md

@@ -69,7 +69,7 @@ It can also execute build plans to produce new data, which are made available to
 A build plan itself is a series of *build tasks*, together with their build inputs.
 
 > **Important**
-> A build task in Nix is called [derivation](../glossary.md#gloss-derivation).
+> A build task in Nix is called [derivation](@docroot@/glossary.md#gloss-derivation).
 
 Each build task has a special build input executed as *build instructions* in order to perform the build.
 The result of a build task can be input to another build task.

doc/manual/src/command-ref/nix-build.md

@@ -41,7 +41,7 @@ expression to a low-level [store derivation]) and [`nix-store
 --realise`](@docroot@/command-ref/nix-store/realise.md) (to build the store
 derivation).
 
-[store derivation]: ../glossary.md#gloss-store-derivation
+[store derivation]: @docroot@/glossary.md#gloss-store-derivation
 
 > **Warning**
 >

doc/manual/src/command-ref/nix-copy-closure.md

@@ -49,7 +49,7 @@ authentication, you can avoid typing the passphrase with `ssh-agent`.
   - `--include-outputs`\
     Also copy the outputs of [store derivation]s included in the closure.
 
-    [store derivation]: ../glossary.md#gloss-store-derivation
+    [store derivation]: @docroot@/glossary.md#gloss-store-derivation
 
   - `--use-substitutes` / `-s`\
     Attempt to download missing paths on the target machine using Nix’s

doc/manual/src/command-ref/nix-instantiate.md

@@ -23,7 +23,7 @@ It evaluates the Nix expressions in each of *files* (which defaults to
 derivation, a list of derivations, or a set of derivations. The paths
 of the resulting store derivations are printed on standard output.
 
-[store derivation]: ../glossary.md#gloss-store-derivation
+[store derivation]: @docroot@/glossary.md#gloss-store-derivation
 
 If *files* is the character `-`, then a Nix expression will be read from
 standard input.

doc/manual/src/command-ref/nix-store/query.md

@@ -40,12 +40,12 @@ symlink.
     derivations *paths*. These are the paths that will be produced when
     the derivation is built.
 
-    [output paths]: ../../glossary.md#gloss-output-path
+    [output paths]: @docroot@/glossary.md#gloss-output-path
 
   - `--requisites`; `-R`\
     Prints out the [closure] of the store path *paths*.
 
-    [closure]: ../../glossary.md#gloss-closure
+    [closure]: @docroot@/glossary.md#gloss-closure
 
     This query has one option:
 
@@ -66,7 +66,7 @@ symlink.
     *paths*, that is, their immediate dependencies. (For *all*
     dependencies, use `--requisites`.)
 
-    [references]: ../../glossary.md#gloss-reference
+    [references]: @docroot@/glossary.md#gloss-reference
 
   - `--referrers`\
     Prints the set of *referrers* of the store paths *paths*, that is,
@@ -90,7 +90,7 @@ symlink.
     example when *paths* were substituted from a binary cache.
     Use `--valid-derivers` instead to obtain valid paths only.
 
-    [deriver]: ../../glossary.md#gloss-deriver
+    [deriver]: @docroot@/glossary.md#gloss-deriver
 
   - `--valid-derivers`\
     Prints a set of derivation files (`.drv`) which are supposed produce

doc/manual/src/contributing/documentation.md

@@ -27,11 +27,9 @@ and open `./result-doc/share/doc/nix/manual/index.html`.
 To build the manual incrementally, [enter the development shell](./hacking.md) and run:
 
 ```console
-make manual-html -j $NIX_BUILD_CORES
+make manual-html-open -j $NIX_BUILD_CORES
 ```
 
-and open `./outputs/out/share/doc/nix/manual/language/index.html`.
-
 In order to reflect changes to the [Makefile for the manual], clear all generated files before re-building:
 
 [Makefile for the manual]: https://github.com/NixOS/nix/blob/master/doc/manual/local.mk
@@ -208,3 +206,22 @@ or inside `nix-shell` or `nix develop`:
 # make internal-api-html
 # xdg-open ./outputs/doc/share/doc/nix/internal-api/html/index.html
 ```
+
+## C API documentation (experimental)
+
+[C API documentation] is available online.
+You can also build and view it yourself:
+
+[C API documentation]: https://hydra.nixos.org/job/nix/master/external-api-docs/latest/download-by-type/doc/external-api-docs
+
+```console
+# nix build .#hydraJobs.external-api-docs
+# xdg-open ./result/share/doc/nix/external-api/html/index.html
+```
+
+or inside `nix-shell` or `nix develop`:
+
+```
+# make external-api-html
+# xdg-open ./outputs/doc/share/doc/nix/external-api/html/index.html
+```

doc/manual/src/contributing/hacking.md

@@ -144,6 +144,7 @@ Nix can be built for various platforms, as specified in [`flake.nix`]:
 - `aarch64-darwin`
 - `armv6l-linux`
 - `armv7l-linux`
+- `riscv64-linux`
 
 In order to build Nix for a different platform than the one you're currently
 on, you need a way for your current Nix installation to build code for that
@@ -166,7 +167,10 @@ or for Nix with the [`flakes`] and [`nix-command`] experimental features enabled
 $ nix build .#packages.aarch64-linux.default
 ```
 
-Cross-compiled builds are available for ARMv6 (`armv6l-linux`) and ARMv7 (`armv7l-linux`).
+Cross-compiled builds are available for:
+- `armv6l-linux`
+- `armv7l-linux`
+- `riscv64-linux`
 Add more [system types](#system-type) to `crossSystems` in `flake.nix` to bootstrap Nix on unsupported platforms.
 
 ### Building for multiple platforms at once
@@ -196,7 +200,7 @@ In order to facilitate this, Nix has some support for being built out of tree
 
 ## System type
 
-Nix uses a string with he following format to identify the *system type* or *platform* it runs on:
+Nix uses a string with the following format to identify the *system type* or *platform* it runs on:
 
 ```
 <cpu>-<os>[-<abi>]
@@ -258,10 +262,10 @@ See [supported compilation environments](#compilation-environments) and instruct
 To use the LSP with your editor, you first need to [set up `clangd`](https://clangd.llvm.org/installation#project-setup) by running:
 
 ```console
-make clean && bear -- make -j$NIX_BUILD_CORES default check install
+make compile_commands.json
 ```
 
-Configure your editor to use the `clangd` from the shell, either by running it inside the development shell, or by using [nix-direnv](https://github.com/nix-community/nix-direnv) and [the appropriate editor plugin](https://github.com/direnv/direnv/wiki#editor-integration).
+Configure your editor to use the `clangd` from the `.#native-clangStdenvPackages` shell. You can do that either by running it inside the development shell, or by using [nix-direnv](https://github.com/nix-community/nix-direnv) and [the appropriate editor plugin](https://github.com/direnv/direnv/wiki#editor-integration).
 
 > **Note**
 >
@@ -269,6 +273,29 @@ Configure your editor to use the `clangd` from the shell, either by running it i
 > Some other editors (e.g. Emacs, Vim) need a plugin to support LSP servers in general (e.g. [lsp-mode](https://github.com/emacs-lsp/lsp-mode) for Emacs and [vim-lsp](https://github.com/prabirshrestha/vim-lsp) for vim).
 > Editor-specific setup is typically opinionated, so we will not cover it here in more detail.
 
+## Formatting and pre-commit hooks
+
+You may run the formatters as a one-off using:
+
+```console
+make format
+```
+
+If you'd like to run the formatters before every commit, install the hooks:
+
+```
+pre-commit-hooks-install
+```
+
+This installs [pre-commit](https://pre-commit.com) using [cachix/git-hooks.nix](https://github.com/cachix/git-hooks.nix).
+
+When making a commit, pay attention to the console output.
+If it fails, run `git add --patch` to approve the suggestions _and commit again_.
+
+To refresh pre-commit hook's config file, do the following:
+1. Exit the development shell and start it again by running `nix develop`.
+2. If you also use the pre-commit hook, also run `pre-commit-hooks-install` again.
+
 ## Add a release note
 
 `doc/manual/rl-next` contains release notes entries for all unreleased changes.

doc/manual/src/favicon.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="587.11" height="516.604" viewBox="0 0 550.416 484.317"><defs><linearGradient id="a"><stop offset="0" style="stop-color:#699ad7;stop-opacity:1"/><stop offset=".243" style="stop-color:#7eb1dd;stop-opacity:1"/><stop offset="1" style="stop-color:#7ebae4;stop-opacity:1"/></linearGradient><linearGradient id="b"><stop offset="0" style="stop-color:#415e9a;stop-opacity:1"/><stop offset=".232" style="stop-color:#4a6baf;stop-opacity:1"/><stop offset="1" style="stop-color:#5277c3;stop-opacity:1"/></linearGradient><linearGradient xlink:href="#a" id="c" x1="200.597" x2="290.087" y1="351.411" y2="506.188" gradientTransform="translate(70.65 -1055.151)" gradientUnits="userSpaceOnUse"/><linearGradient xlink:href="#b" id="e" x1="-584.199" x2="-496.297" y1="782.336" y2="937.714" gradientTransform="translate(864.696 -1491.34)" gradientUnits="userSpaceOnUse"/></defs><g style="display:inline;opacity:1" transform="translate(-132.651 958.04)"><path id="d" d="m309.549-710.388 122.197 211.675-56.157.527-32.624-56.87-32.856 56.566-27.903-.011-14.29-24.69 46.81-80.49-33.23-57.826z" style="opacity:1;fill:url(#c);fill-opacity:1;fill-rule:evenodd;stroke:none;stroke-width:3;stroke-linecap:butt;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"/><use xlink:href="#d" width="100%" height="100%" transform="rotate(60 407.112 -715.787)"/><use xlink:href="#d" width="100%" height="100%" transform="rotate(-60 407.312 -715.7)"/><use xlink:href="#d" width="100%" height="100%" transform="rotate(180 407.419 -715.756)"/><path id="f" d="m309.549-710.388 122.197 211.675-56.157.527-32.624-56.87-32.856 56.566-27.903-.011-14.29-24.69 46.81-80.49-33.23-57.826z" style="color:#000;clip-rule:nonzero;display:inline;overflow:visible;visibility:visible;opacity:1;isolation:auto;mix-blend-mode:normal;color-interpolation:sRGB;color-interpolation-filters:linearRGB;solid-color:#000;solid-opacity:1;fill:url(#e);fill-opacity:1;fill-rule:evenodd;stroke:none;stroke-width:3;stroke-linecap:butt;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1;color-rendering:auto;image-rendering:auto;shape-rendering:auto;text-rendering:auto;enable-background:accumulate"/><use xlink:href="#f" width="100%" height="100%" style="display:inline" transform="rotate(120 407.34 -716.084)"/><use xlink:href="#f" width="100%" height="100%" style="display:inline" transform="rotate(-120 407.288 -715.87)"/></g></svg>

doc/manual/src/glossary.md

@@ -86,7 +86,7 @@
 
   [store path]: #gloss-store-path
 
-- [file system object]{#gloss-store-object}
+- [file system object]{#gloss-file-system-object}
 
   The Nix data model for representing simplified file system data.
 
@@ -215,6 +215,9 @@
 
   [output path]: #gloss-output-path
 
+- [output closure]{#gloss-output-closure}\
+  The [closure] of an [output path]. It only contains what is [reachable] from the output.
+
 - [deriver]{#gloss-deriver}
 
   The [store derivation] that produced an [output path].
@@ -292,6 +295,25 @@
   [path]: ./language/values.md#type-path
   [attribute name]: ./language/values.md#attribute-set
 
+- [base directory]{#gloss-base-directory}
+
+  The location from which relative paths are resolved.
+
+  - For expressions in a file, the base directory is the directory containing that file.
+    This is analogous to the directory of a [base URL](https://datatracker.ietf.org/doc/html/rfc1808#section-3.3).
+    <!-- which is sufficient for resolving non-empty URLs -->
+
+  <!--
+    The wording here may look awkward, but it's for these reasons:
+      * "with --expr": it's a flag, and not an option with an accompanying value
+      * "written in": the expression itself must be written as an argument,
+        whereas the more natural "passed as an argument" allows an interpretation
+        where the expression could be passed by file name.
+    -->
+  - For expressions written in command line arguments with [`--expr`](@docroot@/command-ref/opt-common.html#opt-expr), the base directory is the current working directory.
+
+  [base directory]: #gloss-base-directory
+
 - [experimental feature]{#gloss-experimental-feature}
 
   Not yet stabilized functionality guarded by named experimental feature flags.

doc/manual/src/installation/installing-binary.md

@@ -50,7 +50,7 @@ Supported systems:
 To explicitly instruct the installer to perform a multi-user installation on your system:
 
 ```console
-$ curl -L https://nixos.org/nix/install | sh -s -- --daemon
+$ bash <(curl -L https://nixos.org/nix/install) --daemon
 ```
 
 You can run this under your usual user account or `root`.
@@ -61,7 +61,7 @@ The script will invoke `sudo` as needed.
 To explicitly select a single-user installation on your system:
 
 ```console
-$ curl -L https://nixos.org/nix/install | sh -s -- --no-daemon
+$ bash <(curl -L https://nixos.org/nix/install) --no-daemon
 ```
 
 In a single-user installation, `/nix` is owned by the invoking user.

doc/manual/src/installation/uninstall.md

@@ -5,8 +5,9 @@
 If you have a [single-user installation](./installing-binary.md#single-user-installation) of Nix, uninstall it by running:
 
 ```console
-$ rm -rf /nix
+$ rm -rf /nix ~/.nix-channels ~/.nix-defexpr ~/.nix-profile
 ```
+You might also want to manually remove references to Nix from your `~/.profile`.
 
 ## Multi User
 

doc/manual/src/installation/upgrading.md

@@ -28,7 +28,7 @@ $ sudo su
 ## macOS multi-user
 
 ```console
-$ sudo nix-env --install --file '<nixpkgs>' --attr nix -I nixpkgs=channel:nixpkgs-unstable
+$ sudo nix-env --install --file '<nixpkgs>' --attr nix cacert -I nixpkgs=channel:nixpkgs-unstable
 $ sudo launchctl remove org.nixos.nix-daemon
 $ sudo launchctl load /Library/LaunchDaemons/org.nixos.nix-daemon.plist
 ```

doc/manual/src/language/advanced-attributes.md

@@ -188,9 +188,13 @@ Derivations can declare some infrequently used optional attributes.
     }
     ```
 
-    The `outputHashAlgo` attribute specifies the hash algorithm used to
-    compute the hash. It can currently be `"sha1"`, `"sha256"` or
-    `"sha512"`.
+    The `outputHash` attribute must be a string containing the hash in either hexadecimal or "nix32" encoding, or following the format for integrity metadata as defined by [SRI](https://www.w3.org/TR/SRI/).
+    The "nix32" encoding is an adaptation of base-32 encoding.
+    The [`convertHash`](@docroot@/language/builtins.md#builtins-convertHash) function shows how to convert between different encodings, and the [`nix-hash` command](../command-ref/nix-hash.md) has information about obtaining the hash for some contents, as well as converting to and from encodings.
+
+    The `outputHashAlgo` attribute specifies the hash algorithm used to compute the hash.
+    It can currently be `"sha1"`, `"sha256"`, `"sha512"`, or `null`.
+    `outputHashAlgo` can only be `null` when `outputHash` follows the SRI format.
 
     The `outputHashMode` attribute determines how the hash is computed.
     It must be one of the following two values:
@@ -203,16 +207,16 @@ Derivations can declare some infrequently used optional attributes.
 
         This is the default.
 
-      - `"recursive"`\
-        The hash is computed over the NAR archive dump of the output
+      - `"recursive"` or `"nar"`\
+        The hash is computed over the [NAR archive](@docroot@/glossary.md#gloss-nar) dump of the output
         (i.e., the result of [`nix-store --dump`](@docroot@/command-ref/nix-store/dump.md)). In
         this case, the output can be anything, including a directory
         tree.
 
-    The `outputHash` attribute, finally, must be a string containing
-    the hash in either hexadecimal or base-32 notation. (See the
-    [`nix-hash` command](../command-ref/nix-hash.md) for information
-    about converting to and from base-32 notation.)
+    `"recursive"` is the traditional way of indicating this,
+    and is supported since 2005 (virtually the entire history of Nix).
+    `"nar"` is more clear, and consistent with other parts of Nix (such as the CLI),
+    however support for it is only added in Nix version 2.21.
 
   - [`__contentAddressed`]{#adv-attr-__contentAddressed}
     > **Warning**
@@ -299,7 +303,7 @@ Derivations can declare some infrequently used optional attributes.
     [`disallowedReferences`](#adv-attr-disallowedReferences) and [`disallowedRequisites`](#adv-attr-disallowedRequisites),
     the following attributes are available:
 
-    - `maxSize` defines the maximum size of the resulting [store object](../glossary.md#gloss-store-object).
+    - `maxSize` defines the maximum size of the resulting [store object](@docroot@/glossary.md#gloss-store-object).
     - `maxClosureSize` defines the maximum size of the output's closure.
     - `ignoreSelfRefs` controls whether self-references should be considered when
       checking for allowed references/requisites.

doc/manual/src/language/index.md

@@ -1,7 +1,13 @@
 # Nix Language
 
 The Nix language is designed for conveniently creating and composing *derivations* – precise descriptions of how contents of existing files are used to derive new files.
-It is:
+
+> **Tip**
+>
+> These pages are written as a reference.
+> If you are learning Nix, nix.dev has a good [introduction to the Nix language](https://nix.dev/tutorials/nix-language).
+
+The language is:
 
 - *domain-specific*
 
@@ -432,6 +438,32 @@ This is an incomplete overview of language features, by example.
 
   </td>
  </tr>
+ <tr>
+  <td>
+
+   `inherit pkgs src;`
+
+  </td>
+  <td>
+
+   Adds the variables to the current scope (attribute set or `let` binding).
+   Desugars to `pkgs = pkgs; src = src;`
+
+  </td>
+ </tr>
+ <tr>
+  <td>
+
+   `inherit (pkgs) lib stdenv;`
+
+  </td>
+  <td>
+
+   Adds the attributes, from the attribute set in parentheses, to the current scope (attribute set or `let` binding).
+   Desugars to `lib = pkgs.lib; stdenv = pkgs.stdenv;`
+
+  </td>
+ </tr>
  <tr>
   <td>
 

doc/manual/src/language/operators.md

@@ -128,8 +128,8 @@ The result is a string.
 > The file or directory at *path* must exist and is copied to the [store].
 > The path appears in the result as the corresponding [store path].
 
-[store path]: ../glossary.md#gloss-store-path
-[store]: ../glossary.md#gloss-store
+[store path]: @docroot@/glossary.md#gloss-store-path
+[store]: @docroot@/glossary.md#gloss-store
 
 [String and path concatenation]: #string-and-path-concatenation
 

doc/manual/src/language/string-interpolation.md

@@ -20,7 +20,7 @@ Rather than writing
 
 (where `freetype` is a [derivation]), you can instead write
 
-[derivation]: ../glossary.md#gloss-derivation
+[derivation]: @docroot@/glossary.md#gloss-derivation
 
 ```nix
 "--with-freetype2-library=${freetype}/lib"
@@ -107,9 +107,9 @@ An expression that is interpolated must evaluate to one of the following:
 
 A string interpolates to itself.
 
-A path in an interpolated expression is first copied into the Nix store, and the resulting string is the [store path] of the newly created [store object](../glossary.md#gloss-store-object).
+A path in an interpolated expression is first copied into the Nix store, and the resulting string is the [store path] of the newly created [store object](@docroot@/glossary.md#gloss-store-object).
 
-[store path]: ../glossary.md#gloss-store-path
+[store path]: @docroot@/glossary.md#gloss-store-path
 
 > **Example**
 >

doc/manual/src/language/values.md

@@ -97,8 +97,8 @@
   is not a path: it's parsed as an expression that selects the
   attribute `sh` from the variable `builder`. If the file name is
   relative, i.e., if it does not begin with a slash, it is made
-  absolute at parse time relative to the directory of the Nix
-  expression that contained it. For instance, if a Nix expression in
+  absolute at parse time relative to the [base directory](@docroot@/glossary.md#gloss-base-directory).
+  For instance, if a Nix expression in
   `/foo/bar/bla.nix` refers to `../xyzzy/fnord.nix`, the absolute path
   is `/foo/xyzzy/fnord.nix`.
 
@@ -107,13 +107,13 @@
   e.g. `~/foo` would be equivalent to `/home/edolstra/foo` for a user
   whose home directory is `/home/edolstra`.
 
-  For instance, evaluating `"${./foo.txt}"` will cause `foo.txt` in the current directory to be copied into the Nix store and result in the string `"/nix/store/<hash>-foo.txt"`.
+  For instance, evaluating `"${./foo.txt}"` will cause `foo.txt` in the base directory to be copied into the Nix store and result in the string `"/nix/store/<hash>-foo.txt"`.
 
   Note that the Nix language assumes that all input files will remain _unchanged_ while  evaluating a Nix expression.
   For example, assume you used a file path in an interpolated string during a `nix repl` session.
   Later in the same session, after having changed the file contents, evaluating the interpolated string with the file path again might not return a new [store path], since Nix might not re-read the file contents.
 
-  [store path]: ../glossary.md#gloss-store-path
+  [store path]: @docroot@/glossary.md#gloss-store-path
 
   Paths can include [string interpolation] and can themselves be [interpolated in other expressions].
 

doc/manual/src/protocols/json/store-object-info.md

@@ -83,7 +83,7 @@ This information is not intrinsic to the store object, but about how it is store
 
 ## Computed closure fields
 
-These fields are not stored at all, but computed by traverising the other other fields across all the store objects in a [closure].
+These fields are not stored at all, but computed by traversing the other fields across all the store objects in a [closure].
 
 * `closureSize`:
 

doc/manual/src/protocols/nix-archive.md

@@ -0,0 +1,42 @@
+# Nix Archive (NAR) format
+
+This is the complete specification of the Nix Archive format.
+The Nix Archive format closely follows the abstract specification of a [file system object] tree,
+because it is designed to serialize exactly that data structure.
+
+[file system object]: @docroot@/store/file-system-object.md
+
+The format of this specification is close to [Extended Backus–Naur form](https://en.wikipedia.org/wiki/Extended_Backus%E2%80%93Naur_form), with the exception of the `str(..)` function / parameterized rule, which length-prefixes and pads strings.
+This makes the resulting binary format easier to parse.
+
+Regular users do *not* need to know this information.
+But for those interested in exactly how Nix works, e.g. if they are reimplementing it, this information can be useful.
+
+```ebnf
+nar = str("nix-archive-1"), nar-obj;
+
+nar-obj = str("("), nar-obj-inner, str(")");
+
+nar-obj-inner
+  = str("type"), str("regular") regular
+  | str("type"), str("symlink") symlink
+  | str("type"), str("directory") directory
+  ;
+
+regular = [ str("executable"), str("") ], str("contents"), str(contents);
+
+symlink = str("target"), str(target);
+
+(* side condition: directory entries must be ordered by their names *)
+directory = str("type"), str("directory") { directory-entry };
+
+directory-entry = str("entry"), str("("), str("name"), str(name), str("node"), nar-obj, str(")");
+```
+
+The `str` function / parameterized rule is defined as follows:
+
+- `str(s)` = `int(|s|), pad(s);`
+
+- `int(n)` = the 64-bit little endian representation of the number `n`
+
+- `pad(s)` = the byte sequence `s`, padded with 0s to a multiple of 8 byte

doc/manual/src/protocols/store-path.md

@@ -89,15 +89,20 @@ where
 
       - `rec` = one of:
 
+        - ```ebnf
+          | ""
+          ```
+          (empty string) for hashes of the flat (single file) serialization
+
         - ```ebnf
           | "r:"
           ```
           hashes of the for [Nix Archive (NAR)] (arbitrary file system object) serialization
 
         - ```ebnf
-          | ""
+          | "git:"
           ```
-          (empty string) for hashes of the flat (single file) serialization
+          hashes of the [Git blob/tree](https://git-scm.com/book/en/v2/Git-Internals-Git-Objects) [Merkel tree](https://en.wikipedia.org/wiki/Merkle_tree) format
 
       - ```ebnf
         algo = "md5" | "sha1" | "sha256"

doc/manual/src/quick-start.md

@@ -34,7 +34,7 @@ For more in-depth information you are kindly referred to subsequent chapters.
    lolcat: command not found
    ```
 
-1. Search for more packages on <search.nixos.org> to try them out.
+1. Search for more packages on [search.nixos.org](https://search.nixos.org/) to try them out.
 
 1. Free up storage space:
 

doc/manual/src/release-notes/index.md

@@ -1,12 +1,13 @@
 # Nix Release Notes
 
+The Nix release cycle is calendar-based as follows:
+
 Nix has a release cycle of roughly 6 weeks.
 Notable changes and additions are announced in the release notes for each version.
 
-Bugfixes can be backported on request to previous Nix releases.
-We typically backport only as far back as the Nix version used in the latest NixOS release, which is announced in the [NixOS release notes](https://nixos.org/manual/nixos/stable/release-notes.html#ch-release-notes).
-
-Backports never skip releases.
-If a feature is backported to version `x.y`, it must also be available in version `x.(y+1)`.
-This ensures that upgrading from an older version with backports is still safe and no backported functionality will go missing.
+The supported Nix versions are:
+- The latest release
+- The version used in the stable NixOS release, which is announced in the [NixOS release notes](https://nixos.org/manual/nixos/stable/release-notes.html#ch-release-notes).
 
+Bugfixes and security issues are backported to every supported version.
+Patch releases are published as needed.

doc/manual/src/release-notes/rl-2.15.md

@@ -11,7 +11,7 @@
   As the choice of hash formats is no longer binary, the `--base16` flag is also added
   to explicitly specify the Base16 format, which is still the default.
 
-* The special handling of an [installable](../command-ref/new-cli/nix.md#installables) with `.drv` suffix being interpreted as all of the given [store derivation](../glossary.md#gloss-store-derivation)'s output paths is removed, and instead taken as the literal store path that it represents.
+* The special handling of an [installable](../command-ref/new-cli/nix.md#installables) with `.drv` suffix being interpreted as all of the given [store derivation](@docroot@/glossary.md#gloss-store-derivation)'s output paths is removed, and instead taken as the literal store path that it represents.
 
   The new `^` syntax for store paths introduced in Nix 2.13 allows explicitly referencing output paths of a derivation.
   Using this is better and more clear than relying on the now-removed `.drv` special handling.

doc/manual/src/release-notes/rl-2.20.md

@@ -200,3 +200,9 @@
   while performing various operations (including `nix develop`, `nix flake
   update`, and so on). With several fixes to Nix's signal handlers, Nix
   commands will now exit quickly after Ctrl-C is pressed.
+
+- `nix copy` to a `ssh-ng` store now needs `--substitute-on-destination` (a.k.a. `-s`)
+  in order to substitute paths on the remote store instead of copying them.
+  The behavior is consistent with `nix copy` to a different kind of remote store.
+  Previously this behavior was controlled by the
+  `builders-use-substitutes` setting and `--substitute-on-destination` was ignored.

doc/manual/src/release-notes/rl-2.21.md

@@ -0,0 +1,302 @@
+# Release 2.21.0 (2024-03-11)
+
+- Fix a fixed-output derivation sandbox escape (CVE-2024-27297)
+
+  Cooperating Nix derivations could send file descriptors to files in the Nix
+  store to each other via Unix domain sockets in the abstract namespace. This
+  allowed one derivation to modify the output of the other derivation, after Nix
+  has registered the path as "valid" and immutable in the Nix database.
+  In particular, this allowed the output of fixed-output derivations to be
+  modified from their expected content.
+
+  This isn't the case any more.
+
+- CLI options `--arg-from-file` and `--arg-from-stdin` [#10122](https://github.com/NixOS/nix/pull/10122)
+
+  The new CLI option `--arg-from-file` *name* *path* passes the contents
+  of file *path* as a string value via the function argument *name* to a
+  Nix expression. Similarly, the new option `--arg-from-stdin` *name*
+  reads the contents of the string from standard input.
+
+- Concise error printing in `nix repl` [#9928](https://github.com/NixOS/nix/pull/9928)
+
+  Previously, if an element of a list or attribute set threw an error while
+  evaluating, `nix repl` would print the entire error (including source location
+  information) inline. This output was clumsy and difficult to parse:
+
+  ```
+  nix-repl> { err = builtins.throw "uh oh!"; }
+  { err = «error:
+         … while calling the 'throw' builtin
+           at «string»:1:9:
+              1| { err = builtins.throw "uh oh!"; }
+               |         ^
+
+         error: uh oh!»; }
+  ```
+
+  Now, only the error message is displayed, making the output much more readable.
+  ```
+  nix-repl> { err = builtins.throw "uh oh!"; }
+  { err = «error: uh oh!»; }
+  ```
+
+  However, if the whole expression being evaluated throws an error, source
+  locations and (if applicable) a stack trace are printed, just like you'd expect:
+
+  ```
+  nix-repl> builtins.throw "uh oh!"
+  error:
+         … while calling the 'throw' builtin
+           at «string»:1:1:
+              1| builtins.throw "uh oh!"
+               | ^
+
+         error: uh oh!
+  ```
+
+- `--debugger` can now access bindings from `let` expressions [#8827](https://github.com/NixOS/nix/issues/8827) [#9918](https://github.com/NixOS/nix/pull/9918)
+
+  Breakpoints and errors in the bindings of a `let` expression can now access
+  those bindings in the debugger. Previously, only the body of `let` expressions
+  could access those bindings.
+
+- Enter the `--debugger` when `builtins.trace` is called if `debugger-on-trace` is set [#9914](https://github.com/NixOS/nix/pull/9914)
+
+  If the `debugger-on-trace` option is set and `--debugger` is given,
+  `builtins.trace` calls will behave similarly to `builtins.break` and will enter
+  the debug REPL. This is useful for determining where warnings are being emitted
+  from.
+
+- Debugger prints source position information [#9913](https://github.com/NixOS/nix/pull/9913)
+
+  The `--debugger` now prints source location information, instead of the
+  pointers of source location information. Before:
+
+  ```
+  nix-repl> :bt
+  0: while evaluating the attribute 'python311.pythonForBuild.pkgs'
+  0x600001522598
+  ```
+
+  After:
+
+  ```
+  0: while evaluating the attribute 'python311.pythonForBuild.pkgs'
+  /nix/store/hg65h51xnp74ikahns9hyf3py5mlbbqq-source/overrides/default.nix:132:27
+
+     131|
+     132|       bootstrappingBase = pkgs.${self.python.pythonAttr}.pythonForBuild.pkgs;
+        |                           ^
+     133|     in
+  ```
+
+- The `--debugger` will start more reliably in `let` expressions and function calls [#6649](https://github.com/NixOS/nix/issues/6649) [#9917](https://github.com/NixOS/nix/pull/9917)
+
+  Previously, if you attempted to evaluate this file with the debugger:
+
+  ```nix
+  let
+    a = builtins.trace "before inner break" (
+      builtins.break "hello"
+    );
+    b = builtins.trace "before outer break" (
+      builtins.break a
+    );
+  in
+    b
+  ```
+
+  Nix would correctly enter the debugger at `builtins.break a`, but if you asked
+  it to `:continue`, it would skip over the `builtins.break "hello"` expression
+  entirely.
+
+  Now, Nix will correctly enter the debugger at both breakpoints.
+
+- Nested debuggers are no longer supported [#9920](https://github.com/NixOS/nix/pull/9920)
+
+  Previously, evaluating an expression that throws an error in the debugger would
+  enter a second, nested debugger:
+
+  ```
+  nix-repl> builtins.throw "what"
+  error: what
+
+
+  Starting REPL to allow you to inspect the current state of the evaluator.
+
+  Welcome to Nix 2.18.1. Type :? for help.
+
+  nix-repl>
+  ```
+
+  Now, it just prints the error message like `nix repl`:
+
+  ```
+  nix-repl> builtins.throw "what"
+  error:
+         … while calling the 'throw' builtin
+           at «string»:1:1:
+              1| builtins.throw "what"
+               | ^
+
+         error: what
+  ```
+
+- Consistent order of function arguments in printed expressions [#9874](https://github.com/NixOS/nix/pull/9874)
+
+  Function arguments are now printed in lexicographic order rather than the internal, creation-time based symbol order.
+
+- Fix duplicate attribute error positions for `inherit` [#9874](https://github.com/NixOS/nix/pull/9874)
+
+  When an `inherit` caused a duplicate attribute error the position of the error was not reported correctly, placing the error with the inherit itself or at the start of the bindings block instead of the offending attribute name.
+
+- `inherit (x) ...` evaluates `x` only once [#9847](https://github.com/NixOS/nix/pull/9847)
+
+  `inherit (x) a b ...` now evaluates the expression `x` only once for all inherited attributes rather than once for each inherited attribute.
+  This does not usually have a measurable impact, but side-effects (such as `builtins.trace`) would be duplicated and expensive expressions (such as derivations) could cause a measurable slowdown.
+
+- Store paths are allowed to start with `.` [#912](https://github.com/NixOS/nix/issues/912) [#9091](https://github.com/NixOS/nix/pull/9091) [#9095](https://github.com/NixOS/nix/pull/9095) [#9120](https://github.com/NixOS/nix/pull/9120) [#9121](https://github.com/NixOS/nix/pull/9121) [#9122](https://github.com/NixOS/nix/pull/9122) [#9130](https://github.com/NixOS/nix/pull/9130) [#9219](https://github.com/NixOS/nix/pull/9219) [#9224](https://github.com/NixOS/nix/pull/9224) [#9867](https://github.com/NixOS/nix/pull/9867)
+
+  Leading periods were allowed by accident in Nix 2.4. The Nix team has considered this to be a bug, but this behavior has since been relied on by users, leading to unnecessary difficulties.
+  From now on, leading periods are supported. The names `.` and `..` are disallowed, as well as those starting with `.-` or `..-`.
+
+  Nix versions that denied leading periods are documented [in the issue](https://github.com/NixOS/nix/issues/912#issuecomment-1919583286).
+
+- `nix repl` pretty-prints values [#9931](https://github.com/NixOS/nix/pull/9931)
+
+  `nix repl` will now pretty-print values:
+
+  ```
+  {
+    attrs = {
+      a = {
+        b = {
+          c = { };
+        };
+      };
+    };
+    list = [ 1 ];
+    list' = [
+      1
+      2
+      3
+    ];
+  }
+  ```
+
+- Introduction of `--regex` and `--all` in `nix profile remove` and `nix profile upgrade` [#10166](https://github.com/NixOS/nix/pull/10166)
+
+  Previously the command-line arguments for `nix profile remove` and `nix profile upgrade` matched the package entries using regular expression.
+  For instance:
+
+  ```
+  nix profile remove '.*vim.*'
+  ```
+
+  This would remove all packages that contain `vim` in their name.
+
+  In most cases, only singular package names were used to remove and upgrade packages. Mixing this with regular expressions sometimes lead to unintended behavior. For instance, `python3.1` could match `python311`.
+
+  To avoid unintended behavior, the arguments are now only matching exact names.
+
+  Matching using regular expressions is still possible by using the new `--regex` flag:
+
+  ```
+  nix profile remove --regex '.*vim.*'
+  ```
+
+  One of the most useful cases for using regular expressions was to upgrade all packages. This was previously accomplished by:
+
+  ```
+  nix profile upgrade '.*'
+  ```
+
+  With the introduction of the `--all` flag, this now becomes more straightforward:
+
+  ```
+  nix profile upgrade --all
+  ```
+
+- Visual clutter in `--debugger` is reduced [#9919](https://github.com/NixOS/nix/pull/9919)
+
+  Before:
+  ```
+  info: breakpoint reached
+
+
+  Starting REPL to allow you to inspect the current state of the evaluator.
+
+  Welcome to Nix 2.20.0pre20231222_dirty. Type :? for help.
+
+  nix-repl> :continue
+  error: uh oh
+
+
+  Starting REPL to allow you to inspect the current state of the evaluator.
+
+  Welcome to Nix 2.20.0pre20231222_dirty. Type :? for help.
+
+  nix-repl>
+  ```
+
+  After:
+
+  ```
+  info: breakpoint reached
+
+  Nix 2.20.0pre20231222_dirty debugger
+  Type :? for help.
+  nix-repl> :continue
+  error: uh oh
+
+  nix-repl>
+  ```
+
+- Cycle detection in `nix repl` is simpler and more reliable [#8672](https://github.com/NixOS/nix/issues/8672) [#9926](https://github.com/NixOS/nix/pull/9926)
+
+  The cycle detection in `nix repl`, `nix eval`, `builtins.trace`, and everywhere
+  else values are printed is now simpler and matches the cycle detection in
+  `nix-instantiate --eval` output.
+
+  Before:
+
+  ```
+  nix eval --expr 'let self = { inherit self; }; in self'
+  { self = { self = «repeated»; }; }
+  ```
+
+  After:
+
+  ```
+  { self = «repeated»; }
+  ```
+
+- In the debugger, `while evaluating the attribute` errors now include position information [#9915](https://github.com/NixOS/nix/pull/9915)
+
+  Before:
+
+  ```
+  0: while evaluating the attribute 'python311.pythonForBuild.pkgs'
+  0x600001522598
+  ```
+
+  After:
+
+  ```
+  0: while evaluating the attribute 'python311.pythonForBuild.pkgs'
+  /nix/store/hg65h51xnp74ikahns9hyf3py5mlbbqq-source/overrides/default.nix:132:27
+
+     131|
+     132|       bootstrappingBase = pkgs.${self.python.pythonAttr}.pythonForBuild.pkgs;
+        |                           ^
+     133|     in
+  ```
+
+- Stack size is increased on macOS [#9860](https://github.com/NixOS/nix/pull/9860)
+
+  Previously, Nix would set the stack size to 64MiB on Linux, but would leave the
+  stack size set to the default (approximately 8KiB) on macOS. Now, the stack
+  size is correctly set to 64MiB on macOS as well, which should reduce stack
+  overflow segfaults in deeply-recursive Nix expressions.
+

doc/manual/src/release-notes/rl-2.22.md

@@ -0,0 +1,21 @@
+# Release 2.22.0 (2024-04-23)
+
+### Significant changes
+
+- Remove experimental repl-flake [#10103](https://github.com/NixOS/nix/issues/10103) [#10299](https://github.com/NixOS/nix/pull/10299)
+
+  The `repl-flake` experimental feature has been removed. The `nix repl` command now works like the rest of the new CLI in that `nix repl {path}` now tries to load a flake at `{path}` (or fails if the `flakes` experimental feature isn't enabled).
+
+### Other changes
+
+- `nix eval` prints derivations as `.drv` paths [#10200](https://github.com/NixOS/nix/pull/10200)
+
+  `nix eval` will now print derivations as their `.drv` paths, rather than as
+  attribute sets. This makes commands like `nix eval nixpkgs#bash` terminate
+  instead of infinitely looping into recursive self-referential attributes:
+
+  ```ShellSession
+  $ nix eval nixpkgs#bash
+  «derivation /nix/store/m32cbgbd598f4w299g0hwyv7gbw6rqcg-bash-5.2p26.drv»
+  ```
+

doc/manual/src/store/store-path.md

@@ -46,7 +46,7 @@ But if the store has a file system representation, the store directory contains
 
 [file system objects]: ./file-system-object.md
 
-This means a store path is not just derived from the referenced store object itself, but depends on the store the store object is in.
+This means a store path is not just derived from the referenced store object itself, but depends on the store that the store object is in.
 
 > **Note**
 >

flake.lock

@@ -16,6 +16,41 @@
         "type": "github"
       }
     },
+    "flake-parts": {
+      "inputs": {
+        "nixpkgs-lib": [
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1712014858,
+        "narHash": "sha256-sB4SWl2lX95bExY2gMFG5HIzvva5AVMJd4Igm+GpZNw=",
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "rev": "9126214d0a59633752a136528f5f3b9aa8565b7d",
+        "type": "github"
+      },
+      "original": {
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "type": "github"
+      }
+    },
+    "flake-utils": {
+      "locked": {
+        "lastModified": 1667395993,
+        "narHash": "sha256-nuEHfE/LcWyuSWnS8t12N1wc105Qtau+/OdUAjtQ0rA=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "5aed5285a952e0b949eb3ba02c12fa4fcfef535f",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
     "libgit2": {
       "flake": false,
       "locked": {
@@ -34,16 +69,16 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1705033721,
-        "narHash": "sha256-K5eJHmL1/kev6WuqyqqbS1cdNnSidIZ3jeqJ7GbrYnQ=",
+        "lastModified": 1709083642,
+        "narHash": "sha256-7kkJQd4rZ+vFrzWu8sTRtta5D1kBG0LSRYAfhtmMlSo=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "a1982c92d8980a0114372973cbdfe0a307f1bdea",
+        "rev": "b550fe4b4776908ac2a861124307045f8e717c8e",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "nixos-23.05-small",
+        "ref": "release-23.11",
         "repo": "nixpkgs",
         "type": "github"
       }
@@ -64,12 +99,40 @@
         "type": "github"
       }
     },
+    "pre-commit-hooks": {
+      "inputs": {
+        "flake-compat": [],
+        "flake-utils": "flake-utils",
+        "gitignore": [],
+        "nixpkgs": [
+          "nixpkgs"
+        ],
+        "nixpkgs-stable": [
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1712897695,
+        "narHash": "sha256-nMirxrGteNAl9sWiOhoN5tIHyjBbVi5e2tgZUgZlK3Y=",
+        "owner": "cachix",
+        "repo": "pre-commit-hooks.nix",
+        "rev": "40e6053ecb65fcbf12863338a6dcefb3f55f1bf8",
+        "type": "github"
+      },
+      "original": {
+        "owner": "cachix",
+        "repo": "pre-commit-hooks.nix",
+        "type": "github"
+      }
+    },
     "root": {
       "inputs": {
         "flake-compat": "flake-compat",
+        "flake-parts": "flake-parts",
         "libgit2": "libgit2",
         "nixpkgs": "nixpkgs",
-        "nixpkgs-regression": "nixpkgs-regression"
+        "nixpkgs-regression": "nixpkgs-regression",
+        "pre-commit-hooks": "pre-commit-hooks"
       }
     }
   },

flake.nix

@@ -1,28 +1,32 @@
 {
   description = "The purely functional package manager";
 
-  inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05-small";
+  # TODO switch to nixos-23.11-small
+  #      https://nixpk.gs/pr-tracker.html?pr=291954
+  inputs.nixpkgs.url = "github:NixOS/nixpkgs/release-23.11";
   inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2";
   inputs.flake-compat = { url = "github:edolstra/flake-compat"; flake = false; };
   inputs.libgit2 = { url = "github:libgit2/libgit2"; flake = false; };
 
-  outputs = { self, nixpkgs, nixpkgs-regression, libgit2, ... }:
+  # dev tooling
+  inputs.flake-parts.url = "github:hercules-ci/flake-parts";
+  inputs.pre-commit-hooks.url = "github:cachix/pre-commit-hooks.nix";
+  # work around https://github.com/NixOS/nix/issues/7730
+  inputs.flake-parts.inputs.nixpkgs-lib.follows = "nixpkgs";
+  inputs.pre-commit-hooks.inputs.nixpkgs.follows = "nixpkgs";
+  inputs.pre-commit-hooks.inputs.nixpkgs-stable.follows = "nixpkgs";
+  # work around 7730 and https://github.com/NixOS/nix/issues/7807
+  inputs.pre-commit-hooks.inputs.flake-compat.follows = "";
+  inputs.pre-commit-hooks.inputs.gitignore.follows = "";
+
+  outputs = inputs@{ self, nixpkgs, nixpkgs-regression, libgit2, ... }:
+
 
     let
       inherit (nixpkgs) lib;
+      inherit (lib) fileset;
 
-      # Experimental fileset library: https://github.com/NixOS/nixpkgs/pull/222981
-      # Not an "idiomatic" flake input because:
-      #  - Propagation to dependent locks: https://github.com/NixOS/nix/issues/7730
-      #  - Subflake would download redundant and huge parent flake
-      #  - No git tree hash support: https://github.com/NixOS/nix/issues/6044
-      inherit (import (builtins.fetchTarball { url = "https://github.com/NixOS/nix/archive/1bdcd7fc8a6a40b2e805bad759b36e64e911036b.tar.gz"; sha256 = "sha256:14ljlpdsp4x7h1fkhbmc4bd3vsqnx8zdql4h3037wh09ad6a0893"; }))
-        fileset;
-
-      officialRelease = false;
-
-      # Set to true to build the release notes for the next release.
-      buildUnreleasedNotes = false;
+      officialRelease = true;
 
       version = lib.fileContents ./.version + versionSuffix;
       versionSuffix =
@@ -39,14 +43,8 @@
       crossSystems = [
         "armv6l-unknown-linux-gnueabihf"
         "armv7l-unknown-linux-gnueabihf"
-        "x86_64-unknown-freebsd13"
+        "riscv64-unknown-linux-gnu"
         "x86_64-unknown-netbsd"
-      ];
-
-      # Nix doesn't yet build on this platform, so we put it in a
-      # separate list. We just use this for `devShells` and
-      # `nixpkgsFor`, which this depends on.
-      shellCrossSystems = crossSystems ++ [
         "x86_64-w64-mingw32"
       ];
 
@@ -71,6 +69,17 @@
             })
             stdenvs);
 
+
+      # We don't apply flake-parts to the whole flake so that non-development attributes
+      # load without fetching any development inputs.
+      devFlake = inputs.flake-parts.lib.mkFlake { inherit inputs; } {
+        imports = [ ./maintainers/flake-module.nix ];
+        systems = lib.subtractLists crossSystems systems;
+        perSystem = { system, ... }: {
+          _module.args.pkgs = nixpkgsFor.${system}.native;
+        };
+      };
+
       # Memoize nixpkgs for different platforms for efficiency.
       nixpkgsFor = forAllSystems
         (system: let
@@ -92,7 +101,7 @@
         in {
           inherit stdenvs native;
           static = native.pkgsStatic;
-          cross = lib.genAttrs shellCrossSystems (crossSystem: make-pkgs crossSystem "stdenv");
+          cross = forAllCrossSystems (crossSystem: make-pkgs crossSystem "stdenv");
         });
 
       installScriptFor = tarballs:
@@ -174,7 +183,7 @@
 
           nix =
             let
-              officialRelease = false;
+              officialRelease = true;
               versionSuffix =
                 if officialRelease
                 then ""
@@ -186,7 +195,7 @@
                 stdenv
                 versionSuffix
                 ;
-              officialRelease = false;
+              officialRelease = true;
               boehmgc = final.boehmgc-nix;
               libgit2 = final.libgit2-nix;
               busybox-sandbox-shell = final.busybox-sandbox-shell or final.default-busybox-sandbox-shell;
@@ -200,6 +209,13 @@
             inherit fileset stdenv;
           };
 
+          # See https://github.com/NixOS/nixpkgs/pull/214409
+          # Remove when fixed in this flake's nixpkgs
+          pre-commit =
+            if prev.stdenv.hostPlatform.system == "i686-linux"
+            then (prev.pre-commit.override (o: { dotnet-sdk = ""; })).overridePythonAttrs (o: { doCheck = false; })
+            else prev.pre-commit;
+
         };
 
     in {
@@ -268,6 +284,7 @@
           # Cross
           self.hydraJobs.binaryTarballCross."x86_64-linux"."armv6l-unknown-linux-gnueabihf"
           self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-unknown-linux-gnueabihf"
+          self.hydraJobs.binaryTarballCross."x86_64-linux"."riscv64-unknown-linux-gnu"
         ];
         installerScriptForGHA = installScriptFor [
           # Native
@@ -276,6 +293,7 @@
           # Cross
           self.hydraJobs.binaryTarballCross."x86_64-linux"."armv6l-unknown-linux-gnueabihf"
           self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-unknown-linux-gnueabihf"
+          self.hydraJobs.binaryTarballCross."x86_64-linux"."riscv64-unknown-linux-gnu"
         ];
 
         # docker image with Nix inside
@@ -294,6 +312,13 @@
           enableInternalAPIDocs = true;
         };
 
+        # API docs for Nix's C bindings.
+        external-api-docs = nixpkgsFor.x86_64-linux.native.callPackage ./package.nix {
+          inherit fileset;
+          doBuild = false;
+          enableExternalAPIDocs = true;
+        };
+
         # System tests.
         tests = import ./tests/nixos { inherit lib nixpkgs nixpkgsFor; } // {
 
@@ -307,8 +332,11 @@
               ''
                 type -p nix-env
                 # Note: we're filtering out nixos-install-tools because https://github.com/NixOS/nixpkgs/pull/153594#issuecomment-1020530593.
-                time nix-env --store dummy:// -f ${nixpkgs-regression} -qaP --drv-path | sort | grep -v nixos-install-tools > packages
-                [[ $(sha1sum < packages | cut -c1-40) = ff451c521e61e4fe72bdbe2d0ca5d1809affa733 ]]
+                (
+                  set -x
+                  time nix-env --store dummy:// -f ${nixpkgs-regression} -qaP --drv-path | sort | grep -v nixos-install-tools > packages
+                  [[ $(sha1sum < packages | cut -c1-40) = e01b031fc9785a572a38be6bc473957e3b6faad7 ]]
+                )
                 mkdir $out
               '';
 
@@ -349,7 +377,6 @@
 
       checks = forAllSystems (system: {
         binaryTarball = self.hydraJobs.binaryTarball.${system};
-        perlBindings = self.hydraJobs.perlBindings.${system};
         installTests = self.hydraJobs.installTests.${system};
         nixpkgsLibTests = self.hydraJobs.tests.nixpkgsLibTests.${system};
         rl-next =
@@ -359,7 +386,13 @@
         '';
       } // (lib.optionalAttrs (builtins.elem system linux64BitSystems)) {
         dockerImage = self.hydraJobs.dockerImage.${system};
-      });
+      } // (lib.optionalAttrs (!(builtins.elem system linux32BitSystems))) {
+        # Some perl dependencies are broken on i686-linux.
+        # Since the support is only best-effort there, disable the perl
+        # bindings
+        perlBindings = self.hydraJobs.perlBindings.${system};
+      } // devFlake.checks.${system} or {}
+      );
 
       packages = forAllSystems (system: rec {
         inherit (nixpkgsFor.${system}.native) nix changelog-d-nix;
@@ -394,7 +427,11 @@
           stdenvs)));
 
       devShells = let
-        makeShell = pkgs: stdenv: (pkgs.nix.override { inherit stdenv; forDevShell = true; }).overrideAttrs (attrs: {
+        makeShell = pkgs: stdenv: (pkgs.nix.override { inherit stdenv; forDevShell = true; }).overrideAttrs (attrs:
+        let
+          modular = devFlake.getSystem stdenv.buildPlatform.system;
+        in {
+          pname = "shell-for-" + attrs.pname;
           installFlags = "sysconfdir=$(out)/etc";
           shellHook = ''
             PATH=$prefix/bin:$PATH
@@ -404,8 +441,25 @@
             # Make bash completion work.
             XDG_DATA_DIRS+=:$out/share
           '';
+
+          # We use this shell with the local checkout, not unpackPhase.
+          src = null;
+
+          env = {
+            # For `make format`, to work without installing pre-commit
+            _NIX_PRE_COMMIT_HOOKS_CONFIG =
+              "${(pkgs.formats.yaml { }).generate "pre-commit-config.yaml" modular.pre-commit.settings.rawConfig}";
+          };
+
           nativeBuildInputs = attrs.nativeBuildInputs or []
-            ++ lib.optional stdenv.cc.isClang pkgs.buildPackages.bear
+            ++ [
+              modular.pre-commit.settings.package
+              (pkgs.writeScriptBin "pre-commit-hooks-install"
+                modular.pre-commit.settings.installationScript)
+            ]
+            # TODO: Remove the darwin check once
+            # https://github.com/NixOS/nixpkgs/pull/291814 is available
+            ++ lib.optional (stdenv.cc.isClang && !stdenv.buildPlatform.isDarwin) pkgs.buildPackages.bear
             ++ lib.optional (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform) pkgs.buildPackages.clang-tools;
         });
         in
@@ -417,8 +471,9 @@
               (forAllStdenvs (stdenvName: makeShell pkgs pkgs.${stdenvName}));
           in
             (makeShells "native" nixpkgsFor.${system}.native) //
-            (makeShells "static" nixpkgsFor.${system}.static) //
-            (lib.genAttrs shellCrossSystems (crossSystem: let pkgs = nixpkgsFor.${system}.cross.${crossSystem}; in makeShell pkgs pkgs.stdenv)) //
+            (lib.optionalAttrs (!nixpkgsFor.${system}.native.stdenv.isDarwin)
+              (makeShells "static" nixpkgsFor.${system}.static) //
+              (forAllCrossSystems (crossSystem: let pkgs = nixpkgsFor.${system}.cross.${crossSystem}; in makeShell pkgs pkgs.stdenv))) //
             {
               default = self.devShells.${system}.native-stdenvPackages;
             }

local.mk

@@ -2,9 +2,14 @@ GLOBAL_CXXFLAGS += -Wno-deprecated-declarations -Werror=switch
 # Allow switch-enum to be overridden for files that do not support it, usually because of dependency headers.
 ERROR_SWITCH_ENUM = -Werror=switch-enum
 
-$(foreach i, config.h $(wildcard src/lib*/*.hh), \
+$(foreach i, config.h $(wildcard src/lib*/*.hh) $(filter-out %_internal.h, $(wildcard src/lib*c/*.h)), \
   $(eval $(call install-file-in, $(i), $(includedir)/nix, 0644)))
 
+ifdef HOST_UNIX
+  $(foreach i, $(wildcard src/lib*/unix/*.hh), \
+    $(eval $(call install-file-in, $(i), $(includedir)/nix, 0644)))
+endif
+
 $(GCH): src/libutil/util.hh config.h
 
-GCH_CXXFLAGS = -I src/libutil
+GCH_CXXFLAGS = $(INCLUDE_libutil)

m4/gcc_bug_80431.m4

@@ -46,11 +46,13 @@ AC_DEFUN([ENSURE_NO_GCC_BUG_80431],
       ]])],
     [status_80431=0],
     [status_80431=$?],
-    [
-      # Assume we're bug-free when cross-compiling
-    ])
+    [status_80431=''])
   AC_LANG_POP(C++)
   AS_CASE([$status_80431],
+    [''],[
+      AC_MSG_RESULT(cannot check because cross compiling)
+      AC_MSG_NOTICE(assume we are bug free)
+    ],
     [0],[
       AC_MSG_RESULT(yes)
     ],

maintainers/flake-module.nix

@@ -0,0 +1,436 @@
+{ lib, getSystem, inputs, ... }:
+
+{
+  imports = [
+    inputs.pre-commit-hooks.flakeModule
+  ];
+
+  perSystem = { config, pkgs, ... }: {
+
+    # https://flake.parts/options/pre-commit-hooks-nix.html#options
+    pre-commit.settings = {
+      hooks = {
+        clang-format.enable = true;
+        # TODO: nixfmt, https://github.com/NixOS/nixfmt/issues/153
+      };
+
+      excludes = [
+        # We don't want to format test data
+        # ''tests/(?!nixos/).*\.nix''
+        ''^tests/.*''
+
+        # Don't format vendored code
+        ''^src/toml11/.*''
+        ''^doc/manual/redirects\.js$''
+        ''^doc/manual/theme/highlight\.js$''
+
+        # We haven't applied formatting to these files yet
+        ''^doc/manual/redirects\.js$''
+        ''^doc/manual/theme/highlight\.js$''
+        ''^precompiled-headers\.h$''
+        ''^src/build-remote/build-remote\.cc$''
+        ''^src/libcmd/built-path\.cc$''
+        ''^src/libcmd/built-path\.hh$''
+        ''^src/libcmd/command\.cc$''
+        ''^src/libcmd/command\.hh$''
+        ''^src/libcmd/common-eval-args\.cc$''
+        ''^src/libcmd/common-eval-args\.hh$''
+        ''^src/libcmd/editor-for\.cc$''
+        ''^src/libcmd/installable-attr-path\.cc$''
+        ''^src/libcmd/installable-attr-path\.hh$''
+        ''^src/libcmd/installable-derived-path\.cc$''
+        ''^src/libcmd/installable-derived-path\.hh$''
+        ''^src/libcmd/installable-flake\.cc$''
+        ''^src/libcmd/installable-flake\.hh$''
+        ''^src/libcmd/installable-value\.cc$''
+        ''^src/libcmd/installable-value\.hh$''
+        ''^src/libcmd/installables\.cc$''
+        ''^src/libcmd/installables\.hh$''
+        ''^src/libcmd/legacy\.hh$''
+        ''^src/libcmd/markdown\.cc$''
+        ''^src/libcmd/misc-store-flags\.cc$''
+        ''^src/libcmd/repl-interacter\.cc$''
+        ''^src/libcmd/repl-interacter\.hh$''
+        ''^src/libcmd/repl\.cc$''
+        ''^src/libcmd/repl\.hh$''
+        ''^src/libexpr-c/nix_api_expr\.cc$''
+        ''^src/libexpr-c/nix_api_external\.cc$''
+        ''^src/libexpr/attr-path\.cc$''
+        ''^src/libexpr/attr-path\.hh$''
+        ''^src/libexpr/attr-set\.cc$''
+        ''^src/libexpr/attr-set\.hh$''
+        ''^src/libexpr/eval-cache\.cc$''
+        ''^src/libexpr/eval-cache\.hh$''
+        ''^src/libexpr/eval-error\.cc$''
+        ''^src/libexpr/eval-inline\.hh$''
+        ''^src/libexpr/eval-settings\.cc$''
+        ''^src/libexpr/eval-settings\.hh$''
+        ''^src/libexpr/eval\.cc$''
+        ''^src/libexpr/eval\.hh$''
+        ''^src/libexpr/flake/config\.cc$''
+        ''^src/libexpr/flake/flake\.cc$''
+        ''^src/libexpr/flake/flake\.hh$''
+        ''^src/libexpr/flake/flakeref\.cc$''
+        ''^src/libexpr/flake/flakeref\.hh$''
+        ''^src/libexpr/flake/lockfile\.cc$''
+        ''^src/libexpr/flake/lockfile\.hh$''
+        ''^src/libexpr/flake/url-name\.cc$''
+        ''^src/libexpr/function-trace\.cc$''
+        ''^src/libexpr/gc-small-vector\.hh$''
+        ''^src/libexpr/get-drvs\.cc$''
+        ''^src/libexpr/get-drvs\.hh$''
+        ''^src/libexpr/json-to-value\.cc$''
+        ''^src/libexpr/nixexpr\.cc$''
+        ''^src/libexpr/nixexpr\.hh$''
+        ''^src/libexpr/parser-state\.hh$''
+        ''^src/libexpr/pos-table\.hh$''
+        ''^src/libexpr/primops\.cc$''
+        ''^src/libexpr/primops\.hh$''
+        ''^src/libexpr/primops/context\.cc$''
+        ''^src/libexpr/primops/fetchClosure\.cc$''
+        ''^src/libexpr/primops/fetchMercurial\.cc$''
+        ''^src/libexpr/primops/fetchTree\.cc$''
+        ''^src/libexpr/primops/fromTOML\.cc$''
+        ''^src/libexpr/print-ambiguous\.cc$''
+        ''^src/libexpr/print-ambiguous\.hh$''
+        ''^src/libexpr/print-options\.hh$''
+        ''^src/libexpr/print\.cc$''
+        ''^src/libexpr/print\.hh$''
+        ''^src/libexpr/search-path\.cc$''
+        ''^src/libexpr/symbol-table\.hh$''
+        ''^src/libexpr/value-to-json\.cc$''
+        ''^src/libexpr/value-to-json\.hh$''
+        ''^src/libexpr/value-to-xml\.cc$''
+        ''^src/libexpr/value-to-xml\.hh$''
+        ''^src/libexpr/value\.hh$''
+        ''^src/libexpr/value/context\.cc$''
+        ''^src/libexpr/value/context\.hh$''
+        ''^src/libfetchers/attrs\.cc$''
+        ''^src/libfetchers/cache\.cc$''
+        ''^src/libfetchers/cache\.hh$''
+        ''^src/libfetchers/fetch-settings\.cc$''
+        ''^src/libfetchers/fetch-settings\.hh$''
+        ''^src/libfetchers/fetch-to-store\.cc$''
+        ''^src/libfetchers/fetchers\.cc$''
+        ''^src/libfetchers/fetchers\.hh$''
+        ''^src/libfetchers/filtering-input-accessor\.cc$''
+        ''^src/libfetchers/filtering-input-accessor\.hh$''
+        ''^src/libfetchers/fs-input-accessor\.cc$''
+        ''^src/libfetchers/fs-input-accessor\.hh$''
+        ''^src/libfetchers/git-utils\.cc$''
+        ''^src/libfetchers/git-utils\.hh$''
+        ''^src/libfetchers/github\.cc$''
+        ''^src/libfetchers/indirect\.cc$''
+        ''^src/libfetchers/memory-input-accessor\.cc$''
+        ''^src/libfetchers/path\.cc$''
+        ''^src/libfetchers/registry\.cc$''
+        ''^src/libfetchers/registry\.hh$''
+        ''^src/libfetchers/tarball\.cc$''
+        ''^src/libfetchers/tarball\.hh$''
+        ''^src/libfetchers/unix/git\.cc$''
+        ''^src/libfetchers/unix/mercurial\.cc$''
+        ''^src/libmain/common-args\.cc$''
+        ''^src/libmain/common-args\.hh$''
+        ''^src/libmain/loggers\.cc$''
+        ''^src/libmain/loggers\.hh$''
+        ''^src/libmain/progress-bar\.cc$''
+        ''^src/libmain/shared\.cc$''
+        ''^src/libmain/shared\.hh$''
+        ''^src/libmain/unix/stack\.cc$''
+        ''^src/libstore/binary-cache-store\.cc$''
+        ''^src/libstore/binary-cache-store\.hh$''
+        ''^src/libstore/build-result\.hh$''
+        ''^src/libstore/builtins\.hh$''
+        ''^src/libstore/builtins/buildenv\.cc$''
+        ''^src/libstore/builtins/buildenv\.hh$''
+        ''^src/libstore/common-protocol-impl\.hh$''
+        ''^src/libstore/common-protocol\.cc$''
+        ''^src/libstore/common-protocol\.hh$''
+        ''^src/libstore/content-address\.cc$''
+        ''^src/libstore/content-address\.hh$''
+        ''^src/libstore/daemon\.cc$''
+        ''^src/libstore/daemon\.hh$''
+        ''^src/libstore/derivations\.cc$''
+        ''^src/libstore/derivations\.hh$''
+        ''^src/libstore/derived-path-map\.cc$''
+        ''^src/libstore/derived-path-map\.hh$''
+        ''^src/libstore/derived-path\.cc$''
+        ''^src/libstore/derived-path\.hh$''
+        ''^src/libstore/downstream-placeholder\.cc$''
+        ''^src/libstore/downstream-placeholder\.hh$''
+        ''^src/libstore/dummy-store\.cc$''
+        ''^src/libstore/export-import\.cc$''
+        ''^src/libstore/filetransfer\.cc$''
+        ''^src/libstore/filetransfer\.hh$''
+        ''^src/libstore/gc-store\.hh$''
+        ''^src/libstore/globals\.cc$''
+        ''^src/libstore/globals\.hh$''
+        ''^src/libstore/http-binary-cache-store\.cc$''
+        ''^src/libstore/legacy-ssh-store\.cc$''
+        ''^src/libstore/legacy-ssh-store\.hh$''
+        ''^src/libstore/length-prefixed-protocol-helper\.hh$''
+        ''^src/libstore/linux/personality\.cc$''
+        ''^src/libstore/linux/personality\.hh$''
+        ''^src/libstore/local-binary-cache-store\.cc$''
+        ''^src/libstore/local-fs-store\.cc$''
+        ''^src/libstore/local-fs-store\.hh$''
+        ''^src/libstore/log-store\.cc$''
+        ''^src/libstore/log-store\.hh$''
+        ''^src/libstore/machines\.cc$''
+        ''^src/libstore/machines\.hh$''
+        ''^src/libstore/make-content-addressed\.cc$''
+        ''^src/libstore/make-content-addressed\.hh$''
+        ''^src/libstore/misc\.cc$''
+        ''^src/libstore/names\.cc$''
+        ''^src/libstore/names\.hh$''
+        ''^src/libstore/nar-accessor\.cc$''
+        ''^src/libstore/nar-accessor\.hh$''
+        ''^src/libstore/nar-info-disk-cache\.cc$''
+        ''^src/libstore/nar-info-disk-cache\.hh$''
+        ''^src/libstore/nar-info\.cc$''
+        ''^src/libstore/nar-info\.hh$''
+        ''^src/libstore/outputs-spec\.cc$''
+        ''^src/libstore/outputs-spec\.hh$''
+        ''^src/libstore/parsed-derivations\.cc$''
+        ''^src/libstore/path-info\.cc$''
+        ''^src/libstore/path-info\.hh$''
+        ''^src/libstore/path-references\.cc$''
+        ''^src/libstore/path-regex\.hh$''
+        ''^src/libstore/path-with-outputs\.cc$''
+        ''^src/libstore/path\.cc$''
+        ''^src/libstore/path\.hh$''
+        ''^src/libstore/pathlocks\.cc$''
+        ''^src/libstore/pathlocks\.hh$''
+        ''^src/libstore/profiles\.cc$''
+        ''^src/libstore/profiles\.hh$''
+        ''^src/libstore/realisation\.cc$''
+        ''^src/libstore/realisation\.hh$''
+        ''^src/libstore/remote-fs-accessor\.cc$''
+        ''^src/libstore/remote-fs-accessor\.hh$''
+        ''^src/libstore/remote-store-connection\.hh$''
+        ''^src/libstore/remote-store\.cc$''
+        ''^src/libstore/remote-store\.hh$''
+        ''^src/libstore/s3-binary-cache-store\.cc$''
+        ''^src/libstore/s3\.hh$''
+        ''^src/libstore/serve-protocol-impl\.cc$''
+        ''^src/libstore/serve-protocol-impl\.hh$''
+        ''^src/libstore/serve-protocol\.cc$''
+        ''^src/libstore/serve-protocol\.hh$''
+        ''^src/libstore/sqlite\.cc$''
+        ''^src/libstore/sqlite\.hh$''
+        ''^src/libstore/ssh-store-config\.hh$''
+        ''^src/libstore/ssh-store\.cc$''
+        ''^src/libstore/ssh\.cc$''
+        ''^src/libstore/ssh\.hh$''
+        ''^src/libstore/store-api\.cc$''
+        ''^src/libstore/store-api\.hh$''
+        ''^src/libstore/store-dir-config\.hh$''
+        ''^src/libstore/unix/build/derivation-goal\.cc$''
+        ''^src/libstore/unix/build/derivation-goal\.hh$''
+        ''^src/libstore/unix/build/drv-output-substitution-goal\.cc$''
+        ''^src/libstore/unix/build/drv-output-substitution-goal\.hh$''
+        ''^src/libstore/unix/build/entry-points\.cc$''
+        ''^src/libstore/unix/build/goal\.cc$''
+        ''^src/libstore/unix/build/goal\.hh$''
+        ''^src/libstore/unix/build/hook-instance\.cc$''
+        ''^src/libstore/unix/build/local-derivation-goal\.cc$''
+        ''^src/libstore/unix/build/local-derivation-goal\.hh$''
+        ''^src/libstore/unix/build/substitution-goal\.cc$''
+        ''^src/libstore/unix/build/substitution-goal\.hh$''
+        ''^src/libstore/unix/build/worker\.cc$''
+        ''^src/libstore/unix/build/worker\.hh$''
+        ''^src/libstore/unix/builtins/fetchurl\.cc$''
+        ''^src/libstore/unix/builtins/unpack-channel\.cc$''
+        ''^src/libstore/unix/gc\.cc$''
+        ''^src/libstore/unix/local-overlay-store\.cc$''
+        ''^src/libstore/unix/local-overlay-store\.hh$''
+        ''^src/libstore/unix/local-store\.cc$''
+        ''^src/libstore/unix/local-store\.hh$''
+        ''^src/libstore/unix/lock\.cc$''
+        ''^src/libstore/unix/lock\.hh$''
+        ''^src/libstore/unix/optimise-store\.cc$''
+        ''^src/libstore/unix/pathlocks\.cc$''
+        ''^src/libstore/unix/posix-fs-canonicalise\.cc$''
+        ''^src/libstore/unix/posix-fs-canonicalise\.hh$''
+        ''^src/libstore/unix/uds-remote-store\.cc$''
+        ''^src/libstore/unix/uds-remote-store\.hh$''
+        ''^src/libstore/windows/build\.cc$''
+        ''^src/libstore/worker-protocol-impl\.hh$''
+        ''^src/libstore/worker-protocol\.cc$''
+        ''^src/libstore/worker-protocol\.hh$''
+        ''^src/libutil-c/nix_api_util_internal\.h$''
+        ''^src/libutil/archive\.cc$''
+        ''^src/libutil/archive\.hh$''
+        ''^src/libutil/args\.cc$''
+        ''^src/libutil/args\.hh$''
+        ''^src/libutil/args/root\.hh$''
+        ''^src/libutil/callback\.hh$''
+        ''^src/libutil/canon-path\.cc$''
+        ''^src/libutil/canon-path\.hh$''
+        ''^src/libutil/chunked-vector\.hh$''
+        ''^src/libutil/closure\.hh$''
+        ''^src/libutil/comparator\.hh$''
+        ''^src/libutil/compute-levels\.cc$''
+        ''^src/libutil/config-impl\.hh$''
+        ''^src/libutil/config\.cc$''
+        ''^src/libutil/config\.hh$''
+        ''^src/libutil/current-process\.cc$''
+        ''^src/libutil/current-process\.hh$''
+        ''^src/libutil/english\.cc$''
+        ''^src/libutil/english\.hh$''
+        ''^src/libutil/environment-variables\.cc$''
+        ''^src/libutil/error\.cc$''
+        ''^src/libutil/error\.hh$''
+        ''^src/libutil/exit\.hh$''
+        ''^src/libutil/experimental-features\.cc$''
+        ''^src/libutil/experimental-features\.hh$''
+        ''^src/libutil/file-content-address\.cc$''
+        ''^src/libutil/file-content-address\.hh$''
+        ''^src/libutil/file-descriptor\.cc$''
+        ''^src/libutil/file-descriptor\.hh$''
+        ''^src/libutil/file-path-impl\.hh$''
+        ''^src/libutil/file-path\.hh$''
+        ''^src/libutil/file-system\.cc$''
+        ''^src/libutil/file-system\.hh$''
+        ''^src/libutil/finally\.hh$''
+        ''^src/libutil/fmt\.hh$''
+        ''^src/libutil/fs-sink\.cc$''
+        ''^src/libutil/fs-sink\.hh$''
+        ''^src/libutil/git\.cc$''
+        ''^src/libutil/git\.hh$''
+        ''^src/libutil/hash\.cc$''
+        ''^src/libutil/hash\.hh$''
+        ''^src/libutil/hilite\.cc$''
+        ''^src/libutil/hilite\.hh$''
+        ''^src/libutil/input-accessor\.hh$''
+        ''^src/libutil/json-impls\.hh$''
+        ''^src/libutil/json-utils\.cc$''
+        ''^src/libutil/json-utils\.hh$''
+        ''^src/libutil/linux/cgroup\.cc$''
+        ''^src/libutil/linux/namespaces\.cc$''
+        ''^src/libutil/logging\.cc$''
+        ''^src/libutil/logging\.hh$''
+        ''^src/libutil/lru-cache\.hh$''
+        ''^src/libutil/memory-source-accessor\.cc$''
+        ''^src/libutil/memory-source-accessor\.hh$''
+        ''^src/libutil/pool\.hh$''
+        ''^src/libutil/position\.cc$''
+        ''^src/libutil/position\.hh$''
+        ''^src/libutil/posix-source-accessor\.cc$''
+        ''^src/libutil/posix-source-accessor\.hh$''
+        ''^src/libutil/processes\.hh$''
+        ''^src/libutil/ref\.hh$''
+        ''^src/libutil/references\.cc$''
+        ''^src/libutil/references\.hh$''
+        ''^src/libutil/regex-combinators\.hh$''
+        ''^src/libutil/serialise\.cc$''
+        ''^src/libutil/serialise\.hh$''
+        ''^src/libutil/signals\.hh$''
+        ''^src/libutil/signature/local-keys\.cc$''
+        ''^src/libutil/signature/local-keys\.hh$''
+        ''^src/libutil/signature/signer\.cc$''
+        ''^src/libutil/signature/signer\.hh$''
+        ''^src/libutil/source-accessor\.cc$''
+        ''^src/libutil/source-accessor\.hh$''
+        ''^src/libutil/source-path\.cc$''
+        ''^src/libutil/source-path\.hh$''
+        ''^src/libutil/split\.hh$''
+        ''^src/libutil/suggestions\.cc$''
+        ''^src/libutil/suggestions\.hh$''
+        ''^src/libutil/sync\.hh$''
+        ''^src/libutil/terminal\.cc$''
+        ''^src/libutil/terminal\.hh$''
+        ''^src/libutil/thread-pool\.cc$''
+        ''^src/libutil/thread-pool\.hh$''
+        ''^src/libutil/topo-sort\.hh$''
+        ''^src/libutil/types\.hh$''
+        ''^src/libutil/unix/file-descriptor\.cc$''
+        ''^src/libutil/unix/file-path\.cc$''
+        ''^src/libutil/unix/monitor-fd\.hh$''
+        ''^src/libutil/unix/processes\.cc$''
+        ''^src/libutil/unix/signals-impl\.hh$''
+        ''^src/libutil/unix/signals\.cc$''
+        ''^src/libutil/unix/unix-domain-socket\.cc$''
+        ''^src/libutil/unix/users\.cc$''
+        ''^src/libutil/url-parts\.hh$''
+        ''^src/libutil/url\.cc$''
+        ''^src/libutil/url\.hh$''
+        ''^src/libutil/users\.cc$''
+        ''^src/libutil/users\.hh$''
+        ''^src/libutil/util\.cc$''
+        ''^src/libutil/util\.hh$''
+        ''^src/libutil/variant-wrapper\.hh$''
+        ''^src/libutil/windows/environment-variables\.cc$''
+        ''^src/libutil/windows/file-descriptor\.cc$''
+        ''^src/libutil/windows/file-path\.cc$''
+        ''^src/libutil/windows/processes\.cc$''
+        ''^src/libutil/windows/users\.cc$''
+        ''^src/libutil/windows/windows-error\.cc$''
+        ''^src/libutil/windows/windows-error\.hh$''
+        ''^src/libutil/xml-writer\.cc$''
+        ''^src/libutil/xml-writer\.hh$''
+        ''^src/nix-build/nix-build\.cc$''
+        ''^src/nix-channel/nix-channel\.cc$''
+        ''^src/nix-collect-garbage/nix-collect-garbage\.cc$''
+        ''^src/nix-env/buildenv.nix$''
+        ''^src/nix-env/nix-env\.cc$''
+        ''^src/nix-env/user-env\.cc$''
+        ''^src/nix-env/user-env\.hh$''
+        ''^src/nix-instantiate/nix-instantiate\.cc$''
+        ''^src/nix-store/dotgraph\.cc$''
+        ''^src/nix-store/graphml\.cc$''
+        ''^src/nix-store/nix-store\.cc$''
+        ''^src/nix/add-to-store\.cc$''
+        ''^src/nix/app\.cc$''
+        ''^src/nix/build\.cc$''
+        ''^src/nix/bundle\.cc$''
+        ''^src/nix/cat\.cc$''
+        ''^src/nix/config-check\.cc$''
+        ''^src/nix/config\.cc$''
+        ''^src/nix/copy\.cc$''
+        ''^src/nix/derivation-add\.cc$''
+        ''^src/nix/derivation-show\.cc$''
+        ''^src/nix/derivation\.cc$''
+        ''^src/nix/develop\.cc$''
+        ''^src/nix/diff-closures\.cc$''
+        ''^src/nix/dump-path\.cc$''
+        ''^src/nix/edit\.cc$''
+        ''^src/nix/eval\.cc$''
+        ''^src/nix/flake\.cc$''
+        ''^src/nix/fmt\.cc$''
+        ''^src/nix/hash\.cc$''
+        ''^src/nix/log\.cc$''
+        ''^src/nix/ls\.cc$''
+        ''^src/nix/main\.cc$''
+        ''^src/nix/make-content-addressed\.cc$''
+        ''^src/nix/nar\.cc$''
+        ''^src/nix/optimise-store\.cc$''
+        ''^src/nix/path-from-hash-part\.cc$''
+        ''^src/nix/path-info\.cc$''
+        ''^src/nix/prefetch\.cc$''
+        ''^src/nix/profile\.cc$''
+        ''^src/nix/realisation\.cc$''
+        ''^src/nix/registry\.cc$''
+        ''^src/nix/repl\.cc$''
+        ''^src/nix/run\.cc$''
+        ''^src/nix/run\.hh$''
+        ''^src/nix/search\.cc$''
+        ''^src/nix/sigs\.cc$''
+        ''^src/nix/store-copy-log\.cc$''
+        ''^src/nix/store-delete\.cc$''
+        ''^src/nix/store-gc\.cc$''
+        ''^src/nix/store-info\.cc$''
+        ''^src/nix/store-repair\.cc$''
+        ''^src/nix/store\.cc$''
+        ''^src/nix/unix/daemon\.cc$''
+        ''^src/nix/upgrade-nix\.cc$''
+        ''^src/nix/verify\.cc$''
+        ''^src/nix/why-depends\.cc$''
+      ];
+    };
+
+  };
+
+  # We'll be pulling from this in the main flake
+  flake.getSystem = getSystem;
+}

maintainers/local.mk

@@ -0,0 +1,15 @@
+
+.PHONY: format
+print-top-help += echo '  format: Format source code'
+
+# This uses the cached .pre-commit-hooks.yaml file
+format:
+	@if ! type -p pre-commit &>/dev/null; then \
+	  echo "make format: pre-commit not found. Please use \`nix develop\`."; \
+	  exit 1; \
+	fi; \
+	if test -z "$$_NIX_PRE_COMMIT_HOOKS_CONFIG"; then \
+	  echo "make format: _NIX_PRE_COMMIT_HOOKS_CONFIG not set. Please use \`nix develop\`."; \
+	  exit 1; \
+	fi; \
+	pre-commit run --config $$_NIX_PRE_COMMIT_HOOKS_CONFIG --all-files

maintainers/upload-release.pl

@@ -11,6 +11,8 @@ use JSON::PP;
 use LWP::UserAgent;
 use Net::Amazon::S3;
 
+delete $ENV{'shell'}; # shut up a LWP::UserAgent.pm warning
+
 my $evalId = $ARGV[0] or die "Usage: $0 EVAL-ID\n";
 
 my $releasesBucketName = "nix-releases";
@@ -36,9 +38,9 @@ sub fetch {
 my $evalUrl = "https://hydra.nixos.org/eval/$evalId";
 my $evalInfo = decode_json(fetch($evalUrl, 'application/json'));
 #print Dumper($evalInfo);
-my $flakeUrl = $evalInfo->{flake} or die;
-my $flakeInfo = decode_json(`nix flake metadata --json "$flakeUrl"` or die);
-my $nixRev = $flakeInfo->{revision} or die;
+my $flakeUrl = $evalInfo->{flake};
+my $flakeInfo = decode_json(`nix flake metadata --json "$flakeUrl"` or die) if $flakeUrl;
+my $nixRev = ($flakeInfo ? $flakeInfo->{revision} : $evalInfo->{jobsetevalinputs}->{nix}->{revision}) or die;
 
 my $buildInfo = decode_json(fetch("$evalUrl/job/build.x86_64-linux", 'application/json'));
 #print Dumper($buildInfo);
@@ -83,12 +85,19 @@ my $channelsBucket = $s3_us->bucket($channelsBucketName) or die;
 sub getStorePath {
     my ($jobName, $output) = @_;
     my $buildInfo = decode_json(fetch("$evalUrl/job/$jobName", 'application/json'));
-    return $buildInfo->{buildoutputs}->{$output or "out"}->{path} or die "cannot get store path for '$jobName'";
+    return $buildInfo->{buildoutputs}->{$output or "out"}->{path} // die "cannot get store path for '$jobName'";
 }
 
 sub copyManual {
-    my $manual = getStorePath("build.x86_64-linux", "doc");
-    print "$manual\n";
+    my $manual;
+    eval {
+        $manual = getStorePath("build.x86_64-linux", "doc");
+    };
+    if ($@) {
+        warn "$@";
+        return;
+    }
+    print "Manual: $manual\n";
 
     my $manualNar = "$tmpDir/$releaseName-manual.nar.xz";
     print "$manualNar\n";
@@ -154,19 +163,37 @@ downloadFile("binaryTarball.x86_64-linux", "1");
 downloadFile("binaryTarball.aarch64-linux", "1");
 downloadFile("binaryTarball.x86_64-darwin", "1");
 downloadFile("binaryTarball.aarch64-darwin", "1");
-downloadFile("binaryTarballCross.x86_64-linux.armv6l-unknown-linux-gnueabihf", "1");
-downloadFile("binaryTarballCross.x86_64-linux.armv7l-unknown-linux-gnueabihf", "1");
+eval {
+    downloadFile("binaryTarballCross.x86_64-linux.armv6l-unknown-linux-gnueabihf", "1");
+};
+warn "$@" if $@;
+eval {
+    downloadFile("binaryTarballCross.x86_64-linux.armv7l-unknown-linux-gnueabihf", "1");
+};
+warn "$@" if $@;
+eval {
+    downloadFile("binaryTarballCross.x86_64-linux.riscv64-unknown-linux-gnu", "1");
+};
+warn "$@" if $@;
 downloadFile("installerScript", "1");
 
 # Upload docker images to dockerhub.
 my $dockerManifest = "";
 my $dockerManifestLatest = "";
+my $haveDocker = 0;
 
 for my $platforms (["x86_64-linux", "amd64"], ["aarch64-linux", "arm64"]) {
     my $system = $platforms->[0];
     my $dockerPlatform = $platforms->[1];
     my $fn = "nix-$version-docker-image-$dockerPlatform.tar.gz";
-    downloadFile("dockerImage.$system", "1", $fn);
+    eval {
+        downloadFile("dockerImage.$system", "1", $fn);
+    };
+    if ($@) {
+        warn "$@" if $@;
+        next;
+    }
+    $haveDocker = 1;
 
     print STDERR "loading docker image for $dockerPlatform...\n";
     system("docker load -i $tmpDir/$fn") == 0 or die;
@@ -194,21 +221,23 @@ for my $platforms (["x86_64-linux", "amd64"], ["aarch64-linux", "arm64"]) {
     $dockerManifestLatest .= " --amend $latestTag"
 }
 
-print STDERR "creating multi-platform docker manifest...\n";
-system("docker manifest rm nixos/nix:$version");
-system("docker manifest create nixos/nix:$version $dockerManifest") == 0 or die;
-if ($isLatest) {
-    print STDERR "creating latest multi-platform docker manifest...\n";
-    system("docker manifest rm nixos/nix:latest");
-    system("docker manifest create nixos/nix:latest $dockerManifestLatest") == 0 or die;
-}
+if ($haveDocker) {
+    print STDERR "creating multi-platform docker manifest...\n";
+    system("docker manifest rm nixos/nix:$version");
+    system("docker manifest create nixos/nix:$version $dockerManifest") == 0 or die;
+    if ($isLatest) {
+        print STDERR "creating latest multi-platform docker manifest...\n";
+        system("docker manifest rm nixos/nix:latest");
+        system("docker manifest create nixos/nix:latest $dockerManifestLatest") == 0 or die;
+    }
 
-print STDERR "pushing multi-platform docker manifest...\n";
-system("docker manifest push nixos/nix:$version") == 0 or die;
+    print STDERR "pushing multi-platform docker manifest...\n";
+    system("docker manifest push nixos/nix:$version") == 0 or die;
 
-if ($isLatest) {
-    print STDERR "pushing latest multi-platform docker manifest...\n";
-    system("docker manifest push nixos/nix:latest") == 0 or die;
+    if ($isLatest) {
+        print STDERR "pushing latest multi-platform docker manifest...\n";
+        system("docker manifest push nixos/nix:latest") == 0 or die;
+    }
 }
 
 # Upload nix-fallback-paths.nix.

mk/compilation-database.mk

@@ -0,0 +1,11 @@
+compile-commands-json-files :=
+
+define write-compile-commands
+  _srcs := $$(sort $$(foreach src, $$($(1)_SOURCES), $$(src)))
+
+  $(1)_COMPILE_COMMANDS_JSON := $$(addprefix $(buildprefix), $$(addsuffix .compile_commands.json, $$(basename $$(_srcs))))
+
+  compile-commands-json-files += $$($(1)_COMPILE_COMMANDS_JSON)
+
+  clean-files += $$($(1)_COMPILE_COMMANDS_JSON)
+endef

mk/cxx-big-literal.mk

@@ -1,5 +1,5 @@
 %.gen.hh: %
-	@echo 'R"foo(' >> $@.tmp
+	@echo 'R"__NIX_STR(' >> $@.tmp
 	$(trace-gen) cat $< >> $@.tmp
-	@echo ')foo"' >> $@.tmp
+	@echo ')__NIX_STR"' >> $@.tmp
 	@mv $@.tmp $@

mk/lib.mk

@@ -68,6 +68,7 @@ include mk/patterns.mk
 include mk/templates.mk
 include mk/cxx-big-literal.mk
 include mk/tests.mk
+include mk/compilation-database.mk
 
 
 # Include all sub-Makefiles.
@@ -97,6 +98,13 @@ $(foreach test-group, $(install-tests-groups), \
     $(eval $(call run-test,$(test),$(install_test_init))) \
     $(eval $(test-group).test-group: $(test).test)))
 
+# Compilation database.
+$(foreach lib, $(libraries), $(eval $(call write-compile-commands,$(lib))))
+$(foreach prog, $(programs), $(eval $(call write-compile-commands,$(prog))))
+
+compile_commands.json: $(compile-commands-json-files)
+	@jq --slurp '.' $^ >$@
+
 # Include makefiles requiring built programs.
 $(foreach mf, $(makefiles-late), $(eval $(call include-sub-makefile,$(mf))))
 

mk/patterns.mk

@@ -1,11 +1,41 @@
+
+# These are the complete command lines we use to compile C and C++ files.
+# - $< is the source file.
+# - $1 is the object file to create.
+CC_CMD=$(CC) -o $1 -c $< $(CPPFLAGS) $(GLOBAL_CFLAGS) $(CFLAGS) $($1_CFLAGS) -MMD -MF $(call filename-to-dep,$1) -MP
+CXX_CMD=$(CXX) -o $1 -c $< $(CPPFLAGS) $(GLOBAL_CXXFLAGS_PCH) $(GLOBAL_CXXFLAGS) $(CXXFLAGS) $($1_CXXFLAGS) $(ERROR_SWITCH_ENUM) -MMD -MF $(call filename-to-dep,$1) -MP
+
+# We use COMPILE_COMMANDS_JSON_CMD to turn a compilation command (like CC_CMD
+# or CXX_CMD above) into a comple_commands.json file. We rely on bash native
+# word splitting to define the positional arguments.
+# - $< is the source file being compiled.
+COMPILE_COMMANDS_JSON_CMD=jq --null-input '{ directory: $$ENV.PWD, file: "$<", arguments: $$ARGS.positional }' --args --
+
+
 $(buildprefix)%.o: %.cc
 	@mkdir -p "$(dir $@)"
-	$(trace-cxx) $(CXX) -o $@ -c $< $(CPPFLAGS) $(GLOBAL_CXXFLAGS_PCH) $(GLOBAL_CXXFLAGS) $(CXXFLAGS) $($@_CXXFLAGS) $(ERROR_SWITCH_ENUM) -MMD -MF $(call filename-to-dep, $@) -MP
+	$(trace-cxx) $(call CXX_CMD,$@)
 
 $(buildprefix)%.o: %.cpp
 	@mkdir -p "$(dir $@)"
-	$(trace-cxx) $(CXX) -o $@ -c $< $(CPPFLAGS) $(GLOBAL_CXXFLAGS_PCH) $(GLOBAL_CXXFLAGS) $(CXXFLAGS) $($@_CXXFLAGS) $(ERROR_SWITCH_ENUM) -MMD -MF $(call filename-to-dep, $@) -MP
+	$(trace-cxx) $(call CXX_CMD,$@)
 
 $(buildprefix)%.o: %.c
 	@mkdir -p "$(dir $@)"
-	$(trace-cc) $(CC) -o $@ -c $< $(CPPFLAGS) $(GLOBAL_CFLAGS) $(CFLAGS) $($@_CFLAGS) -MMD -MF $(call filename-to-dep, $@) -MP
+	$(trace-cc) $(call CC_CMD,$@)
+
+# In the following we need to replace the .compile_commands.json extension in $@ with .o
+# to make the object file. This is needed because CC_CMD and CXX_CMD do further expansions
+# based on the object file name (i.e. *_CXXFLAGS and filename-to-dep).
+
+$(buildprefix)%.compile_commands.json: %.cc
+	@mkdir -p "$(dir $@)"
+	$(trace-jq) $(COMPILE_COMMANDS_JSON_CMD) $(call CXX_CMD,$(@:.compile_commands.json=.o)) > $@
+
+$(buildprefix)%.compile_commands.json: %.cpp
+	@mkdir -p "$(dir $@)"
+	$(trace-jq) $(COMPILE_COMMANDS_JSON_CMD) $(call CXX_CMD,$(@:.compile_commands.json=.o)) > $@
+
+$(buildprefix)%.compile_commands.json: %.c
+	@mkdir -p "$(dir $@)"
+	$(trace-jq) $(COMPILE_COMMANDS_JSON_CMD) $(call CC_CMD,$(@:.compile_commands.json=.o)) > $@

mk/tracing.mk

@@ -10,6 +10,8 @@ ifeq ($(V), 0)
   trace-install = @echo "  INST  " $@;
   trace-mkdir   = @echo "  MKDIR " $@;
   trace-test    = @echo "  TEST  " $@;
+  trace-sh      = @echo "  SH    " $@;
+  trace-jq      = @echo "  JQ    " $@;
 
   suppress  = @
 

package.nix

@@ -1,10 +1,12 @@
 { lib
+, fetchurl
 , stdenv
 , releaseTools
 , autoconf-archive
 , autoreconfHook
 , aws-sdk-cpp
 , boehmgc
+, buildPackages
 , nlohmann_json
 , bison
 , boost
@@ -24,6 +26,7 @@
 , libgit2
 , libseccomp
 , libsodium
+, man
 , lowdown
 , mdbook
 , mdbook-linkcheck
@@ -74,7 +77,10 @@
 # sounds so long as evaluation just takes places within short-lived
 # processes. (When the process exits, the memory is reclaimed; it is
 # only leaked *within* the process.)
-, enableGC ? true
+#
+# Temporarily disabled on Windows because the `GC_throw_bad_alloc`
+# symbol is missing during linking.
+, enableGC ? !stdenv.hostPlatform.isWindows
 
 # Whether to enable Markdown rendering in the Nix binary.
 , enableMarkdown ? !stdenv.hostPlatform.isWindows
@@ -87,9 +93,10 @@
 # - readline
 , readlineFlavor ? if stdenv.hostPlatform.isWindows then "readline" else "editline"
 
-# Whether to build the internal API docs, can be done separately from
+# Whether to build the internal/external API docs, can be done separately from
 # everything else.
-, enableInternalAPIDocs ? false
+, enableInternalAPIDocs ? forDevShell
+, enableExternalAPIDocs ? forDevShell
 
 # Whether to install unit tests. This is useful when cross compiling
 # since we cannot run them natively during the build, but can do so
@@ -154,13 +161,15 @@ in {
     in
       fileset.toSource {
         root = ./.;
-        fileset = fileset.intersect baseFiles (fileset.unions ([
+        fileset = fileset.intersection baseFiles (fileset.unions ([
           # For configure
           ./.version
           ./configure.ac
           ./m4
           # TODO: do we really need README.md? It doesn't seem used in the build.
           ./README.md
+          # This could be put behind a conditional
+          ./maintainers/local.mk
           # For make, regardless of what we are building
           ./local.mk
           ./Makefile
@@ -178,6 +187,9 @@ in {
           ./doc/manual
         ] ++ lib.optionals enableInternalAPIDocs [
           ./doc/internal-api
+        ] ++ lib.optionals enableExternalAPIDocs [
+          ./doc/external-api
+        ] ++ lib.optionals (enableInternalAPIDocs || enableExternalAPIDocs) [
           # Source might not be compiled, but still must be available
           # for Doxygen to gather comments.
           ./src
@@ -195,7 +207,7 @@ in {
     ++ lib.optional doBuild "dev"
     # If we are doing just build or just docs, the one thing will use
     # "out". We only need additional outputs if we are doing both.
-    ++ lib.optional (doBuild && (enableManual || enableInternalAPIDocs)) "doc"
+    ++ lib.optional (doBuild && (enableManual || enableInternalAPIDocs || enableExternalAPIDocs)) "doc"
     ++ lib.optional installUnitTests "check";
 
   nativeBuildInputs = [
@@ -209,10 +221,15 @@ in {
     (lib.getBin lowdown)
     mdbook
     mdbook-linkcheck
+  ] ++ lib.optionals doInstallCheck [
+    git
+    mercurial
+    openssh
+    man # for testing `nix-* --help`
   ] ++ lib.optionals (doInstallCheck || enableManual) [
     jq # Also for custom mdBook preprocessor.
   ] ++ lib.optional stdenv.hostPlatform.isLinux util-linux
-    ++ lib.optional enableInternalAPIDocs doxygen
+    ++ lib.optional (enableInternalAPIDocs || enableExternalAPIDocs) doxygen
   ;
 
   buildInputs = lib.optionals doBuild [
@@ -232,7 +249,13 @@ in {
   ] ++ lib.optionals buildUnitTests [
     gtest
     rapidcheck
-  ] ++ lib.optional stdenv.isLinux libseccomp
+  ] ++ lib.optional stdenv.isLinux (libseccomp.overrideAttrs (_: rec {
+    version = "2.5.5";
+    src = fetchurl {
+      url = "https://github.com/seccomp/libseccomp/releases/download/v${version}/libseccomp-${version}.tar.gz";
+      hash = "sha256-JIosik2bmFiqa69ScSw0r+/PnJ6Ut23OAsHJqiX7M3U=";
+    };
+  }))
     ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid
     # There have been issues building these dependencies
     ++ lib.optional (stdenv.hostPlatform == stdenv.buildPlatform && (stdenv.isLinux || stdenv.isDarwin))
@@ -249,12 +272,6 @@ in {
   dontBuild = !attrs.doBuild;
   doCheck = attrs.doCheck;
 
-  nativeCheckInputs = [
-    git
-    mercurial
-    openssh
-  ];
-
   disallowedReferences = [ boost ];
 
   preConfigure = lib.optionalString (doBuild && ! stdenv.hostPlatform.isStatic) (
@@ -282,6 +299,7 @@ in {
     (lib.enableFeature buildUnitTests "unit-tests")
     (lib.enableFeature doInstallCheck "functional-tests")
     (lib.enableFeature enableInternalAPIDocs "internal-api-docs")
+    (lib.enableFeature enableExternalAPIDocs "external-api-docs")
     (lib.enableFeature enableManual "doc-gen")
     (lib.enableFeature enableGC "gc")
     (lib.enableFeature enableMarkdown "markdown")
@@ -306,7 +324,8 @@ in {
   makeFlags = "profiledir=$(out)/etc/profile.d PRECOMPILE_HEADERS=1";
 
   installTargets = lib.optional doBuild "install"
-    ++ lib.optional enableInternalAPIDocs "internal-api-html";
+    ++ lib.optional enableInternalAPIDocs "internal-api-html"
+    ++ lib.optional enableExternalAPIDocs "external-api-html";
 
   installFlags = "sysconfdir=$(out)/etc";
 
@@ -333,6 +352,16 @@ in {
   '' + lib.optionalString enableInternalAPIDocs ''
     mkdir -p ''${!outputDoc}/nix-support
     echo "doc internal-api-docs $out/share/doc/nix/internal-api/html" >> ''${!outputDoc}/nix-support/hydra-build-products
+  ''
+    + lib.optionalString enableExternalAPIDocs ''
+    mkdir -p ''${!outputDoc}/nix-support
+    echo "doc external-api-docs $out/share/doc/nix/external-api/html" >> ''${!outputDoc}/nix-support/hydra-build-products
+  '';
+
+  # So the check output gets links for DLLs in the out output.
+  preFixup = lib.optionalString (stdenv.hostPlatform.isWindows && builtins.elem "check" finalAttrs.outputs) ''
+    ln -s "$check/lib/"*.dll "$check/bin"
+    ln -s "$out/bin/"*.dll "$check/bin"
   '';
 
   doInstallCheck = attrs.doInstallCheck;
@@ -343,15 +372,22 @@ in {
 
   # Work around weird bug where it doesn't think there is a Makefile.
   installCheckPhase = if (!doBuild && doInstallCheck) then ''
+    runHook preInstallCheck
     mkdir -p src/nix-channel
     make installcheck -j$NIX_BUILD_CORES -l$NIX_BUILD_CORES
   '' else null;
 
   # Needed for tests if we are not doing a build, but testing existing
   # built Nix.
-  preInstallCheck = lib.optionalString (! doBuild) ''
-    mkdir -p src/nix-channel
-  '';
+  preInstallCheck =
+    lib.optionalString (! doBuild) ''
+      mkdir -p src/nix-channel
+    ''
+    # See https://github.com/NixOS/nix/issues/2523
+    # Occurs often in tests since https://github.com/NixOS/nix/pull/9900
+    + lib.optionalString stdenv.hostPlatform.isDarwin ''
+      export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES
+    '';
 
   separateDebugInfo = !stdenv.hostPlatform.isStatic;
 

perl/lib/Nix/Store.xs

@@ -259,7 +259,7 @@ hashPath(char * algo, int base32, char * path)
             auto [accessor, canonPath] = PosixSourceAccessor::createAtRoot(path);
             Hash h = hashPath(
                 accessor, canonPath,
-                FileIngestionMethod::Recursive, parseHashAlgo(algo)).first;
+                FileIngestionMethod::Recursive, parseHashAlgo(algo));
             auto s = h.to_string(base32 ? HashFormat::Nix32 : HashFormat::Base16, false);
             XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
         } catch (Error & e) {

precompiled-headers.h

@@ -42,19 +42,22 @@
 #include <dirent.h>
 #include <errno.h>
 #include <fcntl.h>
-#include <grp.h>
-#include <netdb.h>
-#include <pwd.h>
 #include <signal.h>
-#include <sys/resource.h>
-#include <sys/select.h>
-#include <sys/socket.h>
 #include <sys/stat.h>
 #include <sys/time.h>
 #include <sys/types.h>
-#include <sys/utsname.h>
-#include <sys/wait.h>
-#include <termios.h>
 #include <unistd.h>
 
+#ifndef _WIN32
+# include <grp.h>
+# include <netdb.h>
+# include <pwd.h>
+# include <sys/resource.h>
+# include <sys/select.h>
+# include <sys/socket.h>
+# include <sys/utsname.h>
+# include <sys/wait.h>
+# include <termios.h>
+#endif
+
 #include <nlohmann/json.hpp>

scripts/nix-profile-daemon.fish.in

@@ -28,7 +28,7 @@ else
 end
 
 # Set $NIX_SSL_CERT_FILE so that Nixpkgs applications like curl work.
-if test -n "$NIX_SSH_CERT_FILE"
+if test -n "$NIX_SSL_CERT_FILE"
   : # Allow users to override the NIX_SSL_CERT_FILE
 else if test -e /etc/ssl/certs/ca-certificates.crt # NixOS, Ubuntu, Debian, Gentoo, Arch
   set --export NIX_SSL_CERT_FILE /etc/ssl/certs/ca-certificates.crt
@@ -44,7 +44,7 @@ else if test -e "$NIX_LINK/etc/ca-bundle.crt" # old cacert in Nix profile
   set --export NIX_SSL_CERT_FILE "$NIX_LINK/etc/ca-bundle.crt"
 else
   # Fall back to what is in the nix profiles, favouring whatever is defined last.
-  for i in $NIX_PROFILES
+  for i in (string split ' ' $NIX_PROFILES)
     if test -e "$i/etc/ssl/certs/ca-bundle.crt"
       set --export NIX_SSL_CERT_FILE "$i/etc/ssl/certs/ca-bundle.crt"
     end

scripts/nix-profile-daemon.sh.in

@@ -69,4 +69,4 @@ else
 fi
 
 export PATH="$NIX_LINK/bin:@localstatedir@/nix/profiles/default/bin:$PATH"
-unset NIX_LINK
+unset NIX_LINK NIX_LINK_NEW

src/build-remote/build-remote.cc

@@ -202,7 +202,7 @@ static int main_build_remote(int argc, char * * argv)
                         else
                             drvstr = "<unknown>";
 
-                        auto error = HintFmt(errorText);
+                        auto error = HintFmt::fromFormatString(errorText);
                         error
                             % drvstr
                             % neededSystem

src/libcmd/command.cc

@@ -128,10 +128,10 @@ ref<EvalState> EvalCommand::getEvalState()
         evalState =
             #if HAVE_BOEHMGC
             std::allocate_shared<EvalState>(traceable_allocator<EvalState>(),
-                searchPath, getEvalStore(), getStore())
+                lookupPath, getEvalStore(), getStore())
             #else
             std::make_shared<EvalState>(
-                searchPath, getEvalStore(), getStore())
+                lookupPath, getEvalStore(), getStore())
             #endif
             ;
 
@@ -148,7 +148,7 @@ MixOperateOnOptions::MixOperateOnOptions()
 {
     addFlag({
         .longName = "derivation",
-        .description = "Operate on the [store derivation](../../glossary.md#gloss-store-derivation) rather than its outputs.",
+        .description = "Operate on the [store derivation](@docroot@/glossary.md#gloss-store-derivation) rather than its outputs.",
         .category = installablesCategory,
         .handler = {&operateOn, OperateOn::Derivation},
     });

src/libcmd/common-eval-args.cc

@@ -20,7 +20,7 @@ MixEvalArgs::MixEvalArgs()
         .description = "Pass the value *expr* as the argument *name* to Nix functions.",
         .category = category,
         .labels = {"name", "expr"},
-        .handler = {[&](std::string name, std::string expr) { autoArgs[name] = 'E' + expr; }}
+        .handler = {[&](std::string name, std::string expr) { autoArgs.insert_or_assign(name, AutoArg{AutoArgExpr(expr)}); }}
     });
 
     addFlag({
@@ -28,7 +28,24 @@ MixEvalArgs::MixEvalArgs()
         .description = "Pass the string *string* as the argument *name* to Nix functions.",
         .category = category,
         .labels = {"name", "string"},
-        .handler = {[&](std::string name, std::string s) { autoArgs[name] = 'S' + s; }},
+        .handler = {[&](std::string name, std::string s) { autoArgs.insert_or_assign(name, AutoArg{AutoArgString(s)}); }},
+    });
+
+    addFlag({
+        .longName = "arg-from-file",
+        .description = "Pass the contents of file *path* as the argument *name* to Nix functions.",
+        .category = category,
+        .labels = {"name", "path"},
+        .handler = {[&](std::string name, std::string path) { autoArgs.insert_or_assign(name, AutoArg{AutoArgFile(path)}); }},
+        .completer = completePath
+    });
+
+    addFlag({
+        .longName = "arg-from-stdin",
+        .description = "Pass the contents of stdin as the argument *name* to Nix functions.",
+        .category = category,
+        .labels = {"name"},
+        .handler = {[&](std::string name) { autoArgs.insert_or_assign(name, AutoArg{AutoArgStdin{}}); }},
     });
 
     addFlag({
@@ -108,7 +125,7 @@ MixEvalArgs::MixEvalArgs()
         .category = category,
         .labels = {"path"},
         .handler = {[&](std::string s) {
-            searchPath.elements.emplace_back(SearchPath::Elem::parse(s));
+            lookupPath.elements.emplace_back(LookupPath::Elem::parse(s));
         }}
     });
 
@@ -154,13 +171,23 @@ MixEvalArgs::MixEvalArgs()
 Bindings * MixEvalArgs::getAutoArgs(EvalState & state)
 {
     auto res = state.buildBindings(autoArgs.size());
-    for (auto & i : autoArgs) {
+    for (auto & [name, arg] : autoArgs) {
         auto v = state.allocValue();
-        if (i.second[0] == 'E')
-            state.mkThunk_(*v, state.parseExprFromString(i.second.substr(1), state.rootPath(".")));
-        else
-            v->mkString(((std::string_view) i.second).substr(1));
-        res.insert(state.symbols.create(i.first), v);
+        std::visit(overloaded {
+            [&](const AutoArgExpr & arg) {
+                state.mkThunk_(*v, state.parseExprFromString(arg.expr, state.rootPath(".")));
+            },
+            [&](const AutoArgString & arg) {
+                v->mkString(arg.s);
+            },
+            [&](const AutoArgFile & arg) {
+                v->mkString(readFile(arg.path.string()));
+            },
+            [&](const AutoArgStdin & arg) {
+                v->mkString(readFile(STDIN_FILENO));
+            }
+        }, arg);
+        res.insert(state.symbols.create(name), v);
     }
     return res.finish();
 }

src/libcmd/common-eval-args.hh

@@ -6,6 +6,8 @@
 #include "common-args.hh"
 #include "search-path.hh"
 
+#include <filesystem>
+
 namespace nix {
 
 class Store;
@@ -21,14 +23,24 @@ struct MixEvalArgs : virtual Args, virtual MixRepair
 
     Bindings * getAutoArgs(EvalState & state);
 
-    SearchPath searchPath;
+    LookupPath lookupPath;
 
     std::optional<std::string> evalStoreUrl;
 
 private:
-    std::map<std::string, std::string> autoArgs;
+    struct AutoArgExpr { std::string expr; };
+    struct AutoArgString { std::string s; };
+    struct AutoArgFile { std::filesystem::path path; };
+    struct AutoArgStdin { };
+
+    using AutoArg = std::variant<AutoArgExpr, AutoArgString, AutoArgFile, AutoArgStdin>;
+
+    std::map<std::string, AutoArg> autoArgs;
 };
 
+/**
+ * @param baseDir Optional [base directory](https://nixos.org/manual/nix/unstable/glossary#gloss-base-directory)
+ */
 SourcePath lookupFileArg(EvalState & state, std::string_view s, const Path * baseDir = nullptr);
 
 }

src/libcmd/installable-flake.cc

@@ -49,7 +49,7 @@ Value * InstallableFlake::getFlakeOutputs(EvalState & state, const flake::Locked
 
     callFlake(state, lockedFlake, *vFlake);
 
-    auto aOutputs = vFlake->attrs->get(state.symbols.create("outputs"));
+    auto aOutputs = vFlake->attrs()->get(state.symbols.create("outputs"));
     assert(aOutputs);
 
     state.forceValue(*aOutputs->value, aOutputs->value->determinePos(noPos));

src/libcmd/installables.cc

@@ -21,6 +21,7 @@
 #include "url.hh"
 #include "registry.hh"
 #include "build-result.hh"
+#include "fs-input-accessor.hh"
 
 #include <regex>
 #include <queue>
@@ -146,7 +147,7 @@ MixFlakeOptions::MixFlakeOptions()
         .category = category,
         .labels = {"flake-lock-path"},
         .handler = {[&](std::string lockFilePath) {
-            lockFlags.referenceLockFilePath = lockFilePath;
+            lockFlags.referenceLockFilePath = getUnfilteredRootPath(CanonPath(absPath(lockFilePath)));
         }},
         .completer = completePath
     });
@@ -288,7 +289,7 @@ void SourceExprCommand::completeInstallable(AddCompletions & completions, std::s
             state->autoCallFunction(*autoArgs, v1, v2);
 
             if (v2.type() == nAttrs) {
-                for (auto & i : *v2.attrs) {
+                for (auto & i : *v2.attrs()) {
                     std::string name = state->symbols[i.name];
                     if (name.find(searchWord) == 0) {
                         if (prefix_ == "")
@@ -442,10 +443,10 @@ ref<eval_cache::EvalCache> openEvalCache(
     EvalState & state,
     std::shared_ptr<flake::LockedFlake> lockedFlake)
 {
-    auto fingerprint = lockedFlake->getFingerprint();
+    auto fingerprint = lockedFlake->getFingerprint(state.store);
     return make_ref<nix::eval_cache::EvalCache>(
         evalSettings.useEvalCache && evalSettings.pureEval
-            ? std::optional { std::cref(fingerprint) }
+            ? fingerprint
             : std::nullopt,
         state,
         [&state, lockedFlake]()
@@ -460,7 +461,7 @@ ref<eval_cache::EvalCache> openEvalCache(
 
             state.forceAttrs(*vFlake, noPos, "while parsing cached flake data");
 
-            auto aOutputs = vFlake->attrs->get(state.symbols.create("outputs"));
+            auto aOutputs = vFlake->attrs()->get(state.symbols.create("outputs"));
             assert(aOutputs);
 
             return aOutputs->value;

src/libcmd/local.mk

@@ -6,7 +6,7 @@ libcmd_DIR := $(d)
 
 libcmd_SOURCES := $(wildcard $(d)/*.cc)
 
-libcmd_CXXFLAGS += -I src/libutil -I src/libstore -I src/libexpr -I src/libmain -I src/libfetchers
+libcmd_CXXFLAGS += $(INCLUDE_libutil) $(INCLUDE_libstore) $(INCLUDE_libfetchers) $(INCLUDE_libexpr) $(INCLUDE_libmain)
 
 libcmd_LDFLAGS = $(EDITLINE_LIBS) $(LOWDOWN_LIBS) $(THREAD_LDFLAGS)
 

src/libcmd/markdown.cc

@@ -3,9 +3,9 @@
 #include "finally.hh"
 #include "terminal.hh"
 
-#include <sys/queue.h>
 #if HAVE_LOWDOWN
-#include <lowdown.h>
+# include <sys/queue.h>
+# include <lowdown.h>
 #endif
 
 namespace nix {
@@ -50,7 +50,7 @@ std::string renderMarkdownToTerminal(std::string_view markdown)
     if (!rndr_res)
         throw Error("allocation error while rendering Markdown");
 
-    return filterANSIEscapes(std::string(buf->data, buf->size), !shouldANSI());
+    return filterANSIEscapes(std::string(buf->data, buf->size), !isTTY());
 #else
     return std::string(markdown);
 #endif

src/libcmd/repl-interacter.cc

@@ -0,0 +1,190 @@
+#include <cstdio>
+
+#ifdef USE_READLINE
+#include <readline/history.h>
+#include <readline/readline.h>
+#else
+// editline < 1.15.2 don't wrap their API for C++ usage
+// (added in https://github.com/troglobit/editline/commit/91398ceb3427b730995357e9d120539fb9bb7461).
+// This results in linker errors due to to name-mangling of editline C symbols.
+// For compatibility with these versions, we wrap the API here
+// (wrapping multiple times on newer versions is no problem).
+extern "C" {
+#include <editline.h>
+}
+#endif
+
+#include "signals.hh"
+#include "finally.hh"
+#include "repl-interacter.hh"
+#include "file-system.hh"
+#include "libcmd/repl.hh"
+
+namespace nix {
+
+namespace {
+// Used to communicate to NixRepl::getLine whether a signal occurred in ::readline.
+volatile sig_atomic_t g_signal_received = 0;
+
+void sigintHandler(int signo)
+{
+    g_signal_received = signo;
+}
+};
+
+static detail::ReplCompleterMixin * curRepl; // ugly
+
+static char * completionCallback(char * s, int * match)
+{
+    auto possible = curRepl->completePrefix(s);
+    if (possible.size() == 1) {
+        *match = 1;
+        auto * res = strdup(possible.begin()->c_str() + strlen(s));
+        if (!res)
+            throw Error("allocation failure");
+        return res;
+    } else if (possible.size() > 1) {
+        auto checkAllHaveSameAt = [&](size_t pos) {
+            auto & first = *possible.begin();
+            for (auto & p : possible) {
+                if (p.size() <= pos || p[pos] != first[pos])
+                    return false;
+            }
+            return true;
+        };
+        size_t start = strlen(s);
+        size_t len = 0;
+        while (checkAllHaveSameAt(start + len))
+            ++len;
+        if (len > 0) {
+            *match = 1;
+            auto * res = strdup(std::string(*possible.begin(), start, len).c_str());
+            if (!res)
+                throw Error("allocation failure");
+            return res;
+        }
+    }
+
+    *match = 0;
+    return nullptr;
+}
+
+static int listPossibleCallback(char * s, char *** avp)
+{
+    auto possible = curRepl->completePrefix(s);
+
+    if (possible.size() > (INT_MAX / sizeof(char *)))
+        throw Error("too many completions");
+
+    int ac = 0;
+    char ** vp = nullptr;
+
+    auto check = [&](auto * p) {
+        if (!p) {
+            if (vp) {
+                while (--ac >= 0)
+                    free(vp[ac]);
+                free(vp);
+            }
+            throw Error("allocation failure");
+        }
+        return p;
+    };
+
+    vp = check((char **) malloc(possible.size() * sizeof(char *)));
+
+    for (auto & p : possible)
+        vp[ac++] = check(strdup(p.c_str()));
+
+    *avp = vp;
+
+    return ac;
+}
+
+ReadlineLikeInteracter::Guard ReadlineLikeInteracter::init(detail::ReplCompleterMixin * repl)
+{
+    // Allow nix-repl specific settings in .inputrc
+    rl_readline_name = "nix-repl";
+    try {
+        createDirs(dirOf(historyFile));
+    } catch (SystemError & e) {
+        logWarning(e.info());
+    }
+#ifndef USE_READLINE
+    el_hist_size = 1000;
+#endif
+    read_history(historyFile.c_str());
+    auto oldRepl = curRepl;
+    curRepl = repl;
+    Guard restoreRepl([oldRepl] { curRepl = oldRepl; });
+#ifndef USE_READLINE
+    rl_set_complete_func(completionCallback);
+    rl_set_list_possib_func(listPossibleCallback);
+#endif
+    return restoreRepl;
+}
+
+static constexpr const char * promptForType(ReplPromptType promptType)
+{
+    switch (promptType) {
+    case ReplPromptType::ReplPrompt:
+        return "nix-repl> ";
+    case ReplPromptType::ContinuationPrompt:
+        return "          ";
+    }
+    assert(false);
+}
+
+bool ReadlineLikeInteracter::getLine(std::string & input, ReplPromptType promptType)
+{
+#ifndef _WIN32 // TODO use more signals.hh for this
+    struct sigaction act, old;
+    sigset_t savedSignalMask, set;
+
+    auto setupSignals = [&]() {
+        act.sa_handler = sigintHandler;
+        sigfillset(&act.sa_mask);
+        act.sa_flags = 0;
+        if (sigaction(SIGINT, &act, &old))
+            throw SysError("installing handler for SIGINT");
+
+        sigemptyset(&set);
+        sigaddset(&set, SIGINT);
+        if (sigprocmask(SIG_UNBLOCK, &set, &savedSignalMask))
+            throw SysError("unblocking SIGINT");
+    };
+    auto restoreSignals = [&]() {
+        if (sigprocmask(SIG_SETMASK, &savedSignalMask, nullptr))
+            throw SysError("restoring signals");
+
+        if (sigaction(SIGINT, &old, 0))
+            throw SysError("restoring handler for SIGINT");
+    };
+
+    setupSignals();
+#endif
+    char * s = readline(promptForType(promptType));
+    Finally doFree([&]() { free(s); });
+#ifndef _WIN32 // TODO use more signals.hh for this
+    restoreSignals();
+#endif
+
+    if (g_signal_received) {
+        g_signal_received = 0;
+        input.clear();
+        return true;
+    }
+
+    if (!s)
+        return false;
+    input += s;
+    input += '\n';
+    return true;
+}
+
+ReadlineLikeInteracter::~ReadlineLikeInteracter()
+{
+    write_history(historyFile.c_str());
+}
+
+};

src/libcmd/repl-interacter.hh

@@ -0,0 +1,48 @@
+#pragma once
+/// @file
+
+#include "finally.hh"
+#include "types.hh"
+#include <functional>
+#include <string>
+
+namespace nix {
+
+namespace detail {
+/** Provides the completion hooks for the repl, without exposing its complete
+ * internals. */
+struct ReplCompleterMixin {
+    virtual StringSet completePrefix(const std::string & prefix) = 0;
+};
+};
+
+enum class ReplPromptType {
+    ReplPrompt,
+    ContinuationPrompt,
+};
+
+class ReplInteracter
+{
+public:
+    using Guard = Finally<std::function<void()>>;
+
+    virtual Guard init(detail::ReplCompleterMixin * repl) = 0;
+    /** Returns a boolean of whether the interacter got EOF */
+    virtual bool getLine(std::string & input, ReplPromptType promptType) = 0;
+    virtual ~ReplInteracter(){};
+};
+
+class ReadlineLikeInteracter : public virtual ReplInteracter
+{
+    std::string historyFile;
+public:
+    ReadlineLikeInteracter(std::string historyFile)
+        : historyFile(historyFile)
+    {
+    }
+    virtual Guard init(detail::ReplCompleterMixin * repl) override;
+    virtual bool getLine(std::string & input, ReplPromptType promptType) override;
+    virtual ~ReadlineLikeInteracter() override;
+};
+
+};

src/libcmd/repl.cc

@@ -3,32 +3,17 @@
 #include <cstring>
 #include <climits>
 
-#include <setjmp.h>
-
-#ifdef USE_READLINE
-#include <readline/history.h>
-#include <readline/readline.h>
-#else
-// editline < 1.15.2 don't wrap their API for C++ usage
-// (added in https://github.com/troglobit/editline/commit/91398ceb3427b730995357e9d120539fb9bb7461).
-// This results in linker errors due to to name-mangling of editline C symbols.
-// For compatibility with these versions, we wrap the API here
-// (wrapping multiple times on newer versions is no problem).
-extern "C" {
-#include <editline.h>
-}
-#endif
-
+#include "libcmd/repl-interacter.hh"
 #include "repl.hh"
 
 #include "ansicolor.hh"
-#include "signals.hh"
 #include "shared.hh"
 #include "eval.hh"
 #include "eval-cache.hh"
 #include "eval-inline.hh"
 #include "eval-settings.hh"
 #include "attr-path.hh"
+#include "signals.hh"
 #include "store-api.hh"
 #include "log-store.hh"
 #include "common-eval-args.hh"
@@ -38,7 +23,6 @@ extern "C" {
 #include "flake/flake.hh"
 #include "flake/lockfile.hh"
 #include "users.hh"
-#include "terminal.hh"
 #include "editor-for.hh"
 #include "finally.hh"
 #include "markdown.hh"
@@ -75,6 +59,7 @@ enum class ProcessLineResult {
 
 struct NixRepl
     : AbstractNixRepl
+    , detail::ReplCompleterMixin
     #if HAVE_BOEHMGC
     , gc
     #endif
@@ -90,17 +75,16 @@ struct NixRepl
     int displ;
     StringSet varNames;
 
-    const Path historyFile;
+    std::unique_ptr<ReplInteracter> interacter;
 
-    NixRepl(const SearchPath & searchPath, nix::ref<Store> store,ref<EvalState> state,
+    NixRepl(const LookupPath & lookupPath, nix::ref<Store> store,ref<EvalState> state,
             std::function<AnnotatedValues()> getValues);
-    virtual ~NixRepl();
+    virtual ~NixRepl() = default;
 
     ReplExitStatus mainLoop() override;
     void initEnv() override;
 
-    StringSet completePrefix(const std::string & prefix);
-    bool getLine(std::string & input, const std::string & prompt);
+    virtual StringSet completePrefix(const std::string & prefix) override;
     StorePath getDerivationPath(Value & v);
     ProcessLineResult processLine(std::string line);
 
@@ -123,7 +107,8 @@ struct NixRepl
             .force = true,
             .derivationPaths = true,
             .maxDepth = maxDepth,
-            .prettyIndent = 2
+            .prettyIndent = 2,
+            .errors = ErrorPrintBehavior::ThrowTopLevel,
         });
     }
 };
@@ -137,22 +122,16 @@ std::string removeWhitespace(std::string s)
 }
 
 
-NixRepl::NixRepl(const SearchPath & searchPath, nix::ref<Store> store, ref<EvalState> state,
+NixRepl::NixRepl(const LookupPath & lookupPath, nix::ref<Store> store, ref<EvalState> state,
             std::function<NixRepl::AnnotatedValues()> getValues)
     : AbstractNixRepl(state)
     , debugTraceIndex(0)
     , getValues(getValues)
     , staticEnv(new StaticEnv(nullptr, state->staticBaseEnv.get()))
-    , historyFile(getDataDir() + "/nix/repl-history")
+    , interacter(make_unique<ReadlineLikeInteracter>(getDataDir() + "/nix/repl-history"))
 {
 }
 
-
-NixRepl::~NixRepl()
-{
-    write_history(historyFile.c_str());
-}
-
 void runNix(Path program, const Strings & args,
     const std::optional<std::string> & input = {})
 {
@@ -169,79 +148,6 @@ void runNix(Path program, const Strings & args,
     return;
 }
 
-static NixRepl * curRepl; // ugly
-
-static char * completionCallback(char * s, int *match) {
-  auto possible = curRepl->completePrefix(s);
-  if (possible.size() == 1) {
-    *match = 1;
-    auto *res = strdup(possible.begin()->c_str() + strlen(s));
-    if (!res) throw Error("allocation failure");
-    return res;
-  } else if (possible.size() > 1) {
-    auto checkAllHaveSameAt = [&](size_t pos) {
-      auto &first = *possible.begin();
-      for (auto &p : possible) {
-        if (p.size() <= pos || p[pos] != first[pos])
-          return false;
-      }
-      return true;
-    };
-    size_t start = strlen(s);
-    size_t len = 0;
-    while (checkAllHaveSameAt(start + len)) ++len;
-    if (len > 0) {
-      *match = 1;
-      auto *res = strdup(std::string(*possible.begin(), start, len).c_str());
-      if (!res) throw Error("allocation failure");
-      return res;
-    }
-  }
-
-  *match = 0;
-  return nullptr;
-}
-
-static int listPossibleCallback(char *s, char ***avp) {
-  auto possible = curRepl->completePrefix(s);
-
-  if (possible.size() > (INT_MAX / sizeof(char*)))
-    throw Error("too many completions");
-
-  int ac = 0;
-  char **vp = nullptr;
-
-  auto check = [&](auto *p) {
-    if (!p) {
-      if (vp) {
-        while (--ac >= 0)
-          free(vp[ac]);
-        free(vp);
-      }
-      throw Error("allocation failure");
-    }
-    return p;
-  };
-
-  vp = check((char **)malloc(possible.size() * sizeof(char*)));
-
-  for (auto & p : possible)
-    vp[ac++] = check(strdup(p.c_str()));
-
-  *avp = vp;
-
-  return ac;
-}
-
-namespace {
-    // Used to communicate to NixRepl::getLine whether a signal occurred in ::readline.
-    volatile sig_atomic_t g_signal_received = 0;
-
-    void sigintHandler(int signo) {
-        g_signal_received = signo;
-    }
-}
-
 static std::ostream & showDebugTrace(std::ostream & out, const PosTable & positions, const DebugTrace & dt)
 {
     if (dt.isError)
@@ -281,24 +187,7 @@ ReplExitStatus NixRepl::mainLoop()
 
     loadFiles();
 
-    // Allow nix-repl specific settings in .inputrc
-    rl_readline_name = "nix-repl";
-    try {
-        createDirs(dirOf(historyFile));
-    } catch (SystemError & e) {
-        logWarning(e.info());
-    }
-#ifndef USE_READLINE
-    el_hist_size = 1000;
-#endif
-    read_history(historyFile.c_str());
-    auto oldRepl = curRepl;
-    curRepl = this;
-    Finally restoreRepl([&] { curRepl = oldRepl; });
-#ifndef USE_READLINE
-    rl_set_complete_func(completionCallback);
-    rl_set_list_possib_func(listPossibleCallback);
-#endif
+    auto _guard = interacter->init(static_cast<detail::ReplCompleterMixin *>(this));
 
     std::string input;
 
@@ -307,7 +196,7 @@ ReplExitStatus NixRepl::mainLoop()
         logger->pause();
         // When continuing input from previous lines, don't print a prompt, just align to the same
         // number of chars as the prompt.
-        if (!getLine(input, input.empty() ? "nix-repl> " : "          ")) {
+        if (!interacter->getLine(input, input.empty() ? ReplPromptType::ReplPrompt : ReplPromptType::ContinuationPrompt)) {
             // Ctrl-D should exit the debugger.
             state->debugStop = false;
             logger->cout("");
@@ -336,13 +225,7 @@ ReplExitStatus NixRepl::mainLoop()
               printMsg(lvlError, e.msg());
             }
         } catch (EvalError & e) {
-            // in debugger mode, an EvalError should trigger another repl session.
-            // when that session returns the exception will land here.  No need to show it again;
-            // show the error for this repl session instead.
-            if (state->debugRepl && !state->debugTraces.empty())
-                showDebugTrace(std::cout, state->positions, state->debugTraces.front());
-            else
-                printMsg(lvlError, e.msg());
+            printMsg(lvlError, e.msg());
         } catch (Error & e) {
             printMsg(lvlError, e.msg());
         } catch (Interrupted & e) {
@@ -356,52 +239,6 @@ ReplExitStatus NixRepl::mainLoop()
     }
 }
 
-
-bool NixRepl::getLine(std::string & input, const std::string & prompt)
-{
-    struct sigaction act, old;
-    sigset_t savedSignalMask, set;
-
-    auto setupSignals = [&]() {
-        act.sa_handler = sigintHandler;
-        sigfillset(&act.sa_mask);
-        act.sa_flags = 0;
-        if (sigaction(SIGINT, &act, &old))
-            throw SysError("installing handler for SIGINT");
-
-        sigemptyset(&set);
-        sigaddset(&set, SIGINT);
-        if (sigprocmask(SIG_UNBLOCK, &set, &savedSignalMask))
-            throw SysError("unblocking SIGINT");
-    };
-    auto restoreSignals = [&]() {
-        if (sigprocmask(SIG_SETMASK, &savedSignalMask, nullptr))
-            throw SysError("restoring signals");
-
-        if (sigaction(SIGINT, &old, 0))
-            throw SysError("restoring handler for SIGINT");
-    };
-
-    setupSignals();
-    Finally resetTerminal([&]() { rl_deprep_terminal(); });
-    char * s = readline(prompt.c_str());
-    Finally doFree([&]() { free(s); });
-    restoreSignals();
-
-    if (g_signal_received) {
-        g_signal_received = 0;
-        input.clear();
-        return true;
-    }
-
-    if (!s)
-      return false;
-    input += s;
-    input += '\n';
-    return true;
-}
-
-
 StringSet NixRepl::completePrefix(const std::string & prefix)
 {
     StringSet completions;
@@ -453,7 +290,7 @@ StringSet NixRepl::completePrefix(const std::string & prefix)
             e->eval(*state, *env, v);
             state->forceAttrs(v, noPos, "while evaluating an attrset for the purpose of completion (this error should not be displayed; file an issue?)");
 
-            for (auto & i : *v.attrs) {
+            for (auto & i : *v.attrs()) {
                 std::string_view name = state->symbols[i.name];
                 if (name.substr(0, cur2.size()) != cur2) continue;
                 completions.insert(concatStrings(prev, expr, ".", name));
@@ -520,7 +357,7 @@ ProcessLineResult NixRepl::processLine(std::string line)
     if (line.empty())
         return ProcessLineResult::PromptAgain;
 
-    _isInterrupted = false;
+    setInterrupted(false);
 
     std::string command, arg;
 
@@ -549,6 +386,7 @@ ProcessLineResult NixRepl::processLine(std::string line)
              << "  :l, :load <path>             Load Nix expression and add it to scope\n"
              << "  :lf, :load-flake <ref>       Load Nix flake and add it to scope\n"
              << "  :p, :print <expr>            Evaluate and print expression recursively\n"
+             << "                               Strings are printed directly, without escaping.\n"
              << "  :q, :quit                    Exit nix-repl\n"
              << "  :r, :reload                  Reload all files\n"
              << "  :sh <expr>                   Build dependencies of derivation, then start\n"
@@ -652,7 +490,7 @@ ProcessLineResult NixRepl::processLine(std::string line)
                 auto path = state->coerceToPath(noPos, v, context, "while evaluating the filename to edit");
                 return {path, 0};
             } else if (v.isLambda()) {
-                auto pos = state->positions[v.lambda.fun->pos];
+                auto pos = state->positions[v.payload.lambda.fun->pos];
                 if (auto path = std::get_if<SourcePath>(&pos.origin))
                     return {*path, pos.line};
                 else
@@ -670,7 +508,7 @@ ProcessLineResult NixRepl::processLine(std::string line)
 
         // runProgram redirects stdout to a StringSink,
         // using runProgram2 to allow editors to display their UI
-        runProgram2(RunOptions { .program = editor, .searchPath = true, .args = args });
+        runProgram2(RunOptions { .program = editor, .lookupPath = true, .args = args });
 
         // Reload right after exiting the editor
         state->resetFileCache();
@@ -756,7 +594,11 @@ ProcessLineResult NixRepl::processLine(std::string line)
     else if (command == ":p" || command == ":print") {
         Value v;
         evalString(arg, v);
-        printValue(std::cout, v);
+        if (v.type() == nString) {
+            std::cout << v.string_view();
+        } else {
+            printValue(std::cout, v);
+        }
         std::cout << std::endl;
     }
 
@@ -900,17 +742,17 @@ void NixRepl::loadFiles()
 void NixRepl::addAttrsToScope(Value & attrs)
 {
     state->forceAttrs(attrs, [&]() { return attrs.determinePos(noPos); }, "while evaluating an attribute set to be merged in the global scope");
-    if (displ + attrs.attrs->size() >= envSize)
+    if (displ + attrs.attrs()->size() >= envSize)
         throw Error("environment full; cannot add more variables");
 
-    for (auto & i : *attrs.attrs) {
+    for (auto & i : *attrs.attrs()) {
         staticEnv->vars.emplace_back(i.name, displ);
         env->values[displ++] = i.value;
         varNames.emplace(state->symbols[i.name]);
     }
     staticEnv->sort();
     staticEnv->deduplicate();
-    notice("Added %1% variables.", attrs.attrs->size());
+    notice("Added %1% variables.", attrs.attrs()->size());
 }
 
 
@@ -942,11 +784,11 @@ void NixRepl::evalString(std::string s, Value & v)
 
 
 std::unique_ptr<AbstractNixRepl> AbstractNixRepl::create(
-   const SearchPath & searchPath, nix::ref<Store> store, ref<EvalState> state,
+   const LookupPath & lookupPath, nix::ref<Store> store, ref<EvalState> state,
    std::function<AnnotatedValues()> getValues)
 {
     return std::make_unique<NixRepl>(
-        searchPath,
+        lookupPath,
         openStore(),
         state,
         getValues
@@ -962,9 +804,9 @@ ReplExitStatus AbstractNixRepl::runSimple(
         NixRepl::AnnotatedValues values;
         return values;
     };
-    SearchPath searchPath = {};
+    LookupPath lookupPath = {};
     auto repl = std::make_unique<NixRepl>(
-            searchPath,
+            lookupPath,
             openStore(),
             evalState,
             getValues

src/libcmd/repl.hh

@@ -3,11 +3,6 @@
 
 #include "eval.hh"
 
-#if HAVE_BOEHMGC
-#define GC_INCLUDE_NEW
-#include <gc/gc_cpp.h>
-#endif
-
 namespace nix {
 
 struct AbstractNixRepl
@@ -25,7 +20,7 @@ struct AbstractNixRepl
     typedef std::vector<std::pair<Value*,std::string>> AnnotatedValues;
 
     static std::unique_ptr<AbstractNixRepl> create(
-        const SearchPath & searchPath, nix::ref<Store> store, ref<EvalState> state,
+        const LookupPath & lookupPath, nix::ref<Store> store, ref<EvalState> state,
         std::function<AnnotatedValues()> getValues);
 
     static ReplExitStatus runSimple(

src/libexpr-c/local.mk

@@ -0,0 +1,25 @@
+libraries += libexprc
+
+libexprc_NAME = libnixexprc
+
+libexprc_DIR := $(d)
+
+libexprc_SOURCES := \
+  $(wildcard $(d)/*.cc) \
+
+# Not just for this library itself, but also for downstream libraries using this library
+
+INCLUDE_libexprc := -I $(d)
+libexprc_CXXFLAGS += $(INCLUDE_libutil) $(INCLUDE_libutilc) \
+                     $(INCLUDE_libfetchers) \
+                     $(INCLUDE_libstore) $(INCLUDE_libstorec) \
+                     $(INCLUDE_libexpr) $(INCLUDE_libexprc)
+
+libexprc_LIBS = libutil libutilc libstore libstorec libexpr
+
+libexprc_LDFLAGS += $(THREAD_LDFLAGS)
+
+$(eval $(call install-file-in, $(d)/nix-expr-c.pc, $(libdir)/pkgconfig, 0644))
+
+libexprc_FORCE_INSTALL := 1
+

src/libexpr-c/nix-expr-c.pc.in

@@ -0,0 +1,10 @@
+prefix=@prefix@
+libdir=@libdir@
+includedir=@includedir@
+
+Name: Nix
+Description: Nix Language Evaluator - C API
+Version: @PACKAGE_VERSION@
+Requires: nix-store-c
+Libs: -L${libdir} -lnixexprc
+Cflags: -I${includedir}/nix

src/libexpr-c/nix_api_expr.cc

@@ -0,0 +1,178 @@
+#include <cstring>
+#include <iostream>
+#include <stdexcept>
+#include <string>
+
+#include "config.hh"
+#include "eval.hh"
+#include "globals.hh"
+#include "util.hh"
+
+#include "nix_api_expr.h"
+#include "nix_api_expr_internal.h"
+#include "nix_api_store.h"
+#include "nix_api_store_internal.h"
+#include "nix_api_util.h"
+#include "nix_api_util_internal.h"
+
+#ifdef HAVE_BOEHMGC
+#include <mutex>
+#define GC_INCLUDE_NEW 1
+#include "gc_cpp.h"
+#endif
+
+nix_err nix_libexpr_init(nix_c_context * context)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    {
+        auto ret = nix_libutil_init(context);
+        if (ret != NIX_OK)
+            return ret;
+    }
+    {
+        auto ret = nix_libstore_init(context);
+        if (ret != NIX_OK)
+            return ret;
+    }
+    try {
+        nix::initGC();
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_expr_eval_from_string(
+    nix_c_context * context, EvalState * state, const char * expr, const char * path, Value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        nix::Expr * parsedExpr = state->state.parseExprFromString(expr, state->state.rootPath(nix::CanonPath(path)));
+        state->state.eval(parsedExpr, *(nix::Value *) value);
+        state->state.forceValue(*(nix::Value *) value, nix::noPos);
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_value_call(nix_c_context * context, EvalState * state, Value * fn, Value * arg, Value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        state->state.callFunction(*(nix::Value *) fn, *(nix::Value *) arg, *(nix::Value *) value, nix::noPos);
+        state->state.forceValue(*(nix::Value *) value, nix::noPos);
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_value_force(nix_c_context * context, EvalState * state, Value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        state->state.forceValue(*(nix::Value *) value, nix::noPos);
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_value_force_deep(nix_c_context * context, EvalState * state, Value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        state->state.forceValueDeep(*(nix::Value *) value);
+    }
+    NIXC_CATCH_ERRS
+}
+
+EvalState * nix_state_create(nix_c_context * context, const char ** lookupPath_c, Store * store)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        nix::Strings lookupPath;
+        if (lookupPath_c != nullptr)
+            for (size_t i = 0; lookupPath_c[i] != nullptr; i++)
+                lookupPath.push_back(lookupPath_c[i]);
+
+        return new EvalState{nix::EvalState(nix::LookupPath::parse(lookupPath), store->ptr)};
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+void nix_state_free(EvalState * state)
+{
+    delete state;
+}
+
+#ifdef HAVE_BOEHMGC
+std::unordered_map<
+    const void *,
+    unsigned int,
+    std::hash<const void *>,
+    std::equal_to<const void *>,
+    traceable_allocator<std::pair<const void * const, unsigned int>>>
+    nix_refcounts;
+
+std::mutex nix_refcount_lock;
+
+nix_err nix_gc_incref(nix_c_context * context, const void * p)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        std::scoped_lock lock(nix_refcount_lock);
+        auto f = nix_refcounts.find(p);
+        if (f != nix_refcounts.end()) {
+            f->second++;
+        } else {
+            nix_refcounts[p] = 1;
+        }
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_gc_decref(nix_c_context * context, const void * p)
+{
+
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        std::scoped_lock lock(nix_refcount_lock);
+        auto f = nix_refcounts.find(p);
+        if (f != nix_refcounts.end()) {
+            if (--f->second == 0)
+                nix_refcounts.erase(f);
+        } else
+            throw std::runtime_error("nix_gc_decref: object was not referenced");
+    }
+    NIXC_CATCH_ERRS
+}
+
+void nix_gc_now()
+{
+    GC_gcollect();
+}
+
+#else
+nix_err nix_gc_incref(nix_c_context * context, const void *)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    return NIX_OK;
+}
+nix_err nix_gc_decref(nix_c_context * context, const void *)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    return NIX_OK;
+}
+void nix_gc_now() {}
+#endif
+
+void nix_gc_register_finalizer(void * obj, void * cd, void (*finalizer)(void * obj, void * cd))
+{
+#ifdef HAVE_BOEHMGC
+    GC_REGISTER_FINALIZER(obj, finalizer, cd, 0, 0);
+#endif
+}

src/libexpr-c/nix_api_expr.h

@@ -0,0 +1,215 @@
+#ifndef NIX_API_EXPR_H
+#define NIX_API_EXPR_H
+/** @defgroup libexpr libexpr
+ * @brief Bindings to the Nix language evaluator
+ *
+ * Example (without error handling):
+ * @code{.c}
+ * int main() {
+ *    nix_libexpr_init(NULL);
+ *
+ *    Store* store = nix_store_open(NULL, "dummy", NULL);
+ *    EvalState* state = nix_state_create(NULL, NULL, store); // empty nix path
+ *    Value *value = nix_alloc_value(NULL, state);
+ *
+ *    nix_expr_eval_from_string(NULL, state, "builtins.nixVersion", ".", value);
+ *    nix_value_force(NULL, state, value);
+ *    printf("nix version: %s\n", nix_get_string(NULL, value));
+ *
+ *    nix_gc_decref(NULL, value);
+ *    nix_state_free(state);
+ *    nix_store_free(store);
+ *    return 0;
+ *    }
+ * @endcode
+ * @{
+ */
+/** @file
+ * @brief Main entry for the libexpr C bindings
+ */
+
+#include "nix_api_store.h"
+#include "nix_api_util.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+// cffi start
+
+// Type definitions
+/**
+ * @brief Represents a state of the Nix language evaluator.
+ *
+ * Multiple states can be created for multi-threaded
+ * operation.
+ * @struct EvalState
+ * @see nix_state_create
+ */
+typedef struct EvalState EvalState; // nix::EvalState
+/**
+ * @brief Represents a value in the Nix language.
+ *
+ * Owned by the garbage collector.
+ * @struct Value
+ * @see value_manip
+ */
+typedef void Value; // nix::Value
+
+// Function prototypes
+/**
+ * @brief Initialize the Nix language evaluator.
+ *
+ * This function must be called at least once,
+ * at some point before constructing a EvalState for the first time.
+ * This function can be called multiple times, and is idempotent.
+ *
+ * @param[out] context Optional, stores error information
+ * @return NIX_OK if the initialization was successful, an error code otherwise.
+ */
+nix_err nix_libexpr_init(nix_c_context * context);
+
+/**
+ * @brief Parses and evaluates a Nix expression from a string.
+ *
+ * @param[out] context Optional, stores error information
+ * @param[in] state The state of the evaluation.
+ * @param[in] expr The Nix expression to parse.
+ * @param[in] path The file path to associate with the expression.
+ * This is required for expressions that contain relative paths (such as `./.`) that are resolved relative to the given
+ * directory.
+ * @param[out] value The result of the evaluation. You must allocate this
+ * yourself.
+ * @return NIX_OK if the evaluation was successful, an error code otherwise.
+ */
+nix_err nix_expr_eval_from_string(
+    nix_c_context * context, EvalState * state, const char * expr, const char * path, Value * value);
+
+/**
+ * @brief Calls a Nix function with an argument.
+ *
+ * @param[out] context Optional, stores error information
+ * @param[in] state The state of the evaluation.
+ * @param[in] fn The Nix function to call.
+ * @param[in] arg The argument to pass to the function.
+ * @param[out] value The result of the function call.
+ * @return NIX_OK if the function call was successful, an error code otherwise.
+ * @see nix_init_apply() for a similar function that does not performs the call immediately, but stores it as a thunk.
+ *      Note the different argument order.
+ */
+nix_err nix_value_call(nix_c_context * context, EvalState * state, Value * fn, Value * arg, Value * value);
+
+/**
+ * @brief Forces the evaluation of a Nix value.
+ *
+ * The Nix interpreter is lazy, and not-yet-evaluated Values can be
+ * of type NIX_TYPE_THUNK instead of their actual value.
+ *
+ * This function converts these Values into their final type.
+ *
+ * @note You don't need this function for basic API usage, since all functions
+ * that return a value call it for you. The only place you will see a
+ * NIX_TYPE_THUNK is in the arguments that are passed to a PrimOp function
+ * you supplied to nix_alloc_primop.
+ *
+ * @param[out] context Optional, stores error information
+ * @param[in] state The state of the evaluation.
+ * @param[in,out] value The Nix value to force.
+ * @post value is not of type NIX_TYPE_THUNK
+ * @return NIX_OK if the force operation was successful, an error code
+ * otherwise.
+ */
+nix_err nix_value_force(nix_c_context * context, EvalState * state, Value * value);
+
+/**
+ * @brief Forces the deep evaluation of a Nix value.
+ *
+ * Recursively calls nix_value_force
+ *
+ * @see nix_value_force
+ * @warning Calling this function on a recursive data structure will cause a
+ * stack overflow.
+ * @param[out] context Optional, stores error information
+ * @param[in] state The state of the evaluation.
+ * @param[in,out] value The Nix value to force.
+ * @return NIX_OK if the deep force operation was successful, an error code
+ * otherwise.
+ */
+nix_err nix_value_force_deep(nix_c_context * context, EvalState * state, Value * value);
+
+/**
+ * @brief Create a new Nix language evaluator state.
+ *
+ * @param[out] context Optional, stores error information
+ * @param[in] lookupPath Array of strings corresponding to entries in NIX_PATH.
+ * @param[in] store The Nix store to use.
+ * @return A new Nix state or NULL on failure.
+ */
+EvalState * nix_state_create(nix_c_context * context, const char ** lookupPath, Store * store);
+
+/**
+ * @brief Frees a Nix state.
+ *
+ * Does not fail.
+ *
+ * @param[in] state The state to free.
+ */
+void nix_state_free(EvalState * state);
+
+/** @addtogroup GC
+ * @brief Reference counting and garbage collector operations
+ *
+ * The Nix language evaluator uses a garbage collector. To ease C interop, we implement
+ * a reference counting scheme, where objects will be deallocated
+ * when there are no references from the Nix side, and the reference count kept
+ * by the C API reaches `0`.
+ *
+ * Functions returning a garbage-collected object will automatically increase
+ * the refcount for you. You should make sure to call `nix_gc_decref` when
+ * you're done with a value returned by the evaluator.
+ * @{
+ */
+/**
+ * @brief Increment the garbage collector reference counter for the given object.
+ *
+ * The Nix language evaluator C API keeps track of alive objects by reference counting.
+ * When you're done with a refcounted pointer, call nix_gc_decref().
+ *
+ * @param[out] context Optional, stores error information
+ * @param[in] object The object to keep alive
+ */
+nix_err nix_gc_incref(nix_c_context * context, const void * object);
+/**
+ * @brief Decrement the garbage collector reference counter for the given object
+ *
+ * @param[out] context Optional, stores error information
+ * @param[in] object The object to stop referencing
+ */
+nix_err nix_gc_decref(nix_c_context * context, const void * object);
+
+/**
+ * @brief Trigger the garbage collector manually
+ *
+ * You should not need to do this, but it can be useful for debugging.
+ */
+void nix_gc_now();
+
+/**
+ * @brief Register a callback that gets called when the object is garbage
+ * collected.
+ * @note Objects can only have a single finalizer. This function overwrites existing values
+ * silently.
+ * @param[in] obj the object to watch
+ * @param[in] cd the data to pass to the finalizer
+ * @param[in] finalizer the callback function, called with obj and cd
+ */
+void nix_gc_register_finalizer(void * obj, void * cd, void (*finalizer)(void * obj, void * cd));
+
+/** @} */
+// cffi end
+#ifdef __cplusplus
+}
+#endif
+
+/** @} */
+
+#endif // NIX_API_EXPR_H

src/libexpr-c/nix_api_expr_internal.h

@@ -0,0 +1,44 @@
+#ifndef NIX_API_EXPR_INTERNAL_H
+#define NIX_API_EXPR_INTERNAL_H
+
+#include "eval.hh"
+#include "attr-set.hh"
+#include "nix_api_value.h"
+
+struct EvalState
+{
+    nix::EvalState state;
+};
+
+struct BindingsBuilder
+{
+    nix::BindingsBuilder builder;
+};
+
+struct ListBuilder
+{
+    nix::ListBuilder builder;
+};
+
+struct nix_string_return
+{
+    std::string str;
+};
+
+struct nix_printer
+{
+    std::ostream & s;
+};
+
+struct nix_string_context
+{
+    nix::NixStringContext & ctx;
+};
+
+struct nix_realised_string
+{
+    std::string str;
+    std::vector<StorePath> storePaths;
+};
+
+#endif // NIX_API_EXPR_INTERNAL_H

src/libexpr-c/nix_api_external.cc

@@ -0,0 +1,198 @@
+#include "attr-set.hh"
+#include "config.hh"
+#include "eval.hh"
+#include "globals.hh"
+#include "value.hh"
+
+#include "nix_api_expr.h"
+#include "nix_api_expr_internal.h"
+#include "nix_api_external.h"
+#include "nix_api_util.h"
+#include "nix_api_util_internal.h"
+#include "nix_api_value.h"
+#include "value/context.hh"
+
+#include <nlohmann/json.hpp>
+
+#ifdef HAVE_BOEHMGC
+# include "gc/gc.h"
+# define GC_INCLUDE_NEW 1
+# include "gc_cpp.h"
+#endif
+
+void nix_set_string_return(nix_string_return * str, const char * c)
+{
+    str->str = c;
+}
+
+nix_err nix_external_print(nix_c_context * context, nix_printer * printer, const char * c)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        printer->s << c;
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_external_add_string_context(nix_c_context * context, nix_string_context * ctx, const char * c)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto r = nix::NixStringContextElem::parse(c);
+        ctx->ctx.insert(r);
+    }
+    NIXC_CATCH_ERRS
+}
+
+class NixCExternalValue : public nix::ExternalValueBase
+{
+    NixCExternalValueDesc & desc;
+    void * v;
+
+public:
+    NixCExternalValue(NixCExternalValueDesc & desc, void * v)
+        : desc(desc)
+        , v(v){};
+    void * get_ptr()
+    {
+        return v;
+    }
+    /**
+     * Print out the value
+     */
+    virtual std::ostream & print(std::ostream & str) const override
+    {
+        nix_printer p{str};
+        desc.print(v, &p);
+        return str;
+    }
+
+    /**
+     * Return a simple string describing the type
+     */
+    virtual std::string showType() const override
+    {
+        nix_string_return res;
+        desc.showType(v, &res);
+        return std::move(res.str);
+    }
+
+    /**
+     * Return a string to be used in builtins.typeOf
+     */
+    virtual std::string typeOf() const override
+    {
+        nix_string_return res;
+        desc.typeOf(v, &res);
+        return std::move(res.str);
+    }
+
+    /**
+     * Coerce the value to a string.
+     */
+    virtual std::string coerceToString(
+        nix::EvalState & state,
+        const nix::PosIdx & pos,
+        nix::NixStringContext & context,
+        bool copyMore,
+        bool copyToStore) const override
+    {
+        if (!desc.coerceToString) {
+            return nix::ExternalValueBase::coerceToString(state, pos, context, copyMore, copyToStore);
+        }
+        nix_string_context ctx{context};
+        nix_string_return res{""};
+        // todo: pos, errors
+        desc.coerceToString(v, &ctx, copyMore, copyToStore, &res);
+        if (res.str.empty()) {
+            return nix::ExternalValueBase::coerceToString(state, pos, context, copyMore, copyToStore);
+        }
+        return std::move(res.str);
+    }
+
+    /**
+     * Compare to another value of the same type.
+     */
+    virtual bool operator==(const ExternalValueBase & b) const override
+    {
+        if (!desc.equal) {
+            return false;
+        }
+        auto r = dynamic_cast<const NixCExternalValue *>(&b);
+        if (!r)
+            return false;
+        return desc.equal(v, r->v);
+    }
+
+    /**
+     * Print the value as JSON.
+     */
+    virtual nlohmann::json printValueAsJSON(
+        nix::EvalState & state, bool strict, nix::NixStringContext & context, bool copyToStore = true) const override
+    {
+        if (!desc.printValueAsJSON) {
+            return nix::ExternalValueBase::printValueAsJSON(state, strict, context, copyToStore);
+        }
+        nix_string_context ctx{context};
+        nix_string_return res{""};
+        desc.printValueAsJSON(v, (EvalState *) &state, strict, &ctx, copyToStore, &res);
+        if (res.str.empty()) {
+            return nix::ExternalValueBase::printValueAsJSON(state, strict, context, copyToStore);
+        }
+        return nlohmann::json::parse(res.str);
+    }
+
+    /**
+     * Print the value as XML.
+     */
+    virtual void printValueAsXML(
+        nix::EvalState & state,
+        bool strict,
+        bool location,
+        nix::XMLWriter & doc,
+        nix::NixStringContext & context,
+        nix::PathSet & drvsSeen,
+        const nix::PosIdx pos) const override
+    {
+        if (!desc.printValueAsXML) {
+            return nix::ExternalValueBase::printValueAsXML(state, strict, location, doc, context, drvsSeen, pos);
+        }
+        nix_string_context ctx{context};
+        desc.printValueAsXML(
+            v, (EvalState *) &state, strict, location, &doc, &ctx, &drvsSeen,
+            *reinterpret_cast<const uint32_t *>(&pos));
+    }
+
+    virtual ~NixCExternalValue() override{};
+};
+
+ExternalValue * nix_create_external_value(nix_c_context * context, NixCExternalValueDesc * desc, void * v)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto ret = new
+#ifdef HAVE_BOEHMGC
+            (GC)
+#endif
+                NixCExternalValue(*desc, v);
+        nix_gc_incref(nullptr, ret);
+        return (ExternalValue *) ret;
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+void * nix_get_external_value_content(nix_c_context * context, ExternalValue * b)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto r = dynamic_cast<NixCExternalValue *>((nix::ExternalValueBase *) b);
+        if (r)
+            return r->get_ptr();
+        return nullptr;
+    }
+    NIXC_CATCH_ERRS_NULL
+}

src/libexpr-c/nix_api_external.h

@@ -0,0 +1,196 @@
+#ifndef NIX_API_EXTERNAL_H
+#define NIX_API_EXTERNAL_H
+/** @ingroup libexpr
+ * @addtogroup Externals
+ * @brief Deal with external values
+ * @{
+ */
+/** @file
+ * @brief libexpr C bindings dealing with external values
+ */
+
+#include "nix_api_expr.h"
+#include "nix_api_util.h"
+#include "nix_api_value.h"
+#include "stdbool.h"
+#include "stddef.h"
+#include "stdint.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+// cffi start
+
+/**
+ * @brief Represents a string owned by the Nix language evaluator.
+ * @see nix_set_owned_string
+ */
+typedef struct nix_string_return nix_string_return;
+/**
+ * @brief Wraps a stream that can output multiple string pieces.
+ */
+typedef struct nix_printer nix_printer;
+/**
+ * @brief A list of string context items
+ */
+typedef struct nix_string_context nix_string_context;
+
+/**
+ * @brief Sets the contents of a nix_string_return
+ *
+ * Copies the passed string.
+ * @param[out] str the nix_string_return to write to
+ * @param[in]  c   The string to copy
+ */
+void nix_set_string_return(nix_string_return * str, const char * c);
+
+/**
+ * Print to the nix_printer
+ *
+ * @param[out] context Optional, stores error information
+ * @param printer The nix_printer to print to
+ * @param[in] str The string to print
+ * @returns NIX_OK if everything worked
+ */
+nix_err nix_external_print(nix_c_context * context, nix_printer * printer, const char * str);
+
+/**
+ * Add string context to the nix_string_context object
+ * @param[out] context Optional, stores error information
+ * @param[out] string_context The nix_string_context to add to
+ * @param[in] c The context string to add
+ * @returns NIX_OK if everything worked
+ */
+nix_err nix_external_add_string_context(nix_c_context * context, nix_string_context * string_context, const char * c);
+
+/**
+ * @brief Definition for a class of external values
+ *
+ * Create and implement one of these, then pass it to nix_create_external_value
+ * Make sure to keep it alive while the external value lives.
+ *
+ * Optional functions can be set to NULL
+ *
+ * @see nix_create_external_value
+ */
+typedef struct NixCExternalValueDesc
+{
+    /**
+     * @brief Called when printing the external value
+     *
+     * @param[in] self the void* passed to nix_create_external_value
+     * @param[out] printer The printer to print to, pass to nix_external_print
+     */
+    void (*print)(void * self, nix_printer * printer);
+    /**
+     * @brief Called on :t
+     * @param[in] self the void* passed to nix_create_external_value
+     * @param[out] res the return value
+     */
+    void (*showType)(void * self, nix_string_return * res);
+    /**
+     * @brief Called on `builtins.typeOf`
+     * @param self the void* passed to nix_create_external_value
+     * @param[out] res the return value
+     */
+    void (*typeOf)(void * self, nix_string_return * res);
+    /**
+     * @brief Called on "${str}" and builtins.toString.
+     *
+     * The latter with coerceMore=true
+     * Optional, the default is to throw an error.
+     * @param[in] self the void* passed to nix_create_external_value
+     * @param[out] c writable string context for the resulting string
+     * @param[in] coerceMore boolean, try to coerce to strings in more cases
+     * instead of throwing an error
+     * @param[in] copyToStore boolean, whether to copy referenced paths to store
+     * or keep them as-is
+     * @param[out] res the return value. Not touching this, or setting it to the
+     * empty string, will make the conversion throw an error.
+     */
+    void (*coerceToString)(
+        void * self, nix_string_context * c, int coerceMore, int copyToStore, nix_string_return * res);
+    /**
+     * @brief Try to compare two external values
+     *
+     * Optional, the default is always false.
+     * If the other object was not a Nix C external value, this comparison will
+     * also return false
+     * @param[in] self the void* passed to nix_create_external_value
+     * @param[in] other the void* passed to the other object's
+     * nix_create_external_value
+     * @returns true if the objects are deemed to be equal
+     */
+    int (*equal)(void * self, void * other);
+    /**
+     * @brief Convert the external value to json
+     *
+     * Optional, the default is to throw an error
+     * @param[in] self the void* passed to nix_create_external_value
+     * @param[in] state The evaluator state
+     * @param[in] strict boolean Whether to force the value before printing
+     * @param[out] c writable string context for the resulting string
+     * @param[in] copyToStore whether to copy referenced paths to store or keep
+     * them as-is
+     * @param[out] res the return value. Gets parsed as JSON. Not touching this,
+     * or setting it to the empty string, will make the conversion throw an error.
+     */
+    void (*printValueAsJSON)(
+        void * self, EvalState *, bool strict, nix_string_context * c, bool copyToStore, nix_string_return * res);
+    /**
+     * @brief Convert the external value to XML
+     *
+     * Optional, the default is to throw an error
+     * @todo The mechanisms for this call are incomplete. There are no C
+     *       bindings to work with XML, pathsets and positions.
+     * @param[in] self the void* passed to nix_create_external_value
+     * @param[in] state The evaluator state
+     * @param[in] strict boolean Whether to force the value before printing
+     * @param[in] location boolean Whether to include position information in the
+     * xml
+     * @param[out] doc XML document to output to
+     * @param[out] c writable string context for the resulting string
+     * @param[in,out] drvsSeen a path set to avoid duplicating derivations
+     * @param[in] pos The position of the call.
+     */
+    void (*printValueAsXML)(
+        void * self,
+        EvalState *,
+        int strict,
+        int location,
+        void * doc,
+        nix_string_context * c,
+        void * drvsSeen,
+        int pos);
+} NixCExternalValueDesc;
+
+/**
+ * @brief Create an external value, that can be given to nix_init_external
+ *
+ * Owned by the GC. Use nix_gc_decref when you're done with the pointer.
+ *
+ * @param[out] context Optional, stores error information
+ * @param[in] desc a NixCExternalValueDesc, you should keep this alive as long
+ * as the ExternalValue lives
+ * @param[in] v the value to store
+ * @returns external value, owned by the garbage collector
+ * @see nix_init_external
+ */
+ExternalValue * nix_create_external_value(nix_c_context * context, NixCExternalValueDesc * desc, void * v);
+
+/**
+ * @brief Extract the pointer from a nix c external value.
+ * @param[out] context Optional, stores error information
+ * @param[in] b The external value
+ * @returns The pointer, or null if the external value was not from nix c.
+ * @see nix_get_external
+ */
+void * nix_get_external_value_content(nix_c_context * context, ExternalValue * b);
+
+// cffi end
+#ifdef __cplusplus
+}
+#endif
+/** @} */
+
+#endif // NIX_API_EXTERNAL_H

src/libexpr-c/nix_api_value.cc

@@ -0,0 +1,595 @@
+#include "attr-set.hh"
+#include "config.hh"
+#include "eval.hh"
+#include "globals.hh"
+#include "path.hh"
+#include "primops.hh"
+#include "value.hh"
+
+#include "nix_api_expr.h"
+#include "nix_api_expr_internal.h"
+#include "nix_api_util.h"
+#include "nix_api_util_internal.h"
+#include "nix_api_store_internal.h"
+#include "nix_api_value.h"
+#include "value/context.hh"
+
+#ifdef HAVE_BOEHMGC
+#  include "gc/gc.h"
+#  define GC_INCLUDE_NEW 1
+#  include "gc_cpp.h"
+#endif
+
+// Helper function to throw an exception if value is null
+static const nix::Value & check_value_not_null(const Value * value)
+{
+    if (!value) {
+        throw std::runtime_error("Value is null");
+    }
+    return *((const nix::Value *) value);
+}
+
+static nix::Value & check_value_not_null(Value * value)
+{
+    if (!value) {
+        throw std::runtime_error("Value is null");
+    }
+    return *((nix::Value *) value);
+}
+
+/**
+ * Helper function to convert calls from nix into C API.
+ *
+ * Deals with errors and converts arguments from C++ into C types.
+ */
+static void nix_c_primop_wrapper(
+    PrimOpFun f, void * userdata, nix::EvalState & state, const nix::PosIdx pos, nix::Value ** args, nix::Value & v)
+{
+    nix_c_context ctx;
+    f(userdata, &ctx, (EvalState *) &state, (Value **) args, (Value *) &v);
+    /* TODO: In the future, this should throw different errors depending on the error code */
+    if (ctx.last_err_code != NIX_OK)
+        state.error<nix::EvalError>("Error from builtin function: %s", *ctx.last_err).atPos(pos).debugThrow();
+}
+
+PrimOp * nix_alloc_primop(
+    nix_c_context * context,
+    PrimOpFun fun,
+    int arity,
+    const char * name,
+    const char ** args,
+    const char * doc,
+    void * user_data)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        using namespace std::placeholders;
+        auto p = new
+#ifdef HAVE_BOEHMGC
+            (GC)
+#endif
+                nix::PrimOp{
+                    .name = name,
+                    .args = {},
+                    .arity = (size_t) arity,
+                    .doc = doc,
+                    .fun = std::bind(nix_c_primop_wrapper, fun, user_data, _1, _2, _3, _4)};
+        if (args)
+            for (size_t i = 0; args[i]; i++)
+                p->args.emplace_back(*args);
+        nix_gc_incref(nullptr, p);
+        return (PrimOp *) p;
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+nix_err nix_register_primop(nix_c_context * context, PrimOp * primOp)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        nix::RegisterPrimOp r(std::move(*((nix::PrimOp *) primOp)));
+    }
+    NIXC_CATCH_ERRS
+}
+
+Value * nix_alloc_value(nix_c_context * context, EvalState * state)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        Value * res = state->state.allocValue();
+        nix_gc_incref(nullptr, res);
+        return res;
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+ValueType nix_get_type(nix_c_context * context, const Value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        using namespace nix;
+        switch (v.type()) {
+        case nThunk:
+            return NIX_TYPE_THUNK;
+        case nInt:
+            return NIX_TYPE_INT;
+        case nFloat:
+            return NIX_TYPE_FLOAT;
+        case nBool:
+            return NIX_TYPE_BOOL;
+        case nString:
+            return NIX_TYPE_STRING;
+        case nPath:
+            return NIX_TYPE_PATH;
+        case nNull:
+            return NIX_TYPE_NULL;
+        case nAttrs:
+            return NIX_TYPE_ATTRS;
+        case nList:
+            return NIX_TYPE_LIST;
+        case nFunction:
+            return NIX_TYPE_FUNCTION;
+        case nExternal:
+            return NIX_TYPE_EXTERNAL;
+        }
+        return NIX_TYPE_NULL;
+    }
+    NIXC_CATCH_ERRS_RES(NIX_TYPE_NULL);
+}
+
+const char * nix_get_typename(nix_c_context * context, const Value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        auto s = nix::showType(v);
+        return strdup(s.c_str());
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+bool nix_get_bool(nix_c_context * context, const Value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        assert(v.type() == nix::nBool);
+        return v.boolean();
+    }
+    NIXC_CATCH_ERRS_RES(false);
+}
+
+nix_err nix_get_string(nix_c_context * context, const Value * value, nix_get_string_callback callback, void * user_data)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        assert(v.type() == nix::nString);
+        call_nix_get_string_callback(v.c_str(), callback, user_data);
+    }
+    NIXC_CATCH_ERRS
+}
+
+const char * nix_get_path_string(nix_c_context * context, const Value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        assert(v.type() == nix::nPath);
+        // NOTE (from @yorickvP)
+        // v._path.path should work but may not be how Eelco intended it.
+        // Long-term this function should be rewritten to copy some data into a
+        // user-allocated string.
+        // We could use v.path().to_string().c_str(), but I'm concerned this
+        // crashes. Looks like .path() allocates a CanonPath with a copy of the
+        // string, then it gets the underlying data from that.
+        return v.payload.path.path;
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+unsigned int nix_get_list_size(nix_c_context * context, const Value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        assert(v.type() == nix::nList);
+        return v.listSize();
+    }
+    NIXC_CATCH_ERRS_RES(0);
+}
+
+unsigned int nix_get_attrs_size(nix_c_context * context, const Value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        assert(v.type() == nix::nAttrs);
+        return v.attrs()->size();
+    }
+    NIXC_CATCH_ERRS_RES(0);
+}
+
+double nix_get_float(nix_c_context * context, const Value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        assert(v.type() == nix::nFloat);
+        return v.fpoint();
+    }
+    NIXC_CATCH_ERRS_RES(0.0);
+}
+
+int64_t nix_get_int(nix_c_context * context, const Value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        assert(v.type() == nix::nInt);
+        return v.integer();
+    }
+    NIXC_CATCH_ERRS_RES(0);
+}
+
+ExternalValue * nix_get_external(nix_c_context * context, Value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        assert(v.type() == nix::nExternal);
+        return (ExternalValue *) v.external();
+    }
+    NIXC_CATCH_ERRS_NULL;
+}
+
+Value * nix_get_list_byidx(nix_c_context * context, const Value * value, EvalState * state, unsigned int ix)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        assert(v.type() == nix::nList);
+        auto * p = v.listElems()[ix];
+        nix_gc_incref(nullptr, p);
+        if (p != nullptr)
+            state->state.forceValue(*p, nix::noPos);
+        return (Value *) p;
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+Value * nix_get_attr_byname(nix_c_context * context, const Value * value, EvalState * state, const char * name)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        assert(v.type() == nix::nAttrs);
+        nix::Symbol s = state->state.symbols.create(name);
+        auto attr = v.attrs()->get(s);
+        if (attr) {
+            nix_gc_incref(nullptr, attr->value);
+            state->state.forceValue(*attr->value, nix::noPos);
+            return attr->value;
+        }
+        nix_set_err_msg(context, NIX_ERR_KEY, "missing attribute");
+        return nullptr;
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+bool nix_has_attr_byname(nix_c_context * context, const Value * value, EvalState * state, const char * name)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        assert(v.type() == nix::nAttrs);
+        nix::Symbol s = state->state.symbols.create(name);
+        auto attr = v.attrs()->get(s);
+        if (attr)
+            return true;
+        return false;
+    }
+    NIXC_CATCH_ERRS_RES(false);
+}
+
+Value *
+nix_get_attr_byidx(nix_c_context * context, const Value * value, EvalState * state, unsigned int i, const char ** name)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        const nix::Attr & a = (*v.attrs())[i];
+        *name = ((const std::string &) (state->state.symbols[a.name])).c_str();
+        nix_gc_incref(nullptr, a.value);
+        state->state.forceValue(*a.value, nix::noPos);
+        return a.value;
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+const char * nix_get_attr_name_byidx(nix_c_context * context, const Value * value, EvalState * state, unsigned int i)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        const nix::Attr & a = (*v.attrs())[i];
+        return ((const std::string &) (state->state.symbols[a.name])).c_str();
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+nix_err nix_init_bool(nix_c_context * context, Value * value, bool b)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        v.mkBool(b);
+    }
+    NIXC_CATCH_ERRS
+}
+
+// todo string context
+nix_err nix_init_string(nix_c_context * context, Value * value, const char * str)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        v.mkString(std::string_view(str));
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_init_path_string(nix_c_context * context, EvalState * s, Value * value, const char * str)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        v.mkPath(s->state.rootPath(nix::CanonPath(str)));
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_init_float(nix_c_context * context, Value * value, double d)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        v.mkFloat(d);
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_init_int(nix_c_context * context, Value * value, int64_t i)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        v.mkInt(i);
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_init_null(nix_c_context * context, Value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        v.mkNull();
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_init_apply(nix_c_context * context, Value * value, Value * fn, Value * arg)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        auto & f = check_value_not_null(fn);
+        auto & a = check_value_not_null(arg);
+        v.mkApp(&f, &a);
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_init_external(nix_c_context * context, Value * value, ExternalValue * val)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        auto r = (nix::ExternalValueBase *) val;
+        v.mkExternal(r);
+    }
+    NIXC_CATCH_ERRS
+}
+
+ListBuilder * nix_make_list_builder(nix_c_context * context, EvalState * state, size_t capacity)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto builder = state->state.buildList(capacity);
+        return new
+#if HAVE_BOEHMGC
+            (NoGC)
+#endif
+                ListBuilder{std::move(builder)};
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+nix_err nix_list_builder_insert(nix_c_context * context, ListBuilder * list_builder, unsigned int index, Value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & e = check_value_not_null(value);
+        list_builder->builder[index] = &e;
+    }
+    NIXC_CATCH_ERRS
+}
+
+void nix_list_builder_free(ListBuilder * list_builder)
+{
+#if HAVE_BOEHMGC
+    GC_FREE(list_builder);
+#else
+    delete list_builder;
+#endif
+}
+
+nix_err nix_make_list(nix_c_context * context, ListBuilder * list_builder, Value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        v.mkList(list_builder->builder);
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_init_primop(nix_c_context * context, Value * value, PrimOp * p)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        v.mkPrimOp((nix::PrimOp *) p);
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_copy_value(nix_c_context * context, Value * value, Value * source)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        auto & s = check_value_not_null(source);
+        v = s;
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_make_attrs(nix_c_context * context, Value * value, BindingsBuilder * b)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        v.mkAttrs(b->builder);
+    }
+    NIXC_CATCH_ERRS
+}
+
+BindingsBuilder * nix_make_bindings_builder(nix_c_context * context, EvalState * state, size_t capacity)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto bb = state->state.buildBindings(capacity);
+        return new
+#if HAVE_BOEHMGC
+            (NoGC)
+#endif
+                BindingsBuilder{std::move(bb)};
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+nix_err nix_bindings_builder_insert(nix_c_context * context, BindingsBuilder * bb, const char * name, Value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        nix::Symbol s = bb->builder.state.symbols.create(name);
+        bb->builder.insert(s, &v);
+    }
+    NIXC_CATCH_ERRS
+}
+
+void nix_bindings_builder_free(BindingsBuilder * bb)
+{
+#if HAVE_BOEHMGC
+    GC_FREE((nix::BindingsBuilder *) bb);
+#else
+    delete (nix::BindingsBuilder *) bb;
+#endif
+}
+
+nix_realised_string * nix_string_realise(nix_c_context * context, EvalState * state, Value * value, bool isIFD)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        nix::NixStringContext stringContext;
+        auto rawStr = state->state.coerceToString(nix::noPos, v, stringContext, "while realising a string").toOwned();
+        nix::StorePathSet storePaths;
+        auto rewrites = state->state.realiseContext(stringContext, &storePaths);
+
+        auto s = nix::rewriteStrings(rawStr, rewrites);
+
+        // Convert to the C API StorePath type and convert to vector for index-based access
+        std::vector<StorePath> vec;
+        for (auto & sp : storePaths) {
+            vec.push_back(StorePath{sp});
+        }
+
+        return new nix_realised_string{.str = s, .storePaths = vec};
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+void nix_realised_string_free(nix_realised_string * s)
+{
+    delete s;
+}
+
+size_t nix_realised_string_get_buffer_size(nix_realised_string * s)
+{
+    return s->str.size();
+}
+
+const char * nix_realised_string_get_buffer_start(nix_realised_string * s)
+{
+    return s->str.data();
+}
+
+size_t nix_realised_string_get_store_path_count(nix_realised_string * s)
+{
+    return s->storePaths.size();
+}
+
+const StorePath * nix_realised_string_get_store_path(nix_realised_string * s, size_t i)
+{
+    return &s->storePaths[i];
+}

src/libexpr-c/nix_api_value.h

@@ -0,0 +1,510 @@
+#ifndef NIX_API_VALUE_H
+#define NIX_API_VALUE_H
+
+/** @addtogroup libexpr
+ * @{
+ */
+/** @file
+ * @brief libexpr C bindings dealing with values
+ */
+
+#include "nix_api_util.h"
+#include "nix_api_store.h"
+#include "stdbool.h"
+#include "stddef.h"
+#include "stdint.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+// cffi start
+
+// Type definitions
+typedef enum {
+    NIX_TYPE_THUNK,
+    NIX_TYPE_INT,
+    NIX_TYPE_FLOAT,
+    NIX_TYPE_BOOL,
+    NIX_TYPE_STRING,
+    NIX_TYPE_PATH,
+    NIX_TYPE_NULL,
+    NIX_TYPE_ATTRS,
+    NIX_TYPE_LIST,
+    NIX_TYPE_FUNCTION,
+    NIX_TYPE_EXTERNAL
+} ValueType;
+
+// forward declarations
+typedef void Value;
+typedef struct EvalState EvalState;
+// type defs
+/** @brief Stores an under-construction set of bindings
+ * @ingroup value_manip
+ *
+ * Do not reuse.
+ * @see nix_make_bindings_builder, nix_bindings_builder_free, nix_make_attrs
+ * @see nix_bindings_builder_insert
+ */
+typedef struct BindingsBuilder BindingsBuilder;
+
+/** @brief Stores an under-construction list
+ * @ingroup value_manip
+ *
+ * Do not reuse.
+ * @see nix_make_list_builder, nix_list_builder_free, nix_make_list
+ * @see nix_list_builder_insert
+ */
+typedef struct ListBuilder ListBuilder;
+
+/** @brief PrimOp function
+ * @ingroup primops
+ *
+ * Owned by the GC
+ * @see nix_alloc_primop, nix_init_primop
+ */
+typedef struct PrimOp PrimOp;
+/** @brief External Value
+ * @ingroup Externals
+ *
+ * Owned by the GC
+ */
+typedef struct ExternalValue ExternalValue;
+
+/** @brief String without placeholders, and realised store paths
+ */
+typedef struct nix_realised_string nix_realised_string;
+
+/** @defgroup primops
+ * @brief Create your own primops
+ * @{
+ */
+/** @brief Function pointer for primops
+ * When you want to return an error, call nix_set_err_msg(context, NIX_ERR_UNKNOWN, "your error message here").
+ *
+ * @param[in] user_data Arbitrary data that was initially supplied to nix_alloc_primop
+ * @param[out] context Stores error information.
+ * @param[in] state Evaluator state
+ * @param[in] args list of arguments. Note that these can be thunks and should be forced using nix_value_force before
+ * use.
+ * @param[out] ret return value
+ * @see nix_alloc_primop, nix_init_primop
+ */
+typedef void (*PrimOpFun)(void * user_data, nix_c_context * context, EvalState * state, Value ** args, Value * ret);
+
+/** @brief Allocate a PrimOp
+ *
+ * Owned by the garbage collector.
+ * Use nix_gc_decref() when you're done with the returned PrimOp.
+ *
+ * @param[out] context Optional, stores error information
+ * @param[in] fun callback
+ * @param[in] arity expected number of function arguments
+ * @param[in] name function name
+ * @param[in] args array of argument names, NULL-terminated
+ * @param[in] doc optional, documentation for this primop
+ * @param[in] user_data optional, arbitrary data, passed to the callback when it's called
+ * @return primop, or null in case of errors
+ * @see nix_init_primop
+ */
+PrimOp * nix_alloc_primop(
+    nix_c_context * context,
+    PrimOpFun fun,
+    int arity,
+    const char * name,
+    const char ** args,
+    const char * doc,
+    void * user_data);
+
+/** @brief add a primop to the `builtins` attribute set
+ *
+ * Only applies to States created after this call.
+ *
+ * Moves your PrimOp content into the global evaluator
+ * registry, meaning your input PrimOp pointer is no longer usable.
+ * You are free to remove your references to it,
+ * after which it will be garbage collected.
+ *
+ * @param[out] context Optional, stores error information
+ * @return primop, or null in case of errors
+ *
+ */
+nix_err nix_register_primop(nix_c_context * context, PrimOp * primOp);
+/** @} */
+
+// Function prototypes
+
+/** @brief Allocate a Nix value
+ *
+ * Owned by the GC. Use nix_gc_decref() when you're done with the pointer
+ * @param[out] context Optional, stores error information
+ * @param[in] state nix evaluator state
+ * @return value, or null in case of errors
+ *
+ */
+Value * nix_alloc_value(nix_c_context * context, EvalState * state);
+
+/** @addtogroup value_manip Manipulating values
+ * @brief Functions to inspect and change Nix language values, represented by Value.
+ * @{
+ */
+/** @name Getters
+ */
+/**@{*/
+/** @brief Get value type
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect
+ * @return type of nix value
+ */
+ValueType nix_get_type(nix_c_context * context, const Value * value);
+
+/** @brief Get type name of value as defined in the evaluator
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect
+ * @return type name, owned string
+ * @todo way to free the result
+ */
+const char * nix_get_typename(nix_c_context * context, const Value * value);
+
+/** @brief Get boolean value
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect
+ * @return true or false, error info via context
+ */
+bool nix_get_bool(nix_c_context * context, const Value * value);
+
+/** @brief Get the raw string
+ *
+ * This may contain placeholders.
+ *
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect
+ * @param[in] callback Called with the string value.
+ * @param[in] user_data optional, arbitrary data, passed to the callback when it's called.
+ * @return string
+ * @return error code, NIX_OK on success.
+ */
+nix_err
+nix_get_string(nix_c_context * context, const Value * value, nix_get_string_callback callback, void * user_data);
+
+/** @brief Get path as string
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect
+ * @return string
+ * @return NULL in case of error.
+ */
+const char * nix_get_path_string(nix_c_context * context, const Value * value);
+
+/** @brief Get the length of a list
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect
+ * @return length of list, error info via context
+ */
+unsigned int nix_get_list_size(nix_c_context * context, const Value * value);
+
+/** @brief Get the element count of an attrset
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect
+ * @return attrset element count, error info via context
+ */
+unsigned int nix_get_attrs_size(nix_c_context * context, const Value * value);
+
+/** @brief Get float value in 64 bits
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect
+ * @return float contents, error info via context
+ */
+double nix_get_float(nix_c_context * context, const Value * value);
+
+/** @brief Get int value
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect
+ * @return int contents, error info via context
+ */
+int64_t nix_get_int(nix_c_context * context, const Value * value);
+
+/** @brief Get external reference
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect
+ * @return reference to external, NULL in case of error
+ */
+ExternalValue * nix_get_external(nix_c_context * context, Value *);
+
+/** @brief Get the ix'th element of a list
+ *
+ * Owned by the GC. Use nix_gc_decref when you're done with the pointer
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect
+ * @param[in] state nix evaluator state
+ * @param[in] ix list element to get
+ * @return value, NULL in case of errors
+ */
+Value * nix_get_list_byidx(nix_c_context * context, const Value * value, EvalState * state, unsigned int ix);
+
+/** @brief Get an attr by name
+ *
+ * Owned by the GC. Use nix_gc_decref when you're done with the pointer
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect
+ * @param[in] state nix evaluator state
+ * @param[in] name attribute name
+ * @return value, NULL in case of errors
+ */
+Value * nix_get_attr_byname(nix_c_context * context, const Value * value, EvalState * state, const char * name);
+
+/** @brief Check if an attribute name exists on a value
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect
+ * @param[in] state nix evaluator state
+ * @param[in] name attribute name
+ * @return value, error info via context
+ */
+bool nix_has_attr_byname(nix_c_context * context, const Value * value, EvalState * state, const char * name);
+
+/** @brief Get an attribute by index in the sorted bindings
+ *
+ * Also gives you the name.
+ *
+ * Owned by the GC. Use nix_gc_decref when you're done with the pointer
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect
+ * @param[in] state nix evaluator state
+ * @param[in] i attribute index
+ * @param[out] name will store a pointer to the attribute name
+ * @return value, NULL in case of errors
+ */
+Value *
+nix_get_attr_byidx(nix_c_context * context, const Value * value, EvalState * state, unsigned int i, const char ** name);
+
+/** @brief Get an attribute name by index in the sorted bindings
+ *
+ * Useful when you want the name but want to avoid evaluation.
+ *
+ * Owned by the nix EvalState
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect
+ * @param[in] state nix evaluator state
+ * @param[in] i attribute index
+ * @return name, NULL in case of errors
+ */
+const char * nix_get_attr_name_byidx(nix_c_context * context, const Value * value, EvalState * state, unsigned int i);
+
+/**@}*/
+/** @name Initializers
+ *
+ * Values are typically "returned" by initializing already allocated memory that serves as the return value.
+ * For this reason, the construction of values is not tied their allocation.
+ * Nix is a language with immutable values. Respect this property by only initializing Values once; and only initialize
+ * Values that are meant to be initialized by you. Failing to adhere to these rules may lead to undefined behavior.
+ */
+/**@{*/
+/** @brief Set boolean value
+ * @param[out] context Optional, stores error information
+ * @param[out] value Nix value to modify
+ * @param[in] b the boolean value
+ * @return error code, NIX_OK on success.
+ */
+nix_err nix_init_bool(nix_c_context * context, Value * value, bool b);
+
+/** @brief Set a string
+ * @param[out] context Optional, stores error information
+ * @param[out] value Nix value to modify
+ * @param[in] str the string, copied
+ * @return error code, NIX_OK on success.
+ */
+nix_err nix_init_string(nix_c_context * context, Value * value, const char * str);
+
+/** @brief Set a path
+ * @param[out] context Optional, stores error information
+ * @param[out] value Nix value to modify
+ * @param[in] str the path string, copied
+ * @return error code, NIX_OK on success.
+ */
+nix_err nix_init_path_string(nix_c_context * context, EvalState * s, Value * value, const char * str);
+
+/** @brief Set a float
+ * @param[out] context Optional, stores error information
+ * @param[out] value Nix value to modify
+ * @param[in] d the float, 64-bits
+ * @return error code, NIX_OK on success.
+ */
+nix_err nix_init_float(nix_c_context * context, Value * value, double d);
+
+/** @brief Set an int
+ * @param[out] context Optional, stores error information
+ * @param[out] value Nix value to modify
+ * @param[in] i the int
+ * @return error code, NIX_OK on success.
+ */
+
+nix_err nix_init_int(nix_c_context * context, Value * value, int64_t i);
+/** @brief Set null
+ * @param[out] context Optional, stores error information
+ * @param[out] value Nix value to modify
+ * @return error code, NIX_OK on success.
+ */
+nix_err nix_init_null(nix_c_context * context, Value * value);
+
+/** @brief Set the value to a thunk that will perform a function application when needed.
+ *
+ * Thunks may be put into attribute sets and lists to perform some computation lazily; on demand.
+ * However, note that in some places, a thunk must not be returned, such as in the return value of a PrimOp.
+ * In such cases, you may use nix_value_call() instead (but note the different argument order).
+ *
+ * @param[out] context Optional, stores error information
+ * @param[out] value Nix value to modify
+ * @param[in] fn function to call
+ * @param[in] arg argument to pass
+ * @return error code, NIX_OK on successful initialization.
+ * @see nix_value_call() for a similar function that performs the call immediately and only stores the return value.
+ *      Note the different argument order.
+ */
+nix_err nix_init_apply(nix_c_context * context, Value * value, Value * fn, Value * arg);
+
+/** @brief Set an external value
+ * @param[out] context Optional, stores error information
+ * @param[out] value Nix value to modify
+ * @param[in] val the external value to set. Will be GC-referenced by the value.
+ * @return error code, NIX_OK on success.
+ */
+nix_err nix_init_external(nix_c_context * context, Value * value, ExternalValue * val);
+
+/** @brief Create a list from a list builder
+ * @param[out] context Optional, stores error information
+ * @param[in] list_builder list builder to use. Make sure to unref this afterwards.
+ * @param[out] value Nix value to modify
+ * @return error code, NIX_OK on success.
+ */
+nix_err nix_make_list(nix_c_context * context, ListBuilder * list_builder, Value * value);
+
+/** @brief Create a list builder
+ * @param[out] context Optional, stores error information
+ * @param[in] state nix evaluator state
+ * @param[in] capacity how many bindings you'll add. Don't exceed.
+ * @return owned reference to a list builder. Make sure to unref when you're done.
+ */
+ListBuilder * nix_make_list_builder(nix_c_context * context, EvalState * state, size_t capacity);
+
+/** @brief Insert bindings into a builder
+ * @param[out] context Optional, stores error information
+ * @param[in] list_builder ListBuilder to insert into
+ * @param[in] index index to manipulate
+ * @param[in] value value to insert
+ * @return error code, NIX_OK on success.
+ */
+nix_err nix_list_builder_insert(nix_c_context * context, ListBuilder * list_builder, unsigned int index, Value * value);
+
+/** @brief Free a list builder
+ *
+ * Does not fail.
+ * @param[in] builder the builder to free
+ */
+void nix_list_builder_free(ListBuilder * list_builder);
+
+/** @brief Create an attribute set from a bindings builder
+ * @param[out] context Optional, stores error information
+ * @param[out] value Nix value to modify
+ * @param[in] b bindings builder to use. Make sure to unref this afterwards.
+ * @return error code, NIX_OK on success.
+ */
+nix_err nix_make_attrs(nix_c_context * context, Value * value, BindingsBuilder * b);
+
+/** @brief Set primop
+ * @param[out] context Optional, stores error information
+ * @param[out] value Nix value to modify
+ * @param[in] op primop, will be gc-referenced by the value
+ * @see nix_alloc_primop
+ * @return error code, NIX_OK on success.
+ */
+nix_err nix_init_primop(nix_c_context * context, Value * value, PrimOp * op);
+/** @brief Copy from another value
+ * @param[out] context Optional, stores error information
+ * @param[out] value Nix value to modify
+ * @param[in] source value to copy from
+ * @return error code, NIX_OK on success.
+ */
+nix_err nix_copy_value(nix_c_context * context, Value * value, Value * source);
+/**@}*/
+
+/** @brief Create a bindings builder
+* @param[out] context Optional, stores error information
+* @param[in] state nix evaluator state
+* @param[in] capacity how many bindings you'll add. Don't exceed.
+* @return owned reference to a bindings builder. Make sure to unref when you're
+done.
+*/
+BindingsBuilder * nix_make_bindings_builder(nix_c_context * context, EvalState * state, size_t capacity);
+
+/** @brief Insert bindings into a builder
+ * @param[out] context Optional, stores error information
+ * @param[in] builder BindingsBuilder to insert into
+ * @param[in] name attribute name, only used for the duration of the call.
+ * @param[in] value value to give the binding
+ * @return error code, NIX_OK on success.
+ */
+nix_err
+nix_bindings_builder_insert(nix_c_context * context, BindingsBuilder * builder, const char * name, Value * value);
+
+/** @brief Free a bindings builder
+ *
+ * Does not fail.
+ * @param[in] builder the builder to free
+ */
+void nix_bindings_builder_free(BindingsBuilder * builder);
+/**@}*/
+
+/** @brief Realise a string context.
+ *
+ * This will
+ *  - realise the store paths referenced by the string's context, and
+ *  - perform the replacement of placeholders.
+ *  - create temporary garbage collection roots for the store paths, for
+ *    the lifetime of the current process.
+ *  - log to stderr
+ *
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value, which must be a string
+ * @param[in] state Nix evaluator state
+ * @param[in] isIFD If true, disallow derivation outputs if setting `allow-import-from-derivation` is false.
+                    You should set this to true when this call is part of a primop.
+                    You should set this to false when building for your application's purpose.
+ * @return NULL if failed, are a new nix_realised_string, which must be freed with nix_realised_string_free
+ */
+nix_realised_string * nix_string_realise(nix_c_context * context, EvalState * state, Value * value, bool isIFD);
+
+/** @brief Start of the string
+ * @param[in] realised_string
+ * @return pointer to the start of the string. It may not be null-terminated.
+ */
+const char * nix_realised_string_get_buffer_start(nix_realised_string * realised_string);
+
+/** @brief Length of the string
+ * @param[in] realised_string
+ * @return length of the string in bytes
+ */
+size_t nix_realised_string_get_buffer_size(nix_realised_string * realised_string);
+
+/** @brief Number of realised store paths
+ * @param[in] realised_string
+ * @return number of realised store paths that were referenced by the string via its context
+ */
+size_t nix_realised_string_get_store_path_count(nix_realised_string * realised_string);
+
+/** @brief Get a store path. The store paths are stored in an arbitrary order.
+ * @param[in] realised_string
+ * @param[in] index index of the store path, must be less than the count
+ * @return store path
+ */
+const StorePath * nix_realised_string_get_store_path(nix_realised_string * realised_string, size_t index);
+
+/** @brief Free a realised string
+ * @param[in] realised_string
+ */
+void nix_realised_string_free(nix_realised_string * realised_string);
+
+// cffi end
+#ifdef __cplusplus
+}
+#endif
+
+/** @} */
+#endif // NIX_API_VALUE_H