set up builder pipe

flake.lock: Update

.github/STALE-BOT.md

@@ -3,7 +3,7 @@
 - Thanks for your contribution!
 - To remove the stale label, just leave a new comment.
 - _How to find the right people to ping?_ → [`git blame`](https://git-scm.com/docs/git-blame) to the rescue! (or GitHub's history and blame buttons.)
-- You can always ask for help on [our Discourse Forum](https://discourse.nixos.org/) or on the [#nixos IRC channel](https://webchat.freenode.net/#nixos).
+- You can always ask for help on [our Discourse Forum](https://discourse.nixos.org/) or on [Matrix - #nix:nixos.org](https://matrix.to/#/#nix:nixos.org).
 
 ## Suggestions for PRs
 

.github/workflows/test.yml

@@ -8,10 +8,61 @@ jobs:
       matrix:
         os: [ubuntu-latest, macos-latest]
     runs-on: ${{ matrix.os }}
+
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v2.3.4
       with:
         fetch-depth: 0
-    - uses: cachix/install-nix-action@v12
-    #- run: nix flake check
-    - run: nix-build -A checks.$(if [[ `uname` = Linux ]]; then echo x86_64-linux; else echo x86_64-darwin; fi)
+    - uses: cachix/install-nix-action@v13
+    - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
+    - uses: cachix/cachix-action@v10
+      with:
+        name: '${{ env.CACHIX_NAME }}'
+        signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}'
+        authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
+    - run: nix-build -A checks.$(nix-instantiate --eval -E '(builtins.currentSystem)')
+  check_cachix:
+    name: Cachix secret present for installer tests
+    runs-on: ubuntu-latest
+    outputs:
+      secret: ${{ steps.secret.outputs.secret }}
+    steps:
+      - name: Check for Cachix secret
+        id: secret
+        env:
+          _CACHIX_SECRETS: ${{ secrets.CACHIX_SIGNING_KEY }}${{ secrets.CACHIX_AUTH_TOKEN }}
+        run: echo "::set-output name=secret::${{ env._CACHIX_SECRETS != '' }}"
+  installer:
+    needs: [tests, check_cachix]
+    if: github.event_name == 'push' && needs.check_cachix.outputs.secret == 'true'
+    runs-on: ubuntu-latest
+    outputs:
+      installerURL: ${{ steps.prepare-installer.outputs.installerURL }}
+    steps:
+    - uses: actions/checkout@v2.3.4
+      with:
+        fetch-depth: 0
+    - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
+    - uses: cachix/install-nix-action@v13
+    - uses: cachix/cachix-action@v10
+      with:
+        name: '${{ env.CACHIX_NAME }}'
+        signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}'
+        authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
+    - id: prepare-installer
+      run: scripts/prepare-installer-for-github-actions
+  installer_test:
+    needs: [installer, check_cachix]
+    if: github.event_name == 'push' && needs.check_cachix.outputs.secret == 'true'
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+    runs-on: ${{ matrix.os }}
+    steps:
+    - uses: actions/checkout@v2.3.4
+    - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
+    - uses: cachix/install-nix-action@v13
+      with:
+        install_url: '${{needs.installer.outputs.installerURL}}'
+        install_options: "--tarball-url-prefix https://${{ env.CACHIX_NAME }}.cachix.org/serve"
+    - run: nix-instantiate -E 'builtins.currentTime' --eval

.gitignore

@@ -82,6 +82,7 @@ perl/Makefile.config
 /tests/shell
 /tests/shell.drv
 /tests/config.nix
+/tests/ca/config.nix
 
 # /tests/lang/
 /tests/lang/*.out

Makefile

@@ -12,6 +12,7 @@ makefiles = \
   src/resolve-system-dependencies/local.mk \
   scripts/local.mk \
   misc/bash/local.mk \
+  misc/zsh/local.mk \
   misc/systemd/local.mk \
   misc/launchd/local.mk \
   misc/upstart/local.mk \

Makefile.config.in

@@ -9,13 +9,14 @@ CXXFLAGS = @CXXFLAGS@
 EDITLINE_LIBS = @EDITLINE_LIBS@
 ENABLE_S3 = @ENABLE_S3@
 GTEST_LIBS = @GTEST_LIBS@
+HAVE_LIBCPUID = @HAVE_LIBCPUID@
 HAVE_SECCOMP = @HAVE_SECCOMP@
 LDFLAGS = @LDFLAGS@
 LIBARCHIVE_LIBS = @LIBARCHIVE_LIBS@
 LIBBROTLI_LIBS = @LIBBROTLI_LIBS@
 LIBCURL_LIBS = @LIBCURL_LIBS@
-LIBLZMA_LIBS = @LIBLZMA_LIBS@
 OPENSSL_LIBS = @OPENSSL_LIBS@
+LIBSECCOMP_LIBS = @LIBSECCOMP_LIBS@
 PACKAGE_NAME = @PACKAGE_NAME@
 PACKAGE_VERSION = @PACKAGE_VERSION@
 SHELL = @bash@

README.md

@@ -28,7 +28,8 @@ build nix from source with nix-build or how to get a development environment.
 - [Nix manual](https://nixos.org/nix/manual)
 - [Nix jobsets on hydra.nixos.org](https://hydra.nixos.org/project/nix)
 - [NixOS Discourse](https://discourse.nixos.org/)
-- [IRC - #nixos on freenode.net](irc://irc.freenode.net/#nixos)
+- [Matrix - #nix:nixos.org](https://matrix.to/#/#nix:nixos.org)
+- [IRC - #nixos on libera.chat](irc://irc.libera.chat/#nixos)
 
 ## License
 

config/config.guess

@@ -1,8 +1,8 @@
 #! /bin/sh
 # Attempt to guess a canonical system name.
-#   Copyright 1992-2020 Free Software Foundation, Inc.
+#   Copyright 1992-2021 Free Software Foundation, Inc.
 
-timestamp='2020-11-19'
+timestamp='2021-01-25'
 
 # This file is free software; you can redistribute it and/or modify it
 # under the terms of the GNU General Public License as published by
@@ -50,7 +50,7 @@ version="\
 GNU config.guess ($timestamp)
 
 Originally written by Per Bothner.
-Copyright 1992-2020 Free Software Foundation, Inc.
+Copyright 1992-2021 Free Software Foundation, Inc.
 
 This is free software; see the source for copying conditions.  There is NO
 warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE."
@@ -188,10 +188,9 @@ case "$UNAME_MACHINE:$UNAME_SYSTEM:$UNAME_RELEASE:$UNAME_VERSION" in
 	#
 	# Note: NetBSD doesn't particularly care about the vendor
 	# portion of the name.  We always set it to "unknown".
-	sysctl="sysctl -n hw.machine_arch"
 	UNAME_MACHINE_ARCH=$( (uname -p 2>/dev/null || \
-	    "/sbin/$sysctl" 2>/dev/null || \
-	    "/usr/sbin/$sysctl" 2>/dev/null || \
+	    /sbin/sysctl -n hw.machine_arch 2>/dev/null || \
+	    /usr/sbin/sysctl -n hw.machine_arch 2>/dev/null || \
 	    echo unknown))
 	case "$UNAME_MACHINE_ARCH" in
 	    aarch64eb) machine=aarch64_be-unknown ;;
@@ -996,6 +995,9 @@ EOF
     k1om:Linux:*:*)
 	echo "$UNAME_MACHINE"-unknown-linux-"$LIBC"
 	exit ;;
+    loongarch32:Linux:*:* | loongarch64:Linux:*:* | loongarchx32:Linux:*:*)
+	echo "$UNAME_MACHINE"-unknown-linux-"$LIBC"
+	exit ;;
     m32r*:Linux:*:*)
 	echo "$UNAME_MACHINE"-unknown-linux-"$LIBC"
 	exit ;;
@@ -1084,7 +1086,7 @@ EOF
     ppcle:Linux:*:*)
 	echo powerpcle-unknown-linux-"$LIBC"
 	exit ;;
-    riscv32:Linux:*:* | riscv64:Linux:*:*)
+    riscv32:Linux:*:* | riscv32be:Linux:*:* | riscv64:Linux:*:* | riscv64be:Linux:*:*)
 	echo "$UNAME_MACHINE"-unknown-linux-"$LIBC"
 	exit ;;
     s390:Linux:*:* | s390x:Linux:*:*)
@@ -1480,8 +1482,8 @@ EOF
     i*86:rdos:*:*)
 	echo "$UNAME_MACHINE"-pc-rdos
 	exit ;;
-    i*86:AROS:*:*)
-	echo "$UNAME_MACHINE"-pc-aros
+    *:AROS:*:*)
+	echo "$UNAME_MACHINE"-unknown-aros
 	exit ;;
     x86_64:VMkernel:*:*)
 	echo "$UNAME_MACHINE"-unknown-esx

config/config.sub

@@ -1,8 +1,8 @@
 #! /bin/sh
 # Configuration validation subroutine script.
-#   Copyright 1992-2020 Free Software Foundation, Inc.
+#   Copyright 1992-2021 Free Software Foundation, Inc.
 
-timestamp='2020-12-02'
+timestamp='2021-01-08'
 
 # This file is free software; you can redistribute it and/or modify it
 # under the terms of the GNU General Public License as published by
@@ -67,7 +67,7 @@ Report bugs and patches to <config-patches@gnu.org>."
 version="\
 GNU config.sub ($timestamp)
 
-Copyright 1992-2020 Free Software Foundation, Inc.
+Copyright 1992-2021 Free Software Foundation, Inc.
 
 This is free software; see the source for copying conditions.  There is NO
 warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE."
@@ -1185,6 +1185,7 @@ case $cpu-$vendor in
 			| k1om \
 			| le32 | le64 \
 			| lm32 \
+			| loongarch32 | loongarch64 | loongarchx32 \
 			| m32c | m32r | m32rle \
 			| m5200 | m68000 | m680[012346]0 | m68360 | m683?2 | m68k \
 			| m6811 | m68hc11 | m6812 | m68hc12 | m68hcs12x \
@@ -1229,7 +1230,7 @@ case $cpu-$vendor in
 			| powerpc | powerpc64 | powerpc64le | powerpcle | powerpcspe \
 			| pru \
 			| pyramid \
-			| riscv | riscv32 | riscv64 \
+			| riscv | riscv32 | riscv32be | riscv64 | riscv64be \
 			| rl78 | romp | rs6000 | rx \
 			| s390 | s390x \
 			| score \
@@ -1682,11 +1683,14 @@ fi
 
 # Now, validate our (potentially fixed-up) OS.
 case $os in
-	# Sometimes we do "kernel-abi", so those need to count as OSes.
+	# Sometimes we do "kernel-libc", so those need to count as OSes.
 	musl* | newlib* | uclibc*)
 		;;
-	# Likewise for "kernel-libc"
-	eabi | eabihf | gnueabi | gnueabihf)
+	# Likewise for "kernel-abi"
+	eabi* | gnueabi*)
+		;;
+	# VxWorks passes extra cpu info in the 4th filed.
+	simlinux | simwindows | spe)
 		;;
 	# Now accept the basic system types.
 	# The portable systems comes first.
@@ -1750,6 +1754,8 @@ case $kernel-$os in
 		;;
 	kfreebsd*-gnu* | kopensolaris*-gnu*)
 		;;
+	vxworks-simlinux | vxworks-simwindows | vxworks-spe)
+		;;
 	nto-qnx*)
 		;;
 	os2-emx)

configure.ac

@@ -1,4 +1,4 @@
-AC_INIT(nix, m4_esyscmd([bash -c "echo -n $(cat ./.version)$VERSION_SUFFIX"]))
+AC_INIT([nix],[m4_esyscmd(bash -c "echo -n $(cat ./.version)$VERSION_SUFFIX")])
 AC_CONFIG_MACRO_DIRS([m4])
 AC_CONFIG_SRCDIR(README.md)
 AC_CONFIG_AUX_DIR(config)
@@ -9,8 +9,7 @@ AC_PROG_SED
 AC_CANONICAL_HOST
 AC_MSG_CHECKING([for the canonical Nix system name])
 
-AC_ARG_WITH(system, AC_HELP_STRING([--with-system=SYSTEM],
-  [Platform identifier (e.g., `i686-linux').]),
+AC_ARG_WITH(system, AS_HELP_STRING([--with-system=SYSTEM],[Platform identifier (e.g., `i686-linux').]),
   [system=$withval],
   [case "$host_cpu" in
      i*86)
@@ -66,7 +65,7 @@ AC_SYS_LARGEFILE
 AC_STRUCT_DIRENT_D_TYPE
 if test "$sys_name" = sunos; then
     # Solaris requires -lsocket -lnsl for network functions
-    LIBS="-lsocket -lnsl $LIBS"
+    LDFLAGS="-lsocket -lnsl $LDFLAGS"
 fi
 
 
@@ -127,8 +126,7 @@ NEED_PROG(jq, jq)
 AC_SUBST(coreutils, [$(dirname $(type -p cat))])
 
 
-AC_ARG_WITH(store-dir, AC_HELP_STRING([--with-store-dir=PATH],
-  [path of the Nix store (defaults to /nix/store)]),
+AC_ARG_WITH(store-dir, AS_HELP_STRING([--with-store-dir=PATH],[path of the Nix store (defaults to /nix/store)]),
   storedir=$withval, storedir='/nix/store')
 AC_SUBST(storedir)
 
@@ -152,13 +150,12 @@ int main() {
 }]])], GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC=no, GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC=yes)
 AC_MSG_RESULT($GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC)
 if test "x$GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC" = xyes; then
-    LIBS="-latomic $LIBS"
+    LDFLAGS="-latomic $LDFLAGS"
 fi
 
 PKG_PROG_PKG_CONFIG
 
-AC_ARG_ENABLE(shared, AC_HELP_STRING([--enable-shared],
-  [Build shared libraries for Nix [default=yes]]),
+AC_ARG_ENABLE(shared, AS_HELP_STRING([--enable-shared],[Build shared libraries for Nix [default=yes]]),
   shared=$enableval, shared=yes)
 if test "$shared" = yes; then
   AC_SUBST(BUILD_SHARED_LIBS, 1, [Whether to build shared libraries.])
@@ -172,11 +169,6 @@ fi
 PKG_CHECK_MODULES([OPENSSL], [libcrypto], [CXXFLAGS="$OPENSSL_CFLAGS $CXXFLAGS"])
 
 
-# Look for libbz2, a required dependency.
-AC_CHECK_LIB([bz2], [BZ2_bzWriteOpen], [true],
-  [AC_MSG_ERROR([Nix requires libbz2, which is part of bzip2.  See https://sourceware.org/bzip2/.])])
-AC_CHECK_HEADERS([bzlib.h], [true],
-  [AC_MSG_ERROR([Nix requires libbz2, which is part of bzip2.  See https://sourceware.org/bzip2/.])])
 # Checks for libarchive
 PKG_CHECK_MODULES([LIBARCHIVE], [libarchive >= 3.1.2], [CXXFLAGS="$LIBARCHIVE_CFLAGS $CXXFLAGS"])
 # Workaround until https://github.com/libarchive/libarchive/issues/1446 is fixed
@@ -205,26 +197,23 @@ PKG_CHECK_MODULES([EDITLINE], [libeditline], [CXXFLAGS="$EDITLINE_CFLAGS $CXXFLA
 # Look for libsodium, an optional dependency.
 PKG_CHECK_MODULES([SODIUM], [libsodium], [CXXFLAGS="$SODIUM_CFLAGS $CXXFLAGS"])
 
-# Look for liblzma, a required dependency.
-PKG_CHECK_MODULES([LIBLZMA], [liblzma], [CXXFLAGS="$LIBLZMA_CFLAGS $CXXFLAGS"])
-AC_CHECK_LIB([lzma], [lzma_stream_encoder_mt],
-  [AC_DEFINE([HAVE_LZMA_MT], [1], [xz multithreaded compression support])])
-
-# Look for zlib, a required dependency.
-PKG_CHECK_MODULES([ZLIB], [zlib], [CXXFLAGS="$ZLIB_CFLAGS $CXXFLAGS"])
-AC_CHECK_HEADER([zlib.h],[:],[AC_MSG_ERROR([could not find the zlib.h header])])
-LDFLAGS="-lz $LDFLAGS"
-
 # Look for libbrotli{enc,dec}.
 PKG_CHECK_MODULES([LIBBROTLI], [libbrotlienc libbrotlidec], [CXXFLAGS="$LIBBROTLI_CFLAGS $CXXFLAGS"])
 
+# Look for libcpuid.
+if test "$machine_name" = "x86_64"; then
+  PKG_CHECK_MODULES([LIBCPUID], [libcpuid], [CXXFLAGS="$LIBCPUID_CFLAGS $CXXFLAGS"])
+  have_libcpuid=1
+  AC_DEFINE([HAVE_LIBCPUID], [1], [Use libcpuid])
+fi
+AC_SUBST(HAVE_LIBCPUID, [$have_libcpuid])
+
 
 # Look for libseccomp, required for Linux sandboxing.
 if test "$sys_name" = linux; then
   AC_ARG_ENABLE([seccomp-sandboxing],
-                AC_HELP_STRING([--disable-seccomp-sandboxing],
-                               [Don't build support for seccomp sandboxing (only recommended if your arch doesn't support libseccomp yet!)]
-                              ))
+                AS_HELP_STRING([--disable-seccomp-sandboxing],[Don't build support for seccomp sandboxing (only recommended if your arch doesn't support libseccomp yet!)
+                              ]))
   if test "x$enable_seccomp_sandboxing" != "xno"; then
     PKG_CHECK_MODULES([LIBSECCOMP], [libseccomp],
                       [CXXFLAGS="$LIBSECCOMP_CFLAGS $CXXFLAGS"])
@@ -242,8 +231,8 @@ AC_SUBST(HAVE_SECCOMP, [$have_seccomp])
 # Look for aws-cpp-sdk-s3.
 AC_LANG_PUSH(C++)
 AC_CHECK_HEADERS([aws/s3/S3Client.h],
-  [AC_DEFINE([ENABLE_S3], [1], [Whether to enable S3 support via aws-sdk-cpp.])
-  enable_s3=1], [enable_s3=])
+  [AC_DEFINE([ENABLE_S3], [1], [Whether to enable S3 support via aws-sdk-cpp.]) enable_s3=1], 
+  [AC_DEFINE([ENABLE_S3], [0], [Whether to enable S3 support via aws-sdk-cpp.]) enable_s3=])
 AC_SUBST(ENABLE_S3, [$enable_s3])
 AC_LANG_POP(C++)
 
@@ -256,8 +245,7 @@ fi
 
 
 # Whether to use the Boehm garbage collector.
-AC_ARG_ENABLE(gc, AC_HELP_STRING([--enable-gc],
-  [enable garbage collection in the Nix expression evaluator (requires Boehm GC) [default=yes]]),
+AC_ARG_ENABLE(gc, AS_HELP_STRING([--enable-gc],[enable garbage collection in the Nix expression evaluator (requires Boehm GC) [default=yes]]),
   gc=$enableval, gc=yes)
 if test "$gc" = yes; then
   PKG_CHECK_MODULES([BDW_GC], [bdw-gc])
@@ -271,8 +259,7 @@ PKG_CHECK_MODULES([GTEST], [gtest_main])
 
 
 # documentation generation switch
-AC_ARG_ENABLE(doc-gen, AC_HELP_STRING([--disable-doc-gen],
-  [disable documentation generation]),
+AC_ARG_ENABLE(doc-gen, AS_HELP_STRING([--disable-doc-gen],[disable documentation generation]),
   doc_generate=$enableval, doc_generate=yes)
 AC_SUBST(doc_generate)
 
@@ -292,19 +279,7 @@ if test "$(uname)" = "Darwin"; then
 fi
 
 
-# Do we have GNU tar?
-AC_MSG_CHECKING([if you have a recent GNU tar])
-if $tar --version 2> /dev/null | grep -q GNU && tar cvf /dev/null --warning=no-timestamp ./config.log > /dev/null; then
-    AC_MSG_RESULT(yes)
-    tarFlags="--warning=no-timestamp"
-else
-    AC_MSG_RESULT(no)
-fi
-AC_SUBST(tarFlags)
-
-
-AC_ARG_WITH(sandbox-shell, AC_HELP_STRING([--with-sandbox-shell=PATH],
-  [path of a statically-linked shell to use as /bin/sh in sandboxes]),
+AC_ARG_WITH(sandbox-shell, AS_HELP_STRING([--with-sandbox-shell=PATH],[path of a statically-linked shell to use as /bin/sh in sandboxes]),
   sandbox_shell=$withval)
 AC_SUBST(sandbox_shell)
 
@@ -319,6 +294,6 @@ done
 
 rm -f Makefile.config
 
-AC_CONFIG_HEADER([config.h])
+AC_CONFIG_HEADERS([config.h])
 AC_CONFIG_FILES([])
 AC_OUTPUT

doc/manual/generate-manpage.nix

@@ -7,7 +7,10 @@ let
 
   showCommand =
     { command, def, filename }:
-    "# Name\n\n"
+    ''
+      **Warning**: This program is **experimental** and its interface is subject to change.
+    ''
+    + "# Name\n\n"
     + "`${command}` - ${def.description}\n\n"
     + "# Synopsis\n\n"
     + showSynopsis { inherit command; args = def.args; }

doc/manual/local.mk

@@ -25,19 +25,19 @@ nix-eval = $(dummy-env) $(bindir)/nix eval --experimental-features nix-command -
 $(d)/%.1: $(d)/src/command-ref/%.md
 	@printf "Title: %s\n\n" "$$(basename $@ .1)" > $^.tmp
 	@cat $^ >> $^.tmp
-	$(trace-gen) lowdown -sT man $^.tmp -o $@
+	$(trace-gen) lowdown -sT man -M section=1 $^.tmp -o $@
 	@rm $^.tmp
 
 $(d)/%.8: $(d)/src/command-ref/%.md
 	@printf "Title: %s\n\n" "$$(basename $@ .8)" > $^.tmp
 	@cat $^ >> $^.tmp
-	$(trace-gen) lowdown -sT man $^.tmp -o $@
+	$(trace-gen) lowdown -sT man -M section=8 $^.tmp -o $@
 	@rm $^.tmp
 
 $(d)/nix.conf.5: $(d)/src/command-ref/conf-file.md
 	@printf "Title: %s\n\n" "$$(basename $@ .5)" > $^.tmp
 	@cat $^ >> $^.tmp
-	$(trace-gen) lowdown -sT man $^.tmp -o $@
+	$(trace-gen) lowdown -sT man -M section=5 $^.tmp -o $@
 	@rm $^.tmp
 
 $(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/command-ref/new-cli
@@ -80,7 +80,7 @@ install: $(d)/src/command-ref/new-cli
 	  if [[ $$name = SUMMARY ]]; then continue; fi; \
 	  printf "Title: %s\n\n" "$$name" > $$i.tmp; \
 	  cat $$i >> $$i.tmp; \
-	  lowdown -sT man $$i.tmp -o $(mandir)/man1/$$name.1; \
+	  lowdown -sT man -M section=1 $$i.tmp -o $(mandir)/man1/$$name.1; \
 	done
 
 $(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/command-ref/conf-file.md $(d)/src/expressions/builtins.md

doc/manual/src/advanced-topics/cores-vs-jobs.md

@@ -4,13 +4,13 @@ Nix has two relevant settings with regards to how your CPU cores will
 be utilized: `cores` and `max-jobs`. This chapter will talk about what
 they are, how they interact, and their configuration trade-offs.
 
-  - `max-jobs`  
+  - `max-jobs`\
     Dictates how many separate derivations will be built at the same
     time. If you set this to zero, the local machine will do no
     builds.  Nix will still substitute from binary caches, and build
     remotely if remote builders are configured.
 
-  - `cores`  
+  - `cores`\
     Suggests how many cores each derivation should use. Similar to
     `make -j`.
 

doc/manual/src/advanced-topics/distributed-builds.md

@@ -37,7 +37,7 @@ then you need to ensure that the `PATH` of non-interactive login shells
 contains Nix.
 
 > **Warning**
-> 
+>
 > If you are building via the Nix daemon, it is the Nix daemon user
 > account (that is, `root`) that should have SSH access to the remote
 > machine. If you can’t or don’t want to configure `root` to be able to
@@ -52,7 +52,7 @@ example, the following command allows you to build a derivation for
 ```console
 $ uname
 Linux
-    
+
 $ nix build \
   '(with import <nixpkgs> { system = "x86_64-darwin"; }; runCommand "foo" {} "uname > $out")' \
   --builders 'ssh://mac x86_64-darwin'
@@ -103,7 +103,7 @@ default, set it to `-`.
     ```nix
     requiredSystemFeatures = [ "kvm" ];
     ```
-    
+
     will cause the build to be performed on a machine that has the `kvm`
     feature.
 
@@ -112,6 +112,10 @@ default, set it to `-`.
     features appear in the derivation’s `requiredSystemFeatures`
     attribute..
 
+8.  The (base64-encoded) public host key of the remote machine. If omitted, SSH
+    will use its regular known-hosts file. Specifically, the field is calculated
+    via `base64 -w0 /etc/ssh/ssh_host_ed25519_key.pub`.
+
 For example, the machine specification
 
     nix@scratchy.labs.cs.uu.nl  i686-linux      /home/nix/.ssh/id_scratchy_auto        8 1 kvm

doc/manual/src/command-ref/env-common.md

@@ -2,45 +2,49 @@
 
 Most Nix commands interpret the following environment variables:
 
-  - `IN_NIX_SHELL`  
+  - `IN_NIX_SHELL`\
     Indicator that tells if the current environment was set up by
     `nix-shell`. Since Nix 2.0 the values are `"pure"` and `"impure"`
 
-  - `NIX_PATH`  
+  - `NIX_PATH`\
     A colon-separated list of directories used to look up Nix
     expressions enclosed in angle brackets (i.e., `<path>`). For
     instance, the value
-    
+
         /home/eelco/Dev:/etc/nixos
-    
+
     will cause Nix to look for paths relative to `/home/eelco/Dev` and
     `/etc/nixos`, in this order. It is also possible to match paths
     against a prefix. For example, the value
-    
+
         nixpkgs=/home/eelco/Dev/nixpkgs-branch:/etc/nixos
-    
+
     will cause Nix to search for `<nixpkgs/path>` in
     `/home/eelco/Dev/nixpkgs-branch/path` and `/etc/nixos/nixpkgs/path`.
-    
+
     If a path in the Nix search path starts with `http://` or
     `https://`, it is interpreted as the URL of a tarball that will be
     downloaded and unpacked to a temporary location. The tarball must
     consist of a single top-level directory. For example, setting
     `NIX_PATH` to
-    
-        nixpkgs=https://github.com/NixOS/nixpkgs/archive/nixos-15.09.tar.gz
-    
-    tells Nix to download the latest revision in the Nixpkgs/NixOS 15.09
-    channel.
-    
-    A following shorthand can be used to refer to the official channels:
-    
-        nixpkgs=channel:nixos-15.09
-    
-    The search path can be extended using the `-I` option, which takes
-    precedence over `NIX_PATH`.
 
-  - `NIX_IGNORE_SYMLINK_STORE`  
+        nixpkgs=https://github.com/NixOS/nixpkgs/archive/master.tar.gz
+
+    tells Nix to download and use the current contents of the
+    `master` branch in the `nixpkgs` repository.
+
+    The URLs of the tarballs from the official nixos.org channels (see
+    [the manual for `nix-channel`](nix-channel.md)) can be abbreviated
+    as `channel:<channel-name>`.  For instance, the following two
+    values of `NIX_PATH` are equivalent:
+
+        nixpkgs=channel:nixos-21.05
+        nixpkgs=https://nixos.org/channels/nixos-21.05/nixexprs.tar.xz
+
+    The Nix search path can also be extended using the `-I` option to
+    many Nix commands, which takes precedence over `NIX_PATH`.
+
+  - `NIX_IGNORE_SYMLINK_STORE`\
     Normally, the Nix store directory (typically `/nix/store`) is not
     allowed to contain any symlink components. This is to prevent
     “impure” builds. Builders sometimes “canonicalise” paths by
@@ -50,7 +54,7 @@ Most Nix commands interpret the following environment variables:
     builds are deployed to machines where `/nix/store` resolves
     differently. If you are sure that you’re not going to do that, you
     can set `NIX_IGNORE_SYMLINK_STORE` to `1`.
-    
+
     Note that if you’re symlinking the Nix store so that you can put it
     on another file system than the root file system, on Linux you’re
     better off using `bind` mount points, e.g.,
@@ -59,44 +63,44 @@ Most Nix commands interpret the following environment variables:
     $ mkdir /nix
     $ mount -o bind /mnt/otherdisk/nix /nix
     ```
-    
+
     Consult the mount 8 manual page for details.
 
-  - `NIX_STORE_DIR`  
+  - `NIX_STORE_DIR`\
     Overrides the location of the Nix store (default `prefix/store`).
 
-  - `NIX_DATA_DIR`  
+  - `NIX_DATA_DIR`\
     Overrides the location of the Nix static data directory (default
     `prefix/share`).
 
-  - `NIX_LOG_DIR`  
+  - `NIX_LOG_DIR`\
     Overrides the location of the Nix log directory (default
     `prefix/var/log/nix`).
 
-  - `NIX_STATE_DIR`  
+  - `NIX_STATE_DIR`\
     Overrides the location of the Nix state directory (default
     `prefix/var/nix`).
 
-  - `NIX_CONF_DIR`  
+  - `NIX_CONF_DIR`\
     Overrides the location of the system Nix configuration directory
     (default `prefix/etc/nix`).
 
-  - `NIX_CONFIG`  
+  - `NIX_CONFIG`\
     Applies settings from Nix configuration from the environment.
     The content is treated as if it was read from a Nix configuration file.
     Settings are separated by the newline character.
 
-  - `NIX_USER_CONF_FILES`  
+  - `NIX_USER_CONF_FILES`\
     Overrides the location of the user Nix configuration files to load
     from (defaults to the XDG spec locations). The variable is treated
     as a list separated by the `:` token.
 
-  - `TMPDIR`  
+  - `TMPDIR`\
     Use the specified directory to store temporary files. In particular,
     this includes temporary build directories; these can take up
     substantial amounts of disk space. The default is `/tmp`.
 
-  - `NIX_REMOTE`  
+  - `NIX_REMOTE`\
     This variable should be set to `daemon` if you want to use the Nix
     daemon to execute Nix operations. This is necessary in [multi-user
     Nix installations](../installation/multi-user.md). If the Nix
@@ -104,16 +108,16 @@ Most Nix commands interpret the following environment variables:
     should be set to `unix://path/to/socket`. Otherwise, it should be
     left unset.
 
-  - `NIX_SHOW_STATS`  
+  - `NIX_SHOW_STATS`\
     If set to `1`, Nix will print some evaluation statistics, such as
     the number of values allocated.
 
-  - `NIX_COUNT_CALLS`  
+  - `NIX_COUNT_CALLS`\
     If set to `1`, Nix will print how often functions were called during
     Nix expression evaluation. This is useful for profiling your Nix
     expressions.
 
-  - `GC_INITIAL_HEAP_SIZE`  
+  - `GC_INITIAL_HEAP_SIZE`\
     If Nix has been configured to use the Boehm garbage collector, this
     variable sets the initial size of the heap in bytes. It defaults to
     384 MiB. Setting it to a low value reduces memory consumption, but

doc/manual/src/command-ref/nix-build.md

@@ -47,16 +47,16 @@ All options not listed here are passed to `nix-store
 --realise`, except for `--arg` and `--attr` / `-A` which are passed to
 `nix-instantiate`.
 
-  - `--no-out-link`  
+  - `--no-out-link`\
     Do not create a symlink to the output path. Note that as a result
     the output does not become a root of the garbage collector, and so
     might be deleted by `nix-store
                     --gc`.
 
-  - `--dry-run`  
+  - `--dry-run`\
     Show what store paths would be built or downloaded.
 
-  - `--out-link` / `-o` *outlink*  
+  - `--out-link` / `-o` *outlink*\
     Change the name of the symlink to the output path created from
     `result` to *outlink*.
 

doc/manual/src/command-ref/nix-channel.md

@@ -17,26 +17,26 @@ To see the list of official NixOS channels, visit
 
 This command has the following operations:
 
-  - `--add` *url* \[*name*\]  
+  - `--add` *url* \[*name*\]\
     Adds a channel named *name* with URL *url* to the list of subscribed
     channels. If *name* is omitted, it defaults to the last component of
     *url*, with the suffixes `-stable` or `-unstable` removed.
 
-  - `--remove` *name*  
+  - `--remove` *name*\
     Removes the channel named *name* from the list of subscribed
     channels.
 
-  - `--list`  
+  - `--list`\
     Prints the names and URLs of all subscribed channels on standard
     output.
 
-  - `--update` \[*names*…\]  
+  - `--update` \[*names*…\]\
     Downloads the Nix expressions of all subscribed channels (or only
     those included in *names* if specified) and makes them the default
     for `nix-env` operations (by symlinking them from the directory
     `~/.nix-defexpr`).
 
-  - `--rollback` \[*generation*\]  
+  - `--rollback` \[*generation*\]\
     Reverts the previous call to `nix-channel
                     --update`. Optionally, you can specify a specific channel generation
     number to restore.
@@ -70,14 +70,14 @@ $ nix-instantiate --eval -E '(import <nixpkgs> {}).lib.version'
 
 # Files
 
-  - `/nix/var/nix/profiles/per-user/username/channels`  
+  - `/nix/var/nix/profiles/per-user/username/channels`\
     `nix-channel` uses a `nix-env` profile to keep track of previous
     versions of the subscribed channels. Every time you run `nix-channel
     --update`, a new channel generation (that is, a symlink to the
     channel Nix expressions in the Nix store) is created. This enables
     `nix-channel --rollback` to revert to previous versions.
 
-  - `~/.nix-defexpr/channels`  
+  - `~/.nix-defexpr/channels`\
     This is a symlink to
     `/nix/var/nix/profiles/per-user/username/channels`. It ensures that
     `nix-env` can find your channels. In a multi-user installation, you
@@ -89,7 +89,7 @@ $ nix-instantiate --eval -E '(import <nixpkgs> {}).lib.version'
 A channel URL should point to a directory containing the following
 files:
 
-  - `nixexprs.tar.xz`  
+  - `nixexprs.tar.xz`\
     A tarball containing Nix expressions and files referenced by them
     (such as build scripts and patches). At the top level, the tarball
     should contain a single directory. That directory must contain a

doc/manual/src/command-ref/nix-copy-closure.md

@@ -35,21 +35,21 @@ and second to send the dump of those paths.  If this bothers you, use
 
 # Options
 
-  - `--to`  
+  - `--to`\
     Copy the closure of _paths_ from the local Nix store to the Nix
     store on _machine_. This is the default.
 
-  - `--from`  
+  - `--from`\
     Copy the closure of _paths_ from the Nix store on _machine_ to the
     local Nix store.
 
-  - `--gzip`  
+  - `--gzip`\
     Enable compression of the SSH connection.
 
-  - `--include-outputs`  
+  - `--include-outputs`\
     Also copy the outputs of store derivations included in the closure.
 
-  - `--use-substitutes` / `-s`  
+  - `--use-substitutes` / `-s`\
     Attempt to download missing paths on the target machine using Nix’s
     substitute mechanism.  Any paths that cannot be substituted on the
     target are still copied normally from the source.  This is useful,
@@ -58,12 +58,12 @@ and second to send the dump of those paths.  If this bothers you, use
     `nixos.org` (the default binary cache server) is
     fast.
 
-  - `-v`  
+  - `-v`\
     Show verbose output.
 
 # Environment variables
 
-  - `NIX_SSHOPTS`  
+  - `NIX_SSHOPTS`\
     Additional options to be passed to `ssh` on the command
     line.
 

doc/manual/src/command-ref/nix-env.md

@@ -36,27 +36,27 @@ case-sensitive. The regular expression can optionally be followed by a
 dash and a version number; if omitted, any version of the package will
 match. Here are some examples:
 
-  - `firefox`  
+  - `firefox`\
     Matches the package name `firefox` and any version.
 
-  - `firefox-32.0`  
+  - `firefox-32.0`\
     Matches the package name `firefox` and version `32.0`.
 
-  - `gtk\\+`  
+  - `gtk\\+`\
     Matches the package name `gtk+`. The `+` character must be escaped
     using a backslash to prevent it from being interpreted as a
     quantifier, and the backslash must be escaped in turn with another
     backslash to ensure that the shell passes it on.
 
-  - `.\*`  
+  - `.\*`\
     Matches any package name. This is the default for most commands.
 
-  - `'.*zip.*'`  
+  - `'.*zip.*'`\
     Matches any package name containing the string `zip`. Note the dots:
     `'*zip*'` does not work, because in a regular expression, the
     character `*` is interpreted as a quantifier.
 
-  - `'.*(firefox|chromium).*'`  
+  - `'.*(firefox|chromium).*'`\
     Matches any package name containing the strings `firefox` or
     `chromium`.
 
@@ -66,7 +66,7 @@ This section lists the options that are common to all operations. These
 options are allowed for every subcommand, though they may not always
 have an effect.
 
-  - `--file` / `-f` *path*  
+  - `--file` / `-f` *path*\
     Specifies the Nix expression (designated below as the *active Nix
     expression*) used by the `--install`, `--upgrade`, and `--query
     --available` operations to obtain derivations. The default is
@@ -77,13 +77,13 @@ have an effect.
     unpacked to a temporary location. The tarball must include a single
     top-level directory containing at least a file named `default.nix`.
 
-  - `--profile` / `-p` *path*  
+  - `--profile` / `-p` *path*\
     Specifies the profile to be used by those operations that operate on
     a profile (designated below as the *active profile*). A profile is a
     sequence of user environments called *generations*, one of which is
     the *current generation*.
 
-  - `--dry-run`  
+  - `--dry-run`\
     For the `--install`, `--upgrade`, `--uninstall`,
     `--switch-generation`, `--delete-generations` and `--rollback`
     operations, this flag will cause `nix-env` to print what *would* be
@@ -93,7 +93,7 @@ have an effect.
     [substituted](../glossary.md) (i.e., downloaded) and which paths
     will be built from source (because no substitute is available).
 
-  - `--system-filter` *system*  
+  - `--system-filter` *system*\
     By default, operations such as `--query
                     --available` show derivations matching any platform. This option
     allows you to use derivations for the specified platform *system*.
@@ -102,7 +102,7 @@ have an effect.
 
 # Files
 
-  - `~/.nix-defexpr`  
+  - `~/.nix-defexpr`\
     The source for the default Nix expressions used by the
     `--install`, `--upgrade`, and `--query --available` operations to
     obtain derivations. The `--file` option may be used to override
@@ -140,7 +140,7 @@ have an effect.
     The command `nix-channel` places symlinks to the downloaded Nix
     expressions from each subscribed channel in this directory.
 
-  - `~/.nix-profile`  
+  - `~/.nix-profile`\
     A symbolic link to the user's current profile. By default, this
     symlink points to `prefix/var/nix/profiles/default`. The `PATH`
     environment variable should include `~/.nix-profile/bin` for the
@@ -217,13 +217,13 @@ a number of possible ways:
 
 ## Flags
 
-  - `--prebuilt-only` / `-b`  
+  - `--prebuilt-only` / `-b`\
     Use only derivations for which a substitute is registered, i.e.,
     there is a pre-built binary available that can be downloaded in lieu
     of building the derivation. Thus, no packages will be built from
     source.
 
-  - `--preserve-installed`; `-P`  
+  - `--preserve-installed`; `-P`\
     Do not remove derivations with a name matching one of the
     derivations being installed. Usually, trying to have two versions of
     the same package installed in the same generation of a profile will
@@ -231,7 +231,7 @@ a number of possible ways:
     clashes between the two versions. However, this is not the case for
     all packages.
 
-  - `--remove-all`; `-r`  
+  - `--remove-all`; `-r`\
     Remove all previously installed packages first. This is equivalent
     to running `nix-env -e '.*'` first, except that everything happens
     in a single transaction.
@@ -346,24 +346,24 @@ version is installed.
 
 ## Flags
 
-  - `--lt`  
+  - `--lt`\
     Only upgrade a derivation to newer versions. This is the default.
 
-  - `--leq`  
+  - `--leq`\
     In addition to upgrading to newer versions, also “upgrade” to
     derivations that have the same version. Version are not a unique
     identification of a derivation, so there may be many derivations
     that have the same version. This flag may be useful to force
     “synchronisation” between the installed and available derivations.
 
-  - `--eq`  
+  - `--eq`\
     *Only* “upgrade” to derivations that have the same version. This may
     not seem very useful, but it actually is, e.g., when there is a new
     release of Nixpkgs and you want to replace installed applications
     with the same versions built against newer dependencies (to reduce
     the number of dependencies floating around on your system).
 
-  - `--always`  
+  - `--always`\
     In addition to upgrading to newer versions, also “upgrade” to
     derivations that have the same or a lower version. I.e., derivations
     may actually be downgraded depending on what is available in the
@@ -578,11 +578,11 @@ The derivations are sorted by their `name` attributes.
 The following flags specify the set of things on which the query
 operates.
 
-  - `--installed`  
+  - `--installed`\
     The query operates on the store paths that are installed in the
     current generation of the active profile. This is the default.
 
-  - `--available`; `-a`  
+  - `--available`; `-a`\
     The query operates on the derivations that are available in the
     active Nix expression.
 
@@ -593,24 +593,24 @@ selected derivations. Multiple flags may be specified, in which case the
 information is shown in the order given here. Note that the name of the
 derivation is shown unless `--no-name` is specified.
 
-  - `--xml`  
+  - `--xml`\
     Print the result in an XML representation suitable for automatic
     processing by other tools. The root element is called `items`, which
     contains a `item` element for each available or installed
     derivation. The fields discussed below are all stored in attributes
     of the `item` elements.
 
-  - `--json`  
+  - `--json`\
     Print the result in a JSON representation suitable for automatic
     processing by other tools.
 
-  - `--prebuilt-only` / `-b`  
+  - `--prebuilt-only` / `-b`\
     Show only derivations for which a substitute is registered, i.e.,
     there is a pre-built binary available that can be downloaded in lieu
     of building the derivation. Thus, this shows all packages that
     probably can be installed quickly.
 
-  - `--status`; `-s`  
+  - `--status`; `-s`\
     Print the *status* of the derivation. The status consists of three
     characters. The first is `I` or `-`, indicating whether the
     derivation is currently installed in the current generation of the
@@ -621,49 +621,49 @@ derivation is shown unless `--no-name` is specified.
     derivation to be built. The third is `S` or `-`, indicating whether
     a substitute is available for the derivation.
 
-  - `--attr-path`; `-P`  
+  - `--attr-path`; `-P`\
     Print the *attribute path* of the derivation, which can be used to
     unambiguously select it using the `--attr` option available in
     commands that install derivations like `nix-env --install`. This
     option only works together with `--available`
 
-  - `--no-name`  
+  - `--no-name`\
     Suppress printing of the `name` attribute of each derivation.
 
-  - `--compare-versions` / `-c`  
+  - `--compare-versions` / `-c`\
     Compare installed versions to available versions, or vice versa (if
     `--available` is given). This is useful for quickly seeing whether
     upgrades for installed packages are available in a Nix expression. A
     column is added with the following meaning:
 
-      - `<` *version*  
+      - `<` *version*\
         A newer version of the package is available or installed.
 
-      - `=` *version*  
+      - `=` *version*\
         At most the same version of the package is available or
         installed.
 
-      - `>` *version*  
+      - `>` *version*\
         Only older versions of the package are available or installed.
 
-      - `- ?`  
+      - `- ?`\
         No version of the package is available or installed.
 
-  - `--system`  
+  - `--system`\
     Print the `system` attribute of the derivation.
 
-  - `--drv-path`  
+  - `--drv-path`\
     Print the path of the store derivation.
 
-  - `--out-path`  
+  - `--out-path`\
     Print the output path of the derivation.
 
-  - `--description`  
+  - `--description`\
     Print a short (one-line) description of the derivation, if
     available. The description is taken from the `meta.description`
     attribute of the derivation.
 
-  - `--meta`  
+  - `--meta`\
     Print all of the meta-attributes of the derivation. This option is
     only available with `--xml` or `--json`.
 
@@ -874,7 +874,7 @@ error: no generation older than the current (91) exists
 
 # Environment variables
 
-  - `NIX_PROFILE`  
+  - `NIX_PROFILE`\
     Location of the Nix profile. Defaults to the target of the symlink
     `~/.nix-profile`, if it exists, or `/nix/var/nix/profiles/default`
     otherwise.

doc/manual/src/command-ref/nix-hash.md

@@ -29,29 +29,29 @@ md5sum`.
 
 # Options
 
-  - `--flat`  
+  - `--flat`\
     Print the cryptographic hash of the contents of each regular file
     *path*. That is, do not compute the hash over the dump of *path*.
     The result is identical to that produced by the GNU commands
     `md5sum` and `sha1sum`.
 
-  - `--base32`  
+  - `--base32`\
     Print the hash in a base-32 representation rather than hexadecimal.
     This base-32 representation is more compact and can be used in Nix
     expressions (such as in calls to `fetchurl`).
 
-  - `--truncate`  
+  - `--truncate`\
     Truncate hashes longer than 160 bits (such as SHA-256) to 160 bits.
 
-  - `--type` *hashAlgo*  
+  - `--type` *hashAlgo*\
     Use the specified cryptographic hash algorithm, which can be one of
     `md5`, `sha1`, `sha256`, and `sha512`.
 
-  - `--to-base16`  
+  - `--to-base16`\
     Don’t hash anything, but convert the base-32 hash representation
     *hash* to hexadecimal.
 
-  - `--to-base32`  
+  - `--to-base32`\
     Don’t hash anything, but convert the hexadecimal hash representation
     *hash* to base-32.
 

doc/manual/src/command-ref/nix-instantiate.md

@@ -29,26 +29,26 @@ standard input.
 
 # Options
 
-  - `--add-root` *path*  
+  - `--add-root` *path*\
     See the [corresponding option](nix-store.md) in `nix-store`.
 
-  - `--parse`  
+  - `--parse`\
     Just parse the input files, and print their abstract syntax trees on
     standard output in ATerm format.
 
-  - `--eval`  
+  - `--eval`\
     Just parse and evaluate the input files, and print the resulting
     values on standard output. No instantiation of store derivations
     takes place.
 
-  - `--find-file`  
+  - `--find-file`\
     Look up the given files in Nix’s search path (as specified by the
     `NIX_PATH` environment variable). If found, print the corresponding
     absolute paths on standard output. For instance, if `NIX_PATH` is
     `nixpkgs=/home/alice/nixpkgs`, then `nix-instantiate --find-file
     nixpkgs/default.nix` will print `/home/alice/nixpkgs/default.nix`.
 
-  - `--strict`  
+  - `--strict`\
     When used with `--eval`, recursively evaluate list elements and
     attributes. Normally, such sub-expressions are left unevaluated
     (since the Nix expression language is lazy).
@@ -58,17 +58,17 @@ standard input.
     > This option can cause non-termination, because lazy data
     > structures can be infinitely large.
 
-  - `--json`  
+  - `--json`\
     When used with `--eval`, print the resulting value as an JSON
     representation of the abstract syntax tree rather than as an ATerm.
 
-  - `--xml`  
+  - `--xml`\
     When used with `--eval`, print the resulting value as an XML
     representation of the abstract syntax tree rather than as an ATerm.
     The schema is the same as that used by the [`toXML`
     built-in](../expressions/builtins.md).
 
-  - `--read-write-mode`  
+  - `--read-write-mode`\
     When used with `--eval`, perform evaluation in read/write mode so
     nix language features that require it will still work (at the cost
     of needing to do instantiation of every evaluated derivation). If

doc/manual/src/command-ref/nix-prefetch-url.md

@@ -37,22 +37,22 @@ Nix store is also printed.
 
 # Options
 
-  - `--type` *hashAlgo*  
+  - `--type` *hashAlgo*\
     Use the specified cryptographic hash algorithm, which can be one of
     `md5`, `sha1`, `sha256`, and `sha512`.
 
-  - `--print-path`  
+  - `--print-path`\
     Print the store path of the downloaded file on standard output.
 
-  - `--unpack`  
+  - `--unpack`\
     Unpack the archive (which must be a tarball or zip file) and add the
     result to the Nix store. The resulting hash can be used with
     functions such as Nixpkgs’s `fetchzip` or `fetchFromGitHub`.
 
-  - `--executable`  
+  - `--executable`\
     Set the executable bit on the downloaded file.
 
-  - `--name` *name*  
+  - `--name` *name*\
     Override the name of the file in the Nix store. By default, this is
     `hash-basename`, where *basename* is the last component of *url*.
     Overriding the name is necessary when *basename* contains characters

doc/manual/src/command-ref/nix-shell.md

@@ -54,7 +54,7 @@ All options not listed here are passed to `nix-store
 --realise`, except for `--arg` and `--attr` / `-A` which are passed to
 `nix-instantiate`.
 
-  - `--command` *cmd*  
+  - `--command` *cmd*\
     In the environment of the derivation, run the shell command *cmd*.
     This command is executed in an interactive shell. (Use `--run` to
     use a non-interactive shell instead.) However, a call to `exit` is
@@ -64,36 +64,34 @@ All options not listed here are passed to `nix-store
     drop you into the interactive shell. This can be useful for doing
     any additional initialisation.
 
-  - `--run` *cmd*  
+  - `--run` *cmd*\
     Like `--command`, but executes the command in a non-interactive
     shell. This means (among other things) that if you hit Ctrl-C while
     the command is running, the shell exits.
 
-  - `--exclude` *regexp*  
+  - `--exclude` *regexp*\
     Do not build any dependencies whose store path matches the regular
     expression *regexp*. This option may be specified multiple times.
 
-  - `--pure`  
+  - `--pure`\
     If this flag is specified, the environment is almost entirely
     cleared before the interactive shell is started, so you get an
     environment that more closely corresponds to the “real” Nix build. A
     few variables, in particular `HOME`, `USER` and `DISPLAY`, are
-    retained. Note that (depending on your Bash
-    installation) `/etc/bashrc` is still sourced, so any variables set
-    there will affect the interactive shell.
+    retained.
 
-  - `--packages` / `-p` *packages*…  
+  - `--packages` / `-p` *packages*…\
     Set up an environment in which the specified packages are present.
     The command line arguments are interpreted as attribute names inside
     the Nix Packages collection. Thus, `nix-shell -p libjpeg openjdk`
     will start a shell in which the packages denoted by the attribute
     names `libjpeg` and `openjdk` are present.
 
-  - `-i` *interpreter*  
+  - `-i` *interpreter*\
     The chained script interpreter to be invoked by `nix-shell`. Only
     applicable in `#!`-scripts (described below).
 
-  - `--keep` *name*  
+  - `--keep` *name*\
     When a `--pure` shell is started, keep the listed environment
     variables.
 
@@ -101,7 +99,7 @@ The following common options are supported:
 
 # Environment variables
 
-  - `NIX_BUILD_SHELL`  
+  - `NIX_BUILD_SHELL`\
     Shell used to start the interactive environment. Defaults to the
     `bash` found in `PATH`.
 
@@ -232,22 +230,23 @@ terraform apply
 > in a nix-shell shebang.
 
 Finally, using the merging of multiple nix-shell shebangs the following
-Haskell script uses a specific branch of Nixpkgs/NixOS (the 18.03 stable
+Haskell script uses a specific branch of Nixpkgs/NixOS (the 20.03 stable
 branch):
 
 ```haskell
 #! /usr/bin/env nix-shell
-#! nix-shell -i runghc -p "haskellPackages.ghcWithPackages (ps: [ps.HTTP ps.tagsoup])"
-#! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/nixos-18.03.tar.gz
+#! nix-shell -i runghc -p "haskellPackages.ghcWithPackages (ps: [ps.download-curl ps.tagsoup])"
+#! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/nixos-20.03.tar.gz
 
-import Network.HTTP
+import Network.Curl.Download
 import Text.HTML.TagSoup
+import Data.Either
+import Data.ByteString.Char8 (unpack)
 
 -- Fetch nixos.org and print all hrefs.
 main = do
-  resp <- Network.HTTP.simpleHTTP (getRequest "http://nixos.org/")
-  body <- getResponseBody resp
-  let tags = filter (isTagOpenName "a") $ parseTags body
+  resp <- openURI "https://nixos.org/"
+  let tags = filter (isTagOpenName "a") $ parseTags $ unpack $ fromRight undefined resp
   let tags' = map (fromAttrib "href") tags
   mapM_ putStrLn $ filter (/= "") tags'
 ```

doc/manual/src/command-ref/nix-store.md

@@ -22,7 +22,7 @@ This section lists the options that are common to all operations. These
 options are allowed for every subcommand, though they may not always
 have an effect.
 
-  - `--add-root` *path*  
+  - `--add-root` *path*\
     Causes the result of a realisation (`--realise` and
     `--force-realise`) to be registered as a root of the garbage
     collector. *path* will be created as a symlink to the resulting
@@ -79,22 +79,22 @@ paths. Realisation is a somewhat overloaded term:
     system). If the path is already valid, we are done immediately.
     Otherwise, the path and any missing paths in its closure may be
     produced through substitutes. If there are no (successful)
-    subsitutes, realisation fails.
+    substitutes, realisation fails.
 
 The output path of each derivation is printed on standard output. (For
 non-derivations argument, the argument itself is printed.)
 
 The following flags are available:
 
-  - `--dry-run`  
+  - `--dry-run`\
     Print on standard error a description of what packages would be
     built or downloaded, without actually performing the operation.
 
-  - `--ignore-unknown`  
+  - `--ignore-unknown`\
     If a non-derivation path does not have a substitute, then silently
     ignore it.
 
-  - `--check`  
+  - `--check`\
     This option allows you to check whether a derivation is
     deterministic. It rebuilds the specified derivation and checks
     whether the result is bitwise-identical with the existing outputs,
@@ -110,20 +110,20 @@ The following flags are available:
 
 Special exit codes:
 
-  - `100`  
+  - `100`\
     Generic build failure, the builder process returned with a non-zero
     exit code.
 
-  - `101`  
+  - `101`\
     Build timeout, the build was aborted because it did not complete
     within the specified `timeout`.
 
-  - `102`  
+  - `102`\
     Hash mismatch, the build output was rejected because it does not
     match the [`outputHash` attribute of the
     derivation](../expressions/advanced-attributes.md).
 
-  - `104`  
+  - `104`\
     Not deterministic, the build succeeded in check mode but the
     resulting output is not binary reproducable.
 
@@ -170,7 +170,7 @@ access to a restricted ssh user.
 
 The following flags are available:
 
-  - `--write`  
+  - `--write`\
     Allow the connected client to request the realization of
     derivations. In effect, this can be used to make the host act as a
     remote builder.
@@ -200,18 +200,18 @@ reachable via file system references from a set of “roots”, are deleted.
 
 The following suboperations may be specified:
 
-  - `--print-roots`  
+  - `--print-roots`\
     This operation prints on standard output the set of roots used by
     the garbage collector.
 
-  - `--print-live`  
+  - `--print-live`\
     This operation prints on standard output the set of “live” store
     paths, which are all the store paths reachable from the roots. Live
     paths should never be deleted, since that would break consistency —
     it would become possible that applications are installed that
     reference things that are no longer present in the store.
 
-  - `--print-dead`  
+  - `--print-dead`\
     This operation prints out on standard output the set of “dead” store
     paths, which is just the opposite of the set of live paths: any path
     in the store that is not live (with respect to the roots) is dead.
@@ -219,7 +219,7 @@ The following suboperations may be specified:
 By default, all unreachable paths are deleted. The following options
 control what gets deleted and in what order:
 
-  - `--max-freed` *bytes*  
+  - `--max-freed` *bytes*\
     Keep deleting paths until at least *bytes* bytes have been deleted,
     then stop. The argument *bytes* can be followed by the
     multiplicative suffix `K`, `M`, `G` or `T`, denoting KiB, MiB, GiB
@@ -300,22 +300,22 @@ symlink.
 
 ## Common query options
 
-  - `--use-output`; `-u`  
+  - `--use-output`; `-u`\
     For each argument to the query that is a store derivation, apply the
     query to the output path of the derivation instead.
 
-  - `--force-realise`; `-f`  
+  - `--force-realise`; `-f`\
     Realise each argument to the query first (see [`nix-store
     --realise`](#operation---realise)).
 
 ## Queries
 
-  - `--outputs`  
+  - `--outputs`\
     Prints out the [output paths](../glossary.md) of the store
     derivations *paths*. These are the paths that will be produced when
     the derivation is built.
 
-  - `--requisites`; `-R`  
+  - `--requisites`; `-R`\
     Prints out the [closure](../glossary.md) of the store path *paths*.
 
     This query has one option:
@@ -332,31 +332,31 @@ symlink.
     dependencies) is obtained by distributing the closure of a store
     derivation and specifying the option `--include-outputs`.
 
-  - `--references`  
+  - `--references`\
     Prints the set of [references](../glossary.md) of the store paths
     *paths*, that is, their immediate dependencies. (For *all*
     dependencies, use `--requisites`.)
 
-  - `--referrers`  
+  - `--referrers`\
     Prints the set of *referrers* of the store paths *paths*, that is,
     the store paths currently existing in the Nix store that refer to
     one of *paths*. Note that contrary to the references, the set of
     referrers is not constant; it can change as store paths are added or
     removed.
 
-  - `--referrers-closure`  
+  - `--referrers-closure`\
     Prints the closure of the set of store paths *paths* under the
     referrers relation; that is, all store paths that directly or
     indirectly refer to one of *paths*. These are all the path currently
     in the Nix store that are dependent on *paths*.
 
-  - `--deriver`; `-d`  
+  - `--deriver`; `-d`\
     Prints the [deriver](../glossary.md) of the store paths *paths*. If
     the path has no deriver (e.g., if it is a source file), or if the
     deriver is not known (e.g., in the case of a binary-only
     deployment), the string `unknown-deriver` is printed.
 
-  - `--graph`  
+  - `--graph`\
     Prints the references graph of the store paths *paths* in the format
     of the `dot` tool of AT\&T's [Graphviz
     package](http://www.graphviz.org/). This can be used to visualise
@@ -364,39 +364,39 @@ symlink.
     this to a store derivation. To obtain a runtime dependency graph,
     apply it to an output path.
 
-  - `--tree`  
+  - `--tree`\
     Prints the references graph of the store paths *paths* as a nested
     ASCII tree. References are ordered by descending closure size; this
     tends to flatten the tree, making it more readable. The query only
     recurses into a store path when it is first encountered; this
     prevents a blowup of the tree representation of the graph.
 
-  - `--graphml`  
+  - `--graphml`\
     Prints the references graph of the store paths *paths* in the
     [GraphML](http://graphml.graphdrawing.org/) file format. This can be
     used to visualise dependency graphs. To obtain a build-time
     dependency graph, apply this to a store derivation. To obtain a
     runtime dependency graph, apply it to an output path.
 
-  - `--binding` *name*; `-b` *name*  
+  - `--binding` *name*; `-b` *name*\
     Prints the value of the attribute *name* (i.e., environment
     variable) of the store derivations *paths*. It is an error for a
     derivation to not have the specified attribute.
 
-  - `--hash`  
+  - `--hash`\
     Prints the SHA-256 hash of the contents of the store paths *paths*
     (that is, the hash of the output of `nix-store --dump` on the given
     paths). Since the hash is stored in the Nix database, this is a fast
     operation.
 
-  - `--size`  
+  - `--size`\
     Prints the size in bytes of the contents of the store paths *paths*
     — to be precise, the size of the output of `nix-store --dump` on
     the given paths. Note that the actual disk space required by the
     store paths may be higher, especially on filesystems with large
     cluster sizes.
 
-  - `--roots`  
+  - `--roots`\
     Prints the garbage collector roots that point, directly or
     indirectly, at the store paths *paths*.
 
@@ -513,7 +513,7 @@ public url or broke since the download expression was written.
 
 This operation has the following options:
 
-  - `--recursive`  
+  - `--recursive`\
     Use recursive instead of flat hashing mode, used when adding
     directories to the store.
 
@@ -540,14 +540,14 @@ being modified by non-Nix tools, or of bugs in Nix itself.
 
 This operation has the following options:
 
-  - `--check-contents`  
+  - `--check-contents`\
     Checks that the contents of every valid store path has not been
     altered by computing a SHA-256 hash of the contents and comparing it
     with the hash stored in the Nix database at build time. Paths that
     have been modified are printed out. For large stores,
     `--check-contents` is obviously quite slow.
 
-  - `--repair`  
+  - `--repair`\
     If any valid path is missing from the store, or (if
     `--check-contents` is given) the contents of a valid path has been
     modified, then try to repair the path by redownloading it. See

doc/manual/src/command-ref/opt-common.md

@@ -2,56 +2,56 @@
 
 Most Nix commands accept the following command-line options:
 
-  - `--help`  
+  - `--help`\
     Prints out a summary of the command syntax and exits.
 
-  - `--version`  
+  - `--version`\
     Prints out the Nix version number on standard output and exits.
 
-  - `--verbose` / `-v`  
+  - `--verbose` / `-v`\
     Increases the level of verbosity of diagnostic messages printed on
     standard error. For each Nix operation, the information printed on
     standard output is well-defined; any diagnostic information is
     printed on standard error, never on standard output.
-    
+
     This option may be specified repeatedly. Currently, the following
     verbosity levels exist:
-    
-      - 0  
+
+      - 0\
         “Errors only”: only print messages explaining why the Nix
         invocation failed.
-    
-      - 1  
+
+      - 1\
         “Informational”: print *useful* messages about what Nix is
         doing. This is the default.
-    
-      - 2  
+
+      - 2\
         “Talkative”: print more informational messages.
-    
-      - 3  
+
+      - 3\
         “Chatty”: print even more informational messages.
-    
-      - 4  
+
+      - 4\
         “Debug”: print debug information.
-    
-      - 5  
+
+      - 5\
         “Vomit”: print vast amounts of debug information.
 
-  - `--quiet`  
+  - `--quiet`\
     Decreases the level of verbosity of diagnostic messages printed on
     standard error. This is the inverse option to `-v` / `--verbose`.
-    
+
     This option may be specified repeatedly. See the previous verbosity
     levels list.
 
-  - `--log-format` *format*  
+  - `--log-format` *format*\
     This option can be used to change the output of the log format, with
     *format* being one of:
-    
-      - raw  
+
+      - raw\
         This is the raw format, as outputted by nix-build.
-    
-      - internal-json  
+
+      - internal-json\
         Outputs the logs in a structured manner.
 
         > **Warning**
@@ -60,30 +60,30 @@ Most Nix commands accept the following command-line options:
         > the error-messages (namely of the `msg`-field) can change
         > between releases.
 
-      - bar  
+      - bar\
         Only display a progress bar during the builds.
-    
-      - bar-with-logs  
+
+      - bar-with-logs\
         Display the raw logs, with the progress bar at the bottom.
 
-  - `--no-build-output` / `-Q`  
+  - `--no-build-output` / `-Q`\
     By default, output written by builders to standard output and
     standard error is echoed to the Nix command's standard error. This
     option suppresses this behaviour. Note that the builder's standard
     output and error are always written to a log file in
     `prefix/nix/var/log/nix`.
 
-  - `--max-jobs` / `-j` *number*  
+  - `--max-jobs` / `-j` *number*\
     Sets the maximum number of build jobs that Nix will perform in
     parallel to the specified number. Specify `auto` to use the number
     of CPUs in the system. The default is specified by the `max-jobs`
     configuration setting, which itself defaults to `1`. A higher
     value is useful on SMP systems or to exploit I/O latency.
-    
+
     Setting it to `0` disallows building on the local machine, which is
     useful when you want builds to happen only on remote builders.
 
-  - `--cores`  
+  - `--cores`\
     Sets the value of the `NIX_BUILD_CORES` environment variable in
     the invocation of builders. Builders can use this variable at
     their discretion to control the maximum amount of parallelism. For
@@ -94,18 +94,18 @@ Most Nix commands accept the following command-line options:
     means that the builder should use all available CPU cores in the
     system.
 
-  - `--max-silent-time`  
+  - `--max-silent-time`\
     Sets the maximum number of seconds that a builder can go without
     producing any data on standard output or standard error. The
     default is specified by the `max-silent-time` configuration
     setting. `0` means no time-out.
 
-  - `--timeout`  
+  - `--timeout`\
     Sets the maximum number of seconds that a builder can run. The
     default is specified by the `timeout` configuration setting. `0`
     means no timeout.
 
-  - `--keep-going` / `-k`  
+  - `--keep-going` / `-k`\
     Keep going in case of failed builds, to the greatest extent
     possible. That is, if building an input of some derivation fails,
     Nix will still build the other inputs, but not the derivation
@@ -113,17 +113,17 @@ Most Nix commands accept the following command-line options:
     for builds of substitutes), possibly killing builds in progress (in
     case of parallel or distributed builds).
 
-  - `--keep-failed` / `-K`  
+  - `--keep-failed` / `-K`\
     Specifies that in case of a build failure, the temporary directory
     (usually in `/tmp`) in which the build takes place should not be
     deleted. The path of the build directory is printed as an
     informational message.
 
-  - `--fallback`  
+  - `--fallback`\
     Whenever Nix attempts to build a derivation for which substitutes
     are known for each output path, but realising the output paths
     through the substitutes fails, fall back on building the derivation.
-    
+
     The most common scenario in which this is useful is when we have
     registered substitutes in order to perform binary distribution from,
     say, a network repository. If the repository is down, the
@@ -134,21 +134,12 @@ Most Nix commands accept the following command-line options:
     failure in obtaining the substitutes to lead to a full build from
     source (with the related consumption of resources).
 
-  - `--no-build-hook`  
-    Disables the build hook mechanism. This allows to ignore remote
-    builders if they are setup on the machine.
-    
-    It's useful in cases where the bandwidth between the client and the
-    remote builder is too low. In that case it can take more time to
-    upload the sources to the remote builder and fetch back the result
-    than to do the computation locally.
-
-  - `--readonly-mode`  
+  - `--readonly-mode`\
     When this option is used, no attempt is made to open the Nix
     database. Most Nix operations do need database access, so those
     operations will fail.
 
-  - `--arg` *name* *value*  
+  - `--arg` *name* *value*\
     This option is accepted by `nix-env`, `nix-instantiate`,
     `nix-shell` and `nix-build`. When evaluating Nix expressions, the
     expression evaluator will automatically try to call functions that
@@ -160,7 +151,7 @@ Most Nix commands accept the following command-line options:
     override a default value). That is, if the evaluator encounters a
     function with an argument named *name*, it will call it with value
     *value*.
-    
+
     For instance, the top-level `default.nix` in Nixpkgs is actually a
     function:
 
@@ -170,7 +161,7 @@ Most Nix commands accept the following command-line options:
       ...
     }: ...
     ```
-    
+
     So if you call this Nix expression (e.g., when you do `nix-env -i
     pkgname`), the function will be called automatically using the
     value [`builtins.currentSystem`](../expressions/builtins.md) for
@@ -179,13 +170,13 @@ Most Nix commands accept the following command-line options:
     since the argument is a Nix string literal, you have to escape the
     quotes.)
 
-  - `--argstr` *name* *value*  
+  - `--argstr` *name* *value*\
     This option is like `--arg`, only the value is not a Nix
     expression but a string. So instead of `--arg system
     \"i686-linux\"` (the outer quotes are to keep the shell happy) you
     can say `--argstr system i686-linux`.
 
-  - `--attr` / `-A` *attrPath*  
+  - `--attr` / `-A` *attrPath*\
     Select an attribute from the top-level Nix expression being
     evaluated. (`nix-env`, `nix-instantiate`, `nix-build` and
     `nix-shell` only.) The *attribute path* *attrPath* is a sequence
@@ -194,34 +185,34 @@ Most Nix commands accept the following command-line options:
     would cause the expression `e.xorg.xorgserver` to be used. See
     [`nix-env --install`](nix-env.md#operation---install) for some
     concrete examples.
-    
+
     In addition to attribute names, you can also specify array indices.
     For instance, the attribute path `foo.3.bar` selects the `bar`
     attribute of the fourth element of the array in the `foo` attribute
     of the top-level expression.
 
-  - `--expr` / `-E`  
+  - `--expr` / `-E`\
     Interpret the command line arguments as a list of Nix expressions to
     be parsed and evaluated, rather than as a list of file names of Nix
     expressions. (`nix-instantiate`, `nix-build` and `nix-shell` only.)
-    
+
     For `nix-shell`, this option is commonly used to give you a shell in
     which you can build the packages returned by the expression. If you
     want to get a shell which contain the *built* packages ready for
     use, give your expression to the `nix-shell -p` convenience flag
     instead.
 
-  - `-I` *path*  
+  - `-I` *path*\
     Add a path to the Nix expression search path. This option may be
     given multiple times. See the `NIX_PATH` environment variable for
     information on the semantics of the Nix search path. Paths added
     through `-I` take precedence over `NIX_PATH`.
 
-  - `--option` *name* *value*  
+  - `--option` *name* *value*\
     Set the Nix configuration option *name* to *value*. This overrides
     settings in the Nix configuration file (see nix.conf5).
 
-  - `--repair`  
+  - `--repair`\
     Fix corrupted or missing store paths by redownloading or rebuilding
     them. Note that this is slow because it requires computing a
     cryptographic hash of the contents of every path in the closure of

doc/manual/src/expressions/advanced-attributes.md

@@ -2,14 +2,14 @@
 
 Derivations can declare some infrequently used optional attributes.
 
-  - `allowedReferences`  
+  - `allowedReferences`\
     The optional attribute `allowedReferences` specifies a list of legal
     references (dependencies) of the output of the builder. For example,
-    
+
     ```nix
     allowedReferences = [];
     ```
-    
+
     enforces that the output of a derivation cannot have any runtime
     dependencies on its inputs. To allow an output to have a runtime
     dependency on itself, use `"out"` as a list item. This is used in
@@ -17,45 +17,45 @@ Derivations can declare some infrequently used optional attributes.
     booting Linux don’t have accidental dependencies on other paths in
     the Nix store.
 
-  - `allowedRequisites`  
+  - `allowedRequisites`\
     This attribute is similar to `allowedReferences`, but it specifies
     the legal requisites of the whole closure, so all the dependencies
     recursively. For example,
-    
+
     ```nix
     allowedRequisites = [ foobar ];
     ```
-    
+
     enforces that the output of a derivation cannot have any other
     runtime dependency than `foobar`, and in addition it enforces that
     `foobar` itself doesn't introduce any other dependency itself.
 
-  - `disallowedReferences`  
+  - `disallowedReferences`\
     The optional attribute `disallowedReferences` specifies a list of
     illegal references (dependencies) of the output of the builder. For
     example,
-    
+
     ```nix
     disallowedReferences = [ foo ];
     ```
-    
+
     enforces that the output of a derivation cannot have a direct
     runtime dependencies on the derivation `foo`.
 
-  - `disallowedRequisites`  
+  - `disallowedRequisites`\
     This attribute is similar to `disallowedReferences`, but it
     specifies illegal requisites for the whole closure, so all the
     dependencies recursively. For example,
-    
+
     ```nix
     disallowedRequisites = [ foobar ];
     ```
-    
+
     enforces that the output of a derivation cannot have any runtime
     dependency on `foobar` or any other derivation depending recursively
     on `foobar`.
 
-  - `exportReferencesGraph`  
+  - `exportReferencesGraph`\
     This attribute allows builders access to the references graph of
     their inputs. The attribute is a list of inputs in the Nix store
     whose references graph the builder needs to know. The value of
@@ -65,17 +65,17 @@ Derivations can declare some infrequently used optional attributes.
     files have the format used by `nix-store --register-validity`
     (with the deriver fields left empty). For example, when the
     following derivation is built:
-    
+
     ```nix
     derivation {
       ...
       exportReferencesGraph = [ "libfoo-graph" libfoo ];
     };
     ```
-    
+
     the references graph of `libfoo` is placed in the file
     `libfoo-graph` in the temporary build directory.
-    
+
     `exportReferencesGraph` is useful for builders that want to do
     something with the closure of a store path. Examples include the
     builders in NixOS that generate the initial ramdisk for booting
@@ -84,66 +84,66 @@ Derivations can declare some infrequently used optional attributes.
     with a Nix store containing the closure of a bootable NixOS
     configuration).
 
-  - `impureEnvVars`  
+  - `impureEnvVars`\
     This attribute allows you to specify a list of environment variables
     that should be passed from the environment of the calling user to
     the builder. Usually, the environment is cleared completely when the
     builder is executed, but with this attribute you can allow specific
     environment variables to be passed unmodified. For example,
     `fetchurl` in Nixpkgs has the line
-    
+
     ```nix
     impureEnvVars = [ "http_proxy" "https_proxy" ... ];
     ```
-    
+
     to make it use the proxy server configuration specified by the user
     in the environment variables `http_proxy` and friends.
-    
+
     This attribute is only allowed in *fixed-output derivations* (see
     below), where impurities such as these are okay since (the hash
     of) the output is known in advance. It is ignored for all other
     derivations.
-    
+
     > **Warning**
-    > 
+    >
     > `impureEnvVars` implementation takes environment variables from
     > the current builder process. When a daemon is building its
     > environmental variables are used. Without the daemon, the
     > environmental variables come from the environment of the
     > `nix-build`.
 
-  - `outputHash`; `outputHashAlgo`; `outputHashMode`  
+  - `outputHash`; `outputHashAlgo`; `outputHashMode`\
     These attributes declare that the derivation is a so-called
     *fixed-output derivation*, which means that a cryptographic hash of
     the output is already known in advance. When the build of a
     fixed-output derivation finishes, Nix computes the cryptographic
     hash of the output and compares it to the hash declared with these
     attributes. If there is a mismatch, the build fails.
-    
+
     The rationale for fixed-output derivations is derivations such as
     those produced by the `fetchurl` function. This function downloads a
     file from a given URL. To ensure that the downloaded file has not
     been modified, the caller must also specify a cryptographic hash of
     the file. For example,
-    
+
     ```nix
     fetchurl {
       url = "http://ftp.gnu.org/pub/gnu/hello/hello-2.1.1.tar.gz";
       sha256 = "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465";
     }
     ```
-    
+
     It sometimes happens that the URL of the file changes, e.g., because
     servers are reorganised or no longer available. We then must update
     the call to `fetchurl`, e.g.,
-    
+
     ```nix
     fetchurl {
       url = "ftp://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz";
       sha256 = "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465";
     }
     ```
-    
+
     If a `fetchurl` derivation was treated like a normal derivation, the
     output paths of the derivation and *all derivations depending on it*
     would change. For instance, if we were to change the URL of the
@@ -151,16 +151,16 @@ Derivations can declare some infrequently used optional attributes.
     other packages depend) massive rebuilds would be needed. This is
     unfortunate for a change which we know cannot have a real effect as
     it propagates upwards through the dependency graph.
-    
+
     For fixed-output derivations, on the other hand, the name of the
     output path only depends on the `outputHash*` and `name` attributes,
     while all other attributes are ignored for the purpose of computing
     the output path. (The `name` attribute is included because it is
     part of the path.)
-    
+
     As an example, here is the (simplified) Nix expression for
     `fetchurl`:
-    
+
     ```nix
     { stdenv, curl }: # The curl program is used for downloading.
 
@@ -180,43 +180,51 @@ Derivations can declare some infrequently used optional attributes.
       inherit url;
     }
     ```
-    
+
     The `outputHashAlgo` attribute specifies the hash algorithm used to
     compute the hash. It can currently be `"sha1"`, `"sha256"` or
     `"sha512"`.
-    
+
     The `outputHashMode` attribute determines how the hash is computed.
     It must be one of the following two values:
-    
-      - `"flat"`  
+
+      - `"flat"`\
         The output must be a non-executable regular file. If it isn’t,
         the build fails. The hash is simply computed over the contents
         of that file (so it’s equal to what Unix commands like
         `sha256sum` or `sha1sum` produce).
-        
+
         This is the default.
-    
-      - `"recursive"`  
+
+      - `"recursive"`\
         The hash is computed over the NAR archive dump of the output
         (i.e., the result of [`nix-store
         --dump`](../command-ref/nix-store.md#operation---dump)). In
         this case, the output can be anything, including a directory
         tree.
-    
+
     The `outputHash` attribute, finally, must be a string containing
     the hash in either hexadecimal or base-32 notation. (See the
     [`nix-hash` command](../command-ref/nix-hash.md) for information
     about converting to and from base-32 notation.)
+    
+  - `__contentAddressed`
+    If this **experimental** attribute is set to true, then the derivation
+    outputs will be stored in a content-addressed location rather than the
+    traditional input-addressed one.
+    This only has an effect if the `ca-derivation` experimental feature is enabled.
+    
+    Setting this attribute also requires setting `outputHashMode` and `outputHashAlgo` like for *fixed-output derivations* (see above).
 
-  - `passAsFile`  
+  - `passAsFile`\
     A list of names of attributes that should be passed via files rather
     than environment variables. For example, if you have
-    
+
     ```nix
     passAsFile = ["big"];
     big = "a very long string";
     ```
-    
+
     then when the builder runs, the environment variable `bigPath`
     will contain the absolute path to a temporary file containing `a
     very long string`. That is, for any attribute *x* listed in
@@ -226,7 +234,7 @@ Derivations can declare some infrequently used optional attributes.
     builder, since most operating systems impose a limit on the size
     of the environment (typically, a few hundred kilobyte).
 
-  - `preferLocalBuild`  
+  - `preferLocalBuild`\
     If this attribute is set to `true` and [distributed building is
     enabled](../advanced-topics/distributed-builds.md), then, if
     possible, the derivaton will be built locally instead of forwarded
@@ -234,14 +242,14 @@ Derivations can declare some infrequently used optional attributes.
     where the cost of doing a download or remote build would exceed
     the cost of building locally.
 
-  - `allowSubstitutes`  
+  - `allowSubstitutes`\
     If this attribute is set to `false`, then Nix will always build this
     derivation; it will not try to substitute its outputs. This is
     useful for very trivial derivations (such as `writeText` in Nixpkgs)
     that are cheaper to build than to substitute from a binary cache.
-    
+
     > **Note**
-    > 
+    >
     > You need to have a builder configured which satisfies the
     > derivation’s `system` attribute, since the derivation cannot be
     > substituted. Thus it is usually a good idea to align `system` with

doc/manual/src/expressions/builtin-constants.md

@@ -2,7 +2,7 @@
 
 Here are the constants built into the Nix expression evaluator:
 
-  - `builtins`  
+  - `builtins`\
     The set `builtins` contains all the built-in functions and values.
     You can use `builtins` to test for the availability of features in
     the Nix installation, e.g.,
@@ -14,7 +14,7 @@ Here are the constants built into the Nix expression evaluator:
     This allows a Nix expression to fall back gracefully on older Nix
     installations that don’t have the desired built-in function.
 
-  - `builtins.currentSystem`  
+  - `builtins.currentSystem`\
     The built-in value `currentSystem` evaluates to the Nix platform
     identifier for the Nix installation on which the expression is being
     evaluated, such as `"i686-linux"` or `"x86_64-darwin"`.

doc/manual/src/expressions/builtins-prefix.md

@@ -9,7 +9,7 @@ scope. Instead, you can access them through the `builtins` built-in
 value, which is a set that contains all built-in functions and values.
 For instance, `derivation` is also available as `builtins.derivation`.
 
-  - `derivation` *attrs*; `builtins.derivation` *attrs*  
+  - `derivation` *attrs*; `builtins.derivation` *attrs*\
 
     `derivation` is described in [its own section](derivations.md).
 

doc/manual/src/glossary.md

@@ -1,48 +1,48 @@
 # Glossary
 
-  - derivation  
+  - derivation\
     A description of a build action. The result of a derivation is a
     store object. Derivations are typically specified in Nix expressions
     using the [`derivation` primitive](expressions/derivations.md). These are
     translated into low-level *store derivations* (implicitly by
     `nix-env` and `nix-build`, or explicitly by `nix-instantiate`).
 
-  - store  
+  - store\
     The location in the file system where store objects live. Typically
     `/nix/store`.
 
-  - store path  
+  - store path\
     The location in the file system of a store object, i.e., an
     immediate child of the Nix store directory.
 
-  - store object  
+  - store object\
     A file that is an immediate child of the Nix store directory. These
     can be regular files, but also entire directory trees. Store objects
     can be sources (objects copied from outside of the store),
     derivation outputs (objects produced by running a build action), or
     derivations (files describing a build action).
 
-  - substitute  
+  - substitute\
     A substitute is a command invocation stored in the Nix database that
     describes how to build a store object, bypassing the normal build
     mechanism (i.e., derivations). Typically, the substitute builds the
     store object by downloading a pre-built version of the store object
     from some server.
 
-  - purity  
+  - purity\
     The assumption that equal Nix derivations when run always produce
     the same output. This cannot be guaranteed in general (e.g., a
     builder can rely on external inputs such as the network or the
     system time) but the Nix model assumes it.
 
-  - Nix expression  
+  - Nix expression\
     A high-level description of software packages and compositions
     thereof. Deploying software using Nix entails writing Nix
     expressions for your packages. Nix expressions are translated to
     derivations that are stored in the Nix store. These derivations can
     then be built.
 
-  - reference  
+  - reference\
     A store path `P` is said to have a reference to a store path `Q` if
     the store object at `P` contains the path `Q` somewhere. The
     *references* of a store path are the set of store paths to which it
@@ -52,11 +52,11 @@
     output paths), whereas an output path only references other output
     paths.
 
-  - reachable  
+  - reachable\
     A store path `Q` is reachable from another store path `P` if `Q`
     is in the *closure* of the *references* relation.
 
-  - closure  
+  - closure\
     The closure of a store path is the set of store paths that are
     directly or indirectly “reachable” from that store path; that is,
     it’s the closure of the path under the *references* relation. For
@@ -71,29 +71,29 @@
     to path `Q`, then `Q` is in the closure of `P`. Further, if `Q`
     references `R` then `R` is also in the closure of `P`.
 
-  - output path  
+  - output path\
     A store path produced by a derivation.
 
-  - deriver  
+  - deriver\
     The deriver of an *output path* is the store
     derivation that built it.
 
-  - validity  
+  - validity\
     A store path is considered *valid* if it exists in the file system,
     is listed in the Nix database as being valid, and if all paths in
     its closure are also valid.
 
-  - user environment  
+  - user environment\
     An automatically generated store object that consists of a set of
     symlinks to “active” applications, i.e., other store paths. These
     are generated automatically by
     [`nix-env`](command-ref/nix-env.md). See *profiles*.
 
-  - profile  
+  - profile\
     A symlink to the current *user environment* of a user, e.g.,
     `/nix/var/nix/profiles/default`.
 
-  - NAR  
+  - NAR\
     A *N*ix *AR*chive. This is a serialisation of a path in the Nix
     store. It can contain regular files, directories and symbolic
     links.  NARs are generated and unpacked using `nix-store --dump`

doc/manual/src/installation/installing-binary.md

@@ -1,18 +1,26 @@
 # Installing a Binary Distribution
 
-If you are using Linux or macOS versions up to 10.14 (Mojave), the
-easiest way to install Nix is to run the following command:
+The easiest way to install Nix is to run the following command:
 
 ```console
 $ sh <(curl -L https://nixos.org/nix/install)
 ```
 
-If you're using macOS 10.15 (Catalina) or newer, consult [the macOS
-installation instructions](#macos-installation) before installing.
+This will run the installer interactively (causing it to explain what
+it is doing more explicitly), and perform the default "type" of install
+for your platform:
+- single-user on Linux
+- multi-user on macOS
 
-As of Nix 2.1.0, the Nix installer will always default to creating a
-single-user installation, however opting in to the multi-user
-installation is highly recommended.
+  > **Notes on read-only filesystem root in macOS 10.15 Catalina +**
+  > 
+  > - It took some time to support this cleanly. You may see posts,
+  >   examples, and tutorials using obsolete workarounds.
+  > - Supporting it cleanly made macOS installs too complex to qualify
+  >   as single-user, so this type is no longer supported on macOS.
+
+We recommend the multi-user install if it supports your platform and
+you can authenticate with `sudo`.
 
 # Single User Installation
 
@@ -50,9 +58,9 @@ $ rm -rf /nix
 The multi-user Nix installation creates system users, and a system
 service for the Nix daemon.
 
-  - Linux running systemd, with SELinux disabled
-
-  - macOS
+**Supported Systems**
+- Linux running systemd, with SELinux disabled
+- macOS
 
 You can instruct the installer to perform a multi-user installation on
 your system:
@@ -96,165 +104,28 @@ sudo rm /Library/LaunchDaemons/org.nixos.nix-daemon.plist
 There may also be references to Nix in `/etc/profile`, `/etc/bashrc`,
 and `/etc/zshrc` which you may remove.
 
-# macOS Installation
+# macOS Installation <a name="sect-macos-installation-change-store-prefix"></a><a name="sect-macos-installation-encrypted-volume"></a><a name="sect-macos-installation-symlink"></a><a name="sect-macos-installation-recommended-notes"></a>
+<!-- Note: anchors above to catch permalinks to old explanations -->
 
-Starting with macOS 10.15 (Catalina), the root filesystem is read-only.
-This means `/nix` can no longer live on your system volume, and that
-you'll need a workaround to install Nix.
+We believe we have ironed out how to cleanly support the read-only root
+on modern macOS. New installs will do this automatically, and you can
+also re-run a new installer to convert your existing setup.
 
-The recommended approach, which creates an unencrypted APFS volume for
-your Nix store and a "synthetic" empty directory to mount it over at
-`/nix`, is least likely to impair Nix or your system.
+This section previously detailed the situation, options, and trade-offs,
+but it now only outlines what the installer does. You don't need to know
+this to run the installer, but it may help if you run into trouble:
 
-> **Note**
-> 
-> With all separate-volume approaches, it's possible something on your
-> system (particularly daemons/services and restored apps) may need
-> access to your Nix store before the volume is mounted. Adding
-> additional encryption makes this more likely.
-
-If you're using a recent Mac with a [T2
-chip](https://www.apple.com/euro/mac/shared/docs/Apple_T2_Security_Chip_Overview.pdf),
-your drive will still be encrypted at rest (in which case "unencrypted"
-is a bit of a misnomer). To use this approach, just install Nix with:
-
-```console
-$ sh <(curl -L https://nixos.org/nix/install) --darwin-use-unencrypted-nix-store-volume
-```
-
-If you don't like the sound of this, you'll want to weigh the other
-approaches and tradeoffs detailed in this section.
-
-> **Note**
-> 
-> All of the known workarounds have drawbacks, but we hope better
-> solutions will be available in the future. Some that we have our eye
-> on are:
-> 
-> 1.  A true firmlink would enable the Nix store to live on the primary
->     data volume without the build problems caused by the symlink
->     approach. End users cannot currently create true firmlinks.
-> 
-> 2.  If the Nix store volume shared FileVault encryption with the
->     primary data volume (probably by using the same volume group and
->     role), FileVault encryption could be easily supported by the
->     installer without requiring manual setup by each user.
-
-## Change the Nix store path prefix
-
-Changing the default prefix for the Nix store is a simple approach which
-enables you to leave it on your root volume, where it can take full
-advantage of FileVault encryption if enabled. Unfortunately, this
-approach also opts your device out of some benefits that are enabled by
-using the same prefix across systems:
-
-  - Your system won't be able to take advantage of the binary cache
-    (unless someone is able to stand up and support duplicate caching
-    infrastructure), which means you'll spend more time waiting for
-    builds.
-
-  - It's harder to build and deploy packages to Linux systems.
-
-It would also possible (and often requested) to just apply this change
-ecosystem-wide, but it's an intrusive process that has side effects we
-want to avoid for now.
-
-## Use a separate encrypted volume
-
-If you like, you can also add encryption to the recommended approach
-taken by the installer. You can do this by pre-creating an encrypted
-volume before you run the installer--or you can run the installer and
-encrypt the volume it creates later.
-
-In either case, adding encryption to a second volume isn't quite as
-simple as enabling FileVault for your boot volume. Before you dive in,
-there are a few things to weigh:
-
-1.  The additional volume won't be encrypted with your existing
-    FileVault key, so you'll need another mechanism to decrypt the
-    volume.
-
-2.  You can store the password in Keychain to automatically decrypt the
-    volume on boot--but it'll have to wait on Keychain and may not mount
-    before your GUI apps restore. If any of your launchd agents or apps
-    depend on Nix-installed software (for example, if you use a
-    Nix-installed login shell), the restore may fail or break.
-    
-    On a case-by-case basis, you may be able to work around this problem
-    by using `wait4path` to block execution until your executable is
-    available.
-    
-    It's also possible to decrypt and mount the volume earlier with a
-    login hook--but this mechanism appears to be deprecated and its
-    future is unclear.
-
-3.  You can hard-code the password in the clear, so that your store
-    volume can be decrypted before Keychain is available.
-
-If you are comfortable navigating these tradeoffs, you can encrypt the
-volume with something along the lines of:
-
-```console
-$ diskutil apfs enableFileVault /nix -user disk
-```
-
-## Symlink the Nix store to a custom location
-
-Another simple approach is using `/etc/synthetic.conf` to symlink the
-Nix store to the data volume. This option also enables your store to
-share any configured FileVault encryption. Unfortunately, builds that
-resolve the symlink may leak the canonical path or even fail.
-
-Because of these downsides, we can't recommend this approach.
-
-## Notes on the recommended approach
-
-This section goes into a little more detail on the recommended approach.
-You don't need to understand it to run the installer, but it can serve
-as a helpful reference if you run into trouble.
-
-1.  In order to compose user-writable locations into the new read-only
-    system root, Apple introduced a new concept called `firmlinks`,
-    which it describes as a "bi-directional wormhole" between two
-    filesystems. You can see the current firmlinks in
-    `/usr/share/firmlinks`. Unfortunately, firmlinks aren't (currently?)
-    user-configurable.
-    
-    For special cases like NFS mount points or package manager roots,
-    [synthetic.conf(5)](https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man5/synthetic.conf.5.html)
-    supports limited user-controlled file-creation (of symlinks, and
-    synthetic empty directories) at `/`. To create a synthetic empty
-    directory for mounting at `/nix`, add the following line to
-    `/etc/synthetic.conf` (create it if necessary):
-    
-        nix
-
-2.  This configuration is applied at boot time, but you can use
-    `apfs.util` to trigger creation (not deletion) of new entries
-    without a reboot:
-    
-    ```console
-    $ /System/Library/Filesystems/apfs.fs/Contents/Resources/apfs.util -B
-    ```
-
-3.  Create the new APFS volume with diskutil:
-    
-    ```console
-    $ sudo diskutil apfs addVolume diskX APFS 'Nix Store' -mountpoint /nix
-    ```
-
-4.  Using `vifs`, add the new mount to `/etc/fstab`. If it doesn't
-    already have other entries, it should look something like:
-    
-        #
-        # Warning - this file should only be modified with vifs(8)
-        #
-        # Failure to do so is unsupported and may be destructive.
-        #
-        LABEL=Nix\040Store /nix apfs rw,nobrowse
-    
-    The nobrowse setting will keep Spotlight from indexing this volume,
-    and keep it from showing up on your desktop.
+- create a new APFS volume for your Nix store
+- update `/etc/synthetic.conf` to direct macOS to create a "synthetic"
+  empty root directory to mount your volume
+- specify mount options for the volume in `/etc/fstab`
+- if you have FileVault enabled
+    - generate an encryption password
+    - put it in your system Keychain
+    - use it to encrypt the volume
+- create a system LaunchDaemon to mount this volume early enough in the
+  boot process to avoid problems loading or restoring any programs that
+  need access to your Nix store
 
 # Installing a pinned Nix version from a URL
 

doc/manual/src/package-management/s3-substituter.md

@@ -7,17 +7,17 @@ cache mechanism that Nix usually uses to fetch prebuilt binaries from
 
 The following options can be specified as URL parameters to the S3 URL:
 
-  - `profile`  
+  - `profile`\
     The name of the AWS configuration profile to use. By default Nix
     will use the `default` profile.
 
-  - `region`  
+  - `region`\
     The region of the S3 bucket. `us–east-1` by default.
     
     If your bucket is not in `us–east-1`, you should always explicitly
     specify the region parameter.
 
-  - `endpoint`  
+  - `endpoint`\
     The URL to your S3-compatible service, for when not using Amazon S3.
     Do not specify this value if you're using Amazon S3.
     
@@ -26,7 +26,7 @@ The following options can be specified as URL parameters to the S3 URL:
     > This endpoint must support HTTPS and will use path-based
     > addressing instead of virtual host based addressing.
 
-  - `scheme`  
+  - `scheme`\
     The scheme used for S3 requests, `https` (default) or `http`. This
     option allows you to disable HTTPS for binary caches which don't
     support it.

doc/manual/src/release-notes/rl-2.4.md

@@ -0,0 +1,8 @@
+# Release 2.4 (202X-XX-XX)
+
+  - It is now an error to modify the `plugin-files` setting via a
+    command-line flag that appears after the first non-flag argument
+    to any command, including a subcommand to `nix`. For example,
+    `nix-instantiate default.nix --plugin-files ""` must now become
+    `nix-instantiate --plugin-files "" default.nix`.
+  - Plugins that add new `nix` subcommands are now actually respected.

flake.lock

@@ -1,22 +1,40 @@
 {
   "nodes": {
+    "lowdown-src": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1617481909,
+        "narHash": "sha256-SqnfOFuLuVRRNeVJr1yeEPJue/qWoCp5N6o5Kr///p4=",
+        "owner": "kristapsdz",
+        "repo": "lowdown",
+        "rev": "148f9b2f586c41b7e36e73009db43ea68c7a1a4d",
+        "type": "github"
+      },
+      "original": {
+        "owner": "kristapsdz",
+        "ref": "VERSION_0_8_4",
+        "repo": "lowdown",
+        "type": "github"
+      }
+    },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1602702596,
-        "narHash": "sha256-fqJ4UgOb4ZUnCDIapDb4gCrtAah5Rnr2/At3IzMitig=",
+        "lastModified": 1624862269,
+        "narHash": "sha256-JFcsh2+7QtfKdJFoPibLFPLgIW6Ycnv8Bts9a7RYme0=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "ad0d20345219790533ebe06571f82ed6b034db31",
+        "rev": "f77036342e2b690c61c97202bf48f2ce13acc022",
         "type": "github"
       },
       "original": {
         "id": "nixpkgs",
-        "ref": "nixos-20.09-small",
+        "ref": "nixos-21.05-small",
         "type": "indirect"
       }
     },
     "root": {
       "inputs": {
+        "lowdown-src": "lowdown-src",
         "nixpkgs": "nixpkgs"
       }
     }

flake.nix

@@ -1,10 +1,10 @@
 {
   description = "The purely functional package manager";
 
-  inputs.nixpkgs.url = "nixpkgs/nixos-20.09-small";
-  #inputs.lowdown-src = { url = "github:kristapsdz/lowdown"; flake = false; };
+  inputs.nixpkgs.url = "nixpkgs/nixos-21.05-small";
+  inputs.lowdown-src = { url = "github:kristapsdz/lowdown/VERSION_0_8_4"; flake = false; };
 
-  outputs = { self, nixpkgs }:
+  outputs = { self, nixpkgs, lowdown-src }:
 
     let
 
@@ -18,7 +18,7 @@
 
       linux64BitSystems = [ "x86_64-linux" "aarch64-linux" ];
       linuxSystems = linux64BitSystems ++ [ "i686-linux" ];
-      systems = linuxSystems ++ [ "x86_64-darwin" ];
+      systems = linuxSystems ++ [ "x86_64-darwin" "aarch64-darwin" ];
 
       forAllSystems = f: nixpkgs.lib.genAttrs systems (system: f system);
 
@@ -78,11 +78,12 @@
             buildPackages.git
             buildPackages.mercurial
             buildPackages.jq
-          ];
+          ]
+          ++ lib.optionals stdenv.isLinux [(pkgs.util-linuxMinimal or pkgs.utillinuxMinimal)];
 
         buildDeps =
           [ curl
-            bzip2 xz brotli zlib editline
+            bzip2 xz brotli editline
             openssl sqlite
             libarchive
             boost
@@ -90,8 +91,9 @@
             lowdown
             gmock
           ]
-          ++ lib.optionals stdenv.isLinux [libseccomp utillinuxMinimal]
-          ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium;
+          ++ lib.optionals stdenv.isLinux [libseccomp]
+          ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium
+          ++ lib.optional stdenv.isx86_64 libcpuid;
 
         awsDeps = lib.optional (stdenv.isLinux || stdenv.isDarwin)
           (aws-sdk-cpp.override {
@@ -109,12 +111,80 @@
           ];
       };
 
+    installScriptFor = systems:
+      with nixpkgsFor.x86_64-linux;
+        runCommand "installer-script"
+          { buildInputs = [ nix ];
+          }
+          ''
+            mkdir -p $out/nix-support
+
+            # Converts /nix/store/50p3qk8kka9dl6wyq40vydq945k0j3kv-nix-2.4pre20201102_550e11f/bin/nix
+            # To 50p3qk8kka9dl6wyq40vydq945k0j3kv/bin/nix
+            tarballPath() {
+              # Remove the store prefix
+              local path=''${1#${builtins.storeDir}/}
+              # Get the path relative to the derivation root
+              local rest=''${path#*/}
+              # Get the derivation hash
+              local drvHash=''${path%%-*}
+              echo "$drvHash/$rest"
+            }
+
+            substitute ${./scripts/install.in} $out/install \
+              ${pkgs.lib.concatMapStrings
+                (system:
+                  '' \
+                  --replace '@tarballHash_${system}@' $(nix --experimental-features nix-command hash-file --base16 --type sha256 ${self.hydraJobs.binaryTarball.${system}}/*.tar.xz) \
+                  --replace '@tarballPath_${system}@' $(tarballPath ${self.hydraJobs.binaryTarball.${system}}/*.tar.xz) \
+                  ''
+                )
+                systems
+              } --replace '@nixVersion@' ${version}
+
+            echo "file installer $out/install" >> $out/nix-support/hydra-build-products
+          '';
+
+      testNixVersions = pkgs: client: daemon: with commonDeps pkgs; pkgs.stdenv.mkDerivation {
+        NIX_DAEMON_PACKAGE = daemon;
+        NIX_CLIENT_PACKAGE = client;
+        # Must keep this name short as OSX has a rather strict limit on the
+        # socket path length, and this name appears in the path of the
+        # nix-daemon socket used in the tests
+        name = "nix-tests";
+        inherit version;
+
+        src = self;
+
+        VERSION_SUFFIX = versionSuffix;
+
+        nativeBuildInputs = nativeBuildDeps;
+        buildInputs = buildDeps ++ awsDeps;
+        propagatedBuildInputs = propagatedDeps;
+
+        enableParallelBuilding = true;
+
+        dontBuild = true;
+        doInstallCheck = true;
+
+        installPhase = ''
+          mkdir -p $out
+        '';
+        installCheckPhase = "make installcheck";
+
+      };
+
     in {
 
       # A Nixpkgs overlay that overrides the 'nix' and
       # 'nix.perl-bindings' packages.
       overlay = final: prev: {
 
+        # An older version of Nix to test against when using the daemon.
+        # Currently using `nixUnstable` as the stable one doesn't respect
+        # `NIX_DAEMON_SOCKET_PATH` which is needed for the tests.
+        nixStable = prev.nix;
+
         nix = with final; with commonDeps pkgs; stdenv.mkDerivation {
           name = "nix-${version}";
           inherit version;
@@ -164,6 +234,8 @@
 
           separateDebugInfo = true;
 
+          strictDeps = true;
+
           passthru.perl-bindings = with final; stdenv.mkDerivation {
             name = "nix-perl-${version}";
 
@@ -184,7 +256,8 @@
                 boost
                 nlohmann_json
               ]
-              ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium;
+              ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium
+              ++ lib.optional stdenv.isDarwin darwin.apple_sdk.frameworks.Security;
 
             configureFlags = ''
               --with-dbi=${perlPackages.DBI}/${pkgs.perl.libPrefix}
@@ -199,21 +272,23 @@
         };
 
         lowdown = with final; stdenv.mkDerivation rec {
-          name = "lowdown-0.8.0";
+          name = "lowdown-0.8.4";
 
+          /*
           src = fetchurl {
             url = "https://kristaps.bsd.lv/lowdown/snapshots/${name}.tar.gz";
             hash = "sha512-U9WeGoInT9vrawwa57t6u9dEdRge4/P+0wLxmQyOL9nhzOEUU2FRz2Be9H0dCjYE7p2v3vCXIYk40M+jjULATw==";
           };
+          */
 
-          #src = lowdown-src;
+          src = lowdown-src;
 
           outputs = [ "out" "bin" "dev" ];
 
           nativeBuildInputs = [ which ];
 
-          configurePhase =
-            ''
+          configurePhase = ''
+              ${if (stdenv.isDarwin && stdenv.isAarch64) then "echo \"HAVE_SANDBOX_INIT=false\" > configure.local" else ""}
               ./configure \
                 PREFIX=${placeholder "dev"} \
                 BINDIR=${placeholder "bin"}/bin
@@ -313,40 +388,8 @@
         # to https://nixos.org/nix/install. It downloads the binary
         # tarball for the user's system and calls the second half of the
         # installation script.
-        installerScript =
-          with nixpkgsFor.x86_64-linux;
-          runCommand "installer-script"
-            { buildInputs = [ nix ];
-            }
-            ''
-              mkdir -p $out/nix-support
-
-              # Converts /nix/store/50p3qk8kka9dl6wyq40vydq945k0j3kv-nix-2.4pre20201102_550e11f/bin/nix
-              # To 50p3qk8kka9dl6wyq40vydq945k0j3kv/bin/nix
-              tarballPath() {
-                # Remove the store prefix
-                local path=''${1#${builtins.storeDir}/}
-                # Get the path relative to the derivation root
-                local rest=''${path#*/}
-                # Get the derivation hash
-                local drvHash=''${path%%-*}
-                echo "$drvHash/$rest"
-              }
-
-              substitute ${./scripts/install.in} $out/install \
-                ${pkgs.lib.concatMapStrings
-                  (system:
-                    '' \
-                    --replace '@tarballHash_${system}@' $(nix --experimental-features nix-command hash-file --base16 --type sha256 ${self.hydraJobs.binaryTarball.${system}}/*.tar.xz) \
-                    --replace '@tarballPath_${system}@' $(tarballPath ${self.hydraJobs.binaryTarball.${system}}/*.tar.xz) \
-                    ''
-                  )
-                  [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ]
-                } \
-                --replace '@nixVersion@' ${version}
-
-              echo "file installer $out/install" >> $out/nix-support/hydra-build-products
-            '';
+        installerScript = installScriptFor [ "x86_64-linux" "i686-linux" "aarch64-linux" "x86_64-darwin" "aarch64-darwin" ];
+        installerScriptForGHA = installScriptFor [ "x86_64-linux" "x86_64-darwin" ];
 
         # Line coverage analysis.
         coverage =
@@ -431,6 +474,15 @@
       checks = forAllSystems (system: {
         binaryTarball = self.hydraJobs.binaryTarball.${system};
         perlBindings = self.hydraJobs.perlBindings.${system};
+        installTests =
+          let pkgs = nixpkgsFor.${system}; in
+          pkgs.runCommand "install-tests" {
+            againstSelf = testNixVersions pkgs pkgs.nix pkgs.pkgs.nix;
+            againstCurrentUnstable = testNixVersions pkgs pkgs.nix pkgs.nixUnstable;
+            # Disabled because the latest stable version doesn't handle
+            # `NIX_DAEMON_SOCKET_PATH` which is required for the tests to work
+            # againstLatestStable = testNixVersions pkgs pkgs.nix pkgs.nixStable;
+          } "touch $out";
       });
 
       packages = forAllSystems (system: {
@@ -471,6 +523,8 @@
           installCheckFlags = "sysconfdir=$(out)/etc";
 
           stripAllList = ["bin"];
+
+          strictDeps = true;
         };
       });
 

maintainers/upload-release.pl

@@ -133,20 +133,8 @@ for my $fn (glob "$tmpDir/*") {
 
 exit if $version =~ /pre/;
 
-# Update Nixpkgs in a very hacky way.
+# Update nix-fallback-paths.nix.
 system("cd $nixpkgsDir && git pull") == 0 or die;
-my $oldName = `nix-instantiate --eval $nixpkgsDir -A nix.name`; chomp $oldName;
-my $oldHash = `nix-instantiate --eval $nixpkgsDir -A nix.src.outputHash`; chomp $oldHash;
-print STDERR "old stable version in Nixpkgs = $oldName / $oldHash\n";
-
-my $fn = "$nixpkgsDir/pkgs/tools/package-management/nix/default.nix";
-my $oldFile = read_file($fn);
-$oldFile =~ s/$oldName/"$releaseName"/g;
-$oldFile =~ s/$oldHash/"$tarballHash"/g;
-write_file($fn, $oldFile);
-
-$oldName =~ s/nix-//g;
-$oldName =~ s/"//g;
 
 sub getStorePath {
     my ($jobName) = @_;
@@ -167,7 +155,7 @@ write_file("$nixpkgsDir/nixos/modules/installer/tools/nix-fallback-paths.nix",
            "  x86_64-darwin = \"" . getStorePath("build.x86_64-darwin") . "\";\n" .
            "}\n");
 
-system("cd $nixpkgsDir && git commit -a -m 'nix: $oldName -> $version'") == 0 or die;
+system("cd $nixpkgsDir && git commit -a -m 'nix-fallback-paths.nix: Update to $version'") == 0 or die;
 
 # Update the "latest" symlink.
 $channelsBucket->add_key(

misc/launchd/org.nixos.nix-daemon.plist.in

@@ -19,7 +19,7 @@
     <array>
       <string>/bin/sh</string>
       <string>-c</string>
-      <string>/bin/wait4path /nix/var/nix/profiles/default/bin/nix-daemon &amp;&amp; /nix/var/nix/profiles/default/bin/nix-daemon</string>
+      <string>/bin/wait4path /nix/var/nix/profiles/default/bin/nix-daemon &amp;&amp; exec /nix/var/nix/profiles/default/bin/nix-daemon</string>
     </array>
     <key>StandardErrorPath</key>
     <string>/var/log/nix-daemon.log</string>

misc/zsh/completion.zsh

@@ -1,3 +1,5 @@
+#compdef nix
+
 function _nix() {
   local ifs_bk="$IFS"
   local input=("${(Q)words[@]}")
@@ -18,4 +20,4 @@ function _nix() {
   _describe 'nix' suggestions
 }
 
-compdef _nix nix
+_nix "$@"

misc/zsh/local.mk

@@ -0,0 +1 @@
+$(eval $(call install-file-as, $(d)/completion.zsh, $(datarootdir)/zsh/site-functions/_nix, 0644))

mk/jars.mk

@@ -1,36 +0,0 @@
-define build-jar
-
-  $(1)_NAME ?= $(1)
-
-  _d := $$(strip $$($(1)_DIR))
-
-  $(1)_PATH := $$(_d)/$$($(1)_NAME).jar
-
-  $(1)_TMPDIR := $$(_d)/.$$($(1)_NAME).jar.tmp
-
-  _jars := $$(foreach jar, $$($(1)_JARS), $$($$(jar)_PATH))
-
-  $$($(1)_PATH): $$($(1)_SOURCES) $$(_jars) $$($(1)_EXTRA_DEPS)| $$($(1)_ORDER_AFTER)
-	@rm -rf $$($(1)_TMPDIR)
-	@mkdir -p $$($(1)_TMPDIR)
-	$$(trace-javac) javac $(GLOBAL_JAVACFLAGS) $$($(1)_JAVACFLAGS) -d $$($(1)_TMPDIR) \
-	  $$(foreach fn, $$($(1)_SOURCES), '$$(fn)') \
-	  -cp "$$(subst $$(space),,$$(foreach jar,$$($(1)_JARS),$$($$(jar)_PATH):))$$$$CLASSPATH"
-	@echo -e '$$(subst $$(newline),\n,$$($(1)_MANIFEST))' > $$($(1)_PATH).manifest
-	$$(trace-jar) jar cfm $$($(1)_PATH) $$($(1)_PATH).manifest -C $$($(1)_TMPDIR) .
-	@rm $$($(1)_PATH).manifest
-	@rm -rf $$($(1)_TMPDIR)
-
-  $(1)_INSTALL_DIR ?= $$(jardir)
-
-  $(1)_INSTALL_PATH := $$($(1)_INSTALL_DIR)/$$($(1)_NAME).jar
-
-  $$(eval $$(call install-file-as, $$($(1)_PATH), $$($(1)_INSTALL_PATH), 0644))
-
-  install: $$($(1)_INSTALL_PATH)
-
-  jars-list += $$($(1)_PATH)
-
-  clean-files += $$($(1)_PATH)
-
-endef

mk/lib.mk

@@ -31,7 +31,6 @@ libdir ?= $(prefix)/lib
 bindir ?= $(prefix)/bin
 libexecdir ?= $(prefix)/libexec
 datadir ?= $(prefix)/share
-jardir ?= $(datadir)/java
 localstatedir ?= $(prefix)/var
 sysconfdir ?= $(prefix)/etc
 mandir ?= $(prefix)/share/man
@@ -74,7 +73,6 @@ BUILD_DEBUG ?= 1
 ifeq ($(BUILD_DEBUG), 1)
   GLOBAL_CFLAGS += -g
   GLOBAL_CXXFLAGS += -g
-  GLOBAL_JAVACFLAGS += -g
 endif
 
 
@@ -84,7 +82,6 @@ include mk/clean.mk
 include mk/install.mk
 include mk/libraries.mk
 include mk/programs.mk
-include mk/jars.mk
 include mk/patterns.mk
 include mk/templates.mk
 include mk/tests.mk
@@ -102,7 +99,6 @@ $(foreach mf, $(makefiles), $(eval $(call include-sub-makefile, $(mf))))
 # Instantiate stuff.
 $(foreach lib, $(libraries), $(eval $(call build-library,$(lib))))
 $(foreach prog, $(programs), $(eval $(call build-program,$(prog))))
-$(foreach jar, $(jars), $(eval $(call build-jar,$(jar))))
 $(foreach script, $(bin-scripts), $(eval $(call install-program-in,$(script),$(bindir))))
 $(foreach script, $(bin-scripts), $(eval programs-list += $(script)))
 $(foreach script, $(noinst-scripts), $(eval programs-list += $(script)))
@@ -113,7 +109,7 @@ $(foreach file, $(man-pages), $(eval $(call install-data-in, $(file), $(mandir)/
 
 .PHONY: default all man help
 
-all: $(programs-list) $(libs-list) $(jars-list) $(man-pages)
+all: $(programs-list) $(libs-list) $(man-pages)
 
 man: $(man-pages)
 
@@ -137,12 +133,6 @@ ifdef libs-list
 	@echo "The following libraries can be built:"
 	@echo ""
 	@for i in $(libs-list); do echo "  $$i"; done
-endif
-ifdef jars-list
-	@echo ""
-	@echo "The following JARs can be built:"
-	@echo ""
-	@for i in $(jars-list); do echo "  $$i"; done
 endif
 	@echo ""
 	@echo "The following variables control the build:"
@@ -153,4 +143,5 @@ endif
 	@echo "  CFLAGS: Flags for the C compiler"
 	@echo "  CXX ($(CXX)): C++ compiler to be used"
 	@echo "  CXXFLAGS: Flags for the C++ compiler"
+	@echo "  CPPFLAGS: C preprocessor flags, used for both CC and CXX"
 	@$(print-var-help)

mk/patterns.mk

@@ -1,11 +1,11 @@
 $(buildprefix)%.o: %.cc
 	@mkdir -p "$(dir $@)"
-	$(trace-cxx) $(CXX) -o $@ -c $< $(GLOBAL_CXXFLAGS_PCH) $(GLOBAL_CXXFLAGS) $(CXXFLAGS) $($@_CXXFLAGS) -MMD -MF $(call filename-to-dep, $@) -MP
+	$(trace-cxx) $(CXX) -o $@ -c $< $(CPPFLAGS) $(GLOBAL_CXXFLAGS_PCH) $(GLOBAL_CXXFLAGS) $(CXXFLAGS) $($@_CXXFLAGS) -MMD -MF $(call filename-to-dep, $@) -MP
 
 $(buildprefix)%.o: %.cpp
 	@mkdir -p "$(dir $@)"
-	$(trace-cxx) $(CXX) -o $@ -c $< $(GLOBAL_CXXFLAGS_PCH) $(GLOBAL_CXXFLAGS) $(CXXFLAGS) $($@_CXXFLAGS) -MMD -MF $(call filename-to-dep, $@) -MP
+	$(trace-cxx) $(CXX) -o $@ -c $< $(CPPFLAGS) $(GLOBAL_CXXFLAGS_PCH) $(GLOBAL_CXXFLAGS) $(CXXFLAGS) $($@_CXXFLAGS) -MMD -MF $(call filename-to-dep, $@) -MP
 
 $(buildprefix)%.o: %.c
 	@mkdir -p "$(dir $@)"
-	$(trace-cc) $(CC) -o $@ -c $< $(GLOBAL_CFLAGS) $(CFLAGS) $($@_CFLAGS) -MMD -MF $(call filename-to-dep, $@) -MP
+	$(trace-cc) $(CC) -o $@ -c $< $(CPPFLAGS) $(GLOBAL_CFLAGS) $(CFLAGS) $($@_CFLAGS) -MMD -MF $(call filename-to-dep, $@) -MP

mk/run_test.sh

@@ -14,7 +14,7 @@ if [ -t 1 ]; then
     yellow=""
     normal=""
 fi
-(cd $(dirname $1) && env ${TESTS_ENVIRONMENT} init.sh 2>/dev/null > /dev/null)
+(cd tests && env ${TESTS_ENVIRONMENT} init.sh 2>/dev/null > /dev/null)
 log="$(cd $(dirname $1) && env ${TESTS_ENVIRONMENT} $(basename $1) 2>&1)"
 status=$?
 if [ $status -eq 0 ]; then

mk/tests.mk

@@ -8,7 +8,7 @@ define run-install-test
 
   .PHONY: $1.test
   $1.test: $1 $(test-deps)
-	@env TEST_NAME=$(notdir $(basename $1)) TESTS_ENVIRONMENT="$(tests-environment)" mk/run_test.sh $1 < /dev/null
+	@env TEST_NAME=$(basename $1) TESTS_ENVIRONMENT="$(tests-environment)" mk/run_test.sh $1 < /dev/null
 
 endef
 

mk/tracing.mk

@@ -8,8 +8,6 @@ ifeq ($(V), 0)
   trace-ld      = @echo "  LD    " $@;
   trace-ar      = @echo "  AR    " $@;
   trace-install = @echo "  INST  " $@;
-  trace-javac   = @echo "  JAVAC " $@;
-  trace-jar     = @echo "  JAR   " $@;
   trace-mkdir   = @echo "  MKDIR " $@;
   trace-test    = @echo "  TEST  " $@;
 

nix-rust/local.mk

@@ -8,8 +8,13 @@ endif
 
 libnixrust_PATH := $(d)/target/$(RUST_DIR)/libnixrust.$(SO_EXT)
 libnixrust_INSTALL_PATH := $(libdir)/libnixrust.$(SO_EXT)
-libnixrust_LDFLAGS_USE := -L$(d)/target/$(RUST_DIR) -lnixrust -ldl
-libnixrust_LDFLAGS_USE_INSTALLED := -L$(libdir) -lnixrust -ldl
+libnixrust_LDFLAGS_USE := -L$(d)/target/$(RUST_DIR) -lnixrust
+libnixrust_LDFLAGS_USE_INSTALLED := -L$(libdir) -lnixrust
+
+ifeq ($(OS), Linux)
+libnixrust_LDFLAGS_USE += -ldl
+libnixrust_LDFLAGS_USE_INSTALLED += -ldl
+endif
 
 ifeq ($(OS), Darwin)
 libnixrust_BUILD_FLAGS = NIX_LDFLAGS="-undefined dynamic_lookup"

scripts/bigsur-nixbld-user-migration.sh

@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+
+((NEW_NIX_FIRST_BUILD_UID=301))
+
+id_available(){
+	dscl . list /Users UniqueID | grep -E '\b'$1'\b' >/dev/null
+}
+
+change_nixbld_names_and_ids(){
+	local name uid next_id
+	((next_id=NEW_NIX_FIRST_BUILD_UID))
+	echo "Attempting to migrate nixbld users."
+	echo "Each user should change from nixbld# to _nixbld#"
+	echo "and their IDs relocated to $next_id+"
+	while read -r name uid; do
+		echo "   Checking $name (uid: $uid)"
+		# iterate for a clean ID
+		while id_available "$next_id"; do
+			((next_id++))
+			if ((next_id >= 400)); then
+				echo "We've hit UID 400 without placing all of your users :("
+				echo "You should use the commands in this script as a starting"
+				echo "point to review your UID-space and manually move the"
+				echo "remaining users (or delete them, if you don't need them)."
+				exit 1
+			fi
+		done
+
+		if [[ $name == _* ]]; then
+			echo "      It looks like $name has already been renamed--skipping."
+		else
+			# first 3 are cleanup, it's OK if they aren't here
+			sudo dscl . delete /Users/$name dsAttrTypeNative:_writers_passwd &>/dev/null || true
+			sudo dscl . change /Users/$name NFSHomeDirectory "/private/var/empty 1" "/var/empty" &>/dev/null || true
+			# remove existing user from group
+			sudo dseditgroup -o edit -t user -d $name nixbld || true
+			sudo dscl . change /Users/$name UniqueID $uid $next_id
+			sudo dscl . change /Users/$name RecordName $name _$name
+			# add renamed user to group
+			sudo dseditgroup -o edit -t user -a _$name nixbld
+			echo "      $name migrated to _$name (uid: $next_id)"
+		fi
+	done < <(dscl . list /Users UniqueID | grep nixbld | sort -n -k2)
+}
+
+change_nixbld_names_and_ids

scripts/create-darwin-volume.sh

@@ -1,33 +1,262 @@
-#!/bin/sh
-set -e
+#!/usr/bin/env bash
+set -eu
+set -o pipefail
 
-root_disk() {
-    diskutil info -plist /
-}
+# I'm a little agnostic on the choices, but supporting a wide
+# slate of uses for now, including:
+# - import-only: `. create-darwin-volume.sh no-main[ ...]`
+# - legacy: `./create-darwin-volume.sh` or `. create-darwin-volume.sh`
+#   (both will run main())
+# - external alt-routine: `./create-darwin-volume.sh no-main func[ ...]`
+if [ "${1-}" = "no-main" ]; then
+    shift
+    readonly _CREATE_VOLUME_NO_MAIN=1
+else
+    readonly _CREATE_VOLUME_NO_MAIN=0
+    # declare some things we expect to inherit from install-multi-user
+    # I don't love this (because it's a bit of a kludge).
+    #
+    # CAUTION: (Dec 19 2020)
+    # This is a stopgap. It doesn't cover the full slate of
+    # identifiers we inherit--just those necessary to:
+    # - avoid breaking direct invocations of this script (here/now)
+    # - avoid hard-to-reverse structural changes before the call to rm
+    #   single-user support is verified
+    #
+    # In the near-mid term, I (personally) think we should:
+    # - decide to deprecate the direct call and add a notice
+    # - fold all of this into install-darwin-multi-user.sh
+    # - intentionally remove the old direct-invocation form (kill the
+    #   routine, replace this script w/ deprecation notice and a note
+    #   on the remove-after date)
+    #
+    readonly NIX_ROOT="${NIX_ROOT:-/nix}"
 
-# i.e., "disk1"
+    _sudo() {
+        shift # throw away the 'explanation'
+        /usr/bin/sudo "$@"
+    }
+    failure() {
+        if [ "$*" = "" ]; then
+            cat
+        else
+            echo "$@"
+        fi
+        exit 1
+    }
+    task() {
+        echo "$@"
+    }
+fi
+
+# usually "disk1"
 root_disk_identifier() {
-    diskutil info -plist / | xmllint --xpath "/plist/dict/key[text()='ParentWholeDisk']/following-sibling::string[1]/text()" -
+    # For performance (~10ms vs 280ms) I'm parsing 'diskX' from stat output
+    # (~diskXsY)--but I'm retaining the more-semantic approach since
+    # it documents intent better.
+    # /usr/sbin/diskutil info -plist / | xmllint --xpath "/plist/dict/key[text()='ParentWholeDisk']/following-sibling::string[1]/text()" -
+    #
+    local special_device
+    special_device="$(/usr/bin/stat -f "%Sd" /)"
+    echo "${special_device%s[0-9]*}"
 }
 
-find_nix_volume() {
-    diskutil apfs list -plist "$1" | xmllint --xpath "(/plist/dict/array/dict/key[text()='Volumes']/following-sibling::array/dict/key[text()='Name']/following-sibling::string[starts-with(translate(text(),'N','n'),'nix')]/text())[1]" - 2>/dev/null || true
+# make it easy to play w/ 'Case-sensitive APFS'
+readonly NIX_VOLUME_FS="${NIX_VOLUME_FS:-APFS}"
+readonly NIX_VOLUME_LABEL="${NIX_VOLUME_LABEL:-Nix Store}"
+# Strongly assuming we'll make a volume on the device / is on
+# But you can override NIX_VOLUME_USE_DISK to create it on some other device
+readonly NIX_VOLUME_USE_DISK="${NIX_VOLUME_USE_DISK:-$(root_disk_identifier)}"
+NIX_VOLUME_USE_SPECIAL="${NIX_VOLUME_USE_SPECIAL:-}"
+NIX_VOLUME_USE_UUID="${NIX_VOLUME_USE_UUID:-}"
+readonly NIX_VOLUME_MOUNTD_DEST="${NIX_VOLUME_MOUNTD_DEST:-/Library/LaunchDaemons/org.nixos.darwin-store.plist}"
+
+if /usr/bin/fdesetup isactive >/dev/null; then
+    test_filevault_in_use() { return 0; }
+    # no readonly; we may modify if user refuses from cure_volume
+    NIX_VOLUME_DO_ENCRYPT="${NIX_VOLUME_DO_ENCRYPT:-1}"
+else
+    test_filevault_in_use() { return 1; }
+    NIX_VOLUME_DO_ENCRYPT="${NIX_VOLUME_DO_ENCRYPT:-0}"
+fi
+
+should_encrypt_volume() {
+    test_filevault_in_use && (( NIX_VOLUME_DO_ENCRYPT == 1 ))
+}
+
+substep() {
+    printf "   %s\n" "" "- $1" "" "${@:2}"
+}
+
+
+volumes_labeled() {
+    local label="$1"
+    xsltproc --novalid --stringparam label "$label" - <(/usr/sbin/ioreg -ra -c "AppleAPFSVolume") <<'EOF'
+<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="1.0">
+  <xsl:output method="text"/>
+  <xsl:template match="/">
+    <xsl:apply-templates select="/plist/array/dict/key[text()='IORegistryEntryName']/following-sibling::*[1][text()=$label]/.."/>
+  </xsl:template>
+  <xsl:template match="dict">
+    <xsl:apply-templates match="string" select="key[text()='BSD Name']/following-sibling::*[1]"/>
+    <xsl:text>=</xsl:text>
+    <xsl:apply-templates match="string" select="key[text()='UUID']/following-sibling::*[1]"/>
+    <xsl:text>&#xA;</xsl:text>
+  </xsl:template>
+</xsl:stylesheet>
+EOF
+    # I cut label out of the extracted values, but here it is for reference:
+    # <xsl:apply-templates match="string" select="key[text()='IORegistryEntryName']/following-sibling::*[1]"/>
+    # <xsl:text>=</xsl:text>
+}
+
+right_disk() {
+    local volume_special="$1" # (i.e., disk1s7)
+    [[ "$volume_special" == "$NIX_VOLUME_USE_DISK"s* ]]
+}
+
+right_volume() {
+    local volume_special="$1" # (i.e., disk1s7)
+    # if set, it must match; otherwise ensure it's on the right disk
+    if [ -z "$NIX_VOLUME_USE_SPECIAL" ]; then
+        if right_disk "$volume_special"; then
+            NIX_VOLUME_USE_SPECIAL="$volume_special" # latch on
+            return 0
+        else
+            return 1
+        fi
+    else
+        [ "$volume_special" = "$NIX_VOLUME_USE_SPECIAL" ]
+    fi
+}
+
+right_uuid() {
+    local volume_uuid="$1"
+    # if set, it must match; otherwise allow
+    if [ -z "$NIX_VOLUME_USE_UUID" ]; then
+        NIX_VOLUME_USE_UUID="$volume_uuid" # latch on
+        return 0
+    else
+        [ "$volume_uuid" = "$NIX_VOLUME_USE_UUID" ]
+    fi
+}
+
+cure_volumes() {
+    local found volume special uuid
+    # loop just in case they have more than one volume
+    # (nothing stops you from doing this)
+    for volume in $(volumes_labeled "$NIX_VOLUME_LABEL"); do
+        # CAUTION: this could (maybe) be a more normal read
+        # loop like:
+        #   while IFS== read -r special uuid; do
+        #       # ...
+        #   done <<<"$(volumes_labeled "$NIX_VOLUME_LABEL")"
+        #
+        # I did it with for to skirt a problem with the obvious
+        # pattern replacing stdin and causing user prompts
+        # inside (which also use read and access stdin) to skip
+        #
+        # If there's an existing encrypted volume we can't find
+        # in keychain, the user never gets prompted to delete
+        # the volume, and the install fails.
+        #
+        # If you change this, a human needs to test a very
+        # specific scenario: you already have an encrypted
+        # Nix Store volume, and have deleted its credential
+        # from keychain. Ensure the script asks you if it can
+        # delete the volume, and then prompts for your sudo
+        # password to confirm.
+        #
+        # shellcheck disable=SC1097
+        IFS== read -r special uuid <<< "$volume"
+        # take the first one that's on the right disk
+        if [ -z "${found:-}" ]; then
+            if right_volume "$special" && right_uuid "$uuid"; then
+                cure_volume "$special" "$uuid"
+                found="${special} (${uuid})"
+            else
+                warning <<EOF
+Ignoring ${special} (${uuid}) because I am looking for:
+disk=${NIX_VOLUME_USE_DISK} special=${NIX_VOLUME_USE_SPECIAL:-${NIX_VOLUME_USE_DISK}sX} uuid=${NIX_VOLUME_USE_UUID:-any}
+EOF
+                # TODO: give chance to delete if ! headless?
+            fi
+        else
+            warning <<EOF
+Ignoring ${special} (${uuid}), already found target: $found
+EOF
+            # TODO reminder? I feel like I want one
+            # idiom that reminds some warnings, or warns
+            # some reminders?
+            # TODO: if ! headless, chance to delete?
+        fi
+    done
+    if [ -z "${found:-}" ]; then
+        readonly NIX_VOLUME_USE_SPECIAL NIX_VOLUME_USE_UUID
+    fi
+}
+
+volume_encrypted() {
+    local volume_special="$1" # (i.e., disk1s7)
+    # Trying to match the first line of output; known first lines:
+    # No cryptographic users for <special>
+    # Cryptographic user for <special> (1 found)
+    # Cryptographic users for <special> (2 found)
+    /usr/sbin/diskutil apfs listCryptoUsers -plist "$volume_special" | /usr/bin/grep -q APFSCryptoUserUUID
 }
 
 test_fstab() {
-    grep -q "/nix apfs rw" /etc/fstab 2>/dev/null
+    /usr/bin/grep -q "$NIX_ROOT apfs rw" /etc/fstab 2>/dev/null
 }
 
-test_nix_symlink() {
-    [ -L "/nix" ] || grep -q "^nix." /etc/synthetic.conf 2>/dev/null
+test_nix_root_is_symlink() {
+    [ -L "$NIX_ROOT" ]
 }
 
-test_synthetic_conf() {
-    grep -q "^nix$" /etc/synthetic.conf 2>/dev/null
+test_synthetic_conf_either(){
+    /usr/bin/grep -qE "^${NIX_ROOT:1}($|\t.{3,}$)" /etc/synthetic.conf 2>/dev/null
+}
+
+test_synthetic_conf_mountable() {
+    /usr/bin/grep -q "^${NIX_ROOT:1}$" /etc/synthetic.conf 2>/dev/null
+}
+
+test_synthetic_conf_symlinked() {
+    /usr/bin/grep -qE "^${NIX_ROOT:1}\t.{3,}$" /etc/synthetic.conf 2>/dev/null
+}
+
+test_nix_volume_mountd_installed() {
+    test -e "$NIX_VOLUME_MOUNTD_DEST"
+}
+
+# current volume password
+test_keychain_by_uuid() {
+    local volume_uuid="$1"
+    # Note: doesn't need sudo just to check; doesn't output pw
+    security find-generic-password -s "$volume_uuid" &>/dev/null
+}
+
+get_volume_pass() {
+    local volume_uuid="$1"
+    _sudo \
+        "to confirm keychain has a password that unlocks this volume" \
+        security find-generic-password -s "$volume_uuid" -w
+}
+
+verify_volume_pass() {
+    local volume_special="$1" # (i.e., disk1s7)
+    local volume_uuid="$2"
+    /usr/sbin/diskutil apfs unlockVolume "$volume_special" -verify -stdinpassphrase -user "$volume_uuid"
+}
+
+volume_pass_works() {
+    local volume_special="$1" # (i.e., disk1s7)
+    local volume_uuid="$2"
+    get_volume_pass "$volume_uuid" | verify_volume_pass "$volume_special" "$volume_uuid"
 }
 
 # Create the paths defined in synthetic.conf, saving us a reboot.
-create_synthetic_objects(){
+create_synthetic_objects() {
     # Big Sur takes away the -B flag we were using and replaces it
     # with a -t flag that appears to do the same thing (but they
     # don't behave exactly the same way in terms of return values).
@@ -41,129 +270,570 @@ create_synthetic_objects(){
 }
 
 test_nix() {
-    test -d "/nix"
+    test -d "$NIX_ROOT"
 }
 
-test_t2_chip_present(){
-    # Use xartutil to see if system has a t2 chip.
-    #
-    # This isn't well-documented on its own; until it is,
-    # let's keep track of knowledge/assumptions.
-    #
-    # Warnings:
-    # - Don't search "xart" if porn will cause you trouble :)
-    # - Other xartutil flags do dangerous things. Don't run them
-    #   naively. If you must, search "xartutil" first.
-    #
-    # Assumptions:
-    # - the "xART session seeds recovery utility"
-    #   appears to interact with xartstorageremoted
-    # - `sudo xartutil --list` lists xART sessions
-    #   and their seeds and exits 0 if successful. If
-    #   not, it exits 1 and prints an error such as:
-    #   xartutil: ERROR: No supported link to the SEP present
-    # - xART sessions/seeds are present when a T2 chip is
-    #   (and not, otherwise)
-    # - the presence of a T2 chip means a newly-created
-    #   volume on the primary drive will be
-    #   encrypted at rest
-    # - all together: `sudo xartutil --list`
-    #   should exit 0 if a new Nix Store volume will
-    #   be encrypted at rest, and exit 1 if not.
-    sudo xartutil --list >/dev/null 2>/dev/null
+test_voldaemon() {
+    test -f "$NIX_VOLUME_MOUNTD_DEST"
 }
 
-test_filevault_in_use() {
-    fdesetup isactive >/dev/null
+generate_mount_command() {
+    local cmd_type="$1" # encrypted|unencrypted
+    local volume_uuid mountpoint cmd=()
+    printf -v volume_uuid "%q" "$2"
+    printf -v mountpoint "%q" "$NIX_ROOT"
+
+    case "$cmd_type" in
+        encrypted)
+            cmd=(/bin/sh -c "/usr/bin/security find-generic-password -s '$volume_uuid' -w | /usr/sbin/diskutil apfs unlockVolume '$volume_uuid' -mountpoint '$mountpoint' -stdinpassphrase");;
+        unencrypted)
+            cmd=(/usr/sbin/diskutil mount -mountPoint "$mountpoint" "$volume_uuid");;
+        *)
+            failure "Invalid first arg $cmd_type to generate_mount_command";;
+    esac
+
+    printf "    <string>%s</string>\n" "${cmd[@]}"
 }
 
-# use after error msg for conditions we don't understand
-suggest_report_error(){
-    # ex "error: something sad happened :(" >&2
-    echo "       please report this @ https://github.com/nixos/nix/issues" >&2
+generate_mount_daemon() {
+    local cmd_type="$1" # encrypted|unencrypted
+    local volume_uuid="$2"
+    cat <<EOF
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple Computer//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+  <key>RunAtLoad</key>
+  <true/>
+  <key>Label</key>
+  <string>org.nixos.darwin-store</string>
+  <key>ProgramArguments</key>
+  <array>
+$(generate_mount_command "$cmd_type" "$volume_uuid")
+  </array>
+</dict>
+</plist>
+EOF
 }
 
-main() {
-    (
-        echo ""
-        echo "     ------------------------------------------------------------------ "
-        echo "    | This installer will create a volume for the nix store and        |"
-        echo "    | configure it to mount at /nix.  Follow these steps to uninstall. |"
-        echo "     ------------------------------------------------------------------ "
-        echo ""
-        echo "  1. Remove the entry from fstab using 'sudo vifs'"
-        echo "  2. Destroy the data volume using 'diskutil apfs deleteVolume'"
-        echo "  3. Remove the 'nix' line from /etc/synthetic.conf or the file"
-        echo ""
-    ) >&2
+_eat_bootout_err() {
+    /usr/bin/grep -v "Boot-out failed: 36: Operation now in progress"
+}
 
-    if test_nix_symlink; then
-        echo "error: /nix is a symlink, please remove it and make sure it's not in synthetic.conf (in which case a reboot is required)" >&2
-        echo "  /nix -> $(readlink "/nix")" >&2
-        exit 2
+# TODO: remove with --uninstall?
+uninstall_launch_daemon_directions() {
+    local daemon_label="$1" # i.e., org.nixos.blah-blah
+    local daemon_plist="$2" # abspath
+    substep "Uninstall LaunchDaemon $daemon_label" \
+      "  sudo launchctl bootout system/$daemon_label" \
+      "  sudo rm $daemon_plist"
+}
+
+uninstall_launch_daemon_prompt() {
+    local daemon_label="$1" # i.e., org.nixos.blah-blah
+    local daemon_plist="$2" # abspath
+    local reason_for_daemon="$3"
+    cat <<EOF
+
+The installer adds a LaunchDaemon to $reason_for_daemon: $daemon_label
+EOF
+    if ui_confirm "Can I remove it?"; then
+        _sudo "to terminate the daemon" \
+            launchctl bootout "system/$daemon_label" 2> >(_eat_bootout_err >&2) || true
+            # this can "fail" with a message like:
+            # Boot-out failed: 36: Operation now in progress
+        _sudo "to remove the daemon definition" rm "$daemon_plist"
+    fi
+}
+
+nix_volume_mountd_uninstall_directions() {
+    uninstall_launch_daemon_directions "org.nixos.darwin-store" \
+        "$NIX_VOLUME_MOUNTD_DEST"
+}
+
+nix_volume_mountd_uninstall_prompt() {
+    uninstall_launch_daemon_prompt "org.nixos.darwin-store" \
+        "$NIX_VOLUME_MOUNTD_DEST" \
+        "mount your Nix volume"
+}
+
+# TODO: move nix_daemon to install-darwin-multi-user if/when uninstall_launch_daemon_prompt moves up to install-multi-user
+nix_daemon_uninstall_prompt() {
+    uninstall_launch_daemon_prompt "org.nixos.nix-daemon" \
+        "$NIX_DAEMON_DEST" \
+        "run the nix-daemon"
+}
+
+# TODO: remove with --uninstall?
+nix_daemon_uninstall_directions() {
+    uninstall_launch_daemon_directions "org.nixos.nix-daemon" \
+        "$NIX_DAEMON_DEST"
+}
+
+
+# TODO: remove with --uninstall?
+synthetic_conf_uninstall_directions() {
+    # :1 to strip leading slash
+    substep "Remove ${NIX_ROOT:1} from /etc/synthetic.conf" \
+        "  If nix is the only entry: sudo rm /etc/synthetic.conf" \
+        "  Otherwise: sudo /usr/bin/sed -i '' -e '/^${NIX_ROOT:1}$/d' /etc/synthetic.conf"
+}
+
+synthetic_conf_uninstall_prompt() {
+    cat <<EOF
+
+During install, I add '${NIX_ROOT:1}' to /etc/synthetic.conf, which instructs
+macOS to create an empty root directory for mounting the Nix volume.
+EOF
+    # make the edit to a copy
+    /usr/bin/grep -vE "^${NIX_ROOT:1}($|\t.{3,}$)" /etc/synthetic.conf > "$SCRATCH/synthetic.conf.edit"
+
+    if test_synthetic_conf_symlinked; then
+        warning <<EOF
+
+/etc/synthetic.conf already contains a line instructing your system
+to make '${NIX_ROOT}' as a symlink:
+    $(/usr/bin/grep -nE "^${NIX_ROOT:1}\t.{3,}$" /etc/synthetic.conf)
+
+This may mean your system has/had a non-standard Nix install.
+
+The volume-creation process in this installer is *not* compatible
+with a symlinked store, so I'll have to remove this instruction to
+continue.
+
+If you want/need to keep this instruction, answer 'n' to abort.
+
+EOF
     fi
 
-    if ! test_synthetic_conf; then
-        echo "Configuring /etc/synthetic.conf..." >&2
-        echo nix | sudo tee -a /etc/synthetic.conf
-        if ! test_synthetic_conf; then
-            echo "error: failed to configure synthetic.conf;" >&2
-            suggest_report_error
-            exit 1
+    # ask to rm if this left the file empty aside from comments, else edit
+    if /usr/bin/diff -q <(:) <(/usr/bin/grep -v "^#" "$SCRATCH/synthetic.conf.edit") &>/dev/null; then
+        if confirm_rm "/etc/synthetic.conf"; then
+            if test_nix_root_is_symlink; then
+                failure >&2 <<EOF
+I removed /etc/synthetic.conf, but $NIX_ROOT is already a symlink
+(-> $(readlink "$NIX_ROOT")). The system should remove it when you reboot.
+Once you've rebooted, run the installer again.
+EOF
+            fi
+            return 0
+        fi
+    else
+        if confirm_edit "$SCRATCH/synthetic.conf.edit" "/etc/synthetic.conf"; then
+            if test_nix_root_is_symlink; then
+                failure >&2 <<EOF
+I edited Nix out of /etc/synthetic.conf, but $NIX_ROOT is already a symlink
+(-> $(readlink "$NIX_ROOT")). The system should remove it when you reboot.
+Once you've rebooted, run the installer again.
+EOF
+            fi
+            return 0
         fi
     fi
+    # fallback instructions
+    echo "Manually remove nix from /etc/synthetic.conf"
+    return 1
+}
 
-    if ! test_nix; then
-        echo "Creating mountpoint for /nix..." >&2
-        create_synthetic_objects # the ones we defined in synthetic.conf
-        if ! test_nix; then
-            sudo mkdir -p /nix 2>/dev/null || true
+add_nix_vol_fstab_line() {
+    local uuid="$1"
+    # shellcheck disable=SC1003,SC2026
+    local escaped_mountpoint="${NIX_ROOT/ /'\\\'040}"
+    shift
+    EDITOR="/usr/bin/ex" _sudo "to add nix to fstab" "$@" <<EOF
+:a
+UUID=$uuid $escaped_mountpoint apfs rw,noauto,nobrowse,suid,owners
+.
+:x
+EOF
+    # TODO: preserving my notes on suid,owners above until resolved
+    # There *may* be some issue regarding volume ownership, see nix#3156
+    #
+    # It seems like the cheapest fix is adding "suid,owners" to fstab, but:
+    # - We don't have much info on this condition yet
+    # - I'm not certain if these cause other problems?
+    # - There's a "chown" component some people claim to need to fix this
+    #   that I don't understand yet
+    #   (Note however that I've had to add a chown step to handle
+    #   single->multi-user reinstalls, which may cover this)
+    #
+    # I'm not sure if it's safe to approach this way?
+    #
+    # I think I think the most-proper way to test for it is:
+    # diskutil info -plist "$NIX_VOLUME_LABEL" | xmllint --xpath "(/plist/dict/key[text()='GlobalPermissionsEnabled'])/following-sibling::*[1][name()='true']" -; echo $?
+    #
+    # There's also `sudo /usr/sbin/vsdbutil -c /path` (which is much faster, but is also
+    # deprecated and needs minor parsing).
+    #
+    # If no one finds a problem with doing so, I think the simplest approach
+    # is to just eagerly set this. I found a few imperative approaches:
+    # (diskutil enableOwnership, ~100ms), a cheap one (/usr/sbin/vsdbutil -a, ~40-50ms),
+    # a very cheap one (append the internal format to /var/db/volinfo.database).
+    #
+    # But vsdbutil's deprecation notice suggests using fstab, so I want to
+    # give that a whirl first.
+    #
+    # TODO: when this is workable, poke infinisil about reproducing the issue
+    # and confirming this fix?
+}
+
+delete_nix_vol_fstab_line() {
+    # TODO: I'm scaffolding this to handle the new nix volumes
+    # but it might be nice to generalize a smidge further to
+    # go ahead and set up a pattern for curing "old" things
+    # we no longer do?
+    EDITOR="/usr/bin/patch" _sudo "to cut nix from fstab" "$@" < <(/usr/bin/diff /etc/fstab <(/usr/bin/grep -v "$NIX_ROOT apfs rw" /etc/fstab))
+    # leaving some parts out of the grep; people may fiddle this a little?
+}
+
+# TODO: hope to remove with --uninstall
+fstab_uninstall_directions() {
+    substep "Remove ${NIX_ROOT} from /etc/fstab" \
+      "  If nix is the only entry: sudo rm /etc/fstab" \
+      "  Otherwise, run 'sudo /usr/sbin/vifs' to remove the nix line"
+}
+
+fstab_uninstall_prompt() {
+    cat <<EOF
+During install, I add '${NIX_ROOT}' to /etc/fstab so that macOS knows what
+mount options to use for the Nix volume.
+EOF
+    cp /etc/fstab "$SCRATCH/fstab.edit"
+    # technically doesn't need the _sudo path, but throwing away the
+    # output is probably better than mostly-duplicating the code...
+    delete_nix_vol_fstab_line patch "$SCRATCH/fstab.edit" &>/dev/null
+
+    # if the patch test edit, minus comment lines, is equal to empty (:)
+    if /usr/bin/diff -q <(:) <(/usr/bin/grep -v "^#" "$SCRATCH/fstab.edit") &>/dev/null; then
+        # this edit would leave it empty; propose deleting it
+        if confirm_rm "/etc/fstab"; then
+            return 0
+        else
+            echo "Remove nix from /etc/fstab (or remove the file)"
         fi
-        if ! test_nix; then
-            echo "error: failed to bootstrap /nix; if a reboot doesn't help," >&2
-            suggest_report_error
-            exit 1
+    else
+        echo "I might be able to help you make this edit. Here's the diff:"
+        if ! _diff "/etc/fstab" "$SCRATCH/fstab.edit" && ui_confirm "Does the change above look right?"; then
+            delete_nix_vol_fstab_line /usr/sbin/vifs
+        else
+            echo "Remove nix from /etc/fstab (or remove the file)"
         fi
     fi
+}
 
-    disk="$(root_disk_identifier)"
-    volume=$(find_nix_volume "$disk")
-    if [ -z "$volume" ]; then
-        echo "Creating a Nix Store volume..." >&2
+remove_volume() {
+    local volume_special="$1" # (i.e., disk1s7)
+    _sudo "to unmount the Nix volume" \
+        /usr/sbin/diskutil unmount force "$volume_special" || true # might not be mounted
+    _sudo "to delete the Nix volume" \
+        /usr/sbin/diskutil apfs deleteVolume "$volume_special"
+}
 
-        if test_filevault_in_use; then
-            # TODO: Not sure if it's in-scope now, but `diskutil apfs list`
-            # shows both filevault and encrypted at rest status, and it
-            # may be the more semantic way to test for this? It'll show
-            # `FileVault:                 No (Encrypted at rest)`
-            # `FileVault:                 No`
-            # `FileVault:                 Yes (Unlocked)`
-            # and so on.
-            if test_t2_chip_present; then
-                echo "warning: boot volume is FileVault-encrypted, but the Nix store volume" >&2
-                echo "         is only encrypted at rest." >&2
-                echo "         See https://nixos.org/nix/manual/#sect-macos-installation" >&2
+# aspiration: robust enough to both fix problems
+# *and* update older darwin volumes
+cure_volume() {
+    local volume_special="$1" # (i.e., disk1s7)
+    local volume_uuid="$2"
+    header "Found existing Nix volume"
+    row "  special" "$volume_special"
+    row "     uuid" "$volume_uuid"
+
+    if volume_encrypted "$volume_special"; then
+        row "encrypted" "yes"
+        if volume_pass_works "$volume_special" "$volume_uuid"; then
+            NIX_VOLUME_DO_ENCRYPT=0
+            ok "Found a working decryption password in keychain :)"
+            echo ""
+        else
+            # - this is a volume we made, and
+            #   - the user encrypted it on their own
+            #   - something deleted the credential
+            # - this is an old or BYO volume and the pw
+            #   just isn't somewhere we can find it.
+            #
+            # We're going to explain why we're freaking out
+            # and prompt them to either delete the volume
+            # (requiring a sudo auth), or abort to fix
+            warning <<EOF
+
+This volume is encrypted, but I don't see a password to decrypt it.
+The quick fix is to let me delete this volume and make you a new one.
+If that's okay, enter your (sudo) password to continue. If not, you
+can ensure the decryption password is in your system keychain with a
+"Where" (service) field set to this volume's UUID:
+  $volume_uuid
+EOF
+            if password_confirm "delete this volume"; then
+                remove_volume "$volume_special"
             else
-                echo "error: refusing to create Nix store volume because the boot volume is" >&2
-                echo "       FileVault encrypted, but encryption-at-rest is not available." >&2
-                echo "       Manually create a volume for the store and re-run this script." >&2
-                echo "       See https://nixos.org/nix/manual/#sect-macos-installation" >&2
-                exit 1
+                # TODO: this is a good design case for a warn-and
+                # remind idiom...
+                failure <<EOF
+Your Nix volume is encrypted, but I couldn't find its password. Either:
+- Delete or rename the volume out of the way
+- Ensure its decryption password is in the system keychain with a
+  "Where" (service) field set to this volume's UUID:
+    $volume_uuid
+EOF
+            fi
+        fi
+    elif test_filevault_in_use; then
+        row "encrypted" "no"
+        warning <<EOF
+FileVault is on, but your $NIX_VOLUME_LABEL volume isn't encrypted.
+EOF
+        # if we're interactive, give them a chance to
+        # encrypt the volume. If not, /shrug
+        if ! headless && (( NIX_VOLUME_DO_ENCRYPT == 1 )); then
+            if ui_confirm "Should I encrypt it and add the decryption key to your keychain?"; then
+                encrypt_volume "$volume_uuid" "$NIX_VOLUME_LABEL"
+                NIX_VOLUME_DO_ENCRYPT=0
+            else
+                NIX_VOLUME_DO_ENCRYPT=0
+                reminder "FileVault is on, but your $NIX_VOLUME_LABEL volume isn't encrypted."
             fi
         fi
-
-        sudo diskutil apfs addVolume "$disk" APFS 'Nix Store' -mountpoint /nix
-        volume="Nix Store"
     else
-        echo "Using existing '$volume' volume" >&2
-    fi
-
-    if ! test_fstab; then
-        echo "Configuring /etc/fstab..." >&2
-        label=$(echo "$volume" | sed 's/ /\\040/g')
-        # shellcheck disable=SC2209
-        printf "\$a\nLABEL=%s /nix apfs rw,nobrowse\n.\nwq\n" "$label" | EDITOR=ed sudo vifs
+        row "encrypted" "no"
     fi
 }
 
-main "$@"
+remove_volume_artifacts() {
+    if test_synthetic_conf_either; then
+        # NIX_ROOT is in synthetic.conf
+        if synthetic_conf_uninstall_prompt; then
+            # TODO: moot until we tackle uninstall, but when we're
+            # actually uninstalling, we should issue:
+            # reminder "macOS will clean up the empty mount-point directory at $NIX_ROOT on reboot."
+            :
+        fi
+    fi
+    if test_fstab; then
+        fstab_uninstall_prompt
+    fi
+
+    if test_nix_volume_mountd_installed; then
+        nix_volume_mountd_uninstall_prompt
+    fi
+}
+
+setup_synthetic_conf() {
+    if test_nix_root_is_symlink; then
+        if ! test_synthetic_conf_symlinked; then
+            failure >&2 <<EOF
+error: $NIX_ROOT is a symlink (-> $(readlink "$NIX_ROOT")).
+Please remove it. If nix is in /etc/synthetic.conf, remove it and reboot.
+EOF
+        fi
+    fi
+    if ! test_synthetic_conf_mountable; then
+        task "Configuring /etc/synthetic.conf to make a mount-point at $NIX_ROOT" >&2
+        # technically /etc/synthetic.d/nix is supported in Big Sur+
+        # but handling both takes even more code...
+        _sudo "to add Nix to /etc/synthetic.conf" \
+            /usr/bin/ex /etc/synthetic.conf <<EOF
+:a
+${NIX_ROOT:1}
+.
+:x
+EOF
+        if ! test_synthetic_conf_mountable; then
+            failure "error: failed to configure synthetic.conf" >&2
+        fi
+        create_synthetic_objects
+        if ! test_nix; then
+            failure >&2 <<EOF
+error: failed to bootstrap $NIX_ROOT
+If you enabled FileVault after booting, this is likely a known issue
+with macOS that you'll have to reboot to fix. If you didn't enable FV,
+though, please open an issue describing how the system that you see
+this error on was set up.
+EOF
+        fi
+    fi
+}
+
+setup_fstab() {
+    local volume_uuid="$1"
+    # fstab used to be responsible for mounting the volume. Now the last
+    # step adds a LaunchDaemon responsible for mounting. This is technically
+    # redundant for mounting, but diskutil appears to pick up mount options
+    # from fstab (and diskutil's support for specifying them directly is not
+    # consistent across versions/subcommands).
+    if ! test_fstab; then
+        task "Configuring /etc/fstab to specify volume mount options" >&2
+        add_nix_vol_fstab_line "$volume_uuid" /usr/sbin/vifs
+    fi
+}
+
+encrypt_volume() {
+    local volume_uuid="$1"
+    local volume_label="$2"
+    local password
+    # Note: mount/unmount are late additions to support the right order
+    # of operations for creating the volume and then baking its uuid into
+    # other artifacts; not as well-trod wrt to potential errors, race
+    # conditions, etc.
+
+    /usr/sbin/diskutil mount "$volume_label"
+
+    password="$(/usr/bin/xxd -l 32 -p -c 256 /dev/random)"
+    _sudo "to add your Nix volume's password to Keychain" \
+        /usr/bin/security -i <<EOF
+add-generic-password -a "$volume_label" -s "$volume_uuid" -l "$volume_label encryption password" -D "Encrypted volume password" -j "Added automatically by the Nix installer for use by $NIX_VOLUME_MOUNTD_DEST" -w "$password" -T /System/Library/CoreServices/APFSUserAgent -T /System/Library/CoreServices/CSUserAgent -T /usr/bin/security "/Library/Keychains/System.keychain"
+EOF
+    builtin printf "%s" "$password" | _sudo "to encrypt your Nix volume" \
+        /usr/sbin/diskutil apfs encryptVolume "$volume_label" -user disk -stdinpassphrase
+
+    /usr/sbin/diskutil unmount force "$volume_label"
+}
+
+create_volume() {
+    # Notes:
+    # 1) using `-nomount` instead of `-mountpoint "$NIX_ROOT"` to get
+    # its UUID and set mount opts in fstab before first mount
+    #
+    # 2) system is in some sense less secure than user keychain... (it's
+    # possible to read the password for decrypting the keychain) but
+    # the user keychain appears to be available too late. As far as I
+    # can tell, the file with this password (/var/db/SystemKey) is
+    # inside the FileVault envelope. If that isn't true, it may make
+    # sense to store the password inside the envelope?
+    #
+    # 3) At some point it would be ideal to have a small binary to serve
+    # as the daemon itself, and for it to replace /usr/bin/security here.
+    #
+    # 4) *UserAgent exemptions should let the system seamlessly supply the
+    # password if noauto is removed from fstab entry. This is intentional;
+    # the user will hopefully look for help if the volume stops mounting,
+    # rather than failing over into subtle race-condition problems.
+    #
+    # 5) If we ever get users griping about not having space to do
+    # anything useful with Nix, it is possibly to specify
+    # `-reserve 10g` or something, which will fail w/o that much
+    #
+    # 6) getting special w/ awk may be fragile, but doing it to:
+    #    - save time over running slow diskutil commands
+    #    - skirt risk we grab wrong volume if multiple match
+    /usr/sbin/diskutil apfs addVolume "$NIX_VOLUME_USE_DISK" "$NIX_VOLUME_FS" "$NIX_VOLUME_LABEL" -nomount | /usr/bin/awk '/Created new APFS Volume/ {print $5}'
+}
+
+volume_uuid_from_special() {
+    local volume_special="$1" # (i.e., disk1s7)
+    # For reasons I won't pretend to fathom, this returns 253 when it works
+    /System/Library/Filesystems/apfs.fs/Contents/Resources/apfs.util -k "$volume_special" || true
+}
+
+# this sometimes clears immediately, and AFAIK clears
+# within about 1s. diskutil info on an unmounted path
+# fails in around 50-100ms and a match takes about
+# 250-300ms. I suspect it's usually ~250-750ms
+await_volume() {
+    # caution: this could, in theory, get stuck
+    until /usr/sbin/diskutil info "$NIX_ROOT" &>/dev/null; do
+        :
+    done
+}
+
+setup_volume() {
+    local use_special use_uuid profile_packages
+    task "Creating a Nix volume" >&2
+    # DOING: I'm tempted to wrap this call in a grep to get the new disk special without doing anything too complex, but this sudo wrapper *is* a little complex, so it'll be a PITA unless maybe we can skip sudo on this. Let's just try it without.
+
+    use_special="${NIX_VOLUME_USE_SPECIAL:-$(create_volume)}"
+
+    use_uuid=${NIX_VOLUME_USE_UUID:-$(volume_uuid_from_special "$use_special")}
+
+    setup_fstab "$use_uuid"
+
+    if should_encrypt_volume; then
+        encrypt_volume "$use_uuid" "$NIX_VOLUME_LABEL"
+        setup_volume_daemon "encrypted" "$use_uuid"
+    # TODO: might be able to save ~60ms by caching or setting
+    # this somewhere rather than re-checking here.
+    elif volume_encrypted "$use_special"; then
+        setup_volume_daemon "encrypted" "$use_uuid"
+    else
+        setup_volume_daemon "unencrypted" "$use_uuid"
+    fi
+
+    await_volume
+
+    # TODO: below is a vague kludge for now; I just don't know
+    # what if any safe action there is to take here. Also, the
+    # reminder isn't very helpful.
+    # I'm less sure where this belongs, but it also wants mounted, pre-install
+    if type -p nix-env; then
+        profile_packages="$(nix-env --query --installed)"
+        # TODO: can probably do below faster w/ read
+        # intentionally unquoted string to eat whitespace in wc output
+        # shellcheck disable=SC2046,SC2059
+        if ! [ $(printf "$profile_packages" | /usr/bin/wc -l) = "0" ]; then
+            reminder <<EOF
+Nix now supports only multi-user installs on Darwin/macOS, and your user's
+Nix profile has some packages in it. These packages may obscure those in the
+default profile, including the Nix this installer will add. You should
+review these packages:
+$profile_packages
+EOF
+        fi
+    fi
+
+}
+
+setup_volume_daemon() {
+    local cmd_type="$1" # encrypted|unencrypted
+    local volume_uuid="$2"
+    if ! test_voldaemon; then
+        task "Configuring LaunchDaemon to mount '$NIX_VOLUME_LABEL'" >&2
+        _sudo "to install the Nix volume mounter" /usr/bin/ex "$NIX_VOLUME_MOUNTD_DEST" <<EOF
+:a
+$(generate_mount_daemon "$cmd_type" "$volume_uuid")
+.
+:x
+EOF
+
+        # TODO: should probably alert the user if this is disabled?
+        _sudo "to launch the Nix volume mounter" \
+            launchctl bootstrap system "$NIX_VOLUME_MOUNTD_DEST" || true
+        # TODO: confirm whether kickstart is necessesary?
+        # I feel a little superstitous, but it can guard
+        # against multiple problems (doesn't start, old
+        # version still running for some reason...)
+        _sudo "to launch the Nix volume mounter" \
+            launchctl kickstart -k system/org.nixos.darwin-store
+    fi
+}
+
+setup_darwin_volume() {
+    setup_synthetic_conf
+    setup_volume
+}
+
+if [ "$_CREATE_VOLUME_NO_MAIN" = 1 ]; then
+    if [ -n "$*" ]; then
+        "$@" # expose functions in case we want multiple routines?
+    fi
+else
+    # no reason to pay for bash to process this
+    main() {
+        {
+            echo ""
+            echo "     ------------------------------------------------------------------ "
+            echo "    | This installer will create a volume for the nix store and        |"
+            echo "    | configure it to mount at $NIX_ROOT.  Follow these steps to uninstall. |"
+            echo "     ------------------------------------------------------------------ "
+            echo ""
+            echo "  1. Remove the entry from fstab using 'sudo /usr/sbin/vifs'"
+            echo "  2. Run 'sudo launchctl bootout system/org.nixos.darwin-store'"
+            echo "  3. Remove $NIX_VOLUME_MOUNTD_DEST"
+            echo "  4. Destroy the data volume using '/usr/sbin/diskutil apfs deleteVolume'"
+            echo "  5. Remove the 'nix' line from /etc/synthetic.conf (or the file)"
+            echo ""
+        } >&2
+
+        setup_darwin_volume
+    }
+
+    main "$@"
+fi

scripts/install-darwin-multi-user.sh

@@ -3,57 +3,99 @@
 set -eu
 set -o pipefail
 
-readonly PLIST_DEST=/Library/LaunchDaemons/org.nixos.nix-daemon.plist
+readonly NIX_DAEMON_DEST=/Library/LaunchDaemons/org.nixos.nix-daemon.plist
+# create by default; set 0 to DIY, use a symlink, etc.
+readonly NIX_VOLUME_CREATE=${NIX_VOLUME_CREATE:-1} # now default
+NIX_FIRST_BUILD_UID="301"
+NIX_BUILD_USER_NAME_TEMPLATE="_nixbld%d"
+
+# caution: may update times on / if not run as normal non-root user
+read_only_root() {
+    # this touch command ~should~ always produce an error
+    # as of this change I confirmed /usr/bin/touch emits:
+    # "touch: /: Read-only file system" Catalina+ and Big Sur
+    # "touch: /: Permission denied" Mojave
+    # (not matching prefix for compat w/ coreutils touch in case using
+    # an explicit path causes problems; its prefix differs)
+    [[ "$(/usr/bin/touch / 2>&1)" = *"Read-only file system" ]]
+
+    # Avoiding the slow semantic way to get this information (~330ms vs ~8ms)
+    # unless using touch causes problems. Just in case, that approach is:
+    # diskutil info -plist / | <find the Writable or WritableVolume keys>, i.e.
+    # diskutil info -plist / | xmllint --xpath "name(/plist/dict/key[text()='Writable']/following-sibling::*[1])" -
+}
+
+if read_only_root && [ "$NIX_VOLUME_CREATE" = 1 ]; then
+    should_create_volume() { return 0; }
+else
+    should_create_volume() { return 1; }
+fi
+
+# shellcheck source=./create-darwin-volume.sh
+. "$EXTRACTED_NIX_PATH/create-darwin-volume.sh" "no-main"
 
 dsclattr() {
     /usr/bin/dscl . -read "$1" \
-        | awk "/$2/ { print \$2 }"
+        | /usr/bin/awk "/$2/ { print \$2 }"
 }
 
-poly_validate_assumptions() {
-    if [ "$(uname -s)" != "Darwin" ]; then
-        failure "This script is for use with macOS!"
+test_nix_daemon_installed() {
+  test -e "$NIX_DAEMON_DEST"
+}
+
+poly_cure_artifacts() {
+    if should_create_volume; then
+        task "Fixing any leftover Nix volume state"
+        cat <<EOF
+Before I try to install, I'll check for any existing Nix volume config
+and ask for your permission to remove it (so that the installer can
+start fresh). I'll also ask for permission to fix any issues I spot.
+EOF
+        cure_volumes
+        remove_volume_artifacts
     fi
 }
 
 poly_service_installed_check() {
-    [ -e "$PLIST_DEST" ]
+    if should_create_volume; then
+        test_nix_daemon_installed || test_nix_volume_mountd_installed
+    else
+        test_nix_daemon_installed
+    fi
 }
 
 poly_service_uninstall_directions() {
-        cat <<EOF
-$1. Delete $PLIST_DEST
-
-  sudo launchctl unload $PLIST_DEST
-  sudo rm $PLIST_DEST
-
-EOF
+    echo "$1. Remove macOS-specific components:"
+    if should_create_volume && test_nix_volume_mountd_installed; then
+        darwin_volume_uninstall_directions
+    fi
+    if test_nix_daemon_installed; then
+        nix_daemon_uninstall_directions
+    fi
 }
 
 poly_service_setup_note() {
-    cat <<EOF
- - load and start a LaunchDaemon (at $PLIST_DEST) for nix-daemon
-
-EOF
+    if should_create_volume; then
+        echo " - create a Nix volume and a LaunchDaemon to mount it"
+    fi
+    echo " - create a LaunchDaemon (at $NIX_DAEMON_DEST) for nix-daemon"
+    echo ""
 }
 
-poly_extra_try_me_commands(){
-  :
-}
-poly_extra_setup_instructions(){
-  :
+poly_extra_try_me_commands() {
+    :
 }
 
 poly_configure_nix_daemon_service() {
+    task "Setting up the nix-daemon LaunchDaemon"
     _sudo "to set up the nix-daemon as a LaunchDaemon" \
-          cp -f "/nix/var/nix/profiles/default$PLIST_DEST" "$PLIST_DEST"
+          /bin/cp -f "/nix/var/nix/profiles/default$NIX_DAEMON_DEST" "$NIX_DAEMON_DEST"
 
     _sudo "to load the LaunchDaemon plist for nix-daemon" \
           launchctl load /Library/LaunchDaemons/org.nixos.nix-daemon.plist
 
     _sudo "to start the nix-daemon" \
-          launchctl start org.nixos.nix-daemon
-
+          launchctl kickstart -k system/org.nixos.nix-daemon
 }
 
 poly_group_exists() {
@@ -94,6 +136,8 @@ poly_user_home_get() {
 }
 
 poly_user_home_set() {
+    # This can trigger a permission prompt now:
+    # "Terminal" would like to administer your computer. Administration can include modifying passwords, networking, and system settings.
     _sudo "in order to give $1 a safe home directory" \
           /usr/bin/dscl . -create "/Users/$1" "NFSHomeDirectory" "$2"
 }
@@ -119,7 +163,7 @@ poly_user_shell_set() {
 poly_user_in_group_check() {
     username=$1
     group=$2
-    dseditgroup -o checkmember -m "$username" "$group" > /dev/null 2>&1
+    /usr/sbin/dseditgroup -o checkmember -m "$username" "$group" > /dev/null 2>&1
 }
 
 poly_user_in_group_set() {
@@ -149,3 +193,17 @@ poly_create_build_user() {
           /usr/bin/dscl . create "/Users/$username" \
           UniqueID "${uid}"
 }
+
+poly_prepare_to_install() {
+    if should_create_volume; then
+        header "Preparing a Nix volume"
+        # intentional indent below to match task indent
+        cat <<EOF
+    Nix traditionally stores its data in the root directory $NIX_ROOT, but
+    macOS now (starting in 10.15 Catalina) has a read-only root directory.
+    To support Nix, I will create a volume and configure macOS to mount it
+    at $NIX_ROOT.
+EOF
+        setup_darwin_volume
+    fi
+}

scripts/install-multi-user.sh

@@ -25,13 +25,15 @@ readonly RED='\033[31m'
 readonly NIX_USER_COUNT=${NIX_USER_COUNT:-32}
 readonly NIX_BUILD_GROUP_ID="30000"
 readonly NIX_BUILD_GROUP_NAME="nixbld"
-readonly NIX_FIRST_BUILD_UID="30001"
+# darwin installer needs to override these
+NIX_FIRST_BUILD_UID="30001"
+NIX_BUILD_USER_NAME_TEMPLATE="nixbld%d"
 # Please don't change this. We don't support it, because the
 # default shell profile that comes with Nix doesn't support it.
 readonly NIX_ROOT="/nix"
 readonly NIX_EXTRA_CONF=${NIX_EXTRA_CONF:-}
 
-readonly PROFILE_TARGETS=("/etc/bashrc" "/etc/profile.d/nix.sh" "/etc/zshenv")
+readonly PROFILE_TARGETS=("/etc/bashrc" "/etc/profile.d/nix.sh" "/etc/zshenv" "/etc/bash.bashrc" "/etc/zsh/zshenv")
 readonly PROFILE_BACKUP_SUFFIX=".backup-before-nix"
 readonly PROFILE_NIX_FILE="$NIX_ROOT/var/nix/profiles/default/etc/profile.d/nix-daemon.sh"
 
@@ -41,7 +43,7 @@ readonly NIX_INSTALLED_CACERT="@cacert@"
 #readonly NIX_INSTALLED_CACERT="/nix/store/7dxhzymvy330i28ii676fl1pqwcahv2f-nss-cacert-3.49.2"
 readonly EXTRACTED_NIX_PATH="$(dirname "$0")"
 
-readonly ROOT_HOME=$(echo ~root)
+readonly ROOT_HOME=~root
 
 if [ -t 0 ]; then
     readonly IS_HEADLESS='no'
@@ -57,14 +59,19 @@ headless() {
     fi
 }
 
-contactme() {
+contact_us() {
+    echo "You can open an issue at https://github.com/nixos/nix/issues"
+    echo ""
+    echo "Or feel free to contact the team:"
+    echo " - Matrix: #nix:nixos.org"
+    echo " - IRC: in #nixos on irc.libera.chat"
+    echo " - twitter: @nixos_org"
+    echo " - forum: https://discourse.nixos.org"
+}
+get_help() {
     echo "We'd love to help if you need it."
     echo ""
-    echo "If you can, open an issue at https://github.com/nixos/nix/issues"
-    echo ""
-    echo "Or feel free to contact the team,"
-    echo " - on IRC #nixos on irc.freenode.net"
-    echo " - on twitter @nixos_org"
+    contact_us
 }
 
 uninstall_directions() {
@@ -100,11 +107,10 @@ $step. Delete the files Nix added to your system:
 and that is it.
 
 EOF
-
 }
 
 nix_user_for_core() {
-    printf "nixbld%d" "$1"
+    printf "$NIX_BUILD_USER_NAME_TEMPLATE" "$1"
 }
 
 nix_uid_for_core() {
@@ -168,7 +174,7 @@ failure() {
     header "oh no!"
     _textout "$RED" "$@"
     echo ""
-    _textout "$RED" "$(contactme)"
+    _textout "$RED" "$(get_help)"
     trap finish_cleanup EXIT
     exit 1
 }
@@ -199,6 +205,95 @@ ui_confirm() {
     return 1
 }
 
+printf -v _UNCHANGED_GRP_FMT "%b" $'\033[2m%='"$ESC" # "dim"
+# bold+invert+red and bold+invert+green just for the +/- below
+# red/green foreground for rest of the line
+printf -v _OLD_LINE_FMT "%b" $'\033[1;7;31m-'"$ESC ${RED}%L${ESC}"
+printf -v _NEW_LINE_FMT "%b" $'\033[1;7;32m+'"$ESC ${GREEN}%L${ESC}"
+
+_diff() {
+    # simple colorized diff comatible w/ pre `--color` versions
+    diff --unchanged-group-format="$_UNCHANGED_GRP_FMT" --old-line-format="$_OLD_LINE_FMT" --new-line-format="$_NEW_LINE_FMT" --unchanged-line-format="  %L" "$@"
+}
+
+confirm_rm() {
+    local path="$1"
+    if ui_confirm "Can I remove $path?"; then
+        _sudo "to remove $path" rm "$path"
+    fi
+}
+
+confirm_edit() {
+    local path="$1"
+    local edit_path="$2"
+    cat <<EOF
+
+Nix isn't the only thing in $path,
+but I think I know how to edit it out.
+Here's the diff:
+EOF
+
+    # could technically test the diff, but caller should do it
+    _diff "$path" "$edit_path"
+    if ui_confirm "Does the change above look right?"; then
+        _sudo "remove nix from $path" cp "$edit_path" "$path"
+    fi
+}
+
+_SERIOUS_BUSINESS="${RED}%s:${ESC} "
+password_confirm() {
+    local do_something_consequential="$1"
+    if ui_confirm "Can I $do_something_consequential?"; then
+        # shellcheck disable=SC2059
+        sudo -kv --prompt="$(printf "${_SERIOUS_BUSINESS}" "Enter your password to $do_something_consequential")"
+    else
+        return 1
+    fi
+}
+
+# Support accumulating reminders over the course of a run and showing
+# them at the end. An example where this helps: the installer changes
+# something, but it won't work without a reboot. If you tell the user
+# when you do it, they may miss it in the stream. The value of the
+# setting isn't enough to decide whether to message because you only
+# need to message if you *changed* it.
+
+# reminders stored in array delimited by empty entry; if ! headless,
+# user is asked to confirm after each delimiter.
+_reminders=()
+((_remind_num=1))
+
+remind() {
+    # (( arithmetic expression ))
+    if (( _remind_num > 1 )); then
+        header "Reminders"
+        for line in "${_reminders[@]}"; do
+            echo "$line"
+            if ! headless && [ "${#line}" = 0 ]; then
+                if read -r -p "Press enter/return to acknowledge."; then
+                    printf $'\033[A\33[2K\r'
+                fi
+            fi
+        done
+    fi
+}
+
+reminder() {
+    printf -v label "${BLUE}[ %d ]${ESC}" "$_remind_num"
+    _reminders+=("$label")
+    if [[ "$*" = "" ]]; then
+        while read -r line; do
+            _reminders+=("$line")
+        done
+    else
+        # this expands each arg to an array entry (and each entry will
+        # ultimately be a separate line in the output)
+        _reminders+=("$@")
+    fi
+    _reminders+=("")
+    ((_remind_num++))
+}
+
 __sudo() {
     local expl="$1"
     local cmd="$2"
@@ -219,18 +314,18 @@ _sudo() {
     local expl="$1"
     shift
     if ! headless; then
-        __sudo "$expl" "$*"
+        __sudo "$expl" "$*" >&2
     fi
     sudo "$@"
 }
 
 
-readonly SCRATCH=$(mktemp -d -t tmp.XXXXXXXXXX)
-function finish_cleanup {
+readonly SCRATCH=$(mktemp -d "${TMPDIR:-/tmp/}tmp.XXXXXXXXXX")
+finish_cleanup() {
     rm -rf "$SCRATCH"
 }
 
-function finish_fail {
+finish_fail() {
     finish_cleanup
 
     failure <<EOF
@@ -242,45 +337,46 @@ EOF
 }
 trap finish_fail EXIT
 
-channel_update_failed=0
-function finish_success {
-    finish_cleanup
-
+finish_success() {
     ok "Alright! We're done!"
-    if [ "x$channel_update_failed" = x1 ]; then
-        echo ""
-        echo "But fetching the nixpkgs channel failed. (Are you offline?)"
-        echo "To try again later, run \"sudo -i nix-channel --update nixpkgs\"."
-    fi
 
     cat <<EOF
-
-Before Nix will work in your existing shells, you'll need to close
-them and open them again. Other than that, you should be ready to go.
-
 Try it! Open a new terminal, and type:
 $(poly_extra_try_me_commands)
   $ nix-shell -p nix-info --run "nix-info -m"
-$(poly_extra_setup_instructions)
-Thank you for using this installer. If you have any feedback, don't
-hesitate:
 
-$(contactme)
+Thank you for using this installer. If you have any feedback or need
+help, don't hesitate:
+
+$(contact_us)
 EOF
-
+    remind
+    finish_cleanup
 }
 
+finish_uninstall_success() {
+    ok "Alright! Nix should be removed!"
+
+    cat <<EOF
+If you spot anything this uninstaller missed or have feedback,
+don't hesitate:
+
+$(contact_us)
+EOF
+    remind
+    finish_cleanup
+}
+
+remove_nix_artifacts() {
+    failure "Not implemented yet"
+}
+
+cure_artifacts() {
+    poly_cure_artifacts
+    # remove_nix_artifacts (LATER)
+}
 
 validate_starting_assumptions() {
-    poly_validate_assumptions
-
-    if [ $EUID -eq 0 ]; then
-        failure <<EOF
-Please do not run this script with root privileges. We will call sudo
-when we need to.
-EOF
-    fi
-
     if type nix-env 2> /dev/null >&2; then
         warning <<EOF
 Nix already appears to be installed. This installer may run into issues.
@@ -442,18 +538,46 @@ create_build_users() {
 
 create_directories() {
     # FIXME: remove all of this because it duplicates LocalStore::LocalStore().
+    task "Setting up the basic directory structure"
+    if [ -d "$NIX_ROOT" ]; then
+        # if /nix already exists, take ownership
+        #
+        # Caution: notes below are macOS-y
+        # This is a bit of a goldilocks zone for taking ownership
+        # if there are already files on the volume; the volume is
+        # now mounted, but we haven't added a bunch of new files
 
+        # this is probably a bit slow; I've been seeing 3.3-4s even
+        # when promptly installed over a fresh single-user install.
+        # In case anyone's aware of a shortcut.
+        # `|| true`: .Trashes errors w/o full disk perm
+
+        # rumor per #4488 that macOS 11.2 may not have
+        # sbin on path, and that's where chown is, but
+        # since this bit is cross-platform:
+        # - first try with `command -vp` to try and find
+        #   chown in the usual places
+        # - fall back on `command -v` which would find
+        #   any chown on path
+        # if we don't find one, the command is already
+        # hiding behind || true, and the general state
+        # should be one the user can repair once they
+        # figure out where chown is...
+        local get_chr_own="$(command -vp chown)"
+        if [[ -z "$get_chr_own" ]]; then
+            get_chr_own="$(command -v chown)"
+        fi
+        _sudo "to take root ownership of existing Nix store files" \
+              "$get_chr_own" -R "root:$NIX_BUILD_GROUP_NAME" "$NIX_ROOT" || true
+    fi
     _sudo "to make the basic directory structure of Nix (part 1)" \
-          mkdir -pv -m 0755 /nix /nix/var /nix/var/log /nix/var/log/nix /nix/var/log/nix/drvs /nix/var/nix{,/db,/gcroots,/profiles,/temproots,/userpool} /nix/var/nix/{gcroots,profiles}/per-user
+          install -dv -m 0755 /nix /nix/var /nix/var/log /nix/var/log/nix /nix/var/log/nix/drvs /nix/var/nix{,/db,/gcroots,/profiles,/temproots,/userpool} /nix/var/nix/{gcroots,profiles}/per-user
 
     _sudo "to make the basic directory structure of Nix (part 2)" \
-          mkdir -pv -m 1775 /nix/store
-
-    _sudo "to make the basic directory structure of Nix (part 3)" \
-          chgrp "$NIX_BUILD_GROUP_NAME" /nix/store
+          install -dv -g "$NIX_BUILD_GROUP_NAME" -m 1775 /nix/store
 
     _sudo "to place the default nix daemon configuration (part 1)" \
-          mkdir -pv -m 0555 /etc/nix
+          install -dv -m 0555 /etc/nix
 }
 
 place_channel_configuration() {
@@ -473,7 +597,7 @@ This installation tool will set up your computer with the Nix package
 manager. This will happen in a few stages:
 
 1. Make sure your computer doesn't already have Nix. If it does, I
-   will show you instructions on how to clean up your old one.
+   will show you instructions on how to clean up your old install.
 
 2. Show you what we are going to install and where. Then we will ask
    if you are ready to continue.
@@ -572,6 +696,7 @@ EOF
 }
 
 install_from_extracted_nix() {
+    task "Installing Nix"
     (
         cd "$EXTRACTED_NIX_PATH"
 
@@ -587,9 +712,8 @@ $NIX_INSTALLED_NIX.
 EOF
         fi
 
-        cat ./.reginfo \
-            | _sudo "to load data for the first time in to the Nix Database" \
-                   "$NIX_INSTALLED_NIX/bin/nix-store" --load-db
+        _sudo "to load data for the first time in to the Nix Database" \
+              "$NIX_INSTALLED_NIX/bin/nix-store" --load-db < ./.reginfo
 
         echo "      Just finished getting the nix database ready."
     )
@@ -608,6 +732,7 @@ EOF
 }
 
 configure_shell_profile() {
+    task "Setting up shell profiles: ${PROFILE_TARGETS[*]}"
     for profile_target in "${PROFILE_TARGETS[@]}"; do
         if [ -e "$profile_target" ]; then
             _sudo "to back up your current $profile_target to $profile_target$PROFILE_BACKUP_SUFFIX" \
@@ -627,14 +752,27 @@ configure_shell_profile() {
                         tee -a "$profile_target"
         fi
     done
+    # TODO: should we suggest '. $PROFILE_NIX_FILE'? It would get them on
+    # their way less disruptively, but a counter-argument is that they won't
+    # immediately notice if something didn't get set up right?
+    reminder "Nix won't work in active shell sessions until you restart them."
+}
+
+cert_in_store() {
+    # in a subshell
+    # - change into the cert-file dir
+    # - get the phyiscal pwd
+    # and test if this path is in the Nix store
+    [[ "$(cd -- "$(dirname "$NIX_SSL_CERT_FILE")" && exec pwd -P)" == "$NIX_ROOT/store/"* ]]
 }
 
 setup_default_profile() {
-    _sudo "to installing a bootstrapping Nix in to the default Profile" \
+    task "Setting up the default profile"
+    _sudo "to install a bootstrapping Nix in to the default profile" \
           HOME="$ROOT_HOME" "$NIX_INSTALLED_NIX/bin/nix-env" -i "$NIX_INSTALLED_NIX"
 
-    if [ -z "${NIX_SSL_CERT_FILE:-}" ] || ! [ -f "${NIX_SSL_CERT_FILE:-}" ]; then
-        _sudo "to installing a bootstrapping SSL certificate just for Nix in to the default Profile" \
+    if [ -z "${NIX_SSL_CERT_FILE:-}" ] || ! [ -f "${NIX_SSL_CERT_FILE:-}" ] || cert_in_store; then
+        _sudo "to install a bootstrapping SSL certificate just for Nix in to the default profile" \
               HOME="$ROOT_HOME" "$NIX_INSTALLED_NIX/bin/nix-env" -i "$NIX_INSTALLED_CACERT"
         export NIX_SSL_CERT_FILE=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt
     fi
@@ -643,9 +781,13 @@ setup_default_profile() {
         # Have to explicitly pass NIX_SSL_CERT_FILE as part of the sudo call,
         # otherwise it will be lost in environments where sudo doesn't pass
         # all the environment variables by default.
-        _sudo "to update the default channel in the default profile" \
-            HOME="$ROOT_HOME" NIX_SSL_CERT_FILE="$NIX_SSL_CERT_FILE" "$NIX_INSTALLED_NIX/bin/nix-channel" --update nixpkgs \
-            || channel_update_failed=1
+        if ! _sudo "to update the default channel in the default profile" \
+            HOME="$ROOT_HOME" NIX_SSL_CERT_FILE="$NIX_SSL_CERT_FILE" "$NIX_INSTALLED_NIX/bin/nix-channel" --update nixpkgs; then
+            reminder <<EOF
+I had trouble fetching the nixpkgs channel (are you offline?)
+To try again later, run: sudo -i nix-channel --update nixpkgs
+EOF
+        fi
     fi
 }
 
@@ -660,6 +802,17 @@ EOF
 }
 
 main() {
+    # TODO: I've moved this out of validate_starting_assumptions so we
+    # can fail faster in this case. Sourcing install-darwin... now runs
+    # `touch /` to detect Read-only root, but it could update times on
+    # pre-Catalina macOS if run as root user.
+    if [ $EUID -eq 0 ]; then
+        failure <<EOF
+Please do not run this script with root privileges. We will call sudo
+when we need to.
+EOF
+    fi
+
     if [ "$(uname -s)" = "Darwin" ]; then
         # shellcheck source=./install-darwin-multi-user.sh
         . "$EXTRACTED_NIX_PATH/install-darwin-multi-user.sh"
@@ -673,17 +826,24 @@ main() {
     welcome_to_nix
     chat_about_sudo
 
+    cure_artifacts
+    # TODO: there's a tension between cure and validate. I moved the
+    # the sudo/root check out of validate to the head of this func.
+    # Cure is *intended* to subsume the validate-and-abort approach,
+    # so it may eventually obsolete it.
     validate_starting_assumptions
 
     setup_report
 
     if ! ui_confirm "Ready to continue?"; then
         ok "Alright, no changes have been made :)"
-        contactme
+        get_help
         trap finish_cleanup EXIT
         exit 1
     fi
 
+    poly_prepare_to_install
+
     create_build_group
     create_build_users
     create_directories
@@ -693,6 +853,7 @@ main() {
     configure_shell_profile
 
     set +eu
+    # shellcheck disable=SC1091
     . /etc/profile
     set -eu
 
@@ -704,5 +865,20 @@ main() {
     trap finish_success EXIT
 }
 
+# set an empty initial arg for bare invocations in case we need to
+# disambiguate someone directly invoking this later.
+if [ "${#@}" = 0 ]; then
+    set ""
+fi
 
-main
+# ACTION for override
+case "${1-}" in
+    # uninstall)
+    #     shift
+    #     uninstall "$@";;
+    # install == same as the no-arg condition for now (but, explicit)
+    ""|install)
+        main;;
+    *) # holding space for future options (like uninstall + install?)
+        failure "install-multi-user: invalid argument";;
+esac

scripts/install-nix-from-closure.sh

@@ -26,18 +26,9 @@ fi
 
 # macOS support for 10.12.6 or higher
 if [ "$(uname -s)" = "Darwin" ]; then
-    IFS='.' read macos_major macos_minor macos_patch << EOF
+    IFS='.' read -r macos_major macos_minor macos_patch << EOF
 $(sw_vers -productVersion)
 EOF
-    # TODO: this is a temporary speed-bump to keep people from naively installing Nix
-    # on macOS Big Sur (11.0+, 10.16+) until nixpkgs updates are ready for them.
-    # *Ideally* this is gone before next Nix release. If you're intentionally working on
-    # Nix + Big Sur, just comment out this block and be on your way :)
-    if [ "$macos_major" -gt 10 ] || { [ "$macos_major" -eq 10 ] && [ "$macos_minor" -gt 15 ]; }; then
-        echo "$0: nixpkgs isn't quite ready to support macOS $(sw_vers -productVersion) yet"
-        exit 1
-    fi
-
     if [ "$macos_major" -lt 10 ] || { [ "$macos_major" -eq 10 ] && [ "$macos_minor" -lt 12 ]; } || { [ "$macos_minor" -eq 12 ] && [ "$macos_patch" -lt 6 ]; }; then
         # patch may not be present; command substitution for simplicity
         echo "$0: macOS $(sw_vers -productVersion) is not supported, upgrade to 10.12.6 or higher"
@@ -46,21 +37,40 @@ EOF
 fi
 
 # Determine if we could use the multi-user installer or not
-if [ "$(uname -s)" = "Darwin" ]; then
-    echo "Note: a multi-user installation is possible. See https://nixos.org/nix/manual/#sect-multi-user-installation" >&2
-elif [ "$(uname -s)" = "Linux" ]; then
+if [ "$(uname -s)" = "Linux" ]; then
     echo "Note: a multi-user installation is possible. See https://nixos.org/nix/manual/#sect-multi-user-installation" >&2
 fi
 
-INSTALL_MODE=no-daemon
-CREATE_DARWIN_VOLUME=0
+case "$(uname -s)" in
+    "Darwin")
+        INSTALL_MODE=daemon;;
+    *)
+        INSTALL_MODE=no-daemon;;
+esac
+
+# space-separated string
+ACTIONS=
+
 # handle the command line flags
 while [ $# -gt 0 ]; do
     case $1 in
         --daemon)
-            INSTALL_MODE=daemon;;
+            INSTALL_MODE=daemon
+            ACTIONS="${ACTIONS}install "
+            ;;
         --no-daemon)
-            INSTALL_MODE=no-daemon;;
+            if [ "$(uname -s)" = "Darwin" ]; then
+                printf '\e[1;31mError: --no-daemon installs are no-longer supported on Darwin/macOS!\e[0m\n' >&2
+                exit 1
+            fi
+            INSTALL_MODE=no-daemon
+            # intentional tail space
+            ACTIONS="${ACTIONS}install "
+            ;;
+        # --uninstall)
+        #     # intentional tail space
+        #     ACTIONS="${ACTIONS}uninstall "
+        #     ;;
         --no-channel-add)
             export NIX_INSTALLER_NO_CHANNEL_ADD=1;;
         --daemon-user-count)
@@ -69,13 +79,18 @@ while [ $# -gt 0 ]; do
         --no-modify-profile)
             NIX_INSTALLER_NO_MODIFY_PROFILE=1;;
         --darwin-use-unencrypted-nix-store-volume)
-            CREATE_DARWIN_VOLUME=1;;
+            {
+                echo "Warning: the flag --darwin-use-unencrypted-nix-store-volume"
+                echo "         is no longer needed and will be removed in the future."
+                echo ""
+            } >&2;;
         --nix-extra-conf-file)
-            export NIX_EXTRA_CONF="$(cat $2)"
+            # shellcheck disable=SC2155
+            export NIX_EXTRA_CONF="$(cat "$2")"
             shift;;
         *)
-            (
-                echo "Nix Installer [--daemon|--no-daemon] [--daemon-user-count INT] [--no-channel-add] [--no-modify-profile] [--darwin-use-unencrypted-nix-store-volume] [--nix-extra-conf-file FILE]"
+            {
+                echo "Nix Installer [--daemon|--no-daemon] [--daemon-user-count INT] [--no-channel-add] [--no-modify-profile] [--nix-extra-conf-file FILE]"
 
                 echo "Choose installation method."
                 echo ""
@@ -101,45 +116,16 @@ while [ $# -gt 0 ]; do
                 if [ -n "${INVOKED_FROM_INSTALL_IN:-}" ]; then
                     echo " --tarball-url-prefix URL: Base URL to download the Nix tarball from."
                 fi
-            ) >&2
+            } >&2
 
-            # darwin and Catalina+
-            if [ "$(uname -s)" = "Darwin" ] && { [ "$macos_major" -gt 10 ] || { [ "$macos_major" -eq 10 ] && [ "$macos_minor" -gt 14 ]; }; }; then
-                (
-                    echo " --darwin-use-unencrypted-nix-store-volume: Create an APFS volume for the Nix"
-                    echo "              store and mount it at /nix. This is the recommended way to create"
-                    echo "              /nix with a read-only / on macOS >=10.15."
-                    echo "              See: https://nixos.org/nix/manual/#sect-macos-installation"
-                    echo ""
-                ) >&2
-            fi
             exit;;
     esac
     shift
 done
 
-if [ "$(uname -s)" = "Darwin" ]; then
-    if [ "$CREATE_DARWIN_VOLUME" = 1 ]; then
-        printf '\e[1;31mCreating volume and mountpoint /nix.\e[0m\n'
-        "$self/create-darwin-volume.sh"
-    fi
-
-    writable="$(diskutil info -plist / | xmllint --xpath "name(/plist/dict/key[text()='Writable']/following-sibling::*[1])" -)"
-    if ! [ -e $dest ] && [ "$writable" = "false" ]; then
-        (
-            echo ""
-            echo "Installing on macOS >=10.15 requires relocating the store to an apfs volume."
-            echo "Use sh <(curl -L https://nixos.org/nix/install) --darwin-use-unencrypted-nix-store-volume or run the preparation steps manually."
-            echo "See https://nixos.org/nix/manual/#sect-macos-installation"
-            echo ""
-        ) >&2
-        exit 1
-    fi
-fi
-
 if [ "$INSTALL_MODE" = "daemon" ]; then
     printf '\e[1;31mSwitching to the Multi-user Installer\e[0m\n'
-    exec "$self/install-multi-user"
+    exec "$self/install-multi-user" $ACTIONS # let ACTIONS split
     exit 0
 fi
 
@@ -194,6 +180,7 @@ if ! "$nix/bin/nix-store" --load-db < "$self/.reginfo"; then
     exit 1
 fi
 
+# shellcheck source=./nix-profile.sh.in
 . "$nix/etc/profile.d/nix.sh"
 
 if ! "$nix/bin/nix-env" -i "$nix"; then

scripts/install-systemd-multi-user.sh

@@ -41,10 +41,8 @@ handle_network_proxy() {
     fi
 }
 
-poly_validate_assumptions() {
-    if [ "$(uname -s)" != "Linux" ]; then
-        failure "This script is for use with Linux!"
-    fi
+poly_cure_artifacts() {
+    :
 }
 
 poly_service_installed_check() {
@@ -72,7 +70,7 @@ poly_service_setup_note() {
 EOF
 }
 
-poly_extra_try_me_commands(){
+poly_extra_try_me_commands() {
     if [ -e /run/systemd/system ]; then
         :
     else
@@ -81,19 +79,10 @@ poly_extra_try_me_commands(){
 EOF
     fi
 }
-poly_extra_setup_instructions(){
-    if [ -e /run/systemd/system ]; then
-        :
-    else
-        cat <<EOF
-Additionally, you may want to add nix-daemon to your init-system.
-
-EOF
-    fi
-}
 
 poly_configure_nix_daemon_service() {
     if [ -e /run/systemd/system ]; then
+        task "Setting up the nix-daemon systemd service"
         _sudo "to set up the nix-daemon service" \
               systemctl link "/nix/var/nix/profiles/default$SERVICE_SRC"
 
@@ -110,6 +99,8 @@ poly_configure_nix_daemon_service() {
 
         _sudo "to start the nix-daemon.service" \
               systemctl restart nix-daemon.service
+    else
+        reminder "I don't support your init system yet; you may want to add nix-daemon manually."
     fi
 }
 
@@ -207,3 +198,7 @@ poly_create_build_user() {
           --password "!" \
           "$username"
 }
+
+poly_prepare_to_install() {
+    :
+}

scripts/install.in

@@ -46,21 +46,15 @@ case "$(uname -s).$(uname -m)" in
         system=x86_64-darwin
         ;;
     Darwin.arm64|Darwin.aarch64)
-        # check for Rosetta 2 support
-        if ! [ -f /Library/Apple/System/Library/LaunchDaemons/com.apple.oahd.plist ]; then
-          oops "Rosetta 2 is not installed on this ARM64 macOS machine. Run softwareupdate --install-rosetta then restart installation"
-        fi
-
-        hash=@binaryTarball_x86_64-darwin@
-        path=@tarballPath_x86_64-darwin@
-        # eventually maybe: aarch64-darwin
-        system=x86_64-darwin
+        hash=@binaryTarball_aarch64-darwin@
+        path=@tarballPath_aarch64-darwin@
+        system=aarch64-darwin
         ;;
     *) oops "sorry, there is no binary distribution of Nix for your platform";;
 esac
 
 # Use this command-line option to fetch the tarballs using nar-serve or Cachix
-if "${1:---tarball-url-prefix}"; then
+if [ "${1:-}" = "--tarball-url-prefix" ]; then
     if [ -z "${2:-}" ]; then
         oops "missing argument for --tarball-url-prefix"
     fi

scripts/prepare-installer-for-github-actions

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+set -e
+
+script=$(nix-build -A outputs.hydraJobs.installerScriptForGHA --no-out-link)
+installerHash=$(echo $script | cut -b12-43 -)
+
+installerURL=https://$CACHIX_NAME.cachix.org/serve/$installerHash/install
+
+echo "::set-output name=installerURL::$installerURL"

src/build-remote/build-remote.cc

@@ -53,6 +53,9 @@ static int main_build_remote(int argc, char * * argv)
         unsetenv("DISPLAY");
         unsetenv("SSH_ASKPASS");
 
+        /* If we ever use the common args framework, make sure to
+           remove initPlugins below and initialize settings first.
+        */
         if (argc != 2)
             throw UsageError("called without required arguments");
 
@@ -248,7 +251,7 @@ connected:
         std::cerr << "# accept\n" << storeUri << "\n";
 
         auto inputs = readStrings<PathSet>(source);
-        auto outputs = readStrings<PathSet>(source);
+        auto wantedOutputs = readStrings<StringSet>(source);
 
         AutoCloseFD uploadLock = openLockFile(currentLoad + "/" + escapeUri(storeUri) + ".upload-lock", true);
 
@@ -273,23 +276,59 @@ connected:
         uploadLock = -1;
 
         auto drv = store->readDerivation(*drvPath);
-        drv.inputSrcs = store->parseStorePathSet(inputs);
+        auto outputHashes = staticOutputHashes(*store, drv);
+
+        // Hijack the inputs paths of the derivation to include all the paths
+        // that come from the `inputDrvs` set.
+        // We don’t do that for the derivations whose `inputDrvs` is empty
+        // because
+        // 1. It’s not needed
+        // 2. Changing the `inputSrcs` set changes the associated output ids,
+        //  which break CA derivations
+        if (!drv.inputDrvs.empty())
+            drv.inputSrcs = store->parseStorePathSet(inputs);
 
         auto result = sshStore->buildDerivation(*drvPath, drv);
 
         if (!result.success())
             throw Error("build of '%s' on '%s' failed: %s", store->printStorePath(*drvPath), storeUri, result.errorMsg);
 
-        StorePathSet missing;
-        for (auto & path : outputs)
-            if (!store->isValidPath(store->parseStorePath(path))) missing.insert(store->parseStorePath(path));
+        std::set<Realisation> missingRealisations;
+        StorePathSet missingPaths;
+        if (settings.isExperimentalFeatureEnabled("ca-derivations") && !derivationHasKnownOutputPaths(drv.type())) {
+            for (auto & outputName : wantedOutputs) {
+                auto thisOutputHash = outputHashes.at(outputName);
+                auto thisOutputId = DrvOutput{ thisOutputHash, outputName };
+                if (!store->queryRealisation(thisOutputId)) {
+                    debug("missing output %s", outputName);
+                    assert(result.builtOutputs.count(thisOutputId));
+                    auto newRealisation = result.builtOutputs.at(thisOutputId);
+                    missingRealisations.insert(newRealisation);
+                    missingPaths.insert(newRealisation.outPath);
+                }
+            }
+        } else {
+            auto outputPaths = drv.outputsAndOptPaths(*store);
+            for (auto & [outputName, hopefullyOutputPath] : outputPaths) {
+                assert(hopefullyOutputPath.second);
+                if (!store->isValidPath(*hopefullyOutputPath.second))
+                    missingPaths.insert(*hopefullyOutputPath.second);
+            }
+        }
 
-        if (!missing.empty()) {
+        if (!missingPaths.empty()) {
             Activity act(*logger, lvlTalkative, actUnknown, fmt("copying outputs from '%s'", storeUri));
             if (auto localStore = store.dynamic_pointer_cast<LocalStore>())
-                for (auto & i : missing)
-                    localStore->locksHeld.insert(store->printStorePath(i)); /* FIXME: ugly */
-            copyPaths(ref<Store>(sshStore), store, missing, NoRepair, NoCheckSigs, NoSubstitute);
+                for (auto & path : missingPaths)
+                    localStore->locksHeld.insert(store->printStorePath(path)); /* FIXME: ugly */
+            copyPaths(ref<Store>(sshStore), store, missingPaths, NoRepair, NoCheckSigs, NoSubstitute);
+        }
+        // XXX: Should be done as part of `copyPaths`
+        for (auto & realisation : missingRealisations) {
+            // Should hold, because if the feature isn't enabled the set
+            // of missing realisations should be empty
+            settings.requireExperimentalFeature("ca-derivations");
+            store->registerDrvOutput(realisation);
         }
 
         return 0;

src/libcmd/command.cc

@@ -54,7 +54,7 @@ void StoreCommand::run()
     run(getStore());
 }
 
-RealisedPathsCommand::RealisedPathsCommand(bool recursive)
+BuiltPathsCommand::BuiltPathsCommand(bool recursive)
     : recursive(recursive)
 {
     if (recursive)
@@ -81,47 +81,51 @@ RealisedPathsCommand::RealisedPathsCommand(bool recursive)
     });
 }
 
-void RealisedPathsCommand::run(ref<Store> store)
+void BuiltPathsCommand::run(ref<Store> store)
 {
-    std::vector<RealisedPath> paths;
+    BuiltPaths paths;
     if (all) {
         if (installables.size())
             throw UsageError("'--all' does not expect arguments");
         // XXX: Only uses opaque paths, ignores all the realisations
         for (auto & p : store->queryAllValidPaths())
-            paths.push_back(p);
+            paths.push_back(BuiltPath::Opaque{p});
     } else {
-        auto pathSet = toRealisedPaths(store, realiseMode, operateOn, installables);
+        paths = toBuiltPaths(store, realiseMode, operateOn, installables);
         if (recursive) {
-            auto roots = std::move(pathSet);
-            pathSet = {};
-            RealisedPath::closure(*store, roots, pathSet);
+            // XXX: This only computes the store path closure, ignoring
+            // intermediate realisations
+            StorePathSet pathsRoots, pathsClosure;
+            for (auto & root: paths) {
+                auto rootFromThis = root.outPaths();
+                pathsRoots.insert(rootFromThis.begin(), rootFromThis.end());
+            }
+            store->computeFSClosure(pathsRoots, pathsClosure);
+            for (auto & path : pathsClosure)
+                paths.push_back(BuiltPath::Opaque{path});
         }
-        for (auto & path : pathSet)
-            paths.push_back(path);
     }
 
     run(store, std::move(paths));
 }
 
 StorePathsCommand::StorePathsCommand(bool recursive)
-    : RealisedPathsCommand(recursive)
+    : BuiltPathsCommand(recursive)
 {
 }
 
-void StorePathsCommand::run(ref<Store> store, std::vector<RealisedPath> paths)
+void StorePathsCommand::run(ref<Store> store, BuiltPaths paths)
 {
     StorePaths storePaths;
-    for (auto & p : paths)
-        storePaths.push_back(p.path());
+    for (auto& builtPath : paths)
+        for (auto& p : builtPath.outPaths())
+            storePaths.push_back(p);
 
     run(store, std::move(storePaths));
 }
 
-void StorePathCommand::run(ref<Store> store)
+void StorePathCommand::run(ref<Store> store, std::vector<StorePath> storePaths)
 {
-    auto storePaths = toStorePaths(store, Realise::Nothing, operateOn, installables);
-
     if (storePaths.size() != 1)
         throw UsageError("this command requires exactly one store path");
 
@@ -164,7 +168,7 @@ void MixProfile::updateProfile(const StorePath & storePath)
             profile2, storePath));
 }
 
-void MixProfile::updateProfile(const Buildables & buildables)
+void MixProfile::updateProfile(const BuiltPaths & buildables)
 {
     if (!profile) return;
 
@@ -172,18 +176,15 @@ void MixProfile::updateProfile(const Buildables & buildables)
 
     for (auto & buildable : buildables) {
         std::visit(overloaded {
-            [&](BuildableOpaque bo) {
+            [&](BuiltPath::Opaque bo) {
                 result.push_back(bo.path);
             },
-            [&](BuildableFromDrv bfd) {
+            [&](BuiltPath::Built bfd) {
                 for (auto & output : bfd.outputs) {
-                    /* Output path should be known because we just tried to
-                       build it. */
-                    assert(output.second);
-                    result.push_back(*output.second);
+                    result.push_back(output.second);
                 }
             },
-        }, buildable);
+        }, buildable.raw());
     }
 
     if (result.size() != 1)

src/libcmd/command.hh

@@ -143,7 +143,7 @@ private:
 };
 
 /* A command that operates on zero or more store paths. */
-struct RealisedPathsCommand : public InstallablesCommand
+struct BuiltPathsCommand : public InstallablesCommand
 {
 private:
 
@@ -156,36 +156,36 @@ protected:
 
 public:
 
-    RealisedPathsCommand(bool recursive = false);
+    BuiltPathsCommand(bool recursive = false);
 
     using StoreCommand::run;
 
-    virtual void run(ref<Store> store, std::vector<RealisedPath> paths) = 0;
+    virtual void run(ref<Store> store, BuiltPaths paths) = 0;
 
     void run(ref<Store> store) override;
 
     bool useDefaultInstallables() override { return !all; }
 };
 
-struct StorePathsCommand : public RealisedPathsCommand
+struct StorePathsCommand : public BuiltPathsCommand
 {
     StorePathsCommand(bool recursive = false);
 
-    using RealisedPathsCommand::run;
+    using BuiltPathsCommand::run;
 
     virtual void run(ref<Store> store, std::vector<StorePath> storePaths) = 0;
 
-    void run(ref<Store> store, std::vector<RealisedPath> paths) override;
+    void run(ref<Store> store, BuiltPaths paths) override;
 };
 
 /* A command that operates on exactly one store path. */
-struct StorePathCommand : public InstallablesCommand
+struct StorePathCommand : public StorePathsCommand
 {
-    using StoreCommand::run;
+    using StorePathsCommand::run;
 
     virtual void run(ref<Store> store, const StorePath & storePath) = 0;
 
-    void run(ref<Store> store) override;
+    void run(ref<Store> store, std::vector<StorePath> storePaths) override;
 };
 
 /* A helper class for registering commands globally. */
@@ -216,7 +216,7 @@ static RegisterCommand registerCommand2(std::vector<std::string> && name)
     return RegisterCommand(std::move(name), [](){ return make_ref<T>(); });
 }
 
-Buildables build(ref<Store> store, Realise mode,
+BuiltPaths build(ref<Store> store, Realise mode,
     std::vector<std::shared_ptr<Installable>> installables, BuildMode bMode = bmNormal);
 
 std::set<StorePath> toStorePaths(ref<Store> store,
@@ -231,7 +231,7 @@ std::set<StorePath> toDerivations(ref<Store> store,
     std::vector<std::shared_ptr<Installable>> installables,
     bool useDeriver = false);
 
-std::set<RealisedPath> toRealisedPaths(
+BuiltPaths toBuiltPaths(
     ref<Store> store,
     Realise mode,
     OperateOn operateOn,
@@ -252,7 +252,7 @@ struct MixProfile : virtual StoreCommand
 
     /* If 'profile' is set, make it point at the store path produced
        by 'buildables'. */
-    void updateProfile(const Buildables & buildables);
+    void updateProfile(const BuiltPaths & buildables);
 };
 
 struct MixDefaultProfile : MixProfile

src/libcmd/installables.cc

@@ -9,6 +9,7 @@
 #include "store-api.hh"
 #include "shared.hh"
 #include "flake/flake.hh"
+#include "eval-cache.hh"
 #include "url.hh"
 #include "registry.hh"
 
@@ -19,31 +20,6 @@
 
 namespace nix {
 
-nlohmann::json BuildableOpaque::toJSON(ref<Store> store) const {
-    nlohmann::json res;
-    res["path"] = store->printStorePath(path);
-    return res;
-}
-
-nlohmann::json BuildableFromDrv::toJSON(ref<Store> store) const {
-    nlohmann::json res;
-    res["drvPath"] = store->printStorePath(drvPath);
-    for (const auto& [output, path] : outputs) {
-        res["outputs"][output] = path ? store->printStorePath(*path) : "";
-    }
-    return res;
-}
-
-nlohmann::json buildablesToJSON(const Buildables & buildables, ref<Store> store) {
-    auto res = nlohmann::json::array();
-    for (const Buildable & buildable : buildables) {
-        std::visit([&res, store](const auto & buildable) {
-            res.push_back(buildable.toJSON(store));
-        }, buildable);
-    }
-    return res;
-}
-
 void completeFlakeInputPath(
     ref<EvalState> evalState,
     const FlakeRef & flakeRef,
@@ -110,10 +86,11 @@ MixFlakeOptions::MixFlakeOptions()
 
     addFlag({
         .longName = "override-input",
-        .description = "Override a specific flake input (e.g. `dwarffs/nixpkgs`).",
+        .description = "Override a specific flake input (e.g. `dwarffs/nixpkgs`). This implies `--no-write-lock-file`.",
         .category = category,
         .labels = {"input-path", "flake-url"},
         .handler = {[&](std::string inputPath, std::string flakeRef) {
+            lockFlags.writeLockFile = false;
             lockFlags.inputOverrides.insert_or_assign(
                 flake::parseInputPath(inputPath),
                 parseFlakeRef(flakeRef, absPath(".")));
@@ -221,8 +198,10 @@ void completeFlakeRefWithFragment(
             // FIXME: do tilde expansion.
             auto flakeRef = parseFlakeRef(flakeRefS, absPath("."));
 
-            auto lockedFlake = lockFlake(*evalState, flakeRef, lockFlags);
-            auto rootValue = getFlakeValue(*evalState, lockedFlake);
+            auto evalCache = openEvalCache(*evalState,
+                std::make_shared<flake::LockedFlake>(lockFlake(*evalState, flakeRef, lockFlags)));
+
+            auto root = evalCache->getRoot();
 
             /* Complete 'fragment' relative to all the
                attrpath prefixes as well as the root of the
@@ -240,18 +219,12 @@ void completeFlakeRefWithFragment(
                     attrPath.pop_back();
                 }
 
-                auto cachedFieldNames = rootValue->getCache().listChildrenAtPath(evalState->symbols, attrPath);
+                auto attr = root->findAlongAttrPath(attrPath);
+                if (!attr) continue;
 
-                if (!cachedFieldNames) {
-                    auto accessResult = evalState->getOptionalAttrField(*rootValue, attrPath, noPos);
-                    if (accessResult.error) continue;
-                    cachedFieldNames = evalState->listAttrFields(*accessResult.value, *accessResult.pos);
-                }
-
-                for (auto & lastFieldName : *cachedFieldNames) {
-                    if (hasPrefix(lastFieldName, lastAttr)) {
-                        auto attrPath2 = attrPath;
-                        attrPath2.push_back(lastFieldName);
+                for (auto & attr2 : attr->getAttrs()) {
+                    if (hasPrefix(attr2, lastAttr)) {
+                        auto attrPath2 = attr->getAttrPath(attr2);
                         /* Strip the attrpath prefix. */
                         attrPath2.erase(attrPath2.begin(), attrPath2.begin() + attrPathPrefix.size());
                         completions->add(flakeRefS + "#" + concatStringsSep(".", attrPath2));
@@ -263,8 +236,8 @@ void completeFlakeRefWithFragment(
                attrpaths. */
             if (fragment.empty()) {
                 for (auto & attrPath : defaultFlakeAttrPaths) {
-                    auto accessResult = evalState->getOptionalAttrField(*rootValue, {evalState->symbols.create(attrPath)}, noPos);
-                    if (accessResult.error) continue;
+                    auto attr = root->findAlongAttrPath(parseAttrPath(*evalState, attrPath));
+                    if (!attr) continue;
                     completions->add(flakeRefS + "#");
                 }
             }
@@ -312,14 +285,32 @@ void completeFlakeRef(ref<Store> store, std::string_view prefix)
     }
 }
 
-Buildable Installable::toBuildable()
+DerivedPath Installable::toDerivedPath()
 {
-    auto buildables = toBuildables();
+    auto buildables = toDerivedPaths();
     if (buildables.size() != 1)
         throw Error("installable '%s' evaluates to %d derivations, where only one is expected", what(), buildables.size());
     return std::move(buildables[0]);
 }
 
+std::vector<std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>>
+Installable::getCursors(EvalState & state)
+{
+    auto evalCache =
+        std::make_shared<nix::eval_cache::EvalCache>(std::nullopt, state,
+            [&]() { return toValue(state).first; });
+    return {{evalCache->getRoot(), ""}};
+}
+
+std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>
+Installable::getCursor(EvalState & state)
+{
+    auto cursors = getCursors(state);
+    if (cursors.empty())
+        throw Error("cannot find flake attribute '%s'", what());
+    return cursors[0];
+}
+
 struct InstallableStorePath : Installable
 {
     ref<Store> store;
@@ -330,22 +321,19 @@ struct InstallableStorePath : Installable
 
     std::string what() override { return store->printStorePath(storePath); }
 
-    Buildables toBuildables() override
+    DerivedPaths toDerivedPaths() override
     {
         if (storePath.isDerivation()) {
-            std::map<std::string, std::optional<StorePath>> outputs;
             auto drv = store->readDerivation(storePath);
-            for (auto & [name, output] : drv.outputsAndOptPaths(*store))
-                outputs.emplace(name, output.second);
             return {
-                BuildableFromDrv {
+                DerivedPath::Built {
                     .drvPath = storePath,
-                    .outputs = std::move(outputs)
+                    .outputs = drv.outputNames(),
                 }
             };
         } else {
             return {
-                BuildableOpaque {
+                DerivedPath::Opaque {
                     .path = storePath,
                 }
             };
@@ -358,22 +346,22 @@ struct InstallableStorePath : Installable
     }
 };
 
-Buildables InstallableValue::toBuildables()
+DerivedPaths InstallableValue::toDerivedPaths()
 {
-    Buildables res;
+    DerivedPaths res;
 
-    std::map<StorePath, std::map<std::string, std::optional<StorePath>>> drvsToOutputs;
+    std::map<StorePath, std::set<std::string>> drvsToOutputs;
 
     // Group by derivation, helps with .all in particular
     for (auto & drv : toDerivations()) {
         auto outputName = drv.outputName;
         if (outputName == "")
             throw Error("derivation '%s' lacks an 'outputName' attribute", state->store->printStorePath(drv.drvPath));
-        drvsToOutputs[drv.drvPath].insert_or_assign(outputName, drv.outPath);
+        drvsToOutputs[drv.drvPath].insert(outputName);
     }
 
     for (auto & i : drvsToOutputs)
-        res.push_back(BuildableFromDrv { i.first, i.second });
+        res.push_back(DerivedPath::Built { i.first, i.second });
 
     return res;
 }
@@ -390,11 +378,11 @@ struct InstallableAttrPath : InstallableValue
 
     std::string what() override { return attrPath; }
 
-    std::vector<Installable::ValueInfo> toValues(EvalState & state) override
+    std::pair<Value *, Pos> toValue(EvalState & state) override
     {
         auto [vRes, pos] = findAlongAttrPath(state, attrPath, *cmd.getAutoArgs(state), **v);
         state.forceValue(*vRes);
-        return {{vRes, pos, attrPath}};
+        return {vRes, pos};
     }
 
     virtual std::vector<InstallableValue::DerivationInfo> toDerivations() override;
@@ -402,7 +390,7 @@ struct InstallableAttrPath : InstallableValue
 
 std::vector<InstallableValue::DerivationInfo> InstallableAttrPath::toDerivations()
 {
-    auto v = toValue(*state).value;
+    auto v = toValue(*state).first;
 
     Bindings & autoArgs = *cmd.getAutoArgs(*state);
 
@@ -436,7 +424,45 @@ std::vector<std::string> InstallableFlake::getActualAttrPaths()
 
 Value * InstallableFlake::getFlakeOutputs(EvalState & state, const flake::LockedFlake & lockedFlake)
 {
-    return getFlakeValue(state, lockedFlake);
+    auto vFlake = state.allocValue();
+
+    callFlake(state, lockedFlake, *vFlake);
+
+    auto aOutputs = vFlake->attrs->get(state.symbols.create("outputs"));
+    assert(aOutputs);
+
+    state.forceValue(*aOutputs->value);
+
+    return aOutputs->value;
+}
+
+ref<eval_cache::EvalCache> openEvalCache(
+    EvalState & state,
+    std::shared_ptr<flake::LockedFlake> lockedFlake)
+{
+    auto fingerprint = lockedFlake->getFingerprint();
+    return make_ref<nix::eval_cache::EvalCache>(
+        evalSettings.useEvalCache && evalSettings.pureEval
+            ? std::optional { std::cref(fingerprint) }
+            : std::nullopt,
+        state,
+        [&state, lockedFlake]()
+        {
+            /* For testing whether the evaluation cache is
+               complete. */
+            if (getEnv("NIX_ALLOW_EVAL").value_or("1") == "0")
+                throw Error("not everything is cached, but evaluation is not allowed");
+
+            auto vFlake = state.allocValue();
+            flake::callFlake(state, *lockedFlake, *vFlake);
+
+            state.forceAttrs(*vFlake);
+
+            auto aOutputs = vFlake->attrs->get(state.symbols.create("outputs"));
+            assert(aOutputs);
+
+            return aOutputs->value;
+        });
 }
 
 static std::string showAttrPaths(const std::vector<std::string> & paths)
@@ -449,24 +475,54 @@ static std::string showAttrPaths(const std::vector<std::string> & paths)
     return s;
 }
 
+InstallableFlake::InstallableFlake(
+    SourceExprCommand * cmd,
+    ref<EvalState> state,
+    FlakeRef && flakeRef,
+    Strings && attrPaths,
+    Strings && prefixes,
+    const flake::LockFlags & lockFlags)
+    : InstallableValue(state),
+      flakeRef(flakeRef),
+      attrPaths(attrPaths),
+      prefixes(prefixes),
+      lockFlags(lockFlags)
+{
+    if (cmd && cmd->getAutoArgs(*state)->size())
+        throw UsageError("'--arg' and '--argstr' are incompatible with flakes");
+}
+
 std::tuple<std::string, FlakeRef, InstallableValue::DerivationInfo> InstallableFlake::toDerivation()
 {
     auto lockedFlake = getLockedFlake();
 
-    auto flakeValue = toValue(*state);
+    auto cache = openEvalCache(*state, lockedFlake);
+    auto root = cache->getRoot();
 
-    auto rawDrvInfo = getDerivation(*state, *flakeValue.value, false);
-    if (!rawDrvInfo)
-        throw Error("flake output attribute '%s' is not a derivation", flakeValue.positionInfo);
+    for (auto & attrPath : getActualAttrPaths()) {
+        auto attr = root->findAlongAttrPath(
+            parseAttrPath(*state, attrPath),
+            true
+        );
 
-    auto drvInfo = InstallableValue::DerivationInfo
-    {
-        state->store->parseStorePath(rawDrvInfo->queryDrvPath()),
-            state->store->maybeParseStorePath(rawDrvInfo->queryOutPath()),
-            rawDrvInfo->queryOutputName()
-    };
+        if (!attr) continue;
 
-    return {flakeValue.positionInfo, lockedFlake->flake.lockedRef, std::move(drvInfo)};
+        if (!attr->isDerivation())
+            throw Error("flake output attribute '%s' is not a derivation", attrPath);
+
+        auto drvPath = attr->forceDerivation();
+
+        auto drvInfo = DerivationInfo{
+            std::move(drvPath),
+            state->store->maybeParseStorePath(attr->getAttr(state->sOutPath)->getString()),
+            attr->getAttr(state->sOutputName)->getString()
+        };
+
+        return {attrPath, lockedFlake->flake.lockedRef, std::move(drvInfo)};
+    }
+
+    throw Error("flake '%s' does not provide attribute %s",
+        flakeRef, showAttrPaths(getActualAttrPaths()));
 }
 
 std::vector<InstallableValue::DerivationInfo> InstallableFlake::toDerivations()
@@ -476,20 +532,8 @@ std::vector<InstallableValue::DerivationInfo> InstallableFlake::toDerivations()
     return res;
 }
 
-Installable::ValueInfo InstallableFlake::toValue(EvalState & state)
+std::pair<Value *, Pos> InstallableFlake::toValue(EvalState & state)
 {
-    auto values = toValues(state);
-    if (values.empty())
-        throw Error("flake '%s' does not provide attribute %s",
-            flakeRef, showAttrPaths(getActualAttrPaths()));
-    return values[0];
-
-}
-
-std::vector<Installable::ValueInfo>
-InstallableFlake::toValues(EvalState & state)
-{
-    std::vector<Installable::ValueInfo> res;
     auto lockedFlake = getLockedFlake();
 
     auto vOutputs = getFlakeOutputs(state, *lockedFlake);
@@ -500,11 +544,30 @@ InstallableFlake::toValues(EvalState & state)
         try {
             auto [v, pos] = findAlongAttrPath(state, attrPath, *emptyArgs, *vOutputs);
             state.forceValue(*v);
-            res.push_back({v, pos, attrPath});
+            return {v, pos};
         } catch (AttrPathNotFound & e) {
         }
     }
 
+    throw Error("flake '%s' does not provide attribute %s",
+        flakeRef, showAttrPaths(getActualAttrPaths()));
+}
+
+std::vector<std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>>
+InstallableFlake::getCursors(EvalState & state)
+{
+    auto evalCache = openEvalCache(state,
+        std::make_shared<flake::LockedFlake>(lockFlake(state, flakeRef, lockFlags)));
+
+    auto root = evalCache->getRoot();
+
+    std::vector<std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>> res;
+
+    for (auto & attrPath : getActualAttrPaths()) {
+        auto attr = root->findAlongAttrPath(parseAttrPath(state, attrPath));
+        if (attr) res.push_back({attr, attrPath});
+    }
+
     return res;
 }
 
@@ -565,9 +628,12 @@ std::vector<std::shared_ptr<Installable>> SourceExprCommand::parseInstallables(
             try {
                 auto [flakeRef, fragment] = parseFlakeRefWithFragment(s, absPath("."));
                 result.push_back(std::make_shared<InstallableFlake>(
-                        getEvalState(), std::move(flakeRef),
+                        this,
+                        getEvalState(),
+                        std::move(flakeRef),
                         fragment == "" ? getDefaultFlakeAttrPaths() : Strings{fragment},
-                        getDefaultFlakeAttrPathPrefixes(), lockFlags));
+                        getDefaultFlakeAttrPathPrefixes(),
+                        lockFlags));
                 continue;
             } catch (...) {
                 ex = std::current_exception();
@@ -606,31 +672,67 @@ std::shared_ptr<Installable> SourceExprCommand::parseInstallable(
     return installables.front();
 }
 
-Buildables build(ref<Store> store, Realise mode,
+BuiltPaths getBuiltPaths(ref<Store> store, DerivedPaths hopefullyBuiltPaths)
+{
+    BuiltPaths res;
+    for (auto& b : hopefullyBuiltPaths)
+        std::visit(
+            overloaded{
+                [&](DerivedPath::Opaque bo) {
+                    res.push_back(BuiltPath::Opaque{bo.path});
+                },
+                [&](DerivedPath::Built bfd) {
+                    OutputPathMap outputs;
+                    auto drv = store->readDerivation(bfd.drvPath);
+                    auto outputHashes = staticOutputHashes(*store, drv);
+                    auto drvOutputs = drv.outputsAndOptPaths(*store);
+                    for (auto& output : bfd.outputs) {
+                        if (!outputHashes.count(output))
+                            throw Error(
+                                "the derivation '%s' doesn't have an output "
+                                "named '%s'",
+                                store->printStorePath(bfd.drvPath), output);
+                        if (settings.isExperimentalFeatureEnabled(
+                                "ca-derivations")) {
+                            auto outputId =
+                                DrvOutput{outputHashes.at(output), output};
+                            auto realisation =
+                                store->queryRealisation(outputId);
+                            if (!realisation)
+                                throw Error(
+                                    "cannot operate on an output of unbuilt "
+                                    "content-addresed derivation '%s'",
+                                    outputId.to_string());
+                            outputs.insert_or_assign(
+                                output, realisation->outPath);
+                        } else {
+                            // If ca-derivations isn't enabled, assume that
+                            // the output path is statically known.
+                            assert(drvOutputs.count(output));
+                            assert(drvOutputs.at(output).second);
+                            outputs.insert_or_assign(
+                                output, *drvOutputs.at(output).second);
+                        }
+                    }
+                    res.push_back(BuiltPath::Built{bfd.drvPath, outputs});
+                },
+            },
+            b.raw());
+
+    return res;
+}
+
+BuiltPaths build(ref<Store> store, Realise mode,
     std::vector<std::shared_ptr<Installable>> installables, BuildMode bMode)
 {
     if (mode == Realise::Nothing)
         settings.readOnlyMode = true;
 
-    Buildables buildables;
-
-    std::vector<StorePathWithOutputs> pathsToBuild;
+    std::vector<DerivedPath> pathsToBuild;
 
     for (auto & i : installables) {
-        for (auto & b : i->toBuildables()) {
-            std::visit(overloaded {
-                [&](BuildableOpaque bo) {
-                    pathsToBuild.push_back({bo.path});
-                },
-                [&](BuildableFromDrv bfd) {
-                    StringSet outputNames;
-                    for (auto & output : bfd.outputs)
-                        outputNames.insert(output.first);
-                    pathsToBuild.push_back({bfd.drvPath, outputNames});
-                },
-            }, b);
-            buildables.push_back(std::move(b));
-        }
+        auto b = i->toDerivedPaths();
+        pathsToBuild.insert(pathsToBuild.end(), b.begin(), b.end());
     }
 
     if (mode == Realise::Nothing)
@@ -638,59 +740,26 @@ Buildables build(ref<Store> store, Realise mode,
     else if (mode == Realise::Outputs)
         store->buildPaths(pathsToBuild, bMode);
 
-    return buildables;
+    return getBuiltPaths(store, pathsToBuild);
 }
 
-std::set<RealisedPath> toRealisedPaths(
+BuiltPaths toBuiltPaths(
     ref<Store> store,
     Realise mode,
     OperateOn operateOn,
     std::vector<std::shared_ptr<Installable>> installables)
 {
-    std::set<RealisedPath> res;
     if (operateOn == OperateOn::Output) {
-        for (auto & b : build(store, mode, installables))
-            std::visit(overloaded {
-                [&](BuildableOpaque bo) {
-                    res.insert(bo.path);
-                },
-                [&](BuildableFromDrv bfd) {
-                    auto drv = store->readDerivation(bfd.drvPath);
-                    auto outputHashes = staticOutputHashes(*store, drv);
-                    for (auto & output : bfd.outputs) {
-                        if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
-                            if (!outputHashes.count(output.first))
-                                throw Error(
-                                    "the derivation '%s' doesn't have an output named '%s'",
-                                    store->printStorePath(bfd.drvPath),
-                                    output.first);
-                            auto outputId = DrvOutput{outputHashes.at(output.first), output.first};
-                            auto realisation = store->queryRealisation(outputId);
-                            if (!realisation)
-                                throw Error("cannot operate on an output of unbuilt content-addresed derivation '%s'", outputId.to_string());
-                            res.insert(RealisedPath{*realisation});
-                        }
-                        else {
-                            // If ca-derivations isn't enabled, behave as if
-                            // all the paths are opaque to keep the default
-                            // behavior
-                            assert(output.second);
-                            res.insert(*output.second);
-                        }
-                    }
-                },
-            }, b);
+        return build(store, mode, installables);
     } else {
         if (mode == Realise::Nothing)
             settings.readOnlyMode = true;
 
-        for (auto & i : installables)
-            for (auto & b : i->toBuildables())
-                if (auto bfd = std::get_if<BuildableFromDrv>(&b))
-                    res.insert(bfd->drvPath);
+        BuiltPaths res;
+        for (auto & drvPath : toDerivations(store, installables, true))
+            res.push_back(BuiltPath::Opaque{drvPath});
+        return res;
     }
-
-    return res;
 }
 
 StorePathSet toStorePaths(ref<Store> store,
@@ -698,8 +767,10 @@ StorePathSet toStorePaths(ref<Store> store,
     std::vector<std::shared_ptr<Installable>> installables)
 {
     StorePathSet outPaths;
-    for (auto & path : toRealisedPaths(store, mode, operateOn, installables))
-            outPaths.insert(path.path());
+    for (auto & path : toBuiltPaths(store, mode, operateOn, installables)) {
+        auto thisOutPaths = path.outPaths();
+        outPaths.insert(thisOutPaths.begin(), thisOutPaths.end());
+    }
     return outPaths;
 }
 
@@ -721,9 +792,9 @@ StorePathSet toDerivations(ref<Store> store,
     StorePathSet drvPaths;
 
     for (auto & i : installables)
-        for (auto & b : i->toBuildables())
+        for (auto & b : i->toDerivedPaths())
             std::visit(overloaded {
-                [&](BuildableOpaque bo) {
+                [&](DerivedPath::Opaque bo) {
                     if (!useDeriver)
                         throw Error("argument '%s' did not evaluate to a derivation", i->what());
                     auto derivers = store->queryValidDerivers(bo.path);
@@ -732,10 +803,10 @@ StorePathSet toDerivations(ref<Store> store,
                     // FIXME: use all derivers?
                     drvPaths.insert(*derivers.begin());
                 },
-                [&](BuildableFromDrv bfd) {
+                [&](DerivedPath::Built bfd) {
                     drvPaths.insert(bfd.drvPath);
                 },
-            }, b);
+            }, b.raw());
 
     return drvPaths;
 }

src/libcmd/installables.hh

@@ -2,13 +2,13 @@
 
 #include "util.hh"
 #include "path.hh"
+#include "path-with-outputs.hh"
+#include "derived-path.hh"
 #include "eval.hh"
 #include "flake/flake.hh"
 
 #include <optional>
 
-#include <nlohmann/json_fwd.hpp>
-
 namespace nix {
 
 struct DrvInfo;
@@ -16,25 +16,6 @@ struct SourceExprCommand;
 
 namespace eval_cache { class EvalCache; class AttrCursor; }
 
-struct BuildableOpaque {
-    StorePath path;
-    nlohmann::json toJSON(ref<Store> store) const;
-};
-
-struct BuildableFromDrv {
-    StorePath drvPath;
-    std::map<std::string, std::optional<StorePath>> outputs;
-    nlohmann::json toJSON(ref<Store> store) const;
-};
-
-typedef std::variant<
-    BuildableOpaque,
-    BuildableFromDrv
-> Buildable;
-
-typedef std::vector<Buildable> Buildables;
-nlohmann::json buildablesToJSON(const Buildables & buildables, ref<Store> store);
-
 struct App
 {
     std::vector<StorePathWithOutputs> context;
@@ -42,35 +23,25 @@ struct App
     // FIXME: add args, sandbox settings, metadata, ...
 };
 
+struct UnresolvedApp
+{
+    App unresolved;
+    App resolve(ref<Store>);
+};
+
 struct Installable
 {
     virtual ~Installable() { }
 
     virtual std::string what() = 0;
 
-    virtual Buildables toBuildables() = 0;
+    virtual DerivedPaths toDerivedPaths() = 0;
 
-    Buildable toBuildable();
+    DerivedPath toDerivedPath();
 
-    App toApp(EvalState & state);
+    UnresolvedApp toApp(EvalState & state);
 
-    struct ValueInfo {
-        Value * value;
-        Pos pos;
-        string positionInfo;
-    };
-
-    virtual ValueInfo toValue(EvalState & state)
-    {
-        auto values = toValues(state);
-        if (values.empty())
-            throw Error("Installable '%s' does not provide a default value",
-                    what());
-        return values[0];
-    }
-
-    virtual std::vector<ValueInfo>
-    toValues(EvalState & state)
+    virtual std::pair<Value *, Pos> toValue(EvalState & state)
     {
         throw Error("argument '%s' cannot be evaluated", what());
     }
@@ -82,6 +53,11 @@ struct Installable
         return {};
     }
 
+    virtual std::vector<std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>>
+    getCursors(EvalState & state);
+
+    std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>
+    getCursor(EvalState & state);
 
     virtual FlakeRef nixpkgsFlakeRef() const
     {
@@ -104,7 +80,7 @@ struct InstallableValue : Installable
 
     virtual std::vector<DerivationInfo> toDerivations() = 0;
 
-    Buildables toBuildables() override;
+    DerivedPaths toDerivedPaths() override;
 };
 
 struct InstallableFlake : InstallableValue
@@ -115,11 +91,13 @@ struct InstallableFlake : InstallableValue
     const flake::LockFlags & lockFlags;
     mutable std::shared_ptr<flake::LockedFlake> _lockedFlake;
 
-    InstallableFlake(ref<EvalState> state, FlakeRef && flakeRef,
-        Strings && attrPaths, Strings && prefixes, const flake::LockFlags & lockFlags)
-        : InstallableValue(state), flakeRef(flakeRef), attrPaths(attrPaths),
-          prefixes(prefixes), lockFlags(lockFlags)
-    { }
+    InstallableFlake(
+        SourceExprCommand * cmd,
+        ref<EvalState> state,
+        FlakeRef && flakeRef,
+        Strings && attrPaths,
+        Strings && prefixes,
+        const flake::LockFlags & lockFlags);
 
     std::string what() override { return flakeRef.to_string() + "#" + *attrPaths.begin(); }
 
@@ -131,14 +109,18 @@ struct InstallableFlake : InstallableValue
 
     std::vector<DerivationInfo> toDerivations() override;
 
-    ValueInfo toValue(EvalState & state) override;
+    std::pair<Value *, Pos> toValue(EvalState & state) override;
 
-    std::vector<ValueInfo>
-    toValues(EvalState & state) override;
+    std::vector<std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>>
+    getCursors(EvalState & state) override;
 
     std::shared_ptr<flake::LockedFlake> getLockedFlake() const;
 
     FlakeRef nixpkgsFlakeRef() const override;
 };
 
+ref<eval_cache::EvalCache> openEvalCache(
+    EvalState & state,
+    std::shared_ptr<flake::LockedFlake> lockedFlake);
+
 }

src/libexpr/attr-path.cc

@@ -73,12 +73,11 @@ std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attr
             if (attr.empty())
                 throw Error("empty attribute name in selection path '%1%'", attrPath);
 
-            Value * vRes = state.allocValue();
-            auto getResult = state.getOptionalAttrField(*v, {state.symbols.create(attr)}, pos, *vRes);
-            if (getResult.error)
+            Bindings::iterator a = v->attrs->find(state.symbols.create(attr));
+            if (a == v->attrs->end())
                 throw AttrPathNotFound("attribute '%1%' in selection path '%2%' not found", attr, attrPath);
-            v = vRes;
-            pos = *getResult.pos;
+            v = &*a->value;
+            pos = *a->pos;
         }
 
         else {

src/libexpr/attr-set.hh

@@ -35,13 +35,10 @@ class Bindings
 {
 public:
     typedef uint32_t size_t;
+    Pos *pos;
 
 private:
     size_t size_, capacity_;
-
-public:
-
-private:
     Attr attrs[0];
 
     Bindings(size_t capacity) : size_(0), capacity_(capacity) { }

src/libexpr/context.cc

@@ -1,21 +0,0 @@
-#include "context.hh"
-
-namespace nix {
-
-/* Decode a context string ‘!<name>!<path>’ into a pair <path,
-   name>. */
-std::pair<string, string> decodeContext(std::string_view s)
-{
-    if (s.at(0) == '!') {
-        size_t index = s.find("!", 1);
-        return {std::string(s.substr(index + 1)), std::string(s.substr(1, index - 1))};
-    } else
-        return {s.at(0) == '/' ? std::string(s) : std::string(s.substr(1)), ""};
-}
-
-std::string encodeContext(std::string_view name, std::string_view path)
-{
-    return "!" + std::string(name) + "!" + std::string(path);
-}
-
-}

src/libexpr/context.hh

@@ -1,11 +0,0 @@
-#include "util.hh"
-
-namespace nix {
-
-/* Decode a context string ‘!<name>!<path>’ into a pair <path,
-   name>. */
-std::pair<string, string> decodeContext(std::string_view s);
-
-std::string encodeContext(std::string_view name, std::string_view path);
-
-}

src/libexpr/eval-cache.cc

@@ -0,0 +1,629 @@
+#include "eval-cache.hh"
+#include "sqlite.hh"
+#include "eval.hh"
+#include "eval-inline.hh"
+#include "store-api.hh"
+
+namespace nix::eval_cache {
+
+static const char * schema = R"sql(
+create table if not exists Attributes (
+    parent      integer not null,
+    name        text,
+    type        integer not null,
+    value       text,
+    context     text,
+    primary key (parent, name)
+);
+)sql";
+
+struct AttrDb
+{
+    std::atomic_bool failed{false};
+
+    struct State
+    {
+        SQLite db;
+        SQLiteStmt insertAttribute;
+        SQLiteStmt insertAttributeWithContext;
+        SQLiteStmt queryAttribute;
+        SQLiteStmt queryAttributes;
+        std::unique_ptr<SQLiteTxn> txn;
+    };
+
+    std::unique_ptr<Sync<State>> _state;
+
+    AttrDb(const Hash & fingerprint)
+        : _state(std::make_unique<Sync<State>>())
+    {
+        auto state(_state->lock());
+
+        Path cacheDir = getCacheDir() + "/nix/eval-cache-v2";
+        createDirs(cacheDir);
+
+        Path dbPath = cacheDir + "/" + fingerprint.to_string(Base16, false) + ".sqlite";
+
+        state->db = SQLite(dbPath);
+        state->db.isCache();
+        state->db.exec(schema);
+
+        state->insertAttribute.create(state->db,
+            "insert or replace into Attributes(parent, name, type, value) values (?, ?, ?, ?)");
+
+        state->insertAttributeWithContext.create(state->db,
+            "insert or replace into Attributes(parent, name, type, value, context) values (?, ?, ?, ?, ?)");
+
+        state->queryAttribute.create(state->db,
+            "select rowid, type, value, context from Attributes where parent = ? and name = ?");
+
+        state->queryAttributes.create(state->db,
+            "select name from Attributes where parent = ?");
+
+        state->txn = std::make_unique<SQLiteTxn>(state->db);
+    }
+
+    ~AttrDb()
+    {
+        try {
+            auto state(_state->lock());
+            if (!failed)
+                state->txn->commit();
+            state->txn.reset();
+        } catch (...) {
+            ignoreException();
+        }
+    }
+
+    template<typename F>
+    AttrId doSQLite(F && fun)
+    {
+        if (failed) return 0;
+        try {
+            return fun();
+        } catch (SQLiteError &) {
+            ignoreException();
+            failed = true;
+            return 0;
+        }
+    }
+
+    AttrId setAttrs(
+        AttrKey key,
+        const std::vector<Symbol> & attrs)
+    {
+        return doSQLite([&]()
+        {
+            auto state(_state->lock());
+
+            state->insertAttribute.use()
+                (key.first)
+                (key.second)
+                (AttrType::FullAttrs)
+                (0, false).exec();
+
+            AttrId rowId = state->db.getLastInsertedRowId();
+            assert(rowId);
+
+            for (auto & attr : attrs)
+                state->insertAttribute.use()
+                    (rowId)
+                    (attr)
+                    (AttrType::Placeholder)
+                    (0, false).exec();
+
+            return rowId;
+        });
+    }
+
+    AttrId setString(
+        AttrKey key,
+        std::string_view s,
+        const char * * context = nullptr)
+    {
+        return doSQLite([&]()
+        {
+            auto state(_state->lock());
+
+            if (context) {
+                std::string ctx;
+                for (const char * * p = context; *p; ++p) {
+                    if (p != context) ctx.push_back(' ');
+                    ctx.append(*p);
+                }
+                state->insertAttributeWithContext.use()
+                    (key.first)
+                    (key.second)
+                    (AttrType::String)
+                    (s)
+                    (ctx).exec();
+            } else {
+                state->insertAttribute.use()
+                    (key.first)
+                    (key.second)
+                    (AttrType::String)
+                (s).exec();
+            }
+
+            return state->db.getLastInsertedRowId();
+        });
+    }
+
+    AttrId setBool(
+        AttrKey key,
+        bool b)
+    {
+        return doSQLite([&]()
+        {
+            auto state(_state->lock());
+
+            state->insertAttribute.use()
+                (key.first)
+                (key.second)
+                (AttrType::Bool)
+                (b ? 1 : 0).exec();
+
+            return state->db.getLastInsertedRowId();
+        });
+    }
+
+    AttrId setPlaceholder(AttrKey key)
+    {
+        return doSQLite([&]()
+        {
+            auto state(_state->lock());
+
+            state->insertAttribute.use()
+                (key.first)
+                (key.second)
+                (AttrType::Placeholder)
+                (0, false).exec();
+
+            return state->db.getLastInsertedRowId();
+        });
+    }
+
+    AttrId setMissing(AttrKey key)
+    {
+        return doSQLite([&]()
+        {
+            auto state(_state->lock());
+
+            state->insertAttribute.use()
+                (key.first)
+                (key.second)
+                (AttrType::Missing)
+                (0, false).exec();
+
+            return state->db.getLastInsertedRowId();
+        });
+    }
+
+    AttrId setMisc(AttrKey key)
+    {
+        return doSQLite([&]()
+        {
+            auto state(_state->lock());
+
+            state->insertAttribute.use()
+                (key.first)
+                (key.second)
+                (AttrType::Misc)
+                (0, false).exec();
+
+            return state->db.getLastInsertedRowId();
+        });
+    }
+
+    AttrId setFailed(AttrKey key)
+    {
+        return doSQLite([&]()
+        {
+            auto state(_state->lock());
+
+            state->insertAttribute.use()
+                (key.first)
+                (key.second)
+                (AttrType::Failed)
+                (0, false).exec();
+
+            return state->db.getLastInsertedRowId();
+        });
+    }
+
+    std::optional<std::pair<AttrId, AttrValue>> getAttr(
+        AttrKey key,
+        SymbolTable & symbols)
+    {
+        auto state(_state->lock());
+
+        auto queryAttribute(state->queryAttribute.use()(key.first)(key.second));
+        if (!queryAttribute.next()) return {};
+
+        auto rowId = (AttrType) queryAttribute.getInt(0);
+        auto type = (AttrType) queryAttribute.getInt(1);
+
+        switch (type) {
+            case AttrType::Placeholder:
+                return {{rowId, placeholder_t()}};
+            case AttrType::FullAttrs: {
+                // FIXME: expensive, should separate this out.
+                std::vector<Symbol> attrs;
+                auto queryAttributes(state->queryAttributes.use()(rowId));
+                while (queryAttributes.next())
+                    attrs.push_back(symbols.create(queryAttributes.getStr(0)));
+                return {{rowId, attrs}};
+            }
+            case AttrType::String: {
+                std::vector<std::pair<Path, std::string>> context;
+                if (!queryAttribute.isNull(3))
+                    for (auto & s : tokenizeString<std::vector<std::string>>(queryAttribute.getStr(3), ";"))
+                        context.push_back(decodeContext(s));
+                return {{rowId, string_t{queryAttribute.getStr(2), context}}};
+            }
+            case AttrType::Bool:
+                return {{rowId, queryAttribute.getInt(2) != 0}};
+            case AttrType::Missing:
+                return {{rowId, missing_t()}};
+            case AttrType::Misc:
+                return {{rowId, misc_t()}};
+            case AttrType::Failed:
+                return {{rowId, failed_t()}};
+            default:
+                throw Error("unexpected type in evaluation cache");
+        }
+    }
+};
+
+static std::shared_ptr<AttrDb> makeAttrDb(const Hash & fingerprint)
+{
+    try {
+        return std::make_shared<AttrDb>(fingerprint);
+    } catch (SQLiteError &) {
+        ignoreException();
+        return nullptr;
+    }
+}
+
+EvalCache::EvalCache(
+    std::optional<std::reference_wrapper<const Hash>> useCache,
+    EvalState & state,
+    RootLoader rootLoader)
+    : db(useCache ? makeAttrDb(*useCache) : nullptr)
+    , state(state)
+    , rootLoader(rootLoader)
+{
+}
+
+Value * EvalCache::getRootValue()
+{
+    if (!value) {
+        debug("getting root value");
+        value = allocRootValue(rootLoader());
+    }
+    return *value;
+}
+
+std::shared_ptr<AttrCursor> EvalCache::getRoot()
+{
+    return std::make_shared<AttrCursor>(ref(shared_from_this()), std::nullopt);
+}
+
+AttrCursor::AttrCursor(
+    ref<EvalCache> root,
+    Parent parent,
+    Value * value,
+    std::optional<std::pair<AttrId, AttrValue>> && cachedValue)
+    : root(root), parent(parent), cachedValue(std::move(cachedValue))
+{
+    if (value)
+        _value = allocRootValue(value);
+}
+
+AttrKey AttrCursor::getKey()
+{
+    if (!parent)
+        return {0, root->state.sEpsilon};
+    if (!parent->first->cachedValue) {
+        parent->first->cachedValue = root->db->getAttr(
+            parent->first->getKey(), root->state.symbols);
+        assert(parent->first->cachedValue);
+    }
+    return {parent->first->cachedValue->first, parent->second};
+}
+
+Value & AttrCursor::getValue()
+{
+    if (!_value) {
+        if (parent) {
+            auto & vParent = parent->first->getValue();
+            root->state.forceAttrs(vParent);
+            auto attr = vParent.attrs->get(parent->second);
+            if (!attr)
+                throw Error("attribute '%s' is unexpectedly missing", getAttrPathStr());
+            _value = allocRootValue(attr->value);
+        } else
+            _value = allocRootValue(root->getRootValue());
+    }
+    return **_value;
+}
+
+std::vector<Symbol> AttrCursor::getAttrPath() const
+{
+    if (parent) {
+        auto attrPath = parent->first->getAttrPath();
+        attrPath.push_back(parent->second);
+        return attrPath;
+    } else
+        return {};
+}
+
+std::vector<Symbol> AttrCursor::getAttrPath(Symbol name) const
+{
+    auto attrPath = getAttrPath();
+    attrPath.push_back(name);
+    return attrPath;
+}
+
+std::string AttrCursor::getAttrPathStr() const
+{
+    return concatStringsSep(".", getAttrPath());
+}
+
+std::string AttrCursor::getAttrPathStr(Symbol name) const
+{
+    return concatStringsSep(".", getAttrPath(name));
+}
+
+Value & AttrCursor::forceValue()
+{
+    debug("evaluating uncached attribute %s", getAttrPathStr());
+
+    auto & v = getValue();
+
+    try {
+        root->state.forceValue(v);
+    } catch (EvalError &) {
+        debug("setting '%s' to failed", getAttrPathStr());
+        if (root->db)
+            cachedValue = {root->db->setFailed(getKey()), failed_t()};
+        throw;
+    }
+
+    if (root->db && (!cachedValue || std::get_if<placeholder_t>(&cachedValue->second))) {
+        if (v.type() == nString)
+            cachedValue = {root->db->setString(getKey(), v.string.s, v.string.context),
+                           string_t{v.string.s, {}}};
+        else if (v.type() == nPath)
+            cachedValue = {root->db->setString(getKey(), v.path), string_t{v.path, {}}};
+        else if (v.type() == nBool)
+            cachedValue = {root->db->setBool(getKey(), v.boolean), v.boolean};
+        else if (v.type() == nAttrs)
+            ; // FIXME: do something?
+        else
+            cachedValue = {root->db->setMisc(getKey()), misc_t()};
+    }
+
+    return v;
+}
+
+std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(Symbol name, bool forceErrors)
+{
+    if (root->db) {
+        if (!cachedValue)
+            cachedValue = root->db->getAttr(getKey(), root->state.symbols);
+
+        if (cachedValue) {
+            if (auto attrs = std::get_if<std::vector<Symbol>>(&cachedValue->second)) {
+                for (auto & attr : *attrs)
+                    if (attr == name)
+                        return std::make_shared<AttrCursor>(root, std::make_pair(shared_from_this(), name));
+                return nullptr;
+            } else if (std::get_if<placeholder_t>(&cachedValue->second)) {
+                auto attr = root->db->getAttr({cachedValue->first, name}, root->state.symbols);
+                if (attr) {
+                    if (std::get_if<missing_t>(&attr->second))
+                        return nullptr;
+                    else if (std::get_if<failed_t>(&attr->second)) {
+                        if (forceErrors)
+                            debug("reevaluating failed cached attribute '%s'");
+                        else
+                            throw CachedEvalError("cached failure of attribute '%s'", getAttrPathStr(name));
+                    } else
+                        return std::make_shared<AttrCursor>(root,
+                            std::make_pair(shared_from_this(), name), nullptr, std::move(attr));
+                }
+                // Incomplete attrset, so need to fall thru and
+                // evaluate to see whether 'name' exists
+            } else
+                return nullptr;
+                //throw TypeError("'%s' is not an attribute set", getAttrPathStr());
+        }
+    }
+
+    auto & v = forceValue();
+
+    if (v.type() != nAttrs)
+        return nullptr;
+        //throw TypeError("'%s' is not an attribute set", getAttrPathStr());
+
+    auto attr = v.attrs->get(name);
+
+    if (!attr) {
+        if (root->db) {
+            if (!cachedValue)
+                cachedValue = {root->db->setPlaceholder(getKey()), placeholder_t()};
+            root->db->setMissing({cachedValue->first, name});
+        }
+        return nullptr;
+    }
+
+    std::optional<std::pair<AttrId, AttrValue>> cachedValue2;
+    if (root->db) {
+        if (!cachedValue)
+            cachedValue = {root->db->setPlaceholder(getKey()), placeholder_t()};
+        cachedValue2 = {root->db->setPlaceholder({cachedValue->first, name}), placeholder_t()};
+    }
+
+    return std::make_shared<AttrCursor>(
+        root, std::make_pair(shared_from_this(), name), attr->value, std::move(cachedValue2));
+}
+
+std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(std::string_view name)
+{
+    return maybeGetAttr(root->state.symbols.create(name));
+}
+
+std::shared_ptr<AttrCursor> AttrCursor::getAttr(Symbol name, bool forceErrors)
+{
+    auto p = maybeGetAttr(name, forceErrors);
+    if (!p)
+        throw Error("attribute '%s' does not exist", getAttrPathStr(name));
+    return p;
+}
+
+std::shared_ptr<AttrCursor> AttrCursor::getAttr(std::string_view name)
+{
+    return getAttr(root->state.symbols.create(name));
+}
+
+std::shared_ptr<AttrCursor> AttrCursor::findAlongAttrPath(const std::vector<Symbol> & attrPath, bool force)
+{
+    auto res = shared_from_this();
+    for (auto & attr : attrPath) {
+        res = res->maybeGetAttr(attr, force);
+        if (!res) return {};
+    }
+    return res;
+}
+
+std::string AttrCursor::getString()
+{
+    if (root->db) {
+        if (!cachedValue)
+            cachedValue = root->db->getAttr(getKey(), root->state.symbols);
+        if (cachedValue && !std::get_if<placeholder_t>(&cachedValue->second)) {
+            if (auto s = std::get_if<string_t>(&cachedValue->second)) {
+                debug("using cached string attribute '%s'", getAttrPathStr());
+                return s->first;
+            } else
+                throw TypeError("'%s' is not a string", getAttrPathStr());
+        }
+    }
+
+    auto & v = forceValue();
+
+    if (v.type() != nString && v.type() != nPath)
+        throw TypeError("'%s' is not a string but %s", getAttrPathStr(), showType(v.type()));
+
+    return v.type() == nString ? v.string.s : v.path;
+}
+
+string_t AttrCursor::getStringWithContext()
+{
+    if (root->db) {
+        if (!cachedValue)
+            cachedValue = root->db->getAttr(getKey(), root->state.symbols);
+        if (cachedValue && !std::get_if<placeholder_t>(&cachedValue->second)) {
+            if (auto s = std::get_if<string_t>(&cachedValue->second)) {
+                bool valid = true;
+                for (auto & c : s->second) {
+                    if (!root->state.store->isValidPath(root->state.store->parseStorePath(c.first))) {
+                        valid = false;
+                        break;
+                    }
+                }
+                if (valid) {
+                    debug("using cached string attribute '%s'", getAttrPathStr());
+                    return *s;
+                }
+            } else
+                throw TypeError("'%s' is not a string", getAttrPathStr());
+        }
+    }
+
+    auto & v = forceValue();
+
+    if (v.type() == nString)
+        return {v.string.s, v.getContext()};
+    else if (v.type() == nPath)
+        return {v.path, {}};
+    else
+        throw TypeError("'%s' is not a string but %s", getAttrPathStr(), showType(v.type()));
+}
+
+bool AttrCursor::getBool()
+{
+    if (root->db) {
+        if (!cachedValue)
+            cachedValue = root->db->getAttr(getKey(), root->state.symbols);
+        if (cachedValue && !std::get_if<placeholder_t>(&cachedValue->second)) {
+            if (auto b = std::get_if<bool>(&cachedValue->second)) {
+                debug("using cached Boolean attribute '%s'", getAttrPathStr());
+                return *b;
+            } else
+                throw TypeError("'%s' is not a Boolean", getAttrPathStr());
+        }
+    }
+
+    auto & v = forceValue();
+
+    if (v.type() != nBool)
+        throw TypeError("'%s' is not a Boolean", getAttrPathStr());
+
+    return v.boolean;
+}
+
+std::vector<Symbol> AttrCursor::getAttrs()
+{
+    if (root->db) {
+        if (!cachedValue)
+            cachedValue = root->db->getAttr(getKey(), root->state.symbols);
+        if (cachedValue && !std::get_if<placeholder_t>(&cachedValue->second)) {
+            if (auto attrs = std::get_if<std::vector<Symbol>>(&cachedValue->second)) {
+                debug("using cached attrset attribute '%s'", getAttrPathStr());
+                return *attrs;
+            } else
+                throw TypeError("'%s' is not an attribute set", getAttrPathStr());
+        }
+    }
+
+    auto & v = forceValue();
+
+    if (v.type() != nAttrs)
+        throw TypeError("'%s' is not an attribute set", getAttrPathStr());
+
+    std::vector<Symbol> attrs;
+    for (auto & attr : *getValue().attrs)
+        attrs.push_back(attr.name);
+    std::sort(attrs.begin(), attrs.end(), [](const Symbol & a, const Symbol & b) {
+        return (const string &) a < (const string &) b;
+    });
+
+    if (root->db)
+        cachedValue = {root->db->setAttrs(getKey(), attrs), attrs};
+
+    return attrs;
+}
+
+bool AttrCursor::isDerivation()
+{
+    auto aType = maybeGetAttr("type");
+    return aType && aType->getString() == "derivation";
+}
+
+StorePath AttrCursor::forceDerivation()
+{
+    auto aDrvPath = getAttr(root->state.sDrvPath, true);
+    auto drvPath = root->state.store->parseStorePath(aDrvPath->getString());
+    if (!root->state.store->isValidPath(drvPath) && !settings.readOnlyMode) {
+        /* The eval cache contains 'drvPath', but the actual path has
+           been garbage-collected. So force it to be regenerated. */
+        aDrvPath->forceValue();
+        if (!root->state.store->isValidPath(drvPath))
+            throw Error("don't know how to recreate store derivation '%s'!",
+                root->state.store->printStorePath(drvPath));
+    }
+    return drvPath;
+}
+
+}

src/libexpr/eval-cache.hh

@@ -0,0 +1,123 @@
+#pragma once
+
+#include "sync.hh"
+#include "hash.hh"
+#include "eval.hh"
+
+#include <functional>
+#include <variant>
+
+namespace nix::eval_cache {
+
+MakeError(CachedEvalError, EvalError);
+
+struct AttrDb;
+class AttrCursor;
+
+class EvalCache : public std::enable_shared_from_this<EvalCache>
+{
+    friend class AttrCursor;
+
+    std::shared_ptr<AttrDb> db;
+    EvalState & state;
+    typedef std::function<Value *()> RootLoader;
+    RootLoader rootLoader;
+    RootValue value;
+
+    Value * getRootValue();
+
+public:
+
+    EvalCache(
+        std::optional<std::reference_wrapper<const Hash>> useCache,
+        EvalState & state,
+        RootLoader rootLoader);
+
+    std::shared_ptr<AttrCursor> getRoot();
+};
+
+enum AttrType {
+    Placeholder = 0,
+    FullAttrs = 1,
+    String = 2,
+    Missing = 3,
+    Misc = 4,
+    Failed = 5,
+    Bool = 6,
+};
+
+struct placeholder_t {};
+struct missing_t {};
+struct misc_t {};
+struct failed_t {};
+typedef uint64_t AttrId;
+typedef std::pair<AttrId, Symbol> AttrKey;
+typedef std::pair<std::string, std::vector<std::pair<Path, std::string>>> string_t;
+
+typedef std::variant<
+    std::vector<Symbol>,
+    string_t,
+    placeholder_t,
+    missing_t,
+    misc_t,
+    failed_t,
+    bool
+    > AttrValue;
+
+class AttrCursor : public std::enable_shared_from_this<AttrCursor>
+{
+    friend class EvalCache;
+
+    ref<EvalCache> root;
+    typedef std::optional<std::pair<std::shared_ptr<AttrCursor>, Symbol>> Parent;
+    Parent parent;
+    RootValue _value;
+    std::optional<std::pair<AttrId, AttrValue>> cachedValue;
+
+    AttrKey getKey();
+
+    Value & getValue();
+
+public:
+
+    AttrCursor(
+        ref<EvalCache> root,
+        Parent parent,
+        Value * value = nullptr,
+        std::optional<std::pair<AttrId, AttrValue>> && cachedValue = {});
+
+    std::vector<Symbol> getAttrPath() const;
+
+    std::vector<Symbol> getAttrPath(Symbol name) const;
+
+    std::string getAttrPathStr() const;
+
+    std::string getAttrPathStr(Symbol name) const;
+
+    std::shared_ptr<AttrCursor> maybeGetAttr(Symbol name, bool forceErrors = false);
+
+    std::shared_ptr<AttrCursor> maybeGetAttr(std::string_view name);
+
+    std::shared_ptr<AttrCursor> getAttr(Symbol name, bool forceErrors = false);
+
+    std::shared_ptr<AttrCursor> getAttr(std::string_view name);
+
+    std::shared_ptr<AttrCursor> findAlongAttrPath(const std::vector<Symbol> & attrPath, bool force = false);
+
+    std::string getString();
+
+    string_t getStringWithContext();
+
+    bool getBool();
+
+    std::vector<Symbol> getAttrs();
+
+    bool isDerivation();
+
+    Value & forceValue();
+
+    /* Force creation of the .drv file in the Nix store. */
+    StorePath forceDerivation();
+};
+
+}

src/libexpr/eval.cc

@@ -201,6 +201,15 @@ string showType(const Value & v)
     }
 }
 
+Pos Value::determinePos(const Pos &pos) const
+{
+    switch (internalType) {
+        case tAttrs: return *attrs->pos;
+        case tLambda: return lambda.fun->pos;
+        case tApp: return app.left->determinePos(pos);
+        default: return pos;
+    }
+}
 
 bool Value::isTrivial() const
 {
@@ -433,9 +442,6 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
 
 EvalState::~EvalState()
 {
-    for (auto [_, cache] : evalCache) {
-        cache->commit();
-    }
 }
 
 
@@ -595,10 +601,8 @@ Value & EvalState::getBuiltin(const string & name)
 
 std::optional<EvalState::Doc> EvalState::getDoc(Value & v)
 {
-    if (v.isPrimOp() || v.isPrimOpApp()) {
+    if (v.isPrimOp()) {
         auto v2 = &v;
-        while (v2->isPrimOpApp())
-            v2 = v2->primOpApp.left;
         if (v2->primOp->doc)
             return Doc {
                 .pos = noPos,
@@ -1065,6 +1069,8 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v)
         v.attrs->push_back(Attr(nameSym, i.valueExpr->maybeThunk(state, *dynamicEnv), &i.pos));
         v.attrs->sort(); // FIXME: inefficient
     }
+
+    v.attrs->pos = &pos;
 }
 
 
@@ -1121,396 +1127,48 @@ static string showAttrPath(EvalState & state, Env & env, const AttrPath & attrPa
 
 unsigned long nrLookups = 0;
 
-tree_cache::AttrValue cachedValueFor(const Value& v)
-{
-    tree_cache::AttrValue valueToCache;
-    switch (v.type()) {
-        case nThunk:
-            valueToCache = tree_cache::thunk_t{};
-            break;
-        case nNull:
-        case nList:
-        case nFunction:
-        case nExternal:
-            valueToCache = tree_cache::unknown_t{};
-            break;
-        case nBool:
-            valueToCache = tree_cache::wrapped_basetype<bool>{v.boolean};
-            break;
-        case nString:
-            valueToCache = tree_cache::string_t{
-                v.string.s,
-                v.getContext()
-            };
-            break;
-        case nPath:
-            valueToCache = tree_cache::string_t{
-                v.path,
-                {}
-            };
-            break;
-        case nAttrs:
-            valueToCache = tree_cache::attributeSet_t{};
-            break;
-        case nInt:
-            valueToCache = tree_cache::wrapped_basetype<int64_t>{v.integer};
-            break;
-        case nFloat:
-            valueToCache = tree_cache::wrapped_basetype<double>{v.fpoint};
-            break;
-    };
-    return valueToCache;
-}
-
-std::optional<std::vector<Symbol>> ValueCache::listChildren(SymbolTable& symbols)
-{
-    auto ret = std::vector<Symbol>();
-    if (rawCache) {
-        auto cachedValue = rawCache->getCachedValue();
-            if (std::get_if<tree_cache::attributeSet_t>(&cachedValue)) {
-                for (auto & fieldStr : rawCache->getChildren())
-                    ret.push_back(symbols.create(fieldStr));
-            }
-            return ret;
-    }
-    return std::nullopt;
-}
-
-std::optional<std::vector<Symbol>> ValueCache::listChildrenAtPath(SymbolTable & symbols, const std::vector<Symbol> & attrPath)
-{
-    auto ret = std::vector<Symbol>();
-    if (rawCache) {
-        auto rawChildren = rawCache->getChildrenAtPath(attrPath);
-        if (!rawChildren) return std::nullopt;
-        for (auto & fieldStr : rawChildren.value())
-            ret.push_back(symbols.create(fieldStr));
-        return ret;
-    }
-    return std::nullopt;
-}
-
-std::vector<Symbol> EvalState::listAttrFields(Value & attrs, const Pos & pos)
-{
-    // First try to get it from the cache
-    if (auto cachedRes = attrs.getCache().listChildren(symbols))
-        return cachedRes.value();
-
-    auto ret = std::vector<Symbol>();
-    forceAttrs(attrs, pos);
-    ret.reserve(attrs.attrs->size());
-    for (auto & attr : *attrs.attrs) {
-        ret.push_back(attr.name);
-    }
-    return ret;
-}
-
-EvalState::AttrAccesResult EvalState::getOptionalAttrField(Value & attrs, const std::vector<Symbol> & selector, const Pos & pos)
-{
-    Value* resValue = allocValue();
-    auto accessResult = getOptionalAttrField(attrs, selector, pos, *resValue);
-    accessResult.value = resValue;
-    return accessResult;
-}
-
-ValueCache::CacheResult ValueCache::getValue(Store & store, const std::vector<Symbol> & selector, Value & dest)
-{
-    if (!rawCache)
-        return { NoCacheKey };
-    auto resultingCursor = rawCache->findAlongAttrPath(selector);
-    if (!resultingCursor)
-        return { CacheMiss };
-
-    dest.setCache(ValueCache(resultingCursor));
-
-    auto cachedValue = resultingCursor->getCachedValue();
-    return std::visit(
-        overloaded{
-            [&](tree_cache::attributeSet_t) { return ValueCache::CacheResult{ Forward }; },
-            [&](tree_cache::unknown_t) { return ValueCache::CacheResult{ UnCacheable }; },
-            [&](tree_cache::thunk_t) { return ValueCache::CacheResult{ CacheMiss }; },
-            [&](tree_cache::failed_t x) -> ValueCache::CacheResult {throw EvalError(x.error); },
-            [&](tree_cache::missing_t x) {
-                return ValueCache::CacheResult{
-                    .returnCode = CacheHit,
-                    .lastQueriedSymbolIfMissing = x.attrName
-                };
-            },
-            [&](tree_cache::string_t s) {
-                PathSet context;
-                for (auto& [pathName, outputName] : s.second) {
-                    // If the cached value depends on some non-existent
-                    // path, we need to discard it and force the evaluation
-                    // to bring back the context in the store
-                    if (!store.isValidPath(
-                            store.parseStorePath(pathName)))
-                        return ValueCache::CacheResult{UnCacheable};
-                    context.insert("!" + outputName + "!" + pathName);
-                }
-                mkString(dest, s.first, context);
-                return ValueCache::CacheResult{CacheHit};
-            },
-            [&](tree_cache::wrapped_basetype<bool> b) {
-                dest.mkBool(b.value);
-                return ValueCache::CacheResult{CacheHit};
-            },
-            [&](tree_cache::wrapped_basetype<int64_t> i) {
-                dest.mkInt(i.value);
-                return ValueCache::CacheResult{CacheHit};
-            },
-            [&](tree_cache::wrapped_basetype<double> d) {
-                dest.mkFloat(d.value);
-                return ValueCache::CacheResult{CacheHit};
-            },
-        },
-        cachedValue);
-}
-
-void EvalState::updateCacheStats(ValueCache::CacheResult cacheResult)
-{
-    switch (cacheResult.returnCode) {
-        case ValueCache::CacheHit:
-            nrCacheHits++;
-            break;
-        case ValueCache::CacheMiss:
-            nrCacheMisses++;
-            break;
-        case ValueCache::UnCacheable:
-            nrUncacheable++;
-            break;
-        case ValueCache::NoCacheKey:
-            nrUncached++;
-            break;
-        case ValueCache::Forward:
-            nrCacheHits++;
-            break;
-    };
-}
-
-struct ExprCastedVar : Expr
-{
-    Value * v;
-    ExprCastedVar(Value * v) : v(v) {};
-    void show(std::ostream & str) const override {
-        std::set<const Value*> active;
-        printValue(str, active, *v);
-    }
-
-    void bindVars(const StaticEnv & env) override {}
-    void eval(EvalState & state, Env & env, Value & v) override {
-        v = std::move(*this->v);
-    }
-    Value * maybeThunk(EvalState & state, Env & env) override {
-        return v;
-    }
-};
-
-EvalState::AttrAccesResult EvalState::lazyGetOptionalAttrField(Value & attrs, const std::vector<Symbol> & selector, const Pos & pos)
-{
-    Value * dest = allocValue();
-    auto evalCache = attrs.getCache();
-    auto cacheResult = evalCache.getValue(*store, selector, *dest);
-    updateCacheStats(cacheResult);
-
-    if (cacheResult.returnCode == ValueCache::CacheHit) {
-        if (cacheResult.lastQueriedSymbolIfMissing)
-            return {
-                .error =
-                    AttrAccessError{*cacheResult.lastQueriedSymbolIfMissing},
-                .pos = &pos,
-            };
-        else
-            return {.pos = &pos, .value = dest};
-    }
-
-    auto & newEnv(allocEnv(0));
-
-    auto recordAsVar = new ExprCastedVar(&attrs);
-
-    auto accessExpr = new ExprSelect(pos, recordAsVar, selector);
-    dest->mkThunk (&newEnv, accessExpr);
-
-    return {
-        .pos = &pos,
-        .value = dest,
-    };
-
-}
-
-EvalState::AttrAccesResult EvalState::getOptionalAttrField(Value & attrs, const std::vector<Symbol> & selector, const Pos & pos, Value & dest)
-{
-    auto evalCache = attrs.getCache();
-
-    if (auto maybeRawVal = evalCache.getRawValue();
-            maybeRawVal.has_value() &&
-            !std::holds_alternative<tree_cache::attributeSet_t>(maybeRawVal.value()) &&
-            !std::holds_alternative<tree_cache::thunk_t>(maybeRawVal.value())
-        )
-        return {
-            .error = AttrAccessError{
-                .attrName = selector[0],
-                .illTypedValue = &attrs,
-            },
-            .pos = &pos,
-        };
-
-    auto cacheResult = evalCache.getValue(*store, selector, dest);
-    updateCacheStats(cacheResult);
-
-    if (cacheResult.returnCode == ValueCache::CacheHit) {
-        if (cacheResult.lastQueriedSymbolIfMissing)
-            return {
-                .error =
-                    AttrAccessError{*cacheResult.lastQueriedSymbolIfMissing},
-                .pos = &pos,
-            };
-        else
-            return {.pos = &pos, .value = &dest};
-    }
-
-    const Pos * pos2 = &pos;
-    Value * currentValue = &attrs;
-    forceValue(*currentValue, *pos2);
-
-    auto resultingCursor = evalCache;
-    for (auto & name : selector) {
-        nrLookups++;
-        Bindings::iterator j;
-        if (currentValue->type() != nAttrs)
-            return {
-                .error = AttrAccessError{
-                    .attrName = name,
-                    .illTypedValue = currentValue,
-                },
-                .pos = pos2,
-            };
-        if ((j = currentValue->attrs->find(name)) == currentValue->attrs->end())
-            return {.error = AttrAccessError { name }, .pos = pos2 };
-        currentValue = j->value;
-        pos2 = j->pos;
-        try {
-            forceValue(*currentValue, *pos2);
-        } catch (EvalError & e) {
-            resultingCursor.addFailedChild(name, e);
-            throw;
-        };
-        if (cacheResult.returnCode == ValueCache::CacheMiss) {
-            resultingCursor = resultingCursor.addChild(name, *currentValue);
-            currentValue->setCache(resultingCursor);
-        }
-        if (countCalls && pos2) attrSelects[*pos2]++;
-    }
-
-    if (cacheResult.returnCode != ValueCache::CacheMiss) {
-        resultingCursor = dest.getCache();
-        currentValue->setCache(resultingCursor);
-    }
-
-    dest = *currentValue;
-    return { .pos = pos2 };
-}
-
-const ValueCache ValueCache::empty = ValueCache(nullptr);
-
-std::optional<tree_cache::AttrValue> ValueCache::getRawValue()
-{
-    if (!rawCache)
-        return std::nullopt;
-    return rawCache->getCachedValue();
-}
-
-ValueCache ValueCache::addChild(const Symbol& name, const Value& value)
-{
-    if (!rawCache)
-        return ValueCache::empty;
-
-    auto cachedValue = cachedValueFor(value);
-    auto ret = ValueCache(rawCache->addChild(name, cachedValue));
-    if (value.type() == nAttrs)
-        ret.addAttrSetChilds(*value.attrs);
-    return ret;
-}
-
-ValueCache ValueCache::addFailedChild(const Symbol& name, const Error & error)
-{
-    if (!rawCache) return ValueCache::empty;
-    return ValueCache(rawCache->addChild(name, tree_cache::failed_t{ .error = error.msg() }));
-}
-
-ValueCache ValueCache::addNumChild(SymbolTable & symbols, int idx, const Value & value)
-{
-    return addChild(symbols.create(std::to_string(idx)), value);
-}
-
-void ValueCache::addAttrSetChilds(Bindings & children)
-{
-    if (!rawCache) return;
-    for (auto & attr : children) {
-        // We could in theory directly store the value, but that would cause
-        // an infinite recursion in case of a cyclic attrset.
-        // So in a first time, just store a thunk
-        rawCache->addChild(attr.name, tree_cache::thunk_t{});
-        /* addChild(attr.name, *attr.value); */
-    }
-
-}
-
-void ValueCache::addListChilds(SymbolTable & symbols, Value** elems, int listSize)
-{
-    if (!rawCache) return;
-    for (auto i = 0; i < listSize; i++) {
-        addNumChild(symbols, i, *elems[i]);
-    }
-}
-
-void EvalState::getAttrField(Value & attrs, const std::vector<Symbol> & selector, const Pos & pos, Value & dest)
-{
-    auto missingFieldInfo = getOptionalAttrField(attrs, selector, pos, dest);
-    if (missingFieldInfo.error) {
-        if (missingFieldInfo.error->illTypedValue) {
-            throwTypeError("value is %1% while a set was expected", **missingFieldInfo.error->illTypedValue);
-        } else {
-            throwEvalError(
-                    *missingFieldInfo.pos,
-                    "attribute '%1%' missing", missingFieldInfo.error->attrName
-            );
-        }
-    }
-}
-
-Value* EvalState::getAttrField(Value & attrs, const std::vector<Symbol> & selector, const Pos & pos)
-{
-    Value* res = allocValue();
-    getAttrField(attrs, selector, pos, *res);
-    return res;
-}
-
 void ExprSelect::eval(EvalState & state, Env & env, Value & v)
 {
     Value vTmp;
+    Pos * pos2 = 0;
+    Value * vAttrs = &vTmp;
 
     e->eval(state, env, vTmp);
 
-    std::vector<Symbol> evaluatedAttrs;
-    for (auto & i : attrPath) {
-        evaluatedAttrs.push_back(getName(i, state, env));
-    }
-
     try {
-        if (def) {
-            auto missingFieldInfo = state.getOptionalAttrField(vTmp, evaluatedAttrs, pos, v);
-            if (missingFieldInfo.error) {
-                def->eval(state, env, v);
-                return;
+
+        for (auto & i : attrPath) {
+            nrLookups++;
+            Bindings::iterator j;
+            Symbol name = getName(i, state, env);
+            if (def) {
+                state.forceValue(*vAttrs, pos);
+                if (vAttrs->type() != nAttrs ||
+                    (j = vAttrs->attrs->find(name)) == vAttrs->attrs->end())
+                {
+                    def->eval(state, env, v);
+                    return;
+                }
+            } else {
+                state.forceAttrs(*vAttrs, pos);
+                if ((j = vAttrs->attrs->find(name)) == vAttrs->attrs->end())
+                    throwEvalError(pos, "attribute '%1%' missing", name);
             }
-            return;
+            vAttrs = j->value;
+            pos2 = j->pos;
+            if (state.countCalls && pos2) state.attrSelects[*pos2]++;
         }
-        state.getAttrField(vTmp, evaluatedAttrs, pos, v);
+
+        state.forceValue(*vAttrs, ( pos2 != NULL ? *pos2 : this->pos ) );
+
     } catch (Error & e) {
-        if (pos.file != state.sDerivationNix)
-            addErrorTrace(e, pos, "while evaluating the attribute '%1%'",
-                    showAttrPath(state, env, attrPath));
+        if (pos2 && pos2->file != state.sDerivationNix)
+            addErrorTrace(e, *pos2, "while evaluating the attribute '%1%'",
+                showAttrPath(state, env, attrPath));
         throw;
     }
+
+    v = *vAttrs;
 }
 
 
@@ -1732,10 +1390,10 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res)
             } else if (!i.def) {
                 throwMissingArgumentError(i.pos, R"(cannot evaluate a function that has an argument without a value ('%1%')
 
-nix attempted to evaluate a function as a top level expression; in this case it must have its
-arguments supplied either by default values, or passed explicitly with --arg or --argstr.
-
-https://nixos.org/manual/nix/stable/#ss-functions)", i.name);
+Nix attempted to evaluate a function as a top level expression; in
+this case it must have its arguments supplied either by default
+values, or passed explicitly with '--arg' or '--argstr'. See
+https://nixos.org/manual/nix/stable/#ss-functions.)", i.name);
 
             }
         }
@@ -2041,6 +1699,18 @@ string EvalState::forceString(Value & v, const Pos & pos)
 }
 
 
+/* Decode a context string ‘!<name>!<path>’ into a pair <path,
+   name>. */
+std::pair<string, string> decodeContext(std::string_view s)
+{
+    if (s.at(0) == '!') {
+        size_t index = s.find("!", 1);
+        return {std::string(s.substr(index + 1)), std::string(s.substr(1, index - 1))};
+    } else
+        return {s.at(0) == '/' ? std::string(s) : std::string(s.substr(1)), ""};
+}
+
+
 void copyContext(const Value & v, PathSet & context)
 {
     if (v.string.context)
@@ -2049,7 +1719,7 @@ void copyContext(const Value & v, PathSet & context)
 }
 
 
-std::vector<std::pair<Path, std::string>> Value::getContext() const
+std::vector<std::pair<Path, std::string>> Value::getContext()
 {
     std::vector<std::pair<Path, std::string>> res;
     assert(internalType == tString);
@@ -2059,15 +1729,6 @@ std::vector<std::pair<Path, std::string>> Value::getContext() const
     return res;
 }
 
-void Value::setCache(ValueCache cache)
-{
-    evalCache = cache;
-}
-
-ValueCache Value::getCache() const
-{
-    return evalCache;
-}
 
 string EvalState::forceString(Value & v, PathSet & context, const Pos & pos)
 {
@@ -2094,12 +1755,12 @@ string EvalState::forceStringNoCtx(Value & v, const Pos & pos)
 
 bool EvalState::isDerivation(Value & v)
 {
-    auto typeAccesRes = getOptionalAttrField(v, {sType}, noPos);
-    if (typeAccesRes.error)
-        return false;
-    forceValue(*typeAccesRes.value);
-    if (typeAccesRes.value->type() != nString) return false;
-    return strcmp(typeAccesRes.value->string.s, "derivation") == 0;
+    if (v.type() != nAttrs) return false;
+    Bindings::iterator i = v.attrs->find(sType);
+    if (i == v.attrs->end()) return false;
+    forceValue(*i->value);
+    if (i->value->type() != nString) return false;
+    return strcmp(i->value->string.s, "derivation") == 0;
 }
 
 
@@ -2138,9 +1799,9 @@ string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context,
         if (maybeString) {
             return *maybeString;
         }
-        auto accessResult = getOptionalAttrField(v, {sOutPath}, pos);
-        if (accessResult.error) throwTypeError(pos, "cannot coerce a set to a string");
-        return coerceToString(pos, *accessResult.value, context, coerceMore, copyToStore);
+        auto i = v.attrs->find(sOutPath);
+        if (i == v.attrs->end()) throwTypeError(pos, "cannot coerce a set to a string");
+        return coerceToString(pos, *i->value, context, coerceMore, copyToStore);
     }
 
     if (v.type() == nExternal)
@@ -2350,13 +2011,6 @@ void EvalState::printStats()
         topObj.attr("nrLookups", nrLookups);
         topObj.attr("nrPrimOpCalls", nrPrimOpCalls);
         topObj.attr("nrFunctionCalls", nrFunctionCalls);
-        {
-            auto cache = topObj.object("evalCache");
-            cache.attr("nrCacheMisses", nrCacheMisses);
-            cache.attr("nrCacheHits", nrCacheHits);
-            cache.attr("nrUncached", nrUncached);
-            cache.attr("nrUncacheable", nrUncacheable);
-        }
 #if HAVE_BOEHMGC
         {
             auto gc = topObj.object("gc");
@@ -2448,9 +2102,12 @@ Strings EvalSettings::getDefaultNixPath()
         }
     };
 
-    add(getHome() + "/.nix-defexpr/channels");
-    add(settings.nixStateDir + "/profiles/per-user/root/channels/nixpkgs", "nixpkgs");
-    add(settings.nixStateDir + "/profiles/per-user/root/channels");
+    if (!evalSettings.restrictEval && !evalSettings.pureEval) {
+        add(getHome() + "/.nix-defexpr/channels");
+        add(settings.nixStateDir + "/profiles/per-user/root/channels/nixpkgs", "nixpkgs");
+        add(settings.nixStateDir + "/profiles/per-user/root/channels");
+    }
+
     return res;
 }
 
@@ -2458,19 +2115,5 @@ EvalSettings evalSettings;
 
 static GlobalConfig::Register rEvalSettings(&evalSettings);
 
-std::shared_ptr<tree_cache::Cache> EvalState::openTreeCache(Hash cacheKey)
-{
-    if (auto iter = evalCache.find(cacheKey); iter != evalCache.end())
-        return iter->second;
-
-    if (!(evalSettings.useEvalCache && evalSettings.pureEval))
-        return nullptr;
-    auto thisCache = tree_cache::Cache::tryCreate(
-            cacheKey,
-            symbols
-    );
-    evalCache.insert({cacheKey, thisCache});
-    return thisCache;
-}
 
 }

src/libexpr/eval.hh

@@ -1,14 +1,10 @@
 #pragma once
 
 #include "attr-set.hh"
-#include "context.hh"
 #include "value.hh"
 #include "nixexpr.hh"
 #include "symbol-table.hh"
 #include "config.hh"
-#include "hash.hh"
-#include "sqlite.hh"
-#include "tree-cache.hh"
 
 #include <map>
 #include <optional>
@@ -18,9 +14,6 @@
 
 namespace nix {
 
-namespace eval_cache {
-    class EvalCache;
-}
 
 class Store;
 class EvalState;
@@ -123,8 +116,6 @@ private:
 #endif
     FileEvalCache fileEvalCache;
 
-    std::map<Hash, std::shared_ptr<tree_cache::Cache>> evalCache;
-
     SearchPath searchPath;
 
     std::map<std::string, std::pair<bool, std::string>> searchPathResolved;
@@ -140,9 +131,6 @@ public:
     EvalState(const Strings & _searchPath, ref<Store> store);
     ~EvalState();
 
-    std::shared_ptr<eval_cache::EvalCache> openCache(Hash, std::function<Value *()> rootLoader);
-    std::shared_ptr<tree_cache::Cache> openTreeCache(Hash);
-
     void addToSearchPath(const string & s);
 
     SearchPath getSearchPath() { return searchPath; }
@@ -315,26 +303,6 @@ public:
     void mkThunk_(Value & v, Expr * expr);
     void mkPos(Value & v, Pos * pos);
 
-    struct AttrAccessError {
-        const Symbol attrName;
-
-        // To distinguish between a missing field in an attribute set
-        // and an access to something that's not an attribute set
-        std::optional<Value* > illTypedValue;
-    };
-    struct AttrAccesResult {
-        std::optional<AttrAccessError> error;
-        const Pos * pos;
-        Value* value;
-    };
-    AttrAccesResult getOptionalAttrField(Value & attrs, const std::vector<Symbol> & selector, const Pos & pos, Value & dest);
-    AttrAccesResult getOptionalAttrField(Value & attrs, const std::vector<Symbol> & selector, const Pos & pos);
-    AttrAccesResult lazyGetOptionalAttrField(Value & attrs, const std::vector<Symbol> & selector, const Pos & pos);
-    void getAttrField(Value & attrs, const std::vector<Symbol> & selector, const Pos & pos, Value & dest);
-    Value* getAttrField(Value & attrs, const std::vector<Symbol> & selector, const Pos & pos);
-
-    std::vector<Symbol> listAttrFields(Value & attrs, const Pos & pos);
-
     void concatLists(Value & v, size_t nrLists, Value * * lists, const Pos & pos);
 
     /* Print statistics. */
@@ -342,8 +310,6 @@ public:
 
     void realiseContext(const PathSet & context);
 
-    void updateCacheStats(ValueCache::CacheResult);
-
 private:
 
     unsigned long nrEnvs = 0;
@@ -357,10 +323,6 @@ private:
     unsigned long nrListConcats = 0;
     unsigned long nrPrimOpCalls = 0;
     unsigned long nrFunctionCalls = 0;
-    unsigned long nrCacheHits = 0;
-    unsigned long nrCacheMisses = 0;
-    unsigned long nrUncacheable = 0;
-    unsigned long nrUncached = 0;
 
     bool countCalls;
 

src/libexpr/flake/config.cc

@@ -22,7 +22,9 @@ static TrustedList readTrustedList()
 
 static void writeTrustedList(const TrustedList & trustedList)
 {
-    writeFile(trustedListPath(), nlohmann::json(trustedList).dump());
+    auto path = trustedListPath();
+    createDirs(dirOf(path));
+    writeFile(path, nlohmann::json(trustedList).dump());
 }
 
 void ConfigFile::apply()

src/libexpr/flake/flake.cc

@@ -617,13 +617,9 @@ void callFlake(EvalState & state,
             , "/"), **vCallFlake);
     }
 
-    vTmp1->mkApp(*vCallFlake, vLocks);
-    vTmp2->mkApp(vTmp1, vRootSrc);
-    vRes.mkApp(vTmp2, vRootSubdir);
-    auto fingerprint = lockedFlake.getFingerprint();
-    auto evalCache = state.openTreeCache(fingerprint);
-    auto cacheRoot = evalCache ? evalCache->getRoot() : nullptr;
-    vRes.setCache(ValueCache(cacheRoot));
+    state.callFunction(**vCallFlake, *vLocks, *vTmp1, noPos);
+    state.callFunction(*vTmp1, *vRootSrc, *vTmp2, noPos);
+    state.callFunction(*vTmp2, *vRootSubdir, vRes, noPos);
 }
 
 static void prim_getFlake(EvalState & state, const Pos & pos, Value * * args, Value & v)
@@ -662,19 +658,4 @@ Fingerprint LockedFlake::getFingerprint() const
 
 Flake::~Flake() { }
 
-Value* getFlakeValue(EvalState & state, const flake::LockedFlake lockedFlake)
-{
-    /* For testing whether the evaluation cache is
-        complete. */
-    if (getEnv("NIX_ALLOW_EVAL").value_or("1") == "0")
-        throw Error("not everything is cached, but evaluation is not allowed");
-
-    auto vFlake = state.allocValue();
-    flake::callFlake(state, lockedFlake, *vFlake);
-
-    auto aOutputs = state.getAttrField(*vFlake, {state.symbols.create("outputs")}, noPos);
-
-    return aOutputs;
-}
-
 }

src/libexpr/flake/flake.hh

@@ -6,6 +6,7 @@
 #include "value.hh"
 
 namespace nix {
+
 class EvalState;
 
 namespace fetchers { struct Tree; }
@@ -112,7 +113,7 @@ struct LockFlags
     /* Whether to commit changes to flake.lock. */
     bool commitLockFile = false;
 
-    /* Flake inputs to be overriden. */
+    /* Flake inputs to be overridden. */
     std::map<InputPath, FlakeRef> inputOverrides;
 
     /* Flake inputs to be updated. This means that any existing lock
@@ -138,5 +139,4 @@ void emitTreeAttrs(
     const fetchers::Input & input,
     Value & v, bool emptyRevFallback = false);
 
-Value* getFlakeValue(EvalState & state, const flake::LockedFlake lockedFlake);
 }

src/libexpr/get-drvs.cc

@@ -2,6 +2,7 @@
 #include "util.hh"
 #include "eval-inline.hh"
 #include "store-api.hh"
+#include "path-with-outputs.hh"
 
 #include <cstring>
 #include <regex>
@@ -10,16 +11,16 @@
 namespace nix {
 
 
-DrvInfo::DrvInfo(EvalState & state, const string & attrPath, std::optional<Value> attrs)
+DrvInfo::DrvInfo(EvalState & state, const string & attrPath, Bindings * attrs)
     : state(&state), attrs(attrs), attrPath(attrPath)
 {
 }
 
 
 DrvInfo::DrvInfo(EvalState & state, ref<Store> store, const std::string & drvPathWithOutputs)
-    : state(&state), attrs(std::nullopt), attrPath("")
+    : state(&state), attrs(nullptr), attrPath("")
 {
-    auto [drvPath, selectedOutputs] = store->parsePathWithOutputs(drvPathWithOutputs);
+    auto [drvPath, selectedOutputs] = parsePathWithOutputs(*store, drvPathWithOutputs);
 
     this->drvPath = store->printStorePath(drvPath);
 
@@ -46,28 +47,12 @@ DrvInfo::DrvInfo(EvalState & state, ref<Store> store, const std::string & drvPat
 }
 
 
-std::optional<Value> DrvInfo::queryOptionalAttr(const Symbol attrName) const
-{
-    // Erk
-    Value * mutableAttrs = const_cast<Value*>(&attrs.value());
-    auto accessResult = state->getOptionalAttrField(*mutableAttrs, {attrName}, noPos);
-    return accessResult.error ? std::nullopt : std::optional{*accessResult.value};
-}
-
-Value DrvInfo::queryAttr(const Symbol attrName) const
-{
-    Value res;
-    // Erk
-    Value * mutableAttrs = const_cast<Value*>(&attrs.value());
-    state->getAttrField(*mutableAttrs, {attrName}, noPos, res);
-    return res;
-}
-
 string DrvInfo::queryName() const
 {
     if (name == "" && attrs) {
-        Value nameVal = queryAttr(state->sName);
-        name = state->forceStringNoCtx(nameVal);
+        auto i = attrs->find(state->sName);
+        if (i == attrs->end()) throw TypeError("derivation name missing");
+        name = state->forceStringNoCtx(*i->value);
     }
     return name;
 }
@@ -76,8 +61,8 @@ string DrvInfo::queryName() const
 string DrvInfo::querySystem() const
 {
     if (system == "" && attrs) {
-        auto maybeSystemVal = queryOptionalAttr(state->sSystem);
-        system = maybeSystemVal ? state->forceStringNoCtx(*maybeSystemVal) : "unknown";
+        auto i = attrs->find(state->sSystem);
+        system = i == attrs->end() ? "unknown" : state->forceStringNoCtx(*i->value, *i->pos);
     }
     return system;
 }
@@ -86,10 +71,9 @@ string DrvInfo::querySystem() const
 string DrvInfo::queryDrvPath() const
 {
     if (drvPath == "" && attrs) {
-        auto drvPathVal = queryOptionalAttr(state->sDrvPath);
+        Bindings::iterator i = attrs->find(state->sDrvPath);
         PathSet context;
-        if (drvPathVal)
-            drvPath = state->coerceToPath(noPos, *drvPathVal, context);
+        drvPath = i != attrs->end() ? state->coerceToPath(*i->pos, *i->value, context) : "";
     }
     return drvPath;
 }
@@ -98,10 +82,10 @@ string DrvInfo::queryDrvPath() const
 string DrvInfo::queryOutPath() const
 {
     if (!outPath && attrs) {
-        auto val = queryOptionalAttr(state->sOutPath);
+        Bindings::iterator i = attrs->find(state->sOutPath);
         PathSet context;
-        if (val)
-            outPath = state->coerceToPath(noPos, *val, context);
+        if (i != attrs->end())
+            outPath = state->coerceToPath(*i->pos, *i->value, context);
     }
     if (!outPath)
         throw UnimplementedError("CA derivations are not yet supported");
@@ -113,23 +97,23 @@ DrvInfo::Outputs DrvInfo::queryOutputs(bool onlyOutputsToInstall)
 {
     if (outputs.empty()) {
         /* Get the ‘outputs’ list. */
-        std::optional<Value> outputsList;
-        if (attrs && (outputsList = queryOptionalAttr(state->sOutputs))) {
-            state->forceList(*outputsList);
+        Bindings::iterator i;
+        if (attrs && (i = attrs->find(state->sOutputs)) != attrs->end()) {
+            state->forceList(*i->value, *i->pos);
 
             /* For each output... */
-            for (unsigned int j = 0; j < outputsList->listSize(); ++j) {
+            for (unsigned int j = 0; j < i->value->listSize(); ++j) {
                 /* Evaluate the corresponding set. */
-                string name = state->forceStringNoCtx(*outputsList->listElems()[j]);
-                auto out = queryOptionalAttr(state->symbols.create(name));
-                if (!out) continue;
-                state->forceAttrs(*out);
+                string name = state->forceStringNoCtx(*i->value->listElems()[j], *i->pos);
+                Bindings::iterator out = attrs->find(state->symbols.create(name));
+                if (out == attrs->end()) continue; // FIXME: throw error?
+                state->forceAttrs(*out->value);
 
                 /* And evaluate its ‘outPath’ attribute. */
-                Value outPath;
-                state->getAttrField(*out, {state->sOutPath}, noPos, outPath);
+                Bindings::iterator outPath = out->value->attrs->find(state->sOutPath);
+                if (outPath == out->value->attrs->end()) continue; // FIXME: throw error?
                 PathSet context;
-                outputs[name] = state->coerceToPath(noPos, outPath, context);
+                outputs[name] = state->coerceToPath(*outPath->pos, *outPath->value, context);
             }
         } else
             outputs["out"] = queryOutPath();
@@ -157,8 +141,8 @@ DrvInfo::Outputs DrvInfo::queryOutputs(bool onlyOutputsToInstall)
 string DrvInfo::queryOutputName() const
 {
     if (outputName == "" && attrs) {
-        auto val = queryOptionalAttr(state->sOutputName);
-        outputName = val ? state->forceStringNoCtx(*val) : "";
+        Bindings::iterator i = attrs->find(state->sOutputName);
+        outputName = i != attrs->end() ? state->forceStringNoCtx(*i->value) : "";
     }
     return outputName;
 }
@@ -168,10 +152,10 @@ Bindings * DrvInfo::getMeta()
 {
     if (meta) return meta;
     if (!attrs) return 0;
-    auto val = queryOptionalAttr(state->sMeta);
-    if (!val) return 0;
-    state->forceAttrs(*val);
-    meta = val->attrs;
+    Bindings::iterator a = attrs->find(state->sMeta);
+    if (a == attrs->end()) return 0;
+    state->forceAttrs(*a->value, *a->pos);
+    meta = a->value->attrs;
     return meta;
 }
 
@@ -302,7 +286,7 @@ static bool getDerivation(EvalState & state, Value & v,
            derivation {...}; y = x;}'. */
         if (!done.insert(v.attrs).second) return false;
 
-        DrvInfo drv(state, attrPath, v);
+        DrvInfo drv(state, attrPath, v.attrs);
 
         drv.queryName();
 

src/libexpr/get-drvs.hh

@@ -26,8 +26,7 @@ private:
 
     bool failed = false; // set if we get an AssertionError
 
-    std::optional<Value> attrs;
-    Bindings * meta = nullptr;
+    Bindings * attrs = nullptr, * meta = nullptr;
 
     Bindings * getMeta();
 
@@ -37,12 +36,9 @@ public:
     string attrPath; /* path towards the derivation */
 
     DrvInfo(EvalState & state) : state(&state) { };
-    DrvInfo(EvalState & state, const string & attrPath, std::optional<Value> attrs);
+    DrvInfo(EvalState & state, const string & attrPath, Bindings * attrs);
     DrvInfo(EvalState & state, ref<Store> store, const std::string & drvPathWithOutputs);
 
-    Value queryAttr(const Symbol attrName) const;
-    std::optional<Value> queryOptionalAttr(const Symbol attrName) const;
-
     string queryName() const;
     string querySystem() const;
     string queryDrvPath() const;

src/libexpr/local.mk

@@ -16,7 +16,7 @@ libexpr_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/lib
 libexpr_LIBS = libutil libstore libfetchers
 
 libexpr_LDFLAGS = -lboost_context
-ifneq ($(OS), FreeBSD)
+ifeq ($(OS), Linux)
  libexpr_LDFLAGS += -ldl
 endif
 

src/libexpr/nixexpr.hh

@@ -17,7 +17,7 @@ MakeError(ThrownError, AssertionError);
 MakeError(Abort, EvalError);
 MakeError(TypeError, EvalError);
 MakeError(UndefinedVarError, Error);
-MakeError(MissingArgumentError, Error);
+MakeError(MissingArgumentError, EvalError);
 MakeError(RestrictedPathError, Error);
 
 
@@ -166,10 +166,6 @@ struct ExprSelect : Expr
     AttrPath attrPath;
     ExprSelect(const Pos & pos, Expr * e, const AttrPath & attrPath, Expr * def) : pos(pos), e(e), def(def), attrPath(attrPath) { };
     ExprSelect(const Pos & pos, Expr * e, const Symbol & name) : pos(pos), e(e), def(0) { attrPath.push_back(AttrName(name)); };
-    ExprSelect(const Pos & pos, Expr * e, const std::vector<Symbol> & names) : pos(pos), e(e), def(0) {
-        for (auto & name : names)
-            attrPath.push_back(AttrName(name));
-    };
     COMMON_METHODS
 };
 
@@ -184,6 +180,7 @@ struct ExprOpHasAttr : Expr
 struct ExprAttrs : Expr
 {
     bool recursive;
+    Pos pos;
     struct AttrDef {
         bool inherited;
         Expr * e;
@@ -203,7 +200,8 @@ struct ExprAttrs : Expr
     };
     typedef std::vector<DynamicAttrDef> DynamicAttrDefs;
     DynamicAttrDefs dynamicAttrs;
-    ExprAttrs() : recursive(false) { };
+    ExprAttrs(const Pos &pos) : recursive(false), pos(pos) { };
+    ExprAttrs() : recursive(false), pos(noPos) { };
     COMMON_METHODS
 };
 

src/libexpr/parser.y

@@ -478,7 +478,7 @@ binds
           $$->attrs[i.symbol] = ExprAttrs::AttrDef(new ExprSelect(CUR_POS, $4, i.symbol), makeCurPos(@6, data));
       }
     }
-  | { $$ = new ExprAttrs; }
+  | { $$ = new ExprAttrs(makeCurPos(@0, data)); }
   ;
 
 attrs

src/libexpr/primops.cc

@@ -21,6 +21,8 @@
 #include <regex>
 #include <dlfcn.h>
 
+#include <cmath>
+
 
 namespace nix {
 
@@ -35,7 +37,7 @@ InvalidPathError::InvalidPathError(const Path & path) :
 
 void EvalState::realiseContext(const PathSet & context)
 {
-    std::vector<StorePathWithOutputs> drvs;
+    std::vector<DerivedPath::Built> drvs;
 
     for (auto & i : context) {
         auto [ctxS, outputName] = decodeContext(i);
@@ -43,7 +45,7 @@ void EvalState::realiseContext(const PathSet & context)
         if (!store->isValidPath(ctx))
             throw InvalidPathError(store->printStorePath(ctx));
         if (!outputName.empty() && ctx.isDerivation()) {
-            drvs.push_back(StorePathWithOutputs{ctx, {outputName}});
+            drvs.push_back({ctx, {outputName}});
         }
     }
 
@@ -51,14 +53,16 @@ void EvalState::realiseContext(const PathSet & context)
 
     if (!evalSettings.enableImportFromDerivation)
         throw EvalError("attempted to realize '%1%' during evaluation but 'allow-import-from-derivation' is false",
-            store->printStorePath(drvs.begin()->path));
+            store->printStorePath(drvs.begin()->drvPath));
 
     /* For performance, prefetch all substitute info. */
     StorePathSet willBuild, willSubstitute, unknown;
     uint64_t downloadSize, narSize;
-    store->queryMissing(drvs, willBuild, willSubstitute, unknown, downloadSize, narSize);
+    std::vector<DerivedPath> buildReqs;
+    for (auto & d : drvs) buildReqs.emplace_back(DerivedPath { d });
+    store->queryMissing(buildReqs, willBuild, willSubstitute, unknown, downloadSize, narSize);
 
-    store->buildPaths(drvs);
+    store->buildPaths(buildReqs);
 
     /* Add the output of this derivations to the allowed
        paths. */
@@ -410,7 +414,7 @@ static RegisterPrimOp primop_isNull({
       Return `true` if *e* evaluates to `null`, and `false` otherwise.
 
       > **Warning**
-      >
+      > 
       > This function is *deprecated*; just write `e == null` instead.
     )",
     .fun = prim_isNull,
@@ -545,18 +549,56 @@ typedef list<Value *> ValueList;
 #endif
 
 
+static Bindings::iterator getAttr(
+    EvalState & state,
+    string funcName,
+    string attrName,
+    Bindings * attrSet,
+    const Pos & pos)
+{
+    Bindings::iterator value = attrSet->find(state.symbols.create(attrName));
+    if (value == attrSet->end()) {
+        hintformat errorMsg = hintfmt(
+            "attribute '%s' missing for call to '%s'",
+            attrName,
+            funcName
+        );
+
+        Pos aPos = *attrSet->pos;
+        if (aPos == noPos) {
+            throw TypeError({
+                .msg = errorMsg,
+                .errPos = pos,
+            });
+        } else {
+            auto e = TypeError({
+                .msg = errorMsg,
+                .errPos = aPos,
+            });
+
+            // Adding another trace for the function name to make it clear
+            // which call received wrong arguments.
+            e.addTrace(pos, hintfmt("while invoking '%s'", funcName));
+            throw e;
+        }
+    }
+
+    return value;
+}
+
 static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     state.forceAttrs(*args[0], pos);
 
     /* Get the start set. */
-    Bindings::iterator startSet =
-        args[0]->attrs->find(state.symbols.create("startSet"));
-    if (startSet == args[0]->attrs->end())
-        throw EvalError({
-            .msg = hintfmt("attribute 'startSet' required"),
-            .errPos = pos
-        });
+    Bindings::iterator startSet = getAttr(
+        state,
+        "genericClosure",
+        "startSet",
+        args[0]->attrs,
+        pos
+    );
+
     state.forceList(*startSet->value, pos);
 
     ValueList workSet;
@@ -564,13 +606,14 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar
         workSet.push_back(startSet->value->listElems()[n]);
 
     /* Get the operator. */
-    Bindings::iterator op =
-        args[0]->attrs->find(state.symbols.create("operator"));
-    if (op == args[0]->attrs->end())
-        throw EvalError({
-            .msg = hintfmt("attribute 'operator' required"),
-            .errPos = pos
-        });
+    Bindings::iterator op = getAttr(
+        state,
+        "genericClosure",
+        "operator",
+        args[0]->attrs,
+        pos
+    );
+
     state.forceValue(*op->value, pos);
 
     /* Construct the closure by applying the operator to element of
@@ -673,6 +716,44 @@ static RegisterPrimOp primop_addErrorContext(RegisterPrimOp::Info {
     .fun = prim_addErrorContext,
 });
 
+static void prim_ceil(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    auto value = state.forceFloat(*args[0], args[0]->determinePos(pos));
+    mkInt(v, ceil(value));
+}
+
+static RegisterPrimOp primop_ceil({
+    .name = "__ceil",
+    .args = {"double"},
+    .doc = R"(
+        Converts an IEEE-754 double-precision floating-point number (*double*) to
+        the next higher integer.
+
+        If the datatype is neither an integer nor a "float", an evaluation error will be
+        thrown.
+    )",
+    .fun = prim_ceil,
+});
+
+static void prim_floor(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    auto value = state.forceFloat(*args[0], args[0]->determinePos(pos));
+    mkInt(v, floor(value));
+}
+
+static RegisterPrimOp primop_floor({
+    .name = "__floor",
+    .args = {"double"},
+    .doc = R"(
+        Converts an IEEE-754 double-precision floating-point number (*double*) to
+        the next lower integer.
+
+        If the datatype is neither an integer nor a "float", an evaluation error will be
+        thrown.
+    )",
+    .fun = prim_floor,
+});
+
 /* Try evaluating the argument. Success => {success=true; value=something;},
  * else => {success=false; value=false;} */
 static void prim_tryEval(EvalState & state, const Pos & pos, Value * * args, Value & v)
@@ -813,19 +894,19 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
 {
     state.forceAttrs(*args[0], pos);
 
-    Value & arg = *(args[0]);
-
-    auto getOptionalAttr= [&](Symbol attrName) -> std::optional<Value> {
-        auto accessResult = state.getOptionalAttrField(arg, {attrName}, pos);
-        return accessResult.error ? std::nullopt : std::optional{*accessResult.value};
-    };
-
     /* Figure out the name first (for stack backtraces). */
-    state.getAttrField(arg, {state.sName}, pos, v);
+    Bindings::iterator attr = getAttr(
+        state,
+        "derivationStrict",
+        state.sName,
+        args[0]->attrs,
+        pos
+    );
+
     string drvName;
-    auto posDrvName = pos; // FIXME: Should be tho position of the `name` attribute
+    Pos & posDrvName(*attr->pos);
     try {
-        drvName = state.forceStringNoCtx(v, pos);
+        drvName = state.forceStringNoCtx(*attr->value, pos);
     } catch (Error & e) {
         e.addTrace(posDrvName, "while evaluating the derivation attribute 'name'");
         throw;
@@ -834,15 +915,15 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
     /* Check whether attributes should be passed as a JSON file. */
     std::ostringstream jsonBuf;
     std::unique_ptr<JSONObject> jsonObject;
-    if (auto maybeRawObj = getOptionalAttr(state.sStructuredAttrs)) {
-        if (state.forceBool(*maybeRawObj, pos))
+    attr = args[0]->attrs->find(state.sStructuredAttrs);
+    if (attr != args[0]->attrs->end() && state.forceBool(*attr->value, pos))
         jsonObject = std::make_unique<JSONObject>(jsonBuf);
-    }
 
     /* Check whether null attributes should be ignored. */
     bool ignoreNulls = false;
-    if (auto maybeRawIgnoreNull = getOptionalAttr(state.sIgnoreNulls))
-        ignoreNulls = state.forceBool(*maybeRawIgnoreNull, pos);
+    attr = args[0]->attrs->find(state.sIgnoreNulls);
+    if (attr != args[0]->attrs->end())
+        ignoreNulls = state.forceBool(*attr->value, pos);
 
     /* Build the derivation expression by processing the attributes. */
     Derivation drv;
@@ -858,9 +939,9 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
     StringSet outputs;
     outputs.insert("out");
 
-    for (auto & currentArg : args[0]->attrs->lexicographicOrder()) {
-        if (currentArg->name == state.sIgnoreNulls) continue;
-        const string & key = currentArg->name;
+    for (auto & i : args[0]->attrs->lexicographicOrder()) {
+        if (i->name == state.sIgnoreNulls) continue;
+        const string & key = i->name;
         vomit("processing attribute '%1%'", key);
 
         auto handleHashMode = [&](const std::string & s) {
@@ -902,26 +983,22 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
 
         try {
 
-            Value argValue;
-            state.getAttrField(arg, {currentArg->name}, pos, argValue);
-
-
             if (ignoreNulls) {
-                state.forceValue(argValue, pos);
-                if (argValue.type() == nNull) continue;
+                state.forceValue(*i->value, pos);
+                if (i->value->type() == nNull) continue;
             }
 
-            if (currentArg->name == state.sContentAddressed) {
+            if (i->name == state.sContentAddressed) {
                 settings.requireExperimentalFeature("ca-derivations");
-                contentAddressed = state.forceBool(argValue, pos);
+                contentAddressed = state.forceBool(*i->value, pos);
             }
 
             /* The `args' attribute is special: it supplies the
                command-line arguments to the builder. */
-            else if (currentArg->name == state.sArgs) {
-                state.forceList(argValue, pos);
-                for (unsigned int n = 0; n < argValue.listSize(); ++n) {
-                    string s = state.coerceToString(posDrvName, *argValue.listElems()[n], context, true);
+            else if (i->name == state.sArgs) {
+                state.forceList(*i->value, pos);
+                for (unsigned int n = 0; n < i->value->listSize(); ++n) {
+                    string s = state.coerceToString(posDrvName, *i->value->listElems()[n], context, true);
                     drv.args.push_back(s);
                 }
             }
@@ -932,39 +1009,39 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
 
                 if (jsonObject) {
 
-                    if (currentArg->name == state.sStructuredAttrs) continue;
+                    if (i->name == state.sStructuredAttrs) continue;
 
                     auto placeholder(jsonObject->placeholder(key));
-                    printValueAsJSON(state, true, argValue, placeholder, context);
+                    printValueAsJSON(state, true, *i->value, placeholder, context);
 
-                    if (currentArg->name == state.sBuilder)
-                        drv.builder = state.forceString(argValue, context, posDrvName);
-                    else if (currentArg->name == state.sSystem)
-                        drv.platform = state.forceStringNoCtx(argValue, posDrvName);
-                    else if (currentArg->name == state.sOutputHash)
-                        outputHash = state.forceStringNoCtx(argValue, posDrvName);
-                    else if (currentArg->name == state.sOutputHashAlgo)
-                        outputHashAlgo = state.forceStringNoCtx(argValue, posDrvName);
-                    else if (currentArg->name == state.sOutputHashMode)
-                        handleHashMode(state.forceStringNoCtx(argValue, posDrvName));
-                    else if (currentArg->name == state.sOutputs) {
+                    if (i->name == state.sBuilder)
+                        drv.builder = state.forceString(*i->value, context, posDrvName);
+                    else if (i->name == state.sSystem)
+                        drv.platform = state.forceStringNoCtx(*i->value, posDrvName);
+                    else if (i->name == state.sOutputHash)
+                        outputHash = state.forceStringNoCtx(*i->value, posDrvName);
+                    else if (i->name == state.sOutputHashAlgo)
+                        outputHashAlgo = state.forceStringNoCtx(*i->value, posDrvName);
+                    else if (i->name == state.sOutputHashMode)
+                        handleHashMode(state.forceStringNoCtx(*i->value, posDrvName));
+                    else if (i->name == state.sOutputs) {
                         /* Require ‘outputs’ to be a list of strings. */
-                        state.forceList(argValue, posDrvName);
+                        state.forceList(*i->value, posDrvName);
                         Strings ss;
-                        for (unsigned int n = 0; n < argValue.listSize(); ++n)
-                            ss.emplace_back(state.forceStringNoCtx(*argValue.listElems()[n], posDrvName));
+                        for (unsigned int n = 0; n < i->value->listSize(); ++n)
+                            ss.emplace_back(state.forceStringNoCtx(*i->value->listElems()[n], posDrvName));
                         handleOutputs(ss);
                     }
 
                 } else {
-                    auto s = state.coerceToString(posDrvName, argValue, context, true);
+                    auto s = state.coerceToString(*i->pos, *i->value, context, true);
                     drv.env.emplace(key, s);
-                    if (currentArg->name == state.sBuilder) drv.builder = s;
-                    else if (currentArg->name == state.sSystem) drv.platform = s;
-                    else if (currentArg->name == state.sOutputHash) outputHash = s;
-                    else if (currentArg->name == state.sOutputHashAlgo) outputHashAlgo = s;
-                    else if (currentArg->name == state.sOutputHashMode) handleHashMode(s);
-                    else if (currentArg->name == state.sOutputs)
+                    if (i->name == state.sBuilder) drv.builder = s;
+                    else if (i->name == state.sSystem) drv.platform = s;
+                    else if (i->name == state.sOutputHash) outputHash = s;
+                    else if (i->name == state.sOutputHashAlgo) outputHashAlgo = s;
+                    else if (i->name == state.sOutputHashMode) handleHashMode(s);
+                    else if (i->name == state.sOutputs)
                         handleOutputs(tokenizeString<Strings>(s));
                 }
 
@@ -1214,7 +1291,10 @@ static RegisterPrimOp primop_toPath({
 static void prim_storePath(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     if (evalSettings.pureEval)
-        throw EvalError("builtins.storePath' is not allowed in pure evaluation mode");
+        throw EvalError({
+            .msg = hintfmt("'%s' is not allowed in pure evaluation mode", "builtins.storePath"),
+            .errPos = pos
+        });
 
     PathSet context;
     Path path = state.checkSourcePath(state.coerceToPath(pos, *args[0], context));
@@ -1373,12 +1453,13 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va
         if (i != v2.attrs->end())
             prefix = state.forceStringNoCtx(*i->value, pos);
 
-        i = v2.attrs->find(state.symbols.create("path"));
-        if (i == v2.attrs->end())
-            throw EvalError({
-                .msg = hintfmt("attribute 'path' missing"),
-                .errPos = pos
-            });
+        i = getAttr(
+            state,
+            "findFile",
+            "path",
+            v2.attrs,
+            pos
+        );
 
         PathSet context;
         string path = state.coerceToString(pos, *i->value, context, false, false);
@@ -1924,26 +2005,26 @@ static RegisterPrimOp primop_path({
       An enrichment of the built-in path type, based on the attributes
       present in *args*. All are optional except `path`:
 
-        - path
+        - path\
           The underlying path.
 
-        - name
+        - name\
           The name of the path when added to the store. This can used to
           reference paths that have nix-illegal characters in their names,
           like `@`.
 
-        - filter
+        - filter\
           A function of the type expected by `builtins.filterSource`,
           with the same semantics.
 
-        - recursive
+        - recursive\
           When `false`, when `path` is added to the store it is with a
           flat hash, rather than a hash of the NAR serialization of the
           file. Thus, `path` must refer to a regular file, not a
           directory. This allows similar behavior to `fetchurl`. Defaults
           to `true`.
 
-        - sha256
+        - sha256\
           When provided, this is the expected hash of the file at the
           path. Evaluation will fail if the hash is incorrect, and
           providing a hash allows `builtins.path` to be used even when the
@@ -1962,14 +2043,13 @@ static RegisterPrimOp primop_path({
    strings. */
 static void prim_attrNames(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
-    auto attrNames = state.listAttrFields(*args[0], pos);
     state.forceAttrs(*args[0], pos);
 
-    state.mkList(v, attrNames.size());
+    state.mkList(v, args[0]->attrs->size());
 
     size_t n = 0;
-    for (auto & name : attrNames)
-        mkString(*(v.listElems()[n++] = state.allocValue()), name);
+    for (auto & i : *args[0]->attrs)
+        mkString(*(v.listElems()[n++] = state.allocValue()), i.name);
 
     std::sort(v.listElems(), v.listElems() + n,
               [](Value * v1, Value * v2) { return strcmp(v1->string.s, v2->string.s) < 0; });
@@ -2020,8 +2100,18 @@ void prim_getAttr(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     string attr = state.forceStringNoCtx(*args[0], pos);
     state.forceAttrs(*args[1], pos);
-    state.getAttrField(*args[1], {state.symbols.create(attr)}, pos, v);
     // !!! Should we create a symbol here or just do a lookup?
+    Bindings::iterator i = getAttr(
+        state,
+        "getAttr",
+        attr,
+        args[1]->attrs,
+        pos
+    );
+    // !!! add to stack trace?
+    if (state.countCalls && i->pos) state.attrSelects[*i->pos]++;
+    state.forceValue(*i->value, pos);
+    v = *i->value;
 }
 
 static RegisterPrimOp primop_getAttr({
@@ -2144,22 +2234,25 @@ static void prim_listToAttrs(EvalState & state, const Pos & pos, Value * * args,
         Value & v2(*args[0]->listElems()[i]);
         state.forceAttrs(v2, pos);
 
-        Bindings::iterator j = v2.attrs->find(state.sName);
-        if (j == v2.attrs->end())
-            throw TypeError({
-                .msg = hintfmt("'name' attribute missing in a call to 'listToAttrs'"),
-                .errPos = pos
-            });
-        string name = state.forceStringNoCtx(*j->value, pos);
+        Bindings::iterator j = getAttr(
+            state,
+            "listToAttrs",
+            state.sName,
+            v2.attrs,
+            pos
+        );
+
+        string name = state.forceStringNoCtx(*j->value, *j->pos);
 
         Symbol sym = state.symbols.create(name);
         if (seen.insert(sym).second) {
-            Bindings::iterator j2 = v2.attrs->find(state.symbols.create(state.sValue));
-            if (j2 == v2.attrs->end())
-                throw TypeError({
-                    .msg = hintfmt("'value' attribute missing in a call to 'listToAttrs'"),
-                    .errPos = pos
-                });
+            Bindings::iterator j2 = getAttr(
+                state,
+                "listToAttrs",
+                state.sValue,
+                v2.attrs,
+                pos
+            );
             v.attrs->push_back(Attr(sym, j2->value, j2->pos));
         }
     }
@@ -2354,13 +2447,7 @@ static RegisterPrimOp primop_isList({
 
 static void elemAt(EvalState & state, const Pos & pos, Value & list, int n, Value & v)
 {
-    auto evalCache = list.getCache();
-    auto atSymbol = state.symbols.create(std::to_string(n));
-    auto cacheResult = evalCache.getValue(*state.store, {atSymbol}, v);
-    if (cacheResult.returnCode == ValueCache::CacheHit) return;
-
     state.forceList(list, pos);
-    evalCache.addListChilds(state.symbols, list.listElems(), list.listSize());
     if (n < 0 || (unsigned int) n >= list.listSize())
         throw Error({
             .msg = hintfmt("list index %1% is out of bounds", n),
@@ -2428,7 +2515,7 @@ static RegisterPrimOp primop_tail({
       the argument isn’t a list or is an empty list.
 
       > **Warning**
-      >
+      > 
       > This function should generally be avoided since it's inefficient:
       > unlike Haskell's `tail`, it takes O(n) time, so recursing over a
       > list by repeatedly calling `tail` takes O(n^2) time.
@@ -2806,7 +2893,12 @@ static void prim_concatMap(EvalState & state, const Pos & pos, Value * * args, V
     for (unsigned int n = 0; n < nrLists; ++n) {
         Value * vElem = args[1]->listElems()[n];
         state.callFunction(*args[0], *vElem, lists[n], pos);
-        state.forceList(lists[n], pos);
+        try {
+            state.forceList(lists[n], lists[n].determinePos(args[0]->determinePos(pos)));
+        } catch (TypeError &e) {
+            e.addTrace(pos, hintfmt("while invoking '%s'", "concatMap"));
+            throw e;
+        }
         len += lists[n].listSize();
     }
 

src/libexpr/primops/fetchTree.cc

@@ -303,17 +303,17 @@ static RegisterPrimOp primop_fetchGit({
       of the repo at that URL is fetched. Otherwise, it can be an
       attribute with the following attributes (all except `url` optional):
 
-        - url  
+        - url\
           The URL of the repo.
 
-        - name  
+        - name\
           The name of the directory the repo should be exported to in the
           store. Defaults to the basename of the URL.
 
-        - rev  
+        - rev\
           The git revision to fetch. Defaults to the tip of `ref`.
 
-        - ref  
+        - ref\
           The git ref to look for the requested revision under. This is
           often a branch or tag name. Defaults to `HEAD`.
 
@@ -321,11 +321,11 @@ static RegisterPrimOp primop_fetchGit({
           of Nix 2.3.0 Nix will not prefix `refs/heads/` if `ref` starts
           with `refs/`.
 
-        - submodules  
+        - submodules\
           A Boolean parameter that specifies whether submodules should be
           checked out. Defaults to `false`.
 
-        - allRefs  
+        - allRefs\
           Whether to fetch all refs of the repository. With this argument being
           true, it's possible to load a `rev` from *any* `ref` (by default only
           `rev`s from the specified `ref` are supported).

src/libexpr/tree-cache.cc

@@ -1,390 +0,0 @@
-#include "tree-cache.hh"
-#include "sqlite.hh"
-#include "store-api.hh"
-#include "context.hh"
-
-namespace nix::tree_cache {
-
-static const char * schema = R"sql(
-create table if not exists Attributes (
-    id          integer primary key autoincrement not null,
-    parent      integer not null,
-    name        text,
-    type        integer not null,
-    value       text,
-    context     text,
-    unique      (parent, name)
-);
-
-create index if not exists IndexByParent on Attributes(parent, name);
-)sql";
-
-struct AttrDb
-{
-    std::atomic_bool failed{false};
-
-    struct State
-    {
-        SQLite db;
-        SQLiteStmt insertAttribute;
-        SQLiteStmt updateAttribute;
-        SQLiteStmt insertAttributeWithContext;
-        SQLiteStmt queryAttribute;
-        SQLiteStmt queryAttributes;
-        std::unique_ptr<SQLiteTxn> txn;
-    };
-
-    std::unique_ptr<Sync<State>> _state;
-
-    AttrDb(const Hash & fingerprint)
-        : _state(std::make_unique<Sync<State>>())
-    {
-        auto state(_state->lock());
-
-        Path cacheDir = getCacheDir() + "/nix/eval-cache-v3";
-        createDirs(cacheDir);
-
-        Path dbPath = cacheDir + "/" + fingerprint.to_string(Base16, false) + ".sqlite";
-
-        state->db = SQLite(dbPath);
-        state->db.isCache();
-        state->db.exec(schema);
-
-        state->insertAttribute.create(state->db,
-            "insert into Attributes(parent, name, type, value) values (?, ?, ?, ?)");
-
-        state->updateAttribute.create(state->db,
-            "update Attributes set type = ?, value = ?, context = ? where id = ?");
-
-        state->insertAttributeWithContext.create(state->db,
-            "insert into Attributes(parent, name, type, value, context) values (?, ?, ?, ?, ?)");
-
-        state->queryAttribute.create(state->db,
-            "select id, type, value, context from Attributes where parent = ? and name = ?");
-
-        state->queryAttributes.create(state->db,
-            "select name from Attributes where parent = ?");
-
-        state->txn = std::make_unique<SQLiteTxn>(state->db);
-    }
-
-    ~AttrDb()
-    {
-        try {
-            auto state(_state->lock());
-            if (!failed)
-                state->txn->commit();
-            state->txn.reset();
-        } catch (...) {
-            ignoreException();
-        }
-    }
-
-    template<typename F>
-    AttrId doSQLite(F && fun)
-    {
-        if (failed) return 0;
-        try {
-            return fun();
-        } catch (SQLiteError &) {
-            ignoreException();
-            failed = true;
-            return 0;
-        }
-    }
-
-    /**
-     * Store a leaf of the tree in the db
-     */
-    AttrId addEntry(
-        const AttrKey & key,
-        const AttrValue & value)
-    {
-        return doSQLite([&]()
-        {
-            auto state(_state->lock());
-            auto rawValue = RawValue::fromVariant(value);
-
-            state->insertAttributeWithContext.use()
-                (key.first)
-                (key.second)
-                (rawValue.type)
-                (rawValue.value.value_or(""), rawValue.value.has_value())
-                (rawValue.serializeContext())
-                .exec();
-            AttrId rowId = state->db.getLastInsertedRowId();
-            assert(rowId);
-            return rowId;
-        });
-    }
-
-    std::optional<AttrId> getId(const AttrKey& key)
-    {
-        auto state(_state->lock());
-
-        auto queryAttribute(state->queryAttribute.use()(key.first)(key.second));
-        if (!queryAttribute.next()) return std::nullopt;
-
-        return (AttrType) queryAttribute.getInt(0);
-    }
-
-    AttrId setOrUpdate(const AttrKey& key, const AttrValue& value)
-    {
-        debug("cache: miss for the attribute %s", key.second);
-        if (auto existingId = getId(key)) {
-            setValue(*existingId, value);
-            return *existingId;
-        }
-        return addEntry(key, value);
-    }
-
-    void setValue(const AttrId & id, const AttrValue & value)
-    {
-        auto state(_state->lock());
-            auto rawValue = RawValue::fromVariant(value);
-
-            state->updateAttribute.use()
-                (rawValue.type)
-                (rawValue.value.value_or(""), rawValue.value.has_value())
-                (rawValue.serializeContext())
-                (id)
-                .exec();
-    }
-
-    std::optional<std::pair<AttrId, AttrValue>> getValue(AttrKey key)
-    {
-        auto state(_state->lock());
-
-        auto queryAttribute(state->queryAttribute.use()(key.first)(key.second));
-        if (!queryAttribute.next()) return {};
-
-        auto rowId = (AttrType) queryAttribute.getInt(0);
-        auto type = (AttrType) queryAttribute.getInt(1);
-
-        switch (type) {
-            case AttrType::Attrs: {
-                return {{rowId, attributeSet_t()}};
-            }
-            case AttrType::String: {
-                std::vector<std::pair<Path, std::string>> context;
-                if (!queryAttribute.isNull(3))
-                    for (auto & s : tokenizeString<std::vector<std::string>>(queryAttribute.getStr(3), ";"))
-                        context.push_back(decodeContext(s));
-                return {{rowId, string_t{queryAttribute.getStr(2), context}}};
-            }
-            case AttrType::Bool:
-                return {{rowId, wrapped_basetype<bool>{queryAttribute.getInt(2) != 0}}};
-            case AttrType::Int:
-                return {{rowId, wrapped_basetype<int64_t>{queryAttribute.getInt(2)}}};
-            case AttrType::Double:
-                return {{rowId, wrapped_basetype<double>{(double)queryAttribute.getInt(2)}}};
-            case AttrType::Unknown:
-                return {{rowId, unknown_t{}}};
-            case AttrType::Thunk:
-                return {{rowId, thunk_t{}}};
-            case AttrType::Missing:
-                return {{rowId, missing_t{key.second}}};
-            case AttrType::Failed:
-                return {{rowId, failed_t{queryAttribute.getStr(2)}}};
-            default:
-                throw Error("unexpected type in evaluation cache");
-        }
-    }
-
-    std::vector<std::string> getChildren(AttrId parentId)
-    {
-        std::vector<std::string> res;
-        auto state(_state->lock());
-
-        auto queryAttributes(state->queryAttributes.use()(parentId));
-
-        while (queryAttributes.next())
-            res.push_back(queryAttributes.getStr(0));
-
-        return res;
-    }
-};
-
-Cache::Cache(const Hash & useCache,
-        SymbolTable & symbols)
-    : db(std::make_shared<AttrDb>(useCache))
-    , symbols(symbols)
-    , rootSymbol(symbols.create(""))
-{
-}
-
-std::shared_ptr<Cache> Cache::tryCreate(const Hash & useCache, SymbolTable & symbols)
-{
-    try {
-        return std::make_shared<Cache>(useCache, symbols);
-    } catch (SQLiteError &) {
-        ignoreException();
-        return nullptr;
-    }
-}
-
-void Cache::commit()
-{
-    if (db) {
-        debug("Saving the cache");
-        auto state(db->_state->lock());
-        if (state->txn->active) {
-            state->txn->commit();
-            state->txn.reset();
-            state->txn = std::make_unique<SQLiteTxn>(state->db);
-        }
-    }
-}
-
-Cursor::Ref Cache::getRoot()
-{
-    return new Cursor(ref(shared_from_this()), std::nullopt, thunk_t{});
-}
-
-Cursor::Cursor(
-    ref<Cache> root,
-    const Parent & parent,
-    const AttrValue& value
-    )
-    : root(root)
-    , parentId(parent ? std::optional{parent->first.cachedValue.first} : std::nullopt)
-    , label(parent ? parent->second : root->rootSymbol)
-    , cachedValue({root->db->setOrUpdate(getKey(), value), value})
-{
-}
-
-Cursor::Cursor(
-    ref<Cache> root,
-    const Parent & parent,
-    const AttrId & id,
-    const AttrValue & value
-    )
-    : root(root)
-    , parentId(parent ? std::optional{parent->first.cachedValue.first} : std::nullopt)
-    , label(parent ? parent->second : root->rootSymbol)
-    , cachedValue({id, value})
-{
-}
-
-
-AttrKey Cursor::getKey()
-{
-    if (!parentId)
-        return {0, root->rootSymbol};
-    return {*parentId, label};
-}
-
-AttrValue Cursor::getCachedValue()
-{
-    return cachedValue.second;
-}
-
-void Cursor::setValue(const AttrValue & v)
-{
-    root->db->setValue(cachedValue.first, v);
-    cachedValue.second = v;
-}
-
-Cursor::Ref Cursor::addChild(const Symbol & attrPath, const AttrValue & v)
-{
-    Parent parent = {{*this, attrPath}};
-    auto childCursor = new Cursor(
-        root,
-        parent,
-        v
-    );
-    return childCursor;
-}
-
-std::vector<std::string> Cursor::getChildren()
-{
-    return root->db->getChildren(cachedValue.first);
-}
-
-std::optional<std::vector<std::string>> Cursor::getChildrenAtPath(const std::vector<Symbol> & attrPath)
-{
-    auto cursorAtPath = findAlongAttrPath(attrPath);
-    if (cursorAtPath)
-        return cursorAtPath->getChildren();
-    return std::nullopt;
-}
-
-Cursor::Ref Cursor::maybeGetAttr(const Symbol & name)
-{
-    auto rawAttr = root->db->getValue({cachedValue.first, name});
-    if (rawAttr) {
-        Parent parent = {{*this, name}};
-        debug("cache: hit for the attribute %s", cachedValue.first);
-        return new Cursor (
-            root, parent, rawAttr->first,
-            rawAttr->second);
-    }
-    if (std::holds_alternative<attributeSet_t>(cachedValue.second)) {
-        // If the parent is an attribute set but we're not present in the db,
-        // then we're not a member of this attribute set. So mark as missing
-        return addChild(name, missing_t{name});
-    }
-    return nullptr;
-}
-
-Cursor::Ref Cursor::findAlongAttrPath(const std::vector<Symbol> & attrPath)
-{
-    auto currentCursor = this;
-    for (auto & currentAccessor : attrPath) {
-        currentCursor = currentCursor->maybeGetAttr(currentAccessor);
-        if (!currentCursor)
-            break;
-        if (std::holds_alternative<missing_t>(currentCursor->cachedValue.second))
-            break;
-        if (std::holds_alternative<failed_t>(currentCursor->cachedValue.second))
-            break;
-    }
-    return currentCursor;
-}
-
-const RawValue RawValue::fromVariant(const AttrValue & value)
-{
-    RawValue res;
-    std::visit(overloaded{
-      [&](attributeSet_t x) { res.type = AttrType::Attrs; },
-      [&](string_t x) {
-        res.type = AttrType::String;
-        res.value = x.first;
-        res.context = x.second;
-      },
-      [&](wrapped_basetype<bool> x) {
-        res.type = AttrType::Bool;
-        res.value = x.value ? "1" : "0";
-      },
-      [&](wrapped_basetype<int64_t> x) {
-        res.type = AttrType::Int;
-        res.value = std::to_string(x.value);
-      },
-      [&](wrapped_basetype<double> x) {
-        res.type = AttrType::Double;
-        res.value = std::to_string(x.value);
-      },
-      [&](unknown_t x) { res.type = AttrType::Unknown; },
-      [&](missing_t x) { res.type = AttrType::Missing; },
-      [&](thunk_t x) { res.type = AttrType::Thunk; },
-      [&](failed_t x) {
-          res.type = AttrType::Failed;
-          res.value = x.error;
-      }
-    }, value);
-    return res;
-}
-
-std::string RawValue::serializeContext() const
-{
-    std::string res;
-    for (auto & elt : context) {
-        res.append(encodeContext(elt.second, elt.first));
-        res.push_back(' ');
-    }
-    if (!res.empty())
-        res.pop_back(); // Remove the trailing space
-    return res;
-}
-
-}

src/libexpr/tree-cache.hh

@@ -1,156 +0,0 @@
-/**
- * caching for a tree-like data structure (like Nix values)
- *
- * The cache is an sqlite db whose rows are the nodes of the tree, with a
- * pointer to their parent (except for the root of course)
- */
-
-#pragma once
-
-#include "sync.hh"
-#include "hash.hh"
-#include "symbol-table.hh"
-
-#include <functional>
-#include <variant>
-
-namespace nix::tree_cache {
-
-struct AttrDb;
-class Cursor;
-
-class Cache : public std::enable_shared_from_this<Cache>
-{
-private:
-    friend class Cursor;
-
-    /**
-     * The database holding the cache
-     */
-    std::shared_ptr<AttrDb> db;
-
-    SymbolTable & symbols;
-
-    /**
-     * Distinguished symbol indicating the root of the tree
-     */
-    const Symbol rootSymbol;
-
-public:
-
-    Cache(
-        const Hash & useCache,
-        SymbolTable & symbols
-    );
-
-    static std::shared_ptr<Cache> tryCreate(const Hash & useCache, SymbolTable & symbols);
-
-    Cursor * getRoot();
-
-    /**
-     * Flush the cache to disk
-     */
-    void commit();
-};
-
-enum AttrType {
-    Unknown = 0,
-    Attrs = 1,
-    String = 2,
-    Bool = 3,
-    Int = 4,
-    Double = 5,
-    Thunk = 6,
-    Missing = 7, // Missing fields of attribute sets
-    Failed = 8,
-};
-
-struct attributeSet_t {};
-struct unknown_t {};
-struct thunk_t {};
-struct failed_t { string error; };
-struct missing_t { Symbol attrName; };
-
-// Putting several different primitive types in an `std::variant` partially
-// breaks the `std::visit(overloaded{...` hackery because of the implicit cast
-// from one to another which breaks the exhaustiveness check.
-// So we wrap them in a trivial class just to force the disambiguation
-template<typename T>
-struct wrapped_basetype{ T value; };
-
-typedef uint64_t AttrId;
-
-typedef std::pair<AttrId, Symbol> AttrKey;
-typedef std::pair<std::string, std::vector<std::pair<Path, std::string>>> string_t;
-
-typedef std::variant<
-    attributeSet_t,
-    string_t,
-    unknown_t,
-    thunk_t,
-    missing_t,
-    failed_t,
-    wrapped_basetype<bool>,
-    wrapped_basetype<int64_t>,
-    wrapped_basetype<double>
-> AttrValue;
-
-struct RawValue {
-    AttrType type;
-    std::optional<std::string> value;
-    std::vector<std::pair<Path, std::string>> context;
-
-    std::string serializeContext() const;
-
-    static const RawValue fromVariant(const AttrValue&);
-    AttrValue toVariant() const;
-};
-
-/**
- * View inside the cache.
- *
- * A `Cursor` represents a node in the cached tree (be it a leaf or not)
- */
-class Cursor : public std::enable_shared_from_this<Cursor>
-{
-    /**
-     * The overall cache of which this cursor is a view
-     */
-    ref<Cache> root;
-
-    typedef std::optional<std::pair<Cursor&, Symbol>> Parent;
-
-    std::optional<AttrId> parentId;
-    Symbol label;
-
-    std::pair<AttrId, AttrValue> cachedValue;
-
-    /**
-     * Get the identifier for this node in the database
-     */
-    AttrKey getKey();
-
-public:
-
-    using Ref = Cursor*;
-
-    // Create a new cache entry
-    Cursor(ref<Cache> root, const Parent & parent, const AttrValue&);
-    // Build a cursor from an existing cache entry
-    Cursor(ref<Cache> root, const Parent & parent, const AttrId& id, const AttrValue&);
-
-    AttrValue getCachedValue();
-
-    void setValue(const AttrValue & v);
-
-    Ref addChild(const Symbol & attrPath, const AttrValue & v);
-
-    Ref findAlongAttrPath(const std::vector<Symbol> & attrPath);
-    Ref maybeGetAttr(const Symbol & attrPath);
-
-
-    std::vector<std::string> getChildren();
-    std::optional<std::vector<std::string>> getChildrenAtPath(const std::vector<Symbol> & attrPath);
-};
-
-}

src/libexpr/value.hh

@@ -1,7 +1,6 @@
 #pragma once
 
 #include "symbol-table.hh"
-#include "tree-cache.hh"
 
 #if HAVE_BOEHMGC
 #include <gc/gc_allocator.h>
@@ -57,51 +56,8 @@ struct Pos;
 class EvalState;
 class XMLWriter;
 class JSONPlaceholder;
-class Store;
 
 
-struct Value;
-class ValueCache {
-    tree_cache::Cursor::Ref rawCache;
-
-public:
-
-    ValueCache(tree_cache::Cursor::Ref rawCache) : rawCache(rawCache) {}
-
-    const static ValueCache empty;
-
-    enum ReturnCode {
-        // The cache result was an attribute set, so we forward it later in the
-        // chain
-        Forward,
-        CacheMiss,
-        CacheHit,
-        UnCacheable,
-        NoCacheKey,
-    };
-
-    struct CacheResult {
-        ReturnCode returnCode;
-
-        // In case the query returns a `missing_t`, the symbol that's missing
-        std::optional<Symbol> lastQueriedSymbolIfMissing;
-    };
-    CacheResult getValue(Store & store, const std::vector<Symbol> & selector, Value & dest);
-
-    ValueCache addChild(const Symbol & attrName, const Value & value);
-    ValueCache addFailedChild(const Symbol & attrName, const Error & error);
-    ValueCache addNumChild(SymbolTable & symbols, int idx, const Value & value);
-    void addAttrSetChilds(Bindings & children);
-    void addListChilds(SymbolTable & symbols, Value** elems, int listSize);
-
-    std::optional<std::vector<Symbol>> listChildren(SymbolTable&);
-    std::optional<std::vector<Symbol>> listChildrenAtPath(SymbolTable&, const std::vector<Symbol> & attrPath);
-
-    std::optional<tree_cache::AttrValue> getRawValue();
-
-    ValueCache() : rawCache(nullptr) {}
-};
-
 typedef int64_t NixInt;
 typedef double NixFloat;
 
@@ -157,12 +113,6 @@ friend std::string showType(const Value & v);
 friend void printValue(std::ostream & str, std::set<const Value *> & active, const Value & v);
 
 public:
-    /*
-     * An optional evaluation cache (for flakes in particular).
-     * If this is set, then trying to get a value from this attrset will first
-     * try to get it from the cache
-     */
-    ValueCache evalCache;
 
     // Functions needed to distinguish the type
     // These should be removed eventually, by putting the functionality that's
@@ -391,24 +341,14 @@ public:
         return internalType == tList1 ? 1 : internalType == tList2 ? 2 : bigList.size;
     }
 
+    Pos determinePos(const Pos &pos) const;
+
     /* Check whether forcing this value requires a trivial amount of
        computation. In particular, function applications are
        non-trivial. */
     bool isTrivial() const;
 
-    std::vector<std::pair<Path, std::string>> getContext() const;
-
-    /*
-     * Set the associated cache view for this value.
-     * This cache will be used to speed-up some operations like accessing
-     * attribute sets elements.
-     */
-    void setCache(ValueCache);
-
-    /*
-     * Get the cache associated with this value, if any.
-     */
-    ValueCache getCache() const;
+    std::vector<std::pair<Path, std::string>> getContext();
 };
 
 

src/libfetchers/attrs.hh

@@ -6,6 +6,8 @@
 
 #include <nlohmann/json_fwd.hpp>
 
+#include <optional>
+
 namespace nix::fetchers {
 
 typedef std::variant<std::string, uint64_t, Explicit<bool>> Attr;

src/libfetchers/git.cc

@@ -6,6 +6,7 @@
 #include "url-parts.hh"
 
 #include <sys/time.h>
+#include <sys/wait.h>
 
 using namespace std::string_literals;
 
@@ -153,12 +154,14 @@ struct GitInputScheme : InputScheme
 
     std::pair<bool, std::string> getActualUrl(const Input & input) const
     {
-        // Don't clone file:// URIs (but otherwise treat them the
-        // same as remote URIs, i.e. don't use the working tree or
-        // HEAD).
+        // file:// URIs are normally not cloned (but otherwise treated the
+        // same as remote URIs, i.e. we don't use the working tree or
+        // HEAD). Exception: If _NIX_FORCE_HTTP is set, or the repo is a bare git
+        // repo, treat as a remote URI to force a clone.
         static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; // for testing
         auto url = parseURL(getStrAttr(input.attrs, "url"));
-        bool isLocal = url.scheme == "file" && !forceHttp;
+        bool isBareRepository = url.scheme == "file" && !pathExists(url.path + "/.git");
+        bool isLocal = url.scheme == "file" && !forceHttp && !isBareRepository;
         return {isLocal, isLocal ? url.path : url.base};
     }
 
@@ -363,7 +366,9 @@ struct GitInputScheme : InputScheme
                         ? "refs/*"
                         : ref->compare(0, 5, "refs/") == 0
                             ? *ref
-                            : "refs/heads/" + *ref;
+                            : ref == "HEAD"
+                                ? *ref
+                                : "refs/heads/" + *ref;
                     runProgram("git", true, { "-C", repoDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", fetchRef, fetchRef) });
                 } catch (Error & e) {
                     if (!pathExists(localRefFile)) throw;

src/libfetchers/registry.cc

@@ -114,7 +114,7 @@ static std::shared_ptr<Registry> getSystemRegistry()
 
 Path getUserRegistryPath()
 {
-    return getHome() + "/.config/nix/registry.json";
+    return getConfigDir() + "/nix/registry.json";
 }
 
 std::shared_ptr<Registry> getUserRegistry()

src/libfetchers/tarball.cc

@@ -178,7 +178,8 @@ struct TarballInputScheme : InputScheme
             && !hasSuffix(url.path, ".tar")
             && !hasSuffix(url.path, ".tar.gz")
             && !hasSuffix(url.path, ".tar.xz")
-            && !hasSuffix(url.path, ".tar.bz2"))
+            && !hasSuffix(url.path, ".tar.bz2")
+            && !hasSuffix(url.path, ".tar.zst"))
             return {};
 
         Input input;

src/libmain/common-args.cc

@@ -79,4 +79,11 @@ MixCommonArgs::MixCommonArgs(const string & programName)
     hiddenCategories.insert(cat);
 }
 
+void MixCommonArgs::initialFlagsProcessed()
+{
+    initPlugins();
+    pluginsInited();
+}
+
+
 }

src/libmain/common-args.hh

@@ -7,10 +7,14 @@ namespace nix {
 //static constexpr auto commonArgsCategory = "Miscellaneous common options";
 static constexpr auto loggingCategory = "Logging-related options";
 
-struct MixCommonArgs : virtual Args
+class MixCommonArgs : public virtual Args
 {
+    void initialFlagsProcessed() override;
+public:
     string programName;
     MixCommonArgs(const string & programName);
+protected:
+    virtual void pluginsInited() {}
 };
 
 struct MixDryRun : virtual Args

src/libmain/progress-bar.cc

@@ -122,6 +122,7 @@ public:
 
     void log(Verbosity lvl, const FormatOrString & fs) override
     {
+        if (lvl > verbosity) return;
         auto state(state_.lock());
         log(*state, lvl, fs.s);
     }

src/libmain/shared.cc

@@ -36,7 +36,7 @@ void printGCWarning()
 }
 
 
-void printMissing(ref<Store> store, const std::vector<StorePathWithOutputs> & paths, Verbosity lvl)
+void printMissing(ref<Store> store, const std::vector<DerivedPath> & paths, Verbosity lvl)
 {
     uint64_t downloadSize, narSize;
     StorePathSet willBuild, willSubstitute, unknown;
@@ -310,7 +310,7 @@ void printVersion(const string & programName)
 
 void showManPage(const string & name)
 {
-    restoreSignals();
+    restoreProcessContext();
     setenv("MANPATH", settings.nixManDir.c_str(), 1);
     execlp("man", "man", name.c_str(), nullptr);
     throw SysError("command 'man %1%' failed", name.c_str());
@@ -373,7 +373,7 @@ RunPager::RunPager()
             throw SysError("dupping stdin");
         if (!getenv("LESS"))
             setenv("LESS", "FRSXMK", 1);
-        restoreSignals();
+        restoreProcessContext();
         if (pager)
             execl("/bin/sh", "sh", "-c", pager, nullptr);
         execlp("pager", "pager", nullptr);

src/libmain/shared.hh

@@ -4,6 +4,7 @@
 #include "args.hh"
 #include "common-args.hh"
 #include "path.hh"
+#include "derived-path.hh"
 
 #include <signal.h>
 
@@ -42,7 +43,7 @@ struct StorePathWithOutputs;
 
 void printMissing(
     ref<Store> store,
-    const std::vector<StorePathWithOutputs> & paths,
+    const std::vector<DerivedPath> & paths,
     Verbosity lvl = lvlInfo);
 
 void printMissing(ref<Store> store, const StorePathSet & willBuild,

src/libstore/binary-cache-store.cc

@@ -179,6 +179,9 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
     narInfo->url = "nar/" + narInfo->fileHash->to_string(Base32, false) + ".nar"
         + (compression == "xz" ? ".xz" :
            compression == "bzip2" ? ".bz2" :
+           compression == "zstd" ? ".zst" :
+           compression == "lzip" ? ".lzip" :
+           compression == "lz4" ? ".lz4" :
            compression == "br" ? ".br" :
            "");
 
@@ -447,18 +450,43 @@ StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s
 
 std::optional<const Realisation> BinaryCacheStore::queryRealisation(const DrvOutput & id)
 {
+    if (diskCache) {
+        auto [cacheOutcome, maybeCachedRealisation] =
+            diskCache->lookupRealisation(getUri(), id);
+        switch (cacheOutcome) {
+            case NarInfoDiskCache::oValid:
+                debug("Returning a cached realisation for %s", id.to_string());
+                return *maybeCachedRealisation;
+            case NarInfoDiskCache::oInvalid:
+                debug("Returning a cached missing realisation for %s", id.to_string());
+                return {};
+            case NarInfoDiskCache::oUnknown:
+                break;
+        }
+    }
+
     auto outputInfoFilePath = realisationsPrefix + "/" + id.to_string() + ".doi";
     auto rawOutputInfo = getFile(outputInfoFilePath);
 
     if (rawOutputInfo) {
-        return {Realisation::fromJSON(
-            nlohmann::json::parse(*rawOutputInfo), outputInfoFilePath)};
+        auto realisation = Realisation::fromJSON(
+            nlohmann::json::parse(*rawOutputInfo), outputInfoFilePath);
+
+        if (diskCache)
+            diskCache->upsertRealisation(
+                getUri(), realisation);
+
+        return {realisation};
     } else {
+        if (diskCache)
+            diskCache->upsertAbsentRealisation(getUri(), id);
         return std::nullopt;
     }
 }
 
 void BinaryCacheStore::registerDrvOutput(const Realisation& info) {
+    if (diskCache)
+        diskCache->upsertRealisation(getUri(), info);
     auto filePath = realisationsPrefix + "/" + info.id.to_string() + ".doi";
     upsertFile(filePath, info.toJSON().dump(), "application/json");
 }

src/libstore/binary-cache-store.hh

@@ -34,7 +34,7 @@ private:
 protected:
 
     // The prefix under which realisation infos will be stored
-    const std::string realisationsPrefix = "/realisations";
+    const std::string realisationsPrefix = "realisations";
 
     BinaryCacheStore(const Params & params);
 

src/libstore/build/derivation-goal.hh

@@ -2,7 +2,8 @@
 
 #include "parsed-derivations.hh"
 #include "lock.hh"
-#include "local-store.hh"
+#include "store-api.hh"
+#include "pathlocks.hh"
 #include "goal.hh"
 
 namespace nix {
@@ -37,6 +38,7 @@ struct InitialOutputStatus {
 
 struct InitialOutput {
     bool wanted;
+    Hash outputHash;
     std::optional<InitialOutputStatus> known;
 };
 
@@ -48,6 +50,9 @@ struct DerivationGoal : public Goal
     /* The path of the derivation. */
     StorePath drvPath;
 
+    /* The path of the corresponding resolved derivation */
+    std::optional<BasicDerivation> resolvedDrv;
+
     /* The specific outputs that we need to build.  Empty means all of
        them. */
     StringSet wantedOutputs;
@@ -60,7 +65,7 @@ struct DerivationGoal : public Goal
     bool retrySubstitution;
 
     /* The derivation stored at drvPath. */
-    std::unique_ptr<BasicDerivation> drv;
+    std::unique_ptr<Derivation> drv;
 
     std::unique_ptr<ParsedDerivation> parsedDrv;
 
@@ -75,18 +80,6 @@ struct DerivationGoal : public Goal
 
     std::map<std::string, InitialOutput> initialOutputs;
 
-    /* User selected for running the builder. */
-    std::unique_ptr<UserLock> buildUser;
-
-    /* The process ID of the builder. */
-    Pid pid;
-
-    /* The temporary directory. */
-    Path tmpDir;
-
-    /* The path of the temporary directory in the sandbox. */
-    Path tmpDirInSandbox;
-
     /* File descriptor for the log file. */
     AutoCloseFD fdLogFile;
     std::shared_ptr<BufferedSink> logFileSink, logSink;
@@ -102,79 +95,15 @@ struct DerivationGoal : public Goal
 
     std::string currentHookLine;
 
-    /* Pipe for the builder's standard output/error. */
-    Pipe builderOut;
-
-    /* Pipe for synchronising updates to the builder namespaces. */
-    Pipe userNamespaceSync;
-
-    /* The mount namespace of the builder, used to add additional
-       paths to the sandbox as a result of recursive Nix calls. */
-    AutoCloseFD sandboxMountNamespace;
-
-    /* On Linux, whether we're doing the build in its own user
-       namespace. */
-    bool usingUserNamespace = true;
-
     /* The build hook. */
     std::unique_ptr<HookInstance> hook;
 
-    /* Whether we're currently doing a chroot build. */
-    bool useChroot = false;
-
-    Path chrootRootDir;
-
-    /* RAII object to delete the chroot directory. */
-    std::shared_ptr<AutoDelete> autoDelChroot;
-
     /* The sort of derivation we are building. */
     DerivationType derivationType;
 
-    /* Whether to run the build in a private network namespace. */
-    bool privateNetwork = false;
-
     typedef void (DerivationGoal::*GoalState)();
     GoalState state;
 
-    /* Stuff we need to pass to initChild(). */
-    struct ChrootPath {
-        Path source;
-        bool optional;
-        ChrootPath(Path source = "", bool optional = false)
-            : source(source), optional(optional)
-        { }
-    };
-    typedef map<Path, ChrootPath> DirsInChroot; // maps target path to source path
-    DirsInChroot dirsInChroot;
-
-    typedef map<string, string> Environment;
-    Environment env;
-
-#if __APPLE__
-    typedef string SandboxProfile;
-    SandboxProfile additionalSandboxProfile;
-#endif
-
-    /* Hash rewriting. */
-    StringMap inputRewrites, outputRewrites;
-    typedef map<StorePath, StorePath> RedirectedOutputs;
-    RedirectedOutputs redirectedOutputs;
-
-    /* The outputs paths used during the build.
-
-       - Input-addressed derivations or fixed content-addressed outputs are
-         sometimes built when some of their outputs already exist, and can not
-         be hidden via sandboxing. We use temporary locations instead and
-         rewrite after the build. Otherwise the regular predetermined paths are
-         put here.
-
-       - Floating content-addressed derivations do not know their final build
-         output paths until the outputs are hashed, so random locations are
-         used, and then renamed. The randomness helps guard against hidden
-         self-references.
-     */
-    OutputPathMap scratchOutputs;
-
     /* The final output paths of the build.
 
        - For input-addressed derivations, always the precomputed paths
@@ -187,11 +116,6 @@ struct DerivationGoal : public Goal
 
     BuildMode buildMode;
 
-    /* If we're repairing without a chroot, there may be outputs that
-       are valid but corrupt.  So we redirect these outputs to
-       temporary paths. */
-    StorePathSet redirectedBadOutputs;
-
     BuildResult result;
 
     /* The current round, if we're building multiple times. */
@@ -199,17 +123,6 @@ struct DerivationGoal : public Goal
 
     size_t nrRounds;
 
-    /* Path registration info from the previous round, if we're
-       building multiple times. Since this contains the hash, it
-       allows us to compare whether two rounds produced the same
-       result. */
-    std::map<Path, ValidPathInfo> prevInfos;
-
-    uid_t sandboxUid() { return usingUserNamespace ? 1000 : buildUser->getUID(); }
-    gid_t sandboxGid() { return usingUserNamespace ?  100 : buildUser->getGID(); }
-
-    const static Path homeDir;
-
     std::unique_ptr<MaintainCount<uint64_t>> mcExpectedBuilds, mcRunningBuilds;
 
     std::unique_ptr<Activity> act;
@@ -222,39 +135,13 @@ struct DerivationGoal : public Goal
     /* The remote machine on which we're building. */
     std::string machineName;
 
-    /* The recursive Nix daemon socket. */
-    AutoCloseFD daemonSocket;
-
-    /* The daemon main thread. */
-    std::thread daemonThread;
-
-    /* The daemon worker threads. */
-    std::vector<std::thread> daemonWorkerThreads;
-
-    /* Paths that were added via recursive Nix calls. */
-    StorePathSet addedPaths;
-
-    /* Recursive Nix calls are only allowed to build or realize paths
-       in the original input closure or added via a recursive Nix call
-       (so e.g. you can't do 'nix-store -r /nix/store/<bla>' where
-       /nix/store/<bla> is some arbitrary path in a binary cache). */
-    bool isAllowed(const StorePath & path)
-    {
-        return inputPaths.count(path) || addedPaths.count(path);
-    }
-
-    friend struct RestrictedStore;
-
     DerivationGoal(const StorePath & drvPath,
         const StringSet & wantedOutputs, Worker & worker,
         BuildMode buildMode = bmNormal);
     DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv,
         const StringSet & wantedOutputs, Worker & worker,
         BuildMode buildMode = bmNormal);
-    ~DerivationGoal();
-
-    /* Whether we need to perform hash rewriting if there are valid output paths. */
-    bool needsHashRewrite();
+    virtual ~DerivationGoal();
 
     void timedOut(Error && ex) override;
 
@@ -276,7 +163,7 @@ struct DerivationGoal : public Goal
     void closureRepaired();
     void inputsRealised();
     void tryToBuild();
-    void tryLocalBuild();
+    virtual void tryLocalBuild();
     void buildDone();
 
     void resolvedFinished();
@@ -284,49 +171,33 @@ struct DerivationGoal : public Goal
     /* Is the build hook willing to perform the build? */
     HookReply tryBuildHook();
 
-    /* Start building a derivation. */
-    void startBuilder();
-
-    /* Fill in the environment for the builder. */
-    void initEnv();
-
-    /* Setup tmp dir location. */
-    void initTmpDir();
-
-    /* Write a JSON file containing the derivation attributes. */
-    void writeStructuredAttrs();
-
-    void startDaemon();
-
-    void stopDaemon();
-
-    /* Add 'path' to the set of paths that may be referenced by the
-       outputs, and make it appear in the sandbox. */
-    void addDependency(const StorePath & path);
-
-    /* Make a file owned by the builder. */
-    void chownToBuilder(const Path & path);
-
-    /* Run the builder's process. */
-    void runChild();
+    virtual int getChildStatus();
 
     /* Check that the derivation outputs all exist and register them
        as valid. */
-    void registerOutputs();
-
-    /* Check that an output meets the requirements specified by the
-       'outputChecks' attribute (or the legacy
-       '{allowed,disallowed}{References,Requisites}' attributes). */
-    void checkOutputs(const std::map<std::string, ValidPathInfo> & outputs);
+    virtual void registerOutputs();
 
     /* Open a log file and a pipe to it. */
     Path openLogFile();
 
+    /* Sign the newly built realisation if the store allows it */
+    virtual void signRealisation(Realisation&) {}
+
     /* Close the log file. */
     void closeLogFile();
 
-    /* Delete the temporary directory, if we have one. */
-    void deleteTmpDir(bool force);
+    /* Close the read side of the logger pipe. */
+    virtual void closeReadPipes();
+
+    /* Cleanup hooks for buildDone() */
+    virtual void cleanupHookFinally();
+    virtual void cleanupPreChildKill();
+    virtual void cleanupPostChildKill();
+    virtual bool cleanupDecideWhetherDiskFull();
+    virtual void cleanupPostOutputsRegisteredModeCheck();
+    virtual void cleanupPostOutputsRegisteredModeNonCheck();
+
+    virtual bool isReadDesc(int fd);
 
     /* Callback used by the worker to write to the log. */
     void handleChildOutput(int fd, const string & data) override;
@@ -343,17 +214,7 @@ struct DerivationGoal : public Goal
     void checkPathValidity();
 
     /* Forcibly kill the child process, if any. */
-    void killChild();
-
-    /* Create alternative path calculated from but distinct from the
-       input, so we can avoid overwriting outputs (or other store paths)
-       that already exist. */
-    StorePath makeFallbackPath(const StorePath & path);
-    /* Make a path to another based on the output name along with the
-       derivation hash. */
-    /* FIXME add option to randomize, so we can audit whether our
-       rewrites caught everything */
-    StorePath makeFallbackPath(std::string_view outputName);
+    virtual void killChild();
 
     void repairClosure();
 
@@ -366,4 +227,6 @@ struct DerivationGoal : public Goal
     StorePathSet exportReferences(const StorePathSet & storePaths);
 };
 
+MakeError(NotDeterministic, BuildError);
+
 }

src/libstore/build/drv-output-substitution-goal.cc

@@ -0,0 +1,122 @@
+#include "drv-output-substitution-goal.hh"
+#include "worker.hh"
+#include "substitution-goal.hh"
+
+namespace nix {
+
+DrvOutputSubstitutionGoal::DrvOutputSubstitutionGoal(const DrvOutput& id, Worker & worker, RepairFlag repair, std::optional<ContentAddress> ca)
+    : Goal(worker)
+    , id(id)
+{
+    state = &DrvOutputSubstitutionGoal::init;
+    name = fmt("substitution of '%s'", id.to_string());
+    trace("created");
+}
+
+
+void DrvOutputSubstitutionGoal::init()
+{
+    trace("init");
+
+    /* If the derivation already exists, we’re done */
+    if (worker.store.queryRealisation(id)) {
+        amDone(ecSuccess);
+        return;
+    }
+
+    subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list<ref<Store>>();
+    tryNext();
+}
+
+void DrvOutputSubstitutionGoal::tryNext()
+{
+    trace("Trying next substituter");
+
+    if (subs.size() == 0) {
+        /* None left.  Terminate this goal and let someone else deal
+           with it. */
+        debug("drv output '%s' is required, but there is no substituter that can provide it", id.to_string());
+
+        /* Hack: don't indicate failure if there were no substituters.
+           In that case the calling derivation should just do a
+           build. */
+        amDone(substituterFailed ? ecFailed : ecNoSubstituters);
+
+        if (substituterFailed) {
+            worker.failedSubstitutions++;
+            worker.updateProgress();
+        }
+
+        return;
+    }
+
+    auto sub = subs.front();
+    subs.pop_front();
+
+    // FIXME: Make async
+    outputInfo = sub->queryRealisation(id);
+    if (!outputInfo) {
+        tryNext();
+        return;
+    }
+
+    for (const auto & [depId, depPath] : outputInfo->dependentRealisations) {
+        if (depId != id) {
+            if (auto localOutputInfo = worker.store.queryRealisation(depId);
+                localOutputInfo && localOutputInfo->outPath != depPath) {
+                warn(
+                    "substituter '%s' has an incompatible realisation for '%s', ignoring.\n"
+                    "Local:  %s\n"
+                    "Remote: %s",
+                    sub->getUri(),
+                    depId.to_string(),
+                    worker.store.printStorePath(localOutputInfo->outPath),
+                    worker.store.printStorePath(depPath)
+                );
+                tryNext();
+                return;
+            }
+            addWaitee(worker.makeDrvOutputSubstitutionGoal(depId));
+        }
+    }
+
+    addWaitee(worker.makePathSubstitutionGoal(outputInfo->outPath));
+
+    if (waitees.empty()) outPathValid();
+    else state = &DrvOutputSubstitutionGoal::outPathValid;
+}
+
+void DrvOutputSubstitutionGoal::outPathValid()
+{
+    assert(outputInfo);
+    trace("Output path substituted");
+
+    if (nrFailed > 0) {
+        debug("The output path of the derivation output '%s' could not be substituted", id.to_string());
+        amDone(nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed);
+        return;
+    }
+
+    worker.store.registerDrvOutput(*outputInfo);
+    finished();
+}
+
+void DrvOutputSubstitutionGoal::finished()
+{
+    trace("finished");
+    amDone(ecSuccess);
+}
+
+string DrvOutputSubstitutionGoal::key()
+{
+    /* "a$" ensures substitution goals happen before derivation
+       goals. */
+    return "a$" + std::string(id.to_string());
+}
+
+void DrvOutputSubstitutionGoal::work()
+{
+    (this->*state)();
+}
+
+}

src/libstore/build/drv-output-substitution-goal.hh

@@ -0,0 +1,50 @@
+#pragma once
+
+#include "store-api.hh"
+#include "goal.hh"
+#include "realisation.hh"
+
+namespace nix {
+
+class Worker;
+
+// Substitution of a derivation output.
+// This is done in three steps:
+// 1. Fetch the output info from a substituter
+// 2. Substitute the corresponding output path
+// 3. Register the output info
+class DrvOutputSubstitutionGoal : public Goal {
+private:
+    // The drv output we're trying to substitue
+    DrvOutput id;
+
+    // The realisation corresponding to the given output id.
+    // Will be filled once we can get it.
+    std::optional<Realisation> outputInfo;
+
+    /* The remaining substituters. */
+    std::list<ref<Store>> subs;
+
+    /* Whether a substituter failed. */
+    bool substituterFailed = false;
+
+public:
+    DrvOutputSubstitutionGoal(const DrvOutput& id, Worker & worker, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
+
+    typedef void (DrvOutputSubstitutionGoal::*GoalState)();
+    GoalState state;
+
+    void init();
+    void tryNext();
+    void outPathValid();
+    void finished();
+
+    void timedOut(Error && ex) override { abort(); };
+
+    string key() override;
+
+    void work() override;
+
+};
+
+}

src/libstore/build/entry-points.cc

@@ -2,19 +2,24 @@
 #include "worker.hh"
 #include "substitution-goal.hh"
 #include "derivation-goal.hh"
+#include "local-store.hh"
 
 namespace nix {
 
-void Store::buildPaths(const std::vector<StorePathWithOutputs> & drvPaths, BuildMode buildMode)
+void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMode)
 {
     Worker worker(*this);
 
     Goals goals;
-    for (auto & path : drvPaths) {
-        if (path.path.isDerivation())
-            goals.insert(worker.makeDerivationGoal(path.path, path.outputs, buildMode));
-        else
-            goals.insert(worker.makeSubstitutionGoal(path.path, buildMode == bmRepair ? Repair : NoRepair));
+    for (auto & br : reqs) {
+        std::visit(overloaded {
+            [&](DerivedPath::Built bfd) {
+                goals.insert(worker.makeDerivationGoal(bfd.drvPath, bfd.outputs, buildMode));
+            },
+            [&](DerivedPath::Opaque bo) {
+                goals.insert(worker.makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair));
+            },
+        }, br.raw());
     }
 
     worker.run(goals);
@@ -30,7 +35,7 @@ void Store::buildPaths(const std::vector<StorePathWithOutputs> & drvPaths, Build
         }
         if (i->exitCode != Goal::ecSuccess) {
             if (auto i2 = dynamic_cast<DerivationGoal *>(i.get())) failed.insert(i2->drvPath);
-            else if (auto i2 = dynamic_cast<SubstitutionGoal *>(i.get())) failed.insert(i2->storePath);
+            else if (auto i2 = dynamic_cast<PathSubstitutionGoal *>(i.get())) failed.insert(i2->storePath);
         }
     }
 
@@ -58,6 +63,26 @@ BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivat
         result.status = BuildResult::MiscFailure;
         result.errorMsg = e.msg();
     }
+    // XXX: Should use `goal->queryPartialDerivationOutputMap()` once it's
+    // extended to return the full realisation for each output
+    auto staticDrvOutputs = drv.outputsAndOptPaths(*this);
+    auto outputHashes = staticOutputHashes(*this, drv);
+    for (auto & [outputName, staticOutput] : staticDrvOutputs) {
+        auto outputId = DrvOutput{outputHashes.at(outputName), outputName};
+        if (staticOutput.second)
+            result.builtOutputs.insert_or_assign(
+                    outputId,
+                    Realisation{ outputId, *staticOutput.second}
+                    );
+        if (settings.isExperimentalFeatureEnabled("ca-derivations") && !derivationHasKnownOutputPaths(drv.type())) {
+            auto realisation = this->queryRealisation(outputId);
+            if (realisation)
+                result.builtOutputs.insert_or_assign(
+                        outputId,
+                        *realisation
+                        );
+        }
+    }
 
     return result;
 }
@@ -69,7 +94,7 @@ void Store::ensurePath(const StorePath & path)
     if (isValidPath(path)) return;
 
     Worker worker(*this);
-    GoalPtr goal = worker.makeSubstitutionGoal(path);
+    GoalPtr goal = worker.makePathSubstitutionGoal(path);
     Goals goals = {goal};
 
     worker.run(goals);
@@ -87,7 +112,7 @@ void Store::ensurePath(const StorePath & path)
 void LocalStore::repairPath(const StorePath & path)
 {
     Worker worker(*this);
-    GoalPtr goal = worker.makeSubstitutionGoal(path, Repair);
+    GoalPtr goal = worker.makePathSubstitutionGoal(path, Repair);
     Goals goals = {goal};
 
     worker.run(goals);

src/libstore/build/goal.cc

@@ -78,6 +78,8 @@ void Goal::amDone(ExitCode result, std::optional<Error> ex)
     }
     waiters.clear();
     worker.removeGoal(shared_from_this());
+
+    cleanup();
 }
 
 

src/libstore/build/goal.hh

@@ -100,6 +100,8 @@ struct Goal : public std::enable_shared_from_this<Goal>
     virtual string key() = 0;
 
     void amDone(ExitCode result, std::optional<Error> ex = {});
+
+    virtual void cleanup() { }
 };
 
 void addToWeakGoals(WeakGoals & goals, GoalPtr p);

src/libstore/build/local-derivation-goal.hh

@@ -0,0 +1,210 @@
+#pragma once
+
+#include "derivation-goal.hh"
+#include "local-store.hh"
+
+namespace nix {
+
+struct LocalDerivationGoal : public DerivationGoal
+{
+    LocalStore & getLocalStore();
+
+    /* User selected for running the builder. */
+    std::unique_ptr<UserLock> buildUser;
+
+    /* The process ID of the builder. */
+    Pid pid;
+
+    /* The temporary directory. */
+    Path tmpDir;
+
+    /* The path of the temporary directory in the sandbox. */
+    Path tmpDirInSandbox;
+
+    /* Pipe for the builder's standard output/error. */
+    Pipe builderOut;
+
+    /* Pipe for synchronising updates to the builder namespaces. */
+    Pipe userNamespaceSync;
+
+    /* The mount namespace of the builder, used to add additional
+       paths to the sandbox as a result of recursive Nix calls. */
+    AutoCloseFD sandboxMountNamespace;
+
+    /* On Linux, whether we're doing the build in its own user
+       namespace. */
+    bool usingUserNamespace = true;
+
+    /* Whether we're currently doing a chroot build. */
+    bool useChroot = false;
+
+    Path chrootRootDir;
+
+    /* RAII object to delete the chroot directory. */
+    std::shared_ptr<AutoDelete> autoDelChroot;
+
+    /* Whether to run the build in a private network namespace. */
+    bool privateNetwork = false;
+
+    /* Stuff we need to pass to initChild(). */
+    struct ChrootPath {
+        Path source;
+        bool optional;
+        ChrootPath(Path source = "", bool optional = false)
+            : source(source), optional(optional)
+        { }
+    };
+    typedef map<Path, ChrootPath> DirsInChroot; // maps target path to source path
+    DirsInChroot dirsInChroot;
+
+    typedef map<string, string> Environment;
+    Environment env;
+
+#if __APPLE__
+    typedef string SandboxProfile;
+    SandboxProfile additionalSandboxProfile;
+#endif
+
+    /* Hash rewriting. */
+    StringMap inputRewrites, outputRewrites;
+    typedef map<StorePath, StorePath> RedirectedOutputs;
+    RedirectedOutputs redirectedOutputs;
+
+    /* The outputs paths used during the build.
+
+       - Input-addressed derivations or fixed content-addressed outputs are
+         sometimes built when some of their outputs already exist, and can not
+         be hidden via sandboxing. We use temporary locations instead and
+         rewrite after the build. Otherwise the regular predetermined paths are
+         put here.
+
+       - Floating content-addressed derivations do not know their final build
+         output paths until the outputs are hashed, so random locations are
+         used, and then renamed. The randomness helps guard against hidden
+         self-references.
+     */
+    OutputPathMap scratchOutputs;
+
+    /* Path registration info from the previous round, if we're
+       building multiple times. Since this contains the hash, it
+       allows us to compare whether two rounds produced the same
+       result. */
+    std::map<Path, ValidPathInfo> prevInfos;
+
+    uid_t sandboxUid() { return usingUserNamespace ? 1000 : buildUser->getUID(); }
+    gid_t sandboxGid() { return usingUserNamespace ?  100 : buildUser->getGID(); }
+
+    const static Path homeDir;
+
+    /* The recursive Nix daemon socket. */
+    AutoCloseFD daemonSocket;
+
+    /* The daemon main thread. */
+    std::thread daemonThread;
+
+    /* The daemon worker threads. */
+    std::vector<std::thread> daemonWorkerThreads;
+
+    /* Paths that were added via recursive Nix calls. */
+    StorePathSet addedPaths;
+
+    /* Realisations that were added via recursive Nix calls. */
+    std::set<DrvOutput> addedDrvOutputs;
+
+    /* Recursive Nix calls are only allowed to build or realize paths
+       in the original input closure or added via a recursive Nix call
+       (so e.g. you can't do 'nix-store -r /nix/store/<bla>' where
+       /nix/store/<bla> is some arbitrary path in a binary cache). */
+    bool isAllowed(const StorePath & path)
+    {
+        return inputPaths.count(path) || addedPaths.count(path);
+    }
+    bool isAllowed(const DrvOutput & id)
+    {
+        return addedDrvOutputs.count(id);
+    }
+
+    bool isAllowed(const DerivedPath & req);
+
+    friend struct RestrictedStore;
+
+    using DerivationGoal::DerivationGoal;
+
+    virtual ~LocalDerivationGoal() override;
+
+    /* Whether we need to perform hash rewriting if there are valid output paths. */
+    bool needsHashRewrite();
+
+    /* The additional states. */
+    void tryLocalBuild() override;
+
+    /* Start building a derivation. */
+    void startBuilder();
+
+    /* Fill in the environment for the builder. */
+    void initEnv();
+
+    /* Setup tmp dir location. */
+    void initTmpDir();
+
+    /* Write a JSON file containing the derivation attributes. */
+    void writeStructuredAttrs();
+
+    void startDaemon();
+
+    void stopDaemon();
+
+    /* Add 'path' to the set of paths that may be referenced by the
+       outputs, and make it appear in the sandbox. */
+    void addDependency(const StorePath & path);
+
+    /* Make a file owned by the builder. */
+    void chownToBuilder(const Path & path);
+
+    int getChildStatus() override;
+
+    /* Run the builder's process. */
+    void runChild();
+
+    /* Check that the derivation outputs all exist and register them
+       as valid. */
+    void registerOutputs() override;
+
+    void signRealisation(Realisation &) override;
+
+    /* Check that an output meets the requirements specified by the
+       'outputChecks' attribute (or the legacy
+       '{allowed,disallowed}{References,Requisites}' attributes). */
+    void checkOutputs(const std::map<std::string, ValidPathInfo> & outputs);
+
+    /* Close the read side of the logger pipe. */
+    void closeReadPipes() override;
+
+    /* Cleanup hooks for buildDone() */
+    void cleanupHookFinally() override;
+    void cleanupPreChildKill() override;
+    void cleanupPostChildKill() override;
+    bool cleanupDecideWhetherDiskFull() override;
+    void cleanupPostOutputsRegisteredModeCheck() override;
+    void cleanupPostOutputsRegisteredModeNonCheck() override;
+
+    bool isReadDesc(int fd) override;
+
+    /* Delete the temporary directory, if we have one. */
+    void deleteTmpDir(bool force);
+
+    /* Forcibly kill the child process, if any. */
+    void killChild() override;
+
+    /* Create alternative path calculated from but distinct from the
+       input, so we can avoid overwriting outputs (or other store paths)
+       that already exist. */
+    StorePath makeFallbackPath(const StorePath & path);
+    /* Make a path to another based on the output name along with the
+       derivation hash. */
+    /* FIXME add option to randomize, so we can audit whether our
+       rewrites caught everything */
+    StorePath makeFallbackPath(std::string_view outputName);
+};
+
+}

src/libstore/build/substitution-goal.cc

@@ -5,40 +5,32 @@
 
 namespace nix {
 
-SubstitutionGoal::SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair, std::optional<ContentAddress> ca)
+PathSubstitutionGoal::PathSubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair, std::optional<ContentAddress> ca)
     : Goal(worker)
     , storePath(storePath)
     , repair(repair)
     , ca(ca)
 {
-    state = &SubstitutionGoal::init;
+    state = &PathSubstitutionGoal::init;
     name = fmt("substitution of '%s'", worker.store.printStorePath(this->storePath));
     trace("created");
     maintainExpectedSubstitutions = std::make_unique<MaintainCount<uint64_t>>(worker.expectedSubstitutions);
 }
 
 
-SubstitutionGoal::~SubstitutionGoal()
+PathSubstitutionGoal::~PathSubstitutionGoal()
 {
-    try {
-        if (thr.joinable()) {
-            // FIXME: signal worker thread to quit.
-            thr.join();
-            worker.childTerminated(this);
-        }
-    } catch (...) {
-        ignoreException();
-    }
+    cleanup();
 }
 
 
-void SubstitutionGoal::work()
+void PathSubstitutionGoal::work()
 {
     (this->*state)();
 }
 
 
-void SubstitutionGoal::init()
+void PathSubstitutionGoal::init()
 {
     trace("init");
 
@@ -59,10 +51,12 @@ void SubstitutionGoal::init()
 }
 
 
-void SubstitutionGoal::tryNext()
+void PathSubstitutionGoal::tryNext()
 {
     trace("trying next substituter");
 
+    cleanup();
+
     if (subs.size() == 0) {
         /* None left.  Terminate this goal and let someone else deal
            with it. */
@@ -142,7 +136,7 @@ void SubstitutionGoal::tryNext()
     /* Bail out early if this substituter lacks a valid
        signature. LocalStore::addToStore() also checks for this, but
        only after we've downloaded the path. */
-    if (!sub->isTrusted && worker.store.pathInfoIsTrusted(*info))
+    if (!sub->isTrusted && worker.store.pathInfoIsUntrusted(*info))
     {
         warn("substituter '%s' does not have a valid signature for path '%s'",
             sub->getUri(), worker.store.printStorePath(storePath));
@@ -154,16 +148,16 @@ void SubstitutionGoal::tryNext()
        paths referenced by this one. */
     for (auto & i : info->references)
         if (i != storePath) /* ignore self-references */
-            addWaitee(worker.makeSubstitutionGoal(i));
+            addWaitee(worker.makePathSubstitutionGoal(i));
 
     if (waitees.empty()) /* to prevent hang (no wake-up event) */
         referencesValid();
     else
-        state = &SubstitutionGoal::referencesValid;
+        state = &PathSubstitutionGoal::referencesValid;
 }
 
 
-void SubstitutionGoal::referencesValid()
+void PathSubstitutionGoal::referencesValid()
 {
     trace("all references realised");
 
@@ -177,12 +171,12 @@ void SubstitutionGoal::referencesValid()
         if (i != storePath) /* ignore self-references */
             assert(worker.store.isValidPath(i));
 
-    state = &SubstitutionGoal::tryToRun;
+    state = &PathSubstitutionGoal::tryToRun;
     worker.wakeUp(shared_from_this());
 }
 
 
-void SubstitutionGoal::tryToRun()
+void PathSubstitutionGoal::tryToRun()
 {
     trace("trying to run");
 
@@ -205,7 +199,7 @@ void SubstitutionGoal::tryToRun()
     thr = std::thread([this]() {
         try {
             /* Wake up the worker loop when we're done. */
-            Finally updateStats([this]() { outPipe.writeSide = -1; });
+            Finally updateStats([this]() { outPipe.writeSide.close(); });
 
             Activity act(*logger, actSubstitute, Logger::Fields{worker.store.printStorePath(storePath), sub->getUri()});
             PushActivity pact(act.id);
@@ -221,11 +215,11 @@ void SubstitutionGoal::tryToRun()
 
     worker.childStarted(shared_from_this(), {outPipe.readSide.get()}, true, false);
 
-    state = &SubstitutionGoal::finished;
+    state = &PathSubstitutionGoal::finished;
 }
 
 
-void SubstitutionGoal::finished()
+void PathSubstitutionGoal::finished()
 {
     trace("substitute finished");
 
@@ -249,7 +243,7 @@ void SubstitutionGoal::finished()
         }
 
         /* Try the next substitute. */
-        state = &SubstitutionGoal::tryNext;
+        state = &PathSubstitutionGoal::tryNext;
         worker.wakeUp(shared_from_this());
         return;
     }
@@ -278,14 +272,31 @@ void SubstitutionGoal::finished()
 }
 
 
-void SubstitutionGoal::handleChildOutput(int fd, const string & data)
+void PathSubstitutionGoal::handleChildOutput(int fd, const string & data)
 {
 }
 
 
-void SubstitutionGoal::handleEOF(int fd)
+void PathSubstitutionGoal::handleEOF(int fd)
 {
     if (fd == outPipe.readSide.get()) worker.wakeUp(shared_from_this());
 }
 
+
+void PathSubstitutionGoal::cleanup()
+{
+    try {
+        if (thr.joinable()) {
+            // FIXME: signal worker thread to quit.
+            thr.join();
+            worker.childTerminated(this);
+        }
+
+        outPipe.close();
+    } catch (...) {
+        ignoreException();
+    }
+}
+
+
 }

src/libstore/build/substitution-goal.hh

@@ -8,13 +8,13 @@ namespace nix {
 
 class Worker;
 
-struct SubstitutionGoal : public Goal
+struct PathSubstitutionGoal : public Goal
 {
     /* The store path that should be realised through a substitute. */
     StorePath storePath;
 
     /* The path the substituter refers to the path as. This will be
-     * different when the stores have different names. */
+       different when the stores have different names. */
     std::optional<StorePath> subPath;
 
     /* The remaining substituters. */
@@ -47,14 +47,15 @@ struct SubstitutionGoal : public Goal
     std::unique_ptr<MaintainCount<uint64_t>> maintainExpectedSubstitutions,
         maintainRunningSubstitutions, maintainExpectedNar, maintainExpectedDownload;
 
-    typedef void (SubstitutionGoal::*GoalState)();
+    typedef void (PathSubstitutionGoal::*GoalState)();
     GoalState state;
 
     /* Content address for recomputing store path */
     std::optional<ContentAddress> ca;
 
-    SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
-    ~SubstitutionGoal();
+public:
+    PathSubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
+    ~PathSubstitutionGoal();
 
     void timedOut(Error && ex) override { abort(); };
 
@@ -78,6 +79,8 @@ struct SubstitutionGoal : public Goal
     /* Callback used by the worker to write to the log. */
     void handleChildOutput(int fd, const string & data) override;
     void handleEOF(int fd) override;
+
+    void cleanup() override;
 };
 
 }