Add 'nix diff-closures' command

This command makes it easier to see what changed between two closures,
i.e. what packages/versions got added or removed, and whether there
were any notable changes in path size.

For example:

  $ nix diff-closures /nix/var/nix/profiles/system-655-link /nix/var/nix/profiles/system-658-link
  blender-bin: 2.83.0 → 2.83.2, -294.2 KiB
  curl: 7.68.0 → 7.70.0, +19.1 KiB
  firmware-linux-nonfree: 2020-01-22 → 2020-05-19, +30827.7 KiB
  ibus: -21.8 KiB
  initrd-linux: 5.4.46 → 5.4.51, +16.9 KiB
  libexif: 0.6.21 → 0.6.22, +497.6 KiB
  linux: 5.4.46 → 5.4.51, +13.2 KiB
  mesa: 19.3.3 → 19.3.5, -183.9 KiB
  nix: 2.4pre20200701_6ff9aa8 → 2.4pre20200708_9223603, +9.7 KiB
  nix-bash-completions: 0.6.8 → ∅, -57.6 KiB
  nixos-system-hagbard: 20.03.20200615.a84b797 → 20.03.20200713.add5529
  nvidia-persistenced: 440.82 → 440.100
  nvidia-settings: 440.82 → 440.100
  nvidia-x11: 440.82-5.4.46 → 440.100-5.4.51, +664.7 KiB
  pcre: 8.43 → 8.44
  php: 7.3.16 → 7.3.20, -26.2 KiB
  python3.7-youtube-dl: 2020.06.06 → 2020.06.16.1, +8.4 KiB
  samba: 4.11.5 → 4.11.9, +30.1 KiB
  sane-backends: 1.0.28 → 1.0.30, +680.5 KiB
  source: -182.0 KiB
  zfs-kernel: 0.8.3-5.4.46 → 0.8.4-5.4.51, +9.9 KiB
  zfs-user: 0.8.3 → 0.8.4, +20.1 KiB

.dir-locals.el

@@ -1,6 +1,7 @@
 ((c++-mode . (
   (c-file-style . "k&r")
   (c-basic-offset . 4)
+  (c-block-comment-prefix . "  ")
   (indent-tabs-mode . nil)
   (tab-width . 4)
   (show-trailing-whitespace . t)

.github/dependabot.yml

@@ -0,0 +1,6 @@
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"

.github/workflows/test.yml

@@ -6,9 +6,19 @@ jobs:
   tests:
     strategy:
       matrix:
-        os: [ubuntu-18.04, macos]
+        os: [ubuntu-latest, macos-latest]
     runs-on: ${{ matrix.os }}
     steps:
     - uses: actions/checkout@v2
-    - uses: cachix/install-nix-action@v8
+    - uses: cachix/install-nix-action@v10
+    - run: nix-build release.nix --arg nix '{ outPath = ./.; revCount = 123; shortRev = "abcdefgh"; }' --arg systems '[ builtins.currentSystem ]' -A installerScript -A perlBindings
+  macos_perf_test:
+    runs-on: macos-latest
+    steps:
+    - name: Disable syspolicy assessments
+      run: |
+        spctl --status
+        sudo spctl --master-disable
+    - uses: actions/checkout@v2
+    - uses: cachix/install-nix-action@v10
     - run: nix-build release.nix --arg nix '{ outPath = ./.; revCount = 123; shortRev = "abcdefgh"; }' --arg systems '[ builtins.currentSystem ]' -A installerScript -A perlBindings

.gitignore

@@ -49,6 +49,9 @@ perl/Makefile.config
 # /src/libstore/
 *.gen.*
 
+# /src/libutil/
+/src/libutil/tests/libutil-tests
+
 /src/nix/nix
 
 # /src/nix-env/

Makefile

@@ -1,8 +1,8 @@
 makefiles = \
   mk/precompiled-headers.mk \
   local.mk \
-  nix-rust/local.mk \
   src/libutil/local.mk \
+  src/libutil/tests/local.mk \
   src/libstore/local.mk \
   src/libfetchers/local.mk \
   src/libmain/local.mk \

Makefile.config.in

@@ -1,36 +1,38 @@
 AR = @AR@
 BDW_GC_LIBS = @BDW_GC_LIBS@
+BOOST_LDFLAGS = @BOOST_LDFLAGS@
 BUILD_SHARED_LIBS = @BUILD_SHARED_LIBS@
 CC = @CC@
 CFLAGS = @CFLAGS@
 CXX = @CXX@
 CXXFLAGS = @CXXFLAGS@
-LDFLAGS = @LDFLAGS@
+EDITLINE_LIBS = @EDITLINE_LIBS@
 ENABLE_S3 = @ENABLE_S3@
-HAVE_SODIUM = @HAVE_SODIUM@
+GTEST_LIBS = @GTEST_LIBS@
 HAVE_SECCOMP = @HAVE_SECCOMP@
-BOOST_LDFLAGS = @BOOST_LDFLAGS@
+HAVE_SODIUM = @HAVE_SODIUM@
+LDFLAGS = @LDFLAGS@
+LIBARCHIVE_LIBS = @LIBARCHIVE_LIBS@
+LIBBROTLI_LIBS = @LIBBROTLI_LIBS@
 LIBCURL_LIBS = @LIBCURL_LIBS@
+LIBLZMA_LIBS = @LIBLZMA_LIBS@
 OPENSSL_LIBS = @OPENSSL_LIBS@
 PACKAGE_NAME = @PACKAGE_NAME@
 PACKAGE_VERSION = @PACKAGE_VERSION@
 SODIUM_LIBS = @SODIUM_LIBS@
-LIBLZMA_LIBS = @LIBLZMA_LIBS@
 SQLITE3_LIBS = @SQLITE3_LIBS@
-LIBBROTLI_LIBS = @LIBBROTLI_LIBS@
-LIBARCHIVE_LIBS = @LIBARCHIVE_LIBS@
-EDITLINE_LIBS = @EDITLINE_LIBS@
 bash = @bash@
 bindir = @bindir@
-lsof = @lsof@
 datadir = @datadir@
 datarootdir = @datarootdir@
+doc_generate = @doc_generate@
 docdir = @docdir@
 exec_prefix = @exec_prefix@
 includedir = @includedir@
 libdir = @libdir@
 libexecdir = @libexecdir@
 localstatedir = @localstatedir@
+lsof = @lsof@
 mandir = @mandir@
 pkglibdir = $(libdir)/$(PACKAGE_NAME)
 prefix = @prefix@
@@ -38,6 +40,5 @@ sandbox_shell = @sandbox_shell@
 storedir = @storedir@
 sysconfdir = @sysconfdir@
 system = @system@
-doc_generate = @doc_generate@
 xmllint = @xmllint@
 xsltproc = @xsltproc@

configure.ac

@@ -266,6 +266,10 @@ if test "$gc" = yes; then
 fi
 
 
+# Look for gtest.
+PKG_CHECK_MODULES([GTEST], [gtest_main])
+
+
 # documentation generation switch
 AC_ARG_ENABLE(doc-gen, AC_HELP_STRING([--disable-doc-gen],
   [disable documentation generation]),

doc/manual/advanced-topics/diff-hook.xml

@@ -70,7 +70,7 @@ path just built.</para>
 
   <screen>
 $ nix-build ./deterministic.nix -A stable
-these derivations will be built:
+this derivation will be built:
   /nix/store/z98fasz2jqy9gs0xbvdj939p27jwda38-stable.drv
 building '/nix/store/z98fasz2jqy9gs0xbvdj939p27jwda38-stable.drv'...
 /nix/store/yyxlzw3vqaas7wfp04g0b1xg51f2czgq-stable
@@ -85,7 +85,7 @@ checking outputs of '/nix/store/z98fasz2jqy9gs0xbvdj939p27jwda38-stable.drv'...
 
   <screen>
 $ nix-build ./deterministic.nix -A unstable
-these derivations will be built:
+this derivation will be built:
   /nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv
 building '/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv'...
 /nix/store/krpqk0l9ib0ibi1d2w52z293zw455cap-unstable
@@ -193,7 +193,7 @@ repeat = 1
     An example output of this configuration:
     <screen>
 $ nix-build ./test.nix -A unstable
-these derivations will be built:
+this derivation will be built:
   /nix/store/ch6llwpr2h8c3jmnf3f2ghkhx59aa97f-unstable.drv
 building '/nix/store/ch6llwpr2h8c3jmnf3f2ghkhx59aa97f-unstable.drv' (round 1/2)...
 building '/nix/store/ch6llwpr2h8c3jmnf3f2ghkhx59aa97f-unstable.drv' (round 2/2)...

doc/manual/advanced-topics/post-build-hook.xml

@@ -61,7 +61,7 @@ substituters = https://cache.nixos.org/ s3://example-nix-cache
 trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= example-nix-cache-1:1/cKDz3QCCOmwcztD2eV6Coggp6rqc9DGjWv7C0G+rM=
 </programlisting>
 
-<para>we will restart the Nix daemon a later step.</para>
+<para>We will restart the Nix daemon in a later step.</para>
 </section>
 
 <section>
@@ -122,7 +122,7 @@ post-build-hook = /etc/nix/upload-to-cache.sh
 
   <screen>
 $ nix-build -E '(import &lt;nixpkgs&gt; {}).writeText "example" (builtins.toString builtins.currentTime)'
-these derivations will be built:
+this derivation will be built:
   /nix/store/s4pnfbkalzy5qz57qs6yybna8wylkig6-example.drv
 building '/nix/store/s4pnfbkalzy5qz57qs6yybna8wylkig6-example.drv'...
 running post-build-hook '/home/grahamc/projects/github.com/NixOS/nix/post-hook.sh'...
@@ -139,7 +139,7 @@ $ nix-store --delete /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example
 
 <para>Now, copy the path back from the cache:</para>
 <screen>
-$ nix store --realize /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example
+$ nix-store --realise /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example
 copying path '/nix/store/m8bmqwrch6l3h8s0k3d673xpmipcdpsa-example from 's3://example-nix-cache'...
 warning: you did not specify '--add-root'; the result might be removed by the garbage collector
 /nix/store/m8bmqwrch6l3h8s0k3d673xpmipcdpsa-example

doc/manual/command-ref/conf-file.xml

@@ -386,7 +386,7 @@ false</literal>.</para>
 
 <programlisting>
 builtins.fetchurl {
-  url = https://example.org/foo-1.2.3.tar.xz;
+  url = "https://example.org/foo-1.2.3.tar.xz";
   sha256 = "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae";
 }
 </programlisting>

doc/manual/command-ref/env-common.xml

@@ -53,7 +53,7 @@ nixpkgs=/home/eelco/Dev/nixpkgs-branch:/etc/nixos</screen>
     <envar>NIX_PATH</envar> to
 
     <screen>
-nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-15.09.tar.gz</screen>
+nixpkgs=https://github.com/NixOS/nixpkgs/archive/nixos-15.09.tar.gz</screen>
 
     tells Nix to download the latest revision in the Nixpkgs/NixOS
     15.09 channel.</para>

doc/manual/command-ref/nix-env.xml

@@ -516,7 +516,7 @@ source:
 $ nix-env -f '<nixpkgs>' -iA hello --dry-run
 (dry run; not doing anything)
 installing ‘hello-2.10’
-these paths will be fetched (0.04 MiB download, 0.19 MiB unpacked):
+this path will be fetched (0.04 MiB download, 0.19 MiB unpacked):
   /nix/store/wkhdf9jinag5750mqlax6z2zbwhqb76n-hello-2.10
   <replaceable>...</replaceable></screen>
 
@@ -526,13 +526,10 @@ these paths will be fetched (0.04 MiB download, 0.19 MiB unpacked):
 14.12 channel:
 
 <screen>
-$ nix-env -f https://github.com/NixOS/nixpkgs-channels/archive/nixos-14.12.tar.gz -iA firefox
+$ nix-env -f https://github.com/NixOS/nixpkgs/archive/nixos-14.12.tar.gz -iA firefox
 </screen>
 
-(The GitHub repository <literal>nixpkgs-channels</literal> is updated
-automatically from the main <literal>nixpkgs</literal> repository
-after certain tests have succeeded and binaries have been built and
-uploaded to the binary cache at <uri>cache.nixos.org</uri>.)</para>
+</para>
 
 </refsection>
 

doc/manual/command-ref/nix-shell.xml

@@ -258,7 +258,7 @@ path. You can override it by passing <option>-I</option> or setting
 containing the Pan package from a specific revision of Nixpkgs:
 
 <screen>
-$ nix-shell -p pan -I nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/8a3eea054838b55aca962c3fbde9c83c102b8bf2.tar.gz
+$ nix-shell -p pan -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/8a3eea054838b55aca962c3fbde9c83c102b8bf2.tar.gz
 
 [nix-shell:~]$ pan --version
 Pan 0.139
@@ -352,7 +352,7 @@ following Haskell script uses a specific branch of Nixpkgs/NixOS (the
 <programlisting><![CDATA[
 #! /usr/bin/env nix-shell
 #! nix-shell -i runghc -p "haskellPackages.ghcWithPackages (ps: [ps.HTTP ps.tagsoup])"
-#! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-18.03.tar.gz
+#! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/nixos-18.03.tar.gz
 
 import Network.HTTP
 import Text.HTML.TagSoup
@@ -370,7 +370,7 @@ If you want to be even more precise, you can specify a specific
 revision of Nixpkgs:
 
 <programlisting>
-#! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/0672315759b3e15e2121365f067c1c8c56bb4722.tar.gz
+#! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/0672315759b3e15e2121365f067c1c8c56bb4722.tar.gz
 </programlisting>
 
 </para>

doc/manual/command-ref/nix-store.xml

@@ -936,7 +936,7 @@ $ nix-store --add ./foo.c
 
 <para>The operation <option>--add-fixed</option> adds the specified paths to
 the Nix store.  Unlike <option>--add</option> paths are registered using the
-specified hashing algorithm, resulting in the same output path as a fixed output
+specified hashing algorithm, resulting in the same output path as a fixed-output
 derivation.  This can be used for sources that are not available from a public
 url or broke since the download expression was written.
 </para>

doc/manual/command-ref/opt-common-syn.xml

@@ -1,5 +1,5 @@
 <nop xmlns="http://docbook.org/ns/docbook">
-  
+
 <arg><option>--help</option></arg>
 <arg><option>--version</option></arg>
 <arg rep='repeat'>
@@ -11,6 +11,10 @@
 <arg>
   <arg choice='plain'><option>--quiet</option></arg>
 </arg>
+<arg>
+  <option>--log-format</option>
+  <replaceable>format</replaceable>
+</arg>
 <arg>
   <group choice='plain'>
     <arg choice='plain'><option>--no-build-output</option></arg>

doc/manual/command-ref/opt-common.xml

@@ -92,6 +92,37 @@
 </varlistentry>
 
 
+<varlistentry xml:id="opt-log-format"><term><option>--log-format</option> <replaceable>format</replaceable></term>
+
+  <listitem>
+
+  <para>This option can be used to change the output of the log format, with
+  <replaceable>format</replaceable> being one of:</para>
+
+  <variablelist>
+
+    <varlistentry><term>raw</term>
+    <listitem><para>This is the raw format, as outputted by nix-build.</para></listitem>
+    </varlistentry>
+
+    <varlistentry><term>internal-json</term>
+    <listitem><para>Outputs the logs in a structured manner. NOTE: the json schema is not guarantees to be stable between releases.</para></listitem>
+    </varlistentry>
+
+    <varlistentry><term>bar</term>
+    <listitem><para>Only display a progress bar during the builds.</para></listitem>
+    </varlistentry>
+
+    <varlistentry><term>bar-with-logs</term>
+    <listitem><para>Display the raw logs, with the progress bar at the bottom.</para></listitem>
+    </varlistentry>
+
+  </variablelist>
+
+  </listitem>
+
+</varlistentry>
+
 <varlistentry><term><option>--no-build-output</option> / <option>-Q</option></term>
 
   <listitem><para>By default, output written by builders to standard

doc/manual/expressions/advanced-attributes.xml

@@ -178,7 +178,7 @@ impureEnvVars = [ "http_proxy" "https_proxy" <replaceable>...</replaceable> ];
 
 <programlisting>
 fetchurl {
-  url = http://ftp.gnu.org/pub/gnu/hello/hello-2.1.1.tar.gz;
+  url = "http://ftp.gnu.org/pub/gnu/hello/hello-2.1.1.tar.gz";
   sha256 = "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465";
 }
 </programlisting>
@@ -189,7 +189,7 @@ fetchurl {
 
 <programlisting>
 fetchurl {
-  url = ftp://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz;
+  url = "ftp://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz";
   sha256 = "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465";
 }
 </programlisting>

doc/manual/expressions/builtins.xml

@@ -324,7 +324,7 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting>
     particular version of Nixpkgs, e.g.
 
 <programlisting>
-with import (fetchTarball https://github.com/NixOS/nixpkgs-channels/archive/nixos-14.12.tar.gz) {};
+with import (fetchTarball https://github.com/NixOS/nixpkgs/archive/nixos-14.12.tar.gz) {};
 
 stdenv.mkDerivation { … }
 </programlisting>
@@ -349,7 +349,7 @@ stdenv.mkDerivation { … }
 
 <programlisting>
 with import (fetchTarball {
-  url = https://github.com/NixOS/nixpkgs-channels/archive/nixos-14.12.tar.gz;
+  url = "https://github.com/NixOS/nixpkgs/archive/nixos-14.12.tar.gz";
   sha256 = "1jppksrfvbk5ypiqdz4cddxdl8z6zyzdb2srq8fcffr327ld5jj2";
 }) {};
 
@@ -1406,7 +1406,7 @@ stdenv.mkDerivation {
   ";
 
   src = fetchurl {
-    url = http://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz;
+    url = "http://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz";
     sha256 = "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465";
   };
   inherit perl;

doc/manual/expressions/expression-syntax.xml

@@ -15,7 +15,7 @@ stdenv.mkDerivation { <co xml:id='ex-hello-nix-co-2' />
   name = "hello-2.1.1"; <co xml:id='ex-hello-nix-co-3' />
   builder = ./builder.sh; <co xml:id='ex-hello-nix-co-4' />
   src = fetchurl { <co xml:id='ex-hello-nix-co-5' />
-    url = ftp://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz;
+    url = "ftp://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz";
     sha256 = "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465";
   };
   inherit perl; <co xml:id='ex-hello-nix-co-6' />

doc/manual/expressions/simple-building-testing.xml

@@ -73,12 +73,4 @@ waiting for lock on `/nix/store/0h5b7hp8d4hqfrw8igvx97x1xawrjnac-hello-2.1.1x'</
 So it is always safe to run multiple instances of Nix in parallel
 (which isn’t the case with, say, <command>make</command>).</para>
 
-<para>If you have a system with multiple CPUs, you may want to have
-Nix build different derivations in parallel (insofar as possible).
-Just pass the option <link linkend='opt-max-jobs'><option>-j
-<replaceable>N</replaceable></option></link>, where
-<replaceable>N</replaceable> is the maximum number of jobs to be run
-in parallel, or set.  Typically this should be the number of
-CPUs.</para>
-
 </section>

doc/manual/installation/env-variables.xml

@@ -39,7 +39,7 @@ bundle.</para>
   <step><para>Set the environment variable and install Nix</para>
     <screen>
 $ export NIX_SSL_CERT_FILE=/etc/ssl/my-certificate-bundle.crt
-$ sh &lt;(curl https://nixos.org/nix/install)
+$ sh &lt;(curl -L https://nixos.org/nix/install)
 </screen></step>
 
   <step><para>In the shell profile and rc files (for example,

doc/manual/installation/installing-binary.xml

@@ -6,16 +6,30 @@
 
 <title>Installing a Binary Distribution</title>
 
-<para>If you are using Linux or macOS, the easiest way to install Nix
-is to run the following command:
+<para>
+  If you are using Linux or macOS versions up to 10.14 (Mojave), the
+  easiest way to install Nix is to run the following command:
+</para>
 
 <screen>
-  $ sh &lt;(curl https://nixos.org/nix/install)
+  $ sh &lt;(curl -L https://nixos.org/nix/install)
 </screen>
 
-As of Nix 2.1.0, the Nix installer will always default to creating a
-single-user installation, however opting in to the multi-user
-installation is highly recommended.
+<para>
+  If you're using macOS 10.15 (Catalina) or newer, consult
+  <link linkend="sect-macos-installation">the macOS installation instructions</link>
+  before installing.
+</para>
+
+<para>
+  As of Nix 2.1.0, the Nix installer will always default to creating a
+  single-user installation, however opting in to the multi-user
+  installation is highly recommended.
+  <!-- TODO: this explains *neither* why the default version is
+  single-user, nor why we'd recommend multi-user over the default.
+  True prospective users don't have much basis for evaluating this.
+  What's it to me? Who should pick which? Why? What if I pick wrong?
+  -->
 </para>
 
 <section xml:id="sect-single-user-installation">
@@ -25,7 +39,7 @@ installation is highly recommended.
     To explicitly select a single-user installation on your system:
 
     <screen>
-  sh &lt;(curl https://nixos.org/nix/install) --no-daemon
+  sh &lt;(curl -L https://nixos.org/nix/install) --no-daemon
 </screen>
   </para>
 
@@ -36,7 +50,7 @@ run this under your usual user account, <emphasis>not</emphasis> as
 root.  The script will invoke <command>sudo</command> to create
 <filename>/nix</filename> if it doesn’t already exist.  If you don’t
 have <command>sudo</command>, you should manually create
-<command>/nix</command> first as root, e.g.:
+<filename>/nix</filename> first as root, e.g.:
 
 <screen>
 $ mkdir /nix
@@ -47,7 +61,7 @@ The install script will modify the first writable file from amongst
 <filename>.bash_profile</filename>, <filename>.bash_login</filename>
 and <filename>.profile</filename> to source
 <filename>~/.nix-profile/etc/profile.d/nix.sh</filename>. You can set
-the <command>NIX_INSTALLER_NO_MODIFY_PROFILE</command> environment
+the <envar>NIX_INSTALLER_NO_MODIFY_PROFILE</envar> environment
 variable before executing the install script to disable this
 behaviour.
 </para>
@@ -81,12 +95,10 @@ $ rm -rf /nix
   <para>
     You can instruct the installer to perform a multi-user
     installation on your system:
-
-    <screen>
-  sh &lt;(curl https://nixos.org/nix/install) --daemon
-</screen>
   </para>
 
+  <screen>sh &lt;(curl -L https://nixos.org/nix/install) --daemon</screen>
+
   <para>
     The multi-user installation of Nix will create build users between
     the user IDs 30001 and 30032, and a group with the group ID 30000.
@@ -136,6 +148,273 @@ sudo rm /Library/LaunchDaemons/org.nixos.nix-daemon.plist
 
 </section>
 
+<section xml:id="sect-macos-installation">
+  <title>macOS Installation</title>
+
+  <para>
+    Starting with macOS 10.15 (Catalina), the root filesystem is read-only.
+    This means <filename>/nix</filename> can no longer live on your system
+    volume, and that you'll need a workaround to install Nix.
+  </para>
+
+  <para>
+    The recommended approach, which creates an unencrypted APFS volume
+    for your Nix store and a "synthetic" empty directory to mount it
+    over at <filename>/nix</filename>, is least likely to impair Nix
+    or your system.
+  </para>
+
+  <note><para>
+    With all separate-volume approaches, it's possible something on
+    your system (particularly daemons/services and restored apps) may
+    need access to your Nix store before the volume is mounted. Adding
+    additional encryption makes this more likely.
+  </para></note>
+
+  <para>
+    If you're using a recent Mac with a
+    <link xlink:href="https://www.apple.com/euro/mac/shared/docs/Apple_T2_Security_Chip_Overview.pdf">T2 chip</link>,
+    your drive will still be encrypted at rest (in which case "unencrypted"
+    is a bit of a misnomer). To use this approach, just install Nix with:
+  </para>
+
+  <screen>$ sh &lt;(curl -L https://nixos.org/nix/install) --darwin-use-unencrypted-nix-store-volume</screen>
+
+  <para>
+    If you don't like the sound of this, you'll want to weigh the
+    other approaches and tradeoffs detailed in this section.
+  </para>
+
+  <note>
+    <title>Eventual solutions?</title>
+    <para>
+      All of the known workarounds have drawbacks, but we hope
+      better solutions will be available in the future. Some that
+      we have our eye on are:
+    </para>
+    <orderedlist>
+      <listitem>
+        <para>
+          A true firmlink would enable the Nix store to live on the
+          primary data volume without the build problems caused by
+          the symlink approach. End users cannot currently
+          create true firmlinks.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          If the Nix store volume shared FileVault encryption
+          with the primary data volume (probably by using the same
+          volume group and role), FileVault encryption could be
+          easily supported by the installer without requiring
+          manual setup by each user.
+        </para>
+      </listitem>
+    </orderedlist>
+  </note>
+
+  <section xml:id="sect-macos-installation-change-store-prefix">
+    <title>Change the Nix store path prefix</title>
+    <para>
+      Changing the default prefix for the Nix store is a simple
+      approach which enables you to leave it on your root volume,
+      where it can take full advantage of FileVault encryption if
+      enabled. Unfortunately, this approach also opts your device out
+      of some benefits that are enabled by using the same prefix
+      across systems:
+
+      <itemizedlist>
+        <listitem>
+          <para>
+            Your system won't be able to take advantage of the binary
+            cache (unless someone is able to stand up and support
+            duplicate caching infrastructure), which means you'll
+            spend more time waiting for builds.
+          </para>
+        </listitem>
+        <listitem>
+          <para>
+            It's harder to build and deploy packages to Linux systems.
+          </para>
+        </listitem>
+        <!-- TODO: may be more here -->
+      </itemizedlist>
+
+      <!-- TODO: Yes, but how?! -->
+
+      It would also possible (and often requested) to just apply this
+      change ecosystem-wide, but it's an intrusive process that has
+      side effects we want to avoid for now.
+      <!-- magnificent hand-wavy gesture -->
+    </para>
+    <para>
+    </para>
+  </section>
+
+  <section xml:id="sect-macos-installation-encrypted-volume">
+    <title>Use a separate encrypted volume</title>
+    <para>
+      If you like, you can also add encryption to the recommended
+      approach taken by the installer. You can do this by pre-creating
+      an encrypted volume before you run the installer--or you can
+      run the installer and encrypt the volume it creates later.
+      <!-- TODO: see later note about whether this needs both add-encryption and from-scratch directions -->
+    </para>
+    <para>
+      In either case, adding encryption to a second volume isn't quite
+      as simple as enabling FileVault for your boot volume. Before you
+      dive in, there are a few things to weigh:
+    </para>
+    <orderedlist>
+      <listitem>
+        <para>
+          The additional volume won't be encrypted with your existing
+          FileVault key, so you'll need another mechanism to decrypt
+          the volume.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          You can store the password in Keychain to automatically
+          decrypt the volume on boot--but it'll have to wait on Keychain
+          and may not mount before your GUI apps restore. If any of
+          your launchd agents or apps depend on Nix-installed software
+          (for example, if you use a Nix-installed login shell), the
+          restore may fail or break.
+        </para>
+        <para>
+          On a case-by-case basis, you may be able to work around this
+          problem by using <command>wait4path</command> to block
+          execution until your executable is available.
+        </para>
+        <para>
+          It's also possible to decrypt and mount the volume earlier
+          with a login hook--but this mechanism appears to be
+          deprecated and its future is unclear.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          You can hard-code the password in the clear, so that your
+          store volume can be decrypted before Keychain is available.
+        </para>
+      </listitem>
+    </orderedlist>
+    <para>
+      If you are comfortable navigating these tradeoffs, you can encrypt the volume with
+      something along the lines of:
+      <!-- TODO:
+      I don't know if this also needs from-scratch instructions?
+      can we just recommend use-the-installer-and-then-encrypt?
+      -->
+    </para>
+    <!--
+    TODO: it looks like this option can be encryptVolume|encrypt|enableFileVault
+
+    It may be more clear to use encryptVolume, here? FileVault seems
+    heavily associated with the boot-volume behavior; I worry
+    a little that it can mislead here, especially as it gets
+    copied around minus doc context...?
+    -->
+    <screen>alice$ diskutil apfs enableFileVault /nix -user disk</screen>
+
+    <!-- TODO: and then go into detail on the mount/decrypt approaches? -->
+  </section>
+
+  <section xml:id="sect-macos-installation-symlink">
+    <!--
+    Maybe a good razor is: if we'd hate having to support someone who
+    installed Nix this way, it shouldn't even be detailed?
+    -->
+    <title>Symlink the Nix store to a custom location</title>
+    <para>
+      Another simple approach is using <filename>/etc/synthetic.conf</filename>
+      to symlink the Nix store to the data volume. This option also
+      enables your store to share any configured FileVault encryption.
+      Unfortunately, builds that resolve the symlink may leak the
+      canonical path or even fail.
+    </para>
+    <para>
+      Because of these downsides, we can't recommend this approach.
+    </para>
+    <!-- Leaving out instructions for this one. -->
+  </section>
+
+  <section xml:id="sect-macos-installation-recommended-notes">
+    <title>Notes on the recommended approach</title>
+    <para>
+      This section goes into a little more detail on the recommended
+      approach. You don't need to understand it to run the installer,
+      but it can serve as a helpful reference if you run into trouble.
+    </para>
+    <orderedlist>
+      <listitem>
+        <para>
+          In order to compose user-writable locations into the new
+          read-only system root, Apple introduced a new concept called
+          <literal>firmlinks</literal>, which it describes as a
+          "bi-directional wormhole" between two filesystems. You can
+          see the current firmlinks in <filename>/usr/share/firmlinks</filename>.
+          Unfortunately, firmlinks aren't (currently?) user-configurable.
+        </para>
+
+        <para>
+          For special cases like NFS mount points or package manager roots,
+          <link xlink:href="https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man5/synthetic.conf.5.html">synthetic.conf(5)</link>
+          supports limited user-controlled file-creation (of symlinks,
+          and synthetic empty directories) at <filename>/</filename>.
+          To create a synthetic empty directory for mounting at <filename>/nix</filename>,
+          add the following line to <filename>/etc/synthetic.conf</filename>
+          (create it if necessary):
+        </para>
+
+        <screen>nix</screen>
+      </listitem>
+
+      <listitem>
+        <para>
+          This configuration is applied at boot time, but you can use
+          <command>apfs.util</command> to trigger creation (not deletion)
+          of new entries without a reboot:
+        </para>
+
+        <screen>alice$ /System/Library/Filesystems/apfs.fs/Contents/Resources/apfs.util -B</screen>
+      </listitem>
+
+      <listitem>
+        <para>
+          Create the new APFS volume with diskutil:
+        </para>
+
+        <screen>alice$ sudo diskutil apfs addVolume diskX APFS 'Nix Store' -mountpoint /nix</screen>
+      </listitem>
+
+      <listitem>
+        <para>
+          Using <command>vifs</command>, add the new mount to
+          <filename>/etc/fstab</filename>. If it doesn't already have
+          other entries, it should look something like:
+        </para>
+
+<screen>
+#
+# Warning - this file should only be modified with vifs(8)
+#
+# Failure to do so is unsupported and may be destructive.
+#
+LABEL=Nix\040Store /nix apfs rw,nobrowse
+</screen>
+
+        <para>
+          The nobrowse setting will keep Spotlight from indexing this
+          volume, and keep it from showing up on your desktop.
+        </para>
+      </listitem>
+    </orderedlist>
+  </section>
+
+</section>
+
 <section xml:id="sect-nix-install-pinned-version-url">
   <title>Installing a pinned Nix version from a URL</title>
 
@@ -150,7 +429,7 @@ sudo rm /Library/LaunchDaemons/org.nixos.nix-daemon.plist
   NixOS.org installation script:
 
   <screen>
-  sh &lt;(curl https://nixos.org/nix/install)
+  sh &lt;(curl -L https://nixos.org/nix/install)
 </screen>
   </para>
 

doc/manual/installation/upgrading.xml

@@ -17,6 +17,11 @@
 
   <para>
     Single-user installations of Nix should run this:
-    <command>nix-channel --update; nix-env -iA nixpkgs.nix</command>
+    <command>nix-channel --update; nix-env -iA nixpkgs.nix nixpkgs.cacert</command>
+  </para>
+      
+  <para>
+    Multi-user Nix users on Linux should run this with sudo:
+    <command>nix-channel --update; nix-env -iA nixpkgs.nix nixpkgs.cacert; systemctl daemon-reload; systemctl restart nix-daemon</command>
   </para>
 </chapter>

doc/manual/introduction/quick-start.xml

@@ -15,7 +15,7 @@ to subsequent chapters.</para>
 <step><para>Install single-user Nix by running the following:
 
 <screen>
-$ bash &lt;(curl https://nixos.org/nix/install)
+$ bash &lt;(curl -L https://nixos.org/nix/install)
 </screen>
 
 This will install Nix in <filename>/nix</filename>. The install script

doc/manual/release-notes/rl-0.8.xml

@@ -8,7 +8,7 @@
 
 <para>NOTE: the hashing scheme in Nix 0.8 changed (as detailed below).
 As a result, <command>nix-pull</command> manifests and channels built
-for Nix 0.7 and below will now work anymore.  However, the Nix
+for Nix 0.7 and below will not work anymore.  However, the Nix
 expression language has not changed, so you can still build from
 source.  Also, existing user environments continue to work.  Nix 0.8
 will automatically upgrade the database schema of previous

maintainers/upload-release.pl

@@ -142,8 +142,12 @@ $oldName =~ s/"//g;
 sub getStorePath {
     my ($jobName) = @_;
     my $buildInfo = decode_json(fetch("$evalUrl/job/$jobName", 'application/json'));
-    die unless $buildInfo->{buildproducts}->{1}->{type} eq "nix-build";
-    return $buildInfo->{buildproducts}->{1}->{path};
+    for my $product (values %{$buildInfo->{buildproducts}}) {
+        next unless $product->{type} eq "nix-build";
+        next if $product->{path} =~ /[a-z]+$/;
+        return $product->{path};
+    }
+    die;
 }
 
 write_file("$nixpkgsDir/nixos/modules/installer/tools/nix-fallback-paths.nix",
@@ -166,15 +170,5 @@ $channelsBucket->add_key(
 chdir("/home/eelco/Dev/nix-pristine") or die;
 system("git remote update origin") == 0 or die;
 system("git tag --force --sign $version $nixRev -m 'Tagging release $version'") == 0 or die;
-
-# Update the website.
-my $siteDir = "/home/eelco/Dev/nixos-homepage-pristine";
-
-system("cd $siteDir && git pull") == 0 or die;
-
-write_file("$siteDir/nix-release.tt",
-           "[%-\n" .
-           "latestNixVersion = \"$version\"\n" .
-           "-%]\n");
-
-system("cd $siteDir && git commit -a -m 'Nix $version released'") == 0 or die;
+system("git push --tags") == 0 or die;
+system("git push --force-with-lease origin $nixRev:refs/heads/latest-release") == 0 or die;

mk/libraries.mk

@@ -125,7 +125,8 @@ define build-library
     $(1)_PATH := $$(_d)/$$($(1)_NAME).a
 
     $$($(1)_PATH): $$($(1)_OBJS) | $$(_d)/
-	$(trace-ar) $(AR) crs $$@ $$?
+	$(trace-ld) $(LD) -Ur -o $$(_d)/$$($(1)_NAME).o $$?
+	$(trace-ar) $(AR) crs $$@ $$(_d)/$$($(1)_NAME).o
 
     $(1)_LDFLAGS_USE += $$($(1)_PATH) $$($(1)_LDFLAGS)
 

mk/programs.mk

@@ -35,24 +35,28 @@ define build-program
 	$$(trace-ld) $(CXX) -o $$@ $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE))
 
   $(1)_INSTALL_DIR ?= $$(bindir)
-  $(1)_INSTALL_PATH := $$($(1)_INSTALL_DIR)/$(1)
 
-  $$(eval $$(call create-dir, $$($(1)_INSTALL_DIR)))
+  ifdef $(1)_INSTALL_DIR
 
-  install: $(DESTDIR)$$($(1)_INSTALL_PATH)
+    $(1)_INSTALL_PATH := $$($(1)_INSTALL_DIR)/$(1)
 
-  ifeq ($(BUILD_SHARED_LIBS), 1)
+    $$(eval $$(call create-dir, $$($(1)_INSTALL_DIR)))
 
-    _libs_final := $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_INSTALL_PATH))
+    install: $(DESTDIR)$$($(1)_INSTALL_PATH)
 
-    $(DESTDIR)$$($(1)_INSTALL_PATH): $$($(1)_OBJS) $$(_libs_final) | $(DESTDIR)$$($(1)_INSTALL_DIR)/
+    ifeq ($(BUILD_SHARED_LIBS), 1)
+
+      _libs_final := $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_INSTALL_PATH))
+
+      $(DESTDIR)$$($(1)_INSTALL_PATH): $$($(1)_OBJS) $$(_libs_final) | $(DESTDIR)$$($(1)_INSTALL_DIR)/
 	$$(trace-ld) $(CXX) -o $$@ $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE_INSTALLED))
 
-  else
+    else
 
-    $(DESTDIR)$$($(1)_INSTALL_PATH): $$($(1)_PATH) | $(DESTDIR)$$($(1)_INSTALL_DIR)/
+      $(DESTDIR)$$($(1)_INSTALL_PATH): $$($(1)_PATH) | $(DESTDIR)$$($(1)_INSTALL_DIR)/
 	install -t $(DESTDIR)$$($(1)_INSTALL_DIR) $$<
 
+    endif
   endif
 
   # Propagate CFLAGS and CXXFLAGS to the individual object files.
@@ -76,4 +80,10 @@ define build-program
   programs-list += $$($(1)_PATH)
   clean-files += $$($(1)_PATH) $$(_d)/*.o $$(_d)/.*.dep $$($(1)_DEPS) $$($(1)_OBJS)
   dist-files += $$(_srcs)
+
+  # Phony target to run this program (typically as a dependency of 'check').
+  .PHONY: $(1)_RUN
+  $(1)_RUN: $$($(1)_PATH)
+	$(trace-test) $$($(1)_PATH)
+
 endef

mk/run_test.sh

@@ -0,0 +1,28 @@
+#!/bin/sh
+
+set -u
+
+red=""
+green=""
+yellow=""
+normal=""
+
+post_run_msg="ran test $1..."
+if [ -t 1 ]; then
+    red=""
+    green=""
+    yellow=""
+    normal=""
+fi
+(cd $(dirname $1) && env ${TESTS_ENVIRONMENT} init.sh 2>/dev/null > /dev/null)
+log="$(cd $(dirname $1) && env ${TESTS_ENVIRONMENT} $(basename $1) 2>&1)"
+status=$?
+if [ $status -eq 0 ]; then
+  echo "$post_run_msg [${green}PASS$normal]"
+elif [ $status -eq 99 ]; then
+  echo "$post_run_msg [${yellow}SKIP$normal]"
+else
+  echo "$post_run_msg [${red}FAIL$normal]"
+  echo "$log" | sed 's/^/    /'
+  exit "$status"
+fi

mk/tests.mk

@@ -1,45 +1,15 @@
 # Run program $1 as part of ‘make installcheck’.
+
+test-deps =
+
 define run-install-test
 
-  installcheck: $1
+  installcheck: $1.test
 
-  _installcheck-list += $1
+  .PHONY: $1.test
+  $1.test: $1 $(test-deps)
+	@env TEST_NAME=$(notdir $(basename $1)) TESTS_ENVIRONMENT="$(tests-environment)" mk/run_test.sh $1
 
 endef
 
-# Color code from https://unix.stackexchange.com/a/10065
-installcheck:
-	@total=0; failed=0; \
-	red=""; \
-	green=""; \
-	yellow=""; \
-	normal=""; \
-	if [ -t 1 ]; then \
-		red=""; \
-		green=""; \
-		yellow=""; \
-		normal=""; \
-	fi; \
-	for i in $(_installcheck-list); do \
-	  total=$$((total + 1)); \
-	  printf "running test $$i..."; \
-	  log="$$(cd $$(dirname $$i) && $(tests-environment) $$(basename $$i) 2>&1)"; \
-	  status=$$?; \
-	  if [ $$status -eq 0 ]; then \
-	    echo " [$${green}PASS$$normal]"; \
-	  elif [ $$status -eq 99 ]; then \
-	    echo " [$${yellow}SKIP$$normal]"; \
-	  else \
-	    echo " [$${red}FAIL$$normal]"; \
-	    echo "$$log" | sed 's/^/    /'; \
-	    failed=$$((failed + 1)); \
-	  fi; \
-	done; \
-	if [ "$$failed" != 0 ]; then \
-	  echo "$${red}$$failed out of $$total tests failed $$normal"; \
-	  exit 1; \
-	else \
-		echo "$${green}All tests succeeded$$normal"; \
-	fi
-
 .PHONY: check installcheck

mk/tracing.mk

@@ -11,6 +11,7 @@ ifeq ($(V), 0)
   trace-javac   = @echo "  JAVAC " $@;
   trace-jar     = @echo "  JAR   " $@;
   trace-mkdir   = @echo "  MKDIR " $@;
+  trace-test    = @echo "  TEST  " $@;
 
   suppress  = @
 

nix-rust/local.mk

@@ -41,5 +41,5 @@ ifneq ($(OS), Darwin)
 check: rust-tests
 
 rust-tests:
-	cd nix-rust && CARGO_HOME=$$(if [[ -d vendor ]]; then echo vendor; fi) cargo test --release $$(if [[ -d vendor ]]; then echo --offline; fi)
+	$(trace-test) cd nix-rust && CARGO_HOME=$$(if [[ -d vendor ]]; then echo vendor; fi) cargo test --release $$(if [[ -d vendor ]]; then echo --offline; fi)
 endif

perl/lib/Nix/Store.xs

@@ -80,7 +80,7 @@ SV * queryReferences(char * path)
 SV * queryPathHash(char * path)
     PPCODE:
         try {
-            auto s = store()->queryPathInfo(store()->parseStorePath(path))->narHash.to_string();
+            auto s = store()->queryPathInfo(store()->parseStorePath(path))->narHash.to_string(Base32, true);
             XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
         } catch (Error & e) {
             croak("%s", e.what());
@@ -106,7 +106,7 @@ SV * queryPathInfo(char * path, int base32)
                 XPUSHs(&PL_sv_undef);
             else
                 XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(*info->deriver).c_str(), 0)));
-            auto s = info->narHash.to_string(base32 ? Base32 : Base16);
+            auto s = info->narHash.to_string(base32 ? Base32 : Base16, true);
             XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
             mXPUSHi(info->registrationTime);
             mXPUSHi(info->narSize);
@@ -182,7 +182,7 @@ void importPaths(int fd, int dontCheckSigs)
     PPCODE:
         try {
             FdSource source(fd);
-            store()->importPaths(source, nullptr, dontCheckSigs ? NoCheckSigs : CheckSigs);
+            store()->importPaths(source, dontCheckSigs ? NoCheckSigs : CheckSigs);
         } catch (Error & e) {
             croak("%s", e.what());
         }
@@ -274,7 +274,8 @@ int checkSignature(SV * publicKey_, SV * sig_, char * msg)
 SV * addToStore(char * srcPath, int recursive, char * algo)
     PPCODE:
         try {
-            auto path = store()->addToStore(std::string(baseNameOf(srcPath)), srcPath, recursive, parseHashType(algo));
+            auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
+            auto path = store()->addToStore(std::string(baseNameOf(srcPath)), srcPath, method, parseHashType(algo));
             XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0)));
         } catch (Error & e) {
             croak("%s", e.what());
@@ -285,7 +286,8 @@ SV * makeFixedOutputPath(int recursive, char * algo, char * hash, char * name)
     PPCODE:
         try {
             Hash h(hash, parseHashType(algo));
-            auto path = store()->makeFixedOutputPath(recursive, h, name);
+            auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
+            auto path = store()->makeFixedOutputPath(method, h, name);
             XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0)));
         } catch (Error & e) {
             croak("%s", e.what());

release-common.nix

@@ -50,11 +50,11 @@ rec {
       libarchive
       boost
       nlohmann_json
-      rustc cargo
 
       # Tests
       git
       mercurial
+      gmock
     ]
     ++ lib.optionals stdenv.isLinux [libseccomp utillinuxMinimal]
     ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium

release.nix

@@ -12,64 +12,8 @@ let
     builtins.readFile ./.version
     + (if officialRelease then "" else "pre${toString nix.revCount}_${nix.shortRev}");
 
-  # Create a "vendor" directory that contains the crates listed in
-  # Cargo.lock. This allows Nix to be built without network access.
-  vendoredCrates' =
-    let
-      lockFile = builtins.fromTOML (builtins.readFile nix-rust/Cargo.lock);
-
-      files = map (pkg: import <nix/fetchurl.nix> {
-        url = "https://crates.io/api/v1/crates/${pkg.name}/${pkg.version}/download";
-        sha256 = lockFile.metadata."checksum ${pkg.name} ${pkg.version} (registry+https://github.com/rust-lang/crates.io-index)";
-      }) (builtins.filter (pkg: pkg.source or "" == "registry+https://github.com/rust-lang/crates.io-index") lockFile.package);
-
-    in pkgs.runCommand "cargo-vendor-dir" {}
-      ''
-        mkdir -p $out/vendor
-
-        cat > $out/vendor/config <<EOF
-        [source.crates-io]
-        replace-with = "vendored-sources"
-
-        [source.vendored-sources]
-        directory = "vendor"
-        EOF
-
-        ${toString (builtins.map (file: ''
-          mkdir $out/vendor/tmp
-          tar xvf ${file} -C $out/vendor/tmp
-          dir=$(echo $out/vendor/tmp/*)
-
-          # Add just enough metadata to keep Cargo happy.
-          printf '{"files":{},"package":"${file.outputHash}"}' > "$dir/.cargo-checksum.json"
-
-          # Clean up some cruft from the winapi crates. FIXME: find
-          # a way to remove winapi* from our dependencies.
-          if [[ $dir =~ /winapi ]]; then
-            find $dir -name "*.a" -print0 | xargs -0 rm -f --
-          fi
-
-          mv "$dir" $out/vendor/
-
-          rm -rf $out/vendor/tmp
-        '') files)}
-      '';
-
   jobs = rec {
 
-    vendoredCrates =
-      with pkgs;
-      runCommand "vendored-crates" {}
-        ''
-          mkdir -p $out/nix-support
-          name=nix-vendored-crates-${version}
-          fn=$out/$name.tar.xz
-            tar cvfJ $fn -C ${vendoredCrates'} vendor \
-              --owner=0 --group=0 --mode=u+rw,uga+r \
-              --transform "s,vendor,$name,"
-            echo "file crates-tarball $fn" >> $out/nix-support/hydra-build-products
-        '';
-
     build = pkgs.lib.genAttrs systems (system:
 
       let pkgs = import nixpkgs { inherit system; }; in
@@ -101,8 +45,6 @@ let
               patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.*
             ''}
 
-            ln -sfn ${vendoredCrates'}/vendor/ nix-rust/vendor
-
             (cd perl; autoreconf --install --force --verbose)
           '';
 
@@ -115,17 +57,17 @@ let
 
         installFlags = "sysconfdir=$(out)/etc";
 
+        postInstall = ''
+          mkdir -p $doc/nix-support
+          echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products
+        '';
+
         doCheck = true;
 
         doInstallCheck = true;
         installCheckFlags = "sysconfdir=$(out)/etc";
 
         separateDebugInfo = true;
-
-        preDist = ''
-          mkdir -p $doc/nix-support
-          echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products
-        '';
       });
 
 
@@ -177,10 +119,10 @@ let
         }
         ''
           cp ${installerClosureInfo}/registration $TMPDIR/reginfo
+          cp ${./scripts/create-darwin-volume.sh} $TMPDIR/create-darwin-volume.sh
           substitute ${./scripts/install-nix-from-closure.sh} $TMPDIR/install \
             --subst-var-by nix ${toplevel} \
             --subst-var-by cacert ${cacert}
-
           substitute ${./scripts/install-darwin-multi-user.sh} $TMPDIR/install-darwin-multi-user.sh \
             --subst-var-by nix ${toplevel} \
             --subst-var-by cacert ${cacert}
@@ -195,6 +137,7 @@ let
             # SC1090: Don't worry about not being able to find
             #         $nix/etc/profile.d/nix.sh
             shellcheck --exclude SC1090 $TMPDIR/install
+            shellcheck $TMPDIR/create-darwin-volume.sh
             shellcheck $TMPDIR/install-darwin-multi-user.sh
             shellcheck $TMPDIR/install-systemd-multi-user.sh
 
@@ -210,6 +153,7 @@ let
           fi
 
           chmod +x $TMPDIR/install
+          chmod +x $TMPDIR/create-darwin-volume.sh
           chmod +x $TMPDIR/install-darwin-multi-user.sh
           chmod +x $TMPDIR/install-systemd-multi-user.sh
           chmod +x $TMPDIR/install-multi-user
@@ -222,11 +166,15 @@ let
             --absolute-names \
             --hard-dereference \
             --transform "s,$TMPDIR/install,$dir/install," \
+            --transform "s,$TMPDIR/create-darwin-volume.sh,$dir/create-darwin-volume.sh," \
             --transform "s,$TMPDIR/reginfo,$dir/.reginfo," \
             --transform "s,$NIX_STORE,$dir/store,S" \
-            $TMPDIR/install $TMPDIR/install-darwin-multi-user.sh \
+            $TMPDIR/install \
+            $TMPDIR/create-darwin-volume.sh \
+            $TMPDIR/install-darwin-multi-user.sh \
             $TMPDIR/install-systemd-multi-user.sh \
-            $TMPDIR/install-multi-user $TMPDIR/reginfo \
+            $TMPDIR/install-multi-user \
+            $TMPDIR/reginfo \
             $(cat ${installerClosureInfo}/store-paths)
         '');
 
@@ -241,11 +189,6 @@ let
 
         src = nix;
 
-        preConfigure =
-          ''
-            ln -sfn ${vendoredCrates'}/vendor/ nix-rust/vendor
-          '';
-
         enableParallelBuilding = true;
 
         buildInputs = buildDeps ++ propagatedDeps;

scripts/create-darwin-volume.sh

@@ -0,0 +1,185 @@
+#!/bin/sh
+set -e
+
+root_disk() {
+    diskutil info -plist /
+}
+
+apfs_volumes_for() {
+    disk=$1
+    diskutil apfs list -plist "$disk"
+}
+
+disk_identifier() {
+    xpath "/plist/dict/key[text()='ParentWholeDisk']/following-sibling::string[1]/text()" 2>/dev/null
+}
+
+volume_list_true() {
+    key=$1
+    xpath "/plist/dict/array/dict/key[text()='Volumes']/following-sibling::array/dict/key[text()='$key']/following-sibling::true[1]" 2> /dev/null
+}
+
+volume_get_string() {
+    key=$1 i=$2
+    xpath "/plist/dict/array/dict/key[text()='Volumes']/following-sibling::array/dict[$i]/key[text()='$key']/following-sibling::string[1]/text()" 2> /dev/null
+}
+
+find_nix_volume() {
+    disk=$1
+    i=1
+    volumes=$(apfs_volumes_for "$disk")
+    while true; do
+        name=$(echo "$volumes" | volume_get_string "Name" "$i")
+        if [ -z "$name" ]; then
+            break
+        fi
+        case "$name" in
+            [Nn]ix*)
+                echo "$name"
+                break
+                ;;
+        esac
+        i=$((i+1))
+    done
+}
+
+test_fstab() {
+    grep -q "/nix apfs rw" /etc/fstab 2>/dev/null
+}
+
+test_nix_symlink() {
+    [ -L "/nix" ] || grep -q "^nix." /etc/synthetic.conf 2>/dev/null
+}
+
+test_synthetic_conf() {
+    grep -q "^nix$" /etc/synthetic.conf 2>/dev/null
+}
+
+test_nix() {
+    test -d "/nix"
+}
+
+test_t2_chip_present(){
+    # Use xartutil to see if system has a t2 chip.
+    #
+    # This isn't well-documented on its own; until it is,
+    # let's keep track of knowledge/assumptions.
+    #
+    # Warnings:
+    # - Don't search "xart" if porn will cause you trouble :)
+    # - Other xartutil flags do dangerous things. Don't run them
+    #   naively. If you must, search "xartutil" first.
+    #
+    # Assumptions:
+    # - the "xART session seeds recovery utility"
+    #   appears to interact with xartstorageremoted
+    # - `sudo xartutil --list` lists xART sessions
+    #   and their seeds and exits 0 if successful. If
+    #   not, it exits 1 and prints an error such as:
+    #   xartutil: ERROR: No supported link to the SEP present
+    # - xART sessions/seeds are present when a T2 chip is
+    #   (and not, otherwise)
+    # - the presence of a T2 chip means a newly-created
+    #   volume on the primary drive will be
+    #   encrypted at rest
+    # - all together: `sudo xartutil --list`
+    #   should exit 0 if a new Nix Store volume will
+    #   be encrypted at rest, and exit 1 if not.
+    sudo xartutil --list >/dev/null 2>/dev/null
+}
+
+test_filevault_in_use() {
+    disk=$1
+    #      list vols on disk | get value of Filevault key | value is true
+    apfs_volumes_for "$disk" | volume_list_true FileVault | grep -q true
+}
+
+# use after error msg for conditions we don't understand
+suggest_report_error(){
+    # ex "error: something sad happened :(" >&2
+    echo "       please report this @ https://github.com/nixos/nix/issues" >&2
+}
+
+main() {
+    (
+        echo ""
+        echo "     ------------------------------------------------------------------ "
+        echo "    | This installer will create a volume for the nix store and        |"
+        echo "    | configure it to mount at /nix.  Follow these steps to uninstall. |"
+        echo "     ------------------------------------------------------------------ "
+        echo ""
+        echo "  1. Remove the entry from fstab using 'sudo vifs'"
+        echo "  2. Destroy the data volume using 'diskutil apfs deleteVolume'"
+        echo "  3. Remove the 'nix' line from /etc/synthetic.conf or the file"
+        echo ""
+    ) >&2
+
+    if test_nix_symlink; then
+        echo "error: /nix is a symlink, please remove it and make sure it's not in synthetic.conf (in which case a reboot is required)" >&2
+        echo "  /nix -> $(readlink "/nix")" >&2
+        exit 2
+    fi
+
+    if ! test_synthetic_conf; then
+        echo "Configuring /etc/synthetic.conf..." >&2
+        echo nix | sudo tee -a /etc/synthetic.conf
+        if ! test_synthetic_conf; then
+            echo "error: failed to configure synthetic.conf;" >&2
+            suggest_report_error
+            exit 1
+        fi
+    fi
+
+    if ! test_nix; then
+        echo "Creating mountpoint for /nix..." >&2
+        /System/Library/Filesystems/apfs.fs/Contents/Resources/apfs.util -B || true
+        if ! test_nix; then
+            sudo mkdir -p /nix 2>/dev/null || true
+        fi
+        if ! test_nix; then
+            echo "error: failed to bootstrap /nix; if a reboot doesn't help," >&2
+            suggest_report_error
+            exit 1
+        fi
+    fi
+
+    disk=$(root_disk | disk_identifier)
+    volume=$(find_nix_volume "$disk")
+    if [ -z "$volume" ]; then
+        echo "Creating a Nix Store volume..." >&2
+
+        if test_filevault_in_use "$disk"; then
+            # TODO: Not sure if it's in-scope now, but `diskutil apfs list`
+            # shows both filevault and encrypted at rest status, and it
+            # may be the more semantic way to test for this? It'll show
+            # `FileVault:                 No (Encrypted at rest)`
+            # `FileVault:                 No`
+            # `FileVault:                 Yes (Unlocked)`
+            # and so on.
+            if test_t2_chip_present; then
+                echo "warning: boot volume is FileVault-encrypted, but the Nix store volume" >&2
+                echo "         is only encrypted at rest." >&2
+                echo "         See https://nixos.org/nix/manual/#sect-macos-installation" >&2
+            else
+                echo "error: refusing to create Nix store volume because the boot volume is" >&2
+                echo "       FileVault encrypted, but encryption-at-rest is not available." >&2
+                echo "       Manually create a volume for the store and re-run this script." >&2
+                echo "       See https://nixos.org/nix/manual/#sect-macos-installation" >&2
+                exit 1
+            fi
+        fi
+
+        sudo diskutil apfs addVolume "$disk" APFS 'Nix Store' -mountpoint /nix
+        volume="Nix Store"
+    else
+        echo "Using existing '$volume' volume" >&2
+    fi
+
+    if ! test_fstab; then
+        echo "Configuring /etc/fstab..." >&2
+        label=$(echo "$volume" | sed 's/ /\\040/g')
+        printf "\$a\nLABEL=%s /nix apfs rw,nobrowse\n.\nwq\n" "$label" | EDITOR=ed sudo vifs
+    fi
+}
+
+main "$@"

scripts/install-multi-user.sh

@@ -20,15 +20,18 @@ readonly GREEN='\033[32m'
 readonly GREEN_UL='\033[4;32m'
 readonly RED='\033[31m'
 
-readonly NIX_USER_COUNT="32"
+# installer allows overriding build user count to speed up installation
+# as creating each user takes non-trivial amount of time on macos
+readonly NIX_USER_COUNT=${NIX_USER_COUNT:-32}
 readonly NIX_BUILD_GROUP_ID="30000"
 readonly NIX_BUILD_GROUP_NAME="nixbld"
 readonly NIX_FIRST_BUILD_UID="30001"
 # Please don't change this. We don't support it, because the
 # default shell profile that comes with Nix doesn't support it.
 readonly NIX_ROOT="/nix"
+readonly NIX_EXTRA_CONF=${NIX_EXTRA_CONF:-}
 
-readonly PROFILE_TARGETS=("/etc/bashrc" "/etc/profile.d/nix.sh" "/etc/zshrc")
+readonly PROFILE_TARGETS=("/etc/bashrc" "/etc/profile.d/nix.sh" "/etc/zshenv")
 readonly PROFILE_BACKUP_SUFFIX=".backup-before-nix"
 readonly PROFILE_NIX_FILE="$NIX_ROOT/var/nix/profiles/default/etc/profile.d/nix-daemon.sh"
 
@@ -450,9 +453,11 @@ create_directories() {
 }
 
 place_channel_configuration() {
-    echo "https://nixos.org/channels/nixpkgs-unstable nixpkgs" > "$SCRATCH/.nix-channels"
-    _sudo "to set up the default system channel (part 1)" \
-          install -m 0664 "$SCRATCH/.nix-channels" "$ROOT_HOME/.nix-channels"
+    if [ -z "${NIX_INSTALLER_NO_CHANNEL_ADD:-}" ]; then
+        echo "https://nixos.org/channels/nixpkgs-unstable nixpkgs" > "$SCRATCH/.nix-channels"
+        _sudo "to set up the default system channel (part 1)" \
+            install -m 0664 "$SCRATCH/.nix-channels" "$ROOT_HOME/.nix-channels"
+    fi
 }
 
 welcome_to_nix() {
@@ -521,7 +526,7 @@ This script is going to call sudo a lot. Normally, it would show you
 exactly what commands it is running and why. However, the script is
 run in a headless fashion, like this:
 
-  $ curl https://nixos.org/nix/install | sh
+  $ curl -L https://nixos.org/nix/install | sh
 
 or maybe in a CI pipeline. Because of that, we're going to skip the
 verbose output in the interest of brevity.
@@ -529,7 +534,7 @@ verbose output in the interest of brevity.
 If you would like to
 see the output, try like this:
 
-  $ curl -o install-nix https://nixos.org/nix/install
+  $ curl -L -o install-nix https://nixos.org/nix/install
   $ sh ./install-nix
 
 EOF
@@ -634,18 +639,20 @@ setup_default_profile() {
         export NIX_SSL_CERT_FILE=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt
     fi
 
-    # Have to explicitly pass NIX_SSL_CERT_FILE as part of the sudo call,
-    # otherwise it will be lost in environments where sudo doesn't pass
-    # all the environment variables by default.
-    _sudo "to update the default channel in the default profile" \
-          HOME="$ROOT_HOME" NIX_SSL_CERT_FILE="$NIX_SSL_CERT_FILE" "$NIX_INSTALLED_NIX/bin/nix-channel" --update nixpkgs \
-          || channel_update_failed=1
-
+    if [ -z "${NIX_INSTALLER_NO_CHANNEL_ADD:-}" ]; then
+        # Have to explicitly pass NIX_SSL_CERT_FILE as part of the sudo call,
+        # otherwise it will be lost in environments where sudo doesn't pass
+        # all the environment variables by default.
+        _sudo "to update the default channel in the default profile" \
+            HOME="$ROOT_HOME" NIX_SSL_CERT_FILE="$NIX_SSL_CERT_FILE" "$NIX_INSTALLED_NIX/bin/nix-channel" --update nixpkgs \
+            || channel_update_failed=1
+    fi
 }
 
 
 place_nix_configuration() {
     cat <<EOF > "$SCRATCH/nix.conf"
+$NIX_EXTRA_CONF
 build-users-group = $NIX_BUILD_GROUP_NAME
 EOF
     _sudo "to place the default nix daemon configuration (part 2)" \

scripts/install-nix-from-closure.sh

@@ -40,29 +40,85 @@ elif [ "$(uname -s)" = "Linux" ] && [ -e /run/systemd/system ]; then
 fi
 
 INSTALL_MODE=no-daemon
-# Trivially handle the --daemon / --no-daemon options
-if [ "x${1:-}" = "x--no-daemon" ]; then
-    INSTALL_MODE=no-daemon
-elif [ "x${1:-}" = "x--daemon" ]; then
-    INSTALL_MODE=daemon
-elif [ "x${1:-}" != "x" ]; then
-    (
-        echo "Nix Installer [--daemon|--no-daemon]"
+CREATE_DARWIN_VOLUME=0
+# handle the command line flags
+while [ $# -gt 0 ]; do
+    case $1 in
+        --daemon)
+            INSTALL_MODE=daemon;;
+        --no-daemon)
+            INSTALL_MODE=no-daemon;;
+        --no-channel-add)
+            export NIX_INSTALLER_NO_CHANNEL_ADD=1;;
+        --daemon-user-count)
+            export NIX_USER_COUNT=$2
+            shift;;
+        --no-modify-profile)
+            NIX_INSTALLER_NO_MODIFY_PROFILE=1;;
+        --darwin-use-unencrypted-nix-store-volume)
+            CREATE_DARWIN_VOLUME=1;;
+        --nix-extra-conf-file)
+            export NIX_EXTRA_CONF="$(cat $2)"
+            shift;;
+        *)
+            (
+                echo "Nix Installer [--daemon|--no-daemon] [--daemon-user-count INT] [--no-channel-add] [--no-modify-profile] [--darwin-use-unencrypted-nix-store-volume] [--nix-extra-conf-file FILE]"
 
-        echo "Choose installation method."
-        echo ""
-        echo " --daemon:    Installs and configures a background daemon that manages the store,"
-        echo "              providing multi-user support and better isolation for local builds."
-        echo "              Both for security and reproducibility, this method is recommended if"
-        echo "              supported on your platform."
-        echo "              See https://nixos.org/nix/manual/#sect-multi-user-installation"
-        echo ""
-        echo " --no-daemon: Simple, single-user installation that does not require root and is"
-        echo "              trivial to uninstall."
-        echo "              (default)"
-        echo ""
-    ) >&2
-    exit
+                echo "Choose installation method."
+                echo ""
+                echo " --daemon:    Installs and configures a background daemon that manages the store,"
+                echo "              providing multi-user support and better isolation for local builds."
+                echo "              Both for security and reproducibility, this method is recommended if"
+                echo "              supported on your platform."
+                echo "              See https://nixos.org/nix/manual/#sect-multi-user-installation"
+                echo ""
+                echo " --no-daemon: Simple, single-user installation that does not require root and is"
+                echo "              trivial to uninstall."
+                echo "              (default)"
+                echo ""
+                echo " --no-channel-add:    Don't add any channels. nixpkgs-unstable is installed by default."
+                echo ""
+                echo " --no-modify-profile: Skip channel installation. When not provided nixpkgs-unstable"
+                echo "                      is installed by default."
+                echo ""
+                echo " --daemon-user-count: Number of build users to create. Defaults to 32."
+                echo ""
+                echo " --nix-extra-conf-file: Path to nix.conf to prepend when installing /etc/nix.conf"
+                echo ""
+            ) >&2
+
+            # darwin and Catalina+
+            if [ "$(uname -s)" = "Darwin" ] && [ "$macos_major" -gt 14 ]; then
+                (
+                    echo " --darwin-use-unencrypted-nix-store-volume: Create an APFS volume for the Nix"
+                    echo "              store and mount it at /nix. This is the recommended way to create"
+                    echo "              /nix with a read-only / on macOS >=10.15."
+                    echo "              See: https://nixos.org/nix/manual/#sect-macos-installation"
+                    echo ""
+                ) >&2
+            fi
+            exit;;
+    esac
+    shift
+done
+
+if [ "$(uname -s)" = "Darwin" ]; then
+    if [ "$CREATE_DARWIN_VOLUME" = 1 ]; then
+        printf '\e[1;31mCreating volume and mountpoint /nix.\e[0m\n'
+        "$self/create-darwin-volume.sh"
+    fi
+
+    info=$(diskutil info -plist / | xpath "/plist/dict/key[text()='Writable']/following-sibling::true[1]" 2> /dev/null)
+    if ! [ -e $dest ] && [ -n "$info" ] && [ "$macos_major" -gt 14 ]; then
+        (
+            echo ""
+            echo "Installing on macOS >=10.15 requires relocating the store to an apfs volume."
+            echo "Use sh <(curl -L https://nixos.org/nix/install) --darwin-use-unencrypted-nix-store-volume or run the preparation steps manually."
+            echo "See https://nixos.org/nix/manual/#sect-macos-installation"
+            echo ""
+        ) >&2
+        exit 1
+    fi
 fi
 
 if [ "$INSTALL_MODE" = "daemon" ]; then
@@ -130,13 +186,15 @@ if [ -z "$NIX_SSL_CERT_FILE" ] || ! [ -f "$NIX_SSL_CERT_FILE" ]; then
 fi
 
 # Subscribe the user to the Nixpkgs channel and fetch it.
-if ! $nix/bin/nix-channel --list | grep -q "^nixpkgs "; then
-    $nix/bin/nix-channel --add https://nixos.org/channels/nixpkgs-unstable
-fi
-if [ -z "$_NIX_INSTALLER_TEST" ]; then
-    if ! $nix/bin/nix-channel --update nixpkgs; then
-        echo "Fetching the nixpkgs channel failed. (Are you offline?)"
-        echo "To try again later, run \"nix-channel --update nixpkgs\"."
+if [ -z "$NIX_INSTALLER_NO_CHANNEL_ADD" ]; then
+    if ! $nix/bin/nix-channel --list | grep -q "^nixpkgs "; then
+        $nix/bin/nix-channel --add https://nixos.org/channels/nixpkgs-unstable
+    fi
+    if [ -z "$_NIX_INSTALLER_TEST" ]; then
+        if ! $nix/bin/nix-channel --update nixpkgs; then
+            echo "Fetching the nixpkgs channel failed. (Are you offline?)"
+            echo "To try again later, run \"nix-channel --update nixpkgs\"."
+        fi
     fi
 fi
 
@@ -155,6 +213,17 @@ if [ -z "$NIX_INSTALLER_NO_MODIFY_PROFILE" ]; then
             break
         fi
     done
+    for i in .zshenv .zshrc; do
+        fn="$HOME/$i"
+        if [ -w "$fn" ]; then
+            if ! grep -q "$p" "$fn"; then
+                echo "modifying $fn..." >&2
+                echo "if [ -e $p ]; then . $p; fi # added by Nix installer" >> "$fn"
+            fi
+            added=1
+            break
+        fi
+    done
 fi
 
 if [ -z "$added" ]; then

scripts/install.in

@@ -36,7 +36,9 @@ tarball="$tmpDir/$(basename "$tmpDir/nix-@nixVersion@-$system.tar.xz")"
 
 require_util curl "download the binary tarball"
 require_util tar "unpack the binary tarball"
-require_util xz "unpack the binary tarball"
+if [ "$(uname -s)" != "Darwin" ]; then
+    require_util xz "unpack the binary tarball"
+fi
 
 echo "downloading Nix @nixVersion@ binary tarball for $system from '$url' to '$tmpDir'..."
 curl -L "$url" -o "$tarball" || oops "failed to download '$url'"

shell.nix

@@ -7,7 +7,7 @@ with import ./release-common.nix { inherit pkgs; };
 (if useClang then clangStdenv else stdenv).mkDerivation {
   name = "nix";
 
-  buildInputs = buildDeps ++ propagatedDeps ++ perlDeps ++ [ pkgs.rustfmt ];
+  buildInputs = buildDeps ++ propagatedDeps ++ perlDeps;
 
   inherit configureFlags;
 

src/build-remote/build-remote.cc

@@ -200,9 +200,12 @@ static int _main(int argc, char * * argv)
 
                 } catch (std::exception & e) {
                     auto msg = chomp(drainFD(5, false));
-                    printError("cannot build on '%s': %s%s",
-                        bestMachine->storeUri, e.what(),
-                        (msg.empty() ? "" : ": " + msg));
+                    logError({
+                        .name = "Remote build",
+                        .hint = hintfmt("cannot build on '%s': %s%s",
+                            bestMachine->storeUri, e.what(),
+                            (msg.empty() ? "" : ": " + msg))
+                    });
                     bestMachine->enabled = false;
                     continue;
                 }
@@ -241,7 +244,7 @@ connected:
 
         uploadLock = -1;
 
-        BasicDerivation drv(readDerivation(*store, store->realStoreDir + "/" + std::string(drvPath->to_string())));
+        auto drv = store->readDerivation(*drvPath);
         drv.inputSrcs = store->parseStorePathSet(inputs);
 
         auto result = sshStore->buildDerivation(*drvPath, drv);

src/error-demo/error-demo.cc

@@ -1,66 +0,0 @@
-#include "error.hh"
-#include "nixexpr.hh"
-
-#include <iostream>
-#include <optional>
-
-int main()
-{
-    using namespace nix;
-
-    // In each program where errors occur, this has to be set.
-    ErrorInfo::programName = std::optional("error-demo");
-
-    // Error in a program; no hint and no nix code.
-    printErrorInfo(
-        ErrorInfo { .level = elError,
-                    .name = "name",
-                    .description = "error description",
-                  });
-
-    // Warning with name, description, and hint.
-    // The hintfmt function makes all the substituted text yellow.
-    printErrorInfo(
-        ErrorInfo { .level = elWarning,
-                    .name = "name",
-                    .description = "error description",
-                    .hint =  std::optional(
-                                 hintfmt("there was a %1%", "warning")),
-                  });
-
-
-    // Warning with nix file, line number, column, and the lines of
-    // code where a warning occurred.
-    SymbolTable testTable;
-    auto problem_file = testTable.create("myfile.nix");
-
-    printErrorInfo(
-        ErrorInfo{
-            .level = elWarning,
-            .name = "warning name",
-            .description = "warning description",
-            .hint = hintfmt("this hint has %1% templated %2%!!", "yellow", "values"),
-            .nixCode = NixCode {
-                .errPos = Pos(problem_file, 40, 13),
-                .prevLineOfCode = std::nullopt,
-                .errLineOfCode = "this is the problem line of code",
-                .nextLineOfCode = std::nullopt
-            }});
-
-    // Error with previous and next lines of code.
-    printErrorInfo(
-        ErrorInfo{
-            .level = elError,
-            .name = "error name",
-            .description = "error description",
-            .hint = hintfmt("this hint has %1% templated %2%!!", "yellow", "values"),
-            .nixCode = NixCode {
-                .errPos = Pos(problem_file, 40, 13),
-                .prevLineOfCode = std::optional("previous line of code"),
-                .errLineOfCode = "this is the problem line of code",
-                .nextLineOfCode = std::optional("next line of code"),
-            }});
-
-
-    return 0;
-}

src/error-demo/local.mk

@@ -1,12 +0,0 @@
-programs += error-demo
-
-error-demo_DIR := $(d)
-
-error-demo_SOURCES := \
-  $(wildcard $(d)/*.cc) \
-
-error-demo_CXXFLAGS += -I src/libutil -I src/libexpr
-
-error-demo_LIBS = libutil libexpr
-
-error-demo_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) -lboost_context -lboost_thread -lboost_system

src/libexpr/attr-path.cc

@@ -6,11 +6,11 @@
 namespace nix {
 
 
-static Strings parseAttrPath(const string & s)
+static Strings parseAttrPath(std::string_view s)
 {
     Strings res;
     string cur;
-    string::const_iterator i = s.begin();
+    auto i = s.begin();
     while (i != s.end()) {
         if (*i == '.') {
             res.push_back(cur);
@@ -19,7 +19,7 @@ static Strings parseAttrPath(const string & s)
             ++i;
             while (1) {
                 if (i == s.end())
-                    throw Error(format("missing closing quote in selection path '%1%'") % s);
+                    throw Error("missing closing quote in selection path '%1%'", s);
                 if (*i == '"') break;
                 cur.push_back(*i++);
             }
@@ -32,6 +32,15 @@ static Strings parseAttrPath(const string & s)
 }
 
 
+std::vector<Symbol> parseAttrPath(EvalState & state, std::string_view s)
+{
+    std::vector<Symbol> res;
+    for (auto & a : parseAttrPath(s))
+        res.push_back(state.symbols.create(a));
+    return res;
+}
+
+
 std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attrPath,
     Bindings & autoArgs, Value & vIn)
 {
@@ -60,11 +69,11 @@ std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attr
 
             if (v->type != tAttrs)
                 throw TypeError(
-                    format("the expression selected by the selection path '%1%' should be a set but is %2%")
-                    % attrPath % showType(*v));
-
+                    "the expression selected by the selection path '%1%' should be a set but is %2%",
+                    attrPath,
+                    showType(*v));
             if (attr.empty())
-                throw Error(format("empty attribute name in selection path '%1%'") % attrPath);
+                throw Error("empty attribute name in selection path '%1%'", attrPath);
 
             Bindings::iterator a = v->attrs->find(state.symbols.create(attr));
             if (a == v->attrs->end())
@@ -77,9 +86,9 @@ std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attr
 
             if (!v->isList())
                 throw TypeError(
-                    format("the expression selected by the selection path '%1%' should be a list but is %2%")
-                    % attrPath % showType(*v));
-
+                    "the expression selected by the selection path '%1%' should be a list but is %2%",
+                    attrPath,
+                    showType(*v));
             if (attrIndex >= v->listSize())
                 throw AttrPathNotFound("list index %1% in selection path '%2%' is out of range", attrIndex, attrPath);
 
@@ -121,7 +130,7 @@ Pos findDerivationFilename(EvalState & state, Value & v, std::string what)
 
     Symbol file = state.symbols.create(filename);
 
-    return { file, lineno, 0 };
+    return { foFile, file, lineno, 0 };
 }
 
 

src/libexpr/attr-path.hh

@@ -16,4 +16,6 @@ std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attr
 /* Heuristic to find the filename and lineno or a nix value. */
 Pos findDerivationFilename(EvalState & state, Value & v, std::string what);
 
+std::vector<Symbol> parseAttrPath(EvalState & state, std::string_view s);
+
 }

src/libexpr/attr-set.hh

@@ -76,7 +76,11 @@ public:
     {
         auto a = get(name);
         if (!a)
-            throw Error("attribute '%s' missing, at %s", name, pos);
+            throw Error({
+                .hint = hintfmt("attribute '%s' missing", name),
+                .errPos = pos
+            });
+
         return *a;
     }
 

src/libexpr/eval-inline.hh

@@ -7,20 +7,26 @@
 
 namespace nix {
 
-LocalNoInlineNoReturn(void throwEvalError(const char * s, const Pos & pos))
+LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s))
 {
-    throw EvalError(format(s) % pos);
+    throw EvalError({
+        .hint = hintfmt(s),
+        .errPos = pos
+    });
 }
 
 LocalNoInlineNoReturn(void throwTypeError(const char * s, const Value & v))
 {
-    throw TypeError(format(s) % showType(v));
+    throw TypeError(s, showType(v));
 }
 
 
-LocalNoInlineNoReturn(void throwTypeError(const char * s, const Value & v, const Pos & pos))
+LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s, const Value & v))
 {
-    throw TypeError(format(s) % showType(v) % pos);
+    throw TypeError({
+        .hint = hintfmt(s, showType(v)),
+        .errPos = pos
+    });
 }
 
 
@@ -43,7 +49,7 @@ void EvalState::forceValue(Value & v, const Pos & pos)
     else if (v.type == tApp)
         callFunction(*v.app.left, *v.app.right, v, noPos);
     else if (v.type == tBlackhole)
-        throwEvalError("infinite recursion encountered, at %1%", pos);
+        throwEvalError(pos, "infinite recursion encountered");
 }
 
 
@@ -59,7 +65,7 @@ inline void EvalState::forceAttrs(Value & v, const Pos & pos)
 {
     forceValue(v, pos);
     if (v.type != tAttrs)
-        throwTypeError("value is %1% while a set was expected, at %2%", v, pos);
+        throwTypeError(pos, "value is %1% while a set was expected", v);
 }
 
 
@@ -75,7 +81,7 @@ inline void EvalState::forceList(Value & v, const Pos & pos)
 {
     forceValue(v, pos);
     if (!v.isList())
-        throwTypeError("value is %1% while a list was expected, at %2%", v, pos);
+        throwTypeError(pos, "value is %1% while a list was expected", v);
 }
 
 /* Note: Various places expect the allocated memory to be zeroed. */

src/libexpr/eval.cc

@@ -161,12 +161,12 @@ const Value *getPrimOp(const Value &v) {
 }
 
 
-string showType(const Value & v)
+string showType(ValueType type)
 {
-    switch (v.type) {
+    switch (type) {
         case tInt: return "an integer";
-        case tBool: return "a boolean";
-        case tString: return v.string.context ? "a string with context" : "a string";
+        case tBool: return "a Boolean";
+        case tString: return "a string";
         case tPath: return "a path";
         case tNull: return "null";
         case tAttrs: return "a set";
@@ -175,14 +175,27 @@ string showType(const Value & v)
         case tApp: return "a function application";
         case tLambda: return "a function";
         case tBlackhole: return "a black hole";
+        case tPrimOp: return "a built-in function";
+        case tPrimOpApp: return "a partially applied built-in function";
+        case tExternal: return "an external value";
+        case tFloat: return "a float";
+    }
+    abort();
+}
+
+
+string showType(const Value & v)
+{
+    switch (v.type) {
+        case tString: return v.string.context ? "a string with context" : "a string";
         case tPrimOp:
             return fmt("the built-in function '%s'", string(v.primOp->name));
         case tPrimOpApp:
             return fmt("the partially applied built-in function '%s'", string(getPrimOp(v)->primOp->name));
         case tExternal: return v.external->showType();
-        case tFloat: return "a float";
+    default:
+        return showType(v.type);
     }
-    abort();
 }
 
 
@@ -323,6 +336,7 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
     , sOutputHash(symbols.create("outputHash"))
     , sOutputHashAlgo(symbols.create("outputHashAlgo"))
     , sOutputHashMode(symbols.create("outputHashMode"))
+    , sRecurseForDerivations(symbols.create("recurseForDerivations"))
     , repair(NoRepair)
     , store(store)
     , baseEnv(allocEnv(128))
@@ -352,7 +366,7 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
 
             if (store->isInStore(r.second)) {
                 StorePathSet closure;
-                store->computeFSClosure(store->parseStorePath(store->toStorePath(r.second)), closure);
+                store->computeFSClosure(store->toStorePath(r.second).first, closure);
                 for (auto & path : closure)
                     allowedPaths->insert(store->printStorePath(path));
             } else
@@ -471,14 +485,21 @@ Value * EvalState::addConstant(const string & name, Value & v)
 Value * EvalState::addPrimOp(const string & name,
     size_t arity, PrimOpFun primOp)
 {
+    auto name2 = string(name, 0, 2) == "__" ? string(name, 2) : name;
+    Symbol sym = symbols.create(name2);
+
+    /* Hack to make constants lazy: turn them into a application of
+       the primop to a dummy value. */
     if (arity == 0) {
+        auto vPrimOp = allocValue();
+        vPrimOp->type = tPrimOp;
+        vPrimOp->primOp = new PrimOp(primOp, 1, sym);
         Value v;
-        primOp(*this, noPos, nullptr, v);
+        mkApp(v, *vPrimOp, *vPrimOp);
         return addConstant(name, v);
     }
+
     Value * v = allocValue();
-    string name2 = string(name, 0, 2) == "__" ? string(name, 2) : name;
-    Symbol sym = symbols.create(name2);
     v->type = tPrimOp;
     v->primOp = new PrimOp(primOp, arity, sym);
     staticBaseEnv.vars[symbols.create(name)] = baseEnvDispl;
@@ -501,67 +522,84 @@ Value & EvalState::getBuiltin(const string & name)
 
 LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2))
 {
-    throw EvalError(format(s) % s2);
+    throw EvalError(s, s2);
 }
 
-LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2, const Pos & pos))
+LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s, const string & s2))
 {
-    throw EvalError(format(s) % s2 % pos);
+    throw EvalError({
+        .hint = hintfmt(s, s2),
+        .errPos = pos
+    });
 }
 
 LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2, const string & s3))
 {
-    throw EvalError(format(s) % s2 % s3);
+    throw EvalError(s, s2, s3);
 }
 
-LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2, const string & s3, const Pos & pos))
+LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s, const string & s2, const string & s3))
 {
-    throw EvalError(format(s) % s2 % s3 % pos);
+    throw EvalError({
+        .hint = hintfmt(s, s2, s3),
+        .errPos = pos
+    });
 }
 
-LocalNoInlineNoReturn(void throwEvalError(const char * s, const Symbol & sym, const Pos & p1, const Pos & p2))
+LocalNoInlineNoReturn(void throwEvalError(const Pos & p1, const char * s, const Symbol & sym, const Pos & p2))
 {
-    throw EvalError(format(s) % sym % p1 % p2);
+    // p1 is where the error occurred; p2 is a position mentioned in the message.
+    throw EvalError({
+        .hint = hintfmt(s, sym, p2),
+        .errPos = p1
+    });
 }
 
-LocalNoInlineNoReturn(void throwTypeError(const char * s, const Pos & pos))
+LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s))
 {
-    throw TypeError(format(s) % pos);
+    throw TypeError({
+        .hint = hintfmt(s),
+        .errPos = pos
+    });
 }
 
 LocalNoInlineNoReturn(void throwTypeError(const char * s, const string & s1))
 {
-    throw TypeError(format(s) % s1);
+    throw TypeError(s, s1);
 }
 
-LocalNoInlineNoReturn(void throwTypeError(const char * s, const ExprLambda & fun, const Symbol & s2, const Pos & pos))
+LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s, const ExprLambda & fun, const Symbol & s2))
 {
-    throw TypeError(format(s) % fun.showNamePos() % s2 % pos);
+    throw TypeError({
+        .hint = hintfmt(s, fun.showNamePos(), s2),
+        .errPos = pos
+    });
 }
 
-LocalNoInlineNoReturn(void throwAssertionError(const char * s, const string & s1, const Pos & pos))
+LocalNoInlineNoReturn(void throwAssertionError(const Pos & pos, const char * s, const string & s1))
 {
-    throw AssertionError(format(s) % s1 % pos);
+    throw AssertionError({
+        .hint = hintfmt(s, s1),
+        .errPos = pos
+    });
 }
 
-LocalNoInlineNoReturn(void throwUndefinedVarError(const char * s, const string & s1, const Pos & pos))
+LocalNoInlineNoReturn(void throwUndefinedVarError(const Pos & pos, const char * s, const string & s1))
 {
-    throw UndefinedVarError(format(s) % s1 % pos);
+    throw UndefinedVarError({
+        .hint = hintfmt(s, s1),
+        .errPos = pos
+    });
 }
 
-LocalNoInline(void addErrorPrefix(Error & e, const char * s, const string & s2))
+LocalNoInline(void addErrorTrace(Error & e, const char * s, const string & s2))
 {
-    e.addPrefix(format(s) % s2);
+    e.addTrace(std::nullopt, s, s2);
 }
 
-LocalNoInline(void addErrorPrefix(Error & e, const char * s, const ExprLambda & fun, const Pos & pos))
+LocalNoInline(void addErrorTrace(Error & e, const Pos & pos, const char * s, const string & s2))
 {
-    e.addPrefix(format(s) % fun.showNamePos() % pos);
-}
-
-LocalNoInline(void addErrorPrefix(Error & e, const char * s, const string & s2, const Pos & pos))
-{
-    e.addPrefix(format(s) % s2 % pos);
+    e.addTrace(pos, s, s2);
 }
 
 
@@ -614,7 +652,7 @@ inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval)
             return j->value;
         }
         if (!env->prevWith)
-            throwUndefinedVarError("undefined variable '%1%' at %2%", var.name, var.pos);
+            throwUndefinedVarError(var.pos, "undefined variable '%1%'", var.name);
         for (size_t l = env->prevWith; l; --l, env = env->up) ;
     }
 }
@@ -775,7 +813,7 @@ void EvalState::evalFile(const Path & path_, Value & v)
     try {
         eval(e, v);
     } catch (Error & e) {
-        addErrorPrefix(e, "while evaluating the file '%1%':\n", path2);
+        addErrorTrace(e, "while evaluating the file '%1%':", path2);
         throw;
     }
 
@@ -812,7 +850,7 @@ inline bool EvalState::evalBool(Env & env, Expr * e, const Pos & pos)
     Value v;
     e->eval(*this, env, v);
     if (v.type != tBool)
-        throwTypeError("value is %1% while a Boolean was expected, at %2%", v, pos);
+        throwTypeError(pos, "value is %1% while a Boolean was expected", v);
     return v.boolean;
 }
 
@@ -926,7 +964,7 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v)
         Symbol nameSym = state.symbols.create(nameVal.string.s);
         Bindings::iterator j = v.attrs->find(nameSym);
         if (j != v.attrs->end())
-            throwEvalError("dynamic attribute '%1%' at %2% already defined at %3%", nameSym, i.pos, *j->pos);
+            throwEvalError(i.pos, "dynamic attribute '%1%' already defined at %2%", nameSym, *j->pos);
 
         i.valueExpr->setName(nameSym);
         /* Keep sorted order so find can catch duplicates */
@@ -1014,7 +1052,7 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v)
             } else {
                 state.forceAttrs(*vAttrs, pos);
                 if ((j = vAttrs->attrs->find(name)) == vAttrs->attrs->end())
-                    throwEvalError("attribute '%1%' missing, at %2%", name, pos);
+                    throwEvalError(pos, "attribute '%1%' missing", name);
             }
             vAttrs = j->value;
             pos2 = j->pos;
@@ -1025,8 +1063,8 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v)
 
     } catch (Error & e) {
         if (pos2 && pos2->file != state.sDerivationNix)
-            addErrorPrefix(e, "while evaluating the attribute '%1%' at %2%:\n",
-                showAttrPath(state, env, attrPath), *pos2);
+            addErrorTrace(e, *pos2, "while evaluating the attribute '%1%'",
+                showAttrPath(state, env, attrPath));
         throw;
     }
 
@@ -1140,7 +1178,7 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po
     }
 
     if (fun.type != tLambda)
-        throwTypeError("attempt to call something which is not a function but %1%, at %2%", fun, pos);
+        throwTypeError(pos, "attempt to call something which is not a function but %1%", fun);
 
     ExprLambda & lambda(*fun.lambda.fun);
 
@@ -1168,8 +1206,8 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po
         for (auto & i : lambda.formals->formals) {
             Bindings::iterator j = arg.attrs->find(i.name);
             if (j == arg.attrs->end()) {
-                if (!i.def) throwTypeError("%1% called without required argument '%2%', at %3%",
-                    lambda, i.name, pos);
+                if (!i.def) throwTypeError(pos, "%1% called without required argument '%2%'",
+                    lambda, i.name);
                 env2.values[displ++] = i.def->maybeThunk(*this, env2);
             } else {
                 attrsUsed++;
@@ -1184,7 +1222,7 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po
                user. */
             for (auto & i : *arg.attrs)
                 if (lambda.formals->argNames.find(i.name) == lambda.formals->argNames.end())
-                    throwTypeError("%1% called with unexpected argument '%2%', at %3%", lambda, i.name, pos);
+                    throwTypeError(pos, "%1% called with unexpected argument '%2%'", lambda, i.name);
             abort(); // can't happen
         }
     }
@@ -1194,11 +1232,15 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po
 
     /* Evaluate the body.  This is conditional on showTrace, because
        catching exceptions makes this function not tail-recursive. */
-    if (settings.showTrace)
+    if (loggerSettings.showTrace.get())
         try {
             lambda.body->eval(*this, env2, v);
         } catch (Error & e) {
-            addErrorPrefix(e, "while evaluating %1%, called from %2%:\n", lambda, pos);
+            addErrorTrace(e, lambda.pos, "while evaluating %s", 
+              (lambda.name.set() 
+                  ? "'" + (string) lambda.name + "'" 
+                  : "anonymous lambdaction"));
+            addErrorTrace(e, pos, "from call site%s", "");
             throw;
         }
     else
@@ -1273,7 +1315,7 @@ void ExprAssert::eval(EvalState & state, Env & env, Value & v)
     if (!state.evalBool(env, cond, pos)) {
         std::ostringstream out;
         cond->show(out);
-        throwAssertionError("assertion '%1%' failed at %2%", out.str(), pos);
+        throwAssertionError(pos, "assertion '%1%' failed at %2%", out.str());
     }
     body->eval(state, env, v);
 }
@@ -1425,14 +1467,14 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
                 nf = n;
                 nf += vTmp.fpoint;
             } else
-                throwEvalError("cannot add %1% to an integer, at %2%", showType(vTmp), pos);
+                throwEvalError(pos, "cannot add %1% to an integer", showType(vTmp));
         } else if (firstType == tFloat) {
             if (vTmp.type == tInt) {
                 nf += vTmp.integer;
             } else if (vTmp.type == tFloat) {
                 nf += vTmp.fpoint;
             } else
-                throwEvalError("cannot add %1% to a float, at %2%", showType(vTmp), pos);
+                throwEvalError(pos, "cannot add %1% to a float", showType(vTmp));
         } else
             s << state.coerceToString(pos, vTmp, context, false, firstType == tString);
     }
@@ -1443,7 +1485,7 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
         mkFloat(v, nf);
     else if (firstType == tPath) {
         if (!context.empty())
-            throwEvalError("a string that refers to a store path cannot be appended to a path, at %1%", pos);
+            throwEvalError(pos, "a string that refers to a store path cannot be appended to a path");
         auto path = canonPath(s.str());
         mkPath(v, path.c_str());
     } else
@@ -1473,7 +1515,7 @@ void EvalState::forceValueDeep(Value & v)
                 try {
                     recurse(*i.value);
                 } catch (Error & e) {
-                    addErrorPrefix(e, "while evaluating the attribute '%1%' at %2%:\n", i.name, *i.pos);
+                    addErrorTrace(e, *i.pos, "while evaluating the attribute '%1%'", i.name);
                     throw;
                 }
         }
@@ -1492,7 +1534,7 @@ NixInt EvalState::forceInt(Value & v, const Pos & pos)
 {
     forceValue(v, pos);
     if (v.type != tInt)
-        throwTypeError("value is %1% while an integer was expected, at %2%", v, pos);
+        throwTypeError(pos, "value is %1% while an integer was expected", v);
     return v.integer;
 }
 
@@ -1503,7 +1545,7 @@ NixFloat EvalState::forceFloat(Value & v, const Pos & pos)
     if (v.type == tInt)
         return v.integer;
     else if (v.type != tFloat)
-        throwTypeError("value is %1% while a float was expected, at %2%", v, pos);
+        throwTypeError(pos, "value is %1% while a float was expected", v);
     return v.fpoint;
 }
 
@@ -1512,7 +1554,7 @@ bool EvalState::forceBool(Value & v, const Pos & pos)
 {
     forceValue(v, pos);
     if (v.type != tBool)
-        throwTypeError("value is %1% while a Boolean was expected, at %2%", v, pos);
+        throwTypeError(pos, "value is %1% while a Boolean was expected", v);
     return v.boolean;
 }
 
@@ -1527,7 +1569,7 @@ void EvalState::forceFunction(Value & v, const Pos & pos)
 {
     forceValue(v, pos);
     if (v.type != tLambda && v.type != tPrimOp && v.type != tPrimOpApp && !isFunctor(v))
-        throwTypeError("value is %1% while a function was expected, at %2%", v, pos);
+        throwTypeError(pos, "value is %1% while a function was expected", v);
 }
 
 
@@ -1536,7 +1578,7 @@ string EvalState::forceString(Value & v, const Pos & pos)
     forceValue(v, pos);
     if (v.type != tString) {
         if (pos)
-            throwTypeError("value is %1% while a string was expected, at %2%", v, pos);
+            throwTypeError(pos, "value is %1% while a string was expected", v);
         else
             throwTypeError("value is %1% while a string was expected", v);
     }
@@ -1565,8 +1607,8 @@ string EvalState::forceStringNoCtx(Value & v, const Pos & pos)
     string s = forceString(v, pos);
     if (v.string.context) {
         if (pos)
-            throwEvalError("the string '%1%' is not allowed to refer to a store path (such as '%2%'), at %3%",
-                v.string.s, v.string.context[0], pos);
+            throwEvalError(pos, "the string '%1%' is not allowed to refer to a store path (such as '%2%')",
+                v.string.s, v.string.context[0]);
         else
             throwEvalError("the string '%1%' is not allowed to refer to a store path (such as '%2%')",
                 v.string.s, v.string.context[0]);
@@ -1622,7 +1664,7 @@ string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context,
             return *maybeString;
         }
         auto i = v.attrs->find(sOutPath);
-        if (i == v.attrs->end()) throwTypeError("cannot coerce a set to a string, at %1%", pos);
+        if (i == v.attrs->end()) throwTypeError(pos, "cannot coerce a set to a string");
         return coerceToString(pos, *i->value, context, coerceMore, copyToStore);
     }
 
@@ -1653,7 +1695,7 @@ string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context,
         }
     }
 
-    throwTypeError("cannot coerce %1% to a string, at %2%", v, pos);
+    throwTypeError(pos, "cannot coerce %1% to a string", v);
 }
 
 
@@ -1669,7 +1711,7 @@ string EvalState::copyPathToStore(PathSet & context, const Path & path)
     else {
         auto p = settings.readOnlyMode
             ? store->computeStorePathForPath(std::string(baseNameOf(path)), checkSourcePath(path)).first
-            : store->addToStore(std::string(baseNameOf(path)), checkSourcePath(path), true, htSHA256, defaultPathFilter, repair);
+            : store->addToStore(std::string(baseNameOf(path)), checkSourcePath(path), FileIngestionMethod::Recursive, htSHA256, defaultPathFilter, repair);
         dstPath = store->printStorePath(p);
         srcToStore.insert_or_assign(path, std::move(p));
         printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, dstPath);
@@ -1684,7 +1726,7 @@ Path EvalState::coerceToPath(const Pos & pos, Value & v, PathSet & context)
 {
     string path = coerceToString(pos, v, context, false, false);
     if (path == "" || path[0] != '/')
-        throwEvalError("string '%1%' doesn't represent an absolute path, at %2%", path, pos);
+        throwEvalError(pos, "string '%1%' doesn't represent an absolute path", path);
     return path;
 }
 
@@ -1891,8 +1933,10 @@ void EvalState::printStats()
 
 string ExternalValueBase::coerceToString(const Pos & pos, PathSet & context, bool copyMore, bool copyToStore) const
 {
-    throw TypeError(format("cannot coerce %1% to a string, at %2%") %
-        showType() % pos);
+    throw TypeError({
+        .hint = hintfmt("cannot coerce %1% to a string", showType()),
+        .errPos = pos
+    });
 }
 
 

src/libexpr/eval.hh

@@ -18,7 +18,7 @@ namespace nix {
 
 class Store;
 class EvalState;
-struct StorePath;
+class StorePath;
 enum RepairFlag : bool;
 
 
@@ -74,7 +74,8 @@ public:
         sSystem, sOverrides, sOutputs, sOutputName, sIgnoreNulls,
         sFile, sLine, sColumn, sFunctor, sToString,
         sRight, sWrong, sStructuredAttrs, sBuilder, sArgs,
-        sOutputHash, sOutputHashAlgo, sOutputHashMode;
+        sOutputHash, sOutputHashAlgo, sOutputHashMode,
+        sRecurseForDerivations;
     Symbol sDerivationNix;
 
     /* If set, force copying files to the Nix store even if they
@@ -249,7 +250,7 @@ private:
     friend struct ExprAttrs;
     friend struct ExprLet;
 
-    Expr * parse(const char * text, const Path & path,
+    Expr * parse(const char * text, FileOrigin origin, const Path & path,
         const Path & basePath, StaticEnv & staticEnv);
 
 public:
@@ -324,6 +325,7 @@ private:
 
 
 /* Return a string representing the type of the value `v'. */
+string showType(ValueType type);
 string showType(const Value & v);
 
 /* Decode a context string ‘!<name>!<path>’ into a pair <path,

src/libexpr/get-drvs.cc

@@ -1,7 +1,7 @@
 #include "get-drvs.hh"
 #include "util.hh"
 #include "eval-inline.hh"
-#include "derivations.hh"
+#include "store-api.hh"
 
 #include <cstring>
 #include <regex>
@@ -348,7 +348,7 @@ static void getDerivations(EvalState & state, Value & vIn,
                    should we recurse into it?  => Only if it has a
                    `recurseForDerivations = true' attribute. */
                 if (i->value->type == tAttrs) {
-                    Bindings::iterator j = i->value->attrs->find(state.symbols.create("recurseForDerivations"));
+                    Bindings::iterator j = i->value->attrs->find(state.sRecurseForDerivations);
                     if (j != i->value->attrs->end() && state.forceBool(*j->value, *j->pos))
                         getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures);
                 }

src/libexpr/lexer.l

@@ -127,14 +127,14 @@ or          { return OR_KW; }
               try {
                   yylval->n = boost::lexical_cast<int64_t>(yytext);
               } catch (const boost::bad_lexical_cast &) {
-                  throw ParseError(format("invalid integer '%1%'") % yytext);
+                  throw ParseError("invalid integer '%1%'", yytext);
               }
               return INT;
             }
 {FLOAT}     { errno = 0;
               yylval->nf = strtod(yytext, 0);
               if (errno != 0)
-                  throw ParseError(format("invalid float '%1%'") % yytext);
+                  throw ParseError("invalid float '%1%'", yytext);
               return FLOAT;
             }
 
@@ -219,4 +219,3 @@ or          { return OR_KW; }
             }
 
 %%
-

src/libexpr/local.mk

@@ -8,7 +8,7 @@ libexpr_SOURCES := $(wildcard $(d)/*.cc) $(wildcard $(d)/primops/*.cc) $(d)/lexe
 
 libexpr_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/libmain -I src/libexpr
 
-libexpr_LIBS = libutil libstore libfetchers libnixrust
+libexpr_LIBS = libutil libstore libfetchers
 
 libexpr_LDFLAGS =
 ifneq ($(OS), FreeBSD)

src/libexpr/nixexpr.cc

@@ -197,7 +197,22 @@ std::ostream & operator << (std::ostream & str, const Pos & pos)
     if (!pos)
         str << "undefined position";
     else
-        str << (format(ANSI_BOLD "%1%" ANSI_NORMAL ":%2%:%3%") % (string) pos.file % pos.line % pos.column).str();
+    {
+        auto f = format(ANSI_BOLD "%1%" ANSI_NORMAL ":%2%:%3%");
+        switch (pos.origin) {
+            case foFile:
+                f % (string) pos.file;
+                break;
+            case foStdin:
+            case foString:
+                f % "(string)";
+                break;
+            default:
+                throw Error("unhandled Pos origin!");
+        }
+        str << (f % pos.line % pos.column).str();
+    }
+
     return str;
 }
 
@@ -267,8 +282,11 @@ void ExprVar::bindVars(const StaticEnv & env)
     /* Otherwise, the variable must be obtained from the nearest
        enclosing `with'.  If there is no `with', then we can issue an
        "undefined variable" error now. */
-    if (withLevel == -1) throw UndefinedVarError(format("undefined variable '%1%' at %2%") % name % pos);
-
+    if (withLevel == -1)
+        throw UndefinedVarError({
+            .hint = hintfmt("undefined variable '%1%'", name),
+            .errPos = pos
+        });
     fromWith = true;
     this->level = withLevel;
 }

src/libexpr/nixexpr.hh

@@ -2,6 +2,7 @@
 
 #include "value.hh"
 #include "symbol-table.hh"
+#include "error.hh"
 
 #include <map>
 
@@ -23,11 +24,12 @@ MakeError(RestrictedPathError, Error);
 
 struct Pos
 {
+    FileOrigin origin;
     Symbol file;
     unsigned int line, column;
-    Pos() : line(0), column(0) { };
-    Pos(const Symbol & file, unsigned int line, unsigned int column)
-        : file(file), line(line), column(column) { };
+    Pos() : origin(foString), line(0), column(0) { };
+    Pos(FileOrigin origin, const Symbol & file, unsigned int line, unsigned int column)
+        : origin(origin), file(file), line(line), column(column) { };
     operator bool() const
     {
         return line != 0;
@@ -235,8 +237,10 @@ struct ExprLambda : Expr
         : pos(pos), arg(arg), matchAttrs(matchAttrs), formals(formals), body(body)
     {
         if (!arg.empty() && formals && formals->argNames.find(arg) != formals->argNames.end())
-            throw ParseError(format("duplicate formal function argument '%1%' at %2%")
-                % arg % pos);
+            throw ParseError({
+                .hint = hintfmt("duplicate formal function argument '%1%'", arg),
+                .errPos = pos
+            });
     };
     void setName(Symbol & name);
     string showNamePos() const;

src/libexpr/parser.y

@@ -30,8 +30,9 @@ namespace nix {
         SymbolTable & symbols;
         Expr * result;
         Path basePath;
-        Symbol path;
-        string error;
+        Symbol file;
+        FileOrigin origin;
+        ErrorInfo error;
         Symbol sLetBody;
         ParseData(EvalState & state)
             : state(state)
@@ -64,15 +65,19 @@ namespace nix {
 
 static void dupAttr(const AttrPath & attrPath, const Pos & pos, const Pos & prevPos)
 {
-    throw ParseError(format("attribute '%1%' at %2% already defined at %3%")
-        % showAttrPath(attrPath) % pos % prevPos);
+    throw ParseError({
+         .hint = hintfmt("attribute '%1%' already defined at %2%",
+            showAttrPath(attrPath), prevPos),
+         .errPos = pos
+    });
 }
 
-
 static void dupAttr(Symbol attr, const Pos & pos, const Pos & prevPos)
 {
-    throw ParseError(format("attribute '%1%' at %2% already defined at %3%")
-        % attr % pos % prevPos);
+    throw ParseError({
+        .hint = hintfmt("attribute '%1%' already defined at %2%", attr, prevPos),
+        .errPos = pos
+    });
 }
 
 
@@ -140,8 +145,11 @@ static void addAttr(ExprAttrs * attrs, AttrPath & attrPath,
 static void addFormal(const Pos & pos, Formals * formals, const Formal & formal)
 {
     if (!formals->argNames.insert(formal.name).second)
-        throw ParseError(format("duplicate formal function argument '%1%' at %2%")
-            % formal.name % pos);
+        throw ParseError({
+            .hint = hintfmt("duplicate formal function argument '%1%'",
+                formal.name),
+            .errPos = pos
+        });
     formals->formals.push_front(formal);
 }
 
@@ -238,7 +246,7 @@ static Expr * stripIndentation(const Pos & pos, SymbolTable & symbols, vector<Ex
 
 static inline Pos makeCurPos(const YYLTYPE & loc, ParseData * data)
 {
-    return Pos(data->path, loc.first_line, loc.first_column);
+    return Pos(data->origin, data->file, loc.first_line, loc.first_column);
 }
 
 #define CUR_POS makeCurPos(*yylocp, data)
@@ -249,8 +257,10 @@ static inline Pos makeCurPos(const YYLTYPE & loc, ParseData * data)
 
 void yyerror(YYLTYPE * loc, yyscan_t scanner, ParseData * data, const char * error)
 {
-    data->error = (format("%1%, at %2%")
-        % error % makeCurPos(*loc, data)).str();
+    data->error = {
+        .hint = hintfmt(error),
+        .errPos = makeCurPos(*loc, data)
+    };
 }
 
 
@@ -327,8 +337,10 @@ expr_function
     { $$ = new ExprWith(CUR_POS, $2, $4); }
   | LET binds IN expr_function
     { if (!$2->dynamicAttrs.empty())
-        throw ParseError(format("dynamic attributes not allowed in let at %1%")
-            % CUR_POS);
+        throw ParseError({
+            .hint = hintfmt("dynamic attributes not allowed in let"),
+            .errPos = CUR_POS
+        });
       $$ = new ExprLet($2, $4);
     }
   | expr_if
@@ -405,7 +417,10 @@ expr_simple
   | URI {
       static bool noURLLiterals = settings.isExperimentalFeatureEnabled("no-url-literals");
       if (noURLLiterals)
-          throw ParseError("URL literals are disabled, at %s", CUR_POS);
+          throw ParseError({
+              .hint = hintfmt("URL literals are disabled"),
+              .errPos = CUR_POS
+          });
       $$ = new ExprString(data->symbols.create($1));
   }
   | '(' expr ')' { $$ = $2; }
@@ -475,8 +490,10 @@ attrs
           $$->push_back(AttrName(str->s));
           delete str;
       } else
-          throw ParseError(format("dynamic attributes not allowed in inherit at %1%")
-              % makeCurPos(@2, data));
+          throw ParseError({
+              .hint = hintfmt("dynamic attributes not allowed in inherit"),
+              .errPos = makeCurPos(@2, data)
+          });
     }
   | { $$ = new AttrPath; }
   ;
@@ -552,13 +569,24 @@ formal
 namespace nix {
 
 
-Expr * EvalState::parse(const char * text,
+Expr * EvalState::parse(const char * text, FileOrigin origin,
     const Path & path, const Path & basePath, StaticEnv & staticEnv)
 {
     yyscan_t scanner;
     ParseData data(*this);
+    data.origin = origin;
+    switch (origin) {
+        case foFile: 
+            data.file = data.symbols.create(path);
+            break;
+        case foStdin:
+        case foString:
+            data.file = data.symbols.create(text);
+            break;
+        default:
+            assert(false);
+    }
     data.basePath = basePath;
-    data.path = data.symbols.create(path);
 
     yylex_init(&scanner);
     yy_scan_string(text, scanner);
@@ -608,13 +636,13 @@ Expr * EvalState::parseExprFromFile(const Path & path)
 
 Expr * EvalState::parseExprFromFile(const Path & path, StaticEnv & staticEnv)
 {
-    return parse(readFile(path).c_str(), path, dirOf(path), staticEnv);
+    return parse(readFile(path).c_str(), foFile, path, dirOf(path), staticEnv);
 }
 
 
 Expr * EvalState::parseExprFromString(std::string_view s, const Path & basePath, StaticEnv & staticEnv)
 {
-    return parse(s.data(), "(string)", basePath, staticEnv);
+    return parse(s.data(), foString, "", basePath, staticEnv);
 }
 
 
@@ -627,7 +655,7 @@ Expr * EvalState::parseExprFromString(std::string_view s, const Path & basePath)
 Expr * EvalState::parseStdin()
 {
     //Activity act(*logger, lvlTalkative, format("parsing standard input"));
-    return parseExprFromString(drainFD(0), absPath("."));
+    return parse(drainFD(0).data(), foStdin, "", absPath("."), staticBaseEnv);
 }
 
 
@@ -671,11 +699,13 @@ Path EvalState::findFile(SearchPath & searchPath, const string & path, const Pos
         Path res = r.second + suffix;
         if (pathExists(res)) return canonPath(res);
     }
-    format f = format(
-        "file '%1%' was not found in the Nix search path (add it using $NIX_PATH or -I)"
-        + string(pos ? ", at %2%" : ""));
-    f.exceptions(boost::io::all_error_bits ^ boost::io::too_many_args_bit);
-    throw ThrownError(f % path % pos);
+    throw ThrownError({
+        .hint = hintfmt(evalSettings.pureEval
+            ? "cannot look up '<%s>' in pure evaluation mode (use '--impure' to override)"
+            : "file '%s' was not found in the Nix search path (add it using $NIX_PATH or -I)",
+            path),
+        .errPos = pos
+    });
 }
 
 
@@ -691,7 +721,10 @@ std::pair<bool, std::string> EvalState::resolveSearchPathElem(const SearchPathEl
             res = { true, store->toRealPath(fetchers::downloadTarball(
                         store, resolveUri(elem.second), "source", false).storePath) };
         } catch (FileTransferError & e) {
-            printError(format("warning: Nix search path entry '%1%' cannot be downloaded, ignoring") % elem.second);
+            logWarning({
+                .name = "Entry download",
+                .hint = hintfmt("Nix search path entry '%1%' cannot be downloaded, ignoring", elem.second)
+            });
             res = { false, "" };
         }
     } else {
@@ -699,7 +732,10 @@ std::pair<bool, std::string> EvalState::resolveSearchPathElem(const SearchPathEl
         if (pathExists(path))
             res = { true, path };
         else {
-            printError(format("warning: Nix search path entry '%1%' does not exist, ignoring") % elem.second);
+            logWarning({
+                .name = "Entry not found",
+                .hint = hintfmt("warning: Nix search path entry '%1%' does not exist, ignoring", elem.second)
+            });
             res = { false, "" };
         }
     }

src/libexpr/primops.cc

@@ -50,20 +50,20 @@ void EvalState::realiseContext(const PathSet & context)
     std::vector<StorePathWithOutputs> drvs;
 
     for (auto & i : context) {
-        std::pair<string, string> decoded = decodeContext(i);
-        auto ctx = store->parseStorePath(decoded.first);
+        auto [ctxS, outputName] = decodeContext(i);
+        auto ctx = store->parseStorePath(ctxS);
         if (!store->isValidPath(ctx))
             throw InvalidPathError(store->printStorePath(ctx));
-        if (!decoded.second.empty() && ctx.isDerivation()) {
-            drvs.push_back(StorePathWithOutputs{ctx.clone(), {decoded.second}});
+        if (!outputName.empty() && ctx.isDerivation()) {
+            drvs.push_back(StorePathWithOutputs{ctx, {outputName}});
 
             /* Add the output of this derivation to the allowed
                paths. */
             if (allowedPaths) {
-                auto drv = store->derivationFromPath(store->parseStorePath(decoded.first));
-                DerivationOutputs::iterator i = drv.outputs.find(decoded.second);
+                auto drv = store->derivationFromPath(ctx);
+                DerivationOutputs::iterator i = drv.outputs.find(outputName);
                 if (i == drv.outputs.end())
-                    throw Error("derivation '%s' does not have an output named '%s'", decoded.first, decoded.second);
+                    throw Error("derivation '%s' does not have an output named '%s'", ctxS, outputName);
                 allowedPaths->insert(store->printStorePath(i->second.path));
             }
         }
@@ -79,6 +79,7 @@ void EvalState::realiseContext(const PathSet & context)
     StorePathSet willBuild, willSubstitute, unknown;
     unsigned long long downloadSize, narSize;
     store->queryMissing(drvs, willBuild, willSubstitute, unknown, downloadSize, narSize);
+
     store->buildPaths(drvs);
 }
 
@@ -93,8 +94,10 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args
     try {
         state.realiseContext(context);
     } catch (InvalidPathError & e) {
-        throw EvalError(format("cannot import '%1%', since path '%2%' is not valid, at %3%")
-            % path % e.path % pos);
+        throw EvalError({
+            .hint = hintfmt("cannot import '%1%', since path '%2%' is not valid", path, e.path),
+            .errPos = pos
+        });
     }
 
     Path realPath = state.checkSourcePath(state.toRealPath(path, context));
@@ -170,8 +173,12 @@ void prim_importNative(EvalState & state, const Pos & pos, Value * * args, Value
     try {
         state.realiseContext(context);
     } catch (InvalidPathError & e) {
-        throw EvalError(format("cannot import '%1%', since path '%2%' is not valid, at %3%")
-            % path % e.path % pos);
+        throw EvalError({
+            .hint = hintfmt(
+                "cannot import '%1%', since path '%2%' is not valid",
+                path, e.path),
+            .errPos = pos
+        });
     }
 
     path = state.checkSourcePath(path);
@@ -180,17 +187,17 @@ void prim_importNative(EvalState & state, const Pos & pos, Value * * args, Value
 
     void *handle = dlopen(path.c_str(), RTLD_LAZY | RTLD_LOCAL);
     if (!handle)
-        throw EvalError(format("could not open '%1%': %2%") % path % dlerror());
+        throw EvalError("could not open '%1%': %2%", path, dlerror());
 
     dlerror();
     ValueInitializer func = (ValueInitializer) dlsym(handle, sym.c_str());
     if(!func) {
         char *message = dlerror();
         if (message)
-            throw EvalError(format("could not load symbol '%1%' from '%2%': %3%") % sym % path % message);
+            throw EvalError("could not load symbol '%1%' from '%2%': %3%", sym, path, message);
         else
-            throw EvalError(format("symbol '%1%' from '%2%' resolved to NULL when a function pointer was expected")
-                    % sym % path);
+            throw EvalError("symbol '%1%' from '%2%' resolved to NULL when a function pointer was expected",
+                sym, path);
     }
 
     (func)(state, v);
@@ -206,7 +213,10 @@ void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v)
     auto elems = args[0]->listElems();
     auto count = args[0]->listSize();
     if (count == 0) {
-        throw EvalError(format("at least one argument to 'exec' required, at %1%") % pos);
+        throw EvalError({
+            .hint = hintfmt("at least one argument to 'exec' required"),
+            .errPos = pos
+        });
     }
     PathSet context;
     auto program = state.coerceToString(pos, *elems[0], context, false, false);
@@ -217,8 +227,11 @@ void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v)
     try {
         state.realiseContext(context);
     } catch (InvalidPathError & e) {
-        throw EvalError(format("cannot execute '%1%', since path '%2%' is not valid, at %3%")
-            % program % e.path % pos);
+        throw EvalError({
+            .hint = hintfmt("cannot execute '%1%', since path '%2%' is not valid",
+                program, e.path),
+            .errPos = pos
+        });
     }
 
     auto output = runProgram(program, true, commandArgs);
@@ -226,13 +239,13 @@ void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v)
     try {
         parsed = state.parseExprFromString(output, pos.file);
     } catch (Error & e) {
-        e.addPrefix(format("While parsing the output from '%1%', at %2%\n") % program % pos);
+        e.addTrace(pos, "While parsing the output from '%1%'", program);
         throw;
     }
     try {
         state.eval(parsed, v);
     } catch (Error & e) {
-        e.addPrefix(format("While evaluating the output from '%1%', at %2%\n") % program % pos);
+        e.addTrace(pos, "While evaluating the output from '%1%'", program);
         throw;
     }
 }
@@ -338,7 +351,7 @@ struct CompareValues
         if (v1->type == tInt && v2->type == tFloat)
             return v1->integer < v2->fpoint;
         if (v1->type != v2->type)
-            throw EvalError(format("cannot compare %1% with %2%") % showType(*v1) % showType(*v2));
+            throw EvalError("cannot compare %1% with %2%", showType(*v1), showType(*v2));
         switch (v1->type) {
             case tInt:
                 return v1->integer < v2->integer;
@@ -349,7 +362,7 @@ struct CompareValues
             case tPath:
                 return strcmp(v1->path, v2->path) < 0;
             default:
-                throw EvalError(format("cannot compare %1% with %2%") % showType(*v1) % showType(*v2));
+                throw EvalError("cannot compare %1% with %2%", showType(*v1), showType(*v2));
         }
     }
 };
@@ -370,7 +383,10 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar
     Bindings::iterator startSet =
         args[0]->attrs->find(state.symbols.create("startSet"));
     if (startSet == args[0]->attrs->end())
-        throw EvalError(format("attribute 'startSet' required, at %1%") % pos);
+        throw EvalError({
+            .hint = hintfmt("attribute 'startSet' required"),
+            .errPos = pos
+        });
     state.forceList(*startSet->value, pos);
 
     ValueList workSet;
@@ -381,7 +397,10 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar
     Bindings::iterator op =
         args[0]->attrs->find(state.symbols.create("operator"));
     if (op == args[0]->attrs->end())
-        throw EvalError(format("attribute 'operator' required, at %1%") % pos);
+        throw EvalError({
+            .hint = hintfmt("attribute 'operator' required"),
+            .errPos = pos
+        });
     state.forceValue(*op->value, pos);
 
     /* Construct the closure by applying the operator to element of
@@ -400,7 +419,10 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar
         Bindings::iterator key =
             e->attrs->find(state.symbols.create("key"));
         if (key == e->attrs->end())
-            throw EvalError(format("attribute 'key' required, at %1%") % pos);
+            throw EvalError({
+                .hint = hintfmt("attribute 'key' required"),
+                .errPos = pos
+            });
         state.forceValue(*key->value, pos);
 
         if (!doneKeys.insert(key->value).second) continue;
@@ -430,7 +452,7 @@ static void prim_abort(EvalState & state, const Pos & pos, Value * * args, Value
 {
     PathSet context;
     string s = state.coerceToString(pos, *args[0], context);
-    throw Abort(format("evaluation aborted with the following error message: '%1%'") % s);
+    throw Abort("evaluation aborted with the following error message: '%1%'", s);
 }
 
 
@@ -449,7 +471,7 @@ static void prim_addErrorContext(EvalState & state, const Pos & pos, Value * * a
         v = *args[1];
     } catch (Error & e) {
         PathSet context;
-        e.addPrefix(format("%1%\n") % state.coerceToString(pos, *args[0], context));
+        e.addTrace(std::nullopt, state.coerceToString(pos, *args[0], context));
         throw;
     }
 }
@@ -505,9 +527,9 @@ static void prim_trace(EvalState & state, const Pos & pos, Value * * args, Value
 {
     state.forceValue(*args[0], pos);
     if (args[0]->type == tString)
-        printError(format("trace: %1%") % args[0]->string.s);
+        printError("trace: %1%", args[0]->string.s);
     else
-        printError(format("trace: %1%") % *args[0]);
+        printError("trace: %1%", *args[0]);
     state.forceValue(*args[1], pos);
     v = *args[1];
 }
@@ -532,13 +554,16 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
     /* Figure out the name first (for stack backtraces). */
     Bindings::iterator attr = args[0]->attrs->find(state.sName);
     if (attr == args[0]->attrs->end())
-        throw EvalError(format("required attribute 'name' missing, at %1%") % pos);
+        throw EvalError({
+            .hint = hintfmt("required attribute 'name' missing"),
+            .errPos = pos
+        });
     string drvName;
     Pos & posDrvName(*attr->pos);
     try {
         drvName = state.forceStringNoCtx(*attr->value, pos);
     } catch (Error & e) {
-        e.addPrefix(format("while evaluating the derivation attribute 'name' at %1%:\n") % posDrvName);
+        e.addTrace(posDrvName, "while evaluating the derivation attribute 'name'");
         throw;
     }
 
@@ -562,7 +587,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
 
     std::optional<std::string> outputHash;
     std::string outputHashAlgo;
-    bool outputHashRecursive = false;
+    auto ingestionMethod = FileIngestionMethod::Flat;
 
     StringSet outputs;
     outputs.insert("out");
@@ -573,27 +598,40 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
         vomit("processing attribute '%1%'", key);
 
         auto handleHashMode = [&](const std::string & s) {
-            if (s == "recursive") outputHashRecursive = true;
-            else if (s == "flat") outputHashRecursive = false;
-            else throw EvalError("invalid value '%s' for 'outputHashMode' attribute, at %s", s, posDrvName);
+            if (s == "recursive") ingestionMethod = FileIngestionMethod::Recursive;
+            else if (s == "flat") ingestionMethod = FileIngestionMethod::Flat;
+            else
+                throw EvalError({
+                    .hint = hintfmt("invalid value '%s' for 'outputHashMode' attribute", s),
+                    .errPos = posDrvName
+                });
         };
 
         auto handleOutputs = [&](const Strings & ss) {
             outputs.clear();
             for (auto & j : ss) {
                 if (outputs.find(j) != outputs.end())
-                    throw EvalError(format("duplicate derivation output '%1%', at %2%") % j % posDrvName);
+                    throw EvalError({
+                        .hint = hintfmt("duplicate derivation output '%1%'", j),
+                        .errPos = posDrvName
+                    });
                 /* !!! Check whether j is a valid attribute
                    name. */
                 /* Derivations cannot be named ‘drv’, because
                    then we'd have an attribute ‘drvPath’ in
                    the resulting set. */
                 if (j == "drv")
-                    throw EvalError(format("invalid derivation output name 'drv', at %1%") % posDrvName);
+                    throw EvalError({
+                        .hint = hintfmt("invalid derivation output name 'drv'" ),
+                        .errPos = posDrvName
+                    });
                 outputs.insert(j);
             }
             if (outputs.empty())
-                throw EvalError(format("derivation cannot have an empty set of outputs, at %1%") % posDrvName);
+                throw EvalError({
+                    .hint = hintfmt("derivation cannot have an empty set of outputs"),
+                    .errPos = posDrvName
+                });
         };
 
         try {
@@ -658,8 +696,9 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
             }
 
         } catch (Error & e) {
-            e.addPrefix(format("while evaluating the attribute '%1%' of the derivation '%2%' at %3%:\n")
-                % key % drvName % posDrvName);
+            e.addTrace(posDrvName, 
+                "while evaluating the attribute '%1%' of the derivation '%2%'",
+                key, drvName);
             throw;
         }
     }
@@ -686,9 +725,9 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
             StorePathSet refs;
             state.store->computeFSClosure(state.store->parseStorePath(std::string_view(path).substr(1)), refs);
             for (auto & j : refs) {
-                drv.inputSrcs.insert(j.clone());
+                drv.inputSrcs.insert(j);
                 if (j.isDerivation())
-                    drv.inputDrvs[j.clone()] = state.store->queryDerivationOutputNames(j);
+                    drv.inputDrvs[j] = state.store->readDerivation(j).outputNames();
             }
         }
 
@@ -705,27 +744,44 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
 
     /* Do we have all required attributes? */
     if (drv.builder == "")
-        throw EvalError(format("required attribute 'builder' missing, at %1%") % posDrvName);
+        throw EvalError({
+            .hint = hintfmt("required attribute 'builder' missing"),
+            .errPos = posDrvName
+        });
+
     if (drv.platform == "")
-        throw EvalError(format("required attribute 'system' missing, at %1%") % posDrvName);
+        throw EvalError({
+            .hint = hintfmt("required attribute 'system' missing"),
+            .errPos = posDrvName
+        });
 
     /* Check whether the derivation name is valid. */
     if (isDerivation(drvName))
-        throw EvalError("derivation names are not allowed to end in '%s', at %s", drvExtension, posDrvName);
+        throw EvalError({
+            .hint = hintfmt("derivation names are not allowed to end in '%s'", drvExtension),
+            .errPos = posDrvName
+        });
 
     if (outputHash) {
         /* Handle fixed-output derivations. */
         if (outputs.size() != 1 || *(outputs.begin()) != "out")
-            throw Error(format("multiple outputs are not supported in fixed-output derivations, at %1%") % posDrvName);
+            throw Error({
+                .hint = hintfmt("multiple outputs are not supported in fixed-output derivations"),
+                .errPos = posDrvName
+            });
 
-        HashType ht = outputHashAlgo.empty() ? htUnknown : parseHashType(outputHashAlgo);
-        Hash h(*outputHash, ht);
+        std::optional<HashType> ht = parseHashTypeOpt(outputHashAlgo);
+        Hash h = newHashAllowEmpty(*outputHash, ht);
 
-        auto outPath = state.store->makeFixedOutputPath(outputHashRecursive, h, drvName);
+        auto outPath = state.store->makeFixedOutputPath(ingestionMethod, h, drvName);
         if (!jsonObject) drv.env["out"] = state.store->printStorePath(outPath);
-        drv.outputs.insert_or_assign("out", DerivationOutput(std::move(outPath),
-                (outputHashRecursive ? "r:" : "") + printHashType(h.type),
-                h.to_string(Base16, false)));
+        drv.outputs.insert_or_assign("out", DerivationOutput {
+            .path = std::move(outPath),
+            .hash = FixedOutputHash {
+                .method = ingestionMethod,
+                .hash = std::move(h),
+            },
+        });
     }
 
     else {
@@ -738,7 +794,10 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
         for (auto & i : outputs) {
             if (!jsonObject) drv.env[i] = "";
             drv.outputs.insert_or_assign(i,
-                DerivationOutput(StorePath::dummy.clone(), "", ""));
+                DerivationOutput {
+                    .path = StorePath::dummy,
+                    .hash = std::optional<FixedOutputHash> {},
+                });
         }
 
         Hash h = hashDerivationModulo(*state.store, Derivation(drv), true);
@@ -747,7 +806,10 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
             auto outPath = state.store->makeOutputPath(i, h, drvName);
             if (!jsonObject) drv.env[i] = state.store->printStorePath(outPath);
             drv.outputs.insert_or_assign(i,
-                DerivationOutput(std::move(outPath), "", ""));
+                DerivationOutput {
+                    .path = std::move(outPath),
+                    .hash = std::optional<FixedOutputHash>(),
+                });
         }
     }
 
@@ -760,7 +822,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
     /* Optimisation, but required in read-only mode! because in that
        case we don't actually write store derivations, so we can't
        read them later. */
-    drvHashes.insert_or_assign(drvPath.clone(),
+    drvHashes.insert_or_assign(drvPath,
         hashDerivationModulo(*state.store, Derivation(drv), false));
 
     state.mkAttrs(v, 1 + drv.outputs.size());
@@ -817,11 +879,14 @@ static void prim_storePath(EvalState & state, const Pos & pos, Value * * args, V
        e.g. nix-push does the right thing. */
     if (!state.store->isStorePath(path)) path = canonPath(path, true);
     if (!state.store->isInStore(path))
-        throw EvalError(format("path '%1%' is not in the Nix store, at %2%") % path % pos);
-    Path path2 = state.store->toStorePath(path);
+        throw EvalError({
+            .hint = hintfmt("path '%1%' is not in the Nix store", path),
+            .errPos = pos
+        });
+    auto path2 = state.store->toStorePath(path).first;
     if (!settings.readOnlyMode)
-        state.store->ensurePath(state.store->parseStorePath(path2));
-    context.insert(path2);
+        state.store->ensurePath(path2);
+    context.insert(state.store->printStorePath(path2));
     mkString(v, path, context);
 }
 
@@ -833,9 +898,12 @@ static void prim_pathExists(EvalState & state, const Pos & pos, Value * * args,
     try {
         state.realiseContext(context);
     } catch (InvalidPathError & e) {
-        throw EvalError(format(
-                "cannot check the existence of '%1%', since path '%2%' is not valid, at %3%")
-            % path % e.path % pos);
+        throw EvalError({
+            .hint = hintfmt(
+                "cannot check the existence of '%1%', since path '%2%' is not valid",
+                path, e.path),
+            .errPos = pos
+        });
     }
 
     try {
@@ -878,12 +946,14 @@ static void prim_readFile(EvalState & state, const Pos & pos, Value * * args, Va
     try {
         state.realiseContext(context);
     } catch (InvalidPathError & e) {
-        throw EvalError(format("cannot read '%1%', since path '%2%' is not valid, at %3%")
-            % path % e.path % pos);
+        throw EvalError({
+            .hint = hintfmt("cannot read '%1%', since path '%2%' is not valid", path, e.path),
+            .errPos = pos
+        });
     }
     string s = readFile(state.checkSourcePath(state.toRealPath(path, context)));
     if (s.find((char) 0) != string::npos)
-        throw Error(format("the contents of the file '%1%' cannot be represented as a Nix string") % path);
+        throw Error("the contents of the file '%1%' cannot be represented as a Nix string", path);
     mkString(v, s.c_str());
 }
 
@@ -907,7 +977,10 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va
 
         i = v2.attrs->find(state.symbols.create("path"));
         if (i == v2.attrs->end())
-            throw EvalError(format("attribute 'path' missing, at %1%") % pos);
+            throw EvalError({
+                .hint = hintfmt("attribute 'path' missing"),
+                .errPos = pos
+            });
 
         PathSet context;
         string path = state.coerceToString(pos, *i->value, context, false, false);
@@ -915,8 +988,10 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va
         try {
             state.realiseContext(context);
         } catch (InvalidPathError & e) {
-            throw EvalError(format("cannot find '%1%', since path '%2%' is not valid, at %3%")
-                % path % e.path % pos);
+            throw EvalError({
+                .hint = hintfmt("cannot find '%1%', since path '%2%' is not valid", path, e.path),
+                .errPos = pos
+            });
         }
 
         searchPath.emplace_back(prefix, path);
@@ -931,14 +1006,17 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va
 static void prim_hashFile(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     string type = state.forceStringNoCtx(*args[0], pos);
-    HashType ht = parseHashType(type);
-    if (ht == htUnknown)
-      throw Error(format("unknown hash type '%1%', at %2%") % type % pos);
+    std::optional<HashType> ht = parseHashType(type);
+    if (!ht)
+      throw Error({
+          .hint = hintfmt("unknown hash type '%1%'", type),
+          .errPos = pos
+      });
 
     PathSet context; // discarded
     Path p = state.coerceToPath(pos, *args[1], context);
 
-    mkString(v, hashFile(ht, state.checkSourcePath(p)).to_string(Base16, false), context);
+    mkString(v, hashFile(*ht, state.checkSourcePath(p)).to_string(Base16, false), context);
 }
 
 /* Read a directory (without . or ..) */
@@ -949,8 +1027,10 @@ static void prim_readDir(EvalState & state, const Pos & pos, Value * * args, Val
     try {
         state.realiseContext(ctx);
     } catch (InvalidPathError & e) {
-        throw EvalError(format("cannot read '%1%', since path '%2%' is not valid, at %3%")
-            % path % e.path % pos);
+        throw EvalError({
+            .hint = hintfmt("cannot read '%1%', since path '%2%' is not valid", path, e.path),
+            .errPos = pos
+        });
     }
 
     DirEntries entries = readDirectory(state.checkSourcePath(path));
@@ -1020,9 +1100,13 @@ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Valu
 
     for (auto path : context) {
         if (path.at(0) != '/')
-            throw EvalError(format(
-                "in 'toFile': the file named '%1%' must not contain a reference "
-                "to a derivation but contains (%2%), at %3%") % name % path % pos);
+            throw EvalError( {
+                .hint = hintfmt(
+                    "in 'toFile': the file named '%1%' must not contain a reference "
+                    "to a derivation but contains (%2%)",
+                    name, path),
+                .errPos = pos
+            });
         refs.insert(state.store->parseStorePath(path));
     }
 
@@ -1039,7 +1123,7 @@ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Valu
 
 
 static void addPath(EvalState & state, const Pos & pos, const string & name, const Path & path_,
-    Value * filterFun, bool recursive, const Hash & expectedHash, Value & v)
+    Value * filterFun, FileIngestionMethod method, const Hash & expectedHash, Value & v)
 {
     const auto path = evalSettings.pureEval && expectedHash ?
         path_ :
@@ -1070,12 +1154,12 @@ static void addPath(EvalState & state, const Pos & pos, const string & name, con
 
     std::optional<StorePath> expectedStorePath;
     if (expectedHash)
-        expectedStorePath = state.store->makeFixedOutputPath(recursive, expectedHash, name);
+        expectedStorePath = state.store->makeFixedOutputPath(method, expectedHash, name);
     Path dstPath;
     if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) {
         dstPath = state.store->printStorePath(settings.readOnlyMode
-            ? state.store->computeStorePathForPath(name, path, recursive, htSHA256, filter).first
-            : state.store->addToStore(name, path, recursive, htSHA256, filter, state.repair));
+            ? state.store->computeStorePathForPath(name, path, method, htSHA256, filter).first
+            : state.store->addToStore(name, path, method, htSHA256, filter, state.repair));
         if (expectedHash && expectedStorePath != state.store->parseStorePath(dstPath))
             throw Error("store path mismatch in (possibly filtered) path added from '%s'", path);
     } else
@@ -1090,13 +1174,21 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args
     PathSet context;
     Path path = state.coerceToPath(pos, *args[1], context);
     if (!context.empty())
-        throw EvalError(format("string '%1%' cannot refer to other paths, at %2%") % path % pos);
+        throw EvalError({
+            .hint = hintfmt("string '%1%' cannot refer to other paths", path),
+            .errPos = pos
+        });
 
     state.forceValue(*args[0], pos);
     if (args[0]->type != tLambda)
-        throw TypeError(format("first argument in call to 'filterSource' is not a function but %1%, at %2%") % showType(*args[0]) % pos);
+        throw TypeError({
+            .hint = hintfmt(
+                "first argument in call to 'filterSource' is not a function but %1%",
+                showType(*args[0])),
+            .errPos = pos
+        });
 
-    addPath(state, pos, std::string(baseNameOf(path)), path, args[0], true, Hash(), v);
+    addPath(state, pos, std::string(baseNameOf(path)), path, args[0], FileIngestionMethod::Recursive, Hash(), v);
 }
 
 static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value & v)
@@ -1105,7 +1197,7 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value
     Path path;
     string name;
     Value * filterFun = nullptr;
-    auto recursive = true;
+    auto method = FileIngestionMethod::Recursive;
     Hash expectedHash;
 
     for (auto & attr : *args[0]->attrs) {
@@ -1114,25 +1206,34 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value
             PathSet context;
             path = state.coerceToPath(*attr.pos, *attr.value, context);
             if (!context.empty())
-                throw EvalError(format("string '%1%' cannot refer to other paths, at %2%") % path % *attr.pos);
+                throw EvalError({
+                    .hint = hintfmt("string '%1%' cannot refer to other paths", path),
+                    .errPos = *attr.pos
+                });
         } else if (attr.name == state.sName)
             name = state.forceStringNoCtx(*attr.value, *attr.pos);
         else if (n == "filter") {
             state.forceValue(*attr.value, pos);
             filterFun = attr.value;
         } else if (n == "recursive")
-            recursive = state.forceBool(*attr.value, *attr.pos);
+            method = FileIngestionMethod { state.forceBool(*attr.value, *attr.pos) };
         else if (n == "sha256")
-            expectedHash = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256);
+            expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256);
         else
-            throw EvalError(format("unsupported argument '%1%' to 'addPath', at %2%") % attr.name % *attr.pos);
+            throw EvalError({
+                .hint = hintfmt("unsupported argument '%1%' to 'addPath'", attr.name),
+                .errPos = *attr.pos
+            });
     }
     if (path.empty())
-        throw EvalError(format("'path' required, at %1%") % pos);
+        throw EvalError({
+            .hint = hintfmt("'path' required"),
+            .errPos = pos
+        });
     if (name.empty())
         name = baseNameOf(path);
 
-    addPath(state, pos, name, path, filterFun, recursive, expectedHash, v);
+    addPath(state, pos, name, path, filterFun, method, expectedHash, v);
 }
 
 
@@ -1186,7 +1287,10 @@ void prim_getAttr(EvalState & state, const Pos & pos, Value * * args, Value & v)
     // !!! Should we create a symbol here or just do a lookup?
     Bindings::iterator i = args[1]->attrs->find(state.symbols.create(attr));
     if (i == args[1]->attrs->end())
-        throw EvalError(format("attribute '%1%' missing, at %2%") % attr % pos);
+        throw EvalError({
+            .hint = hintfmt("attribute '%1%' missing", attr),
+            .errPos = pos
+        });
     // !!! add to stack trace?
     if (state.countCalls && i->pos) state.attrSelects[*i->pos]++;
     state.forceValue(*i->value, pos);
@@ -1266,15 +1370,20 @@ static void prim_listToAttrs(EvalState & state, const Pos & pos, Value * * args,
 
         Bindings::iterator j = v2.attrs->find(state.sName);
         if (j == v2.attrs->end())
-            throw TypeError(format("'name' attribute missing in a call to 'listToAttrs', at %1%") % pos);
+            throw TypeError({
+                .hint = hintfmt("'name' attribute missing in a call to 'listToAttrs'"),
+                .errPos = pos
+            });
         string name = state.forceStringNoCtx(*j->value, pos);
 
         Symbol sym = state.symbols.create(name);
         if (seen.insert(sym).second) {
             Bindings::iterator j2 = v2.attrs->find(state.symbols.create(state.sValue));
             if (j2 == v2.attrs->end())
-                throw TypeError(format("'value' attribute missing in a call to 'listToAttrs', at %1%") % pos);
-
+                throw TypeError({
+                    .hint = hintfmt("'value' attribute missing in a call to 'listToAttrs'"),
+                    .errPos = pos
+                });
             v.attrs->push_back(Attr(sym, j2->value, j2->pos));
         }
     }
@@ -1347,7 +1456,10 @@ static void prim_functionArgs(EvalState & state, const Pos & pos, Value * * args
 {
     state.forceValue(*args[0], pos);
     if (args[0]->type != tLambda)
-        throw TypeError(format("'functionArgs' requires a function, at %1%") % pos);
+        throw TypeError({
+            .hint = hintfmt("'functionArgs' requires a function"),
+            .errPos = pos
+        });
 
     if (!args[0]->lambda.fun->matchAttrs) {
         state.mkAttrs(v, 0);
@@ -1400,7 +1512,10 @@ static void elemAt(EvalState & state, const Pos & pos, Value & list, int n, Valu
 {
     state.forceList(list, pos);
     if (n < 0 || (unsigned int) n >= list.listSize())
-        throw Error(format("list index %1% is out of bounds, at %2%") % n % pos);
+        throw Error({
+            .hint = hintfmt("list index %1% is out of bounds", n),
+            .errPos = pos
+        });
     state.forceValue(*list.listElems()[n], pos);
     v = *list.listElems()[n];
 }
@@ -1427,7 +1542,11 @@ static void prim_tail(EvalState & state, const Pos & pos, Value * * args, Value
 {
     state.forceList(*args[0], pos);
     if (args[0]->listSize() == 0)
-        throw Error(format("'tail' called on an empty list, at %1%") % pos);
+        throw Error({
+            .hint = hintfmt("'tail' called on an empty list"),
+            .errPos = pos
+        });
+
     state.mkList(v, args[0]->listSize() - 1);
     for (unsigned int n = 0; n < v.listSize(); ++n)
         v.listElems()[n] = args[0]->listElems()[n + 1];
@@ -1568,7 +1687,10 @@ static void prim_genList(EvalState & state, const Pos & pos, Value * * args, Val
     auto len = state.forceInt(*args[1], pos);
 
     if (len < 0)
-        throw EvalError(format("cannot create list of size %1%, at %2%") % len % pos);
+        throw EvalError({
+            .hint = hintfmt("cannot create list of size %1%", len),
+            .errPos = pos
+        });
 
     state.mkList(v, len);
 
@@ -1726,7 +1848,11 @@ static void prim_div(EvalState & state, const Pos & pos, Value * * args, Value &
     state.forceValue(*args[1], pos);
 
     NixFloat f2 = state.forceFloat(*args[1], pos);
-    if (f2 == 0) throw EvalError(format("division by zero, at %1%") % pos);
+    if (f2 == 0)
+        throw EvalError({
+            .hint = hintfmt("division by zero"),
+            .errPos = pos
+        });
 
     if (args[0]->type == tFloat || args[1]->type == tFloat) {
         mkFloat(v, state.forceFloat(*args[0], pos) / state.forceFloat(*args[1], pos));
@@ -1735,7 +1861,11 @@ static void prim_div(EvalState & state, const Pos & pos, Value * * args, Value &
         NixInt i2 = state.forceInt(*args[1], pos);
         /* Avoid division overflow as it might raise SIGFPE. */
         if (i1 == std::numeric_limits<NixInt>::min() && i2 == -1)
-            throw EvalError(format("overflow in integer division, at %1%") % pos);
+            throw EvalError({
+                .hint = hintfmt("overflow in integer division"),
+                .errPos = pos
+            });
+
         mkInt(v, i1 / i2);
     }
 }
@@ -1791,7 +1921,11 @@ static void prim_substring(EvalState & state, const Pos & pos, Value * * args, V
     PathSet context;
     string s = state.coerceToString(pos, *args[2], context);
 
-    if (start < 0) throw EvalError(format("negative start position in 'substring', at %1%") % pos);
+    if (start < 0)
+        throw EvalError({
+            .hint = hintfmt("negative start position in 'substring'"),
+            .errPos = pos
+        });
 
     mkString(v, (unsigned int) start >= s.size() ? "" : string(s, start, len), context);
 }
@@ -1809,14 +1943,17 @@ static void prim_stringLength(EvalState & state, const Pos & pos, Value * * args
 static void prim_hashString(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     string type = state.forceStringNoCtx(*args[0], pos);
-    HashType ht = parseHashType(type);
-    if (ht == htUnknown)
-      throw Error(format("unknown hash type '%1%', at %2%") % type % pos);
+    std::optional<HashType> ht = parseHashType(type);
+    if (!ht)
+        throw Error({
+            .hint = hintfmt("unknown hash type '%1%'", type),
+            .errPos = pos
+        });
 
     PathSet context; // discarded
     string s = state.forceString(*args[1], context, pos);
 
-    mkString(v, hashString(ht, s).to_string(Base16, false), context);
+    mkString(v, hashString(*ht, s).to_string(Base16, false), context);
 }
 
 
@@ -1853,10 +1990,16 @@ void prim_match(EvalState & state, const Pos & pos, Value * * args, Value & v)
 
     } catch (std::regex_error &e) {
         if (e.code() == std::regex_constants::error_space) {
-          // limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++
-          throw EvalError("memory limit exceeded by regular expression '%s', at %s", re, pos);
+            // limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++
+            throw EvalError({
+                .hint = hintfmt("memory limit exceeded by regular expression '%s'", re),
+                .errPos = pos
+            });
         } else {
-          throw EvalError("invalid regular expression '%s', at %s", re, pos);
+            throw EvalError({
+                .hint = hintfmt("invalid regular expression '%s'", re),
+                .errPos = pos
+            });
         }
     }
 }
@@ -1920,10 +2063,16 @@ static void prim_split(EvalState & state, const Pos & pos, Value * * args, Value
 
     } catch (std::regex_error &e) {
         if (e.code() == std::regex_constants::error_space) {
-          // limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++
-          throw EvalError("memory limit exceeded by regular expression '%s', at %s", re, pos);
+            // limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++
+            throw EvalError({
+                .hint = hintfmt("memory limit exceeded by regular expression '%s'", re),
+                .errPos = pos
+            });
         } else {
-          throw EvalError("invalid regular expression '%s', at %s", re, pos);
+            throw EvalError({
+                .hint = hintfmt("invalid regular expression '%s'", re),
+                .errPos = pos
+            });
         }
     }
 }
@@ -1954,7 +2103,10 @@ static void prim_replaceStrings(EvalState & state, const Pos & pos, Value * * ar
     state.forceList(*args[0], pos);
     state.forceList(*args[1], pos);
     if (args[0]->listSize() != args[1]->listSize())
-        throw EvalError(format("'from' and 'to' arguments to 'replaceStrings' have different lengths, at %1%") % pos);
+        throw EvalError({
+            .hint = hintfmt("'from' and 'to' arguments to 'replaceStrings' have different lengths"),
+            .errPos = pos
+        });
 
     vector<string> from;
     from.reserve(args[0]->listSize());
@@ -2057,10 +2209,11 @@ static void prim_splitVersion(EvalState & state, const Pos & pos, Value * * args
 RegisterPrimOp::PrimOps * RegisterPrimOp::primOps;
 
 
-RegisterPrimOp::RegisterPrimOp(std::string name, size_t arity, PrimOpFun fun)
+RegisterPrimOp::RegisterPrimOp(std::string name, size_t arity, PrimOpFun fun,
+    std::optional<std::string> requiredFeature)
 {
     if (!primOps) primOps = new PrimOps;
-    primOps->emplace_back(name, arity, fun);
+    primOps->push_back({name, arity, fun, requiredFeature});
 }
 
 
@@ -2252,7 +2405,8 @@ void EvalState::createBaseEnv()
 
     if (RegisterPrimOp::primOps)
         for (auto & primOp : *RegisterPrimOp::primOps)
-            addPrimOp(std::get<0>(primOp), std::get<1>(primOp), std::get<2>(primOp));
+            if (!primOp.requiredFeature || settings.isExperimentalFeatureEnabled(*primOp.requiredFeature))
+                addPrimOp(primOp.name, primOp.arity, primOp.primOp);
 
     /* Now that we've added all primops, sort the `builtins' set,
        because attribute lookups expect it to be sorted. */

src/libexpr/primops.hh

@@ -7,12 +7,25 @@ namespace nix {
 
 struct RegisterPrimOp
 {
-    typedef std::vector<std::tuple<std::string, size_t, PrimOpFun>> PrimOps;
+    struct Info
+    {
+        std::string name;
+        size_t arity;
+        PrimOpFun primOp;
+        std::optional<std::string> requiredFeature;
+    };
+
+    typedef std::vector<Info> PrimOps;
     static PrimOps * primOps;
+
     /* You can register a constant by passing an arity of 0. fun
        will get called during EvalState initialization, so there
        may be primops not yet added and builtins is not yet sorted. */
-    RegisterPrimOp(std::string name, size_t arity, PrimOpFun fun);
+    RegisterPrimOp(
+        std::string name,
+        size_t arity,
+        PrimOpFun fun,
+        std::optional<std::string> requiredFeature = {});
 };
 
 /* These primops are disabled without enableNativeCode, but plugins

src/libexpr/primops/context.cc

@@ -1,6 +1,6 @@
 #include "primops.hh"
 #include "eval-inline.hh"
-#include "derivations.hh"
+#include "store-api.hh"
 
 namespace nix {
 
@@ -146,7 +146,10 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg
     auto sAllOutputs = state.symbols.create("allOutputs");
     for (auto & i : *args[1]->attrs) {
         if (!state.store->isStorePath(i.name))
-            throw EvalError("Context key '%s' is not a store path, at %s", i.name, i.pos);
+            throw EvalError({
+                .hint = hintfmt("Context key '%s' is not a store path", i.name),
+                .errPos = *i.pos
+            });
         if (!settings.readOnlyMode)
             state.store->ensurePath(state.store->parseStorePath(i.name));
         state.forceAttrs(*i.value, *i.pos);
@@ -160,7 +163,10 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg
         if (iter != i.value->attrs->end()) {
             if (state.forceBool(*iter->value, *iter->pos)) {
                 if (!isDerivation(i.name)) {
-                    throw EvalError("Tried to add all-outputs context of %s, which is not a derivation, to a string, at %s", i.name, i.pos);
+                    throw EvalError({
+                        .hint = hintfmt("Tried to add all-outputs context of %s, which is not a derivation, to a string", i.name),
+                        .errPos = *i.pos
+                    });
                 }
                 context.insert("=" + string(i.name));
             }
@@ -170,7 +176,10 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg
         if (iter != i.value->attrs->end()) {
             state.forceList(*iter->value, *iter->pos);
             if (iter->value->listSize() && !isDerivation(i.name)) {
-                throw EvalError("Tried to add derivation output context of %s, which is not a derivation, to a string, at %s", i.name, i.pos);
+                throw EvalError({
+                    .hint = hintfmt("Tried to add derivation output context of %s, which is not a derivation, to a string", i.name),
+                    .errPos = *i.pos
+                });
             }
             for (unsigned int n = 0; n < iter->value->listSize(); ++n) {
                 auto name = state.forceStringNoCtx(*iter->value->listElems()[n], *iter->pos);

src/libexpr/primops/fetchGit.cc

@@ -35,11 +35,17 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va
             else if (n == "submodules")
                 fetchSubmodules = state.forceBool(*attr.value, *attr.pos);
             else
-                throw EvalError("unsupported argument '%s' to 'fetchGit', at %s", attr.name, *attr.pos);
+                throw EvalError({
+                    .hint = hintfmt("unsupported argument '%s' to 'fetchGit'", attr.name),
+                    .errPos = *attr.pos
+                });
         }
 
         if (url.empty())
-            throw EvalError(format("'url' argument required, at %1%") % pos);
+            throw EvalError({
+                .hint = hintfmt("'url' argument required"),
+                .errPos = pos
+            });
 
     } else
         url = state.coerceToString(pos, *args[0], context, false, false);

src/libexpr/primops/fetchMercurial.cc

@@ -38,11 +38,17 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar
             else if (n == "name")
                 name = state.forceStringNoCtx(*attr.value, *attr.pos);
             else
-                throw EvalError("unsupported argument '%s' to 'fetchMercurial', at %s", attr.name, *attr.pos);
+                throw EvalError({
+                    .hint = hintfmt("unsupported argument '%s' to 'fetchMercurial'", attr.name),
+                    .errPos = *attr.pos
+                });
         }
 
         if (url.empty())
-            throw EvalError(format("'url' argument required, at %1%") % pos);
+            throw EvalError({
+                .hint = hintfmt("'url' argument required"),
+                .errPos = pos
+            });
 
     } else
         url = state.coerceToString(pos, *args[0], context, false, false);

src/libexpr/primops/fetchTree.cc

@@ -23,7 +23,7 @@ void emitTreeAttrs(
 
     assert(tree.info.narHash);
     mkString(*state.allocAttr(v, state.symbols.create("narHash")),
-        tree.info.narHash.to_string(SRI));
+        tree.info.narHash.to_string(SRI, true));
 
     if (input->getRev()) {
         mkString(*state.allocAttr(v, state.symbols.create("rev")), input->getRev()->gitRev());
@@ -66,7 +66,10 @@ static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, V
         }
 
         if (!attrs.count("type"))
-            throw Error("attribute 'type' is missing in call to 'fetchTree', at %s", pos);
+            throw Error({
+                .hint = hintfmt("attribute 'type' is missing in call to 'fetchTree'"),
+                .errPos = pos
+            });
 
         input = fetchers::inputFromAttrs(attrs);
     } else
@@ -103,17 +106,21 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
             if (n == "url")
                 url = state.forceStringNoCtx(*attr.value, *attr.pos);
             else if (n == "sha256")
-                expectedHash = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256);
+                expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256);
             else if (n == "name")
                 name = state.forceStringNoCtx(*attr.value, *attr.pos);
             else
-                throw EvalError("unsupported argument '%s' to '%s', at %s",
-                    attr.name, who, *attr.pos);
-        }
+                throw EvalError({
+                    .hint = hintfmt("unsupported argument '%s' to '%s'", attr.name, who),
+                    .errPos = *attr.pos
+                });
+            }
 
         if (!url)
-            throw EvalError("'url' argument required, at %s", pos);
-
+            throw EvalError({
+                .hint = hintfmt("'url' argument required"),
+                .errPos = pos
+            });
     } else
         url = state.forceStringNoCtx(*args[0], pos);
 
@@ -140,7 +147,7 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
             : hashFile(htSHA256, path);
         if (hash != *expectedHash)
             throw Error((unsigned int) 102, "hash mismatch in file downloaded from '%s':\n  wanted: %s\n  got:    %s",
-                *url, expectedHash->to_string(), hash.to_string());
+                *url, expectedHash->to_string(Base32, true), hash.to_string(Base32, true));
     }
 
     if (state.allowedPaths)

src/libexpr/primops/fromTOML.cc

@@ -81,7 +81,10 @@ static void prim_fromTOML(EvalState & state, const Pos & pos, Value * * args, Va
     try {
         visit(v, parser(tomlStream).parse());
     } catch (std::runtime_error & e) {
-        throw EvalError("while parsing a TOML string at %s: %s", pos, e.what());
+        throw EvalError({
+            .hint = hintfmt("while parsing a TOML string: %s", e.what()),
+            .errPos = pos
+        });
     }
 }
 

src/libexpr/value-to-json.cc

@@ -79,7 +79,7 @@ void printValueAsJSON(EvalState & state, bool strict,
             break;
 
         default:
-            throw TypeError(format("cannot convert %1% to JSON") % showType(v));
+            throw TypeError("cannot convert %1% to JSON", showType(v));
     }
 }
 
@@ -93,7 +93,7 @@ void printValueAsJSON(EvalState & state, bool strict,
 void ExternalValueBase::printValueAsJSON(EvalState & state, bool strict,
     JSONPlaceholder & out, PathSet & context) const
 {
-    throw TypeError(format("cannot convert %1% to JSON") % showType());
+    throw TypeError("cannot convert %1% to JSON", showType());
 }
 
 

src/libfetchers/cache.hh

@@ -6,6 +6,8 @@ namespace nix::fetchers {
 
 struct Cache
 {
+    virtual ~Cache() { }
+
     virtual void add(
         ref<Store> store,
         const Attrs & inAttrs,

src/libfetchers/fetchers.cc

@@ -36,7 +36,7 @@ std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs)
         if (res) {
             if (auto narHash = maybeGetStrAttr(attrs, "narHash"))
                 // FIXME: require SRI hash.
-                res->narHash = Hash(*narHash);
+                res->narHash = newHashAllowEmpty(*narHash, {});
             return res;
         }
     }
@@ -47,7 +47,7 @@ Attrs Input::toAttrs() const
 {
     auto attrs = toAttrsInternal();
     if (narHash)
-        attrs.emplace("narHash", narHash->to_string(SRI));
+        attrs.emplace("narHash", narHash->to_string(SRI, true));
     attrs.emplace("type", type());
     return attrs;
 }
@@ -67,7 +67,7 @@ std::pair<Tree, std::shared_ptr<const Input>> Input::fetchTree(ref<Store> store)
 
     if (narHash && narHash != input->narHash)
         throw Error("NAR hash mismatch in input '%s' (%s), expected '%s', got '%s'",
-            to_string(), tree.actualPath, narHash->to_string(SRI), input->narHash->to_string(SRI));
+            to_string(), tree.actualPath, narHash->to_string(SRI, true), input->narHash->to_string(SRI, true));
 
     return {std::move(tree), input};
 }

src/libfetchers/git.cc

@@ -195,7 +195,7 @@ struct GitInput : Input
                     return files.count(file);
                 };
 
-                auto storePath = store->addToStore("source", actualUrl, true, htSHA256, filter);
+                auto storePath = store->addToStore("source", actualUrl, FileIngestionMethod::Recursive, htSHA256, filter);
 
                 auto tree = Tree {
                     .actualPath = store->printStorePath(storePath),
@@ -282,7 +282,10 @@ struct GitInput : Input
                 // FIXME: git stderr messes up our progress indicator, so
                 // we're using --quiet for now. Should process its stderr.
                 try {
-                    runProgram("git", true, { "-C", repoDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", *input->ref, *input->ref) });
+                    auto fetchRef = input->ref->compare(0, 5, "refs/") == 0
+                        ? *input->ref
+                        : "refs/heads/" + *input->ref;
+                    runProgram("git", true, { "-C", repoDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", fetchRef, fetchRef) });
                 } catch (Error & e) {
                     if (!pathExists(localRefFile)) throw;
                     warn("could not update local clone of Git repository '%s'; continuing with the most recent version", actualUrl);
@@ -347,7 +350,7 @@ struct GitInput : Input
             unpackTarfile(*source, tmpDir);
         }
 
-        auto storePath = store->addToStore(name, tmpDir, true, htSHA256, filter);
+        auto storePath = store->addToStore(name, tmpDir, FileIngestionMethod::Recursive, htSHA256, filter);
 
         auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "log", "-1", "--format=%ct", input->rev->gitRev() }));
 
@@ -418,7 +421,7 @@ struct GitInputScheme : InputScheme
 
         auto input = std::make_unique<GitInput>(parseURL(getStrAttr(attrs, "url")));
         if (auto ref = maybeGetStrAttr(attrs, "ref")) {
-            if (!std::regex_match(*ref, refRegex))
+            if (std::regex_search(*ref, badGitRefRegex))
                 throw BadURL("invalid Git branch/tag name '%s'", *ref);
             input->ref = *ref;
         }

src/libfetchers/github.cc

@@ -76,7 +76,7 @@ struct GitHubInput : Input
                 readFile(
                     store->toRealPath(
                         downloadFile(store, url, "source", false).storePath)));
-            rev = Hash(json["sha"], htSHA1);
+            rev = Hash(std::string { json["sha"] }, htSHA1);
             debug("HEAD revision for '%s' is %s", url, rev->gitRev());
         }
 

src/libfetchers/local.mk

@@ -8,4 +8,4 @@ libfetchers_SOURCES := $(wildcard $(d)/*.cc)
 
 libfetchers_CXXFLAGS += -I src/libutil -I src/libstore
 
-libfetchers_LIBS = libutil libstore libnixrust
+libfetchers_LIBS = libutil libstore

src/libfetchers/mercurial.cc

@@ -114,7 +114,7 @@ struct MercurialInput : Input
                     return files.count(file);
                 };
 
-                auto storePath = store->addToStore("source", actualUrl, true, htSHA256, filter);
+                auto storePath = store->addToStore("source", actualUrl, FileIngestionMethod::Recursive, htSHA256, filter);
 
                 return {Tree {
                     .actualPath = store->printStorePath(storePath),

src/libfetchers/tarball.cc

@@ -67,11 +67,15 @@ DownloadFileResult downloadFile(
         StringSink sink;
         dumpString(*res.data, sink);
         auto hash = hashString(htSHA256, *res.data);
-        ValidPathInfo info(store->makeFixedOutputPath(false, hash, name));
+        ValidPathInfo info(store->makeFixedOutputPath(FileIngestionMethod::Flat, hash, name));
         info.narHash = hashString(htSHA256, *sink.s);
         info.narSize = sink.s->size();
-        info.ca = makeFixedOutputCA(false, hash);
-        store->addToStore(info, sink.s, NoRepair, NoCheckSigs);
+        info.ca = FixedOutputHash {
+            .method = FileIngestionMethod::Flat,
+            .hash = hash,
+        };
+        auto source = StringSource { *sink.s };
+        store->addToStore(info, source, NoRepair, NoCheckSigs);
         storePath = std::move(info.path);
     }
 
@@ -141,7 +145,7 @@ Tree downloadTarball(
             throw nix::Error("tarball '%s' contains an unexpected number of top-level files", url);
         auto topDir = tmpDir + "/" + members.begin()->name;
         lastModified = lstat(topDir).st_mtime;
-        unpackedStorePath = store->addToStore(name, topDir, true, htSHA256, defaultPathFilter, NoRepair);
+        unpackedStorePath = store->addToStore(name, topDir, FileIngestionMethod::Recursive, htSHA256, defaultPathFilter, NoRepair);
     }
 
     Attrs infoAttrs({
@@ -195,9 +199,9 @@ struct TarballInput : Input
         // NAR hashes are preferred over file hashes since tar/zip files
         // don't have a canonical representation.
         if (narHash)
-            url2.query.insert_or_assign("narHash", narHash->to_string(SRI));
+            url2.query.insert_or_assign("narHash", narHash->to_string(SRI, true));
         else if (hash)
-            url2.query.insert_or_assign("hash", hash->to_string(SRI));
+            url2.query.insert_or_assign("hash", hash->to_string(SRI, true));
         return url2;
     }
 
@@ -206,7 +210,7 @@ struct TarballInput : Input
         Attrs attrs;
         attrs.emplace("url", url.to_string());
         if (hash)
-            attrs.emplace("hash", hash->to_string(SRI));
+            attrs.emplace("hash", hash->to_string(SRI, true));
         return attrs;
     }
 
@@ -263,8 +267,7 @@ struct TarballInputScheme : InputScheme
 
         auto input = std::make_unique<TarballInput>(parseURL(getStrAttr(attrs, "url")));
         if (auto hash = maybeGetStrAttr(attrs, "hash"))
-            // FIXME: require SRI hash.
-            input->hash = Hash(*hash);
+            input->hash = newHashAllowEmpty(*hash, {});
 
         return input;
     }

src/libfetchers/tree-info.cc

@@ -8,7 +8,7 @@ namespace nix::fetchers {
 StorePath TreeInfo::computeStorePath(Store & store) const
 {
     assert(narHash);
-    return store.makeFixedOutputPath(true, narHash, "source");
+    return store.makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, "source");
 }
 
 }

src/libmain/common-args.cc

@@ -1,5 +1,6 @@
 #include "common-args.hh"
 #include "globals.hh"
+#include "loggers.hh"
 
 namespace nix {
 
@@ -38,6 +39,14 @@ MixCommonArgs::MixCommonArgs(const string & programName)
         }},
     });
 
+    addFlag({
+        .longName = "log-format",
+        .description = "format of log output; \"raw\", \"internal-json\", \"bar\" "
+                        "or \"bar-with-logs\"",
+        .labels = {"format"},
+        .handler = {[](std::string format) { setLogFormat(format); }},
+    });
+
     addFlag({
         .longName = "max-jobs",
         .shortName = 'j',

src/libmain/loggers.cc

@@ -0,0 +1,52 @@
+#include "loggers.hh"
+#include "progress-bar.hh"
+
+namespace nix {
+
+LogFormat defaultLogFormat = LogFormat::raw;
+
+LogFormat parseLogFormat(const std::string & logFormatStr) {
+    if (logFormatStr == "raw")
+        return LogFormat::raw;
+    else if (logFormatStr == "raw-with-logs")
+        return LogFormat::rawWithLogs;
+    else if (logFormatStr == "internal-json")
+        return LogFormat::internalJson;
+    else if (logFormatStr == "bar")
+        return LogFormat::bar;
+    else if (logFormatStr == "bar-with-logs")
+        return LogFormat::barWithLogs;
+    throw Error("option 'log-format' has an invalid value '%s'", logFormatStr);
+}
+
+Logger * makeDefaultLogger() {
+    switch (defaultLogFormat) {
+    case LogFormat::raw:
+        return makeSimpleLogger(false);
+    case LogFormat::rawWithLogs:
+        return makeSimpleLogger(true);
+    case LogFormat::internalJson:
+        return makeJSONLogger(*makeSimpleLogger(true));
+    case LogFormat::bar:
+        return makeProgressBar();
+    case LogFormat::barWithLogs:
+        return makeProgressBar(true);
+    default:
+        abort();
+    }
+}
+
+void setLogFormat(const std::string & logFormatStr) {
+    setLogFormat(parseLogFormat(logFormatStr));
+}
+
+void setLogFormat(const LogFormat & logFormat) {
+    defaultLogFormat = logFormat;
+    createDefaultLogger();
+}
+
+void createDefaultLogger() {
+    logger = makeDefaultLogger();
+}
+
+}

src/libmain/loggers.hh

@@ -0,0 +1,20 @@
+#pragma once
+
+#include "types.hh"
+
+namespace nix {
+
+enum class LogFormat {
+  raw,
+  rawWithLogs,
+  internalJson,
+  bar,
+  barWithLogs,
+};
+
+void setLogFormat(const std::string & logFormatStr);
+void setLogFormat(const LogFormat & logFormat);
+
+void createDefaultLogger();
+
+}

src/libmain/progress-bar.cc

@@ -106,25 +106,36 @@ public:
         updateThread.join();
     }
 
-    void stop()
+    void stop() override
     {
         auto state(state_.lock());
         if (!state->active) return;
         state->active = false;
-        std::string status = getStatus(*state);
         writeToStderr("\r\e[K");
-        if (status != "")
-            writeToStderr("[" + status + "]\n");
         updateCV.notify_one();
         quitCV.notify_one();
     }
 
+    bool isVerbose() override {
+        return printBuildLogs;
+    }
+
     void log(Verbosity lvl, const FormatOrString & fs) override
     {
         auto state(state_.lock());
         log(*state, lvl, fs.s);
     }
 
+    void logEI(const ErrorInfo &ei) override
+    {
+        auto state(state_.lock());
+
+        std::stringstream oss;
+        showErrorInfo(oss, ei, loggerSettings.showTrace.get());
+
+        log(*state, ei.level, oss.str());
+    }
+
     void log(State & state, Verbosity lvl, const std::string & s)
     {
         if (state.active) {
@@ -142,7 +153,7 @@ public:
     {
         auto state(state_.lock());
 
-        if (lvl <= verbosity && !s.empty())
+        if (lvl <= verbosity && !s.empty() && type != actBuildWaiting)
             log(*state, lvl, s + "...");
 
         state->activities.emplace_back(ActInfo());
@@ -154,7 +165,7 @@ public:
         state->activitiesByType[type].its.emplace(act, i);
 
         if (type == actBuild) {
-            auto name = storePathToName(getS(fields, 0));
+            std::string name(storePathToName(getS(fields, 0)));
             if (hasSuffix(name, ".drv"))
                 name = name.substr(0, name.size() - 4);
             i->s = fmt("building " ANSI_BOLD "%s" ANSI_NORMAL, name);
@@ -457,11 +468,17 @@ public:
     }
 };
 
+Logger * makeProgressBar(bool printBuildLogs)
+{
+    return new ProgressBar(
+        printBuildLogs,
+        isatty(STDERR_FILENO) && getEnv("TERM").value_or("dumb") != "dumb"
+    );
+}
+
 void startProgressBar(bool printBuildLogs)
 {
-    logger = new ProgressBar(
-        printBuildLogs,
-        isatty(STDERR_FILENO) && getEnv("TERM").value_or("dumb") != "dumb");
+    logger = makeProgressBar(printBuildLogs);
 }
 
 void stopProgressBar()

src/libmain/progress-bar.hh

@@ -4,6 +4,8 @@
 
 namespace nix {
 
+Logger * makeProgressBar(bool printBuildLogs = false);
+
 void startProgressBar(bool printBuildLogs = false);
 
 void stopProgressBar();

src/libmain/shared.cc

@@ -2,6 +2,7 @@
 #include "shared.hh"
 #include "store-api.hh"
 #include "util.hh"
+#include "loggers.hh"
 
 #include <algorithm>
 #include <cctype>
@@ -47,7 +48,10 @@ void printMissing(ref<Store> store, const StorePathSet & willBuild,
     unsigned long long downloadSize, unsigned long long narSize, Verbosity lvl)
 {
     if (!willBuild.empty()) {
-        printMsg(lvl, "these derivations will be built:");
+        if (willBuild.size() == 1)
+            printMsg(lvl, fmt("this derivation will be built:"));
+        else
+            printMsg(lvl, fmt("these %d derivations will be built:", willBuild.size()));
         auto sorted = store->topoSortPaths(willBuild);
         reverse(sorted.begin(), sorted.end());
         for (auto & i : sorted)
@@ -55,9 +59,18 @@ void printMissing(ref<Store> store, const StorePathSet & willBuild,
     }
 
     if (!willSubstitute.empty()) {
-        printMsg(lvl, fmt("these paths will be fetched (%.2f MiB download, %.2f MiB unpacked):",
-                downloadSize / (1024.0 * 1024.0),
-                narSize / (1024.0 * 1024.0)));
+        const float downloadSizeMiB = downloadSize / (1024.f * 1024.f);
+        const float narSizeMiB = narSize / (1024.f * 1024.f);
+        if (willSubstitute.size() == 1) {
+            printMsg(lvl, fmt("this path will be fetched (%.2f MiB download, %.2f MiB unpacked):",
+                downloadSizeMiB,
+                narSizeMiB));
+        } else {
+            printMsg(lvl, fmt("these %d paths will be fetched (%.2f MiB download, %.2f MiB unpacked):",
+                willSubstitute.size(),
+                downloadSizeMiB,
+                narSizeMiB));
+        }
         for (auto & i : willSubstitute)
             printMsg(lvl, fmt("  %s", store->printStorePath(i)));
     }
@@ -75,7 +88,7 @@ string getArg(const string & opt,
     Strings::iterator & i, const Strings::iterator & end)
 {
     ++i;
-    if (i == end) throw UsageError(format("'%1%' requires an argument") % opt);
+    if (i == end) throw UsageError("'%1%' requires an argument", opt);
     return *i;
 }
 
@@ -169,7 +182,7 @@ LegacyArgs::LegacyArgs(const std::string & programName,
         .longName = "no-build-output",
         .shortName = 'Q',
         .description = "do not show build output",
-        .handler = {&settings.verboseBuild, false},
+        .handler = {[&]() {setLogFormat(LogFormat::raw); }},
     });
 
     addFlag({
@@ -234,7 +247,7 @@ bool LegacyArgs::processArgs(const Strings & args, bool finish)
     Strings ss(args);
     auto pos = ss.begin();
     if (!parseArg(pos, ss.end()))
-        throw UsageError(format("unexpected argument '%1%'") % args.front());
+        throw UsageError("unexpected argument '%1%'", args.front());
     return true;
 }
 
@@ -281,7 +294,7 @@ void showManPage(const string & name)
     restoreSignals();
     setenv("MANPATH", settings.nixManDir.c_str(), 1);
     execlp("man", "man", name.c_str(), nullptr);
-    throw SysError(format("command 'man %1%' failed") % name.c_str());
+    throw SysError("command 'man %1%' failed", name.c_str());
 }
 
 
@@ -289,6 +302,8 @@ int handleExceptions(const string & programName, std::function<void()> fun)
 {
     ReceiveInterrupts receiveInterrupts; // FIXME: need better place for this
 
+    ErrorInfo::programName = baseNameOf(programName);
+
     string error = ANSI_RED "error:" ANSI_NORMAL " ";
     try {
         try {
@@ -304,13 +319,12 @@ int handleExceptions(const string & programName, std::function<void()> fun)
     } catch (Exit & e) {
         return e.status;
     } catch (UsageError & e) {
-        printError(
-            format(error + "%1%\nTry '%2% --help' for more information.")
-            % e.what() % programName);
+        logError(e.info());
+        printError("Try '%1% --help' for more information.", programName);
         return 1;
     } catch (BaseError & e) {
-        printError(format(error + "%1%%2%") % (settings.showTrace ? e.prefix() : "") % e.msg());
-        if (e.prefix() != "" && !settings.showTrace)
+        logError(e.info());
+        if (e.hasTrace() && !loggerSettings.showTrace.get())
             printError("(use '--show-trace' to show detailed location information)");
         return e.status;
     } catch (std::bad_alloc & e) {
@@ -346,7 +360,7 @@ RunPager::RunPager()
         execlp("pager", "pager", nullptr);
         execlp("less", "less", nullptr);
         execlp("more", "more", nullptr);
-        throw SysError(format("executing '%1%'") % pager);
+        throw SysError("executing '%1%'", pager);
     });
 
     pid.setKillSignal(SIGINT);

src/libmain/shared.hh

@@ -56,7 +56,7 @@ template<class N> N getIntArg(const string & opt,
     Strings::iterator & i, const Strings::iterator & end, bool allowUnit)
 {
     ++i;
-    if (i == end) throw UsageError(format("'%1%' requires an argument") % opt);
+    if (i == end) throw UsageError("'%1%' requires an argument", opt);
     string s = *i;
     N multiplier = 1;
     if (allowUnit && !s.empty()) {
@@ -66,13 +66,13 @@ template<class N> N getIntArg(const string & opt,
             else if (u == 'M') multiplier = 1ULL << 20;
             else if (u == 'G') multiplier = 1ULL << 30;
             else if (u == 'T') multiplier = 1ULL << 40;
-            else throw UsageError(format("invalid unit specifier '%1%'") % u);
+            else throw UsageError("invalid unit specifier '%1%'", u);
             s.resize(s.size() - 1);
         }
     }
     N n;
     if (!string2Int(s, n))
-        throw UsageError(format("'%1%' requires an integer argument") % opt);
+        throw UsageError("'%1%' requires an integer argument", opt);
     return n * multiplier;
 }
 

src/libmain/stack.cc

@@ -1,4 +1,4 @@
-#include "types.hh"
+#include "error.hh"
 
 #include <cstring>
 #include <cstddef>

src/libstore/binary-cache-store.cc

@@ -15,6 +15,7 @@
 #include <chrono>
 #include <future>
 #include <regex>
+#include <fstream>
 
 #include <nlohmann/json.hpp>
 
@@ -40,14 +41,14 @@ void BinaryCacheStore::init()
         upsertFile(cacheInfoFile, "StoreDir: " + storeDir + "\n", "text/x-nix-cache-info");
     } else {
         for (auto & line : tokenizeString<Strings>(*cacheInfo, "\n")) {
-            size_t colon = line.find(':');
-            if (colon == std::string::npos) continue;
+            size_t colon= line.find(':');
+            if (colon ==std::string::npos) continue;
             auto name = line.substr(0, colon);
             auto value = trim(line.substr(colon + 1, std::string::npos));
             if (name == "StoreDir") {
                 if (value != storeDir)
-                    throw Error(format("binary cache '%s' is for Nix stores with prefix '%s', not '%s'")
-                        % getUri() % value % storeDir);
+                    throw Error("binary cache '%s' is for Nix stores with prefix '%s', not '%s'",
+                        getUri(), value, storeDir);
             } else if (name == "WantMassQuery") {
                 wantMassQuery.setDefault(value == "1" ? "true" : "false");
             } else if (name == "Priority") {
@@ -57,6 +58,13 @@ void BinaryCacheStore::init()
     }
 }
 
+void BinaryCacheStore::upsertFile(const std::string & path,
+    std::string && data,
+    const std::string & mimeType)
+{
+    upsertFile(path, std::make_shared<std::stringstream>(std::move(data)), mimeType);
+}
+
 void BinaryCacheStore::getFile(const std::string & path,
     Callback<std::shared_ptr<std::string>> callback) noexcept
 {
@@ -93,7 +101,7 @@ std::shared_ptr<std::string> BinaryCacheStore::getFile(const std::string & path)
 
 std::string BinaryCacheStore::narInfoFileFor(const StorePath & storePath)
 {
-    return storePathToHash(printStorePath(storePath)) + ".narinfo";
+    return std::string(storePath.hashPart()) + ".narinfo";
 }
 
 void BinaryCacheStore::writeNarInfo(ref<NarInfo> narInfo)
@@ -102,7 +110,7 @@ void BinaryCacheStore::writeNarInfo(ref<NarInfo> narInfo)
 
     upsertFile(narInfoFile, narInfo->to_string(*this), "text/x-nix-narinfo");
 
-    auto hashPart = storePathToHash(printStorePath(narInfo->path));
+    std::string hashPart(narInfo->path.hashPart());
 
     {
         auto state_(state.lock());
@@ -113,10 +121,74 @@ void BinaryCacheStore::writeNarInfo(ref<NarInfo> narInfo)
         diskCache->upsertNarInfo(getUri(), hashPart, std::shared_ptr<NarInfo>(narInfo));
 }
 
-void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::string> & nar,
-    RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor)
+AutoCloseFD openFile(const Path & path)
 {
-    if (!repair && isValidPath(info.path)) return;
+    auto fd = open(path.c_str(), O_RDONLY | O_CLOEXEC);
+    if (!fd)
+        throw SysError("opening file '%1%'", path);
+    return fd;
+}
+
+struct FileSource : FdSource
+{
+    AutoCloseFD fd2;
+
+    FileSource(const Path & path)
+        : fd2(openFile(path))
+    {
+        fd = fd2.get();
+    }
+};
+
+void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource,
+    RepairFlag repair, CheckSigsFlag checkSigs)
+{
+    assert(info.narHash && info.narSize);
+
+    if (!repair && isValidPath(info.path)) {
+        // FIXME: copyNAR -> null sink
+        narSource.drain();
+        return;
+    }
+
+    auto [fdTemp, fnTemp] = createTempFile();
+
+    auto now1 = std::chrono::steady_clock::now();
+
+    /* Read the NAR simultaneously into a CompressionSink+FileSink (to
+       write the compressed NAR to disk), into a HashSink (to get the
+       NAR hash), and into a NarAccessor (to get the NAR listing). */
+    HashSink fileHashSink(htSHA256);
+    std::shared_ptr<FSAccessor> narAccessor;
+    {
+    FdSink fileSink(fdTemp.get());
+    TeeSink teeSink(fileSink, fileHashSink);
+    auto compressionSink = makeCompressionSink(compression, teeSink);
+    TeeSource teeSource(narSource, *compressionSink);
+    narAccessor = makeNarAccessor(teeSource);
+    compressionSink->finish();
+    }
+
+    auto now2 = std::chrono::steady_clock::now();
+
+    auto narInfo = make_ref<NarInfo>(info);
+    narInfo->narSize = info.narSize;
+    narInfo->narHash = info.narHash;
+    narInfo->compression = compression;
+    auto [fileHash, fileSize] = fileHashSink.finish();
+    narInfo->fileHash = fileHash;
+    narInfo->fileSize = fileSize;
+    narInfo->url = "nar/" + narInfo->fileHash.to_string(Base32, false) + ".nar"
+        + (compression == "xz" ? ".xz" :
+           compression == "bzip2" ? ".bz2" :
+           compression == "br" ? ".br" :
+           "");
+
+    auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
+    printMsg(lvlTalkative, "copying path '%1%' (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache",
+        printStorePath(narInfo->path), info.narSize,
+        ((1.0 - (double) fileSize / info.narSize) * 100.0),
+        duration);
 
     /* Verify that all references are valid. This may do some .narinfo
        reads, but typically they'll already be cached. */
@@ -129,23 +201,6 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
                 printStorePath(info.path), printStorePath(ref));
         }
 
-    assert(nar->compare(0, narMagic.size(), narMagic) == 0);
-
-    auto narInfo = make_ref<NarInfo>(info);
-
-    narInfo->narSize = nar->size();
-    narInfo->narHash = hashString(htSHA256, *nar);
-
-    if (info.narHash && info.narHash != narInfo->narHash)
-        throw Error("refusing to copy corrupted path '%1%' to binary cache", printStorePath(info.path));
-
-    auto accessor_ = std::dynamic_pointer_cast<RemoteFSAccessor>(accessor);
-
-    auto narAccessor = makeNarAccessor(nar);
-
-    if (accessor_)
-        accessor_->addToCache(printStorePath(info.path), *nar, narAccessor);
-
     /* Optionally write a JSON file containing a listing of the
        contents of the NAR. */
     if (writeNARListing) {
@@ -157,33 +212,13 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
 
             {
                 auto res = jsonRoot.placeholder("root");
-                listNar(res, narAccessor, "", true);
+                listNar(res, ref<FSAccessor>(narAccessor), "", true);
             }
         }
 
-        upsertFile(storePathToHash(printStorePath(info.path)) + ".ls", jsonOut.str(), "application/json");
+        upsertFile(std::string(info.path.to_string()) + ".ls", jsonOut.str(), "application/json");
     }
 
-    /* Compress the NAR. */
-    narInfo->compression = compression;
-    auto now1 = std::chrono::steady_clock::now();
-    auto narCompressed = compress(compression, *nar, parallelCompression);
-    auto now2 = std::chrono::steady_clock::now();
-    narInfo->fileHash = hashString(htSHA256, *narCompressed);
-    narInfo->fileSize = narCompressed->size();
-
-    auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
-    printMsg(lvlTalkative, "copying path '%1%' (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache",
-        printStorePath(narInfo->path), narInfo->narSize,
-        ((1.0 - (double) narCompressed->size() / nar->size()) * 100.0),
-        duration);
-
-    narInfo->url = "nar/" + narInfo->fileHash.to_string(Base32, false) + ".nar"
-        + (compression == "xz" ? ".xz" :
-           compression == "bzip2" ? ".bz2" :
-           compression == "br" ? ".br" :
-           "");
-
     /* Optionally maintain an index of DWARF debug info files
        consisting of JSON files named 'debuginfo/<build-id>' that
        specify the NAR file and member containing the debug info. */
@@ -244,12 +279,14 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
     /* Atomically write the NAR file. */
     if (repair || !fileExists(narInfo->url)) {
         stats.narWrite++;
-        upsertFile(narInfo->url, *narCompressed, "application/x-nix-nar");
+        upsertFile(narInfo->url,
+            std::make_shared<std::fstream>(fnTemp, std::ios_base::in),
+            "application/x-nix-nar");
     } else
         stats.narWriteAverted++;
 
-    stats.narWriteBytes += nar->size();
-    stats.narWriteCompressedBytes += narCompressed->size();
+    stats.narWriteBytes += info.narSize;
+    stats.narWriteCompressedBytes += fileSize;
     stats.narWriteCompressionTimeMs += duration;
 
     /* Atomically write the NAR info file.*/
@@ -284,7 +321,7 @@ void BinaryCacheStore::narFromPath(const StorePath & storePath, Sink & sink)
     try {
         getFile(info->url, *decompressor);
     } catch (NoSuchBinaryCacheFile & e) {
-        throw SubstituteGone(e.what());
+        throw SubstituteGone(e.info());
     }
 
     decompressor->finish();
@@ -327,7 +364,7 @@ void BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath,
 }
 
 StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath,
-    bool recursive, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
+    FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
 {
     // FIXME: some cut&paste from LocalStore::addToStore().
 
@@ -336,7 +373,7 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath
        small files. */
     StringSink sink;
     Hash h;
-    if (recursive) {
+    if (method == FileIngestionMethod::Recursive) {
         dumpPath(srcPath, sink, filter);
         h = hashString(hashAlgo, *sink.s);
     } else {
@@ -345,9 +382,10 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath
         h = hashString(hashAlgo, s);
     }
 
-    ValidPathInfo info(makeFixedOutputPath(recursive, h, name));
+    ValidPathInfo info(makeFixedOutputPath(method, h, name));
 
-    addToStore(info, sink.s, repair, CheckSigs, nullptr);
+    auto source = StringSource { *sink.s };
+    addToStore(info, source, repair, CheckSigs);
 
     return std::move(info.path);
 }
@@ -356,12 +394,13 @@ StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s
     const StorePathSet & references, RepairFlag repair)
 {
     ValidPathInfo info(computeStorePathForText(name, s, references));
-    info.references = cloneStorePathSet(references);
+    info.references = references;
 
     if (repair || !isValidPath(info.path)) {
         StringSink sink;
         dumpString(s, sink);
-        addToStore(info, sink.s, repair, CheckSigs, nullptr);
+        auto source = StringSource { *sink.s };
+        addToStore(info, source, repair, CheckSigs);
     }
 
     return std::move(info.path);
@@ -383,21 +422,19 @@ void BinaryCacheStore::addSignatures(const StorePath & storePath, const StringSe
 
     narInfo->sigs.insert(sigs.begin(), sigs.end());
 
-    auto narInfoFile = narInfoFileFor(narInfo->path);
-
     writeNarInfo(narInfo);
 }
 
 std::shared_ptr<std::string> BinaryCacheStore::getBuildLog(const StorePath & path)
 {
-    auto drvPath = path.clone();
+    auto drvPath = path;
 
     if (!path.isDerivation()) {
         try {
             auto info = queryPathInfo(path);
             // FIXME: add a "Log" field to .narinfo
             if (!info->deriver) return nullptr;
-            drvPath = info->deriver->clone();
+            drvPath = *info->deriver;
         } catch (InvalidPath &) {
             return nullptr;
         }

src/libstore/binary-cache-store.hh

@@ -36,9 +36,13 @@ public:
     virtual bool fileExists(const std::string & path) = 0;
 
     virtual void upsertFile(const std::string & path,
-        const std::string & data,
+        std::shared_ptr<std::basic_iostream<char>> istream,
         const std::string & mimeType) = 0;
 
+    void upsertFile(const std::string & path,
+        std::string && data,
+        const std::string & mimeType);
+
     /* Note: subclasses must implement at least one of the two
        following getFile() methods. */
 
@@ -74,12 +78,11 @@ public:
     std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override
     { unsupported("queryPathFromHashPart"); }
 
-    void addToStore(const ValidPathInfo & info, const ref<std::string> & nar,
-        RepairFlag repair, CheckSigsFlag checkSigs,
-        std::shared_ptr<FSAccessor> accessor) override;
+    void addToStore(const ValidPathInfo & info, Source & narSource,
+        RepairFlag repair, CheckSigsFlag checkSigs) override;
 
     StorePath addToStore(const string & name, const Path & srcPath,
-        bool recursive, HashType hashAlgo,
+        FileIngestionMethod method, HashType hashAlgo,
         PathFilter & filter, RepairFlag repair) override;
 
     StorePath addTextToStore(const string & name, const string & s,

src/libstore/builtins/buildenv.cc

@@ -22,7 +22,10 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir,
         srcFiles = readDirectory(srcDir);
     } catch (SysError & e) {
         if (e.errNo == ENOTDIR) {
-            printError("warning: not including '%s' in the user environment because it's not a directory", srcDir);
+            logWarning({
+                .name = "Create links - directory",
+                .hint = hintfmt("not including '%s' in the user environment because it's not a directory", srcDir)
+            });
             return;
         }
         throw;
@@ -41,7 +44,10 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir,
                 throw SysError("getting status of '%1%'", srcFile);
         } catch (SysError & e) {
             if (e.errNo == ENOENT || e.errNo == ENOTDIR) {
-                printError("warning: skipping dangling symlink '%s'", dstFile);
+                logWarning({
+                    .name = "Create links - skipping symlink",
+                    .hint = hintfmt("skipping dangling symlink '%s'", dstFile)
+                });
                 continue;
             }
             throw;
@@ -72,15 +78,15 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir,
                     if (!S_ISDIR(lstat(target).st_mode))
                         throw Error("collision between '%1%' and non-directory '%2%'", srcFile, target);
                     if (unlink(dstFile.c_str()) == -1)
-                        throw SysError(format("unlinking '%1%'") % dstFile);
+                        throw SysError("unlinking '%1%'", dstFile);
                     if (mkdir(dstFile.c_str(), 0755) == -1)
-                        throw SysError(format("creating directory '%1%'"));
+                        throw SysError("creating directory '%1%'", dstFile);
                     createLinks(state, target, dstFile, state.priorities[dstFile]);
                     createLinks(state, srcFile, dstFile, priority);
                     continue;
                 }
             } else if (errno != ENOENT)
-                throw SysError(format("getting status of '%1%'") % dstFile);
+                throw SysError("getting status of '%1%'", dstFile);
         }
 
         else {
@@ -99,11 +105,11 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir,
                     if (prevPriority < priority)
                         continue;
                     if (unlink(dstFile.c_str()) == -1)
-                        throw SysError(format("unlinking '%1%'") % dstFile);
+                        throw SysError("unlinking '%1%'", dstFile);
                 } else if (S_ISDIR(dstSt.st_mode))
                     throw Error("collision between non-directory '%1%' and directory '%2%'", srcFile, dstFile);
             } else if (errno != ENOENT)
-                throw SysError(format("getting status of '%1%'") % dstFile);
+                throw SysError("getting status of '%1%'", dstFile);
         }
 
         createSymlink(srcFile, dstFile);

src/libstore/builtins/fetchurl.cc

@@ -18,7 +18,7 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
 
     auto getAttr = [&](const string & name) {
         auto i = drv.env.find(name);
-        if (i == drv.env.end()) throw Error(format("attribute '%s' missing") % name);
+        if (i == drv.env.end()) throw Error("attribute '%s' missing", name);
         return i->second;
     };
 
@@ -54,18 +54,21 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
         auto executable = drv.env.find("executable");
         if (executable != drv.env.end() && executable->second == "1") {
             if (chmod(storePath.c_str(), 0755) == -1)
-                throw SysError(format("making '%1%' executable") % storePath);
+                throw SysError("making '%1%' executable", storePath);
         }
     };
 
+    /* We always have one output, and if it's a fixed-output derivation (as
+       checked below) it must be the only output */
+    auto & output = drv.outputs.begin()->second;
+
     /* Try the hashed mirrors first. */
-    if (getAttr("outputHashMode") == "flat")
+    if (output.hash && output.hash->method == FileIngestionMethod::Flat)
         for (auto hashedMirror : settings.hashedMirrors.get())
             try {
                 if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/';
-                auto ht = parseHashType(getAttr("outputHashAlgo"));
-                auto h = Hash(getAttr("outputHash"), ht);
-                fetch(hashedMirror + printHashType(h.type) + "/" + h.to_string(Base16, false));
+                auto & h = output.hash->hash;
+                fetch(hashedMirror + printHashType(*h.type) + "/" + h.to_string(Base16, false));
                 return;
             } catch (Error & e) {
                 debug(e.what());

src/libstore/content-address.cc

@@ -0,0 +1,97 @@
+#include "content-address.hh"
+
+namespace nix {
+
+std::string FixedOutputHash::printMethodAlgo() const {
+    return makeFileIngestionPrefix(method) + printHashType(*hash.type);
+}
+
+std::string makeFileIngestionPrefix(const FileIngestionMethod m) {
+    switch (m) {
+    case FileIngestionMethod::Flat:
+        return "";
+    case FileIngestionMethod::Recursive:
+        return "r:";
+    default:
+        throw Error("impossible, caught both cases");
+    }
+}
+
+std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash)
+{
+    return "fixed:"
+        + makeFileIngestionPrefix(method)
+        + hash.to_string(Base32, true);
+}
+
+// FIXME Put this somewhere?
+template<class... Ts> struct overloaded : Ts... { using Ts::operator()...; };
+template<class... Ts> overloaded(Ts...) -> overloaded<Ts...>;
+
+std::string renderContentAddress(ContentAddress ca) {
+    return std::visit(overloaded {
+        [](TextHash th) {
+            return "text:" + th.hash.to_string(Base32, true);
+        },
+        [](FixedOutputHash fsh) {
+            return makeFixedOutputCA(fsh.method, fsh.hash);
+        }
+    }, ca);
+}
+
+ContentAddress parseContentAddress(std::string_view rawCa) {
+    auto prefixSeparator = rawCa.find(':');
+    if (prefixSeparator != string::npos) {
+        auto prefix = string(rawCa, 0, prefixSeparator);
+        if (prefix == "text") {
+            auto hashTypeAndHash = rawCa.substr(prefixSeparator+1, string::npos);
+            Hash hash = Hash(string(hashTypeAndHash));
+            if (*hash.type != htSHA256) {
+                throw Error("parseContentAddress: the text hash should have type SHA256");
+            }
+            return TextHash { hash };
+        } else if (prefix == "fixed") {
+            // This has to be an inverse of makeFixedOutputCA
+            auto methodAndHash = rawCa.substr(prefixSeparator+1, string::npos);
+            if (methodAndHash.substr(0,2) == "r:") {
+                std::string_view hashRaw = methodAndHash.substr(2,string::npos);
+                return FixedOutputHash {
+                    .method = FileIngestionMethod::Recursive,
+                    .hash = Hash(string(hashRaw)),
+                };
+            } else {
+                std::string_view hashRaw = methodAndHash;
+                return FixedOutputHash {
+                    .method = FileIngestionMethod::Flat,
+                    .hash = Hash(string(hashRaw)),
+                };
+            }
+        } else {
+            throw Error("parseContentAddress: format not recognized; has to be text or fixed");
+        }
+    } else {
+        throw Error("Not a content address because it lacks an appropriate prefix");
+    }
+};
+
+std::optional<ContentAddress> parseContentAddressOpt(std::string_view rawCaOpt) {
+    return rawCaOpt == "" ? std::optional<ContentAddress> {} : parseContentAddress(rawCaOpt);
+};
+
+std::string renderContentAddress(std::optional<ContentAddress> ca) {
+    return ca ? renderContentAddress(*ca) : "";
+}
+
+Hash getContentAddressHash(const ContentAddress & ca)
+{
+    return std::visit(overloaded {
+        [](TextHash th) {
+            return th.hash;
+        },
+        [](FixedOutputHash fsh) {
+            return fsh.hash;
+        }
+    }, ca);
+}
+
+}

src/libstore/content-address.hh

@@ -0,0 +1,58 @@
+#pragma once
+
+#include <variant>
+#include "hash.hh"
+
+namespace nix {
+
+enum struct FileIngestionMethod : uint8_t {
+    Flat = false,
+    Recursive = true
+};
+
+struct TextHash {
+    Hash hash;
+};
+
+/// Pair of a hash, and how the file system was ingested
+struct FixedOutputHash {
+    FileIngestionMethod method;
+    Hash hash;
+    std::string printMethodAlgo() const;
+};
+
+/*
+  We've accumulated several types of content-addressed paths over the years;
+  fixed-output derivations support multiple hash algorithms and serialisation
+  methods (flat file vs NAR). Thus, ‘ca’ has one of the following forms:
+
+  * ‘text:sha256:<sha256 hash of file contents>’: For paths
+    computed by makeTextPath() / addTextToStore().
+
+  * ‘fixed:<r?>:<ht>:<h>’: For paths computed by
+    makeFixedOutputPath() / addToStore().
+*/
+typedef std::variant<
+    TextHash, // for paths computed by makeTextPath() / addTextToStore
+    FixedOutputHash // for path computed by makeFixedOutputPath
+> ContentAddress;
+
+/* Compute the prefix to the hash algorithm which indicates how the files were
+   ingested. */
+std::string makeFileIngestionPrefix(const FileIngestionMethod m);
+
+/* Compute the content-addressability assertion (ValidPathInfo::ca)
+   for paths created by makeFixedOutputPath() / addToStore(). */
+std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash);
+
+std::string renderContentAddress(ContentAddress ca);
+
+std::string renderContentAddress(std::optional<ContentAddress> ca);
+
+ContentAddress parseContentAddress(std::string_view rawCa);
+
+std::optional<ContentAddress> parseContentAddressOpt(std::string_view rawCaOpt);
+
+Hash getContentAddressHash(const ContentAddress & ca);
+
+}

src/libstore/daemon.cc

@@ -73,6 +73,18 @@ struct TunnelLogger : public Logger
         enqueueMsg(*buf.s);
     }
 
+    void logEI(const ErrorInfo & ei) override
+    {
+        if (ei.level > verbosity) return;
+
+        std::stringstream oss;
+        showErrorInfo(oss, ei, false);
+
+        StringSink buf;
+        buf << STDERR_NEXT << oss.str();
+        enqueueMsg(*buf.s);
+    }
+
     /* startWork() means that we're starting an operation for which we
       want to send out stderr to the client. */
     void startWork()
@@ -281,7 +293,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         auto path = store->parseStorePath(readString(from));
         logger->startWork();
         StorePathSet paths; // FIXME
-        paths.insert(path.clone());
+        paths.insert(path);
         auto res = store->querySubstitutablePaths(paths);
         logger->stopWork();
         to << (res.count(path) != 0);
@@ -315,7 +327,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         StorePathSet paths;
         if (op == wopQueryReferences)
             for (auto & i : store->queryPathInfo(path)->references)
-                paths.insert(i.clone());
+                paths.insert(i);
         else if (op == wopQueryReferrers)
             store->queryReferrers(path, paths);
         else if (op == wopQueryValidDerivers)
@@ -329,13 +341,21 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     case wopQueryDerivationOutputNames: {
         auto path = store->parseStorePath(readString(from));
         logger->startWork();
-        StringSet names;
-        names = store->queryDerivationOutputNames(path);
+        auto names = store->readDerivation(path).outputNames();
         logger->stopWork();
         to << names;
         break;
     }
 
+    case wopQueryDerivationOutputMap: {
+        auto path = store->parseStorePath(readString(from));
+        logger->startWork();
+        OutputPathMap outputs = store->queryDerivationOutputMap(path);
+        logger->stopWork();
+        writeOutputPathMap(*store, to, outputs);
+        break;
+    }
+
     case wopQueryDeriver: {
         auto path = store->parseStorePath(readString(from));
         logger->startWork();
@@ -355,32 +375,43 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     }
 
     case wopAddToStore: {
-        bool fixed, recursive;
         std::string s, baseName;
-        from >> baseName >> fixed /* obsolete */ >> recursive >> s;
-        /* Compatibility hack. */
-        if (!fixed) {
-            s = "sha256";
-            recursive = true;
+        FileIngestionMethod method;
+        {
+            bool fixed; uint8_t recursive;
+            from >> baseName >> fixed /* obsolete */ >> recursive >> s;
+            if (recursive > (uint8_t) FileIngestionMethod::Recursive)
+                throw Error("unsupported FileIngestionMethod with value of %i; you may need to upgrade nix-daemon", recursive);
+            method = FileIngestionMethod { recursive };
+            /* Compatibility hack. */
+            if (!fixed) {
+                s = "sha256";
+                method = FileIngestionMethod::Recursive;
+            }
         }
         HashType hashAlgo = parseHashType(s);
 
-        TeeSource savedNAR(from);
+        StringSink savedNAR;
+        TeeSource savedNARSource(from, savedNAR);
         RetrieveRegularNARSink savedRegular;
 
-        if (recursive) {
+        if (method == FileIngestionMethod::Recursive) {
             /* Get the entire NAR dump from the client and save it to
                a string so that we can pass it to
                addToStoreFromDump(). */
             ParseSink sink; /* null sink; just parse the NAR */
-            parseDump(sink, savedNAR);
+            parseDump(sink, savedNARSource);
         } else
             parseDump(savedRegular, from);
 
         logger->startWork();
         if (!savedRegular.regular) throw Error("regular file expected");
 
-        auto path = store->addToStoreFromDump(recursive ? *savedNAR.data : savedRegular.s, baseName, recursive, hashAlgo);
+        auto path = store->addToStoreFromDump(
+            method == FileIngestionMethod::Recursive ? *savedNAR.s : savedRegular.s,
+            baseName,
+            method,
+            hashAlgo);
         logger->stopWork();
 
         to << store->printStorePath(path);
@@ -412,7 +443,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     case wopImportPaths: {
         logger->startWork();
         TunnelSource source(from, to);
-        auto paths = store->importPaths(source, nullptr,
+        auto paths = store->importPaths(source,
             trusted ? NoCheckSigs : CheckSigs);
         logger->stopWork();
         Strings paths2;
@@ -572,9 +603,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         auto path = store->parseStorePath(readString(from));
         logger->startWork();
         SubstitutablePathInfos infos;
-        StorePathSet paths;
-        paths.insert(path.clone()); // FIXME
-        store->querySubstitutablePathInfos(paths, infos);
+        store->querySubstitutablePathInfos({path}, infos);
         logger->stopWork();
         auto i = infos.find(path);
         if (i == infos.end())
@@ -633,7 +662,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
             if (GET_PROTOCOL_MINOR(clientVersion) >= 16) {
                 to << info->ultimate
                    << info->sigs
-                   << info->ca;
+                   << renderContentAddress(info->ca);
             }
         } else {
             assert(GET_PROTOCOL_MINOR(clientVersion) >= 17);
@@ -691,7 +720,8 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         info.references = readStorePaths<StorePathSet>(*store, from);
         from >> info.registrationTime >> info.narSize >> info.ultimate;
         info.sigs = readStrings<StringSet>(from);
-        from >> info.ca >> repair >> dontCheckSigs;
+        info.ca = parseContentAddressOpt(readString(from));
+        from >> repair >> dontCheckSigs;
         if (!trusted && dontCheckSigs)
             dontCheckSigs = false;
         if (!trusted)
@@ -702,9 +732,9 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         if (GET_PROTOCOL_MINOR(clientVersion) >= 21)
             source = std::make_unique<TunnelSource>(from, to);
         else {
-            TeeSink tee(from);
+            TeeParseSink tee(from);
             parseDump(tee, tee.source);
-            saved = std::move(*tee.source.data);
+            saved = std::move(*tee.saved.s);
             source = std::make_unique<StringSource>(saved);
         }
 
@@ -712,7 +742,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
 
         // FIXME: race if addToStore doesn't read source?
         store->addToStore(info, *source, (RepairFlag) repair,
-            dontCheckSigs ? NoCheckSigs : CheckSigs, nullptr);
+            dontCheckSigs ? NoCheckSigs : CheckSigs);
 
         logger->stopWork();
         break;
@@ -735,7 +765,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     }
 
     default:
-        throw Error(format("invalid operation %1%") % op);
+        throw Error("invalid operation %1%", op);
     }
 }
 

src/libstore/derivations.cc

@@ -4,51 +4,9 @@
 #include "util.hh"
 #include "worker-protocol.hh"
 #include "fs-accessor.hh"
-#include "istringstream_nocopy.hh"
 
 namespace nix {
 
-
-void DerivationOutput::parseHashInfo(bool & recursive, Hash & hash) const
-{
-    recursive = false;
-    string algo = hashAlgo;
-
-    if (string(algo, 0, 2) == "r:") {
-        recursive = true;
-        algo = string(algo, 2);
-    }
-
-    HashType hashType = parseHashType(algo);
-    if (hashType == htUnknown)
-        throw Error("unknown hash algorithm '%s'", algo);
-
-    hash = Hash(this->hash, hashType);
-}
-
-
-BasicDerivation::BasicDerivation(const BasicDerivation & other)
-    : platform(other.platform)
-    , builder(other.builder)
-    , args(other.args)
-    , env(other.env)
-{
-    for (auto & i : other.outputs)
-        outputs.insert_or_assign(i.first,
-            DerivationOutput(i.second.path.clone(), std::string(i.second.hashAlgo), std::string(i.second.hash)));
-    for (auto & i : other.inputSrcs)
-        inputSrcs.insert(i.clone());
-}
-
-
-Derivation::Derivation(const Derivation & other)
-    : BasicDerivation(other)
-{
-    for (auto & i : other.inputDrvs)
-        inputDrvs.insert_or_assign(i.first.clone(), i.second);
-}
-
-
 const StorePath & BasicDerivation::findOutput(const string & id) const
 {
     auto i = outputs.find(id);
@@ -67,9 +25,9 @@ bool BasicDerivation::isBuiltin() const
 StorePath writeDerivation(ref<Store> store,
     const Derivation & drv, std::string_view name, RepairFlag repair)
 {
-    auto references = cloneStorePathSet(drv.inputSrcs);
+    auto references = drv.inputSrcs;
     for (auto & i : drv.inputDrvs)
-        references.insert(i.first.clone());
+        references.insert(i.first);
     /* Note that the outputs of a derivation are *not* references
        (that can be missing (of course) and should not necessarily be
        held during a garbage collection). */
@@ -87,7 +45,7 @@ static void expect(std::istream & str, const string & s)
     char s2[s.size()];
     str.read(s2, s.size());
     if (string(s2, s.size()) != s)
-        throw FormatError(format("expected string '%1%'") % s);
+        throw FormatError("expected string '%1%'", s);
 }
 
 
@@ -114,7 +72,7 @@ static Path parsePath(std::istream & str)
 {
     string s = parseString(str);
     if (s.size() == 0 || s[0] != '/')
-        throw FormatError(format("bad path '%1%' in derivation") % s);
+        throw FormatError("bad path '%1%' in derivation", s);
     return s;
 }
 
@@ -142,20 +100,45 @@ static StringSet parseStrings(std::istream & str, bool arePaths)
 }
 
 
-static Derivation parseDerivation(const Store & store, const string & s)
+static DerivationOutput parseDerivationOutput(const Store & store, std::istringstream & str)
+{
+    expect(str, ","); auto path = store.parseStorePath(parsePath(str));
+    expect(str, ","); auto hashAlgo = parseString(str);
+    expect(str, ","); const auto hash = parseString(str);
+    expect(str, ")");
+
+    std::optional<FixedOutputHash> fsh;
+    if (hashAlgo != "") {
+        auto method = FileIngestionMethod::Flat;
+        if (string(hashAlgo, 0, 2) == "r:") {
+            method = FileIngestionMethod::Recursive;
+            hashAlgo = string(hashAlgo, 2);
+        }
+        const HashType hashType = parseHashType(hashAlgo);
+        fsh = FixedOutputHash {
+            .method = std::move(method),
+            .hash = Hash(hash, hashType),
+        };
+    }
+
+    return DerivationOutput {
+        .path = std::move(path),
+        .hash = std::move(fsh),
+    };
+}
+
+
+static Derivation parseDerivation(const Store & store, std::string && s)
 {
     Derivation drv;
-    istringstream_nocopy str(s);
+    std::istringstream str(std::move(s));
     expect(str, "Derive([");
 
     /* Parse the list of outputs. */
     while (!endOfList(str)) {
         expect(str, "("); std::string id = parseString(str);
-        expect(str, ","); auto path = store.parseStorePath(parsePath(str));
-        expect(str, ","); auto hashAlgo = parseString(str);
-        expect(str, ","); auto hash = parseString(str);
-        expect(str, ")");
-        drv.outputs.emplace(id, DerivationOutput(std::move(path), std::move(hashAlgo), std::move(hash)));
+        auto output = parseDerivationOutput(store, str);
+        drv.outputs.emplace(std::move(id), std::move(output));
     }
 
     /* Parse the list of input derivations. */
@@ -196,7 +179,7 @@ Derivation readDerivation(const Store & store, const Path & drvPath)
     try {
         return parseDerivation(store, readFile(drvPath));
     } catch (FormatError & e) {
-        throw Error(format("error parsing derivation '%1%': %2%") % drvPath % e.msg());
+        throw Error("error parsing derivation '%1%': %2%", drvPath, e.msg());
     }
 }
 
@@ -204,6 +187,12 @@ Derivation readDerivation(const Store & store, const Path & drvPath)
 Derivation Store::derivationFromPath(const StorePath & drvPath)
 {
     ensurePath(drvPath);
+    return readDerivation(drvPath);
+}
+
+
+Derivation Store::readDerivation(const StorePath & drvPath)
+{
     auto accessor = getFSAccessor();
     try {
         return parseDerivation(*this, accessor->readFile(printStorePath(drvPath)));
@@ -275,8 +264,9 @@ string Derivation::unparse(const Store & store, bool maskOutputs,
         if (first) first = false; else s += ',';
         s += '('; printUnquotedString(s, i.first);
         s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(i.second.path));
-        s += ','; printUnquotedString(s, i.second.hashAlgo);
-        s += ','; printUnquotedString(s, i.second.hash);
+        s += ','; printUnquotedString(s, i.second.hash ? i.second.hash->printMethodAlgo() : "");
+        s += ','; printUnquotedString(s,
+            i.second.hash ? i.second.hash->hash.to_string(Base16, false) : "");
         s += ')';
     }
 
@@ -332,7 +322,7 @@ bool BasicDerivation::isFixedOutput() const
 {
     return outputs.size() == 1 &&
         outputs.begin()->first == "out" &&
-        outputs.begin()->second.hash != "";
+        outputs.begin()->second.hash;
 }
 
 
@@ -365,8 +355,8 @@ Hash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutput
     if (drv.isFixedOutput()) {
         DerivationOutputs::const_iterator i = drv.outputs.begin();
         return hashString(htSHA256, "fixed:out:"
-            + i->second.hashAlgo + ":"
-            + i->second.hash + ":"
+            + i->second.hash->printMethodAlgo() + ":"
+            + i->second.hash->hash.to_string(Base16, false) + ":"
             + store.printStorePath(i->second.path));
     }
 
@@ -377,8 +367,8 @@ Hash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutput
         auto h = drvHashes.find(i.first);
         if (h == drvHashes.end()) {
             assert(store.isValidPath(i.first));
-            h = drvHashes.insert_or_assign(i.first.clone(), hashDerivationModulo(store,
-                readDerivation(store, store.toRealPath(i.first)), false)).first;
+            h = drvHashes.insert_or_assign(i.first, hashDerivationModulo(store,
+                store.readDerivation(i.first), false)).first;
         }
         inputs2.insert_or_assign(h->second.to_string(Base16, false), i.second);
     }
@@ -405,10 +395,44 @@ StorePathSet BasicDerivation::outputPaths() const
 {
     StorePathSet paths;
     for (auto & i : outputs)
-        paths.insert(i.second.path.clone());
+        paths.insert(i.second.path);
     return paths;
 }
 
+static DerivationOutput readDerivationOutput(Source & in, const Store & store)
+{
+    auto path = store.parseStorePath(readString(in));
+    auto hashAlgo = readString(in);
+    auto hash = readString(in);
+
+    std::optional<FixedOutputHash> fsh;
+    if (hashAlgo != "") {
+        auto method = FileIngestionMethod::Flat;
+        if (string(hashAlgo, 0, 2) == "r:") {
+            method = FileIngestionMethod::Recursive;
+            hashAlgo = string(hashAlgo, 2);
+        }
+        auto hashType = parseHashType(hashAlgo);
+        fsh = FixedOutputHash {
+            .method = std::move(method),
+            .hash = Hash(hash, hashType),
+        };
+    }
+
+    return DerivationOutput {
+        .path = std::move(path),
+        .hash = std::move(fsh),
+    };
+}
+
+StringSet BasicDerivation::outputNames() const
+{
+    StringSet names;
+    for (auto & i : outputs)
+        names.insert(i.first);
+    return names;
+}
+
 
 Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv)
 {
@@ -416,10 +440,8 @@ Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv)
     auto nr = readNum<size_t>(in);
     for (size_t n = 0; n < nr; n++) {
         auto name = readString(in);
-        auto path = store.parseStorePath(readString(in));
-        auto hashAlgo = readString(in);
-        auto hash = readString(in);
-        drv.outputs.emplace(name, DerivationOutput(std::move(path), std::move(hashAlgo), std::move(hash)));
+        auto output = readDerivationOutput(in, store);
+        drv.outputs.emplace(std::move(name), std::move(output));
     }
 
     drv.inputSrcs = readStorePaths<StorePathSet>(store, in);
@@ -440,8 +462,16 @@ Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv)
 void writeDerivation(Sink & out, const Store & store, const BasicDerivation & drv)
 {
     out << drv.outputs.size();
-    for (auto & i : drv.outputs)
-        out << i.first << store.printStorePath(i.second.path) << i.second.hashAlgo << i.second.hash;
+    for (auto & i : drv.outputs) {
+        out << i.first
+            << store.printStorePath(i.second.path);
+        if (i.second.hash) {
+            out << i.second.hash->printMethodAlgo()
+                << i.second.hash->hash.to_string(Base16, false);
+        } else {
+            out << "" << "";
+        }
+    }
     writeStorePaths(store, out, drv.inputSrcs);
     out << drv.platform << drv.builder << drv.args;
     out << drv.env.size();

src/libstore/derivations.hh

@@ -1,8 +1,9 @@
 #pragma once
 
+#include "path.hh"
 #include "types.hh"
 #include "hash.hh"
-#include "store-api.hh"
+#include "content-address.hh"
 
 #include <map>
 
@@ -15,14 +16,7 @@ namespace nix {
 struct DerivationOutput
 {
     StorePath path;
-    std::string hashAlgo; /* hash used for expected hash computation */
-    std::string hash; /* expected hash, may be null */
-    DerivationOutput(StorePath && path, std::string && hashAlgo, std::string && hash)
-        : path(std::move(path))
-        , hashAlgo(std::move(hashAlgo))
-        , hash(std::move(hash))
-    { }
-    void parseHashInfo(bool & recursive, Hash & hash) const;
+    std::optional<FixedOutputHash> hash; /* hash used for expected hash computation */
 };
 
 typedef std::map<string, DerivationOutput> DerivationOutputs;
@@ -43,7 +37,6 @@ struct BasicDerivation
     StringPairs env;
 
     BasicDerivation() { }
-    explicit BasicDerivation(const BasicDerivation & other);
     virtual ~BasicDerivation() { };
 
     /* Return the path corresponding to the output identifier `id' in
@@ -58,6 +51,8 @@ struct BasicDerivation
     /* Return the output paths of a derivation. */
     StorePathSet outputPaths() const;
 
+    /* Return the output names of a derivation. */
+    StringSet outputNames() const;
 };
 
 struct Derivation : BasicDerivation
@@ -69,13 +64,12 @@ struct Derivation : BasicDerivation
         std::map<std::string, StringSet> * actualInputs = nullptr) const;
 
     Derivation() { }
-    Derivation(Derivation && other) = default;
-    explicit Derivation(const Derivation & other);
 };
 
 
 class Store;
 
+enum RepairFlag : bool { NoRepair = false, Repair = true };
 
 /* Write a derivation to the Nix store, and return its path. */
 StorePath writeDerivation(ref<Store> store,

src/libstore/export-import.cc

@@ -1,3 +1,4 @@
+#include "serialise.hh"
 #include "store-api.hh"
 #include "archive.hh"
 #include "worker-protocol.hh"
@@ -6,24 +7,6 @@
 
 namespace nix {
 
-struct HashAndWriteSink : Sink
-{
-    Sink & writeSink;
-    HashSink hashSink;
-    HashAndWriteSink(Sink & writeSink) : writeSink(writeSink), hashSink(htSHA256)
-    {
-    }
-    virtual void operator () (const unsigned char * data, size_t len)
-    {
-        writeSink(data, len);
-        hashSink(data, len);
-    }
-    Hash currentHash()
-    {
-        return hashSink.currentHash().first;
-    }
-};
-
 void Store::exportPaths(const StorePathSet & paths, Sink & sink)
 {
     auto sorted = topoSortPaths(paths);
@@ -46,28 +29,29 @@ void Store::exportPath(const StorePath & path, Sink & sink)
 {
     auto info = queryPathInfo(path);
 
-    HashAndWriteSink hashAndWriteSink(sink);
+    HashSink hashSink(htSHA256);
+    TeeSink teeSink(sink, hashSink);
 
-    narFromPath(path, hashAndWriteSink);
+    narFromPath(path, teeSink);
 
     /* Refuse to export paths that have changed.  This prevents
        filesystem corruption from spreading to other machines.
        Don't complain if the stored hash is zero (unknown). */
-    Hash hash = hashAndWriteSink.currentHash();
-    if (hash != info->narHash && info->narHash != Hash(info->narHash.type))
+    Hash hash = hashSink.currentHash().first;
+    if (hash != info->narHash && info->narHash != Hash(*info->narHash.type))
         throw Error("hash of path '%s' has changed from '%s' to '%s'!",
-            printStorePath(path), info->narHash.to_string(), hash.to_string());
+            printStorePath(path), info->narHash.to_string(Base32, true), hash.to_string(Base32, true));
 
-    hashAndWriteSink
+    teeSink
         << exportMagic
         << printStorePath(path);
-    writeStorePaths(*this, hashAndWriteSink, info->references);
-    hashAndWriteSink
+    writeStorePaths(*this, teeSink, info->references);
+    teeSink
         << (info->deriver ? printStorePath(*info->deriver) : "")
         << 0;
 }
 
-StorePaths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> accessor, CheckSigsFlag checkSigs)
+StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs)
 {
     StorePaths res;
     while (true) {
@@ -76,7 +60,7 @@ StorePaths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> acces
         if (n != 1) throw Error("input doesn't look like something created by 'nix-store --export'");
 
         /* Extract the NAR from the source. */
-        TeeSink tee(source);
+        TeeParseSink tee(source);
         parseDump(tee, tee.source);
 
         uint32_t magic = readInt(source);
@@ -93,16 +77,18 @@ StorePaths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> acces
         if (deriver != "")
             info.deriver = parseStorePath(deriver);
 
-        info.narHash = hashString(htSHA256, *tee.source.data);
-        info.narSize = tee.source.data->size();
+        info.narHash = hashString(htSHA256, *tee.saved.s);
+        info.narSize = tee.saved.s->size();
 
         // Ignore optional legacy signature.
         if (readInt(source) == 1)
             readString(source);
 
-        addToStore(info, tee.source.data, NoRepair, checkSigs, accessor);
+        // Can't use underlying source, which would have been exhausted
+        auto source = StringSource { *tee.saved.s };
+        addToStore(info, source, NoRepair, checkSigs);
 
-        res.push_back(info.path.clone());
+        res.push_back(info.path);
     }
 
     return res;

src/libstore/filetransfer.cc

@@ -72,6 +72,18 @@ struct curlFileTransfer : public FileTransfer
 
         curl_off_t writtenToSink = 0;
 
+        inline static const std::set<long> successfulStatuses {200, 201, 204, 206, 304, 0 /* other protocol */};
+        /* Get the HTTP status code, or 0 for other protocols. */
+        long getHTTPStatus()
+        {
+            long httpStatus = 0;
+            long protocol = 0;
+            curl_easy_getinfo(req, CURLINFO_PROTOCOL, &protocol);
+            if (protocol == CURLPROTO_HTTP || protocol == CURLPROTO_HTTPS)
+                curl_easy_getinfo(req, CURLINFO_RESPONSE_CODE, &httpStatus);
+            return httpStatus;
+        }
+
         TransferItem(curlFileTransfer & fileTransfer,
             const FileTransferRequest & request,
             Callback<FileTransferResult> && callback)
@@ -83,12 +95,11 @@ struct curlFileTransfer : public FileTransfer
             , callback(std::move(callback))
             , finalSink([this](const unsigned char * data, size_t len) {
                 if (this->request.dataCallback) {
-                    long httpStatus = 0;
-                    curl_easy_getinfo(req, CURLINFO_RESPONSE_CODE, &httpStatus);
+                    auto httpStatus = getHTTPStatus();
 
                     /* Only write data to the sink if this is a
                        successful response. */
-                    if (httpStatus == 0 || httpStatus == 200 || httpStatus == 201 || httpStatus == 206) {
+                    if (successfulStatuses.count(httpStatus)) {
                         writtenToSink += len;
                         this->request.dataCallback((char *) data, len);
                     }
@@ -112,7 +123,7 @@ struct curlFileTransfer : public FileTransfer
             if (requestHeaders) curl_slist_free_all(requestHeaders);
             try {
                 if (!done)
-                    fail(FileTransferError(Interrupted, format("download of '%s' was interrupted") % request.uri));
+                    fail(FileTransferError(Interrupted, "download of '%s' was interrupted", request.uri));
             } catch (...) {
                 ignoreException();
             }
@@ -316,8 +327,7 @@ struct curlFileTransfer : public FileTransfer
 
         void finish(CURLcode code)
         {
-            long httpStatus = 0;
-            curl_easy_getinfo(req, CURLINFO_RESPONSE_CODE, &httpStatus);
+            auto httpStatus = getHTTPStatus();
 
             char * effectiveUriCStr;
             curl_easy_getinfo(req, CURLINFO_EFFECTIVE_URL, &effectiveUriCStr);
@@ -343,8 +353,7 @@ struct curlFileTransfer : public FileTransfer
             if (writeException)
                 failEx(writeException);
 
-            else if (code == CURLE_OK &&
-                (httpStatus == 200 || httpStatus == 201 || httpStatus == 204 || httpStatus == 206 || httpStatus == 304 || httpStatus == 226 /* FTP */ || httpStatus == 0 /* other protocol */))
+            else if (code == CURLE_OK && successfulStatuses.count(httpStatus))
             {
                 result.cached = httpStatus == 304;
                 act.progress(result.bodySize, result.bodySize);
@@ -517,7 +526,7 @@ struct curlFileTransfer : public FileTransfer
             int running;
             CURLMcode mc = curl_multi_perform(curlm, &running);
             if (mc != CURLM_OK)
-                throw nix::Error(format("unexpected error from curl_multi_perform(): %s") % curl_multi_strerror(mc));
+                throw nix::Error("unexpected error from curl_multi_perform(): %s", curl_multi_strerror(mc));
 
             /* Set the promises of any finished requests. */
             CURLMsg * msg;
@@ -547,7 +556,7 @@ struct curlFileTransfer : public FileTransfer
             vomit("download thread waiting for %d ms", sleepTimeMs);
             mc = curl_multi_wait(curlm, extraFDs, 1, sleepTimeMs, &numfds);
             if (mc != CURLM_OK)
-                throw nix::Error(format("unexpected error from curl_multi_wait(): %s") % curl_multi_strerror(mc));
+                throw nix::Error("unexpected error from curl_multi_wait(): %s", curl_multi_strerror(mc));
 
             nextWakeup = std::chrono::steady_clock::time_point();
 
@@ -599,7 +608,11 @@ struct curlFileTransfer : public FileTransfer
             workerThreadMain();
         } catch (nix::Interrupted & e) {
         } catch (std::exception & e) {
-            printError("unexpected error in download thread: %s", e.what());
+            logError({
+                .name = "File transfer",
+                .hint = hintfmt("unexpected error in download thread: %s",
+                                e.what())
+            });
         }
 
         {

src/libstore/filetransfer.hh

@@ -103,8 +103,9 @@ class FileTransferError : public Error
 {
 public:
     FileTransfer::Error error;
-    FileTransferError(FileTransfer::Error error, const FormatOrString & fs)
-        : Error(fs), error(error)
+    template<typename... Args>
+    FileTransferError(FileTransfer::Error error, const Args & ... args)
+        : Error(args...), error(error)
     { }
 };
 

src/libstore/gc.cc

@@ -38,10 +38,10 @@ AutoCloseFD LocalStore::openGCLock(LockType lockType)
 
     AutoCloseFD fdGCLock = open(fnGCLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600);
     if (!fdGCLock)
-        throw SysError(format("opening global GC lock '%1%'") % fnGCLock);
+        throw SysError("opening global GC lock '%1%'", fnGCLock);
 
     if (!lockFile(fdGCLock.get(), lockType, false)) {
-        printError(format("waiting for the big garbage collector lock..."));
+        printInfo("waiting for the big garbage collector lock...");
         lockFile(fdGCLock.get(), lockType, true);
     }
 
@@ -65,8 +65,8 @@ static void makeSymlink(const Path & link, const Path & target)
 
     /* Atomically replace the old one. */
     if (rename(tempLink.c_str(), link.c_str()) == -1)
-        throw SysError(format("cannot rename '%1%' to '%2%'")
-            % tempLink % link);
+        throw SysError("cannot rename '%1%' to '%2%'",
+            tempLink , link);
 }
 
 
@@ -91,15 +91,15 @@ Path LocalFSStore::addPermRoot(const StorePath & storePath,
     Path gcRoot(canonPath(_gcRoot));
 
     if (isInStore(gcRoot))
-        throw Error(format(
+        throw Error(
                 "creating a garbage collector root (%1%) in the Nix store is forbidden "
-                "(are you running nix-build inside the store?)") % gcRoot);
+                "(are you running nix-build inside the store?)", gcRoot);
 
     if (indirect) {
         /* Don't clobber the link if it already exists and doesn't
            point to the Nix store. */
         if (pathExists(gcRoot) && (!isLink(gcRoot) || !isInStore(readLink(gcRoot))))
-            throw Error(format("cannot create symlink '%1%'; already exists") % gcRoot);
+            throw Error("cannot create symlink '%1%'; already exists", gcRoot);
         makeSymlink(gcRoot, printStorePath(storePath));
         addIndirectRoot(gcRoot);
     }
@@ -109,10 +109,10 @@ Path LocalFSStore::addPermRoot(const StorePath & storePath,
             Path rootsDir = canonPath((format("%1%/%2%") % stateDir % gcRootsDir).str());
 
             if (string(gcRoot, 0, rootsDir.size() + 1) != rootsDir + "/")
-                throw Error(format(
+                throw Error(
                     "path '%1%' is not a valid garbage collector root; "
-                    "it's not in the directory '%2%'")
-                    % gcRoot % rootsDir);
+                    "it's not in the directory '%2%'",
+                    gcRoot, rootsDir);
         }
 
         if (baseNameOf(gcRoot) == std::string(storePath.to_string()))
@@ -128,11 +128,13 @@ Path LocalFSStore::addPermRoot(const StorePath & storePath,
        gcroots directory. */
     if (settings.checkRootReachability) {
         auto roots = findRoots(false);
-        if (roots[storePath.clone()].count(gcRoot) == 0)
-            printError(
-                "warning: '%1%' is not in a directory where the garbage collector looks for roots; "
-                "therefore, '%2%' might be removed by the garbage collector",
-                gcRoot, printStorePath(storePath));
+        if (roots[storePath].count(gcRoot) == 0)
+            logWarning({
+                .name = "GC root",
+                .hint = hintfmt("warning: '%1%' is not in a directory where the garbage collector looks for roots; "
+                    "therefore, '%2%' might be removed by the garbage collector",
+                    gcRoot, printStorePath(storePath))
+            });
     }
 
     /* Grab the global GC root, causing us to block while a GC is in
@@ -170,7 +172,7 @@ void LocalStore::addTempRoot(const StorePath & path)
                way. */
             struct stat st;
             if (fstat(state->fdTempRoots.get(), &st) == -1)
-                throw SysError(format("statting '%1%'") % fnTempRoots);
+                throw SysError("statting '%1%'", fnTempRoots);
             if (st.st_size == 0) break;
 
             /* The garbage collector deleted this file before we could
@@ -216,7 +218,7 @@ void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor)
         if (!*fd) {
             /* It's okay if the file has disappeared. */
             if (errno == ENOENT) continue;
-            throw SysError(format("opening temporary roots file '%1%'") % path);
+            throw SysError("opening temporary roots file '%1%'", path);
         }
 
         /* This should work, but doesn't, for some reason. */
@@ -227,7 +229,7 @@ void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor)
            only succeed if the owning process has died.  In that case
            we don't care about its temporary roots. */
         if (lockFile(fd->get(), ltWrite, false)) {
-            printError(format("removing stale temporary roots file '%1%'") % path);
+            printInfo("removing stale temporary roots file '%1%'", path);
             unlink(path.c_str());
             writeFull(fd->get(), "d");
             continue;
@@ -260,11 +262,13 @@ void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor)
 void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots)
 {
     auto foundRoot = [&](const Path & path, const Path & target) {
-        auto storePath = maybeParseStorePath(toStorePath(target));
-        if (storePath && isValidPath(*storePath))
-            roots[std::move(*storePath)].emplace(path);
-        else
-            printInfo("skipping invalid root from '%1%' to '%2%'", path, target);
+        try {
+            auto storePath = toStorePath(target).first;
+            if (isValidPath(storePath))
+                roots[std::move(storePath)].emplace(path);
+            else
+                printInfo("skipping invalid root from '%1%' to '%2%'", path, target);
+        } catch (BadStorePath &) { }
     };
 
     try {
@@ -403,7 +407,7 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
                 if (!fdDir) {
                     if (errno == ENOENT || errno == EACCES)
                         continue;
-                    throw SysError(format("opening %1%") % fdStr);
+                    throw SysError("opening %1%", fdStr);
                 }
                 struct dirent * fd_ent;
                 while (errno = 0, fd_ent = readdir(fdDir.get())) {
@@ -413,7 +417,7 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
                 if (errno) {
                     if (errno == ESRCH)
                         continue;
-                    throw SysError(format("iterating /proc/%1%/fd") % ent->d_name);
+                    throw SysError("iterating /proc/%1%/fd", ent->d_name);
                 }
                 fdDir.reset();
 
@@ -470,15 +474,15 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
 
     for (auto & [target, links] : unchecked) {
         if (!isInStore(target)) continue;
-        Path pathS = toStorePath(target);
-        if (!isStorePath(pathS)) continue;
-        auto path = parseStorePath(pathS);
-        if (!isValidPath(path)) continue;
-        debug("got additional root '%1%'", pathS);
-        if (censor)
-            roots[path.clone()].insert(censored);
-        else
-            roots[path.clone()].insert(links.begin(), links.end());
+        try {
+            auto path = toStorePath(target).first;
+            if (!isValidPath(path)) continue;
+            debug("got additional root '%1%'", printStorePath(path));
+            if (censor)
+                roots[path].insert(censored);
+            else
+                roots[path].insert(links.begin(), links.end());
+        } catch (BadStorePath &) { }
     }
 }
 
@@ -541,7 +545,7 @@ void LocalStore::deletePathRecursive(GCState & state, const Path & path)
     struct stat st;
     if (lstat(realPath.c_str(), &st)) {
         if (errno == ENOENT) return;
-        throw SysError(format("getting status of %1%") % realPath);
+        throw SysError("getting status of %1%", realPath);
     }
 
     printInfo(format("deleting '%1%'") % path);
@@ -559,10 +563,10 @@ void LocalStore::deletePathRecursive(GCState & state, const Path & path)
         // size.
         try {
             if (chmod(realPath.c_str(), st.st_mode | S_IWUSR) == -1)
-                throw SysError(format("making '%1%' writable") % realPath);
+                throw SysError("making '%1%' writable", realPath);
             Path tmp = trashDir + "/" + std::string(baseNameOf(path));
             if (rename(realPath.c_str(), tmp.c_str()))
-                throw SysError(format("unable to rename '%1%' to '%2%'") % realPath % tmp);
+                throw SysError("unable to rename '%1%' to '%2%'", realPath, tmp);
             state.bytesInvalidated += size;
         } catch (SysError & e) {
             if (e.errNo == ENOSPC) {
@@ -590,11 +594,11 @@ bool LocalStore::canReachRoot(GCState & state, StorePathSet & visited, const Sto
 
     if (state.roots.count(path)) {
         debug("cannot delete '%1%' because it's a root", printStorePath(path));
-        state.alive.insert(path.clone());
+        state.alive.insert(path);
         return true;
     }
 
-    visited.insert(path.clone());
+    visited.insert(path);
 
     if (!isValidPath(path)) return false;
 
@@ -608,7 +612,7 @@ bool LocalStore::canReachRoot(GCState & state, StorePathSet & visited, const Sto
     if (state.gcKeepDerivations && path.isDerivation()) {
         for (auto & i : queryDerivationOutputs(path))
             if (isValidPath(i) && queryPathInfo(i)->deriver == path)
-                incoming.insert(i.clone());
+                incoming.insert(i);
     }
 
     /* If keep-outputs is set, then don't delete this path if there
@@ -616,13 +620,13 @@ bool LocalStore::canReachRoot(GCState & state, StorePathSet & visited, const Sto
     if (state.gcKeepOutputs) {
         auto derivers = queryValidDerivers(path);
         for (auto & i : derivers)
-            incoming.insert(i.clone());
+            incoming.insert(i);
     }
 
     for (auto & i : incoming)
         if (i != path)
             if (canReachRoot(state, visited, i)) {
-                state.alive.insert(path.clone());
+                state.alive.insert(path);
                 return true;
             }
 
@@ -666,7 +670,7 @@ void LocalStore::tryToDelete(GCState & state, const Path & path)
            ‘nix-store --delete’ doesn't have the unexpected effect of
            recursing into derivations and outputs. */
         for (auto & i : visited)
-            state.dead.insert(i.clone());
+            state.dead.insert(i);
         if (state.shouldDelete)
             deletePathRecursive(state, path);
     }
@@ -681,7 +685,7 @@ void LocalStore::tryToDelete(GCState & state, const Path & path)
 void LocalStore::removeUnusedLinks(const GCState & state)
 {
     AutoCloseDir dir(opendir(linksDir.c_str()));
-    if (!dir) throw SysError(format("opening directory '%1%'") % linksDir);
+    if (!dir) throw SysError("opening directory '%1%'", linksDir);
 
     long long actualSize = 0, unsharedSize = 0;
 
@@ -694,7 +698,7 @@ void LocalStore::removeUnusedLinks(const GCState & state)
 
         struct stat st;
         if (lstat(path.c_str(), &st) == -1)
-            throw SysError(format("statting '%1%'") % path);
+            throw SysError("statting '%1%'", path);
 
         if (st.st_nlink != 1) {
             actualSize += st.st_size;
@@ -705,14 +709,14 @@ void LocalStore::removeUnusedLinks(const GCState & state)
         printMsg(lvlTalkative, format("deleting unused link '%1%'") % path);
 
         if (unlink(path.c_str()) == -1)
-            throw SysError(format("deleting '%1%'") % path);
+            throw SysError("deleting '%1%'", path);
 
         state.results.bytesFreed += st.st_size;
     }
 
     struct stat st;
     if (stat(linksDir.c_str(), &st) == -1)
-        throw SysError(format("statting '%1%'") % linksDir);
+        throw SysError("statting '%1%'", linksDir);
     long long overhead = st.st_blocks * 512ULL;
 
     printInfo(format("note: currently hard linking saves %.2f MiB")
@@ -747,12 +751,12 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
 
     /* Find the roots.  Since we've grabbed the GC lock, the set of
        permanent roots cannot increase now. */
-    printError("finding garbage collector roots...");
+    printInfo("finding garbage collector roots...");
     Roots rootMap;
     if (!options.ignoreLiveness)
         findRootsNoTemp(rootMap, true);
 
-    for (auto & i : rootMap) state.roots.insert(i.first.clone());
+    for (auto & i : rootMap) state.roots.insert(i.first);
 
     /* Read the temporary roots.  This acquires read locks on all
        per-process temporary root files.  So after this point no paths
@@ -761,8 +765,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
     Roots tempRoots;
     findTempRoots(fds, tempRoots, true);
     for (auto & root : tempRoots) {
-        state.tempRoots.insert(root.first.clone());
-        state.roots.insert(root.first.clone());
+        state.tempRoots.insert(root.first);
+        state.roots.insert(root.first);
     }
 
     /* After this point the set of roots or temporary roots cannot
@@ -799,14 +803,14 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
     } else if (options.maxFreed > 0) {
 
         if (state.shouldDelete)
-            printError("deleting garbage...");
+            printInfo("deleting garbage...");
         else
-            printError("determining live/dead paths...");
+            printInfo("determining live/dead paths...");
 
         try {
 
             AutoCloseDir dir(opendir(realStoreDir.c_str()));
-            if (!dir) throw SysError(format("opening directory '%1%'") % realStoreDir);
+            if (!dir) throw SysError("opening directory '%1%'", realStoreDir);
 
             /* Read the store and immediately delete all paths that
                aren't valid.  When using --max-freed etc., deleting
@@ -868,7 +872,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
 
     /* Clean up the links directory. */
     if (options.action == GCOptions::gcDeleteDead || options.action == GCOptions::gcDeleteSpecific) {
-        printError("deleting unused links...");
+        printInfo("deleting unused links...");
         removeUnusedLinks(state);
     }
 

src/libstore/globals.cc

@@ -35,7 +35,7 @@ Settings::Settings()
     , nixLibexecDir(canonPath(getEnv("NIX_LIBEXEC_DIR").value_or(NIX_LIBEXEC_DIR)))
     , nixBinDir(canonPath(getEnv("NIX_BIN_DIR").value_or(NIX_BIN_DIR)))
     , nixManDir(canonPath(NIX_MAN_DIR))
-    , nixDaemonSocketFile(canonPath(nixStateDir + DEFAULT_SOCKET_PATH))
+    , nixDaemonSocketFile(canonPath(getEnv("NIX_DAEMON_SOCKET_PATH").value_or(nixStateDir + DEFAULT_SOCKET_PATH)))
 {
     buildUsersGroup = getuid() == 0 ? "nixbld" : "";
     lockCPU = getEnv("NIX_AFFINITY_HACK") == "1";
@@ -130,7 +130,7 @@ bool Settings::isExperimentalFeatureEnabled(const std::string & name)
 void Settings::requireExperimentalFeature(const std::string & name)
 {
     if (!isExperimentalFeatureEnabled(name))
-        throw Error("experimental Nix feature '%s' is disabled", name);
+        throw Error("experimental Nix feature '%1%' is disabled; use '--experimental-features %1%' to override", name);
 }
 
 bool Settings::isWSL1()

src/libstore/globals.hh

@@ -196,10 +196,6 @@ public:
     /* Whether to lock the Nix client and worker to the same CPU. */
     bool lockCPU;
 
-    /* Whether to show a stack trace if Nix evaluation fails. */
-    Setting<bool> showTrace{this, false, "show-trace",
-        "Whether to show a stack trace on evaluation errors."};
-
     Setting<SandboxMode> sandboxMode{this,
         #if __linux__
           smEnabled
@@ -271,7 +267,7 @@ public:
         "listed in 'trusted-public-keys'."};
 
     Setting<StringSet> extraPlatforms{this,
-        std::string{SYSTEM} == "x86_64-linux" ? StringSet{"i686-linux"} : StringSet{},
+        std::string{SYSTEM} == "x86_64-linux" && !isWSL1() ? StringSet{"i686-linux"} : StringSet{},
         "extra-platforms",
         "Additional platforms that can be built on the local system. "
         "These may be supported natively (e.g. armv7 on some aarch64 CPUs "
@@ -369,6 +365,9 @@ public:
 
     Setting<bool> warnDirty{this, true, "warn-dirty",
         "Whether to warn about dirty Git/Mercurial trees."};
+
+    Setting<size_t> narBufferSize{this, 32 * 1024 * 1024, "nar-buffer-size",
+        "Maximum size of NARs before spilling them to disk."};
 };
 
 

src/libstore/http-binary-cache-store.cc

@@ -100,11 +100,11 @@ protected:
     }
 
     void upsertFile(const std::string & path,
-        const std::string & data,
+        std::shared_ptr<std::basic_iostream<char>> istream,
         const std::string & mimeType) override
     {
         auto req = FileTransferRequest(cacheUri + "/" + path);
-        req.data = std::make_shared<string>(data); // FIXME: inefficient
+        req.data = std::make_shared<string>(StreamToSourceAdapter(istream).drain());
         req.mimeType = mimeType;
         try {
             getFileTransfer()->upload(req);

src/libstore/legacy-ssh-store.cc

@@ -114,7 +114,7 @@ struct LegacySSHStore : public Store
             if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 4) {
                 auto s = readString(conn->from);
                 info->narHash = s.empty() ? Hash() : Hash(s);
-                conn->from >> info->ca;
+                info->ca = parseContentAddressOpt(readString(conn->from));
                 info->sigs = readStrings<StringSet>(conn->from);
             }
 
@@ -126,8 +126,7 @@ struct LegacySSHStore : public Store
     }
 
     void addToStore(const ValidPathInfo & info, Source & source,
-        RepairFlag repair, CheckSigsFlag checkSigs,
-        std::shared_ptr<FSAccessor> accessor) override
+        RepairFlag repair, CheckSigsFlag checkSigs) override
     {
         debug("adding path '%s' to remote host '%s'", printStorePath(info.path), host);
 
@@ -146,7 +145,7 @@ struct LegacySSHStore : public Store
                 << info.narSize
                 << info.ultimate
                 << info.sigs
-                << info.ca;
+                << renderContentAddress(info.ca);
             try {
                 copyNAR(source, conn->to);
             } catch (...) {
@@ -195,7 +194,7 @@ struct LegacySSHStore : public Store
     { unsupported("queryPathFromHashPart"); }
 
     StorePath addToStore(const string & name, const Path & srcPath,
-        bool recursive, HashType hashAlgo,
+        FileIngestionMethod method, HashType hashAlgo,
         PathFilter & filter, RepairFlag repair) override
     { unsupported("addToStore"); }
 
@@ -256,7 +255,7 @@ struct LegacySSHStore : public Store
         conn->to.flush();
 
         for (auto & i : readStorePaths<StorePathSet>(*this, conn->from))
-            out.insert(i.clone());
+            out.insert(i);
     }
 
     StorePathSet queryValidPaths(const StorePathSet & paths,

src/libstore/local-binary-cache-store.cc

@@ -31,8 +31,18 @@ protected:
     bool fileExists(const std::string & path) override;
 
     void upsertFile(const std::string & path,
-        const std::string & data,
-        const std::string & mimeType) override;
+        std::shared_ptr<std::basic_iostream<char>> istream,
+        const std::string & mimeType) override
+    {
+        auto path2 = binaryCacheDir + "/" + path;
+        Path tmp = path2 + ".tmp." + std::to_string(getpid());
+        AutoDelete del(tmp, false);
+        StreamToSourceAdapter source(istream);
+        writeFile(tmp, source);
+        if (rename(tmp.c_str(), path2.c_str()))
+            throw SysError("renaming '%1%' to '%2%'", tmp, path2);
+        del.cancel();
+    }
 
     void getFile(const std::string & path, Sink & sink) override
     {
@@ -52,7 +62,9 @@ protected:
             if (entry.name.size() != 40 ||
                 !hasSuffix(entry.name, ".narinfo"))
                 continue;
-            paths.insert(parseStorePath(storeDir + "/" + entry.name.substr(0, entry.name.size() - 8)));
+            paths.insert(parseStorePath(
+                    storeDir + "/" + entry.name.substr(0, entry.name.size() - 8)
+                    + "-" + MissingName));
         }
 
         return paths;
@@ -68,28 +80,11 @@ void LocalBinaryCacheStore::init()
     BinaryCacheStore::init();
 }
 
-static void atomicWrite(const Path & path, const std::string & s)
-{
-    Path tmp = path + ".tmp." + std::to_string(getpid());
-    AutoDelete del(tmp, false);
-    writeFile(tmp, s);
-    if (rename(tmp.c_str(), path.c_str()))
-        throw SysError(format("renaming '%1%' to '%2%'") % tmp % path);
-    del.cancel();
-}
-
 bool LocalBinaryCacheStore::fileExists(const std::string & path)
 {
     return pathExists(binaryCacheDir + "/" + path);
 }
 
-void LocalBinaryCacheStore::upsertFile(const std::string & path,
-    const std::string & data,
-    const std::string & mimeType)
-{
-    atomicWrite(binaryCacheDir + "/" + path, data);
-}
-
 static RegisterStoreImplementation regStore([](
     const std::string & uri, const Store::Params & params)
     -> std::shared_ptr<Store>

src/libstore/local-fs-store.cc

@@ -20,9 +20,9 @@ struct LocalStoreAccessor : public FSAccessor
 
     Path toRealPath(const Path & path)
     {
-        Path storePath = store->toStorePath(path);
-        if (!store->isValidPath(store->parseStorePath(storePath)))
-            throw InvalidPath(format("path '%1%' is not a valid store path") % storePath);
+        auto storePath = store->toStorePath(path).first;
+        if (!store->isValidPath(storePath))
+            throw InvalidPath("path '%1%' is not a valid store path", store->printStorePath(storePath));
         return store->getRealStoreDir() + std::string(path, store->storeDir.size());
     }
 
@@ -33,11 +33,11 @@ struct LocalStoreAccessor : public FSAccessor
         struct stat st;
         if (lstat(realPath.c_str(), &st)) {
             if (errno == ENOENT || errno == ENOTDIR) return {Type::tMissing, 0, false};
-            throw SysError(format("getting status of '%1%'") % path);
+            throw SysError("getting status of '%1%'", path);
         }
 
         if (!S_ISREG(st.st_mode) && !S_ISDIR(st.st_mode) && !S_ISLNK(st.st_mode))
-            throw Error(format("file '%1%' has unsupported type") % path);
+            throw Error("file '%1%' has unsupported type", path);
 
         return {
             S_ISREG(st.st_mode) ? Type::tRegular :
@@ -90,13 +90,13 @@ const string LocalFSStore::drvsLogDir = "drvs";
 
 std::shared_ptr<std::string> LocalFSStore::getBuildLog(const StorePath & path_)
 {
-    auto path = path_.clone();
+    auto path = path_;
 
     if (!path.isDerivation()) {
         try {
             auto info = queryPathInfo(path);
             if (!info->deriver) return nullptr;
-            path = info->deriver->clone();
+            path = *info->deriver;
         } catch (InvalidPath &) {
             return nullptr;
         }

src/libstore/local-store.cc

@@ -87,18 +87,22 @@ LocalStore::LocalStore(const Params & params)
 
         struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str());
         if (!gr)
-            printError(format("warning: the group '%1%' specified in 'build-users-group' does not exist")
-                % settings.buildUsersGroup);
+            logError({
+                .name = "'build-users-group' not found",
+                .hint = hintfmt(
+                    "warning: the group '%1%' specified in 'build-users-group' does not exist",
+                    settings.buildUsersGroup)
+            });
         else {
             struct stat st;
             if (stat(realStoreDir.c_str(), &st))
-                throw SysError(format("getting attributes of path '%1%'") % realStoreDir);
+                throw SysError("getting attributes of path '%1%'", realStoreDir);
 
             if (st.st_uid != 0 || st.st_gid != gr->gr_gid || (st.st_mode & ~S_IFMT) != perm) {
                 if (chown(realStoreDir.c_str(), 0, gr->gr_gid) == -1)
-                    throw SysError(format("changing ownership of path '%1%'") % realStoreDir);
+                    throw SysError("changing ownership of path '%1%'", realStoreDir);
                 if (chmod(realStoreDir.c_str(), perm) == -1)
-                    throw SysError(format("changing permissions on path '%1%'") % realStoreDir);
+                    throw SysError("changing permissions on path '%1%'", realStoreDir);
             }
         }
     }
@@ -109,12 +113,12 @@ LocalStore::LocalStore(const Params & params)
         struct stat st;
         while (path != "/") {
             if (lstat(path.c_str(), &st))
-                throw SysError(format("getting status of '%1%'") % path);
+                throw SysError("getting status of '%1%'", path);
             if (S_ISLNK(st.st_mode))
-                throw Error(format(
+                throw Error(
                         "the path '%1%' is a symlink; "
-                        "this is not allowed for the Nix store and its parent directories")
-                    % path);
+                        "this is not allowed for the Nix store and its parent directories",
+                        path);
             path = dirOf(path);
         }
     }
@@ -147,7 +151,7 @@ LocalStore::LocalStore(const Params & params)
     globalLock = openLockFile(globalLockPath.c_str(), true);
 
     if (!lockFile(globalLock.get(), ltRead, false)) {
-        printError("waiting for the big Nix store lock...");
+        printInfo("waiting for the big Nix store lock...");
         lockFile(globalLock.get(), ltRead, true);
     }
 
@@ -155,8 +159,8 @@ LocalStore::LocalStore(const Params & params)
        upgrade.  */
     int curSchema = getSchema();
     if (curSchema > nixSchemaVersion)
-        throw Error(format("current Nix store schema is version %1%, but I only support %2%")
-            % curSchema % nixSchemaVersion);
+        throw Error("current Nix store schema is version %1%, but I only support %2%",
+             curSchema, nixSchemaVersion);
 
     else if (curSchema == 0) { /* new store */
         curSchema = nixSchemaVersion;
@@ -178,7 +182,7 @@ LocalStore::LocalStore(const Params & params)
                 "please upgrade Nix to version 1.11 first.");
 
         if (!lockFile(globalLock.get(), ltWrite, false)) {
-            printError("waiting for exclusive access to the Nix store...");
+            printInfo("waiting for exclusive access to the Nix store...");
             lockFile(globalLock.get(), ltWrite, true);
         }
 
@@ -256,7 +260,7 @@ LocalStore::~LocalStore()
     }
 
     if (future.valid()) {
-        printError("waiting for auto-GC to finish on exit...");
+        printInfo("waiting for auto-GC to finish on exit...");
         future.get();
     }
 
@@ -284,7 +288,7 @@ int LocalStore::getSchema()
     if (pathExists(schemaPath)) {
         string s = readFile(schemaPath);
         if (!string2Int(s, curSchema))
-            throw Error(format("'%1%' is corrupt") % schemaPath);
+            throw Error("'%1%' is corrupt", schemaPath);
     }
     return curSchema;
 }
@@ -293,7 +297,7 @@ int LocalStore::getSchema()
 void LocalStore::openDB(State & state, bool create)
 {
     if (access(dbDir.c_str(), R_OK | W_OK))
-        throw SysError(format("Nix database directory '%1%' is not writable") % dbDir);
+        throw SysError("Nix database directory '%1%' is not writable", dbDir);
 
     /* Open the Nix database. */
     string dbPath = dbDir + "/db.sqlite";
@@ -367,7 +371,7 @@ void LocalStore::makeStoreWritable()
             throw SysError("setting up a private mount namespace");
 
         if (mount(0, realStoreDir.c_str(), "none", MS_REMOUNT | MS_BIND, 0) == -1)
-            throw SysError(format("remounting %1% writable") % realStoreDir);
+            throw SysError("remounting %1% writable", realStoreDir);
     }
 #endif
 }
@@ -388,7 +392,7 @@ static void canonicaliseTimestampAndPermissions(const Path & path, const struct
                  | 0444
                  | (st.st_mode & S_IXUSR ? 0111 : 0);
             if (chmod(path.c_str(), mode) == -1)
-                throw SysError(format("changing mode of '%1%' to %2$o") % path % mode);
+                throw SysError("changing mode of '%1%' to %2$o", path, mode);
         }
 
     }
@@ -406,7 +410,7 @@ static void canonicaliseTimestampAndPermissions(const Path & path, const struct
 #else
         if (!S_ISLNK(st.st_mode) && utimes(path.c_str(), times) == -1)
 #endif
-            throw SysError(format("changing modification time of '%1%'") % path);
+            throw SysError("changing modification time of '%1%'", path);
     }
 }
 
@@ -415,7 +419,7 @@ void canonicaliseTimestampAndPermissions(const Path & path)
 {
     struct stat st;
     if (lstat(path.c_str(), &st))
-        throw SysError(format("getting attributes of path '%1%'") % path);
+        throw SysError("getting attributes of path '%1%'", path);
     canonicaliseTimestampAndPermissions(path, st);
 }
 
@@ -430,17 +434,17 @@ static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSe
        setattrlist() to remove other attributes as well. */
     if (lchflags(path.c_str(), 0)) {
         if (errno != ENOTSUP)
-            throw SysError(format("clearing flags of path '%1%'") % path);
+            throw SysError("clearing flags of path '%1%'", path);
     }
 #endif
 
     struct stat st;
     if (lstat(path.c_str(), &st))
-        throw SysError(format("getting attributes of path '%1%'") % path);
+        throw SysError("getting attributes of path '%1%'", path);
 
     /* Really make sure that the path is of a supported type. */
     if (!(S_ISREG(st.st_mode) || S_ISDIR(st.st_mode) || S_ISLNK(st.st_mode)))
-        throw Error(format("file '%1%' has an unsupported type") % path);
+        throw Error("file '%1%' has an unsupported type", path);
 
 #if __linux__
     /* Remove extended attributes / ACLs. */
@@ -474,7 +478,7 @@ static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSe
     if (fromUid != (uid_t) -1 && st.st_uid != fromUid) {
         assert(!S_ISDIR(st.st_mode));
         if (inodesSeen.find(Inode(st.st_dev, st.st_ino)) == inodesSeen.end())
-            throw BuildError(format("invalid ownership on file '%1%'") % path);
+            throw BuildError("invalid ownership on file '%1%'", path);
         mode_t mode = st.st_mode & ~S_IFMT;
         assert(S_ISLNK(st.st_mode) || (st.st_uid == geteuid() && (mode == 0444 || mode == 0555) && st.st_mtime == mtimeStore));
         return;
@@ -498,8 +502,8 @@ static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSe
         if (!S_ISLNK(st.st_mode) &&
             chown(path.c_str(), geteuid(), getegid()) == -1)
 #endif
-            throw SysError(format("changing owner of '%1%' to %2%")
-                % path % geteuid());
+            throw SysError("changing owner of '%1%' to %2%",
+                path, geteuid());
     }
 
     if (S_ISDIR(st.st_mode)) {
@@ -518,11 +522,11 @@ void canonicalisePathMetaData(const Path & path, uid_t fromUid, InodesSeen & ino
        be a symlink, since we can't change its ownership. */
     struct stat st;
     if (lstat(path.c_str(), &st))
-        throw SysError(format("getting attributes of path '%1%'") % path);
+        throw SysError("getting attributes of path '%1%'", path);
 
     if (st.st_uid != geteuid()) {
         assert(S_ISLNK(st.st_mode));
-        throw Error(format("wrong ownership of top-level store path '%1%'") % path);
+        throw Error("wrong ownership of top-level store path '%1%'", path);
     }
 }
 
@@ -557,10 +561,12 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
         if (out == drv.outputs.end())
             throw Error("derivation '%s' does not have an output named 'out'", printStorePath(drvPath));
 
-        bool recursive; Hash h;
-        out->second.parseHashInfo(recursive, h);
-
-        check(makeFixedOutputPath(recursive, h, drvName), out->second.path, "out");
+        check(
+            makeFixedOutputPath(
+                out->second.hash->method,
+                out->second.hash->hash,
+                drvName),
+            out->second.path, "out");
     }
 
     else {
@@ -574,19 +580,19 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
 uint64_t LocalStore::addValidPath(State & state,
     const ValidPathInfo & info, bool checkOutputs)
 {
-    if (info.ca != "" && !info.isContentAddressed(*this))
+    if (info.ca.has_value() && !info.isContentAddressed(*this))
         throw Error("cannot add path '%s' to the Nix store because it claims to be content-addressed but isn't",
             printStorePath(info.path));
 
     state.stmtRegisterValidPath.use()
         (printStorePath(info.path))
-        (info.narHash.to_string(Base16))
+        (info.narHash.to_string(Base16, true))
         (info.registrationTime == 0 ? time(0) : info.registrationTime)
         (info.deriver ? printStorePath(*info.deriver) : "", (bool) info.deriver)
         (info.narSize, info.narSize != 0)
         (info.ultimate ? 1 : 0, info.ultimate)
         (concatStringsSep(" ", info.sigs), !info.sigs.empty())
-        (info.ca, !info.ca.empty())
+        (renderContentAddress(info.ca), (bool) info.ca)
         .exec();
     uint64_t id = sqlite3_last_insert_rowid(state.db);
 
@@ -595,7 +601,7 @@ uint64_t LocalStore::addValidPath(State & state,
        efficiently query whether a path is an output of some
        derivation. */
     if (info.path.isDerivation()) {
-        auto drv = readDerivation(*this, realStoreDir + "/" + std::string(info.path.to_string()));
+        auto drv = readDerivation(info.path);
 
         /* Verify that the output paths in the derivation are correct
            (i.e., follow the scheme for computing output paths from
@@ -615,7 +621,7 @@ uint64_t LocalStore::addValidPath(State & state,
 
     {
         auto state_(Store::state.lock());
-        state_->pathInfoCache.upsert(storePathToHash(printStorePath(info.path)),
+        state_->pathInfoCache.upsert(std::string(info.path.hashPart()),
             PathInfoCacheValue{ .value = std::make_shared<const ValidPathInfo>(info) });
     }
 
@@ -627,7 +633,7 @@ void LocalStore::queryPathInfoUncached(const StorePath & path,
     Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept
 {
     try {
-        auto info = std::make_shared<ValidPathInfo>(path.clone());
+        auto info = std::make_shared<ValidPathInfo>(path);
 
         callback(retrySQLite<std::shared_ptr<ValidPathInfo>>([&]() {
             auto state(_state.lock());
@@ -660,7 +666,7 @@ void LocalStore::queryPathInfoUncached(const StorePath & path,
             if (s) info->sigs = tokenizeString<StringSet>(s, " ");
 
             s = (const char *) sqlite3_column_text(state->stmtQueryPathInfo, 7);
-            if (s) info->ca = s;
+            if (s) info->ca = parseContentAddressOpt(s);
 
             /* Get the references. */
             auto useQueryReferences(state->stmtQueryReferences.use()(info->id));
@@ -680,10 +686,10 @@ void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info)
 {
     state.stmtUpdatePathInfo.use()
         (info.narSize, info.narSize != 0)
-        (info.narHash.to_string(Base16))
+        (info.narHash.to_string(Base16, true))
         (info.ultimate ? 1 : 0, info.ultimate)
         (concatStringsSep(" ", info.sigs), !info.sigs.empty())
-        (info.ca, !info.ca.empty())
+        (renderContentAddress(info.ca), (bool) info.ca)
         (printStorePath(info.path))
         .exec();
 }
@@ -717,7 +723,7 @@ StorePathSet LocalStore::queryValidPaths(const StorePathSet & paths, SubstituteF
 {
     StorePathSet res;
     for (auto & i : paths)
-        if (isValidPath(i)) res.insert(i.clone());
+        if (isValidPath(i)) res.insert(i);
     return res;
 }
 
@@ -768,43 +774,29 @@ StorePathSet LocalStore::queryValidDerivers(const StorePath & path)
 }
 
 
-StorePathSet LocalStore::queryDerivationOutputs(const StorePath & path)
+OutputPathMap LocalStore::queryDerivationOutputMap(const StorePath & path)
 {
-    return retrySQLite<StorePathSet>([&]() {
+    return retrySQLite<OutputPathMap>([&]() {
         auto state(_state.lock());
 
         auto useQueryDerivationOutputs(state->stmtQueryDerivationOutputs.use()
             (queryValidPathId(*state, path)));
 
-        StorePathSet outputs;
+        OutputPathMap outputs;
         while (useQueryDerivationOutputs.next())
-            outputs.insert(parseStorePath(useQueryDerivationOutputs.getStr(1)));
+            outputs.emplace(
+                useQueryDerivationOutputs.getStr(0),
+                parseStorePath(useQueryDerivationOutputs.getStr(1))
+            );
 
         return outputs;
     });
 }
 
 
-StringSet LocalStore::queryDerivationOutputNames(const StorePath & path)
-{
-    return retrySQLite<StringSet>([&]() {
-        auto state(_state.lock());
-
-        auto useQueryDerivationOutputs(state->stmtQueryDerivationOutputs.use()
-            (queryValidPathId(*state, path)));
-
-        StringSet outputNames;
-        while (useQueryDerivationOutputs.next())
-            outputNames.insert(useQueryDerivationOutputs.getStr(0));
-
-        return outputNames;
-    });
-}
-
-
 std::optional<StorePath> LocalStore::queryPathFromHashPart(const std::string & hashPart)
 {
-    if (hashPart.size() != storePathHashLen) throw Error("invalid hash part");
+    if (hashPart.size() != StorePath::HashLen) throw Error("invalid hash part");
 
     Path prefix = storeDir + "/" + hashPart;
 
@@ -829,7 +821,7 @@ StorePathSet LocalStore::querySubstitutablePaths(const StorePathSet & paths)
 
     StorePathSet remaining;
     for (auto & i : paths)
-        remaining.insert(i.clone());
+        remaining.insert(i);
 
     StorePathSet res;
 
@@ -843,9 +835,9 @@ StorePathSet LocalStore::querySubstitutablePaths(const StorePathSet & paths)
         StorePathSet remaining2;
         for (auto & path : remaining)
             if (valid.count(path))
-                res.insert(path.clone());
+                res.insert(path);
             else
-                remaining2.insert(path.clone());
+                remaining2.insert(path);
 
         std::swap(remaining, remaining2);
     }
@@ -867,16 +859,16 @@ void LocalStore::querySubstitutablePathInfos(const StorePathSet & paths,
                 auto info = sub->queryPathInfo(path);
                 auto narInfo = std::dynamic_pointer_cast<const NarInfo>(
                     std::shared_ptr<const ValidPathInfo>(info));
-                infos.insert_or_assign(path.clone(), SubstitutablePathInfo{
-                    info->deriver ? info->deriver->clone() : std::optional<StorePath>(),
-                    cloneStorePathSet(info->references),
+                infos.insert_or_assign(path, SubstitutablePathInfo{
+                    info->deriver,
+                    info->references,
                     narInfo ? narInfo->fileSize : 0,
                     info->narSize});
             } catch (InvalidPath &) {
             } catch (SubstituterDisabled &) {
             } catch (Error & e) {
                 if (settings.tryFallback)
-                    printError(e.what());
+                    logError(e.info());
                 else
                     throw;
             }
@@ -913,7 +905,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos)
                 updatePathInfo(*state, i);
             else
                 addValidPath(*state, i, false);
-            paths.insert(i.path.clone());
+            paths.insert(i.path);
         }
 
         for (auto & i : infos) {
@@ -928,8 +920,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos)
         for (auto & i : infos)
             if (i.path.isDerivation()) {
                 // FIXME: inefficient; we already loaded the derivation in addValidPath().
-                checkDerivationOutputs(i.path,
-                    readDerivation(*this, realStoreDir + "/" + std::string(i.path.to_string())));
+                checkDerivationOutputs(i.path, readDerivation(i.path));
             }
 
         /* Do a topological sort of the paths.  This will throw an
@@ -956,7 +947,7 @@ void LocalStore::invalidatePath(State & state, const StorePath & path)
 
     {
         auto state_(Store::state.lock());
-        state_->pathInfoCache.erase(storePathToHash(printStorePath(path)));
+        state_->pathInfoCache.erase(std::string(path.hashPart()));
     }
 }
 
@@ -971,7 +962,7 @@ const PublicKeys & LocalStore::getPublicKeys()
 
 
 void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
-    RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor)
+    RepairFlag repair, CheckSigsFlag checkSigs)
 {
     if (!info.narHash)
         throw Error("cannot add path '%s' because it lacks a hash", printStorePath(info.path));
@@ -985,7 +976,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
 
         PathLocks outputLock;
 
-        Path realPath = realStoreDir + "/" + std::string(info.path.to_string());
+        auto realPath = Store::toRealPath(info.path);
 
         /* Lock the output path.  But don't lock if we're being called
            from a build hook (whose parent process already acquired a
@@ -997,18 +988,18 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
 
             deletePath(realPath);
 
-            if (info.ca != "" &&
-                !((hasPrefix(info.ca, "text:") && !info.references.count(info.path))
-                    || info.references.empty()))
+            // text hashing has long been allowed to have non-self-references because it is used for drv files.
+            bool refersToSelf = info.references.count(info.path) > 0;
+            if (info.ca.has_value() && !info.references.empty() && !(std::holds_alternative<TextHash>(*info.ca) && !refersToSelf))
                 settings.requireExperimentalFeature("ca-references");
 
             /* While restoring the path from the NAR, compute the hash
                of the NAR. */
             std::unique_ptr<AbstractHashSink> hashSink;
-            if (info.ca == "" || !info.references.count(info.path))
+            if (!info.ca.has_value() || !info.references.count(info.path))
                 hashSink = std::make_unique<HashSink>(htSHA256);
             else
-                hashSink = std::make_unique<HashModuloSink>(htSHA256, storePathToHash(printStorePath(info.path)));
+                hashSink = std::make_unique<HashModuloSink>(htSHA256, std::string(info.path.hashPart()));
 
             LambdaSource wrapperSource([&](unsigned char * data, size_t len) -> size_t {
                 size_t n = source.read(data, len);
@@ -1022,7 +1013,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
 
             if (hashResult.first != info.narHash)
                 throw Error("hash mismatch importing path '%s';\n  wanted: %s\n  got:    %s",
-                    printStorePath(info.path), info.narHash.to_string(), hashResult.first.to_string());
+                    printStorePath(info.path), info.narHash.to_string(Base32, true), hashResult.first.to_string(Base32, true));
 
             if (hashResult.second != info.narSize)
                 throw Error("size mismatch importing path '%s';\n  wanted: %s\n  got:   %s",
@@ -1043,11 +1034,11 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
 
 
 StorePath LocalStore::addToStoreFromDump(const string & dump, const string & name,
-    bool recursive, HashType hashAlgo, RepairFlag repair)
+    FileIngestionMethod method, HashType hashAlgo, RepairFlag repair)
 {
     Hash h = hashString(hashAlgo, dump);
 
-    auto dstPath = makeFixedOutputPath(recursive, h, name);
+    auto dstPath = makeFixedOutputPath(method, h, name);
 
     addTempRoot(dstPath);
 
@@ -1056,8 +1047,7 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam
         /* The first check above is an optimisation to prevent
            unnecessary lock acquisition. */
 
-        Path realPath = realStoreDir + "/";
-        realPath += dstPath.to_string();
+        auto realPath = Store::toRealPath(dstPath);
 
         PathLocks outputLock({realPath});
 
@@ -1067,7 +1057,7 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam
 
             autoGC();
 
-            if (recursive) {
+            if (method == FileIngestionMethod::Recursive) {
                 StringSource source(dump);
                 restorePath(realPath, source);
             } else
@@ -1080,7 +1070,7 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam
                above (if called with recursive == true and hashAlgo ==
                sha256); otherwise, compute it here. */
             HashResult hash;
-            if (recursive) {
+            if (method == FileIngestionMethod::Recursive) {
                 hash.first = hashAlgo == htSHA256 ? h : hashString(htSHA256, dump);
                 hash.second = dump.size();
             } else
@@ -1088,10 +1078,10 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam
 
             optimisePath(realPath); // FIXME: combine with hashPath()
 
-            ValidPathInfo info(dstPath.clone());
+            ValidPathInfo info(dstPath);
             info.narHash = hash.first;
             info.narSize = hash.second;
-            info.ca = makeFixedOutputCA(recursive, h);
+            info.ca = FixedOutputHash { .method = method, .hash = h };
             registerValidPath(info);
         }
 
@@ -1103,20 +1093,123 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam
 
 
 StorePath LocalStore::addToStore(const string & name, const Path & _srcPath,
-    bool recursive, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
+    FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
 {
     Path srcPath(absPath(_srcPath));
 
-    /* Read the whole path into memory. This is not a very scalable
-       method for very large paths, but `copyPath' is mainly used for
-       small files. */
-    StringSink sink;
-    if (recursive)
-        dumpPath(srcPath, sink, filter);
-    else
-        sink.s = make_ref<std::string>(readFile(srcPath));
+    if (method != FileIngestionMethod::Recursive)
+        return addToStoreFromDump(readFile(srcPath), name, method, hashAlgo, repair);
 
-    return addToStoreFromDump(*sink.s, name, recursive, hashAlgo, repair);
+    /* For computing the NAR hash. */
+    auto sha256Sink = std::make_unique<HashSink>(htSHA256);
+
+    /* For computing the store path. In recursive SHA-256 mode, this
+       is the same as the NAR hash, so no need to do it again. */
+    std::unique_ptr<HashSink> hashSink =
+        hashAlgo == htSHA256
+        ? nullptr
+        : std::make_unique<HashSink>(hashAlgo);
+
+    /* Read the source path into memory, but only if it's up to
+       narBufferSize bytes. If it's larger, write it to a temporary
+       location in the Nix store. If the subsequently computed
+       destination store path is already valid, we just delete the
+       temporary path. Otherwise, we move it to the destination store
+       path. */
+    bool inMemory = true;
+    std::string nar;
+
+    auto source = sinkToSource([&](Sink & sink) {
+
+        LambdaSink sink2([&](const unsigned char * buf, size_t len) {
+            (*sha256Sink)(buf, len);
+            if (hashSink) (*hashSink)(buf, len);
+
+            if (inMemory) {
+                if (nar.size() + len > settings.narBufferSize) {
+                    inMemory = false;
+                    sink << 1;
+                    sink((const unsigned char *) nar.data(), nar.size());
+                    nar.clear();
+                } else {
+                    nar.append((const char *) buf, len);
+                }
+            }
+
+            if (!inMemory) sink(buf, len);
+        });
+
+        dumpPath(srcPath, sink2, filter);
+    });
+
+    std::unique_ptr<AutoDelete> delTempDir;
+    Path tempPath;
+
+    try {
+        /* Wait for the source coroutine to give us some dummy
+           data. This is so that we don't create the temporary
+           directory if the NAR fits in memory. */
+        readInt(*source);
+
+        auto tempDir = createTempDir(realStoreDir, "add");
+        delTempDir = std::make_unique<AutoDelete>(tempDir);
+        tempPath = tempDir + "/x";
+
+        restorePath(tempPath, *source);
+
+    } catch (EndOfFile &) {
+        if (!inMemory) throw;
+        /* The NAR fits in memory, so we didn't do restorePath(). */
+    }
+
+    auto sha256 = sha256Sink->finish();
+
+    Hash hash = hashSink ? hashSink->finish().first : sha256.first;
+
+    auto dstPath = makeFixedOutputPath(method, hash, name);
+
+    addTempRoot(dstPath);
+
+    if (repair || !isValidPath(dstPath)) {
+
+        /* The first check above is an optimisation to prevent
+           unnecessary lock acquisition. */
+
+        auto realPath = Store::toRealPath(dstPath);
+
+        PathLocks outputLock({realPath});
+
+        if (repair || !isValidPath(dstPath)) {
+
+            deletePath(realPath);
+
+            autoGC();
+
+            if (inMemory) {
+                /* Restore from the NAR in memory. */
+                StringSource source(nar);
+                restorePath(realPath, source);
+            } else {
+                /* Move the temporary path we restored above. */
+                if (rename(tempPath.c_str(), realPath.c_str()))
+                    throw Error("renaming '%s' to '%s'", tempPath, realPath);
+            }
+
+            canonicalisePathMetaData(realPath, -1); // FIXME: merge into restorePath
+
+            optimisePath(realPath);
+
+            ValidPathInfo info(dstPath);
+            info.narHash = sha256.first;
+            info.narSize = sha256.second;
+            info.ca = FixedOutputHash { .method = method, .hash = hash };
+            registerValidPath(info);
+        }
+
+        outputLock.setDeletion(true);
+    }
+
+    return dstPath;
 }
 
 
@@ -1130,8 +1223,7 @@ StorePath LocalStore::addTextToStore(const string & name, const string & s,
 
     if (repair || !isValidPath(dstPath)) {
 
-        Path realPath = realStoreDir + "/";
-        realPath += dstPath.to_string();
+        auto realPath = Store::toRealPath(dstPath);
 
         PathLocks outputLock({realPath});
 
@@ -1151,11 +1243,11 @@ StorePath LocalStore::addTextToStore(const string & name, const string & s,
 
             optimisePath(realPath);
 
-            ValidPathInfo info(dstPath.clone());
+            ValidPathInfo info(dstPath);
             info.narHash = narHash;
             info.narSize = sink.s->size();
-            info.references = cloneStorePathSet(references);
-            info.ca = "text:" + hash.to_string();
+            info.references = references;
+            info.ca = TextHash { .hash = hash };
             registerValidPath(info);
         }
 
@@ -1205,7 +1297,7 @@ void LocalStore::invalidatePathChecked(const StorePath & path)
 
 bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
 {
-    printError(format("reading the Nix store..."));
+    printInfo(format("reading the Nix store..."));
 
     bool errors = false;
 
@@ -1237,12 +1329,15 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
             Path linkPath = linksDir + "/" + link.name;
             string hash = hashPath(htSHA256, linkPath).first.to_string(Base32, false);
             if (hash != link.name) {
-                printError(
-                    "link '%s' was modified! expected hash '%s', got '%s'",
-                    linkPath, link.name, hash);
+                logError({
+                    .name = "Invalid hash",
+                    .hint = hintfmt(
+                        "link '%s' was modified! expected hash '%s', got '%s'",
+                        linkPath, link.name, hash)
+                });
                 if (repair) {
                     if (unlink(linkPath.c_str()) == 0)
-                        printError("removed link '%s'", linkPath);
+                        printInfo("removed link '%s'", linkPath);
                     else
                         throw SysError("removing corrupt link '%s'", linkPath);
                 } else {
@@ -1263,17 +1358,20 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
                 printMsg(lvlTalkative, "checking contents of '%s'", printStorePath(i));
 
                 std::unique_ptr<AbstractHashSink> hashSink;
-                if (info->ca == "" || !info->references.count(info->path))
-                    hashSink = std::make_unique<HashSink>(info->narHash.type);
+                if (!info->ca || !info->references.count(info->path))
+                    hashSink = std::make_unique<HashSink>(*info->narHash.type);
                 else
-                    hashSink = std::make_unique<HashModuloSink>(info->narHash.type, storePathToHash(printStorePath(info->path)));
+                    hashSink = std::make_unique<HashModuloSink>(*info->narHash.type, std::string(info->path.hashPart()));
 
                 dumpPath(Store::toRealPath(i), *hashSink);
                 auto current = hashSink->finish();
 
                 if (info->narHash != nullHash && info->narHash != current.first) {
-                    printError("path '%s' was modified! expected hash '%s', got '%s'",
-                        printStorePath(i), info->narHash.to_string(), current.first.to_string());
+                    logError({
+                        .name = "Invalid hash - path modified",
+                        .hint = hintfmt("path '%s' was modified! expected hash '%s', got '%s'",
+                        printStorePath(i), info->narHash.to_string(Base32, true), current.first.to_string(Base32, true))
+                    });
                     if (repair) repairPath(i); else errors = true;
                 } else {
 
@@ -1281,14 +1379,14 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
 
                     /* Fill in missing hashes. */
                     if (info->narHash == nullHash) {
-                        printError("fixing missing hash on '%s'", printStorePath(i));
+                        printInfo("fixing missing hash on '%s'", printStorePath(i));
                         info->narHash = current.first;
                         update = true;
                     }
 
                     /* Fill in missing narSize fields (from old stores). */
                     if (info->narSize == 0) {
-                        printError("updating size field on '%s' to %s", printStorePath(i), current.second);
+                        printInfo("updating size field on '%s' to %s", printStorePath(i), current.second);
                         info->narSize = current.second;
                         update = true;
                     }
@@ -1304,7 +1402,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
                 /* It's possible that the path got GC'ed, so ignore
                    errors on invalid paths. */
                 if (isValidPath(i))
-                    printError("error: %s", e.msg());
+                    logError(e.info());
                 else
                     warn(e.msg());
                 errors = true;
@@ -1324,7 +1422,10 @@ void LocalStore::verifyPath(const Path & pathS, const StringSet & store,
     if (!done.insert(pathS).second) return;
 
     if (!isStorePath(pathS)) {
-        printError("path '%s' is not in the Nix store", pathS);
+        logError({
+            .name = "Nix path not found",
+            .hint = hintfmt("path '%s' is not in the Nix store", pathS)
+        });
         return;
     }
 
@@ -1343,16 +1444,19 @@ void LocalStore::verifyPath(const Path & pathS, const StringSet & store,
             }
 
         if (canInvalidate) {
-            printError("path '%s' disappeared, removing from database...", pathS);
+            printInfo("path '%s' disappeared, removing from database...", pathS);
             auto state(_state.lock());
             invalidatePath(*state, path);
         } else {
-            printError("path '%s' disappeared, but it still has valid referrers!", pathS);
+            logError({
+                .name = "Missing path with referrers",
+                .hint = hintfmt("path '%s' disappeared, but it still has valid referrers!", pathS)
+            });
             if (repair)
                 try {
                     repairPath(path);
                 } catch (Error & e) {
-                    warn(e.msg());
+                    logWarning(e.info());
                     errors = true;
                 }
             else errors = true;
@@ -1392,7 +1496,7 @@ static void makeMutable(const Path & path)
     AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_NOFOLLOW | O_CLOEXEC);
     if (fd == -1) {
         if (errno == ELOOP) return; // it's a symlink
-        throw SysError(format("opening file '%1%'") % path);
+        throw SysError("opening file '%1%'", path);
     }
 
     unsigned int flags = 0, old;
@@ -1410,7 +1514,7 @@ static void makeMutable(const Path & path)
 void LocalStore::upgradeStore7()
 {
     if (getuid() != 0) return;
-    printError("removing immutable bits from the Nix store (this may take a while)...");
+    printInfo("removing immutable bits from the Nix store (this may take a while)...");
     makeMutable(realStoreDir);
 }