nix: Fix examples

For instance, 'nix why-depends --use-derivation nixpkgs#hello
nixpkgs#glibc' shows why hello's .drv depends on glibc's .drv.

.dir-locals.el

@@ -1,6 +1,7 @@
 ((c++-mode . (
   (c-file-style . "k&r")
   (c-basic-offset . 4)
+  (c-block-comment-prefix . "  ")
   (indent-tabs-mode . nil)
   (tab-width . 4)
   (show-trailing-whitespace . t)

.github/dependabot.yml

@@ -0,0 +1,6 @@
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"

.github/workflows/test.yml

@@ -6,12 +6,12 @@ jobs:
   tests:
     strategy:
       matrix:
-        os: [ubuntu-18.04, macos]
+        os: [ubuntu-latest, macos-latest]
     runs-on: ${{ matrix.os }}
     steps:
     - uses: actions/checkout@v2
       with:
         fetch-depth: 0
-    - uses: cachix/install-nix-action@v8
+    - uses: cachix/install-nix-action@v10
     #- run: nix flake check
     - run: nix-build -A checks.$(if [[ `uname` = Linux ]]; then echo x86_64-linux; else echo x86_64-darwin; fi)

.gitignore

@@ -49,6 +49,9 @@ perl/Makefile.config
 # /src/libstore/
 *.gen.*
 
+# /src/libutil/
+/src/libutil/tests/libutil-tests
+
 /src/nix/nix
 
 # /src/nix-env/

Makefile

@@ -1,8 +1,8 @@
 makefiles = \
   mk/precompiled-headers.mk \
   local.mk \
-  nix-rust/local.mk \
   src/libutil/local.mk \
+  src/libutil/tests/local.mk \
   src/libstore/local.mk \
   src/libfetchers/local.mk \
   src/libmain/local.mk \

Makefile.config.in

@@ -1,36 +1,38 @@
 AR = @AR@
 BDW_GC_LIBS = @BDW_GC_LIBS@
+BOOST_LDFLAGS = @BOOST_LDFLAGS@
 BUILD_SHARED_LIBS = @BUILD_SHARED_LIBS@
 CC = @CC@
 CFLAGS = @CFLAGS@
 CXX = @CXX@
 CXXFLAGS = @CXXFLAGS@
-LDFLAGS = @LDFLAGS@
+EDITLINE_LIBS = @EDITLINE_LIBS@
 ENABLE_S3 = @ENABLE_S3@
-HAVE_SODIUM = @HAVE_SODIUM@
+GTEST_LIBS = @GTEST_LIBS@
 HAVE_SECCOMP = @HAVE_SECCOMP@
-BOOST_LDFLAGS = @BOOST_LDFLAGS@
+HAVE_SODIUM = @HAVE_SODIUM@
+LDFLAGS = @LDFLAGS@
+LIBARCHIVE_LIBS = @LIBARCHIVE_LIBS@
+LIBBROTLI_LIBS = @LIBBROTLI_LIBS@
 LIBCURL_LIBS = @LIBCURL_LIBS@
+LIBLZMA_LIBS = @LIBLZMA_LIBS@
 OPENSSL_LIBS = @OPENSSL_LIBS@
 PACKAGE_NAME = @PACKAGE_NAME@
 PACKAGE_VERSION = @PACKAGE_VERSION@
 SODIUM_LIBS = @SODIUM_LIBS@
-LIBLZMA_LIBS = @LIBLZMA_LIBS@
 SQLITE3_LIBS = @SQLITE3_LIBS@
-LIBBROTLI_LIBS = @LIBBROTLI_LIBS@
-LIBARCHIVE_LIBS = @LIBARCHIVE_LIBS@
-EDITLINE_LIBS = @EDITLINE_LIBS@
 bash = @bash@
 bindir = @bindir@
-lsof = @lsof@
 datadir = @datadir@
 datarootdir = @datarootdir@
+doc_generate = @doc_generate@
 docdir = @docdir@
 exec_prefix = @exec_prefix@
 includedir = @includedir@
 libdir = @libdir@
 libexecdir = @libexecdir@
 localstatedir = @localstatedir@
+lsof = @lsof@
 mandir = @mandir@
 pkglibdir = $(libdir)/$(PACKAGE_NAME)
 prefix = @prefix@
@@ -38,6 +40,5 @@ sandbox_shell = @sandbox_shell@
 storedir = @storedir@
 sysconfdir = @sysconfdir@
 system = @system@
-doc_generate = @doc_generate@
 xmllint = @xmllint@
 xsltproc = @xsltproc@

configure.ac

@@ -267,6 +267,10 @@ if test "$gc" = yes; then
 fi
 
 
+# Look for gtest.
+PKG_CHECK_MODULES([GTEST], [gtest_main])
+
+
 # documentation generation switch
 AC_ARG_ENABLE(doc-gen, AC_HELP_STRING([--disable-doc-gen],
   [disable documentation generation]),

default.nix

@@ -1,3 +1,3 @@
 (import (fetchTarball https://github.com/edolstra/flake-compat/archive/master.tar.gz) {
-  src = builtins.fetchGit ./.;
+  src = ./.;
 }).defaultNix

doc/manual/advanced-topics/diff-hook.xml

@@ -70,7 +70,7 @@ path just built.</para>
 
   <screen>
 $ nix-build ./deterministic.nix -A stable
-these derivations will be built:
+this derivation will be built:
   /nix/store/z98fasz2jqy9gs0xbvdj939p27jwda38-stable.drv
 building '/nix/store/z98fasz2jqy9gs0xbvdj939p27jwda38-stable.drv'...
 /nix/store/yyxlzw3vqaas7wfp04g0b1xg51f2czgq-stable
@@ -85,7 +85,7 @@ checking outputs of '/nix/store/z98fasz2jqy9gs0xbvdj939p27jwda38-stable.drv'...
 
   <screen>
 $ nix-build ./deterministic.nix -A unstable
-these derivations will be built:
+this derivation will be built:
   /nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv
 building '/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv'...
 /nix/store/krpqk0l9ib0ibi1d2w52z293zw455cap-unstable
@@ -193,7 +193,7 @@ repeat = 1
     An example output of this configuration:
     <screen>
 $ nix-build ./test.nix -A unstable
-these derivations will be built:
+this derivation will be built:
   /nix/store/ch6llwpr2h8c3jmnf3f2ghkhx59aa97f-unstable.drv
 building '/nix/store/ch6llwpr2h8c3jmnf3f2ghkhx59aa97f-unstable.drv' (round 1/2)...
 building '/nix/store/ch6llwpr2h8c3jmnf3f2ghkhx59aa97f-unstable.drv' (round 2/2)...

doc/manual/advanced-topics/post-build-hook.xml

@@ -61,7 +61,7 @@ substituters = https://cache.nixos.org/ s3://example-nix-cache
 trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= example-nix-cache-1:1/cKDz3QCCOmwcztD2eV6Coggp6rqc9DGjWv7C0G+rM=
 </programlisting>
 
-<para>we will restart the Nix daemon a later step.</para>
+<para>We will restart the Nix daemon in a later step.</para>
 </section>
 
 <section>
@@ -122,7 +122,7 @@ post-build-hook = /etc/nix/upload-to-cache.sh
 
   <screen>
 $ nix-build -E '(import &lt;nixpkgs&gt; {}).writeText "example" (builtins.toString builtins.currentTime)'
-these derivations will be built:
+this derivation will be built:
   /nix/store/s4pnfbkalzy5qz57qs6yybna8wylkig6-example.drv
 building '/nix/store/s4pnfbkalzy5qz57qs6yybna8wylkig6-example.drv'...
 running post-build-hook '/home/grahamc/projects/github.com/NixOS/nix/post-hook.sh'...
@@ -139,7 +139,7 @@ $ nix-store --delete /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example
 
 <para>Now, copy the path back from the cache:</para>
 <screen>
-$ nix store --realize /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example
+$ nix-store --realise /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example
 copying path '/nix/store/m8bmqwrch6l3h8s0k3d673xpmipcdpsa-example from 's3://example-nix-cache'...
 warning: you did not specify '--add-root'; the result might be removed by the garbage collector
 /nix/store/m8bmqwrch6l3h8s0k3d673xpmipcdpsa-example

doc/manual/command-ref/conf-file.xml

@@ -386,7 +386,7 @@ false</literal>.</para>
 
 <programlisting>
 builtins.fetchurl {
-  url = https://example.org/foo-1.2.3.tar.xz;
+  url = "https://example.org/foo-1.2.3.tar.xz";
   sha256 = "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae";
 }
 </programlisting>

doc/manual/command-ref/env-common.xml

@@ -53,7 +53,7 @@ nixpkgs=/home/eelco/Dev/nixpkgs-branch:/etc/nixos</screen>
     <envar>NIX_PATH</envar> to
 
     <screen>
-nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-15.09.tar.gz</screen>
+nixpkgs=https://github.com/NixOS/nixpkgs/archive/nixos-15.09.tar.gz</screen>
 
     tells Nix to download the latest revision in the Nixpkgs/NixOS
     15.09 channel.</para>

doc/manual/command-ref/nix-env.xml

@@ -516,7 +516,7 @@ source:
 $ nix-env -f '<nixpkgs>' -iA hello --dry-run
 (dry run; not doing anything)
 installing ‘hello-2.10’
-these paths will be fetched (0.04 MiB download, 0.19 MiB unpacked):
+this path will be fetched (0.04 MiB download, 0.19 MiB unpacked):
   /nix/store/wkhdf9jinag5750mqlax6z2zbwhqb76n-hello-2.10
   <replaceable>...</replaceable></screen>
 
@@ -526,13 +526,10 @@ these paths will be fetched (0.04 MiB download, 0.19 MiB unpacked):
 14.12 channel:
 
 <screen>
-$ nix-env -f https://github.com/NixOS/nixpkgs-channels/archive/nixos-14.12.tar.gz -iA firefox
+$ nix-env -f https://github.com/NixOS/nixpkgs/archive/nixos-14.12.tar.gz -iA firefox
 </screen>
 
-(The GitHub repository <literal>nixpkgs-channels</literal> is updated
-automatically from the main <literal>nixpkgs</literal> repository
-after certain tests have succeeded and binaries have been built and
-uploaded to the binary cache at <uri>cache.nixos.org</uri>.)</para>
+</para>
 
 </refsection>
 

doc/manual/command-ref/nix-shell.xml

@@ -258,7 +258,7 @@ path. You can override it by passing <option>-I</option> or setting
 containing the Pan package from a specific revision of Nixpkgs:
 
 <screen>
-$ nix-shell -p pan -I nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/8a3eea054838b55aca962c3fbde9c83c102b8bf2.tar.gz
+$ nix-shell -p pan -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/8a3eea054838b55aca962c3fbde9c83c102b8bf2.tar.gz
 
 [nix-shell:~]$ pan --version
 Pan 0.139
@@ -352,7 +352,7 @@ following Haskell script uses a specific branch of Nixpkgs/NixOS (the
 <programlisting><![CDATA[
 #! /usr/bin/env nix-shell
 #! nix-shell -i runghc -p "haskellPackages.ghcWithPackages (ps: [ps.HTTP ps.tagsoup])"
-#! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-18.03.tar.gz
+#! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/nixos-18.03.tar.gz
 
 import Network.HTTP
 import Text.HTML.TagSoup
@@ -370,7 +370,7 @@ If you want to be even more precise, you can specify a specific
 revision of Nixpkgs:
 
 <programlisting>
-#! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/0672315759b3e15e2121365f067c1c8c56bb4722.tar.gz
+#! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/0672315759b3e15e2121365f067c1c8c56bb4722.tar.gz
 </programlisting>
 
 </para>

doc/manual/command-ref/nix-store.xml

@@ -936,7 +936,7 @@ $ nix-store --add ./foo.c
 
 <para>The operation <option>--add-fixed</option> adds the specified paths to
 the Nix store.  Unlike <option>--add</option> paths are registered using the
-specified hashing algorithm, resulting in the same output path as a fixed output
+specified hashing algorithm, resulting in the same output path as a fixed-output
 derivation.  This can be used for sources that are not available from a public
 url or broke since the download expression was written.
 </para>

doc/manual/command-ref/opt-common-syn.xml

@@ -1,5 +1,5 @@
 <nop xmlns="http://docbook.org/ns/docbook">
-  
+
 <arg><option>--help</option></arg>
 <arg><option>--version</option></arg>
 <arg rep='repeat'>
@@ -11,6 +11,10 @@
 <arg>
   <arg choice='plain'><option>--quiet</option></arg>
 </arg>
+<arg>
+  <option>--log-format</option>
+  <replaceable>format</replaceable>
+</arg>
 <arg>
   <group choice='plain'>
     <arg choice='plain'><option>--no-build-output</option></arg>

doc/manual/command-ref/opt-common.xml

@@ -92,6 +92,37 @@
 </varlistentry>
 
 
+<varlistentry xml:id="opt-log-format"><term><option>--log-format</option> <replaceable>format</replaceable></term>
+
+  <listitem>
+
+  <para>This option can be used to change the output of the log format, with
+  <replaceable>format</replaceable> being one of:</para>
+
+  <variablelist>
+
+    <varlistentry><term>raw</term>
+    <listitem><para>This is the raw format, as outputted by nix-build.</para></listitem>
+    </varlistentry>
+
+    <varlistentry><term>internal-json</term>
+    <listitem><para>Outputs the logs in a structured manner. NOTE: the json schema is not guarantees to be stable between releases.</para></listitem>
+    </varlistentry>
+
+    <varlistentry><term>bar</term>
+    <listitem><para>Only display a progress bar during the builds.</para></listitem>
+    </varlistentry>
+
+    <varlistentry><term>bar-with-logs</term>
+    <listitem><para>Display the raw logs, with the progress bar at the bottom.</para></listitem>
+    </varlistentry>
+
+  </variablelist>
+
+  </listitem>
+
+</varlistentry>
+
 <varlistentry><term><option>--no-build-output</option> / <option>-Q</option></term>
 
   <listitem><para>By default, output written by builders to standard

doc/manual/expressions/advanced-attributes.xml

@@ -178,7 +178,7 @@ impureEnvVars = [ "http_proxy" "https_proxy" <replaceable>...</replaceable> ];
 
 <programlisting>
 fetchurl {
-  url = http://ftp.gnu.org/pub/gnu/hello/hello-2.1.1.tar.gz;
+  url = "http://ftp.gnu.org/pub/gnu/hello/hello-2.1.1.tar.gz";
   sha256 = "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465";
 }
 </programlisting>
@@ -189,7 +189,7 @@ fetchurl {
 
 <programlisting>
 fetchurl {
-  url = ftp://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz;
+  url = "ftp://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz";
   sha256 = "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465";
 }
 </programlisting>

doc/manual/expressions/builtins.xml

@@ -324,7 +324,7 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting>
     particular version of Nixpkgs, e.g.
 
 <programlisting>
-with import (fetchTarball https://github.com/NixOS/nixpkgs-channels/archive/nixos-14.12.tar.gz) {};
+with import (fetchTarball https://github.com/NixOS/nixpkgs/archive/nixos-14.12.tar.gz) {};
 
 stdenv.mkDerivation { … }
 </programlisting>
@@ -349,7 +349,7 @@ stdenv.mkDerivation { … }
 
 <programlisting>
 with import (fetchTarball {
-  url = https://github.com/NixOS/nixpkgs-channels/archive/nixos-14.12.tar.gz;
+  url = "https://github.com/NixOS/nixpkgs/archive/nixos-14.12.tar.gz";
   sha256 = "1jppksrfvbk5ypiqdz4cddxdl8z6zyzdb2srq8fcffr327ld5jj2";
 }) {};
 
@@ -1406,7 +1406,7 @@ stdenv.mkDerivation {
   ";
 
   src = fetchurl {
-    url = http://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz;
+    url = "http://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz";
     sha256 = "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465";
   };
   inherit perl;

doc/manual/expressions/expression-syntax.xml

@@ -15,7 +15,7 @@ stdenv.mkDerivation { <co xml:id='ex-hello-nix-co-2' />
   name = "hello-2.1.1"; <co xml:id='ex-hello-nix-co-3' />
   builder = ./builder.sh; <co xml:id='ex-hello-nix-co-4' />
   src = fetchurl { <co xml:id='ex-hello-nix-co-5' />
-    url = ftp://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz;
+    url = "ftp://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz";
     sha256 = "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465";
   };
   inherit perl; <co xml:id='ex-hello-nix-co-6' />

doc/manual/expressions/simple-building-testing.xml

@@ -73,12 +73,4 @@ waiting for lock on `/nix/store/0h5b7hp8d4hqfrw8igvx97x1xawrjnac-hello-2.1.1x'</
 So it is always safe to run multiple instances of Nix in parallel
 (which isn’t the case with, say, <command>make</command>).</para>
 
-<para>If you have a system with multiple CPUs, you may want to have
-Nix build different derivations in parallel (insofar as possible).
-Just pass the option <link linkend='opt-max-jobs'><option>-j
-<replaceable>N</replaceable></option></link>, where
-<replaceable>N</replaceable> is the maximum number of jobs to be run
-in parallel, or set.  Typically this should be the number of
-CPUs.</para>
-
 </section>

doc/manual/installation/env-variables.xml

@@ -39,7 +39,7 @@ bundle.</para>
   <step><para>Set the environment variable and install Nix</para>
     <screen>
 $ export NIX_SSL_CERT_FILE=/etc/ssl/my-certificate-bundle.crt
-$ sh &lt;(curl https://nixos.org/nix/install)
+$ sh &lt;(curl -L https://nixos.org/nix/install)
 </screen></step>
 
   <step><para>In the shell profile and rc files (for example,

doc/manual/installation/installing-binary.xml

@@ -6,16 +6,30 @@
 
 <title>Installing a Binary Distribution</title>
 
-<para>If you are using Linux or macOS, the easiest way to install Nix
-is to run the following command:
+<para>
+  If you are using Linux or macOS versions up to 10.14 (Mojave), the
+  easiest way to install Nix is to run the following command:
+</para>
 
 <screen>
-  $ sh &lt;(curl https://nixos.org/nix/install)
+  $ sh &lt;(curl -L https://nixos.org/nix/install)
 </screen>
 
-As of Nix 2.1.0, the Nix installer will always default to creating a
-single-user installation, however opting in to the multi-user
-installation is highly recommended.
+<para>
+  If you're using macOS 10.15 (Catalina) or newer, consult
+  <link linkend="sect-macos-installation">the macOS installation instructions</link>
+  before installing.
+</para>
+
+<para>
+  As of Nix 2.1.0, the Nix installer will always default to creating a
+  single-user installation, however opting in to the multi-user
+  installation is highly recommended.
+  <!-- TODO: this explains *neither* why the default version is
+  single-user, nor why we'd recommend multi-user over the default.
+  True prospective users don't have much basis for evaluating this.
+  What's it to me? Who should pick which? Why? What if I pick wrong?
+  -->
 </para>
 
 <section xml:id="sect-single-user-installation">
@@ -25,7 +39,7 @@ installation is highly recommended.
     To explicitly select a single-user installation on your system:
 
     <screen>
-  sh &lt;(curl https://nixos.org/nix/install) --no-daemon
+  sh &lt;(curl -L https://nixos.org/nix/install) --no-daemon
 </screen>
   </para>
 
@@ -36,7 +50,7 @@ run this under your usual user account, <emphasis>not</emphasis> as
 root.  The script will invoke <command>sudo</command> to create
 <filename>/nix</filename> if it doesn’t already exist.  If you don’t
 have <command>sudo</command>, you should manually create
-<command>/nix</command> first as root, e.g.:
+<filename>/nix</filename> first as root, e.g.:
 
 <screen>
 $ mkdir /nix
@@ -47,7 +61,7 @@ The install script will modify the first writable file from amongst
 <filename>.bash_profile</filename>, <filename>.bash_login</filename>
 and <filename>.profile</filename> to source
 <filename>~/.nix-profile/etc/profile.d/nix.sh</filename>. You can set
-the <command>NIX_INSTALLER_NO_MODIFY_PROFILE</command> environment
+the <envar>NIX_INSTALLER_NO_MODIFY_PROFILE</envar> environment
 variable before executing the install script to disable this
 behaviour.
 </para>
@@ -81,12 +95,10 @@ $ rm -rf /nix
   <para>
     You can instruct the installer to perform a multi-user
     installation on your system:
-
-    <screen>
-  sh &lt;(curl https://nixos.org/nix/install) --daemon
-</screen>
   </para>
 
+  <screen>sh &lt;(curl -L https://nixos.org/nix/install) --daemon</screen>
+
   <para>
     The multi-user installation of Nix will create build users between
     the user IDs 30001 and 30032, and a group with the group ID 30000.
@@ -136,6 +148,273 @@ sudo rm /Library/LaunchDaemons/org.nixos.nix-daemon.plist
 
 </section>
 
+<section xml:id="sect-macos-installation">
+  <title>macOS Installation</title>
+
+  <para>
+    Starting with macOS 10.15 (Catalina), the root filesystem is read-only.
+    This means <filename>/nix</filename> can no longer live on your system
+    volume, and that you'll need a workaround to install Nix.
+  </para>
+
+  <para>
+    The recommended approach, which creates an unencrypted APFS volume
+    for your Nix store and a "synthetic" empty directory to mount it
+    over at <filename>/nix</filename>, is least likely to impair Nix
+    or your system.
+  </para>
+
+  <note><para>
+    With all separate-volume approaches, it's possible something on
+    your system (particularly daemons/services and restored apps) may
+    need access to your Nix store before the volume is mounted. Adding
+    additional encryption makes this more likely.
+  </para></note>
+
+  <para>
+    If you're using a recent Mac with a
+    <link xlink:href="https://www.apple.com/euro/mac/shared/docs/Apple_T2_Security_Chip_Overview.pdf">T2 chip</link>,
+    your drive will still be encrypted at rest (in which case "unencrypted"
+    is a bit of a misnomer). To use this approach, just install Nix with:
+  </para>
+
+  <screen>$ sh &lt;(curl -L https://nixos.org/nix/install) --darwin-use-unencrypted-nix-store-volume</screen>
+
+  <para>
+    If you don't like the sound of this, you'll want to weigh the
+    other approaches and tradeoffs detailed in this section.
+  </para>
+
+  <note>
+    <title>Eventual solutions?</title>
+    <para>
+      All of the known workarounds have drawbacks, but we hope
+      better solutions will be available in the future. Some that
+      we have our eye on are:
+    </para>
+    <orderedlist>
+      <listitem>
+        <para>
+          A true firmlink would enable the Nix store to live on the
+          primary data volume without the build problems caused by
+          the symlink approach. End users cannot currently
+          create true firmlinks.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          If the Nix store volume shared FileVault encryption
+          with the primary data volume (probably by using the same
+          volume group and role), FileVault encryption could be
+          easily supported by the installer without requiring
+          manual setup by each user.
+        </para>
+      </listitem>
+    </orderedlist>
+  </note>
+
+  <section xml:id="sect-macos-installation-change-store-prefix">
+    <title>Change the Nix store path prefix</title>
+    <para>
+      Changing the default prefix for the Nix store is a simple
+      approach which enables you to leave it on your root volume,
+      where it can take full advantage of FileVault encryption if
+      enabled. Unfortunately, this approach also opts your device out
+      of some benefits that are enabled by using the same prefix
+      across systems:
+
+      <itemizedlist>
+        <listitem>
+          <para>
+            Your system won't be able to take advantage of the binary
+            cache (unless someone is able to stand up and support
+            duplicate caching infrastructure), which means you'll
+            spend more time waiting for builds.
+          </para>
+        </listitem>
+        <listitem>
+          <para>
+            It's harder to build and deploy packages to Linux systems.
+          </para>
+        </listitem>
+        <!-- TODO: may be more here -->
+      </itemizedlist>
+
+      <!-- TODO: Yes, but how?! -->
+
+      It would also possible (and often requested) to just apply this
+      change ecosystem-wide, but it's an intrusive process that has
+      side effects we want to avoid for now.
+      <!-- magnificent hand-wavy gesture -->
+    </para>
+    <para>
+    </para>
+  </section>
+
+  <section xml:id="sect-macos-installation-encrypted-volume">
+    <title>Use a separate encrypted volume</title>
+    <para>
+      If you like, you can also add encryption to the recommended
+      approach taken by the installer. You can do this by pre-creating
+      an encrypted volume before you run the installer--or you can
+      run the installer and encrypt the volume it creates later.
+      <!-- TODO: see later note about whether this needs both add-encryption and from-scratch directions -->
+    </para>
+    <para>
+      In either case, adding encryption to a second volume isn't quite
+      as simple as enabling FileVault for your boot volume. Before you
+      dive in, there are a few things to weigh:
+    </para>
+    <orderedlist>
+      <listitem>
+        <para>
+          The additional volume won't be encrypted with your existing
+          FileVault key, so you'll need another mechanism to decrypt
+          the volume.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          You can store the password in Keychain to automatically
+          decrypt the volume on boot--but it'll have to wait on Keychain
+          and may not mount before your GUI apps restore. If any of
+          your launchd agents or apps depend on Nix-installed software
+          (for example, if you use a Nix-installed login shell), the
+          restore may fail or break.
+        </para>
+        <para>
+          On a case-by-case basis, you may be able to work around this
+          problem by using <command>wait4path</command> to block
+          execution until your executable is available.
+        </para>
+        <para>
+          It's also possible to decrypt and mount the volume earlier
+          with a login hook--but this mechanism appears to be
+          deprecated and its future is unclear.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          You can hard-code the password in the clear, so that your
+          store volume can be decrypted before Keychain is available.
+        </para>
+      </listitem>
+    </orderedlist>
+    <para>
+      If you are comfortable navigating these tradeoffs, you can encrypt the volume with
+      something along the lines of:
+      <!-- TODO:
+      I don't know if this also needs from-scratch instructions?
+      can we just recommend use-the-installer-and-then-encrypt?
+      -->
+    </para>
+    <!--
+    TODO: it looks like this option can be encryptVolume|encrypt|enableFileVault
+
+    It may be more clear to use encryptVolume, here? FileVault seems
+    heavily associated with the boot-volume behavior; I worry
+    a little that it can mislead here, especially as it gets
+    copied around minus doc context...?
+    -->
+    <screen>alice$ diskutil apfs enableFileVault /nix -user disk</screen>
+
+    <!-- TODO: and then go into detail on the mount/decrypt approaches? -->
+  </section>
+
+  <section xml:id="sect-macos-installation-symlink">
+    <!--
+    Maybe a good razor is: if we'd hate having to support someone who
+    installed Nix this way, it shouldn't even be detailed?
+    -->
+    <title>Symlink the Nix store to a custom location</title>
+    <para>
+      Another simple approach is using <filename>/etc/synthetic.conf</filename>
+      to symlink the Nix store to the data volume. This option also
+      enables your store to share any configured FileVault encryption.
+      Unfortunately, builds that resolve the symlink may leak the
+      canonical path or even fail.
+    </para>
+    <para>
+      Because of these downsides, we can't recommend this approach.
+    </para>
+    <!-- Leaving out instructions for this one. -->
+  </section>
+
+  <section xml:id="sect-macos-installation-recommended-notes">
+    <title>Notes on the recommended approach</title>
+    <para>
+      This section goes into a little more detail on the recommended
+      approach. You don't need to understand it to run the installer,
+      but it can serve as a helpful reference if you run into trouble.
+    </para>
+    <orderedlist>
+      <listitem>
+        <para>
+          In order to compose user-writable locations into the new
+          read-only system root, Apple introduced a new concept called
+          <literal>firmlinks</literal>, which it describes as a
+          "bi-directional wormhole" between two filesystems. You can
+          see the current firmlinks in <filename>/usr/share/firmlinks</filename>.
+          Unfortunately, firmlinks aren't (currently?) user-configurable.
+        </para>
+
+        <para>
+          For special cases like NFS mount points or package manager roots,
+          <link xlink:href="https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man5/synthetic.conf.5.html">synthetic.conf(5)</link>
+          supports limited user-controlled file-creation (of symlinks,
+          and synthetic empty directories) at <filename>/</filename>.
+          To create a synthetic empty directory for mounting at <filename>/nix</filename>,
+          add the following line to <filename>/etc/synthetic.conf</filename>
+          (create it if necessary):
+        </para>
+
+        <screen>nix</screen>
+      </listitem>
+
+      <listitem>
+        <para>
+          This configuration is applied at boot time, but you can use
+          <command>apfs.util</command> to trigger creation (not deletion)
+          of new entries without a reboot:
+        </para>
+
+        <screen>alice$ /System/Library/Filesystems/apfs.fs/Contents/Resources/apfs.util -B</screen>
+      </listitem>
+
+      <listitem>
+        <para>
+          Create the new APFS volume with diskutil:
+        </para>
+
+        <screen>alice$ sudo diskutil apfs addVolume diskX APFS 'Nix Store' -mountpoint /nix</screen>
+      </listitem>
+
+      <listitem>
+        <para>
+          Using <command>vifs</command>, add the new mount to
+          <filename>/etc/fstab</filename>. If it doesn't already have
+          other entries, it should look something like:
+        </para>
+
+<screen>
+#
+# Warning - this file should only be modified with vifs(8)
+#
+# Failure to do so is unsupported and may be destructive.
+#
+LABEL=Nix\040Store /nix apfs rw,nobrowse
+</screen>
+
+        <para>
+          The nobrowse setting will keep Spotlight from indexing this
+          volume, and keep it from showing up on your desktop.
+        </para>
+      </listitem>
+    </orderedlist>
+  </section>
+
+</section>
+
 <section xml:id="sect-nix-install-pinned-version-url">
   <title>Installing a pinned Nix version from a URL</title>
 
@@ -150,7 +429,7 @@ sudo rm /Library/LaunchDaemons/org.nixos.nix-daemon.plist
   NixOS.org installation script:
 
   <screen>
-  sh &lt;(curl https://nixos.org/nix/install)
+  sh &lt;(curl -L https://nixos.org/nix/install)
 </screen>
   </para>
 

doc/manual/installation/upgrading.xml

@@ -17,6 +17,11 @@
 
   <para>
     Single-user installations of Nix should run this:
-    <command>nix-channel --update; nix-env -iA nixpkgs.nix</command>
+    <command>nix-channel --update; nix-env -iA nixpkgs.nix nixpkgs.cacert</command>
+  </para>
+      
+  <para>
+    Multi-user Nix users on Linux should run this with sudo:
+    <command>nix-channel --update; nix-env -iA nixpkgs.nix nixpkgs.cacert; systemctl daemon-reload; systemctl restart nix-daemon</command>
   </para>
 </chapter>

doc/manual/introduction/quick-start.xml

@@ -15,7 +15,7 @@ to subsequent chapters.</para>
 <step><para>Install single-user Nix by running the following:
 
 <screen>
-$ bash &lt;(curl https://nixos.org/nix/install)
+$ bash &lt;(curl -L https://nixos.org/nix/install)
 </screen>
 
 This will install Nix in <filename>/nix</filename>. The install script

doc/manual/release-notes/rl-0.8.xml

@@ -8,7 +8,7 @@
 
 <para>NOTE: the hashing scheme in Nix 0.8 changed (as detailed below).
 As a result, <command>nix-pull</command> manifests and channels built
-for Nix 0.7 and below will now work anymore.  However, the Nix
+for Nix 0.7 and below will not work anymore.  However, the Nix
 expression language has not changed, so you can still build from
 source.  Also, existing user environments continue to work.  Nix 0.8
 will automatically upgrade the database schema of previous

flake.lock

@@ -1,14 +1,12 @@
 {
   "nodes": {
     "nixpkgs": {
-      "info": {
-        "lastModified": 1585405475,
-        "narHash": "sha256-bESW0n4KgPmZ0luxvwJ+UyATrC6iIltVCsGdLiphVeE="
-      },
       "locked": {
+        "lastModified": 1591633336,
+        "narHash": "sha256-oVXv4xAnDJB03LvZGbC72vSVlIbbJr8tpjEW5o/Fdek=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "b88ff468e9850410070d4e0ccd68c7011f15b2be",
+        "rev": "70717a337f7ae4e486ba71a500367cad697e5f09",
         "type": "github"
       },
       "original": {
@@ -24,5 +22,5 @@
     }
   },
   "root": "root",
-  "version": 5
+  "version": 6
 }

flake.nix

@@ -1,8 +1,6 @@
 {
   description = "The purely functional package manager";
 
-  edition = 201909; # FIXME: remove
-
   inputs.nixpkgs.url = "nixpkgs/nixos-20.03-small";
 
   outputs = { self, nixpkgs }:
@@ -80,12 +78,12 @@
              then nlohmann_json
              else nlohmann_json.override { multipleHeaders = true; })
             nlohmann_json
-            rustc cargo
 
             # Tests
             git
             mercurial
             jq
+            gmock
           ]
           ++ lib.optionals stdenv.isLinux [libseccomp utillinuxMinimal]
           ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium
@@ -135,8 +133,6 @@
                 chmod u+w $out/lib/*.so.*
                 patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.*
               ''}
-
-              ln -sfn ${final.nixVendoredCrates}/vendor/ nix-rust/vendor
             '';
 
           configureFlags = configureFlags ++
@@ -150,15 +146,15 @@
 
           installFlags = "sysconfdir=$(out)/etc";
 
+          postInstall = ''
+            mkdir -p $doc/nix-support
+            echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products
+          '';
+
           doInstallCheck = true;
           installCheckFlags = "sysconfdir=$(out)/etc";
 
           separateDebugInfo = true;
-
-          preDist = ''
-            mkdir -p $doc/nix-support
-            echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products
-          '';
         }) // {
 
           perl-bindings = with final; stdenv.mkDerivation {
@@ -191,68 +187,10 @@
 
         };
 
-        # Create a "vendor" directory that contains the crates listed in
-        # Cargo.lock, and include it in the Nix tarball. This allows Nix
-        # to be built without network access.
-        nixVendoredCrates =
-          let
-            lockFile = builtins.fromTOML (builtins.readFile nix-rust/Cargo.lock);
-
-            files = map (pkg: import <nix/fetchurl.nix> {
-              url = "https://crates.io/api/v1/crates/${pkg.name}/${pkg.version}/download";
-              sha256 = lockFile.metadata."checksum ${pkg.name} ${pkg.version} (registry+https://github.com/rust-lang/crates.io-index)";
-            }) (builtins.filter (pkg: pkg.source or "" == "registry+https://github.com/rust-lang/crates.io-index") lockFile.package);
-
-          in final.runCommand "cargo-vendor-dir" {}
-            ''
-              mkdir -p $out/vendor
-
-              cat > $out/vendor/config <<EOF
-              [source.crates-io]
-              replace-with = "vendored-sources"
-
-              [source.vendored-sources]
-              directory = "vendor"
-              EOF
-
-              ${toString (builtins.map (file: ''
-                mkdir $out/vendor/tmp
-                tar xvf ${file} -C $out/vendor/tmp
-                dir=$(echo $out/vendor/tmp/*)
-
-                # Add just enough metadata to keep Cargo happy.
-                printf '{"files":{},"package":"${file.outputHash}"}' > "$dir/.cargo-checksum.json"
-
-                # Clean up some cruft from the winapi crates. FIXME: find
-                # a way to remove winapi* from our dependencies.
-                if [[ $dir =~ /winapi ]]; then
-                  find $dir -name "*.a" -print0 | xargs -0 rm -f --
-                fi
-
-                mv "$dir" $out/vendor/
-
-                rm -rf $out/vendor/tmp
-              '') files)}
-            '';
-
       };
 
       hydraJobs = {
 
-        vendoredCrates =
-          with nixpkgsFor.x86_64-linux;
-
-          runCommand "vendored-crates" {}
-            ''
-              mkdir -p $out/nix-support
-              name=nix-vendored-crates-${version}
-              fn=$out/$name.tar.xz
-              tar cvfJ $fn -C ${nixVendoredCrates} vendor \
-                --owner=0 --group=0 --mode=u+rw,uga+r \
-                --transform "s,vendor,$name,"
-              echo "file crates-tarball $fn" >> $out/nix-support/hydra-build-products
-            '';
-
         # Binary package for various platforms.
         build = nixpkgs.lib.genAttrs systems (system: nixpkgsFor.${system}.nix);
 
@@ -361,11 +299,6 @@
 
             src = self;
 
-            preConfigure =
-              ''
-                ln -sfn ${nixVendoredCrates}/vendor/ nix-rust/vendor
-              '';
-
             enableParallelBuilding = true;
 
             buildInputs = buildDeps ++ propagatedDeps;
@@ -488,7 +421,7 @@
         stdenv.mkDerivation {
           name = "nix";
 
-          buildInputs = buildDeps ++ propagatedDeps ++ perlDeps ++ [ pkgs.rustfmt ];
+          buildInputs = buildDeps ++ propagatedDeps ++ perlDeps;
 
           inherit configureFlags;
 

maintainers/upload-release.pl

@@ -142,8 +142,12 @@ $oldName =~ s/"//g;
 sub getStorePath {
     my ($jobName) = @_;
     my $buildInfo = decode_json(fetch("$evalUrl/job/$jobName", 'application/json'));
-    die unless $buildInfo->{buildproducts}->{1}->{type} eq "nix-build";
-    return $buildInfo->{buildproducts}->{1}->{path};
+    for my $product (values %{$buildInfo->{buildproducts}}) {
+        next unless $product->{type} eq "nix-build";
+        next if $product->{path} =~ /[a-z]+$/;
+        return $product->{path};
+    }
+    die;
 }
 
 write_file("$nixpkgsDir/nixos/modules/installer/tools/nix-fallback-paths.nix",
@@ -166,15 +170,5 @@ $channelsBucket->add_key(
 chdir("/home/eelco/Dev/nix-pristine") or die;
 system("git remote update origin") == 0 or die;
 system("git tag --force --sign $version $nixRev -m 'Tagging release $version'") == 0 or die;
-
-# Update the website.
-my $siteDir = "/home/eelco/Dev/nixos-homepage-pristine";
-
-system("cd $siteDir && git pull") == 0 or die;
-
-write_file("$siteDir/nix-release.tt",
-           "[%-\n" .
-           "latestNixVersion = \"$version\"\n" .
-           "-%]\n");
-
-system("cd $siteDir && git commit -a -m 'Nix $version released'") == 0 or die;
+system("git push --tags") == 0 or die;
+system("git push --force-with-lease origin $nixRev:refs/heads/latest-release") == 0 or die;

misc/bash/local.mk

@@ -1 +1 @@
-$(eval $(call install-file-as, $(d)/completion.sh, $(datarootdir)/bash-completion/completions/_nix3, 0644))
+$(eval $(call install-file-as, $(d)/completion.sh, $(datarootdir)/bash-completion/completions/nix, 0644))

mk/libraries.mk

@@ -125,7 +125,8 @@ define build-library
     $(1)_PATH := $$(_d)/$$($(1)_NAME).a
 
     $$($(1)_PATH): $$($(1)_OBJS) | $$(_d)/
-	$(trace-ar) $(AR) crs $$@ $$?
+	$(trace-ld) $(LD) -Ur -o $$(_d)/$$($(1)_NAME).o $$?
+	$(trace-ar) $(AR) crs $$@ $$(_d)/$$($(1)_NAME).o
 
     $(1)_LDFLAGS_USE += $$($(1)_PATH) $$($(1)_LDFLAGS)
 

mk/programs.mk

@@ -35,24 +35,28 @@ define build-program
 	$$(trace-ld) $(CXX) -o $$@ $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE))
 
   $(1)_INSTALL_DIR ?= $$(bindir)
-  $(1)_INSTALL_PATH := $$($(1)_INSTALL_DIR)/$(1)
 
-  $$(eval $$(call create-dir, $$($(1)_INSTALL_DIR)))
+  ifdef $(1)_INSTALL_DIR
 
-  install: $(DESTDIR)$$($(1)_INSTALL_PATH)
+    $(1)_INSTALL_PATH := $$($(1)_INSTALL_DIR)/$(1)
 
-  ifeq ($(BUILD_SHARED_LIBS), 1)
+    $$(eval $$(call create-dir, $$($(1)_INSTALL_DIR)))
 
-    _libs_final := $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_INSTALL_PATH))
+    install: $(DESTDIR)$$($(1)_INSTALL_PATH)
 
-    $(DESTDIR)$$($(1)_INSTALL_PATH): $$($(1)_OBJS) $$(_libs_final) | $(DESTDIR)$$($(1)_INSTALL_DIR)/
+    ifeq ($(BUILD_SHARED_LIBS), 1)
+
+      _libs_final := $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_INSTALL_PATH))
+
+      $(DESTDIR)$$($(1)_INSTALL_PATH): $$($(1)_OBJS) $$(_libs_final) | $(DESTDIR)$$($(1)_INSTALL_DIR)/
 	$$(trace-ld) $(CXX) -o $$@ $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE_INSTALLED))
 
-  else
+    else
 
-    $(DESTDIR)$$($(1)_INSTALL_PATH): $$($(1)_PATH) | $(DESTDIR)$$($(1)_INSTALL_DIR)/
+      $(DESTDIR)$$($(1)_INSTALL_PATH): $$($(1)_PATH) | $(DESTDIR)$$($(1)_INSTALL_DIR)/
 	install -t $(DESTDIR)$$($(1)_INSTALL_DIR) $$<
 
+    endif
   endif
 
   # Propagate CFLAGS and CXXFLAGS to the individual object files.
@@ -76,4 +80,10 @@ define build-program
   programs-list += $$($(1)_PATH)
   clean-files += $$($(1)_PATH) $$(_d)/*.o $$(_d)/.*.dep $$($(1)_DEPS) $$($(1)_OBJS)
   dist-files += $$(_srcs)
+
+  # Phony target to run this program (typically as a dependency of 'check').
+  .PHONY: $(1)_RUN
+  $(1)_RUN: $$($(1)_PATH)
+	$(trace-test) $$($(1)_PATH)
+
 endef

mk/run_test.sh

@@ -0,0 +1,28 @@
+#!/bin/sh
+
+set -u
+
+red=""
+green=""
+yellow=""
+normal=""
+
+post_run_msg="ran test $1..."
+if [ -t 1 ]; then
+    red=""
+    green=""
+    yellow=""
+    normal=""
+fi
+(cd $(dirname $1) && env ${TESTS_ENVIRONMENT} init.sh 2>/dev/null > /dev/null)
+log="$(cd $(dirname $1) && env ${TESTS_ENVIRONMENT} $(basename $1) 2>&1)"
+status=$?
+if [ $status -eq 0 ]; then
+  echo "$post_run_msg [${green}PASS$normal]"
+elif [ $status -eq 99 ]; then
+  echo "$post_run_msg [${yellow}SKIP$normal]"
+else
+  echo "$post_run_msg [${red}FAIL$normal]"
+  echo "$log" | sed 's/^/    /'
+  exit "$status"
+fi

mk/tests.mk

@@ -1,45 +1,15 @@
 # Run program $1 as part of ‘make installcheck’.
+
+test-deps =
+
 define run-install-test
 
-  installcheck: $1
+  installcheck: $1.test
 
-  _installcheck-list += $1
+  .PHONY: $1.test
+  $1.test: $1 $(test-deps)
+	@env TEST_NAME=$(notdir $(basename $1)) TESTS_ENVIRONMENT="$(tests-environment)" mk/run_test.sh $1
 
 endef
 
-# Color code from https://unix.stackexchange.com/a/10065
-installcheck:
-	@total=0; failed=0; \
-	red=""; \
-	green=""; \
-	yellow=""; \
-	normal=""; \
-	if [ -t 1 ]; then \
-		red=""; \
-		green=""; \
-		yellow=""; \
-		normal=""; \
-	fi; \
-	for i in $(_installcheck-list); do \
-	  total=$$((total + 1)); \
-	  printf "running test $$i..."; \
-	  log="$$(cd $$(dirname $$i) && $(tests-environment) $$(basename $$i) 2>&1)"; \
-	  status=$$?; \
-	  if [ $$status -eq 0 ]; then \
-	    echo " [$${green}PASS$$normal]"; \
-	  elif [ $$status -eq 99 ]; then \
-	    echo " [$${yellow}SKIP$$normal]"; \
-	  else \
-	    echo " [$${red}FAIL$$normal]"; \
-	    echo "$$log" | sed 's/^/    /'; \
-	    failed=$$((failed + 1)); \
-	  fi; \
-	done; \
-	if [ "$$failed" != 0 ]; then \
-	  echo "$${red}$$failed out of $$total tests failed $$normal"; \
-	  exit 1; \
-	else \
-		echo "$${green}All tests succeeded$$normal"; \
-	fi
-
 .PHONY: check installcheck

mk/tracing.mk

@@ -11,6 +11,7 @@ ifeq ($(V), 0)
   trace-javac   = @echo "  JAVAC " $@;
   trace-jar     = @echo "  JAR   " $@;
   trace-mkdir   = @echo "  MKDIR " $@;
+  trace-test    = @echo "  TEST  " $@;
 
   suppress  = @
 

nix-rust/local.mk

@@ -41,5 +41,5 @@ ifneq ($(OS), Darwin)
 check: rust-tests
 
 rust-tests:
-	cd nix-rust && CARGO_HOME=$$(if [[ -d vendor ]]; then echo vendor; fi) cargo test --release $$(if [[ -d vendor ]]; then echo --offline; fi)
+	$(trace-test) cd nix-rust && CARGO_HOME=$$(if [[ -d vendor ]]; then echo vendor; fi) cargo test --release $$(if [[ -d vendor ]]; then echo --offline; fi)
 endif

perl/lib/Nix/Store.xs

@@ -80,7 +80,7 @@ SV * queryReferences(char * path)
 SV * queryPathHash(char * path)
     PPCODE:
         try {
-            auto s = store()->queryPathInfo(store()->parseStorePath(path))->narHash.to_string();
+            auto s = store()->queryPathInfo(store()->parseStorePath(path))->narHash.to_string(Base32, true);
             XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
         } catch (Error & e) {
             croak("%s", e.what());
@@ -106,7 +106,7 @@ SV * queryPathInfo(char * path, int base32)
                 XPUSHs(&PL_sv_undef);
             else
                 XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(*info->deriver).c_str(), 0)));
-            auto s = info->narHash.to_string(base32 ? Base32 : Base16);
+            auto s = info->narHash.to_string(base32 ? Base32 : Base16, true);
             XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
             mXPUSHi(info->registrationTime);
             mXPUSHi(info->narSize);
@@ -182,7 +182,7 @@ void importPaths(int fd, int dontCheckSigs)
     PPCODE:
         try {
             FdSource source(fd);
-            store()->importPaths(source, nullptr, dontCheckSigs ? NoCheckSigs : CheckSigs);
+            store()->importPaths(source, dontCheckSigs ? NoCheckSigs : CheckSigs);
         } catch (Error & e) {
             croak("%s", e.what());
         }
@@ -274,7 +274,8 @@ int checkSignature(SV * publicKey_, SV * sig_, char * msg)
 SV * addToStore(char * srcPath, int recursive, char * algo)
     PPCODE:
         try {
-            auto path = store()->addToStore(std::string(baseNameOf(srcPath)), srcPath, recursive, parseHashType(algo));
+            auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
+            auto path = store()->addToStore(std::string(baseNameOf(srcPath)), srcPath, method, parseHashType(algo));
             XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0)));
         } catch (Error & e) {
             croak("%s", e.what());
@@ -285,7 +286,8 @@ SV * makeFixedOutputPath(int recursive, char * algo, char * hash, char * name)
     PPCODE:
         try {
             Hash h(hash, parseHashType(algo));
-            auto path = store()->makeFixedOutputPath(recursive, h, name);
+            auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
+            auto path = store()->makeFixedOutputPath(method, h, name);
             XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0)));
         } catch (Error & e) {
             croak("%s", e.what());

scripts/create-darwin-volume.sh

@@ -0,0 +1,185 @@
+#!/bin/sh
+set -e
+
+root_disk() {
+    diskutil info -plist /
+}
+
+apfs_volumes_for() {
+    disk=$1
+    diskutil apfs list -plist "$disk"
+}
+
+disk_identifier() {
+    xpath "/plist/dict/key[text()='ParentWholeDisk']/following-sibling::string[1]/text()" 2>/dev/null
+}
+
+volume_list_true() {
+    key=$1
+    xpath "/plist/dict/array/dict/key[text()='Volumes']/following-sibling::array/dict/key[text()='$key']/following-sibling::true[1]" 2> /dev/null
+}
+
+volume_get_string() {
+    key=$1 i=$2
+    xpath "/plist/dict/array/dict/key[text()='Volumes']/following-sibling::array/dict[$i]/key[text()='$key']/following-sibling::string[1]/text()" 2> /dev/null
+}
+
+find_nix_volume() {
+    disk=$1
+    i=1
+    volumes=$(apfs_volumes_for "$disk")
+    while true; do
+        name=$(echo "$volumes" | volume_get_string "Name" "$i")
+        if [ -z "$name" ]; then
+            break
+        fi
+        case "$name" in
+            [Nn]ix*)
+                echo "$name"
+                break
+                ;;
+        esac
+        i=$((i+1))
+    done
+}
+
+test_fstab() {
+    grep -q "/nix apfs rw" /etc/fstab 2>/dev/null
+}
+
+test_nix_symlink() {
+    [ -L "/nix" ] || grep -q "^nix." /etc/synthetic.conf 2>/dev/null
+}
+
+test_synthetic_conf() {
+    grep -q "^nix$" /etc/synthetic.conf 2>/dev/null
+}
+
+test_nix() {
+    test -d "/nix"
+}
+
+test_t2_chip_present(){
+    # Use xartutil to see if system has a t2 chip.
+    #
+    # This isn't well-documented on its own; until it is,
+    # let's keep track of knowledge/assumptions.
+    #
+    # Warnings:
+    # - Don't search "xart" if porn will cause you trouble :)
+    # - Other xartutil flags do dangerous things. Don't run them
+    #   naively. If you must, search "xartutil" first.
+    #
+    # Assumptions:
+    # - the "xART session seeds recovery utility"
+    #   appears to interact with xartstorageremoted
+    # - `sudo xartutil --list` lists xART sessions
+    #   and their seeds and exits 0 if successful. If
+    #   not, it exits 1 and prints an error such as:
+    #   xartutil: ERROR: No supported link to the SEP present
+    # - xART sessions/seeds are present when a T2 chip is
+    #   (and not, otherwise)
+    # - the presence of a T2 chip means a newly-created
+    #   volume on the primary drive will be
+    #   encrypted at rest
+    # - all together: `sudo xartutil --list`
+    #   should exit 0 if a new Nix Store volume will
+    #   be encrypted at rest, and exit 1 if not.
+    sudo xartutil --list >/dev/null 2>/dev/null
+}
+
+test_filevault_in_use() {
+    disk=$1
+    #      list vols on disk | get value of Filevault key | value is true
+    apfs_volumes_for "$disk" | volume_list_true FileVault | grep -q true
+}
+
+# use after error msg for conditions we don't understand
+suggest_report_error(){
+    # ex "error: something sad happened :(" >&2
+    echo "       please report this @ https://github.com/nixos/nix/issues" >&2
+}
+
+main() {
+    (
+        echo ""
+        echo "     ------------------------------------------------------------------ "
+        echo "    | This installer will create a volume for the nix store and        |"
+        echo "    | configure it to mount at /nix.  Follow these steps to uninstall. |"
+        echo "     ------------------------------------------------------------------ "
+        echo ""
+        echo "  1. Remove the entry from fstab using 'sudo vifs'"
+        echo "  2. Destroy the data volume using 'diskutil apfs deleteVolume'"
+        echo "  3. Remove the 'nix' line from /etc/synthetic.conf or the file"
+        echo ""
+    ) >&2
+
+    if test_nix_symlink; then
+        echo "error: /nix is a symlink, please remove it and make sure it's not in synthetic.conf (in which case a reboot is required)" >&2
+        echo "  /nix -> $(readlink "/nix")" >&2
+        exit 2
+    fi
+
+    if ! test_synthetic_conf; then
+        echo "Configuring /etc/synthetic.conf..." >&2
+        echo nix | sudo tee -a /etc/synthetic.conf
+        if ! test_synthetic_conf; then
+            echo "error: failed to configure synthetic.conf;" >&2
+            suggest_report_error
+            exit 1
+        fi
+    fi
+
+    if ! test_nix; then
+        echo "Creating mountpoint for /nix..." >&2
+        /System/Library/Filesystems/apfs.fs/Contents/Resources/apfs.util -B || true
+        if ! test_nix; then
+            sudo mkdir -p /nix 2>/dev/null || true
+        fi
+        if ! test_nix; then
+            echo "error: failed to bootstrap /nix; if a reboot doesn't help," >&2
+            suggest_report_error
+            exit 1
+        fi
+    fi
+
+    disk=$(root_disk | disk_identifier)
+    volume=$(find_nix_volume "$disk")
+    if [ -z "$volume" ]; then
+        echo "Creating a Nix Store volume..." >&2
+
+        if test_filevault_in_use "$disk"; then
+            # TODO: Not sure if it's in-scope now, but `diskutil apfs list`
+            # shows both filevault and encrypted at rest status, and it
+            # may be the more semantic way to test for this? It'll show
+            # `FileVault:                 No (Encrypted at rest)`
+            # `FileVault:                 No`
+            # `FileVault:                 Yes (Unlocked)`
+            # and so on.
+            if test_t2_chip_present; then
+                echo "warning: boot volume is FileVault-encrypted, but the Nix store volume" >&2
+                echo "         is only encrypted at rest." >&2
+                echo "         See https://nixos.org/nix/manual/#sect-macos-installation" >&2
+            else
+                echo "error: refusing to create Nix store volume because the boot volume is" >&2
+                echo "       FileVault encrypted, but encryption-at-rest is not available." >&2
+                echo "       Manually create a volume for the store and re-run this script." >&2
+                echo "       See https://nixos.org/nix/manual/#sect-macos-installation" >&2
+                exit 1
+            fi
+        fi
+
+        sudo diskutil apfs addVolume "$disk" APFS 'Nix Store' -mountpoint /nix
+        volume="Nix Store"
+    else
+        echo "Using existing '$volume' volume" >&2
+    fi
+
+    if ! test_fstab; then
+        echo "Configuring /etc/fstab..." >&2
+        label=$(echo "$volume" | sed 's/ /\\040/g')
+        printf "\$a\nLABEL=%s /nix apfs rw,nobrowse\n.\nwq\n" "$label" | EDITOR=ed sudo vifs
+    fi
+}
+
+main "$@"

scripts/install-multi-user.sh

@@ -20,15 +20,18 @@ readonly GREEN='\033[32m'
 readonly GREEN_UL='\033[4;32m'
 readonly RED='\033[31m'
 
-readonly NIX_USER_COUNT="32"
+# installer allows overriding build user count to speed up installation
+# as creating each user takes non-trivial amount of time on macos
+readonly NIX_USER_COUNT=${NIX_USER_COUNT:-32}
 readonly NIX_BUILD_GROUP_ID="30000"
 readonly NIX_BUILD_GROUP_NAME="nixbld"
 readonly NIX_FIRST_BUILD_UID="30001"
 # Please don't change this. We don't support it, because the
 # default shell profile that comes with Nix doesn't support it.
 readonly NIX_ROOT="/nix"
+readonly NIX_EXTRA_CONF=${NIX_EXTRA_CONF:-}
 
-readonly PROFILE_TARGETS=("/etc/bashrc" "/etc/profile.d/nix.sh" "/etc/zshrc")
+readonly PROFILE_TARGETS=("/etc/bashrc" "/etc/profile.d/nix.sh" "/etc/zshenv")
 readonly PROFILE_BACKUP_SUFFIX=".backup-before-nix"
 readonly PROFILE_NIX_FILE="$NIX_ROOT/var/nix/profiles/default/etc/profile.d/nix-daemon.sh"
 
@@ -450,9 +453,11 @@ create_directories() {
 }
 
 place_channel_configuration() {
-    echo "https://nixos.org/channels/nixpkgs-unstable nixpkgs" > "$SCRATCH/.nix-channels"
-    _sudo "to set up the default system channel (part 1)" \
-          install -m 0664 "$SCRATCH/.nix-channels" "$ROOT_HOME/.nix-channels"
+    if [ -z "${NIX_INSTALLER_NO_CHANNEL_ADD:-}" ]; then
+        echo "https://nixos.org/channels/nixpkgs-unstable nixpkgs" > "$SCRATCH/.nix-channels"
+        _sudo "to set up the default system channel (part 1)" \
+            install -m 0664 "$SCRATCH/.nix-channels" "$ROOT_HOME/.nix-channels"
+    fi
 }
 
 welcome_to_nix() {
@@ -521,7 +526,7 @@ This script is going to call sudo a lot. Normally, it would show you
 exactly what commands it is running and why. However, the script is
 run in a headless fashion, like this:
 
-  $ curl https://nixos.org/nix/install | sh
+  $ curl -L https://nixos.org/nix/install | sh
 
 or maybe in a CI pipeline. Because of that, we're going to skip the
 verbose output in the interest of brevity.
@@ -529,7 +534,7 @@ verbose output in the interest of brevity.
 If you would like to
 see the output, try like this:
 
-  $ curl -o install-nix https://nixos.org/nix/install
+  $ curl -L -o install-nix https://nixos.org/nix/install
   $ sh ./install-nix
 
 EOF
@@ -634,18 +639,20 @@ setup_default_profile() {
         export NIX_SSL_CERT_FILE=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt
     fi
 
-    # Have to explicitly pass NIX_SSL_CERT_FILE as part of the sudo call,
-    # otherwise it will be lost in environments where sudo doesn't pass
-    # all the environment variables by default.
-    _sudo "to update the default channel in the default profile" \
-          HOME="$ROOT_HOME" NIX_SSL_CERT_FILE="$NIX_SSL_CERT_FILE" "$NIX_INSTALLED_NIX/bin/nix-channel" --update nixpkgs \
-          || channel_update_failed=1
-
+    if [ -z "${NIX_INSTALLER_NO_CHANNEL_ADD:-}" ]; then
+        # Have to explicitly pass NIX_SSL_CERT_FILE as part of the sudo call,
+        # otherwise it will be lost in environments where sudo doesn't pass
+        # all the environment variables by default.
+        _sudo "to update the default channel in the default profile" \
+            HOME="$ROOT_HOME" NIX_SSL_CERT_FILE="$NIX_SSL_CERT_FILE" "$NIX_INSTALLED_NIX/bin/nix-channel" --update nixpkgs \
+            || channel_update_failed=1
+    fi
 }
 
 
 place_nix_configuration() {
     cat <<EOF > "$SCRATCH/nix.conf"
+$NIX_EXTRA_CONF
 build-users-group = $NIX_BUILD_GROUP_NAME
 EOF
     _sudo "to place the default nix daemon configuration (part 2)" \

scripts/install-nix-from-closure.sh

@@ -40,29 +40,85 @@ elif [ "$(uname -s)" = "Linux" ] && [ -e /run/systemd/system ]; then
 fi
 
 INSTALL_MODE=no-daemon
-# Trivially handle the --daemon / --no-daemon options
-if [ "x${1:-}" = "x--no-daemon" ]; then
-    INSTALL_MODE=no-daemon
-elif [ "x${1:-}" = "x--daemon" ]; then
-    INSTALL_MODE=daemon
-elif [ "x${1:-}" != "x" ]; then
-    (
-        echo "Nix Installer [--daemon|--no-daemon]"
+CREATE_DARWIN_VOLUME=0
+# handle the command line flags
+while [ $# -gt 0 ]; do
+    case $1 in
+        --daemon)
+            INSTALL_MODE=daemon;;
+        --no-daemon)
+            INSTALL_MODE=no-daemon;;
+        --no-channel-add)
+            export NIX_INSTALLER_NO_CHANNEL_ADD=1;;
+        --daemon-user-count)
+            export NIX_USER_COUNT=$2
+            shift;;
+        --no-modify-profile)
+            NIX_INSTALLER_NO_MODIFY_PROFILE=1;;
+        --darwin-use-unencrypted-nix-store-volume)
+            CREATE_DARWIN_VOLUME=1;;
+        --nix-extra-conf-file)
+            export NIX_EXTRA_CONF="$(cat $2)"
+            shift;;
+        *)
+            (
+                echo "Nix Installer [--daemon|--no-daemon] [--daemon-user-count INT] [--no-channel-add] [--no-modify-profile] [--darwin-use-unencrypted-nix-store-volume] [--nix-extra-conf-file FILE]"
 
-        echo "Choose installation method."
-        echo ""
-        echo " --daemon:    Installs and configures a background daemon that manages the store,"
-        echo "              providing multi-user support and better isolation for local builds."
-        echo "              Both for security and reproducibility, this method is recommended if"
-        echo "              supported on your platform."
-        echo "              See https://nixos.org/nix/manual/#sect-multi-user-installation"
-        echo ""
-        echo " --no-daemon: Simple, single-user installation that does not require root and is"
-        echo "              trivial to uninstall."
-        echo "              (default)"
-        echo ""
-    ) >&2
-    exit
+                echo "Choose installation method."
+                echo ""
+                echo " --daemon:    Installs and configures a background daemon that manages the store,"
+                echo "              providing multi-user support and better isolation for local builds."
+                echo "              Both for security and reproducibility, this method is recommended if"
+                echo "              supported on your platform."
+                echo "              See https://nixos.org/nix/manual/#sect-multi-user-installation"
+                echo ""
+                echo " --no-daemon: Simple, single-user installation that does not require root and is"
+                echo "              trivial to uninstall."
+                echo "              (default)"
+                echo ""
+                echo " --no-channel-add:    Don't add any channels. nixpkgs-unstable is installed by default."
+                echo ""
+                echo " --no-modify-profile: Skip channel installation. When not provided nixpkgs-unstable"
+                echo "                      is installed by default."
+                echo ""
+                echo " --daemon-user-count: Number of build users to create. Defaults to 32."
+                echo ""
+                echo " --nix-extra-conf-file: Path to nix.conf to prepend when installing /etc/nix.conf"
+                echo ""
+            ) >&2
+
+            # darwin and Catalina+
+            if [ "$(uname -s)" = "Darwin" ] && [ "$macos_major" -gt 14 ]; then
+                (
+                    echo " --darwin-use-unencrypted-nix-store-volume: Create an APFS volume for the Nix"
+                    echo "              store and mount it at /nix. This is the recommended way to create"
+                    echo "              /nix with a read-only / on macOS >=10.15."
+                    echo "              See: https://nixos.org/nix/manual/#sect-macos-installation"
+                    echo ""
+                ) >&2
+            fi
+            exit;;
+    esac
+    shift
+done
+
+if [ "$(uname -s)" = "Darwin" ]; then
+    if [ "$CREATE_DARWIN_VOLUME" = 1 ]; then
+        printf '\e[1;31mCreating volume and mountpoint /nix.\e[0m\n'
+        "$self/create-darwin-volume.sh"
+    fi
+
+    info=$(diskutil info -plist / | xpath "/plist/dict/key[text()='Writable']/following-sibling::true[1]" 2> /dev/null)
+    if ! [ -e $dest ] && [ -n "$info" ] && [ "$macos_major" -gt 14 ]; then
+        (
+            echo ""
+            echo "Installing on macOS >=10.15 requires relocating the store to an apfs volume."
+            echo "Use sh <(curl -L https://nixos.org/nix/install) --darwin-use-unencrypted-nix-store-volume or run the preparation steps manually."
+            echo "See https://nixos.org/nix/manual/#sect-macos-installation"
+            echo ""
+        ) >&2
+        exit 1
+    fi
 fi
 
 if [ "$INSTALL_MODE" = "daemon" ]; then
@@ -130,13 +186,15 @@ if [ -z "$NIX_SSL_CERT_FILE" ] || ! [ -f "$NIX_SSL_CERT_FILE" ]; then
 fi
 
 # Subscribe the user to the Nixpkgs channel and fetch it.
-if ! $nix/bin/nix-channel --list | grep -q "^nixpkgs "; then
-    $nix/bin/nix-channel --add https://nixos.org/channels/nixpkgs-unstable
-fi
-if [ -z "$_NIX_INSTALLER_TEST" ]; then
-    if ! $nix/bin/nix-channel --update nixpkgs; then
-        echo "Fetching the nixpkgs channel failed. (Are you offline?)"
-        echo "To try again later, run \"nix-channel --update nixpkgs\"."
+if [ -z "$NIX_INSTALLER_NO_CHANNEL_ADD" ]; then
+    if ! $nix/bin/nix-channel --list | grep -q "^nixpkgs "; then
+        $nix/bin/nix-channel --add https://nixos.org/channels/nixpkgs-unstable
+    fi
+    if [ -z "$_NIX_INSTALLER_TEST" ]; then
+        if ! $nix/bin/nix-channel --update nixpkgs; then
+            echo "Fetching the nixpkgs channel failed. (Are you offline?)"
+            echo "To try again later, run \"nix-channel --update nixpkgs\"."
+        fi
     fi
 fi
 
@@ -155,6 +213,17 @@ if [ -z "$NIX_INSTALLER_NO_MODIFY_PROFILE" ]; then
             break
         fi
     done
+    for i in .zshenv .zshrc; do
+        fn="$HOME/$i"
+        if [ -w "$fn" ]; then
+            if ! grep -q "$p" "$fn"; then
+                echo "modifying $fn..." >&2
+                echo "if [ -e $p ]; then . $p; fi # added by Nix installer" >> "$fn"
+            fi
+            added=1
+            break
+        fi
+    done
 fi
 
 if [ -z "$added" ]; then

scripts/install.in

@@ -36,7 +36,9 @@ tarball="$tmpDir/$(basename "$tmpDir/nix-@nixVersion@-$system.tar.xz")"
 
 require_util curl "download the binary tarball"
 require_util tar "unpack the binary tarball"
-require_util xz "unpack the binary tarball"
+if [ "$(uname -s)" != "Darwin" ]; then
+    require_util xz "unpack the binary tarball"
+fi
 
 echo "downloading Nix @nixVersion@ binary tarball for $system from '$url' to '$tmpDir'..."
 curl -L "$url" -o "$tarball" || oops "failed to download '$url'"

shell.nix

@@ -1,3 +0,0 @@
-(import (fetchTarball https://github.com/edolstra/flake-compat/archive/master.tar.gz) {
-  src = builtins.fetchGit ./.;
-}).shellNix

src/build-remote/build-remote.cc

@@ -200,9 +200,12 @@ static int _main(int argc, char * * argv)
 
                 } catch (std::exception & e) {
                     auto msg = chomp(drainFD(5, false));
-                    printError("cannot build on '%s': %s%s",
-                        bestMachine->storeUri, e.what(),
-                        (msg.empty() ? "" : ": " + msg));
+                    logError({
+                        .name = "Remote build",
+                        .hint = hintfmt("cannot build on '%s': %s%s",
+                            bestMachine->storeUri, e.what(),
+                            (msg.empty() ? "" : ": " + msg))
+                    });
                     bestMachine->enabled = false;
                     continue;
                 }
@@ -241,7 +244,7 @@ connected:
 
         uploadLock = -1;
 
-        BasicDerivation drv(readDerivation(*store, store->realStoreDir + "/" + std::string(drvPath->to_string())));
+        auto drv = store->readDerivation(*drvPath);
         drv.inputSrcs = store->parseStorePathSet(inputs);
 
         auto result = sshStore->buildDerivation(*drvPath, drv);

src/error-demo/error-demo.cc

@@ -1,66 +0,0 @@
-#include "error.hh"
-#include "nixexpr.hh"
-
-#include <iostream>
-#include <optional>
-
-int main()
-{
-    using namespace nix;
-
-    // In each program where errors occur, this has to be set.
-    ErrorInfo::programName = std::optional("error-demo");
-
-    // Error in a program; no hint and no nix code.
-    printErrorInfo(
-        ErrorInfo { .level = elError,
-                    .name = "name",
-                    .description = "error description",
-                  });
-
-    // Warning with name, description, and hint.
-    // The hintfmt function makes all the substituted text yellow.
-    printErrorInfo(
-        ErrorInfo { .level = elWarning,
-                    .name = "name",
-                    .description = "error description",
-                    .hint =  std::optional(
-                                 hintfmt("there was a %1%", "warning")),
-                  });
-
-
-    // Warning with nix file, line number, column, and the lines of
-    // code where a warning occurred.
-    SymbolTable testTable;
-    auto problem_file = testTable.create("myfile.nix");
-
-    printErrorInfo(
-        ErrorInfo{
-            .level = elWarning,
-            .name = "warning name",
-            .description = "warning description",
-            .hint = hintfmt("this hint has %1% templated %2%!!", "yellow", "values"),
-            .nixCode = NixCode {
-                .errPos = Pos(problem_file, 40, 13),
-                .prevLineOfCode = std::nullopt,
-                .errLineOfCode = "this is the problem line of code",
-                .nextLineOfCode = std::nullopt
-            }});
-
-    // Error with previous and next lines of code.
-    printErrorInfo(
-        ErrorInfo{
-            .level = elError,
-            .name = "error name",
-            .description = "error description",
-            .hint = hintfmt("this hint has %1% templated %2%!!", "yellow", "values"),
-            .nixCode = NixCode {
-                .errPos = Pos(problem_file, 40, 13),
-                .prevLineOfCode = std::optional("previous line of code"),
-                .errLineOfCode = "this is the problem line of code",
-                .nextLineOfCode = std::optional("next line of code"),
-            }});
-
-
-    return 0;
-}

src/error-demo/local.mk

@@ -1,12 +0,0 @@
-programs += error-demo
-
-error-demo_DIR := $(d)
-
-error-demo_SOURCES := \
-  $(wildcard $(d)/*.cc) \
-
-error-demo_CXXFLAGS += -I src/libutil -I src/libexpr
-
-error-demo_LIBS = libutil libexpr
-
-error-demo_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) -lboost_context -lboost_thread -lboost_system

src/libexpr/attr-path.cc

@@ -19,7 +19,7 @@ static Strings parseAttrPath(std::string_view s)
             ++i;
             while (1) {
                 if (i == s.end())
-                    throw Error(format("missing closing quote in selection path '%1%'") % s);
+                    throw Error("missing closing quote in selection path '%1%'", s);
                 if (*i == '"') break;
                 cur.push_back(*i++);
             }
@@ -69,11 +69,11 @@ std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attr
 
             if (v->type != tAttrs)
                 throw TypeError(
-                    format("the expression selected by the selection path '%1%' should be a set but is %2%")
-                    % attrPath % showType(*v));
-
+                    "the expression selected by the selection path '%1%' should be a set but is %2%",
+                    attrPath,
+                    showType(*v));
             if (attr.empty())
-                throw Error(format("empty attribute name in selection path '%1%'") % attrPath);
+                throw Error("empty attribute name in selection path '%1%'", attrPath);
 
             Bindings::iterator a = v->attrs->find(state.symbols.create(attr));
             if (a == v->attrs->end())
@@ -86,9 +86,9 @@ std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attr
 
             if (!v->isList())
                 throw TypeError(
-                    format("the expression selected by the selection path '%1%' should be a list but is %2%")
-                    % attrPath % showType(*v));
-
+                    "the expression selected by the selection path '%1%' should be a list but is %2%",
+                    attrPath,
+                    showType(*v));
             if (attrIndex >= v->listSize())
                 throw AttrPathNotFound("list index %1% in selection path '%2%' is out of range", attrIndex, attrPath);
 
@@ -130,7 +130,7 @@ Pos findDerivationFilename(EvalState & state, Value & v, std::string what)
 
     Symbol file = state.symbols.create(filename);
 
-    return { file, lineno, 0 };
+    return { foFile, file, lineno, 0 };
 }
 
 

src/libexpr/attr-set.hh

@@ -76,7 +76,11 @@ public:
     {
         auto a = get(name);
         if (!a)
-            throw Error("attribute '%s' missing, at %s", name, pos);
+            throw Error({
+                .hint = hintfmt("attribute '%s' missing", name),
+                .errPos = pos
+            });
+
         return *a;
     }
 

src/libexpr/common-eval-args.cc

@@ -76,7 +76,7 @@ Path lookupFileArg(EvalState & state, string s)
     if (isUri(s)) {
         return state.store->toRealPath(
             fetchers::downloadTarball(
-                state.store, resolveUri(s), "source", false).storePath);
+                state.store, resolveUri(s), "source", false).first.storePath);
     } else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') {
         Path p = s.substr(1, s.size() - 2);
         return state.findFile(p);

src/libexpr/eval-cache.cc

@@ -2,6 +2,7 @@
 #include "sqlite.hh"
 #include "eval.hh"
 #include "eval-inline.hh"
+#include "store-api.hh"
 
 namespace nix::eval_cache {
 
@@ -11,16 +12,20 @@ create table if not exists Attributes (
     name        text,
     type        integer not null,
     value       text,
+    context     text,
     primary key (parent, name)
 );
 )sql";
 
 struct AttrDb
 {
+    std::atomic_bool failed{false};
+
     struct State
     {
         SQLite db;
         SQLiteStmt insertAttribute;
+        SQLiteStmt insertAttributeWithContext;
         SQLiteStmt queryAttribute;
         SQLiteStmt queryAttributes;
         std::unique_ptr<SQLiteTxn> txn;
@@ -33,7 +38,7 @@ struct AttrDb
     {
         auto state(_state->lock());
 
-        Path cacheDir = getCacheDir() + "/nix/eval-cache-v1";
+        Path cacheDir = getCacheDir() + "/nix/eval-cache-v2";
         createDirs(cacheDir);
 
         Path dbPath = cacheDir + "/" + fingerprint.to_string(Base16, false) + ".sqlite";
@@ -45,8 +50,11 @@ struct AttrDb
         state->insertAttribute.create(state->db,
             "insert or replace into Attributes(parent, name, type, value) values (?, ?, ?, ?)");
 
+        state->insertAttributeWithContext.create(state->db,
+            "insert or replace into Attributes(parent, name, type, value, context) values (?, ?, ?, ?, ?)");
+
         state->queryAttribute.create(state->db,
-            "select rowid, type, value from Attributes where parent = ? and name = ?");
+            "select rowid, type, value, context from Attributes where parent = ? and name = ?");
 
         state->queryAttributes.create(state->db,
             "select name from Attributes where parent = ?");
@@ -58,118 +66,168 @@ struct AttrDb
     {
         try {
             auto state(_state->lock());
-            state->txn->commit();
+            if (!failed)
+                state->txn->commit();
             state->txn.reset();
         } catch (...) {
             ignoreException();
         }
     }
 
+    template<typename F>
+    AttrId doSQLite(F && fun)
+    {
+        if (failed) return 0;
+        try {
+            return fun();
+        } catch (SQLiteError &) {
+            ignoreException();
+            failed = true;
+            return 0;
+        }
+    }
+
     AttrId setAttrs(
         AttrKey key,
         const std::vector<Symbol> & attrs)
     {
-        auto state(_state->lock());
+        return doSQLite([&]()
+        {
+            auto state(_state->lock());
 
-        state->insertAttribute.use()
-            (key.first)
-            (key.second)
-            (AttrType::FullAttrs)
-            (0, false).exec();
-
-        AttrId rowId = state->db.getLastInsertedRowId();
-        assert(rowId);
-
-        for (auto & attr : attrs)
             state->insertAttribute.use()
-                (rowId)
-                (attr)
-                (AttrType::Placeholder)
+                (key.first)
+                (key.second)
+                (AttrType::FullAttrs)
                 (0, false).exec();
 
-        return rowId;
+            AttrId rowId = state->db.getLastInsertedRowId();
+            assert(rowId);
+
+            for (auto & attr : attrs)
+                state->insertAttribute.use()
+                    (rowId)
+                    (attr)
+                    (AttrType::Placeholder)
+                    (0, false).exec();
+
+            return rowId;
+        });
     }
 
     AttrId setString(
         AttrKey key,
-        std::string_view s)
+        std::string_view s,
+        const char * * context = nullptr)
     {
-        auto state(_state->lock());
+        return doSQLite([&]()
+        {
+            auto state(_state->lock());
 
-        state->insertAttribute.use()
-            (key.first)
-            (key.second)
-            (AttrType::String)
-            (s).exec();
+            if (context) {
+                std::string ctx;
+                for (const char * * p = context; *p; ++p) {
+                    if (p != context) ctx.push_back(' ');
+                    ctx.append(*p);
+                }
+                state->insertAttributeWithContext.use()
+                    (key.first)
+                    (key.second)
+                    (AttrType::String)
+                    (s)
+                    (ctx).exec();
+            } else {
+                state->insertAttribute.use()
+                    (key.first)
+                    (key.second)
+                    (AttrType::String)
+                (s).exec();
+            }
 
-        return state->db.getLastInsertedRowId();
+            return state->db.getLastInsertedRowId();
+        });
     }
 
     AttrId setBool(
         AttrKey key,
         bool b)
     {
-        auto state(_state->lock());
+        return doSQLite([&]()
+        {
+            auto state(_state->lock());
 
-        state->insertAttribute.use()
-            (key.first)
-            (key.second)
-            (AttrType::Bool)
-            (b ? 1 : 0).exec();
+            state->insertAttribute.use()
+                (key.first)
+                (key.second)
+                (AttrType::Bool)
+                (b ? 1 : 0).exec();
 
-        return state->db.getLastInsertedRowId();
+            return state->db.getLastInsertedRowId();
+        });
     }
 
     AttrId setPlaceholder(AttrKey key)
     {
-        auto state(_state->lock());
+        return doSQLite([&]()
+        {
+            auto state(_state->lock());
 
-        state->insertAttribute.use()
-            (key.first)
-            (key.second)
-            (AttrType::Placeholder)
-            (0, false).exec();
+            state->insertAttribute.use()
+                (key.first)
+                (key.second)
+                (AttrType::Placeholder)
+                (0, false).exec();
 
-        return state->db.getLastInsertedRowId();
+            return state->db.getLastInsertedRowId();
+        });
     }
 
     AttrId setMissing(AttrKey key)
     {
-        auto state(_state->lock());
+        return doSQLite([&]()
+        {
+            auto state(_state->lock());
 
-        state->insertAttribute.use()
-            (key.first)
-            (key.second)
-            (AttrType::Missing)
-            (0, false).exec();
+            state->insertAttribute.use()
+                (key.first)
+                (key.second)
+                (AttrType::Missing)
+                (0, false).exec();
 
-        return state->db.getLastInsertedRowId();
+            return state->db.getLastInsertedRowId();
+        });
     }
 
     AttrId setMisc(AttrKey key)
     {
-        auto state(_state->lock());
+        return doSQLite([&]()
+        {
+            auto state(_state->lock());
 
-        state->insertAttribute.use()
-            (key.first)
-            (key.second)
-            (AttrType::Misc)
-            (0, false).exec();
+            state->insertAttribute.use()
+                (key.first)
+                (key.second)
+                (AttrType::Misc)
+                (0, false).exec();
 
-        return state->db.getLastInsertedRowId();
+            return state->db.getLastInsertedRowId();
+        });
     }
 
     AttrId setFailed(AttrKey key)
     {
-        auto state(_state->lock());
+        return doSQLite([&]()
+        {
+            auto state(_state->lock());
 
-        state->insertAttribute.use()
-            (key.first)
-            (key.second)
-            (AttrType::Failed)
-            (0, false).exec();
+            state->insertAttribute.use()
+                (key.first)
+                (key.second)
+                (AttrType::Failed)
+                (0, false).exec();
 
-        return state->db.getLastInsertedRowId();
+            return state->db.getLastInsertedRowId();
+        });
     }
 
     std::optional<std::pair<AttrId, AttrValue>> getAttr(
@@ -195,8 +253,13 @@ struct AttrDb
                     attrs.push_back(symbols.create(queryAttributes.getStr(0)));
                 return {{rowId, attrs}};
             }
-            case AttrType::String:
-                return {{rowId, queryAttribute.getStr(2)}};
+            case AttrType::String: {
+                std::vector<std::pair<Path, std::string>> context;
+                if (!queryAttribute.isNull(3))
+                    for (auto & s : tokenizeString<std::vector<std::string>>(queryAttribute.getStr(3), ";"))
+                        context.push_back(decodeContext(s));
+                return {{rowId, string_t{queryAttribute.getStr(2), context}}};
+            }
             case AttrType::Bool:
                 return {{rowId, queryAttribute.getInt(2) != 0}};
             case AttrType::Missing:
@@ -211,12 +274,22 @@ struct AttrDb
     }
 };
 
+static std::shared_ptr<AttrDb> makeAttrDb(const Hash & fingerprint)
+{
+    try {
+        return std::make_shared<AttrDb>(fingerprint);
+    } catch (SQLiteError &) {
+        ignoreException();
+        return nullptr;
+    }
+}
+
 EvalCache::EvalCache(
     bool useCache,
     const Hash & fingerprint,
     EvalState & state,
     RootLoader rootLoader)
-    : db(useCache ? std::make_shared<AttrDb>(fingerprint) : nullptr)
+    : db(useCache ? makeAttrDb(fingerprint) : nullptr)
     , state(state)
     , rootLoader(rootLoader)
 {
@@ -319,7 +392,9 @@ Value & AttrCursor::forceValue()
 
     if (root->db && (!cachedValue || std::get_if<placeholder_t>(&cachedValue->second))) {
         if (v.type == tString)
-            cachedValue = {root->db->setString(getKey(), v.string.s), v.string.s};
+            cachedValue = {root->db->setString(getKey(), v.string.s, v.string.context), v.string.s};
+        else if (v.type == tPath)
+            cachedValue = {root->db->setString(getKey(), v.path), v.path};
         else if (v.type == tBool)
             cachedValue = {root->db->setBool(getKey(), v.boolean), v.boolean};
         else if (v.type == tAttrs)
@@ -424,7 +499,29 @@ std::string AttrCursor::getString()
         if (!cachedValue)
             cachedValue = root->db->getAttr(getKey(), root->state.symbols);
         if (cachedValue && !std::get_if<placeholder_t>(&cachedValue->second)) {
-            if (auto s = std::get_if<std::string>(&cachedValue->second)) {
+            if (auto s = std::get_if<string_t>(&cachedValue->second)) {
+                debug("using cached string attribute '%s'", getAttrPathStr());
+                return s->first;
+            } else
+                throw TypeError("'%s' is not a string", getAttrPathStr());
+        }
+    }
+
+    auto & v = forceValue();
+
+    if (v.type != tString && v.type != tPath)
+        throw TypeError("'%s' is not a string but %s", getAttrPathStr(), showType(v.type));
+
+    return v.type == tString ? v.string.s : v.path;
+}
+
+string_t AttrCursor::getStringWithContext()
+{
+    if (root->db) {
+        if (!cachedValue)
+            cachedValue = root->db->getAttr(getKey(), root->state.symbols);
+        if (cachedValue && !std::get_if<placeholder_t>(&cachedValue->second)) {
+            if (auto s = std::get_if<string_t>(&cachedValue->second)) {
                 debug("using cached string attribute '%s'", getAttrPathStr());
                 return *s;
             } else
@@ -434,10 +531,12 @@ std::string AttrCursor::getString()
 
     auto & v = forceValue();
 
-    if (v.type != tString)
-        throw TypeError("'%s' is not a string", getAttrPathStr());
-
-    return v.string.s;
+    if (v.type == tString)
+        return {v.string.s, v.getContext()};
+    else if (v.type == tPath)
+        return {v.path, {}};
+    else
+        throw TypeError("'%s' is not a string but %s", getAttrPathStr(), showType(v.type));
 }
 
 bool AttrCursor::getBool()
@@ -500,4 +599,19 @@ bool AttrCursor::isDerivation()
     return aType && aType->getString() == "derivation";
 }
 
+StorePath AttrCursor::forceDerivation()
+{
+    auto aDrvPath = getAttr(root->state.sDrvPath);
+    auto drvPath = root->state.store->parseStorePath(aDrvPath->getString());
+    if (!root->state.store->isValidPath(drvPath) && !settings.readOnlyMode) {
+        /* The eval cache contains 'drvPath', but the actual path has
+           been garbage-collected. So force it to be regenerated. */
+        aDrvPath->forceValue();
+        if (!root->state.store->isValidPath(drvPath))
+            throw Error("don't know how to recreate store derivation '%s'!",
+                root->state.store->printStorePath(drvPath));
+    }
+    return drvPath;
+}
+
 }

src/libexpr/eval-cache.hh

@@ -50,7 +50,17 @@ struct misc_t {};
 struct failed_t {};
 typedef uint64_t AttrId;
 typedef std::pair<AttrId, Symbol> AttrKey;
-typedef std::variant<std::vector<Symbol>, std::string, placeholder_t, missing_t, misc_t, failed_t, bool> AttrValue;
+typedef std::pair<std::string, std::vector<std::pair<Path, std::string>>> string_t;
+
+typedef std::variant<
+    std::vector<Symbol>,
+    string_t,
+    placeholder_t,
+    missing_t,
+    misc_t,
+    failed_t,
+    bool
+    > AttrValue;
 
 class AttrCursor : public std::enable_shared_from_this<AttrCursor>
 {
@@ -94,6 +104,8 @@ public:
 
     std::string getString();
 
+    string_t getStringWithContext();
+
     bool getBool();
 
     std::vector<Symbol> getAttrs();
@@ -101,6 +113,9 @@ public:
     bool isDerivation();
 
     Value & forceValue();
+
+    /* Force creation of the .drv file in the Nix store. */
+    StorePath forceDerivation();
 };
 
 }

src/libexpr/eval-inline.hh

@@ -7,20 +7,26 @@
 
 namespace nix {
 
-LocalNoInlineNoReturn(void throwEvalError(const char * s, const Pos & pos))
+LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s))
 {
-    throw EvalError(format(s) % pos);
+    throw EvalError({
+        .hint = hintfmt(s),
+        .errPos = pos
+    });
 }
 
 LocalNoInlineNoReturn(void throwTypeError(const char * s, const Value & v))
 {
-    throw TypeError(format(s) % showType(v));
+    throw TypeError(s, showType(v));
 }
 
 
-LocalNoInlineNoReturn(void throwTypeError(const char * s, const Value & v, const Pos & pos))
+LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s, const Value & v))
 {
-    throw TypeError(format(s) % showType(v) % pos);
+    throw TypeError({
+        .hint = hintfmt(s, showType(v)),
+        .errPos = pos
+    });
 }
 
 
@@ -43,7 +49,7 @@ void EvalState::forceValue(Value & v, const Pos & pos)
     else if (v.type == tApp)
         callFunction(*v.app.left, *v.app.right, v, noPos);
     else if (v.type == tBlackhole)
-        throwEvalError("infinite recursion encountered, at %1%", pos);
+        throwEvalError(pos, "infinite recursion encountered");
 }
 
 
@@ -59,7 +65,7 @@ inline void EvalState::forceAttrs(Value & v, const Pos & pos)
 {
     forceValue(v, pos);
     if (v.type != tAttrs)
-        throwTypeError("value is %1% while a set was expected, at %2%", v, pos);
+        throwTypeError(pos, "value is %1% while a set was expected", v);
 }
 
 
@@ -75,7 +81,7 @@ inline void EvalState::forceList(Value & v, const Pos & pos)
 {
     forceValue(v, pos);
     if (!v.isList())
-        throwTypeError("value is %1% while a list was expected, at %2%", v, pos);
+        throwTypeError(pos, "value is %1% while a list was expected", v);
 }
 
 /* Note: Various places expect the allocated memory to be zeroed. */

src/libexpr/eval.cc

@@ -8,7 +8,6 @@
 #include "filetransfer.hh"
 #include "json.hh"
 #include "function-trace.hh"
-#include "flake/flake.hh"
 
 #include <algorithm>
 #include <chrono>
@@ -349,10 +348,10 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
     , sOutputHash(symbols.create("outputHash"))
     , sOutputHashAlgo(symbols.create("outputHashAlgo"))
     , sOutputHashMode(symbols.create("outputHashMode"))
+    , sRecurseForDerivations(symbols.create("recurseForDerivations"))
     , sDescription(symbols.create("description"))
     , sSelf(symbols.create("self"))
     , sEpsilon(symbols.create(""))
-    , sRecurseForDerivations(symbols.create("recurseForDerivations"))
     , repair(NoRepair)
     , store(store)
     , baseEnv(allocEnv(128))
@@ -382,7 +381,7 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
 
             if (store->isInStore(r.second)) {
                 StorePathSet closure;
-                store->computeFSClosure(store->parseStorePath(store->toStorePath(r.second)), closure);
+                store->computeFSClosure(store->toStorePath(r.second).first, closure);
                 for (auto & path : closure)
                     allowedPaths->insert(store->printStorePath(path));
             } else
@@ -538,67 +537,84 @@ Value & EvalState::getBuiltin(const string & name)
 
 LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2))
 {
-    throw EvalError(format(s) % s2);
+    throw EvalError(s, s2);
 }
 
-LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2, const Pos & pos))
+LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s, const string & s2))
 {
-    throw EvalError(format(s) % s2 % pos);
+    throw EvalError({
+        .hint = hintfmt(s, s2),
+        .errPos = pos
+    });
 }
 
 LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2, const string & s3))
 {
-    throw EvalError(format(s) % s2 % s3);
+    throw EvalError(s, s2, s3);
 }
 
-LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2, const string & s3, const Pos & pos))
+LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s, const string & s2, const string & s3))
 {
-    throw EvalError(format(s) % s2 % s3 % pos);
+    throw EvalError({
+        .hint = hintfmt(s, s2, s3),
+        .errPos = pos
+    });
 }
 
-LocalNoInlineNoReturn(void throwEvalError(const char * s, const Symbol & sym, const Pos & p1, const Pos & p2))
+LocalNoInlineNoReturn(void throwEvalError(const Pos & p1, const char * s, const Symbol & sym, const Pos & p2))
 {
-    throw EvalError(format(s) % sym % p1 % p2);
+    // p1 is where the error occurred; p2 is a position mentioned in the message.
+    throw EvalError({
+        .hint = hintfmt(s, sym, p2),
+        .errPos = p1
+    });
 }
 
-LocalNoInlineNoReturn(void throwTypeError(const char * s, const Pos & pos))
+LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s))
 {
-    throw TypeError(format(s) % pos);
+    throw TypeError({
+        .hint = hintfmt(s),
+        .errPos = pos
+    });
 }
 
 LocalNoInlineNoReturn(void throwTypeError(const char * s, const string & s1))
 {
-    throw TypeError(format(s) % s1);
+    throw TypeError(s, s1);
 }
 
-LocalNoInlineNoReturn(void throwTypeError(const char * s, const ExprLambda & fun, const Symbol & s2, const Pos & pos))
+LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s, const ExprLambda & fun, const Symbol & s2))
 {
-    throw TypeError(format(s) % fun.showNamePos() % s2 % pos);
+    throw TypeError({
+        .hint = hintfmt(s, fun.showNamePos(), s2),
+        .errPos = pos
+    });
 }
 
-LocalNoInlineNoReturn(void throwAssertionError(const char * s, const string & s1, const Pos & pos))
+LocalNoInlineNoReturn(void throwAssertionError(const Pos & pos, const char * s, const string & s1))
 {
-    throw AssertionError(format(s) % s1 % pos);
+    throw AssertionError({
+        .hint = hintfmt(s, s1),
+        .errPos = pos
+    });
 }
 
-LocalNoInlineNoReturn(void throwUndefinedVarError(const char * s, const string & s1, const Pos & pos))
+LocalNoInlineNoReturn(void throwUndefinedVarError(const Pos & pos, const char * s, const string & s1))
 {
-    throw UndefinedVarError(format(s) % s1 % pos);
+    throw UndefinedVarError({
+        .hint = hintfmt(s, s1),
+        .errPos = pos
+    });
 }
 
-LocalNoInline(void addErrorPrefix(Error & e, const char * s, const string & s2))
+LocalNoInline(void addErrorTrace(Error & e, const char * s, const string & s2))
 {
-    e.addPrefix(format(s) % s2);
+    e.addTrace(std::nullopt, s, s2);
 }
 
-LocalNoInline(void addErrorPrefix(Error & e, const char * s, const ExprLambda & fun, const Pos & pos))
+LocalNoInline(void addErrorTrace(Error & e, const Pos & pos, const char * s, const string & s2))
 {
-    e.addPrefix(format(s) % fun.showNamePos() % pos);
-}
-
-LocalNoInline(void addErrorPrefix(Error & e, const char * s, const string & s2, const Pos & pos))
-{
-    e.addPrefix(format(s) % s2 % pos);
+    e.addTrace(pos, s, s2);
 }
 
 
@@ -651,7 +667,7 @@ inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval)
             return j->value;
         }
         if (!env->prevWith)
-            throwUndefinedVarError("undefined variable '%1%' at %2%", var.name, var.pos);
+            throwUndefinedVarError(var.pos, "undefined variable '%1%'", var.name);
         for (size_t l = env->prevWith; l; --l, env = env->up) ;
     }
 }
@@ -817,7 +833,7 @@ void EvalState::evalFile(const Path & path_, Value & v, bool mustBeTrivial)
             throw Error("file '%s' must be an attribute set", path);
         eval(e, v);
     } catch (Error & e) {
-        addErrorPrefix(e, "while evaluating the file '%1%':\n", path2);
+        addErrorTrace(e, "while evaluating the file '%1%':", path2);
         throw;
     }
 
@@ -854,7 +870,7 @@ inline bool EvalState::evalBool(Env & env, Expr * e, const Pos & pos)
     Value v;
     e->eval(*this, env, v);
     if (v.type != tBool)
-        throwTypeError("value is %1% while a Boolean was expected, at %2%", v, pos);
+        throwTypeError(pos, "value is %1% while a Boolean was expected", v);
     return v.boolean;
 }
 
@@ -968,7 +984,7 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v)
         Symbol nameSym = state.symbols.create(nameVal.string.s);
         Bindings::iterator j = v.attrs->find(nameSym);
         if (j != v.attrs->end())
-            throwEvalError("dynamic attribute '%1%' at %2% already defined at %3%", nameSym, i.pos, *j->pos);
+            throwEvalError(i.pos, "dynamic attribute '%1%' already defined at %2%", nameSym, *j->pos);
 
         i.valueExpr->setName(nameSym);
         /* Keep sorted order so find can catch duplicates */
@@ -1056,7 +1072,7 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v)
             } else {
                 state.forceAttrs(*vAttrs, pos);
                 if ((j = vAttrs->attrs->find(name)) == vAttrs->attrs->end())
-                    throwEvalError("attribute '%1%' missing, at %2%", name, pos);
+                    throwEvalError(pos, "attribute '%1%' missing", name);
             }
             vAttrs = j->value;
             pos2 = j->pos;
@@ -1067,8 +1083,8 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v)
 
     } catch (Error & e) {
         if (pos2 && pos2->file != state.sDerivationNix)
-            addErrorPrefix(e, "while evaluating the attribute '%1%' at %2%:\n",
-                showAttrPath(state, env, attrPath), *pos2);
+            addErrorTrace(e, *pos2, "while evaluating the attribute '%1%'",
+                showAttrPath(state, env, attrPath));
         throw;
     }
 
@@ -1182,7 +1198,7 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po
     }
 
     if (fun.type != tLambda)
-        throwTypeError("attempt to call something which is not a function but %1%, at %2%", fun, pos);
+        throwTypeError(pos, "attempt to call something which is not a function but %1%", fun);
 
     ExprLambda & lambda(*fun.lambda.fun);
 
@@ -1210,8 +1226,8 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po
         for (auto & i : lambda.formals->formals) {
             Bindings::iterator j = arg.attrs->find(i.name);
             if (j == arg.attrs->end()) {
-                if (!i.def) throwTypeError("%1% called without required argument '%2%', at %3%",
-                    lambda, i.name, pos);
+                if (!i.def) throwTypeError(pos, "%1% called without required argument '%2%'",
+                    lambda, i.name);
                 env2.values[displ++] = i.def->maybeThunk(*this, env2);
             } else {
                 attrsUsed++;
@@ -1226,7 +1242,7 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po
                user. */
             for (auto & i : *arg.attrs)
                 if (lambda.formals->argNames.find(i.name) == lambda.formals->argNames.end())
-                    throwTypeError("%1% called with unexpected argument '%2%', at %3%", lambda, i.name, pos);
+                    throwTypeError(pos, "%1% called with unexpected argument '%2%'", lambda, i.name);
             abort(); // can't happen
         }
     }
@@ -1236,11 +1252,15 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po
 
     /* Evaluate the body.  This is conditional on showTrace, because
        catching exceptions makes this function not tail-recursive. */
-    if (settings.showTrace)
+    if (loggerSettings.showTrace.get())
         try {
             lambda.body->eval(*this, env2, v);
         } catch (Error & e) {
-            addErrorPrefix(e, "while evaluating %1%, called from %2%:\n", lambda, pos);
+            addErrorTrace(e, lambda.pos, "while evaluating %s", 
+              (lambda.name.set() 
+                  ? "'" + (string) lambda.name + "'" 
+                  : "anonymous lambdaction"));
+            addErrorTrace(e, pos, "from call site%s", "");
             throw;
         }
     else
@@ -1315,7 +1335,7 @@ void ExprAssert::eval(EvalState & state, Env & env, Value & v)
     if (!state.evalBool(env, cond, pos)) {
         std::ostringstream out;
         cond->show(out);
-        throwAssertionError("assertion '%1%' failed at %2%", out.str(), pos);
+        throwAssertionError(pos, "assertion '%1%' failed at %2%", out.str());
     }
     body->eval(state, env, v);
 }
@@ -1467,14 +1487,14 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
                 nf = n;
                 nf += vTmp.fpoint;
             } else
-                throwEvalError("cannot add %1% to an integer, at %2%", showType(vTmp), pos);
+                throwEvalError(pos, "cannot add %1% to an integer", showType(vTmp));
         } else if (firstType == tFloat) {
             if (vTmp.type == tInt) {
                 nf += vTmp.integer;
             } else if (vTmp.type == tFloat) {
                 nf += vTmp.fpoint;
             } else
-                throwEvalError("cannot add %1% to a float, at %2%", showType(vTmp), pos);
+                throwEvalError(pos, "cannot add %1% to a float", showType(vTmp));
         } else
             s << state.coerceToString(pos, vTmp, context, false, firstType == tString);
     }
@@ -1485,7 +1505,7 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
         mkFloat(v, nf);
     else if (firstType == tPath) {
         if (!context.empty())
-            throwEvalError("a string that refers to a store path cannot be appended to a path, at %1%", pos);
+            throwEvalError(pos, "a string that refers to a store path cannot be appended to a path");
         auto path = canonPath(s.str());
         mkPath(v, path.c_str());
     } else
@@ -1515,7 +1535,7 @@ void EvalState::forceValueDeep(Value & v)
                 try {
                     recurse(*i.value);
                 } catch (Error & e) {
-                    addErrorPrefix(e, "while evaluating the attribute '%1%' at %2%:\n", i.name, *i.pos);
+                    addErrorTrace(e, *i.pos, "while evaluating the attribute '%1%'", i.name);
                     throw;
                 }
         }
@@ -1534,7 +1554,7 @@ NixInt EvalState::forceInt(Value & v, const Pos & pos)
 {
     forceValue(v, pos);
     if (v.type != tInt)
-        throwTypeError("value is %1% while an integer was expected, at %2%", v, pos);
+        throwTypeError(pos, "value is %1% while an integer was expected", v);
     return v.integer;
 }
 
@@ -1545,7 +1565,7 @@ NixFloat EvalState::forceFloat(Value & v, const Pos & pos)
     if (v.type == tInt)
         return v.integer;
     else if (v.type != tFloat)
-        throwTypeError("value is %1% while a float was expected, at %2%", v, pos);
+        throwTypeError(pos, "value is %1% while a float was expected", v);
     return v.fpoint;
 }
 
@@ -1554,7 +1574,7 @@ bool EvalState::forceBool(Value & v, const Pos & pos)
 {
     forceValue(v, pos);
     if (v.type != tBool)
-        throwTypeError("value is %1% while a Boolean was expected, at %2%", v, pos);
+        throwTypeError(pos, "value is %1% while a Boolean was expected", v);
     return v.boolean;
 }
 
@@ -1569,7 +1589,7 @@ void EvalState::forceFunction(Value & v, const Pos & pos)
 {
     forceValue(v, pos);
     if (v.type != tLambda && v.type != tPrimOp && v.type != tPrimOpApp && !isFunctor(v))
-        throwTypeError("value is %1% while a function was expected, at %2%", v, pos);
+        throwTypeError(pos, "value is %1% while a function was expected", v);
 }
 
 
@@ -1578,7 +1598,7 @@ string EvalState::forceString(Value & v, const Pos & pos)
     forceValue(v, pos);
     if (v.type != tString) {
         if (pos)
-            throwTypeError("value is %1% while a string was expected, at %2%", v, pos);
+            throwTypeError(pos, "value is %1% while a string was expected", v);
         else
             throwTypeError("value is %1% while a string was expected", v);
     }
@@ -1586,6 +1606,18 @@ string EvalState::forceString(Value & v, const Pos & pos)
 }
 
 
+/* Decode a context string ‘!<name>!<path>’ into a pair <path,
+   name>. */
+std::pair<string, string> decodeContext(std::string_view s)
+{
+    if (s.at(0) == '!') {
+        size_t index = s.find("!", 1);
+        return {std::string(s.substr(index + 1)), std::string(s.substr(1, index - 1))};
+    } else
+        return {s.at(0) == '/' ? std::string(s) : std::string(s.substr(1)), ""};
+}
+
+
 void copyContext(const Value & v, PathSet & context)
 {
     if (v.string.context)
@@ -1594,6 +1626,17 @@ void copyContext(const Value & v, PathSet & context)
 }
 
 
+std::vector<std::pair<Path, std::string>> Value::getContext()
+{
+    std::vector<std::pair<Path, std::string>> res;
+    assert(type == tString);
+    if (string.context)
+        for (const char * * p = string.context; *p; ++p)
+            res.push_back(decodeContext(*p));
+    return res;
+}
+
+
 string EvalState::forceString(Value & v, PathSet & context, const Pos & pos)
 {
     string s = forceString(v, pos);
@@ -1607,8 +1650,8 @@ string EvalState::forceStringNoCtx(Value & v, const Pos & pos)
     string s = forceString(v, pos);
     if (v.string.context) {
         if (pos)
-            throwEvalError("the string '%1%' is not allowed to refer to a store path (such as '%2%'), at %3%",
-                v.string.s, v.string.context[0], pos);
+            throwEvalError(pos, "the string '%1%' is not allowed to refer to a store path (such as '%2%')",
+                v.string.s, v.string.context[0]);
         else
             throwEvalError("the string '%1%' is not allowed to refer to a store path (such as '%2%')",
                 v.string.s, v.string.context[0]);
@@ -1664,7 +1707,7 @@ string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context,
             return *maybeString;
         }
         auto i = v.attrs->find(sOutPath);
-        if (i == v.attrs->end()) throwTypeError("cannot coerce a set to a string, at %1%", pos);
+        if (i == v.attrs->end()) throwTypeError(pos, "cannot coerce a set to a string");
         return coerceToString(pos, *i->value, context, coerceMore, copyToStore);
     }
 
@@ -1695,7 +1738,7 @@ string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context,
         }
     }
 
-    throwTypeError("cannot coerce %1% to a string, at %2%", v, pos);
+    throwTypeError(pos, "cannot coerce %1% to a string", v);
 }
 
 
@@ -1711,7 +1754,7 @@ string EvalState::copyPathToStore(PathSet & context, const Path & path)
     else {
         auto p = settings.readOnlyMode
             ? store->computeStorePathForPath(std::string(baseNameOf(path)), checkSourcePath(path)).first
-            : store->addToStore(std::string(baseNameOf(path)), checkSourcePath(path), true, htSHA256, defaultPathFilter, repair);
+            : store->addToStore(std::string(baseNameOf(path)), checkSourcePath(path), FileIngestionMethod::Recursive, htSHA256, defaultPathFilter, repair);
         dstPath = store->printStorePath(p);
         srcToStore.insert_or_assign(path, std::move(p));
         printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, dstPath);
@@ -1726,7 +1769,7 @@ Path EvalState::coerceToPath(const Pos & pos, Value & v, PathSet & context)
 {
     string path = coerceToString(pos, v, context, false, false);
     if (path == "" || path[0] != '/')
-        throwEvalError("string '%1%' doesn't represent an absolute path, at %2%", path, pos);
+        throwEvalError(pos, "string '%1%' doesn't represent an absolute path", path);
     return path;
 }
 
@@ -1933,8 +1976,10 @@ void EvalState::printStats()
 
 string ExternalValueBase::coerceToString(const Pos & pos, PathSet & context, bool copyMore, bool copyToStore) const
 {
-    throw TypeError(format("cannot coerce %1% to a string, at %2%") %
-        showType() % pos);
+    throw TypeError({
+        .hint = hintfmt("cannot coerce %1% to a string", showType()),
+        .errPos = pos
+    });
 }
 
 

src/libexpr/eval.hh

@@ -18,7 +18,7 @@ namespace nix {
 
 class Store;
 class EvalState;
-struct StorePath;
+class StorePath;
 enum RepairFlag : bool;
 
 
@@ -75,7 +75,8 @@ public:
         sFile, sLine, sColumn, sFunctor, sToString,
         sRight, sWrong, sStructuredAttrs, sBuilder, sArgs,
         sOutputHash, sOutputHashAlgo, sOutputHashMode,
-        sDescription, sSelf, sEpsilon, sRecurseForDerivations;
+        sRecurseForDerivations,
+        sDescription, sSelf, sEpsilon;
     Symbol sDerivationNix;
 
     /* If set, force copying files to the Nix store even if they
@@ -252,7 +253,7 @@ private:
     friend struct ExprAttrs;
     friend struct ExprLet;
 
-    Expr * parse(const char * text, const Path & path,
+    Expr * parse(const char * text, FileOrigin origin, const Path & path,
         const Path & basePath, StaticEnv & staticEnv);
 
 public:
@@ -332,7 +333,7 @@ string showType(const Value & v);
 
 /* Decode a context string ‘!<name>!<path>’ into a pair <path,
    name>. */
-std::pair<string, string> decodeContext(const string & s);
+std::pair<string, string> decodeContext(std::string_view s);
 
 /* If `path' refers to a directory, then append "/default.nix". */
 Path resolveExprPath(Path path);

src/libexpr/flake/call-flake.nix

@@ -8,14 +8,41 @@ let
     builtins.mapAttrs
       (key: node:
         let
+
           sourceInfo =
             if key == lockFile.root
             then rootSrc
-            else fetchTree ({ inherit (node.info) narHash; } // removeAttrs node.locked ["dir"]);
+            else fetchTree (node.info or {} // removeAttrs node.locked ["dir"]);
+
           subdir = if key == lockFile.root then rootSubdir else node.locked.dir or "";
+
           flake = import (sourceInfo + (if subdir != "" then "/" else "") + subdir + "/flake.nix");
-          inputs = builtins.mapAttrs (inputName: key: allNodes.${key}) (node.inputs or {});
+
+          inputs = builtins.mapAttrs
+            (inputName: inputSpec: allNodes.${resolveInput inputSpec})
+            (node.inputs or {});
+
+          # Resolve a input spec into a node name. An input spec is
+          # either a node name, or a 'follows' path from the root
+          # node.
+          resolveInput = inputSpec:
+              if builtins.isList inputSpec
+              then getInputByPath lockFile.root inputSpec
+              else inputSpec;
+
+          # Follow an input path (e.g. ["dwarffs" "nixpkgs"]) from the
+          # root node, returning the final node.
+          getInputByPath = nodeName: path:
+            if path == []
+            then nodeName
+            else
+              getInputByPath
+                # Since this could be a 'follows' input, call resolveInput.
+                (resolveInput lockFile.nodes.${nodeName}.inputs.${builtins.head path})
+                (builtins.tail path);
+
           outputs = flake.outputs (inputs // { self = result; });
+
           result = outputs // sourceInfo // { inherit inputs; inherit outputs; inherit sourceInfo; };
         in
           if node.flake or true then

src/libexpr/flake/flake.cc

@@ -12,25 +12,10 @@ using namespace flake;
 
 namespace flake {
 
-/* If 'allowLookup' is true, then resolve 'flakeRef' using the
-   registries. */
-static FlakeRef maybeLookupFlake(
-    ref<Store> store,
-    const FlakeRef & flakeRef,
-    bool allowLookup)
-{
-    if (!flakeRef.input->isDirect()) {
-        if (allowLookup)
-            return flakeRef.resolve(store);
-        else
-            throw Error("'%s' is an indirect flake reference, but registry lookups are not allowed", flakeRef);
-    } else
-        return flakeRef;
-}
+typedef std::pair<Tree, FlakeRef> FetchedFlake;
+typedef std::vector<std::pair<FlakeRef, FetchedFlake>> FlakeCache;
 
-typedef std::vector<std::pair<FlakeRef, FlakeRef>> FlakeCache;
-
-static FlakeRef lookupInFlakeCache(
+static std::optional<FetchedFlake> lookupInFlakeCache(
     const FlakeCache & flakeCache,
     const FlakeRef & flakeRef)
 {
@@ -38,69 +23,50 @@ static FlakeRef lookupInFlakeCache(
     for (auto & i : flakeCache) {
         if (flakeRef == i.first) {
             debug("mapping '%s' to previously seen input '%s' -> '%s",
-                flakeRef, i.first, i.second);
+                flakeRef, i.first, i.second.second);
             return i.second;
         }
     }
 
-    return flakeRef;
+    return std::nullopt;
 }
 
 static std::tuple<fetchers::Tree, FlakeRef, FlakeRef> fetchOrSubstituteTree(
     EvalState & state,
     const FlakeRef & originalRef,
-    std::optional<TreeInfo> treeInfo,
     bool allowLookup,
     FlakeCache & flakeCache)
 {
-    /* The tree may already be in the Nix store, or it could be
-       substituted (which is often faster than fetching from the
-       original source). So check that. */
-    if (treeInfo && originalRef.input->isDirect() && originalRef.input->isImmutable()) {
-        try {
-            auto storePath = treeInfo->computeStorePath(*state.store);
+    auto fetched = lookupInFlakeCache(flakeCache, originalRef);
+    FlakeRef resolvedRef = originalRef;
 
-            state.store->ensurePath(storePath);
-
-            debug("using substituted/cached input '%s' in '%s'",
-                originalRef, state.store->printStorePath(storePath));
-
-            auto actualPath = state.store->toRealPath(storePath);
-
-            if (state.allowedPaths)
-                state.allowedPaths->insert(actualPath);
-
-            return {
-                Tree {
-                    .actualPath = actualPath,
-                    .storePath = std::move(storePath),
-                    .info = *treeInfo,
-                },
-                originalRef,
-                originalRef
-            };
-        } catch (Error & e) {
-            debug("substitution of input '%s' failed: %s", originalRef, e.what());
+    if (!fetched) {
+        if (originalRef.input.isDirect()) {
+            fetched.emplace(originalRef.fetchTree(state.store));
+        } else {
+            if (allowLookup) {
+                resolvedRef = originalRef.resolve(state.store);
+                auto fetchedResolved = lookupInFlakeCache(flakeCache, originalRef);
+                if (!fetchedResolved) fetchedResolved.emplace(resolvedRef.fetchTree(state.store));
+                flakeCache.push_back({resolvedRef, fetchedResolved.value()});
+                fetched.emplace(fetchedResolved.value());
+            }
+            else {
+                throw Error("'%s' is an indirect flake reference, but registry lookups are not allowed", originalRef);
+            }
         }
+        flakeCache.push_back({originalRef, fetched.value()});
     }
 
-    auto resolvedRef = lookupInFlakeCache(flakeCache,
-        maybeLookupFlake(state.store,
-            lookupInFlakeCache(flakeCache, originalRef), allowLookup));
-
-    auto [tree, lockedRef] = resolvedRef.fetchTree(state.store);
+    auto [tree, lockedRef] = fetched.value();
 
     debug("got tree '%s' from '%s'",
         state.store->printStorePath(tree.storePath), lockedRef);
 
-    flakeCache.push_back({originalRef, lockedRef});
-    flakeCache.push_back({resolvedRef, lockedRef});
-
     if (state.allowedPaths)
         state.allowedPaths->insert(tree.actualPath);
 
-    if (treeInfo)
-        assert(tree.storePath == treeInfo->computeStorePath(*state.store));
+    assert(!originalRef.input.getNarHash() || tree.storePath == originalRef.input.computeStorePath(*state.store));
 
     return {std::move(tree), resolvedRef, lockedRef};
 }
@@ -123,9 +89,7 @@ static FlakeInput parseFlakeInput(EvalState & state,
 {
     expectType(state, tAttrs, *value, pos);
 
-    FlakeInput input {
-        .ref = FlakeRef::fromAttrs({{"type", "indirect"}, {"id", inputName}})
-    };
+    FlakeInput input;
 
     auto sInputs = state.symbols.create("inputs");
     auto sUrl = state.symbols.create("url");
@@ -158,7 +122,7 @@ static FlakeInput parseFlakeInput(EvalState & state,
                         attr.name, showType(*attr.value));
             }
         } catch (Error & e) {
-            e.addPrefix(fmt("in flake attribute '%s' at '%s':\n", attr.name, *attr.pos));
+            e.addTrace(*attr.pos, hintfmt("in flake attribute '%s'", attr.name));
             throw;
         }
     }
@@ -167,7 +131,7 @@ static FlakeInput parseFlakeInput(EvalState & state,
         try {
             input.ref = FlakeRef::fromAttrs(attrs);
         } catch (Error & e) {
-            e.addPrefix(fmt("in flake input at '%s':\n", pos));
+            e.addTrace(pos, hintfmt("in flake input"));
             throw;
         }
     else {
@@ -178,6 +142,9 @@ static FlakeInput parseFlakeInput(EvalState & state,
             input.ref = parseFlakeRef(*url, {}, true);
     }
 
+    if (!input.follows && !input.ref)
+        input.ref = FlakeRef::fromAttrs({{"type", "indirect"}, {"id", inputName}});
+
     return input;
 }
 
@@ -202,12 +169,11 @@ static std::map<FlakeId, FlakeInput> parseFlakeInputs(
 static Flake getFlake(
     EvalState & state,
     const FlakeRef & originalRef,
-    std::optional<TreeInfo> treeInfo,
     bool allowLookup,
     FlakeCache & flakeCache)
 {
     auto [sourceInfo, resolvedRef, lockedRef] = fetchOrSubstituteTree(
-        state, originalRef, treeInfo, allowLookup, flakeCache);
+        state, originalRef, allowLookup, flakeCache);
 
     // Guard against symlink attacks.
     auto flakeFile = canonPath(sourceInfo.actualPath + "/" + lockedRef.subdir + "/flake.nix");
@@ -228,7 +194,7 @@ static Flake getFlake(
     Value vInfo;
     state.evalFile(flakeFile, vInfo, true); // FIXME: symlink attack
 
-    expectType(state, tAttrs, vInfo, Pos(state.symbols.create(flakeFile), 0, 0));
+    expectType(state, tAttrs, vInfo, Pos(foFile, state.symbols.create(flakeFile), 0, 0));
 
     auto sEdition = state.symbols.create("edition"); // FIXME: remove soon
 
@@ -278,7 +244,7 @@ static Flake getFlake(
 Flake getFlake(EvalState & state, const FlakeRef & originalRef, bool allowLookup)
 {
     FlakeCache flakeCache;
-    return getFlake(state, originalRef, {}, allowLookup, flakeCache);
+    return getFlake(state, originalRef, allowLookup, flakeCache);
 }
 
 /* Compute an in-memory lock file for the specified top-level flake,
@@ -292,7 +258,7 @@ LockedFlake lockFlake(
 
     FlakeCache flakeCache;
 
-    auto flake = getFlake(state, topRef, {}, lockFlags.useRegistries, flakeCache);
+    auto flake = getFlake(state, topRef, lockFlags.useRegistries, flakeCache);
 
     // FIXME: symlink attack
     auto oldLockFile = LockFile::read(
@@ -310,7 +276,6 @@ LockedFlake lockFlake(
     LockFile newLockFile;
 
     std::vector<FlakeRef> parents;
-    std::map<InputPath, InputPath> follows;
 
     std::function<void(
         const FlakeInputs & flakeInputs,
@@ -325,7 +290,7 @@ LockedFlake lockFlake(
         const InputPath & inputPathPrefix,
         std::shared_ptr<const Node> oldNode)
     {
-        debug("computing lock file node '%s'", concatStringsSep("/", inputPathPrefix));
+        debug("computing lock file node '%s'", printInputPath(inputPathPrefix));
 
         /* Get the overrides (i.e. attributes of the form
            'inputs.nixops.inputs.nixpkgs.url = ...'). */
@@ -345,8 +310,8 @@ LockedFlake lockFlake(
         for (auto & [id, input2] : flakeInputs) {
             auto inputPath(inputPathPrefix);
             inputPath.push_back(id);
-            auto inputPathS = concatStringsSep("/", inputPath);
-            debug("computing input '%s'", concatStringsSep("/", inputPath));
+            auto inputPathS = printInputPath(inputPath);
+            debug("computing input '%s'", inputPathS);
 
             /* Do we have an override for this input from one of the
                ancestors? */
@@ -358,33 +323,36 @@ LockedFlake lockFlake(
             /* Resolve 'follows' later (since it may refer to an input
                path we haven't processed yet. */
             if (input.follows) {
-                if (hasOverride)
+                InputPath target;
+                if (hasOverride || input.absolute)
                     /* 'follows' from an override is relative to the
                        root of the graph. */
-                    follows.insert_or_assign(inputPath, *input.follows);
+                    target = *input.follows;
                 else {
                     /* Otherwise, it's relative to the current flake. */
-                    InputPath path(inputPathPrefix);
-                    for (auto & i : *input.follows) path.push_back(i);
-                    follows.insert_or_assign(inputPath, path);
+                    target = inputPathPrefix;
+                    for (auto & i : *input.follows) target.push_back(i);
                 }
+                debug("input '%s' follows '%s'", inputPathS, printInputPath(target));
+                node->inputs.insert_or_assign(id, target);
                 continue;
             }
 
+            assert(input.ref);
+
             /* Do we have an entry in the existing lock file? And we
                don't have a --update-input flag for this input? */
-            std::shared_ptr<const LockedNode> oldLock;
+            std::shared_ptr<LockedNode> oldLock;
 
             updatesUsed.insert(inputPath);
 
-            if (oldNode && !lockFlags.inputUpdates.count(inputPath)) {
-                auto oldLockIt = oldNode->inputs.find(id);
-                if (oldLockIt != oldNode->inputs.end())
-                    oldLock = std::dynamic_pointer_cast<const LockedNode>(oldLockIt->second);
-            }
+            if (oldNode && !lockFlags.inputUpdates.count(inputPath))
+                if (auto oldLock2 = get(oldNode->inputs, id))
+                    if (auto oldLock3 = std::get_if<0>(&*oldLock2))
+                        oldLock = *oldLock3;
 
             if (oldLock
-                && oldLock->originalRef == input.ref
+                && oldLock->originalRef == *input.ref
                 && !hasOverride)
             {
                 debug("keeping existing input '%s'", inputPathS);
@@ -393,7 +361,7 @@ LockedFlake lockFlake(
                    didn't change and there is no override from a
                    higher level flake. */
                 auto childNode = std::make_shared<LockedNode>(
-                    oldLock->lockedRef, oldLock->originalRef, oldLock->info, oldLock->isFlake);
+                    oldLock->lockedRef, oldLock->originalRef, oldLock->isFlake);
 
                 node->inputs.insert_or_assign(id, childNode);
 
@@ -409,7 +377,7 @@ LockedFlake lockFlake(
 
                 if (hasChildUpdate) {
                     auto inputFlake = getFlake(
-                        state, oldLock->lockedRef, oldLock->info, false, flakeCache);
+                        state, oldLock->lockedRef, false, flakeCache);
                     computeLocks(inputFlake.inputs, childNode, inputPath, oldLock);
                 } else {
                     /* No need to fetch this flake, we can be
@@ -419,17 +387,16 @@ LockedFlake lockFlake(
                     FlakeInputs fakeInputs;
 
                     for (auto & i : oldLock->inputs) {
-                        auto lockedNode = std::dynamic_pointer_cast<LockedNode>(i.second);
-                        // Note: this node is not locked in case
-                        // of a circular reference back to the root.
-                        if (lockedNode)
+                        if (auto lockedNode = std::get_if<0>(&i.second)) {
                             fakeInputs.emplace(i.first, FlakeInput {
-                                .ref = lockedNode->originalRef
+                                .ref = (*lockedNode)->originalRef,
+                                .isFlake = (*lockedNode)->isFlake,
+                            });
+                        } else if (auto follows = std::get_if<1>(&i.second)) {
+                            fakeInputs.emplace(i.first, FlakeInput {
+                                .follows = *follows,
+                                .absolute = true
                             });
-                        else {
-                            InputPath path(inputPath);
-                            path.push_back(i.first);
-                            follows.insert_or_assign(path, InputPath());
                         }
                     }
 
@@ -439,12 +406,13 @@ LockedFlake lockFlake(
             } else {
                 /* We need to create a new lock file entry. So fetch
                    this input. */
+                debug("creating new input '%s'", inputPathS);
 
-                if (!lockFlags.allowMutable && !input.ref.input->isImmutable())
+                if (!lockFlags.allowMutable && !input.ref->input.isImmutable())
                     throw Error("cannot update flake input '%s' in pure mode", inputPathS);
 
                 if (input.isFlake) {
-                    auto inputFlake = getFlake(state, input.ref, {}, lockFlags.useRegistries, flakeCache);
+                    auto inputFlake = getFlake(state, *input.ref, lockFlags.useRegistries, flakeCache);
 
                     /* Note: in case of an --override-input, we use
                        the *original* ref (input2.ref) for the
@@ -454,15 +422,15 @@ LockedFlake lockFlake(
                        file. That is, overrides are sticky unless you
                        use --no-write-lock-file. */
                     auto childNode = std::make_shared<LockedNode>(
-                        inputFlake.lockedRef, input2.ref, inputFlake.sourceInfo->info);
+                        inputFlake.lockedRef, input2.ref ? *input2.ref : *input.ref);
 
                     node->inputs.insert_or_assign(id, childNode);
 
                     /* Guard against circular flake imports. */
                     for (auto & parent : parents)
-                        if (parent == input.ref)
+                        if (parent == *input.ref)
                             throw Error("found circular import of flake '%s'", parent);
-                    parents.push_back(input.ref);
+                    parents.push_back(*input.ref);
                     Finally cleanup([&]() { parents.pop_back(); });
 
                     /* Recursively process the inputs of this
@@ -479,9 +447,9 @@ LockedFlake lockFlake(
 
                 else {
                     auto [sourceInfo, resolvedRef, lockedRef] = fetchOrSubstituteTree(
-                        state, input.ref, {}, lockFlags.useRegistries, flakeCache);
+                        state, *input.ref, lockFlags.useRegistries, flakeCache);
                     node->inputs.insert_or_assign(id,
-                        std::make_shared<LockedNode>(lockedRef, input.ref, sourceInfo.info, false));
+                        std::make_shared<LockedNode>(lockedRef, *input.ref, false));
                 }
             }
         }
@@ -491,50 +459,27 @@ LockedFlake lockFlake(
         flake.inputs, newLockFile.root, {},
         lockFlags.recreateLockFile ? nullptr : oldLockFile.root);
 
-    /* Insert edges for 'follows' overrides. */
-    for (auto & [from, to] : follows) {
-        debug("adding 'follows' node from '%s' to '%s'",
-            concatStringsSep("/", from),
-            concatStringsSep("/", to));
-
-        assert(!from.empty());
-
-        InputPath fromParent(from);
-        fromParent.pop_back();
-
-        auto fromParentNode = newLockFile.root->findInput(fromParent);
-        assert(fromParentNode);
-
-        auto toNode = newLockFile.root->findInput(to);
-        if (!toNode)
-            throw Error("flake input '%s' follows non-existent flake input '%s'",
-                concatStringsSep("/", from),
-                concatStringsSep("/", to));
-
-        fromParentNode->inputs.insert_or_assign(from.back(), toNode);
-    }
-
     for (auto & i : lockFlags.inputOverrides)
         if (!overridesUsed.count(i.first))
             warn("the flag '--override-input %s %s' does not match any input",
-                concatStringsSep("/", i.first), i.second);
+                printInputPath(i.first), i.second);
 
     for (auto & i : lockFlags.inputUpdates)
         if (!updatesUsed.count(i))
-            warn("the flag '--update-input %s' does not match any input", concatStringsSep("/", i));
+            warn("the flag '--update-input %s' does not match any input", printInputPath(i));
+
+    /* Check 'follows' inputs. */
+    newLockFile.check();
 
     debug("new lock file: %s", newLockFile);
 
     /* Check whether we need to / can write the new lock file. */
     if (!(newLockFile == oldLockFile)) {
 
-        auto diff = diffLockFiles(oldLockFile, newLockFile);
-
-        if (!(oldLockFile == LockFile()))
-            printInfo("inputs of flake '%s' changed:\n%s", topRef, chomp(diff));
+        auto diff = LockFile::diff(oldLockFile, newLockFile);
 
         if (lockFlags.writeLockFile) {
-            if (auto sourcePath = topRef.input->getSourcePath()) {
+            if (auto sourcePath = topRef.input.getSourcePath()) {
                 if (!newLockFile.isImmutable()) {
                     if (settings.warnDirty)
                         warn("will not write lock file of flake '%s' because it has a mutable input", topRef);
@@ -548,14 +493,18 @@ LockedFlake lockFlake(
 
                     bool lockFileExists = pathExists(path);
 
-                    if (lockFileExists)
-                        warn("updating lock file '%s'", path);
-                    else
+                    if (lockFileExists) {
+                        auto s = chomp(diff);
+                        if (s.empty())
+                            warn("updating lock file '%s'", path);
+                        else
+                            warn("updating lock file '%s':\n%s", path, s);
+                    } else
                         warn("creating lock file '%s'", path);
 
                     newLockFile.write(path);
 
-                    topRef.input->markChangedFile(
+                    topRef.input.markChangedFile(
                         (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock",
                         lockFlags.commitLockFile
                         ? std::optional<std::string>(fmt("%s: %s\n\nFlake input changes:\n\n%s",
@@ -567,25 +516,25 @@ LockedFlake lockFlake(
                        also just clear the 'rev' field... */
                     auto prevLockedRef = flake.lockedRef;
                     FlakeCache dummyCache;
-                    flake = getFlake(state, topRef, {}, lockFlags.useRegistries, dummyCache);
+                    flake = getFlake(state, topRef, lockFlags.useRegistries, dummyCache);
 
                     if (lockFlags.commitLockFile &&
-                        flake.lockedRef.input->getRev() &&
-                        prevLockedRef.input->getRev() != flake.lockedRef.input->getRev())
-                        warn("committed new revision '%s'", flake.lockedRef.input->getRev()->gitRev());
+                        flake.lockedRef.input.getRev() &&
+                        prevLockedRef.input.getRev() != flake.lockedRef.input.getRev())
+                        warn("committed new revision '%s'", flake.lockedRef.input.getRev()->gitRev());
 
                     /* Make sure that we picked up the change,
                        i.e. the tree should usually be dirty
                        now. Corner case: we could have reverted from a
                        dirty to a clean tree! */
                     if (flake.lockedRef.input == prevLockedRef.input
-                        && !flake.lockedRef.input->isImmutable())
+                        && !flake.lockedRef.input.isImmutable())
                         throw Error("'%s' did not change after I updated its 'flake.lock' file; is 'flake.lock' under version control?", flake.originalRef);
                 }
             } else
                 throw Error("cannot write modified lock file of flake '%s' (use '--no-write-lock-file' to ignore)", topRef);
         } else
-            warn("not writing modified lock file of flake '%s'", topRef);
+            warn("not writing modified lock file of flake '%s':\n%s", topRef, chomp(diff));
     }
 
     return LockedFlake { .flake = std::move(flake), .lockFile = std::move(newLockFile) };
@@ -625,7 +574,7 @@ static void prim_getFlake(EvalState & state, const Pos & pos, Value * * args, Va
 {
     auto flakeRefS = state.forceStringNoCtx(*args[0], pos);
     auto flakeRef = parseFlakeRef(flakeRefS, {}, true);
-    if (evalSettings.pureEval && !flakeRef.input->isImmutable())
+    if (evalSettings.pureEval && !flakeRef.input.isImmutable())
         throw Error("cannot call 'getFlake' on mutable flake reference '%s', at %s (use --impure to override)", flakeRefS, pos);
 
     callFlake(state,
@@ -638,7 +587,7 @@ static void prim_getFlake(EvalState & state, const Pos & pos, Value * * args, Va
         v);
 }
 
-static RegisterPrimOp r2("getFlake", 1, prim_getFlake);
+static RegisterPrimOp r2("__getFlake", 1, prim_getFlake, "flakes");
 
 }
 
@@ -650,8 +599,8 @@ Fingerprint LockedFlake::getFingerprint() const
     return hashString(htSHA256,
         fmt("%s;%d;%d;%s",
             flake.sourceInfo->storePath.to_string(),
-            flake.sourceInfo->info.revCount.value_or(0),
-            flake.sourceInfo->info.lastModified.value_or(0),
+            flake.lockedRef.input.getRevCount().value_or(0),
+            flake.lockedRef.input.getLastModified().value_or(0),
             lockFile));
 }
 

src/libexpr/flake/flake.hh

@@ -19,9 +19,10 @@ typedef std::map<FlakeId, FlakeInput> FlakeInputs;
 
 struct FlakeInput
 {
-    FlakeRef ref;
+    std::optional<FlakeRef> ref;
     bool isFlake = true;
     std::optional<InputPath> follows;
+    bool absolute = false; // whether 'follows' is relative to the flake root
     FlakeInputs overrides;
 };
 
@@ -104,7 +105,7 @@ void callFlake(
 void emitTreeAttrs(
     EvalState & state,
     const fetchers::Tree & tree,
-    std::shared_ptr<const fetchers::Input> input,
+    const fetchers::Input & input,
     Value & v);
 
 }

src/libexpr/flake/flakeref.cc

@@ -15,7 +15,7 @@ const static std::string subDirRegex = subDirElemRegex + "(?:/" + subDirElemRege
 
 std::string FlakeRef::to_string() const
 {
-    auto url = input->toURL();
+    auto url = input.toURL();
     if (subdir != "")
         url.query.insert_or_assign("dir", subdir);
     return url.to_string();
@@ -23,7 +23,7 @@ std::string FlakeRef::to_string() const
 
 fetchers::Attrs FlakeRef::toAttrs() const
 {
-    auto attrs = input->toAttrs();
+    auto attrs = input.toAttrs();
     if (subdir != "")
         attrs.emplace("dir", subdir);
     return attrs;
@@ -37,13 +37,13 @@ std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef)
 
 bool FlakeRef::operator ==(const FlakeRef & other) const
 {
-    return *input == *other.input && subdir == other.subdir;
+    return input == other.input && subdir == other.subdir;
 }
 
 FlakeRef FlakeRef::resolve(ref<Store> store) const
 {
     auto [input2, extraAttrs] = lookupInRegistries(store, input);
-    return FlakeRef(input2, fetchers::maybeGetStrAttr(extraAttrs, "dir").value_or(subdir));
+    return FlakeRef(std::move(input2), fetchers::maybeGetStrAttr(extraAttrs, "dir").value_or(subdir));
 }
 
 FlakeRef parseFlakeRef(
@@ -98,7 +98,7 @@ std::pair<FlakeRef, std::string> parseFlakeRefWithFragment(
         };
 
         return std::make_pair(
-            FlakeRef(inputFromURL(parsedURL), ""),
+            FlakeRef(Input::fromURL(parsedURL), ""),
             percentDecode(std::string(match[6])));
     }
 
@@ -142,8 +142,11 @@ std::pair<FlakeRef, std::string> parseFlakeRefWithFragment(
                     parsedURL.query.insert_or_assign("dir", subdir);
                 }
 
+                if (pathExists(flakeRoot + "/.git/shallow"))
+                    parsedURL.query.insert_or_assign("shallow", "1");
+
                 return std::make_pair(
-                    FlakeRef(inputFromURL(parsedURL), get(parsedURL.query, "dir").value_or("")),
+                    FlakeRef(Input::fromURL(parsedURL), get(parsedURL.query, "dir").value_or("")),
                     fragment);
             }
 
@@ -155,7 +158,7 @@ std::pair<FlakeRef, std::string> parseFlakeRefWithFragment(
         attrs.insert_or_assign("type", "path");
         attrs.insert_or_assign("path", path);
 
-        return std::make_pair(FlakeRef(inputFromAttrs(attrs), ""), fragment);
+        return std::make_pair(FlakeRef(Input::fromAttrs(std::move(attrs)), ""), fragment);
     }
 
     else {
@@ -163,7 +166,7 @@ std::pair<FlakeRef, std::string> parseFlakeRefWithFragment(
         std::string fragment;
         std::swap(fragment, parsedURL.fragment);
         return std::make_pair(
-            FlakeRef(inputFromURL(parsedURL), get(parsedURL.query, "dir").value_or("")),
+            FlakeRef(Input::fromURL(parsedURL), get(parsedURL.query, "dir").value_or("")),
             fragment);
     }
 }
@@ -183,14 +186,14 @@ FlakeRef FlakeRef::fromAttrs(const fetchers::Attrs & attrs)
     auto attrs2(attrs);
     attrs2.erase("dir");
     return FlakeRef(
-        fetchers::inputFromAttrs(attrs2),
+        fetchers::Input::fromAttrs(std::move(attrs2)),
         fetchers::maybeGetStrAttr(attrs, "dir").value_or(""));
 }
 
 std::pair<fetchers::Tree, FlakeRef> FlakeRef::fetchTree(ref<Store> store) const
 {
-    auto [tree, lockedInput] = input->fetchTree(store);
-    return {std::move(tree), FlakeRef(lockedInput, subdir)};
+    auto [tree, lockedInput] = input.fetch(store);
+    return {std::move(tree), FlakeRef(std::move(lockedInput), subdir)};
 }
 
 }

src/libexpr/flake/flakeref.hh

@@ -14,17 +14,15 @@ typedef std::string FlakeId;
 
 struct FlakeRef
 {
-    std::shared_ptr<const fetchers::Input> input;
+    fetchers::Input input;
 
     Path subdir;
 
     bool operator==(const FlakeRef & other) const;
 
-    FlakeRef(const std::shared_ptr<const fetchers::Input> & input, const Path & subdir)
-        : input(input), subdir(subdir)
-    {
-        assert(input);
-    }
+    FlakeRef(fetchers::Input && input, const Path & subdir)
+        : input(std::move(input)), subdir(subdir)
+    { }
 
     // FIXME: change to operator <<.
     std::string to_string() const;

src/libexpr/flake/lockfile.cc

@@ -5,46 +5,58 @@
 
 namespace nix::flake {
 
-FlakeRef flakeRefFromJson(const nlohmann::json & json)
-{
-    return FlakeRef::fromAttrs(jsonToAttrs(json));
-}
-
 FlakeRef getFlakeRef(
     const nlohmann::json & json,
-    const char * attr)
+    const char * attr,
+    const char * info)
 {
     auto i = json.find(attr);
-    if (i != json.end())
-        return flakeRefFromJson(*i);
+    if (i != json.end()) {
+        auto attrs = jsonToAttrs(*i);
+        // FIXME: remove when we drop support for version 5.
+        if (info) {
+            auto j = json.find(info);
+            if (j != json.end()) {
+                for (auto k : jsonToAttrs(*j))
+                    attrs.insert_or_assign(k.first, k.second);
+            }
+        }
+        return FlakeRef::fromAttrs(attrs);
+    }
 
     throw Error("attribute '%s' missing in lock file", attr);
 }
 
 LockedNode::LockedNode(const nlohmann::json & json)
-    : lockedRef(getFlakeRef(json, "locked"))
-    , originalRef(getFlakeRef(json, "original"))
-    , info(TreeInfo::fromJson(json))
+    : lockedRef(getFlakeRef(json, "locked", "info"))
+    , originalRef(getFlakeRef(json, "original", nullptr))
     , isFlake(json.find("flake") != json.end() ? (bool) json["flake"] : true)
 {
-    if (!lockedRef.input->isImmutable())
-        throw Error("lockfile contains mutable flakeref '%s'", lockedRef);
+    if (!lockedRef.input.isImmutable())
+        throw Error("lockfile contains mutable lock '%s'", attrsToJson(lockedRef.input.toAttrs()));
 }
 
 StorePath LockedNode::computeStorePath(Store & store) const
 {
-    return info.computeStorePath(store);
+    return lockedRef.input.computeStorePath(store);
 }
 
-std::shared_ptr<Node> Node::findInput(const InputPath & path)
+std::shared_ptr<Node> LockFile::findInput(const InputPath & path)
 {
-    auto pos = shared_from_this();
+    auto pos = root;
+
+    if (!pos) return {};
 
     for (auto & elem : path) {
-        auto i = pos->inputs.find(elem);
-        if (i == pos->inputs.end())
+        if (auto i = get(pos->inputs, elem)) {
+            if (auto node = std::get_if<0>(&*i))
+                pos = *node;
+            else if (auto follows = std::get_if<1>(&*i)) {
+                pos = findInput(*follows);
+                if (!pos) return {};
+            }
+        } else
             return {};
-        pos = i->second;
     }
 
     return pos;
@@ -53,7 +65,7 @@ std::shared_ptr<Node> Node::findInput(const InputPath & path)
 LockFile::LockFile(const nlohmann::json & json, const Path & path)
 {
     auto version = json.value("version", 0);
-    if (version != 5)
+    if (version < 5 || version > 7)
         throw Error("lock file '%s' has unsupported version %d", path, version);
 
     std::unordered_map<std::string, std::shared_ptr<Node>> nodeMap;
@@ -64,21 +76,37 @@ LockFile::LockFile(const nlohmann::json & json, const Path & path)
     {
         if (jsonNode.find("inputs") == jsonNode.end()) return;
         for (auto & i : jsonNode["inputs"].items()) {
-            std::string inputKey = i.value();
-            auto k = nodeMap.find(inputKey);
-            if (k == nodeMap.end()) {
-                auto jsonNode2 = json["nodes"][inputKey];
-                auto input = std::make_shared<LockedNode>(jsonNode2);
-                k = nodeMap.insert_or_assign(inputKey, input).first;
-                getInputs(*input, jsonNode2);
+            if (i.value().is_array()) {
+                InputPath path;
+                for (auto & j : i.value())
+                    path.push_back(j);
+                node.inputs.insert_or_assign(i.key(), path);
+            } else {
+                std::string inputKey = i.value();
+                auto k = nodeMap.find(inputKey);
+                if (k == nodeMap.end()) {
+                    auto jsonNode2 = json["nodes"][inputKey];
+                    auto input = std::make_shared<LockedNode>(jsonNode2);
+                    k = nodeMap.insert_or_assign(inputKey, input).first;
+                    getInputs(*input, jsonNode2);
+                }
+                if (auto child = std::dynamic_pointer_cast<LockedNode>(k->second))
+                    node.inputs.insert_or_assign(i.key(), child);
+                else
+                    // FIXME: replace by follows node
+                    throw Error("lock file contains cycle to root node");
             }
-            node.inputs.insert_or_assign(i.key(), k->second);
         }
     };
 
     std::string rootKey = json["root"];
     nodeMap.insert_or_assign(rootKey, root);
     getInputs(*root, json["nodes"][rootKey]);
+
+    // FIXME: check that there are no cycles in version >= 7. Cycles
+    // between inputs are only possible using 'follows' indirections.
+    // Once we drop support for version <= 6, we can simplify the code
+    // a bit since we don't need to worry about cycles.
 }
 
 nlohmann::json LockFile::toJson() const
@@ -111,15 +139,22 @@ nlohmann::json LockFile::toJson() const
 
         if (!node->inputs.empty()) {
             auto inputs = nlohmann::json::object();
-            for (auto & i : node->inputs)
-                inputs[i.first] = dumpNode(i.first, i.second);
+            for (auto & i : node->inputs) {
+                if (auto child = std::get_if<0>(&i.second)) {
+                    inputs[i.first] = dumpNode(i.first, *child);
+                } else if (auto follows = std::get_if<1>(&i.second)) {
+                    auto arr = nlohmann::json::array();
+                    for (auto & x : *follows)
+                        arr.push_back(x);
+                    inputs[i.first] = std::move(arr);
+                }
+            }
             n["inputs"] = std::move(inputs);
         }
 
         if (auto lockedNode = std::dynamic_pointer_cast<const LockedNode>(node)) {
             n["original"] = fetchers::attrsToJson(lockedNode->originalRef.toAttrs());
             n["locked"] = fetchers::attrsToJson(lockedNode->lockedRef.toAttrs());
-            n["info"] = lockedNode->info.toJson();
             if (!lockedNode->isFlake) n["flake"] = false;
         }
 
@@ -129,7 +164,7 @@ nlohmann::json LockFile::toJson() const
     };
 
     nlohmann::json json;
-    json["version"] = 5;
+    json["version"] = 7;
     json["root"] = dumpNode("root", root);
     json["nodes"] = std::move(nodes);
 
@@ -168,7 +203,9 @@ bool LockFile::isImmutable() const
     visit = [&](std::shared_ptr<const Node> node)
     {
         if (!nodes.insert(node).second) return;
-        for (auto & i : node->inputs) visit(i.second);
+        for (auto & i : node->inputs)
+            if (auto child = std::get_if<0>(&i.second))
+                visit(*child);
     };
 
     visit(root);
@@ -176,7 +213,7 @@ bool LockFile::isImmutable() const
     for (auto & i : nodes) {
         if (i == root) continue;
         auto lockedNode = std::dynamic_pointer_cast<const LockedNode>(i);
-        if (lockedNode && !lockedNode->lockedRef.input->isImmutable()) return false;
+        if (lockedNode && !lockedNode->lockedRef.input.isImmutable()) return false;
     }
 
     return true;
@@ -194,37 +231,62 @@ InputPath parseInputPath(std::string_view s)
 
     for (auto & elem : tokenizeString<std::vector<std::string>>(s, "/")) {
         if (!std::regex_match(elem, flakeIdRegex))
-            throw Error("invalid flake input path element '%s'", elem);
+            throw UsageError("invalid flake input path element '%s'", elem);
         path.push_back(elem);
     }
 
     return path;
 }
 
-static void flattenLockFile(
-    std::shared_ptr<const Node> node,
-    const InputPath & prefix,
-    std::unordered_set<std::shared_ptr<const Node>> & done,
-    std::map<InputPath, std::shared_ptr<const LockedNode>> & res)
+std::map<InputPath, Node::Edge> LockFile::getAllInputs() const
 {
-    if (!done.insert(node).second) return;
+    std::unordered_set<std::shared_ptr<Node>> done;
+    std::map<InputPath, Node::Edge> res;
 
-    for (auto &[id, input] : node->inputs) {
-        auto inputPath(prefix);
-        inputPath.push_back(id);
-        if (auto lockedInput = std::dynamic_pointer_cast<const LockedNode>(input))
-            res.emplace(inputPath, lockedInput);
-        flattenLockFile(input, inputPath, done, res);
-    }
+    std::function<void(const InputPath & prefix, std::shared_ptr<Node> node)> recurse;
+
+    recurse = [&](const InputPath & prefix, std::shared_ptr<Node> node)
+    {
+        if (!done.insert(node).second) return;
+
+        for (auto &[id, input] : node->inputs) {
+            auto inputPath(prefix);
+            inputPath.push_back(id);
+            res.emplace(inputPath, input);
+            if (auto child = std::get_if<0>(&input))
+                recurse(inputPath, *child);
+        }
+    };
+
+    recurse({}, root);
+
+    return res;
 }
 
-std::string diffLockFiles(const LockFile & oldLocks, const LockFile & newLocks)
+std::ostream & operator <<(std::ostream & stream, const Node::Edge & edge)
 {
-    std::unordered_set<std::shared_ptr<const Node>> done;
-    std::map<InputPath, std::shared_ptr<const LockedNode>> oldFlat, newFlat;
-    flattenLockFile(oldLocks.root, {}, done, oldFlat);
-    done.clear();
-    flattenLockFile(newLocks.root, {}, done, newFlat);
+    if (auto node = std::get_if<0>(&edge))
+        stream << "'" << (*node)->lockedRef << "'";
+    else if (auto follows = std::get_if<1>(&edge))
+        stream << fmt("follows '%s'", printInputPath(*follows));
+    return stream;
+}
+
+static bool equals(const Node::Edge & e1, const Node::Edge & e2)
+{
+    if (auto n1 = std::get_if<0>(&e1))
+        if (auto n2 = std::get_if<0>(&e2))
+            return (*n1)->lockedRef == (*n2)->lockedRef;
+    if (auto f1 = std::get_if<1>(&e1))
+        if (auto f2 = std::get_if<1>(&e2))
+            return *f1 == *f2;
+    return false;
+}
+
+std::string LockFile::diff(const LockFile & oldLocks, const LockFile & newLocks)
+{
+    auto oldFlat = oldLocks.getAllInputs();
+    auto newFlat = newLocks.getAllInputs();
 
     auto i = oldFlat.begin();
     auto j = newFlat.begin();
@@ -232,18 +294,17 @@ std::string diffLockFiles(const LockFile & oldLocks, const LockFile & newLocks)
 
     while (i != oldFlat.end() || j != newFlat.end()) {
         if (j != newFlat.end() && (i == oldFlat.end() || i->first > j->first)) {
-            res += fmt("* Added '%s': '%s'\n", concatStringsSep("/", j->first), j->second->lockedRef);
+            res += fmt("* Added '%s': %s\n", printInputPath(j->first), j->second);
             ++j;
         } else if (i != oldFlat.end() && (j == newFlat.end() || i->first < j->first)) {
-            res += fmt("* Removed '%s'\n", concatStringsSep("/", i->first));
+            res += fmt("* Removed '%s'\n", printInputPath(i->first));
             ++i;
         } else {
-            if (!(i->second->lockedRef == j->second->lockedRef)) {
-                assert(i->second->lockedRef.to_string() != j->second->lockedRef.to_string());
-                res += fmt("* Updated '%s': '%s' -> '%s'\n",
-                    concatStringsSep("/", i->first),
-                    i->second->lockedRef,
-                    j->second->lockedRef);
+            if (!equals(i->second, j->second)) {
+                res += fmt("* Updated '%s': %s -> %s\n",
+                    printInputPath(i->first),
+                    i->second,
+                    j->second);
             }
             ++i;
             ++j;
@@ -253,4 +314,25 @@ std::string diffLockFiles(const LockFile & oldLocks, const LockFile & newLocks)
     return res;
 }
 
+void LockFile::check()
+{
+    auto inputs = getAllInputs();
+
+    for (auto & [inputPath, input] : inputs) {
+        if (auto follows = std::get_if<1>(&input)) {
+            if (!follows->empty() && !get(inputs, *follows))
+                throw Error("input '%s' follows a non-existent input '%s'",
+                    printInputPath(inputPath),
+                    printInputPath(*follows));
+        }
+    }
+}
+
+void check();
+
+std::string printInputPath(const InputPath & path)
+{
+    return concatStringsSep("/", path);
+}
+
 }

src/libexpr/flake/lockfile.hh

@@ -15,31 +15,31 @@ using namespace fetchers;
 
 typedef std::vector<FlakeId> InputPath;
 
+struct LockedNode;
+
 /* A node in the lock file. It has outgoing edges to other nodes (its
    inputs). Only the root node has this type; all other nodes have
    type LockedNode. */
 struct Node : std::enable_shared_from_this<Node>
 {
-    std::map<FlakeId, std::shared_ptr<Node>> inputs;
+    typedef std::variant<std::shared_ptr<LockedNode>, InputPath> Edge;
+
+    std::map<FlakeId, Edge> inputs;
 
     virtual ~Node() { }
-
-    std::shared_ptr<Node> findInput(const InputPath & path);
 };
 
 /* A non-root node in the lock file. */
 struct LockedNode : Node
 {
     FlakeRef lockedRef, originalRef;
-    TreeInfo info;
     bool isFlake = true;
 
     LockedNode(
         const FlakeRef & lockedRef,
         const FlakeRef & originalRef,
-        const TreeInfo & info,
         bool isFlake = true)
-        : lockedRef(lockedRef), originalRef(originalRef), info(info), isFlake(isFlake)
+        : lockedRef(lockedRef), originalRef(originalRef), isFlake(isFlake)
     { }
 
     LockedNode(const nlohmann::json & json);
@@ -65,13 +65,21 @@ struct LockFile
     bool isImmutable() const;
 
     bool operator ==(const LockFile & other) const;
+
+    std::shared_ptr<Node> findInput(const InputPath & path);
+
+    std::map<InputPath, Node::Edge> getAllInputs() const;
+
+    static std::string diff(const LockFile & oldLocks, const LockFile & newLocks);
+
+    /* Check that every 'follows' input target exists. */
+    void check();
 };
 
 std::ostream & operator <<(std::ostream & stream, const LockFile & lockFile);
 
 InputPath parseInputPath(std::string_view s);
 
-std::string diffLockFiles(const LockFile & oldLocks, const LockFile & newLocks);
+std::string printInputPath(const InputPath & path);
 
 }
-

src/libexpr/get-drvs.cc

@@ -1,7 +1,7 @@
 #include "get-drvs.hh"
 #include "util.hh"
 #include "eval-inline.hh"
-#include "derivations.hh"
+#include "store-api.hh"
 
 #include <cstring>
 #include <regex>

src/libexpr/lexer.l

@@ -127,14 +127,14 @@ or          { return OR_KW; }
               try {
                   yylval->n = boost::lexical_cast<int64_t>(yytext);
               } catch (const boost::bad_lexical_cast &) {
-                  throw ParseError(format("invalid integer '%1%'") % yytext);
+                  throw ParseError("invalid integer '%1%'", yytext);
               }
               return INT;
             }
 {FLOAT}     { errno = 0;
               yylval->nf = strtod(yytext, 0);
               if (errno != 0)
-                  throw ParseError(format("invalid float '%1%'") % yytext);
+                  throw ParseError("invalid float '%1%'", yytext);
               return FLOAT;
             }
 
@@ -219,4 +219,3 @@ or          { return OR_KW; }
             }
 
 %%
-

src/libexpr/local.mk

@@ -13,7 +13,7 @@ libexpr_SOURCES := \
 
 libexpr_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/libmain -I src/libexpr
 
-libexpr_LIBS = libutil libstore libfetchers libnixrust
+libexpr_LIBS = libutil libstore libfetchers
 
 libexpr_LDFLAGS =
 ifneq ($(OS), FreeBSD)

src/libexpr/nixexpr.cc

@@ -197,7 +197,22 @@ std::ostream & operator << (std::ostream & str, const Pos & pos)
     if (!pos)
         str << "undefined position";
     else
-        str << (format(ANSI_BOLD "%1%" ANSI_NORMAL ":%2%:%3%") % (string) pos.file % pos.line % pos.column).str();
+    {
+        auto f = format(ANSI_BOLD "%1%" ANSI_NORMAL ":%2%:%3%");
+        switch (pos.origin) {
+            case foFile:
+                f % (string) pos.file;
+                break;
+            case foStdin:
+            case foString:
+                f % "(string)";
+                break;
+            default:
+                throw Error("unhandled Pos origin!");
+        }
+        str << (f % pos.line % pos.column).str();
+    }
+
     return str;
 }
 
@@ -267,8 +282,11 @@ void ExprVar::bindVars(const StaticEnv & env)
     /* Otherwise, the variable must be obtained from the nearest
        enclosing `with'.  If there is no `with', then we can issue an
        "undefined variable" error now. */
-    if (withLevel == -1) throw UndefinedVarError(format("undefined variable '%1%' at %2%") % name % pos);
-
+    if (withLevel == -1)
+        throw UndefinedVarError({
+            .hint = hintfmt("undefined variable '%1%'", name),
+            .errPos = pos
+        });
     fromWith = true;
     this->level = withLevel;
 }

src/libexpr/nixexpr.hh

@@ -2,6 +2,7 @@
 
 #include "value.hh"
 #include "symbol-table.hh"
+#include "error.hh"
 
 #include <map>
 
@@ -23,11 +24,12 @@ MakeError(RestrictedPathError, Error);
 
 struct Pos
 {
+    FileOrigin origin;
     Symbol file;
     unsigned int line, column;
-    Pos() : line(0), column(0) { };
-    Pos(const Symbol & file, unsigned int line, unsigned int column)
-        : file(file), line(line), column(column) { };
+    Pos() : origin(foString), line(0), column(0) { };
+    Pos(FileOrigin origin, const Symbol & file, unsigned int line, unsigned int column)
+        : origin(origin), file(file), line(line), column(column) { };
     operator bool() const
     {
         return line != 0;
@@ -235,8 +237,10 @@ struct ExprLambda : Expr
         : pos(pos), arg(arg), matchAttrs(matchAttrs), formals(formals), body(body)
     {
         if (!arg.empty() && formals && formals->argNames.find(arg) != formals->argNames.end())
-            throw ParseError(format("duplicate formal function argument '%1%' at %2%")
-                % arg % pos);
+            throw ParseError({
+                .hint = hintfmt("duplicate formal function argument '%1%'", arg),
+                .errPos = pos
+            });
     };
     void setName(Symbol & name);
     string showNamePos() const;

src/libexpr/parser.y

@@ -30,8 +30,9 @@ namespace nix {
         SymbolTable & symbols;
         Expr * result;
         Path basePath;
-        Symbol path;
-        string error;
+        Symbol file;
+        FileOrigin origin;
+        ErrorInfo error;
         Symbol sLetBody;
         ParseData(EvalState & state)
             : state(state)
@@ -64,15 +65,19 @@ namespace nix {
 
 static void dupAttr(const AttrPath & attrPath, const Pos & pos, const Pos & prevPos)
 {
-    throw ParseError(format("attribute '%1%' at %2% already defined at %3%")
-        % showAttrPath(attrPath) % pos % prevPos);
+    throw ParseError({
+         .hint = hintfmt("attribute '%1%' already defined at %2%",
+            showAttrPath(attrPath), prevPos),
+         .errPos = pos
+    });
 }
 
-
 static void dupAttr(Symbol attr, const Pos & pos, const Pos & prevPos)
 {
-    throw ParseError(format("attribute '%1%' at %2% already defined at %3%")
-        % attr % pos % prevPos);
+    throw ParseError({
+        .hint = hintfmt("attribute '%1%' already defined at %2%", attr, prevPos),
+        .errPos = pos
+    });
 }
 
 
@@ -140,8 +145,11 @@ static void addAttr(ExprAttrs * attrs, AttrPath & attrPath,
 static void addFormal(const Pos & pos, Formals * formals, const Formal & formal)
 {
     if (!formals->argNames.insert(formal.name).second)
-        throw ParseError(format("duplicate formal function argument '%1%' at %2%")
-            % formal.name % pos);
+        throw ParseError({
+            .hint = hintfmt("duplicate formal function argument '%1%'",
+                formal.name),
+            .errPos = pos
+        });
     formals->formals.push_front(formal);
 }
 
@@ -238,7 +246,7 @@ static Expr * stripIndentation(const Pos & pos, SymbolTable & symbols, vector<Ex
 
 static inline Pos makeCurPos(const YYLTYPE & loc, ParseData * data)
 {
-    return Pos(data->path, loc.first_line, loc.first_column);
+    return Pos(data->origin, data->file, loc.first_line, loc.first_column);
 }
 
 #define CUR_POS makeCurPos(*yylocp, data)
@@ -249,8 +257,10 @@ static inline Pos makeCurPos(const YYLTYPE & loc, ParseData * data)
 
 void yyerror(YYLTYPE * loc, yyscan_t scanner, ParseData * data, const char * error)
 {
-    data->error = (format("%1%, at %2%")
-        % error % makeCurPos(*loc, data)).str();
+    data->error = {
+        .hint = hintfmt(error),
+        .errPos = makeCurPos(*loc, data)
+    };
 }
 
 
@@ -327,8 +337,10 @@ expr_function
     { $$ = new ExprWith(CUR_POS, $2, $4); }
   | LET binds IN expr_function
     { if (!$2->dynamicAttrs.empty())
-        throw ParseError(format("dynamic attributes not allowed in let at %1%")
-            % CUR_POS);
+        throw ParseError({
+            .hint = hintfmt("dynamic attributes not allowed in let"),
+            .errPos = CUR_POS
+        });
       $$ = new ExprLet($2, $4);
     }
   | expr_if
@@ -405,7 +417,10 @@ expr_simple
   | URI {
       static bool noURLLiterals = settings.isExperimentalFeatureEnabled("no-url-literals");
       if (noURLLiterals)
-          throw ParseError("URL literals are disabled, at %s", CUR_POS);
+          throw ParseError({
+              .hint = hintfmt("URL literals are disabled"),
+              .errPos = CUR_POS
+          });
       $$ = new ExprString(data->symbols.create($1));
   }
   | '(' expr ')' { $$ = $2; }
@@ -475,8 +490,10 @@ attrs
           $$->push_back(AttrName(str->s));
           delete str;
       } else
-          throw ParseError(format("dynamic attributes not allowed in inherit at %1%")
-              % makeCurPos(@2, data));
+          throw ParseError({
+              .hint = hintfmt("dynamic attributes not allowed in inherit"),
+              .errPos = makeCurPos(@2, data)
+          });
     }
   | { $$ = new AttrPath; }
   ;
@@ -552,13 +569,24 @@ formal
 namespace nix {
 
 
-Expr * EvalState::parse(const char * text,
+Expr * EvalState::parse(const char * text, FileOrigin origin,
     const Path & path, const Path & basePath, StaticEnv & staticEnv)
 {
     yyscan_t scanner;
     ParseData data(*this);
+    data.origin = origin;
+    switch (origin) {
+        case foFile: 
+            data.file = data.symbols.create(path);
+            break;
+        case foStdin:
+        case foString:
+            data.file = data.symbols.create(text);
+            break;
+        default:
+            assert(false);
+    }
     data.basePath = basePath;
-    data.path = data.symbols.create(path);
 
     yylex_init(&scanner);
     yy_scan_string(text, scanner);
@@ -608,13 +636,13 @@ Expr * EvalState::parseExprFromFile(const Path & path)
 
 Expr * EvalState::parseExprFromFile(const Path & path, StaticEnv & staticEnv)
 {
-    return parse(readFile(path).c_str(), path, dirOf(path), staticEnv);
+    return parse(readFile(path).c_str(), foFile, path, dirOf(path), staticEnv);
 }
 
 
 Expr * EvalState::parseExprFromString(std::string_view s, const Path & basePath, StaticEnv & staticEnv)
 {
-    return parse(s.data(), "(string)", basePath, staticEnv);
+    return parse(s.data(), foString, "", basePath, staticEnv);
 }
 
 
@@ -627,7 +655,7 @@ Expr * EvalState::parseExprFromString(std::string_view s, const Path & basePath)
 Expr * EvalState::parseStdin()
 {
     //Activity act(*logger, lvlTalkative, format("parsing standard input"));
-    return parseExprFromString(drainFD(0), absPath("."));
+    return parse(drainFD(0).data(), foStdin, "", absPath("."), staticBaseEnv);
 }
 
 
@@ -671,11 +699,13 @@ Path EvalState::findFile(SearchPath & searchPath, const string & path, const Pos
         Path res = r.second + suffix;
         if (pathExists(res)) return canonPath(res);
     }
-    format f = format(
-        "file '%1%' was not found in the Nix search path (add it using $NIX_PATH or -I)"
-        + string(pos ? ", at %2%" : ""));
-    f.exceptions(boost::io::all_error_bits ^ boost::io::too_many_args_bit);
-    throw ThrownError(f % path % pos);
+    throw ThrownError({
+        .hint = hintfmt(evalSettings.pureEval
+            ? "cannot look up '<%s>' in pure evaluation mode (use '--impure' to override)"
+            : "file '%s' was not found in the Nix search path (add it using $NIX_PATH or -I)",
+            path),
+        .errPos = pos
+    });
 }
 
 
@@ -689,9 +719,12 @@ std::pair<bool, std::string> EvalState::resolveSearchPathElem(const SearchPathEl
     if (isUri(elem.second)) {
         try {
             res = { true, store->toRealPath(fetchers::downloadTarball(
-                        store, resolveUri(elem.second), "source", false).storePath) };
+                        store, resolveUri(elem.second), "source", false).first.storePath) };
         } catch (FileTransferError & e) {
-            printError(format("warning: Nix search path entry '%1%' cannot be downloaded, ignoring") % elem.second);
+            logWarning({
+                .name = "Entry download",
+                .hint = hintfmt("Nix search path entry '%1%' cannot be downloaded, ignoring", elem.second)
+            });
             res = { false, "" };
         }
     } else {
@@ -699,7 +732,10 @@ std::pair<bool, std::string> EvalState::resolveSearchPathElem(const SearchPathEl
         if (pathExists(path))
             res = { true, path };
         else {
-            printError(format("warning: Nix search path entry '%1%' does not exist, ignoring") % elem.second);
+            logWarning({
+                .name = "Entry not found",
+                .hint = hintfmt("warning: Nix search path entry '%1%' does not exist, ignoring", elem.second)
+            });
             res = { false, "" };
         }
     }

src/libexpr/primops.cc

@@ -30,18 +30,6 @@ namespace nix {
  *************************************************************/
 
 
-/* Decode a context string ‘!<name>!<path>’ into a pair <path,
-   name>. */
-std::pair<string, string> decodeContext(const string & s)
-{
-    if (s.at(0) == '!') {
-        size_t index = s.find("!", 1);
-        return std::pair<string, string>(string(s, index + 1), string(s, 1, index - 1));
-    } else
-        return std::pair<string, string>(s.at(0) == '/' ? s : string(s, 1), "");
-}
-
-
 InvalidPathError::InvalidPathError(const Path & path) :
     EvalError("path '%s' is not valid", path), path(path) {}
 
@@ -55,7 +43,7 @@ void EvalState::realiseContext(const PathSet & context)
         if (!store->isValidPath(ctx))
             throw InvalidPathError(store->printStorePath(ctx));
         if (!outputName.empty() && ctx.isDerivation()) {
-            drvs.push_back(StorePathWithOutputs{ctx.clone(), {outputName}});
+            drvs.push_back(StorePathWithOutputs{ctx, {outputName}});
 
             /* Add the output of this derivation to the allowed
                paths. */
@@ -94,8 +82,10 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args
     try {
         state.realiseContext(context);
     } catch (InvalidPathError & e) {
-        throw EvalError(format("cannot import '%1%', since path '%2%' is not valid, at %3%")
-            % path % e.path % pos);
+        throw EvalError({
+            .hint = hintfmt("cannot import '%1%', since path '%2%' is not valid", path, e.path),
+            .errPos = pos
+        });
     }
 
     Path realPath = state.checkSourcePath(state.toRealPath(path, context));
@@ -171,8 +161,12 @@ void prim_importNative(EvalState & state, const Pos & pos, Value * * args, Value
     try {
         state.realiseContext(context);
     } catch (InvalidPathError & e) {
-        throw EvalError(format("cannot import '%1%', since path '%2%' is not valid, at %3%")
-            % path % e.path % pos);
+        throw EvalError({
+            .hint = hintfmt(
+                "cannot import '%1%', since path '%2%' is not valid",
+                path, e.path),
+            .errPos = pos
+        });
     }
 
     path = state.checkSourcePath(path);
@@ -181,17 +175,17 @@ void prim_importNative(EvalState & state, const Pos & pos, Value * * args, Value
 
     void *handle = dlopen(path.c_str(), RTLD_LAZY | RTLD_LOCAL);
     if (!handle)
-        throw EvalError(format("could not open '%1%': %2%") % path % dlerror());
+        throw EvalError("could not open '%1%': %2%", path, dlerror());
 
     dlerror();
     ValueInitializer func = (ValueInitializer) dlsym(handle, sym.c_str());
     if(!func) {
         char *message = dlerror();
         if (message)
-            throw EvalError(format("could not load symbol '%1%' from '%2%': %3%") % sym % path % message);
+            throw EvalError("could not load symbol '%1%' from '%2%': %3%", sym, path, message);
         else
-            throw EvalError(format("symbol '%1%' from '%2%' resolved to NULL when a function pointer was expected")
-                    % sym % path);
+            throw EvalError("symbol '%1%' from '%2%' resolved to NULL when a function pointer was expected",
+                sym, path);
     }
 
     (func)(state, v);
@@ -207,7 +201,10 @@ void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v)
     auto elems = args[0]->listElems();
     auto count = args[0]->listSize();
     if (count == 0) {
-        throw EvalError(format("at least one argument to 'exec' required, at %1%") % pos);
+        throw EvalError({
+            .hint = hintfmt("at least one argument to 'exec' required"),
+            .errPos = pos
+        });
     }
     PathSet context;
     auto program = state.coerceToString(pos, *elems[0], context, false, false);
@@ -218,8 +215,11 @@ void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v)
     try {
         state.realiseContext(context);
     } catch (InvalidPathError & e) {
-        throw EvalError(format("cannot execute '%1%', since path '%2%' is not valid, at %3%")
-            % program % e.path % pos);
+        throw EvalError({
+            .hint = hintfmt("cannot execute '%1%', since path '%2%' is not valid",
+                program, e.path),
+            .errPos = pos
+        });
     }
 
     auto output = runProgram(program, true, commandArgs);
@@ -227,13 +227,13 @@ void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v)
     try {
         parsed = state.parseExprFromString(output, pos.file);
     } catch (Error & e) {
-        e.addPrefix(format("While parsing the output from '%1%', at %2%\n") % program % pos);
+        e.addTrace(pos, "While parsing the output from '%1%'", program);
         throw;
     }
     try {
         state.eval(parsed, v);
     } catch (Error & e) {
-        e.addPrefix(format("While evaluating the output from '%1%', at %2%\n") % program % pos);
+        e.addTrace(pos, "While evaluating the output from '%1%'", program);
         throw;
     }
 }
@@ -339,7 +339,7 @@ struct CompareValues
         if (v1->type == tInt && v2->type == tFloat)
             return v1->integer < v2->fpoint;
         if (v1->type != v2->type)
-            throw EvalError(format("cannot compare %1% with %2%") % showType(*v1) % showType(*v2));
+            throw EvalError("cannot compare %1% with %2%", showType(*v1), showType(*v2));
         switch (v1->type) {
             case tInt:
                 return v1->integer < v2->integer;
@@ -350,7 +350,7 @@ struct CompareValues
             case tPath:
                 return strcmp(v1->path, v2->path) < 0;
             default:
-                throw EvalError(format("cannot compare %1% with %2%") % showType(*v1) % showType(*v2));
+                throw EvalError("cannot compare %1% with %2%", showType(*v1), showType(*v2));
         }
     }
 };
@@ -371,7 +371,10 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar
     Bindings::iterator startSet =
         args[0]->attrs->find(state.symbols.create("startSet"));
     if (startSet == args[0]->attrs->end())
-        throw EvalError(format("attribute 'startSet' required, at %1%") % pos);
+        throw EvalError({
+            .hint = hintfmt("attribute 'startSet' required"),
+            .errPos = pos
+        });
     state.forceList(*startSet->value, pos);
 
     ValueList workSet;
@@ -382,7 +385,10 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar
     Bindings::iterator op =
         args[0]->attrs->find(state.symbols.create("operator"));
     if (op == args[0]->attrs->end())
-        throw EvalError(format("attribute 'operator' required, at %1%") % pos);
+        throw EvalError({
+            .hint = hintfmt("attribute 'operator' required"),
+            .errPos = pos
+        });
     state.forceValue(*op->value, pos);
 
     /* Construct the closure by applying the operator to element of
@@ -401,7 +407,10 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar
         Bindings::iterator key =
             e->attrs->find(state.symbols.create("key"));
         if (key == e->attrs->end())
-            throw EvalError(format("attribute 'key' required, at %1%") % pos);
+            throw EvalError({
+                .hint = hintfmt("attribute 'key' required"),
+                .errPos = pos
+            });
         state.forceValue(*key->value, pos);
 
         if (!doneKeys.insert(key->value).second) continue;
@@ -431,7 +440,7 @@ static void prim_abort(EvalState & state, const Pos & pos, Value * * args, Value
 {
     PathSet context;
     string s = state.coerceToString(pos, *args[0], context);
-    throw Abort(format("evaluation aborted with the following error message: '%1%'") % s);
+    throw Abort("evaluation aborted with the following error message: '%1%'", s);
 }
 
 
@@ -450,7 +459,7 @@ static void prim_addErrorContext(EvalState & state, const Pos & pos, Value * * a
         v = *args[1];
     } catch (Error & e) {
         PathSet context;
-        e.addPrefix(format("%1%\n") % state.coerceToString(pos, *args[0], context));
+        e.addTrace(std::nullopt, state.coerceToString(pos, *args[0], context));
         throw;
     }
 }
@@ -506,9 +515,9 @@ static void prim_trace(EvalState & state, const Pos & pos, Value * * args, Value
 {
     state.forceValue(*args[0], pos);
     if (args[0]->type == tString)
-        printError(format("trace: %1%") % args[0]->string.s);
+        printError("trace: %1%", args[0]->string.s);
     else
-        printError(format("trace: %1%") % *args[0]);
+        printError("trace: %1%", *args[0]);
     state.forceValue(*args[1], pos);
     v = *args[1];
 }
@@ -533,13 +542,16 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
     /* Figure out the name first (for stack backtraces). */
     Bindings::iterator attr = args[0]->attrs->find(state.sName);
     if (attr == args[0]->attrs->end())
-        throw EvalError(format("required attribute 'name' missing, at %1%") % pos);
+        throw EvalError({
+            .hint = hintfmt("required attribute 'name' missing"),
+            .errPos = pos
+        });
     string drvName;
     Pos & posDrvName(*attr->pos);
     try {
         drvName = state.forceStringNoCtx(*attr->value, pos);
     } catch (Error & e) {
-        e.addPrefix(format("while evaluating the derivation attribute 'name' at %1%:\n") % posDrvName);
+        e.addTrace(posDrvName, "while evaluating the derivation attribute 'name'");
         throw;
     }
 
@@ -563,7 +575,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
 
     std::optional<std::string> outputHash;
     std::string outputHashAlgo;
-    bool outputHashRecursive = false;
+    auto ingestionMethod = FileIngestionMethod::Flat;
 
     StringSet outputs;
     outputs.insert("out");
@@ -574,27 +586,40 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
         vomit("processing attribute '%1%'", key);
 
         auto handleHashMode = [&](const std::string & s) {
-            if (s == "recursive") outputHashRecursive = true;
-            else if (s == "flat") outputHashRecursive = false;
-            else throw EvalError("invalid value '%s' for 'outputHashMode' attribute, at %s", s, posDrvName);
+            if (s == "recursive") ingestionMethod = FileIngestionMethod::Recursive;
+            else if (s == "flat") ingestionMethod = FileIngestionMethod::Flat;
+            else
+                throw EvalError({
+                    .hint = hintfmt("invalid value '%s' for 'outputHashMode' attribute", s),
+                    .errPos = posDrvName
+                });
         };
 
         auto handleOutputs = [&](const Strings & ss) {
             outputs.clear();
             for (auto & j : ss) {
                 if (outputs.find(j) != outputs.end())
-                    throw EvalError(format("duplicate derivation output '%1%', at %2%") % j % posDrvName);
+                    throw EvalError({
+                        .hint = hintfmt("duplicate derivation output '%1%'", j),
+                        .errPos = posDrvName
+                    });
                 /* !!! Check whether j is a valid attribute
                    name. */
                 /* Derivations cannot be named ‘drv’, because
                    then we'd have an attribute ‘drvPath’ in
                    the resulting set. */
                 if (j == "drv")
-                    throw EvalError(format("invalid derivation output name 'drv', at %1%") % posDrvName);
+                    throw EvalError({
+                        .hint = hintfmt("invalid derivation output name 'drv'" ),
+                        .errPos = posDrvName
+                    });
                 outputs.insert(j);
             }
             if (outputs.empty())
-                throw EvalError(format("derivation cannot have an empty set of outputs, at %1%") % posDrvName);
+                throw EvalError({
+                    .hint = hintfmt("derivation cannot have an empty set of outputs"),
+                    .errPos = posDrvName
+                });
         };
 
         try {
@@ -659,8 +684,9 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
             }
 
         } catch (Error & e) {
-            e.addPrefix(format("while evaluating the attribute '%1%' of the derivation '%2%' at %3%:\n")
-                % key % drvName % posDrvName);
+            e.addTrace(posDrvName, 
+                "while evaluating the attribute '%1%' of the derivation '%2%'",
+                key, drvName);
             throw;
         }
     }
@@ -687,9 +713,9 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
             StorePathSet refs;
             state.store->computeFSClosure(state.store->parseStorePath(std::string_view(path).substr(1)), refs);
             for (auto & j : refs) {
-                drv.inputSrcs.insert(j.clone());
+                drv.inputSrcs.insert(j);
                 if (j.isDerivation())
-                    drv.inputDrvs[j.clone()] = state.store->queryDerivationOutputNames(j);
+                    drv.inputDrvs[j] = state.store->readDerivation(j).outputNames();
             }
         }
 
@@ -706,27 +732,44 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
 
     /* Do we have all required attributes? */
     if (drv.builder == "")
-        throw EvalError(format("required attribute 'builder' missing, at %1%") % posDrvName);
+        throw EvalError({
+            .hint = hintfmt("required attribute 'builder' missing"),
+            .errPos = posDrvName
+        });
+
     if (drv.platform == "")
-        throw EvalError(format("required attribute 'system' missing, at %1%") % posDrvName);
+        throw EvalError({
+            .hint = hintfmt("required attribute 'system' missing"),
+            .errPos = posDrvName
+        });
 
     /* Check whether the derivation name is valid. */
     if (isDerivation(drvName))
-        throw EvalError("derivation names are not allowed to end in '%s', at %s", drvExtension, posDrvName);
+        throw EvalError({
+            .hint = hintfmt("derivation names are not allowed to end in '%s'", drvExtension),
+            .errPos = posDrvName
+        });
 
     if (outputHash) {
         /* Handle fixed-output derivations. */
         if (outputs.size() != 1 || *(outputs.begin()) != "out")
-            throw Error(format("multiple outputs are not supported in fixed-output derivations, at %1%") % posDrvName);
+            throw Error({
+                .hint = hintfmt("multiple outputs are not supported in fixed-output derivations"),
+                .errPos = posDrvName
+            });
 
-        HashType ht = outputHashAlgo.empty() ? htUnknown : parseHashType(outputHashAlgo);
-        Hash h(*outputHash, ht);
+        std::optional<HashType> ht = parseHashTypeOpt(outputHashAlgo);
+        Hash h = newHashAllowEmpty(*outputHash, ht);
 
-        auto outPath = state.store->makeFixedOutputPath(outputHashRecursive, h, drvName);
+        auto outPath = state.store->makeFixedOutputPath(ingestionMethod, h, drvName);
         if (!jsonObject) drv.env["out"] = state.store->printStorePath(outPath);
-        drv.outputs.insert_or_assign("out", DerivationOutput(std::move(outPath),
-                (outputHashRecursive ? "r:" : "") + printHashType(h.type),
-                h.to_string(Base16, false)));
+        drv.outputs.insert_or_assign("out", DerivationOutput {
+            .path = std::move(outPath),
+            .hash = FixedOutputHash {
+                .method = ingestionMethod,
+                .hash = std::move(h),
+            },
+        });
     }
 
     else {
@@ -739,7 +782,10 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
         for (auto & i : outputs) {
             if (!jsonObject) drv.env[i] = "";
             drv.outputs.insert_or_assign(i,
-                DerivationOutput(StorePath::dummy.clone(), "", ""));
+                DerivationOutput {
+                    .path = StorePath::dummy,
+                    .hash = std::optional<FixedOutputHash> {},
+                });
         }
 
         Hash h = hashDerivationModulo(*state.store, Derivation(drv), true);
@@ -748,7 +794,10 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
             auto outPath = state.store->makeOutputPath(i, h, drvName);
             if (!jsonObject) drv.env[i] = state.store->printStorePath(outPath);
             drv.outputs.insert_or_assign(i,
-                DerivationOutput(std::move(outPath), "", ""));
+                DerivationOutput {
+                    .path = std::move(outPath),
+                    .hash = std::optional<FixedOutputHash>(),
+                });
         }
     }
 
@@ -761,7 +810,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
     /* Optimisation, but required in read-only mode! because in that
        case we don't actually write store derivations, so we can't
        read them later. */
-    drvHashes.insert_or_assign(drvPath.clone(),
+    drvHashes.insert_or_assign(drvPath,
         hashDerivationModulo(*state.store, Derivation(drv), false));
 
     state.mkAttrs(v, 1 + drv.outputs.size());
@@ -818,11 +867,14 @@ static void prim_storePath(EvalState & state, const Pos & pos, Value * * args, V
        e.g. nix-push does the right thing. */
     if (!state.store->isStorePath(path)) path = canonPath(path, true);
     if (!state.store->isInStore(path))
-        throw EvalError(format("path '%1%' is not in the Nix store, at %2%") % path % pos);
-    Path path2 = state.store->toStorePath(path);
+        throw EvalError({
+            .hint = hintfmt("path '%1%' is not in the Nix store", path),
+            .errPos = pos
+        });
+    auto path2 = state.store->toStorePath(path).first;
     if (!settings.readOnlyMode)
-        state.store->ensurePath(state.store->parseStorePath(path2));
-    context.insert(path2);
+        state.store->ensurePath(path2);
+    context.insert(state.store->printStorePath(path2));
     mkString(v, path, context);
 }
 
@@ -834,9 +886,12 @@ static void prim_pathExists(EvalState & state, const Pos & pos, Value * * args,
     try {
         state.realiseContext(context);
     } catch (InvalidPathError & e) {
-        throw EvalError(format(
-                "cannot check the existence of '%1%', since path '%2%' is not valid, at %3%")
-            % path % e.path % pos);
+        throw EvalError({
+            .hint = hintfmt(
+                "cannot check the existence of '%1%', since path '%2%' is not valid",
+                path, e.path),
+            .errPos = pos
+        });
     }
 
     try {
@@ -879,12 +934,14 @@ static void prim_readFile(EvalState & state, const Pos & pos, Value * * args, Va
     try {
         state.realiseContext(context);
     } catch (InvalidPathError & e) {
-        throw EvalError(format("cannot read '%1%', since path '%2%' is not valid, at %3%")
-            % path % e.path % pos);
+        throw EvalError({
+            .hint = hintfmt("cannot read '%1%', since path '%2%' is not valid", path, e.path),
+            .errPos = pos
+        });
     }
     string s = readFile(state.checkSourcePath(state.toRealPath(path, context)));
     if (s.find((char) 0) != string::npos)
-        throw Error(format("the contents of the file '%1%' cannot be represented as a Nix string") % path);
+        throw Error("the contents of the file '%1%' cannot be represented as a Nix string", path);
     mkString(v, s.c_str());
 }
 
@@ -908,7 +965,10 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va
 
         i = v2.attrs->find(state.symbols.create("path"));
         if (i == v2.attrs->end())
-            throw EvalError(format("attribute 'path' missing, at %1%") % pos);
+            throw EvalError({
+                .hint = hintfmt("attribute 'path' missing"),
+                .errPos = pos
+            });
 
         PathSet context;
         string path = state.coerceToString(pos, *i->value, context, false, false);
@@ -916,8 +976,10 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va
         try {
             state.realiseContext(context);
         } catch (InvalidPathError & e) {
-            throw EvalError(format("cannot find '%1%', since path '%2%' is not valid, at %3%")
-                % path % e.path % pos);
+            throw EvalError({
+                .hint = hintfmt("cannot find '%1%', since path '%2%' is not valid", path, e.path),
+                .errPos = pos
+            });
         }
 
         searchPath.emplace_back(prefix, path);
@@ -932,14 +994,17 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va
 static void prim_hashFile(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     string type = state.forceStringNoCtx(*args[0], pos);
-    HashType ht = parseHashType(type);
-    if (ht == htUnknown)
-      throw Error(format("unknown hash type '%1%', at %2%") % type % pos);
+    std::optional<HashType> ht = parseHashType(type);
+    if (!ht)
+      throw Error({
+          .hint = hintfmt("unknown hash type '%1%'", type),
+          .errPos = pos
+      });
 
     PathSet context; // discarded
     Path p = state.coerceToPath(pos, *args[1], context);
 
-    mkString(v, hashFile(ht, state.checkSourcePath(p)).to_string(Base16, false), context);
+    mkString(v, hashFile(*ht, state.checkSourcePath(p)).to_string(Base16, false), context);
 }
 
 /* Read a directory (without . or ..) */
@@ -950,8 +1015,10 @@ static void prim_readDir(EvalState & state, const Pos & pos, Value * * args, Val
     try {
         state.realiseContext(ctx);
     } catch (InvalidPathError & e) {
-        throw EvalError(format("cannot read '%1%', since path '%2%' is not valid, at %3%")
-            % path % e.path % pos);
+        throw EvalError({
+            .hint = hintfmt("cannot read '%1%', since path '%2%' is not valid", path, e.path),
+            .errPos = pos
+        });
     }
 
     DirEntries entries = readDirectory(state.checkSourcePath(path));
@@ -1021,9 +1088,13 @@ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Valu
 
     for (auto path : context) {
         if (path.at(0) != '/')
-            throw EvalError(format(
-                "in 'toFile': the file named '%1%' must not contain a reference "
-                "to a derivation but contains (%2%), at %3%") % name % path % pos);
+            throw EvalError( {
+                .hint = hintfmt(
+                    "in 'toFile': the file named '%1%' must not contain a reference "
+                    "to a derivation but contains (%2%)",
+                    name, path),
+                .errPos = pos
+            });
         refs.insert(state.store->parseStorePath(path));
     }
 
@@ -1040,7 +1111,7 @@ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Valu
 
 
 static void addPath(EvalState & state, const Pos & pos, const string & name, const Path & path_,
-    Value * filterFun, bool recursive, const Hash & expectedHash, Value & v)
+    Value * filterFun, FileIngestionMethod method, const Hash & expectedHash, Value & v)
 {
     const auto path = evalSettings.pureEval && expectedHash ?
         path_ :
@@ -1071,12 +1142,12 @@ static void addPath(EvalState & state, const Pos & pos, const string & name, con
 
     std::optional<StorePath> expectedStorePath;
     if (expectedHash)
-        expectedStorePath = state.store->makeFixedOutputPath(recursive, expectedHash, name);
+        expectedStorePath = state.store->makeFixedOutputPath(method, expectedHash, name);
     Path dstPath;
     if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) {
         dstPath = state.store->printStorePath(settings.readOnlyMode
-            ? state.store->computeStorePathForPath(name, path, recursive, htSHA256, filter).first
-            : state.store->addToStore(name, path, recursive, htSHA256, filter, state.repair));
+            ? state.store->computeStorePathForPath(name, path, method, htSHA256, filter).first
+            : state.store->addToStore(name, path, method, htSHA256, filter, state.repair));
         if (expectedHash && expectedStorePath != state.store->parseStorePath(dstPath))
             throw Error("store path mismatch in (possibly filtered) path added from '%s'", path);
     } else
@@ -1091,13 +1162,21 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args
     PathSet context;
     Path path = state.coerceToPath(pos, *args[1], context);
     if (!context.empty())
-        throw EvalError(format("string '%1%' cannot refer to other paths, at %2%") % path % pos);
+        throw EvalError({
+            .hint = hintfmt("string '%1%' cannot refer to other paths", path),
+            .errPos = pos
+        });
 
     state.forceValue(*args[0], pos);
     if (args[0]->type != tLambda)
-        throw TypeError(format("first argument in call to 'filterSource' is not a function but %1%, at %2%") % showType(*args[0]) % pos);
+        throw TypeError({
+            .hint = hintfmt(
+                "first argument in call to 'filterSource' is not a function but %1%",
+                showType(*args[0])),
+            .errPos = pos
+        });
 
-    addPath(state, pos, std::string(baseNameOf(path)), path, args[0], true, Hash(), v);
+    addPath(state, pos, std::string(baseNameOf(path)), path, args[0], FileIngestionMethod::Recursive, Hash(), v);
 }
 
 static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value & v)
@@ -1106,7 +1185,7 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value
     Path path;
     string name;
     Value * filterFun = nullptr;
-    auto recursive = true;
+    auto method = FileIngestionMethod::Recursive;
     Hash expectedHash;
 
     for (auto & attr : *args[0]->attrs) {
@@ -1115,25 +1194,34 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value
             PathSet context;
             path = state.coerceToPath(*attr.pos, *attr.value, context);
             if (!context.empty())
-                throw EvalError(format("string '%1%' cannot refer to other paths, at %2%") % path % *attr.pos);
+                throw EvalError({
+                    .hint = hintfmt("string '%1%' cannot refer to other paths", path),
+                    .errPos = *attr.pos
+                });
         } else if (attr.name == state.sName)
             name = state.forceStringNoCtx(*attr.value, *attr.pos);
         else if (n == "filter") {
             state.forceValue(*attr.value, pos);
             filterFun = attr.value;
         } else if (n == "recursive")
-            recursive = state.forceBool(*attr.value, *attr.pos);
+            method = FileIngestionMethod { state.forceBool(*attr.value, *attr.pos) };
         else if (n == "sha256")
-            expectedHash = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256);
+            expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256);
         else
-            throw EvalError(format("unsupported argument '%1%' to 'addPath', at %2%") % attr.name % *attr.pos);
+            throw EvalError({
+                .hint = hintfmt("unsupported argument '%1%' to 'addPath'", attr.name),
+                .errPos = *attr.pos
+            });
     }
     if (path.empty())
-        throw EvalError(format("'path' required, at %1%") % pos);
+        throw EvalError({
+            .hint = hintfmt("'path' required"),
+            .errPos = pos
+        });
     if (name.empty())
         name = baseNameOf(path);
 
-    addPath(state, pos, name, path, filterFun, recursive, expectedHash, v);
+    addPath(state, pos, name, path, filterFun, method, expectedHash, v);
 }
 
 
@@ -1187,7 +1275,10 @@ void prim_getAttr(EvalState & state, const Pos & pos, Value * * args, Value & v)
     // !!! Should we create a symbol here or just do a lookup?
     Bindings::iterator i = args[1]->attrs->find(state.symbols.create(attr));
     if (i == args[1]->attrs->end())
-        throw EvalError(format("attribute '%1%' missing, at %2%") % attr % pos);
+        throw EvalError({
+            .hint = hintfmt("attribute '%1%' missing", attr),
+            .errPos = pos
+        });
     // !!! add to stack trace?
     if (state.countCalls && i->pos) state.attrSelects[*i->pos]++;
     state.forceValue(*i->value, pos);
@@ -1267,15 +1358,20 @@ static void prim_listToAttrs(EvalState & state, const Pos & pos, Value * * args,
 
         Bindings::iterator j = v2.attrs->find(state.sName);
         if (j == v2.attrs->end())
-            throw TypeError(format("'name' attribute missing in a call to 'listToAttrs', at %1%") % pos);
+            throw TypeError({
+                .hint = hintfmt("'name' attribute missing in a call to 'listToAttrs'"),
+                .errPos = pos
+            });
         string name = state.forceStringNoCtx(*j->value, pos);
 
         Symbol sym = state.symbols.create(name);
         if (seen.insert(sym).second) {
             Bindings::iterator j2 = v2.attrs->find(state.symbols.create(state.sValue));
             if (j2 == v2.attrs->end())
-                throw TypeError(format("'value' attribute missing in a call to 'listToAttrs', at %1%") % pos);
-
+                throw TypeError({
+                    .hint = hintfmt("'value' attribute missing in a call to 'listToAttrs'"),
+                    .errPos = pos
+                });
             v.attrs->push_back(Attr(sym, j2->value, j2->pos));
         }
     }
@@ -1348,7 +1444,10 @@ static void prim_functionArgs(EvalState & state, const Pos & pos, Value * * args
 {
     state.forceValue(*args[0], pos);
     if (args[0]->type != tLambda)
-        throw TypeError(format("'functionArgs' requires a function, at %1%") % pos);
+        throw TypeError({
+            .hint = hintfmt("'functionArgs' requires a function"),
+            .errPos = pos
+        });
 
     if (!args[0]->lambda.fun->matchAttrs) {
         state.mkAttrs(v, 0);
@@ -1401,7 +1500,10 @@ static void elemAt(EvalState & state, const Pos & pos, Value & list, int n, Valu
 {
     state.forceList(list, pos);
     if (n < 0 || (unsigned int) n >= list.listSize())
-        throw Error(format("list index %1% is out of bounds, at %2%") % n % pos);
+        throw Error({
+            .hint = hintfmt("list index %1% is out of bounds", n),
+            .errPos = pos
+        });
     state.forceValue(*list.listElems()[n], pos);
     v = *list.listElems()[n];
 }
@@ -1428,7 +1530,11 @@ static void prim_tail(EvalState & state, const Pos & pos, Value * * args, Value
 {
     state.forceList(*args[0], pos);
     if (args[0]->listSize() == 0)
-        throw Error(format("'tail' called on an empty list, at %1%") % pos);
+        throw Error({
+            .hint = hintfmt("'tail' called on an empty list"),
+            .errPos = pos
+        });
+
     state.mkList(v, args[0]->listSize() - 1);
     for (unsigned int n = 0; n < v.listSize(); ++n)
         v.listElems()[n] = args[0]->listElems()[n + 1];
@@ -1569,7 +1675,10 @@ static void prim_genList(EvalState & state, const Pos & pos, Value * * args, Val
     auto len = state.forceInt(*args[1], pos);
 
     if (len < 0)
-        throw EvalError(format("cannot create list of size %1%, at %2%") % len % pos);
+        throw EvalError({
+            .hint = hintfmt("cannot create list of size %1%", len),
+            .errPos = pos
+        });
 
     state.mkList(v, len);
 
@@ -1727,7 +1836,11 @@ static void prim_div(EvalState & state, const Pos & pos, Value * * args, Value &
     state.forceValue(*args[1], pos);
 
     NixFloat f2 = state.forceFloat(*args[1], pos);
-    if (f2 == 0) throw EvalError(format("division by zero, at %1%") % pos);
+    if (f2 == 0)
+        throw EvalError({
+            .hint = hintfmt("division by zero"),
+            .errPos = pos
+        });
 
     if (args[0]->type == tFloat || args[1]->type == tFloat) {
         mkFloat(v, state.forceFloat(*args[0], pos) / state.forceFloat(*args[1], pos));
@@ -1736,7 +1849,11 @@ static void prim_div(EvalState & state, const Pos & pos, Value * * args, Value &
         NixInt i2 = state.forceInt(*args[1], pos);
         /* Avoid division overflow as it might raise SIGFPE. */
         if (i1 == std::numeric_limits<NixInt>::min() && i2 == -1)
-            throw EvalError(format("overflow in integer division, at %1%") % pos);
+            throw EvalError({
+                .hint = hintfmt("overflow in integer division"),
+                .errPos = pos
+            });
+
         mkInt(v, i1 / i2);
     }
 }
@@ -1792,7 +1909,11 @@ static void prim_substring(EvalState & state, const Pos & pos, Value * * args, V
     PathSet context;
     string s = state.coerceToString(pos, *args[2], context);
 
-    if (start < 0) throw EvalError(format("negative start position in 'substring', at %1%") % pos);
+    if (start < 0)
+        throw EvalError({
+            .hint = hintfmt("negative start position in 'substring'"),
+            .errPos = pos
+        });
 
     mkString(v, (unsigned int) start >= s.size() ? "" : string(s, start, len), context);
 }
@@ -1810,14 +1931,17 @@ static void prim_stringLength(EvalState & state, const Pos & pos, Value * * args
 static void prim_hashString(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     string type = state.forceStringNoCtx(*args[0], pos);
-    HashType ht = parseHashType(type);
-    if (ht == htUnknown)
-      throw Error(format("unknown hash type '%1%', at %2%") % type % pos);
+    std::optional<HashType> ht = parseHashType(type);
+    if (!ht)
+        throw Error({
+            .hint = hintfmt("unknown hash type '%1%'", type),
+            .errPos = pos
+        });
 
     PathSet context; // discarded
     string s = state.forceString(*args[1], context, pos);
 
-    mkString(v, hashString(ht, s).to_string(Base16, false), context);
+    mkString(v, hashString(*ht, s).to_string(Base16, false), context);
 }
 
 
@@ -1854,10 +1978,16 @@ void prim_match(EvalState & state, const Pos & pos, Value * * args, Value & v)
 
     } catch (std::regex_error &e) {
         if (e.code() == std::regex_constants::error_space) {
-          // limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++
-          throw EvalError("memory limit exceeded by regular expression '%s', at %s", re, pos);
+            // limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++
+            throw EvalError({
+                .hint = hintfmt("memory limit exceeded by regular expression '%s'", re),
+                .errPos = pos
+            });
         } else {
-          throw EvalError("invalid regular expression '%s', at %s", re, pos);
+            throw EvalError({
+                .hint = hintfmt("invalid regular expression '%s'", re),
+                .errPos = pos
+            });
         }
     }
 }
@@ -1921,10 +2051,16 @@ static void prim_split(EvalState & state, const Pos & pos, Value * * args, Value
 
     } catch (std::regex_error &e) {
         if (e.code() == std::regex_constants::error_space) {
-          // limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++
-          throw EvalError("memory limit exceeded by regular expression '%s', at %s", re, pos);
+            // limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++
+            throw EvalError({
+                .hint = hintfmt("memory limit exceeded by regular expression '%s'", re),
+                .errPos = pos
+            });
         } else {
-          throw EvalError("invalid regular expression '%s', at %s", re, pos);
+            throw EvalError({
+                .hint = hintfmt("invalid regular expression '%s'", re),
+                .errPos = pos
+            });
         }
     }
 }
@@ -1955,7 +2091,10 @@ static void prim_replaceStrings(EvalState & state, const Pos & pos, Value * * ar
     state.forceList(*args[0], pos);
     state.forceList(*args[1], pos);
     if (args[0]->listSize() != args[1]->listSize())
-        throw EvalError(format("'from' and 'to' arguments to 'replaceStrings' have different lengths, at %1%") % pos);
+        throw EvalError({
+            .hint = hintfmt("'from' and 'to' arguments to 'replaceStrings' have different lengths"),
+            .errPos = pos
+        });
 
     vector<string> from;
     from.reserve(args[0]->listSize());
@@ -2058,10 +2197,11 @@ static void prim_splitVersion(EvalState & state, const Pos & pos, Value * * args
 RegisterPrimOp::PrimOps * RegisterPrimOp::primOps;
 
 
-RegisterPrimOp::RegisterPrimOp(std::string name, size_t arity, PrimOpFun fun)
+RegisterPrimOp::RegisterPrimOp(std::string name, size_t arity, PrimOpFun fun,
+    std::optional<std::string> requiredFeature)
 {
     if (!primOps) primOps = new PrimOps;
-    primOps->emplace_back(name, arity, fun);
+    primOps->push_back({name, arity, fun, requiredFeature});
 }
 
 
@@ -2253,7 +2393,8 @@ void EvalState::createBaseEnv()
 
     if (RegisterPrimOp::primOps)
         for (auto & primOp : *RegisterPrimOp::primOps)
-            addPrimOp(std::get<0>(primOp), std::get<1>(primOp), std::get<2>(primOp));
+            if (!primOp.requiredFeature || settings.isExperimentalFeatureEnabled(*primOp.requiredFeature))
+                addPrimOp(primOp.name, primOp.arity, primOp.primOp);
 
     /* Now that we've added all primops, sort the `builtins' set,
        because attribute lookups expect it to be sorted. */

src/libexpr/primops.hh

@@ -7,12 +7,25 @@ namespace nix {
 
 struct RegisterPrimOp
 {
-    typedef std::vector<std::tuple<std::string, size_t, PrimOpFun>> PrimOps;
+    struct Info
+    {
+        std::string name;
+        size_t arity;
+        PrimOpFun primOp;
+        std::optional<std::string> requiredFeature;
+    };
+
+    typedef std::vector<Info> PrimOps;
     static PrimOps * primOps;
+
     /* You can register a constant by passing an arity of 0. fun
        will get called during EvalState initialization, so there
        may be primops not yet added and builtins is not yet sorted. */
-    RegisterPrimOp(std::string name, size_t arity, PrimOpFun fun);
+    RegisterPrimOp(
+        std::string name,
+        size_t arity,
+        PrimOpFun fun,
+        std::optional<std::string> requiredFeature = {});
 };
 
 /* These primops are disabled without enableNativeCode, but plugins

src/libexpr/primops/context.cc

@@ -1,6 +1,6 @@
 #include "primops.hh"
 #include "eval-inline.hh"
-#include "derivations.hh"
+#include "store-api.hh"
 
 namespace nix {
 
@@ -146,7 +146,10 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg
     auto sAllOutputs = state.symbols.create("allOutputs");
     for (auto & i : *args[1]->attrs) {
         if (!state.store->isStorePath(i.name))
-            throw EvalError("Context key '%s' is not a store path, at %s", i.name, i.pos);
+            throw EvalError({
+                .hint = hintfmt("Context key '%s' is not a store path", i.name),
+                .errPos = *i.pos
+            });
         if (!settings.readOnlyMode)
             state.store->ensurePath(state.store->parseStorePath(i.name));
         state.forceAttrs(*i.value, *i.pos);
@@ -160,7 +163,10 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg
         if (iter != i.value->attrs->end()) {
             if (state.forceBool(*iter->value, *iter->pos)) {
                 if (!isDerivation(i.name)) {
-                    throw EvalError("Tried to add all-outputs context of %s, which is not a derivation, to a string, at %s", i.name, i.pos);
+                    throw EvalError({
+                        .hint = hintfmt("Tried to add all-outputs context of %s, which is not a derivation, to a string", i.name),
+                        .errPos = *i.pos
+                    });
                 }
                 context.insert("=" + string(i.name));
             }
@@ -170,7 +176,10 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg
         if (iter != i.value->attrs->end()) {
             state.forceList(*iter->value, *iter->pos);
             if (iter->value->listSize() && !isDerivation(i.name)) {
-                throw EvalError("Tried to add derivation output context of %s, which is not a derivation, to a string, at %s", i.name, i.pos);
+                throw EvalError({
+                    .hint = hintfmt("Tried to add derivation output context of %s, which is not a derivation, to a string", i.name),
+                    .errPos = *i.pos
+                });
             }
             for (unsigned int n = 0; n < iter->value->listSize(); ++n) {
                 auto name = state.forceStringNoCtx(*iter->value->listElems()[n], *iter->pos);

src/libexpr/primops/fetchGit.cc

@@ -35,11 +35,17 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va
             else if (n == "submodules")
                 fetchSubmodules = state.forceBool(*attr.value, *attr.pos);
             else
-                throw EvalError("unsupported argument '%s' to 'fetchGit', at %s", attr.name, *attr.pos);
+                throw EvalError({
+                    .hint = hintfmt("unsupported argument '%s' to 'fetchGit'", attr.name),
+                    .errPos = *attr.pos
+                });
         }
 
         if (url.empty())
-            throw EvalError(format("'url' argument required, at %1%") % pos);
+            throw EvalError({
+                .hint = hintfmt("'url' argument required"),
+                .errPos = pos
+            });
 
     } else
         url = state.coerceToString(pos, *args[0], context, false, false);
@@ -56,23 +62,23 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va
     attrs.insert_or_assign("url", url.find("://") != std::string::npos ? url : "file://" + url);
     if (ref) attrs.insert_or_assign("ref", *ref);
     if (rev) attrs.insert_or_assign("rev", rev->gitRev());
-    if (fetchSubmodules) attrs.insert_or_assign("submodules", true);
-    auto input = fetchers::inputFromAttrs(attrs);
+    if (fetchSubmodules) attrs.insert_or_assign("submodules", fetchers::Explicit<bool>{true});
+    auto input = fetchers::Input::fromAttrs(std::move(attrs));
 
     // FIXME: use name?
-    auto [tree, input2] = input->fetchTree(state.store);
+    auto [tree, input2] = input.fetch(state.store);
 
     state.mkAttrs(v, 8);
     auto storePath = state.store->printStorePath(tree.storePath);
     mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath}));
     // Backward compatibility: set 'rev' to
     // 0000000000000000000000000000000000000000 for a dirty tree.
-    auto rev2 = input2->getRev().value_or(Hash(htSHA1));
+    auto rev2 = input2.getRev().value_or(Hash(htSHA1));
     mkString(*state.allocAttr(v, state.symbols.create("rev")), rev2.gitRev());
     mkString(*state.allocAttr(v, state.symbols.create("shortRev")), rev2.gitShortRev());
     // Backward compatibility: set 'revCount' to 0 for a dirty tree.
     mkInt(*state.allocAttr(v, state.symbols.create("revCount")),
-        tree.info.revCount.value_or(0));
+        input2.getRevCount().value_or(0));
     mkBool(*state.allocAttr(v, state.symbols.create("submodules")), fetchSubmodules);
     v.attrs->sort();
 

src/libexpr/primops/fetchMercurial.cc

@@ -38,11 +38,17 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar
             else if (n == "name")
                 name = state.forceStringNoCtx(*attr.value, *attr.pos);
             else
-                throw EvalError("unsupported argument '%s' to 'fetchMercurial', at %s", attr.name, *attr.pos);
+                throw EvalError({
+                    .hint = hintfmt("unsupported argument '%s' to 'fetchMercurial'", attr.name),
+                    .errPos = *attr.pos
+                });
         }
 
         if (url.empty())
-            throw EvalError(format("'url' argument required, at %1%") % pos);
+            throw EvalError({
+                .hint = hintfmt("'url' argument required"),
+                .errPos = pos
+            });
 
     } else
         url = state.coerceToString(pos, *args[0], context, false, false);
@@ -59,23 +65,23 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar
     attrs.insert_or_assign("url", url.find("://") != std::string::npos ? url : "file://" + url);
     if (ref) attrs.insert_or_assign("ref", *ref);
     if (rev) attrs.insert_or_assign("rev", rev->gitRev());
-    auto input = fetchers::inputFromAttrs(attrs);
+    auto input = fetchers::Input::fromAttrs(std::move(attrs));
 
     // FIXME: use name
-    auto [tree, input2] = input->fetchTree(state.store);
+    auto [tree, input2] = input.fetch(state.store);
 
     state.mkAttrs(v, 8);
     auto storePath = state.store->printStorePath(tree.storePath);
     mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath}));
-    if (input2->getRef())
-        mkString(*state.allocAttr(v, state.symbols.create("branch")), *input2->getRef());
+    if (input2.getRef())
+        mkString(*state.allocAttr(v, state.symbols.create("branch")), *input2.getRef());
     // Backward compatibility: set 'rev' to
     // 0000000000000000000000000000000000000000 for a dirty tree.
-    auto rev2 = input2->getRev().value_or(Hash(htSHA1));
+    auto rev2 = input2.getRev().value_or(Hash(htSHA1));
     mkString(*state.allocAttr(v, state.symbols.create("rev")), rev2.gitRev());
     mkString(*state.allocAttr(v, state.symbols.create("shortRev")), std::string(rev2.gitRev(), 0, 12));
-    if (tree.info.revCount)
-        mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *tree.info.revCount);
+    if (auto revCount = input2.getRevCount())
+        mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *revCount);
     v.attrs->sort();
 
     if (state.allowedPaths)

src/libexpr/primops/fetchTree.cc

@@ -13,31 +13,36 @@ namespace nix {
 void emitTreeAttrs(
     EvalState & state,
     const fetchers::Tree & tree,
-    std::shared_ptr<const fetchers::Input> input,
+    const fetchers::Input & input,
     Value & v)
 {
+    assert(input.isImmutable());
+
     state.mkAttrs(v, 8);
 
     auto storePath = state.store->printStorePath(tree.storePath);
 
     mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath}));
 
-    assert(tree.info.narHash);
-    mkString(*state.allocAttr(v, state.symbols.create("narHash")),
-        tree.info.narHash.to_string(SRI));
+    // FIXME: support arbitrary input attributes.
 
-    if (input->getRev()) {
-        mkString(*state.allocAttr(v, state.symbols.create("rev")), input->getRev()->gitRev());
-        mkString(*state.allocAttr(v, state.symbols.create("shortRev")), input->getRev()->gitShortRev());
+    auto narHash = input.getNarHash();
+    assert(narHash);
+    mkString(*state.allocAttr(v, state.symbols.create("narHash")),
+        narHash->to_string(SRI, true));
+
+    if (auto rev = input.getRev()) {
+        mkString(*state.allocAttr(v, state.symbols.create("rev")), rev->gitRev());
+        mkString(*state.allocAttr(v, state.symbols.create("shortRev")), rev->gitShortRev());
     }
 
-    if (tree.info.revCount)
-        mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *tree.info.revCount);
+    if (auto revCount = input.getRevCount())
+        mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *revCount);
 
-    if (tree.info.lastModified) {
-        mkInt(*state.allocAttr(v, state.symbols.create("lastModified")), *tree.info.lastModified);
+    if (auto lastModified = input.getLastModified()) {
+        mkInt(*state.allocAttr(v, state.symbols.create("lastModified")), *lastModified);
         mkString(*state.allocAttr(v, state.symbols.create("lastModifiedDate")),
-            fmt("%s", std::put_time(std::gmtime(&*tree.info.lastModified), "%Y%m%d%H%M%S")));
+            fmt("%s", std::put_time(std::gmtime(&*lastModified), "%Y%m%d%H%M%S")));
     }
 
     v.attrs->sort();
@@ -47,7 +52,7 @@ static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, V
 {
     settings.requireExperimentalFeature("flakes");
 
-    std::shared_ptr<const fetchers::Input> input;
+    fetchers::Input input;
     PathSet context;
 
     state.forceValue(*args[0]);
@@ -62,7 +67,7 @@ static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, V
             if (attr.value->type == tString)
                 attrs.emplace(attr.name, attr.value->string.s);
             else if (attr.value->type == tBool)
-                attrs.emplace(attr.name, attr.value->boolean);
+                attrs.emplace(attr.name, fetchers::Explicit<bool>{attr.value->boolean});
             else if (attr.value->type == tInt)
                 attrs.emplace(attr.name, attr.value->integer);
             else
@@ -71,20 +76,22 @@ static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, V
         }
 
         if (!attrs.count("type"))
-            throw Error("attribute 'type' is missing in call to 'fetchTree', at %s", pos);
+            throw Error({
+                .hint = hintfmt("attribute 'type' is missing in call to 'fetchTree'"),
+                .errPos = pos
+            });
 
-        input = fetchers::inputFromAttrs(attrs);
+        input = fetchers::Input::fromAttrs(std::move(attrs));
     } else
-        input = fetchers::inputFromURL(state.coerceToString(pos, *args[0], context, false, false));
+        input = fetchers::Input::fromURL(state.coerceToString(pos, *args[0], context, false, false));
 
-    if (!evalSettings.pureEval && !input->isDirect())
+    if (!evalSettings.pureEval && !input.isDirect())
         input = lookupInRegistries(state.store, input).first;
 
-    if (evalSettings.pureEval && !input->isImmutable())
+    if (evalSettings.pureEval && !input.isImmutable())
         throw Error("in pure evaluation mode, 'fetchTree' requires an immutable input, at %s", pos);
 
-    // FIXME: use fetchOrSubstituteTree
-    auto [tree, input2] = input->fetchTree(state.store);
+    auto [tree, input2] = input.fetch(state.store);
 
     if (state.allowedPaths)
         state.allowedPaths->insert(tree.actualPath);
@@ -111,17 +118,21 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
             if (n == "url")
                 url = state.forceStringNoCtx(*attr.value, *attr.pos);
             else if (n == "sha256")
-                expectedHash = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256);
+                expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256);
             else if (n == "name")
                 name = state.forceStringNoCtx(*attr.value, *attr.pos);
             else
-                throw EvalError("unsupported argument '%s' to '%s', at %s",
-                    attr.name, who, *attr.pos);
-        }
+                throw EvalError({
+                    .hint = hintfmt("unsupported argument '%s' to '%s'", attr.name, who),
+                    .errPos = *attr.pos
+                });
+            }
 
         if (!url)
-            throw EvalError("'url' argument required, at %s", pos);
-
+            throw EvalError({
+                .hint = hintfmt("'url' argument required"),
+                .errPos = pos
+            });
     } else
         url = state.forceStringNoCtx(*args[0], pos);
 
@@ -137,7 +148,7 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
 
     auto storePath =
         unpack
-        ? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).storePath
+        ? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).first.storePath
         : fetchers::downloadFile(state.store, *url, name, (bool) expectedHash).storePath;
 
     auto path = state.store->toRealPath(storePath);
@@ -148,7 +159,7 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
             : hashFile(htSHA256, path);
         if (hash != *expectedHash)
             throw Error((unsigned int) 102, "hash mismatch in file downloaded from '%s':\n  wanted: %s\n  got:    %s",
-                *url, expectedHash->to_string(), hash.to_string());
+                *url, expectedHash->to_string(Base32, true), hash.to_string(Base32, true));
     }
 
     if (state.allowedPaths)

src/libexpr/primops/fromTOML.cc

@@ -81,7 +81,10 @@ static void prim_fromTOML(EvalState & state, const Pos & pos, Value * * args, Va
     try {
         visit(v, parser(tomlStream).parse());
     } catch (std::runtime_error & e) {
-        throw EvalError("while parsing a TOML string at %s: %s", pos, e.what());
+        throw EvalError({
+            .hint = hintfmt("while parsing a TOML string: %s", e.what()),
+            .errPos = pos
+        });
     }
 }
 

src/libexpr/value-to-json.cc

@@ -79,7 +79,7 @@ void printValueAsJSON(EvalState & state, bool strict,
             break;
 
         default:
-            throw TypeError(format("cannot convert %1% to JSON") % showType(v));
+            throw TypeError("cannot convert %1% to JSON", showType(v));
     }
 }
 
@@ -93,7 +93,7 @@ void printValueAsJSON(EvalState & state, bool strict,
 void ExternalValueBase::printValueAsJSON(EvalState & state, bool strict,
     JSONPlaceholder & out, PathSet & context) const
 {
-    throw TypeError(format("cannot convert %1% to JSON") % showType());
+    throw TypeError("cannot convert %1% to JSON", showType());
 }
 
 

src/libexpr/value.hh

@@ -171,6 +171,8 @@ struct Value
        computation. In particular, function applications are
        non-trivial. */
     bool isTrivial() const;
+
+    std::vector<std::pair<Path, std::string>> getContext();
 };
 
 

src/libfetchers/attrs.cc

@@ -27,7 +27,7 @@ nlohmann::json attrsToJson(const Attrs & attrs)
 {
     nlohmann::json json;
     for (auto & attr : attrs) {
-        if (auto v = std::get_if<int64_t>(&attr.second)) {
+        if (auto v = std::get_if<uint64_t>(&attr.second)) {
             json[attr.first] = *v;
         } else if (auto v = std::get_if<std::string>(&attr.second)) {
             json[attr.first] = *v;
@@ -55,16 +55,16 @@ std::string getStrAttr(const Attrs & attrs, const std::string & name)
     return *s;
 }
 
-std::optional<int64_t> maybeGetIntAttr(const Attrs & attrs, const std::string & name)
+std::optional<uint64_t> maybeGetIntAttr(const Attrs & attrs, const std::string & name)
 {
     auto i = attrs.find(name);
     if (i == attrs.end()) return {};
-    if (auto v = std::get_if<int64_t>(&i->second))
+    if (auto v = std::get_if<uint64_t>(&i->second))
         return *v;
     throw Error("input attribute '%s' is not an integer", name);
 }
 
-int64_t getIntAttr(const Attrs & attrs, const std::string & name)
+uint64_t getIntAttr(const Attrs & attrs, const std::string & name)
 {
     auto s = maybeGetIntAttr(attrs, name);
     if (!s)
@@ -76,8 +76,8 @@ std::optional<bool> maybeGetBoolAttr(const Attrs & attrs, const std::string & na
 {
     auto i = attrs.find(name);
     if (i == attrs.end()) return {};
-    if (auto v = std::get_if<int64_t>(&i->second))
-        return *v;
+    if (auto v = std::get_if<Explicit<bool>>(&i->second))
+        return v->t;
     throw Error("input attribute '%s' is not a Boolean", name);
 }
 
@@ -93,7 +93,7 @@ std::map<std::string, std::string> attrsToQuery(const Attrs & attrs)
 {
     std::map<std::string, std::string> query;
     for (auto & attr : attrs) {
-        if (auto v = std::get_if<int64_t>(&attr.second)) {
+        if (auto v = std::get_if<uint64_t>(&attr.second)) {
             query.insert_or_assign(attr.first, fmt("%d", *v));
         } else if (auto v = std::get_if<std::string>(&attr.second)) {
             query.insert_or_assign(attr.first, *v);

src/libfetchers/attrs.hh

@@ -13,9 +13,14 @@ namespace nix::fetchers {
 template<typename T>
 struct Explicit {
     T t;
+
+    bool operator ==(const Explicit<T> & other) const
+    {
+        return t == other.t;
+    }
 };
 
-typedef std::variant<std::string, int64_t, Explicit<bool>> Attr;
+typedef std::variant<std::string, uint64_t, Explicit<bool>> Attr;
 typedef std::map<std::string, Attr> Attrs;
 
 Attrs jsonToAttrs(const nlohmann::json & json);
@@ -26,9 +31,9 @@ std::optional<std::string> maybeGetStrAttr(const Attrs & attrs, const std::strin
 
 std::string getStrAttr(const Attrs & attrs, const std::string & name);
 
-std::optional<int64_t> maybeGetIntAttr(const Attrs & attrs, const std::string & name);
+std::optional<uint64_t> maybeGetIntAttr(const Attrs & attrs, const std::string & name);
 
-int64_t getIntAttr(const Attrs & attrs, const std::string & name);
+uint64_t getIntAttr(const Attrs & attrs, const std::string & name);
 
 std::optional<bool> maybeGetBoolAttr(const Attrs & attrs, const std::string & name);
 

src/libfetchers/cache.hh

@@ -6,6 +6,8 @@ namespace nix::fetchers {
 
 struct Cache
 {
+    virtual ~Cache() { }
+
     virtual void add(
         ref<Store> store,
         const Attrs & inAttrs,

src/libfetchers/fetchers.cc

@@ -5,82 +5,265 @@
 
 namespace nix::fetchers {
 
-std::unique_ptr<std::vector<std::unique_ptr<InputScheme>>> inputSchemes = nullptr;
+std::unique_ptr<std::vector<std::shared_ptr<InputScheme>>> inputSchemes = nullptr;
 
-void registerInputScheme(std::unique_ptr<InputScheme> && inputScheme)
+void registerInputScheme(std::shared_ptr<InputScheme> && inputScheme)
 {
-    if (!inputSchemes) inputSchemes = std::make_unique<std::vector<std::unique_ptr<InputScheme>>>();
+    if (!inputSchemes) inputSchemes = std::make_unique<std::vector<std::shared_ptr<InputScheme>>>();
     inputSchemes->push_back(std::move(inputScheme));
 }
 
-std::unique_ptr<Input> inputFromURL(const ParsedURL & url)
+Input Input::fromURL(const std::string & url)
+{
+    return fromURL(parseURL(url));
+}
+
+static void fixupInput(Input & input)
+{
+    // Check common attributes.
+    input.getType();
+    input.getRef();
+    if (input.getRev())
+        input.immutable = true;
+    input.getRevCount();
+    input.getLastModified();
+    if (input.getNarHash())
+        input.immutable = true;
+}
+
+Input Input::fromURL(const ParsedURL & url)
 {
     for (auto & inputScheme : *inputSchemes) {
         auto res = inputScheme->inputFromURL(url);
-        if (res) return res;
+        if (res) {
+            res->scheme = inputScheme;
+            fixupInput(*res);
+            return std::move(*res);
+        }
     }
+
     throw Error("input '%s' is unsupported", url.url);
 }
 
-std::unique_ptr<Input> inputFromURL(const std::string & url)
+Input Input::fromAttrs(Attrs && attrs)
 {
-    return inputFromURL(parseURL(url));
-}
-
-std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs)
-{
-    auto attrs2(attrs);
-    attrs2.erase("narHash");
     for (auto & inputScheme : *inputSchemes) {
-        auto res = inputScheme->inputFromAttrs(attrs2);
+        auto res = inputScheme->inputFromAttrs(attrs);
         if (res) {
-            if (auto narHash = maybeGetStrAttr(attrs, "narHash"))
-                // FIXME: require SRI hash.
-                res->narHash = Hash(*narHash);
-            return res;
+            res->scheme = inputScheme;
+            fixupInput(*res);
+            return std::move(*res);
         }
     }
-    throw Error("input '%s' is unsupported", attrsToJson(attrs));
+
+    Input input;
+    input.attrs = attrs;
+    fixupInput(input);
+    return input;
+}
+
+ParsedURL Input::toURL() const
+{
+    if (!scheme)
+        throw Error("cannot show unsupported input '%s'", attrsToJson(attrs));
+    return scheme->toURL(*this);
+}
+
+std::string Input::to_string() const
+{
+    return toURL().to_string();
 }
 
 Attrs Input::toAttrs() const
 {
-    auto attrs = toAttrsInternal();
-    if (narHash)
-        attrs.emplace("narHash", narHash->to_string(SRI));
-    attrs.emplace("type", type());
     return attrs;
 }
 
-std::pair<Tree, std::shared_ptr<const Input>> Input::fetchTree(ref<Store> store) const
+bool Input::hasAllInfo() const
 {
-    auto [tree, input] = fetchTreeInternal(store);
+    return getNarHash() && scheme && scheme->hasAllInfo(*this);
+}
+
+bool Input::operator ==(const Input & other) const
+{
+    return attrs == other.attrs;
+}
+
+bool Input::contains(const Input & other) const
+{
+    if (*this == other) return true;
+    auto other2(other);
+    other2.attrs.erase("ref");
+    other2.attrs.erase("rev");
+    if (*this == other2) return true;
+    return false;
+}
+
+std::pair<Tree, Input> Input::fetch(ref<Store> store) const
+{
+    if (!scheme)
+        throw Error("cannot fetch unsupported input '%s'", attrsToJson(toAttrs()));
+
+    /* The tree may already be in the Nix store, or it could be
+       substituted (which is often faster than fetching from the
+       original source). So check that. */
+    if (hasAllInfo()) {
+        try {
+            auto storePath = computeStorePath(*store);
+
+            store->ensurePath(storePath);
+
+            debug("using substituted/cached input '%s' in '%s'",
+                to_string(), store->printStorePath(storePath));
+
+            auto actualPath = store->toRealPath(storePath);
+
+            return {fetchers::Tree(std::move(actualPath), std::move(storePath)), *this};
+        } catch (Error & e) {
+            debug("substitution of input '%s' failed: %s", to_string(), e.what());
+        }
+    }
+
+    auto [tree, input] = scheme->fetch(store, *this);
 
     if (tree.actualPath == "")
         tree.actualPath = store->toRealPath(tree.storePath);
 
-    if (!tree.info.narHash)
-        tree.info.narHash = store->queryPathInfo(tree.storePath)->narHash;
+    auto narHash = store->queryPathInfo(tree.storePath)->narHash;
+    input.attrs.insert_or_assign("narHash", narHash.to_string(SRI, true));
 
-    if (input->narHash)
-        assert(input->narHash == tree.info.narHash);
+    if (auto prevNarHash = getNarHash()) {
+        if (narHash != *prevNarHash)
+            throw Error("NAR hash mismatch in input '%s' (%s), expected '%s', got '%s'",
+                to_string(), tree.actualPath, prevNarHash->to_string(SRI, true), narHash.to_string(SRI, true));
+    }
 
-    if (narHash && narHash != input->narHash)
-        throw Error("NAR hash mismatch in input '%s' (%s), expected '%s', got '%s'",
-            to_string(), tree.actualPath, narHash->to_string(SRI), input->narHash->to_string(SRI));
+    if (auto prevLastModified = getLastModified()) {
+        if (input.getLastModified() != prevLastModified)
+            throw Error("'lastModified' attribute mismatch in input '%s', expected %d",
+                input.to_string(), *prevLastModified);
+    }
+
+    if (auto prevRevCount = getRevCount()) {
+        if (input.getRevCount() != prevRevCount)
+            throw Error("'revCount' attribute mismatch in input '%s', expected %d",
+                input.to_string(), *prevRevCount);
+    }
+
+    input.immutable = true;
+
+    assert(input.hasAllInfo());
 
     return {std::move(tree), input};
 }
 
-std::shared_ptr<const Input> Input::applyOverrides(
+Input Input::applyOverrides(
     std::optional<std::string> ref,
     std::optional<Hash> rev) const
+{
+    if (!scheme) return *this;
+    return scheme->applyOverrides(*this, ref, rev);
+}
+
+void Input::clone(const Path & destDir) const
+{
+    assert(scheme);
+    scheme->clone(*this, destDir);
+}
+
+std::optional<Path> Input::getSourcePath() const
+{
+    assert(scheme);
+    return scheme->getSourcePath(*this);
+}
+
+void Input::markChangedFile(
+    std::string_view file,
+    std::optional<std::string> commitMsg) const
+{
+    assert(scheme);
+    return scheme->markChangedFile(*this, file, commitMsg);
+}
+
+StorePath Input::computeStorePath(Store & store) const
+{
+    auto narHash = getNarHash();
+    if (!narHash)
+        throw Error("cannot compute store path for mutable input '%s'", to_string());
+    return store.makeFixedOutputPath(FileIngestionMethod::Recursive, *narHash, "source");
+}
+
+std::string Input::getType() const
+{
+    return getStrAttr(attrs, "type");
+}
+
+std::optional<Hash> Input::getNarHash() const
+{
+    if (auto s = maybeGetStrAttr(attrs, "narHash"))
+        // FIXME: require SRI hash.
+        return newHashAllowEmpty(*s, htSHA256);
+    return {};
+}
+
+std::optional<std::string> Input::getRef() const
+{
+    if (auto s = maybeGetStrAttr(attrs, "ref"))
+        return *s;
+    return {};
+}
+
+std::optional<Hash> Input::getRev() const
+{
+    if (auto s = maybeGetStrAttr(attrs, "rev"))
+        return Hash(*s, htSHA1);
+    return {};
+}
+
+std::optional<uint64_t> Input::getRevCount() const
+{
+    if (auto n = maybeGetIntAttr(attrs, "revCount"))
+        return *n;
+    return {};
+}
+
+std::optional<time_t> Input::getLastModified() const
+{
+    if (auto n = maybeGetIntAttr(attrs, "lastModified"))
+        return *n;
+    return {};
+}
+
+ParsedURL InputScheme::toURL(const Input & input)
+{
+    throw Error("don't know how to convert input '%s' to a URL", attrsToJson(input.attrs));
+}
+
+Input InputScheme::applyOverrides(
+    const Input & input,
+    std::optional<std::string> ref,
+    std::optional<Hash> rev)
 {
     if (ref)
-        throw Error("don't know how to apply '%s' to '%s'", *ref, to_string());
+        throw Error("don't know how to set branch/tag name of input '%s' to '%s'", input.to_string(), *ref);
     if (rev)
-        throw Error("don't know how to apply '%s' to '%s'", rev->to_string(Base16, false), to_string());
-    return shared_from_this();
+        throw Error("don't know how to set revision of input '%s' to '%s'", input.to_string(), rev->gitRev());
+    return input;
+}
+
+std::optional<Path> InputScheme::getSourcePath(const Input & input)
+{
+    return {};
+}
+
+void InputScheme::markChangedFile(const Input & input, std::string_view file, std::optional<std::string> commitMsg)
+{
+    assert(false);
+}
+
+void InputScheme::clone(const Input & input, const Path & destDir)
+{
+    throw Error("do not know how to clone input '%s'", input.to_string());
 }
 
 }

src/libfetchers/fetchers.hh

@@ -3,7 +3,6 @@
 #include "types.hh"
 #include "hash.hh"
 #include "path.hh"
-#include "tree-info.hh"
 #include "attrs.hh"
 #include "url.hh"
 
@@ -13,89 +12,101 @@ namespace nix { class Store; }
 
 namespace nix::fetchers {
 
-struct Input;
-
 struct Tree
 {
     Path actualPath;
     StorePath storePath;
-    TreeInfo info;
+    Tree(Path && actualPath, StorePath && storePath) : actualPath(actualPath), storePath(std::move(storePath)) {}
 };
 
-struct Input : std::enable_shared_from_this<Input>
+struct InputScheme;
+
+struct Input
 {
-    std::optional<Hash> narHash; // FIXME: implement
+    friend class InputScheme;
 
-    virtual std::string type() const = 0;
+    std::shared_ptr<InputScheme> scheme; // note: can be null
+    Attrs attrs;
+    bool immutable = false;
+    bool direct = true;
 
-    virtual ~Input() { }
+public:
+    static Input fromURL(const std::string & url);
 
-    virtual bool operator ==(const Input & other) const { return false; }
+    static Input fromURL(const ParsedURL & url);
 
-    /* Check whether this is a "direct" input, that is, not
-       one that goes through a registry. */
-    virtual bool isDirect() const { return true; }
+    static Input fromAttrs(Attrs && attrs);
 
-    /* Check whether this is an "immutable" input, that is,
-       one that contains a commit hash or content hash. */
-    virtual bool isImmutable() const { return (bool) narHash; }
+    ParsedURL toURL() const;
 
-    virtual bool contains(const Input & other) const { return false; }
-
-    virtual std::optional<std::string> getRef() const { return {}; }
-
-    virtual std::optional<Hash> getRev() const { return {}; }
-
-    virtual ParsedURL toURL() const = 0;
-
-    std::string to_string() const
-    {
-        return toURL().to_string();
-    }
+    std::string to_string() const;
 
     Attrs toAttrs() const;
 
-    std::pair<Tree, std::shared_ptr<const Input>> fetchTree(ref<Store> store) const;
+    /* Check whether this is a "direct" input, that is, not
+       one that goes through a registry. */
+    bool isDirect() const { return direct; }
 
-    virtual std::shared_ptr<const Input> applyOverrides(
+    /* Check whether this is an "immutable" input, that is,
+       one that contains a commit hash or content hash. */
+    bool isImmutable() const { return immutable; }
+
+    bool hasAllInfo() const;
+
+    bool operator ==(const Input & other) const;
+
+    bool contains(const Input & other) const;
+
+    std::pair<Tree, Input> fetch(ref<Store> store) const;
+
+    Input applyOverrides(
         std::optional<std::string> ref,
         std::optional<Hash> rev) const;
 
-    virtual std::optional<Path> getSourcePath() const { return {}; }
+    void clone(const Path & destDir) const;
 
-    virtual void markChangedFile(
+    std::optional<Path> getSourcePath() const;
+
+    void markChangedFile(
         std::string_view file,
-        std::optional<std::string> commitMsg) const
-    { assert(false); }
+        std::optional<std::string> commitMsg) const;
 
-    virtual void clone(const Path & destDir) const
-    {
-        throw Error("do not know how to clone input '%s'", to_string());
-    }
+    StorePath computeStorePath(Store & store) const;
 
-private:
-
-    virtual std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(ref<Store> store) const = 0;
-
-    virtual Attrs toAttrsInternal() const = 0;
+    // Convience functions for common attributes.
+    std::string getType() const;
+    std::optional<Hash> getNarHash() const;
+    std::optional<std::string> getRef() const;
+    std::optional<Hash> getRev() const;
+    std::optional<uint64_t> getRevCount() const;
+    std::optional<time_t> getLastModified() const;
 };
 
 struct InputScheme
 {
-    virtual ~InputScheme() { }
+    virtual std::optional<Input> inputFromURL(const ParsedURL & url) = 0;
 
-    virtual std::unique_ptr<Input> inputFromURL(const ParsedURL & url) = 0;
+    virtual std::optional<Input> inputFromAttrs(const Attrs & attrs) = 0;
 
-    virtual std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) = 0;
+    virtual ParsedURL toURL(const Input & input);
+
+    virtual bool hasAllInfo(const Input & input) = 0;
+
+    virtual Input applyOverrides(
+        const Input & input,
+        std::optional<std::string> ref,
+        std::optional<Hash> rev);
+
+    virtual void clone(const Input & input, const Path & destDir);
+
+    virtual std::optional<Path> getSourcePath(const Input & input);
+
+    virtual void markChangedFile(const Input & input, std::string_view file, std::optional<std::string> commitMsg);
+
+    virtual std::pair<Tree, Input> fetch(ref<Store> store, const Input & input) = 0;
 };
 
-std::unique_ptr<Input> inputFromURL(const ParsedURL & url);
-
-std::unique_ptr<Input> inputFromURL(const std::string & url);
-
-std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs);
-
-void registerInputScheme(std::unique_ptr<InputScheme> && fetcher);
+void registerInputScheme(std::shared_ptr<InputScheme> && fetcher);
 
 struct DownloadFileResult
 {
@@ -110,7 +121,7 @@ DownloadFileResult downloadFile(
     const std::string & name,
     bool immutable);
 
-Tree downloadTarball(
+std::pair<Tree, time_t> downloadTarball(
     ref<Store> store,
     const std::string & url,
     const std::string & name,

src/libfetchers/git.cc

@@ -22,137 +22,152 @@ static bool isNotDotGitDirectory(const Path & path)
     return not std::regex_match(path, gitDirRegex);
 }
 
-struct GitInput : Input
+struct GitInputScheme : InputScheme
 {
-    ParsedURL url;
-    std::optional<std::string> ref;
-    std::optional<Hash> rev;
-    bool shallow = false;
-    bool submodules = false;
-
-    GitInput(const ParsedURL & url) : url(url)
-    { }
-
-    std::string type() const override { return "git"; }
-
-    bool operator ==(const Input & other) const override
+    std::optional<Input> inputFromURL(const ParsedURL & url) override
     {
-        auto other2 = dynamic_cast<const GitInput *>(&other);
-        return
-            other2
-            && url == other2->url
-            && rev == other2->rev
-            && ref == other2->ref;
-    }
+        if (url.scheme != "git" &&
+            url.scheme != "git+http" &&
+            url.scheme != "git+https" &&
+            url.scheme != "git+ssh" &&
+            url.scheme != "git+file") return {};
 
-    bool isImmutable() const override
-    {
-        return (bool) rev || narHash;
-    }
+        auto url2(url);
+        if (hasPrefix(url2.scheme, "git+")) url2.scheme = std::string(url2.scheme, 4);
+        url2.query.clear();
 
-    std::optional<std::string> getRef() const override { return ref; }
-
-    std::optional<Hash> getRev() const override { return rev; }
-
-    ParsedURL toURL() const override
-    {
-        ParsedURL url2(url);
-        if (url2.scheme != "git") url2.scheme = "git+" + url2.scheme;
-        if (rev) url2.query.insert_or_assign("rev", rev->gitRev());
-        if (ref) url2.query.insert_or_assign("ref", *ref);
-        if (shallow) url2.query.insert_or_assign("shallow", "1");
-        return url2;
-    }
-
-    Attrs toAttrsInternal() const override
-    {
         Attrs attrs;
-        attrs.emplace("url", url.to_string());
-        if (ref)
-            attrs.emplace("ref", *ref);
-        if (rev)
-            attrs.emplace("rev", rev->gitRev());
-        if (shallow)
-            attrs.emplace("shallow", true);
-        if (submodules)
-            attrs.emplace("submodules", true);
-        return attrs;
+        attrs.emplace("type", "git");
+
+        for (auto &[name, value] : url.query) {
+            if (name == "rev" || name == "ref")
+                attrs.emplace(name, value);
+            else if (name == "shallow")
+                attrs.emplace(name, Explicit<bool> { value == "1" });
+            else
+                url2.query.emplace(name, value);
+        }
+
+        attrs.emplace("url", url2.to_string());
+
+        return inputFromAttrs(attrs);
     }
 
-    void clone(const Path & destDir) const override
+    std::optional<Input> inputFromAttrs(const Attrs & attrs) override
     {
-        auto [isLocal, actualUrl] = getActualUrl();
+        if (maybeGetStrAttr(attrs, "type") != "git") return {};
+
+        for (auto & [name, value] : attrs)
+            if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules" && name != "lastModified" && name != "revCount" && name != "narHash")
+                throw Error("unsupported Git input attribute '%s'", name);
+
+        parseURL(getStrAttr(attrs, "url"));
+        maybeGetBoolAttr(attrs, "shallow");
+        maybeGetBoolAttr(attrs, "submodules");
+
+        if (auto ref = maybeGetStrAttr(attrs, "ref")) {
+            if (std::regex_search(*ref, badGitRefRegex))
+                throw BadURL("invalid Git branch/tag name '%s'", *ref);
+        }
+
+        Input input;
+        input.attrs = attrs;
+        return input;
+    }
+
+    ParsedURL toURL(const Input & input) override
+    {
+        auto url = parseURL(getStrAttr(input.attrs, "url"));
+        if (url.scheme != "git") url.scheme = "git+" + url.scheme;
+        if (auto rev = input.getRev()) url.query.insert_or_assign("rev", rev->gitRev());
+        if (auto ref = input.getRef()) url.query.insert_or_assign("ref", *ref);
+        if (maybeGetBoolAttr(input.attrs, "shallow").value_or(false))
+            url.query.insert_or_assign("shallow", "1");
+        return url;
+    }
+
+    bool hasAllInfo(const Input & input) override
+    {
+        bool maybeDirty = !input.getRef();
+        bool shallow = maybeGetBoolAttr(input.attrs, "shallow").value_or(false);
+        return
+            maybeGetIntAttr(input.attrs, "lastModified")
+            && (shallow || maybeDirty || maybeGetIntAttr(input.attrs, "revCount"));
+    }
+
+    Input applyOverrides(
+        const Input & input,
+        std::optional<std::string> ref,
+        std::optional<Hash> rev) override
+    {
+        auto res(input);
+        if (rev) res.attrs.insert_or_assign("rev", rev->gitRev());
+        if (ref) res.attrs.insert_or_assign("ref", *ref);
+        if (!res.getRef() && res.getRev())
+            throw Error("Git input '%s' has a commit hash but no branch/tag name", res.to_string());
+        return res;
+    }
+
+    void clone(const Input & input, const Path & destDir) override
+    {
+        auto [isLocal, actualUrl] = getActualUrl(input);
 
         Strings args = {"clone"};
 
         args.push_back(actualUrl);
 
-        if (ref) {
+        if (auto ref = input.getRef()) {
             args.push_back("--branch");
             args.push_back(*ref);
         }
 
-        if (rev) throw Error("cloning a specific revision is not implemented");
+        if (input.getRev()) throw Error("cloning a specific revision is not implemented");
 
         args.push_back(destDir);
 
         runProgram("git", true, args);
     }
 
-    std::shared_ptr<const Input> applyOverrides(
-        std::optional<std::string> ref,
-        std::optional<Hash> rev) const override
+    std::optional<Path> getSourcePath(const Input & input) override
     {
-        if (!ref && !rev) return shared_from_this();
-
-        auto res = std::make_shared<GitInput>(*this);
-
-        if (ref) res->ref = ref;
-        if (rev) res->rev = rev;
-
-        if (!res->ref && res->rev)
-            throw Error("Git input '%s' has a commit hash but no branch/tag name", res->to_string());
-
-        return res;
-    }
-
-    std::optional<Path> getSourcePath() const override
-    {
-        if (url.scheme == "file" && !ref && !rev)
+        auto url = parseURL(getStrAttr(input.attrs, "url"));
+        if (url.scheme == "file" && !input.getRef() && !input.getRev())
             return url.path;
         return {};
     }
 
-    void markChangedFile(std::string_view file, std::optional<std::string> commitMsg) const override
+    void markChangedFile(const Input & input, std::string_view file, std::optional<std::string> commitMsg) override
     {
-        auto sourcePath = getSourcePath();
+        auto sourcePath = getSourcePath(input);
         assert(sourcePath);
 
         runProgram("git", true,
-            { "-C", *sourcePath, "add", "--force", "--intent-to-add", std::string(file) });
+            { "-C", *sourcePath, "add", "--force", "--intent-to-add", "--", std::string(file) });
 
         if (commitMsg)
             runProgram("git", true,
                 { "-C", *sourcePath, "commit", std::string(file), "-m", *commitMsg });
     }
 
-    std::pair<bool, std::string> getActualUrl() const
+    std::pair<bool, std::string> getActualUrl(const Input & input) const
     {
         // Don't clone file:// URIs (but otherwise treat them the
         // same as remote URIs, i.e. don't use the working tree or
         // HEAD).
         static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; // for testing
+        auto url = parseURL(getStrAttr(input.attrs, "url"));
         bool isLocal = url.scheme == "file" && !forceHttp;
         return {isLocal, isLocal ? url.path : url.base};
     }
 
-    std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override
+    std::pair<Tree, Input> fetch(ref<Store> store, const Input & _input) override
     {
         auto name = "source";
 
-        auto input = std::make_shared<GitInput>(*this);
+        Input input(_input);
 
-        assert(!rev || rev->type == htSHA1);
+        bool shallow = maybeGetBoolAttr(input.attrs, "shallow").value_or(false);
+        bool submodules = maybeGetBoolAttr(input.attrs, "submodules").value_or(false);
 
         std::string cacheType = "git";
         if (shallow) cacheType += "-shallow";
@@ -163,39 +178,35 @@ struct GitInput : Input
             return Attrs({
                 {"type", cacheType},
                 {"name", name},
-                {"rev", input->rev->gitRev()},
+                {"rev", input.getRev()->gitRev()},
             });
         };
 
         auto makeResult = [&](const Attrs & infoAttrs, StorePath && storePath)
-            -> std::pair<Tree, std::shared_ptr<const Input>>
+            -> std::pair<Tree, Input>
         {
-            assert(input->rev);
-            assert(!rev || rev == input->rev);
+            assert(input.getRev());
+            assert(!_input.getRev() || _input.getRev() == input.getRev());
+            if (!shallow)
+                input.attrs.insert_or_assign("revCount", getIntAttr(infoAttrs, "revCount"));
+            input.attrs.insert_or_assign("lastModified", getIntAttr(infoAttrs, "lastModified"));
             return {
-                Tree {
-                    .actualPath = store->toRealPath(storePath),
-                    .storePath = std::move(storePath),
-                    .info = TreeInfo {
-                        .revCount = shallow ? std::nullopt : std::optional(getIntAttr(infoAttrs, "revCount")),
-                        .lastModified = getIntAttr(infoAttrs, "lastModified"),
-                    },
-                },
+                Tree(store->toRealPath(storePath), std::move(storePath)),
                 input
             };
         };
 
-        if (rev) {
+        if (input.getRev()) {
             if (auto res = getCache()->lookup(store, getImmutableAttrs()))
                 return makeResult(res->first, std::move(res->second));
         }
 
-        auto [isLocal, actualUrl_] = getActualUrl();
+        auto [isLocal, actualUrl_] = getActualUrl(input);
         auto actualUrl = actualUrl_; // work around clang bug
 
         // If this is a local directory and no ref or revision is
         // given, then allow the use of an unclean working tree.
-        if (!input->ref && !input->rev && isLocal) {
+        if (!input.getRef() && !input.getRev() && isLocal) {
             bool clean = false;
 
             /* Check whether this repo has any commits. There are
@@ -252,37 +263,37 @@ struct GitInput : Input
                     return files.count(file);
                 };
 
-                auto storePath = store->addToStore("source", actualUrl, true, htSHA256, filter);
+                auto storePath = store->addToStore("source", actualUrl, FileIngestionMethod::Recursive, htSHA256, filter);
 
-                auto tree = Tree {
-                    .actualPath = store->printStorePath(storePath),
-                    .storePath = std::move(storePath),
-                    .info = TreeInfo {
-                        // FIXME: maybe we should use the timestamp of the last
-                        // modified dirty file?
-                        .lastModified = haveCommits ? std::stoull(runProgram("git", true, { "-C", actualUrl, "log", "-1", "--format=%ct", "HEAD" })) : 0,
-                    }
+                // FIXME: maybe we should use the timestamp of the last
+                // modified dirty file?
+                input.attrs.insert_or_assign(
+                    "lastModified",
+                    haveCommits ? std::stoull(runProgram("git", true, { "-C", actualUrl, "log", "-1", "--format=%ct", "HEAD" })) : 0);
+
+                return {
+                    Tree(store->printStorePath(storePath), std::move(storePath)),
+                    input
                 };
-
-                return {std::move(tree), input};
             }
         }
 
-        if (!input->ref) input->ref = isLocal ? readHead(actualUrl) : "master";
+        if (!input.getRef()) input.attrs.insert_or_assign("ref", isLocal ? readHead(actualUrl) : "master");
 
         Attrs mutableAttrs({
             {"type", cacheType},
             {"name", name},
             {"url", actualUrl},
-            {"ref", *input->ref},
+            {"ref", *input.getRef()},
         });
 
         Path repoDir;
 
         if (isLocal) {
 
-            if (!input->rev)
-                input->rev = Hash(chomp(runProgram("git", true, { "-C", actualUrl, "rev-parse", *input->ref })), htSHA1);
+            if (!input.getRev())
+                input.attrs.insert_or_assign("rev",
+                    Hash(chomp(runProgram("git", true, { "-C", actualUrl, "rev-parse", *input.getRef() })), htSHA1).gitRev());
 
             repoDir = actualUrl;
 
@@ -290,8 +301,8 @@ struct GitInput : Input
 
             if (auto res = getCache()->lookup(store, mutableAttrs)) {
                 auto rev2 = Hash(getStrAttr(res->first, "rev"), htSHA1);
-                if (!rev || rev == rev2) {
-                    input->rev = rev2;
+                if (!input.getRev() || input.getRev() == rev2) {
+                    input.attrs.insert_or_assign("rev", rev2.gitRev());
                     return makeResult(res->first, std::move(res->second));
                 }
             }
@@ -305,18 +316,18 @@ struct GitInput : Input
             }
 
             Path localRefFile =
-                input->ref->compare(0, 5, "refs/") == 0
-                ? cacheDir + "/" + *input->ref
-                : cacheDir + "/refs/heads/" + *input->ref;
+                input.getRef()->compare(0, 5, "refs/") == 0
+                ? cacheDir + "/" + *input.getRef()
+                : cacheDir + "/refs/heads/" + *input.getRef();
 
             bool doFetch;
             time_t now = time(0);
 
             /* If a rev was specified, we need to fetch if it's not in the
                repo. */
-            if (input->rev) {
+            if (input.getRev()) {
                 try {
-                    runProgram("git", true, { "-C", repoDir, "cat-file", "-e", input->rev->gitRev() });
+                    runProgram("git", true, { "-C", repoDir, "cat-file", "-e", input.getRev()->gitRev() });
                     doFetch = false;
                 } catch (ExecError & e) {
                     if (WIFEXITED(e.status)) {
@@ -339,7 +350,11 @@ struct GitInput : Input
                 // FIXME: git stderr messes up our progress indicator, so
                 // we're using --quiet for now. Should process its stderr.
                 try {
-                    runProgram("git", true, { "-C", repoDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", *input->ref, *input->ref) });
+                    auto ref = input.getRef();
+                    auto fetchRef = ref->compare(0, 5, "refs/") == 0
+                        ? *ref
+                        : "refs/heads/" + *ref;
+                    runProgram("git", true, { "-C", repoDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", fetchRef, fetchRef) });
                 } catch (Error & e) {
                     if (!pathExists(localRefFile)) throw;
                     warn("could not update local clone of Git repository '%s'; continuing with the most recent version", actualUrl);
@@ -354,8 +369,8 @@ struct GitInput : Input
                 utimes(localRefFile.c_str(), times);
             }
 
-            if (!input->rev)
-                input->rev = Hash(chomp(readFile(localRefFile)), htSHA1);
+            if (!input.getRev())
+                input.attrs.insert_or_assign("rev", Hash(chomp(readFile(localRefFile)), htSHA1).gitRev());
         }
 
         bool isShallow = chomp(runProgram("git", true, { "-C", repoDir, "rev-parse", "--is-shallow-repository" })) == "true";
@@ -365,7 +380,7 @@ struct GitInput : Input
 
         // FIXME: check whether rev is an ancestor of ref.
 
-        printTalkative("using revision %s of repo '%s'", input->rev->gitRev(), actualUrl);
+        printTalkative("using revision %s of repo '%s'", input.getRev()->gitRev(), actualUrl);
 
         /* Now that we know the ref, check again whether we have it in
            the store. */
@@ -387,7 +402,7 @@ struct GitInput : Input
             runProgram("git", true, { "-C", tmpDir, "fetch", "--quiet", "--force",
                                       "--update-head-ok", "--", repoDir, "refs/*:refs/*" });
 
-            runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", input->rev->gitRev() });
+            runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", input.getRev()->gitRev() });
             runProgram("git", true, { "-C", tmpDir, "remote", "add", "origin", actualUrl });
             runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init", "--recursive" });
 
@@ -396,7 +411,7 @@ struct GitInput : Input
             // FIXME: should pipe this, or find some better way to extract a
             // revision.
             auto source = sinkToSource([&](Sink & sink) {
-                RunOptions gitOptions("git", { "-C", repoDir, "archive", input->rev->gitRev() });
+                RunOptions gitOptions("git", { "-C", repoDir, "archive", input.getRev()->gitRev() });
                 gitOptions.standardOut = &sink;
                 runProgram2(gitOptions);
             });
@@ -404,20 +419,20 @@ struct GitInput : Input
             unpackTarfile(*source, tmpDir);
         }
 
-        auto storePath = store->addToStore(name, tmpDir, true, htSHA256, filter);
+        auto storePath = store->addToStore(name, tmpDir, FileIngestionMethod::Recursive, htSHA256, filter);
 
-        auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "log", "-1", "--format=%ct", input->rev->gitRev() }));
+        auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "log", "-1", "--format=%ct", input.getRev()->gitRev() }));
 
         Attrs infoAttrs({
-            {"rev", input->rev->gitRev()},
+            {"rev", input.getRev()->gitRev()},
             {"lastModified", lastModified},
         });
 
         if (!shallow)
             infoAttrs.insert_or_assign("revCount",
-                std::stoull(runProgram("git", true, { "-C", repoDir, "rev-list", "--count", input->rev->gitRev() })));
+                std::stoull(runProgram("git", true, { "-C", repoDir, "rev-list", "--count", input.getRev()->gitRev() })));
 
-        if (!this->rev)
+        if (!_input.getRev())
             getCache()->add(
                 store,
                 mutableAttrs,
@@ -436,60 +451,6 @@ struct GitInput : Input
     }
 };
 
-struct GitInputScheme : InputScheme
-{
-    std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override
-    {
-        if (url.scheme != "git" &&
-            url.scheme != "git+http" &&
-            url.scheme != "git+https" &&
-            url.scheme != "git+ssh" &&
-            url.scheme != "git+file") return nullptr;
-
-        auto url2(url);
-        if (hasPrefix(url2.scheme, "git+")) url2.scheme = std::string(url2.scheme, 4);
-        url2.query.clear();
-
-        Attrs attrs;
-        attrs.emplace("type", "git");
-
-        for (auto &[name, value] : url.query) {
-            if (name == "rev" || name == "ref")
-                attrs.emplace(name, value);
-            else
-                url2.query.emplace(name, value);
-        }
-
-        attrs.emplace("url", url2.to_string());
-
-        return inputFromAttrs(attrs);
-    }
-
-    std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override
-    {
-        if (maybeGetStrAttr(attrs, "type") != "git") return {};
-
-        for (auto & [name, value] : attrs)
-            if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules")
-                throw Error("unsupported Git input attribute '%s'", name);
-
-        auto input = std::make_unique<GitInput>(parseURL(getStrAttr(attrs, "url")));
-        if (auto ref = maybeGetStrAttr(attrs, "ref")) {
-            if (!std::regex_match(*ref, refRegex))
-                throw BadURL("invalid Git branch/tag name '%s'", *ref);
-            input->ref = *ref;
-        }
-        if (auto rev = maybeGetStrAttr(attrs, "rev"))
-            input->rev = Hash(*rev, htSHA1);
-
-        input->shallow = maybeGetBoolAttr(attrs, "shallow").value_or(false);
-
-        input->submodules = maybeGetBoolAttr(attrs, "submodules").value_or(false);
-
-        return input;
-    }
-};
-
 static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<GitInputScheme>()); });
 
 }

src/libfetchers/github.cc

@@ -8,88 +8,142 @@
 
 namespace nix::fetchers {
 
-std::regex ownerRegex("[a-zA-Z][a-zA-Z0-9_-]*", std::regex::ECMAScript);
-std::regex repoRegex("[a-zA-Z][a-zA-Z0-9_-]*", std::regex::ECMAScript);
+// A github or gitlab url
+const static std::string urlRegexS = "[a-zA-Z0-9.]*"; // FIXME: check
+std::regex urlRegex(urlRegexS, std::regex::ECMAScript);
 
-struct GitHubInput : Input
+struct GitArchiveInputScheme : InputScheme
 {
-    std::string owner;
-    std::string repo;
-    std::optional<std::string> ref;
-    std::optional<Hash> rev;
+    virtual std::string type() = 0;
 
-    std::string type() const override { return "github"; }
-
-    bool operator ==(const Input & other) const override
+    std::optional<Input> inputFromURL(const ParsedURL & url) override
     {
-        auto other2 = dynamic_cast<const GitHubInput *>(&other);
-        return
-            other2
-            && owner == other2->owner
-            && repo == other2->repo
-            && rev == other2->rev
-            && ref == other2->ref;
+        if (url.scheme != type()) return {};
+
+        auto path = tokenizeString<std::vector<std::string>>(url.path, "/");
+
+        std::optional<Hash> rev;
+        std::optional<std::string> ref;
+        std::optional<std::string> host_url;
+
+        if (path.size() == 2) {
+        } else if (path.size() == 3) {
+            if (std::regex_match(path[2], revRegex))
+                rev = Hash(path[2], htSHA1);
+            else if (std::regex_match(path[2], refRegex))
+                ref = path[2];
+            else
+                throw BadURL("in URL '%s', '%s' is not a commit hash or branch/tag name", url.url, path[2]);
+        } else
+            throw BadURL("URL '%s' is invalid", url.url);
+
+        for (auto &[name, value] : url.query) {
+            if (name == "rev") {
+                if (rev)
+                    throw BadURL("URL '%s' contains multiple commit hashes", url.url);
+                rev = Hash(value, htSHA1);
+            }
+            else if (name == "ref") {
+                if (!std::regex_match(value, refRegex))
+                    throw BadURL("URL '%s' contains an invalid branch/tag name", url.url);
+                if (ref)
+                    throw BadURL("URL '%s' contains multiple branch/tag names", url.url);
+                ref = value;
+            }
+            else if (name == "url") {
+                if (!std::regex_match(value, urlRegex))
+                    throw BadURL("URL '%s' contains an invalid instance url", url.url);
+                host_url = value;
+            }
+            // FIXME: barf on unsupported attributes
+        }
+
+        if (ref && rev)
+            throw BadURL("URL '%s' contains both a commit hash and a branch/tag name %s %s", url.url, *ref, rev->gitRev());
+
+        Input input;
+        input.attrs.insert_or_assign("type", type());
+        input.attrs.insert_or_assign("owner", path[0]);
+        input.attrs.insert_or_assign("repo", path[1]);
+        if (rev) input.attrs.insert_or_assign("rev", rev->gitRev());
+        if (ref) input.attrs.insert_or_assign("ref", *ref);
+        if (host_url) input.attrs.insert_or_assign("url", *host_url);
+
+        return input;
     }
 
-    bool isImmutable() const override
+    std::optional<Input> inputFromAttrs(const Attrs & attrs) override
     {
-        return (bool) rev || narHash;
+        if (maybeGetStrAttr(attrs, "type") != type()) return {};
+
+        for (auto & [name, value] : attrs)
+            if (name != "type" && name != "owner" && name != "repo" && name != "ref" && name != "rev" && name != "narHash" && name != "lastModified")
+                throw Error("unsupported input attribute '%s'", name);
+
+        getStrAttr(attrs, "owner");
+        getStrAttr(attrs, "repo");
+
+        Input input;
+        input.attrs = attrs;
+        return input;
     }
 
-    std::optional<std::string> getRef() const override { return ref; }
-
-    std::optional<Hash> getRev() const override { return rev; }
-
-    ParsedURL toURL() const override
+    ParsedURL toURL(const Input & input) override
     {
+        auto owner = getStrAttr(input.attrs, "owner");
+        auto repo = getStrAttr(input.attrs, "repo");
+        auto ref = input.getRef();
+        auto rev = input.getRev();
         auto path = owner + "/" + repo;
         assert(!(ref && rev));
         if (ref) path += "/" + *ref;
         if (rev) path += "/" + rev->to_string(Base16, false);
         return ParsedURL {
-            .scheme = "github",
+            .scheme = type(),
             .path = path,
         };
     }
 
-    Attrs toAttrsInternal() const override
+    bool hasAllInfo(const Input & input) override
     {
-        Attrs attrs;
-        attrs.emplace("owner", owner);
-        attrs.emplace("repo", repo);
-        if (ref)
-            attrs.emplace("ref", *ref);
-        if (rev)
-            attrs.emplace("rev", rev->gitRev());
-        return attrs;
+        return input.getRev() && maybeGetIntAttr(input.attrs, "lastModified");
     }
 
-    void clone(const Path & destDir) const override
+    Input applyOverrides(
+        const Input & _input,
+        std::optional<std::string> ref,
+        std::optional<Hash> rev) override
     {
-        std::shared_ptr<const Input> input = inputFromURL(fmt("git+ssh://git@github.com/%s/%s.git", owner, repo));
-        input = input->applyOverrides(ref.value_or("master"), rev);
-        input->clone(destDir);
-    }
-
-    std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override
-    {
-        auto rev = this->rev;
-        auto ref = this->ref.value_or("master");
-
-        if (!rev) {
-            auto url = fmt("https://api.github.com/repos/%s/%s/commits/%s",
-                owner, repo, ref);
-            auto json = nlohmann::json::parse(
-                readFile(
-                    store->toRealPath(
-                        downloadFile(store, url, "source", false).storePath)));
-            rev = Hash(json["sha"], htSHA1);
-            debug("HEAD revision for '%s' is %s", url, rev->gitRev());
+        auto input(_input);
+        if (rev && ref)
+            throw BadURL("cannot apply both a commit hash (%s) and a branch/tag name ('%s') to input '%s'",
+                rev->gitRev(), *ref, input.to_string());
+        if (rev) {
+            input.attrs.insert_or_assign("rev", rev->gitRev());
+            input.attrs.erase("ref");
         }
+        if (ref) {
+            input.attrs.insert_or_assign("ref", *ref);
+            input.attrs.erase("rev");
+        }
+        return input;
+    }
 
-        auto input = std::make_shared<GitHubInput>(*this);
-        input->ref = {};
-        input->rev = *rev;
+    virtual Hash getRevFromRef(nix::ref<Store> store, const Input & input) const = 0;
+
+    virtual std::string getDownloadUrl(const Input & input) const = 0;
+
+    std::pair<Tree, Input> fetch(ref<Store> store, const Input & _input) override
+    {
+        Input input(_input);
+
+        if (!maybeGetStrAttr(input.attrs, "ref")) input.attrs.insert_or_assign("ref", "HEAD");
+
+        auto rev = input.getRev();
+        if (!rev) rev = getRevFromRef(store, input);
+
+        input.attrs.erase("ref");
+        input.attrs.insert_or_assign("rev", rev->gitRev());
 
         Attrs immutableAttrs({
             {"type", "git-tarball"},
@@ -97,120 +151,123 @@ struct GitHubInput : Input
         });
 
         if (auto res = getCache()->lookup(store, immutableAttrs)) {
+            input.attrs.insert_or_assign("lastModified", getIntAttr(res->first, "lastModified"));
             return {
-                Tree{
-                    .actualPath = store->toRealPath(res->second),
-                    .storePath = std::move(res->second),
-                    .info = TreeInfo {
-                        .lastModified = getIntAttr(res->first, "lastModified"),
-                    },
-                },
+                Tree(store->toRealPath(res->second), std::move(res->second)),
                 input
             };
         }
 
-        // FIXME: use regular /archive URLs instead? api.github.com
-        // might have stricter rate limits.
+        auto url = getDownloadUrl(input);
 
-        auto url = fmt("https://api.github.com/repos/%s/%s/tarball/%s",
-            owner, repo, rev->to_string(Base16, false));
+        auto [tree, lastModified] = downloadTarball(store, url, "source", true);
 
-        std::string accessToken = settings.githubAccessToken.get();
-        if (accessToken != "")
-            url += "?access_token=" + accessToken;
-
-        auto tree = downloadTarball(store, url, "source", true);
+        input.attrs.insert_or_assign("lastModified", lastModified);
 
         getCache()->add(
             store,
             immutableAttrs,
             {
                 {"rev", rev->gitRev()},
-                {"lastModified", *tree.info.lastModified}
+                {"lastModified", lastModified}
             },
             tree.storePath,
             true);
 
         return {std::move(tree), input};
     }
+};
 
-    std::shared_ptr<const Input> applyOverrides(
-        std::optional<std::string> ref,
-        std::optional<Hash> rev) const override
+struct GitHubInputScheme : GitArchiveInputScheme
+{
+    std::string type() override { return "github"; }
+
+    Hash getRevFromRef(nix::ref<Store> store, const Input & input) const override
     {
-        if (!ref && !rev) return shared_from_this();
+        auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("github.com");
+        auto url = fmt("https://api.%s/repos/%s/%s/commits/%s", // FIXME: check
+            host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef());
+        auto json = nlohmann::json::parse(
+            readFile(
+                store->toRealPath(
+                    downloadFile(store, url, "source", false).storePath)));
+        auto rev = Hash(std::string { json["sha"] }, htSHA1);
+        debug("HEAD revision for '%s' is %s", url, rev.gitRev());
+        return rev;
+    }
 
-        auto res = std::make_shared<GitHubInput>(*this);
+    std::string getDownloadUrl(const Input & input) const override
+    {
+        // FIXME: use regular /archive URLs instead? api.github.com
+        // might have stricter rate limits.
+        auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("github.com");
+        auto url = fmt("https://api.%s/repos/%s/%s/tarball/%s", // FIXME: check if this is correct for self hosted instances
+            host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"),
+            input.getRev()->to_string(Base16, false));
 
-        if (ref) res->ref = ref;
-        if (rev) res->rev = rev;
+        std::string accessToken = settings.githubAccessToken.get();
+        if (accessToken != "")
+            url += "?access_token=" + accessToken;
 
-        return res;
+        return url;
+    }
+
+    void clone(const Input & input, const Path & destDir) override
+    {
+        auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("github.com");
+        Input::fromURL(fmt("git+ssh://git@%s/%s/%s.git",
+                host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo")))
+            .applyOverrides(input.getRef().value_or("HEAD"), input.getRev())
+            .clone(destDir);
     }
 };
 
-struct GitHubInputScheme : InputScheme
+struct GitLabInputScheme : GitArchiveInputScheme
 {
-    std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override
+    std::string type() override { return "gitlab"; }
+
+    Hash getRevFromRef(nix::ref<Store> store, const Input & input) const override
     {
-        if (url.scheme != "github") return nullptr;
-
-        auto path = tokenizeString<std::vector<std::string>>(url.path, "/");
-        auto input = std::make_unique<GitHubInput>();
-
-        if (path.size() == 2) {
-        } else if (path.size() == 3) {
-            if (std::regex_match(path[2], revRegex))
-                input->rev = Hash(path[2], htSHA1);
-            else if (std::regex_match(path[2], refRegex))
-                input->ref = path[2];
-            else
-                throw BadURL("in GitHub URL '%s', '%s' is not a commit hash or branch/tag name", url.url, path[2]);
-        } else
-            throw BadURL("GitHub URL '%s' is invalid", url.url);
-
-        for (auto &[name, value] : url.query) {
-            if (name == "rev") {
-                if (input->rev)
-                    throw BadURL("GitHub URL '%s' contains multiple commit hashes", url.url);
-                input->rev = Hash(value, htSHA1);
-            }
-            else if (name == "ref") {
-                if (!std::regex_match(value, refRegex))
-                    throw BadURL("GitHub URL '%s' contains an invalid branch/tag name", url.url);
-                if (input->ref)
-                    throw BadURL("GitHub URL '%s' contains multiple branch/tag names", url.url);
-                input->ref = value;
-            }
-        }
-
-        if (input->ref && input->rev)
-            throw BadURL("GitHub URL '%s' contains both a commit hash and a branch/tag name", url.url);
-
-        input->owner = path[0];
-        input->repo = path[1];
-
-        return input;
+        auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("gitlab.com");
+        auto url = fmt("https://%s/api/v4/projects/%s%%2F%s/repository/commits?ref_name=%s",
+            host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef());
+        auto json = nlohmann::json::parse(
+            readFile(
+                store->toRealPath(
+                    downloadFile(store, url, "source", false).storePath)));
+        auto rev = Hash(std::string(json[0]["id"]), htSHA1);
+        debug("HEAD revision for '%s' is %s", url, rev.gitRev());
+        return rev;
     }
 
-    std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override
+    std::string getDownloadUrl(const Input & input) const override
     {
-        if (maybeGetStrAttr(attrs, "type") != "github") return {};
+        // FIXME: This endpoint has a rate limit threshold of 5 requests per minute
+        auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("gitlab.com");
+        auto url = fmt("https://%s/api/v4/projects/%s%%2F%s/repository/archive.tar.gz?sha=%s",
+            host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"),
+            input.getRev()->to_string(Base16, false));
 
-        for (auto & [name, value] : attrs)
-            if (name != "type" && name != "owner" && name != "repo" && name != "ref" && name != "rev")
-                throw Error("unsupported GitHub input attribute '%s'", name);
+        /* # FIXME: add privat token auth (`curl --header "PRIVATE-TOKEN: <your_access_token>"`)
+        std::string accessToken = settings.githubAccessToken.get();
+        if (accessToken != "")
+            url += "?access_token=" + accessToken;*/
 
-        auto input = std::make_unique<GitHubInput>();
-        input->owner = getStrAttr(attrs, "owner");
-        input->repo = getStrAttr(attrs, "repo");
-        input->ref = maybeGetStrAttr(attrs, "ref");
-        if (auto rev = maybeGetStrAttr(attrs, "rev"))
-            input->rev = Hash(*rev, htSHA1);
-        return input;
+        return url;
+    }
+
+    void clone(const Input & input, const Path & destDir) override
+    {
+        auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("gitlab.com");
+        // FIXME: get username somewhere
+        Input::fromURL(fmt("git+ssh://git@%s/%s/%s.git",
+                host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo")))
+            .applyOverrides(input.getRef().value_or("HEAD"), input.getRev())
+            .clone(destDir);
     }
 };
 
 static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<GitHubInputScheme>()); });
+static auto r2 = OnStartup([] { registerInputScheme(std::make_unique<GitLabInputScheme>()); });
 
 }

src/libfetchers/indirect.cc

@@ -4,135 +4,99 @@ namespace nix::fetchers {
 
 std::regex flakeRegex("[a-zA-Z][a-zA-Z0-9_-]*", std::regex::ECMAScript);
 
-struct IndirectInput : Input
-{
-    std::string id;
-    std::optional<Hash> rev;
-    std::optional<std::string> ref;
-
-    std::string type() const override { return "indirect"; }
-
-    bool operator ==(const Input & other) const override
-    {
-        auto other2 = dynamic_cast<const IndirectInput *>(&other);
-        return
-            other2
-            && id == other2->id
-            && rev == other2->rev
-            && ref == other2->ref;
-    }
-
-    bool isDirect() const override
-    {
-        return false;
-    }
-
-    std::optional<std::string> getRef() const override { return ref; }
-
-    std::optional<Hash> getRev() const override { return rev; }
-
-    bool contains(const Input & other) const override
-    {
-        auto other2 = dynamic_cast<const IndirectInput *>(&other);
-        return
-            other2
-            && id == other2->id
-            && (!ref || ref == other2->ref)
-            && (!rev || rev == other2->rev);
-    }
-
-    ParsedURL toURL() const override
-    {
-        ParsedURL url;
-        url.scheme = "flake";
-        url.path = id;
-        if (ref) { url.path += '/'; url.path += *ref; };
-        if (rev) { url.path += '/'; url.path += rev->gitRev(); };
-        return url;
-    }
-
-    Attrs toAttrsInternal() const override
-    {
-        Attrs attrs;
-        attrs.emplace("id", id);
-        if (ref)
-            attrs.emplace("ref", *ref);
-        if (rev)
-            attrs.emplace("rev", rev->gitRev());
-        return attrs;
-    }
-
-    std::shared_ptr<const Input> applyOverrides(
-        std::optional<std::string> ref,
-        std::optional<Hash> rev) const override
-    {
-        if (!ref && !rev) return shared_from_this();
-
-        auto res = std::make_shared<IndirectInput>(*this);
-
-        if (ref) res->ref = ref;
-        if (rev) res->rev = rev;
-
-        return res;
-    }
-
-    std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override
-    {
-        throw Error("indirect input '%s' cannot be fetched directly", to_string());
-    }
-};
-
 struct IndirectInputScheme : InputScheme
 {
-    std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override
+    std::optional<Input> inputFromURL(const ParsedURL & url) override
     {
-        if (url.scheme != "flake") return nullptr;
+        if (url.scheme != "flake") return {};
 
         auto path = tokenizeString<std::vector<std::string>>(url.path, "/");
-        auto input = std::make_unique<IndirectInput>();
+
+        std::optional<Hash> rev;
+        std::optional<std::string> ref;
 
         if (path.size() == 1) {
         } else if (path.size() == 2) {
             if (std::regex_match(path[1], revRegex))
-                input->rev = Hash(path[1], htSHA1);
+                rev = Hash(path[1], htSHA1);
             else if (std::regex_match(path[1], refRegex))
-                input->ref = path[1];
+                ref = path[1];
             else
                 throw BadURL("in flake URL '%s', '%s' is not a commit hash or branch/tag name", url.url, path[1]);
         } else if (path.size() == 3) {
             if (!std::regex_match(path[1], refRegex))
                 throw BadURL("in flake URL '%s', '%s' is not a branch/tag name", url.url, path[1]);
-            input->ref = path[1];
+            ref = path[1];
             if (!std::regex_match(path[2], revRegex))
                 throw BadURL("in flake URL '%s', '%s' is not a commit hash", url.url, path[2]);
-            input->rev = Hash(path[2], htSHA1);
+            rev = Hash(path[2], htSHA1);
         } else
             throw BadURL("GitHub URL '%s' is invalid", url.url);
 
+        std::string id = path[0];
+        if (!std::regex_match(id, flakeRegex))
+            throw BadURL("'%s' is not a valid flake ID", id);
+
         // FIXME: forbid query params?
 
-        input->id = path[0];
-        if (!std::regex_match(input->id, flakeRegex))
-            throw BadURL("'%s' is not a valid flake ID", input->id);
+        Input input;
+        input.direct = false;
+        input.attrs.insert_or_assign("type", "indirect");
+        input.attrs.insert_or_assign("id", id);
+        if (rev) input.attrs.insert_or_assign("rev", rev->gitRev());
+        if (ref) input.attrs.insert_or_assign("ref", *ref);
 
         return input;
     }
 
-    std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override
+    std::optional<Input> inputFromAttrs(const Attrs & attrs) override
     {
         if (maybeGetStrAttr(attrs, "type") != "indirect") return {};
 
         for (auto & [name, value] : attrs)
-            if (name != "type" && name != "id" && name != "ref" && name != "rev")
+            if (name != "type" && name != "id" && name != "ref" && name != "rev" && name != "narHash")
                 throw Error("unsupported indirect input attribute '%s'", name);
 
-        auto input = std::make_unique<IndirectInput>();
-        input->id = getStrAttr(attrs, "id");
-        input->ref = maybeGetStrAttr(attrs, "ref");
-        if (auto rev = maybeGetStrAttr(attrs, "rev"))
-            input->rev = Hash(*rev, htSHA1);
+        auto id = getStrAttr(attrs, "id");
+        if (!std::regex_match(id, flakeRegex))
+            throw BadURL("'%s' is not a valid flake ID", id);
+
+        Input input;
+        input.direct = false;
+        input.attrs = attrs;
         return input;
     }
+
+    ParsedURL toURL(const Input & input) override
+    {
+        ParsedURL url;
+        url.scheme = "flake";
+        url.path = getStrAttr(input.attrs, "id");
+        if (auto ref = input.getRef()) { url.path += '/'; url.path += *ref; };
+        if (auto rev = input.getRev()) { url.path += '/'; url.path += rev->gitRev(); };
+        return url;
+    }
+
+    bool hasAllInfo(const Input & input) override
+    {
+        return false;
+    }
+
+    Input applyOverrides(
+        const Input & _input,
+        std::optional<std::string> ref,
+        std::optional<Hash> rev) override
+    {
+        auto input(_input);
+        if (rev) input.attrs.insert_or_assign("rev", rev->gitRev());
+        if (ref) input.attrs.insert_or_assign("ref", *ref);
+        return input;
+    }
+
+    std::pair<Tree, Input> fetch(ref<Store> store, const Input & input) override
+    {
+        throw Error("indirect input '%s' cannot be fetched directly", input.to_string());
+    }
 };
 
 static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<IndirectInputScheme>()); });

src/libfetchers/local.mk

@@ -8,4 +8,4 @@ libfetchers_SOURCES := $(wildcard $(d)/*.cc)
 
 libfetchers_CXXFLAGS += -I src/libutil -I src/libstore
 
-libfetchers_LIBS = libutil libstore libnixrust
+libfetchers_LIBS = libutil libstore

src/libfetchers/mercurial.cc

@@ -10,80 +10,92 @@ using namespace std::string_literals;
 
 namespace nix::fetchers {
 
-struct MercurialInput : Input
+struct MercurialInputScheme : InputScheme
 {
-    ParsedURL url;
-    std::optional<std::string> ref;
-    std::optional<Hash> rev;
-
-    MercurialInput(const ParsedURL & url) : url(url)
-    { }
-
-    std::string type() const override { return "hg"; }
-
-    bool operator ==(const Input & other) const override
+    std::optional<Input> inputFromURL(const ParsedURL & url) override
     {
-        auto other2 = dynamic_cast<const MercurialInput *>(&other);
-        return
-            other2
-            && url == other2->url
-            && rev == other2->rev
-            && ref == other2->ref;
+        if (url.scheme != "hg+http" &&
+            url.scheme != "hg+https" &&
+            url.scheme != "hg+ssh" &&
+            url.scheme != "hg+file") return {};
+
+        auto url2(url);
+        url2.scheme = std::string(url2.scheme, 3);
+        url2.query.clear();
+
+        Attrs attrs;
+        attrs.emplace("type", "hg");
+
+        for (auto &[name, value] : url.query) {
+            if (name == "rev" || name == "ref")
+                attrs.emplace(name, value);
+            else
+                url2.query.emplace(name, value);
+        }
+
+        attrs.emplace("url", url2.to_string());
+
+        return inputFromAttrs(attrs);
     }
 
-    bool isImmutable() const override
+    std::optional<Input> inputFromAttrs(const Attrs & attrs) override
     {
-        return (bool) rev || narHash;
+        if (maybeGetStrAttr(attrs, "type") != "hg") return {};
+
+        for (auto & [name, value] : attrs)
+            if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "revCount" && name != "narHash")
+                throw Error("unsupported Mercurial input attribute '%s'", name);
+
+        parseURL(getStrAttr(attrs, "url"));
+
+        if (auto ref = maybeGetStrAttr(attrs, "ref")) {
+            if (!std::regex_match(*ref, refRegex))
+                throw BadURL("invalid Mercurial branch/tag name '%s'", *ref);
+        }
+
+        Input input;
+        input.attrs = attrs;
+        return input;
     }
 
-    std::optional<std::string> getRef() const override { return ref; }
-
-    std::optional<Hash> getRev() const override { return rev; }
-
-    ParsedURL toURL() const override
+    ParsedURL toURL(const Input & input) override
     {
-        ParsedURL url2(url);
-        url2.scheme = "hg+" + url2.scheme;
-        if (rev) url2.query.insert_or_assign("rev", rev->gitRev());
-        if (ref) url2.query.insert_or_assign("ref", *ref);
+        auto url = parseURL(getStrAttr(input.attrs, "url"));
+        url.scheme = "hg+" + url.scheme;
+        if (auto rev = input.getRev()) url.query.insert_or_assign("rev", rev->gitRev());
+        if (auto ref = input.getRef()) url.query.insert_or_assign("ref", *ref);
         return url;
     }
 
-    Attrs toAttrsInternal() const override
+    bool hasAllInfo(const Input & input) override
     {
-        Attrs attrs;
-        attrs.emplace("url", url.to_string());
-        if (ref)
-            attrs.emplace("ref", *ref);
-        if (rev)
-            attrs.emplace("rev", rev->gitRev());
-        return attrs;
+        // FIXME: ugly, need to distinguish between dirty and clean
+        // default trees.
+        return input.getRef() == "default" || maybeGetIntAttr(input.attrs, "revCount");
     }
 
-    std::shared_ptr<const Input> applyOverrides(
+    Input applyOverrides(
+        const Input & input,
         std::optional<std::string> ref,
-        std::optional<Hash> rev) const override
+        std::optional<Hash> rev) override
     {
-        if (!ref && !rev) return shared_from_this();
-
-        auto res = std::make_shared<MercurialInput>(*this);
-
-        if (ref) res->ref = ref;
-        if (rev) res->rev = rev;
-
+        auto res(input);
+        if (rev) res.attrs.insert_or_assign("rev", rev->gitRev());
+        if (ref) res.attrs.insert_or_assign("ref", *ref);
         return res;
     }
 
-    std::optional<Path> getSourcePath() const
+    std::optional<Path> getSourcePath(const Input & input) override
     {
-        if (url.scheme == "file" && !ref && !rev)
+        auto url = parseURL(getStrAttr(input.attrs, "url"));
+        if (url.scheme == "file" && !input.getRef() && !input.getRev())
             return url.path;
         return {};
     }
 
-    void markChangedFile(std::string_view file, std::optional<std::string> commitMsg) const override
+    void markChangedFile(const Input & input, std::string_view file, std::optional<std::string> commitMsg) override
     {
-        auto sourcePath = getSourcePath();
+        auto sourcePath = getSourcePath(input);
         assert(sourcePath);
 
         // FIXME: shut up if file is already tracked.
@@ -95,26 +107,27 @@ struct MercurialInput : Input
                 { "commit", *sourcePath + "/" + std::string(file), "-m", *commitMsg });
     }
 
-    std::pair<bool, std::string> getActualUrl() const
+    std::pair<bool, std::string> getActualUrl(const Input & input) const
     {
+        auto url = parseURL(getStrAttr(input.attrs, "url"));
         bool isLocal = url.scheme == "file";
         return {isLocal, isLocal ? url.path : url.base};
     }
 
-    std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override
+    std::pair<Tree, Input> fetch(ref<Store> store, const Input & _input) override
     {
         auto name = "source";
 
-        auto input = std::make_shared<MercurialInput>(*this);
+        Input input(_input);
 
-        auto [isLocal, actualUrl_] = getActualUrl();
+        auto [isLocal, actualUrl_] = getActualUrl(input);
         auto actualUrl = actualUrl_; // work around clang bug
 
         // FIXME: return lastModified.
 
         // FIXME: don't clone local repositories.
 
-        if (!input->ref && !input->rev && isLocal && pathExists(actualUrl + "/.hg")) {
+        if (!input.getRef() && !input.getRev() && isLocal && pathExists(actualUrl + "/.hg")) {
 
             bool clean = runProgram("hg", true, { "status", "-R", actualUrl, "--modified", "--added", "--removed" }) == "";
 
@@ -129,7 +142,7 @@ struct MercurialInput : Input
                 if (settings.warnDirty)
                     warn("Mercurial tree '%s' is unclean", actualUrl);
 
-                input->ref = chomp(runProgram("hg", true, { "branch", "-R", actualUrl }));
+                input.attrs.insert_or_assign("ref", chomp(runProgram("hg", true, { "branch", "-R", actualUrl })));
 
                 auto files = tokenizeString<std::set<std::string>>(
                     runProgram("hg", true, { "status", "-R", actualUrl, "--clean", "--modified", "--added", "--no-status", "--print0" }), "\0"s);
@@ -149,62 +162,56 @@ struct MercurialInput : Input
                     return files.count(file);
                 };
 
-                auto storePath = store->addToStore("source", actualUrl, true, htSHA256, filter);
+                auto storePath = store->addToStore("source", actualUrl, FileIngestionMethod::Recursive, htSHA256, filter);
 
-                return {Tree {
-                    .actualPath = store->printStorePath(storePath),
-                    .storePath = std::move(storePath),
-                }, input};
+                return {
+                    Tree(store->printStorePath(storePath), std::move(storePath)),
+                    input
+                };
             }
         }
 
-        if (!input->ref) input->ref = "default";
+        if (!input.getRef()) input.attrs.insert_or_assign("ref", "default");
 
         auto getImmutableAttrs = [&]()
         {
             return Attrs({
                 {"type", "hg"},
                 {"name", name},
-                {"rev", input->rev->gitRev()},
+                {"rev", input.getRev()->gitRev()},
             });
         };
 
         auto makeResult = [&](const Attrs & infoAttrs, StorePath && storePath)
-            -> std::pair<Tree, std::shared_ptr<const Input>>
+            -> std::pair<Tree, Input>
         {
-            assert(input->rev);
-            assert(!rev || rev == input->rev);
+            assert(input.getRev());
+            assert(!_input.getRev() || _input.getRev() == input.getRev());
+            input.attrs.insert_or_assign("revCount", getIntAttr(infoAttrs, "revCount"));
             return {
-                Tree{
-                    .actualPath = store->toRealPath(storePath),
-                    .storePath = std::move(storePath),
-                    .info = TreeInfo {
-                        .revCount = getIntAttr(infoAttrs, "revCount"),
-                    },
-                },
+                Tree(store->toRealPath(storePath), std::move(storePath)),
                 input
             };
         };
 
-        if (input->rev) {
+        if (input.getRev()) {
             if (auto res = getCache()->lookup(store, getImmutableAttrs()))
                 return makeResult(res->first, std::move(res->second));
         }
 
-        assert(input->rev || input->ref);
-        auto revOrRef = input->rev ? input->rev->gitRev() : *input->ref;
+        auto revOrRef = input.getRev() ? input.getRev()->gitRev() : *input.getRef();
 
         Attrs mutableAttrs({
             {"type", "hg"},
             {"name", name},
             {"url", actualUrl},
-            {"ref", *input->ref},
+            {"ref", *input.getRef()},
         });
 
         if (auto res = getCache()->lookup(store, mutableAttrs)) {
             auto rev2 = Hash(getStrAttr(res->first, "rev"), htSHA1);
-            if (!rev || rev == rev2) {
-                input->rev = rev2;
+            if (!input.getRev() || input.getRev() == rev2) {
+                input.attrs.insert_or_assign("rev", rev2.gitRev());
                 return makeResult(res->first, std::move(res->second));
             }
         }
@@ -213,10 +220,10 @@ struct MercurialInput : Input
 
         /* If this is a commit hash that we already have, we don't
            have to pull again. */
-        if (!(input->rev
+        if (!(input.getRev()
                 && pathExists(cacheDir)
                 && runProgram(
-                    RunOptions("hg", { "log", "-R", cacheDir, "-r", input->rev->gitRev(), "--template", "1" })
+                    RunOptions("hg", { "log", "-R", cacheDir, "-r", input.getRev()->gitRev(), "--template", "1" })
                     .killStderr(true)).second == "1"))
         {
             Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Mercurial repository '%s'", actualUrl));
@@ -245,9 +252,9 @@ struct MercurialInput : Input
             runProgram("hg", true, { "log", "-R", cacheDir, "-r", revOrRef, "--template", "{node} {rev} {branch}" }));
         assert(tokens.size() == 3);
 
-        input->rev = Hash(tokens[0], htSHA1);
+        input.attrs.insert_or_assign("rev", Hash(tokens[0], htSHA1).gitRev());
         auto revCount = std::stoull(tokens[1]);
-        input->ref = tokens[2];
+        input.attrs.insert_or_assign("ref", tokens[2]);
 
         if (auto res = getCache()->lookup(store, getImmutableAttrs()))
             return makeResult(res->first, std::move(res->second));
@@ -255,18 +262,18 @@ struct MercurialInput : Input
         Path tmpDir = createTempDir();
         AutoDelete delTmpDir(tmpDir, true);
 
-        runProgram("hg", true, { "archive", "-R", cacheDir, "-r", input->rev->gitRev(), tmpDir });
+        runProgram("hg", true, { "archive", "-R", cacheDir, "-r", input.getRev()->gitRev(), tmpDir });
 
         deletePath(tmpDir + "/.hg_archival.txt");
 
         auto storePath = store->addToStore(name, tmpDir);
 
         Attrs infoAttrs({
-            {"rev", input->rev->gitRev()},
+            {"rev", input.getRev()->gitRev()},
             {"revCount", (int64_t) revCount},
         });
 
-        if (!this->rev)
+        if (!_input.getRev())
             getCache()->add(
                 store,
                 mutableAttrs,
@@ -285,54 +292,6 @@ struct MercurialInput : Input
     }
 };
 
-struct MercurialInputScheme : InputScheme
-{
-    std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override
-    {
-        if (url.scheme != "hg+http" &&
-            url.scheme != "hg+https" &&
-            url.scheme != "hg+ssh" &&
-            url.scheme != "hg+file") return nullptr;
-
-        auto url2(url);
-        url2.scheme = std::string(url2.scheme, 3);
-        url2.query.clear();
-
-        Attrs attrs;
-        attrs.emplace("type", "hg");
-
-        for (auto &[name, value] : url.query) {
-            if (name == "rev" || name == "ref")
-                attrs.emplace(name, value);
-            else
-                url2.query.emplace(name, value);
-        }
-
-        attrs.emplace("url", url2.to_string());
-
-        return inputFromAttrs(attrs);
-    }
-
-    std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override
-    {
-        if (maybeGetStrAttr(attrs, "type") != "hg") return {};
-
-        for (auto & [name, value] : attrs)
-            if (name != "type" && name != "url" && name != "ref" && name != "rev")
-                throw Error("unsupported Mercurial input attribute '%s'", name);
-
-        auto input = std::make_unique<MercurialInput>(parseURL(getStrAttr(attrs, "url")));
-        if (auto ref = maybeGetStrAttr(attrs, "ref")) {
-            if (!std::regex_match(*ref, refRegex))
-                throw BadURL("invalid Mercurial branch/tag name '%s'", *ref);
-            input->ref = *ref;
-        }
-        if (auto rev = maybeGetStrAttr(attrs, "rev"))
-            input->rev = Hash(*rev, htSHA1);
-        return input;
-    }
-};
-
 static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<MercurialInputScheme>()); });
 
 }

src/libfetchers/path.cc

@@ -3,76 +3,86 @@
 
 namespace nix::fetchers {
 
-struct PathInput : Input
+struct PathInputScheme : InputScheme
 {
-    Path path;
-
-    /* Allow the user to pass in "fake" tree info attributes. This is
-       useful for making a pinned tree work the same as the repository
-       from which is exported
-       (e.g. path:/nix/store/...-source?lastModified=1585388205&rev=b0c285...). */
-    std::optional<Hash> rev;
-    std::optional<uint64_t> revCount;
-    std::optional<time_t> lastModified;
-
-    std::string type() const override { return "path"; }
-
-    std::optional<Hash> getRev() const override { return rev; }
-
-    bool operator ==(const Input & other) const override
+    std::optional<Input> inputFromURL(const ParsedURL & url) override
     {
-        auto other2 = dynamic_cast<const PathInput *>(&other);
-        return
-            other2
-            && path == other2->path
-            && rev == other2->rev
-            && revCount == other2->revCount
-            && lastModified == other2->lastModified;
+        if (url.scheme != "path") return {};
+
+        if (url.authority && *url.authority != "")
+            throw Error("path URL '%s' should not have an authority ('%s')", url.url, *url.authority);
+
+        Input input;
+        input.attrs.insert_or_assign("type", "path");
+        input.attrs.insert_or_assign("path", url.path);
+
+        for (auto & [name, value] : url.query)
+            if (name == "rev" || name == "narHash")
+                input.attrs.insert_or_assign(name, value);
+            else if (name == "revCount" || name == "lastModified") {
+                uint64_t n;
+                if (!string2Int(value, n))
+                    throw Error("path URL '%s' has invalid parameter '%s'", url.to_string(), name);
+                input.attrs.insert_or_assign(name, n);
+            }
+            else
+                throw Error("path URL '%s' has unsupported parameter '%s'", url.to_string(), name);
+
+        return input;
     }
 
-    bool isImmutable() const override
+    std::optional<Input> inputFromAttrs(const Attrs & attrs) override
     {
-        return narHash || rev;
+        if (maybeGetStrAttr(attrs, "type") != "path") return {};
+
+        getStrAttr(attrs, "path");
+
+        for (auto & [name, value] : attrs)
+            /* Allow the user to pass in "fake" tree info
+               attributes. This is useful for making a pinned tree
+               work the same as the repository from which is exported
+               (e.g. path:/nix/store/...-source?lastModified=1585388205&rev=b0c285...). */
+            if (name == "type" || name == "rev" || name == "revCount" || name == "lastModified" || name == "narHash" || name == "path")
+                // checked in Input::fromAttrs
+                ;
+            else
+                throw Error("unsupported path input attribute '%s'", name);
+
+        Input input;
+        input.attrs = attrs;
+        return input;
     }
 
-    ParsedURL toURL() const override
+    ParsedURL toURL(const Input & input) override
     {
-        auto query = attrsToQuery(toAttrsInternal());
+        auto query = attrsToQuery(input.attrs);
         query.erase("path");
+        query.erase("type");
         return ParsedURL {
             .scheme = "path",
-            .path = path,
+            .path = getStrAttr(input.attrs, "path"),
             .query = query,
         };
     }
 
-    Attrs toAttrsInternal() const override
+    bool hasAllInfo(const Input & input) override
     {
-        Attrs attrs;
-        attrs.emplace("path", path);
-        if (rev)
-            attrs.emplace("rev", rev->gitRev());
-        if (revCount)
-            attrs.emplace("revCount", *revCount);
-        if (lastModified)
-            attrs.emplace("lastModified", *lastModified);
-        if (!rev && narHash)
-            attrs.emplace("narHash", narHash->to_string(SRI));
-        return attrs;
+        return true;
     }
 
-    std::optional<Path> getSourcePath() const override
+    std::optional<Path> getSourcePath(const Input & input) override
     {
-        return path;
+        return getStrAttr(input.attrs, "path");
     }
 
-    void markChangedFile(std::string_view file, std::optional<std::string> commitMsg) const override
+    void markChangedFile(const Input & input, std::string_view file, std::optional<std::string> commitMsg) override
     {
+        // nothing to do
     }
 
-    std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override
+    std::pair<Tree, Input> fetch(ref<Store> store, const Input & input) override
     {
-        auto input = std::make_shared<PathInput>(*this);
+        auto path = getStrAttr(input.attrs, "path");
 
         // FIXME: check whether access to 'path' is allowed.
 
@@ -85,83 +95,10 @@ struct PathInput : Input
             // FIXME: try to substitute storePath.
             storePath = store->addToStore("source", path);
 
-        input->narHash = store->queryPathInfo(*storePath)->narHash;
-
-        return
-            {
-                Tree {
-                    .actualPath = store->toRealPath(*storePath),
-                    .storePath = std::move(*storePath),
-                    .info = TreeInfo {
-                        .revCount = revCount,
-                        .lastModified = lastModified
-                    }
-                },
-                input
-            };
-    }
-
-};
-
-struct PathInputScheme : InputScheme
-{
-    std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override
-    {
-        if (url.scheme != "path") return nullptr;
-
-        auto input = std::make_unique<PathInput>();
-        input->path = url.path;
-
-        if (url.authority && *url.authority != "")
-            throw Error("path URL '%s' should not have an authority ('%s')", url.url, *url.authority);
-
-        for (auto & [name, value] : url.query)
-            if (name == "rev")
-                input->rev = Hash(value, htSHA1);
-            else if (name == "revCount") {
-                uint64_t revCount;
-                if (!string2Int(value, revCount))
-                    throw Error("path URL '%s' has invalid parameter '%s'", url.to_string(), name);
-                input->revCount = revCount;
-            }
-            else if (name == "lastModified") {
-                time_t lastModified;
-                if (!string2Int(value, lastModified))
-                    throw Error("path URL '%s' has invalid parameter '%s'", url.to_string(), name);
-                input->lastModified = lastModified;
-            }
-            else if (name == "narHash")
-                // FIXME: require SRI hash.
-                input->narHash = Hash(value);
-            else
-                throw Error("path URL '%s' has unsupported parameter '%s'", url.to_string(), name);
-
-        return input;
-    }
-
-    std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override
-    {
-        if (maybeGetStrAttr(attrs, "type") != "path") return {};
-
-        auto input = std::make_unique<PathInput>();
-        input->path = getStrAttr(attrs, "path");
-
-        for (auto & [name, value] : attrs)
-            if (name == "rev")
-                input->rev = Hash(getStrAttr(attrs, "rev"), htSHA1);
-            else if (name == "revCount")
-                input->revCount = getIntAttr(attrs, "revCount");
-            else if (name == "lastModified")
-                input->lastModified = getIntAttr(attrs, "lastModified");
-            else if (name == "narHash")
-                // FIXME: require SRI hash.
-                input->narHash = Hash(getStrAttr(attrs, "narHash"));
-            else if (name == "type" || name == "path")
-                ;
-            else
-                throw Error("unsupported path input attribute '%s'", name);
-
-        return input;
+        return {
+            Tree(store->toRealPath(*storePath), std::move(*storePath)),
+            input
+        };
     }
 };
 

src/libfetchers/registry.cc

@@ -22,20 +22,7 @@ std::shared_ptr<Registry> Registry::read(
 
         auto version = json.value("version", 0);
 
-        // FIXME: remove soon
-        if (version == 1) {
-            auto flakes = json["flakes"];
-            for (auto i = flakes.begin(); i != flakes.end(); ++i) {
-                auto url = i->value("url", i->value("uri", ""));
-                if (url.empty())
-                    throw Error("flake registry '%s' lacks a 'url' attribute for entry '%s'",
-                        path, i.key());
-                registry->entries.push_back(
-                    {inputFromURL(i.key()), inputFromURL(url), {}});
-            }
-        }
-
-        else if (version == 2) {
+        if (version == 2) {
             for (auto & i : json["flakes"]) {
                 auto toAttrs = jsonToAttrs(i["to"]);
                 Attrs extraAttrs;
@@ -47,8 +34,8 @@ std::shared_ptr<Registry> Registry::read(
                 auto exact = i.find("exact");
                 registry->entries.push_back(
                     Entry {
-                        .from = inputFromAttrs(jsonToAttrs(i["from"])),
-                        .to = inputFromAttrs(toAttrs),
+                        .from = Input::fromAttrs(jsonToAttrs(i["from"])),
+                        .to = Input::fromAttrs(std::move(toAttrs)),
                         .extraAttrs = extraAttrs,
                         .exact = exact != i.end() && exact.value()
                     });
@@ -72,8 +59,8 @@ void Registry::write(const Path & path)
     nlohmann::json arr;
     for (auto & entry : entries) {
         nlohmann::json obj;
-        obj["from"] = attrsToJson(entry.from->toAttrs());
-        obj["to"] = attrsToJson(entry.to->toAttrs());
+        obj["from"] = attrsToJson(entry.from.toAttrs());
+        obj["to"] = attrsToJson(entry.to.toAttrs());
         if (!entry.extraAttrs.empty())
             obj["to"].update(attrsToJson(entry.extraAttrs));
         if (entry.exact)
@@ -90,8 +77,8 @@ void Registry::write(const Path & path)
 }
 
 void Registry::add(
-    const std::shared_ptr<const Input> & from,
-    const std::shared_ptr<const Input> & to,
+    const Input & from,
+    const Input & to,
     const Attrs & extraAttrs)
 {
     entries.emplace_back(
@@ -102,11 +89,11 @@ void Registry::add(
         });
 }
 
-void Registry::remove(const std::shared_ptr<const Input> & input)
+void Registry::remove(const Input & input)
 {
     // FIXME: use C++20 std::erase.
     for (auto i = entries.begin(); i != entries.end(); )
-        if (*i->from == *input)
+        if (i->from == input)
             i = entries.erase(i);
         else
             ++i;
@@ -145,8 +132,8 @@ std::shared_ptr<Registry> getFlagRegistry()
 }
 
 void overrideRegistry(
-    const std::shared_ptr<const Input> & from,
-    const std::shared_ptr<const Input> & to,
+    const Input & from,
+    const Input & to,
     const Attrs & extraAttrs)
 {
     flagRegistry->add(from, to, extraAttrs);
@@ -180,32 +167,33 @@ Registries getRegistries(ref<Store> store)
     return registries;
 }
 
-std::pair<std::shared_ptr<const Input>, Attrs> lookupInRegistries(
+std::pair<Input, Attrs> lookupInRegistries(
     ref<Store> store,
-    std::shared_ptr<const Input> input)
+    const Input & _input)
 {
     Attrs extraAttrs;
     int n = 0;
+    Input input(_input);
 
  restart:
 
     n++;
-    if (n > 100) throw Error("cycle detected in flake registr for '%s'", input);
+    if (n > 100) throw Error("cycle detected in flake registry for '%s'", input.to_string());
 
     for (auto & registry : getRegistries(store)) {
         // FIXME: O(n)
         for (auto & entry : registry->entries) {
             if (entry.exact) {
-                if (*entry.from == *input) {
+                if (entry.from == input) {
                     input = entry.to;
                     extraAttrs = entry.extraAttrs;
                     goto restart;
                 }
             } else {
-                if (entry.from->contains(*input)) {
-                    input = entry.to->applyOverrides(
-                        !entry.from->getRef() && input->getRef() ? input->getRef() : std::optional<std::string>(),
-                        !entry.from->getRev() && input->getRev() ? input->getRev() : std::optional<Hash>());
+                if (entry.from.contains(input)) {
+                    input = entry.to.applyOverrides(
+                        !entry.from.getRef() && input.getRef() ? input.getRef() : std::optional<std::string>(),
+                        !entry.from.getRev() && input.getRev() ? input.getRev() : std::optional<Hash>());
                     extraAttrs = entry.extraAttrs;
                     goto restart;
                 }
@@ -213,8 +201,10 @@ std::pair<std::shared_ptr<const Input>, Attrs> lookupInRegistries(
         }
     }
 
-    if (!input->isDirect())
-        throw Error("cannot find flake '%s' in the flake registries", input->to_string());
+    if (!input.isDirect())
+        throw Error("cannot find flake '%s' in the flake registries", input.to_string());
+
+    debug("looked up '%s' -> '%s'", _input.to_string(), input.to_string());
 
     return {input, extraAttrs};
 }

src/libfetchers/registry.hh

@@ -20,8 +20,7 @@ struct Registry
 
     struct Entry
     {
-        std::shared_ptr<const Input> from;
-        std::shared_ptr<const Input> to;
+        Input from, to;
         Attrs extraAttrs;
         bool exact = false;
     };
@@ -38,11 +37,11 @@ struct Registry
     void write(const Path & path);
 
     void add(
-        const std::shared_ptr<const Input> & from,
-        const std::shared_ptr<const Input> & to,
+        const Input & from,
+        const Input & to,
         const Attrs & extraAttrs);
 
-    void remove(const std::shared_ptr<const Input> & input);
+    void remove(const Input & input);
 };
 
 typedef std::vector<std::shared_ptr<Registry>> Registries;
@@ -54,12 +53,12 @@ Path getUserRegistryPath();
 Registries getRegistries(ref<Store> store);
 
 void overrideRegistry(
-    const std::shared_ptr<const Input> & from,
-    const std::shared_ptr<const Input> & to,
+    const Input & from,
+    const Input & to,
     const Attrs & extraAttrs);
 
-std::pair<std::shared_ptr<const Input>, Attrs> lookupInRegistries(
+std::pair<Input, Attrs> lookupInRegistries(
     ref<Store> store,
-    std::shared_ptr<const Input> input);
+    const Input & input);
 
 }

src/libfetchers/tarball.cc

@@ -67,11 +67,15 @@ DownloadFileResult downloadFile(
         StringSink sink;
         dumpString(*res.data, sink);
         auto hash = hashString(htSHA256, *res.data);
-        ValidPathInfo info(store->makeFixedOutputPath(false, hash, name));
+        ValidPathInfo info(store->makeFixedOutputPath(FileIngestionMethod::Flat, hash, name));
         info.narHash = hashString(htSHA256, *sink.s);
         info.narSize = sink.s->size();
-        info.ca = makeFixedOutputCA(false, hash);
-        store->addToStore(info, sink.s, NoRepair, NoCheckSigs);
+        info.ca = FixedOutputHash {
+            .method = FileIngestionMethod::Flat,
+            .hash = hash,
+        };
+        auto source = StringSource { *sink.s };
+        store->addToStore(info, source, NoRepair, NoCheckSigs);
         storePath = std::move(info.path);
     }
 
@@ -101,7 +105,7 @@ DownloadFileResult downloadFile(
     };
 }
 
-Tree downloadTarball(
+std::pair<Tree, time_t> downloadTarball(
     ref<Store> store,
     const std::string & url,
     const std::string & name,
@@ -116,12 +120,9 @@ Tree downloadTarball(
     auto cached = getCache()->lookupExpired(store, inAttrs);
 
     if (cached && !cached->expired)
-        return Tree {
-            .actualPath = store->toRealPath(cached->storePath),
-            .storePath = std::move(cached->storePath),
-            .info = TreeInfo {
-                .lastModified = getIntAttr(cached->infoAttrs, "lastModified"),
-            },
+        return {
+            Tree(store->toRealPath(cached->storePath), std::move(cached->storePath)),
+            getIntAttr(cached->infoAttrs, "lastModified")
         };
 
     auto res = downloadFile(store, url, name, immutable);
@@ -141,7 +142,7 @@ Tree downloadTarball(
             throw nix::Error("tarball '%s' contains an unexpected number of top-level files", url);
         auto topDir = tmpDir + "/" + members.begin()->name;
         lastModified = lstat(topDir).st_mtime;
-        unpackedStorePath = store->addToStore(name, topDir, true, htSHA256, defaultPathFilter, NoRepair);
+        unpackedStorePath = store->addToStore(name, topDir, FileIngestionMethod::Recursive, htSHA256, defaultPathFilter, NoRepair);
     }
 
     Attrs infoAttrs({
@@ -156,118 +157,72 @@ Tree downloadTarball(
         *unpackedStorePath,
         immutable);
 
-    return Tree {
-        .actualPath = store->toRealPath(*unpackedStorePath),
-        .storePath = std::move(*unpackedStorePath),
-        .info = TreeInfo {
-            .lastModified = lastModified,
-        },
+    return {
+        Tree(store->toRealPath(*unpackedStorePath), std::move(*unpackedStorePath)),
+        lastModified,
     };
 }
 
-struct TarballInput : Input
-{
-    ParsedURL url;
-    std::optional<Hash> hash;
-
-    TarballInput(const ParsedURL & url) : url(url)
-    { }
-
-    std::string type() const override { return "tarball"; }
-
-    bool operator ==(const Input & other) const override
-    {
-        auto other2 = dynamic_cast<const TarballInput *>(&other);
-        return
-            other2
-            && to_string() == other2->to_string()
-            && hash == other2->hash;
-    }
-
-    bool isImmutable() const override
-    {
-        return hash || narHash;
-    }
-
-    ParsedURL toURL() const override
-    {
-        auto url2(url);
-        // NAR hashes are preferred over file hashes since tar/zip files
-        // don't have a canonical representation.
-        if (narHash)
-            url2.query.insert_or_assign("narHash", narHash->to_string(SRI));
-        else if (hash)
-            url2.query.insert_or_assign("hash", hash->to_string(SRI));
-        return url2;
-    }
-
-    Attrs toAttrsInternal() const override
-    {
-        Attrs attrs;
-        attrs.emplace("url", url.to_string());
-        if (hash)
-            attrs.emplace("hash", hash->to_string(SRI));
-        return attrs;
-    }
-
-    std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override
-    {
-        auto tree = downloadTarball(store, url.to_string(), "source", false);
-
-        auto input = std::make_shared<TarballInput>(*this);
-        input->narHash = store->queryPathInfo(tree.storePath)->narHash;
-
-        return {std::move(tree), input};
-    }
-};
-
 struct TarballInputScheme : InputScheme
 {
-    std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override
+    std::optional<Input> inputFromURL(const ParsedURL & url) override
     {
-        if (url.scheme != "file" && url.scheme != "http" && url.scheme != "https") return nullptr;
+        if (url.scheme != "file" && url.scheme != "http" && url.scheme != "https") return {};
 
         if (!hasSuffix(url.path, ".zip")
             && !hasSuffix(url.path, ".tar")
             && !hasSuffix(url.path, ".tar.gz")
             && !hasSuffix(url.path, ".tar.xz")
             && !hasSuffix(url.path, ".tar.bz2"))
-            return nullptr;
-
-        auto input = std::make_unique<TarballInput>(url);
-
-        auto hash = input->url.query.find("hash");
-        if (hash != input->url.query.end()) {
-            // FIXME: require SRI hash.
-            input->hash = Hash(hash->second);
-            input->url.query.erase(hash);
-        }
-
-        auto narHash = input->url.query.find("narHash");
-        if (narHash != input->url.query.end()) {
-            // FIXME: require SRI hash.
-            input->narHash = Hash(narHash->second);
-            input->url.query.erase(narHash);
-        }
+            return {};
 
+        Input input;
+        input.attrs.insert_or_assign("type", "tarball");
+        input.attrs.insert_or_assign("url", url.to_string());
+        auto narHash = url.query.find("narHash");
+        if (narHash != url.query.end())
+            input.attrs.insert_or_assign("narHash", narHash->second);
         return input;
     }
 
-    std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override
+    std::optional<Input> inputFromAttrs(const Attrs & attrs) override
     {
         if (maybeGetStrAttr(attrs, "type") != "tarball") return {};
 
         for (auto & [name, value] : attrs)
-            if (name != "type" && name != "url" && name != "hash")
+            if (name != "type" && name != "url" && /* name != "hash" && */ name != "narHash")
                 throw Error("unsupported tarball input attribute '%s'", name);
 
-        auto input = std::make_unique<TarballInput>(parseURL(getStrAttr(attrs, "url")));
-        if (auto hash = maybeGetStrAttr(attrs, "hash"))
-            // FIXME: require SRI hash.
-            input->hash = Hash(*hash);
-
+        Input input;
+        input.attrs = attrs;
+        //input.immutable = (bool) maybeGetStrAttr(input.attrs, "hash");
         return input;
     }
+
+    ParsedURL toURL(const Input & input) override
+    {
+        auto url = parseURL(getStrAttr(input.attrs, "url"));
+        // NAR hashes are preferred over file hashes since tar/zip files
+        // don't have a canonical representation.
+        if (auto narHash = input.getNarHash())
+            url.query.insert_or_assign("narHash", narHash->to_string(SRI, true));
+        /*
+        else if (auto hash = maybeGetStrAttr(input.attrs, "hash"))
+            url.query.insert_or_assign("hash", Hash(*hash).to_string(SRI, true));
+        */
+        return url;
+    }
+
+    bool hasAllInfo(const Input & input) override
+    {
+        return true;
+    }
+
+    std::pair<Tree, Input> fetch(ref<Store> store, const Input & input) override
+    {
+        auto tree = downloadTarball(store, getStrAttr(input.attrs, "url"), "source", false).first;
+        return {std::move(tree), input};
+    }
 };
 
 static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<TarballInputScheme>()); });

src/libfetchers/tree-info.cc

@@ -1,60 +0,0 @@
-#include "tree-info.hh"
-#include "store-api.hh"
-
-#include <nlohmann/json.hpp>
-
-namespace nix::fetchers {
-
-StorePath TreeInfo::computeStorePath(Store & store) const
-{
-    assert(narHash);
-    return store.makeFixedOutputPath(true, narHash, "source");
-}
-
-TreeInfo TreeInfo::fromJson(const nlohmann::json & json)
-{
-    TreeInfo info;
-
-    auto i = json.find("info");
-    if (i != json.end()) {
-        const nlohmann::json & i2(*i);
-
-        auto j = i2.find("narHash");
-        if (j != i2.end())
-            info.narHash = Hash((std::string) *j);
-        else
-            throw Error("attribute 'narHash' missing in lock file");
-
-        j = i2.find("revCount");
-        if (j != i2.end())
-            info.revCount = *j;
-
-        j = i2.find("lastModified");
-        if (j != i2.end())
-            info.lastModified = *j;
-
-        return info;
-    }
-
-    i = json.find("narHash");
-    if (i != json.end()) {
-        info.narHash = Hash((std::string) *i);
-        return info;
-    }
-
-    throw Error("attribute 'info' missing in lock file");
-}
-
-nlohmann::json TreeInfo::toJson() const
-{
-    nlohmann::json json;
-    assert(narHash);
-    json["narHash"] = narHash.to_string(SRI);
-    if (revCount)
-        json["revCount"] = *revCount;
-    if (lastModified)
-        json["lastModified"] = *lastModified;
-    return json;
-}
-
-}

src/libfetchers/tree-info.hh

@@ -1,33 +0,0 @@
-#pragma once
-
-#include "path.hh"
-#include "hash.hh"
-
-#include <nlohmann/json_fwd.hpp>
-
-namespace nix { class Store; }
-
-namespace nix::fetchers {
-
-struct TreeInfo
-{
-    Hash narHash;
-    std::optional<uint64_t> revCount;
-    std::optional<time_t> lastModified;
-
-    bool operator ==(const TreeInfo & other) const
-    {
-        return
-            narHash == other.narHash
-            && revCount == other.revCount
-            && lastModified == other.lastModified;
-    }
-
-    StorePath computeStorePath(Store & store) const;
-
-    static TreeInfo fromJson(const nlohmann::json & json);
-
-    nlohmann::json toJson() const;
-};
-
-}

src/libmain/common-args.cc

@@ -1,5 +1,6 @@
 #include "common-args.hh"
 #include "globals.hh"
+#include "loggers.hh"
 
 namespace nix {
 
@@ -48,6 +49,14 @@ MixCommonArgs::MixCommonArgs(const string & programName)
         }
     });
 
+    addFlag({
+        .longName = "log-format",
+        .description = "format of log output; \"raw\", \"internal-json\", \"bar\" "
+                        "or \"bar-with-logs\"",
+        .labels = {"format"},
+        .handler = {[](std::string format) { setLogFormat(format); }},
+    });
+
     addFlag({
         .longName = "max-jobs",
         .shortName = 'j',

src/libmain/loggers.cc

@@ -0,0 +1,53 @@
+#include "loggers.hh"
+#include "progress-bar.hh"
+#include "util.hh"
+
+namespace nix {
+
+LogFormat defaultLogFormat = LogFormat::raw;
+
+LogFormat parseLogFormat(const std::string & logFormatStr) {
+    if (logFormatStr == "raw" || getEnv("NIX_GET_COMPLETIONS"))
+        return LogFormat::raw;
+    else if (logFormatStr == "raw-with-logs")
+        return LogFormat::rawWithLogs;
+    else if (logFormatStr == "internal-json")
+        return LogFormat::internalJson;
+    else if (logFormatStr == "bar")
+        return LogFormat::bar;
+    else if (logFormatStr == "bar-with-logs")
+        return LogFormat::barWithLogs;
+    throw Error("option 'log-format' has an invalid value '%s'", logFormatStr);
+}
+
+Logger * makeDefaultLogger() {
+    switch (defaultLogFormat) {
+    case LogFormat::raw:
+        return makeSimpleLogger(false);
+    case LogFormat::rawWithLogs:
+        return makeSimpleLogger(true);
+    case LogFormat::internalJson:
+        return makeJSONLogger(*makeSimpleLogger(true));
+    case LogFormat::bar:
+        return makeProgressBar();
+    case LogFormat::barWithLogs:
+        return makeProgressBar(true);
+    default:
+        abort();
+    }
+}
+
+void setLogFormat(const std::string & logFormatStr) {
+    setLogFormat(parseLogFormat(logFormatStr));
+}
+
+void setLogFormat(const LogFormat & logFormat) {
+    defaultLogFormat = logFormat;
+    createDefaultLogger();
+}
+
+void createDefaultLogger() {
+    logger = makeDefaultLogger();
+}
+
+}

src/libmain/loggers.hh

@@ -0,0 +1,20 @@
+#pragma once
+
+#include "types.hh"
+
+namespace nix {
+
+enum class LogFormat {
+  raw,
+  rawWithLogs,
+  internalJson,
+  bar,
+  barWithLogs,
+};
+
+void setLogFormat(const std::string & logFormatStr);
+void setLogFormat(const LogFormat & logFormat);
+
+void createDefaultLogger();
+
+}

src/libmain/progress-bar.cc

@@ -106,27 +106,36 @@ public:
         updateThread.join();
     }
 
-    void stop()
+    void stop() override
     {
         auto state(state_.lock());
         if (!state->active) return;
         state->active = false;
-        std::string status = getStatus(*state);
         writeToStderr("\r\e[K");
-        /*
-        if (status != "")
-            writeToStderr("[" + status + "]\n");
-        */
         updateCV.notify_one();
         quitCV.notify_one();
     }
 
+    bool isVerbose() override {
+        return printBuildLogs;
+    }
+
     void log(Verbosity lvl, const FormatOrString & fs) override
     {
         auto state(state_.lock());
         log(*state, lvl, fs.s);
     }
 
+    void logEI(const ErrorInfo &ei) override
+    {
+        auto state(state_.lock());
+
+        std::stringstream oss;
+        showErrorInfo(oss, ei, loggerSettings.showTrace.get());
+
+        log(*state, ei.level, oss.str());
+    }
+
     void log(State & state, Verbosity lvl, const std::string & s)
     {
         if (state.active) {
@@ -144,7 +153,7 @@ public:
     {
         auto state(state_.lock());
 
-        if (lvl <= verbosity && !s.empty())
+        if (lvl <= verbosity && !s.empty() && type != actBuildWaiting)
             log(*state, lvl, s + "...");
 
         state->activities.emplace_back(ActInfo());
@@ -156,7 +165,7 @@ public:
         state->activitiesByType[type].its.emplace(act, i);
 
         if (type == actBuild) {
-            auto name = storePathToName(getS(fields, 0));
+            std::string name(storePathToName(getS(fields, 0)));
             if (hasSuffix(name, ".drv"))
                 name = name.substr(0, name.size() - 4);
             i->s = fmt("building " ANSI_BOLD "%s" ANSI_NORMAL, name);
@@ -459,11 +468,17 @@ public:
     }
 };
 
+Logger * makeProgressBar(bool printBuildLogs)
+{
+    return new ProgressBar(
+        printBuildLogs,
+        isatty(STDERR_FILENO) && getEnv("TERM").value_or("dumb") != "dumb"
+    );
+}
+
 void startProgressBar(bool printBuildLogs)
 {
-    logger = new ProgressBar(
-        printBuildLogs,
-        isatty(STDERR_FILENO) && getEnv("TERM").value_or("dumb") != "dumb");
+    logger = makeProgressBar(printBuildLogs);
 }
 
 void stopProgressBar()

src/libmain/progress-bar.hh

@@ -4,6 +4,8 @@
 
 namespace nix {
 
+Logger * makeProgressBar(bool printBuildLogs = false);
+
 void startProgressBar(bool printBuildLogs = false);
 
 void stopProgressBar();

src/libmain/shared.cc

@@ -2,6 +2,7 @@
 #include "shared.hh"
 #include "store-api.hh"
 #include "util.hh"
+#include "loggers.hh"
 
 #include <algorithm>
 #include <cctype>
@@ -47,7 +48,10 @@ void printMissing(ref<Store> store, const StorePathSet & willBuild,
     unsigned long long downloadSize, unsigned long long narSize, Verbosity lvl)
 {
     if (!willBuild.empty()) {
-        printMsg(lvl, "these derivations will be built:");
+        if (willBuild.size() == 1)
+            printMsg(lvl, fmt("this derivation will be built:"));
+        else
+            printMsg(lvl, fmt("these %d derivations will be built:", willBuild.size()));
         auto sorted = store->topoSortPaths(willBuild);
         reverse(sorted.begin(), sorted.end());
         for (auto & i : sorted)
@@ -55,9 +59,18 @@ void printMissing(ref<Store> store, const StorePathSet & willBuild,
     }
 
     if (!willSubstitute.empty()) {
-        printMsg(lvl, fmt("these paths will be fetched (%.2f MiB download, %.2f MiB unpacked):",
-                downloadSize / (1024.0 * 1024.0),
-                narSize / (1024.0 * 1024.0)));
+        const float downloadSizeMiB = downloadSize / (1024.f * 1024.f);
+        const float narSizeMiB = narSize / (1024.f * 1024.f);
+        if (willSubstitute.size() == 1) {
+            printMsg(lvl, fmt("this path will be fetched (%.2f MiB download, %.2f MiB unpacked):",
+                downloadSizeMiB,
+                narSizeMiB));
+        } else {
+            printMsg(lvl, fmt("these %d paths will be fetched (%.2f MiB download, %.2f MiB unpacked):",
+                willSubstitute.size(),
+                downloadSizeMiB,
+                narSizeMiB));
+        }
         for (auto & i : willSubstitute)
             printMsg(lvl, fmt("  %s", store->printStorePath(i)));
     }
@@ -75,7 +88,7 @@ string getArg(const string & opt,
     Strings::iterator & i, const Strings::iterator & end)
 {
     ++i;
-    if (i == end) throw UsageError(format("'%1%' requires an argument") % opt);
+    if (i == end) throw UsageError("'%1%' requires an argument", opt);
     return *i;
 }
 
@@ -169,7 +182,7 @@ LegacyArgs::LegacyArgs(const std::string & programName,
         .longName = "no-build-output",
         .shortName = 'Q',
         .description = "do not show build output",
-        .handler = {&settings.verboseBuild, false},
+        .handler = {[&]() {setLogFormat(LogFormat::raw); }},
     });
 
     addFlag({
@@ -234,7 +247,7 @@ bool LegacyArgs::processArgs(const Strings & args, bool finish)
     Strings ss(args);
     auto pos = ss.begin();
     if (!parseArg(pos, ss.end()))
-        throw UsageError(format("unexpected argument '%1%'") % args.front());
+        throw UsageError("unexpected argument '%1%'", args.front());
     return true;
 }
 
@@ -281,7 +294,7 @@ void showManPage(const string & name)
     restoreSignals();
     setenv("MANPATH", settings.nixManDir.c_str(), 1);
     execlp("man", "man", name.c_str(), nullptr);
-    throw SysError(format("command 'man %1%' failed") % name.c_str());
+    throw SysError("command 'man %1%' failed", name.c_str());
 }
 
 
@@ -289,6 +302,8 @@ int handleExceptions(const string & programName, std::function<void()> fun)
 {
     ReceiveInterrupts receiveInterrupts; // FIXME: need better place for this
 
+    ErrorInfo::programName = baseNameOf(programName);
+
     string error = ANSI_RED "error:" ANSI_NORMAL " ";
     try {
         try {
@@ -304,13 +319,12 @@ int handleExceptions(const string & programName, std::function<void()> fun)
     } catch (Exit & e) {
         return e.status;
     } catch (UsageError & e) {
-        printError(
-            format(error + "%1%\nTry '%2% --help' for more information.")
-            % e.what() % programName);
+        logError(e.info());
+        printError("Try '%1% --help' for more information.", programName);
         return 1;
     } catch (BaseError & e) {
-        printError(format(error + "%1%%2%") % (settings.showTrace ? e.prefix() : "") % e.msg());
-        if (e.prefix() != "" && !settings.showTrace)
+        logError(e.info());
+        if (e.hasTrace() && !loggerSettings.showTrace.get())
             printError("(use '--show-trace' to show detailed location information)");
         return e.status;
     } catch (std::bad_alloc & e) {
@@ -346,7 +360,7 @@ RunPager::RunPager()
         execlp("pager", "pager", nullptr);
         execlp("less", "less", nullptr);
         execlp("more", "more", nullptr);
-        throw SysError(format("executing '%1%'") % pager);
+        throw SysError("executing '%1%'", pager);
     });
 
     pid.setKillSignal(SIGINT);

src/libmain/shared.hh

@@ -56,7 +56,7 @@ template<class N> N getIntArg(const string & opt,
     Strings::iterator & i, const Strings::iterator & end, bool allowUnit)
 {
     ++i;
-    if (i == end) throw UsageError(format("'%1%' requires an argument") % opt);
+    if (i == end) throw UsageError("'%1%' requires an argument", opt);
     string s = *i;
     N multiplier = 1;
     if (allowUnit && !s.empty()) {
@@ -66,13 +66,13 @@ template<class N> N getIntArg(const string & opt,
             else if (u == 'M') multiplier = 1ULL << 20;
             else if (u == 'G') multiplier = 1ULL << 30;
             else if (u == 'T') multiplier = 1ULL << 40;
-            else throw UsageError(format("invalid unit specifier '%1%'") % u);
+            else throw UsageError("invalid unit specifier '%1%'", u);
             s.resize(s.size() - 1);
         }
     }
     N n;
     if (!string2Int(s, n))
-        throw UsageError(format("'%1%' requires an integer argument") % opt);
+        throw UsageError("'%1%' requires an integer argument", opt);
     return n * multiplier;
 }
 

src/libmain/stack.cc

@@ -1,4 +1,4 @@
-#include "types.hh"
+#include "error.hh"
 
 #include <cstring>
 #include <cstddef>

src/libstore/binary-cache-store.cc

@@ -15,6 +15,7 @@
 #include <chrono>
 #include <future>
 #include <regex>
+#include <fstream>
 
 #include <nlohmann/json.hpp>
 
@@ -40,14 +41,14 @@ void BinaryCacheStore::init()
         upsertFile(cacheInfoFile, "StoreDir: " + storeDir + "\n", "text/x-nix-cache-info");
     } else {
         for (auto & line : tokenizeString<Strings>(*cacheInfo, "\n")) {
-            size_t colon = line.find(':');
-            if (colon == std::string::npos) continue;
+            size_t colon= line.find(':');
+            if (colon ==std::string::npos) continue;
             auto name = line.substr(0, colon);
             auto value = trim(line.substr(colon + 1, std::string::npos));
             if (name == "StoreDir") {
                 if (value != storeDir)
-                    throw Error(format("binary cache '%s' is for Nix stores with prefix '%s', not '%s'")
-                        % getUri() % value % storeDir);
+                    throw Error("binary cache '%s' is for Nix stores with prefix '%s', not '%s'",
+                        getUri(), value, storeDir);
             } else if (name == "WantMassQuery") {
                 wantMassQuery.setDefault(value == "1" ? "true" : "false");
             } else if (name == "Priority") {
@@ -57,6 +58,13 @@ void BinaryCacheStore::init()
     }
 }
 
+void BinaryCacheStore::upsertFile(const std::string & path,
+    std::string && data,
+    const std::string & mimeType)
+{
+    upsertFile(path, std::make_shared<std::stringstream>(std::move(data)), mimeType);
+}
+
 void BinaryCacheStore::getFile(const std::string & path,
     Callback<std::shared_ptr<std::string>> callback) noexcept
 {
@@ -93,7 +101,7 @@ std::shared_ptr<std::string> BinaryCacheStore::getFile(const std::string & path)
 
 std::string BinaryCacheStore::narInfoFileFor(const StorePath & storePath)
 {
-    return storePathToHash(printStorePath(storePath)) + ".narinfo";
+    return std::string(storePath.hashPart()) + ".narinfo";
 }
 
 void BinaryCacheStore::writeNarInfo(ref<NarInfo> narInfo)
@@ -102,7 +110,7 @@ void BinaryCacheStore::writeNarInfo(ref<NarInfo> narInfo)
 
     upsertFile(narInfoFile, narInfo->to_string(*this), "text/x-nix-narinfo");
 
-    auto hashPart = storePathToHash(printStorePath(narInfo->path));
+    std::string hashPart(narInfo->path.hashPart());
 
     {
         auto state_(state.lock());
@@ -113,10 +121,74 @@ void BinaryCacheStore::writeNarInfo(ref<NarInfo> narInfo)
         diskCache->upsertNarInfo(getUri(), hashPart, std::shared_ptr<NarInfo>(narInfo));
 }
 
-void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::string> & nar,
-    RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor)
+AutoCloseFD openFile(const Path & path)
 {
-    if (!repair && isValidPath(info.path)) return;
+    auto fd = open(path.c_str(), O_RDONLY | O_CLOEXEC);
+    if (!fd)
+        throw SysError("opening file '%1%'", path);
+    return fd;
+}
+
+struct FileSource : FdSource
+{
+    AutoCloseFD fd2;
+
+    FileSource(const Path & path)
+        : fd2(openFile(path))
+    {
+        fd = fd2.get();
+    }
+};
+
+void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource,
+    RepairFlag repair, CheckSigsFlag checkSigs)
+{
+    assert(info.narHash && info.narSize);
+
+    if (!repair && isValidPath(info.path)) {
+        // FIXME: copyNAR -> null sink
+        narSource.drain();
+        return;
+    }
+
+    auto [fdTemp, fnTemp] = createTempFile();
+
+    auto now1 = std::chrono::steady_clock::now();
+
+    /* Read the NAR simultaneously into a CompressionSink+FileSink (to
+       write the compressed NAR to disk), into a HashSink (to get the
+       NAR hash), and into a NarAccessor (to get the NAR listing). */
+    HashSink fileHashSink(htSHA256);
+    std::shared_ptr<FSAccessor> narAccessor;
+    {
+    FdSink fileSink(fdTemp.get());
+    TeeSink teeSink(fileSink, fileHashSink);
+    auto compressionSink = makeCompressionSink(compression, teeSink);
+    TeeSource teeSource(narSource, *compressionSink);
+    narAccessor = makeNarAccessor(teeSource);
+    compressionSink->finish();
+    }
+
+    auto now2 = std::chrono::steady_clock::now();
+
+    auto narInfo = make_ref<NarInfo>(info);
+    narInfo->narSize = info.narSize;
+    narInfo->narHash = info.narHash;
+    narInfo->compression = compression;
+    auto [fileHash, fileSize] = fileHashSink.finish();
+    narInfo->fileHash = fileHash;
+    narInfo->fileSize = fileSize;
+    narInfo->url = "nar/" + narInfo->fileHash.to_string(Base32, false) + ".nar"
+        + (compression == "xz" ? ".xz" :
+           compression == "bzip2" ? ".bz2" :
+           compression == "br" ? ".br" :
+           "");
+
+    auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
+    printMsg(lvlTalkative, "copying path '%1%' (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache",
+        printStorePath(narInfo->path), info.narSize,
+        ((1.0 - (double) fileSize / info.narSize) * 100.0),
+        duration);
 
     /* Verify that all references are valid. This may do some .narinfo
        reads, but typically they'll already be cached. */
@@ -129,23 +201,6 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
                 printStorePath(info.path), printStorePath(ref));
         }
 
-    assert(nar->compare(0, narMagic.size(), narMagic) == 0);
-
-    auto narInfo = make_ref<NarInfo>(info);
-
-    narInfo->narSize = nar->size();
-    narInfo->narHash = hashString(htSHA256, *nar);
-
-    if (info.narHash && info.narHash != narInfo->narHash)
-        throw Error("refusing to copy corrupted path '%1%' to binary cache", printStorePath(info.path));
-
-    auto accessor_ = std::dynamic_pointer_cast<RemoteFSAccessor>(accessor);
-
-    auto narAccessor = makeNarAccessor(nar);
-
-    if (accessor_)
-        accessor_->addToCache(printStorePath(info.path), *nar, narAccessor);
-
     /* Optionally write a JSON file containing a listing of the
        contents of the NAR. */
     if (writeNARListing) {
@@ -157,33 +212,13 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
 
             {
                 auto res = jsonRoot.placeholder("root");
-                listNar(res, narAccessor, "", true);
+                listNar(res, ref<FSAccessor>(narAccessor), "", true);
             }
         }
 
-        upsertFile(storePathToHash(printStorePath(info.path)) + ".ls", jsonOut.str(), "application/json");
+        upsertFile(std::string(info.path.to_string()) + ".ls", jsonOut.str(), "application/json");
     }
 
-    /* Compress the NAR. */
-    narInfo->compression = compression;
-    auto now1 = std::chrono::steady_clock::now();
-    auto narCompressed = compress(compression, *nar, parallelCompression);
-    auto now2 = std::chrono::steady_clock::now();
-    narInfo->fileHash = hashString(htSHA256, *narCompressed);
-    narInfo->fileSize = narCompressed->size();
-
-    auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
-    printMsg(lvlTalkative, "copying path '%1%' (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache",
-        printStorePath(narInfo->path), narInfo->narSize,
-        ((1.0 - (double) narCompressed->size() / nar->size()) * 100.0),
-        duration);
-
-    narInfo->url = "nar/" + narInfo->fileHash.to_string(Base32, false) + ".nar"
-        + (compression == "xz" ? ".xz" :
-           compression == "bzip2" ? ".bz2" :
-           compression == "br" ? ".br" :
-           "");
-
     /* Optionally maintain an index of DWARF debug info files
        consisting of JSON files named 'debuginfo/<build-id>' that
        specify the NAR file and member containing the debug info. */
@@ -244,12 +279,14 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
     /* Atomically write the NAR file. */
     if (repair || !fileExists(narInfo->url)) {
         stats.narWrite++;
-        upsertFile(narInfo->url, *narCompressed, "application/x-nix-nar");
+        upsertFile(narInfo->url,
+            std::make_shared<std::fstream>(fnTemp, std::ios_base::in),
+            "application/x-nix-nar");
     } else
         stats.narWriteAverted++;
 
-    stats.narWriteBytes += nar->size();
-    stats.narWriteCompressedBytes += narCompressed->size();
+    stats.narWriteBytes += info.narSize;
+    stats.narWriteCompressedBytes += fileSize;
     stats.narWriteCompressionTimeMs += duration;
 
     /* Atomically write the NAR info file.*/
@@ -284,7 +321,7 @@ void BinaryCacheStore::narFromPath(const StorePath & storePath, Sink & sink)
     try {
         getFile(info->url, *decompressor);
     } catch (NoSuchBinaryCacheFile & e) {
-        throw SubstituteGone(e.what());
+        throw SubstituteGone(e.info());
     }
 
     decompressor->finish();
@@ -327,7 +364,7 @@ void BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath,
 }
 
 StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath,
-    bool recursive, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
+    FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
 {
     // FIXME: some cut&paste from LocalStore::addToStore().
 
@@ -336,7 +373,7 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath
        small files. */
     StringSink sink;
     Hash h;
-    if (recursive) {
+    if (method == FileIngestionMethod::Recursive) {
         dumpPath(srcPath, sink, filter);
         h = hashString(hashAlgo, *sink.s);
     } else {
@@ -345,9 +382,10 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath
         h = hashString(hashAlgo, s);
     }
 
-    ValidPathInfo info(makeFixedOutputPath(recursive, h, name));
+    ValidPathInfo info(makeFixedOutputPath(method, h, name));
 
-    addToStore(info, sink.s, repair, CheckSigs, nullptr);
+    auto source = StringSource { *sink.s };
+    addToStore(info, source, repair, CheckSigs);
 
     return std::move(info.path);
 }
@@ -356,12 +394,13 @@ StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s
     const StorePathSet & references, RepairFlag repair)
 {
     ValidPathInfo info(computeStorePathForText(name, s, references));
-    info.references = cloneStorePathSet(references);
+    info.references = references;
 
     if (repair || !isValidPath(info.path)) {
         StringSink sink;
         dumpString(s, sink);
-        addToStore(info, sink.s, repair, CheckSigs, nullptr);
+        auto source = StringSource { *sink.s };
+        addToStore(info, source, repair, CheckSigs);
     }
 
     return std::move(info.path);
@@ -383,21 +422,19 @@ void BinaryCacheStore::addSignatures(const StorePath & storePath, const StringSe
 
     narInfo->sigs.insert(sigs.begin(), sigs.end());
 
-    auto narInfoFile = narInfoFileFor(narInfo->path);
-
     writeNarInfo(narInfo);
 }
 
 std::shared_ptr<std::string> BinaryCacheStore::getBuildLog(const StorePath & path)
 {
-    auto drvPath = path.clone();
+    auto drvPath = path;
 
     if (!path.isDerivation()) {
         try {
             auto info = queryPathInfo(path);
             // FIXME: add a "Log" field to .narinfo
             if (!info->deriver) return nullptr;
-            drvPath = info->deriver->clone();
+            drvPath = *info->deriver;
         } catch (InvalidPath &) {
             return nullptr;
         }