downloader: when retry but can't resume, retry the whole file

.dir-locals.el

@@ -1,7 +1,6 @@
 ((c++-mode . (
   (c-file-style . "k&r")
   (c-basic-offset . 4)
-  (c-block-comment-prefix . "  ")
   (indent-tabs-mode . nil)
   (tab-width . 4)
   (show-trailing-whitespace . t)

.github/ISSUE_TEMPLATE.md

@@ -0,0 +1,27 @@
+<!--
+
+# Filing a Nix issue
+
+*WAIT* Are you sure you're filing your issue in the right repository?
+
+We appreciate you taking the time to tell us about issues you encounter, but routing the issue to the right place will get you help sooner and save everyone time.
+
+This is the Nix repository, and issues here should be about Nix the build and package management *_tool_*.
+
+If you have a problem with a specific package on NixOS or when using Nix, you probably want to file an issue with _nixpkgs_, whose issue tracker is over at https://github.com/NixOS/nixpkgs/issues.
+
+Examples of _Nix_ issues:
+
+- Nix segfaults when I run `nix-build -A blahblah`
+- The Nix language needs a new builtin: `builtins.foobar`
+- Regression in the behavior of `nix-env` in Nix 2.0
+
+Examples of _nixpkgs_ issues:
+
+- glibc is b0rked on aarch64
+- chromium in NixOS doesn't support U2F but google-chrome does!
+- The OpenJDK package on macOS is missing a key component
+
+Chances are if you're a newcomer to the Nix world, you'll probably want the [nixpkgs tracker](https://github.com/NixOS/nixpkgs/issues). It also gets a lot more eyeball traffic so you'll probably get a response a lot more quickly.
+
+-->

.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,32 +0,0 @@
----
-name: Bug report
-about: Create a report to help us improve
-title: ''
-labels: bug
-assignees: ''
-
----
-
-**Describe the bug**
-
-A clear and concise description of what the bug is.
-
-If you have a problem with a specific package or NixOS,
-you probably want to file an issue at https://github.com/NixOS/nixpkgs/issues. 
-
-**Steps To Reproduce**
-
-1. Go to '...'
-2. Click on '....'
-3. Scroll down to '....'
-4. See error
-
-**Expected behavior**
-
-A clear and concise description of what you expected to happen.
-
-**`nix-env --version` output**
-
-**Additional context**
-
-Add any other context about the problem here.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -1,20 +0,0 @@
----
-name: Feature request
-about: Suggest an idea for this project
-title: ''
-labels: improvement
-assignees: ''
-
----
-
-**Is your feature request related to a problem? Please describe.**
-A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
-
-**Describe the solution you'd like**
-A clear and concise description of what you want to happen.
-
-**Describe alternatives you've considered**
-A clear and concise description of any alternative solutions or features you've considered.
-
-**Additional context**
-Add any other context or screenshots about the feature request here.

.github/dependabot.yml

@@ -1,6 +0,0 @@
-version: 2
-updates:
-  - package-ecosystem: "github-actions"
-    directory: "/"
-    schedule:
-      interval: "weekly"

.github/workflows/test.yml

@@ -6,19 +6,9 @@ jobs:
   tests:
     strategy:
       matrix:
-        os: [ubuntu-latest, macos-latest]
+        os: [ubuntu-18.04, macos]
     runs-on: ${{ matrix.os }}
     steps:
     - uses: actions/checkout@v2
-    - uses: cachix/install-nix-action@v10
-    - run: nix-build release.nix --arg nix '{ outPath = ./.; revCount = 123; shortRev = "abcdefgh"; }' --arg systems '[ builtins.currentSystem ]' -A installerScript -A perlBindings
-  macos_perf_test:
-    runs-on: macos-latest
-    steps:
-    - name: Disable syspolicy assessments
-      run: |
-        spctl --status
-        sudo spctl --master-disable
-    - uses: actions/checkout@v2
-    - uses: cachix/install-nix-action@v10
+    - uses: cachix/install-nix-action@v8
     - run: nix-build release.nix --arg nix '{ outPath = ./.; revCount = 123; shortRev = "abcdefgh"; }' --arg systems '[ builtins.currentSystem ]' -A installerScript -A perlBindings

.gitignore

@@ -49,9 +49,6 @@ perl/Makefile.config
 # /src/libstore/
 *.gen.*
 
-# /src/libutil/
-/src/libutil/tests/libutil-tests
-
 /src/nix/nix
 
 # /src/nix-env/
@@ -78,8 +75,6 @@ perl/Makefile.config
 
 /src/nix-copy-closure/nix-copy-closure
 
-/src/error-demo/error-demo
-
 /src/build-remote/build-remote
 
 # /tests/

.travis.yml

@@ -0,0 +1,8 @@
+matrix:
+  include:
+    - language: osx
+      script: ./tests/install-darwin.sh
+    - language: nix
+      script: nix-build release.nix -A build.x86_64-linux
+notifications:
+  email: false

Makefile

@@ -1,10 +1,9 @@
 makefiles = \
   mk/precompiled-headers.mk \
   local.mk \
+  nix-rust/local.mk \
   src/libutil/local.mk \
-  src/libutil/tests/local.mk \
   src/libstore/local.mk \
-  src/libfetchers/local.mk \
   src/libmain/local.mk \
   src/libexpr/local.mk \
   src/nix/local.mk \

Makefile.config.in

@@ -1,38 +1,36 @@
 AR = @AR@
 BDW_GC_LIBS = @BDW_GC_LIBS@
-BOOST_LDFLAGS = @BOOST_LDFLAGS@
 BUILD_SHARED_LIBS = @BUILD_SHARED_LIBS@
 CC = @CC@
 CFLAGS = @CFLAGS@
 CXX = @CXX@
 CXXFLAGS = @CXXFLAGS@
-EDITLINE_LIBS = @EDITLINE_LIBS@
-ENABLE_S3 = @ENABLE_S3@
-GTEST_LIBS = @GTEST_LIBS@
-HAVE_SECCOMP = @HAVE_SECCOMP@
-HAVE_SODIUM = @HAVE_SODIUM@
 LDFLAGS = @LDFLAGS@
-LIBARCHIVE_LIBS = @LIBARCHIVE_LIBS@
-LIBBROTLI_LIBS = @LIBBROTLI_LIBS@
+ENABLE_S3 = @ENABLE_S3@
+HAVE_SODIUM = @HAVE_SODIUM@
+HAVE_SECCOMP = @HAVE_SECCOMP@
+BOOST_LDFLAGS = @BOOST_LDFLAGS@
 LIBCURL_LIBS = @LIBCURL_LIBS@
-LIBLZMA_LIBS = @LIBLZMA_LIBS@
 OPENSSL_LIBS = @OPENSSL_LIBS@
 PACKAGE_NAME = @PACKAGE_NAME@
 PACKAGE_VERSION = @PACKAGE_VERSION@
 SODIUM_LIBS = @SODIUM_LIBS@
+LIBLZMA_LIBS = @LIBLZMA_LIBS@
 SQLITE3_LIBS = @SQLITE3_LIBS@
+LIBBROTLI_LIBS = @LIBBROTLI_LIBS@
+LIBARCHIVE_LIBS = @LIBARCHIVE_LIBS@
+EDITLINE_LIBS = @EDITLINE_LIBS@
 bash = @bash@
 bindir = @bindir@
+lsof = @lsof@
 datadir = @datadir@
 datarootdir = @datarootdir@
-doc_generate = @doc_generate@
 docdir = @docdir@
 exec_prefix = @exec_prefix@
 includedir = @includedir@
 libdir = @libdir@
 libexecdir = @libexecdir@
 localstatedir = @localstatedir@
-lsof = @lsof@
 mandir = @mandir@
 pkglibdir = $(libdir)/$(PACKAGE_NAME)
 prefix = @prefix@
@@ -40,5 +38,6 @@ sandbox_shell = @sandbox_shell@
 storedir = @storedir@
 sysconfdir = @sysconfdir@
 system = @system@
+doc_generate = @doc_generate@
 xmllint = @xmllint@
 xsltproc = @xsltproc@

README.md

@@ -1,54 +1,21 @@
-# Nix
-
 [![Open Collective supporters](https://opencollective.com/nixos/tiers/supporter/badge.svg?label=Supporters&color=brightgreen)](https://opencollective.com/nixos)
-[![Test](https://github.com/NixOS/nix/workflows/Test/badge.svg)](https://github.com/NixOS/nix/actions)
 
-Nix is a powerful package manager for Linux and other Unix systems that makes package
-management reliable and reproducible. Please refer to the [Nix manual](https://nixos.org/nix/manual)
-for more details.
+Nix, the purely functional package manager
+------------------------------------------
 
-## Installation
+Nix is a new take on package management that is fairly unique. Because of its
+purity aspects, a lot of issues found in traditional package managers don't
+appear with Nix.
 
-On Linux and macOS the easiest way to Install Nix is to run the following shell command
-(as a user other than root):
+To find out more about the tool, usage and installation instructions, please
+read the manual, which is available on the Nix website at
+<https://nixos.org/nix/manual>.
 
-```
-$ curl -L https://nixos.org/nix/install | sh
-```
+## Contributing
 
-Information on additional installation methods is available on the [Nix download page](https://nixos.org/download.html).
-
-## Building And Developing
-
-### Building Nix
-
-You can build Nix using one of the targets provided by [release.nix](./release.nix):
-
-```
-$ nix-build ./release.nix -A build.aarch64-linux
-$ nix-build ./release.nix -A build.x86_64-darwin
-$ nix-build ./release.nix -A build.i686-linux
-$ nix-build ./release.nix -A build.x86_64-linux
-```
-
-### Development Environment
-
-You can use the provided `shell.nix` to get a working development environment:
-
-```
-$ nix-shell
-$ ./bootstrap.sh
-$ ./configure
-$ make
-```
-
-## Additional Resources
-
-- [Nix manual](https://nixos.org/nix/manual)
-- [Nix jobsets on hydra.nixos.org](https://hydra.nixos.org/project/nix)
-- [NixOS Discourse](https://discourse.nixos.org/)
-- [IRC - #nixos on freenode.net](irc://irc.freenode.net/#nixos)
+Take a look at the [Hacking Section](https://nixos.org/nix/manual/#chap-hacking)
+of the manual. It helps you to get started with building Nix from source.
 
 ## License
 
-Nix is released under the [LGPL v2.1](./COPYING).
+Nix is released under the LGPL v2.1

configure.ac

@@ -266,10 +266,6 @@ if test "$gc" = yes; then
 fi
 
 
-# Look for gtest.
-PKG_CHECK_MODULES([GTEST], [gtest_main])
-
-
 # documentation generation switch
 AC_ARG_ENABLE(doc-gen, AC_HELP_STRING([--disable-doc-gen],
   [disable documentation generation]),

doc/manual/advanced-topics/diff-hook.xml

@@ -70,7 +70,7 @@ path just built.</para>
 
   <screen>
 $ nix-build ./deterministic.nix -A stable
-this derivation will be built:
+these derivations will be built:
   /nix/store/z98fasz2jqy9gs0xbvdj939p27jwda38-stable.drv
 building '/nix/store/z98fasz2jqy9gs0xbvdj939p27jwda38-stable.drv'...
 /nix/store/yyxlzw3vqaas7wfp04g0b1xg51f2czgq-stable
@@ -85,7 +85,7 @@ checking outputs of '/nix/store/z98fasz2jqy9gs0xbvdj939p27jwda38-stable.drv'...
 
   <screen>
 $ nix-build ./deterministic.nix -A unstable
-this derivation will be built:
+these derivations will be built:
   /nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv
 building '/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv'...
 /nix/store/krpqk0l9ib0ibi1d2w52z293zw455cap-unstable
@@ -193,7 +193,7 @@ repeat = 1
     An example output of this configuration:
     <screen>
 $ nix-build ./test.nix -A unstable
-this derivation will be built:
+these derivations will be built:
   /nix/store/ch6llwpr2h8c3jmnf3f2ghkhx59aa97f-unstable.drv
 building '/nix/store/ch6llwpr2h8c3jmnf3f2ghkhx59aa97f-unstable.drv' (round 1/2)...
 building '/nix/store/ch6llwpr2h8c3jmnf3f2ghkhx59aa97f-unstable.drv' (round 2/2)...

doc/manual/advanced-topics/post-build-hook.xml

@@ -61,7 +61,7 @@ substituters = https://cache.nixos.org/ s3://example-nix-cache
 trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= example-nix-cache-1:1/cKDz3QCCOmwcztD2eV6Coggp6rqc9DGjWv7C0G+rM=
 </programlisting>
 
-<para>We will restart the Nix daemon in a later step.</para>
+<para>we will restart the Nix daemon a later step.</para>
 </section>
 
 <section>
@@ -122,7 +122,7 @@ post-build-hook = /etc/nix/upload-to-cache.sh
 
   <screen>
 $ nix-build -E '(import &lt;nixpkgs&gt; {}).writeText "example" (builtins.toString builtins.currentTime)'
-this derivation will be built:
+these derivations will be built:
   /nix/store/s4pnfbkalzy5qz57qs6yybna8wylkig6-example.drv
 building '/nix/store/s4pnfbkalzy5qz57qs6yybna8wylkig6-example.drv'...
 running post-build-hook '/home/grahamc/projects/github.com/NixOS/nix/post-hook.sh'...
@@ -139,7 +139,7 @@ $ nix-store --delete /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example
 
 <para>Now, copy the path back from the cache:</para>
 <screen>
-$ nix-store --realise /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example
+$ nix store --realize /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example
 copying path '/nix/store/m8bmqwrch6l3h8s0k3d673xpmipcdpsa-example from 's3://example-nix-cache'...
 warning: you did not specify '--add-root'; the result might be removed by the garbage collector
 /nix/store/m8bmqwrch6l3h8s0k3d673xpmipcdpsa-example

doc/manual/command-ref/conf-file.xml

@@ -19,30 +19,26 @@
 
 <refsection><title>Description</title>
 
-<para>By default Nix reads settings from the following places:</para>
+<para>Nix reads settings from two configuration files:</para>
 
-<para>The system-wide configuration file
-<filename><replaceable>sysconfdir</replaceable>/nix/nix.conf</filename>
-(i.e. <filename>/etc/nix/nix.conf</filename> on most systems), or
-<filename>$NIX_CONF_DIR/nix.conf</filename> if
-<envar>NIX_CONF_DIR</envar> is set. Values loaded in this file are not forwarded to the Nix daemon. The
-client assumes that the daemon has already loaded them.
-</para>
+<itemizedlist>
 
-<para>User-specific configuration files:</para>
+  <listitem>
+    <para>The system-wide configuration file
+    <filename><replaceable>sysconfdir</replaceable>/nix/nix.conf</filename>
+    (i.e. <filename>/etc/nix/nix.conf</filename> on most systems), or
+    <filename>$NIX_CONF_DIR/nix.conf</filename> if
+    <envar>NIX_CONF_DIR</envar> is set.</para>
+  </listitem>
 
-<para>
-  If <envar>NIX_USER_CONF_FILES</envar> is set, then each path separated by
-  <literal>:</literal> will be loaded in reverse order.
-</para>
+  <listitem>
+    <para>The user configuration file
+    <filename>$XDG_CONFIG_HOME/nix/nix.conf</filename>, or
+    <filename>~/.config/nix/nix.conf</filename> if
+    <envar>XDG_CONFIG_HOME</envar> is not set.</para>
+  </listitem>
 
-<para>
-  Otherwise it will look for <filename>nix/nix.conf</filename> files in
-  <envar>XDG_CONFIG_DIRS</envar> and <envar>XDG_CONFIG_HOME</envar>.
-
-  The default location is <filename>$HOME/.config/nix.conf</filename> if
-  those environment variables are unset.
-</para>
+</itemizedlist>
 
 <para>The configuration files consist of
 <literal><replaceable>name</replaceable> =
@@ -386,7 +382,7 @@ false</literal>.</para>
 
 <programlisting>
 builtins.fetchurl {
-  url = "https://example.org/foo-1.2.3.tar.xz";
+  url = https://example.org/foo-1.2.3.tar.xz;
   sha256 = "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae";
 }
 </programlisting>

doc/manual/command-ref/env-common.xml

@@ -33,7 +33,7 @@
 
     will cause Nix to look for paths relative to
     <filename>/home/eelco/Dev</filename> and
-    <filename>/etc/nixos</filename>, in this order.  It is also
+    <filename>/etc/nixos</filename>, in that order.  It is also
     possible to match paths against a prefix.  For example, the value
 
     <screen>
@@ -53,13 +53,13 @@ nixpkgs=/home/eelco/Dev/nixpkgs-branch:/etc/nixos</screen>
     <envar>NIX_PATH</envar> to
 
     <screen>
-nixpkgs=https://github.com/NixOS/nixpkgs/archive/nixos-15.09.tar.gz</screen>
+nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-15.09.tar.gz</screen>
 
     tells Nix to download the latest revision in the Nixpkgs/NixOS
     15.09 channel.</para>
 
     <para>A following shorthand can be used to refer to the official channels:
-
+    
     <screen>nixpkgs=channel:nixos-15.09</screen>
     </para>
 
@@ -137,19 +137,12 @@ $ mount -o bind /mnt/otherdisk/nix /nix</screen>
 
 <varlistentry><term><envar>NIX_CONF_DIR</envar></term>
 
-  <listitem><para>Overrides the location of the system Nix configuration
+  <listitem><para>Overrides the location of the Nix configuration
   directory (default
   <filename><replaceable>prefix</replaceable>/etc/nix</filename>).</para></listitem>
 
 </varlistentry>
 
-<varlistentry><term><envar>NIX_USER_CONF_FILES</envar></term>
-
-  <listitem><para>Overrides the location of the user Nix configuration files
-  to load from (defaults to the XDG spec locations). The variable is treated
-  as a list separated by the <literal>:</literal> token.</para></listitem>
-
-</varlistentry>
 
 <varlistentry><term><envar>TMPDIR</envar></term>
 

doc/manual/command-ref/nix-env.xml

@@ -516,7 +516,7 @@ source:
 $ nix-env -f '<nixpkgs>' -iA hello --dry-run
 (dry run; not doing anything)
 installing ‘hello-2.10’
-this path will be fetched (0.04 MiB download, 0.19 MiB unpacked):
+these paths will be fetched (0.04 MiB download, 0.19 MiB unpacked):
   /nix/store/wkhdf9jinag5750mqlax6z2zbwhqb76n-hello-2.10
   <replaceable>...</replaceable></screen>
 
@@ -526,10 +526,13 @@ this path will be fetched (0.04 MiB download, 0.19 MiB unpacked):
 14.12 channel:
 
 <screen>
-$ nix-env -f https://github.com/NixOS/nixpkgs/archive/nixos-14.12.tar.gz -iA firefox
+$ nix-env -f https://github.com/NixOS/nixpkgs-channels/archive/nixos-14.12.tar.gz -iA firefox
 </screen>
 
-</para>
+(The GitHub repository <literal>nixpkgs-channels</literal> is updated
+automatically from the main <literal>nixpkgs</literal> repository
+after certain tests have succeeded and binaries have been built and
+uploaded to the binary cache at <uri>cache.nixos.org</uri>.)</para>
 
 </refsection>
 

doc/manual/command-ref/nix-shell.xml

@@ -258,7 +258,7 @@ path. You can override it by passing <option>-I</option> or setting
 containing the Pan package from a specific revision of Nixpkgs:
 
 <screen>
-$ nix-shell -p pan -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/8a3eea054838b55aca962c3fbde9c83c102b8bf2.tar.gz
+$ nix-shell -p pan -I nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/8a3eea054838b55aca962c3fbde9c83c102b8bf2.tar.gz
 
 [nix-shell:~]$ pan --version
 Pan 0.139
@@ -352,7 +352,7 @@ following Haskell script uses a specific branch of Nixpkgs/NixOS (the
 <programlisting><![CDATA[
 #! /usr/bin/env nix-shell
 #! nix-shell -i runghc -p "haskellPackages.ghcWithPackages (ps: [ps.HTTP ps.tagsoup])"
-#! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/nixos-18.03.tar.gz
+#! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-18.03.tar.gz
 
 import Network.HTTP
 import Text.HTML.TagSoup
@@ -370,7 +370,7 @@ If you want to be even more precise, you can specify a specific
 revision of Nixpkgs:
 
 <programlisting>
-#! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/0672315759b3e15e2121365f067c1c8c56bb4722.tar.gz
+#! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/0672315759b3e15e2121365f067c1c8c56bb4722.tar.gz
 </programlisting>
 
 </para>

doc/manual/command-ref/nix-store.xml

@@ -936,7 +936,7 @@ $ nix-store --add ./foo.c
 
 <para>The operation <option>--add-fixed</option> adds the specified paths to
 the Nix store.  Unlike <option>--add</option> paths are registered using the
-specified hashing algorithm, resulting in the same output path as a fixed-output
+specified hashing algorithm, resulting in the same output path as a fixed output
 derivation.  This can be used for sources that are not available from a public
 url or broke since the download expression was written.
 </para>

doc/manual/command-ref/opt-common-syn.xml

@@ -1,5 +1,5 @@
 <nop xmlns="http://docbook.org/ns/docbook">
-
+  
 <arg><option>--help</option></arg>
 <arg><option>--version</option></arg>
 <arg rep='repeat'>
@@ -11,10 +11,6 @@
 <arg>
   <arg choice='plain'><option>--quiet</option></arg>
 </arg>
-<arg>
-  <option>--log-format</option>
-  <replaceable>format</replaceable>
-</arg>
 <arg>
   <group choice='plain'>
     <arg choice='plain'><option>--no-build-output</option></arg>

doc/manual/command-ref/opt-common.xml

@@ -92,37 +92,6 @@
 </varlistentry>
 
 
-<varlistentry xml:id="opt-log-format"><term><option>--log-format</option> <replaceable>format</replaceable></term>
-
-  <listitem>
-
-  <para>This option can be used to change the output of the log format, with
-  <replaceable>format</replaceable> being one of:</para>
-
-  <variablelist>
-
-    <varlistentry><term>raw</term>
-    <listitem><para>This is the raw format, as outputted by nix-build.</para></listitem>
-    </varlistentry>
-
-    <varlistentry><term>internal-json</term>
-    <listitem><para>Outputs the logs in a structured manner. NOTE: the json schema is not guarantees to be stable between releases.</para></listitem>
-    </varlistentry>
-
-    <varlistentry><term>bar</term>
-    <listitem><para>Only display a progress bar during the builds.</para></listitem>
-    </varlistentry>
-
-    <varlistentry><term>bar-with-logs</term>
-    <listitem><para>Display the raw logs, with the progress bar at the bottom.</para></listitem>
-    </varlistentry>
-
-  </variablelist>
-
-  </listitem>
-
-</varlistentry>
-
 <varlistentry><term><option>--no-build-output</option> / <option>-Q</option></term>
 
   <listitem><para>By default, output written by builders to standard

doc/manual/expressions/advanced-attributes.xml

@@ -178,7 +178,7 @@ impureEnvVars = [ "http_proxy" "https_proxy" <replaceable>...</replaceable> ];
 
 <programlisting>
 fetchurl {
-  url = "http://ftp.gnu.org/pub/gnu/hello/hello-2.1.1.tar.gz";
+  url = http://ftp.gnu.org/pub/gnu/hello/hello-2.1.1.tar.gz;
   sha256 = "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465";
 }
 </programlisting>
@@ -189,7 +189,7 @@ fetchurl {
 
 <programlisting>
 fetchurl {
-  url = "ftp://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz";
+  url = ftp://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz;
   sha256 = "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465";
 }
 </programlisting>

doc/manual/expressions/builtins.xml

@@ -324,7 +324,7 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting>
     particular version of Nixpkgs, e.g.
 
 <programlisting>
-with import (fetchTarball https://github.com/NixOS/nixpkgs/archive/nixos-14.12.tar.gz) {};
+with import (fetchTarball https://github.com/NixOS/nixpkgs-channels/archive/nixos-14.12.tar.gz) {};
 
 stdenv.mkDerivation { … }
 </programlisting>
@@ -349,7 +349,7 @@ stdenv.mkDerivation { … }
 
 <programlisting>
 with import (fetchTarball {
-  url = "https://github.com/NixOS/nixpkgs/archive/nixos-14.12.tar.gz";
+  url = https://github.com/NixOS/nixpkgs-channels/archive/nixos-14.12.tar.gz;
   sha256 = "1jppksrfvbk5ypiqdz4cddxdl8z6zyzdb2srq8fcffr327ld5jj2";
 }) {};
 
@@ -422,16 +422,6 @@ stdenv.mkDerivation { … }
             </para>
           </listitem>
         </varlistentry>
-        <varlistentry>
-          <term>submodules</term>
-          <listitem>
-            <para>
-              A Boolean parameter that specifies whether submodules
-              should be checked out. Defaults to
-              <literal>false</literal>.
-            </para>
-          </listitem>
-        </varlistentry>
       </variablelist>
 
       <example>
@@ -1406,7 +1396,7 @@ stdenv.mkDerivation {
   ";
 
   src = fetchurl {
-    url = "http://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz";
+    url = http://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz;
     sha256 = "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465";
   };
   inherit perl;

doc/manual/expressions/expression-syntax.xml

@@ -15,7 +15,7 @@ stdenv.mkDerivation { <co xml:id='ex-hello-nix-co-2' />
   name = "hello-2.1.1"; <co xml:id='ex-hello-nix-co-3' />
   builder = ./builder.sh; <co xml:id='ex-hello-nix-co-4' />
   src = fetchurl { <co xml:id='ex-hello-nix-co-5' />
-    url = "ftp://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz";
+    url = ftp://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz;
     sha256 = "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465";
   };
   inherit perl; <co xml:id='ex-hello-nix-co-6' />

doc/manual/expressions/simple-building-testing.xml

@@ -73,4 +73,12 @@ waiting for lock on `/nix/store/0h5b7hp8d4hqfrw8igvx97x1xawrjnac-hello-2.1.1x'</
 So it is always safe to run multiple instances of Nix in parallel
 (which isn’t the case with, say, <command>make</command>).</para>
 
+<para>If you have a system with multiple CPUs, you may want to have
+Nix build different derivations in parallel (insofar as possible).
+Just pass the option <link linkend='opt-max-jobs'><option>-j
+<replaceable>N</replaceable></option></link>, where
+<replaceable>N</replaceable> is the maximum number of jobs to be run
+in parallel, or set.  Typically this should be the number of
+CPUs.</para>
+
 </section>

doc/manual/installation/env-variables.xml

@@ -39,7 +39,7 @@ bundle.</para>
   <step><para>Set the environment variable and install Nix</para>
     <screen>
 $ export NIX_SSL_CERT_FILE=/etc/ssl/my-certificate-bundle.crt
-$ sh &lt;(curl -L https://nixos.org/nix/install)
+$ sh &lt;(curl https://nixos.org/nix/install)
 </screen></step>
 
   <step><para>In the shell profile and rc files (for example,

doc/manual/installation/installing-binary.xml

@@ -6,30 +6,16 @@
 
 <title>Installing a Binary Distribution</title>
 
-<para>
-  If you are using Linux or macOS versions up to 10.14 (Mojave), the
-  easiest way to install Nix is to run the following command:
-</para>
+<para>If you are using Linux or macOS, the easiest way to install Nix
+is to run the following command:
 
 <screen>
-  $ sh &lt;(curl -L https://nixos.org/nix/install)
+  $ sh &lt;(curl https://nixos.org/nix/install)
 </screen>
 
-<para>
-  If you're using macOS 10.15 (Catalina) or newer, consult
-  <link linkend="sect-macos-installation">the macOS installation instructions</link>
-  before installing.
-</para>
-
-<para>
-  As of Nix 2.1.0, the Nix installer will always default to creating a
-  single-user installation, however opting in to the multi-user
-  installation is highly recommended.
-  <!-- TODO: this explains *neither* why the default version is
-  single-user, nor why we'd recommend multi-user over the default.
-  True prospective users don't have much basis for evaluating this.
-  What's it to me? Who should pick which? Why? What if I pick wrong?
-  -->
+As of Nix 2.1.0, the Nix installer will always default to creating a
+single-user installation, however opting in to the multi-user
+installation is highly recommended.
 </para>
 
 <section xml:id="sect-single-user-installation">
@@ -39,7 +25,7 @@
     To explicitly select a single-user installation on your system:
 
     <screen>
-  sh &lt;(curl -L https://nixos.org/nix/install) --no-daemon
+  sh &lt;(curl https://nixos.org/nix/install) --no-daemon
 </screen>
   </para>
 
@@ -50,7 +36,7 @@ run this under your usual user account, <emphasis>not</emphasis> as
 root.  The script will invoke <command>sudo</command> to create
 <filename>/nix</filename> if it doesn’t already exist.  If you don’t
 have <command>sudo</command>, you should manually create
-<filename>/nix</filename> first as root, e.g.:
+<command>/nix</command> first as root, e.g.:
 
 <screen>
 $ mkdir /nix
@@ -61,7 +47,7 @@ The install script will modify the first writable file from amongst
 <filename>.bash_profile</filename>, <filename>.bash_login</filename>
 and <filename>.profile</filename> to source
 <filename>~/.nix-profile/etc/profile.d/nix.sh</filename>. You can set
-the <envar>NIX_INSTALLER_NO_MODIFY_PROFILE</envar> environment
+the <command>NIX_INSTALLER_NO_MODIFY_PROFILE</command> environment
 variable before executing the install script to disable this
 behaviour.
 </para>
@@ -95,9 +81,11 @@ $ rm -rf /nix
   <para>
     You can instruct the installer to perform a multi-user
     installation on your system:
-  </para>
 
-  <screen>sh &lt;(curl -L https://nixos.org/nix/install) --daemon</screen>
+    <screen>
+  sh &lt;(curl https://nixos.org/nix/install) --daemon
+</screen>
+  </para>
 
   <para>
     The multi-user installation of Nix will create build users between
@@ -148,273 +136,6 @@ sudo rm /Library/LaunchDaemons/org.nixos.nix-daemon.plist
 
 </section>
 
-<section xml:id="sect-macos-installation">
-  <title>macOS Installation</title>
-
-  <para>
-    Starting with macOS 10.15 (Catalina), the root filesystem is read-only.
-    This means <filename>/nix</filename> can no longer live on your system
-    volume, and that you'll need a workaround to install Nix.
-  </para>
-
-  <para>
-    The recommended approach, which creates an unencrypted APFS volume
-    for your Nix store and a "synthetic" empty directory to mount it
-    over at <filename>/nix</filename>, is least likely to impair Nix
-    or your system.
-  </para>
-
-  <note><para>
-    With all separate-volume approaches, it's possible something on
-    your system (particularly daemons/services and restored apps) may
-    need access to your Nix store before the volume is mounted. Adding
-    additional encryption makes this more likely.
-  </para></note>
-
-  <para>
-    If you're using a recent Mac with a
-    <link xlink:href="https://www.apple.com/euro/mac/shared/docs/Apple_T2_Security_Chip_Overview.pdf">T2 chip</link>,
-    your drive will still be encrypted at rest (in which case "unencrypted"
-    is a bit of a misnomer). To use this approach, just install Nix with:
-  </para>
-
-  <screen>$ sh &lt;(curl -L https://nixos.org/nix/install) --darwin-use-unencrypted-nix-store-volume</screen>
-
-  <para>
-    If you don't like the sound of this, you'll want to weigh the
-    other approaches and tradeoffs detailed in this section.
-  </para>
-
-  <note>
-    <title>Eventual solutions?</title>
-    <para>
-      All of the known workarounds have drawbacks, but we hope
-      better solutions will be available in the future. Some that
-      we have our eye on are:
-    </para>
-    <orderedlist>
-      <listitem>
-        <para>
-          A true firmlink would enable the Nix store to live on the
-          primary data volume without the build problems caused by
-          the symlink approach. End users cannot currently
-          create true firmlinks.
-        </para>
-      </listitem>
-      <listitem>
-        <para>
-          If the Nix store volume shared FileVault encryption
-          with the primary data volume (probably by using the same
-          volume group and role), FileVault encryption could be
-          easily supported by the installer without requiring
-          manual setup by each user.
-        </para>
-      </listitem>
-    </orderedlist>
-  </note>
-
-  <section xml:id="sect-macos-installation-change-store-prefix">
-    <title>Change the Nix store path prefix</title>
-    <para>
-      Changing the default prefix for the Nix store is a simple
-      approach which enables you to leave it on your root volume,
-      where it can take full advantage of FileVault encryption if
-      enabled. Unfortunately, this approach also opts your device out
-      of some benefits that are enabled by using the same prefix
-      across systems:
-
-      <itemizedlist>
-        <listitem>
-          <para>
-            Your system won't be able to take advantage of the binary
-            cache (unless someone is able to stand up and support
-            duplicate caching infrastructure), which means you'll
-            spend more time waiting for builds.
-          </para>
-        </listitem>
-        <listitem>
-          <para>
-            It's harder to build and deploy packages to Linux systems.
-          </para>
-        </listitem>
-        <!-- TODO: may be more here -->
-      </itemizedlist>
-
-      <!-- TODO: Yes, but how?! -->
-
-      It would also possible (and often requested) to just apply this
-      change ecosystem-wide, but it's an intrusive process that has
-      side effects we want to avoid for now.
-      <!-- magnificent hand-wavy gesture -->
-    </para>
-    <para>
-    </para>
-  </section>
-
-  <section xml:id="sect-macos-installation-encrypted-volume">
-    <title>Use a separate encrypted volume</title>
-    <para>
-      If you like, you can also add encryption to the recommended
-      approach taken by the installer. You can do this by pre-creating
-      an encrypted volume before you run the installer--or you can
-      run the installer and encrypt the volume it creates later.
-      <!-- TODO: see later note about whether this needs both add-encryption and from-scratch directions -->
-    </para>
-    <para>
-      In either case, adding encryption to a second volume isn't quite
-      as simple as enabling FileVault for your boot volume. Before you
-      dive in, there are a few things to weigh:
-    </para>
-    <orderedlist>
-      <listitem>
-        <para>
-          The additional volume won't be encrypted with your existing
-          FileVault key, so you'll need another mechanism to decrypt
-          the volume.
-        </para>
-      </listitem>
-      <listitem>
-        <para>
-          You can store the password in Keychain to automatically
-          decrypt the volume on boot--but it'll have to wait on Keychain
-          and may not mount before your GUI apps restore. If any of
-          your launchd agents or apps depend on Nix-installed software
-          (for example, if you use a Nix-installed login shell), the
-          restore may fail or break.
-        </para>
-        <para>
-          On a case-by-case basis, you may be able to work around this
-          problem by using <command>wait4path</command> to block
-          execution until your executable is available.
-        </para>
-        <para>
-          It's also possible to decrypt and mount the volume earlier
-          with a login hook--but this mechanism appears to be
-          deprecated and its future is unclear.
-        </para>
-      </listitem>
-      <listitem>
-        <para>
-          You can hard-code the password in the clear, so that your
-          store volume can be decrypted before Keychain is available.
-        </para>
-      </listitem>
-    </orderedlist>
-    <para>
-      If you are comfortable navigating these tradeoffs, you can encrypt the volume with
-      something along the lines of:
-      <!-- TODO:
-      I don't know if this also needs from-scratch instructions?
-      can we just recommend use-the-installer-and-then-encrypt?
-      -->
-    </para>
-    <!--
-    TODO: it looks like this option can be encryptVolume|encrypt|enableFileVault
-
-    It may be more clear to use encryptVolume, here? FileVault seems
-    heavily associated with the boot-volume behavior; I worry
-    a little that it can mislead here, especially as it gets
-    copied around minus doc context...?
-    -->
-    <screen>alice$ diskutil apfs enableFileVault /nix -user disk</screen>
-
-    <!-- TODO: and then go into detail on the mount/decrypt approaches? -->
-  </section>
-
-  <section xml:id="sect-macos-installation-symlink">
-    <!--
-    Maybe a good razor is: if we'd hate having to support someone who
-    installed Nix this way, it shouldn't even be detailed?
-    -->
-    <title>Symlink the Nix store to a custom location</title>
-    <para>
-      Another simple approach is using <filename>/etc/synthetic.conf</filename>
-      to symlink the Nix store to the data volume. This option also
-      enables your store to share any configured FileVault encryption.
-      Unfortunately, builds that resolve the symlink may leak the
-      canonical path or even fail.
-    </para>
-    <para>
-      Because of these downsides, we can't recommend this approach.
-    </para>
-    <!-- Leaving out instructions for this one. -->
-  </section>
-
-  <section xml:id="sect-macos-installation-recommended-notes">
-    <title>Notes on the recommended approach</title>
-    <para>
-      This section goes into a little more detail on the recommended
-      approach. You don't need to understand it to run the installer,
-      but it can serve as a helpful reference if you run into trouble.
-    </para>
-    <orderedlist>
-      <listitem>
-        <para>
-          In order to compose user-writable locations into the new
-          read-only system root, Apple introduced a new concept called
-          <literal>firmlinks</literal>, which it describes as a
-          "bi-directional wormhole" between two filesystems. You can
-          see the current firmlinks in <filename>/usr/share/firmlinks</filename>.
-          Unfortunately, firmlinks aren't (currently?) user-configurable.
-        </para>
-
-        <para>
-          For special cases like NFS mount points or package manager roots,
-          <link xlink:href="https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man5/synthetic.conf.5.html">synthetic.conf(5)</link>
-          supports limited user-controlled file-creation (of symlinks,
-          and synthetic empty directories) at <filename>/</filename>.
-          To create a synthetic empty directory for mounting at <filename>/nix</filename>,
-          add the following line to <filename>/etc/synthetic.conf</filename>
-          (create it if necessary):
-        </para>
-
-        <screen>nix</screen>
-      </listitem>
-
-      <listitem>
-        <para>
-          This configuration is applied at boot time, but you can use
-          <command>apfs.util</command> to trigger creation (not deletion)
-          of new entries without a reboot:
-        </para>
-
-        <screen>alice$ /System/Library/Filesystems/apfs.fs/Contents/Resources/apfs.util -B</screen>
-      </listitem>
-
-      <listitem>
-        <para>
-          Create the new APFS volume with diskutil:
-        </para>
-
-        <screen>alice$ sudo diskutil apfs addVolume diskX APFS 'Nix Store' -mountpoint /nix</screen>
-      </listitem>
-
-      <listitem>
-        <para>
-          Using <command>vifs</command>, add the new mount to
-          <filename>/etc/fstab</filename>. If it doesn't already have
-          other entries, it should look something like:
-        </para>
-
-<screen>
-#
-# Warning - this file should only be modified with vifs(8)
-#
-# Failure to do so is unsupported and may be destructive.
-#
-LABEL=Nix\040Store /nix apfs rw,nobrowse
-</screen>
-
-        <para>
-          The nobrowse setting will keep Spotlight from indexing this
-          volume, and keep it from showing up on your desktop.
-        </para>
-      </listitem>
-    </orderedlist>
-  </section>
-
-</section>
-
 <section xml:id="sect-nix-install-pinned-version-url">
   <title>Installing a pinned Nix version from a URL</title>
 
@@ -429,7 +150,7 @@ LABEL=Nix\040Store /nix apfs rw,nobrowse
   NixOS.org installation script:
 
   <screen>
-  sh &lt;(curl -L https://nixos.org/nix/install)
+  sh &lt;(curl https://nixos.org/nix/install)
 </screen>
   </para>
 

doc/manual/installation/upgrading.xml

@@ -17,11 +17,6 @@
 
   <para>
     Single-user installations of Nix should run this:
-    <command>nix-channel --update; nix-env -iA nixpkgs.nix nixpkgs.cacert</command>
-  </para>
-      
-  <para>
-    Multi-user Nix users on Linux should run this with sudo:
-    <command>nix-channel --update; nix-env -iA nixpkgs.nix nixpkgs.cacert; systemctl daemon-reload; systemctl restart nix-daemon</command>
+    <command>nix-channel --update; nix-env -iA nixpkgs.nix</command>
   </para>
 </chapter>

doc/manual/introduction/quick-start.xml

@@ -15,7 +15,7 @@ to subsequent chapters.</para>
 <step><para>Install single-user Nix by running the following:
 
 <screen>
-$ bash &lt;(curl -L https://nixos.org/nix/install)
+$ bash &lt;(curl https://nixos.org/nix/install)
 </screen>
 
 This will install Nix in <filename>/nix</filename>. The install script

doc/manual/release-notes/rl-0.8.xml

@@ -8,7 +8,7 @@
 
 <para>NOTE: the hashing scheme in Nix 0.8 changed (as detailed below).
 As a result, <command>nix-pull</command> manifests and channels built
-for Nix 0.7 and below will not work anymore.  However, the Nix
+for Nix 0.7 and below will now work anymore.  However, the Nix
 expression language has not changed, so you can still build from
 source.  Also, existing user environments continue to work.  Nix 0.8
 will automatically upgrade the database schema of previous

maintainers/upload-release.pl

@@ -1,5 +1,5 @@
 #! /usr/bin/env nix-shell
-#! nix-shell -i perl -p perl perlPackages.LWPUserAgent perlPackages.LWPProtocolHttps perlPackages.FileSlurp perlPackages.NetAmazonS3 gnupg1
+#! nix-shell -i perl -p perl perlPackages.LWPUserAgent perlPackages.LWPProtocolHttps perlPackages.FileSlurp gnupg1
 
 use strict;
 use Data::Dumper;
@@ -9,16 +9,12 @@ use File::Slurp;
 use File::Copy;
 use JSON::PP;
 use LWP::UserAgent;
-use Net::Amazon::S3;
 
 my $evalId = $ARGV[0] or die "Usage: $0 EVAL-ID\n";
 
-my $releasesBucketName = "nix-releases";
-my $channelsBucketName = "nix-channels";
+my $releasesDir = "/home/eelco/mnt/releases";
 my $nixpkgsDir = "/home/eelco/Dev/nixpkgs-pristine";
 
-my $TMPDIR = $ENV{'TMPDIR'} // "/tmp";
-
 # FIXME: cut&paste from nixos-channel-scripts.
 sub fetch {
     my ($url, $type) = @_;
@@ -46,31 +42,13 @@ my $version = $1;
 
 print STDERR "Nix revision is $nixRev, version is $version\n";
 
-my $releaseDir = "nix/$releaseName";
+File::Path::make_path($releasesDir);
+if (system("mountpoint -q $releasesDir") != 0) {
+    system("sshfs hydra-mirror\@nixos.org:/releases $releasesDir") == 0 or die;
+}
 
-my $tmpDir = "$TMPDIR/nix-release/$releaseName";
-File::Path::make_path($tmpDir);
-
-# S3 setup.
-my $aws_access_key_id = $ENV{'AWS_ACCESS_KEY_ID'} or die "No AWS_ACCESS_KEY_ID given.";
-my $aws_secret_access_key = $ENV{'AWS_SECRET_ACCESS_KEY'} or die "No AWS_SECRET_ACCESS_KEY given.";
-
-my $s3 = Net::Amazon::S3->new(
-    { aws_access_key_id     => $aws_access_key_id,
-      aws_secret_access_key => $aws_secret_access_key,
-      retry                 => 1,
-      host                  => "s3-eu-west-1.amazonaws.com",
-    });
-
-my $releasesBucket = $s3->bucket($releasesBucketName) or die;
-
-my $s3_us = Net::Amazon::S3->new(
-    { aws_access_key_id     => $aws_access_key_id,
-      aws_secret_access_key => $aws_secret_access_key,
-      retry                 => 1,
-    });
-
-my $channelsBucket = $s3_us->bucket($channelsBucketName) or die;
+my $releaseDir = "$releasesDir/nix/$releaseName";
+File::Path::make_path($releaseDir);
 
 sub downloadFile {
     my ($jobName, $productNr, $dstName) = @_;
@@ -79,49 +57,40 @@ sub downloadFile {
 
     my $srcFile = $buildInfo->{buildproducts}->{$productNr}->{path} or die "job '$jobName' lacks product $productNr\n";
     $dstName //= basename($srcFile);
-    my $tmpFile = "$tmpDir/$dstName";
+    my $dstFile = "$releaseDir/" . $dstName;
 
-    if (!-e $tmpFile) {
-        print STDERR "downloading $srcFile to $tmpFile...\n";
-        system("NIX_REMOTE=https://cache.nixos.org/ nix cat-store '$srcFile' > '$tmpFile'") == 0
+    if (! -e $dstFile) {
+        print STDERR "downloading $srcFile to $dstFile...\n";
+        system("NIX_REMOTE=https://cache.nixos.org/ nix cat-store '$srcFile' > '$dstFile.tmp'") == 0
             or die "unable to fetch $srcFile\n";
+        rename("$dstFile.tmp", $dstFile) or die;
     }
 
     my $sha256_expected = $buildInfo->{buildproducts}->{$productNr}->{sha256hash} or die;
-    my $sha256_actual = `nix hash-file --base16 --type sha256 '$tmpFile'`;
+    my $sha256_actual = `nix hash-file --base16 --type sha256 '$dstFile'`;
     chomp $sha256_actual;
     if ($sha256_expected ne $sha256_actual) {
-        print STDERR "file $tmpFile is corrupt, got $sha256_actual, expected $sha256_expected\n";
+        print STDERR "file $dstFile is corrupt, got $sha256_actual, expected $sha256_expected\n";
         exit 1;
     }
 
-    write_file("$tmpFile.sha256", $sha256_expected);
+    write_file("$dstFile.sha256", $sha256_expected);
 
-    if (! -e "$tmpFile.asc") {
-        system("gpg2 --detach-sign --armor $tmpFile") == 0 or die "unable to sign $tmpFile\n";
+    if (! -e "$dstFile.asc") {
+        system("gpg2 --detach-sign --armor $dstFile") == 0 or die "unable to sign $dstFile\n";
     }
 
-    return $sha256_expected;
+    return ($dstFile, $sha256_expected);
 }
 
 downloadFile("tarball", "2"); # .tar.bz2
-my $tarballHash = downloadFile("tarball", "3"); # .tar.xz
+my ($tarball, $tarballHash) = downloadFile("tarball", "3"); # .tar.xz
 downloadFile("binaryTarball.i686-linux", "1");
 downloadFile("binaryTarball.x86_64-linux", "1");
 downloadFile("binaryTarball.aarch64-linux", "1");
 downloadFile("binaryTarball.x86_64-darwin", "1");
 downloadFile("installerScript", "1");
 
-for my $fn (glob "$tmpDir/*") {
-    my $name = basename($fn);
-    my $dstKey = "$releaseDir/" . $name;
-    unless (defined $releasesBucket->head_key($dstKey)) {
-        print STDERR "uploading $fn to s3://$releasesBucketName/$dstKey...\n";
-        $releasesBucket->add_key_filename($dstKey, $fn)
-            or die $releasesBucket->err . ": " . $releasesBucket->errstr;
-    }
-}
-
 exit if $version =~ /pre/;
 
 # Update Nixpkgs in a very hacky way.
@@ -142,12 +111,8 @@ $oldName =~ s/"//g;
 sub getStorePath {
     my ($jobName) = @_;
     my $buildInfo = decode_json(fetch("$evalUrl/job/$jobName", 'application/json'));
-    for my $product (values %{$buildInfo->{buildproducts}}) {
-        next unless $product->{type} eq "nix-build";
-        next if $product->{path} =~ /[a-z]+$/;
-        return $product->{path};
-    }
-    die;
+    die unless $buildInfo->{buildproducts}->{1}->{type} eq "nix-build";
+    return $buildInfo->{buildproducts}->{1}->{path};
 }
 
 write_file("$nixpkgsDir/nixos/modules/installer/tools/nix-fallback-paths.nix",
@@ -160,15 +125,32 @@ write_file("$nixpkgsDir/nixos/modules/installer/tools/nix-fallback-paths.nix",
 
 system("cd $nixpkgsDir && git commit -a -m 'nix: $oldName -> $version'") == 0 or die;
 
+# Extract the HTML manual.
+File::Path::make_path("$releaseDir/manual");
+
+system("tar xvf $tarball --strip-components=3 -C $releaseDir/manual --wildcards '*/doc/manual/*.html' '*/doc/manual/*.css' '*/doc/manual/*.gif' '*/doc/manual/*.png'") == 0 or die;
+
+if (! -e "$releaseDir/manual/index.html") {
+    symlink("manual.html", "$releaseDir/manual/index.html") or die;
+}
+
 # Update the "latest" symlink.
-$channelsBucket->add_key(
-    "nix-latest/install", "",
-    { "x-amz-website-redirect-location" => "https://releases.nixos.org/$releaseDir/install" })
-    or die $channelsBucket->err . ": " . $channelsBucket->errstr;
+symlink("$releaseName", "$releasesDir/nix/latest-tmp") or die;
+rename("$releasesDir/nix/latest-tmp", "$releasesDir/nix/latest") or die;
 
 # Tag the release in Git.
 chdir("/home/eelco/Dev/nix-pristine") or die;
 system("git remote update origin") == 0 or die;
 system("git tag --force --sign $version $nixRev -m 'Tagging release $version'") == 0 or die;
-system("git push --tags") == 0 or die;
-system("git push --force-with-lease origin $nixRev:refs/heads/latest-release") == 0 or die;
+
+# Update the website.
+my $siteDir = "/home/eelco/Dev/nixos-homepage-pristine";
+
+system("cd $siteDir && git pull") == 0 or die;
+
+write_file("$siteDir/nix-release.tt",
+           "[%-\n" .
+           "latestNixVersion = \"$version\"\n" .
+           "-%]\n");
+
+system("cd $siteDir && git commit -a -m 'Nix $version released'") == 0 or die;

mk/libraries.mk

@@ -125,8 +125,7 @@ define build-library
     $(1)_PATH := $$(_d)/$$($(1)_NAME).a
 
     $$($(1)_PATH): $$($(1)_OBJS) | $$(_d)/
-	$(trace-ld) $(LD) -Ur -o $$(_d)/$$($(1)_NAME).o $$?
-	$(trace-ar) $(AR) crs $$@ $$(_d)/$$($(1)_NAME).o
+	$(trace-ar) $(AR) crs $$@ $$?
 
     $(1)_LDFLAGS_USE += $$($(1)_PATH) $$($(1)_LDFLAGS)
 

mk/programs.mk

@@ -35,28 +35,24 @@ define build-program
 	$$(trace-ld) $(CXX) -o $$@ $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE))
 
   $(1)_INSTALL_DIR ?= $$(bindir)
+  $(1)_INSTALL_PATH := $$($(1)_INSTALL_DIR)/$(1)
 
-  ifdef $(1)_INSTALL_DIR
+  $$(eval $$(call create-dir, $$($(1)_INSTALL_DIR)))
 
-    $(1)_INSTALL_PATH := $$($(1)_INSTALL_DIR)/$(1)
+  install: $(DESTDIR)$$($(1)_INSTALL_PATH)
 
-    $$(eval $$(call create-dir, $$($(1)_INSTALL_DIR)))
+  ifeq ($(BUILD_SHARED_LIBS), 1)
 
-    install: $(DESTDIR)$$($(1)_INSTALL_PATH)
+    _libs_final := $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_INSTALL_PATH))
 
-    ifeq ($(BUILD_SHARED_LIBS), 1)
-
-      _libs_final := $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_INSTALL_PATH))
-
-      $(DESTDIR)$$($(1)_INSTALL_PATH): $$($(1)_OBJS) $$(_libs_final) | $(DESTDIR)$$($(1)_INSTALL_DIR)/
+    $(DESTDIR)$$($(1)_INSTALL_PATH): $$($(1)_OBJS) $$(_libs_final) | $(DESTDIR)$$($(1)_INSTALL_DIR)/
 	$$(trace-ld) $(CXX) -o $$@ $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE_INSTALLED))
 
-    else
+  else
 
-      $(DESTDIR)$$($(1)_INSTALL_PATH): $$($(1)_PATH) | $(DESTDIR)$$($(1)_INSTALL_DIR)/
+    $(DESTDIR)$$($(1)_INSTALL_PATH): $$($(1)_PATH) | $(DESTDIR)$$($(1)_INSTALL_DIR)/
 	install -t $(DESTDIR)$$($(1)_INSTALL_DIR) $$<
 
-    endif
   endif
 
   # Propagate CFLAGS and CXXFLAGS to the individual object files.
@@ -80,10 +76,4 @@ define build-program
   programs-list += $$($(1)_PATH)
   clean-files += $$($(1)_PATH) $$(_d)/*.o $$(_d)/.*.dep $$($(1)_DEPS) $$($(1)_OBJS)
   dist-files += $$(_srcs)
-
-  # Phony target to run this program (typically as a dependency of 'check').
-  .PHONY: $(1)_RUN
-  $(1)_RUN: $$($(1)_PATH)
-	$(trace-test) $$($(1)_PATH)
-
 endef

mk/run_test.sh

@@ -1,28 +0,0 @@
-#!/bin/sh
-
-set -u
-
-red=""
-green=""
-yellow=""
-normal=""
-
-post_run_msg="ran test $1..."
-if [ -t 1 ]; then
-    red=""
-    green=""
-    yellow=""
-    normal=""
-fi
-(cd $(dirname $1) && env ${TESTS_ENVIRONMENT} init.sh 2>/dev/null > /dev/null)
-log="$(cd $(dirname $1) && env ${TESTS_ENVIRONMENT} $(basename $1) 2>&1)"
-status=$?
-if [ $status -eq 0 ]; then
-  echo "$post_run_msg [${green}PASS$normal]"
-elif [ $status -eq 99 ]; then
-  echo "$post_run_msg [${yellow}SKIP$normal]"
-else
-  echo "$post_run_msg [${red}FAIL$normal]"
-  echo "$log" | sed 's/^/    /'
-  exit "$status"
-fi

mk/tests.mk

@@ -1,15 +1,45 @@
 # Run program $1 as part of ‘make installcheck’.
-
-test-deps =
-
 define run-install-test
 
-  installcheck: $1.test
+  installcheck: $1
 
-  .PHONY: $1.test
-  $1.test: $1 $(test-deps)
-	@env TEST_NAME=$(notdir $(basename $1)) TESTS_ENVIRONMENT="$(tests-environment)" mk/run_test.sh $1
+  _installcheck-list += $1
 
 endef
 
+# Color code from https://unix.stackexchange.com/a/10065
+installcheck:
+	@total=0; failed=0; \
+	red=""; \
+	green=""; \
+	yellow=""; \
+	normal=""; \
+	if [ -t 1 ]; then \
+		red=""; \
+		green=""; \
+		yellow=""; \
+		normal=""; \
+	fi; \
+	for i in $(_installcheck-list); do \
+	  total=$$((total + 1)); \
+	  printf "running test $$i..."; \
+	  log="$$(cd $$(dirname $$i) && $(tests-environment) $$(basename $$i) 2>&1)"; \
+	  status=$$?; \
+	  if [ $$status -eq 0 ]; then \
+	    echo " [$${green}PASS$$normal]"; \
+	  elif [ $$status -eq 99 ]; then \
+	    echo " [$${yellow}SKIP$$normal]"; \
+	  else \
+	    echo " [$${red}FAIL$$normal]"; \
+	    echo "$$log" | sed 's/^/    /'; \
+	    failed=$$((failed + 1)); \
+	  fi; \
+	done; \
+	if [ "$$failed" != 0 ]; then \
+	  echo "$${red}$$failed out of $$total tests failed $$normal"; \
+	  exit 1; \
+	else \
+		echo "$${green}All tests succeeded$$normal"; \
+	fi
+
 .PHONY: check installcheck

mk/tracing.mk

@@ -11,7 +11,6 @@ ifeq ($(V), 0)
   trace-javac   = @echo "  JAVAC " $@;
   trace-jar     = @echo "  JAR   " $@;
   trace-mkdir   = @echo "  MKDIR " $@;
-  trace-test    = @echo "  TEST  " $@;
 
   suppress  = @
 

nix-rust/local.mk

@@ -41,5 +41,5 @@ ifneq ($(OS), Darwin)
 check: rust-tests
 
 rust-tests:
-	$(trace-test) cd nix-rust && CARGO_HOME=$$(if [[ -d vendor ]]; then echo vendor; fi) cargo test --release $$(if [[ -d vendor ]]; then echo --offline; fi)
+	cd nix-rust && CARGO_HOME=$$(if [[ -d vendor ]]; then echo vendor; fi) cargo test --release $$(if [[ -d vendor ]]; then echo --offline; fi)
 endif

perl/lib/Nix/Store.xs

@@ -80,7 +80,7 @@ SV * queryReferences(char * path)
 SV * queryPathHash(char * path)
     PPCODE:
         try {
-            auto s = store()->queryPathInfo(store()->parseStorePath(path))->narHash.to_string(Base32, true);
+            auto s = store()->queryPathInfo(store()->parseStorePath(path))->narHash.to_string();
             XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
         } catch (Error & e) {
             croak("%s", e.what());
@@ -106,7 +106,7 @@ SV * queryPathInfo(char * path, int base32)
                 XPUSHs(&PL_sv_undef);
             else
                 XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(*info->deriver).c_str(), 0)));
-            auto s = info->narHash.to_string(base32 ? Base32 : Base16, true);
+            auto s = info->narHash.to_string(base32 ? Base32 : Base16);
             XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
             mXPUSHi(info->registrationTime);
             mXPUSHi(info->narSize);
@@ -182,7 +182,7 @@ void importPaths(int fd, int dontCheckSigs)
     PPCODE:
         try {
             FdSource source(fd);
-            store()->importPaths(source, dontCheckSigs ? NoCheckSigs : CheckSigs);
+            store()->importPaths(source, nullptr, dontCheckSigs ? NoCheckSigs : CheckSigs);
         } catch (Error & e) {
             croak("%s", e.what());
         }
@@ -274,8 +274,7 @@ int checkSignature(SV * publicKey_, SV * sig_, char * msg)
 SV * addToStore(char * srcPath, int recursive, char * algo)
     PPCODE:
         try {
-            auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
-            auto path = store()->addToStore(std::string(baseNameOf(srcPath)), srcPath, method, parseHashType(algo));
+            auto path = store()->addToStore(std::string(baseNameOf(srcPath)), srcPath, recursive, parseHashType(algo));
             XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0)));
         } catch (Error & e) {
             croak("%s", e.what());
@@ -286,8 +285,7 @@ SV * makeFixedOutputPath(int recursive, char * algo, char * hash, char * name)
     PPCODE:
         try {
             Hash h(hash, parseHashType(algo));
-            auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
-            auto path = store()->makeFixedOutputPath(method, h, name);
+            auto path = store()->makeFixedOutputPath(recursive, h, name);
             XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0)));
         } catch (Error & e) {
             croak("%s", e.what());

precompiled-headers.h

@@ -56,3 +56,6 @@
 #include <sys/wait.h>
 #include <termios.h>
 #include <unistd.h>
+
+#include "util.hh"
+#include "args.hh"

release-common.nix

@@ -50,11 +50,11 @@ rec {
       libarchive
       boost
       nlohmann_json
+      rustc cargo
 
       # Tests
       git
       mercurial
-      gmock
     ]
     ++ lib.optionals stdenv.isLinux [libseccomp utillinuxMinimal]
     ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium

release.nix

@@ -14,6 +14,50 @@ let
 
   jobs = rec {
 
+    # Create a "vendor" directory that contains the crates listed in
+    # Cargo.lock. This allows Nix to be built without network access.
+    vendoredCrates =
+      let
+        lockFile = builtins.fromTOML (builtins.readFile nix-rust/Cargo.lock);
+
+        files = map (pkg: import <nix/fetchurl.nix> {
+          url = "https://crates.io/api/v1/crates/${pkg.name}/${pkg.version}/download";
+          sha256 = lockFile.metadata."checksum ${pkg.name} ${pkg.version} (registry+https://github.com/rust-lang/crates.io-index)";
+        }) (builtins.filter (pkg: pkg.source or "" == "registry+https://github.com/rust-lang/crates.io-index") lockFile.package);
+
+      in pkgs.runCommand "cargo-vendor-dir" {}
+        ''
+          mkdir -p $out/vendor
+
+          cat > $out/vendor/config <<EOF
+          [source.crates-io]
+          replace-with = "vendored-sources"
+
+          [source.vendored-sources]
+          directory = "vendor"
+          EOF
+
+          ${toString (builtins.map (file: ''
+            mkdir $out/vendor/tmp
+            tar xvf ${file} -C $out/vendor/tmp
+            dir=$(echo $out/vendor/tmp/*)
+
+            # Add just enough metadata to keep Cargo happy.
+            printf '{"files":{},"package":"${file.outputHash}"}' > "$dir/.cargo-checksum.json"
+
+            # Clean up some cruft from the winapi crates. FIXME: find
+            # a way to remove winapi* from our dependencies.
+            if [[ $dir =~ /winapi ]]; then
+              find $dir -name "*.a" -print0 | xargs -0 rm -f --
+            fi
+
+            mv "$dir" $out/vendor/
+
+            rm -rf $out/vendor/tmp
+          '') files)}
+        '';
+
+
     build = pkgs.lib.genAttrs systems (system:
 
       let pkgs = import nixpkgs { inherit system; }; in
@@ -45,6 +89,8 @@ let
               patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.*
             ''}
 
+            ln -sfn ${vendoredCrates}/vendor/ nix-rust/vendor
+
             (cd perl; autoreconf --install --force --verbose)
           '';
 
@@ -57,17 +103,17 @@ let
 
         installFlags = "sysconfdir=$(out)/etc";
 
-        postInstall = ''
-          mkdir -p $doc/nix-support
-          echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products
-        '';
-
         doCheck = true;
 
         doInstallCheck = true;
         installCheckFlags = "sysconfdir=$(out)/etc";
 
         separateDebugInfo = true;
+
+        preDist = ''
+          mkdir -p $doc/nix-support
+          echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products
+        '';
       });
 
 
@@ -119,10 +165,10 @@ let
         }
         ''
           cp ${installerClosureInfo}/registration $TMPDIR/reginfo
-          cp ${./scripts/create-darwin-volume.sh} $TMPDIR/create-darwin-volume.sh
           substitute ${./scripts/install-nix-from-closure.sh} $TMPDIR/install \
             --subst-var-by nix ${toplevel} \
             --subst-var-by cacert ${cacert}
+
           substitute ${./scripts/install-darwin-multi-user.sh} $TMPDIR/install-darwin-multi-user.sh \
             --subst-var-by nix ${toplevel} \
             --subst-var-by cacert ${cacert}
@@ -137,7 +183,6 @@ let
             # SC1090: Don't worry about not being able to find
             #         $nix/etc/profile.d/nix.sh
             shellcheck --exclude SC1090 $TMPDIR/install
-            shellcheck $TMPDIR/create-darwin-volume.sh
             shellcheck $TMPDIR/install-darwin-multi-user.sh
             shellcheck $TMPDIR/install-systemd-multi-user.sh
 
@@ -153,7 +198,6 @@ let
           fi
 
           chmod +x $TMPDIR/install
-          chmod +x $TMPDIR/create-darwin-volume.sh
           chmod +x $TMPDIR/install-darwin-multi-user.sh
           chmod +x $TMPDIR/install-systemd-multi-user.sh
           chmod +x $TMPDIR/install-multi-user
@@ -166,15 +210,11 @@ let
             --absolute-names \
             --hard-dereference \
             --transform "s,$TMPDIR/install,$dir/install," \
-            --transform "s,$TMPDIR/create-darwin-volume.sh,$dir/create-darwin-volume.sh," \
             --transform "s,$TMPDIR/reginfo,$dir/.reginfo," \
             --transform "s,$NIX_STORE,$dir/store,S" \
-            $TMPDIR/install \
-            $TMPDIR/create-darwin-volume.sh \
-            $TMPDIR/install-darwin-multi-user.sh \
+            $TMPDIR/install $TMPDIR/install-darwin-multi-user.sh \
             $TMPDIR/install-systemd-multi-user.sh \
-            $TMPDIR/install-multi-user \
-            $TMPDIR/reginfo \
+            $TMPDIR/install-multi-user $TMPDIR/reginfo \
             $(cat ${installerClosureInfo}/store-paths)
         '');
 

scripts/create-darwin-volume.sh

@@ -1,185 +0,0 @@
-#!/bin/sh
-set -e
-
-root_disk() {
-    diskutil info -plist /
-}
-
-apfs_volumes_for() {
-    disk=$1
-    diskutil apfs list -plist "$disk"
-}
-
-disk_identifier() {
-    xpath "/plist/dict/key[text()='ParentWholeDisk']/following-sibling::string[1]/text()" 2>/dev/null
-}
-
-volume_list_true() {
-    key=$1
-    xpath "/plist/dict/array/dict/key[text()='Volumes']/following-sibling::array/dict/key[text()='$key']/following-sibling::true[1]" 2> /dev/null
-}
-
-volume_get_string() {
-    key=$1 i=$2
-    xpath "/plist/dict/array/dict/key[text()='Volumes']/following-sibling::array/dict[$i]/key[text()='$key']/following-sibling::string[1]/text()" 2> /dev/null
-}
-
-find_nix_volume() {
-    disk=$1
-    i=1
-    volumes=$(apfs_volumes_for "$disk")
-    while true; do
-        name=$(echo "$volumes" | volume_get_string "Name" "$i")
-        if [ -z "$name" ]; then
-            break
-        fi
-        case "$name" in
-            [Nn]ix*)
-                echo "$name"
-                break
-                ;;
-        esac
-        i=$((i+1))
-    done
-}
-
-test_fstab() {
-    grep -q "/nix apfs rw" /etc/fstab 2>/dev/null
-}
-
-test_nix_symlink() {
-    [ -L "/nix" ] || grep -q "^nix." /etc/synthetic.conf 2>/dev/null
-}
-
-test_synthetic_conf() {
-    grep -q "^nix$" /etc/synthetic.conf 2>/dev/null
-}
-
-test_nix() {
-    test -d "/nix"
-}
-
-test_t2_chip_present(){
-    # Use xartutil to see if system has a t2 chip.
-    #
-    # This isn't well-documented on its own; until it is,
-    # let's keep track of knowledge/assumptions.
-    #
-    # Warnings:
-    # - Don't search "xart" if porn will cause you trouble :)
-    # - Other xartutil flags do dangerous things. Don't run them
-    #   naively. If you must, search "xartutil" first.
-    #
-    # Assumptions:
-    # - the "xART session seeds recovery utility"
-    #   appears to interact with xartstorageremoted
-    # - `sudo xartutil --list` lists xART sessions
-    #   and their seeds and exits 0 if successful. If
-    #   not, it exits 1 and prints an error such as:
-    #   xartutil: ERROR: No supported link to the SEP present
-    # - xART sessions/seeds are present when a T2 chip is
-    #   (and not, otherwise)
-    # - the presence of a T2 chip means a newly-created
-    #   volume on the primary drive will be
-    #   encrypted at rest
-    # - all together: `sudo xartutil --list`
-    #   should exit 0 if a new Nix Store volume will
-    #   be encrypted at rest, and exit 1 if not.
-    sudo xartutil --list >/dev/null 2>/dev/null
-}
-
-test_filevault_in_use() {
-    disk=$1
-    #      list vols on disk | get value of Filevault key | value is true
-    apfs_volumes_for "$disk" | volume_list_true FileVault | grep -q true
-}
-
-# use after error msg for conditions we don't understand
-suggest_report_error(){
-    # ex "error: something sad happened :(" >&2
-    echo "       please report this @ https://github.com/nixos/nix/issues" >&2
-}
-
-main() {
-    (
-        echo ""
-        echo "     ------------------------------------------------------------------ "
-        echo "    | This installer will create a volume for the nix store and        |"
-        echo "    | configure it to mount at /nix.  Follow these steps to uninstall. |"
-        echo "     ------------------------------------------------------------------ "
-        echo ""
-        echo "  1. Remove the entry from fstab using 'sudo vifs'"
-        echo "  2. Destroy the data volume using 'diskutil apfs deleteVolume'"
-        echo "  3. Remove the 'nix' line from /etc/synthetic.conf or the file"
-        echo ""
-    ) >&2
-
-    if test_nix_symlink; then
-        echo "error: /nix is a symlink, please remove it and make sure it's not in synthetic.conf (in which case a reboot is required)" >&2
-        echo "  /nix -> $(readlink "/nix")" >&2
-        exit 2
-    fi
-
-    if ! test_synthetic_conf; then
-        echo "Configuring /etc/synthetic.conf..." >&2
-        echo nix | sudo tee -a /etc/synthetic.conf
-        if ! test_synthetic_conf; then
-            echo "error: failed to configure synthetic.conf;" >&2
-            suggest_report_error
-            exit 1
-        fi
-    fi
-
-    if ! test_nix; then
-        echo "Creating mountpoint for /nix..." >&2
-        /System/Library/Filesystems/apfs.fs/Contents/Resources/apfs.util -B || true
-        if ! test_nix; then
-            sudo mkdir -p /nix 2>/dev/null || true
-        fi
-        if ! test_nix; then
-            echo "error: failed to bootstrap /nix; if a reboot doesn't help," >&2
-            suggest_report_error
-            exit 1
-        fi
-    fi
-
-    disk=$(root_disk | disk_identifier)
-    volume=$(find_nix_volume "$disk")
-    if [ -z "$volume" ]; then
-        echo "Creating a Nix Store volume..." >&2
-
-        if test_filevault_in_use "$disk"; then
-            # TODO: Not sure if it's in-scope now, but `diskutil apfs list`
-            # shows both filevault and encrypted at rest status, and it
-            # may be the more semantic way to test for this? It'll show
-            # `FileVault:                 No (Encrypted at rest)`
-            # `FileVault:                 No`
-            # `FileVault:                 Yes (Unlocked)`
-            # and so on.
-            if test_t2_chip_present; then
-                echo "warning: boot volume is FileVault-encrypted, but the Nix store volume" >&2
-                echo "         is only encrypted at rest." >&2
-                echo "         See https://nixos.org/nix/manual/#sect-macos-installation" >&2
-            else
-                echo "error: refusing to create Nix store volume because the boot volume is" >&2
-                echo "       FileVault encrypted, but encryption-at-rest is not available." >&2
-                echo "       Manually create a volume for the store and re-run this script." >&2
-                echo "       See https://nixos.org/nix/manual/#sect-macos-installation" >&2
-                exit 1
-            fi
-        fi
-
-        sudo diskutil apfs addVolume "$disk" APFS 'Nix Store' -mountpoint /nix
-        volume="Nix Store"
-    else
-        echo "Using existing '$volume' volume" >&2
-    fi
-
-    if ! test_fstab; then
-        echo "Configuring /etc/fstab..." >&2
-        label=$(echo "$volume" | sed 's/ /\\040/g')
-        printf "\$a\nLABEL=%s /nix apfs rw,nobrowse\n.\nwq\n" "$label" | EDITOR=ed sudo vifs
-    fi
-}
-
-main "$@"

scripts/install-multi-user.sh

@@ -20,18 +20,15 @@ readonly GREEN='\033[32m'
 readonly GREEN_UL='\033[4;32m'
 readonly RED='\033[31m'
 
-# installer allows overriding build user count to speed up installation
-# as creating each user takes non-trivial amount of time on macos
-readonly NIX_USER_COUNT=${NIX_USER_COUNT:-32}
+readonly NIX_USER_COUNT="32"
 readonly NIX_BUILD_GROUP_ID="30000"
 readonly NIX_BUILD_GROUP_NAME="nixbld"
 readonly NIX_FIRST_BUILD_UID="30001"
 # Please don't change this. We don't support it, because the
 # default shell profile that comes with Nix doesn't support it.
 readonly NIX_ROOT="/nix"
-readonly NIX_EXTRA_CONF=${NIX_EXTRA_CONF:-}
 
-readonly PROFILE_TARGETS=("/etc/bashrc" "/etc/profile.d/nix.sh" "/etc/zshenv")
+readonly PROFILE_TARGETS=("/etc/bashrc" "/etc/profile.d/nix.sh" "/etc/zshrc")
 readonly PROFILE_BACKUP_SUFFIX=".backup-before-nix"
 readonly PROFILE_NIX_FILE="$NIX_ROOT/var/nix/profiles/default/etc/profile.d/nix-daemon.sh"
 
@@ -453,11 +450,9 @@ create_directories() {
 }
 
 place_channel_configuration() {
-    if [ -z "${NIX_INSTALLER_NO_CHANNEL_ADD:-}" ]; then
-        echo "https://nixos.org/channels/nixpkgs-unstable nixpkgs" > "$SCRATCH/.nix-channels"
-        _sudo "to set up the default system channel (part 1)" \
-            install -m 0664 "$SCRATCH/.nix-channels" "$ROOT_HOME/.nix-channels"
-    fi
+    echo "https://nixos.org/channels/nixpkgs-unstable nixpkgs" > "$SCRATCH/.nix-channels"
+    _sudo "to set up the default system channel (part 1)" \
+          install -m 0664 "$SCRATCH/.nix-channels" "$ROOT_HOME/.nix-channels"
 }
 
 welcome_to_nix() {
@@ -526,7 +521,7 @@ This script is going to call sudo a lot. Normally, it would show you
 exactly what commands it is running and why. However, the script is
 run in a headless fashion, like this:
 
-  $ curl -L https://nixos.org/nix/install | sh
+  $ curl https://nixos.org/nix/install | sh
 
 or maybe in a CI pipeline. Because of that, we're going to skip the
 verbose output in the interest of brevity.
@@ -534,7 +529,7 @@ verbose output in the interest of brevity.
 If you would like to
 see the output, try like this:
 
-  $ curl -L -o install-nix https://nixos.org/nix/install
+  $ curl -o install-nix https://nixos.org/nix/install
   $ sh ./install-nix
 
 EOF
@@ -639,20 +634,18 @@ setup_default_profile() {
         export NIX_SSL_CERT_FILE=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt
     fi
 
-    if [ -z "${NIX_INSTALLER_NO_CHANNEL_ADD:-}" ]; then
-        # Have to explicitly pass NIX_SSL_CERT_FILE as part of the sudo call,
-        # otherwise it will be lost in environments where sudo doesn't pass
-        # all the environment variables by default.
-        _sudo "to update the default channel in the default profile" \
-            HOME="$ROOT_HOME" NIX_SSL_CERT_FILE="$NIX_SSL_CERT_FILE" "$NIX_INSTALLED_NIX/bin/nix-channel" --update nixpkgs \
-            || channel_update_failed=1
-    fi
+    # Have to explicitly pass NIX_SSL_CERT_FILE as part of the sudo call,
+    # otherwise it will be lost in environments where sudo doesn't pass
+    # all the environment variables by default.
+    _sudo "to update the default channel in the default profile" \
+          HOME="$ROOT_HOME" NIX_SSL_CERT_FILE="$NIX_SSL_CERT_FILE" "$NIX_INSTALLED_NIX/bin/nix-channel" --update nixpkgs \
+          || channel_update_failed=1
+
 }
 
 
 place_nix_configuration() {
     cat <<EOF > "$SCRATCH/nix.conf"
-$NIX_EXTRA_CONF
 build-users-group = $NIX_BUILD_GROUP_NAME
 EOF
     _sudo "to place the default nix daemon configuration (part 2)" \

scripts/install-nix-from-closure.sh

@@ -40,85 +40,29 @@ elif [ "$(uname -s)" = "Linux" ] && [ -e /run/systemd/system ]; then
 fi
 
 INSTALL_MODE=no-daemon
-CREATE_DARWIN_VOLUME=0
-# handle the command line flags
-while [ $# -gt 0 ]; do
-    case $1 in
-        --daemon)
-            INSTALL_MODE=daemon;;
-        --no-daemon)
-            INSTALL_MODE=no-daemon;;
-        --no-channel-add)
-            export NIX_INSTALLER_NO_CHANNEL_ADD=1;;
-        --daemon-user-count)
-            export NIX_USER_COUNT=$2
-            shift;;
-        --no-modify-profile)
-            NIX_INSTALLER_NO_MODIFY_PROFILE=1;;
-        --darwin-use-unencrypted-nix-store-volume)
-            CREATE_DARWIN_VOLUME=1;;
-        --nix-extra-conf-file)
-            export NIX_EXTRA_CONF="$(cat $2)"
-            shift;;
-        *)
-            (
-                echo "Nix Installer [--daemon|--no-daemon] [--daemon-user-count INT] [--no-channel-add] [--no-modify-profile] [--darwin-use-unencrypted-nix-store-volume] [--nix-extra-conf-file FILE]"
+# Trivially handle the --daemon / --no-daemon options
+if [ "x${1:-}" = "x--no-daemon" ]; then
+    INSTALL_MODE=no-daemon
+elif [ "x${1:-}" = "x--daemon" ]; then
+    INSTALL_MODE=daemon
+elif [ "x${1:-}" != "x" ]; then
+    (
+        echo "Nix Installer [--daemon|--no-daemon]"
 
-                echo "Choose installation method."
-                echo ""
-                echo " --daemon:    Installs and configures a background daemon that manages the store,"
-                echo "              providing multi-user support and better isolation for local builds."
-                echo "              Both for security and reproducibility, this method is recommended if"
-                echo "              supported on your platform."
-                echo "              See https://nixos.org/nix/manual/#sect-multi-user-installation"
-                echo ""
-                echo " --no-daemon: Simple, single-user installation that does not require root and is"
-                echo "              trivial to uninstall."
-                echo "              (default)"
-                echo ""
-                echo " --no-channel-add:    Don't add any channels. nixpkgs-unstable is installed by default."
-                echo ""
-                echo " --no-modify-profile: Skip channel installation. When not provided nixpkgs-unstable"
-                echo "                      is installed by default."
-                echo ""
-                echo " --daemon-user-count: Number of build users to create. Defaults to 32."
-                echo ""
-                echo " --nix-extra-conf-file: Path to nix.conf to prepend when installing /etc/nix.conf"
-                echo ""
-            ) >&2
-
-            # darwin and Catalina+
-            if [ "$(uname -s)" = "Darwin" ] && [ "$macos_major" -gt 14 ]; then
-                (
-                    echo " --darwin-use-unencrypted-nix-store-volume: Create an APFS volume for the Nix"
-                    echo "              store and mount it at /nix. This is the recommended way to create"
-                    echo "              /nix with a read-only / on macOS >=10.15."
-                    echo "              See: https://nixos.org/nix/manual/#sect-macos-installation"
-                    echo ""
-                ) >&2
-            fi
-            exit;;
-    esac
-    shift
-done
-
-if [ "$(uname -s)" = "Darwin" ]; then
-    if [ "$CREATE_DARWIN_VOLUME" = 1 ]; then
-        printf '\e[1;31mCreating volume and mountpoint /nix.\e[0m\n'
-        "$self/create-darwin-volume.sh"
-    fi
-
-    info=$(diskutil info -plist / | xpath "/plist/dict/key[text()='Writable']/following-sibling::true[1]" 2> /dev/null)
-    if ! [ -e $dest ] && [ -n "$info" ] && [ "$macos_major" -gt 14 ]; then
-        (
-            echo ""
-            echo "Installing on macOS >=10.15 requires relocating the store to an apfs volume."
-            echo "Use sh <(curl -L https://nixos.org/nix/install) --darwin-use-unencrypted-nix-store-volume or run the preparation steps manually."
-            echo "See https://nixos.org/nix/manual/#sect-macos-installation"
-            echo ""
-        ) >&2
-        exit 1
-    fi
+        echo "Choose installation method."
+        echo ""
+        echo " --daemon:    Installs and configures a background daemon that manages the store,"
+        echo "              providing multi-user support and better isolation for local builds."
+        echo "              Both for security and reproducibility, this method is recommended if"
+        echo "              supported on your platform."
+        echo "              See https://nixos.org/nix/manual/#sect-multi-user-installation"
+        echo ""
+        echo " --no-daemon: Simple, single-user installation that does not require root and is"
+        echo "              trivial to uninstall."
+        echo "              (default)"
+        echo ""
+    ) >&2
+    exit
 fi
 
 if [ "$INSTALL_MODE" = "daemon" ]; then
@@ -186,15 +130,13 @@ if [ -z "$NIX_SSL_CERT_FILE" ] || ! [ -f "$NIX_SSL_CERT_FILE" ]; then
 fi
 
 # Subscribe the user to the Nixpkgs channel and fetch it.
-if [ -z "$NIX_INSTALLER_NO_CHANNEL_ADD" ]; then
-    if ! $nix/bin/nix-channel --list | grep -q "^nixpkgs "; then
-        $nix/bin/nix-channel --add https://nixos.org/channels/nixpkgs-unstable
-    fi
-    if [ -z "$_NIX_INSTALLER_TEST" ]; then
-        if ! $nix/bin/nix-channel --update nixpkgs; then
-            echo "Fetching the nixpkgs channel failed. (Are you offline?)"
-            echo "To try again later, run \"nix-channel --update nixpkgs\"."
-        fi
+if ! $nix/bin/nix-channel --list | grep -q "^nixpkgs "; then
+    $nix/bin/nix-channel --add https://nixos.org/channels/nixpkgs-unstable
+fi
+if [ -z "$_NIX_INSTALLER_TEST" ]; then
+    if ! $nix/bin/nix-channel --update nixpkgs; then
+        echo "Fetching the nixpkgs channel failed. (Are you offline?)"
+        echo "To try again later, run \"nix-channel --update nixpkgs\"."
     fi
 fi
 
@@ -213,17 +155,6 @@ if [ -z "$NIX_INSTALLER_NO_MODIFY_PROFILE" ]; then
             break
         fi
     done
-    for i in .zshenv .zshrc; do
-        fn="$HOME/$i"
-        if [ -w "$fn" ]; then
-            if ! grep -q "$p" "$fn"; then
-                echo "modifying $fn..." >&2
-                echo "if [ -e $p ]; then . $p; fi # added by Nix installer" >> "$fn"
-            fi
-            added=1
-            break
-        fi
-    done
 fi
 
 if [ -z "$added" ]; then

scripts/install.in

@@ -36,9 +36,7 @@ tarball="$tmpDir/$(basename "$tmpDir/nix-@nixVersion@-$system.tar.xz")"
 
 require_util curl "download the binary tarball"
 require_util tar "unpack the binary tarball"
-if [ "$(uname -s)" != "Darwin" ]; then
-    require_util xz "unpack the binary tarball"
-fi
+require_util xz "unpack the binary tarball"
 
 echo "downloading Nix @nixVersion@ binary tarball for $system from '$url' to '$tmpDir'..."
 curl -L "$url" -o "$tarball" || oops "failed to download '$url'"

shell.nix

@@ -7,7 +7,7 @@ with import ./release-common.nix { inherit pkgs; };
 (if useClang then clangStdenv else stdenv).mkDerivation {
   name = "nix";
 
-  buildInputs = buildDeps ++ propagatedDeps ++ perlDeps;
+  buildInputs = buildDeps ++ propagatedDeps ++ perlDeps ++ [ pkgs.rustfmt ];
 
   inherit configureFlags;
 

src/build-remote/build-remote.cc

@@ -200,12 +200,9 @@ static int _main(int argc, char * * argv)
 
                 } catch (std::exception & e) {
                     auto msg = chomp(drainFD(5, false));
-                    logError({
-                        .name = "Remote build",
-                        .hint = hintfmt("cannot build on '%s': %s%s",
-                            bestMachine->storeUri, e.what(),
-                            (msg.empty() ? "" : ": " + msg))
-                    });
+                    printError("cannot build on '%s': %s%s",
+                        bestMachine->storeUri, e.what(),
+                        (msg.empty() ? "" : ": " + msg));
                     bestMachine->enabled = false;
                     continue;
                 }
@@ -244,7 +241,7 @@ connected:
 
         uploadLock = -1;
 
-        auto drv = store->readDerivation(*drvPath);
+        BasicDerivation drv(readDerivation(*store, store->realStoreDir + "/" + std::string(drvPath->to_string())));
         drv.inputSrcs = store->parseStorePathSet(inputs);
 
         auto result = sshStore->buildDerivation(*drvPath, drv);

src/libexpr/attr-path.cc

@@ -6,11 +6,11 @@
 namespace nix {
 
 
-static Strings parseAttrPath(std::string_view s)
+static Strings parseAttrPath(const string & s)
 {
     Strings res;
     string cur;
-    auto i = s.begin();
+    string::const_iterator i = s.begin();
     while (i != s.end()) {
         if (*i == '.') {
             res.push_back(cur);
@@ -19,7 +19,7 @@ static Strings parseAttrPath(std::string_view s)
             ++i;
             while (1) {
                 if (i == s.end())
-                    throw Error("missing closing quote in selection path '%1%'", s);
+                    throw Error(format("missing closing quote in selection path '%1%'") % s);
                 if (*i == '"') break;
                 cur.push_back(*i++);
             }
@@ -32,20 +32,14 @@ static Strings parseAttrPath(std::string_view s)
 }
 
 
-std::vector<Symbol> parseAttrPath(EvalState & state, std::string_view s)
-{
-    std::vector<Symbol> res;
-    for (auto & a : parseAttrPath(s))
-        res.push_back(state.symbols.create(a));
-    return res;
-}
-
-
 std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attrPath,
     Bindings & autoArgs, Value & vIn)
 {
     Strings tokens = parseAttrPath(attrPath);
 
+    Error attrError =
+        Error(format("attribute selection path '%1%' does not match expression") % attrPath);
+
     Value * v = &vIn;
     Pos pos = noPos;
 
@@ -69,11 +63,11 @@ std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attr
 
             if (v->type != tAttrs)
                 throw TypeError(
-                    "the expression selected by the selection path '%1%' should be a set but is %2%",
-                    attrPath,
-                    showType(*v));
+                    format("the expression selected by the selection path '%1%' should be a set but is %2%")
+                    % attrPath % showType(*v));
+
             if (attr.empty())
-                throw Error("empty attribute name in selection path '%1%'", attrPath);
+                throw Error(format("empty attribute name in selection path '%1%'") % attrPath);
 
             Bindings::iterator a = v->attrs->find(state.symbols.create(attr));
             if (a == v->attrs->end())
@@ -86,9 +80,9 @@ std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attr
 
             if (!v->isList())
                 throw TypeError(
-                    "the expression selected by the selection path '%1%' should be a list but is %2%",
-                    attrPath,
-                    showType(*v));
+                    format("the expression selected by the selection path '%1%' should be a list but is %2%")
+                    % attrPath % showType(*v));
+
             if (attrIndex >= v->listSize())
                 throw AttrPathNotFound("list index %1% in selection path '%2%' is out of range", attrIndex, attrPath);
 
@@ -130,7 +124,7 @@ Pos findDerivationFilename(EvalState & state, Value & v, std::string what)
 
     Symbol file = state.symbols.create(filename);
 
-    return { foFile, file, lineno, 0 };
+    return { file, lineno, 0 };
 }
 
 

src/libexpr/attr-path.hh

@@ -16,6 +16,4 @@ std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attr
 /* Heuristic to find the filename and lineno or a nix value. */
 Pos findDerivationFilename(EvalState & state, Value & v, std::string what);
 
-std::vector<Symbol> parseAttrPath(EvalState & state, std::string_view s);
-
 }

src/libexpr/attr-set.hh

@@ -76,11 +76,7 @@ public:
     {
         auto a = get(name);
         if (!a)
-            throw Error({
-                .hint = hintfmt("attribute '%s' missing", name),
-                .errPos = pos
-            });
-
+            throw Error("attribute '%s' missing, at %s", name, pos);
         return *a;
     }
 

src/libexpr/common-eval-args.cc

@@ -1,36 +1,31 @@
 #include "common-eval-args.hh"
 #include "shared.hh"
-#include "filetransfer.hh"
+#include "download.hh"
 #include "util.hh"
 #include "eval.hh"
-#include "fetchers.hh"
-#include "store-api.hh"
 
 namespace nix {
 
 MixEvalArgs::MixEvalArgs()
 {
-    addFlag({
-        .longName = "arg",
-        .description = "argument to be passed to Nix functions",
-        .labels = {"name", "expr"},
-        .handler = {[&](std::string name, std::string expr) { autoArgs[name] = 'E' + expr; }}
-    });
+    mkFlag()
+        .longName("arg")
+        .description("argument to be passed to Nix functions")
+        .labels({"name", "expr"})
+        .handler([&](std::vector<std::string> ss) { autoArgs[ss[0]] = 'E' + ss[1]; });
 
-    addFlag({
-        .longName = "argstr",
-        .description = "string-valued argument to be passed to Nix functions",
-        .labels = {"name", "string"},
-        .handler = {[&](std::string name, std::string s) { autoArgs[name] = 'S' + s; }},
-    });
+    mkFlag()
+        .longName("argstr")
+        .description("string-valued argument to be passed to Nix functions")
+        .labels({"name", "string"})
+        .handler([&](std::vector<std::string> ss) { autoArgs[ss[0]] = 'S' + ss[1]; });
 
-    addFlag({
-        .longName = "include",
-        .shortName = 'I',
-        .description = "add a path to the list of locations used to look up <...> file names",
-        .labels = {"path"},
-        .handler = {[&](std::string s) { searchPath.push_back(s); }}
-    });
+    mkFlag()
+        .shortName('I')
+        .longName("include")
+        .description("add a path to the list of locations used to look up <...> file names")
+        .label("path")
+        .handler([&](std::string s) { searchPath.push_back(s); });
 }
 
 Bindings * MixEvalArgs::getAutoArgs(EvalState & state)
@@ -51,9 +46,9 @@ Bindings * MixEvalArgs::getAutoArgs(EvalState & state)
 Path lookupFileArg(EvalState & state, string s)
 {
     if (isUri(s)) {
-        return state.store->toRealPath(
-            fetchers::downloadTarball(
-                state.store, resolveUri(s), "source", false).storePath);
+        CachedDownloadRequest request(s);
+        request.unpack = true;
+        return getDownloader()->downloadCached(state.store, request).path;
     } else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') {
         Path p = s.substr(1, s.size() - 2);
         return state.findFile(p);

src/libexpr/eval-inline.hh

@@ -7,26 +7,20 @@
 
 namespace nix {
 
-LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s))
+LocalNoInlineNoReturn(void throwEvalError(const char * s, const Pos & pos))
 {
-    throw EvalError({
-        .hint = hintfmt(s),
-        .errPos = pos
-    });
+    throw EvalError(format(s) % pos);
 }
 
 LocalNoInlineNoReturn(void throwTypeError(const char * s, const Value & v))
 {
-    throw TypeError(s, showType(v));
+    throw TypeError(format(s) % showType(v));
 }
 
 
-LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s, const Value & v))
+LocalNoInlineNoReturn(void throwTypeError(const char * s, const Value & v, const Pos & pos))
 {
-    throw TypeError({
-        .hint = hintfmt(s, showType(v)),
-        .errPos = pos
-    });
+    throw TypeError(format(s) % showType(v) % pos);
 }
 
 
@@ -49,7 +43,7 @@ void EvalState::forceValue(Value & v, const Pos & pos)
     else if (v.type == tApp)
         callFunction(*v.app.left, *v.app.right, v, noPos);
     else if (v.type == tBlackhole)
-        throwEvalError(pos, "infinite recursion encountered");
+        throwEvalError("infinite recursion encountered, at %1%", pos);
 }
 
 
@@ -63,9 +57,9 @@ inline void EvalState::forceAttrs(Value & v)
 
 inline void EvalState::forceAttrs(Value & v, const Pos & pos)
 {
-    forceValue(v, pos);
+    forceValue(v);
     if (v.type != tAttrs)
-        throwTypeError(pos, "value is %1% while a set was expected", v);
+        throwTypeError("value is %1% while a set was expected, at %2%", v, pos);
 }
 
 
@@ -79,9 +73,9 @@ inline void EvalState::forceList(Value & v)
 
 inline void EvalState::forceList(Value & v, const Pos & pos)
 {
-    forceValue(v, pos);
+    forceValue(v);
     if (!v.isList())
-        throwTypeError(pos, "value is %1% while a list was expected", v);
+        throwTypeError("value is %1% while a list was expected, at %2%", v, pos);
 }
 
 /* Note: Various places expect the allocated memory to be zeroed. */

src/libexpr/eval.cc

@@ -5,7 +5,7 @@
 #include "derivations.hh"
 #include "globals.hh"
 #include "eval-inline.hh"
-#include "filetransfer.hh"
+#include "download.hh"
 #include "json.hh"
 #include "function-trace.hh"
 
@@ -22,8 +22,6 @@
 
 #if HAVE_BOEHMGC
 
-#define GC_INCLUDE_NEW
-
 #include <gc/gc.h>
 #include <gc/gc_cpp.h>
 
@@ -58,12 +56,6 @@ static char * dupStringWithLen(const char * s, size_t size)
 }
 
 
-RootValue allocRootValue(Value * v)
-{
-    return std::allocate_shared<Value *>(traceable_allocator<Value *>(), v);
-}
-
-
 static void printValue(std::ostream & str, std::set<const Value *> & active, const Value & v)
 {
     checkInterrupt();
@@ -161,12 +153,12 @@ const Value *getPrimOp(const Value &v) {
 }
 
 
-string showType(ValueType type)
+string showType(const Value & v)
 {
-    switch (type) {
+    switch (v.type) {
         case tInt: return "an integer";
-        case tBool: return "a Boolean";
-        case tString: return "a string";
+        case tBool: return "a boolean";
+        case tString: return v.string.context ? "a string with context" : "a string";
         case tPath: return "a path";
         case tNull: return "null";
         case tAttrs: return "a set";
@@ -175,27 +167,14 @@ string showType(ValueType type)
         case tApp: return "a function application";
         case tLambda: return "a function";
         case tBlackhole: return "a black hole";
-        case tPrimOp: return "a built-in function";
-        case tPrimOpApp: return "a partially applied built-in function";
-        case tExternal: return "an external value";
-        case tFloat: return "a float";
-    }
-    abort();
-}
-
-
-string showType(const Value & v)
-{
-    switch (v.type) {
-        case tString: return v.string.context ? "a string with context" : "a string";
         case tPrimOp:
             return fmt("the built-in function '%s'", string(v.primOp->name));
         case tPrimOpApp:
             return fmt("the partially applied built-in function '%s'", string(getPrimOp(v)->primOp->name));
         case tExternal: return v.external->showType();
-    default:
-        return showType(v.type);
+        case tFloat: return "a float";
     }
+    abort();
 }
 
 
@@ -336,7 +315,6 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
     , sOutputHash(symbols.create("outputHash"))
     , sOutputHashAlgo(symbols.create("outputHashAlgo"))
     , sOutputHashMode(symbols.create("outputHashMode"))
-    , sRecurseForDerivations(symbols.create("recurseForDerivations"))
     , repair(NoRepair)
     , store(store)
     , baseEnv(allocEnv(128))
@@ -366,7 +344,7 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
 
             if (store->isInStore(r.second)) {
                 StorePathSet closure;
-                store->computeFSClosure(store->toStorePath(r.second).first, closure);
+                store->computeFSClosure(store->parseStorePath(store->toStorePath(r.second)), closure);
                 for (auto & path : closure)
                     allowedPaths->insert(store->printStorePath(path));
             } else
@@ -485,21 +463,14 @@ Value * EvalState::addConstant(const string & name, Value & v)
 Value * EvalState::addPrimOp(const string & name,
     size_t arity, PrimOpFun primOp)
 {
-    auto name2 = string(name, 0, 2) == "__" ? string(name, 2) : name;
-    Symbol sym = symbols.create(name2);
-
-    /* Hack to make constants lazy: turn them into a application of
-       the primop to a dummy value. */
     if (arity == 0) {
-        auto vPrimOp = allocValue();
-        vPrimOp->type = tPrimOp;
-        vPrimOp->primOp = new PrimOp(primOp, 1, sym);
         Value v;
-        mkApp(v, *vPrimOp, *vPrimOp);
+        primOp(*this, noPos, nullptr, v);
         return addConstant(name, v);
     }
-
     Value * v = allocValue();
+    string name2 = string(name, 0, 2) == "__" ? string(name, 2) : name;
+    Symbol sym = symbols.create(name2);
     v->type = tPrimOp;
     v->primOp = new PrimOp(primOp, arity, sym);
     staticBaseEnv.vars[symbols.create(name)] = baseEnvDispl;
@@ -522,84 +493,67 @@ Value & EvalState::getBuiltin(const string & name)
 
 LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2))
 {
-    throw EvalError(s, s2);
+    throw EvalError(format(s) % s2);
 }
 
-LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s, const string & s2))
+LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2, const Pos & pos))
 {
-    throw EvalError({
-        .hint = hintfmt(s, s2),
-        .errPos = pos
-    });
+    throw EvalError(format(s) % s2 % pos);
 }
 
 LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2, const string & s3))
 {
-    throw EvalError(s, s2, s3);
+    throw EvalError(format(s) % s2 % s3);
 }
 
-LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s, const string & s2, const string & s3))
+LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2, const string & s3, const Pos & pos))
 {
-    throw EvalError({
-        .hint = hintfmt(s, s2, s3),
-        .errPos = pos
-    });
+    throw EvalError(format(s) % s2 % s3 % pos);
 }
 
-LocalNoInlineNoReturn(void throwEvalError(const Pos & p1, const char * s, const Symbol & sym, const Pos & p2))
+LocalNoInlineNoReturn(void throwEvalError(const char * s, const Symbol & sym, const Pos & p1, const Pos & p2))
 {
-    // p1 is where the error occurred; p2 is a position mentioned in the message.
-    throw EvalError({
-        .hint = hintfmt(s, sym, p2),
-        .errPos = p1
-    });
+    throw EvalError(format(s) % sym % p1 % p2);
 }
 
-LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s))
+LocalNoInlineNoReturn(void throwTypeError(const char * s, const Pos & pos))
 {
-    throw TypeError({
-        .hint = hintfmt(s),
-        .errPos = pos
-    });
+    throw TypeError(format(s) % pos);
 }
 
 LocalNoInlineNoReturn(void throwTypeError(const char * s, const string & s1))
 {
-    throw TypeError(s, s1);
+    throw TypeError(format(s) % s1);
 }
 
-LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s, const ExprLambda & fun, const Symbol & s2))
+LocalNoInlineNoReturn(void throwTypeError(const char * s, const ExprLambda & fun, const Symbol & s2, const Pos & pos))
 {
-    throw TypeError({
-        .hint = hintfmt(s, fun.showNamePos(), s2),
-        .errPos = pos
-    });
+    throw TypeError(format(s) % fun.showNamePos() % s2 % pos);
 }
 
-LocalNoInlineNoReturn(void throwAssertionError(const Pos & pos, const char * s, const string & s1))
+LocalNoInlineNoReturn(void throwAssertionError(const char * s, const string & s1, const Pos & pos))
 {
-    throw AssertionError({
-        .hint = hintfmt(s, s1),
-        .errPos = pos
-    });
+    throw AssertionError(format(s) % s1 % pos);
 }
 
-LocalNoInlineNoReturn(void throwUndefinedVarError(const Pos & pos, const char * s, const string & s1))
+LocalNoInlineNoReturn(void throwUndefinedVarError(const char * s, const string & s1, const Pos & pos))
 {
-    throw UndefinedVarError({
-        .hint = hintfmt(s, s1),
-        .errPos = pos
-    });
+    throw UndefinedVarError(format(s) % s1 % pos);
 }
 
-LocalNoInline(void addErrorTrace(Error & e, const char * s, const string & s2))
+LocalNoInline(void addErrorPrefix(Error & e, const char * s, const string & s2))
 {
-    e.addTrace(std::nullopt, s, s2);
+    e.addPrefix(format(s) % s2);
 }
 
-LocalNoInline(void addErrorTrace(Error & e, const Pos & pos, const char * s, const string & s2))
+LocalNoInline(void addErrorPrefix(Error & e, const char * s, const ExprLambda & fun, const Pos & pos))
 {
-    e.addTrace(pos, s, s2);
+    e.addPrefix(format(s) % fun.showNamePos() % pos);
+}
+
+LocalNoInline(void addErrorPrefix(Error & e, const char * s, const string & s2, const Pos & pos))
+{
+    e.addPrefix(format(s) % s2 % pos);
 }
 
 
@@ -652,7 +606,7 @@ inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval)
             return j->value;
         }
         if (!env->prevWith)
-            throwUndefinedVarError(var.pos, "undefined variable '%1%'", var.name);
+            throwUndefinedVarError("undefined variable '%1%' at %2%", var.name, var.pos);
         for (size_t l = env->prevWith; l; --l, env = env->up) ;
     }
 }
@@ -813,7 +767,7 @@ void EvalState::evalFile(const Path & path_, Value & v)
     try {
         eval(e, v);
     } catch (Error & e) {
-        addErrorTrace(e, "while evaluating the file '%1%':", path2);
+        addErrorPrefix(e, "while evaluating the file '%1%':\n", path2);
         throw;
     }
 
@@ -850,7 +804,7 @@ inline bool EvalState::evalBool(Env & env, Expr * e, const Pos & pos)
     Value v;
     e->eval(*this, env, v);
     if (v.type != tBool)
-        throwTypeError(pos, "value is %1% while a Boolean was expected", v);
+        throwTypeError("value is %1% while a Boolean was expected, at %2%", v, pos);
     return v.boolean;
 }
 
@@ -964,7 +918,7 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v)
         Symbol nameSym = state.symbols.create(nameVal.string.s);
         Bindings::iterator j = v.attrs->find(nameSym);
         if (j != v.attrs->end())
-            throwEvalError(i.pos, "dynamic attribute '%1%' already defined at %2%", nameSym, *j->pos);
+            throwEvalError("dynamic attribute '%1%' at %2% already defined at %3%", nameSym, i.pos, *j->pos);
 
         i.valueExpr->setName(nameSym);
         /* Keep sorted order so find can catch duplicates */
@@ -1052,7 +1006,7 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v)
             } else {
                 state.forceAttrs(*vAttrs, pos);
                 if ((j = vAttrs->attrs->find(name)) == vAttrs->attrs->end())
-                    throwEvalError(pos, "attribute '%1%' missing", name);
+                    throwEvalError("attribute '%1%' missing, at %2%", name, pos);
             }
             vAttrs = j->value;
             pos2 = j->pos;
@@ -1063,8 +1017,8 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v)
 
     } catch (Error & e) {
         if (pos2 && pos2->file != state.sDerivationNix)
-            addErrorTrace(e, *pos2, "while evaluating the attribute '%1%'",
-                showAttrPath(state, env, attrPath));
+            addErrorPrefix(e, "while evaluating the attribute '%1%' at %2%:\n",
+                showAttrPath(state, env, attrPath), *pos2);
         throw;
     }
 
@@ -1178,7 +1132,7 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po
     }
 
     if (fun.type != tLambda)
-        throwTypeError(pos, "attempt to call something which is not a function but %1%", fun);
+        throwTypeError("attempt to call something which is not a function but %1%, at %2%", fun, pos);
 
     ExprLambda & lambda(*fun.lambda.fun);
 
@@ -1206,8 +1160,8 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po
         for (auto & i : lambda.formals->formals) {
             Bindings::iterator j = arg.attrs->find(i.name);
             if (j == arg.attrs->end()) {
-                if (!i.def) throwTypeError(pos, "%1% called without required argument '%2%'",
-                    lambda, i.name);
+                if (!i.def) throwTypeError("%1% called without required argument '%2%', at %3%",
+                    lambda, i.name, pos);
                 env2.values[displ++] = i.def->maybeThunk(*this, env2);
             } else {
                 attrsUsed++;
@@ -1222,7 +1176,7 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po
                user. */
             for (auto & i : *arg.attrs)
                 if (lambda.formals->argNames.find(i.name) == lambda.formals->argNames.end())
-                    throwTypeError(pos, "%1% called with unexpected argument '%2%'", lambda, i.name);
+                    throwTypeError("%1% called with unexpected argument '%2%', at %3%", lambda, i.name, pos);
             abort(); // can't happen
         }
     }
@@ -1232,15 +1186,11 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po
 
     /* Evaluate the body.  This is conditional on showTrace, because
        catching exceptions makes this function not tail-recursive. */
-    if (loggerSettings.showTrace.get())
+    if (settings.showTrace)
         try {
             lambda.body->eval(*this, env2, v);
         } catch (Error & e) {
-            addErrorTrace(e, lambda.pos, "while evaluating %s", 
-              (lambda.name.set() 
-                  ? "'" + (string) lambda.name + "'" 
-                  : "anonymous lambdaction"));
-            addErrorTrace(e, pos, "from call site%s", "");
+            addErrorPrefix(e, "while evaluating %1%, called from %2%:\n", lambda, pos);
             throw;
         }
     else
@@ -1306,7 +1256,7 @@ void ExprWith::eval(EvalState & state, Env & env, Value & v)
 
 void ExprIf::eval(EvalState & state, Env & env, Value & v)
 {
-    (state.evalBool(env, cond, pos) ? then : else_)->eval(state, env, v);
+    (state.evalBool(env, cond) ? then : else_)->eval(state, env, v);
 }
 
 
@@ -1315,7 +1265,7 @@ void ExprAssert::eval(EvalState & state, Env & env, Value & v)
     if (!state.evalBool(env, cond, pos)) {
         std::ostringstream out;
         cond->show(out);
-        throwAssertionError(pos, "assertion '%1%' failed at %2%", out.str());
+        throwAssertionError("assertion '%1%' failed at %2%", out.str(), pos);
     }
     body->eval(state, env, v);
 }
@@ -1467,14 +1417,14 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
                 nf = n;
                 nf += vTmp.fpoint;
             } else
-                throwEvalError(pos, "cannot add %1% to an integer", showType(vTmp));
+                throwEvalError("cannot add %1% to an integer, at %2%", showType(vTmp), pos);
         } else if (firstType == tFloat) {
             if (vTmp.type == tInt) {
                 nf += vTmp.integer;
             } else if (vTmp.type == tFloat) {
                 nf += vTmp.fpoint;
             } else
-                throwEvalError(pos, "cannot add %1% to a float", showType(vTmp));
+                throwEvalError("cannot add %1% to a float, at %2%", showType(vTmp), pos);
         } else
             s << state.coerceToString(pos, vTmp, context, false, firstType == tString);
     }
@@ -1485,7 +1435,7 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
         mkFloat(v, nf);
     else if (firstType == tPath) {
         if (!context.empty())
-            throwEvalError(pos, "a string that refers to a store path cannot be appended to a path");
+            throwEvalError("a string that refers to a store path cannot be appended to a path, at %1%", pos);
         auto path = canonPath(s.str());
         mkPath(v, path.c_str());
     } else
@@ -1515,7 +1465,7 @@ void EvalState::forceValueDeep(Value & v)
                 try {
                     recurse(*i.value);
                 } catch (Error & e) {
-                    addErrorTrace(e, *i.pos, "while evaluating the attribute '%1%'", i.name);
+                    addErrorPrefix(e, "while evaluating the attribute '%1%' at %2%:\n", i.name, *i.pos);
                     throw;
                 }
         }
@@ -1534,7 +1484,7 @@ NixInt EvalState::forceInt(Value & v, const Pos & pos)
 {
     forceValue(v, pos);
     if (v.type != tInt)
-        throwTypeError(pos, "value is %1% while an integer was expected", v);
+        throwTypeError("value is %1% while an integer was expected, at %2%", v, pos);
     return v.integer;
 }
 
@@ -1545,16 +1495,16 @@ NixFloat EvalState::forceFloat(Value & v, const Pos & pos)
     if (v.type == tInt)
         return v.integer;
     else if (v.type != tFloat)
-        throwTypeError(pos, "value is %1% while a float was expected", v);
+        throwTypeError("value is %1% while a float was expected, at %2%", v, pos);
     return v.fpoint;
 }
 
 
 bool EvalState::forceBool(Value & v, const Pos & pos)
 {
-    forceValue(v, pos);
+    forceValue(v);
     if (v.type != tBool)
-        throwTypeError(pos, "value is %1% while a Boolean was expected", v);
+        throwTypeError("value is %1% while a Boolean was expected, at %2%", v, pos);
     return v.boolean;
 }
 
@@ -1567,9 +1517,9 @@ bool EvalState::isFunctor(Value & fun)
 
 void EvalState::forceFunction(Value & v, const Pos & pos)
 {
-    forceValue(v, pos);
+    forceValue(v);
     if (v.type != tLambda && v.type != tPrimOp && v.type != tPrimOpApp && !isFunctor(v))
-        throwTypeError(pos, "value is %1% while a function was expected", v);
+        throwTypeError("value is %1% while a function was expected, at %2%", v, pos);
 }
 
 
@@ -1578,7 +1528,7 @@ string EvalState::forceString(Value & v, const Pos & pos)
     forceValue(v, pos);
     if (v.type != tString) {
         if (pos)
-            throwTypeError(pos, "value is %1% while a string was expected", v);
+            throwTypeError("value is %1% while a string was expected, at %2%", v, pos);
         else
             throwTypeError("value is %1% while a string was expected", v);
     }
@@ -1607,8 +1557,8 @@ string EvalState::forceStringNoCtx(Value & v, const Pos & pos)
     string s = forceString(v, pos);
     if (v.string.context) {
         if (pos)
-            throwEvalError(pos, "the string '%1%' is not allowed to refer to a store path (such as '%2%')",
-                v.string.s, v.string.context[0]);
+            throwEvalError("the string '%1%' is not allowed to refer to a store path (such as '%2%'), at %3%",
+                v.string.s, v.string.context[0], pos);
         else
             throwEvalError("the string '%1%' is not allowed to refer to a store path (such as '%2%')",
                 v.string.s, v.string.context[0]);
@@ -1644,7 +1594,7 @@ std::optional<string> EvalState::tryAttrsToString(const Pos & pos, Value & v,
 string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context,
     bool coerceMore, bool copyToStore)
 {
-    forceValue(v, pos);
+    forceValue(v);
 
     string s;
 
@@ -1664,7 +1614,7 @@ string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context,
             return *maybeString;
         }
         auto i = v.attrs->find(sOutPath);
-        if (i == v.attrs->end()) throwTypeError(pos, "cannot coerce a set to a string");
+        if (i == v.attrs->end()) throwTypeError("cannot coerce a set to a string, at %1%", pos);
         return coerceToString(pos, *i->value, context, coerceMore, copyToStore);
     }
 
@@ -1695,7 +1645,7 @@ string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context,
         }
     }
 
-    throwTypeError(pos, "cannot coerce %1% to a string", v);
+    throwTypeError("cannot coerce %1% to a string, at %2%", v, pos);
 }
 
 
@@ -1711,7 +1661,7 @@ string EvalState::copyPathToStore(PathSet & context, const Path & path)
     else {
         auto p = settings.readOnlyMode
             ? store->computeStorePathForPath(std::string(baseNameOf(path)), checkSourcePath(path)).first
-            : store->addToStore(std::string(baseNameOf(path)), checkSourcePath(path), FileIngestionMethod::Recursive, htSHA256, defaultPathFilter, repair);
+            : store->addToStore(std::string(baseNameOf(path)), checkSourcePath(path), true, htSHA256, defaultPathFilter, repair);
         dstPath = store->printStorePath(p);
         srcToStore.insert_or_assign(path, std::move(p));
         printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, dstPath);
@@ -1726,7 +1676,7 @@ Path EvalState::coerceToPath(const Pos & pos, Value & v, PathSet & context)
 {
     string path = coerceToString(pos, v, context, false, false);
     if (path == "" || path[0] != '/')
-        throwEvalError(pos, "string '%1%' doesn't represent an absolute path", path);
+        throwEvalError("string '%1%' doesn't represent an absolute path, at %2%", path, pos);
     return path;
 }
 
@@ -1933,10 +1883,8 @@ void EvalState::printStats()
 
 string ExternalValueBase::coerceToString(const Pos & pos, PathSet & context, bool copyMore, bool copyToStore) const
 {
-    throw TypeError({
-        .hint = hintfmt("cannot coerce %1% to a string", showType()),
-        .errPos = pos
-    });
+    throw TypeError(format("cannot coerce %1% to a string, at %2%") %
+        showType() % pos);
 }
 
 

src/libexpr/eval.hh

@@ -18,7 +18,7 @@ namespace nix {
 
 class Store;
 class EvalState;
-class StorePath;
+struct StorePath;
 enum RepairFlag : bool;
 
 
@@ -74,8 +74,7 @@ public:
         sSystem, sOverrides, sOutputs, sOutputName, sIgnoreNulls,
         sFile, sLine, sColumn, sFunctor, sToString,
         sRight, sWrong, sStructuredAttrs, sBuilder, sArgs,
-        sOutputHash, sOutputHashAlgo, sOutputHashMode,
-        sRecurseForDerivations;
+        sOutputHash, sOutputHashAlgo, sOutputHashMode;
     Symbol sDerivationNix;
 
     /* If set, force copying files to the Nix store even if they
@@ -250,7 +249,7 @@ private:
     friend struct ExprAttrs;
     friend struct ExprLet;
 
-    Expr * parse(const char * text, FileOrigin origin, const Path & path,
+    Expr * parse(const char * text, const Path & path,
         const Path & basePath, StaticEnv & staticEnv);
 
 public:
@@ -325,7 +324,6 @@ private:
 
 
 /* Return a string representing the type of the value `v'. */
-string showType(ValueType type);
 string showType(const Value & v);
 
 /* Decode a context string ‘!<name>!<path>’ into a pair <path,

src/libexpr/get-drvs.cc

@@ -1,7 +1,7 @@
 #include "get-drvs.hh"
 #include "util.hh"
 #include "eval-inline.hh"
-#include "store-api.hh"
+#include "derivations.hh"
 
 #include <cstring>
 #include <regex>
@@ -348,7 +348,7 @@ static void getDerivations(EvalState & state, Value & vIn,
                    should we recurse into it?  => Only if it has a
                    `recurseForDerivations = true' attribute. */
                 if (i->value->type == tAttrs) {
-                    Bindings::iterator j = i->value->attrs->find(state.sRecurseForDerivations);
+                    Bindings::iterator j = i->value->attrs->find(state.symbols.create("recurseForDerivations"));
                     if (j != i->value->attrs->end() && state.forceBool(*j->value, *j->pos))
                         getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures);
                 }

src/libexpr/json-to-value.cc

@@ -4,6 +4,7 @@
 #include <nlohmann/json.hpp>
 
 using json = nlohmann::json;
+using std::unique_ptr;
 
 namespace nix {
 
@@ -12,69 +13,69 @@ namespace nix {
 class JSONSax : nlohmann::json_sax<json> {
     class JSONState {
     protected:
-        std::unique_ptr<JSONState> parent;
-        RootValue v;
+        unique_ptr<JSONState> parent;
+        Value * v;
     public:
-        virtual std::unique_ptr<JSONState> resolve(EvalState &)
+        virtual unique_ptr<JSONState> resolve(EvalState &)
         {
             throw std::logic_error("tried to close toplevel json parser state");
-        }
-        explicit JSONState(std::unique_ptr<JSONState> && p) : parent(std::move(p)) {}
-        explicit JSONState(Value * v) : v(allocRootValue(v)) {}
-        JSONState(JSONState & p) = delete;
-        Value & value(EvalState & state)
+        };
+        explicit JSONState(unique_ptr<JSONState>&& p) : parent(std::move(p)), v(nullptr) {};
+        explicit JSONState(Value* v) : v(v) {};
+        JSONState(JSONState& p) = delete;
+        Value& value(EvalState & state)
         {
-            if (!v)
-                v = allocRootValue(state.allocValue());
-            return **v;
-        }
-        virtual ~JSONState() {}
-        virtual void add() {}
+            if (v == nullptr)
+                v = state.allocValue();
+            return *v;
+        };
+        virtual ~JSONState() {};
+        virtual void add() {};
     };
 
     class JSONObjectState : public JSONState {
         using JSONState::JSONState;
-        ValueMap attrs;
-        std::unique_ptr<JSONState> resolve(EvalState & state) override
+        ValueMap attrs = ValueMap();
+        virtual unique_ptr<JSONState> resolve(EvalState & state) override
         {
-            Value & v = parent->value(state);
+            Value& v = parent->value(state);
             state.mkAttrs(v, attrs.size());
             for (auto & i : attrs)
                 v.attrs->push_back(Attr(i.first, i.second));
             return std::move(parent);
         }
-        void add() override { v = nullptr; }
+        virtual void add() override { v = nullptr; };
     public:
-        void key(string_t & name, EvalState & state)
+        void key(string_t& name, EvalState & state)
         {
-            attrs.insert_or_assign(state.symbols.create(name), &value(state));
+            attrs[state.symbols.create(name)] = &value(state);
         }
     };
 
     class JSONListState : public JSONState {
-        ValueVector values;
-        std::unique_ptr<JSONState> resolve(EvalState & state) override
+        ValueVector values = ValueVector();
+        virtual unique_ptr<JSONState> resolve(EvalState & state) override
         {
-            Value & v = parent->value(state);
+            Value& v = parent->value(state);
             state.mkList(v, values.size());
             for (size_t n = 0; n < values.size(); ++n) {
                 v.listElems()[n] = values[n];
             }
             return std::move(parent);
         }
-        void add() override {
-            values.push_back(*v);
+        virtual void add() override {
+            values.push_back(v);
             v = nullptr;
-        }
+        };
     public:
-        JSONListState(std::unique_ptr<JSONState> && p, std::size_t reserve) : JSONState(std::move(p))
+        JSONListState(unique_ptr<JSONState>&& p, std::size_t reserve) : JSONState(std::move(p))
         {
             values.reserve(reserve);
         }
     };
 
     EvalState & state;
-    std::unique_ptr<JSONState> rs;
+    unique_ptr<JSONState> rs;
 
     template<typename T, typename... Args> inline bool handle_value(T f, Args... args)
     {
@@ -106,12 +107,12 @@ public:
         return handle_value(mkInt, val);
     }
 
-    bool number_float(number_float_t val, const string_t & s)
+    bool number_float(number_float_t val, const string_t& s)
     {
         return handle_value(mkFloat, val);
     }
 
-    bool string(string_t & val)
+    bool string(string_t& val)
     {
         return handle_value<void(Value&, const char*)>(mkString, val.c_str());
     }
@@ -122,7 +123,7 @@ public:
         return true;
     }
 
-    bool key(string_t & name)
+    bool key(string_t& name)
     {
         dynamic_cast<JSONObjectState*>(rs.get())->key(name, state);
         return true;

src/libexpr/lexer.l

@@ -127,14 +127,14 @@ or          { return OR_KW; }
               try {
                   yylval->n = boost::lexical_cast<int64_t>(yytext);
               } catch (const boost::bad_lexical_cast &) {
-                  throw ParseError("invalid integer '%1%'", yytext);
+                  throw ParseError(format("invalid integer '%1%'") % yytext);
               }
               return INT;
             }
 {FLOAT}     { errno = 0;
               yylval->nf = strtod(yytext, 0);
               if (errno != 0)
-                  throw ParseError("invalid float '%1%'", yytext);
+                  throw ParseError(format("invalid float '%1%'") % yytext);
               return FLOAT;
             }
 
@@ -219,3 +219,4 @@ or          { return OR_KW; }
             }
 
 %%
+

src/libexpr/local.mk

@@ -6,9 +6,9 @@ libexpr_DIR := $(d)
 
 libexpr_SOURCES := $(wildcard $(d)/*.cc) $(wildcard $(d)/primops/*.cc) $(d)/lexer-tab.cc $(d)/parser-tab.cc
 
-libexpr_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/libmain -I src/libexpr
+libexpr_CXXFLAGS += -I src/libutil -I src/libstore -I src/libmain -I src/libexpr
 
-libexpr_LIBS = libutil libstore libfetchers
+libexpr_LIBS = libutil libstore libnixrust
 
 libexpr_LDFLAGS =
 ifneq ($(OS), FreeBSD)

src/libexpr/nixexpr.cc

@@ -197,22 +197,7 @@ std::ostream & operator << (std::ostream & str, const Pos & pos)
     if (!pos)
         str << "undefined position";
     else
-    {
-        auto f = format(ANSI_BOLD "%1%" ANSI_NORMAL ":%2%:%3%");
-        switch (pos.origin) {
-            case foFile:
-                f % (string) pos.file;
-                break;
-            case foStdin:
-            case foString:
-                f % "(string)";
-                break;
-            default:
-                throw Error("unhandled Pos origin!");
-        }
-        str << (f % pos.line % pos.column).str();
-    }
-
+        str << (format(ANSI_BOLD "%1%" ANSI_NORMAL ":%2%:%3%") % (string) pos.file % pos.line % pos.column).str();
     return str;
 }
 
@@ -282,11 +267,8 @@ void ExprVar::bindVars(const StaticEnv & env)
     /* Otherwise, the variable must be obtained from the nearest
        enclosing `with'.  If there is no `with', then we can issue an
        "undefined variable" error now. */
-    if (withLevel == -1)
-        throw UndefinedVarError({
-            .hint = hintfmt("undefined variable '%1%'", name),
-            .errPos = pos
-        });
+    if (withLevel == -1) throw UndefinedVarError(format("undefined variable '%1%' at %2%") % name % pos);
+
     fromWith = true;
     this->level = withLevel;
 }

src/libexpr/nixexpr.hh

@@ -2,7 +2,6 @@
 
 #include "value.hh"
 #include "symbol-table.hh"
-#include "error.hh"
 
 #include <map>
 
@@ -24,12 +23,11 @@ MakeError(RestrictedPathError, Error);
 
 struct Pos
 {
-    FileOrigin origin;
     Symbol file;
     unsigned int line, column;
-    Pos() : origin(foString), line(0), column(0) { };
-    Pos(FileOrigin origin, const Symbol & file, unsigned int line, unsigned int column)
-        : origin(origin), file(file), line(line), column(column) { };
+    Pos() : line(0), column(0) { };
+    Pos(const Symbol & file, unsigned int line, unsigned int column)
+        : file(file), line(line), column(column) { };
     operator bool() const
     {
         return line != 0;
@@ -211,10 +209,9 @@ struct ExprList : Expr
 
 struct Formal
 {
-    Pos pos;
     Symbol name;
     Expr * def;
-    Formal(const Pos & pos, const Symbol & name, Expr * def) : pos(pos), name(name), def(def) { };
+    Formal(const Symbol & name, Expr * def) : name(name), def(def) { };
 };
 
 struct Formals
@@ -237,10 +234,8 @@ struct ExprLambda : Expr
         : pos(pos), arg(arg), matchAttrs(matchAttrs), formals(formals), body(body)
     {
         if (!arg.empty() && formals && formals->argNames.find(arg) != formals->argNames.end())
-            throw ParseError({
-                .hint = hintfmt("duplicate formal function argument '%1%'", arg),
-                .errPos = pos
-            });
+            throw ParseError(format("duplicate formal function argument '%1%' at %2%")
+                % arg % pos);
     };
     void setName(Symbol & name);
     string showNamePos() const;
@@ -266,9 +261,8 @@ struct ExprWith : Expr
 
 struct ExprIf : Expr
 {
-    Pos pos;
     Expr * cond, * then, * else_;
-    ExprIf(const Pos & pos, Expr * cond, Expr * then, Expr * else_) : pos(pos), cond(cond), then(then), else_(else_) { };
+    ExprIf(Expr * cond, Expr * then, Expr * else_) : cond(cond), then(then), else_(else_) { };
     COMMON_METHODS
 };
 

src/libexpr/parser.y

@@ -30,9 +30,8 @@ namespace nix {
         SymbolTable & symbols;
         Expr * result;
         Path basePath;
-        Symbol file;
-        FileOrigin origin;
-        ErrorInfo error;
+        Symbol path;
+        string error;
         Symbol sLetBody;
         ParseData(EvalState & state)
             : state(state)
@@ -65,19 +64,15 @@ namespace nix {
 
 static void dupAttr(const AttrPath & attrPath, const Pos & pos, const Pos & prevPos)
 {
-    throw ParseError({
-         .hint = hintfmt("attribute '%1%' already defined at %2%",
-            showAttrPath(attrPath), prevPos),
-         .errPos = pos
-    });
+    throw ParseError(format("attribute '%1%' at %2% already defined at %3%")
+        % showAttrPath(attrPath) % pos % prevPos);
 }
 
+
 static void dupAttr(Symbol attr, const Pos & pos, const Pos & prevPos)
 {
-    throw ParseError({
-        .hint = hintfmt("attribute '%1%' already defined at %2%", attr, prevPos),
-        .errPos = pos
-    });
+    throw ParseError(format("attribute '%1%' at %2% already defined at %3%")
+        % attr % pos % prevPos);
 }
 
 
@@ -145,11 +140,8 @@ static void addAttr(ExprAttrs * attrs, AttrPath & attrPath,
 static void addFormal(const Pos & pos, Formals * formals, const Formal & formal)
 {
     if (!formals->argNames.insert(formal.name).second)
-        throw ParseError({
-            .hint = hintfmt("duplicate formal function argument '%1%'",
-                formal.name),
-            .errPos = pos
-        });
+        throw ParseError(format("duplicate formal function argument '%1%' at %2%")
+            % formal.name % pos);
     formals->formals.push_front(formal);
 }
 
@@ -246,7 +238,7 @@ static Expr * stripIndentation(const Pos & pos, SymbolTable & symbols, vector<Ex
 
 static inline Pos makeCurPos(const YYLTYPE & loc, ParseData * data)
 {
-    return Pos(data->origin, data->file, loc.first_line, loc.first_column);
+    return Pos(data->path, loc.first_line, loc.first_column);
 }
 
 #define CUR_POS makeCurPos(*yylocp, data)
@@ -257,10 +249,8 @@ static inline Pos makeCurPos(const YYLTYPE & loc, ParseData * data)
 
 void yyerror(YYLTYPE * loc, yyscan_t scanner, ParseData * data, const char * error)
 {
-    data->error = {
-        .hint = hintfmt(error),
-        .errPos = makeCurPos(*loc, data)
-    };
+    data->error = (format("%1%, at %2%")
+        % error % makeCurPos(*loc, data)).str();
 }
 
 
@@ -337,17 +327,15 @@ expr_function
     { $$ = new ExprWith(CUR_POS, $2, $4); }
   | LET binds IN expr_function
     { if (!$2->dynamicAttrs.empty())
-        throw ParseError({
-            .hint = hintfmt("dynamic attributes not allowed in let"),
-            .errPos = CUR_POS
-        });
+        throw ParseError(format("dynamic attributes not allowed in let at %1%")
+            % CUR_POS);
       $$ = new ExprLet($2, $4);
     }
   | expr_if
   ;
 
 expr_if
-  : IF expr THEN expr ELSE expr { $$ = new ExprIf(CUR_POS, $2, $4, $6); }
+  : IF expr THEN expr ELSE expr { $$ = new ExprIf($2, $4, $6); }
   | expr_op
   ;
 
@@ -417,10 +405,7 @@ expr_simple
   | URI {
       static bool noURLLiterals = settings.isExperimentalFeatureEnabled("no-url-literals");
       if (noURLLiterals)
-          throw ParseError({
-              .hint = hintfmt("URL literals are disabled"),
-              .errPos = CUR_POS
-          });
+          throw ParseError("URL literals are disabled, at %s", CUR_POS);
       $$ = new ExprString(data->symbols.create($1));
   }
   | '(' expr ')' { $$ = $2; }
@@ -490,10 +475,8 @@ attrs
           $$->push_back(AttrName(str->s));
           delete str;
       } else
-          throw ParseError({
-              .hint = hintfmt("dynamic attributes not allowed in inherit"),
-              .errPos = makeCurPos(@2, data)
-          });
+          throw ParseError(format("dynamic attributes not allowed in inherit at %1%")
+              % makeCurPos(@2, data));
     }
   | { $$ = new AttrPath; }
   ;
@@ -548,8 +531,8 @@ formals
   ;
 
 formal
-  : ID { $$ = new Formal(CUR_POS, data->symbols.create($1), 0); }
-  | ID '?' expr { $$ = new Formal(CUR_POS, data->symbols.create($1), $3); }
+  : ID { $$ = new Formal(data->symbols.create($1), 0); }
+  | ID '?' expr { $$ = new Formal(data->symbols.create($1), $3); }
   ;
 
 %%
@@ -561,32 +544,20 @@ formal
 #include <unistd.h>
 
 #include "eval.hh"
-#include "filetransfer.hh"
-#include "fetchers.hh"
+#include "download.hh"
 #include "store-api.hh"
 
 
 namespace nix {
 
 
-Expr * EvalState::parse(const char * text, FileOrigin origin,
+Expr * EvalState::parse(const char * text,
     const Path & path, const Path & basePath, StaticEnv & staticEnv)
 {
     yyscan_t scanner;
     ParseData data(*this);
-    data.origin = origin;
-    switch (origin) {
-        case foFile: 
-            data.file = data.symbols.create(path);
-            break;
-        case foStdin:
-        case foString:
-            data.file = data.symbols.create(text);
-            break;
-        default:
-            assert(false);
-    }
     data.basePath = basePath;
+    data.path = data.symbols.create(path);
 
     yylex_init(&scanner);
     yy_scan_string(text, scanner);
@@ -636,13 +607,13 @@ Expr * EvalState::parseExprFromFile(const Path & path)
 
 Expr * EvalState::parseExprFromFile(const Path & path, StaticEnv & staticEnv)
 {
-    return parse(readFile(path).c_str(), foFile, path, dirOf(path), staticEnv);
+    return parse(readFile(path).c_str(), path, dirOf(path), staticEnv);
 }
 
 
 Expr * EvalState::parseExprFromString(std::string_view s, const Path & basePath, StaticEnv & staticEnv)
 {
-    return parse(s.data(), foString, "", basePath, staticEnv);
+    return parse(s.data(), "(string)", basePath, staticEnv);
 }
 
 
@@ -655,7 +626,7 @@ Expr * EvalState::parseExprFromString(std::string_view s, const Path & basePath)
 Expr * EvalState::parseStdin()
 {
     //Activity act(*logger, lvlTalkative, format("parsing standard input"));
-    return parse(drainFD(0).data(), foStdin, "", absPath("."), staticBaseEnv);
+    return parseExprFromString(drainFD(0), absPath("."));
 }
 
 
@@ -699,13 +670,11 @@ Path EvalState::findFile(SearchPath & searchPath, const string & path, const Pos
         Path res = r.second + suffix;
         if (pathExists(res)) return canonPath(res);
     }
-    throw ThrownError({
-        .hint = hintfmt(evalSettings.pureEval
-            ? "cannot look up '<%s>' in pure evaluation mode (use '--impure' to override)"
-            : "file '%s' was not found in the Nix search path (add it using $NIX_PATH or -I)",
-            path),
-        .errPos = pos
-    });
+    format f = format(
+        "file '%1%' was not found in the Nix search path (add it using $NIX_PATH or -I)"
+        + string(pos ? ", at %2%" : ""));
+    f.exceptions(boost::io::all_error_bits ^ boost::io::too_many_args_bit);
+    throw ThrownError(f % path % pos);
 }
 
 
@@ -718,13 +687,11 @@ std::pair<bool, std::string> EvalState::resolveSearchPathElem(const SearchPathEl
 
     if (isUri(elem.second)) {
         try {
-            res = { true, store->toRealPath(fetchers::downloadTarball(
-                        store, resolveUri(elem.second), "source", false).storePath) };
-        } catch (FileTransferError & e) {
-            logWarning({
-                .name = "Entry download",
-                .hint = hintfmt("Nix search path entry '%1%' cannot be downloaded, ignoring", elem.second)
-            });
+            CachedDownloadRequest request(elem.second);
+            request.unpack = true;
+            res = { true, getDownloader()->downloadCached(store, request).path };
+        } catch (DownloadError & e) {
+            printError(format("warning: Nix search path entry '%1%' cannot be downloaded, ignoring") % elem.second);
             res = { false, "" };
         }
     } else {
@@ -732,10 +699,7 @@ std::pair<bool, std::string> EvalState::resolveSearchPathElem(const SearchPathEl
         if (pathExists(path))
             res = { true, path };
         else {
-            logWarning({
-                .name = "Entry not found",
-                .hint = hintfmt("warning: Nix search path entry '%1%' does not exist, ignoring", elem.second)
-            });
+            printError(format("warning: Nix search path entry '%1%' does not exist, ignoring") % elem.second);
             res = { false, "" };
         }
     }

src/libexpr/primops.hh

@@ -7,25 +7,12 @@ namespace nix {
 
 struct RegisterPrimOp
 {
-    struct Info
-    {
-        std::string name;
-        size_t arity;
-        PrimOpFun primOp;
-        std::optional<std::string> requiredFeature;
-    };
-
-    typedef std::vector<Info> PrimOps;
+    typedef std::vector<std::tuple<std::string, size_t, PrimOpFun>> PrimOps;
     static PrimOps * primOps;
-
     /* You can register a constant by passing an arity of 0. fun
        will get called during EvalState initialization, so there
        may be primops not yet added and builtins is not yet sorted. */
-    RegisterPrimOp(
-        std::string name,
-        size_t arity,
-        PrimOpFun fun,
-        std::optional<std::string> requiredFeature = {});
+    RegisterPrimOp(std::string name, size_t arity, PrimOpFun fun);
 };
 
 /* These primops are disabled without enableNativeCode, but plugins
@@ -33,7 +20,6 @@ struct RegisterPrimOp
    them. */
 /* Load a ValueInitializer from a DSO and return whatever it initializes */
 void prim_importNative(EvalState & state, const Pos & pos, Value * * args, Value & v);
-
 /* Execute a program and parse its output */
 void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v);
 

src/libexpr/primops/context.cc

@@ -1,6 +1,6 @@
 #include "primops.hh"
 #include "eval-inline.hh"
-#include "store-api.hh"
+#include "derivations.hh"
 
 namespace nix {
 
@@ -146,10 +146,7 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg
     auto sAllOutputs = state.symbols.create("allOutputs");
     for (auto & i : *args[1]->attrs) {
         if (!state.store->isStorePath(i.name))
-            throw EvalError({
-                .hint = hintfmt("Context key '%s' is not a store path", i.name),
-                .errPos = *i.pos
-            });
+            throw EvalError("Context key '%s' is not a store path, at %s", i.name, i.pos);
         if (!settings.readOnlyMode)
             state.store->ensurePath(state.store->parseStorePath(i.name));
         state.forceAttrs(*i.value, *i.pos);
@@ -163,10 +160,7 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg
         if (iter != i.value->attrs->end()) {
             if (state.forceBool(*iter->value, *iter->pos)) {
                 if (!isDerivation(i.name)) {
-                    throw EvalError({
-                        .hint = hintfmt("Tried to add all-outputs context of %s, which is not a derivation, to a string", i.name),
-                        .errPos = *i.pos
-                    });
+                    throw EvalError("Tried to add all-outputs context of %s, which is not a derivation, to a string, at %s", i.name, i.pos);
                 }
                 context.insert("=" + string(i.name));
             }
@@ -176,10 +170,7 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg
         if (iter != i.value->attrs->end()) {
             state.forceList(*iter->value, *iter->pos);
             if (iter->value->listSize() && !isDerivation(i.name)) {
-                throw EvalError({
-                    .hint = hintfmt("Tried to add derivation output context of %s, which is not a derivation, to a string", i.name),
-                    .errPos = *i.pos
-                });
+                throw EvalError("Tried to add derivation output context of %s, which is not a derivation, to a string, at %s", i.name, i.pos);
             }
             for (unsigned int n = 0; n < iter->value->listSize(); ++n) {
                 auto name = state.forceStringNoCtx(*iter->value->listElems()[n], *iter->pos);

src/libexpr/primops/fetchGit.cc

@@ -1,19 +1,203 @@
 #include "primops.hh"
 #include "eval-inline.hh"
+#include "download.hh"
 #include "store-api.hh"
+#include "pathlocks.hh"
 #include "hash.hh"
-#include "fetchers.hh"
-#include "url.hh"
+#include "tarfile.hh"
+
+#include <sys/time.h>
+
+#include <regex>
+
+#include <nlohmann/json.hpp>
+
+using namespace std::string_literals;
 
 namespace nix {
 
+struct GitInfo
+{
+    Path storePath;
+    std::string rev;
+    std::string shortRev;
+    uint64_t revCount = 0;
+};
+
+std::regex revRegex("^[0-9a-fA-F]{40}$");
+
+GitInfo exportGit(ref<Store> store, const std::string & uri,
+    std::optional<std::string> ref, std::string rev,
+    const std::string & name)
+{
+    if (evalSettings.pureEval && rev == "")
+        throw Error("in pure evaluation mode, 'fetchGit' requires a Git revision");
+
+    if (!ref && rev == "" && hasPrefix(uri, "/") && pathExists(uri + "/.git")) {
+
+        bool clean = true;
+
+        try {
+            runProgram("git", true, { "-C", uri, "diff-index", "--quiet", "HEAD", "--" });
+        } catch (ExecError & e) {
+            if (!WIFEXITED(e.status) || WEXITSTATUS(e.status) != 1) throw;
+            clean = false;
+        }
+
+        if (!clean) {
+
+            /* This is an unclean working tree. So copy all tracked files. */
+            GitInfo gitInfo;
+            gitInfo.rev = "0000000000000000000000000000000000000000";
+            gitInfo.shortRev = std::string(gitInfo.rev, 0, 7);
+
+            auto files = tokenizeString<std::set<std::string>>(
+                runProgram("git", true, { "-C", uri, "ls-files", "-z" }), "\0"s);
+
+            PathFilter filter = [&](const Path & p) -> bool {
+                assert(hasPrefix(p, uri));
+                std::string file(p, uri.size() + 1);
+
+                auto st = lstat(p);
+
+                if (S_ISDIR(st.st_mode)) {
+                    auto prefix = file + "/";
+                    auto i = files.lower_bound(prefix);
+                    return i != files.end() && hasPrefix(*i, prefix);
+                }
+
+                return files.count(file);
+            };
+
+            gitInfo.storePath = store->printStorePath(store->addToStore("source", uri, true, htSHA256, filter));
+
+            return gitInfo;
+        }
+
+        // clean working tree, but no ref or rev specified.  Use 'HEAD'.
+        rev = chomp(runProgram("git", true, { "-C", uri, "rev-parse", "HEAD" }));
+        ref = "HEAD"s;
+    }
+
+    if (!ref) ref = "HEAD"s;
+
+    if (rev != "" && !std::regex_match(rev, revRegex))
+        throw Error("invalid Git revision '%s'", rev);
+
+    deletePath(getCacheDir() + "/nix/git");
+
+    Path cacheDir = getCacheDir() + "/nix/gitv2/" + hashString(htSHA256, uri).to_string(Base32, false);
+
+    if (!pathExists(cacheDir)) {
+        createDirs(dirOf(cacheDir));
+        runProgram("git", true, { "init", "--bare", cacheDir });
+    }
+
+    Path localRefFile;
+    if (ref->compare(0, 5, "refs/") == 0)
+        localRefFile = cacheDir + "/" + *ref;
+    else
+        localRefFile = cacheDir + "/refs/heads/" + *ref;
+
+    bool doFetch;
+    time_t now = time(0);
+    /* If a rev was specified, we need to fetch if it's not in the
+       repo. */
+    if (rev != "") {
+        try {
+            runProgram("git", true, { "-C", cacheDir, "cat-file", "-e", rev });
+            doFetch = false;
+        } catch (ExecError & e) {
+            if (WIFEXITED(e.status)) {
+                doFetch = true;
+            } else {
+                throw;
+            }
+        }
+    } else {
+        /* If the local ref is older than ‘tarball-ttl’ seconds, do a
+           git fetch to update the local ref to the remote ref. */
+        struct stat st;
+        doFetch = stat(localRefFile.c_str(), &st) != 0 ||
+            (uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now;
+    }
+    if (doFetch)
+    {
+        Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Git repository '%s'", uri));
+
+        // FIXME: git stderr messes up our progress indicator, so
+        // we're using --quiet for now. Should process its stderr.
+        runProgram("git", true, { "-C", cacheDir, "fetch", "--quiet", "--force", "--", uri, fmt("%s:%s", *ref, *ref) });
+
+        struct timeval times[2];
+        times[0].tv_sec = now;
+        times[0].tv_usec = 0;
+        times[1].tv_sec = now;
+        times[1].tv_usec = 0;
+
+        utimes(localRefFile.c_str(), times);
+    }
+
+    // FIXME: check whether rev is an ancestor of ref.
+    GitInfo gitInfo;
+    gitInfo.rev = rev != "" ? rev : chomp(readFile(localRefFile));
+    gitInfo.shortRev = std::string(gitInfo.rev, 0, 7);
+
+    printTalkative("using revision %s of repo '%s'", gitInfo.rev, uri);
+
+    std::string storeLinkName = hashString(htSHA512, name + std::string("\0"s) + gitInfo.rev).to_string(Base32, false);
+    Path storeLink = cacheDir + "/" + storeLinkName + ".link";
+    PathLocks storeLinkLock({storeLink}, fmt("waiting for lock on '%1%'...", storeLink)); // FIXME: broken
+
+    try {
+        auto json = nlohmann::json::parse(readFile(storeLink));
+
+        assert(json["name"] == name && json["rev"] == gitInfo.rev);
+
+        gitInfo.storePath = json["storePath"];
+
+        if (store->isValidPath(store->parseStorePath(gitInfo.storePath))) {
+            gitInfo.revCount = json["revCount"];
+            return gitInfo;
+        }
+
+    } catch (SysError & e) {
+        if (e.errNo != ENOENT) throw;
+    }
+
+    auto source = sinkToSource([&](Sink & sink) {
+        RunOptions gitOptions("git", { "-C", cacheDir, "archive", gitInfo.rev });
+        gitOptions.standardOut = &sink;
+        runProgram2(gitOptions);
+    });
+
+    Path tmpDir = createTempDir();
+    AutoDelete delTmpDir(tmpDir, true);
+
+    unpackTarfile(*source, tmpDir);
+
+    gitInfo.storePath = store->printStorePath(store->addToStore(name, tmpDir));
+
+    gitInfo.revCount = std::stoull(runProgram("git", true, { "-C", cacheDir, "rev-list", "--count", gitInfo.rev }));
+
+    nlohmann::json json;
+    json["storePath"] = gitInfo.storePath;
+    json["uri"] = uri;
+    json["name"] = name;
+    json["rev"] = gitInfo.rev;
+    json["revCount"] = gitInfo.revCount;
+
+    writeFile(storeLink, json.dump());
+
+    return gitInfo;
+}
+
 static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     std::string url;
     std::optional<std::string> ref;
-    std::optional<Hash> rev;
+    std::string rev;
     std::string name = "source";
-    bool fetchSubmodules = false;
     PathSet context;
 
     state.forceValue(*args[0]);
@@ -29,23 +213,15 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va
             else if (n == "ref")
                 ref = state.forceStringNoCtx(*attr.value, *attr.pos);
             else if (n == "rev")
-                rev = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA1);
+                rev = state.forceStringNoCtx(*attr.value, *attr.pos);
             else if (n == "name")
                 name = state.forceStringNoCtx(*attr.value, *attr.pos);
-            else if (n == "submodules")
-                fetchSubmodules = state.forceBool(*attr.value, *attr.pos);
             else
-                throw EvalError({
-                    .hint = hintfmt("unsupported argument '%s' to 'fetchGit'", attr.name),
-                    .errPos = *attr.pos
-                });
+                throw EvalError("unsupported argument '%s' to 'fetchGit', at %s", attr.name, *attr.pos);
         }
 
         if (url.empty())
-            throw EvalError({
-                .hint = hintfmt("'url' argument required"),
-                .errPos = pos
-            });
+            throw EvalError(format("'url' argument required, at %1%") % pos);
 
     } else
         url = state.coerceToString(pos, *args[0], context, false, false);
@@ -54,36 +230,17 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va
     // whitelist. Ah well.
     state.checkURI(url);
 
-    if (evalSettings.pureEval && !rev)
-        throw Error("in pure evaluation mode, 'fetchGit' requires a Git revision");
-
-    fetchers::Attrs attrs;
-    attrs.insert_or_assign("type", "git");
-    attrs.insert_or_assign("url", url.find("://") != std::string::npos ? url : "file://" + url);
-    if (ref) attrs.insert_or_assign("ref", *ref);
-    if (rev) attrs.insert_or_assign("rev", rev->gitRev());
-    if (fetchSubmodules) attrs.insert_or_assign("submodules", true);
-    auto input = fetchers::inputFromAttrs(attrs);
-
-    // FIXME: use name?
-    auto [tree, input2] = input->fetchTree(state.store);
+    auto gitInfo = exportGit(state.store, url, ref, rev, name);
 
     state.mkAttrs(v, 8);
-    auto storePath = state.store->printStorePath(tree.storePath);
-    mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath}));
-    // Backward compatibility: set 'rev' to
-    // 0000000000000000000000000000000000000000 for a dirty tree.
-    auto rev2 = input2->getRev().value_or(Hash(htSHA1));
-    mkString(*state.allocAttr(v, state.symbols.create("rev")), rev2.gitRev());
-    mkString(*state.allocAttr(v, state.symbols.create("shortRev")), rev2.gitShortRev());
-    // Backward compatibility: set 'revCount' to 0 for a dirty tree.
-    mkInt(*state.allocAttr(v, state.symbols.create("revCount")),
-        tree.info.revCount.value_or(0));
-    mkBool(*state.allocAttr(v, state.symbols.create("submodules")), fetchSubmodules);
+    mkString(*state.allocAttr(v, state.sOutPath), gitInfo.storePath, PathSet({gitInfo.storePath}));
+    mkString(*state.allocAttr(v, state.symbols.create("rev")), gitInfo.rev);
+    mkString(*state.allocAttr(v, state.symbols.create("shortRev")), gitInfo.shortRev);
+    mkInt(*state.allocAttr(v, state.symbols.create("revCount")), gitInfo.revCount);
     v.attrs->sort();
 
     if (state.allowedPaths)
-        state.allowedPaths->insert(tree.actualPath);
+        state.allowedPaths->insert(state.store->toRealPath(gitInfo.storePath));
 }
 
 static RegisterPrimOp r("fetchGit", 1, prim_fetchGit);

src/libexpr/primops/fetchMercurial.cc

@@ -1,18 +1,174 @@
 #include "primops.hh"
 #include "eval-inline.hh"
+#include "download.hh"
 #include "store-api.hh"
-#include "fetchers.hh"
-#include "url.hh"
+#include "pathlocks.hh"
+
+#include <sys/time.h>
 
 #include <regex>
 
+#include <nlohmann/json.hpp>
+
+using namespace std::string_literals;
+
 namespace nix {
 
+struct HgInfo
+{
+    Path storePath;
+    std::string branch;
+    std::string rev;
+    uint64_t revCount = 0;
+};
+
+std::regex commitHashRegex("^[0-9a-fA-F]{40}$");
+
+HgInfo exportMercurial(ref<Store> store, const std::string & uri,
+    std::string rev, const std::string & name)
+{
+    if (evalSettings.pureEval && rev == "")
+        throw Error("in pure evaluation mode, 'fetchMercurial' requires a Mercurial revision");
+
+    if (rev == "" && hasPrefix(uri, "/") && pathExists(uri + "/.hg")) {
+
+        bool clean = runProgram("hg", true, { "status", "-R", uri, "--modified", "--added", "--removed" }) == "";
+
+        if (!clean) {
+
+            /* This is an unclean working tree. So copy all tracked
+               files. */
+
+            printTalkative("copying unclean Mercurial working tree '%s'", uri);
+
+            HgInfo hgInfo;
+            hgInfo.rev = "0000000000000000000000000000000000000000";
+            hgInfo.branch = chomp(runProgram("hg", true, { "branch", "-R", uri }));
+
+            auto files = tokenizeString<std::set<std::string>>(
+                runProgram("hg", true, { "status", "-R", uri, "--clean", "--modified", "--added", "--no-status", "--print0" }), "\0"s);
+
+            PathFilter filter = [&](const Path & p) -> bool {
+                assert(hasPrefix(p, uri));
+                std::string file(p, uri.size() + 1);
+
+                auto st = lstat(p);
+
+                if (S_ISDIR(st.st_mode)) {
+                    auto prefix = file + "/";
+                    auto i = files.lower_bound(prefix);
+                    return i != files.end() && hasPrefix(*i, prefix);
+                }
+
+                return files.count(file);
+            };
+
+            hgInfo.storePath = store->printStorePath(store->addToStore("source", uri, true, htSHA256, filter));
+
+            return hgInfo;
+        }
+    }
+
+    if (rev == "") rev = "default";
+
+    Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), hashString(htSHA256, uri).to_string(Base32, false));
+
+    Path stampFile = fmt("%s/.hg/%s.stamp", cacheDir, hashString(htSHA512, rev).to_string(Base32, false));
+
+    /* If we haven't pulled this repo less than ‘tarball-ttl’ seconds,
+       do so now. */
+    time_t now = time(0);
+    struct stat st;
+    if (stat(stampFile.c_str(), &st) != 0 ||
+        (uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now)
+    {
+        /* Except that if this is a commit hash that we already have,
+           we don't have to pull again. */
+        if (!(std::regex_match(rev, commitHashRegex)
+                && pathExists(cacheDir)
+                && runProgram(
+                    RunOptions("hg", { "log", "-R", cacheDir, "-r", rev, "--template", "1" })
+                    .killStderr(true)).second == "1"))
+        {
+            Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Mercurial repository '%s'", uri));
+
+            if (pathExists(cacheDir)) {
+                try {
+                    runProgram("hg", true, { "pull", "-R", cacheDir, "--", uri });
+                }
+                catch (ExecError & e) {
+                    string transJournal = cacheDir + "/.hg/store/journal";
+                    /* hg throws "abandoned transaction" error only if this file exists */
+                    if (pathExists(transJournal)) {
+                        runProgram("hg", true, { "recover", "-R", cacheDir });
+                        runProgram("hg", true, { "pull", "-R", cacheDir, "--", uri });
+                    } else {
+                        throw ExecError(e.status, fmt("'hg pull' %s", statusToString(e.status)));
+                    }
+                }
+            } else {
+                createDirs(dirOf(cacheDir));
+                runProgram("hg", true, { "clone", "--noupdate", "--", uri, cacheDir });
+            }
+        }
+
+        writeFile(stampFile, "");
+    }
+
+    auto tokens = tokenizeString<std::vector<std::string>>(
+        runProgram("hg", true, { "log", "-R", cacheDir, "-r", rev, "--template", "{node} {rev} {branch}" }));
+    assert(tokens.size() == 3);
+
+    HgInfo hgInfo;
+    hgInfo.rev = tokens[0];
+    hgInfo.revCount = std::stoull(tokens[1]);
+    hgInfo.branch = tokens[2];
+
+    std::string storeLinkName = hashString(htSHA512, name + std::string("\0"s) + hgInfo.rev).to_string(Base32, false);
+    Path storeLink = fmt("%s/.hg/%s.link", cacheDir, storeLinkName);
+
+    try {
+        auto json = nlohmann::json::parse(readFile(storeLink));
+
+        assert(json["name"] == name && json["rev"] == hgInfo.rev);
+
+        hgInfo.storePath = json["storePath"];
+
+        if (store->isValidPath(store->parseStorePath(hgInfo.storePath))) {
+            printTalkative("using cached Mercurial store path '%s'", hgInfo.storePath);
+            return hgInfo;
+        }
+
+    } catch (SysError & e) {
+        if (e.errNo != ENOENT) throw;
+    }
+
+    Path tmpDir = createTempDir();
+    AutoDelete delTmpDir(tmpDir, true);
+
+    runProgram("hg", true, { "archive", "-R", cacheDir, "-r", rev, tmpDir });
+
+    deletePath(tmpDir + "/.hg_archival.txt");
+
+    hgInfo.storePath = store->printStorePath(store->addToStore(name, tmpDir));
+
+    nlohmann::json json;
+    json["storePath"] = hgInfo.storePath;
+    json["uri"] = uri;
+    json["name"] = name;
+    json["branch"] = hgInfo.branch;
+    json["rev"] = hgInfo.rev;
+    json["revCount"] = hgInfo.revCount;
+
+    writeFile(storeLink, json.dump());
+
+    return hgInfo;
+}
+
 static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     std::string url;
-    std::optional<Hash> rev;
-    std::optional<std::string> ref;
+    std::string rev;
     std::string name = "source";
     PathSet context;
 
@@ -26,29 +182,16 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar
             string n(attr.name);
             if (n == "url")
                 url = state.coerceToString(*attr.pos, *attr.value, context, false, false);
-            else if (n == "rev") {
-                // Ugly: unlike fetchGit, here the "rev" attribute can
-                // be both a revision or a branch/tag name.
-                auto value = state.forceStringNoCtx(*attr.value, *attr.pos);
-                if (std::regex_match(value, revRegex))
-                    rev = Hash(value, htSHA1);
-                else
-                    ref = value;
-            }
+            else if (n == "rev")
+                rev = state.forceStringNoCtx(*attr.value, *attr.pos);
             else if (n == "name")
                 name = state.forceStringNoCtx(*attr.value, *attr.pos);
             else
-                throw EvalError({
-                    .hint = hintfmt("unsupported argument '%s' to 'fetchMercurial'", attr.name),
-                    .errPos = *attr.pos
-                });
+                throw EvalError("unsupported argument '%s' to 'fetchMercurial', at %s", attr.name, *attr.pos);
         }
 
         if (url.empty())
-            throw EvalError({
-                .hint = hintfmt("'url' argument required"),
-                .errPos = pos
-            });
+            throw EvalError(format("'url' argument required, at %1%") % pos);
 
     } else
         url = state.coerceToString(pos, *args[0], context, false, false);
@@ -57,35 +200,18 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar
     // whitelist. Ah well.
     state.checkURI(url);
 
-    if (evalSettings.pureEval && !rev)
-        throw Error("in pure evaluation mode, 'fetchMercurial' requires a Mercurial revision");
-
-    fetchers::Attrs attrs;
-    attrs.insert_or_assign("type", "hg");
-    attrs.insert_or_assign("url", url.find("://") != std::string::npos ? url : "file://" + url);
-    if (ref) attrs.insert_or_assign("ref", *ref);
-    if (rev) attrs.insert_or_assign("rev", rev->gitRev());
-    auto input = fetchers::inputFromAttrs(attrs);
-
-    // FIXME: use name
-    auto [tree, input2] = input->fetchTree(state.store);
+    auto hgInfo = exportMercurial(state.store, url, rev, name);
 
     state.mkAttrs(v, 8);
-    auto storePath = state.store->printStorePath(tree.storePath);
-    mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath}));
-    if (input2->getRef())
-        mkString(*state.allocAttr(v, state.symbols.create("branch")), *input2->getRef());
-    // Backward compatibility: set 'rev' to
-    // 0000000000000000000000000000000000000000 for a dirty tree.
-    auto rev2 = input2->getRev().value_or(Hash(htSHA1));
-    mkString(*state.allocAttr(v, state.symbols.create("rev")), rev2.gitRev());
-    mkString(*state.allocAttr(v, state.symbols.create("shortRev")), std::string(rev2.gitRev(), 0, 12));
-    if (tree.info.revCount)
-        mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *tree.info.revCount);
+    mkString(*state.allocAttr(v, state.sOutPath), hgInfo.storePath, PathSet({hgInfo.storePath}));
+    mkString(*state.allocAttr(v, state.symbols.create("branch")), hgInfo.branch);
+    mkString(*state.allocAttr(v, state.symbols.create("rev")), hgInfo.rev);
+    mkString(*state.allocAttr(v, state.symbols.create("shortRev")), std::string(hgInfo.rev, 0, 12));
+    mkInt(*state.allocAttr(v, state.symbols.create("revCount")), hgInfo.revCount);
     v.attrs->sort();
 
     if (state.allowedPaths)
-        state.allowedPaths->insert(tree.actualPath);
+        state.allowedPaths->insert(state.store->toRealPath(hgInfo.storePath));
 }
 
 static RegisterPrimOp r("fetchMercurial", 1, prim_fetchMercurial);

src/libexpr/primops/fetchTree.cc

@@ -1,172 +0,0 @@
-#include "primops.hh"
-#include "eval-inline.hh"
-#include "store-api.hh"
-#include "fetchers.hh"
-#include "filetransfer.hh"
-
-#include <ctime>
-#include <iomanip>
-
-namespace nix {
-
-void emitTreeAttrs(
-    EvalState & state,
-    const fetchers::Tree & tree,
-    std::shared_ptr<const fetchers::Input> input,
-    Value & v)
-{
-    state.mkAttrs(v, 8);
-
-    auto storePath = state.store->printStorePath(tree.storePath);
-
-    mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath}));
-
-    assert(tree.info.narHash);
-    mkString(*state.allocAttr(v, state.symbols.create("narHash")),
-        tree.info.narHash.to_string(SRI, true));
-
-    if (input->getRev()) {
-        mkString(*state.allocAttr(v, state.symbols.create("rev")), input->getRev()->gitRev());
-        mkString(*state.allocAttr(v, state.symbols.create("shortRev")), input->getRev()->gitShortRev());
-    }
-
-    if (tree.info.revCount)
-        mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *tree.info.revCount);
-
-    if (tree.info.lastModified)
-        mkString(*state.allocAttr(v, state.symbols.create("lastModified")),
-            fmt("%s", std::put_time(std::gmtime(&*tree.info.lastModified), "%Y%m%d%H%M%S")));
-
-    v.attrs->sort();
-}
-
-static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, Value & v)
-{
-    settings.requireExperimentalFeature("flakes");
-
-    std::shared_ptr<const fetchers::Input> input;
-    PathSet context;
-
-    state.forceValue(*args[0]);
-
-    if (args[0]->type == tAttrs) {
-        state.forceAttrs(*args[0], pos);
-
-        fetchers::Attrs attrs;
-
-        for (auto & attr : *args[0]->attrs) {
-            state.forceValue(*attr.value);
-            if (attr.value->type == tString)
-                attrs.emplace(attr.name, attr.value->string.s);
-            else if (attr.value->type == tBool)
-                attrs.emplace(attr.name, attr.value->boolean);
-            else
-                throw TypeError("fetchTree argument '%s' is %s while a string or Boolean is expected",
-                    attr.name, showType(*attr.value));
-        }
-
-        if (!attrs.count("type"))
-            throw Error({
-                .hint = hintfmt("attribute 'type' is missing in call to 'fetchTree'"),
-                .errPos = pos
-            });
-
-        input = fetchers::inputFromAttrs(attrs);
-    } else
-        input = fetchers::inputFromURL(state.coerceToString(pos, *args[0], context, false, false));
-
-    if (evalSettings.pureEval && !input->isImmutable())
-        throw Error("in pure evaluation mode, 'fetchTree' requires an immutable input");
-
-    // FIXME: use fetchOrSubstituteTree
-    auto [tree, input2] = input->fetchTree(state.store);
-
-    if (state.allowedPaths)
-        state.allowedPaths->insert(tree.actualPath);
-
-    emitTreeAttrs(state, tree, input2, v);
-}
-
-static RegisterPrimOp r("fetchTree", 1, prim_fetchTree);
-
-static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
-    const string & who, bool unpack, std::string name)
-{
-    std::optional<std::string> url;
-    std::optional<Hash> expectedHash;
-
-    state.forceValue(*args[0]);
-
-    if (args[0]->type == tAttrs) {
-
-        state.forceAttrs(*args[0], pos);
-
-        for (auto & attr : *args[0]->attrs) {
-            string n(attr.name);
-            if (n == "url")
-                url = state.forceStringNoCtx(*attr.value, *attr.pos);
-            else if (n == "sha256")
-                expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256);
-            else if (n == "name")
-                name = state.forceStringNoCtx(*attr.value, *attr.pos);
-            else
-                throw EvalError({
-                    .hint = hintfmt("unsupported argument '%s' to '%s'", attr.name, who),
-                    .errPos = *attr.pos
-                });
-            }
-
-        if (!url)
-            throw EvalError({
-                .hint = hintfmt("'url' argument required"),
-                .errPos = pos
-            });
-    } else
-        url = state.forceStringNoCtx(*args[0], pos);
-
-    url = resolveUri(*url);
-
-    state.checkURI(*url);
-
-    if (name == "")
-        name = baseNameOf(*url);
-
-    if (evalSettings.pureEval && !expectedHash)
-        throw Error("in pure evaluation mode, '%s' requires a 'sha256' argument", who);
-
-    auto storePath =
-        unpack
-        ? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).storePath
-        : fetchers::downloadFile(state.store, *url, name, (bool) expectedHash).storePath;
-
-    auto path = state.store->toRealPath(storePath);
-
-    if (expectedHash) {
-        auto hash = unpack
-            ? state.store->queryPathInfo(storePath)->narHash
-            : hashFile(htSHA256, path);
-        if (hash != *expectedHash)
-            throw Error((unsigned int) 102, "hash mismatch in file downloaded from '%s':\n  wanted: %s\n  got:    %s",
-                *url, expectedHash->to_string(Base32, true), hash.to_string(Base32, true));
-    }
-
-    if (state.allowedPaths)
-        state.allowedPaths->insert(path);
-
-    mkString(v, path, PathSet({path}));
-}
-
-static void prim_fetchurl(EvalState & state, const Pos & pos, Value * * args, Value & v)
-{
-    fetch(state, pos, args, v, "fetchurl", false, "");
-}
-
-static void prim_fetchTarball(EvalState & state, const Pos & pos, Value * * args, Value & v)
-{
-    fetch(state, pos, args, v, "fetchTarball", true, "source");
-}
-
-static RegisterPrimOp r2("__fetchurl", 1, prim_fetchurl);
-static RegisterPrimOp r3("fetchTarball", 1, prim_fetchTarball);
-
-}

src/libexpr/primops/fromTOML.cc

@@ -81,10 +81,7 @@ static void prim_fromTOML(EvalState & state, const Pos & pos, Value * * args, Va
     try {
         visit(v, parser(tomlStream).parse());
     } catch (std::runtime_error & e) {
-        throw EvalError({
-            .hint = hintfmt("while parsing a TOML string: %s", e.what()),
-            .errPos = pos
-        });
+        throw EvalError("while parsing a TOML string at %s: %s", pos, e.what());
     }
 }
 

src/libexpr/value-to-json.cc

@@ -79,7 +79,7 @@ void printValueAsJSON(EvalState & state, bool strict,
             break;
 
         default:
-            throw TypeError("cannot convert %1% to JSON", showType(v));
+            throw TypeError(format("cannot convert %1% to JSON") % showType(v));
     }
 }
 
@@ -93,7 +93,7 @@ void printValueAsJSON(EvalState & state, bool strict,
 void ExternalValueBase::printValueAsJSON(EvalState & state, bool strict,
     JSONPlaceholder & out, PathSet & context) const
 {
-    throw TypeError("cannot convert %1% to JSON", showType());
+    throw TypeError(format("cannot convert %1% to JSON") % showType());
 }
 
 

src/libexpr/value.hh

@@ -253,17 +253,12 @@ void mkPath(Value & v, const char * s);
 
 
 #if HAVE_BOEHMGC
-typedef std::vector<Value *, traceable_allocator<Value *> > ValueVector;
-typedef std::map<Symbol, Value *, std::less<Symbol>, traceable_allocator<std::pair<const Symbol, Value *> > > ValueMap;
+typedef std::vector<Value *, gc_allocator<Value *> > ValueVector;
+typedef std::map<Symbol, Value *, std::less<Symbol>, gc_allocator<std::pair<const Symbol, Value *> > > ValueMap;
 #else
 typedef std::vector<Value *> ValueVector;
 typedef std::map<Symbol, Value *> ValueMap;
 #endif
 
 
-/* A value allocated in traceable memory. */
-typedef std::shared_ptr<Value *> RootValue;
-
-RootValue allocRootValue(Value * v);
-
 }

src/libfetchers/attrs.cc

@@ -1,107 +0,0 @@
-#include "attrs.hh"
-#include "fetchers.hh"
-
-#include <nlohmann/json.hpp>
-
-namespace nix::fetchers {
-
-Attrs jsonToAttrs(const nlohmann::json & json)
-{
-    Attrs attrs;
-
-    for (auto & i : json.items()) {
-        if (i.value().is_number())
-            attrs.emplace(i.key(), i.value().get<int64_t>());
-        else if (i.value().is_string())
-            attrs.emplace(i.key(), i.value().get<std::string>());
-        else if (i.value().is_boolean())
-            attrs.emplace(i.key(), i.value().get<bool>());
-        else
-            throw Error("unsupported input attribute type in lock file");
-    }
-
-    return attrs;
-}
-
-nlohmann::json attrsToJson(const Attrs & attrs)
-{
-    nlohmann::json json;
-    for (auto & attr : attrs) {
-        if (auto v = std::get_if<int64_t>(&attr.second)) {
-            json[attr.first] = *v;
-        } else if (auto v = std::get_if<std::string>(&attr.second)) {
-            json[attr.first] = *v;
-        } else if (auto v = std::get_if<Explicit<bool>>(&attr.second)) {
-            json[attr.first] = v->t;
-        } else abort();
-    }
-    return json;
-}
-
-std::optional<std::string> maybeGetStrAttr(const Attrs & attrs, const std::string & name)
-{
-    auto i = attrs.find(name);
-    if (i == attrs.end()) return {};
-    if (auto v = std::get_if<std::string>(&i->second))
-        return *v;
-    throw Error("input attribute '%s' is not a string %s", name, attrsToJson(attrs).dump());
-}
-
-std::string getStrAttr(const Attrs & attrs, const std::string & name)
-{
-    auto s = maybeGetStrAttr(attrs, name);
-    if (!s)
-        throw Error("input attribute '%s' is missing", name);
-    return *s;
-}
-
-std::optional<int64_t> maybeGetIntAttr(const Attrs & attrs, const std::string & name)
-{
-    auto i = attrs.find(name);
-    if (i == attrs.end()) return {};
-    if (auto v = std::get_if<int64_t>(&i->second))
-        return *v;
-    throw Error("input attribute '%s' is not an integer", name);
-}
-
-int64_t getIntAttr(const Attrs & attrs, const std::string & name)
-{
-    auto s = maybeGetIntAttr(attrs, name);
-    if (!s)
-        throw Error("input attribute '%s' is missing", name);
-    return *s;
-}
-
-std::optional<bool> maybeGetBoolAttr(const Attrs & attrs, const std::string & name)
-{
-    auto i = attrs.find(name);
-    if (i == attrs.end()) return {};
-    if (auto v = std::get_if<int64_t>(&i->second))
-        return *v;
-    throw Error("input attribute '%s' is not a Boolean", name);
-}
-
-bool getBoolAttr(const Attrs & attrs, const std::string & name)
-{
-    auto s = maybeGetBoolAttr(attrs, name);
-    if (!s)
-        throw Error("input attribute '%s' is missing", name);
-    return *s;
-}
-
-std::map<std::string, std::string> attrsToQuery(const Attrs & attrs)
-{
-    std::map<std::string, std::string> query;
-    for (auto & attr : attrs) {
-        if (auto v = std::get_if<int64_t>(&attr.second)) {
-            query.insert_or_assign(attr.first, fmt("%d", *v));
-        } else if (auto v = std::get_if<std::string>(&attr.second)) {
-            query.insert_or_assign(attr.first, *v);
-        } else if (auto v = std::get_if<Explicit<bool>>(&attr.second)) {
-            query.insert_or_assign(attr.first, v->t ? "1" : "0");
-        } else abort();
-    }
-    return query;
-}
-
-}

src/libfetchers/attrs.hh

@@ -1,39 +0,0 @@
-#pragma once
-
-#include "types.hh"
-
-#include <variant>
-
-#include <nlohmann/json_fwd.hpp>
-
-namespace nix::fetchers {
-
-/* Wrap bools to prevent string literals (i.e. 'char *') from being
-   cast to a bool in Attr. */
-template<typename T>
-struct Explicit {
-    T t;
-};
-
-typedef std::variant<std::string, int64_t, Explicit<bool>> Attr;
-typedef std::map<std::string, Attr> Attrs;
-
-Attrs jsonToAttrs(const nlohmann::json & json);
-
-nlohmann::json attrsToJson(const Attrs & attrs);
-
-std::optional<std::string> maybeGetStrAttr(const Attrs & attrs, const std::string & name);
-
-std::string getStrAttr(const Attrs & attrs, const std::string & name);
-
-std::optional<int64_t> maybeGetIntAttr(const Attrs & attrs, const std::string & name);
-
-int64_t getIntAttr(const Attrs & attrs, const std::string & name);
-
-std::optional<bool> maybeGetBoolAttr(const Attrs & attrs, const std::string & name);
-
-bool getBoolAttr(const Attrs & attrs, const std::string & name);
-
-std::map<std::string, std::string> attrsToQuery(const Attrs & attrs);
-
-}

src/libfetchers/cache.cc

@@ -1,121 +0,0 @@
-#include "cache.hh"
-#include "sqlite.hh"
-#include "sync.hh"
-#include "store-api.hh"
-
-#include <nlohmann/json.hpp>
-
-namespace nix::fetchers {
-
-static const char * schema = R"sql(
-
-create table if not exists Cache (
-    input     text not null,
-    info      text not null,
-    path      text not null,
-    immutable integer not null,
-    timestamp integer not null,
-    primary key (input)
-);
-)sql";
-
-struct CacheImpl : Cache
-{
-    struct State
-    {
-        SQLite db;
-        SQLiteStmt add, lookup;
-    };
-
-    Sync<State> _state;
-
-    CacheImpl()
-    {
-        auto state(_state.lock());
-
-        auto dbPath = getCacheDir() + "/nix/fetcher-cache-v1.sqlite";
-        createDirs(dirOf(dbPath));
-
-        state->db = SQLite(dbPath);
-        state->db.isCache();
-        state->db.exec(schema);
-
-        state->add.create(state->db,
-            "insert or replace into Cache(input, info, path, immutable, timestamp) values (?, ?, ?, ?, ?)");
-
-        state->lookup.create(state->db,
-            "select info, path, immutable, timestamp from Cache where input = ?");
-    }
-
-    void add(
-        ref<Store> store,
-        const Attrs & inAttrs,
-        const Attrs & infoAttrs,
-        const StorePath & storePath,
-        bool immutable) override
-    {
-        _state.lock()->add.use()
-            (attrsToJson(inAttrs).dump())
-            (attrsToJson(infoAttrs).dump())
-            (store->printStorePath(storePath))
-            (immutable)
-            (time(0)).exec();
-    }
-
-    std::optional<std::pair<Attrs, StorePath>> lookup(
-        ref<Store> store,
-        const Attrs & inAttrs) override
-    {
-        if (auto res = lookupExpired(store, inAttrs)) {
-            if (!res->expired)
-                return std::make_pair(std::move(res->infoAttrs), std::move(res->storePath));
-            debug("ignoring expired cache entry '%s'",
-                attrsToJson(inAttrs).dump());
-        }
-        return {};
-    }
-
-    std::optional<Result> lookupExpired(
-        ref<Store> store,
-        const Attrs & inAttrs) override
-    {
-        auto state(_state.lock());
-
-        auto inAttrsJson = attrsToJson(inAttrs).dump();
-
-        auto stmt(state->lookup.use()(inAttrsJson));
-        if (!stmt.next()) {
-            debug("did not find cache entry for '%s'", inAttrsJson);
-            return {};
-        }
-
-        auto infoJson = stmt.getStr(0);
-        auto storePath = store->parseStorePath(stmt.getStr(1));
-        auto immutable = stmt.getInt(2) != 0;
-        auto timestamp = stmt.getInt(3);
-
-        store->addTempRoot(storePath);
-        if (!store->isValidPath(storePath)) {
-            // FIXME: we could try to substitute 'storePath'.
-            debug("ignoring disappeared cache entry '%s'", inAttrsJson);
-            return {};
-        }
-
-        debug("using cache entry '%s' -> '%s', '%s'",
-            inAttrsJson, infoJson, store->printStorePath(storePath));
-
-        return Result {
-            .expired = !immutable && (settings.tarballTtl.get() == 0 || timestamp + settings.tarballTtl < time(0)),
-            .infoAttrs = jsonToAttrs(nlohmann::json::parse(infoJson)),
-            .storePath = std::move(storePath)
-        };
-    }
-};
-
-ref<Cache> getCache()
-{
-    static auto cache = std::make_shared<CacheImpl>();
-    return ref<Cache>(cache);
-}
-
-}

src/libfetchers/cache.hh

@@ -1,36 +0,0 @@
-#pragma once
-
-#include "fetchers.hh"
-
-namespace nix::fetchers {
-
-struct Cache
-{
-    virtual ~Cache() { }
-
-    virtual void add(
-        ref<Store> store,
-        const Attrs & inAttrs,
-        const Attrs & infoAttrs,
-        const StorePath & storePath,
-        bool immutable) = 0;
-
-    virtual std::optional<std::pair<Attrs, StorePath>> lookup(
-        ref<Store> store,
-        const Attrs & inAttrs) = 0;
-
-    struct Result
-    {
-        bool expired = false;
-        Attrs infoAttrs;
-        StorePath storePath;
-    };
-
-    virtual std::optional<Result> lookupExpired(
-        ref<Store> store,
-        const Attrs & inAttrs) = 0;
-};
-
-ref<Cache> getCache();
-
-}

src/libfetchers/fetchers.cc

@@ -1,75 +0,0 @@
-#include "fetchers.hh"
-#include "store-api.hh"
-
-#include <nlohmann/json.hpp>
-
-namespace nix::fetchers {
-
-std::unique_ptr<std::vector<std::unique_ptr<InputScheme>>> inputSchemes = nullptr;
-
-void registerInputScheme(std::unique_ptr<InputScheme> && inputScheme)
-{
-    if (!inputSchemes) inputSchemes = std::make_unique<std::vector<std::unique_ptr<InputScheme>>>();
-    inputSchemes->push_back(std::move(inputScheme));
-}
-
-std::unique_ptr<Input> inputFromURL(const ParsedURL & url)
-{
-    for (auto & inputScheme : *inputSchemes) {
-        auto res = inputScheme->inputFromURL(url);
-        if (res) return res;
-    }
-    throw Error("input '%s' is unsupported", url.url);
-}
-
-std::unique_ptr<Input> inputFromURL(const std::string & url)
-{
-    return inputFromURL(parseURL(url));
-}
-
-std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs)
-{
-    auto attrs2(attrs);
-    attrs2.erase("narHash");
-    for (auto & inputScheme : *inputSchemes) {
-        auto res = inputScheme->inputFromAttrs(attrs2);
-        if (res) {
-            if (auto narHash = maybeGetStrAttr(attrs, "narHash"))
-                // FIXME: require SRI hash.
-                res->narHash = newHashAllowEmpty(*narHash, {});
-            return res;
-        }
-    }
-    throw Error("input '%s' is unsupported", attrsToJson(attrs));
-}
-
-Attrs Input::toAttrs() const
-{
-    auto attrs = toAttrsInternal();
-    if (narHash)
-        attrs.emplace("narHash", narHash->to_string(SRI, true));
-    attrs.emplace("type", type());
-    return attrs;
-}
-
-std::pair<Tree, std::shared_ptr<const Input>> Input::fetchTree(ref<Store> store) const
-{
-    auto [tree, input] = fetchTreeInternal(store);
-
-    if (tree.actualPath == "")
-        tree.actualPath = store->toRealPath(tree.storePath);
-
-    if (!tree.info.narHash)
-        tree.info.narHash = store->queryPathInfo(tree.storePath)->narHash;
-
-    if (input->narHash)
-        assert(input->narHash == tree.info.narHash);
-
-    if (narHash && narHash != input->narHash)
-        throw Error("NAR hash mismatch in input '%s' (%s), expected '%s', got '%s'",
-            to_string(), tree.actualPath, narHash->to_string(SRI, true), input->narHash->to_string(SRI, true));
-
-    return {std::move(tree), input};
-}
-
-}

src/libfetchers/fetchers.hh

@@ -1,103 +0,0 @@
-#pragma once
-
-#include "types.hh"
-#include "hash.hh"
-#include "path.hh"
-#include "tree-info.hh"
-#include "attrs.hh"
-#include "url.hh"
-
-#include <memory>
-
-namespace nix { class Store; }
-
-namespace nix::fetchers {
-
-struct Input;
-
-struct Tree
-{
-    Path actualPath;
-    StorePath storePath;
-    TreeInfo info;
-};
-
-struct Input : std::enable_shared_from_this<Input>
-{
-    std::optional<Hash> narHash; // FIXME: implement
-
-    virtual std::string type() const = 0;
-
-    virtual ~Input() { }
-
-    virtual bool operator ==(const Input & other) const { return false; }
-
-    /* Check whether this is a "direct" input, that is, not
-       one that goes through a registry. */
-    virtual bool isDirect() const { return true; }
-
-    /* Check whether this is an "immutable" input, that is,
-       one that contains a commit hash or content hash. */
-    virtual bool isImmutable() const { return (bool) narHash; }
-
-    virtual bool contains(const Input & other) const { return false; }
-
-    virtual std::optional<std::string> getRef() const { return {}; }
-
-    virtual std::optional<Hash> getRev() const { return {}; }
-
-    virtual ParsedURL toURL() const = 0;
-
-    std::string to_string() const
-    {
-        return toURL().to_string();
-    }
-
-    Attrs toAttrs() const;
-
-    std::pair<Tree, std::shared_ptr<const Input>> fetchTree(ref<Store> store) const;
-
-private:
-
-    virtual std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(ref<Store> store) const = 0;
-
-    virtual Attrs toAttrsInternal() const = 0;
-};
-
-struct InputScheme
-{
-    virtual ~InputScheme() { }
-
-    virtual std::unique_ptr<Input> inputFromURL(const ParsedURL & url) = 0;
-
-    virtual std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) = 0;
-};
-
-std::unique_ptr<Input> inputFromURL(const ParsedURL & url);
-
-std::unique_ptr<Input> inputFromURL(const std::string & url);
-
-std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs);
-
-void registerInputScheme(std::unique_ptr<InputScheme> && fetcher);
-
-struct DownloadFileResult
-{
-    StorePath storePath;
-    std::string etag;
-    std::string effectiveUrl;
-};
-
-DownloadFileResult downloadFile(
-    ref<Store> store,
-    const std::string & url,
-    const std::string & name,
-    bool immutable);
-
-Tree downloadTarball(
-    ref<Store> store,
-    const std::string & url,
-    const std::string & name,
-    bool immutable);
-
-}

src/libfetchers/git.cc

@@ -1,441 +0,0 @@
-#include "fetchers.hh"
-#include "cache.hh"
-#include "globals.hh"
-#include "tarfile.hh"
-#include "store-api.hh"
-
-#include <sys/time.h>
-
-using namespace std::string_literals;
-
-namespace nix::fetchers {
-
-static std::string readHead(const Path & path)
-{
-    return chomp(runProgram("git", true, { "-C", path, "rev-parse", "--abbrev-ref", "HEAD" }));
-}
-
-static bool isNotDotGitDirectory(const Path & path)
-{
-    static const std::regex gitDirRegex("^(?:.*/)?\\.git$");
-
-    return not std::regex_match(path, gitDirRegex);
-}
-
-struct GitInput : Input
-{
-    ParsedURL url;
-    std::optional<std::string> ref;
-    std::optional<Hash> rev;
-    bool shallow = false;
-    bool submodules = false;
-
-    GitInput(const ParsedURL & url) : url(url)
-    { }
-
-    std::string type() const override { return "git"; }
-
-    bool operator ==(const Input & other) const override
-    {
-        auto other2 = dynamic_cast<const GitInput *>(&other);
-        return
-            other2
-            && url == other2->url
-            && rev == other2->rev
-            && ref == other2->ref;
-    }
-
-    bool isImmutable() const override
-    {
-        return (bool) rev || narHash;
-    }
-
-    std::optional<std::string> getRef() const override { return ref; }
-
-    std::optional<Hash> getRev() const override { return rev; }
-
-    ParsedURL toURL() const override
-    {
-        ParsedURL url2(url);
-        if (url2.scheme != "git") url2.scheme = "git+" + url2.scheme;
-        if (rev) url2.query.insert_or_assign("rev", rev->gitRev());
-        if (ref) url2.query.insert_or_assign("ref", *ref);
-        if (shallow) url2.query.insert_or_assign("shallow", "1");
-        return url2;
-    }
-
-    Attrs toAttrsInternal() const override
-    {
-        Attrs attrs;
-        attrs.emplace("url", url.to_string());
-        if (ref)
-            attrs.emplace("ref", *ref);
-        if (rev)
-            attrs.emplace("rev", rev->gitRev());
-        if (shallow)
-            attrs.emplace("shallow", true);
-        if (submodules)
-            attrs.emplace("submodules", true);
-        return attrs;
-    }
-
-    std::pair<bool, std::string> getActualUrl() const
-    {
-        // Don't clone file:// URIs (but otherwise treat them the
-        // same as remote URIs, i.e. don't use the working tree or
-        // HEAD).
-        static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; // for testing
-        bool isLocal = url.scheme == "file" && !forceHttp;
-        return {isLocal, isLocal ? url.path : url.base};
-    }
-
-    std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override
-    {
-        auto name = "source";
-
-        auto input = std::make_shared<GitInput>(*this);
-
-        assert(!rev || rev->type == htSHA1);
-
-        std::string cacheType = "git";
-        if (shallow) cacheType += "-shallow";
-        if (submodules) cacheType += "-submodules";
-
-        auto getImmutableAttrs = [&]()
-        {
-            return Attrs({
-                {"type", cacheType},
-                {"name", name},
-                {"rev", input->rev->gitRev()},
-            });
-        };
-
-        auto makeResult = [&](const Attrs & infoAttrs, StorePath && storePath)
-            -> std::pair<Tree, std::shared_ptr<const Input>>
-        {
-            assert(input->rev);
-            assert(!rev || rev == input->rev);
-            return {
-                Tree {
-                    .actualPath = store->toRealPath(storePath),
-                    .storePath = std::move(storePath),
-                    .info = TreeInfo {
-                        .revCount = shallow ? std::nullopt : std::optional(getIntAttr(infoAttrs, "revCount")),
-                        .lastModified = getIntAttr(infoAttrs, "lastModified"),
-                    },
-                },
-                input
-            };
-        };
-
-        if (rev) {
-            if (auto res = getCache()->lookup(store, getImmutableAttrs()))
-                return makeResult(res->first, std::move(res->second));
-        }
-
-        auto [isLocal, actualUrl_] = getActualUrl();
-        auto actualUrl = actualUrl_; // work around clang bug
-
-        // If this is a local directory and no ref or revision is
-        // given, then allow the use of an unclean working tree.
-        if (!input->ref && !input->rev && isLocal) {
-            bool clean = false;
-
-            /* Check whether this repo has any commits. There are
-               probably better ways to do this. */
-            auto gitDir = actualUrl + "/.git";
-            auto commonGitDir = chomp(runProgram(
-                "git",
-                true,
-                { "-C", actualUrl, "rev-parse", "--git-common-dir" }
-            ));
-            if (commonGitDir != ".git")
-                    gitDir = commonGitDir;
-
-            bool haveCommits = !readDirectory(gitDir + "/refs/heads").empty();
-
-            try {
-                if (haveCommits) {
-                    runProgram("git", true, { "-C", actualUrl, "diff-index", "--quiet", "HEAD", "--" });
-                    clean = true;
-                }
-            } catch (ExecError & e) {
-                if (!WIFEXITED(e.status) || WEXITSTATUS(e.status) != 1) throw;
-            }
-
-            if (!clean) {
-
-                /* This is an unclean working tree. So copy all tracked files. */
-
-                if (!settings.allowDirty)
-                    throw Error("Git tree '%s' is dirty", actualUrl);
-
-                if (settings.warnDirty)
-                    warn("Git tree '%s' is dirty", actualUrl);
-
-                auto gitOpts = Strings({ "-C", actualUrl, "ls-files", "-z" });
-                if (submodules)
-                    gitOpts.emplace_back("--recurse-submodules");
-
-                auto files = tokenizeString<std::set<std::string>>(
-                    runProgram("git", true, gitOpts), "\0"s);
-
-                PathFilter filter = [&](const Path & p) -> bool {
-                    assert(hasPrefix(p, actualUrl));
-                    std::string file(p, actualUrl.size() + 1);
-
-                    auto st = lstat(p);
-
-                    if (S_ISDIR(st.st_mode)) {
-                        auto prefix = file + "/";
-                        auto i = files.lower_bound(prefix);
-                        return i != files.end() && hasPrefix(*i, prefix);
-                    }
-
-                    return files.count(file);
-                };
-
-                auto storePath = store->addToStore("source", actualUrl, FileIngestionMethod::Recursive, htSHA256, filter);
-
-                auto tree = Tree {
-                    .actualPath = store->printStorePath(storePath),
-                    .storePath = std::move(storePath),
-                    .info = TreeInfo {
-                        // FIXME: maybe we should use the timestamp of the last
-                        // modified dirty file?
-                        .lastModified = haveCommits ? std::stoull(runProgram("git", true, { "-C", actualUrl, "log", "-1", "--format=%ct", "HEAD" })) : 0,
-                    }
-                };
-
-                return {std::move(tree), input};
-            }
-        }
-
-        if (!input->ref) input->ref = isLocal ? readHead(actualUrl) : "master";
-
-        Attrs mutableAttrs({
-            {"type", cacheType},
-            {"name", name},
-            {"url", actualUrl},
-            {"ref", *input->ref},
-        });
-
-        Path repoDir;
-
-        if (isLocal) {
-
-            if (!input->rev)
-                input->rev = Hash(chomp(runProgram("git", true, { "-C", actualUrl, "rev-parse", *input->ref })), htSHA1);
-
-            repoDir = actualUrl;
-
-        } else {
-
-            if (auto res = getCache()->lookup(store, mutableAttrs)) {
-                auto rev2 = Hash(getStrAttr(res->first, "rev"), htSHA1);
-                if (!rev || rev == rev2) {
-                    input->rev = rev2;
-                    return makeResult(res->first, std::move(res->second));
-                }
-            }
-
-            Path cacheDir = getCacheDir() + "/nix/gitv3/" + hashString(htSHA256, actualUrl).to_string(Base32, false);
-            repoDir = cacheDir;
-
-            if (!pathExists(cacheDir)) {
-                createDirs(dirOf(cacheDir));
-                runProgram("git", true, { "init", "--bare", repoDir });
-            }
-
-            Path localRefFile =
-                input->ref->compare(0, 5, "refs/") == 0
-                ? cacheDir + "/" + *input->ref
-                : cacheDir + "/refs/heads/" + *input->ref;
-
-            bool doFetch;
-            time_t now = time(0);
-
-            /* If a rev was specified, we need to fetch if it's not in the
-               repo. */
-            if (input->rev) {
-                try {
-                    runProgram("git", true, { "-C", repoDir, "cat-file", "-e", input->rev->gitRev() });
-                    doFetch = false;
-                } catch (ExecError & e) {
-                    if (WIFEXITED(e.status)) {
-                        doFetch = true;
-                    } else {
-                        throw;
-                    }
-                }
-            } else {
-                /* If the local ref is older than ‘tarball-ttl’ seconds, do a
-                   git fetch to update the local ref to the remote ref. */
-                struct stat st;
-                doFetch = stat(localRefFile.c_str(), &st) != 0 ||
-                    (uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now;
-            }
-
-            if (doFetch) {
-                Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Git repository '%s'", actualUrl));
-
-                // FIXME: git stderr messes up our progress indicator, so
-                // we're using --quiet for now. Should process its stderr.
-                try {
-                    auto fetchRef = input->ref->compare(0, 5, "refs/") == 0
-                        ? *input->ref
-                        : "refs/heads/" + *input->ref;
-                    runProgram("git", true, { "-C", repoDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", fetchRef, fetchRef) });
-                } catch (Error & e) {
-                    if (!pathExists(localRefFile)) throw;
-                    warn("could not update local clone of Git repository '%s'; continuing with the most recent version", actualUrl);
-                }
-
-                struct timeval times[2];
-                times[0].tv_sec = now;
-                times[0].tv_usec = 0;
-                times[1].tv_sec = now;
-                times[1].tv_usec = 0;
-
-                utimes(localRefFile.c_str(), times);
-            }
-
-            if (!input->rev)
-                input->rev = Hash(chomp(readFile(localRefFile)), htSHA1);
-        }
-
-        bool isShallow = chomp(runProgram("git", true, { "-C", repoDir, "rev-parse", "--is-shallow-repository" })) == "true";
-
-        if (isShallow && !shallow)
-            throw Error("'%s' is a shallow Git repository, but a non-shallow repository is needed", actualUrl);
-
-        // FIXME: check whether rev is an ancestor of ref.
-
-        printTalkative("using revision %s of repo '%s'", input->rev->gitRev(), actualUrl);
-
-        /* Now that we know the ref, check again whether we have it in
-           the store. */
-        if (auto res = getCache()->lookup(store, getImmutableAttrs()))
-            return makeResult(res->first, std::move(res->second));
-
-        Path tmpDir = createTempDir();
-        AutoDelete delTmpDir(tmpDir, true);
-        PathFilter filter = defaultPathFilter;
-
-        if (submodules) {
-            Path tmpGitDir = createTempDir();
-            AutoDelete delTmpGitDir(tmpGitDir, true);
-
-            runProgram("git", true, { "init", tmpDir, "--separate-git-dir", tmpGitDir });
-            // TODO: repoDir might lack the ref (it only checks if rev
-            // exists, see FIXME above) so use a big hammer and fetch
-            // everything to ensure we get the rev.
-            runProgram("git", true, { "-C", tmpDir, "fetch", "--quiet", "--force",
-                                      "--update-head-ok", "--", repoDir, "refs/*:refs/*" });
-
-            runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", input->rev->gitRev() });
-            runProgram("git", true, { "-C", tmpDir, "remote", "add", "origin", actualUrl });
-            runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init", "--recursive" });
-
-            filter = isNotDotGitDirectory;
-        } else {
-            // FIXME: should pipe this, or find some better way to extract a
-            // revision.
-            auto source = sinkToSource([&](Sink & sink) {
-                RunOptions gitOptions("git", { "-C", repoDir, "archive", input->rev->gitRev() });
-                gitOptions.standardOut = &sink;
-                runProgram2(gitOptions);
-            });
-
-            unpackTarfile(*source, tmpDir);
-        }
-
-        auto storePath = store->addToStore(name, tmpDir, FileIngestionMethod::Recursive, htSHA256, filter);
-
-        auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "log", "-1", "--format=%ct", input->rev->gitRev() }));
-
-        Attrs infoAttrs({
-            {"rev", input->rev->gitRev()},
-            {"lastModified", lastModified},
-        });
-
-        if (!shallow)
-            infoAttrs.insert_or_assign("revCount",
-                std::stoull(runProgram("git", true, { "-C", repoDir, "rev-list", "--count", input->rev->gitRev() })));
-
-        if (!this->rev)
-            getCache()->add(
-                store,
-                mutableAttrs,
-                infoAttrs,
-                storePath,
-                false);
-
-        getCache()->add(
-            store,
-            getImmutableAttrs(),
-            infoAttrs,
-            storePath,
-            true);
-
-        return makeResult(infoAttrs, std::move(storePath));
-    }
-};
-
-struct GitInputScheme : InputScheme
-{
-    std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override
-    {
-        if (url.scheme != "git" &&
-            url.scheme != "git+http" &&
-            url.scheme != "git+https" &&
-            url.scheme != "git+ssh" &&
-            url.scheme != "git+file") return nullptr;
-
-        auto url2(url);
-        if (hasPrefix(url2.scheme, "git+")) url2.scheme = std::string(url2.scheme, 4);
-        url2.query.clear();
-
-        Attrs attrs;
-        attrs.emplace("type", "git");
-
-        for (auto &[name, value] : url.query) {
-            if (name == "rev" || name == "ref")
-                attrs.emplace(name, value);
-            else
-                url2.query.emplace(name, value);
-        }
-
-        attrs.emplace("url", url2.to_string());
-
-        return inputFromAttrs(attrs);
-    }
-
-    std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override
-    {
-        if (maybeGetStrAttr(attrs, "type") != "git") return {};
-
-        for (auto & [name, value] : attrs)
-            if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules")
-                throw Error("unsupported Git input attribute '%s'", name);
-
-        auto input = std::make_unique<GitInput>(parseURL(getStrAttr(attrs, "url")));
-        if (auto ref = maybeGetStrAttr(attrs, "ref")) {
-            if (std::regex_search(*ref, badGitRefRegex))
-                throw BadURL("invalid Git branch/tag name '%s'", *ref);
-            input->ref = *ref;
-        }
-        if (auto rev = maybeGetStrAttr(attrs, "rev"))
-            input->rev = Hash(*rev, htSHA1);
-
-        input->shallow = maybeGetBoolAttr(attrs, "shallow").value_or(false);
-
-        input->submodules = maybeGetBoolAttr(attrs, "submodules").value_or(false);
-
-        return input;
-    }
-};
-
-static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<GitInputScheme>()); });
-
-}

src/libfetchers/github.cc

@@ -1,195 +0,0 @@
-#include "filetransfer.hh"
-#include "cache.hh"
-#include "fetchers.hh"
-#include "globals.hh"
-#include "store-api.hh"
-
-#include <nlohmann/json.hpp>
-
-namespace nix::fetchers {
-
-std::regex ownerRegex("[a-zA-Z][a-zA-Z0-9_-]*", std::regex::ECMAScript);
-std::regex repoRegex("[a-zA-Z][a-zA-Z0-9_-]*", std::regex::ECMAScript);
-
-struct GitHubInput : Input
-{
-    std::string owner;
-    std::string repo;
-    std::optional<std::string> ref;
-    std::optional<Hash> rev;
-
-    std::string type() const override { return "github"; }
-
-    bool operator ==(const Input & other) const override
-    {
-        auto other2 = dynamic_cast<const GitHubInput *>(&other);
-        return
-            other2
-            && owner == other2->owner
-            && repo == other2->repo
-            && rev == other2->rev
-            && ref == other2->ref;
-    }
-
-    bool isImmutable() const override
-    {
-        return (bool) rev || narHash;
-    }
-
-    std::optional<std::string> getRef() const override { return ref; }
-
-    std::optional<Hash> getRev() const override { return rev; }
-
-    ParsedURL toURL() const override
-    {
-        auto path = owner + "/" + repo;
-        assert(!(ref && rev));
-        if (ref) path += "/" + *ref;
-        if (rev) path += "/" + rev->to_string(Base16, false);
-        return ParsedURL {
-            .scheme = "github",
-            .path = path,
-        };
-    }
-
-    Attrs toAttrsInternal() const override
-    {
-        Attrs attrs;
-        attrs.emplace("owner", owner);
-        attrs.emplace("repo", repo);
-        if (ref)
-            attrs.emplace("ref", *ref);
-        if (rev)
-            attrs.emplace("rev", rev->gitRev());
-        return attrs;
-    }
-
-    std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override
-    {
-        auto rev = this->rev;
-        auto ref = this->ref.value_or("master");
-
-        if (!rev) {
-            auto url = fmt("https://api.github.com/repos/%s/%s/commits/%s",
-                owner, repo, ref);
-            auto json = nlohmann::json::parse(
-                readFile(
-                    store->toRealPath(
-                        downloadFile(store, url, "source", false).storePath)));
-            rev = Hash(std::string { json["sha"] }, htSHA1);
-            debug("HEAD revision for '%s' is %s", url, rev->gitRev());
-        }
-
-        auto input = std::make_shared<GitHubInput>(*this);
-        input->ref = {};
-        input->rev = *rev;
-
-        Attrs immutableAttrs({
-            {"type", "git-tarball"},
-            {"rev", rev->gitRev()},
-        });
-
-        if (auto res = getCache()->lookup(store, immutableAttrs)) {
-            return {
-                Tree{
-                    .actualPath = store->toRealPath(res->second),
-                    .storePath = std::move(res->second),
-                    .info = TreeInfo {
-                        .lastModified = getIntAttr(res->first, "lastModified"),
-                    },
-                },
-                input
-            };
-        }
-
-        // FIXME: use regular /archive URLs instead? api.github.com
-        // might have stricter rate limits.
-
-        auto url = fmt("https://api.github.com/repos/%s/%s/tarball/%s",
-            owner, repo, rev->to_string(Base16, false));
-
-        std::string accessToken = settings.githubAccessToken.get();
-        if (accessToken != "")
-            url += "?access_token=" + accessToken;
-
-        auto tree = downloadTarball(store, url, "source", true);
-
-        getCache()->add(
-            store,
-            immutableAttrs,
-            {
-                {"rev", rev->gitRev()},
-                {"lastModified", *tree.info.lastModified}
-            },
-            tree.storePath,
-            true);
-
-        return {std::move(tree), input};
-    }
-};
-
-struct GitHubInputScheme : InputScheme
-{
-    std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override
-    {
-        if (url.scheme != "github") return nullptr;
-
-        auto path = tokenizeString<std::vector<std::string>>(url.path, "/");
-        auto input = std::make_unique<GitHubInput>();
-
-        if (path.size() == 2) {
-        } else if (path.size() == 3) {
-            if (std::regex_match(path[2], revRegex))
-                input->rev = Hash(path[2], htSHA1);
-            else if (std::regex_match(path[2], refRegex))
-                input->ref = path[2];
-            else
-                throw BadURL("in GitHub URL '%s', '%s' is not a commit hash or branch/tag name", url.url, path[2]);
-        } else
-            throw BadURL("GitHub URL '%s' is invalid", url.url);
-
-        for (auto &[name, value] : url.query) {
-            if (name == "rev") {
-                if (input->rev)
-                    throw BadURL("GitHub URL '%s' contains multiple commit hashes", url.url);
-                input->rev = Hash(value, htSHA1);
-            }
-            else if (name == "ref") {
-                if (!std::regex_match(value, refRegex))
-                    throw BadURL("GitHub URL '%s' contains an invalid branch/tag name", url.url);
-                if (input->ref)
-                    throw BadURL("GitHub URL '%s' contains multiple branch/tag names", url.url);
-                input->ref = value;
-            }
-        }
-
-        if (input->ref && input->rev)
-            throw BadURL("GitHub URL '%s' contains both a commit hash and a branch/tag name", url.url);
-
-        input->owner = path[0];
-        input->repo = path[1];
-
-        return input;
-    }
-
-    std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override
-    {
-        if (maybeGetStrAttr(attrs, "type") != "github") return {};
-
-        for (auto & [name, value] : attrs)
-            if (name != "type" && name != "owner" && name != "repo" && name != "ref" && name != "rev")
-                throw Error("unsupported GitHub input attribute '%s'", name);
-
-        auto input = std::make_unique<GitHubInput>();
-        input->owner = getStrAttr(attrs, "owner");
-        input->repo = getStrAttr(attrs, "repo");
-        input->ref = maybeGetStrAttr(attrs, "ref");
-        if (auto rev = maybeGetStrAttr(attrs, "rev"))
-            input->rev = Hash(*rev, htSHA1);
-        return input;
-    }
-};
-
-static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<GitHubInputScheme>()); });
-
-}

src/libfetchers/local.mk

@@ -1,11 +0,0 @@
-libraries += libfetchers
-
-libfetchers_NAME = libnixfetchers
-
-libfetchers_DIR := $(d)
-
-libfetchers_SOURCES := $(wildcard $(d)/*.cc)
-
-libfetchers_CXXFLAGS += -I src/libutil -I src/libstore
-
-libfetchers_LIBS = libutil libstore

src/libfetchers/mercurial.cc

@@ -1,303 +0,0 @@
-#include "fetchers.hh"
-#include "cache.hh"
-#include "globals.hh"
-#include "tarfile.hh"
-#include "store-api.hh"
-
-#include <sys/time.h>
-
-using namespace std::string_literals;
-
-namespace nix::fetchers {
-
-struct MercurialInput : Input
-{
-    ParsedURL url;
-    std::optional<std::string> ref;
-    std::optional<Hash> rev;
-
-    MercurialInput(const ParsedURL & url) : url(url)
-    { }
-
-    std::string type() const override { return "hg"; }
-
-    bool operator ==(const Input & other) const override
-    {
-        auto other2 = dynamic_cast<const MercurialInput *>(&other);
-        return
-            other2
-            && url == other2->url
-            && rev == other2->rev
-            && ref == other2->ref;
-    }
-
-    bool isImmutable() const override
-    {
-        return (bool) rev || narHash;
-    }
-
-    std::optional<std::string> getRef() const override { return ref; }
-
-    std::optional<Hash> getRev() const override { return rev; }
-
-    ParsedURL toURL() const override
-    {
-        ParsedURL url2(url);
-        url2.scheme = "hg+" + url2.scheme;
-        if (rev) url2.query.insert_or_assign("rev", rev->gitRev());
-        if (ref) url2.query.insert_or_assign("ref", *ref);
-        return url;
-    }
-
-    Attrs toAttrsInternal() const override
-    {
-        Attrs attrs;
-        attrs.emplace("url", url.to_string());
-        if (ref)
-            attrs.emplace("ref", *ref);
-        if (rev)
-            attrs.emplace("rev", rev->gitRev());
-        return attrs;
-    }
-
-    std::pair<bool, std::string> getActualUrl() const
-    {
-        bool isLocal = url.scheme == "file";
-        return {isLocal, isLocal ? url.path : url.base};
-    }
-
-    std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override
-    {
-        auto name = "source";
-
-        auto input = std::make_shared<MercurialInput>(*this);
-
-        auto [isLocal, actualUrl_] = getActualUrl();
-        auto actualUrl = actualUrl_; // work around clang bug
-
-        // FIXME: return lastModified.
-
-        // FIXME: don't clone local repositories.
-
-        if (!input->ref && !input->rev && isLocal && pathExists(actualUrl + "/.hg")) {
-
-            bool clean = runProgram("hg", true, { "status", "-R", actualUrl, "--modified", "--added", "--removed" }) == "";
-
-            if (!clean) {
-
-                /* This is an unclean working tree. So copy all tracked
-                   files. */
-
-                if (!settings.allowDirty)
-                    throw Error("Mercurial tree '%s' is unclean", actualUrl);
-
-                if (settings.warnDirty)
-                    warn("Mercurial tree '%s' is unclean", actualUrl);
-
-                input->ref = chomp(runProgram("hg", true, { "branch", "-R", actualUrl }));
-
-                auto files = tokenizeString<std::set<std::string>>(
-                    runProgram("hg", true, { "status", "-R", actualUrl, "--clean", "--modified", "--added", "--no-status", "--print0" }), "\0"s);
-
-                PathFilter filter = [&](const Path & p) -> bool {
-                    assert(hasPrefix(p, actualUrl));
-                    std::string file(p, actualUrl.size() + 1);
-
-                    auto st = lstat(p);
-
-                    if (S_ISDIR(st.st_mode)) {
-                        auto prefix = file + "/";
-                        auto i = files.lower_bound(prefix);
-                        return i != files.end() && hasPrefix(*i, prefix);
-                    }
-
-                    return files.count(file);
-                };
-
-                auto storePath = store->addToStore("source", actualUrl, FileIngestionMethod::Recursive, htSHA256, filter);
-
-                return {Tree {
-                    .actualPath = store->printStorePath(storePath),
-                    .storePath = std::move(storePath),
-                }, input};
-            }
-        }
-
-        if (!input->ref) input->ref = "default";
-
-        auto getImmutableAttrs = [&]()
-        {
-            return Attrs({
-                {"type", "hg"},
-                {"name", name},
-                {"rev", input->rev->gitRev()},
-            });
-        };
-
-        auto makeResult = [&](const Attrs & infoAttrs, StorePath && storePath)
-            -> std::pair<Tree, std::shared_ptr<const Input>>
-        {
-            assert(input->rev);
-            assert(!rev || rev == input->rev);
-            return {
-                Tree{
-                    .actualPath = store->toRealPath(storePath),
-                    .storePath = std::move(storePath),
-                    .info = TreeInfo {
-                        .revCount = getIntAttr(infoAttrs, "revCount"),
-                    },
-                },
-                input
-            };
-        };
-
-        if (input->rev) {
-            if (auto res = getCache()->lookup(store, getImmutableAttrs()))
-                return makeResult(res->first, std::move(res->second));
-        }
-
-        assert(input->rev || input->ref);
-        auto revOrRef = input->rev ? input->rev->gitRev() : *input->ref;
-
-        Attrs mutableAttrs({
-            {"type", "hg"},
-            {"name", name},
-            {"url", actualUrl},
-            {"ref", *input->ref},
-        });
-
-        if (auto res = getCache()->lookup(store, mutableAttrs)) {
-            auto rev2 = Hash(getStrAttr(res->first, "rev"), htSHA1);
-            if (!rev || rev == rev2) {
-                input->rev = rev2;
-                return makeResult(res->first, std::move(res->second));
-            }
-        }
-
-        Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), hashString(htSHA256, actualUrl).to_string(Base32, false));
-
-        /* If this is a commit hash that we already have, we don't
-           have to pull again. */
-        if (!(input->rev
-                && pathExists(cacheDir)
-                && runProgram(
-                    RunOptions("hg", { "log", "-R", cacheDir, "-r", input->rev->gitRev(), "--template", "1" })
-                    .killStderr(true)).second == "1"))
-        {
-            Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Mercurial repository '%s'", actualUrl));
-
-            if (pathExists(cacheDir)) {
-                try {
-                    runProgram("hg", true, { "pull", "-R", cacheDir, "--", actualUrl });
-                }
-                catch (ExecError & e) {
-                    string transJournal = cacheDir + "/.hg/store/journal";
-                    /* hg throws "abandoned transaction" error only if this file exists */
-                    if (pathExists(transJournal)) {
-                        runProgram("hg", true, { "recover", "-R", cacheDir });
-                        runProgram("hg", true, { "pull", "-R", cacheDir, "--", actualUrl });
-                    } else {
-                        throw ExecError(e.status, fmt("'hg pull' %s", statusToString(e.status)));
-                    }
-                }
-            } else {
-                createDirs(dirOf(cacheDir));
-                runProgram("hg", true, { "clone", "--noupdate", "--", actualUrl, cacheDir });
-            }
-        }
-
-        auto tokens = tokenizeString<std::vector<std::string>>(
-            runProgram("hg", true, { "log", "-R", cacheDir, "-r", revOrRef, "--template", "{node} {rev} {branch}" }));
-        assert(tokens.size() == 3);
-
-        input->rev = Hash(tokens[0], htSHA1);
-        auto revCount = std::stoull(tokens[1]);
-        input->ref = tokens[2];
-
-        if (auto res = getCache()->lookup(store, getImmutableAttrs()))
-            return makeResult(res->first, std::move(res->second));
-
-        Path tmpDir = createTempDir();
-        AutoDelete delTmpDir(tmpDir, true);
-
-        runProgram("hg", true, { "archive", "-R", cacheDir, "-r", input->rev->gitRev(), tmpDir });
-
-        deletePath(tmpDir + "/.hg_archival.txt");
-
-        auto storePath = store->addToStore(name, tmpDir);
-
-        Attrs infoAttrs({
-            {"rev", input->rev->gitRev()},
-            {"revCount", (int64_t) revCount},
-        });
-
-        if (!this->rev)
-            getCache()->add(
-                store,
-                mutableAttrs,
-                infoAttrs,
-                storePath,
-                false);
-
-        getCache()->add(
-            store,
-            getImmutableAttrs(),
-            infoAttrs,
-            storePath,
-            true);
-
-        return makeResult(infoAttrs, std::move(storePath));
-    }
-};
-
-struct MercurialInputScheme : InputScheme
-{
-    std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override
-    {
-        if (url.scheme != "hg+http" &&
-            url.scheme != "hg+https" &&
-            url.scheme != "hg+ssh" &&
-            url.scheme != "hg+file") return nullptr;
-
-        auto url2(url);
-        url2.scheme = std::string(url2.scheme, 3);
-        url2.query.clear();
-
-        Attrs attrs;
-        attrs.emplace("type", "hg");
-
-        for (auto &[name, value] : url.query) {
-            if (name == "rev" || name == "ref")
-                attrs.emplace(name, value);
-            else
-                url2.query.emplace(name, value);
-        }
-
-        attrs.emplace("url", url2.to_string());
-
-        return inputFromAttrs(attrs);
-    }
-
-    std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override
-    {
-        if (maybeGetStrAttr(attrs, "type") != "hg") return {};
-
-        for (auto & [name, value] : attrs)
-            if (name != "type" && name != "url" && name != "ref" && name != "rev")
-                throw Error("unsupported Mercurial input attribute '%s'", name);
-
-        auto input = std::make_unique<MercurialInput>(parseURL(getStrAttr(attrs, "url")));
-        if (auto ref = maybeGetStrAttr(attrs, "ref")) {
-            if (!std::regex_match(*ref, refRegex))
-                throw BadURL("invalid Mercurial branch/tag name '%s'", *ref);
-            input->ref = *ref;
-        }
-        if (auto rev = maybeGetStrAttr(attrs, "rev"))
-            input->rev = Hash(*rev, htSHA1);
-        return input;
-    }
-};
-
-static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<MercurialInputScheme>()); });
-
-}

src/libfetchers/path.cc

@@ -1,148 +0,0 @@
-#include "fetchers.hh"
-#include "store-api.hh"
-
-namespace nix::fetchers {
-
-struct PathInput : Input
-{
-    Path path;
-
-    /* Allow the user to pass in "fake" tree info attributes. This is
-       useful for making a pinned tree work the same as the repository
-       from which is exported
-       (e.g. path:/nix/store/...-source?lastModified=1585388205&rev=b0c285...). */
-    std::optional<Hash> rev;
-    std::optional<uint64_t> revCount;
-    std::optional<time_t> lastModified;
-
-    std::string type() const override { return "path"; }
-
-    std::optional<Hash> getRev() const override { return rev; }
-
-    bool operator ==(const Input & other) const override
-    {
-        auto other2 = dynamic_cast<const PathInput *>(&other);
-        return
-            other2
-            && path == other2->path
-            && rev == other2->rev
-            && revCount == other2->revCount
-            && lastModified == other2->lastModified;
-    }
-
-    bool isImmutable() const override
-    {
-        return (bool) narHash;
-    }
-
-    ParsedURL toURL() const override
-    {
-        auto query = attrsToQuery(toAttrsInternal());
-        query.erase("path");
-        return ParsedURL {
-            .scheme = "path",
-            .path = path,
-            .query = query,
-        };
-    }
-
-    Attrs toAttrsInternal() const override
-    {
-        Attrs attrs;
-        attrs.emplace("path", path);
-        if (rev)
-            attrs.emplace("rev", rev->gitRev());
-        if (revCount)
-            attrs.emplace("revCount", *revCount);
-        if (lastModified)
-            attrs.emplace("lastModified", *lastModified);
-        return attrs;
-    }
-
-    std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override
-    {
-        auto input = std::make_shared<PathInput>(*this);
-
-        // FIXME: check whether access to 'path' is allowed.
-
-        auto storePath = store->maybeParseStorePath(path);
-
-        if (storePath)
-            store->addTempRoot(*storePath);
-
-        if (!storePath || storePath->name() != "source" || !store->isValidPath(*storePath))
-            // FIXME: try to substitute storePath.
-            storePath = store->addToStore("source", path);
-
-        return
-            {
-                Tree {
-                    .actualPath = store->toRealPath(*storePath),
-                    .storePath = std::move(*storePath),
-                    .info = TreeInfo {
-                        .revCount = revCount,
-                        .lastModified = lastModified
-                    }
-                },
-                input
-            };
-    }
-
-};
-
-struct PathInputScheme : InputScheme
-{
-    std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override
-    {
-        if (url.scheme != "path") return nullptr;
-
-        auto input = std::make_unique<PathInput>();
-        input->path = url.path;
-
-        for (auto & [name, value] : url.query)
-            if (name == "rev")
-                input->rev = Hash(value, htSHA1);
-            else if (name == "revCount") {
-                uint64_t revCount;
-                if (!string2Int(value, revCount))
-                    throw Error("path URL '%s' has invalid parameter '%s'", url.to_string(), name);
-                input->revCount = revCount;
-            }
-            else if (name == "lastModified") {
-                time_t lastModified;
-                if (!string2Int(value, lastModified))
-                    throw Error("path URL '%s' has invalid parameter '%s'", url.to_string(), name);
-                input->lastModified = lastModified;
-            }
-            else
-                throw Error("path URL '%s' has unsupported parameter '%s'", url.to_string(), name);
-
-        return input;
-    }
-
-    std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override
-    {
-        if (maybeGetStrAttr(attrs, "type") != "path") return {};
-
-        auto input = std::make_unique<PathInput>();
-        input->path = getStrAttr(attrs, "path");
-
-        for (auto & [name, value] : attrs)
-            if (name == "rev")
-                input->rev = Hash(getStrAttr(attrs, "rev"), htSHA1);
-            else if (name == "revCount")
-                input->revCount = getIntAttr(attrs, "revCount");
-            else if (name == "lastModified")
-                input->lastModified = getIntAttr(attrs, "lastModified");
-            else if (name == "type" || name == "path")
-                ;
-            else
-                throw Error("unsupported path input attribute '%s'", name);
-
-        return input;
-    }
-};
-
-static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<PathInputScheme>()); });
-
-}

src/libfetchers/tarball.cc

@@ -1,278 +0,0 @@
-#include "fetchers.hh"
-#include "cache.hh"
-#include "filetransfer.hh"
-#include "globals.hh"
-#include "store-api.hh"
-#include "archive.hh"
-#include "tarfile.hh"
-
-namespace nix::fetchers {
-
-DownloadFileResult downloadFile(
-    ref<Store> store,
-    const std::string & url,
-    const std::string & name,
-    bool immutable)
-{
-    // FIXME: check store
-
-    Attrs inAttrs({
-        {"type", "file"},
-        {"url", url},
-        {"name", name},
-    });
-
-    auto cached = getCache()->lookupExpired(store, inAttrs);
-
-    auto useCached = [&]() -> DownloadFileResult
-    {
-        return {
-            .storePath = std::move(cached->storePath),
-            .etag = getStrAttr(cached->infoAttrs, "etag"),
-            .effectiveUrl = getStrAttr(cached->infoAttrs, "url")
-        };
-    };
-
-    if (cached && !cached->expired)
-        return useCached();
-
-    FileTransferRequest request(url);
-    if (cached)
-        request.expectedETag = getStrAttr(cached->infoAttrs, "etag");
-    FileTransferResult res;
-    try {
-        res = getFileTransfer()->download(request);
-    } catch (FileTransferError & e) {
-        if (cached) {
-            warn("%s; using cached version", e.msg());
-            return useCached();
-        } else
-            throw;
-    }
-
-    // FIXME: write to temporary file.
-
-    Attrs infoAttrs({
-        {"etag", res.etag},
-        {"url", res.effectiveUri},
-    });
-
-    std::optional<StorePath> storePath;
-
-    if (res.cached) {
-        assert(cached);
-        assert(request.expectedETag == res.etag);
-        storePath = std::move(cached->storePath);
-    } else {
-        StringSink sink;
-        dumpString(*res.data, sink);
-        auto hash = hashString(htSHA256, *res.data);
-        ValidPathInfo info(store->makeFixedOutputPath(FileIngestionMethod::Flat, hash, name));
-        info.narHash = hashString(htSHA256, *sink.s);
-        info.narSize = sink.s->size();
-        info.ca = FixedOutputHash {
-            .method = FileIngestionMethod::Flat,
-            .hash = hash,
-        };
-        auto source = StringSource { *sink.s };
-        store->addToStore(info, source, NoRepair, NoCheckSigs);
-        storePath = std::move(info.path);
-    }
-
-    getCache()->add(
-        store,
-        inAttrs,
-        infoAttrs,
-        *storePath,
-        immutable);
-
-    if (url != res.effectiveUri)
-        getCache()->add(
-            store,
-            {
-                {"type", "file"},
-                {"url", res.effectiveUri},
-                {"name", name},
-            },
-            infoAttrs,
-            *storePath,
-            immutable);
-
-    return {
-        .storePath = std::move(*storePath),
-        .etag = res.etag,
-        .effectiveUrl = res.effectiveUri,
-    };
-}
-
-Tree downloadTarball(
-    ref<Store> store,
-    const std::string & url,
-    const std::string & name,
-    bool immutable)
-{
-    Attrs inAttrs({
-        {"type", "tarball"},
-        {"url", url},
-        {"name", name},
-    });
-
-    auto cached = getCache()->lookupExpired(store, inAttrs);
-
-    if (cached && !cached->expired)
-        return Tree {
-            .actualPath = store->toRealPath(cached->storePath),
-            .storePath = std::move(cached->storePath),
-            .info = TreeInfo {
-                .lastModified = getIntAttr(cached->infoAttrs, "lastModified"),
-            },
-        };
-
-    auto res = downloadFile(store, url, name, immutable);
-
-    std::optional<StorePath> unpackedStorePath;
-    time_t lastModified;
-
-    if (cached && res.etag != "" && getStrAttr(cached->infoAttrs, "etag") == res.etag) {
-        unpackedStorePath = std::move(cached->storePath);
-        lastModified = getIntAttr(cached->infoAttrs, "lastModified");
-    } else {
-        Path tmpDir = createTempDir();
-        AutoDelete autoDelete(tmpDir, true);
-        unpackTarfile(store->toRealPath(res.storePath), tmpDir);
-        auto members = readDirectory(tmpDir);
-        if (members.size() != 1)
-            throw nix::Error("tarball '%s' contains an unexpected number of top-level files", url);
-        auto topDir = tmpDir + "/" + members.begin()->name;
-        lastModified = lstat(topDir).st_mtime;
-        unpackedStorePath = store->addToStore(name, topDir, FileIngestionMethod::Recursive, htSHA256, defaultPathFilter, NoRepair);
-    }
-
-    Attrs infoAttrs({
-        {"lastModified", lastModified},
-        {"etag", res.etag},
-    });
-
-    getCache()->add(
-        store,
-        inAttrs,
-        infoAttrs,
-        *unpackedStorePath,
-        immutable);
-
-    return Tree {
-        .actualPath = store->toRealPath(*unpackedStorePath),
-        .storePath = std::move(*unpackedStorePath),
-        .info = TreeInfo {
-            .lastModified = lastModified,
-        },
-    };
-}
-
-struct TarballInput : Input
-{
-    ParsedURL url;
-    std::optional<Hash> hash;
-
-    TarballInput(const ParsedURL & url) : url(url)
-    { }
-
-    std::string type() const override { return "tarball"; }
-
-    bool operator ==(const Input & other) const override
-    {
-        auto other2 = dynamic_cast<const TarballInput *>(&other);
-        return
-            other2
-            && to_string() == other2->to_string()
-            && hash == other2->hash;
-    }
-
-    bool isImmutable() const override
-    {
-        return hash || narHash;
-    }
-
-    ParsedURL toURL() const override
-    {
-        auto url2(url);
-        // NAR hashes are preferred over file hashes since tar/zip files
-        // don't have a canonical representation.
-        if (narHash)
-            url2.query.insert_or_assign("narHash", narHash->to_string(SRI, true));
-        else if (hash)
-            url2.query.insert_or_assign("hash", hash->to_string(SRI, true));
-        return url2;
-    }
-
-    Attrs toAttrsInternal() const override
-    {
-        Attrs attrs;
-        attrs.emplace("url", url.to_string());
-        if (hash)
-            attrs.emplace("hash", hash->to_string(SRI, true));
-        return attrs;
-    }
-
-    std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override
-    {
-        auto tree = downloadTarball(store, url.to_string(), "source", false);
-
-        auto input = std::make_shared<TarballInput>(*this);
-        input->narHash = store->queryPathInfo(tree.storePath)->narHash;
-
-        return {std::move(tree), input};
-    }
-};
-
-struct TarballInputScheme : InputScheme
-{
-    std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override
-    {
-        if (url.scheme != "file" && url.scheme != "http" && url.scheme != "https") return nullptr;
-
-        if (!hasSuffix(url.path, ".zip")
-            && !hasSuffix(url.path, ".tar")
-            && !hasSuffix(url.path, ".tar.gz")
-            && !hasSuffix(url.path, ".tar.xz")
-            && !hasSuffix(url.path, ".tar.bz2"))
-            return nullptr;
-
-        auto input = std::make_unique<TarballInput>(url);
-
-        auto hash = input->url.query.find("hash");
-        if (hash != input->url.query.end()) {
-            // FIXME: require SRI hash.
-            input->hash = Hash(hash->second);
-            input->url.query.erase(hash);
-        }
-
-        auto narHash = input->url.query.find("narHash");
-        if (narHash != input->url.query.end()) {
-            // FIXME: require SRI hash.
-            input->narHash = Hash(narHash->second);
-            input->url.query.erase(narHash);
-        }
-
-        return input;
-    }
-
-    std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override
-    {
-        if (maybeGetStrAttr(attrs, "type") != "tarball") return {};
-
-        for (auto & [name, value] : attrs)
-            if (name != "type" && name != "url" && name != "hash")
-                throw Error("unsupported tarball input attribute '%s'", name);
-
-        auto input = std::make_unique<TarballInput>(parseURL(getStrAttr(attrs, "url")));
-        if (auto hash = maybeGetStrAttr(attrs, "hash"))
-            input->hash = newHashAllowEmpty(*hash, {});
-
-        return input;
-    }
-};
-
-static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<TarballInputScheme>()); });
-
-}

src/libfetchers/tree-info.cc

@@ -1,14 +0,0 @@
-#include "tree-info.hh"
-#include "store-api.hh"
-
-#include <nlohmann/json.hpp>
-
-namespace nix::fetchers {
-
-StorePath TreeInfo::computeStorePath(Store & store) const
-{
-    assert(narHash);
-    return store.makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, "source");
-}
-
-}

src/libfetchers/tree-info.hh

@@ -1,29 +0,0 @@
-#pragma once
-
-#include "path.hh"
-#include "hash.hh"
-
-#include <nlohmann/json_fwd.hpp>
-
-namespace nix { class Store; }
-
-namespace nix::fetchers {
-
-struct TreeInfo
-{
-    Hash narHash;
-    std::optional<uint64_t> revCount;
-    std::optional<time_t> lastModified;
-
-    bool operator ==(const TreeInfo & other) const
-    {
-        return
-            narHash == other.narHash
-            && revCount == other.revCount
-            && lastModified == other.lastModified;
-    }
-
-    StorePath computeStorePath(Store & store) const;
-};
-
-}

src/libmain/common-args.cc

@@ -1,61 +1,48 @@
 #include "common-args.hh"
 #include "globals.hh"
-#include "loggers.hh"
 
 namespace nix {
 
 MixCommonArgs::MixCommonArgs(const string & programName)
     : programName(programName)
 {
-    addFlag({
-        .longName = "verbose",
-        .shortName = 'v',
-        .description = "increase verbosity level",
-        .handler = {[]() { verbosity = (Verbosity) (verbosity + 1); }},
-    });
+    mkFlag()
+        .longName("verbose")
+        .shortName('v')
+        .description("increase verbosity level")
+        .handler([]() { verbosity = (Verbosity) (verbosity + 1); });
 
-    addFlag({
-        .longName = "quiet",
-        .description = "decrease verbosity level",
-        .handler = {[]() { verbosity = verbosity > lvlError ? (Verbosity) (verbosity - 1) : lvlError; }},
-    });
+    mkFlag()
+        .longName("quiet")
+        .description("decrease verbosity level")
+        .handler([]() { verbosity = verbosity > lvlError ? (Verbosity) (verbosity - 1) : lvlError; });
 
-    addFlag({
-        .longName = "debug",
-        .description = "enable debug output",
-        .handler = {[]() { verbosity = lvlDebug; }},
-    });
+    mkFlag()
+        .longName("debug")
+        .description("enable debug output")
+        .handler([]() { verbosity = lvlDebug; });
 
-    addFlag({
-        .longName = "option",
-        .description = "set a Nix configuration option (overriding nix.conf)",
-        .labels = {"name", "value"},
-        .handler = {[](std::string name, std::string value) {
+    mkFlag()
+        .longName("option")
+        .labels({"name", "value"})
+        .description("set a Nix configuration option (overriding nix.conf)")
+        .arity(2)
+        .handler([](std::vector<std::string> ss) {
             try {
-                globalConfig.set(name, value);
+                globalConfig.set(ss[0], ss[1]);
             } catch (UsageError & e) {
                 warn(e.what());
             }
-        }},
-    });
+        });
 
-    addFlag({
-        .longName = "log-format",
-        .description = "format of log output; \"raw\", \"internal-json\", \"bar\" "
-                        "or \"bar-with-logs\"",
-        .labels = {"format"},
-        .handler = {[](std::string format) { setLogFormat(format); }},
-    });
-
-    addFlag({
-        .longName = "max-jobs",
-        .shortName = 'j',
-        .description = "maximum number of parallel builds",
-        .labels = Strings{"jobs"},
-        .handler = {[=](std::string s) {
+    mkFlag()
+        .longName("max-jobs")
+        .shortName('j')
+        .label("jobs")
+        .description("maximum number of parallel builds")
+        .handler([=](std::string s) {
             settings.set("max-jobs", s);
-        }}
-    });
+        });
 
     std::string cat = "config";
     globalConfig.convertToArgs(*this, cat);

src/libmain/loggers.cc

@@ -1,52 +0,0 @@
-#include "loggers.hh"
-#include "progress-bar.hh"
-
-namespace nix {
-
-LogFormat defaultLogFormat = LogFormat::raw;
-
-LogFormat parseLogFormat(const std::string & logFormatStr) {
-    if (logFormatStr == "raw")
-        return LogFormat::raw;
-    else if (logFormatStr == "raw-with-logs")
-        return LogFormat::rawWithLogs;
-    else if (logFormatStr == "internal-json")
-        return LogFormat::internalJson;
-    else if (logFormatStr == "bar")
-        return LogFormat::bar;
-    else if (logFormatStr == "bar-with-logs")
-        return LogFormat::barWithLogs;
-    throw Error("option 'log-format' has an invalid value '%s'", logFormatStr);
-}
-
-Logger * makeDefaultLogger() {
-    switch (defaultLogFormat) {
-    case LogFormat::raw:
-        return makeSimpleLogger(false);
-    case LogFormat::rawWithLogs:
-        return makeSimpleLogger(true);
-    case LogFormat::internalJson:
-        return makeJSONLogger(*makeSimpleLogger(true));
-    case LogFormat::bar:
-        return makeProgressBar();
-    case LogFormat::barWithLogs:
-        return makeProgressBar(true);
-    default:
-        abort();
-    }
-}
-
-void setLogFormat(const std::string & logFormatStr) {
-    setLogFormat(parseLogFormat(logFormatStr));
-}
-
-void setLogFormat(const LogFormat & logFormat) {
-    defaultLogFormat = logFormat;
-    createDefaultLogger();
-}
-
-void createDefaultLogger() {
-    logger = makeDefaultLogger();
-}
-
-}

src/libmain/loggers.hh

@@ -1,20 +0,0 @@
-#pragma once
-
-#include "types.hh"
-
-namespace nix {
-
-enum class LogFormat {
-  raw,
-  rawWithLogs,
-  internalJson,
-  bar,
-  barWithLogs,
-};
-
-void setLogFormat(const std::string & logFormatStr);
-void setLogFormat(const LogFormat & logFormat);
-
-void createDefaultLogger();
-
-}

src/libmain/shared.cc

@@ -2,7 +2,6 @@
 #include "shared.hh"
 #include "store-api.hh"
 #include "util.hh"
-#include "loggers.hh"
 
 #include <algorithm>
 #include <cctype>
@@ -48,10 +47,7 @@ void printMissing(ref<Store> store, const StorePathSet & willBuild,
     unsigned long long downloadSize, unsigned long long narSize, Verbosity lvl)
 {
     if (!willBuild.empty()) {
-        if (willBuild.size() == 1)
-            printMsg(lvl, fmt("this derivation will be built:"));
-        else
-            printMsg(lvl, fmt("these %d derivations will be built:", willBuild.size()));
+        printMsg(lvl, "these derivations will be built:");
         auto sorted = store->topoSortPaths(willBuild);
         reverse(sorted.begin(), sorted.end());
         for (auto & i : sorted)
@@ -59,18 +55,9 @@ void printMissing(ref<Store> store, const StorePathSet & willBuild,
     }
 
     if (!willSubstitute.empty()) {
-        const float downloadSizeMiB = downloadSize / (1024.f * 1024.f);
-        const float narSizeMiB = narSize / (1024.f * 1024.f);
-        if (willSubstitute.size() == 1) {
-            printMsg(lvl, fmt("this path will be fetched (%.2f MiB download, %.2f MiB unpacked):",
-                downloadSizeMiB,
-                narSizeMiB));
-        } else {
-            printMsg(lvl, fmt("these %d paths will be fetched (%.2f MiB download, %.2f MiB unpacked):",
-                willSubstitute.size(),
-                downloadSizeMiB,
-                narSizeMiB));
-        }
+        printMsg(lvl, fmt("these paths will be fetched (%.2f MiB download, %.2f MiB unpacked):",
+                downloadSize / (1024.0 * 1024.0),
+                narSize / (1024.0 * 1024.0)));
         for (auto & i : willSubstitute)
             printMsg(lvl, fmt("  %s", store->printStorePath(i)));
     }
@@ -88,7 +75,7 @@ string getArg(const string & opt,
     Strings::iterator & i, const Strings::iterator & end)
 {
     ++i;
-    if (i == end) throw UsageError("'%1%' requires an argument", opt);
+    if (i == end) throw UsageError(format("'%1%' requires an argument") % opt);
     return *i;
 }
 
@@ -168,7 +155,7 @@ void initNix()
        sshd). This breaks build users because they don't have access
        to the TMPDIR, in particular in ‘nix-store --serve’. */
 #if __APPLE__
-    if (hasPrefix(getEnv("TMPDIR").value_or("/tmp"), "/var/folders/"))
+    if (getuid() == 0 && hasPrefix(getEnv("TMPDIR").value_or("/tmp"), "/var/folders/"))
         unsetenv("TMPDIR");
 #endif
 }
@@ -178,32 +165,28 @@ LegacyArgs::LegacyArgs(const std::string & programName,
     std::function<bool(Strings::iterator & arg, const Strings::iterator & end)> parseArg)
     : MixCommonArgs(programName), parseArg(parseArg)
 {
-    addFlag({
-        .longName = "no-build-output",
-        .shortName = 'Q',
-        .description = "do not show build output",
-        .handler = {[&]() {setLogFormat(LogFormat::raw); }},
-    });
+    mkFlag()
+        .longName("no-build-output")
+        .shortName('Q')
+        .description("do not show build output")
+        .set(&settings.verboseBuild, false);
 
-    addFlag({
-        .longName = "keep-failed",
-        .shortName ='K',
-        .description = "keep temporary directories of failed builds",
-        .handler = {&(bool&) settings.keepFailed, true},
-    });
+    mkFlag()
+        .longName("keep-failed")
+        .shortName('K')
+        .description("keep temporary directories of failed builds")
+        .set(&(bool&) settings.keepFailed, true);
 
-    addFlag({
-        .longName = "keep-going",
-        .shortName ='k',
-        .description = "keep going after a build fails",
-        .handler = {&(bool&) settings.keepGoing, true},
-    });
+    mkFlag()
+        .longName("keep-going")
+        .shortName('k')
+        .description("keep going after a build fails")
+        .set(&(bool&) settings.keepGoing, true);
 
-    addFlag({
-        .longName = "fallback",
-        .description = "build from source if substitution fails",
-        .handler = {&(bool&) settings.tryFallback, true},
-    });
+    mkFlag()
+        .longName("fallback")
+        .description("build from source if substitution fails")
+        .set(&(bool&) settings.tryFallback, true);
 
     auto intSettingAlias = [&](char shortName, const std::string & longName,
         const std::string & description, const std::string & dest) {
@@ -222,12 +205,11 @@ LegacyArgs::LegacyArgs(const std::string & programName,
     mkFlag(0, "no-gc-warning", "disable warning about not using '--add-root'",
         &gcWarning, false);
 
-    addFlag({
-        .longName = "store",
-        .description = "URI of the Nix store to use",
-        .labels = {"store-uri"},
-        .handler = {&(std::string&) settings.storeUri},
-    });
+    mkFlag()
+        .longName("store")
+        .label("store-uri")
+        .description("URI of the Nix store to use")
+        .dest(&(std::string&) settings.storeUri);
 }
 
 
@@ -247,7 +229,7 @@ bool LegacyArgs::processArgs(const Strings & args, bool finish)
     Strings ss(args);
     auto pos = ss.begin();
     if (!parseArg(pos, ss.end()))
-        throw UsageError("unexpected argument '%1%'", args.front());
+        throw UsageError(format("unexpected argument '%1%'") % args.front());
     return true;
 }
 
@@ -278,10 +260,7 @@ void printVersion(const string & programName)
         cfg.push_back("signed-caches");
 #endif
         std::cout << "Features: " << concatStringsSep(", ", cfg) << "\n";
-        std::cout << "System configuration file: " << settings.nixConfDir + "/nix.conf" << "\n";
-        std::cout << "User configuration files: " <<
-            concatStringsSep(":", settings.nixUserConfFiles)
-            << "\n";
+        std::cout << "Configuration file: " << settings.nixConfDir + "/nix.conf" << "\n";
         std::cout << "Store directory: " << settings.nixStore << "\n";
         std::cout << "State directory: " << settings.nixStateDir << "\n";
     }
@@ -294,7 +273,7 @@ void showManPage(const string & name)
     restoreSignals();
     setenv("MANPATH", settings.nixManDir.c_str(), 1);
     execlp("man", "man", name.c_str(), nullptr);
-    throw SysError("command 'man %1%' failed", name.c_str());
+    throw SysError(format("command 'man %1%' failed") % name.c_str());
 }
 
 
@@ -302,8 +281,6 @@ int handleExceptions(const string & programName, std::function<void()> fun)
 {
     ReceiveInterrupts receiveInterrupts; // FIXME: need better place for this
 
-    ErrorInfo::programName = baseNameOf(programName);
-
     string error = ANSI_RED "error:" ANSI_NORMAL " ";
     try {
         try {
@@ -319,12 +296,13 @@ int handleExceptions(const string & programName, std::function<void()> fun)
     } catch (Exit & e) {
         return e.status;
     } catch (UsageError & e) {
-        logError(e.info());
-        printError("Try '%1% --help' for more information.", programName);
+        printError(
+            format(error + "%1%\nTry '%2% --help' for more information.")
+            % e.what() % programName);
         return 1;
     } catch (BaseError & e) {
-        logError(e.info());
-        if (e.hasTrace() && !loggerSettings.showTrace.get())
+        printError(format(error + "%1%%2%") % (settings.showTrace ? e.prefix() : "") % e.msg());
+        if (e.prefix() != "" && !settings.showTrace)
             printError("(use '--show-trace' to show detailed location information)");
         return e.status;
     } catch (std::bad_alloc & e) {
@@ -360,7 +338,7 @@ RunPager::RunPager()
         execlp("pager", "pager", nullptr);
         execlp("less", "less", nullptr);
         execlp("more", "more", nullptr);
-        throw SysError("executing '%1%'", pager);
+        throw SysError(format("executing '%1%'") % pager);
     });
 
     pid.setKillSignal(SIGINT);

src/libmain/shared.hh

@@ -56,7 +56,7 @@ template<class N> N getIntArg(const string & opt,
     Strings::iterator & i, const Strings::iterator & end, bool allowUnit)
 {
     ++i;
-    if (i == end) throw UsageError("'%1%' requires an argument", opt);
+    if (i == end) throw UsageError(format("'%1%' requires an argument") % opt);
     string s = *i;
     N multiplier = 1;
     if (allowUnit && !s.empty()) {
@@ -66,13 +66,13 @@ template<class N> N getIntArg(const string & opt,
             else if (u == 'M') multiplier = 1ULL << 20;
             else if (u == 'G') multiplier = 1ULL << 30;
             else if (u == 'T') multiplier = 1ULL << 40;
-            else throw UsageError("invalid unit specifier '%1%'", u);
+            else throw UsageError(format("invalid unit specifier '%1%'") % u);
             s.resize(s.size() - 1);
         }
     }
     N n;
     if (!string2Int(s, n))
-        throw UsageError("'%1%' requires an integer argument", opt);
+        throw UsageError(format("'%1%' requires an integer argument") % opt);
     return n * multiplier;
 }
 

src/libmain/stack.cc

@@ -1,4 +1,4 @@
-#include "error.hh"
+#include "types.hh"
 
 #include <cstring>
 #include <cstddef>

src/libstore/binary-cache-store.cc

@@ -15,7 +15,6 @@
 #include <chrono>
 #include <future>
 #include <regex>
-#include <fstream>
 
 #include <nlohmann/json.hpp>
 
@@ -41,14 +40,14 @@ void BinaryCacheStore::init()
         upsertFile(cacheInfoFile, "StoreDir: " + storeDir + "\n", "text/x-nix-cache-info");
     } else {
         for (auto & line : tokenizeString<Strings>(*cacheInfo, "\n")) {
-            size_t colon= line.find(':');
-            if (colon ==std::string::npos) continue;
+            size_t colon = line.find(':');
+            if (colon == std::string::npos) continue;
             auto name = line.substr(0, colon);
             auto value = trim(line.substr(colon + 1, std::string::npos));
             if (name == "StoreDir") {
                 if (value != storeDir)
-                    throw Error("binary cache '%s' is for Nix stores with prefix '%s', not '%s'",
-                        getUri(), value, storeDir);
+                    throw Error(format("binary cache '%s' is for Nix stores with prefix '%s', not '%s'")
+                        % getUri() % value % storeDir);
             } else if (name == "WantMassQuery") {
                 wantMassQuery.setDefault(value == "1" ? "true" : "false");
             } else if (name == "Priority") {
@@ -58,13 +57,6 @@ void BinaryCacheStore::init()
     }
 }
 
-void BinaryCacheStore::upsertFile(const std::string & path,
-    std::string && data,
-    const std::string & mimeType)
-{
-    upsertFile(path, std::make_shared<std::stringstream>(std::move(data)), mimeType);
-}
-
 void BinaryCacheStore::getFile(const std::string & path,
     Callback<std::shared_ptr<std::string>> callback) noexcept
 {
@@ -101,7 +93,7 @@ std::shared_ptr<std::string> BinaryCacheStore::getFile(const std::string & path)
 
 std::string BinaryCacheStore::narInfoFileFor(const StorePath & storePath)
 {
-    return std::string(storePath.hashPart()) + ".narinfo";
+    return storePathToHash(printStorePath(storePath)) + ".narinfo";
 }
 
 void BinaryCacheStore::writeNarInfo(ref<NarInfo> narInfo)
@@ -110,7 +102,7 @@ void BinaryCacheStore::writeNarInfo(ref<NarInfo> narInfo)
 
     upsertFile(narInfoFile, narInfo->to_string(*this), "text/x-nix-narinfo");
 
-    std::string hashPart(narInfo->path.hashPart());
+    auto hashPart = storePathToHash(printStorePath(narInfo->path));
 
     {
         auto state_(state.lock());
@@ -121,74 +113,10 @@ void BinaryCacheStore::writeNarInfo(ref<NarInfo> narInfo)
         diskCache->upsertNarInfo(getUri(), hashPart, std::shared_ptr<NarInfo>(narInfo));
 }
 
-AutoCloseFD openFile(const Path & path)
+void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::string> & nar,
+    RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor)
 {
-    auto fd = open(path.c_str(), O_RDONLY | O_CLOEXEC);
-    if (!fd)
-        throw SysError("opening file '%1%'", path);
-    return fd;
-}
-
-struct FileSource : FdSource
-{
-    AutoCloseFD fd2;
-
-    FileSource(const Path & path)
-        : fd2(openFile(path))
-    {
-        fd = fd2.get();
-    }
-};
-
-void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource,
-    RepairFlag repair, CheckSigsFlag checkSigs)
-{
-    assert(info.narHash && info.narSize);
-
-    if (!repair && isValidPath(info.path)) {
-        // FIXME: copyNAR -> null sink
-        narSource.drain();
-        return;
-    }
-
-    auto [fdTemp, fnTemp] = createTempFile();
-
-    auto now1 = std::chrono::steady_clock::now();
-
-    /* Read the NAR simultaneously into a CompressionSink+FileSink (to
-       write the compressed NAR to disk), into a HashSink (to get the
-       NAR hash), and into a NarAccessor (to get the NAR listing). */
-    HashSink fileHashSink(htSHA256);
-    std::shared_ptr<FSAccessor> narAccessor;
-    {
-    FdSink fileSink(fdTemp.get());
-    TeeSink teeSink(fileSink, fileHashSink);
-    auto compressionSink = makeCompressionSink(compression, teeSink);
-    TeeSource teeSource(narSource, *compressionSink);
-    narAccessor = makeNarAccessor(teeSource);
-    compressionSink->finish();
-    }
-
-    auto now2 = std::chrono::steady_clock::now();
-
-    auto narInfo = make_ref<NarInfo>(info);
-    narInfo->narSize = info.narSize;
-    narInfo->narHash = info.narHash;
-    narInfo->compression = compression;
-    auto [fileHash, fileSize] = fileHashSink.finish();
-    narInfo->fileHash = fileHash;
-    narInfo->fileSize = fileSize;
-    narInfo->url = "nar/" + narInfo->fileHash.to_string(Base32, false) + ".nar"
-        + (compression == "xz" ? ".xz" :
-           compression == "bzip2" ? ".bz2" :
-           compression == "br" ? ".br" :
-           "");
-
-    auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
-    printMsg(lvlTalkative, "copying path '%1%' (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache",
-        printStorePath(narInfo->path), info.narSize,
-        ((1.0 - (double) fileSize / info.narSize) * 100.0),
-        duration);
+    if (!repair && isValidPath(info.path)) return;
 
     /* Verify that all references are valid. This may do some .narinfo
        reads, but typically they'll already be cached. */
@@ -201,6 +129,23 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
                 printStorePath(info.path), printStorePath(ref));
         }
 
+    assert(nar->compare(0, narMagic.size(), narMagic) == 0);
+
+    auto narInfo = make_ref<NarInfo>(info);
+
+    narInfo->narSize = nar->size();
+    narInfo->narHash = hashString(htSHA256, *nar);
+
+    if (info.narHash && info.narHash != narInfo->narHash)
+        throw Error("refusing to copy corrupted path '%1%' to binary cache", printStorePath(info.path));
+
+    auto accessor_ = std::dynamic_pointer_cast<RemoteFSAccessor>(accessor);
+
+    auto narAccessor = makeNarAccessor(nar);
+
+    if (accessor_)
+        accessor_->addToCache(printStorePath(info.path), *nar, narAccessor);
+
     /* Optionally write a JSON file containing a listing of the
        contents of the NAR. */
     if (writeNARListing) {
@@ -212,13 +157,33 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
 
             {
                 auto res = jsonRoot.placeholder("root");
-                listNar(res, ref<FSAccessor>(narAccessor), "", true);
+                listNar(res, narAccessor, "", true);
             }
         }
 
-        upsertFile(std::string(info.path.to_string()) + ".ls", jsonOut.str(), "application/json");
+        upsertFile(storePathToHash(printStorePath(info.path)) + ".ls", jsonOut.str(), "application/json");
     }
 
+    /* Compress the NAR. */
+    narInfo->compression = compression;
+    auto now1 = std::chrono::steady_clock::now();
+    auto narCompressed = compress(compression, *nar, parallelCompression);
+    auto now2 = std::chrono::steady_clock::now();
+    narInfo->fileHash = hashString(htSHA256, *narCompressed);
+    narInfo->fileSize = narCompressed->size();
+
+    auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
+    printMsg(lvlTalkative, "copying path '%1%' (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache",
+        printStorePath(narInfo->path), narInfo->narSize,
+        ((1.0 - (double) narCompressed->size() / nar->size()) * 100.0),
+        duration);
+
+    narInfo->url = "nar/" + narInfo->fileHash.to_string(Base32, false) + ".nar"
+        + (compression == "xz" ? ".xz" :
+           compression == "bzip2" ? ".bz2" :
+           compression == "br" ? ".br" :
+           "");
+
     /* Optionally maintain an index of DWARF debug info files
        consisting of JSON files named 'debuginfo/<build-id>' that
        specify the NAR file and member containing the debug info. */
@@ -279,14 +244,12 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
     /* Atomically write the NAR file. */
     if (repair || !fileExists(narInfo->url)) {
         stats.narWrite++;
-        upsertFile(narInfo->url,
-            std::make_shared<std::fstream>(fnTemp, std::ios_base::in),
-            "application/x-nix-nar");
+        upsertFile(narInfo->url, *narCompressed, "application/x-nix-nar");
     } else
         stats.narWriteAverted++;
 
-    stats.narWriteBytes += info.narSize;
-    stats.narWriteCompressedBytes += fileSize;
+    stats.narWriteBytes += nar->size();
+    stats.narWriteCompressedBytes += narCompressed->size();
     stats.narWriteCompressionTimeMs += duration;
 
     /* Atomically write the NAR info file.*/
@@ -321,7 +284,7 @@ void BinaryCacheStore::narFromPath(const StorePath & storePath, Sink & sink)
     try {
         getFile(info->url, *decompressor);
     } catch (NoSuchBinaryCacheFile & e) {
-        throw SubstituteGone(e.info());
+        throw SubstituteGone(e.what());
     }
 
     decompressor->finish();
@@ -364,7 +327,7 @@ void BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath,
 }
 
 StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath,
-    FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
+    bool recursive, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
 {
     // FIXME: some cut&paste from LocalStore::addToStore().
 
@@ -373,7 +336,7 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath
        small files. */
     StringSink sink;
     Hash h;
-    if (method == FileIngestionMethod::Recursive) {
+    if (recursive) {
         dumpPath(srcPath, sink, filter);
         h = hashString(hashAlgo, *sink.s);
     } else {
@@ -382,10 +345,9 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath
         h = hashString(hashAlgo, s);
     }
 
-    ValidPathInfo info(makeFixedOutputPath(method, h, name));
+    ValidPathInfo info(makeFixedOutputPath(recursive, h, name));
 
-    auto source = StringSource { *sink.s };
-    addToStore(info, source, repair, CheckSigs);
+    addToStore(info, sink.s, repair, CheckSigs, nullptr);
 
     return std::move(info.path);
 }
@@ -394,13 +356,12 @@ StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s
     const StorePathSet & references, RepairFlag repair)
 {
     ValidPathInfo info(computeStorePathForText(name, s, references));
-    info.references = references;
+    info.references = cloneStorePathSet(references);
 
     if (repair || !isValidPath(info.path)) {
         StringSink sink;
         dumpString(s, sink);
-        auto source = StringSource { *sink.s };
-        addToStore(info, source, repair, CheckSigs);
+        addToStore(info, sink.s, repair, CheckSigs, nullptr);
     }
 
     return std::move(info.path);
@@ -422,19 +383,21 @@ void BinaryCacheStore::addSignatures(const StorePath & storePath, const StringSe
 
     narInfo->sigs.insert(sigs.begin(), sigs.end());
 
+    auto narInfoFile = narInfoFileFor(narInfo->path);
+
     writeNarInfo(narInfo);
 }
 
 std::shared_ptr<std::string> BinaryCacheStore::getBuildLog(const StorePath & path)
 {
-    auto drvPath = path;
+    auto drvPath = path.clone();
 
     if (!path.isDerivation()) {
         try {
             auto info = queryPathInfo(path);
             // FIXME: add a "Log" field to .narinfo
             if (!info->deriver) return nullptr;
-            drvPath = *info->deriver;
+            drvPath = info->deriver->clone();
         } catch (InvalidPath &) {
             return nullptr;
         }

src/libstore/binary-cache-store.hh

@@ -36,13 +36,9 @@ public:
     virtual bool fileExists(const std::string & path) = 0;
 
     virtual void upsertFile(const std::string & path,
-        std::shared_ptr<std::basic_iostream<char>> istream,
+        const std::string & data,
         const std::string & mimeType) = 0;
 
-    void upsertFile(const std::string & path,
-        std::string && data,
-        const std::string & mimeType);
-
     /* Note: subclasses must implement at least one of the two
        following getFile() methods. */
 
@@ -78,11 +74,12 @@ public:
     std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override
     { unsupported("queryPathFromHashPart"); }
 
-    void addToStore(const ValidPathInfo & info, Source & narSource,
-        RepairFlag repair, CheckSigsFlag checkSigs) override;
+    void addToStore(const ValidPathInfo & info, const ref<std::string> & nar,
+        RepairFlag repair, CheckSigsFlag checkSigs,
+        std::shared_ptr<FSAccessor> accessor) override;
 
     StorePath addToStore(const string & name, const Path & srcPath,
-        FileIngestionMethod method, HashType hashAlgo,
+        bool recursive, HashType hashAlgo,
         PathFilter & filter, RepairFlag repair) override;
 
     StorePath addTextToStore(const string & name, const string & s,

src/libstore/builtins/buildenv.cc

@@ -22,10 +22,7 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir,
         srcFiles = readDirectory(srcDir);
     } catch (SysError & e) {
         if (e.errNo == ENOTDIR) {
-            logWarning({
-                .name = "Create links - directory",
-                .hint = hintfmt("not including '%s' in the user environment because it's not a directory", srcDir)
-            });
+            printError("warning: not including '%s' in the user environment because it's not a directory", srcDir);
             return;
         }
         throw;
@@ -44,10 +41,7 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir,
                 throw SysError("getting status of '%1%'", srcFile);
         } catch (SysError & e) {
             if (e.errNo == ENOENT || e.errNo == ENOTDIR) {
-                logWarning({
-                    .name = "Create links - skipping symlink",
-                    .hint = hintfmt("skipping dangling symlink '%s'", dstFile)
-                });
+                printError("warning: skipping dangling symlink '%s'", dstFile);
                 continue;
             }
             throw;
@@ -78,15 +72,15 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir,
                     if (!S_ISDIR(lstat(target).st_mode))
                         throw Error("collision between '%1%' and non-directory '%2%'", srcFile, target);
                     if (unlink(dstFile.c_str()) == -1)
-                        throw SysError("unlinking '%1%'", dstFile);
+                        throw SysError(format("unlinking '%1%'") % dstFile);
                     if (mkdir(dstFile.c_str(), 0755) == -1)
-                        throw SysError("creating directory '%1%'", dstFile);
+                        throw SysError(format("creating directory '%1%'"));
                     createLinks(state, target, dstFile, state.priorities[dstFile]);
                     createLinks(state, srcFile, dstFile, priority);
                     continue;
                 }
             } else if (errno != ENOENT)
-                throw SysError("getting status of '%1%'", dstFile);
+                throw SysError(format("getting status of '%1%'") % dstFile);
         }
 
         else {
@@ -105,11 +99,11 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir,
                     if (prevPriority < priority)
                         continue;
                     if (unlink(dstFile.c_str()) == -1)
-                        throw SysError("unlinking '%1%'", dstFile);
+                        throw SysError(format("unlinking '%1%'") % dstFile);
                 } else if (S_ISDIR(dstSt.st_mode))
                     throw Error("collision between non-directory '%1%' and directory '%2%'", srcFile, dstFile);
             } else if (errno != ENOENT)
-                throw SysError("getting status of '%1%'", dstFile);
+                throw SysError(format("getting status of '%1%'") % dstFile);
         }
 
         createSymlink(srcFile, dstFile);

src/libstore/builtins/fetchurl.cc

@@ -1,5 +1,5 @@
 #include "builtins.hh"
-#include "filetransfer.hh"
+#include "download.hh"
 #include "store-api.hh"
 #include "archive.hh"
 #include "compression.hh"
@@ -18,7 +18,7 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
 
     auto getAttr = [&](const string & name) {
         auto i = drv.env.find(name);
-        if (i == drv.env.end()) throw Error("attribute '%s' missing", name);
+        if (i == drv.env.end()) throw Error(format("attribute '%s' missing") % name);
         return i->second;
     };
 
@@ -26,9 +26,9 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
     auto mainUrl = getAttr("url");
     bool unpack = get(drv.env, "unpack").value_or("") == "1";
 
-    /* Note: have to use a fresh fileTransfer here because we're in
+    /* Note: have to use a fresh downloader here because we're in
        a forked process. */
-    auto fileTransfer = makeFileTransfer();
+    auto downloader = makeDownloader();
 
     auto fetch = [&](const std::string & url) {
 
@@ -36,13 +36,13 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
 
             /* No need to do TLS verification, because we check the hash of
                the result anyway. */
-            FileTransferRequest request(url);
+            DownloadRequest request(url);
             request.verifyTLS = false;
             request.decompress = false;
 
             auto decompressor = makeDecompressionSink(
                 unpack && hasSuffix(mainUrl, ".xz") ? "xz" : "none", sink);
-            fileTransfer->download(std::move(request), *decompressor);
+            downloader->download(std::move(request), *decompressor);
             decompressor->finish();
         });
 
@@ -54,21 +54,18 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
         auto executable = drv.env.find("executable");
         if (executable != drv.env.end() && executable->second == "1") {
             if (chmod(storePath.c_str(), 0755) == -1)
-                throw SysError("making '%1%' executable", storePath);
+                throw SysError(format("making '%1%' executable") % storePath);
         }
     };
 
-    /* We always have one output, and if it's a fixed-output derivation (as
-       checked below) it must be the only output */
-    auto & output = drv.outputs.begin()->second;
-
     /* Try the hashed mirrors first. */
-    if (output.hash && output.hash->method == FileIngestionMethod::Flat)
+    if (getAttr("outputHashMode") == "flat")
         for (auto hashedMirror : settings.hashedMirrors.get())
             try {
                 if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/';
-                auto & h = output.hash->hash;
-                fetch(hashedMirror + printHashType(*h.type) + "/" + h.to_string(Base16, false));
+                auto ht = parseHashType(getAttr("outputHashAlgo"));
+                auto h = Hash(getAttr("outputHash"), ht);
+                fetch(hashedMirror + printHashType(h.type) + "/" + h.to_string(Base16, false));
                 return;
             } catch (Error & e) {
                 debug(e.what());

src/libstore/content-address.cc

@@ -1,97 +0,0 @@
-#include "content-address.hh"
-
-namespace nix {
-
-std::string FixedOutputHash::printMethodAlgo() const {
-    return makeFileIngestionPrefix(method) + printHashType(*hash.type);
-}
-
-std::string makeFileIngestionPrefix(const FileIngestionMethod m) {
-    switch (m) {
-    case FileIngestionMethod::Flat:
-        return "";
-    case FileIngestionMethod::Recursive:
-        return "r:";
-    default:
-        throw Error("impossible, caught both cases");
-    }
-}
-
-std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash)
-{
-    return "fixed:"
-        + makeFileIngestionPrefix(method)
-        + hash.to_string(Base32, true);
-}
-
-// FIXME Put this somewhere?
-template<class... Ts> struct overloaded : Ts... { using Ts::operator()...; };
-template<class... Ts> overloaded(Ts...) -> overloaded<Ts...>;
-
-std::string renderContentAddress(ContentAddress ca) {
-    return std::visit(overloaded {
-        [](TextHash th) {
-            return "text:" + th.hash.to_string(Base32, true);
-        },
-        [](FixedOutputHash fsh) {
-            return makeFixedOutputCA(fsh.method, fsh.hash);
-        }
-    }, ca);
-}
-
-ContentAddress parseContentAddress(std::string_view rawCa) {
-    auto prefixSeparator = rawCa.find(':');
-    if (prefixSeparator != string::npos) {
-        auto prefix = string(rawCa, 0, prefixSeparator);
-        if (prefix == "text") {
-            auto hashTypeAndHash = rawCa.substr(prefixSeparator+1, string::npos);
-            Hash hash = Hash(string(hashTypeAndHash));
-            if (*hash.type != htSHA256) {
-                throw Error("parseContentAddress: the text hash should have type SHA256");
-            }
-            return TextHash { hash };
-        } else if (prefix == "fixed") {
-            // This has to be an inverse of makeFixedOutputCA
-            auto methodAndHash = rawCa.substr(prefixSeparator+1, string::npos);
-            if (methodAndHash.substr(0,2) == "r:") {
-                std::string_view hashRaw = methodAndHash.substr(2,string::npos);
-                return FixedOutputHash {
-                    .method = FileIngestionMethod::Recursive,
-                    .hash = Hash(string(hashRaw)),
-                };
-            } else {
-                std::string_view hashRaw = methodAndHash;
-                return FixedOutputHash {
-                    .method = FileIngestionMethod::Flat,
-                    .hash = Hash(string(hashRaw)),
-                };
-            }
-        } else {
-            throw Error("parseContentAddress: format not recognized; has to be text or fixed");
-        }
-    } else {
-        throw Error("Not a content address because it lacks an appropriate prefix");
-    }
-};
-
-std::optional<ContentAddress> parseContentAddressOpt(std::string_view rawCaOpt) {
-    return rawCaOpt == "" ? std::optional<ContentAddress> {} : parseContentAddress(rawCaOpt);
-};
-
-std::string renderContentAddress(std::optional<ContentAddress> ca) {
-    return ca ? renderContentAddress(*ca) : "";
-}
-
-Hash getContentAddressHash(const ContentAddress & ca)
-{
-    return std::visit(overloaded {
-        [](TextHash th) {
-            return th.hash;
-        },
-        [](FixedOutputHash fsh) {
-            return fsh.hash;
-        }
-    }, ca);
-}
-
-}

src/libstore/content-address.hh

@@ -1,58 +0,0 @@
-#pragma once
-
-#include <variant>
-#include "hash.hh"
-
-namespace nix {
-
-enum struct FileIngestionMethod : uint8_t {
-    Flat = false,
-    Recursive = true
-};
-
-struct TextHash {
-    Hash hash;
-};
-
-/// Pair of a hash, and how the file system was ingested
-struct FixedOutputHash {
-    FileIngestionMethod method;
-    Hash hash;
-    std::string printMethodAlgo() const;
-};
-
-/*
-  We've accumulated several types of content-addressed paths over the years;
-  fixed-output derivations support multiple hash algorithms and serialisation
-  methods (flat file vs NAR). Thus, ‘ca’ has one of the following forms:
-
-  * ‘text:sha256:<sha256 hash of file contents>’: For paths
-    computed by makeTextPath() / addTextToStore().
-
-  * ‘fixed:<r?>:<ht>:<h>’: For paths computed by
-    makeFixedOutputPath() / addToStore().
-*/
-typedef std::variant<
-    TextHash, // for paths computed by makeTextPath() / addTextToStore
-    FixedOutputHash // for path computed by makeFixedOutputPath
-> ContentAddress;
-
-/* Compute the prefix to the hash algorithm which indicates how the files were
-   ingested. */
-std::string makeFileIngestionPrefix(const FileIngestionMethod m);
-
-/* Compute the content-addressability assertion (ValidPathInfo::ca)
-   for paths created by makeFixedOutputPath() / addToStore(). */
-std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash);
-
-std::string renderContentAddress(ContentAddress ca);
-
-std::string renderContentAddress(std::optional<ContentAddress> ca);
-
-ContentAddress parseContentAddress(std::string_view rawCa);
-
-std::optional<ContentAddress> parseContentAddressOpt(std::string_view rawCaOpt);
-
-Hash getContentAddressHash(const ContentAddress & ca);
-
-}

src/libstore/daemon.cc

@@ -73,18 +73,6 @@ struct TunnelLogger : public Logger
         enqueueMsg(*buf.s);
     }
 
-    void logEI(const ErrorInfo & ei) override
-    {
-        if (ei.level > verbosity) return;
-
-        std::stringstream oss;
-        showErrorInfo(oss, ei, false);
-
-        StringSink buf;
-        buf << STDERR_NEXT << oss.str();
-        enqueueMsg(*buf.s);
-    }
-
     /* startWork() means that we're starting an operation for which we
       want to send out stderr to the client. */
     void startWork()
@@ -293,7 +281,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         auto path = store->parseStorePath(readString(from));
         logger->startWork();
         StorePathSet paths; // FIXME
-        paths.insert(path);
+        paths.insert(path.clone());
         auto res = store->querySubstitutablePaths(paths);
         logger->stopWork();
         to << (res.count(path) != 0);
@@ -327,7 +315,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         StorePathSet paths;
         if (op == wopQueryReferences)
             for (auto & i : store->queryPathInfo(path)->references)
-                paths.insert(i);
+                paths.insert(i.clone());
         else if (op == wopQueryReferrers)
             store->queryReferrers(path, paths);
         else if (op == wopQueryValidDerivers)
@@ -341,21 +329,13 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     case wopQueryDerivationOutputNames: {
         auto path = store->parseStorePath(readString(from));
         logger->startWork();
-        auto names = store->readDerivation(path).outputNames();
+        StringSet names;
+        names = store->queryDerivationOutputNames(path);
         logger->stopWork();
         to << names;
         break;
     }
 
-    case wopQueryDerivationOutputMap: {
-        auto path = store->parseStorePath(readString(from));
-        logger->startWork();
-        OutputPathMap outputs = store->queryDerivationOutputMap(path);
-        logger->stopWork();
-        writeOutputPathMap(*store, to, outputs);
-        break;
-    }
-
     case wopQueryDeriver: {
         auto path = store->parseStorePath(readString(from));
         logger->startWork();
@@ -375,43 +355,32 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     }
 
     case wopAddToStore: {
+        bool fixed, recursive;
         std::string s, baseName;
-        FileIngestionMethod method;
-        {
-            bool fixed; uint8_t recursive;
-            from >> baseName >> fixed /* obsolete */ >> recursive >> s;
-            if (recursive > (uint8_t) FileIngestionMethod::Recursive)
-                throw Error("unsupported FileIngestionMethod with value of %i; you may need to upgrade nix-daemon", recursive);
-            method = FileIngestionMethod { recursive };
-            /* Compatibility hack. */
-            if (!fixed) {
-                s = "sha256";
-                method = FileIngestionMethod::Recursive;
-            }
+        from >> baseName >> fixed /* obsolete */ >> recursive >> s;
+        /* Compatibility hack. */
+        if (!fixed) {
+            s = "sha256";
+            recursive = true;
         }
         HashType hashAlgo = parseHashType(s);
 
-        StringSink savedNAR;
-        TeeSource savedNARSource(from, savedNAR);
+        TeeSource savedNAR(from);
         RetrieveRegularNARSink savedRegular;
 
-        if (method == FileIngestionMethod::Recursive) {
+        if (recursive) {
             /* Get the entire NAR dump from the client and save it to
                a string so that we can pass it to
                addToStoreFromDump(). */
             ParseSink sink; /* null sink; just parse the NAR */
-            parseDump(sink, savedNARSource);
+            parseDump(sink, savedNAR);
         } else
             parseDump(savedRegular, from);
 
         logger->startWork();
         if (!savedRegular.regular) throw Error("regular file expected");
 
-        auto path = store->addToStoreFromDump(
-            method == FileIngestionMethod::Recursive ? *savedNAR.s : savedRegular.s,
-            baseName,
-            method,
-            hashAlgo);
+        auto path = store->addToStoreFromDump(recursive ? *savedNAR.data : savedRegular.s, baseName, recursive, hashAlgo);
         logger->stopWork();
 
         to << store->printStorePath(path);
@@ -443,7 +412,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     case wopImportPaths: {
         logger->startWork();
         TunnelSource source(from, to);
-        auto paths = store->importPaths(source,
+        auto paths = store->importPaths(source, nullptr,
             trusted ? NoCheckSigs : CheckSigs);
         logger->stopWork();
         Strings paths2;
@@ -603,7 +572,9 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         auto path = store->parseStorePath(readString(from));
         logger->startWork();
         SubstitutablePathInfos infos;
-        store->querySubstitutablePathInfos({path}, infos);
+        StorePathSet paths;
+        paths.insert(path.clone()); // FIXME
+        store->querySubstitutablePathInfos(paths, infos);
         logger->stopWork();
         auto i = infos.find(path);
         if (i == infos.end())
@@ -662,7 +633,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
             if (GET_PROTOCOL_MINOR(clientVersion) >= 16) {
                 to << info->ultimate
                    << info->sigs
-                   << renderContentAddress(info->ca);
+                   << info->ca;
             }
         } else {
             assert(GET_PROTOCOL_MINOR(clientVersion) >= 17);
@@ -720,8 +691,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         info.references = readStorePaths<StorePathSet>(*store, from);
         from >> info.registrationTime >> info.narSize >> info.ultimate;
         info.sigs = readStrings<StringSet>(from);
-        info.ca = parseContentAddressOpt(readString(from));
-        from >> repair >> dontCheckSigs;
+        from >> info.ca >> repair >> dontCheckSigs;
         if (!trusted && dontCheckSigs)
             dontCheckSigs = false;
         if (!trusted)
@@ -732,9 +702,9 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         if (GET_PROTOCOL_MINOR(clientVersion) >= 21)
             source = std::make_unique<TunnelSource>(from, to);
         else {
-            TeeParseSink tee(from);
+            TeeSink tee(from);
             parseDump(tee, tee.source);
-            saved = std::move(*tee.saved.s);
+            saved = std::move(*tee.source.data);
             source = std::make_unique<StringSource>(saved);
         }
 
@@ -742,7 +712,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
 
         // FIXME: race if addToStore doesn't read source?
         store->addToStore(info, *source, (RepairFlag) repair,
-            dontCheckSigs ? NoCheckSigs : CheckSigs);
+            dontCheckSigs ? NoCheckSigs : CheckSigs, nullptr);
 
         logger->stopWork();
         break;
@@ -765,7 +735,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     }
 
     default:
-        throw Error("invalid operation %1%", op);
+        throw Error(format("invalid operation %1%") % op);
     }
 }