Add 'nix diff-closures' command

This command makes it easier to see what changed between two closures,
i.e. what packages/versions got added or removed, and whether there
were any notable changes in path size.

For example:

  $ nix diff-closures /nix/var/nix/profiles/system-655-link /nix/var/nix/profiles/system-658-link
  blender-bin: 2.83.0 → 2.83.2, -294.2 KiB
  curl: 7.68.0 → 7.70.0, +19.1 KiB
  firmware-linux-nonfree: 2020-01-22 → 2020-05-19, +30827.7 KiB
  ibus: -21.8 KiB
  initrd-linux: 5.4.46 → 5.4.51, +16.9 KiB
  libexif: 0.6.21 → 0.6.22, +497.6 KiB
  linux: 5.4.46 → 5.4.51, +13.2 KiB
  mesa: 19.3.3 → 19.3.5, -183.9 KiB
  nix: 2.4pre20200701_6ff9aa8 → 2.4pre20200708_9223603, +9.7 KiB
  nix-bash-completions: 0.6.8 → ∅, -57.6 KiB
  nixos-system-hagbard: 20.03.20200615.a84b797 → 20.03.20200713.add5529
  nvidia-persistenced: 440.82 → 440.100
  nvidia-settings: 440.82 → 440.100
  nvidia-x11: 440.82-5.4.46 → 440.100-5.4.51, +664.7 KiB
  pcre: 8.43 → 8.44
  php: 7.3.16 → 7.3.20, -26.2 KiB
  python3.7-youtube-dl: 2020.06.06 → 2020.06.16.1, +8.4 KiB
  samba: 4.11.5 → 4.11.9, +30.1 KiB
  sane-backends: 1.0.28 → 1.0.30, +680.5 KiB
  source: -182.0 KiB
  zfs-kernel: 0.8.3-5.4.46 → 0.8.4-5.4.51, +9.9 KiB
  zfs-user: 0.8.3 → 0.8.4, +20.1 KiB

.dir-locals.el

@@ -1,6 +1,7 @@
 ((c++-mode . (
   (c-file-style . "k&r")
   (c-basic-offset . 4)
+  (c-block-comment-prefix . "  ")
   (indent-tabs-mode . nil)
   (tab-width . 4)
   (show-trailing-whitespace . t)
@@ -13,4 +14,5 @@
   (eval . (c-set-offset 'arglist-cont-nonempty '+))
   (eval . (c-set-offset 'substatement-open 0))
   (eval . (c-set-offset 'access-label '-))
+  (eval . (c-set-offset 'inlambda 0))
   )))

.github/ISSUE_TEMPLATE.md

@@ -1,27 +0,0 @@
-<!--
-
-# Filing a Nix issue
-
-*WAIT* Are you sure you're filing your issue in the right repository?
-
-We appreciate you taking the time to tell us about issues you encounter, but routing the issue to the right place will get you help sooner and save everyone time.
-
-This is the Nix repository, and issues here should be about Nix the build and package management *_tool_*.
-
-If you have a problem with a specific package on NixOS or when using Nix, you probably want to file an issue with _nixpkgs_, whose issue tracker is over at https://github.com/NixOS/nixpkgs/issues.
-
-Examples of _Nix_ issues:
-
-- Nix segfaults when I run `nix-build -A blahblah`
-- The Nix language needs a new builtin: `builtins.foobar`
-- Regression in the behavior of `nix-env` in Nix 2.0
-
-Examples of _nixpkgs_ issues:
-
-- glibc is b0rked on aarch64
-- chromium in NixOS doesn't support U2F but google-chrome does!
-- The OpenJDK package on macOS is missing a key component
-
-Chances are if you're a newcomer to the Nix world, you'll probably want the [nixpkgs tracker](https://github.com/NixOS/nixpkgs/issues). It also gets a lot more eyeball traffic so you'll probably get a response a lot more quickly.
-
--->

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,32 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: bug
+assignees: ''
+
+---
+
+**Describe the bug**
+
+A clear and concise description of what the bug is.
+
+If you have a problem with a specific package or NixOS,
+you probably want to file an issue at https://github.com/NixOS/nixpkgs/issues. 
+
+**Steps To Reproduce**
+
+1. Go to '...'
+2. Click on '....'
+3. Scroll down to '....'
+4. See error
+
+**Expected behavior**
+
+A clear and concise description of what you expected to happen.
+
+**`nix-env --version` output**
+
+**Additional context**
+
+Add any other context about the problem here.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,20 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: improvement
+assignees: ''
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

.github/dependabot.yml

@@ -0,0 +1,6 @@
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"

.github/workflows/test.yml

@@ -0,0 +1,24 @@
+name: "Test"
+on:
+  pull_request:
+  push:
+jobs:
+  tests:
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+    runs-on: ${{ matrix.os }}
+    steps:
+    - uses: actions/checkout@v2
+    - uses: cachix/install-nix-action@v10
+    - run: nix-build release.nix --arg nix '{ outPath = ./.; revCount = 123; shortRev = "abcdefgh"; }' --arg systems '[ builtins.currentSystem ]' -A installerScript -A perlBindings
+  macos_perf_test:
+    runs-on: macos-latest
+    steps:
+    - name: Disable syspolicy assessments
+      run: |
+        spctl --status
+        sudo spctl --master-disable
+    - uses: actions/checkout@v2
+    - uses: cachix/install-nix-action@v10
+    - run: nix-build release.nix --arg nix '{ outPath = ./.; revCount = 123; shortRev = "abcdefgh"; }' --arg systems '[ builtins.currentSystem ]' -A installerScript -A perlBindings

.gitignore

@@ -47,7 +47,10 @@ perl/Makefile.config
 /src/libexpr/nix.tbl
 
 # /src/libstore/
-/src/libstore/*.gen.hh
+*.gen.*
+
+# /src/libutil/
+/src/libutil/tests/libutil-tests
 
 /src/nix/nix
 
@@ -75,6 +78,8 @@ perl/Makefile.config
 
 /src/nix-copy-closure/nix-copy-closure
 
+/src/error-demo/error-demo
+
 /src/build-remote/build-remote
 
 # /tests/

.travis.yml

@@ -1,8 +0,0 @@
-matrix:
-  include:
-    - language: osx
-      script: ./tests/install-darwin.sh
-    - language: nix
-      script: nix-build release.nix -A build.x86_64-linux
-notifications:
-  email: false

Makefile

@@ -1,9 +1,10 @@
 makefiles = \
   mk/precompiled-headers.mk \
   local.mk \
-  nix-rust/local.mk \
   src/libutil/local.mk \
+  src/libutil/tests/local.mk \
   src/libstore/local.mk \
+  src/libfetchers/local.mk \
   src/libmain/local.mk \
   src/libexpr/local.mk \
   src/nix/local.mk \

Makefile.config.in

@@ -1,41 +1,44 @@
 AR = @AR@
+BDW_GC_LIBS = @BDW_GC_LIBS@
+BOOST_LDFLAGS = @BOOST_LDFLAGS@
 BUILD_SHARED_LIBS = @BUILD_SHARED_LIBS@
 CC = @CC@
 CFLAGS = @CFLAGS@
 CXX = @CXX@
 CXXFLAGS = @CXXFLAGS@
-LDFLAGS = @LDFLAGS@
+EDITLINE_LIBS = @EDITLINE_LIBS@
 ENABLE_S3 = @ENABLE_S3@
-HAVE_SODIUM = @HAVE_SODIUM@
+GTEST_LIBS = @GTEST_LIBS@
 HAVE_SECCOMP = @HAVE_SECCOMP@
-BOOST_LDFLAGS = @BOOST_LDFLAGS@
+HAVE_SODIUM = @HAVE_SODIUM@
+LDFLAGS = @LDFLAGS@
+LIBARCHIVE_LIBS = @LIBARCHIVE_LIBS@
+LIBBROTLI_LIBS = @LIBBROTLI_LIBS@
 LIBCURL_LIBS = @LIBCURL_LIBS@
+LIBLZMA_LIBS = @LIBLZMA_LIBS@
 OPENSSL_LIBS = @OPENSSL_LIBS@
 PACKAGE_NAME = @PACKAGE_NAME@
 PACKAGE_VERSION = @PACKAGE_VERSION@
 SODIUM_LIBS = @SODIUM_LIBS@
-LIBLZMA_LIBS = @LIBLZMA_LIBS@
 SQLITE3_LIBS = @SQLITE3_LIBS@
-LIBBROTLI_LIBS = @LIBBROTLI_LIBS@
-LIBARCHIVE_LIBS = @LIBARCHIVE_LIBS@
-EDITLINE_LIBS = @EDITLINE_LIBS@
 bash = @bash@
 bindir = @bindir@
-lsof = @lsof@
 datadir = @datadir@
 datarootdir = @datarootdir@
+doc_generate = @doc_generate@
 docdir = @docdir@
 exec_prefix = @exec_prefix@
 includedir = @includedir@
 libdir = @libdir@
 libexecdir = @libexecdir@
 localstatedir = @localstatedir@
+lsof = @lsof@
 mandir = @mandir@
 pkglibdir = $(libdir)/$(PACKAGE_NAME)
 prefix = @prefix@
 sandbox_shell = @sandbox_shell@
 storedir = @storedir@
 sysconfdir = @sysconfdir@
-doc_generate = @doc_generate@
+system = @system@
 xmllint = @xmllint@
 xsltproc = @xsltproc@

README.md

@@ -1,24 +1,54 @@
+# Nix
+
 [![Open Collective supporters](https://opencollective.com/nixos/tiers/supporter/badge.svg?label=Supporters&color=brightgreen)](https://opencollective.com/nixos)
+[![Test](https://github.com/NixOS/nix/workflows/Test/badge.svg)](https://github.com/NixOS/nix/actions)
 
-Nix, the purely functional package manager
-------------------------------------------
+Nix is a powerful package manager for Linux and other Unix systems that makes package
+management reliable and reproducible. Please refer to the [Nix manual](https://nixos.org/nix/manual)
+for more details.
 
-Nix is a new take on package management that is fairly unique. Because of its
-purity aspects, a lot of issues found in traditional package managers don't
-appear with Nix.
+## Installation
 
-To find out more about the tool, usage and installation instructions, please
-read the manual, which is available on the Nix website at
-<http://nixos.org/nix/manual>.
+On Linux and macOS the easiest way to Install Nix is to run the following shell command
+(as a user other than root):
 
-## Contributing
+```
+$ curl -L https://nixos.org/nix/install | sh
+```
 
-Take a look at the [Hacking Section](http://nixos.org/nix/manual/#chap-hacking)
-of the manual. It helps you to get started with building Nix from source.
+Information on additional installation methods is available on the [Nix download page](https://nixos.org/download.html).
+
+## Building And Developing
+
+### Building Nix
+
+You can build Nix using one of the targets provided by [release.nix](./release.nix):
+
+```
+$ nix-build ./release.nix -A build.aarch64-linux
+$ nix-build ./release.nix -A build.x86_64-darwin
+$ nix-build ./release.nix -A build.i686-linux
+$ nix-build ./release.nix -A build.x86_64-linux
+```
+
+### Development Environment
+
+You can use the provided `shell.nix` to get a working development environment:
+
+```
+$ nix-shell
+$ ./bootstrap.sh
+$ ./configure
+$ make
+```
+
+## Additional Resources
+
+- [Nix manual](https://nixos.org/nix/manual)
+- [Nix jobsets on hydra.nixos.org](https://hydra.nixos.org/project/nix)
+- [NixOS Discourse](https://discourse.nixos.org/)
+- [IRC - #nixos on freenode.net](irc://irc.freenode.net/#nixos)
 
 ## License
 
-Nix is released under the LGPL v2.1
-
-This product includes software developed by the OpenSSL Project for
-use in the [OpenSSL Toolkit](http://www.OpenSSL.org/).
+Nix is released under the [LGPL v2.1](./COPYING).

configure.ac

@@ -255,6 +255,21 @@ if test -n "$enable_s3"; then
 fi
 
 
+# Whether to use the Boehm garbage collector.
+AC_ARG_ENABLE(gc, AC_HELP_STRING([--enable-gc],
+  [enable garbage collection in the Nix expression evaluator (requires Boehm GC) [default=yes]]),
+  gc=$enableval, gc=yes)
+if test "$gc" = yes; then
+  PKG_CHECK_MODULES([BDW_GC], [bdw-gc])
+  CXXFLAGS="$BDW_GC_CFLAGS $CXXFLAGS"
+  AC_DEFINE(HAVE_BOEHMGC, 1, [Whether to use the Boehm garbage collector.])
+fi
+
+
+# Look for gtest.
+PKG_CHECK_MODULES([GTEST], [gtest_main])
+
+
 # documentation generation switch
 AC_ARG_ENABLE(doc-gen, AC_HELP_STRING([--disable-doc-gen],
   [disable documentation generation]),

corepkgs/local.mk

@@ -1,4 +1,7 @@
-corepkgs_FILES = buildenv.nix unpack-channel.nix derivation.nix fetchurl.nix imported-drv-to-derivation.nix
+corepkgs_FILES = \
+  unpack-channel.nix \
+  derivation.nix \
+  fetchurl.nix
 
 $(foreach file,config.nix $(corepkgs_FILES),$(eval $(call install-data-in,$(d)/$(file),$(datadir)/nix/corepkgs)))
 

doc/manual/advanced-topics/diff-hook.xml

@@ -70,7 +70,7 @@ path just built.</para>
 
   <screen>
 $ nix-build ./deterministic.nix -A stable
-these derivations will be built:
+this derivation will be built:
   /nix/store/z98fasz2jqy9gs0xbvdj939p27jwda38-stable.drv
 building '/nix/store/z98fasz2jqy9gs0xbvdj939p27jwda38-stable.drv'...
 /nix/store/yyxlzw3vqaas7wfp04g0b1xg51f2czgq-stable
@@ -85,7 +85,7 @@ checking outputs of '/nix/store/z98fasz2jqy9gs0xbvdj939p27jwda38-stable.drv'...
 
   <screen>
 $ nix-build ./deterministic.nix -A unstable
-these derivations will be built:
+this derivation will be built:
   /nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv
 building '/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv'...
 /nix/store/krpqk0l9ib0ibi1d2w52z293zw455cap-unstable
@@ -193,7 +193,7 @@ repeat = 1
     An example output of this configuration:
     <screen>
 $ nix-build ./test.nix -A unstable
-these derivations will be built:
+this derivation will be built:
   /nix/store/ch6llwpr2h8c3jmnf3f2ghkhx59aa97f-unstable.drv
 building '/nix/store/ch6llwpr2h8c3jmnf3f2ghkhx59aa97f-unstable.drv' (round 1/2)...
 building '/nix/store/ch6llwpr2h8c3jmnf3f2ghkhx59aa97f-unstable.drv' (round 2/2)...

doc/manual/advanced-topics/post-build-hook.xml

@@ -61,7 +61,7 @@ substituters = https://cache.nixos.org/ s3://example-nix-cache
 trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= example-nix-cache-1:1/cKDz3QCCOmwcztD2eV6Coggp6rqc9DGjWv7C0G+rM=
 </programlisting>
 
-<para>we will restart the Nix daemon a later step.</para>
+<para>We will restart the Nix daemon in a later step.</para>
 </section>
 
 <section>
@@ -122,7 +122,7 @@ post-build-hook = /etc/nix/upload-to-cache.sh
 
   <screen>
 $ nix-build -E '(import &lt;nixpkgs&gt; {}).writeText "example" (builtins.toString builtins.currentTime)'
-these derivations will be built:
+this derivation will be built:
   /nix/store/s4pnfbkalzy5qz57qs6yybna8wylkig6-example.drv
 building '/nix/store/s4pnfbkalzy5qz57qs6yybna8wylkig6-example.drv'...
 running post-build-hook '/home/grahamc/projects/github.com/NixOS/nix/post-hook.sh'...
@@ -139,7 +139,7 @@ $ nix-store --delete /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example
 
 <para>Now, copy the path back from the cache:</para>
 <screen>
-$ nix store --realize /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example
+$ nix-store --realise /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example
 copying path '/nix/store/m8bmqwrch6l3h8s0k3d673xpmipcdpsa-example from 's3://example-nix-cache'...
 warning: you did not specify '--add-root'; the result might be removed by the garbage collector
 /nix/store/m8bmqwrch6l3h8s0k3d673xpmipcdpsa-example

doc/manual/command-ref/conf-file.xml

@@ -19,26 +19,30 @@
 
 <refsection><title>Description</title>
 
-<para>Nix reads settings from two configuration files:</para>
+<para>By default Nix reads settings from the following places:</para>
 
-<itemizedlist>
+<para>The system-wide configuration file
+<filename><replaceable>sysconfdir</replaceable>/nix/nix.conf</filename>
+(i.e. <filename>/etc/nix/nix.conf</filename> on most systems), or
+<filename>$NIX_CONF_DIR/nix.conf</filename> if
+<envar>NIX_CONF_DIR</envar> is set. Values loaded in this file are not forwarded to the Nix daemon. The
+client assumes that the daemon has already loaded them.
+</para>
 
-  <listitem>
-    <para>The system-wide configuration file
-    <filename><replaceable>sysconfdir</replaceable>/nix/nix.conf</filename>
-    (i.e. <filename>/etc/nix/nix.conf</filename> on most systems), or
-    <filename>$NIX_CONF_DIR/nix.conf</filename> if
-    <envar>NIX_CONF_DIR</envar> is set.</para>
-  </listitem>
+<para>User-specific configuration files:</para>
 
-  <listitem>
-    <para>The user configuration file
-    <filename>$XDG_CONFIG_HOME/nix/nix.conf</filename>, or
-    <filename>~/.config/nix/nix.conf</filename> if
-    <envar>XDG_CONFIG_HOME</envar> is not set.</para>
-  </listitem>
+<para>
+  If <envar>NIX_USER_CONF_FILES</envar> is set, then each path separated by
+  <literal>:</literal> will be loaded in reverse order.
+</para>
 
-</itemizedlist>
+<para>
+  Otherwise it will look for <filename>nix/nix.conf</filename> files in
+  <envar>XDG_CONFIG_DIRS</envar> and <envar>XDG_CONFIG_HOME</envar>.
+
+  The default location is <filename>$HOME/.config/nix.conf</filename> if
+  those environment variables are unset.
+</para>
 
 <para>The configuration files consist of
 <literal><replaceable>name</replaceable> =
@@ -382,7 +386,7 @@ false</literal>.</para>
 
 <programlisting>
 builtins.fetchurl {
-  url = https://example.org/foo-1.2.3.tar.xz;
+  url = "https://example.org/foo-1.2.3.tar.xz";
   sha256 = "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae";
 }
 </programlisting>

doc/manual/command-ref/env-common.xml

@@ -33,7 +33,7 @@
 
     will cause Nix to look for paths relative to
     <filename>/home/eelco/Dev</filename> and
-    <filename>/etc/nixos</filename>, in that order.  It is also
+    <filename>/etc/nixos</filename>, in this order.  It is also
     possible to match paths against a prefix.  For example, the value
 
     <screen>
@@ -53,13 +53,13 @@ nixpkgs=/home/eelco/Dev/nixpkgs-branch:/etc/nixos</screen>
     <envar>NIX_PATH</envar> to
 
     <screen>
-nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-15.09.tar.gz</screen>
+nixpkgs=https://github.com/NixOS/nixpkgs/archive/nixos-15.09.tar.gz</screen>
 
     tells Nix to download the latest revision in the Nixpkgs/NixOS
     15.09 channel.</para>
 
     <para>A following shorthand can be used to refer to the official channels:
-    
+
     <screen>nixpkgs=channel:nixos-15.09</screen>
     </para>
 
@@ -137,12 +137,19 @@ $ mount -o bind /mnt/otherdisk/nix /nix</screen>
 
 <varlistentry><term><envar>NIX_CONF_DIR</envar></term>
 
-  <listitem><para>Overrides the location of the Nix configuration
+  <listitem><para>Overrides the location of the system Nix configuration
   directory (default
   <filename><replaceable>prefix</replaceable>/etc/nix</filename>).</para></listitem>
 
 </varlistentry>
 
+<varlistentry><term><envar>NIX_USER_CONF_FILES</envar></term>
+
+  <listitem><para>Overrides the location of the user Nix configuration files
+  to load from (defaults to the XDG spec locations). The variable is treated
+  as a list separated by the <literal>:</literal> token.</para></listitem>
+
+</varlistentry>
 
 <varlistentry><term><envar>TMPDIR</envar></term>
 

doc/manual/command-ref/nix-env.xml

@@ -516,7 +516,7 @@ source:
 $ nix-env -f '<nixpkgs>' -iA hello --dry-run
 (dry run; not doing anything)
 installing ‘hello-2.10’
-these paths will be fetched (0.04 MiB download, 0.19 MiB unpacked):
+this path will be fetched (0.04 MiB download, 0.19 MiB unpacked):
   /nix/store/wkhdf9jinag5750mqlax6z2zbwhqb76n-hello-2.10
   <replaceable>...</replaceable></screen>
 
@@ -526,13 +526,10 @@ these paths will be fetched (0.04 MiB download, 0.19 MiB unpacked):
 14.12 channel:
 
 <screen>
-$ nix-env -f https://github.com/NixOS/nixpkgs-channels/archive/nixos-14.12.tar.gz -iA firefox
+$ nix-env -f https://github.com/NixOS/nixpkgs/archive/nixos-14.12.tar.gz -iA firefox
 </screen>
 
-(The GitHub repository <literal>nixpkgs-channels</literal> is updated
-automatically from the main <literal>nixpkgs</literal> repository
-after certain tests have succeeded and binaries have been built and
-uploaded to the binary cache at <uri>cache.nixos.org</uri>.)</para>
+</para>
 
 </refsection>
 
@@ -1066,7 +1063,8 @@ user environment elements, etc. -->
     the derivation, which can be used to unambiguously select it using
     the <link linkend="opt-attr"><option>--attr</option> option</link>
     available in commands that install derivations like
-    <literal>nix-env --install</literal>.</para></listitem>
+    <literal>nix-env --install</literal>. This option only works
+    together with <option>--available</option></para></listitem>
 
   </varlistentry>
 

doc/manual/command-ref/nix-shell.xml

@@ -258,7 +258,7 @@ path. You can override it by passing <option>-I</option> or setting
 containing the Pan package from a specific revision of Nixpkgs:
 
 <screen>
-$ nix-shell -p pan -I nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/8a3eea054838b55aca962c3fbde9c83c102b8bf2.tar.gz
+$ nix-shell -p pan -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/8a3eea054838b55aca962c3fbde9c83c102b8bf2.tar.gz
 
 [nix-shell:~]$ pan --version
 Pan 0.139
@@ -352,7 +352,7 @@ following Haskell script uses a specific branch of Nixpkgs/NixOS (the
 <programlisting><![CDATA[
 #! /usr/bin/env nix-shell
 #! nix-shell -i runghc -p "haskellPackages.ghcWithPackages (ps: [ps.HTTP ps.tagsoup])"
-#! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-18.03.tar.gz
+#! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/nixos-18.03.tar.gz
 
 import Network.HTTP
 import Text.HTML.TagSoup
@@ -370,7 +370,7 @@ If you want to be even more precise, you can specify a specific
 revision of Nixpkgs:
 
 <programlisting>
-#! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/0672315759b3e15e2121365f067c1c8c56bb4722.tar.gz
+#! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/0672315759b3e15e2121365f067c1c8c56bb4722.tar.gz
 </programlisting>
 
 </para>

doc/manual/command-ref/nix-store.xml

@@ -360,7 +360,6 @@ EOF
     <arg choice='plain'><option>--print-roots</option></arg>
     <arg choice='plain'><option>--print-live</option></arg>
     <arg choice='plain'><option>--print-dead</option></arg>
-    <arg choice='plain'><option>--delete</option></arg>
   </group>
   <arg><option>--max-freed</option> <replaceable>bytes</replaceable></arg>
 </cmdsynopsis>
@@ -407,14 +406,6 @@ the Nix store not reachable via file system references from a set of
 
   </varlistentry>
 
-  <varlistentry><term><option>--delete</option></term>
-
-    <listitem><para>This operation performs an actual garbage
-    collection.  All dead paths are removed from the
-    store.  This is the default.</para></listitem>
-
-  </varlistentry>
-
 </variablelist>
 
 <para>By default, all unreachable paths are deleted.  The following
@@ -444,10 +435,10 @@ and <link
 linkend="conf-keep-derivations"><literal>keep-derivations</literal></link>
 variables in the Nix configuration file.</para>
 
-<para>With <option>--delete</option>, the collector prints the total
-number of freed bytes when it finishes (or when it is interrupted).
-With <option>--print-dead</option>, it prints the number of bytes that
-would be freed.</para>
+<para>By default, the collector prints the total number of freed bytes
+when it finishes (or when it is interrupted). With
+<option>--print-dead</option>, it prints the number of bytes that would
+be freed.</para>
 
 </refsection>
 
@@ -945,7 +936,7 @@ $ nix-store --add ./foo.c
 
 <para>The operation <option>--add-fixed</option> adds the specified paths to
 the Nix store.  Unlike <option>--add</option> paths are registered using the
-specified hashing algorithm, resulting in the same output path as a fixed output
+specified hashing algorithm, resulting in the same output path as a fixed-output
 derivation.  This can be used for sources that are not available from a public
 url or broke since the download expression was written.
 </para>
@@ -1148,7 +1139,7 @@ the information that Nix considers important.  For instance,
 timestamps are elided because all files in the Nix store have their
 timestamp set to 0 anyway.  Likewise, all permissions are left out
 except for the execute bit, because all files in the Nix store have
-644 or 755 permission.</para>
+444 or 555 permission.</para>
 
 <para>Also, a NAR archive is <emphasis>canonical</emphasis>, meaning
 that “equal” paths always produce the same NAR archive.  For instance,

doc/manual/command-ref/opt-common-syn.xml

@@ -1,5 +1,5 @@
 <nop xmlns="http://docbook.org/ns/docbook">
-  
+
 <arg><option>--help</option></arg>
 <arg><option>--version</option></arg>
 <arg rep='repeat'>
@@ -11,6 +11,10 @@
 <arg>
   <arg choice='plain'><option>--quiet</option></arg>
 </arg>
+<arg>
+  <option>--log-format</option>
+  <replaceable>format</replaceable>
+</arg>
 <arg>
   <group choice='plain'>
     <arg choice='plain'><option>--no-build-output</option></arg>

doc/manual/command-ref/opt-common.xml

@@ -92,6 +92,37 @@
 </varlistentry>
 
 
+<varlistentry xml:id="opt-log-format"><term><option>--log-format</option> <replaceable>format</replaceable></term>
+
+  <listitem>
+
+  <para>This option can be used to change the output of the log format, with
+  <replaceable>format</replaceable> being one of:</para>
+
+  <variablelist>
+
+    <varlistentry><term>raw</term>
+    <listitem><para>This is the raw format, as outputted by nix-build.</para></listitem>
+    </varlistentry>
+
+    <varlistentry><term>internal-json</term>
+    <listitem><para>Outputs the logs in a structured manner. NOTE: the json schema is not guarantees to be stable between releases.</para></listitem>
+    </varlistentry>
+
+    <varlistentry><term>bar</term>
+    <listitem><para>Only display a progress bar during the builds.</para></listitem>
+    </varlistentry>
+
+    <varlistentry><term>bar-with-logs</term>
+    <listitem><para>Display the raw logs, with the progress bar at the bottom.</para></listitem>
+    </varlistentry>
+
+  </variablelist>
+
+  </listitem>
+
+</varlistentry>
+
 <varlistentry><term><option>--no-build-output</option> / <option>-Q</option></term>
 
   <listitem><para>By default, output written by builders to standard

doc/manual/expressions/advanced-attributes.xml

@@ -178,7 +178,7 @@ impureEnvVars = [ "http_proxy" "https_proxy" <replaceable>...</replaceable> ];
 
 <programlisting>
 fetchurl {
-  url = http://ftp.gnu.org/pub/gnu/hello/hello-2.1.1.tar.gz;
+  url = "http://ftp.gnu.org/pub/gnu/hello/hello-2.1.1.tar.gz";
   sha256 = "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465";
 }
 </programlisting>
@@ -189,7 +189,7 @@ fetchurl {
 
 <programlisting>
 fetchurl {
-  url = ftp://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz;
+  url = "ftp://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz";
   sha256 = "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465";
 }
 </programlisting>

doc/manual/expressions/builtins.xml

@@ -324,7 +324,7 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting>
     particular version of Nixpkgs, e.g.
 
 <programlisting>
-with import (fetchTarball https://github.com/NixOS/nixpkgs-channels/archive/nixos-14.12.tar.gz) {};
+with import (fetchTarball https://github.com/NixOS/nixpkgs/archive/nixos-14.12.tar.gz) {};
 
 stdenv.mkDerivation { … }
 </programlisting>
@@ -349,7 +349,7 @@ stdenv.mkDerivation { … }
 
 <programlisting>
 with import (fetchTarball {
-  url = https://github.com/NixOS/nixpkgs-channels/archive/nixos-14.12.tar.gz;
+  url = "https://github.com/NixOS/nixpkgs/archive/nixos-14.12.tar.gz";
   sha256 = "1jppksrfvbk5ypiqdz4cddxdl8z6zyzdb2srq8fcffr327ld5jj2";
 }) {};
 
@@ -422,6 +422,16 @@ stdenv.mkDerivation { … }
             </para>
           </listitem>
         </varlistentry>
+        <varlistentry>
+          <term>submodules</term>
+          <listitem>
+            <para>
+              A Boolean parameter that specifies whether submodules
+              should be checked out. Defaults to
+              <literal>false</literal>.
+            </para>
+          </listitem>
+        </varlistentry>
       </variablelist>
 
       <example>
@@ -1396,7 +1406,7 @@ stdenv.mkDerivation {
   ";
 
   src = fetchurl {
-    url = http://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz;
+    url = "http://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz";
     sha256 = "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465";
   };
   inherit perl;

doc/manual/expressions/expression-syntax.xml

@@ -15,7 +15,7 @@ stdenv.mkDerivation { <co xml:id='ex-hello-nix-co-2' />
   name = "hello-2.1.1"; <co xml:id='ex-hello-nix-co-3' />
   builder = ./builder.sh; <co xml:id='ex-hello-nix-co-4' />
   src = fetchurl { <co xml:id='ex-hello-nix-co-5' />
-    url = ftp://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz;
+    url = "ftp://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz";
     sha256 = "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465";
   };
   inherit perl; <co xml:id='ex-hello-nix-co-6' />

doc/manual/expressions/simple-building-testing.xml

@@ -73,12 +73,4 @@ waiting for lock on `/nix/store/0h5b7hp8d4hqfrw8igvx97x1xawrjnac-hello-2.1.1x'</
 So it is always safe to run multiple instances of Nix in parallel
 (which isn’t the case with, say, <command>make</command>).</para>
 
-<para>If you have a system with multiple CPUs, you may want to have
-Nix build different derivations in parallel (insofar as possible).
-Just pass the option <link linkend='opt-max-jobs'><option>-j
-<replaceable>N</replaceable></option></link>, where
-<replaceable>N</replaceable> is the maximum number of jobs to be run
-in parallel, or set.  Typically this should be the number of
-CPUs.</para>
-
 </section>

doc/manual/installation/env-variables.xml

@@ -39,7 +39,7 @@ bundle.</para>
   <step><para>Set the environment variable and install Nix</para>
     <screen>
 $ export NIX_SSL_CERT_FILE=/etc/ssl/my-certificate-bundle.crt
-$ sh &lt;(curl https://nixos.org/nix/install)
+$ sh &lt;(curl -L https://nixos.org/nix/install)
 </screen></step>
 
   <step><para>In the shell profile and rc files (for example,

doc/manual/installation/installing-binary.xml

@@ -6,16 +6,30 @@
 
 <title>Installing a Binary Distribution</title>
 
-<para>If you are using Linux or macOS, the easiest way to install Nix
-is to run the following command:
+<para>
+  If you are using Linux or macOS versions up to 10.14 (Mojave), the
+  easiest way to install Nix is to run the following command:
+</para>
 
 <screen>
-  $ sh &lt;(curl https://nixos.org/nix/install)
+  $ sh &lt;(curl -L https://nixos.org/nix/install)
 </screen>
 
-As of Nix 2.1.0, the Nix installer will always default to creating a
-single-user installation, however opting in to the multi-user
-installation is highly recommended.
+<para>
+  If you're using macOS 10.15 (Catalina) or newer, consult
+  <link linkend="sect-macos-installation">the macOS installation instructions</link>
+  before installing.
+</para>
+
+<para>
+  As of Nix 2.1.0, the Nix installer will always default to creating a
+  single-user installation, however opting in to the multi-user
+  installation is highly recommended.
+  <!-- TODO: this explains *neither* why the default version is
+  single-user, nor why we'd recommend multi-user over the default.
+  True prospective users don't have much basis for evaluating this.
+  What's it to me? Who should pick which? Why? What if I pick wrong?
+  -->
 </para>
 
 <section xml:id="sect-single-user-installation">
@@ -25,7 +39,7 @@ installation is highly recommended.
     To explicitly select a single-user installation on your system:
 
     <screen>
-  sh &lt;(curl https://nixos.org/nix/install) --no-daemon
+  sh &lt;(curl -L https://nixos.org/nix/install) --no-daemon
 </screen>
   </para>
 
@@ -36,7 +50,7 @@ run this under your usual user account, <emphasis>not</emphasis> as
 root.  The script will invoke <command>sudo</command> to create
 <filename>/nix</filename> if it doesn’t already exist.  If you don’t
 have <command>sudo</command>, you should manually create
-<command>/nix</command> first as root, e.g.:
+<filename>/nix</filename> first as root, e.g.:
 
 <screen>
 $ mkdir /nix
@@ -47,7 +61,7 @@ The install script will modify the first writable file from amongst
 <filename>.bash_profile</filename>, <filename>.bash_login</filename>
 and <filename>.profile</filename> to source
 <filename>~/.nix-profile/etc/profile.d/nix.sh</filename>. You can set
-the <command>NIX_INSTALLER_NO_MODIFY_PROFILE</command> environment
+the <envar>NIX_INSTALLER_NO_MODIFY_PROFILE</envar> environment
 variable before executing the install script to disable this
 behaviour.
 </para>
@@ -81,12 +95,10 @@ $ rm -rf /nix
   <para>
     You can instruct the installer to perform a multi-user
     installation on your system:
-
-    <screen>
-  sh &lt;(curl https://nixos.org/nix/install) --daemon
-</screen>
   </para>
 
+  <screen>sh &lt;(curl -L https://nixos.org/nix/install) --daemon</screen>
+
   <para>
     The multi-user installation of Nix will create build users between
     the user IDs 30001 and 30032, and a group with the group ID 30000.
@@ -136,13 +148,280 @@ sudo rm /Library/LaunchDaemons/org.nixos.nix-daemon.plist
 
 </section>
 
+<section xml:id="sect-macos-installation">
+  <title>macOS Installation</title>
+
+  <para>
+    Starting with macOS 10.15 (Catalina), the root filesystem is read-only.
+    This means <filename>/nix</filename> can no longer live on your system
+    volume, and that you'll need a workaround to install Nix.
+  </para>
+
+  <para>
+    The recommended approach, which creates an unencrypted APFS volume
+    for your Nix store and a "synthetic" empty directory to mount it
+    over at <filename>/nix</filename>, is least likely to impair Nix
+    or your system.
+  </para>
+
+  <note><para>
+    With all separate-volume approaches, it's possible something on
+    your system (particularly daemons/services and restored apps) may
+    need access to your Nix store before the volume is mounted. Adding
+    additional encryption makes this more likely.
+  </para></note>
+
+  <para>
+    If you're using a recent Mac with a
+    <link xlink:href="https://www.apple.com/euro/mac/shared/docs/Apple_T2_Security_Chip_Overview.pdf">T2 chip</link>,
+    your drive will still be encrypted at rest (in which case "unencrypted"
+    is a bit of a misnomer). To use this approach, just install Nix with:
+  </para>
+
+  <screen>$ sh &lt;(curl -L https://nixos.org/nix/install) --darwin-use-unencrypted-nix-store-volume</screen>
+
+  <para>
+    If you don't like the sound of this, you'll want to weigh the
+    other approaches and tradeoffs detailed in this section.
+  </para>
+
+  <note>
+    <title>Eventual solutions?</title>
+    <para>
+      All of the known workarounds have drawbacks, but we hope
+      better solutions will be available in the future. Some that
+      we have our eye on are:
+    </para>
+    <orderedlist>
+      <listitem>
+        <para>
+          A true firmlink would enable the Nix store to live on the
+          primary data volume without the build problems caused by
+          the symlink approach. End users cannot currently
+          create true firmlinks.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          If the Nix store volume shared FileVault encryption
+          with the primary data volume (probably by using the same
+          volume group and role), FileVault encryption could be
+          easily supported by the installer without requiring
+          manual setup by each user.
+        </para>
+      </listitem>
+    </orderedlist>
+  </note>
+
+  <section xml:id="sect-macos-installation-change-store-prefix">
+    <title>Change the Nix store path prefix</title>
+    <para>
+      Changing the default prefix for the Nix store is a simple
+      approach which enables you to leave it on your root volume,
+      where it can take full advantage of FileVault encryption if
+      enabled. Unfortunately, this approach also opts your device out
+      of some benefits that are enabled by using the same prefix
+      across systems:
+
+      <itemizedlist>
+        <listitem>
+          <para>
+            Your system won't be able to take advantage of the binary
+            cache (unless someone is able to stand up and support
+            duplicate caching infrastructure), which means you'll
+            spend more time waiting for builds.
+          </para>
+        </listitem>
+        <listitem>
+          <para>
+            It's harder to build and deploy packages to Linux systems.
+          </para>
+        </listitem>
+        <!-- TODO: may be more here -->
+      </itemizedlist>
+
+      <!-- TODO: Yes, but how?! -->
+
+      It would also possible (and often requested) to just apply this
+      change ecosystem-wide, but it's an intrusive process that has
+      side effects we want to avoid for now.
+      <!-- magnificent hand-wavy gesture -->
+    </para>
+    <para>
+    </para>
+  </section>
+
+  <section xml:id="sect-macos-installation-encrypted-volume">
+    <title>Use a separate encrypted volume</title>
+    <para>
+      If you like, you can also add encryption to the recommended
+      approach taken by the installer. You can do this by pre-creating
+      an encrypted volume before you run the installer--or you can
+      run the installer and encrypt the volume it creates later.
+      <!-- TODO: see later note about whether this needs both add-encryption and from-scratch directions -->
+    </para>
+    <para>
+      In either case, adding encryption to a second volume isn't quite
+      as simple as enabling FileVault for your boot volume. Before you
+      dive in, there are a few things to weigh:
+    </para>
+    <orderedlist>
+      <listitem>
+        <para>
+          The additional volume won't be encrypted with your existing
+          FileVault key, so you'll need another mechanism to decrypt
+          the volume.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          You can store the password in Keychain to automatically
+          decrypt the volume on boot--but it'll have to wait on Keychain
+          and may not mount before your GUI apps restore. If any of
+          your launchd agents or apps depend on Nix-installed software
+          (for example, if you use a Nix-installed login shell), the
+          restore may fail or break.
+        </para>
+        <para>
+          On a case-by-case basis, you may be able to work around this
+          problem by using <command>wait4path</command> to block
+          execution until your executable is available.
+        </para>
+        <para>
+          It's also possible to decrypt and mount the volume earlier
+          with a login hook--but this mechanism appears to be
+          deprecated and its future is unclear.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          You can hard-code the password in the clear, so that your
+          store volume can be decrypted before Keychain is available.
+        </para>
+      </listitem>
+    </orderedlist>
+    <para>
+      If you are comfortable navigating these tradeoffs, you can encrypt the volume with
+      something along the lines of:
+      <!-- TODO:
+      I don't know if this also needs from-scratch instructions?
+      can we just recommend use-the-installer-and-then-encrypt?
+      -->
+    </para>
+    <!--
+    TODO: it looks like this option can be encryptVolume|encrypt|enableFileVault
+
+    It may be more clear to use encryptVolume, here? FileVault seems
+    heavily associated with the boot-volume behavior; I worry
+    a little that it can mislead here, especially as it gets
+    copied around minus doc context...?
+    -->
+    <screen>alice$ diskutil apfs enableFileVault /nix -user disk</screen>
+
+    <!-- TODO: and then go into detail on the mount/decrypt approaches? -->
+  </section>
+
+  <section xml:id="sect-macos-installation-symlink">
+    <!--
+    Maybe a good razor is: if we'd hate having to support someone who
+    installed Nix this way, it shouldn't even be detailed?
+    -->
+    <title>Symlink the Nix store to a custom location</title>
+    <para>
+      Another simple approach is using <filename>/etc/synthetic.conf</filename>
+      to symlink the Nix store to the data volume. This option also
+      enables your store to share any configured FileVault encryption.
+      Unfortunately, builds that resolve the symlink may leak the
+      canonical path or even fail.
+    </para>
+    <para>
+      Because of these downsides, we can't recommend this approach.
+    </para>
+    <!-- Leaving out instructions for this one. -->
+  </section>
+
+  <section xml:id="sect-macos-installation-recommended-notes">
+    <title>Notes on the recommended approach</title>
+    <para>
+      This section goes into a little more detail on the recommended
+      approach. You don't need to understand it to run the installer,
+      but it can serve as a helpful reference if you run into trouble.
+    </para>
+    <orderedlist>
+      <listitem>
+        <para>
+          In order to compose user-writable locations into the new
+          read-only system root, Apple introduced a new concept called
+          <literal>firmlinks</literal>, which it describes as a
+          "bi-directional wormhole" between two filesystems. You can
+          see the current firmlinks in <filename>/usr/share/firmlinks</filename>.
+          Unfortunately, firmlinks aren't (currently?) user-configurable.
+        </para>
+
+        <para>
+          For special cases like NFS mount points or package manager roots,
+          <link xlink:href="https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man5/synthetic.conf.5.html">synthetic.conf(5)</link>
+          supports limited user-controlled file-creation (of symlinks,
+          and synthetic empty directories) at <filename>/</filename>.
+          To create a synthetic empty directory for mounting at <filename>/nix</filename>,
+          add the following line to <filename>/etc/synthetic.conf</filename>
+          (create it if necessary):
+        </para>
+
+        <screen>nix</screen>
+      </listitem>
+
+      <listitem>
+        <para>
+          This configuration is applied at boot time, but you can use
+          <command>apfs.util</command> to trigger creation (not deletion)
+          of new entries without a reboot:
+        </para>
+
+        <screen>alice$ /System/Library/Filesystems/apfs.fs/Contents/Resources/apfs.util -B</screen>
+      </listitem>
+
+      <listitem>
+        <para>
+          Create the new APFS volume with diskutil:
+        </para>
+
+        <screen>alice$ sudo diskutil apfs addVolume diskX APFS 'Nix Store' -mountpoint /nix</screen>
+      </listitem>
+
+      <listitem>
+        <para>
+          Using <command>vifs</command>, add the new mount to
+          <filename>/etc/fstab</filename>. If it doesn't already have
+          other entries, it should look something like:
+        </para>
+
+<screen>
+#
+# Warning - this file should only be modified with vifs(8)
+#
+# Failure to do so is unsupported and may be destructive.
+#
+LABEL=Nix\040Store /nix apfs rw,nobrowse
+</screen>
+
+        <para>
+          The nobrowse setting will keep Spotlight from indexing this
+          volume, and keep it from showing up on your desktop.
+        </para>
+      </listitem>
+    </orderedlist>
+  </section>
+
+</section>
+
 <section xml:id="sect-nix-install-pinned-version-url">
   <title>Installing a pinned Nix version from a URL</title>
 
   <para>
     NixOS.org hosts version-specific installation URLs for all Nix
     versions since 1.11.16, at
-    <literal>https://nixos.org/releases/nix/nix-VERSION/install</literal>.
+    <literal>https://releases.nixos.org/nix/nix-<replaceable>version</replaceable>/install</literal>.
   </para>
 
   <para>
@@ -150,7 +429,7 @@ sudo rm /Library/LaunchDaemons/org.nixos.nix-daemon.plist
   NixOS.org installation script:
 
   <screen>
-  sh &lt;(curl https://nixos.org/nix/install)
+  sh &lt;(curl -L https://nixos.org/nix/install)
 </screen>
   </para>
 

doc/manual/installation/prerequisites-source.xml

@@ -8,6 +8,14 @@
 
 <itemizedlist>
 
+  <listitem><para>GNU Autoconf
+  (<link xlink:href="https://www.gnu.org/software/autoconf/"/>)
+  and the autoconf-archive macro collection
+  (<link xlink:href="https://www.gnu.org/software/autoconf-archive/"/>).
+  These are only needed to run the bootstrap script, and are not necessary
+  if your source distribution came with a pre-built
+  <literal>./configure</literal> script.</para></listitem>
+
   <listitem><para>GNU Make.</para></listitem>
   
   <listitem><para>Bash Shell. The <literal>./configure</literal> script
@@ -50,6 +58,14 @@
   or higher.  If your distribution does not provide it, please install
   it from <link xlink:href="http://www.sqlite.org/" />.</para></listitem>
 
+  <listitem><para>The <link
+  xlink:href="http://www.hboehm.info/gc/">Boehm
+  garbage collector</link> to reduce the evaluator’s memory
+  consumption (optional).  To enable it, install
+  <literal>pkgconfig</literal> and the Boehm garbage collector, and
+  pass the flag <option>--enable-gc</option> to
+  <command>configure</command>.</para></listitem>
+
   <listitem><para>The <literal>boost</literal> library of version
   1.66.0 or higher. It can be obtained from the official web site
   <link xlink:href="https://www.boost.org/" />.</para></listitem>

doc/manual/installation/upgrading.xml

@@ -17,6 +17,11 @@
 
   <para>
     Single-user installations of Nix should run this:
-    <command>nix-channel --update; nix-env -iA nixpkgs.nix</command>
+    <command>nix-channel --update; nix-env -iA nixpkgs.nix nixpkgs.cacert</command>
+  </para>
+      
+  <para>
+    Multi-user Nix users on Linux should run this with sudo:
+    <command>nix-channel --update; nix-env -iA nixpkgs.nix nixpkgs.cacert; systemctl daemon-reload; systemctl restart nix-daemon</command>
   </para>
 </chapter>

doc/manual/introduction/quick-start.xml

@@ -15,7 +15,7 @@ to subsequent chapters.</para>
 <step><para>Install single-user Nix by running the following:
 
 <screen>
-$ bash &lt;(curl https://nixos.org/nix/install)
+$ bash &lt;(curl -L https://nixos.org/nix/install)
 </screen>
 
 This will install Nix in <filename>/nix</filename>. The install script

doc/manual/local.mk

@@ -4,11 +4,10 @@ ifeq ($(doc_generate),yes)
 XSLTPROC = $(xsltproc) --nonet $(xmlflags) \
   --param section.autolabel 1 \
   --param section.label.includes.component.label 1 \
-  --param html.stylesheet \'style.css\' \
   --param xref.with.number.and.title 1 \
   --param toc.section.depth 3 \
   --param admon.style \'\' \
-  --param callout.graphics.extension \'.gif\' \
+  --param callout.graphics 0 \
   --param contrib.inline.enabled 0 \
   --stringparam generate.toc "book toc" \
   --param keep.relative.image.uris 0
@@ -66,12 +65,10 @@ $(d)/manual.html: $(d)/manual.xml $(MANUAL_SRCS) $(d)/manual.is-valid
 	  $(docbookxsl)/profiling/profile.xsl $< | \
 	  $(XSLTPROC) --output $@ $(docbookxsl)/xhtml/docbook.xsl -
 
-$(foreach file, $(d)/manual.html $(d)/style.css, $(eval $(call install-data-in, $(file), $(docdir)/manual)))
+$(foreach file, $(d)/manual.html, $(eval $(call install-data-in, $(file), $(docdir)/manual)))
 
 $(foreach file, $(wildcard $(d)/figures/*.png), $(eval $(call install-data-in, $(file), $(docdir)/manual/figures)))
 
-$(foreach file, $(wildcard $(d)/images/callouts/*.gif), $(eval $(call install-data-in, $(file), $(docdir)/manual/images/callouts)))
-
 $(eval $(call install-symlink, manual.html, $(docdir)/manual/index.html))
 
 

doc/manual/release-notes/rl-0.8.xml

@@ -8,7 +8,7 @@
 
 <para>NOTE: the hashing scheme in Nix 0.8 changed (as detailed below).
 As a result, <command>nix-pull</command> manifests and channels built
-for Nix 0.7 and below will now work anymore.  However, the Nix
+for Nix 0.7 and below will not work anymore.  However, the Nix
 expression language has not changed, so you can still build from
 source.  Also, existing user environments continue to work.  Nix 0.8
 will automatically upgrade the database schema of previous

doc/manual/release-notes/rl-2.0.xml

@@ -503,14 +503,14 @@
   </listitem>
 
   <listitem>
-    <para><emphasis>Pure evaluation mode</emphasis>. This is a variant
-    of the existing restricted evaluation mode. In pure mode, the Nix
-    evaluator forbids access to anything that could cause different
-    evaluations of the same command line arguments to produce a
+    <para><emphasis>Pure evaluation mode</emphasis>. With the
+    <literal>--pure-eval</literal> flag, Nix enables a variant of the existing
+    restricted evaluation mode that forbids access to anything that could cause
+    different evaluations of the same command line arguments to produce a
     different result. This includes builtin functions such as
     <function>builtins.getEnv</function>, but more importantly,
-    <emphasis>all</emphasis> filesystem or network access unless a
-    content hash or commit hash is specified. For example, calls to
+    <emphasis>all</emphasis> filesystem or network access unless a content hash
+    or commit hash is specified. For example, calls to
     <function>builtins.fetchGit</function> are only allowed if a
     <varname>rev</varname> attribute is specified.</para>
 

doc/manual/style.css

@@ -1,263 +0,0 @@
-/* Copied from http://bakefile.sourceforge.net/, which appears
-   licensed under the GNU GPL. */
-
-
-/***************************************************************************
-                             Basic headers and text:
- ***************************************************************************/
-
-body
-{
-    font-family: "Nimbus Sans L", sans-serif;
-    background: white;
-    margin: 2em 1em 2em 1em;
-}
-
-h1, h2, h3, h4
-{
-    color: #005aa0;
-}
-
-h1 /* title */
-{
-    font-size: 200%;
-}
-
-div.part h1
-{
-    font-size: 240%;
-}
-
-h2 /* chapters, appendices, subtitle */
-{
-    font-size: 180%;
-}
-
-div.part
-{
-    margin-top: 4em;
-}
-
-/* Extra space between chapters, appendices. */
-div.chapter > div.titlepage h2, div.appendix > div.titlepage h2 
-{ 
-    margin-top: 1.5em;
-}
-
-div.section > div.titlepage h2 /* sections */
-{
-    font-size: 150%;
-    margin-top: 1.5em;
-}
-
-h3 /* subsections */
-{
-    font-size: 125%;
-}
-
-div.simplesect h2
-{
-    font-size: 110%;
-}
-
-div.appendix h3
-{
-    font-size: 150%;
-    margin-top: 1.5em;
-}
-
-div.refentry\.separator
-{
-    margin-top: 2.5em;
-    margin-bottom: 2em;
-}
-
-div.refnamediv h2, div.refsynopsisdiv h2, div.refsection h2 /* refentry parts */
-{
-    margin-top: 1.4em;
-    font-size: 125%;
-}
-
-div.refsection h3
-{
-    font-size: 110%;
-}
-
-
-/***************************************************************************
-                               Examples:
- ***************************************************************************/
-
-div.example
-{
-    border: 1px solid #b0b0b0;
-    padding: 6px 6px;
-    margin-left: 1.5em;
-    margin-right: 1.5em;
-    background: #f4f4f8;
-    border-radius: 0.4em;
-}
-
-div.example p.title
-{
-    margin-top: 0em;
-}
-
-div.example pre
-{
-}
-
-
-/***************************************************************************
-                            Screen dumps:
- ***************************************************************************/
-
-pre.screen, pre.programlisting
-{
-    padding: 6px 6px;
-    margin-left: 1.5em;
-    margin-right: 1.5em;
-    color: #600000;
-    background: #f4f4f8;
-    font-family: monospace;
-}
-
-div.example pre.programlisting
-{
-    border: 0px;
-    padding: 0 0;
-    margin: 0 0 0 0;
-}
-
-
-/***************************************************************************
-                               Notes, warnings etc:
- ***************************************************************************/
-
-.note, .warning
-{
-    border: 1px solid #b0b0b0;
-    padding: 3px 3px;
-    margin-left: 1.5em;
-    margin-right: 1.5em;
-    margin-bottom: 1em;
-    padding: 0.3em 0.3em 0.3em 0.3em;
-    background: #fffff5;
-    border-radius: 0.4em;
-}
-
-div.note, div.warning
-{
-    font-style: italic;
-}
-
-div.note h3, div.warning h3
-{
-    color: red;
-    font-size: 100%;
-    padding-right: 0.5em;
-    display: inline;
-}
-
-div.note p, div.warning p
-{
-    margin-bottom: 0em;
-}
-
-div.note h3 + p, div.warning h3 + p
-{
-    display: inline;
-}
-
-div.note h3
-{
-    color: blue;
-    font-size: 100%;
-}
-
-div.navfooter *
-{
-    font-size: 90%;
-}
-
-
-/***************************************************************************
-                        Links colors and highlighting: 
- ***************************************************************************/
-
-a { text-decoration: none; }
-a:hover { text-decoration: underline; }
-a:link { color: #0048b3; }
-a:visited { color: #002a6a; }
-
-
-/***************************************************************************
-                              Table of contents:
- ***************************************************************************/
-
-div.toc
-{
-    font-size: 90%;
-}
-
-div.toc dl
-{
-    margin-top: 0em;
-    margin-bottom: 0em;
-}
-
-
-/***************************************************************************
-                               Special elements:
- ***************************************************************************/
-
-tt, code
-{
-    color: #400000;
-}
-
-.term
-{
-    font-weight: bold;
-    
-}
-
-div.variablelist dd p, div.glosslist dd p
-{
-    margin-top: 0em;
-}
-
-div.variablelist dd, div.glosslist dd
-{
-    margin-left: 1.5em;
-}
-
-div.glosslist dt
-{
-    font-style: italic;
-}
-
-.varname
-{
-    color: #400000;
-}
-
-span.command strong
-{
-    font-weight: normal;
-    color: #400000;
-}
-
-div.calloutlist table
-{
-}
-
-table
-{
-    border-collapse: collapse;
-}
-
-div.affiliation
-{
-    font-style: italic;
-}

local.mk

@@ -6,9 +6,11 @@ dist-files += configure config.h.in perl/configure
 
 clean-files += Makefile.config
 
-GLOBAL_CXXFLAGS += -I . -I src -I src/libutil -I src/libstore -I src/libmain -I src/libexpr -I src/nix -Wno-deprecated-declarations
+GLOBAL_CXXFLAGS += -Wno-deprecated-declarations
 
 $(foreach i, config.h $(call rwildcard, src/lib*, *.hh), \
   $(eval $(call install-file-in, $(i), $(includedir)/nix, 0644)))
 
 $(GCH) $(PCH): src/libutil/util.hh config.h
+
+GCH_CXXFLAGS = -I src/libutil

maintainers/upload-release.pl

@@ -1,5 +1,5 @@
 #! /usr/bin/env nix-shell
-#! nix-shell -i perl -p perl perlPackages.LWPUserAgent perlPackages.LWPProtocolHttps perlPackages.FileSlurp gnupg1
+#! nix-shell -i perl -p perl perlPackages.LWPUserAgent perlPackages.LWPProtocolHttps perlPackages.FileSlurp perlPackages.NetAmazonS3 gnupg1
 
 use strict;
 use Data::Dumper;
@@ -9,12 +9,16 @@ use File::Slurp;
 use File::Copy;
 use JSON::PP;
 use LWP::UserAgent;
+use Net::Amazon::S3;
 
 my $evalId = $ARGV[0] or die "Usage: $0 EVAL-ID\n";
 
-my $releasesDir = "/home/eelco/mnt/releases";
+my $releasesBucketName = "nix-releases";
+my $channelsBucketName = "nix-channels";
 my $nixpkgsDir = "/home/eelco/Dev/nixpkgs-pristine";
 
+my $TMPDIR = $ENV{'TMPDIR'} // "/tmp";
+
 # FIXME: cut&paste from nixos-channel-scripts.
 sub fetch {
     my ($url, $type) = @_;
@@ -42,13 +46,31 @@ my $version = $1;
 
 print STDERR "Nix revision is $nixRev, version is $version\n";
 
-File::Path::make_path($releasesDir);
-if (system("mountpoint -q $releasesDir") != 0) {
-    system("sshfs hydra-mirror\@nixos.org:/releases $releasesDir") == 0 or die;
-}
+my $releaseDir = "nix/$releaseName";
 
-my $releaseDir = "$releasesDir/nix/$releaseName";
-File::Path::make_path($releaseDir);
+my $tmpDir = "$TMPDIR/nix-release/$releaseName";
+File::Path::make_path($tmpDir);
+
+# S3 setup.
+my $aws_access_key_id = $ENV{'AWS_ACCESS_KEY_ID'} or die "No AWS_ACCESS_KEY_ID given.";
+my $aws_secret_access_key = $ENV{'AWS_SECRET_ACCESS_KEY'} or die "No AWS_SECRET_ACCESS_KEY given.";
+
+my $s3 = Net::Amazon::S3->new(
+    { aws_access_key_id     => $aws_access_key_id,
+      aws_secret_access_key => $aws_secret_access_key,
+      retry                 => 1,
+      host                  => "s3-eu-west-1.amazonaws.com",
+    });
+
+my $releasesBucket = $s3->bucket($releasesBucketName) or die;
+
+my $s3_us = Net::Amazon::S3->new(
+    { aws_access_key_id     => $aws_access_key_id,
+      aws_secret_access_key => $aws_secret_access_key,
+      retry                 => 1,
+    });
+
+my $channelsBucket = $s3_us->bucket($channelsBucketName) or die;
 
 sub downloadFile {
     my ($jobName, $productNr, $dstName) = @_;
@@ -57,40 +79,49 @@ sub downloadFile {
 
     my $srcFile = $buildInfo->{buildproducts}->{$productNr}->{path} or die "job '$jobName' lacks product $productNr\n";
     $dstName //= basename($srcFile);
-    my $dstFile = "$releaseDir/" . $dstName;
+    my $tmpFile = "$tmpDir/$dstName";
 
-    if (! -e $dstFile) {
-        print STDERR "downloading $srcFile to $dstFile...\n";
-        system("NIX_REMOTE=https://cache.nixos.org/ nix cat-store '$srcFile' > '$dstFile.tmp'") == 0
+    if (!-e $tmpFile) {
+        print STDERR "downloading $srcFile to $tmpFile...\n";
+        system("NIX_REMOTE=https://cache.nixos.org/ nix cat-store '$srcFile' > '$tmpFile'") == 0
             or die "unable to fetch $srcFile\n";
-        rename("$dstFile.tmp", $dstFile) or die;
     }
 
     my $sha256_expected = $buildInfo->{buildproducts}->{$productNr}->{sha256hash} or die;
-    my $sha256_actual = `nix hash-file --base16 --type sha256 '$dstFile'`;
+    my $sha256_actual = `nix hash-file --base16 --type sha256 '$tmpFile'`;
     chomp $sha256_actual;
     if ($sha256_expected ne $sha256_actual) {
-        print STDERR "file $dstFile is corrupt, got $sha256_actual, expected $sha256_expected\n";
+        print STDERR "file $tmpFile is corrupt, got $sha256_actual, expected $sha256_expected\n";
         exit 1;
     }
 
-    write_file("$dstFile.sha256", $sha256_expected);
+    write_file("$tmpFile.sha256", $sha256_expected);
 
-    if (! -e "$dstFile.asc") {
-        system("gpg2 --detach-sign --armor $dstFile") == 0 or die "unable to sign $dstFile\n";
+    if (! -e "$tmpFile.asc") {
+        system("gpg2 --detach-sign --armor $tmpFile") == 0 or die "unable to sign $tmpFile\n";
     }
 
-    return ($dstFile, $sha256_expected);
+    return $sha256_expected;
 }
 
 downloadFile("tarball", "2"); # .tar.bz2
-my ($tarball, $tarballHash) = downloadFile("tarball", "3"); # .tar.xz
+my $tarballHash = downloadFile("tarball", "3"); # .tar.xz
 downloadFile("binaryTarball.i686-linux", "1");
 downloadFile("binaryTarball.x86_64-linux", "1");
 downloadFile("binaryTarball.aarch64-linux", "1");
 downloadFile("binaryTarball.x86_64-darwin", "1");
 downloadFile("installerScript", "1");
 
+for my $fn (glob "$tmpDir/*") {
+    my $name = basename($fn);
+    my $dstKey = "$releaseDir/" . $name;
+    unless (defined $releasesBucket->head_key($dstKey)) {
+        print STDERR "uploading $fn to s3://$releasesBucketName/$dstKey...\n";
+        $releasesBucket->add_key_filename($dstKey, $fn)
+            or die $releasesBucket->err . ": " . $releasesBucket->errstr;
+    }
+}
+
 exit if $version =~ /pre/;
 
 # Update Nixpkgs in a very hacky way.
@@ -111,8 +142,12 @@ $oldName =~ s/"//g;
 sub getStorePath {
     my ($jobName) = @_;
     my $buildInfo = decode_json(fetch("$evalUrl/job/$jobName", 'application/json'));
-    die unless $buildInfo->{buildproducts}->{1}->{type} eq "nix-build";
-    return $buildInfo->{buildproducts}->{1}->{path};
+    for my $product (values %{$buildInfo->{buildproducts}}) {
+        next unless $product->{type} eq "nix-build";
+        next if $product->{path} =~ /[a-z]+$/;
+        return $product->{path};
+    }
+    die;
 }
 
 write_file("$nixpkgsDir/nixos/modules/installer/tools/nix-fallback-paths.nix",
@@ -125,32 +160,15 @@ write_file("$nixpkgsDir/nixos/modules/installer/tools/nix-fallback-paths.nix",
 
 system("cd $nixpkgsDir && git commit -a -m 'nix: $oldName -> $version'") == 0 or die;
 
-# Extract the HTML manual.
-File::Path::make_path("$releaseDir/manual");
-
-system("tar xvf $tarball --strip-components=3 -C $releaseDir/manual --wildcards '*/doc/manual/*.html' '*/doc/manual/*.css' '*/doc/manual/*.gif' '*/doc/manual/*.png'") == 0 or die;
-
-if (! -e "$releaseDir/manual/index.html") {
-    symlink("manual.html", "$releaseDir/manual/index.html") or die;
-}
-
 # Update the "latest" symlink.
-symlink("$releaseName", "$releasesDir/nix/latest-tmp") or die;
-rename("$releasesDir/nix/latest-tmp", "$releasesDir/nix/latest") or die;
+$channelsBucket->add_key(
+    "nix-latest/install", "",
+    { "x-amz-website-redirect-location" => "https://releases.nixos.org/$releaseDir/install" })
+    or die $channelsBucket->err . ": " . $channelsBucket->errstr;
 
 # Tag the release in Git.
 chdir("/home/eelco/Dev/nix-pristine") or die;
 system("git remote update origin") == 0 or die;
 system("git tag --force --sign $version $nixRev -m 'Tagging release $version'") == 0 or die;
-
-# Update the website.
-my $siteDir = "/home/eelco/Dev/nixos-homepage-pristine";
-
-system("cd $siteDir && git pull") == 0 or die;
-
-write_file("$siteDir/nix-release.tt",
-           "[%-\n" .
-           "latestNixVersion = \"$version\"\n" .
-           "-%]\n");
-
-system("cd $siteDir && git commit -a -m 'Nix $version released'") == 0 or die;
+system("git push --tags") == 0 or die;
+system("git push --force-with-lease origin $nixRev:refs/heads/latest-release") == 0 or die;

mk/README.md

@@ -1,6 +0,0 @@
-This is a set of helper Makefiles for doing non-recursive builds with
-GNU Make.  The canonical source can be found at
-https://github.com/edolstra/make-rules.  You should copy the files
-into the `mk` subdirectory of your project.
-
-TODO: write more documentation.

mk/libraries.mk

@@ -125,7 +125,8 @@ define build-library
     $(1)_PATH := $$(_d)/$$($(1)_NAME).a
 
     $$($(1)_PATH): $$($(1)_OBJS) | $$(_d)/
-	$(trace-ar) $(AR) crs $$@ $$?
+	$(trace-ld) $(LD) -Ur -o $$(_d)/$$($(1)_NAME).o $$?
+	$(trace-ar) $(AR) crs $$@ $$(_d)/$$($(1)_NAME).o
 
     $(1)_LDFLAGS_USE += $$($(1)_PATH) $$($(1)_LDFLAGS)
 

mk/precompiled-headers.mk

@@ -8,14 +8,14 @@ GCH = $(buildprefix)precompiled-headers.h.gch
 $(GCH): precompiled-headers.h
 	@rm -f $@
 	@mkdir -p "$(dir $@)"
-	$(trace-gen) $(CXX) -x c++-header -o $@ $< $(GLOBAL_CXXFLAGS)
+	$(trace-gen) $(CXX) -x c++-header -o $@ $< $(GLOBAL_CXXFLAGS) $(GCH_CXXFLAGS)
 
 PCH = $(buildprefix)precompiled-headers.h.pch
 
 $(PCH): precompiled-headers.h
 	@rm -f $@
 	@mkdir -p "$(dir $@)"
-	$(trace-gen) $(CXX) -x c++-header -o $@ $< $(GLOBAL_CXXFLAGS)
+	$(trace-gen) $(CXX) -x c++-header -o $@ $< $(GLOBAL_CXXFLAGS) $(GCH_CXXFLAGS)
 
 clean-files += $(GCH) $(PCH)
 

mk/programs.mk

@@ -35,24 +35,28 @@ define build-program
 	$$(trace-ld) $(CXX) -o $$@ $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE))
 
   $(1)_INSTALL_DIR ?= $$(bindir)
-  $(1)_INSTALL_PATH := $$($(1)_INSTALL_DIR)/$(1)
 
-  $$(eval $$(call create-dir, $$($(1)_INSTALL_DIR)))
+  ifdef $(1)_INSTALL_DIR
 
-  install: $(DESTDIR)$$($(1)_INSTALL_PATH)
+    $(1)_INSTALL_PATH := $$($(1)_INSTALL_DIR)/$(1)
 
-  ifeq ($(BUILD_SHARED_LIBS), 1)
+    $$(eval $$(call create-dir, $$($(1)_INSTALL_DIR)))
 
-    _libs_final := $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_INSTALL_PATH))
+    install: $(DESTDIR)$$($(1)_INSTALL_PATH)
 
-    $(DESTDIR)$$($(1)_INSTALL_PATH): $$($(1)_OBJS) $$(_libs_final) | $(DESTDIR)$$($(1)_INSTALL_DIR)/
+    ifeq ($(BUILD_SHARED_LIBS), 1)
+
+      _libs_final := $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_INSTALL_PATH))
+
+      $(DESTDIR)$$($(1)_INSTALL_PATH): $$($(1)_OBJS) $$(_libs_final) | $(DESTDIR)$$($(1)_INSTALL_DIR)/
 	$$(trace-ld) $(CXX) -o $$@ $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE_INSTALLED))
 
-  else
+    else
 
-    $(DESTDIR)$$($(1)_INSTALL_PATH): $$($(1)_PATH) | $(DESTDIR)$$($(1)_INSTALL_DIR)/
+      $(DESTDIR)$$($(1)_INSTALL_PATH): $$($(1)_PATH) | $(DESTDIR)$$($(1)_INSTALL_DIR)/
 	install -t $(DESTDIR)$$($(1)_INSTALL_DIR) $$<
 
+    endif
   endif
 
   # Propagate CFLAGS and CXXFLAGS to the individual object files.
@@ -76,4 +80,10 @@ define build-program
   programs-list += $$($(1)_PATH)
   clean-files += $$($(1)_PATH) $$(_d)/*.o $$(_d)/.*.dep $$($(1)_DEPS) $$($(1)_OBJS)
   dist-files += $$(_srcs)
+
+  # Phony target to run this program (typically as a dependency of 'check').
+  .PHONY: $(1)_RUN
+  $(1)_RUN: $$($(1)_PATH)
+	$(trace-test) $$($(1)_PATH)
+
 endef

mk/run_test.sh

@@ -0,0 +1,28 @@
+#!/bin/sh
+
+set -u
+
+red=""
+green=""
+yellow=""
+normal=""
+
+post_run_msg="ran test $1..."
+if [ -t 1 ]; then
+    red=""
+    green=""
+    yellow=""
+    normal=""
+fi
+(cd $(dirname $1) && env ${TESTS_ENVIRONMENT} init.sh 2>/dev/null > /dev/null)
+log="$(cd $(dirname $1) && env ${TESTS_ENVIRONMENT} $(basename $1) 2>&1)"
+status=$?
+if [ $status -eq 0 ]; then
+  echo "$post_run_msg [${green}PASS$normal]"
+elif [ $status -eq 99 ]; then
+  echo "$post_run_msg [${yellow}SKIP$normal]"
+else
+  echo "$post_run_msg [${red}FAIL$normal]"
+  echo "$log" | sed 's/^/    /'
+  exit "$status"
+fi

mk/tests.mk

@@ -1,45 +1,15 @@
 # Run program $1 as part of ‘make installcheck’.
+
+test-deps =
+
 define run-install-test
 
-  installcheck: $1
+  installcheck: $1.test
 
-  _installcheck-list += $1
+  .PHONY: $1.test
+  $1.test: $1 $(test-deps)
+	@env TEST_NAME=$(notdir $(basename $1)) TESTS_ENVIRONMENT="$(tests-environment)" mk/run_test.sh $1
 
 endef
 
-# Color code from https://unix.stackexchange.com/a/10065
-installcheck:
-	@total=0; failed=0; \
-	red=""; \
-	green=""; \
-	yellow=""; \
-	normal=""; \
-	if [ -t 1 ]; then \
-		red=""; \
-		green=""; \
-		yellow=""; \
-		normal=""; \
-	fi; \
-	for i in $(_installcheck-list); do \
-	  total=$$((total + 1)); \
-	  printf "running test $$i..."; \
-	  log="$$(cd $$(dirname $$i) && $(tests-environment) $$(basename $$i) 2>&1)"; \
-	  status=$$?; \
-	  if [ $$status -eq 0 ]; then \
-	    echo " [$${green}PASS$$normal]"; \
-	  elif [ $$status -eq 99 ]; then \
-	    echo " [$${yellow}SKIP$$normal]"; \
-	  else \
-	    echo " [$${red}FAIL$$normal]"; \
-	    echo "$$log" | sed 's/^/    /'; \
-	    failed=$$((failed + 1)); \
-	  fi; \
-	done; \
-	if [ "$$failed" != 0 ]; then \
-	  echo "$${red}$$failed out of $$total tests failed $$normal"; \
-	  exit 1; \
-	else \
-		echo "$${green}All tests succeeded$$normal"; \
-	fi
-
 .PHONY: check installcheck

mk/tracing.mk

@@ -11,6 +11,7 @@ ifeq ($(V), 0)
   trace-javac   = @echo "  JAVAC " $@;
   trace-jar     = @echo "  JAR   " $@;
   trace-mkdir   = @echo "  MKDIR " $@;
+  trace-test    = @echo "  TEST  " $@;
 
   suppress  = @
 

nix-rust/local.mk

@@ -41,5 +41,5 @@ ifneq ($(OS), Darwin)
 check: rust-tests
 
 rust-tests:
-	cd nix-rust && CARGO_HOME=$$(if [[ -d vendor ]]; then echo vendor; fi) cargo test --release $$(if [[ -d vendor ]]; then echo --offline; fi)
+	$(trace-test) cd nix-rust && CARGO_HOME=$$(if [[ -d vendor ]]; then echo vendor; fi) cargo test --release $$(if [[ -d vendor ]]; then echo --offline; fi)
 endif

perl/lib/Nix/Store.xs

@@ -80,7 +80,7 @@ SV * queryReferences(char * path)
 SV * queryPathHash(char * path)
     PPCODE:
         try {
-            auto s = store()->queryPathInfo(store()->parseStorePath(path))->narHash.to_string();
+            auto s = store()->queryPathInfo(store()->parseStorePath(path))->narHash.to_string(Base32, true);
             XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
         } catch (Error & e) {
             croak("%s", e.what());
@@ -102,11 +102,11 @@ SV * queryPathInfo(char * path, int base32)
     PPCODE:
         try {
             auto info = store()->queryPathInfo(store()->parseStorePath(path));
-            if (info->deriver)
+            if (!info->deriver)
                 XPUSHs(&PL_sv_undef);
             else
                 XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(*info->deriver).c_str(), 0)));
-            auto s = info->narHash.to_string(base32 ? Base32 : Base16);
+            auto s = info->narHash.to_string(base32 ? Base32 : Base16, true);
             XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
             mXPUSHi(info->registrationTime);
             mXPUSHi(info->narSize);
@@ -182,7 +182,7 @@ void importPaths(int fd, int dontCheckSigs)
     PPCODE:
         try {
             FdSource source(fd);
-            store()->importPaths(source, nullptr, dontCheckSigs ? NoCheckSigs : CheckSigs);
+            store()->importPaths(source, dontCheckSigs ? NoCheckSigs : CheckSigs);
         } catch (Error & e) {
             croak("%s", e.what());
         }
@@ -274,7 +274,8 @@ int checkSignature(SV * publicKey_, SV * sig_, char * msg)
 SV * addToStore(char * srcPath, int recursive, char * algo)
     PPCODE:
         try {
-            auto path = store()->addToStore(std::string(baseNameOf(srcPath)), srcPath, recursive, parseHashType(algo));
+            auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
+            auto path = store()->addToStore(std::string(baseNameOf(srcPath)), srcPath, method, parseHashType(algo));
             XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0)));
         } catch (Error & e) {
             croak("%s", e.what());
@@ -285,7 +286,8 @@ SV * makeFixedOutputPath(int recursive, char * algo, char * hash, char * name)
     PPCODE:
         try {
             Hash h(hash, parseHashType(algo));
-            auto path = store()->makeFixedOutputPath(recursive, h, name);
+            auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
+            auto path = store()->makeFixedOutputPath(method, h, name);
             XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0)));
         } catch (Error & e) {
             croak("%s", e.what());

precompiled-headers.h

@@ -56,6 +56,3 @@
 #include <sys/wait.h>
 #include <termios.h>
 #include <unistd.h>
-
-#include "util.hh"
-#include "args.hh"

release-common.nix

@@ -34,7 +34,7 @@ rec {
       "--with-sandbox-shell=${sh}/bin/busybox"
     ];
 
-  tarballDeps =
+  buildDeps =
     [ bison
       flex
       libxml2
@@ -43,20 +43,18 @@ rec {
       docbook_xsl_ns
       autoconf-archive
       autoreconfHook
-    ];
 
-  buildDeps =
-    [ curl
+      curl
       bzip2 xz brotli zlib editline
       openssl pkgconfig sqlite
       libarchive
       boost
       nlohmann_json
-      rustc cargo
 
       # Tests
       git
       mercurial
+      gmock
     ]
     ++ lib.optionals stdenv.isLinux [libseccomp utillinuxMinimal]
     ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium
@@ -73,6 +71,10 @@ rec {
         */
       }));
 
+  propagatedDeps =
+    [ (boehmgc.override { enableLargeConfig = true; })
+    ];
+
   perlDeps =
     [ perl
       perlPackages.DBDSQLite

release.nix

@@ -1,5 +1,5 @@
 { nix ? builtins.fetchGit ./.
-, nixpkgs ? builtins.fetchTarball https://github.com/NixOS/nixpkgs/archive/nixos-19.09.tar.gz
+, nixpkgs ? builtins.fetchTarball https://github.com/NixOS/nixpkgs/archive/nixos-20.03-small.tar.gz
 , officialRelease ? false
 , systems ? [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ]
 }:
@@ -8,95 +8,12 @@ let
 
   pkgs = import nixpkgs { system = builtins.currentSystem or "x86_64-linux"; };
 
+  version =
+    builtins.readFile ./.version
+    + (if officialRelease then "" else "pre${toString nix.revCount}_${nix.shortRev}");
+
   jobs = rec {
 
-    # Create a "vendor" directory that contains the crates listed in
-    # Cargo.lock, and include it in the Nix tarball. This allows Nix
-    # to be built without network access.
-    vendoredCrates =
-      let
-        lockFile = builtins.fromTOML (builtins.readFile nix-rust/Cargo.lock);
-
-        files = map (pkg: import <nix/fetchurl.nix> {
-          url = "https://crates.io/api/v1/crates/${pkg.name}/${pkg.version}/download";
-          sha256 = lockFile.metadata."checksum ${pkg.name} ${pkg.version} (registry+https://github.com/rust-lang/crates.io-index)";
-        }) (builtins.filter (pkg: pkg.source or "" == "registry+https://github.com/rust-lang/crates.io-index") lockFile.package);
-
-      in pkgs.runCommand "cargo-vendor-dir" {}
-        ''
-          mkdir -p $out/vendor
-
-          cat > $out/vendor/config <<EOF
-          [source.crates-io]
-          replace-with = "vendored-sources"
-
-          [source.vendored-sources]
-          directory = "vendor"
-          EOF
-
-          ${toString (builtins.map (file: ''
-            mkdir $out/vendor/tmp
-            tar xvf ${file} -C $out/vendor/tmp
-            dir=$(echo $out/vendor/tmp/*)
-
-            # Add just enough metadata to keep Cargo happy.
-            printf '{"files":{},"package":"${file.outputHash}"}' > "$dir/.cargo-checksum.json"
-
-            # Clean up some cruft from the winapi crates. FIXME: find
-            # a way to remove winapi* from our dependencies.
-            if [[ $dir =~ /winapi ]]; then
-              find $dir -name "*.a" -print0 | xargs -0 rm -f --
-            fi
-
-            mv "$dir" $out/vendor/
-
-            rm -rf $out/vendor/tmp
-          '') files)}
-        '';
-
-
-    tarball =
-      with pkgs;
-
-      with import ./release-common.nix { inherit pkgs; };
-
-      releaseTools.sourceTarball {
-        name = "nix-tarball";
-        version = builtins.readFile ./.version;
-        versionSuffix = if officialRelease then "" else "pre${toString nix.revCount}_${nix.shortRev}";
-        src = nix;
-        inherit officialRelease;
-
-        buildInputs = tarballDeps ++ buildDeps;
-
-        postUnpack = ''
-          (cd $sourceRoot && find . -type f) | cut -c3- > $sourceRoot/.dist-files
-          cat $sourceRoot/.dist-files
-        '';
-
-        preConfigure = ''
-          (cd perl ; autoreconf --install --force --verbose)
-          # TeX needs a writable font cache.
-          export VARTEXFONTS=$TMPDIR/texfonts
-        '';
-
-        distPhase =
-          ''
-            cp -prd ${vendoredCrates}/vendor/ nix-rust/vendor/
-
-            runHook preDist
-            make dist
-            mkdir -p $out/tarballs
-            cp *.tar.* $out/tarballs
-          '';
-
-        preDist = ''
-          make install docdir=$out/share/doc/nix makefiles=doc/manual/local.mk
-          echo "doc manual $out/share/doc/nix/manual" >> $out/nix-support/hydra-build-products
-        '';
-      };
-
-
     build = pkgs.lib.genAttrs systems (system:
 
       let pkgs = import nixpkgs { inherit system; }; in
@@ -105,16 +22,21 @@ let
 
       with import ./release-common.nix { inherit pkgs; };
 
-      releaseTools.nixBuild {
-        name = "nix";
-        src = tarball;
+      stdenv.mkDerivation {
+        name = "nix-${version}";
+
+        src = nix;
+
+        outputs = [ "out" "dev" "doc" ];
 
         buildInputs = buildDeps;
 
+        propagatedBuildInputs = propagatedDeps;
+
         preConfigure =
-          # Copy libboost_context so we don't get all of Boost in our closure.
-          # https://github.com/NixOS/nixpkgs/issues/45462
           ''
+            # Copy libboost_context so we don't get all of Boost in our closure.
+            # https://github.com/NixOS/nixpkgs/issues/45462
             mkdir -p $out/lib
             cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib
             rm -f $out/lib/*.a
@@ -122,6 +44,8 @@ let
               chmod u+w $out/lib/*.so.*
               patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.*
             ''}
+
+            (cd perl; autoreconf --install --force --verbose)
           '';
 
         configureFlags = configureFlags ++
@@ -133,8 +57,17 @@ let
 
         installFlags = "sysconfdir=$(out)/etc";
 
+        postInstall = ''
+          mkdir -p $doc/nix-support
+          echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products
+        '';
+
+        doCheck = true;
+
         doInstallCheck = true;
         installCheckFlags = "sysconfdir=$(out)/etc";
+
+        separateDebugInfo = true;
       });
 
 
@@ -143,11 +76,21 @@ let
       let pkgs = import nixpkgs { inherit system; }; in with pkgs;
 
       releaseTools.nixBuild {
-        name = "nix-perl";
-        src = tarball;
+        name = "nix-perl-${version}";
+
+        src = nix;
 
         buildInputs =
-          [ jobs.build.${system} curl bzip2 xz pkgconfig pkgs.perl boost ]
+          [ autoconf-archive
+            autoreconfHook
+            jobs.build.${system}
+            curl
+            bzip2
+            xz
+            pkgconfig
+            pkgs.perl
+            boost
+          ]
           ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium;
 
         configureFlags = ''
@@ -167,7 +110,6 @@ let
 
       let
         toplevel = builtins.getAttr system jobs.build;
-        version = toplevel.src.version;
         installerClosureInfo = closureInfo { rootPaths = [ toplevel cacert ]; };
       in
 
@@ -177,10 +119,10 @@ let
         }
         ''
           cp ${installerClosureInfo}/registration $TMPDIR/reginfo
+          cp ${./scripts/create-darwin-volume.sh} $TMPDIR/create-darwin-volume.sh
           substitute ${./scripts/install-nix-from-closure.sh} $TMPDIR/install \
             --subst-var-by nix ${toplevel} \
             --subst-var-by cacert ${cacert}
-
           substitute ${./scripts/install-darwin-multi-user.sh} $TMPDIR/install-darwin-multi-user.sh \
             --subst-var-by nix ${toplevel} \
             --subst-var-by cacert ${cacert}
@@ -195,6 +137,7 @@ let
             # SC1090: Don't worry about not being able to find
             #         $nix/etc/profile.d/nix.sh
             shellcheck --exclude SC1090 $TMPDIR/install
+            shellcheck $TMPDIR/create-darwin-volume.sh
             shellcheck $TMPDIR/install-darwin-multi-user.sh
             shellcheck $TMPDIR/install-systemd-multi-user.sh
 
@@ -210,6 +153,7 @@ let
           fi
 
           chmod +x $TMPDIR/install
+          chmod +x $TMPDIR/create-darwin-volume.sh
           chmod +x $TMPDIR/install-darwin-multi-user.sh
           chmod +x $TMPDIR/install-systemd-multi-user.sh
           chmod +x $TMPDIR/install-multi-user
@@ -222,11 +166,15 @@ let
             --absolute-names \
             --hard-dereference \
             --transform "s,$TMPDIR/install,$dir/install," \
+            --transform "s,$TMPDIR/create-darwin-volume.sh,$dir/create-darwin-volume.sh," \
             --transform "s,$TMPDIR/reginfo,$dir/.reginfo," \
             --transform "s,$NIX_STORE,$dir/store,S" \
-            $TMPDIR/install $TMPDIR/install-darwin-multi-user.sh \
+            $TMPDIR/install \
+            $TMPDIR/create-darwin-volume.sh \
+            $TMPDIR/install-darwin-multi-user.sh \
             $TMPDIR/install-systemd-multi-user.sh \
-            $TMPDIR/install-multi-user $TMPDIR/reginfo \
+            $TMPDIR/install-multi-user \
+            $TMPDIR/reginfo \
             $(cat ${installerClosureInfo}/store-paths)
         '');
 
@@ -237,12 +185,13 @@ let
       with import ./release-common.nix { inherit pkgs; };
 
       releaseTools.coverageAnalysis {
-        name = "nix-build";
-        src = tarball;
+        name = "nix-coverage-${version}";
+
+        src = nix;
 
         enableParallelBuilding = true;
 
-        buildInputs = buildDeps;
+        buildInputs = buildDeps ++ propagatedDeps;
 
         dontInstall = false;
 
@@ -334,45 +283,20 @@ let
 
     installerScript =
       pkgs.runCommand "installer-script"
-        { buildInputs = [ build.x86_64-linux ];
-        }
+        { buildInputs = [ build.${builtins.currentSystem or "x86_64-linux"} ]; }
         ''
           mkdir -p $out/nix-support
 
           substitute ${./scripts/install.in} $out/install \
             ${pkgs.lib.concatMapStrings
               (system: "--replace '@binaryTarball_${system}@' $(nix --experimental-features nix-command hash-file --base16 --type sha256 ${binaryTarball.${system}}/*.tar.xz) ")
-              [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ]
+              systems
             } \
-            --replace '@nixVersion@' ${build.x86_64-linux.src.version}
+            --replace '@nixVersion@' ${version}
 
           echo "file installer $out/install" >> $out/nix-support/hydra-build-products
         '';
 
-
-    # Aggregate job containing the release-critical jobs.
-    release = pkgs.releaseTools.aggregate {
-      name = "nix-${tarball.version}";
-      meta.description = "Release-critical builds";
-      constituents =
-        [ tarball
-          build.i686-linux
-          build.x86_64-darwin
-          build.x86_64-linux
-          build.aarch64-linux
-          binaryTarball.i686-linux
-          binaryTarball.x86_64-darwin
-          binaryTarball.x86_64-linux
-          binaryTarball.aarch64-linux
-          tests.remoteBuilds
-          tests.nix-copy-closure
-          tests.binaryTarball
-          #tests.evalNixpkgs
-          #tests.evalNixOS
-          installerScript
-        ];
-    };
-
   };
 
 

scripts/create-darwin-volume.sh

@@ -0,0 +1,185 @@
+#!/bin/sh
+set -e
+
+root_disk() {
+    diskutil info -plist /
+}
+
+apfs_volumes_for() {
+    disk=$1
+    diskutil apfs list -plist "$disk"
+}
+
+disk_identifier() {
+    xpath "/plist/dict/key[text()='ParentWholeDisk']/following-sibling::string[1]/text()" 2>/dev/null
+}
+
+volume_list_true() {
+    key=$1
+    xpath "/plist/dict/array/dict/key[text()='Volumes']/following-sibling::array/dict/key[text()='$key']/following-sibling::true[1]" 2> /dev/null
+}
+
+volume_get_string() {
+    key=$1 i=$2
+    xpath "/plist/dict/array/dict/key[text()='Volumes']/following-sibling::array/dict[$i]/key[text()='$key']/following-sibling::string[1]/text()" 2> /dev/null
+}
+
+find_nix_volume() {
+    disk=$1
+    i=1
+    volumes=$(apfs_volumes_for "$disk")
+    while true; do
+        name=$(echo "$volumes" | volume_get_string "Name" "$i")
+        if [ -z "$name" ]; then
+            break
+        fi
+        case "$name" in
+            [Nn]ix*)
+                echo "$name"
+                break
+                ;;
+        esac
+        i=$((i+1))
+    done
+}
+
+test_fstab() {
+    grep -q "/nix apfs rw" /etc/fstab 2>/dev/null
+}
+
+test_nix_symlink() {
+    [ -L "/nix" ] || grep -q "^nix." /etc/synthetic.conf 2>/dev/null
+}
+
+test_synthetic_conf() {
+    grep -q "^nix$" /etc/synthetic.conf 2>/dev/null
+}
+
+test_nix() {
+    test -d "/nix"
+}
+
+test_t2_chip_present(){
+    # Use xartutil to see if system has a t2 chip.
+    #
+    # This isn't well-documented on its own; until it is,
+    # let's keep track of knowledge/assumptions.
+    #
+    # Warnings:
+    # - Don't search "xart" if porn will cause you trouble :)
+    # - Other xartutil flags do dangerous things. Don't run them
+    #   naively. If you must, search "xartutil" first.
+    #
+    # Assumptions:
+    # - the "xART session seeds recovery utility"
+    #   appears to interact with xartstorageremoted
+    # - `sudo xartutil --list` lists xART sessions
+    #   and their seeds and exits 0 if successful. If
+    #   not, it exits 1 and prints an error such as:
+    #   xartutil: ERROR: No supported link to the SEP present
+    # - xART sessions/seeds are present when a T2 chip is
+    #   (and not, otherwise)
+    # - the presence of a T2 chip means a newly-created
+    #   volume on the primary drive will be
+    #   encrypted at rest
+    # - all together: `sudo xartutil --list`
+    #   should exit 0 if a new Nix Store volume will
+    #   be encrypted at rest, and exit 1 if not.
+    sudo xartutil --list >/dev/null 2>/dev/null
+}
+
+test_filevault_in_use() {
+    disk=$1
+    #      list vols on disk | get value of Filevault key | value is true
+    apfs_volumes_for "$disk" | volume_list_true FileVault | grep -q true
+}
+
+# use after error msg for conditions we don't understand
+suggest_report_error(){
+    # ex "error: something sad happened :(" >&2
+    echo "       please report this @ https://github.com/nixos/nix/issues" >&2
+}
+
+main() {
+    (
+        echo ""
+        echo "     ------------------------------------------------------------------ "
+        echo "    | This installer will create a volume for the nix store and        |"
+        echo "    | configure it to mount at /nix.  Follow these steps to uninstall. |"
+        echo "     ------------------------------------------------------------------ "
+        echo ""
+        echo "  1. Remove the entry from fstab using 'sudo vifs'"
+        echo "  2. Destroy the data volume using 'diskutil apfs deleteVolume'"
+        echo "  3. Remove the 'nix' line from /etc/synthetic.conf or the file"
+        echo ""
+    ) >&2
+
+    if test_nix_symlink; then
+        echo "error: /nix is a symlink, please remove it and make sure it's not in synthetic.conf (in which case a reboot is required)" >&2
+        echo "  /nix -> $(readlink "/nix")" >&2
+        exit 2
+    fi
+
+    if ! test_synthetic_conf; then
+        echo "Configuring /etc/synthetic.conf..." >&2
+        echo nix | sudo tee -a /etc/synthetic.conf
+        if ! test_synthetic_conf; then
+            echo "error: failed to configure synthetic.conf;" >&2
+            suggest_report_error
+            exit 1
+        fi
+    fi
+
+    if ! test_nix; then
+        echo "Creating mountpoint for /nix..." >&2
+        /System/Library/Filesystems/apfs.fs/Contents/Resources/apfs.util -B || true
+        if ! test_nix; then
+            sudo mkdir -p /nix 2>/dev/null || true
+        fi
+        if ! test_nix; then
+            echo "error: failed to bootstrap /nix; if a reboot doesn't help," >&2
+            suggest_report_error
+            exit 1
+        fi
+    fi
+
+    disk=$(root_disk | disk_identifier)
+    volume=$(find_nix_volume "$disk")
+    if [ -z "$volume" ]; then
+        echo "Creating a Nix Store volume..." >&2
+
+        if test_filevault_in_use "$disk"; then
+            # TODO: Not sure if it's in-scope now, but `diskutil apfs list`
+            # shows both filevault and encrypted at rest status, and it
+            # may be the more semantic way to test for this? It'll show
+            # `FileVault:                 No (Encrypted at rest)`
+            # `FileVault:                 No`
+            # `FileVault:                 Yes (Unlocked)`
+            # and so on.
+            if test_t2_chip_present; then
+                echo "warning: boot volume is FileVault-encrypted, but the Nix store volume" >&2
+                echo "         is only encrypted at rest." >&2
+                echo "         See https://nixos.org/nix/manual/#sect-macos-installation" >&2
+            else
+                echo "error: refusing to create Nix store volume because the boot volume is" >&2
+                echo "       FileVault encrypted, but encryption-at-rest is not available." >&2
+                echo "       Manually create a volume for the store and re-run this script." >&2
+                echo "       See https://nixos.org/nix/manual/#sect-macos-installation" >&2
+                exit 1
+            fi
+        fi
+
+        sudo diskutil apfs addVolume "$disk" APFS 'Nix Store' -mountpoint /nix
+        volume="Nix Store"
+    else
+        echo "Using existing '$volume' volume" >&2
+    fi
+
+    if ! test_fstab; then
+        echo "Configuring /etc/fstab..." >&2
+        label=$(echo "$volume" | sed 's/ /\\040/g')
+        printf "\$a\nLABEL=%s /nix apfs rw,nobrowse\n.\nwq\n" "$label" | EDITOR=ed sudo vifs
+    fi
+}
+
+main "$@"

scripts/install-multi-user.sh

@@ -13,22 +13,25 @@ set -o pipefail
 # however tracking which bits came from which would be impossible.
 
 readonly ESC='\033[0m'
-readonly BOLD='\033[38;1m'
-readonly BLUE='\033[38;34m'
-readonly BLUE_UL='\033[38;4;34m'
-readonly GREEN='\033[38;32m'
-readonly GREEN_UL='\033[38;4;32m'
-readonly RED='\033[38;31m'
+readonly BOLD='\033[1m'
+readonly BLUE='\033[34m'
+readonly BLUE_UL='\033[4;34m'
+readonly GREEN='\033[32m'
+readonly GREEN_UL='\033[4;32m'
+readonly RED='\033[31m'
 
-readonly NIX_USER_COUNT="32"
+# installer allows overriding build user count to speed up installation
+# as creating each user takes non-trivial amount of time on macos
+readonly NIX_USER_COUNT=${NIX_USER_COUNT:-32}
 readonly NIX_BUILD_GROUP_ID="30000"
 readonly NIX_BUILD_GROUP_NAME="nixbld"
 readonly NIX_FIRST_BUILD_UID="30001"
 # Please don't change this. We don't support it, because the
 # default shell profile that comes with Nix doesn't support it.
 readonly NIX_ROOT="/nix"
+readonly NIX_EXTRA_CONF=${NIX_EXTRA_CONF:-}
 
-readonly PROFILE_TARGETS=("/etc/bashrc" "/etc/profile.d/nix.sh" "/etc/zshrc")
+readonly PROFILE_TARGETS=("/etc/bashrc" "/etc/profile.d/nix.sh" "/etc/zshenv")
 readonly PROFILE_BACKUP_SUFFIX=".backup-before-nix"
 readonly PROFILE_NIX_FILE="$NIX_ROOT/var/nix/profiles/default/etc/profile.d/nix-daemon.sh"
 
@@ -450,9 +453,11 @@ create_directories() {
 }
 
 place_channel_configuration() {
-    echo "https://nixos.org/channels/nixpkgs-unstable nixpkgs" > "$SCRATCH/.nix-channels"
-    _sudo "to set up the default system channel (part 1)" \
-          install -m 0664 "$SCRATCH/.nix-channels" "$ROOT_HOME/.nix-channels"
+    if [ -z "${NIX_INSTALLER_NO_CHANNEL_ADD:-}" ]; then
+        echo "https://nixos.org/channels/nixpkgs-unstable nixpkgs" > "$SCRATCH/.nix-channels"
+        _sudo "to set up the default system channel (part 1)" \
+            install -m 0664 "$SCRATCH/.nix-channels" "$ROOT_HOME/.nix-channels"
+    fi
 }
 
 welcome_to_nix() {
@@ -521,7 +526,7 @@ This script is going to call sudo a lot. Normally, it would show you
 exactly what commands it is running and why. However, the script is
 run in a headless fashion, like this:
 
-  $ curl https://nixos.org/nix/install | sh
+  $ curl -L https://nixos.org/nix/install | sh
 
 or maybe in a CI pipeline. Because of that, we're going to skip the
 verbose output in the interest of brevity.
@@ -529,7 +534,7 @@ verbose output in the interest of brevity.
 If you would like to
 see the output, try like this:
 
-  $ curl -o install-nix https://nixos.org/nix/install
+  $ curl -L -o install-nix https://nixos.org/nix/install
   $ sh ./install-nix
 
 EOF
@@ -567,7 +572,7 @@ install_from_extracted_nix() {
         cd "$EXTRACTED_NIX_PATH"
 
         _sudo "to copy the basic Nix files to the new store at $NIX_ROOT/store" \
-              rsync -rlpt ./store/* "$NIX_ROOT/store/"
+              rsync -rlpt --chmod=-w ./store/* "$NIX_ROOT/store/"
 
         if [ -d "$NIX_INSTALLED_NIX" ]; then
             echo "      Alright! We have our first nix at $NIX_INSTALLED_NIX"
@@ -634,18 +639,20 @@ setup_default_profile() {
         export NIX_SSL_CERT_FILE=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt
     fi
 
-    # Have to explicitly pass NIX_SSL_CERT_FILE as part of the sudo call,
-    # otherwise it will be lost in environments where sudo doesn't pass
-    # all the environment variables by default.
-    _sudo "to update the default channel in the default profile" \
-          HOME="$ROOT_HOME" NIX_SSL_CERT_FILE="$NIX_SSL_CERT_FILE" "$NIX_INSTALLED_NIX/bin/nix-channel" --update nixpkgs \
-          || channel_update_failed=1
-
+    if [ -z "${NIX_INSTALLER_NO_CHANNEL_ADD:-}" ]; then
+        # Have to explicitly pass NIX_SSL_CERT_FILE as part of the sudo call,
+        # otherwise it will be lost in environments where sudo doesn't pass
+        # all the environment variables by default.
+        _sudo "to update the default channel in the default profile" \
+            HOME="$ROOT_HOME" NIX_SSL_CERT_FILE="$NIX_SSL_CERT_FILE" "$NIX_INSTALLED_NIX/bin/nix-channel" --update nixpkgs \
+            || channel_update_failed=1
+    fi
 }
 
 
 place_nix_configuration() {
     cat <<EOF > "$SCRATCH/nix.conf"
+$NIX_EXTRA_CONF
 build-users-group = $NIX_BUILD_GROUP_NAME
 EOF
     _sudo "to place the default nix daemon configuration (part 2)" \

scripts/install-nix-from-closure.sh

@@ -40,29 +40,85 @@ elif [ "$(uname -s)" = "Linux" ] && [ -e /run/systemd/system ]; then
 fi
 
 INSTALL_MODE=no-daemon
-# Trivially handle the --daemon / --no-daemon options
-if [ "x${1:-}" = "x--no-daemon" ]; then
-    INSTALL_MODE=no-daemon
-elif [ "x${1:-}" = "x--daemon" ]; then
-    INSTALL_MODE=daemon
-elif [ "x${1:-}" != "x" ]; then
-    (
-        echo "Nix Installer [--daemon|--no-daemon]"
+CREATE_DARWIN_VOLUME=0
+# handle the command line flags
+while [ $# -gt 0 ]; do
+    case $1 in
+        --daemon)
+            INSTALL_MODE=daemon;;
+        --no-daemon)
+            INSTALL_MODE=no-daemon;;
+        --no-channel-add)
+            export NIX_INSTALLER_NO_CHANNEL_ADD=1;;
+        --daemon-user-count)
+            export NIX_USER_COUNT=$2
+            shift;;
+        --no-modify-profile)
+            NIX_INSTALLER_NO_MODIFY_PROFILE=1;;
+        --darwin-use-unencrypted-nix-store-volume)
+            CREATE_DARWIN_VOLUME=1;;
+        --nix-extra-conf-file)
+            export NIX_EXTRA_CONF="$(cat $2)"
+            shift;;
+        *)
+            (
+                echo "Nix Installer [--daemon|--no-daemon] [--daemon-user-count INT] [--no-channel-add] [--no-modify-profile] [--darwin-use-unencrypted-nix-store-volume] [--nix-extra-conf-file FILE]"
 
-        echo "Choose installation method."
-        echo ""
-        echo " --daemon:    Installs and configures a background daemon that manages the store,"
-        echo "              providing multi-user support and better isolation for local builds."
-        echo "              Both for security and reproducibility, this method is recommended if"
-        echo "              supported on your platform."
-        echo "              See https://nixos.org/nix/manual/#sect-multi-user-installation"
-        echo ""
-        echo " --no-daemon: Simple, single-user installation that does not require root and is"
-        echo "              trivial to uninstall."
-        echo "              (default)"
-        echo ""
-    ) >&2
-    exit
+                echo "Choose installation method."
+                echo ""
+                echo " --daemon:    Installs and configures a background daemon that manages the store,"
+                echo "              providing multi-user support and better isolation for local builds."
+                echo "              Both for security and reproducibility, this method is recommended if"
+                echo "              supported on your platform."
+                echo "              See https://nixos.org/nix/manual/#sect-multi-user-installation"
+                echo ""
+                echo " --no-daemon: Simple, single-user installation that does not require root and is"
+                echo "              trivial to uninstall."
+                echo "              (default)"
+                echo ""
+                echo " --no-channel-add:    Don't add any channels. nixpkgs-unstable is installed by default."
+                echo ""
+                echo " --no-modify-profile: Skip channel installation. When not provided nixpkgs-unstable"
+                echo "                      is installed by default."
+                echo ""
+                echo " --daemon-user-count: Number of build users to create. Defaults to 32."
+                echo ""
+                echo " --nix-extra-conf-file: Path to nix.conf to prepend when installing /etc/nix.conf"
+                echo ""
+            ) >&2
+
+            # darwin and Catalina+
+            if [ "$(uname -s)" = "Darwin" ] && [ "$macos_major" -gt 14 ]; then
+                (
+                    echo " --darwin-use-unencrypted-nix-store-volume: Create an APFS volume for the Nix"
+                    echo "              store and mount it at /nix. This is the recommended way to create"
+                    echo "              /nix with a read-only / on macOS >=10.15."
+                    echo "              See: https://nixos.org/nix/manual/#sect-macos-installation"
+                    echo ""
+                ) >&2
+            fi
+            exit;;
+    esac
+    shift
+done
+
+if [ "$(uname -s)" = "Darwin" ]; then
+    if [ "$CREATE_DARWIN_VOLUME" = 1 ]; then
+        printf '\e[1;31mCreating volume and mountpoint /nix.\e[0m\n'
+        "$self/create-darwin-volume.sh"
+    fi
+
+    info=$(diskutil info -plist / | xpath "/plist/dict/key[text()='Writable']/following-sibling::true[1]" 2> /dev/null)
+    if ! [ -e $dest ] && [ -n "$info" ] && [ "$macos_major" -gt 14 ]; then
+        (
+            echo ""
+            echo "Installing on macOS >=10.15 requires relocating the store to an apfs volume."
+            echo "Use sh <(curl -L https://nixos.org/nix/install) --darwin-use-unencrypted-nix-store-volume or run the preparation steps manually."
+            echo "See https://nixos.org/nix/manual/#sect-macos-installation"
+            echo ""
+        ) >&2
+        exit 1
+    fi
 fi
 
 if [ "$INSTALL_MODE" = "daemon" ]; then
@@ -87,7 +143,7 @@ if ! [ -e $dest ]; then
 fi
 
 if ! [ -w $dest ]; then
-    echo "$0: directory $dest exists, but is not writable by you. This could indicate that another user has already performed a single-user installation of Nix on this system. If you wish to enable multi-user support see http://nixos.org/nix/manual/#ssec-multi-user. If you wish to continue with a single-user install for $USER please run 'chown -R $USER $dest' as root." >&2
+    echo "$0: directory $dest exists, but is not writable by you. This could indicate that another user has already performed a single-user installation of Nix on this system. If you wish to enable multi-user support see https://nixos.org/nix/manual/#ssec-multi-user. If you wish to continue with a single-user install for $USER please run 'chown -R $USER $dest' as root." >&2
     exit 1
 fi
 
@@ -130,13 +186,15 @@ if [ -z "$NIX_SSL_CERT_FILE" ] || ! [ -f "$NIX_SSL_CERT_FILE" ]; then
 fi
 
 # Subscribe the user to the Nixpkgs channel and fetch it.
-if ! $nix/bin/nix-channel --list | grep -q "^nixpkgs "; then
-    $nix/bin/nix-channel --add https://nixos.org/channels/nixpkgs-unstable
-fi
-if [ -z "$_NIX_INSTALLER_TEST" ]; then
-    if ! $nix/bin/nix-channel --update nixpkgs; then
-        echo "Fetching the nixpkgs channel failed. (Are you offline?)"
-        echo "To try again later, run \"nix-channel --update nixpkgs\"."
+if [ -z "$NIX_INSTALLER_NO_CHANNEL_ADD" ]; then
+    if ! $nix/bin/nix-channel --list | grep -q "^nixpkgs "; then
+        $nix/bin/nix-channel --add https://nixos.org/channels/nixpkgs-unstable
+    fi
+    if [ -z "$_NIX_INSTALLER_TEST" ]; then
+        if ! $nix/bin/nix-channel --update nixpkgs; then
+            echo "Fetching the nixpkgs channel failed. (Are you offline?)"
+            echo "To try again later, run \"nix-channel --update nixpkgs\"."
+        fi
     fi
 fi
 
@@ -155,6 +213,17 @@ if [ -z "$NIX_INSTALLER_NO_MODIFY_PROFILE" ]; then
             break
         fi
     done
+    for i in .zshenv .zshrc; do
+        fn="$HOME/$i"
+        if [ -w "$fn" ]; then
+            if ! grep -q "$p" "$fn"; then
+                echo "modifying $fn..." >&2
+                echo "if [ -e $p ]; then . $p; fi # added by Nix installer" >> "$fn"
+            fi
+            added=1
+            break
+        fi
+    done
 fi
 
 if [ -z "$added" ]; then

scripts/install-systemd-multi-user.sh

@@ -88,7 +88,7 @@ poly_configure_nix_daemon_service() {
           systemctl start nix-daemon.socket
 
     _sudo "to start the nix-daemon.service" \
-          systemctl start nix-daemon.service
+          systemctl restart nix-daemon.service
 
 }
 

scripts/install.in

@@ -30,12 +30,15 @@ case "$(uname -s).$(uname -m)" in
     *) oops "sorry, there is no binary distribution of Nix for your platform";;
 esac
 
-url="https://nixos.org/releases/nix/nix-@nixVersion@/nix-@nixVersion@-$system.tar.xz"
+url="https://releases.nixos.org/nix/nix-@nixVersion@/nix-@nixVersion@-$system.tar.xz"
 
 tarball="$tmpDir/$(basename "$tmpDir/nix-@nixVersion@-$system.tar.xz")"
 
 require_util curl "download the binary tarball"
 require_util tar "unpack the binary tarball"
+if [ "$(uname -s)" != "Darwin" ]; then
+    require_util xz "unpack the binary tarball"
+fi
 
 echo "downloading Nix @nixVersion@ binary tarball for $system from '$url' to '$tmpDir'..."
 curl -L "$url" -o "$tarball" || oops "failed to download '$url'"

shell.nix

@@ -1,13 +1,13 @@
 { useClang ? false }:
 
-with import (builtins.fetchTarball https://github.com/NixOS/nixpkgs/archive/nixos-19.09.tar.gz) {};
+with import (builtins.fetchTarball https://github.com/NixOS/nixpkgs/archive/nixos-20.03-small.tar.gz) {};
 
 with import ./release-common.nix { inherit pkgs; };
 
 (if useClang then clangStdenv else stdenv).mkDerivation {
   name = "nix";
 
-  buildInputs = buildDeps ++ tarballDeps ++ perlDeps ++ [ pkgs.rustfmt ];
+  buildInputs = buildDeps ++ propagatedDeps ++ perlDeps;
 
   inherit configureFlags;
 

src/build-remote/build-remote.cc

@@ -17,7 +17,7 @@
 #include "store-api.hh"
 #include "derivations.hh"
 #include "local-store.hh"
-#include "legacy.hh"
+#include "../nix/legacy.hh"
 
 using namespace nix;
 using std::cin;
@@ -200,9 +200,12 @@ static int _main(int argc, char * * argv)
 
                 } catch (std::exception & e) {
                     auto msg = chomp(drainFD(5, false));
-                    printError("cannot build on '%s': %s%s",
-                        bestMachine->storeUri, e.what(),
-                        (msg.empty() ? "" : ": " + msg));
+                    logError({
+                        .name = "Remote build",
+                        .hint = hintfmt("cannot build on '%s': %s%s",
+                            bestMachine->storeUri, e.what(),
+                            (msg.empty() ? "" : ": " + msg))
+                    });
                     bestMachine->enabled = false;
                     continue;
                 }
@@ -241,7 +244,7 @@ connected:
 
         uploadLock = -1;
 
-        BasicDerivation drv(readDerivation(*store, store->realStoreDir + "/" + std::string(drvPath->to_string())));
+        auto drv = store->readDerivation(*drvPath);
         drv.inputSrcs = store->parseStorePathSet(inputs);
 
         auto result = sshStore->buildDerivation(*drvPath, drv);

src/libexpr/attr-path.cc

@@ -6,11 +6,11 @@
 namespace nix {
 
 
-static Strings parseAttrPath(const string & s)
+static Strings parseAttrPath(std::string_view s)
 {
     Strings res;
     string cur;
-    string::const_iterator i = s.begin();
+    auto i = s.begin();
     while (i != s.end()) {
         if (*i == '.') {
             res.push_back(cur);
@@ -19,7 +19,7 @@ static Strings parseAttrPath(const string & s)
             ++i;
             while (1) {
                 if (i == s.end())
-                    throw Error(format("missing closing quote in selection path '%1%'") % s);
+                    throw Error("missing closing quote in selection path '%1%'", s);
                 if (*i == '"') break;
                 cur.push_back(*i++);
             }
@@ -32,15 +32,22 @@ static Strings parseAttrPath(const string & s)
 }
 
 
-Ptr<Value> findAlongAttrPath(EvalState & state, const string & attrPath,
+std::vector<Symbol> parseAttrPath(EvalState & state, std::string_view s)
+{
+    std::vector<Symbol> res;
+    for (auto & a : parseAttrPath(s))
+        res.push_back(state.symbols.create(a));
+    return res;
+}
+
+
+std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attrPath,
     Bindings & autoArgs, Value & vIn)
 {
     Strings tokens = parseAttrPath(attrPath);
 
-    Error attrError =
-        Error(format("attribute selection path '%1%' does not match expression") % attrPath);
-
-    Ptr<Value> v(&vIn);
+    Value * v = &vIn;
+    Pos pos = noPos;
 
     for (auto & attr : tokens) {
 
@@ -50,7 +57,7 @@ Ptr<Value> findAlongAttrPath(EvalState & state, const string & attrPath,
         if (string2Int(attr, attrIndex)) apType = apIndex;
 
         /* Evaluate the expression. */
-        auto vNew = state.allocValue();
+        Value * vNew = state.allocValue();
         state.autoCallFunction(autoArgs, *v, *vNew);
         v = vNew;
         state.forceValue(*v);
@@ -62,34 +69,36 @@ Ptr<Value> findAlongAttrPath(EvalState & state, const string & attrPath,
 
             if (v->type != tAttrs)
                 throw TypeError(
-                    format("the expression selected by the selection path '%1%' should be a set but is %2%")
-                    % attrPath % showType(*v));
-
+                    "the expression selected by the selection path '%1%' should be a set but is %2%",
+                    attrPath,
+                    showType(*v));
             if (attr.empty())
-                throw Error(format("empty attribute name in selection path '%1%'") % attrPath);
+                throw Error("empty attribute name in selection path '%1%'", attrPath);
 
             Bindings::iterator a = v->attrs->find(state.symbols.create(attr));
             if (a == v->attrs->end())
-                throw Error(format("attribute '%1%' in selection path '%2%' not found") % attr % attrPath);
-            v = a->value;
+                throw AttrPathNotFound("attribute '%1%' in selection path '%2%' not found", attr, attrPath);
+            v = &*a->value;
+            pos = *a->pos;
         }
 
         else if (apType == apIndex) {
 
             if (!v->isList())
                 throw TypeError(
-                    format("the expression selected by the selection path '%1%' should be a list but is %2%")
-                    % attrPath % showType(*v));
-
+                    "the expression selected by the selection path '%1%' should be a list but is %2%",
+                    attrPath,
+                    showType(*v));
             if (attrIndex >= v->listSize())
-                throw Error(format("list index %1% in selection path '%2%' is out of range") % attrIndex % attrPath);
+                throw AttrPathNotFound("list index %1% in selection path '%2%' is out of range", attrIndex, attrPath);
 
             v = v->listElems()[attrIndex];
+            pos = noPos;
         }
 
     }
 
-    return v;
+    return {v, pos};
 }
 
 
@@ -97,9 +106,10 @@ Pos findDerivationFilename(EvalState & state, Value & v, std::string what)
 {
     Value * v2;
     try {
-        v2 = findAlongAttrPath(state, "meta.position", *state.emptyBindings, v);
+        auto dummyArgs = state.allocBindings(0);
+        v2 = findAlongAttrPath(state, "meta.position", *dummyArgs, v).first;
     } catch (Error &) {
-        throw Error("package '%s' has no source location information", what);
+        throw NoPositionInfo("package '%s' has no source location information", what);
     }
 
     // FIXME: is it possible to extract the Pos object instead of doing this
@@ -120,7 +130,7 @@ Pos findDerivationFilename(EvalState & state, Value & v, std::string what)
 
     Symbol file = state.symbols.create(filename);
 
-    return { file, lineno, 0 };
+    return { foFile, file, lineno, 0 };
 }
 
 

src/libexpr/attr-path.hh

@@ -7,10 +7,15 @@
 
 namespace nix {
 
-Ptr<Value> findAlongAttrPath(EvalState & state, const string & attrPath,
+MakeError(AttrPathNotFound, Error);
+MakeError(NoPositionInfo, Error);
+
+std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attrPath,
     Bindings & autoArgs, Value & vIn);
 
 /* Heuristic to find the filename and lineno or a nix value. */
 Pos findDerivationFilename(EvalState & state, Value & v, std::string what);
 
+std::vector<Symbol> parseAttrPath(EvalState & state, std::string_view s);
+
 }

src/libexpr/attr-set.cc

@@ -7,28 +7,28 @@
 namespace nix {
 
 
-/* Allocate a new array of attributes for an attribute set with a
-   specific capacity. The space is implicitly reserved after the
-   Bindings structure. */
-Ptr<Bindings> Bindings::allocBindings(size_t capacity)
+/* Allocate a new array of attributes for an attribute set with a specific
+   capacity. The space is implicitly reserved after the Bindings
+   structure. */
+Bindings * EvalState::allocBindings(size_t capacity)
 {
-    if (capacity >= 1UL << Object::miscBytes * 8)
+    if (capacity > std::numeric_limits<Bindings::size_t>::max())
         throw Error("attribute set of size %d is too big", capacity);
-    return gc.alloc<Bindings>(Bindings::wordsFor(capacity), capacity);
+    return new (allocBytes(sizeof(Bindings) + sizeof(Attr) * capacity)) Bindings((Bindings::size_t) capacity);
 }
 
 
 void EvalState::mkAttrs(Value & v, size_t capacity)
 {
     if (capacity == 0) {
-        v.attrs = emptyBindings;
-        v.type = tAttrs;
-    } else {
-        v.attrs = Bindings::allocBindings(capacity);
-        v.type = tAttrs;
-        nrAttrsets++;
-        nrAttrsInAttrsets += capacity;
+        v = vEmptySet;
+        return;
     }
+    clearValue(v);
+    v.type = tAttrs;
+    v.attrs = allocBindings(capacity);
+    nrAttrsets++;
+    nrAttrsInAttrsets += capacity;
 }
 
 
@@ -37,12 +37,18 @@ void EvalState::mkAttrs(Value & v, size_t capacity)
    this attribute. */
 Value * EvalState::allocAttr(Value & vAttrs, const Symbol & name)
 {
-    auto v = allocValue();
+    Value * v = allocValue();
     vAttrs.attrs->push_back(Attr(name, v));
     return v;
 }
 
 
+Value * EvalState::allocAttr(Value & vAttrs, const std::string & name)
+{
+    return allocAttr(vAttrs, symbols.create(name));
+}
+
+
 void Bindings::sort()
 {
     std::sort(begin(), end());

src/libexpr/attr-set.hh

@@ -2,7 +2,6 @@
 
 #include "nixexpr.hh"
 #include "symbol-table.hh"
-#include "gc.hh"
 
 #include <algorithm>
 #include <optional>
@@ -32,22 +31,19 @@ struct Attr
    by its size and its capacity, the capacity being the number of Attr
    elements allocated after this structure, while the size corresponds to
    the number of elements already inserted in this structure. */
-class Bindings : public Object
+class Bindings
 {
 public:
     typedef uint32_t size_t;
 
 private:
-    // FIXME: eliminate size_. We can just rely on capacity by sorting
-    // null entries towards the end of the vector.
-    size_t size_;
+    size_t size_, capacity_;
     Attr attrs[0];
 
-    Bindings(size_t capacity) : Object(tBindings, capacity), size_(0) {}
+    Bindings(size_t capacity) : size_(0), capacity_(capacity) { }
     Bindings(const Bindings & bindings) = delete;
 
 public:
-
     size_t size() const { return size_; }
 
     bool empty() const { return !size_; }
@@ -56,8 +52,7 @@ public:
 
     void push_back(const Attr & attr)
     {
-        assert(size_ < capacity());
-        gc.assertObject(attr.value);
+        assert(size_ < capacity_);
         attrs[size_++] = attr;
     }
 
@@ -81,7 +76,11 @@ public:
     {
         auto a = get(name);
         if (!a)
-            throw Error("attribute '%s' missing, at %s", name, pos);
+            throw Error({
+                .hint = hintfmt("attribute '%s' missing", name),
+                .errPos = pos
+            });
+
         return *a;
     }
 
@@ -95,7 +94,7 @@ public:
 
     void sort();
 
-    size_t capacity() const { return getMisc(); }
+    size_t capacity() { return capacity_; }
 
     /* Returns the attributes in lexicographically sorted order. */
     std::vector<const Attr *> lexicographicOrder() const
@@ -110,19 +109,7 @@ public:
         return res;
     }
 
-    size_t words() const
-    {
-        return wordsFor(capacity());
-    }
-
-    static size_t wordsFor(size_t capacity)
-    {
-        return 2 + 3 * capacity; // FIXME
-    }
-
-    static Ptr<Bindings> allocBindings(size_t capacity);
-
-    friend class GC;
+    friend class EvalState;
 };
 
 

src/libexpr/common-eval-args.cc

@@ -1,56 +1,59 @@
 #include "common-eval-args.hh"
 #include "shared.hh"
-#include "download.hh"
+#include "filetransfer.hh"
 #include "util.hh"
 #include "eval.hh"
+#include "fetchers.hh"
+#include "store-api.hh"
 
 namespace nix {
 
 MixEvalArgs::MixEvalArgs()
 {
-    mkFlag()
-        .longName("arg")
-        .description("argument to be passed to Nix functions")
-        .labels({"name", "expr"})
-        .handler([&](std::vector<std::string> ss) { autoArgs[ss[0]] = 'E' + ss[1]; });
+    addFlag({
+        .longName = "arg",
+        .description = "argument to be passed to Nix functions",
+        .labels = {"name", "expr"},
+        .handler = {[&](std::string name, std::string expr) { autoArgs[name] = 'E' + expr; }}
+    });
 
-    mkFlag()
-        .longName("argstr")
-        .description("string-valued argument to be passed to Nix functions")
-        .labels({"name", "string"})
-        .handler([&](std::vector<std::string> ss) { autoArgs[ss[0]] = 'S' + ss[1]; });
+    addFlag({
+        .longName = "argstr",
+        .description = "string-valued argument to be passed to Nix functions",
+        .labels = {"name", "string"},
+        .handler = {[&](std::string name, std::string s) { autoArgs[name] = 'S' + s; }},
+    });
 
-    mkFlag()
-        .shortName('I')
-        .longName("include")
-        .description("add a path to the list of locations used to look up <...> file names")
-        .label("path")
-        .handler([&](std::string s) { searchPath.push_back(s); });
+    addFlag({
+        .longName = "include",
+        .shortName = 'I',
+        .description = "add a path to the list of locations used to look up <...> file names",
+        .labels = {"path"},
+        .handler = {[&](std::string s) { searchPath.push_back(s); }}
+    });
 }
 
-Ptr<Bindings> MixEvalArgs::getAutoArgs(EvalState & state)
+Bindings * MixEvalArgs::getAutoArgs(EvalState & state)
 {
-    if (!bindings) {
-        bindings = Bindings::allocBindings(autoArgs.size());
-        for (auto & i : autoArgs) {
-            Value * v = state.allocValue();
-            if (i.second[0] == 'E')
-                state.mkThunk_(*v, state.parseExprFromString(string(i.second, 1), absPath(".")));
-            else
-                mkString(*v, string(i.second, 1));
-            bindings->push_back(Attr(state.symbols.create(i.first), v));
-        }
-        bindings->sort();
+    Bindings * res = state.allocBindings(autoArgs.size());
+    for (auto & i : autoArgs) {
+        Value * v = state.allocValue();
+        if (i.second[0] == 'E')
+            state.mkThunk_(*v, state.parseExprFromString(string(i.second, 1), absPath(".")));
+        else
+            mkString(*v, string(i.second, 1));
+        res->push_back(Attr(state.symbols.create(i.first), v));
     }
-    return bindings;
+    res->sort();
+    return res;
 }
 
 Path lookupFileArg(EvalState & state, string s)
 {
     if (isUri(s)) {
-        CachedDownloadRequest request(s);
-        request.unpack = true;
-        return getDownloader()->downloadCached(state.store, request).path;
+        return state.store->toRealPath(
+            fetchers::downloadTarball(
+                state.store, resolveUri(s), "source", false).storePath);
     } else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') {
         Path p = s.substr(1, s.size() - 2);
         return state.findFile(p);

src/libexpr/common-eval-args.hh

@@ -1,28 +1,24 @@
 #pragma once
 
 #include "args.hh"
-#include "eval.hh"
 
 namespace nix {
 
 class Store;
-
-template<class T>
-struct Ptr;
+class EvalState;
+class Bindings;
 
 struct MixEvalArgs : virtual Args
 {
     MixEvalArgs();
 
-    Ptr<Bindings> getAutoArgs(EvalState & state);
+    Bindings * getAutoArgs(EvalState & state);
 
     Strings searchPath;
 
 private:
 
     std::map<std::string, std::string> autoArgs;
-
-    Ptr<Bindings> bindings;
 };
 
 Path lookupFileArg(EvalState & state, string s);

src/libexpr/eval-inline.hh

@@ -7,49 +7,49 @@
 
 namespace nix {
 
-LocalNoInlineNoReturn(void throwEvalError(const char * s, const Pos & pos))
+LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s))
 {
-    throw EvalError(format(s) % pos);
+    throw EvalError({
+        .hint = hintfmt(s),
+        .errPos = pos
+    });
 }
 
 LocalNoInlineNoReturn(void throwTypeError(const char * s, const Value & v))
 {
-    throw TypeError(format(s) % showType(v));
+    throw TypeError(s, showType(v));
 }
 
 
-LocalNoInlineNoReturn(void throwTypeError(const char * s, const Value & v, const Pos & pos))
+LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s, const Value & v))
 {
-    throw TypeError(format(s) % showType(v) % pos);
+    throw TypeError({
+        .hint = hintfmt(s, showType(v)),
+        .errPos = pos
+    });
 }
 
 
 void EvalState::forceValue(Value & v, const Pos & pos)
 {
     if (v.type == tThunk) {
-        // FIXME: this is necessary because some values (like vList2)
-        // are created non-atomically.
-        Ptr<Env> env(v.thunk.env);
+        Env * env = v.thunk.env;
         Expr * expr = v.thunk.expr;
         try {
             v.type = tBlackhole;
             //checkInterrupt();
             expr->eval(*this, *env, v);
         } catch (...) {
+            v.type = tThunk;
             v.thunk.env = env;
             v.thunk.expr = expr;
-            v.type = tThunk;
             throw;
         }
     }
-    else if (v.type == tApp) {
-        // FIXME: idem.
-        Ptr<Value> left(v.app.left);
-        Ptr<Value> right(v.app.right);
+    else if (v.type == tApp)
         callFunction(*v.app.left, *v.app.right, v, noPos);
-    }
     else if (v.type == tBlackhole)
-        throwEvalError("infinite recursion encountered, at %1%", pos);
+        throwEvalError(pos, "infinite recursion encountered");
 }
 
 
@@ -63,9 +63,9 @@ inline void EvalState::forceAttrs(Value & v)
 
 inline void EvalState::forceAttrs(Value & v, const Pos & pos)
 {
-    forceValue(v);
+    forceValue(v, pos);
     if (v.type != tAttrs)
-        throwTypeError("value is %1% while a set was expected, at %2%", v, pos);
+        throwTypeError(pos, "value is %1% while a set was expected", v);
 }
 
 
@@ -79,9 +79,22 @@ inline void EvalState::forceList(Value & v)
 
 inline void EvalState::forceList(Value & v, const Pos & pos)
 {
-    forceValue(v);
+    forceValue(v, pos);
     if (!v.isList())
-        throwTypeError("value is %1% while a list was expected, at %2%", v, pos);
+        throwTypeError(pos, "value is %1% while a list was expected", v);
+}
+
+/* Note: Various places expect the allocated memory to be zeroed. */
+inline void * allocBytes(size_t n)
+{
+    void * p;
+#if HAVE_BOEHMGC
+    p = GC_MALLOC(n);
+#else
+    p = calloc(n, 1);
+#endif
+    if (!p) throw std::bad_alloc();
+    return p;
 }
 
 

src/libexpr/eval.hh

@@ -6,8 +6,8 @@
 #include "symbol-table.hh"
 #include "hash.hh"
 #include "config.hh"
-#include "gc.hh"
 
+#include <regex>
 #include <map>
 #include <optional>
 #include <unordered_map>
@@ -18,8 +18,7 @@ namespace nix {
 
 class Store;
 class EvalState;
-struct Derivation; // FIXME: remove
-struct StorePath;
+class StorePath;
 enum RepairFlag : bool;
 
 
@@ -36,56 +35,18 @@ struct PrimOp
 };
 
 
-struct Env : Object
+struct Env
 {
     Env * up;
+    unsigned short prevWith:14; // nr of levels up to next `with' environment
+    enum { Plain = 0, HasWithExpr, HasWithAttrs } type:2;
     Value * values[0];
-
-private:
-
-    constexpr static size_t maxSize = 1 << 16;
-    constexpr static size_t maxPrevWith = 1 << 10;
-
-    Env(Tag type, size_t size, size_t prevWith)
-        : Object(type, size | (prevWith << 16))
-    {
-        if (size >= maxSize)
-            throw Error("environment size %d is too big", size);
-
-        if (prevWith >= maxPrevWith)
-            throw Error("too many nesting levels");
-
-        for (size_t i = 0; i < size; i++)
-            values[i] = nullptr;
-    }
-
-    friend class GC;
-
-public:
-
-    unsigned short getPrevWith() const
-    {
-        return getMisc() >> 16;
-    }
-
-    unsigned short getSize() const
-    {
-        return getMisc() & 0xffff;
-    }
-
-    size_t words() const
-    {
-        return wordsFor(getSize());
-    }
-
-    static size_t wordsFor(unsigned short size)
-    {
-        return 2 + size;
-    }
 };
 
 
-Value & mkString(Value & v, std::string_view s, const PathSet & context);
+Value & mkString(Value & v, std::string_view s, const PathSet & context = PathSet());
+
+void copyContext(const Value & v, PathSet & context);
 
 
 /* Cache for calls to addToStore(); maps source paths to the store
@@ -100,16 +61,21 @@ typedef std::pair<std::string, std::string> SearchPathElem;
 typedef std::list<SearchPathElem> SearchPath;
 
 
+/* Initialise the Boehm GC, if applicable. */
+void initGC();
+
+
 class EvalState
 {
 public:
-    SymbolTable & symbols{nix::symbols}; // FIXME: remove
+    SymbolTable symbols;
 
     const Symbol sWith, sOutPath, sDrvPath, sType, sMeta, sName, sValue,
         sSystem, sOverrides, sOutputs, sOutputName, sIgnoreNulls,
         sFile, sLine, sColumn, sFunctor, sToString,
         sRight, sWrong, sStructuredAttrs, sBuilder, sArgs,
-        sOutputHash, sOutputHashAlgo, sOutputHashMode;
+        sOutputHash, sOutputHashAlgo, sOutputHashMode,
+        sRecurseForDerivations;
     Symbol sDerivationNix;
 
     /* If set, force copying files to the Nix store even if they
@@ -120,7 +86,7 @@ public:
        mode. */
     std::optional<PathSet> allowedPaths;
 
-    Ptr<Bindings> emptyBindings;
+    Value vEmptySet;
 
     const ref<Store> store;
 
@@ -128,11 +94,19 @@ private:
     SrcToStore srcToStore;
 
     /* A cache from path names to parse trees. */
+#if HAVE_BOEHMGC
+    typedef std::map<Path, Expr *, std::less<Path>, traceable_allocator<std::pair<const Path, Expr *> > > FileParseCache;
+#else
     typedef std::map<Path, Expr *> FileParseCache;
+#endif
     FileParseCache fileParseCache;
 
     /* A cache from path names to values. */
-    typedef std::map<Path, Ptr<Value>> FileEvalCache;
+#if HAVE_BOEHMGC
+    typedef std::map<Path, Value, std::less<Path>, traceable_allocator<std::pair<const Path, Value> > > FileEvalCache;
+#else
+    typedef std::map<Path, Value> FileEvalCache;
+#endif
     FileEvalCache fileEvalCache;
 
     SearchPath searchPath;
@@ -142,6 +116,9 @@ private:
     /* Cache used by checkSourcePath(). */
     std::unordered_map<Path, Path> resolvedPaths;
 
+    /* Cache used by prim_match(). */
+    std::unordered_map<std::string, std::regex> regexCache;
+
 public:
 
     EvalState(const Strings & _searchPath, ref<Store> store);
@@ -169,8 +146,8 @@ public:
     Expr * parseExprFromFile(const Path & path, StaticEnv & staticEnv);
 
     /* Parse a Nix expression from the specified string. */
-    Expr * parseExprFromString(const string & s, const Path & basePath, StaticEnv & staticEnv);
-    Expr * parseExprFromString(const string & s, const Path & basePath);
+    Expr * parseExprFromString(std::string_view s, const Path & basePath, StaticEnv & staticEnv);
+    Expr * parseExprFromString(std::string_view s, const Path & basePath);
 
     Expr * parseStdin();
 
@@ -245,7 +222,7 @@ public:
 
     /* The base environment, containing the builtin functions and
        values. */
-    Ptr<Env> baseEnv;
+    Env & baseEnv;
 
     /* The same, but used during parsing to resolve variables. */
     StaticEnv staticBaseEnv; // !!! should be private
@@ -273,7 +250,7 @@ private:
     friend struct ExprAttrs;
     friend struct ExprLet;
 
-    Expr * parse(const char * text, const Path & path,
+    Expr * parse(const char * text, FileOrigin origin, const Path & path,
         const Path & basePath, StaticEnv & staticEnv);
 
 public:
@@ -292,17 +269,13 @@ public:
     void autoCallFunction(Bindings & args, Value & fun, Value & res);
 
     /* Allocation primitives. */
-    Ptr<Value> allocValue()
-    {
-        nrValues++;
-        return gc.alloc<Value>(Value::words());
-    }
+    Value * allocValue();
+    Env & allocEnv(size_t size);
 
-    Ptr<Env> allocEnv(size_t size, size_t prevWith = 0, Tag type = tEnv);
-
-    // Note: the resulting Value is only reachable as long as vAttrs
-    // is reachable.
     Value * allocAttr(Value & vAttrs, const Symbol & name);
+    Value * allocAttr(Value & vAttrs, const std::string & name);
+
+    Bindings * allocBindings(size_t capacity);
 
     void mkList(Value & v, size_t length);
     void mkAttrs(Value & v, size_t capacity);
@@ -347,14 +320,12 @@ private:
     friend struct ExprOpConcatLists;
     friend struct ExprSelect;
     friend void prim_getAttr(EvalState & state, const Pos & pos, Value * * args, Value & v);
-
-public:
-
-    std::function<void(const StorePath & drvPath, const Derivation & drv)> derivationHook;
+    friend void prim_match(EvalState & state, const Pos & pos, Value * * args, Value & v);
 };
 
 
 /* Return a string representing the type of the value `v'. */
+string showType(ValueType type);
 string showType(const Value & v);
 
 /* Decode a context string ‘!<name>!<path>’ into a pair <path,
@@ -399,7 +370,7 @@ struct EvalSettings : Config
         "Prefixes of URIs that builtin functions such as fetchurl and fetchGit are allowed to fetch."};
 
     Setting<bool> traceFunctionCalls{this, false, "trace-function-calls",
-        "Emit log messages for each function entry and exit at the 'vomit' log level (-vvvv)"};
+        "Emit log messages for each function entry and exit at the 'vomit' log level (-vvvv)."};
 };
 
 extern EvalSettings evalSettings;

src/libexpr/function-trace.cc

@@ -1,4 +1,5 @@
 #include "function-trace.hh"
+#include "logging.hh"
 
 namespace nix {
 

src/libexpr/gc.cc

@@ -1,537 +0,0 @@
-#include "gc.hh"
-#include "value.hh"
-#include "attr-set.hh"
-#include "eval.hh"
-
-#include <chrono>
-
-namespace nix {
-
-GC gc;
-
-GC::GC()
-{
-    nextSize = std::max((size_t) 2, parseSize<size_t>(getEnv("GC_INITIAL_HEAP_SIZE").value_or("131072")) / WORD_SIZE);
-
-    // FIXME: placement new
-    frontPtrSentinel = (Ptr<Object> *) malloc(sizeof(Ptr<Object>));
-    backPtrSentinel = (Ptr<Object> *) malloc(sizeof(Ptr<Object>));
-
-    frontPtrSentinel->prev = nullptr;
-    frontPtrSentinel->next = backPtrSentinel;
-
-    backPtrSentinel->prev = frontPtrSentinel;
-    backPtrSentinel->next = nullptr;
-
-    frontRootSentinel = (Root<Object> *) malloc(sizeof(Root<Object>));
-    backRootSentinel = (Root<Object> *) malloc(sizeof(Root<Object>));
-
-    frontRootSentinel->prev = nullptr;
-    frontRootSentinel->next = backRootSentinel;
-
-    backRootSentinel->prev = frontRootSentinel;
-    backRootSentinel->next = nullptr;
-
-    freeLists[0].minSize = 2;
-    freeLists[1].minSize = 3;
-    freeLists[2].minSize = 4;
-    freeLists[3].minSize = 8;
-    freeLists[4].minSize = 16;
-    freeLists[5].minSize = 32;
-    freeLists[6].minSize = 64;
-    freeLists[7].minSize = 128;
-
-    addArena(nextSize);
-}
-
-GC::~GC()
-{
-    debug("%d bytes in arenas, %d bytes allocated, %d bytes reclaimed, in %d ms",
-        totalSize * WORD_SIZE,
-        allTimeWordsAllocated * WORD_SIZE,
-        allTimeWordsFreed * WORD_SIZE,
-        totalDurationMs);
-
-    size_t n = 0;
-    for (Ptr<Object> * p = frontPtrSentinel->next; p != backPtrSentinel; p = p->next)
-        n++;
-    if (n)
-        warn("%d GC root pointers still exist on exit", n);
-
-    n = 0;
-    for (Root<Object> * p = frontRootSentinel->next; p != backRootSentinel; p = p->next)
-        n++;
-    if (n)
-        warn("%d GC root objects still exist on exit", n);
-
-    assert(!frontPtrSentinel->prev);
-    assert(!backPtrSentinel->next);
-    assert(!frontRootSentinel->prev);
-    assert(!backRootSentinel->next);
-}
-
-void GC::addArena(size_t arenaSize)
-{
-    debug("allocating arena of %d bytes", arenaSize * WORD_SIZE);
-
-    auto arena = Arena(arenaSize);
-
-    // Add this arena to a freelist as a single block.
-    addToFreeList(new (arena.start) Free(arenaSize));
-
-    arenas.emplace_back(std::move(arena));
-
-    totalSize += arenaSize;
-
-    nextSize = arenaSize * 1.5; // FIXME: overflow, clamp
-}
-
-void GC::addToFreeList(Free * obj)
-{
-    auto size = obj->words();
-    for (auto i = freeLists.rbegin(); i != freeLists.rend(); ++i)
-        if (size >= i->minSize) {
-            obj->next = i->front;
-            i->front = obj;
-            return;
-        }
-    abort();
-}
-
-void GC::gc()
-{
-    typedef std::chrono::time_point<std::chrono::steady_clock> steady_time_point;
-
-    auto before = steady_time_point::clock::now();
-
-    size_t marked = 0;
-
-    std::stack<Object *> stack;
-
-    // FIXME: ensure this gets inlined.
-    auto push = [&](Object * p) { if (p) { assertObject(p); stack.push(p); } };
-
-    auto pushPointers = [&](Object * obj) {
-        switch (obj->type) {
-
-        case tFree:
-            printError("reached a freed object at %x", obj);
-            abort();
-
-        case tBindings: {
-            auto obj2 = (Bindings *) obj;
-            for (auto i = obj2->attrs; i < obj2->attrs + obj2->size_; ++i)
-                push(i->value);
-            break;
-        }
-
-        case tValueList: {
-            auto obj2 = (PtrList<Object> *) obj;
-            for (auto i = obj2->elems; i < obj2->elems + obj2->size(); ++i)
-                push(*i);
-            break;
-        }
-
-        case tEnv: {
-            auto obj2 = (Env *) obj;
-            push(obj2->up);
-            for (auto i = obj2->values; i < obj2->values + obj2->getSize(); ++i)
-                push(*i);
-            break;
-        }
-
-        case tWithExprEnv: {
-            auto obj2 = (Env *) obj;
-            push(obj2->up);
-            break;
-        }
-
-        case tWithAttrsEnv: {
-            auto obj2 = (Env *) obj;
-            push(obj2->up);
-            push(obj2->values[0]);
-            break;
-        }
-
-        case tString:
-        case tContext:
-        case tInt:
-        case tBool:
-        case tNull:
-        case tList0:
-        case tFloat:
-        case tShortString:
-        case tStaticString:
-            break;
-
-        case tLongString: {
-            auto obj2 = (Value *) obj;
-            push(obj2->string.s);
-            // See setContext().
-            if (!(((ptrdiff_t) obj2->string.context) & 1))
-                push(obj2->string.context);
-            break;
-        }
-
-        case tPath:
-            push(((Value *) obj)->path);
-            break;
-
-        case tAttrs:
-            push(((Value *) obj)->attrs);
-            break;
-
-        case tList1:
-            push(((Value *) obj)->smallList[0]);
-            break;
-
-        case tList2:
-            push(((Value *) obj)->smallList[0]);
-            push(((Value *) obj)->smallList[1]);
-            break;
-
-        case tListN:
-            push(((Value *) obj)->bigList);
-            break;
-
-        case tThunk:
-        case tBlackhole:
-            push(((Value *) obj)->thunk.env);
-            break;
-
-        case tApp:
-        case tPrimOpApp:
-            push(((Value *) obj)->app.left);
-            push(((Value *) obj)->app.right);
-            break;
-
-        case tLambda:
-            push(((Value *) obj)->lambda.env);
-            break;
-
-        case tPrimOp:
-            // FIXME: GC primops?
-            break;
-
-        default:
-            printError("don't know how to traverse object at %x (tag %d)", obj, obj->type);
-            abort();
-        }
-    };
-
-    auto processStack = [&]() {
-        while (!stack.empty()) {
-            auto obj = stack.top();
-            stack.pop();
-
-            //printError("MARK %x", obj);
-
-            if (!obj->isMarked()) {
-                marked++;
-                obj->mark();
-                pushPointers(obj);
-            }
-        }
-    };
-
-    for (Root<Object> * p = frontRootSentinel->next; p != backRootSentinel; p = p->next) {
-        pushPointers(&p->value);
-        processStack();
-    }
-
-    for (Ptr<Object> * p = frontPtrSentinel->next; p != backPtrSentinel; p = p->next) {
-        if (!p->value) continue;
-        stack.push(p->value);
-        processStack();
-    }
-
-    auto afterMark = steady_time_point::clock::now();
-
-    // Reset all the freelists.
-    for (auto & freeList : freeLists)
-        freeList.front = nullptr;
-
-    // Go through all the arenas and add free objects to the
-    // appropriate freelists.
-    size_t totalObjectsFreed = 0;
-    size_t totalWordsFreed = 0;
-    size_t totalObjectsKept = 0;
-    size_t totalWordsKept = 0;
-
-    for (auto & arena : arenas) {
-        auto [objectsFreed, wordsFreed, objectsKept, wordsKept] = freeUnmarked(arena);
-        totalObjectsFreed += objectsFreed;
-        totalWordsFreed += wordsFreed;
-        totalObjectsKept += objectsKept;
-        totalWordsKept += wordsKept;
-    }
-
-    auto after = steady_time_point::clock::now();
-
-    auto markDurationMs = std::chrono::duration_cast<std::chrono::milliseconds>(afterMark - before).count();
-    auto sweepDurationMs = std::chrono::duration_cast<std::chrono::milliseconds>(after - afterMark).count();
-
-    debug("freed %d dead objects (%d bytes), keeping %d/%d objects (%d bytes), marked in %d ms, swept in %d ms",
-        totalObjectsFreed, totalWordsFreed * WORD_SIZE,
-        marked, totalObjectsKept, totalWordsKept * WORD_SIZE,
-        markDurationMs, sweepDurationMs);
-
-    allTimeWordsFreed += totalWordsFreed;
-    totalDurationMs += markDurationMs + sweepDurationMs;
-}
-
-size_t GC::getObjectSize(Object * obj)
-{
-    auto tag = obj->type;
-    if (tag >= tInt && tag <= tFloat) {
-        return ((Value *) obj)->words();
-    } else {
-        switch (tag) {
-        case tFree:
-            return ((Free *) obj)->words();
-            break;
-        case tString:
-            return ((String *) obj)->words();
-            break;
-        case tBindings:
-            return ((Bindings *) obj)->words();
-            break;
-        case tValueList:
-            return ((PtrList<Value> *) obj)->words();
-            break;
-        case tEnv:
-        case tWithExprEnv:
-        case tWithAttrsEnv:
-            return ((Env *) obj)->words();
-            break;
-        case tContext:
-            return ((Context *) obj)->getSize() + 1;
-            break;
-        default:
-            printError("GC encountered invalid object with tag %d", tag);
-            abort();
-        }
-    }
-}
-
-std::tuple<size_t, size_t, size_t, size_t> GC::freeUnmarked(Arena & arena)
-{
-    size_t objectsFreed = 0;
-    size_t wordsFreed = 0;
-    size_t objectsKept = 0;
-    size_t wordsKept = 0;
-
-    auto end = arena.start + arena.size;
-    auto pos = arena.start;
-
-    Free * curFree = nullptr;
-
-    auto linkCurFree = [&]() {
-        if (curFree && curFree->words() > 1)
-            addToFreeList(curFree);
-        curFree = nullptr;
-    };
-
-    while (pos < end) {
-        auto obj = (Object *) pos;
-
-        auto objSize = getObjectSize(obj);
-
-        // Merge current object into the previous free object.
-        auto mergeFree = [&]() {
-            //printError("MERGE %x %x %d", curFree, obj, curFree->size() + objSize);
-            assert(curFree->words() >= 1);
-            curFree->setSize(curFree->words() + objSize);
-        };
-
-        if (obj->type == tFree) {
-            //debug("KEEP FREE %x %d", obj, obj->getMisc());
-            if (curFree) {
-                // Merge this object into the previous free
-                // object.
-                mergeFree();
-            } else {
-                curFree = (Free *) obj;
-            }
-        } else {
-
-            if (obj->isMarked()) {
-                // Unmark to prepare for the next GC run.
-                //debug("KEEP OBJECT %x %d %d", obj, obj->type, objSize);
-                linkCurFree();
-                obj->unmark();
-                objectsKept += 1;
-                wordsKept += objSize;
-            } else {
-                //debug("FREE OBJECT %x %d %d", obj, obj->type, objSize);
-                #if GC_DEBUG
-                for (size_t i = 0; i < objSize; ++i)
-                    ((Word *) obj)[i] = 0xdeadc0dedeadbeefULL;
-                #endif
-                objectsFreed += 1;
-                wordsFreed += objSize;
-                if (curFree) {
-                    mergeFree();
-                } else {
-                    // Convert to a free object.
-                    curFree = (Free *) obj;
-                    curFree->type = tFree;
-                    curFree->setSize(objSize);
-                }
-            }
-        }
-
-        pos += objSize;
-    }
-
-    linkCurFree();
-
-    assert(pos == end);
-
-    return {objectsFreed, wordsFreed, objectsKept, wordsKept};
-}
-
-bool GC::isObject(void * p)
-{
-    for (auto & arena : arenas)
-        if (p >= arena.start && p < arena.start + arena.size)
-            return true;
-    return false;
-}
-
-std::tuple<size_t, size_t> GC::getObjectClosureSize(Object * p)
-{
-    std::unordered_set<Object *> seen;
-
-    // FIXME: cut&paste.
-
-    std::stack<Object *> stack;
-
-    // FIXME: ensure this gets inlined.
-    auto push = [&](Object * p) { if (p) { assertObject(p); stack.push(p); } };
-
-    auto pushPointers = [&](Object * obj) {
-        switch (obj->type) {
-
-        case tFree:
-            printError("reached a freed object at %x", obj);
-            abort();
-
-        case tBindings: {
-            auto obj2 = (Bindings *) obj;
-            for (auto i = obj2->attrs; i < obj2->attrs + obj2->size_; ++i)
-                push(i->value);
-            break;
-        }
-
-        case tValueList: {
-            auto obj2 = (PtrList<Object> *) obj;
-            for (auto i = obj2->elems; i < obj2->elems + obj2->size(); ++i)
-                push(*i);
-            break;
-        }
-
-        case tEnv: {
-            auto obj2 = (Env *) obj;
-            push(obj2->up);
-            for (auto i = obj2->values; i < obj2->values + obj2->getSize(); ++i)
-                push(*i);
-            break;
-        }
-
-        case tWithExprEnv: {
-            auto obj2 = (Env *) obj;
-            push(obj2->up);
-            break;
-        }
-
-        case tWithAttrsEnv: {
-            auto obj2 = (Env *) obj;
-            push(obj2->up);
-            push(obj2->values[0]);
-            break;
-        }
-
-        case tString:
-        case tContext:
-        case tInt:
-        case tBool:
-        case tNull:
-        case tList0:
-        case tFloat:
-        case tShortString:
-        case tStaticString:
-            break;
-
-        case tLongString: {
-            auto obj2 = (Value *) obj;
-            push(obj2->string.s);
-            // See setContext().
-            if (!(((ptrdiff_t) obj2->string.context) & 1))
-                push(obj2->string.context);
-            break;
-        }
-
-        case tPath:
-            push(((Value *) obj)->path);
-            break;
-
-        case tAttrs:
-            push(((Value *) obj)->attrs);
-            break;
-
-        case tList1:
-            push(((Value *) obj)->smallList[0]);
-            break;
-
-        case tList2:
-            push(((Value *) obj)->smallList[0]);
-            push(((Value *) obj)->smallList[1]);
-            break;
-
-        case tListN:
-            push(((Value *) obj)->bigList);
-            break;
-
-        case tThunk:
-        case tBlackhole:
-            push(((Value *) obj)->thunk.env);
-            break;
-
-        case tApp:
-        case tPrimOpApp:
-            push(((Value *) obj)->app.left);
-            push(((Value *) obj)->app.right);
-            break;
-
-        case tLambda:
-            push(((Value *) obj)->lambda.env);
-            break;
-
-        case tPrimOp:
-            // FIXME: GC primops?
-            break;
-
-        default:
-            printError("don't know how to traverse object at %x (tag %d)", obj, obj->type);
-            abort();
-        }
-    };
-
-    stack.push(p);
-
-    size_t totalSize = 0;
-
-    while (!stack.empty()) {
-        auto obj = stack.top();
-        stack.pop();
-        if (seen.insert(obj).second) {
-            pushPointers(obj);
-            totalSize += getObjectSize(obj);
-        }
-    }
-
-    return {seen.size(), totalSize};
-}
-
-}

src/libexpr/gc.hh

@@ -1,482 +0,0 @@
-#pragma once
-
-#include "logging.hh"
-
-#include <stack>
-#include <limits>
-#include <cassert>
-#include <cstring>
-
-//#define GC_DEBUG 1
-
-namespace nix {
-
-typedef unsigned long Word;
-
-enum Tag {
-    tFree = 3,
-
-    // Misc types
-    tString,
-    tBindings,
-    tValueList,
-    tEnv,
-    tWithExprEnv,
-    tWithAttrsEnv,
-    tContext,
-
-    // Value tags
-    tInt,
-    tBool,
-    tShortString,
-    tStaticString,
-    tLongString,
-    tPath,
-    tNull,
-    tAttrs,
-    tList0,
-    tList1,
-    tList2,
-    tListN,
-    tThunk,
-    tApp,
-    tLambda,
-    tBlackhole,
-    tPrimOp,
-    tPrimOpApp,
-    tExternal,
-    tFloat
-};
-
-constexpr size_t WORD_SIZE = 8;
-
-struct Object
-{
-    friend class GC;
-
-public:
-    constexpr static size_t miscBytes = 7;
-
-public: // FIXME
-    Tag type:7;
-
-private:
-    bool marked:1;
-
-    unsigned long misc:56;
-
-    void unmark()
-    {
-        marked = false;
-    }
-
-protected:
-
-    Object(Tag type, unsigned long misc) : type(type), marked(false), misc(misc) { }
-
-    bool isMarked()
-    {
-        return marked;
-    }
-
-    void mark()
-    {
-        marked = true;
-    }
-
-    void setMisc(unsigned int misc)
-    {
-        this->misc = misc;
-    }
-
-    unsigned int getMisc() const
-    {
-        return misc;
-    }
-
-    char * getMiscData() const
-    {
-        return ((char *) this) + 1;
-    }
-};
-
-template<class T>
-struct PtrList : Object
-{
-    T * elems[0];
-
-    PtrList(Tag type, size_t size) : Object(type, size)
-    {
-        for (size_t i = 0; i < size; i++)
-            elems[i] = nullptr;
-    }
-
-    size_t size() const { return getMisc(); }
-    size_t words() const { return wordsFor(size()); }
-    static size_t wordsFor(size_t size) { return 1 + size; }
-};
-
-struct Free : Object
-{
-    Free * next;
-
-    Free(size_t size) : Object(tFree, size), next(nullptr) { }
-
-    // return size in words
-    size_t words() const { return getMisc(); }
-
-    void setSize(size_t size) { assert(size >= 1); setMisc(size); }
-};
-
-template<class T>
-struct Ptr;
-
-template<class T>
-struct Root;
-
-struct GC
-{
-
-private:
-
-    Ptr<Object> * frontPtrSentinel;
-    Ptr<Object> * backPtrSentinel;
-
-    Root<Object> * frontRootSentinel;
-    Root<Object> * backRootSentinel;
-
-    template<class T>
-    friend class Ptr;
-
-    template<class T>
-    friend class Root;
-
-    struct Arena
-    {
-        size_t size; // in words
-        Word * start;
-
-        Arena(size_t size)
-            : size(size)
-            , start(new Word[size])
-        {
-            assert(size >= 2);
-        }
-
-        Arena(const Arena & arena) = delete;
-
-        Arena(Arena && arena)
-        {
-            size = arena.size;
-            start = arena.start;
-            arena.start = nullptr;
-        }
-
-        ~Arena()
-        {
-            delete[] start;
-        }
-    };
-
-    size_t totalSize = 0;
-    size_t nextSize;
-
-    std::vector<Arena> arenas;
-
-    struct FreeList
-    {
-        size_t minSize;
-        Free * front = nullptr;
-    };
-
-    std::array<FreeList, 8> freeLists;
-
-public:
-
-    size_t getHeapSize() { return totalSize * WORD_SIZE; }
-
-    size_t allTimeWordsAllocated = 0;
-    size_t allTimeWordsFreed = 0;
-    uint64_t totalDurationMs = 0;
-
-private:
-
-    Object * allocObject(size_t size)
-    {
-        assert(size >= 2);
-
-        for (int attempt = 0; attempt < 3; attempt++) {
-
-            for (size_t i = 0; i < freeLists.size(); ++i) {
-                auto & freeList = freeLists[i];
-
-                if ((size <= freeList.minSize || i == freeLists.size() - 1) && freeList.front) {
-                    //printError("TRY %d %d %d", size, i, freeList.minSize);
-
-                    Free * * prev = &freeList.front;
-
-                    while (Free * freeObj = *prev) {
-                        //printError("LOOK %x %d %x", freeObj, freeObj->words(), freeObj->next);
-                        assert(freeObj->words() >= freeList.minSize);
-                        if (freeObj->words() == size) {
-                            // Convert the free object.
-                            *prev = freeObj->next;
-                            return (Object *) freeObj;
-                        } else if (freeObj->words() >= size + 2) {
-                            // Split the free object.
-                            auto newSize = freeObj->words() - size;
-                            freeObj->setSize(newSize);
-                            if (newSize < freeList.minSize) {
-                                /* The free object is now smaller than
-                                   the minimum size for this freelist,
-                                   so move it to another one. */
-                                //printError("MOVE %x %d -> %d", freeObj, newSize + size, newSize);
-                                *prev = freeObj->next;
-                                addToFreeList(freeObj);
-                            }
-                            return (Object *) (((Word *) freeObj) + newSize);
-                        } else if (freeObj->words() == size + 1) {
-                            // Return the free object and add a padding word.
-                            *prev = freeObj->next;
-                            freeObj->setSize(1);
-                            return (Object *) (((Word *) freeObj) + 1);
-                        } else {
-                            assert(freeObj->words() < size);
-                            prev = &freeObj->next;
-                        }
-                    }
-                }
-            }
-
-            if (attempt == 0) {
-                debug("allocation of %d bytes failed, GCing...", size * WORD_SIZE);
-                gc();
-            } else if (attempt == 1) {
-                addArena(std::max(nextSize, size));
-            }
-        }
-
-        throw Error("allocation of %d bytes failed", size);
-    }
-
-public:
-
-    GC();
-    ~GC();
-
-    template<typename T, typename... Args>
-    Ptr<T> alloc(size_t size, const Args & ... args)
-    {
-        auto raw = allocObject(size);
-        allTimeWordsAllocated += size;
-        return new (raw) T(args...);
-    }
-
-    void gc();
-
-    bool isObject(void * p);
-
-    void assertObject(void * p)
-    {
-        #if GC_DEBUG
-        if (!isObject(p)) {
-            printError("object %p is not an object", p);
-            abort();
-        }
-        #endif
-    }
-
-    /* Return the size in words of object 'obj'. */
-    static size_t getObjectSize(Object * obj);
-
-    /* Return the number and size in words of the objects reachable
-       from object 'obj'. */
-    std::tuple<size_t, size_t> getObjectClosureSize(Object * obj);
-
-private:
-
-    void addArena(size_t arenaSize);
-
-    void addToFreeList(Free * obj);
-
-    std::tuple<size_t, size_t, size_t, size_t> freeUnmarked(Arena & arena);
-};
-
-extern GC gc;
-
-template<class T>
-struct Ptr
-{
-private:
-
-    friend class GC;
-
-    Ptr * prev = nullptr, * next = nullptr;
-    T * value = nullptr;
-
-    void link()
-    {
-        prev = (Ptr *) gc.frontPtrSentinel;
-        next = prev->next;
-        next->prev = this;
-        prev->next = this;
-    }
-
-public:
-
-    Ptr() {
-        link();
-    }
-
-    Ptr(T * value) : value(value)
-    {
-        link();
-    }
-
-    Ptr(const Ptr & p)
-    {
-        auto & p2 = const_cast<Ptr &>(p);
-        value = p2.value;
-        next = &p2;
-        prev = p2.prev;
-        prev->next = this;
-        p2.prev = this;
-    }
-
-    Ptr(Ptr && p)
-    {
-        link();
-        value = p.value;
-        p.value = nullptr;
-    }
-
-    Ptr & operator =(const Ptr & p)
-    {
-        value = p.value;
-        return *this;
-    }
-
-    Ptr & operator =(Ptr && p)
-    {
-        value = p.value;
-        p.value = nullptr;
-        return *this;
-    }
-
-    Ptr & operator =(T * v)
-    {
-        value = v;
-        return *this;
-    }
-
-    ~Ptr()
-    {
-        assert(next);
-        assert(prev);
-        assert(next->prev == this);
-        next->prev = prev;
-        assert(prev->next == this);
-        prev->next = next;
-    }
-
-    T * operator ->()
-    {
-        return value;
-    }
-
-    T * operator ->() const // FIXME
-    {
-        return value;
-    }
-
-    operator T * ()
-    {
-        return value;
-    }
-
-    operator T & ()
-    {
-        assert(value);
-        return *value;
-    }
-
-    operator bool() const
-    {
-        return value != nullptr;
-    }
-};
-
-template<class T>
-struct Root
-{
-    Root * prev = nullptr, * next = nullptr;
-    T value;
-
-    template<typename... Args>
-    Root(const Args & ... args)
-        : value{args... }
-    {
-        prev = (Root *) gc.frontRootSentinel;
-        next = prev->next;
-        next->prev = this;
-        prev->next = this;
-    }
-
-    Root(const Root & p) = delete;
-    Root(Root && p) = delete;
-
-    Root & operator =(const T & v) { value = v; return *this; }
-
-    ~Root()
-    {
-        assert(next);
-        assert(prev);
-        assert(next->prev == this);
-        next->prev = prev;
-        assert(prev->next == this);
-        prev->next = next;
-    }
-
-    T * operator ->()
-    {
-        return &value;
-    }
-
-    operator T * ()
-    {
-        return &value;
-    }
-
-    operator T & ()
-    {
-        return value;
-    }
-};
-
-struct String : Object
-{
-    char s[0];
-
-    String(size_t len, const char * src)
-        : Object(tString, len)
-    {
-        std::memcpy(s, src, len + 1);
-    }
-
-    size_t words() const { return wordsFor(getMisc()); }
-
-    /* Return the number of words needed to store a string of 'len'
-       characters (where 'len' excludes the terminator). */
-    static size_t wordsFor(size_t len)
-    {
-        return len / WORD_SIZE + 2;
-    }
-
-    static String * alloc(const char * src)
-    {
-        auto len = strlen(src);
-        return gc.alloc<String>(wordsFor(len), len, src);
-    }
-};
-
-}

src/libexpr/get-drvs.cc

@@ -1,7 +1,7 @@
 #include "get-drvs.hh"
 #include "util.hh"
 #include "eval-inline.hh"
-#include "derivations.hh"
+#include "store-api.hh"
 
 #include <cstring>
 #include <regex>
@@ -115,15 +115,15 @@ DrvInfo::Outputs DrvInfo::queryOutputs(bool onlyOutputsToInstall)
         return outputs;
 
     /* Check for `meta.outputsToInstall` and return `outputs` reduced to that. */
-    const auto outTI = queryMeta("outputsToInstall");
+    const Value * outTI = queryMeta("outputsToInstall");
     if (!outTI) return outputs;
     const auto errMsg = Error("this derivation has bad 'meta.outputsToInstall'");
         /* ^ this shows during `nix-env -i` right under the bad derivation */
     if (!outTI->isList()) throw errMsg;
     Outputs result;
     for (auto i = outTI->listElems(); i != outTI->listElems() + outTI->listSize(); ++i) {
-        if (!(*i)->isString()) throw errMsg;
-        auto out = outputs.find((*i)->getString());
+        if ((*i)->type != tString) throw errMsg;
+        auto out = outputs.find((*i)->string.s);
         if (out == outputs.end()) throw errMsg;
         result.insert(*out);
     }
@@ -178,7 +178,8 @@ bool DrvInfo::checkMeta(Value & v)
             if (!checkMeta(*i.value)) return false;
         return true;
     }
-    else return v.type == tInt || v.type == tBool || v.isString() || v.type == tFloat;
+    else return v.type == tInt || v.type == tBool || v.type == tString ||
+                v.type == tFloat;
 }
 
 
@@ -193,36 +194,36 @@ Value * DrvInfo::queryMeta(const string & name)
 
 string DrvInfo::queryMetaString(const string & name)
 {
-    auto v = queryMeta(name);
-    if (!v || !v->isString()) return "";
-    return v->getString();
+    Value * v = queryMeta(name);
+    if (!v || v->type != tString) return "";
+    return v->string.s;
 }
 
 
 NixInt DrvInfo::queryMetaInt(const string & name, NixInt def)
 {
-    auto v = queryMeta(name);
+    Value * v = queryMeta(name);
     if (!v) return def;
     if (v->type == tInt) return v->integer;
-    if (v->isString()) {
+    if (v->type == tString) {
         /* Backwards compatibility with before we had support for
            integer meta fields. */
         NixInt n;
-        if (string2Int(v->getString(), n)) return n;
+        if (string2Int(v->string.s, n)) return n;
     }
     return def;
 }
 
 NixFloat DrvInfo::queryMetaFloat(const string & name, NixFloat def)
 {
-    auto v = queryMeta(name);
+    Value * v = queryMeta(name);
     if (!v) return def;
     if (v->type == tFloat) return v->fpoint;
-    if (v->isString()) {
+    if (v->type == tString) {
         /* Backwards compatibility with before we had support for
            float meta fields. */
         NixFloat n;
-        if (string2Float(v->getString(), n)) return n;
+        if (string2Float(v->string.s, n)) return n;
     }
     return def;
 }
@@ -230,15 +231,14 @@ NixFloat DrvInfo::queryMetaFloat(const string & name, NixFloat def)
 
 bool DrvInfo::queryMetaBool(const string & name, bool def)
 {
-    auto v = queryMeta(name);
+    Value * v = queryMeta(name);
     if (!v) return def;
     if (v->type == tBool) return v->boolean;
-    if (v->isString()) {
+    if (v->type == tString) {
         /* Backwards compatibility with before we had support for
            Boolean meta fields. */
-        auto s = v->getString();
-        if (strcmp(s, "true") == 0) return true;
-        if (strcmp(s, "false") == 0) return false;
+        if (strcmp(v->string.s, "true") == 0) return true;
+        if (strcmp(v->string.s, "false") == 0) return false;
     }
     return def;
 }
@@ -247,8 +247,8 @@ bool DrvInfo::queryMetaBool(const string & name, bool def)
 void DrvInfo::setMeta(const string & name, Value * v)
 {
     getMeta();
-    Ptr<Bindings> old = meta;
-    meta = Bindings::allocBindings(1 + (old ? old->size() : 0));
+    Bindings * old = meta;
+    meta = state->allocBindings(1 + (old ? old->size() : 0));
     Symbol sym = state->symbols.create(name);
     if (old)
         for (auto i : *old)
@@ -260,7 +260,6 @@ void DrvInfo::setMeta(const string & name, Value * v)
 
 
 /* Cache for already considered attrsets. */
-// FIXME: Use Ptr?
 typedef set<Bindings *> Done;
 
 
@@ -320,24 +319,24 @@ static void getDerivations(EvalState & state, Value & vIn,
     DrvInfos & drvs, Done & done,
     bool ignoreAssertionFailures)
 {
-    Root<Value> v;
+    Value v;
     state.autoCallFunction(autoArgs, vIn, v);
 
     /* Process the expression. */
     if (!getDerivation(state, v, pathPrefix, drvs, done, ignoreAssertionFailures)) ;
 
-    else if (v->type == tAttrs) {
+    else if (v.type == tAttrs) {
 
         /* !!! undocumented hackery to support combining channels in
            nix-env.cc. */
-        bool combineChannels = v->attrs->find(state.symbols.create("_combineChannels")) != v->attrs->end();
+        bool combineChannels = v.attrs->find(state.symbols.create("_combineChannels")) != v.attrs->end();
 
         /* Consider the attributes in sorted order to get more
            deterministic behaviour in nix-env operations (e.g. when
            there are names clashes between derivations, the derivation
            bound to the attribute with the "lower" name should take
            precedence). */
-        for (auto & i : v->attrs->lexicographicOrder()) {
+        for (auto & i : v.attrs->lexicographicOrder()) {
             debug("evaluating attribute '%1%'", i->name);
             if (!std::regex_match(std::string(i->name), attrRegex))
                 continue;
@@ -349,7 +348,7 @@ static void getDerivations(EvalState & state, Value & vIn,
                    should we recurse into it?  => Only if it has a
                    `recurseForDerivations = true' attribute. */
                 if (i->value->type == tAttrs) {
-                    Bindings::iterator j = i->value->attrs->find(state.symbols.create("recurseForDerivations"));
+                    Bindings::iterator j = i->value->attrs->find(state.sRecurseForDerivations);
                     if (j != i->value->attrs->end() && state.forceBool(*j->value, *j->pos))
                         getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures);
                 }
@@ -357,11 +356,11 @@ static void getDerivations(EvalState & state, Value & vIn,
         }
     }
 
-    else if (v->isList()) {
-        for (unsigned int n = 0; n < v->listSize(); ++n) {
+    else if (v.isList()) {
+        for (unsigned int n = 0; n < v.listSize(); ++n) {
             string pathPrefix2 = addToPath(pathPrefix, (format("%1%") % n).str());
-            if (getDerivation(state, *v->listElems()[n], pathPrefix2, drvs, done, ignoreAssertionFailures))
-                getDerivations(state, *v->listElems()[n], pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures);
+            if (getDerivation(state, *v.listElems()[n], pathPrefix2, drvs, done, ignoreAssertionFailures))
+                getDerivations(state, *v.listElems()[n], pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures);
         }
     }
 

src/libexpr/get-drvs.hh

@@ -26,8 +26,7 @@ private:
 
     bool failed = false; // set if we get an AssertionError
 
-    Ptr<Bindings> attrs;
-    Ptr<Bindings> meta;
+    Bindings * attrs = nullptr, * meta = nullptr;
 
     Bindings * getMeta();
 
@@ -70,7 +69,11 @@ public:
 };
 
 
-typedef std::list<DrvInfo> DrvInfos;
+#if HAVE_BOEHMGC
+typedef list<DrvInfo, traceable_allocator<DrvInfo> > DrvInfos;
+#else
+typedef list<DrvInfo> DrvInfos;
+#endif
 
 
 /* If value `v' denotes a derivation, return a DrvInfo object

src/libexpr/json-to-value.cc

@@ -4,40 +4,38 @@
 #include <nlohmann/json.hpp>
 
 using json = nlohmann::json;
-using std::unique_ptr;
 
 namespace nix {
 
 // for more information, refer to
 // https://github.com/nlohmann/json/blob/master/include/nlohmann/detail/input/json_sax.hpp
 class JSONSax : nlohmann::json_sax<json> {
-
     class JSONState {
     protected:
-        unique_ptr<JSONState> parent;
-        Ptr<Value> v;
+        std::unique_ptr<JSONState> parent;
+        RootValue v;
     public:
-        virtual unique_ptr<JSONState> resolve(EvalState &)
+        virtual std::unique_ptr<JSONState> resolve(EvalState &)
         {
             throw std::logic_error("tried to close toplevel json parser state");
-        };
-        explicit JSONState(unique_ptr<JSONState> && p) : parent(std::move(p)), v(nullptr) {};
-        explicit JSONState(Value * v) : v(v) {};
+        }
+        explicit JSONState(std::unique_ptr<JSONState> && p) : parent(std::move(p)) {}
+        explicit JSONState(Value * v) : v(allocRootValue(v)) {}
         JSONState(JSONState & p) = delete;
         Value & value(EvalState & state)
         {
             if (!v)
-                v = state.allocValue();
-            return *v;
-        };
-        virtual ~JSONState() {};
-        virtual void add() {};
+                v = allocRootValue(state.allocValue());
+            return **v;
+        }
+        virtual ~JSONState() {}
+        virtual void add() {}
     };
 
     class JSONObjectState : public JSONState {
         using JSONState::JSONState;
-        ValueMap attrs = ValueMap();
-        virtual unique_ptr<JSONState> resolve(EvalState & state) override
+        ValueMap attrs;
+        std::unique_ptr<JSONState> resolve(EvalState & state) override
         {
             Value & v = parent->value(state);
             state.mkAttrs(v, attrs.size());
@@ -45,17 +43,17 @@ class JSONSax : nlohmann::json_sax<json> {
                 v.attrs->push_back(Attr(i.first, i.second));
             return std::move(parent);
         }
-        virtual void add() override { v = nullptr; };
+        void add() override { v = nullptr; }
     public:
         void key(string_t & name, EvalState & state)
         {
-            attrs[state.symbols.create(name)] = &value(state);
+            attrs.insert_or_assign(state.symbols.create(name), &value(state));
         }
     };
 
     class JSONListState : public JSONState {
-        ValueVector values = ValueVector();
-        virtual unique_ptr<JSONState> resolve(EvalState & state) override
+        ValueVector values;
+        std::unique_ptr<JSONState> resolve(EvalState & state) override
         {
             Value & v = parent->value(state);
             state.mkList(v, values.size());
@@ -64,19 +62,19 @@ class JSONSax : nlohmann::json_sax<json> {
             }
             return std::move(parent);
         }
-        virtual void add() override {
-            values.push_back(v);
+        void add() override {
+            values.push_back(*v);
             v = nullptr;
-        };
+        }
     public:
-        JSONListState(unique_ptr<JSONState> && p, std::size_t reserve) : JSONState(std::move(p))
+        JSONListState(std::unique_ptr<JSONState> && p, std::size_t reserve) : JSONState(std::move(p))
         {
             values.reserve(reserve);
         }
     };
 
     EvalState & state;
-    unique_ptr<JSONState> rs;
+    std::unique_ptr<JSONState> rs;
 
     template<typename T, typename... Args> inline bool handle_value(T f, Args... args)
     {
@@ -108,14 +106,14 @@ public:
         return handle_value(mkInt, val);
     }
 
-    bool number_float(number_float_t val, const string_t& s)
+    bool number_float(number_float_t val, const string_t & s)
     {
         return handle_value(mkFloat, val);
     }
 
-    bool string(string_t& val)
+    bool string(string_t & val)
     {
-        return handle_value<Value & (Value &, std::string_view)>(mkString, (std::string_view) val);
+        return handle_value<void(Value&, const char*)>(mkString, val.c_str());
     }
 
     bool start_object(std::size_t len)
@@ -124,7 +122,7 @@ public:
         return true;
     }
 
-    bool key(string_t& name)
+    bool key(string_t & name)
     {
         dynamic_cast<JSONObjectState*>(rs.get())->key(name, state);
         return true;

src/libexpr/lexer.l

@@ -127,14 +127,14 @@ or          { return OR_KW; }
               try {
                   yylval->n = boost::lexical_cast<int64_t>(yytext);
               } catch (const boost::bad_lexical_cast &) {
-                  throw ParseError(format("invalid integer '%1%'") % yytext);
+                  throw ParseError("invalid integer '%1%'", yytext);
               }
               return INT;
             }
 {FLOAT}     { errno = 0;
               yylval->nf = strtod(yytext, 0);
               if (errno != 0)
-                  throw ParseError(format("invalid float '%1%'") % yytext);
+                  throw ParseError("invalid float '%1%'", yytext);
               return FLOAT;
             }
 
@@ -219,4 +219,3 @@ or          { return OR_KW; }
             }
 
 %%
-

src/libexpr/local.mk

@@ -6,13 +6,20 @@ libexpr_DIR := $(d)
 
 libexpr_SOURCES := $(wildcard $(d)/*.cc) $(wildcard $(d)/primops/*.cc) $(d)/lexer-tab.cc $(d)/parser-tab.cc
 
-libexpr_LIBS = libutil libstore libnixrust
+libexpr_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/libmain -I src/libexpr
+
+libexpr_LIBS = libutil libstore libfetchers
 
 libexpr_LDFLAGS =
 ifneq ($(OS), FreeBSD)
  libexpr_LDFLAGS += -ldl
 endif
 
+# The dependency on libgc must be propagated (i.e. meaning that
+# programs/libraries that use libexpr must explicitly pass -lgc),
+# because inline functions in libexpr's header files call libgc.
+libexpr_LDFLAGS_PROPAGATED = $(BDW_GC_LIBS)
+
 libexpr_ORDER_AFTER := $(d)/parser-tab.cc $(d)/parser-tab.hh $(d)/lexer-tab.cc $(d)/lexer-tab.hh
 
 $(d)/parser-tab.cc $(d)/parser-tab.hh: $(d)/parser.y
@@ -26,3 +33,5 @@ clean-files += $(d)/parser-tab.cc $(d)/parser-tab.hh $(d)/lexer-tab.cc $(d)/lexe
 dist-files += $(d)/parser-tab.cc $(d)/parser-tab.hh $(d)/lexer-tab.cc $(d)/lexer-tab.hh
 
 $(eval $(call install-file-in, $(d)/nix-expr.pc, $(prefix)/lib/pkgconfig, 0644))
+
+$(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh

src/libexpr/nixexpr.cc

@@ -80,7 +80,7 @@ void ExprString::show(std::ostream & str) const
 
 void ExprPath::show(std::ostream & str) const
 {
-    str << v->path->s;
+    str << s;
 }
 
 void ExprVar::show(std::ostream & str) const
@@ -197,7 +197,22 @@ std::ostream & operator << (std::ostream & str, const Pos & pos)
     if (!pos)
         str << "undefined position";
     else
-        str << (format(ANSI_BOLD "%1%" ANSI_NORMAL ":%2%:%3%") % (string) pos.file % pos.line % pos.column).str();
+    {
+        auto f = format(ANSI_BOLD "%1%" ANSI_NORMAL ":%2%:%3%");
+        switch (pos.origin) {
+            case foFile:
+                f % (string) pos.file;
+                break;
+            case foStdin:
+            case foString:
+                f % "(string)";
+                break;
+            default:
+                throw Error("unhandled Pos origin!");
+        }
+        str << (f % pos.line % pos.column).str();
+    }
+
     return str;
 }
 
@@ -267,8 +282,11 @@ void ExprVar::bindVars(const StaticEnv & env)
     /* Otherwise, the variable must be obtained from the nearest
        enclosing `with'.  If there is no `with', then we can issue an
        "undefined variable" error now. */
-    if (withLevel == -1) throw UndefinedVarError(format("undefined variable '%1%' at %2%") % name % pos);
-
+    if (withLevel == -1)
+        throw UndefinedVarError({
+            .hint = hintfmt("undefined variable '%1%'", name),
+            .errPos = pos
+        });
     fromWith = true;
     this->level = withLevel;
 }

src/libexpr/nixexpr.hh

@@ -2,6 +2,7 @@
 
 #include "value.hh"
 #include "symbol-table.hh"
+#include "error.hh"
 
 #include <map>
 
@@ -23,11 +24,12 @@ MakeError(RestrictedPathError, Error);
 
 struct Pos
 {
+    FileOrigin origin;
     Symbol file;
     unsigned int line, column;
-    Pos() : line(0), column(0) { };
-    Pos(const Symbol & file, unsigned int line, unsigned int column)
-        : file(file), line(line), column(column) { };
+    Pos() : origin(foString), line(0), column(0) { };
+    Pos(FileOrigin origin, const Symbol & file, unsigned int line, unsigned int column)
+        : origin(origin), file(file), line(line), column(column) { };
     operator bool() const
     {
         return line != 0;
@@ -78,7 +80,7 @@ struct Expr
     virtual void show(std::ostream & str) const;
     virtual void bindVars(const StaticEnv & env);
     virtual void eval(EvalState & state, Env & env, Value & v);
-    virtual Ptr<Value> maybeThunk(EvalState & state, Env & env);
+    virtual Value * maybeThunk(EvalState & state, Env & env);
     virtual void setName(Symbol & name);
 };
 
@@ -92,37 +94,28 @@ std::ostream & operator << (std::ostream & str, const Expr & e);
 struct ExprInt : Expr
 {
     NixInt n;
-    Ptr<Value> v;
-    ExprInt(NixInt n) : n(n) {
-        v = gc.alloc<Value>(Value::words());
-        mkInt(v, n);
-    };
+    Value v;
+    ExprInt(NixInt n) : n(n) { mkInt(v, n); };
     COMMON_METHODS
-    Ptr<Value> maybeThunk(EvalState & state, Env & env) override;
+    Value * maybeThunk(EvalState & state, Env & env);
 };
 
 struct ExprFloat : Expr
 {
     NixFloat nf;
-    Ptr<Value> v;
-    ExprFloat(NixFloat nf) : nf(nf) {
-        v = gc.alloc<Value>(Value::words());
-        mkFloat(v, nf);
-    };
+    Value v;
+    ExprFloat(NixFloat nf) : nf(nf) { mkFloat(v, nf); };
     COMMON_METHODS
-    Ptr<Value> maybeThunk(EvalState & state, Env & env) override;
+    Value * maybeThunk(EvalState & state, Env & env);
 };
 
 struct ExprString : Expr
 {
     Symbol s;
-    Ptr<Value> v;
-    ExprString(const Symbol & s) : s(s) {
-        v = gc.alloc<Value>(Value::words());
-        mkString(v, s);
-    };
+    Value v;
+    ExprString(const Symbol & s) : s(s) { mkString(v, s); };
     COMMON_METHODS
-    Ptr<Value> maybeThunk(EvalState & state, Env & env) override;
+    Value * maybeThunk(EvalState & state, Env & env);
 };
 
 /* Temporary class used during parsing of indented strings. */
@@ -134,13 +127,11 @@ struct ExprIndStr : Expr
 
 struct ExprPath : Expr
 {
-    Ptr<Value> v;
-    ExprPath(const string & s) {
-        v = gc.alloc<Value>(Value::words());
-        mkPath(v, s);
-    };
+    string s;
+    Value v;
+    ExprPath(const string & s) : s(s) { mkPathNoCopy(v, this->s.c_str()); };
     COMMON_METHODS
-    Ptr<Value> maybeThunk(EvalState & state, Env & env) override;
+    Value * maybeThunk(EvalState & state, Env & env);
 };
 
 struct ExprVar : Expr
@@ -164,7 +155,7 @@ struct ExprVar : Expr
     ExprVar(const Symbol & name) : name(name) { };
     ExprVar(const Pos & pos, const Symbol & name) : pos(pos), name(name) { };
     COMMON_METHODS
-    Ptr<Value> maybeThunk(EvalState & state, Env & env) override;
+    Value * maybeThunk(EvalState & state, Env & env);
 };
 
 struct ExprSelect : Expr
@@ -220,9 +211,10 @@ struct ExprList : Expr
 
 struct Formal
 {
+    Pos pos;
     Symbol name;
     Expr * def;
-    Formal(const Symbol & name, Expr * def) : name(name), def(def) { };
+    Formal(const Pos & pos, const Symbol & name, Expr * def) : pos(pos), name(name), def(def) { };
 };
 
 struct Formals
@@ -245,8 +237,10 @@ struct ExprLambda : Expr
         : pos(pos), arg(arg), matchAttrs(matchAttrs), formals(formals), body(body)
     {
         if (!arg.empty() && formals && formals->argNames.find(arg) != formals->argNames.end())
-            throw ParseError(format("duplicate formal function argument '%1%' at %2%")
-                % arg % pos);
+            throw ParseError({
+                .hint = hintfmt("duplicate formal function argument '%1%'", arg),
+                .errPos = pos
+            });
     };
     void setName(Symbol & name);
     string showNamePos() const;
@@ -272,8 +266,9 @@ struct ExprWith : Expr
 
 struct ExprIf : Expr
 {
+    Pos pos;
     Expr * cond, * then, * else_;
-    ExprIf(Expr * cond, Expr * then, Expr * else_) : cond(cond), then(then), else_(else_) { };
+    ExprIf(const Pos & pos, Expr * cond, Expr * then, Expr * else_) : pos(pos), cond(cond), then(then), else_(else_) { };
     COMMON_METHODS
 };
 

src/libexpr/parser.y

@@ -30,8 +30,9 @@ namespace nix {
         SymbolTable & symbols;
         Expr * result;
         Path basePath;
-        Symbol path;
-        string error;
+        Symbol file;
+        FileOrigin origin;
+        ErrorInfo error;
         Symbol sLetBody;
         ParseData(EvalState & state)
             : state(state)
@@ -64,15 +65,19 @@ namespace nix {
 
 static void dupAttr(const AttrPath & attrPath, const Pos & pos, const Pos & prevPos)
 {
-    throw ParseError(format("attribute '%1%' at %2% already defined at %3%")
-        % showAttrPath(attrPath) % pos % prevPos);
+    throw ParseError({
+         .hint = hintfmt("attribute '%1%' already defined at %2%",
+            showAttrPath(attrPath), prevPos),
+         .errPos = pos
+    });
 }
 
-
 static void dupAttr(Symbol attr, const Pos & pos, const Pos & prevPos)
 {
-    throw ParseError(format("attribute '%1%' at %2% already defined at %3%")
-        % attr % pos % prevPos);
+    throw ParseError({
+        .hint = hintfmt("attribute '%1%' already defined at %2%", attr, prevPos),
+        .errPos = pos
+    });
 }
 
 
@@ -140,8 +145,11 @@ static void addAttr(ExprAttrs * attrs, AttrPath & attrPath,
 static void addFormal(const Pos & pos, Formals * formals, const Formal & formal)
 {
     if (!formals->argNames.insert(formal.name).second)
-        throw ParseError(format("duplicate formal function argument '%1%' at %2%")
-            % formal.name % pos);
+        throw ParseError({
+            .hint = hintfmt("duplicate formal function argument '%1%'",
+                formal.name),
+            .errPos = pos
+        });
     formals->formals.push_front(formal);
 }
 
@@ -238,7 +246,7 @@ static Expr * stripIndentation(const Pos & pos, SymbolTable & symbols, vector<Ex
 
 static inline Pos makeCurPos(const YYLTYPE & loc, ParseData * data)
 {
-    return Pos(data->path, loc.first_line, loc.first_column);
+    return Pos(data->origin, data->file, loc.first_line, loc.first_column);
 }
 
 #define CUR_POS makeCurPos(*yylocp, data)
@@ -249,8 +257,10 @@ static inline Pos makeCurPos(const YYLTYPE & loc, ParseData * data)
 
 void yyerror(YYLTYPE * loc, yyscan_t scanner, ParseData * data, const char * error)
 {
-    data->error = (format("%1%, at %2%")
-        % error % makeCurPos(*loc, data)).str();
+    data->error = {
+        .hint = hintfmt(error),
+        .errPos = makeCurPos(*loc, data)
+    };
 }
 
 
@@ -327,15 +337,17 @@ expr_function
     { $$ = new ExprWith(CUR_POS, $2, $4); }
   | LET binds IN expr_function
     { if (!$2->dynamicAttrs.empty())
-        throw ParseError(format("dynamic attributes not allowed in let at %1%")
-            % CUR_POS);
+        throw ParseError({
+            .hint = hintfmt("dynamic attributes not allowed in let"),
+            .errPos = CUR_POS
+        });
       $$ = new ExprLet($2, $4);
     }
   | expr_if
   ;
 
 expr_if
-  : IF expr THEN expr ELSE expr { $$ = new ExprIf($2, $4, $6); }
+  : IF expr THEN expr ELSE expr { $$ = new ExprIf(CUR_POS, $2, $4, $6); }
   | expr_op
   ;
 
@@ -405,7 +417,10 @@ expr_simple
   | URI {
       static bool noURLLiterals = settings.isExperimentalFeatureEnabled("no-url-literals");
       if (noURLLiterals)
-          throw ParseError("URL literals are disabled, at %s", CUR_POS);
+          throw ParseError({
+              .hint = hintfmt("URL literals are disabled"),
+              .errPos = CUR_POS
+          });
       $$ = new ExprString(data->symbols.create($1));
   }
   | '(' expr ')' { $$ = $2; }
@@ -475,8 +490,10 @@ attrs
           $$->push_back(AttrName(str->s));
           delete str;
       } else
-          throw ParseError(format("dynamic attributes not allowed in inherit at %1%")
-              % makeCurPos(@2, data));
+          throw ParseError({
+              .hint = hintfmt("dynamic attributes not allowed in inherit"),
+              .errPos = makeCurPos(@2, data)
+          });
     }
   | { $$ = new AttrPath; }
   ;
@@ -531,8 +548,8 @@ formals
   ;
 
 formal
-  : ID { $$ = new Formal(data->symbols.create($1), 0); }
-  | ID '?' expr { $$ = new Formal(data->symbols.create($1), $3); }
+  : ID { $$ = new Formal(CUR_POS, data->symbols.create($1), 0); }
+  | ID '?' expr { $$ = new Formal(CUR_POS, data->symbols.create($1), $3); }
   ;
 
 %%
@@ -544,20 +561,32 @@ formal
 #include <unistd.h>
 
 #include "eval.hh"
-#include "download.hh"
+#include "filetransfer.hh"
+#include "fetchers.hh"
 #include "store-api.hh"
 
 
 namespace nix {
 
 
-Expr * EvalState::parse(const char * text,
+Expr * EvalState::parse(const char * text, FileOrigin origin,
     const Path & path, const Path & basePath, StaticEnv & staticEnv)
 {
     yyscan_t scanner;
     ParseData data(*this);
+    data.origin = origin;
+    switch (origin) {
+        case foFile: 
+            data.file = data.symbols.create(path);
+            break;
+        case foStdin:
+        case foString:
+            data.file = data.symbols.create(text);
+            break;
+        default:
+            assert(false);
+    }
     data.basePath = basePath;
-    data.path = data.symbols.create(path);
 
     yylex_init(&scanner);
     yy_scan_string(text, scanner);
@@ -607,17 +636,17 @@ Expr * EvalState::parseExprFromFile(const Path & path)
 
 Expr * EvalState::parseExprFromFile(const Path & path, StaticEnv & staticEnv)
 {
-    return parse(readFile(path).c_str(), path, dirOf(path), staticEnv);
+    return parse(readFile(path).c_str(), foFile, path, dirOf(path), staticEnv);
 }
 
 
-Expr * EvalState::parseExprFromString(const string & s, const Path & basePath, StaticEnv & staticEnv)
+Expr * EvalState::parseExprFromString(std::string_view s, const Path & basePath, StaticEnv & staticEnv)
 {
-    return parse(s.c_str(), "(string)", basePath, staticEnv);
+    return parse(s.data(), foString, "", basePath, staticEnv);
 }
 
 
-Expr * EvalState::parseExprFromString(const string & s, const Path & basePath)
+Expr * EvalState::parseExprFromString(std::string_view s, const Path & basePath)
 {
     return parseExprFromString(s, basePath, staticBaseEnv);
 }
@@ -626,7 +655,7 @@ Expr * EvalState::parseExprFromString(const string & s, const Path & basePath)
 Expr * EvalState::parseStdin()
 {
     //Activity act(*logger, lvlTalkative, format("parsing standard input"));
-    return parseExprFromString(drainFD(0), absPath("."));
+    return parse(drainFD(0).data(), foStdin, "", absPath("."), staticBaseEnv);
 }
 
 
@@ -670,11 +699,13 @@ Path EvalState::findFile(SearchPath & searchPath, const string & path, const Pos
         Path res = r.second + suffix;
         if (pathExists(res)) return canonPath(res);
     }
-    format f = format(
-        "file '%1%' was not found in the Nix search path (add it using $NIX_PATH or -I)"
-        + string(pos ? ", at %2%" : ""));
-    f.exceptions(boost::io::all_error_bits ^ boost::io::too_many_args_bit);
-    throw ThrownError(f % path % pos);
+    throw ThrownError({
+        .hint = hintfmt(evalSettings.pureEval
+            ? "cannot look up '<%s>' in pure evaluation mode (use '--impure' to override)"
+            : "file '%s' was not found in the Nix search path (add it using $NIX_PATH or -I)",
+            path),
+        .errPos = pos
+    });
 }
 
 
@@ -687,11 +718,13 @@ std::pair<bool, std::string> EvalState::resolveSearchPathElem(const SearchPathEl
 
     if (isUri(elem.second)) {
         try {
-            CachedDownloadRequest request(elem.second);
-            request.unpack = true;
-            res = { true, getDownloader()->downloadCached(store, request).path };
-        } catch (DownloadError & e) {
-            printError(format("warning: Nix search path entry '%1%' cannot be downloaded, ignoring") % elem.second);
+            res = { true, store->toRealPath(fetchers::downloadTarball(
+                        store, resolveUri(elem.second), "source", false).storePath) };
+        } catch (FileTransferError & e) {
+            logWarning({
+                .name = "Entry download",
+                .hint = hintfmt("Nix search path entry '%1%' cannot be downloaded, ignoring", elem.second)
+            });
             res = { false, "" };
         }
     } else {
@@ -699,7 +732,10 @@ std::pair<bool, std::string> EvalState::resolveSearchPathElem(const SearchPathEl
         if (pathExists(path))
             res = { true, path };
         else {
-            printError(format("warning: Nix search path entry '%1%' does not exist, ignoring") % elem.second);
+            logWarning({
+                .name = "Entry not found",
+                .hint = hintfmt("warning: Nix search path entry '%1%' does not exist, ignoring", elem.second)
+            });
             res = { false, "" };
         }
     }

src/libexpr/primops.hh

@@ -7,12 +7,25 @@ namespace nix {
 
 struct RegisterPrimOp
 {
-    typedef std::vector<std::tuple<std::string, size_t, PrimOpFun>> PrimOps;
+    struct Info
+    {
+        std::string name;
+        size_t arity;
+        PrimOpFun primOp;
+        std::optional<std::string> requiredFeature;
+    };
+
+    typedef std::vector<Info> PrimOps;
     static PrimOps * primOps;
+
     /* You can register a constant by passing an arity of 0. fun
        will get called during EvalState initialization, so there
        may be primops not yet added and builtins is not yet sorted. */
-    RegisterPrimOp(std::string name, size_t arity, PrimOpFun fun);
+    RegisterPrimOp(
+        std::string name,
+        size_t arity,
+        PrimOpFun fun,
+        std::optional<std::string> requiredFeature = {});
 };
 
 /* These primops are disabled without enableNativeCode, but plugins
@@ -20,6 +33,7 @@ struct RegisterPrimOp
    them. */
 /* Load a ValueInitializer from a DSO and return whatever it initializes */
 void prim_importNative(EvalState & state, const Pos & pos, Value * * args, Value & v);
+
 /* Execute a program and parse its output */
 void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v);
 

src/libexpr/primops/context.cc

@@ -1,6 +1,6 @@
 #include "primops.hh"
 #include "eval-inline.hh"
-#include "derivations.hh"
+#include "store-api.hh"
 
 namespace nix {
 
@@ -146,7 +146,10 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg
     auto sAllOutputs = state.symbols.create("allOutputs");
     for (auto & i : *args[1]->attrs) {
         if (!state.store->isStorePath(i.name))
-            throw EvalError("Context key '%s' is not a store path, at %s", i.name, i.pos);
+            throw EvalError({
+                .hint = hintfmt("Context key '%s' is not a store path", i.name),
+                .errPos = *i.pos
+            });
         if (!settings.readOnlyMode)
             state.store->ensurePath(state.store->parseStorePath(i.name));
         state.forceAttrs(*i.value, *i.pos);
@@ -160,7 +163,10 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg
         if (iter != i.value->attrs->end()) {
             if (state.forceBool(*iter->value, *iter->pos)) {
                 if (!isDerivation(i.name)) {
-                    throw EvalError("tried to add all-outputs context of %s, which is not a derivation, to a string, at %s", i.name, *i.pos);
+                    throw EvalError({
+                        .hint = hintfmt("Tried to add all-outputs context of %s, which is not a derivation, to a string", i.name),
+                        .errPos = *i.pos
+                    });
                 }
                 context.insert("=" + string(i.name));
             }
@@ -170,7 +176,10 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg
         if (iter != i.value->attrs->end()) {
             state.forceList(*iter->value, *iter->pos);
             if (iter->value->listSize() && !isDerivation(i.name)) {
-                throw EvalError("tried to add derivation output context of %s, which is not a derivation, to a string, at %s", i.name, *i.pos);
+                throw EvalError({
+                    .hint = hintfmt("Tried to add derivation output context of %s, which is not a derivation, to a string", i.name),
+                    .errPos = *i.pos
+                });
             }
             for (unsigned int n = 0; n < iter->value->listSize(); ++n) {
                 auto name = state.forceStringNoCtx(*iter->value->listElems()[n], *iter->pos);

src/libexpr/primops/fetchGit.cc

@@ -1,203 +1,19 @@
 #include "primops.hh"
 #include "eval-inline.hh"
-#include "download.hh"
 #include "store-api.hh"
-#include "pathlocks.hh"
 #include "hash.hh"
-#include "tarfile.hh"
-
-#include <sys/time.h>
-
-#include <regex>
-
-#include <nlohmann/json.hpp>
-
-using namespace std::string_literals;
+#include "fetchers.hh"
+#include "url.hh"
 
 namespace nix {
 
-struct GitInfo
-{
-    Path storePath;
-    std::string rev;
-    std::string shortRev;
-    uint64_t revCount = 0;
-};
-
-std::regex revRegex("^[0-9a-fA-F]{40}$");
-
-GitInfo exportGit(ref<Store> store, const std::string & uri,
-    std::optional<std::string> ref, std::string rev,
-    const std::string & name)
-{
-    if (evalSettings.pureEval && rev == "")
-        throw Error("in pure evaluation mode, 'fetchGit' requires a Git revision");
-
-    if (!ref && rev == "" && hasPrefix(uri, "/") && pathExists(uri + "/.git")) {
-
-        bool clean = true;
-
-        try {
-            runProgram("git", true, { "-C", uri, "diff-index", "--quiet", "HEAD", "--" });
-        } catch (ExecError & e) {
-            if (!WIFEXITED(e.status) || WEXITSTATUS(e.status) != 1) throw;
-            clean = false;
-        }
-
-        if (!clean) {
-
-            /* This is an unclean working tree. So copy all tracked files. */
-            GitInfo gitInfo;
-            gitInfo.rev = "0000000000000000000000000000000000000000";
-            gitInfo.shortRev = std::string(gitInfo.rev, 0, 7);
-
-            auto files = tokenizeString<std::set<std::string>>(
-                runProgram("git", true, { "-C", uri, "ls-files", "-z" }), "\0"s);
-
-            PathFilter filter = [&](const Path & p) -> bool {
-                assert(hasPrefix(p, uri));
-                std::string file(p, uri.size() + 1);
-
-                auto st = lstat(p);
-
-                if (S_ISDIR(st.st_mode)) {
-                    auto prefix = file + "/";
-                    auto i = files.lower_bound(prefix);
-                    return i != files.end() && hasPrefix(*i, prefix);
-                }
-
-                return files.count(file);
-            };
-
-            gitInfo.storePath = store->printStorePath(store->addToStore("source", uri, true, htSHA256, filter));
-
-            return gitInfo;
-        }
-
-        // clean working tree, but no ref or rev specified.  Use 'HEAD'.
-        rev = chomp(runProgram("git", true, { "-C", uri, "rev-parse", "HEAD" }));
-        ref = "HEAD"s;
-    }
-
-    if (!ref) ref = "HEAD"s;
-
-    if (rev != "" && !std::regex_match(rev, revRegex))
-        throw Error("invalid Git revision '%s'", rev);
-
-    deletePath(getCacheDir() + "/nix/git");
-
-    Path cacheDir = getCacheDir() + "/nix/gitv2/" + hashString(htSHA256, uri).to_string(Base32, false);
-
-    if (!pathExists(cacheDir)) {
-        createDirs(dirOf(cacheDir));
-        runProgram("git", true, { "init", "--bare", cacheDir });
-    }
-
-    Path localRefFile;
-    if (ref->compare(0, 5, "refs/") == 0)
-        localRefFile = cacheDir + "/" + *ref;
-    else
-        localRefFile = cacheDir + "/refs/heads/" + *ref;
-
-    bool doFetch;
-    time_t now = time(0);
-    /* If a rev was specified, we need to fetch if it's not in the
-       repo. */
-    if (rev != "") {
-        try {
-            runProgram("git", true, { "-C", cacheDir, "cat-file", "-e", rev });
-            doFetch = false;
-        } catch (ExecError & e) {
-            if (WIFEXITED(e.status)) {
-                doFetch = true;
-            } else {
-                throw;
-            }
-        }
-    } else {
-        /* If the local ref is older than ‘tarball-ttl’ seconds, do a
-           git fetch to update the local ref to the remote ref. */
-        struct stat st;
-        doFetch = stat(localRefFile.c_str(), &st) != 0 ||
-            (uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now;
-    }
-    if (doFetch)
-    {
-        Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Git repository '%s'", uri));
-
-        // FIXME: git stderr messes up our progress indicator, so
-        // we're using --quiet for now. Should process its stderr.
-        runProgram("git", true, { "-C", cacheDir, "fetch", "--quiet", "--force", "--", uri, fmt("%s:%s", *ref, *ref) });
-
-        struct timeval times[2];
-        times[0].tv_sec = now;
-        times[0].tv_usec = 0;
-        times[1].tv_sec = now;
-        times[1].tv_usec = 0;
-
-        utimes(localRefFile.c_str(), times);
-    }
-
-    // FIXME: check whether rev is an ancestor of ref.
-    GitInfo gitInfo;
-    gitInfo.rev = rev != "" ? rev : chomp(readFile(localRefFile));
-    gitInfo.shortRev = std::string(gitInfo.rev, 0, 7);
-
-    printTalkative("using revision %s of repo '%s'", gitInfo.rev, uri);
-
-    std::string storeLinkName = hashString(htSHA512, name + std::string("\0"s) + gitInfo.rev).to_string(Base32, false);
-    Path storeLink = cacheDir + "/" + storeLinkName + ".link";
-    PathLocks storeLinkLock({storeLink}, fmt("waiting for lock on '%1%'...", storeLink)); // FIXME: broken
-
-    try {
-        auto json = nlohmann::json::parse(readFile(storeLink));
-
-        assert(json["name"] == name && json["rev"] == gitInfo.rev);
-
-        gitInfo.storePath = json["storePath"];
-
-        if (store->isValidPath(store->parseStorePath(gitInfo.storePath))) {
-            gitInfo.revCount = json["revCount"];
-            return gitInfo;
-        }
-
-    } catch (SysError & e) {
-        if (e.errNo != ENOENT) throw;
-    }
-
-    auto source = sinkToSource([&](Sink & sink) {
-        RunOptions gitOptions("git", { "-C", cacheDir, "archive", gitInfo.rev });
-        gitOptions.standardOut = &sink;
-        runProgram2(gitOptions);
-    });
-
-    Path tmpDir = createTempDir();
-    AutoDelete delTmpDir(tmpDir, true);
-
-    unpackTarfile(*source, tmpDir);
-
-    gitInfo.storePath = store->printStorePath(store->addToStore(name, tmpDir));
-
-    gitInfo.revCount = std::stoull(runProgram("git", true, { "-C", cacheDir, "rev-list", "--count", gitInfo.rev }));
-
-    nlohmann::json json;
-    json["storePath"] = gitInfo.storePath;
-    json["uri"] = uri;
-    json["name"] = name;
-    json["rev"] = gitInfo.rev;
-    json["revCount"] = gitInfo.revCount;
-
-    writeFile(storeLink, json.dump());
-
-    return gitInfo;
-}
-
 static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     std::string url;
     std::optional<std::string> ref;
-    std::string rev;
+    std::optional<Hash> rev;
     std::string name = "source";
+    bool fetchSubmodules = false;
     PathSet context;
 
     state.forceValue(*args[0]);
@@ -213,15 +29,23 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va
             else if (n == "ref")
                 ref = state.forceStringNoCtx(*attr.value, *attr.pos);
             else if (n == "rev")
-                rev = state.forceStringNoCtx(*attr.value, *attr.pos);
+                rev = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA1);
             else if (n == "name")
                 name = state.forceStringNoCtx(*attr.value, *attr.pos);
+            else if (n == "submodules")
+                fetchSubmodules = state.forceBool(*attr.value, *attr.pos);
             else
-                throw EvalError("unsupported argument '%s' to 'fetchGit', at %s", attr.name, *attr.pos);
+                throw EvalError({
+                    .hint = hintfmt("unsupported argument '%s' to 'fetchGit'", attr.name),
+                    .errPos = *attr.pos
+                });
         }
 
         if (url.empty())
-            throw EvalError(format("'url' argument required, at %1%") % pos);
+            throw EvalError({
+                .hint = hintfmt("'url' argument required"),
+                .errPos = pos
+            });
 
     } else
         url = state.coerceToString(pos, *args[0], context, false, false);
@@ -230,17 +54,36 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va
     // whitelist. Ah well.
     state.checkURI(url);
 
-    auto gitInfo = exportGit(state.store, url, ref, rev, name);
+    if (evalSettings.pureEval && !rev)
+        throw Error("in pure evaluation mode, 'fetchGit' requires a Git revision");
+
+    fetchers::Attrs attrs;
+    attrs.insert_or_assign("type", "git");
+    attrs.insert_or_assign("url", url.find("://") != std::string::npos ? url : "file://" + url);
+    if (ref) attrs.insert_or_assign("ref", *ref);
+    if (rev) attrs.insert_or_assign("rev", rev->gitRev());
+    if (fetchSubmodules) attrs.insert_or_assign("submodules", true);
+    auto input = fetchers::inputFromAttrs(attrs);
+
+    // FIXME: use name?
+    auto [tree, input2] = input->fetchTree(state.store);
 
     state.mkAttrs(v, 8);
-    mkString(*state.allocAttr(v, state.sOutPath), gitInfo.storePath, PathSet({gitInfo.storePath}));
-    mkString(*state.allocAttr(v, state.symbols.create("rev")), gitInfo.rev);
-    mkString(*state.allocAttr(v, state.symbols.create("shortRev")), gitInfo.shortRev);
-    mkInt(*state.allocAttr(v, state.symbols.create("revCount")), gitInfo.revCount);
+    auto storePath = state.store->printStorePath(tree.storePath);
+    mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath}));
+    // Backward compatibility: set 'rev' to
+    // 0000000000000000000000000000000000000000 for a dirty tree.
+    auto rev2 = input2->getRev().value_or(Hash(htSHA1));
+    mkString(*state.allocAttr(v, state.symbols.create("rev")), rev2.gitRev());
+    mkString(*state.allocAttr(v, state.symbols.create("shortRev")), rev2.gitShortRev());
+    // Backward compatibility: set 'revCount' to 0 for a dirty tree.
+    mkInt(*state.allocAttr(v, state.symbols.create("revCount")),
+        tree.info.revCount.value_or(0));
+    mkBool(*state.allocAttr(v, state.symbols.create("submodules")), fetchSubmodules);
     v.attrs->sort();
 
     if (state.allowedPaths)
-        state.allowedPaths->insert(state.store->toRealPath(gitInfo.storePath));
+        state.allowedPaths->insert(tree.actualPath);
 }
 
 static RegisterPrimOp r("fetchGit", 1, prim_fetchGit);

src/libexpr/primops/fetchMercurial.cc

@@ -1,174 +1,18 @@
 #include "primops.hh"
 #include "eval-inline.hh"
-#include "download.hh"
 #include "store-api.hh"
-#include "pathlocks.hh"
-
-#include <sys/time.h>
+#include "fetchers.hh"
+#include "url.hh"
 
 #include <regex>
 
-#include <nlohmann/json.hpp>
-
-using namespace std::string_literals;
-
 namespace nix {
 
-struct HgInfo
-{
-    Path storePath;
-    std::string branch;
-    std::string rev;
-    uint64_t revCount = 0;
-};
-
-std::regex commitHashRegex("^[0-9a-fA-F]{40}$");
-
-HgInfo exportMercurial(ref<Store> store, const std::string & uri,
-    std::string rev, const std::string & name)
-{
-    if (evalSettings.pureEval && rev == "")
-        throw Error("in pure evaluation mode, 'fetchMercurial' requires a Mercurial revision");
-
-    if (rev == "" && hasPrefix(uri, "/") && pathExists(uri + "/.hg")) {
-
-        bool clean = runProgram("hg", true, { "status", "-R", uri, "--modified", "--added", "--removed" }) == "";
-
-        if (!clean) {
-
-            /* This is an unclean working tree. So copy all tracked
-               files. */
-
-            printTalkative("copying unclean Mercurial working tree '%s'", uri);
-
-            HgInfo hgInfo;
-            hgInfo.rev = "0000000000000000000000000000000000000000";
-            hgInfo.branch = chomp(runProgram("hg", true, { "branch", "-R", uri }));
-
-            auto files = tokenizeString<std::set<std::string>>(
-                runProgram("hg", true, { "status", "-R", uri, "--clean", "--modified", "--added", "--no-status", "--print0" }), "\0"s);
-
-            PathFilter filter = [&](const Path & p) -> bool {
-                assert(hasPrefix(p, uri));
-                std::string file(p, uri.size() + 1);
-
-                auto st = lstat(p);
-
-                if (S_ISDIR(st.st_mode)) {
-                    auto prefix = file + "/";
-                    auto i = files.lower_bound(prefix);
-                    return i != files.end() && hasPrefix(*i, prefix);
-                }
-
-                return files.count(file);
-            };
-
-            hgInfo.storePath = store->printStorePath(store->addToStore("source", uri, true, htSHA256, filter));
-
-            return hgInfo;
-        }
-    }
-
-    if (rev == "") rev = "default";
-
-    Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), hashString(htSHA256, uri).to_string(Base32, false));
-
-    Path stampFile = fmt("%s/.hg/%s.stamp", cacheDir, hashString(htSHA512, rev).to_string(Base32, false));
-
-    /* If we haven't pulled this repo less than ‘tarball-ttl’ seconds,
-       do so now. */
-    time_t now = time(0);
-    struct stat st;
-    if (stat(stampFile.c_str(), &st) != 0 ||
-        (uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now)
-    {
-        /* Except that if this is a commit hash that we already have,
-           we don't have to pull again. */
-        if (!(std::regex_match(rev, commitHashRegex)
-                && pathExists(cacheDir)
-                && runProgram(
-                    RunOptions("hg", { "log", "-R", cacheDir, "-r", rev, "--template", "1" })
-                    .killStderr(true)).second == "1"))
-        {
-            Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Mercurial repository '%s'", uri));
-
-            if (pathExists(cacheDir)) {
-                try {
-                    runProgram("hg", true, { "pull", "-R", cacheDir, "--", uri });
-                }
-                catch (ExecError & e) {
-                    string transJournal = cacheDir + "/.hg/store/journal";
-                    /* hg throws "abandoned transaction" error only if this file exists */
-                    if (pathExists(transJournal)) {
-                        runProgram("hg", true, { "recover", "-R", cacheDir });
-                        runProgram("hg", true, { "pull", "-R", cacheDir, "--", uri });
-                    } else {
-                        throw ExecError(e.status, fmt("'hg pull' %s", statusToString(e.status)));
-                    }
-                }
-            } else {
-                createDirs(dirOf(cacheDir));
-                runProgram("hg", true, { "clone", "--noupdate", "--", uri, cacheDir });
-            }
-        }
-
-        writeFile(stampFile, "");
-    }
-
-    auto tokens = tokenizeString<std::vector<std::string>>(
-        runProgram("hg", true, { "log", "-R", cacheDir, "-r", rev, "--template", "{node} {rev} {branch}" }));
-    assert(tokens.size() == 3);
-
-    HgInfo hgInfo;
-    hgInfo.rev = tokens[0];
-    hgInfo.revCount = std::stoull(tokens[1]);
-    hgInfo.branch = tokens[2];
-
-    std::string storeLinkName = hashString(htSHA512, name + std::string("\0"s) + hgInfo.rev).to_string(Base32, false);
-    Path storeLink = fmt("%s/.hg/%s.link", cacheDir, storeLinkName);
-
-    try {
-        auto json = nlohmann::json::parse(readFile(storeLink));
-
-        assert(json["name"] == name && json["rev"] == hgInfo.rev);
-
-        hgInfo.storePath = json["storePath"];
-
-        if (store->isValidPath(store->parseStorePath(hgInfo.storePath))) {
-            printTalkative("using cached Mercurial store path '%s'", hgInfo.storePath);
-            return hgInfo;
-        }
-
-    } catch (SysError & e) {
-        if (e.errNo != ENOENT) throw;
-    }
-
-    Path tmpDir = createTempDir();
-    AutoDelete delTmpDir(tmpDir, true);
-
-    runProgram("hg", true, { "archive", "-R", cacheDir, "-r", rev, tmpDir });
-
-    deletePath(tmpDir + "/.hg_archival.txt");
-
-    hgInfo.storePath = store->printStorePath(store->addToStore(name, tmpDir));
-
-    nlohmann::json json;
-    json["storePath"] = hgInfo.storePath;
-    json["uri"] = uri;
-    json["name"] = name;
-    json["branch"] = hgInfo.branch;
-    json["rev"] = hgInfo.rev;
-    json["revCount"] = hgInfo.revCount;
-
-    writeFile(storeLink, json.dump());
-
-    return hgInfo;
-}
-
 static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     std::string url;
-    std::string rev;
+    std::optional<Hash> rev;
+    std::optional<std::string> ref;
     std::string name = "source";
     PathSet context;
 
@@ -182,16 +26,29 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar
             string n(attr.name);
             if (n == "url")
                 url = state.coerceToString(*attr.pos, *attr.value, context, false, false);
-            else if (n == "rev")
-                rev = state.forceStringNoCtx(*attr.value, *attr.pos);
+            else if (n == "rev") {
+                // Ugly: unlike fetchGit, here the "rev" attribute can
+                // be both a revision or a branch/tag name.
+                auto value = state.forceStringNoCtx(*attr.value, *attr.pos);
+                if (std::regex_match(value, revRegex))
+                    rev = Hash(value, htSHA1);
+                else
+                    ref = value;
+            }
             else if (n == "name")
                 name = state.forceStringNoCtx(*attr.value, *attr.pos);
             else
-                throw EvalError("unsupported argument '%s' to 'fetchMercurial', at %s", attr.name, *attr.pos);
+                throw EvalError({
+                    .hint = hintfmt("unsupported argument '%s' to 'fetchMercurial'", attr.name),
+                    .errPos = *attr.pos
+                });
         }
 
         if (url.empty())
-            throw EvalError(format("'url' argument required, at %1%") % pos);
+            throw EvalError({
+                .hint = hintfmt("'url' argument required"),
+                .errPos = pos
+            });
 
     } else
         url = state.coerceToString(pos, *args[0], context, false, false);
@@ -200,18 +57,35 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar
     // whitelist. Ah well.
     state.checkURI(url);
 
-    auto hgInfo = exportMercurial(state.store, url, rev, name);
+    if (evalSettings.pureEval && !rev)
+        throw Error("in pure evaluation mode, 'fetchMercurial' requires a Mercurial revision");
+
+    fetchers::Attrs attrs;
+    attrs.insert_or_assign("type", "hg");
+    attrs.insert_or_assign("url", url.find("://") != std::string::npos ? url : "file://" + url);
+    if (ref) attrs.insert_or_assign("ref", *ref);
+    if (rev) attrs.insert_or_assign("rev", rev->gitRev());
+    auto input = fetchers::inputFromAttrs(attrs);
+
+    // FIXME: use name
+    auto [tree, input2] = input->fetchTree(state.store);
 
     state.mkAttrs(v, 8);
-    mkString(*state.allocAttr(v, state.sOutPath), hgInfo.storePath, PathSet({hgInfo.storePath}));
-    mkString(*state.allocAttr(v, state.symbols.create("branch")), hgInfo.branch);
-    mkString(*state.allocAttr(v, state.symbols.create("rev")), hgInfo.rev);
-    mkString(*state.allocAttr(v, state.symbols.create("shortRev")), std::string(hgInfo.rev, 0, 12));
-    mkInt(*state.allocAttr(v, state.symbols.create("revCount")), hgInfo.revCount);
+    auto storePath = state.store->printStorePath(tree.storePath);
+    mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath}));
+    if (input2->getRef())
+        mkString(*state.allocAttr(v, state.symbols.create("branch")), *input2->getRef());
+    // Backward compatibility: set 'rev' to
+    // 0000000000000000000000000000000000000000 for a dirty tree.
+    auto rev2 = input2->getRev().value_or(Hash(htSHA1));
+    mkString(*state.allocAttr(v, state.symbols.create("rev")), rev2.gitRev());
+    mkString(*state.allocAttr(v, state.symbols.create("shortRev")), std::string(rev2.gitRev(), 0, 12));
+    if (tree.info.revCount)
+        mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *tree.info.revCount);
     v.attrs->sort();
 
     if (state.allowedPaths)
-        state.allowedPaths->insert(state.store->toRealPath(hgInfo.storePath));
+        state.allowedPaths->insert(tree.actualPath);
 }
 
 static RegisterPrimOp r("fetchMercurial", 1, prim_fetchMercurial);

src/libexpr/primops/fetchTree.cc

@@ -0,0 +1,172 @@
+#include "primops.hh"
+#include "eval-inline.hh"
+#include "store-api.hh"
+#include "fetchers.hh"
+#include "filetransfer.hh"
+
+#include <ctime>
+#include <iomanip>
+
+namespace nix {
+
+void emitTreeAttrs(
+    EvalState & state,
+    const fetchers::Tree & tree,
+    std::shared_ptr<const fetchers::Input> input,
+    Value & v)
+{
+    state.mkAttrs(v, 8);
+
+    auto storePath = state.store->printStorePath(tree.storePath);
+
+    mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath}));
+
+    assert(tree.info.narHash);
+    mkString(*state.allocAttr(v, state.symbols.create("narHash")),
+        tree.info.narHash.to_string(SRI, true));
+
+    if (input->getRev()) {
+        mkString(*state.allocAttr(v, state.symbols.create("rev")), input->getRev()->gitRev());
+        mkString(*state.allocAttr(v, state.symbols.create("shortRev")), input->getRev()->gitShortRev());
+    }
+
+    if (tree.info.revCount)
+        mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *tree.info.revCount);
+
+    if (tree.info.lastModified)
+        mkString(*state.allocAttr(v, state.symbols.create("lastModified")),
+            fmt("%s", std::put_time(std::gmtime(&*tree.info.lastModified), "%Y%m%d%H%M%S")));
+
+    v.attrs->sort();
+}
+
+static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    settings.requireExperimentalFeature("flakes");
+
+    std::shared_ptr<const fetchers::Input> input;
+    PathSet context;
+
+    state.forceValue(*args[0]);
+
+    if (args[0]->type == tAttrs) {
+        state.forceAttrs(*args[0], pos);
+
+        fetchers::Attrs attrs;
+
+        for (auto & attr : *args[0]->attrs) {
+            state.forceValue(*attr.value);
+            if (attr.value->type == tString)
+                attrs.emplace(attr.name, attr.value->string.s);
+            else if (attr.value->type == tBool)
+                attrs.emplace(attr.name, attr.value->boolean);
+            else
+                throw TypeError("fetchTree argument '%s' is %s while a string or Boolean is expected",
+                    attr.name, showType(*attr.value));
+        }
+
+        if (!attrs.count("type"))
+            throw Error({
+                .hint = hintfmt("attribute 'type' is missing in call to 'fetchTree'"),
+                .errPos = pos
+            });
+
+        input = fetchers::inputFromAttrs(attrs);
+    } else
+        input = fetchers::inputFromURL(state.coerceToString(pos, *args[0], context, false, false));
+
+    if (evalSettings.pureEval && !input->isImmutable())
+        throw Error("in pure evaluation mode, 'fetchTree' requires an immutable input");
+
+    // FIXME: use fetchOrSubstituteTree
+    auto [tree, input2] = input->fetchTree(state.store);
+
+    if (state.allowedPaths)
+        state.allowedPaths->insert(tree.actualPath);
+
+    emitTreeAttrs(state, tree, input2, v);
+}
+
+static RegisterPrimOp r("fetchTree", 1, prim_fetchTree);
+
+static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
+    const string & who, bool unpack, std::string name)
+{
+    std::optional<std::string> url;
+    std::optional<Hash> expectedHash;
+
+    state.forceValue(*args[0]);
+
+    if (args[0]->type == tAttrs) {
+
+        state.forceAttrs(*args[0], pos);
+
+        for (auto & attr : *args[0]->attrs) {
+            string n(attr.name);
+            if (n == "url")
+                url = state.forceStringNoCtx(*attr.value, *attr.pos);
+            else if (n == "sha256")
+                expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256);
+            else if (n == "name")
+                name = state.forceStringNoCtx(*attr.value, *attr.pos);
+            else
+                throw EvalError({
+                    .hint = hintfmt("unsupported argument '%s' to '%s'", attr.name, who),
+                    .errPos = *attr.pos
+                });
+            }
+
+        if (!url)
+            throw EvalError({
+                .hint = hintfmt("'url' argument required"),
+                .errPos = pos
+            });
+    } else
+        url = state.forceStringNoCtx(*args[0], pos);
+
+    url = resolveUri(*url);
+
+    state.checkURI(*url);
+
+    if (name == "")
+        name = baseNameOf(*url);
+
+    if (evalSettings.pureEval && !expectedHash)
+        throw Error("in pure evaluation mode, '%s' requires a 'sha256' argument", who);
+
+    auto storePath =
+        unpack
+        ? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).storePath
+        : fetchers::downloadFile(state.store, *url, name, (bool) expectedHash).storePath;
+
+    auto path = state.store->toRealPath(storePath);
+
+    if (expectedHash) {
+        auto hash = unpack
+            ? state.store->queryPathInfo(storePath)->narHash
+            : hashFile(htSHA256, path);
+        if (hash != *expectedHash)
+            throw Error((unsigned int) 102, "hash mismatch in file downloaded from '%s':\n  wanted: %s\n  got:    %s",
+                *url, expectedHash->to_string(Base32, true), hash.to_string(Base32, true));
+    }
+
+    if (state.allowedPaths)
+        state.allowedPaths->insert(path);
+
+    mkString(v, path, PathSet({path}));
+}
+
+static void prim_fetchurl(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    fetch(state, pos, args, v, "fetchurl", false, "");
+}
+
+static void prim_fetchTarball(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    fetch(state, pos, args, v, "fetchTarball", true, "source");
+}
+
+static RegisterPrimOp r2("__fetchurl", 1, prim_fetchurl);
+static RegisterPrimOp r3("fetchTarball", 1, prim_fetchTarball);
+
+}

src/libexpr/primops/fromTOML.cc

@@ -1,7 +1,7 @@
 #include "primops.hh"
 #include "eval-inline.hh"
 
-#include "cpptoml/cpptoml.h"
+#include "../../cpptoml/cpptoml.h"
 
 namespace nix {
 
@@ -81,7 +81,10 @@ static void prim_fromTOML(EvalState & state, const Pos & pos, Value * * args, Va
     try {
         visit(v, parser(tomlStream).parse());
     } catch (std::runtime_error & e) {
-        throw EvalError("while parsing a TOML string at %s: %s", pos, e.what());
+        throw EvalError({
+            .hint = hintfmt("while parsing a TOML string: %s", e.what()),
+            .errPos = pos
+        });
     }
 }
 

src/libexpr/symbol-table.hh

@@ -19,7 +19,6 @@ private:
     const string * s; // pointer into SymbolTable
     Symbol(const string * s) : s(s) { };
     friend class SymbolTable;
-    friend class Value;
 
 public:
     Symbol() : s(0) { };
@@ -90,6 +89,4 @@ public:
     }
 };
 
-extern SymbolTable symbols;
-
 }