nix eval-hydra-jobs: Support parallel evaluation

Example usage:

  $ nix eval-hydra-jobs -f '<nixpkgs/pkgs/top-level/release.nix>' '' \
      --max-memory-size 2048 --workers 8

.dir-locals.el

@@ -13,5 +13,4 @@
   (eval . (c-set-offset 'arglist-cont-nonempty '+))
   (eval . (c-set-offset 'substatement-open 0))
   (eval . (c-set-offset 'access-label '-))
-  (eval . (c-set-offset 'inlambda 0))
   )))

.github/ISSUE_TEMPLATE.md

@@ -0,0 +1,27 @@
+<!--
+
+# Filing a Nix issue
+
+*WAIT* Are you sure you're filing your issue in the right repository?
+
+We appreciate you taking the time to tell us about issues you encounter, but routing the issue to the right place will get you help sooner and save everyone time.
+
+This is the Nix repository, and issues here should be about Nix the build and package management *_tool_*.
+
+If you have a problem with a specific package on NixOS or when using Nix, you probably want to file an issue with _nixpkgs_, whose issue tracker is over at https://github.com/NixOS/nixpkgs/issues.
+
+Examples of _Nix_ issues:
+
+- Nix segfaults when I run `nix-build -A blahblah`
+- The Nix language needs a new builtin: `builtins.foobar`
+- Regression in the behavior of `nix-env` in Nix 2.0
+
+Examples of _nixpkgs_ issues:
+
+- glibc is b0rked on aarch64
+- chromium in NixOS doesn't support U2F but google-chrome does!
+- The OpenJDK package on macOS is missing a key component
+
+Chances are if you're a newcomer to the Nix world, you'll probably want the [nixpkgs tracker](https://github.com/NixOS/nixpkgs/issues). It also gets a lot more eyeball traffic so you'll probably get a response a lot more quickly.
+
+-->

.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,32 +0,0 @@
----
-name: Bug report
-about: Create a report to help us improve
-title: ''
-labels: bug
-assignees: ''
-
----
-
-**Describe the bug**
-
-A clear and concise description of what the bug is.
-
-If you have a problem with a specific package or NixOS,
-you probably want to file an issue at https://github.com/NixOS/nixpkgs/issues. 
-
-**Steps To Reproduce**
-
-1. Go to '...'
-2. Click on '....'
-3. Scroll down to '....'
-4. See error
-
-**Expected behavior**
-
-A clear and concise description of what you expected to happen.
-
-**`nix-env --version` output**
-
-**Additional context**
-
-Add any other context about the problem here.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -1,20 +0,0 @@
----
-name: Feature request
-about: Suggest an idea for this project
-title: ''
-labels: improvement
-assignees: ''
-
----
-
-**Is your feature request related to a problem? Please describe.**
-A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
-
-**Describe the solution you'd like**
-A clear and concise description of what you want to happen.
-
-**Describe alternatives you've considered**
-A clear and concise description of any alternative solutions or features you've considered.
-
-**Additional context**
-Add any other context or screenshots about the feature request here.

.github/workflows/test.yml

@@ -1,17 +0,0 @@
-name: "Test"
-on:
-  pull_request:
-  push:
-jobs:
-  tests:
-    strategy:
-      matrix:
-        os: [ubuntu-18.04, macos]
-    runs-on: ${{ matrix.os }}
-    steps:
-    - uses: actions/checkout@v2
-      with:
-        fetch-depth: 0
-    - uses: cachix/install-nix-action@v8
-    #- run: nix flake check
-    - run: nix-build -A checks.$(if [[ `uname` = Linux ]]; then echo x86_64-linux; else echo x86_64-darwin; fi)

.gitignore

@@ -47,7 +47,7 @@ perl/Makefile.config
 /src/libexpr/nix.tbl
 
 # /src/libstore/
-*.gen.*
+/src/libstore/*.gen.hh
 
 /src/nix/nix
 
@@ -75,8 +75,6 @@ perl/Makefile.config
 
 /src/nix-copy-closure/nix-copy-closure
 
-/src/error-demo/error-demo
-
 /src/build-remote/build-remote
 
 # /tests/

.travis.yml

@@ -0,0 +1,8 @@
+matrix:
+  include:
+    - language: osx
+      script: ./tests/install-darwin.sh
+    - language: nix
+      script: nix-build release.nix -A build.x86_64-linux
+notifications:
+  email: false

Makefile

@@ -4,14 +4,12 @@ makefiles = \
   nix-rust/local.mk \
   src/libutil/local.mk \
   src/libstore/local.mk \
-  src/libfetchers/local.mk \
   src/libmain/local.mk \
   src/libexpr/local.mk \
   src/nix/local.mk \
   src/resolve-system-dependencies/local.mk \
   scripts/local.mk \
   corepkgs/local.mk \
-  misc/bash/local.mk \
   misc/systemd/local.mk \
   misc/launchd/local.mk \
   misc/upstart/local.mk \

Makefile.config.in

@@ -1,5 +1,4 @@
 AR = @AR@
-BDW_GC_LIBS = @BDW_GC_LIBS@
 BUILD_SHARED_LIBS = @BUILD_SHARED_LIBS@
 CC = @CC@
 CFLAGS = @CFLAGS@
@@ -37,7 +36,6 @@ prefix = @prefix@
 sandbox_shell = @sandbox_shell@
 storedir = @storedir@
 sysconfdir = @sysconfdir@
-system = @system@
 doc_generate = @doc_generate@
 xmllint = @xmllint@
 xsltproc = @xsltproc@

README.md

@@ -1,54 +1,24 @@
-# Nix
-
 [![Open Collective supporters](https://opencollective.com/nixos/tiers/supporter/badge.svg?label=Supporters&color=brightgreen)](https://opencollective.com/nixos)
-[![Test](https://github.com/NixOS/nix/workflows/Test/badge.svg)](https://github.com/NixOS/nix/actions)
 
-Nix is a powerful package manager for Linux and other Unix systems that makes package
-management reliable and reproducible. Please refer to the [Nix manual](https://nixos.org/nix/manual)
-for more details.
+Nix, the purely functional package manager
+------------------------------------------
 
-## Installation
+Nix is a new take on package management that is fairly unique. Because of its
+purity aspects, a lot of issues found in traditional package managers don't
+appear with Nix.
 
-On Linux and macOS the easiest way to Install Nix is to run the following shell command
-(as a user other than root):
+To find out more about the tool, usage and installation instructions, please
+read the manual, which is available on the Nix website at
+<http://nixos.org/nix/manual>.
 
-```
-$ curl -L https://nixos.org/nix/install | sh
-```
+## Contributing
 
-Information on additional installation methods is available on the [Nix download page](https://nixos.org/download.html).
-
-## Building And Developing
-
-### Building Nix
-
-You can build Nix using one of the targets provided by [release.nix](./release.nix):
-
-```
-$ nix-build ./release.nix -A build.aarch64-linux
-$ nix-build ./release.nix -A build.x86_64-darwin
-$ nix-build ./release.nix -A build.i686-linux
-$ nix-build ./release.nix -A build.x86_64-linux
-```
-
-### Development Environment
-
-You can use the provided `shell.nix` to get a working development environment:
-
-```
-$ nix-shell
-$ ./bootstrap.sh
-$ ./configure
-$ make
-```
-
-## Additional Resources
-
-- [Nix manual](https://nixos.org/nix/manual)
-- [Nix jobsets on hydra.nixos.org](https://hydra.nixos.org/project/nix)
-- [NixOS Discourse](https://discourse.nixos.org/)
-- [IRC - #nixos on freenode.net](irc://irc.freenode.net/#nixos)
+Take a look at the [Hacking Section](http://nixos.org/nix/manual/#chap-hacking)
+of the manual. It helps you to get started with building Nix from source.
 
 ## License
 
-Nix is released under the [LGPL v2.1](./COPYING).
+Nix is released under the LGPL v2.1
+
+This product includes software developed by the OpenSSL Project for
+use in the [OpenSSL Toolkit](http://www.OpenSSL.org/).

configure.ac

@@ -123,7 +123,6 @@ AC_PATH_PROG(flex, flex, false)
 AC_PATH_PROG(bison, bison, false)
 AC_PATH_PROG(dot, dot)
 AC_PATH_PROG(lsof, lsof, lsof)
-NEED_PROG(jq, jq)
 
 
 AC_SUBST(coreutils, [$(dirname $(type -p cat))])
@@ -256,17 +255,6 @@ if test -n "$enable_s3"; then
 fi
 
 
-# Whether to use the Boehm garbage collector.
-AC_ARG_ENABLE(gc, AC_HELP_STRING([--enable-gc],
-  [enable garbage collection in the Nix expression evaluator (requires Boehm GC) [default=yes]]),
-  gc=$enableval, gc=yes)
-if test "$gc" = yes; then
-  PKG_CHECK_MODULES([BDW_GC], [bdw-gc])
-  CXXFLAGS="$BDW_GC_CFLAGS $CXXFLAGS"
-  AC_DEFINE(HAVE_BOEHMGC, 1, [Whether to use the Boehm garbage collector.])
-fi
-
-
 # documentation generation switch
 AC_ARG_ENABLE(doc-gen, AC_HELP_STRING([--disable-doc-gen],
   [disable documentation generation]),

corepkgs/local.mk

@@ -1,7 +1,4 @@
-corepkgs_FILES = \
-  unpack-channel.nix \
-  derivation.nix \
-  fetchurl.nix
+corepkgs_FILES = buildenv.nix unpack-channel.nix derivation.nix fetchurl.nix imported-drv-to-derivation.nix
 
 $(foreach file,config.nix $(corepkgs_FILES),$(eval $(call install-data-in,$(d)/$(file),$(datadir)/nix/corepkgs)))
 

default.nix

@@ -1,3 +0,0 @@
-(import (fetchTarball https://github.com/edolstra/flake-compat/archive/master.tar.gz) {
-  src = builtins.fetchGit ./.;
-}).defaultNix

doc/manual/command-ref/conf-file.xml

@@ -19,30 +19,26 @@
 
 <refsection><title>Description</title>
 
-<para>By default Nix reads settings from the following places:</para>
+<para>Nix reads settings from two configuration files:</para>
 
-<para>The system-wide configuration file
-<filename><replaceable>sysconfdir</replaceable>/nix/nix.conf</filename>
-(i.e. <filename>/etc/nix/nix.conf</filename> on most systems), or
-<filename>$NIX_CONF_DIR/nix.conf</filename> if
-<envar>NIX_CONF_DIR</envar> is set. Values loaded in this file are not forwarded to the Nix daemon. The
-client assumes that the daemon has already loaded them.
-</para>
+<itemizedlist>
 
-<para>User-specific configuration files:</para>
+  <listitem>
+    <para>The system-wide configuration file
+    <filename><replaceable>sysconfdir</replaceable>/nix/nix.conf</filename>
+    (i.e. <filename>/etc/nix/nix.conf</filename> on most systems), or
+    <filename>$NIX_CONF_DIR/nix.conf</filename> if
+    <envar>NIX_CONF_DIR</envar> is set.</para>
+  </listitem>
 
-<para>
-  If <envar>NIX_USER_CONF_FILES</envar> is set, then each path separated by
-  <literal>:</literal> will be loaded in reverse order.
-</para>
+  <listitem>
+    <para>The user configuration file
+    <filename>$XDG_CONFIG_HOME/nix/nix.conf</filename>, or
+    <filename>~/.config/nix/nix.conf</filename> if
+    <envar>XDG_CONFIG_HOME</envar> is not set.</para>
+  </listitem>
 
-<para>
-  Otherwise it will look for <filename>nix/nix.conf</filename> files in
-  <envar>XDG_CONFIG_DIRS</envar> and <envar>XDG_CONFIG_HOME</envar>.
-
-  The default location is <filename>$HOME/.config/nix.conf</filename> if
-  those environment variables are unset.
-</para>
+</itemizedlist>
 
 <para>The configuration files consist of
 <literal><replaceable>name</replaceable> =

doc/manual/command-ref/env-common.xml

@@ -33,7 +33,7 @@
 
     will cause Nix to look for paths relative to
     <filename>/home/eelco/Dev</filename> and
-    <filename>/etc/nixos</filename>, in this order.  It is also
+    <filename>/etc/nixos</filename>, in that order.  It is also
     possible to match paths against a prefix.  For example, the value
 
     <screen>
@@ -59,7 +59,7 @@ nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-15.09.tar.gz</sc
     15.09 channel.</para>
 
     <para>A following shorthand can be used to refer to the official channels:
-
+    
     <screen>nixpkgs=channel:nixos-15.09</screen>
     </para>
 
@@ -137,19 +137,12 @@ $ mount -o bind /mnt/otherdisk/nix /nix</screen>
 
 <varlistentry><term><envar>NIX_CONF_DIR</envar></term>
 
-  <listitem><para>Overrides the location of the system Nix configuration
+  <listitem><para>Overrides the location of the Nix configuration
   directory (default
   <filename><replaceable>prefix</replaceable>/etc/nix</filename>).</para></listitem>
 
 </varlistentry>
 
-<varlistentry><term><envar>NIX_USER_CONF_FILES</envar></term>
-
-  <listitem><para>Overrides the location of the user Nix configuration files
-  to load from (defaults to the XDG spec locations). The variable is treated
-  as a list separated by the <literal>:</literal> token.</para></listitem>
-
-</varlistentry>
 
 <varlistentry><term><envar>TMPDIR</envar></term>
 

doc/manual/command-ref/nix-env.xml

@@ -1066,8 +1066,7 @@ user environment elements, etc. -->
     the derivation, which can be used to unambiguously select it using
     the <link linkend="opt-attr"><option>--attr</option> option</link>
     available in commands that install derivations like
-    <literal>nix-env --install</literal>. This option only works
-    together with <option>--available</option></para></listitem>
+    <literal>nix-env --install</literal>.</para></listitem>
 
   </varlistentry>
 

doc/manual/command-ref/nix-store.xml

@@ -360,6 +360,7 @@ EOF
     <arg choice='plain'><option>--print-roots</option></arg>
     <arg choice='plain'><option>--print-live</option></arg>
     <arg choice='plain'><option>--print-dead</option></arg>
+    <arg choice='plain'><option>--delete</option></arg>
   </group>
   <arg><option>--max-freed</option> <replaceable>bytes</replaceable></arg>
 </cmdsynopsis>
@@ -406,6 +407,14 @@ the Nix store not reachable via file system references from a set of
 
   </varlistentry>
 
+  <varlistentry><term><option>--delete</option></term>
+
+    <listitem><para>This operation performs an actual garbage
+    collection.  All dead paths are removed from the
+    store.  This is the default.</para></listitem>
+
+  </varlistentry>
+
 </variablelist>
 
 <para>By default, all unreachable paths are deleted.  The following
@@ -435,10 +444,10 @@ and <link
 linkend="conf-keep-derivations"><literal>keep-derivations</literal></link>
 variables in the Nix configuration file.</para>
 
-<para>By default, the collector prints the total number of freed bytes
-when it finishes (or when it is interrupted). With
-<option>--print-dead</option>, it prints the number of bytes that would
-be freed.</para>
+<para>With <option>--delete</option>, the collector prints the total
+number of freed bytes when it finishes (or when it is interrupted).
+With <option>--print-dead</option>, it prints the number of bytes that
+would be freed.</para>
 
 </refsection>
 
@@ -1139,7 +1148,7 @@ the information that Nix considers important.  For instance,
 timestamps are elided because all files in the Nix store have their
 timestamp set to 0 anyway.  Likewise, all permissions are left out
 except for the execute bit, because all files in the Nix store have
-444 or 555 permission.</para>
+644 or 755 permission.</para>
 
 <para>Also, a NAR archive is <emphasis>canonical</emphasis>, meaning
 that “equal” paths always produce the same NAR archive.  For instance,

doc/manual/expressions/builtins.xml

@@ -422,16 +422,6 @@ stdenv.mkDerivation { … }
             </para>
           </listitem>
         </varlistentry>
-        <varlistentry>
-          <term>submodules</term>
-          <listitem>
-            <para>
-              A Boolean parameter that specifies whether submodules
-              should be checked out. Defaults to
-              <literal>false</literal>.
-            </para>
-          </listitem>
-        </varlistentry>
       </variablelist>
 
       <example>

doc/manual/installation/installing-binary.xml

@@ -142,7 +142,7 @@ sudo rm /Library/LaunchDaemons/org.nixos.nix-daemon.plist
   <para>
     NixOS.org hosts version-specific installation URLs for all Nix
     versions since 1.11.16, at
-    <literal>https://releases.nixos.org/nix/nix-<replaceable>version</replaceable>/install</literal>.
+    <literal>https://nixos.org/releases/nix/nix-VERSION/install</literal>.
   </para>
 
   <para>

doc/manual/installation/prerequisites-source.xml

@@ -8,14 +8,6 @@
 
 <itemizedlist>
 
-  <listitem><para>GNU Autoconf
-  (<link xlink:href="https://www.gnu.org/software/autoconf/"/>)
-  and the autoconf-archive macro collection
-  (<link xlink:href="https://www.gnu.org/software/autoconf-archive/"/>).
-  These are only needed to run the bootstrap script, and are not necessary
-  if your source distribution came with a pre-built
-  <literal>./configure</literal> script.</para></listitem>
-
   <listitem><para>GNU Make.</para></listitem>
   
   <listitem><para>Bash Shell. The <literal>./configure</literal> script
@@ -58,14 +50,6 @@
   or higher.  If your distribution does not provide it, please install
   it from <link xlink:href="http://www.sqlite.org/" />.</para></listitem>
 
-  <listitem><para>The <link
-  xlink:href="http://www.hboehm.info/gc/">Boehm
-  garbage collector</link> to reduce the evaluator’s memory
-  consumption (optional).  To enable it, install
-  <literal>pkgconfig</literal> and the Boehm garbage collector, and
-  pass the flag <option>--enable-gc</option> to
-  <command>configure</command>.</para></listitem>
-
   <listitem><para>The <literal>boost</literal> library of version
   1.66.0 or higher. It can be obtained from the official web site
   <link xlink:href="https://www.boost.org/" />.</para></listitem>

doc/manual/local.mk

@@ -4,10 +4,11 @@ ifeq ($(doc_generate),yes)
 XSLTPROC = $(xsltproc) --nonet $(xmlflags) \
   --param section.autolabel 1 \
   --param section.label.includes.component.label 1 \
+  --param html.stylesheet \'style.css\' \
   --param xref.with.number.and.title 1 \
   --param toc.section.depth 3 \
   --param admon.style \'\' \
-  --param callout.graphics 0 \
+  --param callout.graphics.extension \'.gif\' \
   --param contrib.inline.enabled 0 \
   --stringparam generate.toc "book toc" \
   --param keep.relative.image.uris 0
@@ -65,10 +66,12 @@ $(d)/manual.html: $(d)/manual.xml $(MANUAL_SRCS) $(d)/manual.is-valid
 	  $(docbookxsl)/profiling/profile.xsl $< | \
 	  $(XSLTPROC) --output $@ $(docbookxsl)/xhtml/docbook.xsl -
 
-$(foreach file, $(d)/manual.html, $(eval $(call install-data-in, $(file), $(docdir)/manual)))
+$(foreach file, $(d)/manual.html $(d)/style.css, $(eval $(call install-data-in, $(file), $(docdir)/manual)))
 
 $(foreach file, $(wildcard $(d)/figures/*.png), $(eval $(call install-data-in, $(file), $(docdir)/manual/figures)))
 
+$(foreach file, $(wildcard $(d)/images/callouts/*.gif), $(eval $(call install-data-in, $(file), $(docdir)/manual/images/callouts)))
+
 $(eval $(call install-symlink, manual.html, $(docdir)/manual/index.html))
 
 

doc/manual/release-notes/rl-2.0.xml

@@ -503,14 +503,14 @@
   </listitem>
 
   <listitem>
-    <para><emphasis>Pure evaluation mode</emphasis>. With the
-    <literal>--pure-eval</literal> flag, Nix enables a variant of the existing
-    restricted evaluation mode that forbids access to anything that could cause
-    different evaluations of the same command line arguments to produce a
+    <para><emphasis>Pure evaluation mode</emphasis>. This is a variant
+    of the existing restricted evaluation mode. In pure mode, the Nix
+    evaluator forbids access to anything that could cause different
+    evaluations of the same command line arguments to produce a
     different result. This includes builtin functions such as
     <function>builtins.getEnv</function>, but more importantly,
-    <emphasis>all</emphasis> filesystem or network access unless a content hash
-    or commit hash is specified. For example, calls to
+    <emphasis>all</emphasis> filesystem or network access unless a
+    content hash or commit hash is specified. For example, calls to
     <function>builtins.fetchGit</function> are only allowed if a
     <varname>rev</varname> attribute is specified.</para>
 

doc/manual/style.css

@@ -0,0 +1,263 @@
+/* Copied from http://bakefile.sourceforge.net/, which appears
+   licensed under the GNU GPL. */
+
+
+/***************************************************************************
+                             Basic headers and text:
+ ***************************************************************************/
+
+body
+{
+    font-family: "Nimbus Sans L", sans-serif;
+    background: white;
+    margin: 2em 1em 2em 1em;
+}
+
+h1, h2, h3, h4
+{
+    color: #005aa0;
+}
+
+h1 /* title */
+{
+    font-size: 200%;
+}
+
+div.part h1
+{
+    font-size: 240%;
+}
+
+h2 /* chapters, appendices, subtitle */
+{
+    font-size: 180%;
+}
+
+div.part
+{
+    margin-top: 4em;
+}
+
+/* Extra space between chapters, appendices. */
+div.chapter > div.titlepage h2, div.appendix > div.titlepage h2 
+{ 
+    margin-top: 1.5em;
+}
+
+div.section > div.titlepage h2 /* sections */
+{
+    font-size: 150%;
+    margin-top: 1.5em;
+}
+
+h3 /* subsections */
+{
+    font-size: 125%;
+}
+
+div.simplesect h2
+{
+    font-size: 110%;
+}
+
+div.appendix h3
+{
+    font-size: 150%;
+    margin-top: 1.5em;
+}
+
+div.refentry\.separator
+{
+    margin-top: 2.5em;
+    margin-bottom: 2em;
+}
+
+div.refnamediv h2, div.refsynopsisdiv h2, div.refsection h2 /* refentry parts */
+{
+    margin-top: 1.4em;
+    font-size: 125%;
+}
+
+div.refsection h3
+{
+    font-size: 110%;
+}
+
+
+/***************************************************************************
+                               Examples:
+ ***************************************************************************/
+
+div.example
+{
+    border: 1px solid #b0b0b0;
+    padding: 6px 6px;
+    margin-left: 1.5em;
+    margin-right: 1.5em;
+    background: #f4f4f8;
+    border-radius: 0.4em;
+}
+
+div.example p.title
+{
+    margin-top: 0em;
+}
+
+div.example pre
+{
+}
+
+
+/***************************************************************************
+                            Screen dumps:
+ ***************************************************************************/
+
+pre.screen, pre.programlisting
+{
+    padding: 6px 6px;
+    margin-left: 1.5em;
+    margin-right: 1.5em;
+    color: #600000;
+    background: #f4f4f8;
+    font-family: monospace;
+}
+
+div.example pre.programlisting
+{
+    border: 0px;
+    padding: 0 0;
+    margin: 0 0 0 0;
+}
+
+
+/***************************************************************************
+                               Notes, warnings etc:
+ ***************************************************************************/
+
+.note, .warning
+{
+    border: 1px solid #b0b0b0;
+    padding: 3px 3px;
+    margin-left: 1.5em;
+    margin-right: 1.5em;
+    margin-bottom: 1em;
+    padding: 0.3em 0.3em 0.3em 0.3em;
+    background: #fffff5;
+    border-radius: 0.4em;
+}
+
+div.note, div.warning
+{
+    font-style: italic;
+}
+
+div.note h3, div.warning h3
+{
+    color: red;
+    font-size: 100%;
+    padding-right: 0.5em;
+    display: inline;
+}
+
+div.note p, div.warning p
+{
+    margin-bottom: 0em;
+}
+
+div.note h3 + p, div.warning h3 + p
+{
+    display: inline;
+}
+
+div.note h3
+{
+    color: blue;
+    font-size: 100%;
+}
+
+div.navfooter *
+{
+    font-size: 90%;
+}
+
+
+/***************************************************************************
+                        Links colors and highlighting: 
+ ***************************************************************************/
+
+a { text-decoration: none; }
+a:hover { text-decoration: underline; }
+a:link { color: #0048b3; }
+a:visited { color: #002a6a; }
+
+
+/***************************************************************************
+                              Table of contents:
+ ***************************************************************************/
+
+div.toc
+{
+    font-size: 90%;
+}
+
+div.toc dl
+{
+    margin-top: 0em;
+    margin-bottom: 0em;
+}
+
+
+/***************************************************************************
+                               Special elements:
+ ***************************************************************************/
+
+tt, code
+{
+    color: #400000;
+}
+
+.term
+{
+    font-weight: bold;
+    
+}
+
+div.variablelist dd p, div.glosslist dd p
+{
+    margin-top: 0em;
+}
+
+div.variablelist dd, div.glosslist dd
+{
+    margin-left: 1.5em;
+}
+
+div.glosslist dt
+{
+    font-style: italic;
+}
+
+.varname
+{
+    color: #400000;
+}
+
+span.command strong
+{
+    font-weight: normal;
+    color: #400000;
+}
+
+div.calloutlist table
+{
+}
+
+table
+{
+    border-collapse: collapse;
+}
+
+div.affiliation
+{
+    font-style: italic;
+}

flake.lock

@@ -1,28 +0,0 @@
-{
-  "nodes": {
-    "nixpkgs": {
-      "info": {
-        "lastModified": 1585405475,
-        "narHash": "sha256-bESW0n4KgPmZ0luxvwJ+UyATrC6iIltVCsGdLiphVeE="
-      },
-      "locked": {
-        "owner": "NixOS",
-        "repo": "nixpkgs",
-        "rev": "b88ff468e9850410070d4e0ccd68c7011f15b2be",
-        "type": "github"
-      },
-      "original": {
-        "id": "nixpkgs",
-        "ref": "nixos-20.03-small",
-        "type": "indirect"
-      }
-    },
-    "root": {
-      "inputs": {
-        "nixpkgs": "nixpkgs"
-      }
-    }
-  },
-  "root": "root",
-  "version": 5
-}

flake.nix

@@ -1,510 +0,0 @@
-{
-  description = "The purely functional package manager";
-
-  edition = 201909; # FIXME: remove
-
-  inputs.nixpkgs.url = "nixpkgs/nixos-20.03-small";
-
-  outputs = { self, nixpkgs }:
-
-    let
-
-      version = builtins.readFile ./.version + versionSuffix;
-      versionSuffix =
-        if officialRelease
-        then ""
-        else "pre${builtins.substring 0 8 (self.lastModifiedDate or self.lastModified)}_${self.shortRev or "dirty"}";
-
-      officialRelease = false;
-
-      systems = [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ];
-
-      forAllSystems = f: nixpkgs.lib.genAttrs systems (system: f system);
-
-      # Memoize nixpkgs for different platforms for efficiency.
-      nixpkgsFor = forAllSystems (system:
-        import nixpkgs {
-          inherit system;
-          overlays = [ self.overlay ];
-        }
-      );
-
-      commonDeps = pkgs: with pkgs; rec {
-        # Use "busybox-sandbox-shell" if present,
-        # if not (legacy) fallback and hope it's sufficient.
-        sh = pkgs.busybox-sandbox-shell or (busybox.override {
-          useMusl = true;
-          enableStatic = true;
-          enableMinimal = true;
-          extraConfig = ''
-            CONFIG_FEATURE_FANCY_ECHO y
-            CONFIG_FEATURE_SH_MATH y
-            CONFIG_FEATURE_SH_MATH_64 y
-
-            CONFIG_ASH y
-            CONFIG_ASH_OPTIMIZE_FOR_SIZE y
-
-            CONFIG_ASH_ALIAS y
-            CONFIG_ASH_BASH_COMPAT y
-            CONFIG_ASH_CMDCMD y
-            CONFIG_ASH_ECHO y
-            CONFIG_ASH_GETOPTS y
-            CONFIG_ASH_INTERNAL_GLOB y
-            CONFIG_ASH_JOB_CONTROL y
-            CONFIG_ASH_PRINTF y
-            CONFIG_ASH_TEST y
-          '';
-        });
-
-        configureFlags =
-          lib.optionals stdenv.isLinux [
-            "--with-sandbox-shell=${sh}/bin/busybox"
-          ];
-
-        buildDeps =
-          [ bison
-            flex
-            libxml2
-            libxslt
-            docbook5
-            docbook_xsl_ns
-            autoconf-archive
-            autoreconfHook
-
-            curl
-            bzip2 xz brotli zlib editline
-            openssl pkgconfig sqlite
-            libarchive
-            boost
-            (if lib.versionAtLeast lib.version "20.03pre"
-             then nlohmann_json
-             else nlohmann_json.override { multipleHeaders = true; })
-            nlohmann_json
-            rustc cargo
-
-            # Tests
-            git
-            mercurial
-            jq
-          ]
-          ++ lib.optionals stdenv.isLinux [libseccomp utillinuxMinimal]
-          ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium
-          ++ lib.optional (stdenv.isLinux || stdenv.isDarwin)
-            (aws-sdk-cpp.override {
-              apis = ["s3" "transfer"];
-              customMemoryManagement = false;
-            });
-
-        propagatedDeps =
-          [ (boehmgc.override { enableLargeConfig = true; })
-          ];
-
-        perlDeps =
-          [ perl
-            perlPackages.DBDSQLite
-          ];
-      };
-
-    in {
-
-      # A Nixpkgs overlay that overrides the 'nix' and
-      # 'nix.perl-bindings' packages.
-      overlay = final: prev: {
-
-        nix = with final; with commonDeps pkgs; (stdenv.mkDerivation {
-          name = "nix-${version}";
-
-          src = self;
-
-          VERSION_SUFFIX = versionSuffix;
-
-          outputs = [ "out" "dev" "doc" ];
-
-          buildInputs = buildDeps;
-
-          propagatedBuildInputs = propagatedDeps;
-
-          preConfigure =
-            ''
-              # Copy libboost_context so we don't get all of Boost in our closure.
-              # https://github.com/NixOS/nixpkgs/issues/45462
-              mkdir -p $out/lib
-              cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib
-              rm -f $out/lib/*.a
-              ${lib.optionalString stdenv.isLinux ''
-                chmod u+w $out/lib/*.so.*
-                patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.*
-              ''}
-
-              ln -sfn ${final.nixVendoredCrates}/vendor/ nix-rust/vendor
-            '';
-
-          configureFlags = configureFlags ++
-            [ "--sysconfdir=/etc" ];
-
-          enableParallelBuilding = true;
-
-          makeFlags = "profiledir=$(out)/etc/profile.d";
-
-          doCheck = true;
-
-          installFlags = "sysconfdir=$(out)/etc";
-
-          doInstallCheck = true;
-          installCheckFlags = "sysconfdir=$(out)/etc";
-
-          separateDebugInfo = true;
-
-          preDist = ''
-            mkdir -p $doc/nix-support
-            echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products
-          '';
-        }) // {
-
-          perl-bindings = with final; stdenv.mkDerivation {
-            name = "nix-perl-${version}";
-
-            src = self;
-
-            buildInputs =
-              [ autoconf-archive
-                autoreconfHook
-                nix
-                curl
-                bzip2
-                xz
-                pkgconfig
-                pkgs.perl
-                boost
-              ]
-              ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium;
-
-            configureFlags = ''
-              --with-dbi=${perlPackages.DBI}/${pkgs.perl.libPrefix}
-              --with-dbd-sqlite=${perlPackages.DBDSQLite}/${pkgs.perl.libPrefix}
-            '';
-
-            enableParallelBuilding = true;
-
-            postUnpack = "sourceRoot=$sourceRoot/perl";
-          };
-
-        };
-
-        # Create a "vendor" directory that contains the crates listed in
-        # Cargo.lock, and include it in the Nix tarball. This allows Nix
-        # to be built without network access.
-        nixVendoredCrates =
-          let
-            lockFile = builtins.fromTOML (builtins.readFile nix-rust/Cargo.lock);
-
-            files = map (pkg: import <nix/fetchurl.nix> {
-              url = "https://crates.io/api/v1/crates/${pkg.name}/${pkg.version}/download";
-              sha256 = lockFile.metadata."checksum ${pkg.name} ${pkg.version} (registry+https://github.com/rust-lang/crates.io-index)";
-            }) (builtins.filter (pkg: pkg.source or "" == "registry+https://github.com/rust-lang/crates.io-index") lockFile.package);
-
-          in final.runCommand "cargo-vendor-dir" {}
-            ''
-              mkdir -p $out/vendor
-
-              cat > $out/vendor/config <<EOF
-              [source.crates-io]
-              replace-with = "vendored-sources"
-
-              [source.vendored-sources]
-              directory = "vendor"
-              EOF
-
-              ${toString (builtins.map (file: ''
-                mkdir $out/vendor/tmp
-                tar xvf ${file} -C $out/vendor/tmp
-                dir=$(echo $out/vendor/tmp/*)
-
-                # Add just enough metadata to keep Cargo happy.
-                printf '{"files":{},"package":"${file.outputHash}"}' > "$dir/.cargo-checksum.json"
-
-                # Clean up some cruft from the winapi crates. FIXME: find
-                # a way to remove winapi* from our dependencies.
-                if [[ $dir =~ /winapi ]]; then
-                  find $dir -name "*.a" -print0 | xargs -0 rm -f --
-                fi
-
-                mv "$dir" $out/vendor/
-
-                rm -rf $out/vendor/tmp
-              '') files)}
-            '';
-
-      };
-
-      hydraJobs = {
-
-        vendoredCrates =
-          with nixpkgsFor.x86_64-linux;
-
-          runCommand "vendored-crates" {}
-            ''
-              mkdir -p $out/nix-support
-              name=nix-vendored-crates-${version}
-              fn=$out/$name.tar.xz
-              tar cvfJ $fn -C ${nixVendoredCrates} vendor \
-                --owner=0 --group=0 --mode=u+rw,uga+r \
-                --transform "s,vendor,$name,"
-              echo "file crates-tarball $fn" >> $out/nix-support/hydra-build-products
-            '';
-
-        # Binary package for various platforms.
-        build = nixpkgs.lib.genAttrs systems (system: nixpkgsFor.${system}.nix);
-
-        # Perl bindings for various platforms.
-        perlBindings = nixpkgs.lib.genAttrs systems (system: nixpkgsFor.${system}.nix.perl-bindings);
-
-        # Binary tarball for various platforms, containing a Nix store
-        # with the closure of 'nix' package, and the second half of
-        # the installation script.
-        binaryTarball = nixpkgs.lib.genAttrs systems (system:
-
-          with nixpkgsFor.${system};
-
-          let
-            installerClosureInfo = closureInfo { rootPaths = [ nix cacert ]; };
-          in
-
-          runCommand "nix-binary-tarball-${version}"
-            { #nativeBuildInputs = lib.optional (system != "aarch64-linux") shellcheck;
-              meta.description = "Distribution-independent Nix bootstrap binaries for ${system}";
-            }
-            ''
-              cp ${installerClosureInfo}/registration $TMPDIR/reginfo
-              substitute ${./scripts/install-nix-from-closure.sh} $TMPDIR/install \
-                --subst-var-by nix ${nix} \
-                --subst-var-by cacert ${cacert}
-
-              substitute ${./scripts/install-darwin-multi-user.sh} $TMPDIR/install-darwin-multi-user.sh \
-                --subst-var-by nix ${nix} \
-                --subst-var-by cacert ${cacert}
-              substitute ${./scripts/install-systemd-multi-user.sh} $TMPDIR/install-systemd-multi-user.sh \
-                --subst-var-by nix ${nix} \
-                --subst-var-by cacert ${cacert}
-              substitute ${./scripts/install-multi-user.sh} $TMPDIR/install-multi-user \
-                --subst-var-by nix ${nix} \
-                --subst-var-by cacert ${cacert}
-
-              if type -p shellcheck; then
-                # SC1090: Don't worry about not being able to find
-                #         $nix/etc/profile.d/nix.sh
-                shellcheck --exclude SC1090 $TMPDIR/install
-                shellcheck $TMPDIR/install-darwin-multi-user.sh
-                shellcheck $TMPDIR/install-systemd-multi-user.sh
-
-                # SC1091: Don't panic about not being able to source
-                #         /etc/profile
-                # SC2002: Ignore "useless cat" "error", when loading
-                #         .reginfo, as the cat is a much cleaner
-                #         implementation, even though it is "useless"
-                # SC2116: Allow ROOT_HOME=$(echo ~root) for resolving
-                #         root's home directory
-                shellcheck --external-sources \
-                  --exclude SC1091,SC2002,SC2116 $TMPDIR/install-multi-user
-              fi
-
-              chmod +x $TMPDIR/install
-              chmod +x $TMPDIR/install-darwin-multi-user.sh
-              chmod +x $TMPDIR/install-systemd-multi-user.sh
-              chmod +x $TMPDIR/install-multi-user
-              dir=nix-${version}-${system}
-              fn=$out/$dir.tar.xz
-              mkdir -p $out/nix-support
-              echo "file binary-dist $fn" >> $out/nix-support/hydra-build-products
-              tar cvfJ $fn \
-                --owner=0 --group=0 --mode=u+rw,uga+r \
-                --absolute-names \
-                --hard-dereference \
-                --transform "s,$TMPDIR/install,$dir/install," \
-                --transform "s,$TMPDIR/reginfo,$dir/.reginfo," \
-                --transform "s,$NIX_STORE,$dir/store,S" \
-                $TMPDIR/install $TMPDIR/install-darwin-multi-user.sh \
-                $TMPDIR/install-systemd-multi-user.sh \
-                $TMPDIR/install-multi-user $TMPDIR/reginfo \
-                $(cat ${installerClosureInfo}/store-paths)
-            '');
-
-        # The first half of the installation script. This is uploaded
-        # to https://nixos.org/nix/install. It downloads the binary
-        # tarball for the user's system and calls the second half of the
-        # installation script.
-        installerScript =
-          with nixpkgsFor.x86_64-linux;
-          runCommand "installer-script"
-            { buildInputs = [ nix ];
-            }
-            ''
-              mkdir -p $out/nix-support
-
-              substitute ${./scripts/install.in} $out/install \
-                ${pkgs.lib.concatMapStrings
-                  (system: "--replace '@binaryTarball_${system}@' $(nix --experimental-features nix-command hash-file --base16 --type sha256 ${self.hydraJobs.binaryTarball.${system}}/*.tar.xz) ")
-                  [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ]
-                } \
-                --replace '@nixVersion@' ${version}
-
-              echo "file installer $out/install" >> $out/nix-support/hydra-build-products
-            '';
-
-        # Line coverage analysis.
-        coverage =
-          with nixpkgsFor.x86_64-linux;
-          with commonDeps pkgs;
-
-          releaseTools.coverageAnalysis {
-            name = "nix-coverage-${version}";
-
-            src = self;
-
-            preConfigure =
-              ''
-                ln -sfn ${nixVendoredCrates}/vendor/ nix-rust/vendor
-              '';
-
-            enableParallelBuilding = true;
-
-            buildInputs = buildDeps ++ propagatedDeps;
-
-            dontInstall = false;
-
-            doInstallCheck = true;
-
-            lcovFilter = [ "*/boost/*" "*-tab.*" ];
-
-            # We call `dot', and even though we just use it to
-            # syntax-check generated dot files, it still requires some
-            # fonts.  So provide those.
-            FONTCONFIG_FILE = texFunctions.fontsConf;
-
-            # To test building without precompiled headers.
-            makeFlagsArray = [ "PRECOMPILE_HEADERS=0" ];
-          };
-
-        # System tests.
-        tests.remoteBuilds = import ./tests/remote-builds.nix {
-          system = "x86_64-linux";
-          inherit nixpkgs;
-          inherit (self) overlay;
-        };
-
-        tests.nix-copy-closure = import ./tests/nix-copy-closure.nix {
-          system = "x86_64-linux";
-          inherit nixpkgs;
-          inherit (self) overlay;
-        };
-
-        tests.githubFlakes = (import ./tests/github-flakes.nix rec {
-          system = "x86_64-linux";
-          inherit nixpkgs;
-          inherit (self) overlay;
-        });
-
-        tests.setuid = nixpkgs.lib.genAttrs
-          ["i686-linux" "x86_64-linux"]
-          (system:
-            import ./tests/setuid.nix rec {
-              inherit nixpkgs system;
-              inherit (self) overlay;
-            });
-
-        # Test whether the binary tarball works in an Ubuntu system.
-        tests.binaryTarball =
-          with nixpkgsFor.x86_64-linux;
-          vmTools.runInLinuxImage (runCommand "nix-binary-tarball-test"
-            { diskImage = vmTools.diskImages.ubuntu1204x86_64;
-            }
-            ''
-              set -x
-              useradd -m alice
-              su - alice -c 'tar xf ${self.hydraJobs.binaryTarball.x86_64-linux}/*.tar.*'
-              mkdir /dest-nix
-              mount -o bind /dest-nix /nix # Provide a writable /nix.
-              chown alice /nix
-              su - alice -c '_NIX_INSTALLER_TEST=1 ./nix-*/install'
-              su - alice -c 'nix-store --verify'
-              su - alice -c 'PAGER= nix-store -qR ${self.hydraJobs.build.x86_64-linux}'
-
-              # Check whether 'nix upgrade-nix' works.
-              cat > /tmp/paths.nix <<EOF
-              {
-                x86_64-linux = "${self.hydraJobs.build.x86_64-linux}";
-              }
-              EOF
-              su - alice -c 'nix --experimental-features nix-command upgrade-nix -vvv --nix-store-paths-url file:///tmp/paths.nix'
-              (! [ -L /home/alice/.profile-1-link ])
-              su - alice -c 'PAGER= nix-store -qR ${self.hydraJobs.build.x86_64-linux}'
-
-              mkdir -p $out/nix-support
-              touch $out/nix-support/hydra-build-products
-              umount /nix
-            '');
-
-        /*
-        # Check whether we can still evaluate all of Nixpkgs.
-        tests.evalNixpkgs =
-          import (nixpkgs + "/pkgs/top-level/make-tarball.nix") {
-            # FIXME: fix pkgs/top-level/make-tarball.nix in NixOS to not require a revCount.
-            inherit nixpkgs;
-            pkgs = nixpkgsFor.x86_64-linux;
-            officialRelease = false;
-          };
-
-        # Check whether we can still evaluate NixOS.
-        tests.evalNixOS =
-          with nixpkgsFor.x86_64-linux;
-          runCommand "eval-nixos" { buildInputs = [ nix ]; }
-            ''
-              export NIX_STATE_DIR=$TMPDIR
-
-              nix-instantiate ${nixpkgs}/nixos/release-combined.nix -A tested --dry-run \
-                --arg nixpkgs '{ outPath = ${nixpkgs}; revCount = 123; shortRev = "abcdefgh"; }'
-
-              touch $out
-            '';
-        */
-
-      };
-
-      checks = forAllSystems (system: {
-        binaryTarball = self.hydraJobs.binaryTarball.${system};
-        perlBindings = self.hydraJobs.perlBindings.${system};
-      });
-
-      packages = forAllSystems (system: {
-        inherit (nixpkgsFor.${system}) nix;
-      });
-
-      defaultPackage = forAllSystems (system: self.packages.${system}.nix);
-
-      devShell = forAllSystems (system:
-        with nixpkgsFor.${system};
-        with commonDeps pkgs;
-
-        stdenv.mkDerivation {
-          name = "nix";
-
-          buildInputs = buildDeps ++ propagatedDeps ++ perlDeps ++ [ pkgs.rustfmt ];
-
-          inherit configureFlags;
-
-          enableParallelBuilding = true;
-
-          installFlags = "sysconfdir=$(out)/etc";
-
-          shellHook =
-            ''
-              export prefix=$(pwd)/inst
-              configureFlags+=" --prefix=$prefix"
-              PKG_CONFIG_PATH=$prefix/lib/pkgconfig:$PKG_CONFIG_PATH
-              PATH=$prefix/bin:$PATH
-              unset PYTHONPATH
-            '';
-        });
-
-  };
-}

local.mk

@@ -6,11 +6,9 @@ dist-files += configure config.h.in perl/configure
 
 clean-files += Makefile.config
 
-GLOBAL_CXXFLAGS += -Wno-deprecated-declarations
+GLOBAL_CXXFLAGS += -I . -I src -I src/libutil -I src/libstore -I src/libmain -I src/libexpr -I src/nix -Wno-deprecated-declarations
 
-$(foreach i, config.h $(wildcard src/lib*/*.hh), \
+$(foreach i, config.h $(call rwildcard, src/lib*, *.hh), \
   $(eval $(call install-file-in, $(i), $(includedir)/nix, 0644)))
 
 $(GCH) $(PCH): src/libutil/util.hh config.h
-
-GCH_CXXFLAGS = -I src/libutil

maintainers/upload-release.pl

@@ -1,5 +1,5 @@
 #! /usr/bin/env nix-shell
-#! nix-shell -i perl -p perl perlPackages.LWPUserAgent perlPackages.LWPProtocolHttps perlPackages.FileSlurp perlPackages.NetAmazonS3 gnupg1
+#! nix-shell -i perl -p perl perlPackages.LWPUserAgent perlPackages.LWPProtocolHttps perlPackages.FileSlurp gnupg1
 
 use strict;
 use Data::Dumper;
@@ -9,16 +9,12 @@ use File::Slurp;
 use File::Copy;
 use JSON::PP;
 use LWP::UserAgent;
-use Net::Amazon::S3;
 
 my $evalId = $ARGV[0] or die "Usage: $0 EVAL-ID\n";
 
-my $releasesBucketName = "nix-releases";
-my $channelsBucketName = "nix-channels";
+my $releasesDir = "/home/eelco/mnt/releases";
 my $nixpkgsDir = "/home/eelco/Dev/nixpkgs-pristine";
 
-my $TMPDIR = $ENV{'TMPDIR'} // "/tmp";
-
 # FIXME: cut&paste from nixos-channel-scripts.
 sub fetch {
     my ($url, $type) = @_;
@@ -46,31 +42,13 @@ my $version = $1;
 
 print STDERR "Nix revision is $nixRev, version is $version\n";
 
-my $releaseDir = "nix/$releaseName";
+File::Path::make_path($releasesDir);
+if (system("mountpoint -q $releasesDir") != 0) {
+    system("sshfs hydra-mirror\@nixos.org:/releases $releasesDir") == 0 or die;
+}
 
-my $tmpDir = "$TMPDIR/nix-release/$releaseName";
-File::Path::make_path($tmpDir);
-
-# S3 setup.
-my $aws_access_key_id = $ENV{'AWS_ACCESS_KEY_ID'} or die "No AWS_ACCESS_KEY_ID given.";
-my $aws_secret_access_key = $ENV{'AWS_SECRET_ACCESS_KEY'} or die "No AWS_SECRET_ACCESS_KEY given.";
-
-my $s3 = Net::Amazon::S3->new(
-    { aws_access_key_id     => $aws_access_key_id,
-      aws_secret_access_key => $aws_secret_access_key,
-      retry                 => 1,
-      host                  => "s3-eu-west-1.amazonaws.com",
-    });
-
-my $releasesBucket = $s3->bucket($releasesBucketName) or die;
-
-my $s3_us = Net::Amazon::S3->new(
-    { aws_access_key_id     => $aws_access_key_id,
-      aws_secret_access_key => $aws_secret_access_key,
-      retry                 => 1,
-    });
-
-my $channelsBucket = $s3_us->bucket($channelsBucketName) or die;
+my $releaseDir = "$releasesDir/nix/$releaseName";
+File::Path::make_path($releaseDir);
 
 sub downloadFile {
     my ($jobName, $productNr, $dstName) = @_;
@@ -79,49 +57,40 @@ sub downloadFile {
 
     my $srcFile = $buildInfo->{buildproducts}->{$productNr}->{path} or die "job '$jobName' lacks product $productNr\n";
     $dstName //= basename($srcFile);
-    my $tmpFile = "$tmpDir/$dstName";
+    my $dstFile = "$releaseDir/" . $dstName;
 
-    if (!-e $tmpFile) {
-        print STDERR "downloading $srcFile to $tmpFile...\n";
-        system("NIX_REMOTE=https://cache.nixos.org/ nix cat-store '$srcFile' > '$tmpFile'") == 0
+    if (! -e $dstFile) {
+        print STDERR "downloading $srcFile to $dstFile...\n";
+        system("NIX_REMOTE=https://cache.nixos.org/ nix cat-store '$srcFile' > '$dstFile.tmp'") == 0
             or die "unable to fetch $srcFile\n";
+        rename("$dstFile.tmp", $dstFile) or die;
     }
 
     my $sha256_expected = $buildInfo->{buildproducts}->{$productNr}->{sha256hash} or die;
-    my $sha256_actual = `nix hash-file --base16 --type sha256 '$tmpFile'`;
+    my $sha256_actual = `nix hash-file --base16 --type sha256 '$dstFile'`;
     chomp $sha256_actual;
     if ($sha256_expected ne $sha256_actual) {
-        print STDERR "file $tmpFile is corrupt, got $sha256_actual, expected $sha256_expected\n";
+        print STDERR "file $dstFile is corrupt, got $sha256_actual, expected $sha256_expected\n";
         exit 1;
     }
 
-    write_file("$tmpFile.sha256", $sha256_expected);
+    write_file("$dstFile.sha256", $sha256_expected);
 
-    if (! -e "$tmpFile.asc") {
-        system("gpg2 --detach-sign --armor $tmpFile") == 0 or die "unable to sign $tmpFile\n";
+    if (! -e "$dstFile.asc") {
+        system("gpg2 --detach-sign --armor $dstFile") == 0 or die "unable to sign $dstFile\n";
     }
 
-    return $sha256_expected;
+    return ($dstFile, $sha256_expected);
 }
 
 downloadFile("tarball", "2"); # .tar.bz2
-my $tarballHash = downloadFile("tarball", "3"); # .tar.xz
+my ($tarball, $tarballHash) = downloadFile("tarball", "3"); # .tar.xz
 downloadFile("binaryTarball.i686-linux", "1");
 downloadFile("binaryTarball.x86_64-linux", "1");
 downloadFile("binaryTarball.aarch64-linux", "1");
 downloadFile("binaryTarball.x86_64-darwin", "1");
 downloadFile("installerScript", "1");
 
-for my $fn (glob "$tmpDir/*") {
-    my $name = basename($fn);
-    my $dstKey = "$releaseDir/" . $name;
-    unless (defined $releasesBucket->head_key($dstKey)) {
-        print STDERR "uploading $fn to s3://$releasesBucketName/$dstKey...\n";
-        $releasesBucket->add_key_filename($dstKey, $fn)
-            or die $releasesBucket->err . ": " . $releasesBucket->errstr;
-    }
-}
-
 exit if $version =~ /pre/;
 
 # Update Nixpkgs in a very hacky way.
@@ -156,11 +125,18 @@ write_file("$nixpkgsDir/nixos/modules/installer/tools/nix-fallback-paths.nix",
 
 system("cd $nixpkgsDir && git commit -a -m 'nix: $oldName -> $version'") == 0 or die;
 
+# Extract the HTML manual.
+File::Path::make_path("$releaseDir/manual");
+
+system("tar xvf $tarball --strip-components=3 -C $releaseDir/manual --wildcards '*/doc/manual/*.html' '*/doc/manual/*.css' '*/doc/manual/*.gif' '*/doc/manual/*.png'") == 0 or die;
+
+if (! -e "$releaseDir/manual/index.html") {
+    symlink("manual.html", "$releaseDir/manual/index.html") or die;
+}
+
 # Update the "latest" symlink.
-$channelsBucket->add_key(
-    "nix-latest/install", "",
-    { "x-amz-website-redirect-location" => "https://releases.nixos.org/$releaseDir/install" })
-    or die $channelsBucket->err . ": " . $channelsBucket->errstr;
+symlink("$releaseName", "$releasesDir/nix/latest-tmp") or die;
+rename("$releasesDir/nix/latest-tmp", "$releasesDir/nix/latest") or die;
 
 # Tag the release in Git.
 chdir("/home/eelco/Dev/nix-pristine") or die;

misc/bash/completion.sh

@@ -1,19 +0,0 @@
-function _complete_nix {
-    local -a words
-    local cword cur
-    _get_comp_words_by_ref -n ':=&' words cword cur
-    local have_type
-    while IFS= read -r line; do
-        if [[ -z $have_type ]]; then
-            have_type=1
-            if [[ $line = filenames ]]; then
-                compopt -o filenames
-            fi
-        else
-            COMPREPLY+=("$line")
-        fi
-    done < <(NIX_GET_COMPLETIONS=$cword "${words[@]}")
-    __ltrim_colon_completions "$cur"
-}
-
-complete -F _complete_nix nix

misc/bash/local.mk

@@ -1 +0,0 @@
-$(eval $(call install-file-as, $(d)/completion.sh, $(datarootdir)/bash-completion/completions/_nix3, 0644))

mk/README.md

@@ -0,0 +1,6 @@
+This is a set of helper Makefiles for doing non-recursive builds with
+GNU Make.  The canonical source can be found at
+https://github.com/edolstra/make-rules.  You should copy the files
+into the `mk` subdirectory of your project.
+
+TODO: write more documentation.

mk/precompiled-headers.mk

@@ -8,14 +8,14 @@ GCH = $(buildprefix)precompiled-headers.h.gch
 $(GCH): precompiled-headers.h
 	@rm -f $@
 	@mkdir -p "$(dir $@)"
-	$(trace-gen) $(CXX) -x c++-header -o $@ $< $(GLOBAL_CXXFLAGS) $(GCH_CXXFLAGS)
+	$(trace-gen) $(CXX) -x c++-header -o $@ $< $(GLOBAL_CXXFLAGS)
 
 PCH = $(buildprefix)precompiled-headers.h.pch
 
 $(PCH): precompiled-headers.h
 	@rm -f $@
 	@mkdir -p "$(dir $@)"
-	$(trace-gen) $(CXX) -x c++-header -o $@ $< $(GLOBAL_CXXFLAGS) $(GCH_CXXFLAGS)
+	$(trace-gen) $(CXX) -x c++-header -o $@ $< $(GLOBAL_CXXFLAGS)
 
 clean-files += $(GCH) $(PCH)
 

perl/lib/Nix/Store.xs

@@ -102,7 +102,7 @@ SV * queryPathInfo(char * path, int base32)
     PPCODE:
         try {
             auto info = store()->queryPathInfo(store()->parseStorePath(path));
-            if (!info->deriver)
+            if (info->deriver)
                 XPUSHs(&PL_sv_undef);
             else
                 XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(*info->deriver).c_str(), 0)));

precompiled-headers.h

@@ -56,3 +56,6 @@
 #include <sys/wait.h>
 #include <termios.h>
 #include <unistd.h>
+
+#include "util.hh"
+#include "args.hh"

release-common.nix

@@ -0,0 +1,80 @@
+{ pkgs }:
+
+with pkgs;
+
+rec {
+  # Use "busybox-sandbox-shell" if present,
+  # if not (legacy) fallback and hope it's sufficient.
+  sh = pkgs.busybox-sandbox-shell or (busybox.override {
+    useMusl = true;
+    enableStatic = true;
+    enableMinimal = true;
+    extraConfig = ''
+      CONFIG_FEATURE_FANCY_ECHO y
+      CONFIG_FEATURE_SH_MATH y
+      CONFIG_FEATURE_SH_MATH_64 y
+
+      CONFIG_ASH y
+      CONFIG_ASH_OPTIMIZE_FOR_SIZE y
+
+      CONFIG_ASH_ALIAS y
+      CONFIG_ASH_BASH_COMPAT y
+      CONFIG_ASH_CMDCMD y
+      CONFIG_ASH_ECHO y
+      CONFIG_ASH_GETOPTS y
+      CONFIG_ASH_INTERNAL_GLOB y
+      CONFIG_ASH_JOB_CONTROL y
+      CONFIG_ASH_PRINTF y
+      CONFIG_ASH_TEST y
+    '';
+  });
+
+  configureFlags =
+    lib.optionals stdenv.isLinux [
+      "--with-sandbox-shell=${sh}/bin/busybox"
+    ];
+
+  tarballDeps =
+    [ bison
+      flex
+      libxml2
+      libxslt
+      docbook5
+      docbook_xsl_ns
+      autoconf-archive
+      autoreconfHook
+    ];
+
+  buildDeps =
+    [ curl
+      bzip2 xz brotli zlib editline
+      openssl pkgconfig sqlite
+      libarchive
+      boost
+      nlohmann_json
+      rustc cargo
+
+      # Tests
+      git
+      mercurial
+    ]
+    ++ lib.optionals stdenv.isLinux [libseccomp utillinuxMinimal]
+    ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium
+    ++ lib.optional (stdenv.isLinux || stdenv.isDarwin)
+      ((aws-sdk-cpp.override {
+        apis = ["s3" "transfer"];
+        customMemoryManagement = false;
+      }).overrideDerivation (args: {
+        /*
+        patches = args.patches or [] ++ [ (fetchpatch {
+          url = https://github.com/edolstra/aws-sdk-cpp/commit/3e07e1f1aae41b4c8b340735ff9e8c735f0c063f.patch;
+          sha256 = "1pij0v449p166f9l29x7ppzk8j7g9k9mp15ilh5qxp29c7fnvxy2";
+        }) ];
+        */
+      }));
+
+  perlDeps =
+    [ perl
+      perlPackages.DBDSQLite
+    ];
+}

release.nix

@@ -0,0 +1,379 @@
+{ nix ? builtins.fetchGit ./.
+, nixpkgs ? builtins.fetchTarball https://github.com/NixOS/nixpkgs/archive/nixos-19.09.tar.gz
+, officialRelease ? false
+, systems ? [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ]
+}:
+
+let
+
+  pkgs = import nixpkgs { system = builtins.currentSystem or "x86_64-linux"; };
+
+  jobs = rec {
+
+    # Create a "vendor" directory that contains the crates listed in
+    # Cargo.lock, and include it in the Nix tarball. This allows Nix
+    # to be built without network access.
+    vendoredCrates =
+      let
+        lockFile = builtins.fromTOML (builtins.readFile nix-rust/Cargo.lock);
+
+        files = map (pkg: import <nix/fetchurl.nix> {
+          url = "https://crates.io/api/v1/crates/${pkg.name}/${pkg.version}/download";
+          sha256 = lockFile.metadata."checksum ${pkg.name} ${pkg.version} (registry+https://github.com/rust-lang/crates.io-index)";
+        }) (builtins.filter (pkg: pkg.source or "" == "registry+https://github.com/rust-lang/crates.io-index") lockFile.package);
+
+      in pkgs.runCommand "cargo-vendor-dir" {}
+        ''
+          mkdir -p $out/vendor
+
+          cat > $out/vendor/config <<EOF
+          [source.crates-io]
+          replace-with = "vendored-sources"
+
+          [source.vendored-sources]
+          directory = "vendor"
+          EOF
+
+          ${toString (builtins.map (file: ''
+            mkdir $out/vendor/tmp
+            tar xvf ${file} -C $out/vendor/tmp
+            dir=$(echo $out/vendor/tmp/*)
+
+            # Add just enough metadata to keep Cargo happy.
+            printf '{"files":{},"package":"${file.outputHash}"}' > "$dir/.cargo-checksum.json"
+
+            # Clean up some cruft from the winapi crates. FIXME: find
+            # a way to remove winapi* from our dependencies.
+            if [[ $dir =~ /winapi ]]; then
+              find $dir -name "*.a" -print0 | xargs -0 rm -f --
+            fi
+
+            mv "$dir" $out/vendor/
+
+            rm -rf $out/vendor/tmp
+          '') files)}
+        '';
+
+
+    tarball =
+      with pkgs;
+
+      with import ./release-common.nix { inherit pkgs; };
+
+      releaseTools.sourceTarball {
+        name = "nix-tarball";
+        version = builtins.readFile ./.version;
+        versionSuffix = if officialRelease then "" else "pre${toString nix.revCount}_${nix.shortRev}";
+        src = nix;
+        inherit officialRelease;
+
+        buildInputs = tarballDeps ++ buildDeps;
+
+        postUnpack = ''
+          (cd $sourceRoot && find . -type f) | cut -c3- > $sourceRoot/.dist-files
+          cat $sourceRoot/.dist-files
+        '';
+
+        preConfigure = ''
+          (cd perl ; autoreconf --install --force --verbose)
+          # TeX needs a writable font cache.
+          export VARTEXFONTS=$TMPDIR/texfonts
+        '';
+
+        distPhase =
+          ''
+            cp -prd ${vendoredCrates}/vendor/ nix-rust/vendor/
+
+            runHook preDist
+            make dist
+            mkdir -p $out/tarballs
+            cp *.tar.* $out/tarballs
+          '';
+
+        preDist = ''
+          make install docdir=$out/share/doc/nix makefiles=doc/manual/local.mk
+          echo "doc manual $out/share/doc/nix/manual" >> $out/nix-support/hydra-build-products
+        '';
+      };
+
+
+    build = pkgs.lib.genAttrs systems (system:
+
+      let pkgs = import nixpkgs { inherit system; }; in
+
+      with pkgs;
+
+      with import ./release-common.nix { inherit pkgs; };
+
+      releaseTools.nixBuild {
+        name = "nix";
+        src = tarball;
+
+        buildInputs = buildDeps;
+
+        preConfigure =
+          # Copy libboost_context so we don't get all of Boost in our closure.
+          # https://github.com/NixOS/nixpkgs/issues/45462
+          ''
+            mkdir -p $out/lib
+            cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib
+            rm -f $out/lib/*.a
+            ${lib.optionalString stdenv.isLinux ''
+              chmod u+w $out/lib/*.so.*
+              patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.*
+            ''}
+          '';
+
+        configureFlags = configureFlags ++
+          [ "--sysconfdir=/etc" ];
+
+        enableParallelBuilding = true;
+
+        makeFlags = "profiledir=$(out)/etc/profile.d";
+
+        installFlags = "sysconfdir=$(out)/etc";
+
+        doInstallCheck = true;
+        installCheckFlags = "sysconfdir=$(out)/etc";
+      });
+
+
+    perlBindings = pkgs.lib.genAttrs systems (system:
+
+      let pkgs = import nixpkgs { inherit system; }; in with pkgs;
+
+      releaseTools.nixBuild {
+        name = "nix-perl";
+        src = tarball;
+
+        buildInputs =
+          [ jobs.build.${system} curl bzip2 xz pkgconfig pkgs.perl boost ]
+          ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium;
+
+        configureFlags = ''
+          --with-dbi=${perlPackages.DBI}/${pkgs.perl.libPrefix}
+          --with-dbd-sqlite=${perlPackages.DBDSQLite}/${pkgs.perl.libPrefix}
+        '';
+
+        enableParallelBuilding = true;
+
+        postUnpack = "sourceRoot=$sourceRoot/perl";
+      });
+
+
+    binaryTarball = pkgs.lib.genAttrs systems (system:
+
+      with import nixpkgs { inherit system; };
+
+      let
+        toplevel = builtins.getAttr system jobs.build;
+        version = toplevel.src.version;
+        installerClosureInfo = closureInfo { rootPaths = [ toplevel cacert ]; };
+      in
+
+      runCommand "nix-binary-tarball-${version}"
+        { #nativeBuildInputs = lib.optional (system != "aarch64-linux") shellcheck;
+          meta.description = "Distribution-independent Nix bootstrap binaries for ${system}";
+        }
+        ''
+          cp ${installerClosureInfo}/registration $TMPDIR/reginfo
+          substitute ${./scripts/install-nix-from-closure.sh} $TMPDIR/install \
+            --subst-var-by nix ${toplevel} \
+            --subst-var-by cacert ${cacert}
+
+          substitute ${./scripts/install-darwin-multi-user.sh} $TMPDIR/install-darwin-multi-user.sh \
+            --subst-var-by nix ${toplevel} \
+            --subst-var-by cacert ${cacert}
+          substitute ${./scripts/install-systemd-multi-user.sh} $TMPDIR/install-systemd-multi-user.sh \
+            --subst-var-by nix ${toplevel} \
+            --subst-var-by cacert ${cacert}
+          substitute ${./scripts/install-multi-user.sh} $TMPDIR/install-multi-user \
+            --subst-var-by nix ${toplevel} \
+            --subst-var-by cacert ${cacert}
+
+          if type -p shellcheck; then
+            # SC1090: Don't worry about not being able to find
+            #         $nix/etc/profile.d/nix.sh
+            shellcheck --exclude SC1090 $TMPDIR/install
+            shellcheck $TMPDIR/install-darwin-multi-user.sh
+            shellcheck $TMPDIR/install-systemd-multi-user.sh
+
+            # SC1091: Don't panic about not being able to source
+            #         /etc/profile
+            # SC2002: Ignore "useless cat" "error", when loading
+            #         .reginfo, as the cat is a much cleaner
+            #         implementation, even though it is "useless"
+            # SC2116: Allow ROOT_HOME=$(echo ~root) for resolving
+            #         root's home directory
+            shellcheck --external-sources \
+              --exclude SC1091,SC2002,SC2116 $TMPDIR/install-multi-user
+          fi
+
+          chmod +x $TMPDIR/install
+          chmod +x $TMPDIR/install-darwin-multi-user.sh
+          chmod +x $TMPDIR/install-systemd-multi-user.sh
+          chmod +x $TMPDIR/install-multi-user
+          dir=nix-${version}-${system}
+          fn=$out/$dir.tar.xz
+          mkdir -p $out/nix-support
+          echo "file binary-dist $fn" >> $out/nix-support/hydra-build-products
+          tar cvfJ $fn \
+            --owner=0 --group=0 --mode=u+rw,uga+r \
+            --absolute-names \
+            --hard-dereference \
+            --transform "s,$TMPDIR/install,$dir/install," \
+            --transform "s,$TMPDIR/reginfo,$dir/.reginfo," \
+            --transform "s,$NIX_STORE,$dir/store,S" \
+            $TMPDIR/install $TMPDIR/install-darwin-multi-user.sh \
+            $TMPDIR/install-systemd-multi-user.sh \
+            $TMPDIR/install-multi-user $TMPDIR/reginfo \
+            $(cat ${installerClosureInfo}/store-paths)
+        '');
+
+
+    coverage =
+      with pkgs;
+
+      with import ./release-common.nix { inherit pkgs; };
+
+      releaseTools.coverageAnalysis {
+        name = "nix-build";
+        src = tarball;
+
+        enableParallelBuilding = true;
+
+        buildInputs = buildDeps;
+
+        dontInstall = false;
+
+        doInstallCheck = true;
+
+        lcovFilter = [ "*/boost/*" "*-tab.*" ];
+
+        # We call `dot', and even though we just use it to
+        # syntax-check generated dot files, it still requires some
+        # fonts.  So provide those.
+        FONTCONFIG_FILE = texFunctions.fontsConf;
+
+        # To test building without precompiled headers.
+        makeFlagsArray = [ "PRECOMPILE_HEADERS=0" ];
+      };
+
+
+    # System tests.
+    tests.remoteBuilds = (import ./tests/remote-builds.nix rec {
+      inherit nixpkgs;
+      nix = build.x86_64-linux; system = "x86_64-linux";
+    });
+
+    tests.nix-copy-closure = (import ./tests/nix-copy-closure.nix rec {
+      inherit nixpkgs;
+      nix = build.x86_64-linux; system = "x86_64-linux";
+    });
+
+    tests.setuid = pkgs.lib.genAttrs
+      ["i686-linux" "x86_64-linux"]
+      (system:
+        import ./tests/setuid.nix rec {
+          inherit nixpkgs;
+          nix = build.${system}; inherit system;
+        });
+
+    tests.binaryTarball =
+      with import nixpkgs { system = "x86_64-linux"; };
+      vmTools.runInLinuxImage (runCommand "nix-binary-tarball-test"
+        { diskImage = vmTools.diskImages.ubuntu1204x86_64;
+        }
+        ''
+          set -x
+          useradd -m alice
+          su - alice -c 'tar xf ${binaryTarball.x86_64-linux}/*.tar.*'
+          mkdir /dest-nix
+          mount -o bind /dest-nix /nix # Provide a writable /nix.
+          chown alice /nix
+          su - alice -c '_NIX_INSTALLER_TEST=1 ./nix-*/install'
+          su - alice -c 'nix-store --verify'
+          su - alice -c 'PAGER= nix-store -qR ${build.x86_64-linux}'
+
+          # Check whether 'nix upgrade-nix' works.
+          cat > /tmp/paths.nix <<EOF
+          {
+            x86_64-linux = "${build.x86_64-linux}";
+          }
+          EOF
+          su - alice -c 'nix --experimental-features nix-command upgrade-nix -vvv --nix-store-paths-url file:///tmp/paths.nix'
+          (! [ -L /home/alice/.profile-1-link ])
+          su - alice -c 'PAGER= nix-store -qR ${build.x86_64-linux}'
+
+          mkdir -p $out/nix-support
+          touch $out/nix-support/hydra-build-products
+          umount /nix
+        ''); # */
+
+    /*
+    tests.evalNixpkgs =
+      import (nixpkgs + "/pkgs/top-level/make-tarball.nix") {
+        inherit nixpkgs;
+        inherit pkgs;
+        nix = build.x86_64-linux;
+        officialRelease = false;
+      };
+
+    tests.evalNixOS =
+      pkgs.runCommand "eval-nixos" { buildInputs = [ build.x86_64-linux ]; }
+        ''
+          export NIX_STATE_DIR=$TMPDIR
+
+          nix-instantiate ${nixpkgs}/nixos/release-combined.nix -A tested --dry-run \
+            --arg nixpkgs '{ outPath = ${nixpkgs}; revCount = 123; shortRev = "abcdefgh"; }'
+
+          touch $out
+        '';
+    */
+
+
+    installerScript =
+      pkgs.runCommand "installer-script"
+        { buildInputs = [ build.x86_64-linux ];
+        }
+        ''
+          mkdir -p $out/nix-support
+
+          substitute ${./scripts/install.in} $out/install \
+            ${pkgs.lib.concatMapStrings
+              (system: "--replace '@binaryTarball_${system}@' $(nix --experimental-features nix-command hash-file --base16 --type sha256 ${binaryTarball.${system}}/*.tar.xz) ")
+              [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ]
+            } \
+            --replace '@nixVersion@' ${build.x86_64-linux.src.version}
+
+          echo "file installer $out/install" >> $out/nix-support/hydra-build-products
+        '';
+
+
+    # Aggregate job containing the release-critical jobs.
+    release = pkgs.releaseTools.aggregate {
+      name = "nix-${tarball.version}";
+      meta.description = "Release-critical builds";
+      constituents =
+        [ tarball
+          build.i686-linux
+          build.x86_64-darwin
+          build.x86_64-linux
+          build.aarch64-linux
+          binaryTarball.i686-linux
+          binaryTarball.x86_64-darwin
+          binaryTarball.x86_64-linux
+          binaryTarball.aarch64-linux
+          tests.remoteBuilds
+          tests.nix-copy-closure
+          tests.binaryTarball
+          #tests.evalNixpkgs
+          #tests.evalNixOS
+          installerScript
+        ];
+    };
+
+  };
+
+
+in jobs

scripts/install-multi-user.sh

@@ -13,12 +13,12 @@ set -o pipefail
 # however tracking which bits came from which would be impossible.
 
 readonly ESC='\033[0m'
-readonly BOLD='\033[1m'
-readonly BLUE='\033[34m'
-readonly BLUE_UL='\033[4;34m'
-readonly GREEN='\033[32m'
-readonly GREEN_UL='\033[4;32m'
-readonly RED='\033[31m'
+readonly BOLD='\033[38;1m'
+readonly BLUE='\033[38;34m'
+readonly BLUE_UL='\033[38;4;34m'
+readonly GREEN='\033[38;32m'
+readonly GREEN_UL='\033[38;4;32m'
+readonly RED='\033[38;31m'
 
 readonly NIX_USER_COUNT="32"
 readonly NIX_BUILD_GROUP_ID="30000"
@@ -567,7 +567,7 @@ install_from_extracted_nix() {
         cd "$EXTRACTED_NIX_PATH"
 
         _sudo "to copy the basic Nix files to the new store at $NIX_ROOT/store" \
-              rsync -rlpt --chmod=-w ./store/* "$NIX_ROOT/store/"
+              rsync -rlpt ./store/* "$NIX_ROOT/store/"
 
         if [ -d "$NIX_INSTALLED_NIX" ]; then
             echo "      Alright! We have our first nix at $NIX_INSTALLED_NIX"

scripts/install-nix-from-closure.sh

@@ -87,7 +87,7 @@ if ! [ -e $dest ]; then
 fi
 
 if ! [ -w $dest ]; then
-    echo "$0: directory $dest exists, but is not writable by you. This could indicate that another user has already performed a single-user installation of Nix on this system. If you wish to enable multi-user support see https://nixos.org/nix/manual/#ssec-multi-user. If you wish to continue with a single-user install for $USER please run 'chown -R $USER $dest' as root." >&2
+    echo "$0: directory $dest exists, but is not writable by you. This could indicate that another user has already performed a single-user installation of Nix on this system. If you wish to enable multi-user support see http://nixos.org/nix/manual/#ssec-multi-user. If you wish to continue with a single-user install for $USER please run 'chown -R $USER $dest' as root." >&2
     exit 1
 fi
 

scripts/install-systemd-multi-user.sh

@@ -88,7 +88,7 @@ poly_configure_nix_daemon_service() {
           systemctl start nix-daemon.socket
 
     _sudo "to start the nix-daemon.service" \
-          systemctl restart nix-daemon.service
+          systemctl start nix-daemon.service
 
 }
 

scripts/install.in

@@ -30,13 +30,12 @@ case "$(uname -s).$(uname -m)" in
     *) oops "sorry, there is no binary distribution of Nix for your platform";;
 esac
 
-url="https://releases.nixos.org/nix/nix-@nixVersion@/nix-@nixVersion@-$system.tar.xz"
+url="https://nixos.org/releases/nix/nix-@nixVersion@/nix-@nixVersion@-$system.tar.xz"
 
 tarball="$tmpDir/$(basename "$tmpDir/nix-@nixVersion@-$system.tar.xz")"
 
 require_util curl "download the binary tarball"
 require_util tar "unpack the binary tarball"
-require_util xz "unpack the binary tarball"
 
 echo "downloading Nix @nixVersion@ binary tarball for $system from '$url' to '$tmpDir'..."
 curl -L "$url" -o "$tarball" || oops "failed to download '$url'"

shell.nix

@@ -1,3 +1,25 @@
-(import (fetchTarball https://github.com/edolstra/flake-compat/archive/master.tar.gz) {
-  src = builtins.fetchGit ./.;
-}).shellNix
+{ useClang ? false }:
+
+with import (builtins.fetchTarball https://github.com/NixOS/nixpkgs/archive/nixos-19.09.tar.gz) {};
+
+with import ./release-common.nix { inherit pkgs; };
+
+(if useClang then clangStdenv else stdenv).mkDerivation {
+  name = "nix";
+
+  buildInputs = buildDeps ++ tarballDeps ++ perlDeps ++ [ pkgs.rustfmt ];
+
+  inherit configureFlags;
+
+  enableParallelBuilding = true;
+
+  installFlags = "sysconfdir=$(out)/etc";
+
+  shellHook =
+    ''
+      export prefix=$(pwd)/inst
+      configureFlags+=" --prefix=$prefix"
+      PKG_CONFIG_PATH=$prefix/lib/pkgconfig:$PKG_CONFIG_PATH
+      PATH=$prefix/bin:$PATH
+    '';
+}

src/build-remote/build-remote.cc

@@ -17,7 +17,7 @@
 #include "store-api.hh"
 #include "derivations.hh"
 #include "local-store.hh"
-#include "../nix/legacy.hh"
+#include "legacy.hh"
 
 using namespace nix;
 using std::cin;

src/error-demo/error-demo.cc

@@ -1,66 +0,0 @@
-#include "error.hh"
-#include "nixexpr.hh"
-
-#include <iostream>
-#include <optional>
-
-int main()
-{
-    using namespace nix;
-
-    // In each program where errors occur, this has to be set.
-    ErrorInfo::programName = std::optional("error-demo");
-
-    // Error in a program; no hint and no nix code.
-    printErrorInfo(
-        ErrorInfo { .level = elError,
-                    .name = "name",
-                    .description = "error description",
-                  });
-
-    // Warning with name, description, and hint.
-    // The hintfmt function makes all the substituted text yellow.
-    printErrorInfo(
-        ErrorInfo { .level = elWarning,
-                    .name = "name",
-                    .description = "error description",
-                    .hint =  std::optional(
-                                 hintfmt("there was a %1%", "warning")),
-                  });
-
-
-    // Warning with nix file, line number, column, and the lines of
-    // code where a warning occurred.
-    SymbolTable testTable;
-    auto problem_file = testTable.create("myfile.nix");
-
-    printErrorInfo(
-        ErrorInfo{
-            .level = elWarning,
-            .name = "warning name",
-            .description = "warning description",
-            .hint = hintfmt("this hint has %1% templated %2%!!", "yellow", "values"),
-            .nixCode = NixCode {
-                .errPos = Pos(problem_file, 40, 13),
-                .prevLineOfCode = std::nullopt,
-                .errLineOfCode = "this is the problem line of code",
-                .nextLineOfCode = std::nullopt
-            }});
-
-    // Error with previous and next lines of code.
-    printErrorInfo(
-        ErrorInfo{
-            .level = elError,
-            .name = "error name",
-            .description = "error description",
-            .hint = hintfmt("this hint has %1% templated %2%!!", "yellow", "values"),
-            .nixCode = NixCode {
-                .errPos = Pos(problem_file, 40, 13),
-                .prevLineOfCode = std::optional("previous line of code"),
-                .errLineOfCode = "this is the problem line of code",
-                .nextLineOfCode = std::optional("next line of code"),
-            }});
-
-
-    return 0;
-}

src/error-demo/local.mk

@@ -1,12 +0,0 @@
-programs += error-demo
-
-error-demo_DIR := $(d)
-
-error-demo_SOURCES := \
-  $(wildcard $(d)/*.cc) \
-
-error-demo_CXXFLAGS += -I src/libutil -I src/libexpr
-
-error-demo_LIBS = libutil libexpr
-
-error-demo_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) -lboost_context -lboost_thread -lboost_system

src/libexpr/attr-path.cc

@@ -6,11 +6,11 @@
 namespace nix {
 
 
-static Strings parseAttrPath(std::string_view s)
+static Strings parseAttrPath(const string & s)
 {
     Strings res;
     string cur;
-    auto i = s.begin();
+    string::const_iterator i = s.begin();
     while (i != s.end()) {
         if (*i == '.') {
             res.push_back(cur);
@@ -32,22 +32,15 @@ static Strings parseAttrPath(std::string_view s)
 }
 
 
-std::vector<Symbol> parseAttrPath(EvalState & state, std::string_view s)
-{
-    std::vector<Symbol> res;
-    for (auto & a : parseAttrPath(s))
-        res.push_back(state.symbols.create(a));
-    return res;
-}
-
-
-std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attrPath,
+Ptr<Value> findAlongAttrPath(EvalState & state, const string & attrPath,
     Bindings & autoArgs, Value & vIn)
 {
     Strings tokens = parseAttrPath(attrPath);
 
-    Value * v = &vIn;
-    Pos pos = noPos;
+    Error attrError =
+        Error(format("attribute selection path '%1%' does not match expression") % attrPath);
+
+    Ptr<Value> v(&vIn);
 
     for (auto & attr : tokens) {
 
@@ -57,7 +50,7 @@ std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attr
         if (string2Int(attr, attrIndex)) apType = apIndex;
 
         /* Evaluate the expression. */
-        Value * vNew = state.allocValue();
+        auto vNew = state.allocValue();
         state.autoCallFunction(autoArgs, *v, *vNew);
         v = vNew;
         state.forceValue(*v);
@@ -77,9 +70,8 @@ std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attr
 
             Bindings::iterator a = v->attrs->find(state.symbols.create(attr));
             if (a == v->attrs->end())
-                throw AttrPathNotFound("attribute '%1%' in selection path '%2%' not found", attr, attrPath);
-            v = &*a->value;
-            pos = *a->pos;
+                throw Error(format("attribute '%1%' in selection path '%2%' not found") % attr % attrPath);
+            v = a->value;
         }
 
         else if (apType == apIndex) {
@@ -90,15 +82,14 @@ std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attr
                     % attrPath % showType(*v));
 
             if (attrIndex >= v->listSize())
-                throw AttrPathNotFound("list index %1% in selection path '%2%' is out of range", attrIndex, attrPath);
+                throw Error(format("list index %1% in selection path '%2%' is out of range") % attrIndex % attrPath);
 
             v = v->listElems()[attrIndex];
-            pos = noPos;
         }
 
     }
 
-    return {v, pos};
+    return v;
 }
 
 
@@ -106,10 +97,9 @@ Pos findDerivationFilename(EvalState & state, Value & v, std::string what)
 {
     Value * v2;
     try {
-        auto dummyArgs = state.allocBindings(0);
-        v2 = findAlongAttrPath(state, "meta.position", *dummyArgs, v).first;
+        v2 = findAlongAttrPath(state, "meta.position", *state.emptyBindings, v);
     } catch (Error &) {
-        throw NoPositionInfo("package '%s' has no source location information", what);
+        throw Error("package '%s' has no source location information", what);
     }
 
     // FIXME: is it possible to extract the Pos object instead of doing this

src/libexpr/attr-path.hh

@@ -7,15 +7,10 @@
 
 namespace nix {
 
-MakeError(AttrPathNotFound, Error);
-MakeError(NoPositionInfo, Error);
-
-std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attrPath,
+Ptr<Value> findAlongAttrPath(EvalState & state, const string & attrPath,
     Bindings & autoArgs, Value & vIn);
 
 /* Heuristic to find the filename and lineno or a nix value. */
 Pos findDerivationFilename(EvalState & state, Value & v, std::string what);
 
-std::vector<Symbol> parseAttrPath(EvalState & state, std::string_view s);
-
 }

src/libexpr/attr-set.cc

@@ -7,28 +7,28 @@
 namespace nix {
 
 
-/* Allocate a new array of attributes for an attribute set with a specific
-   capacity. The space is implicitly reserved after the Bindings
-   structure. */
-Bindings * EvalState::allocBindings(size_t capacity)
+/* Allocate a new array of attributes for an attribute set with a
+   specific capacity. The space is implicitly reserved after the
+   Bindings structure. */
+Ptr<Bindings> Bindings::allocBindings(size_t capacity)
 {
-    if (capacity > std::numeric_limits<Bindings::size_t>::max())
+    if (capacity >= 1UL << Object::miscBytes * 8)
         throw Error("attribute set of size %d is too big", capacity);
-    return new (allocBytes(sizeof(Bindings) + sizeof(Attr) * capacity)) Bindings((Bindings::size_t) capacity);
+    return gc.alloc<Bindings>(Bindings::wordsFor(capacity), capacity);
 }
 
 
 void EvalState::mkAttrs(Value & v, size_t capacity)
 {
     if (capacity == 0) {
-        v = vEmptySet;
-        return;
+        v.attrs = emptyBindings;
+        v.type = tAttrs;
+    } else {
+        v.attrs = Bindings::allocBindings(capacity);
+        v.type = tAttrs;
+        nrAttrsets++;
+        nrAttrsInAttrsets += capacity;
     }
-    clearValue(v);
-    v.type = tAttrs;
-    v.attrs = allocBindings(capacity);
-    nrAttrsets++;
-    nrAttrsInAttrsets += capacity;
 }
 
 
@@ -37,18 +37,12 @@ void EvalState::mkAttrs(Value & v, size_t capacity)
    this attribute. */
 Value * EvalState::allocAttr(Value & vAttrs, const Symbol & name)
 {
-    Value * v = allocValue();
+    auto v = allocValue();
     vAttrs.attrs->push_back(Attr(name, v));
     return v;
 }
 
 
-Value * EvalState::allocAttr(Value & vAttrs, const std::string & name)
-{
-    return allocAttr(vAttrs, symbols.create(name));
-}
-
-
 void Bindings::sort()
 {
     std::sort(begin(), end());

src/libexpr/attr-set.hh

@@ -2,6 +2,7 @@
 
 #include "nixexpr.hh"
 #include "symbol-table.hh"
+#include "gc.hh"
 
 #include <algorithm>
 #include <optional>
@@ -31,19 +32,22 @@ struct Attr
    by its size and its capacity, the capacity being the number of Attr
    elements allocated after this structure, while the size corresponds to
    the number of elements already inserted in this structure. */
-class Bindings
+class Bindings : public Object
 {
 public:
     typedef uint32_t size_t;
 
 private:
-    size_t size_, capacity_;
+    // FIXME: eliminate size_. We can just rely on capacity by sorting
+    // null entries towards the end of the vector.
+    size_t size_;
     Attr attrs[0];
 
-    Bindings(size_t capacity) : size_(0), capacity_(capacity) { }
+    Bindings(size_t capacity) : Object(tBindings, capacity), size_(0) {}
     Bindings(const Bindings & bindings) = delete;
 
 public:
+
     size_t size() const { return size_; }
 
     bool empty() const { return !size_; }
@@ -52,7 +56,8 @@ public:
 
     void push_back(const Attr & attr)
     {
-        assert(size_ < capacity_);
+        assert(size_ < capacity());
+        gc.assertObject(attr.value);
         attrs[size_++] = attr;
     }
 
@@ -90,7 +95,7 @@ public:
 
     void sort();
 
-    size_t capacity() { return capacity_; }
+    size_t capacity() const { return getMisc(); }
 
     /* Returns the attributes in lexicographically sorted order. */
     std::vector<const Attr *> lexicographicOrder() const
@@ -105,7 +110,19 @@ public:
         return res;
     }
 
-    friend class EvalState;
+    size_t words() const
+    {
+        return wordsFor(capacity());
+    }
+
+    static size_t wordsFor(size_t capacity)
+    {
+        return 2 + 3 * capacity; // FIXME
+    }
+
+    static Ptr<Bindings> allocBindings(size_t capacity);
+
+    friend class GC;
 };
 
 

src/libexpr/common-eval-args.cc

@@ -1,82 +1,56 @@
 #include "common-eval-args.hh"
 #include "shared.hh"
-#include "filetransfer.hh"
+#include "download.hh"
 #include "util.hh"
 #include "eval.hh"
-#include "fetchers.hh"
-#include "registry.hh"
-#include "flake/flakeref.hh"
-#include "store-api.hh"
 
 namespace nix {
 
 MixEvalArgs::MixEvalArgs()
 {
-    addFlag({
-        .longName = "arg",
-        .description = "argument to be passed to Nix functions",
-        .labels = {"name", "expr"},
-        .handler = {[&](std::string name, std::string expr) { autoArgs[name] = 'E' + expr; }}
-    });
+    mkFlag()
+        .longName("arg")
+        .description("argument to be passed to Nix functions")
+        .labels({"name", "expr"})
+        .handler([&](std::vector<std::string> ss) { autoArgs[ss[0]] = 'E' + ss[1]; });
 
-    addFlag({
-        .longName = "argstr",
-        .description = "string-valued argument to be passed to Nix functions",
-        .labels = {"name", "string"},
-        .handler = {[&](std::string name, std::string s) { autoArgs[name] = 'S' + s; }},
-    });
+    mkFlag()
+        .longName("argstr")
+        .description("string-valued argument to be passed to Nix functions")
+        .labels({"name", "string"})
+        .handler([&](std::vector<std::string> ss) { autoArgs[ss[0]] = 'S' + ss[1]; });
 
-    addFlag({
-        .longName = "include",
-        .shortName = 'I',
-        .description = "add a path to the list of locations used to look up <...> file names",
-        .labels = {"path"},
-        .handler = {[&](std::string s) { searchPath.push_back(s); }}
-    });
-
-    addFlag({
-        .longName = "impure",
-        .description = "allow access to mutable paths and repositories",
-        .handler = {[&]() {
-            evalSettings.pureEval = false;
-        }},
-    });
-
-    addFlag({
-        .longName = "override-flake",
-        .description = "override a flake registry value",
-        .labels = {"original-ref", "resolved-ref"},
-        .handler = {[&](std::string _from, std::string _to) {
-            auto from = parseFlakeRef(_from, absPath("."));
-            auto to = parseFlakeRef(_to, absPath("."));
-            fetchers::Attrs extraAttrs;
-            if (to.subdir != "") extraAttrs["dir"] = to.subdir;
-            fetchers::overrideRegistry(from.input, to.input, extraAttrs);
-        }}
-    });
+    mkFlag()
+        .shortName('I')
+        .longName("include")
+        .description("add a path to the list of locations used to look up <...> file names")
+        .label("path")
+        .handler([&](std::string s) { searchPath.push_back(s); });
 }
 
-Bindings * MixEvalArgs::getAutoArgs(EvalState & state)
+Ptr<Bindings> MixEvalArgs::getAutoArgs(EvalState & state)
 {
-    Bindings * res = state.allocBindings(autoArgs.size());
-    for (auto & i : autoArgs) {
-        Value * v = state.allocValue();
-        if (i.second[0] == 'E')
-            state.mkThunk_(*v, state.parseExprFromString(string(i.second, 1), absPath(".")));
-        else
-            mkString(*v, string(i.second, 1));
-        res->push_back(Attr(state.symbols.create(i.first), v));
+    if (!bindings) {
+        bindings = Bindings::allocBindings(autoArgs.size());
+        for (auto & i : autoArgs) {
+            Value * v = state.allocValue();
+            if (i.second[0] == 'E')
+                state.mkThunk_(*v, state.parseExprFromString(string(i.second, 1), absPath(".")));
+            else
+                mkString(*v, string(i.second, 1));
+            bindings->push_back(Attr(state.symbols.create(i.first), v));
+        }
+        bindings->sort();
     }
-    res->sort();
-    return res;
+    return bindings;
 }
 
 Path lookupFileArg(EvalState & state, string s)
 {
     if (isUri(s)) {
-        return state.store->toRealPath(
-            fetchers::downloadTarball(
-                state.store, resolveUri(s), "source", false).storePath);
+        CachedDownloadRequest request(s);
+        request.unpack = true;
+        return getDownloader()->downloadCached(state.store, request).path;
     } else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') {
         Path p = s.substr(1, s.size() - 2);
         return state.findFile(p);

src/libexpr/common-eval-args.hh

@@ -1,24 +1,28 @@
 #pragma once
 
 #include "args.hh"
+#include "eval.hh"
 
 namespace nix {
 
 class Store;
-class EvalState;
-class Bindings;
+
+template<class T>
+struct Ptr;
 
 struct MixEvalArgs : virtual Args
 {
     MixEvalArgs();
 
-    Bindings * getAutoArgs(EvalState & state);
+    Ptr<Bindings> getAutoArgs(EvalState & state);
 
     Strings searchPath;
 
 private:
 
     std::map<std::string, std::string> autoArgs;
+
+    Ptr<Bindings> bindings;
 };
 
 Path lookupFileArg(EvalState & state, string s);

src/libexpr/eval-cache.cc

@@ -1,503 +0,0 @@
-#include "eval-cache.hh"
-#include "sqlite.hh"
-#include "eval.hh"
-#include "eval-inline.hh"
-
-namespace nix::eval_cache {
-
-static const char * schema = R"sql(
-create table if not exists Attributes (
-    parent      integer not null,
-    name        text,
-    type        integer not null,
-    value       text,
-    primary key (parent, name)
-);
-)sql";
-
-struct AttrDb
-{
-    struct State
-    {
-        SQLite db;
-        SQLiteStmt insertAttribute;
-        SQLiteStmt queryAttribute;
-        SQLiteStmt queryAttributes;
-        std::unique_ptr<SQLiteTxn> txn;
-    };
-
-    std::unique_ptr<Sync<State>> _state;
-
-    AttrDb(const Hash & fingerprint)
-        : _state(std::make_unique<Sync<State>>())
-    {
-        auto state(_state->lock());
-
-        Path cacheDir = getCacheDir() + "/nix/eval-cache-v1";
-        createDirs(cacheDir);
-
-        Path dbPath = cacheDir + "/" + fingerprint.to_string(Base16, false) + ".sqlite";
-
-        state->db = SQLite(dbPath);
-        state->db.isCache();
-        state->db.exec(schema);
-
-        state->insertAttribute.create(state->db,
-            "insert or replace into Attributes(parent, name, type, value) values (?, ?, ?, ?)");
-
-        state->queryAttribute.create(state->db,
-            "select rowid, type, value from Attributes where parent = ? and name = ?");
-
-        state->queryAttributes.create(state->db,
-            "select name from Attributes where parent = ?");
-
-        state->txn = std::make_unique<SQLiteTxn>(state->db);
-    }
-
-    ~AttrDb()
-    {
-        try {
-            auto state(_state->lock());
-            state->txn->commit();
-            state->txn.reset();
-        } catch (...) {
-            ignoreException();
-        }
-    }
-
-    AttrId setAttrs(
-        AttrKey key,
-        const std::vector<Symbol> & attrs)
-    {
-        auto state(_state->lock());
-
-        state->insertAttribute.use()
-            (key.first)
-            (key.second)
-            (AttrType::FullAttrs)
-            (0, false).exec();
-
-        AttrId rowId = state->db.getLastInsertedRowId();
-        assert(rowId);
-
-        for (auto & attr : attrs)
-            state->insertAttribute.use()
-                (rowId)
-                (attr)
-                (AttrType::Placeholder)
-                (0, false).exec();
-
-        return rowId;
-    }
-
-    AttrId setString(
-        AttrKey key,
-        std::string_view s)
-    {
-        auto state(_state->lock());
-
-        state->insertAttribute.use()
-            (key.first)
-            (key.second)
-            (AttrType::String)
-            (s).exec();
-
-        return state->db.getLastInsertedRowId();
-    }
-
-    AttrId setBool(
-        AttrKey key,
-        bool b)
-    {
-        auto state(_state->lock());
-
-        state->insertAttribute.use()
-            (key.first)
-            (key.second)
-            (AttrType::Bool)
-            (b ? 1 : 0).exec();
-
-        return state->db.getLastInsertedRowId();
-    }
-
-    AttrId setPlaceholder(AttrKey key)
-    {
-        auto state(_state->lock());
-
-        state->insertAttribute.use()
-            (key.first)
-            (key.second)
-            (AttrType::Placeholder)
-            (0, false).exec();
-
-        return state->db.getLastInsertedRowId();
-    }
-
-    AttrId setMissing(AttrKey key)
-    {
-        auto state(_state->lock());
-
-        state->insertAttribute.use()
-            (key.first)
-            (key.second)
-            (AttrType::Missing)
-            (0, false).exec();
-
-        return state->db.getLastInsertedRowId();
-    }
-
-    AttrId setMisc(AttrKey key)
-    {
-        auto state(_state->lock());
-
-        state->insertAttribute.use()
-            (key.first)
-            (key.second)
-            (AttrType::Misc)
-            (0, false).exec();
-
-        return state->db.getLastInsertedRowId();
-    }
-
-    AttrId setFailed(AttrKey key)
-    {
-        auto state(_state->lock());
-
-        state->insertAttribute.use()
-            (key.first)
-            (key.second)
-            (AttrType::Failed)
-            (0, false).exec();
-
-        return state->db.getLastInsertedRowId();
-    }
-
-    std::optional<std::pair<AttrId, AttrValue>> getAttr(
-        AttrKey key,
-        SymbolTable & symbols)
-    {
-        auto state(_state->lock());
-
-        auto queryAttribute(state->queryAttribute.use()(key.first)(key.second));
-        if (!queryAttribute.next()) return {};
-
-        auto rowId = (AttrType) queryAttribute.getInt(0);
-        auto type = (AttrType) queryAttribute.getInt(1);
-
-        switch (type) {
-            case AttrType::Placeholder:
-                return {{rowId, placeholder_t()}};
-            case AttrType::FullAttrs: {
-                // FIXME: expensive, should separate this out.
-                std::vector<Symbol> attrs;
-                auto queryAttributes(state->queryAttributes.use()(rowId));
-                while (queryAttributes.next())
-                    attrs.push_back(symbols.create(queryAttributes.getStr(0)));
-                return {{rowId, attrs}};
-            }
-            case AttrType::String:
-                return {{rowId, queryAttribute.getStr(2)}};
-            case AttrType::Bool:
-                return {{rowId, queryAttribute.getInt(2) != 0}};
-            case AttrType::Missing:
-                return {{rowId, missing_t()}};
-            case AttrType::Misc:
-                return {{rowId, misc_t()}};
-            case AttrType::Failed:
-                return {{rowId, failed_t()}};
-            default:
-                throw Error("unexpected type in evaluation cache");
-        }
-    }
-};
-
-EvalCache::EvalCache(
-    bool useCache,
-    const Hash & fingerprint,
-    EvalState & state,
-    RootLoader rootLoader)
-    : db(useCache ? std::make_shared<AttrDb>(fingerprint) : nullptr)
-    , state(state)
-    , rootLoader(rootLoader)
-{
-}
-
-Value * EvalCache::getRootValue()
-{
-    if (!value) {
-        debug("getting root value");
-        value = allocRootValue(rootLoader());
-    }
-    return *value;
-}
-
-std::shared_ptr<AttrCursor> EvalCache::getRoot()
-{
-    return std::make_shared<AttrCursor>(ref(shared_from_this()), std::nullopt);
-}
-
-AttrCursor::AttrCursor(
-    ref<EvalCache> root,
-    Parent parent,
-    Value * value,
-    std::optional<std::pair<AttrId, AttrValue>> && cachedValue)
-    : root(root), parent(parent), cachedValue(std::move(cachedValue))
-{
-    if (value)
-        _value = allocRootValue(value);
-}
-
-AttrKey AttrCursor::getKey()
-{
-    if (!parent)
-        return {0, root->state.sEpsilon};
-    if (!parent->first->cachedValue) {
-        parent->first->cachedValue = root->db->getAttr(
-            parent->first->getKey(), root->state.symbols);
-        assert(parent->first->cachedValue);
-    }
-    return {parent->first->cachedValue->first, parent->second};
-}
-
-Value & AttrCursor::getValue()
-{
-    if (!_value) {
-        if (parent) {
-            auto & vParent = parent->first->getValue();
-            root->state.forceAttrs(vParent);
-            auto attr = vParent.attrs->get(parent->second);
-            if (!attr)
-                throw Error("attribute '%s' is unexpectedly missing", getAttrPathStr());
-            _value = allocRootValue(attr->value);
-        } else
-            _value = allocRootValue(root->getRootValue());
-    }
-    return **_value;
-}
-
-std::vector<Symbol> AttrCursor::getAttrPath() const
-{
-    if (parent) {
-        auto attrPath = parent->first->getAttrPath();
-        attrPath.push_back(parent->second);
-        return attrPath;
-    } else
-        return {};
-}
-
-std::vector<Symbol> AttrCursor::getAttrPath(Symbol name) const
-{
-    auto attrPath = getAttrPath();
-    attrPath.push_back(name);
-    return attrPath;
-}
-
-std::string AttrCursor::getAttrPathStr() const
-{
-    return concatStringsSep(".", getAttrPath());
-}
-
-std::string AttrCursor::getAttrPathStr(Symbol name) const
-{
-    return concatStringsSep(".", getAttrPath(name));
-}
-
-Value & AttrCursor::forceValue()
-{
-    debug("evaluating uncached attribute %s", getAttrPathStr());
-
-    auto & v = getValue();
-
-    try {
-        root->state.forceValue(v);
-    } catch (EvalError &) {
-        debug("setting '%s' to failed", getAttrPathStr());
-        if (root->db)
-            cachedValue = {root->db->setFailed(getKey()), failed_t()};
-        throw;
-    }
-
-    if (root->db && (!cachedValue || std::get_if<placeholder_t>(&cachedValue->second))) {
-        if (v.type == tString)
-            cachedValue = {root->db->setString(getKey(), v.string.s), v.string.s};
-        else if (v.type == tBool)
-            cachedValue = {root->db->setBool(getKey(), v.boolean), v.boolean};
-        else if (v.type == tAttrs)
-            ; // FIXME: do something?
-        else
-            cachedValue = {root->db->setMisc(getKey()), misc_t()};
-    }
-
-    return v;
-}
-
-std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(Symbol name)
-{
-    if (root->db) {
-        if (!cachedValue)
-            cachedValue = root->db->getAttr(getKey(), root->state.symbols);
-
-        if (cachedValue) {
-            if (auto attrs = std::get_if<std::vector<Symbol>>(&cachedValue->second)) {
-                for (auto & attr : *attrs)
-                    if (attr == name)
-                        return std::make_shared<AttrCursor>(root, std::make_pair(shared_from_this(), name));
-                return nullptr;
-            } else if (std::get_if<placeholder_t>(&cachedValue->second)) {
-                auto attr = root->db->getAttr({cachedValue->first, name}, root->state.symbols);
-                if (attr) {
-                    if (std::get_if<missing_t>(&attr->second))
-                        return nullptr;
-                    else if (std::get_if<failed_t>(&attr->second))
-                        throw EvalError("cached failure of attribute '%s'", getAttrPathStr(name));
-                    else
-                        return std::make_shared<AttrCursor>(root,
-                            std::make_pair(shared_from_this(), name), nullptr, std::move(attr));
-                }
-                // Incomplete attrset, so need to fall thru and
-                // evaluate to see whether 'name' exists
-            } else
-                return nullptr;
-                //throw TypeError("'%s' is not an attribute set", getAttrPathStr());
-        }
-    }
-
-    auto & v = forceValue();
-
-    if (v.type != tAttrs)
-        return nullptr;
-        //throw TypeError("'%s' is not an attribute set", getAttrPathStr());
-
-    auto attr = v.attrs->get(name);
-
-    if (!attr) {
-        if (root->db) {
-            if (!cachedValue)
-                cachedValue = {root->db->setPlaceholder(getKey()), placeholder_t()};
-            root->db->setMissing({cachedValue->first, name});
-        }
-        return nullptr;
-    }
-
-    std::optional<std::pair<AttrId, AttrValue>> cachedValue2;
-    if (root->db) {
-        if (!cachedValue)
-            cachedValue = {root->db->setPlaceholder(getKey()), placeholder_t()};
-        cachedValue2 = {root->db->setPlaceholder({cachedValue->first, name}), placeholder_t()};
-    }
-
-    return std::make_shared<AttrCursor>(
-        root, std::make_pair(shared_from_this(), name), attr->value, std::move(cachedValue2));
-}
-
-std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(std::string_view name)
-{
-    return maybeGetAttr(root->state.symbols.create(name));
-}
-
-std::shared_ptr<AttrCursor> AttrCursor::getAttr(Symbol name)
-{
-    auto p = maybeGetAttr(name);
-    if (!p)
-        throw Error("attribute '%s' does not exist", getAttrPathStr(name));
-    return p;
-}
-
-std::shared_ptr<AttrCursor> AttrCursor::getAttr(std::string_view name)
-{
-    return getAttr(root->state.symbols.create(name));
-}
-
-std::shared_ptr<AttrCursor> AttrCursor::findAlongAttrPath(const std::vector<Symbol> & attrPath)
-{
-    auto res = shared_from_this();
-    for (auto & attr : attrPath) {
-        res = res->maybeGetAttr(attr);
-        if (!res) return {};
-    }
-    return res;
-}
-
-std::string AttrCursor::getString()
-{
-    if (root->db) {
-        if (!cachedValue)
-            cachedValue = root->db->getAttr(getKey(), root->state.symbols);
-        if (cachedValue && !std::get_if<placeholder_t>(&cachedValue->second)) {
-            if (auto s = std::get_if<std::string>(&cachedValue->second)) {
-                debug("using cached string attribute '%s'", getAttrPathStr());
-                return *s;
-            } else
-                throw TypeError("'%s' is not a string", getAttrPathStr());
-        }
-    }
-
-    auto & v = forceValue();
-
-    if (v.type != tString)
-        throw TypeError("'%s' is not a string", getAttrPathStr());
-
-    return v.string.s;
-}
-
-bool AttrCursor::getBool()
-{
-    if (root->db) {
-        if (!cachedValue)
-            cachedValue = root->db->getAttr(getKey(), root->state.symbols);
-        if (cachedValue && !std::get_if<placeholder_t>(&cachedValue->second)) {
-            if (auto b = std::get_if<bool>(&cachedValue->second)) {
-                debug("using cached Boolean attribute '%s'", getAttrPathStr());
-                return *b;
-            } else
-                throw TypeError("'%s' is not a Boolean", getAttrPathStr());
-        }
-    }
-
-    auto & v = forceValue();
-
-    if (v.type != tBool)
-        throw TypeError("'%s' is not a Boolean", getAttrPathStr());
-
-    return v.boolean;
-}
-
-std::vector<Symbol> AttrCursor::getAttrs()
-{
-    if (root->db) {
-        if (!cachedValue)
-            cachedValue = root->db->getAttr(getKey(), root->state.symbols);
-        if (cachedValue && !std::get_if<placeholder_t>(&cachedValue->second)) {
-            if (auto attrs = std::get_if<std::vector<Symbol>>(&cachedValue->second)) {
-                debug("using cached attrset attribute '%s'", getAttrPathStr());
-                return *attrs;
-            } else
-                throw TypeError("'%s' is not an attribute set", getAttrPathStr());
-        }
-    }
-
-    auto & v = forceValue();
-
-    if (v.type != tAttrs)
-        throw TypeError("'%s' is not an attribute set", getAttrPathStr());
-
-    std::vector<Symbol> attrs;
-    for (auto & attr : *getValue().attrs)
-        attrs.push_back(attr.name);
-    std::sort(attrs.begin(), attrs.end(), [](const Symbol & a, const Symbol & b) {
-        return (const string &) a < (const string &) b;
-    });
-
-    if (root->db)
-        cachedValue = {root->db->setAttrs(getKey(), attrs), attrs};
-
-    return attrs;
-}
-
-bool AttrCursor::isDerivation()
-{
-    auto aType = maybeGetAttr("type");
-    return aType && aType->getString() == "derivation";
-}
-
-}

src/libexpr/eval-cache.hh

@@ -1,106 +0,0 @@
-#pragma once
-
-#include "sync.hh"
-#include "hash.hh"
-#include "eval.hh"
-
-#include <variant>
-
-namespace nix::eval_cache {
-
-class AttrDb;
-class AttrCursor;
-
-class EvalCache : public std::enable_shared_from_this<EvalCache>
-{
-    friend class AttrCursor;
-
-    std::shared_ptr<AttrDb> db;
-    EvalState & state;
-    typedef std::function<Value *()> RootLoader;
-    RootLoader rootLoader;
-    RootValue value;
-
-    Value * getRootValue();
-
-public:
-
-    EvalCache(
-        bool useCache,
-        const Hash & fingerprint,
-        EvalState & state,
-        RootLoader rootLoader);
-
-    std::shared_ptr<AttrCursor> getRoot();
-};
-
-enum AttrType {
-    Placeholder = 0,
-    FullAttrs = 1,
-    String = 2,
-    Missing = 3,
-    Misc = 4,
-    Failed = 5,
-    Bool = 6,
-};
-
-struct placeholder_t {};
-struct missing_t {};
-struct misc_t {};
-struct failed_t {};
-typedef uint64_t AttrId;
-typedef std::pair<AttrId, Symbol> AttrKey;
-typedef std::variant<std::vector<Symbol>, std::string, placeholder_t, missing_t, misc_t, failed_t, bool> AttrValue;
-
-class AttrCursor : public std::enable_shared_from_this<AttrCursor>
-{
-    friend class EvalCache;
-
-    ref<EvalCache> root;
-    typedef std::optional<std::pair<std::shared_ptr<AttrCursor>, Symbol>> Parent;
-    Parent parent;
-    RootValue _value;
-    std::optional<std::pair<AttrId, AttrValue>> cachedValue;
-
-    AttrKey getKey();
-
-    Value & getValue();
-
-public:
-
-    AttrCursor(
-        ref<EvalCache> root,
-        Parent parent,
-        Value * value = nullptr,
-        std::optional<std::pair<AttrId, AttrValue>> && cachedValue = {});
-
-    std::vector<Symbol> getAttrPath() const;
-
-    std::vector<Symbol> getAttrPath(Symbol name) const;
-
-    std::string getAttrPathStr() const;
-
-    std::string getAttrPathStr(Symbol name) const;
-
-    std::shared_ptr<AttrCursor> maybeGetAttr(Symbol name);
-
-    std::shared_ptr<AttrCursor> maybeGetAttr(std::string_view name);
-
-    std::shared_ptr<AttrCursor> getAttr(Symbol name);
-
-    std::shared_ptr<AttrCursor> getAttr(std::string_view name);
-
-    std::shared_ptr<AttrCursor> findAlongAttrPath(const std::vector<Symbol> & attrPath);
-
-    std::string getString();
-
-    bool getBool();
-
-    std::vector<Symbol> getAttrs();
-
-    bool isDerivation();
-
-    Value & forceValue();
-};
-
-}

src/libexpr/eval-inline.hh

@@ -27,21 +27,27 @@ LocalNoInlineNoReturn(void throwTypeError(const char * s, const Value & v, const
 void EvalState::forceValue(Value & v, const Pos & pos)
 {
     if (v.type == tThunk) {
-        Env * env = v.thunk.env;
+        // FIXME: this is necessary because some values (like vList2)
+        // are created non-atomically.
+        Ptr<Env> env(v.thunk.env);
         Expr * expr = v.thunk.expr;
         try {
             v.type = tBlackhole;
             //checkInterrupt();
             expr->eval(*this, *env, v);
         } catch (...) {
-            v.type = tThunk;
             v.thunk.env = env;
             v.thunk.expr = expr;
+            v.type = tThunk;
             throw;
         }
     }
-    else if (v.type == tApp)
+    else if (v.type == tApp) {
+        // FIXME: idem.
+        Ptr<Value> left(v.app.left);
+        Ptr<Value> right(v.app.right);
         callFunction(*v.app.left, *v.app.right, v, noPos);
+    }
     else if (v.type == tBlackhole)
         throwEvalError("infinite recursion encountered, at %1%", pos);
 }
@@ -57,7 +63,7 @@ inline void EvalState::forceAttrs(Value & v)
 
 inline void EvalState::forceAttrs(Value & v, const Pos & pos)
 {
-    forceValue(v, pos);
+    forceValue(v);
     if (v.type != tAttrs)
         throwTypeError("value is %1% while a set was expected, at %2%", v, pos);
 }
@@ -73,23 +79,10 @@ inline void EvalState::forceList(Value & v)
 
 inline void EvalState::forceList(Value & v, const Pos & pos)
 {
-    forceValue(v, pos);
+    forceValue(v);
     if (!v.isList())
         throwTypeError("value is %1% while a list was expected, at %2%", v, pos);
 }
 
-/* Note: Various places expect the allocated memory to be zeroed. */
-inline void * allocBytes(size_t n)
-{
-    void * p;
-#if HAVE_BOEHMGC
-    p = GC_MALLOC(n);
-#else
-    p = calloc(n, 1);
-#endif
-    if (!p) throw std::bad_alloc();
-    return p;
-}
-
 
 }

src/libexpr/eval.hh

@@ -4,13 +4,13 @@
 #include "value.hh"
 #include "nixexpr.hh"
 #include "symbol-table.hh"
+#include "hash.hh"
 #include "config.hh"
+#include "gc.hh"
 
-#include <regex>
 #include <map>
 #include <optional>
 #include <unordered_map>
-#include <mutex>
 
 
 namespace nix {
@@ -18,6 +18,7 @@ namespace nix {
 
 class Store;
 class EvalState;
+struct Derivation; // FIXME: remove
 struct StorePath;
 enum RepairFlag : bool;
 
@@ -35,18 +36,56 @@ struct PrimOp
 };
 
 
-struct Env
+struct Env : Object
 {
     Env * up;
-    unsigned short prevWith:14; // nr of levels up to next `with' environment
-    enum { Plain = 0, HasWithExpr, HasWithAttrs } type:2;
     Value * values[0];
+
+private:
+
+    constexpr static size_t maxSize = 1 << 16;
+    constexpr static size_t maxPrevWith = 1 << 10;
+
+    Env(Tag type, size_t size, size_t prevWith)
+        : Object(type, size | (prevWith << 16))
+    {
+        if (size >= maxSize)
+            throw Error("environment size %d is too big", size);
+
+        if (prevWith >= maxPrevWith)
+            throw Error("too many nesting levels");
+
+        for (size_t i = 0; i < size; i++)
+            values[i] = nullptr;
+    }
+
+    friend class GC;
+
+public:
+
+    unsigned short getPrevWith() const
+    {
+        return getMisc() >> 16;
+    }
+
+    unsigned short getSize() const
+    {
+        return getMisc() & 0xffff;
+    }
+
+    size_t words() const
+    {
+        return wordsFor(getSize());
+    }
+
+    static size_t wordsFor(unsigned short size)
+    {
+        return 2 + size;
+    }
 };
 
 
-Value & mkString(Value & v, std::string_view s, const PathSet & context = PathSet());
-
-void copyContext(const Value & v, PathSet & context);
+Value & mkString(Value & v, std::string_view s, const PathSet & context);
 
 
 /* Cache for calls to addToStore(); maps source paths to the store
@@ -61,21 +100,16 @@ typedef std::pair<std::string, std::string> SearchPathElem;
 typedef std::list<SearchPathElem> SearchPath;
 
 
-/* Initialise the Boehm GC, if applicable. */
-void initGC();
-
-
 class EvalState
 {
 public:
-    SymbolTable symbols;
+    SymbolTable & symbols{nix::symbols}; // FIXME: remove
 
     const Symbol sWith, sOutPath, sDrvPath, sType, sMeta, sName, sValue,
         sSystem, sOverrides, sOutputs, sOutputName, sIgnoreNulls,
         sFile, sLine, sColumn, sFunctor, sToString,
         sRight, sWrong, sStructuredAttrs, sBuilder, sArgs,
-        sOutputHash, sOutputHashAlgo, sOutputHashMode,
-        sDescription, sSelf, sEpsilon, sRecurseForDerivations;
+        sOutputHash, sOutputHashAlgo, sOutputHashMode;
     Symbol sDerivationNix;
 
     /* If set, force copying files to the Nix store even if they
@@ -86,28 +120,19 @@ public:
        mode. */
     std::optional<PathSet> allowedPaths;
 
-    Value vEmptySet;
+    Ptr<Bindings> emptyBindings;
 
     const ref<Store> store;
 
-
 private:
     SrcToStore srcToStore;
 
     /* A cache from path names to parse trees. */
-#if HAVE_BOEHMGC
-    typedef std::map<Path, Expr *, std::less<Path>, traceable_allocator<std::pair<const Path, Expr *> > > FileParseCache;
-#else
     typedef std::map<Path, Expr *> FileParseCache;
-#endif
     FileParseCache fileParseCache;
 
     /* A cache from path names to values. */
-#if HAVE_BOEHMGC
-    typedef std::map<Path, Value, std::less<Path>, traceable_allocator<std::pair<const Path, Value> > > FileEvalCache;
-#else
-    typedef std::map<Path, Value> FileEvalCache;
-#endif
+    typedef std::map<Path, Ptr<Value>> FileEvalCache;
     FileEvalCache fileEvalCache;
 
     SearchPath searchPath;
@@ -117,9 +142,6 @@ private:
     /* Cache used by checkSourcePath(). */
     std::unordered_map<Path, Path> resolvedPaths;
 
-    /* Cache used by prim_match(). */
-    std::unordered_map<std::string, std::regex> regexCache;
-
 public:
 
     EvalState(const Strings & _searchPath, ref<Store> store);
@@ -147,15 +169,14 @@ public:
     Expr * parseExprFromFile(const Path & path, StaticEnv & staticEnv);
 
     /* Parse a Nix expression from the specified string. */
-    Expr * parseExprFromString(std::string_view s, const Path & basePath, StaticEnv & staticEnv);
-    Expr * parseExprFromString(std::string_view s, const Path & basePath);
+    Expr * parseExprFromString(const string & s, const Path & basePath, StaticEnv & staticEnv);
+    Expr * parseExprFromString(const string & s, const Path & basePath);
 
     Expr * parseStdin();
 
     /* Evaluate an expression read from the given file to normal
-       form. Optionally enforce that the top-level expression is
-       trivial (i.e. doesn't require arbitrary computation). */
-    void evalFile(const Path & path, Value & v, bool mustBeTrivial = false);
+       form. */
+    void evalFile(const Path & path, Value & v);
 
     void resetFileCache();
 
@@ -224,7 +245,7 @@ public:
 
     /* The base environment, containing the builtin functions and
        values. */
-    Env & baseEnv;
+    Ptr<Env> baseEnv;
 
     /* The same, but used during parsing to resolve variables. */
     StaticEnv staticBaseEnv; // !!! should be private
@@ -271,13 +292,17 @@ public:
     void autoCallFunction(Bindings & args, Value & fun, Value & res);
 
     /* Allocation primitives. */
-    Value * allocValue();
-    Env & allocEnv(size_t size);
+    Ptr<Value> allocValue()
+    {
+        nrValues++;
+        return gc.alloc<Value>(Value::words());
+    }
 
+    Ptr<Env> allocEnv(size_t size, size_t prevWith = 0, Tag type = tEnv);
+
+    // Note: the resulting Value is only reachable as long as vAttrs
+    // is reachable.
     Value * allocAttr(Value & vAttrs, const Symbol & name);
-    Value * allocAttr(Value & vAttrs, const std::string & name);
-
-    Bindings * allocBindings(size_t capacity);
 
     void mkList(Value & v, size_t length);
     void mkAttrs(Value & v, size_t capacity);
@@ -322,12 +347,14 @@ private:
     friend struct ExprOpConcatLists;
     friend struct ExprSelect;
     friend void prim_getAttr(EvalState & state, const Pos & pos, Value * * args, Value & v);
-    friend void prim_match(EvalState & state, const Pos & pos, Value * * args, Value & v);
+
+public:
+
+    std::function<void(const StorePath & drvPath, const Derivation & drv)> derivationHook;
 };
 
 
 /* Return a string representing the type of the value `v'. */
-string showType(ValueType type);
 string showType(const Value & v);
 
 /* Decode a context string ‘!<name>!<path>’ into a pair <path,
@@ -372,7 +399,7 @@ struct EvalSettings : Config
         "Prefixes of URIs that builtin functions such as fetchurl and fetchGit are allowed to fetch."};
 
     Setting<bool> traceFunctionCalls{this, false, "trace-function-calls",
-        "Emit log messages for each function entry and exit at the 'vomit' log level (-vvvv)."};
+        "Emit log messages for each function entry and exit at the 'vomit' log level (-vvvv)"};
 };
 
 extern EvalSettings evalSettings;

src/libexpr/flake/call-flake.nix

@@ -1,29 +0,0 @@
-lockFileStr: rootSrc: rootSubdir:
-
-let
-
-  lockFile = builtins.fromJSON lockFileStr;
-
-  allNodes =
-    builtins.mapAttrs
-      (key: node:
-        let
-          sourceInfo =
-            if key == lockFile.root
-            then rootSrc
-            else fetchTree ({ inherit (node.info) narHash; } // removeAttrs node.locked ["dir"]);
-          subdir = if key == lockFile.root then rootSubdir else node.locked.dir or "";
-          flake = import (sourceInfo + (if subdir != "" then "/" else "") + subdir + "/flake.nix");
-          inputs = builtins.mapAttrs (inputName: key: allNodes.${key}) (node.inputs or {});
-          outputs = flake.outputs (inputs // { self = result; });
-          result = outputs // sourceInfo // { inherit inputs; inherit outputs; inherit sourceInfo; };
-        in
-          if node.flake or true then
-            assert builtins.isFunction flake.outputs;
-            result
-          else
-            sourceInfo
-      )
-      lockFile.nodes;
-
-in allNodes.${lockFile.root}

src/libexpr/flake/flake.cc

@@ -1,660 +0,0 @@
-#include "flake.hh"
-#include "lockfile.hh"
-#include "primops.hh"
-#include "eval-inline.hh"
-#include "store-api.hh"
-#include "fetchers.hh"
-#include "finally.hh"
-
-namespace nix {
-
-using namespace flake;
-
-namespace flake {
-
-/* If 'allowLookup' is true, then resolve 'flakeRef' using the
-   registries. */
-static FlakeRef maybeLookupFlake(
-    ref<Store> store,
-    const FlakeRef & flakeRef,
-    bool allowLookup)
-{
-    if (!flakeRef.input->isDirect()) {
-        if (allowLookup)
-            return flakeRef.resolve(store);
-        else
-            throw Error("'%s' is an indirect flake reference, but registry lookups are not allowed", flakeRef);
-    } else
-        return flakeRef;
-}
-
-typedef std::vector<std::pair<FlakeRef, FlakeRef>> FlakeCache;
-
-static FlakeRef lookupInFlakeCache(
-    const FlakeCache & flakeCache,
-    const FlakeRef & flakeRef)
-{
-    // FIXME: inefficient.
-    for (auto & i : flakeCache) {
-        if (flakeRef == i.first) {
-            debug("mapping '%s' to previously seen input '%s' -> '%s",
-                flakeRef, i.first, i.second);
-            return i.second;
-        }
-    }
-
-    return flakeRef;
-}
-
-static std::tuple<fetchers::Tree, FlakeRef, FlakeRef> fetchOrSubstituteTree(
-    EvalState & state,
-    const FlakeRef & originalRef,
-    std::optional<TreeInfo> treeInfo,
-    bool allowLookup,
-    FlakeCache & flakeCache)
-{
-    /* The tree may already be in the Nix store, or it could be
-       substituted (which is often faster than fetching from the
-       original source). So check that. */
-    if (treeInfo && originalRef.input->isDirect() && originalRef.input->isImmutable()) {
-        try {
-            auto storePath = treeInfo->computeStorePath(*state.store);
-
-            state.store->ensurePath(storePath);
-
-            debug("using substituted/cached input '%s' in '%s'",
-                originalRef, state.store->printStorePath(storePath));
-
-            auto actualPath = state.store->toRealPath(storePath);
-
-            if (state.allowedPaths)
-                state.allowedPaths->insert(actualPath);
-
-            return {
-                Tree {
-                    .actualPath = actualPath,
-                    .storePath = std::move(storePath),
-                    .info = *treeInfo,
-                },
-                originalRef,
-                originalRef
-            };
-        } catch (Error & e) {
-            debug("substitution of input '%s' failed: %s", originalRef, e.what());
-        }
-    }
-
-    auto resolvedRef = lookupInFlakeCache(flakeCache,
-        maybeLookupFlake(state.store,
-            lookupInFlakeCache(flakeCache, originalRef), allowLookup));
-
-    auto [tree, lockedRef] = resolvedRef.fetchTree(state.store);
-
-    debug("got tree '%s' from '%s'",
-        state.store->printStorePath(tree.storePath), lockedRef);
-
-    flakeCache.push_back({originalRef, lockedRef});
-    flakeCache.push_back({resolvedRef, lockedRef});
-
-    if (state.allowedPaths)
-        state.allowedPaths->insert(tree.actualPath);
-
-    if (treeInfo)
-        assert(tree.storePath == treeInfo->computeStorePath(*state.store));
-
-    return {std::move(tree), resolvedRef, lockedRef};
-}
-
-static void expectType(EvalState & state, ValueType type,
-    Value & value, const Pos & pos)
-{
-    if (value.type == tThunk && value.isTrivial())
-        state.forceValue(value, pos);
-    if (value.type != type)
-        throw Error("expected %s but got %s at %s",
-            showType(type), showType(value.type), pos);
-}
-
-static std::map<FlakeId, FlakeInput> parseFlakeInputs(
-    EvalState & state, Value * value, const Pos & pos);
-
-static FlakeInput parseFlakeInput(EvalState & state,
-    const std::string & inputName, Value * value, const Pos & pos)
-{
-    expectType(state, tAttrs, *value, pos);
-
-    FlakeInput input {
-        .ref = FlakeRef::fromAttrs({{"type", "indirect"}, {"id", inputName}})
-    };
-
-    auto sInputs = state.symbols.create("inputs");
-    auto sUrl = state.symbols.create("url");
-    auto sFlake = state.symbols.create("flake");
-    auto sFollows = state.symbols.create("follows");
-
-    fetchers::Attrs attrs;
-    std::optional<std::string> url;
-
-    for (nix::Attr attr : *(value->attrs)) {
-        try {
-            if (attr.name == sUrl) {
-                expectType(state, tString, *attr.value, *attr.pos);
-                url = attr.value->string.s;
-                attrs.emplace("url", *url);
-            } else if (attr.name == sFlake) {
-                expectType(state, tBool, *attr.value, *attr.pos);
-                input.isFlake = attr.value->boolean;
-            } else if (attr.name == sInputs) {
-                input.overrides = parseFlakeInputs(state, attr.value, *attr.pos);
-            } else if (attr.name == sFollows) {
-                expectType(state, tString, *attr.value, *attr.pos);
-                input.follows = parseInputPath(attr.value->string.s);
-            } else {
-                state.forceValue(*attr.value);
-                if (attr.value->type == tString)
-                    attrs.emplace(attr.name, attr.value->string.s);
-                else
-                    throw TypeError("flake input attribute '%s' is %s while a string is expected",
-                        attr.name, showType(*attr.value));
-            }
-        } catch (Error & e) {
-            e.addPrefix(fmt("in flake attribute '%s' at '%s':\n", attr.name, *attr.pos));
-            throw;
-        }
-    }
-
-    if (attrs.count("type"))
-        try {
-            input.ref = FlakeRef::fromAttrs(attrs);
-        } catch (Error & e) {
-            e.addPrefix(fmt("in flake input at '%s':\n", pos));
-            throw;
-        }
-    else {
-        attrs.erase("url");
-        if (!attrs.empty())
-            throw Error("unexpected flake input attribute '%s', at %s", attrs.begin()->first, pos);
-        if (url)
-            input.ref = parseFlakeRef(*url, {}, true);
-    }
-
-    return input;
-}
-
-static std::map<FlakeId, FlakeInput> parseFlakeInputs(
-    EvalState & state, Value * value, const Pos & pos)
-{
-    std::map<FlakeId, FlakeInput> inputs;
-
-    expectType(state, tAttrs, *value, pos);
-
-    for (nix::Attr & inputAttr : *(*value).attrs) {
-        inputs.emplace(inputAttr.name,
-            parseFlakeInput(state,
-                inputAttr.name,
-                inputAttr.value,
-                *inputAttr.pos));
-    }
-
-    return inputs;
-}
-
-static Flake getFlake(
-    EvalState & state,
-    const FlakeRef & originalRef,
-    std::optional<TreeInfo> treeInfo,
-    bool allowLookup,
-    FlakeCache & flakeCache)
-{
-    auto [sourceInfo, resolvedRef, lockedRef] = fetchOrSubstituteTree(
-        state, originalRef, treeInfo, allowLookup, flakeCache);
-
-    // Guard against symlink attacks.
-    auto flakeFile = canonPath(sourceInfo.actualPath + "/" + lockedRef.subdir + "/flake.nix");
-    if (!isInDir(flakeFile, sourceInfo.actualPath))
-        throw Error("'flake.nix' file of flake '%s' escapes from '%s'",
-            lockedRef, state.store->printStorePath(sourceInfo.storePath));
-
-    Flake flake {
-        .originalRef = originalRef,
-        .resolvedRef = resolvedRef,
-        .lockedRef = lockedRef,
-        .sourceInfo = std::make_shared<fetchers::Tree>(std::move(sourceInfo))
-    };
-
-    if (!pathExists(flakeFile))
-        throw Error("source tree referenced by '%s' does not contain a '%s/flake.nix' file", lockedRef, lockedRef.subdir);
-
-    Value vInfo;
-    state.evalFile(flakeFile, vInfo, true); // FIXME: symlink attack
-
-    expectType(state, tAttrs, vInfo, Pos(state.symbols.create(flakeFile), 0, 0));
-
-    auto sEdition = state.symbols.create("edition"); // FIXME: remove soon
-
-    if (vInfo.attrs->get(sEdition))
-        warn("flake '%s' has deprecated attribute 'edition'", lockedRef);
-
-    if (auto description = vInfo.attrs->get(state.sDescription)) {
-        expectType(state, tString, *description->value, *description->pos);
-        flake.description = description->value->string.s;
-    }
-
-    auto sInputs = state.symbols.create("inputs");
-
-    if (auto inputs = vInfo.attrs->get(sInputs))
-        flake.inputs = parseFlakeInputs(state, inputs->value, *inputs->pos);
-
-    auto sOutputs = state.symbols.create("outputs");
-
-    if (auto outputs = vInfo.attrs->get(sOutputs)) {
-        expectType(state, tLambda, *outputs->value, *outputs->pos);
-        flake.vOutputs = allocRootValue(outputs->value);
-
-        if ((*flake.vOutputs)->lambda.fun->matchAttrs) {
-            for (auto & formal : (*flake.vOutputs)->lambda.fun->formals->formals) {
-                if (formal.name != state.sSelf)
-                    flake.inputs.emplace(formal.name, FlakeInput {
-                        .ref = parseFlakeRef(formal.name)
-                    });
-            }
-        }
-
-    } else
-        throw Error("flake '%s' lacks attribute 'outputs'", lockedRef);
-
-    for (auto & attr : *vInfo.attrs) {
-        if (attr.name != sEdition &&
-            attr.name != state.sDescription &&
-            attr.name != sInputs &&
-            attr.name != sOutputs)
-            throw Error("flake '%s' has an unsupported attribute '%s', at %s",
-                lockedRef, attr.name, *attr.pos);
-    }
-
-    return flake;
-}
-
-Flake getFlake(EvalState & state, const FlakeRef & originalRef, bool allowLookup)
-{
-    FlakeCache flakeCache;
-    return getFlake(state, originalRef, {}, allowLookup, flakeCache);
-}
-
-/* Compute an in-memory lock file for the specified top-level flake,
-   and optionally write it to file, it the flake is writable. */
-LockedFlake lockFlake(
-    EvalState & state,
-    const FlakeRef & topRef,
-    const LockFlags & lockFlags)
-{
-    settings.requireExperimentalFeature("flakes");
-
-    FlakeCache flakeCache;
-
-    auto flake = getFlake(state, topRef, {}, lockFlags.useRegistries, flakeCache);
-
-    // FIXME: symlink attack
-    auto oldLockFile = LockFile::read(
-        flake.sourceInfo->actualPath + "/" + flake.lockedRef.subdir + "/flake.lock");
-
-    debug("old lock file: %s", oldLockFile);
-
-    // FIXME: check whether all overrides are used.
-    std::map<InputPath, FlakeInput> overrides;
-    std::set<InputPath> overridesUsed, updatesUsed;
-
-    for (auto & i : lockFlags.inputOverrides)
-        overrides.insert_or_assign(i.first, FlakeInput { .ref = i.second });
-
-    LockFile newLockFile;
-
-    std::vector<FlakeRef> parents;
-    std::map<InputPath, InputPath> follows;
-
-    std::function<void(
-        const FlakeInputs & flakeInputs,
-        std::shared_ptr<Node> node,
-        const InputPath & inputPathPrefix,
-        std::shared_ptr<const Node> oldNode)>
-        computeLocks;
-
-    computeLocks = [&](
-        const FlakeInputs & flakeInputs,
-        std::shared_ptr<Node> node,
-        const InputPath & inputPathPrefix,
-        std::shared_ptr<const Node> oldNode)
-    {
-        debug("computing lock file node '%s'", concatStringsSep("/", inputPathPrefix));
-
-        /* Get the overrides (i.e. attributes of the form
-           'inputs.nixops.inputs.nixpkgs.url = ...'). */
-        // FIXME: check this
-        for (auto & [id, input] : flake.inputs) {
-            for (auto & [idOverride, inputOverride] : input.overrides) {
-                auto inputPath(inputPathPrefix);
-                inputPath.push_back(id);
-                inputPath.push_back(idOverride);
-                overrides.insert_or_assign(inputPath, inputOverride);
-            }
-        }
-
-        /* Go over the flake inputs, resolve/fetch them if
-           necessary (i.e. if they're new or the flakeref changed
-           from what's in the lock file). */
-        for (auto & [id, input2] : flakeInputs) {
-            auto inputPath(inputPathPrefix);
-            inputPath.push_back(id);
-            auto inputPathS = concatStringsSep("/", inputPath);
-            debug("computing input '%s'", concatStringsSep("/", inputPath));
-
-            /* Do we have an override for this input from one of the
-               ancestors? */
-            auto i = overrides.find(inputPath);
-            bool hasOverride = i != overrides.end();
-            if (hasOverride) overridesUsed.insert(inputPath);
-            auto & input = hasOverride ? i->second : input2;
-
-            /* Resolve 'follows' later (since it may refer to an input
-               path we haven't processed yet. */
-            if (input.follows) {
-                if (hasOverride)
-                    /* 'follows' from an override is relative to the
-                       root of the graph. */
-                    follows.insert_or_assign(inputPath, *input.follows);
-                else {
-                    /* Otherwise, it's relative to the current flake. */
-                    InputPath path(inputPathPrefix);
-                    for (auto & i : *input.follows) path.push_back(i);
-                    follows.insert_or_assign(inputPath, path);
-                }
-                continue;
-            }
-
-            /* Do we have an entry in the existing lock file? And we
-               don't have a --update-input flag for this input? */
-            std::shared_ptr<const LockedNode> oldLock;
-
-            updatesUsed.insert(inputPath);
-
-            if (oldNode && !lockFlags.inputUpdates.count(inputPath)) {
-                auto oldLockIt = oldNode->inputs.find(id);
-                if (oldLockIt != oldNode->inputs.end())
-                    oldLock = std::dynamic_pointer_cast<const LockedNode>(oldLockIt->second);
-            }
-
-            if (oldLock
-                && oldLock->originalRef == input.ref
-                && !hasOverride)
-            {
-                debug("keeping existing input '%s'", inputPathS);
-
-                /* Copy the input from the old lock since its flakeref
-                   didn't change and there is no override from a
-                   higher level flake. */
-                auto childNode = std::make_shared<LockedNode>(
-                    oldLock->lockedRef, oldLock->originalRef, oldLock->info, oldLock->isFlake);
-
-                node->inputs.insert_or_assign(id, childNode);
-
-                /* If we have an --update-input flag for an input
-                   of this input, then we must fetch the flake to
-                   to update it. */
-                auto lb = lockFlags.inputUpdates.lower_bound(inputPath);
-
-                auto hasChildUpdate =
-                    lb != lockFlags.inputUpdates.end()
-                    && lb->size() > inputPath.size()
-                    && std::equal(inputPath.begin(), inputPath.end(), lb->begin());
-
-                if (hasChildUpdate) {
-                    auto inputFlake = getFlake(
-                        state, oldLock->lockedRef, oldLock->info, false, flakeCache);
-                    computeLocks(inputFlake.inputs, childNode, inputPath, oldLock);
-                } else {
-                    /* No need to fetch this flake, we can be
-                       lazy. However there may be new overrides on the
-                       inputs of this flake, so we need to check
-                       those. */
-                    FlakeInputs fakeInputs;
-
-                    for (auto & i : oldLock->inputs) {
-                        auto lockedNode = std::dynamic_pointer_cast<LockedNode>(i.second);
-                        // Note: this node is not locked in case
-                        // of a circular reference back to the root.
-                        if (lockedNode)
-                            fakeInputs.emplace(i.first, FlakeInput {
-                                .ref = lockedNode->originalRef
-                            });
-                        else {
-                            InputPath path(inputPath);
-                            path.push_back(i.first);
-                            follows.insert_or_assign(path, InputPath());
-                        }
-                    }
-
-                    computeLocks(fakeInputs, childNode, inputPath, oldLock);
-                }
-
-            } else {
-                /* We need to create a new lock file entry. So fetch
-                   this input. */
-
-                if (!lockFlags.allowMutable && !input.ref.input->isImmutable())
-                    throw Error("cannot update flake input '%s' in pure mode", inputPathS);
-
-                if (input.isFlake) {
-                    auto inputFlake = getFlake(state, input.ref, {}, lockFlags.useRegistries, flakeCache);
-
-                    /* Note: in case of an --override-input, we use
-                       the *original* ref (input2.ref) for the
-                       "original" field, rather than the
-                       override. This ensures that the override isn't
-                       nuked the next time we update the lock
-                       file. That is, overrides are sticky unless you
-                       use --no-write-lock-file. */
-                    auto childNode = std::make_shared<LockedNode>(
-                        inputFlake.lockedRef, input2.ref, inputFlake.sourceInfo->info);
-
-                    node->inputs.insert_or_assign(id, childNode);
-
-                    /* Guard against circular flake imports. */
-                    for (auto & parent : parents)
-                        if (parent == input.ref)
-                            throw Error("found circular import of flake '%s'", parent);
-                    parents.push_back(input.ref);
-                    Finally cleanup([&]() { parents.pop_back(); });
-
-                    /* Recursively process the inputs of this
-                       flake. Also, unless we already have this flake
-                       in the top-level lock file, use this flake's
-                       own lock file. */
-                    computeLocks(
-                        inputFlake.inputs, childNode, inputPath,
-                        oldLock
-                        ? std::dynamic_pointer_cast<const Node>(oldLock)
-                        : LockFile::read(
-                            inputFlake.sourceInfo->actualPath + "/" + inputFlake.lockedRef.subdir + "/flake.lock").root);
-                }
-
-                else {
-                    auto [sourceInfo, resolvedRef, lockedRef] = fetchOrSubstituteTree(
-                        state, input.ref, {}, lockFlags.useRegistries, flakeCache);
-                    node->inputs.insert_or_assign(id,
-                        std::make_shared<LockedNode>(lockedRef, input.ref, sourceInfo.info, false));
-                }
-            }
-        }
-    };
-
-    computeLocks(
-        flake.inputs, newLockFile.root, {},
-        lockFlags.recreateLockFile ? nullptr : oldLockFile.root);
-
-    /* Insert edges for 'follows' overrides. */
-    for (auto & [from, to] : follows) {
-        debug("adding 'follows' node from '%s' to '%s'",
-            concatStringsSep("/", from),
-            concatStringsSep("/", to));
-
-        assert(!from.empty());
-
-        InputPath fromParent(from);
-        fromParent.pop_back();
-
-        auto fromParentNode = newLockFile.root->findInput(fromParent);
-        assert(fromParentNode);
-
-        auto toNode = newLockFile.root->findInput(to);
-        if (!toNode)
-            throw Error("flake input '%s' follows non-existent flake input '%s'",
-                concatStringsSep("/", from),
-                concatStringsSep("/", to));
-
-        fromParentNode->inputs.insert_or_assign(from.back(), toNode);
-    }
-
-    for (auto & i : lockFlags.inputOverrides)
-        if (!overridesUsed.count(i.first))
-            warn("the flag '--override-input %s %s' does not match any input",
-                concatStringsSep("/", i.first), i.second);
-
-    for (auto & i : lockFlags.inputUpdates)
-        if (!updatesUsed.count(i))
-            warn("the flag '--update-input %s' does not match any input", concatStringsSep("/", i));
-
-    debug("new lock file: %s", newLockFile);
-
-    /* Check whether we need to / can write the new lock file. */
-    if (!(newLockFile == oldLockFile)) {
-
-        auto diff = diffLockFiles(oldLockFile, newLockFile);
-
-        if (!(oldLockFile == LockFile()))
-            printInfo("inputs of flake '%s' changed:\n%s", topRef, chomp(diff));
-
-        if (lockFlags.writeLockFile) {
-            if (auto sourcePath = topRef.input->getSourcePath()) {
-                if (!newLockFile.isImmutable()) {
-                    if (settings.warnDirty)
-                        warn("will not write lock file of flake '%s' because it has a mutable input", topRef);
-                } else {
-                    if (!lockFlags.updateLockFile)
-                        throw Error("flake '%s' requires lock file changes but they're not allowed due to '--no-update-lock-file'", topRef);
-
-                    auto relPath = (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock";
-
-                    auto path = *sourcePath + "/" + relPath;
-
-                    bool lockFileExists = pathExists(path);
-
-                    if (lockFileExists)
-                        warn("updating lock file '%s'", path);
-                    else
-                        warn("creating lock file '%s'", path);
-
-                    newLockFile.write(path);
-
-                    topRef.input->markChangedFile(
-                        (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock",
-                        lockFlags.commitLockFile
-                        ? std::optional<std::string>(fmt("%s: %s\n\nFlake input changes:\n\n%s",
-                                relPath, lockFileExists ? "Update" : "Add", diff))
-                        : std::nullopt);
-
-                    /* Rewriting the lockfile changed the top-level
-                       repo, so we should re-read it. FIXME: we could
-                       also just clear the 'rev' field... */
-                    auto prevLockedRef = flake.lockedRef;
-                    FlakeCache dummyCache;
-                    flake = getFlake(state, topRef, {}, lockFlags.useRegistries, dummyCache);
-
-                    if (lockFlags.commitLockFile &&
-                        flake.lockedRef.input->getRev() &&
-                        prevLockedRef.input->getRev() != flake.lockedRef.input->getRev())
-                        warn("committed new revision '%s'", flake.lockedRef.input->getRev()->gitRev());
-
-                    /* Make sure that we picked up the change,
-                       i.e. the tree should usually be dirty
-                       now. Corner case: we could have reverted from a
-                       dirty to a clean tree! */
-                    if (flake.lockedRef.input == prevLockedRef.input
-                        && !flake.lockedRef.input->isImmutable())
-                        throw Error("'%s' did not change after I updated its 'flake.lock' file; is 'flake.lock' under version control?", flake.originalRef);
-                }
-            } else
-                throw Error("cannot write modified lock file of flake '%s' (use '--no-write-lock-file' to ignore)", topRef);
-        } else
-            warn("not writing modified lock file of flake '%s'", topRef);
-    }
-
-    return LockedFlake { .flake = std::move(flake), .lockFile = std::move(newLockFile) };
-}
-
-void callFlake(EvalState & state,
-    const LockedFlake & lockedFlake,
-    Value & vRes)
-{
-    auto vLocks = state.allocValue();
-    auto vRootSrc = state.allocValue();
-    auto vRootSubdir = state.allocValue();
-    auto vTmp1 = state.allocValue();
-    auto vTmp2 = state.allocValue();
-
-    mkString(*vLocks, lockedFlake.lockFile.to_string());
-
-    emitTreeAttrs(state, *lockedFlake.flake.sourceInfo, lockedFlake.flake.lockedRef.input, *vRootSrc);
-
-    mkString(*vRootSubdir, lockedFlake.flake.lockedRef.subdir);
-
-    static RootValue vCallFlake = nullptr;
-
-    if (!vCallFlake) {
-        vCallFlake = allocRootValue(state.allocValue());
-        state.eval(state.parseExprFromString(
-            #include "call-flake.nix.gen.hh"
-            , "/"), **vCallFlake);
-    }
-
-    state.callFunction(**vCallFlake, *vLocks, *vTmp1, noPos);
-    state.callFunction(*vTmp1, *vRootSrc, *vTmp2, noPos);
-    state.callFunction(*vTmp2, *vRootSubdir, vRes, noPos);
-}
-
-static void prim_getFlake(EvalState & state, const Pos & pos, Value * * args, Value & v)
-{
-    auto flakeRefS = state.forceStringNoCtx(*args[0], pos);
-    auto flakeRef = parseFlakeRef(flakeRefS, {}, true);
-    if (evalSettings.pureEval && !flakeRef.input->isImmutable())
-        throw Error("cannot call 'getFlake' on mutable flake reference '%s', at %s (use --impure to override)", flakeRefS, pos);
-
-    callFlake(state,
-        lockFlake(state, flakeRef,
-            LockFlags {
-                .updateLockFile = false,
-                .useRegistries = !evalSettings.pureEval,
-                .allowMutable  = !evalSettings.pureEval,
-            }),
-        v);
-}
-
-static RegisterPrimOp r2("getFlake", 1, prim_getFlake);
-
-}
-
-Fingerprint LockedFlake::getFingerprint() const
-{
-    // FIXME: as an optimization, if the flake contains a lock file
-    // and we haven't changed it, then it's sufficient to use
-    // flake.sourceInfo.storePath for the fingerprint.
-    return hashString(htSHA256,
-        fmt("%s;%d;%d;%s",
-            flake.sourceInfo->storePath.to_string(),
-            flake.sourceInfo->info.revCount.value_or(0),
-            flake.sourceInfo->info.lastModified.value_or(0),
-            lockFile));
-}
-
-Flake::~Flake() { }
-
-}

src/libexpr/flake/flake.hh

@@ -1,110 +0,0 @@
-#pragma once
-
-#include "types.hh"
-#include "flakeref.hh"
-#include "lockfile.hh"
-#include "value.hh"
-
-namespace nix {
-
-class EvalState;
-
-namespace fetchers { struct Tree; }
-
-namespace flake {
-
-struct FlakeInput;
-
-typedef std::map<FlakeId, FlakeInput> FlakeInputs;
-
-struct FlakeInput
-{
-    FlakeRef ref;
-    bool isFlake = true;
-    std::optional<InputPath> follows;
-    FlakeInputs overrides;
-};
-
-struct Flake
-{
-    FlakeRef originalRef;
-    FlakeRef resolvedRef;
-    FlakeRef lockedRef;
-    std::optional<std::string> description;
-    std::shared_ptr<const fetchers::Tree> sourceInfo;
-    FlakeInputs inputs;
-    RootValue vOutputs;
-    ~Flake();
-};
-
-Flake getFlake(EvalState & state, const FlakeRef & flakeRef, bool allowLookup);
-
-/* Fingerprint of a locked flake; used as a cache key. */
-typedef Hash Fingerprint;
-
-struct LockedFlake
-{
-    Flake flake;
-    LockFile lockFile;
-
-    Fingerprint getFingerprint() const;
-};
-
-struct LockFlags
-{
-    /* Whether to ignore the existing lock file, creating a new one
-       from scratch. */
-    bool recreateLockFile = false;
-
-    /* Whether to update the lock file at all. If set to false, if any
-       change to the lock file is needed (e.g. when an input has been
-       added to flake.nix), you get a fatal error. */
-    bool updateLockFile = true;
-
-    /* Whether to write the lock file to disk. If set to true, if the
-       any changes to the lock file are needed and the flake is not
-       writable (i.e. is not a local Git working tree or similar), you
-       get a fatal error. If set to false, Nix will use the modified
-       lock file in memory only, without writing it to disk. */
-    bool writeLockFile = true;
-
-    /* Whether to use the registries to lookup indirect flake
-       references like 'nixpkgs'. */
-    bool useRegistries = true;
-
-    /* Whether mutable flake references (i.e. those without a Git
-       revision or similar) without a corresponding lock are
-       allowed. Mutable flake references with a lock are always
-       allowed. */
-    bool allowMutable = true;
-
-    /* Whether to commit changes to flake.lock. */
-    bool commitLockFile = false;
-
-    /* Flake inputs to be overriden. */
-    std::map<InputPath, FlakeRef> inputOverrides;
-
-    /* Flake inputs to be updated. This means that any existing lock
-       for those inputs will be ignored. */
-    std::set<InputPath> inputUpdates;
-};
-
-LockedFlake lockFlake(
-    EvalState & state,
-    const FlakeRef & flakeRef,
-    const LockFlags & lockFlags);
-
-void callFlake(
-    EvalState & state,
-    const LockedFlake & lockedFlake,
-    Value & v);
-
-}
-
-void emitTreeAttrs(
-    EvalState & state,
-    const fetchers::Tree & tree,
-    std::shared_ptr<const fetchers::Input> input,
-    Value & v);
-
-}

src/libexpr/flake/flakeref.cc

@@ -1,196 +0,0 @@
-#include "flakeref.hh"
-#include "store-api.hh"
-#include "url.hh"
-#include "fetchers.hh"
-#include "registry.hh"
-
-namespace nix {
-
-#if 0
-// 'dir' path elements cannot start with a '.'. We also reject
-// potentially dangerous characters like ';'.
-const static std::string subDirElemRegex = "(?:[a-zA-Z0-9_-]+[a-zA-Z0-9._-]*)";
-const static std::string subDirRegex = subDirElemRegex + "(?:/" + subDirElemRegex + ")*";
-#endif
-
-std::string FlakeRef::to_string() const
-{
-    auto url = input->toURL();
-    if (subdir != "")
-        url.query.insert_or_assign("dir", subdir);
-    return url.to_string();
-}
-
-fetchers::Attrs FlakeRef::toAttrs() const
-{
-    auto attrs = input->toAttrs();
-    if (subdir != "")
-        attrs.emplace("dir", subdir);
-    return attrs;
-}
-
-std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef)
-{
-    str << flakeRef.to_string();
-    return str;
-}
-
-bool FlakeRef::operator ==(const FlakeRef & other) const
-{
-    return *input == *other.input && subdir == other.subdir;
-}
-
-FlakeRef FlakeRef::resolve(ref<Store> store) const
-{
-    auto [input2, extraAttrs] = lookupInRegistries(store, input);
-    return FlakeRef(input2, fetchers::maybeGetStrAttr(extraAttrs, "dir").value_or(subdir));
-}
-
-FlakeRef parseFlakeRef(
-    const std::string & url, const std::optional<Path> & baseDir, bool allowMissing)
-{
-    auto [flakeRef, fragment] = parseFlakeRefWithFragment(url, baseDir, allowMissing);
-    if (fragment != "")
-        throw Error("unexpected fragment '%s' in flake reference '%s'", fragment, url);
-    return flakeRef;
-}
-
-std::optional<FlakeRef> maybeParseFlakeRef(
-    const std::string & url, const std::optional<Path> & baseDir)
-{
-    try {
-        return parseFlakeRef(url, baseDir);
-    } catch (Error &) {
-        return {};
-    }
-}
-
-std::pair<FlakeRef, std::string> parseFlakeRefWithFragment(
-    const std::string & url, const std::optional<Path> & baseDir, bool allowMissing)
-{
-    using namespace fetchers;
-
-    static std::string fnRegex = "[0-9a-zA-Z-._~!$&'\"()*+,;=]+";
-
-    static std::regex pathUrlRegex(
-        "(/?" + fnRegex + "(?:/" + fnRegex + ")*/?)"
-        + "(?:\\?(" + queryRegex + "))?"
-        + "(?:#(" + queryRegex + "))?",
-        std::regex::ECMAScript);
-
-    static std::regex flakeRegex(
-        "((" + flakeIdRegexS + ")(?:/(?:" + refAndOrRevRegex + "))?)"
-        + "(?:#(" + queryRegex + "))?",
-        std::regex::ECMAScript);
-
-    std::smatch match;
-
-    /* Check if 'url' is a flake ID. This is an abbreviated syntax for
-       'flake:<flake-id>?ref=<ref>&rev=<rev>'. */
-
-    if (std::regex_match(url, match, flakeRegex)) {
-        auto parsedURL = ParsedURL{
-            .url = url,
-            .base = "flake:" + std::string(match[1]),
-            .scheme = "flake",
-            .authority = "",
-            .path = match[1],
-        };
-
-        return std::make_pair(
-            FlakeRef(inputFromURL(parsedURL), ""),
-            percentDecode(std::string(match[6])));
-    }
-
-    /* Check if 'url' is a path (either absolute or relative to
-       'baseDir'). If so, search upward to the root of the repo
-       (i.e. the directory containing .git). */
-
-    else if (std::regex_match(url, match, pathUrlRegex)) {
-        std::string path = match[1];
-        if (!baseDir && !hasPrefix(path, "/"))
-            throw BadURL("flake reference '%s' is not an absolute path", url);
-        path = absPath(path, baseDir, true);
-
-        if (!S_ISDIR(lstat(path).st_mode))
-            throw BadURL("path '%s' is not a flake (because it's not a directory)", path);
-
-        if (!allowMissing && !pathExists(path + "/flake.nix"))
-            throw BadURL("path '%s' is not a flake (because it doesn't contain a 'flake.nix' file)", path);
-
-        auto fragment = percentDecode(std::string(match[3]));
-
-        auto flakeRoot = path;
-        std::string subdir;
-
-        while (flakeRoot != "/") {
-            if (pathExists(flakeRoot + "/.git")) {
-                auto base = std::string("git+file://") + flakeRoot;
-
-                auto parsedURL = ParsedURL{
-                    .url = base, // FIXME
-                    .base = base,
-                    .scheme = "git+file",
-                    .authority = "",
-                    .path = flakeRoot,
-                    .query = decodeQuery(match[2]),
-                };
-
-                if (subdir != "") {
-                    if (parsedURL.query.count("dir"))
-                        throw Error("flake URL '%s' has an inconsistent 'dir' parameter", url);
-                    parsedURL.query.insert_or_assign("dir", subdir);
-                }
-
-                return std::make_pair(
-                    FlakeRef(inputFromURL(parsedURL), get(parsedURL.query, "dir").value_or("")),
-                    fragment);
-            }
-
-            subdir = std::string(baseNameOf(flakeRoot)) + (subdir.empty() ? "" : "/" + subdir);
-            flakeRoot = dirOf(flakeRoot);
-        }
-
-        fetchers::Attrs attrs;
-        attrs.insert_or_assign("type", "path");
-        attrs.insert_or_assign("path", path);
-
-        return std::make_pair(FlakeRef(inputFromAttrs(attrs), ""), fragment);
-    }
-
-    else {
-        auto parsedURL = parseURL(url);
-        std::string fragment;
-        std::swap(fragment, parsedURL.fragment);
-        return std::make_pair(
-            FlakeRef(inputFromURL(parsedURL), get(parsedURL.query, "dir").value_or("")),
-            fragment);
-    }
-}
-
-std::optional<std::pair<FlakeRef, std::string>> maybeParseFlakeRefWithFragment(
-    const std::string & url, const std::optional<Path> & baseDir)
-{
-    try {
-        return parseFlakeRefWithFragment(url, baseDir);
-    } catch (Error & e) {
-        return {};
-    }
-}
-
-FlakeRef FlakeRef::fromAttrs(const fetchers::Attrs & attrs)
-{
-    auto attrs2(attrs);
-    attrs2.erase("dir");
-    return FlakeRef(
-        fetchers::inputFromAttrs(attrs2),
-        fetchers::maybeGetStrAttr(attrs, "dir").value_or(""));
-}
-
-std::pair<fetchers::Tree, FlakeRef> FlakeRef::fetchTree(ref<Store> store) const
-{
-    auto [tree, lockedInput] = input->fetchTree(store);
-    return {std::move(tree), FlakeRef(lockedInput, subdir)};
-}
-
-}

src/libexpr/flake/flakeref.hh

@@ -1,55 +0,0 @@
-#pragma once
-
-#include "types.hh"
-#include "hash.hh"
-#include "fetchers.hh"
-
-#include <variant>
-
-namespace nix {
-
-class Store;
-
-typedef std::string FlakeId;
-
-struct FlakeRef
-{
-    std::shared_ptr<const fetchers::Input> input;
-
-    Path subdir;
-
-    bool operator==(const FlakeRef & other) const;
-
-    FlakeRef(const std::shared_ptr<const fetchers::Input> & input, const Path & subdir)
-        : input(input), subdir(subdir)
-    {
-        assert(input);
-    }
-
-    // FIXME: change to operator <<.
-    std::string to_string() const;
-
-    fetchers::Attrs toAttrs() const;
-
-    FlakeRef resolve(ref<Store> store) const;
-
-    static FlakeRef fromAttrs(const fetchers::Attrs & attrs);
-
-    std::pair<fetchers::Tree, FlakeRef> fetchTree(ref<Store> store) const;
-};
-
-std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef);
-
-FlakeRef parseFlakeRef(
-    const std::string & url, const std::optional<Path> & baseDir = {}, bool allowMissing = false);
-
-std::optional<FlakeRef> maybeParseFlake(
-    const std::string & url, const std::optional<Path> & baseDir = {});
-
-std::pair<FlakeRef, std::string> parseFlakeRefWithFragment(
-    const std::string & url, const std::optional<Path> & baseDir = {}, bool allowMissing = false);
-
-std::optional<std::pair<FlakeRef, std::string>> maybeParseFlakeRefWithFragment(
-    const std::string & url, const std::optional<Path> & baseDir = {});
-
-}

src/libexpr/flake/lockfile.cc

@@ -1,256 +0,0 @@
-#include "lockfile.hh"
-#include "store-api.hh"
-
-#include <nlohmann/json.hpp>
-
-namespace nix::flake {
-
-FlakeRef flakeRefFromJson(const nlohmann::json & json)
-{
-    return FlakeRef::fromAttrs(jsonToAttrs(json));
-}
-
-FlakeRef getFlakeRef(
-    const nlohmann::json & json,
-    const char * attr)
-{
-    auto i = json.find(attr);
-    if (i != json.end())
-        return flakeRefFromJson(*i);
-
-    throw Error("attribute '%s' missing in lock file", attr);
-}
-
-LockedNode::LockedNode(const nlohmann::json & json)
-    : lockedRef(getFlakeRef(json, "locked"))
-    , originalRef(getFlakeRef(json, "original"))
-    , info(TreeInfo::fromJson(json))
-    , isFlake(json.find("flake") != json.end() ? (bool) json["flake"] : true)
-{
-    if (!lockedRef.input->isImmutable())
-        throw Error("lockfile contains mutable flakeref '%s'", lockedRef);
-}
-
-StorePath LockedNode::computeStorePath(Store & store) const
-{
-    return info.computeStorePath(store);
-}
-
-std::shared_ptr<Node> Node::findInput(const InputPath & path)
-{
-    auto pos = shared_from_this();
-
-    for (auto & elem : path) {
-        auto i = pos->inputs.find(elem);
-        if (i == pos->inputs.end())
-            return {};
-        pos = i->second;
-    }
-
-    return pos;
-}
-
-LockFile::LockFile(const nlohmann::json & json, const Path & path)
-{
-    auto version = json.value("version", 0);
-    if (version != 5)
-        throw Error("lock file '%s' has unsupported version %d", path, version);
-
-    std::unordered_map<std::string, std::shared_ptr<Node>> nodeMap;
-
-    std::function<void(Node & node, const nlohmann::json & jsonNode)> getInputs;
-
-    getInputs = [&](Node & node, const nlohmann::json & jsonNode)
-    {
-        if (jsonNode.find("inputs") == jsonNode.end()) return;
-        for (auto & i : jsonNode["inputs"].items()) {
-            std::string inputKey = i.value();
-            auto k = nodeMap.find(inputKey);
-            if (k == nodeMap.end()) {
-                auto jsonNode2 = json["nodes"][inputKey];
-                auto input = std::make_shared<LockedNode>(jsonNode2);
-                k = nodeMap.insert_or_assign(inputKey, input).first;
-                getInputs(*input, jsonNode2);
-            }
-            node.inputs.insert_or_assign(i.key(), k->second);
-        }
-    };
-
-    std::string rootKey = json["root"];
-    nodeMap.insert_or_assign(rootKey, root);
-    getInputs(*root, json["nodes"][rootKey]);
-}
-
-nlohmann::json LockFile::toJson() const
-{
-    nlohmann::json nodes;
-    std::unordered_map<std::shared_ptr<const Node>, std::string> nodeKeys;
-    std::unordered_set<std::string> keys;
-
-    std::function<std::string(const std::string & key, std::shared_ptr<const Node> node)> dumpNode;
-
-    dumpNode = [&](std::string key, std::shared_ptr<const Node> node) -> std::string
-    {
-        auto k = nodeKeys.find(node);
-        if (k != nodeKeys.end())
-            return k->second;
-
-        if (!keys.insert(key).second) {
-            for (int n = 2; ; ++n) {
-                auto k = fmt("%s_%d", key, n);
-                if (keys.insert(k).second) {
-                    key = k;
-                    break;
-                }
-            }
-        }
-
-        nodeKeys.insert_or_assign(node, key);
-
-        auto n = nlohmann::json::object();
-
-        if (!node->inputs.empty()) {
-            auto inputs = nlohmann::json::object();
-            for (auto & i : node->inputs)
-                inputs[i.first] = dumpNode(i.first, i.second);
-            n["inputs"] = std::move(inputs);
-        }
-
-        if (auto lockedNode = std::dynamic_pointer_cast<const LockedNode>(node)) {
-            n["original"] = fetchers::attrsToJson(lockedNode->originalRef.toAttrs());
-            n["locked"] = fetchers::attrsToJson(lockedNode->lockedRef.toAttrs());
-            n["info"] = lockedNode->info.toJson();
-            if (!lockedNode->isFlake) n["flake"] = false;
-        }
-
-        nodes[key] = std::move(n);
-
-        return key;
-    };
-
-    nlohmann::json json;
-    json["version"] = 5;
-    json["root"] = dumpNode("root", root);
-    json["nodes"] = std::move(nodes);
-
-    return json;
-}
-
-std::string LockFile::to_string() const
-{
-    return toJson().dump(2);
-}
-
-LockFile LockFile::read(const Path & path)
-{
-    if (!pathExists(path)) return LockFile();
-    return LockFile(nlohmann::json::parse(readFile(path)), path);
-}
-
-std::ostream & operator <<(std::ostream & stream, const LockFile & lockFile)
-{
-    stream << lockFile.toJson().dump(2);
-    return stream;
-}
-
-void LockFile::write(const Path & path) const
-{
-    createDirs(dirOf(path));
-    writeFile(path, fmt("%s\n", *this));
-}
-
-bool LockFile::isImmutable() const
-{
-    std::unordered_set<std::shared_ptr<const Node>> nodes;
-
-    std::function<void(std::shared_ptr<const Node> node)> visit;
-
-    visit = [&](std::shared_ptr<const Node> node)
-    {
-        if (!nodes.insert(node).second) return;
-        for (auto & i : node->inputs) visit(i.second);
-    };
-
-    visit(root);
-
-    for (auto & i : nodes) {
-        if (i == root) continue;
-        auto lockedNode = std::dynamic_pointer_cast<const LockedNode>(i);
-        if (lockedNode && !lockedNode->lockedRef.input->isImmutable()) return false;
-    }
-
-    return true;
-}
-
-bool LockFile::operator ==(const LockFile & other) const
-{
-    // FIXME: slow
-    return toJson() == other.toJson();
-}
-
-InputPath parseInputPath(std::string_view s)
-{
-    InputPath path;
-
-    for (auto & elem : tokenizeString<std::vector<std::string>>(s, "/")) {
-        if (!std::regex_match(elem, flakeIdRegex))
-            throw Error("invalid flake input path element '%s'", elem);
-        path.push_back(elem);
-    }
-
-    return path;
-}
-
-static void flattenLockFile(
-    std::shared_ptr<const Node> node,
-    const InputPath & prefix,
-    std::unordered_set<std::shared_ptr<const Node>> & done,
-    std::map<InputPath, std::shared_ptr<const LockedNode>> & res)
-{
-    if (!done.insert(node).second) return;
-
-    for (auto &[id, input] : node->inputs) {
-        auto inputPath(prefix);
-        inputPath.push_back(id);
-        if (auto lockedInput = std::dynamic_pointer_cast<const LockedNode>(input))
-            res.emplace(inputPath, lockedInput);
-        flattenLockFile(input, inputPath, done, res);
-    }
-}
-
-std::string diffLockFiles(const LockFile & oldLocks, const LockFile & newLocks)
-{
-    std::unordered_set<std::shared_ptr<const Node>> done;
-    std::map<InputPath, std::shared_ptr<const LockedNode>> oldFlat, newFlat;
-    flattenLockFile(oldLocks.root, {}, done, oldFlat);
-    done.clear();
-    flattenLockFile(newLocks.root, {}, done, newFlat);
-
-    auto i = oldFlat.begin();
-    auto j = newFlat.begin();
-    std::string res;
-
-    while (i != oldFlat.end() || j != newFlat.end()) {
-        if (j != newFlat.end() && (i == oldFlat.end() || i->first > j->first)) {
-            res += fmt("* Added '%s': '%s'\n", concatStringsSep("/", j->first), j->second->lockedRef);
-            ++j;
-        } else if (i != oldFlat.end() && (j == newFlat.end() || i->first < j->first)) {
-            res += fmt("* Removed '%s'\n", concatStringsSep("/", i->first));
-            ++i;
-        } else {
-            if (!(i->second->lockedRef == j->second->lockedRef)) {
-                assert(i->second->lockedRef.to_string() != j->second->lockedRef.to_string());
-                res += fmt("* Updated '%s': '%s' -> '%s'\n",
-                    concatStringsSep("/", i->first),
-                    i->second->lockedRef,
-                    j->second->lockedRef);
-            }
-            ++i;
-            ++j;
-        }
-    }
-
-    return res;
-}
-
-}

src/libexpr/flake/lockfile.hh

@@ -1,77 +0,0 @@
-#pragma once
-
-#include "flakeref.hh"
-
-#include <nlohmann/json_fwd.hpp>
-
-namespace nix {
-class Store;
-struct StorePath;
-}
-
-namespace nix::flake {
-
-using namespace fetchers;
-
-typedef std::vector<FlakeId> InputPath;
-
-/* A node in the lock file. It has outgoing edges to other nodes (its
-   inputs). Only the root node has this type; all other nodes have
-   type LockedNode. */
-struct Node : std::enable_shared_from_this<Node>
-{
-    std::map<FlakeId, std::shared_ptr<Node>> inputs;
-
-    virtual ~Node() { }
-
-    std::shared_ptr<Node> findInput(const InputPath & path);
-};
-
-/* A non-root node in the lock file. */
-struct LockedNode : Node
-{
-    FlakeRef lockedRef, originalRef;
-    TreeInfo info;
-    bool isFlake = true;
-
-    LockedNode(
-        const FlakeRef & lockedRef,
-        const FlakeRef & originalRef,
-        const TreeInfo & info,
-        bool isFlake = true)
-        : lockedRef(lockedRef), originalRef(originalRef), info(info), isFlake(isFlake)
-    { }
-
-    LockedNode(const nlohmann::json & json);
-
-    StorePath computeStorePath(Store & store) const;
-};
-
-struct LockFile
-{
-    std::shared_ptr<Node> root = std::make_shared<Node>();
-
-    LockFile() {};
-    LockFile(const nlohmann::json & json, const Path & path);
-
-    nlohmann::json toJson() const;
-
-    std::string to_string() const;
-
-    static LockFile read(const Path & path);
-
-    void write(const Path & path) const;
-
-    bool isImmutable() const;
-
-    bool operator ==(const LockFile & other) const;
-};
-
-std::ostream & operator <<(std::ostream & stream, const LockFile & lockFile);
-
-InputPath parseInputPath(std::string_view s);
-
-std::string diffLockFiles(const LockFile & oldLocks, const LockFile & newLocks);
-
-}
-

src/libexpr/function-trace.cc

@@ -1,5 +1,4 @@
 #include "function-trace.hh"
-#include "logging.hh"
 
 namespace nix {
 

src/libexpr/gc.cc

@@ -0,0 +1,537 @@
+#include "gc.hh"
+#include "value.hh"
+#include "attr-set.hh"
+#include "eval.hh"
+
+#include <chrono>
+
+namespace nix {
+
+GC gc;
+
+GC::GC()
+{
+    nextSize = std::max((size_t) 2, parseSize<size_t>(getEnv("GC_INITIAL_HEAP_SIZE").value_or("131072")) / WORD_SIZE);
+
+    // FIXME: placement new
+    frontPtrSentinel = (Ptr<Object> *) malloc(sizeof(Ptr<Object>));
+    backPtrSentinel = (Ptr<Object> *) malloc(sizeof(Ptr<Object>));
+
+    frontPtrSentinel->prev = nullptr;
+    frontPtrSentinel->next = backPtrSentinel;
+
+    backPtrSentinel->prev = frontPtrSentinel;
+    backPtrSentinel->next = nullptr;
+
+    frontRootSentinel = (Root<Object> *) malloc(sizeof(Root<Object>));
+    backRootSentinel = (Root<Object> *) malloc(sizeof(Root<Object>));
+
+    frontRootSentinel->prev = nullptr;
+    frontRootSentinel->next = backRootSentinel;
+
+    backRootSentinel->prev = frontRootSentinel;
+    backRootSentinel->next = nullptr;
+
+    freeLists[0].minSize = 2;
+    freeLists[1].minSize = 3;
+    freeLists[2].minSize = 4;
+    freeLists[3].minSize = 8;
+    freeLists[4].minSize = 16;
+    freeLists[5].minSize = 32;
+    freeLists[6].minSize = 64;
+    freeLists[7].minSize = 128;
+
+    addArena(nextSize);
+}
+
+GC::~GC()
+{
+    debug("%d bytes in arenas, %d bytes allocated, %d bytes reclaimed, in %d ms",
+        totalSize * WORD_SIZE,
+        allTimeWordsAllocated * WORD_SIZE,
+        allTimeWordsFreed * WORD_SIZE,
+        totalDurationMs);
+
+    size_t n = 0;
+    for (Ptr<Object> * p = frontPtrSentinel->next; p != backPtrSentinel; p = p->next)
+        n++;
+    if (n)
+        warn("%d GC root pointers still exist on exit", n);
+
+    n = 0;
+    for (Root<Object> * p = frontRootSentinel->next; p != backRootSentinel; p = p->next)
+        n++;
+    if (n)
+        warn("%d GC root objects still exist on exit", n);
+
+    assert(!frontPtrSentinel->prev);
+    assert(!backPtrSentinel->next);
+    assert(!frontRootSentinel->prev);
+    assert(!backRootSentinel->next);
+}
+
+void GC::addArena(size_t arenaSize)
+{
+    debug("allocating arena of %d bytes", arenaSize * WORD_SIZE);
+
+    auto arena = Arena(arenaSize);
+
+    // Add this arena to a freelist as a single block.
+    addToFreeList(new (arena.start) Free(arenaSize));
+
+    arenas.emplace_back(std::move(arena));
+
+    totalSize += arenaSize;
+
+    nextSize = arenaSize * 1.5; // FIXME: overflow, clamp
+}
+
+void GC::addToFreeList(Free * obj)
+{
+    auto size = obj->words();
+    for (auto i = freeLists.rbegin(); i != freeLists.rend(); ++i)
+        if (size >= i->minSize) {
+            obj->next = i->front;
+            i->front = obj;
+            return;
+        }
+    abort();
+}
+
+void GC::gc()
+{
+    typedef std::chrono::time_point<std::chrono::steady_clock> steady_time_point;
+
+    auto before = steady_time_point::clock::now();
+
+    size_t marked = 0;
+
+    std::stack<Object *> stack;
+
+    // FIXME: ensure this gets inlined.
+    auto push = [&](Object * p) { if (p) { assertObject(p); stack.push(p); } };
+
+    auto pushPointers = [&](Object * obj) {
+        switch (obj->type) {
+
+        case tFree:
+            printError("reached a freed object at %x", obj);
+            abort();
+
+        case tBindings: {
+            auto obj2 = (Bindings *) obj;
+            for (auto i = obj2->attrs; i < obj2->attrs + obj2->size_; ++i)
+                push(i->value);
+            break;
+        }
+
+        case tValueList: {
+            auto obj2 = (PtrList<Object> *) obj;
+            for (auto i = obj2->elems; i < obj2->elems + obj2->size(); ++i)
+                push(*i);
+            break;
+        }
+
+        case tEnv: {
+            auto obj2 = (Env *) obj;
+            push(obj2->up);
+            for (auto i = obj2->values; i < obj2->values + obj2->getSize(); ++i)
+                push(*i);
+            break;
+        }
+
+        case tWithExprEnv: {
+            auto obj2 = (Env *) obj;
+            push(obj2->up);
+            break;
+        }
+
+        case tWithAttrsEnv: {
+            auto obj2 = (Env *) obj;
+            push(obj2->up);
+            push(obj2->values[0]);
+            break;
+        }
+
+        case tString:
+        case tContext:
+        case tInt:
+        case tBool:
+        case tNull:
+        case tList0:
+        case tFloat:
+        case tShortString:
+        case tStaticString:
+            break;
+
+        case tLongString: {
+            auto obj2 = (Value *) obj;
+            push(obj2->string.s);
+            // See setContext().
+            if (!(((ptrdiff_t) obj2->string.context) & 1))
+                push(obj2->string.context);
+            break;
+        }
+
+        case tPath:
+            push(((Value *) obj)->path);
+            break;
+
+        case tAttrs:
+            push(((Value *) obj)->attrs);
+            break;
+
+        case tList1:
+            push(((Value *) obj)->smallList[0]);
+            break;
+
+        case tList2:
+            push(((Value *) obj)->smallList[0]);
+            push(((Value *) obj)->smallList[1]);
+            break;
+
+        case tListN:
+            push(((Value *) obj)->bigList);
+            break;
+
+        case tThunk:
+        case tBlackhole:
+            push(((Value *) obj)->thunk.env);
+            break;
+
+        case tApp:
+        case tPrimOpApp:
+            push(((Value *) obj)->app.left);
+            push(((Value *) obj)->app.right);
+            break;
+
+        case tLambda:
+            push(((Value *) obj)->lambda.env);
+            break;
+
+        case tPrimOp:
+            // FIXME: GC primops?
+            break;
+
+        default:
+            printError("don't know how to traverse object at %x (tag %d)", obj, obj->type);
+            abort();
+        }
+    };
+
+    auto processStack = [&]() {
+        while (!stack.empty()) {
+            auto obj = stack.top();
+            stack.pop();
+
+            //printError("MARK %x", obj);
+
+            if (!obj->isMarked()) {
+                marked++;
+                obj->mark();
+                pushPointers(obj);
+            }
+        }
+    };
+
+    for (Root<Object> * p = frontRootSentinel->next; p != backRootSentinel; p = p->next) {
+        pushPointers(&p->value);
+        processStack();
+    }
+
+    for (Ptr<Object> * p = frontPtrSentinel->next; p != backPtrSentinel; p = p->next) {
+        if (!p->value) continue;
+        stack.push(p->value);
+        processStack();
+    }
+
+    auto afterMark = steady_time_point::clock::now();
+
+    // Reset all the freelists.
+    for (auto & freeList : freeLists)
+        freeList.front = nullptr;
+
+    // Go through all the arenas and add free objects to the
+    // appropriate freelists.
+    size_t totalObjectsFreed = 0;
+    size_t totalWordsFreed = 0;
+    size_t totalObjectsKept = 0;
+    size_t totalWordsKept = 0;
+
+    for (auto & arena : arenas) {
+        auto [objectsFreed, wordsFreed, objectsKept, wordsKept] = freeUnmarked(arena);
+        totalObjectsFreed += objectsFreed;
+        totalWordsFreed += wordsFreed;
+        totalObjectsKept += objectsKept;
+        totalWordsKept += wordsKept;
+    }
+
+    auto after = steady_time_point::clock::now();
+
+    auto markDurationMs = std::chrono::duration_cast<std::chrono::milliseconds>(afterMark - before).count();
+    auto sweepDurationMs = std::chrono::duration_cast<std::chrono::milliseconds>(after - afterMark).count();
+
+    debug("freed %d dead objects (%d bytes), keeping %d/%d objects (%d bytes), marked in %d ms, swept in %d ms",
+        totalObjectsFreed, totalWordsFreed * WORD_SIZE,
+        marked, totalObjectsKept, totalWordsKept * WORD_SIZE,
+        markDurationMs, sweepDurationMs);
+
+    allTimeWordsFreed += totalWordsFreed;
+    totalDurationMs += markDurationMs + sweepDurationMs;
+}
+
+size_t GC::getObjectSize(Object * obj)
+{
+    auto tag = obj->type;
+    if (tag >= tInt && tag <= tFloat) {
+        return ((Value *) obj)->words();
+    } else {
+        switch (tag) {
+        case tFree:
+            return ((Free *) obj)->words();
+            break;
+        case tString:
+            return ((String *) obj)->words();
+            break;
+        case tBindings:
+            return ((Bindings *) obj)->words();
+            break;
+        case tValueList:
+            return ((PtrList<Value> *) obj)->words();
+            break;
+        case tEnv:
+        case tWithExprEnv:
+        case tWithAttrsEnv:
+            return ((Env *) obj)->words();
+            break;
+        case tContext:
+            return ((Context *) obj)->getSize() + 1;
+            break;
+        default:
+            printError("GC encountered invalid object with tag %d", tag);
+            abort();
+        }
+    }
+}
+
+std::tuple<size_t, size_t, size_t, size_t> GC::freeUnmarked(Arena & arena)
+{
+    size_t objectsFreed = 0;
+    size_t wordsFreed = 0;
+    size_t objectsKept = 0;
+    size_t wordsKept = 0;
+
+    auto end = arena.start + arena.size;
+    auto pos = arena.start;
+
+    Free * curFree = nullptr;
+
+    auto linkCurFree = [&]() {
+        if (curFree && curFree->words() > 1)
+            addToFreeList(curFree);
+        curFree = nullptr;
+    };
+
+    while (pos < end) {
+        auto obj = (Object *) pos;
+
+        auto objSize = getObjectSize(obj);
+
+        // Merge current object into the previous free object.
+        auto mergeFree = [&]() {
+            //printError("MERGE %x %x %d", curFree, obj, curFree->size() + objSize);
+            assert(curFree->words() >= 1);
+            curFree->setSize(curFree->words() + objSize);
+        };
+
+        if (obj->type == tFree) {
+            //debug("KEEP FREE %x %d", obj, obj->getMisc());
+            if (curFree) {
+                // Merge this object into the previous free
+                // object.
+                mergeFree();
+            } else {
+                curFree = (Free *) obj;
+            }
+        } else {
+
+            if (obj->isMarked()) {
+                // Unmark to prepare for the next GC run.
+                //debug("KEEP OBJECT %x %d %d", obj, obj->type, objSize);
+                linkCurFree();
+                obj->unmark();
+                objectsKept += 1;
+                wordsKept += objSize;
+            } else {
+                //debug("FREE OBJECT %x %d %d", obj, obj->type, objSize);
+                #if GC_DEBUG
+                for (size_t i = 0; i < objSize; ++i)
+                    ((Word *) obj)[i] = 0xdeadc0dedeadbeefULL;
+                #endif
+                objectsFreed += 1;
+                wordsFreed += objSize;
+                if (curFree) {
+                    mergeFree();
+                } else {
+                    // Convert to a free object.
+                    curFree = (Free *) obj;
+                    curFree->type = tFree;
+                    curFree->setSize(objSize);
+                }
+            }
+        }
+
+        pos += objSize;
+    }
+
+    linkCurFree();
+
+    assert(pos == end);
+
+    return {objectsFreed, wordsFreed, objectsKept, wordsKept};
+}
+
+bool GC::isObject(void * p)
+{
+    for (auto & arena : arenas)
+        if (p >= arena.start && p < arena.start + arena.size)
+            return true;
+    return false;
+}
+
+std::tuple<size_t, size_t> GC::getObjectClosureSize(Object * p)
+{
+    std::unordered_set<Object *> seen;
+
+    // FIXME: cut&paste.
+
+    std::stack<Object *> stack;
+
+    // FIXME: ensure this gets inlined.
+    auto push = [&](Object * p) { if (p) { assertObject(p); stack.push(p); } };
+
+    auto pushPointers = [&](Object * obj) {
+        switch (obj->type) {
+
+        case tFree:
+            printError("reached a freed object at %x", obj);
+            abort();
+
+        case tBindings: {
+            auto obj2 = (Bindings *) obj;
+            for (auto i = obj2->attrs; i < obj2->attrs + obj2->size_; ++i)
+                push(i->value);
+            break;
+        }
+
+        case tValueList: {
+            auto obj2 = (PtrList<Object> *) obj;
+            for (auto i = obj2->elems; i < obj2->elems + obj2->size(); ++i)
+                push(*i);
+            break;
+        }
+
+        case tEnv: {
+            auto obj2 = (Env *) obj;
+            push(obj2->up);
+            for (auto i = obj2->values; i < obj2->values + obj2->getSize(); ++i)
+                push(*i);
+            break;
+        }
+
+        case tWithExprEnv: {
+            auto obj2 = (Env *) obj;
+            push(obj2->up);
+            break;
+        }
+
+        case tWithAttrsEnv: {
+            auto obj2 = (Env *) obj;
+            push(obj2->up);
+            push(obj2->values[0]);
+            break;
+        }
+
+        case tString:
+        case tContext:
+        case tInt:
+        case tBool:
+        case tNull:
+        case tList0:
+        case tFloat:
+        case tShortString:
+        case tStaticString:
+            break;
+
+        case tLongString: {
+            auto obj2 = (Value *) obj;
+            push(obj2->string.s);
+            // See setContext().
+            if (!(((ptrdiff_t) obj2->string.context) & 1))
+                push(obj2->string.context);
+            break;
+        }
+
+        case tPath:
+            push(((Value *) obj)->path);
+            break;
+
+        case tAttrs:
+            push(((Value *) obj)->attrs);
+            break;
+
+        case tList1:
+            push(((Value *) obj)->smallList[0]);
+            break;
+
+        case tList2:
+            push(((Value *) obj)->smallList[0]);
+            push(((Value *) obj)->smallList[1]);
+            break;
+
+        case tListN:
+            push(((Value *) obj)->bigList);
+            break;
+
+        case tThunk:
+        case tBlackhole:
+            push(((Value *) obj)->thunk.env);
+            break;
+
+        case tApp:
+        case tPrimOpApp:
+            push(((Value *) obj)->app.left);
+            push(((Value *) obj)->app.right);
+            break;
+
+        case tLambda:
+            push(((Value *) obj)->lambda.env);
+            break;
+
+        case tPrimOp:
+            // FIXME: GC primops?
+            break;
+
+        default:
+            printError("don't know how to traverse object at %x (tag %d)", obj, obj->type);
+            abort();
+        }
+    };
+
+    stack.push(p);
+
+    size_t totalSize = 0;
+
+    while (!stack.empty()) {
+        auto obj = stack.top();
+        stack.pop();
+        if (seen.insert(obj).second) {
+            pushPointers(obj);
+            totalSize += getObjectSize(obj);
+        }
+    }
+
+    return {seen.size(), totalSize};
+}
+
+}

src/libexpr/gc.hh

@@ -0,0 +1,482 @@
+#pragma once
+
+#include "logging.hh"
+
+#include <stack>
+#include <limits>
+#include <cassert>
+#include <cstring>
+
+//#define GC_DEBUG 1
+
+namespace nix {
+
+typedef unsigned long Word;
+
+enum Tag {
+    tFree = 3,
+
+    // Misc types
+    tString,
+    tBindings,
+    tValueList,
+    tEnv,
+    tWithExprEnv,
+    tWithAttrsEnv,
+    tContext,
+
+    // Value tags
+    tInt,
+    tBool,
+    tShortString,
+    tStaticString,
+    tLongString,
+    tPath,
+    tNull,
+    tAttrs,
+    tList0,
+    tList1,
+    tList2,
+    tListN,
+    tThunk,
+    tApp,
+    tLambda,
+    tBlackhole,
+    tPrimOp,
+    tPrimOpApp,
+    tExternal,
+    tFloat
+};
+
+constexpr size_t WORD_SIZE = 8;
+
+struct Object
+{
+    friend class GC;
+
+public:
+    constexpr static size_t miscBytes = 7;
+
+public: // FIXME
+    Tag type:7;
+
+private:
+    bool marked:1;
+
+    unsigned long misc:56;
+
+    void unmark()
+    {
+        marked = false;
+    }
+
+protected:
+
+    Object(Tag type, unsigned long misc) : type(type), marked(false), misc(misc) { }
+
+    bool isMarked()
+    {
+        return marked;
+    }
+
+    void mark()
+    {
+        marked = true;
+    }
+
+    void setMisc(unsigned int misc)
+    {
+        this->misc = misc;
+    }
+
+    unsigned int getMisc() const
+    {
+        return misc;
+    }
+
+    char * getMiscData() const
+    {
+        return ((char *) this) + 1;
+    }
+};
+
+template<class T>
+struct PtrList : Object
+{
+    T * elems[0];
+
+    PtrList(Tag type, size_t size) : Object(type, size)
+    {
+        for (size_t i = 0; i < size; i++)
+            elems[i] = nullptr;
+    }
+
+    size_t size() const { return getMisc(); }
+    size_t words() const { return wordsFor(size()); }
+    static size_t wordsFor(size_t size) { return 1 + size; }
+};
+
+struct Free : Object
+{
+    Free * next;
+
+    Free(size_t size) : Object(tFree, size), next(nullptr) { }
+
+    // return size in words
+    size_t words() const { return getMisc(); }
+
+    void setSize(size_t size) { assert(size >= 1); setMisc(size); }
+};
+
+template<class T>
+struct Ptr;
+
+template<class T>
+struct Root;
+
+struct GC
+{
+
+private:
+
+    Ptr<Object> * frontPtrSentinel;
+    Ptr<Object> * backPtrSentinel;
+
+    Root<Object> * frontRootSentinel;
+    Root<Object> * backRootSentinel;
+
+    template<class T>
+    friend class Ptr;
+
+    template<class T>
+    friend class Root;
+
+    struct Arena
+    {
+        size_t size; // in words
+        Word * start;
+
+        Arena(size_t size)
+            : size(size)
+            , start(new Word[size])
+        {
+            assert(size >= 2);
+        }
+
+        Arena(const Arena & arena) = delete;
+
+        Arena(Arena && arena)
+        {
+            size = arena.size;
+            start = arena.start;
+            arena.start = nullptr;
+        }
+
+        ~Arena()
+        {
+            delete[] start;
+        }
+    };
+
+    size_t totalSize = 0;
+    size_t nextSize;
+
+    std::vector<Arena> arenas;
+
+    struct FreeList
+    {
+        size_t minSize;
+        Free * front = nullptr;
+    };
+
+    std::array<FreeList, 8> freeLists;
+
+public:
+
+    size_t getHeapSize() { return totalSize * WORD_SIZE; }
+
+    size_t allTimeWordsAllocated = 0;
+    size_t allTimeWordsFreed = 0;
+    uint64_t totalDurationMs = 0;
+
+private:
+
+    Object * allocObject(size_t size)
+    {
+        assert(size >= 2);
+
+        for (int attempt = 0; attempt < 3; attempt++) {
+
+            for (size_t i = 0; i < freeLists.size(); ++i) {
+                auto & freeList = freeLists[i];
+
+                if ((size <= freeList.minSize || i == freeLists.size() - 1) && freeList.front) {
+                    //printError("TRY %d %d %d", size, i, freeList.minSize);
+
+                    Free * * prev = &freeList.front;
+
+                    while (Free * freeObj = *prev) {
+                        //printError("LOOK %x %d %x", freeObj, freeObj->words(), freeObj->next);
+                        assert(freeObj->words() >= freeList.minSize);
+                        if (freeObj->words() == size) {
+                            // Convert the free object.
+                            *prev = freeObj->next;
+                            return (Object *) freeObj;
+                        } else if (freeObj->words() >= size + 2) {
+                            // Split the free object.
+                            auto newSize = freeObj->words() - size;
+                            freeObj->setSize(newSize);
+                            if (newSize < freeList.minSize) {
+                                /* The free object is now smaller than
+                                   the minimum size for this freelist,
+                                   so move it to another one. */
+                                //printError("MOVE %x %d -> %d", freeObj, newSize + size, newSize);
+                                *prev = freeObj->next;
+                                addToFreeList(freeObj);
+                            }
+                            return (Object *) (((Word *) freeObj) + newSize);
+                        } else if (freeObj->words() == size + 1) {
+                            // Return the free object and add a padding word.
+                            *prev = freeObj->next;
+                            freeObj->setSize(1);
+                            return (Object *) (((Word *) freeObj) + 1);
+                        } else {
+                            assert(freeObj->words() < size);
+                            prev = &freeObj->next;
+                        }
+                    }
+                }
+            }
+
+            if (attempt == 0) {
+                debug("allocation of %d bytes failed, GCing...", size * WORD_SIZE);
+                gc();
+            } else if (attempt == 1) {
+                addArena(std::max(nextSize, size));
+            }
+        }
+
+        throw Error("allocation of %d bytes failed", size);
+    }
+
+public:
+
+    GC();
+    ~GC();
+
+    template<typename T, typename... Args>
+    Ptr<T> alloc(size_t size, const Args & ... args)
+    {
+        auto raw = allocObject(size);
+        allTimeWordsAllocated += size;
+        return new (raw) T(args...);
+    }
+
+    void gc();
+
+    bool isObject(void * p);
+
+    void assertObject(void * p)
+    {
+        #if GC_DEBUG
+        if (!isObject(p)) {
+            printError("object %p is not an object", p);
+            abort();
+        }
+        #endif
+    }
+
+    /* Return the size in words of object 'obj'. */
+    static size_t getObjectSize(Object * obj);
+
+    /* Return the number and size in words of the objects reachable
+       from object 'obj'. */
+    std::tuple<size_t, size_t> getObjectClosureSize(Object * obj);
+
+private:
+
+    void addArena(size_t arenaSize);
+
+    void addToFreeList(Free * obj);
+
+    std::tuple<size_t, size_t, size_t, size_t> freeUnmarked(Arena & arena);
+};
+
+extern GC gc;
+
+template<class T>
+struct Ptr
+{
+private:
+
+    friend class GC;
+
+    Ptr * prev = nullptr, * next = nullptr;
+    T * value = nullptr;
+
+    void link()
+    {
+        prev = (Ptr *) gc.frontPtrSentinel;
+        next = prev->next;
+        next->prev = this;
+        prev->next = this;
+    }
+
+public:
+
+    Ptr() {
+        link();
+    }
+
+    Ptr(T * value) : value(value)
+    {
+        link();
+    }
+
+    Ptr(const Ptr & p)
+    {
+        auto & p2 = const_cast<Ptr &>(p);
+        value = p2.value;
+        next = &p2;
+        prev = p2.prev;
+        prev->next = this;
+        p2.prev = this;
+    }
+
+    Ptr(Ptr && p)
+    {
+        link();
+        value = p.value;
+        p.value = nullptr;
+    }
+
+    Ptr & operator =(const Ptr & p)
+    {
+        value = p.value;
+        return *this;
+    }
+
+    Ptr & operator =(Ptr && p)
+    {
+        value = p.value;
+        p.value = nullptr;
+        return *this;
+    }
+
+    Ptr & operator =(T * v)
+    {
+        value = v;
+        return *this;
+    }
+
+    ~Ptr()
+    {
+        assert(next);
+        assert(prev);
+        assert(next->prev == this);
+        next->prev = prev;
+        assert(prev->next == this);
+        prev->next = next;
+    }
+
+    T * operator ->()
+    {
+        return value;
+    }
+
+    T * operator ->() const // FIXME
+    {
+        return value;
+    }
+
+    operator T * ()
+    {
+        return value;
+    }
+
+    operator T & ()
+    {
+        assert(value);
+        return *value;
+    }
+
+    operator bool() const
+    {
+        return value != nullptr;
+    }
+};
+
+template<class T>
+struct Root
+{
+    Root * prev = nullptr, * next = nullptr;
+    T value;
+
+    template<typename... Args>
+    Root(const Args & ... args)
+        : value{args... }
+    {
+        prev = (Root *) gc.frontRootSentinel;
+        next = prev->next;
+        next->prev = this;
+        prev->next = this;
+    }
+
+    Root(const Root & p) = delete;
+    Root(Root && p) = delete;
+
+    Root & operator =(const T & v) { value = v; return *this; }
+
+    ~Root()
+    {
+        assert(next);
+        assert(prev);
+        assert(next->prev == this);
+        next->prev = prev;
+        assert(prev->next == this);
+        prev->next = next;
+    }
+
+    T * operator ->()
+    {
+        return &value;
+    }
+
+    operator T * ()
+    {
+        return &value;
+    }
+
+    operator T & ()
+    {
+        return value;
+    }
+};
+
+struct String : Object
+{
+    char s[0];
+
+    String(size_t len, const char * src)
+        : Object(tString, len)
+    {
+        std::memcpy(s, src, len + 1);
+    }
+
+    size_t words() const { return wordsFor(getMisc()); }
+
+    /* Return the number of words needed to store a string of 'len'
+       characters (where 'len' excludes the terminator). */
+    static size_t wordsFor(size_t len)
+    {
+        return len / WORD_SIZE + 2;
+    }
+
+    static String * alloc(const char * src)
+    {
+        auto len = strlen(src);
+        return gc.alloc<String>(wordsFor(len), len, src);
+    }
+};
+
+}

src/libexpr/get-drvs.cc

@@ -115,15 +115,15 @@ DrvInfo::Outputs DrvInfo::queryOutputs(bool onlyOutputsToInstall)
         return outputs;
 
     /* Check for `meta.outputsToInstall` and return `outputs` reduced to that. */
-    const Value * outTI = queryMeta("outputsToInstall");
+    const auto outTI = queryMeta("outputsToInstall");
     if (!outTI) return outputs;
     const auto errMsg = Error("this derivation has bad 'meta.outputsToInstall'");
         /* ^ this shows during `nix-env -i` right under the bad derivation */
     if (!outTI->isList()) throw errMsg;
     Outputs result;
     for (auto i = outTI->listElems(); i != outTI->listElems() + outTI->listSize(); ++i) {
-        if ((*i)->type != tString) throw errMsg;
-        auto out = outputs.find((*i)->string.s);
+        if (!(*i)->isString()) throw errMsg;
+        auto out = outputs.find((*i)->getString());
         if (out == outputs.end()) throw errMsg;
         result.insert(*out);
     }
@@ -178,8 +178,7 @@ bool DrvInfo::checkMeta(Value & v)
             if (!checkMeta(*i.value)) return false;
         return true;
     }
-    else return v.type == tInt || v.type == tBool || v.type == tString ||
-                v.type == tFloat;
+    else return v.type == tInt || v.type == tBool || v.isString() || v.type == tFloat;
 }
 
 
@@ -194,36 +193,36 @@ Value * DrvInfo::queryMeta(const string & name)
 
 string DrvInfo::queryMetaString(const string & name)
 {
-    Value * v = queryMeta(name);
-    if (!v || v->type != tString) return "";
-    return v->string.s;
+    auto v = queryMeta(name);
+    if (!v || !v->isString()) return "";
+    return v->getString();
 }
 
 
 NixInt DrvInfo::queryMetaInt(const string & name, NixInt def)
 {
-    Value * v = queryMeta(name);
+    auto v = queryMeta(name);
     if (!v) return def;
     if (v->type == tInt) return v->integer;
-    if (v->type == tString) {
+    if (v->isString()) {
         /* Backwards compatibility with before we had support for
            integer meta fields. */
         NixInt n;
-        if (string2Int(v->string.s, n)) return n;
+        if (string2Int(v->getString(), n)) return n;
     }
     return def;
 }
 
 NixFloat DrvInfo::queryMetaFloat(const string & name, NixFloat def)
 {
-    Value * v = queryMeta(name);
+    auto v = queryMeta(name);
     if (!v) return def;
     if (v->type == tFloat) return v->fpoint;
-    if (v->type == tString) {
+    if (v->isString()) {
         /* Backwards compatibility with before we had support for
            float meta fields. */
         NixFloat n;
-        if (string2Float(v->string.s, n)) return n;
+        if (string2Float(v->getString(), n)) return n;
     }
     return def;
 }
@@ -231,14 +230,15 @@ NixFloat DrvInfo::queryMetaFloat(const string & name, NixFloat def)
 
 bool DrvInfo::queryMetaBool(const string & name, bool def)
 {
-    Value * v = queryMeta(name);
+    auto v = queryMeta(name);
     if (!v) return def;
     if (v->type == tBool) return v->boolean;
-    if (v->type == tString) {
+    if (v->isString()) {
         /* Backwards compatibility with before we had support for
            Boolean meta fields. */
-        if (strcmp(v->string.s, "true") == 0) return true;
-        if (strcmp(v->string.s, "false") == 0) return false;
+        auto s = v->getString();
+        if (strcmp(s, "true") == 0) return true;
+        if (strcmp(s, "false") == 0) return false;
     }
     return def;
 }
@@ -247,8 +247,8 @@ bool DrvInfo::queryMetaBool(const string & name, bool def)
 void DrvInfo::setMeta(const string & name, Value * v)
 {
     getMeta();
-    Bindings * old = meta;
-    meta = state->allocBindings(1 + (old ? old->size() : 0));
+    Ptr<Bindings> old = meta;
+    meta = Bindings::allocBindings(1 + (old ? old->size() : 0));
     Symbol sym = state->symbols.create(name);
     if (old)
         for (auto i : *old)
@@ -260,6 +260,7 @@ void DrvInfo::setMeta(const string & name, Value * v)
 
 
 /* Cache for already considered attrsets. */
+// FIXME: Use Ptr?
 typedef set<Bindings *> Done;
 
 
@@ -319,24 +320,24 @@ static void getDerivations(EvalState & state, Value & vIn,
     DrvInfos & drvs, Done & done,
     bool ignoreAssertionFailures)
 {
-    Value v;
+    Root<Value> v;
     state.autoCallFunction(autoArgs, vIn, v);
 
     /* Process the expression. */
     if (!getDerivation(state, v, pathPrefix, drvs, done, ignoreAssertionFailures)) ;
 
-    else if (v.type == tAttrs) {
+    else if (v->type == tAttrs) {
 
         /* !!! undocumented hackery to support combining channels in
            nix-env.cc. */
-        bool combineChannels = v.attrs->find(state.symbols.create("_combineChannels")) != v.attrs->end();
+        bool combineChannels = v->attrs->find(state.symbols.create("_combineChannels")) != v->attrs->end();
 
         /* Consider the attributes in sorted order to get more
            deterministic behaviour in nix-env operations (e.g. when
            there are names clashes between derivations, the derivation
            bound to the attribute with the "lower" name should take
            precedence). */
-        for (auto & i : v.attrs->lexicographicOrder()) {
+        for (auto & i : v->attrs->lexicographicOrder()) {
             debug("evaluating attribute '%1%'", i->name);
             if (!std::regex_match(std::string(i->name), attrRegex))
                 continue;
@@ -348,7 +349,7 @@ static void getDerivations(EvalState & state, Value & vIn,
                    should we recurse into it?  => Only if it has a
                    `recurseForDerivations = true' attribute. */
                 if (i->value->type == tAttrs) {
-                    Bindings::iterator j = i->value->attrs->find(state.sRecurseForDerivations);
+                    Bindings::iterator j = i->value->attrs->find(state.symbols.create("recurseForDerivations"));
                     if (j != i->value->attrs->end() && state.forceBool(*j->value, *j->pos))
                         getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures);
                 }
@@ -356,11 +357,11 @@ static void getDerivations(EvalState & state, Value & vIn,
         }
     }
 
-    else if (v.isList()) {
-        for (unsigned int n = 0; n < v.listSize(); ++n) {
+    else if (v->isList()) {
+        for (unsigned int n = 0; n < v->listSize(); ++n) {
             string pathPrefix2 = addToPath(pathPrefix, (format("%1%") % n).str());
-            if (getDerivation(state, *v.listElems()[n], pathPrefix2, drvs, done, ignoreAssertionFailures))
-                getDerivations(state, *v.listElems()[n], pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures);
+            if (getDerivation(state, *v->listElems()[n], pathPrefix2, drvs, done, ignoreAssertionFailures))
+                getDerivations(state, *v->listElems()[n], pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures);
         }
     }
 

src/libexpr/get-drvs.hh

@@ -26,7 +26,8 @@ private:
 
     bool failed = false; // set if we get an AssertionError
 
-    Bindings * attrs = nullptr, * meta = nullptr;
+    Ptr<Bindings> attrs;
+    Ptr<Bindings> meta;
 
     Bindings * getMeta();
 
@@ -69,11 +70,7 @@ public:
 };
 
 
-#if HAVE_BOEHMGC
-typedef list<DrvInfo, traceable_allocator<DrvInfo> > DrvInfos;
-#else
-typedef list<DrvInfo> DrvInfos;
-#endif
+typedef std::list<DrvInfo> DrvInfos;
 
 
 /* If value `v' denotes a derivation, return a DrvInfo object

src/libexpr/json-to-value.cc

@@ -4,38 +4,40 @@
 #include <nlohmann/json.hpp>
 
 using json = nlohmann::json;
+using std::unique_ptr;
 
 namespace nix {
 
 // for more information, refer to
 // https://github.com/nlohmann/json/blob/master/include/nlohmann/detail/input/json_sax.hpp
 class JSONSax : nlohmann::json_sax<json> {
+
     class JSONState {
     protected:
-        std::unique_ptr<JSONState> parent;
-        RootValue v;
+        unique_ptr<JSONState> parent;
+        Ptr<Value> v;
     public:
-        virtual std::unique_ptr<JSONState> resolve(EvalState &)
+        virtual unique_ptr<JSONState> resolve(EvalState &)
         {
             throw std::logic_error("tried to close toplevel json parser state");
-        }
-        explicit JSONState(std::unique_ptr<JSONState> && p) : parent(std::move(p)) {}
-        explicit JSONState(Value * v) : v(allocRootValue(v)) {}
+        };
+        explicit JSONState(unique_ptr<JSONState> && p) : parent(std::move(p)), v(nullptr) {};
+        explicit JSONState(Value * v) : v(v) {};
         JSONState(JSONState & p) = delete;
         Value & value(EvalState & state)
         {
             if (!v)
-                v = allocRootValue(state.allocValue());
-            return **v;
-        }
-        virtual ~JSONState() {}
-        virtual void add() {}
+                v = state.allocValue();
+            return *v;
+        };
+        virtual ~JSONState() {};
+        virtual void add() {};
     };
 
     class JSONObjectState : public JSONState {
         using JSONState::JSONState;
-        ValueMap attrs;
-        std::unique_ptr<JSONState> resolve(EvalState & state) override
+        ValueMap attrs = ValueMap();
+        virtual unique_ptr<JSONState> resolve(EvalState & state) override
         {
             Value & v = parent->value(state);
             state.mkAttrs(v, attrs.size());
@@ -43,17 +45,17 @@ class JSONSax : nlohmann::json_sax<json> {
                 v.attrs->push_back(Attr(i.first, i.second));
             return std::move(parent);
         }
-        void add() override { v = nullptr; }
+        virtual void add() override { v = nullptr; };
     public:
         void key(string_t & name, EvalState & state)
         {
-            attrs.insert_or_assign(state.symbols.create(name), &value(state));
+            attrs[state.symbols.create(name)] = &value(state);
         }
     };
 
     class JSONListState : public JSONState {
-        ValueVector values;
-        std::unique_ptr<JSONState> resolve(EvalState & state) override
+        ValueVector values = ValueVector();
+        virtual unique_ptr<JSONState> resolve(EvalState & state) override
         {
             Value & v = parent->value(state);
             state.mkList(v, values.size());
@@ -62,19 +64,19 @@ class JSONSax : nlohmann::json_sax<json> {
             }
             return std::move(parent);
         }
-        void add() override {
-            values.push_back(*v);
+        virtual void add() override {
+            values.push_back(v);
             v = nullptr;
-        }
+        };
     public:
-        JSONListState(std::unique_ptr<JSONState> && p, std::size_t reserve) : JSONState(std::move(p))
+        JSONListState(unique_ptr<JSONState> && p, std::size_t reserve) : JSONState(std::move(p))
         {
             values.reserve(reserve);
         }
     };
 
     EvalState & state;
-    std::unique_ptr<JSONState> rs;
+    unique_ptr<JSONState> rs;
 
     template<typename T, typename... Args> inline bool handle_value(T f, Args... args)
     {
@@ -106,14 +108,14 @@ public:
         return handle_value(mkInt, val);
     }
 
-    bool number_float(number_float_t val, const string_t & s)
+    bool number_float(number_float_t val, const string_t& s)
     {
         return handle_value(mkFloat, val);
     }
 
-    bool string(string_t & val)
+    bool string(string_t& val)
     {
-        return handle_value<void(Value&, const char*)>(mkString, val.c_str());
+        return handle_value<Value & (Value &, std::string_view)>(mkString, (std::string_view) val);
     }
 
     bool start_object(std::size_t len)
@@ -122,7 +124,7 @@ public:
         return true;
     }
 
-    bool key(string_t & name)
+    bool key(string_t& name)
     {
         dynamic_cast<JSONObjectState*>(rs.get())->key(name, state);
         return true;

src/libexpr/local.mk

@@ -4,27 +4,15 @@ libexpr_NAME = libnixexpr
 
 libexpr_DIR := $(d)
 
-libexpr_SOURCES := \
-  $(wildcard $(d)/*.cc) \
-  $(wildcard $(d)/primops/*.cc) \
-  $(wildcard $(d)/flake/*.cc) \
-  $(d)/lexer-tab.cc \
-  $(d)/parser-tab.cc
+libexpr_SOURCES := $(wildcard $(d)/*.cc) $(wildcard $(d)/primops/*.cc) $(d)/lexer-tab.cc $(d)/parser-tab.cc
 
-libexpr_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/libmain -I src/libexpr
-
-libexpr_LIBS = libutil libstore libfetchers libnixrust
+libexpr_LIBS = libutil libstore libnixrust
 
 libexpr_LDFLAGS =
 ifneq ($(OS), FreeBSD)
  libexpr_LDFLAGS += -ldl
 endif
 
-# The dependency on libgc must be propagated (i.e. meaning that
-# programs/libraries that use libexpr must explicitly pass -lgc),
-# because inline functions in libexpr's header files call libgc.
-libexpr_LDFLAGS_PROPAGATED = $(BDW_GC_LIBS)
-
 libexpr_ORDER_AFTER := $(d)/parser-tab.cc $(d)/parser-tab.hh $(d)/lexer-tab.cc $(d)/lexer-tab.hh
 
 $(d)/parser-tab.cc $(d)/parser-tab.hh: $(d)/parser.y
@@ -38,10 +26,3 @@ clean-files += $(d)/parser-tab.cc $(d)/parser-tab.hh $(d)/lexer-tab.cc $(d)/lexe
 dist-files += $(d)/parser-tab.cc $(d)/parser-tab.hh $(d)/lexer-tab.cc $(d)/lexer-tab.hh
 
 $(eval $(call install-file-in, $(d)/nix-expr.pc, $(prefix)/lib/pkgconfig, 0644))
-
-$(foreach i, $(wildcard src/libexpr/flake/*.hh), \
-  $(eval $(call install-file-in, $(i), $(includedir)/nix/flake, 0644)))
-
-$(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh
-
-$(d)/flake/flake.cc: $(d)/flake/call-flake.nix.gen.hh

src/libexpr/nixexpr.cc

@@ -80,7 +80,7 @@ void ExprString::show(std::ostream & str) const
 
 void ExprPath::show(std::ostream & str) const
 {
-    str << s;
+    str << v->path->s;
 }
 
 void ExprVar::show(std::ostream & str) const

src/libexpr/nixexpr.hh

@@ -78,7 +78,7 @@ struct Expr
     virtual void show(std::ostream & str) const;
     virtual void bindVars(const StaticEnv & env);
     virtual void eval(EvalState & state, Env & env, Value & v);
-    virtual Value * maybeThunk(EvalState & state, Env & env);
+    virtual Ptr<Value> maybeThunk(EvalState & state, Env & env);
     virtual void setName(Symbol & name);
 };
 
@@ -92,28 +92,37 @@ std::ostream & operator << (std::ostream & str, const Expr & e);
 struct ExprInt : Expr
 {
     NixInt n;
-    Value v;
-    ExprInt(NixInt n) : n(n) { mkInt(v, n); };
+    Ptr<Value> v;
+    ExprInt(NixInt n) : n(n) {
+        v = gc.alloc<Value>(Value::words());
+        mkInt(v, n);
+    };
     COMMON_METHODS
-    Value * maybeThunk(EvalState & state, Env & env);
+    Ptr<Value> maybeThunk(EvalState & state, Env & env) override;
 };
 
 struct ExprFloat : Expr
 {
     NixFloat nf;
-    Value v;
-    ExprFloat(NixFloat nf) : nf(nf) { mkFloat(v, nf); };
+    Ptr<Value> v;
+    ExprFloat(NixFloat nf) : nf(nf) {
+        v = gc.alloc<Value>(Value::words());
+        mkFloat(v, nf);
+    };
     COMMON_METHODS
-    Value * maybeThunk(EvalState & state, Env & env);
+    Ptr<Value> maybeThunk(EvalState & state, Env & env) override;
 };
 
 struct ExprString : Expr
 {
     Symbol s;
-    Value v;
-    ExprString(const Symbol & s) : s(s) { mkString(v, s); };
+    Ptr<Value> v;
+    ExprString(const Symbol & s) : s(s) {
+        v = gc.alloc<Value>(Value::words());
+        mkString(v, s);
+    };
     COMMON_METHODS
-    Value * maybeThunk(EvalState & state, Env & env);
+    Ptr<Value> maybeThunk(EvalState & state, Env & env) override;
 };
 
 /* Temporary class used during parsing of indented strings. */
@@ -125,11 +134,13 @@ struct ExprIndStr : Expr
 
 struct ExprPath : Expr
 {
-    string s;
-    Value v;
-    ExprPath(const string & s) : s(s) { mkPathNoCopy(v, this->s.c_str()); };
+    Ptr<Value> v;
+    ExprPath(const string & s) {
+        v = gc.alloc<Value>(Value::words());
+        mkPath(v, s);
+    };
     COMMON_METHODS
-    Value * maybeThunk(EvalState & state, Env & env);
+    Ptr<Value> maybeThunk(EvalState & state, Env & env) override;
 };
 
 struct ExprVar : Expr
@@ -153,7 +164,7 @@ struct ExprVar : Expr
     ExprVar(const Symbol & name) : name(name) { };
     ExprVar(const Pos & pos, const Symbol & name) : pos(pos), name(name) { };
     COMMON_METHODS
-    Value * maybeThunk(EvalState & state, Env & env);
+    Ptr<Value> maybeThunk(EvalState & state, Env & env) override;
 };
 
 struct ExprSelect : Expr
@@ -209,10 +220,9 @@ struct ExprList : Expr
 
 struct Formal
 {
-    Pos pos;
     Symbol name;
     Expr * def;
-    Formal(const Pos & pos, const Symbol & name, Expr * def) : pos(pos), name(name), def(def) { };
+    Formal(const Symbol & name, Expr * def) : name(name), def(def) { };
 };
 
 struct Formals
@@ -262,9 +272,8 @@ struct ExprWith : Expr
 
 struct ExprIf : Expr
 {
-    Pos pos;
     Expr * cond, * then, * else_;
-    ExprIf(const Pos & pos, Expr * cond, Expr * then, Expr * else_) : pos(pos), cond(cond), then(then), else_(else_) { };
+    ExprIf(Expr * cond, Expr * then, Expr * else_) : cond(cond), then(then), else_(else_) { };
     COMMON_METHODS
 };
 

src/libexpr/parser.y

@@ -335,7 +335,7 @@ expr_function
   ;
 
 expr_if
-  : IF expr THEN expr ELSE expr { $$ = new ExprIf(CUR_POS, $2, $4, $6); }
+  : IF expr THEN expr ELSE expr { $$ = new ExprIf($2, $4, $6); }
   | expr_op
   ;
 
@@ -531,8 +531,8 @@ formals
   ;
 
 formal
-  : ID { $$ = new Formal(CUR_POS, data->symbols.create($1), 0); }
-  | ID '?' expr { $$ = new Formal(CUR_POS, data->symbols.create($1), $3); }
+  : ID { $$ = new Formal(data->symbols.create($1), 0); }
+  | ID '?' expr { $$ = new Formal(data->symbols.create($1), $3); }
   ;
 
 %%
@@ -544,8 +544,7 @@ formal
 #include <unistd.h>
 
 #include "eval.hh"
-#include "filetransfer.hh"
-#include "fetchers.hh"
+#include "download.hh"
 #include "store-api.hh"
 
 
@@ -612,13 +611,13 @@ Expr * EvalState::parseExprFromFile(const Path & path, StaticEnv & staticEnv)
 }
 
 
-Expr * EvalState::parseExprFromString(std::string_view s, const Path & basePath, StaticEnv & staticEnv)
+Expr * EvalState::parseExprFromString(const string & s, const Path & basePath, StaticEnv & staticEnv)
 {
-    return parse(s.data(), "(string)", basePath, staticEnv);
+    return parse(s.c_str(), "(string)", basePath, staticEnv);
 }
 
 
-Expr * EvalState::parseExprFromString(std::string_view s, const Path & basePath)
+Expr * EvalState::parseExprFromString(const string & s, const Path & basePath)
 {
     return parseExprFromString(s, basePath, staticBaseEnv);
 }
@@ -688,9 +687,10 @@ std::pair<bool, std::string> EvalState::resolveSearchPathElem(const SearchPathEl
 
     if (isUri(elem.second)) {
         try {
-            res = { true, store->toRealPath(fetchers::downloadTarball(
-                        store, resolveUri(elem.second), "source", false).storePath) };
-        } catch (FileTransferError & e) {
+            CachedDownloadRequest request(elem.second);
+            request.unpack = true;
+            res = { true, getDownloader()->downloadCached(store, request).path };
+        } catch (DownloadError & e) {
             printError(format("warning: Nix search path entry '%1%' cannot be downloaded, ignoring") % elem.second);
             res = { false, "" };
         }

src/libexpr/primops.cc

@@ -1,5 +1,6 @@
 #include "archive.hh"
 #include "derivations.hh"
+#include "download.hh"
 #include "eval-inline.hh"
 #include "eval.hh"
 #include "globals.hh"
@@ -50,20 +51,20 @@ void EvalState::realiseContext(const PathSet & context)
     std::vector<StorePathWithOutputs> drvs;
 
     for (auto & i : context) {
-        auto [ctxS, outputName] = decodeContext(i);
-        auto ctx = store->parseStorePath(ctxS);
+        std::pair<string, string> decoded = decodeContext(i);
+        auto ctx = store->parseStorePath(decoded.first);
         if (!store->isValidPath(ctx))
             throw InvalidPathError(store->printStorePath(ctx));
-        if (!outputName.empty() && ctx.isDerivation()) {
-            drvs.push_back(StorePathWithOutputs{ctx.clone(), {outputName}});
+        if (!decoded.second.empty() && ctx.isDerivation()) {
+            drvs.push_back(StorePathWithOutputs{ctx.clone(), {decoded.second}});
 
             /* Add the output of this derivation to the allowed
                paths. */
             if (allowedPaths) {
-                auto drv = store->derivationFromPath(ctx);
-                DerivationOutputs::iterator i = drv.outputs.find(outputName);
+                auto drv = store->derivationFromPath(store->parseStorePath(decoded.first));
+                DerivationOutputs::iterator i = drv.outputs.find(decoded.second);
                 if (i == drv.outputs.end())
-                    throw Error("derivation '%s' does not have an output named '%s'", ctxS, outputName);
+                    throw Error("derivation '%s' does not have an output named '%s'", decoded.first, decoded.second);
                 allowedPaths->insert(store->printStorePath(i->second.path));
             }
         }
@@ -79,7 +80,6 @@ void EvalState::realiseContext(const PathSet & context)
     StorePathSet willBuild, willSubstitute, unknown;
     unsigned long long downloadSize, narSize;
     store->queryMissing(drvs, willBuild, willSubstitute, unknown, downloadSize, narSize);
-
     store->buildPaths(drvs);
 }
 
@@ -103,14 +103,13 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args
     // FIXME
     if (state.store->isStorePath(path) && state.store->isValidPath(state.store->parseStorePath(path)) && isDerivation(path)) {
         Derivation drv = readDerivation(*state.store, realPath);
-        Value & w = *state.allocValue();
+        auto w = state.allocValue();
         state.mkAttrs(w, 3 + drv.outputs.size());
-        Value * v2 = state.allocAttr(w, state.sDrvPath);
+        auto v2 = state.allocAttr(w, state.sDrvPath);
         mkString(*v2, path, {"=" + path});
         v2 = state.allocAttr(w, state.sName);
         mkString(*v2, drv.env["name"]);
-        Value * outputsVal =
-            state.allocAttr(w, state.symbols.create("outputs"));
+        auto outputsVal = state.allocAttr(w, state.symbols.create("outputs"));
         state.mkList(*outputsVal, drv.outputs.size());
         unsigned int outputs_index = 0;
 
@@ -120,26 +119,19 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args
             outputsVal->listElems()[outputs_index] = state.allocValue();
             mkString(*(outputsVal->listElems()[outputs_index++]), o.first);
         }
-        w.attrs->sort();
-
-        static RootValue fun;
-        if (!fun) {
-            fun = allocRootValue(state.allocValue());
-            state.eval(state.parseExprFromString(
-                #include "imported-drv-to-derivation.nix.gen.hh"
-                , "/"), **fun);
-        }
-
-        state.forceFunction(**fun, pos);
-        mkApp(v, **fun, w);
+        w->attrs->sort();
+        auto fun = state.allocValue();
+        state.evalFile(settings.nixDataDir + "/nix/corepkgs/imported-drv-to-derivation.nix", fun);
+        state.forceFunction(fun, pos);
+        mkApp(v, fun, w);
         state.forceAttrs(v, pos);
     } else {
         state.forceAttrs(*args[0]);
         if (args[0]->attrs->empty())
             state.evalFile(realPath, v);
         else {
-            Env * env = &state.allocEnv(args[0]->attrs->size());
-            env->up = &state.baseEnv;
+            auto env = state.allocEnv(args[0]->attrs->size());
+            env->up = state.baseEnv;
 
             StaticEnv staticEnv(false, &state.staticBaseEnv);
 
@@ -242,24 +234,29 @@ void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v)
 /* Return a string representing the type of the expression. */
 static void prim_typeOf(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
-    state.forceValue(*args[0], pos);
+    state.forceValue(*args[0]);
     string t;
     switch (args[0]->type) {
         case tInt: t = "int"; break;
         case tBool: t = "bool"; break;
-        case tString: t = "string"; break;
+        case tShortString:
+        case tStaticString:
+        case tLongString:
+            t = "string"; break;
         case tPath: t = "path"; break;
         case tNull: t = "null"; break;
         case tAttrs: t = "set"; break;
-        case tList1: case tList2: case tListN: t = "list"; break;
+        case tList0: case tList1: case tList2: case tListN: t = "list"; break;
         case tLambda:
         case tPrimOp:
         case tPrimOpApp:
             t = "lambda";
             break;
+#if 0
         case tExternal:
             t = args[0]->external->typeOf();
             break;
+#endif
         case tFloat: t = "float"; break;
         default: abort();
     }
@@ -270,7 +267,7 @@ static void prim_typeOf(EvalState & state, const Pos & pos, Value * * args, Valu
 /* Determine whether the argument is the null value. */
 static void prim_isNull(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
-    state.forceValue(*args[0], pos);
+    state.forceValue(*args[0]);
     mkBool(v, args[0]->type == tNull);
 }
 
@@ -278,7 +275,7 @@ static void prim_isNull(EvalState & state, const Pos & pos, Value * * args, Valu
 /* Determine whether the argument is a function. */
 static void prim_isFunction(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
-    state.forceValue(*args[0], pos);
+    state.forceValue(*args[0]);
     bool res;
     switch (args[0]->type) {
         case tLambda:
@@ -297,36 +294,36 @@ static void prim_isFunction(EvalState & state, const Pos & pos, Value * * args,
 /* Determine whether the argument is an integer. */
 static void prim_isInt(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
-    state.forceValue(*args[0], pos);
+    state.forceValue(*args[0]);
     mkBool(v, args[0]->type == tInt);
 }
 
 /* Determine whether the argument is a float. */
 static void prim_isFloat(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
-    state.forceValue(*args[0], pos);
+    state.forceValue(*args[0]);
     mkBool(v, args[0]->type == tFloat);
 }
 
 /* Determine whether the argument is a string. */
 static void prim_isString(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
-    state.forceValue(*args[0], pos);
-    mkBool(v, args[0]->type == tString);
+    state.forceValue(*args[0]);
+    mkBool(v, args[0]->isString());
 }
 
 
 /* Determine whether the argument is a Boolean. */
 static void prim_isBool(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
-    state.forceValue(*args[0], pos);
+    state.forceValue(*args[0]);
     mkBool(v, args[0]->type == tBool);
 }
 
 /* Determine whether the argument is a path. */
 static void prim_isPath(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
-    state.forceValue(*args[0], pos);
+    state.forceValue(*args[0]);
     mkBool(v, args[0]->type == tPath);
 }
 
@@ -338,6 +335,8 @@ struct CompareValues
             return v1->fpoint < v2->integer;
         if (v1->type == tInt && v2->type == tFloat)
             return v1->integer < v2->fpoint;
+        if (v1->isString() && v2->isString())
+            return strcmp(v1->getString(), v2->getString()) < 0;
         if (v1->type != v2->type)
             throw EvalError(format("cannot compare %1% with %2%") % showType(*v1) % showType(*v2));
         switch (v1->type) {
@@ -345,10 +344,8 @@ struct CompareValues
                 return v1->integer < v2->integer;
             case tFloat:
                 return v1->fpoint < v2->fpoint;
-            case tString:
-                return strcmp(v1->string.s, v2->string.s) < 0;
             case tPath:
-                return strcmp(v1->path, v2->path) < 0;
+                return strcmp(v1->path->s, v2->path->s) < 0;
             default:
                 throw EvalError(format("cannot compare %1% with %2%") % showType(*v1) % showType(*v2));
         }
@@ -356,11 +353,7 @@ struct CompareValues
 };
 
 
-#if HAVE_BOEHMGC
-typedef list<Value *, gc_allocator<Value *> > ValueList;
-#else
-typedef list<Value *> ValueList;
-#endif
+typedef list<Ptr<Value>> ValueList;
 
 
 static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * args, Value & v)
@@ -383,17 +376,17 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar
         args[0]->attrs->find(state.symbols.create("operator"));
     if (op == args[0]->attrs->end())
         throw EvalError(format("attribute 'operator' required, at %1%") % pos);
-    state.forceValue(*op->value, pos);
+    state.forceValue(*op->value);
 
     /* Construct the closure by applying the operator to element of
        `workSet', adding the result to `workSet', continuing until
        no new elements are found. */
     ValueList res;
     // `doneKeys' doesn't need to be a GC root, because its values are
-    // reachable from res.
+    // reachable from res. FIXME: dubious.
     set<Value *, CompareValues> doneKeys;
     while (!workSet.empty()) {
-        Value * e = *(workSet.begin());
+        auto e = *(workSet.begin());
         workSet.pop_front();
 
         state.forceAttrs(*e, pos);
@@ -402,20 +395,20 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar
             e->attrs->find(state.symbols.create("key"));
         if (key == e->attrs->end())
             throw EvalError(format("attribute 'key' required, at %1%") % pos);
-        state.forceValue(*key->value, pos);
+        state.forceValue(*key->value);
 
         if (!doneKeys.insert(key->value).second) continue;
         res.push_back(e);
 
         /* Call the `operator' function with `e' as argument. */
-        Value call;
+        Root<Value> call;
         mkApp(call, *op->value, *e);
         state.forceList(call, pos);
 
         /* Add the values returned by the operator to the work set. */
-        for (unsigned int n = 0; n < call.listSize(); ++n) {
-            state.forceValue(*call.listElems()[n], pos);
-            workSet.push_back(call.listElems()[n]);
+        for (unsigned int n = 0; n < call->listSize(); ++n) {
+            state.forceValue(*call->listElems()[n]);
+            workSet.push_back(call->listElems()[n]);
         }
     }
 
@@ -446,7 +439,7 @@ static void prim_throw(EvalState & state, const Pos & pos, Value * * args, Value
 static void prim_addErrorContext(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     try {
-        state.forceValue(*args[1], pos);
+        state.forceValue(*args[1]);
         v = *args[1];
     } catch (Error & e) {
         PathSet context;
@@ -462,7 +455,7 @@ static void prim_tryEval(EvalState & state, const Pos & pos, Value * * args, Val
 {
     state.mkAttrs(v, 2);
     try {
-        state.forceValue(*args[0], pos);
+        state.forceValue(*args[0]);
         v.attrs->push_back(Attr(state.sValue, args[0]));
         mkBool(*state.allocAttr(v, state.symbols.create("success")), true);
     } catch (AssertionError & e) {
@@ -484,8 +477,8 @@ static void prim_getEnv(EvalState & state, const Pos & pos, Value * * args, Valu
 /* Evaluate the first argument, then return the second argument. */
 static void prim_seq(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
-    state.forceValue(*args[0], pos);
-    state.forceValue(*args[1], pos);
+    state.forceValue(*args[0]);
+    state.forceValue(*args[1]);
     v = *args[1];
 }
 
@@ -495,7 +488,7 @@ static void prim_seq(EvalState & state, const Pos & pos, Value * * args, Value &
 static void prim_deepSeq(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     state.forceValueDeep(*args[0]);
-    state.forceValue(*args[1], pos);
+    state.forceValue(*args[1]);
     v = *args[1];
 }
 
@@ -504,12 +497,12 @@ static void prim_deepSeq(EvalState & state, const Pos & pos, Value * * args, Val
    return the second expression.  Useful for debugging. */
 static void prim_trace(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
-    state.forceValue(*args[0], pos);
-    if (args[0]->type == tString)
-        printError(format("trace: %1%") % args[0]->string.s);
+    state.forceValue(*args[0]);
+    if (args[0]->isString())
+        printError("trace: %s", args[0]->getString());
     else
-        printError(format("trace: %1%") % *args[0]);
-    state.forceValue(*args[1], pos);
+        printError("trace: %s", *args[0]);
+    state.forceValue(*args[1]);
     v = *args[1];
 }
 
@@ -600,7 +593,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
         try {
 
             if (ignoreNulls) {
-                state.forceValue(*i->value, pos);
+                state.forceValue(*i->value);
                 if (i->value->type == tNull) continue;
             }
 
@@ -756,6 +749,8 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
     auto drvPath = writeDerivation(state.store, drv, drvName, state.repair);
     auto drvPathS = state.store->printStorePath(drvPath);
 
+    if (state.derivationHook) state.derivationHook(drvPath, drv);
+
     printMsg(lvlChatty, "instantiated '%1%' -> '%2%'", drvName, drvPathS);
 
     /* Optimisation, but required in read-only mode! because in that
@@ -957,15 +952,20 @@ static void prim_readDir(EvalState & state, const Pos & pos, Value * * args, Val
     DirEntries entries = readDirectory(state.checkSourcePath(path));
     state.mkAttrs(v, entries.size());
 
+    auto sRegular = state.symbols.create("regular");
+    auto sDirectory = state.symbols.create("directory");
+    auto sSymlink = state.symbols.create("symlink");
+    auto sUnknown = state.symbols.create("unknown");
+
     for (auto & ent : entries) {
-        Value * ent_val = state.allocAttr(v, state.symbols.create(ent.name));
+        auto ent_val = state.allocAttr(v, state.symbols.create(ent.name));
         if (ent.type == DT_UNKNOWN)
             ent.type = getFileType(path + "/" + ent.name);
-        mkStringNoCopy(*ent_val,
-            ent.type == DT_REG ? "regular" :
-            ent.type == DT_DIR ? "directory" :
-            ent.type == DT_LNK ? "symlink" :
-            "unknown");
+        mkString(*ent_val,
+            ent.type == DT_REG ? sRegular :
+            ent.type == DT_DIR ? sDirectory :
+            ent.type == DT_LNK ? sSymlink :
+            sUnknown);
     }
 
     v.attrs->sort();
@@ -1021,9 +1021,7 @@ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Valu
 
     for (auto path : context) {
         if (path.at(0) != '/')
-            throw EvalError(format(
-                "in 'toFile': the file named '%1%' must not contain a reference "
-                "to a derivation but contains (%2%), at %3%") % name % path % pos);
+            throw EvalError(format("in 'toFile': the file '%1%' cannot refer to derivation outputs, at %2%") % name % pos);
         refs.insert(state.store->parseStorePath(path));
     }
 
@@ -1050,20 +1048,20 @@ static void addPath(EvalState & state, const Pos & pos, const string & name, con
 
         /* Call the filter function.  The first argument is the path,
            the second is a string indicating the type of the file. */
-        Value arg1;
+        auto arg1 = state.allocValue();
         mkString(arg1, path);
 
-        Value fun2;
+        Root<Value> fun2;
         state.callFunction(*filterFun, arg1, fun2, noPos);
 
-        Value arg2;
+        auto arg2 = state.allocValue();
         mkString(arg2,
             S_ISREG(st.st_mode) ? "regular" :
             S_ISDIR(st.st_mode) ? "directory" :
             S_ISLNK(st.st_mode) ? "symlink" :
             "unknown" /* not supported, will fail! */);
 
-        Value res;
+        Root<Value> res;
         state.callFunction(fun2, arg2, res, noPos);
 
         return state.forceBool(res, pos);
@@ -1093,7 +1091,7 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args
     if (!context.empty())
         throw EvalError(format("string '%1%' cannot refer to other paths, at %2%") % path % pos);
 
-    state.forceValue(*args[0], pos);
+    state.forceValue(*args[0]);
     if (args[0]->type != tLambda)
         throw TypeError(format("first argument in call to 'filterSource' is not a function but %1%, at %2%") % showType(*args[0]) % pos);
 
@@ -1119,7 +1117,7 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value
         } else if (attr.name == state.sName)
             name = state.forceStringNoCtx(*attr.value, *attr.pos);
         else if (n == "filter") {
-            state.forceValue(*attr.value, pos);
+            state.forceValue(*attr.value);
             filterFun = attr.value;
         } else if (n == "recursive")
             recursive = state.forceBool(*attr.value, *attr.pos);
@@ -1155,7 +1153,7 @@ static void prim_attrNames(EvalState & state, const Pos & pos, Value * * args, V
         mkString(*(v.listElems()[n++] = state.allocValue()), i.name);
 
     std::sort(v.listElems(), v.listElems() + n,
-              [](Value * v1, Value * v2) { return strcmp(v1->string.s, v2->string.s) < 0; });
+        [](Value * v1, Value * v2) { return strcmp(v1->getString(), v2->getString()) < 0; });
 }
 
 
@@ -1190,7 +1188,7 @@ void prim_getAttr(EvalState & state, const Pos & pos, Value * * args, Value & v)
         throw EvalError(format("attribute '%1%' missing, at %2%") % attr % pos);
     // !!! add to stack trace?
     if (state.countCalls && i->pos) state.attrSelects[*i->pos]++;
-    state.forceValue(*i->value, pos);
+    state.forceValue(*i->value);
     v = *i->value;
 }
 
@@ -1220,7 +1218,7 @@ static void prim_hasAttr(EvalState & state, const Pos & pos, Value * * args, Val
 /* Determine whether the argument is a set. */
 static void prim_isAttrs(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
-    state.forceValue(*args[0], pos);
+    state.forceValue(*args[0]);
     mkBool(v, args[0]->type == tAttrs);
 }
 
@@ -1232,10 +1230,9 @@ static void prim_removeAttrs(EvalState & state, const Pos & pos, Value * * args,
 
     /* Get the attribute names to be removed. */
     std::set<Symbol> names;
-    for (unsigned int i = 0; i < args[1]->listSize(); ++i) {
-        state.forceStringNoCtx(*args[1]->listElems()[i], pos);
-        names.insert(state.symbols.create(args[1]->listElems()[i]->string.s));
-    }
+    for (unsigned int i = 0; i < args[1]->listSize(); ++i)
+        names.insert(state.symbols.create(
+                state.forceStringNoCtx(*args[1]->listElems()[i], pos)));
 
     /* Copy all attributes not in that set.  Note that we don't need
        to sort v.attrs because it's a subset of an already sorted
@@ -1346,7 +1343,7 @@ static void prim_catAttrs(EvalState & state, const Pos & pos, Value * * args, Va
 */
 static void prim_functionArgs(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
-    state.forceValue(*args[0], pos);
+    state.forceValue(*args[0]);
     if (args[0]->type != tLambda)
         throw TypeError(format("'functionArgs' requires a function, at %1%") % pos);
 
@@ -1356,12 +1353,9 @@ static void prim_functionArgs(EvalState & state, const Pos & pos, Value * * args
     }
 
     state.mkAttrs(v, args[0]->lambda.fun->formals->formals.size());
-    for (auto & i : args[0]->lambda.fun->formals->formals) {
+    for (auto & i : args[0]->lambda.fun->formals->formals)
         // !!! should optimise booleans (allocate only once)
-        Value * value = state.allocValue();
-        v.attrs->push_back(Attr(i.name, value, &i.pos));
-        mkBool(*value, i.def);
-    }
+        mkBool(*state.allocAttr(v, i.name), i.def);
     v.attrs->sort();
 }
 
@@ -1374,8 +1368,8 @@ static void prim_mapAttrs(EvalState & state, const Pos & pos, Value * * args, Va
     state.mkAttrs(v, args[1]->attrs->size());
 
     for (auto & i : *args[1]->attrs) {
-        Value * vName = state.allocValue();
-        Value * vFun2 = state.allocValue();
+        auto vName = state.allocValue();
+        auto vFun2 = state.allocValue();
         mkString(*vName, i.name);
         mkApp(*vFun2, *args[0], *vName);
         mkApp(*state.allocAttr(v, i.name), *vFun2, *i.value);
@@ -1392,7 +1386,7 @@ static void prim_mapAttrs(EvalState & state, const Pos & pos, Value * * args, Va
 /* Determine whether the argument is a list. */
 static void prim_isList(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
-    state.forceValue(*args[0], pos);
+    state.forceValue(*args[0]);
     mkBool(v, args[0]->isList());
 }
 
@@ -1402,7 +1396,7 @@ static void elemAt(EvalState & state, const Pos & pos, Value & list, int n, Valu
     state.forceList(list, pos);
     if (n < 0 || (unsigned int) n >= list.listSize())
         throw Error(format("list index %1% is out of bounds, at %2%") % n % pos);
-    state.forceValue(*list.listElems()[n], pos);
+    state.forceValue(*list.listElems()[n]);
     v = *list.listElems()[n];
 }
 
@@ -1462,7 +1456,7 @@ static void prim_filter(EvalState & state, const Pos & pos, Value * * args, Valu
 
     bool same = true;
     for (unsigned int n = 0; n < args[1]->listSize(); ++n) {
-        Value res;
+        Root<Value> res;
         state.callFunction(*args[0], *args[1]->listElems()[n], res, noPos);
         if (state.forceBool(res, pos))
             vs[k++] = args[1]->listElems()[n];
@@ -1517,17 +1511,17 @@ static void prim_foldlStrict(EvalState & state, const Pos & pos, Value * * args,
     state.forceList(*args[2], pos);
 
     if (args[2]->listSize()) {
-        Value * vCur = args[1];
+        Ptr<Value> vCur = args[1];
 
         for (unsigned int n = 0; n < args[2]->listSize(); ++n) {
-            Value vTmp;
+            Root<Value> vTmp; // FIXME: correct?
             state.callFunction(*args[0], *vCur, vTmp, pos);
-            vCur = n == args[2]->listSize() - 1 ? &v : state.allocValue();
+            vCur = n == args[2]->listSize() - 1 ? Ptr(&v) : state.allocValue();
             state.callFunction(vTmp, *args[2]->listElems()[n], *vCur, pos);
         }
-        state.forceValue(v, pos);
+        state.forceValue(v);
     } else {
-        state.forceValue(*args[1], pos);
+        state.forceValue(*args[1]);
         v = *args[1];
     }
 }
@@ -1538,7 +1532,7 @@ static void anyOrAll(bool any, EvalState & state, const Pos & pos, Value * * arg
     state.forceFunction(*args[0], pos);
     state.forceList(*args[1], pos);
 
-    Value vTmp;
+    Root<Value> vTmp;
     for (unsigned int n = 0; n < args[1]->listSize(); ++n) {
         state.callFunction(*args[0], *args[1]->listElems()[n], vTmp, pos);
         bool res = state.forceBool(vTmp, pos);
@@ -1574,7 +1568,7 @@ static void prim_genList(EvalState & state, const Pos & pos, Value * * args, Val
     state.mkList(v, len);
 
     for (unsigned int n = 0; n < (unsigned int) len; ++n) {
-        Value * arg = state.allocValue();
+        auto arg = state.allocValue();
         mkInt(*arg, n);
         mkApp(*(v.listElems()[n] = state.allocValue()), *args[0], *arg);
     }
@@ -1592,7 +1586,7 @@ static void prim_sort(EvalState & state, const Pos & pos, Value * * args, Value
     auto len = args[1]->listSize();
     state.mkList(v, len);
     for (unsigned int n = 0; n < len; ++n) {
-        state.forceValue(*args[1]->listElems()[n], pos);
+        state.forceValue(*args[1]->listElems()[n]);
         v.listElems()[n] = args[1]->listElems()[n];
     }
 
@@ -1603,7 +1597,7 @@ static void prim_sort(EvalState & state, const Pos & pos, Value * * args, Value
         if (args[0]->type == tPrimOp && args[0]->primOp->fun == prim_lessThan)
             return CompareValues()(a, b);
 
-        Value vTmp1, vTmp2;
+        Root<Value> vTmp1, vTmp2; // FIXME
         state.callFunction(*args[0], *a, vTmp1, pos);
         state.callFunction(vTmp1, *b, vTmp2, pos);
         return state.forceBool(vTmp2, pos);
@@ -1623,12 +1617,13 @@ static void prim_partition(EvalState & state, const Pos & pos, Value * * args, V
 
     auto len = args[1]->listSize();
 
-    ValueVector right, wrong;
+    // Note: these Values are reachable via args[0].
+    std::vector<Value *> right, wrong;
 
     for (unsigned int n = 0; n < len; ++n) {
         auto vElem = args[1]->listElems()[n];
-        state.forceValue(*vElem, pos);
-        Value res;
+        state.forceValue(*vElem);
+        Root<Value> res;
         state.callFunction(*args[0], *vElem, res, pos);
         if (state.forceBool(res, pos))
             right.push_back(vElem);
@@ -1638,13 +1633,13 @@ static void prim_partition(EvalState & state, const Pos & pos, Value * * args, V
 
     state.mkAttrs(v, 2);
 
-    Value * vRight = state.allocAttr(v, state.sRight);
+    auto vRight = state.allocAttr(v, state.sRight);
     auto rsize = right.size();
     state.mkList(*vRight, rsize);
     if (rsize)
         memcpy(vRight->listElems(), right.data(), sizeof(Value *) * rsize);
 
-    Value * vWrong = state.allocAttr(v, state.sWrong);
+    auto vWrong = state.allocAttr(v, state.sWrong);
     auto wsize = wrong.size();
     state.mkList(*vWrong, wsize);
     if (wsize)
@@ -1662,22 +1657,23 @@ static void prim_concatMap(EvalState & state, const Pos & pos, Value * * args, V
     state.forceList(*args[1], pos);
     auto nrLists = args[1]->listSize();
 
-    Value lists[nrLists];
+    // FIXME: Root<>[] is inefficient
+    Root<Value> lists[nrLists];
     size_t len = 0;
 
     for (unsigned int n = 0; n < nrLists; ++n) {
         Value * vElem = args[1]->listElems()[n];
         state.callFunction(*args[0], *vElem, lists[n], pos);
         state.forceList(lists[n], pos);
-        len += lists[n].listSize();
+        len += lists[n]->listSize();
     }
 
     state.mkList(v, len);
     auto out = v.listElems();
     for (unsigned int n = 0, pos = 0; n < nrLists; ++n) {
-        auto l = lists[n].listSize();
+        auto l = lists[n]->listSize();
         if (l)
-            memcpy(out + pos, lists[n].listElems(), l * sizeof(Value *));
+            memcpy(out + pos, lists[n]->listElems(), l * sizeof(Value *));
         pos += l;
     }
 }
@@ -1758,8 +1754,8 @@ static void prim_bitXor(EvalState & state, const Pos & pos, Value * * args, Valu
 
 static void prim_lessThan(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
-    state.forceValue(*args[0], pos);
-    state.forceValue(*args[1], pos);
+    state.forceValue(*args[0]);
+    state.forceValue(*args[1]);
     CompareValues comp;
     mkBool(v, comp(args[0], args[1]));
 }
@@ -1823,21 +1819,19 @@ static void prim_hashString(EvalState & state, const Pos & pos, Value * * args,
 
 /* Match a regular expression against a string and return either
    ‘null’ or a list containing substring matches. */
-void prim_match(EvalState & state, const Pos & pos, Value * * args, Value & v)
+static void prim_match(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     auto re = state.forceStringNoCtx(*args[0], pos);
 
     try {
 
-        auto regex = state.regexCache.find(re);
-        if (regex == state.regexCache.end())
-            regex = state.regexCache.emplace(re, std::regex(re, std::regex::extended)).first;
+        std::regex regex(re, std::regex::extended);
 
         PathSet context;
         const std::string str = state.forceString(*args[1], context, pos);
 
         std::smatch match;
-        if (!std::regex_match(str, match, regex->second)) {
+        if (!std::regex_match(str, match, regex)) {
             mkNull(v);
             return;
         }
@@ -2050,6 +2044,68 @@ static void prim_splitVersion(EvalState & state, const Pos & pos, Value * * args
 }
 
 
+/*************************************************************
+ * Networking
+ *************************************************************/
+
+
+void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
+    const string & who, bool unpack, const std::string & defaultName)
+{
+    CachedDownloadRequest request("");
+    request.unpack = unpack;
+    request.name = defaultName;
+
+    state.forceValue(*args[0]);
+
+    if (args[0]->type == tAttrs) {
+
+        state.forceAttrs(*args[0], pos);
+
+        for (auto & attr : *args[0]->attrs) {
+            string n(attr.name);
+            if (n == "url")
+                request.uri = state.forceStringNoCtx(*attr.value, *attr.pos);
+            else if (n == "sha256")
+                request.expectedHash = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256);
+            else if (n == "name")
+                request.name = state.forceStringNoCtx(*attr.value, *attr.pos);
+            else
+                throw EvalError(format("unsupported argument '%1%' to '%2%', at %3%") % attr.name % who % attr.pos);
+        }
+
+        if (request.uri.empty())
+            throw EvalError(format("'url' argument required, at %1%") % pos);
+
+    } else
+        request.uri = state.forceStringNoCtx(*args[0], pos);
+
+    state.checkURI(request.uri);
+
+    if (evalSettings.pureEval && !request.expectedHash)
+        throw Error("in pure evaluation mode, '%s' requires a 'sha256' argument", who);
+
+    auto res = getDownloader()->downloadCached(state.store, request);
+
+    if (state.allowedPaths)
+        state.allowedPaths->insert(res.path);
+
+    mkString(v, res.storePath, PathSet({res.storePath}));
+}
+
+
+static void prim_fetchurl(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    fetch(state, pos, args, v, "fetchurl", false, "");
+}
+
+
+static void prim_fetchTarball(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    fetch(state, pos, args, v, "fetchTarball", true, "source");
+}
+
+
 /*************************************************************
  * Primop registration
  *************************************************************/
@@ -2067,10 +2123,10 @@ RegisterPrimOp::RegisterPrimOp(std::string name, size_t arity, PrimOpFun fun)
 
 void EvalState::createBaseEnv()
 {
-    baseEnv.up = 0;
+    baseEnv->up = 0;
 
     /* Add global constants such as `true' to the base environment. */
-    Value v;
+    auto v = allocValue();
 
     /* `builtins' must be first! */
     mkAttrs(v, 128);
@@ -2088,7 +2144,7 @@ void EvalState::createBaseEnv()
     auto vThrow = addPrimOp("throw", 1, prim_throw);
 
     auto addPurityError = [&](const std::string & name) {
-        Value * v2 = allocValue();
+        auto v2 = allocValue();
         mkString(*v2, fmt("'%s' is not allowed in pure evaluation mode", name));
         mkApp(v, *vThrow, *v2);
         addConstant(name, v);
@@ -2119,7 +2175,7 @@ void EvalState::createBaseEnv()
 
     // Miscellaneous
     auto vScopedImport = addPrimOp("scopedImport", 2, prim_scopedImport);
-    Value * v2 = allocValue();
+    auto v2 = allocValue();
     mkAttrs(*v2, 0);
     mkApp(v, *vScopedImport, *v2);
     forceValue(v);
@@ -2232,6 +2288,10 @@ void EvalState::createBaseEnv()
     addPrimOp("derivationStrict", 1, prim_derivationStrict);
     addPrimOp("placeholder", 1, prim_placeholder);
 
+    // Networking
+    addPrimOp("__fetchurl", 1, prim_fetchurl);
+    addPrimOp("fetchTarball", 1, prim_fetchTarball);
+
     /* Add a wrapper around the derivation primop that computes the
        `drvPath' and `outPath' attributes lazily. */
     string path = canonPath(settings.nixDataDir + "/nix/corepkgs/derivation.nix", true);
@@ -2243,7 +2303,7 @@ void EvalState::createBaseEnv()
     mkList(v, searchPath.size());
     int n = 0;
     for (auto & i : searchPath) {
-        v2 = v.listElems()[n++] = allocValue();
+        v2 = v->listElems()[n++] = allocValue();
         mkAttrs(*v2, 2);
         mkString(*allocAttr(*v2, symbols.create("path")), i.second);
         mkString(*allocAttr(*v2, symbols.create("prefix")), i.first);
@@ -2257,7 +2317,7 @@ void EvalState::createBaseEnv()
 
     /* Now that we've added all primops, sort the `builtins' set,
        because attribute lookups expect it to be sorted. */
-    baseEnv.values[0]->attrs->sort();
+    baseEnv->values[0]->attrs->sort();
 }
 
 

src/libexpr/primops.hh

@@ -20,7 +20,6 @@ struct RegisterPrimOp
    them. */
 /* Load a ValueInitializer from a DSO and return whatever it initializes */
 void prim_importNative(EvalState & state, const Pos & pos, Value * * args, Value & v);
-
 /* Execute a program and parse its output */
 void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v);
 

src/libexpr/primops/context.cc

@@ -160,7 +160,7 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg
         if (iter != i.value->attrs->end()) {
             if (state.forceBool(*iter->value, *iter->pos)) {
                 if (!isDerivation(i.name)) {
-                    throw EvalError("Tried to add all-outputs context of %s, which is not a derivation, to a string, at %s", i.name, i.pos);
+                    throw EvalError("tried to add all-outputs context of %s, which is not a derivation, to a string, at %s", i.name, *i.pos);
                 }
                 context.insert("=" + string(i.name));
             }
@@ -170,7 +170,7 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg
         if (iter != i.value->attrs->end()) {
             state.forceList(*iter->value, *iter->pos);
             if (iter->value->listSize() && !isDerivation(i.name)) {
-                throw EvalError("Tried to add derivation output context of %s, which is not a derivation, to a string, at %s", i.name, i.pos);
+                throw EvalError("tried to add derivation output context of %s, which is not a derivation, to a string, at %s", i.name, *i.pos);
             }
             for (unsigned int n = 0; n < iter->value->listSize(); ++n) {
                 auto name = state.forceStringNoCtx(*iter->value->listElems()[n], *iter->pos);

src/libexpr/primops/fetchGit.cc

@@ -1,19 +1,203 @@
 #include "primops.hh"
 #include "eval-inline.hh"
+#include "download.hh"
 #include "store-api.hh"
+#include "pathlocks.hh"
 #include "hash.hh"
-#include "fetchers.hh"
-#include "url.hh"
+#include "tarfile.hh"
+
+#include <sys/time.h>
+
+#include <regex>
+
+#include <nlohmann/json.hpp>
+
+using namespace std::string_literals;
 
 namespace nix {
 
+struct GitInfo
+{
+    Path storePath;
+    std::string rev;
+    std::string shortRev;
+    uint64_t revCount = 0;
+};
+
+std::regex revRegex("^[0-9a-fA-F]{40}$");
+
+GitInfo exportGit(ref<Store> store, const std::string & uri,
+    std::optional<std::string> ref, std::string rev,
+    const std::string & name)
+{
+    if (evalSettings.pureEval && rev == "")
+        throw Error("in pure evaluation mode, 'fetchGit' requires a Git revision");
+
+    if (!ref && rev == "" && hasPrefix(uri, "/") && pathExists(uri + "/.git")) {
+
+        bool clean = true;
+
+        try {
+            runProgram("git", true, { "-C", uri, "diff-index", "--quiet", "HEAD", "--" });
+        } catch (ExecError & e) {
+            if (!WIFEXITED(e.status) || WEXITSTATUS(e.status) != 1) throw;
+            clean = false;
+        }
+
+        if (!clean) {
+
+            /* This is an unclean working tree. So copy all tracked files. */
+            GitInfo gitInfo;
+            gitInfo.rev = "0000000000000000000000000000000000000000";
+            gitInfo.shortRev = std::string(gitInfo.rev, 0, 7);
+
+            auto files = tokenizeString<std::set<std::string>>(
+                runProgram("git", true, { "-C", uri, "ls-files", "-z" }), "\0"s);
+
+            PathFilter filter = [&](const Path & p) -> bool {
+                assert(hasPrefix(p, uri));
+                std::string file(p, uri.size() + 1);
+
+                auto st = lstat(p);
+
+                if (S_ISDIR(st.st_mode)) {
+                    auto prefix = file + "/";
+                    auto i = files.lower_bound(prefix);
+                    return i != files.end() && hasPrefix(*i, prefix);
+                }
+
+                return files.count(file);
+            };
+
+            gitInfo.storePath = store->printStorePath(store->addToStore("source", uri, true, htSHA256, filter));
+
+            return gitInfo;
+        }
+
+        // clean working tree, but no ref or rev specified.  Use 'HEAD'.
+        rev = chomp(runProgram("git", true, { "-C", uri, "rev-parse", "HEAD" }));
+        ref = "HEAD"s;
+    }
+
+    if (!ref) ref = "HEAD"s;
+
+    if (rev != "" && !std::regex_match(rev, revRegex))
+        throw Error("invalid Git revision '%s'", rev);
+
+    deletePath(getCacheDir() + "/nix/git");
+
+    Path cacheDir = getCacheDir() + "/nix/gitv2/" + hashString(htSHA256, uri).to_string(Base32, false);
+
+    if (!pathExists(cacheDir)) {
+        createDirs(dirOf(cacheDir));
+        runProgram("git", true, { "init", "--bare", cacheDir });
+    }
+
+    Path localRefFile;
+    if (ref->compare(0, 5, "refs/") == 0)
+        localRefFile = cacheDir + "/" + *ref;
+    else
+        localRefFile = cacheDir + "/refs/heads/" + *ref;
+
+    bool doFetch;
+    time_t now = time(0);
+    /* If a rev was specified, we need to fetch if it's not in the
+       repo. */
+    if (rev != "") {
+        try {
+            runProgram("git", true, { "-C", cacheDir, "cat-file", "-e", rev });
+            doFetch = false;
+        } catch (ExecError & e) {
+            if (WIFEXITED(e.status)) {
+                doFetch = true;
+            } else {
+                throw;
+            }
+        }
+    } else {
+        /* If the local ref is older than ‘tarball-ttl’ seconds, do a
+           git fetch to update the local ref to the remote ref. */
+        struct stat st;
+        doFetch = stat(localRefFile.c_str(), &st) != 0 ||
+            (uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now;
+    }
+    if (doFetch)
+    {
+        Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Git repository '%s'", uri));
+
+        // FIXME: git stderr messes up our progress indicator, so
+        // we're using --quiet for now. Should process its stderr.
+        runProgram("git", true, { "-C", cacheDir, "fetch", "--quiet", "--force", "--", uri, fmt("%s:%s", *ref, *ref) });
+
+        struct timeval times[2];
+        times[0].tv_sec = now;
+        times[0].tv_usec = 0;
+        times[1].tv_sec = now;
+        times[1].tv_usec = 0;
+
+        utimes(localRefFile.c_str(), times);
+    }
+
+    // FIXME: check whether rev is an ancestor of ref.
+    GitInfo gitInfo;
+    gitInfo.rev = rev != "" ? rev : chomp(readFile(localRefFile));
+    gitInfo.shortRev = std::string(gitInfo.rev, 0, 7);
+
+    printTalkative("using revision %s of repo '%s'", gitInfo.rev, uri);
+
+    std::string storeLinkName = hashString(htSHA512, name + std::string("\0"s) + gitInfo.rev).to_string(Base32, false);
+    Path storeLink = cacheDir + "/" + storeLinkName + ".link";
+    PathLocks storeLinkLock({storeLink}, fmt("waiting for lock on '%1%'...", storeLink)); // FIXME: broken
+
+    try {
+        auto json = nlohmann::json::parse(readFile(storeLink));
+
+        assert(json["name"] == name && json["rev"] == gitInfo.rev);
+
+        gitInfo.storePath = json["storePath"];
+
+        if (store->isValidPath(store->parseStorePath(gitInfo.storePath))) {
+            gitInfo.revCount = json["revCount"];
+            return gitInfo;
+        }
+
+    } catch (SysError & e) {
+        if (e.errNo != ENOENT) throw;
+    }
+
+    auto source = sinkToSource([&](Sink & sink) {
+        RunOptions gitOptions("git", { "-C", cacheDir, "archive", gitInfo.rev });
+        gitOptions.standardOut = &sink;
+        runProgram2(gitOptions);
+    });
+
+    Path tmpDir = createTempDir();
+    AutoDelete delTmpDir(tmpDir, true);
+
+    unpackTarfile(*source, tmpDir);
+
+    gitInfo.storePath = store->printStorePath(store->addToStore(name, tmpDir));
+
+    gitInfo.revCount = std::stoull(runProgram("git", true, { "-C", cacheDir, "rev-list", "--count", gitInfo.rev }));
+
+    nlohmann::json json;
+    json["storePath"] = gitInfo.storePath;
+    json["uri"] = uri;
+    json["name"] = name;
+    json["rev"] = gitInfo.rev;
+    json["revCount"] = gitInfo.revCount;
+
+    writeFile(storeLink, json.dump());
+
+    return gitInfo;
+}
+
 static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     std::string url;
     std::optional<std::string> ref;
-    std::optional<Hash> rev;
+    std::string rev;
     std::string name = "source";
-    bool fetchSubmodules = false;
     PathSet context;
 
     state.forceValue(*args[0]);
@@ -29,11 +213,9 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va
             else if (n == "ref")
                 ref = state.forceStringNoCtx(*attr.value, *attr.pos);
             else if (n == "rev")
-                rev = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA1);
+                rev = state.forceStringNoCtx(*attr.value, *attr.pos);
             else if (n == "name")
                 name = state.forceStringNoCtx(*attr.value, *attr.pos);
-            else if (n == "submodules")
-                fetchSubmodules = state.forceBool(*attr.value, *attr.pos);
             else
                 throw EvalError("unsupported argument '%s' to 'fetchGit', at %s", attr.name, *attr.pos);
         }
@@ -48,36 +230,17 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va
     // whitelist. Ah well.
     state.checkURI(url);
 
-    if (evalSettings.pureEval && !rev)
-        throw Error("in pure evaluation mode, 'fetchGit' requires a Git revision");
-
-    fetchers::Attrs attrs;
-    attrs.insert_or_assign("type", "git");
-    attrs.insert_or_assign("url", url.find("://") != std::string::npos ? url : "file://" + url);
-    if (ref) attrs.insert_or_assign("ref", *ref);
-    if (rev) attrs.insert_or_assign("rev", rev->gitRev());
-    if (fetchSubmodules) attrs.insert_or_assign("submodules", true);
-    auto input = fetchers::inputFromAttrs(attrs);
-
-    // FIXME: use name?
-    auto [tree, input2] = input->fetchTree(state.store);
+    auto gitInfo = exportGit(state.store, url, ref, rev, name);
 
     state.mkAttrs(v, 8);
-    auto storePath = state.store->printStorePath(tree.storePath);
-    mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath}));
-    // Backward compatibility: set 'rev' to
-    // 0000000000000000000000000000000000000000 for a dirty tree.
-    auto rev2 = input2->getRev().value_or(Hash(htSHA1));
-    mkString(*state.allocAttr(v, state.symbols.create("rev")), rev2.gitRev());
-    mkString(*state.allocAttr(v, state.symbols.create("shortRev")), rev2.gitShortRev());
-    // Backward compatibility: set 'revCount' to 0 for a dirty tree.
-    mkInt(*state.allocAttr(v, state.symbols.create("revCount")),
-        tree.info.revCount.value_or(0));
-    mkBool(*state.allocAttr(v, state.symbols.create("submodules")), fetchSubmodules);
+    mkString(*state.allocAttr(v, state.sOutPath), gitInfo.storePath, PathSet({gitInfo.storePath}));
+    mkString(*state.allocAttr(v, state.symbols.create("rev")), gitInfo.rev);
+    mkString(*state.allocAttr(v, state.symbols.create("shortRev")), gitInfo.shortRev);
+    mkInt(*state.allocAttr(v, state.symbols.create("revCount")), gitInfo.revCount);
     v.attrs->sort();
 
     if (state.allowedPaths)
-        state.allowedPaths->insert(tree.actualPath);
+        state.allowedPaths->insert(state.store->toRealPath(gitInfo.storePath));
 }
 
 static RegisterPrimOp r("fetchGit", 1, prim_fetchGit);

src/libexpr/primops/fetchMercurial.cc

@@ -1,18 +1,174 @@
 #include "primops.hh"
 #include "eval-inline.hh"
+#include "download.hh"
 #include "store-api.hh"
-#include "fetchers.hh"
-#include "url.hh"
+#include "pathlocks.hh"
+
+#include <sys/time.h>
 
 #include <regex>
 
+#include <nlohmann/json.hpp>
+
+using namespace std::string_literals;
+
 namespace nix {
 
+struct HgInfo
+{
+    Path storePath;
+    std::string branch;
+    std::string rev;
+    uint64_t revCount = 0;
+};
+
+std::regex commitHashRegex("^[0-9a-fA-F]{40}$");
+
+HgInfo exportMercurial(ref<Store> store, const std::string & uri,
+    std::string rev, const std::string & name)
+{
+    if (evalSettings.pureEval && rev == "")
+        throw Error("in pure evaluation mode, 'fetchMercurial' requires a Mercurial revision");
+
+    if (rev == "" && hasPrefix(uri, "/") && pathExists(uri + "/.hg")) {
+
+        bool clean = runProgram("hg", true, { "status", "-R", uri, "--modified", "--added", "--removed" }) == "";
+
+        if (!clean) {
+
+            /* This is an unclean working tree. So copy all tracked
+               files. */
+
+            printTalkative("copying unclean Mercurial working tree '%s'", uri);
+
+            HgInfo hgInfo;
+            hgInfo.rev = "0000000000000000000000000000000000000000";
+            hgInfo.branch = chomp(runProgram("hg", true, { "branch", "-R", uri }));
+
+            auto files = tokenizeString<std::set<std::string>>(
+                runProgram("hg", true, { "status", "-R", uri, "--clean", "--modified", "--added", "--no-status", "--print0" }), "\0"s);
+
+            PathFilter filter = [&](const Path & p) -> bool {
+                assert(hasPrefix(p, uri));
+                std::string file(p, uri.size() + 1);
+
+                auto st = lstat(p);
+
+                if (S_ISDIR(st.st_mode)) {
+                    auto prefix = file + "/";
+                    auto i = files.lower_bound(prefix);
+                    return i != files.end() && hasPrefix(*i, prefix);
+                }
+
+                return files.count(file);
+            };
+
+            hgInfo.storePath = store->printStorePath(store->addToStore("source", uri, true, htSHA256, filter));
+
+            return hgInfo;
+        }
+    }
+
+    if (rev == "") rev = "default";
+
+    Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), hashString(htSHA256, uri).to_string(Base32, false));
+
+    Path stampFile = fmt("%s/.hg/%s.stamp", cacheDir, hashString(htSHA512, rev).to_string(Base32, false));
+
+    /* If we haven't pulled this repo less than ‘tarball-ttl’ seconds,
+       do so now. */
+    time_t now = time(0);
+    struct stat st;
+    if (stat(stampFile.c_str(), &st) != 0 ||
+        (uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now)
+    {
+        /* Except that if this is a commit hash that we already have,
+           we don't have to pull again. */
+        if (!(std::regex_match(rev, commitHashRegex)
+                && pathExists(cacheDir)
+                && runProgram(
+                    RunOptions("hg", { "log", "-R", cacheDir, "-r", rev, "--template", "1" })
+                    .killStderr(true)).second == "1"))
+        {
+            Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Mercurial repository '%s'", uri));
+
+            if (pathExists(cacheDir)) {
+                try {
+                    runProgram("hg", true, { "pull", "-R", cacheDir, "--", uri });
+                }
+                catch (ExecError & e) {
+                    string transJournal = cacheDir + "/.hg/store/journal";
+                    /* hg throws "abandoned transaction" error only if this file exists */
+                    if (pathExists(transJournal)) {
+                        runProgram("hg", true, { "recover", "-R", cacheDir });
+                        runProgram("hg", true, { "pull", "-R", cacheDir, "--", uri });
+                    } else {
+                        throw ExecError(e.status, fmt("'hg pull' %s", statusToString(e.status)));
+                    }
+                }
+            } else {
+                createDirs(dirOf(cacheDir));
+                runProgram("hg", true, { "clone", "--noupdate", "--", uri, cacheDir });
+            }
+        }
+
+        writeFile(stampFile, "");
+    }
+
+    auto tokens = tokenizeString<std::vector<std::string>>(
+        runProgram("hg", true, { "log", "-R", cacheDir, "-r", rev, "--template", "{node} {rev} {branch}" }));
+    assert(tokens.size() == 3);
+
+    HgInfo hgInfo;
+    hgInfo.rev = tokens[0];
+    hgInfo.revCount = std::stoull(tokens[1]);
+    hgInfo.branch = tokens[2];
+
+    std::string storeLinkName = hashString(htSHA512, name + std::string("\0"s) + hgInfo.rev).to_string(Base32, false);
+    Path storeLink = fmt("%s/.hg/%s.link", cacheDir, storeLinkName);
+
+    try {
+        auto json = nlohmann::json::parse(readFile(storeLink));
+
+        assert(json["name"] == name && json["rev"] == hgInfo.rev);
+
+        hgInfo.storePath = json["storePath"];
+
+        if (store->isValidPath(store->parseStorePath(hgInfo.storePath))) {
+            printTalkative("using cached Mercurial store path '%s'", hgInfo.storePath);
+            return hgInfo;
+        }
+
+    } catch (SysError & e) {
+        if (e.errNo != ENOENT) throw;
+    }
+
+    Path tmpDir = createTempDir();
+    AutoDelete delTmpDir(tmpDir, true);
+
+    runProgram("hg", true, { "archive", "-R", cacheDir, "-r", rev, tmpDir });
+
+    deletePath(tmpDir + "/.hg_archival.txt");
+
+    hgInfo.storePath = store->printStorePath(store->addToStore(name, tmpDir));
+
+    nlohmann::json json;
+    json["storePath"] = hgInfo.storePath;
+    json["uri"] = uri;
+    json["name"] = name;
+    json["branch"] = hgInfo.branch;
+    json["rev"] = hgInfo.rev;
+    json["revCount"] = hgInfo.revCount;
+
+    writeFile(storeLink, json.dump());
+
+    return hgInfo;
+}
+
 static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     std::string url;
-    std::optional<Hash> rev;
-    std::optional<std::string> ref;
+    std::string rev;
     std::string name = "source";
     PathSet context;
 
@@ -26,15 +182,8 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar
             string n(attr.name);
             if (n == "url")
                 url = state.coerceToString(*attr.pos, *attr.value, context, false, false);
-            else if (n == "rev") {
-                // Ugly: unlike fetchGit, here the "rev" attribute can
-                // be both a revision or a branch/tag name.
-                auto value = state.forceStringNoCtx(*attr.value, *attr.pos);
-                if (std::regex_match(value, revRegex))
-                    rev = Hash(value, htSHA1);
-                else
-                    ref = value;
-            }
+            else if (n == "rev")
+                rev = state.forceStringNoCtx(*attr.value, *attr.pos);
             else if (n == "name")
                 name = state.forceStringNoCtx(*attr.value, *attr.pos);
             else
@@ -51,35 +200,18 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar
     // whitelist. Ah well.
     state.checkURI(url);
 
-    if (evalSettings.pureEval && !rev)
-        throw Error("in pure evaluation mode, 'fetchMercurial' requires a Mercurial revision");
-
-    fetchers::Attrs attrs;
-    attrs.insert_or_assign("type", "hg");
-    attrs.insert_or_assign("url", url.find("://") != std::string::npos ? url : "file://" + url);
-    if (ref) attrs.insert_or_assign("ref", *ref);
-    if (rev) attrs.insert_or_assign("rev", rev->gitRev());
-    auto input = fetchers::inputFromAttrs(attrs);
-
-    // FIXME: use name
-    auto [tree, input2] = input->fetchTree(state.store);
+    auto hgInfo = exportMercurial(state.store, url, rev, name);
 
     state.mkAttrs(v, 8);
-    auto storePath = state.store->printStorePath(tree.storePath);
-    mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath}));
-    if (input2->getRef())
-        mkString(*state.allocAttr(v, state.symbols.create("branch")), *input2->getRef());
-    // Backward compatibility: set 'rev' to
-    // 0000000000000000000000000000000000000000 for a dirty tree.
-    auto rev2 = input2->getRev().value_or(Hash(htSHA1));
-    mkString(*state.allocAttr(v, state.symbols.create("rev")), rev2.gitRev());
-    mkString(*state.allocAttr(v, state.symbols.create("shortRev")), std::string(rev2.gitRev(), 0, 12));
-    if (tree.info.revCount)
-        mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *tree.info.revCount);
+    mkString(*state.allocAttr(v, state.sOutPath), hgInfo.storePath, PathSet({hgInfo.storePath}));
+    mkString(*state.allocAttr(v, state.symbols.create("branch")), hgInfo.branch);
+    mkString(*state.allocAttr(v, state.symbols.create("rev")), hgInfo.rev);
+    mkString(*state.allocAttr(v, state.symbols.create("shortRev")), std::string(hgInfo.rev, 0, 12));
+    mkInt(*state.allocAttr(v, state.symbols.create("revCount")), hgInfo.revCount);
     v.attrs->sort();
 
     if (state.allowedPaths)
-        state.allowedPaths->insert(tree.actualPath);
+        state.allowedPaths->insert(state.store->toRealPath(hgInfo.storePath));
 }
 
 static RegisterPrimOp r("fetchMercurial", 1, prim_fetchMercurial);

src/libexpr/primops/fetchTree.cc

@@ -1,173 +0,0 @@
-#include "primops.hh"
-#include "eval-inline.hh"
-#include "store-api.hh"
-#include "fetchers.hh"
-#include "filetransfer.hh"
-#include "registry.hh"
-
-#include <ctime>
-#include <iomanip>
-
-namespace nix {
-
-void emitTreeAttrs(
-    EvalState & state,
-    const fetchers::Tree & tree,
-    std::shared_ptr<const fetchers::Input> input,
-    Value & v)
-{
-    state.mkAttrs(v, 8);
-
-    auto storePath = state.store->printStorePath(tree.storePath);
-
-    mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath}));
-
-    assert(tree.info.narHash);
-    mkString(*state.allocAttr(v, state.symbols.create("narHash")),
-        tree.info.narHash.to_string(SRI));
-
-    if (input->getRev()) {
-        mkString(*state.allocAttr(v, state.symbols.create("rev")), input->getRev()->gitRev());
-        mkString(*state.allocAttr(v, state.symbols.create("shortRev")), input->getRev()->gitShortRev());
-    }
-
-    if (tree.info.revCount)
-        mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *tree.info.revCount);
-
-    if (tree.info.lastModified) {
-        mkInt(*state.allocAttr(v, state.symbols.create("lastModified")), *tree.info.lastModified);
-        mkString(*state.allocAttr(v, state.symbols.create("lastModifiedDate")),
-            fmt("%s", std::put_time(std::gmtime(&*tree.info.lastModified), "%Y%m%d%H%M%S")));
-    }
-
-    v.attrs->sort();
-}
-
-static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, Value & v)
-{
-    settings.requireExperimentalFeature("flakes");
-
-    std::shared_ptr<const fetchers::Input> input;
-    PathSet context;
-
-    state.forceValue(*args[0]);
-
-    if (args[0]->type == tAttrs) {
-        state.forceAttrs(*args[0], pos);
-
-        fetchers::Attrs attrs;
-
-        for (auto & attr : *args[0]->attrs) {
-            state.forceValue(*attr.value);
-            if (attr.value->type == tString)
-                attrs.emplace(attr.name, attr.value->string.s);
-            else if (attr.value->type == tBool)
-                attrs.emplace(attr.name, attr.value->boolean);
-            else if (attr.value->type == tInt)
-                attrs.emplace(attr.name, attr.value->integer);
-            else
-                throw TypeError("fetchTree argument '%s' is %s while a string, Boolean or integer is expected",
-                    attr.name, showType(*attr.value));
-        }
-
-        if (!attrs.count("type"))
-            throw Error("attribute 'type' is missing in call to 'fetchTree', at %s", pos);
-
-        input = fetchers::inputFromAttrs(attrs);
-    } else
-        input = fetchers::inputFromURL(state.coerceToString(pos, *args[0], context, false, false));
-
-    if (!evalSettings.pureEval && !input->isDirect())
-        input = lookupInRegistries(state.store, input).first;
-
-    if (evalSettings.pureEval && !input->isImmutable())
-        throw Error("in pure evaluation mode, 'fetchTree' requires an immutable input, at %s", pos);
-
-    // FIXME: use fetchOrSubstituteTree
-    auto [tree, input2] = input->fetchTree(state.store);
-
-    if (state.allowedPaths)
-        state.allowedPaths->insert(tree.actualPath);
-
-    emitTreeAttrs(state, tree, input2, v);
-}
-
-static RegisterPrimOp r("fetchTree", 1, prim_fetchTree);
-
-static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
-    const string & who, bool unpack, std::string name)
-{
-    std::optional<std::string> url;
-    std::optional<Hash> expectedHash;
-
-    state.forceValue(*args[0]);
-
-    if (args[0]->type == tAttrs) {
-
-        state.forceAttrs(*args[0], pos);
-
-        for (auto & attr : *args[0]->attrs) {
-            string n(attr.name);
-            if (n == "url")
-                url = state.forceStringNoCtx(*attr.value, *attr.pos);
-            else if (n == "sha256")
-                expectedHash = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256);
-            else if (n == "name")
-                name = state.forceStringNoCtx(*attr.value, *attr.pos);
-            else
-                throw EvalError("unsupported argument '%s' to '%s', at %s",
-                    attr.name, who, *attr.pos);
-        }
-
-        if (!url)
-            throw EvalError("'url' argument required, at %s", pos);
-
-    } else
-        url = state.forceStringNoCtx(*args[0], pos);
-
-    url = resolveUri(*url);
-
-    state.checkURI(*url);
-
-    if (name == "")
-        name = baseNameOf(*url);
-
-    if (evalSettings.pureEval && !expectedHash)
-        throw Error("in pure evaluation mode, '%s' requires a 'sha256' argument", who);
-
-    auto storePath =
-        unpack
-        ? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).storePath
-        : fetchers::downloadFile(state.store, *url, name, (bool) expectedHash).storePath;
-
-    auto path = state.store->toRealPath(storePath);
-
-    if (expectedHash) {
-        auto hash = unpack
-            ? state.store->queryPathInfo(storePath)->narHash
-            : hashFile(htSHA256, path);
-        if (hash != *expectedHash)
-            throw Error((unsigned int) 102, "hash mismatch in file downloaded from '%s':\n  wanted: %s\n  got:    %s",
-                *url, expectedHash->to_string(), hash.to_string());
-    }
-
-    if (state.allowedPaths)
-        state.allowedPaths->insert(path);
-
-    mkString(v, path, PathSet({path}));
-}
-
-static void prim_fetchurl(EvalState & state, const Pos & pos, Value * * args, Value & v)
-{
-    fetch(state, pos, args, v, "fetchurl", false, "");
-}
-
-static void prim_fetchTarball(EvalState & state, const Pos & pos, Value * * args, Value & v)
-{
-    fetch(state, pos, args, v, "fetchTarball", true, "source");
-}
-
-static RegisterPrimOp r2("__fetchurl", 1, prim_fetchurl);
-static RegisterPrimOp r3("fetchTarball", 1, prim_fetchTarball);
-
-}

src/libexpr/primops/fromTOML.cc

@@ -1,7 +1,7 @@
 #include "primops.hh"
 #include "eval-inline.hh"
 
-#include "../../cpptoml/cpptoml.h"
+#include "cpptoml/cpptoml.h"
 
 namespace nix {
 

src/libexpr/symbol-table.hh

@@ -19,6 +19,7 @@ private:
     const string * s; // pointer into SymbolTable
     Symbol(const string * s) : s(s) { };
     friend class SymbolTable;
+    friend class Value;
 
 public:
     Symbol() : s(0) { };
@@ -28,12 +29,6 @@ public:
         return s == s2.s;
     }
 
-    // FIXME: remove
-    bool operator == (std::string_view s2) const
-    {
-        return s->compare(s2) == 0;
-    }
-
     bool operator != (const Symbol & s2) const
     {
         return s != s2.s;
@@ -74,10 +69,9 @@ private:
     Symbols symbols;
 
 public:
-    Symbol create(std::string_view s)
+    Symbol create(const string & s)
     {
-        // FIXME: avoid allocation if 's' already exists in the symbol table.
-        std::pair<Symbols::iterator, bool> res = symbols.emplace(std::string(s));
+        std::pair<Symbols::iterator, bool> res = symbols.insert(s);
         return Symbol(&*res.first);
     }
 
@@ -96,4 +90,6 @@ public:
     }
 };
 
+extern SymbolTable symbols;
+
 }

src/libexpr/value-to-json.cc

@@ -26,13 +26,15 @@ void printValueAsJSON(EvalState & state, bool strict,
             out.write(v.boolean);
             break;
 
-        case tString:
-            copyContext(v, context);
-            out.write(v.string.s);
+        case tShortString:
+        case tStaticString:
+        case tLongString:
+            v.getContext(context);
+            out.write(v.getString());
             break;
 
         case tPath:
-            out.write(state.copyPathToStore(context, v.path));
+            out.write(state.copyPathToStore(context, v.path->s));
             break;
 
         case tNull:
@@ -61,7 +63,7 @@ void printValueAsJSON(EvalState & state, bool strict,
             break;
         }
 
-        case tList1: case tList2: case tListN: {
+        case tList0: case tList1: case tList2: case tListN: {
             auto list(out.list());
             for (unsigned int n = 0; n < v.listSize(); ++n) {
                 auto placeholder(list.placeholder());
@@ -70,9 +72,11 @@ void printValueAsJSON(EvalState & state, bool strict,
             break;
         }
 
+#if 0
         case tExternal:
             v.external->printValueAsJSON(state, strict, out, context);
             break;
+#endif
 
         case tFloat:
             out.write(v.fpoint);
@@ -90,11 +94,13 @@ void printValueAsJSON(EvalState & state, bool strict,
     printValueAsJSON(state, strict, v, out, context);
 }
 
+#if 0
 void ExternalValueBase::printValueAsJSON(EvalState & state, bool strict,
     JSONPlaceholder & out, PathSet & context) const
 {
     throw TypeError(format("cannot convert %1% to JSON") % showType());
 }
+#endif
 
 
 }

src/libexpr/value-to-xml.cc

@@ -68,14 +68,16 @@ static void printValueAsXML(EvalState & state, bool strict, bool location,
             doc.writeEmptyElement("bool", singletonAttrs("value", v.boolean ? "true" : "false"));
             break;
 
-        case tString:
+        case tShortString:
+        case tStaticString:
+        case tLongString:
             /* !!! show the context? */
-            copyContext(v, context);
-            doc.writeEmptyElement("string", singletonAttrs("value", v.string.s));
+            v.getContext(context);
+            doc.writeEmptyElement("string", singletonAttrs("value", v.getString()));
             break;
 
         case tPath:
-            doc.writeEmptyElement("path", singletonAttrs("value", v.path));
+            doc.writeEmptyElement("path", singletonAttrs("value", v.path->s));
             break;
 
         case tNull:
@@ -92,15 +94,15 @@ static void printValueAsXML(EvalState & state, bool strict, bool location,
                 a = v.attrs->find(state.sDrvPath);
                 if (a != v.attrs->end()) {
                     if (strict) state.forceValue(*a->value);
-                    if (a->value->type == tString)
-                        xmlAttrs["drvPath"] = drvPath = a->value->string.s;
+                    if (a->value->isString())
+                        xmlAttrs["drvPath"] = drvPath = a->value->getString();
                 }
 
                 a = v.attrs->find(state.sOutPath);
                 if (a != v.attrs->end()) {
                     if (strict) state.forceValue(*a->value);
-                    if (a->value->type == tString)
-                        xmlAttrs["outPath"] = a->value->string.s;
+                    if (a->value->isString())
+                        xmlAttrs["outPath"] = a->value->getString();
                 }
 
                 XMLOpenElement _(doc, "derivation", xmlAttrs);
@@ -118,7 +120,7 @@ static void printValueAsXML(EvalState & state, bool strict, bool location,
 
             break;
 
-        case tList1: case tList2: case tListN: {
+        case tList0: case tList1: case tList2: case tListN: {
             XMLOpenElement _(doc, "list");
             for (unsigned int n = 0; n < v.listSize(); ++n)
                 printValueAsXML(state, strict, location, *v.listElems()[n], doc, context, drvsSeen);
@@ -143,9 +145,11 @@ static void printValueAsXML(EvalState & state, bool strict, bool location,
             break;
         }
 
+#if 0
         case tExternal:
             v.external->printValueAsXML(state, strict, location, doc, context, drvsSeen);
             break;
+#endif
 
         case tFloat:
             doc.writeEmptyElement("float", singletonAttrs("value", (format("%1%") % v.fpoint).str()));
@@ -157,11 +161,13 @@ static void printValueAsXML(EvalState & state, bool strict, bool location,
 }
 
 
+#if 0
 void ExternalValueBase::printValueAsXML(EvalState & state, bool strict,
     bool location, XMLWriter & doc, PathSet & context, PathSet & drvsSeen) const
 {
     doc.writeEmptyElement("unevaluated");
 }
+#endif
 
 
 void printValueAsXML(EvalState & state, bool strict, bool location,

src/libexpr/value.hh

@@ -1,35 +1,12 @@
 #pragma once
 
 #include "symbol-table.hh"
+#include "gc.hh"
 
-#if HAVE_BOEHMGC
-#include <gc/gc_allocator.h>
-#endif
+#include <cstring>
 
 namespace nix {
 
-
-typedef enum {
-    tInt = 1,
-    tBool,
-    tString,
-    tPath,
-    tNull,
-    tAttrs,
-    tList1,
-    tList2,
-    tListN,
-    tThunk,
-    tApp,
-    tLambda,
-    tBlackhole,
-    tPrimOp,
-    tPrimOpApp,
-    tExternal,
-    tFloat
-} ValueType;
-
-
 class Bindings;
 struct Env;
 struct Expr;
@@ -45,13 +22,15 @@ class JSONPlaceholder;
 typedef int64_t NixInt;
 typedef double NixFloat;
 
+#if 0
 /* External values must descend from ExternalValueBase, so that
  * type-agnostic nix functions (e.g. showType) can be implemented
  */
 class ExternalValueBase
 {
     friend std::ostream & operator << (std::ostream & str, const ExternalValueBase & v);
-    protected:
+
+protected:
     /* Print out the value */
     virtual std::ostream & print(std::ostream & str) const = 0;
 
@@ -86,11 +65,29 @@ class ExternalValueBase
 };
 
 std::ostream & operator << (std::ostream & str, const ExternalValueBase & v);
+#endif
 
 
-struct Value
+class Context : Object
+{
+    friend class Value;
+    friend class GC;
+
+    Symbol members[0];
+
+    Context(const PathSet & context) : Object(tContext, context.size())
+    {
+        size_t n = 0;
+        for (auto & i : context)
+            members[n++] = symbols.create(i);
+    }
+
+    size_t getSize() { return getMisc(); }
+};
+
+
+struct Value : Object
 {
-    ValueType type;
     union
     {
         NixInt integer;
@@ -113,20 +110,15 @@ struct Value
            with context C is used as a derivation attribute, then the
            derivations in C will be added to the inputDrvs of the
            derivation, and the other store paths in C will be added to
-           the inputSrcs of the derivations.
-
-           For canonicity, the store paths should be in sorted order. */
+           the inputSrcs of the derivations. */
         struct {
-            const char * s;
-            const char * * context; // must be in sorted order
+            String * s;
+            Context * context;
         } string;
-
-        const char * path;
+        const char * staticString;
+        String * path;
         Bindings * attrs;
-        struct {
-            size_t size;
-            Value * * elems;
-        } bigList;
+        PtrList<Value> * bigList;
         Value * smallList[2];
         struct {
             Env * env;
@@ -140,51 +132,99 @@ struct Value
             ExprLambda * fun;
         } lambda;
         PrimOp * primOp;
-        struct {
-            Value * left, * right;
-        } primOpApp;
-        ExternalValueBase * external;
+        //ExternalValueBase * external;
         NixFloat fpoint;
     };
 
+private:
+
+    Value() : Object(tNull, 0) {}
+
+    friend class GC;
+    template<typename T> friend class Root;
+
+public:
+
     bool isList() const
     {
-        return type == tList1 || type == tList2 || type == tListN;
+        return type >= tList0 && type <= tListN;
     }
 
     Value * * listElems()
     {
-        return type == tList1 || type == tList2 ? smallList : bigList.elems;
+        return type == tList0 || type == tList1 || type == tList2 ? smallList : bigList->elems;
     }
 
     const Value * const * listElems() const
     {
-        return type == tList1 || type == tList2 ? smallList : bigList.elems;
+        return type == tList0 || type == tList1 || type == tList2 ? smallList : bigList->elems;
     }
 
     size_t listSize() const
     {
-        return type == tList1 ? 1 : type == tList2 ? 2 : bigList.size;
+        return type == tList0 ? 0 : type == tList1 ? 1 : type == tList2 ? 2 : bigList->size();
     }
 
-    /* Check whether forcing this value requires a trivial amount of
-       computation. In particular, function applications are
-       non-trivial. */
-    bool isTrivial() const;
+    constexpr static size_t words() { return 3; } // FIXME
+
+    void setContext(const PathSet & context)
+    {
+        if (context.size() == 0)
+            string.context = nullptr;
+        else if (context.size() == 1) {
+            // If we have a single context, then store it
+            // directly. This saves allocating a Context object (16
+            // bytes).
+            auto symbol = symbols.create(*context.begin());
+            string.context = (Context *) (((ptrdiff_t) symbol.s) | 1);
+        } else {
+            string.context = gc.alloc<Context>(1 + context.size(), context);
+        }
+    }
+
+    void getContext(PathSet & context)
+    {
+        if (type == tLongString && string.context) {
+            if (((ptrdiff_t) string.context) & 1) {
+                auto s = (const std::string *) (((ptrdiff_t) string.context) & ~1UL);
+                context.insert(*s);
+            } else {
+                auto size = string.context->getSize();
+                for (size_t i = 0; i < size; ++i)
+                    context.insert(string.context->members[i]);
+            }
+        }
+    }
+
+    bool isString() const
+    {
+        return type == tShortString || type == tStaticString || type == tLongString;
+    }
+
+    void setShortString(std::string_view s)
+    {
+        // FIXME: can't use strcpy here because gcc flags it as a
+        // buffer overflow on 'misc'.
+        auto p = getMiscData();
+        memcpy(p, s.data(), s.size());
+        p[s.size()] = 0;
+        type = tShortString;
+    }
+
+    const char * getString() const
+    {
+        if (type == tShortString)
+            return getMiscData();
+        else if (type == tStaticString)
+            return staticString;
+        else
+            return string.s->s;
+    }
 };
 
 
-/* After overwriting an app node, be sure to clear pointers in the
-   Value to ensure that the target isn't kept alive unnecessarily. */
-static inline void clearValue(Value & v)
-{
-    v.app.left = v.app.right = 0;
-}
-
-
 static inline void mkInt(Value & v, NixInt n)
 {
-    clearValue(v);
     v.type = tInt;
     v.integer = n;
 }
@@ -192,7 +232,6 @@ static inline void mkInt(Value & v, NixInt n)
 
 static inline void mkFloat(Value & v, NixFloat n)
 {
-    clearValue(v);
     v.type = tFloat;
     v.fpoint = n;
 }
@@ -200,7 +239,6 @@ static inline void mkFloat(Value & v, NixFloat n)
 
 static inline void mkBool(Value & v, bool b)
 {
-    clearValue(v);
     v.type = tBool;
     v.boolean = b;
 }
@@ -208,7 +246,6 @@ static inline void mkBool(Value & v, bool b)
 
 static inline void mkNull(Value & v)
 {
-    clearValue(v);
     v.type = tNull;
 }
 
@@ -229,46 +266,35 @@ static inline void mkPrimOpApp(Value & v, Value & left, Value & right)
 }
 
 
-static inline void mkStringNoCopy(Value & v, const char * s)
+static inline Value & mkString(Value & v, std::string_view s)
 {
-    v.type = tString;
-    v.string.s = s;
-    v.string.context = 0;
+    if (s.size() < WORD_SIZE * 2 + Object::miscBytes)
+        v.setShortString(s);
+    else {
+        v.string.s = gc.alloc<String>(String::wordsFor(s.size()), s.size(), s.data());
+        v.string.context = 0;
+        v.type = tLongString;
+    }
+    return v;
 }
 
 
 static inline void mkString(Value & v, const Symbol & s)
 {
-    mkStringNoCopy(v, ((const string &) s).c_str());
+    v.staticString = ((const string &) s).c_str();
+    v.type = tStaticString;
 }
 
 
-void mkString(Value & v, const char * s);
-
-
-static inline void mkPathNoCopy(Value & v, const char * s)
+static inline void mkPath(Value & v, const std::string & s)
 {
-    clearValue(v);
+    v.path = String::alloc(s.c_str());
     v.type = tPath;
-    v.path = s;
 }
 
 
-void mkPath(Value & v, const char * s);
+typedef std::vector<Ptr<Value>> ValueVector; // FIXME: make more efficient
+typedef std::map<Symbol, Ptr<Value>> ValueMap; // FIXME: use Bindings?
 
 
-#if HAVE_BOEHMGC
-typedef std::vector<Value *, traceable_allocator<Value *> > ValueVector;
-typedef std::map<Symbol, Value *, std::less<Symbol>, traceable_allocator<std::pair<const Symbol, Value *> > > ValueMap;
-#else
-typedef std::vector<Value *> ValueVector;
-typedef std::map<Symbol, Value *> ValueMap;
-#endif
-
-
-/* A value allocated in traceable memory. */
-typedef std::shared_ptr<Value *> RootValue;
-
-RootValue allocRootValue(Value * v);
-
 }

src/libfetchers/attrs.cc

@@ -1,107 +0,0 @@
-#include "attrs.hh"
-#include "fetchers.hh"
-
-#include <nlohmann/json.hpp>
-
-namespace nix::fetchers {
-
-Attrs jsonToAttrs(const nlohmann::json & json)
-{
-    Attrs attrs;
-
-    for (auto & i : json.items()) {
-        if (i.value().is_number())
-            attrs.emplace(i.key(), i.value().get<int64_t>());
-        else if (i.value().is_string())
-            attrs.emplace(i.key(), i.value().get<std::string>());
-        else if (i.value().is_boolean())
-            attrs.emplace(i.key(), i.value().get<bool>());
-        else
-            throw Error("unsupported input attribute type in lock file");
-    }
-
-    return attrs;
-}
-
-nlohmann::json attrsToJson(const Attrs & attrs)
-{
-    nlohmann::json json;
-    for (auto & attr : attrs) {
-        if (auto v = std::get_if<int64_t>(&attr.second)) {
-            json[attr.first] = *v;
-        } else if (auto v = std::get_if<std::string>(&attr.second)) {
-            json[attr.first] = *v;
-        } else if (auto v = std::get_if<Explicit<bool>>(&attr.second)) {
-            json[attr.first] = v->t;
-        } else abort();
-    }
-    return json;
-}
-
-std::optional<std::string> maybeGetStrAttr(const Attrs & attrs, const std::string & name)
-{
-    auto i = attrs.find(name);
-    if (i == attrs.end()) return {};
-    if (auto v = std::get_if<std::string>(&i->second))
-        return *v;
-    throw Error("input attribute '%s' is not a string %s", name, attrsToJson(attrs).dump());
-}
-
-std::string getStrAttr(const Attrs & attrs, const std::string & name)
-{
-    auto s = maybeGetStrAttr(attrs, name);
-    if (!s)
-        throw Error("input attribute '%s' is missing", name);
-    return *s;
-}
-
-std::optional<int64_t> maybeGetIntAttr(const Attrs & attrs, const std::string & name)
-{
-    auto i = attrs.find(name);
-    if (i == attrs.end()) return {};
-    if (auto v = std::get_if<int64_t>(&i->second))
-        return *v;
-    throw Error("input attribute '%s' is not an integer", name);
-}
-
-int64_t getIntAttr(const Attrs & attrs, const std::string & name)
-{
-    auto s = maybeGetIntAttr(attrs, name);
-    if (!s)
-        throw Error("input attribute '%s' is missing", name);
-    return *s;
-}
-
-std::optional<bool> maybeGetBoolAttr(const Attrs & attrs, const std::string & name)
-{
-    auto i = attrs.find(name);
-    if (i == attrs.end()) return {};
-    if (auto v = std::get_if<int64_t>(&i->second))
-        return *v;
-    throw Error("input attribute '%s' is not a Boolean", name);
-}
-
-bool getBoolAttr(const Attrs & attrs, const std::string & name)
-{
-    auto s = maybeGetBoolAttr(attrs, name);
-    if (!s)
-        throw Error("input attribute '%s' is missing", name);
-    return *s;
-}
-
-std::map<std::string, std::string> attrsToQuery(const Attrs & attrs)
-{
-    std::map<std::string, std::string> query;
-    for (auto & attr : attrs) {
-        if (auto v = std::get_if<int64_t>(&attr.second)) {
-            query.insert_or_assign(attr.first, fmt("%d", *v));
-        } else if (auto v = std::get_if<std::string>(&attr.second)) {
-            query.insert_or_assign(attr.first, *v);
-        } else if (auto v = std::get_if<Explicit<bool>>(&attr.second)) {
-            query.insert_or_assign(attr.first, v->t ? "1" : "0");
-        } else abort();
-    }
-    return query;
-}
-
-}