Fix InstallableCommand

macOS doesn't have GLOB_ONLYDIR.

.dir-locals.el

@@ -13,4 +13,5 @@
   (eval . (c-set-offset 'arglist-cont-nonempty '+))
   (eval . (c-set-offset 'substatement-open 0))
   (eval . (c-set-offset 'access-label '-))
+  (eval . (c-set-offset 'inlambda 0))
   )))

.github/ISSUE_TEMPLATE.md

@@ -1,27 +0,0 @@
-<!--
-
-# Filing a Nix issue
-
-*WAIT* Are you sure you're filing your issue in the right repository?
-
-We appreciate you taking the time to tell us about issues you encounter, but routing the issue to the right place will get you help sooner and save everyone time.
-
-This is the Nix repository, and issues here should be about Nix the build and package management *_tool_*.
-
-If you have a problem with a specific package on NixOS or when using Nix, you probably want to file an issue with _nixpkgs_, whose issue tracker is over at https://github.com/NixOS/nixpkgs/issues.
-
-Examples of _Nix_ issues:
-
-- Nix segfaults when I run `nix-build -A blahblah`
-- The Nix language needs a new builtin: `builtins.foobar`
-- Regression in the behavior of `nix-env` in Nix 2.0
-
-Examples of _nixpkgs_ issues:
-
-- glibc is b0rked on aarch64
-- chromium in NixOS doesn't support U2F but google-chrome does!
-- The OpenJDK package on macOS is missing a key component
-
-Chances are if you're a newcomer to the Nix world, you'll probably want the [nixpkgs tracker](https://github.com/NixOS/nixpkgs/issues). It also gets a lot more eyeball traffic so you'll probably get a response a lot more quickly.
-
--->

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,32 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: bug
+assignees: ''
+
+---
+
+**Describe the bug**
+
+A clear and concise description of what the bug is.
+
+If you have a problem with a specific package or NixOS,
+you probably want to file an issue at https://github.com/NixOS/nixpkgs/issues. 
+
+**Steps To Reproduce**
+
+1. Go to '...'
+2. Click on '....'
+3. Scroll down to '....'
+4. See error
+
+**Expected behavior**
+
+A clear and concise description of what you expected to happen.
+
+**`nix-env --version` output**
+
+**Additional context**
+
+Add any other context about the problem here.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,20 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: improvement
+assignees: ''
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

.github/workflows/test.yml

@@ -0,0 +1,17 @@
+name: "Test"
+on:
+  pull_request:
+  push:
+jobs:
+  tests:
+    strategy:
+      matrix:
+        os: [ubuntu-18.04, macos]
+    runs-on: ${{ matrix.os }}
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
+    - uses: cachix/install-nix-action@v8
+    #- run: nix flake check
+    - run: nix-build -A checks.$(if [[ `uname` = Linux ]]; then echo x86_64-linux; else echo x86_64-darwin; fi)

.gitignore

@@ -47,7 +47,7 @@ perl/Makefile.config
 /src/libexpr/nix.tbl
 
 # /src/libstore/
-/src/libstore/*.gen.hh
+*.gen.*
 
 /src/nix/nix
 
@@ -75,6 +75,8 @@ perl/Makefile.config
 
 /src/nix-copy-closure/nix-copy-closure
 
+/src/error-demo/error-demo
+
 /src/build-remote/build-remote
 
 # /tests/

.travis.yml

@@ -1,8 +0,0 @@
-matrix:
-  include:
-    - language: osx
-      script: ./tests/install-darwin.sh
-    - language: nix
-      script: nix-build release.nix -A build.x86_64-linux
-notifications:
-  email: false

Makefile

@@ -4,12 +4,14 @@ makefiles = \
   nix-rust/local.mk \
   src/libutil/local.mk \
   src/libstore/local.mk \
+  src/libfetchers/local.mk \
   src/libmain/local.mk \
   src/libexpr/local.mk \
   src/nix/local.mk \
   src/resolve-system-dependencies/local.mk \
   scripts/local.mk \
   corepkgs/local.mk \
+  misc/bash/local.mk \
   misc/systemd/local.mk \
   misc/launchd/local.mk \
   misc/upstart/local.mk \

Makefile.config.in

@@ -37,6 +37,7 @@ prefix = @prefix@
 sandbox_shell = @sandbox_shell@
 storedir = @storedir@
 sysconfdir = @sysconfdir@
+system = @system@
 doc_generate = @doc_generate@
 xmllint = @xmllint@
 xsltproc = @xsltproc@

README.md

@@ -1,24 +1,54 @@
+# Nix
+
 [![Open Collective supporters](https://opencollective.com/nixos/tiers/supporter/badge.svg?label=Supporters&color=brightgreen)](https://opencollective.com/nixos)
+[![Test](https://github.com/NixOS/nix/workflows/Test/badge.svg)](https://github.com/NixOS/nix/actions)
 
-Nix, the purely functional package manager
-------------------------------------------
+Nix is a powerful package manager for Linux and other Unix systems that makes package
+management reliable and reproducible. Please refer to the [Nix manual](https://nixos.org/nix/manual)
+for more details.
 
-Nix is a new take on package management that is fairly unique. Because of its
-purity aspects, a lot of issues found in traditional package managers don't
-appear with Nix.
+## Installation
 
-To find out more about the tool, usage and installation instructions, please
-read the manual, which is available on the Nix website at
-<http://nixos.org/nix/manual>.
+On Linux and macOS the easiest way to Install Nix is to run the following shell command
+(as a user other than root):
 
-## Contributing
+```
+$ curl -L https://nixos.org/nix/install | sh
+```
 
-Take a look at the [Hacking Section](http://nixos.org/nix/manual/#chap-hacking)
-of the manual. It helps you to get started with building Nix from source.
+Information on additional installation methods is available on the [Nix download page](https://nixos.org/download.html).
+
+## Building And Developing
+
+### Building Nix
+
+You can build Nix using one of the targets provided by [release.nix](./release.nix):
+
+```
+$ nix-build ./release.nix -A build.aarch64-linux
+$ nix-build ./release.nix -A build.x86_64-darwin
+$ nix-build ./release.nix -A build.i686-linux
+$ nix-build ./release.nix -A build.x86_64-linux
+```
+
+### Development Environment
+
+You can use the provided `shell.nix` to get a working development environment:
+
+```
+$ nix-shell
+$ ./bootstrap.sh
+$ ./configure
+$ make
+```
+
+## Additional Resources
+
+- [Nix manual](https://nixos.org/nix/manual)
+- [Nix jobsets on hydra.nixos.org](https://hydra.nixos.org/project/nix)
+- [NixOS Discourse](https://discourse.nixos.org/)
+- [IRC - #nixos on freenode.net](irc://irc.freenode.net/#nixos)
 
 ## License
 
-Nix is released under the LGPL v2.1
-
-This product includes software developed by the OpenSSL Project for
-use in the [OpenSSL Toolkit](http://www.OpenSSL.org/).
+Nix is released under the [LGPL v2.1](./COPYING).

configure.ac

@@ -123,6 +123,7 @@ AC_PATH_PROG(flex, flex, false)
 AC_PATH_PROG(bison, bison, false)
 AC_PATH_PROG(dot, dot)
 AC_PATH_PROG(lsof, lsof, lsof)
+NEED_PROG(jq, jq)
 
 
 AC_SUBST(coreutils, [$(dirname $(type -p cat))])

corepkgs/local.mk

@@ -1,4 +1,7 @@
-corepkgs_FILES = buildenv.nix unpack-channel.nix derivation.nix fetchurl.nix imported-drv-to-derivation.nix
+corepkgs_FILES = \
+  unpack-channel.nix \
+  derivation.nix \
+  fetchurl.nix
 
 $(foreach file,config.nix $(corepkgs_FILES),$(eval $(call install-data-in,$(d)/$(file),$(datadir)/nix/corepkgs)))
 

default.nix

@@ -0,0 +1,3 @@
+(import (fetchTarball https://github.com/edolstra/flake-compat/archive/master.tar.gz) {
+  src = builtins.fetchGit ./.;
+}).defaultNix

doc/manual/command-ref/conf-file.xml

@@ -19,26 +19,30 @@
 
 <refsection><title>Description</title>
 
-<para>Nix reads settings from two configuration files:</para>
+<para>By default Nix reads settings from the following places:</para>
 
-<itemizedlist>
+<para>The system-wide configuration file
+<filename><replaceable>sysconfdir</replaceable>/nix/nix.conf</filename>
+(i.e. <filename>/etc/nix/nix.conf</filename> on most systems), or
+<filename>$NIX_CONF_DIR/nix.conf</filename> if
+<envar>NIX_CONF_DIR</envar> is set. Values loaded in this file are not forwarded to the Nix daemon. The
+client assumes that the daemon has already loaded them.
+</para>
 
-  <listitem>
-    <para>The system-wide configuration file
-    <filename><replaceable>sysconfdir</replaceable>/nix/nix.conf</filename>
-    (i.e. <filename>/etc/nix/nix.conf</filename> on most systems), or
-    <filename>$NIX_CONF_DIR/nix.conf</filename> if
-    <envar>NIX_CONF_DIR</envar> is set.</para>
-  </listitem>
+<para>User-specific configuration files:</para>
 
-  <listitem>
-    <para>The user configuration file
-    <filename>$XDG_CONFIG_HOME/nix/nix.conf</filename>, or
-    <filename>~/.config/nix/nix.conf</filename> if
-    <envar>XDG_CONFIG_HOME</envar> is not set.</para>
-  </listitem>
+<para>
+  If <envar>NIX_USER_CONF_FILES</envar> is set, then each path separated by
+  <literal>:</literal> will be loaded in reverse order.
+</para>
 
-</itemizedlist>
+<para>
+  Otherwise it will look for <filename>nix/nix.conf</filename> files in
+  <envar>XDG_CONFIG_DIRS</envar> and <envar>XDG_CONFIG_HOME</envar>.
+
+  The default location is <filename>$HOME/.config/nix.conf</filename> if
+  those environment variables are unset.
+</para>
 
 <para>The configuration files consist of
 <literal><replaceable>name</replaceable> =

doc/manual/command-ref/env-common.xml

@@ -33,7 +33,7 @@
 
     will cause Nix to look for paths relative to
     <filename>/home/eelco/Dev</filename> and
-    <filename>/etc/nixos</filename>, in that order.  It is also
+    <filename>/etc/nixos</filename>, in this order.  It is also
     possible to match paths against a prefix.  For example, the value
 
     <screen>
@@ -59,7 +59,7 @@ nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-15.09.tar.gz</sc
     15.09 channel.</para>
 
     <para>A following shorthand can be used to refer to the official channels:
-    
+
     <screen>nixpkgs=channel:nixos-15.09</screen>
     </para>
 
@@ -137,12 +137,19 @@ $ mount -o bind /mnt/otherdisk/nix /nix</screen>
 
 <varlistentry><term><envar>NIX_CONF_DIR</envar></term>
 
-  <listitem><para>Overrides the location of the Nix configuration
+  <listitem><para>Overrides the location of the system Nix configuration
   directory (default
   <filename><replaceable>prefix</replaceable>/etc/nix</filename>).</para></listitem>
 
 </varlistentry>
 
+<varlistentry><term><envar>NIX_USER_CONF_FILES</envar></term>
+
+  <listitem><para>Overrides the location of the user Nix configuration files
+  to load from (defaults to the XDG spec locations). The variable is treated
+  as a list separated by the <literal>:</literal> token.</para></listitem>
+
+</varlistentry>
 
 <varlistentry><term><envar>TMPDIR</envar></term>
 

doc/manual/command-ref/nix-env.xml

@@ -1066,7 +1066,8 @@ user environment elements, etc. -->
     the derivation, which can be used to unambiguously select it using
     the <link linkend="opt-attr"><option>--attr</option> option</link>
     available in commands that install derivations like
-    <literal>nix-env --install</literal>.</para></listitem>
+    <literal>nix-env --install</literal>. This option only works
+    together with <option>--available</option></para></listitem>
 
   </varlistentry>
 

doc/manual/command-ref/nix-store.xml

@@ -360,7 +360,6 @@ EOF
     <arg choice='plain'><option>--print-roots</option></arg>
     <arg choice='plain'><option>--print-live</option></arg>
     <arg choice='plain'><option>--print-dead</option></arg>
-    <arg choice='plain'><option>--delete</option></arg>
   </group>
   <arg><option>--max-freed</option> <replaceable>bytes</replaceable></arg>
 </cmdsynopsis>
@@ -407,14 +406,6 @@ the Nix store not reachable via file system references from a set of
 
   </varlistentry>
 
-  <varlistentry><term><option>--delete</option></term>
-
-    <listitem><para>This operation performs an actual garbage
-    collection.  All dead paths are removed from the
-    store.  This is the default.</para></listitem>
-
-  </varlistentry>
-
 </variablelist>
 
 <para>By default, all unreachable paths are deleted.  The following
@@ -444,10 +435,10 @@ and <link
 linkend="conf-keep-derivations"><literal>keep-derivations</literal></link>
 variables in the Nix configuration file.</para>
 
-<para>With <option>--delete</option>, the collector prints the total
-number of freed bytes when it finishes (or when it is interrupted).
-With <option>--print-dead</option>, it prints the number of bytes that
-would be freed.</para>
+<para>By default, the collector prints the total number of freed bytes
+when it finishes (or when it is interrupted). With
+<option>--print-dead</option>, it prints the number of bytes that would
+be freed.</para>
 
 </refsection>
 
@@ -1148,7 +1139,7 @@ the information that Nix considers important.  For instance,
 timestamps are elided because all files in the Nix store have their
 timestamp set to 0 anyway.  Likewise, all permissions are left out
 except for the execute bit, because all files in the Nix store have
-644 or 755 permission.</para>
+444 or 555 permission.</para>
 
 <para>Also, a NAR archive is <emphasis>canonical</emphasis>, meaning
 that “equal” paths always produce the same NAR archive.  For instance,

doc/manual/expressions/builtins.xml

@@ -422,6 +422,16 @@ stdenv.mkDerivation { … }
             </para>
           </listitem>
         </varlistentry>
+        <varlistentry>
+          <term>submodules</term>
+          <listitem>
+            <para>
+              A Boolean parameter that specifies whether submodules
+              should be checked out. Defaults to
+              <literal>false</literal>.
+            </para>
+          </listitem>
+        </varlistentry>
       </variablelist>
 
       <example>

doc/manual/installation/installing-binary.xml

@@ -142,7 +142,7 @@ sudo rm /Library/LaunchDaemons/org.nixos.nix-daemon.plist
   <para>
     NixOS.org hosts version-specific installation URLs for all Nix
     versions since 1.11.16, at
-    <literal>https://nixos.org/releases/nix/nix-VERSION/install</literal>.
+    <literal>https://releases.nixos.org/nix/nix-<replaceable>version</replaceable>/install</literal>.
   </para>
 
   <para>

doc/manual/installation/prerequisites-source.xml

@@ -8,6 +8,14 @@
 
 <itemizedlist>
 
+  <listitem><para>GNU Autoconf
+  (<link xlink:href="https://www.gnu.org/software/autoconf/"/>)
+  and the autoconf-archive macro collection
+  (<link xlink:href="https://www.gnu.org/software/autoconf-archive/"/>).
+  These are only needed to run the bootstrap script, and are not necessary
+  if your source distribution came with a pre-built
+  <literal>./configure</literal> script.</para></listitem>
+
   <listitem><para>GNU Make.</para></listitem>
   
   <listitem><para>Bash Shell. The <literal>./configure</literal> script

doc/manual/local.mk

@@ -4,11 +4,10 @@ ifeq ($(doc_generate),yes)
 XSLTPROC = $(xsltproc) --nonet $(xmlflags) \
   --param section.autolabel 1 \
   --param section.label.includes.component.label 1 \
-  --param html.stylesheet \'style.css\' \
   --param xref.with.number.and.title 1 \
   --param toc.section.depth 3 \
   --param admon.style \'\' \
-  --param callout.graphics.extension \'.gif\' \
+  --param callout.graphics 0 \
   --param contrib.inline.enabled 0 \
   --stringparam generate.toc "book toc" \
   --param keep.relative.image.uris 0
@@ -66,12 +65,10 @@ $(d)/manual.html: $(d)/manual.xml $(MANUAL_SRCS) $(d)/manual.is-valid
 	  $(docbookxsl)/profiling/profile.xsl $< | \
 	  $(XSLTPROC) --output $@ $(docbookxsl)/xhtml/docbook.xsl -
 
-$(foreach file, $(d)/manual.html $(d)/style.css, $(eval $(call install-data-in, $(file), $(docdir)/manual)))
+$(foreach file, $(d)/manual.html, $(eval $(call install-data-in, $(file), $(docdir)/manual)))
 
 $(foreach file, $(wildcard $(d)/figures/*.png), $(eval $(call install-data-in, $(file), $(docdir)/manual/figures)))
 
-$(foreach file, $(wildcard $(d)/images/callouts/*.gif), $(eval $(call install-data-in, $(file), $(docdir)/manual/images/callouts)))
-
 $(eval $(call install-symlink, manual.html, $(docdir)/manual/index.html))
 
 

doc/manual/release-notes/rl-2.0.xml

@@ -503,14 +503,14 @@
   </listitem>
 
   <listitem>
-    <para><emphasis>Pure evaluation mode</emphasis>. This is a variant
-    of the existing restricted evaluation mode. In pure mode, the Nix
-    evaluator forbids access to anything that could cause different
-    evaluations of the same command line arguments to produce a
+    <para><emphasis>Pure evaluation mode</emphasis>. With the
+    <literal>--pure-eval</literal> flag, Nix enables a variant of the existing
+    restricted evaluation mode that forbids access to anything that could cause
+    different evaluations of the same command line arguments to produce a
     different result. This includes builtin functions such as
     <function>builtins.getEnv</function>, but more importantly,
-    <emphasis>all</emphasis> filesystem or network access unless a
-    content hash or commit hash is specified. For example, calls to
+    <emphasis>all</emphasis> filesystem or network access unless a content hash
+    or commit hash is specified. For example, calls to
     <function>builtins.fetchGit</function> are only allowed if a
     <varname>rev</varname> attribute is specified.</para>
 

doc/manual/style.css

@@ -1,263 +0,0 @@
-/* Copied from http://bakefile.sourceforge.net/, which appears
-   licensed under the GNU GPL. */
-
-
-/***************************************************************************
-                             Basic headers and text:
- ***************************************************************************/
-
-body
-{
-    font-family: "Nimbus Sans L", sans-serif;
-    background: white;
-    margin: 2em 1em 2em 1em;
-}
-
-h1, h2, h3, h4
-{
-    color: #005aa0;
-}
-
-h1 /* title */
-{
-    font-size: 200%;
-}
-
-div.part h1
-{
-    font-size: 240%;
-}
-
-h2 /* chapters, appendices, subtitle */
-{
-    font-size: 180%;
-}
-
-div.part
-{
-    margin-top: 4em;
-}
-
-/* Extra space between chapters, appendices. */
-div.chapter > div.titlepage h2, div.appendix > div.titlepage h2 
-{ 
-    margin-top: 1.5em;
-}
-
-div.section > div.titlepage h2 /* sections */
-{
-    font-size: 150%;
-    margin-top: 1.5em;
-}
-
-h3 /* subsections */
-{
-    font-size: 125%;
-}
-
-div.simplesect h2
-{
-    font-size: 110%;
-}
-
-div.appendix h3
-{
-    font-size: 150%;
-    margin-top: 1.5em;
-}
-
-div.refentry\.separator
-{
-    margin-top: 2.5em;
-    margin-bottom: 2em;
-}
-
-div.refnamediv h2, div.refsynopsisdiv h2, div.refsection h2 /* refentry parts */
-{
-    margin-top: 1.4em;
-    font-size: 125%;
-}
-
-div.refsection h3
-{
-    font-size: 110%;
-}
-
-
-/***************************************************************************
-                               Examples:
- ***************************************************************************/
-
-div.example
-{
-    border: 1px solid #b0b0b0;
-    padding: 6px 6px;
-    margin-left: 1.5em;
-    margin-right: 1.5em;
-    background: #f4f4f8;
-    border-radius: 0.4em;
-}
-
-div.example p.title
-{
-    margin-top: 0em;
-}
-
-div.example pre
-{
-}
-
-
-/***************************************************************************
-                            Screen dumps:
- ***************************************************************************/
-
-pre.screen, pre.programlisting
-{
-    padding: 6px 6px;
-    margin-left: 1.5em;
-    margin-right: 1.5em;
-    color: #600000;
-    background: #f4f4f8;
-    font-family: monospace;
-}
-
-div.example pre.programlisting
-{
-    border: 0px;
-    padding: 0 0;
-    margin: 0 0 0 0;
-}
-
-
-/***************************************************************************
-                               Notes, warnings etc:
- ***************************************************************************/
-
-.note, .warning
-{
-    border: 1px solid #b0b0b0;
-    padding: 3px 3px;
-    margin-left: 1.5em;
-    margin-right: 1.5em;
-    margin-bottom: 1em;
-    padding: 0.3em 0.3em 0.3em 0.3em;
-    background: #fffff5;
-    border-radius: 0.4em;
-}
-
-div.note, div.warning
-{
-    font-style: italic;
-}
-
-div.note h3, div.warning h3
-{
-    color: red;
-    font-size: 100%;
-    padding-right: 0.5em;
-    display: inline;
-}
-
-div.note p, div.warning p
-{
-    margin-bottom: 0em;
-}
-
-div.note h3 + p, div.warning h3 + p
-{
-    display: inline;
-}
-
-div.note h3
-{
-    color: blue;
-    font-size: 100%;
-}
-
-div.navfooter *
-{
-    font-size: 90%;
-}
-
-
-/***************************************************************************
-                        Links colors and highlighting: 
- ***************************************************************************/
-
-a { text-decoration: none; }
-a:hover { text-decoration: underline; }
-a:link { color: #0048b3; }
-a:visited { color: #002a6a; }
-
-
-/***************************************************************************
-                              Table of contents:
- ***************************************************************************/
-
-div.toc
-{
-    font-size: 90%;
-}
-
-div.toc dl
-{
-    margin-top: 0em;
-    margin-bottom: 0em;
-}
-
-
-/***************************************************************************
-                               Special elements:
- ***************************************************************************/
-
-tt, code
-{
-    color: #400000;
-}
-
-.term
-{
-    font-weight: bold;
-    
-}
-
-div.variablelist dd p, div.glosslist dd p
-{
-    margin-top: 0em;
-}
-
-div.variablelist dd, div.glosslist dd
-{
-    margin-left: 1.5em;
-}
-
-div.glosslist dt
-{
-    font-style: italic;
-}
-
-.varname
-{
-    color: #400000;
-}
-
-span.command strong
-{
-    font-weight: normal;
-    color: #400000;
-}
-
-div.calloutlist table
-{
-}
-
-table
-{
-    border-collapse: collapse;
-}
-
-div.affiliation
-{
-    font-style: italic;
-}

flake.lock

@@ -0,0 +1,28 @@
+{
+  "nodes": {
+    "nixpkgs": {
+      "info": {
+        "lastModified": 1585405475,
+        "narHash": "sha256-bESW0n4KgPmZ0luxvwJ+UyATrC6iIltVCsGdLiphVeE="
+      },
+      "locked": {
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "b88ff468e9850410070d4e0ccd68c7011f15b2be",
+        "type": "github"
+      },
+      "original": {
+        "id": "nixpkgs",
+        "ref": "nixos-20.03-small",
+        "type": "indirect"
+      }
+    },
+    "root": {
+      "inputs": {
+        "nixpkgs": "nixpkgs"
+      }
+    }
+  },
+  "root": "root",
+  "version": 5
+}

flake.nix

@@ -0,0 +1,510 @@
+{
+  description = "The purely functional package manager";
+
+  edition = 201909; # FIXME: remove
+
+  inputs.nixpkgs.url = "nixpkgs/nixos-20.03-small";
+
+  outputs = { self, nixpkgs }:
+
+    let
+
+      version = builtins.readFile ./.version + versionSuffix;
+      versionSuffix =
+        if officialRelease
+        then ""
+        else "pre${builtins.substring 0 8 (self.lastModifiedDate or self.lastModified)}_${self.shortRev or "dirty"}";
+
+      officialRelease = false;
+
+      systems = [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ];
+
+      forAllSystems = f: nixpkgs.lib.genAttrs systems (system: f system);
+
+      # Memoize nixpkgs for different platforms for efficiency.
+      nixpkgsFor = forAllSystems (system:
+        import nixpkgs {
+          inherit system;
+          overlays = [ self.overlay ];
+        }
+      );
+
+      commonDeps = pkgs: with pkgs; rec {
+        # Use "busybox-sandbox-shell" if present,
+        # if not (legacy) fallback and hope it's sufficient.
+        sh = pkgs.busybox-sandbox-shell or (busybox.override {
+          useMusl = true;
+          enableStatic = true;
+          enableMinimal = true;
+          extraConfig = ''
+            CONFIG_FEATURE_FANCY_ECHO y
+            CONFIG_FEATURE_SH_MATH y
+            CONFIG_FEATURE_SH_MATH_64 y
+
+            CONFIG_ASH y
+            CONFIG_ASH_OPTIMIZE_FOR_SIZE y
+
+            CONFIG_ASH_ALIAS y
+            CONFIG_ASH_BASH_COMPAT y
+            CONFIG_ASH_CMDCMD y
+            CONFIG_ASH_ECHO y
+            CONFIG_ASH_GETOPTS y
+            CONFIG_ASH_INTERNAL_GLOB y
+            CONFIG_ASH_JOB_CONTROL y
+            CONFIG_ASH_PRINTF y
+            CONFIG_ASH_TEST y
+          '';
+        });
+
+        configureFlags =
+          lib.optionals stdenv.isLinux [
+            "--with-sandbox-shell=${sh}/bin/busybox"
+          ];
+
+        buildDeps =
+          [ bison
+            flex
+            libxml2
+            libxslt
+            docbook5
+            docbook_xsl_ns
+            autoconf-archive
+            autoreconfHook
+
+            curl
+            bzip2 xz brotli zlib editline
+            openssl pkgconfig sqlite
+            libarchive
+            boost
+            (if lib.versionAtLeast lib.version "20.03pre"
+             then nlohmann_json
+             else nlohmann_json.override { multipleHeaders = true; })
+            nlohmann_json
+            rustc cargo
+
+            # Tests
+            git
+            mercurial
+            jq
+          ]
+          ++ lib.optionals stdenv.isLinux [libseccomp utillinuxMinimal]
+          ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium
+          ++ lib.optional (stdenv.isLinux || stdenv.isDarwin)
+            (aws-sdk-cpp.override {
+              apis = ["s3" "transfer"];
+              customMemoryManagement = false;
+            });
+
+        propagatedDeps =
+          [ (boehmgc.override { enableLargeConfig = true; })
+          ];
+
+        perlDeps =
+          [ perl
+            perlPackages.DBDSQLite
+          ];
+      };
+
+    in {
+
+      # A Nixpkgs overlay that overrides the 'nix' and
+      # 'nix.perl-bindings' packages.
+      overlay = final: prev: {
+
+        nix = with final; with commonDeps pkgs; (stdenv.mkDerivation {
+          name = "nix-${version}";
+
+          src = self;
+
+          VERSION_SUFFIX = versionSuffix;
+
+          outputs = [ "out" "dev" "doc" ];
+
+          buildInputs = buildDeps;
+
+          propagatedBuildInputs = propagatedDeps;
+
+          preConfigure =
+            ''
+              # Copy libboost_context so we don't get all of Boost in our closure.
+              # https://github.com/NixOS/nixpkgs/issues/45462
+              mkdir -p $out/lib
+              cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib
+              rm -f $out/lib/*.a
+              ${lib.optionalString stdenv.isLinux ''
+                chmod u+w $out/lib/*.so.*
+                patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.*
+              ''}
+
+              ln -sfn ${final.nixVendoredCrates}/vendor/ nix-rust/vendor
+            '';
+
+          configureFlags = configureFlags ++
+            [ "--sysconfdir=/etc" ];
+
+          enableParallelBuilding = true;
+
+          makeFlags = "profiledir=$(out)/etc/profile.d";
+
+          doCheck = true;
+
+          installFlags = "sysconfdir=$(out)/etc";
+
+          doInstallCheck = true;
+          installCheckFlags = "sysconfdir=$(out)/etc";
+
+          separateDebugInfo = true;
+
+          preDist = ''
+            mkdir -p $doc/nix-support
+            echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products
+          '';
+        }) // {
+
+          perl-bindings = with final; stdenv.mkDerivation {
+            name = "nix-perl-${version}";
+
+            src = self;
+
+            buildInputs =
+              [ autoconf-archive
+                autoreconfHook
+                nix
+                curl
+                bzip2
+                xz
+                pkgconfig
+                pkgs.perl
+                boost
+              ]
+              ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium;
+
+            configureFlags = ''
+              --with-dbi=${perlPackages.DBI}/${pkgs.perl.libPrefix}
+              --with-dbd-sqlite=${perlPackages.DBDSQLite}/${pkgs.perl.libPrefix}
+            '';
+
+            enableParallelBuilding = true;
+
+            postUnpack = "sourceRoot=$sourceRoot/perl";
+          };
+
+        };
+
+        # Create a "vendor" directory that contains the crates listed in
+        # Cargo.lock, and include it in the Nix tarball. This allows Nix
+        # to be built without network access.
+        nixVendoredCrates =
+          let
+            lockFile = builtins.fromTOML (builtins.readFile nix-rust/Cargo.lock);
+
+            files = map (pkg: import <nix/fetchurl.nix> {
+              url = "https://crates.io/api/v1/crates/${pkg.name}/${pkg.version}/download";
+              sha256 = lockFile.metadata."checksum ${pkg.name} ${pkg.version} (registry+https://github.com/rust-lang/crates.io-index)";
+            }) (builtins.filter (pkg: pkg.source or "" == "registry+https://github.com/rust-lang/crates.io-index") lockFile.package);
+
+          in final.runCommand "cargo-vendor-dir" {}
+            ''
+              mkdir -p $out/vendor
+
+              cat > $out/vendor/config <<EOF
+              [source.crates-io]
+              replace-with = "vendored-sources"
+
+              [source.vendored-sources]
+              directory = "vendor"
+              EOF
+
+              ${toString (builtins.map (file: ''
+                mkdir $out/vendor/tmp
+                tar xvf ${file} -C $out/vendor/tmp
+                dir=$(echo $out/vendor/tmp/*)
+
+                # Add just enough metadata to keep Cargo happy.
+                printf '{"files":{},"package":"${file.outputHash}"}' > "$dir/.cargo-checksum.json"
+
+                # Clean up some cruft from the winapi crates. FIXME: find
+                # a way to remove winapi* from our dependencies.
+                if [[ $dir =~ /winapi ]]; then
+                  find $dir -name "*.a" -print0 | xargs -0 rm -f --
+                fi
+
+                mv "$dir" $out/vendor/
+
+                rm -rf $out/vendor/tmp
+              '') files)}
+            '';
+
+      };
+
+      hydraJobs = {
+
+        vendoredCrates =
+          with nixpkgsFor.x86_64-linux;
+
+          runCommand "vendored-crates" {}
+            ''
+              mkdir -p $out/nix-support
+              name=nix-vendored-crates-${version}
+              fn=$out/$name.tar.xz
+              tar cvfJ $fn -C ${nixVendoredCrates} vendor \
+                --owner=0 --group=0 --mode=u+rw,uga+r \
+                --transform "s,vendor,$name,"
+              echo "file crates-tarball $fn" >> $out/nix-support/hydra-build-products
+            '';
+
+        # Binary package for various platforms.
+        build = nixpkgs.lib.genAttrs systems (system: nixpkgsFor.${system}.nix);
+
+        # Perl bindings for various platforms.
+        perlBindings = nixpkgs.lib.genAttrs systems (system: nixpkgsFor.${system}.nix.perl-bindings);
+
+        # Binary tarball for various platforms, containing a Nix store
+        # with the closure of 'nix' package, and the second half of
+        # the installation script.
+        binaryTarball = nixpkgs.lib.genAttrs systems (system:
+
+          with nixpkgsFor.${system};
+
+          let
+            installerClosureInfo = closureInfo { rootPaths = [ nix cacert ]; };
+          in
+
+          runCommand "nix-binary-tarball-${version}"
+            { #nativeBuildInputs = lib.optional (system != "aarch64-linux") shellcheck;
+              meta.description = "Distribution-independent Nix bootstrap binaries for ${system}";
+            }
+            ''
+              cp ${installerClosureInfo}/registration $TMPDIR/reginfo
+              substitute ${./scripts/install-nix-from-closure.sh} $TMPDIR/install \
+                --subst-var-by nix ${nix} \
+                --subst-var-by cacert ${cacert}
+
+              substitute ${./scripts/install-darwin-multi-user.sh} $TMPDIR/install-darwin-multi-user.sh \
+                --subst-var-by nix ${nix} \
+                --subst-var-by cacert ${cacert}
+              substitute ${./scripts/install-systemd-multi-user.sh} $TMPDIR/install-systemd-multi-user.sh \
+                --subst-var-by nix ${nix} \
+                --subst-var-by cacert ${cacert}
+              substitute ${./scripts/install-multi-user.sh} $TMPDIR/install-multi-user \
+                --subst-var-by nix ${nix} \
+                --subst-var-by cacert ${cacert}
+
+              if type -p shellcheck; then
+                # SC1090: Don't worry about not being able to find
+                #         $nix/etc/profile.d/nix.sh
+                shellcheck --exclude SC1090 $TMPDIR/install
+                shellcheck $TMPDIR/install-darwin-multi-user.sh
+                shellcheck $TMPDIR/install-systemd-multi-user.sh
+
+                # SC1091: Don't panic about not being able to source
+                #         /etc/profile
+                # SC2002: Ignore "useless cat" "error", when loading
+                #         .reginfo, as the cat is a much cleaner
+                #         implementation, even though it is "useless"
+                # SC2116: Allow ROOT_HOME=$(echo ~root) for resolving
+                #         root's home directory
+                shellcheck --external-sources \
+                  --exclude SC1091,SC2002,SC2116 $TMPDIR/install-multi-user
+              fi
+
+              chmod +x $TMPDIR/install
+              chmod +x $TMPDIR/install-darwin-multi-user.sh
+              chmod +x $TMPDIR/install-systemd-multi-user.sh
+              chmod +x $TMPDIR/install-multi-user
+              dir=nix-${version}-${system}
+              fn=$out/$dir.tar.xz
+              mkdir -p $out/nix-support
+              echo "file binary-dist $fn" >> $out/nix-support/hydra-build-products
+              tar cvfJ $fn \
+                --owner=0 --group=0 --mode=u+rw,uga+r \
+                --absolute-names \
+                --hard-dereference \
+                --transform "s,$TMPDIR/install,$dir/install," \
+                --transform "s,$TMPDIR/reginfo,$dir/.reginfo," \
+                --transform "s,$NIX_STORE,$dir/store,S" \
+                $TMPDIR/install $TMPDIR/install-darwin-multi-user.sh \
+                $TMPDIR/install-systemd-multi-user.sh \
+                $TMPDIR/install-multi-user $TMPDIR/reginfo \
+                $(cat ${installerClosureInfo}/store-paths)
+            '');
+
+        # The first half of the installation script. This is uploaded
+        # to https://nixos.org/nix/install. It downloads the binary
+        # tarball for the user's system and calls the second half of the
+        # installation script.
+        installerScript =
+          with nixpkgsFor.x86_64-linux;
+          runCommand "installer-script"
+            { buildInputs = [ nix ];
+            }
+            ''
+              mkdir -p $out/nix-support
+
+              substitute ${./scripts/install.in} $out/install \
+                ${pkgs.lib.concatMapStrings
+                  (system: "--replace '@binaryTarball_${system}@' $(nix --experimental-features nix-command hash-file --base16 --type sha256 ${self.hydraJobs.binaryTarball.${system}}/*.tar.xz) ")
+                  [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ]
+                } \
+                --replace '@nixVersion@' ${version}
+
+              echo "file installer $out/install" >> $out/nix-support/hydra-build-products
+            '';
+
+        # Line coverage analysis.
+        coverage =
+          with nixpkgsFor.x86_64-linux;
+          with commonDeps pkgs;
+
+          releaseTools.coverageAnalysis {
+            name = "nix-coverage-${version}";
+
+            src = self;
+
+            preConfigure =
+              ''
+                ln -sfn ${nixVendoredCrates}/vendor/ nix-rust/vendor
+              '';
+
+            enableParallelBuilding = true;
+
+            buildInputs = buildDeps ++ propagatedDeps;
+
+            dontInstall = false;
+
+            doInstallCheck = true;
+
+            lcovFilter = [ "*/boost/*" "*-tab.*" ];
+
+            # We call `dot', and even though we just use it to
+            # syntax-check generated dot files, it still requires some
+            # fonts.  So provide those.
+            FONTCONFIG_FILE = texFunctions.fontsConf;
+
+            # To test building without precompiled headers.
+            makeFlagsArray = [ "PRECOMPILE_HEADERS=0" ];
+          };
+
+        # System tests.
+        tests.remoteBuilds = import ./tests/remote-builds.nix {
+          system = "x86_64-linux";
+          inherit nixpkgs;
+          inherit (self) overlay;
+        };
+
+        tests.nix-copy-closure = import ./tests/nix-copy-closure.nix {
+          system = "x86_64-linux";
+          inherit nixpkgs;
+          inherit (self) overlay;
+        };
+
+        tests.githubFlakes = (import ./tests/github-flakes.nix rec {
+          system = "x86_64-linux";
+          inherit nixpkgs;
+          inherit (self) overlay;
+        });
+
+        tests.setuid = nixpkgs.lib.genAttrs
+          ["i686-linux" "x86_64-linux"]
+          (system:
+            import ./tests/setuid.nix rec {
+              inherit nixpkgs system;
+              inherit (self) overlay;
+            });
+
+        # Test whether the binary tarball works in an Ubuntu system.
+        tests.binaryTarball =
+          with nixpkgsFor.x86_64-linux;
+          vmTools.runInLinuxImage (runCommand "nix-binary-tarball-test"
+            { diskImage = vmTools.diskImages.ubuntu1204x86_64;
+            }
+            ''
+              set -x
+              useradd -m alice
+              su - alice -c 'tar xf ${self.hydraJobs.binaryTarball.x86_64-linux}/*.tar.*'
+              mkdir /dest-nix
+              mount -o bind /dest-nix /nix # Provide a writable /nix.
+              chown alice /nix
+              su - alice -c '_NIX_INSTALLER_TEST=1 ./nix-*/install'
+              su - alice -c 'nix-store --verify'
+              su - alice -c 'PAGER= nix-store -qR ${self.hydraJobs.build.x86_64-linux}'
+
+              # Check whether 'nix upgrade-nix' works.
+              cat > /tmp/paths.nix <<EOF
+              {
+                x86_64-linux = "${self.hydraJobs.build.x86_64-linux}";
+              }
+              EOF
+              su - alice -c 'nix --experimental-features nix-command upgrade-nix -vvv --nix-store-paths-url file:///tmp/paths.nix'
+              (! [ -L /home/alice/.profile-1-link ])
+              su - alice -c 'PAGER= nix-store -qR ${self.hydraJobs.build.x86_64-linux}'
+
+              mkdir -p $out/nix-support
+              touch $out/nix-support/hydra-build-products
+              umount /nix
+            '');
+
+        /*
+        # Check whether we can still evaluate all of Nixpkgs.
+        tests.evalNixpkgs =
+          import (nixpkgs + "/pkgs/top-level/make-tarball.nix") {
+            # FIXME: fix pkgs/top-level/make-tarball.nix in NixOS to not require a revCount.
+            inherit nixpkgs;
+            pkgs = nixpkgsFor.x86_64-linux;
+            officialRelease = false;
+          };
+
+        # Check whether we can still evaluate NixOS.
+        tests.evalNixOS =
+          with nixpkgsFor.x86_64-linux;
+          runCommand "eval-nixos" { buildInputs = [ nix ]; }
+            ''
+              export NIX_STATE_DIR=$TMPDIR
+
+              nix-instantiate ${nixpkgs}/nixos/release-combined.nix -A tested --dry-run \
+                --arg nixpkgs '{ outPath = ${nixpkgs}; revCount = 123; shortRev = "abcdefgh"; }'
+
+              touch $out
+            '';
+        */
+
+      };
+
+      checks = forAllSystems (system: {
+        binaryTarball = self.hydraJobs.binaryTarball.${system};
+        perlBindings = self.hydraJobs.perlBindings.${system};
+      });
+
+      packages = forAllSystems (system: {
+        inherit (nixpkgsFor.${system}) nix;
+      });
+
+      defaultPackage = forAllSystems (system: self.packages.${system}.nix);
+
+      devShell = forAllSystems (system:
+        with nixpkgsFor.${system};
+        with commonDeps pkgs;
+
+        stdenv.mkDerivation {
+          name = "nix";
+
+          buildInputs = buildDeps ++ propagatedDeps ++ perlDeps ++ [ pkgs.rustfmt ];
+
+          inherit configureFlags;
+
+          enableParallelBuilding = true;
+
+          installFlags = "sysconfdir=$(out)/etc";
+
+          shellHook =
+            ''
+              export prefix=$(pwd)/inst
+              configureFlags+=" --prefix=$prefix"
+              PKG_CONFIG_PATH=$prefix/lib/pkgconfig:$PKG_CONFIG_PATH
+              PATH=$prefix/bin:$PATH
+              unset PYTHONPATH
+            '';
+        });
+
+  };
+}

local.mk

@@ -6,9 +6,11 @@ dist-files += configure config.h.in perl/configure
 
 clean-files += Makefile.config
 
-GLOBAL_CXXFLAGS += -I . -I src -I src/libutil -I src/libstore -I src/libmain -I src/libexpr -I src/nix -Wno-deprecated-declarations
+GLOBAL_CXXFLAGS += -Wno-deprecated-declarations
 
-$(foreach i, config.h $(call rwildcard, src/lib*, *.hh), \
+$(foreach i, config.h $(wildcard src/lib*/*.hh), \
   $(eval $(call install-file-in, $(i), $(includedir)/nix, 0644)))
 
 $(GCH) $(PCH): src/libutil/util.hh config.h
+
+GCH_CXXFLAGS = -I src/libutil

maintainers/upload-release.pl

@@ -1,5 +1,5 @@
 #! /usr/bin/env nix-shell
-#! nix-shell -i perl -p perl perlPackages.LWPUserAgent perlPackages.LWPProtocolHttps perlPackages.FileSlurp gnupg1
+#! nix-shell -i perl -p perl perlPackages.LWPUserAgent perlPackages.LWPProtocolHttps perlPackages.FileSlurp perlPackages.NetAmazonS3 gnupg1
 
 use strict;
 use Data::Dumper;
@@ -9,12 +9,16 @@ use File::Slurp;
 use File::Copy;
 use JSON::PP;
 use LWP::UserAgent;
+use Net::Amazon::S3;
 
 my $evalId = $ARGV[0] or die "Usage: $0 EVAL-ID\n";
 
-my $releasesDir = "/home/eelco/mnt/releases";
+my $releasesBucketName = "nix-releases";
+my $channelsBucketName = "nix-channels";
 my $nixpkgsDir = "/home/eelco/Dev/nixpkgs-pristine";
 
+my $TMPDIR = $ENV{'TMPDIR'} // "/tmp";
+
 # FIXME: cut&paste from nixos-channel-scripts.
 sub fetch {
     my ($url, $type) = @_;
@@ -42,13 +46,31 @@ my $version = $1;
 
 print STDERR "Nix revision is $nixRev, version is $version\n";
 
-File::Path::make_path($releasesDir);
-if (system("mountpoint -q $releasesDir") != 0) {
-    system("sshfs hydra-mirror\@nixos.org:/releases $releasesDir") == 0 or die;
-}
+my $releaseDir = "nix/$releaseName";
 
-my $releaseDir = "$releasesDir/nix/$releaseName";
-File::Path::make_path($releaseDir);
+my $tmpDir = "$TMPDIR/nix-release/$releaseName";
+File::Path::make_path($tmpDir);
+
+# S3 setup.
+my $aws_access_key_id = $ENV{'AWS_ACCESS_KEY_ID'} or die "No AWS_ACCESS_KEY_ID given.";
+my $aws_secret_access_key = $ENV{'AWS_SECRET_ACCESS_KEY'} or die "No AWS_SECRET_ACCESS_KEY given.";
+
+my $s3 = Net::Amazon::S3->new(
+    { aws_access_key_id     => $aws_access_key_id,
+      aws_secret_access_key => $aws_secret_access_key,
+      retry                 => 1,
+      host                  => "s3-eu-west-1.amazonaws.com",
+    });
+
+my $releasesBucket = $s3->bucket($releasesBucketName) or die;
+
+my $s3_us = Net::Amazon::S3->new(
+    { aws_access_key_id     => $aws_access_key_id,
+      aws_secret_access_key => $aws_secret_access_key,
+      retry                 => 1,
+    });
+
+my $channelsBucket = $s3_us->bucket($channelsBucketName) or die;
 
 sub downloadFile {
     my ($jobName, $productNr, $dstName) = @_;
@@ -57,40 +79,49 @@ sub downloadFile {
 
     my $srcFile = $buildInfo->{buildproducts}->{$productNr}->{path} or die "job '$jobName' lacks product $productNr\n";
     $dstName //= basename($srcFile);
-    my $dstFile = "$releaseDir/" . $dstName;
+    my $tmpFile = "$tmpDir/$dstName";
 
-    if (! -e $dstFile) {
-        print STDERR "downloading $srcFile to $dstFile...\n";
-        system("NIX_REMOTE=https://cache.nixos.org/ nix cat-store '$srcFile' > '$dstFile.tmp'") == 0
+    if (!-e $tmpFile) {
+        print STDERR "downloading $srcFile to $tmpFile...\n";
+        system("NIX_REMOTE=https://cache.nixos.org/ nix cat-store '$srcFile' > '$tmpFile'") == 0
             or die "unable to fetch $srcFile\n";
-        rename("$dstFile.tmp", $dstFile) or die;
     }
 
     my $sha256_expected = $buildInfo->{buildproducts}->{$productNr}->{sha256hash} or die;
-    my $sha256_actual = `nix hash-file --base16 --type sha256 '$dstFile'`;
+    my $sha256_actual = `nix hash-file --base16 --type sha256 '$tmpFile'`;
     chomp $sha256_actual;
     if ($sha256_expected ne $sha256_actual) {
-        print STDERR "file $dstFile is corrupt, got $sha256_actual, expected $sha256_expected\n";
+        print STDERR "file $tmpFile is corrupt, got $sha256_actual, expected $sha256_expected\n";
         exit 1;
     }
 
-    write_file("$dstFile.sha256", $sha256_expected);
+    write_file("$tmpFile.sha256", $sha256_expected);
 
-    if (! -e "$dstFile.asc") {
-        system("gpg2 --detach-sign --armor $dstFile") == 0 or die "unable to sign $dstFile\n";
+    if (! -e "$tmpFile.asc") {
+        system("gpg2 --detach-sign --armor $tmpFile") == 0 or die "unable to sign $tmpFile\n";
     }
 
-    return ($dstFile, $sha256_expected);
+    return $sha256_expected;
 }
 
 downloadFile("tarball", "2"); # .tar.bz2
-my ($tarball, $tarballHash) = downloadFile("tarball", "3"); # .tar.xz
+my $tarballHash = downloadFile("tarball", "3"); # .tar.xz
 downloadFile("binaryTarball.i686-linux", "1");
 downloadFile("binaryTarball.x86_64-linux", "1");
 downloadFile("binaryTarball.aarch64-linux", "1");
 downloadFile("binaryTarball.x86_64-darwin", "1");
 downloadFile("installerScript", "1");
 
+for my $fn (glob "$tmpDir/*") {
+    my $name = basename($fn);
+    my $dstKey = "$releaseDir/" . $name;
+    unless (defined $releasesBucket->head_key($dstKey)) {
+        print STDERR "uploading $fn to s3://$releasesBucketName/$dstKey...\n";
+        $releasesBucket->add_key_filename($dstKey, $fn)
+            or die $releasesBucket->err . ": " . $releasesBucket->errstr;
+    }
+}
+
 exit if $version =~ /pre/;
 
 # Update Nixpkgs in a very hacky way.
@@ -125,18 +156,11 @@ write_file("$nixpkgsDir/nixos/modules/installer/tools/nix-fallback-paths.nix",
 
 system("cd $nixpkgsDir && git commit -a -m 'nix: $oldName -> $version'") == 0 or die;
 
-# Extract the HTML manual.
-File::Path::make_path("$releaseDir/manual");
-
-system("tar xvf $tarball --strip-components=3 -C $releaseDir/manual --wildcards '*/doc/manual/*.html' '*/doc/manual/*.css' '*/doc/manual/*.gif' '*/doc/manual/*.png'") == 0 or die;
-
-if (! -e "$releaseDir/manual/index.html") {
-    symlink("manual.html", "$releaseDir/manual/index.html") or die;
-}
-
 # Update the "latest" symlink.
-symlink("$releaseName", "$releasesDir/nix/latest-tmp") or die;
-rename("$releasesDir/nix/latest-tmp", "$releasesDir/nix/latest") or die;
+$channelsBucket->add_key(
+    "nix-latest/install", "",
+    { "x-amz-website-redirect-location" => "https://releases.nixos.org/$releaseDir/install" })
+    or die $channelsBucket->err . ": " . $channelsBucket->errstr;
 
 # Tag the release in Git.
 chdir("/home/eelco/Dev/nix-pristine") or die;

misc/bash/completion.sh

@@ -0,0 +1,19 @@
+function _complete_nix {
+    local -a words
+    local cword cur
+    _get_comp_words_by_ref -n ':=&' words cword cur
+    local have_type
+    while IFS= read -r line; do
+        if [[ -z $have_type ]]; then
+            have_type=1
+            if [[ $line = filenames ]]; then
+                compopt -o filenames
+            fi
+        else
+            COMPREPLY+=("$line")
+        fi
+    done < <(NIX_GET_COMPLETIONS=$cword "${words[@]}")
+    __ltrim_colon_completions "$cur"
+}
+
+complete -F _complete_nix nix

misc/bash/local.mk

@@ -0,0 +1 @@
+$(eval $(call install-file-as, $(d)/completion.sh, $(datarootdir)/bash-completion/completions/_nix3, 0644))

mk/README.md

@@ -1,6 +0,0 @@
-This is a set of helper Makefiles for doing non-recursive builds with
-GNU Make.  The canonical source can be found at
-https://github.com/edolstra/make-rules.  You should copy the files
-into the `mk` subdirectory of your project.
-
-TODO: write more documentation.

mk/precompiled-headers.mk

@@ -8,14 +8,14 @@ GCH = $(buildprefix)precompiled-headers.h.gch
 $(GCH): precompiled-headers.h
 	@rm -f $@
 	@mkdir -p "$(dir $@)"
-	$(trace-gen) $(CXX) -x c++-header -o $@ $< $(GLOBAL_CXXFLAGS)
+	$(trace-gen) $(CXX) -x c++-header -o $@ $< $(GLOBAL_CXXFLAGS) $(GCH_CXXFLAGS)
 
 PCH = $(buildprefix)precompiled-headers.h.pch
 
 $(PCH): precompiled-headers.h
 	@rm -f $@
 	@mkdir -p "$(dir $@)"
-	$(trace-gen) $(CXX) -x c++-header -o $@ $< $(GLOBAL_CXXFLAGS)
+	$(trace-gen) $(CXX) -x c++-header -o $@ $< $(GLOBAL_CXXFLAGS) $(GCH_CXXFLAGS)
 
 clean-files += $(GCH) $(PCH)
 

perl/lib/Nix/Store.xs

@@ -102,7 +102,7 @@ SV * queryPathInfo(char * path, int base32)
     PPCODE:
         try {
             auto info = store()->queryPathInfo(store()->parseStorePath(path));
-            if (info->deriver)
+            if (!info->deriver)
                 XPUSHs(&PL_sv_undef);
             else
                 XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(*info->deriver).c_str(), 0)));

precompiled-headers.h

@@ -56,6 +56,3 @@
 #include <sys/wait.h>
 #include <termios.h>
 #include <unistd.h>
-
-#include "util.hh"
-#include "args.hh"

release-common.nix

@@ -1,80 +0,0 @@
-{ pkgs }:
-
-with pkgs;
-
-rec {
-  # Use "busybox-sandbox-shell" if present,
-  # if not (legacy) fallback and hope it's sufficient.
-  sh = pkgs.busybox-sandbox-shell or (busybox.override {
-    useMusl = true;
-    enableStatic = true;
-    enableMinimal = true;
-    extraConfig = ''
-      CONFIG_FEATURE_FANCY_ECHO y
-      CONFIG_FEATURE_SH_MATH y
-      CONFIG_FEATURE_SH_MATH_64 y
-
-      CONFIG_ASH y
-      CONFIG_ASH_OPTIMIZE_FOR_SIZE y
-
-      CONFIG_ASH_ALIAS y
-      CONFIG_ASH_BASH_COMPAT y
-      CONFIG_ASH_CMDCMD y
-      CONFIG_ASH_ECHO y
-      CONFIG_ASH_GETOPTS y
-      CONFIG_ASH_INTERNAL_GLOB y
-      CONFIG_ASH_JOB_CONTROL y
-      CONFIG_ASH_PRINTF y
-      CONFIG_ASH_TEST y
-    '';
-  });
-
-  configureFlags =
-    lib.optionals stdenv.isLinux [
-      "--with-sandbox-shell=${sh}/bin/busybox"
-    ];
-
-  tarballDeps =
-    [ bison
-      flex
-      libxml2
-      libxslt
-      docbook5
-      docbook_xsl_ns
-      autoconf-archive
-      autoreconfHook
-    ];
-
-  buildDeps =
-    [ curl
-      bzip2 xz brotli zlib editline
-      openssl pkgconfig sqlite boehmgc
-      libarchive
-      boost
-      nlohmann_json
-      rustc cargo
-
-      # Tests
-      git
-      mercurial
-    ]
-    ++ lib.optionals stdenv.isLinux [libseccomp utillinuxMinimal]
-    ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium
-    ++ lib.optional (stdenv.isLinux || stdenv.isDarwin)
-      ((aws-sdk-cpp.override {
-        apis = ["s3" "transfer"];
-        customMemoryManagement = false;
-      }).overrideDerivation (args: {
-        /*
-        patches = args.patches or [] ++ [ (fetchpatch {
-          url = https://github.com/edolstra/aws-sdk-cpp/commit/3e07e1f1aae41b4c8b340735ff9e8c735f0c063f.patch;
-          sha256 = "1pij0v449p166f9l29x7ppzk8j7g9k9mp15ilh5qxp29c7fnvxy2";
-        }) ];
-        */
-      }));
-
-  perlDeps =
-    [ perl
-      perlPackages.DBDSQLite
-    ];
-}

release.nix

@@ -1,379 +0,0 @@
-{ nix ? builtins.fetchGit ./.
-, nixpkgs ? builtins.fetchTarball https://github.com/NixOS/nixpkgs/archive/nixos-19.09.tar.gz
-, officialRelease ? false
-, systems ? [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ]
-}:
-
-let
-
-  pkgs = import nixpkgs { system = builtins.currentSystem or "x86_64-linux"; };
-
-  jobs = rec {
-
-    # Create a "vendor" directory that contains the crates listed in
-    # Cargo.lock, and include it in the Nix tarball. This allows Nix
-    # to be built without network access.
-    vendoredCrates =
-      let
-        lockFile = builtins.fromTOML (builtins.readFile nix-rust/Cargo.lock);
-
-        files = map (pkg: import <nix/fetchurl.nix> {
-          url = "https://crates.io/api/v1/crates/${pkg.name}/${pkg.version}/download";
-          sha256 = lockFile.metadata."checksum ${pkg.name} ${pkg.version} (registry+https://github.com/rust-lang/crates.io-index)";
-        }) (builtins.filter (pkg: pkg.source or "" == "registry+https://github.com/rust-lang/crates.io-index") lockFile.package);
-
-      in pkgs.runCommand "cargo-vendor-dir" {}
-        ''
-          mkdir -p $out/vendor
-
-          cat > $out/vendor/config <<EOF
-          [source.crates-io]
-          replace-with = "vendored-sources"
-
-          [source.vendored-sources]
-          directory = "vendor"
-          EOF
-
-          ${toString (builtins.map (file: ''
-            mkdir $out/vendor/tmp
-            tar xvf ${file} -C $out/vendor/tmp
-            dir=$(echo $out/vendor/tmp/*)
-
-            # Add just enough metadata to keep Cargo happy.
-            printf '{"files":{},"package":"${file.outputHash}"}' > "$dir/.cargo-checksum.json"
-
-            # Clean up some cruft from the winapi crates. FIXME: find
-            # a way to remove winapi* from our dependencies.
-            if [[ $dir =~ /winapi ]]; then
-              find $dir -name "*.a" -print0 | xargs -0 rm -f --
-            fi
-
-            mv "$dir" $out/vendor/
-
-            rm -rf $out/vendor/tmp
-          '') files)}
-        '';
-
-
-    tarball =
-      with pkgs;
-
-      with import ./release-common.nix { inherit pkgs; };
-
-      releaseTools.sourceTarball {
-        name = "nix-tarball";
-        version = builtins.readFile ./.version;
-        versionSuffix = if officialRelease then "" else "pre${toString nix.revCount}_${nix.shortRev}";
-        src = nix;
-        inherit officialRelease;
-
-        buildInputs = tarballDeps ++ buildDeps;
-
-        postUnpack = ''
-          (cd $sourceRoot && find . -type f) | cut -c3- > $sourceRoot/.dist-files
-          cat $sourceRoot/.dist-files
-        '';
-
-        preConfigure = ''
-          (cd perl ; autoreconf --install --force --verbose)
-          # TeX needs a writable font cache.
-          export VARTEXFONTS=$TMPDIR/texfonts
-        '';
-
-        distPhase =
-          ''
-            cp -prd ${vendoredCrates}/vendor/ nix-rust/vendor/
-
-            runHook preDist
-            make dist
-            mkdir -p $out/tarballs
-            cp *.tar.* $out/tarballs
-          '';
-
-        preDist = ''
-          make install docdir=$out/share/doc/nix makefiles=doc/manual/local.mk
-          echo "doc manual $out/share/doc/nix/manual" >> $out/nix-support/hydra-build-products
-        '';
-      };
-
-
-    build = pkgs.lib.genAttrs systems (system:
-
-      let pkgs = import nixpkgs { inherit system; }; in
-
-      with pkgs;
-
-      with import ./release-common.nix { inherit pkgs; };
-
-      releaseTools.nixBuild {
-        name = "nix";
-        src = tarball;
-
-        buildInputs = buildDeps;
-
-        preConfigure =
-          # Copy libboost_context so we don't get all of Boost in our closure.
-          # https://github.com/NixOS/nixpkgs/issues/45462
-          ''
-            mkdir -p $out/lib
-            cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib
-            rm -f $out/lib/*.a
-            ${lib.optionalString stdenv.isLinux ''
-              chmod u+w $out/lib/*.so.*
-              patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.*
-            ''}
-          '';
-
-        configureFlags = configureFlags ++
-          [ "--sysconfdir=/etc" ];
-
-        enableParallelBuilding = true;
-
-        makeFlags = "profiledir=$(out)/etc/profile.d";
-
-        installFlags = "sysconfdir=$(out)/etc";
-
-        doInstallCheck = true;
-        installCheckFlags = "sysconfdir=$(out)/etc";
-      });
-
-
-    perlBindings = pkgs.lib.genAttrs systems (system:
-
-      let pkgs = import nixpkgs { inherit system; }; in with pkgs;
-
-      releaseTools.nixBuild {
-        name = "nix-perl";
-        src = tarball;
-
-        buildInputs =
-          [ jobs.build.${system} curl bzip2 xz pkgconfig pkgs.perl boost ]
-          ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium;
-
-        configureFlags = ''
-          --with-dbi=${perlPackages.DBI}/${pkgs.perl.libPrefix}
-          --with-dbd-sqlite=${perlPackages.DBDSQLite}/${pkgs.perl.libPrefix}
-        '';
-
-        enableParallelBuilding = true;
-
-        postUnpack = "sourceRoot=$sourceRoot/perl";
-      });
-
-
-    binaryTarball = pkgs.lib.genAttrs systems (system:
-
-      with import nixpkgs { inherit system; };
-
-      let
-        toplevel = builtins.getAttr system jobs.build;
-        version = toplevel.src.version;
-        installerClosureInfo = closureInfo { rootPaths = [ toplevel cacert ]; };
-      in
-
-      runCommand "nix-binary-tarball-${version}"
-        { #nativeBuildInputs = lib.optional (system != "aarch64-linux") shellcheck;
-          meta.description = "Distribution-independent Nix bootstrap binaries for ${system}";
-        }
-        ''
-          cp ${installerClosureInfo}/registration $TMPDIR/reginfo
-          substitute ${./scripts/install-nix-from-closure.sh} $TMPDIR/install \
-            --subst-var-by nix ${toplevel} \
-            --subst-var-by cacert ${cacert}
-
-          substitute ${./scripts/install-darwin-multi-user.sh} $TMPDIR/install-darwin-multi-user.sh \
-            --subst-var-by nix ${toplevel} \
-            --subst-var-by cacert ${cacert}
-          substitute ${./scripts/install-systemd-multi-user.sh} $TMPDIR/install-systemd-multi-user.sh \
-            --subst-var-by nix ${toplevel} \
-            --subst-var-by cacert ${cacert}
-          substitute ${./scripts/install-multi-user.sh} $TMPDIR/install-multi-user \
-            --subst-var-by nix ${toplevel} \
-            --subst-var-by cacert ${cacert}
-
-          if type -p shellcheck; then
-            # SC1090: Don't worry about not being able to find
-            #         $nix/etc/profile.d/nix.sh
-            shellcheck --exclude SC1090 $TMPDIR/install
-            shellcheck $TMPDIR/install-darwin-multi-user.sh
-            shellcheck $TMPDIR/install-systemd-multi-user.sh
-
-            # SC1091: Don't panic about not being able to source
-            #         /etc/profile
-            # SC2002: Ignore "useless cat" "error", when loading
-            #         .reginfo, as the cat is a much cleaner
-            #         implementation, even though it is "useless"
-            # SC2116: Allow ROOT_HOME=$(echo ~root) for resolving
-            #         root's home directory
-            shellcheck --external-sources \
-              --exclude SC1091,SC2002,SC2116 $TMPDIR/install-multi-user
-          fi
-
-          chmod +x $TMPDIR/install
-          chmod +x $TMPDIR/install-darwin-multi-user.sh
-          chmod +x $TMPDIR/install-systemd-multi-user.sh
-          chmod +x $TMPDIR/install-multi-user
-          dir=nix-${version}-${system}
-          fn=$out/$dir.tar.xz
-          mkdir -p $out/nix-support
-          echo "file binary-dist $fn" >> $out/nix-support/hydra-build-products
-          tar cvfJ $fn \
-            --owner=0 --group=0 --mode=u+rw,uga+r \
-            --absolute-names \
-            --hard-dereference \
-            --transform "s,$TMPDIR/install,$dir/install," \
-            --transform "s,$TMPDIR/reginfo,$dir/.reginfo," \
-            --transform "s,$NIX_STORE,$dir/store,S" \
-            $TMPDIR/install $TMPDIR/install-darwin-multi-user.sh \
-            $TMPDIR/install-systemd-multi-user.sh \
-            $TMPDIR/install-multi-user $TMPDIR/reginfo \
-            $(cat ${installerClosureInfo}/store-paths)
-        '');
-
-
-    coverage =
-      with pkgs;
-
-      with import ./release-common.nix { inherit pkgs; };
-
-      releaseTools.coverageAnalysis {
-        name = "nix-build";
-        src = tarball;
-
-        enableParallelBuilding = true;
-
-        buildInputs = buildDeps;
-
-        dontInstall = false;
-
-        doInstallCheck = true;
-
-        lcovFilter = [ "*/boost/*" "*-tab.*" ];
-
-        # We call `dot', and even though we just use it to
-        # syntax-check generated dot files, it still requires some
-        # fonts.  So provide those.
-        FONTCONFIG_FILE = texFunctions.fontsConf;
-
-        # To test building without precompiled headers.
-        makeFlagsArray = [ "PRECOMPILE_HEADERS=0" ];
-      };
-
-
-    # System tests.
-    tests.remoteBuilds = (import ./tests/remote-builds.nix rec {
-      inherit nixpkgs;
-      nix = build.x86_64-linux; system = "x86_64-linux";
-    });
-
-    tests.nix-copy-closure = (import ./tests/nix-copy-closure.nix rec {
-      inherit nixpkgs;
-      nix = build.x86_64-linux; system = "x86_64-linux";
-    });
-
-    tests.setuid = pkgs.lib.genAttrs
-      ["i686-linux" "x86_64-linux"]
-      (system:
-        import ./tests/setuid.nix rec {
-          inherit nixpkgs;
-          nix = build.${system}; inherit system;
-        });
-
-    tests.binaryTarball =
-      with import nixpkgs { system = "x86_64-linux"; };
-      vmTools.runInLinuxImage (runCommand "nix-binary-tarball-test"
-        { diskImage = vmTools.diskImages.ubuntu1204x86_64;
-        }
-        ''
-          set -x
-          useradd -m alice
-          su - alice -c 'tar xf ${binaryTarball.x86_64-linux}/*.tar.*'
-          mkdir /dest-nix
-          mount -o bind /dest-nix /nix # Provide a writable /nix.
-          chown alice /nix
-          su - alice -c '_NIX_INSTALLER_TEST=1 ./nix-*/install'
-          su - alice -c 'nix-store --verify'
-          su - alice -c 'PAGER= nix-store -qR ${build.x86_64-linux}'
-
-          # Check whether 'nix upgrade-nix' works.
-          cat > /tmp/paths.nix <<EOF
-          {
-            x86_64-linux = "${build.x86_64-linux}";
-          }
-          EOF
-          su - alice -c 'nix --experimental-features nix-command upgrade-nix -vvv --nix-store-paths-url file:///tmp/paths.nix'
-          (! [ -L /home/alice/.profile-1-link ])
-          su - alice -c 'PAGER= nix-store -qR ${build.x86_64-linux}'
-
-          mkdir -p $out/nix-support
-          touch $out/nix-support/hydra-build-products
-          umount /nix
-        ''); # */
-
-    /*
-    tests.evalNixpkgs =
-      import (nixpkgs + "/pkgs/top-level/make-tarball.nix") {
-        inherit nixpkgs;
-        inherit pkgs;
-        nix = build.x86_64-linux;
-        officialRelease = false;
-      };
-
-    tests.evalNixOS =
-      pkgs.runCommand "eval-nixos" { buildInputs = [ build.x86_64-linux ]; }
-        ''
-          export NIX_STATE_DIR=$TMPDIR
-
-          nix-instantiate ${nixpkgs}/nixos/release-combined.nix -A tested --dry-run \
-            --arg nixpkgs '{ outPath = ${nixpkgs}; revCount = 123; shortRev = "abcdefgh"; }'
-
-          touch $out
-        '';
-    */
-
-
-    installerScript =
-      pkgs.runCommand "installer-script"
-        { buildInputs = [ build.x86_64-linux ];
-        }
-        ''
-          mkdir -p $out/nix-support
-
-          substitute ${./scripts/install.in} $out/install \
-            ${pkgs.lib.concatMapStrings
-              (system: "--replace '@binaryTarball_${system}@' $(nix --experimental-features nix-command hash-file --base16 --type sha256 ${binaryTarball.${system}}/*.tar.xz) ")
-              [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ]
-            } \
-            --replace '@nixVersion@' ${build.x86_64-linux.src.version}
-
-          echo "file installer $out/install" >> $out/nix-support/hydra-build-products
-        '';
-
-
-    # Aggregate job containing the release-critical jobs.
-    release = pkgs.releaseTools.aggregate {
-      name = "nix-${tarball.version}";
-      meta.description = "Release-critical builds";
-      constituents =
-        [ tarball
-          build.i686-linux
-          build.x86_64-darwin
-          build.x86_64-linux
-          build.aarch64-linux
-          binaryTarball.i686-linux
-          binaryTarball.x86_64-darwin
-          binaryTarball.x86_64-linux
-          binaryTarball.aarch64-linux
-          tests.remoteBuilds
-          tests.nix-copy-closure
-          tests.binaryTarball
-          #tests.evalNixpkgs
-          #tests.evalNixOS
-          installerScript
-        ];
-    };
-
-  };
-
-
-in jobs

scripts/install-multi-user.sh

@@ -13,12 +13,12 @@ set -o pipefail
 # however tracking which bits came from which would be impossible.
 
 readonly ESC='\033[0m'
-readonly BOLD='\033[38;1m'
-readonly BLUE='\033[38;34m'
-readonly BLUE_UL='\033[38;4;34m'
-readonly GREEN='\033[38;32m'
-readonly GREEN_UL='\033[38;4;32m'
-readonly RED='\033[38;31m'
+readonly BOLD='\033[1m'
+readonly BLUE='\033[34m'
+readonly BLUE_UL='\033[4;34m'
+readonly GREEN='\033[32m'
+readonly GREEN_UL='\033[4;32m'
+readonly RED='\033[31m'
 
 readonly NIX_USER_COUNT="32"
 readonly NIX_BUILD_GROUP_ID="30000"
@@ -567,7 +567,7 @@ install_from_extracted_nix() {
         cd "$EXTRACTED_NIX_PATH"
 
         _sudo "to copy the basic Nix files to the new store at $NIX_ROOT/store" \
-              rsync -rlpt ./store/* "$NIX_ROOT/store/"
+              rsync -rlpt --chmod=-w ./store/* "$NIX_ROOT/store/"
 
         if [ -d "$NIX_INSTALLED_NIX" ]; then
             echo "      Alright! We have our first nix at $NIX_INSTALLED_NIX"

scripts/install-nix-from-closure.sh

@@ -87,7 +87,7 @@ if ! [ -e $dest ]; then
 fi
 
 if ! [ -w $dest ]; then
-    echo "$0: directory $dest exists, but is not writable by you. This could indicate that another user has already performed a single-user installation of Nix on this system. If you wish to enable multi-user support see http://nixos.org/nix/manual/#ssec-multi-user. If you wish to continue with a single-user install for $USER please run 'chown -R $USER $dest' as root." >&2
+    echo "$0: directory $dest exists, but is not writable by you. This could indicate that another user has already performed a single-user installation of Nix on this system. If you wish to enable multi-user support see https://nixos.org/nix/manual/#ssec-multi-user. If you wish to continue with a single-user install for $USER please run 'chown -R $USER $dest' as root." >&2
     exit 1
 fi
 

scripts/install-systemd-multi-user.sh

@@ -88,7 +88,7 @@ poly_configure_nix_daemon_service() {
           systemctl start nix-daemon.socket
 
     _sudo "to start the nix-daemon.service" \
-          systemctl start nix-daemon.service
+          systemctl restart nix-daemon.service
 
 }
 

scripts/install.in

@@ -30,12 +30,13 @@ case "$(uname -s).$(uname -m)" in
     *) oops "sorry, there is no binary distribution of Nix for your platform";;
 esac
 
-url="https://nixos.org/releases/nix/nix-@nixVersion@/nix-@nixVersion@-$system.tar.xz"
+url="https://releases.nixos.org/nix/nix-@nixVersion@/nix-@nixVersion@-$system.tar.xz"
 
 tarball="$tmpDir/$(basename "$tmpDir/nix-@nixVersion@-$system.tar.xz")"
 
 require_util curl "download the binary tarball"
 require_util tar "unpack the binary tarball"
+require_util xz "unpack the binary tarball"
 
 echo "downloading Nix @nixVersion@ binary tarball for $system from '$url' to '$tmpDir'..."
 curl -L "$url" -o "$tarball" || oops "failed to download '$url'"

shell.nix

@@ -1,25 +1,3 @@
-{ useClang ? false }:
-
-with import (builtins.fetchTarball https://github.com/NixOS/nixpkgs/archive/nixos-19.09.tar.gz) {};
-
-with import ./release-common.nix { inherit pkgs; };
-
-(if useClang then clangStdenv else stdenv).mkDerivation {
-  name = "nix";
-
-  buildInputs = buildDeps ++ tarballDeps ++ perlDeps ++ [ pkgs.rustfmt ];
-
-  inherit configureFlags;
-
-  enableParallelBuilding = true;
-
-  installFlags = "sysconfdir=$(out)/etc";
-
-  shellHook =
-    ''
-      export prefix=$(pwd)/inst
-      configureFlags+=" --prefix=$prefix"
-      PKG_CONFIG_PATH=$prefix/lib/pkgconfig:$PKG_CONFIG_PATH
-      PATH=$prefix/bin:$PATH
-    '';
-}
+(import (fetchTarball https://github.com/edolstra/flake-compat/archive/master.tar.gz) {
+  src = builtins.fetchGit ./.;
+}).shellNix

src/build-remote/build-remote.cc

@@ -17,7 +17,7 @@
 #include "store-api.hh"
 #include "derivations.hh"
 #include "local-store.hh"
-#include "legacy.hh"
+#include "../nix/legacy.hh"
 
 using namespace nix;
 using std::cin;

src/error-demo/error-demo.cc

@@ -0,0 +1,66 @@
+#include "error.hh"
+#include "nixexpr.hh"
+
+#include <iostream>
+#include <optional>
+
+int main()
+{
+    using namespace nix;
+
+    // In each program where errors occur, this has to be set.
+    ErrorInfo::programName = std::optional("error-demo");
+
+    // Error in a program; no hint and no nix code.
+    printErrorInfo(
+        ErrorInfo { .level = elError,
+                    .name = "name",
+                    .description = "error description",
+                  });
+
+    // Warning with name, description, and hint.
+    // The hintfmt function makes all the substituted text yellow.
+    printErrorInfo(
+        ErrorInfo { .level = elWarning,
+                    .name = "name",
+                    .description = "error description",
+                    .hint =  std::optional(
+                                 hintfmt("there was a %1%", "warning")),
+                  });
+
+
+    // Warning with nix file, line number, column, and the lines of
+    // code where a warning occurred.
+    SymbolTable testTable;
+    auto problem_file = testTable.create("myfile.nix");
+
+    printErrorInfo(
+        ErrorInfo{
+            .level = elWarning,
+            .name = "warning name",
+            .description = "warning description",
+            .hint = hintfmt("this hint has %1% templated %2%!!", "yellow", "values"),
+            .nixCode = NixCode {
+                .errPos = Pos(problem_file, 40, 13),
+                .prevLineOfCode = std::nullopt,
+                .errLineOfCode = "this is the problem line of code",
+                .nextLineOfCode = std::nullopt
+            }});
+
+    // Error with previous and next lines of code.
+    printErrorInfo(
+        ErrorInfo{
+            .level = elError,
+            .name = "error name",
+            .description = "error description",
+            .hint = hintfmt("this hint has %1% templated %2%!!", "yellow", "values"),
+            .nixCode = NixCode {
+                .errPos = Pos(problem_file, 40, 13),
+                .prevLineOfCode = std::optional("previous line of code"),
+                .errLineOfCode = "this is the problem line of code",
+                .nextLineOfCode = std::optional("next line of code"),
+            }});
+
+
+    return 0;
+}

src/error-demo/local.mk

@@ -0,0 +1,12 @@
+programs += error-demo
+
+error-demo_DIR := $(d)
+
+error-demo_SOURCES := \
+  $(wildcard $(d)/*.cc) \
+
+error-demo_CXXFLAGS += -I src/libutil -I src/libexpr
+
+error-demo_LIBS = libutil libexpr
+
+error-demo_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) -lboost_context -lboost_thread -lboost_system

src/libexpr/attr-path.cc

@@ -6,11 +6,11 @@
 namespace nix {
 
 
-static Strings parseAttrPath(const string & s)
+static Strings parseAttrPath(std::string_view s)
 {
     Strings res;
     string cur;
-    string::const_iterator i = s.begin();
+    auto i = s.begin();
     while (i != s.end()) {
         if (*i == '.') {
             res.push_back(cur);
@@ -32,15 +32,22 @@ static Strings parseAttrPath(const string & s)
 }
 
 
-Value * findAlongAttrPath(EvalState & state, const string & attrPath,
+std::vector<Symbol> parseAttrPath(EvalState & state, std::string_view s)
+{
+    std::vector<Symbol> res;
+    for (auto & a : parseAttrPath(s))
+        res.push_back(state.symbols.create(a));
+    return res;
+}
+
+
+std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attrPath,
     Bindings & autoArgs, Value & vIn)
 {
     Strings tokens = parseAttrPath(attrPath);
 
-    Error attrError =
-        Error(format("attribute selection path '%1%' does not match expression") % attrPath);
-
     Value * v = &vIn;
+    Pos pos = noPos;
 
     for (auto & attr : tokens) {
 
@@ -70,8 +77,9 @@ Value * findAlongAttrPath(EvalState & state, const string & attrPath,
 
             Bindings::iterator a = v->attrs->find(state.symbols.create(attr));
             if (a == v->attrs->end())
-                throw Error(format("attribute '%1%' in selection path '%2%' not found") % attr % attrPath);
+                throw AttrPathNotFound("attribute '%1%' in selection path '%2%' not found", attr, attrPath);
             v = &*a->value;
+            pos = *a->pos;
         }
 
         else if (apType == apIndex) {
@@ -82,14 +90,15 @@ Value * findAlongAttrPath(EvalState & state, const string & attrPath,
                     % attrPath % showType(*v));
 
             if (attrIndex >= v->listSize())
-                throw Error(format("list index %1% in selection path '%2%' is out of range") % attrIndex % attrPath);
+                throw AttrPathNotFound("list index %1% in selection path '%2%' is out of range", attrIndex, attrPath);
 
             v = v->listElems()[attrIndex];
+            pos = noPos;
         }
 
     }
 
-    return v;
+    return {v, pos};
 }
 
 
@@ -98,9 +107,9 @@ Pos findDerivationFilename(EvalState & state, Value & v, std::string what)
     Value * v2;
     try {
         auto dummyArgs = state.allocBindings(0);
-        v2 = findAlongAttrPath(state, "meta.position", *dummyArgs, v);
+        v2 = findAlongAttrPath(state, "meta.position", *dummyArgs, v).first;
     } catch (Error &) {
-        throw Error("package '%s' has no source location information", what);
+        throw NoPositionInfo("package '%s' has no source location information", what);
     }
 
     // FIXME: is it possible to extract the Pos object instead of doing this

src/libexpr/attr-path.hh

@@ -7,10 +7,15 @@
 
 namespace nix {
 
-Value * findAlongAttrPath(EvalState & state, const string & attrPath,
+MakeError(AttrPathNotFound, Error);
+MakeError(NoPositionInfo, Error);
+
+std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attrPath,
     Bindings & autoArgs, Value & vIn);
 
 /* Heuristic to find the filename and lineno or a nix value. */
 Pos findDerivationFilename(EvalState & state, Value & v, std::string what);
 
+std::vector<Symbol> parseAttrPath(EvalState & state, std::string_view s);
+
 }

src/libexpr/attr-set.cc

@@ -43,6 +43,12 @@ Value * EvalState::allocAttr(Value & vAttrs, const Symbol & name)
 }
 
 
+Value * EvalState::allocAttr(Value & vAttrs, const std::string & name)
+{
+    return allocAttr(vAttrs, symbols.create(name));
+}
+
+
 void Bindings::sort()
 {
     std::sort(begin(), end());

src/libexpr/common-eval-args.cc

@@ -1,31 +1,59 @@
 #include "common-eval-args.hh"
 #include "shared.hh"
-#include "download.hh"
+#include "filetransfer.hh"
 #include "util.hh"
 #include "eval.hh"
+#include "fetchers.hh"
+#include "registry.hh"
+#include "flake/flakeref.hh"
+#include "store-api.hh"
 
 namespace nix {
 
 MixEvalArgs::MixEvalArgs()
 {
-    mkFlag()
-        .longName("arg")
-        .description("argument to be passed to Nix functions")
-        .labels({"name", "expr"})
-        .handler([&](std::vector<std::string> ss) { autoArgs[ss[0]] = 'E' + ss[1]; });
+    addFlag({
+        .longName = "arg",
+        .description = "argument to be passed to Nix functions",
+        .labels = {"name", "expr"},
+        .handler = {[&](std::string name, std::string expr) { autoArgs[name] = 'E' + expr; }}
+    });
 
-    mkFlag()
-        .longName("argstr")
-        .description("string-valued argument to be passed to Nix functions")
-        .labels({"name", "string"})
-        .handler([&](std::vector<std::string> ss) { autoArgs[ss[0]] = 'S' + ss[1]; });
+    addFlag({
+        .longName = "argstr",
+        .description = "string-valued argument to be passed to Nix functions",
+        .labels = {"name", "string"},
+        .handler = {[&](std::string name, std::string s) { autoArgs[name] = 'S' + s; }},
+    });
 
-    mkFlag()
-        .shortName('I')
-        .longName("include")
-        .description("add a path to the list of locations used to look up <...> file names")
-        .label("path")
-        .handler([&](std::string s) { searchPath.push_back(s); });
+    addFlag({
+        .longName = "include",
+        .shortName = 'I',
+        .description = "add a path to the list of locations used to look up <...> file names",
+        .labels = {"path"},
+        .handler = {[&](std::string s) { searchPath.push_back(s); }}
+    });
+
+    addFlag({
+        .longName = "impure",
+        .description = "allow access to mutable paths and repositories",
+        .handler = {[&]() {
+            evalSettings.pureEval = false;
+        }},
+    });
+
+    addFlag({
+        .longName = "override-flake",
+        .description = "override a flake registry value",
+        .labels = {"original-ref", "resolved-ref"},
+        .handler = {[&](std::string _from, std::string _to) {
+            auto from = parseFlakeRef(_from, absPath("."));
+            auto to = parseFlakeRef(_to, absPath("."));
+            fetchers::Attrs extraAttrs;
+            if (to.subdir != "") extraAttrs["dir"] = to.subdir;
+            fetchers::overrideRegistry(from.input, to.input, extraAttrs);
+        }}
+    });
 }
 
 Bindings * MixEvalArgs::getAutoArgs(EvalState & state)
@@ -46,9 +74,9 @@ Bindings * MixEvalArgs::getAutoArgs(EvalState & state)
 Path lookupFileArg(EvalState & state, string s)
 {
     if (isUri(s)) {
-        CachedDownloadRequest request(s);
-        request.unpack = true;
-        return getDownloader()->downloadCached(state.store, request).path;
+        return state.store->toRealPath(
+            fetchers::downloadTarball(
+                state.store, resolveUri(s), "source", false).storePath);
     } else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') {
         Path p = s.substr(1, s.size() - 2);
         return state.findFile(p);

src/libexpr/eval-cache.cc

@@ -0,0 +1,503 @@
+#include "eval-cache.hh"
+#include "sqlite.hh"
+#include "eval.hh"
+#include "eval-inline.hh"
+
+namespace nix::eval_cache {
+
+static const char * schema = R"sql(
+create table if not exists Attributes (
+    parent      integer not null,
+    name        text,
+    type        integer not null,
+    value       text,
+    primary key (parent, name)
+);
+)sql";
+
+struct AttrDb
+{
+    struct State
+    {
+        SQLite db;
+        SQLiteStmt insertAttribute;
+        SQLiteStmt queryAttribute;
+        SQLiteStmt queryAttributes;
+        std::unique_ptr<SQLiteTxn> txn;
+    };
+
+    std::unique_ptr<Sync<State>> _state;
+
+    AttrDb(const Hash & fingerprint)
+        : _state(std::make_unique<Sync<State>>())
+    {
+        auto state(_state->lock());
+
+        Path cacheDir = getCacheDir() + "/nix/eval-cache-v1";
+        createDirs(cacheDir);
+
+        Path dbPath = cacheDir + "/" + fingerprint.to_string(Base16, false) + ".sqlite";
+
+        state->db = SQLite(dbPath);
+        state->db.isCache();
+        state->db.exec(schema);
+
+        state->insertAttribute.create(state->db,
+            "insert or replace into Attributes(parent, name, type, value) values (?, ?, ?, ?)");
+
+        state->queryAttribute.create(state->db,
+            "select rowid, type, value from Attributes where parent = ? and name = ?");
+
+        state->queryAttributes.create(state->db,
+            "select name from Attributes where parent = ?");
+
+        state->txn = std::make_unique<SQLiteTxn>(state->db);
+    }
+
+    ~AttrDb()
+    {
+        try {
+            auto state(_state->lock());
+            state->txn->commit();
+            state->txn.reset();
+        } catch (...) {
+            ignoreException();
+        }
+    }
+
+    AttrId setAttrs(
+        AttrKey key,
+        const std::vector<Symbol> & attrs)
+    {
+        auto state(_state->lock());
+
+        state->insertAttribute.use()
+            (key.first)
+            (key.second)
+            (AttrType::FullAttrs)
+            (0, false).exec();
+
+        AttrId rowId = state->db.getLastInsertedRowId();
+        assert(rowId);
+
+        for (auto & attr : attrs)
+            state->insertAttribute.use()
+                (rowId)
+                (attr)
+                (AttrType::Placeholder)
+                (0, false).exec();
+
+        return rowId;
+    }
+
+    AttrId setString(
+        AttrKey key,
+        std::string_view s)
+    {
+        auto state(_state->lock());
+
+        state->insertAttribute.use()
+            (key.first)
+            (key.second)
+            (AttrType::String)
+            (s).exec();
+
+        return state->db.getLastInsertedRowId();
+    }
+
+    AttrId setBool(
+        AttrKey key,
+        bool b)
+    {
+        auto state(_state->lock());
+
+        state->insertAttribute.use()
+            (key.first)
+            (key.second)
+            (AttrType::Bool)
+            (b ? 1 : 0).exec();
+
+        return state->db.getLastInsertedRowId();
+    }
+
+    AttrId setPlaceholder(AttrKey key)
+    {
+        auto state(_state->lock());
+
+        state->insertAttribute.use()
+            (key.first)
+            (key.second)
+            (AttrType::Placeholder)
+            (0, false).exec();
+
+        return state->db.getLastInsertedRowId();
+    }
+
+    AttrId setMissing(AttrKey key)
+    {
+        auto state(_state->lock());
+
+        state->insertAttribute.use()
+            (key.first)
+            (key.second)
+            (AttrType::Missing)
+            (0, false).exec();
+
+        return state->db.getLastInsertedRowId();
+    }
+
+    AttrId setMisc(AttrKey key)
+    {
+        auto state(_state->lock());
+
+        state->insertAttribute.use()
+            (key.first)
+            (key.second)
+            (AttrType::Misc)
+            (0, false).exec();
+
+        return state->db.getLastInsertedRowId();
+    }
+
+    AttrId setFailed(AttrKey key)
+    {
+        auto state(_state->lock());
+
+        state->insertAttribute.use()
+            (key.first)
+            (key.second)
+            (AttrType::Failed)
+            (0, false).exec();
+
+        return state->db.getLastInsertedRowId();
+    }
+
+    std::optional<std::pair<AttrId, AttrValue>> getAttr(
+        AttrKey key,
+        SymbolTable & symbols)
+    {
+        auto state(_state->lock());
+
+        auto queryAttribute(state->queryAttribute.use()(key.first)(key.second));
+        if (!queryAttribute.next()) return {};
+
+        auto rowId = (AttrType) queryAttribute.getInt(0);
+        auto type = (AttrType) queryAttribute.getInt(1);
+
+        switch (type) {
+            case AttrType::Placeholder:
+                return {{rowId, placeholder_t()}};
+            case AttrType::FullAttrs: {
+                // FIXME: expensive, should separate this out.
+                std::vector<Symbol> attrs;
+                auto queryAttributes(state->queryAttributes.use()(rowId));
+                while (queryAttributes.next())
+                    attrs.push_back(symbols.create(queryAttributes.getStr(0)));
+                return {{rowId, attrs}};
+            }
+            case AttrType::String:
+                return {{rowId, queryAttribute.getStr(2)}};
+            case AttrType::Bool:
+                return {{rowId, queryAttribute.getInt(2) != 0}};
+            case AttrType::Missing:
+                return {{rowId, missing_t()}};
+            case AttrType::Misc:
+                return {{rowId, misc_t()}};
+            case AttrType::Failed:
+                return {{rowId, failed_t()}};
+            default:
+                throw Error("unexpected type in evaluation cache");
+        }
+    }
+};
+
+EvalCache::EvalCache(
+    bool useCache,
+    const Hash & fingerprint,
+    EvalState & state,
+    RootLoader rootLoader)
+    : db(useCache ? std::make_shared<AttrDb>(fingerprint) : nullptr)
+    , state(state)
+    , rootLoader(rootLoader)
+{
+}
+
+Value * EvalCache::getRootValue()
+{
+    if (!value) {
+        debug("getting root value");
+        value = allocRootValue(rootLoader());
+    }
+    return *value;
+}
+
+std::shared_ptr<AttrCursor> EvalCache::getRoot()
+{
+    return std::make_shared<AttrCursor>(ref(shared_from_this()), std::nullopt);
+}
+
+AttrCursor::AttrCursor(
+    ref<EvalCache> root,
+    Parent parent,
+    Value * value,
+    std::optional<std::pair<AttrId, AttrValue>> && cachedValue)
+    : root(root), parent(parent), cachedValue(std::move(cachedValue))
+{
+    if (value)
+        _value = allocRootValue(value);
+}
+
+AttrKey AttrCursor::getKey()
+{
+    if (!parent)
+        return {0, root->state.sEpsilon};
+    if (!parent->first->cachedValue) {
+        parent->first->cachedValue = root->db->getAttr(
+            parent->first->getKey(), root->state.symbols);
+        assert(parent->first->cachedValue);
+    }
+    return {parent->first->cachedValue->first, parent->second};
+}
+
+Value & AttrCursor::getValue()
+{
+    if (!_value) {
+        if (parent) {
+            auto & vParent = parent->first->getValue();
+            root->state.forceAttrs(vParent);
+            auto attr = vParent.attrs->get(parent->second);
+            if (!attr)
+                throw Error("attribute '%s' is unexpectedly missing", getAttrPathStr());
+            _value = allocRootValue(attr->value);
+        } else
+            _value = allocRootValue(root->getRootValue());
+    }
+    return **_value;
+}
+
+std::vector<Symbol> AttrCursor::getAttrPath() const
+{
+    if (parent) {
+        auto attrPath = parent->first->getAttrPath();
+        attrPath.push_back(parent->second);
+        return attrPath;
+    } else
+        return {};
+}
+
+std::vector<Symbol> AttrCursor::getAttrPath(Symbol name) const
+{
+    auto attrPath = getAttrPath();
+    attrPath.push_back(name);
+    return attrPath;
+}
+
+std::string AttrCursor::getAttrPathStr() const
+{
+    return concatStringsSep(".", getAttrPath());
+}
+
+std::string AttrCursor::getAttrPathStr(Symbol name) const
+{
+    return concatStringsSep(".", getAttrPath(name));
+}
+
+Value & AttrCursor::forceValue()
+{
+    debug("evaluating uncached attribute %s", getAttrPathStr());
+
+    auto & v = getValue();
+
+    try {
+        root->state.forceValue(v);
+    } catch (EvalError &) {
+        debug("setting '%s' to failed", getAttrPathStr());
+        if (root->db)
+            cachedValue = {root->db->setFailed(getKey()), failed_t()};
+        throw;
+    }
+
+    if (root->db && (!cachedValue || std::get_if<placeholder_t>(&cachedValue->second))) {
+        if (v.type == tString)
+            cachedValue = {root->db->setString(getKey(), v.string.s), v.string.s};
+        else if (v.type == tBool)
+            cachedValue = {root->db->setBool(getKey(), v.boolean), v.boolean};
+        else if (v.type == tAttrs)
+            ; // FIXME: do something?
+        else
+            cachedValue = {root->db->setMisc(getKey()), misc_t()};
+    }
+
+    return v;
+}
+
+std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(Symbol name)
+{
+    if (root->db) {
+        if (!cachedValue)
+            cachedValue = root->db->getAttr(getKey(), root->state.symbols);
+
+        if (cachedValue) {
+            if (auto attrs = std::get_if<std::vector<Symbol>>(&cachedValue->second)) {
+                for (auto & attr : *attrs)
+                    if (attr == name)
+                        return std::make_shared<AttrCursor>(root, std::make_pair(shared_from_this(), name));
+                return nullptr;
+            } else if (std::get_if<placeholder_t>(&cachedValue->second)) {
+                auto attr = root->db->getAttr({cachedValue->first, name}, root->state.symbols);
+                if (attr) {
+                    if (std::get_if<missing_t>(&attr->second))
+                        return nullptr;
+                    else if (std::get_if<failed_t>(&attr->second))
+                        throw EvalError("cached failure of attribute '%s'", getAttrPathStr(name));
+                    else
+                        return std::make_shared<AttrCursor>(root,
+                            std::make_pair(shared_from_this(), name), nullptr, std::move(attr));
+                }
+                // Incomplete attrset, so need to fall thru and
+                // evaluate to see whether 'name' exists
+            } else
+                return nullptr;
+                //throw TypeError("'%s' is not an attribute set", getAttrPathStr());
+        }
+    }
+
+    auto & v = forceValue();
+
+    if (v.type != tAttrs)
+        return nullptr;
+        //throw TypeError("'%s' is not an attribute set", getAttrPathStr());
+
+    auto attr = v.attrs->get(name);
+
+    if (!attr) {
+        if (root->db) {
+            if (!cachedValue)
+                cachedValue = {root->db->setPlaceholder(getKey()), placeholder_t()};
+            root->db->setMissing({cachedValue->first, name});
+        }
+        return nullptr;
+    }
+
+    std::optional<std::pair<AttrId, AttrValue>> cachedValue2;
+    if (root->db) {
+        if (!cachedValue)
+            cachedValue = {root->db->setPlaceholder(getKey()), placeholder_t()};
+        cachedValue2 = {root->db->setPlaceholder({cachedValue->first, name}), placeholder_t()};
+    }
+
+    return std::make_shared<AttrCursor>(
+        root, std::make_pair(shared_from_this(), name), attr->value, std::move(cachedValue2));
+}
+
+std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(std::string_view name)
+{
+    return maybeGetAttr(root->state.symbols.create(name));
+}
+
+std::shared_ptr<AttrCursor> AttrCursor::getAttr(Symbol name)
+{
+    auto p = maybeGetAttr(name);
+    if (!p)
+        throw Error("attribute '%s' does not exist", getAttrPathStr(name));
+    return p;
+}
+
+std::shared_ptr<AttrCursor> AttrCursor::getAttr(std::string_view name)
+{
+    return getAttr(root->state.symbols.create(name));
+}
+
+std::shared_ptr<AttrCursor> AttrCursor::findAlongAttrPath(const std::vector<Symbol> & attrPath)
+{
+    auto res = shared_from_this();
+    for (auto & attr : attrPath) {
+        res = res->maybeGetAttr(attr);
+        if (!res) return {};
+    }
+    return res;
+}
+
+std::string AttrCursor::getString()
+{
+    if (root->db) {
+        if (!cachedValue)
+            cachedValue = root->db->getAttr(getKey(), root->state.symbols);
+        if (cachedValue && !std::get_if<placeholder_t>(&cachedValue->second)) {
+            if (auto s = std::get_if<std::string>(&cachedValue->second)) {
+                debug("using cached string attribute '%s'", getAttrPathStr());
+                return *s;
+            } else
+                throw TypeError("'%s' is not a string", getAttrPathStr());
+        }
+    }
+
+    auto & v = forceValue();
+
+    if (v.type != tString)
+        throw TypeError("'%s' is not a string", getAttrPathStr());
+
+    return v.string.s;
+}
+
+bool AttrCursor::getBool()
+{
+    if (root->db) {
+        if (!cachedValue)
+            cachedValue = root->db->getAttr(getKey(), root->state.symbols);
+        if (cachedValue && !std::get_if<placeholder_t>(&cachedValue->second)) {
+            if (auto b = std::get_if<bool>(&cachedValue->second)) {
+                debug("using cached Boolean attribute '%s'", getAttrPathStr());
+                return *b;
+            } else
+                throw TypeError("'%s' is not a Boolean", getAttrPathStr());
+        }
+    }
+
+    auto & v = forceValue();
+
+    if (v.type != tBool)
+        throw TypeError("'%s' is not a Boolean", getAttrPathStr());
+
+    return v.boolean;
+}
+
+std::vector<Symbol> AttrCursor::getAttrs()
+{
+    if (root->db) {
+        if (!cachedValue)
+            cachedValue = root->db->getAttr(getKey(), root->state.symbols);
+        if (cachedValue && !std::get_if<placeholder_t>(&cachedValue->second)) {
+            if (auto attrs = std::get_if<std::vector<Symbol>>(&cachedValue->second)) {
+                debug("using cached attrset attribute '%s'", getAttrPathStr());
+                return *attrs;
+            } else
+                throw TypeError("'%s' is not an attribute set", getAttrPathStr());
+        }
+    }
+
+    auto & v = forceValue();
+
+    if (v.type != tAttrs)
+        throw TypeError("'%s' is not an attribute set", getAttrPathStr());
+
+    std::vector<Symbol> attrs;
+    for (auto & attr : *getValue().attrs)
+        attrs.push_back(attr.name);
+    std::sort(attrs.begin(), attrs.end(), [](const Symbol & a, const Symbol & b) {
+        return (const string &) a < (const string &) b;
+    });
+
+    if (root->db)
+        cachedValue = {root->db->setAttrs(getKey(), attrs), attrs};
+
+    return attrs;
+}
+
+bool AttrCursor::isDerivation()
+{
+    auto aType = maybeGetAttr("type");
+    return aType && aType->getString() == "derivation";
+}
+
+}

src/libexpr/eval-cache.hh

@@ -0,0 +1,106 @@
+#pragma once
+
+#include "sync.hh"
+#include "hash.hh"
+#include "eval.hh"
+
+#include <variant>
+
+namespace nix::eval_cache {
+
+class AttrDb;
+class AttrCursor;
+
+class EvalCache : public std::enable_shared_from_this<EvalCache>
+{
+    friend class AttrCursor;
+
+    std::shared_ptr<AttrDb> db;
+    EvalState & state;
+    typedef std::function<Value *()> RootLoader;
+    RootLoader rootLoader;
+    RootValue value;
+
+    Value * getRootValue();
+
+public:
+
+    EvalCache(
+        bool useCache,
+        const Hash & fingerprint,
+        EvalState & state,
+        RootLoader rootLoader);
+
+    std::shared_ptr<AttrCursor> getRoot();
+};
+
+enum AttrType {
+    Placeholder = 0,
+    FullAttrs = 1,
+    String = 2,
+    Missing = 3,
+    Misc = 4,
+    Failed = 5,
+    Bool = 6,
+};
+
+struct placeholder_t {};
+struct missing_t {};
+struct misc_t {};
+struct failed_t {};
+typedef uint64_t AttrId;
+typedef std::pair<AttrId, Symbol> AttrKey;
+typedef std::variant<std::vector<Symbol>, std::string, placeholder_t, missing_t, misc_t, failed_t, bool> AttrValue;
+
+class AttrCursor : public std::enable_shared_from_this<AttrCursor>
+{
+    friend class EvalCache;
+
+    ref<EvalCache> root;
+    typedef std::optional<std::pair<std::shared_ptr<AttrCursor>, Symbol>> Parent;
+    Parent parent;
+    RootValue _value;
+    std::optional<std::pair<AttrId, AttrValue>> cachedValue;
+
+    AttrKey getKey();
+
+    Value & getValue();
+
+public:
+
+    AttrCursor(
+        ref<EvalCache> root,
+        Parent parent,
+        Value * value = nullptr,
+        std::optional<std::pair<AttrId, AttrValue>> && cachedValue = {});
+
+    std::vector<Symbol> getAttrPath() const;
+
+    std::vector<Symbol> getAttrPath(Symbol name) const;
+
+    std::string getAttrPathStr() const;
+
+    std::string getAttrPathStr(Symbol name) const;
+
+    std::shared_ptr<AttrCursor> maybeGetAttr(Symbol name);
+
+    std::shared_ptr<AttrCursor> maybeGetAttr(std::string_view name);
+
+    std::shared_ptr<AttrCursor> getAttr(Symbol name);
+
+    std::shared_ptr<AttrCursor> getAttr(std::string_view name);
+
+    std::shared_ptr<AttrCursor> findAlongAttrPath(const std::vector<Symbol> & attrPath);
+
+    std::string getString();
+
+    bool getBool();
+
+    std::vector<Symbol> getAttrs();
+
+    bool isDerivation();
+
+    Value & forceValue();
+};
+
+}

src/libexpr/eval-inline.hh

@@ -57,7 +57,7 @@ inline void EvalState::forceAttrs(Value & v)
 
 inline void EvalState::forceAttrs(Value & v, const Pos & pos)
 {
-    forceValue(v);
+    forceValue(v, pos);
     if (v.type != tAttrs)
         throwTypeError("value is %1% while a set was expected, at %2%", v, pos);
 }
@@ -73,7 +73,7 @@ inline void EvalState::forceList(Value & v)
 
 inline void EvalState::forceList(Value & v, const Pos & pos)
 {
-    forceValue(v);
+    forceValue(v, pos);
     if (!v.isList())
         throwTypeError("value is %1% while a list was expected, at %2%", v, pos);
 }

src/libexpr/eval.cc

@@ -5,9 +5,10 @@
 #include "derivations.hh"
 #include "globals.hh"
 #include "eval-inline.hh"
-#include "download.hh"
+#include "filetransfer.hh"
 #include "json.hh"
 #include "function-trace.hh"
+#include "flake/flake.hh"
 
 #include <algorithm>
 #include <chrono>
@@ -22,6 +23,8 @@
 
 #if HAVE_BOEHMGC
 
+#define GC_INCLUDE_NEW
+
 #include <gc/gc.h>
 #include <gc/gc_cpp.h>
 
@@ -56,6 +59,12 @@ static char * dupStringWithLen(const char * s, size_t size)
 }
 
 
+RootValue allocRootValue(Value * v)
+{
+    return std::allocate_shared<Value *>(traceable_allocator<Value *>(), v);
+}
+
+
 static void printValue(std::ostream & str, std::set<const Value *> & active, const Value & v)
 {
     checkInterrupt();
@@ -153,12 +162,12 @@ const Value *getPrimOp(const Value &v) {
 }
 
 
-string showType(const Value & v)
+string showType(ValueType type)
 {
-    switch (v.type) {
+    switch (type) {
         case tInt: return "an integer";
-        case tBool: return "a boolean";
-        case tString: return v.string.context ? "a string with context" : "a string";
+        case tBool: return "a Boolean";
+        case tString: return "a string";
         case tPath: return "a path";
         case tNull: return "null";
         case tAttrs: return "a set";
@@ -167,14 +176,39 @@ string showType(const Value & v)
         case tApp: return "a function application";
         case tLambda: return "a function";
         case tBlackhole: return "a black hole";
+        case tPrimOp: return "a built-in function";
+        case tPrimOpApp: return "a partially applied built-in function";
+        case tExternal: return "an external value";
+        case tFloat: return "a float";
+    }
+    abort();
+}
+
+
+string showType(const Value & v)
+{
+    switch (v.type) {
+        case tString: return v.string.context ? "a string with context" : "a string";
         case tPrimOp:
             return fmt("the built-in function '%s'", string(v.primOp->name));
         case tPrimOpApp:
             return fmt("the partially applied built-in function '%s'", string(getPrimOp(v)->primOp->name));
         case tExternal: return v.external->showType();
-        case tFloat: return "a float";
+    default:
+        return showType(v.type);
     }
-    abort();
+}
+
+
+bool Value::isTrivial() const
+{
+    return
+        type != tApp
+        && type != tPrimOpApp
+        && (type != tThunk
+            || (dynamic_cast<ExprAttrs *>(thunk.expr)
+                && ((ExprAttrs *) thunk.expr)->dynamicAttrs.empty())
+            || dynamic_cast<ExprLambda *>(thunk.expr));
 }
 
 
@@ -315,6 +349,10 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
     , sOutputHash(symbols.create("outputHash"))
     , sOutputHashAlgo(symbols.create("outputHashAlgo"))
     , sOutputHashMode(symbols.create("outputHashMode"))
+    , sDescription(symbols.create("description"))
+    , sSelf(symbols.create("self"))
+    , sEpsilon(symbols.create(""))
+    , sRecurseForDerivations(symbols.create("recurseForDerivations"))
     , repair(NoRepair)
     , store(store)
     , baseEnv(allocEnv(128))
@@ -463,14 +501,21 @@ Value * EvalState::addConstant(const string & name, Value & v)
 Value * EvalState::addPrimOp(const string & name,
     size_t arity, PrimOpFun primOp)
 {
+    auto name2 = string(name, 0, 2) == "__" ? string(name, 2) : name;
+    Symbol sym = symbols.create(name2);
+
+    /* Hack to make constants lazy: turn them into a application of
+       the primop to a dummy value. */
     if (arity == 0) {
+        auto vPrimOp = allocValue();
+        vPrimOp->type = tPrimOp;
+        vPrimOp->primOp = new PrimOp(primOp, 1, sym);
         Value v;
-        primOp(*this, noPos, nullptr, v);
+        mkApp(v, *vPrimOp, *vPrimOp);
         return addConstant(name, v);
     }
+
     Value * v = allocValue();
-    string name2 = string(name, 0, 2) == "__" ? string(name, 2) : name;
-    Symbol sym = symbols.create(name2);
     v->type = tPrimOp;
     v->primOp = new PrimOp(primOp, arity, sym);
     staticBaseEnv.vars[symbols.create(name)] = baseEnvDispl;
@@ -736,7 +781,7 @@ Value * ExprPath::maybeThunk(EvalState & state, Env & env)
 }
 
 
-void EvalState::evalFile(const Path & path_, Value & v)
+void EvalState::evalFile(const Path & path_, Value & v, bool mustBeTrivial)
 {
     auto path = checkSourcePath(path_);
 
@@ -765,6 +810,11 @@ void EvalState::evalFile(const Path & path_, Value & v)
     fileParseCache[path2] = e;
 
     try {
+        // Enforce that 'flake.nix' is a direct attrset, not a
+        // computation.
+        if (mustBeTrivial &&
+            !(dynamic_cast<ExprAttrs *>(e)))
+            throw Error("file '%s' must be an attribute set", path);
         eval(e, v);
     } catch (Error & e) {
         addErrorPrefix(e, "while evaluating the file '%1%':\n", path2);
@@ -1256,7 +1306,7 @@ void ExprWith::eval(EvalState & state, Env & env, Value & v)
 
 void ExprIf::eval(EvalState & state, Env & env, Value & v)
 {
-    (state.evalBool(env, cond) ? then : else_)->eval(state, env, v);
+    (state.evalBool(env, cond, pos) ? then : else_)->eval(state, env, v);
 }
 
 
@@ -1502,7 +1552,7 @@ NixFloat EvalState::forceFloat(Value & v, const Pos & pos)
 
 bool EvalState::forceBool(Value & v, const Pos & pos)
 {
-    forceValue(v);
+    forceValue(v, pos);
     if (v.type != tBool)
         throwTypeError("value is %1% while a Boolean was expected, at %2%", v, pos);
     return v.boolean;
@@ -1517,7 +1567,7 @@ bool EvalState::isFunctor(Value & fun)
 
 void EvalState::forceFunction(Value & v, const Pos & pos)
 {
-    forceValue(v);
+    forceValue(v, pos);
     if (v.type != tLambda && v.type != tPrimOp && v.type != tPrimOpApp && !isFunctor(v))
         throwTypeError("value is %1% while a function was expected, at %2%", v, pos);
 }
@@ -1594,7 +1644,7 @@ std::optional<string> EvalState::tryAttrsToString(const Pos & pos, Value & v,
 string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context,
     bool coerceMore, bool copyToStore)
 {
-    forceValue(v);
+    forceValue(v, pos);
 
     string s;
 

src/libexpr/eval.hh

@@ -4,12 +4,13 @@
 #include "value.hh"
 #include "nixexpr.hh"
 #include "symbol-table.hh"
-#include "hash.hh"
 #include "config.hh"
 
+#include <regex>
 #include <map>
 #include <optional>
 #include <unordered_map>
+#include <mutex>
 
 
 namespace nix {
@@ -73,7 +74,8 @@ public:
         sSystem, sOverrides, sOutputs, sOutputName, sIgnoreNulls,
         sFile, sLine, sColumn, sFunctor, sToString,
         sRight, sWrong, sStructuredAttrs, sBuilder, sArgs,
-        sOutputHash, sOutputHashAlgo, sOutputHashMode;
+        sOutputHash, sOutputHashAlgo, sOutputHashMode,
+        sDescription, sSelf, sEpsilon, sRecurseForDerivations;
     Symbol sDerivationNix;
 
     /* If set, force copying files to the Nix store even if they
@@ -88,6 +90,7 @@ public:
 
     const ref<Store> store;
 
+
 private:
     SrcToStore srcToStore;
 
@@ -114,6 +117,9 @@ private:
     /* Cache used by checkSourcePath(). */
     std::unordered_map<Path, Path> resolvedPaths;
 
+    /* Cache used by prim_match(). */
+    std::unordered_map<std::string, std::regex> regexCache;
+
 public:
 
     EvalState(const Strings & _searchPath, ref<Store> store);
@@ -141,14 +147,15 @@ public:
     Expr * parseExprFromFile(const Path & path, StaticEnv & staticEnv);
 
     /* Parse a Nix expression from the specified string. */
-    Expr * parseExprFromString(const string & s, const Path & basePath, StaticEnv & staticEnv);
-    Expr * parseExprFromString(const string & s, const Path & basePath);
+    Expr * parseExprFromString(std::string_view s, const Path & basePath, StaticEnv & staticEnv);
+    Expr * parseExprFromString(std::string_view s, const Path & basePath);
 
     Expr * parseStdin();
 
     /* Evaluate an expression read from the given file to normal
-       form. */
-    void evalFile(const Path & path, Value & v);
+       form. Optionally enforce that the top-level expression is
+       trivial (i.e. doesn't require arbitrary computation). */
+    void evalFile(const Path & path, Value & v, bool mustBeTrivial = false);
 
     void resetFileCache();
 
@@ -268,6 +275,7 @@ public:
     Env & allocEnv(size_t size);
 
     Value * allocAttr(Value & vAttrs, const Symbol & name);
+    Value * allocAttr(Value & vAttrs, const std::string & name);
 
     Bindings * allocBindings(size_t capacity);
 
@@ -314,10 +322,12 @@ private:
     friend struct ExprOpConcatLists;
     friend struct ExprSelect;
     friend void prim_getAttr(EvalState & state, const Pos & pos, Value * * args, Value & v);
+    friend void prim_match(EvalState & state, const Pos & pos, Value * * args, Value & v);
 };
 
 
 /* Return a string representing the type of the value `v'. */
+string showType(ValueType type);
 string showType(const Value & v);
 
 /* Decode a context string ‘!<name>!<path>’ into a pair <path,
@@ -362,7 +372,7 @@ struct EvalSettings : Config
         "Prefixes of URIs that builtin functions such as fetchurl and fetchGit are allowed to fetch."};
 
     Setting<bool> traceFunctionCalls{this, false, "trace-function-calls",
-        "Emit log messages for each function entry and exit at the 'vomit' log level (-vvvv)"};
+        "Emit log messages for each function entry and exit at the 'vomit' log level (-vvvv)."};
 };
 
 extern EvalSettings evalSettings;

src/libexpr/flake/call-flake.nix

@@ -0,0 +1,29 @@
+lockFileStr: rootSrc: rootSubdir:
+
+let
+
+  lockFile = builtins.fromJSON lockFileStr;
+
+  allNodes =
+    builtins.mapAttrs
+      (key: node:
+        let
+          sourceInfo =
+            if key == lockFile.root
+            then rootSrc
+            else fetchTree ({ inherit (node.info) narHash; } // removeAttrs node.locked ["dir"]);
+          subdir = if key == lockFile.root then rootSubdir else node.locked.dir or "";
+          flake = import (sourceInfo + (if subdir != "" then "/" else "") + subdir + "/flake.nix");
+          inputs = builtins.mapAttrs (inputName: key: allNodes.${key}) (node.inputs or {});
+          outputs = flake.outputs (inputs // { self = result; });
+          result = outputs // sourceInfo // { inherit inputs; inherit outputs; inherit sourceInfo; };
+        in
+          if node.flake or true then
+            assert builtins.isFunction flake.outputs;
+            result
+          else
+            sourceInfo
+      )
+      lockFile.nodes;
+
+in allNodes.${lockFile.root}

src/libexpr/flake/flake.cc

@@ -0,0 +1,660 @@
+#include "flake.hh"
+#include "lockfile.hh"
+#include "primops.hh"
+#include "eval-inline.hh"
+#include "store-api.hh"
+#include "fetchers.hh"
+#include "finally.hh"
+
+namespace nix {
+
+using namespace flake;
+
+namespace flake {
+
+/* If 'allowLookup' is true, then resolve 'flakeRef' using the
+   registries. */
+static FlakeRef maybeLookupFlake(
+    ref<Store> store,
+    const FlakeRef & flakeRef,
+    bool allowLookup)
+{
+    if (!flakeRef.input->isDirect()) {
+        if (allowLookup)
+            return flakeRef.resolve(store);
+        else
+            throw Error("'%s' is an indirect flake reference, but registry lookups are not allowed", flakeRef);
+    } else
+        return flakeRef;
+}
+
+typedef std::vector<std::pair<FlakeRef, FlakeRef>> FlakeCache;
+
+static FlakeRef lookupInFlakeCache(
+    const FlakeCache & flakeCache,
+    const FlakeRef & flakeRef)
+{
+    // FIXME: inefficient.
+    for (auto & i : flakeCache) {
+        if (flakeRef == i.first) {
+            debug("mapping '%s' to previously seen input '%s' -> '%s",
+                flakeRef, i.first, i.second);
+            return i.second;
+        }
+    }
+
+    return flakeRef;
+}
+
+static std::tuple<fetchers::Tree, FlakeRef, FlakeRef> fetchOrSubstituteTree(
+    EvalState & state,
+    const FlakeRef & originalRef,
+    std::optional<TreeInfo> treeInfo,
+    bool allowLookup,
+    FlakeCache & flakeCache)
+{
+    /* The tree may already be in the Nix store, or it could be
+       substituted (which is often faster than fetching from the
+       original source). So check that. */
+    if (treeInfo && originalRef.input->isDirect() && originalRef.input->isImmutable()) {
+        try {
+            auto storePath = treeInfo->computeStorePath(*state.store);
+
+            state.store->ensurePath(storePath);
+
+            debug("using substituted/cached input '%s' in '%s'",
+                originalRef, state.store->printStorePath(storePath));
+
+            auto actualPath = state.store->toRealPath(storePath);
+
+            if (state.allowedPaths)
+                state.allowedPaths->insert(actualPath);
+
+            return {
+                Tree {
+                    .actualPath = actualPath,
+                    .storePath = std::move(storePath),
+                    .info = *treeInfo,
+                },
+                originalRef,
+                originalRef
+            };
+        } catch (Error & e) {
+            debug("substitution of input '%s' failed: %s", originalRef, e.what());
+        }
+    }
+
+    auto resolvedRef = lookupInFlakeCache(flakeCache,
+        maybeLookupFlake(state.store,
+            lookupInFlakeCache(flakeCache, originalRef), allowLookup));
+
+    auto [tree, lockedRef] = resolvedRef.fetchTree(state.store);
+
+    debug("got tree '%s' from '%s'",
+        state.store->printStorePath(tree.storePath), lockedRef);
+
+    flakeCache.push_back({originalRef, lockedRef});
+    flakeCache.push_back({resolvedRef, lockedRef});
+
+    if (state.allowedPaths)
+        state.allowedPaths->insert(tree.actualPath);
+
+    if (treeInfo)
+        assert(tree.storePath == treeInfo->computeStorePath(*state.store));
+
+    return {std::move(tree), resolvedRef, lockedRef};
+}
+
+static void expectType(EvalState & state, ValueType type,
+    Value & value, const Pos & pos)
+{
+    if (value.type == tThunk && value.isTrivial())
+        state.forceValue(value, pos);
+    if (value.type != type)
+        throw Error("expected %s but got %s at %s",
+            showType(type), showType(value.type), pos);
+}
+
+static std::map<FlakeId, FlakeInput> parseFlakeInputs(
+    EvalState & state, Value * value, const Pos & pos);
+
+static FlakeInput parseFlakeInput(EvalState & state,
+    const std::string & inputName, Value * value, const Pos & pos)
+{
+    expectType(state, tAttrs, *value, pos);
+
+    FlakeInput input {
+        .ref = FlakeRef::fromAttrs({{"type", "indirect"}, {"id", inputName}})
+    };
+
+    auto sInputs = state.symbols.create("inputs");
+    auto sUrl = state.symbols.create("url");
+    auto sFlake = state.symbols.create("flake");
+    auto sFollows = state.symbols.create("follows");
+
+    fetchers::Attrs attrs;
+    std::optional<std::string> url;
+
+    for (nix::Attr attr : *(value->attrs)) {
+        try {
+            if (attr.name == sUrl) {
+                expectType(state, tString, *attr.value, *attr.pos);
+                url = attr.value->string.s;
+                attrs.emplace("url", *url);
+            } else if (attr.name == sFlake) {
+                expectType(state, tBool, *attr.value, *attr.pos);
+                input.isFlake = attr.value->boolean;
+            } else if (attr.name == sInputs) {
+                input.overrides = parseFlakeInputs(state, attr.value, *attr.pos);
+            } else if (attr.name == sFollows) {
+                expectType(state, tString, *attr.value, *attr.pos);
+                input.follows = parseInputPath(attr.value->string.s);
+            } else {
+                state.forceValue(*attr.value);
+                if (attr.value->type == tString)
+                    attrs.emplace(attr.name, attr.value->string.s);
+                else
+                    throw TypeError("flake input attribute '%s' is %s while a string is expected",
+                        attr.name, showType(*attr.value));
+            }
+        } catch (Error & e) {
+            e.addPrefix(fmt("in flake attribute '%s' at '%s':\n", attr.name, *attr.pos));
+            throw;
+        }
+    }
+
+    if (attrs.count("type"))
+        try {
+            input.ref = FlakeRef::fromAttrs(attrs);
+        } catch (Error & e) {
+            e.addPrefix(fmt("in flake input at '%s':\n", pos));
+            throw;
+        }
+    else {
+        attrs.erase("url");
+        if (!attrs.empty())
+            throw Error("unexpected flake input attribute '%s', at %s", attrs.begin()->first, pos);
+        if (url)
+            input.ref = parseFlakeRef(*url, {}, true);
+    }
+
+    return input;
+}
+
+static std::map<FlakeId, FlakeInput> parseFlakeInputs(
+    EvalState & state, Value * value, const Pos & pos)
+{
+    std::map<FlakeId, FlakeInput> inputs;
+
+    expectType(state, tAttrs, *value, pos);
+
+    for (nix::Attr & inputAttr : *(*value).attrs) {
+        inputs.emplace(inputAttr.name,
+            parseFlakeInput(state,
+                inputAttr.name,
+                inputAttr.value,
+                *inputAttr.pos));
+    }
+
+    return inputs;
+}
+
+static Flake getFlake(
+    EvalState & state,
+    const FlakeRef & originalRef,
+    std::optional<TreeInfo> treeInfo,
+    bool allowLookup,
+    FlakeCache & flakeCache)
+{
+    auto [sourceInfo, resolvedRef, lockedRef] = fetchOrSubstituteTree(
+        state, originalRef, treeInfo, allowLookup, flakeCache);
+
+    // Guard against symlink attacks.
+    auto flakeFile = canonPath(sourceInfo.actualPath + "/" + lockedRef.subdir + "/flake.nix");
+    if (!isInDir(flakeFile, sourceInfo.actualPath))
+        throw Error("'flake.nix' file of flake '%s' escapes from '%s'",
+            lockedRef, state.store->printStorePath(sourceInfo.storePath));
+
+    Flake flake {
+        .originalRef = originalRef,
+        .resolvedRef = resolvedRef,
+        .lockedRef = lockedRef,
+        .sourceInfo = std::make_shared<fetchers::Tree>(std::move(sourceInfo))
+    };
+
+    if (!pathExists(flakeFile))
+        throw Error("source tree referenced by '%s' does not contain a '%s/flake.nix' file", lockedRef, lockedRef.subdir);
+
+    Value vInfo;
+    state.evalFile(flakeFile, vInfo, true); // FIXME: symlink attack
+
+    expectType(state, tAttrs, vInfo, Pos(state.symbols.create(flakeFile), 0, 0));
+
+    auto sEdition = state.symbols.create("edition"); // FIXME: remove soon
+
+    if (vInfo.attrs->get(sEdition))
+        warn("flake '%s' has deprecated attribute 'edition'", lockedRef);
+
+    if (auto description = vInfo.attrs->get(state.sDescription)) {
+        expectType(state, tString, *description->value, *description->pos);
+        flake.description = description->value->string.s;
+    }
+
+    auto sInputs = state.symbols.create("inputs");
+
+    if (auto inputs = vInfo.attrs->get(sInputs))
+        flake.inputs = parseFlakeInputs(state, inputs->value, *inputs->pos);
+
+    auto sOutputs = state.symbols.create("outputs");
+
+    if (auto outputs = vInfo.attrs->get(sOutputs)) {
+        expectType(state, tLambda, *outputs->value, *outputs->pos);
+        flake.vOutputs = allocRootValue(outputs->value);
+
+        if ((*flake.vOutputs)->lambda.fun->matchAttrs) {
+            for (auto & formal : (*flake.vOutputs)->lambda.fun->formals->formals) {
+                if (formal.name != state.sSelf)
+                    flake.inputs.emplace(formal.name, FlakeInput {
+                        .ref = parseFlakeRef(formal.name)
+                    });
+            }
+        }
+
+    } else
+        throw Error("flake '%s' lacks attribute 'outputs'", lockedRef);
+
+    for (auto & attr : *vInfo.attrs) {
+        if (attr.name != sEdition &&
+            attr.name != state.sDescription &&
+            attr.name != sInputs &&
+            attr.name != sOutputs)
+            throw Error("flake '%s' has an unsupported attribute '%s', at %s",
+                lockedRef, attr.name, *attr.pos);
+    }
+
+    return flake;
+}
+
+Flake getFlake(EvalState & state, const FlakeRef & originalRef, bool allowLookup)
+{
+    FlakeCache flakeCache;
+    return getFlake(state, originalRef, {}, allowLookup, flakeCache);
+}
+
+/* Compute an in-memory lock file for the specified top-level flake,
+   and optionally write it to file, it the flake is writable. */
+LockedFlake lockFlake(
+    EvalState & state,
+    const FlakeRef & topRef,
+    const LockFlags & lockFlags)
+{
+    settings.requireExperimentalFeature("flakes");
+
+    FlakeCache flakeCache;
+
+    auto flake = getFlake(state, topRef, {}, lockFlags.useRegistries, flakeCache);
+
+    // FIXME: symlink attack
+    auto oldLockFile = LockFile::read(
+        flake.sourceInfo->actualPath + "/" + flake.lockedRef.subdir + "/flake.lock");
+
+    debug("old lock file: %s", oldLockFile);
+
+    // FIXME: check whether all overrides are used.
+    std::map<InputPath, FlakeInput> overrides;
+    std::set<InputPath> overridesUsed, updatesUsed;
+
+    for (auto & i : lockFlags.inputOverrides)
+        overrides.insert_or_assign(i.first, FlakeInput { .ref = i.second });
+
+    LockFile newLockFile;
+
+    std::vector<FlakeRef> parents;
+    std::map<InputPath, InputPath> follows;
+
+    std::function<void(
+        const FlakeInputs & flakeInputs,
+        std::shared_ptr<Node> node,
+        const InputPath & inputPathPrefix,
+        std::shared_ptr<const Node> oldNode)>
+        computeLocks;
+
+    computeLocks = [&](
+        const FlakeInputs & flakeInputs,
+        std::shared_ptr<Node> node,
+        const InputPath & inputPathPrefix,
+        std::shared_ptr<const Node> oldNode)
+    {
+        debug("computing lock file node '%s'", concatStringsSep("/", inputPathPrefix));
+
+        /* Get the overrides (i.e. attributes of the form
+           'inputs.nixops.inputs.nixpkgs.url = ...'). */
+        // FIXME: check this
+        for (auto & [id, input] : flake.inputs) {
+            for (auto & [idOverride, inputOverride] : input.overrides) {
+                auto inputPath(inputPathPrefix);
+                inputPath.push_back(id);
+                inputPath.push_back(idOverride);
+                overrides.insert_or_assign(inputPath, inputOverride);
+            }
+        }
+
+        /* Go over the flake inputs, resolve/fetch them if
+           necessary (i.e. if they're new or the flakeref changed
+           from what's in the lock file). */
+        for (auto & [id, input2] : flakeInputs) {
+            auto inputPath(inputPathPrefix);
+            inputPath.push_back(id);
+            auto inputPathS = concatStringsSep("/", inputPath);
+            debug("computing input '%s'", concatStringsSep("/", inputPath));
+
+            /* Do we have an override for this input from one of the
+               ancestors? */
+            auto i = overrides.find(inputPath);
+            bool hasOverride = i != overrides.end();
+            if (hasOverride) overridesUsed.insert(inputPath);
+            auto & input = hasOverride ? i->second : input2;
+
+            /* Resolve 'follows' later (since it may refer to an input
+               path we haven't processed yet. */
+            if (input.follows) {
+                if (hasOverride)
+                    /* 'follows' from an override is relative to the
+                       root of the graph. */
+                    follows.insert_or_assign(inputPath, *input.follows);
+                else {
+                    /* Otherwise, it's relative to the current flake. */
+                    InputPath path(inputPathPrefix);
+                    for (auto & i : *input.follows) path.push_back(i);
+                    follows.insert_or_assign(inputPath, path);
+                }
+                continue;
+            }
+
+            /* Do we have an entry in the existing lock file? And we
+               don't have a --update-input flag for this input? */
+            std::shared_ptr<const LockedNode> oldLock;
+
+            updatesUsed.insert(inputPath);
+
+            if (oldNode && !lockFlags.inputUpdates.count(inputPath)) {
+                auto oldLockIt = oldNode->inputs.find(id);
+                if (oldLockIt != oldNode->inputs.end())
+                    oldLock = std::dynamic_pointer_cast<const LockedNode>(oldLockIt->second);
+            }
+
+            if (oldLock
+                && oldLock->originalRef == input.ref
+                && !hasOverride)
+            {
+                debug("keeping existing input '%s'", inputPathS);
+
+                /* Copy the input from the old lock since its flakeref
+                   didn't change and there is no override from a
+                   higher level flake. */
+                auto childNode = std::make_shared<LockedNode>(
+                    oldLock->lockedRef, oldLock->originalRef, oldLock->info, oldLock->isFlake);
+
+                node->inputs.insert_or_assign(id, childNode);
+
+                /* If we have an --update-input flag for an input
+                   of this input, then we must fetch the flake to
+                   to update it. */
+                auto lb = lockFlags.inputUpdates.lower_bound(inputPath);
+
+                auto hasChildUpdate =
+                    lb != lockFlags.inputUpdates.end()
+                    && lb->size() > inputPath.size()
+                    && std::equal(inputPath.begin(), inputPath.end(), lb->begin());
+
+                if (hasChildUpdate) {
+                    auto inputFlake = getFlake(
+                        state, oldLock->lockedRef, oldLock->info, false, flakeCache);
+                    computeLocks(inputFlake.inputs, childNode, inputPath, oldLock);
+                } else {
+                    /* No need to fetch this flake, we can be
+                       lazy. However there may be new overrides on the
+                       inputs of this flake, so we need to check
+                       those. */
+                    FlakeInputs fakeInputs;
+
+                    for (auto & i : oldLock->inputs) {
+                        auto lockedNode = std::dynamic_pointer_cast<LockedNode>(i.second);
+                        // Note: this node is not locked in case
+                        // of a circular reference back to the root.
+                        if (lockedNode)
+                            fakeInputs.emplace(i.first, FlakeInput {
+                                .ref = lockedNode->originalRef
+                            });
+                        else {
+                            InputPath path(inputPath);
+                            path.push_back(i.first);
+                            follows.insert_or_assign(path, InputPath());
+                        }
+                    }
+
+                    computeLocks(fakeInputs, childNode, inputPath, oldLock);
+                }
+
+            } else {
+                /* We need to create a new lock file entry. So fetch
+                   this input. */
+
+                if (!lockFlags.allowMutable && !input.ref.input->isImmutable())
+                    throw Error("cannot update flake input '%s' in pure mode", inputPathS);
+
+                if (input.isFlake) {
+                    auto inputFlake = getFlake(state, input.ref, {}, lockFlags.useRegistries, flakeCache);
+
+                    /* Note: in case of an --override-input, we use
+                       the *original* ref (input2.ref) for the
+                       "original" field, rather than the
+                       override. This ensures that the override isn't
+                       nuked the next time we update the lock
+                       file. That is, overrides are sticky unless you
+                       use --no-write-lock-file. */
+                    auto childNode = std::make_shared<LockedNode>(
+                        inputFlake.lockedRef, input2.ref, inputFlake.sourceInfo->info);
+
+                    node->inputs.insert_or_assign(id, childNode);
+
+                    /* Guard against circular flake imports. */
+                    for (auto & parent : parents)
+                        if (parent == input.ref)
+                            throw Error("found circular import of flake '%s'", parent);
+                    parents.push_back(input.ref);
+                    Finally cleanup([&]() { parents.pop_back(); });
+
+                    /* Recursively process the inputs of this
+                       flake. Also, unless we already have this flake
+                       in the top-level lock file, use this flake's
+                       own lock file. */
+                    computeLocks(
+                        inputFlake.inputs, childNode, inputPath,
+                        oldLock
+                        ? std::dynamic_pointer_cast<const Node>(oldLock)
+                        : LockFile::read(
+                            inputFlake.sourceInfo->actualPath + "/" + inputFlake.lockedRef.subdir + "/flake.lock").root);
+                }
+
+                else {
+                    auto [sourceInfo, resolvedRef, lockedRef] = fetchOrSubstituteTree(
+                        state, input.ref, {}, lockFlags.useRegistries, flakeCache);
+                    node->inputs.insert_or_assign(id,
+                        std::make_shared<LockedNode>(lockedRef, input.ref, sourceInfo.info, false));
+                }
+            }
+        }
+    };
+
+    computeLocks(
+        flake.inputs, newLockFile.root, {},
+        lockFlags.recreateLockFile ? nullptr : oldLockFile.root);
+
+    /* Insert edges for 'follows' overrides. */
+    for (auto & [from, to] : follows) {
+        debug("adding 'follows' node from '%s' to '%s'",
+            concatStringsSep("/", from),
+            concatStringsSep("/", to));
+
+        assert(!from.empty());
+
+        InputPath fromParent(from);
+        fromParent.pop_back();
+
+        auto fromParentNode = newLockFile.root->findInput(fromParent);
+        assert(fromParentNode);
+
+        auto toNode = newLockFile.root->findInput(to);
+        if (!toNode)
+            throw Error("flake input '%s' follows non-existent flake input '%s'",
+                concatStringsSep("/", from),
+                concatStringsSep("/", to));
+
+        fromParentNode->inputs.insert_or_assign(from.back(), toNode);
+    }
+
+    for (auto & i : lockFlags.inputOverrides)
+        if (!overridesUsed.count(i.first))
+            warn("the flag '--override-input %s %s' does not match any input",
+                concatStringsSep("/", i.first), i.second);
+
+    for (auto & i : lockFlags.inputUpdates)
+        if (!updatesUsed.count(i))
+            warn("the flag '--update-input %s' does not match any input", concatStringsSep("/", i));
+
+    debug("new lock file: %s", newLockFile);
+
+    /* Check whether we need to / can write the new lock file. */
+    if (!(newLockFile == oldLockFile)) {
+
+        auto diff = diffLockFiles(oldLockFile, newLockFile);
+
+        if (!(oldLockFile == LockFile()))
+            printInfo("inputs of flake '%s' changed:\n%s", topRef, chomp(diff));
+
+        if (lockFlags.writeLockFile) {
+            if (auto sourcePath = topRef.input->getSourcePath()) {
+                if (!newLockFile.isImmutable()) {
+                    if (settings.warnDirty)
+                        warn("will not write lock file of flake '%s' because it has a mutable input", topRef);
+                } else {
+                    if (!lockFlags.updateLockFile)
+                        throw Error("flake '%s' requires lock file changes but they're not allowed due to '--no-update-lock-file'", topRef);
+
+                    auto relPath = (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock";
+
+                    auto path = *sourcePath + "/" + relPath;
+
+                    bool lockFileExists = pathExists(path);
+
+                    if (lockFileExists)
+                        warn("updating lock file '%s'", path);
+                    else
+                        warn("creating lock file '%s'", path);
+
+                    newLockFile.write(path);
+
+                    topRef.input->markChangedFile(
+                        (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock",
+                        lockFlags.commitLockFile
+                        ? std::optional<std::string>(fmt("%s: %s\n\nFlake input changes:\n\n%s",
+                                relPath, lockFileExists ? "Update" : "Add", diff))
+                        : std::nullopt);
+
+                    /* Rewriting the lockfile changed the top-level
+                       repo, so we should re-read it. FIXME: we could
+                       also just clear the 'rev' field... */
+                    auto prevLockedRef = flake.lockedRef;
+                    FlakeCache dummyCache;
+                    flake = getFlake(state, topRef, {}, lockFlags.useRegistries, dummyCache);
+
+                    if (lockFlags.commitLockFile &&
+                        flake.lockedRef.input->getRev() &&
+                        prevLockedRef.input->getRev() != flake.lockedRef.input->getRev())
+                        warn("committed new revision '%s'", flake.lockedRef.input->getRev()->gitRev());
+
+                    /* Make sure that we picked up the change,
+                       i.e. the tree should usually be dirty
+                       now. Corner case: we could have reverted from a
+                       dirty to a clean tree! */
+                    if (flake.lockedRef.input == prevLockedRef.input
+                        && !flake.lockedRef.input->isImmutable())
+                        throw Error("'%s' did not change after I updated its 'flake.lock' file; is 'flake.lock' under version control?", flake.originalRef);
+                }
+            } else
+                throw Error("cannot write modified lock file of flake '%s' (use '--no-write-lock-file' to ignore)", topRef);
+        } else
+            warn("not writing modified lock file of flake '%s'", topRef);
+    }
+
+    return LockedFlake { .flake = std::move(flake), .lockFile = std::move(newLockFile) };
+}
+
+void callFlake(EvalState & state,
+    const LockedFlake & lockedFlake,
+    Value & vRes)
+{
+    auto vLocks = state.allocValue();
+    auto vRootSrc = state.allocValue();
+    auto vRootSubdir = state.allocValue();
+    auto vTmp1 = state.allocValue();
+    auto vTmp2 = state.allocValue();
+
+    mkString(*vLocks, lockedFlake.lockFile.to_string());
+
+    emitTreeAttrs(state, *lockedFlake.flake.sourceInfo, lockedFlake.flake.lockedRef.input, *vRootSrc);
+
+    mkString(*vRootSubdir, lockedFlake.flake.lockedRef.subdir);
+
+    static RootValue vCallFlake = nullptr;
+
+    if (!vCallFlake) {
+        vCallFlake = allocRootValue(state.allocValue());
+        state.eval(state.parseExprFromString(
+            #include "call-flake.nix.gen.hh"
+            , "/"), **vCallFlake);
+    }
+
+    state.callFunction(**vCallFlake, *vLocks, *vTmp1, noPos);
+    state.callFunction(*vTmp1, *vRootSrc, *vTmp2, noPos);
+    state.callFunction(*vTmp2, *vRootSubdir, vRes, noPos);
+}
+
+static void prim_getFlake(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    auto flakeRefS = state.forceStringNoCtx(*args[0], pos);
+    auto flakeRef = parseFlakeRef(flakeRefS, {}, true);
+    if (evalSettings.pureEval && !flakeRef.input->isImmutable())
+        throw Error("cannot call 'getFlake' on mutable flake reference '%s', at %s (use --impure to override)", flakeRefS, pos);
+
+    callFlake(state,
+        lockFlake(state, flakeRef,
+            LockFlags {
+                .updateLockFile = false,
+                .useRegistries = !evalSettings.pureEval,
+                .allowMutable  = !evalSettings.pureEval,
+            }),
+        v);
+}
+
+static RegisterPrimOp r2("getFlake", 1, prim_getFlake);
+
+}
+
+Fingerprint LockedFlake::getFingerprint() const
+{
+    // FIXME: as an optimization, if the flake contains a lock file
+    // and we haven't changed it, then it's sufficient to use
+    // flake.sourceInfo.storePath for the fingerprint.
+    return hashString(htSHA256,
+        fmt("%s;%d;%d;%s",
+            flake.sourceInfo->storePath.to_string(),
+            flake.sourceInfo->info.revCount.value_or(0),
+            flake.sourceInfo->info.lastModified.value_or(0),
+            lockFile));
+}
+
+Flake::~Flake() { }
+
+}

src/libexpr/flake/flake.hh

@@ -0,0 +1,110 @@
+#pragma once
+
+#include "types.hh"
+#include "flakeref.hh"
+#include "lockfile.hh"
+#include "value.hh"
+
+namespace nix {
+
+class EvalState;
+
+namespace fetchers { struct Tree; }
+
+namespace flake {
+
+struct FlakeInput;
+
+typedef std::map<FlakeId, FlakeInput> FlakeInputs;
+
+struct FlakeInput
+{
+    FlakeRef ref;
+    bool isFlake = true;
+    std::optional<InputPath> follows;
+    FlakeInputs overrides;
+};
+
+struct Flake
+{
+    FlakeRef originalRef;
+    FlakeRef resolvedRef;
+    FlakeRef lockedRef;
+    std::optional<std::string> description;
+    std::shared_ptr<const fetchers::Tree> sourceInfo;
+    FlakeInputs inputs;
+    RootValue vOutputs;
+    ~Flake();
+};
+
+Flake getFlake(EvalState & state, const FlakeRef & flakeRef, bool allowLookup);
+
+/* Fingerprint of a locked flake; used as a cache key. */
+typedef Hash Fingerprint;
+
+struct LockedFlake
+{
+    Flake flake;
+    LockFile lockFile;
+
+    Fingerprint getFingerprint() const;
+};
+
+struct LockFlags
+{
+    /* Whether to ignore the existing lock file, creating a new one
+       from scratch. */
+    bool recreateLockFile = false;
+
+    /* Whether to update the lock file at all. If set to false, if any
+       change to the lock file is needed (e.g. when an input has been
+       added to flake.nix), you get a fatal error. */
+    bool updateLockFile = true;
+
+    /* Whether to write the lock file to disk. If set to true, if the
+       any changes to the lock file are needed and the flake is not
+       writable (i.e. is not a local Git working tree or similar), you
+       get a fatal error. If set to false, Nix will use the modified
+       lock file in memory only, without writing it to disk. */
+    bool writeLockFile = true;
+
+    /* Whether to use the registries to lookup indirect flake
+       references like 'nixpkgs'. */
+    bool useRegistries = true;
+
+    /* Whether mutable flake references (i.e. those without a Git
+       revision or similar) without a corresponding lock are
+       allowed. Mutable flake references with a lock are always
+       allowed. */
+    bool allowMutable = true;
+
+    /* Whether to commit changes to flake.lock. */
+    bool commitLockFile = false;
+
+    /* Flake inputs to be overriden. */
+    std::map<InputPath, FlakeRef> inputOverrides;
+
+    /* Flake inputs to be updated. This means that any existing lock
+       for those inputs will be ignored. */
+    std::set<InputPath> inputUpdates;
+};
+
+LockedFlake lockFlake(
+    EvalState & state,
+    const FlakeRef & flakeRef,
+    const LockFlags & lockFlags);
+
+void callFlake(
+    EvalState & state,
+    const LockedFlake & lockedFlake,
+    Value & v);
+
+}
+
+void emitTreeAttrs(
+    EvalState & state,
+    const fetchers::Tree & tree,
+    std::shared_ptr<const fetchers::Input> input,
+    Value & v);
+
+}

src/libexpr/flake/flakeref.cc

@@ -0,0 +1,196 @@
+#include "flakeref.hh"
+#include "store-api.hh"
+#include "url.hh"
+#include "fetchers.hh"
+#include "registry.hh"
+
+namespace nix {
+
+#if 0
+// 'dir' path elements cannot start with a '.'. We also reject
+// potentially dangerous characters like ';'.
+const static std::string subDirElemRegex = "(?:[a-zA-Z0-9_-]+[a-zA-Z0-9._-]*)";
+const static std::string subDirRegex = subDirElemRegex + "(?:/" + subDirElemRegex + ")*";
+#endif
+
+std::string FlakeRef::to_string() const
+{
+    auto url = input->toURL();
+    if (subdir != "")
+        url.query.insert_or_assign("dir", subdir);
+    return url.to_string();
+}
+
+fetchers::Attrs FlakeRef::toAttrs() const
+{
+    auto attrs = input->toAttrs();
+    if (subdir != "")
+        attrs.emplace("dir", subdir);
+    return attrs;
+}
+
+std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef)
+{
+    str << flakeRef.to_string();
+    return str;
+}
+
+bool FlakeRef::operator ==(const FlakeRef & other) const
+{
+    return *input == *other.input && subdir == other.subdir;
+}
+
+FlakeRef FlakeRef::resolve(ref<Store> store) const
+{
+    auto [input2, extraAttrs] = lookupInRegistries(store, input);
+    return FlakeRef(input2, fetchers::maybeGetStrAttr(extraAttrs, "dir").value_or(subdir));
+}
+
+FlakeRef parseFlakeRef(
+    const std::string & url, const std::optional<Path> & baseDir, bool allowMissing)
+{
+    auto [flakeRef, fragment] = parseFlakeRefWithFragment(url, baseDir, allowMissing);
+    if (fragment != "")
+        throw Error("unexpected fragment '%s' in flake reference '%s'", fragment, url);
+    return flakeRef;
+}
+
+std::optional<FlakeRef> maybeParseFlakeRef(
+    const std::string & url, const std::optional<Path> & baseDir)
+{
+    try {
+        return parseFlakeRef(url, baseDir);
+    } catch (Error &) {
+        return {};
+    }
+}
+
+std::pair<FlakeRef, std::string> parseFlakeRefWithFragment(
+    const std::string & url, const std::optional<Path> & baseDir, bool allowMissing)
+{
+    using namespace fetchers;
+
+    static std::string fnRegex = "[0-9a-zA-Z-._~!$&'\"()*+,;=]+";
+
+    static std::regex pathUrlRegex(
+        "(/?" + fnRegex + "(?:/" + fnRegex + ")*/?)"
+        + "(?:\\?(" + queryRegex + "))?"
+        + "(?:#(" + queryRegex + "))?",
+        std::regex::ECMAScript);
+
+    static std::regex flakeRegex(
+        "((" + flakeIdRegexS + ")(?:/(?:" + refAndOrRevRegex + "))?)"
+        + "(?:#(" + queryRegex + "))?",
+        std::regex::ECMAScript);
+
+    std::smatch match;
+
+    /* Check if 'url' is a flake ID. This is an abbreviated syntax for
+       'flake:<flake-id>?ref=<ref>&rev=<rev>'. */
+
+    if (std::regex_match(url, match, flakeRegex)) {
+        auto parsedURL = ParsedURL{
+            .url = url,
+            .base = "flake:" + std::string(match[1]),
+            .scheme = "flake",
+            .authority = "",
+            .path = match[1],
+        };
+
+        return std::make_pair(
+            FlakeRef(inputFromURL(parsedURL), ""),
+            percentDecode(std::string(match[6])));
+    }
+
+    /* Check if 'url' is a path (either absolute or relative to
+       'baseDir'). If so, search upward to the root of the repo
+       (i.e. the directory containing .git). */
+
+    else if (std::regex_match(url, match, pathUrlRegex)) {
+        std::string path = match[1];
+        if (!baseDir && !hasPrefix(path, "/"))
+            throw BadURL("flake reference '%s' is not an absolute path", url);
+        path = absPath(path, baseDir, true);
+
+        if (!S_ISDIR(lstat(path).st_mode))
+            throw BadURL("path '%s' is not a flake (because it's not a directory)", path);
+
+        if (!allowMissing && !pathExists(path + "/flake.nix"))
+            throw BadURL("path '%s' is not a flake (because it doesn't contain a 'flake.nix' file)", path);
+
+        auto fragment = percentDecode(std::string(match[3]));
+
+        auto flakeRoot = path;
+        std::string subdir;
+
+        while (flakeRoot != "/") {
+            if (pathExists(flakeRoot + "/.git")) {
+                auto base = std::string("git+file://") + flakeRoot;
+
+                auto parsedURL = ParsedURL{
+                    .url = base, // FIXME
+                    .base = base,
+                    .scheme = "git+file",
+                    .authority = "",
+                    .path = flakeRoot,
+                    .query = decodeQuery(match[2]),
+                };
+
+                if (subdir != "") {
+                    if (parsedURL.query.count("dir"))
+                        throw Error("flake URL '%s' has an inconsistent 'dir' parameter", url);
+                    parsedURL.query.insert_or_assign("dir", subdir);
+                }
+
+                return std::make_pair(
+                    FlakeRef(inputFromURL(parsedURL), get(parsedURL.query, "dir").value_or("")),
+                    fragment);
+            }
+
+            subdir = std::string(baseNameOf(flakeRoot)) + (subdir.empty() ? "" : "/" + subdir);
+            flakeRoot = dirOf(flakeRoot);
+        }
+
+        fetchers::Attrs attrs;
+        attrs.insert_or_assign("type", "path");
+        attrs.insert_or_assign("path", path);
+
+        return std::make_pair(FlakeRef(inputFromAttrs(attrs), ""), fragment);
+    }
+
+    else {
+        auto parsedURL = parseURL(url);
+        std::string fragment;
+        std::swap(fragment, parsedURL.fragment);
+        return std::make_pair(
+            FlakeRef(inputFromURL(parsedURL), get(parsedURL.query, "dir").value_or("")),
+            fragment);
+    }
+}
+
+std::optional<std::pair<FlakeRef, std::string>> maybeParseFlakeRefWithFragment(
+    const std::string & url, const std::optional<Path> & baseDir)
+{
+    try {
+        return parseFlakeRefWithFragment(url, baseDir);
+    } catch (Error & e) {
+        return {};
+    }
+}
+
+FlakeRef FlakeRef::fromAttrs(const fetchers::Attrs & attrs)
+{
+    auto attrs2(attrs);
+    attrs2.erase("dir");
+    return FlakeRef(
+        fetchers::inputFromAttrs(attrs2),
+        fetchers::maybeGetStrAttr(attrs, "dir").value_or(""));
+}
+
+std::pair<fetchers::Tree, FlakeRef> FlakeRef::fetchTree(ref<Store> store) const
+{
+    auto [tree, lockedInput] = input->fetchTree(store);
+    return {std::move(tree), FlakeRef(lockedInput, subdir)};
+}
+
+}

src/libexpr/flake/flakeref.hh

@@ -0,0 +1,55 @@
+#pragma once
+
+#include "types.hh"
+#include "hash.hh"
+#include "fetchers.hh"
+
+#include <variant>
+
+namespace nix {
+
+class Store;
+
+typedef std::string FlakeId;
+
+struct FlakeRef
+{
+    std::shared_ptr<const fetchers::Input> input;
+
+    Path subdir;
+
+    bool operator==(const FlakeRef & other) const;
+
+    FlakeRef(const std::shared_ptr<const fetchers::Input> & input, const Path & subdir)
+        : input(input), subdir(subdir)
+    {
+        assert(input);
+    }
+
+    // FIXME: change to operator <<.
+    std::string to_string() const;
+
+    fetchers::Attrs toAttrs() const;
+
+    FlakeRef resolve(ref<Store> store) const;
+
+    static FlakeRef fromAttrs(const fetchers::Attrs & attrs);
+
+    std::pair<fetchers::Tree, FlakeRef> fetchTree(ref<Store> store) const;
+};
+
+std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef);
+
+FlakeRef parseFlakeRef(
+    const std::string & url, const std::optional<Path> & baseDir = {}, bool allowMissing = false);
+
+std::optional<FlakeRef> maybeParseFlake(
+    const std::string & url, const std::optional<Path> & baseDir = {});
+
+std::pair<FlakeRef, std::string> parseFlakeRefWithFragment(
+    const std::string & url, const std::optional<Path> & baseDir = {}, bool allowMissing = false);
+
+std::optional<std::pair<FlakeRef, std::string>> maybeParseFlakeRefWithFragment(
+    const std::string & url, const std::optional<Path> & baseDir = {});
+
+}

src/libexpr/flake/lockfile.cc

@@ -0,0 +1,256 @@
+#include "lockfile.hh"
+#include "store-api.hh"
+
+#include <nlohmann/json.hpp>
+
+namespace nix::flake {
+
+FlakeRef flakeRefFromJson(const nlohmann::json & json)
+{
+    return FlakeRef::fromAttrs(jsonToAttrs(json));
+}
+
+FlakeRef getFlakeRef(
+    const nlohmann::json & json,
+    const char * attr)
+{
+    auto i = json.find(attr);
+    if (i != json.end())
+        return flakeRefFromJson(*i);
+
+    throw Error("attribute '%s' missing in lock file", attr);
+}
+
+LockedNode::LockedNode(const nlohmann::json & json)
+    : lockedRef(getFlakeRef(json, "locked"))
+    , originalRef(getFlakeRef(json, "original"))
+    , info(TreeInfo::fromJson(json))
+    , isFlake(json.find("flake") != json.end() ? (bool) json["flake"] : true)
+{
+    if (!lockedRef.input->isImmutable())
+        throw Error("lockfile contains mutable flakeref '%s'", lockedRef);
+}
+
+StorePath LockedNode::computeStorePath(Store & store) const
+{
+    return info.computeStorePath(store);
+}
+
+std::shared_ptr<Node> Node::findInput(const InputPath & path)
+{
+    auto pos = shared_from_this();
+
+    for (auto & elem : path) {
+        auto i = pos->inputs.find(elem);
+        if (i == pos->inputs.end())
+            return {};
+        pos = i->second;
+    }
+
+    return pos;
+}
+
+LockFile::LockFile(const nlohmann::json & json, const Path & path)
+{
+    auto version = json.value("version", 0);
+    if (version != 5)
+        throw Error("lock file '%s' has unsupported version %d", path, version);
+
+    std::unordered_map<std::string, std::shared_ptr<Node>> nodeMap;
+
+    std::function<void(Node & node, const nlohmann::json & jsonNode)> getInputs;
+
+    getInputs = [&](Node & node, const nlohmann::json & jsonNode)
+    {
+        if (jsonNode.find("inputs") == jsonNode.end()) return;
+        for (auto & i : jsonNode["inputs"].items()) {
+            std::string inputKey = i.value();
+            auto k = nodeMap.find(inputKey);
+            if (k == nodeMap.end()) {
+                auto jsonNode2 = json["nodes"][inputKey];
+                auto input = std::make_shared<LockedNode>(jsonNode2);
+                k = nodeMap.insert_or_assign(inputKey, input).first;
+                getInputs(*input, jsonNode2);
+            }
+            node.inputs.insert_or_assign(i.key(), k->second);
+        }
+    };
+
+    std::string rootKey = json["root"];
+    nodeMap.insert_or_assign(rootKey, root);
+    getInputs(*root, json["nodes"][rootKey]);
+}
+
+nlohmann::json LockFile::toJson() const
+{
+    nlohmann::json nodes;
+    std::unordered_map<std::shared_ptr<const Node>, std::string> nodeKeys;
+    std::unordered_set<std::string> keys;
+
+    std::function<std::string(const std::string & key, std::shared_ptr<const Node> node)> dumpNode;
+
+    dumpNode = [&](std::string key, std::shared_ptr<const Node> node) -> std::string
+    {
+        auto k = nodeKeys.find(node);
+        if (k != nodeKeys.end())
+            return k->second;
+
+        if (!keys.insert(key).second) {
+            for (int n = 2; ; ++n) {
+                auto k = fmt("%s_%d", key, n);
+                if (keys.insert(k).second) {
+                    key = k;
+                    break;
+                }
+            }
+        }
+
+        nodeKeys.insert_or_assign(node, key);
+
+        auto n = nlohmann::json::object();
+
+        if (!node->inputs.empty()) {
+            auto inputs = nlohmann::json::object();
+            for (auto & i : node->inputs)
+                inputs[i.first] = dumpNode(i.first, i.second);
+            n["inputs"] = std::move(inputs);
+        }
+
+        if (auto lockedNode = std::dynamic_pointer_cast<const LockedNode>(node)) {
+            n["original"] = fetchers::attrsToJson(lockedNode->originalRef.toAttrs());
+            n["locked"] = fetchers::attrsToJson(lockedNode->lockedRef.toAttrs());
+            n["info"] = lockedNode->info.toJson();
+            if (!lockedNode->isFlake) n["flake"] = false;
+        }
+
+        nodes[key] = std::move(n);
+
+        return key;
+    };
+
+    nlohmann::json json;
+    json["version"] = 5;
+    json["root"] = dumpNode("root", root);
+    json["nodes"] = std::move(nodes);
+
+    return json;
+}
+
+std::string LockFile::to_string() const
+{
+    return toJson().dump(2);
+}
+
+LockFile LockFile::read(const Path & path)
+{
+    if (!pathExists(path)) return LockFile();
+    return LockFile(nlohmann::json::parse(readFile(path)), path);
+}
+
+std::ostream & operator <<(std::ostream & stream, const LockFile & lockFile)
+{
+    stream << lockFile.toJson().dump(2);
+    return stream;
+}
+
+void LockFile::write(const Path & path) const
+{
+    createDirs(dirOf(path));
+    writeFile(path, fmt("%s\n", *this));
+}
+
+bool LockFile::isImmutable() const
+{
+    std::unordered_set<std::shared_ptr<const Node>> nodes;
+
+    std::function<void(std::shared_ptr<const Node> node)> visit;
+
+    visit = [&](std::shared_ptr<const Node> node)
+    {
+        if (!nodes.insert(node).second) return;
+        for (auto & i : node->inputs) visit(i.second);
+    };
+
+    visit(root);
+
+    for (auto & i : nodes) {
+        if (i == root) continue;
+        auto lockedNode = std::dynamic_pointer_cast<const LockedNode>(i);
+        if (lockedNode && !lockedNode->lockedRef.input->isImmutable()) return false;
+    }
+
+    return true;
+}
+
+bool LockFile::operator ==(const LockFile & other) const
+{
+    // FIXME: slow
+    return toJson() == other.toJson();
+}
+
+InputPath parseInputPath(std::string_view s)
+{
+    InputPath path;
+
+    for (auto & elem : tokenizeString<std::vector<std::string>>(s, "/")) {
+        if (!std::regex_match(elem, flakeIdRegex))
+            throw Error("invalid flake input path element '%s'", elem);
+        path.push_back(elem);
+    }
+
+    return path;
+}
+
+static void flattenLockFile(
+    std::shared_ptr<const Node> node,
+    const InputPath & prefix,
+    std::unordered_set<std::shared_ptr<const Node>> & done,
+    std::map<InputPath, std::shared_ptr<const LockedNode>> & res)
+{
+    if (!done.insert(node).second) return;
+
+    for (auto &[id, input] : node->inputs) {
+        auto inputPath(prefix);
+        inputPath.push_back(id);
+        if (auto lockedInput = std::dynamic_pointer_cast<const LockedNode>(input))
+            res.emplace(inputPath, lockedInput);
+        flattenLockFile(input, inputPath, done, res);
+    }
+}
+
+std::string diffLockFiles(const LockFile & oldLocks, const LockFile & newLocks)
+{
+    std::unordered_set<std::shared_ptr<const Node>> done;
+    std::map<InputPath, std::shared_ptr<const LockedNode>> oldFlat, newFlat;
+    flattenLockFile(oldLocks.root, {}, done, oldFlat);
+    done.clear();
+    flattenLockFile(newLocks.root, {}, done, newFlat);
+
+    auto i = oldFlat.begin();
+    auto j = newFlat.begin();
+    std::string res;
+
+    while (i != oldFlat.end() || j != newFlat.end()) {
+        if (j != newFlat.end() && (i == oldFlat.end() || i->first > j->first)) {
+            res += fmt("* Added '%s': '%s'\n", concatStringsSep("/", j->first), j->second->lockedRef);
+            ++j;
+        } else if (i != oldFlat.end() && (j == newFlat.end() || i->first < j->first)) {
+            res += fmt("* Removed '%s'\n", concatStringsSep("/", i->first));
+            ++i;
+        } else {
+            if (!(i->second->lockedRef == j->second->lockedRef)) {
+                assert(i->second->lockedRef.to_string() != j->second->lockedRef.to_string());
+                res += fmt("* Updated '%s': '%s' -> '%s'\n",
+                    concatStringsSep("/", i->first),
+                    i->second->lockedRef,
+                    j->second->lockedRef);
+            }
+            ++i;
+            ++j;
+        }
+    }
+
+    return res;
+}
+
+}

src/libexpr/flake/lockfile.hh

@@ -0,0 +1,77 @@
+#pragma once
+
+#include "flakeref.hh"
+
+#include <nlohmann/json_fwd.hpp>
+
+namespace nix {
+class Store;
+struct StorePath;
+}
+
+namespace nix::flake {
+
+using namespace fetchers;
+
+typedef std::vector<FlakeId> InputPath;
+
+/* A node in the lock file. It has outgoing edges to other nodes (its
+   inputs). Only the root node has this type; all other nodes have
+   type LockedNode. */
+struct Node : std::enable_shared_from_this<Node>
+{
+    std::map<FlakeId, std::shared_ptr<Node>> inputs;
+
+    virtual ~Node() { }
+
+    std::shared_ptr<Node> findInput(const InputPath & path);
+};
+
+/* A non-root node in the lock file. */
+struct LockedNode : Node
+{
+    FlakeRef lockedRef, originalRef;
+    TreeInfo info;
+    bool isFlake = true;
+
+    LockedNode(
+        const FlakeRef & lockedRef,
+        const FlakeRef & originalRef,
+        const TreeInfo & info,
+        bool isFlake = true)
+        : lockedRef(lockedRef), originalRef(originalRef), info(info), isFlake(isFlake)
+    { }
+
+    LockedNode(const nlohmann::json & json);
+
+    StorePath computeStorePath(Store & store) const;
+};
+
+struct LockFile
+{
+    std::shared_ptr<Node> root = std::make_shared<Node>();
+
+    LockFile() {};
+    LockFile(const nlohmann::json & json, const Path & path);
+
+    nlohmann::json toJson() const;
+
+    std::string to_string() const;
+
+    static LockFile read(const Path & path);
+
+    void write(const Path & path) const;
+
+    bool isImmutable() const;
+
+    bool operator ==(const LockFile & other) const;
+};
+
+std::ostream & operator <<(std::ostream & stream, const LockFile & lockFile);
+
+InputPath parseInputPath(std::string_view s);
+
+std::string diffLockFiles(const LockFile & oldLocks, const LockFile & newLocks);
+
+}
+

src/libexpr/function-trace.cc

@@ -1,4 +1,5 @@
 #include "function-trace.hh"
+#include "logging.hh"
 
 namespace nix {
 

src/libexpr/get-drvs.cc

@@ -348,7 +348,7 @@ static void getDerivations(EvalState & state, Value & vIn,
                    should we recurse into it?  => Only if it has a
                    `recurseForDerivations = true' attribute. */
                 if (i->value->type == tAttrs) {
-                    Bindings::iterator j = i->value->attrs->find(state.symbols.create("recurseForDerivations"));
+                    Bindings::iterator j = i->value->attrs->find(state.sRecurseForDerivations);
                     if (j != i->value->attrs->end() && state.forceBool(*j->value, *j->pos))
                         getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures);
                 }

src/libexpr/json-to-value.cc

@@ -4,7 +4,6 @@
 #include <nlohmann/json.hpp>
 
 using json = nlohmann::json;
-using std::unique_ptr;
 
 namespace nix {
 
@@ -13,69 +12,69 @@ namespace nix {
 class JSONSax : nlohmann::json_sax<json> {
     class JSONState {
     protected:
-        unique_ptr<JSONState> parent;
-        Value * v;
+        std::unique_ptr<JSONState> parent;
+        RootValue v;
     public:
-        virtual unique_ptr<JSONState> resolve(EvalState &)
+        virtual std::unique_ptr<JSONState> resolve(EvalState &)
         {
             throw std::logic_error("tried to close toplevel json parser state");
-        };
-        explicit JSONState(unique_ptr<JSONState>&& p) : parent(std::move(p)), v(nullptr) {};
-        explicit JSONState(Value* v) : v(v) {};
-        JSONState(JSONState& p) = delete;
-        Value& value(EvalState & state)
+        }
+        explicit JSONState(std::unique_ptr<JSONState> && p) : parent(std::move(p)) {}
+        explicit JSONState(Value * v) : v(allocRootValue(v)) {}
+        JSONState(JSONState & p) = delete;
+        Value & value(EvalState & state)
         {
-            if (v == nullptr)
-                v = state.allocValue();
-            return *v;
-        };
-        virtual ~JSONState() {};
-        virtual void add() {};
+            if (!v)
+                v = allocRootValue(state.allocValue());
+            return **v;
+        }
+        virtual ~JSONState() {}
+        virtual void add() {}
     };
 
     class JSONObjectState : public JSONState {
         using JSONState::JSONState;
-        ValueMap attrs = ValueMap();
-        virtual unique_ptr<JSONState> resolve(EvalState & state) override
+        ValueMap attrs;
+        std::unique_ptr<JSONState> resolve(EvalState & state) override
         {
-            Value& v = parent->value(state);
+            Value & v = parent->value(state);
             state.mkAttrs(v, attrs.size());
             for (auto & i : attrs)
                 v.attrs->push_back(Attr(i.first, i.second));
             return std::move(parent);
         }
-        virtual void add() override { v = nullptr; };
+        void add() override { v = nullptr; }
     public:
-        void key(string_t& name, EvalState & state)
+        void key(string_t & name, EvalState & state)
         {
-            attrs[state.symbols.create(name)] = &value(state);
+            attrs.insert_or_assign(state.symbols.create(name), &value(state));
         }
     };
 
     class JSONListState : public JSONState {
-        ValueVector values = ValueVector();
-        virtual unique_ptr<JSONState> resolve(EvalState & state) override
+        ValueVector values;
+        std::unique_ptr<JSONState> resolve(EvalState & state) override
         {
-            Value& v = parent->value(state);
+            Value & v = parent->value(state);
             state.mkList(v, values.size());
             for (size_t n = 0; n < values.size(); ++n) {
                 v.listElems()[n] = values[n];
             }
             return std::move(parent);
         }
-        virtual void add() override {
-            values.push_back(v);
+        void add() override {
+            values.push_back(*v);
             v = nullptr;
-        };
+        }
     public:
-        JSONListState(unique_ptr<JSONState>&& p, std::size_t reserve) : JSONState(std::move(p))
+        JSONListState(std::unique_ptr<JSONState> && p, std::size_t reserve) : JSONState(std::move(p))
         {
             values.reserve(reserve);
         }
     };
 
     EvalState & state;
-    unique_ptr<JSONState> rs;
+    std::unique_ptr<JSONState> rs;
 
     template<typename T, typename... Args> inline bool handle_value(T f, Args... args)
     {
@@ -107,12 +106,12 @@ public:
         return handle_value(mkInt, val);
     }
 
-    bool number_float(number_float_t val, const string_t& s)
+    bool number_float(number_float_t val, const string_t & s)
     {
         return handle_value(mkFloat, val);
     }
 
-    bool string(string_t& val)
+    bool string(string_t & val)
     {
         return handle_value<void(Value&, const char*)>(mkString, val.c_str());
     }
@@ -123,7 +122,7 @@ public:
         return true;
     }
 
-    bool key(string_t& name)
+    bool key(string_t & name)
     {
         dynamic_cast<JSONObjectState*>(rs.get())->key(name, state);
         return true;

src/libexpr/local.mk

@@ -4,9 +4,16 @@ libexpr_NAME = libnixexpr
 
 libexpr_DIR := $(d)
 
-libexpr_SOURCES := $(wildcard $(d)/*.cc) $(wildcard $(d)/primops/*.cc) $(d)/lexer-tab.cc $(d)/parser-tab.cc
+libexpr_SOURCES := \
+  $(wildcard $(d)/*.cc) \
+  $(wildcard $(d)/primops/*.cc) \
+  $(wildcard $(d)/flake/*.cc) \
+  $(d)/lexer-tab.cc \
+  $(d)/parser-tab.cc
 
-libexpr_LIBS = libutil libstore libnixrust
+libexpr_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/libmain -I src/libexpr
+
+libexpr_LIBS = libutil libstore libfetchers libnixrust
 
 libexpr_LDFLAGS =
 ifneq ($(OS), FreeBSD)
@@ -31,3 +38,10 @@ clean-files += $(d)/parser-tab.cc $(d)/parser-tab.hh $(d)/lexer-tab.cc $(d)/lexe
 dist-files += $(d)/parser-tab.cc $(d)/parser-tab.hh $(d)/lexer-tab.cc $(d)/lexer-tab.hh
 
 $(eval $(call install-file-in, $(d)/nix-expr.pc, $(prefix)/lib/pkgconfig, 0644))
+
+$(foreach i, $(wildcard src/libexpr/flake/*.hh), \
+  $(eval $(call install-file-in, $(i), $(includedir)/nix/flake, 0644)))
+
+$(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh
+
+$(d)/flake/flake.cc: $(d)/flake/call-flake.nix.gen.hh

src/libexpr/nixexpr.hh

@@ -209,9 +209,10 @@ struct ExprList : Expr
 
 struct Formal
 {
+    Pos pos;
     Symbol name;
     Expr * def;
-    Formal(const Symbol & name, Expr * def) : name(name), def(def) { };
+    Formal(const Pos & pos, const Symbol & name, Expr * def) : pos(pos), name(name), def(def) { };
 };
 
 struct Formals
@@ -261,8 +262,9 @@ struct ExprWith : Expr
 
 struct ExprIf : Expr
 {
+    Pos pos;
     Expr * cond, * then, * else_;
-    ExprIf(Expr * cond, Expr * then, Expr * else_) : cond(cond), then(then), else_(else_) { };
+    ExprIf(const Pos & pos, Expr * cond, Expr * then, Expr * else_) : pos(pos), cond(cond), then(then), else_(else_) { };
     COMMON_METHODS
 };
 

src/libexpr/parser.y

@@ -335,7 +335,7 @@ expr_function
   ;
 
 expr_if
-  : IF expr THEN expr ELSE expr { $$ = new ExprIf($2, $4, $6); }
+  : IF expr THEN expr ELSE expr { $$ = new ExprIf(CUR_POS, $2, $4, $6); }
   | expr_op
   ;
 
@@ -531,8 +531,8 @@ formals
   ;
 
 formal
-  : ID { $$ = new Formal(data->symbols.create($1), 0); }
-  | ID '?' expr { $$ = new Formal(data->symbols.create($1), $3); }
+  : ID { $$ = new Formal(CUR_POS, data->symbols.create($1), 0); }
+  | ID '?' expr { $$ = new Formal(CUR_POS, data->symbols.create($1), $3); }
   ;
 
 %%
@@ -544,7 +544,8 @@ formal
 #include <unistd.h>
 
 #include "eval.hh"
-#include "download.hh"
+#include "filetransfer.hh"
+#include "fetchers.hh"
 #include "store-api.hh"
 
 
@@ -611,13 +612,13 @@ Expr * EvalState::parseExprFromFile(const Path & path, StaticEnv & staticEnv)
 }
 
 
-Expr * EvalState::parseExprFromString(const string & s, const Path & basePath, StaticEnv & staticEnv)
+Expr * EvalState::parseExprFromString(std::string_view s, const Path & basePath, StaticEnv & staticEnv)
 {
-    return parse(s.c_str(), "(string)", basePath, staticEnv);
+    return parse(s.data(), "(string)", basePath, staticEnv);
 }
 
 
-Expr * EvalState::parseExprFromString(const string & s, const Path & basePath)
+Expr * EvalState::parseExprFromString(std::string_view s, const Path & basePath)
 {
     return parseExprFromString(s, basePath, staticBaseEnv);
 }
@@ -687,10 +688,9 @@ std::pair<bool, std::string> EvalState::resolveSearchPathElem(const SearchPathEl
 
     if (isUri(elem.second)) {
         try {
-            CachedDownloadRequest request(elem.second);
-            request.unpack = true;
-            res = { true, getDownloader()->downloadCached(store, request).path };
-        } catch (DownloadError & e) {
+            res = { true, store->toRealPath(fetchers::downloadTarball(
+                        store, resolveUri(elem.second), "source", false).storePath) };
+        } catch (FileTransferError & e) {
             printError(format("warning: Nix search path entry '%1%' cannot be downloaded, ignoring") % elem.second);
             res = { false, "" };
         }

src/libexpr/primops.cc

@@ -1,6 +1,5 @@
 #include "archive.hh"
 #include "derivations.hh"
-#include "download.hh"
 #include "eval-inline.hh"
 #include "eval.hh"
 #include "globals.hh"
@@ -51,20 +50,20 @@ void EvalState::realiseContext(const PathSet & context)
     std::vector<StorePathWithOutputs> drvs;
 
     for (auto & i : context) {
-        std::pair<string, string> decoded = decodeContext(i);
-        auto ctx = store->parseStorePath(decoded.first);
+        auto [ctxS, outputName] = decodeContext(i);
+        auto ctx = store->parseStorePath(ctxS);
         if (!store->isValidPath(ctx))
             throw InvalidPathError(store->printStorePath(ctx));
-        if (!decoded.second.empty() && ctx.isDerivation()) {
-            drvs.push_back(StorePathWithOutputs{ctx.clone(), {decoded.second}});
+        if (!outputName.empty() && ctx.isDerivation()) {
+            drvs.push_back(StorePathWithOutputs{ctx.clone(), {outputName}});
 
             /* Add the output of this derivation to the allowed
                paths. */
             if (allowedPaths) {
-                auto drv = store->derivationFromPath(store->parseStorePath(decoded.first));
-                DerivationOutputs::iterator i = drv.outputs.find(decoded.second);
+                auto drv = store->derivationFromPath(ctx);
+                DerivationOutputs::iterator i = drv.outputs.find(outputName);
                 if (i == drv.outputs.end())
-                    throw Error("derivation '%s' does not have an output named '%s'", decoded.first, decoded.second);
+                    throw Error("derivation '%s' does not have an output named '%s'", ctxS, outputName);
                 allowedPaths->insert(store->printStorePath(i->second.path));
             }
         }
@@ -80,6 +79,7 @@ void EvalState::realiseContext(const PathSet & context)
     StorePathSet willBuild, willSubstitute, unknown;
     unsigned long long downloadSize, narSize;
     store->queryMissing(drvs, willBuild, willSubstitute, unknown, downloadSize, narSize);
+
     store->buildPaths(drvs);
 }
 
@@ -121,10 +121,17 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args
             mkString(*(outputsVal->listElems()[outputs_index++]), o.first);
         }
         w.attrs->sort();
-        Value fun;
-        state.evalFile(settings.nixDataDir + "/nix/corepkgs/imported-drv-to-derivation.nix", fun);
-        state.forceFunction(fun, pos);
-        mkApp(v, fun, w);
+
+        static RootValue fun;
+        if (!fun) {
+            fun = allocRootValue(state.allocValue());
+            state.eval(state.parseExprFromString(
+                #include "imported-drv-to-derivation.nix.gen.hh"
+                , "/"), **fun);
+        }
+
+        state.forceFunction(**fun, pos);
+        mkApp(v, **fun, w);
         state.forceAttrs(v, pos);
     } else {
         state.forceAttrs(*args[0]);
@@ -235,7 +242,7 @@ void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v)
 /* Return a string representing the type of the expression. */
 static void prim_typeOf(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
-    state.forceValue(*args[0]);
+    state.forceValue(*args[0], pos);
     string t;
     switch (args[0]->type) {
         case tInt: t = "int"; break;
@@ -263,7 +270,7 @@ static void prim_typeOf(EvalState & state, const Pos & pos, Value * * args, Valu
 /* Determine whether the argument is the null value. */
 static void prim_isNull(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
-    state.forceValue(*args[0]);
+    state.forceValue(*args[0], pos);
     mkBool(v, args[0]->type == tNull);
 }
 
@@ -271,7 +278,7 @@ static void prim_isNull(EvalState & state, const Pos & pos, Value * * args, Valu
 /* Determine whether the argument is a function. */
 static void prim_isFunction(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
-    state.forceValue(*args[0]);
+    state.forceValue(*args[0], pos);
     bool res;
     switch (args[0]->type) {
         case tLambda:
@@ -290,21 +297,21 @@ static void prim_isFunction(EvalState & state, const Pos & pos, Value * * args,
 /* Determine whether the argument is an integer. */
 static void prim_isInt(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
-    state.forceValue(*args[0]);
+    state.forceValue(*args[0], pos);
     mkBool(v, args[0]->type == tInt);
 }
 
 /* Determine whether the argument is a float. */
 static void prim_isFloat(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
-    state.forceValue(*args[0]);
+    state.forceValue(*args[0], pos);
     mkBool(v, args[0]->type == tFloat);
 }
 
 /* Determine whether the argument is a string. */
 static void prim_isString(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
-    state.forceValue(*args[0]);
+    state.forceValue(*args[0], pos);
     mkBool(v, args[0]->type == tString);
 }
 
@@ -312,14 +319,14 @@ static void prim_isString(EvalState & state, const Pos & pos, Value * * args, Va
 /* Determine whether the argument is a Boolean. */
 static void prim_isBool(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
-    state.forceValue(*args[0]);
+    state.forceValue(*args[0], pos);
     mkBool(v, args[0]->type == tBool);
 }
 
 /* Determine whether the argument is a path. */
 static void prim_isPath(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
-    state.forceValue(*args[0]);
+    state.forceValue(*args[0], pos);
     mkBool(v, args[0]->type == tPath);
 }
 
@@ -376,7 +383,7 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar
         args[0]->attrs->find(state.symbols.create("operator"));
     if (op == args[0]->attrs->end())
         throw EvalError(format("attribute 'operator' required, at %1%") % pos);
-    state.forceValue(*op->value);
+    state.forceValue(*op->value, pos);
 
     /* Construct the closure by applying the operator to element of
        `workSet', adding the result to `workSet', continuing until
@@ -395,7 +402,7 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar
             e->attrs->find(state.symbols.create("key"));
         if (key == e->attrs->end())
             throw EvalError(format("attribute 'key' required, at %1%") % pos);
-        state.forceValue(*key->value);
+        state.forceValue(*key->value, pos);
 
         if (!doneKeys.insert(key->value).second) continue;
         res.push_back(e);
@@ -407,7 +414,7 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar
 
         /* Add the values returned by the operator to the work set. */
         for (unsigned int n = 0; n < call.listSize(); ++n) {
-            state.forceValue(*call.listElems()[n]);
+            state.forceValue(*call.listElems()[n], pos);
             workSet.push_back(call.listElems()[n]);
         }
     }
@@ -439,7 +446,7 @@ static void prim_throw(EvalState & state, const Pos & pos, Value * * args, Value
 static void prim_addErrorContext(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     try {
-        state.forceValue(*args[1]);
+        state.forceValue(*args[1], pos);
         v = *args[1];
     } catch (Error & e) {
         PathSet context;
@@ -455,7 +462,7 @@ static void prim_tryEval(EvalState & state, const Pos & pos, Value * * args, Val
 {
     state.mkAttrs(v, 2);
     try {
-        state.forceValue(*args[0]);
+        state.forceValue(*args[0], pos);
         v.attrs->push_back(Attr(state.sValue, args[0]));
         mkBool(*state.allocAttr(v, state.symbols.create("success")), true);
     } catch (AssertionError & e) {
@@ -477,8 +484,8 @@ static void prim_getEnv(EvalState & state, const Pos & pos, Value * * args, Valu
 /* Evaluate the first argument, then return the second argument. */
 static void prim_seq(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
-    state.forceValue(*args[0]);
-    state.forceValue(*args[1]);
+    state.forceValue(*args[0], pos);
+    state.forceValue(*args[1], pos);
     v = *args[1];
 }
 
@@ -488,7 +495,7 @@ static void prim_seq(EvalState & state, const Pos & pos, Value * * args, Value &
 static void prim_deepSeq(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     state.forceValueDeep(*args[0]);
-    state.forceValue(*args[1]);
+    state.forceValue(*args[1], pos);
     v = *args[1];
 }
 
@@ -497,12 +504,12 @@ static void prim_deepSeq(EvalState & state, const Pos & pos, Value * * args, Val
    return the second expression.  Useful for debugging. */
 static void prim_trace(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
-    state.forceValue(*args[0]);
+    state.forceValue(*args[0], pos);
     if (args[0]->type == tString)
         printError(format("trace: %1%") % args[0]->string.s);
     else
         printError(format("trace: %1%") % *args[0]);
-    state.forceValue(*args[1]);
+    state.forceValue(*args[1], pos);
     v = *args[1];
 }
 
@@ -593,7 +600,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
         try {
 
             if (ignoreNulls) {
-                state.forceValue(*i->value);
+                state.forceValue(*i->value, pos);
                 if (i->value->type == tNull) continue;
             }
 
@@ -1014,7 +1021,9 @@ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Valu
 
     for (auto path : context) {
         if (path.at(0) != '/')
-            throw EvalError(format("in 'toFile': the file '%1%' cannot refer to derivation outputs, at %2%") % name % pos);
+            throw EvalError(format(
+                "in 'toFile': the file named '%1%' must not contain a reference "
+                "to a derivation but contains (%2%), at %3%") % name % path % pos);
         refs.insert(state.store->parseStorePath(path));
     }
 
@@ -1084,7 +1093,7 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args
     if (!context.empty())
         throw EvalError(format("string '%1%' cannot refer to other paths, at %2%") % path % pos);
 
-    state.forceValue(*args[0]);
+    state.forceValue(*args[0], pos);
     if (args[0]->type != tLambda)
         throw TypeError(format("first argument in call to 'filterSource' is not a function but %1%, at %2%") % showType(*args[0]) % pos);
 
@@ -1110,7 +1119,7 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value
         } else if (attr.name == state.sName)
             name = state.forceStringNoCtx(*attr.value, *attr.pos);
         else if (n == "filter") {
-            state.forceValue(*attr.value);
+            state.forceValue(*attr.value, pos);
             filterFun = attr.value;
         } else if (n == "recursive")
             recursive = state.forceBool(*attr.value, *attr.pos);
@@ -1181,7 +1190,7 @@ void prim_getAttr(EvalState & state, const Pos & pos, Value * * args, Value & v)
         throw EvalError(format("attribute '%1%' missing, at %2%") % attr % pos);
     // !!! add to stack trace?
     if (state.countCalls && i->pos) state.attrSelects[*i->pos]++;
-    state.forceValue(*i->value);
+    state.forceValue(*i->value, pos);
     v = *i->value;
 }
 
@@ -1211,7 +1220,7 @@ static void prim_hasAttr(EvalState & state, const Pos & pos, Value * * args, Val
 /* Determine whether the argument is a set. */
 static void prim_isAttrs(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
-    state.forceValue(*args[0]);
+    state.forceValue(*args[0], pos);
     mkBool(v, args[0]->type == tAttrs);
 }
 
@@ -1337,7 +1346,7 @@ static void prim_catAttrs(EvalState & state, const Pos & pos, Value * * args, Va
 */
 static void prim_functionArgs(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
-    state.forceValue(*args[0]);
+    state.forceValue(*args[0], pos);
     if (args[0]->type != tLambda)
         throw TypeError(format("'functionArgs' requires a function, at %1%") % pos);
 
@@ -1347,9 +1356,12 @@ static void prim_functionArgs(EvalState & state, const Pos & pos, Value * * args
     }
 
     state.mkAttrs(v, args[0]->lambda.fun->formals->formals.size());
-    for (auto & i : args[0]->lambda.fun->formals->formals)
+    for (auto & i : args[0]->lambda.fun->formals->formals) {
         // !!! should optimise booleans (allocate only once)
-        mkBool(*state.allocAttr(v, i.name), i.def);
+        Value * value = state.allocValue();
+        v.attrs->push_back(Attr(i.name, value, &i.pos));
+        mkBool(*value, i.def);
+    }
     v.attrs->sort();
 }
 
@@ -1380,7 +1392,7 @@ static void prim_mapAttrs(EvalState & state, const Pos & pos, Value * * args, Va
 /* Determine whether the argument is a list. */
 static void prim_isList(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
-    state.forceValue(*args[0]);
+    state.forceValue(*args[0], pos);
     mkBool(v, args[0]->isList());
 }
 
@@ -1390,7 +1402,7 @@ static void elemAt(EvalState & state, const Pos & pos, Value & list, int n, Valu
     state.forceList(list, pos);
     if (n < 0 || (unsigned int) n >= list.listSize())
         throw Error(format("list index %1% is out of bounds, at %2%") % n % pos);
-    state.forceValue(*list.listElems()[n]);
+    state.forceValue(*list.listElems()[n], pos);
     v = *list.listElems()[n];
 }
 
@@ -1513,9 +1525,9 @@ static void prim_foldlStrict(EvalState & state, const Pos & pos, Value * * args,
             vCur = n == args[2]->listSize() - 1 ? &v : state.allocValue();
             state.callFunction(vTmp, *args[2]->listElems()[n], *vCur, pos);
         }
-        state.forceValue(v);
+        state.forceValue(v, pos);
     } else {
-        state.forceValue(*args[1]);
+        state.forceValue(*args[1], pos);
         v = *args[1];
     }
 }
@@ -1580,7 +1592,7 @@ static void prim_sort(EvalState & state, const Pos & pos, Value * * args, Value
     auto len = args[1]->listSize();
     state.mkList(v, len);
     for (unsigned int n = 0; n < len; ++n) {
-        state.forceValue(*args[1]->listElems()[n]);
+        state.forceValue(*args[1]->listElems()[n], pos);
         v.listElems()[n] = args[1]->listElems()[n];
     }
 
@@ -1615,7 +1627,7 @@ static void prim_partition(EvalState & state, const Pos & pos, Value * * args, V
 
     for (unsigned int n = 0; n < len; ++n) {
         auto vElem = args[1]->listElems()[n];
-        state.forceValue(*vElem);
+        state.forceValue(*vElem, pos);
         Value res;
         state.callFunction(*args[0], *vElem, res, pos);
         if (state.forceBool(res, pos))
@@ -1746,8 +1758,8 @@ static void prim_bitXor(EvalState & state, const Pos & pos, Value * * args, Valu
 
 static void prim_lessThan(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
-    state.forceValue(*args[0]);
-    state.forceValue(*args[1]);
+    state.forceValue(*args[0], pos);
+    state.forceValue(*args[1], pos);
     CompareValues comp;
     mkBool(v, comp(args[0], args[1]));
 }
@@ -1811,19 +1823,21 @@ static void prim_hashString(EvalState & state, const Pos & pos, Value * * args,
 
 /* Match a regular expression against a string and return either
    ‘null’ or a list containing substring matches. */
-static void prim_match(EvalState & state, const Pos & pos, Value * * args, Value & v)
+void prim_match(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     auto re = state.forceStringNoCtx(*args[0], pos);
 
     try {
 
-        std::regex regex(re, std::regex::extended);
+        auto regex = state.regexCache.find(re);
+        if (regex == state.regexCache.end())
+            regex = state.regexCache.emplace(re, std::regex(re, std::regex::extended)).first;
 
         PathSet context;
         const std::string str = state.forceString(*args[1], context, pos);
 
         std::smatch match;
-        if (!std::regex_match(str, match, regex)) {
+        if (!std::regex_match(str, match, regex->second)) {
             mkNull(v);
             return;
         }
@@ -2036,68 +2050,6 @@ static void prim_splitVersion(EvalState & state, const Pos & pos, Value * * args
 }
 
 
-/*************************************************************
- * Networking
- *************************************************************/
-
-
-void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
-    const string & who, bool unpack, const std::string & defaultName)
-{
-    CachedDownloadRequest request("");
-    request.unpack = unpack;
-    request.name = defaultName;
-
-    state.forceValue(*args[0]);
-
-    if (args[0]->type == tAttrs) {
-
-        state.forceAttrs(*args[0], pos);
-
-        for (auto & attr : *args[0]->attrs) {
-            string n(attr.name);
-            if (n == "url")
-                request.uri = state.forceStringNoCtx(*attr.value, *attr.pos);
-            else if (n == "sha256")
-                request.expectedHash = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256);
-            else if (n == "name")
-                request.name = state.forceStringNoCtx(*attr.value, *attr.pos);
-            else
-                throw EvalError(format("unsupported argument '%1%' to '%2%', at %3%") % attr.name % who % attr.pos);
-        }
-
-        if (request.uri.empty())
-            throw EvalError(format("'url' argument required, at %1%") % pos);
-
-    } else
-        request.uri = state.forceStringNoCtx(*args[0], pos);
-
-    state.checkURI(request.uri);
-
-    if (evalSettings.pureEval && !request.expectedHash)
-        throw Error("in pure evaluation mode, '%s' requires a 'sha256' argument", who);
-
-    auto res = getDownloader()->downloadCached(state.store, request);
-
-    if (state.allowedPaths)
-        state.allowedPaths->insert(res.path);
-
-    mkString(v, res.storePath, PathSet({res.storePath}));
-}
-
-
-static void prim_fetchurl(EvalState & state, const Pos & pos, Value * * args, Value & v)
-{
-    fetch(state, pos, args, v, "fetchurl", false, "");
-}
-
-
-static void prim_fetchTarball(EvalState & state, const Pos & pos, Value * * args, Value & v)
-{
-    fetch(state, pos, args, v, "fetchTarball", true, "source");
-}
-
-
 /*************************************************************
  * Primop registration
  *************************************************************/
@@ -2280,10 +2232,6 @@ void EvalState::createBaseEnv()
     addPrimOp("derivationStrict", 1, prim_derivationStrict);
     addPrimOp("placeholder", 1, prim_placeholder);
 
-    // Networking
-    addPrimOp("__fetchurl", 1, prim_fetchurl);
-    addPrimOp("fetchTarball", 1, prim_fetchTarball);
-
     /* Add a wrapper around the derivation primop that computes the
        `drvPath' and `outPath' attributes lazily. */
     string path = canonPath(settings.nixDataDir + "/nix/corepkgs/derivation.nix", true);

src/libexpr/primops.hh

@@ -20,6 +20,7 @@ struct RegisterPrimOp
    them. */
 /* Load a ValueInitializer from a DSO and return whatever it initializes */
 void prim_importNative(EvalState & state, const Pos & pos, Value * * args, Value & v);
+
 /* Execute a program and parse its output */
 void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v);
 

src/libexpr/primops/fetchGit.cc

@@ -1,203 +1,19 @@
 #include "primops.hh"
 #include "eval-inline.hh"
-#include "download.hh"
 #include "store-api.hh"
-#include "pathlocks.hh"
 #include "hash.hh"
-#include "tarfile.hh"
-
-#include <sys/time.h>
-
-#include <regex>
-
-#include <nlohmann/json.hpp>
-
-using namespace std::string_literals;
+#include "fetchers.hh"
+#include "url.hh"
 
 namespace nix {
 
-struct GitInfo
-{
-    Path storePath;
-    std::string rev;
-    std::string shortRev;
-    uint64_t revCount = 0;
-};
-
-std::regex revRegex("^[0-9a-fA-F]{40}$");
-
-GitInfo exportGit(ref<Store> store, const std::string & uri,
-    std::optional<std::string> ref, std::string rev,
-    const std::string & name)
-{
-    if (evalSettings.pureEval && rev == "")
-        throw Error("in pure evaluation mode, 'fetchGit' requires a Git revision");
-
-    if (!ref && rev == "" && hasPrefix(uri, "/") && pathExists(uri + "/.git")) {
-
-        bool clean = true;
-
-        try {
-            runProgram("git", true, { "-C", uri, "diff-index", "--quiet", "HEAD", "--" });
-        } catch (ExecError & e) {
-            if (!WIFEXITED(e.status) || WEXITSTATUS(e.status) != 1) throw;
-            clean = false;
-        }
-
-        if (!clean) {
-
-            /* This is an unclean working tree. So copy all tracked files. */
-            GitInfo gitInfo;
-            gitInfo.rev = "0000000000000000000000000000000000000000";
-            gitInfo.shortRev = std::string(gitInfo.rev, 0, 7);
-
-            auto files = tokenizeString<std::set<std::string>>(
-                runProgram("git", true, { "-C", uri, "ls-files", "-z" }), "\0"s);
-
-            PathFilter filter = [&](const Path & p) -> bool {
-                assert(hasPrefix(p, uri));
-                std::string file(p, uri.size() + 1);
-
-                auto st = lstat(p);
-
-                if (S_ISDIR(st.st_mode)) {
-                    auto prefix = file + "/";
-                    auto i = files.lower_bound(prefix);
-                    return i != files.end() && hasPrefix(*i, prefix);
-                }
-
-                return files.count(file);
-            };
-
-            gitInfo.storePath = store->printStorePath(store->addToStore("source", uri, true, htSHA256, filter));
-
-            return gitInfo;
-        }
-
-        // clean working tree, but no ref or rev specified.  Use 'HEAD'.
-        rev = chomp(runProgram("git", true, { "-C", uri, "rev-parse", "HEAD" }));
-        ref = "HEAD"s;
-    }
-
-    if (!ref) ref = "HEAD"s;
-
-    if (rev != "" && !std::regex_match(rev, revRegex))
-        throw Error("invalid Git revision '%s'", rev);
-
-    deletePath(getCacheDir() + "/nix/git");
-
-    Path cacheDir = getCacheDir() + "/nix/gitv2/" + hashString(htSHA256, uri).to_string(Base32, false);
-
-    if (!pathExists(cacheDir)) {
-        createDirs(dirOf(cacheDir));
-        runProgram("git", true, { "init", "--bare", cacheDir });
-    }
-
-    Path localRefFile;
-    if (ref->compare(0, 5, "refs/") == 0)
-        localRefFile = cacheDir + "/" + *ref;
-    else
-        localRefFile = cacheDir + "/refs/heads/" + *ref;
-
-    bool doFetch;
-    time_t now = time(0);
-    /* If a rev was specified, we need to fetch if it's not in the
-       repo. */
-    if (rev != "") {
-        try {
-            runProgram("git", true, { "-C", cacheDir, "cat-file", "-e", rev });
-            doFetch = false;
-        } catch (ExecError & e) {
-            if (WIFEXITED(e.status)) {
-                doFetch = true;
-            } else {
-                throw;
-            }
-        }
-    } else {
-        /* If the local ref is older than ‘tarball-ttl’ seconds, do a
-           git fetch to update the local ref to the remote ref. */
-        struct stat st;
-        doFetch = stat(localRefFile.c_str(), &st) != 0 ||
-            (uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now;
-    }
-    if (doFetch)
-    {
-        Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Git repository '%s'", uri));
-
-        // FIXME: git stderr messes up our progress indicator, so
-        // we're using --quiet for now. Should process its stderr.
-        runProgram("git", true, { "-C", cacheDir, "fetch", "--quiet", "--force", "--", uri, fmt("%s:%s", *ref, *ref) });
-
-        struct timeval times[2];
-        times[0].tv_sec = now;
-        times[0].tv_usec = 0;
-        times[1].tv_sec = now;
-        times[1].tv_usec = 0;
-
-        utimes(localRefFile.c_str(), times);
-    }
-
-    // FIXME: check whether rev is an ancestor of ref.
-    GitInfo gitInfo;
-    gitInfo.rev = rev != "" ? rev : chomp(readFile(localRefFile));
-    gitInfo.shortRev = std::string(gitInfo.rev, 0, 7);
-
-    printTalkative("using revision %s of repo '%s'", gitInfo.rev, uri);
-
-    std::string storeLinkName = hashString(htSHA512, name + std::string("\0"s) + gitInfo.rev).to_string(Base32, false);
-    Path storeLink = cacheDir + "/" + storeLinkName + ".link";
-    PathLocks storeLinkLock({storeLink}, fmt("waiting for lock on '%1%'...", storeLink)); // FIXME: broken
-
-    try {
-        auto json = nlohmann::json::parse(readFile(storeLink));
-
-        assert(json["name"] == name && json["rev"] == gitInfo.rev);
-
-        gitInfo.storePath = json["storePath"];
-
-        if (store->isValidPath(store->parseStorePath(gitInfo.storePath))) {
-            gitInfo.revCount = json["revCount"];
-            return gitInfo;
-        }
-
-    } catch (SysError & e) {
-        if (e.errNo != ENOENT) throw;
-    }
-
-    auto source = sinkToSource([&](Sink & sink) {
-        RunOptions gitOptions("git", { "-C", cacheDir, "archive", gitInfo.rev });
-        gitOptions.standardOut = &sink;
-        runProgram2(gitOptions);
-    });
-
-    Path tmpDir = createTempDir();
-    AutoDelete delTmpDir(tmpDir, true);
-
-    unpackTarfile(*source, tmpDir);
-
-    gitInfo.storePath = store->printStorePath(store->addToStore(name, tmpDir));
-
-    gitInfo.revCount = std::stoull(runProgram("git", true, { "-C", cacheDir, "rev-list", "--count", gitInfo.rev }));
-
-    nlohmann::json json;
-    json["storePath"] = gitInfo.storePath;
-    json["uri"] = uri;
-    json["name"] = name;
-    json["rev"] = gitInfo.rev;
-    json["revCount"] = gitInfo.revCount;
-
-    writeFile(storeLink, json.dump());
-
-    return gitInfo;
-}
-
 static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     std::string url;
     std::optional<std::string> ref;
-    std::string rev;
+    std::optional<Hash> rev;
     std::string name = "source";
+    bool fetchSubmodules = false;
     PathSet context;
 
     state.forceValue(*args[0]);
@@ -213,9 +29,11 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va
             else if (n == "ref")
                 ref = state.forceStringNoCtx(*attr.value, *attr.pos);
             else if (n == "rev")
-                rev = state.forceStringNoCtx(*attr.value, *attr.pos);
+                rev = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA1);
             else if (n == "name")
                 name = state.forceStringNoCtx(*attr.value, *attr.pos);
+            else if (n == "submodules")
+                fetchSubmodules = state.forceBool(*attr.value, *attr.pos);
             else
                 throw EvalError("unsupported argument '%s' to 'fetchGit', at %s", attr.name, *attr.pos);
         }
@@ -230,17 +48,36 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va
     // whitelist. Ah well.
     state.checkURI(url);
 
-    auto gitInfo = exportGit(state.store, url, ref, rev, name);
+    if (evalSettings.pureEval && !rev)
+        throw Error("in pure evaluation mode, 'fetchGit' requires a Git revision");
+
+    fetchers::Attrs attrs;
+    attrs.insert_or_assign("type", "git");
+    attrs.insert_or_assign("url", url.find("://") != std::string::npos ? url : "file://" + url);
+    if (ref) attrs.insert_or_assign("ref", *ref);
+    if (rev) attrs.insert_or_assign("rev", rev->gitRev());
+    if (fetchSubmodules) attrs.insert_or_assign("submodules", true);
+    auto input = fetchers::inputFromAttrs(attrs);
+
+    // FIXME: use name?
+    auto [tree, input2] = input->fetchTree(state.store);
 
     state.mkAttrs(v, 8);
-    mkString(*state.allocAttr(v, state.sOutPath), gitInfo.storePath, PathSet({gitInfo.storePath}));
-    mkString(*state.allocAttr(v, state.symbols.create("rev")), gitInfo.rev);
-    mkString(*state.allocAttr(v, state.symbols.create("shortRev")), gitInfo.shortRev);
-    mkInt(*state.allocAttr(v, state.symbols.create("revCount")), gitInfo.revCount);
+    auto storePath = state.store->printStorePath(tree.storePath);
+    mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath}));
+    // Backward compatibility: set 'rev' to
+    // 0000000000000000000000000000000000000000 for a dirty tree.
+    auto rev2 = input2->getRev().value_or(Hash(htSHA1));
+    mkString(*state.allocAttr(v, state.symbols.create("rev")), rev2.gitRev());
+    mkString(*state.allocAttr(v, state.symbols.create("shortRev")), rev2.gitShortRev());
+    // Backward compatibility: set 'revCount' to 0 for a dirty tree.
+    mkInt(*state.allocAttr(v, state.symbols.create("revCount")),
+        tree.info.revCount.value_or(0));
+    mkBool(*state.allocAttr(v, state.symbols.create("submodules")), fetchSubmodules);
     v.attrs->sort();
 
     if (state.allowedPaths)
-        state.allowedPaths->insert(state.store->toRealPath(gitInfo.storePath));
+        state.allowedPaths->insert(tree.actualPath);
 }
 
 static RegisterPrimOp r("fetchGit", 1, prim_fetchGit);

src/libexpr/primops/fetchMercurial.cc

@@ -1,174 +1,18 @@
 #include "primops.hh"
 #include "eval-inline.hh"
-#include "download.hh"
 #include "store-api.hh"
-#include "pathlocks.hh"
-
-#include <sys/time.h>
+#include "fetchers.hh"
+#include "url.hh"
 
 #include <regex>
 
-#include <nlohmann/json.hpp>
-
-using namespace std::string_literals;
-
 namespace nix {
 
-struct HgInfo
-{
-    Path storePath;
-    std::string branch;
-    std::string rev;
-    uint64_t revCount = 0;
-};
-
-std::regex commitHashRegex("^[0-9a-fA-F]{40}$");
-
-HgInfo exportMercurial(ref<Store> store, const std::string & uri,
-    std::string rev, const std::string & name)
-{
-    if (evalSettings.pureEval && rev == "")
-        throw Error("in pure evaluation mode, 'fetchMercurial' requires a Mercurial revision");
-
-    if (rev == "" && hasPrefix(uri, "/") && pathExists(uri + "/.hg")) {
-
-        bool clean = runProgram("hg", true, { "status", "-R", uri, "--modified", "--added", "--removed" }) == "";
-
-        if (!clean) {
-
-            /* This is an unclean working tree. So copy all tracked
-               files. */
-
-            printTalkative("copying unclean Mercurial working tree '%s'", uri);
-
-            HgInfo hgInfo;
-            hgInfo.rev = "0000000000000000000000000000000000000000";
-            hgInfo.branch = chomp(runProgram("hg", true, { "branch", "-R", uri }));
-
-            auto files = tokenizeString<std::set<std::string>>(
-                runProgram("hg", true, { "status", "-R", uri, "--clean", "--modified", "--added", "--no-status", "--print0" }), "\0"s);
-
-            PathFilter filter = [&](const Path & p) -> bool {
-                assert(hasPrefix(p, uri));
-                std::string file(p, uri.size() + 1);
-
-                auto st = lstat(p);
-
-                if (S_ISDIR(st.st_mode)) {
-                    auto prefix = file + "/";
-                    auto i = files.lower_bound(prefix);
-                    return i != files.end() && hasPrefix(*i, prefix);
-                }
-
-                return files.count(file);
-            };
-
-            hgInfo.storePath = store->printStorePath(store->addToStore("source", uri, true, htSHA256, filter));
-
-            return hgInfo;
-        }
-    }
-
-    if (rev == "") rev = "default";
-
-    Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), hashString(htSHA256, uri).to_string(Base32, false));
-
-    Path stampFile = fmt("%s/.hg/%s.stamp", cacheDir, hashString(htSHA512, rev).to_string(Base32, false));
-
-    /* If we haven't pulled this repo less than ‘tarball-ttl’ seconds,
-       do so now. */
-    time_t now = time(0);
-    struct stat st;
-    if (stat(stampFile.c_str(), &st) != 0 ||
-        (uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now)
-    {
-        /* Except that if this is a commit hash that we already have,
-           we don't have to pull again. */
-        if (!(std::regex_match(rev, commitHashRegex)
-                && pathExists(cacheDir)
-                && runProgram(
-                    RunOptions("hg", { "log", "-R", cacheDir, "-r", rev, "--template", "1" })
-                    .killStderr(true)).second == "1"))
-        {
-            Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Mercurial repository '%s'", uri));
-
-            if (pathExists(cacheDir)) {
-                try {
-                    runProgram("hg", true, { "pull", "-R", cacheDir, "--", uri });
-                }
-                catch (ExecError & e) {
-                    string transJournal = cacheDir + "/.hg/store/journal";
-                    /* hg throws "abandoned transaction" error only if this file exists */
-                    if (pathExists(transJournal)) {
-                        runProgram("hg", true, { "recover", "-R", cacheDir });
-                        runProgram("hg", true, { "pull", "-R", cacheDir, "--", uri });
-                    } else {
-                        throw ExecError(e.status, fmt("'hg pull' %s", statusToString(e.status)));
-                    }
-                }
-            } else {
-                createDirs(dirOf(cacheDir));
-                runProgram("hg", true, { "clone", "--noupdate", "--", uri, cacheDir });
-            }
-        }
-
-        writeFile(stampFile, "");
-    }
-
-    auto tokens = tokenizeString<std::vector<std::string>>(
-        runProgram("hg", true, { "log", "-R", cacheDir, "-r", rev, "--template", "{node} {rev} {branch}" }));
-    assert(tokens.size() == 3);
-
-    HgInfo hgInfo;
-    hgInfo.rev = tokens[0];
-    hgInfo.revCount = std::stoull(tokens[1]);
-    hgInfo.branch = tokens[2];
-
-    std::string storeLinkName = hashString(htSHA512, name + std::string("\0"s) + hgInfo.rev).to_string(Base32, false);
-    Path storeLink = fmt("%s/.hg/%s.link", cacheDir, storeLinkName);
-
-    try {
-        auto json = nlohmann::json::parse(readFile(storeLink));
-
-        assert(json["name"] == name && json["rev"] == hgInfo.rev);
-
-        hgInfo.storePath = json["storePath"];
-
-        if (store->isValidPath(store->parseStorePath(hgInfo.storePath))) {
-            printTalkative("using cached Mercurial store path '%s'", hgInfo.storePath);
-            return hgInfo;
-        }
-
-    } catch (SysError & e) {
-        if (e.errNo != ENOENT) throw;
-    }
-
-    Path tmpDir = createTempDir();
-    AutoDelete delTmpDir(tmpDir, true);
-
-    runProgram("hg", true, { "archive", "-R", cacheDir, "-r", rev, tmpDir });
-
-    deletePath(tmpDir + "/.hg_archival.txt");
-
-    hgInfo.storePath = store->printStorePath(store->addToStore(name, tmpDir));
-
-    nlohmann::json json;
-    json["storePath"] = hgInfo.storePath;
-    json["uri"] = uri;
-    json["name"] = name;
-    json["branch"] = hgInfo.branch;
-    json["rev"] = hgInfo.rev;
-    json["revCount"] = hgInfo.revCount;
-
-    writeFile(storeLink, json.dump());
-
-    return hgInfo;
-}
-
 static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     std::string url;
-    std::string rev;
+    std::optional<Hash> rev;
+    std::optional<std::string> ref;
     std::string name = "source";
     PathSet context;
 
@@ -182,8 +26,15 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar
             string n(attr.name);
             if (n == "url")
                 url = state.coerceToString(*attr.pos, *attr.value, context, false, false);
-            else if (n == "rev")
-                rev = state.forceStringNoCtx(*attr.value, *attr.pos);
+            else if (n == "rev") {
+                // Ugly: unlike fetchGit, here the "rev" attribute can
+                // be both a revision or a branch/tag name.
+                auto value = state.forceStringNoCtx(*attr.value, *attr.pos);
+                if (std::regex_match(value, revRegex))
+                    rev = Hash(value, htSHA1);
+                else
+                    ref = value;
+            }
             else if (n == "name")
                 name = state.forceStringNoCtx(*attr.value, *attr.pos);
             else
@@ -200,18 +51,35 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar
     // whitelist. Ah well.
     state.checkURI(url);
 
-    auto hgInfo = exportMercurial(state.store, url, rev, name);
+    if (evalSettings.pureEval && !rev)
+        throw Error("in pure evaluation mode, 'fetchMercurial' requires a Mercurial revision");
+
+    fetchers::Attrs attrs;
+    attrs.insert_or_assign("type", "hg");
+    attrs.insert_or_assign("url", url.find("://") != std::string::npos ? url : "file://" + url);
+    if (ref) attrs.insert_or_assign("ref", *ref);
+    if (rev) attrs.insert_or_assign("rev", rev->gitRev());
+    auto input = fetchers::inputFromAttrs(attrs);
+
+    // FIXME: use name
+    auto [tree, input2] = input->fetchTree(state.store);
 
     state.mkAttrs(v, 8);
-    mkString(*state.allocAttr(v, state.sOutPath), hgInfo.storePath, PathSet({hgInfo.storePath}));
-    mkString(*state.allocAttr(v, state.symbols.create("branch")), hgInfo.branch);
-    mkString(*state.allocAttr(v, state.symbols.create("rev")), hgInfo.rev);
-    mkString(*state.allocAttr(v, state.symbols.create("shortRev")), std::string(hgInfo.rev, 0, 12));
-    mkInt(*state.allocAttr(v, state.symbols.create("revCount")), hgInfo.revCount);
+    auto storePath = state.store->printStorePath(tree.storePath);
+    mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath}));
+    if (input2->getRef())
+        mkString(*state.allocAttr(v, state.symbols.create("branch")), *input2->getRef());
+    // Backward compatibility: set 'rev' to
+    // 0000000000000000000000000000000000000000 for a dirty tree.
+    auto rev2 = input2->getRev().value_or(Hash(htSHA1));
+    mkString(*state.allocAttr(v, state.symbols.create("rev")), rev2.gitRev());
+    mkString(*state.allocAttr(v, state.symbols.create("shortRev")), std::string(rev2.gitRev(), 0, 12));
+    if (tree.info.revCount)
+        mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *tree.info.revCount);
     v.attrs->sort();
 
     if (state.allowedPaths)
-        state.allowedPaths->insert(state.store->toRealPath(hgInfo.storePath));
+        state.allowedPaths->insert(tree.actualPath);
 }
 
 static RegisterPrimOp r("fetchMercurial", 1, prim_fetchMercurial);

src/libexpr/primops/fetchTree.cc

@@ -0,0 +1,173 @@
+#include "primops.hh"
+#include "eval-inline.hh"
+#include "store-api.hh"
+#include "fetchers.hh"
+#include "filetransfer.hh"
+#include "registry.hh"
+
+#include <ctime>
+#include <iomanip>
+
+namespace nix {
+
+void emitTreeAttrs(
+    EvalState & state,
+    const fetchers::Tree & tree,
+    std::shared_ptr<const fetchers::Input> input,
+    Value & v)
+{
+    state.mkAttrs(v, 8);
+
+    auto storePath = state.store->printStorePath(tree.storePath);
+
+    mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath}));
+
+    assert(tree.info.narHash);
+    mkString(*state.allocAttr(v, state.symbols.create("narHash")),
+        tree.info.narHash.to_string(SRI));
+
+    if (input->getRev()) {
+        mkString(*state.allocAttr(v, state.symbols.create("rev")), input->getRev()->gitRev());
+        mkString(*state.allocAttr(v, state.symbols.create("shortRev")), input->getRev()->gitShortRev());
+    }
+
+    if (tree.info.revCount)
+        mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *tree.info.revCount);
+
+    if (tree.info.lastModified) {
+        mkInt(*state.allocAttr(v, state.symbols.create("lastModified")), *tree.info.lastModified);
+        mkString(*state.allocAttr(v, state.symbols.create("lastModifiedDate")),
+            fmt("%s", std::put_time(std::gmtime(&*tree.info.lastModified), "%Y%m%d%H%M%S")));
+    }
+
+    v.attrs->sort();
+}
+
+static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    settings.requireExperimentalFeature("flakes");
+
+    std::shared_ptr<const fetchers::Input> input;
+    PathSet context;
+
+    state.forceValue(*args[0]);
+
+    if (args[0]->type == tAttrs) {
+        state.forceAttrs(*args[0], pos);
+
+        fetchers::Attrs attrs;
+
+        for (auto & attr : *args[0]->attrs) {
+            state.forceValue(*attr.value);
+            if (attr.value->type == tString)
+                attrs.emplace(attr.name, attr.value->string.s);
+            else if (attr.value->type == tBool)
+                attrs.emplace(attr.name, attr.value->boolean);
+            else if (attr.value->type == tInt)
+                attrs.emplace(attr.name, attr.value->integer);
+            else
+                throw TypeError("fetchTree argument '%s' is %s while a string, Boolean or integer is expected",
+                    attr.name, showType(*attr.value));
+        }
+
+        if (!attrs.count("type"))
+            throw Error("attribute 'type' is missing in call to 'fetchTree', at %s", pos);
+
+        input = fetchers::inputFromAttrs(attrs);
+    } else
+        input = fetchers::inputFromURL(state.coerceToString(pos, *args[0], context, false, false));
+
+    if (!evalSettings.pureEval && !input->isDirect())
+        input = lookupInRegistries(state.store, input).first;
+
+    if (evalSettings.pureEval && !input->isImmutable())
+        throw Error("in pure evaluation mode, 'fetchTree' requires an immutable input, at %s", pos);
+
+    // FIXME: use fetchOrSubstituteTree
+    auto [tree, input2] = input->fetchTree(state.store);
+
+    if (state.allowedPaths)
+        state.allowedPaths->insert(tree.actualPath);
+
+    emitTreeAttrs(state, tree, input2, v);
+}
+
+static RegisterPrimOp r("fetchTree", 1, prim_fetchTree);
+
+static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
+    const string & who, bool unpack, std::string name)
+{
+    std::optional<std::string> url;
+    std::optional<Hash> expectedHash;
+
+    state.forceValue(*args[0]);
+
+    if (args[0]->type == tAttrs) {
+
+        state.forceAttrs(*args[0], pos);
+
+        for (auto & attr : *args[0]->attrs) {
+            string n(attr.name);
+            if (n == "url")
+                url = state.forceStringNoCtx(*attr.value, *attr.pos);
+            else if (n == "sha256")
+                expectedHash = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256);
+            else if (n == "name")
+                name = state.forceStringNoCtx(*attr.value, *attr.pos);
+            else
+                throw EvalError("unsupported argument '%s' to '%s', at %s",
+                    attr.name, who, *attr.pos);
+        }
+
+        if (!url)
+            throw EvalError("'url' argument required, at %s", pos);
+
+    } else
+        url = state.forceStringNoCtx(*args[0], pos);
+
+    url = resolveUri(*url);
+
+    state.checkURI(*url);
+
+    if (name == "")
+        name = baseNameOf(*url);
+
+    if (evalSettings.pureEval && !expectedHash)
+        throw Error("in pure evaluation mode, '%s' requires a 'sha256' argument", who);
+
+    auto storePath =
+        unpack
+        ? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).storePath
+        : fetchers::downloadFile(state.store, *url, name, (bool) expectedHash).storePath;
+
+    auto path = state.store->toRealPath(storePath);
+
+    if (expectedHash) {
+        auto hash = unpack
+            ? state.store->queryPathInfo(storePath)->narHash
+            : hashFile(htSHA256, path);
+        if (hash != *expectedHash)
+            throw Error((unsigned int) 102, "hash mismatch in file downloaded from '%s':\n  wanted: %s\n  got:    %s",
+                *url, expectedHash->to_string(), hash.to_string());
+    }
+
+    if (state.allowedPaths)
+        state.allowedPaths->insert(path);
+
+    mkString(v, path, PathSet({path}));
+}
+
+static void prim_fetchurl(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    fetch(state, pos, args, v, "fetchurl", false, "");
+}
+
+static void prim_fetchTarball(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    fetch(state, pos, args, v, "fetchTarball", true, "source");
+}
+
+static RegisterPrimOp r2("__fetchurl", 1, prim_fetchurl);
+static RegisterPrimOp r3("fetchTarball", 1, prim_fetchTarball);
+
+}

src/libexpr/primops/fromTOML.cc

@@ -1,7 +1,7 @@
 #include "primops.hh"
 #include "eval-inline.hh"
 
-#include "cpptoml/cpptoml.h"
+#include "../../cpptoml/cpptoml.h"
 
 namespace nix {
 

src/libexpr/symbol-table.hh

@@ -28,6 +28,12 @@ public:
         return s == s2.s;
     }
 
+    // FIXME: remove
+    bool operator == (std::string_view s2) const
+    {
+        return s->compare(s2) == 0;
+    }
+
     bool operator != (const Symbol & s2) const
     {
         return s != s2.s;
@@ -68,9 +74,10 @@ private:
     Symbols symbols;
 
 public:
-    Symbol create(const string & s)
+    Symbol create(std::string_view s)
     {
-        std::pair<Symbols::iterator, bool> res = symbols.insert(s);
+        // FIXME: avoid allocation if 's' already exists in the symbol table.
+        std::pair<Symbols::iterator, bool> res = symbols.emplace(std::string(s));
         return Symbol(&*res.first);
     }
 

src/libexpr/value.hh

@@ -166,6 +166,11 @@ struct Value
     {
         return type == tList1 ? 1 : type == tList2 ? 2 : bigList.size;
     }
+
+    /* Check whether forcing this value requires a trivial amount of
+       computation. In particular, function applications are
+       non-trivial. */
+    bool isTrivial() const;
 };
 
 
@@ -253,12 +258,17 @@ void mkPath(Value & v, const char * s);
 
 
 #if HAVE_BOEHMGC
-typedef std::vector<Value *, gc_allocator<Value *> > ValueVector;
-typedef std::map<Symbol, Value *, std::less<Symbol>, gc_allocator<std::pair<const Symbol, Value *> > > ValueMap;
+typedef std::vector<Value *, traceable_allocator<Value *> > ValueVector;
+typedef std::map<Symbol, Value *, std::less<Symbol>, traceable_allocator<std::pair<const Symbol, Value *> > > ValueMap;
 #else
 typedef std::vector<Value *> ValueVector;
 typedef std::map<Symbol, Value *> ValueMap;
 #endif
 
 
+/* A value allocated in traceable memory. */
+typedef std::shared_ptr<Value *> RootValue;
+
+RootValue allocRootValue(Value * v);
+
 }

src/libfetchers/attrs.cc

@@ -0,0 +1,107 @@
+#include "attrs.hh"
+#include "fetchers.hh"
+
+#include <nlohmann/json.hpp>
+
+namespace nix::fetchers {
+
+Attrs jsonToAttrs(const nlohmann::json & json)
+{
+    Attrs attrs;
+
+    for (auto & i : json.items()) {
+        if (i.value().is_number())
+            attrs.emplace(i.key(), i.value().get<int64_t>());
+        else if (i.value().is_string())
+            attrs.emplace(i.key(), i.value().get<std::string>());
+        else if (i.value().is_boolean())
+            attrs.emplace(i.key(), i.value().get<bool>());
+        else
+            throw Error("unsupported input attribute type in lock file");
+    }
+
+    return attrs;
+}
+
+nlohmann::json attrsToJson(const Attrs & attrs)
+{
+    nlohmann::json json;
+    for (auto & attr : attrs) {
+        if (auto v = std::get_if<int64_t>(&attr.second)) {
+            json[attr.first] = *v;
+        } else if (auto v = std::get_if<std::string>(&attr.second)) {
+            json[attr.first] = *v;
+        } else if (auto v = std::get_if<Explicit<bool>>(&attr.second)) {
+            json[attr.first] = v->t;
+        } else abort();
+    }
+    return json;
+}
+
+std::optional<std::string> maybeGetStrAttr(const Attrs & attrs, const std::string & name)
+{
+    auto i = attrs.find(name);
+    if (i == attrs.end()) return {};
+    if (auto v = std::get_if<std::string>(&i->second))
+        return *v;
+    throw Error("input attribute '%s' is not a string %s", name, attrsToJson(attrs).dump());
+}
+
+std::string getStrAttr(const Attrs & attrs, const std::string & name)
+{
+    auto s = maybeGetStrAttr(attrs, name);
+    if (!s)
+        throw Error("input attribute '%s' is missing", name);
+    return *s;
+}
+
+std::optional<int64_t> maybeGetIntAttr(const Attrs & attrs, const std::string & name)
+{
+    auto i = attrs.find(name);
+    if (i == attrs.end()) return {};
+    if (auto v = std::get_if<int64_t>(&i->second))
+        return *v;
+    throw Error("input attribute '%s' is not an integer", name);
+}
+
+int64_t getIntAttr(const Attrs & attrs, const std::string & name)
+{
+    auto s = maybeGetIntAttr(attrs, name);
+    if (!s)
+        throw Error("input attribute '%s' is missing", name);
+    return *s;
+}
+
+std::optional<bool> maybeGetBoolAttr(const Attrs & attrs, const std::string & name)
+{
+    auto i = attrs.find(name);
+    if (i == attrs.end()) return {};
+    if (auto v = std::get_if<int64_t>(&i->second))
+        return *v;
+    throw Error("input attribute '%s' is not a Boolean", name);
+}
+
+bool getBoolAttr(const Attrs & attrs, const std::string & name)
+{
+    auto s = maybeGetBoolAttr(attrs, name);
+    if (!s)
+        throw Error("input attribute '%s' is missing", name);
+    return *s;
+}
+
+std::map<std::string, std::string> attrsToQuery(const Attrs & attrs)
+{
+    std::map<std::string, std::string> query;
+    for (auto & attr : attrs) {
+        if (auto v = std::get_if<int64_t>(&attr.second)) {
+            query.insert_or_assign(attr.first, fmt("%d", *v));
+        } else if (auto v = std::get_if<std::string>(&attr.second)) {
+            query.insert_or_assign(attr.first, *v);
+        } else if (auto v = std::get_if<Explicit<bool>>(&attr.second)) {
+            query.insert_or_assign(attr.first, v->t ? "1" : "0");
+        } else abort();
+    }
+    return query;
+}
+
+}

src/libfetchers/attrs.hh

@@ -0,0 +1,39 @@
+#pragma once
+
+#include "types.hh"
+
+#include <variant>
+
+#include <nlohmann/json_fwd.hpp>
+
+namespace nix::fetchers {
+
+/* Wrap bools to prevent string literals (i.e. 'char *') from being
+   cast to a bool in Attr. */
+template<typename T>
+struct Explicit {
+    T t;
+};
+
+typedef std::variant<std::string, int64_t, Explicit<bool>> Attr;
+typedef std::map<std::string, Attr> Attrs;
+
+Attrs jsonToAttrs(const nlohmann::json & json);
+
+nlohmann::json attrsToJson(const Attrs & attrs);
+
+std::optional<std::string> maybeGetStrAttr(const Attrs & attrs, const std::string & name);
+
+std::string getStrAttr(const Attrs & attrs, const std::string & name);
+
+std::optional<int64_t> maybeGetIntAttr(const Attrs & attrs, const std::string & name);
+
+int64_t getIntAttr(const Attrs & attrs, const std::string & name);
+
+std::optional<bool> maybeGetBoolAttr(const Attrs & attrs, const std::string & name);
+
+bool getBoolAttr(const Attrs & attrs, const std::string & name);
+
+std::map<std::string, std::string> attrsToQuery(const Attrs & attrs);
+
+}

src/libfetchers/cache.cc

@@ -0,0 +1,121 @@
+#include "cache.hh"
+#include "sqlite.hh"
+#include "sync.hh"
+#include "store-api.hh"
+
+#include <nlohmann/json.hpp>
+
+namespace nix::fetchers {
+
+static const char * schema = R"sql(
+
+create table if not exists Cache (
+    input     text not null,
+    info      text not null,
+    path      text not null,
+    immutable integer not null,
+    timestamp integer not null,
+    primary key (input)
+);
+)sql";
+
+struct CacheImpl : Cache
+{
+    struct State
+    {
+        SQLite db;
+        SQLiteStmt add, lookup;
+    };
+
+    Sync<State> _state;
+
+    CacheImpl()
+    {
+        auto state(_state.lock());
+
+        auto dbPath = getCacheDir() + "/nix/fetcher-cache-v1.sqlite";
+        createDirs(dirOf(dbPath));
+
+        state->db = SQLite(dbPath);
+        state->db.isCache();
+        state->db.exec(schema);
+
+        state->add.create(state->db,
+            "insert or replace into Cache(input, info, path, immutable, timestamp) values (?, ?, ?, ?, ?)");
+
+        state->lookup.create(state->db,
+            "select info, path, immutable, timestamp from Cache where input = ?");
+    }
+
+    void add(
+        ref<Store> store,
+        const Attrs & inAttrs,
+        const Attrs & infoAttrs,
+        const StorePath & storePath,
+        bool immutable) override
+    {
+        _state.lock()->add.use()
+            (attrsToJson(inAttrs).dump())
+            (attrsToJson(infoAttrs).dump())
+            (store->printStorePath(storePath))
+            (immutable)
+            (time(0)).exec();
+    }
+
+    std::optional<std::pair<Attrs, StorePath>> lookup(
+        ref<Store> store,
+        const Attrs & inAttrs) override
+    {
+        if (auto res = lookupExpired(store, inAttrs)) {
+            if (!res->expired)
+                return std::make_pair(std::move(res->infoAttrs), std::move(res->storePath));
+            debug("ignoring expired cache entry '%s'",
+                attrsToJson(inAttrs).dump());
+        }
+        return {};
+    }
+
+    std::optional<Result> lookupExpired(
+        ref<Store> store,
+        const Attrs & inAttrs) override
+    {
+        auto state(_state.lock());
+
+        auto inAttrsJson = attrsToJson(inAttrs).dump();
+
+        auto stmt(state->lookup.use()(inAttrsJson));
+        if (!stmt.next()) {
+            debug("did not find cache entry for '%s'", inAttrsJson);
+            return {};
+        }
+
+        auto infoJson = stmt.getStr(0);
+        auto storePath = store->parseStorePath(stmt.getStr(1));
+        auto immutable = stmt.getInt(2) != 0;
+        auto timestamp = stmt.getInt(3);
+
+        store->addTempRoot(storePath);
+        if (!store->isValidPath(storePath)) {
+            // FIXME: we could try to substitute 'storePath'.
+            debug("ignoring disappeared cache entry '%s'", inAttrsJson);
+            return {};
+        }
+
+        debug("using cache entry '%s' -> '%s', '%s'",
+            inAttrsJson, infoJson, store->printStorePath(storePath));
+
+        return Result {
+            .expired = !immutable && (settings.tarballTtl.get() == 0 || timestamp + settings.tarballTtl < time(0)),
+            .infoAttrs = jsonToAttrs(nlohmann::json::parse(infoJson)),
+            .storePath = std::move(storePath)
+        };
+    }
+};
+
+ref<Cache> getCache()
+{
+    static auto cache = std::make_shared<CacheImpl>();
+    return ref<Cache>(cache);
+}
+
+}

src/libfetchers/cache.hh

@@ -0,0 +1,34 @@
+#pragma once
+
+#include "fetchers.hh"
+
+namespace nix::fetchers {
+
+struct Cache
+{
+    virtual void add(
+        ref<Store> store,
+        const Attrs & inAttrs,
+        const Attrs & infoAttrs,
+        const StorePath & storePath,
+        bool immutable) = 0;
+
+    virtual std::optional<std::pair<Attrs, StorePath>> lookup(
+        ref<Store> store,
+        const Attrs & inAttrs) = 0;
+
+    struct Result
+    {
+        bool expired = false;
+        Attrs infoAttrs;
+        StorePath storePath;
+    };
+
+    virtual std::optional<Result> lookupExpired(
+        ref<Store> store,
+        const Attrs & inAttrs) = 0;
+};
+
+ref<Cache> getCache();
+
+}

src/libfetchers/fetchers.cc

@@ -0,0 +1,86 @@
+#include "fetchers.hh"
+#include "store-api.hh"
+
+#include <nlohmann/json.hpp>
+
+namespace nix::fetchers {
+
+std::unique_ptr<std::vector<std::unique_ptr<InputScheme>>> inputSchemes = nullptr;
+
+void registerInputScheme(std::unique_ptr<InputScheme> && inputScheme)
+{
+    if (!inputSchemes) inputSchemes = std::make_unique<std::vector<std::unique_ptr<InputScheme>>>();
+    inputSchemes->push_back(std::move(inputScheme));
+}
+
+std::unique_ptr<Input> inputFromURL(const ParsedURL & url)
+{
+    for (auto & inputScheme : *inputSchemes) {
+        auto res = inputScheme->inputFromURL(url);
+        if (res) return res;
+    }
+    throw Error("input '%s' is unsupported", url.url);
+}
+
+std::unique_ptr<Input> inputFromURL(const std::string & url)
+{
+    return inputFromURL(parseURL(url));
+}
+
+std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs)
+{
+    auto attrs2(attrs);
+    attrs2.erase("narHash");
+    for (auto & inputScheme : *inputSchemes) {
+        auto res = inputScheme->inputFromAttrs(attrs2);
+        if (res) {
+            if (auto narHash = maybeGetStrAttr(attrs, "narHash"))
+                // FIXME: require SRI hash.
+                res->narHash = Hash(*narHash);
+            return res;
+        }
+    }
+    throw Error("input '%s' is unsupported", attrsToJson(attrs));
+}
+
+Attrs Input::toAttrs() const
+{
+    auto attrs = toAttrsInternal();
+    if (narHash)
+        attrs.emplace("narHash", narHash->to_string(SRI));
+    attrs.emplace("type", type());
+    return attrs;
+}
+
+std::pair<Tree, std::shared_ptr<const Input>> Input::fetchTree(ref<Store> store) const
+{
+    auto [tree, input] = fetchTreeInternal(store);
+
+    if (tree.actualPath == "")
+        tree.actualPath = store->toRealPath(tree.storePath);
+
+    if (!tree.info.narHash)
+        tree.info.narHash = store->queryPathInfo(tree.storePath)->narHash;
+
+    if (input->narHash)
+        assert(input->narHash == tree.info.narHash);
+
+    if (narHash && narHash != input->narHash)
+        throw Error("NAR hash mismatch in input '%s' (%s), expected '%s', got '%s'",
+            to_string(), tree.actualPath, narHash->to_string(SRI), input->narHash->to_string(SRI));
+
+    return {std::move(tree), input};
+}
+
+std::shared_ptr<const Input> Input::applyOverrides(
+    std::optional<std::string> ref,
+    std::optional<Hash> rev) const
+{
+    if (ref)
+        throw Error("don't know how to apply '%s' to '%s'", *ref, to_string());
+    if (rev)
+        throw Error("don't know how to apply '%s' to '%s'", rev->to_string(Base16, false), to_string());
+    return shared_from_this();
+}
+
+}

src/libfetchers/fetchers.hh

@@ -0,0 +1,119 @@
+#pragma once
+
+#include "types.hh"
+#include "hash.hh"
+#include "path.hh"
+#include "tree-info.hh"
+#include "attrs.hh"
+#include "url.hh"
+
+#include <memory>
+
+namespace nix { class Store; }
+
+namespace nix::fetchers {
+
+struct Input;
+
+struct Tree
+{
+    Path actualPath;
+    StorePath storePath;
+    TreeInfo info;
+};
+
+struct Input : std::enable_shared_from_this<Input>
+{
+    std::optional<Hash> narHash; // FIXME: implement
+
+    virtual std::string type() const = 0;
+
+    virtual ~Input() { }
+
+    virtual bool operator ==(const Input & other) const { return false; }
+
+    /* Check whether this is a "direct" input, that is, not
+       one that goes through a registry. */
+    virtual bool isDirect() const { return true; }
+
+    /* Check whether this is an "immutable" input, that is,
+       one that contains a commit hash or content hash. */
+    virtual bool isImmutable() const { return (bool) narHash; }
+
+    virtual bool contains(const Input & other) const { return false; }
+
+    virtual std::optional<std::string> getRef() const { return {}; }
+
+    virtual std::optional<Hash> getRev() const { return {}; }
+
+    virtual ParsedURL toURL() const = 0;
+
+    std::string to_string() const
+    {
+        return toURL().to_string();
+    }
+
+    Attrs toAttrs() const;
+
+    std::pair<Tree, std::shared_ptr<const Input>> fetchTree(ref<Store> store) const;
+
+    virtual std::shared_ptr<const Input> applyOverrides(
+        std::optional<std::string> ref,
+        std::optional<Hash> rev) const;
+
+    virtual std::optional<Path> getSourcePath() const { return {}; }
+
+    virtual void markChangedFile(
+        std::string_view file,
+        std::optional<std::string> commitMsg) const
+    { assert(false); }
+
+    virtual void clone(const Path & destDir) const
+    {
+        throw Error("do not know how to clone input '%s'", to_string());
+    }
+
+private:
+
+    virtual std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(ref<Store> store) const = 0;
+
+    virtual Attrs toAttrsInternal() const = 0;
+};
+
+struct InputScheme
+{
+    virtual ~InputScheme() { }
+
+    virtual std::unique_ptr<Input> inputFromURL(const ParsedURL & url) = 0;
+
+    virtual std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) = 0;
+};
+
+std::unique_ptr<Input> inputFromURL(const ParsedURL & url);
+
+std::unique_ptr<Input> inputFromURL(const std::string & url);
+
+std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs);
+
+void registerInputScheme(std::unique_ptr<InputScheme> && fetcher);
+
+struct DownloadFileResult
+{
+    StorePath storePath;
+    std::string etag;
+    std::string effectiveUrl;
+};
+
+DownloadFileResult downloadFile(
+    ref<Store> store,
+    const std::string & url,
+    const std::string & name,
+    bool immutable);
+
+Tree downloadTarball(
+    ref<Store> store,
+    const std::string & url,
+    const std::string & name,
+    bool immutable);
+
+}

src/libfetchers/git.cc

@@ -0,0 +1,495 @@
+#include "fetchers.hh"
+#include "cache.hh"
+#include "globals.hh"
+#include "tarfile.hh"
+#include "store-api.hh"
+
+#include <sys/time.h>
+
+using namespace std::string_literals;
+
+namespace nix::fetchers {
+
+static std::string readHead(const Path & path)
+{
+    return chomp(runProgram("git", true, { "-C", path, "rev-parse", "--abbrev-ref", "HEAD" }));
+}
+
+static bool isNotDotGitDirectory(const Path & path)
+{
+    static const std::regex gitDirRegex("^(?:.*/)?\\.git$");
+
+    return not std::regex_match(path, gitDirRegex);
+}
+
+struct GitInput : Input
+{
+    ParsedURL url;
+    std::optional<std::string> ref;
+    std::optional<Hash> rev;
+    bool shallow = false;
+    bool submodules = false;
+
+    GitInput(const ParsedURL & url) : url(url)
+    { }
+
+    std::string type() const override { return "git"; }
+
+    bool operator ==(const Input & other) const override
+    {
+        auto other2 = dynamic_cast<const GitInput *>(&other);
+        return
+            other2
+            && url == other2->url
+            && rev == other2->rev
+            && ref == other2->ref;
+    }
+
+    bool isImmutable() const override
+    {
+        return (bool) rev || narHash;
+    }
+
+    std::optional<std::string> getRef() const override { return ref; }
+
+    std::optional<Hash> getRev() const override { return rev; }
+
+    ParsedURL toURL() const override
+    {
+        ParsedURL url2(url);
+        if (url2.scheme != "git") url2.scheme = "git+" + url2.scheme;
+        if (rev) url2.query.insert_or_assign("rev", rev->gitRev());
+        if (ref) url2.query.insert_or_assign("ref", *ref);
+        if (shallow) url2.query.insert_or_assign("shallow", "1");
+        return url2;
+    }
+
+    Attrs toAttrsInternal() const override
+    {
+        Attrs attrs;
+        attrs.emplace("url", url.to_string());
+        if (ref)
+            attrs.emplace("ref", *ref);
+        if (rev)
+            attrs.emplace("rev", rev->gitRev());
+        if (shallow)
+            attrs.emplace("shallow", true);
+        if (submodules)
+            attrs.emplace("submodules", true);
+        return attrs;
+    }
+
+    void clone(const Path & destDir) const override
+    {
+        auto [isLocal, actualUrl] = getActualUrl();
+
+        Strings args = {"clone"};
+
+        args.push_back(actualUrl);
+
+        if (ref) {
+            args.push_back("--branch");
+            args.push_back(*ref);
+        }
+
+        if (rev) throw Error("cloning a specific revision is not implemented");
+
+        args.push_back(destDir);
+
+        runProgram("git", true, args);
+    }
+
+    std::shared_ptr<const Input> applyOverrides(
+        std::optional<std::string> ref,
+        std::optional<Hash> rev) const override
+    {
+        if (!ref && !rev) return shared_from_this();
+
+        auto res = std::make_shared<GitInput>(*this);
+
+        if (ref) res->ref = ref;
+        if (rev) res->rev = rev;
+
+        if (!res->ref && res->rev)
+            throw Error("Git input '%s' has a commit hash but no branch/tag name", res->to_string());
+
+        return res;
+    }
+
+    std::optional<Path> getSourcePath() const override
+    {
+        if (url.scheme == "file" && !ref && !rev)
+            return url.path;
+        return {};
+    }
+
+    void markChangedFile(std::string_view file, std::optional<std::string> commitMsg) const override
+    {
+        auto sourcePath = getSourcePath();
+        assert(sourcePath);
+
+        runProgram("git", true,
+            { "-C", *sourcePath, "add", "--force", "--intent-to-add", std::string(file) });
+
+        if (commitMsg)
+            runProgram("git", true,
+                { "-C", *sourcePath, "commit", std::string(file), "-m", *commitMsg });
+    }
+
+    std::pair<bool, std::string> getActualUrl() const
+    {
+        // Don't clone file:// URIs (but otherwise treat them the
+        // same as remote URIs, i.e. don't use the working tree or
+        // HEAD).
+        static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; // for testing
+        bool isLocal = url.scheme == "file" && !forceHttp;
+        return {isLocal, isLocal ? url.path : url.base};
+    }
+
+    std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override
+    {
+        auto name = "source";
+
+        auto input = std::make_shared<GitInput>(*this);
+
+        assert(!rev || rev->type == htSHA1);
+
+        std::string cacheType = "git";
+        if (shallow) cacheType += "-shallow";
+        if (submodules) cacheType += "-submodules";
+
+        auto getImmutableAttrs = [&]()
+        {
+            return Attrs({
+                {"type", cacheType},
+                {"name", name},
+                {"rev", input->rev->gitRev()},
+            });
+        };
+
+        auto makeResult = [&](const Attrs & infoAttrs, StorePath && storePath)
+            -> std::pair<Tree, std::shared_ptr<const Input>>
+        {
+            assert(input->rev);
+            assert(!rev || rev == input->rev);
+            return {
+                Tree {
+                    .actualPath = store->toRealPath(storePath),
+                    .storePath = std::move(storePath),
+                    .info = TreeInfo {
+                        .revCount = shallow ? std::nullopt : std::optional(getIntAttr(infoAttrs, "revCount")),
+                        .lastModified = getIntAttr(infoAttrs, "lastModified"),
+                    },
+                },
+                input
+            };
+        };
+
+        if (rev) {
+            if (auto res = getCache()->lookup(store, getImmutableAttrs()))
+                return makeResult(res->first, std::move(res->second));
+        }
+
+        auto [isLocal, actualUrl_] = getActualUrl();
+        auto actualUrl = actualUrl_; // work around clang bug
+
+        // If this is a local directory and no ref or revision is
+        // given, then allow the use of an unclean working tree.
+        if (!input->ref && !input->rev && isLocal) {
+            bool clean = false;
+
+            /* Check whether this repo has any commits. There are
+               probably better ways to do this. */
+            auto gitDir = actualUrl + "/.git";
+            auto commonGitDir = chomp(runProgram(
+                "git",
+                true,
+                { "-C", actualUrl, "rev-parse", "--git-common-dir" }
+            ));
+            if (commonGitDir != ".git")
+                    gitDir = commonGitDir;
+
+            bool haveCommits = !readDirectory(gitDir + "/refs/heads").empty();
+
+            try {
+                if (haveCommits) {
+                    runProgram("git", true, { "-C", actualUrl, "diff-index", "--quiet", "HEAD", "--" });
+                    clean = true;
+                }
+            } catch (ExecError & e) {
+                if (!WIFEXITED(e.status) || WEXITSTATUS(e.status) != 1) throw;
+            }
+
+            if (!clean) {
+
+                /* This is an unclean working tree. So copy all tracked files. */
+
+                if (!settings.allowDirty)
+                    throw Error("Git tree '%s' is dirty", actualUrl);
+
+                if (settings.warnDirty)
+                    warn("Git tree '%s' is dirty", actualUrl);
+
+                auto gitOpts = Strings({ "-C", actualUrl, "ls-files", "-z" });
+                if (submodules)
+                    gitOpts.emplace_back("--recurse-submodules");
+
+                auto files = tokenizeString<std::set<std::string>>(
+                    runProgram("git", true, gitOpts), "\0"s);
+
+                PathFilter filter = [&](const Path & p) -> bool {
+                    assert(hasPrefix(p, actualUrl));
+                    std::string file(p, actualUrl.size() + 1);
+
+                    auto st = lstat(p);
+
+                    if (S_ISDIR(st.st_mode)) {
+                        auto prefix = file + "/";
+                        auto i = files.lower_bound(prefix);
+                        return i != files.end() && hasPrefix(*i, prefix);
+                    }
+
+                    return files.count(file);
+                };
+
+                auto storePath = store->addToStore("source", actualUrl, true, htSHA256, filter);
+
+                auto tree = Tree {
+                    .actualPath = store->printStorePath(storePath),
+                    .storePath = std::move(storePath),
+                    .info = TreeInfo {
+                        // FIXME: maybe we should use the timestamp of the last
+                        // modified dirty file?
+                        .lastModified = haveCommits ? std::stoull(runProgram("git", true, { "-C", actualUrl, "log", "-1", "--format=%ct", "HEAD" })) : 0,
+                    }
+                };
+
+                return {std::move(tree), input};
+            }
+        }
+
+        if (!input->ref) input->ref = isLocal ? readHead(actualUrl) : "master";
+
+        Attrs mutableAttrs({
+            {"type", cacheType},
+            {"name", name},
+            {"url", actualUrl},
+            {"ref", *input->ref},
+        });
+
+        Path repoDir;
+
+        if (isLocal) {
+
+            if (!input->rev)
+                input->rev = Hash(chomp(runProgram("git", true, { "-C", actualUrl, "rev-parse", *input->ref })), htSHA1);
+
+            repoDir = actualUrl;
+
+        } else {
+
+            if (auto res = getCache()->lookup(store, mutableAttrs)) {
+                auto rev2 = Hash(getStrAttr(res->first, "rev"), htSHA1);
+                if (!rev || rev == rev2) {
+                    input->rev = rev2;
+                    return makeResult(res->first, std::move(res->second));
+                }
+            }
+
+            Path cacheDir = getCacheDir() + "/nix/gitv3/" + hashString(htSHA256, actualUrl).to_string(Base32, false);
+            repoDir = cacheDir;
+
+            if (!pathExists(cacheDir)) {
+                createDirs(dirOf(cacheDir));
+                runProgram("git", true, { "init", "--bare", repoDir });
+            }
+
+            Path localRefFile =
+                input->ref->compare(0, 5, "refs/") == 0
+                ? cacheDir + "/" + *input->ref
+                : cacheDir + "/refs/heads/" + *input->ref;
+
+            bool doFetch;
+            time_t now = time(0);
+
+            /* If a rev was specified, we need to fetch if it's not in the
+               repo. */
+            if (input->rev) {
+                try {
+                    runProgram("git", true, { "-C", repoDir, "cat-file", "-e", input->rev->gitRev() });
+                    doFetch = false;
+                } catch (ExecError & e) {
+                    if (WIFEXITED(e.status)) {
+                        doFetch = true;
+                    } else {
+                        throw;
+                    }
+                }
+            } else {
+                /* If the local ref is older than ‘tarball-ttl’ seconds, do a
+                   git fetch to update the local ref to the remote ref. */
+                struct stat st;
+                doFetch = stat(localRefFile.c_str(), &st) != 0 ||
+                    (uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now;
+            }
+
+            if (doFetch) {
+                Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Git repository '%s'", actualUrl));
+
+                // FIXME: git stderr messes up our progress indicator, so
+                // we're using --quiet for now. Should process its stderr.
+                try {
+                    runProgram("git", true, { "-C", repoDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", *input->ref, *input->ref) });
+                } catch (Error & e) {
+                    if (!pathExists(localRefFile)) throw;
+                    warn("could not update local clone of Git repository '%s'; continuing with the most recent version", actualUrl);
+                }
+
+                struct timeval times[2];
+                times[0].tv_sec = now;
+                times[0].tv_usec = 0;
+                times[1].tv_sec = now;
+                times[1].tv_usec = 0;
+
+                utimes(localRefFile.c_str(), times);
+            }
+
+            if (!input->rev)
+                input->rev = Hash(chomp(readFile(localRefFile)), htSHA1);
+        }
+
+        bool isShallow = chomp(runProgram("git", true, { "-C", repoDir, "rev-parse", "--is-shallow-repository" })) == "true";
+
+        if (isShallow && !shallow)
+            throw Error("'%s' is a shallow Git repository, but a non-shallow repository is needed", actualUrl);
+
+        // FIXME: check whether rev is an ancestor of ref.
+
+        printTalkative("using revision %s of repo '%s'", input->rev->gitRev(), actualUrl);
+
+        /* Now that we know the ref, check again whether we have it in
+           the store. */
+        if (auto res = getCache()->lookup(store, getImmutableAttrs()))
+            return makeResult(res->first, std::move(res->second));
+
+        Path tmpDir = createTempDir();
+        AutoDelete delTmpDir(tmpDir, true);
+        PathFilter filter = defaultPathFilter;
+
+        if (submodules) {
+            Path tmpGitDir = createTempDir();
+            AutoDelete delTmpGitDir(tmpGitDir, true);
+
+            runProgram("git", true, { "init", tmpDir, "--separate-git-dir", tmpGitDir });
+            // TODO: repoDir might lack the ref (it only checks if rev
+            // exists, see FIXME above) so use a big hammer and fetch
+            // everything to ensure we get the rev.
+            runProgram("git", true, { "-C", tmpDir, "fetch", "--quiet", "--force",
+                                      "--update-head-ok", "--", repoDir, "refs/*:refs/*" });
+
+            runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", input->rev->gitRev() });
+            runProgram("git", true, { "-C", tmpDir, "remote", "add", "origin", actualUrl });
+            runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init", "--recursive" });
+
+            filter = isNotDotGitDirectory;
+        } else {
+            // FIXME: should pipe this, or find some better way to extract a
+            // revision.
+            auto source = sinkToSource([&](Sink & sink) {
+                RunOptions gitOptions("git", { "-C", repoDir, "archive", input->rev->gitRev() });
+                gitOptions.standardOut = &sink;
+                runProgram2(gitOptions);
+            });
+
+            unpackTarfile(*source, tmpDir);
+        }
+
+        auto storePath = store->addToStore(name, tmpDir, true, htSHA256, filter);
+
+        auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "log", "-1", "--format=%ct", input->rev->gitRev() }));
+
+        Attrs infoAttrs({
+            {"rev", input->rev->gitRev()},
+            {"lastModified", lastModified},
+        });
+
+        if (!shallow)
+            infoAttrs.insert_or_assign("revCount",
+                std::stoull(runProgram("git", true, { "-C", repoDir, "rev-list", "--count", input->rev->gitRev() })));
+
+        if (!this->rev)
+            getCache()->add(
+                store,
+                mutableAttrs,
+                infoAttrs,
+                storePath,
+                false);
+
+        getCache()->add(
+            store,
+            getImmutableAttrs(),
+            infoAttrs,
+            storePath,
+            true);
+
+        return makeResult(infoAttrs, std::move(storePath));
+    }
+};
+
+struct GitInputScheme : InputScheme
+{
+    std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override
+    {
+        if (url.scheme != "git" &&
+            url.scheme != "git+http" &&
+            url.scheme != "git+https" &&
+            url.scheme != "git+ssh" &&
+            url.scheme != "git+file") return nullptr;
+
+        auto url2(url);
+        if (hasPrefix(url2.scheme, "git+")) url2.scheme = std::string(url2.scheme, 4);
+        url2.query.clear();
+
+        Attrs attrs;
+        attrs.emplace("type", "git");
+
+        for (auto &[name, value] : url.query) {
+            if (name == "rev" || name == "ref")
+                attrs.emplace(name, value);
+            else
+                url2.query.emplace(name, value);
+        }
+
+        attrs.emplace("url", url2.to_string());
+
+        return inputFromAttrs(attrs);
+    }
+
+    std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override
+    {
+        if (maybeGetStrAttr(attrs, "type") != "git") return {};
+
+        for (auto & [name, value] : attrs)
+            if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules")
+                throw Error("unsupported Git input attribute '%s'", name);
+
+        auto input = std::make_unique<GitInput>(parseURL(getStrAttr(attrs, "url")));
+        if (auto ref = maybeGetStrAttr(attrs, "ref")) {
+            if (!std::regex_match(*ref, refRegex))
+                throw BadURL("invalid Git branch/tag name '%s'", *ref);
+            input->ref = *ref;
+        }
+        if (auto rev = maybeGetStrAttr(attrs, "rev"))
+            input->rev = Hash(*rev, htSHA1);
+
+        input->shallow = maybeGetBoolAttr(attrs, "shallow").value_or(false);
+
+        input->submodules = maybeGetBoolAttr(attrs, "submodules").value_or(false);
+
+        return input;
+    }
+};
+
+static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<GitInputScheme>()); });
+
+}

src/libfetchers/github.cc

@@ -0,0 +1,216 @@
+#include "filetransfer.hh"
+#include "cache.hh"
+#include "fetchers.hh"
+#include "globals.hh"
+#include "store-api.hh"
+
+#include <nlohmann/json.hpp>
+
+namespace nix::fetchers {
+
+std::regex ownerRegex("[a-zA-Z][a-zA-Z0-9_-]*", std::regex::ECMAScript);
+std::regex repoRegex("[a-zA-Z][a-zA-Z0-9_-]*", std::regex::ECMAScript);
+
+struct GitHubInput : Input
+{
+    std::string owner;
+    std::string repo;
+    std::optional<std::string> ref;
+    std::optional<Hash> rev;
+
+    std::string type() const override { return "github"; }
+
+    bool operator ==(const Input & other) const override
+    {
+        auto other2 = dynamic_cast<const GitHubInput *>(&other);
+        return
+            other2
+            && owner == other2->owner
+            && repo == other2->repo
+            && rev == other2->rev
+            && ref == other2->ref;
+    }
+
+    bool isImmutable() const override
+    {
+        return (bool) rev || narHash;
+    }
+
+    std::optional<std::string> getRef() const override { return ref; }
+
+    std::optional<Hash> getRev() const override { return rev; }
+
+    ParsedURL toURL() const override
+    {
+        auto path = owner + "/" + repo;
+        assert(!(ref && rev));
+        if (ref) path += "/" + *ref;
+        if (rev) path += "/" + rev->to_string(Base16, false);
+        return ParsedURL {
+            .scheme = "github",
+            .path = path,
+        };
+    }
+
+    Attrs toAttrsInternal() const override
+    {
+        Attrs attrs;
+        attrs.emplace("owner", owner);
+        attrs.emplace("repo", repo);
+        if (ref)
+            attrs.emplace("ref", *ref);
+        if (rev)
+            attrs.emplace("rev", rev->gitRev());
+        return attrs;
+    }
+
+    void clone(const Path & destDir) const override
+    {
+        std::shared_ptr<const Input> input = inputFromURL(fmt("git+ssh://git@github.com/%s/%s.git", owner, repo));
+        input = input->applyOverrides(ref.value_or("master"), rev);
+        input->clone(destDir);
+    }
+
+    std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override
+    {
+        auto rev = this->rev;
+        auto ref = this->ref.value_or("master");
+
+        if (!rev) {
+            auto url = fmt("https://api.github.com/repos/%s/%s/commits/%s",
+                owner, repo, ref);
+            auto json = nlohmann::json::parse(
+                readFile(
+                    store->toRealPath(
+                        downloadFile(store, url, "source", false).storePath)));
+            rev = Hash(json["sha"], htSHA1);
+            debug("HEAD revision for '%s' is %s", url, rev->gitRev());
+        }
+
+        auto input = std::make_shared<GitHubInput>(*this);
+        input->ref = {};
+        input->rev = *rev;
+
+        Attrs immutableAttrs({
+            {"type", "git-tarball"},
+            {"rev", rev->gitRev()},
+        });
+
+        if (auto res = getCache()->lookup(store, immutableAttrs)) {
+            return {
+                Tree{
+                    .actualPath = store->toRealPath(res->second),
+                    .storePath = std::move(res->second),
+                    .info = TreeInfo {
+                        .lastModified = getIntAttr(res->first, "lastModified"),
+                    },
+                },
+                input
+            };
+        }
+
+        // FIXME: use regular /archive URLs instead? api.github.com
+        // might have stricter rate limits.
+
+        auto url = fmt("https://api.github.com/repos/%s/%s/tarball/%s",
+            owner, repo, rev->to_string(Base16, false));
+
+        std::string accessToken = settings.githubAccessToken.get();
+        if (accessToken != "")
+            url += "?access_token=" + accessToken;
+
+        auto tree = downloadTarball(store, url, "source", true);
+
+        getCache()->add(
+            store,
+            immutableAttrs,
+            {
+                {"rev", rev->gitRev()},
+                {"lastModified", *tree.info.lastModified}
+            },
+            tree.storePath,
+            true);
+
+        return {std::move(tree), input};
+    }
+
+    std::shared_ptr<const Input> applyOverrides(
+        std::optional<std::string> ref,
+        std::optional<Hash> rev) const override
+    {
+        if (!ref && !rev) return shared_from_this();
+
+        auto res = std::make_shared<GitHubInput>(*this);
+
+        if (ref) res->ref = ref;
+        if (rev) res->rev = rev;
+
+        return res;
+    }
+};
+
+struct GitHubInputScheme : InputScheme
+{
+    std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override
+    {
+        if (url.scheme != "github") return nullptr;
+
+        auto path = tokenizeString<std::vector<std::string>>(url.path, "/");
+        auto input = std::make_unique<GitHubInput>();
+
+        if (path.size() == 2) {
+        } else if (path.size() == 3) {
+            if (std::regex_match(path[2], revRegex))
+                input->rev = Hash(path[2], htSHA1);
+            else if (std::regex_match(path[2], refRegex))
+                input->ref = path[2];
+            else
+                throw BadURL("in GitHub URL '%s', '%s' is not a commit hash or branch/tag name", url.url, path[2]);
+        } else
+            throw BadURL("GitHub URL '%s' is invalid", url.url);
+
+        for (auto &[name, value] : url.query) {
+            if (name == "rev") {
+                if (input->rev)
+                    throw BadURL("GitHub URL '%s' contains multiple commit hashes", url.url);
+                input->rev = Hash(value, htSHA1);
+            }
+            else if (name == "ref") {
+                if (!std::regex_match(value, refRegex))
+                    throw BadURL("GitHub URL '%s' contains an invalid branch/tag name", url.url);
+                if (input->ref)
+                    throw BadURL("GitHub URL '%s' contains multiple branch/tag names", url.url);
+                input->ref = value;
+            }
+        }
+
+        if (input->ref && input->rev)
+            throw BadURL("GitHub URL '%s' contains both a commit hash and a branch/tag name", url.url);
+
+        input->owner = path[0];
+        input->repo = path[1];
+
+        return input;
+    }
+
+    std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override
+    {
+        if (maybeGetStrAttr(attrs, "type") != "github") return {};
+
+        for (auto & [name, value] : attrs)
+            if (name != "type" && name != "owner" && name != "repo" && name != "ref" && name != "rev")
+                throw Error("unsupported GitHub input attribute '%s'", name);
+
+        auto input = std::make_unique<GitHubInput>();
+        input->owner = getStrAttr(attrs, "owner");
+        input->repo = getStrAttr(attrs, "repo");
+        input->ref = maybeGetStrAttr(attrs, "ref");
+        if (auto rev = maybeGetStrAttr(attrs, "rev"))
+            input->rev = Hash(*rev, htSHA1);
+        return input;
+    }
+};
+
+static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<GitHubInputScheme>()); });
+
+}

src/libfetchers/indirect.cc

@@ -0,0 +1,140 @@
+#include "fetchers.hh"
+
+namespace nix::fetchers {
+
+std::regex flakeRegex("[a-zA-Z][a-zA-Z0-9_-]*", std::regex::ECMAScript);
+
+struct IndirectInput : Input
+{
+    std::string id;
+    std::optional<Hash> rev;
+    std::optional<std::string> ref;
+
+    std::string type() const override { return "indirect"; }
+
+    bool operator ==(const Input & other) const override
+    {
+        auto other2 = dynamic_cast<const IndirectInput *>(&other);
+        return
+            other2
+            && id == other2->id
+            && rev == other2->rev
+            && ref == other2->ref;
+    }
+
+    bool isDirect() const override
+    {
+        return false;
+    }
+
+    std::optional<std::string> getRef() const override { return ref; }
+
+    std::optional<Hash> getRev() const override { return rev; }
+
+    bool contains(const Input & other) const override
+    {
+        auto other2 = dynamic_cast<const IndirectInput *>(&other);
+        return
+            other2
+            && id == other2->id
+            && (!ref || ref == other2->ref)
+            && (!rev || rev == other2->rev);
+    }
+
+    ParsedURL toURL() const override
+    {
+        ParsedURL url;
+        url.scheme = "flake";
+        url.path = id;
+        if (ref) { url.path += '/'; url.path += *ref; };
+        if (rev) { url.path += '/'; url.path += rev->gitRev(); };
+        return url;
+    }
+
+    Attrs toAttrsInternal() const override
+    {
+        Attrs attrs;
+        attrs.emplace("id", id);
+        if (ref)
+            attrs.emplace("ref", *ref);
+        if (rev)
+            attrs.emplace("rev", rev->gitRev());
+        return attrs;
+    }
+
+    std::shared_ptr<const Input> applyOverrides(
+        std::optional<std::string> ref,
+        std::optional<Hash> rev) const override
+    {
+        if (!ref && !rev) return shared_from_this();
+
+        auto res = std::make_shared<IndirectInput>(*this);
+
+        if (ref) res->ref = ref;
+        if (rev) res->rev = rev;
+
+        return res;
+    }
+
+    std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override
+    {
+        throw Error("indirect input '%s' cannot be fetched directly", to_string());
+    }
+};
+
+struct IndirectInputScheme : InputScheme
+{
+    std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override
+    {
+        if (url.scheme != "flake") return nullptr;
+
+        auto path = tokenizeString<std::vector<std::string>>(url.path, "/");
+        auto input = std::make_unique<IndirectInput>();
+
+        if (path.size() == 1) {
+        } else if (path.size() == 2) {
+            if (std::regex_match(path[1], revRegex))
+                input->rev = Hash(path[1], htSHA1);
+            else if (std::regex_match(path[1], refRegex))
+                input->ref = path[1];
+            else
+                throw BadURL("in flake URL '%s', '%s' is not a commit hash or branch/tag name", url.url, path[1]);
+        } else if (path.size() == 3) {
+            if (!std::regex_match(path[1], refRegex))
+                throw BadURL("in flake URL '%s', '%s' is not a branch/tag name", url.url, path[1]);
+            input->ref = path[1];
+            if (!std::regex_match(path[2], revRegex))
+                throw BadURL("in flake URL '%s', '%s' is not a commit hash", url.url, path[2]);
+            input->rev = Hash(path[2], htSHA1);
+        } else
+            throw BadURL("GitHub URL '%s' is invalid", url.url);
+
+        // FIXME: forbid query params?
+
+        input->id = path[0];
+        if (!std::regex_match(input->id, flakeRegex))
+            throw BadURL("'%s' is not a valid flake ID", input->id);
+
+        return input;
+    }
+
+    std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override
+    {
+        if (maybeGetStrAttr(attrs, "type") != "indirect") return {};
+
+        for (auto & [name, value] : attrs)
+            if (name != "type" && name != "id" && name != "ref" && name != "rev")
+                throw Error("unsupported indirect input attribute '%s'", name);
+
+        auto input = std::make_unique<IndirectInput>();
+        input->id = getStrAttr(attrs, "id");
+        input->ref = maybeGetStrAttr(attrs, "ref");
+        if (auto rev = maybeGetStrAttr(attrs, "rev"))
+            input->rev = Hash(*rev, htSHA1);
+        return input;
+    }
+};
+
+static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<IndirectInputScheme>()); });
+
+}

src/libfetchers/local.mk

@@ -0,0 +1,11 @@
+libraries += libfetchers
+
+libfetchers_NAME = libnixfetchers
+
+libfetchers_DIR := $(d)
+
+libfetchers_SOURCES := $(wildcard $(d)/*.cc)
+
+libfetchers_CXXFLAGS += -I src/libutil -I src/libstore
+
+libfetchers_LIBS = libutil libstore libnixrust

src/libfetchers/mercurial.cc

@@ -0,0 +1,338 @@
+#include "fetchers.hh"
+#include "cache.hh"
+#include "globals.hh"
+#include "tarfile.hh"
+#include "store-api.hh"
+
+#include <sys/time.h>
+
+using namespace std::string_literals;
+
+namespace nix::fetchers {
+
+struct MercurialInput : Input
+{
+    ParsedURL url;
+    std::optional<std::string> ref;
+    std::optional<Hash> rev;
+
+    MercurialInput(const ParsedURL & url) : url(url)
+    { }
+
+    std::string type() const override { return "hg"; }
+
+    bool operator ==(const Input & other) const override
+    {
+        auto other2 = dynamic_cast<const MercurialInput *>(&other);
+        return
+            other2
+            && url == other2->url
+            && rev == other2->rev
+            && ref == other2->ref;
+    }
+
+    bool isImmutable() const override
+    {
+        return (bool) rev || narHash;
+    }
+
+    std::optional<std::string> getRef() const override { return ref; }
+
+    std::optional<Hash> getRev() const override { return rev; }
+
+    ParsedURL toURL() const override
+    {
+        ParsedURL url2(url);
+        url2.scheme = "hg+" + url2.scheme;
+        if (rev) url2.query.insert_or_assign("rev", rev->gitRev());
+        if (ref) url2.query.insert_or_assign("ref", *ref);
+        return url;
+    }
+
+    Attrs toAttrsInternal() const override
+    {
+        Attrs attrs;
+        attrs.emplace("url", url.to_string());
+        if (ref)
+            attrs.emplace("ref", *ref);
+        if (rev)
+            attrs.emplace("rev", rev->gitRev());
+        return attrs;
+    }
+
+    std::shared_ptr<const Input> applyOverrides(
+        std::optional<std::string> ref,
+        std::optional<Hash> rev) const override
+    {
+        if (!ref && !rev) return shared_from_this();
+
+        auto res = std::make_shared<MercurialInput>(*this);
+
+        if (ref) res->ref = ref;
+        if (rev) res->rev = rev;
+
+        return res;
+    }
+
+    std::optional<Path> getSourcePath() const
+    {
+        if (url.scheme == "file" && !ref && !rev)
+            return url.path;
+        return {};
+    }
+
+    void markChangedFile(std::string_view file, std::optional<std::string> commitMsg) const override
+    {
+        auto sourcePath = getSourcePath();
+        assert(sourcePath);
+
+        // FIXME: shut up if file is already tracked.
+        runProgram("hg", true,
+            { "add", *sourcePath + "/" + std::string(file) });
+
+        if (commitMsg)
+            runProgram("hg", true,
+                { "commit", *sourcePath + "/" + std::string(file), "-m", *commitMsg });
+    }
+
+    std::pair<bool, std::string> getActualUrl() const
+    {
+        bool isLocal = url.scheme == "file";
+        return {isLocal, isLocal ? url.path : url.base};
+    }
+
+    std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override
+    {
+        auto name = "source";
+
+        auto input = std::make_shared<MercurialInput>(*this);
+
+        auto [isLocal, actualUrl_] = getActualUrl();
+        auto actualUrl = actualUrl_; // work around clang bug
+
+        // FIXME: return lastModified.
+
+        // FIXME: don't clone local repositories.
+
+        if (!input->ref && !input->rev && isLocal && pathExists(actualUrl + "/.hg")) {
+
+            bool clean = runProgram("hg", true, { "status", "-R", actualUrl, "--modified", "--added", "--removed" }) == "";
+
+            if (!clean) {
+
+                /* This is an unclean working tree. So copy all tracked
+                   files. */
+
+                if (!settings.allowDirty)
+                    throw Error("Mercurial tree '%s' is unclean", actualUrl);
+
+                if (settings.warnDirty)
+                    warn("Mercurial tree '%s' is unclean", actualUrl);
+
+                input->ref = chomp(runProgram("hg", true, { "branch", "-R", actualUrl }));
+
+                auto files = tokenizeString<std::set<std::string>>(
+                    runProgram("hg", true, { "status", "-R", actualUrl, "--clean", "--modified", "--added", "--no-status", "--print0" }), "\0"s);
+
+                PathFilter filter = [&](const Path & p) -> bool {
+                    assert(hasPrefix(p, actualUrl));
+                    std::string file(p, actualUrl.size() + 1);
+
+                    auto st = lstat(p);
+
+                    if (S_ISDIR(st.st_mode)) {
+                        auto prefix = file + "/";
+                        auto i = files.lower_bound(prefix);
+                        return i != files.end() && hasPrefix(*i, prefix);
+                    }
+
+                    return files.count(file);
+                };
+
+                auto storePath = store->addToStore("source", actualUrl, true, htSHA256, filter);
+
+                return {Tree {
+                    .actualPath = store->printStorePath(storePath),
+                    .storePath = std::move(storePath),
+                }, input};
+            }
+        }
+
+        if (!input->ref) input->ref = "default";
+
+        auto getImmutableAttrs = [&]()
+        {
+            return Attrs({
+                {"type", "hg"},
+                {"name", name},
+                {"rev", input->rev->gitRev()},
+            });
+        };
+
+        auto makeResult = [&](const Attrs & infoAttrs, StorePath && storePath)
+            -> std::pair<Tree, std::shared_ptr<const Input>>
+        {
+            assert(input->rev);
+            assert(!rev || rev == input->rev);
+            return {
+                Tree{
+                    .actualPath = store->toRealPath(storePath),
+                    .storePath = std::move(storePath),
+                    .info = TreeInfo {
+                        .revCount = getIntAttr(infoAttrs, "revCount"),
+                    },
+                },
+                input
+            };
+        };
+
+        if (input->rev) {
+            if (auto res = getCache()->lookup(store, getImmutableAttrs()))
+                return makeResult(res->first, std::move(res->second));
+        }
+
+        assert(input->rev || input->ref);
+        auto revOrRef = input->rev ? input->rev->gitRev() : *input->ref;
+
+        Attrs mutableAttrs({
+            {"type", "hg"},
+            {"name", name},
+            {"url", actualUrl},
+            {"ref", *input->ref},
+        });
+
+        if (auto res = getCache()->lookup(store, mutableAttrs)) {
+            auto rev2 = Hash(getStrAttr(res->first, "rev"), htSHA1);
+            if (!rev || rev == rev2) {
+                input->rev = rev2;
+                return makeResult(res->first, std::move(res->second));
+            }
+        }
+
+        Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), hashString(htSHA256, actualUrl).to_string(Base32, false));
+
+        /* If this is a commit hash that we already have, we don't
+           have to pull again. */
+        if (!(input->rev
+                && pathExists(cacheDir)
+                && runProgram(
+                    RunOptions("hg", { "log", "-R", cacheDir, "-r", input->rev->gitRev(), "--template", "1" })
+                    .killStderr(true)).second == "1"))
+        {
+            Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Mercurial repository '%s'", actualUrl));
+
+            if (pathExists(cacheDir)) {
+                try {
+                    runProgram("hg", true, { "pull", "-R", cacheDir, "--", actualUrl });
+                }
+                catch (ExecError & e) {
+                    string transJournal = cacheDir + "/.hg/store/journal";
+                    /* hg throws "abandoned transaction" error only if this file exists */
+                    if (pathExists(transJournal)) {
+                        runProgram("hg", true, { "recover", "-R", cacheDir });
+                        runProgram("hg", true, { "pull", "-R", cacheDir, "--", actualUrl });
+                    } else {
+                        throw ExecError(e.status, fmt("'hg pull' %s", statusToString(e.status)));
+                    }
+                }
+            } else {
+                createDirs(dirOf(cacheDir));
+                runProgram("hg", true, { "clone", "--noupdate", "--", actualUrl, cacheDir });
+            }
+        }
+
+        auto tokens = tokenizeString<std::vector<std::string>>(
+            runProgram("hg", true, { "log", "-R", cacheDir, "-r", revOrRef, "--template", "{node} {rev} {branch}" }));
+        assert(tokens.size() == 3);
+
+        input->rev = Hash(tokens[0], htSHA1);
+        auto revCount = std::stoull(tokens[1]);
+        input->ref = tokens[2];
+
+        if (auto res = getCache()->lookup(store, getImmutableAttrs()))
+            return makeResult(res->first, std::move(res->second));
+
+        Path tmpDir = createTempDir();
+        AutoDelete delTmpDir(tmpDir, true);
+
+        runProgram("hg", true, { "archive", "-R", cacheDir, "-r", input->rev->gitRev(), tmpDir });
+
+        deletePath(tmpDir + "/.hg_archival.txt");
+
+        auto storePath = store->addToStore(name, tmpDir);
+
+        Attrs infoAttrs({
+            {"rev", input->rev->gitRev()},
+            {"revCount", (int64_t) revCount},
+        });
+
+        if (!this->rev)
+            getCache()->add(
+                store,
+                mutableAttrs,
+                infoAttrs,
+                storePath,
+                false);
+
+        getCache()->add(
+            store,
+            getImmutableAttrs(),
+            infoAttrs,
+            storePath,
+            true);
+
+        return makeResult(infoAttrs, std::move(storePath));
+    }
+};
+
+struct MercurialInputScheme : InputScheme
+{
+    std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override
+    {
+        if (url.scheme != "hg+http" &&
+            url.scheme != "hg+https" &&
+            url.scheme != "hg+ssh" &&
+            url.scheme != "hg+file") return nullptr;
+
+        auto url2(url);
+        url2.scheme = std::string(url2.scheme, 3);
+        url2.query.clear();
+
+        Attrs attrs;
+        attrs.emplace("type", "hg");
+
+        for (auto &[name, value] : url.query) {
+            if (name == "rev" || name == "ref")
+                attrs.emplace(name, value);
+            else
+                url2.query.emplace(name, value);
+        }
+
+        attrs.emplace("url", url2.to_string());
+
+        return inputFromAttrs(attrs);
+    }
+
+    std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override
+    {
+        if (maybeGetStrAttr(attrs, "type") != "hg") return {};
+
+        for (auto & [name, value] : attrs)
+            if (name != "type" && name != "url" && name != "ref" && name != "rev")
+                throw Error("unsupported Mercurial input attribute '%s'", name);
+
+        auto input = std::make_unique<MercurialInput>(parseURL(getStrAttr(attrs, "url")));
+        if (auto ref = maybeGetStrAttr(attrs, "ref")) {
+            if (!std::regex_match(*ref, refRegex))
+                throw BadURL("invalid Mercurial branch/tag name '%s'", *ref);
+            input->ref = *ref;
+        }
+        if (auto rev = maybeGetStrAttr(attrs, "rev"))
+            input->rev = Hash(*rev, htSHA1);
+        return input;
+    }
+};
+
+static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<MercurialInputScheme>()); });
+
+}