Bump version

The install-multi-user script uses blue, green, and red colors, as
well as bold and underline, to add helpful formatting that helps
structure its rather voluminous output.

Unfortunately, the terminal escape sequences it uses are not quite
well-formed.  The relevant information is all there, just obscured
by some extra noise, a leading parameter `38`.  Empirically, the
result is:

 * On macOS, in both Terminal.app and iTerm2, the spurious `38` is
   ignored, the rest of the escape sequence is applied, and the colors
   show up as intended.

 * On Linux, in at least gnome-terminal and xterm, the spurious `38`
   and the next parameter after it are ignored, and what's left is
   applied.  So in the sequence `38;4;32`, the 4 (underline) is
   ignored but the 32 (green) takes effect; in a more typical sequence
   like `38;34`, the 34 (blue) is ignored and nothing happens.

These codes are all unchanged since this script's origins as a
Darwin-only script -- so the fact that they work fine in common macOS
terminals goes some way to explain how the bug arose.

Happily, we can make the colors work as intended by just deleting the
extra `38;`.  Tested in all four terminals mentioned above; the new
codes work correctly on all of them, and on the two macOS terminals
they work exactly the same as before.

---

In a bit more technical detail -- perhaps more than anyone, me
included, ever wanted to know, but now that I've gone and learned it
I'll write it down anyway :) -- here's what's happening in these codes:

An ECMA-48 "control sequence" begins with `\033[` aka "CSI", contains
any number of parameters as semicolon-separated decimal numbers (plus
sometimes other wrinkles), and ends with a byte from 0x40..0x7e.  In
our case, with `m` aka "SGR", "Select Graphic Rendition".

An SGR control sequence `\033[...m` sets colors, fonts, text styles,
etc.  In particular a parameter `31` means red, `32` green, `34` blue,
`4` underline, and `0` means reset to normal.  Those are all we use.

There is also a `38`.  This is used for setting colors too... but it
needs arguments.  `38;5;nn` is color nn from a 256-color palette, and
`38;2;rr;gg;bb` has the given RGB values.

There is no meaning defined for `38;1` or `38;34` etc.  On seeing a
parameter `38` followed by an unrecognized argument for it, apparently
some implementations (as seen on macOS) discard only the `38` and
others (as seen on Linux) discard the argument too before resuming.

(cherry picked from commit 7313aa267b)

.gitignore

@@ -81,6 +81,9 @@ perl/Makefile.config
 /tests/common.sh
 /tests/dummy
 /tests/result*
+/tests/restricted-innocent
+/tests/shell
+/tests/shell.drv
 
 # /tests/lang/
 /tests/lang/*.out

.version

@@ -0,0 +1 @@
+2.3.4

Makefile

@@ -19,11 +19,4 @@ GLOBAL_CXXFLAGS += -g -Wall -include config.h
 
 -include Makefile.config
 
-OPTIMIZE = 1
-
-ifeq ($(OPTIMIZE), 1)
-  GLOBAL_CFLAGS += -O3
-  GLOBAL_CXXFLAGS += -O3
-endif
-
 include mk/lib.mk

Makefile.config.in

@@ -1,12 +1,15 @@
+AR = @AR@
 BDW_GC_LIBS = @BDW_GC_LIBS@
+BUILD_SHARED_LIBS = @BUILD_SHARED_LIBS@
 CC = @CC@
 CFLAGS = @CFLAGS@
 CXX = @CXX@
 CXXFLAGS = @CXXFLAGS@
+LDFLAGS = @LDFLAGS@
 ENABLE_S3 = @ENABLE_S3@
 HAVE_SODIUM = @HAVE_SODIUM@
-HAVE_READLINE = @HAVE_READLINE@
 HAVE_SECCOMP = @HAVE_SECCOMP@
+BOOST_LDFLAGS = @BOOST_LDFLAGS@
 LIBCURL_LIBS = @LIBCURL_LIBS@
 OPENSSL_LIBS = @OPENSSL_LIBS@
 PACKAGE_NAME = @PACKAGE_NAME@

README.md

@@ -1,3 +1,5 @@
+[![Open Collective supporters](https://opencollective.com/nixos/tiers/supporter/badge.svg?label=Supporters&color=brightgreen)](https://opencollective.com/nixos)
+
 Nix, the purely functional package manager
 ------------------------------------------
 

configure.ac

@@ -1,4 +1,5 @@
-AC_INIT(nix, m4_esyscmd([bash -c "echo -n $(cat ./version)$VERSION_SUFFIX"]))
+AC_INIT(nix, m4_esyscmd([bash -c "echo -n $(cat ./.version)$VERSION_SUFFIX"]))
+AC_CONFIG_MACRO_DIRS([m4])
 AC_CONFIG_SRCDIR(README.md)
 AC_CONFIG_AUX_DIR(config)
 
@@ -42,13 +43,28 @@ esac
 
 AC_MSG_RESULT($system)
 AC_SUBST(system)
-AC_DEFINE_UNQUOTED(SYSTEM, ["$system"], [platform identifier (`cpu-os')])
+AC_DEFINE_UNQUOTED(SYSTEM, ["$system"], [platform identifier ('cpu-os')])
 
 
 # State should be stored in /nix/var, unless the user overrides it explicitly.
 test "$localstatedir" = '${prefix}/var' && localstatedir=/nix/var
 
 
+# Set default flags for nix (as per AC_PROG_CC/CXX docs),
+# while still allowing the user to override them from the command line.
+: ${CFLAGS="-O3"}
+: ${CXXFLAGS="-O3"}
+AC_PROG_CC
+AC_PROG_CXX
+AC_PROG_CPP
+AX_CXX_COMPILE_STDCXX_17([noext], [mandatory])
+
+AC_CHECK_TOOL([AR], [ar])
+
+# Use 64-bit file system calls so that we can support files > 2 GiB.
+AC_SYS_LARGEFILE
+
+
 # Solaris-specific stuff.
 AC_STRUCT_DIRENT_D_TYPE
 if test "$sys_name" = sunos; then
@@ -57,18 +73,6 @@ if test "$sys_name" = sunos; then
 fi
 
 
-CFLAGS=
-CXXFLAGS=
-AC_PROG_CC
-AC_PROG_CXX
-AC_PROG_CPP
-AX_CXX_COMPILE_STDCXX_14
-
-
-# Use 64-bit file system calls so that we can support files > 2 GiB.
-AC_SYS_LARGEFILE
-
-
 # Check for pubsetbuf.
 AC_MSG_CHECKING([for pubsetbuf])
 AC_LANG_PUSH(C++)
@@ -144,6 +148,16 @@ AC_ARG_WITH(store-dir, AC_HELP_STRING([--with-store-dir=PATH],
 AC_SUBST(storedir)
 
 
+# Look for boost, a required dependency.
+# Note that AX_BOOST_BASE only exports *CPP* BOOST_CPPFLAGS, no CXX flags,
+# and CPPFLAGS are not passed to the C++ compiler automatically.
+# Thus we append the returned CPPFLAGS to the CXXFLAGS here.
+AX_BOOST_BASE([1.66], [CXXFLAGS="$BOOST_CPPFLAGS $CXXFLAGS"], [AC_MSG_ERROR([Nix requires boost.])])
+# For unknown reasons, setting this directly in the ACTION-IF-FOUND above
+# ends up with LDFLAGS being empty, so we set it afterwards.
+LDFLAGS="$BOOST_LDFLAGS $LDFLAGS"
+
+
 # Look for OpenSSL, a required dependency.
 PKG_CHECK_MODULES([OPENSSL], [libcrypto], [CXXFLAGS="$OPENSSL_CFLAGS $CXXFLAGS"])
 
@@ -163,7 +177,16 @@ PKG_CHECK_MODULES([SQLITE3], [sqlite3 >= 3.6.19], [CXXFLAGS="$SQLITE3_CFLAGS $CX
 PKG_CHECK_MODULES([LIBCURL], [libcurl], [CXXFLAGS="$LIBCURL_CFLAGS $CXXFLAGS"])
 
 # Look for editline, a required dependency.
-PKG_CHECK_MODULES([EDITLINE], [libeditline], [CXXFLAGS="$EDITLINE_CFLAGS $CXXFLAGS"])
+# The the libeditline.pc file was added only in libeditline >= 1.15.2,
+# see https://github.com/troglobit/editline/commit/0a8f2ef4203c3a4a4726b9dd1336869cd0da8607,
+# but e.g. Ubuntu 16.04 has an older version, so we fall back to searching for
+# editline.h when the pkg-config approach fails.
+PKG_CHECK_MODULES([EDITLINE], [libeditline], [CXXFLAGS="$EDITLINE_CFLAGS $CXXFLAGS"], [
+  AC_CHECK_HEADERS([editline.h], [true],
+    [AC_MSG_ERROR([Nix requires libeditline; it was found neither via pkg-config nor its normal header.])])
+  AC_SEARCH_LIBS([readline read_history], [editline], [],
+    [AC_MSG_ERROR([Nix requires libeditline; it was not found via pkg-config, but via its header, but required functions do not work. Maybe it is too old? >= 1.14 is required.])])
+])
 
 # Look for libsodium, an optional dependency.
 PKG_CHECK_MODULES([SODIUM], [libsodium],
@@ -267,6 +290,15 @@ AC_ARG_WITH(sandbox-shell, AC_HELP_STRING([--with-sandbox-shell=PATH],
   sandbox_shell=$withval)
 AC_SUBST(sandbox_shell)
 
+AC_ARG_ENABLE(shared, AC_HELP_STRING([--enable-shared],
+  [Build shared libraries for Nix [default=yes]]),
+  shared=$enableval, shared=yes)
+if test "$shared" = yes; then
+  AC_SUBST(BUILD_SHARED_LIBS, 1, [Whether to build shared libraries.])
+else
+  AC_SUBST(BUILD_SHARED_LIBS, 0, [Whether to build shared libraries.])
+fi
+
 
 # Expand all variables in config.status.
 test "$prefix" = NONE && prefix=$ac_default_prefix

contrib/stack-collapse.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env nix-shell
+#!nix-shell -i python3 -p python3 --pure
+
+# To be used with `--trace-function-calls` and `flamegraph.pl`.
+#
+# For example:
+#
+# nix-instantiate --trace-function-calls '<nixpkgs>' -A hello 2> nix-function-calls.trace
+# ./contrib/stack-collapse.py nix-function-calls.trace > nix-function-calls.folded
+# nix-shell -p flamegraph --run "flamegraph.pl nix-function-calls.folded > nix-function-calls.svg"
+
+import sys
+from pprint import pprint
+import fileinput
+
+stack = []
+timestack = []
+
+for line in fileinput.input():
+    components = line.strip().split(" ", 2)
+    if components[0] != "function-trace":
+        continue
+
+    direction = components[1]
+    components = components[2].rsplit(" ", 2)
+
+    loc = components[0]
+    _at = components[1]
+    time = int(components[2])
+
+    if direction == "entered":
+        stack.append(loc)
+        timestack.append(time)
+    elif direction == "exited":
+        dur = time - timestack.pop()
+        vst = ";".join(stack)
+        print(f"{vst} {dur}")
+        stack.pop()

corepkgs/unpack-channel.nix

@@ -18,21 +18,17 @@ let
       if [ * != $channelName ]; then
         mv * $out/$channelName
       fi
-      if [ -n "$binaryCacheURL" ]; then
-        mkdir $out/binary-caches
-        echo -n "$binaryCacheURL" > $out/binary-caches/$channelName
-      fi
     '';
 
 in
 
-{ name, channelName, src, binaryCacheURL ? "" }:
+{ name, channelName, src }:
 
 derivation {
   system = builtins.currentSystem;
   builder = shell;
   args = [ "-e" builder ];
-  inherit name channelName src binaryCacheURL;
+  inherit name channelName src;
 
   PATH = "${nixBinDir}:${coreutils}";
 

doc/manual/advanced-topics/advanced-topics.xml

@@ -7,5 +7,8 @@
 <title>Advanced Topics</title>
 
 <xi:include href="distributed-builds.xml" />
+<xi:include href="cores-vs-jobs.xml" />
+<xi:include href="diff-hook.xml" />
+<xi:include href="post-build-hook.xml" />
 
 </part>

doc/manual/advanced-topics/cores-vs-jobs.xml

@@ -0,0 +1,121 @@
+<chapter xmlns="http://docbook.org/ns/docbook"
+      xmlns:xlink="http://www.w3.org/1999/xlink"
+      xmlns:xi="http://www.w3.org/2001/XInclude"
+      version="5.0"
+      xml:id="chap-tuning-cores-and-jobs">
+
+<title>Tuning Cores and Jobs</title>
+
+<para>Nix has two relevant settings with regards to how your CPU cores
+will be utilized: <xref linkend="conf-cores" /> and
+<xref linkend="conf-max-jobs" />. This chapter will talk about what
+they are, how they interact, and their configuration trade-offs.</para>
+
+<variablelist>
+  <varlistentry>
+    <term><xref linkend="conf-max-jobs" /></term>
+    <listitem><para>
+      Dictates how many separate derivations will be built at the same
+      time. If you set this to zero, the local machine will do no
+      builds. Nix will still substitute from binary caches, and build
+      remotely if remote builders are configured.
+    </para></listitem>
+  </varlistentry>
+  <varlistentry>
+    <term><xref linkend="conf-cores" /></term>
+    <listitem><para>
+      Suggests how many cores each derivation should use. Similar to
+      <command>make -j</command>.
+    </para></listitem>
+  </varlistentry>
+</variablelist>
+
+<para>The <xref linkend="conf-cores" /> setting determines the value of
+<envar>NIX_BUILD_CORES</envar>. <envar>NIX_BUILD_CORES</envar> is equal
+to <xref linkend="conf-cores" />, unless <xref linkend="conf-cores" />
+equals <literal>0</literal>, in which case <envar>NIX_BUILD_CORES</envar>
+will be the total number of cores in the system.</para>
+
+<para>The total number of consumed cores is a simple multiplication,
+<xref linkend="conf-cores" /> * <envar>NIX_BUILD_CORES</envar>.</para>
+
+<para>The balance on how to set these two independent variables depends
+upon each builder's workload and hardware. Here are a few example
+scenarios on a machine with 24 cores:</para>
+
+<table>
+  <caption>Balancing 24 Build Cores</caption>
+  <thead>
+    <tr>
+      <th><xref linkend="conf-max-jobs" /></th>
+      <th><xref linkend="conf-cores" /></th>
+      <th><envar>NIX_BUILD_CORES</envar></th>
+      <th>Maximum Processes</th>
+      <th>Result</th>
+    </tr>
+  </thead>
+  <tbody>
+    <tr>
+      <td>1</td>
+      <td>24</td>
+      <td>24</td>
+      <td>24</td>
+      <td>
+        One derivation will be built at a time, each one can use 24
+        cores. Undersold if a job can’t use 24 cores.
+      </td>
+    </tr>
+
+    <tr>
+      <td>4</td>
+      <td>6</td>
+      <td>6</td>
+      <td>24</td>
+      <td>
+        Four derivations will be built at once, each given access to
+        six cores.
+      </td>
+    </tr>
+    <tr>
+      <td>12</td>
+      <td>6</td>
+      <td>6</td>
+      <td>72</td>
+      <td>
+        12 derivations will be built at once, each given access to six
+        cores. This configuration is over-sold. If all 12 derivations
+        being built simultaneously try to use all six cores, the
+        machine's performance will be degraded due to extensive context
+        switching between the 12 builds.
+      </td>
+    </tr>
+    <tr>
+      <td>24</td>
+      <td>1</td>
+      <td>1</td>
+      <td>24</td>
+      <td>
+        24 derivations can build at the same time, each using a single
+        core. Never oversold, but derivations which require many cores
+        will be very slow to compile.
+      </td>
+    </tr>
+    <tr>
+      <td>24</td>
+      <td>0</td>
+      <td>24</td>
+      <td>576</td>
+      <td>
+        24 derivations can build at the same time, each using all the
+        available cores of the machine. Very likely to be oversold,
+        and very likely to suffer context switches.
+      </td>
+    </tr>
+  </tbody>
+</table>
+
+<para>It is up to the derivations' build script to respect
+host's requested cores-per-build by following the value of the
+<envar>NIX_BUILD_CORES</envar> environment variable.</para>
+
+</chapter>

doc/manual/advanced-topics/diff-hook.xml

@@ -0,0 +1,205 @@
+<chapter xmlns="http://docbook.org/ns/docbook"
+      xmlns:xlink="http://www.w3.org/1999/xlink"
+      xmlns:xi="http://www.w3.org/2001/XInclude"
+      xml:id="chap-diff-hook"
+      version="5.0"
+      >
+
+<title>Verifying Build Reproducibility with <option linkend="conf-diff-hook">diff-hook</option></title>
+
+<subtitle>Check build reproducibility by running builds multiple times
+and comparing their results.</subtitle>
+
+<para>Specify a program with Nix's <xref linkend="conf-diff-hook" /> to
+compare build results when two builds produce different results. Note:
+this hook is only executed if the results are not the same, this hook
+is not used for determining if the results are the same.</para>
+
+<para>For purposes of demonstration, we'll use the following Nix file,
+<filename>deterministic.nix</filename> for testing:</para>
+
+<programlisting>
+let
+  inherit (import &lt;nixpkgs&gt; {}) runCommand;
+in {
+  stable = runCommand "stable" {} ''
+    touch $out
+  '';
+
+  unstable = runCommand "unstable" {} ''
+    echo $RANDOM > $out
+  '';
+}
+</programlisting>
+
+<para>Additionally, <filename>nix.conf</filename> contains:
+
+<programlisting>
+diff-hook = /etc/nix/my-diff-hook
+run-diff-hook = true
+</programlisting>
+
+where <filename>/etc/nix/my-diff-hook</filename> is an executable
+file containing:
+
+<programlisting>
+#!/bin/sh
+exec &gt;&amp;2
+echo "For derivation $3:"
+/run/current-system/sw/bin/diff -r "$1" "$2"
+</programlisting>
+
+</para>
+
+<para>The diff hook is executed by the same user and group who ran the
+build. However, the diff hook does not have write access to the store
+path just built.</para>
+
+<section>
+  <title>
+    Spot-Checking Build Determinism
+  </title>
+
+  <para>
+    Verify a path which already exists in the Nix store by passing
+    <option>--check</option> to the build command.
+  </para>
+
+  <para>If the build passes and is deterministic, Nix will exit with a
+  status code of 0:</para>
+
+  <screen>
+$ nix-build ./deterministic.nix -A stable
+these derivations will be built:
+  /nix/store/z98fasz2jqy9gs0xbvdj939p27jwda38-stable.drv
+building '/nix/store/z98fasz2jqy9gs0xbvdj939p27jwda38-stable.drv'...
+/nix/store/yyxlzw3vqaas7wfp04g0b1xg51f2czgq-stable
+
+$ nix-build ./deterministic.nix -A stable --check
+checking outputs of '/nix/store/z98fasz2jqy9gs0xbvdj939p27jwda38-stable.drv'...
+/nix/store/yyxlzw3vqaas7wfp04g0b1xg51f2czgq-stable
+</screen>
+
+  <para>If the build is not deterministic, Nix will exit with a status
+  code of 1:</para>
+
+  <screen>
+$ nix-build ./deterministic.nix -A unstable
+these derivations will be built:
+  /nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv
+building '/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv'...
+/nix/store/krpqk0l9ib0ibi1d2w52z293zw455cap-unstable
+
+$ nix-build ./deterministic.nix -A unstable --check
+checking outputs of '/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv'...
+error: derivation '/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv' may not be deterministic: output '/nix/store/krpqk0l9ib0ibi1d2w52z293zw455cap-unstable' differs
+</screen>
+
+<para>In the Nix daemon's log, we will now see:
+<screen>
+For derivation /nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv:
+1c1
+&lt; 8108
+---
+&gt; 30204
+</screen>
+</para>
+
+  <para>Using <option>--check</option> with <option>--keep-failed</option>
+  will cause Nix to keep the second build's output in a special,
+  <literal>.check</literal> path:</para>
+
+  <screen>
+$ nix-build ./deterministic.nix -A unstable --check --keep-failed
+checking outputs of '/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv'...
+note: keeping build directory '/tmp/nix-build-unstable.drv-0'
+error: derivation '/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv' may not be deterministic: output '/nix/store/krpqk0l9ib0ibi1d2w52z293zw455cap-unstable' differs from '/nix/store/krpqk0l9ib0ibi1d2w52z293zw455cap-unstable.check'
+</screen>
+
+  <para>In particular, notice the
+  <literal>/nix/store/krpqk0l9ib0ibi1d2w52z293zw455cap-unstable.check</literal>
+  output. Nix has copied the build results to that directory where you
+  can examine it.</para>
+
+  <note xml:id="check-dirs-are-unregistered">
+    <title><literal>.check</literal> paths are not registered store paths</title>
+
+    <para>Check paths are not protected against garbage collection,
+    and this path will be deleted on the next garbage collection.</para>
+
+    <para>The path is guaranteed to be alive for the duration of
+    <xref linkend="conf-diff-hook" />'s execution, but may be deleted
+    any time after.</para>
+
+    <para>If the comparison is performed as part of automated tooling,
+    please use the diff-hook or author your tooling to handle the case
+    where the build was not deterministic and also a check path does
+    not exist.</para>
+  </note>
+
+  <para>
+    <option>--check</option> is only usable if the derivation has
+    been built on the system already. If the derivation has not been
+    built Nix will fail with the error:
+    <screen>
+error: some outputs of '/nix/store/hzi1h60z2qf0nb85iwnpvrai3j2w7rr6-unstable.drv' are not valid, so checking is not possible
+</screen>
+
+    Run the build without <option>--check</option>, and then try with
+    <option>--check</option> again.
+  </para>
+</section>
+
+<section>
+  <title>
+    Automatic and Optionally Enforced Determinism Verification
+  </title>
+
+  <para>
+    Automatically verify every build at build time by executing the
+    build multiple times.
+  </para>
+
+  <para>
+    Setting <xref linkend="conf-repeat" /> and
+    <xref linkend="conf-enforce-determinism" /> in your
+    <filename>nix.conf</filename> permits the automated verification
+    of every build Nix performs.
+  </para>
+
+  <para>
+    The following configuration will run each build three times, and
+    will require the build to be deterministic:
+
+    <programlisting>
+enforce-determinism = true
+repeat = 2
+</programlisting>
+  </para>
+
+  <para>
+    Setting <xref linkend="conf-enforce-determinism" /> to false as in
+    the following configuration will run the build multiple times,
+    execute the build hook, but will allow the build to succeed even
+    if it does not build reproducibly:
+
+    <programlisting>
+enforce-determinism = false
+repeat = 1
+</programlisting>
+  </para>
+
+  <para>
+    An example output of this configuration:
+    <screen>
+$ nix-build ./test.nix -A unstable
+these derivations will be built:
+  /nix/store/ch6llwpr2h8c3jmnf3f2ghkhx59aa97f-unstable.drv
+building '/nix/store/ch6llwpr2h8c3jmnf3f2ghkhx59aa97f-unstable.drv' (round 1/2)...
+building '/nix/store/ch6llwpr2h8c3jmnf3f2ghkhx59aa97f-unstable.drv' (round 2/2)...
+output '/nix/store/6xg356v9gl03hpbbg8gws77n19qanh02-unstable' of '/nix/store/ch6llwpr2h8c3jmnf3f2ghkhx59aa97f-unstable.drv' differs from '/nix/store/6xg356v9gl03hpbbg8gws77n19qanh02-unstable.check' from previous round
+/nix/store/6xg356v9gl03hpbbg8gws77n19qanh02-unstable
+</screen>
+  </para>
+</section>
+</chapter>

doc/manual/advanced-topics/distributed-builds.xml

@@ -180,4 +180,11 @@ builders = @/etc/nix/machines
 causes the list of machines in <filename>/etc/nix/machines</filename>
 to be included. (This is the default.)</para>
 
+<para>If you want the builders to use caches, you likely want to set
+the option <link linkend='conf-builders-use-substitutes'><literal>builders-use-substitutes</literal></link>
+in your local <filename>nix.conf</filename>.</para>
+
+<para>To build only on remote builders and disable building on the local machine,
+you can use the option <option>--max-jobs 0</option>.</para>
+
 </chapter>

doc/manual/advanced-topics/post-build-hook.xml

@@ -0,0 +1,160 @@
+<chapter xmlns="http://docbook.org/ns/docbook"
+      xmlns:xlink="http://www.w3.org/1999/xlink"
+      xmlns:xi="http://www.w3.org/2001/XInclude"
+      xml:id="chap-post-build-hook"
+      version="5.0"
+      >
+
+<title>Using the <xref linkend="conf-post-build-hook" /></title>
+<subtitle>Uploading to an S3-compatible binary cache after each build</subtitle>
+
+
+<section xml:id="chap-post-build-hook-caveats">
+  <title>Implementation Caveats</title>
+  <para>Here we use the post-build hook to upload to a binary cache.
+  This is a simple and working example, but it is not suitable for all
+  use cases.</para>
+
+  <para>The post build hook program runs after each executed build,
+  and blocks the build loop. The build loop exits if the hook program
+  fails.</para>
+
+  <para>Concretely, this implementation will make Nix slow or unusable
+  when the internet is slow or unreliable.</para>
+
+  <para>A more advanced implementation might pass the store paths to a
+  user-supplied daemon or queue for processing the store paths outside
+  of the build loop.</para>
+</section>
+
+<section>
+  <title>Prerequisites</title>
+
+  <para>
+    This tutorial assumes you have configured an S3-compatible binary cache
+    according to the instructions at
+    <xref linkend="ssec-s3-substituter-authenticated-writes" />, and
+    that the <literal>root</literal> user's default AWS profile can
+    upload to the bucket.
+  </para>
+</section>
+
+<section>
+  <title>Set up a Signing Key</title>
+  <para>Use <command>nix-store --generate-binary-cache-key</command> to
+  create our public and private signing keys. We will sign paths
+  with the private key, and distribute the public key for verifying
+  the authenticity of the paths.</para>
+
+  <screen>
+# nix-store --generate-binary-cache-key example-nix-cache-1 /etc/nix/key.private /etc/nix/key.public
+# cat /etc/nix/key.public
+example-nix-cache-1:1/cKDz3QCCOmwcztD2eV6Coggp6rqc9DGjWv7C0G+rM=
+</screen>
+
+<para>Then, add the public key and the cache URL to your
+<filename>nix.conf</filename>'s <xref linkend="conf-trusted-public-keys" />
+and <xref linkend="conf-substituters" /> like:</para>
+
+<programlisting>
+substituters = https://cache.nixos.org/ s3://example-nix-cache
+trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= example-nix-cache-1:1/cKDz3QCCOmwcztD2eV6Coggp6rqc9DGjWv7C0G+rM=
+</programlisting>
+
+<para>we will restart the Nix daemon a later step.</para>
+</section>
+
+<section>
+  <title>Implementing the build hook</title>
+  <para>Write the following script to
+  <filename>/etc/nix/upload-to-cache.sh</filename>:
+  </para>
+
+  <programlisting>
+#!/bin/sh
+
+set -eu
+set -f # disable globbing
+export IFS=' '
+
+echo "Signing paths" $OUT_PATHS
+nix sign-paths --key-file /etc/nix/key.private $OUT_PATHS
+echo "Uploading paths" $OUT_PATHS
+exec nix copy --to 's3://example-nix-cache' $OUT_PATHS
+</programlisting>
+
+  <note>
+    <title>Should <literal>$OUT_PATHS</literal> be quoted?</title>
+    <para>
+      The <literal>$OUT_PATHS</literal> variable is a space-separated
+      list of Nix store paths. In this case, we expect and want the
+      shell to perform word splitting to make each output path its
+      own argument to <command>nix sign-paths</command>. Nix guarantees
+      the paths will not contain any spaces, however a store path
+      might contain glob characters. The <command>set -f</command>
+      disables globbing in the shell.
+    </para>
+  </note>
+  <para>
+    Then make sure the hook program is executable by the <literal>root</literal> user:
+    <screen>
+# chmod +x /etc/nix/upload-to-cache.sh
+</screen></para>
+</section>
+
+<section>
+  <title>Updating Nix Configuration</title>
+
+  <para>Edit <filename>/etc/nix/nix.conf</filename> to run our hook,
+  by adding the following configuration snippet at the end:</para>
+
+  <programlisting>
+post-build-hook = /etc/nix/upload-to-cache.sh
+</programlisting>
+
+<para>Then, restart the <command>nix-daemon</command>.</para>
+</section>
+
+<section>
+  <title>Testing</title>
+
+  <para>Build any derivation, for example:</para>
+
+  <screen>
+$ nix-build -E '(import &lt;nixpkgs&gt; {}).writeText "example" (builtins.toString builtins.currentTime)'
+these derivations will be built:
+  /nix/store/s4pnfbkalzy5qz57qs6yybna8wylkig6-example.drv
+building '/nix/store/s4pnfbkalzy5qz57qs6yybna8wylkig6-example.drv'...
+running post-build-hook '/home/grahamc/projects/github.com/NixOS/nix/post-hook.sh'...
+post-build-hook: Signing paths /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example
+post-build-hook: Uploading paths /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example
+/nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example
+</screen>
+
+  <para>Then delete the path from the store, and try substituting it from the binary cache:</para>
+  <screen>
+$ rm ./result
+$ nix-store --delete /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example
+</screen>
+
+<para>Now, copy the path back from the cache:</para>
+<screen>
+$ nix store --realize /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example
+copying path '/nix/store/m8bmqwrch6l3h8s0k3d673xpmipcdpsa-example from 's3://example-nix-cache'...
+warning: you did not specify '--add-root'; the result might be removed by the garbage collector
+/nix/store/m8bmqwrch6l3h8s0k3d673xpmipcdpsa-example
+</screen>
+</section>
+<section>
+  <title>Conclusion</title>
+  <para>
+    We now have a Nix installation configured to automatically sign and
+    upload every local build to a remote binary cache.
+  </para>
+
+  <para>
+    Before deploying this to production, be sure to consider the
+    implementation caveats in <xref linkend="chap-post-build-hook-caveats" />.
+  </para>
+</section>
+</chapter>

doc/manual/command-ref/conf-file.xml

@@ -1,7 +1,9 @@
+<?xml version="1.0" encoding="utf-8"?>
 <refentry xmlns="http://docbook.org/ns/docbook"
           xmlns:xlink="http://www.w3.org/1999/xlink"
           xmlns:xi="http://www.w3.org/2001/XInclude"
-          xml:id="sec-conf-file">
+          xml:id="sec-conf-file"
+          version="5">
 
 <refmeta>
   <refentrytitle>nix.conf</refentrytitle>
@@ -236,8 +238,74 @@ false</literal>.</para>
     linkend='opt-cores'>--cores</option> command line switch and
     defaults to <literal>1</literal>.  The value <literal>0</literal>
     means that the builder should use all available CPU cores in the
-    system.</para></listitem>
+    system.</para>
 
+    <para>See also <xref linkend="chap-tuning-cores-and-jobs" />.</para></listitem>
+  </varlistentry>
+
+  <varlistentry xml:id="conf-diff-hook"><term><literal>diff-hook</literal></term>
+  <listitem>
+    <para>
+      Absolute path to an executable capable of diffing build results.
+      The hook executes if <xref linkend="conf-run-diff-hook" /> is
+      true, and the output of a build is known to not be the same.
+      This program is not executed to determine if two results are the
+      same.
+    </para>
+
+    <para>
+      The diff hook is executed by the same user and group who ran the
+      build. However, the diff hook does not have write access to the
+      store path just built.
+    </para>
+
+    <para>The diff hook program receives three parameters:</para>
+
+    <orderedlist>
+      <listitem>
+        <para>
+          A path to the previous build's results
+        </para>
+      </listitem>
+
+      <listitem>
+        <para>
+          A path to the current build's results
+        </para>
+      </listitem>
+
+      <listitem>
+        <para>
+          The path to the build's derivation
+        </para>
+      </listitem>
+
+      <listitem>
+        <para>
+          The path to the build's scratch directory. This directory
+          will exist only if the build was run with
+          <option>--keep-failed</option>.
+        </para>
+      </listitem>
+    </orderedlist>
+
+    <para>
+      The stderr and stdout output from the diff hook will not be
+      displayed to the user. Instead, it will print to the nix-daemon's
+      log.
+    </para>
+
+    <para>When using the Nix daemon, <literal>diff-hook</literal> must
+    be set in the <filename>nix.conf</filename> configuration file, and
+    cannot be passed at the command line.
+    </para>
+  </listitem>
+  </varlistentry>
+
+  <varlistentry xml:id="conf-enforce-determinism">
+    <term><literal>enforce-determinism</literal></term>
+
+    <listitem><para>See <xref linkend="conf-repeat" />.</para></listitem>
   </varlistentry>
 
   <varlistentry xml:id="conf-extra-sandbox-paths">
@@ -365,7 +433,7 @@ builtins.fetchurl {
   <varlistentry xml:id="conf-keep-env-derivations"><term><literal>keep-env-derivations</literal></term>
 
     <listitem><para>If <literal>false</literal> (default), derivations
-    are not stored in Nix user environments.  That is, the derivation
+    are not stored in Nix user environments.  That is, the derivations of
     any build-time-only dependencies may be garbage-collected.</para>
 
     <para>If <literal>true</literal>, when you add a Nix derivation to
@@ -415,8 +483,10 @@ builtins.fetchurl {
 
   <varlistentry xml:id="conf-max-free"><term><literal>max-free</literal></term>
 
-    <listitem><para>This option defines after how many free bytes to stop collecting
-    garbage once the <literal>min-free</literal> condition gets triggered.</para></listitem>
+    <listitem><para>When a garbage collection is triggered by the
+    <literal>min-free</literal> option, it stops as soon as
+    <literal>max-free</literal> bytes are available. The default is
+    infinity (i.e. delete all garbage).</para></listitem>
 
   </varlistentry>
 
@@ -431,7 +501,10 @@ builtins.fetchurl {
     regardless).  It can be
     overridden using the <option
     linkend='opt-max-jobs'>--max-jobs</option> (<option>-j</option>)
-    command line switch.</para></listitem>
+    command line switch.</para>
+
+    <para>See also <xref linkend="chap-tuning-cores-and-jobs" />.</para>
+    </listitem>
   </varlistentry>
 
   <varlistentry xml:id="conf-max-silent-time"><term><literal>max-silent-time</literal></term>
@@ -457,9 +530,11 @@ builtins.fetchurl {
   <varlistentry xml:id="conf-min-free"><term><literal>min-free</literal></term>
 
     <listitem>
-      <para>When the disk reaches <literal>min-free</literal> bytes of free disk space during a build, nix
-        will start to garbage-collection until <literal>max-free</literal> bytes are available on the disk.
-        A value of <literal>0</literal> (the default) means that this feature is disabled.</para>
+      <para>When free disk space in <filename>/nix/store</filename>
+      drops below <literal>min-free</literal> during a build, Nix
+      performs a garbage-collection until <literal>max-free</literal>
+      bytes are available or there is no more garbage.  A value of
+      <literal>0</literal> (the default) disables this feature.</para>
     </listitem>
 
   </varlistentry>
@@ -589,15 +664,71 @@ password <replaceable>my-password</replaceable>
 
   </varlistentry>
 
+  <varlistentry xml:id="conf-post-build-hook">
+    <term><literal>post-build-hook</literal></term>
+    <listitem>
+      <para>Optional. The path to a program to execute after each build.</para>
+
+      <para>This option is only settable in the global
+      <filename>nix.conf</filename>, or on the command line by trusted
+      users.</para>
+
+      <para>When using the nix-daemon, the daemon executes the hook as
+      <literal>root</literal>. If the nix-daemon is not involved, the
+      hook runs as the user executing the nix-build.</para>
+
+      <itemizedlist>
+        <listitem><para>The hook executes after an evaluation-time build.</para></listitem>
+        <listitem><para>The hook does not execute on substituted paths.</para></listitem>
+        <listitem><para>The hook's output always goes to the user's terminal.</para></listitem>
+        <listitem><para>If the hook fails, the build succeeds but no further builds execute.</para></listitem>
+        <listitem><para>The hook executes synchronously, and blocks other builds from progressing while it runs.</para></listitem>
+      </itemizedlist>
+
+      <para>The program executes with no arguments. The program's environment
+      contains the following environment variables:</para>
+
+      <variablelist>
+        <varlistentry>
+          <term><envar>DRV_PATH</envar></term>
+          <listitem>
+            <para>The derivation for the built paths.</para>
+            <para>Example:
+            <literal>/nix/store/5nihn1a7pa8b25l9zafqaqibznlvvp3f-bash-4.4-p23.drv</literal>
+            </para>
+          </listitem>
+        </varlistentry>
+
+        <varlistentry>
+          <term><envar>OUT_PATHS</envar></term>
+          <listitem>
+            <para>Output paths of the built derivation, separated by a space character.</para>
+            <para>Example:
+            <literal>/nix/store/zf5lbh336mnzf1nlswdn11g4n2m8zh3g-bash-4.4-p23-dev
+            /nix/store/rjxwxwv1fpn9wa2x5ssk5phzwlcv4mna-bash-4.4-p23-doc
+            /nix/store/6bqvbzjkcp9695dq0dpl5y43nvy37pq1-bash-4.4-p23-info
+            /nix/store/r7fng3kk3vlpdlh2idnrbn37vh4imlj2-bash-4.4-p23-man
+            /nix/store/xfghy8ixrhz3kyy6p724iv3cxji088dx-bash-4.4-p23</literal>.
+            </para>
+          </listitem>
+        </varlistentry>
+      </variablelist>
+
+      <para>See <xref linkend="chap-post-build-hook" /> for an example
+      implementation.</para>
+
+    </listitem>
+  </varlistentry>
+
   <varlistentry xml:id="conf-repeat"><term><literal>repeat</literal></term>
 
     <listitem><para>How many times to repeat builds to check whether
     they are deterministic. The default value is 0. If the value is
     non-zero, every build is repeated the specified number of
     times. If the contents of any of the runs differs from the
-    previous ones, the build is rejected and the resulting store paths
-    are not registered as “valid” in Nix’s database.</para></listitem>
-
+    previous ones and <xref linkend="conf-enforce-determinism" /> is
+    true, the build is rejected and the resulting store paths are not
+    registered as “valid” in Nix’s database.</para></listitem>
   </varlistentry>
 
   <varlistentry xml:id="conf-require-sigs"><term><literal>require-sigs</literal></term>
@@ -628,6 +759,19 @@ password <replaceable>my-password</replaceable>
 
   </varlistentry>
 
+  <varlistentry xml:id="conf-run-diff-hook"><term><literal>run-diff-hook</literal></term>
+  <listitem>
+    <para>
+      If true, enable the execution of <xref linkend="conf-diff-hook" />.
+    </para>
+
+    <para>
+      When using the Nix daemon, <literal>run-diff-hook</literal> must
+      be set in the <filename>nix.conf</filename> configuration file,
+      and cannot be passed at the command line.
+    </para>
+  </listitem>
+  </varlistentry>
 
   <varlistentry xml:id="conf-sandbox"><term><literal>sandbox</literal></term>
 
@@ -657,7 +801,8 @@ password <replaceable>my-password</replaceable>
     <varname>__noChroot</varname> attribute set to
     <literal>true</literal> do not run in sandboxes.</para>
 
-    <para>The default is <literal>false</literal>.</para>
+    <para>The default is <literal>true</literal> on Linux and
+    <literal>false</literal> on all other platforms.</para>
 
     </listitem>
 
@@ -728,6 +873,14 @@ password <replaceable>my-password</replaceable>
 
   </varlistentry>
 
+  <varlistentry xml:id="conf-stalled-download-timeout"><term><literal>stalled-download-timeout</literal></term>
+    <listitem>
+      <para>The timeout (in seconds) for receiving data from servers
+      during download. Nix cancels idle downloads after this timeout's
+      duration.</para>
+    </listitem>
+  </varlistentry>
+
   <varlistentry xml:id="conf-substituters"><term><literal>substituters</literal></term>
 
     <listitem><para>A list of URLs of substituters, separated by
@@ -783,6 +936,31 @@ requiredSystemFeatures = [ "kvm" ];
 
   </varlistentry>
 
+  <varlistentry xml:id="conf-tarball-ttl"><term><literal>tarball-ttl</literal></term>
+
+    <listitem>
+      <para>Default: <literal>3600</literal> seconds.</para>
+
+      <para>The number of seconds a downloaded tarball is considered
+      fresh. If the cached tarball is stale, Nix will check whether
+      it is still up to date using the ETag header. Nix will download
+      a new version if the ETag header is unsupported, or the
+      cached ETag doesn't match.
+      </para>
+
+      <para>Setting the TTL to <literal>0</literal> forces Nix to always
+      check if the tarball is up to date.</para>
+
+      <para>Nix caches tarballs in
+      <filename>$XDG_CACHE_HOME/nix/tarballs</filename>.</para>
+
+      <para>Files fetched via <envar>NIX_PATH</envar>,
+      <function>fetchGit</function>, <function>fetchMercurial</function>,
+      <function>fetchTarball</function>, and <function>fetchurl</function>
+      respect this TTL.
+      </para>
+    </listitem>
+  </varlistentry>
 
   <varlistentry xml:id="conf-timeout"><term><literal>timeout</literal></term>
 
@@ -803,6 +981,34 @@ requiredSystemFeatures = [ "kvm" ];
 
   </varlistentry>
 
+  <varlistentry xml:id="conf-trace-function-calls"><term><literal>trace-function-calls</literal></term>
+
+    <listitem>
+
+      <para>Default: <literal>false</literal>.</para>
+
+      <para>If set to <literal>true</literal>, the Nix evaluator will
+      trace every function call. Nix will print a log message at the
+      "vomit" level for every function entrance and function exit.</para>
+
+      <informalexample><screen>
+function-trace entered undefined position at 1565795816999559622
+function-trace exited undefined position at 1565795816999581277
+function-trace entered /nix/store/.../example.nix:226:41 at 1565795253249935150
+function-trace exited /nix/store/.../example.nix:226:41 at 1565795253249941684
+</screen></informalexample>
+
+      <para>The <literal>undefined position</literal> means the function
+      call is a builtin.</para>
+
+      <para>Use the <literal>contrib/stack-collapse.py</literal> script
+      distributed with the Nix source code to convert the trace logs
+      in to a format suitable for <command>flamegraph.pl</command>.</para>
+
+    </listitem>
+
+  </varlistentry>
+
   <varlistentry xml:id="conf-trusted-public-keys"><term><literal>trusted-public-keys</literal></term>
 
     <listitem><para>A whitespace-separated list of public keys. When

doc/manual/command-ref/env-common.xml

@@ -14,7 +14,8 @@
 <varlistentry><term><envar>IN_NIX_SHELL</envar></term>
 
   <listitem><para>Indicator that tells if the current environment was set up by
-  <command>nix-shell</command>.</para></listitem>
+  <command>nix-shell</command>.  Since Nix 2.0 the values are
+  <literal>"pure"</literal> and <literal>"impure"</literal></para></listitem>
 
 </varlistentry>
 
@@ -52,10 +53,15 @@ nixpkgs=/home/eelco/Dev/nixpkgs-branch:/etc/nixos</screen>
     <envar>NIX_PATH</envar> to
 
     <screen>
-nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-14.12.tar.gz</screen>
+nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-15.09.tar.gz</screen>
 
     tells Nix to download the latest revision in the Nixpkgs/NixOS
-    14.12 channel.</para>
+    15.09 channel.</para>
+
+    <para>A following shorthand can be used to refer to the official channels:
+    
+    <screen>nixpkgs=channel:nixos-15.09</screen>
+    </para>
 
     <para>The search path can be extended using the <option
     linkend="opt-I">-I</option> option, which takes precedence over
@@ -116,7 +122,7 @@ $ mount -o bind /mnt/otherdisk/nix /nix</screen>
 <varlistentry><term><envar>NIX_LOG_DIR</envar></term>
 
   <listitem><para>Overrides the location of the Nix log directory
-  (default <filename><replaceable>prefix</replaceable>/log/nix</filename>).</para></listitem>
+  (default <filename><replaceable>prefix</replaceable>/var/log/nix</filename>).</para></listitem>
 
 </varlistentry>
 

doc/manual/command-ref/nix-build.xml

@@ -30,6 +30,7 @@
       <replaceable>attrPath</replaceable>
     </arg>
     <arg><option>--no-out-link</option></arg>
+    <arg><option>--dry-run</option></arg>
     <arg>
       <group choice='req'>
         <arg choice='plain'><option>--out-link</option></arg>
@@ -98,6 +99,10 @@ also <xref linkend="sec-common-options" />.</phrase></para>
 
   </varlistentry>
 
+  <varlistentry><term><option>--dry-run</option></term>
+   <listitem><para>Show what store paths would be built or downloaded</para></listitem>
+  </varlistentry>
+
   <varlistentry xml:id='opt-out-link'><term><option>--out-link</option> /
   <option>-o</option> <replaceable>outlink</replaceable></term>
 

doc/manual/command-ref/nix-channel.xml

@@ -31,12 +31,11 @@
 
 <refsection><title>Description</title>
 
-<para>A Nix channel is a mechanism that allows you to automatically stay
-up-to-date with a set of pre-built Nix expressions.  A Nix channel is
-just a URL that points to a place containing both a set of Nix
-expressions and a pointer to a binary cache.  <phrase
-condition="manual">See also <xref linkend="sec-channels"
-/>.</phrase></para>
+<para>A Nix channel is a mechanism that allows you to automatically
+stay up-to-date with a set of pre-built Nix expressions.  A Nix
+channel is just a URL that points to a place containing a set of Nix
+expressions.  <phrase condition="manual">See also <xref
+linkend="sec-channels" />.</phrase></para>
 
 <para>This command has the following operations:
 
@@ -172,18 +171,6 @@ following files:</para>
 
   </varlistentry>
 
-  <varlistentry><term><filename>binary-cache-url</filename></term>
-
-    <listitem><para>A file containing the URL to a binary cache (such
-    as <uri>https://cache.nixos.org</uri>). Nix will automatically
-    check this cache for pre-built binaries, if the user has
-    sufficient rights to add binary caches. For instance, in a
-    multi-user Nix setup, the binary caches provided by the channels
-    of the root user are used automatically, but caches corresponding
-    to the channels of non-root users are ignored.</para></listitem>
-
-  </varlistentry>
-
 </variablelist>
 
 </refsection>

doc/manual/command-ref/nix-env.xml

@@ -221,31 +221,53 @@ also <xref linkend="sec-common-options" />.</phrase></para>
 
   <varlistentry><term><filename>~/.nix-defexpr</filename></term>
 
-    <listitem><para>A directory that contains the default Nix
+    <listitem><para>The source for the default Nix
     expressions used by the <option>--install</option>,
     <option>--upgrade</option>, and <option>--query
-    --available</option> operations to obtain derivations.  The
+    --available</option> operations to obtain derivations. The
     <option>--file</option> option may be used to override this
     default.</para>
 
-    <para>The Nix expressions in this directory are combined into a
-    single set, with each file as an attribute that has the name of
-    the file.  Thus, if <filename>~/.nix-defexpr</filename> contains
-    two files, <filename>foo</filename> and <filename>bar</filename>,
+    <para>If <filename>~/.nix-defexpr</filename> is a file,
+    it is loaded as a Nix expression. If the expression
+    is a set, it is used as the default Nix expression.
+    If the expression is a function, an empty set is passed
+    as argument and the return value is used as
+    the default Nix expression.</para>
+
+    <para>If <filename>~/.nix-defexpr</filename> is a directory
+    containing a <filename>default.nix</filename> file, that file
+    is loaded as in the above paragraph.</para>
+
+    <para>If <filename>~/.nix-defexpr</filename> is a directory without
+    a <filename>default.nix</filename> file, then its contents
+    (both files and subdirectories) are loaded as Nix expressions.
+    The expressions are combined into a single set, each expression
+    under an attribute with the same name as the original file
+    or subdirectory.
+    </para>
+
+    <para>For example, if <filename>~/.nix-defexpr</filename> contains
+    two files, <filename>foo.nix</filename> and <filename>bar.nix</filename>,
     then the default Nix expression will essentially be
 
 <programlisting>
 {
-  foo = import ~/.nix-defexpr/foo;
-  bar = import ~/.nix-defexpr/bar;
+  foo = import ~/.nix-defexpr/foo.nix;
+  bar = import ~/.nix-defexpr/bar.nix;
 }</programlisting>
 
     </para>
 
+    <para>The file <filename>manifest.nix</filename> is always ignored.
+    Subdirectories without a <filename>default.nix</filename> file
+    are traversed recursively in search of more Nix expressions,
+    but the names of these intermediate directories are not
+    added to the attribute paths of the default Nix expression.</para>
+
     <para>The command <command>nix-channel</command> places symlinks
     to the downloaded Nix expressions from each subscribed channel in
     this directory.</para>
-
     </listitem>
 
   </varlistentry>
@@ -637,7 +659,7 @@ upgrading `mozilla-1.2' to `mozilla-1.4'</screen>
 <literal>gcc-3.3.1</literal> are split into two parts: the package
 name (<literal>gcc</literal>), and the version
 (<literal>3.3.1</literal>).  The version part starts after the first
-dash not following by a letter.  <varname>x</varname> is considered an
+dash not followed by a letter.  <varname>x</varname> is considered an
 upgrade of <varname>y</varname> if their package names match, and the
 version of <varname>y</varname> is higher that that of
 <varname>x</varname>.</para>
@@ -1348,10 +1370,13 @@ profile.  The generations can be a list of generation numbers, the
 special value <literal>old</literal> to delete all non-current
 generations,  a value such as <literal>30d</literal> to delete all
 generations older than the specified number of days (except for the
-generation that was active at that point in time), or a value such as.
-<literal>+5</literal> to only keep the specified items older than the
-current generation. Periodically deleting old generations is important
-to make garbage collection effective.</para>
+generation that was active at that point in time), or a value such as
+<literal>+5</literal> to keep the last <literal>5</literal> generations
+ignoring any newer than current, e.g., if <literal>30</literal> is the current
+generation <literal>+5</literal> will delete generation <literal>25</literal>
+and all older generations.
+Periodically deleting old generations is important to make garbage collection
+effective.</para>
 
 </refsection>
 

doc/manual/command-ref/nix-prefetch-url.xml

@@ -53,7 +53,7 @@ avoided.</para>
 <para>If <replaceable>hash</replaceable> is specified, then a download
 is not performed if the Nix store already contains a file with the
 same hash and base name.  Otherwise, the file is downloaded, and an
-error if signaled if the actual hash of the file does not match the
+error is signaled if the actual hash of the file does not match the
 specified hash.</para>
 
 <para>This command prints the hash on standard output.  Additionally,

doc/manual/command-ref/nix-shell.xml

@@ -337,7 +337,7 @@ following Haskell script uses a specific branch of Nixpkgs/NixOS (the
 
 <programlisting><![CDATA[
 #! /usr/bin/env nix-shell
-#! nix-shell -i runghc -p haskellPackages.ghc haskellPackages.HTTP haskellPackages.tagsoup
+#! nix-shell -i runghc -p "haskellPackages.ghcWithPackages (ps: [ps.HTTP ps.tagsoup])"
 #! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-18.03.tar.gz
 
 import Network.HTTP

doc/manual/command-ref/nix-store.xml

@@ -215,6 +215,48 @@ printed.)</para>
 
 </variablelist>
 
+<para>Special exit codes:</para>
+
+<variablelist>
+
+  <varlistentry><term><literal>100</literal></term>
+    <listitem><para>Generic build failure, the builder process
+    returned with a non-zero exit code.</para></listitem>
+  </varlistentry>
+
+  <varlistentry><term><literal>101</literal></term>
+    <listitem><para>Build timeout, the build was aborted because it
+    did not complete within the specified <link
+    linkend='conf-timeout'><literal>timeout</literal></link>.
+    </para></listitem>
+  </varlistentry>
+
+  <varlistentry><term><literal>102</literal></term>
+    <listitem><para>Hash mismatch, the build output was rejected
+    because it does not match the specified <link
+    linkend="fixed-output-drvs"><varname>outputHash</varname></link>.
+    </para></listitem>
+  </varlistentry>
+
+  <varlistentry><term><literal>104</literal></term>
+    <listitem><para>Not deterministic, the build succeeded in check
+    mode but the resulting output is not binary reproducable.</para>
+    </listitem>
+  </varlistentry>
+
+</variablelist>
+
+<para>With the <option>--keep-going</option> flag it's possible for
+multiple failures to occur, in this case the 1xx status codes are or combined
+using binary or. <screen>
+1100100
+   ^^^^
+   |||`- timeout
+   ||`-- output hash mismatch
+   |`--- build failure
+   `---- not deterministic
+</screen></para>
+
 </refsection>
 
 
@@ -883,6 +925,60 @@ $ nix-store --add ./foo.c
 
 </refsection>
 
+<!--######################################################################-->
+
+<refsection><title>Operation <option>--add-fixed</option></title>
+
+<refsection><title>Synopsis</title>
+
+<cmdsynopsis>
+  <command>nix-store</command>
+  <arg><option>--recursive</option></arg>
+  <arg choice='plain'><option>--add-fixed</option></arg>
+  <arg choice='plain'><replaceable>algorithm</replaceable></arg>
+  <arg choice='plain' rep='repeat'><replaceable>paths</replaceable></arg>
+</cmdsynopsis>
+
+</refsection>
+
+<refsection><title>Description</title>
+
+<para>The operation <option>--add-fixed</option> adds the specified paths to
+the Nix store.  Unlike <option>--add</option> paths are registered using the
+specified hashing algorithm, resulting in the same output path as a fixed output
+derivation.  This can be used for sources that are not available from a public
+url or broke since the download expression was written.
+</para>
+
+<para>This operation has the following options:
+
+<variablelist>
+
+  <varlistentry><term><option>--recursive</option></term>
+
+    <listitem><para>
+      Use recursive instead of flat hashing mode, used when adding directories
+      to the store.
+    </para></listitem>
+
+  </varlistentry>
+
+</variablelist>
+
+</para>
+
+</refsection>
+
+<refsection><title>Example</title>
+
+<screen>
+$ nix-store --add-fixed sha256 ./hello-2.10.tar.gz
+/nix/store/3x7dwzq014bblazs7kq20p9hyzz0qh8g-hello-2.10.tar.gz</screen>
+
+</refsection>
+
+</refsection>
+
 
 
 <!--######################################################################-->
@@ -1282,6 +1378,7 @@ ktorrent-2.2.1/NEWS
   <cmdsynopsis>
     <command>nix-store</command>
     <arg choice='plain'><option>--dump-db</option></arg>
+    <arg rep='repeat'><replaceable>paths</replaceable></arg>
   </cmdsynopsis>
 </refsection>
 
@@ -1292,6 +1389,13 @@ Nix database to standard output.  It can be loaded into an empty Nix
 store using <option>--load-db</option>.  This is useful for making
 backups and when migrating to different database schemas.</para>
 
+<para>By default, <option>--dump-db</option> will dump the entire Nix
+database.  When one or more store paths is passed, only the subset of
+the Nix database for those store paths is dumped.  As with
+<option>--export</option>, the user is responsible for passing all the
+store paths for a closure.  See <option>--export</option> for an
+example.</para>
+
 </refsection>
 
 </refsection>

doc/manual/command-ref/opt-common.xml

@@ -107,14 +107,22 @@
 <varlistentry xml:id="opt-max-jobs"><term><option>--max-jobs</option> / <option>-j</option>
 <replaceable>number</replaceable></term>
 
-  <listitem><para>Sets the maximum number of build jobs that Nix will
+  <listitem>
+
+  <para>Sets the maximum number of build jobs that Nix will
   perform in parallel to the specified number.  Specify
   <literal>auto</literal> to use the number of CPUs in the system.
   The default is specified by the <link
   linkend='conf-max-jobs'><literal>max-jobs</literal></link>
   configuration setting, which itself defaults to
   <literal>1</literal>.  A higher value is useful on SMP systems or to
-  exploit I/O latency.</para></listitem>
+  exploit I/O latency.</para>
+
+  <para> Setting it to <literal>0</literal> disallows building on the local
+  machine, which is useful when you want builds to happen only on remote
+  builders.</para>
+
+  </listitem>
 
 </varlistentry>
 

doc/manual/expressions/builtins.xml

@@ -23,6 +23,7 @@ available as <function>builtins.derivation</function>.</para>
 
   <varlistentry xml:id='builtin-abort'>
     <term><function>abort</function> <replaceable>s</replaceable></term>
+    <term><function>builtins.abort</function> <replaceable>s</replaceable></term>
 
     <listitem><para>Abort Nix expression evaluation, print error
     message <replaceable>s</replaceable>.</para></listitem>
@@ -169,18 +170,6 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting>
   </varlistentry>
 
 
-  <varlistentry xml:id='builtin-splitVersion'>
-    <term><function>builtins.splitVersion</function>
-    <replaceable>s</replaceable></term>
-
-    <listitem><para>Split a string representing a version into its
-    components, by the same version splitting logic underlying the
-    version comparison in <link linkend="ssec-version-comparisons">
-    <command>nix-env -u</command></link>.</para></listitem>
-
-  </varlistentry>
-
-
   <varlistentry xml:id='builtin-concatLists'>
     <term><function>builtins.concatLists</function>
     <replaceable>lists</replaceable></term>
@@ -251,6 +240,8 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting>
   <varlistentry xml:id='builtin-derivation'>
     <term><function>derivation</function>
     <replaceable>attrs</replaceable></term>
+    <term><function>builtins.derivation</function>
+    <replaceable>attrs</replaceable></term>
 
     <listitem><para><function>derivation</function> is described in
     <xref linkend='ssec-derivation' />.</para></listitem>
@@ -260,6 +251,7 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting>
 
   <varlistentry xml:id='builtin-dirOf'>
     <term><function>dirOf</function> <replaceable>s</replaceable></term>
+    <term><function>builtins.dirOf</function> <replaceable>s</replaceable></term>
 
     <listitem><para>Return the directory part of the string
     <replaceable>s</replaceable>, that is, everything before the final
@@ -297,7 +289,7 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting>
 
     <listitem><para>Return element <replaceable>n</replaceable> from
     the list <replaceable>xs</replaceable>.  Elements are counted
-    starting from 0.  A fatal error occurs in the index is out of
+    starting from 0.  A fatal error occurs if the index is out of
     bounds.</para></listitem>
 
   </varlistentry>
@@ -318,6 +310,8 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting>
   <varlistentry xml:id='builtin-fetchTarball'>
     <term><function>fetchTarball</function>
     <replaceable>url</replaceable></term>
+    <term><function>builtins.fetchTarball</function>
+    <replaceable>url</replaceable></term>
 
     <listitem><para>Download the specified URL, unpack it and return
     the path of the unpacked tree. The file must be a tape archive
@@ -341,7 +335,7 @@ stdenv.mkDerivation { … }
     You can change the cache timeout either on the command line with
     <option>--option tarball-ttl <replaceable>number of seconds</replaceable></option> or
     in the Nix configuration file with this option:
-    <literal>tarball-ttl <replaceable>number of seconds to cache</replaceable></literal>.
+    <literal><xref linkend="conf-tarball-ttl" /> <replaceable>number of seconds to cache</replaceable></literal>.
     </para>
 
     <para>Note that when obtaining the hash with <varname>nix-prefetch-url
@@ -419,6 +413,13 @@ stdenv.mkDerivation { … }
               This is often a branch or tag name. Defaults to
               <literal>HEAD</literal>.
             </para>
+
+            <para>
+              By default, the <varname>ref</varname> value is prefixed
+              with <literal>refs/heads/</literal>. As of Nix 2.3.0
+              Nix will not prefix <literal>refs/heads/</literal> if
+              <varname>ref</varname> starts with <literal>refs/</literal>.
+            </para>
           </listitem>
         </varlistentry>
       </variablelist>
@@ -432,6 +433,14 @@ stdenv.mkDerivation { … }
 }</programlisting>
       </example>
 
+      <example>
+        <title>Fetching an arbitrary ref</title>
+        <programlisting>builtins.fetchGit {
+  url = "https://github.com/NixOS/nix.git";
+  ref = "refs/heads/0.5-release";
+}</programlisting>
+      </example>
+
       <example>
         <title>Fetching a repository's specific commit on an arbitrary branch</title>
         <para>
@@ -478,11 +487,8 @@ stdenv.mkDerivation { … }
         <title>Fetching a tag</title>
         <programlisting>builtins.fetchGit {
   url = "https://github.com/nixos/nix.git";
-  ref = "tags/1.9";
+  ref = "refs/tags/1.9";
 }</programlisting>
-        <note><para>Due to a bug (<link
-        xlink:href="https://github.com/NixOS/nix/issues/2385">#2385</link>),
-        only non-annotated tags can be fetched.</para></note>
       </example>
 
       <example>
@@ -492,7 +498,7 @@ stdenv.mkDerivation { … }
            fetch the latest version of a remote branch.
         </para>
         <note><para>Nix will refetch the branch in accordance to
-        <option>tarball-ttl</option>.</para></note>
+        <xref linkend="conf-tarball-ttl" />.</para></note>
         <note><para>This behavior is disabled in
         <emphasis>Pure evaluation mode</emphasis>.</para></note>
         <programlisting>builtins.fetchGit {
@@ -693,8 +699,21 @@ builtins.genList (x: x * x) 5
     <listitem><para>Return a base-16 representation of the
     cryptographic hash of string <replaceable>s</replaceable>.  The
     hash algorithm specified by <replaceable>type</replaceable> must
-    be one of <literal>"md5"</literal>, <literal>"sha1"</literal> or
-    <literal>"sha256"</literal>.</para></listitem>
+    be one of <literal>"md5"</literal>, <literal>"sha1"</literal>,
+    <literal>"sha256"</literal> or <literal>"sha512"</literal>.</para></listitem>
+
+  </varlistentry>
+
+
+  <varlistentry xml:id='builtin-hashFile'>
+    <term><function>builtins.hashFile</function>
+    <replaceable>type</replaceable> <replaceable>p</replaceable></term>
+
+    <listitem><para>Return a base-16 representation of the
+    cryptographic hash of the file at path <replaceable>p</replaceable>.  The
+    hash algorithm specified by <replaceable>type</replaceable> must
+    be one of <literal>"md5"</literal>, <literal>"sha1"</literal>,
+    <literal>"sha256"</literal> or <literal>"sha512"</literal>.</para></listitem>
 
   </varlistentry>
 
@@ -714,6 +733,8 @@ builtins.genList (x: x * x) 5
   <varlistentry xml:id='builtin-import'>
     <term><function>import</function>
     <replaceable>path</replaceable></term>
+    <term><function>builtins.import</function>
+    <replaceable>path</replaceable></term>
 
     <listitem><para>Load, parse and return the Nix expression in the
     file <replaceable>path</replaceable>.  If <replaceable>path
@@ -725,6 +746,11 @@ builtins.genList (x: x * x) 5
     separate file, and use it from Nix expressions in other
     files.</para>
 
+    <note><para>Unlike some languages, <function>import</function> is a regular
+    function in Nix. Paths using the angle bracket syntax (e.g., <function>
+    import</function> <replaceable>&lt;foo&gt;</replaceable>) are normal path
+    values (see <xref linkend='ssec-values' />).</para></note>
+
     <para>A Nix expression loaded by <function>import</function> must
     not contain any <emphasis>free variables</emphasis> (identifiers
     that are not defined in the Nix expression itself and are not
@@ -853,10 +879,20 @@ x: x + 456</programlisting>
 
   </varlistentry>
 
+  <varlistentry><term><function>builtins.isPath</function>
+  <replaceable>e</replaceable></term>
+
+    <listitem><para>Return <literal>true</literal> if
+    <replaceable>e</replaceable> evaluates to a path, and
+    <literal>false</literal> otherwise.</para></listitem>
+
+  </varlistentry>
 
   <varlistentry xml:id='builtin-isNull'>
     <term><function>isNull</function>
     <replaceable>e</replaceable></term>
+    <term><function>builtins.isNull</function>
+    <replaceable>e</replaceable></term>
 
     <listitem><para>Return <literal>true</literal> if
     <replaceable>e</replaceable> evaluates to <literal>null</literal>,
@@ -925,6 +961,8 @@ builtins.listToAttrs
   <varlistentry xml:id='builtin-map'>
     <term><function>map</function>
     <replaceable>f</replaceable> <replaceable>list</replaceable></term>
+    <term><function>builtins.map</function>
+    <replaceable>f</replaceable> <replaceable>list</replaceable></term>
 
     <listitem><para>Apply the function <replaceable>f</replaceable> to
     each element in the list <replaceable>list</replaceable>.  For
@@ -1082,6 +1120,16 @@ Evaluates to <literal>[ "foo" ]</literal>.
 
   </varlistentry>
 
+  <varlistentry xml:id='builtin-placeholder'>
+    <term><function>builtins.placeholder</function>
+    <replaceable>output</replaceable></term>
+
+    <listitem><para>Return a placeholder string for the specified
+    <replaceable>output</replaceable> that will be substituted by the
+    corresponding output path at build time. Typical outputs would be
+    <literal>"out"</literal>, <literal>"bin"</literal> or
+    <literal>"dev"</literal>.</para></listitem>
+  </varlistentry>
 
   <varlistentry xml:id='builtin-readDir'>
     <term><function>builtins.readDir</function>
@@ -1119,6 +1167,8 @@ Evaluates to <literal>[ "foo" ]</literal>.
   <varlistentry xml:id='builtin-removeAttrs'>
     <term><function>removeAttrs</function>
     <replaceable>set</replaceable> <replaceable>list</replaceable></term>
+    <term><function>builtins.removeAttrs</function>
+    <replaceable>set</replaceable> <replaceable>list</replaceable></term>
 
     <listitem><para>Remove the attributes listed in
     <replaceable>list</replaceable> from
@@ -1225,6 +1275,19 @@ Evaluates to <literal>[ "  " [ "FOO" ] "   " ]</literal>.
     </para></listitem>
   </varlistentry>
 
+
+  <varlistentry xml:id='builtin-splitVersion'>
+    <term><function>builtins.splitVersion</function>
+    <replaceable>s</replaceable></term>
+
+    <listitem><para>Split a string representing a version into its
+    components, by the same version splitting logic underlying the
+    version comparison in <link linkend="ssec-version-comparisons">
+    <command>nix-env -u</command></link>.</para></listitem>
+
+  </varlistentry>
+
+
   <varlistentry xml:id='builtin-stringLength'>
     <term><function>builtins.stringLength</function>
     <replaceable>e</replaceable></term>
@@ -1287,6 +1350,8 @@ builtins.substring 0 3 "nixos"
   <varlistentry xml:id='builtin-throw'>
     <term><function>throw</function>
     <replaceable>s</replaceable></term>
+    <term><function>builtins.throw</function>
+    <replaceable>s</replaceable></term>
 
     <listitem><para>Throw an error message
     <replaceable>s</replaceable>.  This usually aborts Nix expression
@@ -1405,6 +1470,7 @@ in foo</programlisting>
 
   <varlistentry xml:id='builtin-toString'>
     <term><function>toString</function> <replaceable>e</replaceable></term>
+    <term><function>builtins.toString</function> <replaceable>e</replaceable></term>
 
     <listitem><para>Convert the expression
     <replaceable>e</replaceable> to a string.
@@ -1415,7 +1481,7 @@ in foo</programlisting>
       <listitem><para>A set containing <literal>{ __toString = self: ...; }</literal>.</para></listitem>
       <listitem><para>An integer.</para></listitem>
       <listitem><para>A list, in which case the string representations of its elements are joined with spaces.</para></listitem>
-      <listitem><para>A Boolean (<literal>false</literal> yields <literal>""</literal>, <literal>true</literal> yields <literal>"1"</literal>.</para></listitem>
+      <listitem><para>A Boolean (<literal>false</literal> yields <literal>""</literal>, <literal>true</literal> yields <literal>"1"</literal>).</para></listitem>
       <listitem><para><literal>null</literal>, which yields the empty string.</para></listitem>
     </itemizedlist>
     </listitem>
@@ -1554,12 +1620,18 @@ stdenv.mkDerivation (rec {
     <term><function>builtins.tryEval</function>
     <replaceable>e</replaceable></term>
 
-    <listitem><para>Try to evaluate <replaceable>e</replaceable>.
+    <listitem><para>Try to shallowly evaluate <replaceable>e</replaceable>.
     Return a set containing the attributes <literal>success</literal>
     (<literal>true</literal> if <replaceable>e</replaceable> evaluated
     successfully, <literal>false</literal> if an error was thrown) and
     <literal>value</literal>, equalling <replaceable>e</replaceable>
-    if successful and <literal>false</literal> otherwise.
+    if successful and <literal>false</literal> otherwise. Note that this
+    doesn't evaluate <replaceable>e</replaceable> deeply, so
+    <literal>let e = { x = throw ""; }; in (builtins.tryEval e).success
+    </literal> will be <literal>true</literal>. Using <literal>builtins.deepSeq
+    </literal> one can get the expected result: <literal>let e = { x = throw "";
+    }; in (builtins.tryEval (builtins.deepSeq e e)).success</literal> will be
+    <literal>false</literal>.
     </para></listitem>
 
   </varlistentry>

doc/manual/expressions/language-constructs.xml

@@ -43,7 +43,7 @@ encountered</quote>).</para></footnote>.</para>
 
 <simplesect xml:id="sect-let-expressions"><title>Let-expressions</title>
 
-<para>A let-expression allows you define local variables for an
+<para>A let-expression allows you to define local variables for an
 expression.  For instance,
 
 <programlisting>
@@ -217,7 +217,25 @@ but can also be written as:
   ellipsis(<literal>...</literal>) as you can access attribute names as 
   <literal>a</literal>, using <literal>args.a</literal>, which was given as an
   additional attribute to the function.
-  </para></listitem>
+  </para>
+
+  <warning>
+   <para>
+    The <literal>args@</literal> expression is bound to the argument passed to the function which
+    means that attributes with defaults that aren't explicitly specified in the function call
+    won't cause an evaluation error, but won't exist in <literal>args</literal>.
+   </para>
+   <para>
+    For instance
+<programlisting>
+let
+  function = args@{ a ? 23, ... }: args;
+in
+ function {}
+</programlisting>
+    will evaluate to an empty attribute set.
+   </para>
+  </warning></listitem>
 
 </itemizedlist>
 

doc/manual/expressions/language-operators.xml

@@ -15,13 +15,16 @@ weakest binding).</para>
   <tgroup cols='3'>
     <thead>
       <row>
+        <entry>Name</entry>
         <entry>Syntax</entry>
         <entry>Associativity</entry>
         <entry>Description</entry>
+        <entry>Precedence</entry>
       </row>
     </thead>
     <tbody>
       <row>
+        <entry>Select</entry>
         <entry><replaceable>e</replaceable> <literal>.</literal>
         <replaceable>attrpath</replaceable>
         [ <literal>or</literal> <replaceable>def</replaceable> ]
@@ -33,19 +36,25 @@ weakest binding).</para>
         dot-separated list of attribute names.)  If the attribute
         doesn’t exist, return <replaceable>def</replaceable> if
         provided, otherwise abort evaluation.</entry>
+        <entry>1</entry>
       </row>
       <row>
+        <entry>Application</entry>
         <entry><replaceable>e1</replaceable> <replaceable>e2</replaceable></entry>
         <entry>left</entry>
         <entry>Call function <replaceable>e1</replaceable> with
         argument <replaceable>e2</replaceable>.</entry>
+        <entry>2</entry>
       </row>
       <row>
+        <entry>Arithmetic Negation</entry>
         <entry><literal>-</literal> <replaceable>e</replaceable></entry>
         <entry>none</entry>
         <entry>Arithmetic negation.</entry>
+        <entry>3</entry>
       </row>
       <row>
+        <entry>Has Attribute</entry>
         <entry><replaceable>e</replaceable> <literal>?</literal>
         <replaceable>attrpath</replaceable></entry>
         <entry>none</entry>
@@ -53,34 +62,69 @@ weakest binding).</para>
         the attribute denoted by <replaceable>attrpath</replaceable>;
         return <literal>true</literal> or
         <literal>false</literal>.</entry>
+        <entry>4</entry>
       </row>
       <row>
+        <entry>List Concatenation</entry>
         <entry><replaceable>e1</replaceable> <literal>++</literal> <replaceable>e2</replaceable></entry>
         <entry>right</entry>
         <entry>List concatenation.</entry>
+        <entry>5</entry>
       </row>
       <row>
+        <entry>Multiplication</entry>
         <entry>
           <replaceable>e1</replaceable> <literal>*</literal> <replaceable>e2</replaceable>,
+        </entry>
+        <entry>left</entry>
+        <entry>Arithmetic multiplication.</entry>
+        <entry>6</entry>
+      </row>
+      <row>
+        <entry>Division</entry>
+        <entry>
           <replaceable>e1</replaceable> <literal>/</literal> <replaceable>e2</replaceable>
         </entry>
         <entry>left</entry>
-        <entry>Arithmetic multiplication and division.</entry>
+        <entry>Arithmetic division.</entry>
+        <entry>6</entry>
       </row>
       <row>
+        <entry>Addition</entry>
+        <entry>
+          <replaceable>e1</replaceable> <literal>+</literal> <replaceable>e2</replaceable>
+        </entry>
+        <entry>left</entry>
+        <entry>Arithmetic addition.</entry>
+        <entry>7</entry>
+      </row>
+      <row>
+        <entry>Subtraction</entry>
         <entry>
-          <replaceable>e1</replaceable> <literal>+</literal> <replaceable>e2</replaceable>,
           <replaceable>e1</replaceable> <literal>-</literal> <replaceable>e2</replaceable>
         </entry>
         <entry>left</entry>
-        <entry>Arithmetic addition and subtraction.  String or path concatenation (only by <literal>+</literal>).</entry>
+        <entry>Arithmetic subtraction.</entry>
+        <entry>7</entry>
       </row>
       <row>
+        <entry>String Concatenation</entry>
+        <entry>
+          <replaceable>string1</replaceable> <literal>+</literal> <replaceable>string2</replaceable>
+        </entry>
+        <entry>left</entry>
+        <entry>String concatenation.</entry>
+        <entry>7</entry>
+      </row>
+      <row>
+        <entry>Not</entry>
         <entry><literal>!</literal> <replaceable>e</replaceable></entry>
         <entry>none</entry>
         <entry>Boolean negation.</entry>
+        <entry>8</entry>
       </row>
       <row>
+        <entry>Update</entry>
         <entry><replaceable>e1</replaceable> <literal>//</literal>
         <replaceable>e2</replaceable></entry>
         <entry>right</entry>
@@ -89,47 +133,90 @@ weakest binding).</para>
         <replaceable>e2</replaceable> (with the latter taking
         precedence over the former in case of equally named
         attributes).</entry>
+        <entry>9</entry>
       </row>
       <row>
+        <entry>Less Than</entry>
         <entry>
           <replaceable>e1</replaceable> <literal>&lt;</literal> <replaceable>e2</replaceable>,
-          <replaceable>e1</replaceable> <literal>&gt;</literal> <replaceable>e2</replaceable>,
-          <replaceable>e1</replaceable> <literal>&lt;=</literal> <replaceable>e2</replaceable>,
+        </entry>
+        <entry>none</entry>
+        <entry>Arithmetic comparison.</entry>
+        <entry>10</entry>
+      </row>
+      <row>
+        <entry>Less Than or Equal To</entry>
+        <entry>
+          <replaceable>e1</replaceable> <literal>&lt;=</literal> <replaceable>e2</replaceable>
+        </entry>
+        <entry>none</entry>
+        <entry>Arithmetic comparison.</entry>
+        <entry>10</entry>
+      </row>
+      <row>
+        <entry>Greater Than</entry>
+        <entry>
+          <replaceable>e1</replaceable> <literal>&gt;</literal> <replaceable>e2</replaceable>
+        </entry>
+        <entry>none</entry>
+        <entry>Arithmetic comparison.</entry>
+        <entry>10</entry>
+      </row>
+      <row>
+        <entry>Greater Than or Equal To</entry>
+        <entry>
           <replaceable>e1</replaceable> <literal>&gt;=</literal> <replaceable>e2</replaceable>
         </entry>
         <entry>none</entry>
         <entry>Arithmetic comparison.</entry>
+        <entry>10</entry>
       </row>
       <row>
+        <entry>Equality</entry>
+        <entry>
+          <replaceable>e1</replaceable> <literal>==</literal> <replaceable>e2</replaceable>
+        </entry>
+        <entry>none</entry>
+        <entry>Equality.</entry>
+        <entry>11</entry>
+      </row>
+      <row>
+        <entry>Inequality</entry>
         <entry>
-          <replaceable>e1</replaceable> <literal>==</literal> <replaceable>e2</replaceable>,
           <replaceable>e1</replaceable> <literal>!=</literal> <replaceable>e2</replaceable>
         </entry>
         <entry>none</entry>
-        <entry>Equality and inequality.</entry>
+        <entry>Inequality.</entry>
+        <entry>11</entry>
       </row>
       <row>
+        <entry>Logical AND</entry>
         <entry><replaceable>e1</replaceable> <literal>&amp;&amp;</literal>
         <replaceable>e2</replaceable></entry>
         <entry>left</entry>
         <entry>Logical AND.</entry>
+        <entry>12</entry>
       </row>
       <row>
+        <entry>Logical OR</entry>
         <entry><replaceable>e1</replaceable> <literal>||</literal>
         <replaceable>e2</replaceable></entry>
         <entry>left</entry>
         <entry>Logical OR.</entry>
+        <entry>13</entry>
       </row>
       <row>
+        <entry>Logical Implication</entry>
         <entry><replaceable>e1</replaceable> <literal>-></literal>
         <replaceable>e2</replaceable></entry>
         <entry>none</entry>
         <entry>Logical implication (equivalent to
         <literal>!<replaceable>e1</replaceable> ||
         <replaceable>e2</replaceable></literal>).</entry>
+        <entry>14</entry>
       </row>
     </tbody>
   </tgroup>
 </table>
 
-</section>
+</section>

doc/manual/expressions/simple-building-testing.xml

@@ -43,7 +43,7 @@ use <command>nix-build</command>’s <option
 linkend='opt-out-link'>-o</option> switch to give the symlink another
 name.</para>
 
-<para>Nix has a transactional semantics.  Once a build finishes
+<para>Nix has transactional semantics.  Once a build finishes
 successfully, Nix makes a note of this in its database: it registers
 that the path denoted by <envar>out</envar> is now
 <quote>valid</quote>.  If you try to build the derivation again, Nix

doc/manual/installation/env-variables.xml

@@ -67,5 +67,23 @@ $ sudo launchctl kickstart -k system/org.nixos.nix-daemon
 </screen>
 </section>
 
+<section xml:id="sec-installer-proxy-settings">
+
+<title>Proxy Environment Variables</title>
+
+<para>The Nix installer has special handling for these proxy-related
+environment variables:
+<varname>http_proxy</varname>, <varname>https_proxy</varname>,
+<varname>ftp_proxy</varname>, <varname>no_proxy</varname>,
+<varname>HTTP_PROXY</varname>, <varname>HTTPS_PROXY</varname>,
+<varname>FTP_PROXY</varname>, <varname>NO_PROXY</varname>.
+</para>
+<para>If any of these variables are set when running the Nix installer,
+then the installer will create an override file at
+<filename>/etc/systemd/system/nix-daemon.service.d/override.conf</filename>
+so <command>nix-daemon</command> will use them.
+</para>
+</section>
+
 </section>
 </chapter>

doc/manual/installation/prerequisites-source.xml

@@ -13,7 +13,7 @@
   <listitem><para>Bash Shell. The <literal>./configure</literal> script
   relies on bashisms, so Bash is required.</para></listitem>
 
-  <listitem><para>A version of GCC or Clang that supports C++14.</para></listitem>
+  <listitem><para>A version of GCC or Clang that supports C++17.</para></listitem>
 
   <listitem><para><command>pkg-config</command> to locate
   dependencies.  If your distribution does not provide it, you can get
@@ -62,6 +62,10 @@
   1.66.0 or higher. It can be obtained from the official web site
   <link xlink:href="https://www.boost.org/" />.</para></listitem>
 
+  <listitem><para>The <literal>editline</literal> library of version
+  1.14.0 or higher. It can be obtained from the its repository
+  <link xlink:href="https://github.com/troglobit/editline" />.</para></listitem>
+
   <listitem><para>The <command>xmllint</command> and
   <command>xsltproc</command> programs to build this manual and the
   man-pages.  These are part of the <literal>libxml2</literal> and

doc/manual/local.mk

@@ -4,7 +4,6 @@ ifeq ($(doc_generate),yes)
 XSLTPROC = $(xsltproc) --nonet $(xmlflags) \
   --param section.autolabel 1 \
   --param section.label.includes.component.label 1 \
-  --param html.stylesheet \'style.css\' \
   --param xref.with.number.and.title 1 \
   --param toc.section.depth 3 \
   --param admon.style \'\' \
@@ -66,7 +65,7 @@ $(d)/manual.html: $(d)/manual.xml $(MANUAL_SRCS) $(d)/manual.is-valid
 	  $(docbookxsl)/profiling/profile.xsl $< | \
 	  $(XSLTPROC) --output $@ $(docbookxsl)/xhtml/docbook.xsl -
 
-$(foreach file, $(d)/manual.html $(d)/style.css, $(eval $(call install-data-in, $(file), $(docdir)/manual)))
+$(foreach file, $(d)/manual.html, $(eval $(call install-data-in, $(file), $(docdir)/manual)))
 
 $(foreach file, $(wildcard $(d)/figures/*.png), $(eval $(call install-data-in, $(file), $(docdir)/manual/figures)))
 

doc/manual/packages/basic-package-mgmt.xml

@@ -24,11 +24,11 @@ symlinks to the files of the active applications.  </para>
 <para>Components are installed from a set of <emphasis>Nix
 expressions</emphasis> that tell Nix how to build those packages,
 including, if necessary, their dependencies.  There is a collection of
-Nix expressions called the Nix Package collection that contains
+Nix expressions called the Nixpkgs package collection that contains
 packages ranging from basic development stuff such as GCC and Glibc,
 to end-user applications like Mozilla Firefox.  (Nix is however not
-tied to the Nix Package collection; you could write your own Nix
-expressions based on it, or completely new ones.)</para>
+tied to the Nixpkgs package collection; you could write your own Nix
+expressions based on Nixpkgs, or completely new ones.)</para>
 
 <para>You can manually download the latest version of Nixpkgs from
 <link xlink:href='http://nixos.org/nixpkgs/download.html'/>. However,

doc/manual/packages/garbage-collection.xml

@@ -52,12 +52,13 @@ garbage collector as follows:
 <screen>
 $ nix-store --gc</screen>
 
-The behaviour of the gargage collector is affected by the <literal>keep-
-derivations</literal> (default: true) and <literal>keep-outputs</literal>
+The behaviour of the gargage collector is affected by the 
+<literal>keep-derivations</literal> (default: true) and <literal>keep-outputs</literal>
 (default: false) options in the Nix configuration file. The defaults will ensure
-that all derivations that are not build-time dependencies of garbage collector roots
-will be collected but that all output paths that are not runtime dependencies
-will be collected. (This is usually what you want, but while you are developing
+that all derivations that are build-time dependencies of garbage collector roots
+will be kept and that all output paths that are runtime dependencies
+will be kept as well. All other derivations or paths will be collected. 
+(This is usually what you want, but while you are developing
 it may make sense to keep outputs to ensure that rebuild times are quick.)
 
 If you are feeling uncertain, you can also first view what files would

doc/manual/packages/s3-substituter.xml

@@ -89,7 +89,7 @@ the S3 URL:</para>
     "Version": "2012-10-17",
     "Statement": [
         {
-            "Sid": "AlowDirectReads",
+            "Sid": "AllowDirectReads",
             "Action": [
                 "s3:GetObject",
                 "s3:GetBucketLocation"
@@ -113,7 +113,7 @@ the S3 URL:</para>
   exactly <uri>s3://example-nix-cache</uri>.</para>
 
   <para>Nix will use the <link
-  xlink:href="https://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/credentials.html#credentials-default.">default
+  xlink:href="https://docs.aws.amazon.com/sdk-for-cpp/v1/developer-guide/credentials.html">default
   credential provider chain</link> for authenticating requests to
   Amazon S3.</para>
 
@@ -138,7 +138,7 @@ the S3 URL:</para>
   be <uri>s3://example-nix-cache</uri>.</para>
 
   <para>Nix will use the <link
-  xlink:href="https://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/credentials.html#credentials-default.">default
+  xlink:href="https://docs.aws.amazon.com/sdk-for-cpp/v1/developer-guide/credentials.html">default
   credential provider chain</link> for authenticating requests to
   Amazon S3.</para>
 
@@ -159,7 +159,6 @@ the S3 URL:</para>
         "s3:ListBucket",
         "s3:ListBucketMultipartUploads",
         "s3:ListMultipartUploadParts",
-        "s3:ListObjects",
         "s3:PutObject"
       ],
       "Resource": [

doc/manual/release-notes/release-notes.xml

@@ -12,6 +12,7 @@
 </partintro>
 -->
 
+<xi:include href="rl-2.3.xml" />
 <xi:include href="rl-2.2.xml" />
 <xi:include href="rl-2.1.xml" />
 <xi:include href="rl-2.0.xml" />

doc/manual/release-notes/rl-2.3.xml

@@ -0,0 +1,91 @@
+<section xmlns="http://docbook.org/ns/docbook"
+      xmlns:xlink="http://www.w3.org/1999/xlink"
+      xmlns:xi="http://www.w3.org/2001/XInclude"
+      version="5.0"
+      xml:id="ssec-relnotes-2.3">
+
+<title>Release 2.3 (2019-09-04)</title>
+
+<para>This is primarily a bug fix release. However, it makes some
+incompatible changes:</para>
+
+<itemizedlist>
+
+  <listitem>
+    <para>Nix now uses BSD file locks instead of POSIX file
+    locks. Because of this, you should not use Nix 2.3 and previous
+    releases at the same time on a Nix store.</para>
+  </listitem>
+
+</itemizedlist>
+
+<para>It also has the following changes:</para>
+
+<itemizedlist>
+
+  <listitem>
+    <para><function>builtins.fetchGit</function>'s <varname>ref</varname>
+    argument now allows specifying an absolute remote ref.
+    Nix will automatically prefix <varname>ref</varname> with
+    <literal>refs/heads</literal> only if <varname>ref</varname> doesn't
+    already begin with <literal>refs/</literal>.
+    </para>
+  </listitem>
+
+  <listitem>
+    <para>The installer now enables sandboxing by default on Linux when the
+    system has the necessary kernel support.
+    </para>
+  </listitem>
+
+  <listitem>
+    <para>The <literal>max-jobs</literal> setting now defaults to 1.</para>
+  </listitem>
+
+  <listitem>
+    <para>New builtin functions:
+    <literal>builtins.isPath</literal>,
+    <literal>builtins.hashFile</literal>.
+    </para>
+  </listitem>
+
+  <listitem>
+    <para>The <command>nix</command> command has a new
+    <option>--print-build-logs</option> (<option>-L</option>) flag to
+    print build log output to stderr, rather than showing the last log
+    line in the progress bar. To distinguish between concurrent
+    builds, log lines are prefixed by the name of the package.
+    </para>
+  </listitem>
+
+  <listitem>
+    <para>Builds are now executed in a pseudo-terminal, and the
+    <envar>TERM</envar> environment variable is set to
+    <literal>xterm-256color</literal>. This allows many programs
+    (e.g. <command>gcc</command>, <command>clang</command>,
+    <command>cmake</command>) to print colorized log output.</para>
+  </listitem>
+
+  <listitem>
+    <para>Add <option>--no-net</option> convenience flag. This flag
+    disables substituters; sets the <literal>tarball-ttl</literal>
+    setting to infinity (ensuring that any previously downloaded files
+    are considered current); and disables retrying downloads and sets
+    the connection timeout to the minimum. This flag is enabled
+    automatically if there are no configured non-loopback network
+    interfaces.</para>
+  </listitem>
+
+  <listitem>
+    <para>Add a <literal>post-build-hook</literal> setting to run a
+    program after a build has succeeded.</para>
+  </listitem>
+
+  <listitem>
+    <para>Add a <literal>trace-function-calls</literal> setting to log
+    the duration of Nix function calls to stderr.</para>
+  </listitem>
+
+</itemizedlist>
+
+</section>

doc/manual/style.css

@@ -1,263 +0,0 @@
-/* Copied from http://bakefile.sourceforge.net/, which appears
-   licensed under the GNU GPL. */
-
-
-/***************************************************************************
-                             Basic headers and text:
- ***************************************************************************/
-
-body
-{
-    font-family: "Nimbus Sans L", sans-serif;
-    background: white;
-    margin: 2em 1em 2em 1em;
-}
-
-h1, h2, h3, h4
-{
-    color: #005aa0;
-}
-
-h1 /* title */
-{
-    font-size: 200%;
-}
-
-div.part h1
-{
-    font-size: 240%;
-}
-
-h2 /* chapters, appendices, subtitle */
-{
-    font-size: 180%;
-}
-
-div.part
-{
-    margin-top: 4em;
-}
-
-/* Extra space between chapters, appendices. */
-div.chapter > div.titlepage h2, div.appendix > div.titlepage h2 
-{ 
-    margin-top: 1.5em;
-}
-
-div.section > div.titlepage h2 /* sections */
-{
-    font-size: 150%;
-    margin-top: 1.5em;
-}
-
-h3 /* subsections */
-{
-    font-size: 125%;
-}
-
-div.simplesect h2
-{
-    font-size: 110%;
-}
-
-div.appendix h3
-{
-    font-size: 150%;
-    margin-top: 1.5em;
-}
-
-div.refentry\.separator
-{
-    margin-top: 2.5em;
-    margin-bottom: 2em;
-}
-
-div.refnamediv h2, div.refsynopsisdiv h2, div.refsection h2 /* refentry parts */
-{
-    margin-top: 1.4em;
-    font-size: 125%;
-}
-
-div.refsection h3
-{
-    font-size: 110%;
-}
-
-
-/***************************************************************************
-                               Examples:
- ***************************************************************************/
-
-div.example
-{
-    border: 1px solid #b0b0b0;
-    padding: 6px 6px;
-    margin-left: 1.5em;
-    margin-right: 1.5em;
-    background: #f4f4f8;
-    border-radius: 0.4em;
-}
-
-div.example p.title
-{
-    margin-top: 0em;
-}
-
-div.example pre
-{
-}
-
-
-/***************************************************************************
-                            Screen dumps:
- ***************************************************************************/
-
-pre.screen, pre.programlisting
-{
-    padding: 6px 6px;
-    margin-left: 1.5em;
-    margin-right: 1.5em;
-    color: #600000;
-    background: #f4f4f8;
-    font-family: monospace;
-}
-
-div.example pre.programlisting
-{
-    border: 0px;
-    padding: 0 0;
-    margin: 0 0 0 0;
-}
-
-
-/***************************************************************************
-                               Notes, warnings etc:
- ***************************************************************************/
-
-.note, .warning
-{
-    border: 1px solid #b0b0b0;
-    padding: 3px 3px;
-    margin-left: 1.5em;
-    margin-right: 1.5em;
-    margin-bottom: 1em;
-    padding: 0.3em 0.3em 0.3em 0.3em;
-    background: #fffff5;
-    border-radius: 0.4em;
-}
-
-div.note, div.warning
-{
-    font-style: italic;
-}
-
-div.note h3, div.warning h3
-{
-    color: red;
-    font-size: 100%;
-    padding-right: 0.5em;
-    display: inline;
-}
-
-div.note p, div.warning p
-{
-    margin-bottom: 0em;
-}
-
-div.note h3 + p, div.warning h3 + p
-{
-    display: inline;
-}
-
-div.note h3
-{
-    color: blue;
-    font-size: 100%;
-}
-
-div.navfooter *
-{
-    font-size: 90%;
-}
-
-
-/***************************************************************************
-                        Links colors and highlighting: 
- ***************************************************************************/
-
-a { text-decoration: none; }
-a:hover { text-decoration: underline; }
-a:link { color: #0048b3; }
-a:visited { color: #002a6a; }
-
-
-/***************************************************************************
-                              Table of contents:
- ***************************************************************************/
-
-div.toc
-{
-    font-size: 90%;
-}
-
-div.toc dl
-{
-    margin-top: 0em;
-    margin-bottom: 0em;
-}
-
-
-/***************************************************************************
-                               Special elements:
- ***************************************************************************/
-
-tt, code
-{
-    color: #400000;
-}
-
-.term
-{
-    font-weight: bold;
-    
-}
-
-div.variablelist dd p, div.glosslist dd p
-{
-    margin-top: 0em;
-}
-
-div.variablelist dd, div.glosslist dd
-{
-    margin-left: 1.5em;
-}
-
-div.glosslist dt
-{
-    font-style: italic;
-}
-
-.varname
-{
-    color: #400000;
-}
-
-span.command strong
-{
-    font-weight: normal;
-    color: #400000;
-}
-
-div.calloutlist table
-{
-}
-
-table
-{
-    border-collapse: collapse;
-}
-
-div.affiliation
-{
-    font-style: italic;
-}

m4/ax_cxx_compile_stdcxx.m4

@@ -0,0 +1,951 @@
+# ===========================================================================
+#  https://www.gnu.org/software/autoconf-archive/ax_cxx_compile_stdcxx.html
+# ===========================================================================
+#
+# SYNOPSIS
+#
+#   AX_CXX_COMPILE_STDCXX(VERSION, [ext|noext], [mandatory|optional])
+#
+# DESCRIPTION
+#
+#   Check for baseline language coverage in the compiler for the specified
+#   version of the C++ standard.  If necessary, add switches to CXX and
+#   CXXCPP to enable support.  VERSION may be '11' (for the C++11 standard)
+#   or '14' (for the C++14 standard).
+#
+#   The second argument, if specified, indicates whether you insist on an
+#   extended mode (e.g. -std=gnu++11) or a strict conformance mode (e.g.
+#   -std=c++11).  If neither is specified, you get whatever works, with
+#   preference for an extended mode.
+#
+#   The third argument, if specified 'mandatory' or if left unspecified,
+#   indicates that baseline support for the specified C++ standard is
+#   required and that the macro should error out if no mode with that
+#   support is found.  If specified 'optional', then configuration proceeds
+#   regardless, after defining HAVE_CXX${VERSION} if and only if a
+#   supporting mode is found.
+#
+# LICENSE
+#
+#   Copyright (c) 2008 Benjamin Kosnik <bkoz@redhat.com>
+#   Copyright (c) 2012 Zack Weinberg <zackw@panix.com>
+#   Copyright (c) 2013 Roy Stogner <roystgnr@ices.utexas.edu>
+#   Copyright (c) 2014, 2015 Google Inc.; contributed by Alexey Sokolov <sokolov@google.com>
+#   Copyright (c) 2015 Paul Norman <penorman@mac.com>
+#   Copyright (c) 2015 Moritz Klammler <moritz@klammler.eu>
+#   Copyright (c) 2016, 2018 Krzesimir Nowak <qdlacz@gmail.com>
+#   Copyright (c) 2019 Enji Cooper <yaneurabeya@gmail.com>
+#
+#   Copying and distribution of this file, with or without modification, are
+#   permitted in any medium without royalty provided the copyright notice
+#   and this notice are preserved.  This file is offered as-is, without any
+#   warranty.
+
+#serial 11
+
+dnl  This macro is based on the code from the AX_CXX_COMPILE_STDCXX_11 macro
+dnl  (serial version number 13).
+
+AC_DEFUN([AX_CXX_COMPILE_STDCXX], [dnl
+  m4_if([$1], [11], [ax_cxx_compile_alternatives="11 0x"],
+        [$1], [14], [ax_cxx_compile_alternatives="14 1y"],
+        [$1], [17], [ax_cxx_compile_alternatives="17 1z"],
+        [m4_fatal([invalid first argument `$1' to AX_CXX_COMPILE_STDCXX])])dnl
+  m4_if([$2], [], [],
+        [$2], [ext], [],
+        [$2], [noext], [],
+        [m4_fatal([invalid second argument `$2' to AX_CXX_COMPILE_STDCXX])])dnl
+  m4_if([$3], [], [ax_cxx_compile_cxx$1_required=true],
+        [$3], [mandatory], [ax_cxx_compile_cxx$1_required=true],
+        [$3], [optional], [ax_cxx_compile_cxx$1_required=false],
+        [m4_fatal([invalid third argument `$3' to AX_CXX_COMPILE_STDCXX])])
+  AC_LANG_PUSH([C++])dnl
+  ac_success=no
+
+  m4_if([$2], [noext], [], [dnl
+  if test x$ac_success = xno; then
+    for alternative in ${ax_cxx_compile_alternatives}; do
+      switch="-std=gnu++${alternative}"
+      cachevar=AS_TR_SH([ax_cv_cxx_compile_cxx$1_$switch])
+      AC_CACHE_CHECK(whether $CXX supports C++$1 features with $switch,
+                     $cachevar,
+        [ac_save_CXX="$CXX"
+         CXX="$CXX $switch"
+         AC_COMPILE_IFELSE([AC_LANG_SOURCE([_AX_CXX_COMPILE_STDCXX_testbody_$1])],
+          [eval $cachevar=yes],
+          [eval $cachevar=no])
+         CXX="$ac_save_CXX"])
+      if eval test x\$$cachevar = xyes; then
+        CXX="$CXX $switch"
+        if test -n "$CXXCPP" ; then
+          CXXCPP="$CXXCPP $switch"
+        fi
+        ac_success=yes
+        break
+      fi
+    done
+  fi])
+
+  m4_if([$2], [ext], [], [dnl
+  if test x$ac_success = xno; then
+    dnl HP's aCC needs +std=c++11 according to:
+    dnl http://h21007.www2.hp.com/portal/download/files/unprot/aCxx/PDF_Release_Notes/769149-001.pdf
+    dnl Cray's crayCC needs "-h std=c++11"
+    for alternative in ${ax_cxx_compile_alternatives}; do
+      for switch in -std=c++${alternative} +std=c++${alternative} "-h std=c++${alternative}"; do
+        cachevar=AS_TR_SH([ax_cv_cxx_compile_cxx$1_$switch])
+        AC_CACHE_CHECK(whether $CXX supports C++$1 features with $switch,
+                       $cachevar,
+          [ac_save_CXX="$CXX"
+           CXX="$CXX $switch"
+           AC_COMPILE_IFELSE([AC_LANG_SOURCE([_AX_CXX_COMPILE_STDCXX_testbody_$1])],
+            [eval $cachevar=yes],
+            [eval $cachevar=no])
+           CXX="$ac_save_CXX"])
+        if eval test x\$$cachevar = xyes; then
+          CXX="$CXX $switch"
+          if test -n "$CXXCPP" ; then
+            CXXCPP="$CXXCPP $switch"
+          fi
+          ac_success=yes
+          break
+        fi
+      done
+      if test x$ac_success = xyes; then
+        break
+      fi
+    done
+  fi])
+  AC_LANG_POP([C++])
+  if test x$ax_cxx_compile_cxx$1_required = xtrue; then
+    if test x$ac_success = xno; then
+      AC_MSG_ERROR([*** A compiler with support for C++$1 language features is required.])
+    fi
+  fi
+  if test x$ac_success = xno; then
+    HAVE_CXX$1=0
+    AC_MSG_NOTICE([No compiler with C++$1 support was found])
+  else
+    HAVE_CXX$1=1
+    AC_DEFINE(HAVE_CXX$1,1,
+              [define if the compiler supports basic C++$1 syntax])
+  fi
+  AC_SUBST(HAVE_CXX$1)
+])
+
+
+dnl  Test body for checking C++11 support
+
+m4_define([_AX_CXX_COMPILE_STDCXX_testbody_11],
+  _AX_CXX_COMPILE_STDCXX_testbody_new_in_11
+)
+
+
+dnl  Test body for checking C++14 support
+
+m4_define([_AX_CXX_COMPILE_STDCXX_testbody_14],
+  _AX_CXX_COMPILE_STDCXX_testbody_new_in_11
+  _AX_CXX_COMPILE_STDCXX_testbody_new_in_14
+)
+
+m4_define([_AX_CXX_COMPILE_STDCXX_testbody_17],
+  _AX_CXX_COMPILE_STDCXX_testbody_new_in_11
+  _AX_CXX_COMPILE_STDCXX_testbody_new_in_14
+  _AX_CXX_COMPILE_STDCXX_testbody_new_in_17
+)
+
+dnl  Tests for new features in C++11
+
+m4_define([_AX_CXX_COMPILE_STDCXX_testbody_new_in_11], [[
+
+// If the compiler admits that it is not ready for C++11, why torture it?
+// Hopefully, this will speed up the test.
+
+#ifndef __cplusplus
+
+#error "This is not a C++ compiler"
+
+#elif __cplusplus < 201103L
+
+#error "This is not a C++11 compiler"
+
+#else
+
+namespace cxx11
+{
+
+  namespace test_static_assert
+  {
+
+    template <typename T>
+    struct check
+    {
+      static_assert(sizeof(int) <= sizeof(T), "not big enough");
+    };
+
+  }
+
+  namespace test_final_override
+  {
+
+    struct Base
+    {
+      virtual ~Base() {}
+      virtual void f() {}
+    };
+
+    struct Derived : public Base
+    {
+      virtual ~Derived() override {}
+      virtual void f() override {}
+    };
+
+  }
+
+  namespace test_double_right_angle_brackets
+  {
+
+    template < typename T >
+    struct check {};
+
+    typedef check<void> single_type;
+    typedef check<check<void>> double_type;
+    typedef check<check<check<void>>> triple_type;
+    typedef check<check<check<check<void>>>> quadruple_type;
+
+  }
+
+  namespace test_decltype
+  {
+
+    int
+    f()
+    {
+      int a = 1;
+      decltype(a) b = 2;
+      return a + b;
+    }
+
+  }
+
+  namespace test_type_deduction
+  {
+
+    template < typename T1, typename T2 >
+    struct is_same
+    {
+      static const bool value = false;
+    };
+
+    template < typename T >
+    struct is_same<T, T>
+    {
+      static const bool value = true;
+    };
+
+    template < typename T1, typename T2 >
+    auto
+    add(T1 a1, T2 a2) -> decltype(a1 + a2)
+    {
+      return a1 + a2;
+    }
+
+    int
+    test(const int c, volatile int v)
+    {
+      static_assert(is_same<int, decltype(0)>::value == true, "");
+      static_assert(is_same<int, decltype(c)>::value == false, "");
+      static_assert(is_same<int, decltype(v)>::value == false, "");
+      auto ac = c;
+      auto av = v;
+      auto sumi = ac + av + 'x';
+      auto sumf = ac + av + 1.0;
+      static_assert(is_same<int, decltype(ac)>::value == true, "");
+      static_assert(is_same<int, decltype(av)>::value == true, "");
+      static_assert(is_same<int, decltype(sumi)>::value == true, "");
+      static_assert(is_same<int, decltype(sumf)>::value == false, "");
+      static_assert(is_same<int, decltype(add(c, v))>::value == true, "");
+      return (sumf > 0.0) ? sumi : add(c, v);
+    }
+
+  }
+
+  namespace test_noexcept
+  {
+
+    int f() { return 0; }
+    int g() noexcept { return 0; }
+
+    static_assert(noexcept(f()) == false, "");
+    static_assert(noexcept(g()) == true, "");
+
+  }
+
+  namespace test_constexpr
+  {
+
+    template < typename CharT >
+    unsigned long constexpr
+    strlen_c_r(const CharT *const s, const unsigned long acc) noexcept
+    {
+      return *s ? strlen_c_r(s + 1, acc + 1) : acc;
+    }
+
+    template < typename CharT >
+    unsigned long constexpr
+    strlen_c(const CharT *const s) noexcept
+    {
+      return strlen_c_r(s, 0UL);
+    }
+
+    static_assert(strlen_c("") == 0UL, "");
+    static_assert(strlen_c("1") == 1UL, "");
+    static_assert(strlen_c("example") == 7UL, "");
+    static_assert(strlen_c("another\0example") == 7UL, "");
+
+  }
+
+  namespace test_rvalue_references
+  {
+
+    template < int N >
+    struct answer
+    {
+      static constexpr int value = N;
+    };
+
+    answer<1> f(int&)       { return answer<1>(); }
+    answer<2> f(const int&) { return answer<2>(); }
+    answer<3> f(int&&)      { return answer<3>(); }
+
+    void
+    test()
+    {
+      int i = 0;
+      const int c = 0;
+      static_assert(decltype(f(i))::value == 1, "");
+      static_assert(decltype(f(c))::value == 2, "");
+      static_assert(decltype(f(0))::value == 3, "");
+    }
+
+  }
+
+  namespace test_uniform_initialization
+  {
+
+    struct test
+    {
+      static const int zero {};
+      static const int one {1};
+    };
+
+    static_assert(test::zero == 0, "");
+    static_assert(test::one == 1, "");
+
+  }
+
+  namespace test_lambdas
+  {
+
+    void
+    test1()
+    {
+      auto lambda1 = [](){};
+      auto lambda2 = lambda1;
+      lambda1();
+      lambda2();
+    }
+
+    int
+    test2()
+    {
+      auto a = [](int i, int j){ return i + j; }(1, 2);
+      auto b = []() -> int { return '0'; }();
+      auto c = [=](){ return a + b; }();
+      auto d = [&](){ return c; }();
+      auto e = [a, &b](int x) mutable {
+        const auto identity = [](int y){ return y; };
+        for (auto i = 0; i < a; ++i)
+          a += b--;
+        return x + identity(a + b);
+      }(0);
+      return a + b + c + d + e;
+    }
+
+    int
+    test3()
+    {
+      const auto nullary = [](){ return 0; };
+      const auto unary = [](int x){ return x; };
+      using nullary_t = decltype(nullary);
+      using unary_t = decltype(unary);
+      const auto higher1st = [](nullary_t f){ return f(); };
+      const auto higher2nd = [unary](nullary_t f1){
+        return [unary, f1](unary_t f2){ return f2(unary(f1())); };
+      };
+      return higher1st(nullary) + higher2nd(nullary)(unary);
+    }
+
+  }
+
+  namespace test_variadic_templates
+  {
+
+    template <int...>
+    struct sum;
+
+    template <int N0, int... N1toN>
+    struct sum<N0, N1toN...>
+    {
+      static constexpr auto value = N0 + sum<N1toN...>::value;
+    };
+
+    template <>
+    struct sum<>
+    {
+      static constexpr auto value = 0;
+    };
+
+    static_assert(sum<>::value == 0, "");
+    static_assert(sum<1>::value == 1, "");
+    static_assert(sum<23>::value == 23, "");
+    static_assert(sum<1, 2>::value == 3, "");
+    static_assert(sum<5, 5, 11>::value == 21, "");
+    static_assert(sum<2, 3, 5, 7, 11, 13>::value == 41, "");
+
+  }
+
+  // http://stackoverflow.com/questions/13728184/template-aliases-and-sfinae
+  // Clang 3.1 fails with headers of libstd++ 4.8.3 when using std::function
+  // because of this.
+  namespace test_template_alias_sfinae
+  {
+
+    struct foo {};
+
+    template<typename T>
+    using member = typename T::member_type;
+
+    template<typename T>
+    void func(...) {}
+
+    template<typename T>
+    void func(member<T>*) {}
+
+    void test();
+
+    void test() { func<foo>(0); }
+
+  }
+
+}  // namespace cxx11
+
+#endif  // __cplusplus >= 201103L
+
+]])
+
+
+dnl  Tests for new features in C++14
+
+m4_define([_AX_CXX_COMPILE_STDCXX_testbody_new_in_14], [[
+
+// If the compiler admits that it is not ready for C++14, why torture it?
+// Hopefully, this will speed up the test.
+
+#ifndef __cplusplus
+
+#error "This is not a C++ compiler"
+
+#elif __cplusplus < 201402L
+
+#error "This is not a C++14 compiler"
+
+#else
+
+namespace cxx14
+{
+
+  namespace test_polymorphic_lambdas
+  {
+
+    int
+    test()
+    {
+      const auto lambda = [](auto&&... args){
+        const auto istiny = [](auto x){
+          return (sizeof(x) == 1UL) ? 1 : 0;
+        };
+        const int aretiny[] = { istiny(args)... };
+        return aretiny[0];
+      };
+      return lambda(1, 1L, 1.0f, '1');
+    }
+
+  }
+
+  namespace test_binary_literals
+  {
+
+    constexpr auto ivii = 0b0000000000101010;
+    static_assert(ivii == 42, "wrong value");
+
+  }
+
+  namespace test_generalized_constexpr
+  {
+
+    template < typename CharT >
+    constexpr unsigned long
+    strlen_c(const CharT *const s) noexcept
+    {
+      auto length = 0UL;
+      for (auto p = s; *p; ++p)
+        ++length;
+      return length;
+    }
+
+    static_assert(strlen_c("") == 0UL, "");
+    static_assert(strlen_c("x") == 1UL, "");
+    static_assert(strlen_c("test") == 4UL, "");
+    static_assert(strlen_c("another\0test") == 7UL, "");
+
+  }
+
+  namespace test_lambda_init_capture
+  {
+
+    int
+    test()
+    {
+      auto x = 0;
+      const auto lambda1 = [a = x](int b){ return a + b; };
+      const auto lambda2 = [a = lambda1(x)](){ return a; };
+      return lambda2();
+    }
+
+  }
+
+  namespace test_digit_separators
+  {
+
+    constexpr auto ten_million = 100'000'000;
+    static_assert(ten_million == 100000000, "");
+
+  }
+
+  namespace test_return_type_deduction
+  {
+
+    auto f(int& x) { return x; }
+    decltype(auto) g(int& x) { return x; }
+
+    template < typename T1, typename T2 >
+    struct is_same
+    {
+      static constexpr auto value = false;
+    };
+
+    template < typename T >
+    struct is_same<T, T>
+    {
+      static constexpr auto value = true;
+    };
+
+    int
+    test()
+    {
+      auto x = 0;
+      static_assert(is_same<int, decltype(f(x))>::value, "");
+      static_assert(is_same<int&, decltype(g(x))>::value, "");
+      return x;
+    }
+
+  }
+
+}  // namespace cxx14
+
+#endif  // __cplusplus >= 201402L
+
+]])
+
+
+dnl  Tests for new features in C++17
+
+m4_define([_AX_CXX_COMPILE_STDCXX_testbody_new_in_17], [[
+
+// If the compiler admits that it is not ready for C++17, why torture it?
+// Hopefully, this will speed up the test.
+
+#ifndef __cplusplus
+
+#error "This is not a C++ compiler"
+
+#elif __cplusplus < 201703L
+
+#error "This is not a C++17 compiler"
+
+#else
+
+#include <initializer_list>
+#include <utility>
+#include <type_traits>
+
+namespace cxx17
+{
+
+  namespace test_constexpr_lambdas
+  {
+
+    constexpr int foo = [](){return 42;}();
+
+  }
+
+  namespace test::nested_namespace::definitions
+  {
+
+  }
+
+  namespace test_fold_expression
+  {
+
+    template<typename... Args>
+    int multiply(Args... args)
+    {
+      return (args * ... * 1);
+    }
+
+    template<typename... Args>
+    bool all(Args... args)
+    {
+      return (args && ...);
+    }
+
+  }
+
+  namespace test_extended_static_assert
+  {
+
+    static_assert (true);
+
+  }
+
+  namespace test_auto_brace_init_list
+  {
+
+    auto foo = {5};
+    auto bar {5};
+
+    static_assert(std::is_same<std::initializer_list<int>, decltype(foo)>::value);
+    static_assert(std::is_same<int, decltype(bar)>::value);
+  }
+
+  namespace test_typename_in_template_template_parameter
+  {
+
+    template<template<typename> typename X> struct D;
+
+  }
+
+  namespace test_fallthrough_nodiscard_maybe_unused_attributes
+  {
+
+    int f1()
+    {
+      return 42;
+    }
+
+    [[nodiscard]] int f2()
+    {
+      [[maybe_unused]] auto unused = f1();
+
+      switch (f1())
+      {
+      case 17:
+        f1();
+        [[fallthrough]];
+      case 42:
+        f1();
+      }
+      return f1();
+    }
+
+  }
+
+  namespace test_extended_aggregate_initialization
+  {
+
+    struct base1
+    {
+      int b1, b2 = 42;
+    };
+
+    struct base2
+    {
+      base2() {
+        b3 = 42;
+      }
+      int b3;
+    };
+
+    struct derived : base1, base2
+    {
+        int d;
+    };
+
+    derived d1 {{1, 2}, {}, 4};  // full initialization
+    derived d2 {{}, {}, 4};      // value-initialized bases
+
+  }
+
+  namespace test_general_range_based_for_loop
+  {
+
+    struct iter
+    {
+      int i;
+
+      int& operator* ()
+      {
+        return i;
+      }
+
+      const int& operator* () const
+      {
+        return i;
+      }
+
+      iter& operator++()
+      {
+        ++i;
+        return *this;
+      }
+    };
+
+    struct sentinel
+    {
+      int i;
+    };
+
+    bool operator== (const iter& i, const sentinel& s)
+    {
+      return i.i == s.i;
+    }
+
+    bool operator!= (const iter& i, const sentinel& s)
+    {
+      return !(i == s);
+    }
+
+    struct range
+    {
+      iter begin() const
+      {
+        return {0};
+      }
+
+      sentinel end() const
+      {
+        return {5};
+      }
+    };
+
+    void f()
+    {
+      range r {};
+
+      for (auto i : r)
+      {
+        [[maybe_unused]] auto v = i;
+      }
+    }
+
+  }
+
+  namespace test_lambda_capture_asterisk_this_by_value
+  {
+
+    struct t
+    {
+      int i;
+      int foo()
+      {
+        return [*this]()
+        {
+          return i;
+        }();
+      }
+    };
+
+  }
+
+  namespace test_enum_class_construction
+  {
+
+    enum class byte : unsigned char
+    {};
+
+    byte foo {42};
+
+  }
+
+  namespace test_constexpr_if
+  {
+
+    template <bool cond>
+    int f ()
+    {
+      if constexpr(cond)
+      {
+        return 13;
+      }
+      else
+      {
+        return 42;
+      }
+    }
+
+  }
+
+  namespace test_selection_statement_with_initializer
+  {
+
+    int f()
+    {
+      return 13;
+    }
+
+    int f2()
+    {
+      if (auto i = f(); i > 0)
+      {
+        return 3;
+      }
+
+      switch (auto i = f(); i + 4)
+      {
+      case 17:
+        return 2;
+
+      default:
+        return 1;
+      }
+    }
+
+  }
+
+  namespace test_template_argument_deduction_for_class_templates
+  {
+
+    template <typename T1, typename T2>
+    struct pair
+    {
+      pair (T1 p1, T2 p2)
+        : m1 {p1},
+          m2 {p2}
+      {}
+
+      T1 m1;
+      T2 m2;
+    };
+
+    void f()
+    {
+      [[maybe_unused]] auto p = pair{13, 42u};
+    }
+
+  }
+
+  namespace test_non_type_auto_template_parameters
+  {
+
+    template <auto n>
+    struct B
+    {};
+
+    B<5> b1;
+    B<'a'> b2;
+
+  }
+
+  namespace test_structured_bindings
+  {
+
+    int arr[2] = { 1, 2 };
+    std::pair<int, int> pr = { 1, 2 };
+
+    auto f1() -> int(&)[2]
+    {
+      return arr;
+    }
+
+    auto f2() -> std::pair<int, int>&
+    {
+      return pr;
+    }
+
+    struct S
+    {
+      int x1 : 2;
+      volatile double y1;
+    };
+
+    S f3()
+    {
+      return {};
+    }
+
+    auto [ x1, y1 ] = f1();
+    auto& [ xr1, yr1 ] = f1();
+    auto [ x2, y2 ] = f2();
+    auto& [ xr2, yr2 ] = f2();
+    const auto [ x3, y3 ] = f3();
+
+  }
+
+  namespace test_exception_spec_type_system
+  {
+
+    struct Good {};
+    struct Bad {};
+
+    void g1() noexcept;
+    void g2();
+
+    template<typename T>
+    Bad
+    f(T*, T*);
+
+    template<typename T1, typename T2>
+    Good
+    f(T1*, T2*);
+
+    static_assert (std::is_same_v<Good, decltype(f(g1, g2))>);
+
+  }
+
+  namespace test_inline_variables
+  {
+
+    template<class T> void f(T)
+    {}
+
+    template<class T> inline T g(T)
+    {
+      return T{};
+    }
+
+    template<> inline void f<>(int)
+    {}
+
+    template<> int g<>(int)
+    {
+      return 5;
+    }
+
+  }
+
+}  // namespace cxx17
+
+#endif  // __cplusplus < 201703L
+
+]])

m4/ax_cxx_compile_stdcxx_17.m4

@@ -0,0 +1,35 @@
+# =============================================================================
+#  https://www.gnu.org/software/autoconf-archive/ax_cxx_compile_stdcxx_17.html
+# =============================================================================
+#
+# SYNOPSIS
+#
+#   AX_CXX_COMPILE_STDCXX_17([ext|noext], [mandatory|optional])
+#
+# DESCRIPTION
+#
+#   Check for baseline language coverage in the compiler for the C++17
+#   standard; if necessary, add switches to CXX and CXXCPP to enable
+#   support.
+#
+#   This macro is a convenience alias for calling the AX_CXX_COMPILE_STDCXX
+#   macro with the version set to C++17.  The two optional arguments are
+#   forwarded literally as the second and third argument respectively.
+#   Please see the documentation for the AX_CXX_COMPILE_STDCXX macro for
+#   more information.  If you want to use this macro, you also need to
+#   download the ax_cxx_compile_stdcxx.m4 file.
+#
+# LICENSE
+#
+#   Copyright (c) 2015 Moritz Klammler <moritz@klammler.eu>
+#   Copyright (c) 2016 Krzesimir Nowak <qdlacz@gmail.com>
+#
+#   Copying and distribution of this file, with or without modification, are
+#   permitted in any medium without royalty provided the copyright notice
+#   and this notice are preserved. This file is offered as-is, without any
+#   warranty.
+
+#serial 2
+
+AX_REQUIRE_DEFINED([AX_CXX_COMPILE_STDCXX])
+AC_DEFUN([AX_CXX_COMPILE_STDCXX_17], [AX_CXX_COMPILE_STDCXX([17], [$1], [$2])])

maintainers/upload-release.pl

@@ -67,10 +67,10 @@ sub downloadFile {
     }
 
     my $sha256_expected = $buildInfo->{buildproducts}->{$productNr}->{sha256hash} or die;
-    my $sha256_actual = `nix hash-file --type sha256 '$dstFile'`;
+    my $sha256_actual = `nix hash-file --base16 --type sha256 '$dstFile'`;
     chomp $sha256_actual;
     if ($sha256_expected ne $sha256_actual) {
-        print STDERR "file $dstFile is corrupt\n";
+        print STDERR "file $dstFile is corrupt, got $sha256_actual, expected $sha256_expected\n";
         exit 1;
     }
 

misc/launchd/org.nixos.nix-daemon.plist.in

@@ -2,14 +2,23 @@
 <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
 <plist version="1.0">
   <dict>
+    <key>EnvironmentVariables</key>
+    <dict>
+      <key>OBJC_DISABLE_INITIALIZE_FORK_SAFETY</key>
+      <string>YES</string>
+    </dict>
     <key>Label</key>
     <string>org.nixos.nix-daemon</string>
     <key>KeepAlive</key>
     <true/>
     <key>RunAtLoad</key>
     <true/>
-    <key>Program</key>
-    <string>@bindir@/nix-daemon</string>
+    <key>ProgramArguments</key>
+    <array>
+      <string>/bin/sh</string>
+      <string>-c</string>
+      <string>/bin/wait4path /nix/var/nix/profiles/default/bin/nix-daemon &amp;&amp; /nix/var/nix/profiles/default/bin/nix-daemon</string>
+    </array>
     <key>StandardErrorPath</key>
     <string>/var/log/nix-daemon.log</string>
     <key>StandardOutPath</key>

misc/systemd/nix-daemon.service.in

@@ -7,3 +7,6 @@ ConditionPathIsReadWrite=@localstatedir@/nix/daemon-socket
 [Service]
 ExecStart=@@bindir@/nix-daemon nix-daemon --daemon
 KillMode=process
+
+[Install]
+WantedBy=multi-user.target

mk/README.md

@@ -1,6 +0,0 @@
-This is a set of helper Makefiles for doing non-recursive builds with
-GNU Make.  The canonical source can be found at
-https://github.com/edolstra/make-rules.  You should copy the files
-into the `mk` subdirectory of your project.
-
-TODO: write more documentation.

mk/libraries.mk

@@ -91,7 +91,7 @@ define build-library
     $(1)_PATH := $$(_d)/$$($(1)_NAME).$(SO_EXT)
 
     $$($(1)_PATH): $$($(1)_OBJS) $$(_libs) | $$(_d)/
-	$$(trace-ld) $(CXX) -o $$(abspath $$@) -shared $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$($(1)_LDFLAGS_PROPAGATED) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE)) $$($(1)_LDFLAGS_UNINSTALLED)
+	$$(trace-ld) $(CXX) -o $$(abspath $$@) -shared $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$($(1)_LDFLAGS_PROPAGATED) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE)) $$($(1)_LDFLAGS_UNINSTALLED)
 
     ifneq ($(OS), Darwin)
       $(1)_LDFLAGS_USE += -Wl,-rpath,$$(abspath $$(_d))
@@ -105,7 +105,7 @@ define build-library
     $$(eval $$(call create-dir, $$($(1)_INSTALL_DIR)))
 
     $$($(1)_INSTALL_PATH): $$($(1)_OBJS) $$(_libs_final) | $(DESTDIR)$$($(1)_INSTALL_DIR)/
-	$$(trace-ld) $(CXX) -o $$@ -shared $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$($(1)_LDFLAGS_PROPAGATED) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE_INSTALLED))
+	$$(trace-ld) $(CXX) -o $$@ -shared $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$($(1)_LDFLAGS_PROPAGATED) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE_INSTALLED))
 
     $(1)_LDFLAGS_USE_INSTALLED += -L$$(DESTDIR)$$($(1)_INSTALL_DIR) -l$$(patsubst lib%,%,$$(strip $$($(1)_NAME)))
     ifneq ($(OS), Darwin)
@@ -125,7 +125,7 @@ define build-library
     $(1)_PATH := $$(_d)/$$($(1)_NAME).a
 
     $$($(1)_PATH): $$($(1)_OBJS) | $$(_d)/
-	$(trace-ar) ar crs $$@ $$?
+	$(trace-ar) $(AR) crs $$@ $$?
 
     $(1)_LDFLAGS_USE += $$($(1)_PATH) $$($(1)_LDFLAGS)
 

mk/programs.mk

@@ -32,7 +32,7 @@ define build-program
   $$(eval $$(call create-dir, $$(_d)))
 
   $$($(1)_PATH): $$($(1)_OBJS) $$(_libs) | $$(_d)/
-	$$(trace-ld) $(CXX) -o $$@ $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE))
+	$$(trace-ld) $(CXX) -o $$@ $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE))
 
   $(1)_INSTALL_DIR ?= $$(bindir)
   $(1)_INSTALL_PATH := $$($(1)_INSTALL_DIR)/$(1)
@@ -46,7 +46,7 @@ define build-program
     _libs_final := $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_INSTALL_PATH))
 
     $(DESTDIR)$$($(1)_INSTALL_PATH): $$($(1)_OBJS) $$(_libs_final) | $(DESTDIR)$$($(1)_INSTALL_DIR)/
-	$$(trace-ld) $(CXX) -o $$@ $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE_INSTALLED))
+	$$(trace-ld) $(CXX) -o $$@ $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE_INSTALLED))
 
   else
 

nix.spec.in

@@ -106,7 +106,7 @@ chmod 1775 $RPM_BUILD_ROOT/nix/store
 for d in profiles gcroots;
 do
   mkdir -p $RPM_BUILD_ROOT/nix/var/nix/$d/per-user
-  chmod 1777 $RPM_BUILD_ROOT/nix/var/nix/$d/per-user
+  chmod 755 $RPM_BUILD_ROOT/nix/var/nix/$d/per-user
 done
 
 # fix permission of nix profile

perl/Makefile

@@ -4,11 +4,4 @@ GLOBAL_CXXFLAGS += -g -Wall
 
 -include Makefile.config
 
-OPTIMIZE = 1
-
-ifeq ($(OPTIMIZE), 1)
-  GLOBAL_CFLAGS += -O3
-  GLOBAL_CXXFLAGS += -O3
-endif
-
 include mk/lib.mk

perl/configure.ac

@@ -1,9 +1,11 @@
-AC_INIT(nix-perl, m4_esyscmd([bash -c "echo -n $(cat ../version)$VERSION_SUFFIX"]))
+AC_INIT(nix-perl, m4_esyscmd([bash -c "echo -n $(cat ../.version)$VERSION_SUFFIX"]))
 AC_CONFIG_SRCDIR(MANIFEST)
 AC_CONFIG_AUX_DIR(../config)
 
-CFLAGS=
-CXXFLAGS=
+# Set default flags for nix (as per AC_PROG_CC/CXX docs),
+# while still allowing the user to override them from the command line.
+: ${CFLAGS="-O3"}
+: ${CXXFLAGS="-O3"}
 AC_PROG_CC
 AC_PROG_CXX
 AX_CXX_COMPILE_STDCXX_11

release-common.nix

@@ -42,7 +42,7 @@ rec {
       libxml2
       libxslt
       docbook5
-      docbook5_xsl
+      docbook_xsl_ns
       autoconf-archive
       autoreconfHook
     ];
@@ -50,7 +50,7 @@ rec {
   buildDeps =
     [ curl
       bzip2 xz brotli editline
-      openssl pkgconfig sqlite boehmgc
+      openssl pkgconfig sqlite
       boost
 
       # Tests
@@ -72,6 +72,10 @@ rec {
         */
       }));
 
+  propagatedDeps =
+    [ (boehmgc.override { enableLargeConfig = true; })
+    ];
+
   perlDeps =
     [ perl
       perlPackages.DBDSQLite

release.nix

@@ -1,5 +1,5 @@
 { nix ? builtins.fetchGit ./.
-, nixpkgs ? builtins.fetchGit { url = https://github.com/NixOS/nixpkgs-channels.git; ref = "nixos-18.09"; }
+, nixpkgs ? builtins.fetchTarball https://github.com/NixOS/nixpkgs-channels/archive/nixos-19.03.tar.gz
 , officialRelease ? false
 , systems ? [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ]
 }:
@@ -18,12 +18,12 @@ let
 
       releaseTools.sourceTarball {
         name = "nix-tarball";
-        version = builtins.readFile ./version;
+        version = builtins.readFile ./.version;
         versionSuffix = if officialRelease then "" else "pre${toString nix.revCount}_${nix.shortRev}";
         src = nix;
         inherit officialRelease;
 
-        buildInputs = tarballDeps ++ buildDeps;
+        buildInputs = tarballDeps ++ buildDeps ++ propagatedDeps;
 
         configureFlags = "--enable-gc";
 
@@ -67,12 +67,19 @@ let
 
         buildInputs = buildDeps;
 
+        propagatedBuildInputs = propagatedDeps;
+
         preConfigure =
           # Copy libboost_context so we don't get all of Boost in our closure.
           # https://github.com/NixOS/nixpkgs/issues/45462
           ''
             mkdir -p $out/lib
-            cp ${boost}/lib/libboost_context* $out/lib
+            cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib
+            rm -f $out/lib/*.a
+            ${lib.optionalString stdenv.isLinux ''
+              chmod u+w $out/lib/*.so.*
+              patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.*
+            ''}
           '';
 
         configureFlags = configureFlags ++
@@ -86,6 +93,8 @@ let
 
         doInstallCheck = true;
         installCheckFlags = "sysconfdir=$(out)/etc";
+
+        separateDebugInfo = true;
       });
 
 
@@ -123,7 +132,7 @@ let
       in
 
       runCommand "nix-binary-tarball-${version}"
-        { nativeBuildInputs = lib.optional (system != "aarch64-linux") shellcheck;
+        { #nativeBuildInputs = lib.optional (system != "aarch64-linux") shellcheck;
           meta.description = "Distribution-independent Nix bootstrap binaries for ${system}";
         }
         ''
@@ -165,10 +174,10 @@ let
           chmod +x $TMPDIR/install-systemd-multi-user.sh
           chmod +x $TMPDIR/install-multi-user
           dir=nix-${version}-${system}
-          fn=$out/$dir.tar.bz2
+          fn=$out/$dir.tar.xz
           mkdir -p $out/nix-support
           echo "file binary-dist $fn" >> $out/nix-support/hydra-build-products
-          tar cvfj $fn \
+          tar cvfJ $fn \
             --owner=0 --group=0 --mode=u+rw,uga+r \
             --absolute-names \
             --hard-dereference \
@@ -191,7 +200,9 @@ let
         name = "nix-build";
         src = tarball;
 
-        buildInputs = buildDeps;
+        enableParallelBuilding = true;
+
+        buildInputs = buildDeps ++ propagatedDeps;
 
         dontInstall = false;
 
@@ -266,6 +277,7 @@ let
           umount /nix
         ''); # */
 
+    /*
     tests.evalNixpkgs =
       import (nixpkgs + "/pkgs/top-level/make-tarball.nix") {
         inherit nixpkgs;
@@ -278,13 +290,13 @@ let
       pkgs.runCommand "eval-nixos" { buildInputs = [ build.x86_64-linux ]; }
         ''
           export NIX_STATE_DIR=$TMPDIR
-          nix-store --init
 
           nix-instantiate ${nixpkgs}/nixos/release-combined.nix -A tested --dry-run \
             --arg nixpkgs '{ outPath = ${nixpkgs}; revCount = 123; shortRev = "abcdefgh"; }'
 
           touch $out
         '';
+    */
 
 
     installerScript =
@@ -296,7 +308,7 @@ let
 
           substitute ${./scripts/install.in} $out/install \
             ${pkgs.lib.concatMapStrings
-              (system: "--replace '@binaryTarball_${system}@' $(nix hash-file --type sha256 ${binaryTarball.${system}}/*.tar.bz2) ")
+              (system: "--replace '@binaryTarball_${system}@' $(nix hash-file --base16 --type sha256 ${binaryTarball.${system}}/*.tar.xz) ")
               [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ]
             } \
             --replace '@nixVersion@' ${build.x86_64-linux.src.version}
@@ -322,8 +334,8 @@ let
           tests.remoteBuilds
           tests.nix-copy-closure
           tests.binaryTarball
-          tests.evalNixpkgs
-          tests.evalNixOS
+          #tests.evalNixpkgs
+          #tests.evalNixOS
           installerScript
         ];
     };

scripts/install-darwin-multi-user.sh

@@ -39,7 +39,7 @@ EOF
 
 poly_configure_nix_daemon_service() {
     _sudo "to set up the nix-daemon as a LaunchDaemon" \
-          ln -sfn "/nix/var/nix/profiles/default$PLIST_DEST" "$PLIST_DEST"
+          cp -f "/nix/var/nix/profiles/default$PLIST_DEST" "$PLIST_DEST"
 
     _sudo "to load the LaunchDaemon plist for nix-daemon" \
           launchctl load /Library/LaunchDaemons/org.nixos.nix-daemon.plist

scripts/install-multi-user.sh

@@ -13,15 +13,12 @@ set -o pipefail
 # however tracking which bits came from which would be impossible.
 
 readonly ESC='\033[0m'
-readonly BOLD='\033[38;1m'
-readonly BLUE='\033[38;34m'
-readonly BLUE_UL='\033[38;4;34m'
-readonly GREEN='\033[38;32m'
-readonly GREEN_UL='\033[38;4;32m'
-readonly RED='\033[38;31m'
-readonly RED_UL='\033[38;4;31m'
-readonly YELLOW='\033[38;33m'
-readonly YELLOW_UL='\033[38;4;33m'
+readonly BOLD='\033[1m'
+readonly BLUE='\033[34m'
+readonly BLUE_UL='\033[4;34m'
+readonly GREEN='\033[32m'
+readonly GREEN_UL='\033[4;32m'
+readonly RED='\033[31m'
 
 readonly NIX_USER_COUNT="32"
 readonly NIX_BUILD_GROUP_ID="30000"
@@ -240,10 +237,16 @@ EOF
 }
 trap finish_fail EXIT
 
+channel_update_failed=0
 function finish_success {
     finish_cleanup
 
     ok "Alright! We're done!"
+    if [ "x$channel_update_failed" = x1 ]; then
+        echo ""
+        echo "But fetching the nixpkgs channel failed. (Are you offline?)"
+        echo "To try again later, run \"sudo -i nix-channel --update nixpkgs\"."
+    fi
     cat <<EOF
 
 Before Nix will work in your existing shells, you'll need to close
@@ -324,7 +327,7 @@ EOF
         fi
     done
 
-    if [ -d /nix ]; then
+    if [ -d /nix/store ] || [ -d /nix/var ]; then
         failure <<EOF
 There are some relics of a previous installation of Nix at /nix, and
 this scripts assumes Nix is _not_ yet installed. Please delete the old
@@ -523,24 +526,17 @@ create_build_users() {
 }
 
 create_directories() {
+    # FIXME: remove all of this because it duplicates LocalStore::LocalStore().
+
     _sudo "to make the basic directory structure of Nix (part 1)" \
-          mkdir -pv -m 0755 /nix /nix/var /nix/var/log /nix/var/log/nix /nix/var/log/nix/drvs /nix/var/nix{,/db,/gcroots,/profiles,/temproots,/userpool}
+          mkdir -pv -m 0755 /nix /nix/var /nix/var/log /nix/var/log/nix /nix/var/log/nix/drvs /nix/var/nix{,/db,/gcroots,/profiles,/temproots,/userpool} /nix/var/nix/{gcroots,profiles}/per-user
 
     _sudo "to make the basic directory structure of Nix (part 2)" \
-          mkdir -pv -m 1777 /nix/var/nix/{gcroots,profiles}/per-user
-
-    _sudo "to make the basic directory structure of Nix (part 3)" \
           mkdir -pv -m 1775 /nix/store
 
-    _sudo "to make the basic directory structure of Nix (part 4)" \
+    _sudo "to make the basic directory structure of Nix (part 3)" \
           chgrp "$NIX_BUILD_GROUP_NAME" /nix/store
 
-    _sudo "to set up the root user's profile (part 1)" \
-          mkdir -pv -m 0755 /nix/var/nix/profiles/per-user/root
-
-    _sudo "to set up the root user's profile (part 2)" \
-          mkdir -pv -m 0700 "$ROOT_HOME/.nix-defexpr"
-
     _sudo "to place the default nix daemon configuration (part 1)" \
           mkdir -pv -m 0555 /etc/nix
 }
@@ -583,7 +579,7 @@ EOF
 We will:
 
  - make sure your computer doesn't already have Nix files
-   (if it does, I  will tell you how to clean them up.)
+   (if it does, I will tell you how to clean them up.)
  - create local users (see the list above for the users we'll make)
  - create a local group ($NIX_BUILD_GROUP_NAME)
  - install Nix in to $NIX_ROOT
@@ -674,9 +670,6 @@ $NIX_INSTALLED_NIX.
 EOF
         fi
 
-        _sudo "to initialize the Nix Database" \
-              $NIX_INSTALLED_NIX/bin/nix-store --init
-
         cat ./.reginfo \
             | _sudo "to load data for the first time in to the Nix Database" \
                    "$NIX_INSTALLED_NIX/bin/nix-store" --load-db
@@ -737,17 +730,15 @@ setup_default_profile() {
     # otherwise it will be lost in environments where sudo doesn't pass
     # all the environment variables by default.
     _sudo "to update the default channel in the default profile" \
-          HOME="$ROOT_HOME" NIX_SSL_CERT_FILE="$NIX_SSL_CERT_FILE" "$NIX_INSTALLED_NIX/bin/nix-channel" --update nixpkgs
+          HOME="$ROOT_HOME" NIX_SSL_CERT_FILE="$NIX_SSL_CERT_FILE" "$NIX_INSTALLED_NIX/bin/nix-channel" --update nixpkgs \
+          || channel_update_failed=1
+
 }
 
 
 place_nix_configuration() {
     cat <<EOF > "$SCRATCH/nix.conf"
 build-users-group = $NIX_BUILD_GROUP_NAME
-
-max-jobs = $NIX_USER_COUNT
-cores = 1
-sandbox = false
 EOF
     _sudo "to place the default nix daemon configuration (part 2)" \
           install -m 0664 "$SCRATCH/nix.conf" /etc/nix/nix.conf
@@ -757,9 +748,13 @@ main() {
     if [ "$(uname -s)" = "Darwin" ]; then
         # shellcheck source=./install-darwin-multi-user.sh
         . "$EXTRACTED_NIX_PATH/install-darwin-multi-user.sh"
-    elif [ "$(uname -s)" = "Linux" ] && [ -e /run/systemd/system ]; then
-        # shellcheck source=./install-systemd-multi-user.sh
-        . "$EXTRACTED_NIX_PATH/install-systemd-multi-user.sh"
+    elif [ "$(uname -s)" = "Linux" ]; then
+        if [ -e /run/systemd/system ]; then
+            # shellcheck source=./install-systemd-multi-user.sh
+            . "$EXTRACTED_NIX_PATH/install-systemd-multi-user.sh"
+        else
+            failure "Sorry, the multi-user installation requires systemd on Linux (detected using /run/systemd/system)"
+        fi
     else
         failure "Sorry, I don't know what to do on $(uname)"
     fi

scripts/install-nix-from-closure.sh

@@ -12,7 +12,7 @@ if ! [ -e "$self/.reginfo" ]; then
     echo "$0: incomplete installer (.reginfo is missing)" >&2
 fi
 
-if [ -z "$USER" ]; then
+if [ -z "$USER" ] && ! USER=$(id -u -n); then
     echo "$0: \$USER is not set" >&2
     exit 1
 fi
@@ -22,10 +22,12 @@ if [ -z "$HOME" ]; then
     exit 1
 fi
 
-# macOS support for 10.10 or higher
+# macOS support for 10.12.6 or higher
 if [ "$(uname -s)" = "Darwin" ]; then
-    if [ $(($(sw_vers -productVersion | cut -d '.' -f 2))) -lt 10 ]; then
-        echo "$0: macOS $(sw_vers -productVersion) is not supported, upgrade to 10.10 or higher"
+    macos_major=$(sw_vers -productVersion | cut -d '.' -f 2)
+    macos_minor=$(sw_vers -productVersion | cut -d '.' -f 3)
+    if [ "$macos_major" -lt 12 ] || { [ "$macos_major" -eq 12 ] && [ "$macos_minor" -lt 6 ]; }; then
+        echo "$0: macOS $(sw_vers -productVersion) is not supported, upgrade to 10.12.6 or higher"
         exit 1
     fi
 fi
@@ -109,12 +111,6 @@ for i in $(cd "$self/store" >/dev/null && echo ./*); do
 done
 echo "" >&2
 
-echo "initialising Nix database..." >&2
-if ! $nix/bin/nix-store --init; then
-    echo "$0: failed to initialize the Nix database" >&2
-    exit 1
-fi
-
 if ! "$nix/bin/nix-store" --load-db < "$self/.reginfo"; then
     echo "$0: unable to register valid paths" >&2
     exit 1
@@ -138,15 +134,16 @@ if ! $nix/bin/nix-channel --list | grep -q "^nixpkgs "; then
     $nix/bin/nix-channel --add https://nixos.org/channels/nixpkgs-unstable
 fi
 if [ -z "$_NIX_INSTALLER_TEST" ]; then
-    $nix/bin/nix-channel --update nixpkgs
+    if ! $nix/bin/nix-channel --update nixpkgs; then
+        echo "Fetching the nixpkgs channel failed. (Are you offline?)"
+        echo "To try again later, run \"nix-channel --update nixpkgs\"."
+    fi
 fi
 
 added=
+p=$HOME/.nix-profile/etc/profile.d/nix.sh
 if [ -z "$NIX_INSTALLER_NO_MODIFY_PROFILE" ]; then
-
     # Make the shell source nix.sh during login.
-    p=$HOME/.nix-profile/etc/profile.d/nix.sh
-
     for i in .bash_profile .bash_login .profile; do
         fn="$HOME/$i"
         if [ -w "$fn" ]; then
@@ -158,7 +155,6 @@ if [ -z "$NIX_INSTALLER_NO_MODIFY_PROFILE" ]; then
             break
         fi
     done
-
 fi
 
 if [ -z "$added" ]; then

scripts/install-systemd-multi-user.sh

@@ -9,6 +9,38 @@ readonly SERVICE_DEST=/etc/systemd/system/nix-daemon.service
 readonly SOCKET_SRC=/lib/systemd/system/nix-daemon.socket
 readonly SOCKET_DEST=/etc/systemd/system/nix-daemon.socket
 
+
+# Path for the systemd override unit file to contain the proxy settings
+readonly SERVICE_OVERRIDE=${SERVICE_DEST}.d/override.conf
+
+create_systemd_override() {
+     header "Configuring proxy for the nix-daemon service"
+    _sudo "create directory for systemd unit override" mkdir -p "$(dirname $SERVICE_OVERRIDE)"
+    cat <<EOF | _sudo "create systemd unit override" tee "$SERVICE_OVERRIDE"
+[Service]
+$1
+EOF
+}
+
+# Gather all non-empty proxy environment variables into a string
+create_systemd_proxy_env() {
+    vars="http_proxy https_proxy ftp_proxy no_proxy HTTP_PROXY HTTPS_PROXY FTP_PROXY NO_PROXY"
+    for v in $vars; do
+        if [ "x${!v:-}" != "x" ]; then
+            echo "Environment=${v}=${!v}"
+        fi
+    done
+}
+
+handle_network_proxy() {
+    # Create a systemd unit override with proxy environment variables
+    # if any proxy environment variables are not empty.
+    PROXY_ENV_STRING=$(create_systemd_proxy_env)
+    if [ -n "${PROXY_ENV_STRING}" ]; then
+        create_systemd_override "${PROXY_ENV_STRING}"
+    fi
+}
+
 poly_validate_assumptions() {
     if [ "$(uname -s)" != "Linux" ]; then
         failure "This script is for use with Linux!"
@@ -47,6 +79,8 @@ poly_configure_nix_daemon_service() {
     _sudo "to set up the nix-daemon socket service" \
           systemctl enable "/nix/var/nix/profiles/default$SOCKET_SRC"
 
+    handle_network_proxy
+
     _sudo "to load the systemd unit for nix-daemon" \
           systemctl daemon-reload
 

scripts/install.in

@@ -18,7 +18,7 @@ cleanup() {
 trap cleanup EXIT INT QUIT TERM
 
 require_util() {
-    type "$1" > /dev/null 2>&1 || command -v "$1" > /dev/null 2>&1 ||
+    command -v "$1" > /dev/null 2>&1 ||
         oops "you do not have '$1' installed, which I need to $2"
 }
 
@@ -30,22 +30,21 @@ case "$(uname -s).$(uname -m)" in
     *) oops "sorry, there is no binary distribution of Nix for your platform";;
 esac
 
-url="https://nixos.org/releases/nix/nix-@nixVersion@/nix-@nixVersion@-$system.tar.bz2"
+url="https://nixos.org/releases/nix/nix-@nixVersion@/nix-@nixVersion@-$system.tar.xz"
 
-tarball="$tmpDir/$(basename "$tmpDir/nix-@nixVersion@-$system.tar.bz2")"
+tarball="$tmpDir/$(basename "$tmpDir/nix-@nixVersion@-$system.tar.xz")"
 
 require_util curl "download the binary tarball"
-require_util bzcat "decompress the binary tarball"
 require_util tar "unpack the binary tarball"
 
 echo "downloading Nix @nixVersion@ binary tarball for $system from '$url' to '$tmpDir'..."
 curl -L "$url" -o "$tarball" || oops "failed to download '$url'"
 
-if type sha256sum > /dev/null 2>&1; then
+if command -v sha256sum > /dev/null 2>&1; then
     hash2="$(sha256sum -b "$tarball" | cut -c1-64)"
-elif type shasum > /dev/null 2>&1; then
+elif command -v shasum > /dev/null 2>&1; then
     hash2="$(shasum -a 256 -b "$tarball" | cut -c1-64)"
-elif type openssl > /dev/null 2>&1; then
+elif command -v openssl > /dev/null 2>&1; then
     hash2="$(openssl dgst -r -sha256 "$tarball" | cut -c1-64)"
 else
     oops "cannot verify the SHA-256 hash of '$url'; you need one of 'shasum', 'sha256sum', or 'openssl'"
@@ -57,7 +56,7 @@ fi
 
 unpack=$tmpDir/unpack
 mkdir -p "$unpack"
-< "$tarball" bzcat | tar -xf - -C "$unpack" || oops "failed to unpack '$url'"
+tar -xf "$tarball" -C "$unpack" || oops "failed to unpack '$url'"
 
 script=$(echo "$unpack"/*/install)
 

scripts/nix-profile-daemon.sh.in

@@ -2,54 +2,9 @@
 if [ -n "${__ETC_PROFILE_NIX_SOURCED:-}" ]; then return; fi
 __ETC_PROFILE_NIX_SOURCED=1
 
-# Set up secure multi-user builds: non-root users build through the
-# Nix daemon.
-if [ "$USER" != root -o ! -w @localstatedir@/nix/db ]; then
-    export NIX_REMOTE=daemon
-fi
-
 export NIX_USER_PROFILE_DIR="@localstatedir@/nix/profiles/per-user/$USER"
 export NIX_PROFILES="@localstatedir@/nix/profiles/default $HOME/.nix-profile"
 
-# Set up the per-user profile.
-mkdir -m 0755 -p $NIX_USER_PROFILE_DIR
-if ! test -O "$NIX_USER_PROFILE_DIR"; then
-    echo "WARNING: bad ownership on $NIX_USER_PROFILE_DIR" >&2
-fi
-
-if test -w $HOME; then
-  if ! test -L $HOME/.nix-profile; then
-      if test "$USER" != root; then
-          ln -s $NIX_USER_PROFILE_DIR/profile $HOME/.nix-profile
-      else
-          # Root installs in the system-wide profile by default.
-          ln -s @localstatedir@/nix/profiles/default $HOME/.nix-profile
-      fi
-  fi
-
-  # Subscribe the root user to the NixOS channel by default.
-  if [ "$USER" = root -a ! -e $HOME/.nix-channels ]; then
-      echo "https://nixos.org/channels/nixpkgs-unstable nixpkgs" > $HOME/.nix-channels
-  fi
-
-  # Create the per-user garbage collector roots directory.
-  NIX_USER_GCROOTS_DIR=@localstatedir@/nix/gcroots/per-user/$USER
-  mkdir -m 0755 -p $NIX_USER_GCROOTS_DIR
-  if ! test -O "$NIX_USER_GCROOTS_DIR"; then
-      echo "WARNING: bad ownership on $NIX_USER_GCROOTS_DIR" >&2
-  fi
-
-  # Set up a default Nix expression from which to install stuff.
-  if [ ! -e $HOME/.nix-defexpr -o -L $HOME/.nix-defexpr ]; then
-      rm -f $HOME/.nix-defexpr
-      mkdir -p $HOME/.nix-defexpr
-      if [ "$USER" != root ]; then
-          ln -s @localstatedir@/nix/profiles/per-user/root/channels $HOME/.nix-defexpr/channels_root
-      fi
-  fi
-fi
-
-
 # Set $NIX_SSL_CERT_FILE so that Nixpkgs applications like curl work.
 if [ ! -z "${NIX_SSL_CERT_FILE:-}" ]; then
     : # Allow users to override the NIX_SSL_CERT_FILE

scripts/nix-profile.sh.in

@@ -1,6 +1,4 @@
 if [ -n "$HOME" ] && [ -n "$USER" ]; then
-    __savedpath="$PATH"
-    export PATH=@coreutils@
 
     # Set up the per-user profile.
     # This part should be kept in sync with nixpkgs:nixos/modules/programs/shell.nix
@@ -9,56 +7,13 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then
 
     NIX_USER_PROFILE_DIR=@localstatedir@/nix/profiles/per-user/$USER
 
-    mkdir -m 0755 -p "$NIX_USER_PROFILE_DIR"
-
-    if [ "$(stat --printf '%u' "$NIX_USER_PROFILE_DIR")" != "$(id -u)" ]; then
-        echo "Nix: WARNING: bad ownership on "$NIX_USER_PROFILE_DIR", should be $(id -u)" >&2
-    fi
-
-    if [ -w "$HOME" ]; then
-        if ! [ -L "$NIX_LINK" ]; then
-            echo "Nix: creating $NIX_LINK" >&2
-            if [ "$USER" != root ]; then
-                if ! ln -s "$NIX_USER_PROFILE_DIR"/profile "$NIX_LINK"; then
-                    echo "Nix: WARNING: could not create $NIX_LINK -> $NIX_USER_PROFILE_DIR/profile" >&2
-                fi
-            else
-                # Root installs in the system-wide profile by default.
-                ln -s @localstatedir@/nix/profiles/default "$NIX_LINK"
-            fi
-        fi
-
-        # Subscribe the user to the unstable Nixpkgs channel by default.
-        if [ ! -e "$HOME/.nix-channels" ]; then
-            echo "https://nixos.org/channels/nixpkgs-unstable nixpkgs" > "$HOME/.nix-channels"
-        fi
-
-        # Create the per-user garbage collector roots directory.
-        __user_gcroots=@localstatedir@/nix/gcroots/per-user/"$USER"
-        mkdir -m 0755 -p "$__user_gcroots"
-        if [ "$(stat --printf '%u' "$__user_gcroots")" != "$(id -u)" ]; then
-            echo "Nix: WARNING: bad ownership on $__user_gcroots, should be $(id -u)" >&2
-        fi
-        unset __user_gcroots
-
-        # Set up a default Nix expression from which to install stuff.
-        __nix_defexpr="$HOME"/.nix-defexpr
-        [ -L "$__nix_defexpr" ] && rm -f "$__nix_defexpr"
-        mkdir -m 0755 -p "$__nix_defexpr"
-        if [ "$USER" != root ] && [ ! -L "$__nix_defexpr"/channels_root ]; then
-            ln -s @localstatedir@/nix/profiles/per-user/root/channels "$__nix_defexpr"/channels_root
-        fi
-        unset __nix_defexpr
-    fi
-
-    # Append ~/.nix-defexpr/channels/nixpkgs to $NIX_PATH so that
-    # <nixpkgs> paths work when the user has fetched the Nixpkgs
-    # channel.
-    export NIX_PATH="${NIX_PATH:+$NIX_PATH:}nixpkgs=$HOME/.nix-defexpr/channels/nixpkgs"
+    # Append ~/.nix-defexpr/channels to $NIX_PATH so that <nixpkgs>
+    # paths work when the user has fetched the Nixpkgs channel.
+    export NIX_PATH=${NIX_PATH:+$NIX_PATH:}$HOME/.nix-defexpr/channels
 
     # Set up environment.
     # This part should be kept in sync with nixpkgs:nixos/modules/programs/environment.nix
-    NIX_PROFILES="@localstatedir@/nix/profiles/default $NIX_USER_PROFILE_DIR"
+    export NIX_PROFILES="@localstatedir@/nix/profiles/default $HOME/.nix-profile"
 
     # Set $NIX_SSL_CERT_FILE so that Nixpkgs applications like curl work.
     if [ -e /etc/ssl/certs/ca-certificates.crt ]; then # NixOS, Ubuntu, Debian, Gentoo, Arch
@@ -75,10 +30,10 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then
         export NIX_SSL_CERT_FILE="$NIX_LINK/etc/ca-bundle.crt"
     fi
 
-    if [ -n "${MANPATH}" ]; then
+    if [ -n "${MANPATH-}" ]; then
         export MANPATH="$NIX_LINK/share/man:$MANPATH"
     fi
 
-    export PATH="$NIX_LINK/bin:$__savedpath"
-    unset __savedpath NIX_LINK NIX_USER_PROFILE_DIR NIX_PROFILES
+    export PATH="$NIX_LINK/bin:$PATH"
+    unset NIX_LINK NIX_USER_PROFILE_DIR
 fi

shell.nix

@@ -1,13 +1,13 @@
 { useClang ? false }:
 
-with import (builtins.fetchGit { url = https://github.com/NixOS/nixpkgs-channels.git; ref = "nixos-18.09"; }) {};
+with import (builtins.fetchTarball https://github.com/NixOS/nixpkgs-channels/archive/nixos-19.03.tar.gz) {};
 
 with import ./release-common.nix { inherit pkgs; };
 
 (if useClang then clangStdenv else stdenv).mkDerivation {
   name = "nix";
 
-  buildInputs = buildDeps ++ tarballDeps ++ perlDeps;
+  buildInputs = buildDeps ++ propagatedDeps ++ tarballDeps ++ perlDeps;
 
   inherit configureFlags;
 

src/build-remote/build-remote.cc

@@ -38,6 +38,12 @@ static AutoCloseFD openSlotLock(const Machine & m, unsigned long long slot)
     return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri), slot), true);
 }
 
+static bool allSupportedLocally(const std::set<std::string>& requiredFeatures) {
+    for (auto & feature : requiredFeatures)
+        if (!settings.systemFeatures.get().count(feature)) return false;
+    return true;
+}
+
 static int _main(int argc, char * * argv)
 {
     {
@@ -97,9 +103,10 @@ static int _main(int argc, char * * argv)
             source >> drvPath;
             auto requiredFeatures = readStrings<std::set<std::string>>(source);
 
-            auto canBuildLocally = amWilling
-                &&  (  neededSystem == settings.thisSystem
-                    || settings.extraPlatforms.get().count(neededSystem) > 0);
+             auto canBuildLocally = amWilling
+                 &&  (  neededSystem == settings.thisSystem
+                     || settings.extraPlatforms.get().count(neededSystem) > 0)
+                 &&  allSupportedLocally(requiredFeatures);
 
             /* Error ignored here, will be caught later */
             mkdir(currentLoad.c_str(), 0777);

src/cpptoml/cpptoml.h

@@ -4,8 +4,8 @@
  * @date May 2013
  */
 
-#ifndef _CPPTOML_H_
-#define _CPPTOML_H_
+#ifndef CPPTOML_H
+#define CPPTOML_H
 
 #include <algorithm>
 #include <cassert>
@@ -84,11 +84,12 @@ class option
         return &value_;
     }
 
-    const T& value_or(const T& alternative) const
+    template <class U>
+    T value_or(U&& alternative) const
     {
         if (!empty_)
             return value_;
-        return alternative;
+        return static_cast<T>(std::forward<U>(alternative));
     }
 
   private:
@@ -295,13 +296,12 @@ struct valid_value_or_string_convertible
 };
 
 template <class T>
-struct value_traits<T, typename std::
-                           enable_if<valid_value_or_string_convertible<T>::
-                                         value>::type>
+struct value_traits<T, typename std::enable_if<
+                           valid_value_or_string_convertible<T>::value>::type>
 {
-    using value_type = typename std::
-        conditional<valid_value<typename std::decay<T>::type>::value,
-                    typename std::decay<T>::type, std::string>::type;
+    using value_type = typename std::conditional<
+        valid_value<typename std::decay<T>::type>::value,
+        typename std::decay<T>::type, std::string>::type;
 
     using type = value<value_type>;
 
@@ -312,12 +312,11 @@ struct value_traits<T, typename std::
 };
 
 template <class T>
-struct value_traits<T,
-                    typename std::
-                        enable_if<!valid_value_or_string_convertible<T>::value
-                                  && std::is_floating_point<
-                                         typename std::decay<T>::type>::value>::
-                            type>
+struct value_traits<
+    T,
+    typename std::enable_if<
+        !valid_value_or_string_convertible<T>::value
+        && std::is_floating_point<typename std::decay<T>::type>::value>::type>
 {
     using value_type = typename std::decay<T>::type;
 
@@ -330,11 +329,11 @@ struct value_traits<T,
 };
 
 template <class T>
-struct value_traits<T,
-                    typename std::
-                        enable_if<!valid_value_or_string_convertible<T>::value
-                                  && std::is_signed<typename std::decay<T>::
-                                                        type>::value>::type>
+struct value_traits<
+    T, typename std::enable_if<
+           !valid_value_or_string_convertible<T>::value
+           && !std::is_floating_point<typename std::decay<T>::type>::value
+           && std::is_signed<typename std::decay<T>::type>::value>::type>
 {
     using value_type = int64_t;
 
@@ -356,11 +355,10 @@ struct value_traits<T,
 };
 
 template <class T>
-struct value_traits<T,
-                    typename std::
-                        enable_if<!valid_value_or_string_convertible<T>::value
-                                  && std::is_unsigned<typename std::decay<T>::
-                                                          type>::value>::type>
+struct value_traits<
+    T, typename std::enable_if<
+           !valid_value_or_string_convertible<T>::value
+           && std::is_unsigned<typename std::decay<T>::type>::value>::type>
 {
     using value_type = int64_t;
 
@@ -395,10 +393,15 @@ struct array_of_trait<array>
 template <class T>
 inline std::shared_ptr<typename value_traits<T>::type> make_value(T&& val);
 inline std::shared_ptr<array> make_array();
+
+namespace detail
+{
 template <class T>
 inline std::shared_ptr<T> make_element();
+}
+
 inline std::shared_ptr<table> make_table();
-inline std::shared_ptr<table_array> make_table_array();
+inline std::shared_ptr<table_array> make_table_array(bool is_inline = false);
 
 #if defined(CPPTOML_NO_RTTI)
 /// Base type used to store underlying data type explicitly if RTTI is disabled
@@ -576,7 +579,7 @@ class base : public std::enable_shared_from_this<base>
 #if defined(CPPTOML_NO_RTTI)
     base_type type() const
     {
-      return type_;
+        return type_;
     }
 
   protected:
@@ -698,7 +701,7 @@ inline std::shared_ptr<value<double>> base::as()
     if (type() == base_type::INT)
     {
         auto v = std::static_pointer_cast<value<int64_t>>(shared_from_this());
-        return make_value<double>(static_cast<double>(v->get()));;
+        return make_value<double>(static_cast<double>(v->get()));
     }
 #else
     if (auto v = std::dynamic_pointer_cast<value<double>>(shared_from_this()))
@@ -731,7 +734,8 @@ inline std::shared_ptr<const value<double>> base::as() const
 {
 #if defined(CPPTOML_NO_RTTI)
     if (type() == base_type::FLOAT)
-        return std::static_pointer_cast<const value<double>>(shared_from_this());
+        return std::static_pointer_cast<const value<double>>(
+            shared_from_this());
 
     if (type() == base_type::INT)
     {
@@ -1027,11 +1031,14 @@ inline std::shared_ptr<array> make_array()
     return std::make_shared<make_shared_enabler>();
 }
 
+namespace detail
+{
 template <>
 inline std::shared_ptr<array> make_element<array>()
 {
     return make_array();
 }
+} // namespace detail
 
 /**
  * Obtains a option<vector<T>>. The option will be empty if the array
@@ -1060,7 +1067,7 @@ class table;
 class table_array : public base
 {
     friend class table;
-    friend std::shared_ptr<table_array> make_table_array();
+    friend std::shared_ptr<table_array> make_table_array(bool);
 
   public:
     std::shared_ptr<base> clone() const override;
@@ -1152,14 +1159,25 @@ class table_array : public base
         array_.reserve(n);
     }
 
+    /**
+     * Whether or not the table array is declared inline. This mostly
+     * matters for parsing, where statically defined arrays cannot be
+     * appended to using the array-of-table syntax.
+     */
+    bool is_inline() const
+    {
+        return is_inline_;
+    }
+
   private:
 #if defined(CPPTOML_NO_RTTI)
-    table_array() : base(base_type::TABLE_ARRAY)
+    table_array(bool is_inline = false)
+        : base(base_type::TABLE_ARRAY), is_inline_(is_inline)
     {
         // nothing
     }
 #else
-    table_array()
+    table_array(bool is_inline = false) : is_inline_(is_inline)
     {
         // nothing
     }
@@ -1169,26 +1187,30 @@ class table_array : public base
     table_array& operator=(const table_array& rhs) = delete;
 
     std::vector<std::shared_ptr<table>> array_;
+    const bool is_inline_ = false;
 };
 
-inline std::shared_ptr<table_array> make_table_array()
+inline std::shared_ptr<table_array> make_table_array(bool is_inline)
 {
     struct make_shared_enabler : public table_array
     {
-        make_shared_enabler()
+        make_shared_enabler(bool mse_is_inline) : table_array(mse_is_inline)
         {
             // nothing
         }
     };
 
-    return std::make_shared<make_shared_enabler>();
+    return std::make_shared<make_shared_enabler>(is_inline);
 }
 
+namespace detail
+{
 template <>
 inline std::shared_ptr<table_array> make_element<table_array>()
 {
-    return make_table_array();
+    return make_table_array(true);
 }
+} // namespace detail
 
 // The below are overloads for fetching specific value types out of a value
 // where special casting behavior (like bounds checking) is desired
@@ -1679,11 +1701,14 @@ std::shared_ptr<table> make_table()
     return std::make_shared<make_shared_enabler>();
 }
 
+namespace detail
+{
 template <>
 inline std::shared_ptr<table> make_element<table>()
 {
     return make_table();
 }
+} // namespace detail
 
 template <class T>
 std::shared_ptr<base> value<T>::clone() const
@@ -1702,7 +1727,7 @@ inline std::shared_ptr<base> array::clone() const
 
 inline std::shared_ptr<base> table_array::clone() const
 {
-    auto result = make_table_array();
+    auto result = make_table_array(is_inline());
     result->reserve(array_.size());
     for (const auto& ptr : array_)
         result->array_.push_back(ptr->clone()->as_table());
@@ -1738,6 +1763,11 @@ inline bool is_number(char c)
     return c >= '0' && c <= '9';
 }
 
+inline bool is_hex(char c)
+{
+    return is_number(c) || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F');
+}
+
 /**
  * Helper object for consuming expected characters.
  */
@@ -1766,6 +1796,13 @@ class consumer
                       [&](char c) { (*this)(c); });
     }
 
+    void eat_or(char a, char b)
+    {
+        if (it_ == end_ || (*it_ != a && *it_ != b))
+            on_error_();
+        ++it_;
+    }
+
     int eat_digits(int len)
     {
         int val = 0;
@@ -1830,7 +1867,7 @@ inline std::istream& getline(std::istream& input, std::string& line)
         line.push_back(static_cast<char>(c));
     }
 }
-}
+} // namespace detail
 
 /**
  * The parser class.
@@ -1914,21 +1951,25 @@ class parser
 
         std::string full_table_name;
         bool inserted = false;
-        while (it != end && *it != ']')
-        {
-            auto part = parse_key(it, end,
-                                  [](char c) { return c == '.' || c == ']'; });
 
+        auto key_end = [](char c) { return c == ']'; };
+
+        auto key_part_handler = [&](const std::string& part) {
             if (part.empty())
                 throw_parse_exception("Empty component of table name");
 
             if (!full_table_name.empty())
-                full_table_name += ".";
+                full_table_name += '.';
             full_table_name += part;
 
             if (curr_table->contains(part))
             {
+#if !defined(__PGI)
                 auto b = curr_table->get(part);
+#else
+                // Workaround for PGI compiler
+                std::shared_ptr<base> b = curr_table->get(part);
+#endif
                 if (b->is_table())
                     curr_table = static_cast<table*>(b.get());
                 else if (b->is_table_array())
@@ -1946,16 +1987,23 @@ class parser
                 curr_table->insert(part, make_table());
                 curr_table = static_cast<table*>(curr_table->get(part).get());
             }
-            consume_whitespace(it, end);
-            if (it != end && *it == '.')
-                ++it;
-            consume_whitespace(it, end);
-        }
+        };
+
+        key_part_handler(parse_key(it, end, key_end, key_part_handler));
 
         if (it == end)
             throw_parse_exception(
                 "Unterminated table declaration; did you forget a ']'?");
 
+        if (*it != ']')
+        {
+            std::string errmsg{"Unexpected character in table definition: "};
+            errmsg += '"';
+            errmsg += *it;
+            errmsg += '"';
+            throw_parse_exception(errmsg);
+        }
+
         // table already existed
         if (!inserted)
         {
@@ -1969,8 +2017,9 @@ class parser
             // since it has already been defined. If there aren't any
             // values, then it was implicitly created by something like
             // [a.b]
-            if (curr_table->empty() || std::any_of(curr_table->begin(),
-                                                   curr_table->end(), is_value))
+            if (curr_table->empty()
+                || std::any_of(curr_table->begin(), curr_table->end(),
+                               is_value))
             {
                 throw_parse_exception("Redefinition of table "
                                       + full_table_name);
@@ -1989,36 +2038,45 @@ class parser
         if (it == end || *it == ']')
             throw_parse_exception("Table array name cannot be empty");
 
-        std::string full_ta_name;
-        while (it != end && *it != ']')
-        {
-            auto part = parse_key(it, end,
-                                  [](char c) { return c == '.' || c == ']'; });
+        auto key_end = [](char c) { return c == ']'; };
 
+        std::string full_ta_name;
+        auto key_part_handler = [&](const std::string& part) {
             if (part.empty())
                 throw_parse_exception("Empty component of table array name");
 
             if (!full_ta_name.empty())
-                full_ta_name += ".";
+                full_ta_name += '.';
             full_ta_name += part;
 
-            consume_whitespace(it, end);
-            if (it != end && *it == '.')
-                ++it;
-            consume_whitespace(it, end);
-
             if (curr_table->contains(part))
             {
+#if !defined(__PGI)
                 auto b = curr_table->get(part);
+#else
+                // Workaround for PGI compiler
+                std::shared_ptr<base> b = curr_table->get(part);
+#endif
 
                 // if this is the end of the table array name, add an
-                // element to the table array that we just looked up
+                // element to the table array that we just looked up,
+                // provided it was not declared inline
                 if (it != end && *it == ']')
                 {
                     if (!b->is_table_array())
+                    {
                         throw_parse_exception("Key " + full_ta_name
                                               + " is not a table array");
+                    }
+
                     auto v = b->as_table_array();
+
+                    if (v->is_inline())
+                    {
+                        throw_parse_exception("Static array " + full_ta_name
+                                              + " cannot be appended to");
+                    }
+
                     v->get().push_back(make_table());
                     curr_table = v->get().back().get();
                 }
@@ -2059,15 +2117,16 @@ class parser
                         = static_cast<table*>(curr_table->get(part).get());
                 }
             }
-        }
+        };
+
+        key_part_handler(parse_key(it, end, key_end, key_part_handler));
 
         // consume the last "]]"
-        if (it == end)
+        auto eat = make_consumer(it, end, [this]() {
             throw_parse_exception("Unterminated table array name");
-        ++it;
-        if (it == end)
-            throw_parse_exception("Unterminated table array name");
-        ++it;
+        });
+        eat(']');
+        eat(']');
 
         consume_whitespace(it, end);
         eol_or_comment(it, end);
@@ -2076,7 +2135,35 @@ class parser
     void parse_key_value(std::string::iterator& it, std::string::iterator& end,
                          table* curr_table)
     {
-        auto key = parse_key(it, end, [](char c) { return c == '='; });
+        auto key_end = [](char c) { return c == '='; };
+
+        auto key_part_handler = [&](const std::string& part) {
+            // two cases: this key part exists already, in which case it must
+            // be a table, or it doesn't exist in which case we must create
+            // an implicitly defined table
+            if (curr_table->contains(part))
+            {
+                auto val = curr_table->get(part);
+                if (val->is_table())
+                {
+                    curr_table = static_cast<table*>(val.get());
+                }
+                else
+                {
+                    throw_parse_exception("Key " + part
+                                          + " already exists as a value");
+                }
+            }
+            else
+            {
+                auto newtable = make_table();
+                curr_table->insert(part, newtable);
+                curr_table = newtable.get();
+            }
+        };
+
+        auto key = parse_key(it, end, key_end, key_part_handler);
+
         if (curr_table->contains(key))
             throw_parse_exception("Key " + key + " already present");
         if (it == end || *it != '=')
@@ -2087,18 +2174,57 @@ class parser
         consume_whitespace(it, end);
     }
 
-    template <class Function>
-    std::string parse_key(std::string::iterator& it,
-                          const std::string::iterator& end, Function&& fun)
+    template <class KeyEndFinder, class KeyPartHandler>
+    std::string
+    parse_key(std::string::iterator& it, const std::string::iterator& end,
+              KeyEndFinder&& key_end, KeyPartHandler&& key_part_handler)
+    {
+        // parse the key as a series of one or more simple-keys joined with '.'
+        while (it != end && !key_end(*it))
+        {
+            auto part = parse_simple_key(it, end);
+            consume_whitespace(it, end);
+
+            if (it == end || key_end(*it))
+            {
+                return part;
+            }
+
+            if (*it != '.')
+            {
+                std::string errmsg{"Unexpected character in key: "};
+                errmsg += '"';
+                errmsg += *it;
+                errmsg += '"';
+                throw_parse_exception(errmsg);
+            }
+
+            key_part_handler(part);
+
+            // consume the dot
+            ++it;
+        }
+
+        throw_parse_exception("Unexpected end of key");
+    }
+
+    std::string parse_simple_key(std::string::iterator& it,
+                                 const std::string::iterator& end)
     {
         consume_whitespace(it, end);
-        if (*it == '"')
+
+        if (it == end)
+            throw_parse_exception("Unexpected end of key (blank key?)");
+
+        if (*it == '"' || *it == '\'')
         {
-            return parse_quoted_key(it, end);
+            return string_literal(it, end, *it);
         }
         else
         {
-            auto bke = std::find_if(it, end, std::forward<Function>(fun));
+            auto bke = std::find_if(it, end, [](char c) {
+                return c == '.' || c == '=' || c == ']';
+            });
             return parse_bare_key(it, bke);
         }
     }
@@ -2142,12 +2268,6 @@ class parser
         return key;
     }
 
-    std::string parse_quoted_key(std::string::iterator& it,
-                                 const std::string::iterator& end)
-    {
-        return string_literal(it, end, '"');
-    }
-
     enum class parse_type
     {
         STRING = 1,
@@ -2193,7 +2313,7 @@ class parser
     parse_type determine_value_type(const std::string::iterator& it,
                                     const std::string::iterator& end)
     {
-        if(it == end)
+        if (it == end)
         {
             throw_parse_exception("Failed to parse value type");
         }
@@ -2209,7 +2329,11 @@ class parser
         {
             return *dtype;
         }
-        else if (is_number(*it) || *it == '-' || *it == '+')
+        else if (is_number(*it) || *it == '-' || *it == '+'
+                 || (*it == 'i' && it + 1 != end && it[1] == 'n'
+                     && it + 2 != end && it[2] == 'f')
+                 || (*it == 'n' && it + 1 != end && it[1] == 'a'
+                     && it + 2 != end && it[2] == 'n'))
         {
             return determine_number_type(it, end);
         }
@@ -2235,6 +2359,13 @@ class parser
         auto check_it = it;
         if (*check_it == '-' || *check_it == '+')
             ++check_it;
+
+        if (check_it == end)
+            throw_parse_exception("Malformed number");
+
+        if (*check_it == 'i' || *check_it == 'n')
+            return parse_type::FLOAT;
+
         while (check_it != end && is_number(*check_it))
             ++check_it;
         if (check_it != end && *check_it == '.')
@@ -2283,57 +2414,56 @@ class parser
         bool consuming = false;
         std::shared_ptr<value<std::string>> ret;
 
-        auto handle_line
-            = [&](std::string::iterator& local_it,
-                  std::string::iterator& local_end) {
-                  if (consuming)
-                  {
-                      local_it = std::find_if_not(local_it, local_end, is_ws);
+        auto handle_line = [&](std::string::iterator& local_it,
+                               std::string::iterator& local_end) {
+            if (consuming)
+            {
+                local_it = std::find_if_not(local_it, local_end, is_ws);
 
-                      // whole line is whitespace
-                      if (local_it == local_end)
-                          return;
-                  }
+                // whole line is whitespace
+                if (local_it == local_end)
+                    return;
+            }
 
-                  consuming = false;
+            consuming = false;
 
-                  while (local_it != local_end)
-                  {
-                      // handle escaped characters
-                      if (delim == '"' && *local_it == '\\')
-                      {
-                          auto check = local_it;
-                          // check if this is an actual escape sequence or a
-                          // whitespace escaping backslash
-                          ++check;
-                          consume_whitespace(check, local_end);
-                          if (check == local_end)
-                          {
-                              consuming = true;
-                              break;
-                          }
+            while (local_it != local_end)
+            {
+                // handle escaped characters
+                if (delim == '"' && *local_it == '\\')
+                {
+                    auto check = local_it;
+                    // check if this is an actual escape sequence or a
+                    // whitespace escaping backslash
+                    ++check;
+                    consume_whitespace(check, local_end);
+                    if (check == local_end)
+                    {
+                        consuming = true;
+                        break;
+                    }
 
-                          ss << parse_escape_code(local_it, local_end);
-                          continue;
-                      }
+                    ss << parse_escape_code(local_it, local_end);
+                    continue;
+                }
 
-                      // if we can end the string
-                      if (std::distance(local_it, local_end) >= 3)
-                      {
-                          auto check = local_it;
-                          // check for """
-                          if (*check++ == delim && *check++ == delim
-                              && *check++ == delim)
-                          {
-                              local_it = check;
-                              ret = make_value<std::string>(ss.str());
-                              break;
-                          }
-                      }
+                // if we can end the string
+                if (std::distance(local_it, local_end) >= 3)
+                {
+                    auto check = local_it;
+                    // check for """
+                    if (*check++ == delim && *check++ == delim
+                        && *check++ == delim)
+                    {
+                        local_it = check;
+                        ret = make_value<std::string>(ss.str());
+                        break;
+                    }
+                }
 
-                      ss << *local_it++;
-                  }
-              };
+                ss << *local_it++;
+            }
+        };
 
         // handle the remainder of the current line
         handle_line(it, end);
@@ -2514,17 +2644,13 @@ class parser
         return value;
     }
 
-    bool is_hex(char c)
-    {
-        return is_number(c) || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F');
-    }
-
     uint32_t hex_to_digit(char c)
     {
         if (is_number(c))
             return static_cast<uint32_t>(c - '0');
-        return 10 + static_cast<uint32_t>(
-                        c - ((c >= 'a' && c <= 'f') ? 'a' : 'A'));
+        return 10
+               + static_cast<uint32_t>(c
+                                       - ((c >= 'a' && c <= 'f') ? 'a' : 'A'));
     }
 
     std::shared_ptr<base> parse_number(std::string::iterator& it,
@@ -2538,25 +2664,6 @@ class parser
                 ++check_it;
         };
 
-        eat_sign();
-
-        auto eat_numbers = [&]() {
-            auto beg = check_it;
-            while (check_it != end && is_number(*check_it))
-            {
-                ++check_it;
-                if (check_it != end && *check_it == '_')
-                {
-                    ++check_it;
-                    if (check_it == end || !is_number(*check_it))
-                        throw_parse_exception("Malformed number");
-                }
-            }
-
-            if (check_it == beg)
-                throw_parse_exception("Malformed number");
-        };
-
         auto check_no_leading_zero = [&]() {
             if (check_it != end && *check_it == '0' && check_it + 1 != check_end
                 && check_it[1] != '.')
@@ -2565,7 +2672,80 @@ class parser
             }
         };
 
+        auto eat_digits = [&](bool (*check_char)(char)) {
+            auto beg = check_it;
+            while (check_it != end && check_char(*check_it))
+            {
+                ++check_it;
+                if (check_it != end && *check_it == '_')
+                {
+                    ++check_it;
+                    if (check_it == end || !check_char(*check_it))
+                        throw_parse_exception("Malformed number");
+                }
+            }
+
+            if (check_it == beg)
+                throw_parse_exception("Malformed number");
+        };
+
+        auto eat_hex = [&]() { eat_digits(&is_hex); };
+
+        auto eat_numbers = [&]() { eat_digits(&is_number); };
+
+        if (check_it != end && *check_it == '0' && check_it + 1 != check_end
+            && (check_it[1] == 'x' || check_it[1] == 'o' || check_it[1] == 'b'))
+        {
+            ++check_it;
+            char base = *check_it;
+            ++check_it;
+            if (base == 'x')
+            {
+                eat_hex();
+                return parse_int(it, check_it, 16);
+            }
+            else if (base == 'o')
+            {
+                auto start = check_it;
+                eat_numbers();
+                auto val = parse_int(start, check_it, 8, "0");
+                it = start;
+                return val;
+            }
+            else // if (base == 'b')
+            {
+                auto start = check_it;
+                eat_numbers();
+                auto val = parse_int(start, check_it, 2);
+                it = start;
+                return val;
+            }
+        }
+
+        eat_sign();
         check_no_leading_zero();
+
+        if (check_it != end && check_it + 1 != end && check_it + 2 != end)
+        {
+            if (check_it[0] == 'i' && check_it[1] == 'n' && check_it[2] == 'f')
+            {
+                auto val = std::numeric_limits<double>::infinity();
+                if (*it == '-')
+                    val = -val;
+                it = check_it + 3;
+                return make_value(val);
+            }
+            else if (check_it[0] == 'n' && check_it[1] == 'a'
+                     && check_it[2] == 'n')
+            {
+                auto val = std::numeric_limits<double>::quiet_NaN();
+                if (*it == '-')
+                    val = -val;
+                it = check_it + 3;
+                return make_value(val);
+            }
+        }
+
         eat_numbers();
 
         if (check_it != end
@@ -2604,14 +2784,17 @@ class parser
     }
 
     std::shared_ptr<value<int64_t>> parse_int(std::string::iterator& it,
-                                              const std::string::iterator& end)
+                                              const std::string::iterator& end,
+                                              int base = 10,
+                                              const char* prefix = "")
     {
         std::string v{it, end};
+        v = prefix + v;
         v.erase(std::remove(v.begin(), v.end(), '_'), v.end());
         it = end;
         try
         {
-            return make_value<int64_t>(std::stoll(v));
+            return make_value<int64_t>(std::stoll(v, nullptr, base));
         }
         catch (const std::invalid_argument& ex)
         {
@@ -2674,18 +2857,33 @@ class parser
     std::string::iterator find_end_of_number(std::string::iterator it,
                                              std::string::iterator end)
     {
-        return std::find_if(it, end, [](char c) {
+        auto ret = std::find_if(it, end, [](char c) {
             return !is_number(c) && c != '_' && c != '.' && c != 'e' && c != 'E'
-                   && c != '-' && c != '+';
+                   && c != '-' && c != '+' && c != 'x' && c != 'o' && c != 'b';
         });
+        if (ret != end && ret + 1 != end && ret + 2 != end)
+        {
+            if ((ret[0] == 'i' && ret[1] == 'n' && ret[2] == 'f')
+                || (ret[0] == 'n' && ret[1] == 'a' && ret[2] == 'n'))
+            {
+                ret = ret + 3;
+            }
+        }
+        return ret;
     }
 
     std::string::iterator find_end_of_date(std::string::iterator it,
                                            std::string::iterator end)
     {
-        return std::find_if(it, end, [](char c) {
-            return !is_number(c) && c != 'T' && c != 'Z' && c != ':' && c != '-'
-                   && c != '+' && c != '.';
+        auto end_of_date = std::find_if(it, end, [](char c) {
+            return !is_number(c) && c != '-';
+        });
+        if (end_of_date != end && *end_of_date == ' ' && end_of_date + 1 != end
+            && is_number(end_of_date[1]))
+            end_of_date++;
+        return std::find_if(end_of_date, end, [](char c) {
+            return !is_number(c) && c != 'T' && c != 'Z' && c != ':'
+                   && c != '-' && c != '+' && c != '.';
         });
     }
 
@@ -2754,7 +2952,7 @@ class parser
         if (it == date_end)
             return make_value(ldate);
 
-        eat('T');
+        eat.eat_or('T', ' ');
 
         local_datetime ldt;
         static_cast<local_date&>(ldt) = ldate;
@@ -2850,9 +3048,9 @@ class parser
         auto arr = make_array();
         while (it != end && *it != ']')
         {
-            auto value = parse_value(it, end);
-            if (auto v = value->as<Value>())
-                arr->get().push_back(value);
+            auto val = parse_value(it, end);
+            if (auto v = val->as<Value>())
+                arr->get().push_back(val);
             else
                 throw_parse_exception("Arrays must be homogeneous");
             skip_whitespace_and_comments(it, end);
@@ -2871,7 +3069,7 @@ class parser
                                                std::string::iterator& it,
                                                std::string::iterator& end)
     {
-        auto arr = make_element<Object>();
+        auto arr = detail::make_element<Object>();
 
         while (it != end && *it != ']')
         {
@@ -2881,7 +3079,7 @@ class parser
             arr->get().push_back(((*this).*fun)(it, end));
             skip_whitespace_and_comments(it, end);
 
-            if (*it != ',')
+            if (it == end || *it != ',')
                 break;
 
             ++it;
@@ -2906,8 +3104,11 @@ class parser
                 throw_parse_exception("Unterminated inline table");
 
             consume_whitespace(it, end);
-            parse_key_value(it, end, tbl.get());
-            consume_whitespace(it, end);
+            if (it != end && *it != '}')
+            {
+                parse_key_value(it, end, tbl.get());
+                consume_whitespace(it, end);
+            }
         } while (*it == ',');
 
         if (it == end || *it != '}')
@@ -2987,7 +3188,8 @@ class parser
         if (it[4] != '-' || it[7] != '-')
             return {};
 
-        if (len >= 19 && it[10] == 'T' && is_time(it + 11, date_end))
+        if (len >= 19 && (it[10] == 'T' || it[10] == ' ')
+            && is_time(it + 11, date_end))
         {
             // datetime type
             auto time_end = find_end_of_time(it + 11, date_end);
@@ -3243,7 +3445,7 @@ class toml_writer
             {
                 res += "\\\\";
             }
-            else if ((const uint32_t)*it <= 0x001f)
+            else if (static_cast<uint32_t>(*it) <= UINT32_C(0x001f))
             {
                 res += "\\u";
                 std::stringstream ss;
@@ -3274,12 +3476,21 @@ class toml_writer
      */
     void write(const value<double>& v)
     {
-        std::ios::fmtflags flags{stream_.flags()};
+        std::stringstream ss;
+        ss << std::showpoint
+           << std::setprecision(std::numeric_limits<double>::max_digits10)
+           << v.get();
 
-        stream_ << std::showpoint;
-        write(v.get());
+        auto double_str = ss.str();
+        auto pos = double_str.find("e0");
+        if (pos != std::string::npos)
+            double_str.replace(pos, 2, "e");
+        pos = double_str.find("e-0");
+        if (pos != std::string::npos)
+            double_str.replace(pos, 3, "e-");
 
-        stream_.flags(flags);
+        stream_ << double_str;
+        has_naked_endline_ = false;
     }
 
     /**
@@ -3287,9 +3498,9 @@ class toml_writer
      * offset_datetime.
      */
     template <class T>
-    typename std::enable_if<is_one_of<T, int64_t, local_date, local_time,
-                                      local_datetime,
-                                      offset_datetime>::value>::type
+    typename std::enable_if<
+        is_one_of<T, int64_t, local_date, local_time, local_datetime,
+                  offset_datetime>::value>::type
     write(const value<T>& v)
     {
         write(v.get());
@@ -3453,5 +3664,5 @@ inline std::ostream& operator<<(std::ostream& stream, const array& a)
     a.accept(writer);
     return stream;
 }
-}
-#endif
+} // namespace cpptoml
+#endif // CPPTOML_H

src/libexpr/common-eval-args.cc

@@ -45,9 +45,11 @@ Bindings * MixEvalArgs::getAutoArgs(EvalState & state)
 
 Path lookupFileArg(EvalState & state, string s)
 {
-    if (isUri(s))
-        return getDownloader()->downloadCached(state.store, s, true);
-    else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') {
+    if (isUri(s)) {
+        CachedDownloadRequest request(s);
+        request.unpack = true;
+        return getDownloader()->downloadCached(state.store, request).path;
+    } else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') {
         Path p = s.substr(1, s.size() - 2);
         return state.findFile(p);
     } else

src/libexpr/eval.cc

@@ -7,8 +7,10 @@
 #include "eval-inline.hh"
 #include "download.hh"
 #include "json.hh"
+#include "function-trace.hh"
 
 #include <algorithm>
+#include <chrono>
 #include <cstring>
 #include <unistd.h>
 #include <sys/time.h>
@@ -16,7 +18,6 @@
 #include <iostream>
 #include <fstream>
 
-#include <sys/time.h>
 #include <sys/resource.h>
 
 #if HAVE_BOEHMGC
@@ -130,6 +131,16 @@ std::ostream & operator << (std::ostream & str, const Value & v)
 }
 
 
+const Value *getPrimOp(const Value &v) {
+    const Value * primOp = &v;
+    while (primOp->type == tPrimOpApp) {
+        primOp = primOp->primOpApp.left;
+    }
+    assert(primOp->type == tPrimOp);
+    return primOp;
+}
+
+
 string showType(const Value & v)
 {
     switch (v.type) {
@@ -144,8 +155,10 @@ string showType(const Value & v)
         case tApp: return "a function application";
         case tLambda: return "a function";
         case tBlackhole: return "a black hole";
-        case tPrimOp: return "a built-in function";
-        case tPrimOpApp: return "a partially applied built-in function";
+        case tPrimOp:
+            return fmt("the built-in function '%s'", string(v.primOp->name));
+        case tPrimOpApp:
+            return fmt("the partially applied built-in function '%s'", string(getPrimOp(v)->primOp->name));
         case tExternal: return v.external->showType();
         case tFloat: return "a float";
     }
@@ -507,9 +520,9 @@ LocalNoInlineNoReturn(void throwTypeError(const char * s, const ExprLambda & fun
     throw TypeError(format(s) % fun.showNamePos() % s2 % pos);
 }
 
-LocalNoInlineNoReturn(void throwAssertionError(const char * s, const Pos & pos))
+LocalNoInlineNoReturn(void throwAssertionError(const char * s, const string & s1, const Pos & pos))
 {
-    throw AssertionError(format(s) % pos);
+    throw AssertionError(format(s) % s1 % pos);
 }
 
 LocalNoInlineNoReturn(void throwUndefinedVarError(const char * s, const string & s1, const Pos & pos))
@@ -865,7 +878,7 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v)
         if (hasOverrides) {
             Value * vOverrides = (*v.attrs)[overrides->second.displ].value;
             state.forceAttrs(*vOverrides);
-            Bindings * newBnds = state.allocBindings(v.attrs->size() + vOverrides->attrs->size());
+            Bindings * newBnds = state.allocBindings(v.attrs->capacity() + vOverrides->attrs->size());
             for (auto & i : *v.attrs)
                 newBnds->push_back(i);
             for (auto & i : *vOverrides->attrs) {
@@ -1082,9 +1095,10 @@ void EvalState::callPrimOp(Value & fun, Value & arg, Value & v, const Pos & pos)
     }
 }
 
-
 void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & pos)
 {
+    auto trace = evalSettings.traceFunctionCalls ? std::make_unique<FunctionCallTrace>(pos) : nullptr;
+
     forceValue(fun, pos);
 
     if (fun.type == tPrimOp || fun.type == tPrimOpApp) {
@@ -1239,8 +1253,11 @@ void ExprIf::eval(EvalState & state, Env & env, Value & v)
 
 void ExprAssert::eval(EvalState & state, Env & env, Value & v)
 {
-    if (!state.evalBool(env, cond, pos))
-        throwAssertionError("assertion failed at %1%", pos);
+    if (!state.evalBool(env, cond, pos)) {
+        std::ostringstream out;
+        cond->show(out);
+        throwAssertionError("assertion %1% failed at %2%", out.str(), pos);
+    }
     body->eval(state, env, v);
 }
 
@@ -1553,6 +1570,19 @@ bool EvalState::isDerivation(Value & v)
 }
 
 
+std::optional<string> EvalState::tryAttrsToString(const Pos & pos, Value & v,
+    PathSet & context, bool coerceMore, bool copyToStore)
+{
+    auto i = v.attrs->find(sToString);
+    if (i != v.attrs->end()) {
+        Value v1;
+        callFunction(*i->value, v, v1, pos);
+        return coerceToString(pos, v1, context, coerceMore, copyToStore);
+    }
+
+    return {};
+}
+
 string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context,
     bool coerceMore, bool copyToStore)
 {
@@ -1571,13 +1601,11 @@ string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context,
     }
 
     if (v.type == tAttrs) {
-        auto i = v.attrs->find(sToString);
-        if (i != v.attrs->end()) {
-            Value v1;
-            callFunction(*i->value, v, v1, pos);
-            return coerceToString(pos, v1, context, coerceMore, copyToStore);
+        auto maybeString = tryAttrsToString(pos, v, context, coerceMore, copyToStore);
+        if (maybeString) {
+            return *maybeString;
         }
-        i = v.attrs->find(sOutPath);
+        auto i = v.attrs->find(sOutPath);
         if (i == v.attrs->end()) throwTypeError("cannot coerce a set to a string, at %1%", pos);
         return coerceToString(pos, *i->value, context, coerceMore, copyToStore);
     }
@@ -1799,6 +1827,7 @@ void EvalState::printStats()
             gc.attr("totalBytes", totalBytes);
         }
 #endif
+
         if (countCalls) {
             {
                 auto obj = topObj.object("primops");
@@ -1834,6 +1863,11 @@ void EvalState::printStats()
                 }
             }
         }
+
+        if (getEnv("NIX_SHOW_SYMBOLS", "0") != "0") {
+            auto list = topObj.list("symbols");
+            symbols.dump([&](const std::string & s) { list.elem(s); });
+        }
     }
 }
 

src/libexpr/eval.hh

@@ -8,6 +8,7 @@
 #include "config.hh"
 
 #include <map>
+#include <optional>
 #include <unordered_map>
 
 
@@ -81,7 +82,7 @@ public:
 
     /* The allowed filesystem paths in restricted or pure evaluation
        mode. */
-    std::experimental::optional<PathSet> allowedPaths;
+    std::optional<PathSet> allowedPaths;
 
     Value vEmptySet;
 
@@ -195,6 +196,9 @@ public:
        set with attribute `type = "derivation"'). */
     bool isDerivation(Value & v);
 
+    std::optional<string> tryAttrsToString(const Pos & pos, Value & v,
+        PathSet & context, bool coerceMore = false, bool copyToStore = true);
+
     /* String coercion.  Converts strings, paths and derivations to a
        string.  If `coerceMore' is set, also converts nulls, integers,
        booleans and lists to a string.  If `copyToStore' is set,
@@ -316,6 +320,9 @@ private:
 /* Return a string representing the type of the value `v'. */
 string showType(const Value & v);
 
+/* Decode a context string ‘!<name>!<path>’ into a pair <path,
+   name>. */
+std::pair<string, string> decodeContext(const string & s);
 
 /* If `path' refers to a directory, then append "/default.nix". */
 Path resolveExprPath(Path path);
@@ -346,6 +353,9 @@ struct EvalSettings : Config
 
     Setting<Strings> allowedUris{this, {}, "allowed-uris",
         "Prefixes of URIs that builtin functions such as fetchurl and fetchGit are allowed to fetch."};
+
+    Setting<bool> traceFunctionCalls{this, false, "trace-function-calls",
+        "Emit log messages for each function entry and exit at the 'vomit' log level (-vvvv)"};
 };
 
 extern EvalSettings evalSettings;

src/libexpr/function-trace.cc

@@ -0,0 +1,17 @@
+#include "function-trace.hh"
+
+namespace nix {
+
+FunctionCallTrace::FunctionCallTrace(const Pos & pos) : pos(pos) {
+    auto duration = std::chrono::high_resolution_clock::now().time_since_epoch();
+    auto ns = std::chrono::duration_cast<std::chrono::nanoseconds>(duration);
+    printMsg(lvlInfo, "function-trace entered %1% at %2%", pos, ns.count());
+}
+
+FunctionCallTrace::~FunctionCallTrace() {
+    auto duration = std::chrono::high_resolution_clock::now().time_since_epoch();
+    auto ns = std::chrono::duration_cast<std::chrono::nanoseconds>(duration);
+    printMsg(lvlInfo, "function-trace exited %1% at %2%", pos, ns.count());
+}
+
+}

src/libexpr/function-trace.hh

@@ -0,0 +1,15 @@
+#pragma once
+
+#include "eval.hh"
+
+#include <chrono>
+
+namespace nix {
+
+struct FunctionCallTrace
+{
+    const Pos & pos;
+    FunctionCallTrace(const Pos & pos);
+    ~FunctionCallTrace();
+};
+}

src/libexpr/get-drvs.cc

@@ -295,7 +295,7 @@ static bool getDerivation(EvalState & state, Value & v,
 }
 
 
-std::experimental::optional<DrvInfo> getDerivation(EvalState & state, Value & v,
+std::optional<DrvInfo> getDerivation(EvalState & state, Value & v,
     bool ignoreAssertionFailures)
 {
     Done done;

src/libexpr/get-drvs.hh

@@ -78,7 +78,7 @@ typedef list<DrvInfo> DrvInfos;
 
 /* If value `v' denotes a derivation, return a DrvInfo object
    describing it. Otherwise return nothing. */
-std::experimental::optional<DrvInfo> getDerivation(EvalState & state,
+std::optional<DrvInfo> getDerivation(EvalState & state,
     Value & v, bool ignoreAssertionFailures);
 
 void getDerivations(EvalState & state, Value & v, const string & pathPrefix,

src/libexpr/json-to-value.cc

@@ -111,9 +111,9 @@ static void parseJSON(EvalState & state, const char * & s, Value & v)
                 mkFloat(v, stod(tmp_number));
             else
                 mkInt(v, stol(tmp_number));
-        } catch (std::invalid_argument e) {
+        } catch (std::invalid_argument & e) {
             throw JSONParseError("invalid JSON number");
-        } catch (std::out_of_range e) {
+        } catch (std::out_of_range & e) {
             throw JSONParseError("out-of-range JSON number");
         }
     }

src/libexpr/nix-expr.pc.in

@@ -7,4 +7,4 @@ Description: Nix Package Manager
 Version: @PACKAGE_VERSION@
 Requires: nix-store bdw-gc
 Libs: -L${libdir} -lnixexpr
-Cflags: -I${includedir}/nix -std=c++14
+Cflags: -I${includedir}/nix -std=c++17

src/libexpr/parser.y

@@ -1,7 +1,7 @@
 %glr-parser
 %pure-parser
 %locations
-%error-verbose
+%define parse.error verbose
 %defines
 /* %no-lines */
 %parse-param { void * scanner }
@@ -81,6 +81,8 @@ static void addAttr(ExprAttrs * attrs, AttrPath & attrPath,
     AttrPath::iterator i;
     // All attrpaths have at least one attr
     assert(!attrPath.empty());
+    // Checking attrPath validity.
+    // ===========================
     for (i = attrPath.begin(); i + 1 < attrPath.end(); i++) {
         if (i->symbol.set()) {
             ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(i->symbol);
@@ -102,11 +104,29 @@ static void addAttr(ExprAttrs * attrs, AttrPath & attrPath,
             attrs = nested;
         }
     }
+    // Expr insertion.
+    // ==========================
     if (i->symbol.set()) {
         ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(i->symbol);
         if (j != attrs->attrs.end()) {
-            dupAttr(attrPath, pos, j->second.pos);
+            // This attr path is already defined. However, if both
+            // e and the expr pointed by the attr path are two attribute sets,
+            // we want to merge them.
+            // Otherwise, throw an error.
+            auto ae = dynamic_cast<ExprAttrs *>(e);
+            auto jAttrs = dynamic_cast<ExprAttrs *>(j->second.e);
+            if (jAttrs && ae) {
+                for (auto & ad : ae->attrs) {
+                    auto j2 = jAttrs->attrs.find(ad.first);
+                    if (j2 != jAttrs->attrs.end()) // Attr already defined in iAttrs, error.
+                        dupAttr(ad.first, j2->second.pos, ad.second.pos);
+                    jAttrs->attrs[ad.first] = ad.second;
+                }
+            } else {
+                dupAttr(attrPath, pos, j->second.pos);
+            }
         } else {
+            // This attr path is not defined. Let's create it.
             attrs->attrs[i->symbol] = ExprAttrs::AttrDef(e, pos);
             e->setName(i->symbol);
         }
@@ -657,7 +677,9 @@ std::pair<bool, std::string> EvalState::resolveSearchPathElem(const SearchPathEl
 
     if (isUri(elem.second)) {
         try {
-            res = { true, getDownloader()->downloadCached(store, elem.second, true) };
+            CachedDownloadRequest request(elem.second);
+            request.unpack = true;
+            res = { true, getDownloader()->downloadCached(store, request).path };
         } catch (DownloadError & e) {
             printError(format("warning: Nix search path entry '%1%' cannot be downloaded, ignoring") % elem.second);
             res = { false, "" };

src/libexpr/primops.cc

@@ -315,6 +315,12 @@ static void prim_isBool(EvalState & state, const Pos & pos, Value * * args, Valu
     mkBool(v, args[0]->type == tBool);
 }
 
+/* Determine whether the argument is a path. */
+static void prim_isPath(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    state.forceValue(*args[0]);
+    mkBool(v, args[0]->type == tPath);
+}
 
 struct CompareValues
 {
@@ -555,7 +561,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
 
     PathSet context;
 
-    std::experimental::optional<std::string> outputHash;
+    std::optional<std::string> outputHash;
     std::string outputHashAlgo;
     bool outputHashRecursive = false;
 
@@ -687,21 +693,12 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
             }
         }
 
-        /* See prim_unsafeDiscardOutputDependency. */
-        else if (path.at(0) == '~')
-            drv.inputSrcs.insert(string(path, 1));
-
         /* Handle derivation outputs of the form ‘!<name>!<path>’. */
         else if (path.at(0) == '!') {
             std::pair<string, string> ctx = decodeContext(path);
             drv.inputDrvs[ctx.first].insert(ctx.second);
         }
 
-        /* Handle derivation contexts returned by
-           ‘builtins.storePath’. */
-        else if (isDerivation(path))
-            drv.inputDrvs[path] = state.store->queryDerivationOutputNames(path);
-
         /* Otherwise it's a source file. */
         else
             drv.inputSrcs.insert(path);
@@ -835,8 +832,14 @@ static void prim_pathExists(EvalState & state, const Pos & pos, Value * * args,
 {
     PathSet context;
     Path path = state.coerceToPath(pos, *args[0], context);
-    if (!context.empty())
-        throw EvalError(format("string '%1%' cannot refer to other paths, at %2%") % path % pos);
+    try {
+        state.realiseContext(context);
+    } catch (InvalidPathError & e) {
+        throw EvalError(format(
+                "cannot check the existence of '%1%', since path '%2%' is not valid, at %3%")
+            % path % e.path % pos);
+    }
+
     try {
         mkBool(v, pathExists(state.checkSourcePath(path)));
     } catch (SysError & e) {
@@ -926,6 +929,20 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va
     mkPath(v, state.checkSourcePath(state.findFile(searchPath, path, pos)).c_str());
 }
 
+/* Return the cryptographic hash of a file in base-16. */
+static void prim_hashFile(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    string type = state.forceStringNoCtx(*args[0], pos);
+    HashType ht = parseHashType(type);
+    if (ht == htUnknown)
+      throw Error(format("unknown hash type '%1%', at %2%") % type % pos);
+
+    PathSet context; // discarded
+    Path p = state.coerceToPath(pos, *args[1], context);
+
+    mkString(v, hashFile(ht, state.checkSourcePath(p)).to_string(Base16, false), context);
+}
+
 /* Read a directory (without . or ..) */
 static void prim_readDir(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
@@ -1004,13 +1021,8 @@ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Valu
     PathSet refs;
 
     for (auto path : context) {
-        if (path.at(0) == '=') path = string(path, 1);
-        if (isDerivation(path)) {
-            /* See prim_unsafeDiscardOutputDependency. */
-            if (path.at(0) != '~')
-                throw EvalError(format("in 'toFile': the file '%1%' cannot refer to derivation outputs, at %2%") % name % pos);
-            path = string(path, 1);
-        }
+        if (path.at(0) != '/')
+            throw EvalError(format("in 'toFile': the file '%1%' cannot refer to derivation outputs, at %2%") % name % pos);
         refs.insert(path);
     }
 
@@ -1794,41 +1806,6 @@ static void prim_stringLength(EvalState & state, const Pos & pos, Value * * args
 }
 
 
-static void prim_unsafeDiscardStringContext(EvalState & state, const Pos & pos, Value * * args, Value & v)
-{
-    PathSet context;
-    string s = state.coerceToString(pos, *args[0], context);
-    mkString(v, s, PathSet());
-}
-
-
-static void prim_hasContext(EvalState & state, const Pos & pos, Value * * args, Value & v)
-{
-    PathSet context;
-    state.forceString(*args[0], context, pos);
-    mkBool(v, !context.empty());
-}
-
-
-/* Sometimes we want to pass a derivation path (i.e. pkg.drvPath) to a
-   builder without causing the derivation to be built (for instance,
-   in the derivation that builds NARs in nix-push, when doing
-   source-only deployment).  This primop marks the string context so
-   that builtins.derivation adds the path to drv.inputSrcs rather than
-   drv.inputDrvs. */
-static void prim_unsafeDiscardOutputDependency(EvalState & state, const Pos & pos, Value * * args, Value & v)
-{
-    PathSet context;
-    string s = state.coerceToString(pos, *args[0], context);
-
-    PathSet context2;
-    for (auto & p : context)
-        context2.insert(p.at(0) == '=' ? "~" + string(p, 1) : p);
-
-    mkString(v, s, context2);
-}
-
-
 /* Return the cryptographic hash of a string in base-16. */
 static void prim_hashString(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
@@ -2079,9 +2056,9 @@ static void prim_splitVersion(EvalState & state, const Pos & pos, Value * * args
 void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
     const string & who, bool unpack, const std::string & defaultName)
 {
-    string url;
-    Hash expectedHash;
-    string name = defaultName;
+    CachedDownloadRequest request("");
+    request.unpack = unpack;
+    request.name = defaultName;
 
     state.forceValue(*args[0]);
 
@@ -2092,32 +2069,32 @@ void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
         for (auto & attr : *args[0]->attrs) {
             string n(attr.name);
             if (n == "url")
-                url = state.forceStringNoCtx(*attr.value, *attr.pos);
+                request.uri = state.forceStringNoCtx(*attr.value, *attr.pos);
             else if (n == "sha256")
-                expectedHash = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256);
+                request.expectedHash = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256);
             else if (n == "name")
-                name = state.forceStringNoCtx(*attr.value, *attr.pos);
+                request.name = state.forceStringNoCtx(*attr.value, *attr.pos);
             else
                 throw EvalError(format("unsupported argument '%1%' to '%2%', at %3%") % attr.name % who % attr.pos);
         }
 
-        if (url.empty())
+        if (request.uri.empty())
             throw EvalError(format("'url' argument required, at %1%") % pos);
 
     } else
-        url = state.forceStringNoCtx(*args[0], pos);
+        request.uri = state.forceStringNoCtx(*args[0], pos);
 
-    state.checkURI(url);
+    state.checkURI(request.uri);
 
-    if (evalSettings.pureEval && !expectedHash)
+    if (evalSettings.pureEval && !request.expectedHash)
         throw Error("in pure evaluation mode, '%s' requires a 'sha256' argument", who);
 
-    Path res = getDownloader()->downloadCached(state.store, url, unpack, name, expectedHash);
+    auto res = getDownloader()->downloadCached(state.store, request);
 
     if (state.allowedPaths)
-        state.allowedPaths->insert(res);
+        state.allowedPaths->insert(res.path);
 
-    mkString(v, res, PathSet({res}));
+    mkString(v, res.storePath, PathSet({res.storePath}));
 }
 
 
@@ -2218,6 +2195,7 @@ void EvalState::createBaseEnv()
     addPrimOp("__isInt", 1, prim_isInt);
     addPrimOp("__isFloat", 1, prim_isFloat);
     addPrimOp("__isBool", 1, prim_isBool);
+    addPrimOp("__isPath", 1, prim_isPath);
     addPrimOp("__genericClosure", 1, prim_genericClosure);
     addPrimOp("abort", 1, prim_abort);
     addPrimOp("__addErrorContext", 2, prim_addErrorContext);
@@ -2244,6 +2222,7 @@ void EvalState::createBaseEnv()
     addPrimOp("__readFile", 1, prim_readFile);
     addPrimOp("__readDir", 1, prim_readDir);
     addPrimOp("__findFile", 2, prim_findFile);
+    addPrimOp("__hashFile", 2, prim_hashFile);
 
     // Creating files
     addPrimOp("__toXML", 1, prim_toXML);
@@ -2299,9 +2278,6 @@ void EvalState::createBaseEnv()
     addPrimOp("toString", 1, prim_toString);
     addPrimOp("__substring", 3, prim_substring);
     addPrimOp("__stringLength", 1, prim_stringLength);
-    addPrimOp("__hasContext", 1, prim_hasContext);
-    addPrimOp("__unsafeDiscardStringContext", 1, prim_unsafeDiscardStringContext);
-    addPrimOp("__unsafeDiscardOutputDependency", 1, prim_unsafeDiscardOutputDependency);
     addPrimOp("__hashString", 2, prim_hashString);
     addPrimOp("__match", 2, prim_match);
     addPrimOp("__split", 2, prim_split);

src/libexpr/primops/context.cc

@@ -0,0 +1,187 @@
+#include "primops.hh"
+#include "eval-inline.hh"
+#include "derivations.hh"
+
+namespace nix {
+
+static void prim_unsafeDiscardStringContext(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    PathSet context;
+    string s = state.coerceToString(pos, *args[0], context);
+    mkString(v, s, PathSet());
+}
+
+static RegisterPrimOp r1("__unsafeDiscardStringContext", 1, prim_unsafeDiscardStringContext);
+
+
+static void prim_hasContext(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    PathSet context;
+    state.forceString(*args[0], context, pos);
+    mkBool(v, !context.empty());
+}
+
+static RegisterPrimOp r2("__hasContext", 1, prim_hasContext);
+
+
+/* Sometimes we want to pass a derivation path (i.e. pkg.drvPath) to a
+   builder without causing the derivation to be built (for instance,
+   in the derivation that builds NARs in nix-push, when doing
+   source-only deployment).  This primop marks the string context so
+   that builtins.derivation adds the path to drv.inputSrcs rather than
+   drv.inputDrvs. */
+static void prim_unsafeDiscardOutputDependency(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    PathSet context;
+    string s = state.coerceToString(pos, *args[0], context);
+
+    PathSet context2;
+    for (auto & p : context)
+        context2.insert(p.at(0) == '=' ? string(p, 1) : p);
+
+    mkString(v, s, context2);
+}
+
+static RegisterPrimOp r3("__unsafeDiscardOutputDependency", 1, prim_unsafeDiscardOutputDependency);
+
+
+/* Extract the context of a string as a structured Nix value.
+
+   The context is represented as an attribute set whose keys are the
+   paths in the context set and whose values are attribute sets with
+   the following keys:
+     path: True if the relevant path is in the context as a plain store
+           path (i.e. the kind of context you get when interpolating
+           a Nix path (e.g. ./.) into a string). False if missing.
+     allOutputs: True if the relevant path is a derivation and it is
+                  in the context as a drv file with all of its outputs
+                  (i.e. the kind of context you get when referencing
+                  .drvPath of some derivation). False if missing.
+     outputs: If a non-empty list, the relevant path is a derivation
+              and the provided outputs are referenced in the context
+              (i.e. the kind of context you get when referencing
+              .outPath of some derivation). Empty list if missing.
+   Note that for a given path any combination of the above attributes
+   may be present.
+*/
+static void prim_getContext(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    struct ContextInfo {
+        bool path = false;
+        bool allOutputs = false;
+        Strings outputs;
+    };
+    PathSet context;
+    state.forceString(*args[0], context, pos);
+    auto contextInfos = std::map<Path, ContextInfo>();
+    for (const auto & p : context) {
+        Path drv;
+        string output;
+        const Path * path = &p;
+        if (p.at(0) == '=') {
+            drv = string(p, 1);
+            path = &drv;
+        } else if (p.at(0) == '!') {
+            std::pair<string, string> ctx = decodeContext(p);
+            drv = ctx.first;
+            output = ctx.second;
+            path = &drv;
+        }
+        auto isPath = drv.empty();
+        auto isAllOutputs = (!drv.empty()) && output.empty();
+
+        auto iter = contextInfos.find(*path);
+        if (iter == contextInfos.end()) {
+            contextInfos.emplace(*path, ContextInfo{isPath, isAllOutputs, output.empty() ? Strings{} : Strings{std::move(output)}});
+        } else {
+            if (isPath)
+                iter->second.path = true;
+            else if (isAllOutputs)
+                iter->second.allOutputs = true;
+            else
+                iter->second.outputs.emplace_back(std::move(output));
+        }
+    }
+
+    state.mkAttrs(v, contextInfos.size());
+
+    auto sPath = state.symbols.create("path");
+    auto sAllOutputs = state.symbols.create("allOutputs");
+    for (const auto & info : contextInfos) {
+        auto & infoVal = *state.allocAttr(v, state.symbols.create(info.first));
+        state.mkAttrs(infoVal, 3);
+        if (info.second.path)
+            mkBool(*state.allocAttr(infoVal, sPath), true);
+        if (info.second.allOutputs)
+            mkBool(*state.allocAttr(infoVal, sAllOutputs), true);
+        if (!info.second.outputs.empty()) {
+            auto & outputsVal = *state.allocAttr(infoVal, state.sOutputs);
+            state.mkList(outputsVal, info.second.outputs.size());
+            size_t i = 0;
+            for (const auto & output : info.second.outputs) {
+                mkString(*(outputsVal.listElems()[i++] = state.allocValue()), output);
+            }
+        }
+        infoVal.attrs->sort();
+    }
+    v.attrs->sort();
+}
+
+static RegisterPrimOp r4("__getContext", 1, prim_getContext);
+
+
+/* Append the given context to a given string.
+
+   See the commentary above unsafeGetContext for details of the
+   context representation.
+*/
+static void prim_appendContext(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    PathSet context;
+    auto orig = state.forceString(*args[0], context, pos);
+
+    state.forceAttrs(*args[1], pos);
+
+    auto sPath = state.symbols.create("path");
+    auto sAllOutputs = state.symbols.create("allOutputs");
+    for (auto & i : *args[1]->attrs) {
+        if (!state.store->isStorePath(i.name))
+            throw EvalError("Context key '%s' is not a store path, at %s", i.name, i.pos);
+        if (!settings.readOnlyMode)
+            state.store->ensurePath(i.name);
+        state.forceAttrs(*i.value, *i.pos);
+        auto iter = i.value->attrs->find(sPath);
+        if (iter != i.value->attrs->end()) {
+            if (state.forceBool(*iter->value, *iter->pos))
+                context.insert(i.name);
+        }
+
+        iter = i.value->attrs->find(sAllOutputs);
+        if (iter != i.value->attrs->end()) {
+            if (state.forceBool(*iter->value, *iter->pos)) {
+                if (!isDerivation(i.name)) {
+                    throw EvalError("Tried to add all-outputs context of %s, which is not a derivation, to a string, at %s", i.name, i.pos);
+                }
+                context.insert("=" + string(i.name));
+            }
+        }
+
+        iter = i.value->attrs->find(state.sOutputs);
+        if (iter != i.value->attrs->end()) {
+            state.forceList(*iter->value, *iter->pos);
+            if (iter->value->listSize() && !isDerivation(i.name)) {
+                throw EvalError("Tried to add derivation output context of %s, which is not a derivation, to a string, at %s", i.name, i.pos);
+            }
+            for (unsigned int n = 0; n < iter->value->listSize(); ++n) {
+                auto name = state.forceStringNoCtx(*iter->value->listElems()[n], *iter->pos);
+                context.insert("!" + name + "!" + string(i.name));
+            }
+        }
+    }
+
+    mkString(v, orig, context);
+}
+
+static RegisterPrimOp r5("__appendContext", 2, prim_appendContext);
+
+}

src/libexpr/primops/fetchGit.cc

@@ -26,7 +26,7 @@ struct GitInfo
 std::regex revRegex("^[0-9a-fA-F]{40}$");
 
 GitInfo exportGit(ref<Store> store, const std::string & uri,
-    std::experimental::optional<std::string> ref, std::string rev,
+    std::optional<std::string> ref, std::string rev,
     const std::string & name)
 {
     if (evalSettings.pureEval && rev == "")
@@ -38,7 +38,7 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
 
         try {
             runProgram("git", true, { "-C", uri, "diff-index", "--quiet", "HEAD", "--" });
-        } catch (ExecError e) {
+        } catch (ExecError & e) {
             if (!WIFEXITED(e.status) || WEXITSTATUS(e.status) != 1) throw;
             clean = false;
         }
@@ -94,7 +94,11 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
         runProgram("git", true, { "init", "--bare", cacheDir });
     }
 
-    Path localRefFile = cacheDir + "/refs/heads/" + *ref;
+    Path localRefFile;
+    if (ref->compare(0, 5, "refs/") == 0)
+        localRefFile = cacheDir + "/" + *ref;
+    else
+        localRefFile = cacheDir + "/refs/heads/" + *ref;
 
     bool doFetch;
     time_t now = time(0);
@@ -116,7 +120,7 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
            git fetch to update the local ref to the remote ref. */
         struct stat st;
         doFetch = stat(localRefFile.c_str(), &st) != 0 ||
-            st.st_mtime + settings.tarballTtl <= now;
+            (uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now;
     }
     if (doFetch)
     {
@@ -190,7 +194,7 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
 static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     std::string url;
-    std::experimental::optional<std::string> ref;
+    std::optional<std::string> ref;
     std::string rev;
     std::string name = "source";
     PathSet context;
@@ -235,7 +239,7 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va
     v.attrs->sort();
 
     if (state.allowedPaths)
-        state.allowedPaths->insert(gitInfo.storePath);
+        state.allowedPaths->insert(state.store->toRealPath(gitInfo.storePath));
 }
 
 static RegisterPrimOp r("fetchGit", 1, prim_fetchGit);

src/libexpr/primops/fetchMercurial.cc

@@ -80,7 +80,7 @@ HgInfo exportMercurial(ref<Store> store, const std::string & uri,
     time_t now = time(0);
     struct stat st;
     if (stat(stampFile.c_str(), &st) != 0 ||
-        st.st_mtime + settings.tarballTtl <= now)
+        (uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now)
     {
         /* Except that if this is a commit hash that we already have,
            we don't have to pull again. */
@@ -96,17 +96,14 @@ HgInfo exportMercurial(ref<Store> store, const std::string & uri,
                 try {
                     runProgram("hg", true, { "pull", "-R", cacheDir, "--", uri });
                 }
-                catch (ExecError & e){
+                catch (ExecError & e) {
                     string transJournal = cacheDir + "/.hg/store/journal";
                     /* hg throws "abandoned transaction" error only if this file exists */
-                    if (pathExists(transJournal))
-                    {
+                    if (pathExists(transJournal)) {
                         runProgram("hg", true, { "recover", "-R", cacheDir });
                         runProgram("hg", true, { "pull", "-R", cacheDir, "--", uri });
-                    }
-                    else 
-                    {
-                        throw ExecError(e.status, fmt("program hg '%1%' ", statusToString(e.status)));
+                    } else {
+                        throw ExecError(e.status, fmt("'hg pull' %s", statusToString(e.status)));
                     }
                 }
             } else {
@@ -214,7 +211,7 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar
     v.attrs->sort();
 
     if (state.allowedPaths)
-        state.allowedPaths->insert(hgInfo.storePath);
+        state.allowedPaths->insert(state.store->toRealPath(hgInfo.storePath));
 }
 
 static RegisterPrimOp r("fetchMercurial", 1, prim_fetchMercurial);

src/libexpr/primops/fromTOML.cc

@@ -49,6 +49,19 @@ static void prim_fromTOML(EvalState & state, const Pos & pos, Value * * args, Va
                 visit(*(v.listElems()[i] = state.allocValue()), t2->get()[i]);
         }
 
+        // Handle cases like 'a = [[{ a = true }]]', which IMHO should be
+        // parsed as a array containing an array containing a table,
+        // but instead are parsed as an array containing a table array
+        // containing a table.
+        else if (auto t2 = t->as_table_array()) {
+            size_t size = t2->get().size();
+
+            state.mkList(v, size);
+
+            for (size_t j = 0; j < size; ++j)
+                visit(*(v.listElems()[j] = state.allocValue()), t2->get()[j]);
+        }
+
         else if (t->is_value()) {
             if (auto val = t->as<int64_t>())
                 mkInt(v, val->get());

src/libexpr/symbol-table.hh

@@ -75,6 +75,13 @@ public:
     }
 
     size_t totalSize() const;
+
+    template<typename T>
+    void dump(T callback)
+    {
+        for (auto & s : symbols)
+            callback(s);
+    }
 };
 
 }

src/libexpr/value-to-json.cc

@@ -40,7 +40,12 @@ void printValueAsJSON(EvalState & state, bool strict,
             break;
 
         case tAttrs: {
-            Bindings::iterator i = v.attrs->find(state.sOutPath);
+            auto maybeString = state.tryAttrsToString(noPos, v, context, false, false);
+            if (maybeString) {
+                out.write(*maybeString);
+                break;
+            }
+            auto i = v.attrs->find(state.sOutPath);
             if (i == v.attrs->end()) {
                 auto obj(out.object());
                 StringSet names;

src/libmain/common-args.cc

@@ -35,6 +35,15 @@ MixCommonArgs::MixCommonArgs(const string & programName)
             }
         });
 
+    mkFlag()
+        .longName("max-jobs")
+        .shortName('j')
+        .label("jobs")
+        .description("maximum number of parallel builds")
+        .handler([=](std::string s) {
+            settings.set("max-jobs", s);
+        });
+
     std::string cat = "config";
     globalConfig.convertToArgs(*this, cat);
 

src/libmain/nix-main.pc.in

@@ -6,4 +6,4 @@ Name: Nix
 Description: Nix Package Manager
 Version: @PACKAGE_VERSION@
 Libs: -L${libdir} -lnixmain
-Cflags: -I${includedir}/nix -std=c++14
+Cflags: -I${includedir}/nix -std=c++17

src/libmain/shared.cc

@@ -80,6 +80,7 @@ string getArg(const string & opt,
 }
 
 
+#if OPENSSL_VERSION_NUMBER < 0x10101000L
 /* OpenSSL is not thread-safe by default - it will randomly crash
    unless the user supplies a mutex locking function. So let's do
    that. */
@@ -92,6 +93,7 @@ static void opensslLockCallback(int mode, int type, const char * file, int line)
     else
         opensslLocks[type].unlock();
 }
+#endif
 
 
 static void sigHandler(int signo) { }
@@ -105,9 +107,11 @@ void initNix()
     std::cerr.rdbuf()->pubsetbuf(buf, sizeof(buf));
 #endif
 
+#if OPENSSL_VERSION_NUMBER < 0x10101000L
     /* Initialise OpenSSL locking. */
     opensslLocks = std::vector<std::mutex>(CRYPTO_num_locks());
     CRYPTO_set_locking_callback(opensslLockCallback);
+#endif
 
     loadConfFile();
 
@@ -125,6 +129,15 @@ void initNix()
     act.sa_handler = sigHandler;
     if (sigaction(SIGUSR1, &act, 0)) throw SysError("handling SIGUSR1");
 
+#if __APPLE__
+    /* HACK: on darwin, we need can’t use sigprocmask with SIGWINCH.
+     * Instead, add a dummy sigaction handler, and signalHandlerThread
+     * can handle the rest. */
+    struct sigaction sa;
+    sa.sa_handler = sigHandler;
+    if (sigaction(SIGWINCH, &sa, 0)) throw SysError("handling SIGWINCH");
+#endif
+
     /* Register a SIGSEGV handler to detect stack overflows. */
     detectStackOverflow();
 
@@ -175,10 +188,6 @@ LegacyArgs::LegacyArgs(const std::string & programName,
         .description("build from source if substitution fails")
         .set(&(bool&) settings.tryFallback, true);
 
-    mkFlag1('j', "max-jobs", "jobs", "maximum number of parallel builds", [=](std::string s) {
-        settings.set("max-jobs", s);
-    });
-
     auto intSettingAlias = [&](char shortName, const std::string & longName,
         const std::string & description, const std::string & dest) {
         mkFlag<unsigned int>(shortName, longName, description, [=](unsigned int n) {

src/libstore/binary-cache-store.cc

@@ -55,7 +55,7 @@ void BinaryCacheStore::init()
 }
 
 void BinaryCacheStore::getFile(const std::string & path,
-    Callback<std::shared_ptr<std::string>> callback)
+    Callback<std::shared_ptr<std::string>> callback) noexcept
 {
     try {
         callback(getFile(path));
@@ -240,7 +240,7 @@ void BinaryCacheStore::narFromPath(const Path & storePath, Sink & sink)
 }
 
 void BinaryCacheStore::queryPathInfoUncached(const Path & storePath,
-    Callback<std::shared_ptr<ValidPathInfo>> callback)
+    Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept
 {
     auto uri = getUri();
     auto act = std::make_shared<Activity>(*logger, lvlTalkative, actQueryPathInfo,
@@ -249,21 +249,23 @@ void BinaryCacheStore::queryPathInfoUncached(const Path & storePath,
 
     auto narInfoFile = narInfoFileFor(storePath);
 
+    auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback));
+
     getFile(narInfoFile,
         {[=](std::future<std::shared_ptr<std::string>> fut) {
             try {
                 auto data = fut.get();
 
-                if (!data) return callback(nullptr);
+                if (!data) return (*callbackPtr)(nullptr);
 
                 stats.narInfoRead++;
 
-                callback((std::shared_ptr<ValidPathInfo>)
+                (*callbackPtr)((std::shared_ptr<ValidPathInfo>)
                     std::make_shared<NarInfo>(*this, *data, narInfoFile));
 
                 (void) act; // force Activity into this lambda to ensure it stays alive
             } catch (...) {
-                callback.rethrow();
+                callbackPtr->rethrow();
             }
         }});
 }

src/libstore/binary-cache-store.hh

@@ -47,7 +47,7 @@ public:
     /* Fetch the specified file and call the specified callback with
        the result. A subclass may implement this asynchronously. */
     virtual void getFile(const std::string & path,
-        Callback<std::shared_ptr<std::string>> callback);
+        Callback<std::shared_ptr<std::string>> callback) noexcept;
 
     std::shared_ptr<std::string> getFile(const std::string & path);
 
@@ -72,24 +72,11 @@ public:
 
     bool isValidPathUncached(const Path & path) override;
 
-    PathSet queryAllValidPaths() override
-    { unsupported(); }
-
     void queryPathInfoUncached(const Path & path,
-        Callback<std::shared_ptr<ValidPathInfo>> callback) override;
-
-    void queryReferrers(const Path & path,
-        PathSet & referrers) override
-    { unsupported(); }
-
-    PathSet queryDerivationOutputs(const Path & path) override
-    { unsupported(); }
-
-    StringSet queryDerivationOutputNames(const Path & path) override
-    { unsupported(); }
+        Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept override;
 
     Path queryPathFromHashPart(const string & hashPart) override
-    { unsupported(); }
+    { unsupported("queryPathFromHashPart"); }
 
     bool wantMassQuery() override { return wantMassQuery_; }
 
@@ -108,22 +95,10 @@ public:
 
     BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv,
         BuildMode buildMode) override
-    { unsupported(); }
+    { unsupported("buildDerivation"); }
 
     void ensurePath(const Path & path) override
-    { unsupported(); }
-
-    void addTempRoot(const Path & path) override
-    { unsupported(); }
-
-    void addIndirectRoot(const Path & path) override
-    { unsupported(); }
-
-    Roots findRoots() override
-    { unsupported(); }
-
-    void collectGarbage(const GCOptions & options, GCResults & results) override
-    { unsupported(); }
+    { unsupported("ensurePath"); }
 
     ref<FSAccessor> getFSAccessor() override;
 

src/libstore/build.cc

@@ -12,6 +12,7 @@
 #include "json.hh"
 #include "nar-info.hh"
 #include "parsed-derivations.hh"
+#include "machines.hh"
 
 #include <algorithm>
 #include <iostream>
@@ -37,6 +38,7 @@
 #include <unistd.h>
 #include <errno.h>
 #include <cstring>
+#include <termios.h>
 
 #include <pwd.h>
 #include <grp.h>
@@ -264,6 +266,12 @@ public:
     /* Set if at least one derivation had a timeout. */
     bool timedOut;
 
+    /* Set if at least one derivation fails with a hash mismatch. */
+    bool hashMismatch;
+
+    /* Set if at least one derivation is not deterministic in check mode. */
+    bool checkMismatch;
+
     LocalStore & store;
 
     std::unique_ptr<HookInstance> hook;
@@ -460,6 +468,28 @@ static void commonChildInit(Pipe & logPipe)
     close(fdDevNull);
 }
 
+void handleDiffHook(uid_t uid, uid_t gid, Path tryA, Path tryB, Path drvPath, Path tmpDir)
+{
+    auto diffHook = settings.diffHook;
+    if (diffHook != "" && settings.runDiffHook) {
+        try {
+            RunOptions diffHookOptions(diffHook,{tryA, tryB, drvPath, tmpDir});
+            diffHookOptions.searchPath = true;
+            diffHookOptions.uid = uid;
+            diffHookOptions.gid = gid;
+            diffHookOptions.chdir = "/";
+
+            auto diffRes = runProgram(diffHookOptions);
+            if (!statusOk(diffRes.first))
+                throw ExecError(diffRes.first, fmt("diff-hook program '%1%' %2%", diffHook, statusToString(diffRes.first)));
+
+            if (diffRes.second != "")
+                printError(chomp(diffRes.second));
+        } catch (Error & error) {
+            printError("diff hook execution failed: %s", error.what());
+        }
+    }
+}
 
 //////////////////////////////////////////////////////////////////////
 
@@ -881,6 +911,9 @@ public:
         Worker & worker, BuildMode buildMode = bmNormal);
     ~DerivationGoal();
 
+    /* Whether we need to perform hash rewriting if there are valid output paths. */
+    bool needsHashRewrite();
+
     void timedOut() override;
 
     string key() override
@@ -924,6 +957,9 @@ private:
     /* Fill in the environment for the builder. */
     void initEnv();
 
+    /* Setup tmp dir location. */
+    void initTmpDir();
+
     /* Write a JSON file containing the derivation attributes. */
     void writeStructuredAttrs();
 
@@ -1033,6 +1069,17 @@ DerivationGoal::~DerivationGoal()
 }
 
 
+inline bool DerivationGoal::needsHashRewrite()
+{
+#if __linux__
+    return !useChroot;
+#else
+    /* Darwin requires hash rewriting even when sandboxing is enabled. */
+    return true;
+#endif
+}
+
+
 void DerivationGoal::killChild()
 {
     if (pid != -1) {
@@ -1153,7 +1200,7 @@ void DerivationGoal::haveDerivation()
     /* We are first going to try to create the invalid output paths
        through substitutes.  If that doesn't work, we'll build
        them. */
-    if (settings.useSubstitutes && drv->substitutesAllowed())
+    if (settings.useSubstitutes && parsedDrv->substitutesAllowed())
         for (auto & i : invalidOutputs)
             addWaitee(worker.makeSubstitutionGoal(i, buildMode == bmRepair ? Repair : NoRepair));
 
@@ -1521,8 +1568,8 @@ void DerivationGoal::buildDone()
     if (hook) {
         hook->builderOut.readSide = -1;
         hook->fromHook.readSide = -1;
-    }
-    else builderOut.readSide = -1;
+    } else
+        builderOut.readSide = -1;
 
     /* Close the log file. */
     closeLogFile();
@@ -1585,6 +1632,61 @@ void DerivationGoal::buildDone()
            being valid. */
         registerOutputs();
 
+        if (settings.postBuildHook != "") {
+            Activity act(*logger, lvlInfo, actPostBuildHook,
+                fmt("running post-build-hook '%s'", settings.postBuildHook),
+                Logger::Fields{drvPath});
+            PushActivity pact(act.id);
+            auto outputPaths = drv->outputPaths();
+            std::map<std::string, std::string> hookEnvironment = getEnv();
+
+            hookEnvironment.emplace("DRV_PATH", drvPath);
+            hookEnvironment.emplace("OUT_PATHS", chomp(concatStringsSep(" ", outputPaths)));
+
+            RunOptions opts(settings.postBuildHook, {});
+            opts.environment = hookEnvironment;
+
+            struct LogSink : Sink {
+                Activity & act;
+                std::string currentLine;
+
+                LogSink(Activity & act) : act(act) { }
+
+                void operator() (const unsigned char * data, size_t len) override {
+                    for (size_t i = 0; i < len; i++) {
+                        auto c = data[i];
+
+                        if (c == '\n') {
+                            flushLine();
+                        } else {
+                            currentLine += c;
+                        }
+                    }
+                }
+
+                void flushLine() {
+                    if (settings.verboseBuild) {
+                        printError("post-build-hook: " + currentLine);
+                    } else {
+                        act.result(resPostBuildLogLine, currentLine);
+                    }
+                    currentLine.clear();
+                }
+
+                ~LogSink() {
+                    if (currentLine != "") {
+                        currentLine += '\n';
+                        flushLine();
+                    }
+                }
+            };
+            LogSink sink(act);
+
+            opts.standardOut = &sink;
+            opts.mergeStderrToStdout = true;
+            runProgram2(opts);
+        }
+
         if (buildMode == bmCheck) {
             done(BuildResult::Built);
             return;
@@ -1862,13 +1964,6 @@ void DerivationGoal::startBuilder()
     auto drvName = storePathToName(drvPath);
     tmpDir = createTempDir("", "nix-build-" + drvName, false, false, 0700);
 
-    /* In a sandbox, for determinism, always use the same temporary
-       directory. */
-#if __linux__
-    tmpDirInSandbox = useChroot ? settings.sandboxBuildDir : tmpDir;
-#else
-    tmpDirInSandbox = tmpDir;
-#endif
     chownToBuilder(tmpDir);
 
     /* Substitute output placeholders with the actual output paths. */
@@ -2072,7 +2167,7 @@ void DerivationGoal::startBuilder()
 #endif
     }
 
-    else {
+    if (needsHashRewrite()) {
 
         if (pathExists(homeDir))
             throw Error(format("directory '%1%' exists; please remove it") % homeDir);
@@ -2144,7 +2239,48 @@ void DerivationGoal::startBuilder()
     Path logFile = openLogFile();
 
     /* Create a pipe to get the output of the builder. */
-    builderOut.create();
+    //builderOut.create();
+
+    builderOut.readSide = posix_openpt(O_RDWR | O_NOCTTY);
+    if (!builderOut.readSide)
+        throw SysError("opening pseudoterminal master");
+
+    std::string slaveName(ptsname(builderOut.readSide.get()));
+
+    if (buildUser) {
+        if (chmod(slaveName.c_str(), 0600))
+            throw SysError("changing mode of pseudoterminal slave");
+
+        if (chown(slaveName.c_str(), buildUser->getUID(), 0))
+            throw SysError("changing owner of pseudoterminal slave");
+    } else {
+        if (grantpt(builderOut.readSide.get()))
+            throw SysError("granting access to pseudoterminal slave");
+    }
+
+    #if 0
+    // Mount the pt in the sandbox so that the "tty" command works.
+    // FIXME: this doesn't work with the new devpts in the sandbox.
+    if (useChroot)
+        dirsInChroot[slaveName] = {slaveName, false};
+    #endif
+
+    if (unlockpt(builderOut.readSide.get()))
+        throw SysError("unlocking pseudoterminal");
+
+    builderOut.writeSide = open(slaveName.c_str(), O_RDWR | O_NOCTTY);
+    if (!builderOut.writeSide)
+        throw SysError("opening pseudoterminal slave");
+
+    // Put the pt into raw mode to prevent \n -> \r\n translation.
+    struct termios term;
+    if (tcgetattr(builderOut.writeSide.get(), &term))
+        throw SysError("getting pseudoterminal attributes");
+
+    cfmakeraw(&term);
+
+    if (tcsetattr(builderOut.writeSide.get(), TCSANOW, &term))
+        throw SysError("putting pseudoterminal into raw mode");
 
     result.startTime = time(0);
 
@@ -2193,7 +2329,6 @@ void DerivationGoal::startBuilder()
         userNamespaceSync.create();
 
         options.allowVfork = false;
-        options.restoreMountNamespace = false;
 
         Pid helper = startProcess([&]() {
 
@@ -2218,17 +2353,37 @@ void DerivationGoal::startBuilder()
                 flags |= CLONE_NEWNET;
 
             pid_t child = clone(childEntry, stack + stackSize, flags, this);
-            if (child == -1 && errno == EINVAL)
+            if (child == -1 && errno == EINVAL) {
                 /* Fallback for Linux < 2.13 where CLONE_NEWPID and
                    CLONE_PARENT are not allowed together. */
-                child = clone(childEntry, stack + stackSize, flags & ~CLONE_NEWPID, this);
+                flags &= ~CLONE_NEWPID;
+                child = clone(childEntry, stack + stackSize, flags, this);
+            }
+            if (child == -1 && (errno == EPERM || errno == EINVAL)) {
+                /* Some distros patch Linux to not allow unpriveleged
+                 * user namespaces. If we get EPERM or EINVAL, try
+                 * without CLONE_NEWUSER and see if that works.
+                 */
+                flags &= ~CLONE_NEWUSER;
+                child = clone(childEntry, stack + stackSize, flags, this);
+            }
+            /* Otherwise exit with EPERM so we can handle this in the
+               parent. This is only done when sandbox-fallback is set
+               to true (the default). */
+            if (child == -1 && (errno == EPERM || errno == EINVAL) && settings.sandboxFallback)
+                _exit(1);
             if (child == -1) throw SysError("cloning builder process");
 
             writeFull(builderOut.writeSide.get(), std::to_string(child) + "\n");
             _exit(0);
         }, options);
 
-        if (helper.wait() != 0)
+        int res = helper.wait();
+        if (res != 0 && settings.sandboxFallback) {
+            useChroot = false;
+            initTmpDir();
+            goto fallback;
+        } else if (res != 0)
             throw Error("unable to start build process");
 
         userNamespaceSync.readSide = -1;
@@ -2259,8 +2414,8 @@ void DerivationGoal::startBuilder()
     } else
 #endif
     {
+    fallback:
         options.allowVfork = !buildUser && !drv->isBuiltin();
-        options.restoreMountNamespace = false;
         pid = startProcess([&]() {
             runChild();
         }, options);
@@ -2283,6 +2438,54 @@ void DerivationGoal::startBuilder()
 }
 
 
+void DerivationGoal::initTmpDir() {
+    /* In a sandbox, for determinism, always use the same temporary
+       directory. */
+#if __linux__
+    tmpDirInSandbox = useChroot ? settings.sandboxBuildDir : tmpDir;
+#else
+    tmpDirInSandbox = tmpDir;
+#endif
+
+    /* In non-structured mode, add all bindings specified in the
+       derivation via the environment, except those listed in the
+       passAsFile attribute. Those are passed as file names pointing
+       to temporary files containing the contents. Note that
+       passAsFile is ignored in structure mode because it's not
+       needed (attributes are not passed through the environment, so
+       there is no size constraint). */
+    if (!parsedDrv->getStructuredAttrs()) {
+
+        StringSet passAsFile = tokenizeString<StringSet>(get(drv->env, "passAsFile"));
+        int fileNr = 0;
+        for (auto & i : drv->env) {
+            if (passAsFile.find(i.first) == passAsFile.end()) {
+                env[i.first] = i.second;
+            } else {
+                string fn = ".attr-" + std::to_string(fileNr++);
+                Path p = tmpDir + "/" + fn;
+                writeFile(p, rewriteStrings(i.second, inputRewrites));
+                chownToBuilder(p);
+                env[i.first + "Path"] = tmpDirInSandbox + "/" + fn;
+            }
+        }
+
+    }
+
+    /* For convenience, set an environment pointing to the top build
+       directory. */
+    env["NIX_BUILD_TOP"] = tmpDirInSandbox;
+
+    /* Also set TMPDIR and variants to point to this directory. */
+    env["TMPDIR"] = env["TEMPDIR"] = env["TMP"] = env["TEMP"] = tmpDirInSandbox;
+
+    /* Explicitly set PWD to prevent problems with chroot builds.  In
+       particular, dietlibc cannot figure out the cwd because the
+       inode of the current directory doesn't appear in .. (because
+       getdents returns the inode of the mount point). */
+    env["PWD"] = tmpDirInSandbox;
+}
+
 void DerivationGoal::initEnv()
 {
     env.clear();
@@ -2309,43 +2512,7 @@ void DerivationGoal::initEnv()
     /* The maximum number of cores to utilize for parallel building. */
     env["NIX_BUILD_CORES"] = (format("%d") % settings.buildCores).str();
 
-    /* In non-structured mode, add all bindings specified in the
-       derivation via the environment, except those listed in the
-       passAsFile attribute. Those are passed as file names pointing
-       to temporary files containing the contents. Note that
-       passAsFile is ignored in structure mode because it's not
-       needed (attributes are not passed through the environment, so
-       there is no size constraint). */
-    if (!parsedDrv->getStructuredAttrs()) {
-
-        StringSet passAsFile = tokenizeString<StringSet>(get(drv->env, "passAsFile"));
-        int fileNr = 0;
-        for (auto & i : drv->env) {
-            if (passAsFile.find(i.first) == passAsFile.end()) {
-                env[i.first] = i.second;
-            } else {
-                string fn = ".attr-" + std::to_string(fileNr++);
-                Path p = tmpDir + "/" + fn;
-                writeFile(p, i.second);
-                chownToBuilder(p);
-                env[i.first + "Path"] = tmpDirInSandbox + "/" + fn;
-            }
-        }
-
-    }
-
-    /* For convenience, set an environment pointing to the top build
-       directory. */
-    env["NIX_BUILD_TOP"] = tmpDirInSandbox;
-
-    /* Also set TMPDIR and variants to point to this directory. */
-    env["TMPDIR"] = env["TEMPDIR"] = env["TMP"] = env["TEMP"] = tmpDirInSandbox;
-
-    /* Explicitly set PWD to prevent problems with chroot builds.  In
-       particular, dietlibc cannot figure out the cwd because the
-       inode of the current directory doesn't appear in .. (because
-       getdents returns the inode of the mount point). */
-    env["PWD"] = tmpDirInSandbox;
+    initTmpDir();
 
     /* Compatibility hack with Nix <= 0.7: if this is a fixed-output
        derivation, tell the builder, so that for instance `fetchurl'
@@ -2371,6 +2538,9 @@ void DerivationGoal::initEnv()
        may change that in the future. So tell the builder which file
        descriptor to use for that. */
     env["NIX_LOG_FD"] = "2";
+
+    /* Trigger colored output in various tools. */
+    env["TERM"] = "xterm-256color";
 }
 
 
@@ -2408,6 +2578,7 @@ void DerivationGoal::writeStructuredAttrs()
     }
 
     writeFile(tmpDir + "/.attrs.json", rewriteStrings(json.dump(), inputRewrites));
+    chownToBuilder(tmpDir + "/.attrs.json");
 
     /* As a convenience to bash scripts, write a shell file that
        maps all attributes that are representable in bash -
@@ -2415,7 +2586,7 @@ void DerivationGoal::writeStructuredAttrs()
        objects consisting entirely of those values. (So nested
        arrays or objects are not supported.) */
 
-    auto handleSimpleType = [](const nlohmann::json & value) -> std::experimental::optional<std::string> {
+    auto handleSimpleType = [](const nlohmann::json & value) -> std::optional<std::string> {
         if (value.is_string())
             return shellEscape(value);
 
@@ -2476,6 +2647,7 @@ void DerivationGoal::writeStructuredAttrs()
     }
 
     writeFile(tmpDir + "/.attrs.sh", rewriteStrings(jsonSh, inputRewrites));
+    chownToBuilder(tmpDir + "/.attrs.sh");
 }
 
 
@@ -2501,17 +2673,17 @@ void setupSeccomp()
         seccomp_release(ctx);
     });
 
-    if (settings.thisSystem == "x86_64-linux" &&
+    if (nativeSystem == "x86_64-linux" &&
         seccomp_arch_add(ctx, SCMP_ARCH_X86) != 0)
         throw SysError("unable to add 32-bit seccomp architecture");
 
-    if (settings.thisSystem == "x86_64-linux" &&
+    if (nativeSystem == "x86_64-linux" &&
         seccomp_arch_add(ctx, SCMP_ARCH_X32) != 0)
         throw SysError("unable to add X32 seccomp architecture");
 
-    if (settings.thisSystem == "aarch64-linux" &&
+    if (nativeSystem == "aarch64-linux" &&
         seccomp_arch_add(ctx, SCMP_ARCH_ARM) != 0)
-        printError("unsable to add ARM seccomp architecture; this may result in spurious build failures if running 32-bit ARM processes.");
+        printError("unable to add ARM seccomp architecture; this may result in spurious build failures if running 32-bit ARM processes");
 
     /* Prevent builders from creating setuid/setgid binaries. */
     for (int perm : { S_ISUID, S_ISGID }) {
@@ -2648,7 +2820,13 @@ void DerivationGoal::runChild()
                on. */
             if (fixedOutput) {
                 ss.push_back("/etc/resolv.conf");
-                ss.push_back("/etc/nsswitch.conf");
+
+                // Only use nss functions to resolve hosts and
+                // services. Don’t use it for anything else that may
+                // be configured for this system. This limits the
+                // potential impurities introduced in fixed outputs.
+                writeFile(chrootRootDir + "/etc/nsswitch.conf", "hosts: files dns\nservices: files\n");
+
                 ss.push_back("/etc/services");
                 ss.push_back("/etc/hosts");
                 if (pathExists("/var/run/nscd/socket"))
@@ -2874,6 +3052,10 @@ void DerivationGoal::runChild()
                 for (auto & i : missingPaths) {
                     sandboxProfile += (format("\t(subpath \"%1%\")\n") % i.c_str()).str();
                 }
+                /* Also add redirected outputs to the chroot */
+                for (auto & i : redirectedOutputs) {
+                    sandboxProfile += (format("\t(subpath \"%1%\")\n") % i.second.c_str()).str();
+                }
                 sandboxProfile += ")\n";
 
                 /* Our inputs (transitive dependencies and any impurities computed above)
@@ -3026,8 +3208,7 @@ void DerivationGoal::registerOutputs()
     InodesSeen inodesSeen;
 
     Path checkSuffix = ".check";
-    bool runDiffHook = settings.runDiffHook;
-    bool keepPreviousRound = settings.keepFailed || runDiffHook;
+    bool keepPreviousRound = settings.keepFailed || settings.runDiffHook;
 
     std::exception_ptr delayedException;
 
@@ -3052,7 +3233,9 @@ void DerivationGoal::registerOutputs()
                         throw SysError(format("moving build output '%1%' from the sandbox to the Nix store") % path);
             }
             if (buildMode != bmCheck) actualPath = worker.store.toRealPath(path);
-        } else {
+        }
+
+        if (needsHashRewrite()) {
             Path redirected = redirectedOutputs[path];
             if (buildMode == bmRepair
                 && redirectedBadOutputs.find(path) != redirectedBadOutputs.end()
@@ -3128,6 +3311,7 @@ void DerivationGoal::registerOutputs()
 
                 /* Throw an error after registering the path as
                    valid. */
+                worker.hashMismatch = true;
                 delayedException = std::make_exception_ptr(
                     BuildError("hash mismatch in fixed-output derivation '%s':\n  wanted: %s\n  got:    %s",
                         dest, h.to_string(), h2.to_string()));
@@ -3170,15 +3354,22 @@ void DerivationGoal::registerOutputs()
             if (!worker.store.isValidPath(path)) continue;
             auto info = *worker.store.queryPathInfo(path);
             if (hash.first != info.narHash) {
-                if (settings.keepFailed) {
+                worker.checkMismatch = true;
+                if (settings.runDiffHook || settings.keepFailed) {
                     Path dst = worker.store.toRealPath(path + checkSuffix);
                     deletePath(dst);
                     if (rename(actualPath.c_str(), dst.c_str()))
                         throw SysError(format("renaming '%1%' to '%2%'") % actualPath % dst);
-                    throw Error(format("derivation '%1%' may not be deterministic: output '%2%' differs from '%3%'")
+
+                    handleDiffHook(
+                        buildUser ? buildUser->getUID() : getuid(),
+                        buildUser ? buildUser->getGID() : getgid(),
+                        path, dst, drvPath, tmpDir);
+
+                    throw NotDeterministic(format("derivation '%1%' may not be deterministic: output '%2%' differs from '%3%'")
                         % drvPath % path % dst);
                 } else
-                    throw Error(format("derivation '%1%' may not be deterministic: output '%2%' differs")
+                    throw NotDeterministic(format("derivation '%1%' may not be deterministic: output '%2%' differs")
                         % drvPath % path);
             }
 
@@ -3239,16 +3430,10 @@ void DerivationGoal::registerOutputs()
                     ? fmt("output '%1%' of '%2%' differs from '%3%' from previous round", i->second.path, drvPath, prev)
                     : fmt("output '%1%' of '%2%' differs from previous round", i->second.path, drvPath);
 
-                auto diffHook = settings.diffHook;
-                if (prevExists && diffHook != "" && runDiffHook) {
-                    try {
-                        auto diff = runProgram(diffHook, true, {prev, i->second.path});
-                        if (diff != "")
-                            printError(chomp(diff));
-                    } catch (Error & error) {
-                        printError("diff hook execution failed: %s", error.what());
-                    }
-                }
+                handleDiffHook(
+                    buildUser ? buildUser->getUID() : getuid(),
+                    buildUser ? buildUser->getGID() : getgid(),
+                    prev, i->second.path, drvPath, tmpDir);
 
                 if (settings.enforceDeterminism)
                     throw NotDeterministic(msg);
@@ -3313,8 +3498,8 @@ void DerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & outputs)
         struct Checks
         {
             bool ignoreSelfRefs = false;
-            std::experimental::optional<uint64_t> maxSize, maxClosureSize;
-            std::experimental::optional<Strings> allowedReferences, allowedRequisites, disallowedReferences, disallowedRequisites;
+            std::optional<uint64_t> maxSize, maxClosureSize;
+            std::optional<Strings> allowedReferences, allowedRequisites, disallowedReferences, disallowedRequisites;
         };
 
         /* Compute the closure and closure size of some output. This
@@ -3361,7 +3546,7 @@ void DerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & outputs)
                         info.path, closureSize, *checks.maxClosureSize);
             }
 
-            auto checkRefs = [&](const std::experimental::optional<Strings> & value, bool allowed, bool recursive)
+            auto checkRefs = [&](const std::optional<Strings> & value, bool allowed, bool recursive)
             {
                 if (!value) return;
 
@@ -3415,7 +3600,7 @@ void DerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & outputs)
                     if (maxClosureSize != output->end())
                         checks.maxClosureSize = maxClosureSize->get<uint64_t>();
 
-                    auto get = [&](const std::string & name) -> std::experimental::optional<Strings> {
+                    auto get = [&](const std::string & name) -> std::optional<Strings> {
                         auto i = output->find(name);
                         if (i != output->end()) {
                             Strings res;
@@ -3885,17 +4070,6 @@ void SubstitutionGoal::tryToRun()
         return;
     }
 
-    /* If the store path is already locked (probably by a
-       DerivationGoal), then put this goal to sleep. Note: we don't
-       acquire a lock here since that breaks addToStore(), so below we
-       handle an AlreadyLocked exception from addToStore(). The check
-       here is just an optimisation to prevent having to redo a
-       download due to a locked path. */
-    if (pathIsLockedByMe(worker.store.toRealPath(storePath))) {
-        worker.waitForAWhile(shared_from_this());
-        return;
-    }
-
     maintainRunningSubstitutions = std::make_unique<MaintainCount<uint64_t>>(worker.runningSubstitutions);
     worker.updateProgress();
 
@@ -3935,12 +4109,6 @@ void SubstitutionGoal::finished()
 
     try {
         promise.get_future().get();
-    } catch (AlreadyLocked & e) {
-        /* Probably a DerivationGoal is already building this store
-           path. Sleep for a while and try again. */
-        state = &SubstitutionGoal::init;
-        worker.waitForAWhile(shared_from_this());
-        return;
     } catch (std::exception & e) {
         printError(e.what());
 
@@ -4016,6 +4184,8 @@ Worker::Worker(LocalStore & store)
     lastWokenUp = steady_time_point::min();
     permanentFailure = false;
     timedOut = false;
+    hashMismatch = false;
+    checkMismatch = false;
 }
 
 
@@ -4321,14 +4491,15 @@ void Worker::waitForInput()
         for (auto & k : fds2) {
             if (FD_ISSET(k, &fds)) {
                 ssize_t rd = read(k, buffer.data(), buffer.size());
-                if (rd == -1) {
-                    if (errno != EINTR)
-                        throw SysError(format("reading from %1%")
-                            % goal->getName());
-                } else if (rd == 0) {
+                // FIXME: is there a cleaner way to handle pt close
+                // than EIO? Is this even standard?
+                if (rd == 0 || (rd == -1 && errno == EIO)) {
                     debug(format("%1%: got EOF") % goal->getName());
                     goal->handleEOF(k);
                     j->fds.erase(k);
+                } else if (rd == -1) {
+                    if (errno != EINTR)
+                        throw SysError("%s: read failed", goal->getName());
                 } else {
                     printMsg(lvlVomit, format("%1%: read %2% bytes")
                         % goal->getName() % rd);
@@ -4375,7 +4546,29 @@ void Worker::waitForInput()
 
 unsigned int Worker::exitStatus()
 {
-    return timedOut ? 101 : (permanentFailure ? 100 : 1);
+    /*
+     * 1100100
+     *    ^^^^
+     *    |||`- timeout
+     *    ||`-- output hash mismatch
+     *    |`--- build failure
+     *    `---- not deterministic
+     */
+    unsigned int mask = 0;
+    bool buildFailure = permanentFailure || timedOut || hashMismatch;
+    if (buildFailure)
+        mask |= 0x04;  // 100
+    if (timedOut)
+        mask |= 0x01;  // 101
+    if (hashMismatch)
+        mask |= 0x02;  // 102
+    if (checkMismatch) {
+        mask |= 0x08;  // 104
+    }
+
+    if (mask)
+        mask |= 0x60;
+    return mask ? mask : 1;
 }
 
 
@@ -4413,6 +4606,11 @@ static void primeCache(Store & store, const PathSet & paths)
     PathSet willBuild, willSubstitute, unknown;
     unsigned long long downloadSize, narSize;
     store.queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize);
+
+    if (!willBuild.empty() && 0 == settings.maxBuildJobs && getMachines().empty())
+        throw Error(
+            "%d derivations need to be built, but neither local builds ('--max-jobs') "
+            "nor remote builds ('--builders') are enabled", willBuild.size());
 }
 
 

src/libstore/builtins/fetchurl.cc

@@ -64,7 +64,8 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
             try {
                 if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/';
                 auto ht = parseHashType(getAttr("outputHashAlgo"));
-                fetch(hashedMirror + printHashType(ht) + "/" + Hash(getAttr("outputHash"), ht).to_string(Base16, false));
+                auto h = Hash(getAttr("outputHash"), ht);
+                fetch(hashedMirror + printHashType(h.type) + "/" + h.to_string(Base16, false));
                 return;
             } catch (Error & e) {
                 debug(e.what());

src/libstore/derivations.cc

@@ -36,12 +36,6 @@ Path BasicDerivation::findOutput(const string & id) const
 }
 
 
-bool BasicDerivation::substitutesAllowed() const
-{
-    return get(env, "allowSubstitutes", "1") == "1";
-}
-
-
 bool BasicDerivation::isBuiltin() const
 {
     return string(builder, 0, 8) == "builtin:";

src/libstore/derivations.hh

@@ -56,8 +56,6 @@ struct BasicDerivation
        the given derivation. */
     Path findOutput(const string & id) const;
 
-    bool substitutesAllowed() const;
-
     bool isBuiltin() const;
 
     /* Return true iff this is a fixed-output derivation. */

src/libstore/download.cc

@@ -30,23 +30,7 @@ using namespace std::string_literals;
 
 namespace nix {
 
-struct DownloadSettings : Config
-{
-    Setting<bool> enableHttp2{this, true, "http2",
-        "Whether to enable HTTP/2 support."};
-
-    Setting<std::string> userAgentSuffix{this, "", "user-agent-suffix",
-        "String appended to the user agent in HTTP requests."};
-
-    Setting<size_t> httpConnections{this, 25, "http-connections",
-        "Number of parallel HTTP connections.",
-        {"binary-caches-parallel-connections"}};
-
-    Setting<unsigned long> connectTimeout{this, 0, "connect-timeout",
-        "Timeout for connecting to servers during downloads. 0 means use curl's builtin default."};
-};
-
-static DownloadSettings downloadSettings;
+DownloadSettings downloadSettings;
 
 static GlobalConfig::Register r1(&downloadSettings);
 
@@ -87,19 +71,31 @@ struct CurlDownloader : public Downloader
 
         std::string encoding;
 
+        bool acceptRanges = false;
+
+        curl_off_t writtenToSink = 0;
+
         DownloadItem(CurlDownloader & downloader,
             const DownloadRequest & request,
-            Callback<DownloadResult> callback)
+            Callback<DownloadResult> && callback)
             : downloader(downloader)
             , request(request)
             , act(*logger, lvlTalkative, actDownload,
                 fmt(request.data ? "uploading '%s'" : "downloading '%s'", request.uri),
                 {request.uri}, request.parentAct)
-            , callback(callback)
+            , callback(std::move(callback))
             , finalSink([this](const unsigned char * data, size_t len) {
-                if (this->request.dataCallback)
-                    this->request.dataCallback((char *) data, len);
-                else
+                if (this->request.dataCallback) {
+                    long httpStatus = 0;
+                    curl_easy_getinfo(req, CURLINFO_RESPONSE_CODE, &httpStatus);
+
+                    /* Only write data to the sink if this is a
+                       successful response. */
+                    if (httpStatus == 0 || httpStatus == 200 || httpStatus == 201 || httpStatus == 206) {
+                        writtenToSink += len;
+                        this->request.dataCallback((char *) data, len);
+                    }
+                } else
                     this->result.data->append((char *) data, len);
               })
         {
@@ -177,6 +173,7 @@ struct CurlDownloader : public Downloader
                 status = ss.size() >= 2 ? ss[1] : "";
                 result.data = std::make_shared<std::string>();
                 result.bodySize = 0;
+                acceptRanges = false;
                 encoding = "";
             } else {
                 auto i = line.find(':');
@@ -194,7 +191,9 @@ struct CurlDownloader : public Downloader
                             return 0;
                         }
                     } else if (name == "content-encoding")
-                        encoding = trim(string(line, i + 1));;
+                        encoding = trim(string(line, i + 1));
+                    else if (name == "accept-ranges" && toLower(trim(std::string(line, i + 1))) == "bytes")
+                        acceptRanges = true;
                 }
             }
             return realSize;
@@ -244,8 +243,6 @@ struct CurlDownloader : public Downloader
             return ((DownloadItem *) userp)->readCallback(buffer, size, nitems);
         }
 
-        long lowSpeedTimeout = 300;
-
         void init()
         {
             if (!req) req = curl_easy_init();
@@ -270,6 +267,8 @@ struct CurlDownloader : public Downloader
             #if LIBCURL_VERSION_NUM >= 0x072f00
             if (downloadSettings.enableHttp2)
                 curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2TLS);
+            else
+                curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1);
             #endif
             curl_easy_setopt(req, CURLOPT_WRITEFUNCTION, DownloadItem::writeCallbackWrapper);
             curl_easy_setopt(req, CURLOPT_WRITEDATA, this);
@@ -303,13 +302,16 @@ struct CurlDownloader : public Downloader
             curl_easy_setopt(req, CURLOPT_CONNECTTIMEOUT, downloadSettings.connectTimeout.get());
 
             curl_easy_setopt(req, CURLOPT_LOW_SPEED_LIMIT, 1L);
-            curl_easy_setopt(req, CURLOPT_LOW_SPEED_TIME, lowSpeedTimeout);
+            curl_easy_setopt(req, CURLOPT_LOW_SPEED_TIME, downloadSettings.stalledDownloadTimeout.get());
 
             /* If no file exist in the specified path, curl continues to work
                anyway as if netrc support was disabled. */
             curl_easy_setopt(req, CURLOPT_NETRC_FILE, settings.netrcFile.get().c_str());
             curl_easy_setopt(req, CURLOPT_NETRC, CURL_NETRC_OPTIONAL);
 
+            if (writtenToSink)
+                curl_easy_setopt(req, CURLOPT_RESUME_FROM_LARGE, writtenToSink);
+
             result.data = std::make_shared<std::string>();
             result.bodySize = 0;
         }
@@ -319,16 +321,21 @@ struct CurlDownloader : public Downloader
             long httpStatus = 0;
             curl_easy_getinfo(req, CURLINFO_RESPONSE_CODE, &httpStatus);
 
-            char * effectiveUrlCStr;
-            curl_easy_getinfo(req, CURLINFO_EFFECTIVE_URL, &effectiveUrlCStr);
-            if (effectiveUrlCStr)
-                result.effectiveUrl = effectiveUrlCStr;
+            char * effectiveUriCStr;
+            curl_easy_getinfo(req, CURLINFO_EFFECTIVE_URL, &effectiveUriCStr);
+            if (effectiveUriCStr)
+                result.effectiveUri = effectiveUriCStr;
 
             debug("finished %s of '%s'; curl status = %d, HTTP status = %d, body = %d bytes",
                 request.verb(), request.uri, code, httpStatus, result.bodySize);
 
-            if (decompressionSink)
-                decompressionSink->finish();
+            if (decompressionSink) {
+                try {
+                    decompressionSink->finish();
+                } catch (...) {
+                    writeException = std::current_exception();
+                }
+            }
 
             if (code == CURLE_WRITE_ERROR && result.etag == request.expectedETag) {
                 code = CURLE_OK;
@@ -339,18 +346,12 @@ struct CurlDownloader : public Downloader
                 failEx(writeException);
 
             else if (code == CURLE_OK &&
-                (httpStatus == 200 || httpStatus == 201 || httpStatus == 204 || httpStatus == 304 || httpStatus == 226 /* FTP */ || httpStatus == 0 /* other protocol */))
+                (httpStatus == 200 || httpStatus == 201 || httpStatus == 204 || httpStatus == 206 || httpStatus == 304 || httpStatus == 226 /* FTP */ || httpStatus == 0 /* other protocol */))
             {
                 result.cached = httpStatus == 304;
+                act.progress(result.bodySize, result.bodySize);
                 done = true;
-
-                try {
-                    act.progress(result.bodySize, result.bodySize);
-                    callback(std::move(result));
-                } catch (...) {
-                    done = true;
-                    callback.rethrow();
-                }
+                callback(std::move(result));
             }
 
             else {
@@ -363,9 +364,10 @@ struct CurlDownloader : public Downloader
                 } else if (httpStatus == 401 || httpStatus == 403 || httpStatus == 407) {
                     // Don't retry on authentication/authorization failures
                     err = Forbidden;
-                } else if (httpStatus >= 400 && httpStatus < 500 && httpStatus != 408) {
+                } else if (httpStatus >= 400 && httpStatus < 500 && httpStatus != 408 && httpStatus != 429) {
                     // Most 4xx errors are client errors and are probably not worth retrying:
                     //   * 408 means the server timed out waiting for us, so we try again
+                    //   * 429 means too many requests, so we retry (with a delay)
                     err = Misc;
                 } else if (httpStatus == 501 || httpStatus == 505 || httpStatus == 511) {
                     // Let's treat most 5xx (server) errors as transient, except for a handful:
@@ -389,6 +391,7 @@ struct CurlDownloader : public Downloader
                         case CURLE_SSL_CACERT_BADFILE:
                         case CURLE_TOO_MANY_REDIRECTS:
                         case CURLE_WRITE_ERROR:
+                        case CURLE_UNSUPPORTED_PROTOCOL:
                             err = Misc;
                             break;
                         default: // Shut up warnings
@@ -412,10 +415,20 @@ struct CurlDownloader : public Downloader
                             request.verb(), request.uri, curl_easy_strerror(code), code));
 
                 /* If this is a transient error, then maybe retry the
-                   download after a while. */
-                if (err == Transient && attempt < request.tries) {
+                   download after a while. If we're writing to a
+                   sink, we can only retry if the server supports
+                   ranged requests. */
+                if (err == Transient
+                    && attempt < request.tries
+                    && (!this->request.dataCallback
+                        || writtenToSink == 0
+                        || (acceptRanges && encoding.empty())))
+                {
                     int ms = request.baseRetryTimeMs * std::pow(2.0f, attempt - 1 + std::uniform_real_distribution<>(0.0, 0.5)(downloader.mt19937));
-                    printError(format("warning: %s; retrying in %d ms") % exc.what() % ms);
+                    if (writtenToSink)
+                        warn("%s; retrying from offset %d in %d ms", exc.what(), writtenToSink, ms);
+                    else
+                        warn("%s; retrying in %d ms", exc.what(), ms);
                     embargo = std::chrono::steady_clock::now() + std::chrono::milliseconds(ms);
                     downloader.enqueueItem(shared_from_this());
                 }
@@ -588,7 +601,7 @@ struct CurlDownloader : public Downloader
             workerThreadMain();
         } catch (nix::Interrupted & e) {
         } catch (std::exception & e) {
-            printError(format("unexpected error in download thread: %s") % e.what());
+            printError("unexpected error in download thread: %s", e.what());
         }
 
         {
@@ -614,6 +627,22 @@ struct CurlDownloader : public Downloader
         writeFull(wakeupPipe.writeSide.get(), " ");
     }
 
+#ifdef ENABLE_S3
+    std::tuple<std::string, std::string, Store::Params> parseS3Uri(std::string uri)
+    {
+        auto [path, params] = splitUriAndParams(uri);
+
+        auto slash = path.find('/', 5); // 5 is the length of "s3://" prefix
+            if (slash == std::string::npos)
+                throw nix::Error("bad S3 URI '%s'", path);
+
+        std::string bucketName(path, 5, slash - 5);
+        std::string key(path, slash + 1);
+
+        return {bucketName, key, params};
+    }
+#endif
+
     void enqueueDownload(const DownloadRequest & request,
         Callback<DownloadResult> callback) override
     {
@@ -622,12 +651,15 @@ struct CurlDownloader : public Downloader
             // FIXME: do this on a worker thread
             try {
 #ifdef ENABLE_S3
-                S3Helper s3Helper("", Aws::Region::US_EAST_1, "", ""); // FIXME: make configurable
-                auto slash = request.uri.find('/', 5);
-                if (slash == std::string::npos)
-                    throw nix::Error("bad S3 URI '%s'", request.uri);
-                std::string bucketName(request.uri, 5, slash - 5);
-                std::string key(request.uri, slash + 1);
+                auto [bucketName, key, params] = parseS3Uri(request.uri);
+
+                std::string profile = get(params, "profile", "");
+                std::string region = get(params, "region", Aws::Region::US_EAST_1);
+                std::string scheme = get(params, "scheme", "");
+                std::string endpoint = get(params, "endpoint", "");
+
+                S3Helper s3Helper(profile, region, scheme, endpoint);
+
                 // FIXME: implement ETag
                 auto s3Res = s3Helper.getObject(bucketName, key);
                 DownloadResult res;
@@ -642,7 +674,7 @@ struct CurlDownloader : public Downloader
             return;
         }
 
-        enqueueItem(std::make_shared<DownloadItem>(*this, request, callback));
+        enqueueItem(std::make_shared<DownloadItem>(*this, request, std::move(callback)));
     }
 };
 
@@ -771,20 +803,26 @@ void Downloader::download(DownloadRequest && request, Sink & sink)
     }
 }
 
-Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpack, string name, const Hash & expectedHash, string * effectiveUrl, int ttl)
+CachedDownloadResult Downloader::downloadCached(
+    ref<Store> store, const CachedDownloadRequest & request)
 {
-    auto url = resolveUri(url_);
+    auto url = resolveUri(request.uri);
 
+    auto name = request.name;
     if (name == "") {
         auto p = url.rfind('/');
         if (p != string::npos) name = string(url, p + 1);
     }
 
     Path expectedStorePath;
-    if (expectedHash) {
-        expectedStorePath = store->makeFixedOutputPath(unpack, expectedHash, name);
-        if (store->isValidPath(expectedStorePath))
-            return store->toRealPath(expectedStorePath);
+    if (request.expectedHash) {
+        expectedStorePath = store->makeFixedOutputPath(request.unpack, request.expectedHash, name);
+        if (store->isValidPath(expectedStorePath)) {
+            CachedDownloadResult result;
+            result.storePath = expectedStorePath;
+            result.path = store->toRealPath(expectedStorePath);
+            return result;
+        }
     }
 
     Path cacheDir = getCacheDir() + "/nix/tarballs";
@@ -803,6 +841,8 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa
 
     bool skip = false;
 
+    CachedDownloadResult result;
+
     if (pathExists(fileLink) && pathExists(dataFile)) {
         storePath = readLink(fileLink);
         store->addTempRoot(storePath);
@@ -810,10 +850,10 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa
             auto ss = tokenizeString<vector<string>>(readFile(dataFile), "\n");
             if (ss.size() >= 3 && ss[0] == url) {
                 time_t lastChecked;
-                if (string2Int(ss[2], lastChecked) && lastChecked + ttl >= time(0)) {
+                if (string2Int(ss[2], lastChecked) && (uint64_t) lastChecked + request.ttl >= (uint64_t) time(0)) {
                     skip = true;
-                    if (effectiveUrl)
-                        *effectiveUrl = url_;
+                    result.effectiveUri = request.uri;
+                    result.etag = ss[1];
                 } else if (!ss[1].empty()) {
                     debug(format("verifying previous ETag '%1%'") % ss[1]);
                     expectedETag = ss[1];
@@ -826,17 +866,17 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa
     if (!skip) {
 
         try {
-            DownloadRequest request(url);
-            request.expectedETag = expectedETag;
-            auto res = download(request);
-            if (effectiveUrl)
-                *effectiveUrl = res.effectiveUrl;
+            DownloadRequest request2(url);
+            request2.expectedETag = expectedETag;
+            auto res = download(request2);
+            result.effectiveUri = res.effectiveUri;
+            result.etag = res.etag;
 
             if (!res.cached) {
                 ValidPathInfo info;
                 StringSink sink;
                 dumpString(*res.data, sink);
-                Hash hash = hashString(expectedHash ? expectedHash.type : htSHA256, *res.data);
+                Hash hash = hashString(request.expectedHash ? request.expectedHash.type : htSHA256, *res.data);
                 info.path = store->makeFixedOutputPath(false, hash, name);
                 info.narHash = hashString(htSHA256, *sink.s);
                 info.narSize = sink.s->size();
@@ -851,11 +891,12 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa
             writeFile(dataFile, url + "\n" + res.etag + "\n" + std::to_string(time(0)) + "\n");
         } catch (DownloadError & e) {
             if (storePath.empty()) throw;
-            printError(format("warning: %1%; using cached result") % e.msg());
+            warn("warning: %s; using cached result", e.msg());
+            result.etag = expectedETag;
         }
     }
 
-    if (unpack) {
+    if (request.unpack) {
         Path unpackedLink = cacheDir + "/" + baseNameOf(storePath) + "-unpacked";
         PathLocks lock2({unpackedLink}, fmt("waiting for lock on '%1%'...", unpackedLink));
         Path unpackedStorePath;
@@ -878,14 +919,17 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa
     }
 
     if (expectedStorePath != "" && storePath != expectedStorePath) {
-        Hash gotHash = unpack
-            ? hashPath(expectedHash.type, store->toRealPath(storePath)).first
-            : hashFile(expectedHash.type, store->toRealPath(storePath));
-        throw nix::Error("hash mismatch in file downloaded from '%s':\n  wanted: %s\n  got:    %s",
-            url, expectedHash.to_string(), gotHash.to_string());
+        unsigned int statusCode = 102;
+        Hash gotHash = request.unpack
+            ? hashPath(request.expectedHash.type, store->toRealPath(storePath)).first
+            : hashFile(request.expectedHash.type, store->toRealPath(storePath));
+        throw nix::Error(statusCode, "hash mismatch in file downloaded from '%s':\n  wanted: %s\n  got:    %s",
+            url, request.expectedHash.to_string(), gotHash.to_string());
     }
 
-    return store->toRealPath(storePath);
+    result.storePath = storePath;
+    result.path = store->toRealPath(storePath);
+    return result;
 }
 
 

src/libstore/download.hh

@@ -9,13 +9,37 @@
 
 namespace nix {
 
+struct DownloadSettings : Config
+{
+    Setting<bool> enableHttp2{this, true, "http2",
+        "Whether to enable HTTP/2 support."};
+
+    Setting<std::string> userAgentSuffix{this, "", "user-agent-suffix",
+        "String appended to the user agent in HTTP requests."};
+
+    Setting<size_t> httpConnections{this, 25, "http-connections",
+        "Number of parallel HTTP connections.",
+        {"binary-caches-parallel-connections"}};
+
+    Setting<unsigned long> connectTimeout{this, 0, "connect-timeout",
+        "Timeout for connecting to servers during downloads. 0 means use curl's builtin default."};
+
+    Setting<unsigned long> stalledDownloadTimeout{this, 300, "stalled-download-timeout",
+        "Timeout (in seconds) for receiving data from servers during download. Nix cancels idle downloads after this timeout's duration."};
+
+    Setting<unsigned int> tries{this, 5, "download-attempts",
+        "How often Nix will attempt to download a file before giving up."};
+};
+
+extern DownloadSettings downloadSettings;
+
 struct DownloadRequest
 {
     std::string uri;
     std::string expectedETag;
     bool verifyTLS = true;
     bool head = false;
-    size_t tries = 5;
+    size_t tries = downloadSettings.tries;
     unsigned int baseRetryTimeMs = 250;
     ActivityId parentAct;
     bool decompress = true;
@@ -36,15 +60,39 @@ struct DownloadResult
 {
     bool cached = false;
     std::string etag;
-    std::string effectiveUrl;
+    std::string effectiveUri;
     std::shared_ptr<std::string> data;
     uint64_t bodySize = 0;
 };
 
+struct CachedDownloadRequest
+{
+    std::string uri;
+    bool unpack = false;
+    std::string name;
+    Hash expectedHash;
+    unsigned int ttl = settings.tarballTtl;
+
+    CachedDownloadRequest(const std::string & uri)
+        : uri(uri) { }
+};
+
+struct CachedDownloadResult
+{
+    // Note: 'storePath' may be different from 'path' when using a
+    // chroot store.
+    Path storePath;
+    Path path;
+    std::optional<std::string> etag;
+    std::string effectiveUri;
+};
+
 class Store;
 
 struct Downloader
 {
+    virtual ~Downloader() { }
+
     /* Enqueue a download request, returning a future to the result of
        the download. The future may throw a DownloadError
        exception. */
@@ -64,8 +112,7 @@ struct Downloader
        and is more recent than ‘tarball-ttl’ seconds. Otherwise,
        use the recorded ETag to verify if the server has a more
        recent version, and if so, download it to the Nix store. */
-    Path downloadCached(ref<Store> store, const string & uri, bool unpack, string name = "",
-        const Hash & expectedHash = Hash(), string * effectiveUri = nullptr, int ttl = settings.tarballTtl);
+    CachedDownloadResult downloadCached(ref<Store> store, const CachedDownloadRequest & request);
 
     enum Error { NotFound, Forbidden, Misc, Transient, Interrupted };
 };

src/libstore/fs-accessor.hh

@@ -19,6 +19,8 @@ public:
         uint64_t narOffset = 0; // regular files only
     };
 
+    virtual ~FSAccessor() { }
+
     virtual Stat stat(const Path & path) = 0;
 
     virtual StringSet readDirectory(const Path & path) = 0;

src/libstore/gc.cc

@@ -29,7 +29,7 @@ static string gcRootsDir = "gcroots";
    read.  To be precise: when they try to create a new temporary root
    file, they will block until the garbage collector has finished /
    yielded the GC lock. */
-int LocalStore::openGCLock(LockType lockType)
+AutoCloseFD LocalStore::openGCLock(LockType lockType)
 {
     Path fnGCLock = (format("%1%/%2%")
         % stateDir % gcLockName).str();
@@ -49,7 +49,7 @@ int LocalStore::openGCLock(LockType lockType)
        process that can open the file for reading can DoS the
        collector. */
 
-    return fdGCLock.release();
+    return fdGCLock;
 }
 
 
@@ -129,8 +129,8 @@ Path LocalFSStore::addPermRoot(const Path & _storePath,
        check if the root is in a directory in or linked from the
        gcroots directory. */
     if (settings.checkRootReachability) {
-        Roots roots = findRoots();
-        if (roots.find(gcRoot) == roots.end())
+        Roots roots = findRoots(false);
+        if (roots[storePath].count(gcRoot) == 0)
             printError(
                 format(
                     "warning: '%1%' is not in a directory where the garbage collector looks for roots; "
@@ -197,10 +197,11 @@ void LocalStore::addTempRoot(const Path & path)
 }
 
 
-std::set<std::pair<pid_t, Path>> LocalStore::readTempRoots(FDs & fds)
-{
-    std::set<std::pair<pid_t, Path>> tempRoots;
+static std::string censored = "{censored}";
 
+
+void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor)
+{
     /* Read the `temproots' directory for per-process temporary root
        files. */
     for (auto & i : readDirectory(tempRootsDir)) {
@@ -220,26 +221,22 @@ std::set<std::pair<pid_t, Path>> LocalStore::readTempRoots(FDs & fds)
         //FDPtr fd(new AutoCloseFD(openLockFile(path, false)));
         //if (*fd == -1) continue;
 
-        if (path != fnTempRoots) {
-
-            /* Try to acquire a write lock without blocking.  This can
-               only succeed if the owning process has died.  In that case
-               we don't care about its temporary roots. */
-            if (lockFile(fd->get(), ltWrite, false)) {
-                printError(format("removing stale temporary roots file '%1%'") % path);
-                unlink(path.c_str());
-                writeFull(fd->get(), "d");
-                continue;
-            }
-
-            /* Acquire a read lock.  This will prevent the owning process
-               from upgrading to a write lock, therefore it will block in
-               addTempRoot(). */
-            debug(format("waiting for read lock on '%1%'") % path);
-            lockFile(fd->get(), ltRead, true);
-
+        /* Try to acquire a write lock without blocking.  This can
+           only succeed if the owning process has died.  In that case
+           we don't care about its temporary roots. */
+        if (lockFile(fd->get(), ltWrite, false)) {
+            printError(format("removing stale temporary roots file '%1%'") % path);
+            unlink(path.c_str());
+            writeFull(fd->get(), "d");
+            continue;
         }
 
+        /* Acquire a read lock.  This will prevent the owning process
+           from upgrading to a write lock, therefore it will block in
+           addTempRoot(). */
+        debug(format("waiting for read lock on '%1%'") % path);
+        lockFile(fd->get(), ltRead, true);
+
         /* Read the entire file. */
         string contents = readFile(fd->get());
 
@@ -250,14 +247,12 @@ std::set<std::pair<pid_t, Path>> LocalStore::readTempRoots(FDs & fds)
             Path root(contents, pos, end - pos);
             debug("got temporary root '%s'", root);
             assertStorePath(root);
-            tempRoots.emplace(pid, root);
+            tempRoots[root].emplace(censor ? censored : fmt("{temp:%d}", pid));
             pos = end + 1;
         }
 
         fds.push_back(fd); /* keep open */
     }
-
-    return tempRoots;
 }
 
 
@@ -266,7 +261,7 @@ void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots)
     auto foundRoot = [&](const Path & path, const Path & target) {
         Path storePath = toStorePath(target);
         if (isStorePath(storePath) && isValidPath(storePath))
-            roots[path] = storePath;
+            roots[storePath].emplace(path);
         else
             printInfo(format("skipping invalid root from '%1%' to '%2%'") % path % storePath);
     };
@@ -306,7 +301,7 @@ void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots)
         else if (type == DT_REG) {
             Path storePath = storeDir + "/" + baseNameOf(path);
             if (isStorePath(storePath) && isValidPath(storePath))
-                roots[path] = storePath;
+                roots[storePath].emplace(path);
         }
 
     }
@@ -321,44 +316,31 @@ void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots)
 }
 
 
-Roots LocalStore::findRootsNoTemp()
+void LocalStore::findRootsNoTemp(Roots & roots, bool censor)
 {
-    Roots roots;
-
     /* Process direct roots in {gcroots,profiles}. */
     findRoots(stateDir + "/" + gcRootsDir, DT_UNKNOWN, roots);
     findRoots(stateDir + "/profiles", DT_UNKNOWN, roots);
 
-    /* Add additional roots returned by the program specified by the
-       NIX_ROOT_FINDER environment variable.  This is typically used
-       to add running programs to the set of roots (to prevent them
-       from being garbage collected). */
-    size_t n = 0;
-    for (auto & root : findRuntimeRoots())
-        roots[fmt("{memory:%d}", n++)] = root;
-
-    return roots;
+    /* Add additional roots returned by different platforms-specific
+       heuristics.  This is typically used to add running programs to
+       the set of roots (to prevent them from being garbage collected). */
+    findRuntimeRoots(roots, censor);
 }
 
 
-Roots LocalStore::findRoots()
+Roots LocalStore::findRoots(bool censor)
 {
-    Roots roots = findRootsNoTemp();
+    Roots roots;
+    findRootsNoTemp(roots, censor);
 
     FDs fds;
-    pid_t prev = -1;
-    size_t n = 0;
-    for (auto & root : readTempRoots(fds)) {
-        if (prev != root.first) n = 0;
-        prev = root.first;
-        roots[fmt("{temp:%d:%d}", root.first, n++)] = root.second;
-    }
+    findTempRoots(fds, roots, censor);
 
     return roots;
 }
 
-
-static void readProcLink(const string & file, StringSet & paths)
+static void readProcLink(const string & file, Roots & roots)
 {
     /* 64 is the starting buffer size gnu readlink uses... */
     auto bufsiz = ssize_t{64};
@@ -377,8 +359,8 @@ try_again:
         goto try_again;
     }
     if (res > 0 && buf[0] == '/')
-        paths.emplace(static_cast<char *>(buf), res);
-    return;
+        roots[std::string(static_cast<char *>(buf), res)]
+            .emplace(file);
 }
 
 static string quoteRegexChars(const string & raw)
@@ -387,20 +369,20 @@ static string quoteRegexChars(const string & raw)
     return std::regex_replace(raw, specialRegex, R"(\$&)");
 }
 
-static void readFileRoots(const char * path, StringSet & paths)
+static void readFileRoots(const char * path, Roots & roots)
 {
     try {
-        paths.emplace(readFile(path));
+        roots[readFile(path)].emplace(path);
     } catch (SysError & e) {
         if (e.errNo != ENOENT && e.errNo != EACCES)
             throw;
     }
 }
 
-PathSet LocalStore::findRuntimeRoots()
+void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
 {
-    PathSet roots;
-    StringSet paths;
+    Roots unchecked;
+
     auto procDir = AutoCloseDir{opendir("/proc")};
     if (procDir) {
         struct dirent * ent;
@@ -410,10 +392,10 @@ PathSet LocalStore::findRuntimeRoots()
         while (errno = 0, ent = readdir(procDir.get())) {
             checkInterrupt();
             if (std::regex_match(ent->d_name, digitsRegex)) {
-                readProcLink((format("/proc/%1%/exe") % ent->d_name).str(), paths);
-                readProcLink((format("/proc/%1%/cwd") % ent->d_name).str(), paths);
+                readProcLink(fmt("/proc/%s/exe" ,ent->d_name), unchecked);
+                readProcLink(fmt("/proc/%s/cwd", ent->d_name), unchecked);
 
-                auto fdStr = (format("/proc/%1%/fd") % ent->d_name).str();
+                auto fdStr = fmt("/proc/%s/fd", ent->d_name);
                 auto fdDir = AutoCloseDir(opendir(fdStr.c_str()));
                 if (!fdDir) {
                     if (errno == ENOENT || errno == EACCES)
@@ -422,9 +404,8 @@ PathSet LocalStore::findRuntimeRoots()
                 }
                 struct dirent * fd_ent;
                 while (errno = 0, fd_ent = readdir(fdDir.get())) {
-                    if (fd_ent->d_name[0] != '.') {
-                        readProcLink((format("%1%/%2%") % fdStr % fd_ent->d_name).str(), paths);
-                    }
+                    if (fd_ent->d_name[0] != '.')
+                        readProcLink(fmt("%s/%s", fdStr, fd_ent->d_name), unchecked);
                 }
                 if (errno) {
                     if (errno == ESRCH)
@@ -434,18 +415,19 @@ PathSet LocalStore::findRuntimeRoots()
                 fdDir.reset();
 
                 try {
-                    auto mapLines =
-                        tokenizeString<std::vector<string>>(readFile((format("/proc/%1%/maps") % ent->d_name).str(), true), "\n");
-                    for (const auto& line : mapLines) {
+                    auto mapFile = fmt("/proc/%s/maps", ent->d_name);
+                    auto mapLines = tokenizeString<std::vector<string>>(readFile(mapFile, true), "\n");
+                    for (const auto & line : mapLines) {
                         auto match = std::smatch{};
                         if (std::regex_match(line, match, mapRegex))
-                            paths.emplace(match[1]);
+                            unchecked[match[1]].emplace(mapFile);
                     }
 
-                    auto envString = readFile((format("/proc/%1%/environ") % ent->d_name).str(), true);
+                    auto envFile = fmt("/proc/%s/environ", ent->d_name);
+                    auto envString = readFile(envFile, true);
                     auto env_end = std::sregex_iterator{};
                     for (auto i = std::sregex_iterator{envString.begin(), envString.end(), storePathRegex}; i != env_end; ++i)
-                        paths.emplace(i->str());
+                        unchecked[i->str()].emplace(envFile);
                 } catch (SysError & e) {
                     if (errno == ENOENT || errno == EACCES || errno == ESRCH)
                         continue;
@@ -458,36 +440,43 @@ PathSet LocalStore::findRuntimeRoots()
     }
 
 #if !defined(__linux__)
-    try {
-        std::regex lsofRegex(R"(^n(/.*)$)");
-        auto lsofLines =
-            tokenizeString<std::vector<string>>(runProgram(LSOF, true, { "-n", "-w", "-F", "n" }), "\n");
-        for (const auto & line : lsofLines) {
-            std::smatch match;
-            if (std::regex_match(line, match, lsofRegex))
-                paths.emplace(match[1]);
+    // lsof is really slow on OS X. This actually causes the gc-concurrent.sh test to fail.
+    // See: https://github.com/NixOS/nix/issues/3011
+    // Because of this we disable lsof when running the tests.
+    if (getEnv("_NIX_TEST_NO_LSOF") == "") {
+        try {
+            std::regex lsofRegex(R"(^n(/.*)$)");
+            auto lsofLines =
+                tokenizeString<std::vector<string>>(runProgram(LSOF, true, { "-n", "-w", "-F", "n" }), "\n");
+            for (const auto & line : lsofLines) {
+                std::smatch match;
+                if (std::regex_match(line, match, lsofRegex))
+                    unchecked[match[1]].emplace("{lsof}");
+            }
+        } catch (ExecError & e) {
+            /* lsof not installed, lsof failed */
         }
-    } catch (ExecError & e) {
-        /* lsof not installed, lsof failed */
     }
 #endif
 
 #if defined(__linux__)
-    readFileRoots("/proc/sys/kernel/modprobe", paths);
-    readFileRoots("/proc/sys/kernel/fbsplash", paths);
-    readFileRoots("/proc/sys/kernel/poweroff_cmd", paths);
+    readFileRoots("/proc/sys/kernel/modprobe", unchecked);
+    readFileRoots("/proc/sys/kernel/fbsplash", unchecked);
+    readFileRoots("/proc/sys/kernel/poweroff_cmd", unchecked);
 #endif
 
-    for (auto & i : paths)
-        if (isInStore(i)) {
-            Path path = toStorePath(i);
-            if (roots.find(path) == roots.end() && isStorePath(path) && isValidPath(path)) {
+    for (auto & [target, links] : unchecked) {
+        if (isInStore(target)) {
+            Path path = toStorePath(target);
+            if (isStorePath(path) && isValidPath(path)) {
                 debug(format("got additional root '%1%'") % path);
-                roots.insert(path);
+                if (censor)
+                    roots[path].insert(censored);
+                else
+                    roots[path].insert(links.begin(), links.end());
             }
         }
-
-    return roots;
+    }
 }
 
 
@@ -701,9 +690,8 @@ void LocalStore::removeUnusedLinks(const GCState & state)
             throw SysError(format("statting '%1%'") % path);
 
         if (st.st_nlink != 1) {
-            unsigned long long size = st.st_blocks * 512ULL;
-            actualSize += size;
-            unsharedSize += (st.st_nlink - 1) * size;
+            actualSize += st.st_size;
+            unsharedSize += (st.st_nlink - 1) * st.st_size;
             continue;
         }
 
@@ -712,7 +700,7 @@ void LocalStore::removeUnusedLinks(const GCState & state)
         if (unlink(path.c_str()) == -1)
             throw SysError(format("deleting '%1%'") % path);
 
-        state.results.bytesFreed += st.st_blocks * 512ULL;
+        state.results.bytesFreed += st.st_size;
     }
 
     struct stat st;
@@ -754,16 +742,20 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
     /* Find the roots.  Since we've grabbed the GC lock, the set of
        permanent roots cannot increase now. */
     printError(format("finding garbage collector roots..."));
-    Roots rootMap = options.ignoreLiveness ? Roots() : findRootsNoTemp();
+    Roots rootMap;
+    if (!options.ignoreLiveness)
+        findRootsNoTemp(rootMap, true);
 
-    for (auto & i : rootMap) state.roots.insert(i.second);
+    for (auto & i : rootMap) state.roots.insert(i.first);
 
     /* Read the temporary roots.  This acquires read locks on all
        per-process temporary root files.  So after this point no paths
        can be added to the set of temporary roots. */
     FDs fds;
-    for (auto & root : readTempRoots(fds))
-        state.tempRoots.insert(root.second);
+    Roots tempRoots;
+    findTempRoots(fds, tempRoots, true);
+    for (auto & root : tempRoots)
+        state.tempRoots.insert(root.first);
     state.roots.insert(state.tempRoots.begin(), state.tempRoots.end());
 
     /* After this point the set of roots or temporary roots cannot
@@ -874,7 +866,12 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
 
 void LocalStore::autoGC(bool sync)
 {
-    auto getAvail = [this]() {
+    static auto fakeFreeSpaceFile = getEnv("_NIX_TEST_FREE_SPACE_FILE", "");
+
+    auto getAvail = [this]() -> uint64_t {
+        if (!fakeFreeSpaceFile.empty())
+            return std::stoll(readFile(fakeFreeSpaceFile));
+
         struct statvfs st;
         if (statvfs(realStoreDir.c_str(), &st))
             throw SysError("getting filesystem info about '%s'", realStoreDir);
@@ -895,7 +892,7 @@ void LocalStore::autoGC(bool sync)
 
         auto now = std::chrono::steady_clock::now();
 
-        if (now < state->lastGCCheck + std::chrono::seconds(5)) return;
+        if (now < state->lastGCCheck + std::chrono::seconds(settings.minFreeCheckInterval)) return;
 
         auto avail = getAvail();
 
@@ -922,11 +919,11 @@ void LocalStore::autoGC(bool sync)
                     promise.set_value();
                 });
 
-                printInfo("running auto-GC to free %d bytes", settings.maxFree - avail);
-
                 GCOptions options;
                 options.maxFreed = settings.maxFree - avail;
 
+                printInfo("running auto-GC to free %d bytes", options.maxFreed);
+
                 GCResults results;
 
                 collectGarbage(options, results);

src/libstore/globals.hh

@@ -209,6 +209,9 @@ public:
         "The paths to make available inside the build sandbox.",
         {"build-chroot-dirs", "build-sandbox-paths"}};
 
+    Setting<bool> sandboxFallback{this, true, "sandbox-fallback",
+        "Whether to disable sandboxing when the kernel doesn't allow it."};
+
     Setting<PathSet> extraSandboxPaths{this, {}, "extra-sandbox-paths",
         "Additional paths to make available inside the build sandbox.",
         {"build-extra-chroot-dirs", "build-extra-sandbox-paths"}};
@@ -255,7 +258,7 @@ public:
         "Secret keys with which to sign local builds."};
 
     Setting<unsigned int> tarballTtl{this, 60 * 60, "tarball-ttl",
-        "How soon to expire files fetched by builtins.fetchTarball and builtins.fetchurl."};
+        "How long downloaded files are considered up-to-date."};
 
     Setting<bool> requireSigs{this, true, "require-sigs",
         "Whether to check that any non-content-addressed path added to the "
@@ -315,6 +318,9 @@ public:
         "pre-build-hook",
         "A program to run just before a build to set derivation-specific build settings."};
 
+    Setting<std::string> postBuildHook{this, "", "post-build-hook",
+        "A program to run just after each successful build."};
+
     Setting<std::string> netrcFile{this, fmt("%s/%s", nixConfDir, "netrc"), "netrc-file",
         "Path to the netrc file used to obtain usernames/passwords for downloads."};
 
@@ -342,6 +348,9 @@ public:
     Setting<uint64_t> maxFree{this, std::numeric_limits<uint64_t>::max(), "max-free",
         "Stop deleting garbage when free disk space is above the specified amount."};
 
+    Setting<uint64_t> minFreeCheckInterval{this, 5, "min-free-check-interval",
+        "Number of seconds between checking free disk space."};
+
     Setting<Paths> pluginFiles{this, {}, "plugin-files",
         "Plugins to dynamically load at nix initialization time."};
 };

src/libstore/http-binary-cache-store.cc

@@ -84,7 +84,6 @@ protected:
         try {
             DownloadRequest request(cacheUri + "/" + path);
             request.head = true;
-            request.tries = 5;
             getDownloader()->download(request);
             return true;
         } catch (DownloadError & e) {
@@ -114,7 +113,6 @@ protected:
     DownloadRequest makeRequest(const std::string & path)
     {
         DownloadRequest request(cacheUri + "/" + path);
-        request.tries = 8;
         return request;
     }
 
@@ -133,23 +131,25 @@ protected:
     }
 
     void getFile(const std::string & path,
-        Callback<std::shared_ptr<std::string>> callback) override
+        Callback<std::shared_ptr<std::string>> callback) noexcept override
     {
         checkEnabled();
 
         auto request(makeRequest(path));
 
+        auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback));
+
         getDownloader()->enqueueDownload(request,
-            {[callback, this](std::future<DownloadResult> result) {
+            {[callbackPtr, this](std::future<DownloadResult> result) {
                 try {
-                    callback(result.get().data);
+                    (*callbackPtr)(result.get().data);
                 } catch (DownloadError & e) {
                     if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden)
-                        return callback(std::shared_ptr<std::string>());
+                        return (*callbackPtr)(std::shared_ptr<std::string>());
                     maybeDisable();
-                    callback.rethrow();
+                    callbackPtr->rethrow();
                 } catch (...) {
-                    callback.rethrow();
+                    callbackPtr->rethrow();
                 }
             }});
     }

src/libstore/legacy-ssh-store.cc

@@ -88,7 +88,7 @@ struct LegacySSHStore : public Store
     }
 
     void queryPathInfoUncached(const Path & path,
-        Callback<std::shared_ptr<ValidPathInfo>> callback) override
+        Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept override
     {
         try {
             auto conn(connections->get());
@@ -187,28 +187,17 @@ struct LegacySSHStore : public Store
         copyNAR(conn->from, sink);
     }
 
-    PathSet queryAllValidPaths() override { unsupported(); }
-
-    void queryReferrers(const Path & path, PathSet & referrers) override
-    { unsupported(); }
-
-    PathSet queryDerivationOutputs(const Path & path) override
-    { unsupported(); }
-
-    StringSet queryDerivationOutputNames(const Path & path) override
-    { unsupported(); }
-
     Path queryPathFromHashPart(const string & hashPart) override
-    { unsupported(); }
+    { unsupported("queryPathFromHashPart"); }
 
     Path addToStore(const string & name, const Path & srcPath,
         bool recursive, HashType hashAlgo,
         PathFilter & filter, RepairFlag repair) override
-    { unsupported(); }
+    { unsupported("addToStore"); }
 
     Path addTextToStore(const string & name, const string & s,
         const PathSet & references, RepairFlag repair) override
-    { unsupported(); }
+    { unsupported("addTextToStore"); }
 
     BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv,
         BuildMode buildMode) override
@@ -242,25 +231,7 @@ struct LegacySSHStore : public Store
     }
 
     void ensurePath(const Path & path) override
-    { unsupported(); }
-
-    void addTempRoot(const Path & path) override
-    { unsupported(); }
-
-    void addIndirectRoot(const Path & path) override
-    { unsupported(); }
-
-    Roots findRoots() override
-    { unsupported(); }
-
-    void collectGarbage(const GCOptions & options, GCResults & results) override
-    { unsupported(); }
-
-    ref<FSAccessor> getFSAccessor() override
-    { unsupported(); }
-
-    void addSignatures(const Path & storePath, const StringSet & sigs) override
-    { unsupported(); }
+    { unsupported("ensurePath"); }
 
     void computeFSClosure(const PathSet & paths,
         PathSet & out, bool flipDirection = false,

src/libstore/local-store.cc

@@ -70,15 +70,17 @@ LocalStore::LocalStore(const Params & params)
         createSymlink(profilesDir, gcRootsDir + "/profiles");
     }
 
+    for (auto & perUserDir : {profilesDir + "/per-user", gcRootsDir + "/per-user"}) {
+        createDirs(perUserDir);
+        if (chmod(perUserDir.c_str(), 0755) == -1)
+            throw SysError("could not set permissions on '%s' to 755", perUserDir);
+    }
+
+    createUser(getUserName(), getuid());
+
     /* Optionally, create directories and set permissions for a
        multi-user install. */
     if (getuid() == 0 && settings.buildUsersGroup != "") {
-
-        Path perUserDir = profilesDir + "/per-user";
-        createDirs(perUserDir);
-        if (chmod(perUserDir.c_str(), 01777) == -1)
-            throw SysError(format("could not set permissions on '%1%' to 1777") % perUserDir);
-
         mode_t perm = 01775;
 
         struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str());
@@ -366,8 +368,6 @@ void LocalStore::makeStoreWritable()
         throw SysError("getting info about the Nix store mount point");
 
     if (stat.f_flag & ST_RDONLY) {
-        saveMountNamespace();
-
         if (unshare(CLONE_NEWNS) == -1)
             throw SysError("setting up a private mount namespace");
 
@@ -631,7 +631,7 @@ uint64_t LocalStore::addValidPath(State & state,
 
 
 void LocalStore::queryPathInfoUncached(const Path & path,
-    Callback<std::shared_ptr<ValidPathInfo>> callback)
+    Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept
 {
     try {
         auto info = std::make_shared<ValidPathInfo>();
@@ -881,8 +881,8 @@ void LocalStore::querySubstitutablePathInfos(const PathSet & paths,
                     info->references,
                     narInfo ? narInfo->fileSize : 0,
                     info->narSize};
-            } catch (InvalidPath) {
-            } catch (SubstituterDisabled) {
+            } catch (InvalidPath &) {
+            } catch (SubstituterDisabled &) {
             } catch (Error & e) {
                 if (settings.tryFallback)
                     printError(e.what());
@@ -1212,7 +1212,8 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
 
     bool errors = false;
 
-    /* Acquire the global GC lock to prevent a garbage collection. */
+    /* Acquire the global GC lock to get a consistent snapshot of
+       existing and valid paths. */
     AutoCloseFD fdGCLock = openGCLock(ltWrite);
 
     PathSet store;
@@ -1223,13 +1224,11 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
 
     PathSet validPaths2 = queryAllValidPaths(), validPaths, done;
 
+    fdGCLock = -1;
+
     for (auto & i : validPaths2)
         verifyPath(i, store, done, validPaths, repair, errors);
 
-    /* Release the GC lock so that checking content hashes (which can
-       take ages) doesn't block the GC or builds. */
-    fdGCLock = -1;
-
     /* Optionally, check the content hashes (slow). */
     if (checkContents) {
         printInfo("checking hashes...");
@@ -1436,4 +1435,19 @@ void LocalStore::signPathInfo(ValidPathInfo & info)
 }
 
 
+void LocalStore::createUser(const std::string & userName, uid_t userId)
+{
+    for (auto & dir : {
+        fmt("%s/profiles/per-user/%s", stateDir, userName),
+        fmt("%s/gcroots/per-user/%s", stateDir, userName)
+    }) {
+        createDirs(dir);
+        if (chmod(dir.c_str(), 0755) == -1)
+            throw SysError("changing permissions of directory '%s'", dir);
+        if (chown(dir.c_str(), userId, getgid()) == -1)
+            throw SysError("changing owner of directory '%s'", dir);
+    }
+}
+
+
 }

src/libstore/local-store.hh

@@ -127,7 +127,7 @@ public:
     PathSet queryAllValidPaths() override;
 
     void queryPathInfoUncached(const Path & path,
-        Callback<std::shared_ptr<ValidPathInfo>> callback) override;
+        Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept override;
 
     void queryReferrers(const Path & path, PathSet & referrers) override;
 
@@ -180,11 +180,11 @@ private:
     typedef std::shared_ptr<AutoCloseFD> FDPtr;
     typedef list<FDPtr> FDs;
 
-    std::set<std::pair<pid_t, Path>> readTempRoots(FDs & fds);
+    void findTempRoots(FDs & fds, Roots & roots, bool censor);
 
 public:
 
-    Roots findRoots() override;
+    Roots findRoots(bool censor) override;
 
     void collectGarbage(const GCOptions & options, GCResults & results) override;
 
@@ -263,13 +263,13 @@ private:
     bool isActiveTempFile(const GCState & state,
         const Path & path, const string & suffix);
 
-    int openGCLock(LockType lockType);
+    AutoCloseFD openGCLock(LockType lockType);
 
     void findRoots(const Path & path, unsigned char type, Roots & roots);
 
-    Roots findRootsNoTemp();
+    void findRootsNoTemp(Roots & roots, bool censor);
 
-    PathSet findRuntimeRoots();
+    void findRuntimeRoots(Roots & roots, bool censor);
 
     void removeUnusedLinks(const GCState & state);
 
@@ -293,6 +293,8 @@ private:
 
     Path getRealStoreDir() override { return realStoreDir; }
 
+    void createUser(const std::string & userName, uid_t userId) override;
+
     friend class DerivationGoal;
     friend class SubstitutionGoal;
 };

src/libstore/local.mk

@@ -39,9 +39,12 @@ libstore_CXXFLAGS = \
  -DNIX_LIBEXEC_DIR=\"$(libexecdir)\" \
  -DNIX_BIN_DIR=\"$(bindir)\" \
  -DNIX_MAN_DIR=\"$(mandir)\" \
- -DSANDBOX_SHELL="\"$(sandbox_shell)\"" \
  -DLSOF=\"$(lsof)\"
 
+ifneq ($(sandbox_shell),)
+libstore_CXXFLAGS += -DSANDBOX_SHELL="\"$(sandbox_shell)\""
+endif
+
 $(d)/local-store.cc: $(d)/schema.sql.gen.hh
 
 $(d)/build.cc:

src/libstore/machines.cc

@@ -89,10 +89,11 @@ void parseMachines(const std::string & s, Machines & machines)
 
 Machines getMachines()
 {
-    Machines machines;
-
-    parseMachines(settings.builders, machines);
-
+    static auto machines = [&]() {
+        Machines machines;
+        parseMachines(settings.builders, machines);
+        return machines;
+    }();
     return machines;
 }
 

src/libstore/misc.cc

@@ -1,4 +1,5 @@
 #include "derivations.hh"
+#include "parsed-derivations.hh"
 #include "globals.hh"
 #include "local-store.hh"
 #include "store-api.hh"
@@ -189,6 +190,7 @@ void Store::queryMissing(const PathSet & targets,
             }
 
             Derivation drv = derivationFromPath(i2.first);
+            ParsedDerivation parsedDrv(i2.first, drv);
 
             PathSet invalid;
             for (auto & j : drv.outputs)
@@ -197,7 +199,7 @@ void Store::queryMissing(const PathSet & targets,
                     invalid.insert(j.second.path);
             if (invalid.empty()) return;
 
-            if (settings.useSubstitutes && drv.substitutesAllowed()) {
+            if (settings.useSubstitutes && parsedDrv.substitutesAllowed()) {
                 auto drvState = make_ref<Sync<DrvState>>(DrvState(invalid.size()));
                 for (auto & output : invalid)
                     pool.enqueue(std::bind(checkOutput, i2.first, make_ref<Derivation>(drv), output, drvState));

src/libstore/nix-store.pc.in

@@ -6,4 +6,4 @@ Name: Nix
 Description: Nix Package Manager
 Version: @PACKAGE_VERSION@
 Libs: -L${libdir} -lnixstore -lnixutil
-Cflags: -I${includedir}/nix -std=c++14
+Cflags: -I${includedir}/nix -std=c++17

src/libstore/parsed-derivations.cc

@@ -16,7 +16,7 @@ ParsedDerivation::ParsedDerivation(const Path & drvPath, BasicDerivation & drv)
     }
 }
 
-std::experimental::optional<std::string> ParsedDerivation::getStringAttr(const std::string & name) const
+std::optional<std::string> ParsedDerivation::getStringAttr(const std::string & name) const
 {
     if (structuredAttrs) {
         auto i = structuredAttrs->find(name);
@@ -56,7 +56,7 @@ bool ParsedDerivation::getBoolAttr(const std::string & name, bool def) const
     }
 }
 
-std::experimental::optional<Strings> ParsedDerivation::getStringsAttr(const std::string & name) const
+std::optional<Strings> ParsedDerivation::getStringsAttr(const std::string & name) const
 {
     if (structuredAttrs) {
         auto i = structuredAttrs->find(name);
@@ -108,4 +108,9 @@ bool ParsedDerivation::willBuildLocally() const
     return getBoolAttr("preferLocalBuild") && canBuildLocally();
 }
 
+bool ParsedDerivation::substitutesAllowed() const
+{
+    return getBoolAttr("allowSubstitutes", true);
+}
+
 }

src/libstore/parsed-derivations.hh

@@ -8,28 +8,30 @@ class ParsedDerivation
 {
     Path drvPath;
     BasicDerivation & drv;
-    std::experimental::optional<nlohmann::json> structuredAttrs;
+    std::optional<nlohmann::json> structuredAttrs;
 
 public:
 
     ParsedDerivation(const Path & drvPath, BasicDerivation & drv);
 
-    const std::experimental::optional<nlohmann::json> & getStructuredAttrs() const
+    const std::optional<nlohmann::json> & getStructuredAttrs() const
     {
         return structuredAttrs;
     }
 
-    std::experimental::optional<std::string> getStringAttr(const std::string & name) const;
+    std::optional<std::string> getStringAttr(const std::string & name) const;
 
     bool getBoolAttr(const std::string & name, bool def = false) const;
 
-    std::experimental::optional<Strings> getStringsAttr(const std::string & name) const;
+    std::optional<Strings> getStringsAttr(const std::string & name) const;
 
     StringSet getRequiredSystemFeatures() const;
 
     bool canBuildLocally() const;
 
     bool willBuildLocally() const;
+
+    bool substitutesAllowed() const;
 };
 
 }

src/libstore/pathlocks.cc

@@ -5,9 +5,10 @@
 #include <cerrno>
 #include <cstdlib>
 
+#include <fcntl.h>
 #include <sys/types.h>
 #include <sys/stat.h>
-#include <fcntl.h>
+#include <sys/file.h>
 
 
 namespace nix {
@@ -40,17 +41,14 @@ void deleteLockFile(const Path & path, int fd)
 
 bool lockFile(int fd, LockType lockType, bool wait)
 {
-    struct flock lock;
-    if (lockType == ltRead) lock.l_type = F_RDLCK;
-    else if (lockType == ltWrite) lock.l_type = F_WRLCK;
-    else if (lockType == ltNone) lock.l_type = F_UNLCK;
+    int type;
+    if (lockType == ltRead) type = LOCK_SH;
+    else if (lockType == ltWrite) type = LOCK_EX;
+    else if (lockType == ltNone) type = LOCK_UN;
     else abort();
-    lock.l_whence = SEEK_SET;
-    lock.l_start = 0;
-    lock.l_len = 0; /* entire file */
 
     if (wait) {
-        while (fcntl(fd, F_SETLKW, &lock) != 0) {
+        while (flock(fd, type) != 0) {
             checkInterrupt();
             if (errno != EINTR)
                 throw SysError(format("acquiring/releasing lock"));
@@ -58,9 +56,9 @@ bool lockFile(int fd, LockType lockType, bool wait)
                 return false;
         }
     } else {
-        while (fcntl(fd, F_SETLK, &lock) != 0) {
+        while (flock(fd, type | LOCK_NB) != 0) {
             checkInterrupt();
-            if (errno == EACCES || errno == EAGAIN) return false;
+            if (errno == EWOULDBLOCK) return false;
             if (errno != EINTR)
                 throw SysError(format("acquiring/releasing lock"));
         }
@@ -70,14 +68,6 @@ bool lockFile(int fd, LockType lockType, bool wait)
 }
 
 
-/* This enables us to check whether are not already holding a lock on
-   a file ourselves.  POSIX locks (fcntl) suck in this respect: if we
-   close a descriptor, the previous lock will be closed as well.  And
-   there is no way to query whether we already have a lock (F_GETLK
-   only works on locks held by other processes). */
-static Sync<StringSet> lockedPaths_;
-
-
 PathLocks::PathLocks()
     : deletePaths(false)
 {
@@ -91,7 +81,7 @@ PathLocks::PathLocks(const PathSet & paths, const string & waitMsg)
 }
 
 
-bool PathLocks::lockPaths(const PathSet & _paths,
+bool PathLocks::lockPaths(const PathSet & paths,
     const string & waitMsg, bool wait)
 {
     assert(fds.empty());
@@ -99,75 +89,54 @@ bool PathLocks::lockPaths(const PathSet & _paths,
     /* Note that `fds' is built incrementally so that the destructor
        will only release those locks that we have already acquired. */
 
-    /* Sort the paths.  This assures that locks are always acquired in
-       the same order, thus preventing deadlocks. */
-    Paths paths(_paths.begin(), _paths.end());
-    paths.sort();
-
-    /* Acquire the lock for each path. */
+    /* Acquire the lock for each path in sorted order. This ensures
+       that locks are always acquired in the same order, thus
+       preventing deadlocks. */
     for (auto & path : paths) {
         checkInterrupt();
         Path lockPath = path + ".lock";
 
         debug(format("locking path '%1%'") % path);
 
-        {
-            auto lockedPaths(lockedPaths_.lock());
-            if (lockedPaths->count(lockPath)) {
-                if (!wait) return false;
-                throw AlreadyLocked("deadlock: trying to re-acquire self-held lock '%s'", lockPath);
-            }
-            lockedPaths->insert(lockPath);
-        }
+        AutoCloseFD fd;
 
-        try {
+        while (1) {
 
-            AutoCloseFD fd;
+            /* Open/create the lock file. */
+            fd = openLockFile(lockPath, true);
 
-            while (1) {
-
-                /* Open/create the lock file. */
-                fd = openLockFile(lockPath, true);
-
-                /* Acquire an exclusive lock. */
-                if (!lockFile(fd.get(), ltWrite, false)) {
-                    if (wait) {
-                        if (waitMsg != "") printError(waitMsg);
-                        lockFile(fd.get(), ltWrite, true);
-                    } else {
-                        /* Failed to lock this path; release all other
-                           locks. */
-                        unlock();
-                        lockedPaths_.lock()->erase(lockPath);
-                        return false;
-                    }
+            /* Acquire an exclusive lock. */
+            if (!lockFile(fd.get(), ltWrite, false)) {
+                if (wait) {
+                    if (waitMsg != "") printError(waitMsg);
+                    lockFile(fd.get(), ltWrite, true);
+                } else {
+                    /* Failed to lock this path; release all other
+                       locks. */
+                    unlock();
+                    return false;
                 }
-
-                debug(format("lock acquired on '%1%'") % lockPath);
-
-                /* Check that the lock file hasn't become stale (i.e.,
-                   hasn't been unlinked). */
-                struct stat st;
-                if (fstat(fd.get(), &st) == -1)
-                    throw SysError(format("statting lock file '%1%'") % lockPath);
-                if (st.st_size != 0)
-                    /* This lock file has been unlinked, so we're holding
-                       a lock on a deleted file.  This means that other
-                       processes may create and acquire a lock on
-                       `lockPath', and proceed.  So we must retry. */
-                    debug(format("open lock file '%1%' has become stale") % lockPath);
-                else
-                    break;
             }
 
-            /* Use borrow so that the descriptor isn't closed. */
-            fds.push_back(FDPair(fd.release(), lockPath));
+            debug(format("lock acquired on '%1%'") % lockPath);
 
-        } catch (...) {
-            lockedPaths_.lock()->erase(lockPath);
-            throw;
+            /* Check that the lock file hasn't become stale (i.e.,
+               hasn't been unlinked). */
+            struct stat st;
+            if (fstat(fd.get(), &st) == -1)
+                throw SysError(format("statting lock file '%1%'") % lockPath);
+            if (st.st_size != 0)
+                /* This lock file has been unlinked, so we're holding
+                   a lock on a deleted file.  This means that other
+                   processes may create and acquire a lock on
+                   `lockPath', and proceed.  So we must retry. */
+                debug(format("open lock file '%1%' has become stale") % lockPath);
+            else
+                break;
         }
 
+        /* Use borrow so that the descriptor isn't closed. */
+        fds.push_back(FDPair(fd.release(), lockPath));
     }
 
     return true;
@@ -189,8 +158,6 @@ void PathLocks::unlock()
     for (auto & i : fds) {
         if (deletePaths) deleteLockFile(i.second, i.first);
 
-        lockedPaths_.lock()->erase(i.second);
-
         if (close(i.first) == -1)
             printError(
                 format("error (ignored): cannot close lock file on '%1%'") % i.second);
@@ -208,11 +175,4 @@ void PathLocks::setDeletion(bool deletePaths)
 }
 
 
-bool pathIsLockedByMe(const Path & path)
-{
-    Path lockPath = path + ".lock";
-    return lockedPaths_.lock()->count(lockPath);
-}
-
-
 }