Release notes tweaks

(cherry picked from commit 4dd09210d7)

Makefile

@@ -5,7 +5,17 @@ makefiles = \
   src/libmain/local.mk \
   src/libexpr/local.mk \
   src/nix/local.mk \
+  src/nix-store/local.mk \
+  src/nix-instantiate/local.mk \
+  src/nix-env/local.mk \
+  src/nix-daemon/local.mk \
+  src/nix-collect-garbage/local.mk \
+  src/nix-copy-closure/local.mk \
+  src/nix-prefetch-url/local.mk \
   src/resolve-system-dependencies/local.mk \
+  src/nix-channel/local.mk \
+  src/nix-build/local.mk \
+  src/build-remote/local.mk \
   scripts/local.mk \
   corepkgs/local.mk \
   misc/systemd/local.mk \

Makefile.config.in

@@ -15,9 +15,9 @@ SODIUM_LIBS = @SODIUM_LIBS@
 LIBLZMA_LIBS = @LIBLZMA_LIBS@
 SQLITE3_LIBS = @SQLITE3_LIBS@
 LIBBROTLI_LIBS = @LIBBROTLI_LIBS@
-EDITLINE_LIBS = @EDITLINE_LIBS@
 bash = @bash@
 bindir = @bindir@
+brotli = @brotli@
 lsof = @lsof@
 datadir = @datadir@
 datarootdir = @datarootdir@

configure.ac

@@ -127,6 +127,8 @@ NEED_PROG(bzip2, bzip2)
 NEED_PROG(gzip, gzip)
 NEED_PROG(xz, xz)
 AC_PATH_PROG(dot, dot)
+AC_PATH_PROG(pv, pv, pv)
+AC_PATH_PROGS(brotli, brotli bro, bro)
 AC_PATH_PROG(lsof, lsof, lsof)
 
 
@@ -150,9 +152,9 @@ PKG_CHECK_MODULES([OPENSSL], [libcrypto], [CXXFLAGS="$OPENSSL_CFLAGS $CXXFLAGS"]
 
 # Look for libbz2, a required dependency.
 AC_CHECK_LIB([bz2], [BZ2_bzWriteOpen], [true],
-  [AC_MSG_ERROR([Nix requires libbz2, which is part of bzip2.  See https://web.archive.org/web/20180624184756/http://www.bzip.org/.])])
+  [AC_MSG_ERROR([Nix requires libbz2, which is part of bzip2.  See http://www.bzip.org/.])])
 AC_CHECK_HEADERS([bzlib.h], [true],
-  [AC_MSG_ERROR([Nix requires libbz2, which is part of bzip2.  See https://web.archive.org/web/20180624184756/http://www.bzip.org/.])])
+  [AC_MSG_ERROR([Nix requires libbz2, which is part of bzip2.  See http://www.bzip.org/.])])
 
 
 # Look for SQLite, a required dependency.
@@ -162,8 +164,6 @@ PKG_CHECK_MODULES([SQLITE3], [sqlite3 >= 3.6.19], [CXXFLAGS="$SQLITE3_CFLAGS $CX
 # Look for libcurl, a required dependency.
 PKG_CHECK_MODULES([LIBCURL], [libcurl], [CXXFLAGS="$LIBCURL_CFLAGS $CXXFLAGS"])
 
-# Look for editline, a required dependency.
-PKG_CHECK_MODULES([EDITLINE], [libeditline], [CXXFLAGS="$EDITLINE_CFLAGS $CXXFLAGS"])
 
 # Look for libsodium, an optional dependency.
 PKG_CHECK_MODULES([SODIUM], [libsodium],
@@ -251,6 +251,11 @@ if test "$(uname)" = "Darwin"; then
 fi
 
 
+# Figure out the extension of dynamic libraries.
+eval dynlib_suffix=$shrext_cmds
+AC_SUBST(dynlib_suffix)
+
+
 # Do we have GNU tar?
 AC_MSG_CHECKING([if you have a recent GNU tar])
 if $tar --version 2> /dev/null | grep -q GNU && tar cvf /dev/null --warning=no-timestamp ./config.log > /dev/null; then

corepkgs/fetchurl.nix

@@ -1,14 +1,10 @@
 { system ? "" # obsolete
 , url
-, hash ? "" # an SRI ash
-
-# Legacy hash specification
 , md5 ? "", sha1 ? "", sha256 ? "", sha512 ? ""
 , outputHash ?
-    if hash != "" then hash else if sha512 != "" then sha512 else if sha1 != "" then sha1 else if md5 != "" then md5 else sha256
+    if sha512 != "" then sha512 else if sha1 != "" then sha1 else if md5 != "" then md5 else sha256
 , outputHashAlgo ?
-    if hash != "" then "" else if sha512 != "" then "sha512" else if sha1 != "" then "sha1" else if md5 != "" then "md5" else "sha256"
-
+    if sha512 != "" then "sha512" else if sha1 != "" then "sha1" else if md5 != "" then "md5" else "sha256"
 , executable ? false
 , unpack ? false
 , name ? baseNameOf (toString url)

doc/manual/command-ref/conf-file.xml

@@ -135,6 +135,7 @@ false</literal>.</para>
 
   </varlistentry>
 
+
   <varlistentry xml:id="conf-builders">
     <term><literal>builders</literal></term>
     <listitem>
@@ -158,6 +159,7 @@ false</literal>.</para>
 
   </varlistentry>
 
+
   <varlistentry xml:id="conf-build-users-group"><term><literal>build-users-group</literal></term>
 
     <listitem><para>This options specifies the Unix group containing
@@ -208,6 +210,7 @@ false</literal>.</para>
 
   </varlistentry>
 
+
   <varlistentry xml:id="conf-connect-timeout"><term><literal>connect-timeout</literal></term>
 
     <listitem>
@@ -240,6 +243,7 @@ false</literal>.</para>
 
   </varlistentry>
 
+
   <varlistentry xml:id="conf-extra-sandbox-paths">
     <term><literal>extra-sandbox-paths</literal></term>
 
@@ -279,6 +283,7 @@ false</literal>.</para>
 
   </varlistentry>
 
+
   <varlistentry xml:id="conf-fallback"><term><literal>fallback</literal></term>
 
     <listitem><para>If set to <literal>true</literal>, Nix will fall
@@ -288,6 +293,7 @@ false</literal>.</para>
 
   </varlistentry>
 
+
   <varlistentry xml:id="conf-fsync-metadata"><term><literal>fsync-metadata</literal></term>
 
     <listitem><para>If set to <literal>true</literal>, changes to the
@@ -298,6 +304,7 @@ false</literal>.</para>
 
   </varlistentry>
 
+
   <varlistentry xml:id="conf-hashed-mirrors"><term><literal>hashed-mirrors</literal></term>
 
     <listitem><para>A list of web servers used by
@@ -360,8 +367,10 @@ builtins.fetchurl {
     options a store path was built), so by default this option is on.
     Turn it off to save a bit of disk space (or a lot if
     <literal>keep-outputs</literal> is also turned on).</para></listitem>
+
   </varlistentry>
 
+
   <varlistentry xml:id="conf-keep-env-derivations"><term><literal>keep-env-derivations</literal></term>
 
     <listitem><para>If <literal>false</literal> (default), derivations
@@ -385,6 +394,7 @@ builtins.fetchurl {
 
   </varlistentry>
 
+
   <varlistentry xml:id="conf-keep-outputs"><term><literal>keep-outputs</literal></term>
 
     <listitem><para>If <literal>true</literal>, the garbage collector
@@ -398,8 +408,10 @@ builtins.fetchurl {
     only at build time (e.g., the C compiler, or source tarballs
     downloaded from the network).  To prevent it from doing so, set
     this option to <literal>true</literal>.</para></listitem>
+
   </varlistentry>
 
+
   <varlistentry xml:id="conf-max-build-log-size"><term><literal>max-build-log-size</literal></term>
 
     <listitem>
@@ -425,15 +437,14 @@ builtins.fetchurl {
     <listitem><para>This option defines the maximum number of jobs
     that Nix will try to build in parallel.  The default is
     <literal>1</literal>. The special value <literal>auto</literal>
-    causes Nix to use the number of CPUs in your system.  <literal>0</literal>
-    is useful when using remote builders to prevent any local builds (except for
-    <literal>preferLocalBuild</literal> derivation attribute which executes locally
-    regardless).  It can be
+    causes Nix to use the number of CPUs in your system.  It can be
     overridden using the <option
     linkend='opt-max-jobs'>--max-jobs</option> (<option>-j</option>)
     command line switch.</para></listitem>
+
   </varlistentry>
 
+
   <varlistentry xml:id="conf-max-silent-time"><term><literal>max-silent-time</literal></term>
 
     <listitem>
@@ -589,6 +600,7 @@ password <replaceable>my-password</replaceable>
 
   </varlistentry>
 
+
   <varlistentry xml:id="conf-repeat"><term><literal>repeat</literal></term>
 
     <listitem><para>How many times to repeat builds to check whether
@@ -600,6 +612,7 @@ password <replaceable>my-password</replaceable>
 
   </varlistentry>
 
+
   <varlistentry xml:id="conf-require-sigs"><term><literal>require-sigs</literal></term>
 
     <listitem><para>If set to <literal>true</literal> (the default),
@@ -663,6 +676,7 @@ password <replaceable>my-password</replaceable>
 
   </varlistentry>
 
+
   <varlistentry xml:id="conf-sandbox-dev-shm-size"><term><literal>sandbox-dev-shm-size</literal></term>
 
     <listitem><para>This option determines the maximum size of the
@@ -728,6 +742,7 @@ password <replaceable>my-password</replaceable>
 
   </varlistentry>
 
+
   <varlistentry xml:id="conf-substituters"><term><literal>substituters</literal></term>
 
     <listitem><para>A list of URLs of substituters, separated by
@@ -736,6 +751,7 @@ password <replaceable>my-password</replaceable>
 
   </varlistentry>
 
+
   <varlistentry xml:id="conf-system"><term><literal>system</literal></term>
 
     <listitem><para>This option specifies the canonical Nix system
@@ -757,33 +773,6 @@ password <replaceable>my-password</replaceable>
   </varlistentry>
 
 
-  <varlistentry xml:id="conf-system-features"><term><literal>system-features</literal></term>
-
-    <listitem><para>A set of system “features” supported by this
-    machine, e.g. <literal>kvm</literal>. Derivations can express a
-    dependency on such features through the derivation attribute
-    <varname>requiredSystemFeatures</varname>. For example, the
-    attribute
-
-<programlisting>
-requiredSystemFeatures = [ "kvm" ];
-</programlisting>
-
-    ensures that the derivation can only be built on a machine with
-    the <literal>kvm</literal> feature.</para>
-
-    <para>This setting by default includes <literal>kvm</literal> if
-    <filename>/dev/kvm</filename> is accessible, and the
-    pseudo-features <literal>nixos-test</literal>,
-    <literal>benchmark</literal> and <literal>big-parallel</literal>
-    that are used in Nixpkgs to route builds to specific
-    machines.</para>
-
-    </listitem>
-
-  </varlistentry>
-
-
   <varlistentry xml:id="conf-timeout"><term><literal>timeout</literal></term>
 
     <listitem>
@@ -803,6 +792,7 @@ requiredSystemFeatures = [ "kvm" ];
 
   </varlistentry>
 
+
   <varlistentry xml:id="conf-trusted-public-keys"><term><literal>trusted-public-keys</literal></term>
 
     <listitem><para>A whitespace-separated list of public keys. When
@@ -813,6 +803,7 @@ requiredSystemFeatures = [ "kvm" ];
 
   </varlistentry>
 
+
   <varlistentry xml:id="conf-trusted-substituters"><term><literal>trusted-substituters</literal></term>
 
     <listitem><para>A list of URLs of substituters, separated by
@@ -825,6 +816,7 @@ requiredSystemFeatures = [ "kvm" ];
 
   </varlistentry>
 
+
   <varlistentry xml:id="conf-trusted-users"><term><literal>trusted-users</literal></term>
 
     <listitem>
@@ -850,178 +842,9 @@ requiredSystemFeatures = [ "kvm" ];
   </varlistentry>
 
 </variablelist>
+
 </para>
 
-<refsection>
-  <title>Deprecated Settings</title>
-
-<para>
-
-<variablelist>
-
-  <varlistentry xml:id="conf-binary-caches">
-    <term><literal>binary-caches</literal></term>
-
-    <listitem><para><emphasis>Deprecated:</emphasis>
-    <literal>binary-caches</literal> is now an alias to
-    <xref linkend="conf-substituters" />.</para></listitem>
-  </varlistentry>
-
-  <varlistentry xml:id="conf-binary-cache-public-keys">
-    <term><literal>binary-cache-public-keys</literal></term>
-
-    <listitem><para><emphasis>Deprecated:</emphasis>
-    <literal>binary-cache-public-keys</literal> is now an alias to
-    <xref linkend="conf-trusted-public-keys" />.</para></listitem>
-  </varlistentry>
-
-  <varlistentry xml:id="conf-build-compress-log">
-    <term><literal>build-compress-log</literal></term>
-
-    <listitem><para><emphasis>Deprecated:</emphasis>
-    <literal>build-compress-log</literal> is now an alias to
-    <xref linkend="conf-compress-build-log" />.</para></listitem>
-  </varlistentry>
-
-  <varlistentry xml:id="conf-build-cores">
-    <term><literal>build-cores</literal></term>
-
-    <listitem><para><emphasis>Deprecated:</emphasis>
-    <literal>build-cores</literal> is now an alias to
-    <xref linkend="conf-cores" />.</para></listitem>
-  </varlistentry>
-
-  <varlistentry xml:id="conf-build-extra-chroot-dirs">
-    <term><literal>build-extra-chroot-dirs</literal></term>
-
-    <listitem><para><emphasis>Deprecated:</emphasis>
-    <literal>build-extra-chroot-dirs</literal> is now an alias to
-    <xref linkend="conf-extra-sandbox-paths" />.</para></listitem>
-  </varlistentry>
-
-  <varlistentry xml:id="conf-build-extra-sandbox-paths">
-    <term><literal>build-extra-sandbox-paths</literal></term>
-
-    <listitem><para><emphasis>Deprecated:</emphasis>
-    <literal>build-extra-sandbox-paths</literal> is now an alias to
-    <xref linkend="conf-extra-sandbox-paths" />.</para></listitem>
-  </varlistentry>
-
-  <varlistentry xml:id="conf-build-fallback">
-    <term><literal>build-fallback</literal></term>
-
-    <listitem><para><emphasis>Deprecated:</emphasis>
-    <literal>build-fallback</literal> is now an alias to
-    <xref linkend="conf-fallback" />.</para></listitem>
-  </varlistentry>
-
-  <varlistentry xml:id="conf-build-max-jobs">
-    <term><literal>build-max-jobs</literal></term>
-
-    <listitem><para><emphasis>Deprecated:</emphasis>
-    <literal>build-max-jobs</literal> is now an alias to
-    <xref linkend="conf-max-jobs" />.</para></listitem>
-  </varlistentry>
-
-  <varlistentry xml:id="conf-build-max-log-size">
-    <term><literal>build-max-log-size</literal></term>
-
-    <listitem><para><emphasis>Deprecated:</emphasis>
-    <literal>build-max-log-size</literal> is now an alias to
-    <xref linkend="conf-max-build-log-size" />.</para></listitem>
-  </varlistentry>
-
-  <varlistentry xml:id="conf-build-max-silent-time">
-    <term><literal>build-max-silent-time</literal></term>
-
-    <listitem><para><emphasis>Deprecated:</emphasis>
-    <literal>build-max-silent-time</literal> is now an alias to
-    <xref linkend="conf-max-silent-time" />.</para></listitem>
-  </varlistentry>
-
-  <varlistentry xml:id="conf-build-repeat">
-    <term><literal>build-repeat</literal></term>
-
-    <listitem><para><emphasis>Deprecated:</emphasis>
-    <literal>build-repeat</literal> is now an alias to
-    <xref linkend="conf-repeat" />.</para></listitem>
-  </varlistentry>
-
-  <varlistentry xml:id="conf-build-timeout">
-    <term><literal>build-timeout</literal></term>
-
-    <listitem><para><emphasis>Deprecated:</emphasis>
-    <literal>build-timeout</literal> is now an alias to
-    <xref linkend="conf-timeout" />.</para></listitem>
-  </varlistentry>
-
-  <varlistentry xml:id="conf-build-use-chroot">
-    <term><literal>build-use-chroot</literal></term>
-
-    <listitem><para><emphasis>Deprecated:</emphasis>
-    <literal>build-use-chroot</literal> is now an alias to
-    <xref linkend="conf-sandbox" />.</para></listitem>
-  </varlistentry>
-
-  <varlistentry xml:id="conf-build-use-sandbox">
-    <term><literal>build-use-sandbox</literal></term>
-
-    <listitem><para><emphasis>Deprecated:</emphasis>
-    <literal>build-use-sandbox</literal> is now an alias to
-    <xref linkend="conf-sandbox" />.</para></listitem>
-  </varlistentry>
-
-  <varlistentry xml:id="conf-build-use-substitutes">
-    <term><literal>build-use-substitutes</literal></term>
-
-    <listitem><para><emphasis>Deprecated:</emphasis>
-    <literal>build-use-substitutes</literal> is now an alias to
-    <xref linkend="conf-substitute" />.</para></listitem>
-  </varlistentry>
-
-  <varlistentry xml:id="conf-gc-keep-derivations">
-    <term><literal>gc-keep-derivations</literal></term>
-
-    <listitem><para><emphasis>Deprecated:</emphasis>
-    <literal>gc-keep-derivations</literal> is now an alias to
-    <xref linkend="conf-keep-derivations" />.</para></listitem>
-  </varlistentry>
-
-  <varlistentry xml:id="conf-gc-keep-outputs">
-    <term><literal>gc-keep-outputs</literal></term>
-
-    <listitem><para><emphasis>Deprecated:</emphasis>
-    <literal>gc-keep-outputs</literal> is now an alias to
-    <xref linkend="conf-keep-outputs" />.</para></listitem>
-  </varlistentry>
-
-  <varlistentry xml:id="conf-env-keep-derivations">
-    <term><literal>env-keep-derivations</literal></term>
-
-    <listitem><para><emphasis>Deprecated:</emphasis>
-    <literal>env-keep-derivations</literal> is now an alias to
-    <xref linkend="conf-keep-env-derivations" />.</para></listitem>
-  </varlistentry>
-
-  <varlistentry xml:id="conf-extra-binary-caches">
-    <term><literal>extra-binary-caches</literal></term>
-
-    <listitem><para><emphasis>Deprecated:</emphasis>
-    <literal>extra-binary-caches</literal> is now an alias to
-    <xref linkend="conf-extra-substituters" />.</para></listitem>
-  </varlistentry>
-
-  <varlistentry xml:id="conf-trusted-binary-caches">
-    <term><literal>trusted-binary-caches</literal></term>
-
-    <listitem><para><emphasis>Deprecated:</emphasis>
-    <literal>trusted-binary-caches</literal> is now an alias to
-    <xref linkend="conf-trusted-substituters" />.</para></listitem>
-  </varlistentry>
-</variablelist>
-</para>
-</refsection>
-
 </refsection>
 
 </refentry>

doc/manual/command-ref/nix-copy-closure.xml

@@ -95,6 +95,15 @@ those paths.  If this bothers you, use
 
   </varlistentry>
 
+  <!--
+  <varlistentry><term><option>- -show-progress</option></term>
+
+    <listitem><para>Show the progress of each path's transfer as it's made.
+    This requires the <command>pv</command> utility to be in <envar>PATH</envar>.</para></listitem>
+
+  </varlistentry>
+  -->
+
   <varlistentry><term><option>--include-outputs</option></term>
 
     <listitem><para>Also copy the outputs of store derivations

doc/manual/command-ref/nix-instantiate.xml

@@ -154,9 +154,7 @@ input.</para>
     <listitem><para>When used with <option>--eval</option>, perform
     evaluation in read/write mode so nix language features that
     require it will still work (at the cost of needing to do
-    instantiation of every evaluated derivation). If this option is
-    not enabled, there may be uninstantiated store paths in the final
-    output.</para>
+    instantiation of every evaluated derivation).</para>
 
     </listitem>
 

doc/manual/command-ref/nix-shell.xml

@@ -317,28 +317,13 @@ while (my $token = $p->get_tag("a")) {
 
 </para>
 
-<para>Sometimes you need to pass a simple Nix expression to customize
-a package like Terraform:
-
-<programlisting><![CDATA[
-#! /usr/bin/env nix-shell
-#! nix-shell -i bash -p "terraform.withPlugins (plugins: [ plugins.openstack ])"
-
-terraform apply
-]]></programlisting>
-
-<note><para>You must use double quotes (<literal>"</literal>) when
-passing a simple Nix expression in a nix-shell shebang.</para></note>
-</para>
-
-<para>Finally, using the merging of multiple nix-shell shebangs the
-following Haskell script uses a specific branch of Nixpkgs/NixOS (the
-18.03 stable branch):
+<para>Finally, the following Haskell script uses a specific branch of
+Nixpkgs/NixOS (the 14.12 stable branch):
 
 <programlisting><![CDATA[
 #! /usr/bin/env nix-shell
 #! nix-shell -i runghc -p haskellPackages.ghc haskellPackages.HTTP haskellPackages.tagsoup
-#! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-18.03.tar.gz
+#! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-14.12.tar.gz
 
 import Network.HTTP
 import Text.HTML.TagSoup

doc/manual/command-ref/nix-store.xml

@@ -275,7 +275,7 @@ as a means of providing Nix store access to a restricted ssh user.
 
     <listitem><para>Allow the connected client to request the realization
     of derivations. In effect, this can be used to make the host act
-    as a remote builder.</para></listitem>
+    as a build slave.</para></listitem>
 
   </varlistentry>
 
@@ -679,18 +679,6 @@ query is applied to the target of the symlink.</para>
 
   </varlistentry>
 
-  <varlistentry><term><option>--graphml</option></term>
-
-    <listitem><para>Prints the references graph of the store paths
-    <replaceable>paths</replaceable> in the <link
-    xlink:href="http://graphml.graphdrawing.org/">GraphML</link> file format.
-    This can be used to visualise dependency graphs. To obtain a
-    build-time dependency graph, apply this to a store derivation. To
-    obtain a runtime dependency graph, apply it to an output
-    path.</para></listitem>
-
-  </varlistentry>
-
   <varlistentry><term><option>--binding</option> <replaceable>name</replaceable></term>
     <term><option>-b</option> <replaceable>name</replaceable></term>
 

doc/manual/expressions/advanced-attributes.xml

@@ -216,7 +216,7 @@ fetchurl {
 <programlisting>
 { stdenv, curl }: # The <command>curl</command> program is used for downloading.
 
-{ url, sha256 }:
+{ url, md5 }:
 
 stdenv.mkDerivation {
   name = baseNameOf (toString url);
@@ -224,10 +224,10 @@ stdenv.mkDerivation {
   buildInputs = [ curl ];
 
   # This is a fixed-output derivation; the output must be a regular
-  # file with SHA256 hash <varname>sha256</varname>.
+  # file with MD5 hash <varname>md5</varname>.
   outputHashMode = "flat";
-  outputHashAlgo = "sha256";
-  outputHash = sha256;
+  outputHashAlgo = "md5";
+  outputHash = md5;
 
   inherit url;
 }
@@ -237,8 +237,8 @@ stdenv.mkDerivation {
 
     <para>The <varname>outputHashAlgo</varname> attribute specifies
     the hash algorithm used to compute the hash.  It can currently be
-    <literal>"sha1"</literal>, <literal>"sha256"</literal> or
-    <literal>"sha512"</literal>.</para>
+    <literal>"md5"</literal>, <literal>"sha1"</literal> or
+    <literal>"sha256"</literal>.</para>
 
     <para>The <varname>outputHashMode</varname> attribute determines
     how the hash is computed.  It must be one of the following two
@@ -251,7 +251,7 @@ stdenv.mkDerivation {
         <listitem><para>The output must be a non-executable regular
         file.  If it isn’t, the build fails.  The hash is simply
         computed over the contents of that file (so it’s equal to what
-        Unix commands like <command>sha256sum</command> or
+        Unix commands like <command>md5sum</command> or
         <command>sha1sum</command> produce).</para>
 
         <para>This is the default.</para></listitem>
@@ -312,7 +312,9 @@ big = "a very long string";
   <varlistentry><term><varname>preferLocalBuild</varname></term>
 
     <listitem><para>If this attribute is set to
-    <literal>true</literal> and <link
+    <literal>true</literal>, it has two effects.  First, the
+    derivation will always be built, not substituted, even if a
+    substitute is available.  Second, if <link
     linkend="chap-distributed-builds">distributed building is
     enabled</link>, then, if possible, the derivaton will be built
     locally instead of forwarded to a remote machine.  This is
@@ -322,19 +324,6 @@ big = "a very long string";
 
   </varlistentry>
 
-
-  <varlistentry><term><varname>allowSubstitutes</varname></term>
-
-    <listitem><para>If this attribute is set to
-    <literal>false</literal>, then Nix will always build this
-    derivation; it will not try to substitute its outputs. This is
-    useful for very trivial derivations (such as
-    <function>writeText</function> in Nixpkgs) that are cheaper to
-    build than to substitute from a binary cache.</para></listitem>
-
-  </varlistentry>
-
-
 </variablelist>
 
 </section>

doc/manual/expressions/builtins.xml

@@ -21,8 +21,7 @@ available as <function>builtins.derivation</function>.</para>
 <variablelist>
 
 
-  <varlistentry xml:id='builtin-abort'>
-    <term><function>abort</function> <replaceable>s</replaceable></term>
+  <varlistentry><term><function>abort</function> <replaceable>s</replaceable></term>
 
     <listitem><para>Abort Nix expression evaluation, print error
     message <replaceable>s</replaceable>.</para></listitem>
@@ -30,10 +29,8 @@ available as <function>builtins.derivation</function>.</para>
   </varlistentry>
 
 
-  <varlistentry xml:id='builtin-add'>
-    <term><function>builtins.add</function>
-    <replaceable>e1</replaceable> <replaceable>e2</replaceable>
-    </term>
+  <varlistentry><term><function>builtins.add</function>
+  <replaceable>e1</replaceable> <replaceable>e2</replaceable></term>
 
     <listitem><para>Return the sum of the numbers
     <replaceable>e1</replaceable> and
@@ -42,9 +39,8 @@ available as <function>builtins.derivation</function>.</para>
   </varlistentry>
 
 
-  <varlistentry xml:id='builtin-all'>
-    <term><function>builtins.all</function>
-    <replaceable>pred</replaceable> <replaceable>list</replaceable></term>
+  <varlistentry><term><function>builtins.all</function>
+  <replaceable>pred</replaceable> <replaceable>list</replaceable></term>
 
     <listitem><para>Return <literal>true</literal> if the function
     <replaceable>pred</replaceable> returns <literal>true</literal>
@@ -54,9 +50,8 @@ available as <function>builtins.derivation</function>.</para>
   </varlistentry>
 
 
-  <varlistentry xml:id='builtin-any'>
-    <term><function>builtins.any</function>
-    <replaceable>pred</replaceable> <replaceable>list</replaceable></term>
+  <varlistentry><term><function>builtins.any</function>
+  <replaceable>pred</replaceable> <replaceable>list</replaceable></term>
 
     <listitem><para>Return <literal>true</literal> if the function
     <replaceable>pred</replaceable> returns <literal>true</literal>
@@ -66,9 +61,8 @@ available as <function>builtins.derivation</function>.</para>
   </varlistentry>
 
 
-  <varlistentry xml:id='builtin-attrNames'>
-    <term><function>builtins.attrNames</function>
-    <replaceable>set</replaceable></term>
+  <varlistentry><term><function>builtins.attrNames</function>
+  <replaceable>set</replaceable></term>
 
     <listitem><para>Return the names of the attributes in the set
     <replaceable>set</replaceable> in an alphabetically sorted list.  For instance,
@@ -78,9 +72,8 @@ available as <function>builtins.derivation</function>.</para>
   </varlistentry>
 
 
-  <varlistentry xml:id='builtin-attrValues'>
-    <term><function>builtins.attrValues</function>
-    <replaceable>set</replaceable></term>
+  <varlistentry><term><function>builtins.attrValues</function>
+  <replaceable>set</replaceable></term>
 
     <listitem><para>Return the values of the attributes in the set
     <replaceable>set</replaceable> in the order corresponding to the
@@ -89,8 +82,7 @@ available as <function>builtins.derivation</function>.</para>
   </varlistentry>
 
 
-  <varlistentry xml:id='builtin-baseNameOf'>
-    <term><function>baseNameOf</function> <replaceable>s</replaceable></term>
+  <varlistentry><term><function>baseNameOf</function> <replaceable>s</replaceable></term>
 
     <listitem><para>Return the <emphasis>base name</emphasis> of the
     string <replaceable>s</replaceable>, that is, everything following
@@ -100,9 +92,8 @@ available as <function>builtins.derivation</function>.</para>
   </varlistentry>
 
 
-  <varlistentry xml:id='builtin-bitAnd'>
-    <term><function>builtins.bitAnd</function>
-    <replaceable>e1</replaceable> <replaceable>e2</replaceable></term>
+  <varlistentry><term><function>builtins.bitAnd</function>
+  <replaceable>e1</replaceable> <replaceable>e2</replaceable></term>
 
     <listitem><para>Return the bitwise AND of the integers
     <replaceable>e1</replaceable> and
@@ -111,9 +102,8 @@ available as <function>builtins.derivation</function>.</para>
   </varlistentry>
 
 
-  <varlistentry xml:id='builtin-bitOr'>
-    <term><function>builtins.bitOr</function>
-    <replaceable>e1</replaceable> <replaceable>e2</replaceable></term>
+  <varlistentry><term><function>builtins.bitOr</function>
+  <replaceable>e1</replaceable> <replaceable>e2</replaceable></term>
 
     <listitem><para>Return the bitwise OR of the integers
     <replaceable>e1</replaceable> and
@@ -122,9 +112,8 @@ available as <function>builtins.derivation</function>.</para>
   </varlistentry>
 
 
-  <varlistentry xml:id='builtin-bitXor'>
-    <term><function>builtins.bitXor</function>
-    <replaceable>e1</replaceable> <replaceable>e2</replaceable></term>
+  <varlistentry><term><function>builtins.bitXor</function>
+  <replaceable>e1</replaceable> <replaceable>e2</replaceable></term>
 
     <listitem><para>Return the bitwise XOR of the integers
     <replaceable>e1</replaceable> and
@@ -133,8 +122,7 @@ available as <function>builtins.derivation</function>.</para>
   </varlistentry>
 
 
-  <varlistentry xml:id='builtin-builtins'>
-    <term><varname>builtins</varname></term>
+  <varlistentry><term><varname>builtins</varname></term>
 
     <listitem><para>The set <varname>builtins</varname> contains all
     the built-in functions and values.  You can use
@@ -151,9 +139,8 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting>
   </varlistentry>
 
 
-  <varlistentry xml:id='builtin-compareVersions'>
-    <term><function>builtins.compareVersions</function>
-    <replaceable>s1</replaceable> <replaceable>s2</replaceable></term>
+  <varlistentry><term><function>builtins.compareVersions</function>
+  <replaceable>s1</replaceable> <replaceable>s2</replaceable></term>
 
     <listitem><para>Compare two strings representing versions and
     return <literal>-1</literal> if version
@@ -169,9 +156,8 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting>
   </varlistentry>
 
 
-  <varlistentry xml:id='builtin-splitVersion'>
-    <term><function>builtins.splitVersion</function>
-    <replaceable>s</replaceable></term>
+  <varlistentry><term><function>builtins.splitVersion</function>
+  <replaceable>s</replaceable></term>
 
     <listitem><para>Split a string representing a version into its
     components, by the same version splitting logic underlying the
@@ -181,18 +167,16 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting>
   </varlistentry>
 
 
-  <varlistentry xml:id='builtin-concatLists'>
-    <term><function>builtins.concatLists</function>
-    <replaceable>lists</replaceable></term>
+  <varlistentry><term><function>builtins.concatLists</function>
+  <replaceable>lists</replaceable></term>
 
     <listitem><para>Concatenate a list of lists into a single
     list.</para></listitem>
 
   </varlistentry>
 
-  <varlistentry xml:id='builtin-concatStringsSep'>
-    <term><function>builtins.concatStringsSep</function>
-    <replaceable>separator</replaceable> <replaceable>list</replaceable></term>
+  <varlistentry><term><function>builtins.concatStringsSep</function>
+  <replaceable>separator</replaceable> <replaceable>list</replaceable></term>
 
     <listitem><para>Concatenate a list of strings with a separator
     between each element, e.g. <literal>concatStringsSep "/"
@@ -200,8 +184,8 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting>
 
   </varlistentry>
 
-  <varlistentry xml:id='builtin-currentSystem'>
-    <term><varname>builtins.currentSystem</varname></term>
+  <varlistentry
+  xml:id='builtin-currentSystem'><term><varname>builtins.currentSystem</varname></term>
 
     <listitem><para>The built-in value <varname>currentSystem</varname>
     evaluates to the Nix platform identifier for the Nix installation
@@ -234,9 +218,8 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting>
   -->
 
 
-  <varlistentry xml:id='builtin-deepSeq'>
-    <term><function>builtins.deepSeq</function>
-    <replaceable>e1</replaceable> <replaceable>e2</replaceable></term>
+  <varlistentry><term><function>builtins.deepSeq</function>
+  <replaceable>e1</replaceable> <replaceable>e2</replaceable></term>
 
     <listitem><para>This is like <literal>seq
     <replaceable>e1</replaceable>
@@ -248,9 +231,8 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting>
   </varlistentry>
 
 
-  <varlistentry xml:id='builtin-derivation'>
-    <term><function>derivation</function>
-    <replaceable>attrs</replaceable></term>
+  <varlistentry><term><function>derivation</function>
+  <replaceable>attrs</replaceable></term>
 
     <listitem><para><function>derivation</function> is described in
     <xref linkend='ssec-derivation' />.</para></listitem>
@@ -258,8 +240,7 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting>
   </varlistentry>
 
 
-  <varlistentry xml:id='builtin-dirOf'>
-    <term><function>dirOf</function> <replaceable>s</replaceable></term>
+  <varlistentry><term><function>dirOf</function> <replaceable>s</replaceable></term>
 
     <listitem><para>Return the directory part of the string
     <replaceable>s</replaceable>, that is, everything before the final
@@ -269,9 +250,8 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting>
   </varlistentry>
 
 
-  <varlistentry xml:id='builtin-div'>
-    <term><function>builtins.div</function>
-    <replaceable>e1</replaceable> <replaceable>e2</replaceable></term>
+  <varlistentry><term><function>builtins.div</function>
+  <replaceable>e1</replaceable> <replaceable>e2</replaceable></term>
 
     <listitem><para>Return the quotient of the numbers
     <replaceable>e1</replaceable> and
@@ -279,9 +259,8 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting>
 
   </varlistentry>
 
-  <varlistentry xml:id='builtin-elem'>
-    <term><function>builtins.elem</function>
-    <replaceable>x</replaceable> <replaceable>xs</replaceable></term>
+  <varlistentry><term><function>builtins.elem</function>
+  <replaceable>x</replaceable> <replaceable>xs</replaceable></term>
 
     <listitem><para>Return <literal>true</literal> if a value equal to
     <replaceable>x</replaceable> occurs in the list
@@ -291,9 +270,8 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting>
   </varlistentry>
 
 
-  <varlistentry xml:id='builtin-elemAt'>
-    <term><function>builtins.elemAt</function>
-    <replaceable>xs</replaceable> <replaceable>n</replaceable></term>
+  <varlistentry><term><function>builtins.elemAt</function>
+  <replaceable>xs</replaceable> <replaceable>n</replaceable></term>
 
     <listitem><para>Return element <replaceable>n</replaceable> from
     the list <replaceable>xs</replaceable>.  Elements are counted
@@ -303,9 +281,8 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting>
   </varlistentry>
 
 
-  <varlistentry xml:id='builtin-fetchurl'>
-    <term><function>builtins.fetchurl</function>
-    <replaceable>url</replaceable></term>
+  <varlistentry><term><function>builtins.fetchurl</function>
+  <replaceable>url</replaceable></term>
 
     <listitem><para>Download the specified URL and return the path of
     the downloaded file. This function is not available if <link
@@ -315,9 +292,8 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting>
   </varlistentry>
 
 
-  <varlistentry xml:id='builtin-fetchTarball'>
-    <term><function>fetchTarball</function>
-    <replaceable>url</replaceable></term>
+  <varlistentry><term><function>fetchTarball</function>
+  <replaceable>url</replaceable></term>
 
     <listitem><para>Download the specified URL, unpack it and return
     the path of the unpacked tree. The file must be a tape archive
@@ -370,7 +346,7 @@ stdenv.mkDerivation { … }
 
   </varlistentry>
 
-  <varlistentry xml:id='builtin-fetchGit'>
+  <varlistentry>
     <term>
       <function>builtins.fetchGit</function>
       <replaceable>args</replaceable>
@@ -436,7 +412,7 @@ stdenv.mkDerivation { … }
         <title>Fetching a repository's specific commit on an arbitrary branch</title>
         <para>
           If the revision you're looking for is in the default branch
-          of the git repository you don't strictly need to specify
+          of the gift repository you don't strictly need to specify
           the branch name in the <varname>ref</varname> attribute.
         </para>
         <para>
@@ -465,7 +441,7 @@ stdenv.mkDerivation { … }
         <title>Fetching a repository's specific commit on the default branch</title>
         <para>
           If the revision you're looking for is in the default branch
-          of the git repository you may omit the
+          of the gift repository you may omit the
           <varname>ref</varname> attribute.
         </para>
         <programlisting>builtins.fetchGit {
@@ -570,8 +546,7 @@ stdenv.mkDerivation {
   </varlistentry>
 
 
-  <varlistentry xml:id='builtin-foldl-prime'>
-    <term><function>builtins.foldl’</function>
+  <varlistentry><term><function>builtins.foldl’</function>
     <replaceable>op</replaceable> <replaceable>nul</replaceable> <replaceable>list</replaceable></term>
 
     <listitem><para>Reduce a list by applying a binary operator, from
@@ -584,8 +559,7 @@ stdenv.mkDerivation {
   </varlistentry>
 
 
-  <varlistentry xml:id='builtin-functionArgs'>
-    <term><function>builtins.functionArgs</function>
+  <varlistentry><term><function>builtins.functionArgs</function>
     <replaceable>f</replaceable></term>
 
     <listitem><para>
@@ -603,8 +577,7 @@ stdenv.mkDerivation {
   </varlistentry>
 
 
-  <varlistentry xml:id='builtin-fromJSON'>
-    <term><function>builtins.fromJSON</function> <replaceable>e</replaceable></term>
+  <varlistentry><term><function>builtins.fromJSON</function> <replaceable>e</replaceable></term>
 
     <listitem><para>Convert a JSON string to a Nix
     value. For example,
@@ -619,9 +592,8 @@ builtins.fromJSON ''{"x": [1, 2, 3], "y": null}''
   </varlistentry>
 
 
-  <varlistentry xml:id='builtin-genList'>
-    <term><function>builtins.genList</function>
-    <replaceable>generator</replaceable> <replaceable>length</replaceable></term>
+  <varlistentry><term><function>builtins.genList</function>
+  <replaceable>generator</replaceable> <replaceable>length</replaceable></term>
 
     <listitem><para>Generate list of size
     <replaceable>length</replaceable>, with each element
@@ -638,9 +610,8 @@ builtins.genList (x: x * x) 5
   </varlistentry>
 
 
-  <varlistentry xml:id='builtin-getAttr'>
-    <term><function>builtins.getAttr</function>
-    <replaceable>s</replaceable> <replaceable>set</replaceable></term>
+  <varlistentry><term><function>builtins.getAttr</function>
+  <replaceable>s</replaceable> <replaceable>set</replaceable></term>
 
     <listitem><para><function>getAttr</function> returns the attribute
     named <replaceable>s</replaceable> from
@@ -652,9 +623,8 @@ builtins.genList (x: x * x) 5
   </varlistentry>
 
 
-  <varlistentry xml:id='builtin-getEnv'>
-    <term><function>builtins.getEnv</function>
-    <replaceable>s</replaceable></term>
+  <varlistentry><term><function>builtins.getEnv</function>
+  <replaceable>s</replaceable></term>
 
     <listitem><para><function>getEnv</function> returns the value of
     the environment variable <replaceable>s</replaceable>, or an empty
@@ -671,9 +641,8 @@ builtins.genList (x: x * x) 5
   </varlistentry>
 
 
-  <varlistentry xml:id='builtin-hasAttr'>
-    <term><function>builtins.hasAttr</function>
-    <replaceable>s</replaceable> <replaceable>set</replaceable></term>
+  <varlistentry><term><function>builtins.hasAttr</function>
+  <replaceable>s</replaceable> <replaceable>set</replaceable></term>
 
     <listitem><para><function>hasAttr</function> returns
     <literal>true</literal> if <replaceable>set</replaceable> has an
@@ -686,9 +655,8 @@ builtins.genList (x: x * x) 5
   </varlistentry>
 
 
-  <varlistentry xml:id='builtin-hashString'>
-    <term><function>builtins.hashString</function>
-    <replaceable>type</replaceable> <replaceable>s</replaceable></term>
+  <varlistentry><term><function>builtins.hashString</function>
+  <replaceable>type</replaceable> <replaceable>s</replaceable></term>
 
     <listitem><para>Return a base-16 representation of the
     cryptographic hash of string <replaceable>s</replaceable>.  The
@@ -699,9 +667,8 @@ builtins.genList (x: x * x) 5
   </varlistentry>
 
 
-  <varlistentry xml:id='builtin-head'>
-    <term><function>builtins.head</function>
-    <replaceable>list</replaceable></term>
+  <varlistentry><term><function>builtins.head</function>
+  <replaceable>list</replaceable></term>
 
     <listitem><para>Return the first element of a list; abort
     evaluation if the argument isn’t a list or is an empty list.  You
@@ -711,9 +678,8 @@ builtins.genList (x: x * x) 5
   </varlistentry>
 
 
-  <varlistentry xml:id='builtin-import'>
-    <term><function>import</function>
-    <replaceable>path</replaceable></term>
+  <varlistentry><term><function>import</function>
+  <replaceable>path</replaceable></term>
 
     <listitem><para>Load, parse and return the Nix expression in the
     file <replaceable>path</replaceable>.  If <replaceable>path
@@ -767,9 +733,8 @@ x: x + 456</programlisting>
   </varlistentry>
 
 
-  <varlistentry xml:id='builtin-intersectAttrs'>
-    <term><function>builtins.intersectAttrs</function>
-    <replaceable>e1</replaceable> <replaceable>e2</replaceable></term>
+  <varlistentry><term><function>builtins.intersectAttrs</function>
+  <replaceable>e1</replaceable> <replaceable>e2</replaceable></term>
 
     <listitem><para>Return a set consisting of the attributes in the
     set <replaceable>e2</replaceable> that also exist in the set
@@ -778,9 +743,8 @@ x: x + 456</programlisting>
   </varlistentry>
 
 
-  <varlistentry xml:id='builtin-isAttrs'>
-    <term><function>builtins.isAttrs</function>
-    <replaceable>e</replaceable></term>
+  <varlistentry><term><function>builtins.isAttrs</function>
+  <replaceable>e</replaceable></term>
 
     <listitem><para>Return <literal>true</literal> if
     <replaceable>e</replaceable> evaluates to a set, and
@@ -789,9 +753,8 @@ x: x + 456</programlisting>
   </varlistentry>
 
 
-  <varlistentry xml:id='builtin-isList'>
-    <term><function>builtins.isList</function>
-    <replaceable>e</replaceable></term>
+  <varlistentry><term><function>builtins.isList</function>
+  <replaceable>e</replaceable></term>
 
     <listitem><para>Return <literal>true</literal> if
     <replaceable>e</replaceable> evaluates to a list, and
@@ -800,7 +763,7 @@ x: x + 456</programlisting>
   </varlistentry>
 
 
-  <varlistentry xml:id='builtin-isFunction'><term><function>builtins.isFunction</function>
+  <varlistentry><term><function>builtins.isFunction</function>
   <replaceable>e</replaceable></term>
 
     <listitem><para>Return <literal>true</literal> if
@@ -810,9 +773,8 @@ x: x + 456</programlisting>
   </varlistentry>
 
 
-  <varlistentry xml:id='builtin-isString'>
-    <term><function>builtins.isString</function>
-    <replaceable>e</replaceable></term>
+  <varlistentry><term><function>builtins.isString</function>
+  <replaceable>e</replaceable></term>
 
     <listitem><para>Return <literal>true</literal> if
     <replaceable>e</replaceable> evaluates to a string, and
@@ -821,9 +783,8 @@ x: x + 456</programlisting>
   </varlistentry>
 
 
-  <varlistentry xml:id='builtin-isInt'>
-    <term><function>builtins.isInt</function>
-    <replaceable>e</replaceable></term>
+  <varlistentry><term><function>builtins.isInt</function>
+  <replaceable>e</replaceable></term>
 
     <listitem><para>Return <literal>true</literal> if
     <replaceable>e</replaceable> evaluates to an int, and
@@ -832,9 +793,8 @@ x: x + 456</programlisting>
   </varlistentry>
 
 
-  <varlistentry xml:id='builtin-isFloat'>
-    <term><function>builtins.isFloat</function>
-    <replaceable>e</replaceable></term>
+  <varlistentry><term><function>builtins.isFloat</function>
+  <replaceable>e</replaceable></term>
 
     <listitem><para>Return <literal>true</literal> if
     <replaceable>e</replaceable> evaluates to a float, and
@@ -843,9 +803,8 @@ x: x + 456</programlisting>
   </varlistentry>
 
 
-  <varlistentry xml:id='builtin-isBool'>
-    <term><function>builtins.isBool</function>
-    <replaceable>e</replaceable></term>
+  <varlistentry><term><function>builtins.isBool</function>
+  <replaceable>e</replaceable></term>
 
     <listitem><para>Return <literal>true</literal> if
     <replaceable>e</replaceable> evaluates to a bool, and
@@ -854,9 +813,8 @@ x: x + 456</programlisting>
   </varlistentry>
 
 
-  <varlistentry xml:id='builtin-isNull'>
-    <term><function>isNull</function>
-    <replaceable>e</replaceable></term>
+  <varlistentry><term><function>isNull</function>
+  <replaceable>e</replaceable></term>
 
     <listitem><para>Return <literal>true</literal> if
     <replaceable>e</replaceable> evaluates to <literal>null</literal>,
@@ -870,9 +828,8 @@ x: x + 456</programlisting>
   </varlistentry>
 
 
-  <varlistentry xml:id='builtin-length'>
-    <term><function>builtins.length</function>
-    <replaceable>e</replaceable></term>
+  <varlistentry><term><function>builtins.length</function>
+  <replaceable>e</replaceable></term>
 
     <listitem><para>Return the length of the list
     <replaceable>e</replaceable>.</para></listitem>
@@ -880,9 +837,8 @@ x: x + 456</programlisting>
   </varlistentry>
 
 
-  <varlistentry xml:id='builtin-lessThan'>
-    <term><function>builtins.lessThan</function>
-    <replaceable>e1</replaceable> <replaceable>e2</replaceable></term>
+  <varlistentry><term><function>builtins.lessThan</function>
+  <replaceable>e1</replaceable> <replaceable>e2</replaceable></term>
 
     <listitem><para>Return <literal>true</literal> if the number
     <replaceable>e1</replaceable> is less than the number
@@ -894,9 +850,8 @@ x: x + 456</programlisting>
   </varlistentry>
 
 
-  <varlistentry xml:id='builtin-listToAttrs'>
-    <term><function>builtins.listToAttrs</function>
-    <replaceable>e</replaceable></term>
+  <varlistentry><term><function>builtins.listToAttrs</function>
+  <replaceable>e</replaceable></term>
 
     <listitem><para>Construct a set from a list specifying the names
     and values of each attribute.  Each element of the list should be
@@ -922,9 +877,8 @@ builtins.listToAttrs
 
   </varlistentry>
 
-  <varlistentry xml:id='builtin-map'>
-    <term><function>map</function>
-    <replaceable>f</replaceable> <replaceable>list</replaceable></term>
+  <varlistentry><term><function>map</function>
+  <replaceable>f</replaceable> <replaceable>list</replaceable></term>
 
     <listitem><para>Apply the function <replaceable>f</replaceable> to
     each element in the list <replaceable>list</replaceable>.  For
@@ -939,15 +893,14 @@ map (x: "foo" + x) [ "bar" "bla" "abc" ]</programlisting>
   </varlistentry>
 
 
-  <varlistentry xml:id='builtin-match'>
-    <term><function>builtins.match</function>
-    <replaceable>regex</replaceable> <replaceable>str</replaceable></term>
+  <varlistentry><term><function>builtins.match</function>
+  <replaceable>regex</replaceable> <replaceable>str</replaceable></term>
 
-    <listitem><para>Returns a list if the <link
-    xlink:href="http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap09.html#tag_09_04">extended
-    POSIX regular expression</link> <replaceable>regex</replaceable>
-    matches <replaceable>str</replaceable> precisely, otherwise returns
-    <literal>null</literal>.  Each item in the list is a regex group.
+  <listitem><para>Returns a list if the <link
+  xlink:href="http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap09.html#tag_09_04">extended
+  POSIX regular expression</link> <replaceable>regex</replaceable>
+  matches <replaceable>str</replaceable> precisely, otherwise returns
+  <literal>null</literal>.  Each item in the list is a regex group.
 
 <programlisting>
 builtins.match "ab" "abc"
@@ -973,12 +926,11 @@ builtins.match "[[:space:]]+([[:upper:]]+)[[:space:]]+" "  FOO   "
 
 Evaluates to <literal>[ "foo" ]</literal>.
 
-    </para></listitem>
+  </para></listitem>
   </varlistentry>
 
-  <varlistentry xml:id='builtin-mul'>
-    <term><function>builtins.mul</function>
-    <replaceable>e1</replaceable> <replaceable>e2</replaceable></term>
+  <varlistentry><term><function>builtins.mul</function>
+  <replaceable>e1</replaceable> <replaceable>e2</replaceable></term>
 
     <listitem><para>Return the product of the numbers
     <replaceable>e1</replaceable> and
@@ -987,9 +939,8 @@ Evaluates to <literal>[ "foo" ]</literal>.
   </varlistentry>
 
 
-  <varlistentry xml:id='builtin-parseDrvName'>
-    <term><function>builtins.parseDrvName</function>
-    <replaceable>s</replaceable></term>
+  <varlistentry><term><function>builtins.parseDrvName</function>
+  <replaceable>s</replaceable></term>
 
     <listitem><para>Split the string <replaceable>s</replaceable> into
     a package name and version.  The package name is everything up to
@@ -1002,7 +953,7 @@ Evaluates to <literal>[ "foo" ]</literal>.
 
   </varlistentry>
 
-  <varlistentry xml:id='builtin-path'>
+  <varlistentry>
     <term>
       <function>builtins.path</function>
       <replaceable>args</replaceable>
@@ -1072,20 +1023,32 @@ Evaluates to <literal>[ "foo" ]</literal>.
     </listitem>
   </varlistentry>
 
-  <varlistentry xml:id='builtin-pathExists'>
-    <term><function>builtins.pathExists</function>
-    <replaceable>path</replaceable></term>
+  <varlistentry><term><function>builtins.pathExists</function>
+  <replaceable>path</replaceable></term>
 
     <listitem><para>Return <literal>true</literal> if the path
-    <replaceable>path</replaceable> exists at evaluation time, and
-    <literal>false</literal> otherwise.</para></listitem>
+    <replaceable>path</replaceable> exists, and
+    <literal>false</literal> otherwise.  One application of this
+    function is to conditionally include a Nix expression containing
+    user configuration:
+
+<programlisting>
+let
+  fileName = builtins.getEnv "CONFIG_FILE";
+  config =
+    if fileName != "" &amp;&amp; builtins.pathExists (builtins.toPath fileName)
+    then import (builtins.toPath fileName)
+    else { someSetting = false; }; <lineannotation># default configuration</lineannotation>
+in config.someSetting</programlisting>
+
+    (Note that <envar>CONFIG_FILE</envar> must be an absolute path for
+    this to work.)</para></listitem>
 
   </varlistentry>
 
 
-  <varlistentry xml:id='builtin-readDir'>
-    <term><function>builtins.readDir</function>
-    <replaceable>path</replaceable></term>
+  <varlistentry><term><function>builtins.readDir</function>
+  <replaceable>path</replaceable></term>
 
     <listitem><para>Return the contents of the directory
     <replaceable>path</replaceable> as a set mapping directory entries
@@ -1106,9 +1069,8 @@ Evaluates to <literal>[ "foo" ]</literal>.
   </varlistentry>
 
 
-  <varlistentry xml:id='builtin-readFile'>
-    <term><function>builtins.readFile</function>
-    <replaceable>path</replaceable></term>
+  <varlistentry><term><function>builtins.readFile</function>
+  <replaceable>path</replaceable></term>
 
     <listitem><para>Return the contents of the file
     <replaceable>path</replaceable> as a string.</para></listitem>
@@ -1116,9 +1078,8 @@ Evaluates to <literal>[ "foo" ]</literal>.
   </varlistentry>
 
 
-  <varlistentry xml:id='builtin-removeAttrs'>
-    <term><function>removeAttrs</function>
-    <replaceable>set</replaceable> <replaceable>list</replaceable></term>
+  <varlistentry><term><function>removeAttrs</function>
+  <replaceable>set</replaceable> <replaceable>list</replaceable></term>
 
     <listitem><para>Remove the attributes listed in
     <replaceable>list</replaceable> from
@@ -1133,9 +1094,8 @@ removeAttrs { x = 1; y = 2; z = 3; } [ "a" "x" "z" ]</programlisting>
   </varlistentry>
 
 
-  <varlistentry xml:id='builtin-replaceStrings'>
-    <term><function>builtins.replaceStrings</function>
-    <replaceable>from</replaceable> <replaceable>to</replaceable> <replaceable>s</replaceable></term>
+  <varlistentry><term><function>builtins.replaceStrings</function>
+  <replaceable>from</replaceable> <replaceable>to</replaceable> <replaceable>s</replaceable></term>
 
     <listitem><para>Given string <replaceable>s</replaceable>, replace
     every occurrence of the strings in <replaceable>from</replaceable>
@@ -1151,9 +1111,8 @@ builtins.replaceStrings ["oo" "a"] ["a" "i"] "foobar"
   </varlistentry>
 
 
-  <varlistentry xml:id='builtin-seq'>
-    <term><function>builtins.seq</function>
-    <replaceable>e1</replaceable> <replaceable>e2</replaceable></term>
+  <varlistentry><term><function>builtins.seq</function>
+  <replaceable>e1</replaceable> <replaceable>e2</replaceable></term>
 
     <listitem><para>Evaluate <replaceable>e1</replaceable>, then
     evaluate and return <replaceable>e2</replaceable>. This ensures
@@ -1163,9 +1122,8 @@ builtins.replaceStrings ["oo" "a"] ["a" "i"] "foobar"
   </varlistentry>
 
 
-  <varlistentry xml:id='builtin-sort'>
-    <term><function>builtins.sort</function>
-    <replaceable>comparator</replaceable> <replaceable>list</replaceable></term>
+  <varlistentry><term><function>builtins.sort</function>
+  <replaceable>comparator</replaceable> <replaceable>list</replaceable></term>
 
     <listitem><para>Return <replaceable>list</replaceable> in sorted
     order. It repeatedly calls the function
@@ -1187,16 +1145,15 @@ builtins.sort builtins.lessThan [ 483 249 526 147 42 77 ]
   </varlistentry>
 
 
-  <varlistentry xml:id='builtin-split'>
-    <term><function>builtins.split</function>
-    <replaceable>regex</replaceable> <replaceable>str</replaceable></term>
+  <varlistentry><term><function>builtins.split</function>
+  <replaceable>regex</replaceable> <replaceable>str</replaceable></term>
 
-    <listitem><para>Returns a list composed of non matched strings interleaved
-    with the lists of the <link
-    xlink:href="http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap09.html#tag_09_04">extended
-    POSIX regular expression</link> <replaceable>regex</replaceable> matches
-    of <replaceable>str</replaceable>. Each item in the lists of matched
-    sequences is a regex group.
+  <listitem><para>Returns a list composed of non matched strings interleaved
+  with the lists of the <link
+  xlink:href="http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap09.html#tag_09_04">extended
+  POSIX regular expression</link> <replaceable>regex</replaceable> matches
+  of <replaceable>str</replaceable>. Each item in the lists of matched
+  sequences is a regex group.
 
 <programlisting>
 builtins.split "(a)b" "abc"
@@ -1222,12 +1179,11 @@ builtins.split "([[:upper:]]+)" "  FOO   "
 
 Evaluates to <literal>[ "  " [ "FOO" ] "   " ]</literal>.
 
-    </para></listitem>
+  </para></listitem>
   </varlistentry>
 
-  <varlistentry xml:id='builtin-stringLength'>
-    <term><function>builtins.stringLength</function>
-    <replaceable>e</replaceable></term>
+  <varlistentry><term><function>builtins.stringLength</function>
+  <replaceable>e</replaceable></term>
 
     <listitem><para>Return the length of the string
     <replaceable>e</replaceable>.  If <replaceable>e</replaceable> is
@@ -1236,9 +1192,8 @@ Evaluates to <literal>[ "  " [ "FOO" ] "   " ]</literal>.
   </varlistentry>
 
 
-  <varlistentry xml:id='builtin-sub'>
-    <term><function>builtins.sub</function>
-    <replaceable>e1</replaceable> <replaceable>e2</replaceable></term>
+  <varlistentry><term><function>builtins.sub</function>
+  <replaceable>e1</replaceable> <replaceable>e2</replaceable></term>
 
     <listitem><para>Return the difference between the numbers
     <replaceable>e1</replaceable> and
@@ -1247,10 +1202,9 @@ Evaluates to <literal>[ "  " [ "FOO" ] "   " ]</literal>.
   </varlistentry>
 
 
-  <varlistentry xml:id='builtin-substring'>
-    <term><function>builtins.substring</function>
-    <replaceable>start</replaceable> <replaceable>len</replaceable>
-    <replaceable>s</replaceable></term>
+  <varlistentry><term><function>builtins.substring</function>
+  <replaceable>start</replaceable> <replaceable>len</replaceable>
+  <replaceable>s</replaceable></term>
 
     <listitem><para>Return the substring of
     <replaceable>s</replaceable> from character position
@@ -1273,9 +1227,8 @@ builtins.substring 0 3 "nixos"
   </varlistentry>
 
 
-  <varlistentry xml:id='builtin-tail'>
-    <term><function>builtins.tail</function>
-    <replaceable>list</replaceable></term>
+  <varlistentry><term><function>builtins.tail</function>
+  <replaceable>list</replaceable></term>
 
     <listitem><para>Return the second to last elements of a list;
     abort evaluation if the argument isn’t a list or is an empty
@@ -1284,9 +1237,8 @@ builtins.substring 0 3 "nixos"
   </varlistentry>
 
 
-  <varlistentry xml:id='builtin-throw'>
-    <term><function>throw</function>
-    <replaceable>s</replaceable></term>
+  <varlistentry><term><function>throw</function>
+  <replaceable>s</replaceable></term>
 
     <listitem><para>Throw an error message
     <replaceable>s</replaceable>.  This usually aborts Nix expression
@@ -1299,10 +1251,9 @@ builtins.substring 0 3 "nixos"
   </varlistentry>
 
 
-  <varlistentry xml:id='builtin-toFile'>
-    <term><function>builtins.toFile</function>
-    <replaceable>name</replaceable>
-    <replaceable>s</replaceable></term>
+  <varlistentry
+  xml:id='builtin-toFile'><term><function>builtins.toFile</function>
+  <replaceable>name</replaceable> <replaceable>s</replaceable></term>
 
     <listitem><para>Store the string <replaceable>s</replaceable> in a
     file in the Nix store and return its path.  The file has suffix
@@ -1378,8 +1329,7 @@ in foo</programlisting>
   </varlistentry>
 
 
-  <varlistentry xml:id='builtin-toJSON'>
-    <term><function>builtins.toJSON</function> <replaceable>e</replaceable></term>
+  <varlistentry><term><function>builtins.toJSON</function> <replaceable>e</replaceable></term>
 
     <listitem><para>Return a string containing a JSON representation
     of <replaceable>e</replaceable>.  Strings, integers, floats, booleans,
@@ -1392,19 +1342,20 @@ in foo</programlisting>
   </varlistentry>
 
 
-  <varlistentry xml:id='builtin-toPath'>
-    <term><function>builtins.toPath</function> <replaceable>s</replaceable></term>
+  <varlistentry><term><function>builtins.toPath</function> <replaceable>s</replaceable></term>
 
-    <listitem><para> DEPRECATED. Use <literal>/. + "/path"</literal>
-    to convert a string into an absolute path. For relative paths,
-    use <literal>./. + "/path"</literal>.
-    </para></listitem>
+    <listitem><para>Convert the string value
+    <replaceable>s</replaceable> into a path value.  The string
+    <replaceable>s</replaceable> must represent an absolute path
+    (i.e., must start with <literal>/</literal>).  The path need not
+    exist.  The resulting path is canonicalised, e.g.,
+    <literal>builtins.toPath "//foo/xyzzy/../bar/"</literal> returns
+    <literal>/foo/bar</literal>.</para></listitem>
 
   </varlistentry>
 
 
-  <varlistentry xml:id='builtin-toString'>
-    <term><function>toString</function> <replaceable>e</replaceable></term>
+  <varlistentry><term><function>toString</function> <replaceable>e</replaceable></term>
 
     <listitem><para>Convert the expression
     <replaceable>e</replaceable> to a string.
@@ -1423,8 +1374,7 @@ in foo</programlisting>
   </varlistentry>
 
 
-  <varlistentry xml:id='builtin-toXML'>
-    <term><function>builtins.toXML</function> <replaceable>e</replaceable></term>
+  <varlistentry xml:id='builtin-toXML'><term><function>builtins.toXML</function> <replaceable>e</replaceable></term>
 
     <listitem><para>Return a string containing an XML representation
     of <replaceable>e</replaceable>.  The main application for
@@ -1480,7 +1430,7 @@ stdenv.mkDerivation (rec {
   builder = builtins.toFile "builder.sh" "
     source $stdenv/setup
     mkdir $out
-    echo "$servlets" | xsltproc ${stylesheet} - > $out/server-conf.xml]]> <co xml:id='ex-toxml-co-apply' /> <![CDATA[
+    echo $servlets | xsltproc ${stylesheet} - > $out/server-conf.xml]]> <co xml:id='ex-toxml-co-apply' /> <![CDATA[
   ";
 
   stylesheet = builtins.toFile "stylesheet.xsl"]]> <co xml:id='ex-toxml-co-stylesheet' /> <![CDATA[
@@ -1539,9 +1489,8 @@ stdenv.mkDerivation (rec {
   </varlistentry>
 
 
-  <varlistentry xml:id='builtin-trace'>
-    <term><function>builtins.trace</function>
-    <replaceable>e1</replaceable> <replaceable>e2</replaceable></term>
+  <varlistentry><term><function>builtins.trace</function>
+  <replaceable>e1</replaceable> <replaceable>e2</replaceable></term>
 
     <listitem><para>Evaluate <replaceable>e1</replaceable> and print its
     abstract syntax representation on standard error.  Then return
@@ -1550,9 +1499,8 @@ stdenv.mkDerivation (rec {
 
   </varlistentry>
 
-  <varlistentry xml:id='builtin-tryEval'>
-    <term><function>builtins.tryEval</function>
-    <replaceable>e</replaceable></term>
+  <varlistentry><term><function>builtins.tryEval</function>
+  <replaceable>e</replaceable></term>
 
     <listitem><para>Try to evaluate <replaceable>e</replaceable>.
     Return a set containing the attributes <literal>success</literal>
@@ -1565,9 +1513,8 @@ stdenv.mkDerivation (rec {
   </varlistentry>
 
 
-  <varlistentry xml:id='builtin-typeOf'>
-    <term><function>builtins.typeOf</function>
-    <replaceable>e</replaceable></term>
+  <varlistentry><term><function>builtins.typeOf</function>
+  <replaceable>e</replaceable></term>
 
     <listitem><para>Return a string representing the type of the value
     <replaceable>e</replaceable>, namely <literal>"int"</literal>,

doc/manual/installation/prerequisites-source.xml

@@ -25,18 +25,11 @@
   If your distribution does not provide it, you can get it from <link
   xlink:href="https://www.openssl.org"/>.</para></listitem>
 
-  <listitem><para>The <literal>libbrotlienc</literal> and
-  <literal>libbrotlidec</literal> libraries to provide implementation
-  of the Brotli compression algorithm. They are available for download
-  from the official repository <link
-  xlink:href="https://github.com/google/brotli" />.</para></listitem>
-
   <listitem><para>The bzip2 compressor program and the
   <literal>libbz2</literal> library.  Thus you must have bzip2
   installed, including development headers and libraries.  If your
   distribution does not provide these, you can obtain bzip2 from <link
-  xlink:href="https://web.archive.org/web/20180624184756/http://www.bzip.org/"
-  />.</para></listitem>
+  xlink:href="http://www.bzip.org/"/>.</para></listitem>
 
   <listitem><para><literal>liblzma</literal>, which is provided by
   XZ Utils. If your distribution does not provide this, you can
@@ -58,10 +51,6 @@
   pass the flag <option>--enable-gc</option> to
   <command>configure</command>.</para></listitem>
 
-  <listitem><para>The <literal>boost</literal> library of version
-  1.66.0 or higher. It can be obtained from the official web site
-  <link xlink:href="https://www.boost.org/" />.</para></listitem>
-
   <listitem><para>The <command>xmllint</command> and
   <command>xsltproc</command> programs to build this manual and the
   man-pages.  These are part of the <literal>libxml2</literal> and
@@ -87,15 +76,6 @@
   modify the parser or when you are building from the Git
   repository.</para></listitem>
 
-  <listitem><para>The <literal>libseccomp</literal> is used to provide
-  syscall filtering on Linux. This is an optional dependency and can
-  be disabled passing a <option>--disable-seccomp-sandboxing</option>
-  option to the <command>configure</command> script (Not recommended
-  unless your system doesn't support
-  <literal>libseccomp</literal>). To get the library, visit <link
-  xlink:href="https://github.com/seccomp/libseccomp"
-  />.</para></listitem>
-
 </itemizedlist>
 
 </section>

doc/manual/installation/upgrading.xml

@@ -7,16 +7,15 @@
   <title>Upgrading Nix</title>
 
   <para>
-    Multi-user Nix users on macOS can upgrade Nix by running:
-    <command>sudo -i sh -c 'nix-channel --update &amp;&amp;
-    nix-env -iA nixpkgs.nix &amp;&amp;
-    launchctl remove org.nixos.nix-daemon &amp;&amp;
-    launchctl load /Library/LaunchDaemons/org.nixos.nix-daemon.plist'</command>
+    Multi-user Nix users on macOS can upgrade Nix by running
+    <command>sudo -i sh -c 'nix-channel --update &amp;&amp; nix-env
+    -iA nixpkgs.nix'; sudo launchctl stop org.nixos.nix-daemon; sudo
+    launchctl start org.nixos.nix-daemon</command>.
   </para>
 
-
   <para>
-    Single-user installations of Nix should run this:
-    <command>nix-channel --update; nix-env -iA nixpkgs.nix</command>
+    Single-user installations of Nix should run <command>nix-channel
+    --update; nix-env -iA nixpkgs.nix</command>.
   </para>
+
 </chapter>

doc/manual/introduction/about-nix.xml

@@ -262,6 +262,12 @@ xlink:href="http://nixos.org/">NixOS homepage</link>.</para>
 xlink:href="http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html">GNU
 LGPLv2.1 or (at your option) any later version</link>.</para>
 
+<para>Nix uses the <link
+xlink:href="https://github.com/arangodb/linenoise-ng">linenoise-ng
+library</link>, which has the following license:</para>
+
+<programlisting><xi:include href="../../../src/linenoise/LICENSE" parse="text" /></programlisting>
+
 </simplesect>
 
 

doc/manual/packages/s3-substituter.xml

@@ -12,61 +12,8 @@ from Amazon S3 and S3 compatible services. This uses the same
 <emphasis>binary</emphasis> cache mechanism that Nix usually uses to
 fetch prebuilt binaries from <uri>cache.nixos.org</uri>.</para>
 
-<para>The following options can be specified as URL parameters to
-the S3 URL:</para>
-
-<variablelist>
-  <varlistentry><term><literal>profile</literal></term>
-  <listitem>
-    <para>
-      The name of the AWS configuration profile to use. By default
-      Nix will use the <literal>default</literal> profile.
-    </para>
-  </listitem>
-  </varlistentry>
-
-  <varlistentry><term><literal>region</literal></term>
-  <listitem>
-    <para>
-      The region of the S3 bucket. <literal>us–east-1</literal> by
-      default.
-    </para>
-
-    <para>
-      If your bucket is not in <literal>us–east-1</literal>, you
-      should always explicitly specify the region parameter.
-    </para>
-  </listitem>
-  </varlistentry>
-
-  <varlistentry><term><literal>endpoint</literal></term>
-  <listitem>
-    <para>
-      The URL to your S3-compatible service, for when not using
-      Amazon S3. Do not specify this value if you're using Amazon
-      S3.
-    </para>
-    <note><para>This endpoint must support HTTPS and will use
-    path-based addressing instead of virtual host based
-    addressing.</para></note>
-  </listitem>
-  </varlistentry>
-
-  <varlistentry><term><literal>scheme</literal></term>
-  <listitem>
-    <para>
-      The scheme used for S3 requests, <literal>https</literal>
-      (default) or <literal>http</literal>.  This option allows you to
-      disable HTTPS for binary caches which don't support it.
-    </para>
-    <note><para>HTTPS should be used if the cache might contain
-    sensitive information.</para></note>
-  </listitem>
-  </varlistentry>
-</variablelist>
-
 <para>In this example we will use the bucket named
-<literal>example-nix-cache</literal>.</para>
+<literal>example-bucket</literal>.</para>
 
 <section xml:id="ssec-s3-substituter-anonymous-reads">
   <title>Anonymous Reads to your S3-compatible binary cache</title>
@@ -77,56 +24,65 @@ the S3 URL:</para>
   cache.</para>
 
   <para>For AWS S3 the binary cache URL for example bucket will be
-  exactly <uri>https://example-nix-cache.s3.amazonaws.com</uri> or
-  <uri>s3://example-nix-cache</uri>. For S3 compatible binary caches,
-  consult that cache's documentation.</para>
+  exactly <uri>https://example-bucket.s3.amazonaws.com</uri>. For S3
+  compatible binary caches ago have to consult your software's
+  documentation.</para>
 
   <para>Your bucket will need the following bucket policy:</para>
 
-  <programlisting><![CDATA[
+  <programlisting>
+<![CDATA[
 {
-    "Id": "DirectReads",
-    "Version": "2012-10-17",
-    "Statement": [
-        {
-            "Sid": "AlowDirectReads",
-            "Action": [
-                "s3:GetObject",
-                "s3:GetBucketLocation"
-            ],
-            "Effect": "Allow",
-            "Resource": [
-                "arn:aws:s3:::example-nix-cache",
-                "arn:aws:s3:::example-nix-cache/*"
-            ],
-            "Principal": "*"
-        }
-    ]
+  "Id": "DirectReads",
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Sid": "AlowDirectReads",
+      "Action": [
+        "s3:GetObject"
+      ],
+      "Effect": "Allow",
+      "Resource": "arn:aws:s3:::example-bucket/*",
+      "Principal": "*"
+    }
+  ]
 }
-]]></programlisting>
+]]>
+</programlisting>
 </section>
 
 <section xml:id="ssec-s3-substituter-authenticated-reads">
   <title>Authenticated Reads to your S3 binary cache</title>
 
   <para>For AWS S3 the binary cache URL for example bucket will be
-  exactly <uri>s3://example-nix-cache</uri>.</para>
+  exactly <uri>s3://example-bucket</uri>.</para>
 
   <para>Nix will use the <link
   xlink:href="https://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/credentials.html#credentials-default.">default
   credential provider chain</link> for authenticating requests to
   Amazon S3.</para>
 
-  <para>Nix supports authenticated reads from Amazon S3 and S3
-  compatible binary caches.</para>
+  <para>Nix supports authenticated writes to S3 compatible binary
+  caches but only supports Authenticated reads from Amazon S3.
+  Additionally, the following limitations are in place for
+  authenticated reads:</para>
+
+  <itemizedlist>
+    <listitem><para>The bucket must actually be hosted by Amazon S3 and
+    <emphasis>not</emphasis> an S3 compatible
+    service.</para></listitem>
+
+    <listitem><para>The bucket must be within the
+    <literal>us-east-1</literal> region.</para></listitem>
+
+    <listitem><para>The Amazon credentials, if stored in a credential
+    profile, must be stored in the <literal>default</literal>
+    profile.</para></listitem>
+  </itemizedlist>
 
   <para>Your bucket will need a bucket policy allowing the desired
-  users to perform the <literal>s3:GetObject</literal> and
-  <literal>s3:GetBucketLocation</literal> action on all objects in the
-  bucket. The anonymous policy in <xref
-  linkend="ssec-s3-substituter-anonymous-reads" /> can be updated to
-  have a restricted <literal>Principal</literal> to support
-  this.</para>
+  users to perform the <literal>s3:GetObject</literal> action on all
+  objects in the bucket.</para>
 </section>
 
 
@@ -135,49 +91,69 @@ the S3 URL:</para>
 
   <para>Nix support fully supports writing to Amazon S3 and S3
   compatible buckets. The binary cache URL for our example bucket will
-  be <uri>s3://example-nix-cache</uri>.</para>
+  be <uri>s3://example-bucket</uri>.</para>
 
   <para>Nix will use the <link
   xlink:href="https://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/credentials.html#credentials-default.">default
   credential provider chain</link> for authenticating requests to
   Amazon S3.</para>
 
-  <para>Your account will need the following IAM policy to
-  upload to the cache:</para>
+  <para>The following options can be specified as URL parameters to
+  the S3 URL:</para>
+  <variablelist>
+    <varlistentry><term><literal>profile</literal></term>
+    <listitem>
+      <para>
+        The name of the AWS configuration profile to use. By default
+        Nix will use the <literal>default</literal> profile.
+      </para>
+    </listitem>
+    </varlistentry>
 
-  <programlisting><![CDATA[
-{
-  "Version": "2012-10-17",
-  "Statement": [
-    {
-      "Sid": "UploadToCache",
-      "Effect": "Allow",
-      "Action": [
-        "s3:AbortMultipartUpload",
-        "s3:GetBucketLocation",
-        "s3:GetObject",
-        "s3:ListBucket",
-        "s3:ListBucketMultipartUploads",
-        "s3:ListMultipartUploadParts",
-        "s3:ListObjects",
-        "s3:PutObject"
-      ],
-      "Resource": [
-        "arn:aws:s3:::example-nix-cache",
-        "arn:aws:s3:::example-nix-cache/*"
-      ]
-    }
-  ]
-}
-]]></programlisting>
+    <varlistentry><term><literal>region</literal></term>
+    <listitem>
+      <para>
+        The region of the S3 bucket. <literal>us–east-1</literal> by
+        default.
+      </para>
+    </listitem>
+    </varlistentry>
 
+    <varlistentry><term><literal>endpoint</literal></term>
+    <listitem>
+      <para>
+        The URL to your S3-compatible service, for when not using
+        Amazon S3. Do not specify this value if you're using Amazon
+        S3.
+      </para>
+      <note><para>This endpoint must support HTTPS and will use
+      path-based addressing instead of virtual host based
+      addressing.</para></note>
+    </listitem>
+    </varlistentry>
+  </variablelist>
 
-  <example><title>Uploading with a specific credential profile for Amazon S3</title>
-    <para><command>nix copy --to 's3://example-nix-cache?profile=cache-upload&amp;region=eu-west-2' nixpkgs.hello</command></para>
+  <example><title>Uploading with non-default credential profile for Amazon S3</title>
+    <para><command>nix copy --to ssh://machine nixpkgs.hello s3://example-bucket?profile=cache-upload</command></para>
   </example>
 
   <example><title>Uploading to an S3-Compatible Binary Cache</title>
-    <para><command>nix copy --to 's3://example-nix-cache?profile=cache-upload&amp;scheme=https&amp;endpoint=minio.example.com' nixpkgs.hello</command></para>
+    <para><command>nix copy --to ssh://machine nixpkgs.hello s3://example-bucket?profile=cache-upload&amp;endpoint=minio.example.com</command></para>
   </example>
+
+  <para>The user writing to the bucket will need to perform the
+  following actions against the bucket:</para>
+
+  <itemizedlist>
+    <listitem><para><literal>s3:ListBucket</literal></para></listitem>
+    <listitem><para><literal>s3:GetBucketLocation</literal></para></listitem>
+    <listitem><para><literal>s3:ListObjects</literal></para></listitem>
+    <listitem><para><literal>s3:GetObject</literal></para></listitem>
+    <listitem><para><literal>s3:PutObject</literal></para></listitem>
+    <listitem><para><literal>s3:ListBucketMultipartUploads</literal></para></listitem>
+    <listitem><para><literal>s3:CreateMultipartUpload</literal></para></listitem>
+    <listitem><para><literal>s3:ListMultipartUploadParts</literal></para></listitem>
+    <listitem><para><literal>s3:AbortMultipartUpload</literal></para></listitem>
+  </itemizedlist>
 </section>
 </section>

doc/manual/release-notes/release-notes.xml

@@ -12,7 +12,6 @@
 </partintro>
 -->
 
-<xi:include href="rl-2.2.xml" />
 <xi:include href="rl-2.1.xml" />
 <xi:include href="rl-2.0.xml" />
 <xi:include href="rl-1.11.10.xml" />

doc/manual/release-notes/rl-2.2.xml

@@ -1,143 +0,0 @@
-<section xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id="ssec-relnotes-2.2">
-
-<title>Release 2.2 (2019-01-11)</title>
-
-<para>This is primarily a bug fix release. It also has the following
-changes:</para>
-
-<itemizedlist>
-
-  <listitem>
-    <para>In derivations that use structured attributes (i.e. that
-    specify set the <varname>__structuredAttrs</varname> attribute to
-    <literal>true</literal> to cause all attributes to be passed to
-    the builder in JSON format), you can now specify closure checks
-    per output, e.g.:
-
-<programlisting>
-outputChecks."out" = {
-  # The closure of 'out' must not be larger than 256 MiB.
-  maxClosureSize = 256 * 1024 * 1024;
-
-  # It must not refer to C compiler or to the 'dev' output.
-  disallowedRequisites = [ stdenv.cc "dev" ];
-};
-
-outputChecks."dev" = {
-  # The 'dev' output must not be larger than 128 KiB.
-  maxSize = 128 * 1024;
-};
-</programlisting>
-
-    </para>
-  </listitem>
-
-
-  <listitem>
-    <para>The derivation attribute
-    <varname>requiredSystemFeatures</varname> is now enforced for
-    local builds, and not just to route builds to remote builders.
-    The supported features of a machine can be specified through the
-    configuration setting <varname>system-features</varname>.</para>
-
-    <para>By default, <varname>system-features</varname> includes
-    <literal>kvm</literal> if <filename>/dev/kvm</filename>
-    exists. For compatibility, it also includes the pseudo-features
-    <literal>nixos-test</literal>, <literal>benchmark</literal> and
-    <literal>big-parallel</literal> which are used by Nixpkgs to route
-    builds to particular Hydra build machines.</para>
-
-  </listitem>
-
-  <listitem>
-    <para>Sandbox builds are now enabled by default on Linux.</para>
-  </listitem>
-
-  <listitem>
-    <para>The new command <command>nix doctor</command> shows
-    potential issues with your Nix installation.</para>
-  </listitem>
-
-  <listitem>
-    <para>The <literal>fetchGit</literal> builtin function now uses a
-    caching scheme that puts different remote repositories in distinct
-    local repositories, rather than a single shared repository. This
-    may require more disk space but is faster.</para>
-  </listitem>
-
-  <listitem>
-    <para>The <literal>dirOf</literal> builtin function now works on
-    relative paths.</para>
-  </listitem>
-
-  <listitem>
-    <para>Nix now supports <link
-    xlink:href="https://www.w3.org/TR/SRI/">SRI hashes</link>,
-    allowing the hash algorithm and hash to be specified in a single
-    string. For example, you can write:
-
-<programlisting>
-import &lt;nix/fetchurl.nix> {
-  url = https://nixos.org/releases/nix/nix-2.1.3/nix-2.1.3.tar.xz;
-  hash = "sha256-XSLa0FjVyADWWhFfkZ2iKTjFDda6mMXjoYMXLRSYQKQ=";
-};
-</programlisting>
-
-    instead of
-
-<programlisting>
-import &lt;nix/fetchurl.nix> {
-  url = https://nixos.org/releases/nix/nix-2.1.3/nix-2.1.3.tar.xz;
-  sha256 = "5d22dad058d5c800d65a115f919da22938c50dd6ba98c5e3a183172d149840a4";
-};
-</programlisting>
-
-    </para>
-
-    <para>In fixed-output derivations, the
-    <varname>outputHashAlgo</varname> attribute is no longer mandatory
-    if <varname>outputHash</varname> specifies the hash.</para>
-
-    <para><command>nix hash-file</command> and <command>nix
-    hash-path</command> now print hashes in SRI format by
-    default. They also use SHA-256 by default instead of SHA-512
-    because that's what we use most of the time in Nixpkgs.</para>
-  </listitem>
-
-  <listitem>
-    <para>Integers are now 64 bits on all platforms.</para>
-  </listitem>
-
-  <listitem>
-    <para>The evaluator now prints profiling statistics (enabled via
-    the <envar>NIX_SHOW_STATS</envar> and
-    <envar>NIX_COUNT_CALLS</envar> environment variables) in JSON
-    format.</para>
-  </listitem>
-
-  <listitem>
-    <para>The option <option>--xml</option> in <command>nix-store
-    --query</command> has been removed. Instead, there now is an
-    option <option>--graphml</option> to output the dependency graph
-    in GraphML format.</para>
-  </listitem>
-
-  <listitem>
-    <para>All <filename>nix-*</filename> commands are now symlinks to
-    <filename>nix</filename>. This saves a bit of disk space.</para>
-  </listitem>
-
-  <listitem>
-    <para><command>nix repl</command> now uses
-    <literal>libeditline</literal> or
-    <literal>libreadline</literal>.</para>
-  </listitem>
-
-</itemizedlist>
-
-</section>
-

local.mk

@@ -6,7 +6,7 @@ dist-files += configure config.h.in nix.spec perl/configure
 
 clean-files += Makefile.config
 
-GLOBAL_CXXFLAGS += -I . -I src -I src/libutil -I src/libstore -I src/libmain -I src/libexpr -I src/nix
+GLOBAL_CXXFLAGS += -I . -I src -I src/libutil -I src/libstore -I src/libmain -I src/libexpr
 
 $(foreach i, config.h $(call rwildcard, src/lib*, *.hh), \
   $(eval $(call install-file-in, $(i), $(includedir)/nix, 0644)))

maintainers/upload-release.pl

@@ -91,8 +91,6 @@ downloadFile("binaryTarball.aarch64-linux", "1");
 downloadFile("binaryTarball.x86_64-darwin", "1");
 downloadFile("installerScript", "1");
 
-exit if $version =~ /pre/;
-
 # Update Nixpkgs in a very hacky way.
 system("cd $nixpkgsDir && git pull") == 0 or die;
 my $oldName = `nix-instantiate --eval $nixpkgsDir -A nix.name`; chomp $oldName;

release-common.nix

@@ -49,7 +49,7 @@ rec {
 
   buildDeps =
     [ curl
-      bzip2 xz brotli editline
+      bzip2 xz brotli
       openssl pkgconfig sqlite boehmgc
       boost
 
@@ -64,12 +64,10 @@ rec {
         apis = ["s3" "transfer"];
         customMemoryManagement = false;
       }).overrideDerivation (args: {
-        /*
         patches = args.patches or [] ++ [ (fetchpatch {
           url = https://github.com/edolstra/aws-sdk-cpp/commit/3e07e1f1aae41b4c8b340735ff9e8c735f0c063f.patch;
           sha256 = "1pij0v449p166f9l29x7ppzk8j7g9k9mp15ilh5qxp29c7fnvxy2";
         }) ];
-        */
       }));
 
   perlDeps =

release.nix

@@ -1,5 +1,5 @@
 { nix ? builtins.fetchGit ./.
-, nixpkgs ? builtins.fetchGit { url = https://github.com/NixOS/nixpkgs-channels.git; ref = "nixos-18.09"; }
+, nixpkgs ? builtins.fetchGit { url = https://github.com/NixOS/nixpkgs-channels.git; ref = "nixos-18.03"; }
 , officialRelease ? false
 , systems ? [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ]
 }:

scripts/install.in

@@ -11,14 +11,14 @@ oops() {
 }
 
 tmpDir="$(mktemp -d -t nix-binary-tarball-unpack.XXXXXXXXXX || \
-          oops "Can't create temporary directory for downloading the Nix binary tarball")"
+          oops "Can\'t create temporary directory for downloading the Nix binary tarball")"
 cleanup() {
     rm -rf "$tmpDir"
 }
 trap cleanup EXIT INT QUIT TERM
 
 require_util() {
-    type "$1" > /dev/null 2>&1 || command -v "$1" > /dev/null 2>&1 ||
+    type "$1" > /dev/null 2>&1 || which "$1" > /dev/null 2>&1 ||
         oops "you do not have '$1' installed, which I need to $2"
 }
 

scripts/nix-profile-daemon.sh.in

@@ -61,14 +61,11 @@ elif [ -e /etc/ssl/certs/ca-bundle.crt ]; then # Old NixOS
     export NIX_SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt
 elif [ -e /etc/pki/tls/certs/ca-bundle.crt ]; then # Fedora, CentOS
     export NIX_SSL_CERT_FILE=/etc/pki/tls/certs/ca-bundle.crt
-else
-  # Fall back to what is in the nix profiles, favouring whatever is defined last.
-  for i in $NIX_PROFILES; do
-    if [ -e $i/etc/ssl/certs/ca-bundle.crt ]; then
-      export NIX_SSL_CERT_FILE=$i/etc/ssl/certs/ca-bundle.crt
-    fi
-  done
+elif [ -e "$NIX_USER_PROFILE_DIR/etc/ssl/certs/ca-bundle.crt" ]; then # fall back to cacert in the user's Nix profile
+    export NIX_SSL_CERT_FILE=$NIX_USER_PROFILE_DIR/etc/ssl/certs/ca-bundle.crt
+elif [ -e "/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt" ]; then # fall back to cacert in the default Nix profile
+    export NIX_SSL_CERT_FILE=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt
 fi
 
 export NIX_PATH="nixpkgs=@localstatedir@/nix/profiles/per-user/root/channels/nixpkgs:@localstatedir@/nix/profiles/per-user/root/channels"
-export PATH="$HOME/.nix-profile/bin:@localstatedir@/nix/profiles/default/bin:$PATH"
+export PATH="$HOME/.nix-profile/bin:$HOME/.nix-profile/lib/kde4/libexec:@localstatedir@/nix/profiles/default/bin:@localstatedir@/nix/profiles/default:@localstatedir@/nix/profiles/default/lib/kde4/libexec:$PATH"

shell.nix

@@ -1,6 +1,6 @@
 { useClang ? false }:
 
-with import (builtins.fetchGit { url = https://github.com/NixOS/nixpkgs-channels.git; ref = "nixos-18.09"; }) {};
+with import (builtins.fetchGit { url = https://github.com/NixOS/nixpkgs-channels.git; ref = "nixos-18.03"; }) {};
 
 with import ./release-common.nix { inherit pkgs; };
 

src/build-remote/build-remote.cc

@@ -17,7 +17,6 @@
 #include "store-api.hh"
 #include "derivations.hh"
 #include "local-store.hh"
-#include "legacy.hh"
 
 using namespace nix;
 using std::cin;
@@ -38,9 +37,11 @@ static AutoCloseFD openSlotLock(const Machine & m, unsigned long long slot)
     return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri), slot), true);
 }
 
-static int _main(int argc, char * * argv)
+int main (int argc, char * * argv)
 {
-    {
+    return handleExceptions(argv[0], [&]() {
+        initNix();
+
         logger = makeJSONLogger(*logger);
 
         /* Ensure we don't get any SSH passphrase or host key popups. */
@@ -79,7 +80,7 @@ static int _main(int argc, char * * argv)
 
         if (machines.empty()) {
             std::cerr << "# decline-permanently\n";
-            return 0;
+            return;
         }
 
         string drvPath;
@@ -89,8 +90,8 @@ static int _main(int argc, char * * argv)
 
             try {
                 auto s = readString(source);
-                if (s != "try") return 0;
-            } catch (EndOfFile &) { return 0; }
+                if (s != "try") return;
+            } catch (EndOfFile &) { return; }
 
             auto amWilling = readInt(source);
             auto neededSystem = readString(source);
@@ -252,8 +253,6 @@ connected:
             copyPaths(ref<Store>(sshStore), store, missing, NoRepair, NoCheckSigs, NoSubstitute);
         }
 
-        return 0;
-    }
+        return;
+    });
 }
-
-static RegisterLegacyCommand s1("build-remote", _main);

src/build-remote/local.mk

@@ -0,0 +1,9 @@
+programs += build-remote
+
+build-remote_DIR := $(d)
+
+build-remote_INSTALL_DIR := $(libexecdir)/nix
+
+build-remote_LIBS = libmain libformat libstore libutil
+
+build-remote_SOURCES := $(d)/build-remote.cc

src/libexpr/eval.cc

@@ -6,15 +6,12 @@
 #include "globals.hh"
 #include "eval-inline.hh"
 #include "download.hh"
-#include "json.hh"
 
 #include <algorithm>
 #include <cstring>
 #include <unistd.h>
 #include <sys/time.h>
 #include <sys/resource.h>
-#include <iostream>
-#include <fstream>
 
 #include <sys/time.h>
 #include <sys/resource.h>
@@ -26,6 +23,7 @@
 
 #endif
 
+
 namespace nix {
 
 
@@ -757,7 +755,6 @@ void EvalState::evalFile(const Path & path_, Value & v)
 void EvalState::resetFileCache()
 {
     fileEvalCache.clear();
-    fileParseCache.clear();
 }
 
 
@@ -1726,9 +1723,12 @@ bool EvalState::eqValues(Value & v1, Value & v2)
     }
 }
 
+
 void EvalState::printStats()
 {
     bool showStats = getEnv("NIX_SHOW_STATS", "0") != "0";
+    Verbosity v = showStats ? lvlInfo : lvlDebug;
+    printMsg(v, "evaluation statistics:");
 
     struct rusage buf;
     getrusage(RUSAGE_SELF, &buf);
@@ -1739,101 +1739,62 @@ void EvalState::printStats()
     uint64_t bValues = nrValues * sizeof(Value);
     uint64_t bAttrsets = nrAttrsets * sizeof(Bindings) + nrAttrsInAttrsets * sizeof(Attr);
 
+    printMsg(v, format("  time elapsed: %1%") % cpuTime);
+    printMsg(v, format("  size of a value: %1%") % sizeof(Value));
+    printMsg(v, format("  size of an attr: %1%") % sizeof(Attr));
+    printMsg(v, format("  environments allocated count: %1%") % nrEnvs);
+    printMsg(v, format("  environments allocated bytes: %1%") % bEnvs);
+    printMsg(v, format("  list elements count: %1%") % nrListElems);
+    printMsg(v, format("  list elements bytes: %1%") % bLists);
+    printMsg(v, format("  list concatenations: %1%") % nrListConcats);
+    printMsg(v, format("  values allocated count: %1%") % nrValues);
+    printMsg(v, format("  values allocated bytes: %1%") % bValues);
+    printMsg(v, format("  sets allocated: %1% (%2% bytes)") % nrAttrsets % bAttrsets);
+    printMsg(v, format("  right-biased unions: %1%") % nrOpUpdates);
+    printMsg(v, format("  values copied in right-biased unions: %1%") % nrOpUpdateValuesCopied);
+    printMsg(v, format("  symbols in symbol table: %1%") % symbols.size());
+    printMsg(v, format("  size of symbol table: %1%") % symbols.totalSize());
+    printMsg(v, format("  number of thunks: %1%") % nrThunks);
+    printMsg(v, format("  number of thunks avoided: %1%") % nrAvoided);
+    printMsg(v, format("  number of attr lookups: %1%") % nrLookups);
+    printMsg(v, format("  number of primop calls: %1%") % nrPrimOpCalls);
+    printMsg(v, format("  number of function calls: %1%") % nrFunctionCalls);
+    printMsg(v, format("  total allocations: %1% bytes") % (bEnvs + bLists + bValues + bAttrsets));
+
 #if HAVE_BOEHMGC
     GC_word heapSize, totalBytes;
     GC_get_heap_usage_safe(&heapSize, 0, 0, 0, &totalBytes);
+    printMsg(v, format("  current Boehm heap size: %1% bytes") % heapSize);
+    printMsg(v, format("  total Boehm heap allocations: %1% bytes") % totalBytes);
 #endif
-    if (showStats) {
-        auto outPath = getEnv("NIX_SHOW_STATS_PATH","-");
-        std::fstream fs;
-        if (outPath != "-")
-            fs.open(outPath, std::fstream::out);
-        JSONObject topObj(outPath == "-" ? std::cerr : fs, true);
-        topObj.attr("cpuTime",cpuTime);
-        {
-            auto envs = topObj.object("envs");
-            envs.attr("number", nrEnvs);
-            envs.attr("elements", nrValuesInEnvs);
-            envs.attr("bytes", bEnvs);
-        }
-        {
-            auto lists = topObj.object("list");
-            lists.attr("elements", nrListElems);
-            lists.attr("bytes", bLists);
-            lists.attr("concats", nrListConcats);
-        }
-        {
-            auto values = topObj.object("values");
-            values.attr("number", nrValues);
-            values.attr("bytes", bValues);
-        }
-        {
-            auto syms = topObj.object("symbols");
-            syms.attr("number", symbols.size());
-            syms.attr("bytes", symbols.totalSize());
-        }
-        {
-            auto sets = topObj.object("sets");
-            sets.attr("number", nrAttrsets);
-            sets.attr("bytes", bAttrsets);
-            sets.attr("elements", nrAttrsInAttrsets);
-        }
-        {
-            auto sizes = topObj.object("sizes");
-            sizes.attr("Env", sizeof(Env));
-            sizes.attr("Value", sizeof(Value));
-            sizes.attr("Bindings", sizeof(Bindings));
-            sizes.attr("Attr", sizeof(Attr));
-        }
-        topObj.attr("nrOpUpdates", nrOpUpdates);
-        topObj.attr("nrOpUpdateValuesCopied", nrOpUpdateValuesCopied);
-        topObj.attr("nrThunks", nrThunks);
-        topObj.attr("nrAvoided", nrAvoided);
-        topObj.attr("nrLookups", nrLookups);
-        topObj.attr("nrPrimOpCalls", nrPrimOpCalls);
-        topObj.attr("nrFunctionCalls", nrFunctionCalls);
-#if HAVE_BOEHMGC
-        {
-            auto gc = topObj.object("gc");
-            gc.attr("heapSize", heapSize);
-            gc.attr("totalBytes", totalBytes);
-        }
-#endif
-        if (countCalls) {
-            {
-                auto obj = topObj.object("primops");
-                for (auto & i : primOpCalls)
-                    obj.attr(i.first, i.second);
-            }
-            {
-                auto list = topObj.list("functions");
-                for (auto & i : functionCalls) {
-                    auto obj = list.object();
-                    if (i.first->name.set())
-                        obj.attr("name", (const string &) i.first->name);
-                    else
-                        obj.attr("name", nullptr);
-                    if (i.first->pos) {
-                        obj.attr("file", (const string &) i.first->pos.file);
-                        obj.attr("line", i.first->pos.line);
-                        obj.attr("column", i.first->pos.column);
-                    }
-                    obj.attr("count", i.second);
-                }
-            }
-            {
-                auto list = topObj.list("attributes");
-                for (auto & i : attrSelects) {
-                    auto obj = list.object();
-                    if (i.first) {
-                        obj.attr("file", (const string &) i.first.file);
-                        obj.attr("line", i.first.line);
-                        obj.attr("column", i.first.column);
-                    }
-                    obj.attr("count", i.second);
-                }
-            }
-        }
+
+    if (countCalls) {
+        v = lvlInfo;
+
+        printMsg(v, format("calls to %1% primops:") % primOpCalls.size());
+        typedef std::multimap<size_t, Symbol> PrimOpCalls_;
+        PrimOpCalls_ primOpCalls_;
+        for (auto & i : primOpCalls)
+            primOpCalls_.insert(std::pair<size_t, Symbol>(i.second, i.first));
+        for (auto i = primOpCalls_.rbegin(); i != primOpCalls_.rend(); ++i)
+            printMsg(v, format("%1$10d %2%") % i->first % i->second);
+
+        printMsg(v, format("calls to %1% functions:") % functionCalls.size());
+        typedef std::multimap<size_t, ExprLambda *> FunctionCalls_;
+        FunctionCalls_ functionCalls_;
+        for (auto & i : functionCalls)
+            functionCalls_.insert(std::pair<size_t, ExprLambda *>(i.second, i.first));
+        for (auto i = functionCalls_.rbegin(); i != functionCalls_.rend(); ++i)
+            printMsg(v, format("%1$10d %2%") % i->first % i->second->showNamePos());
+
+        printMsg(v, format("evaluations of %1% attributes:") % attrSelects.size());
+        typedef std::multimap<size_t, Pos> AttrSelects_;
+        AttrSelects_ attrSelects_;
+        for (auto & i : attrSelects)
+            attrSelects_.insert(std::pair<size_t, Pos>(i.second, i.first));
+        for (auto i = attrSelects_.rbegin(); i != attrSelects_.rend(); ++i)
+            printMsg(v, format("%1$10d %2%") % i->first % i->second);
+
     }
 }
 

src/libexpr/get-drvs.hh

@@ -44,7 +44,7 @@ public:
     string queryDrvPath() const;
     string queryOutPath() const;
     string queryOutputName() const;
-    /** Return the list of outputs. The "outputs to install" are determined by `meta.outputsToInstall`. */
+    /** Return the list of outputs. The "outputs to install" are determined by `mesa.outputsToInstall`. */
     Outputs queryOutputs(bool onlyOutputsToInstall = false);
 
     StringSet queryMetaNames();

src/libexpr/lexer.l

@@ -6,14 +6,12 @@
 %option nounput noyy_top_state
 
 
-%s DEFAULT
 %x STRING
 %x IND_STRING
+%x INSIDE_DOLLAR_CURLY
 
 
 %{
-#include <boost/lexical_cast.hpp>
-
 #include "nixexpr.hh"
 #include "parser-tab.hh"
 
@@ -99,6 +97,8 @@ URI         [a-zA-Z][a-zA-Z0-9\+\-\.]*\:[a-zA-Z0-9\%\/\?\:\@\&\=\+\$\,\-\_\.\!\~
 
 %%
 
+<INITIAL,INSIDE_DOLLAR_CURLY>{
+
 
 if          { return IF; }
 then        { return THEN; }
@@ -124,11 +124,9 @@ or          { return OR_KW; }
 
 {ID}        { yylval->id = strdup(yytext); return ID; }
 {INT}       { errno = 0;
-              try {
-                  yylval->n = boost::lexical_cast<int64_t>(yytext);
-              } catch (const boost::bad_lexical_cast &) {
+              yylval->n = strtol(yytext, 0, 10);
+              if (errno != 0)
                   throw ParseError(format("invalid integer '%1%'") % yytext);
-              }
               return INT;
             }
 {FLOAT}     { errno = 0;
@@ -138,19 +136,17 @@ or          { return OR_KW; }
               return FLOAT;
             }
 
-\$\{        { PUSH_STATE(DEFAULT); return DOLLAR_CURLY; }
+\$\{        { PUSH_STATE(INSIDE_DOLLAR_CURLY); return DOLLAR_CURLY; }
+}
 
-\}          { /* State INITIAL only exists at the bottom of the stack and is
-                 used as a marker. DEFAULT replaces it everywhere else.
-                 Popping when in INITIAL state causes an empty stack exception,
-                 so don't */
-              if (YYSTATE != INITIAL)
-                POP_STATE();
-              return '}';
-            }
-\{          { PUSH_STATE(DEFAULT); return '{'; }
+\}                           { return '}'; }
+<INSIDE_DOLLAR_CURLY>\}      { POP_STATE(); return '}'; }
+\{                           { return '{'; }
+<INSIDE_DOLLAR_CURLY>\{      { PUSH_STATE(INSIDE_DOLLAR_CURLY); return '{'; }
 
-\"          { PUSH_STATE(STRING); return '"'; }
+<INITIAL,INSIDE_DOLLAR_CURLY>\" {
+                PUSH_STATE(STRING); return '"';
+              }
 <STRING>([^\$\"\\]|\$[^\{\"\\]|\\{ANY}|\$\\{ANY})*\$/\" |
 <STRING>([^\$\"\\]|\$[^\{\"\\]|\\{ANY}|\$\\{ANY})+ {
                 /* It is impossible to match strings ending with '$' with one
@@ -159,7 +155,7 @@ or          { return OR_KW; }
                 yylval->e = unescapeStr(data->symbols, yytext, yyleng);
                 return STR;
               }
-<STRING>\$\{  { PUSH_STATE(DEFAULT); return DOLLAR_CURLY; }
+<STRING>\$\{  { PUSH_STATE(INSIDE_DOLLAR_CURLY); return DOLLAR_CURLY; }
 <STRING>\"    { POP_STATE(); return '"'; }
 <STRING>\$|\\|\$\\ {
                 /* This can only occur when we reach EOF, otherwise the above
@@ -169,7 +165,7 @@ or          { return OR_KW; }
                 return STR;
               }
 
-\'\'(\ *\n)?     { PUSH_STATE(IND_STRING); return IND_STRING_OPEN; }
+<INITIAL,INSIDE_DOLLAR_CURLY>\'\'(\ *\n)?     { PUSH_STATE(IND_STRING); return IND_STRING_OPEN; }
 <IND_STRING>([^\$\']|\$[^\{\']|\'[^\'\$])+ {
                    yylval->e = new ExprIndStr(yytext);
                    return IND_STR;
@@ -187,13 +183,14 @@ or          { return OR_KW; }
                    yylval->e = unescapeStr(data->symbols, yytext + 2, yyleng - 2);
                    return IND_STR;
                  }
-<IND_STRING>\$\{ { PUSH_STATE(DEFAULT); return DOLLAR_CURLY; }
+<IND_STRING>\$\{ { PUSH_STATE(INSIDE_DOLLAR_CURLY); return DOLLAR_CURLY; }
 <IND_STRING>\'\' { POP_STATE(); return IND_STRING_CLOSE; }
 <IND_STRING>\'   {
                    yylval->e = new ExprIndStr("'");
                    return IND_STR;
                  }
 
+<INITIAL,INSIDE_DOLLAR_CURLY>{
 
 {PATH}      { if (yytext[yyleng-1] == '/')
                   throw ParseError("path '%s' has a trailing slash", yytext);
@@ -218,5 +215,7 @@ or          { return OR_KW; }
               return (unsigned char) yytext[0];
             }
 
+}
+
 %%
 

src/libexpr/local.mk

@@ -6,7 +6,7 @@ libexpr_DIR := $(d)
 
 libexpr_SOURCES := $(wildcard $(d)/*.cc) $(wildcard $(d)/primops/*.cc) $(d)/lexer-tab.cc $(d)/parser-tab.cc
 
-libexpr_LIBS = libutil libstore
+libexpr_LIBS = libutil libstore libformat
 
 libexpr_LDFLAGS =
 ifneq ($(OS), FreeBSD)

src/libexpr/primops.cc

@@ -724,14 +724,16 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
         if (outputs.size() != 1 || *(outputs.begin()) != "out")
             throw Error(format("multiple outputs are not supported in fixed-output derivations, at %1%") % posDrvName);
 
-        HashType ht = outputHashAlgo.empty() ? htUnknown : parseHashType(outputHashAlgo);
+        HashType ht = parseHashType(outputHashAlgo);
+        if (ht == htUnknown)
+            throw EvalError(format("unknown hash algorithm '%1%', at %2%") % outputHashAlgo % posDrvName);
         Hash h(*outputHash, ht);
+        outputHash = h.to_string(Base16, false);
+        if (outputHashRecursive) outputHashAlgo = "r:" + outputHashAlgo;
 
         Path outPath = state.store->makeFixedOutputPath(outputHashRecursive, h, drvName);
         if (!jsonObject) drv.env["out"] = outPath;
-        drv.outputs["out"] = DerivationOutput(outPath,
-            (outputHashRecursive ? "r:" : "") + printHashType(h.type),
-            h.to_string(Base16, false));
+        drv.outputs["out"] = DerivationOutput(outPath, outputHashAlgo, *outputHash);
     }
 
     else {
@@ -864,7 +866,7 @@ static void prim_baseNameOf(EvalState & state, const Pos & pos, Value * * args,
 static void prim_dirOf(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     PathSet context;
-    Path dir = dirOf(state.coerceToString(pos, *args[0], context, false, false));
+    Path dir = dirOf(state.coerceToPath(pos, *args[0], context));
     if (args[0]->type == tPath) mkPath(v, dir.c_str()); else mkString(v, dir, context);
 }
 

src/libexpr/primops/fetchGit.cc

@@ -3,7 +3,6 @@
 #include "download.hh"
 #include "store-api.hh"
 #include "pathlocks.hh"
-#include "hash.hh"
 
 #include <sys/time.h>
 
@@ -85,16 +84,15 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
     if (rev != "" && !std::regex_match(rev, revRegex))
         throw Error("invalid Git revision '%s'", rev);
 
-    deletePath(getCacheDir() + "/nix/git");
-
-    Path cacheDir = getCacheDir() + "/nix/gitv2/" + hashString(htSHA256, uri).to_string(Base32, false);
+    Path cacheDir = getCacheDir() + "/nix/git";
 
     if (!pathExists(cacheDir)) {
-        createDirs(dirOf(cacheDir));
         runProgram("git", true, { "init", "--bare", cacheDir });
     }
 
-    Path localRefFile = cacheDir + "/refs/heads/" + *ref;
+    std::string localRef = hashString(htSHA256, fmt("%s-%s", uri, *ref)).to_string(Base32, false);
+
+    Path localRefFile = cacheDir + "/refs/heads/" + localRef;
 
     bool doFetch;
     time_t now = time(0);
@@ -124,7 +122,7 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
 
         // FIXME: git stderr messes up our progress indicator, so
         // we're using --quiet for now. Should process its stderr.
-        runProgram("git", true, { "-C", cacheDir, "fetch", "--quiet", "--force", "--", uri, fmt("%s:%s", *ref, *ref) });
+        runProgram("git", true, { "-C", cacheDir, "fetch", "--quiet", "--force", "--", uri, *ref + ":" + localRef });
 
         struct timeval times[2];
         times[0].tv_sec = now;

src/libexpr/primops/fetchMercurial.cc

@@ -93,22 +93,7 @@ HgInfo exportMercurial(ref<Store> store, const std::string & uri,
             Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Mercurial repository '%s'", uri));
 
             if (pathExists(cacheDir)) {
-                try {
-                    runProgram("hg", true, { "pull", "-R", cacheDir, "--", uri });
-                }
-                catch (ExecError & e){
-                    string transJournal = cacheDir + "/.hg/store/journal";
-                    /* hg throws "abandoned transaction" error only if this file exists */
-                    if (pathExists(transJournal))
-                    {
-                        runProgram("hg", true, { "recover", "-R", cacheDir });
-                        runProgram("hg", true, { "pull", "-R", cacheDir, "--", uri });
-                    }
-                    else 
-                    {
-                        throw ExecError(e.status, fmt("program hg '%1%' ", statusToString(e.status)));
-                    }
-                }
+                runProgram("hg", true, { "pull", "-R", cacheDir, "--", uri });
             } else {
                 createDirs(dirOf(cacheDir));
                 runProgram("hg", true, { "clone", "--noupdate", "--", uri, cacheDir });

src/libexpr/value.hh

@@ -43,7 +43,7 @@ class XMLWriter;
 class JSONPlaceholder;
 
 
-typedef int64_t NixInt;
+typedef long NixInt;
 typedef double NixFloat;
 
 /* External values must descend from ExternalValueBase, so that

src/libmain/local.mk

@@ -8,7 +8,7 @@ libmain_SOURCES := $(wildcard $(d)/*.cc)
 
 libmain_LDFLAGS = $(OPENSSL_LIBS)
 
-libmain_LIBS = libstore libutil
+libmain_LIBS = libstore libutil libformat
 
 libmain_ALLOW_UNDEFINED = 1
 

src/libmain/stack.cc

@@ -63,7 +63,7 @@ void detectStackOverflow()
     act.sa_sigaction = sigsegvHandler;
     act.sa_flags = SA_SIGINFO | SA_ONSTACK;
     if (sigaction(SIGSEGV, &act, 0))
-        throw SysError("resetting SIGSEGV");
+        throw SysError("resetting SIGCHLD");
 #endif
 }
 

src/libstore/build.cc

@@ -11,7 +11,6 @@
 #include "compression.hh"
 #include "json.hh"
 #include "nar-info.hh"
-#include "parsed-derivations.hh"
 
 #include <algorithm>
 #include <iostream>
@@ -21,7 +20,6 @@
 #include <future>
 #include <chrono>
 #include <regex>
-#include <queue>
 
 #include <limits.h>
 #include <sys/time.h>
@@ -742,8 +740,6 @@ private:
     /* The derivation stored at drvPath. */
     std::unique_ptr<BasicDerivation> drv;
 
-    std::unique_ptr<ParsedDerivation> parsedDrv;
-
     /* The remainder is state held during the build. */
 
     /* Locks on the output paths. */
@@ -858,7 +854,7 @@ private:
        building multiple times. Since this contains the hash, it
        allows us to compare whether two rounds produced the same
        result. */
-    std::map<Path, ValidPathInfo> prevInfos;
+    ValidPathInfos prevInfos;
 
     const uid_t sandboxUid = 1000;
     const gid_t sandboxGid = 100;
@@ -939,11 +935,6 @@ private:
        as valid. */
     void registerOutputs();
 
-    /* Check that an output meets the requirements specified by the
-       'outputChecks' attribute (or the legacy
-       '{allowed,disallowed}{References,Requisites}' attributes). */
-    void checkOutputs(const std::map<std::string, ValidPathInfo> & outputs);
-
     /* Open a log file and a pipe to it. */
     Path openLogFile();
 
@@ -1148,8 +1139,6 @@ void DerivationGoal::haveDerivation()
         return;
     }
 
-    parsedDrv = std::make_unique<ParsedDerivation>(drvPath, *drv);
-
     /* We are first going to try to create the invalid output paths
        through substitutes.  If that doesn't work, we'll build
        them. */
@@ -1406,7 +1395,7 @@ void DerivationGoal::tryToBuild()
     /* Don't do a remote build if the derivation has the attribute
        `preferLocalBuild' set.  Also, check and repair modes are only
        supported for local builds. */
-    bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally();
+    bool buildLocally = buildMode != bmNormal || drv->willBuildLocally();
 
     auto started = [&]() {
         auto msg = fmt(
@@ -1652,13 +1641,19 @@ HookReply DerivationGoal::tryBuildHook()
 
     try {
 
+        /* Tell the hook about system features (beyond the system type)
+           required from the build machine.  (The hook could parse the
+           drv file itself, but this is easier.) */
+        Strings features = tokenizeString<Strings>(get(drv->env, "requiredSystemFeatures"));
+        for (auto & i : features) checkStoreName(i); /* !!! abuse */
+
         /* Send the request to the hook. */
         worker.hook->sink
             << "try"
             << (worker.getNrLocalBuilds() < settings.maxBuildJobs ? 1 : 0)
             << drv->platform
             << drvPath
-            << parsedDrv->getRequiredSystemFeatures();
+            << features;
         worker.hook->sink.flush();
 
         /* Read the first line of input, which should be a word indicating
@@ -1798,26 +1793,23 @@ static void preloadNSS() {
 void DerivationGoal::startBuilder()
 {
     /* Right platform? */
-    if (!parsedDrv->canBuildLocally())
-        throw Error("a '%s' with features {%s} is required to build '%s', but I am a '%s' with features {%s}",
-            drv->platform,
-            concatStringsSep(", ", parsedDrv->getRequiredSystemFeatures()),
-            drvPath,
-            settings.thisSystem,
-            concatStringsSep(", ", settings.systemFeatures));
+    if (!drv->canBuildLocally()) {
+        throw Error(
+            format("a '%1%' is required to build '%3%', but I am a '%2%'")
+            % drv->platform % settings.thisSystem % drvPath);
+    }
 
     if (drv->isBuiltin())
         preloadNSS();
 
 #if __APPLE__
-    additionalSandboxProfile = parsedDrv->getStringAttr("__sandboxProfile").value_or("");
+    additionalSandboxProfile = get(drv->env, "__sandboxProfile");
 #endif
 
     /* Are we doing a chroot build? */
     {
-        auto noChroot = parsedDrv->getBoolAttr("__noChroot");
         if (settings.sandboxMode == smEnabled) {
-            if (noChroot)
+            if (get(drv->env, "__noChroot") == "1")
                 throw Error(format("derivation '%1%' has '__noChroot' set, "
                     "but that's not allowed when 'sandbox' is 'true'") % drvPath);
 #if __APPLE__
@@ -1830,7 +1822,7 @@ void DerivationGoal::startBuilder()
         else if (settings.sandboxMode == smDisabled)
             useChroot = false;
         else if (settings.sandboxMode == smRelaxed)
-            useChroot = !fixedOutput && !noChroot;
+            useChroot = !fixedOutput && get(drv->env, "__noChroot") != "1";
     }
 
     if (worker.store.storeDir != worker.store.realStoreDir) {
@@ -1881,7 +1873,7 @@ void DerivationGoal::startBuilder()
     writeStructuredAttrs();
 
     /* Handle exportReferencesGraph(), if set. */
-    if (!parsedDrv->getStructuredAttrs()) {
+    if (!drv->env.count("__json")) {
         /* The `exportReferencesGraph' feature allows the references graph
            to be passed to a builder.  This attribute should be a list of
            pairs [name1 path1 name2 path2 ...].  The references graph of
@@ -1946,7 +1938,7 @@ void DerivationGoal::startBuilder()
         PathSet allowedPaths = settings.allowedImpureHostPrefixes;
 
         /* This works like the above, except on a per-derivation level */
-        auto impurePaths = parsedDrv->getStringsAttr("__impureHostDeps").value_or(Strings());
+        Strings impurePaths = tokenizeString<Strings>(get(drv->env, "__impureHostDeps"));
 
         for (auto & i : impurePaths) {
             bool found = false;
@@ -2015,7 +2007,7 @@ void DerivationGoal::startBuilder()
 
         /* Create /etc/hosts with localhost entry. */
         if (!fixedOutput)
-            writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n::1 localhost\n");
+            writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n");
 
         /* Make the closure of the inputs available in the chroot,
            rather than the whole Nix store.  This prevents any access
@@ -2193,7 +2185,6 @@ void DerivationGoal::startBuilder()
         userNamespaceSync.create();
 
         options.allowVfork = false;
-        options.restoreMountNamespace = false;
 
         Pid helper = startProcess([&]() {
 
@@ -2260,7 +2251,6 @@ void DerivationGoal::startBuilder()
 #endif
     {
         options.allowVfork = !buildUser && !drv->isBuiltin();
-        options.restoreMountNamespace = false;
         pid = startProcess([&]() {
             runChild();
         }, options);
@@ -2316,7 +2306,7 @@ void DerivationGoal::initEnv()
        passAsFile is ignored in structure mode because it's not
        needed (attributes are not passed through the environment, so
        there is no size constraint). */
-    if (!parsedDrv->getStructuredAttrs()) {
+    if (!drv->env.count("__json")) {
 
         StringSet passAsFile = tokenizeString<StringSet>(get(drv->env, "passAsFile"));
         int fileNr = 0;
@@ -2363,8 +2353,8 @@ void DerivationGoal::initEnv()
        fixed-output derivations is by definition pure (since we
        already know the cryptographic hash of the output). */
     if (fixedOutput) {
-        for (auto & i : parsedDrv->getStringsAttr("impureEnvVars").value_or(Strings()))
-            env[i] = getEnv(i);
+        Strings varNames = tokenizeString<Strings>(get(drv->env, "impureEnvVars"));
+        for (auto & i : varNames) env[i] = getEnv(i);
     }
 
     /* Currently structured log messages piggyback on stderr, but we
@@ -2379,103 +2369,111 @@ static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*");
 
 void DerivationGoal::writeStructuredAttrs()
 {
-    auto & structuredAttrs = parsedDrv->getStructuredAttrs();
-    if (!structuredAttrs) return;
+    auto jsonAttr = drv->env.find("__json");
+    if (jsonAttr == drv->env.end()) return;
 
-    auto json = *structuredAttrs;
+    try {
 
-    /* Add an "outputs" object containing the output paths. */
-    nlohmann::json outputs;
-    for (auto & i : drv->outputs)
-        outputs[i.first] = rewriteStrings(i.second.path, inputRewrites);
-    json["outputs"] = outputs;
+        auto jsonStr = rewriteStrings(jsonAttr->second, inputRewrites);
 
-    /* Handle exportReferencesGraph. */
-    auto e = json.find("exportReferencesGraph");
-    if (e != json.end() && e->is_object()) {
-        for (auto i = e->begin(); i != e->end(); ++i) {
-            std::ostringstream str;
-            {
-                JSONPlaceholder jsonRoot(str, true);
-                PathSet storePaths;
-                for (auto & p : *i)
-                    storePaths.insert(p.get<std::string>());
-                worker.store.pathInfoToJSON(jsonRoot,
-                    exportReferences(storePaths), false, true);
+        auto json = nlohmann::json::parse(jsonStr);
+
+        /* Add an "outputs" object containing the output paths. */
+        nlohmann::json outputs;
+        for (auto & i : drv->outputs)
+            outputs[i.first] = rewriteStrings(i.second.path, inputRewrites);
+        json["outputs"] = outputs;
+
+        /* Handle exportReferencesGraph. */
+        auto e = json.find("exportReferencesGraph");
+        if (e != json.end() && e->is_object()) {
+            for (auto i = e->begin(); i != e->end(); ++i) {
+                std::ostringstream str;
+                {
+                    JSONPlaceholder jsonRoot(str, true);
+                    PathSet storePaths;
+                    for (auto & p : *i)
+                        storePaths.insert(p.get<std::string>());
+                    worker.store.pathInfoToJSON(jsonRoot,
+                        exportReferences(storePaths), false, true);
+                }
+                json[i.key()] = nlohmann::json::parse(str.str()); // urgh
             }
-            json[i.key()] = nlohmann::json::parse(str.str()); // urgh
         }
+
+        writeFile(tmpDir + "/.attrs.json", json.dump());
+
+        /* As a convenience to bash scripts, write a shell file that
+           maps all attributes that are representable in bash -
+           namely, strings, integers, nulls, Booleans, and arrays and
+           objects consisting entirely of those values. (So nested
+           arrays or objects are not supported.) */
+
+        auto handleSimpleType = [](const nlohmann::json & value) -> std::experimental::optional<std::string> {
+            if (value.is_string())
+                return shellEscape(value);
+
+            if (value.is_number()) {
+                auto f = value.get<float>();
+                if (std::ceil(f) == f)
+                    return std::to_string(value.get<int>());
+            }
+
+            if (value.is_null())
+                return std::string("''");
+
+            if (value.is_boolean())
+                return value.get<bool>() ? std::string("1") : std::string("");
+
+            return {};
+        };
+
+        std::string jsonSh;
+
+        for (auto i = json.begin(); i != json.end(); ++i) {
+
+            if (!std::regex_match(i.key(), shVarName)) continue;
+
+            auto & value = i.value();
+
+            auto s = handleSimpleType(value);
+            if (s)
+                jsonSh += fmt("declare %s=%s\n", i.key(), *s);
+
+            else if (value.is_array()) {
+                std::string s2;
+                bool good = true;
+
+                for (auto i = value.begin(); i != value.end(); ++i) {
+                    auto s3 = handleSimpleType(i.value());
+                    if (!s3) { good = false; break; }
+                    s2 += *s3; s2 += ' ';
+                }
+
+                if (good)
+                    jsonSh += fmt("declare -a %s=(%s)\n", i.key(), s2);
+            }
+
+            else if (value.is_object()) {
+                std::string s2;
+                bool good = true;
+
+                for (auto i = value.begin(); i != value.end(); ++i) {
+                    auto s3 = handleSimpleType(i.value());
+                    if (!s3) { good = false; break; }
+                    s2 += fmt("[%s]=%s ", shellEscape(i.key()), *s3);
+                }
+
+                if (good)
+                    jsonSh += fmt("declare -A %s=(%s)\n", i.key(), s2);
+            }
+        }
+
+        writeFile(tmpDir + "/.attrs.sh", jsonSh);
+
+    } catch (std::exception & e) {
+        throw Error("cannot process __json attribute of '%s': %s", drvPath, e.what());
     }
-
-    writeFile(tmpDir + "/.attrs.json", rewriteStrings(json.dump(), inputRewrites));
-
-    /* As a convenience to bash scripts, write a shell file that
-       maps all attributes that are representable in bash -
-       namely, strings, integers, nulls, Booleans, and arrays and
-       objects consisting entirely of those values. (So nested
-       arrays or objects are not supported.) */
-
-    auto handleSimpleType = [](const nlohmann::json & value) -> std::experimental::optional<std::string> {
-        if (value.is_string())
-            return shellEscape(value);
-
-        if (value.is_number()) {
-            auto f = value.get<float>();
-            if (std::ceil(f) == f)
-                return std::to_string(value.get<int>());
-        }
-
-        if (value.is_null())
-            return std::string("''");
-
-        if (value.is_boolean())
-            return value.get<bool>() ? std::string("1") : std::string("");
-
-        return {};
-    };
-
-    std::string jsonSh;
-
-    for (auto i = json.begin(); i != json.end(); ++i) {
-
-        if (!std::regex_match(i.key(), shVarName)) continue;
-
-        auto & value = i.value();
-
-        auto s = handleSimpleType(value);
-        if (s)
-            jsonSh += fmt("declare %s=%s\n", i.key(), *s);
-
-        else if (value.is_array()) {
-            std::string s2;
-            bool good = true;
-
-            for (auto i = value.begin(); i != value.end(); ++i) {
-                auto s3 = handleSimpleType(i.value());
-                if (!s3) { good = false; break; }
-                s2 += *s3; s2 += ' ';
-            }
-
-            if (good)
-                jsonSh += fmt("declare -a %s=(%s)\n", i.key(), s2);
-        }
-
-        else if (value.is_object()) {
-            std::string s2;
-            bool good = true;
-
-            for (auto i = value.begin(); i != value.end(); ++i) {
-                auto s3 = handleSimpleType(i.value());
-                if (!s3) { good = false; break; }
-                s2 += fmt("[%s]=%s ", shellEscape(i.key()), *s3);
-            }
-
-            if (good)
-                jsonSh += fmt("declare -A %s=(%s)\n", i.key(), s2);
-        }
-    }
-
-    writeFile(tmpDir + "/.attrs.sh", rewriteStrings(jsonSh, inputRewrites));
 }
 
 
@@ -2630,7 +2628,7 @@ void DerivationGoal::runChild()
                 createDirs(chrootRootDir + "/dev/shm");
                 createDirs(chrootRootDir + "/dev/pts");
                 ss.push_back("/dev/full");
-                if (settings.systemFeatures.get().count("kvm") && pathExists("/dev/kvm"))
+                if (pathExists("/dev/kvm"))
                     ss.push_back("/dev/kvm");
                 ss.push_back("/dev/null");
                 ss.push_back("/dev/random");
@@ -2919,7 +2917,7 @@ void DerivationGoal::runChild()
 
             writeFile(sandboxFile, sandboxProfile);
 
-            bool allowLocalNetworking = parsedDrv->getBoolAttr("__darwinAllowLocalNetworking");
+            bool allowLocalNetworking = get(drv->env, "__darwinAllowLocalNetworking") == "1";
 
             /* The tmpDir in scope points at the temporary build directory for our derivation. Some packages try different mechanisms
                to find temporary directories, so we want to open up a broader place for them to dump their files, if needed. */
@@ -2991,9 +2989,10 @@ void DerivationGoal::runChild()
 /* Parse a list of reference specifiers.  Each element must either be
    a store path, or the symbolic name of the output of the derivation
    (such as `out'). */
-PathSet parseReferenceSpecifiers(Store & store, const BasicDerivation & drv, const Strings & paths)
+PathSet parseReferenceSpecifiers(Store & store, const BasicDerivation & drv, string attr)
 {
     PathSet result;
+    Paths paths = tokenizeString<Paths>(attr);
     for (auto & i : paths) {
         if (store.isStorePath(i))
             result.insert(i);
@@ -3018,7 +3017,7 @@ void DerivationGoal::registerOutputs()
         if (allValid) return;
     }
 
-    std::map<std::string, ValidPathInfo> infos;
+    ValidPathInfos infos;
 
     /* Set of inodes seen during calls to canonicalisePathMetaData()
        for this build's outputs.  This needs to be shared between
@@ -3122,15 +3121,15 @@ void DerivationGoal::registerOutputs()
                the derivation to its content-addressed location. */
             Hash h2 = recursive ? hashPath(h.type, actualPath).first : hashFile(h.type, actualPath);
 
-            Path dest = worker.store.makeFixedOutputPath(recursive, h2, storePathToName(path));
+            Path dest = worker.store.makeFixedOutputPath(recursive, h2, drv->env["name"]);
 
             if (h != h2) {
 
                 /* Throw an error after registering the path as
                    valid. */
                 delayedException = std::make_exception_ptr(
-                    BuildError("hash mismatch in fixed-output derivation '%s':\n  wanted: %s\n  got:    %s",
-                        dest, h.to_string(), h2.to_string()));
+                    BuildError("fixed-output derivation produced path '%s' with %s hash '%s' instead of the expected hash '%s'",
+                        dest, printHashType(h.type), printHash16or32(h2), printHash16or32(h)));
 
                 Path actualDest = worker.store.toRealPath(dest);
 
@@ -3203,6 +3202,48 @@ void DerivationGoal::registerOutputs()
                 debug(format("referenced input: '%1%'") % i);
         }
 
+        /* Enforce `allowedReferences' and friends. */
+        auto checkRefs = [&](const string & attrName, bool allowed, bool recursive) {
+            if (drv->env.find(attrName) == drv->env.end()) return;
+
+            PathSet spec = parseReferenceSpecifiers(worker.store, *drv, get(drv->env, attrName));
+
+            PathSet used;
+            if (recursive) {
+                /* Our requisites are the union of the closures of our references. */
+                for (auto & i : references)
+                    /* Don't call computeFSClosure on ourselves. */
+                    if (path != i)
+                        worker.store.computeFSClosure(i, used);
+            } else
+                used = references;
+
+            PathSet badPaths;
+
+            for (auto & i : used)
+                if (allowed) {
+                    if (spec.find(i) == spec.end())
+                        badPaths.insert(i);
+                } else {
+                    if (spec.find(i) != spec.end())
+                        badPaths.insert(i);
+                }
+
+            if (!badPaths.empty()) {
+                string badPathsStr;
+                for (auto & i : badPaths) {
+                    badPathsStr += "\n\t";
+                    badPathsStr += i;
+                }
+                throw BuildError(format("output '%1%' is not allowed to refer to the following paths:%2%") % actualPath % badPathsStr);
+            }
+        };
+
+        checkRefs("allowedReferences", true, false);
+        checkRefs("allowedRequisites", true, true);
+        checkRefs("disallowedReferences", false, false);
+        checkRefs("disallowedRequisites", false, true);
+
         if (curRound == nrRounds) {
             worker.store.optimisePath(actualPath); // FIXME: combine with scanForReferences()
             worker.markContentsGood(path);
@@ -3218,14 +3259,11 @@ void DerivationGoal::registerOutputs()
 
         if (!info.references.empty()) info.ca.clear();
 
-        infos[i.first] = info;
+        infos.push_back(info);
     }
 
     if (buildMode == bmCheck) return;
 
-    /* Apply output checks. */
-    checkOutputs(infos);
-
     /* Compare the result with the previous round, and report which
        path is different, if any.*/
     if (curRound > 1 && prevInfos != infos) {
@@ -3233,16 +3271,16 @@ void DerivationGoal::registerOutputs()
         for (auto i = prevInfos.begin(), j = infos.begin(); i != prevInfos.end(); ++i, ++j)
             if (!(*i == *j)) {
                 result.isNonDeterministic = true;
-                Path prev = i->second.path + checkSuffix;
+                Path prev = i->path + checkSuffix;
                 bool prevExists = keepPreviousRound && pathExists(prev);
                 auto msg = prevExists
-                    ? fmt("output '%1%' of '%2%' differs from '%3%' from previous round", i->second.path, drvPath, prev)
-                    : fmt("output '%1%' of '%2%' differs from previous round", i->second.path, drvPath);
+                    ? fmt("output '%1%' of '%2%' differs from '%3%' from previous round", i->path, drvPath, prev)
+                    : fmt("output '%1%' of '%2%' differs from previous round", i->path, drvPath);
 
                 auto diffHook = settings.diffHook;
                 if (prevExists && diffHook != "" && runDiffHook) {
                     try {
-                        auto diff = runProgram(diffHook, true, {prev, i->second.path});
+                        auto diff = runProgram(diffHook, true, {prev, i->path});
                         if (diff != "")
                             printError(chomp(diff));
                     } catch (Error & error) {
@@ -3287,11 +3325,7 @@ void DerivationGoal::registerOutputs()
     /* Register each output path as valid, and register the sets of
        paths referenced by each of them.  If there are cycles in the
        outputs, this will fail. */
-    {
-        ValidPathInfos infos2;
-        for (auto & i : infos) infos2.push_back(i.second);
-        worker.store.registerValidPaths(infos2);
-    }
+    worker.store.registerValidPaths(infos);
 
     /* In case of a fixed-output derivation hash mismatch, throw an
        exception now that we have registered the output as valid. */
@@ -3300,158 +3334,6 @@ void DerivationGoal::registerOutputs()
 }
 
 
-void DerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & outputs)
-{
-    std::map<Path, const ValidPathInfo &> outputsByPath;
-    for (auto & output : outputs)
-        outputsByPath.emplace(output.second.path, output.second);
-
-    for (auto & output : outputs) {
-        auto & outputName = output.first;
-        auto & info = output.second;
-
-        struct Checks
-        {
-            bool ignoreSelfRefs = false;
-            std::experimental::optional<uint64_t> maxSize, maxClosureSize;
-            std::experimental::optional<Strings> allowedReferences, allowedRequisites, disallowedReferences, disallowedRequisites;
-        };
-
-        /* Compute the closure and closure size of some output. This
-           is slightly tricky because some of its references (namely
-           other outputs) may not be valid yet. */
-        auto getClosure = [&](const Path & path)
-        {
-            uint64_t closureSize = 0;
-            PathSet pathsDone;
-            std::queue<Path> pathsLeft;
-            pathsLeft.push(path);
-
-            while (!pathsLeft.empty()) {
-                auto path = pathsLeft.front();
-                pathsLeft.pop();
-                if (!pathsDone.insert(path).second) continue;
-
-                auto i = outputsByPath.find(path);
-                if (i != outputsByPath.end()) {
-                    closureSize += i->second.narSize;
-                    for (auto & ref : i->second.references)
-                        pathsLeft.push(ref);
-                } else {
-                    auto info = worker.store.queryPathInfo(path);
-                    closureSize += info->narSize;
-                    for (auto & ref : info->references)
-                        pathsLeft.push(ref);
-                }
-            }
-
-            return std::make_pair(pathsDone, closureSize);
-        };
-
-        auto applyChecks = [&](const Checks & checks)
-        {
-            if (checks.maxSize && info.narSize > *checks.maxSize)
-                throw BuildError("path '%s' is too large at %d bytes; limit is %d bytes",
-                    info.path, info.narSize, *checks.maxSize);
-
-            if (checks.maxClosureSize) {
-                uint64_t closureSize = getClosure(info.path).second;
-                if (closureSize > *checks.maxClosureSize)
-                    throw BuildError("closure of path '%s' is too large at %d bytes; limit is %d bytes",
-                        info.path, closureSize, *checks.maxClosureSize);
-            }
-
-            auto checkRefs = [&](const std::experimental::optional<Strings> & value, bool allowed, bool recursive)
-            {
-                if (!value) return;
-
-                PathSet spec = parseReferenceSpecifiers(worker.store, *drv, *value);
-
-                PathSet used = recursive ? getClosure(info.path).first : info.references;
-
-                if (recursive && checks.ignoreSelfRefs)
-                    used.erase(info.path);
-
-                PathSet badPaths;
-
-                for (auto & i : used)
-                    if (allowed) {
-                        if (!spec.count(i))
-                            badPaths.insert(i);
-                    } else {
-                        if (spec.count(i))
-                            badPaths.insert(i);
-                    }
-
-                if (!badPaths.empty()) {
-                    string badPathsStr;
-                    for (auto & i : badPaths) {
-                        badPathsStr += "\n  ";
-                        badPathsStr += i;
-                    }
-                    throw BuildError("output '%s' is not allowed to refer to the following paths:%s", info.path, badPathsStr);
-                }
-            };
-
-            checkRefs(checks.allowedReferences, true, false);
-            checkRefs(checks.allowedRequisites, true, true);
-            checkRefs(checks.disallowedReferences, false, false);
-            checkRefs(checks.disallowedRequisites, false, true);
-        };
-
-        if (auto structuredAttrs = parsedDrv->getStructuredAttrs()) {
-            auto outputChecks = structuredAttrs->find("outputChecks");
-            if (outputChecks != structuredAttrs->end()) {
-                auto output = outputChecks->find(outputName);
-
-                if (output != outputChecks->end()) {
-                    Checks checks;
-
-                    auto maxSize = output->find("maxSize");
-                    if (maxSize != output->end())
-                        checks.maxSize = maxSize->get<uint64_t>();
-
-                    auto maxClosureSize = output->find("maxClosureSize");
-                    if (maxClosureSize != output->end())
-                        checks.maxClosureSize = maxClosureSize->get<uint64_t>();
-
-                    auto get = [&](const std::string & name) -> std::experimental::optional<Strings> {
-                        auto i = output->find(name);
-                        if (i != output->end()) {
-                            Strings res;
-                            for (auto j = i->begin(); j != i->end(); ++j) {
-                                if (!j->is_string())
-                                    throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, drvPath);
-                                res.push_back(j->get<std::string>());
-                            }
-                            checks.disallowedRequisites = res;
-                            return res;
-                        }
-                        return {};
-                    };
-
-                    checks.allowedReferences = get("allowedReferences");
-                    checks.allowedRequisites = get("allowedRequisites");
-                    checks.disallowedReferences = get("disallowedReferences");
-                    checks.disallowedRequisites = get("disallowedRequisites");
-
-                    applyChecks(checks);
-                }
-            }
-        } else {
-            // legacy non-structured-attributes case
-            Checks checks;
-            checks.ignoreSelfRefs = true;
-            checks.allowedReferences = parsedDrv->getStringsAttr("allowedReferences");
-            checks.allowedRequisites = parsedDrv->getStringsAttr("allowedRequisites");
-            checks.disallowedReferences = parsedDrv->getStringsAttr("disallowedReferences");
-            checks.disallowedRequisites = parsedDrv->getStringsAttr("disallowedRequisites");
-            applyChecks(checks);
-        }
-    }
-}
-
-
 Path DerivationGoal::openLogFile()
 {
     logSize = 0;
@@ -3800,19 +3682,6 @@ void SubstitutionGoal::tryNext()
     } catch (InvalidPath &) {
         tryNext();
         return;
-    } catch (SubstituterDisabled &) {
-        if (settings.tryFallback) {
-            tryNext();
-            return;
-        }
-        throw;
-    } catch (Error & e) {
-        if (settings.tryFallback) {
-            printError(e.what());
-            tryNext();
-            return;
-        }
-        throw;
     }
 
     /* Update the total expected download size. */

src/libstore/builtins/fetchurl.cc

@@ -24,7 +24,6 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
 
     Path storePath = getAttr("out");
     auto mainUrl = getAttr("url");
-    bool unpack = get(drv.env, "unpack", "") == "1";
 
     /* Note: have to use a fresh downloader here because we're in
        a forked process. */
@@ -41,12 +40,12 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
             request.decompress = false;
 
             auto decompressor = makeDecompressionSink(
-                unpack && hasSuffix(mainUrl, ".xz") ? "xz" : "none", sink);
+                hasSuffix(mainUrl, ".xz") ? "xz" : "none", sink);
             downloader->download(std::move(request), *decompressor);
             decompressor->finish();
         });
 
-        if (unpack)
+        if (get(drv.env, "unpack", "") == "1")
             restorePath(storePath, *source);
         else
             writeFile(storePath, *source);

src/libstore/derivations.cc

@@ -36,6 +36,12 @@ Path BasicDerivation::findOutput(const string & id) const
 }
 
 
+bool BasicDerivation::willBuildLocally() const
+{
+    return get(env, "preferLocalBuild") == "1" && canBuildLocally();
+}
+
+
 bool BasicDerivation::substitutesAllowed() const
 {
     return get(env, "allowSubstitutes", "1") == "1";
@@ -48,6 +54,14 @@ bool BasicDerivation::isBuiltin() const
 }
 
 
+bool BasicDerivation::canBuildLocally() const
+{
+    return platform == settings.thisSystem
+        || settings.extraPlatforms.get().count(platform) > 0
+        || isBuiltin();
+}
+
+
 Path writeDerivation(ref<Store> store,
     const Derivation & drv, const string & name, RepairFlag repair)
 {

src/libstore/derivations.hh

@@ -56,10 +56,14 @@ struct BasicDerivation
        the given derivation. */
     Path findOutput(const string & id) const;
 
+    bool willBuildLocally() const;
+
     bool substitutesAllowed() const;
 
     bool isBuiltin() const;
 
+    bool canBuildLocally() const;
+
     /* Return true iff this is a fixed-output derivation. */
     bool isFixedOutput() const;
 

src/libstore/download.cc

@@ -345,7 +345,7 @@ struct CurlDownloader : public Downloader
                 done = true;
 
                 try {
-                    act.progress(result.bodySize, result.bodySize);
+                    act.progress(result.data->size(), result.data->size());
                     callback(std::move(result));
                 } catch (...) {
                     done = true;
@@ -528,11 +528,10 @@ struct CurlDownloader : public Downloader
             extraFDs[0].fd = wakeupPipe.readSide.get();
             extraFDs[0].events = CURL_WAIT_POLLIN;
             extraFDs[0].revents = 0;
-            long maxSleepTimeMs = items.empty() ? 10000 : 100;
             auto sleepTimeMs =
                 nextWakeup != std::chrono::steady_clock::time_point()
                 ? std::max(0, (int) std::chrono::duration_cast<std::chrono::milliseconds>(nextWakeup - std::chrono::steady_clock::now()).count())
-                : maxSleepTimeMs;
+                : 10000;
             vomit("download thread waiting for %d ms", sleepTimeMs);
             mc = curl_multi_wait(curlm, extraFDs, 1, sleepTimeMs, &numfds);
             if (mc != CURLM_OK)
@@ -622,7 +621,7 @@ struct CurlDownloader : public Downloader
             // FIXME: do this on a worker thread
             try {
 #ifdef ENABLE_S3
-                S3Helper s3Helper("", Aws::Region::US_EAST_1, "", ""); // FIXME: make configurable
+                S3Helper s3Helper("", Aws::Region::US_EAST_1, ""); // FIXME: make configurable
                 auto slash = request.uri.find('/', 5);
                 if (slash == std::string::npos)
                     throw nix::Error("bad S3 URI '%s'", request.uri);
@@ -711,12 +710,11 @@ void Downloader::download(DownloadRequest && request, Sink & sink)
 
         /* If the buffer is full, then go to sleep until the calling
            thread wakes us up (i.e. when it has removed data from the
-           buffer). We don't wait forever to prevent stalling the
-           download thread. (Hopefully sleeping will throttle the
-           sender.) */
-        if (state->data.size() > 1024 * 1024) {
+           buffer). Note: this does stall the download thread. */
+        while (state->data.size() > 1024 * 1024) {
+            if (state->quit) return;
             debug("download buffer is full; going to sleep");
-            state.wait_for(state->request, std::chrono::seconds(10));
+            state.wait(state->request);
         }
 
         /* Append data to the buffer and wake up the calling
@@ -738,36 +736,30 @@ void Downloader::download(DownloadRequest && request, Sink & sink)
             state->request.notify_one();
         }});
 
+    auto state(_state->lock());
+
     while (true) {
         checkInterrupt();
 
-        std::string chunk;
+        /* If no data is available, then wait for the download thread
+           to wake us up. */
+        if (state->data.empty()) {
 
-        /* Grab data if available, otherwise wait for the download
-           thread to wake us up. */
-        {
-            auto state(_state->lock());
-
-            while (state->data.empty()) {
-
-                if (state->quit) {
-                    if (state->exc) std::rethrow_exception(state->exc);
-                    return;
-                }
-
-                state.wait(state->avail);
+            if (state->quit) {
+                if (state->exc) std::rethrow_exception(state->exc);
+                break;
             }
 
-            chunk = std::move(state->data);
-
-            state->request.notify_one();
+            state.wait(state->avail);
         }
 
-        /* Flush the data to the sink and wake up the download thread
-           if it's blocked on a full buffer. We don't hold the state
-           lock while doing this to prevent blocking the download
-           thread if sink() takes a long time. */
-        sink((unsigned char *) chunk.data(), chunk.size());
+        /* If data is available, then flush it to the sink and wake up
+           the download thread if it's blocked on a full buffer. */
+        if (!state->data.empty()) {
+            sink((unsigned char *) state->data.data(), state->data.size());
+            state->data.clear();
+            state->request.notify_one();
+        }
     }
 }
 
@@ -881,8 +873,8 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa
         Hash gotHash = unpack
             ? hashPath(expectedHash.type, store->toRealPath(storePath)).first
             : hashFile(expectedHash.type, store->toRealPath(storePath));
-        throw nix::Error("hash mismatch in file downloaded from '%s':\n  wanted: %s\n  got:    %s",
-            url, expectedHash.to_string(), gotHash.to_string());
+        throw nix::Error("hash mismatch in file downloaded from '%s': got hash '%s' instead of the expected hash '%s'",
+            url, gotHash.to_string(), expectedHash.to_string());
     }
 
     return store->toRealPath(storePath);

src/libstore/globals.cc

@@ -78,11 +78,7 @@ void loadConfFile()
        ~/.nix/nix.conf or the command line. */
     globalConfig.resetOverriden();
 
-    auto dirs = getConfigDirs();
-    // Iterate over them in reverse so that the ones appearing first in the path take priority
-    for (auto dir = dirs.rbegin(); dir != dirs.rend(); dir++) {
-        globalConfig.applyConfigFile(*dir + "/nix/nix.conf");
-    }
+    globalConfig.applyConfigFile(getConfigDir() + "/nix/nix.conf");
 }
 
 unsigned int Settings::getDefaultCores()
@@ -90,21 +86,6 @@ unsigned int Settings::getDefaultCores()
     return std::max(1U, std::thread::hardware_concurrency());
 }
 
-StringSet Settings::getDefaultSystemFeatures()
-{
-    /* For backwards compatibility, accept some "features" that are
-       used in Nixpkgs to route builds to certain machines but don't
-       actually require anything special on the machines. */
-    StringSet features{"nixos-test", "benchmark", "big-parallel"};
-
-    #if __linux__
-    if (access("/dev/kvm", R_OK | W_OK) == 0)
-        features.insert("kvm");
-    #endif
-
-    return features;
-}
-
 const string nixVersion = PACKAGE_VERSION;
 
 template<> void BaseSetting<SandboxMode>::set(const std::string & str)

src/libstore/globals.hh

@@ -32,8 +32,6 @@ class Settings : public Config {
 
     unsigned int getDefaultCores();
 
-    StringSet getDefaultSystemFeatures();
-
 public:
 
     Settings();
@@ -82,9 +80,9 @@ public:
     /* Whether to show build log output in real time. */
     bool verboseBuild = true;
 
-    Setting<size_t> logLines{this, 10, "log-lines",
-        "If verbose-build is false, the number of lines of the tail of "
-        "the log to show if a build fails."};
+    /* If verboseBuild is false, the number of lines of the tail of
+       the log to show if a build fails. */
+    size_t logLines = 10;
 
     MaxBuildJobsSetting maxBuildJobs{this, 1, "max-jobs",
         "Maximum number of parallel build jobs. \"auto\" means use number of cores.",
@@ -195,13 +193,7 @@ public:
     Setting<bool> showTrace{this, false, "show-trace",
         "Whether to show a stack trace on evaluation errors."};
 
-    Setting<SandboxMode> sandboxMode{this,
-        #if __linux__
-          smEnabled
-        #else
-          smDisabled
-        #endif
-        , "sandbox",
+    Setting<SandboxMode> sandboxMode{this, smDisabled, "sandbox",
         "Whether to enable sandboxed builds. Can be \"true\", \"false\" or \"relaxed\".",
         {"build-use-chroot", "build-use-sandbox"}};
 
@@ -269,10 +261,6 @@ public:
         "These may be supported natively (e.g. armv7 on some aarch64 CPUs "
         "or using hacks like qemu-user."};
 
-    Setting<StringSet> systemFeatures{this, getDefaultSystemFeatures(),
-        "system-features",
-        "Optional features that this system implements (like \"kvm\")."};
-
     Setting<Strings> substituters{this,
         nixStore == "/nix/store" ? Strings{"https://cache.nixos.org/"} : Strings(),
         "substituters",

src/libstore/http-binary-cache-store.cc

@@ -13,14 +13,6 @@ private:
 
     Path cacheUri;
 
-    struct State
-    {
-        bool enabled = true;
-        std::chrono::steady_clock::time_point disabledUntil;
-    };
-
-    Sync<State> _state;
-
 public:
 
     HttpBinaryCacheStore(
@@ -54,33 +46,8 @@ public:
 
 protected:
 
-    void maybeDisable()
-    {
-        auto state(_state.lock());
-        if (state->enabled && settings.tryFallback) {
-            int t = 60;
-            printError("disabling binary cache '%s' for %s seconds", getUri(), t);
-            state->enabled = false;
-            state->disabledUntil = std::chrono::steady_clock::now() + std::chrono::seconds(t);
-        }
-    }
-
-    void checkEnabled()
-    {
-        auto state(_state.lock());
-        if (state->enabled) return;
-        if (std::chrono::steady_clock::now() > state->disabledUntil) {
-            state->enabled = true;
-            debug("re-enabling binary cache '%s'", getUri());
-            return;
-        }
-        throw SubstituterDisabled("substituter '%s' is disabled", getUri());
-    }
-
     bool fileExists(const std::string & path) override
     {
-        checkEnabled();
-
         try {
             DownloadRequest request(cacheUri + "/" + path);
             request.head = true;
@@ -92,7 +59,6 @@ protected:
                bucket is unlistable, so treat 403 as 404. */
             if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden)
                 return false;
-            maybeDisable();
             throw;
         }
     }
@@ -120,14 +86,12 @@ protected:
 
     void getFile(const std::string & path, Sink & sink) override
     {
-        checkEnabled();
         auto request(makeRequest(path));
         try {
             getDownloader()->download(std::move(request), sink);
         } catch (DownloadError & e) {
             if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden)
                 throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache '%s'", path, getUri());
-            maybeDisable();
             throw;
         }
     }
@@ -135,18 +99,15 @@ protected:
     void getFile(const std::string & path,
         Callback<std::shared_ptr<std::string>> callback) override
     {
-        checkEnabled();
-
         auto request(makeRequest(path));
 
         getDownloader()->enqueueDownload(request,
-            {[callback, this](std::future<DownloadResult> result) {
+            {[callback](std::future<DownloadResult> result) {
                 try {
                     callback(result.get().data);
                 } catch (DownloadError & e) {
                     if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden)
                         return callback(std::shared_ptr<std::string>());
-                    maybeDisable();
                     callback.rethrow();
                 } catch (...) {
                     callback.rethrow();

src/libstore/legacy-ssh-store.cc

@@ -303,12 +303,6 @@ struct LegacySSHStore : public Store
     {
         auto conn(connections->get());
     }
-
-    unsigned int getProtocol() override
-    {
-        auto conn(connections->get());
-        return conn->remoteVersion;
-    }
 };
 
 static RegisterStoreImplementation regStore([](

src/libstore/local-store.cc

@@ -366,8 +366,6 @@ void LocalStore::makeStoreWritable()
         throw SysError("getting info about the Nix store mount point");
 
     if (stat.f_flag & ST_RDONLY) {
-        saveMountNamespace();
-
         if (unshare(CLONE_NEWNS) == -1)
             throw SysError("setting up a private mount namespace");
 
@@ -882,12 +880,6 @@ void LocalStore::querySubstitutablePathInfos(const PathSet & paths,
                     narInfo ? narInfo->fileSize : 0,
                     info->narSize};
             } catch (InvalidPath) {
-            } catch (SubstituterDisabled) {
-            } catch (Error & e) {
-                if (settings.tryFallback)
-                    printError(e.what());
-                else
-                    throw;
             }
         }
     }
@@ -1022,11 +1014,11 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
             auto hashResult = hashSink.finish();
 
             if (hashResult.first != info.narHash)
-                throw Error("hash mismatch importing path '%s';\n  wanted: %s\n  got:    %s",
+                throw Error("hash mismatch importing path '%s'; expected hash '%s', got '%s'",
                     info.path, info.narHash.to_string(), hashResult.first.to_string());
 
             if (hashResult.second != info.narSize)
-                throw Error("size mismatch importing path '%s';\n  wanted: %s\n  got:   %s",
+                throw Error("size mismatch importing path '%s'; expected %s, got %s",
                     info.path, info.narSize, hashResult.second);
 
             autoGC();
@@ -1340,12 +1332,6 @@ void LocalStore::verifyPath(const Path & path, const PathSet & store,
 }
 
 
-unsigned int LocalStore::getProtocol()
-{
-    return PROTOCOL_VERSION;
-}
-
-
 #if defined(FS_IOC_SETFLAGS) && defined(FS_IOC_GETFLAGS) && defined(FS_IMMUTABLE_FL)
 
 static void makeMutable(const Path & path)

src/libstore/local-store.hh

@@ -209,8 +209,6 @@ public:
 
     void registerValidPaths(const ValidPathInfos & infos);
 
-    unsigned int getProtocol() override;
-
     void vacuumDB();
 
     /* Repair the contents of the given path by redownloading it using

src/libstore/local.mk

@@ -6,7 +6,7 @@ libstore_DIR := $(d)
 
 libstore_SOURCES := $(wildcard $(d)/*.cc $(d)/builtins/*.cc)
 
-libstore_LIBS = libutil
+libstore_LIBS = libutil libformat
 
 libstore_LDFLAGS = $(SQLITE3_LIBS) -lbz2 $(LIBCURL_LIBS) $(SODIUM_LIBS) -pthread
 ifneq ($(OS), FreeBSD)

src/libstore/nar-info-disk-cache.cc

@@ -31,7 +31,6 @@ create table if not exists NARs (
     refs             text,
     deriver          text,
     sigs             text,
-    ca               text,
     timestamp        integer not null,
     present          integer not null,
     primary key (cache, hashPart),
@@ -73,7 +72,7 @@ public:
     {
         auto state(_state.lock());
 
-        Path dbPath = getCacheDir() + "/nix/binary-cache-v6.sqlite";
+        Path dbPath = getCacheDir() + "/nix/binary-cache-v5.sqlite";
         createDirs(dirOf(dbPath));
 
         state->db = SQLite(dbPath);
@@ -95,13 +94,13 @@ public:
 
         state->insertNAR.create(state->db,
             "insert or replace into NARs(cache, hashPart, namePart, url, compression, fileHash, fileSize, narHash, "
-            "narSize, refs, deriver, sigs, ca, timestamp, present) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1)");
+            "narSize, refs, deriver, sigs, timestamp, present) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1)");
 
         state->insertMissingNAR.create(state->db,
             "insert or replace into NARs(cache, hashPart, timestamp, present) values (?, ?, ?, 0)");
 
         state->queryNAR.create(state->db,
-            "select present, namePart, url, compression, fileHash, fileSize, narHash, narSize, refs, deriver, sigs, ca from NARs where cache = ? and hashPart = ? and ((present = 0 and timestamp > ?) or (present = 1 and timestamp > ?))");
+            "select * from NARs where cache = ? and hashPart = ? and ((present = 0 and timestamp > ?) or (present = 1 and timestamp > ?))");
 
         /* Periodically purge expired entries from the database. */
         retrySQLite<void>([&]() {
@@ -190,28 +189,27 @@ public:
             if (!queryNAR.next())
                 return {oUnknown, 0};
 
-            if (!queryNAR.getInt(0))
+            if (!queryNAR.getInt(13))
                 return {oInvalid, 0};
 
             auto narInfo = make_ref<NarInfo>();
 
-            auto namePart = queryNAR.getStr(1);
+            auto namePart = queryNAR.getStr(2);
             narInfo->path = cache.storeDir + "/" +
                 hashPart + (namePart.empty() ? "" : "-" + namePart);
-            narInfo->url = queryNAR.getStr(2);
-            narInfo->compression = queryNAR.getStr(3);
-            if (!queryNAR.isNull(4))
-                narInfo->fileHash = Hash(queryNAR.getStr(4));
-            narInfo->fileSize = queryNAR.getInt(5);
-            narInfo->narHash = Hash(queryNAR.getStr(6));
-            narInfo->narSize = queryNAR.getInt(7);
-            for (auto & r : tokenizeString<Strings>(queryNAR.getStr(8), " "))
+            narInfo->url = queryNAR.getStr(3);
+            narInfo->compression = queryNAR.getStr(4);
+            if (!queryNAR.isNull(5))
+                narInfo->fileHash = Hash(queryNAR.getStr(5));
+            narInfo->fileSize = queryNAR.getInt(6);
+            narInfo->narHash = Hash(queryNAR.getStr(7));
+            narInfo->narSize = queryNAR.getInt(8);
+            for (auto & r : tokenizeString<Strings>(queryNAR.getStr(9), " "))
                 narInfo->references.insert(cache.storeDir + "/" + r);
-            if (!queryNAR.isNull(9))
-                narInfo->deriver = cache.storeDir + "/" + queryNAR.getStr(9);
-            for (auto & sig : tokenizeString<Strings>(queryNAR.getStr(10), " "))
+            if (!queryNAR.isNull(10))
+                narInfo->deriver = cache.storeDir + "/" + queryNAR.getStr(10);
+            for (auto & sig : tokenizeString<Strings>(queryNAR.getStr(11), " "))
                 narInfo->sigs.insert(sig);
-            narInfo->ca = queryNAR.getStr(11);
 
             return {oValid, narInfo};
         });
@@ -245,7 +243,6 @@ public:
                     (concatStringsSep(" ", info->shortRefs()))
                     (info->deriver != "" ? baseNameOf(info->deriver) : "", info->deriver != "")
                     (concatStringsSep(" ", info->sigs))
-                    (info->ca)
                     (time(0)).exec();
 
             } else {

src/libstore/parsed-derivations.cc

@@ -1,111 +0,0 @@
-#include "parsed-derivations.hh"
-
-namespace nix {
-
-ParsedDerivation::ParsedDerivation(const Path & drvPath, BasicDerivation & drv)
-    : drvPath(drvPath), drv(drv)
-{
-    /* Parse the __json attribute, if any. */
-    auto jsonAttr = drv.env.find("__json");
-    if (jsonAttr != drv.env.end()) {
-        try {
-            structuredAttrs = nlohmann::json::parse(jsonAttr->second);
-        } catch (std::exception & e) {
-            throw Error("cannot process __json attribute of '%s': %s", drvPath, e.what());
-        }
-    }
-}
-
-std::experimental::optional<std::string> ParsedDerivation::getStringAttr(const std::string & name) const
-{
-    if (structuredAttrs) {
-        auto i = structuredAttrs->find(name);
-        if (i == structuredAttrs->end())
-            return {};
-        else {
-            if (!i->is_string())
-                throw Error("attribute '%s' of derivation '%s' must be a string", name, drvPath);
-            return i->get<std::string>();
-        }
-    } else {
-        auto i = drv.env.find(name);
-        if (i == drv.env.end())
-            return {};
-        else
-            return i->second;
-    }
-}
-
-bool ParsedDerivation::getBoolAttr(const std::string & name, bool def) const
-{
-    if (structuredAttrs) {
-        auto i = structuredAttrs->find(name);
-        if (i == structuredAttrs->end())
-            return def;
-        else {
-            if (!i->is_boolean())
-                throw Error("attribute '%s' of derivation '%s' must be a Boolean", name, drvPath);
-            return i->get<bool>();
-        }
-    } else {
-        auto i = drv.env.find(name);
-        if (i == drv.env.end())
-            return def;
-        else
-            return i->second == "1";
-    }
-}
-
-std::experimental::optional<Strings> ParsedDerivation::getStringsAttr(const std::string & name) const
-{
-    if (structuredAttrs) {
-        auto i = structuredAttrs->find(name);
-        if (i == structuredAttrs->end())
-            return {};
-        else {
-            if (!i->is_array())
-                throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, drvPath);
-            Strings res;
-            for (auto j = i->begin(); j != i->end(); ++j) {
-                if (!j->is_string())
-                    throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, drvPath);
-                res.push_back(j->get<std::string>());
-            }
-            return res;
-        }
-    } else {
-        auto i = drv.env.find(name);
-        if (i == drv.env.end())
-            return {};
-        else
-            return tokenizeString<Strings>(i->second);
-    }
-}
-
-StringSet ParsedDerivation::getRequiredSystemFeatures() const
-{
-    StringSet res;
-    for (auto & i : getStringsAttr("requiredSystemFeatures").value_or(Strings()))
-        res.insert(i);
-    return res;
-}
-
-bool ParsedDerivation::canBuildLocally() const
-{
-    if (drv.platform != settings.thisSystem.get()
-        && !settings.extraPlatforms.get().count(drv.platform)
-        && !drv.isBuiltin())
-        return false;
-
-    for (auto & feature : getRequiredSystemFeatures())
-        if (!settings.systemFeatures.get().count(feature)) return false;
-
-    return true;
-}
-
-bool ParsedDerivation::willBuildLocally() const
-{
-    return getBoolAttr("preferLocalBuild") && canBuildLocally();
-}
-
-}

src/libstore/parsed-derivations.hh

@@ -1,35 +0,0 @@
-#include "derivations.hh"
-
-#include <nlohmann/json.hpp>
-
-namespace nix {
-
-class ParsedDerivation
-{
-    Path drvPath;
-    BasicDerivation & drv;
-    std::experimental::optional<nlohmann::json> structuredAttrs;
-
-public:
-
-    ParsedDerivation(const Path & drvPath, BasicDerivation & drv);
-
-    const std::experimental::optional<nlohmann::json> & getStructuredAttrs() const
-    {
-        return structuredAttrs;
-    }
-
-    std::experimental::optional<std::string> getStringAttr(const std::string & name) const;
-
-    bool getBoolAttr(const std::string & name, bool def = false) const;
-
-    std::experimental::optional<Strings> getStringsAttr(const std::string & name) const;
-
-    StringSet getRequiredSystemFeatures() const;
-
-    bool canBuildLocally() const;
-
-    bool willBuildLocally() const;
-};
-
-}

src/libstore/remote-store.cc

@@ -161,8 +161,7 @@ void RemoteStore::initConnection(Connection & conn)
         if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 11)
             conn.to << false;
 
-        auto ex = conn.processStderr();
-        if (ex) std::rethrow_exception(ex);
+        conn.processStderr();
     }
     catch (Error & e) {
         throw Error("cannot open connection to remote store '%s': %s", getUri(), e.what());
@@ -196,68 +195,22 @@ void RemoteStore::setOptions(Connection & conn)
             conn.to << i.first << i.second.value;
     }
 
-    auto ex = conn.processStderr();
-    if (ex) std::rethrow_exception(ex);
-}
-
-
-/* A wrapper around Pool<RemoteStore::Connection>::Handle that marks
-   the connection as bad (causing it to be closed) if a non-daemon
-   exception is thrown before the handle is closed. Such an exception
-   causes a deviation from the expected protocol and therefore a
-   desynchronization between the client and daemon. */
-struct ConnectionHandle
-{
-    Pool<RemoteStore::Connection>::Handle handle;
-    bool daemonException = false;
-
-    ConnectionHandle(Pool<RemoteStore::Connection>::Handle && handle)
-        : handle(std::move(handle))
-    { }
-
-    ConnectionHandle(ConnectionHandle && h)
-        : handle(std::move(h.handle))
-    { }
-
-    ~ConnectionHandle()
-    {
-        if (!daemonException && std::uncaught_exception()) {
-            handle.markBad();
-            debug("closing daemon connection because of an exception");
-        }
-    }
-
-    RemoteStore::Connection * operator -> () { return &*handle; }
-
-    void processStderr(Sink * sink = 0, Source * source = 0)
-    {
-        auto ex = handle->processStderr(sink, source);
-        if (ex) {
-            daemonException = true;
-            std::rethrow_exception(ex);
-        }
-    }
-};
-
-
-ConnectionHandle RemoteStore::getConnection()
-{
-    return ConnectionHandle(connections->get());
+    conn.processStderr();
 }
 
 
 bool RemoteStore::isValidPathUncached(const Path & path)
 {
-    auto conn(getConnection());
+    auto conn(connections->get());
     conn->to << wopIsValidPath << path;
-    conn.processStderr();
+    conn->processStderr();
     return readInt(conn->from);
 }
 
 
 PathSet RemoteStore::queryValidPaths(const PathSet & paths, SubstituteFlag maybeSubstitute)
 {
-    auto conn(getConnection());
+    auto conn(connections->get());
     if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) {
         PathSet res;
         for (auto & i : paths)
@@ -265,7 +218,7 @@ PathSet RemoteStore::queryValidPaths(const PathSet & paths, SubstituteFlag maybe
         return res;
     } else {
         conn->to << wopQueryValidPaths << paths;
-        conn.processStderr();
+        conn->processStderr();
         return readStorePaths<PathSet>(*this, conn->from);
     }
 }
@@ -273,27 +226,27 @@ PathSet RemoteStore::queryValidPaths(const PathSet & paths, SubstituteFlag maybe
 
 PathSet RemoteStore::queryAllValidPaths()
 {
-    auto conn(getConnection());
+    auto conn(connections->get());
     conn->to << wopQueryAllValidPaths;
-    conn.processStderr();
+    conn->processStderr();
     return readStorePaths<PathSet>(*this, conn->from);
 }
 
 
 PathSet RemoteStore::querySubstitutablePaths(const PathSet & paths)
 {
-    auto conn(getConnection());
+    auto conn(connections->get());
     if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) {
         PathSet res;
         for (auto & i : paths) {
             conn->to << wopHasSubstitutes << i;
-            conn.processStderr();
+            conn->processStderr();
             if (readInt(conn->from)) res.insert(i);
         }
         return res;
     } else {
         conn->to << wopQuerySubstitutablePaths << paths;
-        conn.processStderr();
+        conn->processStderr();
         return readStorePaths<PathSet>(*this, conn->from);
     }
 }
@@ -304,14 +257,14 @@ void RemoteStore::querySubstitutablePathInfos(const PathSet & paths,
 {
     if (paths.empty()) return;
 
-    auto conn(getConnection());
+    auto conn(connections->get());
 
     if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) {
 
         for (auto & i : paths) {
             SubstitutablePathInfo info;
             conn->to << wopQuerySubstitutablePathInfo << i;
-            conn.processStderr();
+            conn->processStderr();
             unsigned int reply = readInt(conn->from);
             if (reply == 0) continue;
             info.deriver = readString(conn->from);
@@ -325,7 +278,7 @@ void RemoteStore::querySubstitutablePathInfos(const PathSet & paths,
     } else {
 
         conn->to << wopQuerySubstitutablePathInfos << paths;
-        conn.processStderr();
+        conn->processStderr();
         size_t count = readNum<size_t>(conn->from);
         for (size_t n = 0; n < count; n++) {
             Path path = readStorePath(*this, conn->from);
@@ -347,10 +300,10 @@ void RemoteStore::queryPathInfoUncached(const Path & path,
     try {
         std::shared_ptr<ValidPathInfo> info;
         {
-            auto conn(getConnection());
+            auto conn(connections->get());
             conn->to << wopQueryPathInfo << path;
             try {
-                conn.processStderr();
+                conn->processStderr();
             } catch (Error & e) {
                 // Ugly backwards compatibility hack.
                 if (e.msg().find("is not valid") != std::string::npos)
@@ -382,9 +335,9 @@ void RemoteStore::queryPathInfoUncached(const Path & path,
 void RemoteStore::queryReferrers(const Path & path,
     PathSet & referrers)
 {
-    auto conn(getConnection());
+    auto conn(connections->get());
     conn->to << wopQueryReferrers << path;
-    conn.processStderr();
+    conn->processStderr();
     PathSet referrers2 = readStorePaths<PathSet>(*this, conn->from);
     referrers.insert(referrers2.begin(), referrers2.end());
 }
@@ -392,36 +345,36 @@ void RemoteStore::queryReferrers(const Path & path,
 
 PathSet RemoteStore::queryValidDerivers(const Path & path)
 {
-    auto conn(getConnection());
+    auto conn(connections->get());
     conn->to << wopQueryValidDerivers << path;
-    conn.processStderr();
+    conn->processStderr();
     return readStorePaths<PathSet>(*this, conn->from);
 }
 
 
 PathSet RemoteStore::queryDerivationOutputs(const Path & path)
 {
-    auto conn(getConnection());
+    auto conn(connections->get());
     conn->to << wopQueryDerivationOutputs << path;
-    conn.processStderr();
+    conn->processStderr();
     return readStorePaths<PathSet>(*this, conn->from);
 }
 
 
 PathSet RemoteStore::queryDerivationOutputNames(const Path & path)
 {
-    auto conn(getConnection());
+    auto conn(connections->get());
     conn->to << wopQueryDerivationOutputNames << path;
-    conn.processStderr();
+    conn->processStderr();
     return readStrings<PathSet>(conn->from);
 }
 
 
 Path RemoteStore::queryPathFromHashPart(const string & hashPart)
 {
-    auto conn(getConnection());
+    auto conn(connections->get());
     conn->to << wopQueryPathFromHashPart << hashPart;
-    conn.processStderr();
+    conn->processStderr();
     Path path = readString(conn->from);
     if (!path.empty()) assertStorePath(path);
     return path;
@@ -431,7 +384,7 @@ Path RemoteStore::queryPathFromHashPart(const string & hashPart)
 void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
     RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor)
 {
-    auto conn(getConnection());
+    auto conn(connections->get());
 
     if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 18) {
         conn->to << wopImportPaths;
@@ -450,7 +403,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
                 ;
         });
 
-        conn.processStderr(0, source2.get());
+        conn->processStderr(0, source2.get());
 
         auto importedPaths = readStorePaths<PathSet>(*this, conn->from);
         assert(importedPaths.size() <= 1);
@@ -464,7 +417,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
                  << repair << !checkSigs;
         bool tunnel = GET_PROTOCOL_MINOR(conn->daemonVersion) >= 21;
         if (!tunnel) copyNAR(source, conn->to);
-        conn.processStderr(0, tunnel ? &source : nullptr);
+        conn->processStderr(0, tunnel ? &source : nullptr);
     }
 }
 
@@ -474,7 +427,7 @@ Path RemoteStore::addToStore(const string & name, const Path & _srcPath,
 {
     if (repair) throw Error("repairing is not supported when building through the Nix daemon");
 
-    auto conn(getConnection());
+    auto conn(connections->get());
 
     Path srcPath(absPath(_srcPath));
 
@@ -492,13 +445,13 @@ Path RemoteStore::addToStore(const string & name, const Path & _srcPath,
             dumpPath(srcPath, conn->to, filter);
         }
         conn->to.warn = false;
-        conn.processStderr();
+        conn->processStderr();
     } catch (SysError & e) {
         /* Daemon closed while we were sending the path. Probably OOM
            or I/O error. */
         if (e.errNo == EPIPE)
             try {
-                conn.processStderr();
+                conn->processStderr();
             } catch (EndOfFile & e) { }
         throw;
     }
@@ -512,17 +465,17 @@ Path RemoteStore::addTextToStore(const string & name, const string & s,
 {
     if (repair) throw Error("repairing is not supported when building through the Nix daemon");
 
-    auto conn(getConnection());
+    auto conn(connections->get());
     conn->to << wopAddTextToStore << name << s << references;
 
-    conn.processStderr();
+    conn->processStderr();
     return readStorePath(*this, conn->from);
 }
 
 
 void RemoteStore::buildPaths(const PathSet & drvPaths, BuildMode buildMode)
 {
-    auto conn(getConnection());
+    auto conn(connections->get());
     conn->to << wopBuildPaths;
     if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 13) {
         conn->to << drvPaths;
@@ -541,7 +494,7 @@ void RemoteStore::buildPaths(const PathSet & drvPaths, BuildMode buildMode)
             drvPaths2.insert(string(i, 0, i.find('!')));
         conn->to << drvPaths2;
     }
-    conn.processStderr();
+    conn->processStderr();
     readInt(conn->from);
 }
 
@@ -549,9 +502,9 @@ void RemoteStore::buildPaths(const PathSet & drvPaths, BuildMode buildMode)
 BuildResult RemoteStore::buildDerivation(const Path & drvPath, const BasicDerivation & drv,
     BuildMode buildMode)
 {
-    auto conn(getConnection());
+    auto conn(connections->get());
     conn->to << wopBuildDerivation << drvPath << drv << buildMode;
-    conn.processStderr();
+    conn->processStderr();
     BuildResult res;
     unsigned int status;
     conn->from >> status >> res.errorMsg;
@@ -562,45 +515,45 @@ BuildResult RemoteStore::buildDerivation(const Path & drvPath, const BasicDeriva
 
 void RemoteStore::ensurePath(const Path & path)
 {
-    auto conn(getConnection());
+    auto conn(connections->get());
     conn->to << wopEnsurePath << path;
-    conn.processStderr();
+    conn->processStderr();
     readInt(conn->from);
 }
 
 
 void RemoteStore::addTempRoot(const Path & path)
 {
-    auto conn(getConnection());
+    auto conn(connections->get());
     conn->to << wopAddTempRoot << path;
-    conn.processStderr();
+    conn->processStderr();
     readInt(conn->from);
 }
 
 
 void RemoteStore::addIndirectRoot(const Path & path)
 {
-    auto conn(getConnection());
+    auto conn(connections->get());
     conn->to << wopAddIndirectRoot << path;
-    conn.processStderr();
+    conn->processStderr();
     readInt(conn->from);
 }
 
 
 void RemoteStore::syncWithGC()
 {
-    auto conn(getConnection());
+    auto conn(connections->get());
     conn->to << wopSyncWithGC;
-    conn.processStderr();
+    conn->processStderr();
     readInt(conn->from);
 }
 
 
 Roots RemoteStore::findRoots()
 {
-    auto conn(getConnection());
+    auto conn(connections->get());
     conn->to << wopFindRoots;
-    conn.processStderr();
+    conn->processStderr();
     size_t count = readNum<size_t>(conn->from);
     Roots result;
     while (count--) {
@@ -614,7 +567,7 @@ Roots RemoteStore::findRoots()
 
 void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results)
 {
-    auto conn(getConnection());
+    auto conn(connections->get());
 
     conn->to
         << wopCollectGarbage << options.action << options.pathsToDelete << options.ignoreLiveness
@@ -622,7 +575,7 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results)
         /* removed options */
         << 0 << 0 << 0;
 
-    conn.processStderr();
+    conn->processStderr();
 
     results.paths = readStrings<PathSet>(conn->from);
     results.bytesFreed = readLongLong(conn->from);
@@ -637,27 +590,27 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results)
 
 void RemoteStore::optimiseStore()
 {
-    auto conn(getConnection());
+    auto conn(connections->get());
     conn->to << wopOptimiseStore;
-    conn.processStderr();
+    conn->processStderr();
     readInt(conn->from);
 }
 
 
 bool RemoteStore::verifyStore(bool checkContents, RepairFlag repair)
 {
-    auto conn(getConnection());
+    auto conn(connections->get());
     conn->to << wopVerifyStore << checkContents << repair;
-    conn.processStderr();
+    conn->processStderr();
     return readInt(conn->from);
 }
 
 
 void RemoteStore::addSignatures(const Path & storePath, const StringSet & sigs)
 {
-    auto conn(getConnection());
+    auto conn(connections->get());
     conn->to << wopAddSignatures << storePath << sigs;
-    conn.processStderr();
+    conn->processStderr();
     readInt(conn->from);
 }
 
@@ -667,13 +620,13 @@ void RemoteStore::queryMissing(const PathSet & targets,
     unsigned long long & downloadSize, unsigned long long & narSize)
 {
     {
-        auto conn(getConnection());
+        auto conn(connections->get());
         if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 19)
             // Don't hold the connection handle in the fallback case
             // to prevent a deadlock.
             goto fallback;
         conn->to << wopQueryMissing << targets;
-        conn.processStderr();
+        conn->processStderr();
         willBuild = readStorePaths<PathSet>(*this, conn->from);
         willSubstitute = readStorePaths<PathSet>(*this, conn->from);
         unknown = readStorePaths<PathSet>(*this, conn->from);
@@ -688,15 +641,8 @@ void RemoteStore::queryMissing(const PathSet & targets,
 
 
 void RemoteStore::connect()
-{
-    auto conn(getConnection());
-}
-
-
-unsigned int RemoteStore::getProtocol()
 {
     auto conn(connections->get());
-    return conn->daemonVersion;
 }
 
 
@@ -733,7 +679,7 @@ static Logger::Fields readFields(Source & from)
 }
 
 
-std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source * source)
+void RemoteStore::Connection::processStderr(Sink * sink, Source * source)
 {
     to.flush();
 
@@ -758,7 +704,7 @@ std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source *
         else if (msg == STDERR_ERROR) {
             string error = readString(from);
             unsigned int status = readInt(from);
-            return std::make_exception_ptr(Error(status, error));
+            throw Error(status, error);
         }
 
         else if (msg == STDERR_NEXT)
@@ -792,8 +738,6 @@ std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source *
         else
             throw Error("got unknown message type %x from Nix daemon", msg);
     }
-
-    return nullptr;
 }
 
 static std::string uriScheme = "unix://";

src/libstore/remote-store.hh

@@ -14,7 +14,6 @@ class Pid;
 struct FdSink;
 struct FdSource;
 template<typename T> class Pool;
-struct ConnectionHandle;
 
 
 /* FIXME: RemoteStore is a misnomer - should be something like
@@ -98,15 +97,12 @@ public:
 
     void connect() override;
 
-    unsigned int getProtocol() override;
-
     void flushBadConnections();
 
 protected:
 
     struct Connection
     {
-        AutoCloseFD fd;
         FdSink to;
         FdSource from;
         unsigned int daemonVersion;
@@ -114,7 +110,7 @@ protected:
 
         virtual ~Connection();
 
-        std::exception_ptr processStderr(Sink * sink = 0, Source * source = 0);
+        void processStderr(Sink * sink = 0, Source * source = 0);
     };
 
     ref<Connection> openConnectionWrapper();
@@ -127,10 +123,6 @@ protected:
 
     virtual void setOptions(Connection & conn);
 
-    ConnectionHandle getConnection();
-
-    friend struct ConnectionHandle;
-
 private:
 
     std::atomic_bool failed{false};
@@ -148,6 +140,11 @@ public:
 
 private:
 
+    struct Connection : RemoteStore::Connection
+    {
+        AutoCloseFD fd;
+    };
+
     ref<RemoteStore::Connection> openConnection() override;
     std::experimental::optional<std::string> path;
 };

src/libstore/s3-binary-cache-store.cc

@@ -19,6 +19,8 @@
 #include <aws/core/utils/logging/LogMacros.h>
 #include <aws/core/utils/threading/Executor.h>
 #include <aws/s3/S3Client.h>
+#include <aws/s3/model/CreateBucketRequest.h>
+#include <aws/s3/model/GetBucketLocationRequest.h>
 #include <aws/s3/model/GetObjectRequest.h>
 #include <aws/s3/model/HeadObjectRequest.h>
 #include <aws/s3/model/ListObjectsRequest.h>
@@ -82,8 +84,8 @@ static void initAWS()
     });
 }
 
-S3Helper::S3Helper(const string & profile, const string & region, const string & scheme, const string & endpoint)
-    : config(makeConfig(region, scheme, endpoint))
+S3Helper::S3Helper(const std::string & profile, const std::string & region, const std::string & endpoint)
+    : config(makeConfig(region, endpoint))
     , client(make_ref<Aws::S3::S3Client>(
             profile == ""
             ? std::dynamic_pointer_cast<Aws::Auth::AWSCredentialsProvider>(
@@ -114,14 +116,11 @@ class RetryStrategy : public Aws::Client::DefaultRetryStrategy
     }
 };
 
-ref<Aws::Client::ClientConfiguration> S3Helper::makeConfig(const string & region, const string & scheme, const string & endpoint)
+ref<Aws::Client::ClientConfiguration> S3Helper::makeConfig(const string & region, const string & endpoint)
 {
     initAWS();
     auto res = make_ref<Aws::Client::ClientConfiguration>();
     res->region = region;
-    if (!scheme.empty()) {
-        res->scheme = Aws::Http::SchemeMapper::FromString(scheme.c_str());
-    }
     if (!endpoint.empty()) {
         res->endpointOverride = endpoint;
     }
@@ -172,13 +171,10 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
 {
     const Setting<std::string> profile{this, "", "profile", "The name of the AWS configuration profile to use."};
     const Setting<std::string> region{this, Aws::Region::US_EAST_1, "region", {"aws-region"}};
-    const Setting<std::string> scheme{this, "", "scheme", "The scheme to use for S3 requests, https by default."};
     const Setting<std::string> endpoint{this, "", "endpoint", "An optional override of the endpoint to use when talking to S3."};
     const Setting<std::string> narinfoCompression{this, "", "narinfo-compression", "compression method for .narinfo files"};
     const Setting<std::string> lsCompression{this, "", "ls-compression", "compression method for .ls files"};
     const Setting<std::string> logCompression{this, "", "log-compression", "compression method for log/* files"};
-    const Setting<bool> multipartUpload{
-        this, false, "multipart-upload", "whether to use multi-part uploads"};
     const Setting<uint64_t> bufferSize{
         this, 5 * 1024 * 1024, "buffer-size", "size (in bytes) of each part in multi-part uploads"};
 
@@ -192,7 +188,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
         const Params & params, const std::string & bucketName)
         : S3BinaryCacheStore(params)
         , bucketName(bucketName)
-        , s3Helper(profile, region, scheme, endpoint)
+        , s3Helper(profile, region, endpoint)
     {
         diskCache = getNarInfoDiskCache();
     }
@@ -206,6 +202,32 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
     {
         if (!diskCache->cacheExists(getUri(), wantMassQuery_, priority)) {
 
+            /* Create the bucket if it doesn't already exists. */
+            // FIXME: HeadBucket would be more appropriate, but doesn't return
+            // an easily parsed 404 message.
+            auto res = s3Helper.client->GetBucketLocation(
+                Aws::S3::Model::GetBucketLocationRequest().WithBucket(bucketName));
+
+            if (!res.IsSuccess()) {
+                if (res.GetError().GetErrorType() != Aws::S3::S3Errors::NO_SUCH_BUCKET)
+                    throw Error(format("AWS error checking bucket '%s': %s") % bucketName % res.GetError().GetMessage());
+
+                printInfo("creating S3 bucket '%s'...", bucketName);
+
+                // Stupid S3 bucket locations.
+                auto bucketConfig = Aws::S3::Model::CreateBucketConfiguration();
+                if (s3Helper.config->region != "us-east-1")
+                    bucketConfig.SetLocationConstraint(
+                        Aws::S3::Model::BucketLocationConstraintMapper::GetBucketLocationConstraintForName(
+                            s3Helper.config->region));
+
+                checkAws(format("AWS error creating bucket '%s'") % bucketName,
+                    s3Helper.client->CreateBucket(
+                        Aws::S3::Model::CreateBucketRequest()
+                        .WithBucket(bucketName)
+                        .WithCreateBucketConfiguration(bucketConfig)));
+            }
+
             BinaryCacheStore::init();
 
             diskCache->createCache(getUri(), storeDir, wantMassQuery_, priority);
@@ -267,73 +289,48 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
         static std::shared_ptr<Aws::Utils::Threading::PooledThreadExecutor>
             executor = std::make_shared<Aws::Utils::Threading::PooledThreadExecutor>(maxThreads);
 
-        std::call_once(transferManagerCreated, [&]()
-        {
-            if (multipartUpload) {
-                TransferManagerConfiguration transferConfig(executor.get());
+        std::call_once(transferManagerCreated, [&]() {
 
-                transferConfig.s3Client = s3Helper.client;
-                transferConfig.bufferSize = bufferSize;
+            TransferManagerConfiguration transferConfig(executor.get());
 
-                transferConfig.uploadProgressCallback =
-                    [](const TransferManager *transferManager,
-                        const std::shared_ptr<const TransferHandle>
-                        &transferHandle)
-                    {
-                        //FIXME: find a way to properly abort the multipart upload.
-                        //checkInterrupt();
-                        debug("upload progress ('%s'): '%d' of '%d' bytes",
-                            transferHandle->GetKey(),
-                            transferHandle->GetBytesTransferred(),
-                            transferHandle->GetBytesTotalSize());
-                    };
+            transferConfig.s3Client = s3Helper.client;
+            transferConfig.bufferSize = bufferSize;
 
-                transferManager = TransferManager::Create(transferConfig);
-            }
+            transferConfig.uploadProgressCallback =
+                [&](const TransferManager *transferManager,
+                    const std::shared_ptr<const TransferHandle>
+                    &transferHandle)
+                {
+                    //FIXME: find a way to properly abort the multipart upload.
+                    //checkInterrupt();
+                    debug("upload progress ('%s'): '%d' of '%d' bytes",
+                        path,
+                        transferHandle->GetBytesTransferred(),
+                        transferHandle->GetBytesTotalSize());
+                };
+
+            transferManager = TransferManager::Create(transferConfig);
         });
 
         auto now1 = std::chrono::steady_clock::now();
 
-        if (transferManager) {
+        std::shared_ptr<TransferHandle> transferHandle =
+            transferManager->UploadFile(
+                stream, bucketName, path, mimeType,
+                Aws::Map<Aws::String, Aws::String>(),
+                nullptr, contentEncoding);
 
-            if (contentEncoding != "")
-                throw Error("setting a content encoding is not supported with S3 multi-part uploads");
+        transferHandle->WaitUntilFinished();
 
-            std::shared_ptr<TransferHandle> transferHandle =
-                transferManager->UploadFile(
-                    stream, bucketName, path, mimeType,
-                    Aws::Map<Aws::String, Aws::String>(),
-                    nullptr /*, contentEncoding */);
+        if (transferHandle->GetStatus() == TransferStatus::FAILED)
+            throw Error("AWS error: failed to upload 's3://%s/%s': %s",
+                bucketName, path, transferHandle->GetLastError().GetMessage());
 
-            transferHandle->WaitUntilFinished();
+        if (transferHandle->GetStatus() != TransferStatus::COMPLETED)
+            throw Error("AWS error: transfer status of 's3://%s/%s' in unexpected state",
+                bucketName, path);
 
-            if (transferHandle->GetStatus() == TransferStatus::FAILED)
-                throw Error("AWS error: failed to upload 's3://%s/%s': %s",
-                    bucketName, path, transferHandle->GetLastError().GetMessage());
-
-            if (transferHandle->GetStatus() != TransferStatus::COMPLETED)
-                throw Error("AWS error: transfer status of 's3://%s/%s' in unexpected state",
-                    bucketName, path);
-
-        } else {
-
-            auto request =
-                Aws::S3::Model::PutObjectRequest()
-                .WithBucket(bucketName)
-                .WithKey(path);
-
-            request.SetContentType(mimeType);
-
-            if (contentEncoding != "")
-                request.SetContentEncoding(contentEncoding);
-
-            auto stream = std::make_shared<istringstream_nocopy>(data);
-
-            request.SetBody(stream);
-
-            auto result = checkAws(fmt("AWS error uploading '%s'", path),
-                s3Helper.client->PutObject(request));
-        }
+        printTalkative("upload of '%s' completed", path);
 
         auto now2 = std::chrono::steady_clock::now();
 

src/libstore/s3.hh

@@ -14,9 +14,9 @@ struct S3Helper
     ref<Aws::Client::ClientConfiguration> config;
     ref<Aws::S3::S3Client> client;
 
-    S3Helper(const std::string & profile, const std::string & region, const std::string & scheme, const std::string & endpoint);
+    S3Helper(const std::string & profile, const std::string & region, const std::string & endpoint);
 
-    ref<Aws::Client::ClientConfiguration> makeConfig(const std::string & region, const std::string & scheme, const std::string & endpoint);
+    ref<Aws::Client::ClientConfiguration> makeConfig(const std::string & region, const std::string & endpoint);
 
     struct DownloadResult
     {

src/libstore/ssh.cc

@@ -1,5 +1,4 @@
 #include "ssh.hh"
-#include "affinity.hh"
 
 namespace nix {
 
@@ -35,9 +34,7 @@ std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(const std::string
 
     auto conn = std::make_unique<Connection>();
     conn->sshPid = startProcess([&]() {
-        restoreAffinity();
         restoreSignals();
-        restoreMountNamespace();
 
         close(in.writeSide.get());
         close(out.readSide.get());

src/libstore/store-api.cc

@@ -320,8 +320,6 @@ ref<const ValidPathInfo> Store::queryPathInfo(const Path & storePath)
 void Store::queryPathInfo(const Path & storePath,
     Callback<ref<ValidPathInfo>> callback)
 {
-    assertStorePath(storePath);
-
     auto hashPart = storePathToHash(storePath);
 
     try {
@@ -588,19 +586,15 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
 
     uint64_t total = 0;
 
+    // FIXME
+#if 0
     if (!info->narHash) {
-        StringSink sink;
-        srcStore->narFromPath({storePath}, sink);
         auto info2 = make_ref<ValidPathInfo>(*info);
         info2->narHash = hashString(htSHA256, *sink.s);
         if (!info->narSize) info2->narSize = sink.s->size();
-        if (info->ultimate) info2->ultimate = false;
         info = info2;
-
-        StringSource source(*sink.s);
-        dstStore->addToStore(*info, source, repair, checkSigs);
-        return;
     }
+#endif
 
     if (info->ultimate) {
         auto info2 = make_ref<ValidPathInfo>(*info);
@@ -616,7 +610,7 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
         });
         srcStore->narFromPath({storePath}, wrapperSink);
     }, [&]() {
-        throw EndOfFile("NAR for '%s' fetched from '%s' is incomplete", storePath, srcStore->getUri());
+	throw EndOfFile("NAR for '%s' fetched from '%s' is incomplete", storePath, srcStore->getUri());
     });
 
     dstStore->addToStore(*info, *source, repair, checkSigs);

src/libstore/store-api.hh

@@ -23,7 +23,6 @@ MakeError(BuildError, Error) /* denotes a permanent build failure */
 MakeError(InvalidPath, Error)
 MakeError(Unsupported, Error)
 MakeError(SubstituteGone, Error)
-MakeError(SubstituterDisabled, Error)
 
 
 struct BasicDerivation;
@@ -599,12 +598,6 @@ public:
        a notion of connection. Otherwise this is a no-op. */
     virtual void connect() { };
 
-    /* Get the protocol version of this store or it's connection. */
-    virtual unsigned int getProtocol()
-    {
-        return 0;
-    };
-
     /* Get the priority of the store, used to order substituters. In
        particular, binary caches can specify a priority field in their
        "nix-cache-info" file. Lower value means higher priority. */

src/libutil/archive.cc

@@ -283,7 +283,7 @@ void parseDump(ParseSink & sink, Source & source)
 {
     string version;
     try {
-        version = readString(source, narVersionMagic1.size());
+        version = readString(source);
     } catch (SerialisationError & e) {
         /* This generally means the integer at the start couldn't be
            decoded.  Ignore and throw the exception below. */

src/libutil/compression.cc

@@ -250,7 +250,7 @@ struct XzCompressionSink : CompressionSink
             ret = lzma_stream_encoder_mt(&strm, &mt_options);
             done = true;
 #else
-            printMsg(lvlError, "warning: parallel XZ compression requested but not supported, falling back to single-threaded compression");
+            printMsg(lvlError, "warning: parallel compression requested but not supported for metho  d '%1%', falling back to single-threaded compression", method);
 #endif
         }
 

src/libutil/hash.cc

@@ -105,9 +105,9 @@ string printHash16or32(const Hash & hash)
 std::string Hash::to_string(Base base, bool includeType) const
 {
     std::string s;
-    if (base == SRI || includeType) {
+    if (includeType) {
         s += printHashType(type);
-        s += base == SRI ? '-' : ':';
+        s += ':';
     }
     switch (base) {
     case Base16:
@@ -117,7 +117,6 @@ std::string Hash::to_string(Base base, bool includeType) const
         s += printHash32(*this);
         break;
     case Base64:
-    case SRI:
         s += base64Encode(std::string((const char *) hash, hashSize));
         break;
     }
@@ -128,33 +127,28 @@ std::string Hash::to_string(Base base, bool includeType) const
 Hash::Hash(const std::string & s, HashType type)
     : type(type)
 {
+    auto colon = s.find(':');
+
     size_t pos = 0;
-    bool isSRI = false;
 
-    auto sep = s.find(':');
-    if (sep == string::npos) {
-        sep = s.find('-');
-        if (sep != string::npos) {
-            isSRI = true;
-        } else if (type == htUnknown)
+    if (colon == string::npos) {
+        if (type == htUnknown)
             throw BadHash("hash '%s' does not include a type", s);
-    }
-
-    if (sep != string::npos) {
-        string hts = string(s, 0, sep);
+    } else {
+        string hts = string(s, 0, colon);
         this->type = parseHashType(hts);
         if (this->type == htUnknown)
             throw BadHash("unknown hash type '%s'", hts);
         if (type != htUnknown && type != this->type)
             throw BadHash("hash '%s' should have type '%s'", s, printHashType(type));
-        pos = sep + 1;
+        pos = colon + 1;
     }
 
     init();
 
     size_t size = s.size() - pos;
 
-    if (!isSRI && size == base16Len()) {
+    if (size == base16Len()) {
 
         auto parseHexDigit = [&](char c) {
             if (c >= '0' && c <= '9') return c - '0';
@@ -170,7 +164,7 @@ Hash::Hash(const std::string & s, HashType type)
         }
     }
 
-    else if (!isSRI && size == base32Len()) {
+    else if (size == base32Len()) {
 
         for (unsigned int n = 0; n < size; ++n) {
             char c = s[pos + size - n - 1];
@@ -193,10 +187,10 @@ Hash::Hash(const std::string & s, HashType type)
         }
     }
 
-    else if (isSRI || size == base64Len()) {
+    else if (size == base64Len()) {
         auto d = base64Decode(std::string(s, pos));
         if (d.size() != hashSize)
-            throw BadHash("invalid %s hash '%s'", isSRI ? "SRI" : "base-64", s);
+            throw BadHash("invalid base-64 hash '%s'", s);
         assert(hashSize);
         memcpy(hash, d.data(), hashSize);
     }

src/libutil/hash.hh

@@ -20,7 +20,7 @@ const int sha512HashSize = 64;
 
 extern const string base32Chars;
 
-enum Base : int { Base64, Base32, Base16, SRI };
+enum Base : int { Base64, Base32, Base16 };
 
 
 struct Hash
@@ -38,9 +38,8 @@ struct Hash
     Hash(HashType type) : type(type) { init(); };
 
     /* Initialize the hash from a string representation, in the format
-       "[<type>:]<base16|base32|base64>" or "<type>-<base64>" (a
-       Subresource Integrity hash expression). If the 'type' argument
-       is htUnknown, then the hash type must be specified in the
+       "[<type>:]<base16|base32|base64>". If the 'type' argument is
+       htUnknown, then the hash type must be specified in the
        string. */
     Hash(const std::string & s, HashType type = htUnknown);
 

src/libutil/local.mk

@@ -7,3 +7,7 @@ libutil_DIR := $(d)
 libutil_SOURCES := $(wildcard $(d)/*.cc)
 
 libutil_LDFLAGS = $(LIBLZMA_LIBS) -lbz2 -pthread $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) -lboost_context
+
+libutil_LIBS = libformat
+
+libutil_CXXFLAGS = -DBROTLI=\"$(brotli)\"

src/libutil/pool.hh

@@ -97,7 +97,6 @@ public:
     private:
         Pool & pool;
         std::shared_ptr<R> r;
-        bool bad = false;
 
         friend Pool;
 
@@ -113,8 +112,7 @@ public:
             if (!r) return;
             {
                 auto state_(pool.state.lock());
-                if (!bad)
-                    state_->idle.push_back(ref<R>(r));
+                state_->idle.push_back(ref<R>(r));
                 assert(state_->inUse);
                 state_->inUse--;
             }
@@ -123,8 +121,6 @@ public:
 
         R * operator -> () { return &*r; }
         R & operator * () { return *r; }
-
-        void markBad() { bad = true; }
     };
 
     Handle get()

src/libutil/serialise.cc

@@ -169,13 +169,17 @@ std::unique_ptr<Source> sinkToSource(
     {
         typedef boost::coroutines2::coroutine<std::string> coro_t;
 
-        std::function<void(Sink &)> fun;
         std::function<void()> eof;
-        std::experimental::optional<coro_t::pull_type> coro;
-        bool started = false;
+        coro_t::pull_type coro;
 
         SinkToSource(std::function<void(Sink &)> fun, std::function<void()> eof)
-            : fun(fun), eof(eof)
+            : eof(eof)
+            , coro([&](coro_t::push_type & yield) {
+                LambdaSink sink([&](const unsigned char * data, size_t len) {
+                    if (len) yield(std::string((const char *) data, len));
+                });
+                fun(sink);
+            })
         {
         }
 
@@ -184,19 +188,11 @@ std::unique_ptr<Source> sinkToSource(
 
         size_t read(unsigned char * data, size_t len) override
         {
-            if (!coro)
-                coro = coro_t::pull_type([&](coro_t::push_type & yield) {
-                    LambdaSink sink([&](const unsigned char * data, size_t len) {
-                            if (len) yield(std::string((const char *) data, len));
-                        });
-                    fun(sink);
-                });
-
-            if (!*coro) { eof(); abort(); }
+            if (!coro) { eof(); abort(); }
 
             if (pos == cur.size()) {
-                if (!cur.empty()) (*coro)();
-                cur = coro->get();
+                if (!cur.empty()) coro();
+                cur = coro.get();
                 pos = 0;
             }
 
@@ -272,17 +268,16 @@ void readPadding(size_t len, Source & source)
 size_t readString(unsigned char * buf, size_t max, Source & source)
 {
     auto len = readNum<size_t>(source);
-    if (len > max) throw SerialisationError("string is too long");
+    if (len > max) throw Error("string is too long");
     source(buf, len);
     readPadding(len, source);
     return len;
 }
 
 
-string readString(Source & source, size_t max)
+string readString(Source & source)
 {
     auto len = readNum<size_t>(source);
-    if (len > max) throw SerialisationError("string is too long");
     std::string res(len, 0);
     source((unsigned char*) res.data(), len);
     readPadding(len, source);

src/libutil/serialise.hh

@@ -284,7 +284,7 @@ inline uint64_t readLongLong(Source & source)
 
 void readPadding(size_t len, Source & source);
 size_t readString(unsigned char * buf, size_t max, Source & source);
-string readString(Source & source, size_t max = std::numeric_limits<size_t>::max());
+string readString(Source & source);
 template<class T> T readStrings(Source & source);
 
 Source & operator >> (Source & in, string & s);

src/libutil/util.cc

@@ -167,7 +167,7 @@ Path dirOf(const Path & path)
 {
     Path::size_type pos = path.rfind('/');
     if (pos == string::npos)
-        return ".";
+        throw Error(format("invalid file name '%1%'") % path);
     return pos == 0 ? "/" : Path(path, 0, pos);
 }
 
@@ -202,7 +202,7 @@ bool isInDir(const Path & path, const Path & dir)
 
 bool isDirOrInDir(const Path & path, const Path & dir)
 {
-    return path == dir || isInDir(path, dir);
+    return path == dir or isInDir(path, dir);
 }
 
 
@@ -468,7 +468,7 @@ static Lazy<Path> getHome2([]() {
         std::vector<char> buf(16384);
         struct passwd pwbuf;
         struct passwd * pw;
-        if (getpwuid_r(geteuid(), &pwbuf, buf.data(), buf.size(), &pw) != 0
+        if (getpwuid_r(getuid(), &pwbuf, buf.data(), buf.size(), &pw) != 0
             || !pw || !pw->pw_dir || !pw->pw_dir[0])
             throw Error("cannot determine user's home directory");
         homeDir = pw->pw_dir;
@@ -496,15 +496,6 @@ Path getConfigDir()
     return configDir;
 }
 
-std::vector<Path> getConfigDirs()
-{
-    Path configHome = getConfigDir();
-    string configDirs = getEnv("XDG_CONFIG_DIRS");
-    std::vector<Path> result = tokenizeString<std::vector<string>>(configDirs, ":");
-    result.insert(result.begin(), configHome);
-    return result;
-}
-
 
 Path getDataDir()
 {
@@ -936,8 +927,6 @@ pid_t startProcess(std::function<void()> fun, const ProcessOptions & options)
                 throw SysError("setting death signal");
 #endif
             restoreAffinity();
-            if (options.restoreMountNamespace)
-                restoreMountNamespace();
             fun();
         } catch (std::exception & e) {
             try {
@@ -1506,26 +1495,4 @@ std::unique_ptr<InterruptCallback> createInterruptCallback(std::function<void()>
     return std::unique_ptr<InterruptCallback>(res.release());
 }
 
-static AutoCloseFD fdSavedMountNamespace;
-
-void saveMountNamespace()
-{
-#if __linux__
-    std::once_flag done;
-    std::call_once(done, []() {
-        fdSavedMountNamespace = open("/proc/self/ns/mnt", O_RDONLY);
-        if (!fdSavedMountNamespace)
-            throw SysError("saving parent mount namespace");
-    });
-#endif
-}
-
-void restoreMountNamespace()
-{
-#if __linux__
-    if (fdSavedMountNamespace && setns(fdSavedMountNamespace.get(), CLONE_NEWNS) == -1)
-        throw SysError("restoring parent mount namespace");
-#endif
-}
-
 }

src/libutil/util.hh

@@ -131,9 +131,6 @@ Path getCacheDir();
 /* Return $XDG_CONFIG_HOME or $HOME/.config. */
 Path getConfigDir();
 
-/* Return the directories to search for user configuration files */
-std::vector<Path> getConfigDirs();
-
 /* Return $XDG_DATA_HOME or $HOME/.local/share. */
 Path getDataDir();
 
@@ -250,7 +247,6 @@ struct ProcessOptions
     bool dieWithParent = true;
     bool runExitHandlers = false;
     bool allowVfork = true;
-    bool restoreMountNamespace = true;
 };
 
 pid_t startProcess(std::function<void()> fun, const ProcessOptions & options = ProcessOptions());
@@ -515,13 +511,4 @@ typedef std::function<bool(const Path & path)> PathFilter;
 extern PathFilter defaultPathFilter;
 
 
-/* Save the current mount namespace. Ignored if called more than
-   once. */
-void saveMountNamespace();
-
-/* Restore the mount namespace saved by saveMountNamespace(). Ignored
-   if saveMountNamespace() was never called. */
-void restoreMountNamespace();
-
-
 }

src/linenoise/ConvertUTF.cpp

@@ -0,0 +1,542 @@
+/*
+ * Copyright 2001-2004 Unicode, Inc.
+ * 
+ * Disclaimer
+ * 
+ * This source code is provided as is by Unicode, Inc. No claims are
+ * made as to fitness for any particular purpose. No warranties of any
+ * kind are expressed or implied. The recipient agrees to determine
+ * applicability of information provided. If this file has been
+ * purchased on magnetic or optical media from Unicode, Inc., the
+ * sole remedy for any claim will be exchange of defective media
+ * within 90 days of receipt.
+ * 
+ * Limitations on Rights to Redistribute This Code
+ * 
+ * Unicode, Inc. hereby grants the right to freely use the information
+ * supplied in this file in the creation of products supporting the
+ * Unicode Standard, and to make copies of this file in any form
+ * for internal or external distribution as long as this notice
+ * remains attached.
+ */
+
+/* ---------------------------------------------------------------------
+
+    Conversions between UTF32, UTF-16, and UTF-8. Source code file.
+    Author: Mark E. Davis, 1994.
+    Rev History: Rick McGowan, fixes & updates May 2001.
+    Sept 2001: fixed const & error conditions per
+        mods suggested by S. Parent & A. Lillich.
+    June 2002: Tim Dodd added detection and handling of incomplete
+        source sequences, enhanced error detection, added casts
+        to eliminate compiler warnings.
+    July 2003: slight mods to back out aggressive FFFE detection.
+    Jan 2004: updated switches in from-UTF8 conversions.
+    Oct 2004: updated to use UNI_MAX_LEGAL_UTF32 in UTF-32 conversions.
+
+    See the header file "ConvertUTF.h" for complete documentation.
+
+------------------------------------------------------------------------ */
+
+#include "ConvertUTF.h"
+#ifdef CVTUTF_DEBUG
+#include <stdio.h>
+#endif
+
+namespace linenoise_ng {
+
+static const int halfShift  = 10; /* used for shifting by 10 bits */
+
+static const UTF32 halfBase = 0x0010000UL;
+static const UTF32 halfMask = 0x3FFUL;
+
+#define UNI_SUR_HIGH_START  (UTF32)0xD800
+#define UNI_SUR_HIGH_END    (UTF32)0xDBFF
+#define UNI_SUR_LOW_START   (UTF32)0xDC00
+#define UNI_SUR_LOW_END     (UTF32)0xDFFF
+#define false      0
+#define true        1
+
+/* --------------------------------------------------------------------- */
+
+ConversionResult ConvertUTF32toUTF16 (
+        const UTF32** sourceStart, const UTF32* sourceEnd, 
+        char16_t** targetStart, char16_t* targetEnd, ConversionFlags flags) {
+    ConversionResult result = conversionOK;
+    const UTF32* source = *sourceStart;
+    char16_t* target = *targetStart;
+    while (source < sourceEnd) {
+        UTF32 ch;
+        if (target >= targetEnd) {
+            result = targetExhausted; break;
+        }
+        ch = *source++;
+        if (ch <= UNI_MAX_BMP) { /* Target is a character <= 0xFFFF */
+            /* UTF-16 surrogate values are illegal in UTF-32; 0xffff or 0xfffe are both reserved values */
+            if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) {
+                if (flags == strictConversion) {
+                    --source; /* return to the illegal value itself */
+                    result = sourceIllegal;
+                    break;
+                } else {
+                    *target++ = UNI_REPLACEMENT_CHAR;
+                }
+            } else {
+                *target++ = (UTF16)ch; /* normal case */
+            }
+        } else if (ch > UNI_MAX_LEGAL_UTF32) {
+            if (flags == strictConversion) {
+                result = sourceIllegal;
+            } else {
+                *target++ = UNI_REPLACEMENT_CHAR;
+            }
+        } else {
+            /* target is a character in range 0xFFFF - 0x10FFFF. */
+            if (target + 1 >= targetEnd) {
+                --source; /* Back up source pointer! */
+                result = targetExhausted; break;
+            }
+            ch -= halfBase;
+            *target++ = (UTF16)((ch >> halfShift) + UNI_SUR_HIGH_START);
+            *target++ = (UTF16)((ch & halfMask) + UNI_SUR_LOW_START);
+        }
+    }
+    *sourceStart = source;
+    *targetStart = target;
+    return result;
+}
+
+/* --------------------------------------------------------------------- */
+
+ConversionResult ConvertUTF16toUTF32 (
+        const UTF16** sourceStart, const UTF16* sourceEnd, 
+        UTF32** targetStart, UTF32* targetEnd, ConversionFlags flags) {
+    ConversionResult result = conversionOK;
+    const UTF16* source = *sourceStart;
+    UTF32* target = *targetStart;
+    UTF32 ch, ch2;
+    while (source < sourceEnd) {
+        const UTF16* oldSource = source; /*  In case we have to back up because of target overflow. */
+        ch = *source++;
+        /* If we have a surrogate pair, convert to UTF32 first. */
+        if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_HIGH_END) {
+            /* If the 16 bits following the high surrogate are in the source buffer... */
+            if (source < sourceEnd) {
+                ch2 = *source;
+                /* If it's a low surrogate, convert to UTF32. */
+                if (ch2 >= UNI_SUR_LOW_START && ch2 <= UNI_SUR_LOW_END) {
+                    ch = ((ch - UNI_SUR_HIGH_START) << halfShift)
+                        + (ch2 - UNI_SUR_LOW_START) + halfBase;
+                    ++source;
+                } else if (flags == strictConversion) { /* it's an unpaired high surrogate */
+                    --source; /* return to the illegal value itself */
+                    result = sourceIllegal;
+                    break;
+                }
+            } else { /* We don't have the 16 bits following the high surrogate. */
+                --source; /* return to the high surrogate */
+                result = sourceExhausted;
+                break;
+            }
+        } else if (flags == strictConversion) {
+            /* UTF-16 surrogate values are illegal in UTF-32 */
+            if (ch >= UNI_SUR_LOW_START && ch <= UNI_SUR_LOW_END) {
+                --source; /* return to the illegal value itself */
+                result = sourceIllegal;
+                break;
+            }
+        }
+        if (target >= targetEnd) {
+            source = oldSource; /* Back up source pointer! */
+            result = targetExhausted; break;
+        }
+        *target++ = ch;
+    }
+    *sourceStart = source;
+    *targetStart = target;
+#ifdef CVTUTF_DEBUG
+if (result == sourceIllegal) {
+    fprintf(stderr, "ConvertUTF16toUTF32 illegal seq 0x%04x,%04x\n", ch, ch2);
+    fflush(stderr);
+}
+#endif
+    return result;
+}
+
+/* --------------------------------------------------------------------- */
+
+/*
+ * Index into the table below with the first byte of a UTF-8 sequence to
+ * get the number of trailing bytes that are supposed to follow it.
+ * Note that *legal* UTF-8 values can't have 4 or 5-bytes. The table is
+ * left as-is for anyone who may want to do such conversion, which was
+ * allowed in earlier algorithms.
+ */
+static const char trailingBytesForUTF8[256] = {
+    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+    1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
+    2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 3,3,3,3,3,3,3,3,4,4,4,4,5,5,5,5
+};
+
+/*
+ * Magic values subtracted from a buffer value during UTF8 conversion.
+ * This table contains as many values as there might be trailing bytes
+ * in a UTF-8 sequence.
+ */
+static const UTF32 offsetsFromUTF8[6] = { 0x00000000UL, 0x00003080UL, 0x000E2080UL, 
+                     0x03C82080UL, 0xFA082080UL, 0x82082080UL };
+
+/*
+ * Once the bits are split out into bytes of UTF-8, this is a mask OR-ed
+ * into the first byte, depending on how many bytes follow.  There are
+ * as many entries in this table as there are UTF-8 sequence types.
+ * (I.e., one byte sequence, two byte... etc.). Remember that sequencs
+ * for *legal* UTF-8 will be 4 or fewer bytes total.
+ */
+static const UTF8 firstByteMark[7] = { 0x00, 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC };
+
+/* --------------------------------------------------------------------- */
+
+/* The interface converts a whole buffer to avoid function-call overhead.
+ * Constants have been gathered. Loops & conditionals have been removed as
+ * much as possible for efficiency, in favor of drop-through switches.
+ * (See "Note A" at the bottom of the file for equivalent code.)
+ * If your compiler supports it, the "isLegalUTF8" call can be turned
+ * into an inline function.
+ */
+
+/* --------------------------------------------------------------------- */
+
+ConversionResult ConvertUTF16toUTF8 (
+        const UTF16** sourceStart, const UTF16* sourceEnd, 
+        UTF8** targetStart, UTF8* targetEnd, ConversionFlags flags) {
+    ConversionResult result = conversionOK;
+    const UTF16* source = *sourceStart;
+    UTF8* target = *targetStart;
+    while (source < sourceEnd) {
+        UTF32 ch;
+        unsigned short bytesToWrite = 0;
+        const UTF32 byteMask = 0xBF;
+        const UTF32 byteMark = 0x80; 
+        const UTF16* oldSource = source; /* In case we have to back up because of target overflow. */
+        ch = *source++;
+        /* If we have a surrogate pair, convert to UTF32 first. */
+        if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_HIGH_END) {
+            /* If the 16 bits following the high surrogate are in the source buffer... */
+            if (source < sourceEnd) {
+                UTF32 ch2 = *source;
+                /* If it's a low surrogate, convert to UTF32. */
+                if (ch2 >= UNI_SUR_LOW_START && ch2 <= UNI_SUR_LOW_END) {
+                    ch = ((ch - UNI_SUR_HIGH_START) << halfShift)
+                        + (ch2 - UNI_SUR_LOW_START) + halfBase;
+                    ++source;
+                } else if (flags == strictConversion) { /* it's an unpaired high surrogate */
+                    --source; /* return to the illegal value itself */
+                    result = sourceIllegal;
+                    break;
+                }
+            } else { /* We don't have the 16 bits following the high surrogate. */
+                --source; /* return to the high surrogate */
+                result = sourceExhausted;
+                break;
+            }
+        } else if (flags == strictConversion) {
+            /* UTF-16 surrogate values are illegal in UTF-32 */
+            if (ch >= UNI_SUR_LOW_START && ch <= UNI_SUR_LOW_END) {
+                --source; /* return to the illegal value itself */
+                result = sourceIllegal;
+                break;
+            }
+        }
+        /* Figure out how many bytes the result will require */
+        if (ch < (UTF32)0x80) {      bytesToWrite = 1;
+        } else if (ch < (UTF32)0x800) {     bytesToWrite = 2;
+        } else if (ch < (UTF32)0x10000) {   bytesToWrite = 3;
+        } else if (ch < (UTF32)0x110000) {  bytesToWrite = 4;
+        } else {                            bytesToWrite = 3;
+                                            ch = UNI_REPLACEMENT_CHAR;
+        }
+
+        target += bytesToWrite;
+        if (target > targetEnd) {
+            source = oldSource; /* Back up source pointer! */
+            target -= bytesToWrite; result = targetExhausted; break;
+        }
+        switch (bytesToWrite) { /* note: everything falls through. */
+            case 4: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;
+            case 3: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;
+            case 2: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;
+            case 1: *--target =  (UTF8)(ch | firstByteMark[bytesToWrite]);
+        }
+        target += bytesToWrite;
+    }
+    *sourceStart = source;
+    *targetStart = target;
+    return result;
+}
+
+/* --------------------------------------------------------------------- */
+
+/*
+ * Utility routine to tell whether a sequence of bytes is legal UTF-8.
+ * This must be called with the length pre-determined by the first byte.
+ * If not calling this from ConvertUTF8to*, then the length can be set by:
+ *  length = trailingBytesForUTF8[*source]+1;
+ * and the sequence is illegal right away if there aren't that many bytes
+ * available.
+ * If presented with a length > 4, this returns false.  The Unicode
+ * definition of UTF-8 goes up to 4-byte sequences.
+ */
+
+static Boolean isLegalUTF8(const UTF8 *source, int length) {
+    UTF8 a;
+    const UTF8 *srcptr = source+length;
+    switch (length) {
+    default: return false;
+        /* Everything else falls through when "true"... */
+    case 4: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return false;
+    case 3: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return false;
+    case 2: if ((a = (*--srcptr)) > 0xBF) return false;
+
+        switch (*source) {
+            /* no fall-through in this inner switch */
+            case 0xE0: if (a < 0xA0) return false; break;
+            case 0xED: if (a > 0x9F) return false; break;
+            case 0xF0: if (a < 0x90) return false; break;
+            case 0xF4: if (a > 0x8F) return false; break;
+            default:   if (a < 0x80) return false;
+        }
+
+    case 1: if (*source >= 0x80 && *source < 0xC2) return false;
+    }
+    if (*source > 0xF4) return false;
+    return true;
+}
+
+/* --------------------------------------------------------------------- */
+
+/*
+ * Exported function to return whether a UTF-8 sequence is legal or not.
+ * This is not used here; it's just exported.
+ */
+Boolean isLegalUTF8Sequence(const UTF8 *source, const UTF8 *sourceEnd) {
+    int length = trailingBytesForUTF8[*source]+1;
+    if (source+length > sourceEnd) {
+        return false;
+    }
+    return isLegalUTF8(source, length);
+}
+
+/* --------------------------------------------------------------------- */
+
+ConversionResult ConvertUTF8toUTF16 (
+        const UTF8** sourceStart, const UTF8* sourceEnd, 
+        UTF16** targetStart, UTF16* targetEnd, ConversionFlags flags) {
+    ConversionResult result = conversionOK;
+    const UTF8* source = *sourceStart;
+    UTF16* target = *targetStart;
+    while (source < sourceEnd) {
+        UTF32 ch = 0;
+        unsigned short extraBytesToRead = trailingBytesForUTF8[*source];
+        if (source + extraBytesToRead >= sourceEnd) {
+            result = sourceExhausted; break;
+        }
+        /* Do this check whether lenient or strict */
+        if (! isLegalUTF8(source, extraBytesToRead+1)) {
+            result = sourceIllegal;
+            break;
+        }
+        /*
+         * The cases all fall through. See "Note A" below.
+         */
+        switch (extraBytesToRead) {
+            case 5: ch += *source++; ch <<= 6; /* remember, illegal UTF-8 */
+            case 4: ch += *source++; ch <<= 6; /* remember, illegal UTF-8 */
+            case 3: ch += *source++; ch <<= 6;
+            case 2: ch += *source++; ch <<= 6;
+            case 1: ch += *source++; ch <<= 6;
+            case 0: ch += *source++;
+        }
+        ch -= offsetsFromUTF8[extraBytesToRead];
+
+        if (target >= targetEnd) {
+            source -= (extraBytesToRead+1); /* Back up source pointer! */
+            result = targetExhausted; break;
+        }
+        if (ch <= UNI_MAX_BMP) { /* Target is a character <= 0xFFFF */
+            /* UTF-16 surrogate values are illegal in UTF-32 */
+            if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) {
+                if (flags == strictConversion) {
+                    source -= (extraBytesToRead+1); /* return to the illegal value itself */
+                    result = sourceIllegal;
+                    break;
+                } else {
+                    *target++ = UNI_REPLACEMENT_CHAR;
+                }
+            } else {
+                *target++ = (UTF16)ch; /* normal case */
+            }
+        } else if (ch > UNI_MAX_UTF16) {
+            if (flags == strictConversion) {
+                result = sourceIllegal;
+                source -= (extraBytesToRead+1); /* return to the start */
+                break; /* Bail out; shouldn't continue */
+            } else {
+                *target++ = UNI_REPLACEMENT_CHAR;
+            }
+        } else {
+            /* target is a character in range 0xFFFF - 0x10FFFF. */
+            if (target + 1 >= targetEnd) {
+                source -= (extraBytesToRead+1); /* Back up source pointer! */
+                result = targetExhausted; break;
+            }
+            ch -= halfBase;
+            *target++ = (UTF16)((ch >> halfShift) + UNI_SUR_HIGH_START);
+            *target++ = (UTF16)((ch & halfMask) + UNI_SUR_LOW_START);
+        }
+    }
+    *sourceStart = source;
+    *targetStart = target;
+    return result;
+}
+
+/* --------------------------------------------------------------------- */
+
+ConversionResult ConvertUTF32toUTF8 (
+        const UTF32** sourceStart, const UTF32* sourceEnd, 
+        UTF8** targetStart, UTF8* targetEnd, ConversionFlags flags) {
+    ConversionResult result = conversionOK;
+    const UTF32* source = *sourceStart;
+    UTF8* target = *targetStart;
+    while (source < sourceEnd) {
+        UTF32 ch;
+        unsigned short bytesToWrite = 0;
+        const UTF32 byteMask = 0xBF;
+        const UTF32 byteMark = 0x80; 
+        ch = *source++;
+        if (flags == strictConversion ) {
+            /* UTF-16 surrogate values are illegal in UTF-32 */
+            if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) {
+                --source; /* return to the illegal value itself */
+                result = sourceIllegal;
+                break;
+            }
+        }
+        /*
+         * Figure out how many bytes the result will require. Turn any
+         * illegally large UTF32 things (> Plane 17) into replacement chars.
+         */
+        if (ch < (UTF32)0x80) {      bytesToWrite = 1;
+        } else if (ch < (UTF32)0x800) {     bytesToWrite = 2;
+        } else if (ch < (UTF32)0x10000) {   bytesToWrite = 3;
+        } else if (ch <= UNI_MAX_LEGAL_UTF32) {  bytesToWrite = 4;
+        } else {                            bytesToWrite = 3;
+                                            ch = UNI_REPLACEMENT_CHAR;
+                                            result = sourceIllegal;
+        }
+        
+        target += bytesToWrite;
+        if (target > targetEnd) {
+            --source; /* Back up source pointer! */
+            target -= bytesToWrite; result = targetExhausted; break;
+        }
+        switch (bytesToWrite) { /* note: everything falls through. */
+            case 4: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;
+            case 3: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;
+            case 2: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;
+            case 1: *--target = (UTF8) (ch | firstByteMark[bytesToWrite]);
+        }
+        target += bytesToWrite;
+    }
+    *sourceStart = source;
+    *targetStart = target;
+    return result;
+}
+
+/* --------------------------------------------------------------------- */
+
+ConversionResult ConvertUTF8toUTF32 (
+        const UTF8** sourceStart, const UTF8* sourceEnd, 
+        UTF32** targetStart, UTF32* targetEnd, ConversionFlags flags) {
+    ConversionResult result = conversionOK;
+    const UTF8* source = *sourceStart;
+    UTF32* target = *targetStart;
+    while (source < sourceEnd) {
+        UTF32 ch = 0;
+        unsigned short extraBytesToRead = trailingBytesForUTF8[*source];
+        if (source + extraBytesToRead >= sourceEnd) {
+            result = sourceExhausted; break;
+        }
+        /* Do this check whether lenient or strict */
+        if (! isLegalUTF8(source, extraBytesToRead+1)) {
+            result = sourceIllegal;
+            break;
+        }
+        /*
+         * The cases all fall through. See "Note A" below.
+         */
+        switch (extraBytesToRead) {
+            case 5: ch += *source++; ch <<= 6;
+            case 4: ch += *source++; ch <<= 6;
+            case 3: ch += *source++; ch <<= 6;
+            case 2: ch += *source++; ch <<= 6;
+            case 1: ch += *source++; ch <<= 6;
+            case 0: ch += *source++;
+        }
+        ch -= offsetsFromUTF8[extraBytesToRead];
+
+        if (target >= targetEnd) {
+            source -= (extraBytesToRead+1); /* Back up the source pointer! */
+            result = targetExhausted; break;
+        }
+        if (ch <= UNI_MAX_LEGAL_UTF32) {
+            /*
+             * UTF-16 surrogate values are illegal in UTF-32, and anything
+             * over Plane 17 (> 0x10FFFF) is illegal.
+             */
+            if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) {
+                if (flags == strictConversion) {
+                    source -= (extraBytesToRead+1); /* return to the illegal value itself */
+                    result = sourceIllegal;
+                    break;
+                } else {
+                    *target++ = UNI_REPLACEMENT_CHAR;
+                }
+            } else {
+                *target++ = ch;
+            }
+        } else { /* i.e., ch > UNI_MAX_LEGAL_UTF32 */
+            result = sourceIllegal;
+            *target++ = UNI_REPLACEMENT_CHAR;
+        }
+    }
+    *sourceStart = source;
+    *targetStart = target;
+    return result;
+}
+
+}
+
+/* ---------------------------------------------------------------------
+
+    Note A.
+    The fall-through switches in UTF-8 reading code save a
+    temp variable, some decrements & conditionals.  The switches
+    are equivalent to the following loop:
+        {
+            int tmpBytesToRead = extraBytesToRead+1;
+            do {
+                ch += *source++;
+                --tmpBytesToRead;
+                if (tmpBytesToRead) ch <<= 6;
+            } while (tmpBytesToRead > 0);
+        }
+    In UTF-8 writing code, the switches on "bytesToWrite" are
+    similarly unrolled loops.
+
+   --------------------------------------------------------------------- */

src/linenoise/ConvertUTF.h

@@ -0,0 +1,162 @@
+/*
+ * Copyright 2001-2004 Unicode, Inc.
+ * 
+ * Disclaimer
+ * 
+ * This source code is provided as is by Unicode, Inc. No claims are
+ * made as to fitness for any particular purpose. No warranties of any
+ * kind are expressed or implied. The recipient agrees to determine
+ * applicability of information provided. If this file has been
+ * purchased on magnetic or optical media from Unicode, Inc., the
+ * sole remedy for any claim will be exchange of defective media
+ * within 90 days of receipt.
+ * 
+ * Limitations on Rights to Redistribute This Code
+ * 
+ * Unicode, Inc. hereby grants the right to freely use the information
+ * supplied in this file in the creation of products supporting the
+ * Unicode Standard, and to make copies of this file in any form
+ * for internal or external distribution as long as this notice
+ * remains attached.
+ */
+
+/* ---------------------------------------------------------------------
+
+    Conversions between UTF32, UTF-16, and UTF-8.  Header file.
+
+    Several funtions are included here, forming a complete set of
+    conversions between the three formats.  UTF-7 is not included
+    here, but is handled in a separate source file.
+
+    Each of these routines takes pointers to input buffers and output
+    buffers.  The input buffers are const.
+
+    Each routine converts the text between *sourceStart and sourceEnd,
+    putting the result into the buffer between *targetStart and
+    targetEnd. Note: the end pointers are *after* the last item: e.g. 
+    *(sourceEnd - 1) is the last item.
+
+    The return result indicates whether the conversion was successful,
+    and if not, whether the problem was in the source or target buffers.
+    (Only the first encountered problem is indicated.)
+
+    After the conversion, *sourceStart and *targetStart are both
+    updated to point to the end of last text successfully converted in
+    the respective buffers.
+
+    Input parameters:
+        sourceStart - pointer to a pointer to the source buffer.
+                The contents of this are modified on return so that
+                it points at the next thing to be converted.
+        targetStart - similarly, pointer to pointer to the target buffer.
+        sourceEnd, targetEnd - respectively pointers to the ends of the
+                two buffers, for overflow checking only.
+
+    These conversion functions take a ConversionFlags argument. When this
+    flag is set to strict, both irregular sequences and isolated surrogates
+    will cause an error.  When the flag is set to lenient, both irregular
+    sequences and isolated surrogates are converted.
+
+    Whether the flag is strict or lenient, all illegal sequences will cause
+    an error return. This includes sequences such as: <F4 90 80 80>, <C0 80>,
+    or <A0> in UTF-8, and values above 0x10FFFF in UTF-32. Conformant code
+    must check for illegal sequences.
+
+    When the flag is set to lenient, characters over 0x10FFFF are converted
+    to the replacement character; otherwise (when the flag is set to strict)
+    they constitute an error.
+
+    Output parameters:
+        The value "sourceIllegal" is returned from some routines if the input
+        sequence is malformed.  When "sourceIllegal" is returned, the source
+        value will point to the illegal value that caused the problem. E.g.,
+        in UTF-8 when a sequence is malformed, it points to the start of the
+        malformed sequence.  
+
+    Author: Mark E. Davis, 1994.
+    Rev History: Rick McGowan, fixes & updates May 2001.
+                 Fixes & updates, Sept 2001.
+
+------------------------------------------------------------------------ */
+
+/* ---------------------------------------------------------------------
+    The following 4 definitions are compiler-specific.
+    The C standard does not guarantee that wchar_t has at least
+    16 bits, so wchar_t is no less portable than unsigned short!
+    All should be unsigned values to avoid sign extension during
+    bit mask & shift operations.
+------------------------------------------------------------------------ */
+
+#if 0
+typedef unsigned long  UTF32; /* at least 32 bits */
+typedef unsigned short UTF16; /* at least 16 bits */
+typedef unsigned char  UTF8;  /* typically 8 bits */
+#endif
+
+#include <stdint.h>
+#include <string>
+
+namespace linenoise_ng {
+
+typedef uint32_t         UTF32;
+typedef uint16_t         UTF16;
+typedef uint8_t          UTF8;
+typedef unsigned char    Boolean; /* 0 or 1 */
+
+/* Some fundamental constants */
+#define UNI_REPLACEMENT_CHAR (UTF32)0x0000FFFD
+#define UNI_MAX_BMP (UTF32)0x0000FFFF
+#define UNI_MAX_UTF16 (UTF32)0x0010FFFF
+#define UNI_MAX_UTF32 (UTF32)0x7FFFFFFF
+#define UNI_MAX_LEGAL_UTF32 (UTF32)0x0010FFFF
+
+typedef enum {
+        conversionOK,           /* conversion successful */
+        sourceExhausted,        /* partial character in source, but hit end */
+        targetExhausted,        /* insuff. room in target for conversion */
+        sourceIllegal           /* source sequence is illegal/malformed */
+} ConversionResult;
+
+typedef enum {
+        strictConversion = 0,
+        lenientConversion
+} ConversionFlags;
+
+// /* This is for C++ and does no harm in C */
+// #ifdef __cplusplus
+// extern "C" {
+// #endif
+
+ConversionResult ConvertUTF8toUTF16 (
+                const UTF8** sourceStart, const UTF8* sourceEnd, 
+                UTF16** targetStart, UTF16* targetEnd, ConversionFlags flags);
+
+ConversionResult ConvertUTF16toUTF8 (
+                const UTF16** sourceStart, const UTF16* sourceEnd, 
+                UTF8** targetStart, UTF8* targetEnd, ConversionFlags flags);
+                
+ConversionResult ConvertUTF8toUTF32 (
+                const UTF8** sourceStart, const UTF8* sourceEnd, 
+                UTF32** targetStart, UTF32* targetEnd, ConversionFlags flags);
+
+ConversionResult ConvertUTF32toUTF8 (
+                const UTF32** sourceStart, const UTF32* sourceEnd, 
+                UTF8** targetStart, UTF8* targetEnd, ConversionFlags flags);
+                
+ConversionResult ConvertUTF16toUTF32 (
+                const UTF16** sourceStart, const UTF16* sourceEnd, 
+                UTF32** targetStart, UTF32* targetEnd, ConversionFlags flags);
+
+ConversionResult ConvertUTF32toUTF16 (
+                const UTF32** sourceStart, const UTF32* sourceEnd, 
+                char16_t** targetStart, char16_t* targetEnd, ConversionFlags flags);
+
+Boolean isLegalUTF8Sequence(const UTF8 *source, const UTF8 *sourceEnd);
+
+// #ifdef __cplusplus
+// }
+// #endif
+
+}
+
+/* --------------------------------------------------------------------- */

src/linenoise/LICENSE

@@ -0,0 +1,66 @@
+linenoise.cpp
+=============
+
+Copyright (c) 2010, Salvatore Sanfilippo <antirez at gmail dot com>
+Copyright (c) 2010, Pieter Noordhuis <pcnoordhuis at gmail dot com>
+
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+  * Redistributions of source code must retain the above copyright notice,
+    this list of conditions and the following disclaimer.
+  * Redistributions in binary form must reproduce the above copyright
+    notice, this list of conditions and the following disclaimer in the
+    documentation and/or other materials provided with the distribution.
+  * Neither the name of Redis nor the names of its contributors may be used
+    to endorse or promote products derived from this software without
+    specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.
+
+
+wcwidth.cpp
+===========
+
+Markus Kuhn -- 2007-05-26 (Unicode 5.0)
+
+Permission to use, copy, modify, and distribute this software
+for any purpose and without fee is hereby granted. The author
+disclaims all warranties with regard to this software.
+
+
+
+ConvertUTF.cpp
+==============
+
+Copyright 2001-2004 Unicode, Inc.
+ 
+Disclaimer
+ 
+This source code is provided as is by Unicode, Inc. No claims are
+made as to fitness for any particular purpose. No warranties of any
+kind are expressed or implied. The recipient agrees to determine
+applicability of information provided. If this file has been
+purchased on magnetic or optical media from Unicode, Inc., the
+sole remedy for any claim will be exchange of defective media
+within 90 days of receipt.
+ 
+Limitations on Rights to Redistribute This Code
+ 
+Unicode, Inc. hereby grants the right to freely use the information
+supplied in this file in the creation of products supporting the
+Unicode Standard, and to make copies of this file in any form
+for internal or external distribution as long as this notice
+remains attached.

src/linenoise/linenoise.h

@@ -0,0 +1,73 @@
+/* linenoise.h -- guerrilla line editing library against the idea that a
+ * line editing lib needs to be 20,000 lines of C code.
+ *
+ * See linenoise.c for more information.
+ *
+ * Copyright (c) 2010, Salvatore Sanfilippo <antirez at gmail dot com>
+ * Copyright (c) 2010, Pieter Noordhuis <pcnoordhuis at gmail dot com>
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *   * Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *   * Neither the name of Redis nor the names of its contributors may be used
+ *     to endorse or promote products derived from this software without
+ *     specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __LINENOISE_H
+#define __LINENOISE_H
+
+#define LINENOISE_VERSION "1.0.0"
+#define LINENOISE_VERSION_MAJOR 1
+#define LINENOISE_VERSION_MINOR 1
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct linenoiseCompletions linenoiseCompletions;
+
+typedef void(linenoiseCompletionCallback)(const char*, linenoiseCompletions*);
+void linenoiseSetCompletionCallback(linenoiseCompletionCallback* fn);
+void linenoiseAddCompletion(linenoiseCompletions* lc, const char* str);
+
+char* linenoise(const char* prompt);
+void linenoisePreloadBuffer(const char* preloadText);
+int linenoiseHistoryAdd(const char* line);
+int linenoiseHistorySetMaxLen(int len);
+char* linenoiseHistoryLine(int index);
+int linenoiseHistorySave(const char* filename);
+int linenoiseHistoryLoad(const char* filename);
+void linenoiseHistoryFree(void);
+void linenoiseClearScreen(void);
+void linenoiseSetMultiLine(int ml);
+void linenoisePrintKeyCodes(void);
+/* the following are extensions to the original linenoise API */
+int linenoiseInstallWindowChangeHandler(void);
+/* returns type of key pressed: 1 = CTRL-C, 2 = CTRL-D, 0 = other */ 
+int linenoiseKeyType(void);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* __LINENOISE_H */

src/linenoise/wcwidth.cpp

@@ -0,0 +1,315 @@
+/*
+ * This is an implementation of wcwidth() and wcswidth() (defined in
+ * IEEE Std 1002.1-2001) for Unicode.
+ *
+ * http://www.opengroup.org/onlinepubs/007904975/functions/wcwidth.html
+ * http://www.opengroup.org/onlinepubs/007904975/functions/wcswidth.html
+ *
+ * In fixed-width output devices, Latin characters all occupy a single
+ * "cell" position of equal width, whereas ideographic CJK characters
+ * occupy two such cells. Interoperability between terminal-line
+ * applications and (teletype-style) character terminals using the
+ * UTF-8 encoding requires agreement on which character should advance
+ * the cursor by how many cell positions. No established formal
+ * standards exist at present on which Unicode character shall occupy
+ * how many cell positions on character terminals. These routines are
+ * a first attempt of defining such behavior based on simple rules
+ * applied to data provided by the Unicode Consortium.
+ *
+ * For some graphical characters, the Unicode standard explicitly
+ * defines a character-cell width via the definition of the East Asian
+ * FullWidth (F), Wide (W), Half-width (H), and Narrow (Na) classes.
+ * In all these cases, there is no ambiguity about which width a
+ * terminal shall use. For characters in the East Asian Ambiguous (A)
+ * class, the width choice depends purely on a preference of backward
+ * compatibility with either historic CJK or Western practice.
+ * Choosing single-width for these characters is easy to justify as
+ * the appropriate long-term solution, as the CJK practice of
+ * displaying these characters as double-width comes from historic
+ * implementation simplicity (8-bit encoded characters were displayed
+ * single-width and 16-bit ones double-width, even for Greek,
+ * Cyrillic, etc.) and not any typographic considerations.
+ *
+ * Much less clear is the choice of width for the Not East Asian
+ * (Neutral) class. Existing practice does not dictate a width for any
+ * of these characters. It would nevertheless make sense
+ * typographically to allocate two character cells to characters such
+ * as for instance EM SPACE or VOLUME INTEGRAL, which cannot be
+ * represented adequately with a single-width glyph. The following
+ * routines at present merely assign a single-cell width to all
+ * neutral characters, in the interest of simplicity. This is not
+ * entirely satisfactory and should be reconsidered before
+ * establishing a formal standard in this area. At the moment, the
+ * decision which Not East Asian (Neutral) characters should be
+ * represented by double-width glyphs cannot yet be answered by
+ * applying a simple rule from the Unicode database content. Setting
+ * up a proper standard for the behavior of UTF-8 character terminals
+ * will require a careful analysis not only of each Unicode character,
+ * but also of each presentation form, something the author of these
+ * routines has avoided to do so far.
+ *
+ * http://www.unicode.org/unicode/reports/tr11/
+ *
+ * Markus Kuhn -- 2007-05-26 (Unicode 5.0)
+ *
+ * Permission to use, copy, modify, and distribute this software
+ * for any purpose and without fee is hereby granted. The author
+ * disclaims all warranties with regard to this software.
+ *
+ * Latest version: http://www.cl.cam.ac.uk/~mgk25/ucs/wcwidth.c
+ */
+
+#include <wchar.h>
+#include <string>
+#include <memory>
+
+namespace linenoise_ng {
+
+struct interval {
+  char32_t first;
+  char32_t last;
+};
+
+/* auxiliary function for binary search in interval table */
+static int bisearch(char32_t ucs, const struct interval *table, int max) {
+  int min = 0;
+  int mid;
+
+  if (ucs < table[0].first || ucs > table[max].last)
+    return 0;
+  while (max >= min) {
+    mid = (min + max) / 2;
+    if (ucs > table[mid].last)
+      min = mid + 1;
+    else if (ucs < table[mid].first)
+      max = mid - 1;
+    else
+      return 1;
+  }
+
+  return 0;
+}
+
+
+/* The following two functions define the column width of an ISO 10646
+ * character as follows:
+ *
+ *    - The null character (U+0000) has a column width of 0.
+ *
+ *    - Other C0/C1 control characters and DEL will lead to a return
+ *      value of -1.
+ *
+ *    - Non-spacing and enclosing combining characters (general
+ *      category code Mn or Me in the Unicode database) have a
+ *      column width of 0.
+ *
+ *    - SOFT HYPHEN (U+00AD) has a column width of 1.
+ *
+ *    - Other format characters (general category code Cf in the Unicode
+ *      database) and ZERO WIDTH SPACE (U+200B) have a column width of 0.
+ *
+ *    - Hangul Jamo medial vowels and final consonants (U+1160-U+11FF)
+ *      have a column width of 0.
+ *
+ *    - Spacing characters in the East Asian Wide (W) or East Asian
+ *      Full-width (F) category as defined in Unicode Technical
+ *      Report #11 have a column width of 2.
+ *
+ *    - All remaining characters (including all printable
+ *      ISO 8859-1 and WGL4 characters, Unicode control characters,
+ *      etc.) have a column width of 1.
+ *
+ * This implementation assumes that wchar_t characters are encoded
+ * in ISO 10646.
+ */
+
+int mk_wcwidth(char32_t ucs)
+{
+  /* sorted list of non-overlapping intervals of non-spacing characters */
+  /* generated by "uniset +cat=Me +cat=Mn +cat=Cf -00AD +1160-11FF +200B c" */
+  static const struct interval combining[] = {
+    { 0x0300, 0x036F }, { 0x0483, 0x0486 }, { 0x0488, 0x0489 },
+    { 0x0591, 0x05BD }, { 0x05BF, 0x05BF }, { 0x05C1, 0x05C2 },
+    { 0x05C4, 0x05C5 }, { 0x05C7, 0x05C7 }, { 0x0600, 0x0603 },
+    { 0x0610, 0x0615 }, { 0x064B, 0x065E }, { 0x0670, 0x0670 },
+    { 0x06D6, 0x06E4 }, { 0x06E7, 0x06E8 }, { 0x06EA, 0x06ED },
+    { 0x070F, 0x070F }, { 0x0711, 0x0711 }, { 0x0730, 0x074A },
+    { 0x07A6, 0x07B0 }, { 0x07EB, 0x07F3 }, { 0x0901, 0x0902 },
+    { 0x093C, 0x093C }, { 0x0941, 0x0948 }, { 0x094D, 0x094D },
+    { 0x0951, 0x0954 }, { 0x0962, 0x0963 }, { 0x0981, 0x0981 },
+    { 0x09BC, 0x09BC }, { 0x09C1, 0x09C4 }, { 0x09CD, 0x09CD },
+    { 0x09E2, 0x09E3 }, { 0x0A01, 0x0A02 }, { 0x0A3C, 0x0A3C },
+    { 0x0A41, 0x0A42 }, { 0x0A47, 0x0A48 }, { 0x0A4B, 0x0A4D },
+    { 0x0A70, 0x0A71 }, { 0x0A81, 0x0A82 }, { 0x0ABC, 0x0ABC },
+    { 0x0AC1, 0x0AC5 }, { 0x0AC7, 0x0AC8 }, { 0x0ACD, 0x0ACD },
+    { 0x0AE2, 0x0AE3 }, { 0x0B01, 0x0B01 }, { 0x0B3C, 0x0B3C },
+    { 0x0B3F, 0x0B3F }, { 0x0B41, 0x0B43 }, { 0x0B4D, 0x0B4D },
+    { 0x0B56, 0x0B56 }, { 0x0B82, 0x0B82 }, { 0x0BC0, 0x0BC0 },
+    { 0x0BCD, 0x0BCD }, { 0x0C3E, 0x0C40 }, { 0x0C46, 0x0C48 },
+    { 0x0C4A, 0x0C4D }, { 0x0C55, 0x0C56 }, { 0x0CBC, 0x0CBC },
+    { 0x0CBF, 0x0CBF }, { 0x0CC6, 0x0CC6 }, { 0x0CCC, 0x0CCD },
+    { 0x0CE2, 0x0CE3 }, { 0x0D41, 0x0D43 }, { 0x0D4D, 0x0D4D },
+    { 0x0DCA, 0x0DCA }, { 0x0DD2, 0x0DD4 }, { 0x0DD6, 0x0DD6 },
+    { 0x0E31, 0x0E31 }, { 0x0E34, 0x0E3A }, { 0x0E47, 0x0E4E },
+    { 0x0EB1, 0x0EB1 }, { 0x0EB4, 0x0EB9 }, { 0x0EBB, 0x0EBC },
+    { 0x0EC8, 0x0ECD }, { 0x0F18, 0x0F19 }, { 0x0F35, 0x0F35 },
+    { 0x0F37, 0x0F37 }, { 0x0F39, 0x0F39 }, { 0x0F71, 0x0F7E },
+    { 0x0F80, 0x0F84 }, { 0x0F86, 0x0F87 }, { 0x0F90, 0x0F97 },
+    { 0x0F99, 0x0FBC }, { 0x0FC6, 0x0FC6 }, { 0x102D, 0x1030 },
+    { 0x1032, 0x1032 }, { 0x1036, 0x1037 }, { 0x1039, 0x1039 },
+    { 0x1058, 0x1059 }, { 0x1160, 0x11FF }, { 0x135F, 0x135F },
+    { 0x1712, 0x1714 }, { 0x1732, 0x1734 }, { 0x1752, 0x1753 },
+    { 0x1772, 0x1773 }, { 0x17B4, 0x17B5 }, { 0x17B7, 0x17BD },
+    { 0x17C6, 0x17C6 }, { 0x17C9, 0x17D3 }, { 0x17DD, 0x17DD },
+    { 0x180B, 0x180D }, { 0x18A9, 0x18A9 }, { 0x1920, 0x1922 },
+    { 0x1927, 0x1928 }, { 0x1932, 0x1932 }, { 0x1939, 0x193B },
+    { 0x1A17, 0x1A18 }, { 0x1B00, 0x1B03 }, { 0x1B34, 0x1B34 },
+    { 0x1B36, 0x1B3A }, { 0x1B3C, 0x1B3C }, { 0x1B42, 0x1B42 },
+    { 0x1B6B, 0x1B73 }, { 0x1DC0, 0x1DCA }, { 0x1DFE, 0x1DFF },
+    { 0x200B, 0x200F }, { 0x202A, 0x202E }, { 0x2060, 0x2063 },
+    { 0x206A, 0x206F }, { 0x20D0, 0x20EF }, { 0x302A, 0x302F },
+    { 0x3099, 0x309A }, { 0xA806, 0xA806 }, { 0xA80B, 0xA80B },
+    { 0xA825, 0xA826 }, { 0xFB1E, 0xFB1E }, { 0xFE00, 0xFE0F },
+    { 0xFE20, 0xFE23 }, { 0xFEFF, 0xFEFF }, { 0xFFF9, 0xFFFB },
+    { 0x10A01, 0x10A03 }, { 0x10A05, 0x10A06 }, { 0x10A0C, 0x10A0F },
+    { 0x10A38, 0x10A3A }, { 0x10A3F, 0x10A3F }, { 0x1D167, 0x1D169 },
+    { 0x1D173, 0x1D182 }, { 0x1D185, 0x1D18B }, { 0x1D1AA, 0x1D1AD },
+    { 0x1D242, 0x1D244 }, { 0xE0001, 0xE0001 }, { 0xE0020, 0xE007F },
+    { 0xE0100, 0xE01EF }
+  };
+
+  /* test for 8-bit control characters */
+  if (ucs == 0)
+    return 0;
+  if (ucs < 32 || (ucs >= 0x7f && ucs < 0xa0))
+    return -1;
+
+  /* binary search in table of non-spacing characters */
+  if (bisearch(ucs, combining,
+               sizeof(combining) / sizeof(struct interval) - 1))
+    return 0;
+
+  /* if we arrive here, ucs is not a combining or C0/C1 control character */
+
+  return 1 + 
+    (ucs >= 0x1100 &&
+     (ucs <= 0x115f ||                    /* Hangul Jamo init. consonants */
+      ucs == 0x2329 || ucs == 0x232a ||
+      (ucs >= 0x2e80 && ucs <= 0xa4cf &&
+       ucs != 0x303f) ||                  /* CJK ... Yi */
+      (ucs >= 0xac00 && ucs <= 0xd7a3) || /* Hangul Syllables */
+      (ucs >= 0xf900 && ucs <= 0xfaff) || /* CJK Compatibility Ideographs */
+      (ucs >= 0xfe10 && ucs <= 0xfe19) || /* Vertical forms */
+      (ucs >= 0xfe30 && ucs <= 0xfe6f) || /* CJK Compatibility Forms */
+      (ucs >= 0xff00 && ucs <= 0xff60) || /* Fullwidth Forms */
+      (ucs >= 0xffe0 && ucs <= 0xffe6) ||
+      (ucs >= 0x20000 && ucs <= 0x2fffd) ||
+      (ucs >= 0x30000 && ucs <= 0x3fffd)));
+}
+
+
+int mk_wcswidth(const char32_t* pwcs, size_t n)
+{
+  int w, width = 0;
+
+  for (;*pwcs && n-- > 0; pwcs++)
+    if ((w = mk_wcwidth(*pwcs)) < 0)
+      return -1;
+    else
+      width += w;
+
+  return width;
+}
+
+
+/*
+ * The following functions are the same as mk_wcwidth() and
+ * mk_wcswidth(), except that spacing characters in the East Asian
+ * Ambiguous (A) category as defined in Unicode Technical Report #11
+ * have a column width of 2. This variant might be useful for users of
+ * CJK legacy encodings who want to migrate to UCS without changing
+ * the traditional terminal character-width behaviour. It is not
+ * otherwise recommended for general use.
+ */
+int mk_wcwidth_cjk(wchar_t ucs)
+{
+  /* sorted list of non-overlapping intervals of East Asian Ambiguous
+   * characters, generated by "uniset +WIDTH-A -cat=Me -cat=Mn -cat=Cf c" */
+  static const struct interval ambiguous[] = {
+    { 0x00A1, 0x00A1 }, { 0x00A4, 0x00A4 }, { 0x00A7, 0x00A8 },
+    { 0x00AA, 0x00AA }, { 0x00AE, 0x00AE }, { 0x00B0, 0x00B4 },
+    { 0x00B6, 0x00BA }, { 0x00BC, 0x00BF }, { 0x00C6, 0x00C6 },
+    { 0x00D0, 0x00D0 }, { 0x00D7, 0x00D8 }, { 0x00DE, 0x00E1 },
+    { 0x00E6, 0x00E6 }, { 0x00E8, 0x00EA }, { 0x00EC, 0x00ED },
+    { 0x00F0, 0x00F0 }, { 0x00F2, 0x00F3 }, { 0x00F7, 0x00FA },
+    { 0x00FC, 0x00FC }, { 0x00FE, 0x00FE }, { 0x0101, 0x0101 },
+    { 0x0111, 0x0111 }, { 0x0113, 0x0113 }, { 0x011B, 0x011B },
+    { 0x0126, 0x0127 }, { 0x012B, 0x012B }, { 0x0131, 0x0133 },
+    { 0x0138, 0x0138 }, { 0x013F, 0x0142 }, { 0x0144, 0x0144 },
+    { 0x0148, 0x014B }, { 0x014D, 0x014D }, { 0x0152, 0x0153 },
+    { 0x0166, 0x0167 }, { 0x016B, 0x016B }, { 0x01CE, 0x01CE },
+    { 0x01D0, 0x01D0 }, { 0x01D2, 0x01D2 }, { 0x01D4, 0x01D4 },
+    { 0x01D6, 0x01D6 }, { 0x01D8, 0x01D8 }, { 0x01DA, 0x01DA },
+    { 0x01DC, 0x01DC }, { 0x0251, 0x0251 }, { 0x0261, 0x0261 },
+    { 0x02C4, 0x02C4 }, { 0x02C7, 0x02C7 }, { 0x02C9, 0x02CB },
+    { 0x02CD, 0x02CD }, { 0x02D0, 0x02D0 }, { 0x02D8, 0x02DB },
+    { 0x02DD, 0x02DD }, { 0x02DF, 0x02DF }, { 0x0391, 0x03A1 },
+    { 0x03A3, 0x03A9 }, { 0x03B1, 0x03C1 }, { 0x03C3, 0x03C9 },
+    { 0x0401, 0x0401 }, { 0x0410, 0x044F }, { 0x0451, 0x0451 },
+    { 0x2010, 0x2010 }, { 0x2013, 0x2016 }, { 0x2018, 0x2019 },
+    { 0x201C, 0x201D }, { 0x2020, 0x2022 }, { 0x2024, 0x2027 },
+    { 0x2030, 0x2030 }, { 0x2032, 0x2033 }, { 0x2035, 0x2035 },
+    { 0x203B, 0x203B }, { 0x203E, 0x203E }, { 0x2074, 0x2074 },
+    { 0x207F, 0x207F }, { 0x2081, 0x2084 }, { 0x20AC, 0x20AC },
+    { 0x2103, 0x2103 }, { 0x2105, 0x2105 }, { 0x2109, 0x2109 },
+    { 0x2113, 0x2113 }, { 0x2116, 0x2116 }, { 0x2121, 0x2122 },
+    { 0x2126, 0x2126 }, { 0x212B, 0x212B }, { 0x2153, 0x2154 },
+    { 0x215B, 0x215E }, { 0x2160, 0x216B }, { 0x2170, 0x2179 },
+    { 0x2190, 0x2199 }, { 0x21B8, 0x21B9 }, { 0x21D2, 0x21D2 },
+    { 0x21D4, 0x21D4 }, { 0x21E7, 0x21E7 }, { 0x2200, 0x2200 },
+    { 0x2202, 0x2203 }, { 0x2207, 0x2208 }, { 0x220B, 0x220B },
+    { 0x220F, 0x220F }, { 0x2211, 0x2211 }, { 0x2215, 0x2215 },
+    { 0x221A, 0x221A }, { 0x221D, 0x2220 }, { 0x2223, 0x2223 },
+    { 0x2225, 0x2225 }, { 0x2227, 0x222C }, { 0x222E, 0x222E },
+    { 0x2234, 0x2237 }, { 0x223C, 0x223D }, { 0x2248, 0x2248 },
+    { 0x224C, 0x224C }, { 0x2252, 0x2252 }, { 0x2260, 0x2261 },
+    { 0x2264, 0x2267 }, { 0x226A, 0x226B }, { 0x226E, 0x226F },
+    { 0x2282, 0x2283 }, { 0x2286, 0x2287 }, { 0x2295, 0x2295 },
+    { 0x2299, 0x2299 }, { 0x22A5, 0x22A5 }, { 0x22BF, 0x22BF },
+    { 0x2312, 0x2312 }, { 0x2460, 0x24E9 }, { 0x24EB, 0x254B },
+    { 0x2550, 0x2573 }, { 0x2580, 0x258F }, { 0x2592, 0x2595 },
+    { 0x25A0, 0x25A1 }, { 0x25A3, 0x25A9 }, { 0x25B2, 0x25B3 },
+    { 0x25B6, 0x25B7 }, { 0x25BC, 0x25BD }, { 0x25C0, 0x25C1 },
+    { 0x25C6, 0x25C8 }, { 0x25CB, 0x25CB }, { 0x25CE, 0x25D1 },
+    { 0x25E2, 0x25E5 }, { 0x25EF, 0x25EF }, { 0x2605, 0x2606 },
+    { 0x2609, 0x2609 }, { 0x260E, 0x260F }, { 0x2614, 0x2615 },
+    { 0x261C, 0x261C }, { 0x261E, 0x261E }, { 0x2640, 0x2640 },
+    { 0x2642, 0x2642 }, { 0x2660, 0x2661 }, { 0x2663, 0x2665 },
+    { 0x2667, 0x266A }, { 0x266C, 0x266D }, { 0x266F, 0x266F },
+    { 0x273D, 0x273D }, { 0x2776, 0x277F }, { 0xE000, 0xF8FF },
+    { 0xFFFD, 0xFFFD }, { 0xF0000, 0xFFFFD }, { 0x100000, 0x10FFFD }
+  };
+
+  /* binary search in table of non-spacing characters */
+  if (bisearch(ucs, ambiguous,
+               sizeof(ambiguous) / sizeof(struct interval) - 1))
+    return 2;
+
+  return mk_wcwidth(ucs);
+}
+
+
+int mk_wcswidth_cjk(const wchar_t *pwcs, size_t n)
+{
+  int w, width = 0;
+
+  for (;*pwcs && n-- > 0; pwcs++)
+    if ((w = mk_wcwidth_cjk(*pwcs)) < 0)
+      return -1;
+    else
+      width += w;
+
+  return width;
+}
+
+}

src/nix-build/local.mk

@@ -0,0 +1,9 @@
+programs += nix-build
+
+nix-build_DIR := $(d)
+
+nix-build_SOURCES := $(d)/nix-build.cc
+
+nix-build_LIBS = libmain libexpr libstore libutil libformat
+
+$(eval $(call install-symlink, nix-build, $(bindir)/nix-shell))

src/nix-build/nix-build.cc

@@ -16,7 +16,6 @@
 #include "get-drvs.hh"
 #include "common-eval-args.hh"
 #include "attr-path.hh"
-#include "legacy.hh"
 
 using namespace nix;
 using namespace std::string_literals;
@@ -67,8 +66,11 @@ std::vector<string> shellwords(const string & s)
     return res;
 }
 
-static void _main(int argc, char * * argv)
+void mainWrapped(int argc, char * * argv)
 {
+    initNix();
+    initGC();
+
     auto dryRun = false;
     auto runEnv = std::regex_search(argv[0], std::regex("nix-shell$"));
     auto pure = false;
@@ -401,6 +403,8 @@ static void _main(int argc, char * * argv)
             } else
                 env[var.first] = var.second;
 
+        restoreAffinity();
+
         /* Run a shell using the derivation's environment.  For
            convenience, source $stdenv/setup to setup additional
            environment variables and shell functions.  Also don't
@@ -413,20 +417,16 @@ static void _main(int argc, char * * argv)
                 "dontAddDisableDepTrack=1; "
                 "[ -e $stdenv/setup ] && source $stdenv/setup; "
                 "%3%"
-                "PATH=\"%4%:$PATH\"; "
-                "SHELL=%5%; "
                 "set +e; "
                 R"s([ -n "$PS1" ] && PS1='\n\[\033[1;32m\][nix-shell:\w]\$\[\033[0m\] '; )s"
                 "if [ \"$(type -t runHook)\" = function ]; then runHook shellHook; fi; "
                 "unset NIX_ENFORCE_PURITY; "
                 "shopt -u nullglob; "
-                "unset TZ; %6%"
-                "%7%",
+                "unset TZ; %4%"
+                "%5%",
                 (Path) tmpDir,
                 (pure ? "" : "p=$PATH; "),
                 (pure ? "" : "PATH=$PATH:$p; unset p; "),
-                dirOf(shell),
-                shell,
                 (getenv("TZ") ? (string("export TZ='") + getenv("TZ") + "'; ") : ""),
                 envCommand));
 
@@ -444,9 +444,7 @@ static void _main(int argc, char * * argv)
 
         auto argPtrs = stringsToCharPtrs(args);
 
-        restoreAffinity();
         restoreSignals();
-        restoreMountNamespace();
 
         execvp(shell.c_str(), argPtrs.data());
 
@@ -502,5 +500,9 @@ static void _main(int argc, char * * argv)
     }
 }
 
-static RegisterLegacyCommand s1("nix-build", _main);
-static RegisterLegacyCommand s2("nix-shell", _main);
+int main(int argc, char * * argv)
+{
+    return handleExceptions(argv[0], [&]() {
+        return mainWrapped(argc, argv);
+    });
+}

src/nix-channel/local.mk

@@ -0,0 +1,7 @@
+programs += nix-channel
+
+nix-channel_DIR := $(d)
+
+nix-channel_LIBS = libmain libformat libstore libutil
+
+nix-channel_SOURCES := $(d)/nix-channel.cc

src/nix-channel/nix-channel.cc

@@ -1,11 +1,9 @@
 #include "shared.hh"
 #include "globals.hh"
 #include "download.hh"
-#include "store-api.hh"
-#include "legacy.hh"
-
 #include <fcntl.h>
 #include <regex>
+#include "store-api.hh"
 #include <pwd.h>
 
 using namespace nix;
@@ -159,9 +157,11 @@ static void update(const StringSet & channelNames)
     replaceSymlink(profile, channelLink);
 }
 
-static int _main(int argc, char ** argv)
+int main(int argc, char ** argv)
 {
-    {
+    return handleExceptions(argv[0], [&]() {
+        initNix();
+
         // Figure out the name of the `.nix-channels' file to use
         auto home = getHome();
         channelsList = home + "/.nix-channels";
@@ -169,7 +169,7 @@ static int _main(int argc, char ** argv)
 
         // Figure out the name of the channels profile.
         ;
-        auto pw = getpwuid(geteuid());
+        auto pw = getpwuid(getuid());
         std::string name = pw ? pw->pw_name : getEnv("USER", "");
         if (name.empty())
             throw Error("cannot figure out user name");
@@ -255,9 +255,5 @@ static int _main(int argc, char ** argv)
                 runProgram(settings.nixBinDir + "/nix-env", false, envArgs);
                 break;
         }
-
-        return 0;
-    }
+    });
 }
-
-static RegisterLegacyCommand s1("nix-channel", _main);

src/nix-collect-garbage/local.mk

@@ -0,0 +1,7 @@
+programs += nix-collect-garbage
+
+nix-collect-garbage_DIR := $(d)
+
+nix-collect-garbage_SOURCES := $(d)/nix-collect-garbage.cc
+
+nix-collect-garbage_LIBS = libmain libstore libutil libformat

src/nix-collect-garbage/nix-collect-garbage.cc

@@ -2,7 +2,6 @@
 #include "profiles.hh"
 #include "shared.hh"
 #include "globals.hh"
-#include "legacy.hh"
 
 #include <iostream>
 #include <cerrno>
@@ -49,10 +48,12 @@ void removeOldGenerations(std::string dir)
     }
 }
 
-static int _main(int argc, char * * argv)
+int main(int argc, char * * argv)
 {
-    {
-        bool removeOld = false;
+    bool removeOld = false;
+
+    return handleExceptions(argv[0], [&]() {
+        initNix();
 
         GCOptions options;
 
@@ -89,9 +90,5 @@ static int _main(int argc, char * * argv)
             PrintFreed freed(true, results);
             store->collectGarbage(options, results);
         }
-
-        return 0;
-    }
+    });
 }
-
-static RegisterLegacyCommand s1("nix-collect-garbage", _main);

src/nix-copy-closure/local.mk

@@ -0,0 +1,7 @@
+programs += nix-copy-closure
+
+nix-copy-closure_DIR := $(d)
+
+nix-copy-closure_LIBS = libmain libformat libstore libutil
+
+nix-copy-closure_SOURCES := $(d)/nix-copy-closure.cc

src/nix-copy-closure/nix-copy-closure.cc

@@ -1,12 +1,13 @@
 #include "shared.hh"
 #include "store-api.hh"
-#include "legacy.hh"
 
 using namespace nix;
 
-static int _main(int argc, char ** argv)
+int main(int argc, char ** argv)
 {
-    {
+    return handleExceptions(argv[0], [&]() {
+        initNix();
+
         auto gzip = false;
         auto toMode = true;
         auto includeOutputs = false;
@@ -60,9 +61,5 @@ static int _main(int argc, char ** argv)
         from->computeFSClosure(storePaths2, closure, false, includeOutputs);
 
         copyPaths(from, to, closure, NoRepair, NoCheckSigs, useSubstitutes);
-
-        return 0;
-    }
+    });
 }
-
-static RegisterLegacyCommand s1("nix-copy-closure", _main);

src/nix-daemon/local.mk

@@ -0,0 +1,13 @@
+programs += nix-daemon
+
+nix-daemon_DIR := $(d)
+
+nix-daemon_SOURCES := $(d)/nix-daemon.cc
+
+nix-daemon_LIBS = libmain libstore libutil libformat
+
+nix-daemon_LDFLAGS = -pthread
+
+ifeq ($(OS), SunOS)
+        nix-daemon_LDFLAGS += -lsocket
+endif

src/nix-daemon/nix-daemon.cc

@@ -9,7 +9,6 @@
 #include "monitor-fd.hh"
 #include "derivations.hh"
 #include "finally.hh"
-#include "legacy.hh"
 
 #include <algorithm>
 
@@ -558,8 +557,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
                     ;
                 else if (trusted
                     || name == settings.buildTimeout.name
-                    || name == "connect-timeout"
-                    || (name == "builders" && value == ""))
+                    || name == "connect-timeout")
                     settings.set(name, value);
                 else if (setSubstituters(settings.substituters))
                     ;
@@ -710,7 +708,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         logger->startWork();
 
         // FIXME: race if addToStore doesn't read source?
-        store->addToStore(info, *source, (RepairFlag) repair,
+        store.cast<Store>()->addToStore(info, *source, (RepairFlag) repair,
             dontCheckSigs ? NoCheckSigs : CheckSigs, nullptr);
 
         logger->stopWork();
@@ -1059,9 +1057,11 @@ static void daemonLoop(char * * argv)
 }
 
 
-static int _main(int argc, char * * argv)
+int main(int argc, char * * argv)
 {
-    {
+    return handleExceptions(argv[0], [&]() {
+        initNix();
+
         auto stdio = false;
 
         parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) {
@@ -1121,7 +1121,7 @@ static int _main(int argc, char * * argv)
                         if (res == -1)
                             throw SysError("splicing data from stdin to daemon socket");
                         else if (res == 0)
-                            return 0;
+                            return;
                     }
                 }
             } else {
@@ -1130,9 +1130,5 @@ static int _main(int argc, char * * argv)
         } else {
             daemonLoop(argv);
         }
-
-        return 0;
-    }
+    });
 }
-
-static RegisterLegacyCommand s1("nix-daemon", _main);

src/nix-env/local.mk

@@ -0,0 +1,7 @@
+programs += nix-env
+
+nix-env_DIR := $(d)
+
+nix-env_SOURCES := $(wildcard $(d)/*.cc)
+
+nix-env_LIBS = libexpr libmain libstore libutil libformat

src/nix-env/nix-env.cc

@@ -13,7 +13,6 @@
 #include "json.hh"
 #include "value-to-json.hh"
 #include "xml-writer.hh"
-#include "legacy.hh"
 
 #include <cerrno>
 #include <ctime>
@@ -151,8 +150,10 @@ static void loadSourceExpr(EvalState & state, const Path & path, Value & v)
     if (stat(path.c_str(), &st) == -1)
         throw SysError(format("getting information about '%1%'") % path);
 
-    if (isNixExpr(path, st))
+    if (isNixExpr(path, st)) {
         state.evalFile(path, v);
+        return;
+    }
 
     /* The path is a directory.  Put the Nix expressions in the
        directory in a set, with the file name of each expression as
@@ -160,15 +161,13 @@ static void loadSourceExpr(EvalState & state, const Path & path, Value & v)
        set flat, not nested, to make it easier for a user to have a
        ~/.nix-defexpr directory that includes some system-wide
        directory). */
-    else if (S_ISDIR(st.st_mode)) {
+    if (S_ISDIR(st.st_mode)) {
         state.mkAttrs(v, 1024);
         state.mkList(*state.allocAttr(v, state.symbols.create("_combineChannels")), 0);
         StringSet attrs;
         getAllExprs(state, path, attrs, v);
         v.attrs->sort();
     }
-
-    else throw Error("path '%s' is not a directory or a Nix expression", path);
 }
 
 
@@ -1312,9 +1311,12 @@ static void opVersion(Globals & globals, Strings opFlags, Strings opArgs)
 }
 
 
-static int _main(int argc, char * * argv)
+int main(int argc, char * * argv)
 {
-    {
+    return handleExceptions(argv[0], [&]() {
+        initNix();
+        initGC();
+
         Strings opFlags, opArgs;
         Operation op = 0;
         RepairFlag repair = NoRepair;
@@ -1426,9 +1428,5 @@ static int _main(int argc, char * * argv)
         op(globals, opFlags, opArgs);
 
         globals.state->printStats();
-
-        return 0;
-    }
+    });
 }
-
-static RegisterLegacyCommand s1("nix-env", _main);

src/nix-instantiate/local.mk

@@ -0,0 +1,7 @@
+programs += nix-instantiate
+
+nix-instantiate_DIR := $(d)
+
+nix-instantiate_SOURCES := $(d)/nix-instantiate.cc
+
+nix-instantiate_LIBS = libexpr libmain libstore libutil libformat

src/nix-instantiate/nix-instantiate.cc

@@ -9,7 +9,6 @@
 #include "util.hh"
 #include "store-api.hh"
 #include "common-eval-args.hh"
-#include "legacy.hh"
 
 #include <map>
 #include <iostream>
@@ -84,9 +83,12 @@ void processExpr(EvalState & state, const Strings & attrPaths,
 }
 
 
-static int _main(int argc, char * * argv)
+int main(int argc, char * * argv)
 {
-    {
+    return handleExceptions(argv[0], [&]() {
+        initNix();
+        initGC();
+
         Strings files;
         bool readStdin = false;
         bool fromArgs = false;
@@ -169,7 +171,7 @@ static int _main(int argc, char * * argv)
                 if (p == "") throw Error(format("unable to find '%1%'") % i);
                 std::cout << p << std::endl;
             }
-            return 0;
+            return;
         }
 
         if (readStdin) {
@@ -188,9 +190,5 @@ static int _main(int argc, char * * argv)
         }
 
         state->printStats();
-
-        return 0;
-    }
+    });
 }
-
-static RegisterLegacyCommand s1("nix-instantiate", _main);

src/nix-prefetch-url/local.mk

@@ -0,0 +1,7 @@
+programs += nix-prefetch-url
+
+nix-prefetch-url_DIR := $(d)
+
+nix-prefetch-url_SOURCES := $(d)/nix-prefetch-url.cc
+
+nix-prefetch-url_LIBS = libmain libexpr libstore libutil libformat

src/nix-prefetch-url/nix-prefetch-url.cc

@@ -6,9 +6,6 @@
 #include "eval-inline.hh"
 #include "common-eval-args.hh"
 #include "attr-path.hh"
-#include "legacy.hh"
-#include "finally.hh"
-#include "progress-bar.hh"
 
 #include <iostream>
 
@@ -47,9 +44,12 @@ string resolveMirrorUri(EvalState & state, string uri)
 }
 
 
-static int _main(int argc, char * * argv)
+int main(int argc, char * * argv)
 {
-    {
+    return handleExceptions(argv[0], [&]() {
+        initNix();
+        initGC();
+
         HashType ht = htSHA256;
         std::vector<string> args;
         bool printPath = getEnv("PRINT_PATH") != "";
@@ -98,11 +98,6 @@ static int _main(int argc, char * * argv)
         if (args.size() > 2)
             throw UsageError("too many arguments");
 
-        Finally f([]() { stopProgressBar(); });
-
-        if (isatty(STDERR_FILENO))
-          startProgressBar();
-
         auto store = openStore();
         auto state = std::make_unique<EvalState>(myArgs.searchPath, store);
 
@@ -220,17 +215,11 @@ static int _main(int argc, char * * argv)
             assert(storePath == store->makeFixedOutputPath(unpack, hash, name));
         }
 
-        stopProgressBar();
-
         if (!printPath)
             printInfo(format("path is '%1%'") % storePath);
 
         std::cout << printHash16or32(hash) << std::endl;
         if (printPath)
             std::cout << storePath << std::endl;
-
-        return 0;
-    }
+    });
 }
-
-static RegisterLegacyCommand s1("nix-prefetch-url", _main);

src/nix-store/graphml.cc

@@ -1,90 +0,0 @@
-#include "graphml.hh"
-#include "util.hh"
-#include "store-api.hh"
-#include "derivations.hh"
-
-#include <iostream>
-
-
-using std::cout;
-
-namespace nix {
-
-
-static inline const string & xmlQuote(const string & s)
-{
-    // Luckily, store paths shouldn't contain any character that needs to be
-    // quoted.
-    return s;
-}
-
-
-static string symbolicName(const string & path)
-{
-    string p = baseNameOf(path);
-    return string(p, p.find('-') + 1);
-}
-
-
-static string makeEdge(const string & src, const string & dst)
-{
-    return fmt("  <edge source=\"%1%\" target=\"%2%\"/>\n",
-        xmlQuote(src), xmlQuote(dst));
-}
-
-
-static string makeNode(const ValidPathInfo & info)
-{
-    return fmt(
-        "  <node id=\"%1%\">\n"
-        "    <data key=\"narSize\">%2%</data>\n"
-        "    <data key=\"name\">%3%</data>\n"
-        "    <data key=\"type\">%4%</data>\n"
-        "  </node>\n",
-        info.path,
-        info.narSize,
-        symbolicName(info.path),
-        (isDerivation(info.path) ? "derivation" : "output-path"));
-}
-
-
-void printGraphML(ref<Store> store, const PathSet & roots)
-{
-    PathSet workList(roots);
-    PathSet doneSet;
-    std::pair<PathSet::iterator,bool> ret;
-
-    cout << "<?xml version='1.0' encoding='utf-8'?>\n"
-         << "<graphml xmlns='http://graphml.graphdrawing.org/xmlns'\n"
-         << "    xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance'\n"
-         << "    xsi:schemaLocation='http://graphml.graphdrawing.org/xmlns/1.0/graphml.xsd'>\n"
-         << "<key id='narSize' for='node' attr.name='narSize' attr.type='int'/>"
-         << "<key id='name' for='node' attr.name='name' attr.type='string'/>"
-         << "<key id='type' for='node' attr.name='type' attr.type='string'/>"
-         << "<graph id='G' edgedefault='directed'>\n";
-
-    while (!workList.empty()) {
-        Path path = *(workList.begin());
-        workList.erase(path);
-
-        ret = doneSet.insert(path);
-        if (ret.second == false) continue;
-
-        ValidPathInfo info = *(store->queryPathInfo(path));
-        cout << makeNode(info);
-
-        for (auto & p : store->queryPathInfo(path)->references) {
-            if (p != path) {
-                workList.insert(p);
-                cout << makeEdge(path, p);
-            }
-        }
-
-    }
-
-    cout << "</graph>\n";
-    cout << "</graphml>\n";
-}
-
-
-}

src/nix-store/local.mk

@@ -0,0 +1,9 @@
+programs += nix-store
+
+nix-store_DIR := $(d)
+
+nix-store_SOURCES := $(wildcard $(d)/*.cc)
+
+nix-store_LIBS = libmain libstore libutil libformat
+
+nix-store_LDFLAGS = -lbz2 -pthread $(SODIUM_LIBS)

src/nix-store/nix-store.cc

@@ -8,8 +8,7 @@
 #include "shared.hh"
 #include "util.hh"
 #include "worker-protocol.hh"
-#include "graphml.hh"
-#include "legacy.hh"
+#include "xmlgraph.hh"
 
 #include <iostream>
 #include <algorithm>
@@ -274,7 +273,7 @@ static void opQuery(Strings opFlags, Strings opArgs)
     enum QueryType
         { qDefault, qOutputs, qRequisites, qReferences, qReferrers
         , qReferrersClosure, qDeriver, qBinding, qHash, qSize
-        , qTree, qGraph, qGraphML, qResolve, qRoots };
+        , qTree, qGraph, qXml, qResolve, qRoots };
     QueryType query = qDefault;
     bool useOutput = false;
     bool includeOutputs = false;
@@ -300,7 +299,7 @@ static void opQuery(Strings opFlags, Strings opArgs)
         else if (i == "--size") query = qSize;
         else if (i == "--tree") query = qTree;
         else if (i == "--graph") query = qGraph;
-        else if (i == "--graphml") query = qGraphML;
+        else if (i == "--xml") query = qXml;
         else if (i == "--resolve") query = qResolve;
         else if (i == "--roots") query = qRoots;
         else if (i == "--use-output" || i == "-u") useOutput = true;
@@ -404,13 +403,13 @@ static void opQuery(Strings opFlags, Strings opArgs)
             break;
         }
 
-        case qGraphML: {
+        case qXml: {
             PathSet roots;
             for (auto & i : opArgs) {
                 PathSet paths = maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise);
                 roots.insert(paths.begin(), paths.end());
             }
-            printGraphML(ref<Store>(store), roots);
+            printXmlGraph(ref<Store>(store), roots);
             break;
         }
 
@@ -994,9 +993,11 @@ static void opVersion(Strings opFlags, Strings opArgs)
 /* Scan the arguments; find the operation, set global flags, put all
    other flags in a list, and put all other arguments in another
    list. */
-static int _main(int argc, char * * argv)
+int main(int argc, char * * argv)
 {
-    {
+    return handleExceptions(argv[0], [&]() {
+        initNix();
+
         Strings opFlags, opArgs;
         Operation op = 0;
 
@@ -1083,9 +1084,5 @@ static int _main(int argc, char * * argv)
             store = openStore();
 
         op(opFlags, opArgs);
-
-        return 0;
-    }
+    });
 }
-
-static RegisterLegacyCommand s1("nix-store", _main);

src/nix-store/xmlgraph.cc

@@ -0,0 +1,66 @@
+#include "xmlgraph.hh"
+#include "util.hh"
+#include "store-api.hh"
+
+#include <iostream>
+
+
+using std::cout;
+
+namespace nix {
+
+
+static inline const string & xmlQuote(const string & s)
+{
+    // Luckily, store paths shouldn't contain any character that needs to be
+    // quoted.
+    return s;
+}
+
+
+static string makeEdge(const string & src, const string & dst)
+{
+    format f = format("  <edge src=\"%1%\" dst=\"%2%\"/>\n")
+      % xmlQuote(src) % xmlQuote(dst);
+    return f.str();
+}
+
+
+static string makeNode(const string & id)
+{
+    format f = format("  <node name=\"%1%\"/>\n") % xmlQuote(id);
+    return f.str();
+}
+
+
+void printXmlGraph(ref<Store> store, const PathSet & roots)
+{
+    PathSet workList(roots);
+    PathSet doneSet;
+
+    cout << "<?xml version='1.0' encoding='utf-8'?>\n"
+         << "<nix>\n";
+
+    while (!workList.empty()) {
+        Path path = *(workList.begin());
+        workList.erase(path);
+
+        if (doneSet.find(path) != doneSet.end()) continue;
+        doneSet.insert(path);
+
+        cout << makeNode(path);
+
+        for (auto & p : store->queryPathInfo(path)->references) {
+            if (p != path) {
+                workList.insert(p);
+                cout << makeEdge(p, path);
+            }
+        }
+
+    }
+
+    cout << "</nix>\n";
+}
+
+
+}

src/nix-store/xmlgraph.hh

@@ -6,6 +6,6 @@ namespace nix {
 
 class Store;
 
-void printGraphML(ref<Store> store, const PathSet & roots);
+void printXmlGraph(ref<Store> store, const PathSet & roots);
 
 }

src/nix/copy.cc

@@ -69,12 +69,12 @@ struct CmdCopy : StorePathsCommand
             },
 #ifdef ENABLE_S3
             Example{
-                "To copy Hello to an S3 binary cache:",
-                "nix copy --to s3://my-bucket?region=eu-west-1 nixpkgs.hello"
+                "To populate the current folder build output to a S3 binary cache:",
+                "nix copy --to s3://my-bucket?region=eu-west-1"
             },
             Example{
-                "To copy Hello to an S3-compatible binary cache:",
-                "nix copy --to s3://my-bucket?region=eu-west-1&endpoint=example.com nixpkgs.hello"
+                "To populate the current folder build output to an S3-compatible binary cache:",
+                "nix copy --to s3://my-bucket?region=eu-west-1&endpoint=example.com"
             },
 #endif
         };

src/nix/doctor.cc

@@ -1,124 +0,0 @@
-#include "command.hh"
-#include "serve-protocol.hh"
-#include "shared.hh"
-#include "store-api.hh"
-#include "worker-protocol.hh"
-
-using namespace nix;
-
-std::string formatProtocol(unsigned int proto)
-{
-    if (proto) {
-        auto major = GET_PROTOCOL_MAJOR(proto) >> 8;
-        auto minor = GET_PROTOCOL_MINOR(proto);
-        return (format("%1%.%2%") % major % minor).str();
-    }
-    return "unknown";
-}
-
-struct CmdDoctor : StoreCommand
-{
-    bool success = true;
-
-    std::string name() override
-    {
-        return "doctor";
-    }
-
-    std::string description() override
-    {
-        return "check your system for potential problems";
-    }
-
-    void run(ref<Store> store) override
-    {
-        std::cout << "Store uri: " << store->getUri() << std::endl;
-        std::cout << std::endl;
-
-        auto type = getStoreType();
-
-        if (type < tOther) {
-            success &= checkNixInPath();
-            success &= checkProfileRoots(store);
-        }
-        success &= checkStoreProtocol(store->getProtocol());
-
-        if (!success)
-            throw Exit(2);
-    }
-
-    bool checkNixInPath()
-    {
-        PathSet dirs;
-
-        for (auto & dir : tokenizeString<Strings>(getEnv("PATH"), ":"))
-            if (pathExists(dir + "/nix-env"))
-                dirs.insert(dirOf(canonPath(dir + "/nix-env", true)));
-
-        if (dirs.size() != 1) {
-            std::cout << "Warning: multiple versions of nix found in PATH." << std::endl;
-            std::cout << std::endl;
-            for (auto & dir : dirs)
-                std::cout << "  " << dir << std::endl;
-            std::cout << std::endl;
-            return false;
-        }
-
-        return true;
-    }
-
-    bool checkProfileRoots(ref<Store> store)
-    {
-        PathSet dirs;
-
-        for (auto & dir : tokenizeString<Strings>(getEnv("PATH"), ":")) {
-            Path profileDir = dirOf(dir);
-            try {
-                Path userEnv = canonPath(profileDir, true);
-
-                if (store->isStorePath(userEnv) && hasSuffix(userEnv, "user-environment")) {
-                    while (profileDir.find("/profiles/") == std::string::npos && isLink(profileDir))
-                        profileDir = absPath(readLink(profileDir), dirOf(profileDir));
-
-                    if (profileDir.find("/profiles/") == std::string::npos)
-                        dirs.insert(dir);
-                }
-            } catch (SysError &) {}
-        }
-
-        if (!dirs.empty()) {
-            std::cout << "Warning: found profiles outside of " << settings.nixStateDir << "/profiles." << std::endl;
-            std::cout << "The generation this profile points to might not have a gcroot and could be" << std::endl;
-            std::cout << "garbage collected, resulting in broken symlinks." << std::endl;
-            std::cout << std::endl;
-            for (auto & dir : dirs)
-                std::cout << "  " << dir << std::endl;
-            std::cout << std::endl;
-            return false;
-        }
-
-        return true;
-    }
-
-    bool checkStoreProtocol(unsigned int storeProto)
-    {
-        unsigned int clientProto = GET_PROTOCOL_MAJOR(SERVE_PROTOCOL_VERSION) == GET_PROTOCOL_MAJOR(storeProto)
-            ? SERVE_PROTOCOL_VERSION
-            : PROTOCOL_VERSION;
-
-        if (clientProto != storeProto) {
-            std::cout << "Warning: protocol version of this client does not match the store." << std::endl;
-            std::cout << "While this is not necessarily a problem it's recommended to keep the client in" << std::endl;
-            std::cout << "sync with the daemon." << std::endl;
-            std::cout << std::endl;
-            std::cout << "Client protocol: " << formatProtocol(clientProto) << std::endl;
-            std::cout << "Store protocol: " << formatProtocol(storeProto) << std::endl;
-            std::cout << std::endl;
-            return false;
-        }
-
-        return true;
-    }
-};
-
-static RegisterCommand r1(make_ref<CmdDoctor>());

src/nix/edit.cc

@@ -3,7 +3,6 @@
 #include "eval.hh"
 #include "attr-path.hh"
 #include "progress-bar.hh"
-#include "affinity.hh"
 
 #include <unistd.h>
 
@@ -73,10 +72,6 @@ struct CmdEdit : InstallableCommand
 
         stopProgressBar();
 
-        restoreAffinity();
-        restoreSignals();
-        restoreMountNamespace();
-
         execvp(args.front().c_str(), stringsToCharPtrs(args).data());
 
         throw SysError("cannot run editor '%s'", editor);

src/nix/hash.cc

@@ -9,14 +9,13 @@ struct CmdHash : Command
 {
     enum Mode { mFile, mPath };
     Mode mode;
-    Base base = SRI;
+    Base base = Base16;
     bool truncate = false;
-    HashType ht = htSHA256;
+    HashType ht = htSHA512;
     std::vector<std::string> paths;
 
     CmdHash(Mode mode) : mode(mode)
     {
-        mkFlag(0, "sri", "print hash in SRI format", &base, SRI);
         mkFlag(0, "base64", "print hash in base-64", &base, Base64);
         mkFlag(0, "base32", "print hash in base-32 (Nix-specific)", &base, Base32);
         mkFlag(0, "base16", "print hash in base-16", &base, Base16);
@@ -44,7 +43,7 @@ struct CmdHash : Command
             Hash h = mode == mFile ? hashFile(ht, path) : hashPath(ht, path).first;
             if (truncate && h.hashSize > 20) h = compressHash(h, 20);
             std::cout << format("%1%\n") %
-                h.to_string(base, base == SRI);
+                h.to_string(base, false);
         }
     }
 };
@@ -55,7 +54,7 @@ static RegisterCommand r2(make_ref<CmdHash>(CmdHash::mPath));
 struct CmdToBase : Command
 {
     Base base;
-    HashType ht = htUnknown;
+    HashType ht = htSHA512;
     std::vector<std::string> args;
 
     CmdToBase(Base base) : base(base)
@@ -71,30 +70,26 @@ struct CmdToBase : Command
         return
             base == Base16 ? "to-base16" :
             base == Base32 ? "to-base32" :
-            base == Base64 ? "to-base64" :
-            "to-sri";
+            "to-base64";
     }
 
     std::string description() override
     {
-        return fmt("convert a hash to %s representation",
-            base == Base16 ? "base-16" :
-            base == Base32 ? "base-32" :
-            base == Base64 ? "base-64" :
-            "SRI");
+        return fmt("convert a hash to base-%d representation",
+            base == Base16 ? 16 :
+            base == Base32 ? 32 : 64);
     }
 
     void run() override
     {
         for (auto s : args)
-            std::cout << fmt("%s\n", Hash(s, ht).to_string(base, base == SRI));
+            std::cout << fmt("%s\n", Hash(s, ht).to_string(base, false));
     }
 };
 
 static RegisterCommand r3(make_ref<CmdToBase>(Base16));
 static RegisterCommand r4(make_ref<CmdToBase>(Base32));
 static RegisterCommand r5(make_ref<CmdToBase>(Base64));
-static RegisterCommand r6(make_ref<CmdToBase>(SRI));
 
 /* Legacy nix-hash command. */
 static int compatNixHash(int argc, char * * argv)

src/nix/local.mk

@@ -2,24 +2,10 @@ programs += nix
 
 nix_DIR := $(d)
 
-nix_SOURCES := \
-  $(wildcard $(d)/*.cc) \
-  $(wildcard src/build-remote/*.cc) \
-  $(wildcard src/nix-build/*.cc) \
-  $(wildcard src/nix-channel/*.cc) \
-  $(wildcard src/nix-collect-garbage/*.cc) \
-  $(wildcard src/nix-copy-closure/*.cc) \
-  $(wildcard src/nix-daemon/*.cc) \
-  $(wildcard src/nix-env/*.cc) \
-  $(wildcard src/nix-instantiate/*.cc) \
-  $(wildcard src/nix-prefetch-url/*.cc) \
-  $(wildcard src/nix-store/*.cc) \
+nix_SOURCES := $(wildcard $(d)/*.cc) $(wildcard src/linenoise/*.cpp)
 
-nix_LIBS = libexpr libmain libstore libutil
+nix_LIBS = libexpr libmain libstore libutil libformat
 
-nix_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS)
+nix_LDFLAGS = -pthread
 
-$(foreach name, \
-  nix-build nix-channel nix-collect-garbage nix-copy-closure nix-daemon nix-env nix-hash nix-instantiate nix-prefetch-url nix-shell nix-store, \
-  $(eval $(call install-symlink, nix, $(bindir)/$(name))))
-$(eval $(call install-symlink, $(bindir)/nix, $(libexecdir)/nix/build-remote))
+$(eval $(call install-symlink, nix, $(bindir)/nix-hash))