Merge branch 'data-dir-non-canon' of https://github.com/shlevy/nix

This reverts commit ddc58e7896.

https://hydra.nixos.org/eval/1435322

.github/ISSUE_TEMPLATE.md

@@ -14,7 +14,7 @@ Examples of _Nix_ issues:
 
 - Nix segfaults when I run `nix-build -A blahblah`
 - The Nix language needs a new builtin: `builtins.foobar`
-- Regression in the behavior of `nix-env` in Nix 1.12
+- Regression in the behavior of `nix-env` in Nix 2.0
 
 Examples of _nixpkgs_ issues:
 
@@ -24,4 +24,4 @@ Examples of _nixpkgs_ issues:
 
 Chances are if you're a newcomer to the Nix world, you'll probably want the [nixpkgs tracker](https://github.com/NixOS/nixpkgs/issues). It also gets a lot more eyeball traffic so you'll probably get a response a lot more quickly.
 
--->
+-->

.gitignore

@@ -38,6 +38,7 @@ perl/Makefile.config
 /scripts/nix-copy-closure
 /scripts/nix-reduce-build
 /scripts/nix-http-export.cgi
+/scripts/nix-profile-daemon.sh
 
 # /src/libexpr/
 /src/libexpr/lexer-tab.cc

Makefile

@@ -24,7 +24,8 @@ makefiles = \
   misc/launchd/local.mk \
   misc/upstart/local.mk \
   doc/manual/local.mk \
-  tests/local.mk
+  tests/local.mk \
+  tests/plugins/local.mk
 
 GLOBAL_CXXFLAGS += -std=c++14 -g -Wall -include config.h
 

Makefile.config.in

@@ -7,6 +7,7 @@ ENABLE_S3 = @ENABLE_S3@
 HAVE_SODIUM = @HAVE_SODIUM@
 HAVE_READLINE = @HAVE_READLINE@
 HAVE_BROTLI = @HAVE_BROTLI@
+HAVE_SECCOMP = @HAVE_SECCOMP@
 LIBCURL_LIBS = @LIBCURL_LIBS@
 OPENSSL_LIBS = @OPENSSL_LIBS@
 PACKAGE_NAME = @PACKAGE_NAME@

configure.ac

@@ -61,6 +61,7 @@ CFLAGS=
 CXXFLAGS=
 AC_PROG_CC
 AC_PROG_CXX
+AC_PROG_CPP
 AX_CXX_COMPILE_STDCXX_11
 
 
@@ -174,6 +175,8 @@ AC_SUBST(HAVE_SODIUM, [$have_sodium])
 
 # Look for liblzma, a required dependency.
 PKG_CHECK_MODULES([LIBLZMA], [liblzma], [CXXFLAGS="$LIBLZMA_CFLAGS $CXXFLAGS"])
+AC_CHECK_LIB([lzma], [lzma_stream_encoder_mt],
+  [AC_DEFINE([HAVE_LZMA_MT], [1], [xz multithreaded compression support])])
 
 
 # Look for libbrotli{enc,dec}, optional dependencies
@@ -185,9 +188,22 @@ AC_SUBST(HAVE_BROTLI, [$have_brotli])
 
 # Look for libseccomp, required for Linux sandboxing.
 if test "$sys_name" = linux; then
-  PKG_CHECK_MODULES([LIBSECCOMP], [libseccomp],
-                    [CXXFLAGS="$LIBSECCOMP_CFLAGS $CXXFLAGS"])
+  AC_ARG_ENABLE([seccomp-sandboxing],
+                AC_HELP_STRING([--disable-seccomp-sandboxing],
+                               [Don't build support for seccomp sandboxing (only recommended if your arch doesn't support libseccomp yet!)]
+                              ))
+  if test "x$enable_seccomp_sandboxing" != "xno"; then
+    PKG_CHECK_MODULES([LIBSECCOMP], [libseccomp],
+                      [CXXFLAGS="$LIBSECCOMP_CFLAGS $CXXFLAGS"])
+    have_seccomp=1
+    AC_DEFINE([HAVE_SECCOMP], [1], [Whether seccomp is available and should be used for sandboxing.])
+  else
+    have_seccomp=
+  fi
+else
+  have_seccomp=
 fi
+AC_SUBST(HAVE_SECCOMP, [$have_seccomp])
 
 
 # Look for aws-cpp-sdk-s3.
@@ -199,7 +215,7 @@ AC_SUBST(ENABLE_S3, [$enable_s3])
 AC_LANG_POP(C++)
 
 if test -n "$enable_s3"; then
-  declare -a aws_version_tokens=($(printf '#include <aws/core/VersionConfig.h>\nAWS_SDK_VERSION_STRING' | cpp -E | grep -v '^#.*' | sed 's/"//g' | tr '.' ' '))
+  declare -a aws_version_tokens=($(printf '#include <aws/core/VersionConfig.h>\nAWS_SDK_VERSION_STRING' | $CPP - | grep -v '^#.*' | sed 's/"//g' | tr '.' ' '))
   AC_DEFINE_UNQUOTED([AWS_VERSION_MAJOR], ${aws_version_tokens@<:@0@:>@}, [Major version of aws-sdk-cpp.])
   AC_DEFINE_UNQUOTED([AWS_VERSION_MINOR], ${aws_version_tokens@<:@1@:>@}, [Minor version of aws-sdk-cpp.])
 fi

corepkgs/fetchurl.nix

@@ -1,4 +1,4 @@
-{ system ? builtins.currentSystem
+{ system ? "" # obsolete
 , url
 , md5 ? "", sha1 ? "", sha256 ? "", sha512 ? ""
 , outputHash ?
@@ -17,7 +17,9 @@ derivation {
   inherit outputHashAlgo outputHash;
   outputHashMode = if unpack || executable then "recursive" else "flat";
 
-  inherit name system url executable unpack;
+  inherit name url executable unpack;
+
+  system = "builtin";
 
   # No need to double the amount of network traffic
   preferLocalBuild = true;

doc/manual/advanced-topics/distributed-builds.xml

@@ -4,71 +4,109 @@
       version="5.0"
       xml:id='chap-distributed-builds'>
 
-<title>Distributed Builds</title>
+<title>Remote Builds</title>
 
-<para>Nix supports distributed builds, where a local Nix installation can
-forward Nix builds to other machines over the network.  This allows
-multiple builds to be performed in parallel (thus improving
-performance) and allows Nix to perform multi-platform builds in a
-semi-transparent way.  For instance, if you perform a build for a
-<literal>x86_64-darwin</literal> on an <literal>i686-linux</literal>
-machine, Nix can automatically forward the build to a
-<literal>x86_64-darwin</literal> machine, if available.</para>
+<para>Nix supports remote builds, where a local Nix installation can
+forward Nix builds to other machines.  This allows multiple builds to
+be performed in parallel and allows Nix to perform multi-platform
+builds in a semi-transparent way.  For instance, if you perform a
+build for a <literal>x86_64-darwin</literal> on an
+<literal>i686-linux</literal> machine, Nix can automatically forward
+the build to a <literal>x86_64-darwin</literal> machine, if
+available.</para>
 
-<para>You can enable distributed builds by setting the environment
-variable <envar>NIX_BUILD_HOOK</envar> to point to a program that Nix
-will call whenever it wants to build a derivation.  The build hook
-(typically a shell or Perl script) can decline the build, in which Nix
-will perform it in the usual way if possible, or it can accept it, in
-which case it is responsible for somehow getting the inputs of the
-build to another machine, doing the build there, and getting the
-results back.</para>
+<para>To forward a build to a remote machine, it’s required that the
+remote machine is accessible via SSH and that it has Nix
+installed. You can test whether connecting to the remote Nix instance
+works, e.g.
 
-<example xml:id='ex-remote-systems'><title>Remote machine configuration:
-<filename>remote-systems.conf</filename></title>
-<programlisting>
-nix@mcflurry.labs.cs.uu.nl  x86_64-darwin   /home/nix/.ssh/id_quarterpounder_auto  2
-nix@scratchy.labs.cs.uu.nl  i686-linux      /home/nix/.ssh/id_scratchy_auto        8 1 kvm
-nix@itchy.labs.cs.uu.nl     i686-linux      /home/nix/.ssh/id_scratchy_auto        8 2
-nix@poochie.labs.cs.uu.nl   i686-linux      /home/nix/.ssh/id_scratchy_auto        8 2 kvm perf
-</programlisting>
-</example>
+<screen>
+$ nix ping-store --store ssh://mac
+</screen>
 
-<para>Nix ships with a build hook that should be suitable for most
-purposes.  It uses <command>ssh</command> and
-<command>nix-copy-closure</command> to copy the build inputs and
-outputs and perform the remote build.  To use it, you should set
-<envar>NIX_BUILD_HOOK</envar> to
-<filename><replaceable>prefix</replaceable>/libexec/nix/build-remote</filename>.
-You should also define a list of available build machines and point
-the environment variable <envar>NIX_REMOTE_SYSTEMS</envar> to
-it. <envar>NIX_REMOTE_SYSTEMS</envar> must be an absolute path. An
-example configuration is shown in <xref linkend='ex-remote-systems'
-/>.  Each line in the file specifies a machine, with the following
-bits of information:
+will try to connect to the machine named <literal>mac</literal>. It is
+possible to specify an SSH identity file as part of the remote store
+URI, e.g.
+
+<screen>
+$ nix ping-store --store ssh://mac?ssh-key=/home/alice/my-key
+</screen>
+
+Since builds should be non-interactive, the key should not have a
+passphrase. Alternatively, you can load identities ahead of time into
+<command>ssh-agent</command> or <command>gpg-agent</command>.</para>
+
+<para>If you get the error
+
+<screen>
+bash: nix-store: command not found
+error: cannot connect to 'mac'
+</screen>
+
+then you need to ensure that the <envar>PATH</envar> of
+non-interactive login shells contains Nix.</para>
+
+<warning><para>If you are building via the Nix daemon, it is the Nix
+daemon user account (that is, <literal>root</literal>) that should
+have SSH access to the remote machine. If you can’t or don’t want to
+configure <literal>root</literal> to be able to access to remote
+machine, you can use a private Nix store instead by passing
+e.g. <literal>--store ~/my-nix</literal>.</para></warning>
+
+<para>The list of remote machines can be specified on the command line
+or in the Nix configuration file. The former is convenient for
+testing. For example, the following command allows you to build a
+derivation for <literal>x86_64-darwin</literal> on a Linux machine:
+
+<screen>
+$ uname
+Linux
+
+$ nix build \
+  '(with import &lt;nixpkgs> { system = "x86_64-darwin"; }; runCommand "foo" {} "uname > $out")' \
+  --builders 'ssh://mac x86_64-darwin'
+[1/0/1 built, 0.0 MiB DL] building foo on ssh://mac
+
+$ cat ./result
+Darwin
+</screen>
+
+It is possible to specify multiple builders separated by a semicolon
+or a newline, e.g.
+
+<screen>
+  --builders 'ssh://mac x86_64-darwin ; ssh://beastie x86_64-freebsd'
+</screen>
+</para>
+
+<para>Each machine specification consists of the following elements,
+separated by spaces. Only the first element is required.
 
 <orderedlist>
 
-  <listitem><para>The name of the remote machine, with optionally the
-  user under which the remote build should be performed.  This is
-  actually passed as an argument to <command>ssh</command>, so it can
-  be an alias defined in your
+  <listitem><para>The URI of the remote store in the format
+  <literal>ssh://[<replaceable>username</replaceable>@]<replaceable>hostname</replaceable></literal>,
+  e.g. <literal>ssh://nix@mac</literal> or
+  <literal>ssh://mac</literal>. For backward compatibility,
+  <literal>ssh://</literal> may be omitted. The hostname may be an
+  alias defined in your
   <filename>~/.ssh/config</filename>.</para></listitem>
 
   <listitem><para>A comma-separated list of Nix platform type
   identifiers, such as <literal>x86_64-darwin</literal>.  It is
   possible for a machine to support multiple platform types, e.g.,
-  <literal>i686-linux,x86_64-linux</literal>.</para></listitem>
+  <literal>i686-linux,x86_64-linux</literal>. If omitted, this
+  defaults to the local platform type.</para></listitem>
 
-  <listitem><para>The SSH private key to be used to log in to the
-  remote machine.  Since builds should be non-interactive, this key
-  should not have a passphrase!</para></listitem>
+  <listitem><para>The SSH identity file to be used to log in to the
+  remote machine. If omitted, SSH will use its regular
+  identities.</para></listitem>
 
-  <listitem><para>The maximum number of builds that
-  <filename>build-remote</filename> will execute in parallel on the
-  machine.  Typically this should be equal to the number of CPU cores.
-  For instance, the machine <literal>itchy</literal> in the example
-  will execute up to 8 builds in parallel.</para></listitem>
+  <listitem><para>The maximum number of builds that Nix will execute
+  in parallel on the machine.  Typically this should be equal to the
+  number of CPU cores.  For instance, the machine
+  <literal>itchy</literal> in the example will execute up to 8 builds
+  in parallel.</para></listitem>
 
   <listitem><para>The “speed factor”, indicating the relative speed of
   the machine.  If there are multiple machines of the right type, Nix
@@ -76,30 +114,69 @@ bits of information:
 
   <listitem><para>A comma-separated list of <emphasis>supported
   features</emphasis>.  If a derivation has the
-  <varname>requiredSystemFeatures</varname> attribute, then
-  <filename>build-remote</filename> will only perform the
-  derivation on a machine that has the specified features.  For
-  instance, the attribute
+  <varname>requiredSystemFeatures</varname> attribute, then Nix will
+  only perform the derivation on a machine that has the specified
+  features.  For instance, the attribute
 
 <programlisting>
 requiredSystemFeatures = [ "kvm" ];
 </programlisting>
 
   will cause the build to be performed on a machine that has the
-  <literal>kvm</literal> feature (i.e., <literal>scratchy</literal> in
-  the example above).</para></listitem>
+  <literal>kvm</literal> feature.</para></listitem>
 
   <listitem><para>A comma-separated list of <emphasis>mandatory
   features</emphasis>.  A machine will only be used to build a
   derivation if all of the machine’s mandatory features appear in the
-  derivation’s <varname>requiredSystemFeatures</varname> attribute.
-  Thus, in the example, the machine <literal>poochie</literal> will
-  only do derivations that have
-  <varname>requiredSystemFeatures</varname> set to <literal>["kvm"
-  "perf"]</literal> or <literal>["perf"]</literal>.</para></listitem>
+  derivation’s <varname>requiredSystemFeatures</varname>
+  attribute..</para></listitem>
 
 </orderedlist>
 
-</para>
+For example, the machine specification
+
+<programlisting>
+nix@scratchy.labs.cs.uu.nl  i686-linux      /home/nix/.ssh/id_scratchy_auto        8 1 kvm
+nix@itchy.labs.cs.uu.nl     i686-linux      /home/nix/.ssh/id_scratchy_auto        8 2
+nix@poochie.labs.cs.uu.nl   i686-linux      /home/nix/.ssh/id_scratchy_auto        1 2 kvm benchmark
+</programlisting>
+
+specifies several machines that can perform
+<literal>i686-linux</literal> builds. However,
+<literal>poochie</literal> will only do builds that have the attribute
+
+<programlisting>
+requiredSystemFeatures = [ "benchmark" ];
+</programlisting>
+
+or
+
+<programlisting>
+requiredSystemFeatures = [ "benchmark" "kvm" ];
+</programlisting>
+
+<literal>itchy</literal> cannot do builds that require
+<literal>kvm</literal>, but <literal>scratchy</literal> does support
+such builds. For regular builds, <literal>itchy</literal> will be
+preferred over <literal>scratchy</literal> because it has a higher
+speed factor.</para>
+
+<para>Remote builders can also be configured in
+<filename>nix.conf</filename>, e.g.
+
+<programlisting>
+builders = ssh://mac x86_64-darwin ; ssh://beastie x86_64-freebsd
+</programlisting>
+
+Finally, remote builders can be configured in a separate configuration
+file included in <option>builders</option> via the syntax
+<literal>@<replaceable>file</replaceable></literal>. For example,
+
+<programlisting>
+builders = @/etc/nix/machines
+</programlisting>
+
+causes the list of machines in <filename>/etc/nix/machines</filename>
+to be included. (This is the default.)</para>
 
 </chapter>

doc/manual/command-ref/env-common.xml

@@ -154,6 +154,8 @@ $ mount -o bind /mnt/otherdisk/nix /nix</screen>
   <literal>daemon</literal> if you want to use the Nix daemon to
   execute Nix operations. This is necessary in <link
   linkend="ssec-multi-user">multi-user Nix installations</link>.
+  If the Nix daemon's Unix socket is at some non-standard path,
+  this variable should be set to <literal>unix://path/to/socket</literal>.
   Otherwise, it should be left unset.</para></listitem>
 
 </varlistentry>

doc/manual/command-ref/nix-build.xml

@@ -29,8 +29,6 @@
       </group>
       <replaceable>attrPath</replaceable>
     </arg>
-    <arg><option>--drv-link</option> <replaceable>drvlink</replaceable></arg>
-    <arg><option>--add-drv-link</option></arg>
     <arg><option>--no-out-link</option></arg>
     <arg>
       <group choice='req'>
@@ -91,25 +89,6 @@ also <xref linkend="sec-common-options" />.</phrase></para>
 
 <variablelist>
 
-  <varlistentry><term><option>--drv-link</option> <replaceable>drvlink</replaceable></term>
-
-    <listitem><para>Add a symlink named
-    <replaceable>drvlink</replaceable> to the store derivation
-    produced by <command>nix-instantiate</command>.  The derivation is
-    a root of the garbage collector until the symlink is deleted or
-    renamed.  If there are multiple derivations, numbers are suffixed
-    to <replaceable>drvlink</replaceable> to distinguish between
-    them.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry><term><option>--add-drv-link</option></term>
-
-    <listitem><para>Shorthand for <option>--drv-link</option>
-    <filename>./derivation</filename>.</para></listitem>
-
-  </varlistentry>
-
   <varlistentry><term><option>--no-out-link</option></term>
 
     <listitem><para>Do not create a symlink to the output path.  Note

doc/manual/command-ref/opt-common-syn.xml

@@ -47,7 +47,6 @@
 </arg>
 <arg><option>--fallback</option></arg>
 <arg><option>--readonly-mode</option></arg>
-<arg><option>--show-trace</option></arg>
 <arg>
   <option>-I</option>
   <replaceable>path</replaceable>

doc/manual/command-ref/opt-common.xml

@@ -301,13 +301,6 @@
 
 </varlistentry>
 
-<varlistentry><term><option>--show-trace</option></term>
-
-  <listitem><para>Causes Nix to print out a stack trace in case of Nix
-  expression evaluation errors.</para></listitem>
-
-</varlistentry>
-
 
 <varlistentry xml:id="opt-I"><term><option>-I</option> <replaceable>path</replaceable></term>
 

doc/manual/expressions/builtins.xml

@@ -126,6 +126,17 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting>
   </varlistentry>
 
 
+  <varlistentry><term><function>builtins.splitVersion</function>
+  <replaceable>s</replaceable></term>
+
+    <listitem><para>Split a string representing a version into its
+    components, by the same version splitting logic underlying the
+    version comparison in <link linkend="ssec-version-comparisons">
+    <command>nix-env -u</command></link>.</para></listitem>
+
+  </varlistentry>
+
+
   <varlistentry><term><function>builtins.concatLists</function>
   <replaceable>lists</replaceable></term>
 
@@ -308,8 +319,9 @@ stdenv.mkDerivation { … }
   </varlistentry>
 
 
-  <varlistentry><term><function>builtins.filterSource</function>
-  <replaceable>e1</replaceable> <replaceable>e2</replaceable></term>
+  <varlistentry xml:id='builtin-filterSource'>
+    <term><function>builtins.filterSource</function>
+    <replaceable>e1</replaceable> <replaceable>e2</replaceable></term>
 
     <listitem>
 
@@ -768,6 +780,75 @@ Evaluates to <literal>[ "foo" ]</literal>.
 
   </varlistentry>
 
+  <varlistentry>
+    <term>
+      <function>builtins.path</function>
+      <replaceable>args</replaceable>
+    </term>
+
+    <listitem>
+      <para>
+        An enrichment of the built-in path type, based on the attributes
+        present in <replaceable>args</replaceable>. All are optional
+        except <varname>path</varname>:
+      </para>
+
+      <variablelist>
+        <varlistentry>
+          <term>path</term>
+          <listitem>
+            <para>The underlying path.</para>
+          </listitem>
+        </varlistentry>
+        <varlistentry>
+          <term>name</term>
+          <listitem>
+            <para>
+              The name of the path when added to the store. This can
+              used to reference paths that have nix-illegal characters
+              in their names, like <literal>@</literal>.
+            </para>
+          </listitem>
+        </varlistentry>
+        <varlistentry>
+          <term>filter</term>
+          <listitem>
+            <para>
+              A function of the type expected by
+              <link linkend="builtin-filterSource">builtins.filterSource</link>,
+              with the same semantics.
+            </para>
+          </listitem>
+        </varlistentry>
+        <varlistentry>
+          <term>recursive</term>
+          <listitem>
+            <para>
+              When <literal>false</literal>, when
+              <varname>path</varname> is added to the store it is with a
+              flat hash, rather than a hash of the NAR serialization of
+              the file. Thus, <varname>path</varname> must refer to a
+              regular file, not a directory. This allows similar
+              behavior to <literal>fetchurl</literal>. Defaults to
+              <literal>true</literal>.
+            </para>
+          </listitem>
+        </varlistentry>
+        <varlistentry>
+          <term>sha256</term>
+          <listitem>
+            <para>
+              When provided, this is the expected hash of the file at
+              the path. Evaluation will fail if the hash is incorrect,
+              and providing a hash allows
+              <literal>builtins.path</literal> to be used even when the
+              <literal>pure-eval</literal> nix config option is on.
+            </para>
+          </listitem>
+        </varlistentry>
+      </variablelist>
+    </listitem>
+  </varlistentry>
 
   <varlistentry><term><function>builtins.pathExists</function>
   <replaceable>path</replaceable></term>

doc/manual/expressions/debug-build.xml

@@ -1,34 +0,0 @@
-<section xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id="sec-debug-build">
-
-<title>Debugging Build Failures</title>
-
-<para>At the beginning of each phase of the build (such as unpacking,
-building or installing), the set of all shell variables is written to
-the file <filename>env-vars</filename> at the top-level build
-directory.  This is useful for debugging: it allows you to recreate
-the environment in which a build was performed.  For instance, if a
-build fails, then assuming you used the <option>-K</option> flag, you
-can go to the output directory and <quote>switch</quote> to the
-environment of the builder:
-
-<screen>
-$ nix-build -K ./foo.nix
-... fails, keeping build directory `/tmp/nix-1234-0'
-
-$ cd /tmp/nix-1234-0
-
-$ source env-vars
-
-<lineannotation>(edit some files...)</lineannotation>
-
-$ make
-
-<lineannotation>(execution continues with the same GCC, make, etc.)</lineannotation></screen>
-
-</para>
-
-</section>

doc/manual/expressions/language-constructs.xml

@@ -61,7 +61,7 @@ evaluates to <literal>"foobar"</literal>.
 
 <simplesect><title>Inheriting attributes</title>
 
-<para>When defining a set it is often convenient to copy variables
+<para>When defining a set or in a let-expression it is often convenient to copy variables
 from the surrounding lexical scope (e.g., when you want to propagate
 attributes).  This can be shortened using the
 <literal>inherit</literal> keyword.  For instance,
@@ -72,7 +72,15 @@ let x = 123; in
   y = 456;
 }</programlisting>
 
-evaluates to <literal>{ x = 123; y = 456; }</literal>.  (Note that
+is equivalent to
+
+<programlisting>
+let x = 123; in
+{ x = x;
+  y = 456;
+}</programlisting>
+
+and both evaluate to <literal>{ x = 123; y = 456; }</literal>. (Note that
 this works because <varname>x</varname> is added to the lexical scope
 by the <literal>let</literal> construct.)  It is also possible to
 inherit attributes from another set.  For instance, in this fragment
@@ -101,6 +109,26 @@ variables from the surrounding scope (<varname>fetchurl</varname>
 <varname>libXaw</varname> (the X Athena Widgets) from the
 <varname>xlibs</varname> (X11 client-side libraries) set.</para>
 
+<para>
+Summarizing the fragment
+
+<programlisting>
+...
+inherit x y z;
+inherit (src-set) a b c;
+...</programlisting>
+
+is equivalent to
+
+<programlisting>
+...
+x = x; y = y; z = z;
+a = src-set.a; b = src-set.b; c = src-set.c;
+...</programlisting>
+
+when used while defining local variables in a let-expression or
+while defining a set.</para>
+
 </simplesect>
 
 

doc/manual/expressions/simple-building-testing.xml

@@ -81,6 +81,4 @@ Just pass the option <link linkend='opt-max-jobs'><option>-j
 in parallel, or set.  Typically this should be the number of
 CPUs.</para>
 
-<xi:include href="debug-build.xml" />
-
 </section>

doc/manual/installation/installing-binary.xml

@@ -79,16 +79,6 @@ alice$ ./install
 
 </para>
 
-<para>Nix can be uninstalled using <command>rpm -e nix</command> or
-<command>dpkg -r nix</command> on RPM- and Dpkg-based systems,
-respectively.  After this you should manually remove the Nix store and
-other auxiliary data, if desired:
-
-<screen>
-$ rm -rf /nix</screen>
-
-</para>
-
 <para>You can uninstall Nix simply by running:
 
 <screen>

doc/manual/installation/multi-user.xml

@@ -52,34 +52,6 @@ This creates 10 build users. There can never be more concurrent builds
 than the number of build users, so you may want to increase this if
 you expect to do many builds at the same time.</para>
 
-<para>On macOS, you can create the required group and users by
-running the following script:
-
-<programlisting>
-#! /bin/bash -e
-
-dseditgroup -o create nixbld -q
-
-gid=$(dscl . -read /Groups/nixbld | awk '($1 == "PrimaryGroupID:") {print $2 }')
-
-echo "created nixbld group with gid $gid"
-
-for i in $(seq 1 10); do
-    user=/Users/nixbld$i
-    uid="$((30000 + $i))"
-    dscl . create $user
-    dscl . create $user RealName "Nix build user $i"
-    dscl . create $user PrimaryGroupID "$gid"
-    dscl . create $user UserShell /usr/bin/false
-    dscl . create $user NFSHomeDirectory /var/empty
-    dscl . create $user UniqueID "$uid"
-    dseditgroup -o edit -a nixbld$i -t user nixbld
-    echo "created nixbld$i user with uid $uid"
-done
-</programlisting>
-
-</para>
-
 </simplesect>
 
 

doc/manual/installation/supported-platforms.xml

@@ -33,7 +33,4 @@
 
 </para>
 
-<para>Nix is fairly portable, so it should work on most platforms that
-support POSIX threads and have a C++11 compiler.</para>
-
 </chapter>

doc/manual/introduction/quick-start.xml

@@ -15,7 +15,7 @@ to subsequent chapters.</para>
 <step><para>Install single-user Nix by running the following:
 
 <screen>
-$ curl https://nixos.org/nix/install | sh
+$ bash &lt;(curl https://nixos.org/nix/install)
 </screen>
 
 This will install Nix in <filename>/nix</filename>. The install script

doc/manual/manual.xml

@@ -12,19 +12,14 @@
         <firstname>Eelco</firstname>
         <surname>Dolstra</surname>
       </personname>
-      <affiliation>
-        <orgname>LogicBlox</orgname>
-      </affiliation>
       <contrib>Author</contrib>
     </author>
 
     <copyright>
-      <year>2004-2014</year>
+      <year>2004-2018</year>
       <holder>Eelco Dolstra</holder>
     </copyright>
 
-    <date>November 2014</date>
-
   </info>
 
   <!--
@@ -41,7 +36,6 @@
   <xi:include href="expressions/writing-nix-expressions.xml" />
   <xi:include href="advanced-topics/advanced-topics.xml" />
   <xi:include href="command-ref/command-ref.xml" />
-  <xi:include href="troubleshooting/troubleshooting.xml" />
   <xi:include href="glossary/glossary.xml" />
   <xi:include href="hacking.xml" />
   <xi:include href="release-notes/release-notes.xml" />

doc/manual/release-notes/release-notes.xml

@@ -12,7 +12,7 @@
 </partintro>
 -->
 
-<xi:include href="rl-1.12.xml" />
+<xi:include href="rl-2.0.xml" />
 <xi:include href="rl-1.11.10.xml" />
 <xi:include href="rl-1.11.xml" />
 <xi:include href="rl-1.10.xml" />

doc/manual/release-notes/rl-1.12.xml

@@ -1,426 +0,0 @@
-<section xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id="ssec-relnotes-1.12">
-
-<title>Release 1.12 (TBA)</title>
-
-<para>This release has the following new features:</para>
-
-<itemizedlist>
-
-  <listitem>
-    <para>Start of new <command>nix</command> command line
-    interface. This is a work in progress and the interface is subject
-    to change.</para>
-
-    <itemizedlist>
-
-      <listitem><para>Self-documenting: <option>--help</option> shows
-      all available command-line arguments.</para></listitem>
-
-      <listitem><para><option>--help-config</option> shows all
-      configuration options.</para></listitem>
-
-      <listitem><para><command>nix build</command>: Replacement for
-      <command>nix-build</command>.</para></listitem>
-
-      <listitem><para><command>nix ls-store</command> and <command>nix
-      ls-nar</command> allow listing the contents of a store path or
-      NAR file.</para></listitem>
-
-      <listitem><para><command>nix cat-store</command> and
-      <command>nix cat-nar</command> allow extracting a file from a
-      store path or NAR file.</para></listitem>
-
-      <listitem><para><command>nix verify</command> checks whether a
-      store path is unmodified and/or is trusted.</para></listitem>
-
-      <listitem><para><command>nix copy-sigs</command> copies
-      signatures from one store to another.</para></listitem>
-
-      <listitem><para><command>nix sign-paths</command> signs store
-      paths.</para></listitem>
-
-      <listitem><para><command>nix copy</command> copies paths between
-      arbitrary Nix stores, generalising
-      <command>nix-copy-closure</command> and
-      <command>nix-push</command>.</para></listitem>
-
-      <listitem><para><command>nix path-info</command> shows
-      information about store paths.</para></listitem>
-
-      <listitem><para><command>nix run</command> starts a shell in
-      which the specified packages are available.</para></listitem>
-
-      <listitem><para><command>nix log</command> shows the build log
-      of a package or path. If the build log is not available locally,
-      it will try to obtain it from a binary cache.</para></listitem>
-
-      <listitem><para><command>nix eval</command> replaces
-      <command>nix-instantiate --eval</command>.</para></listitem>
-
-      <listitem><para><command>nix dump-path</command> to get a NAR
-      from a store path.</para></listitem>
-
-      <listitem><para><command>nix edit</command> opens the source
-      code of a package in an editor.</para></listitem>
-
-      <listitem><para><command>nix search</command> replaces
-      <command>nix-env -qa</command>. It searches the available
-      packages for occurences of a search string in the attribute
-      name, package name or description. It caches available packages
-      to speed up searches.</para></listitem>
-
-      <listitem><para><command>nix why-depends</command> (d41c5eb13f4f3a37d80dbc6d3888644170c3b44a).</para></listitem>
-
-      <listitem><para><command>nix show-derivation</command> (e8d6ee7c1b90a2fe6d824f1a875acc56799ae6e2).</para></listitem>
-
-      <listitem><para><command>nix add-to-store</command> (970366266b8df712f5f9cedb45af183ef5a8357f).</para></listitem>
-
-      <listitem><para>Progress indicator.</para></listitem>
-
-      <listitem><para>All options are available as flags now
-      (b8283773bd64d7da6859ed520ee19867742a03ba).</para></listitem>
-
-    </itemizedlist>
-
-  </listitem>
-
-  <listitem>
-    <para>The external program <command>nix-repl</command> has been
-    integrated into Nix as <command>nix repl</command>.</para>
-  </listitem>
-
-  <listitem>
-    <para>New build mode <command>nix-build --hash</command> that
-    builds a derivation, computes the hash of the output, and moves
-    the output to the store path corresponding to what a fixed-output
-    derivation with that hash would produce.
-    (Add docs and examples; see d367b8e7875161e655deaa96bf8a5dd0bcf8229e)</para>
-  </listitem>
-
-  <listitem>
-    <para>It is no longer necessary to set the
-    <envar>NIX_REMOTE</envar> environment variable if you need to use
-    the Nix daemon. Nix will use the daemon automatically if you don’t
-    have write access to the Nix database.</para>
-  </listitem>
-
-  <listitem>
-    <para>The Nix language now supports floating point numbers. They are
-    based on regular C++ <literal>float</literal> and compatible with
-    existing integers and number-related operations. Export and import to and
-    from JSON and XML works, too.</para>
-  </listitem>
-
-  <listitem>
-    <para><command>nix-shell</command> now sets the
-    <varname>IN_NIX_SHELL</varname> environment variable during
-    evaluation and in the shell itself. This can be used to perform
-    different actions depending on whether you’re in a Nix shell or in
-    a regular build. Nixpkgs provides
-    <varname>lib.inNixShell</varname> to check this variable during
-    evaluation. (bb36a1a3cf3fbe6bc9d0afcc5fa0f928bed03170)</para>
-  </listitem>
-
-  <listitem>
-    <para>Internal: all <classname>Store</classname> classes are now
-    thread-safe. <classname>RemoteStore</classname> supports multiple
-    concurrent connections to the daemon. This is primarily useful in
-    multi-threaded programs such as
-    <command>hydra-queue-runner</command>.</para>
-  </listitem>
-
-  <listitem>
-    <para>The dependency on Perl has been removed. As a result, some
-    (obsolete) programs have been removed: <command>nix-push</command>
-    (replaced by <command>nix copy</command>),
-    <command>nix-pull</command> (obsoleted by binary caches),
-    <command>nix-generate-patches</command>,
-    <command>bsdiff</command>, <command>bspatch</command>.</para>
-  </listitem>
-
-  <listitem>
-    <para>Improved store abstraction. Substituters
-    eliminated. BinaryCacheStore, LocalBinaryCacheStore,
-    HttpBinaryCacheStore, S3BinaryCacheStore (compile-time
-    optional), SSHStore. Add docs + examples?
-    </para>
-  </listitem>
-
-  <listitem>
-    <para>Nix now stores signatures for local store
-    paths. Locally-built paths are now signed automatically using the
-    secret keys specified by the <option>secret-key-files</option>
-    store option.</para>
-
-    <para>In addition, store paths that have been built locally are
-    marked as “ultimately trusted”, and content-addressable store
-    paths carry a “content-addressability assertion” that allow them
-    to be trusted without any signatures.</para>
-  </listitem>
-
-  <listitem>
-    <para><envar>NIX_PATH</envar> is now lazy, so URIs in the path are
-    only downloaded if they are needed for evaluation.</para>
-  </listitem>
-
-  <listitem>
-    <para>You can now use
-    <uri>channel:<replaceable>channel-name</replaceable></uri> as a
-    short-hand for
-    <uri>https://nixos.org/channels/<replaceable>channel-name</replaceable>/nixexprs.tar.xz</uri>. For
-    example, <literal>nix-build channel:nixos-15.09 -A hello</literal>
-    will build the GNU Hello package from the
-    <literal>nixos-15.09</literal> channel.</para>
-  </listitem>
-
-  <listitem>
-    <para>When <option>--no-build-output</option> is given, the last
-    10 lines of the build log will be shown if a build
-    fails.</para>
-  </listitem>
-
-  <listitem>
-    <para><function>builtins.fetchGit</function>.
-    (38539b943a060d9cdfc24d6e5d997c0885b8aa2f)</para>
-  </listitem>
-
-  <listitem>
-    <para><literal>&lt;nix/fetchurl.nix&gt;</literal> now uses the
-    content-addressable tarball cache at
-    <uri>http://tarballs.nixos.org/</uri>, just like
-    <function>fetchurl</function> in
-    Nixpkgs. (f2682e6e18a76ecbfb8a12c17e3a0ca15c084197)</para>
-  </listitem>
-
-  <listitem>
-    <para>Chroot Nix stores: allow the “physical” location of the Nix
-    store (e.g. <filename>/home/alice/nix/store</filename>) to differ
-    from its “logical” location (typically
-    <filename>/nix/store</filename>). This allows non-root users to
-    use Nix while still getting the benefits from prebuilt binaries
-    from
-    <uri>cache.nixos.org</uri>. (4494000e04122f24558e1436e66d20d89028b4bd,
-    3eb621750848e0e6b30e5a79f76afbb096bb6c8a)</para>
-  </listitem>
-
-  <listitem>
-    <para>On Linux, builds are now executed in a user
-    namespace with uid 1000 and gid 100.</para>
-  </listitem>
-
-  <listitem>
-    <para><function>builtins.fetchurl</function> and
-    <function>builtins.fetchTarball</function> now support
-    <varname>sha256</varname> and <varname>name</varname>
-    attributes.</para>
-  </listitem>
-
-  <listitem>
-    <para><literal>HttpBinaryCacheStore</literal> (the replacement of
-    <command>download-from-binary-cache</command>) now retries
-    automatically on certain HTTP error codes.</para>
-  </listitem>
-
-  <listitem>
-    <para>Derivation attributes can now reference the outputs of the
-    derivation using the <function>placeholder</function> builtin
-    function. For example, the attribute
-
-<programlisting>
-configureFlags = "--prefix=${placeholder "out"} --includedir=${placeholder "dev"}";
-</programlisting>
-
-    will cause the <envar>configureFlags</envar> environment variable
-    to contain the actual store paths corresponding to the
-    <literal>out</literal> and <literal>dev</literal> outputs. TODO:
-    add docs.</para>
-  </listitem>
-
-  <listitem>
-    <para>Support for HTTP/2. This makes binary cache lookups much
-    more efficient. (90ad02bf626b885a5dd8967894e2eafc953bdf92)</para>
-  </listitem>
-
-  <listitem>
-    <para>The <option>build-sandbox-paths</option> configuration
-    option can now specify optional paths by appending a
-    <literal>?</literal>, e.g. <literal>/dev/nvidiactl?</literal> will
-    bind-mount <varname>/dev/nvidiactl</varname> only if it
-    exists.</para>
-  </listitem>
-
-  <listitem>
-    <para>More support for testing build reproducibility: when
-    <option>enforce-determinism</option> is set to
-    <literal>false</literal>, it’s no longer a fatal error build
-    rounds produce different output
-    (8bdf83f936adae6f2c907a6d2541e80d4120f051); add a hook to run
-    diffoscope when build rounds produce different output
-    (9a313469a4bdea2d1e8df24d16289dc2a172a169w).</para>
-  </listitem>
-
-  <listitem>
-    <para>Kill builds as soon as stdout/stderr is closed. This fixes a
-    bug that allowed builds to hang Nix indefinitely (regardless of
-    timeouts). (21948deed99a3295e4d5666e027a6ca42dc00b40)</para>
-  </listitem>
-
-  <listitem>
-    <para>Add support for passing structured data to builders. TODO:
-    document. (6de33a9c675b187437a2e1abbcb290981a89ecb1)</para>
-  </listitem>
-
-  <listitem>
-    <para><varname>exportReferencesGraph</varname>: Export more
-    complete info in JSON
-    format. (c2b0d8749f7e77afc1c4b3e8dd36b7ee9720af4a)</para>
-  </listitem>
-
-  <listitem>
-    <para>Support for
-    netrc. (e6e74f987f0fa284d220432d426eb965269a97d6,
-    302386f775eea309679654e5ea7c972fb6e7b9af)</para>
-  </listitem>
-
-  <listitem>
-    <para>Support <uri>s3://</uri> URIs in all places where Nix allows
-    URIs. (9ff9c3f2f80ba4108e9c945bbfda2c64735f987b)</para>
-  </listitem>
-
-  <listitem>
-    <para>The <option>build-max-jobs</option> option can be set to
-    <literal>auto</literal> to use the number of CPUs in the
-    system. (7251d048fa812d2551b7003bc9f13a8f5d4c95a5)</para>
-  </listitem>
-
-  <listitem>
-    <para>Add support for Brotli compression.
-    <uri>cache.nixos.org</uri> compresses build logs using
-    Brotli.</para>
-  </listitem>
-
-  <listitem>
-    <para>Substitutions from binary caches now require signatures by
-    default. This was already the case on
-    NixOS. (ecbc3fedd3d5bdc5a0e1a0a51b29062f2874ac8b)</para>
-  </listitem>
-
-  <listitem>
-    <para><command>nix-env</command> now ignores packages with bad
-    derivation names (in particular those starting with a digit or
-    containing a
-    dot). (b0cb11722626e906a73f10dd9a0c9eea29faf43a)</para>
-  </listitem>
-
-  <listitem>
-    <para>Renamed various configuration options. (TODO: in progress)</para>
-  </listitem>
-
-  <listitem>
-    <para>Remote machines can now be specified on the command
-    line. TODO:
-    document. (1a68710d4dff609bbaf61db3e17a2573f0aadf17)</para>
-  </listitem>
-
-  <listitem>
-    <para>In Linux sandbox builds, we now use
-    <filename>/build</filename> instead of <filename>/tmp</filename>
-    as the temporary build directory. This fixes potential security
-    problems when a build accidentally stores its
-    <envar>TMPDIR</envar> in some critical place, such as an
-    RPATH. (eba840c8a13b465ace90172ff76a0db2899ab11b)</para>
-  </listitem>
-
-  <listitem>
-    <para>In Linux sandbox builds, we now provide a default
-    <filename>/bin/sh</filename> (namely <filename>ash</filename> from
-    BusyBox). (a2d92bb20e82a0957067ede60e91fab256948b41)</para>
-  </listitem>
-
-  <listitem>
-    <para>Make all configuration options available as command line
-    flags (b8283773bd64d7da6859ed520ee19867742a03ba).</para>
-  </listitem>
-
-  <listitem>
-    <para>Support base-64
-    hashes. (c0015e87af70f539f24d2aa2bc224a9d8b84276b)</para>
-  </listitem>
-
-  <listitem>
-    <para><command>nix-shell</command> now uses
-    <varname>bashInteractive</varname> from Nixpkgs, rather than the
-    <command>bash</command> command that happens to be in the caller’s
-    <envar>PATH</envar>. This is especially important on macOS where
-    the <command>bash</command> provided by the system is seriously
-    outdated and cannot execute <literal>stdenv</literal>’s setup
-    script.</para>
-  </listitem>
-
-  <listitem>
-    <para>New builtin functions: <function>builtins.split</function>
-    (b8867a0239b1930a16f9ef3f7f3e864b01416dff),
-    <function>builtins.partition</function>.</para>
-  </listitem>
-
-  <listitem>
-    <para>Automatic garbage collection.</para>
-  </listitem>
-
-  <listitem>
-    <para><command>nix-store -q --roots</command> and
-    <command>nix-store --gc --print-roots</command> now show temporary
-    and in-memory roots.</para>
-  </listitem>
-
-  <listitem>
-    <para>Builders can now communicate what build phase they are in by
-    writing messages to the file descriptor specified in
-    <envar>NIX_LOG_FD</envar>. (88e6bb76de5564b3217be9688677d1c89101b2a3)
-    </para>
-  </listitem>
-
-</itemizedlist>
-
-<para>Some features were removed:</para>
-
-<itemizedlist>
-
-  <listitem>
-    <para>“Nested” log output. As a result,
-    <command>nix-log2xml</command> was also removed.</para>
-  </listitem>
-
-  <listitem>
-    <para>OpenSSL-based signing. (f435f8247553656774dd1b2c88e9de5d59cab203)</para>
-  </listitem>
-
-  <listitem>
-    <para>Caching of failed
-    builds. (8cffec84859cec8b610a2a22ab0c4d462a9351ff)</para>
-  </listitem>
-
-  <listitem>
-    <para><filename>nix-mode.el</filename> has been removed from
-    Nix. It is now a separate repository in
-    <uri>https://github.com/NixOS/nix-mode</uri> and can be installed
-    through the MELPA package repository.</para>
-  </listitem>
-
-  <listitem>
-    <para>In restricted evaluation mode
-    (<option>--restrict-eval</option>), builtin functions that
-    download from the network (such as <function>fetchGit</function>)
-    are permitted to fetch underneath the list of URI prefixes
-    specified in the option <option>allowed-uris</option>.</para>
-  </listitem>
-
-</itemizedlist>
-
-<para>This release has contributions from TBD.</para>
-
-</section>

doc/manual/style.css

@@ -96,7 +96,6 @@ div.example
     margin-right: 1.5em;
     background: #f4f4f8;
     border-radius: 0.4em;
-    box-shadow: 0.4em 0.4em 0.5em #e0e0e0;
 }
 
 div.example p.title
@@ -106,7 +105,6 @@ div.example p.title
 
 div.example pre
 {
-    box-shadow: none;
 }
 
 
@@ -116,15 +114,12 @@ div.example pre
 
 pre.screen, pre.programlisting
 {
-    border: 1px solid #b0b0b0;
-    padding: 3px 3px;
+    padding: 6px 6px;
     margin-left: 1.5em;
     margin-right: 1.5em;
     color: #600000;
     background: #f4f4f8;
     font-family: monospace;
-    border-radius: 0.4em;
-    box-shadow: 0.4em 0.4em 0.5em #e0e0e0;
 }
 
 div.example pre.programlisting
@@ -149,7 +144,6 @@ div.example pre.programlisting
     padding: 0.3em 0.3em 0.3em 0.3em;
     background: #fffff5;
     border-radius: 0.4em;
-    box-shadow: 0.4em 0.4em 0.5em #e0e0e0;
 }
 
 div.note, div.warning
@@ -256,16 +250,14 @@ span.command strong
 
 div.calloutlist table
 {
-    box-shadow: none;
 }
 
 table
 {
     border-collapse: collapse;
-    box-shadow: 0.4em 0.4em 0.5em #e0e0e0;
 }
 
 div.affiliation
 {
     font-style: italic;
-}
+}

doc/manual/troubleshooting/collisions-nixenv.xml

@@ -1,38 +0,0 @@
-<section xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id="sec-collisions-nixenv">
-
-<title>Collisions in <command>nix-env</command></title>
-
-<para>Symptom: when installing or upgrading, you get an error message such as
-
-<screen>
-$ nix-env -i docbook-xml
-...
-adding /nix/store/s5hyxgm62gk2...-docbook-xml-4.2
-collision between `/nix/store/s5hyxgm62gk2...-docbook-xml-4.2/xml/dtd/docbook/calstblx.dtd'
-  and `/nix/store/06h377hr4b33...-docbook-xml-4.3/xml/dtd/docbook/calstblx.dtd'
-  at /nix/store/...-builder.pl line 62.</screen>
-
-</para>
-
-<para>The cause is that two installed packages in the user environment
-have overlapping filenames (e.g.,
-<filename>xml/dtd/docbook/calstblx.dtd</filename>.  This usually
-happens when you accidentally try to install two versions of the same
-package.  For instance, in the example above, the Nix Packages
-collection contains two versions of <literal>docbook-xml</literal>, so
-<command>nix-env -i</command> will try to install both.  The default
-user environment builder has no way to way to resolve such conflicts,
-so it just gives up.</para>
-
-<para>Solution: remove one of the offending packages from the user
-environment (if already installed) using <command>nix-env
--e</command>, or specify exactly which version should be installed
-(e.g., <literal>nix-env -i docbook-xml-4.2</literal>).</para>
-
-<!-- FIXME: describe priorities -->
-
-</section>

doc/manual/troubleshooting/links-nix-store.xml

@@ -1,43 +0,0 @@
-<section xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id="sec-links-nix-store">
-
-<title><quote>Too many links</quote> Error in the Nix store</title>
-
-
-<para>Symptom: when building something, you get an error message such as
-
-<screen>
-...
-<literal>mkdir: cannot create directory `/nix/store/<replaceable>name</replaceable>': Too many links</literal></screen>
-
-</para>
-
-<para>This is usually because you have more than 32,000 subdirectories
-in <filename>/nix/store</filename>, as can be seen using <command>ls
--l</command>:
-
-<screen>
-$ ls -ld /nix/store
-drwxrwxrwt 32000 nix nix 4620288 Sep 8 15:08 store</screen>
-
-The <literal>ext2</literal> file system is limited to an inode link
-count of 32,000 (each subdirectory increasing the count by one).
-Furthermore, the <literal>st_nlink</literal> field of the
-<function>stat</function> system call is a 16-bit value.</para>
-
-<para>This only happens on very large Nix installations (such as build
-machines).</para>
-
-<para>Quick solution: run the garbage collector.  You may want to use
-the <option>--max-links</option> option.</para>
-
-<para>Real solution: put the Nix store on a file system that supports
-more than 32,000 subdirectories per directory, such as ext4.  (This
-doesn’t solve the <literal>st_nlink</literal> limit, but ext4 lies to
-the kernel by reporting a link count of 1 if it exceeds the
-limit.)</para>
-
-</section>

doc/manual/troubleshooting/troubleshooting.xml

@@ -1,16 +0,0 @@
-<appendix xmlns="http://docbook.org/ns/docbook"
-          xmlns:xlink="http://www.w3.org/1999/xlink"
-          xmlns:xi="http://www.w3.org/2001/XInclude"
-          version="5.0"
-          xml:id="ch-troubleshooting">
-
-<title>Troubleshooting</title>
-
-<para>This section provides solutions for some common problems.  See
-the <link xlink:href="https://github.com/NixOS/nix/issues">Nix bug
-tracker</link> for a list of currently known issues.</para>
-
-<xi:include href="collisions-nixenv.xml" />
-<xi:include href="links-nix-store.xml" />
-
-</appendix>

mk/libraries.mk

@@ -45,6 +45,11 @@ endif
 # - $(1)_INSTALL_DIR: the directory where the library will be
 #   installed.  Defaults to $(libdir).
 #
+# - $(1)_EXCLUDE_FROM_LIBRARY_LIST: if defined, the library will not
+#   be automatically marked as a dependency of the top-level all
+#   target andwill not be listed in the make help output. This is
+#   useful for libraries built solely for testing, for example.
+#
 # - BUILD_SHARED_LIBS: if equal to ‘1’, a dynamic library will be
 #   built, otherwise a static library.
 define build-library
@@ -149,7 +154,9 @@ define build-library
   $(1)_DEPS := $$(foreach fn, $$($(1)_OBJS), $$(call filename-to-dep, $$(fn)))
   -include $$($(1)_DEPS)
 
+  ifndef $(1)_EXCLUDE_FROM_LIBRARY_LIST
   libs-list += $$($(1)_PATH)
+  endif
   clean-files += $$(_d)/*.a $$(_d)/*.$(SO_EXT) $$(_d)/*.o $$(_d)/.*.dep $$($(1)_DEPS) $$($(1)_OBJS)
   dist-files += $$(_srcs)
 endef

mk/tests.mk

@@ -39,7 +39,7 @@ installcheck:
 	  echo "$${red}$$failed out of $$total tests failed $$normal"; \
 	  exit 1; \
 	else \
-		echo "$${green}All tests succeeded"; \
+		echo "$${green}All tests succeeded$$normal"; \
 	fi
 
 .PHONY: check installcheck

release-common.nix

@@ -1,17 +1,31 @@
 { pkgs }:
 
 rec {
-  sh = pkgs.busybox.override {
+  # Use "busybox-sandbox-shell" if present,
+  # if not (legacy) fallback and hope it's sufficient.
+  sh = pkgs.busybox-sandbox-shell or (pkgs.busybox.override {
     useMusl = true;
     enableStatic = true;
     enableMinimal = true;
     extraConfig = ''
+      CONFIG_FEATURE_FANCY_ECHO y
+      CONFIG_FEATURE_SH_MATH y
+      CONFIG_FEATURE_SH_MATH_64 y
+
       CONFIG_ASH y
-      CONFIG_ASH_ECHO y
-      CONFIG_ASH_TEST y
       CONFIG_ASH_OPTIMIZE_FOR_SIZE y
+
+      CONFIG_ASH_ALIAS y
+      CONFIG_ASH_BASH_COMPAT y
+      CONFIG_ASH_CMDCMD y
+      CONFIG_ASH_ECHO y
+      CONFIG_ASH_GETOPTS y
+      CONFIG_ASH_INTERNAL_GLOB y
+      CONFIG_ASH_JOB_CONTROL y
+      CONFIG_ASH_PRINTF y
+      CONFIG_ASH_TEST y
     '';
-  };
+  });
 
   configureFlags =
     [ "--disable-init-state"

release.nix

@@ -1,12 +1,12 @@
 { nix ? builtins.fetchGit ./.
-, nixpkgs ? fetchTarball channel:nixos-17.09
+, nixpkgs ? builtins.fetchGit { url = https://github.com/NixOS/nixpkgs.git; ref = "nix-2.0"; }
 , officialRelease ? false
 , systems ? [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ]
 }:
 
 let
 
-  pkgs = import nixpkgs {};
+  pkgs = import nixpkgs { system = builtins.currentSystem or "x86_64-linux"; };
 
   jobs = rec {
 
@@ -127,7 +127,6 @@ let
 
     binaryTarball = pkgs.lib.genAttrs systems (system:
 
-      # FIXME: temporarily use a different branch for the Darwin build.
       with import nixpkgs { inherit system; };
 
       let
@@ -137,7 +136,7 @@ let
 
       runCommand "nix-binary-tarball-${version}"
         { exportReferencesGraph = [ "closure1" toplevel "closure2" cacert ];
-          buildInputs = [ perl shellcheck ];
+          buildInputs = [ perl ] ++ lib.optional (system != "aarch64-linux") shellcheck;
           meta.description = "Distribution-independent Nix bootstrap binaries for ${system}";
         }
         ''
@@ -150,8 +149,10 @@ let
             --subst-var-by nix ${toplevel} \
             --subst-var-by cacert ${cacert}
 
-          shellcheck -e SC1090 $TMPDIR/install
-          shellcheck -e SC1091,SC2002 $TMPDIR/install-darwin-multi-user
+          if type -p shellcheck; then
+            shellcheck -e SC1090 $TMPDIR/install
+            shellcheck -e SC1091,SC2002 $TMPDIR/install-darwin-multi-user
+          fi
 
           chmod +x $TMPDIR/install
           chmod +x $TMPDIR/install-darwin-multi-user
@@ -224,11 +225,13 @@ let
       nix = build.x86_64-linux; system = "x86_64-linux";
     });
 
-    tests.setuid = pkgs.lib.genAttrs (pkgs.lib.filter (pkgs.lib.hasSuffix "-linux") systems) (system:
-      import ./tests/setuid.nix rec {
-        inherit nixpkgs;
-        nix = build.${system}; inherit system;
-      });
+    tests.setuid = pkgs.lib.genAttrs
+      ["i686-linux" "x86_64-linux"]
+      (system:
+        import ./tests/setuid.nix rec {
+          inherit nixpkgs;
+          nix = build.${system}; inherit system;
+        });
 
     tests.binaryTarball =
       with import nixpkgs { system = "x86_64-linux"; };

scripts/install-darwin-multi-user.sh

@@ -24,7 +24,7 @@ readonly YELLOW='\033[38;33m'
 readonly YELLOW_UL='\033[38;4;33m'
 
 readonly CORES=$(sysctl -n hw.ncpu)
-readonly NIX_USER_COUNT="$CORES"
+readonly NIX_USER_COUNT="32"
 readonly NIX_BUILD_GROUP_ID="30000"
 readonly NIX_BUILD_GROUP_NAME="nixbld"
 readonly NIX_FIRST_BUILD_UID="30001"
@@ -647,7 +647,7 @@ chat_about_sudo() {
         cat <<EOF
 This script is going to call sudo a lot. Normally, it would show you
 exactly what commands it is running and why. However, the script is
-run in  a headless fashion, like this:
+run in a headless fashion, like this:
 
   $ curl https://nixos.org/nix/install | sh
 
@@ -695,7 +695,7 @@ install_from_extracted_nix() {
         cd "$EXTRACTED_NIX_PATH"
 
         _sudo "to copy the basic Nix files to the new store at $NIX_ROOT/store" \
-              rsync -rlpt "$(pwd)/store/" "$NIX_ROOT/store/"
+              rsync -rlpt ./store/* "$NIX_ROOT/store/"
 
         if [ -d "$NIX_INSTALLED_NIX" ]; then
             echo "      Alright! We have our first nix at $NIX_INSTALLED_NIX"

scripts/nix-profile.sh.in

@@ -75,7 +75,7 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then
         export NIX_SSL_CERT_FILE="$NIX_LINK/etc/ca-bundle.crt"
     fi
 
-    if [ -n ${MANPATH} ]; then
+    if [ -n "${MANPATH}" ]; then
         export MANPATH="$NIX_LINK/share/man:$MANPATH"
     fi
 

src/build-remote/build-remote.cc

@@ -64,6 +64,8 @@ int main (int argc, char * * argv)
 
         settings.maxBuildJobs.set("1"); // hack to make tests with local?root= work
 
+        initPlugins();
+
         auto store = openStore().cast<LocalStore>();
 
         /* It would be more appropriate to use $XDG_RUNTIME_DIR, since
@@ -241,7 +243,7 @@ connected:
 
         if (!missing.empty()) {
             Activity act(*logger, lvlTalkative, actUnknown, fmt("copying outputs from '%s'", storeUri));
-            setenv("NIX_HELD_LOCKS", concatStringsSep(" ", missing).c_str(), 1); /* FIXME: ugly */
+            store->locksHeld.insert(missing.begin(), missing.end()); /* FIXME: ugly */
             copyPaths(ref<Store>(sshStore), store, missing, NoRepair, NoCheckSigs, substitute);
         }
 

src/libexpr/attr-set.cc

@@ -7,13 +7,14 @@
 namespace nix {
 
 
+/* Note: Various places expect the allocated memory to be zeroed. */
 static void * allocBytes(size_t n)
 {
     void * p;
 #if HAVE_BOEHMGC
     p = GC_malloc(n);
 #else
-    p = malloc(n);
+    p = calloc(n, 1);
 #endif
     if (!p) throw std::bad_alloc();
     return p;

src/libexpr/attr-set.hh

@@ -83,7 +83,7 @@ public:
         for (size_t n = 0; n < size_; n++)
             res.emplace_back(&attrs[n]);
         std::sort(res.begin(), res.end(), [](const Attr * a, const Attr * b) {
-            return (string) a->name < (string) b->name;
+            return (const string &) a->name < (const string &) b->name;
         });
         return res;
     }

src/libexpr/eval.cc

@@ -43,13 +43,14 @@ static char * dupString(const char * s)
 }
 
 
+/* Note: Various places expect the allocated memory to be zeroed. */
 static void * allocBytes(size_t n)
 {
     void * p;
 #if HAVE_BOEHMGC
     p = GC_malloc(n);
 #else
-    p = malloc(n);
+    p = calloc(n, 1);
 #endif
     if (!p) throw std::bad_alloc();
     return p;
@@ -293,6 +294,10 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
     , sWrong(symbols.create("wrong"))
     , sStructuredAttrs(symbols.create("__structuredAttrs"))
     , sBuilder(symbols.create("builder"))
+    , sArgs(symbols.create("args"))
+    , sOutputHash(symbols.create("outputHash"))
+    , sOutputHashAlgo(symbols.create("outputHashAlgo"))
+    , sOutputHashMode(symbols.create("outputHashMode"))
     , repair(NoRepair)
     , store(store)
     , baseEnv(allocEnv(128))
@@ -300,15 +305,24 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
 {
     countCalls = getEnv("NIX_COUNT_CALLS", "0") != "0";
 
-    restricted = settings.restrictEval;
-
     assert(gcInitialised);
 
     /* Initialise the Nix expression search path. */
-    Strings paths = parseNixPath(getEnv("NIX_PATH", ""));
-    for (auto & i : _searchPath) addToSearchPath(i);
-    for (auto & i : paths) addToSearchPath(i);
-    addToSearchPath("nix=" + settings.nixDataDir + "/nix/corepkgs");
+    if (!settings.pureEval) {
+        Strings paths = parseNixPath(getEnv("NIX_PATH", ""));
+        for (auto & i : _searchPath) addToSearchPath(i);
+        for (auto & i : paths) addToSearchPath(i);
+    }
+    addToSearchPath("nix=" + canonPath(settings.nixDataDir + "/nix/corepkgs"));
+
+    if (settings.restrictEval || settings.pureEval) {
+        allowedPaths = PathSet();
+        for (auto & i : searchPath) {
+            auto r = resolveSearchPathElem(i);
+            if (!r.first) continue;
+            allowedPaths->insert(r.second);
+        }
+    }
 
     clearValue(vEmptySet);
     vEmptySet.type = tAttrs;
@@ -326,38 +340,36 @@ EvalState::~EvalState()
 
 Path EvalState::checkSourcePath(const Path & path_)
 {
-    if (!restricted) return path_;
+    if (!allowedPaths) return path_;
+
+    bool found = false;
+
+    for (auto & i : *allowedPaths) {
+        if (isDirOrInDir(path_, i)) {
+            found = true;
+            break;
+        }
+    }
+
+    if (!found)
+        throw RestrictedPathError("access to path '%1%' is forbidden in restricted mode", path_);
 
     /* Resolve symlinks. */
     debug(format("checking access to '%s'") % path_);
     Path path = canonPath(path_, true);
 
-    for (auto & i : searchPath) {
-        auto r = resolveSearchPathElem(i);
-        if (!r.first) continue;
-        if (path == r.second || isInDir(path, r.second))
+    for (auto & i : *allowedPaths) {
+        if (isDirOrInDir(path, i))
             return path;
     }
 
-    /* To support import-from-derivation, allow access to anything in
-       the store. FIXME: only allow access to paths that have been
-       constructed by this evaluation. */
-    if (store->isInStore(path)) return path;
-
-#if 0
-    /* Hack to support the chroot dependencies of corepkgs (see
-       corepkgs/config.nix.in). */
-    if (path == settings.nixPrefix && isStorePath(settings.nixPrefix))
-        return path;
-#endif
-
-    throw RestrictedPathError(format("access to path '%1%' is forbidden in restricted mode") % path_);
+    throw RestrictedPathError("access to path '%1%' is forbidden in restricted mode", path);
 }
 
 
 void EvalState::checkURI(const std::string & uri)
 {
-    if (!restricted) return;
+    if (!settings.restrictEval) return;
 
     /* 'uri' should be equal to a prefix, or in a subdirectory of a
        prefix. Thus, the prefix https://github.co does not permit
@@ -371,11 +383,33 @@ void EvalState::checkURI(const std::string & uri)
             && (prefix[prefix.size() - 1] == '/' || uri[prefix.size()] == '/')))
             return;
 
+    /* If the URI is a path, then check it against allowedPaths as
+       well. */
+    if (hasPrefix(uri, "/")) {
+        checkSourcePath(uri);
+        return;
+    }
+
+    if (hasPrefix(uri, "file://")) {
+        checkSourcePath(std::string(uri, 7));
+        return;
+    }
+
     throw RestrictedPathError("access to URI '%s' is forbidden in restricted mode", uri);
 }
 
 
-void EvalState::addConstant(const string & name, Value & v)
+Path EvalState::toRealPath(const Path & path, const PathSet & context)
+{
+    // FIXME: check whether 'path' is in 'context'.
+    return
+        !context.empty() && store->isInStore(path)
+        ? store->toRealPath(path)
+        : path;
+};
+
+
+Value * EvalState::addConstant(const string & name, Value & v)
 {
     Value * v2 = allocValue();
     *v2 = v;
@@ -383,12 +417,18 @@ void EvalState::addConstant(const string & name, Value & v)
     baseEnv.values[baseEnvDispl++] = v2;
     string name2 = string(name, 0, 2) == "__" ? string(name, 2) : name;
     baseEnv.values[0]->attrs->push_back(Attr(symbols.create(name2), v2));
+    return v2;
 }
 
 
-void EvalState::addPrimOp(const string & name,
+Value * EvalState::addPrimOp(const string & name,
     unsigned int arity, PrimOpFun primOp)
 {
+    if (arity == 0) {
+        Value v;
+        primOp(*this, noPos, nullptr, v);
+        return addConstant(name, v);
+    }
     Value * v = allocValue();
     string name2 = string(name, 0, 2) == "__" ? string(name, 2) : name;
     Symbol sym = symbols.create(name2);
@@ -397,6 +437,7 @@ void EvalState::addPrimOp(const string & name,
     staticBaseEnv.vars[symbols.create(name)] = baseEnvDispl;
     baseEnv.values[baseEnvDispl++] = v;
     baseEnv.values[0]->attrs->push_back(Attr(sym, v));
+    return v;
 }
 
 
@@ -546,9 +587,7 @@ Env & EvalState::allocEnv(unsigned int size)
     Env * env = (Env *) allocBytes(sizeof(Env) + size * sizeof(Value *));
     env->size = size;
 
-    /* Clear the values because maybeThunk() and lookupVar fromWith expect this. */
-    for (unsigned i = 0; i < size; ++i)
-        env->values[i] = 0;
+    /* We assume that env->values has been cleared by the allocator; maybeThunk() and lookupVar fromWith expect this. */
 
     return *env;
 }
@@ -649,8 +688,10 @@ Value * ExprPath::maybeThunk(EvalState & state, Env & env)
 }
 
 
-void EvalState::evalFile(const Path & path, Value & v)
+void EvalState::evalFile(const Path & path_, Value & v)
 {
+    auto path = checkSourcePath(path_);
+
     FileEvalCache::iterator i;
     if ((i = fileEvalCache.find(path)) != fileEvalCache.end()) {
         v = i->second;
@@ -1546,7 +1587,7 @@ string EvalState::copyPathToStore(PathSet & context, const Path & path)
         dstPath = srcToStore[path];
     else {
         dstPath = settings.readOnlyMode
-            ? store->computeStorePathForPath(checkSourcePath(path)).first
+            ? store->computeStorePathForPath(baseNameOf(path), checkSourcePath(path)).first
             : store->addToStore(baseNameOf(path), checkSourcePath(path), true, htSHA256, defaultPathFilter, repair);
         srcToStore[path] = dstPath;
         printMsg(lvlChatty, format("copied source '%1%' -> '%2%'")
@@ -1668,10 +1709,13 @@ void EvalState::printStats()
     printMsg(v, format("  time elapsed: %1%") % cpuTime);
     printMsg(v, format("  size of a value: %1%") % sizeof(Value));
     printMsg(v, format("  size of an attr: %1%") % sizeof(Attr));
-    printMsg(v, format("  environments allocated: %1% (%2% bytes)") % nrEnvs % bEnvs);
-    printMsg(v, format("  list elements: %1% (%2% bytes)") % nrListElems % bLists);
+    printMsg(v, format("  environments allocated count: %1%") % nrEnvs);
+    printMsg(v, format("  environments allocated bytes: %1%") % bEnvs);
+    printMsg(v, format("  list elements count: %1%") % nrListElems);
+    printMsg(v, format("  list elements bytes: %1%") % bLists);
     printMsg(v, format("  list concatenations: %1%") % nrListConcats);
-    printMsg(v, format("  values allocated: %1% (%2% bytes)") % nrValues % bValues);
+    printMsg(v, format("  values allocated count: %1%") % nrValues);
+    printMsg(v, format("  values allocated bytes: %1%") % bValues);
     printMsg(v, format("  sets allocated: %1% (%2% bytes)") % nrAttrsets % bAttrsets);
     printMsg(v, format("  right-biased unions: %1%") % nrOpUpdates);
     printMsg(v, format("  values copied in right-biased unions: %1%") % nrOpUpdateValuesCopied);

src/libexpr/eval.hh

@@ -69,16 +69,17 @@ public:
     const Symbol sWith, sOutPath, sDrvPath, sType, sMeta, sName, sValue,
         sSystem, sOverrides, sOutputs, sOutputName, sIgnoreNulls,
         sFile, sLine, sColumn, sFunctor, sToString,
-        sRight, sWrong, sStructuredAttrs, sBuilder;
+        sRight, sWrong, sStructuredAttrs, sBuilder, sArgs,
+        sOutputHash, sOutputHashAlgo, sOutputHashMode;
     Symbol sDerivationNix;
 
     /* If set, force copying files to the Nix store even if they
        already exist there. */
     RepairFlag repair;
 
-    /* If set, don't allow access to files outside of the Nix search
-       path or to environment variables. */
-    bool restricted;
+    /* The allowed filesystem paths in restricted or pure evaluation
+       mode. */
+    std::experimental::optional<PathSet> allowedPaths;
 
     Value vEmptySet;
 
@@ -112,6 +113,15 @@ public:
 
     void checkURI(const std::string & uri);
 
+    /* When using a diverted store and 'path' is in the Nix store, map
+       'path' to the diverted location (e.g. /nix/store/foo is mapped
+       to /home/alice/my-nix/nix/store/foo). However, this is only
+       done if the context is not empty, since otherwise we're
+       probably trying to read from the actual /nix/store. This is
+       intended to distinguish between import-from-derivation and
+       sources stored in the actual /nix/store. */
+    Path toRealPath(const Path & path, const PathSet & context);
+
     /* Parse a Nix expression from the specified file. */
     Expr * parseExprFromFile(const Path & path);
     Expr * parseExprFromFile(const Path & path, StaticEnv & staticEnv);
@@ -201,9 +211,9 @@ private:
 
     void createBaseEnv();
 
-    void addConstant(const string & name, Value & v);
+    Value * addConstant(const string & name, Value & v);
 
-    void addPrimOp(const string & name,
+    Value * addPrimOp(const string & name,
         unsigned int arity, PrimOpFun primOp);
 
 public:

src/libexpr/lexer.l

@@ -49,9 +49,10 @@ static void adjustLoc(YYLTYPE * loc, const char * s, size_t len)
 }
 
 
-static Expr * unescapeStr(SymbolTable & symbols, const char * s)
+static Expr * unescapeStr(SymbolTable & symbols, const char * s, size_t length)
 {
     string t;
+    t.reserve(length);
     char c;
     while ((c = *s++)) {
         if (c == '\\') {
@@ -150,7 +151,7 @@ or          { return OR_KW; }
                 /* It is impossible to match strings ending with '$' with one
                    regex because trailing contexts are only valid at the end
                    of a rule. (A sane but undocumented limitation.) */
-                yylval->e = unescapeStr(data->symbols, yytext);
+                yylval->e = unescapeStr(data->symbols, yytext, yyleng);
                 return STR;
               }
 <STRING>\$\{  { PUSH_STATE(INSIDE_DOLLAR_CURLY); return DOLLAR_CURLY; }
@@ -178,7 +179,7 @@ or          { return OR_KW; }
                    return IND_STR;
                  }
 <IND_STRING>\'\'\\. {
-                   yylval->e = unescapeStr(data->symbols, yytext + 2);
+                   yylval->e = unescapeStr(data->symbols, yytext + 2, yyleng - 2);
                    return IND_STR;
                  }
 <IND_STRING>\$\{ { PUSH_STATE(INSIDE_DOLLAR_CURLY); return DOLLAR_CURLY; }

src/libexpr/names.cc

@@ -41,7 +41,7 @@ bool DrvName::matches(DrvName & n)
 }
 
 
-static string nextComponent(string::const_iterator & p,
+string nextComponent(string::const_iterator & p,
     const string::const_iterator end)
 {
     /* Skip any dots and dashes (component separators). */

src/libexpr/names.hh

@@ -24,6 +24,8 @@ private:
 
 typedef list<DrvName> DrvNames;
 
+string nextComponent(string::const_iterator & p,
+    const string::const_iterator end);
 int compareVersions(const string & v1, const string & v2);
 DrvNames drvNamesFromArgs(const Strings & opArgs);
 

src/libexpr/primops.cc

@@ -39,7 +39,7 @@ std::pair<string, string> decodeContext(const string & s)
         size_t index = s.find("!", 1);
         return std::pair<string, string>(string(s, index + 1), string(s, 1, index - 1));
     } else
-        return std::pair<string, string>(s.at(0) == '/' ? s: string(s, 1), "");
+        return std::pair<string, string>(s.at(0) == '/' ? s : string(s, 1), "");
 }
 
 
@@ -49,24 +49,38 @@ InvalidPathError::InvalidPathError(const Path & path) :
 void EvalState::realiseContext(const PathSet & context)
 {
     PathSet drvs;
+
     for (auto & i : context) {
         std::pair<string, string> decoded = decodeContext(i);
         Path ctx = decoded.first;
         assert(store->isStorePath(ctx));
         if (!store->isValidPath(ctx))
             throw InvalidPathError(ctx);
-        if (!decoded.second.empty() && nix::isDerivation(ctx))
+        if (!decoded.second.empty() && nix::isDerivation(ctx)) {
             drvs.insert(decoded.first + "!" + decoded.second);
+
+            /* Add the output of this derivation to the allowed
+               paths. */
+            if (allowedPaths) {
+                auto drv = store->derivationFromPath(decoded.first);
+                DerivationOutputs::iterator i = drv.outputs.find(decoded.second);
+                if (i == drv.outputs.end())
+                    throw Error("derivation '%s' does not have an output named '%s'", decoded.first, decoded.second);
+                allowedPaths->insert(i->second.path);
+            }
+        }
     }
-    if (!drvs.empty()) {
-        if (!settings.enableImportFromDerivation)
-            throw EvalError(format("attempted to realize '%1%' during evaluation but 'allow-import-from-derivation' is false") % *(drvs.begin()));
-        /* For performance, prefetch all substitute info. */
-        PathSet willBuild, willSubstitute, unknown;
-        unsigned long long downloadSize, narSize;
-        store->queryMissing(drvs, willBuild, willSubstitute, unknown, downloadSize, narSize);
-        store->buildPaths(drvs);
-    }
+
+    if (drvs.empty()) return;
+
+    if (!settings.enableImportFromDerivation)
+        throw EvalError(format("attempted to realize '%1%' during evaluation but 'allow-import-from-derivation' is false") % *(drvs.begin()));
+
+    /* For performance, prefetch all substitute info. */
+    PathSet willBuild, willSubstitute, unknown;
+    unsigned long long downloadSize, narSize;
+    store->queryMissing(drvs, willBuild, willSubstitute, unknown, downloadSize, narSize);
+    store->buildPaths(drvs);
 }
 
 
@@ -84,10 +98,10 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args
             % path % e.path % pos);
     }
 
-    path = state.checkSourcePath(path);
+    Path realPath = state.checkSourcePath(state.toRealPath(path, context));
 
     if (state.store->isStorePath(path) && state.store->isValidPath(path) && isDerivation(path)) {
-        Derivation drv = readDerivation(path);
+        Derivation drv = readDerivation(realPath);
         Value & w = *state.allocValue();
         state.mkAttrs(w, 3 + drv.outputs.size());
         Value * v2 = state.allocAttr(w, state.sDrvPath);
@@ -114,7 +128,7 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args
     } else {
         state.forceAttrs(*args[0]);
         if (args[0]->attrs->empty())
-            state.evalFile(path, v);
+            state.evalFile(realPath, v);
         else {
             Env * env = &state.allocEnv(args[0]->attrs->size());
             env->up = &state.baseEnv;
@@ -127,8 +141,8 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args
                 env->values[displ++] = attr.value;
             }
 
-            printTalkative("evaluating file '%1%'", path);
-            Expr * e = state.parseExprFromFile(resolveExprPath(path), staticEnv);
+            printTalkative("evaluating file '%1%'", realPath);
+            Expr * e = state.parseExprFromFile(resolveExprPath(realPath), staticEnv);
 
             e->eval(state, *env, v);
         }
@@ -439,7 +453,7 @@ static void prim_tryEval(EvalState & state, const Pos & pos, Value * * args, Val
 static void prim_getEnv(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     string name = state.forceStringNoCtx(*args[0], pos);
-    mkString(v, state.restricted ? "" : getEnv(name));
+    mkString(v, settings.restrictEval || settings.pureEval ? "" : getEnv(name));
 }
 
 
@@ -539,7 +553,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
 
     for (auto & i : args[0]->attrs->lexicographicOrder()) {
         if (i->name == state.sIgnoreNulls) continue;
-        string key = i->name;
+        const string & key = i->name;
         vomit("processing attribute '%1%'", key);
 
         auto handleHashMode = [&](const std::string & s) {
@@ -575,7 +589,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
 
             /* The `args' attribute is special: it supplies the
                command-line arguments to the builder. */
-            if (key == "args") {
+            if (i->name == state.sArgs) {
                 state.forceList(*i->value, pos);
                 for (unsigned int n = 0; n < i->value->listSize(); ++n) {
                     string s = state.coerceToString(posDrvName, *i->value->listElems()[n], context, true);
@@ -598,15 +612,13 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
                         drv.builder = state.forceString(*i->value, context, posDrvName);
                     else if (i->name == state.sSystem)
                         drv.platform = state.forceStringNoCtx(*i->value, posDrvName);
-                    else if (i->name == state.sName)
-                        drvName = state.forceStringNoCtx(*i->value, posDrvName);
-                    else if (key == "outputHash")
+                    else if (i->name == state.sOutputHash)
                         outputHash = state.forceStringNoCtx(*i->value, posDrvName);
-                    else if (key == "outputHashAlgo")
+                    else if (i->name == state.sOutputHashAlgo)
                         outputHashAlgo = state.forceStringNoCtx(*i->value, posDrvName);
-                    else if (key == "outputHashMode")
+                    else if (i->name == state.sOutputHashMode)
                         handleHashMode(state.forceStringNoCtx(*i->value, posDrvName));
-                    else if (key == "outputs") {
+                    else if (i->name == state.sOutputs) {
                         /* Require ‘outputs’ to be a list of strings. */
                         state.forceList(*i->value, posDrvName);
                         Strings ss;
@@ -620,14 +632,10 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
                     drv.env.emplace(key, s);
                     if (i->name == state.sBuilder) drv.builder = s;
                     else if (i->name == state.sSystem) drv.platform = s;
-                    else if (i->name == state.sName) {
-                        drvName = s;
-                        printMsg(lvlVomit, format("derivation name is '%1%'") % drvName);
-                    }
-                    else if (key == "outputHash") outputHash = s;
-                    else if (key == "outputHashAlgo") outputHashAlgo = s;
-                    else if (key == "outputHashMode") handleHashMode(s);
-                    else if (key == "outputs")
+                    else if (i->name == state.sOutputHash) outputHash = s;
+                    else if (i->name == state.sOutputHashAlgo) outputHashAlgo = s;
+                    else if (i->name == state.sOutputHashMode) handleHashMode(s);
+                    else if (i->name == state.sOutputs)
                         handleOutputs(tokenizeString<Strings>(s));
                 }
 
@@ -863,7 +871,7 @@ static void prim_readFile(EvalState & state, const Pos & pos, Value * * args, Va
         throw EvalError(format("cannot read '%1%', since path '%2%' is not valid, at %3%")
             % path % e.path % pos);
     }
-    string s = readFile(state.checkSourcePath(path));
+    string s = readFile(state.checkSourcePath(state.toRealPath(path, context)));
     if (s.find((char) 0) != string::npos)
         throw Error(format("the contents of the file '%1%' cannot be represented as a Nix string") % path);
     mkString(v, s.c_str());
@@ -1009,20 +1017,13 @@ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Valu
 }
 
 
-static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args, Value & v)
+static void addPath(EvalState & state, const Pos & pos, const string & name, const Path & path_,
+    Value * filterFun, bool recursive, const Hash & expectedHash, Value & v)
 {
-    PathSet context;
-    Path path = state.coerceToPath(pos, *args[1], context);
-    if (!context.empty())
-        throw EvalError(format("string '%1%' cannot refer to other paths, at %2%") % path % pos);
-
-    state.forceValue(*args[0]);
-    if (args[0]->type != tLambda)
-        throw TypeError(format("first argument in call to 'filterSource' is not a function but %1%, at %2%") % showType(*args[0]) % pos);
-
-    path = state.checkSourcePath(path);
-
-    PathFilter filter = [&](const Path & path) {
+    const auto path = settings.pureEval && expectedHash ?
+        path_ :
+        state.checkSourcePath(path_);
+    PathFilter filter = filterFun ? ([&](const Path & path) {
         auto st = lstat(path);
 
         /* Call the filter function.  The first argument is the path,
@@ -1031,7 +1032,7 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args
         mkString(arg1, path);
 
         Value fun2;
-        state.callFunction(*args[0], arg1, fun2, noPos);
+        state.callFunction(*filterFun, arg1, fun2, noPos);
 
         Value arg2;
         mkString(arg2,
@@ -1044,16 +1045,79 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args
         state.callFunction(fun2, arg2, res, noPos);
 
         return state.forceBool(res, pos);
-    };
+    }) : defaultPathFilter;
 
-    Path dstPath = settings.readOnlyMode
-        ? state.store->computeStorePathForPath(path, true, htSHA256, filter).first
-        : state.store->addToStore(baseNameOf(path), path, true, htSHA256, filter, state.repair);
+    Path expectedStorePath;
+    if (expectedHash) {
+        expectedStorePath =
+            state.store->makeFixedOutputPath(recursive, expectedHash, name);
+    }
+    Path dstPath;
+    if (!expectedHash || !state.store->isValidPath(expectedStorePath)) {
+        dstPath = settings.readOnlyMode
+            ? state.store->computeStorePathForPath(name, path, recursive, htSHA256, filter).first
+            : state.store->addToStore(name, path, recursive, htSHA256, filter, state.repair);
+        if (expectedHash && expectedStorePath != dstPath) {
+            throw Error(format("store path mismatch in (possibly filtered) path added from '%1%'") % path);
+        }
+    } else
+        dstPath = expectedStorePath;
 
     mkString(v, dstPath, {dstPath});
 }
 
 
+static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    PathSet context;
+    Path path = state.coerceToPath(pos, *args[1], context);
+    if (!context.empty())
+        throw EvalError(format("string '%1%' cannot refer to other paths, at %2%") % path % pos);
+
+    state.forceValue(*args[0]);
+    if (args[0]->type != tLambda)
+        throw TypeError(format("first argument in call to 'filterSource' is not a function but %1%, at %2%") % showType(*args[0]) % pos);
+
+    addPath(state, pos, baseNameOf(path), path, args[0], true, Hash(), v);
+}
+
+static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    state.forceAttrs(*args[0], pos);
+    Path path;
+    string name;
+    Value * filterFun = nullptr;
+    auto recursive = true;
+    Hash expectedHash;
+
+    for (auto & attr : *args[0]->attrs) {
+        const string & n(attr.name);
+        if (n == "path") {
+            PathSet context;
+            path = state.coerceToPath(*attr.pos, *attr.value, context);
+            if (!context.empty())
+                throw EvalError(format("string '%1%' cannot refer to other paths, at %2%") % path % *attr.pos);
+        } else if (attr.name == state.sName)
+            name = state.forceStringNoCtx(*attr.value, *attr.pos);
+        else if (n == "filter") {
+            state.forceValue(*attr.value);
+            filterFun = attr.value;
+        } else if (n == "recursive")
+            recursive = state.forceBool(*attr.value, *attr.pos);
+        else if (n == "sha256")
+            expectedHash = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256);
+        else
+            throw EvalError(format("unsupported argument '%1%' to 'addPath', at %2%") % attr.name % *attr.pos);
+    }
+    if (path.empty())
+        throw EvalError(format("'path' required, at %1%") % pos);
+    if (name.empty())
+        name = baseNameOf(path);
+
+    addPath(state, pos, name, path, filterFun, recursive, expectedHash, v);
+}
+
+
 /*************************************************************
  * Sets
  *************************************************************/
@@ -1068,8 +1132,11 @@ static void prim_attrNames(EvalState & state, const Pos & pos, Value * * args, V
     state.mkList(v, args[0]->attrs->size());
 
     size_t n = 0;
-    for (auto & i : args[0]->attrs->lexicographicOrder())
-        mkString(*(v.listElems()[n++] = state.allocValue()), i->name);
+    for (auto & i : *args[0]->attrs)
+        mkString(*(v.listElems()[n++] = state.allocValue()), i.name);
+
+    std::sort(v.listElems(), v.listElems() + n,
+              [](Value * v1, Value * v2) { return strcmp(v1->string.s, v2->string.s) < 0; });
 }
 
 
@@ -1891,6 +1958,26 @@ static void prim_compareVersions(EvalState & state, const Pos & pos, Value * * a
 }
 
 
+static void prim_splitVersion(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    string version = state.forceStringNoCtx(*args[0], pos);
+    auto iter = version.cbegin();
+    Strings components;
+    while (iter != version.cend()) {
+        auto component = nextComponent(iter, version.cend());
+        if (component.empty())
+            break;
+        components.emplace_back(std::move(component));
+    }
+    state.mkList(v, components.size());
+    unsigned int n = 0;
+    for (auto & component : components) {
+        auto listElem = v.listElems()[n++] = state.allocValue();
+        mkString(*listElem, std::move(component));
+    }
+}
+
+
 /*************************************************************
  * Networking
  *************************************************************/
@@ -1929,7 +2016,14 @@ void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
 
     state.checkURI(url);
 
+    if (settings.pureEval && !expectedHash)
+        throw Error("in pure evaluation mode, '%s' requires a 'sha256' argument", who);
+
     Path res = getDownloader()->downloadCached(state.store, url, unpack, name, expectedHash);
+
+    if (state.allowedPaths)
+        state.allowedPaths->insert(res);
+
     mkString(v, res, PathSet({res}));
 }
 
@@ -1981,11 +2075,24 @@ void EvalState::createBaseEnv()
     mkNull(v);
     addConstant("null", v);
 
-    mkInt(v, time(0));
-    addConstant("__currentTime", v);
+    auto vThrow = addPrimOp("throw", 1, prim_throw);
 
-    mkString(v, settings.thisSystem);
-    addConstant("__currentSystem", v);
+    auto addPurityError = [&](const std::string & name) {
+        Value * v2 = allocValue();
+        mkString(*v2, fmt("'%s' is not allowed in pure evaluation mode", name));
+        mkApp(v, *vThrow, *v2);
+        addConstant(name, v);
+    };
+
+    if (!settings.pureEval) {
+        mkInt(v, time(0));
+        addConstant("__currentTime", v);
+    }
+
+    if (!settings.pureEval) {
+        mkString(v, settings.thisSystem);
+        addConstant("__currentSystem", v);
+    }
 
     mkString(v, nixVersion);
     addConstant("__nixVersion", v);
@@ -2001,10 +2108,10 @@ void EvalState::createBaseEnv()
     addConstant("__langVersion", v);
 
     // Miscellaneous
-    addPrimOp("scopedImport", 2, prim_scopedImport);
+    auto vScopedImport = addPrimOp("scopedImport", 2, prim_scopedImport);
     Value * v2 = allocValue();
     mkAttrs(*v2, 0);
-    mkApp(v, *baseEnv.values[baseEnvDispl - 1], *v2);
+    mkApp(v, *vScopedImport, *v2);
     forceValue(v);
     addConstant("import", v);
     if (settings.enableNativeCode) {
@@ -2020,7 +2127,6 @@ void EvalState::createBaseEnv()
     addPrimOp("__isBool", 1, prim_isBool);
     addPrimOp("__genericClosure", 1, prim_genericClosure);
     addPrimOp("abort", 1, prim_abort);
-    addPrimOp("throw", 1, prim_throw);
     addPrimOp("__addErrorContext", 2, prim_addErrorContext);
     addPrimOp("__tryEval", 1, prim_tryEval);
     addPrimOp("__getEnv", 1, prim_getEnv);
@@ -2035,7 +2141,10 @@ void EvalState::createBaseEnv()
 
     // Paths
     addPrimOp("__toPath", 1, prim_toPath);
-    addPrimOp("__storePath", 1, prim_storePath);
+    if (settings.pureEval)
+        addPurityError("__storePath");
+    else
+        addPrimOp("__storePath", 1, prim_storePath);
     addPrimOp("__pathExists", 1, prim_pathExists);
     addPrimOp("baseNameOf", 1, prim_baseNameOf);
     addPrimOp("dirOf", 1, prim_dirOf);
@@ -2049,6 +2158,7 @@ void EvalState::createBaseEnv()
     addPrimOp("__fromJSON", 1, prim_fromJSON);
     addPrimOp("__toFile", 2, prim_toFile);
     addPrimOp("__filterSource", 2, prim_filterSource);
+    addPrimOp("__path", 1, prim_path);
 
     // Sets
     addPrimOp("__attrNames", 1, prim_attrNames);
@@ -2103,6 +2213,7 @@ void EvalState::createBaseEnv()
     // Versions
     addPrimOp("__parseDrvName", 1, prim_parseDrvName);
     addPrimOp("__compareVersions", 2, prim_compareVersions);
+    addPrimOp("__splitVersion", 1, prim_splitVersion);
 
     // Derivations
     addPrimOp("derivationStrict", 1, prim_derivationStrict);

src/libexpr/primops.hh

@@ -9,6 +9,9 @@ struct RegisterPrimOp
 {
     typedef std::vector<std::tuple<std::string, size_t, PrimOpFun>> PrimOps;
     static PrimOps * primOps;
+    /* You can register a constant by passing an arity of 0. fun
+       will get called during EvalState initialization, so there
+       may be primops not yet added and builtins is not yet sorted. */
     RegisterPrimOp(std::string name, size_t arity, PrimOpFun fun);
 };
 

src/libexpr/primops/fetchGit.cc

@@ -22,10 +22,15 @@ struct GitInfo
     uint64_t revCount = 0;
 };
 
+std::regex revRegex("^[0-9a-fA-F]{40}$");
+
 GitInfo exportGit(ref<Store> store, const std::string & uri,
     std::experimental::optional<std::string> ref, std::string rev,
     const std::string & name)
 {
+    if (settings.pureEval && rev == "")
+        throw Error("in pure evaluation mode, 'fetchGit' requires a Git revision");
+
     if (!ref && rev == "" && hasPrefix(uri, "/") && pathExists(uri + "/.git")) {
 
         bool clean = true;
@@ -76,11 +81,8 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
 
     if (!ref) ref = "master"s;
 
-    if (rev != "") {
-        std::regex revRegex("^[0-9a-fA-F]{40}$");
-        if (!std::regex_match(rev, revRegex))
-            throw Error("invalid Git revision '%s'", rev);
-    }
+    if (rev != "" && !std::regex_match(rev, revRegex))
+        throw Error("invalid Git revision '%s'", rev);
 
     Path cacheDir = getCacheDir() + "/nix/git";
 
@@ -231,6 +233,9 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va
     mkString(*state.allocAttr(v, state.symbols.create("shortRev")), gitInfo.shortRev);
     mkInt(*state.allocAttr(v, state.symbols.create("revCount")), gitInfo.revCount);
     v.attrs->sort();
+
+    if (state.allowedPaths)
+        state.allowedPaths->insert(gitInfo.storePath);
 }
 
 static RegisterPrimOp r("fetchGit", 1, prim_fetchGit);

src/libexpr/primops/fetchMercurial.cc

@@ -27,6 +27,9 @@ std::regex commitHashRegex("^[0-9a-fA-F]{40}$");
 HgInfo exportMercurial(ref<Store> store, const std::string & uri,
     std::string rev, const std::string & name)
 {
+    if (settings.pureEval && rev == "")
+        throw Error("in pure evaluation mode, 'fetchMercurial' requires a Mercurial revision");
+
     if (rev == "" && hasPrefix(uri, "/") && pathExists(uri + "/.hg")) {
 
         bool clean = runProgram("hg", true, { "status", "-R", uri, "--modified", "--added", "--removed" }) == "";
@@ -196,6 +199,9 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar
     mkString(*state.allocAttr(v, state.symbols.create("shortRev")), std::string(hgInfo.rev, 0, 12));
     mkInt(*state.allocAttr(v, state.symbols.create("revCount")), hgInfo.revCount);
     v.attrs->sort();
+
+    if (state.allowedPaths)
+        state.allowedPaths->insert(hgInfo.storePath);
 }
 
 static RegisterPrimOp r("fetchMercurial", 1, prim_fetchMercurial);

src/libmain/common-args.cc

@@ -37,6 +37,10 @@ MixCommonArgs::MixCommonArgs(const string & programName)
 
     std::string cat = "config";
     settings.convertToArgs(*this, cat);
+
+    // Backward compatibility hack: nix-env already had a --system flag.
+    if (programName == "nix-env") longFlags.erase("system");
+
     hiddenCategories.insert(cat);
 }
 

src/libmain/shared.cc

@@ -193,9 +193,6 @@ LegacyArgs::LegacyArgs(const std::string & programName,
     mkFlag(0, "readonly-mode", "do not write to the Nix store",
         &settings.readOnlyMode);
 
-    mkFlag(0, "show-trace", "show Nix expression stack trace in evaluation errors",
-        &settings.showTrace);
-
     mkFlag(0, "no-gc-warning", "disable warning about not using '--add-root'",
         &gcWarning, false);
 
@@ -265,6 +262,7 @@ void printVersion(const string & programName)
 void showManPage(const string & name)
 {
     restoreSignals();
+    setenv("MANPATH", settings.nixManDir.c_str(), 1);
     execlp("man", "man", name.c_str(), NULL);
     throw SysError(format("command 'man %1%' failed") % name.c_str());
 }

src/libmain/shared.hh

@@ -22,6 +22,7 @@ public:
 
 int handleExceptions(const string & programName, std::function<void()> fun);
 
+/* Don't forget to call initPlugins() after settings are initialized! */
 void initNix();
 
 void parseCmdLine(int argc, char * * argv,

src/libstore/binary-cache-store.cc

@@ -149,7 +149,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
     /* Compress the NAR. */
     narInfo->compression = compression;
     auto now1 = std::chrono::steady_clock::now();
-    auto narCompressed = compress(compression, *nar);
+    auto narCompressed = compress(compression, *nar, parallelCompression);
     auto now2 = std::chrono::steady_clock::now();
     narInfo->fileHash = hashString(htSHA256, *narCompressed);
     narInfo->fileSize = narCompressed->size();

src/libstore/binary-cache-store.hh

@@ -19,6 +19,8 @@ public:
     const Setting<bool> writeNARListing{this, false, "write-nar-listing", "whether to write a JSON file listing the files in each NAR"};
     const Setting<Path> secretKeyFile{this, "", "secret-key", "path to secret key used to sign the binary cache"};
     const Setting<Path> localNarCache{this, "", "local-nar-cache", "path to a local cache of NARs"};
+    const Setting<bool> parallelCompression{this, false, "parallel-compression",
+        "enable multi-threading compression, available for xz only currently"};
 
 private:
 

src/libstore/build.cc

@@ -6,6 +6,7 @@
 #include "archive.hh"
 #include "affinity.hh"
 #include "builtins.hh"
+#include "download.hh"
 #include "finally.hh"
 #include "compression.hh"
 #include "json.hh"
@@ -48,7 +49,9 @@
 #include <sys/param.h>
 #include <sys/mount.h>
 #include <sys/syscall.h>
+#if HAVE_SECCOMP
 #include <seccomp.h>
+#endif
 #define pivot_root(new_root, put_old) (syscall(SYS_pivot_root, new_root, put_old))
 #endif
 
@@ -1123,11 +1126,6 @@ void DerivationGoal::haveDerivation()
         return;
     }
 
-    /* Reject doing a hash build of anything other than a fixed-output
-       derivation. */
-    if (buildMode == bmHash && !drv->isFixedOutput())
-        throw Error("cannot do a hash build of non-fixed-output derivation '%1%'", drvPath);
-
     /* We are first going to try to create the invalid output paths
        through substitutes.  If that doesn't work, we'll build
        them. */
@@ -1319,9 +1317,7 @@ void DerivationGoal::inputsRealised()
     allPaths.insert(inputPaths.begin(), inputPaths.end());
 
     /* Is this a fixed-output derivation? */
-    fixedOutput = true;
-    for (auto & i : drv->outputs)
-        if (i.second.hash == "") fixedOutput = false;
+    fixedOutput = drv->isFixedOutput();
 
     /* Don't repeat fixed-output derivations since they're already
        verified by their output hash.*/
@@ -1341,19 +1337,6 @@ void DerivationGoal::tryToBuild()
 {
     trace("trying to build");
 
-    /* Check for the possibility that some other goal in this process
-       has locked the output since we checked in haveDerivation().
-       (It can't happen between here and the lockPaths() call below
-       because we're not allowing multi-threading.)  If so, put this
-       goal to sleep until another goal finishes, then try again. */
-    for (auto & i : drv->outputs)
-        if (pathIsLockedByMe(worker.store.toRealPath(i.second.path))) {
-            debug(format("putting derivation '%1%' to sleep because '%2%' is locked by another goal")
-                % drvPath % i.second.path);
-            worker.waitForAnyGoal(shared_from_this());
-            return;
-        }
-
     /* Obtain locks on all output paths.  The locks are automatically
        released when we exit this function or Nix crashes.  If we
        can't acquire the lock, then continue; hopefully some other
@@ -1777,6 +1760,19 @@ PathSet exportReferences(Store & store, PathSet storePaths)
     return paths;
 }
 
+static std::once_flag dns_resolve_flag;
+
+static void preloadNSS() {
+    /* builtin:fetchurl can trigger a DNS lookup, which with glibc can trigger a dynamic library load of
+       one of the glibc NSS libraries in a sandboxed child, which will fail unless the library's already
+       been loaded in the parent. So we force a download of an invalid URL to force the NSS machinery to
+       load its lookup libraries in the parent before any child gets a chance to. */
+    std::call_once(dns_resolve_flag, []() {
+        DownloadRequest request("http://this.pre-initializes.the.dns.resolvers.invalid");
+        request.tries = 1; // We only need to do it once, and this also suppresses an annoying warning
+        try { getDownloader()->download(request); } catch (...) {}
+    });
+}
 
 void DerivationGoal::startBuilder()
 {
@@ -1787,6 +1783,9 @@ void DerivationGoal::startBuilder()
             % drv->platform % settings.thisSystem % drvPath);
     }
 
+    if (drv->isBuiltin())
+        preloadNSS();
+
 #if __APPLE__
     additionalSandboxProfile = get(drv->env, "__sandboxProfile");
 #endif
@@ -1810,8 +1809,13 @@ void DerivationGoal::startBuilder()
             useChroot = !fixedOutput && get(drv->env, "__noChroot") != "1";
     }
 
-    if (worker.store.storeDir != worker.store.realStoreDir)
-        useChroot = true;
+    if (worker.store.storeDir != worker.store.realStoreDir) {
+        #if __linux__
+            useChroot = true;
+        #else
+            throw Error("building using a diverted store is not supported on this platform");
+        #endif
+    }
 
     /* If `build-users-group' is not empty, then we have to build as
        one of the members of that group. */
@@ -2469,7 +2473,7 @@ void setupSeccomp()
 {
 #if __linux__
     if (!settings.filterSyscalls) return;
-
+#if HAVE_SECCOMP
     scmp_filter_ctx ctx;
 
     if (!(ctx = seccomp_init(SCMP_ACT_ALLOW)))
@@ -2515,6 +2519,11 @@ void setupSeccomp()
 
     if (seccomp_load(ctx) != 0)
         throw SysError("unable to load seccomp BPF program");
+#else
+    throw Error(
+        "seccomp is not supported on this platform; "
+        "you can bypass this error by setting the option 'filter-syscalls' to false, but note that untrusted builds can then create setuid binaries!");
+#endif
 #endif
 }
 
@@ -2928,8 +2937,13 @@ void DerivationGoal::runChild()
         if (drv->isBuiltin()) {
             try {
                 logger = makeJSONLogger(*logger);
+
+                BasicDerivation drv2(*drv);
+                for (auto & e : drv2.env)
+                    e.second = rewriteStrings(e.second, inputRewrites);
+
                 if (drv->builder == "builtin:fetchurl")
-                    builtinFetchurl(*drv, netrcData);
+                    builtinFetchurl(drv2, netrcData);
                 else
                     throw Error(format("unsupported builtin function '%1%'") % string(drv->builder, 8));
                 _exit(0);
@@ -2992,6 +3006,8 @@ void DerivationGoal::registerOutputs()
     bool runDiffHook = settings.runDiffHook;
     bool keepPreviousRound = settings.keepFailed || runDiffHook;
 
+    std::exception_ptr delayedException;
+
     /* Check whether the output paths were created, and grep each
        output path to determine what other paths it references.  Also make all
        output paths read-only. */
@@ -3066,7 +3082,7 @@ void DerivationGoal::registerOutputs()
         /* Check that fixed-output derivations produced the right
            outputs (i.e., the content hash should match the specified
            hash). */
-        if (i.second.hash != "") {
+        if (fixedOutput) {
 
             bool recursive; Hash h;
             i.second.parseHashInfo(recursive, h);
@@ -3082,27 +3098,34 @@ void DerivationGoal::registerOutputs()
             /* Check the hash. In hash mode, move the path produced by
                the derivation to its content-addressed location. */
             Hash h2 = recursive ? hashPath(h.type, actualPath).first : hashFile(h.type, actualPath);
-            if (buildMode == bmHash) {
-                Path dest = worker.store.makeFixedOutputPath(recursive, h2, drv->env["name"]);
-                printError(format("build produced path '%1%' with %2% hash '%3%'")
-                    % dest % printHashType(h.type) % printHash16or32(h2));
-                if (worker.store.isValidPath(dest))
-                    return;
+
+            Path dest = worker.store.makeFixedOutputPath(recursive, h2, drv->env["name"]);
+
+            if (h != h2) {
+
+                /* Throw an error after registering the path as
+                   valid. */
+                delayedException = std::make_exception_ptr(
+                    BuildError("fixed-output derivation produced path '%s' with %s hash '%s' instead of the expected hash '%s'",
+                        dest, printHashType(h.type), printHash16or32(h2), printHash16or32(h)));
+
                 Path actualDest = worker.store.toRealPath(dest);
+
+                if (worker.store.isValidPath(dest))
+                    std::rethrow_exception(delayedException);
+
                 if (actualPath != actualDest) {
                     PathLocks outputLocks({actualDest});
                     deletePath(actualDest);
                     if (rename(actualPath.c_str(), actualDest.c_str()) == -1)
                         throw SysError(format("moving '%1%' to '%2%'") % actualPath % dest);
                 }
+
                 path = dest;
                 actualPath = actualDest;
-            } else {
-                if (h != h2)
-                    throw BuildError(
-                        format("output path '%1%' has %2% hash '%3%' when '%4%' was expected")
-                        % path % i.second.hashAlgo % printHash16or32(h2) % printHash16or32(h));
             }
+            else
+                assert(path == dest);
 
             info.ca = makeFixedOutputCA(recursive, h2);
         }
@@ -3279,6 +3302,11 @@ void DerivationGoal::registerOutputs()
        paths referenced by each of them.  If there are cycles in the
        outputs, this will fail. */
     worker.store.registerValidPaths(infos);
+
+    /* In case of a fixed-output derivation hash mismatch, throw an
+       exception now that we have registered the output as valid. */
+    if (delayedException)
+        std::rethrow_exception(delayedException);
 }
 
 
@@ -3394,7 +3422,7 @@ void DerivationGoal::flushLine()
     else {
         if (settings.verboseBuild &&
             (settings.printRepeatedBuilds || curRound == 1))
-            printError(filterANSIEscapes(currentLogLine, true));
+            printError(currentLogLine);
         else {
             logTail.push_back(currentLogLine);
             if (logTail.size() > settings.logLines) logTail.pop_front();
@@ -3636,7 +3664,7 @@ void SubstitutionGoal::tryNext()
     /* Update the total expected download size. */
     auto narInfo = std::dynamic_pointer_cast<const NarInfo>(info);
 
-    maintainExpectedNar = std::make_unique<MaintainCount<uint64_t>>(worker.expectedNarSize, narInfo->narSize);
+    maintainExpectedNar = std::make_unique<MaintainCount<uint64_t>>(worker.expectedNarSize, info->narSize);
 
     maintainExpectedDownload =
         narInfo && narInfo->fileSize
@@ -3650,9 +3678,12 @@ void SubstitutionGoal::tryNext()
     /* Bail out early if this substituter lacks a valid
        signature. LocalStore::addToStore() also checks for this, but
        only after we've downloaded the path. */
-    if (worker.store.requireSigs && !info->checkSignatures(worker.store, worker.store.publicKeys)) {
-        printInfo(format("warning: substituter '%s' does not have a valid signature for path '%s'")
-            % sub->getUri() % storePath);
+    if (worker.store.requireSigs
+        && !sub->isTrusted
+        && !info->checkSignatures(worker.store, worker.store.publicKeys))
+    {
+        printError("warning: substituter '%s' does not have a valid signature for path '%s'",
+            sub->getUri(), storePath);
         tryNext();
         return;
     }
@@ -3702,6 +3733,17 @@ void SubstitutionGoal::tryToRun()
         return;
     }
 
+    /* If the store path is already locked (probably by a
+       DerivationGoal), then put this goal to sleep. Note: we don't
+       acquire a lock here since that breaks addToStore(), so below we
+       handle an AlreadyLocked exception from addToStore(). The check
+       here is just an optimisation to prevent having to redo a
+       download due to a locked path. */
+    if (pathIsLockedByMe(worker.store.toRealPath(storePath))) {
+        worker.waitForAWhile(shared_from_this());
+        return;
+    }
+
     maintainRunningSubstitutions = std::make_unique<MaintainCount<uint64_t>>(worker.runningSubstitutions);
     worker.updateProgress();
 
@@ -3718,7 +3760,7 @@ void SubstitutionGoal::tryToRun()
             PushActivity pact(act.id);
 
             copyStorePath(ref<Store>(sub), ref<Store>(worker.store.shared_from_this()),
-                storePath, repair);
+                storePath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs);
 
             promise.set_value();
         } catch (...) {
@@ -3741,8 +3783,14 @@ void SubstitutionGoal::finished()
 
     try {
         promise.get_future().get();
+    } catch (AlreadyLocked & e) {
+        /* Probably a DerivationGoal is already building this store
+           path. Sleep for a while and try again. */
+        state = &SubstitutionGoal::init;
+        worker.waitForAWhile(shared_from_this());
+        return;
     } catch (Error & e) {
-        printInfo(e.msg());
+        printError(e.msg());
 
         /* Try the next substitute. */
         state = &SubstitutionGoal::tryNext;

src/libstore/download.cc

@@ -17,11 +17,13 @@
 
 #include <curl/curl.h>
 
-#include <queue>
-#include <iostream>
-#include <thread>
+#include <algorithm>
 #include <cmath>
+#include <cstring>
+#include <iostream>
+#include <queue>
 #include <random>
+#include <thread>
 
 using namespace std::string_literals;
 
@@ -91,6 +93,8 @@ struct CurlDownloader : public Downloader
         {
             if (!request.expectedETag.empty())
                 requestHeaders = curl_slist_append(requestHeaders, ("If-None-Match: " + request.expectedETag).c_str());
+            if (!request.mimeType.empty())
+                requestHeaders = curl_slist_append(requestHeaders, ("Content-Type: " + request.mimeType).c_str());
         }
 
         ~DownloadItem()
@@ -185,6 +189,22 @@ struct CurlDownloader : public Downloader
             return 0;
         }
 
+        size_t readOffset = 0;
+        int readCallback(char *buffer, size_t size, size_t nitems)
+        {
+            if (readOffset == request.data->length())
+                return 0;
+            auto count = std::min(size * nitems, request.data->length() - readOffset);
+            memcpy(buffer, request.data->data() + readOffset, count);
+            readOffset += count;
+            return count;
+        }
+
+        static int readCallbackWrapper(char *buffer, size_t size, size_t nitems, void * userp)
+        {
+            return ((DownloadItem *) userp)->readCallback(buffer, size, nitems);
+        }
+
         long lowSpeedTimeout = 300;
 
         void init()
@@ -225,6 +245,13 @@ struct CurlDownloader : public Downloader
             if (request.head)
                 curl_easy_setopt(req, CURLOPT_NOBODY, 1);
 
+            if (request.data) {
+                curl_easy_setopt(req, CURLOPT_UPLOAD, 1L);
+                curl_easy_setopt(req, CURLOPT_READFUNCTION, readCallbackWrapper);
+                curl_easy_setopt(req, CURLOPT_READDATA, this);
+                curl_easy_setopt(req, CURLOPT_INFILESIZE_LARGE, (curl_off_t) request.data->length());
+            }
+
             if (request.verifyTLS) {
                 if (settings.caFile != "")
                     curl_easy_setopt(req, CURLOPT_CAINFO, settings.caFile.c_str());
@@ -265,7 +292,7 @@ struct CurlDownloader : public Downloader
             }
 
             if (code == CURLE_OK &&
-                (httpStatus == 200 || httpStatus == 304 || httpStatus == 226 /* FTP */ || httpStatus == 0 /* other protocol */))
+                (httpStatus == 200 || httpStatus == 201 || httpStatus == 204 || httpStatus == 304 || httpStatus == 226 /* FTP */ || httpStatus == 0 /* other protocol */))
             {
                 result.cached = httpStatus == 304;
                 done = true;
@@ -303,6 +330,7 @@ struct CurlDownloader : public Downloader
                     // Don't bother retrying on certain cURL errors either
                     switch (code) {
                         case CURLE_FAILED_INIT:
+                        case CURLE_URL_MALFORMAT:
                         case CURLE_NOT_BUILT_IN:
                         case CURLE_REMOTE_ACCESS_DENIED:
                         case CURLE_FILE_COULDNT_READ_FILE:
@@ -311,10 +339,10 @@ struct CurlDownloader : public Downloader
                         case CURLE_BAD_FUNCTION_ARGUMENT:
                         case CURLE_INTERFACE_FAILED:
                         case CURLE_UNKNOWN_OPTION:
-                        err = Misc;
-                        break;
+                            err = Misc;
+                            break;
                         default: // Shut up warnings
-                        break;
+                            break;
                     }
                 }
 

src/libstore/download.hh

@@ -18,6 +18,8 @@ struct DownloadRequest
     unsigned int baseRetryTimeMs = 250;
     ActivityId parentAct;
     bool decompress = true;
+    std::shared_ptr<std::string> data;
+    std::string mimeType;
 
     DownloadRequest(const std::string & uri)
         : uri(uri), parentAct(curActivity) { }

src/libstore/gc.cc

@@ -324,10 +324,8 @@ Roots LocalStore::findRootsNoTemp()
 {
     Roots roots;
 
-    /* Process direct roots in {gcroots,manifests,profiles}. */
+    /* Process direct roots in {gcroots,profiles}. */
     findRoots(stateDir + "/" + gcRootsDir, DT_UNKNOWN, roots);
-    if (pathExists(stateDir + "/manifests"))
-        findRoots(stateDir + "/manifests", DT_UNKNOWN, roots);
     findRoots(stateDir + "/profiles", DT_UNKNOWN, roots);
 
     /* Add additional roots returned by the program specified by the

src/libstore/globals.cc

@@ -6,6 +6,7 @@
 #include <algorithm>
 #include <map>
 #include <thread>
+#include <dlfcn.h>
 
 
 namespace nix {
@@ -37,6 +38,7 @@ Settings::Settings()
     , nixConfDir(canonPath(getEnv("NIX_CONF_DIR", NIX_CONF_DIR)))
     , nixLibexecDir(canonPath(getEnv("NIX_LIBEXEC_DIR", NIX_LIBEXEC_DIR)))
     , nixBinDir(canonPath(getEnv("NIX_BIN_DIR", NIX_BIN_DIR)))
+    , nixManDir(canonPath(NIX_MAN_DIR))
     , nixDaemonSocketFile(canonPath(nixStateDir + DEFAULT_SOCKET_PATH))
 {
     buildUsersGroup = getuid() == 0 ? "nixbld" : "";
@@ -137,4 +139,46 @@ void MaxBuildJobsSetting::set(const std::string & str)
         throw UsageError("configuration setting '%s' should be 'auto' or an integer", name);
 }
 
+
+void initPlugins()
+{
+    for (const auto & pluginFile : settings.pluginFiles.get()) {
+        Paths pluginFiles;
+        try {
+            auto ents = readDirectory(pluginFile);
+            for (const auto & ent : ents)
+                pluginFiles.emplace_back(pluginFile + "/" + ent.name);
+        } catch (SysError & e) {
+            if (e.errNo != ENOTDIR)
+                throw;
+            pluginFiles.emplace_back(pluginFile);
+        }
+        for (const auto & file : pluginFiles) {
+            /* handle is purposefully leaked as there may be state in the
+               DSO needed by the action of the plugin. */
+            void *handle =
+                dlopen(file.c_str(), RTLD_LAZY | RTLD_LOCAL);
+            if (!handle)
+                throw Error("could not dynamically open plugin file '%s%': %s%", file, dlerror());
+        }
+    }
+    /* We handle settings registrations here, since plugins can add settings */
+    if (RegisterSetting::settingRegistrations) {
+        for (auto & registration : *RegisterSetting::settingRegistrations)
+            settings.addSetting(registration);
+        delete RegisterSetting::settingRegistrations;
+    }
+    settings.handleUnknownSettings();
+}
+
+RegisterSetting::SettingRegistrations * RegisterSetting::settingRegistrations;
+
+RegisterSetting::RegisterSetting(AbstractSetting * s)
+{
+    if (!settingRegistrations)
+        settingRegistrations = new SettingRegistrations;
+    settingRegistrations->emplace_back(s);
+}
+
+
 }

src/libstore/globals.hh

@@ -82,6 +82,9 @@ public:
     /* The directory where the main programs are stored. */
     Path nixBinDir;
 
+    /* The directory where the man pages are stored. */
+    Path nixManDir;
+
     /* File name of the socket the daemon listens to.  */
     Path nixDaemonSocketFile;
 
@@ -211,7 +214,8 @@ public:
     bool lockCPU;
 
     /* Whether to show a stack trace if Nix evaluation fails. */
-    bool showTrace = false;
+    Setting<bool> showTrace{this, false, "show-trace",
+        "Whether to show a stack trace on evaluation errors."};
 
     Setting<bool> enableNativeCode{this, false, "allow-unsafe-native-code-during-evaluation",
         "Whether builtin functions that allow executing native code should be enabled."};
@@ -232,6 +236,9 @@ public:
         "Whether to restrict file system access to paths in $NIX_PATH, "
         "and network access to the URI prefixes listed in 'allowed-uris'."};
 
+    Setting<bool> pureEval{this, false, "pure-eval",
+        "Whether to restrict file system and network access to files specified by cryptographic hash."};
+
     Setting<size_t> buildRepeat{this, 0, "repeat",
         "The number of times to repeat a build in order to verify determinism.",
         {"build-repeat"}};
@@ -283,10 +290,7 @@ public:
     Setting<unsigned int> tarballTtl{this, 60 * 60, "tarball-ttl",
         "How soon to expire files fetched by builtins.fetchTarball and builtins.fetchurl."};
 
-    Setting<std::string> signedBinaryCaches{this, "*", "signed-binary-caches",
-        "Obsolete."};
-
-    Setting<bool> requireSigs{this, signedBinaryCaches == "*", "require-sigs",
+    Setting<bool> requireSigs{this, true, "require-sigs",
         "Whether to check that any non-content-addressed path added to the "
         "Nix store has a valid signature (that is, one signed using a key "
         "listed in 'trusted-public-keys'."};
@@ -366,14 +370,28 @@ public:
 
     Setting<Strings> allowedUris{this, {}, "allowed-uris",
         "Prefixes of URIs that builtin functions such as fetchurl and fetchGit are allowed to fetch."};
+
+    Setting<Paths> pluginFiles{this, {}, "plugin-files",
+        "Plugins to dynamically load at nix initialization time."};
 };
 
 
 // FIXME: don't use a global variable.
 extern Settings settings;
 
+/* This should be called after settings are initialized, but before
+   anything else */
+void initPlugins();
+
 
 extern const string nixVersion;
 
+struct RegisterSetting
+{
+    typedef std::vector<AbstractSetting *> SettingRegistrations;
+    static SettingRegistrations * settingRegistrations;
+    RegisterSetting(AbstractSetting * s);
+};
+
 
 }

src/libstore/http-binary-cache-store.cc

@@ -38,7 +38,7 @@ public:
             try {
                 BinaryCacheStore::init();
             } catch (UploadToHTTP &) {
-                throw Error(format("'%s' does not appear to be a binary cache") % cacheUri);
+                throw Error("'%s' does not appear to be a binary cache", cacheUri);
             }
             diskCache->createCache(cacheUri, storeDir, wantMassQuery_, priority);
         }
@@ -67,7 +67,14 @@ protected:
         const std::string & data,
         const std::string & mimeType) override
     {
-        throw UploadToHTTP("uploading to an HTTP binary cache is not supported");
+        auto req = DownloadRequest(cacheUri + "/" + path);
+        req.data = std::make_shared<string>(data); // FIXME: inefficient
+        req.mimeType = mimeType;
+        try {
+            getDownloader()->download(req);
+        } catch (DownloadError & e) {
+            throw UploadToHTTP(format("uploading to HTTP binary cache at %1% not supported: %2%") % cacheUri % e.msg());
+        }
     }
 
     void getFile(const std::string & path,

src/libstore/local-store.cc

@@ -992,8 +992,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, const ref<std::string> &
         /* Lock the output path.  But don't lock if we're being called
            from a build hook (whose parent process already acquired a
            lock on this path). */
-        Strings locksHeld = tokenizeString<Strings>(getEnv("NIX_HELD_LOCKS"));
-        if (find(locksHeld.begin(), locksHeld.end(), info.path) == locksHeld.end())
+        if (!locksHeld.count(info.path))
             outputLock.lockPaths({realPath});
 
         if (repair || !isValidPath(info.path)) {

src/libstore/local-store.hh

@@ -19,7 +19,7 @@ namespace nix {
 /* Nix store and database schema version.  Version 1 (or 0) was Nix <=
    0.7.  Version 2 was Nix 0.8 and 0.9.  Version 3 is Nix 0.10.
    Version 4 is Nix 0.11.  Version 5 is Nix 0.12-0.16.  Version 6 is
-   Nix 1.0.  Version 7 is Nix 1.3. Version 10 is 1.12. */
+   Nix 1.0.  Version 7 is Nix 1.3. Version 10 is 2.0. */
 const int nixSchemaVersion = 10;
 
 
@@ -104,6 +104,9 @@ private:
 
 public:
 
+    // Hack for build-remote.cc.
+    PathSet locksHeld = tokenizeString<PathSet>(getEnv("NIX_HELD_LOCKS"));
+
     /* Initialise the local store, upgrading the schema if
        necessary. */
     LocalStore(const Params & params);

src/libstore/local.mk

@@ -9,6 +9,9 @@ libstore_SOURCES := $(wildcard $(d)/*.cc)
 libstore_LIBS = libutil libformat
 
 libstore_LDFLAGS = $(SQLITE3_LIBS) -lbz2 $(LIBCURL_LIBS) $(SODIUM_LIBS) -pthread
+ifneq ($(OS), FreeBSD)
+ libstore_LDFLAGS += -ldl
+endif
 
 libstore_FILES = sandbox-defaults.sb sandbox-minimal.sb sandbox-network.sb
 
@@ -22,7 +25,7 @@ ifeq ($(OS), SunOS)
 	libstore_LDFLAGS += -lsocket
 endif
 
-ifeq ($(OS), Linux)
+ifeq ($(HAVE_SECCOMP), 1)
 	libstore_LDFLAGS += -lseccomp
 endif
 
@@ -35,6 +38,7 @@ libstore_CXXFLAGS = \
  -DNIX_CONF_DIR=\"$(sysconfdir)/nix\" \
  -DNIX_LIBEXEC_DIR=\"$(libexecdir)\" \
  -DNIX_BIN_DIR=\"$(bindir)\" \
+ -DNIX_MAN_DIR=\"$(mandir)\" \
  -DSANDBOX_SHELL="\"$(sandbox_shell)\"" \
  -DLSOF=\"$(lsof)\"
 

src/libstore/pathlocks.cc

@@ -113,8 +113,10 @@ bool PathLocks::lockPaths(const PathSet & _paths,
 
         {
             auto lockedPaths(lockedPaths_.lock());
-            if (lockedPaths->count(lockPath))
-                throw Error("deadlock: trying to re-acquire self-held lock '%s'", lockPath);
+            if (lockedPaths->count(lockPath)) {
+                if (!wait) return false;
+                throw AlreadyLocked("deadlock: trying to re-acquire self-held lock '%s'", lockPath);
+            }
             lockedPaths->insert(lockPath);
         }
 

src/libstore/pathlocks.hh

@@ -2,10 +2,8 @@
 
 #include "util.hh"
 
-
 namespace nix {
 
-
 /* Open (possibly create) a lock file and return the file descriptor.
    -1 is returned if create is false and the lock could not be opened
    because it doesn't exist.  Any other error throws an exception. */
@@ -18,6 +16,7 @@ enum LockType { ltRead, ltWrite, ltNone };
 
 bool lockFile(int fd, LockType lockType, bool wait);
 
+MakeError(AlreadyLocked, Error);
 
 class PathLocks
 {
@@ -38,9 +37,6 @@ public:
     void setDeletion(bool deletePaths);
 };
 
-
-// FIXME: not thread-safe!
 bool pathIsLockedByMe(const Path & path);
 
-
 }

src/libstore/remote-store.cc

@@ -78,9 +78,22 @@ UDSRemoteStore::UDSRemoteStore(const Params & params)
 }
 
 
+UDSRemoteStore::UDSRemoteStore(std::string socket_path, const Params & params)
+    : Store(params)
+    , LocalFSStore(params)
+    , RemoteStore(params)
+    , path(socket_path)
+{
+}
+
+
 std::string UDSRemoteStore::getUri()
 {
-    return "daemon";
+    if (path) {
+        return std::string("unix://") + *path;
+    } else {
+        return "daemon";
+    }
 }
 
 
@@ -98,7 +111,7 @@ ref<RemoteStore::Connection> UDSRemoteStore::openConnection()
         throw SysError("cannot create Unix domain socket");
     closeOnExec(conn->fd.get());
 
-    string socketPath = settings.nixDaemonSocketFile;
+    string socketPath = path ? *path : settings.nixDaemonSocketFile;
 
     struct sockaddr_un addr;
     addr.sun_family = AF_UNIX;
@@ -721,5 +734,14 @@ void RemoteStore::Connection::processStderr(Sink * sink, Source * source)
     }
 }
 
+static std::string uriScheme = "unix://";
+
+static RegisterStoreImplementation regStore([](
+    const std::string & uri, const Store::Params & params)
+    -> std::shared_ptr<Store>
+{
+    if (std::string(uri, 0, uriScheme.size()) != uriScheme) return 0;
+    return std::make_shared<UDSRemoteStore>(std::string(uri, uriScheme.size()), params);
+});
 
 }

src/libstore/remote-store.hh

@@ -134,6 +134,7 @@ class UDSRemoteStore : public LocalFSStore, public RemoteStore
 public:
 
     UDSRemoteStore(const Params & params);
+    UDSRemoteStore(std::string path, const Params & params);
 
     std::string getUri() override;
 
@@ -145,6 +146,7 @@ private:
     };
 
     ref<RemoteStore::Connection> openConnection() override;
+    std::experimental::optional<std::string> path;
 };
 
 

src/libstore/store-api.cc

@@ -222,11 +222,10 @@ Path Store::makeTextPath(const string & name, const Hash & hash,
 }
 
 
-std::pair<Path, Hash> Store::computeStorePathForPath(const Path & srcPath,
-    bool recursive, HashType hashAlgo, PathFilter & filter) const
+std::pair<Path, Hash> Store::computeStorePathForPath(const string & name,
+    const Path & srcPath, bool recursive, HashType hashAlgo, PathFilter & filter) const
 {
     Hash h = recursive ? hashPath(hashAlgo, srcPath, filter).first : hashFile(hashAlgo, srcPath);
-    string name = baseNameOf(srcPath);
     Path dstPath = makeFixedOutputPath(recursive, h, name);
     return std::pair<Path, Hash>(dstPath, h);
 }
@@ -840,7 +839,7 @@ ref<Store> openStore(const std::string & uri_,
     for (auto fun : *RegisterStoreImplementation::implementations) {
         auto store = fun(uri, params);
         if (store) {
-            store->warnUnknownSettings();
+            store->handleUnknownSettings();
             return ref<Store>(store);
         }
     }
@@ -897,7 +896,11 @@ std::list<ref<Store>> getDefaultSubstituters()
         auto addStore = [&](const std::string & uri) {
             if (done.count(uri)) return;
             done.insert(uri);
-            stores.push_back(openStore(uri));
+            try {
+                stores.push_back(openStore(uri));
+            } catch (Error & e) {
+                printError("warning: %s", e.what());
+            }
         };
 
         for (auto uri : settings.substituters.get())

src/libstore/store-api.hh

@@ -192,7 +192,7 @@ struct ValidPathInfo
 typedef list<ValidPathInfo> ValidPathInfos;
 
 
-enum BuildMode { bmNormal, bmRepair, bmCheck, bmHash };
+enum BuildMode { bmNormal, bmRepair, bmCheck };
 
 
 struct BuildResult
@@ -248,6 +248,8 @@ public:
 
     const Setting<int> pathInfoCacheSize{this, 65536, "path-info-cache-size", "size of the in-memory store path information cache"};
 
+    const Setting<bool> isTrusted{this, false, "trusted", "whether paths from this store can be used as substitutes even when they lack trusted signatures"};
+
 protected:
 
     struct State
@@ -305,9 +307,9 @@ public:
     /* This is the preparatory part of addToStore(); it computes the
        store path to which srcPath is to be copied.  Returns the store
        path and the cryptographic hash of the contents of srcPath. */
-    std::pair<Path, Hash> computeStorePathForPath(const Path & srcPath,
-        bool recursive = true, HashType hashAlgo = htSHA256,
-        PathFilter & filter = defaultPathFilter) const;
+    std::pair<Path, Hash> computeStorePathForPath(const string & name,
+        const Path & srcPath, bool recursive = true,
+        HashType hashAlgo = htSHA256, PathFilter & filter = defaultPathFilter) const;
 
     /* Preparatory part of addTextToStore().
 
@@ -597,6 +599,11 @@ public:
        "nix-cache-info" file. Lower value means higher priority. */
     virtual int getPriority() { return 0; }
 
+    virtual Path toRealPath(const Path & storePath)
+    {
+        return storePath;
+    }
+
 protected:
 
     Stats stats;
@@ -639,9 +646,10 @@ public:
 
     virtual Path getRealStoreDir() { return storeDir; }
 
-    Path toRealPath(const Path & storePath)
+    Path toRealPath(const Path & storePath) override
     {
-        return getRealStoreDir() + "/" + baseNameOf(storePath);
+        assert(isInStore(storePath));
+        return getRealStoreDir() + "/" + std::string(storePath, storeDir.size() + 1);
     }
 
     std::shared_ptr<std::string> getBuildLog(const Path & path) override;
@@ -699,6 +707,9 @@ void removeTempRoots();
    * ‘daemon’: The Nix store accessed via a Unix domain socket
      connection to nix-daemon.
 
+   * ‘unix://<path>’: The Nix store accessed via a Unix domain socket
+     connection to nix-daemon, with the socket located at <path>.
+
    * ‘auto’ or ‘’: Equivalent to ‘local’ or ‘daemon’ depending on
      whether the user has write access to the local Nix
      store/database.

src/libutil/compression.cc

@@ -1,6 +1,7 @@
 #include "compression.hh"
 #include "util.hh"
 #include "finally.hh"
+#include "logging.hh"
 
 #include <lzma.h>
 #include <bzlib.h>
@@ -151,10 +152,10 @@ static ref<std::string> decompressBrotli(const std::string & in)
 #endif // HAVE_BROTLI
 }
 
-ref<std::string> compress(const std::string & method, const std::string & in)
+ref<std::string> compress(const std::string & method, const std::string & in, const bool parallel)
 {
     StringSink ssink;
-    auto sink = makeCompressionSink(method, ssink);
+    auto sink = makeCompressionSink(method, ssink, parallel);
     (*sink)(in);
     sink->finish();
     return ssink.s;
@@ -189,10 +190,9 @@ struct XzSink : CompressionSink
     lzma_stream strm = LZMA_STREAM_INIT;
     bool finished = false;
 
-    XzSink(Sink & nextSink) : nextSink(nextSink)
-    {
-        lzma_ret ret = lzma_easy_encoder(
-            &strm, 6, LZMA_CHECK_CRC64);
+    template <typename F>
+    XzSink(Sink & nextSink, F&& initEncoder) : nextSink(nextSink) {
+        lzma_ret ret = initEncoder();
         if (ret != LZMA_OK)
             throw CompressionError("unable to initialise lzma encoder");
         // FIXME: apply the x86 BCJ filter?
@@ -200,6 +200,9 @@ struct XzSink : CompressionSink
         strm.next_out = outbuf;
         strm.avail_out = sizeof(outbuf);
     }
+    XzSink(Sink & nextSink) : XzSink(nextSink, [this]() {
+        return lzma_easy_encoder(&strm, 6, LZMA_CHECK_CRC64);
+    }) {}
 
     ~XzSink()
     {
@@ -253,6 +256,27 @@ struct XzSink : CompressionSink
     }
 };
 
+#ifdef HAVE_LZMA_MT
+struct ParallelXzSink : public XzSink
+{
+  ParallelXzSink(Sink &nextSink) : XzSink(nextSink, [this]() {
+        lzma_mt mt_options = {};
+        mt_options.flags = 0;
+        mt_options.timeout = 300; // Using the same setting as the xz cmd line
+        mt_options.preset = LZMA_PRESET_DEFAULT;
+        mt_options.filters = NULL;
+        mt_options.check = LZMA_CHECK_CRC64;
+        mt_options.threads = lzma_cputhreads();
+        mt_options.block_size = 0;
+        if (mt_options.threads == 0)
+            mt_options.threads = 1;
+        // FIXME: maybe use lzma_stream_encoder_mt_memusage() to control the
+        // number of threads.
+        return lzma_stream_encoder_mt(&strm, &mt_options);
+  }) {}
+};
+#endif
+
 struct BzipSink : CompressionSink
 {
     Sink & nextSink;
@@ -449,8 +473,16 @@ struct BrotliSink : CompressionSink
 };
 #endif // HAVE_BROTLI
 
-ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink)
+ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel)
 {
+    if (parallel) {
+#ifdef HAVE_LZMA_MT
+        if (method == "xz")
+            return make_ref<ParallelXzSink>(nextSink);
+#endif
+        printMsg(lvlError, format("Warning: parallel compression requested but not supported for method '%1%', falling back to single-threaded compression") % method);
+    }
+
     if (method == "none")
         return make_ref<NoneSink>(nextSink);
     else if (method == "xz")

src/libutil/compression.hh

@@ -8,7 +8,7 @@
 
 namespace nix {
 
-ref<std::string> compress(const std::string & method, const std::string & in);
+ref<std::string> compress(const std::string & method, const std::string & in, const bool parallel = false);
 
 ref<std::string> decompress(const std::string & method, const std::string & in);
 
@@ -17,7 +17,7 @@ struct CompressionSink : BufferedSink
     virtual void finish() = 0;
 };
 
-ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink);
+ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel = false);
 
 MakeError(UnknownCompressionMethod, Error);
 

src/libutil/config.cc

@@ -7,10 +7,12 @@ namespace nix {
 void Config::set(const std::string & name, const std::string & value)
 {
     auto i = _settings.find(name);
-    if (i == _settings.end())
-        throw UsageError("unknown setting '%s'", name);
-    i->second.setting->set(value);
-    i->second.setting->overriden = true;
+    if (i == _settings.end()) {
+        extras.emplace(name, value);
+    } else {
+        i->second.setting->set(value);
+        i->second.setting->overriden = true;
+    }
 }
 
 void Config::addSetting(AbstractSetting * setting)
@@ -21,34 +23,34 @@ void Config::addSetting(AbstractSetting * setting)
 
     bool set = false;
 
-    auto i = initials.find(setting->name);
-    if (i != initials.end()) {
+    auto i = extras.find(setting->name);
+    if (i != extras.end()) {
         setting->set(i->second);
         setting->overriden = true;
-        initials.erase(i);
+        extras.erase(i);
         set = true;
     }
 
     for (auto & alias : setting->aliases) {
-        auto i = initials.find(alias);
-        if (i != initials.end()) {
+        auto i = extras.find(alias);
+        if (i != extras.end()) {
             if (set)
                 warn("setting '%s' is set, but it's an alias of '%s' which is also set",
                     alias, setting->name);
             else {
                 setting->set(i->second);
                 setting->overriden = true;
-                initials.erase(i);
+                extras.erase(i);
                 set = true;
             }
         }
     }
 }
 
-void Config::warnUnknownSettings()
+void Config::handleUnknownSettings()
 {
-    for (auto & i : initials)
-        warn("unknown setting '%s'", i.first);
+    for (auto & s : extras)
+        warn("unknown setting '%s'", s.first);
 }
 
 StringMap Config::getSettings(bool overridenOnly)
@@ -60,7 +62,7 @@ StringMap Config::getSettings(bool overridenOnly)
     return res;
 }
 
-void Config::applyConfigFile(const Path & path, bool fatal)
+void Config::applyConfigFile(const Path & path)
 {
     try {
         string contents = readFile(path);
@@ -80,7 +82,31 @@ void Config::applyConfigFile(const Path & path, bool fatal)
             vector<string> tokens = tokenizeString<vector<string> >(line);
             if (tokens.empty()) continue;
 
-            if (tokens.size() < 2 || tokens[1] != "=")
+            if (tokens.size() < 2)
+                throw UsageError("illegal configuration line '%1%' in '%2%'", line, path);
+
+            auto include = false;
+            auto ignoreMissing = false;
+            if (tokens[0] == "include")
+                include = true;
+            else if (tokens[0] == "!include") {
+                include = true;
+                ignoreMissing = true;
+            }
+
+            if (include) {
+                if (tokens.size() != 2)
+                    throw UsageError("illegal configuration line '%1%' in '%2%'", line, path);
+                auto p = absPath(tokens[1], dirOf(path));
+                if (pathExists(p)) {
+                    applyConfigFile(p);
+                } else if (!ignoreMissing) {
+                    throw Error("file '%1%' included from '%2%' not found", p, path);
+                }
+                continue;
+            }
+
+            if (tokens[1] != "=")
                 throw UsageError("illegal configuration line '%1%' in '%2%'", line, path);
 
             string name = tokens[0];
@@ -88,12 +114,7 @@ void Config::applyConfigFile(const Path & path, bool fatal)
             vector<string>::iterator i = tokens.begin();
             advance(i, 2);
 
-            try {
-                set(name, concatStringsSep(" ", Strings(i, tokens.end()))); // FIXME: slow
-            } catch (UsageError & e) {
-                if (fatal) throw;
-                warn("in configuration file '%s': %s", path, e.what());
-            }
+            set(name, concatStringsSep(" ", Strings(i, tokens.end()))); // FIXME: slow
         };
     } catch (SysError &) { }
 }

src/libutil/config.hh

@@ -48,25 +48,25 @@ private:
 
     Settings _settings;
 
-    StringMap initials;
+    StringMap extras;
 
 public:
 
     Config(const StringMap & initials)
-        : initials(initials)
+        : extras(initials)
     { }
 
     void set(const std::string & name, const std::string & value);
 
     void addSetting(AbstractSetting * setting);
 
-    void warnUnknownSettings();
+    void handleUnknownSettings();
 
     StringMap getSettings(bool overridenOnly = false);
 
     const Settings & _getSettings() { return _settings; }
 
-    void applyConfigFile(const Path & path, bool fatal = false);
+    void applyConfigFile(const Path & path);
 
     void resetOverriden();
 

src/libutil/hash.cc

@@ -189,7 +189,8 @@ Hash::Hash(const std::string & s, HashType type)
 
     else if (size == base64Len()) {
         auto d = base64Decode(std::string(s, pos));
-        assert(d.size() == hashSize);
+        if (d.size() != hashSize)
+            throw BadHash("invalid base-64 hash '%s'", s);
         memcpy(hash, d.data(), hashSize);
     }
 

src/libutil/logging.cc

@@ -44,7 +44,7 @@ public:
             prefix = std::string("<") + c + ">";
         }
 
-        writeToStderr(prefix + (tty ? fs.s : filterANSIEscapes(fs.s)) + "\n");
+        writeToStderr(prefix + filterANSIEscapes(fs.s) + "\n");
     }
 
     void startActivity(ActivityId act, Verbosity lvl, ActivityType type,

src/libutil/monitor-fd.hh

@@ -21,13 +21,29 @@ public:
     MonitorFdHup(int fd)
     {
         thread = std::thread([fd]() {
-            /* Wait indefinitely until a POLLHUP occurs. */
-            struct pollfd fds[1];
-            fds[0].fd = fd;
-            fds[0].events = 0;
-            if (poll(fds, 1, -1) == -1) abort(); // can't happen
-            assert(fds[0].revents & POLLHUP);
-            triggerInterrupt();
+            while (true) {
+              /* Wait indefinitely until a POLLHUP occurs. */
+              struct pollfd fds[1];
+              fds[0].fd = fd;
+              /* This shouldn't be necessary, but macOS doesn't seem to
+                 like a zeroed out events field.
+                 See rdar://37537852.
+              */
+              fds[0].events = POLLHUP;
+              auto count = poll(fds, 1, -1);
+              if (count == -1) abort(); // can't happen
+              /* This shouldn't happen, but can on macOS due to a bug.
+                 See rdar://37550628.
+
+                 This may eventually need a delay or further
+                 coordination with the main thread if spinning proves
+                 too harmful.
+               */
+              if (count == 0) continue;
+              assert(fds[0].revents & POLLHUP);
+              triggerInterrupt();
+              break;
+            }
         });
     };
 

src/libutil/util.cc

@@ -192,6 +192,12 @@ bool isInDir(const Path & path, const Path & dir)
 }
 
 
+bool isDirOrInDir(const Path & path, const Path & dir)
+{
+    return path == dir or isInDir(path, dir);
+}
+
+
 struct stat lstat(const Path & path)
 {
     struct stat st;
@@ -1172,36 +1178,51 @@ void ignoreException()
 }
 
 
-string filterANSIEscapes(const string & s, bool nixOnly)
+std::string filterANSIEscapes(const std::string & s, unsigned int width)
 {
-    string t, r;
-    enum { stTop, stEscape, stCSI } state = stTop;
-    for (auto c : s) {
-        if (state == stTop) {
-            if (c == '\e') {
-                state = stEscape;
-                r = c;
-            } else
-                t += c;
-        } else if (state == stEscape) {
-            r += c;
-            if (c == '[')
-                state = stCSI;
-            else {
-                t += r;
-                state = stTop;
+    std::string t, e;
+    size_t w = 0;
+    auto i = s.begin();
+
+    while (w < (size_t) width && i != s.end()) {
+
+        if (*i == '\e') {
+            std::string e;
+            e += *i++;
+            char last = 0;
+
+            if (i != s.end() && *i == '[') {
+                e += *i++;
+                // eat parameter bytes
+                while (i != s.end() && *i >= 0x30 && *i <= 0x3f) e += *i++;
+                // eat intermediate bytes
+                while (i != s.end() && *i >= 0x20 && *i <= 0x2f) e += *i++;
+                // eat final byte
+                if (i != s.end() && *i >= 0x40 && *i <= 0x7e) e += last = *i++;
+            } else {
+                if (i != s.end() && *i >= 0x40 && *i <= 0x5f) e += *i++;
             }
-        } else {
-            r += c;
-            if (c >= 0x40 && c <= 0x7e) {
-                if (nixOnly && (c != 'p' && c != 'q' && c != 's' && c != 'a' && c != 'b'))
-                    t += r;
-                state = stTop;
-                r.clear();
+
+            if (last == 'm')
+                t += e;
+        }
+
+        else if (*i == '\t') {
+            i++; t += ' '; w++;
+            while (w < (size_t) width && w % 8) {
+                t += ' '; w++;
             }
         }
+
+        else if (*i == '\r')
+            // do nothing for now
+            i++;
+
+        else {
+            t += *i++; w++;
+        }
     }
-    t += r;
+
     return t;
 }
 

src/libutil/util.hh

@@ -53,10 +53,12 @@ Path dirOf(const Path & path);
    following the final `/'. */
 string baseNameOf(const Path & path);
 
-/* Check whether a given path is a descendant of the given
-   directory. */
+/* Check whether 'path' is a descendant of 'dir'. */
 bool isInDir(const Path & path, const Path & dir);
 
+/* Check whether 'path' is equal to 'dir' or a descendant of 'dir'. */
+bool isDirOrInDir(const Path & path, const Path & dir);
+
 /* Get status of `path'. */
 struct stat lstat(const Path & path);
 
@@ -386,10 +388,12 @@ void ignoreException();
 #define ANSI_BLUE "\e[34;1m"
 
 
-/* Filter out ANSI escape codes from the given string. If ‘nixOnly’ is
-   set, only filter escape codes generated by Nixpkgs' stdenv (used to
-   denote nesting etc.). */
-string filterANSIEscapes(const string & s, bool nixOnly = false);
+/* Truncate a string to 'width' printable characters. Certain ANSI
+   escape sequences (such as colour setting) are copied but not
+   included in the character count. Other ANSI escape sequences are
+   filtered. Also, tabs are expanded to spaces. */
+std::string filterANSIEscapes(const std::string & s,
+    unsigned int width = std::numeric_limits<unsigned int>::max());
 
 
 /* Base64 encoding/decoding. */

src/nix-build/nix-build.cc

@@ -141,7 +141,7 @@ void mainWrapped(int argc, char * * argv)
         else if (*arg == "--version")
             printVersion(myName);
 
-        else if (*arg == "--add-drv-link")
+        else if (*arg == "--add-drv-link" || *arg == "--indirect")
             ; // obsolete
 
         else if (*arg == "--no-out-link" || *arg == "--no-link")
@@ -167,9 +167,6 @@ void mainWrapped(int argc, char * * argv)
             buildMode = bmRepair;
         }
 
-        else if (*arg == "--hash")
-            buildMode = bmHash;
-
         else if (*arg == "--run-env") // obsolete
             runEnv = true;
 
@@ -235,6 +232,8 @@ void mainWrapped(int argc, char * * argv)
 
     myArgs.parseCmdline(args);
 
+    initPlugins();
+
     if (packages && fromArgs)
         throw UsageError("'-p' and '-E' are mutually exclusive");
 
@@ -279,8 +278,8 @@ void mainWrapped(int argc, char * * argv)
             else
                 /* If we're in a #! script, interpret filenames
                    relative to the script. */
-                exprs.push_back(state.parseExprFromFile(resolveExprPath(lookupFileArg(state,
-                    inShebang && !packages ? absPath(i, absPath(dirOf(script))) : i))));
+                exprs.push_back(state.parseExprFromFile(resolveExprPath(state.checkSourcePath(lookupFileArg(state,
+                    inShebang && !packages ? absPath(i, absPath(dirOf(script))) : i)))));
         }
 
     /* Evaluate them into derivations. */

src/nix-channel/nix-channel.cc

@@ -213,6 +213,9 @@ int main(int argc, char ** argv)
             }
             return true;
         });
+
+        initPlugins();
+
         switch (cmd) {
             case cNone:
                 throw UsageError("no command specified");

src/nix-collect-garbage/nix-collect-garbage.cc

@@ -77,6 +77,8 @@ int main(int argc, char * * argv)
             return true;
         });
 
+        initPlugins();
+
         auto profilesDir = settings.nixStateDir + "/profiles";
         if (removeOld) removeOldGenerations(profilesDir);
 

src/nix-copy-closure/nix-copy-closure.cc

@@ -44,6 +44,8 @@ int main(int argc, char ** argv)
             return true;
         });
 
+        initPlugins();
+
         if (sshHost.empty())
             throw UsageError("no host name specified");
 

src/nix-daemon/nix-daemon.cc

@@ -994,7 +994,7 @@ static void daemonLoop(char * * argv)
             if (matchUser(user, group, trustedUsers))
                 trusted = true;
 
-            if (!trusted && !matchUser(user, group, allowedUsers))
+            if ((!trusted && !matchUser(user, group, allowedUsers)) || group == settings.buildUsersGroup)
                 throw Error(format("user '%1%' is not allowed to connect to the Nix daemon") % user);
 
             printInfo(format((string) "accepted connection from pid %1%, user %2%" + (trusted ? " (trusted)" : ""))
@@ -1060,6 +1060,8 @@ int main(int argc, char * * argv)
             return true;
         });
 
+        initPlugins();
+
         if (stdio) {
             if (getStoreType() == tDaemon) {
                 /* Forward on this connection to the real daemon */

src/nix-env/nix-env.cc

@@ -1393,6 +1393,8 @@ int main(int argc, char * * argv)
 
         myArgs.parseCmdline(argvToStrings(argc, argv));
 
+        initPlugins();
+
         if (!op) throw UsageError("no operation specified");
 
         auto store = openStore();

src/nix-instantiate/nix-instantiate.cc

@@ -151,6 +151,8 @@ int main(int argc, char * * argv)
 
         myArgs.parseCmdline(argvToStrings(argc, argv));
 
+        initPlugins();
+
         if (evalOnly && !wantsReadWrite)
             settings.readOnlyMode = true;
 
@@ -182,7 +184,7 @@ int main(int argc, char * * argv)
         for (auto & i : files) {
             Expr * e = fromArgs
                 ? state.parseExprFromString(i, absPath("."))
-                : state.parseExprFromFile(resolveExprPath(lookupFileArg(state, i)));
+                : state.parseExprFromFile(resolveExprPath(state.checkSourcePath(lookupFileArg(state, i))));
             processExpr(state, attrPaths, parseOnly, strict, autoArgs,
                 evalOnly, outputKind, xmlOutputSourceLocation, e);
         }

src/nix-prefetch-url/nix-prefetch-url.cc

@@ -89,6 +89,8 @@ int main(int argc, char * * argv)
 
         myArgs.parseCmdline(argvToStrings(argc, argv));
 
+        initPlugins();
+
         if (args.size() > 2)
             throw UsageError("too many arguments");
 

src/nix-store/nix-store.cc

@@ -122,7 +122,6 @@ static void opRealise(Strings opFlags, Strings opArgs)
         if (i == "--dry-run") dryRun = true;
         else if (i == "--repair") buildMode = bmRepair;
         else if (i == "--check") buildMode = bmCheck;
-        else if (i == "--hash") buildMode = bmHash;
         else if (i == "--ignore-unknown") ignoreUnknown = true;
         else throw UsageError(format("unknown flag '%1%'") % i);
 
@@ -1053,6 +1052,8 @@ int main(int argc, char * * argv)
             return true;
         });
 
+        initPlugins();
+
         if (!op) throw UsageError("no operation specified");
 
         if (op != opDump && op != opRestore) /* !!! hack */

src/nix/build.cc

@@ -50,7 +50,9 @@ struct CmdBuild : MixDryRun, InstallablesCommand
 
     void run(ref<Store> store) override
     {
-        auto buildables = toBuildables(store, dryRun ? DryRun : Build, installables);
+        auto buildables = build(store, dryRun ? DryRun : Build, installables);
+
+        if (dryRun) return;
 
         for (size_t i = 0; i < buildables.size(); ++i) {
             auto & b(buildables[i]);

src/nix/command.hh

@@ -5,6 +5,8 @@
 
 namespace nix {
 
+extern std::string programPath;
+
 struct Value;
 class Bindings;
 class EvalState;
@@ -196,7 +198,7 @@ std::shared_ptr<Installable> parseInstallable(
     SourceExprCommand & cmd, ref<Store> store, const std::string & installable,
     bool useDefaultInstallables);
 
-Buildables toBuildables(ref<Store> store, RealiseMode mode,
+Buildables build(ref<Store> store, RealiseMode mode,
     std::vector<std::shared_ptr<Installable>> installables);
 
 PathSet toStorePaths(ref<Store> store, RealiseMode mode,

src/nix/copy.cc

@@ -57,15 +57,15 @@ struct CmdCopy : StorePathsCommand
         return {
             Example{
                 "To copy Firefox from the local store to a binary cache in file:///tmp/cache:",
-                "nix copy --to file:///tmp/cache -r $(type -p firefox)"
+                "nix copy --to file:///tmp/cache $(type -p firefox)"
             },
             Example{
                 "To copy the entire current NixOS system closure to another machine via SSH:",
-                "nix copy --to ssh://server -r /run/current-system"
+                "nix copy --to ssh://server /run/current-system"
             },
             Example{
                 "To copy a closure from another machine via SSH:",
-                "nix copy --from ssh://server -r /nix/store/a6cnl93nk1wxnq84brbbwr6hxw9gp2w9-blender-2.79-rc2"
+                "nix copy --from ssh://server /nix/store/a6cnl93nk1wxnq84brbbwr6hxw9gp2w9-blender-2.79-rc2"
             },
         };
     }

src/nix/eval.cc

@@ -5,10 +5,11 @@
 #include "eval.hh"
 #include "json.hh"
 #include "value-to-json.hh"
+#include "progress-bar.hh"
 
 using namespace nix;
 
-struct CmdEval : MixJSON, InstallablesCommand
+struct CmdEval : MixJSON, InstallableCommand
 {
     bool raw = false;
 
@@ -56,20 +57,19 @@ struct CmdEval : MixJSON, InstallablesCommand
 
         auto state = getEvalState();
 
-        auto jsonOut = json ? std::make_unique<JSONList>(std::cout) : nullptr;
+        auto v = installable->toValue(*state);
+        PathSet context;
 
-        for (auto & i : installables) {
-            auto v = i->toValue(*state);
-            PathSet context;
-            if (raw) {
-                std::cout << state->coerceToString(noPos, *v, context);
-            } else if (json) {
-                auto jsonElem = jsonOut->placeholder();
-                printValueAsJSON(*state, true, *v, jsonElem, context);
-            } else {
-                state->forceValueDeep(*v);
-                std::cout << *v << "\n";
-            }
+        stopProgressBar();
+
+        if (raw) {
+            std::cout << state->coerceToString(noPos, *v, context);
+        } else if (json) {
+            JSONPlaceholder jsonOut(std::cout);
+            printValueAsJSON(*state, true, *v, jsonOut, context);
+        } else {
+            state->forceValueDeep(*v);
+            std::cout << *v << "\n";
         }
     }
 };

src/nix/installables.cc

@@ -30,10 +30,8 @@ Value * SourceExprCommand::getSourceExpr(EvalState & state)
 
     vSourceExpr = state.allocValue();
 
-    if (file != "") {
-        Expr * e = state.parseExprFromFile(resolveExprPath(lookupFileArg(state, file)));
-        state.eval(e, *vSourceExpr);
-    }
+    if (file != "")
+        state.evalFile(lookupFileArg(state, file), *vSourceExpr);
 
     else {
 
@@ -255,7 +253,7 @@ std::shared_ptr<Installable> parseInstallable(
     return installables.front();
 }
 
-Buildables toBuildables(ref<Store> store, RealiseMode mode,
+Buildables build(ref<Store> store, RealiseMode mode,
     std::vector<std::shared_ptr<Installable>> installables)
 {
     if (mode != Build)
@@ -293,7 +291,7 @@ PathSet toStorePaths(ref<Store> store, RealiseMode mode,
 {
     PathSet outPaths;
 
-    for (auto & b : toBuildables(store, mode, installables))
+    for (auto & b : build(store, mode, installables))
         for (auto & output : b.outputs)
             outPaths.insert(output.second);
 

src/nix/local.mk

@@ -6,4 +6,6 @@ nix_SOURCES := $(wildcard $(d)/*.cc) $(wildcard src/linenoise/*.cpp)
 
 nix_LIBS = libexpr libmain libstore libutil libformat
 
+nix_LDFLAGS = -pthread
+
 $(eval $(call install-symlink, nix, $(bindir)/nix-hash))

src/nix/log.cc

@@ -50,6 +50,7 @@ struct CmdLog : InstallableCommand
 
         auto b = installable->toBuildable();
 
+        RunPager pager;
         for (auto & sub : subs) {
             auto log = b.drvPath != "" ? sub->getBuildLog(b.drvPath) : nullptr;
             for (auto & output : b.outputs) {

src/nix/ls.cc

@@ -90,6 +90,16 @@ struct CmdLsStore : StoreCommand, MixLs
         expectArg("path", &path);
     }
 
+    Examples examples() override
+    {
+        return {
+            Example{
+                "To list the contents of a store path in a binary cache:",
+                "nix ls-store --store https://cache.nixos.org/ -lR /nix/store/0i2jd68mp5g6h2sa5k9c85rb80sn8hi9-hello-2.10"
+            },
+        };
+    }
+
     std::string name() override
     {
         return "ls-store";
@@ -116,6 +126,16 @@ struct CmdLsNar : Command, MixLs
         expectArg("path", &path);
     }
 
+    Examples examples() override
+    {
+        return {
+            Example{
+                "To list a specific file in a NAR:",
+                "nix ls-nar -l hello.nar /bin/hello"
+            },
+        };
+    }
+
     std::string name() override
     {
         return "ls-nar";

src/nix/main.cc

@@ -16,6 +16,8 @@ void chrootHelper(int argc, char * * argv);
 
 namespace nix {
 
+std::string programPath;
+
 struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
 {
     NixArgs() : MultiCommand(*RegisterCommand::commands), MixCommonArgs("nix")
@@ -78,7 +80,8 @@ void mainWrapped(int argc, char * * argv)
     initNix();
     initGC();
 
-    string programName = baseNameOf(argv[0]);
+    programPath = argv[0];
+    string programName = baseNameOf(programPath);
 
     {
         auto legacy = (*RegisterLegacyCommand::commands)[programName];
@@ -89,6 +92,8 @@ void mainWrapped(int argc, char * * argv)
 
     args.parseCmdline(argvToStrings(argc, argv));
 
+    initPlugins();
+
     if (!args.command) args.showHelpAndExit();
 
     Finally f([]() { stopProgressBar(); });

src/nix/ping-store.cc

@@ -0,0 +1,35 @@
+#include "command.hh"
+#include "shared.hh"
+#include "store-api.hh"
+
+using namespace nix;
+
+struct CmdPingStore : StoreCommand
+{
+    std::string name() override
+    {
+        return "ping-store";
+    }
+
+    std::string description() override
+    {
+        return "test whether a store can be opened";
+    }
+
+    Examples examples() override
+    {
+        return {
+            Example{
+                "To test whether connecting to a remote Nix store via SSH works:",
+                "nix ping-store --store ssh://mac1"
+            },
+        };
+    }
+
+    void run(ref<Store> store) override
+    {
+        store->connect();
+    }
+};
+
+static RegisterCommand r1(make_ref<CmdPingStore>());

src/nix/progress-bar.cc

@@ -3,8 +3,9 @@
 #include "sync.hh"
 #include "store-api.hh"
 
-#include <map>
 #include <atomic>
+#include <map>
+#include <thread>
 
 namespace nix {
 
@@ -22,44 +23,6 @@ static uint64_t getI(const std::vector<Logger::Field> & fields, size_t n)
     return fields[n].i;
 }
 
-/* Truncate a string to 'width' printable characters. ANSI escape
-   sequences are copied but not included in the character count. Also,
-   tabs are expanded to spaces. */
-static std::string ansiTruncate(const std::string & s, int width)
-{
-    if (width <= 0) return s;
-
-    std::string t;
-    size_t w = 0;
-    auto i = s.begin();
-
-    while (w < (size_t) width && i != s.end()) {
-        if (*i == '\e') {
-            t += *i++;
-            if (i != s.end() && *i == '[') {
-                t += *i++;
-                while (i != s.end() && (*i < 0x40 || *i > 0x7e)) {
-                    t += *i++;
-                }
-                if (i != s.end()) t += *i++;
-            }
-        }
-
-        else if (*i == '\t') {
-            t += ' '; w++;
-            while (w < (size_t) width && w & 8) {
-                t += ' '; w++;
-            }
-        }
-
-        else {
-            t += *i++; w++;
-        }
-    }
-
-    return t;
-}
-
 class ProgressBar : public Logger
 {
 private:
@@ -101,15 +64,28 @@ private:
 
     Sync<State> state_;
 
+    std::thread updateThread;
+
+    std::condition_variable quitCV, updateCV;
+
 public:
 
     ProgressBar()
     {
+        updateThread = std::thread([&]() {
+            auto state(state_.lock());
+            while (state->active) {
+                state.wait(updateCV);
+                draw(*state);
+                state.wait_for(quitCV, std::chrono::milliseconds(50));
+            }
+        });
     }
 
     ~ProgressBar()
     {
         stop();
+        updateThread.join();
     }
 
     void stop()
@@ -121,6 +97,8 @@ public:
         writeToStderr("\r\e[K");
         if (status != "")
             writeToStderr("[" + status + "]\n");
+        updateCV.notify_one();
+        quitCV.notify_one();
     }
 
     void log(Verbosity lvl, const FormatOrString & fs) override
@@ -132,7 +110,7 @@ public:
     void log(State & state, Verbosity lvl, const std::string & s)
     {
         writeToStderr("\r\e[K" + s + ANSI_NORMAL "\n");
-        update(state);
+        draw(state);
     }
 
     void startActivity(ActivityId act, Verbosity lvl, ActivityType type,
@@ -167,7 +145,12 @@ public:
 
         if (type == actSubstitute) {
             auto name = storePathToName(getS(fields, 0));
-            i->s = fmt("fetching " ANSI_BOLD "%s" ANSI_NORMAL " from %s", name, getS(fields, 1));
+            auto sub = getS(fields, 1);
+            i->s = fmt(
+                hasPrefix(sub, "local")
+                ? "copying " ANSI_BOLD "%s" ANSI_NORMAL " from %s"
+                : "fetching " ANSI_BOLD "%s" ANSI_NORMAL " from %s",
+                name, sub);
         }
 
         if (type == actQueryPathInfo) {
@@ -180,7 +163,7 @@ public:
             || (type == actCopyPath && hasAncestor(*state, actSubstitute, parent)))
             i->visible = false;
 
-        update(*state);
+        update();
     }
 
     /* Check whether an activity has an ancestore with the specified
@@ -215,7 +198,7 @@ public:
             state->its.erase(i);
         }
 
-        update(*state);
+        update();
     }
 
     void result(ActivityId act, ResultType type, const std::vector<Field> & fields) override
@@ -225,7 +208,7 @@ public:
         if (type == resFileLinked) {
             state->filesLinked++;
             state->bytesLinked += getI(fields, 0);
-            update(*state);
+            update();
         }
 
         else if (type == resBuildLogLine) {
@@ -238,25 +221,25 @@ public:
                 info.lastLine = lastLine;
                 state->activities.emplace_back(info);
                 i->second = std::prev(state->activities.end());
-                update(*state);
+                update();
             }
         }
 
         else if (type == resUntrustedPath) {
             state->untrustedPaths++;
-            update(*state);
+            update();
         }
 
         else if (type == resCorruptedPath) {
             state->corruptedPaths++;
-            update(*state);
+            update();
         }
 
         else if (type == resSetPhase) {
             auto i = state->its.find(act);
             assert(i != state->its.end());
             i->second->phase = getS(fields, 0);
-            update(*state);
+            update();
         }
 
         else if (type == resProgress) {
@@ -267,7 +250,7 @@ public:
             actInfo.expected = getI(fields, 1);
             actInfo.running = getI(fields, 2);
             actInfo.failed = getI(fields, 3);
-            update(*state);
+            update();
         }
 
         else if (type == resSetExpected) {
@@ -279,17 +262,16 @@ public:
             state->activitiesByType[type].expected -= j;
             j = getI(fields, 1);
             state->activitiesByType[type].expected += j;
-            update(*state);
+            update();
         }
     }
 
     void update()
     {
-        auto state(state_.lock());
-        update(*state);
+        updateCV.notify_one();
     }
 
-    void update(State & state)
+    void draw(State & state)
     {
         if (!state.active) return;
 
@@ -323,7 +305,10 @@ public:
             }
         }
 
-        writeToStderr("\r" + ansiTruncate(line, getWindowSize().second) + "\e[K");
+        auto width = getWindowSize().second;
+        if (width <= 0) std::numeric_limits<decltype(width)>::max();
+
+        writeToStderr("\r" + filterANSIEscapes(line, width) + "\e[K");
     }
 
     std::string getStatus(State & state)

src/nix/run.cc

@@ -85,6 +85,10 @@ struct CmdRun : InstallablesCommand
                 "To run GNU Hello:",
                 "nix run nixpkgs.hello -c hello --greeting 'Hi everybody!'"
             },
+            Example{
+                "To run GNU Hello in a chroot store:",
+                "nix run --store ~/my-nix nixpkgs.hello -c hello"
+            },
         };
     }
 

src/nix/upgrade-nix.cc

@@ -0,0 +1,131 @@
+#include "command.hh"
+#include "store-api.hh"
+#include "download.hh"
+#include "eval.hh"
+#include "attr-path.hh"
+
+using namespace nix;
+
+struct CmdUpgradeNix : StoreCommand
+{
+    Path profileDir;
+
+    CmdUpgradeNix()
+    {
+        mkFlag()
+            .longName("profile")
+            .shortName('p')
+            .labels({"profile-dir"})
+            .description("the Nix profile to upgrade")
+            .dest(&profileDir);
+    }
+
+    std::string name() override
+    {
+        return "upgrade-nix";
+    }
+
+    std::string description() override
+    {
+        return "upgrade Nix to the latest stable version";
+    }
+
+    Examples examples() override
+    {
+        return {
+            Example{
+                "To upgrade Nix to the latest stable version:",
+                "nix upgrade-nix"
+            },
+            Example{
+                "To upgrade Nix in a specific profile:",
+                "nix upgrade-nix -p /nix/var/nix/profiles/per-user/alice/profile"
+            },
+        };
+    }
+
+    void run(ref<Store> store) override
+    {
+        settings.pureEval = true;
+
+        if (profileDir == "")
+            profileDir = getProfileDir(store);
+
+        printInfo("upgrading Nix in profile '%s'", profileDir);
+
+        Path storePath;
+        {
+            Activity act(*logger, lvlInfo, actUnknown, "querying latest Nix version");
+            storePath = getLatestNix(store);
+        }
+
+        {
+            Activity act(*logger, lvlInfo, actUnknown, fmt("downloading '%s'...", storePath));
+            store->ensurePath(storePath);
+        }
+
+        {
+            Activity act(*logger, lvlInfo, actUnknown, fmt("verifying that '%s' works...", storePath));
+            auto program = storePath + "/bin/nix-env";
+            auto s = runProgram(program, false, {"--version"});
+            if (s.find("Nix") == std::string::npos)
+                throw Error("could not verify that '%s' works", program);
+        }
+
+        {
+            Activity act(*logger, lvlInfo, actUnknown, fmt("installing '%s' into profile '%s'...", storePath, profileDir));
+            runProgram(settings.nixBinDir + "/nix-env", false,
+                {"--profile", profileDir, "-i", storePath, "--no-sandbox"});
+        }
+    }
+
+    /* Return the profile in which Nix is installed. */
+    Path getProfileDir(ref<Store> store)
+    {
+        Path where;
+
+        for (auto & dir : tokenizeString<Strings>(getEnv("PATH"), ":"))
+            if (pathExists(dir + "/nix-env")) {
+                where = dir;
+                break;
+            }
+
+        if (where == "")
+            throw Error("couldn't figure out how Nix is installed, so I can't upgrade it");
+
+        printInfo("found Nix in '%s'", where);
+
+        if (hasPrefix(where, "/run/current-system"))
+            throw Error("Nix on NixOS must be upgraded via 'nixos-rebuild'");
+
+        Path profileDir;
+        Path userEnv;
+
+        if (baseNameOf(where) != "bin" ||
+            !hasSuffix(userEnv = canonPath(profileDir = dirOf(where), true), "user-environment"))
+            throw Error("directory '%s' does not appear to be part of a Nix profile", where);
+
+        if (!store->isValidPath(userEnv))
+            throw Error("directory '%s' is not in the Nix store", userEnv);
+
+        return profileDir;
+    }
+
+    /* Return the store path of the latest stable Nix. */
+    Path getLatestNix(ref<Store> store)
+    {
+        // FIXME: use nixos.org?
+        auto req = DownloadRequest("https://github.com/NixOS/nixpkgs/raw/master/nixos/modules/installer/tools/nix-fallback-paths.nix");
+        auto res = getDownloader()->download(req);
+
+        EvalState state(Strings(), store);
+        auto v = state.allocValue();
+        state.eval(state.parseExprFromString(*res.data, "/no-such-path"), *v);
+        Bindings & bindings(*state.allocBindings(0));
+        auto v2 = findAlongAttrPath(state, settings.thisSystem, bindings, *v);
+
+        return state.forceString(*v2);
+    }
+};
+
+static RegisterCommand r1(make_ref<CmdUpgradeNix>());

tests/build-dry.sh

@@ -0,0 +1,52 @@
+source common.sh
+
+###################################################
+# Check that --dry-run isn't confused with read-only mode
+# https://github.com/NixOS/nix/issues/1795
+
+clearStore
+clearCache
+
+# Ensure this builds successfully first
+nix build -f dependencies.nix
+
+clearStore
+clearCache
+
+# Try --dry-run using old command first
+nix-build dependencies.nix --dry-run 2>&1 | grep "will be built"
+# Now new command:
+nix build -f dependencies.nix --dry-run 2>&1 | grep "will be built"
+
+# TODO: XXX: FIXME: #1793
+# Disable this part of the test until the problem is resolved:
+if [ -n "$ISSUE_1795_IS_FIXED" ]; then
+clearStore
+clearCache
+
+# Try --dry-run using new command first
+nix build -f dependencies.nix --dry-run 2>&1 | grep "will be built"
+# Now old command:
+nix-build dependencies.nix --dry-run 2>&1 | grep "will be built"
+fi
+
+###################################################
+# Check --dry-run doesn't create links with --dry-run
+# https://github.com/NixOS/nix/issues/1849
+clearStore
+clearCache
+
+RESULT=$TEST_ROOT/result-link
+rm -f $RESULT
+
+nix-build dependencies.nix -o $RESULT --dry-run
+
+[[ ! -h $RESULT ]] || fail "nix-build --dry-run created output link"
+
+nix build -f dependencies.nix -o $RESULT --dry-run
+
+[[ ! -h $RESULT ]] || fail "nix build --dry-run created output link"
+
+nix build -f dependencies.nix -o $RESULT
+
+[[ -h $RESULT ]]