Add builtins.unique [WIP]

This cuts ~4.5s from "nix-env -qaP -f `<nixpkgs>` --out-path" and
reduces memory allocations from 7.49 to 7.16 GiB.

Note that builtins.unique requires a less-than comparator function,
which is a change from the 'unique' function in Nixpkgs which uses an
equality function.

.gitignore

@@ -13,9 +13,6 @@ perl/Makefile.config
 
 /corepkgs/config.nix
 
-# /corepkgs/buildenv/
-/corepkgs/buildenv/builder.pl
-
 # /corepkgs/channels/
 /corepkgs/channels/unpack.sh
 
@@ -72,9 +69,6 @@ perl/Makefile.config
 # /src/nix-channel/
 /src/nix-channel/nix-channel
 
-# /src/buildenv/
-/src/buildenv/buildenv
-
 # /src/nix-build/
 /src/nix-build/nix-build
 
@@ -87,6 +81,9 @@ perl/Makefile.config
 /tests/common.sh
 /tests/dummy
 /tests/result*
+/tests/restricted-innocent
+/tests/shell
+/tests/shell.drv
 
 # /tests/lang/
 /tests/lang/*.out

.version

@@ -0,0 +1 @@
+2.3

Makefile

@@ -1,23 +1,11 @@
 makefiles = \
   local.mk \
-  src/boost/format/local.mk \
   src/libutil/local.mk \
   src/libstore/local.mk \
   src/libmain/local.mk \
   src/libexpr/local.mk \
   src/nix/local.mk \
-  src/nix-store/local.mk \
-  src/nix-instantiate/local.mk \
-  src/nix-env/local.mk \
-  src/nix-daemon/local.mk \
-  src/nix-collect-garbage/local.mk \
-  src/nix-copy-closure/local.mk \
-  src/nix-prefetch-url/local.mk \
-  src/buildenv/local.mk \
   src/resolve-system-dependencies/local.mk \
-  src/nix-channel/local.mk \
-  src/nix-build/local.mk \
-  src/build-remote/local.mk \
   scripts/local.mk \
   corepkgs/local.mk \
   misc/systemd/local.mk \
@@ -27,7 +15,7 @@ makefiles = \
   tests/local.mk \
   tests/plugins/local.mk
 
-GLOBAL_CXXFLAGS += -std=c++14 -g -Wall -include config.h
+GLOBAL_CXXFLAGS += -g -Wall -include config.h
 
 -include Makefile.config
 

Makefile.config.in

@@ -1,4 +1,6 @@
+AR = @AR@
 BDW_GC_LIBS = @BDW_GC_LIBS@
+BUILD_SHARED_LIBS = @BUILD_SHARED_LIBS@
 CC = @CC@
 CFLAGS = @CFLAGS@
 CXX = @CXX@
@@ -6,7 +8,6 @@ CXXFLAGS = @CXXFLAGS@
 ENABLE_S3 = @ENABLE_S3@
 HAVE_SODIUM = @HAVE_SODIUM@
 HAVE_READLINE = @HAVE_READLINE@
-HAVE_BROTLI = @HAVE_BROTLI@
 HAVE_SECCOMP = @HAVE_SECCOMP@
 LIBCURL_LIBS = @LIBCURL_LIBS@
 OPENSSL_LIBS = @OPENSSL_LIBS@
@@ -16,9 +17,9 @@ SODIUM_LIBS = @SODIUM_LIBS@
 LIBLZMA_LIBS = @LIBLZMA_LIBS@
 SQLITE3_LIBS = @SQLITE3_LIBS@
 LIBBROTLI_LIBS = @LIBBROTLI_LIBS@
+EDITLINE_LIBS = @EDITLINE_LIBS@
 bash = @bash@
 bindir = @bindir@
-brotli = @brotli@
 lsof = @lsof@
 datadir = @datadir@
 datarootdir = @datarootdir@

configure.ac

@@ -1,4 +1,4 @@
-AC_INIT(nix, m4_esyscmd([bash -c "echo -n $(cat ./version)$VERSION_SUFFIX"]))
+AC_INIT(nix, m4_esyscmd([bash -c "echo -n $(cat ./.version)$VERSION_SUFFIX"]))
 AC_CONFIG_SRCDIR(README.md)
 AC_CONFIG_AUX_DIR(config)
 
@@ -62,8 +62,9 @@ CXXFLAGS=
 AC_PROG_CC
 AC_PROG_CXX
 AC_PROG_CPP
-AX_CXX_COMPILE_STDCXX_11
+AX_CXX_COMPILE_STDCXX_17
 
+AC_CHECK_TOOL([AR], [ar])
 
 # Use 64-bit file system calls so that we can support files > 2 GiB.
 AC_SYS_LARGEFILE
@@ -127,8 +128,6 @@ NEED_PROG(bzip2, bzip2)
 NEED_PROG(gzip, gzip)
 NEED_PROG(xz, xz)
 AC_PATH_PROG(dot, dot)
-AC_PATH_PROG(pv, pv, pv)
-AC_PATH_PROGS(brotli, brotli bro, bro)
 AC_PATH_PROG(lsof, lsof, lsof)
 
 
@@ -152,9 +151,9 @@ PKG_CHECK_MODULES([OPENSSL], [libcrypto], [CXXFLAGS="$OPENSSL_CFLAGS $CXXFLAGS"]
 
 # Look for libbz2, a required dependency.
 AC_CHECK_LIB([bz2], [BZ2_bzWriteOpen], [true],
-  [AC_MSG_ERROR([Nix requires libbz2, which is part of bzip2.  See http://www.bzip.org/.])])
+  [AC_MSG_ERROR([Nix requires libbz2, which is part of bzip2.  See https://web.archive.org/web/20180624184756/http://www.bzip.org/.])])
 AC_CHECK_HEADERS([bzlib.h], [true],
-  [AC_MSG_ERROR([Nix requires libbz2, which is part of bzip2.  See http://www.bzip.org/.])])
+  [AC_MSG_ERROR([Nix requires libbz2, which is part of bzip2.  See https://web.archive.org/web/20180624184756/http://www.bzip.org/.])])
 
 
 # Look for SQLite, a required dependency.
@@ -164,6 +163,8 @@ PKG_CHECK_MODULES([SQLITE3], [sqlite3 >= 3.6.19], [CXXFLAGS="$SQLITE3_CFLAGS $CX
 # Look for libcurl, a required dependency.
 PKG_CHECK_MODULES([LIBCURL], [libcurl], [CXXFLAGS="$LIBCURL_CFLAGS $CXXFLAGS"])
 
+# Look for editline, a required dependency.
+PKG_CHECK_MODULES([EDITLINE], [libeditline], [CXXFLAGS="$EDITLINE_CFLAGS $CXXFLAGS"])
 
 # Look for libsodium, an optional dependency.
 PKG_CHECK_MODULES([SODIUM], [libsodium],
@@ -179,12 +180,9 @@ AC_CHECK_LIB([lzma], [lzma_stream_encoder_mt],
   [AC_DEFINE([HAVE_LZMA_MT], [1], [xz multithreaded compression support])])
 
 
-# Look for libbrotli{enc,dec}, optional dependencies
-PKG_CHECK_MODULES([LIBBROTLI], [libbrotlienc libbrotlidec],
-  [AC_DEFINE([HAVE_BROTLI], [1], [Whether to use libbrotli.])
-   CXXFLAGS="$LIBBROTLI_CFLAGS $CXXFLAGS"]
-   have_brotli=1], [have_brotli=])
-AC_SUBST(HAVE_BROTLI, [$have_brotli])
+# Look for libbrotli{enc,dec}.
+PKG_CHECK_MODULES([LIBBROTLI], [libbrotlienc libbrotlidec], [CXXFLAGS="$LIBBROTLI_CFLAGS $CXXFLAGS"])
+
 
 # Look for libseccomp, required for Linux sandboxing.
 if test "$sys_name" = linux; then
@@ -215,7 +213,7 @@ AC_SUBST(ENABLE_S3, [$enable_s3])
 AC_LANG_POP(C++)
 
 if test -n "$enable_s3"; then
-  declare -a aws_version_tokens=($(printf '#include <aws/core/VersionConfig.h>\nAWS_SDK_VERSION_STRING' | $CPP - | grep -v '^#.*' | sed 's/"//g' | tr '.' ' '))
+  declare -a aws_version_tokens=($(printf '#include <aws/core/VersionConfig.h>\nAWS_SDK_VERSION_STRING' | $CPP $CPPFLAGS - | grep -v '^#.*' | sed 's/"//g' | tr '.' ' '))
   AC_DEFINE_UNQUOTED([AWS_VERSION_MAJOR], ${aws_version_tokens@<:@0@:>@}, [Major version of aws-sdk-cpp.])
   AC_DEFINE_UNQUOTED([AWS_VERSION_MINOR], ${aws_version_tokens@<:@1@:>@}, [Minor version of aws-sdk-cpp.])
 fi
@@ -232,12 +230,6 @@ if test "$gc" = yes; then
 fi
 
 
-AC_ARG_ENABLE(init-state, AC_HELP_STRING([--disable-init-state],
-  [do not initialise DB etc. in `make install']),
-  init_state=$enableval, init_state=yes)
-#AM_CONDITIONAL(INIT_STATE, test "$init_state" = "yes")
-
-
 # documentation generation switch
 AC_ARG_ENABLE(doc-gen, AC_HELP_STRING([--disable-doc-gen],
   [disable documentation generation]),
@@ -260,11 +252,6 @@ if test "$(uname)" = "Darwin"; then
 fi
 
 
-# Figure out the extension of dynamic libraries.
-eval dynlib_suffix=$shrext_cmds
-AC_SUBST(dynlib_suffix)
-
-
 # Do we have GNU tar?
 AC_MSG_CHECKING([if you have a recent GNU tar])
 if $tar --version 2> /dev/null | grep -q GNU && tar cvf /dev/null --warning=no-timestamp ./config.log > /dev/null; then
@@ -281,6 +268,15 @@ AC_ARG_WITH(sandbox-shell, AC_HELP_STRING([--with-sandbox-shell=PATH],
   sandbox_shell=$withval)
 AC_SUBST(sandbox_shell)
 
+AC_ARG_ENABLE(shared, AC_HELP_STRING([--enable-shared],
+  [Build shared libraries for Nix [default=yes]]),
+  shared=$enableval, shared=yes)
+if test "$shared" = yes; then
+  AC_SUBST(BUILD_SHARED_LIBS, 1, [Whether to build shared libraries.])
+else
+  AC_SUBST(BUILD_SHARED_LIBS, 0, [Whether to build shared libraries.])
+fi
+
 
 # Expand all variables in config.status.
 test "$prefix" = NONE && prefix=$ac_default_prefix

corepkgs/buildenv.nix

@@ -1,11 +1,9 @@
-with import <nix/config.nix>;
-
 { derivations, manifest }:
 
 derivation {
   name = "user-environment";
-  system = builtins.currentSystem;
-  builder = nixLibexecDir + "/nix/buildenv";
+  system = "builtin";
+  builder = "builtin:buildenv";
 
   inherit manifest;
 
@@ -24,21 +22,4 @@ derivation {
 
   # Also don't bother substituting.
   allowSubstitutes = false;
-
-  __sandboxProfile = ''
-    (allow sysctl-read)
-    (allow file-read*
-           (literal "/usr/lib/libSystem.dylib")
-           (literal "/usr/lib/libSystem.B.dylib")
-           (literal "/usr/lib/libobjc.A.dylib")
-           (literal "/usr/lib/libobjc.dylib")
-           (literal "/usr/lib/libauto.dylib")
-           (literal "/usr/lib/libc++abi.dylib")
-           (literal "/usr/lib/libc++.1.dylib")
-           (literal "/usr/lib/libDiagnosticMessagesClient.dylib")
-           (subpath "/usr/lib/system")
-           (subpath "/dev"))
-  '';
-
-  inherit chrootDeps;
 }

corepkgs/fetchurl.nix

@@ -1,10 +1,14 @@
 { system ? "" # obsolete
 , url
+, hash ? "" # an SRI ash
+
+# Legacy hash specification
 , md5 ? "", sha1 ? "", sha256 ? "", sha512 ? ""
 , outputHash ?
-    if sha512 != "" then sha512 else if sha1 != "" then sha1 else if md5 != "" then md5 else sha256
+    if hash != "" then hash else if sha512 != "" then sha512 else if sha1 != "" then sha1 else if md5 != "" then md5 else sha256
 , outputHashAlgo ?
-    if sha512 != "" then "sha512" else if sha1 != "" then "sha1" else if md5 != "" then "md5" else "sha256"
+    if hash != "" then "" else if sha512 != "" then "sha512" else if sha1 != "" then "sha1" else if md5 != "" then "md5" else "sha256"
+
 , executable ? false
 , unpack ? false
 , name ? baseNameOf (toString url)

doc/manual/advanced-topics/advanced-topics.xml

@@ -1,6 +1,7 @@
 <part xmlns="http://docbook.org/ns/docbook"
       xmlns:xlink="http://www.w3.org/1999/xlink"
       xmlns:xi="http://www.w3.org/2001/XInclude"
+      xml:id="part-advanced-topics"
       version="5.0">
 
 <title>Advanced Topics</title>

doc/manual/advanced-topics/distributed-builds.xml

@@ -81,6 +81,7 @@ or a newline, e.g.
 
 <para>Each machine specification consists of the following elements,
 separated by spaces. Only the first element is required.
+To leave a field at its default, set it to <literal>-</literal>.
 
 <orderedlist>
 
@@ -179,4 +180,8 @@ builders = @/etc/nix/machines
 causes the list of machines in <filename>/etc/nix/machines</filename>
 to be included. (This is the default.)</para>
 
+<para>If you want the builders to use caches, you likely want to set
+the option <link linkend='conf-builders-use-substitutes'><literal>builders-use-substitutes</literal></link>
+in your local <filename>nix.conf</filename>.</para>
+
 </chapter>

doc/manual/command-ref/conf-file.xml

@@ -123,7 +123,7 @@ false</literal>.</para>
   </varlistentry>
 
 
-  <varlistentry><term><literal>auto-optimise-store</literal></term>
+  <varlistentry xml:id="conf-auto-optimise-store"><term><literal>auto-optimise-store</literal></term>
 
     <listitem><para>If set to <literal>true</literal>, Nix
     automatically detects files in the store that have identical
@@ -135,7 +135,6 @@ false</literal>.</para>
 
   </varlistentry>
 
-
   <varlistentry xml:id="conf-builders">
     <term><literal>builders</literal></term>
     <listitem>
@@ -146,7 +145,7 @@ false</literal>.</para>
   </varlistentry>
 
 
-  <varlistentry><term><literal>builders-use-substitutes</literal></term>
+  <varlistentry xml:id="conf-builders-use-substitutes"><term><literal>builders-use-substitutes</literal></term>
 
     <listitem><para>If set to <literal>true</literal>, Nix will instruct
     remote build machines to use their own binary substitutes if available. In
@@ -159,7 +158,6 @@ false</literal>.</para>
 
   </varlistentry>
 
-
   <varlistentry xml:id="conf-build-users-group"><term><literal>build-users-group</literal></term>
 
     <listitem><para>This options specifies the Unix group containing
@@ -201,7 +199,7 @@ false</literal>.</para>
   </varlistentry>
 
 
-  <varlistentry><term><literal>compress-build-log</literal></term>
+  <varlistentry xml:id="conf-compress-build-log"><term><literal>compress-build-log</literal></term>
 
     <listitem><para>If set to <literal>true</literal> (the default),
     build logs written to <filename>/nix/var/log/nix/drvs</filename>
@@ -210,7 +208,6 @@ false</literal>.</para>
 
   </varlistentry>
 
-
   <varlistentry xml:id="conf-connect-timeout"><term><literal>connect-timeout</literal></term>
 
     <listitem>
@@ -243,7 +240,6 @@ false</literal>.</para>
 
   </varlistentry>
 
-
   <varlistentry xml:id="conf-extra-sandbox-paths">
     <term><literal>extra-sandbox-paths</literal></term>
 
@@ -254,7 +250,26 @@ false</literal>.</para>
   </varlistentry>
 
 
-  <varlistentry><term><literal>extra-substituters</literal></term>
+  <varlistentry xml:id="conf-extra-platforms"><term><literal>extra-platforms</literal></term>
+
+    <listitem><para>Platforms other than the native one which
+    this machine is capable of building for. This can be useful for
+    supporting additional architectures on compatible machines:
+    i686-linux can be built on x86_64-linux machines (and the default
+    for this setting reflects this); armv7 is backwards-compatible with
+    armv6 and armv5tel; some aarch64 machines can also natively run
+    32-bit ARM code; and qemu-user may be used to support non-native
+    platforms (though this may be slow and buggy). Most values for this
+    are not enabled by default because build systems will often
+    misdetect the target platform and generate incompatible code, so you
+    may wish to cross-check the results of using this option against
+    proper natively-built versions of your
+    derivations.</para></listitem>
+
+  </varlistentry>
+
+
+  <varlistentry xml:id="conf-extra-substituters"><term><literal>extra-substituters</literal></term>
 
     <listitem><para>Additional binary caches appended to those
     specified in <option>substituters</option>.  When used by
@@ -264,8 +279,7 @@ false</literal>.</para>
 
   </varlistentry>
 
-
-  <varlistentry><term><literal>fallback</literal></term>
+  <varlistentry xml:id="conf-fallback"><term><literal>fallback</literal></term>
 
     <listitem><para>If set to <literal>true</literal>, Nix will fall
     back to building from source if a binary substitute fails.  This
@@ -274,8 +288,7 @@ false</literal>.</para>
 
   </varlistentry>
 
-
-  <varlistentry><term><literal>fsync-metadata</literal></term>
+  <varlistentry xml:id="conf-fsync-metadata"><term><literal>fsync-metadata</literal></term>
 
     <listitem><para>If set to <literal>true</literal>, changes to the
     Nix store metadata (in <filename>/nix/var/nix/db</filename>) are
@@ -285,7 +298,6 @@ false</literal>.</para>
 
   </varlistentry>
 
-
   <varlistentry xml:id="conf-hashed-mirrors"><term><literal>hashed-mirrors</literal></term>
 
     <listitem><para>A list of web servers used by
@@ -314,7 +326,7 @@ builtins.fetchurl {
   </varlistentry>
 
 
-  <varlistentry><term><literal>http-connections</literal></term>
+  <varlistentry xml:id="conf-http-connections"><term><literal>http-connections</literal></term>
 
     <listitem><para>The maximum number of parallel TCP connections
     used to fetch files from binary caches and by other downloads. It
@@ -323,7 +335,7 @@ builtins.fetchurl {
   </varlistentry>
 
 
-  <varlistentry><term><literal>keep-build-log</literal></term>
+  <varlistentry xml:id="conf-keep-build-log"><term><literal>keep-build-log</literal></term>
 
     <listitem><para>If set to <literal>true</literal> (the default),
     Nix will write the build log of a derivation (i.e. the standard
@@ -348,11 +360,9 @@ builtins.fetchurl {
     options a store path was built), so by default this option is on.
     Turn it off to save a bit of disk space (or a lot if
     <literal>keep-outputs</literal> is also turned on).</para></listitem>
-
   </varlistentry>
 
-
-  <varlistentry><term><literal>keep-env-derivations</literal></term>
+  <varlistentry xml:id="conf-keep-env-derivations"><term><literal>keep-env-derivations</literal></term>
 
     <listitem><para>If <literal>false</literal> (default), derivations
     are not stored in Nix user environments.  That is, the derivation
@@ -375,7 +385,6 @@ builtins.fetchurl {
 
   </varlistentry>
 
-
   <varlistentry xml:id="conf-keep-outputs"><term><literal>keep-outputs</literal></term>
 
     <listitem><para>If <literal>true</literal>, the garbage collector
@@ -389,10 +398,8 @@ builtins.fetchurl {
     only at build time (e.g., the C compiler, or source tarballs
     downloaded from the network).  To prevent it from doing so, set
     this option to <literal>true</literal>.</para></listitem>
-
   </varlistentry>
 
-
   <varlistentry xml:id="conf-max-build-log-size"><term><literal>max-build-log-size</literal></term>
 
     <listitem>
@@ -406,20 +413,27 @@ builtins.fetchurl {
 
   </varlistentry>
 
+  <varlistentry xml:id="conf-max-free"><term><literal>max-free</literal></term>
+
+    <listitem><para>This option defines after how many free bytes to stop collecting
+    garbage once the <literal>min-free</literal> condition gets triggered.</para></listitem>
+
+  </varlistentry>
 
   <varlistentry xml:id="conf-max-jobs"><term><literal>max-jobs</literal></term>
 
     <listitem><para>This option defines the maximum number of jobs
     that Nix will try to build in parallel.  The default is
     <literal>1</literal>. The special value <literal>auto</literal>
-    causes Nix to use the number of CPUs in your system.  It can be
+    causes Nix to use the number of CPUs in your system.  <literal>0</literal>
+    is useful when using remote builders to prevent any local builds (except for
+    <literal>preferLocalBuild</literal> derivation attribute which executes locally
+    regardless).  It can be
     overridden using the <option
     linkend='opt-max-jobs'>--max-jobs</option> (<option>-j</option>)
     command line switch.</para></listitem>
-
   </varlistentry>
 
-
   <varlistentry xml:id="conf-max-silent-time"><term><literal>max-silent-time</literal></term>
 
     <listitem>
@@ -440,8 +454,48 @@ builtins.fetchurl {
 
   </varlistentry>
 
+  <varlistentry xml:id="conf-min-free"><term><literal>min-free</literal></term>
 
-  <varlistentry><term><literal>netrc-file</literal></term>
+    <listitem>
+      <para>When the disk reaches <literal>min-free</literal> bytes of free disk space during a build, nix
+        will start to garbage-collection until <literal>max-free</literal> bytes are available on the disk.
+        A value of <literal>0</literal> (the default) means that this feature is disabled.</para>
+    </listitem>
+
+  </varlistentry>
+
+  <varlistentry xml:id="conf-narinfo-cache-negative-ttl"><term><literal>narinfo-cache-negative-ttl</literal></term>
+
+    <listitem>
+
+      <para>The TTL in seconds for negative lookups. If a store path is
+      queried from a substituter but was not found, there will be a
+      negative lookup cached in the local disk cache database for the
+      specified duration.</para>
+
+    </listitem>
+
+  </varlistentry>
+
+  <varlistentry xml:id="conf-narinfo-cache-positive-ttl"><term><literal>narinfo-cache-positive-ttl</literal></term>
+
+    <listitem>
+
+      <para>The TTL in seconds for positive lookups. If a store path is
+      queried from a substituter, the result of the query will be cached
+      in the local disk cache database including some of the NAR
+      metadata. The default TTL is a month, setting a shorter TTL for
+      positive lookups can be useful for binary caches that have
+      frequent garbage collection, in which case having a more frequent
+      cache invalidation would prevent trying to pull the path again and
+      failing with a hash mismatch if the build isn't reproducible.
+      </para>
+
+    </listitem>
+
+  </varlistentry>
+
+  <varlistentry xml:id="conf-netrc-file"><term><literal>netrc-file</literal></term>
 
     <listitem><para>If set to an absolute path to a <filename>netrc</filename>
     file, Nix will use the HTTP authentication credentials in this file when
@@ -459,7 +513,12 @@ password <replaceable>my-password</replaceable>
 
     For the exact syntax, see <link
     xlink:href="https://ec.haxx.se/usingcurl-netrc.html">the
-    <literal>curl</literal> documentation.</link></para></listitem>
+    <literal>curl</literal> documentation.</link></para>
+
+    <note><para>This must be an absolute path, and <literal>~</literal>
+    is not resolved. For example, <filename>~/.netrc</filename> won't
+    resolve to your home directory's <filename>.netrc</filename>.</para></note>
+    </listitem>
 
   </varlistentry>
 
@@ -495,7 +554,6 @@ password <replaceable>my-password</replaceable>
 
   </varlistentry>
 
-
   <varlistentry xml:id="conf-pre-build-hook"><term><literal>pre-build-hook</literal></term>
 
     <listitem>
@@ -531,7 +589,6 @@ password <replaceable>my-password</replaceable>
 
   </varlistentry>
 
-
   <varlistentry xml:id="conf-repeat"><term><literal>repeat</literal></term>
 
     <listitem><para>How many times to repeat builds to check whether
@@ -543,8 +600,7 @@ password <replaceable>my-password</replaceable>
 
   </varlistentry>
 
-
-  <varlistentry><term><literal>require-sigs</literal></term>
+  <varlistentry xml:id="conf-require-sigs"><term><literal>require-sigs</literal></term>
 
     <listitem><para>If set to <literal>true</literal> (the default),
     any non-content-addressed path added or copied to the Nix store
@@ -573,7 +629,7 @@ password <replaceable>my-password</replaceable>
   </varlistentry>
 
 
-  <varlistentry><term><literal>sandbox</literal></term>
+  <varlistentry xml:id="conf-sandbox"><term><literal>sandbox</literal></term>
 
     <listitem><para>If set to <literal>true</literal>, builds will be
     performed in a <emphasis>sandboxed environment</emphasis>, i.e.,
@@ -601,13 +657,13 @@ password <replaceable>my-password</replaceable>
     <varname>__noChroot</varname> attribute set to
     <literal>true</literal> do not run in sandboxes.</para>
 
-    <para>The default is <literal>false</literal>.</para>
+    <para>The default is <literal>true</literal> on Linux and
+    <literal>false</literal> on all other platforms.</para>
 
     </listitem>
 
   </varlistentry>
 
-
   <varlistentry xml:id="conf-sandbox-dev-shm-size"><term><literal>sandbox-dev-shm-size</literal></term>
 
     <listitem><para>This option determines the maximum size of the
@@ -644,7 +700,7 @@ password <replaceable>my-password</replaceable>
   </varlistentry>
 
 
-  <varlistentry><term><literal>secret-key-files</literal></term>
+  <varlistentry xml:id="conf-secret-key-files"><term><literal>secret-key-files</literal></term>
 
     <listitem><para>A whitespace-separated list of files containing
     secret (private) keys. These are used to sign locally-built
@@ -665,7 +721,7 @@ password <replaceable>my-password</replaceable>
   </varlistentry>
 
 
-  <varlistentry><term><literal>substitute</literal></term>
+  <varlistentry xml:id="conf-substitute"><term><literal>substitute</literal></term>
 
     <listitem><para>If set to <literal>true</literal> (default), Nix
     will use binary substitutes if available.  This option can be
@@ -673,8 +729,7 @@ password <replaceable>my-password</replaceable>
 
   </varlistentry>
 
-
-  <varlistentry><term><literal>substituters</literal></term>
+  <varlistentry xml:id="conf-substituters"><term><literal>substituters</literal></term>
 
     <listitem><para>A list of URLs of substituters, separated by
     whitespace.  The default is
@@ -682,8 +737,7 @@ password <replaceable>my-password</replaceable>
 
   </varlistentry>
 
-
-  <varlistentry><term><literal>system</literal></term>
+  <varlistentry xml:id="conf-system"><term><literal>system</literal></term>
 
     <listitem><para>This option specifies the canonical Nix system
     name of the current installation, such as
@@ -704,6 +758,33 @@ password <replaceable>my-password</replaceable>
   </varlistentry>
 
 
+  <varlistentry xml:id="conf-system-features"><term><literal>system-features</literal></term>
+
+    <listitem><para>A set of system “features” supported by this
+    machine, e.g. <literal>kvm</literal>. Derivations can express a
+    dependency on such features through the derivation attribute
+    <varname>requiredSystemFeatures</varname>. For example, the
+    attribute
+
+<programlisting>
+requiredSystemFeatures = [ "kvm" ];
+</programlisting>
+
+    ensures that the derivation can only be built on a machine with
+    the <literal>kvm</literal> feature.</para>
+
+    <para>This setting by default includes <literal>kvm</literal> if
+    <filename>/dev/kvm</filename> is accessible, and the
+    pseudo-features <literal>nixos-test</literal>,
+    <literal>benchmark</literal> and <literal>big-parallel</literal>
+    that are used in Nixpkgs to route builds to specific
+    machines.</para>
+
+    </listitem>
+
+  </varlistentry>
+
+
   <varlistentry xml:id="conf-timeout"><term><literal>timeout</literal></term>
 
     <listitem>
@@ -723,8 +804,7 @@ password <replaceable>my-password</replaceable>
 
   </varlistentry>
 
-
-  <varlistentry><term><literal>trusted-public-keys</literal></term>
+  <varlistentry xml:id="conf-trusted-public-keys"><term><literal>trusted-public-keys</literal></term>
 
     <listitem><para>A whitespace-separated list of public keys. When
     paths are copied from another Nix store (such as a binary cache),
@@ -734,8 +814,7 @@ password <replaceable>my-password</replaceable>
 
   </varlistentry>
 
-
-  <varlistentry><term><literal>trusted-substituters</literal></term>
+  <varlistentry xml:id="conf-trusted-substituters"><term><literal>trusted-substituters</literal></term>
 
     <listitem><para>A list of URLs of substituters, separated by
     whitespace.  These are not used by default, but can be enabled by
@@ -747,7 +826,6 @@ password <replaceable>my-password</replaceable>
 
   </varlistentry>
 
-
   <varlistentry xml:id="conf-trusted-users"><term><literal>trusted-users</literal></term>
 
     <listitem>
@@ -772,11 +850,179 @@ password <replaceable>my-password</replaceable>
 
   </varlistentry>
 
-
 </variablelist>
-
 </para>
 
+<refsection>
+  <title>Deprecated Settings</title>
+
+<para>
+
+<variablelist>
+
+  <varlistentry xml:id="conf-binary-caches">
+    <term><literal>binary-caches</literal></term>
+
+    <listitem><para><emphasis>Deprecated:</emphasis>
+    <literal>binary-caches</literal> is now an alias to
+    <xref linkend="conf-substituters" />.</para></listitem>
+  </varlistentry>
+
+  <varlistentry xml:id="conf-binary-cache-public-keys">
+    <term><literal>binary-cache-public-keys</literal></term>
+
+    <listitem><para><emphasis>Deprecated:</emphasis>
+    <literal>binary-cache-public-keys</literal> is now an alias to
+    <xref linkend="conf-trusted-public-keys" />.</para></listitem>
+  </varlistentry>
+
+  <varlistentry xml:id="conf-build-compress-log">
+    <term><literal>build-compress-log</literal></term>
+
+    <listitem><para><emphasis>Deprecated:</emphasis>
+    <literal>build-compress-log</literal> is now an alias to
+    <xref linkend="conf-compress-build-log" />.</para></listitem>
+  </varlistentry>
+
+  <varlistentry xml:id="conf-build-cores">
+    <term><literal>build-cores</literal></term>
+
+    <listitem><para><emphasis>Deprecated:</emphasis>
+    <literal>build-cores</literal> is now an alias to
+    <xref linkend="conf-cores" />.</para></listitem>
+  </varlistentry>
+
+  <varlistentry xml:id="conf-build-extra-chroot-dirs">
+    <term><literal>build-extra-chroot-dirs</literal></term>
+
+    <listitem><para><emphasis>Deprecated:</emphasis>
+    <literal>build-extra-chroot-dirs</literal> is now an alias to
+    <xref linkend="conf-extra-sandbox-paths" />.</para></listitem>
+  </varlistentry>
+
+  <varlistentry xml:id="conf-build-extra-sandbox-paths">
+    <term><literal>build-extra-sandbox-paths</literal></term>
+
+    <listitem><para><emphasis>Deprecated:</emphasis>
+    <literal>build-extra-sandbox-paths</literal> is now an alias to
+    <xref linkend="conf-extra-sandbox-paths" />.</para></listitem>
+  </varlistentry>
+
+  <varlistentry xml:id="conf-build-fallback">
+    <term><literal>build-fallback</literal></term>
+
+    <listitem><para><emphasis>Deprecated:</emphasis>
+    <literal>build-fallback</literal> is now an alias to
+    <xref linkend="conf-fallback" />.</para></listitem>
+  </varlistentry>
+
+  <varlistentry xml:id="conf-build-max-jobs">
+    <term><literal>build-max-jobs</literal></term>
+
+    <listitem><para><emphasis>Deprecated:</emphasis>
+    <literal>build-max-jobs</literal> is now an alias to
+    <xref linkend="conf-max-jobs" />.</para></listitem>
+  </varlistentry>
+
+  <varlistentry xml:id="conf-build-max-log-size">
+    <term><literal>build-max-log-size</literal></term>
+
+    <listitem><para><emphasis>Deprecated:</emphasis>
+    <literal>build-max-log-size</literal> is now an alias to
+    <xref linkend="conf-max-build-log-size" />.</para></listitem>
+  </varlistentry>
+
+  <varlistentry xml:id="conf-build-max-silent-time">
+    <term><literal>build-max-silent-time</literal></term>
+
+    <listitem><para><emphasis>Deprecated:</emphasis>
+    <literal>build-max-silent-time</literal> is now an alias to
+    <xref linkend="conf-max-silent-time" />.</para></listitem>
+  </varlistentry>
+
+  <varlistentry xml:id="conf-build-repeat">
+    <term><literal>build-repeat</literal></term>
+
+    <listitem><para><emphasis>Deprecated:</emphasis>
+    <literal>build-repeat</literal> is now an alias to
+    <xref linkend="conf-repeat" />.</para></listitem>
+  </varlistentry>
+
+  <varlistentry xml:id="conf-build-timeout">
+    <term><literal>build-timeout</literal></term>
+
+    <listitem><para><emphasis>Deprecated:</emphasis>
+    <literal>build-timeout</literal> is now an alias to
+    <xref linkend="conf-timeout" />.</para></listitem>
+  </varlistentry>
+
+  <varlistentry xml:id="conf-build-use-chroot">
+    <term><literal>build-use-chroot</literal></term>
+
+    <listitem><para><emphasis>Deprecated:</emphasis>
+    <literal>build-use-chroot</literal> is now an alias to
+    <xref linkend="conf-sandbox" />.</para></listitem>
+  </varlistentry>
+
+  <varlistentry xml:id="conf-build-use-sandbox">
+    <term><literal>build-use-sandbox</literal></term>
+
+    <listitem><para><emphasis>Deprecated:</emphasis>
+    <literal>build-use-sandbox</literal> is now an alias to
+    <xref linkend="conf-sandbox" />.</para></listitem>
+  </varlistentry>
+
+  <varlistentry xml:id="conf-build-use-substitutes">
+    <term><literal>build-use-substitutes</literal></term>
+
+    <listitem><para><emphasis>Deprecated:</emphasis>
+    <literal>build-use-substitutes</literal> is now an alias to
+    <xref linkend="conf-substitute" />.</para></listitem>
+  </varlistentry>
+
+  <varlistentry xml:id="conf-gc-keep-derivations">
+    <term><literal>gc-keep-derivations</literal></term>
+
+    <listitem><para><emphasis>Deprecated:</emphasis>
+    <literal>gc-keep-derivations</literal> is now an alias to
+    <xref linkend="conf-keep-derivations" />.</para></listitem>
+  </varlistentry>
+
+  <varlistentry xml:id="conf-gc-keep-outputs">
+    <term><literal>gc-keep-outputs</literal></term>
+
+    <listitem><para><emphasis>Deprecated:</emphasis>
+    <literal>gc-keep-outputs</literal> is now an alias to
+    <xref linkend="conf-keep-outputs" />.</para></listitem>
+  </varlistentry>
+
+  <varlistentry xml:id="conf-env-keep-derivations">
+    <term><literal>env-keep-derivations</literal></term>
+
+    <listitem><para><emphasis>Deprecated:</emphasis>
+    <literal>env-keep-derivations</literal> is now an alias to
+    <xref linkend="conf-keep-env-derivations" />.</para></listitem>
+  </varlistentry>
+
+  <varlistentry xml:id="conf-extra-binary-caches">
+    <term><literal>extra-binary-caches</literal></term>
+
+    <listitem><para><emphasis>Deprecated:</emphasis>
+    <literal>extra-binary-caches</literal> is now an alias to
+    <xref linkend="conf-extra-substituters" />.</para></listitem>
+  </varlistentry>
+
+  <varlistentry xml:id="conf-trusted-binary-caches">
+    <term><literal>trusted-binary-caches</literal></term>
+
+    <listitem><para><emphasis>Deprecated:</emphasis>
+    <literal>trusted-binary-caches</literal> is now an alias to
+    <xref linkend="conf-trusted-substituters" />.</para></listitem>
+  </varlistentry>
+</variablelist>
+</para>
+</refsection>
+
 </refsection>
 
 </refentry>

doc/manual/command-ref/env-common.xml

@@ -52,10 +52,15 @@ nixpkgs=/home/eelco/Dev/nixpkgs-branch:/etc/nixos</screen>
     <envar>NIX_PATH</envar> to
 
     <screen>
-nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-14.12.tar.gz</screen>
+nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-15.09.tar.gz</screen>
 
     tells Nix to download the latest revision in the Nixpkgs/NixOS
-    14.12 channel.</para>
+    15.09 channel.</para>
+
+    <para>A following shorthand can be used to refer to the official channels:
+    
+    <screen>nixpkgs=channel:nixos-15.09</screen>
+    </para>
 
     <para>The search path can be extended using the <option
     linkend="opt-I">-I</option> option, which takes precedence over

doc/manual/command-ref/nix-channel.xml

@@ -31,7 +31,7 @@
 
 <refsection><title>Description</title>
 
-<para>A Nix channel is mechanism that allows you to automatically stay
+<para>A Nix channel is a mechanism that allows you to automatically stay
 up-to-date with a set of pre-built Nix expressions.  A Nix channel is
 just a URL that points to a place containing both a set of Nix
 expressions and a pointer to a binary cache.  <phrase
@@ -165,8 +165,8 @@ following files:</para>
   <varlistentry><term><filename>nixexprs.tar.xz</filename></term>
 
     <listitem><para>A tarball containing Nix expressions and files
-    referenced by them (such as build scripts and patches). At
-    top-level, the tarball should contain a single directory. That
+    referenced by them (such as build scripts and patches). At the
+    top level, the tarball should contain a single directory. That
     directory must contain a file <filename>default.nix</filename>
     that serves as the channel’s “entry point”.</para></listitem>
 
@@ -175,7 +175,7 @@ following files:</para>
   <varlistentry><term><filename>binary-cache-url</filename></term>
 
     <listitem><para>A file containing the URL to a binary cache (such
-    as <uri>https://cache.nixos.org</uri>. Nix will automatically
+    as <uri>https://cache.nixos.org</uri>). Nix will automatically
     check this cache for pre-built binaries, if the user has
     sufficient rights to add binary caches. For instance, in a
     multi-user Nix setup, the binary caches provided by the channels

doc/manual/command-ref/nix-collect-garbage.xml

@@ -22,12 +22,6 @@
     <arg><option>--delete-old</option></arg>
     <arg><option>-d</option></arg>
     <arg><option>--delete-older-than</option> <replaceable>period</replaceable></arg>
-    <group choice='opt'>
-      <arg choice='plain'><option>--print-roots</option></arg>
-      <arg choice='plain'><option>--print-live</option></arg>
-      <arg choice='plain'><option>--print-dead</option></arg>
-      <arg choice='plain'><option>--delete</option></arg>
-    </group>
     <arg><option>--max-freed</option> <replaceable>bytes</replaceable></arg>
     <arg><option>--dry-run</option></arg>
   </cmdsynopsis>

doc/manual/command-ref/nix-copy-closure.xml

@@ -95,15 +95,6 @@ those paths.  If this bothers you, use
 
   </varlistentry>
 
-  <!--
-  <varlistentry><term><option>- -show-progress</option></term>
-
-    <listitem><para>Show the progress of each path's transfer as it's made.
-    This requires the <command>pv</command> utility to be in <envar>PATH</envar>.</para></listitem>
-
-  </varlistentry>
-  -->
-
   <varlistentry><term><option>--include-outputs</option></term>
 
     <listitem><para>Also copy the outputs of store derivations

doc/manual/command-ref/nix-env.xml

@@ -456,7 +456,7 @@ $ nix-env -f ~/foo.nix -i '.*'</screen>
 from another profile:
 
 <screen>
-$ nix-env -i --from-profile /nix/var/nix/profiles/foo -i gcc</screen>
+$ nix-env -i --from-profile /nix/var/nix/profiles/foo gcc</screen>
 
 </para>
 
@@ -1346,11 +1346,12 @@ $ nix-env --list-generations
 <para>This operation deletes the specified generations of the current
 profile.  The generations can be a list of generation numbers, the
 special value <literal>old</literal> to delete all non-current
-generations, or a value such as <literal>30d</literal> to delete all
+generations,  a value such as <literal>30d</literal> to delete all
 generations older than the specified number of days (except for the
-generation that was active at that point in time).
-Periodically deleting old generations is important to make garbage
-collection effective.</para>
+generation that was active at that point in time), or a value such as.
+<literal>+5</literal> to only keep the specified items older than the
+current generation. Periodically deleting old generations is important
+to make garbage collection effective.</para>
 
 </refsection>
 
@@ -1359,6 +1360,8 @@ collection effective.</para>
 <screen>
 $ nix-env --delete-generations 3 4 8
 
+$ nix-env --delete-generations +5
+
 $ nix-env --delete-generations 30d
 
 $ nix-env -p other_profile --delete-generations old</screen>
@@ -1458,7 +1461,7 @@ error: no generation older than the current (91) exists</screen>
 <refsection condition="manpage"><title>Environment variables</title>
 
 <variablelist>
-  
+
   <varlistentry><term><envar>NIX_PROFILE</envar></term>
 
     <listitem><para>Location of the Nix profile.  Defaults to the
@@ -1472,6 +1475,6 @@ error: no generation older than the current (91) exists</screen>
 </variablelist>
 
 </refsection>
-  
+
 
 </refentry>

doc/manual/command-ref/nix-instantiate.xml

@@ -154,7 +154,9 @@ input.</para>
     <listitem><para>When used with <option>--eval</option>, perform
     evaluation in read/write mode so nix language features that
     require it will still work (at the cost of needing to do
-    instantiation of every evaluated derivation).</para>
+    instantiation of every evaluated derivation). If this option is
+    not enabled, there may be uninstantiated store paths in the final
+    output.</para>
 
     </listitem>
 

doc/manual/command-ref/nix-shell.xml

@@ -32,6 +32,7 @@
     <arg><option>--run</option> <replaceable>cmd</replaceable></arg>
     <arg><option>--exclude</option> <replaceable>regexp</replaceable></arg>
     <arg><option>--pure</option></arg>
+    <arg><option>--keep</option> <replaceable>name</replaceable></arg>
     <group choice='req'>
       <arg choice='plain'>
         <group choice='req'>
@@ -165,6 +166,13 @@ also <xref linkend="sec-common-options" />.</phrase></para>
 
     </listitem></varlistentry>
 
+  <varlistentry><term><option>--keep</option> <replaceable>name</replaceable></term>
+
+    <listitem><para>When a <option>--pure</option> shell is started,
+    keep the listed environment variables.</para></listitem>
+
+  </varlistentry>
+
 </variablelist>
 
 <para>The following common options are supported:</para>
@@ -309,13 +317,28 @@ while (my $token = $p->get_tag("a")) {
 
 </para>
 
-<para>Finally, the following Haskell script uses a specific branch of
-Nixpkgs/NixOS (the 14.12 stable branch):
+<para>Sometimes you need to pass a simple Nix expression to customize
+a package like Terraform:
 
 <programlisting><![CDATA[
 #! /usr/bin/env nix-shell
-#! nix-shell -i runghc -p haskellPackages.ghc haskellPackages.HTTP haskellPackages.tagsoup
-#! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-14.12.tar.gz
+#! nix-shell -i bash -p "terraform.withPlugins (plugins: [ plugins.openstack ])"
+
+terraform apply
+]]></programlisting>
+
+<note><para>You must use double quotes (<literal>"</literal>) when
+passing a simple Nix expression in a nix-shell shebang.</para></note>
+</para>
+
+<para>Finally, using the merging of multiple nix-shell shebangs the
+following Haskell script uses a specific branch of Nixpkgs/NixOS (the
+18.03 stable branch):
+
+<programlisting><![CDATA[
+#! /usr/bin/env nix-shell
+#! nix-shell -i runghc -p "haskellPackages.ghcWithPackages (ps: [ps.HTTP ps.tagsoup])"
+#! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-18.03.tar.gz
 
 import Network.HTTP
 import Text.HTML.TagSoup

doc/manual/command-ref/nix-store.xml

@@ -204,7 +204,7 @@ printed.)</para>
     with <option>-K</option>, if an output path is not identical to
     the corresponding output from the previous build, the new output
     path is left in
-    <filename>/nix/store/<replaceable>name</replaceable>-check.</filename></para>
+    <filename>/nix/store/<replaceable>name</replaceable>.check.</filename></para>
 
     <para>See also the <option>build-repeat</option> configuration
     option, which repeats a derivation a number of times and prevents
@@ -275,7 +275,7 @@ as a means of providing Nix store access to a restricted ssh user.
 
     <listitem><para>Allow the connected client to request the realization
     of derivations. In effect, this can be used to make the host act
-    as a build slave.</para></listitem>
+    as a remote builder.</para></listitem>
 
   </varlistentry>
 
@@ -501,10 +501,11 @@ error: cannot delete path `/nix/store/zq0h41l75vlb4z45kzgjjmsjxvcv1qk7-mesa-6.4'
     <arg choice='plain'><option>--referrers</option></arg>
     <arg choice='plain'><option>--referrers-closure</option></arg>
     <arg choice='plain'><option>--deriver</option></arg>
-    <arg choice='plain'><option>--deriver</option></arg>
+    <arg choice='plain'><option>-d</option></arg>
     <arg choice='plain'><option>--graph</option></arg>
     <arg choice='plain'><option>--tree</option></arg>
     <arg choice='plain'><option>--binding</option> <replaceable>name</replaceable></arg>
+    <arg choice='plain'><option>-b</option> <replaceable>name</replaceable></arg>
     <arg choice='plain'><option>--hash</option></arg>
     <arg choice='plain'><option>--size</option></arg>
     <arg choice='plain'><option>--roots</option></arg>
@@ -642,6 +643,7 @@ query is applied to the target of the symlink.</para>
   </varlistentry>
 
   <varlistentry><term><option>--deriver</option></term>
+    <term><option>-d</option></term>
 
     <listitem><para>Prints the <link
     linkend="gloss-deriver">deriver</link> of the store paths
@@ -677,7 +679,20 @@ query is applied to the target of the symlink.</para>
 
   </varlistentry>
 
+  <varlistentry><term><option>--graphml</option></term>
+
+    <listitem><para>Prints the references graph of the store paths
+    <replaceable>paths</replaceable> in the <link
+    xlink:href="http://graphml.graphdrawing.org/">GraphML</link> file format.
+    This can be used to visualise dependency graphs. To obtain a
+    build-time dependency graph, apply this to a store derivation. To
+    obtain a runtime dependency graph, apply it to an output
+    path.</para></listitem>
+
+  </varlistentry>
+
   <varlistentry><term><option>--binding</option> <replaceable>name</replaceable></term>
+    <term><option>-b</option> <replaceable>name</replaceable></term>
 
     <listitem><para>Prints the value of the attribute
     <replaceable>name</replaceable> (i.e., environment variable) of
@@ -1267,6 +1282,7 @@ ktorrent-2.2.1/NEWS
   <cmdsynopsis>
     <command>nix-store</command>
     <arg choice='plain'><option>--dump-db</option></arg>
+    <arg rep='repeat'><replaceable>paths</replaceable></arg>
   </cmdsynopsis>
 </refsection>
 
@@ -1277,6 +1293,13 @@ Nix database to standard output.  It can be loaded into an empty Nix
 store using <option>--load-db</option>.  This is useful for making
 backups and when migrating to different database schemas.</para>
 
+<para>By default, <option>--dump-db</option> will dump the entire Nix
+database.  When one or more store paths is passed, only the subset of
+the Nix database for those store paths is dumped.  As with
+<option>--export</option>, the user is responsible for passing all the
+store paths for a closure.  See <option>--export</option> for an
+example.</para>
+
 </refsection>
 
 </refsection>

doc/manual/command-ref/opt-common-syn.xml

@@ -8,6 +8,9 @@
     <arg choice='plain'><option>-v</option></arg>
   </group>
 </arg>
+<arg>
+  <arg choice='plain'><option>--quiet</option></arg>
+</arg>
 <arg>
   <group choice='plain'>
     <arg choice='plain'><option>--no-build-output</option></arg>

doc/manual/command-ref/opt-common.xml

@@ -75,6 +75,23 @@
 </varlistentry>
 
 
+<varlistentry><term><option>--quiet</option></term>
+
+  <listitem>
+
+  <para>Decreases the level of verbosity of diagnostic messages
+  printed on standard error.  This is the inverse option to
+  <option>-v</option> / <option>--verbose</option>.
+  </para>
+
+  <para>This option may be specified repeatedly.  See the previous
+  verbosity levels list.</para>
+
+  </listitem>
+
+</varlistentry>
+
+
 <varlistentry><term><option>--no-build-output</option> / <option>-Q</option></term>
 
   <listitem><para>By default, output written by builders to standard

doc/manual/expressions/advanced-attributes.xml

@@ -50,6 +50,40 @@ allowedRequisites = [ foobar ];
 
   </varlistentry>
 
+  <varlistentry><term><varname>disallowedReferences</varname></term>
+
+    <listitem><para>The optional attribute
+    <varname>disallowedReferences</varname> specifies a list of illegal
+    references (dependencies) of the output of the builder.  For
+    example,
+
+<programlisting>
+disallowedReferences = [ foo ];
+</programlisting>
+
+    enforces that the output of a derivation cannot have a direct runtime
+    dependencies on the derivation <varname>foo</varname>.</para></listitem>
+
+  </varlistentry>
+
+
+  <varlistentry><term><varname>disallowedRequisites</varname></term>
+
+    <listitem><para>This attribute is similar to
+    <varname>disallowedReferences</varname>, but it specifies illegal
+    requisites for the whole closure, so all the dependencies
+    recursively.  For example,
+
+<programlisting>
+disallowedRequisites = [ foobar ];
+</programlisting>
+
+    enforces that the output of a derivation cannot have any
+    runtime dependency on <varname>foobar</varname> or any other derivation
+    depending recursively on <varname>foobar</varname>.</para></listitem>
+
+  </varlistentry>
+
 
   <varlistentry><term><varname>exportReferencesGraph</varname></term>
 
@@ -112,7 +146,13 @@ impureEnvVars = [ "http_proxy" "https_proxy" <replaceable>...</replaceable> ];
     linkend="fixed-output-drvs">fixed-output derivations</link>, where
     impurities such as these are okay since (the hash of) the output
     is known in advance.  It is ignored for all other
-    derivations.</para></listitem>
+    derivations.</para>
+
+    <warning><para><varname>impureEnvVars</varname> implementation takes
+    environment variables from the current builder process. When a daemon is
+    building its environmental variables are used. Without the daemon, the
+    environmental variables come from the environment of the
+    <command>nix-build</command>.</para></warning></listitem>
 
   </varlistentry>
 
@@ -176,7 +216,7 @@ fetchurl {
 <programlisting>
 { stdenv, curl }: # The <command>curl</command> program is used for downloading.
 
-{ url, md5 }:
+{ url, sha256 }:
 
 stdenv.mkDerivation {
   name = baseNameOf (toString url);
@@ -184,10 +224,10 @@ stdenv.mkDerivation {
   buildInputs = [ curl ];
 
   # This is a fixed-output derivation; the output must be a regular
-  # file with MD5 hash <varname>md5</varname>.
+  # file with SHA256 hash <varname>sha256</varname>.
   outputHashMode = "flat";
-  outputHashAlgo = "md5";
-  outputHash = md5;
+  outputHashAlgo = "sha256";
+  outputHash = sha256;
 
   inherit url;
 }
@@ -197,8 +237,8 @@ stdenv.mkDerivation {
 
     <para>The <varname>outputHashAlgo</varname> attribute specifies
     the hash algorithm used to compute the hash.  It can currently be
-    <literal>"md5"</literal>, <literal>"sha1"</literal> or
-    <literal>"sha256"</literal>.</para>
+    <literal>"sha1"</literal>, <literal>"sha256"</literal> or
+    <literal>"sha512"</literal>.</para>
 
     <para>The <varname>outputHashMode</varname> attribute determines
     how the hash is computed.  It must be one of the following two
@@ -211,7 +251,7 @@ stdenv.mkDerivation {
         <listitem><para>The output must be a non-executable regular
         file.  If it isn’t, the build fails.  The hash is simply
         computed over the contents of that file (so it’s equal to what
-        Unix commands like <command>md5sum</command> or
+        Unix commands like <command>sha256sum</command> or
         <command>sha1sum</command> produce).</para>
 
         <para>This is the default.</para></listitem>
@@ -272,9 +312,7 @@ big = "a very long string";
   <varlistentry><term><varname>preferLocalBuild</varname></term>
 
     <listitem><para>If this attribute is set to
-    <literal>true</literal>, it has two effects.  First, the
-    derivation will always be built, not substituted, even if a
-    substitute is available.  Second, if <link
+    <literal>true</literal> and <link
     linkend="chap-distributed-builds">distributed building is
     enabled</link>, then, if possible, the derivaton will be built
     locally instead of forwarded to a remote machine.  This is
@@ -284,6 +322,19 @@ big = "a very long string";
 
   </varlistentry>
 
+
+  <varlistentry><term><varname>allowSubstitutes</varname></term>
+
+    <listitem><para>If this attribute is set to
+    <literal>false</literal>, then Nix will always build this
+    derivation; it will not try to substitute its outputs. This is
+    useful for very trivial derivations (such as
+    <function>writeText</function> in Nixpkgs) that are cheaper to
+    build than to substitute from a binary cache.</para></listitem>
+
+  </varlistentry>
+
+
 </variablelist>
 
 </section>

doc/manual/expressions/language-constructs.xml

@@ -41,7 +41,7 @@ encountered</quote>).</para></footnote>.</para>
 </simplesect>
 
 
-<simplesect><title>Let-expressions</title>
+<simplesect xml:id="sect-let-expressions"><title>Let-expressions</title>
 
 <para>A let-expression allows you define local variables for an
 expression.  For instance,

doc/manual/glossary/glossary.xml

@@ -1,5 +1,6 @@
 <appendix xmlns="http://docbook.org/ns/docbook"
-          xmlns:xlink="http://www.w3.org/1999/xlink">
+          xmlns:xlink="http://www.w3.org/1999/xlink"
+          xml:id="part-glossary">
 
 <title>Glossary</title>
 
@@ -85,29 +86,48 @@
 
 <glossentry xml:id="gloss-reference"><glossterm>reference</glossterm>
 
-  <glossdef><para>A store path <varname>P</varname> is said to have a
-  reference to a store path <varname>Q</varname> if the store object
-  at <varname>P</varname> contains the path <varname>Q</varname>
-  somewhere.  This implies than an execution involving
-  <varname>P</varname> potentially needs <varname>Q</varname> to be
-  present.  The <emphasis>references</emphasis> of a store path are
-  the set of store paths to which it has a reference.</para></glossdef>
+  <glossdef>
+    <para>A store path <varname>P</varname> is said to have a
+    reference to a store path <varname>Q</varname> if the store object
+    at <varname>P</varname> contains the path <varname>Q</varname>
+    somewhere. The <emphasis>references</emphasis> of a store path are
+    the set of store paths to which it has a reference.
+    </para>
+    <para>A derivation can reference other derivations and sources
+    (but not output paths), whereas an output path only references other
+    output paths.
+    </para>
+  </glossdef>
 
 </glossentry>
 
+<glossentry xml:id="gloss-reachable"><glossterm>reachable</glossterm>
+
+  <glossdef><para>A store path <varname>Q</varname> is reachable from
+  another store path <varname>P</varname> if <varname>Q</varname> is in the
+  <link linkend="gloss-closure">closure</link> of the
+  <link linkend="gloss-reference">references</link> relation.
+  </para></glossdef>
+</glossentry>
 
 <glossentry xml:id="gloss-closure"><glossterm>closure</glossterm>
 
   <glossdef><para>The closure of a store path is the set of store
   paths that are directly or indirectly “reachable” from that store
   path; that is, it’s the closure of the path under the <link
-  linkend="gloss-reference">references</link> relation.  For instance,
-  if the store object at path <varname>P</varname> contains a
-  reference to path <varname>Q</varname>, then <varname>Q</varname> is
-  in the closure of <varname>P</varname>.  For correct deployment it
-  is necessary to deploy whole closures, since otherwise at runtime
-  files could be missing.  The command <command>nix-store
-  -qR</command> prints out closures of store paths.</para></glossdef>
+  linkend="gloss-reference">references</link> relation. For a package, the
+  closure of its derivation is equivalent to the build-time
+  dependencies, while the closure of its output path is equivalent to its
+  runtime dependencies. For correct deployment it is necessary to deploy whole
+  closures, since otherwise at runtime files could be missing. The command
+  <command>nix-store -qR</command> prints out closures of store paths.
+  </para>
+  <para>As an example, if the store object at path <varname>P</varname> contains
+  a reference to path <varname>Q</varname>, then <varname>Q</varname> is
+  in the closure of <varname>P</varname>. Further, if <varname>Q</varname>
+  references <varname>R</varname> then <varname>R</varname> is also in
+  the closure of <varname>P</varname>.
+  </para></glossdef>
 
 </glossentry>
 
@@ -147,7 +167,7 @@
   linkend="sec-profiles" />.</para>
 
   </glossdef>
-  
+
 </glossentry>
 
 

doc/manual/hacking.xml

@@ -30,7 +30,7 @@ To build Nix itself in this shell:
 [nix-shell]$ configurePhase
 [nix-shell]$ make
 </screen>
-To install it in <literal>$(pwd)/nix</literal> and test it:
+To install it in <literal>$(pwd)/inst</literal> and test it:
 <screen>
 [nix-shell]$ make install
 [nix-shell]$ make installcheck

doc/manual/installation/env-variables.xml

@@ -21,4 +21,51 @@ in your <filename>~/.profile</filename> (or similar), like this:</para>
 <screen>
 source <replaceable>prefix</replaceable>/etc/profile.d/nix.sh</screen>
 
-</chapter>
+<section xml:id="sec-nix-ssl-cert-file">
+
+<title><envar>NIX_SSL_CERT_FILE</envar></title>
+
+<para>If you need to specify a custom certificate bundle to account
+for an HTTPS-intercepting man in the middle proxy, you must specify
+the path to the certificate bundle in the environment variable
+<envar>NIX_SSL_CERT_FILE</envar>.</para>
+
+
+<para>If you don't specify a <envar>NIX_SSL_CERT_FILE</envar>
+manually, Nix will install and use its own certificate
+bundle.</para>
+
+<procedure>
+  <step><para>Set the environment variable and install Nix</para>
+    <screen>
+$ export NIX_SSL_CERT_FILE=/etc/ssl/my-certificate-bundle.crt
+$ sh &lt;(curl https://nixos.org/nix/install)
+</screen></step>
+
+  <step><para>In the shell profile and rc files (for example,
+  <filename>/etc/bashrc</filename>, <filename>/etc/zshrc</filename>),
+  add the following line:</para>
+<programlisting>
+export NIX_SSL_CERT_FILE=/etc/ssl/my-certificate-bundle.crt
+</programlisting>
+</step>
+</procedure>
+
+<note><para>You must not add the export and then do the install, as
+the Nix installer will detect the presense of Nix configuration, and
+abort.</para></note>
+
+<section xml:id="sec-nix-ssl-cert-file-with-nix-daemon-and-macos">
+<title><envar>NIX_SSL_CERT_FILE</envar> with macOS and the Nix daemon</title>
+
+<para>On macOS you must specify the environment variable for the Nix
+daemon service, then restart it:</para>
+
+<screen>
+$ sudo launchctl setenv NIX_SSL_CERT_FILE /etc/ssl/my-certificate-bundle.crt
+$ sudo launchctl kickstart -k system/org.nixos.nix-daemon
+</screen>
+</section>
+
+</section>
+</chapter>

doc/manual/installation/installing-binary.xml

@@ -6,13 +6,30 @@
 
 <title>Installing a Binary Distribution</title>
 
-<para>If you are using Linux or macOS, the easiest way to install
-Nix is to run the following command:
+<para>If you are using Linux or macOS, the easiest way to install Nix
+is to run the following command:
 
 <screen>
-$ bash &lt;(curl https://nixos.org/nix/install)
+  $ sh &lt;(curl https://nixos.org/nix/install)
 </screen>
 
+As of Nix 2.1.0, the Nix installer will always default to creating a
+single-user installation, however opting in to the multi-user
+installation is highly recommended.
+</para>
+
+<section xml:id="sect-single-user-installation">
+  <title>Single User Installation</title>
+
+  <para>
+    To explicitly select a single-user installation on your system:
+
+    <screen>
+  sh &lt;(curl https://nixos.org/nix/install) --no-daemon
+</screen>
+  </para>
+
+<para>
 This will perform a single-user installation of Nix, meaning that
 <filename>/nix</filename> is owned by the invoking user.  You should
 run this under your usual user account, <emphasis>not</emphasis> as
@@ -33,51 +50,8 @@ and <filename>.profile</filename> to source
 the <command>NIX_INSTALLER_NO_MODIFY_PROFILE</command> environment
 variable before executing the install script to disable this
 behaviour.
-
 </para>
 
-<!--
-<para>You can also manually download and install a binary package.
-Binary packages of the latest stable release are available for Fedora,
-Debian, Ubuntu, macOS and various other systems from the <link
-xlink:href="http://nixos.org/nix/download.html">Nix homepage</link>.
-You can also get builds of the latest development release from our
-<link
-xlink:href="http://hydra.nixos.org/job/nix/master/release/latest-finished#tabs-constituents">continuous
-build system</link>.</para>
-
-<para>For Fedora, RPM packages are available.  These can be installed
-or upgraded using <command>rpm -U</command>.  For example,
-
-<screen>
-$ rpm -U nix-1.8-1.i386.rpm</screen>
-
-</para>
-
-<para>For Debian and Ubuntu, you can download a Deb package and
-install it like this:
-
-<screen>
-$ dpkg -i nix_1.8-1_amd64.deb</screen>
-
-</para>
--->
-
-<para>You can also download a binary tarball that contains Nix and all
-its dependencies.  (This is what the install script at
-<uri>https://nixos.org/nix/install</uri> does automatically.)  You
-should unpack it somewhere (e.g. in <filename>/tmp</filename>), and
-then run the script named <command>install</command> inside the binary
-tarball:
-
-<screen>
-alice$ cd /tmp
-alice$ tar xfj nix-1.8-x86_64-darwin.tar.bz2
-alice$ cd nix-1.8-x86_64-darwin
-alice$ ./install
-</screen>
-
-</para>
 
 <para>You can uninstall Nix simply by running:
 
@@ -86,5 +60,131 @@ $ rm -rf /nix
 </screen>
 
 </para>
+</section>
 
+<section xml:id="sect-multi-user-installation">
+  <title>Multi User Installation</title>
+  <para>
+    The multi-user Nix installation creates system users, and a system
+    service for the Nix daemon.
+  </para>
+
+  <itemizedlist>
+    <title>Supported Systems</title>
+
+    <listitem>
+      <para>Linux running systemd, with SELinux disabled</para>
+    </listitem>
+    <listitem><para>macOS</para></listitem>
+  </itemizedlist>
+
+  <para>
+    You can instruct the installer to perform a multi-user
+    installation on your system:
+
+    <screen>
+  sh &lt;(curl https://nixos.org/nix/install) --daemon
+</screen>
+  </para>
+
+  <para>
+    The multi-user installation of Nix will create build users between
+    the user IDs 30001 and 30032, and a group with the group ID 30000.
+
+    You should run this under your usual user account,
+    <emphasis>not</emphasis> as root. The script will invoke
+    <command>sudo</command> as needed.
+  </para>
+
+  <note><para>
+    If you need Nix to use a different group ID or user ID set, you
+    will have to download the tarball manually and <link
+    linkend="sect-nix-install-binary-tarball">edit the install
+    script</link>.
+  </para></note>
+
+  <para>
+    The installer will modify <filename>/etc/bashrc</filename>, and
+    <filename>/etc/zshrc</filename> if they exist. The installer will
+    first back up these files with a
+    <literal>.backup-before-nix</literal> extension. The installer
+    will also create <filename>/etc/profile.d/nix.sh</filename>.
+  </para>
+
+  <para>You can uninstall Nix with the following commands:
+
+<screen>
+sudo rm -rf /etc/profile/nix.sh /etc/nix /nix ~root/.nix-profile ~root/.nix-defexpr ~root/.nix-channels ~/.nix-profile ~/.nix-defexpr ~/.nix-channels
+
+# If you are on Linux with systemd, you will need to run:
+sudo systemctl stop nix-daemon.socket
+sudo systemctl stop nix-daemon.service
+sudo systemctl disable nix-daemon.socket
+sudo systemctl disable nix-daemon.service
+sudo systemctl daemon-reload
+
+# If you are on macOS, you will need to run:
+sudo launchctl unload /Library/LaunchDaemons/org.nixos.nix-daemon.plist
+sudo rm /Library/LaunchDaemons/org.nixos.nix-daemon.plist
+</screen>
+
+    There may also be references to Nix in
+    <filename>/etc/profile</filename>,
+    <filename>/etc/bashrc</filename>, and
+    <filename>/etc/zshrc</filename> which you may remove.
+  </para>
+
+</section>
+
+<section xml:id="sect-nix-install-pinned-version-url">
+  <title>Installing a pinned Nix version from a URL</title>
+
+  <para>
+    NixOS.org hosts version-specific installation URLs for all Nix
+    versions since 1.11.16, at
+    <literal>https://nixos.org/releases/nix/nix-VERSION/install</literal>.
+  </para>
+
+  <para>
+    These install scripts can be used the same as the main
+  NixOS.org installation script:
+
+  <screen>
+  sh &lt;(curl https://nixos.org/nix/install)
+</screen>
+  </para>
+
+  <para>
+    In the same directory of the install script are sha256 sums, and
+    gpg signature files.
+  </para>
+</section>
+
+<section xml:id="sect-nix-install-binary-tarball">
+  <title>Installing from a binary tarball</title>
+
+  <para>
+    You can also download a binary tarball that contains Nix and all
+    its dependencies.  (This is what the install script at
+    <uri>https://nixos.org/nix/install</uri> does automatically.)  You
+    should unpack it somewhere (e.g. in <filename>/tmp</filename>),
+    and then run the script named <command>install</command> inside
+    the binary tarball:
+
+
+<screen>
+alice$ cd /tmp
+alice$ tar xfj nix-1.8-x86_64-darwin.tar.bz2
+alice$ cd nix-1.8-x86_64-darwin
+alice$ ./install
+</screen>
+  </para>
+
+  <para>
+    If you need to edit the multi-user installation script to use
+    different group ID or a different user ID range, modify the
+    variables set in the file named
+    <filename>install-multi-user</filename>.
+  </para>
+</section>
 </chapter>

doc/manual/installation/prerequisites-source.xml

@@ -9,6 +9,9 @@
 <itemizedlist>
 
   <listitem><para>GNU Make.</para></listitem>
+  
+  <listitem><para>Bash Shell. The <literal>./configure</literal> script
+  relies on bashisms, so Bash is required.</para></listitem>
 
   <listitem><para>A version of GCC or Clang that supports C++14.</para></listitem>
 
@@ -22,12 +25,27 @@
   If your distribution does not provide it, you can get it from <link
   xlink:href="https://www.openssl.org"/>.</para></listitem>
 
+  <listitem><para>The <literal>libbrotlienc</literal> and
+  <literal>libbrotlidec</literal> libraries to provide implementation
+  of the Brotli compression algorithm. They are available for download
+  from the official repository <link
+  xlink:href="https://github.com/google/brotli" />.</para></listitem>
+
   <listitem><para>The bzip2 compressor program and the
   <literal>libbz2</literal> library.  Thus you must have bzip2
   installed, including development headers and libraries.  If your
   distribution does not provide these, you can obtain bzip2 from <link
-  xlink:href="http://www.bzip.org/"/>.</para></listitem>
+  xlink:href="https://web.archive.org/web/20180624184756/http://www.bzip.org/"
+  />.</para></listitem>
 
+  <listitem><para><literal>liblzma</literal>, which is provided by
+  XZ Utils. If your distribution does not provide this, you can
+  get it from <link xlink:href="https://tukaani.org/xz/"/>.</para></listitem>
+  
+  <listitem><para>cURL and its library. If your distribution does not
+  provide it, you can get it from <link
+  xlink:href="https://curl.haxx.se/"/>.</para></listitem>
+      
   <listitem><para>The SQLite embedded database library, version 3.6.19
   or higher.  If your distribution does not provide it, please install
   it from <link xlink:href="http://www.sqlite.org/" />.</para></listitem>
@@ -40,6 +58,10 @@
   pass the flag <option>--enable-gc</option> to
   <command>configure</command>.</para></listitem>
 
+  <listitem><para>The <literal>boost</literal> library of version
+  1.66.0 or higher. It can be obtained from the official web site
+  <link xlink:href="https://www.boost.org/" />.</para></listitem>
+
   <listitem><para>The <command>xmllint</command> and
   <command>xsltproc</command> programs to build this manual and the
   man-pages.  These are part of the <literal>libxml2</literal> and
@@ -65,6 +87,15 @@
   modify the parser or when you are building from the Git
   repository.</para></listitem>
 
+  <listitem><para>The <literal>libseccomp</literal> is used to provide
+  syscall filtering on Linux. This is an optional dependency and can
+  be disabled passing a <option>--disable-seccomp-sandboxing</option>
+  option to the <command>configure</command> script (Not recommended
+  unless your system doesn't support
+  <literal>libseccomp</literal>). To get the library, visit <link
+  xlink:href="https://github.com/seccomp/libseccomp"
+  />.</para></listitem>
+
 </itemizedlist>
 
 </section>

doc/manual/installation/supported-platforms.xml

@@ -10,7 +10,7 @@
 
 <itemizedlist>
 
-  <listitem><para>Linux (i686, x86_64).</para></listitem>
+  <listitem><para>Linux (i686, x86_64, aarch64).</para></listitem>
 
   <listitem><para>macOS (x86_64).</para></listitem>
 

doc/manual/installation/upgrading.xml

@@ -0,0 +1,22 @@
+<chapter xmlns="http://docbook.org/ns/docbook"
+      xmlns:xlink="http://www.w3.org/1999/xlink"
+      xmlns:xi="http://www.w3.org/2001/XInclude"
+      version="5.0"
+      xml:id="ch-upgrading-nix">
+
+  <title>Upgrading Nix</title>
+
+  <para>
+    Multi-user Nix users on macOS can upgrade Nix by running:
+    <command>sudo -i sh -c 'nix-channel --update &amp;&amp;
+    nix-env -iA nixpkgs.nix &amp;&amp;
+    launchctl remove org.nixos.nix-daemon &amp;&amp;
+    launchctl load /Library/LaunchDaemons/org.nixos.nix-daemon.plist'</command>
+  </para>
+
+
+  <para>
+    Single-user installations of Nix should run this:
+    <command>nix-channel --update; nix-env -iA nixpkgs.nix</command>
+  </para>
+</chapter>

doc/manual/introduction/about-nix.xml

@@ -60,7 +60,8 @@ This is because tools such as compilers don’t search in per-packages
 directories such as
 <filename>/nix/store/5lbfaxb722zp…-openssl-0.9.8d/include</filename>,
 so if a package builds correctly on your system, this is because you
-specified the dependency explicitly.</para>
+specified the dependency explicitly. This takes care of the build-time
+dependencies.</para>
 
 <para>Once a package is built, runtime dependencies are found by
 scanning binaries for the hash parts of Nix store paths (such as
@@ -261,12 +262,6 @@ xlink:href="http://nixos.org/">NixOS homepage</link>.</para>
 xlink:href="http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html">GNU
 LGPLv2.1 or (at your option) any later version</link>.</para>
 
-<para>Nix uses the <link
-xlink:href="https://github.com/arangodb/linenoise-ng">linenoise-ng
-library</link>, which has the following license:</para>
-
-<programlisting><xi:include href="../../../src/linenoise/LICENSE" parse="text" /></programlisting>
-
 </simplesect>
 
 

doc/manual/manual.xml

@@ -32,6 +32,7 @@
 
   <xi:include href="introduction/introduction.xml" />
   <xi:include href="installation/installation.xml" />
+  <xi:include href="installation/upgrading.xml" />
   <xi:include href="packages/package-management.xml" />
   <xi:include href="expressions/writing-nix-expressions.xml" />
   <xi:include href="advanced-topics/advanced-topics.xml" />

doc/manual/packages/garbage-collection.xml

@@ -52,6 +52,14 @@ garbage collector as follows:
 <screen>
 $ nix-store --gc</screen>
 
+The behaviour of the gargage collector is affected by the <literal>keep-
+derivations</literal> (default: true) and <literal>keep-outputs</literal>
+(default: false) options in the Nix configuration file. The defaults will ensure
+that all derivations that are not build-time dependencies of garbage collector roots
+will be collected but that all output paths that are not runtime dependencies
+will be collected. (This is usually what you want, but while you are developing
+it may make sense to keep outputs to ensure that rebuild times are quick.)
+
 If you are feeling uncertain, you can also first view what files would
 be deleted:
 

doc/manual/packages/s3-substituter.xml

@@ -0,0 +1,183 @@
+<?xml version="1.0" encoding="utf-8"?>
+<section xmlns="http://docbook.org/ns/docbook"
+         xmlns:xlink="http://www.w3.org/1999/xlink"
+         xmlns:xi="http://www.w3.org/2001/XInclude"
+         version="5.0"
+         xml:id="ssec-s3-substituter">
+
+<title>Serving a Nix store via AWS S3 or S3-compatible Service</title>
+
+<para>Nix has built-in support for storing and fetching store paths
+from Amazon S3 and S3 compatible services. This uses the same
+<emphasis>binary</emphasis> cache mechanism that Nix usually uses to
+fetch prebuilt binaries from <uri>cache.nixos.org</uri>.</para>
+
+<para>The following options can be specified as URL parameters to
+the S3 URL:</para>
+
+<variablelist>
+  <varlistentry><term><literal>profile</literal></term>
+  <listitem>
+    <para>
+      The name of the AWS configuration profile to use. By default
+      Nix will use the <literal>default</literal> profile.
+    </para>
+  </listitem>
+  </varlistentry>
+
+  <varlistentry><term><literal>region</literal></term>
+  <listitem>
+    <para>
+      The region of the S3 bucket. <literal>us–east-1</literal> by
+      default.
+    </para>
+
+    <para>
+      If your bucket is not in <literal>us–east-1</literal>, you
+      should always explicitly specify the region parameter.
+    </para>
+  </listitem>
+  </varlistentry>
+
+  <varlistentry><term><literal>endpoint</literal></term>
+  <listitem>
+    <para>
+      The URL to your S3-compatible service, for when not using
+      Amazon S3. Do not specify this value if you're using Amazon
+      S3.
+    </para>
+    <note><para>This endpoint must support HTTPS and will use
+    path-based addressing instead of virtual host based
+    addressing.</para></note>
+  </listitem>
+  </varlistentry>
+
+  <varlistentry><term><literal>scheme</literal></term>
+  <listitem>
+    <para>
+      The scheme used for S3 requests, <literal>https</literal>
+      (default) or <literal>http</literal>.  This option allows you to
+      disable HTTPS for binary caches which don't support it.
+    </para>
+    <note><para>HTTPS should be used if the cache might contain
+    sensitive information.</para></note>
+  </listitem>
+  </varlistentry>
+</variablelist>
+
+<para>In this example we will use the bucket named
+<literal>example-nix-cache</literal>.</para>
+
+<section xml:id="ssec-s3-substituter-anonymous-reads">
+  <title>Anonymous Reads to your S3-compatible binary cache</title>
+
+  <para>If your binary cache is publicly accessible and does not
+  require authentication, the simplest and easiest way to use Nix with
+  your S3 compatible binary cache is to use the HTTP URL for that
+  cache.</para>
+
+  <para>For AWS S3 the binary cache URL for example bucket will be
+  exactly <uri>https://example-nix-cache.s3.amazonaws.com</uri> or
+  <uri>s3://example-nix-cache</uri>. For S3 compatible binary caches,
+  consult that cache's documentation.</para>
+
+  <para>Your bucket will need the following bucket policy:</para>
+
+  <programlisting><![CDATA[
+{
+    "Id": "DirectReads",
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Sid": "AllowDirectReads",
+            "Action": [
+                "s3:GetObject",
+                "s3:GetBucketLocation"
+            ],
+            "Effect": "Allow",
+            "Resource": [
+                "arn:aws:s3:::example-nix-cache",
+                "arn:aws:s3:::example-nix-cache/*"
+            ],
+            "Principal": "*"
+        }
+    ]
+}
+]]></programlisting>
+</section>
+
+<section xml:id="ssec-s3-substituter-authenticated-reads">
+  <title>Authenticated Reads to your S3 binary cache</title>
+
+  <para>For AWS S3 the binary cache URL for example bucket will be
+  exactly <uri>s3://example-nix-cache</uri>.</para>
+
+  <para>Nix will use the <link
+  xlink:href="https://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/credentials.html#credentials-default.">default
+  credential provider chain</link> for authenticating requests to
+  Amazon S3.</para>
+
+  <para>Nix supports authenticated reads from Amazon S3 and S3
+  compatible binary caches.</para>
+
+  <para>Your bucket will need a bucket policy allowing the desired
+  users to perform the <literal>s3:GetObject</literal> and
+  <literal>s3:GetBucketLocation</literal> action on all objects in the
+  bucket. The anonymous policy in <xref
+  linkend="ssec-s3-substituter-anonymous-reads" /> can be updated to
+  have a restricted <literal>Principal</literal> to support
+  this.</para>
+</section>
+
+
+<section xml:id="ssec-s3-substituter-authenticated-writes">
+  <title>Authenticated Writes to your S3-compatible binary cache</title>
+
+  <para>Nix support fully supports writing to Amazon S3 and S3
+  compatible buckets. The binary cache URL for our example bucket will
+  be <uri>s3://example-nix-cache</uri>.</para>
+
+  <para>Nix will use the <link
+  xlink:href="https://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/credentials.html#credentials-default.">default
+  credential provider chain</link> for authenticating requests to
+  Amazon S3.</para>
+
+  <para>Your account will need the following IAM policy to
+  upload to the cache:</para>
+
+  <programlisting><![CDATA[
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Sid": "UploadToCache",
+      "Effect": "Allow",
+      "Action": [
+        "s3:AbortMultipartUpload",
+        "s3:GetBucketLocation",
+        "s3:GetObject",
+        "s3:ListBucket",
+        "s3:ListBucketMultipartUploads",
+        "s3:ListMultipartUploadParts",
+        "s3:ListObjects",
+        "s3:PutObject"
+      ],
+      "Resource": [
+        "arn:aws:s3:::example-nix-cache",
+        "arn:aws:s3:::example-nix-cache/*"
+      ]
+    }
+  ]
+}
+]]></programlisting>
+
+
+  <example><title>Uploading with a specific credential profile for Amazon S3</title>
+    <para><command>nix copy --to 's3://example-nix-cache?profile=cache-upload&amp;region=eu-west-2' nixpkgs.hello</command></para>
+  </example>
+
+  <example><title>Uploading to an S3-Compatible Binary Cache</title>
+    <para><command>nix copy --to 's3://example-nix-cache?profile=cache-upload&amp;scheme=https&amp;endpoint=minio.example.com' nixpkgs.hello</command></para>
+  </example>
+</section>
+</section>

doc/manual/packages/sharing-packages.xml

@@ -15,5 +15,6 @@ packages between machines.</para>
 <xi:include href="binary-cache-substituter.xml" />
 <xi:include href="copy-closure.xml" />
 <xi:include href="ssh-substituter.xml" />
+<xi:include href="s3-substituter.xml" />
 
 </chapter>

doc/manual/packages/ssh-substituter.xml

@@ -12,7 +12,7 @@ automatically fetching any store paths in Firefox’s closure if they
 are available on the server <literal>avalon</literal>:
 
 <screen>
-$ nix-env -i firefox --option ssh-substituter-hosts alice@avalon
+$ nix-env -i firefox --substituters ssh://alice@avalon
 </screen>
 
 This works similar to the binary cache substituter that Nix usually
@@ -31,7 +31,7 @@ an SSH passphrase interactively. Therefore, you should use
 installing it into your profile, e.g.
 
 <screen>
-$ nix-store -r /nix/store/m85bxg…-firefox-34.0.5 --option ssh-substituter-hosts alice@avalon
+$ nix-store -r /nix/store/m85bxg…-firefox-34.0.5 --substituters ssh://alice@avalon
 </screen>
 
 This is essentially equivalent to doing

doc/manual/release-notes/release-notes.xml

@@ -12,6 +12,8 @@
 </partintro>
 -->
 
+<xi:include href="rl-2.2.xml" />
+<xi:include href="rl-2.1.xml" />
 <xi:include href="rl-2.0.xml" />
 <xi:include href="rl-1.11.10.xml" />
 <xi:include href="rl-1.11.xml" />

doc/manual/release-notes/rl-2.0.xml

@@ -35,14 +35,14 @@
     available paths from the installation medium. The replacement is
     to use a chroot store as a substituter
     (e.g. <literal>--substituters /mnt</literal>), or to build into a
-    chroot store (e.g. <literal>--store /mnt --substituter /</literal>).</para>
+    chroot store (e.g. <literal>--store /mnt --substituters /</literal>).</para>
   </listitem>
 
   <listitem>
     <para>The command <command>nix-push</command> has been removed as
     part of the effort to eliminate Nix's dependency on Perl. You can
     use <command>nix copy</command> instead, e.g. <literal>nix copy
-    --to /tmp/my-binary-cache <replaceable>paths…</replaceable></literal></para>
+    --to file:///tmp/my-binary-cache <replaceable>paths…</replaceable></literal></para>
   </listitem>
 
   <listitem>
@@ -323,13 +323,12 @@
 
       <listitem>
         <para><command>nix sign-paths</command> signs store
-        paths. (TODO: add examples)</para>
+        paths.</para>
       </listitem>
 
       <listitem>
         <para><command>nix copy-sigs</command> copies signatures from
-        one store to another. (TODO: add examples and
-        tests)</para>
+        one store to another.</para>
       </listitem>
 
       <listitem>
@@ -460,7 +459,7 @@
         secret keys specified by the <option>secret-key-files</option>
         store option. Secret/public key pairs can be generated using
         <command>nix-store
-        --generate-binary-cache-key</command>. (TODO: rename)</para>
+        --generate-binary-cache-key</command>.</para>
 
         <para>In addition, locally-built store paths are marked as
         “ultimately trusted”, but this bit is not propagated when
@@ -537,7 +536,7 @@
     xlink:href="https://github.com/NixOS/nix/commit/8bdf83f936adae6f2c907a6d2541e80d4120f051">no
     longer</link> a fatal error if build rounds produce different
     output. Also, a hook named <option>diff-hook</option> is <link
-    xlink:href="https://github.com/NixOS/nix/commit/9a313469a4bdea2d1e8df24d16289dc2a172a169w">provided</link>
+    xlink:href="https://github.com/NixOS/nix/commit/9a313469a4bdea2d1e8df24d16289dc2a172a169">provided</link>
     to allow you to run tools such as <command>diffoscope</command>
     when build rounds produce different output.</para>
   </listitem>
@@ -723,8 +722,7 @@ configureFlags = "--prefix=${placeholder "out"} --includedir=${placeholder "dev"
 
     will cause the <envar>configureFlags</envar> environment variable
     to contain the actual store paths corresponding to the
-    <literal>out</literal> and <literal>dev</literal> outputs. TODO:
-    add docs.</para>
+    <literal>out</literal> and <literal>dev</literal> outputs.</para>
   </listitem>
 
 </itemizedlist>

doc/manual/release-notes/rl-2.1.xml

@@ -0,0 +1,133 @@
+<section xmlns="http://docbook.org/ns/docbook"
+      xmlns:xlink="http://www.w3.org/1999/xlink"
+      xmlns:xi="http://www.w3.org/2001/XInclude"
+      version="5.0"
+      xml:id="ssec-relnotes-2.1">
+
+<title>Release 2.1 (2018-09-02)</title>
+
+<para>This is primarily a bug fix release. It also reduces memory
+consumption in certain situations. In addition, it has the following
+new features:</para>
+
+<itemizedlist>
+
+  <listitem>
+    <para>The Nix installer will no longer default to the Multi-User
+    installation for macOS. You can still <link
+    linkend="sect-multi-user-installation">instruct the installer to
+    run in multi-user mode</link>.
+    </para>
+  </listitem>
+
+  <listitem>
+    <para>The Nix installer now supports performing a Multi-User
+    installation for Linux computers which are running systemd. You
+    can <link
+    linkend="sect-multi-user-installation">select a Multi-User installation</link> by passing the
+    <option>--daemon</option> flag to the installer: <command>sh &lt;(curl
+    https://nixos.org/nix/install) --daemon</command>.
+    </para>
+
+    <para>The multi-user installer cannot handle systems with SELinux.
+    If your system has SELinux enabled, you can <link
+    linkend="sect-single-user-installation">force the installer to run
+    in single-user mode</link>.</para>
+  </listitem>
+
+  <listitem>
+    <para>New builtin functions:
+    <literal>builtins.bitAnd</literal>,
+    <literal>builtins.bitOr</literal>,
+    <literal>builtins.bitXor</literal>,
+    <literal>builtins.fromTOML</literal>,
+    <literal>builtins.concatMap</literal>,
+    <literal>builtins.mapAttrs</literal>.
+    </para>
+  </listitem>
+
+  <listitem>
+    <para>The S3 binary cache store now supports uploading NARs larger
+    than 5 GiB.</para>
+  </listitem>
+
+  <listitem>
+    <para>The S3 binary cache store now supports uploading to
+    S3-compatible services with the <literal>endpoint</literal>
+    option.</para>
+  </listitem>
+
+  <listitem>
+    <para>The flag <option>--fallback</option> is no longer required
+    to recover from disappeared NARs in binary caches.</para>
+  </listitem>
+
+  <listitem>
+    <para><command>nix-daemon</command> now respects
+    <option>--store</option>.</para>
+  </listitem>
+
+  <listitem>
+    <para><command>nix run</command> now respects
+    <varname>nix-support/propagated-user-env-packages</varname>.</para>
+  </listitem>
+
+</itemizedlist>
+
+<para>This release has contributions from
+
+Adrien Devresse,
+Aleksandr Pashkov,
+Alexandre Esteves,
+Amine Chikhaoui,
+Andrew Dunham,
+Asad Saeeduddin,
+aszlig,
+Ben Challenor,
+Ben Gamari,
+Benjamin Hipple,
+Bogdan Seniuc,
+Corey O'Connor,
+Daiderd Jordan,
+Daniel Peebles,
+Daniel Poelzleithner,
+Danylo Hlynskyi,
+Dmitry Kalinkin,
+Domen Kožar,
+Doug Beardsley,
+Eelco Dolstra,
+Erik Arvstedt,
+Félix Baylac-Jacqué,
+Gleb Peregud,
+Graham Christensen,
+Guillaume Maudoux,
+Ivan Kozik,
+John Arnold,
+Justin Humm,
+Linus Heckemann,
+Lorenzo Manacorda,
+Matthew Justin Bauer,
+Matthew O'Gorman,
+Maximilian Bosch,
+Michael Bishop,
+Michael Fiano,
+Michael Mercier,
+Michael Raskin,
+Michael Weiss,
+Nicolas Dudebout,
+Peter Simons,
+Ryan Trinkle,
+Samuel Dionne-Riel,
+Sean Seefried,
+Shea Levy,
+Symphorien Gibol,
+Tim Engler,
+Tim Sears,
+Tuomas Tynkkynen,
+volth,
+Will Dietz,
+Yorick van Pelt and
+zimbatm.
+</para>
+
+</section>

doc/manual/release-notes/rl-2.2.xml

@@ -0,0 +1,143 @@
+<section xmlns="http://docbook.org/ns/docbook"
+      xmlns:xlink="http://www.w3.org/1999/xlink"
+      xmlns:xi="http://www.w3.org/2001/XInclude"
+      version="5.0"
+      xml:id="ssec-relnotes-2.2">
+
+<title>Release 2.2 (2019-01-11)</title>
+
+<para>This is primarily a bug fix release. It also has the following
+changes:</para>
+
+<itemizedlist>
+
+  <listitem>
+    <para>In derivations that use structured attributes (i.e. that
+    specify set the <varname>__structuredAttrs</varname> attribute to
+    <literal>true</literal> to cause all attributes to be passed to
+    the builder in JSON format), you can now specify closure checks
+    per output, e.g.:
+
+<programlisting>
+outputChecks."out" = {
+  # The closure of 'out' must not be larger than 256 MiB.
+  maxClosureSize = 256 * 1024 * 1024;
+
+  # It must not refer to C compiler or to the 'dev' output.
+  disallowedRequisites = [ stdenv.cc "dev" ];
+};
+
+outputChecks."dev" = {
+  # The 'dev' output must not be larger than 128 KiB.
+  maxSize = 128 * 1024;
+};
+</programlisting>
+
+    </para>
+  </listitem>
+
+
+  <listitem>
+    <para>The derivation attribute
+    <varname>requiredSystemFeatures</varname> is now enforced for
+    local builds, and not just to route builds to remote builders.
+    The supported features of a machine can be specified through the
+    configuration setting <varname>system-features</varname>.</para>
+
+    <para>By default, <varname>system-features</varname> includes
+    <literal>kvm</literal> if <filename>/dev/kvm</filename>
+    exists. For compatibility, it also includes the pseudo-features
+    <literal>nixos-test</literal>, <literal>benchmark</literal> and
+    <literal>big-parallel</literal> which are used by Nixpkgs to route
+    builds to particular Hydra build machines.</para>
+
+  </listitem>
+
+  <listitem>
+    <para>Sandbox builds are now enabled by default on Linux.</para>
+  </listitem>
+
+  <listitem>
+    <para>The new command <command>nix doctor</command> shows
+    potential issues with your Nix installation.</para>
+  </listitem>
+
+  <listitem>
+    <para>The <literal>fetchGit</literal> builtin function now uses a
+    caching scheme that puts different remote repositories in distinct
+    local repositories, rather than a single shared repository. This
+    may require more disk space but is faster.</para>
+  </listitem>
+
+  <listitem>
+    <para>The <literal>dirOf</literal> builtin function now works on
+    relative paths.</para>
+  </listitem>
+
+  <listitem>
+    <para>Nix now supports <link
+    xlink:href="https://www.w3.org/TR/SRI/">SRI hashes</link>,
+    allowing the hash algorithm and hash to be specified in a single
+    string. For example, you can write:
+
+<programlisting>
+import &lt;nix/fetchurl.nix> {
+  url = https://nixos.org/releases/nix/nix-2.1.3/nix-2.1.3.tar.xz;
+  hash = "sha256-XSLa0FjVyADWWhFfkZ2iKTjFDda6mMXjoYMXLRSYQKQ=";
+};
+</programlisting>
+
+    instead of
+
+<programlisting>
+import &lt;nix/fetchurl.nix> {
+  url = https://nixos.org/releases/nix/nix-2.1.3/nix-2.1.3.tar.xz;
+  sha256 = "5d22dad058d5c800d65a115f919da22938c50dd6ba98c5e3a183172d149840a4";
+};
+</programlisting>
+
+    </para>
+
+    <para>In fixed-output derivations, the
+    <varname>outputHashAlgo</varname> attribute is no longer mandatory
+    if <varname>outputHash</varname> specifies the hash.</para>
+
+    <para><command>nix hash-file</command> and <command>nix
+    hash-path</command> now print hashes in SRI format by
+    default. They also use SHA-256 by default instead of SHA-512
+    because that's what we use most of the time in Nixpkgs.</para>
+  </listitem>
+
+  <listitem>
+    <para>Integers are now 64 bits on all platforms.</para>
+  </listitem>
+
+  <listitem>
+    <para>The evaluator now prints profiling statistics (enabled via
+    the <envar>NIX_SHOW_STATS</envar> and
+    <envar>NIX_COUNT_CALLS</envar> environment variables) in JSON
+    format.</para>
+  </listitem>
+
+  <listitem>
+    <para>The option <option>--xml</option> in <command>nix-store
+    --query</command> has been removed. Instead, there now is an
+    option <option>--graphml</option> to output the dependency graph
+    in GraphML format.</para>
+  </listitem>
+
+  <listitem>
+    <para>All <filename>nix-*</filename> commands are now symlinks to
+    <filename>nix</filename>. This saves a bit of disk space.</para>
+  </listitem>
+
+  <listitem>
+    <para><command>nix repl</command> now uses
+    <literal>libeditline</literal> or
+    <literal>libreadline</literal>.</para>
+  </listitem>
+
+</itemizedlist>
+
+</section>
+

local.mk

@@ -6,9 +6,7 @@ dist-files += configure config.h.in nix.spec perl/configure
 
 clean-files += Makefile.config
 
-GLOBAL_CXXFLAGS += -I . -I src -I src/libutil -I src/libstore -I src/libmain -I src/libexpr
+GLOBAL_CXXFLAGS += -I . -I src -I src/libutil -I src/libstore -I src/libmain -I src/libexpr -I src/nix
 
 $(foreach i, config.h $(call rwildcard, src/lib*, *.hh), \
   $(eval $(call install-file-in, $(i), $(includedir)/nix, 0644)))
-
-$(foreach i, $(call rwildcard, src/boost, *.hpp), $(eval $(call install-file-in, $(i), $(includedir)/nix/$(patsubst src/%/,%,$(dir $(i))), 0644)))

maintainers/upload-release.pl

@@ -6,6 +6,7 @@ use Data::Dumper;
 use File::Basename;
 use File::Path;
 use File::Slurp;
+use File::Copy;
 use JSON::PP;
 use LWP::UserAgent;
 
@@ -54,7 +55,7 @@ sub downloadFile {
 
     my $buildInfo = decode_json(fetch("$evalUrl/job/$jobName", 'application/json'));
 
-    my $srcFile = $buildInfo->{buildproducts}->{$productNr}->{path} or die;
+    my $srcFile = $buildInfo->{buildproducts}->{$productNr}->{path} or die "job '$jobName' lacks product $productNr\n";
     $dstName //= basename($srcFile);
     my $dstFile = "$releaseDir/" . $dstName;
 
@@ -75,16 +76,22 @@ sub downloadFile {
 
     write_file("$dstFile.sha256", $sha256_expected);
 
+    if (! -e "$dstFile.asc") {
+        system("gpg2 --detach-sign --armor $dstFile") == 0 or die "unable to sign $dstFile\n";
+    }
+
     return ($dstFile, $sha256_expected);
 }
 
-downloadFile("tarball", "2"); # PDF
-downloadFile("tarball", "3"); # .tar.bz2
-my ($tarball, $tarballHash) = downloadFile("tarball", "4"); # .tar.xz
-my ($tarball_i686_linux, $tarball_i686_linux_hash) = downloadFile("binaryTarball.i686-linux", "1");
-my ($tarball_x86_64_linux, $tarball_x86_64_linux_hash) = downloadFile("binaryTarball.x86_64-linux", "1");
-my ($tarball_aarch64_linux, $tarball_aarch64_linux_hash) = downloadFile("binaryTarball.aarch64-linux", "1");
-my ($tarball_x86_64_darwin, $tarball_x86_64_darwin_hash) = downloadFile("binaryTarball.x86_64-darwin", "1");
+downloadFile("tarball", "2"); # .tar.bz2
+my ($tarball, $tarballHash) = downloadFile("tarball", "3"); # .tar.xz
+downloadFile("binaryTarball.i686-linux", "1");
+downloadFile("binaryTarball.x86_64-linux", "1");
+downloadFile("binaryTarball.aarch64-linux", "1");
+downloadFile("binaryTarball.x86_64-darwin", "1");
+downloadFile("installerScript", "1");
+
+exit if $version =~ /pre/;
 
 # Update Nixpkgs in a very hacky way.
 system("cd $nixpkgsDir && git pull") == 0 or die;
@@ -144,12 +151,6 @@ system("cd $siteDir && git pull") == 0 or die;
 write_file("$siteDir/nix-release.tt",
            "[%-\n" .
            "latestNixVersion = \"$version\"\n" .
-           "nix_hash_i686_linux = \"$tarball_i686_linux_hash\"\n" .
-           "nix_hash_x86_64_linux = \"$tarball_x86_64_linux_hash\"\n" .
-           "nix_hash_aarch64_linux = \"$tarball_aarch64_linux_hash\"\n" .
-           "nix_hash_x86_64_darwin = \"$tarball_x86_64_darwin_hash\"\n" .
            "-%]\n");
 
-system("cd $siteDir && nix-shell --run 'make nix/install nix/install.sig'") == 0 or die;
-
 system("cd $siteDir && git commit -a -m 'Nix $version released'") == 0 or die;

misc/docker/Dockerfile

@@ -1,26 +0,0 @@
-FROM alpine
-
-# Enable HTTPS support in wget.
-RUN apk add --update openssl
-
-# Download Nix and install it into the system.
-RUN wget -O- https://nixos.org/releases/nix/nix-1.11.14/nix-1.11.14-x86_64-linux.tar.bz2 | bzcat - | tar xf - \
-  && addgroup -g 30000 -S nixbld \
-  && for i in $(seq 1 30); do adduser -S -D -h /var/empty -g "Nix build user $i" -u $((30000 + i)) -G nixbld nixbld$i ; done \
-  && mkdir -m 0755 /nix && USER=root sh nix-*-x86_64-linux/install \
-  && ln -s /nix/var/nix/profiles/default/etc/profile.d/nix.sh /etc/profile.d/ \
-  && rm -r /nix-*-x86_64-linux \
-  && rm -r /var/cache/apk/*
-
-ONBUILD ENV \
-    ENV=/etc/profile \
-    PATH=/nix/var/nix/profiles/default/bin:/nix/var/nix/profiles/default/sbin:/bin:/sbin:/usr/bin:/usr/sbin \
-    GIT_SSL_CAINFO=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt \
-    NIX_SSL_CERT_FILE=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt
-
-ENV \
-    ENV=/etc/profile \
-    PATH=/nix/var/nix/profiles/default/bin:/nix/var/nix/profiles/default/sbin:/bin:/sbin:/usr/bin:/usr/sbin \
-    GIT_SSL_CAINFO=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt \
-    NIX_SSL_CERT_FILE=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt \
-    NIX_PATH=/nix/var/nix/profiles/per-user/root/channels

misc/launchd/org.nixos.nix-daemon.plist.in

@@ -2,8 +2,15 @@
 <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
 <plist version="1.0">
   <dict>
+    <key>EnvironmentVariables</key>
+    <dict>
+      <key>OBJC_DISABLE_INITIALIZE_FORK_SAFETY</key>
+      <string>YES</string>
+    </dict>
     <key>Label</key>
     <string>org.nixos.nix-daemon</string>
+    <key>KeepAlive</key>
+    <true/>
     <key>RunAtLoad</key>
     <true/>
     <key>Program</key>

mk/libraries.mk

@@ -125,7 +125,7 @@ define build-library
     $(1)_PATH := $$(_d)/$$($(1)_NAME).a
 
     $$($(1)_PATH): $$($(1)_OBJS) | $$(_d)/
-	$(trace-ar) ar crs $$@ $$?
+	$(trace-ar) $(AR) crs $$@ $$?
 
     $(1)_LDFLAGS_USE += $$($(1)_PATH) $$($(1)_LDFLAGS)
 

mk/programs.mk

@@ -51,7 +51,7 @@ define build-program
   else
 
     $(DESTDIR)$$($(1)_INSTALL_PATH): $$($(1)_PATH) | $(DESTDIR)$$($(1)_INSTALL_DIR)/
-	install -t $$($(1)_INSTALL_DIR) $$<
+	install -t $(DESTDIR)$$($(1)_INSTALL_DIR) $$<
 
   endif
 

nix.spec.in

@@ -3,31 +3,47 @@
 %global nixbld_user "nix-builder-"
 %global nixbld_group "nixbld"
 
+# NOTE: BUILD on EL7 requires
+# - Centos / RHEL7 software collection repository 
+#   yum install centos-release-scl
+#
+# - Recent boost backport
+#   curl https://copr.fedorainfracloud.org/coprs/whosthere/boost/repo/epel-7/whosthere-boost-epel-7.repo -o /etc/yum.repos.d/whosthere-boost-epel-7.repo
+#
+
+# Disable documentation generation
+# necessary on some platforms
+%bcond_without docgen
+
 Summary: The Nix software deployment system
 Name: nix
 Version: @PACKAGE_VERSION@
 Release: 2%{?dist}
 License: LGPLv2+
-%if 0%{?rhel} && 0%{?rhel} < 7
 Group: Applications/System
-%endif
 URL: http://nixos.org/
 Source0: %{name}-%{version}.tar.bz2
-%if 0%{?el5}
-BuildRoot: %(mktemp -ud %{_tmppath}/%{name}-%{version}-%{release}-XXXXXX)
-%endif
+
 Requires: curl
 Requires: bzip2
 Requires: gzip
 Requires: xz
-Requires: libseccomp
+BuildRequires: bison
+BuildRequires: boost-devel >= 1.60
 BuildRequires: bzip2-devel
-BuildRequires: sqlite-devel
+
+# for RHEL <= 7, we need software collections for a C++14 compatible compatible compiler
+%if 0%{?rhel}
+BuildRequires: devtoolset-7-gcc
+BuildRequires: devtoolset-7-gcc-c++
+%endif
+
+BuildRequires: flex
 BuildRequires: libcurl-devel
 BuildRequires: libseccomp-devel
-
-# Hack to make that shitty RPM scanning hack shut up.
-Provides: perl(Nix::SSH)
+BuildRequires: openssl-devel
+BuildRequires: sqlite-devel
+BuildRequires: xz-devel
 
 %description
 Nix is a purely functional package manager. It allows multiple
@@ -39,9 +55,6 @@ it can be used equally well under other Unix systems.
 
 %package        devel
 Summary:        Development files for %{name}
-%if 0%{?rhel} && 0%{?rhel} < 7
-Group:          Development/Libraries
-%endif
 Requires:       %{name}%{?_isa} = %{version}-%{release}
 
 %description   devel
@@ -51,9 +64,6 @@ developing applications that use %{name}.
 
 %package doc
 Summary:        Documentation files for %{name}
-%if 0%{?rhel} && 0%{?rhel} < 7
-Group:          Documentation
-%endif
 BuildArch:      noarch
 Requires:       %{name} = %{version}-%{release}
 
@@ -65,20 +75,25 @@ The %{name}-doc package contains documentation files for %{name}.
 
 
 %build
+%if 0%{?rhel}
+source /opt/rh/devtoolset-7/enable 
+%endif
 extraFlags=
 # - override docdir so large documentation files are owned by the
 #   -doc subpackage
 # - set localstatedir by hand to the preferred nix value
 %configure --localstatedir=/nix/var \
+	   %{!?without_docgen:--disable-doc-gen} \
            --docdir=%{_defaultdocdir}/%{name}-doc-%{version} \
            $extraFlags
-make -j$NIX_BUILD_CORES -l$NIX_BUILD_CORES
+make V=1 %{?_smp_mflags}
 
 
 %install
-%if 0%{?el5}
-rm -rf $RPM_BUILD_ROOT
+%if 0%{?rhel}
+source /opt/rh/devtoolset-7/enable 
 %endif
+
 make DESTDIR=$RPM_BUILD_ROOT install
 
 find $RPM_BUILD_ROOT -name '*.la' -exec rm -f {} ';'
@@ -128,6 +143,7 @@ systemctl start  nix-daemon.socket
 %endif
 
 %files
+%license COPYING
 %{_bindir}/nix*
 %{_libdir}/*.so
 %{_prefix}/libexec/*
@@ -136,9 +152,11 @@ systemctl start  nix-daemon.socket
 %{_prefix}/lib/systemd/system/nix-daemon.service
 %endif
 %{_datadir}/nix
-%{_mandir}/man1/*.1*
-%{_mandir}/man5/*.5*
-%{_mandir}/man8/*.8*
+#%if ! %{without docgen}
+#%{_mandir}/man1/*.1*
+#%{_mandir}/man5/*.5*
+#%{_mandir}/man8/*.8*
+#%endif
 %config(noreplace) %{_sysconfdir}/profile.d/nix.sh
 %config(noreplace) %{_sysconfdir}/profile.d/nix-daemon.sh
 /nix
@@ -147,6 +165,9 @@ systemctl start  nix-daemon.socket
 %{_includedir}/nix
 %{_prefix}/lib/pkgconfig/*.pc
 
-%files doc
-%docdir %{_defaultdocdir}/%{name}-doc-%{version}
-%{_defaultdocdir}/%{name}-doc-%{version}
+
+#%if ! %{without docgen}
+#%files doc
+#%docdir %{_defaultdocdir}/%{name}-doc-%{version}
+#%{_defaultdocdir}/%{name}-doc-%{version}
+#%endif

perl/Makefile

@@ -1,6 +1,6 @@
 makefiles = local.mk
 
-GLOBAL_CXXFLAGS += -std=c++14 -g -Wall
+GLOBAL_CXXFLAGS += -g -Wall
 
 -include Makefile.config
 

perl/configure.ac

@@ -1,4 +1,4 @@
-AC_INIT(nix-perl, m4_esyscmd([bash -c "echo -n $(cat ../version)$VERSION_SUFFIX"]))
+AC_INIT(nix-perl, m4_esyscmd([bash -c "echo -n $(cat ../.version)$VERSION_SUFFIX"]))
 AC_CONFIG_SRCDIR(MANIFEST)
 AC_CONFIG_AUX_DIR(../config)
 

perl/lib/Nix/Store.xs

@@ -27,7 +27,7 @@ static ref<Store> store()
     static std::shared_ptr<Store> _store;
     if (!_store) {
         try {
-            settings.loadConfFile();
+            loadConfFile();
             settings.lockCPU = false;
             _store = openStore();
         } catch (Error & e) {

release-common.nix

@@ -1,9 +1,11 @@
 { pkgs }:
 
+with pkgs;
+
 rec {
   # Use "busybox-sandbox-shell" if present,
   # if not (legacy) fallback and hope it's sufficient.
-  sh = pkgs.busybox-sandbox-shell or (pkgs.busybox.override {
+  sh = pkgs.busybox-sandbox-shell or (busybox.override {
     useMusl = true;
     enableStatic = true;
     enableMinimal = true;
@@ -28,9 +30,50 @@ rec {
   });
 
   configureFlags =
-    [ "--disable-init-state"
+    [
       "--enable-gc"
-    ] ++ pkgs.lib.optionals pkgs.stdenv.isLinux [
+    ] ++ lib.optionals stdenv.isLinux [
       "--with-sandbox-shell=${sh}/bin/busybox"
     ];
+
+  tarballDeps =
+    [ bison
+      flex
+      libxml2
+      libxslt
+      docbook5
+      docbook5_xsl
+      autoconf-archive
+      autoreconfHook
+    ];
+
+  buildDeps =
+    [ curl
+      bzip2 xz brotli editline
+      openssl pkgconfig sqlite boehmgc
+      boost
+
+      # Tests
+      git
+      mercurial
+    ]
+    ++ lib.optionals stdenv.isLinux [libseccomp utillinuxMinimal]
+    ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium
+    ++ lib.optional (stdenv.isLinux || stdenv.isDarwin)
+      ((aws-sdk-cpp.override {
+        apis = ["s3" "transfer"];
+        customMemoryManagement = false;
+      }).overrideDerivation (args: {
+        /*
+        patches = args.patches or [] ++ [ (fetchpatch {
+          url = https://github.com/edolstra/aws-sdk-cpp/commit/3e07e1f1aae41b4c8b340735ff9e8c735f0c063f.patch;
+          sha256 = "1pij0v449p166f9l29x7ppzk8j7g9k9mp15ilh5qxp29c7fnvxy2";
+        }) ];
+        */
+      }));
+
+  perlDeps =
+    [ perl
+      perlPackages.DBDSQLite
+    ];
 }

release.nix

@@ -1,5 +1,5 @@
 { nix ? builtins.fetchGit ./.
-, nixpkgs ? builtins.fetchGit { url = https://github.com/NixOS/nixpkgs.git; ref = "nix-2.0"; }
+, nixpkgs ? builtins.fetchGit { url = https://github.com/NixOS/nixpkgs-channels.git; ref = "nixos-19.03"; }
 , officialRelease ? false
 , systems ? [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ]
 }:
@@ -14,26 +14,22 @@ let
     tarball =
       with pkgs;
 
+      with import ./release-common.nix { inherit pkgs; };
+
       releaseTools.sourceTarball {
         name = "nix-tarball";
-        version = builtins.readFile ./version;
+        version = builtins.readFile ./.version;
         versionSuffix = if officialRelease then "" else "pre${toString nix.revCount}_${nix.shortRev}";
         src = nix;
         inherit officialRelease;
 
-        buildInputs =
-          [ curl bison flex libxml2 libxslt
-            bzip2 xz brotli
-            pkgconfig sqlite libsodium boehmgc
-            docbook5 docbook5_xsl
-            autoconf-archive
-          ] ++ lib.optional stdenv.isLinux libseccomp;
+        buildInputs = tarballDeps ++ buildDeps;
 
         configureFlags = "--enable-gc";
 
         postUnpack = ''
-          (cd source && find . -type f) | cut -c3- > source/.dist-files
-          cat source/.dist-files
+          (cd $sourceRoot && find . -type f) | cut -c3- > $sourceRoot/.dist-files
+          cat $sourceRoot/.dist-files
         '';
 
         preConfigure = ''
@@ -59,7 +55,9 @@ let
 
     build = pkgs.lib.genAttrs systems (system:
 
-      with import nixpkgs { inherit system; };
+      let pkgs = import nixpkgs { inherit system; }; in
+
+      with pkgs;
 
       with import ./release-common.nix { inherit pkgs; };
 
@@ -67,22 +65,15 @@ let
         name = "nix";
         src = tarball;
 
-        buildInputs =
-          [ curl
-            bzip2 xz brotli
-            openssl pkgconfig sqlite boehmgc
+        buildInputs = buildDeps;
 
-            # Tests
-            git
-            mercurial
-          ]
-          ++ lib.optional stdenv.isLinux libseccomp
-          ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium
-          ++ lib.optional (stdenv.isLinux || stdenv.isDarwin)
-            (aws-sdk-cpp.override {
-              apis = ["s3"];
-              customMemoryManagement = false;
-            });
+        preConfigure =
+          # Copy libboost_context so we don't get all of Boost in our closure.
+          # https://github.com/NixOS/nixpkgs/issues/45462
+          ''
+            mkdir -p $out/lib
+            cp ${boost}/lib/libboost_context* $out/lib
+          '';
 
         configureFlags = configureFlags ++
           [ "--sysconfdir=/etc" ];
@@ -91,8 +82,6 @@ let
 
         makeFlags = "profiledir=$(out)/etc/profile.d";
 
-        preBuild = "unset NIX_INDENT_MAKE";
-
         installFlags = "sysconfdir=$(out)/etc";
 
         doInstallCheck = true;
@@ -109,7 +98,7 @@ let
         src = tarball;
 
         buildInputs =
-          [ (builtins.getAttr system jobs.build) curl bzip2 xz pkgconfig pkgs.perl ]
+          [ jobs.build.${system} curl bzip2 xz pkgconfig pkgs.perl boost ]
           ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium;
 
         configureFlags = ''
@@ -120,8 +109,6 @@ let
         enableParallelBuilding = true;
 
         postUnpack = "sourceRoot=$sourceRoot/perl";
-
-        preBuild = "unset NIX_INDENT_MAKE";
       });
 
 
@@ -132,30 +119,51 @@ let
       let
         toplevel = builtins.getAttr system jobs.build;
         version = toplevel.src.version;
+        installerClosureInfo = closureInfo { rootPaths = [ toplevel cacert ]; };
       in
 
       runCommand "nix-binary-tarball-${version}"
-        { exportReferencesGraph = [ "closure1" toplevel "closure2" cacert ];
-          buildInputs = [ perl ] ++ lib.optional (system != "aarch64-linux") shellcheck;
+        { nativeBuildInputs = lib.optional (system != "aarch64-linux") shellcheck;
           meta.description = "Distribution-independent Nix bootstrap binaries for ${system}";
         }
         ''
-          storePaths=$(perl ${pathsFromGraph} ./closure1 ./closure2)
-          printRegistration=1 perl ${pathsFromGraph} ./closure1 ./closure2 > $TMPDIR/reginfo
+          cp ${installerClosureInfo}/registration $TMPDIR/reginfo
           substitute ${./scripts/install-nix-from-closure.sh} $TMPDIR/install \
             --subst-var-by nix ${toplevel} \
             --subst-var-by cacert ${cacert}
-          substitute ${./scripts/install-darwin-multi-user.sh} $TMPDIR/install-darwin-multi-user \
+
+          substitute ${./scripts/install-darwin-multi-user.sh} $TMPDIR/install-darwin-multi-user.sh \
+            --subst-var-by nix ${toplevel} \
+            --subst-var-by cacert ${cacert}
+          substitute ${./scripts/install-systemd-multi-user.sh} $TMPDIR/install-systemd-multi-user.sh \
+            --subst-var-by nix ${toplevel} \
+            --subst-var-by cacert ${cacert}
+          substitute ${./scripts/install-multi-user.sh} $TMPDIR/install-multi-user \
             --subst-var-by nix ${toplevel} \
             --subst-var-by cacert ${cacert}
 
           if type -p shellcheck; then
-            shellcheck -e SC1090 $TMPDIR/install
-            shellcheck -e SC1091,SC2002 $TMPDIR/install-darwin-multi-user
+            # SC1090: Don't worry about not being able to find
+            #         $nix/etc/profile.d/nix.sh
+            shellcheck --exclude SC1090 $TMPDIR/install
+            shellcheck $TMPDIR/install-darwin-multi-user.sh
+            shellcheck $TMPDIR/install-systemd-multi-user.sh
+
+            # SC1091: Don't panic about not being able to source
+            #         /etc/profile
+            # SC2002: Ignore "useless cat" "error", when loading
+            #         .reginfo, as the cat is a much cleaner
+            #         implementation, even though it is "useless"
+            # SC2116: Allow ROOT_HOME=$(echo ~root) for resolving
+            #         root's home directory
+            shellcheck --external-sources \
+              --exclude SC1091,SC2002,SC2116 $TMPDIR/install-multi-user
           fi
 
           chmod +x $TMPDIR/install
-          chmod +x $TMPDIR/install-darwin-multi-user
+          chmod +x $TMPDIR/install-darwin-multi-user.sh
+          chmod +x $TMPDIR/install-systemd-multi-user.sh
+          chmod +x $TMPDIR/install-multi-user
           dir=nix-${version}-${system}
           fn=$out/$dir.tar.bz2
           mkdir -p $out/nix-support
@@ -167,26 +175,23 @@ let
             --transform "s,$TMPDIR/install,$dir/install," \
             --transform "s,$TMPDIR/reginfo,$dir/.reginfo," \
             --transform "s,$NIX_STORE,$dir/store,S" \
-            $TMPDIR/install $TMPDIR/install-darwin-multi-user $TMPDIR/reginfo $storePaths
+            $TMPDIR/install $TMPDIR/install-darwin-multi-user.sh \
+            $TMPDIR/install-systemd-multi-user.sh \
+            $TMPDIR/install-multi-user $TMPDIR/reginfo \
+            $(cat ${installerClosureInfo}/store-paths)
         '');
 
 
     coverage =
-      with import nixpkgs { system = "x86_64-linux"; };
+      with pkgs;
+
+      with import ./release-common.nix { inherit pkgs; };
 
       releaseTools.coverageAnalysis {
         name = "nix-build";
         src = tarball;
 
-        buildInputs =
-          [ curl bzip2 openssl pkgconfig sqlite xz libsodium libseccomp
-            # These are for "make check" only:
-            graphviz libxml2 libxslt git mercurial
-          ];
-
-        configureFlags = ''
-          --disable-init-state
-        '';
+        buildInputs = buildDeps;
 
         dontInstall = false;
 
@@ -201,17 +206,14 @@ let
       };
 
 
-    rpm_fedora25i386 = makeRPM_i686 (diskImageFuns: diskImageFuns.fedora25i386) [ "libsodium-devel" ];
-    rpm_fedora25x86_64 = makeRPM_x86_64 (diskImageFunsFun: diskImageFunsFun.fedora25x86_64) [ "libsodium-devel" ];
+    #rpm_fedora27x86_64 = makeRPM_x86_64 (diskImageFunsFun: diskImageFunsFun.fedora27x86_64) [ ];
 
 
     #deb_debian8i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.debian8i386) [ "libsodium-dev" ] [ "libsodium13" ];
     #deb_debian8x86_64 = makeDeb_x86_64 (diskImageFunsFun: diskImageFunsFun.debian8x86_64) [ "libsodium-dev" ] [ "libsodium13" ];
 
-    deb_ubuntu1604i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1604i386) [ "libsodium-dev" ] [ "libsodium18" ];
-    deb_ubuntu1604x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1604x86_64) [ "libsodium-dev" ] [ "libsodium18" ];
-    deb_ubuntu1610i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1610i386) [ "libsodium-dev" ] [ "libsodium18" ];
-    deb_ubuntu1610x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1610x86_64) [ "libsodium-dev" ] [ "libsodium18" ];
+    #deb_ubuntu1710i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1710i386) [ ] [ "libsodium18" ];
+    #deb_ubuntu1710x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1710x86_64) [ ] [ "libsodium18" "libboost-context1.62.0" ];
 
 
     # System tests.
@@ -239,6 +241,7 @@ let
         { diskImage = vmTools.diskImages.ubuntu1204x86_64;
         }
         ''
+          set -x
           useradd -m alice
           su - alice -c 'tar xf ${binaryTarball.x86_64-linux}/*.tar.*'
           mkdir /dest-nix
@@ -247,6 +250,17 @@ let
           su - alice -c '_NIX_INSTALLER_TEST=1 ./nix-*/install'
           su - alice -c 'nix-store --verify'
           su - alice -c 'PAGER= nix-store -qR ${build.x86_64-linux}'
+
+          # Check whether 'nix upgrade-nix' works.
+          cat > /tmp/paths.nix <<EOF
+          {
+            x86_64-linux = "${build.x86_64-linux}";
+          }
+          EOF
+          su - alice -c 'nix upgrade-nix -vvv --nix-store-paths-url file:///tmp/paths.nix'
+          (! [ -L /home/alice/.profile-1-link ])
+          su - alice -c 'PAGER= nix-store -qR ${build.x86_64-linux}'
+
           mkdir -p $out/nix-support
           touch $out/nix-support/hydra-build-products
           umount /nix
@@ -264,14 +278,32 @@ let
       pkgs.runCommand "eval-nixos" { buildInputs = [ build.x86_64-linux ]; }
         ''
           export NIX_STATE_DIR=$TMPDIR
-          nix-store --init
 
-          nix-instantiate ${nixpkgs}/nixos/release-combined.nix -A tested --dry-run
+          nix-instantiate ${nixpkgs}/nixos/release-combined.nix -A tested --dry-run \
+            --arg nixpkgs '{ outPath = ${nixpkgs}; revCount = 123; shortRev = "abcdefgh"; }'
 
           touch $out
         '';
 
 
+    installerScript =
+      pkgs.runCommand "installer-script"
+        { buildInputs = [ build.x86_64-linux ];
+        }
+        ''
+          mkdir -p $out/nix-support
+
+          substitute ${./scripts/install.in} $out/install \
+            ${pkgs.lib.concatMapStrings
+              (system: "--replace '@binaryTarball_${system}@' $(nix hash-file --base16 --type sha256 ${binaryTarball.${system}}/*.tar.bz2) ")
+              [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ]
+            } \
+            --replace '@nixVersion@' ${build.x86_64-linux.src.version}
+
+          echo "file installer $out/install" >> $out/nix-support/hydra-build-products
+        '';
+
+
     # Aggregate job containing the release-critical jobs.
     release = pkgs.releaseTools.aggregate {
       name = "nix-${tarball.version}";
@@ -281,20 +313,17 @@ let
           build.i686-linux
           build.x86_64-darwin
           build.x86_64-linux
+          build.aarch64-linux
           binaryTarball.i686-linux
           binaryTarball.x86_64-darwin
           binaryTarball.x86_64-linux
-          #deb_debian8i386
-          #deb_debian8x86_64
-          deb_ubuntu1604i386
-          deb_ubuntu1604x86_64
-          rpm_fedora25i386
-          rpm_fedora25x86_64
+          binaryTarball.aarch64-linux
           tests.remoteBuilds
           tests.nix-copy-closure
           tests.binaryTarball
           tests.evalNixpkgs
           tests.evalNixOS
+          installerScript
         ];
     };
 
@@ -314,7 +343,7 @@ let
       src = jobs.tarball;
       diskImage = (diskImageFun vmTools.diskImageFuns)
         { extraPackages =
-            [ "sqlite" "sqlite-devel" "bzip2-devel" "libcurl-devel" "openssl-devel" "xz-devel" "libseccomp-devel" ]
+            [ "sqlite" "sqlite-devel" "bzip2-devel" "libcurl-devel" "openssl-devel" "xz-devel" "libseccomp-devel" "libsodium-devel" "boost-devel" "bison" "flex" ]
             ++ extraPackages; };
       # At most 2047MB can be simulated in qemu-system-i386
       memSize = 2047;
@@ -337,9 +366,9 @@ let
       src = jobs.tarball;
       diskImage = (diskImageFun vmTools.diskImageFuns)
         { extraPackages =
-            [ "libsqlite3-dev" "libbz2-dev" "libcurl-dev" "libcurl3-nss" "libssl-dev" "liblzma-dev" "libseccomp-dev" ]
+            [ "libsqlite3-dev" "libbz2-dev" "libcurl-dev" "libcurl3-nss" "libssl-dev" "liblzma-dev" "libseccomp-dev" "libsodium-dev" "libboost-all-dev" ]
             ++ extraPackages; };
-      memSize = 1024;
+      memSize = 2047;
       meta.schedulingPriority = 50;
       postInstall = "make installcheck";
       configureFlags = "--sysconfdir=/etc";

scripts/install-darwin-multi-user.sh

@@ -1,772 +1,43 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 set -eu
 set -o pipefail
 
-# Sourced from:
-# - https://github.com/LnL7/nix-darwin/blob/8c29d0985d74b4a990238497c47a2542a5616b3c/bootstrap.sh
-# - https://gist.github.com/expipiplus1/e571ce88c608a1e83547c918591b149f/ac504c6c1b96e65505fbda437a28ce563408ecb0
-# - https://github.com/NixOS/nixos-org-configurations/blob/a122f418797713d519aadf02e677fce0dc1cb446/delft/scripts/nix-mac-installer.sh
-# - https://github.com/matthewbauer/macNixOS/blob/f6045394f9153edea417be90c216788e754feaba/install-macNixOS.sh
-# - https://gist.github.com/LnL7/9717bd6cdcb30b086fd7f2093e5f8494/86b26f852ce563e973acd30f796a9a416248c34a
-#
-# however tracking which bits came from which would be impossible.
-
-readonly ESC='\033[0m'
-readonly BOLD='\033[38;1m'
-readonly BLUE='\033[38;34m'
-readonly BLUE_UL='\033[38;4;34m'
-readonly GREEN='\033[38;32m'
-readonly GREEN_UL='\033[38;4;32m'
-readonly RED='\033[38;31m'
-readonly RED_UL='\033[38;4;31m'
-readonly YELLOW='\033[38;33m'
-readonly YELLOW_UL='\033[38;4;33m'
-
-readonly CORES=$(sysctl -n hw.ncpu)
-readonly NIX_USER_COUNT="32"
-readonly NIX_BUILD_GROUP_ID="30000"
-readonly NIX_BUILD_GROUP_NAME="nixbld"
-readonly NIX_FIRST_BUILD_UID="30001"
-# Please don't change this. We don't support it, because the
-# default shell profile that comes with Nix doesn't support it.
-readonly NIX_ROOT="/nix"
 readonly PLIST_DEST=/Library/LaunchDaemons/org.nixos.nix-daemon.plist
 
-readonly PROFILE_TARGETS=("/etc/bashrc" "/etc/zshrc")
-readonly PROFILE_BACKUP_SUFFIX=".backup-before-nix"
-readonly PROFILE_NIX_FILE="$NIX_ROOT/var/nix/profiles/default/etc/profile.d/nix-daemon.sh"
-
-readonly NIX_INSTALLED_NIX="@nix@"
-readonly NIX_INSTALLED_CACERT="@cacert@"
-readonly EXTRACTED_NIX_PATH="$(dirname "$0")"
-
-readonly ROOT_HOME="/var/root"
-
-if [ -t 0 ]; then
-    readonly IS_HEADLESS='no'
-else
-    readonly IS_HEADLESS='yes'
-fi
-
-headless() {
-    if [ "$IS_HEADLESS" = "yes" ]; then
-        return 0
-    else
-        return 1
-    fi
-}
-
-contactme() {
-    echo "We'd love to help if you need it."
-    echo ""
-    echo "If you can, open an issue at https://github.com/nixos/nix/issues"
-    echo ""
-    echo "Or feel free to contact the team,"
-    echo " - on IRC #nixos on irc.freenode.net"
-    echo " - on twitter @nixos_org"
-}
-
-uninstall_directions() {
-    subheader "Uninstalling nix:"
-    local step=0
-
-    if [ -e "$PLIST_DEST" ]; then
-        step=$((step + 1))
-        cat <<EOF
-$step. Delete $PLIST_DEST
-
-  sudo launchctl unload $PLIST_DEST
-  sudo rm $PLIST_DEST
-
-EOF
-    fi
-
-    for profile_target in "${PROFILE_TARGETS[@]}"; do
-        if [ -e "$profile_target" ] && [ -e "$profile_target$PROFILE_BACKUP_SUFFIX" ]; then
-            step=$((step + 1))
-            cat <<EOF
-$step. Restore $profile_target$PROFILE_BACKUP_SUFFIX back to $profile_target
-
-  sudo mv $profile_target$PROFILE_BACKUP_SUFFIX $profile_target
-
-(after this one, you may need to re-open any terminals that were
-opened while it existed.)
-
-EOF
-        fi
-    done
-
-    step=$((step + 1))
-    cat <<EOF
-$step. Delete the files Nix added to your system:
-
-  sudo rm -rf /etc/nix $NIX_ROOT $ROOT_HOME/.nix-profile $ROOT_HOME/.nix-defexpr $ROOT_HOME/.nix-channels $HOME/.nix-profile $HOME/.nix-defexpr $HOME/.nix-channels
-
-and that is it.
-
-EOF
-
-}
-
-nix_user_for_core() {
-    printf "nixbld%d" "$1"
-}
-
-nix_uid_for_core() {
-    echo $((NIX_FIRST_BUILD_UID + $1 - 1))
-}
-
 dsclattr() {
     /usr/bin/dscl . -read "$1" \
         | awk "/$2/ { print \$2 }"
 }
 
-_textout() {
-    echo -en "$1"
-    shift
-    if [ "$*" = "" ]; then
-        cat
-    else
-        echo "$@"
-    fi
-    echo -en "$ESC"
-}
-
-header() {
-    follow="---------------------------------------------------------"
-    header=$(echo "---- $* $follow$follow$follow" | head -c 80)
-    echo ""
-    _textout "$BLUE" "$header"
-}
-
-warningheader() {
-    follow="---------------------------------------------------------"
-    header=$(echo "---- $* $follow$follow$follow" | head -c 80)
-    echo ""
-    _textout "$RED" "$header"
-}
-
-subheader() {
-    echo ""
-    _textout "$BLUE_UL" "$*"
-}
-
-row() {
-    printf "$BOLD%s$ESC:\\t%s\\n" "$1" "$2"
-}
-
-task() {
-    echo ""
-    ok "~~> $1"
-}
-
-bold() {
-    echo "$BOLD$*$ESC"
-}
-
-ok() {
-    _textout "$GREEN" "$@"
-}
-
-warning() {
-    warningheader "warning!"
-    cat
-    echo ""
-}
-
-failure() {
-    header "oh no!"
-    _textout "$RED" "$@"
-    echo ""
-    _textout "$RED" "$(contactme)"
-    trap finish_cleanup EXIT
-    exit 1
-}
-
-ui_confirm() {
-    _textout "$GREEN$GREEN_UL" "$1"
-
-    if headless; then
-        echo "No TTY, assuming you would say yes :)"
-        return 0
-    fi
-
-    local prompt="[y/n] "
-    echo -n "$prompt"
-    while read -r y; do
-        if [ "$y" = "y" ]; then
-            echo ""
-            return 0
-        elif [ "$y" = "n" ]; then
-            echo ""
-            return 1
-        else
-            _textout "$RED" "Sorry, I didn't understand. I can only understand answers of y or n"
-            echo -n "$prompt"
-        fi
-    done
-    echo ""
-    return 1
-}
-
-__sudo() {
-    local expl="$1"
-    local cmd="$2"
-    shift
-    header "sudo execution"
-
-    echo "I am executing:"
-    echo ""
-    printf "    $ sudo %s\\n" "$cmd"
-    echo ""
-    echo "$expl"
-    echo ""
-
-    return 0
-}
-
-_sudo() {
-    local expl="$1"
-    shift
-    if ! headless; then
-        __sudo "$expl" "$*"
-    fi
-    sudo "$@"
-}
-
-
-readonly SCRATCH=$(mktemp -d -t tmp.XXXXXXXXXX)
-function finish_cleanup {
-    rm -rf "$SCRATCH"
-}
-
-function finish_fail {
-    finish_cleanup
-
-    failure <<EOF
-Jeeze, something went wrong. If you can take all the output and open
-an issue, we'd love to fix the problem so nobody else has this issue.
-
-:(
-EOF
-}
-trap finish_fail EXIT
-
-function finish_success {
-    finish_cleanup
-
-    ok "Alright! We're done!"
-    cat <<EOF
-
-Before Nix will work in your existing shells, you'll need to close
-them and open them again. Other than that, you should be ready to go.
-
-Try it! Open a new terminal, and type:
-
-  $ nix-shell -p nix-info --run "nix-info -m"
-
-Thank you for using this installer. If you have any feedback, don't
-hesitate:
-
-$(contactme)
-EOF
-}
-
-
-validate_starting_assumptions() {
+poly_validate_assumptions() {
     if [ "$(uname -s)" != "Darwin" ]; then
         failure "This script is for use with macOS!"
     fi
-
-    if [ $EUID -eq 0 ]; then
-        failure <<EOF
-Please do not run this script with root privileges. We will call sudo
-when we need to.
-EOF
-    fi
-
-    if type nix-env 2> /dev/null >&2; then
-        failure <<EOF
-Nix already appears to be installed, and this tool assumes it is
-_not_ yet installed.
-
-$(uninstall_directions)
-EOF
-    fi
-
-    if [ "${NIX_REMOTE:-}" != "" ]; then
-        failure <<EOF
-For some reason, \$NIX_REMOTE is set. It really should not be set
-before this installer runs, and it hints that Nix is currently
-installed. Please delete the old Nix installation and start again.
-
-Note: You might need to close your shell window and open a new shell
-to clear the variable.
-EOF
-    fi
-
-    if echo "${SSL_CERT_FILE:-}" | grep -qE "(nix/var/nix|nix-profile)"; then
-        failure <<EOF
-It looks like \$SSL_CERT_FILE is set to a path that used to be part of
-the old Nix installation. Please unset that variable and try again:
-
-  $ unset SSL_CERT_FILE
-
-EOF
-    fi
-
-    for file in ~/.bash_profile ~/.bash_login ~/.profile ~/.zshenv ~/.zprofile ~/.zshrc ~/.zlogin; do
-        if [ -f "$file" ]; then
-            if grep -l "^[^#].*.nix-profile" "$file"; then
-                failure <<EOF
-I found a reference to a ".nix-profile" in $file.
-This has a high chance of breaking a new nix installation. It was most
-likely put there by a previous Nix installer.
-
-Please remove this reference and try running this again. You should
-also look for similar references in:
-
- - ~/.bash_profile
- - ~/.bash_login
- - ~/.profile
-
-or other shell init files that you may have.
-
-$(uninstall_directions)
-EOF
-            fi
-        fi
-    done
-
-    if [ -d /nix ]; then
-        failure <<EOF
-There are some relics of a previous installation of Nix at /nix, and
-this scripts assumes Nix is _not_ yet installed. Please delete the old
-Nix installation and start again.
-
-$(uninstall_directions)
-EOF
-    fi
-
-    if [ -d /etc/nix ]; then
-        failure <<EOF
-There are some relics of a previous installation of Nix at /etc/nix, and
-this scripts assumes Nix is _not_ yet installed. Please delete the old
-Nix installation and start again.
-
-$(uninstall_directions)
-EOF
-    fi
-
-    for profile_target in "${PROFILE_TARGETS[@]}"; do
-        if [ -e "$profile_target$PROFILE_BACKUP_SUFFIX" ]; then
-        failure <<EOF
-When this script runs, it backs up the current $profile_target to
-$profile_target$PROFILE_BACKUP_SUFFIX. This backup file already exists, though.
-
-Please follow these instructions to clean up the old backup file:
-
-1. Copy $profile_target and $profile_target$PROFILE_BACKUP_SUFFIX to another place, just
-in case.
-
-2. Take care to make sure that $profile_target$PROFILE_BACKUP_SUFFIX doesn't look like
-it has anything nix-related in it. If it does, something is probably
-quite wrong. Please open an issue or get in touch immediately.
-
-3. Take care to make sure that $profile_target doesn't look like it has
-anything nix-related in it. If it does, and $profile_target _did not_,
-run:
-
-  $ /usr/bin/sudo /bin/mv $profile_target$PROFILE_BACKUP_SUFFIX $profile_target
-
-and try again.
-EOF
-        fi
-
-        if grep -qi "nix" "$profile_target"; then
-            failure <<EOF
-It looks like $profile_target already has some Nix configuration in
-there. There should be no reason to run this again. If you're having
-trouble, please open an issue.
-EOF
-        fi
-    done
-
-    danger_paths=("$ROOT_HOME/.nix-defexpr" "$ROOT_HOME/.nix-channels" "$ROOT_HOME/.nix-profile")
-    for danger_path in "${danger_paths[@]}"; do
-        if _sudo "making sure that $danger_path doesn't exist" \
-           test -e "$danger_path"; then
-            failure <<EOF
-I found a file at $danger_path, which is a relic of a previous
-installation. You must first delete this file before continuing.
-
-$(uninstall_directions)
-EOF
-        fi
-    done
 }
 
-setup_report() {
-    header "hardware report"
-    row "           Cores" "$CORES"
-
-    header "Nix config report"
-    row "        Temp Dir" "$SCRATCH"
-    row "        Nix Root" "$NIX_ROOT"
-    row "     Build Users" "$NIX_USER_COUNT"
-    row "  Build Group ID" "$NIX_BUILD_GROUP_ID"
-    row "Build Group Name" "$NIX_BUILD_GROUP_NAME"
-    if [ "${ALLOW_PREEXISTING_INSTALLATION:-}" != "" ]; then
-        row "Preexisting Install" "Allowed"
-    fi
-
-    subheader "build users:"
-
-    row "    Username" "UID"
-    for i in $(seq 1 "$NIX_USER_COUNT"); do
-        row "     $(nix_user_for_core "$i")" "$(nix_uid_for_core "$i")"
-    done
-    echo ""
+poly_service_installed_check() {
+    [ -e "$PLIST_DEST" ]
 }
 
-create_build_group() {
-    local primary_group_id
+poly_service_uninstall_directions() {
+        cat <<EOF
+$1. Delete $PLIST_DEST
 
-    task "Setting up the build group $NIX_BUILD_GROUP_NAME"
-    if ! /usr/bin/dscl . -read "/Groups/$NIX_BUILD_GROUP_NAME" > /dev/null 2>&1; then
-        _sudo "Create the Nix build group, $NIX_BUILD_GROUP_NAME" \
-              /usr/sbin/dseditgroup -o create \
-              -r "Nix build group for nix-daemon" \
-              -i "$NIX_BUILD_GROUP_ID" \
-              "$NIX_BUILD_GROUP_NAME" >&2
-        row "            Created" "Yes"
-    else
-        primary_group_id=$(dsclattr "/Groups/$NIX_BUILD_GROUP_NAME" "PrimaryGroupID")
-        if [ "$primary_group_id" -ne "$NIX_BUILD_GROUP_ID" ]; then
-            failure <<EOF
-It seems the build group $NIX_BUILD_GROUP_NAME already exists, but
-with the UID $primary_group_id. This script can't really handle
-that right now, so I'm going to give up.
+  sudo launchctl unload $PLIST_DEST
+  sudo rm $PLIST_DEST
 
-You can fix this by editing this script and changing the
-NIX_BUILD_GROUP_ID variable near the top to from $NIX_BUILD_GROUP_ID
-to $primary_group_id and re-run.
 EOF
-        else
-            row "            Exists" "Yes"
-        fi
-    fi
 }
 
-create_build_user_for_core() {
-    local coreid
-    local username
-    local uid
-
-    coreid="$1"
-    username=$(nix_user_for_core "$coreid")
-    uid=$(nix_uid_for_core "$coreid")
-    dsclpath="/Users/$username"
-
-    task "Setting up the build user $username"
-
-    if ! /usr/bin/dscl . -read "$dsclpath" > /dev/null 2>&1; then
-        _sudo "Creating the Nix build user, $username" \
-              /usr/bin/dscl . create "$dsclpath" \
-              UniqueID "${uid}"
-        row "           Created" "Yes"
-    else
-        actual_uid=$(dsclattr "$dsclpath" "UniqueID")
-        if [ "$actual_uid" -ne "$uid" ]; then
-            failure <<EOF
-It seems the build user $username already exists, but with the UID
-with the UID $actual_uid. This script can't really handle that right
-now, so I'm going to give up.
-
-If you already created the users and you know they start from
-$actual_uid and go up from there, you can edit this script and change
-NIX_FIRST_BUILD_UID near the top of the file to $actual_uid and try
-again.
-EOF
-        else
-            row "            Exists" "Yes"
-        fi
-    fi
-
-    if [ "$(dsclattr "$dsclpath" "IsHidden")" = "1" ]; then
-        row "          IsHidden" "Yes"
-    else
-        _sudo "in order to make $username a hidden user" \
-              /usr/bin/dscl . -create "$dsclpath" "IsHidden" "1"
-        row "          IsHidden" "Yes"
-    fi
-
-    if [ "$(dsclattr "$dsclpath" "NFSHomeDirectory")" = "/var/empty" ]; then
-        row "          NFSHomeDirectory" "/var/empty"
-    else
-        _sudo "in order to give $username a safe home directory" \
-              /usr/bin/dscl . -create "$dsclpath" "NFSHomeDirectory" "/var/empty"
-        row "          NFSHomeDirectory" "/var/empty"
-    fi
-
-    if [ "$(dsclattr "$dsclpath" "RealName")" = "Nix build user $coreid" ]; then
-        row "          RealName" "Nix build user $coreid"
-    else
-        _sudo "in order to give $username a useful name" \
-              /usr/bin/dscl . -create "$dsclpath" "RealName" "Nix build user $coreid"
-        row "          RealName" "Nix build user $coreid"
-    fi
-
-    if [ "$(dsclattr "$dsclpath" "UserShell")" = "/sbin/nologin" ]; then
-        row "   Logins Disabled" "Yes"
-    else
-        _sudo "in order to prevent $username from logging in" \
-              /usr/bin/dscl . -create "$dsclpath" "UserShell" "/sbin/nologin"
-        row "   Logins Disabled" "Yes"
-    fi
-
-    if dseditgroup -o checkmember -m "$username" "$NIX_BUILD_GROUP_NAME" > /dev/null 2>&1 ; then
-        row "  Member of $NIX_BUILD_GROUP_NAME" "Yes"
-    else
-        _sudo "Add $username to the $NIX_BUILD_GROUP_NAME group"\
-            /usr/sbin/dseditgroup -o edit -t user \
-            -a "$username" "$NIX_BUILD_GROUP_NAME"
-        row "  Member of $NIX_BUILD_GROUP_NAME" "Yes"
-    fi
-
-    if [ "$(dsclattr "$dsclpath" "PrimaryGroupID")" = "$NIX_BUILD_GROUP_ID" ]; then
-        row "    PrimaryGroupID" "$NIX_BUILD_GROUP_ID"
-    else
-        _sudo "to let the nix daemon use this user for builds (this might seem redundant, but there are two concepts of group membership)" \
-              /usr/bin/dscl . -create "$dsclpath" "PrimaryGroupID" "$NIX_BUILD_GROUP_ID"
-        row "    PrimaryGroupID" "$NIX_BUILD_GROUP_ID"
-
-    fi
-}
-
-create_build_users() {
-    for i in $(seq 1 "$NIX_USER_COUNT"); do
-        create_build_user_for_core "$i"
-    done
-}
-
-create_directories() {
-    _sudo "to make the basic directory structure of Nix (part 1)" \
-          mkdir -pv -m 0755 /nix /nix/var /nix/var/log /nix/var/log/nix /nix/var/log/nix/drvs /nix/var/nix{,/db,/gcroots,/profiles,/temproots,/userpool}
-
-    _sudo "to make the basic directory structure of Nix (part 2)" \
-          mkdir -pv -m 1777 /nix/var/nix/{gcroots,profiles}/per-user
-
-    _sudo "to make the basic directory structure of Nix (part 3)" \
-          mkdir -pv -m 1775 /nix/store
-
-    _sudo "to make the basic directory structure of Nix (part 4)" \
-          chgrp "$NIX_BUILD_GROUP_NAME" /nix/store
-
-    _sudo "to set up the root user's profile (part 1)" \
-          mkdir -pv -m 0755 /nix/var/nix/profiles/per-user/root
-
-    _sudo "to set up the root user's profile (part 2)" \
-          mkdir -pv -m 0700 "$ROOT_HOME/.nix-defexpr"
-
-    _sudo "to place the default nix daemon configuration (part 1)" \
-          mkdir -pv -m 0555 /etc/nix
-}
-
-place_channel_configuration() {
-    echo "https://nixos.org/channels/nixpkgs-unstable nixpkgs" > "$SCRATCH/.nix-channels"
-    _sudo "to set up the default system channel (part 1)" \
-          install -m 0664 "$SCRATCH/.nix-channels" "$ROOT_HOME/.nix-channels"
-}
-
-welcome_to_nix() {
-    ok "Welcome to the Multi-User Nix Installation"
-
+poly_service_setup_note() {
     cat <<EOF
-
-This installation tool will set up your computer with the Nix package
-manager. This will happen in a few stages:
-
-1. Make sure your computer doesn't already have Nix. If it does, I
-   will show you instructions on how to clean up your old one.
-
-2. Show you what we are going to install and where. Then we will ask
-   if you are ready to continue.
-
-3. Create the system users and groups that the Nix daemon uses to run
-   builds.
-
-4. Perform the basic installation of the Nix files daemon.
-
-5. Configure your shell to import special Nix Profile files, so you
-   can use Nix.
-
-6. Start the Nix daemon.
-
-EOF
-
-    if ui_confirm "Would you like to see a more detailed list of what we will do?"; then
-        cat <<EOF
-
-We will:
-
- - make sure your computer doesn't already have Nix files
-   (if it does, I  will tell you how to clean them up.)
- - create local users (see the list above for the users we'll make)
- - create a local group ($NIX_BUILD_GROUP_NAME)
- - install Nix in to $NIX_ROOT
- - create a configuration file in /etc/nix
- - set up the "default profile" by creating some Nix-related files in
-   $ROOT_HOME
-EOF
-        for profile_target in "${PROFILE_TARGETS[@]}"; do
-            if [ -e "$profile_target" ]; then
-                cat <<EOF
- - back up $profile_target to $profile_target$PROFILE_BACKUP_SUFFIX
- - update $profile_target to include some Nix configuration
-EOF
-            fi
-        done
-        cat <<EOF
  - load and start a LaunchDaemon (at $PLIST_DEST) for nix-daemon
 
 EOF
-        if ! ui_confirm "Ready to continue?"; then
-            failure <<EOF
-Okay, maybe you would like to talk to the team.
-EOF
-        fi
-    fi
 }
 
-chat_about_sudo() {
-    header "let's talk about sudo"
-
-    if headless; then
-        cat <<EOF
-This script is going to call sudo a lot. Normally, it would show you
-exactly what commands it is running and why. However, the script is
-run in a headless fashion, like this:
-
-  $ curl https://nixos.org/nix/install | sh
-
-or maybe in a CI pipeline. Because of that, we're going to skip the
-verbose output in the interest of brevity.
-
-If you would like to
-see the output, try like this:
-
-  $ curl -o install-nix https://nixos.org/nix/install
-  $ sh ./install-nix
-
-EOF
-        return 0
-    fi
-
-    cat <<EOF
-This script is going to call sudo a lot. Every time we do, it'll
-output exactly what it'll do, and why.
-
-Just like this:
-EOF
-
-    __sudo "to demonstrate how our sudo prompts look" \
-           echo "this is a sudo prompt"
-
-    cat <<EOF
-
-This might look scary, but everything can be undone by running just a
-few commands. We used to ask you to confirm each time sudo ran, but it
-was too many times. Instead, I'll just ask you this one time:
-
-EOF
-    if ui_confirm "Can we use sudo?"; then
-        ok "Yay! Thanks! Let's get going!"
-    else
-        failure <<EOF
-That is okay, but we can't install.
-EOF
-    fi
-}
-
-install_from_extracted_nix() {
-    (
-        cd "$EXTRACTED_NIX_PATH"
-
-        _sudo "to copy the basic Nix files to the new store at $NIX_ROOT/store" \
-              rsync -rlpt ./store/* "$NIX_ROOT/store/"
-
-        if [ -d "$NIX_INSTALLED_NIX" ]; then
-            echo "      Alright! We have our first nix at $NIX_INSTALLED_NIX"
-        else
-            failure <<EOF
-Something went wrong, and I didn't find Nix installed at
-$NIX_INSTALLED_NIX.
-EOF
-        fi
-
-        _sudo "to initialize the Nix Database" \
-              $NIX_INSTALLED_NIX/bin/nix-store --init
-
-        cat ./.reginfo \
-            | _sudo "to load data for the first time in to the Nix Database" \
-                   "$NIX_INSTALLED_NIX/bin/nix-store" --load-db
-
-        echo "      Just finished getting the nix database ready."
-    )
-}
-
-shell_source_lines() {
-    cat <<EOF
-
-# Nix
-if [ -e '$PROFILE_NIX_FILE' ]; then
-  . '$PROFILE_NIX_FILE'
-fi
-# End Nix
-
-EOF
-}
-configure_shell_profile() {
-    for profile_target in "${PROFILE_TARGETS[@]}"; do
-        if [ -e "$profile_target" ]; then
-            _sudo "to back up your current $profile_target to $profile_target$PROFILE_BACKUP_SUFFIX" \
-                  cp "$profile_target" "$profile_target$PROFILE_BACKUP_SUFFIX"
-
-            shell_source_lines \
-                | _sudo "extend your $profile_target with nix-daemon settings" \
-                        tee -a "$profile_target"
-        fi
-    done
-
-}
-
-setup_default_profile() {
-    _sudo "to installing a bootstrapping Nix in to the default Profile" \
-          HOME=$ROOT_HOME "$NIX_INSTALLED_NIX/bin/nix-env" -i "$NIX_INSTALLED_NIX"
-
-    _sudo "to installing a bootstrapping SSL certificate just for Nix in to the default Profile" \
-          HOME=$ROOT_HOME "$NIX_INSTALLED_NIX/bin/nix-env" -i "$NIX_INSTALLED_CACERT"
-
-    _sudo "to update the default channel in the default profile" \
-          HOME=$ROOT_HOME NIX_SSL_CERT_FILE=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt "$NIX_INSTALLED_NIX/bin/nix-channel" --update nixpkgs
-}
-
-
-place_nix_configuration() {
-    cat <<EOF > "$SCRATCH/nix.conf"
-build-users-group = $NIX_BUILD_GROUP_NAME
-
-max-jobs = $NIX_USER_COUNT
-cores = 1
-sandbox = false
-EOF
-    _sudo "to place the default nix daemon configuration (part 2)" \
-          install -m 0664 "$SCRATCH/nix.conf" /etc/nix/nix.conf
-}
-
-configure_nix_daemon_plist() {
+poly_configure_nix_daemon_service() {
     _sudo "to set up the nix-daemon as a LaunchDaemon" \
           ln -sfn "/nix/var/nix/profiles/default$PLIST_DEST" "$PLIST_DEST"
 
@@ -778,42 +49,96 @@ configure_nix_daemon_plist() {
 
 }
 
-
-main() {
-    welcome_to_nix
-    chat_about_sudo
-
-    if [ "${ALLOW_PREEXISTING_INSTALLATION:-}" = "" ]; then
-        validate_starting_assumptions
-    fi
-
-    setup_report
-
-    if ! ui_confirm "Ready to continue?"; then
-        ok "Alright, no changes have been made :)"
-        contactme
-        trap finish_cleanup EXIT
-        exit 1
-    fi
-
-    create_build_group
-    create_build_users
-    create_directories
-    place_channel_configuration
-    install_from_extracted_nix
-
-    configure_shell_profile
-
-    set +eu
-    . /etc/profile
-    set -eu
-
-    setup_default_profile
-    place_nix_configuration
-    configure_nix_daemon_plist
-
-    trap finish_success EXIT
+poly_group_exists() {
+    /usr/bin/dscl . -read "/Groups/$1" > /dev/null 2>&1
 }
 
+poly_group_id_get() {
+    dsclattr "/Groups/$1" "PrimaryGroupID"
+}
 
-main
+poly_create_build_group() {
+    _sudo "Create the Nix build group, $NIX_BUILD_GROUP_NAME" \
+          /usr/sbin/dseditgroup -o create \
+          -r "Nix build group for nix-daemon" \
+          -i "$NIX_BUILD_GROUP_ID" \
+          "$NIX_BUILD_GROUP_NAME" >&2
+}
+
+poly_user_exists() {
+    /usr/bin/dscl . -read "/Users/$1" > /dev/null 2>&1
+}
+
+poly_user_id_get() {
+    dsclattr "/Users/$1" "UniqueID"
+}
+
+poly_user_hidden_get() {
+    dsclattr "/Users/$1" "IsHidden"
+}
+
+poly_user_hidden_set() {
+    _sudo "in order to make $1 a hidden user" \
+          /usr/bin/dscl . -create "/Users/$1" "IsHidden" "1"
+}
+
+poly_user_home_get() {
+    dsclattr "/Users/$1" "NFSHomeDirectory"
+}
+
+poly_user_home_set() {
+    _sudo "in order to give $1 a safe home directory" \
+          /usr/bin/dscl . -create "/Users/$1" "NFSHomeDirectory" "$2"
+}
+
+poly_user_note_get() {
+    dsclattr "/Users/$1" "RealName"
+}
+
+poly_user_note_set() {
+    _sudo "in order to give $username a useful note" \
+          /usr/bin/dscl . -create "/Users/$1" "RealName" "$2"
+}
+
+poly_user_shell_get() {
+    dsclattr "/Users/$1" "UserShell"
+}
+
+poly_user_shell_set() {
+    _sudo "in order to give $1 a safe home directory" \
+          /usr/bin/dscl . -create "/Users/$1" "UserShell" "$2"
+}
+
+poly_user_in_group_check() {
+    username=$1
+    group=$2
+    dseditgroup -o checkmember -m "$username" "$group" > /dev/null 2>&1
+}
+
+poly_user_in_group_set() {
+    username=$1
+    group=$2
+
+    _sudo "Add $username to the $group group"\
+          /usr/sbin/dseditgroup -o edit -t user \
+          -a "$username" "$group"
+}
+
+poly_user_primary_group_get() {
+    dsclattr "/Users/$1" "PrimaryGroupID"
+}
+
+poly_user_primary_group_set() {
+    _sudo "to let the nix daemon use this user for builds (this might seem redundant, but there are two concepts of group membership)" \
+          /usr/bin/dscl . -create "/Users/$1" "PrimaryGroupID" "$2"
+}
+
+poly_create_build_user() {
+    username=$1
+    uid=$2
+    builder_num=$3
+
+    _sudo "Creating the Nix build user (#$builder_num), $username" \
+          /usr/bin/dscl . create "/Users/$username" \
+          UniqueID "${uid}"
+}

scripts/install-multi-user.sh

@@ -0,0 +1,799 @@
+#!/usr/bin/env bash
+
+set -eu
+set -o pipefail
+
+# Sourced from:
+# - https://github.com/LnL7/nix-darwin/blob/8c29d0985d74b4a990238497c47a2542a5616b3c/bootstrap.sh
+# - https://gist.github.com/expipiplus1/e571ce88c608a1e83547c918591b149f/ac504c6c1b96e65505fbda437a28ce563408ecb0
+# - https://github.com/NixOS/nixos-org-configurations/blob/a122f418797713d519aadf02e677fce0dc1cb446/delft/scripts/nix-mac-installer.sh
+# - https://github.com/matthewbauer/macNixOS/blob/f6045394f9153edea417be90c216788e754feaba/install-macNixOS.sh
+# - https://gist.github.com/LnL7/9717bd6cdcb30b086fd7f2093e5f8494/86b26f852ce563e973acd30f796a9a416248c34a
+#
+# however tracking which bits came from which would be impossible.
+
+readonly ESC='\033[0m'
+readonly BOLD='\033[38;1m'
+readonly BLUE='\033[38;34m'
+readonly BLUE_UL='\033[38;4;34m'
+readonly GREEN='\033[38;32m'
+readonly GREEN_UL='\033[38;4;32m'
+readonly RED='\033[38;31m'
+readonly RED_UL='\033[38;4;31m'
+readonly YELLOW='\033[38;33m'
+readonly YELLOW_UL='\033[38;4;33m'
+
+readonly NIX_USER_COUNT="32"
+readonly NIX_BUILD_GROUP_ID="30000"
+readonly NIX_BUILD_GROUP_NAME="nixbld"
+readonly NIX_FIRST_BUILD_UID="30001"
+# Please don't change this. We don't support it, because the
+# default shell profile that comes with Nix doesn't support it.
+readonly NIX_ROOT="/nix"
+
+readonly PROFILE_TARGETS=("/etc/bashrc" "/etc/profile.d/nix.sh" "/etc/zshrc")
+readonly PROFILE_BACKUP_SUFFIX=".backup-before-nix"
+readonly PROFILE_NIX_FILE="$NIX_ROOT/var/nix/profiles/default/etc/profile.d/nix-daemon.sh"
+
+readonly NIX_INSTALLED_NIX="@nix@"
+readonly NIX_INSTALLED_CACERT="@cacert@"
+readonly EXTRACTED_NIX_PATH="$(dirname "$0")"
+
+readonly ROOT_HOME=$(echo ~root)
+
+if [ -t 0 ]; then
+    readonly IS_HEADLESS='no'
+else
+    readonly IS_HEADLESS='yes'
+fi
+
+headless() {
+    if [ "$IS_HEADLESS" = "yes" ]; then
+        return 0
+    else
+        return 1
+    fi
+}
+
+contactme() {
+    echo "We'd love to help if you need it."
+    echo ""
+    echo "If you can, open an issue at https://github.com/nixos/nix/issues"
+    echo ""
+    echo "Or feel free to contact the team,"
+    echo " - on IRC #nixos on irc.freenode.net"
+    echo " - on twitter @nixos_org"
+}
+
+uninstall_directions() {
+    subheader "Uninstalling nix:"
+    local step=0
+
+    if poly_service_installed_check; then
+        step=$((step + 1))
+        poly_service_uninstall_directions "$step"
+    fi
+
+    for profile_target in "${PROFILE_TARGETS[@]}"; do
+        if [ -e "$profile_target" ] && [ -e "$profile_target$PROFILE_BACKUP_SUFFIX" ]; then
+            step=$((step + 1))
+            cat <<EOF
+$step. Restore $profile_target$PROFILE_BACKUP_SUFFIX back to $profile_target
+
+  sudo mv $profile_target$PROFILE_BACKUP_SUFFIX $profile_target
+
+(after this one, you may need to re-open any terminals that were
+opened while it existed.)
+
+EOF
+        fi
+    done
+
+    step=$((step + 1))
+    cat <<EOF
+$step. Delete the files Nix added to your system:
+
+  sudo rm -rf /etc/nix $NIX_ROOT $ROOT_HOME/.nix-profile $ROOT_HOME/.nix-defexpr $ROOT_HOME/.nix-channels $HOME/.nix-profile $HOME/.nix-defexpr $HOME/.nix-channels
+
+and that is it.
+
+EOF
+
+}
+
+nix_user_for_core() {
+    printf "nixbld%d" "$1"
+}
+
+nix_uid_for_core() {
+    echo $((NIX_FIRST_BUILD_UID + $1 - 1))
+}
+
+_textout() {
+    echo -en "$1"
+    shift
+    if [ "$*" = "" ]; then
+        cat
+    else
+        echo "$@"
+    fi
+    echo -en "$ESC"
+}
+
+header() {
+    follow="---------------------------------------------------------"
+    header=$(echo "---- $* $follow$follow$follow" | head -c 80)
+    echo ""
+    _textout "$BLUE" "$header"
+}
+
+warningheader() {
+    follow="---------------------------------------------------------"
+    header=$(echo "---- $* $follow$follow$follow" | head -c 80)
+    echo ""
+    _textout "$RED" "$header"
+}
+
+subheader() {
+    echo ""
+    _textout "$BLUE_UL" "$*"
+}
+
+row() {
+    printf "$BOLD%s$ESC:\\t%s\\n" "$1" "$2"
+}
+
+task() {
+    echo ""
+    ok "~~> $1"
+}
+
+bold() {
+    echo "$BOLD$*$ESC"
+}
+
+ok() {
+    _textout "$GREEN" "$@"
+}
+
+warning() {
+    warningheader "warning!"
+    cat
+    echo ""
+}
+
+failure() {
+    header "oh no!"
+    _textout "$RED" "$@"
+    echo ""
+    _textout "$RED" "$(contactme)"
+    trap finish_cleanup EXIT
+    exit 1
+}
+
+ui_confirm() {
+    _textout "$GREEN$GREEN_UL" "$1"
+
+    if headless; then
+        echo "No TTY, assuming you would say yes :)"
+        return 0
+    fi
+
+    local prompt="[y/n] "
+    echo -n "$prompt"
+    while read -r y; do
+        if [ "$y" = "y" ]; then
+            echo ""
+            return 0
+        elif [ "$y" = "n" ]; then
+            echo ""
+            return 1
+        else
+            _textout "$RED" "Sorry, I didn't understand. I can only understand answers of y or n"
+            echo -n "$prompt"
+        fi
+    done
+    echo ""
+    return 1
+}
+
+__sudo() {
+    local expl="$1"
+    local cmd="$2"
+    shift
+    header "sudo execution"
+
+    echo "I am executing:"
+    echo ""
+    printf "    $ sudo %s\\n" "$cmd"
+    echo ""
+    echo "$expl"
+    echo ""
+
+    return 0
+}
+
+_sudo() {
+    local expl="$1"
+    shift
+    if ! headless; then
+        __sudo "$expl" "$*"
+    fi
+    sudo "$@"
+}
+
+
+readonly SCRATCH=$(mktemp -d -t tmp.XXXXXXXXXX)
+function finish_cleanup {
+    rm -rf "$SCRATCH"
+}
+
+function finish_fail {
+    finish_cleanup
+
+    failure <<EOF
+Jeeze, something went wrong. If you can take all the output and open
+an issue, we'd love to fix the problem so nobody else has this issue.
+
+:(
+EOF
+}
+trap finish_fail EXIT
+
+function finish_success {
+    finish_cleanup
+
+    ok "Alright! We're done!"
+    cat <<EOF
+
+Before Nix will work in your existing shells, you'll need to close
+them and open them again. Other than that, you should be ready to go.
+
+Try it! Open a new terminal, and type:
+
+  $ nix-shell -p nix-info --run "nix-info -m"
+
+Thank you for using this installer. If you have any feedback, don't
+hesitate:
+
+$(contactme)
+EOF
+}
+
+
+validate_starting_assumptions() {
+    poly_validate_assumptions
+
+    if [ $EUID -eq 0 ]; then
+        failure <<EOF
+Please do not run this script with root privileges. We will call sudo
+when we need to.
+EOF
+    fi
+
+    if type nix-env 2> /dev/null >&2; then
+        failure <<EOF
+Nix already appears to be installed, and this tool assumes it is
+_not_ yet installed.
+
+$(uninstall_directions)
+EOF
+    fi
+
+    if [ "${NIX_REMOTE:-}" != "" ]; then
+        failure <<EOF
+For some reason, \$NIX_REMOTE is set. It really should not be set
+before this installer runs, and it hints that Nix is currently
+installed. Please delete the old Nix installation and start again.
+
+Note: You might need to close your shell window and open a new shell
+to clear the variable.
+EOF
+    fi
+
+    if echo "${SSL_CERT_FILE:-}" | grep -qE "(nix/var/nix|nix-profile)"; then
+        failure <<EOF
+It looks like \$SSL_CERT_FILE is set to a path that used to be part of
+the old Nix installation. Please unset that variable and try again:
+
+  $ unset SSL_CERT_FILE
+
+EOF
+    fi
+
+    for file in ~/.bash_profile ~/.bash_login ~/.profile ~/.zshenv ~/.zprofile ~/.zshrc ~/.zlogin; do
+        if [ -f "$file" ]; then
+            if grep -l "^[^#].*.nix-profile" "$file"; then
+                failure <<EOF
+I found a reference to a ".nix-profile" in $file.
+This has a high chance of breaking a new nix installation. It was most
+likely put there by a previous Nix installer.
+
+Please remove this reference and try running this again. You should
+also look for similar references in:
+
+ - ~/.bash_profile
+ - ~/.bash_login
+ - ~/.profile
+
+or other shell init files that you may have.
+
+$(uninstall_directions)
+EOF
+            fi
+        fi
+    done
+
+    if [ -d /nix ]; then
+        failure <<EOF
+There are some relics of a previous installation of Nix at /nix, and
+this scripts assumes Nix is _not_ yet installed. Please delete the old
+Nix installation and start again.
+
+$(uninstall_directions)
+EOF
+    fi
+
+    if [ -d /etc/nix ]; then
+        failure <<EOF
+There are some relics of a previous installation of Nix at /etc/nix, and
+this scripts assumes Nix is _not_ yet installed. Please delete the old
+Nix installation and start again.
+
+$(uninstall_directions)
+EOF
+    fi
+
+    for profile_target in "${PROFILE_TARGETS[@]}"; do
+        if [ -e "$profile_target$PROFILE_BACKUP_SUFFIX" ]; then
+        failure <<EOF
+When this script runs, it backs up the current $profile_target to
+$profile_target$PROFILE_BACKUP_SUFFIX. This backup file already exists, though.
+
+Please follow these instructions to clean up the old backup file:
+
+1. Copy $profile_target and $profile_target$PROFILE_BACKUP_SUFFIX to another place, just
+in case.
+
+2. Take care to make sure that $profile_target$PROFILE_BACKUP_SUFFIX doesn't look like
+it has anything nix-related in it. If it does, something is probably
+quite wrong. Please open an issue or get in touch immediately.
+
+3. Take care to make sure that $profile_target doesn't look like it has
+anything nix-related in it. If it does, and $profile_target _did not_,
+run:
+
+  $ /usr/bin/sudo /bin/mv $profile_target$PROFILE_BACKUP_SUFFIX $profile_target
+
+and try again.
+EOF
+        fi
+
+        if [ -e "$profile_target" ] && grep -qi "nix" "$profile_target"; then
+            failure <<EOF
+It looks like $profile_target already has some Nix configuration in
+there. There should be no reason to run this again. If you're having
+trouble, please open an issue.
+EOF
+        fi
+    done
+
+    danger_paths=("$ROOT_HOME/.nix-defexpr" "$ROOT_HOME/.nix-channels" "$ROOT_HOME/.nix-profile")
+    for danger_path in "${danger_paths[@]}"; do
+        if _sudo "making sure that $danger_path doesn't exist" \
+           test -e "$danger_path"; then
+            failure <<EOF
+I found a file at $danger_path, which is a relic of a previous
+installation. You must first delete this file before continuing.
+
+$(uninstall_directions)
+EOF
+        fi
+    done
+}
+
+setup_report() {
+    header "Nix config report"
+    row "        Temp Dir" "$SCRATCH"
+    row "        Nix Root" "$NIX_ROOT"
+    row "     Build Users" "$NIX_USER_COUNT"
+    row "  Build Group ID" "$NIX_BUILD_GROUP_ID"
+    row "Build Group Name" "$NIX_BUILD_GROUP_NAME"
+    if [ "${ALLOW_PREEXISTING_INSTALLATION:-}" != "" ]; then
+        row "Preexisting Install" "Allowed"
+    fi
+
+    subheader "build users:"
+
+    row "    Username" "UID"
+    for i in $(seq 1 "$NIX_USER_COUNT"); do
+        row "     $(nix_user_for_core "$i")" "$(nix_uid_for_core "$i")"
+    done
+    echo ""
+}
+
+create_build_group() {
+    local primary_group_id
+
+    task "Setting up the build group $NIX_BUILD_GROUP_NAME"
+    if ! poly_group_exists "$NIX_BUILD_GROUP_NAME"; then
+        poly_create_build_group
+        row "            Created" "Yes"
+    else
+        primary_group_id=$(poly_group_id_get "$NIX_BUILD_GROUP_NAME")
+        if [ "$primary_group_id" -ne "$NIX_BUILD_GROUP_ID" ]; then
+            failure <<EOF
+It seems the build group $NIX_BUILD_GROUP_NAME already exists, but
+with the UID $primary_group_id. This script can't really handle
+that right now, so I'm going to give up.
+
+You can fix this by editing this script and changing the
+NIX_BUILD_GROUP_ID variable near the top to from $NIX_BUILD_GROUP_ID
+to $primary_group_id and re-run.
+EOF
+        else
+            row "            Exists" "Yes"
+        fi
+    fi
+}
+
+create_build_user_for_core() {
+    local coreid
+    local username
+    local uid
+
+    coreid="$1"
+    username=$(nix_user_for_core "$coreid")
+    uid=$(nix_uid_for_core "$coreid")
+
+    task "Setting up the build user $username"
+
+    if ! poly_user_exists "$username"; then
+        poly_create_build_user "$username" "$uid" "$coreid"
+        row "           Created" "Yes"
+    else
+        actual_uid=$(poly_user_id_get "$username")
+        if [ "$actual_uid" != "$uid" ]; then
+            failure <<EOF
+It seems the build user $username already exists, but with the UID
+with the UID '$actual_uid'. This script can't really handle that right
+now, so I'm going to give up.
+
+If you already created the users and you know they start from
+$actual_uid and go up from there, you can edit this script and change
+NIX_FIRST_BUILD_UID near the top of the file to $actual_uid and try
+again.
+EOF
+        else
+            row "            Exists" "Yes"
+        fi
+    fi
+
+    if [ "$(poly_user_hidden_get "$username")" = "1" ]; then
+        row "            Hidden" "Yes"
+    else
+        poly_user_hidden_set "$username"
+        row "            Hidden" "Yes"
+    fi
+
+    if [ "$(poly_user_home_get "$username")" = "/var/empty" ]; then
+        row "    Home Directory" "/var/empty"
+    else
+        poly_user_home_set "$username" "/var/empty"
+        row "    Home Directory" "/var/empty"
+    fi
+
+    # We use grep instead of an equality check because it is difficult
+    # to extract _just_ the user's note, instead it is prefixed with
+    # some plist junk. This was causing the user note to always be set,
+    # even if there was no reason for it.
+    if ! poly_user_note_get "$username" | grep -q "Nix build user $coreid"; then
+        row "              Note" "Nix build user $coreid"
+    else
+        poly_user_note_set "$username" "Nix build user $coreid"
+        row "              Note" "Nix build user $coreid"
+    fi
+
+    if [ "$(poly_user_shell_get "$username")" = "/sbin/nologin" ]; then
+        row "   Logins Disabled" "Yes"
+    else
+        poly_user_shell_set "$username" "/sbin/nologin"
+        row "   Logins Disabled" "Yes"
+    fi
+
+    if poly_user_in_group_check "$username" "$NIX_BUILD_GROUP_NAME"; then
+        row "  Member of $NIX_BUILD_GROUP_NAME" "Yes"
+    else
+        poly_user_in_group_set "$username" "$NIX_BUILD_GROUP_NAME"
+        row "  Member of $NIX_BUILD_GROUP_NAME" "Yes"
+    fi
+
+    if [ "$(poly_user_primary_group_get "$username")" = "$NIX_BUILD_GROUP_ID" ]; then
+        row "    PrimaryGroupID" "$NIX_BUILD_GROUP_ID"
+    else
+        poly_user_primary_group_set "$username" "$NIX_BUILD_GROUP_ID"
+        row "    PrimaryGroupID" "$NIX_BUILD_GROUP_ID"
+    fi
+}
+
+create_build_users() {
+    for i in $(seq 1 "$NIX_USER_COUNT"); do
+        create_build_user_for_core "$i"
+    done
+}
+
+create_directories() {
+    _sudo "to make the basic directory structure of Nix (part 1)" \
+          mkdir -pv -m 0755 /nix /nix/var /nix/var/log /nix/var/log/nix /nix/var/log/nix/drvs /nix/var/nix{,/db,/gcroots,/profiles,/temproots,/userpool}
+
+    _sudo "to make the basic directory structure of Nix (part 2)" \
+          mkdir -pv -m 1777 /nix/var/nix/{gcroots,profiles}/per-user
+
+    _sudo "to make the basic directory structure of Nix (part 3)" \
+          mkdir -pv -m 1775 /nix/store
+
+    _sudo "to make the basic directory structure of Nix (part 4)" \
+          chgrp "$NIX_BUILD_GROUP_NAME" /nix/store
+
+    _sudo "to set up the root user's profile (part 1)" \
+          mkdir -pv -m 0755 /nix/var/nix/profiles/per-user/root
+
+    _sudo "to set up the root user's profile (part 2)" \
+          mkdir -pv -m 0700 "$ROOT_HOME/.nix-defexpr"
+
+    _sudo "to place the default nix daemon configuration (part 1)" \
+          mkdir -pv -m 0555 /etc/nix
+}
+
+place_channel_configuration() {
+    echo "https://nixos.org/channels/nixpkgs-unstable nixpkgs" > "$SCRATCH/.nix-channels"
+    _sudo "to set up the default system channel (part 1)" \
+          install -m 0664 "$SCRATCH/.nix-channels" "$ROOT_HOME/.nix-channels"
+}
+
+welcome_to_nix() {
+    ok "Welcome to the Multi-User Nix Installation"
+
+    cat <<EOF
+
+This installation tool will set up your computer with the Nix package
+manager. This will happen in a few stages:
+
+1. Make sure your computer doesn't already have Nix. If it does, I
+   will show you instructions on how to clean up your old one.
+
+2. Show you what we are going to install and where. Then we will ask
+   if you are ready to continue.
+
+3. Create the system users and groups that the Nix daemon uses to run
+   builds.
+
+4. Perform the basic installation of the Nix files daemon.
+
+5. Configure your shell to import special Nix Profile files, so you
+   can use Nix.
+
+6. Start the Nix daemon.
+
+EOF
+
+    if ui_confirm "Would you like to see a more detailed list of what we will do?"; then
+        cat <<EOF
+
+We will:
+
+ - make sure your computer doesn't already have Nix files
+   (if it does, I  will tell you how to clean them up.)
+ - create local users (see the list above for the users we'll make)
+ - create a local group ($NIX_BUILD_GROUP_NAME)
+ - install Nix in to $NIX_ROOT
+ - create a configuration file in /etc/nix
+ - set up the "default profile" by creating some Nix-related files in
+   $ROOT_HOME
+EOF
+        for profile_target in "${PROFILE_TARGETS[@]}"; do
+            if [ -e "$profile_target" ]; then
+                cat <<EOF
+ - back up $profile_target to $profile_target$PROFILE_BACKUP_SUFFIX
+ - update $profile_target to include some Nix configuration
+EOF
+            fi
+        done
+        poly_service_setup_note
+        if ! ui_confirm "Ready to continue?"; then
+            failure <<EOF
+Okay, maybe you would like to talk to the team.
+EOF
+        fi
+    fi
+}
+
+chat_about_sudo() {
+    header "let's talk about sudo"
+
+    if headless; then
+        cat <<EOF
+This script is going to call sudo a lot. Normally, it would show you
+exactly what commands it is running and why. However, the script is
+run in a headless fashion, like this:
+
+  $ curl https://nixos.org/nix/install | sh
+
+or maybe in a CI pipeline. Because of that, we're going to skip the
+verbose output in the interest of brevity.
+
+If you would like to
+see the output, try like this:
+
+  $ curl -o install-nix https://nixos.org/nix/install
+  $ sh ./install-nix
+
+EOF
+        return 0
+    fi
+
+    cat <<EOF
+This script is going to call sudo a lot. Every time we do, it'll
+output exactly what it'll do, and why.
+
+Just like this:
+EOF
+
+    __sudo "to demonstrate how our sudo prompts look" \
+           echo "this is a sudo prompt"
+
+    cat <<EOF
+
+This might look scary, but everything can be undone by running just a
+few commands. We used to ask you to confirm each time sudo ran, but it
+was too many times. Instead, I'll just ask you this one time:
+
+EOF
+    if ui_confirm "Can we use sudo?"; then
+        ok "Yay! Thanks! Let's get going!"
+    else
+        failure <<EOF
+That is okay, but we can't install.
+EOF
+    fi
+}
+
+install_from_extracted_nix() {
+    (
+        cd "$EXTRACTED_NIX_PATH"
+
+        _sudo "to copy the basic Nix files to the new store at $NIX_ROOT/store" \
+              rsync -rlpt ./store/* "$NIX_ROOT/store/"
+
+        if [ -d "$NIX_INSTALLED_NIX" ]; then
+            echo "      Alright! We have our first nix at $NIX_INSTALLED_NIX"
+        else
+            failure <<EOF
+Something went wrong, and I didn't find Nix installed at
+$NIX_INSTALLED_NIX.
+EOF
+        fi
+
+        cat ./.reginfo \
+            | _sudo "to load data for the first time in to the Nix Database" \
+                   "$NIX_INSTALLED_NIX/bin/nix-store" --load-db
+
+        echo "      Just finished getting the nix database ready."
+    )
+}
+
+shell_source_lines() {
+    cat <<EOF
+
+# Nix
+if [ -e '$PROFILE_NIX_FILE' ]; then
+  . '$PROFILE_NIX_FILE'
+fi
+# End Nix
+
+EOF
+}
+
+configure_shell_profile() {
+    # If there is an /etc/profile.d directory, we want to ensure there
+    # is a nix.sh within it, so we can use the following loop to add
+    # the source lines to it. Note that I'm _not_ adding the source
+    # lines here, because we want to be using the regular machinery.
+    #
+    # If we go around that machinery, it becomes more complicated and
+    # adds complications to the uninstall instruction generator and
+    # old instruction sniffer as well.
+    if [ -d /etc/profile.d ]; then
+        _sudo "create a stub /etc/profile.d/nix.sh which will be updated" \
+              touch /etc/profile.d/nix.sh
+    fi
+
+    for profile_target in "${PROFILE_TARGETS[@]}"; do
+        if [ -e "$profile_target" ]; then
+            _sudo "to back up your current $profile_target to $profile_target$PROFILE_BACKUP_SUFFIX" \
+                  cp "$profile_target" "$profile_target$PROFILE_BACKUP_SUFFIX"
+
+            shell_source_lines \
+                | _sudo "extend your $profile_target with nix-daemon settings" \
+                        tee -a "$profile_target"
+        fi
+    done
+}
+
+setup_default_profile() {
+    _sudo "to installing a bootstrapping Nix in to the default Profile" \
+          HOME="$ROOT_HOME" "$NIX_INSTALLED_NIX/bin/nix-env" -i "$NIX_INSTALLED_NIX"
+
+    if [ -z "${NIX_SSL_CERT_FILE:-}" ] || ! [ -f "${NIX_SSL_CERT_FILE:-}" ]; then
+        _sudo "to installing a bootstrapping SSL certificate just for Nix in to the default Profile" \
+              HOME="$ROOT_HOME" "$NIX_INSTALLED_NIX/bin/nix-env" -i "$NIX_INSTALLED_CACERT"
+        export NIX_SSL_CERT_FILE=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt
+    fi
+
+    # Have to explicitly pass NIX_SSL_CERT_FILE as part of the sudo call,
+    # otherwise it will be lost in environments where sudo doesn't pass
+    # all the environment variables by default.
+    _sudo "to update the default channel in the default profile" \
+          HOME="$ROOT_HOME" NIX_SSL_CERT_FILE="$NIX_SSL_CERT_FILE" "$NIX_INSTALLED_NIX/bin/nix-channel" --update nixpkgs
+}
+
+
+place_nix_configuration() {
+    cat <<EOF > "$SCRATCH/nix.conf"
+build-users-group = $NIX_BUILD_GROUP_NAME
+
+max-jobs = $NIX_USER_COUNT
+cores = 1
+EOF
+    _sudo "to place the default nix daemon configuration (part 2)" \
+          install -m 0664 "$SCRATCH/nix.conf" /etc/nix/nix.conf
+}
+
+main() {
+    if [ "$(uname -s)" = "Darwin" ]; then
+        # shellcheck source=./install-darwin-multi-user.sh
+        . "$EXTRACTED_NIX_PATH/install-darwin-multi-user.sh"
+    elif [ "$(uname -s)" = "Linux" ] && [ -e /run/systemd/system ]; then
+        # shellcheck source=./install-systemd-multi-user.sh
+        . "$EXTRACTED_NIX_PATH/install-systemd-multi-user.sh"
+    else
+        failure "Sorry, I don't know what to do on $(uname)"
+    fi
+
+    welcome_to_nix
+    chat_about_sudo
+
+    if [ "${ALLOW_PREEXISTING_INSTALLATION:-}" = "" ]; then
+        validate_starting_assumptions
+    fi
+
+    setup_report
+
+    if ! ui_confirm "Ready to continue?"; then
+        ok "Alright, no changes have been made :)"
+        contactme
+        trap finish_cleanup EXIT
+        exit 1
+    fi
+
+    create_build_group
+    create_build_users
+    create_directories
+    place_channel_configuration
+    install_from_extracted_nix
+
+    configure_shell_profile
+
+    set +eu
+    . /etc/profile
+    set -eu
+
+    setup_default_profile
+    place_nix_configuration
+    poly_configure_nix_daemon_service
+
+    trap finish_success EXIT
+}
+
+
+main

scripts/install-nix-from-closure.sh

@@ -28,9 +28,44 @@ if [ "$(uname -s)" = "Darwin" ]; then
         echo "$0: macOS $(sw_vers -productVersion) is not supported, upgrade to 10.10 or higher"
         exit 1
     fi
+fi
 
-    printf '\e[1;31mSwitching to the Multi-User Darwin Installer\e[0m\n'
-    exec "$self/install-darwin-multi-user"
+# Determine if we could use the multi-user installer or not
+if [ "$(uname -s)" = "Darwin" ]; then
+    echo "Note: a multi-user installation is possible. See https://nixos.org/nix/manual/#sect-multi-user-installation" >&2
+elif [ "$(uname -s)" = "Linux" ] && [ -e /run/systemd/system ]; then
+    echo "Note: a multi-user installation is possible. See https://nixos.org/nix/manual/#sect-multi-user-installation" >&2
+fi
+
+INSTALL_MODE=no-daemon
+# Trivially handle the --daemon / --no-daemon options
+if [ "x${1:-}" = "x--no-daemon" ]; then
+    INSTALL_MODE=no-daemon
+elif [ "x${1:-}" = "x--daemon" ]; then
+    INSTALL_MODE=daemon
+elif [ "x${1:-}" != "x" ]; then
+    (
+        echo "Nix Installer [--daemon|--no-daemon]"
+
+        echo "Choose installation method."
+        echo ""
+        echo " --daemon:    Installs and configures a background daemon that manages the store,"
+        echo "              providing multi-user support and better isolation for local builds."
+        echo "              Both for security and reproducibility, this method is recommended if"
+        echo "              supported on your platform."
+        echo "              See https://nixos.org/nix/manual/#sect-multi-user-installation"
+        echo ""
+        echo " --no-daemon: Simple, single-user installation that does not require root and is"
+        echo "              trivial to uninstall."
+        echo "              (default)"
+        echo ""
+    ) >&2
+    exit
+fi
+
+if [ "$INSTALL_MODE" = "daemon" ]; then
+    printf '\e[1;31mSwitching to the Daemon-based Installer\e[0m\n'
+    exec "$self/install-multi-user"
     exit 0
 fi
 
@@ -74,12 +109,6 @@ for i in $(cd "$self/store" >/dev/null && echo ./*); do
 done
 echo "" >&2
 
-echo "initialising Nix database..." >&2
-if ! $nix/bin/nix-store --init; then
-    echo "$0: failed to initialize the Nix database" >&2
-    exit 1
-fi
-
 if ! "$nix/bin/nix-store" --load-db < "$self/.reginfo"; then
     echo "$0: unable to register valid paths" >&2
     exit 1

scripts/install-systemd-multi-user.sh

@@ -0,0 +1,154 @@
+#!/usr/bin/env bash
+
+set -eu
+set -o pipefail
+
+readonly SERVICE_SRC=/lib/systemd/system/nix-daemon.service
+readonly SERVICE_DEST=/etc/systemd/system/nix-daemon.service
+
+readonly SOCKET_SRC=/lib/systemd/system/nix-daemon.socket
+readonly SOCKET_DEST=/etc/systemd/system/nix-daemon.socket
+
+poly_validate_assumptions() {
+    if [ "$(uname -s)" != "Linux" ]; then
+        failure "This script is for use with Linux!"
+    fi
+}
+
+poly_service_installed_check() {
+    [ "$(systemctl is-enabled nix-daemon.service)" = "linked" ] \
+        || [ "$(systemctl is-enabled nix-daemon.socket)" = "enabled" ]
+}
+
+poly_service_uninstall_directions() {
+        cat <<EOF
+$1. Delete the systemd service and socket units
+
+  sudo systemctl stop nix-daemon.socket
+  sudo systemctl stop nix-daemon.service
+  sudo systemctl disable nix-daemon.socket
+  sudo systemctl disable nix-daemon.service
+  sudo systemctl daemon-reload
+EOF
+}
+
+poly_service_setup_note() {
+    cat <<EOF
+ - load and start a service (at $SERVICE_DEST
+   and $SOCKET_DEST) for nix-daemon
+
+EOF
+}
+
+poly_configure_nix_daemon_service() {
+    _sudo "to set up the nix-daemon service" \
+          systemctl link "/nix/var/nix/profiles/default$SERVICE_SRC"
+
+    _sudo "to set up the nix-daemon socket service" \
+          systemctl enable "/nix/var/nix/profiles/default$SOCKET_SRC"
+
+    _sudo "to load the systemd unit for nix-daemon" \
+          systemctl daemon-reload
+
+    _sudo "to start the nix-daemon.socket" \
+          systemctl start nix-daemon.socket
+
+    _sudo "to start the nix-daemon.service" \
+          systemctl start nix-daemon.service
+
+}
+
+poly_group_exists() {
+    getent group "$1" > /dev/null 2>&1
+}
+
+poly_group_id_get() {
+    getent group "$1" | cut -d: -f3
+}
+
+poly_create_build_group() {
+    _sudo "Create the Nix build group, $NIX_BUILD_GROUP_NAME" \
+          groupadd -g "$NIX_BUILD_GROUP_ID" --system \
+          "$NIX_BUILD_GROUP_NAME" >&2
+}
+
+poly_user_exists() {
+    getent passwd "$1" > /dev/null 2>&1
+}
+
+poly_user_id_get() {
+    getent passwd "$1" | cut -d: -f3
+}
+
+poly_user_hidden_get() {
+    echo "1"
+}
+
+poly_user_hidden_set() {
+    true
+}
+
+poly_user_home_get() {
+    getent passwd "$1" | cut -d: -f6
+}
+
+poly_user_home_set() {
+    _sudo "in order to give $1 a safe home directory" \
+          usermod --home "$2" "$1"
+}
+
+poly_user_note_get() {
+    getent passwd "$1" | cut -d: -f5
+}
+
+poly_user_note_set() {
+    _sudo "in order to give $1 a useful comment" \
+          usermod --comment "$2" "$1"
+}
+
+poly_user_shell_get() {
+    getent passwd "$1" | cut -d: -f7
+}
+
+poly_user_shell_set() {
+    _sudo "in order to prevent $1 from logging in" \
+          usermod --shell "$2" "$1"
+}
+
+poly_user_in_group_check() {
+    groups "$1" | grep -q "$2" > /dev/null 2>&1
+}
+
+poly_user_in_group_set() {
+    _sudo "Add $1 to the $2 group"\
+          usermod --append --groups "$2" "$1"
+}
+
+poly_user_primary_group_get() {
+    getent passwd "$1" | cut -d: -f4
+}
+
+poly_user_primary_group_set() {
+    _sudo "to let the nix daemon use this user for builds (this might seem redundant, but there are two concepts of group membership)" \
+          usermod --gid "$2" "$1"
+
+}
+
+poly_create_build_user() {
+    username=$1
+    uid=$2
+    builder_num=$3
+
+    _sudo "Creating the Nix build user, $username" \
+          useradd \
+          --home-dir /var/empty \
+          --comment "Nix build user $builder_num" \
+          --gid "$NIX_BUILD_GROUP_ID" \
+          --groups "$NIX_BUILD_GROUP_NAME" \
+          --no-user-group \
+          --system \
+          --shell /sbin/nologin \
+          --uid "$uid" \
+          --password "!" \
+          "$username"
+}

scripts/install.in

@@ -0,0 +1,67 @@
+#!/bin/sh
+
+# This script installs the Nix package manager on your system by
+# downloading a binary distribution and running its installer script
+# (which in turn creates and populates /nix).
+
+{ # Prevent execution if this script was only partially downloaded
+oops() {
+    echo "$0:" "$@" >&2
+    exit 1
+}
+
+tmpDir="$(mktemp -d -t nix-binary-tarball-unpack.XXXXXXXXXX || \
+          oops "Can't create temporary directory for downloading the Nix binary tarball")"
+cleanup() {
+    rm -rf "$tmpDir"
+}
+trap cleanup EXIT INT QUIT TERM
+
+require_util() {
+    type "$1" > /dev/null 2>&1 || command -v "$1" > /dev/null 2>&1 ||
+        oops "you do not have '$1' installed, which I need to $2"
+}
+
+case "$(uname -s).$(uname -m)" in
+    Linux.x86_64) system=x86_64-linux; hash=@binaryTarball_x86_64-linux@;;
+    Linux.i?86) system=i686-linux; hash=@binaryTarball_i686-linux@;;
+    Linux.aarch64) system=aarch64-linux; hash=@binaryTarball_aarch64-linux@;;
+    Darwin.x86_64) system=x86_64-darwin; hash=@binaryTarball_x86_64-darwin@;;
+    *) oops "sorry, there is no binary distribution of Nix for your platform";;
+esac
+
+url="https://nixos.org/releases/nix/nix-@nixVersion@/nix-@nixVersion@-$system.tar.bz2"
+
+tarball="$tmpDir/$(basename "$tmpDir/nix-@nixVersion@-$system.tar.bz2")"
+
+require_util curl "download the binary tarball"
+require_util bzcat "decompress the binary tarball"
+require_util tar "unpack the binary tarball"
+
+echo "downloading Nix @nixVersion@ binary tarball for $system from '$url' to '$tmpDir'..."
+curl -L "$url" -o "$tarball" || oops "failed to download '$url'"
+
+if type sha256sum > /dev/null 2>&1; then
+    hash2="$(sha256sum -b "$tarball" | cut -c1-64)"
+elif type shasum > /dev/null 2>&1; then
+    hash2="$(shasum -a 256 -b "$tarball" | cut -c1-64)"
+elif type openssl > /dev/null 2>&1; then
+    hash2="$(openssl dgst -r -sha256 "$tarball" | cut -c1-64)"
+else
+    oops "cannot verify the SHA-256 hash of '$url'; you need one of 'shasum', 'sha256sum', or 'openssl'"
+fi
+
+if [ "$hash" != "$hash2" ]; then
+    oops "SHA-256 hash mismatch in '$url'; expected $hash, got $hash2"
+fi
+
+unpack=$tmpDir/unpack
+mkdir -p "$unpack"
+< "$tarball" bzcat | tar -xf - -C "$unpack" || oops "failed to unpack '$url'"
+
+script=$(echo "$unpack"/*/install)
+
+[ -e "$script" ] || oops "installation script is missing from the binary tarball!"
+"$script" "$@"
+
+} # End of wrapping

scripts/nix-profile-daemon.sh.in

@@ -1,5 +1,5 @@
 # Only execute this file once per shell.
-if [ -n "$__ETC_PROFILE_NIX_SOURCED" ]; then return; fi
+if [ -n "${__ETC_PROFILE_NIX_SOURCED:-}" ]; then return; fi
 __ETC_PROFILE_NIX_SOURCED=1
 
 # Set up secure multi-user builds: non-root users build through the
@@ -49,6 +49,26 @@ if test -w $HOME; then
   fi
 fi
 
-export NIX_SSL_CERT_FILE="@localstatedir@/nix/profiles/default/etc/ssl/certs/ca-bundle.crt"
-export NIX_PATH="@localstatedir@/nix/profiles/per-user/root/channels"
-export PATH="$HOME/.nix-profile/bin:$HOME/.nix-profile/lib/kde4/libexec:@localstatedir@/nix/profiles/default/bin:@localstatedir@/nix/profiles/default:@localstatedir@/nix/profiles/default/lib/kde4/libexec:$PATH"
+
+# Set $NIX_SSL_CERT_FILE so that Nixpkgs applications like curl work.
+if [ ! -z "${NIX_SSL_CERT_FILE:-}" ]; then
+    : # Allow users to override the NIX_SSL_CERT_FILE
+elif [ -e /etc/ssl/certs/ca-certificates.crt ]; then # NixOS, Ubuntu, Debian, Gentoo, Arch
+    export NIX_SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt
+elif [ -e /etc/ssl/ca-bundle.pem ]; then # openSUSE Tumbleweed
+    export NIX_SSL_CERT_FILE=/etc/ssl/ca-bundle.pem
+elif [ -e /etc/ssl/certs/ca-bundle.crt ]; then # Old NixOS
+    export NIX_SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt
+elif [ -e /etc/pki/tls/certs/ca-bundle.crt ]; then # Fedora, CentOS
+    export NIX_SSL_CERT_FILE=/etc/pki/tls/certs/ca-bundle.crt
+else
+  # Fall back to what is in the nix profiles, favouring whatever is defined last.
+  for i in $NIX_PROFILES; do
+    if [ -e $i/etc/ssl/certs/ca-bundle.crt ]; then
+      export NIX_SSL_CERT_FILE=$i/etc/ssl/certs/ca-bundle.crt
+    fi
+  done
+fi
+
+export NIX_PATH="nixpkgs=@localstatedir@/nix/profiles/per-user/root/channels/nixpkgs:@localstatedir@/nix/profiles/per-user/root/channels"
+export PATH="$HOME/.nix-profile/bin:@localstatedir@/nix/profiles/default/bin:$PATH"

scripts/nix-profile.sh.in

@@ -75,7 +75,7 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then
         export NIX_SSL_CERT_FILE="$NIX_LINK/etc/ca-bundle.crt"
     fi
 
-    if [ -n "${MANPATH}" ]; then
+    if [ -n "${MANPATH-}" ]; then
         export MANPATH="$NIX_LINK/share/man:$MANPATH"
     fi
 

shell.nix

@@ -1,33 +1,13 @@
 { useClang ? false }:
 
-with import <nixpkgs> {};
+with import (builtins.fetchGit { url = https://github.com/NixOS/nixpkgs-channels.git; ref = "nixos-19.03"; }) {};
 
 with import ./release-common.nix { inherit pkgs; };
 
 (if useClang then clangStdenv else stdenv).mkDerivation {
   name = "nix";
 
-  buildInputs =
-    [ curl bison flex libxml2 libxslt
-      bzip2 xz brotli
-      pkgconfig sqlite libsodium boehmgc
-      docbook5 docbook5_xsl
-      autoconf-archive
-      (aws-sdk-cpp.override {
-        apis = ["s3"];
-        customMemoryManagement = false;
-      })
-      autoreconfHook
-
-      # For nix-perl
-      perl
-      perlPackages.DBDSQLite
-
-      # Tests
-      git
-      mercurial
-    ]
-    ++ lib.optional stdenv.isLinux libseccomp;
+  buildInputs = buildDeps ++ tarballDeps ++ perlDeps;
 
   inherit configureFlags;
 

src/boost/assert.hpp

@@ -1,38 +0,0 @@
-//
-//  boost/assert.hpp - BOOST_ASSERT(expr)
-//
-//  Copyright (c) 2001, 2002 Peter Dimov and Multi Media Ltd.
-//
-//  Permission to copy, use, modify, sell and distribute this software
-//  is granted provided this copyright notice appears in all copies.
-//  This software is provided "as is" without express or implied
-//  warranty, and with no claim as to its suitability for any purpose.
-//
-//  Note: There are no include guards. This is intentional.
-//
-//  See http://www.boost.org/libs/utility/assert.html for documentation.
-//
-
-#undef BOOST_ASSERT
-
-#if defined(BOOST_DISABLE_ASSERTS)
-
-# define BOOST_ASSERT(expr) ((void)0)
-
-#elif defined(BOOST_ENABLE_ASSERT_HANDLER)
-
-#include <boost/current_function.hpp>
-
-namespace boost
-{
-
-void assertion_failed(char const * expr, char const * function, char const * file, long line); // user defined
-
-} // namespace boost
-
-#define BOOST_ASSERT(expr) ((expr)? ((void)0): ::boost::assertion_failed(#expr, BOOST_CURRENT_FUNCTION, __FILE__, __LINE__))
-
-#else
-# include <assert.h>
-# define BOOST_ASSERT(expr) assert(expr)
-#endif

src/boost/format.hpp

@@ -1,64 +0,0 @@
-// -*- C++ -*-
-//  Boost general library 'format'   ---------------------------
-//  See http://www.boost.org for updates, documentation, and revision history.
-
-//  (C) Samuel Krempp 2001
-//                  krempp@crans.ens-cachan.fr
-//  Permission to copy, use, modify, sell and
-//  distribute this software is granted provided this copyright notice appears
-//  in all copies. This software is provided "as is" without express or implied
-//  warranty, and with no claim as to its suitability for any purpose.
-
-// ideas taken from Rüdiger Loos's format class
-// and Karl Nelson's ofstream
-
-// ----------------------------------------------------------------------------
-// format.hpp :  primary header
-// ----------------------------------------------------------------------------
-
-#ifndef BOOST_FORMAT_HPP
-#define BOOST_FORMAT_HPP
-
-#include <vector>
-#include <string>
-#include <sstream>
-#include <cassert>
-
-#if HAVE_LOCALE
-#include <locale>
-#else
-#define BOOST_NO_STD_LOCALE
-#define BOOST_NO_LOCALE_ISIDIGIT
-#include <cctype>
-#endif
-
-#include <boost/format/macros_default.hpp>
-
-
-// ****  Forward declarations ----------------------------------
-#include <boost/format/format_fwd.hpp>           // basic_format<Ch,Tr>, and other frontends
-#include <boost/format/internals_fwd.hpp>        // misc forward declarations for internal use
-
-
-// ****  Auxiliary structs (stream_format_state<Ch,Tr> , and format_item<Ch,Tr> )
-#include <boost/format/internals.hpp>    
-
-// ****  Format  class  interface --------------------------------
-#include <boost/format/format_class.hpp>
-
-// **** Exceptions -----------------------------------------------
-#include <boost/format/exceptions.hpp>
-
-// **** Implementation -------------------------------------------
-//#include <boost/format/format_implementation.hpp>   // member functions
-
-#include <boost/format/group.hpp>                   // class for grouping arguments
-
-#include <boost/format/feed_args.hpp>               // argument-feeding functions
-//#include <boost/format/parsing.hpp>                 // format-string parsing (member-)functions
-
-// **** Implementation of the free functions ----------------------
-//#include <boost/format/free_funcs.hpp>
-
-
-#endif // BOOST_FORMAT_HPP

src/boost/format/exceptions.hpp

@@ -1,96 +0,0 @@
-// -*- C++ -*-
-//  Boost general library 'format'   ---------------------------
-//  See http://www.boost.org for updates, documentation, and revision history.
-
-//  (C) Samuel Krempp 2001
-//                  krempp@crans.ens-cachan.fr
-//  Permission to copy, use, modify, sell and
-//  distribute this software is granted provided this copyright notice appears
-//  in all copies. This software is provided "as is" without express or implied
-//  warranty, and with no claim as to its suitability for any purpose.
-
-// ideas taken from Rüdiger Loos's format class
-// and Karl Nelson's ofstream (also took its parsing code as basis for printf parsing)
-
-// ------------------------------------------------------------------------------
-// exceptions.hpp 
-// ------------------------------------------------------------------------------
-
-
-#ifndef BOOST_FORMAT_EXCEPTIONS_HPP
-#define BOOST_FORMAT_EXCEPTIONS_HPP
-
-
-#include <stdexcept>
-
-
-namespace boost {
-
-namespace io {
-
-// **** exceptions -----------------------------------------------
-
-class format_error : public std::exception
-{
-public:
-  format_error() { abort(); }
-  virtual const char *what() const throw()
-  {
-    return "boost::format_error: "
-      "format generic failure";
-  }
-};
-
-class bad_format_string : public format_error
-{
-public:
-  bad_format_string() { abort(); }
-  virtual const char *what() const throw()
-  {
-    return "boost::bad_format_string: "
-      "format-string is ill-formed";
-  }
-};
-
-class too_few_args : public format_error
-{
-public:
-  too_few_args() { abort(); }
-  virtual const char *what() const throw()
-  {
-    return "boost::too_few_args: "
-      "format-string refered to more arguments than were passed";
-  }
-};
-
-class too_many_args : public format_error
-{
-public:
-  too_many_args() { abort(); }
-  virtual const char *what() const throw()
-  {
-    return "boost::too_many_args: "
-      "format-string refered to less arguments than were passed";
-  }
-};
-
-
-class  out_of_range : public format_error
-{
-public:
-  out_of_range() { abort(); }
-  virtual const char *what() const throw()
-  {
-    return "boost::out_of_range: "
-      "tried to refer to an argument (or item) number which is out of range, "
-      "according to the format string.";
-  }
-};
-
-
-} // namespace io
-
-} // namespace boost
-
-
-#endif // BOOST_FORMAT_EXCEPTIONS_HPP

src/boost/format/feed_args.hpp

@@ -1,254 +0,0 @@
-// -*- C++ -*-
-//  Boost general library 'format'   ---------------------------
-//  See http://www.boost.org for updates, documentation, and revision history.
-
-//  (C) Samuel Krempp 2001
-//                  krempp@crans.ens-cachan.fr
-//  Permission to copy, use, modify, sell and
-//  distribute this software is granted provided this copyright notice appears
-//  in all copies. This software is provided "as is" without express or implied
-//  warranty, and with no claim as to its suitability for any purpose.
-
-// ideas taken from Rüdiger Loos's format class
-// and Karl Nelson's ofstream
-
-// ----------------------------------------------------------------------------
-// feed_args.hpp :  functions for processing each argument 
-//                      (feed, feed_manip, and distribute)
-// ----------------------------------------------------------------------------
-
-
-#ifndef BOOST_FORMAT_FEED_ARGS_HPP
-#define BOOST_FORMAT_FEED_ARGS_HPP
-
-#include "boost/format/format_class.hpp"
-#include "boost/format/group.hpp"
-
-#include "boost/throw_exception.hpp"
-
-namespace boost {
-namespace io {
-namespace detail {
-namespace  { 
-
-  inline
-  void empty_buf(BOOST_IO_STD ostringstream & os) { 
-    static const std::string emptyStr;
-    os.str(emptyStr); 
-  }
-
-  void do_pad( std::string & s, 
-                std::streamsize w, 
-                const char c, 
-                std::ios::fmtflags f, 
-                bool center)
-    __attribute__ ((unused));
-
-  void do_pad( std::string & s, 
-                std::streamsize w, 
-                const char c, 
-                std::ios::fmtflags f, 
-                bool center) 
-    // applies centered / left / right  padding  to the string s.
-    // Effects : string s is padded.
-  {
-    std::streamsize n=w-s.size();
-    if(n<=0) {
-      return;
-    }
-    if(center) 
-      {
-        s.reserve(w); // allocate once for the 2 inserts
-        const std::streamsize n1 = n /2, n0 = n - n1; 
-        s.insert(s.begin(), n0, c);
-        s.append(n1, c);
-      } 
-    else 
-      {
-        if(f & std::ios::left) {
-          s.append(n, c);
-        }
-        else {
-          s.insert(s.begin(), n, c);
-        }
-      }
-  } // -do_pad(..) 
-
-
-  template<class T> inline
-  void put_head(BOOST_IO_STD ostream& , const T& ) {
-  }
-
-  template<class T> inline
-  void put_head( BOOST_IO_STD ostream& os, const group1<T>& x ) {
-    os << group_head(x.a1_); // send the first N-1 items, not the last
-  }
-
-  template<class T> inline
-  void put_last( BOOST_IO_STD ostream& os, const T& x ) {
-    os << x ;
-  }
-
-  template<class T> inline
-  void put_last( BOOST_IO_STD ostream& os, const group1<T>& x ) {
-    os << group_last(x.a1_); // this selects the last element
-  }
-
-#ifndef BOOST_NO_OVERLOAD_FOR_NON_CONST 
-  template<class T> inline
-  void put_head( BOOST_IO_STD ostream& , T& ) {
-  }
-
-  template<class T> inline
-  void put_last( BOOST_IO_STD ostream& os, T& x ) {
-    os << x ;
-  }
-#endif
-
-
-
-  
-template<class T> 
-void put( T x, 
-          const format_item& specs, 
-          std::string & res, 
-          BOOST_IO_STD ostringstream& oss_ )
-{
-  // does the actual conversion of x, with given params, into a string
-  // using the *supplied* strinstream. (the stream state is important)
-
-  typedef std::string string_t;
-  typedef format_item  format_item_t;
-
-  stream_format_state   prev_state(oss_);
-    
-  specs.state_.apply_on(oss_);
-
-  // in case x is a group, apply the manip part of it, 
-  // in order to find width
-  put_head( oss_, x );
-  empty_buf( oss_);
-
-  const std::streamsize w=oss_.width();
-  const std::ios::fmtflags fl=oss_.flags();
-  const bool internal = (fl & std::ios::internal) != 0;
-  const bool two_stepped_padding = internal
-    &&  ! ( specs.pad_scheme_ & format_item_t::spacepad ) 
-    && specs.truncate_ < 0 ;
-      
-
-  if(! two_stepped_padding) 
-    {
-      if(w>0) // handle simple padding via do_pad, not natively in stream 
-        oss_.width(0);
-      put_last( oss_, x);
-      res = oss_.str();
-
-      if (specs.truncate_ >= 0)
-        res.erase(specs.truncate_);
-
-      // complex pads :
-      if(specs.pad_scheme_ & format_item_t::spacepad)
-        {
-          if( res.size()==0 ||   ( res[0]!='+' && res[0]!='-'  ))
-            {
-              res.insert(res.begin(), 1, ' '); // insert 1 space at  pos 0
-            }
-        }
-      if(w > 0) // need do_pad
-        {
-          do_pad(res,w,oss_.fill(), fl, (specs.pad_scheme_ & format_item_t::centered) !=0 );
-        }
-    } 
-  else  // 2-stepped padding
-    {
-      put_last( oss_, x); // oss_.width() may result in padding.
-      res = oss_.str();
-      
-      if (specs.truncate_ >= 0)
-        res.erase(specs.truncate_);
-
-      if( res.size() - w > 0)
-        { //   length w exceeded
-          // either it was multi-output with first output padding up all width..
-          // either it was one big arg and we are fine.
-          empty_buf( oss_);
-          oss_.width(0);
-          put_last(oss_, x );
-          string_t tmp = oss_.str();  // minimal-length output
-          std::streamsize d;
-          if( (d=w - tmp.size()) <=0 ) 
-            {
-              // minimal length is already >= w, so no padding  (cool!)
-              res.swap(tmp);
-            }
-          else
-            { // hum..  we need to pad (it was necessarily multi-output)
-              typedef typename string_t::size_type size_type;
-              size_type i = 0;
-              while( i<tmp.size() && tmp[i] == res[i] ) // find where we should pad.
-                ++i;
-              tmp.insert(i, static_cast<size_type>( d ), oss_.fill());
-              res.swap( tmp );
-            }
-        }
-      else 
-        { // okay, only one thing was printed and padded, so res is fine.
-        }
-    }
-
-  prev_state.apply_on(oss_);
-  empty_buf( oss_);
-  oss_.clear();
-} // end- put(..)
-
-
-}  // local namespace
-
-
-
-
-
-template<class T> 
-void distribute(basic_format& self, T x) 
-  // call put(x, ..) on every occurence of the current argument :
-{
-  if(self.cur_arg_ >= self.num_args_)
-    {
-      if( self.exceptions() & too_many_args_bit )
-        boost::throw_exception(too_many_args()); // too many variables have been supplied !
-      else return;
-    }
-  for(unsigned long i=0; i < self.items_.size(); ++i)
-    {
-      if(self.items_[i].argN_ == self.cur_arg_)
-        {
-          put<T> (x, self.items_[i], self.items_[i].res_, self.oss_ );
-        }
-    }
-}
-
-template<class T> 
-basic_format&  feed(basic_format& self, T x) 
-{
-  if(self.dumped_) self.clear();
-  distribute<T> (self, x);
-  ++self.cur_arg_;
-  if(self.bound_.size() != 0)
-    {
-      while( self.cur_arg_ < self.num_args_ && self.bound_[self.cur_arg_] )
-        ++self.cur_arg_;
-    }
-
-  // this arg is finished, reset the stream's format state
-  self.state0_.apply_on(self.oss_);
-  return self;
-}
-    
-
-} // namespace detail
-} // namespace io
-} // namespace boost
-
-
-#endif //  BOOST_FORMAT_FEED_ARGS_HPP

src/boost/format/format_class.hpp

@@ -1,135 +0,0 @@
-// -*- C++ -*-
-//  Boost general library 'format'   ---------------------------
-//  See http://www.boost.org for updates, documentation, and revision history.
-
-//  (C) Samuel Krempp 2001
-//                  krempp@crans.ens-cachan.fr
-//  Permission to copy, use, modify, sell and
-//  distribute this software is granted provided this copyright notice appears
-//  in all copies. This software is provided "as is" without express or implied
-//  warranty, and with no claim as to its suitability for any purpose.
-
-// ideas taken from Rüdiger Loos's format class
-// and Karl Nelson's ofstream (also took its parsing code as basis for printf parsing)
-
-// ------------------------------------------------------------------------------
-// format_class.hpp :  class interface
-// ------------------------------------------------------------------------------
-
-
-#ifndef BOOST_FORMAT_CLASS_HPP
-#define BOOST_FORMAT_CLASS_HPP
-
-#include <vector>
-#include <string>
-
-#include <boost/format/format_fwd.hpp>
-#include <boost/format/internals_fwd.hpp>
-
-#include <boost/format/internals.hpp>
-
-namespace boost {
-
-class basic_format 
-{
-public:
-  typedef std::string                string_t;
-  typedef BOOST_IO_STD ostringstream internal_stream_t;
-private:
-  typedef BOOST_IO_STD ostream       stream_t;
-  typedef io::detail::stream_format_state  stream_format_state;
-  typedef io::detail::format_item          format_item_t;
-
-public:
-  basic_format(const char* str);
-  basic_format(const string_t& s);
-#ifndef BOOST_NO_STD_LOCALE
-  basic_format(const char* str, const std::locale & loc);
-  basic_format(const string_t& s, const std::locale & loc);
-#endif // no locale
-  basic_format(const basic_format& x);
-  basic_format& operator= (const basic_format& x);
-
-  basic_format& clear(); // empty the string buffers (except bound arguments, see clear_binds() )
-
-  // pass arguments through those operators :
-  template<class T>  basic_format&   operator%(const T& x) 
-  { 
-    return io::detail::feed<const T&>(*this,x);
-  }
-
-#ifndef BOOST_NO_OVERLOAD_FOR_NON_CONST
-  template<class T>  basic_format&   operator%(T& x) 
-  {
-    return io::detail::feed<T&>(*this,x);
-  }
-#endif
-
-
-  // system for binding arguments :
-  template<class T>  
-  basic_format&         bind_arg(int argN, const T& val) 
-  {
-    return io::detail::bind_arg_body(*this, argN, val); 
-  }
-  basic_format&         clear_bind(int argN);
-  basic_format&         clear_binds();
-
-  // modify the params of a directive, by applying a manipulator :
-  template<class T> 
-  basic_format&  modify_item(int itemN, const T& manipulator) 
-  {
-    return io::detail::modify_item_body(*this, itemN, manipulator) ;
-  }
-
-  // Choosing which errors will throw exceptions :
-  unsigned char exceptions() const;
-  unsigned char exceptions(unsigned char newexcept);
-
-  // final output
-  string_t str() const;
-  friend BOOST_IO_STD ostream& 
-  operator<< ( BOOST_IO_STD ostream& , const basic_format& ); 
-                      
-
-  template<class T>  friend basic_format&  
-  io::detail::feed(basic_format&, T);
-    
-  template<class T>  friend   
-  void io::detail::distribute(basic_format&, T);
-  
-  template<class T>  friend
-  basic_format&  io::detail::modify_item_body(basic_format&, int, const T&);
-
-  template<class T> friend
-  basic_format&  io::detail::bind_arg_body(basic_format&, int, const T&);
-
-// make the members private only if the friend templates are supported
-private:
-
-  // flag bits, used for style_
-  enum style_values  { ordered = 1,        // set only if all directives are  positional directives
-                       special_needs = 4 };     
-
-  // parse the format string :
-  void parse(const string_t&);
-
-  int                           style_;         // style of format-string :  positional or not, etc
-  int                           cur_arg_;       // keep track of wich argument will come
-  int                           num_args_;      // number of expected arguments
-  mutable bool                  dumped_;        // true only after call to str() or <<
-  std::vector<format_item_t>    items_;         // vector of directives (aka items)
-  string_t                      prefix_;        // piece of string to insert before first item
-
-  std::vector<bool>             bound_;         // stores which arguments were bound
-                                                //   size = num_args OR zero
-  internal_stream_t             oss_;           // the internal stream.
-  stream_format_state           state0_;        // reference state for oss_
-  unsigned char                 exceptions_;
-}; // class basic_format
-
-
-} // namespace boost
-
-
-#endif // BOOST_FORMAT_CLASS_HPP

src/boost/format/format_fwd.hpp

@@ -1,49 +0,0 @@
-// -*- C++ -*-
-//  Boost general library 'format'   ---------------------------
-//  See http://www.boost.org for updates, documentation, and revision history.
-
-//  (C) Samuel Krempp 2001
-//                  krempp@crans.ens-cachan.fr
-//  Permission to copy, use, modify, sell and
-//  distribute this software is granted provided this copyright notice appears
-//  in all copies. This software is provided "as is" without express or implied
-//  warranty, and with no claim as to its suitability for any purpose.
-
-// ideas taken from Rüdiger Loos's format class
-// and Karl Nelson's ofstream (also took its parsing code as basis for printf parsing)
-
-// ------------------------------------------------------------------------------
-// format_fwd.hpp :  forward declarations, for primary header format.hpp
-// ------------------------------------------------------------------------------
-
-#ifndef BOOST_FORMAT_FWD_HPP
-#define BOOST_FORMAT_FWD_HPP
-
-#include <string>
-#include <iosfwd>
-
-namespace boost {
-
-class basic_format;
-
-typedef basic_format    format;
-
-namespace io {
-enum format_error_bits { bad_format_string_bit = 1, 
-                         too_few_args_bit = 2, too_many_args_bit = 4,
-                         out_of_range_bit = 8,
-                         all_error_bits = 255, no_error_bits=0 };
-                  
-// Convertion:  format   to   string
-std::string     str(const basic_format& ) ;
-
-} // namespace io
-
-
-BOOST_IO_STD ostream& 
-operator<<( BOOST_IO_STD ostream&, const basic_format&);
-
-
-} // namespace boost
-
-#endif // BOOST_FORMAT_FWD_HPP

src/boost/format/format_implementation.cc

@@ -1,256 +0,0 @@
-// -*- C++ -*-
-//  Boost general library format ---------------------------
-//  See http://www.boost.org for updates, documentation, and revision history.
-
-//  (C) Samuel Krempp 2001
-//                  krempp@crans.ens-cachan.fr
-//  Permission to copy, use, modify, sell and
-//  distribute this software is granted provided this copyright notice appears
-//  in all copies. This software is provided "as is" without express or implied
-//  warranty, and with no claim as to its suitability for any purpose.
-
-// ideas taken from Rüdiger Loos's format class
-// and Karl Nelson's ofstream
-
-// ----------------------------------------------------------------------------
-// format_implementation.hpp  Implementation of the basic_format class
-// ----------------------------------------------------------------------------
-
-
-#ifndef BOOST_FORMAT_IMPLEMENTATION_HPP
-#define BOOST_FORMAT_IMPLEMENTATION_HPP
-
-#include <boost/throw_exception.hpp>
-#include <boost/assert.hpp>
-#include <boost/format.hpp>
-
-namespace boost {
-
-// --------  format:: -------------------------------------------
-basic_format::basic_format(const char* str)
-    : style_(0), cur_arg_(0), num_args_(0), dumped_(false),
-      items_(), oss_(), exceptions_(io::all_error_bits)
-{
-    state0_.set_by_stream(oss_);
-    string_t emptyStr;
-    if( !str) str = emptyStr.c_str();
-    parse( str );
-}
-
-#ifndef BOOST_NO_STD_LOCALE
-basic_format::basic_format(const char* str, const std::locale & loc)
-    : style_(0), cur_arg_(0), num_args_(0), dumped_(false),
-      items_(), oss_(), exceptions_(io::all_error_bits)
-{
-    oss_.imbue( loc );
-    state0_.set_by_stream(oss_);
-    string_t emptyStr;
-    if( !str) str = emptyStr.c_str();
-    parse( str );
-}
-
-basic_format::basic_format(const string_t& s, const std::locale & loc)
-    : style_(0), cur_arg_(0), num_args_(0), dumped_(false),
-      items_(),  oss_(), exceptions_(io::all_error_bits)
-{
-    oss_.imbue( loc );
-    state0_.set_by_stream(oss_);
-    parse(s);  
-}
-#endif //BOOST_NO_STD_LOCALE
-
-basic_format::basic_format(const string_t& s)
-    : style_(0), cur_arg_(0), num_args_(0), dumped_(false),
-      items_(),  oss_(), exceptions_(io::all_error_bits)
-{
-    state0_.set_by_stream(oss_);
-    parse(s);  
-}
-
-basic_format:: basic_format(const basic_format& x)
-    : style_(x.style_), cur_arg_(x.cur_arg_), num_args_(x.num_args_), dumped_(false), 
-      items_(x.items_), prefix_(x.prefix_), bound_(x.bound_), 
-      oss_(),   // <- we obviously can't copy x.oss_
-      state0_(x.state0_), exceptions_(x.exceptions_)
-{ 
-    state0_.apply_on(oss_);
-} 
-
-basic_format& basic_format::operator= (const basic_format& x)
-{
-    if(this == &x)
-      return *this;
-    state0_ = x.state0_;
-    state0_.apply_on(oss_);
-
-    // plus all the other (trivial) assignments :
-    exceptions_ = x.exceptions_;
-    items_ = x.items_;
-    prefix_ = x.prefix_;
-    bound_=x.bound_;
-    style_=x.style_; 
-    cur_arg_=x.cur_arg_; 
-    num_args_=x.num_args_;
-    dumped_=x.dumped_;
-    return *this;
-}
-
-
-unsigned char basic_format::exceptions() const 
-{
-  return exceptions_; 
-}
-
-unsigned char basic_format::exceptions(unsigned char newexcept) 
-{ 
-  unsigned char swp = exceptions_; 
-  exceptions_ = newexcept; 
-  return swp; 
-}
-
-
-basic_format& basic_format ::clear()
-  // empty the string buffers (except bound arguments, see clear_binds() )
-  // and make the format object ready for formatting a new set of arguments
-{
-    BOOST_ASSERT( bound_.size()==0 || num_args_ == static_cast<int>(bound_.size()) );
-
-    for(unsigned long i=0; i<items_.size(); ++i){
-      items_[i].state_ = items_[i].ref_state_;
-      // clear converted strings only if the corresponding argument is not  bound :
-      if( bound_.size()==0 || !bound_[ items_[i].argN_ ] )  items_[i].res_.resize(0);
-    }
-    cur_arg_=0; dumped_=false;
-    // maybe first arg is bound:
-    if(bound_.size() != 0)
-      {
-        while(cur_arg_ < num_args_ && bound_[cur_arg_] )      ++cur_arg_;
-      }
-    return *this;
-}
-
-basic_format& basic_format ::clear_binds() 
-  // cancel all bindings, and clear()
-{
-    bound_.resize(0);
-    clear();
-    return *this;
-}
-
-basic_format& basic_format::clear_bind(int argN) 
-  // cancel the binding of ONE argument, and clear()
-{
-    if(argN<1 || argN > num_args_ || bound_.size()==0 || !bound_[argN-1] ) 
-      {
-        if( exceptions() & io::out_of_range_bit )
-          boost::throw_exception(io::out_of_range()); // arg not in range.
-        else return *this;
-      }
-    bound_[argN-1]=false;
-    clear();
-    return *this;
-}
-
-
-
-std::string basic_format::str() const
-{
-  dumped_=true;
-  if(items_.size()==0)
-    return prefix_;
-  if( cur_arg_ < num_args_)
-      if( exceptions() & io::too_few_args_bit )
-        boost::throw_exception(io::too_few_args()); // not enough variables have been supplied !
-
-  unsigned long sz = prefix_.size();
-  unsigned long i;
-  for(i=0; i < items_.size(); ++i) 
-    sz += items_[i].res_.size() + items_[i].appendix_.size();
-  string_t res;
-  res.reserve(sz);
-
-  res += prefix_;
-  for(i=0; i < items_.size(); ++i) 
-  {
-    const format_item_t& item = items_[i];
-    res += item.res_;
-    if( item.argN_ == format_item_t::argN_tabulation) 
-    { 
-      BOOST_ASSERT( item.pad_scheme_ & format_item_t::tabulation);
-      std::streamsize  n = item.state_.width_ - res.size();
-      if( n > 0 )
-        res.append( n, item.state_.fill_ );
-    }
-    res += item.appendix_;
-  }
-  return res;
-}
-
-namespace io {
-namespace detail {
-
-template<class T>
-basic_format&  bind_arg_body( basic_format& self, 
-                                      int argN, 
-                                      const T& val)
-  // bind one argument to a fixed value
-  // this is persistent over clear() calls, thus also over str() and <<
-{
-    if(self.dumped_) self.clear(); // needed, because we will modify cur_arg_..
-    if(argN<1 || argN > self.num_args_) 
-      {
-        if( self.exceptions() & io::out_of_range_bit )
-          boost::throw_exception(io::out_of_range()); // arg not in range.
-        else return self;
-      }
-    if(self.bound_.size()==0) 
-      self.bound_.assign(self.num_args_,false);
-    else 
-      BOOST_ASSERT( self.num_args_ == static_cast<signed int>(self.bound_.size()) );
-    int o_cur_arg = self.cur_arg_;
-    self.cur_arg_ = argN-1; // arrays begin at 0
-
-    self.bound_[self.cur_arg_]=false; // if already set, we unset and re-sets..
-    self.operator%(val); // put val at the right place, because cur_arg is set
-    
-
-    // Now re-position cur_arg before leaving :
-    self.cur_arg_ = o_cur_arg; 
-    self.bound_[argN-1]=true;
-    if(self.cur_arg_ == argN-1 )
-      // hum, now this arg is bound, so move to next free arg
-      {
-        while(self.cur_arg_ < self.num_args_ && self.bound_[self.cur_arg_])   ++self.cur_arg_;
-      }
-    // In any case, we either have all args, or are on a non-binded arg :
-    BOOST_ASSERT( self.cur_arg_ >= self.num_args_ || ! self.bound_[self.cur_arg_]);
-    return self;
-}
-
-template<class T>
-basic_format&  modify_item_body( basic_format& self,
-                                      int itemN, 
-                                      const T& manipulator)
-  // applies a manipulator to the format_item describing a given directive.
-  // this is a permanent change, clear or clear_binds won't cancel that.
-{
-  if(itemN<1 || itemN >= static_cast<signed int>(self.items_.size() )) 
-    {
-      if( self.exceptions() & io::out_of_range_bit ) 
-        boost::throw_exception(io::out_of_range()); // item not in range.
-      else return self;
-    }
-  self.items_[itemN-1].ref_state_.apply_manip( manipulator );
-  self.items_[itemN-1].state_ = self.items_[itemN-1].ref_state_;
-  return self;
-}
-
-} // namespace detail
-
-} // namespace io
-
-} // namespace boost
-
-
-
-#endif  // BOOST_FORMAT_IMPLEMENTATION_HPP

src/boost/format/free_funcs.cc

@@ -1,71 +0,0 @@
-// -*- C++ -*-
-//  Boost general library 'format'   ---------------------------
-//  See http://www.boost.org for updates, documentation, and revision history.
-
-//  (C) Samuel Krempp 2001
-//                  krempp@crans.ens-cachan.fr
-//  Permission to copy, use, modify, sell and
-//  distribute this software is granted provided this copyright notice appears
-//  in all copies. This software is provided "as is" without express or implied
-//  warranty, and with no claim as to its suitability for any purpose.
-
-// ideas taken from Rüdiger Loos's format class
-// and Karl Nelson's ofstream (also took its parsing code as basis for printf parsing)
-
-// ------------------------------------------------------------------------------
-// free_funcs.hpp :  implementation of the free functions declared in namespace format
-// ------------------------------------------------------------------------------
-
-#ifndef BOOST_FORMAT_FUNCS_HPP
-#define BOOST_FORMAT_FUNCS_HPP
-
-#include "boost/format.hpp"
-#include "boost/throw_exception.hpp"
-
-namespace boost {
-
-namespace io {
-  inline 
-  std::string str(const basic_format& f) 
-    // adds up all pieces of strings and converted items, and return the formatted string
-  {
-    return f.str();
-  }
-}   // - namespace io
-
-BOOST_IO_STD ostream& 
-operator<<( BOOST_IO_STD ostream& os, 
-            const boost::basic_format& f) 
-  // effect: "return os << str(f);" but we can try to do it faster
-{
-  typedef boost::basic_format   format_t;
-  if(f.items_.size()==0) 
-    os << f.prefix_;
-  else {
-    if(f.cur_arg_ < f.num_args_)
-      if( f.exceptions() & io::too_few_args_bit )
-        boost::throw_exception(io::too_few_args()); // not enough variables have been supplied !
-    if(f.style_ & format_t::special_needs) 
-        os << f.str();
-    else {
-    // else we dont have to count chars output, so we dump directly to os :
-      os << f.prefix_;
-      for(unsigned long i=0; i<f.items_.size(); ++i) 
-        {
-          const format_t::format_item_t& item = f.items_[i];
-          os << item.res_;
-          os << item.appendix_;
-
-        }
-    }
-  }
-  f.dumped_=true;
-  return os;
-}
-
-
-
-} // namespace boost
-
-
-#endif // BOOST_FORMAT_FUNCS_HPP

src/boost/format/group.hpp

@@ -1,680 +0,0 @@
-
-// -*- C++ -*-
-//  Boost general library 'format'   ---------------------------
-//  See http://www.boost.org for updates, documentation, and revision history.
-
-//  (C) Samuel Krempp 2001
-//                  krempp@crans.ens-cachan.fr
-//  Permission to copy, use, modify, sell and
-//  distribute this software is granted provided this copyright notice appears
-//  in all copies. This software is provided "as is" without express or implied
-//  warranty, and with no claim as to its suitability for any purpose.
-
-// ideas taken from Rüdiger Loos's format class
-// and Karl Nelson's ofstream
-
-// ----------------------------------------------------------------------------
-
-// group.hpp :  encapsulates a group of manipulators along with an argument
-//                      
-// group_head : cut the last element of a group out.
-// (is overloaded below on each type of group)
-
-// group_last : returns the last element of a group
-// (is overloaded below on each type of group)
-
-// ----------------------------------------------------------------------------
-
-
-#ifndef BOOST_FORMAT_GROUP_HPP
-#define BOOST_FORMAT_GROUP_HPP
-
-
-namespace boost {
-namespace io {
-
-
-namespace detail {
-
-
-// empty group, but useful even though.
-struct group0 
-{
-    group0()      {}
-};
-
-template <class Ch, class Tr>
-inline
-BOOST_IO_STD ostream&
-operator << ( BOOST_IO_STD ostream& os,
-             const group0& )
-{ 
-   return os; 
-}
-
-template <class T1>
-struct group1
-{
-    T1 a1_;
-    group1(T1 a1)
-      : a1_(a1)
-      {}
-};
-
-template <class Ch, class Tr, class T1>
-inline
-BOOST_IO_STD ostream&
-operator << (BOOST_IO_STD ostream& os,
-             const group1<T1>& x)
-{ 
-   os << x.a1_;  
-   return os; 
-}
-
-
-
-
-template <class T1,class T2>
-struct group2
-{
-    T1 a1_;
-    T2 a2_;
-    group2(T1 a1,T2 a2)
-      : a1_(a1),a2_(a2)
-      {}
-};
-
-template <class Ch, class Tr, class T1,class T2>
-inline
-BOOST_IO_STD ostream&
-operator << (BOOST_IO_STD ostream& os,
-             const group2<T1,T2>& x)
-{ 
-   os << x.a1_<< x.a2_;  
-   return os; 
-}
-
-template <class T1,class T2,class T3>
-struct group3
-{
-    T1 a1_;
-    T2 a2_;
-    T3 a3_;
-    group3(T1 a1,T2 a2,T3 a3)
-      : a1_(a1),a2_(a2),a3_(a3)
-      {}
-};
-
-template <class Ch, class Tr, class T1,class T2,class T3>
-inline
-BOOST_IO_STD ostream&
-operator << (BOOST_IO_STD ostream& os,
-             const group3<T1,T2,T3>& x)
-{ 
-   os << x.a1_<< x.a2_<< x.a3_;  
-   return os; 
-}
-
-template <class T1,class T2,class T3,class T4>
-struct group4
-{
-    T1 a1_;
-    T2 a2_;
-    T3 a3_;
-    T4 a4_;
-    group4(T1 a1,T2 a2,T3 a3,T4 a4)
-      : a1_(a1),a2_(a2),a3_(a3),a4_(a4)
-      {}
-};
-
-template <class Ch, class Tr, class T1,class T2,class T3,class T4>
-inline
-BOOST_IO_STD ostream&
-operator << (BOOST_IO_STD ostream& os,
-             const group4<T1,T2,T3,T4>& x)
-{ 
-   os << x.a1_<< x.a2_<< x.a3_<< x.a4_;  
-   return os; 
-}
-
-template <class T1,class T2,class T3,class T4,class T5>
-struct group5
-{
-    T1 a1_;
-    T2 a2_;
-    T3 a3_;
-    T4 a4_;
-    T5 a5_;
-    group5(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5)
-      : a1_(a1),a2_(a2),a3_(a3),a4_(a4),a5_(a5)
-      {}
-};
-
-template <class Ch, class Tr, class T1,class T2,class T3,class T4,class T5>
-inline
-BOOST_IO_STD ostream&
-operator << (BOOST_IO_STD ostream& os,
-             const group5<T1,T2,T3,T4,T5>& x)
-{ 
-   os << x.a1_<< x.a2_<< x.a3_<< x.a4_<< x.a5_;  
-   return os; 
-}
-
-template <class T1,class T2,class T3,class T4,class T5,class T6>
-struct group6
-{
-    T1 a1_;
-    T2 a2_;
-    T3 a3_;
-    T4 a4_;
-    T5 a5_;
-    T6 a6_;
-    group6(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5,T6 a6)
-      : a1_(a1),a2_(a2),a3_(a3),a4_(a4),a5_(a5),a6_(a6)
-      {}
-};
-
-template <class Ch, class Tr, class T1,class T2,class T3,class T4,class T5,class T6>
-inline
-BOOST_IO_STD ostream&
-operator << (BOOST_IO_STD ostream& os,
-             const group6<T1,T2,T3,T4,T5,T6>& x)
-{ 
-   os << x.a1_<< x.a2_<< x.a3_<< x.a4_<< x.a5_<< x.a6_;  
-   return os; 
-}
-
-template <class T1,class T2,class T3,class T4,class T5,class T6,class T7>
-struct group7
-{
-    T1 a1_;
-    T2 a2_;
-    T3 a3_;
-    T4 a4_;
-    T5 a5_;
-    T6 a6_;
-    T7 a7_;
-    group7(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5,T6 a6,T7 a7)
-      : a1_(a1),a2_(a2),a3_(a3),a4_(a4),a5_(a5),a6_(a6),a7_(a7)
-      {}
-};
-
-template <class Ch, class Tr, class T1,class T2,class T3,class T4,class T5,class T6,class T7>
-inline
-BOOST_IO_STD ostream&
-operator << (BOOST_IO_STD ostream& os,
-             const group7<T1,T2,T3,T4,T5,T6,T7>& x)
-{ 
-   os << x.a1_<< x.a2_<< x.a3_<< x.a4_<< x.a5_<< x.a6_<< x.a7_;  
-   return os; 
-}
-
-template <class T1,class T2,class T3,class T4,class T5,class T6,class T7,class T8>
-struct group8
-{
-    T1 a1_;
-    T2 a2_;
-    T3 a3_;
-    T4 a4_;
-    T5 a5_;
-    T6 a6_;
-    T7 a7_;
-    T8 a8_;
-    group8(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5,T6 a6,T7 a7,T8 a8)
-      : a1_(a1),a2_(a2),a3_(a3),a4_(a4),a5_(a5),a6_(a6),a7_(a7),a8_(a8)
-      {}
-};
-
-template <class Ch, class Tr, class T1,class T2,class T3,class T4,class T5,class T6,class T7,class T8>
-inline
-BOOST_IO_STD ostream&
-operator << (BOOST_IO_STD ostream& os,
-             const group8<T1,T2,T3,T4,T5,T6,T7,T8>& x)
-{ 
-   os << x.a1_<< x.a2_<< x.a3_<< x.a4_<< x.a5_<< x.a6_<< x.a7_<< x.a8_;  
-   return os; 
-}
-
-template <class T1,class T2,class T3,class T4,class T5,class T6,class T7,class T8,class T9>
-struct group9
-{
-    T1 a1_;
-    T2 a2_;
-    T3 a3_;
-    T4 a4_;
-    T5 a5_;
-    T6 a6_;
-    T7 a7_;
-    T8 a8_;
-    T9 a9_;
-    group9(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5,T6 a6,T7 a7,T8 a8,T9 a9)
-      : a1_(a1),a2_(a2),a3_(a3),a4_(a4),a5_(a5),a6_(a6),a7_(a7),a8_(a8),a9_(a9)
-      {}
-};
-
-template <class Ch, class Tr, class T1,class T2,class T3,class T4,class T5,class T6,class T7,class T8,class T9>
-inline
-BOOST_IO_STD ostream&
-operator << (BOOST_IO_STD ostream& os,
-             const group9<T1,T2,T3,T4,T5,T6,T7,T8,T9>& x)
-{ 
-   os << x.a1_<< x.a2_<< x.a3_<< x.a4_<< x.a5_<< x.a6_<< x.a7_<< x.a8_<< x.a9_;  
-   return os; 
-}
-
-template <class T1,class T2,class T3,class T4,class T5,class T6,class T7,class T8,class T9,class T10>
-struct group10
-{
-    T1 a1_;
-    T2 a2_;
-    T3 a3_;
-    T4 a4_;
-    T5 a5_;
-    T6 a6_;
-    T7 a7_;
-    T8 a8_;
-    T9 a9_;
-    T10 a10_;
-    group10(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5,T6 a6,T7 a7,T8 a8,T9 a9,T10 a10)
-      : a1_(a1),a2_(a2),a3_(a3),a4_(a4),a5_(a5),a6_(a6),a7_(a7),a8_(a8),a9_(a9),a10_(a10)
-      {}
-};
-
-template <class Ch, class Tr, class T1,class T2,class T3,class T4,class T5,class T6,class T7,class T8,class T9,class T10>
-inline
-BOOST_IO_STD ostream&
-operator << (BOOST_IO_STD ostream& os,
-             const group10<T1,T2,T3,T4,T5,T6,T7,T8,T9,T10>& x)
-{ 
-   os << x.a1_<< x.a2_<< x.a3_<< x.a4_<< x.a5_<< x.a6_<< x.a7_<< x.a8_<< x.a9_<< x.a10_;  
-   return os; 
-}
-
-
-
-
-template <class T1,class T2>
-inline
-group1<T1> 
-group_head( group2<T1,T2> const& x)
-{
-   return group1<T1> (x.a1_); 
-}
-
-template <class T1,class T2>
-inline
-group1<T2> 
-group_last( group2<T1,T2> const& x)
-{
-   return group1<T2> (x.a2_); 
-}
-
-
-
-template <class T1,class T2,class T3>
-inline
-group2<T1,T2> 
-group_head( group3<T1,T2,T3> const& x)
-{
-   return group2<T1,T2> (x.a1_,x.a2_); 
-}
-
-template <class T1,class T2,class T3>
-inline
-group1<T3> 
-group_last( group3<T1,T2,T3> const& x)
-{
-   return group1<T3> (x.a3_); 
-}
-
-
-
-template <class T1,class T2,class T3,class T4>
-inline
-group3<T1,T2,T3> 
-group_head( group4<T1,T2,T3,T4> const& x)
-{
-   return group3<T1,T2,T3> (x.a1_,x.a2_,x.a3_); 
-}
-
-template <class T1,class T2,class T3,class T4>
-inline
-group1<T4> 
-group_last( group4<T1,T2,T3,T4> const& x)
-{
-   return group1<T4> (x.a4_); 
-}
-
-
-
-template <class T1,class T2,class T3,class T4,class T5>
-inline
-group4<T1,T2,T3,T4> 
-group_head( group5<T1,T2,T3,T4,T5> const& x)
-{
-   return group4<T1,T2,T3,T4> (x.a1_,x.a2_,x.a3_,x.a4_); 
-}
-
-template <class T1,class T2,class T3,class T4,class T5>
-inline
-group1<T5> 
-group_last( group5<T1,T2,T3,T4,T5> const& x)
-{
-   return group1<T5> (x.a5_); 
-}
-
-
-
-template <class T1,class T2,class T3,class T4,class T5,class T6>
-inline
-group5<T1,T2,T3,T4,T5> 
-group_head( group6<T1,T2,T3,T4,T5,T6> const& x)
-{
-   return group5<T1,T2,T3,T4,T5> (x.a1_,x.a2_,x.a3_,x.a4_,x.a5_); 
-}
-
-template <class T1,class T2,class T3,class T4,class T5,class T6>
-inline
-group1<T6> 
-group_last( group6<T1,T2,T3,T4,T5,T6> const& x)
-{
-   return group1<T6> (x.a6_); 
-}
-
-
-
-template <class T1,class T2,class T3,class T4,class T5,class T6,class T7>
-inline
-group6<T1,T2,T3,T4,T5,T6> 
-group_head( group7<T1,T2,T3,T4,T5,T6,T7> const& x)
-{
-   return group6<T1,T2,T3,T4,T5,T6> (x.a1_,x.a2_,x.a3_,x.a4_,x.a5_,x.a6_); 
-}
-
-template <class T1,class T2,class T3,class T4,class T5,class T6,class T7>
-inline
-group1<T7> 
-group_last( group7<T1,T2,T3,T4,T5,T6,T7> const& x)
-{
-   return group1<T7> (x.a7_); 
-}
-
-
-
-template <class T1,class T2,class T3,class T4,class T5,class T6,class T7,class T8>
-inline
-group7<T1,T2,T3,T4,T5,T6,T7> 
-group_head( group8<T1,T2,T3,T4,T5,T6,T7,T8> const& x)
-{
-   return group7<T1,T2,T3,T4,T5,T6,T7> (x.a1_,x.a2_,x.a3_,x.a4_,x.a5_,x.a6_,x.a7_); 
-}
-
-template <class T1,class T2,class T3,class T4,class T5,class T6,class T7,class T8>
-inline
-group1<T8> 
-group_last( group8<T1,T2,T3,T4,T5,T6,T7,T8> const& x)
-{
-   return group1<T8> (x.a8_); 
-}
-
-
-
-template <class T1,class T2,class T3,class T4,class T5,class T6,class T7,class T8,class T9>
-inline
-group8<T1,T2,T3,T4,T5,T6,T7,T8> 
-group_head( group9<T1,T2,T3,T4,T5,T6,T7,T8,T9> const& x)
-{
-   return group8<T1,T2,T3,T4,T5,T6,T7,T8> (x.a1_,x.a2_,x.a3_,x.a4_,x.a5_,x.a6_,x.a7_,x.a8_); 
-}
-
-template <class T1,class T2,class T3,class T4,class T5,class T6,class T7,class T8,class T9>
-inline
-group1<T9> 
-group_last( group9<T1,T2,T3,T4,T5,T6,T7,T8,T9> const& x)
-{
-   return group1<T9> (x.a9_); 
-}
-
-
-
-template <class T1,class T2,class T3,class T4,class T5,class T6,class T7,class T8,class T9,class T10>
-inline
-group9<T1,T2,T3,T4,T5,T6,T7,T8,T9> 
-group_head( group10<T1,T2,T3,T4,T5,T6,T7,T8,T9,T10> const& x)
-{
-   return group9<T1,T2,T3,T4,T5,T6,T7,T8,T9> (x.a1_,x.a2_,x.a3_,x.a4_,x.a5_,x.a6_,x.a7_,x.a8_,x.a9_); 
-}
-
-template <class T1,class T2,class T3,class T4,class T5,class T6,class T7,class T8,class T9,class T10>
-inline
-group1<T10> 
-group_last( group10<T1,T2,T3,T4,T5,T6,T7,T8,T9,T10> const& x)
-{
-   return group1<T10> (x.a10_); 
-}
-
-
-
-
-
-} // namespace detail
-
-
-
-// helper functions
-
-
-inline detail::group1< detail::group0 >  
-group() { return detail::group1< detail::group0 > ( detail::group0() ); }
-
-template  <class T1, class Var> 
-inline
-detail::group1< detail::group2<T1, Var const&> >
-  group(T1 a1, Var const& var)
-{ 
-   return detail::group1< detail::group2<T1, Var const&> >
-                   ( detail::group2<T1, Var const&> 
-                        (a1, var) 
-                  );
-}
-
-template  <class T1,class T2, class Var> 
-inline
-detail::group1< detail::group3<T1,T2, Var const&> >
-  group(T1 a1,T2 a2, Var const& var)
-{ 
-   return detail::group1< detail::group3<T1,T2, Var const&> >
-                   ( detail::group3<T1,T2, Var const&> 
-                        (a1,a2, var) 
-                  );
-}
-
-template  <class T1,class T2,class T3, class Var> 
-inline
-detail::group1< detail::group4<T1,T2,T3, Var const&> >
-  group(T1 a1,T2 a2,T3 a3, Var const& var)
-{ 
-   return detail::group1< detail::group4<T1,T2,T3, Var const&> >
-                   ( detail::group4<T1,T2,T3, Var const&> 
-                        (a1,a2,a3, var) 
-                  );
-}
-
-template  <class T1,class T2,class T3,class T4, class Var> 
-inline
-detail::group1< detail::group5<T1,T2,T3,T4, Var const&> >
-  group(T1 a1,T2 a2,T3 a3,T4 a4, Var const& var)
-{ 
-   return detail::group1< detail::group5<T1,T2,T3,T4, Var const&> >
-                   ( detail::group5<T1,T2,T3,T4, Var const&> 
-                        (a1,a2,a3,a4, var) 
-                  );
-}
-
-template  <class T1,class T2,class T3,class T4,class T5, class Var> 
-inline
-detail::group1< detail::group6<T1,T2,T3,T4,T5, Var const&> >
-  group(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5, Var const& var)
-{ 
-   return detail::group1< detail::group6<T1,T2,T3,T4,T5, Var const&> >
-                   ( detail::group6<T1,T2,T3,T4,T5, Var const&> 
-                        (a1,a2,a3,a4,a5, var) 
-                  );
-}
-
-template  <class T1,class T2,class T3,class T4,class T5,class T6, class Var> 
-inline
-detail::group1< detail::group7<T1,T2,T3,T4,T5,T6, Var const&> >
-  group(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5,T6 a6, Var const& var)
-{ 
-   return detail::group1< detail::group7<T1,T2,T3,T4,T5,T6, Var const&> >
-                   ( detail::group7<T1,T2,T3,T4,T5,T6, Var const&> 
-                        (a1,a2,a3,a4,a5,a6, var) 
-                  );
-}
-
-template  <class T1,class T2,class T3,class T4,class T5,class T6,class T7, class Var> 
-inline
-detail::group1< detail::group8<T1,T2,T3,T4,T5,T6,T7, Var const&> >
-  group(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5,T6 a6,T7 a7, Var const& var)
-{ 
-   return detail::group1< detail::group8<T1,T2,T3,T4,T5,T6,T7, Var const&> >
-                   ( detail::group8<T1,T2,T3,T4,T5,T6,T7, Var const&> 
-                        (a1,a2,a3,a4,a5,a6,a7, var) 
-                  );
-}
-
-template  <class T1,class T2,class T3,class T4,class T5,class T6,class T7,class T8, class Var> 
-inline
-detail::group1< detail::group9<T1,T2,T3,T4,T5,T6,T7,T8, Var const&> >
-  group(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5,T6 a6,T7 a7,T8 a8, Var const& var)
-{ 
-   return detail::group1< detail::group9<T1,T2,T3,T4,T5,T6,T7,T8, Var const&> >
-                   ( detail::group9<T1,T2,T3,T4,T5,T6,T7,T8, Var const&> 
-                        (a1,a2,a3,a4,a5,a6,a7,a8, var) 
-                  );
-}
-
-template  <class T1,class T2,class T3,class T4,class T5,class T6,class T7,class T8,class T9, class Var> 
-inline
-detail::group1< detail::group10<T1,T2,T3,T4,T5,T6,T7,T8,T9, Var const&> >
-  group(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5,T6 a6,T7 a7,T8 a8,T9 a9, Var const& var)
-{ 
-   return detail::group1< detail::group10<T1,T2,T3,T4,T5,T6,T7,T8,T9, Var const&> >
-                   ( detail::group10<T1,T2,T3,T4,T5,T6,T7,T8,T9, Var const&> 
-                        (a1,a2,a3,a4,a5,a6,a7,a8,a9, var) 
-                  );
-}
-
-
-#ifndef BOOST_NO_OVERLOAD_FOR_NON_CONST
-
-template  <class T1, class Var> 
-inline
-detail::group1< detail::group2<T1, Var&> >
-  group(T1 a1, Var& var)
-{ 
-   return detail::group1< detail::group2<T1, Var&> >
-                   ( detail::group2<T1, Var&> 
-                        (a1, var) 
-                  );
-}
-
-template  <class T1,class T2, class Var> 
-inline
-detail::group1< detail::group3<T1,T2, Var&> >
-  group(T1 a1,T2 a2, Var& var)
-{ 
-   return detail::group1< detail::group3<T1,T2, Var&> >
-                   ( detail::group3<T1,T2, Var&> 
-                        (a1,a2, var) 
-                  );
-}
-
-template  <class T1,class T2,class T3, class Var> 
-inline
-detail::group1< detail::group4<T1,T2,T3, Var&> >
-  group(T1 a1,T2 a2,T3 a3, Var& var)
-{ 
-   return detail::group1< detail::group4<T1,T2,T3, Var&> >
-                   ( detail::group4<T1,T2,T3, Var&> 
-                        (a1,a2,a3, var) 
-                  );
-}
-
-template  <class T1,class T2,class T3,class T4, class Var> 
-inline
-detail::group1< detail::group5<T1,T2,T3,T4, Var&> >
-  group(T1 a1,T2 a2,T3 a3,T4 a4, Var& var)
-{ 
-   return detail::group1< detail::group5<T1,T2,T3,T4, Var&> >
-                   ( detail::group5<T1,T2,T3,T4, Var&> 
-                        (a1,a2,a3,a4, var) 
-                  );
-}
-
-template  <class T1,class T2,class T3,class T4,class T5, class Var> 
-inline
-detail::group1< detail::group6<T1,T2,T3,T4,T5, Var&> >
-  group(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5, Var& var)
-{ 
-   return detail::group1< detail::group6<T1,T2,T3,T4,T5, Var&> >
-                   ( detail::group6<T1,T2,T3,T4,T5, Var&> 
-                        (a1,a2,a3,a4,a5, var) 
-                  );
-}
-
-template  <class T1,class T2,class T3,class T4,class T5,class T6, class Var> 
-inline
-detail::group1< detail::group7<T1,T2,T3,T4,T5,T6, Var&> >
-  group(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5,T6 a6, Var& var)
-{ 
-   return detail::group1< detail::group7<T1,T2,T3,T4,T5,T6, Var&> >
-                   ( detail::group7<T1,T2,T3,T4,T5,T6, Var&> 
-                        (a1,a2,a3,a4,a5,a6, var) 
-                  );
-}
-
-template  <class T1,class T2,class T3,class T4,class T5,class T6,class T7, class Var> 
-inline
-detail::group1< detail::group8<T1,T2,T3,T4,T5,T6,T7, Var&> >
-  group(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5,T6 a6,T7 a7, Var& var)
-{ 
-   return detail::group1< detail::group8<T1,T2,T3,T4,T5,T6,T7, Var&> >
-                   ( detail::group8<T1,T2,T3,T4,T5,T6,T7, Var&> 
-                        (a1,a2,a3,a4,a5,a6,a7, var) 
-                  );
-}
-
-template  <class T1,class T2,class T3,class T4,class T5,class T6,class T7,class T8, class Var> 
-inline
-detail::group1< detail::group9<T1,T2,T3,T4,T5,T6,T7,T8, Var&> >
-  group(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5,T6 a6,T7 a7,T8 a8, Var& var)
-{ 
-   return detail::group1< detail::group9<T1,T2,T3,T4,T5,T6,T7,T8, Var&> >
-                   ( detail::group9<T1,T2,T3,T4,T5,T6,T7,T8, Var&> 
-                        (a1,a2,a3,a4,a5,a6,a7,a8, var) 
-                  );
-}
-
-template  <class T1,class T2,class T3,class T4,class T5,class T6,class T7,class T8,class T9, class Var> 
-inline
-detail::group1< detail::group10<T1,T2,T3,T4,T5,T6,T7,T8,T9, Var&> >
-  group(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5,T6 a6,T7 a7,T8 a8,T9 a9, Var& var)
-{ 
-   return detail::group1< detail::group10<T1,T2,T3,T4,T5,T6,T7,T8,T9, Var&> >
-                   ( detail::group10<T1,T2,T3,T4,T5,T6,T7,T8,T9, Var&> 
-                        (a1,a2,a3,a4,a5,a6,a7,a8,a9, var) 
-                  );
-}
-
-
-#endif  //end- #ifndef BOOST_NO_OVERLOAD_FOR_NON_CONST
-
-
-} // namespace io
-
-} // namespace boost
-
-
-#endif   // BOOST_FORMAT_GROUP_HPP

src/boost/format/internals.hpp

@@ -1,167 +0,0 @@
-// -*- C++ -*-
-//  Boost general library 'format'   ---------------------------
-//  See http://www.boost.org for updates, documentation, and revision history.
-
-//  (C) Samuel Krempp 2001
-//                  krempp@crans.ens-cachan.fr
-//  Permission to copy, use, modify, sell and
-//  distribute this software is granted provided this copyright notice appears
-//  in all copies. This software is provided "as is" without express or implied
-//  warranty, and with no claim as to its suitability for any purpose.
-
-// ideas taken from Rüdiger Loos's format class
-// and Karl Nelson's ofstream
-
-// ----------------------------------------------------------------------------
-// internals.hpp :  internal structs. included by format.hpp
-//                              stream_format_state, and format_item
-// ----------------------------------------------------------------------------
-
-
-#ifndef BOOST_FORMAT_INTERNALS_HPP
-#define BOOST_FORMAT_INTERNALS_HPP
-
-
-#include <string>
-#include <sstream>
-
-namespace boost {
-namespace io {
-namespace detail {
-
-
-// --------------
-// set of params that define the format state of a stream
-
-struct stream_format_state 
-{
-  typedef std::ios   basic_ios;
-
-  std::streamsize width_;
-  std::streamsize precision_;
-  char fill_; 
-  std::ios::fmtflags flags_;
-
-  stream_format_state()       : width_(-1), precision_(-1), fill_(0), flags_(std::ios::dec)  {}
-  stream_format_state(basic_ios& os)                  {set_by_stream(os); }
-
-  void apply_on(basic_ios & os) const;                //- applies format_state to the stream
-  template<class T> void apply_manip(T manipulator)   //- modifies state by applying manipulator.
-       { apply_manip_body<T>( *this, manipulator) ; }
-  void reset();                                       //- sets to default state.
-  void set_by_stream(const basic_ios& os);            //- sets to os's state.
-};  
-
-
-
-// --------------
-// format_item : stores all parameters that can be defined by directives in the format-string
-
-struct format_item 
-{     
-  enum pad_values { zeropad = 1, spacepad =2, centered=4, tabulation = 8 };
-
-  enum arg_values { argN_no_posit   = -1, // non-positional directive. argN will be set later.
-                    argN_tabulation = -2, // tabulation directive. (no argument read) 
-                    argN_ignored    = -3  // ignored directive. (no argument read)
-  };
-  typedef BOOST_IO_STD ios              basic_ios;
-  typedef detail::stream_format_state         stream_format_state;
-  typedef std::string           string_t;
-  typedef BOOST_IO_STD ostringstream    internal_stream_t;
-
-
-  int         argN_;           //- argument number (starts at 0,  eg : %1 => argN=0)
-                               //  negative values are used for items that don't process
-                               //  an argument
-  string_t    res_;            //- result of the formatting of this item
-  string_t    appendix_;       //- piece of string between this item and the next
-
-  stream_format_state ref_state_;// set by parsing the format_string, is only affected by modify_item
-  stream_format_state state_;  // always same as ref_state, _unless_ modified by manipulators 'group(..)'
-
-  // non-stream format-state parameters
-  signed int truncate_;        //- is >=0 for directives like %.5s (take 5 chars from the string)
-  unsigned int pad_scheme_;    //- several possible padding schemes can mix. see pad_values
-
-  format_item() : argN_(argN_no_posit), truncate_(-1), pad_scheme_(0)  {}
-
-  void compute_states();      // sets states  according to truncate and pad_scheme.
-}; 
-
-
-
-// -----------------------------------------------------------
-// Definitions
-// -----------------------------------------------------------
-
-// --- stream_format_state:: -------------------------------------------
-inline
-void stream_format_state::apply_on(basic_ios & os) const
-  // set the state of this stream according to our params
-{
-      if(width_ != -1)
-        os.width(width_);
-      if(precision_ != -1)
-        os.precision(precision_);
-      if(fill_ != 0)
-        os.fill(fill_);
-      os.flags(flags_);
-}
-
-inline
-void stream_format_state::set_by_stream(const basic_ios& os) 
-  // set our params according to the state of this stream
-{
-      flags_ = os.flags();
-      width_ = os.width();
-      precision_ = os.precision();
-      fill_ = os.fill();
-}
-
-template<class T>  inline
-void apply_manip_body( stream_format_state& self,
-                       T manipulator) 
-  // modify our params according to the manipulator
-{
-      BOOST_IO_STD stringstream  ss;
-      self.apply_on( ss );
-      ss << manipulator;
-      self.set_by_stream( ss );
-}
-
-inline
-void stream_format_state::reset() 
-  // set our params to standard's default state
-{
-      width_=-1; precision_=-1; fill_=0; 
-      flags_ = std::ios::dec; 
-}
-
-
-// --- format_items:: -------------------------------------------
-inline
-void format_item::compute_states() 
-  // reflect pad_scheme_   on  state_ and ref_state_ 
-  //   because some pad_schemes has complex consequences on several state params.
-{
-  if(pad_scheme_ & zeropad) 
-  {
-    if(ref_state_.flags_ & std::ios::left) 
-    {
-      pad_scheme_ = pad_scheme_ & (~zeropad); // ignore zeropad in left alignment
-    }
-    else 
-    { 
-      ref_state_.fill_='0'; 
-      ref_state_.flags_ |= std::ios::internal;
-    }
-  }
-  state_ = ref_state_;
-}
-
-
-} } } // namespaces boost :: io :: detail
-
-
-#endif // BOOST_FORMAT_INTERNALS_HPP

src/boost/format/internals_fwd.hpp

@@ -1,65 +0,0 @@
-// -*- C++ -*-
-//  Boost general library 'format'   ---------------------------
-//  See http://www.boost.org for updates, documentation, and revision history.
-
-//  (C) Samuel Krempp 2001
-//                  krempp@crans.ens-cachan.fr
-//  Permission to copy, use, modify, sell and
-//  distribute this software is granted provided this copyright notice appears
-//  in all copies. This software is provided "as is" without express or implied
-//  warranty, and with no claim as to its suitability for any purpose.
-
-// ideas taken from Rüdiger Loos's format class
-// and Karl Nelson's ofstream (also took its parsing code as basis for printf parsing)
-
-// ------------------------------------------------------------------------------
-// internals_fwd.hpp :  forward declarations, for internal headers
-// ------------------------------------------------------------------------------
-
-#ifndef BOOST_FORMAT_INTERNAL_FWD_HPP
-#define BOOST_FORMAT_INTERNAL_FWD_HPP
-
-#include "boost/format/format_fwd.hpp"
-
-
-namespace boost {
-namespace io {
-
-namespace detail {
-  struct stream_format_state;
-  struct format_item;
-}
-
-
-namespace detail {
-
-  // these functions were intended as methods, 
-  // but MSVC have problems with template member functions :
-
-  // defined in format_implementation.hpp :
-     template<class T> 
-     basic_format&  modify_item_body( basic_format& self, 
-                                          int itemN, const T& manipulator);
-
-     template<class T> 
-     basic_format&  bind_arg_body( basic_format& self,
-                                           int argN, const T& val);
-
-    template<class T> 
-    void apply_manip_body( stream_format_state& self,
-                           T manipulator);
-
-  // argument feeding (defined in feed_args.hpp ) :
-     template<class T> 
-     void distribute(basic_format& self, T x);
-
-     template<class T> 
-     basic_format& feed(basic_format& self, T x);
- 
-} // namespace detail
-
-} // namespace io
-} // namespace boost
-
-
-#endif //  BOOST_FORMAT_INTERNAL_FWD_HPP

src/boost/format/local.mk

@@ -1,7 +0,0 @@
-libraries += libformat
-
-libformat_NAME = libnixformat
-
-libformat_DIR := $(d)
-
-libformat_SOURCES := $(wildcard $(d)/*.cc)

src/boost/format/macros_default.hpp

@@ -1,48 +0,0 @@
-// -*- C++ -*-
-//  Boost general library 'format'   ---------------------------
-//  See http://www.boost.org for updates, documentation, and revision history.
-
-//  (C) Samuel Krempp 2001
-//                  krempp@crans.ens-cachan.fr
-//  Permission to copy, use, modify, sell and
-//  distribute this software is granted provided this copyright notice appears
-//  in all copies. This software is provided "as is" without express or implied
-//  warranty, and with no claim as to its suitability for any purpose.
-
-// ideas taken from Rüdiger Loos's format class
-// and Karl Nelson's ofstream (also took its parsing code as basis for printf parsing)
-
-// ------------------------------------------------------------------------------
-// macros_default.hpp : configuration for the format library
-//                       provides default values for the stl workaround macros
-// ------------------------------------------------------------------------------
-
-#ifndef BOOST_FORMAT_MACROS_DEFAULT_HPP
-#define BOOST_FORMAT_MACROS_DEFAULT_HPP
-
-// *** This should go to "boost/config/suffix.hpp".
-
-#ifndef BOOST_IO_STD
-#  define BOOST_IO_STD std::
-#endif
-
-// **** Workaround for io streams, stlport and msvc.
-#ifdef BOOST_IO_NEEDS_USING_DECLARATION
-namespace boost {
-  using std::char_traits;
-  using std::basic_ostream;
-  using std::basic_ostringstream;
-  namespace io {
-    using std::basic_ostream;
-    namespace detail {
-      using std::basic_ios;
-      using std::basic_ostream;
-      using std::basic_ostringstream;
-    }
-  }
-}
-#endif
-
-// ------------------------------------------------------------------------------
-
-#endif // BOOST_FORMAT_MACROS_DEFAULT_HPP

src/boost/format/parsing.cc

@@ -1,454 +0,0 @@
-// -*- C++ -*-
-//  Boost general library 'format'   ---------------------------
-//  See http://www.boost.org for updates, documentation, and revision history.
-
-//  (C) Samuel Krempp 2001
-//                  krempp@crans.ens-cachan.fr
-//  Permission to copy, use, modify, sell and
-//  distribute this software is granted provided this copyright notice appears
-//  in all copies. This software is provided "as is" without express or implied
-//  warranty, and with no claim as to its suitability for any purpose.
-
-// ideas taken from Rudiger Loos's format class
-// and Karl Nelson's ofstream (also took its parsing code as basis for printf parsing)
-
-// ------------------------------------------------------------------------------
-// parsing.hpp :  implementation of the parsing member functions
-//                      ( parse, parse_printf_directive)
-// ------------------------------------------------------------------------------
-
-
-#ifndef BOOST_FORMAT_PARSING_HPP
-#define BOOST_FORMAT_PARSING_HPP
-
-
-#include <boost/format.hpp>
-#include <boost/throw_exception.hpp>
-#include <boost/assert.hpp>
-
-
-namespace boost {
-namespace io {
-namespace detail {
-
-  template<class Stream> inline
-  bool wrap_isdigit(char c, Stream &os) 
-  {
-#ifndef BOOST_NO_LOCALE_ISIDIGIT
-    return std::isdigit(c, os.rdbuf()->getloc() );
-# else
-    using namespace std;
-    return isdigit(c); 
-#endif 
-  } //end- wrap_isdigit(..)
-
-  template<class Res> inline
-  Res str2int(const std::string& s, 
-              std::string::size_type start, 
-              BOOST_IO_STD ios &os,
-              const Res = Res(0)  ) 
-    // Input : char string, with starting index
-    //         a basic_ios& merely to call its widen/narrow member function in the desired locale.
-    // Effects : reads s[start:] and converts digits into an integral n, of type Res
-    // Returns : n
-  {
-    Res n = 0;
-    while(start<s.size() && wrap_isdigit(s[start], os) ) {
-      char cur_ch = s[start];
-      BOOST_ASSERT(cur_ch != 0 ); // since we called isdigit, this should not happen.
-      n *= 10;
-      n += cur_ch - '0'; // 22.2.1.1.2 of the C++ standard
-      ++start;
-    }
-    return n;
-  }
-
-  void skip_asterisk(const std::string & buf, 
-                     std::string::size_type * pos_p,
-                     BOOST_IO_STD ios &os)
-    // skip printf's "asterisk-fields" directives in the format-string buf
-    // Input : char string, with starting index *pos_p
-    //         a basic_ios& merely to call its widen/narrow member function in the desired locale.
-    // Effects : advance *pos_p by skipping printf's asterisk fields.
-    // Returns : nothing
-  {
-    using namespace std;
-    BOOST_ASSERT( pos_p != 0);
-    if(*pos_p >= buf.size() ) return;
-    if(buf[ *pos_p]=='*') {
-      ++ (*pos_p);
-      while (*pos_p < buf.size() && wrap_isdigit(buf[*pos_p],os)) ++(*pos_p);
-      if(buf[*pos_p]=='$') ++(*pos_p);
-    }
-  }
-
-
-  inline void maybe_throw_exception( unsigned char exceptions)
-    // auxiliary func called by parse_printf_directive
-    // for centralising error handling
-    // it either throws if user sets the corresponding flag, or does nothing.
-  {
-    if(exceptions & io::bad_format_string_bit)
-          boost::throw_exception(io::bad_format_string());
-  }
-    
-
-
-  bool parse_printf_directive(const std::string & buf,
-                              std::string::size_type * pos_p,
-                              detail::format_item * fpar,
-                              BOOST_IO_STD ios &os,
-                              unsigned char exceptions)
-    // Input   : a 'printf-directive' in the format-string, starting at buf[ *pos_p ]
-    //           a basic_ios& merely to call its widen/narrow member function in the desired locale.
-    //           a bitset'excpetions' telling whether to throw exceptions on errors.
-    // Returns : true if parse somehow succeeded (possibly ignoring errors if exceptions disabled) 
-    //           false if it failed so bad that the directive should be printed verbatim
-    // Effects : - *pos_p is incremented so that buf[*pos_p] is the first char after the directive
-    //           - *fpar is set with the parameters read in the directive
-  {
-    typedef format_item  format_item_t;
-    BOOST_ASSERT( pos_p != 0);
-    std::string::size_type       &i1 = *pos_p,      
-                                                        i0; 
-    fpar->argN_ = format_item_t::argN_no_posit;  // if no positional-directive
-
-    bool in_brackets=false;
-    if(buf[i1]=='|')
-      {
-        in_brackets=true;
-        if( ++i1 >= buf.size() ) {
-          maybe_throw_exception(exceptions);
-          return false;
-        }
-      }
-
-    // the flag '0' would be picked as a digit for argument order, but here it's a flag :
-    if(buf[i1]=='0') 
-      goto parse_flags;
-
-    // handle argument order (%2$d)  or possibly width specification: %2d
-    i0 = i1;  // save position before digits
-    while (i1 < buf.size() && wrap_isdigit(buf[i1], os))
-      ++i1;
-    if (i1!=i0) 
-      {
-        if( i1 >= buf.size() ) {
-          maybe_throw_exception(exceptions);
-          return false;
-        }
-        int n=str2int(buf,i0, os, int(0) );
-        
-        // %N% case : this is already the end of the directive
-        if( buf[i1] == '%' ) 
-          {
-            fpar->argN_ = n-1;
-            ++i1;
-            if( in_brackets) 
-              maybe_throw_exception(exceptions); 
-              // but don't return.  maybe "%" was used in lieu of '$', so we go on.
-            else return true;
-          }
-
-        if ( buf[i1]=='$' ) 
-          {
-            fpar->argN_ = n-1;
-            ++i1;
-          } 
-        else  
-          {
-            // non-positionnal directive
-            fpar->ref_state_.width_ = n;
-            fpar->argN_  = format_item_t::argN_no_posit;
-            goto parse_precision;
-          }
-      }
-    
-  parse_flags: 
-    // handle flags
-    while ( i1 <buf.size()) // as long as char is one of + - = # 0 l h   or ' '
-      {  
-        // misc switches
-        switch (buf[i1]) 
-          {
-          case '\'' : break; // no effect yet. (painful to implement)
-          case 'l':
-          case 'h':  // short/long modifier : for printf-comaptibility (no action needed)
-             break;
-          case '-':
-            fpar->ref_state_.flags_ |= std::ios::left;
-            break;
-          case '=':
-            fpar->pad_scheme_ |= format_item_t::centered;
-            break;
-          case ' ':
-            fpar->pad_scheme_ |= format_item_t::spacepad;
-            break;
-          case '+':
-            fpar->ref_state_.flags_ |= std::ios::showpos;
-            break;
-          case '0':
-            fpar->pad_scheme_ |= format_item_t::zeropad; 
-            // need to know alignment before really setting flags,
-            // so just add 'zeropad' flag for now, it will be processed later.
-            break;
-          case '#':
-            fpar->ref_state_.flags_ |= std::ios::showpoint | std::ios::showbase;
-            break;
-          default:
-            goto parse_width;
-          }
-        ++i1;
-      } // loop on flag.
-    if( i1>=buf.size()) {
-      maybe_throw_exception(exceptions);
-      return true; 
-    }
-
-  parse_width:
-    // handle width spec
-    skip_asterisk(buf, &i1, os); // skips 'asterisk fields' :  *, or *N$
-    i0 = i1;  // save position before digits
-    while (i1<buf.size() && wrap_isdigit(buf[i1], os))
-      i1++;
-    
-    if (i1!=i0) 
-      { fpar->ref_state_.width_ = str2int( buf,i0, os, std::streamsize(0) ); }
-
-  parse_precision:
-    if( i1>=buf.size()) { 
-      maybe_throw_exception(exceptions);
-      return true;
-    }
-    // handle precision spec
-    if (buf[i1]=='.')  
-      {
-        ++i1;
-        skip_asterisk(buf, &i1, os);
-        i0 = i1;  // save position before digits
-        while (i1<buf.size() && wrap_isdigit(buf[i1], os))
-          ++i1;
-
-        if(i1==i0)
-          fpar->ref_state_.precision_ = 0;
-        else 
-          fpar->ref_state_.precision_ = str2int(buf,i0, os, std::streamsize(0) );
-      }
-    
-    // handle  formatting-type flags :
-    while( i1<buf.size() && 
-           ( buf[i1]=='l' || buf[i1]=='L' || buf[i1]=='h') )
-      ++i1;
-    if( i1>=buf.size()) {
-      maybe_throw_exception(exceptions);
-      return true;
-    }
-    
-    if( in_brackets && buf[i1]=='|' ) 
-      {
-        ++i1;
-        return true;
-      }
-    switch (buf[i1])  
-      {
-      case 'X':
-        fpar->ref_state_.flags_ |= std::ios::uppercase;
-      case 'p': // pointer => set hex.
-      case 'x':
-        fpar->ref_state_.flags_ &= ~std::ios::basefield;
-        fpar->ref_state_.flags_ |= std::ios::hex;
-        break;
-      
-      case 'o':
-        fpar->ref_state_.flags_ &= ~std::ios::basefield;
-        fpar->ref_state_.flags_ |=  std::ios::oct;
-        break;
-
-      case 'E':
-        fpar->ref_state_.flags_ |=  std::ios::uppercase;
-      case 'e':
-        fpar->ref_state_.flags_ &= ~std::ios::floatfield;
-        fpar->ref_state_.flags_ |=  std::ios::scientific;
-
-        fpar->ref_state_.flags_ &= ~std::ios::basefield;
-        fpar->ref_state_.flags_ |=  std::ios::dec;
-        break;
-      
-      case 'f':
-        fpar->ref_state_.flags_ &= ~std::ios::floatfield;
-        fpar->ref_state_.flags_ |=  std::ios::fixed;
-      case 'u':
-      case 'd':
-      case 'i':
-        fpar->ref_state_.flags_ &= ~std::ios::basefield;
-        fpar->ref_state_.flags_ |=  std::ios::dec;
-        break;
-
-      case 'T':
-        ++i1;
-        if( i1 >= buf.size())
-          maybe_throw_exception(exceptions);
-        else
-          fpar->ref_state_.fill_ = buf[i1];
-        fpar->pad_scheme_ |= format_item_t::tabulation;
-        fpar->argN_ = format_item_t::argN_tabulation; 
-        break;
-      case 't': 
-        fpar->ref_state_.fill_ = ' ';
-        fpar->pad_scheme_ |= format_item_t::tabulation;
-        fpar->argN_ = format_item_t::argN_tabulation; 
-        break;
-
-      case 'G':
-        fpar->ref_state_.flags_ |= std::ios::uppercase;
-        break;
-      case 'g': // 'g' conversion is default for floats.
-        fpar->ref_state_.flags_ &= ~std::ios::basefield;
-        fpar->ref_state_.flags_ |=  std::ios::dec;
-
-        // CLEAR all floatield flags, so stream will CHOOSE
-        fpar->ref_state_.flags_ &= ~std::ios::floatfield; 
-        break;
-
-      case 'C':
-      case 'c': 
-        fpar->truncate_ = 1;
-        break;
-      case 'S':
-      case 's': 
-        fpar->truncate_ = fpar->ref_state_.precision_;
-        fpar->ref_state_.precision_ = -1;
-        break;
-      case 'n' :  
-        fpar->argN_ = format_item_t::argN_ignored;
-        break;
-      default: 
-        maybe_throw_exception(exceptions);
-      }
-    ++i1;
-
-    if( in_brackets )
-      {
-        if( i1<buf.size() && buf[i1]=='|' ) 
-          {
-            ++i1;
-            return true;
-          }
-        else  maybe_throw_exception(exceptions);
-      }
-    return true;
-  }
-
-} // detail namespace
-} // io namespace
-
-
-// -----------------------------------------------
-//  format :: parse(..)
-
-void basic_format::parse(const string_t & buf) 
-  // parse the format-string
-{
-    using namespace std;
-    const char arg_mark = '%';
-    bool ordered_args=true; 
-    int max_argN=-1;
-    string_t::size_type i1=0;
-    int num_items=0;
-    
-    // A: find upper_bound on num_items and allocates arrays
-    i1=0; 
-    while( (i1=buf.find(arg_mark,i1)) != string::npos ) 
-    {
-      if( i1+1 >= buf.size() ) {
-        if(exceptions() & io::bad_format_string_bit)
-          boost::throw_exception(io::bad_format_string()); // must not end in "bla bla %"
-        else break; // stop there, ignore last '%'
-      }
-      if(buf[i1+1] == buf[i1] ) { i1+=2; continue; } // escaped "%%" / "##"
-      ++i1;
-      
-      // in case of %N% directives, dont count it double (wastes allocations..) :
-      while(i1 < buf.size() && io::detail::wrap_isdigit(buf[i1],oss_)) ++i1;
-      if( i1 < buf.size() && buf[i1] == arg_mark ) ++ i1;
-
-      ++num_items;
-    }
-    items_.assign( num_items, format_item_t() );
-    
-    // B: Now the real parsing of the format string :
-    num_items=0;
-    i1 = 0;
-    string_t::size_type i0 = i1;
-    bool special_things=false;
-    int cur_it=0;
-    while( (i1=buf.find(arg_mark,i1)) != string::npos ) 
-    {
-      string_t & piece = (cur_it==0) ? prefix_ : items_[cur_it-1].appendix_;
-
-      if( buf[i1+1] == buf[i1] ) // escaped mark, '%%'
-      {
-        piece += buf.substr(i0, i1-i0) + buf[i1]; 
-        i1+=2; i0=i1;
-        continue; 
-      }
-      BOOST_ASSERT(  static_cast<unsigned int>(cur_it) < items_.size() || cur_it==0);
-
-      if(i1!=i0) piece += buf.substr(i0, i1-i0);
-      ++i1;
-      
-      bool parse_ok;
-      parse_ok = io::detail::parse_printf_directive(buf, &i1, &items_[cur_it], oss_, exceptions());
-      if( ! parse_ok ) continue; // the directive will be printed verbatim
-
-      i0=i1;
-      items_[cur_it].compute_states(); // process complex options, like zeropad, into stream params.
-
-      int argN=items_[cur_it].argN_;
-      if(argN == format_item_t::argN_ignored)
-        continue;
-      if(argN ==format_item_t::argN_no_posit)
-        ordered_args=false;
-      else if(argN == format_item_t::argN_tabulation) special_things=true;
-      else if(argN > max_argN) max_argN = argN;
-      ++num_items;
-      ++cur_it;
-    } // loop on %'s
-    BOOST_ASSERT(cur_it == num_items);
-    
-    // store the final piece of string
-    string_t & piece = (cur_it==0) ? prefix_ : items_[cur_it-1].appendix_;
-    piece += buf.substr(i0);
-    
-    if( !ordered_args) 
-    {
-      if(max_argN >= 0 )  // dont mix positional with non-positionnal directives
-        {
-          if(exceptions() & io::bad_format_string_bit)
-            boost::throw_exception(io::bad_format_string());
-          // else do nothing. => positionnal arguments are processed as non-positionnal
-        }
-      // set things like it would have been with positional directives :
-      int non_ordered_items = 0;
-      for(int i=0; i< num_items; ++i)
-        if(items_[i].argN_ == format_item_t::argN_no_posit) 
-          {
-            items_[i].argN_ = non_ordered_items;
-            ++non_ordered_items;
-          }
-      max_argN = non_ordered_items-1;
-    }
-    
-    // C: set some member data :
-    items_.resize(num_items);
-
-    if(special_things) style_ |= special_needs;
-    num_args_ = max_argN + 1;
-    if(ordered_args) style_ |=  ordered;
-    else style_ &= ~ordered;
-}
-
-} // namespace boost
-
-
-#endif //  BOOST_FORMAT_PARSING_HPP

src/boost/throw_exception.hpp

@@ -1,47 +0,0 @@
-#ifndef BOOST_THROW_EXCEPTION_HPP_INCLUDED
-#define BOOST_THROW_EXCEPTION_HPP_INCLUDED
-
-// MS compatible compilers support #pragma once
-
-#if defined(_MSC_VER) && (_MSC_VER >= 1020)
-# pragma once
-#endif
-
-//
-//  boost/throw_exception.hpp
-//
-//  Copyright (c) 2002 Peter Dimov and Multi Media Ltd.
-//
-//  Permission to copy, use, modify, sell and distribute this software
-//  is granted provided this copyright notice appears in all copies.
-//  This software is provided "as is" without express or implied
-//  warranty, and with no claim as to its suitability for any purpose.
-//
-//  http://www.boost.org/libs/utility/throw_exception.html
-//
-
-//#include <boost/config.hpp>
-
-#ifdef BOOST_NO_EXCEPTIONS
-# include <exception>
-#endif
-
-namespace boost
-{
-
-#ifdef BOOST_NO_EXCEPTIONS
-
-void throw_exception(std::exception const & e); // user defined
-
-#else
-
-template<class E> void throw_exception(E const & e)
-{
-    throw e;
-}
-
-#endif
-
-} // namespace boost
-
-#endif // #ifndef BOOST_THROW_EXCEPTION_HPP_INCLUDED

src/build-remote/build-remote.cc

@@ -17,6 +17,7 @@
 #include "store-api.hh"
 #include "derivations.hh"
 #include "local-store.hh"
+#include "legacy.hh"
 
 using namespace nix;
 using std::cin;
@@ -37,11 +38,15 @@ static AutoCloseFD openSlotLock(const Machine & m, unsigned long long slot)
     return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri), slot), true);
 }
 
-int main (int argc, char * * argv)
-{
-    return handleExceptions(argv[0], [&]() {
-        initNix();
+static bool allSupportedLocally(const std::set<std::string>& requiredFeatures) {
+    for (auto & feature : requiredFeatures)
+        if (!settings.systemFeatures.get().count(feature)) return false;
+    return true;
+}
 
+static int _main(int argc, char * * argv)
+{
+    {
         logger = makeJSONLogger(*logger);
 
         /* Ensure we don't get any SSH passphrase or host key popups. */
@@ -80,7 +85,7 @@ int main (int argc, char * * argv)
 
         if (machines.empty()) {
             std::cerr << "# decline-permanently\n";
-            return;
+            return 0;
         }
 
         string drvPath;
@@ -90,15 +95,18 @@ int main (int argc, char * * argv)
 
             try {
                 auto s = readString(source);
-                if (s != "try") return;
-            } catch (EndOfFile &) { return; }
+                if (s != "try") return 0;
+            } catch (EndOfFile &) { return 0; }
 
             auto amWilling = readInt(source);
             auto neededSystem = readString(source);
             source >> drvPath;
             auto requiredFeatures = readStrings<std::set<std::string>>(source);
 
-            auto canBuildLocally = amWilling && (neededSystem == settings.thisSystem);
+             auto canBuildLocally = amWilling
+                 &&  (  neededSystem == settings.thisSystem
+                     || settings.extraPlatforms.get().count(neededSystem) > 0)
+                 &&  allSupportedLocally(requiredFeatures);
 
             /* Error ignored here, will be caught later */
             mkdir(currentLoad.c_str(), 0777);
@@ -191,8 +199,10 @@ int main (int argc, char * * argv)
                     storeUri = bestMachine->storeUri;
 
                 } catch (std::exception & e) {
-                    printError("unable to open SSH connection to '%s': %s; trying other available machines...",
-                        bestMachine->storeUri, e.what());
+                    auto msg = chomp(drainFD(5, false));
+                    printError("cannot build on '%s': %s%s",
+                        bestMachine->storeUri, e.what(),
+                        (msg.empty() ? "" : ": " + msg));
                     bestMachine->enabled = false;
                     continue;
                 }
@@ -202,6 +212,8 @@ int main (int argc, char * * argv)
         }
 
 connected:
+        close(5);
+
         std::cerr << "# accept\n" << storeUri << "\n";
 
         auto inputs = readStrings<PathSet>(source);
@@ -244,9 +256,11 @@ connected:
         if (!missing.empty()) {
             Activity act(*logger, lvlTalkative, actUnknown, fmt("copying outputs from '%s'", storeUri));
             store->locksHeld.insert(missing.begin(), missing.end()); /* FIXME: ugly */
-            copyPaths(ref<Store>(sshStore), store, missing, NoRepair, NoCheckSigs, substitute);
+            copyPaths(ref<Store>(sshStore), store, missing, NoRepair, NoCheckSigs, NoSubstitute);
         }
 
-        return;
-    });
+        return 0;
+    }
 }
+
+static RegisterLegacyCommand s1("build-remote", _main);

src/build-remote/local.mk

@@ -1,9 +0,0 @@
-programs += build-remote
-
-build-remote_DIR := $(d)
-
-build-remote_INSTALL_DIR := $(libexecdir)/nix
-
-build-remote_LIBS = libmain libutil libformat libstore
-
-build-remote_SOURCES := $(d)/build-remote.cc

src/buildenv/buildenv.cc

@@ -1,187 +0,0 @@
-#include "shared.hh"
-#include <sys/stat.h>
-#include <sys/types.h>
-#include <fcntl.h>
-#include <algorithm>
-
-using namespace nix;
-
-typedef std::map<Path,int> Priorities;
-
-static bool isDirectory (const Path & path)
-{
-    struct stat st;
-    if (stat(path.c_str(), &st) == -1)
-        throw SysError(format("getting status of '%1%'") % path);
-    return S_ISDIR(st.st_mode);
-}
-
-static auto priorities = Priorities{};
-
-static auto symlinks = 0;
-
-/* For each activated package, create symlinks */
-static void createLinks(const Path & srcDir, const Path & dstDir, int priority)
-{
-    auto srcFiles = readDirectory(srcDir);
-    for (const auto & ent : srcFiles) {
-        if (ent.name[0] == '.')
-            /* not matched by glob */
-            continue;
-        const auto & srcFile = srcDir + "/" + ent.name;
-        auto dstFile = dstDir + "/" + ent.name;
-
-        /* The files below are special-cased to that they don't show up
-         * in user profiles, either because they are useless, or
-         * because they would cauase pointless collisions (e.g., each
-         * Python package brings its own
-         * `$out/lib/pythonX.Y/site-packages/easy-install.pth'.)
-         */
-        if (hasSuffix(srcFile, "/propagated-build-inputs") ||
-            hasSuffix(srcFile, "/nix-support") ||
-            hasSuffix(srcFile, "/perllocal.pod") ||
-            hasSuffix(srcFile, "/info/dir") ||
-            hasSuffix(srcFile, "/log")) {
-            continue;
-        } else if (isDirectory(srcFile)) {
-            struct stat dstSt;
-            auto res = lstat(dstFile.c_str(), &dstSt);
-            if (res == 0) {
-                if (S_ISDIR(dstSt.st_mode)) {
-                    createLinks(srcFile, dstFile, priority);
-                    continue;
-                } else if (S_ISLNK(dstSt.st_mode)) {
-                    auto target = readLink(dstFile);
-                    if (!isDirectory(target))
-                        throw Error(format("collision between '%1%' and non-directory '%2%'")
-                            % srcFile % target);
-                    if (unlink(dstFile.c_str()) == -1)
-                        throw SysError(format("unlinking '%1%'") % dstFile);
-                    if (mkdir(dstFile.c_str(), 0755) == -1)
-                        throw SysError(format("creating directory '%1%'"));
-                    createLinks(target, dstFile, priorities[dstFile]);
-                    createLinks(srcFile, dstFile, priority);
-                    continue;
-                }
-            } else if (errno != ENOENT)
-                throw SysError(format("getting status of '%1%'") % dstFile);
-        } else {
-            struct stat dstSt;
-            auto res = lstat(dstFile.c_str(), &dstSt);
-            if (res == 0) {
-                if (S_ISLNK(dstSt.st_mode)) {
-                    auto target = readLink(dstFile);
-                    auto prevPriority = priorities[dstFile];
-                    if (prevPriority == priority)
-                        throw Error(format(
-                                "Packages '%1%' and '%2%' have the same priority '%3%'"
-                                "use 'nix-env --set-flag priority NUMBER INSTALLED_PKGNAME' "
-                                "to change the priority of one of the conflicting packages"
-                                " ('0' being the highest priority)"
-                                ) % srcFile % target % priority);
-                    if (prevPriority < priority)
-                        continue;
-                    if (unlink(dstFile.c_str()) == -1)
-                        throw SysError(format("unlinking '%1%'") % dstFile);
-                }
-            } else if (errno != ENOENT)
-                throw SysError(format("getting status of '%1%'") % dstFile);
-        }
-        createSymlink(srcFile, dstFile);
-        priorities[dstFile] = priority;
-        symlinks++;
-    }
-}
-
-typedef std::set<Path> FileProp;
-
-static auto done = FileProp{};
-static auto postponed = FileProp{};
-
-static auto out = string{};
-
-static void addPkg(const Path & pkgDir, int priority)
-{
-    if (done.find(pkgDir) != done.end())
-        return;
-    done.insert(pkgDir);
-    createLinks(pkgDir, out, priority);
-    auto propagatedFN = pkgDir + "/nix-support/propagated-user-env-packages";
-    auto propagated = string{};
-    {
-        AutoCloseFD fd = open(propagatedFN.c_str(), O_RDONLY | O_CLOEXEC);
-        if (!fd) {
-            if (errno == ENOENT)
-                return;
-            throw SysError(format("opening '%1%'") % propagatedFN);
-        }
-        propagated = readFile(fd.get());
-    }
-    for (const auto & p : tokenizeString<std::vector<string>>(propagated, " \n"))
-        if (done.find(p) == done.end())
-            postponed.insert(p);
-}
-
-struct Package {
-    Path path;
-    bool active;
-    int priority;
-    Package(Path path, bool active, int priority) : path{std::move(path)}, active{active}, priority{priority} {}
-};
-
-typedef std::vector<Package> Packages;
-
-int main(int argc, char ** argv)
-{
-    return handleExceptions(argv[0], [&]() {
-        initNix();
-        out = getEnv("out");
-        if (mkdir(out.c_str(), 0755) == -1)
-            throw SysError(format("creating %1%") % out);
-
-        /* Convert the stuff we get from the environment back into a coherent
-         * data type.
-         */
-        auto pkgs = Packages{};
-        auto derivations = tokenizeString<Strings>(getEnv("derivations"));
-        while (!derivations.empty()) {
-            /* !!! We're trusting the caller to structure derivations env var correctly */
-            auto active = derivations.front(); derivations.pop_front();
-            auto priority = stoi(derivations.front()); derivations.pop_front();
-            auto outputs = stoi(derivations.front()); derivations.pop_front();
-            for (auto n = 0; n < outputs; n++) {
-                auto path = derivations.front(); derivations.pop_front();
-                pkgs.emplace_back(path, active != "false", priority);
-            }
-        }
-
-        /* Symlink to the packages that have been installed explicitly by the
-         * user. Process in priority order to reduce unnecessary
-         * symlink/unlink steps.
-         */
-        std::sort(pkgs.begin(), pkgs.end(), [](const Package & a, const Package & b) {
-            return a.priority < b.priority || (a.priority == b.priority && a.path < b.path);
-        });
-        for (const auto & pkg : pkgs)
-            if (pkg.active)
-                addPkg(pkg.path, pkg.priority);
-
-        /* Symlink to the packages that have been "propagated" by packages
-         * installed by the user (i.e., package X declares that it wants Y
-         * installed as well). We do these later because they have a lower
-         * priority in case of collisions.
-         */
-        auto priorityCounter = 1000;
-        while (!postponed.empty()) {
-            auto pkgDirs = postponed;
-            postponed = FileProp{};
-            for (const auto & pkgDir : pkgDirs)
-                addPkg(pkgDir, priorityCounter++);
-        }
-
-        std::cerr << "created " << symlinks << " symlinks in user environment\n";
-
-        createSymlink(getEnv("manifest"), out + "/manifest.nix");
-    });
-}
-

src/buildenv/local.mk

@@ -1,9 +0,0 @@
-programs += buildenv
-
-buildenv_DIR := $(d)
-
-buildenv_INSTALL_DIR := $(libexecdir)/nix
-
-buildenv_LIBS = libmain libstore libutil libformat
-
-buildenv_SOURCES := $(d)/buildenv.cc

src/cpptoml/LICENSE

@@ -0,0 +1,18 @@
+Copyright (c) 2014 Chase Geigle
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

src/libexpr/attr-set.cc

@@ -1,5 +1,5 @@
 #include "attr-set.hh"
-#include "eval.hh"
+#include "eval-inline.hh"
 
 #include <algorithm>
 
@@ -7,30 +7,18 @@
 namespace nix {
 
 
-/* Note: Various places expect the allocated memory to be zeroed. */
-static void * allocBytes(size_t n)
-{
-    void * p;
-#if HAVE_BOEHMGC
-    p = GC_malloc(n);
-#else
-    p = calloc(n, 1);
-#endif
-    if (!p) throw std::bad_alloc();
-    return p;
-}
-
-
 /* Allocate a new array of attributes for an attribute set with a specific
    capacity. The space is implicitly reserved after the Bindings
    structure. */
-Bindings * EvalState::allocBindings(Bindings::size_t capacity)
+Bindings * EvalState::allocBindings(size_t capacity)
 {
-    return new (allocBytes(sizeof(Bindings) + sizeof(Attr) * capacity)) Bindings(capacity);
+    if (capacity > std::numeric_limits<Bindings::size_t>::max())
+        throw Error("attribute set of size %d is too big", capacity);
+    return new (allocBytes(sizeof(Bindings) + sizeof(Attr) * capacity)) Bindings((Bindings::size_t) capacity);
 }
 
 
-void EvalState::mkAttrs(Value & v, unsigned int capacity)
+void EvalState::mkAttrs(Value & v, size_t capacity)
 {
     if (capacity == 0) {
         v = vEmptySet;

src/libexpr/eval-inline.hh

@@ -78,4 +78,18 @@ inline void EvalState::forceList(Value & v, const Pos & pos)
         throwTypeError("value is %1% while a list was expected, at %2%", v, pos);
 }
 
+/* Note: Various places expect the allocated memory to be zeroed. */
+inline void * allocBytes(size_t n)
+{
+    void * p;
+#if HAVE_BOEHMGC
+    p = GC_MALLOC(n);
+#else
+    p = calloc(n, 1);
+#endif
+    if (!p) throw std::bad_alloc();
+    return p;
+}
+
+
 }

src/libexpr/eval.cc

@@ -6,10 +6,16 @@
 #include "globals.hh"
 #include "eval-inline.hh"
 #include "download.hh"
+#include "json.hh"
 
 #include <algorithm>
 #include <cstring>
 #include <unistd.h>
+#include <sys/time.h>
+#include <sys/resource.h>
+#include <iostream>
+#include <fstream>
+
 #include <sys/time.h>
 #include <sys/resource.h>
 
@@ -18,15 +24,8 @@
 #include <gc/gc.h>
 #include <gc/gc_cpp.h>
 
-#define NEW new (UseGC)
-
-#else
-
-#define NEW new
-
 #endif
 
-
 namespace nix {
 
 
@@ -34,7 +33,7 @@ static char * dupString(const char * s)
 {
     char * t;
 #if HAVE_BOEHMGC
-    t = GC_strdup(s);
+    t = GC_STRDUP(s);
 #else
     t = strdup(s);
 #endif
@@ -43,20 +42,6 @@ static char * dupString(const char * s)
 }
 
 
-/* Note: Various places expect the allocated memory to be zeroed. */
-static void * allocBytes(size_t n)
-{
-    void * p;
-#if HAVE_BOEHMGC
-    p = GC_malloc(n);
-#else
-    p = calloc(n, 1);
-#endif
-    if (!p) throw std::bad_alloc();
-    return p;
-}
-
-
 static void printValue(std::ostream & str, std::set<const Value *> & active, const Value & v)
 {
     checkInterrupt();
@@ -145,6 +130,16 @@ std::ostream & operator << (std::ostream & str, const Value & v)
 }
 
 
+const Value *getPrimOp(const Value &v) {
+    const Value * primOp = &v;
+    while (primOp->type == tPrimOpApp) {
+        primOp = primOp->primOpApp.left;
+    }
+    assert(primOp->type == tPrimOp);
+    return primOp;
+}
+
+
 string showType(const Value & v)
 {
     switch (v.type) {
@@ -159,8 +154,10 @@ string showType(const Value & v)
         case tApp: return "a function application";
         case tLambda: return "a function";
         case tBlackhole: return "a black hole";
-        case tPrimOp: return "a built-in function";
-        case tPrimOpApp: return "a partially applied built-in function";
+        case tPrimOp:
+            return fmt("the built-in function '%s'", string(v.primOp->name));
+        case tPrimOpApp:
+            return fmt("the partially applied built-in function '%s'", string(getPrimOp(v)->primOp->name));
         case tExternal: return v.external->showType();
         case tFloat: return "a float";
     }
@@ -199,8 +196,15 @@ void initGC()
 
 #if HAVE_BOEHMGC
     /* Initialise the Boehm garbage collector. */
+
+    /* Don't look for interior pointers. This reduces the odds of
+       misdetection a bit. */
     GC_set_all_interior_pointers(0);
 
+    /* We don't have any roots in data segments, so don't scan from
+       there. */
+    GC_set_no_dls(1);
+
     GC_INIT();
 
     GC_set_oom_fn(oomHandler);
@@ -307,20 +311,32 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
 
     assert(gcInitialised);
 
+    static_assert(sizeof(Env) <= 16, "environment must be <= 16 bytes");
+
     /* Initialise the Nix expression search path. */
-    if (!settings.pureEval) {
+    if (!evalSettings.pureEval) {
         Strings paths = parseNixPath(getEnv("NIX_PATH", ""));
         for (auto & i : _searchPath) addToSearchPath(i);
         for (auto & i : paths) addToSearchPath(i);
     }
-    addToSearchPath("nix=" + canonPath(settings.nixDataDir + "/nix/corepkgs"));
+    addToSearchPath("nix=" + canonPath(settings.nixDataDir + "/nix/corepkgs", true));
 
-    if (settings.restrictEval || settings.pureEval) {
+    if (evalSettings.restrictEval || evalSettings.pureEval) {
         allowedPaths = PathSet();
+
         for (auto & i : searchPath) {
             auto r = resolveSearchPathElem(i);
             if (!r.first) continue;
-            allowedPaths->insert(r.second);
+
+            auto path = r.second;
+
+            if (store->isInStore(r.second)) {
+                PathSet closure;
+                store->computeFSClosure(store->toStorePath(r.second), closure);
+                for (auto & path : closure)
+                    allowedPaths->insert(path);
+            } else
+                allowedPaths->insert(r.second);
         }
     }
 
@@ -334,7 +350,6 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
 
 EvalState::~EvalState()
 {
-    fileEvalCache.clear();
 }
 
 
@@ -342,25 +357,37 @@ Path EvalState::checkSourcePath(const Path & path_)
 {
     if (!allowedPaths) return path_;
 
+    auto i = resolvedPaths.find(path_);
+    if (i != resolvedPaths.end())
+        return i->second;
+
     bool found = false;
 
+    /* First canonicalize the path without symlinks, so we make sure an
+     * attacker can't append ../../... to a path that would be in allowedPaths
+     * and thus leak symlink targets.
+     */
+    Path abspath = canonPath(path_);
+
     for (auto & i : *allowedPaths) {
-        if (isDirOrInDir(path_, i)) {
+        if (isDirOrInDir(abspath, i)) {
             found = true;
             break;
         }
     }
 
     if (!found)
-        throw RestrictedPathError("access to path '%1%' is forbidden in restricted mode", path_);
+        throw RestrictedPathError("access to path '%1%' is forbidden in restricted mode", abspath);
 
     /* Resolve symlinks. */
-    debug(format("checking access to '%s'") % path_);
-    Path path = canonPath(path_, true);
+    debug(format("checking access to '%s'") % abspath);
+    Path path = canonPath(abspath, true);
 
     for (auto & i : *allowedPaths) {
-        if (isDirOrInDir(path, i))
+        if (isDirOrInDir(path, i)) {
+            resolvedPaths[path_] = path;
             return path;
+        }
     }
 
     throw RestrictedPathError("access to path '%1%' is forbidden in restricted mode", path);
@@ -369,13 +396,13 @@ Path EvalState::checkSourcePath(const Path & path_)
 
 void EvalState::checkURI(const std::string & uri)
 {
-    if (!settings.restrictEval) return;
+    if (!evalSettings.restrictEval) return;
 
     /* 'uri' should be equal to a prefix, or in a subdirectory of a
        prefix. Thus, the prefix https://github.co does not permit
        access to https://github.com. Note: this allows 'http://' and
        'https://' as prefixes for any http/https URI. */
-    for (auto & prefix : settings.allowedUris.get())
+    for (auto & prefix : evalSettings.allowedUris.get())
         if (uri == prefix ||
             (uri.size() > prefix.size()
             && prefix.size() > 0
@@ -422,7 +449,7 @@ Value * EvalState::addConstant(const string & name, Value & v)
 
 
 Value * EvalState::addPrimOp(const string & name,
-    unsigned int arity, PrimOpFun primOp)
+    size_t arity, PrimOpFun primOp)
 {
     if (arity == 0) {
         Value v;
@@ -433,7 +460,7 @@ Value * EvalState::addPrimOp(const string & name,
     string name2 = string(name, 0, 2) == "__" ? string(name, 2) : name;
     Symbol sym = symbols.create(name2);
     v->type = tPrimOp;
-    v->primOp = NEW PrimOp(primOp, arity, sym);
+    v->primOp = new PrimOp(primOp, arity, sym);
     staticBaseEnv.vars[symbols.create(name)] = baseEnvDispl;
     baseEnv.values[baseEnvDispl++] = v;
     baseEnv.values[0]->attrs->push_back(Attr(sym, v));
@@ -528,7 +555,7 @@ Value & mkString(Value & v, const string & s, const PathSet & context)
 {
     mkString(v, s.c_str());
     if (!context.empty()) {
-        unsigned int n = 0;
+        size_t n = 0;
         v.string.context = (const char * *)
             allocBytes((context.size() + 1) * sizeof(char *));
         for (auto & i : context)
@@ -547,17 +574,17 @@ void mkPath(Value & v, const char * s)
 
 inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval)
 {
-    for (unsigned int l = var.level; l; --l, env = env->up) ;
+    for (size_t l = var.level; l; --l, env = env->up) ;
 
     if (!var.fromWith) return env->values[var.displ];
 
     while (1) {
-        if (!env->haveWithAttrs) {
+        if (env->type == Env::HasWithExpr) {
             if (noEval) return 0;
             Value * v = allocValue();
             evalAttrs(*env->up, (Expr *) env->values[0], *v);
             env->values[0] = v;
-            env->haveWithAttrs = true;
+            env->type = Env::HasWithAttrs;
         }
         Bindings::iterator j = env->values[0]->attrs->find(var.name);
         if (j != env->values[0]->attrs->end()) {
@@ -566,26 +593,37 @@ inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval)
         }
         if (!env->prevWith)
             throwUndefinedVarError("undefined variable '%1%' at %2%", var.name, var.pos);
-        for (unsigned int l = env->prevWith; l; --l, env = env->up) ;
+        for (size_t l = env->prevWith; l; --l, env = env->up) ;
     }
 }
 
 
+std::atomic<uint64_t> nrValuesFreed{0};
+
+void finalizeValue(void * obj, void * data)
+{
+    nrValuesFreed++;
+}
+
 Value * EvalState::allocValue()
 {
     nrValues++;
-    return (Value *) allocBytes(sizeof(Value));
+    auto v = (Value *) allocBytes(sizeof(Value));
+    //GC_register_finalizer_no_order(v, finalizeValue, nullptr, nullptr, nullptr);
+    return v;
 }
 
 
-Env & EvalState::allocEnv(unsigned int size)
+Env & EvalState::allocEnv(size_t size)
 {
-    assert(size <= std::numeric_limits<decltype(Env::size)>::max());
+    if (size > std::numeric_limits<decltype(Env::size)>::max())
+        throw Error("environment size %d is too big", size);
 
     nrEnvs++;
     nrValuesInEnvs += size;
     Env * env = (Env *) allocBytes(sizeof(Env) + size * sizeof(Value *));
-    env->size = size;
+    env->size = (decltype(Env::size)) size;
+    env->type = Env::Plain;
 
     /* We assume that env->values has been cleared by the allocator; maybeThunk() and lookupVar fromWith expect this. */
 
@@ -593,7 +631,7 @@ Env & EvalState::allocEnv(unsigned int size)
 }
 
 
-void EvalState::mkList(Value & v, unsigned int size)
+void EvalState::mkList(Value & v, size_t size)
 {
     clearValue(v);
     if (size == 1)
@@ -628,7 +666,7 @@ void EvalState::mkThunk_(Value & v, Expr * expr)
 
 void EvalState::mkPos(Value & v, Pos * pos)
 {
-    if (pos) {
+    if (pos && pos->file.set()) {
         mkAttrs(v, 3);
         mkString(*allocAttr(v, sFile), pos->file);
         mkInt(*allocAttr(v, sLine), pos->line);
@@ -705,7 +743,17 @@ void EvalState::evalFile(const Path & path_, Value & v)
     }
 
     printTalkative("evaluating file '%1%'", path2);
-    Expr * e = parseExprFromFile(checkSourcePath(path2));
+    Expr * e = nullptr;
+
+    auto j = fileParseCache.find(path2);
+    if (j != fileParseCache.end())
+        e = j->second;
+
+    if (!e)
+        e = parseExprFromFile(checkSourcePath(path2));
+
+    fileParseCache[path2] = e;
+
     try {
         eval(e, v);
     } catch (Error & e) {
@@ -721,6 +769,7 @@ void EvalState::evalFile(const Path & path_, Value & v)
 void EvalState::resetFileCache()
 {
     fileEvalCache.clear();
+    fileParseCache.clear();
 }
 
 
@@ -805,7 +854,7 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v)
         /* The recursive attributes are evaluated in the new
            environment, while the inherited attributes are evaluated
            in the original environment. */
-        unsigned int displ = 0;
+        size_t displ = 0;
         for (auto & i : attrs) {
             Value * vAttr;
             if (hasOverrides && !i.second.inherited) {
@@ -879,7 +928,7 @@ void ExprLet::eval(EvalState & state, Env & env, Value & v)
     /* The recursive attributes are evaluated in the new environment,
        while the inherited attributes are evaluated in the original
        environment. */
-    unsigned int displ = 0;
+    size_t displ = 0;
     for (auto & i : attrs->attrs)
         env2.values[displ++] = i.second.e->maybeThunk(state, i.second.inherited ? env : env2);
 
@@ -890,7 +939,7 @@ void ExprLet::eval(EvalState & state, Env & env, Value & v)
 void ExprList::eval(EvalState & state, Env & env, Value & v)
 {
     state.mkList(v, elems.size());
-    for (unsigned int n = 0; n < elems.size(); ++n)
+    for (size_t n = 0; n < elems.size(); ++n)
         v.listElems()[n] = elems[n]->maybeThunk(state, env);
 }
 
@@ -1012,22 +1061,22 @@ void ExprApp::eval(EvalState & state, Env & env, Value & v)
 void EvalState::callPrimOp(Value & fun, Value & arg, Value & v, const Pos & pos)
 {
     /* Figure out the number of arguments still needed. */
-    unsigned int argsDone = 0;
+    size_t argsDone = 0;
     Value * primOp = &fun;
     while (primOp->type == tPrimOpApp) {
         argsDone++;
         primOp = primOp->primOpApp.left;
     }
     assert(primOp->type == tPrimOp);
-    unsigned int arity = primOp->primOp->arity;
-    unsigned int argsLeft = arity - argsDone;
+    auto arity = primOp->primOp->arity;
+    auto argsLeft = arity - argsDone;
 
     if (argsLeft == 1) {
         /* We have all the arguments, so call the primop. */
 
         /* Put all the arguments in an array. */
         Value * vArgs[arity];
-        unsigned int n = arity - 1;
+        auto n = arity - 1;
         vArgs[n--] = &arg;
         for (Value * arg = &fun; arg->type == tPrimOpApp; arg = arg->primOpApp.left)
             vArgs[n--] = arg->primOpApp.right;
@@ -1048,6 +1097,8 @@ void EvalState::callPrimOp(Value & fun, Value & arg, Value & v, const Pos & pos)
 
 void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & pos)
 {
+    forceValue(fun, pos);
+
     if (fun.type == tPrimOp || fun.type == tPrimOpApp) {
         callPrimOp(fun, arg, v, pos);
         return;
@@ -1063,10 +1114,8 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po
         auto & fun2 = *allocValue();
         fun2 = fun;
         /* !!! Should we use the attr pos here? */
-        forceValue(*found->value, pos);
         Value v2;
         callFunction(*found->value, fun2, v2, pos);
-        forceValue(v2, pos);
         return callFunction(v2, arg, v, pos);
       }
     }
@@ -1076,13 +1125,13 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po
 
     ExprLambda & lambda(*fun.lambda.fun);
 
-    unsigned int size =
+    auto size =
         (lambda.arg.empty() ? 0 : 1) +
         (lambda.matchAttrs ? lambda.formals->formals.size() : 0);
     Env & env2(allocEnv(size));
     env2.up = fun.lambda.env;
 
-    unsigned int displ = 0;
+    size_t displ = 0;
 
     if (!lambda.matchAttrs)
         env2.values[displ++] = &arg;
@@ -1096,7 +1145,7 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po
         /* For each formal argument, get the actual argument.  If
            there is no matching actual argument but the formal
            argument has a default, use the default. */
-        unsigned int attrsUsed = 0;
+        size_t attrsUsed = 0;
         for (auto & i : lambda.formals->formals) {
             Bindings::iterator j = arg.attrs->find(i.name);
             if (j == arg.attrs->end()) {
@@ -1153,7 +1202,6 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res)
     if (fun.type == tAttrs) {
         auto found = fun.attrs->find(sFunctor);
         if (found != fun.attrs->end()) {
-            forceValue(*found->value);
             Value * v = allocValue();
             callFunction(*found->value, fun, *v, noPos);
             forceValue(*v);
@@ -1188,7 +1236,7 @@ void ExprWith::eval(EvalState & state, Env & env, Value & v)
     Env & env2(state.allocEnv(1));
     env2.up = &env;
     env2.prevWith = prevWith;
-    env2.haveWithAttrs = false;
+    env2.type = Env::HasWithExpr;
     env2.values[0] = (Value *) attrs;
 
     body->eval(state, env2, v);
@@ -1294,15 +1342,15 @@ void ExprOpConcatLists::eval(EvalState & state, Env & env, Value & v)
 }
 
 
-void EvalState::concatLists(Value & v, unsigned int nrLists, Value * * lists, const Pos & pos)
+void EvalState::concatLists(Value & v, size_t nrLists, Value * * lists, const Pos & pos)
 {
     nrListConcats++;
 
     Value * nonEmpty = 0;
-    unsigned int len = 0;
-    for (unsigned int n = 0; n < nrLists; ++n) {
+    size_t len = 0;
+    for (size_t n = 0; n < nrLists; ++n) {
         forceList(*lists[n], pos);
-        unsigned int l = lists[n]->listSize();
+        auto l = lists[n]->listSize();
         len += l;
         if (l) nonEmpty = lists[n];
     }
@@ -1314,9 +1362,10 @@ void EvalState::concatLists(Value & v, unsigned int nrLists, Value * * lists, co
 
     mkList(v, len);
     auto out = v.listElems();
-    for (unsigned int n = 0, pos = 0; n < nrLists; ++n) {
-        unsigned int l = lists[n]->listSize();
-        memcpy(out + pos, lists[n]->listElems(), l * sizeof(Value *));
+    for (size_t n = 0, pos = 0; n < nrLists; ++n) {
+        auto l = lists[n]->listSize();
+        if (l)
+            memcpy(out + pos, lists[n]->listElems(), l * sizeof(Value *));
         pos += l;
     }
 }
@@ -1409,7 +1458,7 @@ void EvalState::forceValueDeep(Value & v)
         }
 
         else if (v.isList()) {
-            for (unsigned int n = 0; n < v.listSize(); ++n)
+            for (size_t n = 0; n < v.listSize(); ++n)
                 recurse(*v.listElems()[n]);
         }
     };
@@ -1536,7 +1585,6 @@ string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context,
     if (v.type == tAttrs) {
         auto i = v.attrs->find(sToString);
         if (i != v.attrs->end()) {
-            forceValue(*i->value, pos);
             Value v1;
             callFunction(*i->value, v, v1, pos);
             return coerceToString(pos, v1, context, coerceMore, copyToStore);
@@ -1561,7 +1609,7 @@ string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context,
 
         if (v.isList()) {
             string result;
-            for (unsigned int n = 0; n < v.listSize(); ++n) {
+            for (size_t n = 0; n < v.listSize(); ++n) {
                 result += coerceToString(pos, *v.listElems()[n],
                     context, coerceMore, copyToStore);
                 if (n < v.listSize() - 1
@@ -1648,7 +1696,7 @@ bool EvalState::eqValues(Value & v1, Value & v2)
         case tList2:
         case tListN:
             if (v1.listSize() != v2.listSize()) return false;
-            for (unsigned int n = 0; n < v1.listSize(); ++n)
+            for (size_t n = 0; n < v1.listSize(); ++n)
                 if (!eqValues(*v1.listElems()[n], *v2.listElems()[n])) return false;
             return true;
 
@@ -1690,12 +1738,9 @@ bool EvalState::eqValues(Value & v1, Value & v2)
     }
 }
 
-
 void EvalState::printStats()
 {
     bool showStats = getEnv("NIX_SHOW_STATS", "0") != "0";
-    Verbosity v = showStats ? lvlInfo : lvlDebug;
-    printMsg(v, "evaluation statistics:");
 
     struct rusage buf;
     getrusage(RUSAGE_SELF, &buf);
@@ -1706,62 +1751,107 @@ void EvalState::printStats()
     uint64_t bValues = nrValues * sizeof(Value);
     uint64_t bAttrsets = nrAttrsets * sizeof(Bindings) + nrAttrsInAttrsets * sizeof(Attr);
 
-    printMsg(v, format("  time elapsed: %1%") % cpuTime);
-    printMsg(v, format("  size of a value: %1%") % sizeof(Value));
-    printMsg(v, format("  size of an attr: %1%") % sizeof(Attr));
-    printMsg(v, format("  environments allocated count: %1%") % nrEnvs);
-    printMsg(v, format("  environments allocated bytes: %1%") % bEnvs);
-    printMsg(v, format("  list elements count: %1%") % nrListElems);
-    printMsg(v, format("  list elements bytes: %1%") % bLists);
-    printMsg(v, format("  list concatenations: %1%") % nrListConcats);
-    printMsg(v, format("  values allocated count: %1%") % nrValues);
-    printMsg(v, format("  values allocated bytes: %1%") % bValues);
-    printMsg(v, format("  sets allocated: %1% (%2% bytes)") % nrAttrsets % bAttrsets);
-    printMsg(v, format("  right-biased unions: %1%") % nrOpUpdates);
-    printMsg(v, format("  values copied in right-biased unions: %1%") % nrOpUpdateValuesCopied);
-    printMsg(v, format("  symbols in symbol table: %1%") % symbols.size());
-    printMsg(v, format("  size of symbol table: %1%") % symbols.totalSize());
-    printMsg(v, format("  number of thunks: %1%") % nrThunks);
-    printMsg(v, format("  number of thunks avoided: %1%") % nrAvoided);
-    printMsg(v, format("  number of attr lookups: %1%") % nrLookups);
-    printMsg(v, format("  number of primop calls: %1%") % nrPrimOpCalls);
-    printMsg(v, format("  number of function calls: %1%") % nrFunctionCalls);
-    printMsg(v, format("  total allocations: %1% bytes") % (bEnvs + bLists + bValues + bAttrsets));
-
 #if HAVE_BOEHMGC
     GC_word heapSize, totalBytes;
     GC_get_heap_usage_safe(&heapSize, 0, 0, 0, &totalBytes);
-    printMsg(v, format("  current Boehm heap size: %1% bytes") % heapSize);
-    printMsg(v, format("  total Boehm heap allocations: %1% bytes") % totalBytes);
+#endif
+    if (showStats) {
+        auto outPath = getEnv("NIX_SHOW_STATS_PATH","-");
+        std::fstream fs;
+        if (outPath != "-")
+            fs.open(outPath, std::fstream::out);
+        JSONObject topObj(outPath == "-" ? std::cerr : fs, true);
+        topObj.attr("cpuTime",cpuTime);
+        {
+            auto envs = topObj.object("envs");
+            envs.attr("number", nrEnvs);
+            envs.attr("elements", nrValuesInEnvs);
+            envs.attr("bytes", bEnvs);
+        }
+        {
+            auto lists = topObj.object("list");
+            lists.attr("elements", nrListElems);
+            lists.attr("bytes", bLists);
+            lists.attr("concats", nrListConcats);
+        }
+        {
+            auto values = topObj.object("values");
+            values.attr("number", nrValues);
+            values.attr("bytes", bValues);
+        }
+        {
+            auto syms = topObj.object("symbols");
+            syms.attr("number", symbols.size());
+            syms.attr("bytes", symbols.totalSize());
+        }
+        {
+            auto sets = topObj.object("sets");
+            sets.attr("number", nrAttrsets);
+            sets.attr("bytes", bAttrsets);
+            sets.attr("elements", nrAttrsInAttrsets);
+        }
+        {
+            auto sizes = topObj.object("sizes");
+            sizes.attr("Env", sizeof(Env));
+            sizes.attr("Value", sizeof(Value));
+            sizes.attr("Bindings", sizeof(Bindings));
+            sizes.attr("Attr", sizeof(Attr));
+        }
+        topObj.attr("nrOpUpdates", nrOpUpdates);
+        topObj.attr("nrOpUpdateValuesCopied", nrOpUpdateValuesCopied);
+        topObj.attr("nrThunks", nrThunks);
+        topObj.attr("nrAvoided", nrAvoided);
+        topObj.attr("nrLookups", nrLookups);
+        topObj.attr("nrPrimOpCalls", nrPrimOpCalls);
+        topObj.attr("nrFunctionCalls", nrFunctionCalls);
+#if HAVE_BOEHMGC
+        {
+            auto gc = topObj.object("gc");
+            gc.attr("heapSize", heapSize);
+            gc.attr("totalBytes", totalBytes);
+        }
 #endif
 
-    if (countCalls) {
-        v = lvlInfo;
-
-        printMsg(v, format("calls to %1% primops:") % primOpCalls.size());
-        typedef std::multimap<unsigned int, Symbol> PrimOpCalls_;
-        PrimOpCalls_ primOpCalls_;
-        for (auto & i : primOpCalls)
-            primOpCalls_.insert(std::pair<unsigned int, Symbol>(i.second, i.first));
-        for (auto i = primOpCalls_.rbegin(); i != primOpCalls_.rend(); ++i)
-            printMsg(v, format("%1$10d %2%") % i->first % i->second);
-
-        printMsg(v, format("calls to %1% functions:") % functionCalls.size());
-        typedef std::multimap<unsigned int, ExprLambda *> FunctionCalls_;
-        FunctionCalls_ functionCalls_;
-        for (auto & i : functionCalls)
-            functionCalls_.insert(std::pair<unsigned int, ExprLambda *>(i.second, i.first));
-        for (auto i = functionCalls_.rbegin(); i != functionCalls_.rend(); ++i)
-            printMsg(v, format("%1$10d %2%") % i->first % i->second->showNamePos());
-
-        printMsg(v, format("evaluations of %1% attributes:") % attrSelects.size());
-        typedef std::multimap<unsigned int, Pos> AttrSelects_;
-        AttrSelects_ attrSelects_;
-        for (auto & i : attrSelects)
-            attrSelects_.insert(std::pair<unsigned int, Pos>(i.second, i.first));
-        for (auto i = attrSelects_.rbegin(); i != attrSelects_.rend(); ++i)
-            printMsg(v, format("%1$10d %2%") % i->first % i->second);
+        if (countCalls) {
+            {
+                auto obj = topObj.object("primops");
+                for (auto & i : primOpCalls)
+                    obj.attr(i.first, i.second);
+            }
+            {
+                auto list = topObj.list("functions");
+                for (auto & i : functionCalls) {
+                    auto obj = list.object();
+                    if (i.first->name.set())
+                        obj.attr("name", (const string &) i.first->name);
+                    else
+                        obj.attr("name", nullptr);
+                    if (i.first->pos) {
+                        obj.attr("file", (const string &) i.first->pos.file);
+                        obj.attr("line", i.first->pos.line);
+                        obj.attr("column", i.first->pos.column);
+                    }
+                    obj.attr("count", i.second);
+                }
+            }
+            {
+                auto list = topObj.list("attributes");
+                for (auto & i : attrSelects) {
+                    auto obj = list.object();
+                    if (i.first) {
+                        obj.attr("file", (const string &) i.first.file);
+                        obj.attr("line", i.first.line);
+                        obj.attr("column", i.first.column);
+                    }
+                    obj.attr("count", i.second);
+                }
+            }
+        }
 
+        if (getEnv("NIX_SHOW_SYMBOLS", "0") != "0") {
+            auto list = topObj.list("symbols");
+            symbols.dump([&](const std::string & s) { list.elem(s); });
+        }
     }
 }
 
@@ -1809,7 +1899,7 @@ size_t valueSize(Value & v)
             if (seen.find(v.listElems()) == seen.end()) {
                 seen.insert(v.listElems());
                 sz += v.listSize() * sizeof(Value *);
-                for (unsigned int n = 0; n < v.listSize(); ++n)
+                for (size_t n = 0; n < v.listSize(); ++n)
                     sz += doValue(*v.listElems()[n]);
             }
             break;
@@ -1845,9 +1935,10 @@ size_t valueSize(Value & v)
 
         size_t sz = sizeof(Env) + sizeof(Value *) * env.size;
 
-        for (unsigned int i = 0; i < env.size; ++i)
-            if (env.values[i])
-                sz += doValue(*env.values[i]);
+        if (env.type != Env::HasWithExpr)
+            for (size_t i = 0; i < env.size; ++i)
+                if (env.values[i])
+                    sz += doValue(*env.values[i]);
 
         if (env.up) sz += doEnv(*env.up);
 
@@ -1876,4 +1967,9 @@ std::ostream & operator << (std::ostream & str, const ExternalValueBase & v) {
 }
 
 
+EvalSettings evalSettings;
+
+static GlobalConfig::Register r1(&evalSettings);
+
+
 }

src/libexpr/eval.hh

@@ -5,8 +5,10 @@
 #include "nixexpr.hh"
 #include "symbol-table.hh"
 #include "hash.hh"
+#include "config.hh"
 
 #include <map>
+#include <unordered_map>
 
 
 namespace nix {
@@ -34,8 +36,8 @@ struct Env
 {
     Env * up;
     unsigned short size; // used by ‘valueSize’
-    unsigned short prevWith:15; // nr of levels up to next `with' environment
-    unsigned short haveWithAttrs:1;
+    unsigned short prevWith:14; // nr of levels up to next `with' environment
+    enum { Plain = 0, HasWithExpr, HasWithAttrs } type:2;
     Value * values[0];
 };
 
@@ -79,7 +81,7 @@ public:
 
     /* The allowed filesystem paths in restricted or pure evaluation
        mode. */
-    std::experimental::optional<PathSet> allowedPaths;
+    std::optional<PathSet> allowedPaths;
 
     Value vEmptySet;
 
@@ -88,6 +90,14 @@ public:
 private:
     SrcToStore srcToStore;
 
+    /* A cache from path names to parse trees. */
+#if HAVE_BOEHMGC
+    typedef std::map<Path, Expr *, std::less<Path>, traceable_allocator<std::pair<const Path, Expr *> > > FileParseCache;
+#else
+    typedef std::map<Path, Expr *> FileParseCache;
+#endif
+    FileParseCache fileParseCache;
+
     /* A cache from path names to values. */
 #if HAVE_BOEHMGC
     typedef std::map<Path, Value, std::less<Path>, traceable_allocator<std::pair<const Path, Value> > > FileEvalCache;
@@ -100,6 +110,9 @@ private:
 
     std::map<std::string, std::pair<bool, std::string>> searchPathResolved;
 
+    /* Cache used by checkSourcePath(). */
+    std::unordered_map<Path, Path> resolvedPaths;
+
 public:
 
     EvalState(const Strings & _searchPath, ref<Store> store);
@@ -214,7 +227,7 @@ private:
     Value * addConstant(const string & name, Value & v);
 
     Value * addPrimOp(const string & name,
-        unsigned int arity, PrimOpFun primOp);
+        size_t arity, PrimOpFun primOp);
 
 public:
 
@@ -248,18 +261,18 @@ public:
 
     /* Allocation primitives. */
     Value * allocValue();
-    Env & allocEnv(unsigned int size);
+    Env & allocEnv(size_t size);
 
     Value * allocAttr(Value & vAttrs, const Symbol & name);
 
-    Bindings * allocBindings(Bindings::size_t capacity);
+    Bindings * allocBindings(size_t capacity);
 
-    void mkList(Value & v, unsigned int length);
-    void mkAttrs(Value & v, unsigned int capacity);
+    void mkList(Value & v, size_t length);
+    void mkAttrs(Value & v, size_t capacity);
     void mkThunk_(Value & v, Expr * expr);
     void mkPos(Value & v, Pos * pos);
 
-    void concatLists(Value & v, unsigned int nrLists, Value * * lists, const Pos & pos);
+    void concatLists(Value & v, size_t nrLists, Value * * lists, const Pos & pos);
 
     /* Print statistics. */
     void printStats();
@@ -282,15 +295,15 @@ private:
 
     bool countCalls;
 
-    typedef std::map<Symbol, unsigned int> PrimOpCalls;
+    typedef std::map<Symbol, size_t> PrimOpCalls;
     PrimOpCalls primOpCalls;
 
-    typedef std::map<ExprLambda *, unsigned int> FunctionCalls;
+    typedef std::map<ExprLambda *, size_t> FunctionCalls;
     FunctionCalls functionCalls;
 
     void incrFunctionCall(ExprLambda * fun);
 
-    typedef std::map<Pos, unsigned int> AttrSelects;
+    typedef std::map<Pos, size_t> AttrSelects;
     AttrSelects attrSelects;
 
     friend struct ExprOpUpdate;
@@ -303,6 +316,9 @@ private:
 /* Return a string representing the type of the value `v'. */
 string showType(const Value & v);
 
+/* Decode a context string ‘!<name>!<path>’ into a pair <path,
+   name>. */
+std::pair<string, string> decodeContext(const string & s);
 
 /* If `path' refers to a directory, then append "/default.nix". */
 Path resolveExprPath(Path path);
@@ -316,4 +332,25 @@ struct InvalidPathError : EvalError
 #endif
 };
 
+struct EvalSettings : Config
+{
+    Setting<bool> enableNativeCode{this, false, "allow-unsafe-native-code-during-evaluation",
+        "Whether builtin functions that allow executing native code should be enabled."};
+
+    Setting<bool> restrictEval{this, false, "restrict-eval",
+        "Whether to restrict file system access to paths in $NIX_PATH, "
+        "and network access to the URI prefixes listed in 'allowed-uris'."};
+
+    Setting<bool> pureEval{this, false, "pure-eval",
+        "Whether to restrict file system and network access to files specified by cryptographic hash."};
+
+    Setting<bool> enableImportFromDerivation{this, true, "allow-import-from-derivation",
+        "Whether the evaluator allows importing the result of a derivation."};
+
+    Setting<Strings> allowedUris{this, {}, "allowed-uris",
+        "Prefixes of URIs that builtin functions such as fetchurl and fetchGit are allowed to fetch."};
+};
+
+extern EvalSettings evalSettings;
+
 }

src/libexpr/get-drvs.cc

@@ -295,7 +295,7 @@ static bool getDerivation(EvalState & state, Value & v,
 }
 
 
-std::experimental::optional<DrvInfo> getDerivation(EvalState & state, Value & v,
+std::optional<DrvInfo> getDerivation(EvalState & state, Value & v,
     bool ignoreAssertionFailures)
 {
     Done done;

src/libexpr/get-drvs.hh

@@ -44,7 +44,7 @@ public:
     string queryDrvPath() const;
     string queryOutPath() const;
     string queryOutputName() const;
-    /** Return the list of outputs. The "outputs to install" are determined by `mesa.outputsToInstall`. */
+    /** Return the list of outputs. The "outputs to install" are determined by `meta.outputsToInstall`. */
     Outputs queryOutputs(bool onlyOutputsToInstall = false);
 
     StringSet queryMetaNames();
@@ -78,7 +78,7 @@ typedef list<DrvInfo> DrvInfos;
 
 /* If value `v' denotes a derivation, return a DrvInfo object
    describing it. Otherwise return nothing. */
-std::experimental::optional<DrvInfo> getDerivation(EvalState & state,
+std::optional<DrvInfo> getDerivation(EvalState & state,
     Value & v, bool ignoreAssertionFailures);
 
 void getDerivations(EvalState & state, Value & v, const string & pathPrefix,

src/libexpr/json-to-value.cc

@@ -110,7 +110,7 @@ static void parseJSON(EvalState & state, const char * & s, Value & v)
             if (number_type == tFloat)
                 mkFloat(v, stod(tmp_number));
             else
-                mkInt(v, stoi(tmp_number));
+                mkInt(v, stol(tmp_number));
         } catch (std::invalid_argument e) {
             throw JSONParseError("invalid JSON number");
         } catch (std::out_of_range e) {

src/libexpr/lexer.l

@@ -6,12 +6,14 @@
 %option nounput noyy_top_state
 
 
+%s DEFAULT
 %x STRING
 %x IND_STRING
-%x INSIDE_DOLLAR_CURLY
 
 
 %{
+#include <boost/lexical_cast.hpp>
+
 #include "nixexpr.hh"
 #include "parser-tab.hh"
 
@@ -85,6 +87,7 @@ static Expr * unescapeStr(SymbolTable & symbols, const char * s, size_t length)
 %}
 
 
+ANY         .|\n
 ID          [a-zA-Z\_][a-zA-Z0-9\_\'\-]*
 INT         [0-9]+
 FLOAT       (([1-9][0-9]*\.[0-9]*)|(0?\.[0-9]+))([Ee][+-]?[0-9]+)?
@@ -96,8 +99,6 @@ URI         [a-zA-Z][a-zA-Z0-9\+\-\.]*\:[a-zA-Z0-9\%\/\?\:\@\&\=\+\$\,\-\_\.\!\~
 
 %%
 
-<INITIAL,INSIDE_DOLLAR_CURLY>{
-
 
 if          { return IF; }
 then        { return THEN; }
@@ -123,9 +124,11 @@ or          { return OR_KW; }
 
 {ID}        { yylval->id = strdup(yytext); return ID; }
 {INT}       { errno = 0;
-              yylval->n = strtol(yytext, 0, 10);
-              if (errno != 0)
+              try {
+                  yylval->n = boost::lexical_cast<int64_t>(yytext);
+              } catch (const boost::bad_lexical_cast &) {
                   throw ParseError(format("invalid integer '%1%'") % yytext);
+              }
               return INT;
             }
 {FLOAT}     { errno = 0;
@@ -135,26 +138,28 @@ or          { return OR_KW; }
               return FLOAT;
             }
 
-\$\{        { PUSH_STATE(INSIDE_DOLLAR_CURLY); return DOLLAR_CURLY; }
-}
+\$\{        { PUSH_STATE(DEFAULT); return DOLLAR_CURLY; }
 
-\}                           { return '}'; }
-<INSIDE_DOLLAR_CURLY>\}      { POP_STATE(); return '}'; }
-\{                           { return '{'; }
-<INSIDE_DOLLAR_CURLY>\{      { PUSH_STATE(INSIDE_DOLLAR_CURLY); return '{'; }
+\}          { /* State INITIAL only exists at the bottom of the stack and is
+                 used as a marker. DEFAULT replaces it everywhere else.
+                 Popping when in INITIAL state causes an empty stack exception,
+                 so don't */
+              if (YYSTATE != INITIAL)
+                POP_STATE();
+              return '}';
+            }
+\{          { PUSH_STATE(DEFAULT); return '{'; }
 
-<INITIAL,INSIDE_DOLLAR_CURLY>\" {
-                PUSH_STATE(STRING); return '"';
-              }
-<STRING>([^\$\"\\]|\$[^\{\"\\]|\\.|\$\\.)*\$/\" |
-<STRING>([^\$\"\\]|\$[^\{\"\\]|\\.|\$\\.)+ {
+\"          { PUSH_STATE(STRING); return '"'; }
+<STRING>([^\$\"\\]|\$[^\{\"\\]|\\{ANY}|\$\\{ANY})*\$/\" |
+<STRING>([^\$\"\\]|\$[^\{\"\\]|\\{ANY}|\$\\{ANY})+ {
                 /* It is impossible to match strings ending with '$' with one
                    regex because trailing contexts are only valid at the end
                    of a rule. (A sane but undocumented limitation.) */
                 yylval->e = unescapeStr(data->symbols, yytext, yyleng);
                 return STR;
               }
-<STRING>\$\{  { PUSH_STATE(INSIDE_DOLLAR_CURLY); return DOLLAR_CURLY; }
+<STRING>\$\{  { PUSH_STATE(DEFAULT); return DOLLAR_CURLY; }
 <STRING>\"    { POP_STATE(); return '"'; }
 <STRING>\$|\\|\$\\ {
                 /* This can only occur when we reach EOF, otherwise the above
@@ -164,7 +169,7 @@ or          { return OR_KW; }
                 return STR;
               }
 
-<INITIAL,INSIDE_DOLLAR_CURLY>\'\'(\ *\n)?     { PUSH_STATE(IND_STRING); return IND_STRING_OPEN; }
+\'\'(\ *\n)?     { PUSH_STATE(IND_STRING); return IND_STRING_OPEN; }
 <IND_STRING>([^\$\']|\$[^\{\']|\'[^\'\$])+ {
                    yylval->e = new ExprIndStr(yytext);
                    return IND_STR;
@@ -178,18 +183,17 @@ or          { return OR_KW; }
                    yylval->e = new ExprIndStr("''");
                    return IND_STR;
                  }
-<IND_STRING>\'\'\\. {
+<IND_STRING>\'\'\\{ANY} {
                    yylval->e = unescapeStr(data->symbols, yytext + 2, yyleng - 2);
                    return IND_STR;
                  }
-<IND_STRING>\$\{ { PUSH_STATE(INSIDE_DOLLAR_CURLY); return DOLLAR_CURLY; }
+<IND_STRING>\$\{ { PUSH_STATE(DEFAULT); return DOLLAR_CURLY; }
 <IND_STRING>\'\' { POP_STATE(); return IND_STRING_CLOSE; }
 <IND_STRING>\'   {
                    yylval->e = new ExprIndStr("'");
                    return IND_STR;
                  }
 
-<INITIAL,INSIDE_DOLLAR_CURLY>{
 
 {PATH}      { if (yytext[yyleng-1] == '/')
                   throw ParseError("path '%s' has a trailing slash", yytext);
@@ -208,11 +212,11 @@ or          { return OR_KW; }
 \#[^\r\n]*    /* single-line comments */
 \/\*([^*]|\*+[^*/])*\*+\/  /* long comments */
 
-.           return yytext[0];
-
-}
-
-<<EOF>> { data->atEnd = true; return 0; }
+{ANY}       {
+              /* Don't return a negative number, as this will cause
+                 Bison to stop parsing without an error. */
+              return (unsigned char) yytext[0];
+            }
 
 %%
 

src/libexpr/local.mk

@@ -6,7 +6,7 @@ libexpr_DIR := $(d)
 
 libexpr_SOURCES := $(wildcard $(d)/*.cc) $(wildcard $(d)/primops/*.cc) $(d)/lexer-tab.cc $(d)/parser-tab.cc
 
-libexpr_LIBS = libutil libstore libformat
+libexpr_LIBS = libutil libstore
 
 libexpr_LDFLAGS =
 ifneq ($(OS), FreeBSD)

src/libexpr/nix-expr.pc.in

@@ -7,4 +7,4 @@ Description: Nix Package Manager
 Version: @PACKAGE_VERSION@
 Requires: nix-store bdw-gc
 Libs: -L${libdir} -lnixexpr
-Cflags: -I${includedir}/nix
+Cflags: -I${includedir}/nix -std=c++17

src/libexpr/nixexpr.cc

@@ -10,7 +10,7 @@ namespace nix {
 
 /* Displaying abstract syntax trees. */
 
-std::ostream & operator << (std::ostream & str, Expr & e)
+std::ostream & operator << (std::ostream & str, const Expr & e)
 {
     e.show(str);
     return str;
@@ -58,48 +58,48 @@ std::ostream & operator << (std::ostream & str, const Symbol & sym)
     return str;
 }
 
-void Expr::show(std::ostream & str)
+void Expr::show(std::ostream & str) const
 {
     abort();
 }
 
-void ExprInt::show(std::ostream & str)
+void ExprInt::show(std::ostream & str) const
 {
     str << n;
 }
 
-void ExprFloat::show(std::ostream & str)
+void ExprFloat::show(std::ostream & str) const
 {
     str << nf;
 }
 
-void ExprString::show(std::ostream & str)
+void ExprString::show(std::ostream & str) const
 {
     showString(str, s);
 }
 
-void ExprPath::show(std::ostream & str)
+void ExprPath::show(std::ostream & str) const
 {
     str << s;
 }
 
-void ExprVar::show(std::ostream & str)
+void ExprVar::show(std::ostream & str) const
 {
     str << name;
 }
 
-void ExprSelect::show(std::ostream & str)
+void ExprSelect::show(std::ostream & str) const
 {
     str << "(" << *e << ")." << showAttrPath(attrPath);
     if (def) str << " or (" << *def << ")";
 }
 
-void ExprOpHasAttr::show(std::ostream & str)
+void ExprOpHasAttr::show(std::ostream & str) const
 {
     str << "((" << *e << ") ? " << showAttrPath(attrPath) << ")";
 }
 
-void ExprAttrs::show(std::ostream & str)
+void ExprAttrs::show(std::ostream & str) const
 {
     if (recursive) str << "rec ";
     str << "{ ";
@@ -113,7 +113,7 @@ void ExprAttrs::show(std::ostream & str)
     str << "}";
 }
 
-void ExprList::show(std::ostream & str)
+void ExprList::show(std::ostream & str) const
 {
     str << "[ ";
     for (auto & i : elems)
@@ -121,7 +121,7 @@ void ExprList::show(std::ostream & str)
     str << "]";
 }
 
-void ExprLambda::show(std::ostream & str)
+void ExprLambda::show(std::ostream & str) const
 {
     str << "(";
     if (matchAttrs) {
@@ -143,7 +143,7 @@ void ExprLambda::show(std::ostream & str)
     str << ": " << *body << ")";
 }
 
-void ExprLet::show(std::ostream & str)
+void ExprLet::show(std::ostream & str) const
 {
     str << "(let ";
     for (auto & i : attrs->attrs)
@@ -155,27 +155,27 @@ void ExprLet::show(std::ostream & str)
     str << "in " << *body << ")";
 }
 
-void ExprWith::show(std::ostream & str)
+void ExprWith::show(std::ostream & str) const
 {
     str << "(with " << *attrs << "; " << *body << ")";
 }
 
-void ExprIf::show(std::ostream & str)
+void ExprIf::show(std::ostream & str) const
 {
     str << "(if " << *cond << " then " << *then << " else " << *else_ << ")";
 }
 
-void ExprAssert::show(std::ostream & str)
+void ExprAssert::show(std::ostream & str) const
 {
     str << "assert " << *cond << "; " << *body;
 }
 
-void ExprOpNot::show(std::ostream & str)
+void ExprOpNot::show(std::ostream & str) const
 {
     str << "(! " << *e << ")";
 }
 
-void ExprConcatStrings::show(std::ostream & str)
+void ExprConcatStrings::show(std::ostream & str) const
 {
     bool first = true;
     str << "(";
@@ -186,7 +186,7 @@ void ExprConcatStrings::show(std::ostream & str)
     str << ")";
 }
 
-void ExprPos::show(std::ostream & str)
+void ExprPos::show(std::ostream & str) const
 {
     str << "__curPos";
 }

src/libexpr/nixexpr.hh

@@ -11,7 +11,6 @@ namespace nix {
 
 MakeError(EvalError, Error)
 MakeError(ParseError, Error)
-MakeError(IncompleteParseError, ParseError)
 MakeError(AssertionError, EvalError)
 MakeError(ThrownError, AssertionError)
 MakeError(Abort, EvalError)
@@ -76,17 +75,17 @@ string showAttrPath(const AttrPath & attrPath);
 struct Expr
 {
     virtual ~Expr() { };
-    virtual void show(std::ostream & str);
+    virtual void show(std::ostream & str) const;
     virtual void bindVars(const StaticEnv & env);
     virtual void eval(EvalState & state, Env & env, Value & v);
     virtual Value * maybeThunk(EvalState & state, Env & env);
     virtual void setName(Symbol & name);
 };
 
-std::ostream & operator << (std::ostream & str, Expr & e);
+std::ostream & operator << (std::ostream & str, const Expr & e);
 
 #define COMMON_METHODS \
-    void show(std::ostream & str); \
+    void show(std::ostream & str) const; \
     void eval(EvalState & state, Env & env, Value & v); \
     void bindVars(const StaticEnv & env);
 
@@ -255,7 +254,7 @@ struct ExprWith : Expr
 {
     Pos pos;
     Expr * attrs, * body;
-    unsigned int prevWith;
+    size_t prevWith;
     ExprWith(const Pos & pos, Expr * attrs, Expr * body) : pos(pos), attrs(attrs), body(body) { };
     COMMON_METHODS
 };
@@ -283,13 +282,13 @@ struct ExprOpNot : Expr
 };
 
 #define MakeBinOp(name, s) \
-    struct Expr##name : Expr \
+    struct name : Expr \
     { \
         Pos pos; \
         Expr * e1, * e2; \
-        Expr##name(Expr * e1, Expr * e2) : e1(e1), e2(e2) { }; \
-        Expr##name(const Pos & pos, Expr * e1, Expr * e2) : pos(pos), e1(e1), e2(e2) { }; \
-        void show(std::ostream & str) \
+        name(Expr * e1, Expr * e2) : e1(e1), e2(e2) { }; \
+        name(const Pos & pos, Expr * e1, Expr * e2) : pos(pos), e1(e1), e2(e2) { }; \
+        void show(std::ostream & str) const \
         { \
             str << "(" << *e1 << " " s " " << *e2 << ")";   \
         } \
@@ -300,14 +299,14 @@ struct ExprOpNot : Expr
         void eval(EvalState & state, Env & env, Value & v); \
     };
 
-MakeBinOp(App, "")
-MakeBinOp(OpEq, "==")
-MakeBinOp(OpNEq, "!=")
-MakeBinOp(OpAnd, "&&")
-MakeBinOp(OpOr, "||")
-MakeBinOp(OpImpl, "->")
-MakeBinOp(OpUpdate, "//")
-MakeBinOp(OpConcatLists, "++")
+MakeBinOp(ExprApp, "")
+MakeBinOp(ExprOpEq, "==")
+MakeBinOp(ExprOpNEq, "!=")
+MakeBinOp(ExprOpAnd, "&&")
+MakeBinOp(ExprOpOr, "||")
+MakeBinOp(ExprOpImpl, "->")
+MakeBinOp(ExprOpUpdate, "//")
+MakeBinOp(ExprOpConcatLists, "++")
 
 struct ExprConcatStrings : Expr
 {

src/libexpr/parser.y

@@ -1,7 +1,7 @@
 %glr-parser
 %pure-parser
 %locations
-%error-verbose
+%define parse.error verbose
 %defines
 /* %no-lines */
 %parse-param { void * scanner }
@@ -31,12 +31,10 @@ namespace nix {
         Path basePath;
         Symbol path;
         string error;
-        bool atEnd;
         Symbol sLetBody;
         ParseData(EvalState & state)
             : state(state)
             , symbols(state.symbols)
-            , atEnd(false)
             , sLetBody(symbols.create("<let-body>"))
             { };
     };
@@ -136,8 +134,8 @@ static Expr * stripIndentation(const Pos & pos, SymbolTable & symbols, vector<Ex
        whitespace-only final lines are not taken into account.  (So
        the " " in "\n ''" is ignored, but the " " in "\n foo''" is.) */
     bool atStartOfLine = true; /* = seen only whitespace in the current line */
-    unsigned int minIndent = 1000000;
-    unsigned int curIndent = 0;
+    size_t minIndent = 1000000;
+    size_t curIndent = 0;
     for (auto & i : es) {
         ExprIndStr * e = dynamic_cast<ExprIndStr *>(i);
         if (!e) {
@@ -148,7 +146,7 @@ static Expr * stripIndentation(const Pos & pos, SymbolTable & symbols, vector<Ex
             }
             continue;
         }
-        for (unsigned int j = 0; j < e->s.size(); ++j) {
+        for (size_t j = 0; j < e->s.size(); ++j) {
             if (atStartOfLine) {
                 if (e->s[j] == ' ')
                     curIndent++;
@@ -170,8 +168,8 @@ static Expr * stripIndentation(const Pos & pos, SymbolTable & symbols, vector<Ex
     /* Strip spaces from each line. */
     vector<Expr *> * es2 = new vector<Expr *>;
     atStartOfLine = true;
-    unsigned int curDropped = 0;
-    unsigned int n = es.size();
+    size_t curDropped = 0;
+    size_t n = es.size();
     for (vector<Expr *>::iterator i = es.begin(); i != es.end(); ++i, --n) {
         ExprIndStr * e = dynamic_cast<ExprIndStr *>(*i);
         if (!e) {
@@ -182,7 +180,7 @@ static Expr * stripIndentation(const Pos & pos, SymbolTable & symbols, vector<Ex
         }
 
         string s2;
-        for (unsigned int j = 0; j < e->s.size(); ++j) {
+        for (size_t j = 0; j < e->s.size(); ++j) {
             if (atStartOfLine) {
                 if (e->s[j] == ' ') {
                     if (curDropped++ >= minIndent)
@@ -275,11 +273,11 @@ void yyerror(YYLTYPE * loc, yyscan_t scanner, ParseData * data, const char * err
 %token IND_STRING_OPEN IND_STRING_CLOSE
 %token ELLIPSIS
 
-%nonassoc IMPL
+%right IMPL
 %left OR
 %left AND
 %nonassoc EQ NEQ
-%left '<' '>' LEQ GEQ
+%nonassoc '<' '>' LEQ GEQ
 %right UPDATE
 %left NOT
 %left '+' '-'
@@ -541,12 +539,7 @@ Expr * EvalState::parse(const char * text,
     int res = yyparse(scanner, &data);
     yylex_destroy(scanner);
 
-    if (res) {
-      if (data.atEnd)
-        throw IncompleteParseError(data.error);
-      else
-        throw ParseError(data.error);
-    }
+    if (res) throw ParseError(data.error);
 
     data.result->bindVars(staticEnv);
 

src/libexpr/primops.cc

@@ -73,7 +73,7 @@ void EvalState::realiseContext(const PathSet & context)
 
     if (drvs.empty()) return;
 
-    if (!settings.enableImportFromDerivation)
+    if (!evalSettings.enableImportFromDerivation)
         throw EvalError(format("attempted to realize '%1%' during evaluation but 'allow-import-from-derivation' is false") % *(drvs.begin()));
 
     /* For performance, prefetch all substitute info. */
@@ -155,7 +155,7 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args
 extern "C" typedef void (*ValueInitializer)(EvalState & state, Value & v);
 
 /* Load a ValueInitializer from a DSO and return whatever it initializes */
-static void prim_importNative(EvalState & state, const Pos & pos, Value * * args, Value & v)
+void prim_importNative(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     PathSet context;
     Path path = state.coerceToPath(pos, *args[0], context);
@@ -193,7 +193,7 @@ static void prim_importNative(EvalState & state, const Pos & pos, Value * * args
 
 
 /* Execute a program and parse its output */
-static void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v)
+void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     state.forceList(*args[0], pos);
     auto elems = args[0]->listElems();
@@ -271,7 +271,18 @@ static void prim_isNull(EvalState & state, const Pos & pos, Value * * args, Valu
 static void prim_isFunction(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     state.forceValue(*args[0]);
-    mkBool(v, args[0]->type == tLambda);
+    bool res;
+    switch (args[0]->type) {
+        case tLambda:
+        case tPrimOp:
+        case tPrimOpApp:
+            res = true;
+            break;
+        default:
+            res = false;
+            break;
+    }
+    mkBool(v, res);
 }
 
 
@@ -304,6 +315,12 @@ static void prim_isBool(EvalState & state, const Pos & pos, Value * * args, Valu
     mkBool(v, args[0]->type == tBool);
 }
 
+/* Determine whether the argument is a path. */
+static void prim_isPath(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    state.forceValue(*args[0]);
+    mkBool(v, args[0]->type == tPath);
+}
 
 struct CompareValues
 {
@@ -324,6 +341,8 @@ struct CompareValues
                 return strcmp(v1->string.s, v2->string.s) < 0;
             case tPath:
                 return strcmp(v1->path, v2->path) < 0;
+            case tAttrs:
+                return v1 < v2; // FIXME
             default:
                 throw EvalError(format("cannot compare %1% with %2%") % showType(*v1) % showType(*v2));
         }
@@ -453,7 +472,7 @@ static void prim_tryEval(EvalState & state, const Pos & pos, Value * * args, Val
 static void prim_getEnv(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     string name = state.forceStringNoCtx(*args[0], pos);
-    mkString(v, settings.restrictEval || settings.pureEval ? "" : getEnv(name));
+    mkString(v, evalSettings.restrictEval || evalSettings.pureEval ? "" : getEnv(name));
 }
 
 
@@ -544,7 +563,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
 
     PathSet context;
 
-    std::experimental::optional<std::string> outputHash;
+    std::optional<std::string> outputHash;
     std::string outputHashAlgo;
     bool outputHashRecursive = false;
 
@@ -676,21 +695,12 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
             }
         }
 
-        /* See prim_unsafeDiscardOutputDependency. */
-        else if (path.at(0) == '~')
-            drv.inputSrcs.insert(string(path, 1));
-
         /* Handle derivation outputs of the form ‘!<name>!<path>’. */
         else if (path.at(0) == '!') {
             std::pair<string, string> ctx = decodeContext(path);
             drv.inputDrvs[ctx.first].insert(ctx.second);
         }
 
-        /* Handle derivation contexts returned by
-           ‘builtins.storePath’. */
-        else if (isDerivation(path))
-            drv.inputDrvs[path] = state.store->queryDerivationOutputNames(path);
-
         /* Otherwise it's a source file. */
         else
             drv.inputSrcs.insert(path);
@@ -713,16 +723,14 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
         if (outputs.size() != 1 || *(outputs.begin()) != "out")
             throw Error(format("multiple outputs are not supported in fixed-output derivations, at %1%") % posDrvName);
 
-        HashType ht = parseHashType(outputHashAlgo);
-        if (ht == htUnknown)
-            throw EvalError(format("unknown hash algorithm '%1%', at %2%") % outputHashAlgo % posDrvName);
+        HashType ht = outputHashAlgo.empty() ? htUnknown : parseHashType(outputHashAlgo);
         Hash h(*outputHash, ht);
-        outputHash = h.to_string(Base16, false);
-        if (outputHashRecursive) outputHashAlgo = "r:" + outputHashAlgo;
 
         Path outPath = state.store->makeFixedOutputPath(outputHashRecursive, h, drvName);
         if (!jsonObject) drv.env["out"] = outPath;
-        drv.outputs["out"] = DerivationOutput(outPath, outputHashAlgo, *outputHash);
+        drv.outputs["out"] = DerivationOutput(outPath,
+            (outputHashRecursive ? "r:" : "") + printHashType(h.type),
+            h.to_string(Base16, false));
     }
 
     else {
@@ -855,7 +863,7 @@ static void prim_baseNameOf(EvalState & state, const Pos & pos, Value * * args,
 static void prim_dirOf(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     PathSet context;
-    Path dir = dirOf(state.coerceToPath(pos, *args[0], context));
+    Path dir = dirOf(state.coerceToString(pos, *args[0], context, false, false));
     if (args[0]->type == tPath) mkPath(v, dir.c_str()); else mkString(v, dir, context);
 }
 
@@ -995,13 +1003,8 @@ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Valu
     PathSet refs;
 
     for (auto path : context) {
-        if (path.at(0) == '=') path = string(path, 1);
-        if (isDerivation(path)) {
-            /* See prim_unsafeDiscardOutputDependency. */
-            if (path.at(0) != '~')
-                throw EvalError(format("in 'toFile': the file '%1%' cannot refer to derivation outputs, at %2%") % name % pos);
-            path = string(path, 1);
-        }
+        if (path.at(0) != '/')
+            throw EvalError(format("in 'toFile': the file '%1%' cannot refer to derivation outputs, at %2%") % name % pos);
         refs.insert(path);
     }
 
@@ -1020,7 +1023,7 @@ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Valu
 static void addPath(EvalState & state, const Pos & pos, const string & name, const Path & path_,
     Value * filterFun, bool recursive, const Hash & expectedHash, Value & v)
 {
-    const auto path = settings.pureEval && expectedHash ?
+    const auto path = evalSettings.pureEval && expectedHash ?
         path_ :
         state.checkSourcePath(path_);
     PathFilter filter = filterFun ? ([&](const Path & path) {
@@ -1345,6 +1348,24 @@ static void prim_functionArgs(EvalState & state, const Pos & pos, Value * * args
 }
 
 
+/* Apply a function to every element of an attribute set. */
+static void prim_mapAttrs(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    state.forceAttrs(*args[1], pos);
+
+    state.mkAttrs(v, args[1]->attrs->size());
+
+    for (auto & i : *args[1]->attrs) {
+        Value * vName = state.allocValue();
+        Value * vFun2 = state.allocValue();
+        mkString(*vName, i.name);
+        mkApp(*vFun2, *args[0], *vName);
+        mkApp(*state.allocAttr(v, i.name), *vFun2, *i.value);
+    }
+}
+
+
+
 /*************************************************************
  * Lists
  *************************************************************/
@@ -1399,7 +1420,6 @@ static void prim_tail(EvalState & state, const Pos & pos, Value * * args, Value
 /* Apply a function to every element of a list. */
 static void prim_map(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
-    state.forceFunction(*args[0], pos);
     state.forceList(*args[1], pos);
 
     state.mkList(v, args[1]->listSize());
@@ -1478,19 +1498,20 @@ static void prim_foldlStrict(EvalState & state, const Pos & pos, Value * * args,
     state.forceFunction(*args[0], pos);
     state.forceList(*args[2], pos);
 
-    Value * vCur = args[1];
+    if (args[2]->listSize()) {
+        Value * vCur = args[1];
 
-    if (args[2]->listSize())
         for (unsigned int n = 0; n < args[2]->listSize(); ++n) {
             Value vTmp;
             state.callFunction(*args[0], *vCur, vTmp, pos);
             vCur = n == args[2]->listSize() - 1 ? &v : state.allocValue();
             state.callFunction(vTmp, *args[2]->listElems()[n], *vCur, pos);
         }
-    else
-        v = *vCur;
-
-    state.forceValue(v);
+        state.forceValue(v);
+    } else {
+        state.forceValue(*args[1]);
+        v = *args[1];
+    }
 }
 
 
@@ -1527,7 +1548,6 @@ static void prim_all(EvalState & state, const Pos & pos, Value * * args, Value &
 
 static void prim_genList(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
-    state.forceFunction(*args[0], pos);
     auto len = state.forceInt(*args[1], pos);
 
     if (len < 0)
@@ -1558,7 +1578,6 @@ static void prim_sort(EvalState & state, const Pos & pos, Value * * args, Value
         v.listElems()[n] = args[1]->listElems()[n];
     }
 
-
     auto comparator = [&](Value * a, Value * b) {
         /* Optimization: if the comparator is lessThan, bypass
            callFunction. */
@@ -1578,6 +1597,62 @@ static void prim_sort(EvalState & state, const Pos & pos, Value * * args, Value
 }
 
 
+static void prim_unique(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    state.forceFunction(*args[0], pos);
+    state.forceList(*args[1], pos);
+
+    auto len = args[1]->listSize();
+
+    Value * vs[len];
+    for (size_t n = 0; n < len; ++n) {
+        auto v2 = args[1]->listElems()[n];
+        state.forceValue(*v2);
+        vs[n] = v2;
+    }
+
+    auto comparator = [&](Value * a, Value * b) {
+        /* Optimization: if the comparator is lessThan, bypass
+           callFunction. */
+        if (args[0]->type == tPrimOp && args[0]->primOp->fun == prim_lessThan)
+            return CompareValues()(a, b);
+
+        Value vTmp1, vTmp2;
+        state.callFunction(*args[0], *a, vTmp1, pos);
+        state.callFunction(vTmp1, *b, vTmp2, pos);
+        return state.forceBool(vTmp2, pos);
+    };
+
+    std::sort(vs, vs + len, comparator);
+
+    auto outLen = std::min((size_t) 1, len);
+    for (size_t n = 1; n < len; n++) {
+        if (comparator(vs[outLen - 1], vs[n])) {
+            vs[outLen++] = vs[n];
+        }
+    }
+
+    std::sort(vs, vs + outLen);
+
+    state.mkList(v, outLen);
+
+    size_t outPos = 0;
+    for (size_t n = 0; n < len; n++) {
+        auto v2 = args[1]->listElems()[n];
+        auto i = std::lower_bound(vs, vs + outLen, v2);
+        if (*i == v2) {
+            // Remove v2 from the set by setting its least significant
+            // bit. (This doesn't change the sort order.)
+            *i = (Value *) (((ptrdiff_t) v2) | 1);
+            assert(outPos < outLen);
+            v.listElems()[outPos++] = v2;
+        }
+    }
+
+    assert(outPos == outLen);
+}
+
+
 static void prim_partition(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     state.forceFunction(*args[0], pos);
@@ -1601,17 +1676,50 @@ static void prim_partition(EvalState & state, const Pos & pos, Value * * args, V
     state.mkAttrs(v, 2);
 
     Value * vRight = state.allocAttr(v, state.sRight);
-    state.mkList(*vRight, right.size());
-    memcpy(vRight->listElems(), right.data(), sizeof(Value *) * right.size());
+    auto rsize = right.size();
+    state.mkList(*vRight, rsize);
+    if (rsize)
+        memcpy(vRight->listElems(), right.data(), sizeof(Value *) * rsize);
 
     Value * vWrong = state.allocAttr(v, state.sWrong);
-    state.mkList(*vWrong, wrong.size());
-    memcpy(vWrong->listElems(), wrong.data(), sizeof(Value *) * wrong.size());
+    auto wsize = wrong.size();
+    state.mkList(*vWrong, wsize);
+    if (wsize)
+        memcpy(vWrong->listElems(), wrong.data(), sizeof(Value *) * wsize);
 
     v.attrs->sort();
 }
 
 
+/* concatMap = f: list: concatLists (map f list); */
+/* C++-version is to avoid allocating `mkApp', call `f' eagerly */
+static void prim_concatMap(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    state.forceFunction(*args[0], pos);
+    state.forceList(*args[1], pos);
+    auto nrLists = args[1]->listSize();
+
+    Value lists[nrLists];
+    size_t len = 0;
+
+    for (unsigned int n = 0; n < nrLists; ++n) {
+        Value * vElem = args[1]->listElems()[n];
+        state.callFunction(*args[0], *vElem, lists[n], pos);
+        state.forceList(lists[n], pos);
+        len += lists[n].listSize();
+    }
+
+    state.mkList(v, len);
+    auto out = v.listElems();
+    for (unsigned int n = 0, pos = 0; n < nrLists; ++n) {
+        auto l = lists[n].listSize();
+        if (l)
+            memcpy(out + pos, lists[n].listElems(), l * sizeof(Value *));
+        pos += l;
+    }
+}
+
+
 /*************************************************************
  * Integer arithmetic
  *************************************************************/
@@ -1619,6 +1727,8 @@ static void prim_partition(EvalState & state, const Pos & pos, Value * * args, V
 
 static void prim_add(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
+    state.forceValue(*args[0], pos);
+    state.forceValue(*args[1], pos);
     if (args[0]->type == tFloat || args[1]->type == tFloat)
         mkFloat(v, state.forceFloat(*args[0], pos) + state.forceFloat(*args[1], pos));
     else
@@ -1628,6 +1738,8 @@ static void prim_add(EvalState & state, const Pos & pos, Value * * args, Value &
 
 static void prim_sub(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
+    state.forceValue(*args[0], pos);
+    state.forceValue(*args[1], pos);
     if (args[0]->type == tFloat || args[1]->type == tFloat)
         mkFloat(v, state.forceFloat(*args[0], pos) - state.forceFloat(*args[1], pos));
     else
@@ -1637,6 +1749,8 @@ static void prim_sub(EvalState & state, const Pos & pos, Value * * args, Value &
 
 static void prim_mul(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
+    state.forceValue(*args[0], pos);
+    state.forceValue(*args[1], pos);
     if (args[0]->type == tFloat || args[1]->type == tFloat)
         mkFloat(v, state.forceFloat(*args[0], pos) * state.forceFloat(*args[1], pos));
     else
@@ -1646,6 +1760,9 @@ static void prim_mul(EvalState & state, const Pos & pos, Value * * args, Value &
 
 static void prim_div(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
+    state.forceValue(*args[0], pos);
+    state.forceValue(*args[1], pos);
+
     NixFloat f2 = state.forceFloat(*args[1], pos);
     if (f2 == 0) throw EvalError(format("division by zero, at %1%") % pos);
 
@@ -1661,6 +1778,20 @@ static void prim_div(EvalState & state, const Pos & pos, Value * * args, Value &
     }
 }
 
+static void prim_bitAnd(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    mkInt(v, state.forceInt(*args[0], pos) & state.forceInt(*args[1], pos));
+}
+
+static void prim_bitOr(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    mkInt(v, state.forceInt(*args[0], pos) | state.forceInt(*args[1], pos));
+}
+
+static void prim_bitXor(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    mkInt(v, state.forceInt(*args[0], pos) ^ state.forceInt(*args[1], pos));
+}
 
 static void prim_lessThan(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
@@ -1712,41 +1843,6 @@ static void prim_stringLength(EvalState & state, const Pos & pos, Value * * args
 }
 
 
-static void prim_unsafeDiscardStringContext(EvalState & state, const Pos & pos, Value * * args, Value & v)
-{
-    PathSet context;
-    string s = state.coerceToString(pos, *args[0], context);
-    mkString(v, s, PathSet());
-}
-
-
-static void prim_hasContext(EvalState & state, const Pos & pos, Value * * args, Value & v)
-{
-    PathSet context;
-    state.forceString(*args[0], context, pos);
-    mkBool(v, !context.empty());
-}
-
-
-/* Sometimes we want to pass a derivation path (i.e. pkg.drvPath) to a
-   builder without causing the derivation to be built (for instance,
-   in the derivation that builds NARs in nix-push, when doing
-   source-only deployment).  This primop marks the string context so
-   that builtins.derivation adds the path to drv.inputSrcs rather than
-   drv.inputDrvs. */
-static void prim_unsafeDiscardOutputDependency(EvalState & state, const Pos & pos, Value * * args, Value & v)
-{
-    PathSet context;
-    string s = state.coerceToString(pos, *args[0], context);
-
-    PathSet context2;
-    for (auto & p : context)
-        context2.insert(p.at(0) == '=' ? "~" + string(p, 1) : p);
-
-    mkString(v, s, context2);
-}
-
-
 /* Return the cryptographic hash of a string in base-16. */
 static void prim_hashString(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
@@ -1913,21 +2009,32 @@ static void prim_replaceStrings(EvalState & state, const Pos & pos, Value * * ar
     auto s = state.forceString(*args[2], context, pos);
 
     string res;
-    for (size_t p = 0; p < s.size(); ) {
+    // Loops one past last character to handle the case where 'from' contains an empty string.
+    for (size_t p = 0; p <= s.size(); ) {
         bool found = false;
         auto i = from.begin();
         auto j = to.begin();
         for (; i != from.end(); ++i, ++j)
             if (s.compare(p, i->size(), *i) == 0) {
                 found = true;
-                p += i->size();
                 res += j->first;
+                if (i->empty()) {
+                    if (p < s.size())
+                        res += s[p];
+                    p++;
+                } else {
+                    p += i->size();
+                }
                 for (auto& path : j->second)
                     context.insert(path);
                 j->second.clear();
                 break;
             }
-        if (!found) res += s[p++];
+        if (!found) {
+            if (p < s.size())
+                res += s[p];
+            p++;
+        }
     }
 
     mkString(v, res, context);
@@ -2016,7 +2123,7 @@ void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
 
     state.checkURI(url);
 
-    if (settings.pureEval && !expectedHash)
+    if (evalSettings.pureEval && !expectedHash)
         throw Error("in pure evaluation mode, '%s' requires a 'sha256' argument", who);
 
     Path res = getDownloader()->downloadCached(state.store, url, unpack, name, expectedHash);
@@ -2084,12 +2191,12 @@ void EvalState::createBaseEnv()
         addConstant(name, v);
     };
 
-    if (!settings.pureEval) {
+    if (!evalSettings.pureEval) {
         mkInt(v, time(0));
         addConstant("__currentTime", v);
     }
 
-    if (!settings.pureEval) {
+    if (!evalSettings.pureEval) {
         mkString(v, settings.thisSystem);
         addConstant("__currentSystem", v);
     }
@@ -2114,7 +2221,7 @@ void EvalState::createBaseEnv()
     mkApp(v, *vScopedImport, *v2);
     forceValue(v);
     addConstant("import", v);
-    if (settings.enableNativeCode) {
+    if (evalSettings.enableNativeCode) {
         addPrimOp("__importNative", 2, prim_importNative);
         addPrimOp("__exec", 1, prim_exec);
     }
@@ -2125,6 +2232,7 @@ void EvalState::createBaseEnv()
     addPrimOp("__isInt", 1, prim_isInt);
     addPrimOp("__isFloat", 1, prim_isFloat);
     addPrimOp("__isBool", 1, prim_isBool);
+    addPrimOp("__isPath", 1, prim_isPath);
     addPrimOp("__genericClosure", 1, prim_genericClosure);
     addPrimOp("abort", 1, prim_abort);
     addPrimOp("__addErrorContext", 2, prim_addErrorContext);
@@ -2141,7 +2249,7 @@ void EvalState::createBaseEnv()
 
     // Paths
     addPrimOp("__toPath", 1, prim_toPath);
-    if (settings.pureEval)
+    if (evalSettings.pureEval)
         addPurityError("__storePath");
     else
         addPrimOp("__storePath", 1, prim_storePath);
@@ -2172,6 +2280,7 @@ void EvalState::createBaseEnv()
     addPrimOp("__intersectAttrs", 2, prim_intersectAttrs);
     addPrimOp("__catAttrs", 2, prim_catAttrs);
     addPrimOp("__functionArgs", 1, prim_functionArgs);
+    addPrimOp("__mapAttrs", 2, prim_mapAttrs);
 
     // Lists
     addPrimOp("__isList", 1, prim_isList);
@@ -2188,22 +2297,24 @@ void EvalState::createBaseEnv()
     addPrimOp("__all", 2, prim_all);
     addPrimOp("__genList", 2, prim_genList);
     addPrimOp("__sort", 2, prim_sort);
+    addPrimOp("__unique", 2, prim_unique);
     addPrimOp("__partition", 2, prim_partition);
+    addPrimOp("__concatMap", 2, prim_concatMap);
 
     // Integer arithmetic
     addPrimOp("__add", 2, prim_add);
     addPrimOp("__sub", 2, prim_sub);
     addPrimOp("__mul", 2, prim_mul);
     addPrimOp("__div", 2, prim_div);
+    addPrimOp("__bitAnd", 2, prim_bitAnd);
+    addPrimOp("__bitOr", 2, prim_bitOr);
+    addPrimOp("__bitXor", 2, prim_bitXor);
     addPrimOp("__lessThan", 2, prim_lessThan);
 
     // String manipulation
     addPrimOp("toString", 1, prim_toString);
     addPrimOp("__substring", 3, prim_substring);
     addPrimOp("__stringLength", 1, prim_stringLength);
-    addPrimOp("__hasContext", 1, prim_hasContext);
-    addPrimOp("__unsafeDiscardStringContext", 1, prim_unsafeDiscardStringContext);
-    addPrimOp("__unsafeDiscardOutputDependency", 1, prim_unsafeDiscardOutputDependency);
     addPrimOp("__hashString", 2, prim_hashString);
     addPrimOp("__match", 2, prim_match);
     addPrimOp("__split", 2, prim_split);
@@ -2225,7 +2336,7 @@ void EvalState::createBaseEnv()
 
     /* Add a wrapper around the derivation primop that computes the
        `drvPath' and `outPath' attributes lazily. */
-    string path = settings.nixDataDir + "/nix/corepkgs/derivation.nix";
+    string path = canonPath(settings.nixDataDir + "/nix/corepkgs/derivation.nix", true);
     sDerivationNix = symbols.create(path);
     evalFile(path, v);
     addConstant("derivation", v);

src/libexpr/primops.hh

@@ -15,4 +15,12 @@ struct RegisterPrimOp
     RegisterPrimOp(std::string name, size_t arity, PrimOpFun fun);
 };
 
+/* These primops are disabled without enableNativeCode, but plugins
+   may wish to use them in limited contexts without globally enabling
+   them. */
+/* Load a ValueInitializer from a DSO and return whatever it initializes */
+void prim_importNative(EvalState & state, const Pos & pos, Value * * args, Value & v);
+/* Execute a program and parse its output */
+void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v);
+
 }

src/libexpr/primops/context.cc

@@ -0,0 +1,187 @@
+#include "primops.hh"
+#include "eval-inline.hh"
+#include "derivations.hh"
+
+namespace nix {
+
+static void prim_unsafeDiscardStringContext(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    PathSet context;
+    string s = state.coerceToString(pos, *args[0], context);
+    mkString(v, s, PathSet());
+}
+
+static RegisterPrimOp r1("__unsafeDiscardStringContext", 1, prim_unsafeDiscardStringContext);
+
+
+static void prim_hasContext(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    PathSet context;
+    state.forceString(*args[0], context, pos);
+    mkBool(v, !context.empty());
+}
+
+static RegisterPrimOp r2("__hasContext", 1, prim_hasContext);
+
+
+/* Sometimes we want to pass a derivation path (i.e. pkg.drvPath) to a
+   builder without causing the derivation to be built (for instance,
+   in the derivation that builds NARs in nix-push, when doing
+   source-only deployment).  This primop marks the string context so
+   that builtins.derivation adds the path to drv.inputSrcs rather than
+   drv.inputDrvs. */
+static void prim_unsafeDiscardOutputDependency(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    PathSet context;
+    string s = state.coerceToString(pos, *args[0], context);
+
+    PathSet context2;
+    for (auto & p : context)
+        context2.insert(p.at(0) == '=' ? string(p, 1) : p);
+
+    mkString(v, s, context2);
+}
+
+static RegisterPrimOp r3("__unsafeDiscardOutputDependency", 1, prim_unsafeDiscardOutputDependency);
+
+
+/* Extract the context of a string as a structured Nix value.
+
+   The context is represented as an attribute set whose keys are the
+   paths in the context set and whose values are attribute sets with
+   the following keys:
+     path: True if the relevant path is in the context as a plain store
+           path (i.e. the kind of context you get when interpolating
+           a Nix path (e.g. ./.) into a string). False if missing.
+     allOutputs: True if the relevant path is a derivation and it is
+                  in the context as a drv file with all of its outputs
+                  (i.e. the kind of context you get when referencing
+                  .drvPath of some derivation). False if missing.
+     outputs: If a non-empty list, the relevant path is a derivation
+              and the provided outputs are referenced in the context
+              (i.e. the kind of context you get when referencing
+              .outPath of some derivation). Empty list if missing.
+   Note that for a given path any combination of the above attributes
+   may be present.
+*/
+static void prim_getContext(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    struct ContextInfo {
+        bool path = false;
+        bool allOutputs = false;
+        Strings outputs;
+    };
+    PathSet context;
+    state.forceString(*args[0], context, pos);
+    auto contextInfos = std::map<Path, ContextInfo>();
+    for (const auto & p : context) {
+        Path drv;
+        string output;
+        const Path * path = &p;
+        if (p.at(0) == '=') {
+            drv = string(p, 1);
+            path = &drv;
+        } else if (p.at(0) == '!') {
+            std::pair<string, string> ctx = decodeContext(p);
+            drv = ctx.first;
+            output = ctx.second;
+            path = &drv;
+        }
+        auto isPath = drv.empty();
+        auto isAllOutputs = (!drv.empty()) && output.empty();
+
+        auto iter = contextInfos.find(*path);
+        if (iter == contextInfos.end()) {
+            contextInfos.emplace(*path, ContextInfo{isPath, isAllOutputs, output.empty() ? Strings{} : Strings{std::move(output)}});
+        } else {
+            if (isPath)
+                iter->second.path = true;
+            else if (isAllOutputs)
+                iter->second.allOutputs = true;
+            else
+                iter->second.outputs.emplace_back(std::move(output));
+        }
+    }
+
+    state.mkAttrs(v, contextInfos.size());
+
+    auto sPath = state.symbols.create("path");
+    auto sAllOutputs = state.symbols.create("allOutputs");
+    for (const auto & info : contextInfos) {
+        auto & infoVal = *state.allocAttr(v, state.symbols.create(info.first));
+        state.mkAttrs(infoVal, 3);
+        if (info.second.path)
+            mkBool(*state.allocAttr(infoVal, sPath), true);
+        if (info.second.allOutputs)
+            mkBool(*state.allocAttr(infoVal, sAllOutputs), true);
+        if (!info.second.outputs.empty()) {
+            auto & outputsVal = *state.allocAttr(infoVal, state.sOutputs);
+            state.mkList(outputsVal, info.second.outputs.size());
+            size_t i = 0;
+            for (const auto & output : info.second.outputs) {
+                mkString(*(outputsVal.listElems()[i++] = state.allocValue()), output);
+            }
+        }
+        infoVal.attrs->sort();
+    }
+    v.attrs->sort();
+}
+
+static RegisterPrimOp r4("__getContext", 1, prim_getContext);
+
+
+/* Append the given context to a given string.
+
+   See the commentary above unsafeGetContext for details of the
+   context representation.
+*/
+static void prim_appendContext(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    PathSet context;
+    auto orig = state.forceString(*args[0], context, pos);
+
+    state.forceAttrs(*args[1], pos);
+
+    auto sPath = state.symbols.create("path");
+    auto sAllOutputs = state.symbols.create("allOutputs");
+    for (auto & i : *args[1]->attrs) {
+        if (!state.store->isStorePath(i.name))
+            throw EvalError("Context key '%s' is not a store path, at %s", i.name, i.pos);
+        if (!settings.readOnlyMode)
+            state.store->ensurePath(i.name);
+        state.forceAttrs(*i.value, *i.pos);
+        auto iter = i.value->attrs->find(sPath);
+        if (iter != i.value->attrs->end()) {
+            if (state.forceBool(*iter->value, *iter->pos))
+                context.insert(i.name);
+        }
+
+        iter = i.value->attrs->find(sAllOutputs);
+        if (iter != i.value->attrs->end()) {
+            if (state.forceBool(*iter->value, *iter->pos)) {
+                if (!isDerivation(i.name)) {
+                    throw EvalError("Tried to add all-outputs context of %s, which is not a derivation, to a string, at %s", i.name, i.pos);
+                }
+                context.insert("=" + string(i.name));
+            }
+        }
+
+        iter = i.value->attrs->find(state.sOutputs);
+        if (iter != i.value->attrs->end()) {
+            state.forceList(*iter->value, *iter->pos);
+            if (iter->value->listSize() && !isDerivation(i.name)) {
+                throw EvalError("Tried to add derivation output context of %s, which is not a derivation, to a string, at %s", i.name, i.pos);
+            }
+            for (unsigned int n = 0; n < iter->value->listSize(); ++n) {
+                auto name = state.forceStringNoCtx(*iter->value->listElems()[n], *iter->pos);
+                context.insert("!" + name + "!" + string(i.name));
+            }
+        }
+    }
+
+    mkString(v, orig, context);
+}
+
+static RegisterPrimOp r5("__appendContext", 2, prim_appendContext);
+
+}

src/libexpr/primops/fetchGit.cc

@@ -3,6 +3,7 @@
 #include "download.hh"
 #include "store-api.hh"
 #include "pathlocks.hh"
+#include "hash.hh"
 
 #include <sys/time.h>
 
@@ -25,10 +26,10 @@ struct GitInfo
 std::regex revRegex("^[0-9a-fA-F]{40}$");
 
 GitInfo exportGit(ref<Store> store, const std::string & uri,
-    std::experimental::optional<std::string> ref, std::string rev,
+    std::optional<std::string> ref, std::string rev,
     const std::string & name)
 {
-    if (settings.pureEval && rev == "")
+    if (evalSettings.pureEval && rev == "")
         throw Error("in pure evaluation mode, 'fetchGit' requires a Git revision");
 
     if (!ref && rev == "" && hasPrefix(uri, "/") && pathExists(uri + "/.git")) {
@@ -79,20 +80,21 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
         ref = "HEAD"s;
     }
 
-    if (!ref) ref = "master"s;
+    if (!ref) ref = "HEAD"s;
 
     if (rev != "" && !std::regex_match(rev, revRegex))
         throw Error("invalid Git revision '%s'", rev);
 
-    Path cacheDir = getCacheDir() + "/nix/git";
+    deletePath(getCacheDir() + "/nix/git");
+
+    Path cacheDir = getCacheDir() + "/nix/gitv2/" + hashString(htSHA256, uri).to_string(Base32, false);
 
     if (!pathExists(cacheDir)) {
+        createDirs(dirOf(cacheDir));
         runProgram("git", true, { "init", "--bare", cacheDir });
     }
 
-    std::string localRef = hashString(htSHA256, fmt("%s-%s", uri, *ref)).to_string(Base32, false);
-
-    Path localRefFile = cacheDir + "/refs/heads/" + localRef;
+    Path localRefFile = cacheDir + "/refs/heads/" + *ref;
 
     bool doFetch;
     time_t now = time(0);
@@ -114,7 +116,7 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
            git fetch to update the local ref to the remote ref. */
         struct stat st;
         doFetch = stat(localRefFile.c_str(), &st) != 0 ||
-            st.st_mtime <= now - settings.tarballTtl;
+            st.st_mtime + settings.tarballTtl <= now;
     }
     if (doFetch)
     {
@@ -122,7 +124,7 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
 
         // FIXME: git stderr messes up our progress indicator, so
         // we're using --quiet for now. Should process its stderr.
-        runProgram("git", true, { "-C", cacheDir, "fetch", "--quiet", "--force", "--", uri, *ref + ":" + localRef });
+        runProgram("git", true, { "-C", cacheDir, "fetch", "--quiet", "--force", "--", uri, fmt("%s:%s", *ref, *ref) });
 
         struct timeval times[2];
         times[0].tv_sec = now;
@@ -138,7 +140,7 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
     gitInfo.rev = rev != "" ? rev : chomp(readFile(localRefFile));
     gitInfo.shortRev = std::string(gitInfo.rev, 0, 7);
 
-    printTalkative("using revision %s of repo '%s'", uri, gitInfo.rev);
+    printTalkative("using revision %s of repo '%s'", gitInfo.rev, uri);
 
     std::string storeLinkName = hashString(htSHA512, name + std::string("\0"s) + gitInfo.rev).to_string(Base32, false);
     Path storeLink = cacheDir + "/" + storeLinkName + ".link";
@@ -188,7 +190,7 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
 static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     std::string url;
-    std::experimental::optional<std::string> ref;
+    std::optional<std::string> ref;
     std::string rev;
     std::string name = "source";
     PathSet context;
@@ -219,8 +221,6 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va
     } else
         url = state.coerceToString(pos, *args[0], context, false, false);
 
-    if (!isUri(url)) url = absPath(url);
-
     // FIXME: git externals probably can be used to bypass the URI
     // whitelist. Ah well.
     state.checkURI(url);