Add memoise primop

'builtins.memoise f x' is equal to 'f x', but uses a cache to speed up
repeated invocations of 'f' with the same 'x'. A typical application
is memoising evaluations of Nixpkgs in NixOps network
specifications. For example, with the patch below, the time to
evaluate a 10-machine NixOps network is reduced from 17.1s to 9.6s,
while memory consumption goes from 4486 MiB to 3089 MiB. (This is with
GC_INITIAL_HEAP_SIZE=16G.)

Nixpkgs patch:

diff --git a/pkgs/top-level/impure.nix b/pkgs/top-level/impure.nix
index a9f21e45aed..f641067e022 100644
--- a/pkgs/top-level/impure.nix
+++ b/pkgs/top-level/impure.nix
@@ -79,7 +79,7 @@ in
 # not be passed.
 assert args ? localSystem -> !(args ? system || args ? platform);

-import ./. (builtins.removeAttrs args [ "system" "platform" ] // {
+builtins.memoise or (x: x) (import ./.) (builtins.removeAttrs args [ "system" "platform" ] // {
   inherit config overlays crossSystem;
   # Fallback: Assume we are building packages on the current (build, in GNU
   # Autotools parlance) system.

Makefile.config.in

@@ -6,6 +6,7 @@ CXXFLAGS = @CXXFLAGS@
 ENABLE_S3 = @ENABLE_S3@
 HAVE_SODIUM = @HAVE_SODIUM@
 HAVE_READLINE = @HAVE_READLINE@
+HAVE_BROTLI = @HAVE_BROTLI@
 LIBCURL_LIBS = @LIBCURL_LIBS@
 OPENSSL_LIBS = @OPENSSL_LIBS@
 PACKAGE_NAME = @PACKAGE_NAME@
@@ -13,9 +14,10 @@ PACKAGE_VERSION = @PACKAGE_VERSION@
 SODIUM_LIBS = @SODIUM_LIBS@
 LIBLZMA_LIBS = @LIBLZMA_LIBS@
 SQLITE3_LIBS = @SQLITE3_LIBS@
+LIBBROTLI_LIBS = @LIBBROTLI_LIBS@
 bash = @bash@
 bindir = @bindir@
-bro = @bro@
+brotli = @brotli@
 lsof = @lsof@
 datadir = @datadir@
 datarootdir = @datarootdir@

configure.ac

@@ -127,7 +127,7 @@ NEED_PROG(gzip, gzip)
 NEED_PROG(xz, xz)
 AC_PATH_PROG(dot, dot)
 AC_PATH_PROG(pv, pv, pv)
-AC_PATH_PROG(bro, bro, bro)
+AC_PATH_PROGS(brotli, brotli bro, bro)
 AC_PATH_PROG(lsof, lsof, lsof)
 
 
@@ -176,6 +176,13 @@ AC_SUBST(HAVE_SODIUM, [$have_sodium])
 PKG_CHECK_MODULES([LIBLZMA], [liblzma], [CXXFLAGS="$LIBLZMA_CFLAGS $CXXFLAGS"])
 
 
+# Look for libbrotli{enc,dec}, optional dependencies
+PKG_CHECK_MODULES([LIBBROTLI], [libbrotlienc libbrotlidec],
+  [AC_DEFINE([HAVE_BROTLI], [1], [Whether to use libbrotli.])
+   CXXFLAGS="$LIBBROTLI_CFLAGS $CXXFLAGS"]
+   have_brotli=1], [have_brotli=])
+AC_SUBST(HAVE_BROTLI, [$have_brotli])
+
 # Look for libseccomp, required for Linux sandboxing.
 if test "$sys_name" = linux; then
   PKG_CHECK_MODULES([LIBSECCOMP], [libseccomp],
@@ -186,11 +193,17 @@ fi
 # Look for aws-cpp-sdk-s3.
 AC_LANG_PUSH(C++)
 AC_CHECK_HEADERS([aws/s3/S3Client.h],
-  [AC_DEFINE([ENABLE_S3], [1], [Whether to enable S3 support via aws-cpp-sdk-s3.])
+  [AC_DEFINE([ENABLE_S3], [1], [Whether to enable S3 support via aws-sdk-cpp.])
   enable_s3=1], [enable_s3=])
 AC_SUBST(ENABLE_S3, [$enable_s3])
 AC_LANG_POP(C++)
 
+if test -n "$enable_s3"; then
+  declare -a aws_version_tokens=($(printf '#include <aws/core/VersionConfig.h>\nAWS_SDK_VERSION_STRING' | cpp -E | grep -v '^#.*' | sed 's/"//g' | tr '.' ' '))
+  AC_DEFINE_UNQUOTED([AWS_VERSION_MAJOR], ${aws_version_tokens@<:@0@:>@}, [Major version of aws-sdk-cpp.])
+  AC_DEFINE_UNQUOTED([AWS_VERSION_MINOR], ${aws_version_tokens@<:@1@:>@}, [Minor version of aws-sdk-cpp.])
+fi
+
 
 # Whether to use the Boehm garbage collector.
 AC_ARG_ENABLE(gc, AC_HELP_STRING([--enable-gc],

doc/manual/command-ref/conf-file.xml

@@ -312,7 +312,7 @@ false</literal>.</para>
   </varlistentry>
 
 
-  <varlistentry><term><literal>use-substitutes</literal></term>
+  <varlistentry><term><literal>substitute</literal></term>
 
     <listitem><para>If set to <literal>true</literal> (default), Nix
     will use binary substitutes if available.  This option can be
@@ -321,6 +321,20 @@ false</literal>.</para>
   </varlistentry>
 
 
+  <varlistentry><term><literal>builders-use-substitutes</literal></term>
+
+    <listitem><para>If set to <literal>true</literal>, Nix will instruct
+    remote build machines to use their own binary substitutes if available. In
+    practical terms, this means that remote hosts will fetch as many build
+    dependencies as possible from their own substitutes (e.g, from
+    <literal>cache.nixos.org</literal>), instead of waiting for this host to
+    upload them all. This can drastically reduce build times if the network
+    connection between this computer and the remote build host is slow. Defaults
+    to <literal>false</literal>.</para></listitem>
+
+  </varlistentry>
+
+
   <varlistentry><term><literal>fallback</literal></term>
 
     <listitem><para>If set to <literal>true</literal>, Nix will fall

local.mk

@@ -1,6 +1,5 @@
 ifeq ($(MAKECMDGOALS), dist)
-  # Make sure we are in repo root with `--git-dir`
-  dist-files += $(shell git --git-dir=.git ls-files || find * -type f)
+  dist-files += $(shell cat .dist-files)
 endif
 
 dist-files += configure config.h.in nix.spec perl/configure

release.nix

@@ -1,12 +1,12 @@
-{ nix ? { outPath = ./.; revCount = 1234; shortRev = "abcdef"; }
-, nixpkgs ? { outPath = <nixpkgs>; revCount = 1234; shortRev = "abcdef"; }
+{ nix ? builtins.fetchGit ./.
+, nixpkgs ? fetchTarball channel:nixos-17.09
 , officialRelease ? false
 , systems ? [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ]
 }:
 
 let
 
-  pkgs = import <nixpkgs> {};
+  pkgs = import nixpkgs {};
 
   jobs = rec {
 
@@ -27,16 +27,13 @@ let
             pkgconfig sqlite libsodium boehmgc
             docbook5 docbook5_xsl
             autoconf-archive
-            git
           ] ++ lib.optional stdenv.isLinux libseccomp;
 
         configureFlags = "--enable-gc";
 
         postUnpack = ''
-          # Clean up when building from a working tree.
-          if [[ -d $sourceRoot/.git ]]; then
-            git -C $sourceRoot clean -fd
-          fi
+          (cd source && find . -type f) | cut -c3- > source/.dist-files
+          cat source/.dist-files
         '';
 
         preConfigure = ''
@@ -62,7 +59,7 @@ let
 
     build = pkgs.lib.genAttrs systems (system:
 
-      with import <nixpkgs> { inherit system; };
+      with import nixpkgs { inherit system; };
 
       with import ./release-common.nix { inherit pkgs; };
 
@@ -105,7 +102,7 @@ let
 
     perlBindings = pkgs.lib.genAttrs systems (system:
 
-      let pkgs = import <nixpkgs> { inherit system; }; in with pkgs;
+      let pkgs = import nixpkgs { inherit system; }; in with pkgs;
 
       releaseTools.nixBuild {
         name = "nix-perl";
@@ -131,7 +128,7 @@ let
     binaryTarball = pkgs.lib.genAttrs systems (system:
 
       # FIXME: temporarily use a different branch for the Darwin build.
-      with import <nixpkgs> { inherit system; };
+      with import nixpkgs { inherit system; };
 
       let
         toplevel = builtins.getAttr system jobs.build;
@@ -174,7 +171,7 @@ let
 
 
     coverage =
-      with import <nixpkgs> { system = "x86_64-linux"; };
+      with import nixpkgs { system = "x86_64-linux"; };
 
       releaseTools.coverageAnalysis {
         name = "nix-build";
@@ -218,20 +215,23 @@ let
 
     # System tests.
     tests.remoteBuilds = (import ./tests/remote-builds.nix rec {
+      inherit nixpkgs;
       nix = build.x86_64-linux; system = "x86_64-linux";
     });
 
     tests.nix-copy-closure = (import ./tests/nix-copy-closure.nix rec {
+      inherit nixpkgs;
       nix = build.x86_64-linux; system = "x86_64-linux";
     });
 
     tests.setuid = pkgs.lib.genAttrs (pkgs.lib.filter (pkgs.lib.hasSuffix "-linux") systems) (system:
       import ./tests/setuid.nix rec {
+        inherit nixpkgs;
         nix = build.${system}; inherit system;
       });
 
     tests.binaryTarball =
-      with import <nixpkgs> { system = "x86_64-linux"; };
+      with import nixpkgs { system = "x86_64-linux"; };
       vmTools.runInLinuxImage (runCommand "nix-binary-tarball-test"
         { diskImage = vmTools.diskImages.ubuntu1204x86_64;
         }
@@ -250,7 +250,7 @@ let
         ''); # */
 
     tests.evalNixpkgs =
-      import <nixpkgs/pkgs/top-level/make-tarball.nix> {
+      import (nixpkgs + "/pkgs/top-level/make-tarball.nix") {
         inherit nixpkgs;
         inherit pkgs;
         nix = build.x86_64-linux;
@@ -304,7 +304,7 @@ let
   makeRPM =
     system: diskImageFun: extraPackages:
 
-    with import <nixpkgs> { inherit system; };
+    with import nixpkgs { inherit system; };
 
     releaseTools.rpmBuild rec {
       name = "nix-rpm";
@@ -313,7 +313,8 @@ let
         { extraPackages =
             [ "sqlite" "sqlite-devel" "bzip2-devel" "libcurl-devel" "openssl-devel" "xz-devel" "libseccomp-devel" ]
             ++ extraPackages; };
-      memSize = 1024;
+      # At most 2047MB can be simulated in qemu-system-i386
+      memSize = 2047;
       meta.schedulingPriority = 50;
       postRPMInstall = "cd /tmp/rpmout/BUILD/nix-* && make installcheck";
       #enableParallelBuilding = true;
@@ -326,7 +327,7 @@ let
   makeDeb =
     system: diskImageFun: extraPackages: extraDebPackages:
 
-    with import <nixpkgs> { inherit system; };
+    with import nixpkgs { inherit system; };
 
     releaseTools.debBuild {
       name = "nix-deb";

scripts/install-darwin-multi-user.sh

@@ -33,7 +33,7 @@ readonly NIX_FIRST_BUILD_UID="30001"
 readonly NIX_ROOT="/nix"
 readonly PLIST_DEST=/Library/LaunchDaemons/org.nixos.nix-daemon.plist
 
-readonly PROFILE_TARGETS=("/etc/profile" "/etc/bashrc" "/etc/zshrc")
+readonly PROFILE_TARGETS=("/etc/bashrc" "/etc/zshrc")
 readonly PROFILE_BACKUP_SUFFIX=".backup-before-nix"
 readonly PROFILE_NIX_FILE="$NIX_ROOT/var/nix/profiles/default/etc/profile.d/nix-daemon.sh"
 

scripts/nix-profile.sh.in

@@ -60,12 +60,6 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then
     # This part should be kept in sync with nixpkgs:nixos/modules/programs/environment.nix
     NIX_PROFILES="@localstatedir@/nix/profiles/default $NIX_USER_PROFILE_DIR"
 
-    for i in $NIX_PROFILES; do
-        if [ -d "$i/lib/aspell" ]; then
-            export ASPELL_CONF="dict-dir $i/lib/aspell"
-        fi
-    done
-
     # Set $NIX_SSL_CERT_FILE so that Nixpkgs applications like curl work.
     if [ -e /etc/ssl/certs/ca-certificates.crt ]; then # NixOS, Ubuntu, Debian, Gentoo, Arch
         export NIX_SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt

src/build-remote/build-remote.cc

@@ -218,9 +218,11 @@ connected:
             signal(SIGALRM, old);
         }
 
+        auto substitute = settings.buildersUseSubstitutes ? Substitute : NoSubstitute;
+
         {
             Activity act(*logger, lvlTalkative, actUnknown, fmt("copying dependencies to '%s'", storeUri));
-            copyPaths(store, ref<Store>(sshStore), inputs, NoRepair, NoCheckSigs);
+            copyPaths(store, ref<Store>(sshStore), inputs, NoRepair, NoCheckSigs, substitute);
         }
 
         uploadLock = -1;
@@ -240,7 +242,7 @@ connected:
         if (!missing.empty()) {
             Activity act(*logger, lvlTalkative, actUnknown, fmt("copying outputs from '%s'", storeUri));
             setenv("NIX_HELD_LOCKS", concatStringsSep(" ", missing).c_str(), 1); /* FIXME: ugly */
-            copyPaths(ref<Store>(sshStore), store, missing, NoRepair, NoCheckSigs);
+            copyPaths(ref<Store>(sshStore), store, missing, NoRepair, NoCheckSigs, substitute);
         }
 
         return;

src/libexpr/eval.cc

@@ -1683,6 +1683,8 @@ void EvalState::printStats()
     printMsg(v, format("  number of primop calls: %1%") % nrPrimOpCalls);
     printMsg(v, format("  number of function calls: %1%") % nrFunctionCalls);
     printMsg(v, format("  total allocations: %1% bytes") % (bEnvs + bLists + bValues + bAttrsets));
+    printMsg(v, format("  memoisation hits: %d") % nrMemoiseHits);
+    printMsg(v, format("  memoisation misses: %d") % nrMemoiseMisses);
 
 #if HAVE_BOEHMGC
     GC_word heapSize, totalBytes;

src/libexpr/eval.hh

@@ -89,7 +89,7 @@ private:
 
     /* A cache from path names to values. */
 #if HAVE_BOEHMGC
-    typedef std::map<Path, Value, std::less<Path>, traceable_allocator<std::pair<const Path, Value> > > FileEvalCache;
+    typedef std::map<Path, Value, std::less<Path>, traceable_allocator<std::pair<const Path, Value>>> FileEvalCache;
 #else
     typedef std::map<Path, Value> FileEvalCache;
 #endif
@@ -269,6 +269,8 @@ private:
     unsigned long nrListConcats = 0;
     unsigned long nrPrimOpCalls = 0;
     unsigned long nrFunctionCalls = 0;
+    unsigned long nrMemoiseHits = 0;
+    unsigned long nrMemoiseMisses = 0;
 
     bool countCalls;
 
@@ -287,6 +289,22 @@ private:
     friend struct ExprOpConcatLists;
     friend struct ExprSelect;
     friend void prim_getAttr(EvalState & state, const Pos & pos, Value * * args, Value & v);
+    friend void prim_memoise(EvalState & state, const Pos & pos, Value * * args, Value & v);
+
+    /* State for builtins.memoise. */
+    struct MemoArgComparator
+    {
+        EvalState & state;
+        MemoArgComparator(EvalState & state) : state(state) { }
+        bool operator()(Value * a, Value * b);
+    };
+
+    typedef std::map<Value *, Value, MemoArgComparator, traceable_allocator<std::pair<const Value *, Value>>> PerLambdaMemo;
+
+    typedef std::pair<Env *, ExprLambda *> LambdaKey;
+
+    // FIXME: use std::unordered_map
+    std::map<LambdaKey, PerLambdaMemo, std::less<LambdaKey>, traceable_allocator<std::pair<const LambdaKey, PerLambdaMemo>>> memos;
 };
 
 

src/libexpr/primops.cc

@@ -1653,6 +1653,14 @@ static void prim_unsafeDiscardStringContext(EvalState & state, const Pos & pos,
 }
 
 
+static void prim_hasContext(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    PathSet context;
+    state.forceString(*args[0], context, pos);
+    mkBool(v, !context.empty());
+}
+
+
 /* Sometimes we want to pass a derivation path (i.e. pkg.drvPath) to a
    builder without causing the derivation to be built (for instance,
    in the derivation that builds NARs in nix-push, when doing
@@ -2083,6 +2091,7 @@ void EvalState::createBaseEnv()
     addPrimOp("toString", 1, prim_toString);
     addPrimOp("__substring", 3, prim_substring);
     addPrimOp("__stringLength", 1, prim_stringLength);
+    addPrimOp("__hasContext", 1, prim_hasContext);
     addPrimOp("__unsafeDiscardStringContext", 1, prim_unsafeDiscardStringContext);
     addPrimOp("__unsafeDiscardOutputDependency", 1, prim_unsafeDiscardOutputDependency);
     addPrimOp("__hashString", 2, prim_hashString);

src/libexpr/primops/fetchGit.cc

@@ -23,7 +23,7 @@ struct GitInfo
 };
 
 GitInfo exportGit(ref<Store> store, const std::string & uri,
-    std::experimental::optional<std::string> ref, const std::string & rev,
+    std::experimental::optional<std::string> ref, std::string rev,
     const std::string & name)
 {
     if (!ref && rev == "" && hasPrefix(uri, "/") && pathExists(uri + "/.git")) {
@@ -68,6 +68,10 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
 
             return gitInfo;
         }
+
+        // clean working tree, but no ref or rev specified.  Use 'HEAD'.
+        rev = chomp(runProgram("git", true, { "-C", uri, "rev-parse", "HEAD" }));
+        ref = "HEAD"s;
     }
 
     if (!ref) ref = "master"s;
@@ -81,7 +85,6 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
     Path cacheDir = getCacheDir() + "/nix/git";
 
     if (!pathExists(cacheDir)) {
-        createDirs(cacheDir);
         runProgram("git", true, { "init", "--bare", cacheDir });
     }
 

src/libexpr/primops/memoise.cc

@@ -0,0 +1,95 @@
+#include "primops.hh"
+#include "eval-inline.hh"
+
+#include <cstring>
+
+namespace nix {
+
+bool EvalState::MemoArgComparator::operator()(Value * v1, Value * v2)
+{
+    if (v1 == v2) return false;
+
+    state.forceValue(*v1);
+    state.forceValue(*v2);
+
+    if (v1->type == v2->type) {
+        switch (v1->type) {
+
+        case tInt:
+            return v1->integer < v2->integer;
+
+        case tBool:
+            return v1->boolean < v2->boolean;
+
+        case tString:
+            return strcmp(v1->string.s, v2->string.s);
+
+        case tPath:
+            return strcmp(v1->path, v2->path);
+
+        case tNull:
+            return false;
+
+        case tList1:
+        case tList2:
+        case tListN:
+            unsigned int n;
+            for (n = 0; n < v1->listSize() && n < v2->listSize(); ++n) {
+                if ((*this)(v1->listElems()[n], v2->listElems()[n])) return true;
+                if ((*this)(v2->listElems()[n], v1->listElems()[n])) return false;
+            }
+
+            return n == v1->listSize() && n < v2->listSize();
+
+        case tAttrs:
+            Bindings::iterator i, j;
+            for (i = v1->attrs->begin(), j = v2->attrs->begin(); i != v1->attrs->end() && j != v2->attrs->end(); ++i, ++j) {
+                if (i->name < j->name) return true;
+                if (j->name < i->name) return false;
+                if ((*this)(i->value, j->value)) return true;
+                if ((*this)(j->value, i->value)) return false;
+            }
+            return i == v1->attrs->end() && j != v2->attrs->end();
+
+        case tLambda:
+            return std::make_pair(v1->lambda.env, v1->lambda.fun) < std::make_pair(v2->lambda.env, v2->lambda.fun);
+
+        case tFloat:
+            return v1->fpoint < v2->fpoint;
+
+        default:
+            break;
+        }
+    }
+
+    // As a fallback, use pointer equality.
+    return v1 < v2;
+}
+
+void prim_memoise(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    state.forceFunction(*args[0], pos);
+
+    EvalState::PerLambdaMemo foo(state);
+
+    auto & memo = state.memos.emplace(std::make_pair(args[0]->lambda.env, args[0]->lambda.fun), state).first->second;
+
+    auto result = memo.find(args[1]);
+
+    if (result != memo.end()) {
+        state.nrMemoiseHits++;
+        v = result->second;
+        return;
+    }
+
+    state.nrMemoiseMisses++;
+
+    state.callFunction(*args[0], *args[1], v, pos);
+
+    memo[args[1]] = v;
+
+}
+
+static RegisterPrimOp r("memoise", 2, prim_memoise);
+
+}

src/libstore/globals.hh

@@ -138,6 +138,11 @@ public:
     Setting<std::string> builders{this, "@" + nixConfDir + "/machines", "builders",
         "A semicolon-separated list of build machines, in the format of nix.machines."};
 
+    Setting<bool> buildersUseSubstitutes{this, false, "builders-use-substitutes",
+        "Whether build machines should use their own substitutes for obtaining "
+        "build dependencies if possible, rather than waiting for this host to "
+        "upload them."};
+
     Setting<off_t> reservedSize{this, 8 * 1024 * 1024, "gc-reserved-space",
         "Amount of reserved disk space for the garbage collector."};
 
@@ -150,7 +155,7 @@ public:
     Setting<bool> syncBeforeRegistering{this, false, "sync-before-registering",
         "Whether to call sync() before registering a path as valid."};
 
-    Setting<bool> useSubstitutes{this, true, "use-substitutes",
+    Setting<bool> useSubstitutes{this, true, "substitute",
         "Whether to use substitutes.",
         {"build-use-substitutes"}};
 

src/libstore/s3-binary-cache-store.cc

@@ -10,6 +10,7 @@
 #include "istringstream_nocopy.hh"
 
 #include <aws/core/Aws.h>
+#include <aws/core/VersionConfig.h>
 #include <aws/core/auth/AWSCredentialsProvider.h>
 #include <aws/core/auth/AWSCredentialsProviderChain.h>
 #include <aws/core/client/ClientConfiguration.h>
@@ -87,7 +88,14 @@ S3Helper::S3Helper(const std::string & profile, const std::string & region)
                 std::make_shared<Aws::Auth::DefaultAWSCredentialsProviderChain>())
             : std::dynamic_pointer_cast<Aws::Auth::AWSCredentialsProvider>(
                 std::make_shared<Aws::Auth::ProfileConfigFileAWSCredentialsProvider>(profile.c_str())),
-            *config, Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, false))
+            *config,
+            // FIXME: https://github.com/aws/aws-sdk-cpp/issues/759
+#if AWS_VERSION_MAJOR == 1 && AWS_VERSION_MINOR < 3
+            false,
+#else
+            Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never,
+#endif
+            false))
 {
 }
 

src/libutil/compression.cc

@@ -7,6 +7,11 @@
 #include <cstdio>
 #include <cstring>
 
+#if HAVE_BROTLI
+#include <brotli/decode.h>
+#include <brotli/encode.h>
+#endif // HAVE_BROTLI
+
 #include <iostream>
 
 namespace nix {
@@ -94,8 +99,56 @@ static ref<std::string> decompressBzip2(const std::string & in)
 
 static ref<std::string> decompressBrotli(const std::string & in)
 {
-    // FIXME: use libbrotli
-    return make_ref<std::string>(runProgram(BRO, true, {"-d"}, {in}));
+#if !HAVE_BROTLI
+    return make_ref<std::string>(runProgram(BROTLI, true, {"-d"}, {in}));
+#else
+    auto *s = BrotliDecoderCreateInstance(nullptr, nullptr, nullptr);
+    if (!s)
+        throw CompressionError("unable to initialize brotli decoder");
+
+    Finally free([s]() { BrotliDecoderDestroyInstance(s); });
+
+    uint8_t outbuf[BUFSIZ];
+    ref<std::string> res = make_ref<std::string>();
+    const uint8_t *next_in = (uint8_t *)in.c_str();
+    size_t avail_in = in.size();
+    uint8_t *next_out = outbuf;
+    size_t avail_out = sizeof(outbuf);
+
+    while (true) {
+        checkInterrupt();
+
+        auto ret = BrotliDecoderDecompressStream(s,
+                &avail_in, &next_in,
+                &avail_out, &next_out,
+                nullptr);
+
+        switch (ret) {
+        case BROTLI_DECODER_RESULT_ERROR:
+            throw CompressionError("error while decompressing brotli file");
+        case BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT:
+            throw CompressionError("incomplete or corrupt brotli file");
+        case BROTLI_DECODER_RESULT_SUCCESS:
+            if (avail_in != 0)
+                throw CompressionError("unexpected input after brotli decompression");
+            break;
+        case BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT:
+            // I'm not sure if this can happen, but abort if this happens with empty buffer
+            if (avail_out == sizeof(outbuf))
+                throw CompressionError("brotli decompression requires larger buffer");
+            break;
+        }
+
+        // Always ensure we have full buffer for next invocation
+        if (avail_out < sizeof(outbuf)) {
+            res->append((char*)outbuf, sizeof(outbuf) - avail_out);
+            next_out = outbuf;
+            avail_out = sizeof(outbuf);
+        }
+
+        if (ret == BROTLI_DECODER_RESULT_SUCCESS) return res;
+    }
+#endif // HAVE_BROTLI
 }
 
 ref<std::string> compress(const std::string & method, const std::string & in)
@@ -270,25 +323,22 @@ struct BzipSink : CompressionSink
     }
 };
 
-struct BrotliSink : CompressionSink
+struct LambdaCompressionSink : CompressionSink
 {
     Sink & nextSink;
     std::string data;
-
-    BrotliSink(Sink & nextSink) : nextSink(nextSink)
+    using CompressFnTy = std::function<std::string(const std::string&)>;
+    CompressFnTy compressFn;
+    LambdaCompressionSink(Sink& nextSink, CompressFnTy compressFn)
+        : nextSink(nextSink)
+        , compressFn(std::move(compressFn))
     {
-    }
-
-    ~BrotliSink()
-    {
-    }
-
-    // FIXME: use libbrotli
+    };
 
     void finish() override
     {
         flush();
-        nextSink(runProgram(BRO, true, {}, data));
+        nextSink(compressFn(data));
     }
 
     void write(const unsigned char * data, size_t len) override
@@ -298,6 +348,107 @@ struct BrotliSink : CompressionSink
     }
 };
 
+struct BrotliCmdSink : LambdaCompressionSink
+{
+    BrotliCmdSink(Sink& nextSink)
+        : LambdaCompressionSink(nextSink, [](const std::string& data) {
+            return runProgram(BROTLI, true, {}, data);
+        })
+    {
+    }
+};
+
+#if HAVE_BROTLI
+struct BrotliSink : CompressionSink
+{
+    Sink & nextSink;
+    uint8_t outbuf[BUFSIZ];
+    BrotliEncoderState *state;
+    bool finished = false;
+
+    BrotliSink(Sink & nextSink) : nextSink(nextSink)
+    {
+        state = BrotliEncoderCreateInstance(nullptr, nullptr, nullptr);
+        if (!state)
+            throw CompressionError("unable to initialise brotli encoder");
+    }
+
+    ~BrotliSink()
+    {
+        BrotliEncoderDestroyInstance(state);
+    }
+
+    void finish() override
+    {
+        flush();
+        assert(!finished);
+
+        const uint8_t *next_in = nullptr;
+        size_t avail_in = 0;
+        uint8_t *next_out = outbuf;
+        size_t avail_out = sizeof(outbuf);
+        while (!finished) {
+            checkInterrupt();
+
+            if (!BrotliEncoderCompressStream(state,
+                        BROTLI_OPERATION_FINISH,
+                        &avail_in, &next_in,
+                        &avail_out, &next_out,
+                        nullptr))
+                throw CompressionError("error while finishing brotli file");
+
+            finished = BrotliEncoderIsFinished(state);
+            if (avail_out == 0 || finished) {
+                nextSink(outbuf, sizeof(outbuf) - avail_out);
+                next_out = outbuf;
+                avail_out = sizeof(outbuf);
+            }
+        }
+    }
+
+    void write(const unsigned char * data, size_t len) override
+    {
+        assert(!finished);
+
+        // Don't feed brotli too much at once
+        const size_t CHUNK_SIZE = sizeof(outbuf) << 2;
+        while (len) {
+          size_t n = std::min(CHUNK_SIZE, len);
+          writeInternal(data, n);
+          data += n;
+          len -= n;
+        }
+    }
+  private:
+    void writeInternal(const unsigned char * data, size_t len)
+    {
+        assert(!finished);
+
+        const uint8_t *next_in = data;
+        size_t avail_in = len;
+        uint8_t *next_out = outbuf;
+        size_t avail_out = sizeof(outbuf);
+
+        while (avail_in > 0) {
+            checkInterrupt();
+
+            if (!BrotliEncoderCompressStream(state,
+                      BROTLI_OPERATION_PROCESS,
+                      &avail_in, &next_in,
+                      &avail_out, &next_out,
+                      nullptr))
+                throw CompressionError("error while compressing brotli file");
+
+            if (avail_out < sizeof(outbuf) || avail_in == 0) {
+                nextSink(outbuf, sizeof(outbuf) - avail_out);
+                next_out = outbuf;
+                avail_out = sizeof(outbuf);
+            }
+        }
+    }
+};
+#endif // HAVE_BROTLI
+
 ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink)
 {
     if (method == "none")
@@ -307,7 +458,11 @@ ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & next
     else if (method == "bzip2")
         return make_ref<BzipSink>(nextSink);
     else if (method == "br")
+#if HAVE_BROTLI
         return make_ref<BrotliSink>(nextSink);
+#else
+        return make_ref<BrotliCmdSink>(nextSink);
+#endif
     else
         throw UnknownCompressionMethod(format("unknown compression method '%s'") % method);
 }

src/libutil/local.mk

@@ -6,8 +6,8 @@ libutil_DIR := $(d)
 
 libutil_SOURCES := $(wildcard $(d)/*.cc)
 
-libutil_LDFLAGS = $(LIBLZMA_LIBS) -lbz2 -pthread $(OPENSSL_LIBS)
+libutil_LDFLAGS = $(LIBLZMA_LIBS) -lbz2 -pthread $(OPENSSL_LIBS) $(LIBBROTLI_LIBS)
 
 libutil_LIBS = libformat
 
-libutil_CXXFLAGS = -DBRO=\"$(bro)\"
+libutil_CXXFLAGS = -DBROTLI=\"$(brotli)\"

src/nix-daemon/nix-daemon.cc

@@ -411,7 +411,7 @@ static void performOp(TunnelLogger * logger, ref<LocalStore> store,
             /* Repairing is not atomic, so disallowed for "untrusted"
                clients.  */
             if (mode == bmRepair && !trusted)
-                throw Error("repairing is not supported when building through the Nix daemon");
+                throw Error("repairing is not allowed because you are not in 'trusted-users'");
         }
         logger->startWork();
         store->buildPaths(drvs, mode);

src/nix/repl.cc

@@ -186,7 +186,16 @@ bool NixRepl::getLine(string & input, const std::string &prompt)
 {
     char * s = linenoise(prompt.c_str());
     Finally doFree([&]() { free(s); });
-    if (!s) return false;
+    if (!s) {
+      switch (auto type = linenoiseKeyType()) {
+        case 1: // ctrl-C
+          return true;
+        case 2: // ctrl-D
+          return false;
+        default:
+          throw Error(format("Unexpected linenoise keytype: %1%") % type);
+      }
+    }
     input += s;
     return true;
 }

tests/brotli.sh

@@ -0,0 +1,28 @@
+source common.sh
+
+
+# Only test if we found brotli libraries
+# (CLI tool is likely unavailable if libraries are missing)
+if [ -n "$HAVE_BROTLI" ]; then
+
+clearStore
+clearCache
+
+cacheURI="file://$cacheDir?compression=br"
+
+outPath=$(nix-build dependencies.nix --no-out-link)
+
+nix copy --to $cacheURI $outPath
+
+HASH=$(nix hash-path $outPath)
+
+clearStore
+clearCacheCache
+
+nix copy --from $cacheURI $outPath --no-check-sigs
+
+HASH2=$(nix hash-path $outPath)
+
+[[ $HASH = $HASH2 ]]
+
+fi # HAVE_BROTLI

tests/common.sh.in

@@ -32,6 +32,7 @@ export xmllint="@xmllint@"
 export SHELL="@bash@"
 export PAGER=cat
 export HAVE_SODIUM="@HAVE_SODIUM@"
+export HAVE_BROTLI="@HAVE_BROTLI@"
 
 export version=@PACKAGE_VERSION@
 export system=@system@

tests/fetchGit.sh

@@ -93,3 +93,42 @@ git -C $repo add hello
 git -C $repo commit -m 'Bla4'
 rev3=$(git -C $repo rev-parse HEAD)
 nix eval --tarball-ttl 3600 "(builtins.fetchGit { url = $repo; rev = \"$rev3\"; })" >/dev/null
+
+# Update 'path' to reflect latest master
+path=$(nix eval --raw "(builtins.fetchGit file://$repo).outPath")
+
+# Check behavior when non-master branch is used
+git -C $repo checkout $rev2 -b dev
+echo dev > $repo/hello
+
+# File URI uses 'master' unless specified otherwise
+path2=$(nix eval --raw "(builtins.fetchGit file://$repo).outPath")
+[[ $path = $path2 ]]
+
+# Using local path with branch other than 'master' should work when clean or dirty
+path3=$(nix eval --raw "(builtins.fetchGit $repo).outPath")
+# (check dirty-tree handling was used)
+[[ $(nix eval --raw "(builtins.fetchGit $repo).rev") = 0000000000000000000000000000000000000000 ]]
+
+# Committing shouldn't change store path, or switch to using 'master'
+git -C $repo commit -m 'Bla5' -a
+path4=$(nix eval --raw "(builtins.fetchGit $repo).outPath")
+[[ $(cat $path4/hello) = dev ]]
+[[ $path3 = $path4 ]]
+
+# Confirm same as 'dev' branch
+path5=$(nix eval --raw "(builtins.fetchGit { url = $repo; ref = \"dev\"; }).outPath")
+[[ $path3 = $path5 ]]
+
+
+# Nuke the cache
+rm -rf $TEST_HOME/.cache/nix/git
+
+# Try again, but without 'git' on PATH
+NIX=$(command -v nix)
+# This should fail
+(! PATH= $NIX eval --raw "(builtins.fetchGit { url = $repo; ref = \"dev\"; }).outPath" )
+
+# Try again, with 'git' available.  This should work.
+path5=$(nix eval --raw "(builtins.fetchGit { url = $repo; ref = \"dev\"; }).outPath")
+[[ $path3 = $path5 ]]

tests/local.mk

@@ -19,7 +19,8 @@ nix_tests = \
   fetchGit.sh \
   fetchMercurial.sh \
   signing.sh \
-  run.sh
+  run.sh \
+  brotli.sh
   # parallel.sh
 
 install-tests += $(foreach x, $(nix_tests), tests/$(x))

tests/nix-copy-closure.nix

@@ -1,8 +1,8 @@
 # Test ‘nix-copy-closure’.
 
-{ system, nix }:
+{ nixpkgs, system, nix }:
 
-with import <nixpkgs/nixos/lib/testing.nix> { inherit system; };
+with import (nixpkgs + /nixos/lib/testing.nix) { inherit system; };
 
 makeTest (let pkgA = pkgs.cowsay; pkgB = pkgs.wget; pkgC = pkgs.hello; in {
 

tests/remote-builds.nix

@@ -1,8 +1,8 @@
 # Test Nix's remote build feature.
 
-{ system, nix }:
+{ nixpkgs, system, nix }:
 
-with import <nixpkgs/nixos/lib/testing.nix> { inherit system; };
+with import (nixpkgs + "/nixos/lib/testing.nix") { inherit system; };
 
 makeTest (
 

tests/run.sh

@@ -18,10 +18,10 @@ if [[ $(uname) = Linux ]]; then
     # Note: we need the sandbox paths to ensure that the shell is
     # visible in the sandbox.
     nix run --sandbox-build-dir /build-tmp \
-        --sandbox-paths '/nix? /bin? /lib? /usr?' \
+        --sandbox-paths '/nix? /bin? /lib? /lib64? /usr?' \
         --store $TEST_ROOT/store0 -f run.nix hello -c hello | grep 'Hello World'
 
-    path2=$(nix run --sandbox-paths '/nix? /bin? /lib? /usr?' --store $TEST_ROOT/store0 -f run.nix hello -c $SHELL -c 'type -p hello')
+    path2=$(nix run --sandbox-paths '/nix? /bin? /lib? /lib64? /usr?' --store $TEST_ROOT/store0 -f run.nix hello -c $SHELL -c 'type -p hello')
 
     [[ $path/bin/hello = $path2 ]]
 

tests/setuid.nix

@@ -1,8 +1,8 @@
 # Verify that Linux builds cannot create setuid or setgid binaries.
 
-{ system, nix }:
+{ nixpkgs, system, nix }:
 
-with import <nixpkgs/nixos/lib/testing.nix> { inherit system; };
+with import (nixpkgs + "/nixos/lib/testing.nix") { inherit system; };
 
 makeTest {
 

tests/shell.shebang.sh

@@ -1,4 +1,4 @@
 #! @ENV_PROG@ nix-shell
-#! nix-shell -I nixpkgs=shell.nix --no-use-substitutes
+#! nix-shell -I nixpkgs=shell.nix --no-substitute
 #! nix-shell --pure -i bash -p foo bar
 echo "$(foo) $(bar) $@"