Add "nix mount-store" command

This mounts an arbitrary Nix store on the specified mount point.

Typical usage:

  $ /nix/store/d0am5d8gwh2kfdcgyxh4y684mb5b2v54-blender-2.79/bin/blender --version
  bash: /nix/store/d0am5d8gwh2kfdcgyxh4y684mb5b2v54-blender-2.79/bin/blender: No such file or directory

  $ nix mount-store /tmp/mp --store https://cache.nixos.org?local-nar-cache=/tmp/nars

  $ unshare -m -r

  $ mount -o bind /tmp/mp /nix/store

  $ /nix/store/d0am5d8gwh2kfdcgyxh4y684mb5b2v54-blender-2.79/bin/blender --version
  [after a lot of downloading...]
  Blender 2.79 (sub 0)

One application is to replace the current remote store file access in
hydra-server implemented via "nix {cat,ls}-store", which doesn't work
all that well (e.g. it doesn't resolve symlinks properly).

Another application would be on-demand fetching of build inputs on
Hydra build slaves (to speed up builds that don't access their entire
closure). However, that will require a lot more machinery.

Makefile.config.in

@@ -6,7 +6,6 @@ CXXFLAGS = @CXXFLAGS@
 ENABLE_S3 = @ENABLE_S3@
 HAVE_SODIUM = @HAVE_SODIUM@
 HAVE_READLINE = @HAVE_READLINE@
-HAVE_BROTLI = @HAVE_BROTLI@
 LIBCURL_LIBS = @LIBCURL_LIBS@
 OPENSSL_LIBS = @OPENSSL_LIBS@
 PACKAGE_NAME = @PACKAGE_NAME@
@@ -14,10 +13,9 @@ PACKAGE_VERSION = @PACKAGE_VERSION@
 SODIUM_LIBS = @SODIUM_LIBS@
 LIBLZMA_LIBS = @LIBLZMA_LIBS@
 SQLITE3_LIBS = @SQLITE3_LIBS@
-LIBBROTLI_LIBS = @LIBBROTLI_LIBS@
 bash = @bash@
 bindir = @bindir@
-brotli = @brotli@
+bro = @bro@
 lsof = @lsof@
 datadir = @datadir@
 datarootdir = @datarootdir@

configure.ac

@@ -127,7 +127,7 @@ NEED_PROG(gzip, gzip)
 NEED_PROG(xz, xz)
 AC_PATH_PROG(dot, dot)
 AC_PATH_PROG(pv, pv, pv)
-AC_PATH_PROGS(brotli, brotli bro, bro)
+AC_PATH_PROG(bro, bro, bro)
 AC_PATH_PROG(lsof, lsof, lsof)
 
 
@@ -176,13 +176,6 @@ AC_SUBST(HAVE_SODIUM, [$have_sodium])
 PKG_CHECK_MODULES([LIBLZMA], [liblzma], [CXXFLAGS="$LIBLZMA_CFLAGS $CXXFLAGS"])
 
 
-# Look for libbrotli{enc,dec}, optional dependencies
-PKG_CHECK_MODULES([LIBBROTLI], [libbrotlienc libbrotlidec],
-  [AC_DEFINE([HAVE_BROTLI], [1], [Whether to use libbrotli.])
-   CXXFLAGS="$LIBBROTLI_CFLAGS $CXXFLAGS"]
-   have_brotli=1], [have_brotli=])
-AC_SUBST(HAVE_BROTLI, [$have_brotli])
-
 # Look for libseccomp, required for Linux sandboxing.
 if test "$sys_name" = linux; then
   PKG_CHECK_MODULES([LIBSECCOMP], [libseccomp],
@@ -193,17 +186,11 @@ fi
 # Look for aws-cpp-sdk-s3.
 AC_LANG_PUSH(C++)
 AC_CHECK_HEADERS([aws/s3/S3Client.h],
-  [AC_DEFINE([ENABLE_S3], [1], [Whether to enable S3 support via aws-sdk-cpp.])
+  [AC_DEFINE([ENABLE_S3], [1], [Whether to enable S3 support via aws-cpp-sdk-s3.])
   enable_s3=1], [enable_s3=])
 AC_SUBST(ENABLE_S3, [$enable_s3])
 AC_LANG_POP(C++)
 
-if test -n "$enable_s3"; then
-  declare -a aws_version_tokens=($(printf '#include <aws/core/VersionConfig.h>\nAWS_SDK_VERSION_STRING' | cpp -E | grep -v '^#.*' | sed 's/"//g' | tr '.' ' '))
-  AC_DEFINE_UNQUOTED([AWS_VERSION_MAJOR], ${aws_version_tokens@<:@0@:>@}, [Major version of aws-sdk-cpp.])
-  AC_DEFINE_UNQUOTED([AWS_VERSION_MINOR], ${aws_version_tokens@<:@1@:>@}, [Minor version of aws-sdk-cpp.])
-fi
-
 
 # Whether to use the Boehm garbage collector.
 AC_ARG_ENABLE(gc, AC_HELP_STRING([--enable-gc],

doc/manual/command-ref/conf-file.xml

@@ -312,7 +312,7 @@ false</literal>.</para>
   </varlistentry>
 
 
-  <varlistentry><term><literal>substitute</literal></term>
+  <varlistentry><term><literal>use-substitutes</literal></term>
 
     <listitem><para>If set to <literal>true</literal> (default), Nix
     will use binary substitutes if available.  This option can be
@@ -321,20 +321,6 @@ false</literal>.</para>
   </varlistentry>
 
 
-  <varlistentry><term><literal>builders-use-substitutes</literal></term>
-
-    <listitem><para>If set to <literal>true</literal>, Nix will instruct
-    remote build machines to use their own binary substitutes if available. In
-    practical terms, this means that remote hosts will fetch as many build
-    dependencies as possible from their own substitutes (e.g, from
-    <literal>cache.nixos.org</literal>), instead of waiting for this host to
-    upload them all. This can drastically reduce build times if the network
-    connection between this computer and the remote build host is slow. Defaults
-    to <literal>false</literal>.</para></listitem>
-
-  </varlistentry>
-
-
   <varlistentry><term><literal>fallback</literal></term>
 
     <listitem><para>If set to <literal>true</literal>, Nix will fall

local.mk

@@ -1,12 +1,13 @@
 ifeq ($(MAKECMDGOALS), dist)
-  dist-files += $(shell cat .dist-files)
+  # Make sure we are in repo root with `--git-dir`
+  dist-files += $(shell git --git-dir=.git ls-files || find * -type f)
 endif
 
 dist-files += configure config.h.in nix.spec perl/configure
 
 clean-files += Makefile.config
 
-GLOBAL_CXXFLAGS += -I . -I src -I src/libutil -I src/libstore -I src/libmain -I src/libexpr
+GLOBAL_CXXFLAGS += -I . -I src -I src/libutil -I src/libstore -I src/libmain -I src/libexpr -D_FILE_OFFSET_BITS=64
 
 $(foreach i, config.h $(call rwildcard, src/lib*, *.hh), \
   $(eval $(call install-file-in, $(i), $(includedir)/nix, 0644)))

release.nix

@@ -1,12 +1,12 @@
-{ nix ? builtins.fetchGit ./.
-, nixpkgs ? fetchTarball channel:nixos-17.09
+{ nix ? { outPath = ./.; revCount = 1234; shortRev = "abcdef"; }
+, nixpkgs ? { outPath = <nixpkgs>; revCount = 1234; shortRev = "abcdef"; }
 , officialRelease ? false
 , systems ? [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ]
 }:
 
 let
 
-  pkgs = import nixpkgs {};
+  pkgs = import <nixpkgs> {};
 
   jobs = rec {
 
@@ -27,13 +27,16 @@ let
             pkgconfig sqlite libsodium boehmgc
             docbook5 docbook5_xsl
             autoconf-archive
+            git
           ] ++ lib.optional stdenv.isLinux libseccomp;
 
         configureFlags = "--enable-gc";
 
         postUnpack = ''
-          (cd source && find . -type f) | cut -c3- > source/.dist-files
-          cat source/.dist-files
+          # Clean up when building from a working tree.
+          if [[ -d $sourceRoot/.git ]]; then
+            git -C $sourceRoot clean -fd
+          fi
         '';
 
         preConfigure = ''
@@ -59,7 +62,7 @@ let
 
     build = pkgs.lib.genAttrs systems (system:
 
-      with import nixpkgs { inherit system; };
+      with import <nixpkgs> { inherit system; };
 
       with import ./release-common.nix { inherit pkgs; };
 
@@ -76,7 +79,7 @@ let
             git
             mercurial
           ]
-          ++ lib.optional stdenv.isLinux libseccomp
+          ++ lib.optionals stdenv.isLinux [ libseccomp fuse ]
           ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium
           ++ lib.optional (stdenv.isLinux || stdenv.isDarwin)
             (aws-sdk-cpp.override {
@@ -102,7 +105,7 @@ let
 
     perlBindings = pkgs.lib.genAttrs systems (system:
 
-      let pkgs = import nixpkgs { inherit system; }; in with pkgs;
+      let pkgs = import <nixpkgs> { inherit system; }; in with pkgs;
 
       releaseTools.nixBuild {
         name = "nix-perl";
@@ -128,7 +131,7 @@ let
     binaryTarball = pkgs.lib.genAttrs systems (system:
 
       # FIXME: temporarily use a different branch for the Darwin build.
-      with import nixpkgs { inherit system; };
+      with import <nixpkgs> { inherit system; };
 
       let
         toplevel = builtins.getAttr system jobs.build;
@@ -171,7 +174,7 @@ let
 
 
     coverage =
-      with import nixpkgs { system = "x86_64-linux"; };
+      with import <nixpkgs> { system = "x86_64-linux"; };
 
       releaseTools.coverageAnalysis {
         name = "nix-build";
@@ -215,23 +218,20 @@ let
 
     # System tests.
     tests.remoteBuilds = (import ./tests/remote-builds.nix rec {
-      inherit nixpkgs;
       nix = build.x86_64-linux; system = "x86_64-linux";
     });
 
     tests.nix-copy-closure = (import ./tests/nix-copy-closure.nix rec {
-      inherit nixpkgs;
       nix = build.x86_64-linux; system = "x86_64-linux";
     });
 
     tests.setuid = pkgs.lib.genAttrs (pkgs.lib.filter (pkgs.lib.hasSuffix "-linux") systems) (system:
       import ./tests/setuid.nix rec {
-        inherit nixpkgs;
         nix = build.${system}; inherit system;
       });
 
     tests.binaryTarball =
-      with import nixpkgs { system = "x86_64-linux"; };
+      with import <nixpkgs> { system = "x86_64-linux"; };
       vmTools.runInLinuxImage (runCommand "nix-binary-tarball-test"
         { diskImage = vmTools.diskImages.ubuntu1204x86_64;
         }
@@ -250,7 +250,7 @@ let
         ''); # */
 
     tests.evalNixpkgs =
-      import (nixpkgs + "/pkgs/top-level/make-tarball.nix") {
+      import <nixpkgs/pkgs/top-level/make-tarball.nix> {
         inherit nixpkgs;
         inherit pkgs;
         nix = build.x86_64-linux;
@@ -304,7 +304,7 @@ let
   makeRPM =
     system: diskImageFun: extraPackages:
 
-    with import nixpkgs { inherit system; };
+    with import <nixpkgs> { inherit system; };
 
     releaseTools.rpmBuild rec {
       name = "nix-rpm";
@@ -313,8 +313,7 @@ let
         { extraPackages =
             [ "sqlite" "sqlite-devel" "bzip2-devel" "libcurl-devel" "openssl-devel" "xz-devel" "libseccomp-devel" ]
             ++ extraPackages; };
-      # At most 2047MB can be simulated in qemu-system-i386
-      memSize = 2047;
+      memSize = 1024;
       meta.schedulingPriority = 50;
       postRPMInstall = "cd /tmp/rpmout/BUILD/nix-* && make installcheck";
       #enableParallelBuilding = true;
@@ -327,7 +326,7 @@ let
   makeDeb =
     system: diskImageFun: extraPackages: extraDebPackages:
 
-    with import nixpkgs { inherit system; };
+    with import <nixpkgs> { inherit system; };
 
     releaseTools.debBuild {
       name = "nix-deb";

scripts/install-darwin-multi-user.sh

@@ -33,7 +33,7 @@ readonly NIX_FIRST_BUILD_UID="30001"
 readonly NIX_ROOT="/nix"
 readonly PLIST_DEST=/Library/LaunchDaemons/org.nixos.nix-daemon.plist
 
-readonly PROFILE_TARGETS=("/etc/bashrc" "/etc/zshrc")
+readonly PROFILE_TARGETS=("/etc/profile" "/etc/bashrc" "/etc/zshrc")
 readonly PROFILE_BACKUP_SUFFIX=".backup-before-nix"
 readonly PROFILE_NIX_FILE="$NIX_ROOT/var/nix/profiles/default/etc/profile.d/nix-daemon.sh"
 

scripts/nix-profile.sh.in

@@ -60,6 +60,12 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then
     # This part should be kept in sync with nixpkgs:nixos/modules/programs/environment.nix
     NIX_PROFILES="@localstatedir@/nix/profiles/default $NIX_USER_PROFILE_DIR"
 
+    for i in $NIX_PROFILES; do
+        if [ -d "$i/lib/aspell" ]; then
+            export ASPELL_CONF="dict-dir $i/lib/aspell"
+        fi
+    done
+
     # Set $NIX_SSL_CERT_FILE so that Nixpkgs applications like curl work.
     if [ -e /etc/ssl/certs/ca-certificates.crt ]; then # NixOS, Ubuntu, Debian, Gentoo, Arch
         export NIX_SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt

shell.nix

@@ -27,7 +27,7 @@ with import ./release-common.nix { inherit pkgs; };
       git
       mercurial
     ]
-    ++ lib.optional stdenv.isLinux libseccomp;
+    ++ lib.optionals stdenv.isLinux [ libseccomp fuse ];
 
   inherit configureFlags;
 

src/build-remote/build-remote.cc

@@ -218,11 +218,9 @@ connected:
             signal(SIGALRM, old);
         }
 
-        auto substitute = settings.buildersUseSubstitutes ? Substitute : NoSubstitute;
-
         {
             Activity act(*logger, lvlTalkative, actUnknown, fmt("copying dependencies to '%s'", storeUri));
-            copyPaths(store, ref<Store>(sshStore), inputs, NoRepair, NoCheckSigs, substitute);
+            copyPaths(store, ref<Store>(sshStore), inputs, NoRepair, NoCheckSigs);
         }
 
         uploadLock = -1;
@@ -242,7 +240,7 @@ connected:
         if (!missing.empty()) {
             Activity act(*logger, lvlTalkative, actUnknown, fmt("copying outputs from '%s'", storeUri));
             setenv("NIX_HELD_LOCKS", concatStringsSep(" ", missing).c_str(), 1); /* FIXME: ugly */
-            copyPaths(ref<Store>(sshStore), store, missing, NoRepair, NoCheckSigs, substitute);
+            copyPaths(ref<Store>(sshStore), store, missing, NoRepair, NoCheckSigs);
         }
 
         return;

src/libexpr/eval.cc

@@ -1683,8 +1683,6 @@ void EvalState::printStats()
     printMsg(v, format("  number of primop calls: %1%") % nrPrimOpCalls);
     printMsg(v, format("  number of function calls: %1%") % nrFunctionCalls);
     printMsg(v, format("  total allocations: %1% bytes") % (bEnvs + bLists + bValues + bAttrsets));
-    printMsg(v, format("  memoisation hits: %d") % nrMemoiseHits);
-    printMsg(v, format("  memoisation misses: %d") % nrMemoiseMisses);
 
 #if HAVE_BOEHMGC
     GC_word heapSize, totalBytes;

src/libexpr/eval.hh

@@ -89,7 +89,7 @@ private:
 
     /* A cache from path names to values. */
 #if HAVE_BOEHMGC
-    typedef std::map<Path, Value, std::less<Path>, traceable_allocator<std::pair<const Path, Value>>> FileEvalCache;
+    typedef std::map<Path, Value, std::less<Path>, traceable_allocator<std::pair<const Path, Value> > > FileEvalCache;
 #else
     typedef std::map<Path, Value> FileEvalCache;
 #endif
@@ -269,8 +269,6 @@ private:
     unsigned long nrListConcats = 0;
     unsigned long nrPrimOpCalls = 0;
     unsigned long nrFunctionCalls = 0;
-    unsigned long nrMemoiseHits = 0;
-    unsigned long nrMemoiseMisses = 0;
 
     bool countCalls;
 
@@ -289,22 +287,6 @@ private:
     friend struct ExprOpConcatLists;
     friend struct ExprSelect;
     friend void prim_getAttr(EvalState & state, const Pos & pos, Value * * args, Value & v);
-    friend void prim_memoise(EvalState & state, const Pos & pos, Value * * args, Value & v);
-
-    /* State for builtins.memoise. */
-    struct MemoArgComparator
-    {
-        EvalState & state;
-        MemoArgComparator(EvalState & state) : state(state) { }
-        bool operator()(Value * a, Value * b);
-    };
-
-    typedef std::map<Value *, Value, MemoArgComparator, traceable_allocator<std::pair<const Value *, Value>>> PerLambdaMemo;
-
-    typedef std::pair<Env *, ExprLambda *> LambdaKey;
-
-    // FIXME: use std::unordered_map
-    std::map<LambdaKey, PerLambdaMemo, std::less<LambdaKey>, traceable_allocator<std::pair<const LambdaKey, PerLambdaMemo>>> memos;
 };
 
 

src/libexpr/primops.cc

@@ -1653,14 +1653,6 @@ static void prim_unsafeDiscardStringContext(EvalState & state, const Pos & pos,
 }
 
 
-static void prim_hasContext(EvalState & state, const Pos & pos, Value * * args, Value & v)
-{
-    PathSet context;
-    state.forceString(*args[0], context, pos);
-    mkBool(v, !context.empty());
-}
-
-
 /* Sometimes we want to pass a derivation path (i.e. pkg.drvPath) to a
    builder without causing the derivation to be built (for instance,
    in the derivation that builds NARs in nix-push, when doing
@@ -2091,7 +2083,6 @@ void EvalState::createBaseEnv()
     addPrimOp("toString", 1, prim_toString);
     addPrimOp("__substring", 3, prim_substring);
     addPrimOp("__stringLength", 1, prim_stringLength);
-    addPrimOp("__hasContext", 1, prim_hasContext);
     addPrimOp("__unsafeDiscardStringContext", 1, prim_unsafeDiscardStringContext);
     addPrimOp("__unsafeDiscardOutputDependency", 1, prim_unsafeDiscardOutputDependency);
     addPrimOp("__hashString", 2, prim_hashString);

src/libexpr/primops/fetchGit.cc

@@ -23,7 +23,7 @@ struct GitInfo
 };
 
 GitInfo exportGit(ref<Store> store, const std::string & uri,
-    std::experimental::optional<std::string> ref, std::string rev,
+    std::experimental::optional<std::string> ref, const std::string & rev,
     const std::string & name)
 {
     if (!ref && rev == "" && hasPrefix(uri, "/") && pathExists(uri + "/.git")) {
@@ -68,10 +68,6 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
 
             return gitInfo;
         }
-
-        // clean working tree, but no ref or rev specified.  Use 'HEAD'.
-        rev = chomp(runProgram("git", true, { "-C", uri, "rev-parse", "HEAD" }));
-        ref = "HEAD"s;
     }
 
     if (!ref) ref = "master"s;
@@ -85,6 +81,7 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
     Path cacheDir = getCacheDir() + "/nix/git";
 
     if (!pathExists(cacheDir)) {
+        createDirs(cacheDir);
         runProgram("git", true, { "init", "--bare", cacheDir });
     }
 

src/libexpr/primops/memoise.cc

@@ -1,95 +0,0 @@
-#include "primops.hh"
-#include "eval-inline.hh"
-
-#include <cstring>
-
-namespace nix {
-
-bool EvalState::MemoArgComparator::operator()(Value * v1, Value * v2)
-{
-    if (v1 == v2) return false;
-
-    state.forceValue(*v1);
-    state.forceValue(*v2);
-
-    if (v1->type == v2->type) {
-        switch (v1->type) {
-
-        case tInt:
-            return v1->integer < v2->integer;
-
-        case tBool:
-            return v1->boolean < v2->boolean;
-
-        case tString:
-            return strcmp(v1->string.s, v2->string.s);
-
-        case tPath:
-            return strcmp(v1->path, v2->path);
-
-        case tNull:
-            return false;
-
-        case tList1:
-        case tList2:
-        case tListN:
-            unsigned int n;
-            for (n = 0; n < v1->listSize() && n < v2->listSize(); ++n) {
-                if ((*this)(v1->listElems()[n], v2->listElems()[n])) return true;
-                if ((*this)(v2->listElems()[n], v1->listElems()[n])) return false;
-            }
-
-            return n == v1->listSize() && n < v2->listSize();
-
-        case tAttrs:
-            Bindings::iterator i, j;
-            for (i = v1->attrs->begin(), j = v2->attrs->begin(); i != v1->attrs->end() && j != v2->attrs->end(); ++i, ++j) {
-                if (i->name < j->name) return true;
-                if (j->name < i->name) return false;
-                if ((*this)(i->value, j->value)) return true;
-                if ((*this)(j->value, i->value)) return false;
-            }
-            return i == v1->attrs->end() && j != v2->attrs->end();
-
-        case tLambda:
-            return std::make_pair(v1->lambda.env, v1->lambda.fun) < std::make_pair(v2->lambda.env, v2->lambda.fun);
-
-        case tFloat:
-            return v1->fpoint < v2->fpoint;
-
-        default:
-            break;
-        }
-    }
-
-    // As a fallback, use pointer equality.
-    return v1 < v2;
-}
-
-void prim_memoise(EvalState & state, const Pos & pos, Value * * args, Value & v)
-{
-    state.forceFunction(*args[0], pos);
-
-    EvalState::PerLambdaMemo foo(state);
-
-    auto & memo = state.memos.emplace(std::make_pair(args[0]->lambda.env, args[0]->lambda.fun), state).first->second;
-
-    auto result = memo.find(args[1]);
-
-    if (result != memo.end()) {
-        state.nrMemoiseHits++;
-        v = result->second;
-        return;
-    }
-
-    state.nrMemoiseMisses++;
-
-    state.callFunction(*args[0], *args[1], v, pos);
-
-    memo[args[1]] = v;
-
-}
-
-static RegisterPrimOp r("memoise", 2, prim_memoise);
-
-}

src/libstore/globals.hh

@@ -138,11 +138,6 @@ public:
     Setting<std::string> builders{this, "@" + nixConfDir + "/machines", "builders",
         "A semicolon-separated list of build machines, in the format of nix.machines."};
 
-    Setting<bool> buildersUseSubstitutes{this, false, "builders-use-substitutes",
-        "Whether build machines should use their own substitutes for obtaining "
-        "build dependencies if possible, rather than waiting for this host to "
-        "upload them."};
-
     Setting<off_t> reservedSize{this, 8 * 1024 * 1024, "gc-reserved-space",
         "Amount of reserved disk space for the garbage collector."};
 
@@ -155,7 +150,7 @@ public:
     Setting<bool> syncBeforeRegistering{this, false, "sync-before-registering",
         "Whether to call sync() before registering a path as valid."};
 
-    Setting<bool> useSubstitutes{this, true, "substitute",
+    Setting<bool> useSubstitutes{this, true, "use-substitutes",
         "Whether to use substitutes.",
         {"build-use-substitutes"}};
 

src/libstore/s3-binary-cache-store.cc

@@ -10,7 +10,6 @@
 #include "istringstream_nocopy.hh"
 
 #include <aws/core/Aws.h>
-#include <aws/core/VersionConfig.h>
 #include <aws/core/auth/AWSCredentialsProvider.h>
 #include <aws/core/auth/AWSCredentialsProviderChain.h>
 #include <aws/core/client/ClientConfiguration.h>
@@ -88,14 +87,7 @@ S3Helper::S3Helper(const std::string & profile, const std::string & region)
                 std::make_shared<Aws::Auth::DefaultAWSCredentialsProviderChain>())
             : std::dynamic_pointer_cast<Aws::Auth::AWSCredentialsProvider>(
                 std::make_shared<Aws::Auth::ProfileConfigFileAWSCredentialsProvider>(profile.c_str())),
-            *config,
-            // FIXME: https://github.com/aws/aws-sdk-cpp/issues/759
-#if AWS_VERSION_MAJOR == 1 && AWS_VERSION_MINOR < 3
-            false,
-#else
-            Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never,
-#endif
-            false))
+            *config, Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, false))
 {
 }
 

src/libutil/compression.cc

@@ -7,11 +7,6 @@
 #include <cstdio>
 #include <cstring>
 
-#if HAVE_BROTLI
-#include <brotli/decode.h>
-#include <brotli/encode.h>
-#endif // HAVE_BROTLI
-
 #include <iostream>
 
 namespace nix {
@@ -99,56 +94,8 @@ static ref<std::string> decompressBzip2(const std::string & in)
 
 static ref<std::string> decompressBrotli(const std::string & in)
 {
-#if !HAVE_BROTLI
-    return make_ref<std::string>(runProgram(BROTLI, true, {"-d"}, {in}));
-#else
-    auto *s = BrotliDecoderCreateInstance(nullptr, nullptr, nullptr);
-    if (!s)
-        throw CompressionError("unable to initialize brotli decoder");
-
-    Finally free([s]() { BrotliDecoderDestroyInstance(s); });
-
-    uint8_t outbuf[BUFSIZ];
-    ref<std::string> res = make_ref<std::string>();
-    const uint8_t *next_in = (uint8_t *)in.c_str();
-    size_t avail_in = in.size();
-    uint8_t *next_out = outbuf;
-    size_t avail_out = sizeof(outbuf);
-
-    while (true) {
-        checkInterrupt();
-
-        auto ret = BrotliDecoderDecompressStream(s,
-                &avail_in, &next_in,
-                &avail_out, &next_out,
-                nullptr);
-
-        switch (ret) {
-        case BROTLI_DECODER_RESULT_ERROR:
-            throw CompressionError("error while decompressing brotli file");
-        case BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT:
-            throw CompressionError("incomplete or corrupt brotli file");
-        case BROTLI_DECODER_RESULT_SUCCESS:
-            if (avail_in != 0)
-                throw CompressionError("unexpected input after brotli decompression");
-            break;
-        case BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT:
-            // I'm not sure if this can happen, but abort if this happens with empty buffer
-            if (avail_out == sizeof(outbuf))
-                throw CompressionError("brotli decompression requires larger buffer");
-            break;
-        }
-
-        // Always ensure we have full buffer for next invocation
-        if (avail_out < sizeof(outbuf)) {
-            res->append((char*)outbuf, sizeof(outbuf) - avail_out);
-            next_out = outbuf;
-            avail_out = sizeof(outbuf);
-        }
-
-        if (ret == BROTLI_DECODER_RESULT_SUCCESS) return res;
-    }
-#endif // HAVE_BROTLI
+    // FIXME: use libbrotli
+    return make_ref<std::string>(runProgram(BRO, true, {"-d"}, {in}));
 }
 
 ref<std::string> compress(const std::string & method, const std::string & in)
@@ -323,22 +270,25 @@ struct BzipSink : CompressionSink
     }
 };
 
-struct LambdaCompressionSink : CompressionSink
+struct BrotliSink : CompressionSink
 {
     Sink & nextSink;
     std::string data;
-    using CompressFnTy = std::function<std::string(const std::string&)>;
-    CompressFnTy compressFn;
-    LambdaCompressionSink(Sink& nextSink, CompressFnTy compressFn)
-        : nextSink(nextSink)
-        , compressFn(std::move(compressFn))
+
+    BrotliSink(Sink & nextSink) : nextSink(nextSink)
     {
-    };
+    }
+
+    ~BrotliSink()
+    {
+    }
+
+    // FIXME: use libbrotli
 
     void finish() override
     {
         flush();
-        nextSink(compressFn(data));
+        nextSink(runProgram(BRO, true, {}, data));
     }
 
     void write(const unsigned char * data, size_t len) override
@@ -348,107 +298,6 @@ struct LambdaCompressionSink : CompressionSink
     }
 };
 
-struct BrotliCmdSink : LambdaCompressionSink
-{
-    BrotliCmdSink(Sink& nextSink)
-        : LambdaCompressionSink(nextSink, [](const std::string& data) {
-            return runProgram(BROTLI, true, {}, data);
-        })
-    {
-    }
-};
-
-#if HAVE_BROTLI
-struct BrotliSink : CompressionSink
-{
-    Sink & nextSink;
-    uint8_t outbuf[BUFSIZ];
-    BrotliEncoderState *state;
-    bool finished = false;
-
-    BrotliSink(Sink & nextSink) : nextSink(nextSink)
-    {
-        state = BrotliEncoderCreateInstance(nullptr, nullptr, nullptr);
-        if (!state)
-            throw CompressionError("unable to initialise brotli encoder");
-    }
-
-    ~BrotliSink()
-    {
-        BrotliEncoderDestroyInstance(state);
-    }
-
-    void finish() override
-    {
-        flush();
-        assert(!finished);
-
-        const uint8_t *next_in = nullptr;
-        size_t avail_in = 0;
-        uint8_t *next_out = outbuf;
-        size_t avail_out = sizeof(outbuf);
-        while (!finished) {
-            checkInterrupt();
-
-            if (!BrotliEncoderCompressStream(state,
-                        BROTLI_OPERATION_FINISH,
-                        &avail_in, &next_in,
-                        &avail_out, &next_out,
-                        nullptr))
-                throw CompressionError("error while finishing brotli file");
-
-            finished = BrotliEncoderIsFinished(state);
-            if (avail_out == 0 || finished) {
-                nextSink(outbuf, sizeof(outbuf) - avail_out);
-                next_out = outbuf;
-                avail_out = sizeof(outbuf);
-            }
-        }
-    }
-
-    void write(const unsigned char * data, size_t len) override
-    {
-        assert(!finished);
-
-        // Don't feed brotli too much at once
-        const size_t CHUNK_SIZE = sizeof(outbuf) << 2;
-        while (len) {
-          size_t n = std::min(CHUNK_SIZE, len);
-          writeInternal(data, n);
-          data += n;
-          len -= n;
-        }
-    }
-  private:
-    void writeInternal(const unsigned char * data, size_t len)
-    {
-        assert(!finished);
-
-        const uint8_t *next_in = data;
-        size_t avail_in = len;
-        uint8_t *next_out = outbuf;
-        size_t avail_out = sizeof(outbuf);
-
-        while (avail_in > 0) {
-            checkInterrupt();
-
-            if (!BrotliEncoderCompressStream(state,
-                      BROTLI_OPERATION_PROCESS,
-                      &avail_in, &next_in,
-                      &avail_out, &next_out,
-                      nullptr))
-                throw CompressionError("error while compressing brotli file");
-
-            if (avail_out < sizeof(outbuf) || avail_in == 0) {
-                nextSink(outbuf, sizeof(outbuf) - avail_out);
-                next_out = outbuf;
-                avail_out = sizeof(outbuf);
-            }
-        }
-    }
-};
-#endif // HAVE_BROTLI
-
 ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink)
 {
     if (method == "none")
@@ -458,11 +307,7 @@ ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & next
     else if (method == "bzip2")
         return make_ref<BzipSink>(nextSink);
     else if (method == "br")
-#if HAVE_BROTLI
         return make_ref<BrotliSink>(nextSink);
-#else
-        return make_ref<BrotliCmdSink>(nextSink);
-#endif
     else
         throw UnknownCompressionMethod(format("unknown compression method '%s'") % method);
 }

src/libutil/local.mk

@@ -6,8 +6,8 @@ libutil_DIR := $(d)
 
 libutil_SOURCES := $(wildcard $(d)/*.cc)
 
-libutil_LDFLAGS = $(LIBLZMA_LIBS) -lbz2 -pthread $(OPENSSL_LIBS) $(LIBBROTLI_LIBS)
+libutil_LDFLAGS = $(LIBLZMA_LIBS) -lbz2 -pthread $(OPENSSL_LIBS)
 
 libutil_LIBS = libformat
 
-libutil_CXXFLAGS = -DBROTLI=\"$(brotli)\"
+libutil_CXXFLAGS = -DBRO=\"$(bro)\"

src/nix-daemon/nix-daemon.cc

@@ -411,7 +411,7 @@ static void performOp(TunnelLogger * logger, ref<LocalStore> store,
             /* Repairing is not atomic, so disallowed for "untrusted"
                clients.  */
             if (mode == bmRepair && !trusted)
-                throw Error("repairing is not allowed because you are not in 'trusted-users'");
+                throw Error("repairing is not supported when building through the Nix daemon");
         }
         logger->startWork();
         store->buildPaths(drvs, mode);

src/nix/local.mk

@@ -6,4 +6,6 @@ nix_SOURCES := $(wildcard $(d)/*.cc) $(wildcard src/linenoise/*.cpp)
 
 nix_LIBS = libexpr libmain libstore libutil libformat
 
+nix_LDFLAGS = -lfuse
+
 $(eval $(call install-symlink, nix, $(bindir)/nix-hash))

src/nix/mount.cc

@@ -0,0 +1,207 @@
+#include "command.hh"
+#include "store-api.hh"
+#include "fs-accessor.hh"
+#include "nar-accessor.hh"
+
+#define FUSE_USE_VERSION 30
+#include <fuse.h>
+
+#include <cstring>
+
+using namespace nix;
+
+std::shared_ptr<Store> store;
+std::shared_ptr<FSAccessor> accessor;
+
+static int op_getattr(const char * path_, struct stat * stbuf)
+{
+    try {
+
+        Path path(path_);
+
+        memset(stbuf, 0, sizeof(struct stat));
+        stbuf->st_uid = 0;
+        stbuf->st_gid = 0;
+        stbuf->st_nlink = 1;
+
+        if (path == "/") {
+            stbuf->st_mode = S_IFDIR | 0111;
+        } else {
+            auto st = accessor->stat(store->storeDir + path);
+
+            switch (st.type) {
+            case FSAccessor::tRegular:
+                stbuf->st_mode = S_IFREG | (st.isExecutable ? 0555 : 0444);
+                stbuf->st_size = st.fileSize;
+                break;
+            case FSAccessor::tSymlink:
+                stbuf->st_mode = S_IFLNK | 0777;
+                break;
+            case FSAccessor::tDirectory:
+                stbuf->st_mode = S_IFDIR | 0555;
+                break;
+            default:
+                return -ENOENT;
+            }
+        }
+
+        return 0;
+
+    } catch (...) {
+        ignoreException();
+        return -EIO;
+    }
+}
+
+static int op_readdir(const char * path_, void * buf, fuse_fill_dir_t filler,
+    off_t offset, struct fuse_file_info * fi)
+{
+    try {
+
+        Path path(path_);
+
+        if (path == "/")
+            /* FIXME: could use queryAllValidPaths(), but it will be
+               superslow for binary caches, and won't include name
+               parts. */
+            return 0;
+
+        auto st = accessor->stat(store->storeDir + path);
+        if (st.type == FSAccessor::tMissing) return -ENOENT;
+        if (st.type != FSAccessor::tDirectory) return -ENOTDIR;
+
+        for (auto & entry : accessor->readDirectory(store->storeDir + path))
+            filler(buf, entry.c_str(), nullptr, 0);
+
+        return 0;
+
+    } catch (...) {
+        ignoreException();
+        return -EIO;
+    }
+}
+
+static int op_open(const char * path_, struct fuse_file_info * fi)
+{
+    try {
+
+        Path path(path_);
+
+        auto st = accessor->stat(store->storeDir + path);
+        if (st.type == FSAccessor::tMissing) return -ENOENT;
+        if (st.type == FSAccessor::tDirectory) return -EISDIR;
+        if (st.type != FSAccessor::tRegular) return -EINVAL;
+
+        return 0;
+
+    } catch (...) {
+        ignoreException();
+        return -EIO;
+    }
+}
+
+static int op_read(const char * path_, char * buf, size_t size, off_t offset,
+    struct fuse_file_info * fi)
+{
+    try {
+
+        Path path(path_);
+
+        // FIXME: absolutely need to cache this and/or provide random
+        // access.
+
+        auto s = accessor->readFile(store->storeDir + path);
+
+        if (offset >= (off_t) s.size()) return 0;
+
+        if (offset + size > s.size())
+            size = s.size() - offset;
+
+        memcpy(buf, s.data() + offset, size);
+
+        return size;
+
+    } catch (...) {
+        ignoreException();
+        return -EIO;
+    }
+}
+
+static int op_readlink(const char * path_, char * buf, size_t size)
+{
+    try {
+
+        Path path(path_);
+
+        auto st = accessor->stat(store->storeDir + path);
+        if (st.type == FSAccessor::tMissing) return -ENOENT;
+        if (st.type != FSAccessor::tSymlink) return -EINVAL;
+
+        auto s = accessor->readLink(store->storeDir + path);
+
+        if (s.size() >= size) return ENAMETOOLONG; // FIXME
+
+        strncpy(buf, s.c_str(), size);
+
+        return 0;
+
+    } catch (...) {
+        ignoreException();
+        return -EIO;
+    }
+}
+
+struct CmdMountStore : StoreCommand
+{
+    Path mountPoint;
+
+    CmdMountStore()
+    {
+        expectArg("mount-point", &mountPoint);
+    }
+
+    std::string name() override
+    {
+        return "mount-store";
+    }
+
+    std::string description() override
+    {
+        return "mount a Nix store as a FUSE file system";
+    }
+
+    void run(ref<Store> store) override
+    {
+        ::store = store;
+        accessor = store->getFSAccessor();
+
+        Strings fuseArgs = { "nix", mountPoint, "-o", "debug" };
+        auto fuseArgs2 = stringsToCharPtrs(fuseArgs);
+
+        struct fuse * fuse;
+        char * mountpoint;
+        int multithreaded;
+
+        fuse_operations oper;
+        memset(&oper, 0, sizeof(oper));
+        oper.getattr = op_getattr;
+        oper.readdir = op_readdir;
+        oper.open = op_open;
+        oper.read = op_read;
+        oper.readlink = op_readlink;
+
+        fuse = fuse_setup(fuseArgs2.size() - 1, fuseArgs2.data(),
+            &oper, sizeof(oper),
+            &mountpoint, &multithreaded, nullptr);
+        if (!fuse) throw Error("FUSE setup failed");
+
+        if (multithreaded)
+            fuse_loop_mt(fuse);
+        else
+            fuse_loop(fuse);
+
+        fuse_teardown(fuse, mountpoint);
+    }
+};
+
+static RegisterCommand r(make_ref<CmdMountStore>());

src/nix/repl.cc

@@ -186,16 +186,7 @@ bool NixRepl::getLine(string & input, const std::string &prompt)
 {
     char * s = linenoise(prompt.c_str());
     Finally doFree([&]() { free(s); });
-    if (!s) {
-      switch (auto type = linenoiseKeyType()) {
-        case 1: // ctrl-C
-          return true;
-        case 2: // ctrl-D
-          return false;
-        default:
-          throw Error(format("Unexpected linenoise keytype: %1%") % type);
-      }
-    }
+    if (!s) return false;
     input += s;
     return true;
 }

tests/brotli.sh

@@ -1,28 +0,0 @@
-source common.sh
-
-
-# Only test if we found brotli libraries
-# (CLI tool is likely unavailable if libraries are missing)
-if [ -n "$HAVE_BROTLI" ]; then
-
-clearStore
-clearCache
-
-cacheURI="file://$cacheDir?compression=br"
-
-outPath=$(nix-build dependencies.nix --no-out-link)
-
-nix copy --to $cacheURI $outPath
-
-HASH=$(nix hash-path $outPath)
-
-clearStore
-clearCacheCache
-
-nix copy --from $cacheURI $outPath --no-check-sigs
-
-HASH2=$(nix hash-path $outPath)
-
-[[ $HASH = $HASH2 ]]
-
-fi # HAVE_BROTLI

tests/common.sh.in

@@ -32,7 +32,6 @@ export xmllint="@xmllint@"
 export SHELL="@bash@"
 export PAGER=cat
 export HAVE_SODIUM="@HAVE_SODIUM@"
-export HAVE_BROTLI="@HAVE_BROTLI@"
 
 export version=@PACKAGE_VERSION@
 export system=@system@

tests/fetchGit.sh

@@ -93,42 +93,3 @@ git -C $repo add hello
 git -C $repo commit -m 'Bla4'
 rev3=$(git -C $repo rev-parse HEAD)
 nix eval --tarball-ttl 3600 "(builtins.fetchGit { url = $repo; rev = \"$rev3\"; })" >/dev/null
-
-# Update 'path' to reflect latest master
-path=$(nix eval --raw "(builtins.fetchGit file://$repo).outPath")
-
-# Check behavior when non-master branch is used
-git -C $repo checkout $rev2 -b dev
-echo dev > $repo/hello
-
-# File URI uses 'master' unless specified otherwise
-path2=$(nix eval --raw "(builtins.fetchGit file://$repo).outPath")
-[[ $path = $path2 ]]
-
-# Using local path with branch other than 'master' should work when clean or dirty
-path3=$(nix eval --raw "(builtins.fetchGit $repo).outPath")
-# (check dirty-tree handling was used)
-[[ $(nix eval --raw "(builtins.fetchGit $repo).rev") = 0000000000000000000000000000000000000000 ]]
-
-# Committing shouldn't change store path, or switch to using 'master'
-git -C $repo commit -m 'Bla5' -a
-path4=$(nix eval --raw "(builtins.fetchGit $repo).outPath")
-[[ $(cat $path4/hello) = dev ]]
-[[ $path3 = $path4 ]]
-
-# Confirm same as 'dev' branch
-path5=$(nix eval --raw "(builtins.fetchGit { url = $repo; ref = \"dev\"; }).outPath")
-[[ $path3 = $path5 ]]
-
-
-# Nuke the cache
-rm -rf $TEST_HOME/.cache/nix/git
-
-# Try again, but without 'git' on PATH
-NIX=$(command -v nix)
-# This should fail
-(! PATH= $NIX eval --raw "(builtins.fetchGit { url = $repo; ref = \"dev\"; }).outPath" )
-
-# Try again, with 'git' available.  This should work.
-path5=$(nix eval --raw "(builtins.fetchGit { url = $repo; ref = \"dev\"; }).outPath")
-[[ $path3 = $path5 ]]

tests/local.mk

@@ -19,8 +19,7 @@ nix_tests = \
   fetchGit.sh \
   fetchMercurial.sh \
   signing.sh \
-  run.sh \
-  brotli.sh
+  run.sh
   # parallel.sh
 
 install-tests += $(foreach x, $(nix_tests), tests/$(x))

tests/nix-copy-closure.nix

@@ -1,8 +1,8 @@
 # Test ‘nix-copy-closure’.
 
-{ nixpkgs, system, nix }:
+{ system, nix }:
 
-with import (nixpkgs + /nixos/lib/testing.nix) { inherit system; };
+with import <nixpkgs/nixos/lib/testing.nix> { inherit system; };
 
 makeTest (let pkgA = pkgs.cowsay; pkgB = pkgs.wget; pkgC = pkgs.hello; in {
 

tests/remote-builds.nix

@@ -1,8 +1,8 @@
 # Test Nix's remote build feature.
 
-{ nixpkgs, system, nix }:
+{ system, nix }:
 
-with import (nixpkgs + "/nixos/lib/testing.nix") { inherit system; };
+with import <nixpkgs/nixos/lib/testing.nix> { inherit system; };
 
 makeTest (
 

tests/run.sh

@@ -18,10 +18,10 @@ if [[ $(uname) = Linux ]]; then
     # Note: we need the sandbox paths to ensure that the shell is
     # visible in the sandbox.
     nix run --sandbox-build-dir /build-tmp \
-        --sandbox-paths '/nix? /bin? /lib? /lib64? /usr?' \
+        --sandbox-paths '/nix? /bin? /lib? /usr?' \
         --store $TEST_ROOT/store0 -f run.nix hello -c hello | grep 'Hello World'
 
-    path2=$(nix run --sandbox-paths '/nix? /bin? /lib? /lib64? /usr?' --store $TEST_ROOT/store0 -f run.nix hello -c $SHELL -c 'type -p hello')
+    path2=$(nix run --sandbox-paths '/nix? /bin? /lib? /usr?' --store $TEST_ROOT/store0 -f run.nix hello -c $SHELL -c 'type -p hello')
 
     [[ $path/bin/hello = $path2 ]]
 

tests/setuid.nix

@@ -1,8 +1,8 @@
 # Verify that Linux builds cannot create setuid or setgid binaries.
 
-{ nixpkgs, system, nix }:
+{ system, nix }:
 
-with import (nixpkgs + "/nixos/lib/testing.nix") { inherit system; };
+with import <nixpkgs/nixos/lib/testing.nix> { inherit system; };
 
 makeTest {
 

tests/shell.shebang.sh

@@ -1,4 +1,4 @@
 #! @ENV_PROG@ nix-shell
-#! nix-shell -I nixpkgs=shell.nix --no-substitute
+#! nix-shell -I nixpkgs=shell.nix --no-use-substitutes
 #! nix-shell --pure -i bash -p foo bar
 echo "$(foo) $(bar) $@"