Warn when experimental features are used

If you use an experimental feature not listed in the
'experimental-features' setting, you now get a warning rather than a
fatal error. This makes things like the 'nix' command a bit less
obnoxious.

However, if you set 'allow-experimental-features = false', it's still
a fatal error. This is primarily for CI systems where use of
experimental features in the daemon could go unnoticed.

.dir-locals.el

@@ -1,7 +1,6 @@
 ((c++-mode . (
   (c-file-style . "k&r")
   (c-basic-offset . 4)
-  (c-block-comment-prefix . "  ")
   (indent-tabs-mode . nil)
   (tab-width . 4)
   (show-trailing-whitespace . t)

.github/dependabot.yml

@@ -1,6 +0,0 @@
-version: 2
-updates:
-  - package-ecosystem: "github-actions"
-    directory: "/"
-    schedule:
-      interval: "weekly"

.github/workflows/test.yml

@@ -10,15 +10,5 @@ jobs:
     runs-on: ${{ matrix.os }}
     steps:
     - uses: actions/checkout@v2
-    - uses: cachix/install-nix-action@v10
-    - run: nix-build release.nix --arg nix '{ outPath = ./.; revCount = 123; shortRev = "abcdefgh"; }' --arg systems '[ builtins.currentSystem ]' -A installerScript -A perlBindings
-  macos_perf_test:
-    runs-on: macos-latest
-    steps:
-    - name: Disable syspolicy assessments
-      run: |
-        spctl --status
-        sudo spctl --master-disable
-    - uses: actions/checkout@v2
-    - uses: cachix/install-nix-action@v10
+    - uses: cachix/install-nix-action@v8
     - run: nix-build release.nix --arg nix '{ outPath = ./.; revCount = 123; shortRev = "abcdefgh"; }' --arg systems '[ builtins.currentSystem ]' -A installerScript -A perlBindings

maintainers/upload-release.pl

@@ -170,5 +170,15 @@ $channelsBucket->add_key(
 chdir("/home/eelco/Dev/nix-pristine") or die;
 system("git remote update origin") == 0 or die;
 system("git tag --force --sign $version $nixRev -m 'Tagging release $version'") == 0 or die;
-system("git push --tags") == 0 or die;
-system("git push --force-with-lease origin $nixRev:refs/heads/latest-release") == 0 or die;
+
+# Update the website.
+my $siteDir = "/home/eelco/Dev/nixos-homepage-pristine";
+
+system("cd $siteDir && git pull") == 0 or die;
+
+write_file("$siteDir/nix-release.tt",
+           "[%-\n" .
+           "latestNixVersion = \"$version\"\n" .
+           "-%]\n");
+
+system("cd $siteDir && git commit -a -m 'Nix $version released'") == 0 or die;

mk/run_test.sh

@@ -1,28 +0,0 @@
-#!/bin/sh
-
-set -u
-
-red=""
-green=""
-yellow=""
-normal=""
-
-post_run_msg="ran test $1..."
-if [ -t 1 ]; then
-    red=""
-    green=""
-    yellow=""
-    normal=""
-fi
-(cd $(dirname $1) && env ${TESTS_ENVIRONMENT} init.sh 2>/dev/null > /dev/null)
-log="$(cd $(dirname $1) && env ${TESTS_ENVIRONMENT} $(basename $1) 2>&1)"
-status=$?
-if [ $status -eq 0 ]; then
-  echo "$post_run_msg [${green}PASS$normal]"
-elif [ $status -eq 99 ]; then
-  echo "$post_run_msg [${yellow}SKIP$normal]"
-else
-  echo "$post_run_msg [${red}FAIL$normal]"
-  echo "$log" | sed 's/^/    /'
-  exit "$status"
-fi

mk/tests.mk

@@ -1,15 +1,45 @@
 # Run program $1 as part of ‘make installcheck’.
-
-test-deps =
-
 define run-install-test
 
-  installcheck: $1.test
+  installcheck: $1
 
-  .PHONY: $1.test
-  $1.test: $1 $(test-deps)
-	@env TEST_NAME=$(notdir $(basename $1)) TESTS_ENVIRONMENT="$(tests-environment)" mk/run_test.sh $1
+  _installcheck-list += $1
 
 endef
 
+# Color code from https://unix.stackexchange.com/a/10065
+installcheck:
+	@total=0; failed=0; \
+	red=""; \
+	green=""; \
+	yellow=""; \
+	normal=""; \
+	if [ -t 1 ]; then \
+		red=""; \
+		green=""; \
+		yellow=""; \
+		normal=""; \
+	fi; \
+	for i in $(_installcheck-list); do \
+	  total=$$((total + 1)); \
+	  printf "running test $$i..."; \
+	  log="$$(cd $$(dirname $$i) && $(tests-environment) $$(basename $$i) 2>&1)"; \
+	  status=$$?; \
+	  if [ $$status -eq 0 ]; then \
+	    echo " [$${green}PASS$$normal]"; \
+	  elif [ $$status -eq 99 ]; then \
+	    echo " [$${yellow}SKIP$$normal]"; \
+	  else \
+	    echo " [$${red}FAIL$$normal]"; \
+	    echo "$$log" | sed 's/^/    /'; \
+	    failed=$$((failed + 1)); \
+	  fi; \
+	done; \
+	if [ "$$failed" != 0 ]; then \
+	  echo "$${red}$$failed out of $$total tests failed $$normal"; \
+	  exit 1; \
+	else \
+		echo "$${green}All tests succeeded$$normal"; \
+	fi
+
 .PHONY: check installcheck

perl/lib/Nix/Store.xs

@@ -182,7 +182,7 @@ void importPaths(int fd, int dontCheckSigs)
     PPCODE:
         try {
             FdSource source(fd);
-            store()->importPaths(source, dontCheckSigs ? NoCheckSigs : CheckSigs);
+            store()->importPaths(source, nullptr, dontCheckSigs ? NoCheckSigs : CheckSigs);
         } catch (Error & e) {
             croak("%s", e.what());
         }

src/libexpr/attr-path.cc

@@ -130,7 +130,7 @@ Pos findDerivationFilename(EvalState & state, Value & v, std::string what)
 
     Symbol file = state.symbols.create(filename);
 
-    return { foFile, file, lineno, 0 };
+    return { file, lineno, 0 };
 }
 
 

src/libexpr/attr-set.hh

@@ -78,7 +78,7 @@ public:
         if (!a)
             throw Error({
                 .hint = hintfmt("attribute '%s' missing", name),
-                .errPos = pos
+                .nixCode = NixCode { .errPos = pos }
             });
 
         return *a;

src/libexpr/eval-inline.hh

@@ -11,7 +11,7 @@ LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s))
 {
     throw EvalError({
         .hint = hintfmt(s),
-        .errPos = pos
+        .nixCode = NixCode { .errPos = pos }
     });
 }
 
@@ -25,7 +25,7 @@ LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s, const
 {
     throw TypeError({
         .hint = hintfmt(s, showType(v)),
-        .errPos = pos
+        .nixCode = NixCode { .errPos = pos }
     });
 }
 

src/libexpr/eval.cc

@@ -366,7 +366,7 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
 
             if (store->isInStore(r.second)) {
                 StorePathSet closure;
-                store->computeFSClosure(store->toStorePath(r.second).first, closure);
+                store->computeFSClosure(store->parseStorePath(store->toStorePath(r.second)), closure);
                 for (auto & path : closure)
                     allowedPaths->insert(store->printStorePath(path));
             } else
@@ -529,7 +529,7 @@ LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s, const
 {
     throw EvalError({
         .hint = hintfmt(s, s2),
-        .errPos = pos
+        .nixCode = NixCode { .errPos = pos }
     });
 }
 
@@ -542,7 +542,7 @@ LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s, const
 {
     throw EvalError({
         .hint = hintfmt(s, s2, s3),
-        .errPos = pos
+        .nixCode = NixCode { .errPos = pos }
     });
 }
 
@@ -551,7 +551,7 @@ LocalNoInlineNoReturn(void throwEvalError(const Pos & p1, const char * s, const
     // p1 is where the error occurred; p2 is a position mentioned in the message.
     throw EvalError({
         .hint = hintfmt(s, sym, p2),
-        .errPos = p1
+        .nixCode = NixCode { .errPos = p1 }
     });
 }
 
@@ -559,7 +559,7 @@ LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s))
 {
     throw TypeError({
         .hint = hintfmt(s),
-        .errPos = pos
+        .nixCode = NixCode { .errPos = pos }
     });
 }
 
@@ -572,7 +572,7 @@ LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s, const
 {
     throw TypeError({
         .hint = hintfmt(s, fun.showNamePos(), s2),
-        .errPos = pos
+        .nixCode = NixCode { .errPos = pos }
     });
 }
 
@@ -580,7 +580,7 @@ LocalNoInlineNoReturn(void throwAssertionError(const Pos & pos, const char * s,
 {
     throw AssertionError({
         .hint = hintfmt(s, s1),
-        .errPos = pos
+        .nixCode = NixCode { .errPos = pos }
     });
 }
 
@@ -588,18 +588,23 @@ LocalNoInlineNoReturn(void throwUndefinedVarError(const Pos & pos, const char *
 {
     throw UndefinedVarError({
         .hint = hintfmt(s, s1),
-        .errPos = pos
+        .nixCode = NixCode { .errPos = pos }
     });
 }
 
-LocalNoInline(void addErrorTrace(Error & e, const char * s, const string & s2))
+LocalNoInline(void addErrorPrefix(Error & e, const char * s, const string & s2))
 {
-    e.addTrace(std::nullopt, s, s2);
+    e.addPrefix(format(s) % s2);
 }
 
-LocalNoInline(void addErrorTrace(Error & e, const Pos & pos, const char * s, const string & s2))
+LocalNoInline(void addErrorPrefix(Error & e, const char * s, const ExprLambda & fun, const Pos & pos))
 {
-    e.addTrace(pos, s, s2);
+    e.addPrefix(format(s) % fun.showNamePos() % pos);
+}
+
+LocalNoInline(void addErrorPrefix(Error & e, const char * s, const string & s2, const Pos & pos))
+{
+    e.addPrefix(format(s) % s2 % pos);
 }
 
 
@@ -813,7 +818,7 @@ void EvalState::evalFile(const Path & path_, Value & v)
     try {
         eval(e, v);
     } catch (Error & e) {
-        addErrorTrace(e, "while evaluating the file '%1%':", path2);
+        addErrorPrefix(e, "while evaluating the file '%1%':\n", path2);
         throw;
     }
 
@@ -1063,8 +1068,8 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v)
 
     } catch (Error & e) {
         if (pos2 && pos2->file != state.sDerivationNix)
-            addErrorTrace(e, *pos2, "while evaluating the attribute '%1%'",
-                showAttrPath(state, env, attrPath));
+            addErrorPrefix(e, "while evaluating the attribute '%1%' at %2%:\n",
+                showAttrPath(state, env, attrPath), *pos2);
         throw;
     }
 
@@ -1232,15 +1237,11 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po
 
     /* Evaluate the body.  This is conditional on showTrace, because
        catching exceptions makes this function not tail-recursive. */
-    if (loggerSettings.showTrace.get())
+    if (settings.showTrace)
         try {
             lambda.body->eval(*this, env2, v);
         } catch (Error & e) {
-            addErrorTrace(e, lambda.pos, "while evaluating %s", 
-              (lambda.name.set() 
-                  ? "'" + (string) lambda.name + "'" 
-                  : "anonymous lambdaction"));
-            addErrorTrace(e, pos, "from call site%s", "");
+            addErrorPrefix(e, "while evaluating %1%, called from %2%:\n", lambda, pos);
             throw;
         }
     else
@@ -1515,7 +1516,7 @@ void EvalState::forceValueDeep(Value & v)
                 try {
                     recurse(*i.value);
                 } catch (Error & e) {
-                    addErrorTrace(e, *i.pos, "while evaluating the attribute '%1%'", i.name);
+                    addErrorPrefix(e, "while evaluating the attribute '%1%' at %2%:\n", i.name, *i.pos);
                     throw;
                 }
         }
@@ -1935,7 +1936,7 @@ string ExternalValueBase::coerceToString(const Pos & pos, PathSet & context, boo
 {
     throw TypeError({
         .hint = hintfmt("cannot coerce %1% to a string", showType()),
-        .errPos = pos
+        .nixCode = NixCode { .errPos = pos }
     });
 }
 

src/libexpr/eval.hh

@@ -250,7 +250,7 @@ private:
     friend struct ExprAttrs;
     friend struct ExprLet;
 
-    Expr * parse(const char * text, FileOrigin origin, const Path & path,
+    Expr * parse(const char * text, const Path & path,
         const Path & basePath, StaticEnv & staticEnv);
 
 public:

src/libexpr/nixexpr.cc

@@ -197,22 +197,7 @@ std::ostream & operator << (std::ostream & str, const Pos & pos)
     if (!pos)
         str << "undefined position";
     else
-    {
-        auto f = format(ANSI_BOLD "%1%" ANSI_NORMAL ":%2%:%3%");
-        switch (pos.origin) {
-            case foFile:
-                f % (string) pos.file;
-                break;
-            case foStdin:
-            case foString:
-                f % "(string)";
-                break;
-            default:
-                throw Error("unhandled Pos origin!");
-        }
-        str << (f % pos.line % pos.column).str();
-    }
-
+        str << (format(ANSI_BOLD "%1%" ANSI_NORMAL ":%2%:%3%") % (string) pos.file % pos.line % pos.column).str();
     return str;
 }
 
@@ -285,7 +270,7 @@ void ExprVar::bindVars(const StaticEnv & env)
     if (withLevel == -1)
         throw UndefinedVarError({
             .hint = hintfmt("undefined variable '%1%'", name),
-            .errPos = pos
+            .nixCode = NixCode { .errPos = pos }
         });
     fromWith = true;
     this->level = withLevel;

src/libexpr/nixexpr.hh

@@ -24,12 +24,11 @@ MakeError(RestrictedPathError, Error);
 
 struct Pos
 {
-    FileOrigin origin;
     Symbol file;
     unsigned int line, column;
-    Pos() : origin(foString), line(0), column(0) { };
-    Pos(FileOrigin origin, const Symbol & file, unsigned int line, unsigned int column)
-        : origin(origin), file(file), line(line), column(column) { };
+    Pos() : line(0), column(0) { };
+    Pos(const Symbol & file, unsigned int line, unsigned int column)
+        : file(file), line(line), column(column) { };
     operator bool() const
     {
         return line != 0;
@@ -239,7 +238,7 @@ struct ExprLambda : Expr
         if (!arg.empty() && formals && formals->argNames.find(arg) != formals->argNames.end())
             throw ParseError({
                 .hint = hintfmt("duplicate formal function argument '%1%'", arg),
-                .errPos = pos
+                .nixCode = NixCode { .errPos = pos }
             });
     };
     void setName(Symbol & name);

src/libexpr/parser.y

@@ -30,8 +30,7 @@ namespace nix {
         SymbolTable & symbols;
         Expr * result;
         Path basePath;
-        Symbol file;
-        FileOrigin origin;
+        Symbol path;
         ErrorInfo error;
         Symbol sLetBody;
         ParseData(EvalState & state)
@@ -66,17 +65,18 @@ namespace nix {
 static void dupAttr(const AttrPath & attrPath, const Pos & pos, const Pos & prevPos)
 {
     throw ParseError({
-         .hint = hintfmt("attribute '%1%' already defined at %2%",
+        .hint = hintfmt("attribute '%1%' already defined at %2%",
             showAttrPath(attrPath), prevPos),
-         .errPos = pos
+        .nixCode = NixCode { .errPos = pos },
     });
 }
 
+
 static void dupAttr(Symbol attr, const Pos & pos, const Pos & prevPos)
 {
     throw ParseError({
         .hint = hintfmt("attribute '%1%' already defined at %2%", attr, prevPos),
-        .errPos = pos
+        .nixCode = NixCode { .errPos = pos },
     });
 }
 
@@ -148,7 +148,7 @@ static void addFormal(const Pos & pos, Formals * formals, const Formal & formal)
         throw ParseError({
             .hint = hintfmt("duplicate formal function argument '%1%'",
                 formal.name),
-            .errPos = pos
+            .nixCode = NixCode { .errPos = pos },
         });
     formals->formals.push_front(formal);
 }
@@ -246,7 +246,7 @@ static Expr * stripIndentation(const Pos & pos, SymbolTable & symbols, vector<Ex
 
 static inline Pos makeCurPos(const YYLTYPE & loc, ParseData * data)
 {
-    return Pos(data->origin, data->file, loc.first_line, loc.first_column);
+    return Pos(data->path, loc.first_line, loc.first_column);
 }
 
 #define CUR_POS makeCurPos(*yylocp, data)
@@ -259,7 +259,7 @@ void yyerror(YYLTYPE * loc, yyscan_t scanner, ParseData * data, const char * err
 {
     data->error = {
         .hint = hintfmt(error),
-        .errPos = makeCurPos(*loc, data)
+        .nixCode = NixCode { .errPos = makeCurPos(*loc, data) }
     };
 }
 
@@ -339,7 +339,7 @@ expr_function
     { if (!$2->dynamicAttrs.empty())
         throw ParseError({
             .hint = hintfmt("dynamic attributes not allowed in let"),
-            .errPos = CUR_POS
+            .nixCode = NixCode { .errPos = CUR_POS },
         });
       $$ = new ExprLet($2, $4);
     }
@@ -419,7 +419,7 @@ expr_simple
       if (noURLLiterals)
           throw ParseError({
               .hint = hintfmt("URL literals are disabled"),
-              .errPos = CUR_POS
+              .nixCode = NixCode { .errPos = CUR_POS }
           });
       $$ = new ExprString(data->symbols.create($1));
   }
@@ -492,7 +492,7 @@ attrs
       } else
           throw ParseError({
               .hint = hintfmt("dynamic attributes not allowed in inherit"),
-              .errPos = makeCurPos(@2, data)
+              .nixCode = NixCode { .errPos = makeCurPos(@2, data) },
           });
     }
   | { $$ = new AttrPath; }
@@ -569,24 +569,13 @@ formal
 namespace nix {
 
 
-Expr * EvalState::parse(const char * text, FileOrigin origin,
+Expr * EvalState::parse(const char * text,
     const Path & path, const Path & basePath, StaticEnv & staticEnv)
 {
     yyscan_t scanner;
     ParseData data(*this);
-    data.origin = origin;
-    switch (origin) {
-        case foFile: 
-            data.file = data.symbols.create(path);
-            break;
-        case foStdin:
-        case foString:
-            data.file = data.symbols.create(text);
-            break;
-        default:
-            assert(false);
-    }
     data.basePath = basePath;
+    data.path = data.symbols.create(path);
 
     yylex_init(&scanner);
     yy_scan_string(text, scanner);
@@ -636,13 +625,13 @@ Expr * EvalState::parseExprFromFile(const Path & path)
 
 Expr * EvalState::parseExprFromFile(const Path & path, StaticEnv & staticEnv)
 {
-    return parse(readFile(path).c_str(), foFile, path, dirOf(path), staticEnv);
+    return parse(readFile(path).c_str(), path, dirOf(path), staticEnv);
 }
 
 
 Expr * EvalState::parseExprFromString(std::string_view s, const Path & basePath, StaticEnv & staticEnv)
 {
-    return parse(s.data(), foString, "", basePath, staticEnv);
+    return parse(s.data(), "(string)", basePath, staticEnv);
 }
 
 
@@ -655,7 +644,7 @@ Expr * EvalState::parseExprFromString(std::string_view s, const Path & basePath)
 Expr * EvalState::parseStdin()
 {
     //Activity act(*logger, lvlTalkative, format("parsing standard input"));
-    return parse(drainFD(0).data(), foStdin, "", absPath("."), staticBaseEnv);
+    return parseExprFromString(drainFD(0), absPath("."));
 }
 
 
@@ -704,7 +693,7 @@ Path EvalState::findFile(SearchPath & searchPath, const string & path, const Pos
             ? "cannot look up '<%s>' in pure evaluation mode (use '--impure' to override)"
             : "file '%s' was not found in the Nix search path (add it using $NIX_PATH or -I)",
             path),
-        .errPos = pos
+        .nixCode = NixCode { .errPos = pos }
     });
 }
 

src/libexpr/primops.cc

@@ -96,7 +96,7 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args
     } catch (InvalidPathError & e) {
         throw EvalError({
             .hint = hintfmt("cannot import '%1%', since path '%2%' is not valid", path, e.path),
-            .errPos = pos
+            .nixCode = NixCode { .errPos = pos }
         });
     }
 
@@ -177,7 +177,7 @@ void prim_importNative(EvalState & state, const Pos & pos, Value * * args, Value
             .hint = hintfmt(
                 "cannot import '%1%', since path '%2%' is not valid",
                 path, e.path),
-            .errPos = pos
+            .nixCode = NixCode { .errPos = pos }
         });
     }
 
@@ -215,7 +215,7 @@ void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v)
     if (count == 0) {
         throw EvalError({
             .hint = hintfmt("at least one argument to 'exec' required"),
-            .errPos = pos
+            .nixCode = NixCode { .errPos = pos }
         });
     }
     PathSet context;
@@ -230,7 +230,7 @@ void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v)
         throw EvalError({
             .hint = hintfmt("cannot execute '%1%', since path '%2%' is not valid",
                 program, e.path),
-            .errPos = pos
+            .nixCode = NixCode { .errPos = pos }
         });
     }
 
@@ -239,13 +239,13 @@ void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v)
     try {
         parsed = state.parseExprFromString(output, pos.file);
     } catch (Error & e) {
-        e.addTrace(pos, "While parsing the output from '%1%'", program);
+        e.addPrefix(fmt("While parsing the output from '%1%', at %2%\n", program, pos));
         throw;
     }
     try {
         state.eval(parsed, v);
     } catch (Error & e) {
-        e.addTrace(pos, "While evaluating the output from '%1%'", program);
+        e.addPrefix(fmt("While evaluating the output from '%1%', at %2%\n", program, pos));
         throw;
     }
 }
@@ -385,7 +385,7 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar
     if (startSet == args[0]->attrs->end())
         throw EvalError({
             .hint = hintfmt("attribute 'startSet' required"),
-            .errPos = pos
+            .nixCode = NixCode { .errPos = pos }
         });
     state.forceList(*startSet->value, pos);
 
@@ -399,7 +399,7 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar
     if (op == args[0]->attrs->end())
         throw EvalError({
             .hint = hintfmt("attribute 'operator' required"),
-            .errPos = pos
+            .nixCode = NixCode { .errPos = pos }
         });
     state.forceValue(*op->value, pos);
 
@@ -421,7 +421,7 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar
         if (key == e->attrs->end())
             throw EvalError({
                 .hint = hintfmt("attribute 'key' required"),
-                .errPos = pos
+                .nixCode = NixCode { .errPos = pos }
             });
         state.forceValue(*key->value, pos);
 
@@ -471,7 +471,7 @@ static void prim_addErrorContext(EvalState & state, const Pos & pos, Value * * a
         v = *args[1];
     } catch (Error & e) {
         PathSet context;
-        e.addTrace(std::nullopt, state.coerceToString(pos, *args[0], context));
+        e.addPrefix(format("%1%\n") % state.coerceToString(pos, *args[0], context));
         throw;
     }
 }
@@ -556,14 +556,14 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
     if (attr == args[0]->attrs->end())
         throw EvalError({
             .hint = hintfmt("required attribute 'name' missing"),
-            .errPos = pos
+            .nixCode = NixCode { .errPos = pos }
         });
     string drvName;
     Pos & posDrvName(*attr->pos);
     try {
         drvName = state.forceStringNoCtx(*attr->value, pos);
     } catch (Error & e) {
-        e.addTrace(posDrvName, "while evaluating the derivation attribute 'name'");
+        e.addPrefix(fmt("while evaluating the derivation attribute 'name' at %1%:\n", posDrvName));
         throw;
     }
 
@@ -603,7 +603,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
             else
                 throw EvalError({
                     .hint = hintfmt("invalid value '%s' for 'outputHashMode' attribute", s),
-                    .errPos = posDrvName
+                    .nixCode = NixCode { .errPos = posDrvName }
                 });
         };
 
@@ -613,7 +613,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
                 if (outputs.find(j) != outputs.end())
                     throw EvalError({
                         .hint = hintfmt("duplicate derivation output '%1%'", j),
-                        .errPos = posDrvName
+                        .nixCode = NixCode { .errPos = posDrvName }
                     });
                 /* !!! Check whether j is a valid attribute
                    name. */
@@ -623,14 +623,14 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
                 if (j == "drv")
                     throw EvalError({
                         .hint = hintfmt("invalid derivation output name 'drv'" ),
-                        .errPos = posDrvName
+                        .nixCode = NixCode { .errPos = posDrvName }
                     });
                 outputs.insert(j);
             }
             if (outputs.empty())
                 throw EvalError({
                     .hint = hintfmt("derivation cannot have an empty set of outputs"),
-                    .errPos = posDrvName
+                    .nixCode = NixCode { .errPos = posDrvName }
                 });
         };
 
@@ -696,9 +696,8 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
             }
 
         } catch (Error & e) {
-            e.addTrace(posDrvName, 
-                "while evaluating the attribute '%1%' of the derivation '%2%'",
-                key, drvName);
+            e.addPrefix(format("while evaluating the attribute '%1%' of the derivation '%2%' at %3%:\n")
+                % key % drvName % posDrvName);
             throw;
         }
     }
@@ -746,20 +745,20 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
     if (drv.builder == "")
         throw EvalError({
             .hint = hintfmt("required attribute 'builder' missing"),
-            .errPos = posDrvName
+            .nixCode = NixCode { .errPos = posDrvName }
         });
 
     if (drv.platform == "")
         throw EvalError({
             .hint = hintfmt("required attribute 'system' missing"),
-            .errPos = posDrvName
+            .nixCode = NixCode { .errPos = posDrvName }
         });
 
     /* Check whether the derivation name is valid. */
     if (isDerivation(drvName))
         throw EvalError({
             .hint = hintfmt("derivation names are not allowed to end in '%s'", drvExtension),
-            .errPos = posDrvName
+            .nixCode = NixCode { .errPos = posDrvName }
         });
 
     if (outputHash) {
@@ -767,7 +766,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
         if (outputs.size() != 1 || *(outputs.begin()) != "out")
             throw Error({
                 .hint = hintfmt("multiple outputs are not supported in fixed-output derivations"),
-                .errPos = posDrvName
+                .nixCode = NixCode { .errPos = posDrvName }
             });
 
         std::optional<HashType> ht = parseHashTypeOpt(outputHashAlgo);
@@ -881,12 +880,12 @@ static void prim_storePath(EvalState & state, const Pos & pos, Value * * args, V
     if (!state.store->isInStore(path))
         throw EvalError({
             .hint = hintfmt("path '%1%' is not in the Nix store", path),
-            .errPos = pos
+            .nixCode = NixCode { .errPos = pos }
         });
-    auto path2 = state.store->toStorePath(path).first;
+    Path path2 = state.store->toStorePath(path);
     if (!settings.readOnlyMode)
-        state.store->ensurePath(path2);
-    context.insert(state.store->printStorePath(path2));
+        state.store->ensurePath(state.store->parseStorePath(path2));
+    context.insert(path2);
     mkString(v, path, context);
 }
 
@@ -902,7 +901,7 @@ static void prim_pathExists(EvalState & state, const Pos & pos, Value * * args,
             .hint = hintfmt(
                 "cannot check the existence of '%1%', since path '%2%' is not valid",
                 path, e.path),
-            .errPos = pos
+            .nixCode = NixCode { .errPos = pos }
         });
     }
 
@@ -948,7 +947,7 @@ static void prim_readFile(EvalState & state, const Pos & pos, Value * * args, Va
     } catch (InvalidPathError & e) {
         throw EvalError({
             .hint = hintfmt("cannot read '%1%', since path '%2%' is not valid", path, e.path),
-            .errPos = pos
+            .nixCode = NixCode { .errPos = pos }
         });
     }
     string s = readFile(state.checkSourcePath(state.toRealPath(path, context)));
@@ -979,7 +978,7 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va
         if (i == v2.attrs->end())
             throw EvalError({
                 .hint = hintfmt("attribute 'path' missing"),
-                .errPos = pos
+                .nixCode = NixCode { .errPos = pos }
             });
 
         PathSet context;
@@ -990,7 +989,7 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va
         } catch (InvalidPathError & e) {
             throw EvalError({
                 .hint = hintfmt("cannot find '%1%', since path '%2%' is not valid", path, e.path),
-                .errPos = pos
+                .nixCode = NixCode { .errPos = pos }
             });
         }
 
@@ -1010,7 +1009,7 @@ static void prim_hashFile(EvalState & state, const Pos & pos, Value * * args, Va
     if (!ht)
       throw Error({
           .hint = hintfmt("unknown hash type '%1%'", type),
-          .errPos = pos
+          .nixCode = NixCode { .errPos = pos }
       });
 
     PathSet context; // discarded
@@ -1029,7 +1028,7 @@ static void prim_readDir(EvalState & state, const Pos & pos, Value * * args, Val
     } catch (InvalidPathError & e) {
         throw EvalError({
             .hint = hintfmt("cannot read '%1%', since path '%2%' is not valid", path, e.path),
-            .errPos = pos
+            .nixCode = NixCode { .errPos = pos }
         });
     }
 
@@ -1105,7 +1104,7 @@ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Valu
                     "in 'toFile': the file named '%1%' must not contain a reference "
                     "to a derivation but contains (%2%)",
                     name, path),
-                .errPos = pos
+                .nixCode = NixCode { .errPos = pos }
             });
         refs.insert(state.store->parseStorePath(path));
     }
@@ -1176,7 +1175,7 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args
     if (!context.empty())
         throw EvalError({
             .hint = hintfmt("string '%1%' cannot refer to other paths", path),
-            .errPos = pos
+            .nixCode = NixCode { .errPos = pos }
         });
 
     state.forceValue(*args[0], pos);
@@ -1185,7 +1184,7 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args
             .hint = hintfmt(
                 "first argument in call to 'filterSource' is not a function but %1%",
                 showType(*args[0])),
-            .errPos = pos
+            .nixCode = NixCode { .errPos = pos }
         });
 
     addPath(state, pos, std::string(baseNameOf(path)), path, args[0], FileIngestionMethod::Recursive, Hash(), v);
@@ -1208,7 +1207,7 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value
             if (!context.empty())
                 throw EvalError({
                     .hint = hintfmt("string '%1%' cannot refer to other paths", path),
-                    .errPos = *attr.pos
+                    .nixCode = NixCode { .errPos = *attr.pos }
                 });
         } else if (attr.name == state.sName)
             name = state.forceStringNoCtx(*attr.value, *attr.pos);
@@ -1222,13 +1221,13 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value
         else
             throw EvalError({
                 .hint = hintfmt("unsupported argument '%1%' to 'addPath'", attr.name),
-                .errPos = *attr.pos
+                .nixCode = NixCode { .errPos = *attr.pos }
             });
     }
     if (path.empty())
         throw EvalError({
             .hint = hintfmt("'path' required"),
-            .errPos = pos
+            .nixCode = NixCode { .errPos = pos }
         });
     if (name.empty())
         name = baseNameOf(path);
@@ -1289,7 +1288,7 @@ void prim_getAttr(EvalState & state, const Pos & pos, Value * * args, Value & v)
     if (i == args[1]->attrs->end())
         throw EvalError({
             .hint = hintfmt("attribute '%1%' missing", attr),
-            .errPos = pos
+            .nixCode = NixCode { .errPos = pos }
         });
     // !!! add to stack trace?
     if (state.countCalls && i->pos) state.attrSelects[*i->pos]++;
@@ -1372,7 +1371,7 @@ static void prim_listToAttrs(EvalState & state, const Pos & pos, Value * * args,
         if (j == v2.attrs->end())
             throw TypeError({
                 .hint = hintfmt("'name' attribute missing in a call to 'listToAttrs'"),
-                .errPos = pos
+                .nixCode = NixCode { .errPos = pos }
             });
         string name = state.forceStringNoCtx(*j->value, pos);
 
@@ -1382,7 +1381,7 @@ static void prim_listToAttrs(EvalState & state, const Pos & pos, Value * * args,
             if (j2 == v2.attrs->end())
                 throw TypeError({
                     .hint = hintfmt("'value' attribute missing in a call to 'listToAttrs'"),
-                    .errPos = pos
+                    .nixCode = NixCode { .errPos = pos }
                 });
             v.attrs->push_back(Attr(sym, j2->value, j2->pos));
         }
@@ -1458,7 +1457,7 @@ static void prim_functionArgs(EvalState & state, const Pos & pos, Value * * args
     if (args[0]->type != tLambda)
         throw TypeError({
             .hint = hintfmt("'functionArgs' requires a function"),
-            .errPos = pos
+            .nixCode = NixCode { .errPos = pos }
         });
 
     if (!args[0]->lambda.fun->matchAttrs) {
@@ -1514,7 +1513,7 @@ static void elemAt(EvalState & state, const Pos & pos, Value & list, int n, Valu
     if (n < 0 || (unsigned int) n >= list.listSize())
         throw Error({
             .hint = hintfmt("list index %1% is out of bounds", n),
-            .errPos = pos
+            .nixCode = NixCode { .errPos = pos }
         });
     state.forceValue(*list.listElems()[n], pos);
     v = *list.listElems()[n];
@@ -1544,7 +1543,7 @@ static void prim_tail(EvalState & state, const Pos & pos, Value * * args, Value
     if (args[0]->listSize() == 0)
         throw Error({
             .hint = hintfmt("'tail' called on an empty list"),
-            .errPos = pos
+            .nixCode = NixCode { .errPos = pos }
         });
 
     state.mkList(v, args[0]->listSize() - 1);
@@ -1689,7 +1688,7 @@ static void prim_genList(EvalState & state, const Pos & pos, Value * * args, Val
     if (len < 0)
         throw EvalError({
             .hint = hintfmt("cannot create list of size %1%", len),
-            .errPos = pos
+            .nixCode = NixCode { .errPos = pos }
         });
 
     state.mkList(v, len);
@@ -1851,7 +1850,7 @@ static void prim_div(EvalState & state, const Pos & pos, Value * * args, Value &
     if (f2 == 0)
         throw EvalError({
             .hint = hintfmt("division by zero"),
-            .errPos = pos
+            .nixCode = NixCode { .errPos = pos }
         });
 
     if (args[0]->type == tFloat || args[1]->type == tFloat) {
@@ -1863,7 +1862,7 @@ static void prim_div(EvalState & state, const Pos & pos, Value * * args, Value &
         if (i1 == std::numeric_limits<NixInt>::min() && i2 == -1)
             throw EvalError({
                 .hint = hintfmt("overflow in integer division"),
-                .errPos = pos
+                .nixCode = NixCode { .errPos = pos }
             });
 
         mkInt(v, i1 / i2);
@@ -1924,7 +1923,7 @@ static void prim_substring(EvalState & state, const Pos & pos, Value * * args, V
     if (start < 0)
         throw EvalError({
             .hint = hintfmt("negative start position in 'substring'"),
-            .errPos = pos
+            .nixCode = NixCode { .errPos = pos }
         });
 
     mkString(v, (unsigned int) start >= s.size() ? "" : string(s, start, len), context);
@@ -1947,7 +1946,7 @@ static void prim_hashString(EvalState & state, const Pos & pos, Value * * args,
     if (!ht)
         throw Error({
             .hint = hintfmt("unknown hash type '%1%'", type),
-            .errPos = pos
+            .nixCode = NixCode { .errPos = pos }
         });
 
     PathSet context; // discarded
@@ -1993,12 +1992,12 @@ void prim_match(EvalState & state, const Pos & pos, Value * * args, Value & v)
             // limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++
             throw EvalError({
                 .hint = hintfmt("memory limit exceeded by regular expression '%s'", re),
-                .errPos = pos
+                .nixCode = NixCode { .errPos = pos }
             });
         } else {
             throw EvalError({
                 .hint = hintfmt("invalid regular expression '%s'", re),
-                .errPos = pos
+                .nixCode = NixCode { .errPos = pos }
             });
         }
     }
@@ -2066,12 +2065,12 @@ static void prim_split(EvalState & state, const Pos & pos, Value * * args, Value
             // limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++
             throw EvalError({
                 .hint = hintfmt("memory limit exceeded by regular expression '%s'", re),
-                .errPos = pos
+                .nixCode = NixCode { .errPos = pos }
             });
         } else {
             throw EvalError({
                 .hint = hintfmt("invalid regular expression '%s'", re),
-                .errPos = pos
+                .nixCode = NixCode { .errPos = pos }
             });
         }
     }
@@ -2105,7 +2104,7 @@ static void prim_replaceStrings(EvalState & state, const Pos & pos, Value * * ar
     if (args[0]->listSize() != args[1]->listSize())
         throw EvalError({
             .hint = hintfmt("'from' and 'to' arguments to 'replaceStrings' have different lengths"),
-            .errPos = pos
+            .nixCode = NixCode { .errPos = pos }
         });
 
     vector<string> from;

src/libexpr/primops/context.cc

@@ -148,7 +148,7 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg
         if (!state.store->isStorePath(i.name))
             throw EvalError({
                 .hint = hintfmt("Context key '%s' is not a store path", i.name),
-                .errPos = *i.pos
+                .nixCode = NixCode { .errPos = *i.pos }
             });
         if (!settings.readOnlyMode)
             state.store->ensurePath(state.store->parseStorePath(i.name));
@@ -165,7 +165,7 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg
                 if (!isDerivation(i.name)) {
                     throw EvalError({
                         .hint = hintfmt("Tried to add all-outputs context of %s, which is not a derivation, to a string", i.name),
-                        .errPos = *i.pos
+                        .nixCode = NixCode { .errPos = *i.pos }
                     });
                 }
                 context.insert("=" + string(i.name));
@@ -178,7 +178,7 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg
             if (iter->value->listSize() && !isDerivation(i.name)) {
                 throw EvalError({
                     .hint = hintfmt("Tried to add derivation output context of %s, which is not a derivation, to a string", i.name),
-                    .errPos = *i.pos
+                    .nixCode = NixCode { .errPos = *i.pos }
                 });
             }
             for (unsigned int n = 0; n < iter->value->listSize(); ++n) {

src/libexpr/primops/fetchGit.cc

@@ -37,14 +37,14 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va
             else
                 throw EvalError({
                     .hint = hintfmt("unsupported argument '%s' to 'fetchGit'", attr.name),
-                    .errPos = *attr.pos
+                    .nixCode = NixCode { .errPos = *attr.pos }
                 });
         }
 
         if (url.empty())
             throw EvalError({
                 .hint = hintfmt("'url' argument required"),
-                .errPos = pos
+                .nixCode = NixCode { .errPos = pos }
             });
 
     } else

src/libexpr/primops/fetchMercurial.cc

@@ -40,14 +40,14 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar
             else
                 throw EvalError({
                     .hint = hintfmt("unsupported argument '%s' to 'fetchMercurial'", attr.name),
-                    .errPos = *attr.pos
+                    .nixCode = NixCode { .errPos = *attr.pos }
                 });
         }
 
         if (url.empty())
             throw EvalError({
                 .hint = hintfmt("'url' argument required"),
-                .errPos = pos
+                .nixCode = NixCode { .errPos = pos }
             });
 
     } else

src/libexpr/primops/fetchTree.cc

@@ -68,7 +68,7 @@ static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, V
         if (!attrs.count("type"))
             throw Error({
                 .hint = hintfmt("attribute 'type' is missing in call to 'fetchTree'"),
-                .errPos = pos
+                .nixCode = NixCode { .errPos = pos }
             });
 
         input = fetchers::inputFromAttrs(attrs);
@@ -112,14 +112,14 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
             else
                 throw EvalError({
                     .hint = hintfmt("unsupported argument '%s' to '%s'", attr.name, who),
-                    .errPos = *attr.pos
+                    .nixCode = NixCode { .errPos = *attr.pos }
                 });
             }
 
         if (!url)
             throw EvalError({
                 .hint = hintfmt("'url' argument required"),
-                .errPos = pos
+                .nixCode = NixCode { .errPos = pos }
             });
     } else
         url = state.forceStringNoCtx(*args[0], pos);

src/libexpr/primops/fromTOML.cc

@@ -83,7 +83,7 @@ static void prim_fromTOML(EvalState & state, const Pos & pos, Value * * args, Va
     } catch (std::runtime_error & e) {
         throw EvalError({
             .hint = hintfmt("while parsing a TOML string: %s", e.what()),
-            .errPos = pos
+            .nixCode = NixCode { .errPos = pos }
         });
     }
 }

src/libfetchers/cache.hh

@@ -6,8 +6,6 @@ namespace nix::fetchers {
 
 struct Cache
 {
-    virtual ~Cache() { }
-
     virtual void add(
         ref<Store> store,
         const Attrs & inAttrs,

src/libmain/loggers.cc

@@ -26,7 +26,7 @@ Logger * makeDefaultLogger() {
     case LogFormat::rawWithLogs:
         return makeSimpleLogger(true);
     case LogFormat::internalJson:
-        return makeJSONLogger(*makeSimpleLogger(true));
+        return makeJSONLogger(*makeSimpleLogger());
     case LogFormat::bar:
         return makeProgressBar();
     case LogFormat::barWithLogs:

src/libmain/progress-bar.cc

@@ -131,7 +131,7 @@ public:
         auto state(state_.lock());
 
         std::stringstream oss;
-        showErrorInfo(oss, ei, loggerSettings.showTrace.get());
+        oss << ei;
 
         log(*state, ei.level, oss.str());
     }

src/libmain/shared.cc

@@ -323,8 +323,10 @@ int handleExceptions(const string & programName, std::function<void()> fun)
         printError("Try '%1% --help' for more information.", programName);
         return 1;
     } catch (BaseError & e) {
+        if (settings.showTrace && e.prefix() != "")
+            printError(e.prefix());
         logError(e.info());
-        if (e.hasTrace() && !loggerSettings.showTrace.get())
+        if (e.prefix() != "" && !settings.showTrace)
             printError("(use '--show-trace' to show detailed location information)");
         return e.status;
     } catch (std::bad_alloc & e) {

src/libstore/binary-cache-store.cc

@@ -15,7 +15,6 @@
 #include <chrono>
 #include <future>
 #include <regex>
-#include <fstream>
 
 #include <nlohmann/json.hpp>
 
@@ -58,13 +57,6 @@ void BinaryCacheStore::init()
     }
 }
 
-void BinaryCacheStore::upsertFile(const std::string & path,
-    std::string && data,
-    const std::string & mimeType)
-{
-    upsertFile(path, std::make_shared<std::stringstream>(std::move(data)), mimeType);
-}
-
 void BinaryCacheStore::getFile(const std::string & path,
     Callback<std::shared_ptr<std::string>> callback) noexcept
 {
@@ -121,74 +113,13 @@ void BinaryCacheStore::writeNarInfo(ref<NarInfo> narInfo)
         diskCache->upsertNarInfo(getUri(), hashPart, std::shared_ptr<NarInfo>(narInfo));
 }
 
-AutoCloseFD openFile(const Path & path)
-{
-    auto fd = open(path.c_str(), O_RDONLY | O_CLOEXEC);
-    if (!fd)
-        throw SysError("opening file '%1%'", path);
-    return fd;
-}
-
-struct FileSource : FdSource
-{
-    AutoCloseFD fd2;
-
-    FileSource(const Path & path)
-        : fd2(openFile(path))
-    {
-        fd = fd2.get();
-    }
-};
-
 void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource,
-    RepairFlag repair, CheckSigsFlag checkSigs)
+    RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor)
 {
-    assert(info.narHash && info.narSize);
+    // FIXME: See if we can use the original source to reduce memory usage.
+    auto nar = make_ref<std::string>(narSource.drain());
 
-    if (!repair && isValidPath(info.path)) {
-        // FIXME: copyNAR -> null sink
-        narSource.drain();
-        return;
-    }
-
-    auto [fdTemp, fnTemp] = createTempFile();
-
-    auto now1 = std::chrono::steady_clock::now();
-
-    /* Read the NAR simultaneously into a CompressionSink+FileSink (to
-       write the compressed NAR to disk), into a HashSink (to get the
-       NAR hash), and into a NarAccessor (to get the NAR listing). */
-    HashSink fileHashSink(htSHA256);
-    std::shared_ptr<FSAccessor> narAccessor;
-    {
-    FdSink fileSink(fdTemp.get());
-    TeeSink teeSink(fileSink, fileHashSink);
-    auto compressionSink = makeCompressionSink(compression, teeSink);
-    TeeSource teeSource(narSource, *compressionSink);
-    narAccessor = makeNarAccessor(teeSource);
-    compressionSink->finish();
-    }
-
-    auto now2 = std::chrono::steady_clock::now();
-
-    auto narInfo = make_ref<NarInfo>(info);
-    narInfo->narSize = info.narSize;
-    narInfo->narHash = info.narHash;
-    narInfo->compression = compression;
-    auto [fileHash, fileSize] = fileHashSink.finish();
-    narInfo->fileHash = fileHash;
-    narInfo->fileSize = fileSize;
-    narInfo->url = "nar/" + narInfo->fileHash.to_string(Base32, false) + ".nar"
-        + (compression == "xz" ? ".xz" :
-           compression == "bzip2" ? ".bz2" :
-           compression == "br" ? ".br" :
-           "");
-
-    auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
-    printMsg(lvlTalkative, "copying path '%1%' (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache",
-        printStorePath(narInfo->path), info.narSize,
-        ((1.0 - (double) fileSize / info.narSize) * 100.0),
-        duration);
+    if (!repair && isValidPath(info.path)) return;
 
     /* Verify that all references are valid. This may do some .narinfo
        reads, but typically they'll already be cached. */
@@ -201,6 +132,23 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
                 printStorePath(info.path), printStorePath(ref));
         }
 
+    assert(nar->compare(0, narMagic.size(), narMagic) == 0);
+
+    auto narInfo = make_ref<NarInfo>(info);
+
+    narInfo->narSize = nar->size();
+    narInfo->narHash = hashString(htSHA256, *nar);
+
+    if (info.narHash && info.narHash != narInfo->narHash)
+        throw Error("refusing to copy corrupted path '%1%' to binary cache", printStorePath(info.path));
+
+    auto accessor_ = std::dynamic_pointer_cast<RemoteFSAccessor>(accessor);
+
+    auto narAccessor = makeNarAccessor(nar);
+
+    if (accessor_)
+        accessor_->addToCache(printStorePath(info.path), *nar, narAccessor);
+
     /* Optionally write a JSON file containing a listing of the
        contents of the NAR. */
     if (writeNARListing) {
@@ -212,13 +160,33 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
 
             {
                 auto res = jsonRoot.placeholder("root");
-                listNar(res, ref<FSAccessor>(narAccessor), "", true);
+                listNar(res, narAccessor, "", true);
             }
         }
 
         upsertFile(std::string(info.path.to_string()) + ".ls", jsonOut.str(), "application/json");
     }
 
+    /* Compress the NAR. */
+    narInfo->compression = compression;
+    auto now1 = std::chrono::steady_clock::now();
+    auto narCompressed = compress(compression, *nar, parallelCompression);
+    auto now2 = std::chrono::steady_clock::now();
+    narInfo->fileHash = hashString(htSHA256, *narCompressed);
+    narInfo->fileSize = narCompressed->size();
+
+    auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
+    printMsg(lvlTalkative, "copying path '%1%' (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache",
+        printStorePath(narInfo->path), narInfo->narSize,
+        ((1.0 - (double) narCompressed->size() / nar->size()) * 100.0),
+        duration);
+
+    narInfo->url = "nar/" + narInfo->fileHash.to_string(Base32, false) + ".nar"
+        + (compression == "xz" ? ".xz" :
+           compression == "bzip2" ? ".bz2" :
+           compression == "br" ? ".br" :
+           "");
+
     /* Optionally maintain an index of DWARF debug info files
        consisting of JSON files named 'debuginfo/<build-id>' that
        specify the NAR file and member containing the debug info. */
@@ -279,14 +247,12 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
     /* Atomically write the NAR file. */
     if (repair || !fileExists(narInfo->url)) {
         stats.narWrite++;
-        upsertFile(narInfo->url,
-            std::make_shared<std::fstream>(fnTemp, std::ios_base::in),
-            "application/x-nix-nar");
+        upsertFile(narInfo->url, *narCompressed, "application/x-nix-nar");
     } else
         stats.narWriteAverted++;
 
-    stats.narWriteBytes += info.narSize;
-    stats.narWriteCompressedBytes += fileSize;
+    stats.narWriteBytes += nar->size();
+    stats.narWriteCompressedBytes += narCompressed->size();
     stats.narWriteCompressionTimeMs += duration;
 
     /* Atomically write the NAR info file.*/
@@ -385,7 +351,7 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath
     ValidPathInfo info(makeFixedOutputPath(method, h, name));
 
     auto source = StringSource { *sink.s };
-    addToStore(info, source, repair, CheckSigs);
+    addToStore(info, source, repair, CheckSigs, nullptr);
 
     return std::move(info.path);
 }
@@ -400,7 +366,7 @@ StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s
         StringSink sink;
         dumpString(s, sink);
         auto source = StringSource { *sink.s };
-        addToStore(info, source, repair, CheckSigs);
+        addToStore(info, source, repair, CheckSigs, nullptr);
     }
 
     return std::move(info.path);

src/libstore/binary-cache-store.hh

@@ -36,13 +36,9 @@ public:
     virtual bool fileExists(const std::string & path) = 0;
 
     virtual void upsertFile(const std::string & path,
-        std::shared_ptr<std::basic_iostream<char>> istream,
+        const std::string & data,
         const std::string & mimeType) = 0;
 
-    void upsertFile(const std::string & path,
-        std::string && data,
-        const std::string & mimeType);
-
     /* Note: subclasses must implement at least one of the two
        following getFile() methods. */
 
@@ -79,7 +75,8 @@ public:
     { unsupported("queryPathFromHashPart"); }
 
     void addToStore(const ValidPathInfo & info, Source & narSource,
-        RepairFlag repair, CheckSigsFlag checkSigs) override;
+        RepairFlag repair, CheckSigsFlag checkSigs,
+        std::shared_ptr<FSAccessor> accessor) override;
 
     StorePath addToStore(const string & name, const Path & srcPath,
         FileIngestionMethod method, HashType hashAlgo,

src/libstore/build.cc

@@ -1950,11 +1950,8 @@ void linkOrCopy(const Path & from, const Path & to)
         /* Hard-linking fails if we exceed the maximum link count on a
            file (e.g. 32000 of ext3), which is quite possible after a
            'nix-store --optimise'. FIXME: actually, why don't we just
-           bind-mount in this case?
-           
-           It can also fail with EPERM in BeegFS v7 and earlier versions
-           which don't allow hard-links to other directories */
-        if (errno != EMLINK && errno != EPERM)
+           bind-mount in this case? */
+        if (errno != EMLINK)
             throw SysError("linking '%s' to '%s'", to, from);
         copyPath(from, to);
     }
@@ -2041,10 +2038,7 @@ void DerivationGoal::startBuilder()
             if (!std::regex_match(fileName, regex))
                 throw Error("invalid file name '%s' in 'exportReferencesGraph'", fileName);
 
-            auto storePathS = *i++;
-            if (!worker.store.isInStore(storePathS))
-                throw BuildError("'exportReferencesGraph' contains a non-store path '%1%'", storePathS);
-            auto storePath = worker.store.toStorePath(storePathS).first;
+            auto storePath = worker.store.parseStorePath(*i++);
 
             /* Write closure info to <fileName>. */
             writeFile(tmpDir + "/" + fileName,
@@ -2083,7 +2077,7 @@ void DerivationGoal::startBuilder()
         for (auto & i : dirsInChroot)
             try {
                 if (worker.store.isInStore(i.second.source))
-                    worker.store.computeFSClosure(worker.store.toStorePath(i.second.source).first, closure);
+                    worker.store.computeFSClosure(worker.store.parseStorePath(worker.store.toStorePath(i.second.source)), closure);
             } catch (InvalidPath & e) {
             } catch (Error & e) {
                 throw Error("while processing 'sandbox-paths': %s", e.what());
@@ -2756,8 +2750,8 @@ struct RestrictedStore : public LocalFSStore
     void queryReferrers(const StorePath & path, StorePathSet & referrers) override
     { }
 
-    OutputPathMap queryDerivationOutputMap(const StorePath & path) override
-    { throw Error("queryDerivationOutputMap"); }
+    StorePathSet queryDerivationOutputs(const StorePath & path) override
+    { throw Error("queryDerivationOutputs"); }
 
     std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override
     { throw Error("queryPathFromHashPart"); }
@@ -2768,9 +2762,10 @@ struct RestrictedStore : public LocalFSStore
     { throw Error("addToStore"); }
 
     void addToStore(const ValidPathInfo & info, Source & narSource,
-        RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs) override
+        RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs,
+        std::shared_ptr<FSAccessor> accessor = 0) override
     {
-        next->addToStore(info, narSource, repair, checkSigs);
+        next->addToStore(info, narSource, repair, checkSigs, accessor);
         goal.addDependency(info.path);
     }
 

src/libstore/builtins/fetchurl.cc

@@ -58,16 +58,13 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
         }
     };
 
-    /* We always have one output, and if it's a fixed-output derivation (as
-       checked below) it must be the only output */
-    auto & output = drv.outputs.begin()->second;
-
     /* Try the hashed mirrors first. */
-    if (output.hash && output.hash->method == FileIngestionMethod::Flat)
+    if (getAttr("outputHashMode") == "flat")
         for (auto hashedMirror : settings.hashedMirrors.get())
             try {
                 if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/';
-                auto & h = output.hash->hash;
+                auto ht = parseHashTypeOpt(getAttr("outputHashAlgo"));
+                auto h = Hash(getAttr("outputHash"), ht);
                 fetch(hashedMirror + printHashType(*h.type) + "/" + h.to_string(Base16, false));
                 return;
             } catch (Error & e) {

src/libstore/content-address.cc

@@ -82,16 +82,4 @@ std::string renderContentAddress(std::optional<ContentAddress> ca) {
     return ca ? renderContentAddress(*ca) : "";
 }
 
-Hash getContentAddressHash(const ContentAddress & ca)
-{
-    return std::visit(overloaded {
-        [](TextHash th) {
-            return th.hash;
-        },
-        [](FixedOutputHash fsh) {
-            return fsh.hash;
-        }
-    }, ca);
-}
-
 }

src/libstore/content-address.hh

@@ -53,6 +53,4 @@ ContentAddress parseContentAddress(std::string_view rawCa);
 
 std::optional<ContentAddress> parseContentAddressOpt(std::string_view rawCaOpt);
 
-Hash getContentAddressHash(const ContentAddress & ca);
-
 }

src/libstore/daemon.cc

@@ -78,10 +78,10 @@ struct TunnelLogger : public Logger
         if (ei.level > verbosity) return;
 
         std::stringstream oss;
-        showErrorInfo(oss, ei, false);
+        oss << ei;
 
         StringSink buf;
-        buf << STDERR_NEXT << oss.str();
+        buf << STDERR_NEXT << oss.str() << "\n"; // (fs.s + "\n");
         enqueueMsg(*buf.s);
     }
 
@@ -347,15 +347,6 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         break;
     }
 
-    case wopQueryDerivationOutputMap: {
-        auto path = store->parseStorePath(readString(from));
-        logger->startWork();
-        OutputPathMap outputs = store->queryDerivationOutputMap(path);
-        logger->stopWork();
-        writeOutputPathMap(*store, to, outputs);
-        break;
-    }
-
     case wopQueryDeriver: {
         auto path = store->parseStorePath(readString(from));
         logger->startWork();
@@ -391,8 +382,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         }
         HashType hashAlgo = parseHashType(s);
 
-        StringSink savedNAR;
-        TeeSource savedNARSource(from, savedNAR);
+        TeeSource savedNAR(from);
         RetrieveRegularNARSink savedRegular;
 
         if (method == FileIngestionMethod::Recursive) {
@@ -400,7 +390,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
                a string so that we can pass it to
                addToStoreFromDump(). */
             ParseSink sink; /* null sink; just parse the NAR */
-            parseDump(sink, savedNARSource);
+            parseDump(sink, savedNAR);
         } else
             parseDump(savedRegular, from);
 
@@ -408,7 +398,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         if (!savedRegular.regular) throw Error("regular file expected");
 
         auto path = store->addToStoreFromDump(
-            method == FileIngestionMethod::Recursive ? *savedNAR.s : savedRegular.s,
+            method == FileIngestionMethod::Recursive ? *savedNAR.data : savedRegular.s,
             baseName,
             method,
             hashAlgo);
@@ -443,7 +433,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     case wopImportPaths: {
         logger->startWork();
         TunnelSource source(from, to);
-        auto paths = store->importPaths(source,
+        auto paths = store->importPaths(source, nullptr,
             trusted ? NoCheckSigs : CheckSigs);
         logger->stopWork();
         Strings paths2;
@@ -732,9 +722,9 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         if (GET_PROTOCOL_MINOR(clientVersion) >= 21)
             source = std::make_unique<TunnelSource>(from, to);
         else {
-            TeeParseSink tee(from);
+            TeeSink tee(from);
             parseDump(tee, tee.source);
-            saved = std::move(*tee.saved.s);
+            saved = std::move(*tee.source.data);
             source = std::make_unique<StringSource>(saved);
         }
 
@@ -742,7 +732,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
 
         // FIXME: race if addToStore doesn't read source?
         store->addToStore(info, *source, (RepairFlag) repair,
-            dontCheckSigs ? NoCheckSigs : CheckSigs);
+            dontCheckSigs ? NoCheckSigs : CheckSigs, nullptr);
 
         logger->stopWork();
         break;

src/libstore/derivations.cc

@@ -4,6 +4,7 @@
 #include "util.hh"
 #include "worker-protocol.hh"
 #include "fs-accessor.hh"
+#include "istringstream_nocopy.hh"
 
 namespace nix {
 
@@ -100,7 +101,7 @@ static StringSet parseStrings(std::istream & str, bool arePaths)
 }
 
 
-static DerivationOutput parseDerivationOutput(const Store & store, std::istringstream & str)
+static DerivationOutput parseDerivationOutput(const Store & store, istringstream_nocopy & str)
 {
     expect(str, ","); auto path = store.parseStorePath(parsePath(str));
     expect(str, ","); auto hashAlgo = parseString(str);
@@ -128,10 +129,10 @@ static DerivationOutput parseDerivationOutput(const Store & store, std::istrings
 }
 
 
-static Derivation parseDerivation(const Store & store, std::string && s)
+static Derivation parseDerivation(const Store & store, const string & s)
 {
     Derivation drv;
-    std::istringstream str(std::move(s));
+    istringstream_nocopy str(s);
     expect(str, "Derive([");
 
     /* Parse the list of outputs. */
@@ -403,7 +404,7 @@ static DerivationOutput readDerivationOutput(Source & in, const Store & store)
 {
     auto path = store.parseStorePath(readString(in));
     auto hashAlgo = readString(in);
-    auto hash = readString(in);
+    const auto hash = readString(in);
 
     std::optional<FixedOutputHash> fsh;
     if (hashAlgo != "") {
@@ -412,7 +413,7 @@ static DerivationOutput readDerivationOutput(Source & in, const Store & store)
             method = FileIngestionMethod::Recursive;
             hashAlgo = string(hashAlgo, 2);
         }
-        auto hashType = parseHashType(hashAlgo);
+        const HashType hashType = parseHashType(hashAlgo);
         fsh = FixedOutputHash {
             .method = std::move(method),
             .hash = Hash(hash, hashType),
@@ -462,16 +463,11 @@ Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv)
 void writeDerivation(Sink & out, const Store & store, const BasicDerivation & drv)
 {
     out << drv.outputs.size();
-    for (auto & i : drv.outputs) {
+    for (auto & i : drv.outputs)
         out << i.first
-            << store.printStorePath(i.second.path);
-        if (i.second.hash) {
-            out << i.second.hash->printMethodAlgo()
-                << i.second.hash->hash.to_string(Base16, false);
-        } else {
-            out << "" << "";
-        }
-    }
+            << store.printStorePath(i.second.path)
+            << i.second.hash->printMethodAlgo()
+            << i.second.hash->hash.to_string(Base16, false);
     writeStorePaths(store, out, drv.inputSrcs);
     out << drv.platform << drv.builder << drv.args;
     out << drv.env.size();

src/libstore/export-import.cc

@@ -7,6 +7,24 @@
 
 namespace nix {
 
+struct HashAndWriteSink : Sink
+{
+    Sink & writeSink;
+    HashSink hashSink;
+    HashAndWriteSink(Sink & writeSink) : writeSink(writeSink), hashSink(htSHA256)
+    {
+    }
+    virtual void operator () (const unsigned char * data, size_t len)
+    {
+        writeSink(data, len);
+        hashSink(data, len);
+    }
+    Hash currentHash()
+    {
+        return hashSink.currentHash().first;
+    }
+};
+
 void Store::exportPaths(const StorePathSet & paths, Sink & sink)
 {
     auto sorted = topoSortPaths(paths);
@@ -29,29 +47,28 @@ void Store::exportPath(const StorePath & path, Sink & sink)
 {
     auto info = queryPathInfo(path);
 
-    HashSink hashSink(htSHA256);
-    TeeSink teeSink(sink, hashSink);
+    HashAndWriteSink hashAndWriteSink(sink);
 
-    narFromPath(path, teeSink);
+    narFromPath(path, hashAndWriteSink);
 
     /* Refuse to export paths that have changed.  This prevents
        filesystem corruption from spreading to other machines.
        Don't complain if the stored hash is zero (unknown). */
-    Hash hash = hashSink.currentHash().first;
+    Hash hash = hashAndWriteSink.currentHash();
     if (hash != info->narHash && info->narHash != Hash(*info->narHash.type))
         throw Error("hash of path '%s' has changed from '%s' to '%s'!",
             printStorePath(path), info->narHash.to_string(Base32, true), hash.to_string(Base32, true));
 
-    teeSink
+    hashAndWriteSink
         << exportMagic
         << printStorePath(path);
-    writeStorePaths(*this, teeSink, info->references);
-    teeSink
+    writeStorePaths(*this, hashAndWriteSink, info->references);
+    hashAndWriteSink
         << (info->deriver ? printStorePath(*info->deriver) : "")
         << 0;
 }
 
-StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs)
+StorePaths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> accessor, CheckSigsFlag checkSigs)
 {
     StorePaths res;
     while (true) {
@@ -60,7 +77,7 @@ StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs)
         if (n != 1) throw Error("input doesn't look like something created by 'nix-store --export'");
 
         /* Extract the NAR from the source. */
-        TeeParseSink tee(source);
+        TeeSink tee(source);
         parseDump(tee, tee.source);
 
         uint32_t magic = readInt(source);
@@ -77,16 +94,16 @@ StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs)
         if (deriver != "")
             info.deriver = parseStorePath(deriver);
 
-        info.narHash = hashString(htSHA256, *tee.saved.s);
-        info.narSize = tee.saved.s->size();
+        info.narHash = hashString(htSHA256, *tee.source.data);
+        info.narSize = tee.source.data->size();
 
         // Ignore optional legacy signature.
         if (readInt(source) == 1)
             readString(source);
 
         // Can't use underlying source, which would have been exhausted
-        auto source = StringSource { *tee.saved.s };
-        addToStore(info, source, NoRepair, checkSigs);
+        auto source = StringSource { *tee.source.data };
+        addToStore(info, source, NoRepair, checkSigs, accessor);
 
         res.push_back(info.path);
     }

src/libstore/gc.cc

@@ -262,13 +262,11 @@ void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor)
 void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots)
 {
     auto foundRoot = [&](const Path & path, const Path & target) {
-        try {
-            auto storePath = toStorePath(target).first;
-            if (isValidPath(storePath))
-                roots[std::move(storePath)].emplace(path);
-            else
-                printInfo("skipping invalid root from '%1%' to '%2%'", path, target);
-        } catch (BadStorePath &) { }
+        auto storePath = maybeParseStorePath(toStorePath(target));
+        if (storePath && isValidPath(*storePath))
+            roots[std::move(*storePath)].emplace(path);
+        else
+            printInfo("skipping invalid root from '%1%' to '%2%'", path, target);
     };
 
     try {
@@ -474,15 +472,15 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
 
     for (auto & [target, links] : unchecked) {
         if (!isInStore(target)) continue;
-        try {
-            auto path = toStorePath(target).first;
-            if (!isValidPath(path)) continue;
-            debug("got additional root '%1%'", printStorePath(path));
-            if (censor)
-                roots[path].insert(censored);
-            else
-                roots[path].insert(links.begin(), links.end());
-        } catch (BadStorePath &) { }
+        Path pathS = toStorePath(target);
+        if (!isStorePath(pathS)) continue;
+        auto path = parseStorePath(pathS);
+        if (!isValidPath(path)) continue;
+        debug("got additional root '%1%'", pathS);
+        if (censor)
+            roots[path].insert(censored);
+        else
+            roots[path].insert(links.begin(), links.end());
     }
 }
 

src/libstore/globals.cc

@@ -2,6 +2,7 @@
 #include "util.hh"
 #include "archive.hh"
 #include "args.hh"
+#include "sync.hh"
 
 #include <algorithm>
 #include <map>
@@ -35,7 +36,7 @@ Settings::Settings()
     , nixLibexecDir(canonPath(getEnv("NIX_LIBEXEC_DIR").value_or(NIX_LIBEXEC_DIR)))
     , nixBinDir(canonPath(getEnv("NIX_BIN_DIR").value_or(NIX_BIN_DIR)))
     , nixManDir(canonPath(NIX_MAN_DIR))
-    , nixDaemonSocketFile(canonPath(getEnv("NIX_DAEMON_SOCKET_PATH").value_or(nixStateDir + DEFAULT_SOCKET_PATH)))
+    , nixDaemonSocketFile(canonPath(nixStateDir + DEFAULT_SOCKET_PATH))
 {
     buildUsersGroup = getuid() == 0 ? "nixbld" : "";
     lockCPU = getEnv("NIX_AFFINITY_HACK") == "1";
@@ -129,8 +130,14 @@ bool Settings::isExperimentalFeatureEnabled(const std::string & name)
 
 void Settings::requireExperimentalFeature(const std::string & name)
 {
-    if (!isExperimentalFeatureEnabled(name))
-        throw Error("experimental Nix feature '%1%' is disabled; use '--experimental-features %1%' to override", name);
+    if (!isExperimentalFeatureEnabled(name)) {
+        if (allowExperimentalFeatures) {
+            static Sync<std::unordered_set<std::string>> warned;
+            if (warned.lock()->insert(name).second)
+                warn("feature '%s' is experimental", name);
+        } else
+            throw Error("experimental Nix feature '%1%' is disabled; use '--experimental-features %1%' to override", name);
+    }
 }
 
 bool Settings::isWSL1()

src/libstore/globals.hh

@@ -196,6 +196,10 @@ public:
     /* Whether to lock the Nix client and worker to the same CPU. */
     bool lockCPU;
 
+    /* Whether to show a stack trace if Nix evaluation fails. */
+    Setting<bool> showTrace{this, false, "show-trace",
+        "Whether to show a stack trace on evaluation errors."};
+
     Setting<SandboxMode> sandboxMode{this,
         #if __linux__
           smEnabled
@@ -353,6 +357,10 @@ public:
     Setting<std::string> githubAccessToken{this, "", "github-access-token",
         "GitHub access token to get access to GitHub data through the GitHub API for github:<..> flakes."};
 
+    Setting<bool> allowExperimentalFeatures{this, true, "allow-experimental-features",
+        "Whether the use of experimental features other than those listed in "
+        "the option 'experimental-features' gives a warning rather than fatal error."};
+
     Setting<Strings> experimentalFeatures{this, {}, "experimental-features",
         "Experimental Nix features to enable."};
 
@@ -365,9 +373,6 @@ public:
 
     Setting<bool> warnDirty{this, true, "warn-dirty",
         "Whether to warn about dirty Git/Mercurial trees."};
-
-    Setting<size_t> narBufferSize{this, 32 * 1024 * 1024, "nar-buffer-size",
-        "Maximum size of NARs before spilling them to disk."};
 };
 
 

src/libstore/http-binary-cache-store.cc

@@ -100,11 +100,11 @@ protected:
     }
 
     void upsertFile(const std::string & path,
-        std::shared_ptr<std::basic_iostream<char>> istream,
+        const std::string & data,
         const std::string & mimeType) override
     {
         auto req = FileTransferRequest(cacheUri + "/" + path);
-        req.data = std::make_shared<string>(StreamToSourceAdapter(istream).drain());
+        req.data = std::make_shared<string>(data); // FIXME: inefficient
         req.mimeType = mimeType;
         try {
             getFileTransfer()->upload(req);

src/libstore/legacy-ssh-store.cc

@@ -126,7 +126,8 @@ struct LegacySSHStore : public Store
     }
 
     void addToStore(const ValidPathInfo & info, Source & source,
-        RepairFlag repair, CheckSigsFlag checkSigs) override
+        RepairFlag repair, CheckSigsFlag checkSigs,
+        std::shared_ptr<FSAccessor> accessor) override
     {
         debug("adding path '%s' to remote host '%s'", printStorePath(info.path), host);
 

src/libstore/local-binary-cache-store.cc

@@ -31,18 +31,8 @@ protected:
     bool fileExists(const std::string & path) override;
 
     void upsertFile(const std::string & path,
-        std::shared_ptr<std::basic_iostream<char>> istream,
-        const std::string & mimeType) override
-    {
-        auto path2 = binaryCacheDir + "/" + path;
-        Path tmp = path2 + ".tmp." + std::to_string(getpid());
-        AutoDelete del(tmp, false);
-        StreamToSourceAdapter source(istream);
-        writeFile(tmp, source);
-        if (rename(tmp.c_str(), path2.c_str()))
-            throw SysError("renaming '%1%' to '%2%'", tmp, path2);
-        del.cancel();
-    }
+        const std::string & data,
+        const std::string & mimeType) override;
 
     void getFile(const std::string & path, Sink & sink) override
     {
@@ -62,9 +52,7 @@ protected:
             if (entry.name.size() != 40 ||
                 !hasSuffix(entry.name, ".narinfo"))
                 continue;
-            paths.insert(parseStorePath(
-                    storeDir + "/" + entry.name.substr(0, entry.name.size() - 8)
-                    + "-" + MissingName));
+            paths.insert(parseStorePath(storeDir + "/" + entry.name.substr(0, entry.name.size() - 8)));
         }
 
         return paths;
@@ -80,11 +68,28 @@ void LocalBinaryCacheStore::init()
     BinaryCacheStore::init();
 }
 
+static void atomicWrite(const Path & path, const std::string & s)
+{
+    Path tmp = path + ".tmp." + std::to_string(getpid());
+    AutoDelete del(tmp, false);
+    writeFile(tmp, s);
+    if (rename(tmp.c_str(), path.c_str()))
+        throw SysError("renaming '%1%' to '%2%'", tmp, path);
+    del.cancel();
+}
+
 bool LocalBinaryCacheStore::fileExists(const std::string & path)
 {
     return pathExists(binaryCacheDir + "/" + path);
 }
 
+void LocalBinaryCacheStore::upsertFile(const std::string & path,
+    const std::string & data,
+    const std::string & mimeType)
+{
+    atomicWrite(binaryCacheDir + "/" + path, data);
+}
+
 static RegisterStoreImplementation regStore([](
     const std::string & uri, const Store::Params & params)
     -> std::shared_ptr<Store>

src/libstore/local-fs-store.cc

@@ -20,9 +20,9 @@ struct LocalStoreAccessor : public FSAccessor
 
     Path toRealPath(const Path & path)
     {
-        auto storePath = store->toStorePath(path).first;
-        if (!store->isValidPath(storePath))
-            throw InvalidPath("path '%1%' is not a valid store path", store->printStorePath(storePath));
+        Path storePath = store->toStorePath(path);
+        if (!store->isValidPath(store->parseStorePath(storePath)))
+            throw InvalidPath("path '%1%' is not a valid store path", storePath);
         return store->getRealStoreDir() + std::string(path, store->storeDir.size());
     }
 

src/libstore/local-store.cc

@@ -774,20 +774,17 @@ StorePathSet LocalStore::queryValidDerivers(const StorePath & path)
 }
 
 
-OutputPathMap LocalStore::queryDerivationOutputMap(const StorePath & path)
+StorePathSet LocalStore::queryDerivationOutputs(const StorePath & path)
 {
-    return retrySQLite<OutputPathMap>([&]() {
+    return retrySQLite<StorePathSet>([&]() {
         auto state(_state.lock());
 
         auto useQueryDerivationOutputs(state->stmtQueryDerivationOutputs.use()
             (queryValidPathId(*state, path)));
 
-        OutputPathMap outputs;
+        StorePathSet outputs;
         while (useQueryDerivationOutputs.next())
-            outputs.emplace(
-                useQueryDerivationOutputs.getStr(0),
-                parseStorePath(useQueryDerivationOutputs.getStr(1))
-            );
+            outputs.insert(parseStorePath(useQueryDerivationOutputs.getStr(1)));
 
         return outputs;
     });
@@ -962,7 +959,7 @@ const PublicKeys & LocalStore::getPublicKeys()
 
 
 void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
-    RepairFlag repair, CheckSigsFlag checkSigs)
+    RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor)
 {
     if (!info.narHash)
         throw Error("cannot add path '%s' because it lacks a hash", printStorePath(info.path));
@@ -976,7 +973,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
 
         PathLocks outputLock;
 
-        auto realPath = Store::toRealPath(info.path);
+        Path realPath = realStoreDir + "/" + std::string(info.path.to_string());
 
         /* Lock the output path.  But don't lock if we're being called
            from a build hook (whose parent process already acquired a
@@ -1047,7 +1044,8 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam
         /* The first check above is an optimisation to prevent
            unnecessary lock acquisition. */
 
-        auto realPath = Store::toRealPath(dstPath);
+        Path realPath = realStoreDir + "/";
+        realPath += dstPath.to_string();
 
         PathLocks outputLock({realPath});
 
@@ -1097,119 +1095,16 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath,
 {
     Path srcPath(absPath(_srcPath));
 
-    if (method != FileIngestionMethod::Recursive)
-        return addToStoreFromDump(readFile(srcPath), name, method, hashAlgo, repair);
+    /* Read the whole path into memory. This is not a very scalable
+       method for very large paths, but `copyPath' is mainly used for
+       small files. */
+    StringSink sink;
+    if (method == FileIngestionMethod::Recursive)
+        dumpPath(srcPath, sink, filter);
+    else
+        sink.s = make_ref<std::string>(readFile(srcPath));
 
-    /* For computing the NAR hash. */
-    auto sha256Sink = std::make_unique<HashSink>(htSHA256);
-
-    /* For computing the store path. In recursive SHA-256 mode, this
-       is the same as the NAR hash, so no need to do it again. */
-    std::unique_ptr<HashSink> hashSink =
-        hashAlgo == htSHA256
-        ? nullptr
-        : std::make_unique<HashSink>(hashAlgo);
-
-    /* Read the source path into memory, but only if it's up to
-       narBufferSize bytes. If it's larger, write it to a temporary
-       location in the Nix store. If the subsequently computed
-       destination store path is already valid, we just delete the
-       temporary path. Otherwise, we move it to the destination store
-       path. */
-    bool inMemory = true;
-    std::string nar;
-
-    auto source = sinkToSource([&](Sink & sink) {
-
-        LambdaSink sink2([&](const unsigned char * buf, size_t len) {
-            (*sha256Sink)(buf, len);
-            if (hashSink) (*hashSink)(buf, len);
-
-            if (inMemory) {
-                if (nar.size() + len > settings.narBufferSize) {
-                    inMemory = false;
-                    sink << 1;
-                    sink((const unsigned char *) nar.data(), nar.size());
-                    nar.clear();
-                } else {
-                    nar.append((const char *) buf, len);
-                }
-            }
-
-            if (!inMemory) sink(buf, len);
-        });
-
-        dumpPath(srcPath, sink2, filter);
-    });
-
-    std::unique_ptr<AutoDelete> delTempDir;
-    Path tempPath;
-
-    try {
-        /* Wait for the source coroutine to give us some dummy
-           data. This is so that we don't create the temporary
-           directory if the NAR fits in memory. */
-        readInt(*source);
-
-        auto tempDir = createTempDir(realStoreDir, "add");
-        delTempDir = std::make_unique<AutoDelete>(tempDir);
-        tempPath = tempDir + "/x";
-
-        restorePath(tempPath, *source);
-
-    } catch (EndOfFile &) {
-        if (!inMemory) throw;
-        /* The NAR fits in memory, so we didn't do restorePath(). */
-    }
-
-    auto sha256 = sha256Sink->finish();
-
-    Hash hash = hashSink ? hashSink->finish().first : sha256.first;
-
-    auto dstPath = makeFixedOutputPath(method, hash, name);
-
-    addTempRoot(dstPath);
-
-    if (repair || !isValidPath(dstPath)) {
-
-        /* The first check above is an optimisation to prevent
-           unnecessary lock acquisition. */
-
-        auto realPath = Store::toRealPath(dstPath);
-
-        PathLocks outputLock({realPath});
-
-        if (repair || !isValidPath(dstPath)) {
-
-            deletePath(realPath);
-
-            autoGC();
-
-            if (inMemory) {
-                /* Restore from the NAR in memory. */
-                StringSource source(nar);
-                restorePath(realPath, source);
-            } else {
-                /* Move the temporary path we restored above. */
-                if (rename(tempPath.c_str(), realPath.c_str()))
-                    throw Error("renaming '%s' to '%s'", tempPath, realPath);
-            }
-
-            canonicalisePathMetaData(realPath, -1); // FIXME: merge into restorePath
-
-            optimisePath(realPath);
-
-            ValidPathInfo info(dstPath);
-            info.narHash = sha256.first;
-            info.narSize = sha256.second;
-            info.ca = FixedOutputHash { .method = method, .hash = hash };
-            registerValidPath(info);
-        }
-
-        outputLock.setDeletion(true);
-    }
-
-    return dstPath;
+    return addToStoreFromDump(*sink.s, name, method, hashAlgo, repair);
 }
 
 
@@ -1223,7 +1118,8 @@ StorePath LocalStore::addTextToStore(const string & name, const string & s,
 
     if (repair || !isValidPath(dstPath)) {
 
-        auto realPath = Store::toRealPath(dstPath);
+        Path realPath = realStoreDir + "/";
+        realPath += dstPath.to_string();
 
         PathLocks outputLock({realPath});
 

src/libstore/local-store.hh

@@ -133,7 +133,7 @@ public:
 
     StorePathSet queryValidDerivers(const StorePath & path) override;
 
-    OutputPathMap queryDerivationOutputMap(const StorePath & path) override;
+    StorePathSet queryDerivationOutputs(const StorePath & path) override;
 
     std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override;
 
@@ -143,7 +143,8 @@ public:
         SubstitutablePathInfos & infos) override;
 
     void addToStore(const ValidPathInfo & info, Source & source,
-        RepairFlag repair, CheckSigsFlag checkSigs) override;
+        RepairFlag repair, CheckSigsFlag checkSigs,
+        std::shared_ptr<FSAccessor> accessor) override;
 
     StorePath addToStore(const string & name, const Path & srcPath,
         FileIngestionMethod method, HashType hashAlgo,

src/libstore/nar-accessor.cc

@@ -18,7 +18,7 @@ struct NarMember
 
     /* If this is a regular file, position of the contents of this
        file in the NAR. */
-    uint64_t start = 0, size = 0;
+    size_t start = 0, size = 0;
 
     std::string target;
 
@@ -34,19 +34,17 @@ struct NarAccessor : public FSAccessor
 
     NarMember root;
 
-    struct NarIndexer : ParseSink, Source
+    struct NarIndexer : ParseSink, StringSource
     {
         NarAccessor & acc;
-        Source & source;
 
         std::stack<NarMember *> parents;
 
+        std::string currentStart;
         bool isExec = false;
 
-        uint64_t pos = 0;
-
-        NarIndexer(NarAccessor & acc, Source & source)
-            : acc(acc), source(source)
+        NarIndexer(NarAccessor & acc, const std::string & nar)
+            : StringSource(nar), acc(acc)
         { }
 
         void createMember(const Path & path, NarMember member) {
@@ -81,38 +79,31 @@ struct NarAccessor : public FSAccessor
 
         void preallocateContents(unsigned long long size) override
         {
-            assert(size <= std::numeric_limits<uint64_t>::max());
-            parents.top()->size = (uint64_t) size;
+            currentStart = string(s, pos, 16);
+            assert(size <= std::numeric_limits<size_t>::max());
+            parents.top()->size = (size_t)size;
             parents.top()->start = pos;
         }
 
         void receiveContents(unsigned char * data, unsigned int len) override
-        { }
+        {
+            // Sanity check
+            if (!currentStart.empty()) {
+                assert(len < 16 || currentStart == string((char *) data, 16));
+                currentStart.clear();
+            }
+        }
 
         void createSymlink(const Path & path, const string & target) override
         {
             createMember(path,
                 NarMember{FSAccessor::Type::tSymlink, false, 0, 0, target});
         }
-
-        size_t read(unsigned char * data, size_t len) override
-        {
-            auto n = source.read(data, len);
-            pos += n;
-            return n;
-        }
     };
 
     NarAccessor(ref<const std::string> nar) : nar(nar)
     {
-        StringSource source(*nar);
-        NarIndexer indexer(*this, source);
-        parseDump(indexer, indexer);
-    }
-
-    NarAccessor(Source & source)
-    {
-        NarIndexer indexer(*this, source);
+        NarIndexer indexer(*this, *nar);
         parseDump(indexer, indexer);
     }
 
@@ -228,11 +219,6 @@ ref<FSAccessor> makeNarAccessor(ref<const std::string> nar)
     return make_ref<NarAccessor>(nar);
 }
 
-ref<FSAccessor> makeNarAccessor(Source & source)
-{
-    return make_ref<NarAccessor>(source);
-}
-
 ref<FSAccessor> makeLazyNarAccessor(const std::string & listing,
     GetNarBytes getNarBytes)
 {

src/libstore/nar-accessor.hh

@@ -6,14 +6,10 @@
 
 namespace nix {
 
-struct Source;
-
 /* Return an object that provides access to the contents of a NAR
    file. */
 ref<FSAccessor> makeNarAccessor(ref<const std::string> nar);
 
-ref<FSAccessor> makeNarAccessor(Source & source);
-
 /* Create a NAR accessor from a NAR listing (in the format produced by
    listNar()). The callback getNarBytes(offset, length) is used by the
    readFile() method of the accessor to get the contents of files

src/libstore/path.cc

@@ -2,6 +2,8 @@
 
 namespace nix {
 
+MakeError(BadStorePath, Error);
+
 static void checkName(std::string_view path, std::string_view name)
 {
     if (name.empty())

src/libstore/path.hh

@@ -62,7 +62,6 @@ public:
 
 typedef std::set<StorePath> StorePathSet;
 typedef std::vector<StorePath> StorePaths;
-typedef std::map<string, StorePath> OutputPathMap;
 
 /* Extension of derivations in the Nix store. */
 const std::string drvExtension = ".drv";

src/libstore/remote-fs-accessor.cc

@@ -16,26 +16,26 @@ RemoteFSAccessor::RemoteFSAccessor(ref<Store> store, const Path & cacheDir)
         createDirs(cacheDir);
 }
 
-Path RemoteFSAccessor::makeCacheFile(std::string_view hashPart, const std::string & ext)
+Path RemoteFSAccessor::makeCacheFile(const Path & storePath, const std::string & ext)
 {
     assert(cacheDir != "");
-    return fmt("%s/%s.%s", cacheDir, hashPart, ext);
+    return fmt("%s/%s.%s", cacheDir, store->parseStorePath(storePath).hashPart(), ext);
 }
 
-void RemoteFSAccessor::addToCache(std::string_view hashPart, const std::string & nar,
+void RemoteFSAccessor::addToCache(const Path & storePath, const std::string & nar,
     ref<FSAccessor> narAccessor)
 {
-    nars.emplace(hashPart, narAccessor);
+    nars.emplace(storePath, narAccessor);
 
     if (cacheDir != "") {
         try {
             std::ostringstream str;
             JSONPlaceholder jsonRoot(str);
             listNar(jsonRoot, narAccessor, "", true);
-            writeFile(makeCacheFile(hashPart, "ls"), str.str());
+            writeFile(makeCacheFile(storePath, "ls"), str.str());
 
             /* FIXME: do this asynchronously. */
-            writeFile(makeCacheFile(hashPart, "nar"), nar);
+            writeFile(makeCacheFile(storePath, "nar"), nar);
 
         } catch (...) {
             ignoreException();
@@ -47,22 +47,23 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_)
 {
     auto path = canonPath(path_);
 
-    auto [storePath, restPath] = store->toStorePath(path);
+    auto storePath = store->toStorePath(path);
+    std::string restPath = std::string(path, storePath.size());
 
-    if (!store->isValidPath(storePath))
-        throw InvalidPath("path '%1%' is not a valid store path", store->printStorePath(storePath));
+    if (!store->isValidPath(store->parseStorePath(storePath)))
+        throw InvalidPath("path '%1%' is not a valid store path", storePath);
 
-    auto i = nars.find(std::string(storePath.hashPart()));
+    auto i = nars.find(storePath);
     if (i != nars.end()) return {i->second, restPath};
 
     StringSink sink;
     std::string listing;
     Path cacheFile;
 
-    if (cacheDir != "" && pathExists(cacheFile = makeCacheFile(storePath.hashPart(), "nar"))) {
+    if (cacheDir != "" && pathExists(cacheFile = makeCacheFile(storePath, "nar"))) {
 
         try {
-            listing = nix::readFile(makeCacheFile(storePath.hashPart(), "ls"));
+            listing = nix::readFile(makeCacheFile(storePath, "ls"));
 
             auto narAccessor = makeLazyNarAccessor(listing,
                 [cacheFile](uint64_t offset, uint64_t length) {
@@ -80,7 +81,7 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_)
                     return buf;
                 });
 
-            nars.emplace(storePath.hashPart(), narAccessor);
+            nars.emplace(storePath, narAccessor);
             return {narAccessor, restPath};
 
         } catch (SysError &) { }
@@ -89,15 +90,15 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_)
             *sink.s = nix::readFile(cacheFile);
 
             auto narAccessor = makeNarAccessor(sink.s);
-            nars.emplace(storePath.hashPart(), narAccessor);
+            nars.emplace(storePath, narAccessor);
             return {narAccessor, restPath};
 
         } catch (SysError &) { }
     }
 
-    store->narFromPath(storePath, sink);
+    store->narFromPath(store->parseStorePath(storePath), sink);
     auto narAccessor = makeNarAccessor(sink.s);
-    addToCache(storePath.hashPart(), *sink.s, narAccessor);
+    addToCache(storePath, *sink.s, narAccessor);
     return {narAccessor, restPath};
 }
 

src/libstore/remote-fs-accessor.hh

@@ -10,7 +10,7 @@ class RemoteFSAccessor : public FSAccessor
 {
     ref<Store> store;
 
-    std::map<std::string, ref<FSAccessor>> nars;
+    std::map<Path, ref<FSAccessor>> nars;
 
     Path cacheDir;
 
@@ -18,9 +18,9 @@ class RemoteFSAccessor : public FSAccessor
 
     friend class BinaryCacheStore;
 
-    Path makeCacheFile(std::string_view hashPart, const std::string & ext);
+    Path makeCacheFile(const Path & storePath, const std::string & ext);
 
-    void addToCache(std::string_view hashPart, const std::string & nar,
+    void addToCache(const Path & storePath, const std::string & nar,
         ref<FSAccessor> narAccessor);
 
 public:

src/libstore/remote-store.cc

@@ -8,7 +8,6 @@
 #include "derivations.hh"
 #include "pool.hh"
 #include "finally.hh"
-#include "logging.hh"
 
 #include <sys/types.h>
 #include <sys/stat.h>
@@ -39,29 +38,6 @@ void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths
         out << store.printStorePath(i);
 }
 
-std::map<string, StorePath> readOutputPathMap(const Store & store, Source & from)
-{
-    std::map<string, StorePath> pathMap;
-    auto rawInput = readStrings<Strings>(from);
-    if (rawInput.size() % 2)
-        throw Error("got an odd number of elements from the daemon when trying to read a output path map");
-    auto curInput = rawInput.begin();
-    while (curInput != rawInput.end()) {
-        auto thisKey = *curInput++;
-        auto thisValue = *curInput++;
-        pathMap.emplace(thisKey, store.parseStorePath(thisValue));
-    }
-    return pathMap;
-}
-
-void writeOutputPathMap(const Store & store, Sink & out, const std::map<string, StorePath> & pathMap)
-{
-    out << 2*pathMap.size();
-    for (auto & i : pathMap) {
-        out << i.first;
-        out << store.printStorePath(i.second);
-    }
-}
 
 /* TODO: Separate these store impls into different files, give them better names */
 RemoteStore::RemoteStore(const Params & params)
@@ -221,7 +197,7 @@ void RemoteStore::setOptions(Connection & conn)
         overrides.erase(settings.maxSilentTime.name);
         overrides.erase(settings.buildCores.name);
         overrides.erase(settings.useSubstitutes.name);
-        overrides.erase(loggerSettings.showTrace.name);
+        overrides.erase(settings.showTrace.name);
         conn.to << overrides.size();
         for (auto & i : overrides)
             conn.to << i.first << i.second.value;
@@ -436,24 +412,12 @@ StorePathSet RemoteStore::queryValidDerivers(const StorePath & path)
 StorePathSet RemoteStore::queryDerivationOutputs(const StorePath & path)
 {
     auto conn(getConnection());
-    if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 0x16) {
-        return Store::queryDerivationOutputs(path);
-    }
     conn->to << wopQueryDerivationOutputs << printStorePath(path);
     conn.processStderr();
     return readStorePaths<StorePathSet>(*this, conn->from);
 }
 
 
-OutputPathMap RemoteStore::queryDerivationOutputMap(const StorePath & path)
-{
-    auto conn(getConnection());
-    conn->to << wopQueryDerivationOutputMap << printStorePath(path);
-    conn.processStderr();
-    return readOutputPathMap(*this, conn->from);
-
-}
-
 std::optional<StorePath> RemoteStore::queryPathFromHashPart(const std::string & hashPart)
 {
     auto conn(getConnection());
@@ -466,7 +430,7 @@ std::optional<StorePath> RemoteStore::queryPathFromHashPart(const std::string &
 
 
 void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
-    RepairFlag repair, CheckSigsFlag checkSigs)
+    RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor)
 {
     auto conn(getConnection());
 

src/libstore/remote-store.hh

@@ -51,7 +51,6 @@ public:
 
     StorePathSet queryDerivationOutputs(const StorePath & path) override;
 
-    OutputPathMap queryDerivationOutputMap(const StorePath & path) override;
     std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override;
 
     StorePathSet querySubstitutablePaths(const StorePathSet & paths) override;
@@ -60,7 +59,8 @@ public:
         SubstitutablePathInfos & infos) override;
 
     void addToStore(const ValidPathInfo & info, Source & nar,
-        RepairFlag repair, CheckSigsFlag checkSigs) override;
+        RepairFlag repair, CheckSigsFlag checkSigs,
+        std::shared_ptr<FSAccessor> accessor) override;
 
     StorePath addToStore(const string & name, const Path & srcPath,
         FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256,

src/libstore/s3-binary-cache-store.cc

@@ -7,6 +7,7 @@
 #include "globals.hh"
 #include "compression.hh"
 #include "filetransfer.hh"
+#include "istringstream_nocopy.hh"
 
 #include <aws/core/Aws.h>
 #include <aws/core/VersionConfig.h>
@@ -261,11 +262,12 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
     std::shared_ptr<TransferManager> transferManager;
     std::once_flag transferManagerCreated;
 
-    void uploadFile(const std::string & path,
-        std::shared_ptr<std::basic_iostream<char>> istream,
+    void uploadFile(const std::string & path, const std::string & data,
         const std::string & mimeType,
         const std::string & contentEncoding)
     {
+        auto stream = std::make_shared<istringstream_nocopy>(data);
+
         auto maxThreads = std::thread::hardware_concurrency();
 
         static std::shared_ptr<Aws::Utils::Threading::PooledThreadExecutor>
@@ -305,7 +307,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
 
             std::shared_ptr<TransferHandle> transferHandle =
                 transferManager->UploadFile(
-                    istream, bucketName, path, mimeType,
+                    stream, bucketName, path, mimeType,
                     Aws::Map<Aws::String, Aws::String>(),
                     nullptr /*, contentEncoding */);
 
@@ -331,7 +333,9 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
             if (contentEncoding != "")
                 request.SetContentEncoding(contentEncoding);
 
-            request.SetBody(istream);
+            auto stream = std::make_shared<istringstream_nocopy>(data);
+
+            request.SetBody(stream);
 
             auto result = checkAws(fmt("AWS error uploading '%s'", path),
                 s3Helper.client->PutObject(request));
@@ -343,34 +347,25 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
             std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1)
                 .count();
 
-        auto size = istream->tellg();
-
-        printInfo("uploaded 's3://%s/%s' (%d bytes) in %d ms",
-            bucketName, path, size, duration);
+        printInfo(format("uploaded 's3://%1%/%2%' (%3% bytes) in %4% ms") %
+                  bucketName % path % data.size() % duration);
 
         stats.putTimeMs += duration;
-        stats.putBytes += size;
+        stats.putBytes += data.size();
         stats.put++;
     }
 
-    void upsertFile(const std::string & path,
-        std::shared_ptr<std::basic_iostream<char>> istream,
+    void upsertFile(const std::string & path, const std::string & data,
         const std::string & mimeType) override
     {
-        auto compress = [&](std::string compression)
-        {
-            auto compressed = nix::compress(compression, StreamToSourceAdapter(istream).drain());
-            return std::make_shared<std::stringstream>(std::move(*compressed));
-        };
-
         if (narinfoCompression != "" && hasSuffix(path, ".narinfo"))
-            uploadFile(path, compress(narinfoCompression), mimeType, narinfoCompression);
+            uploadFile(path, *compress(narinfoCompression, data), mimeType, narinfoCompression);
         else if (lsCompression != "" && hasSuffix(path, ".ls"))
-            uploadFile(path, compress(lsCompression), mimeType, lsCompression);
+            uploadFile(path, *compress(lsCompression, data), mimeType, lsCompression);
         else if (logCompression != "" && hasPrefix(path, "log/"))
-            uploadFile(path, compress(logCompression), mimeType, logCompression);
+            uploadFile(path, *compress(logCompression, data), mimeType, logCompression);
         else
-            uploadFile(path, istream, mimeType, "");
+            uploadFile(path, data, mimeType, "");
     }
 
     void getFile(const std::string & path, Sink & sink) override
@@ -415,7 +410,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
             for (auto object : contents) {
                 auto & key = object.GetKey();
                 if (key.size() != 40 || !hasSuffix(key, ".narinfo")) continue;
-                paths.insert(parseStorePath(storeDir + "/" + key.substr(0, key.size() - 8) + "-" + MissingName));
+                paths.insert(parseStorePath(storeDir + "/" + key.substr(0, key.size() - 8) + "-unknown"));
             }
 
             marker = res.GetNextMarker();

src/libstore/store-api.cc

@@ -7,7 +7,6 @@
 #include "json.hh"
 #include "derivations.hh"
 #include "url.hh"
-#include "archive.hh"
 
 #include <future>
 
@@ -21,15 +20,15 @@ bool Store::isInStore(const Path & path) const
 }
 
 
-std::pair<StorePath, Path> Store::toStorePath(const Path & path) const
+Path Store::toStorePath(const Path & path) const
 {
     if (!isInStore(path))
         throw Error("path '%1%' is not in the Nix store", path);
     Path::size_type slash = path.find('/', storeDir.size() + 1);
     if (slash == Path::npos)
-        return {parseStorePath(path), ""};
+        return path;
     else
-        return {parseStorePath(std::string_view(path).substr(0, slash)), path.substr(slash)};
+        return Path(path, 0, slash);
 }
 
 
@@ -42,14 +41,14 @@ Path Store::followLinksToStore(std::string_view _path) const
         path = absPath(target, dirOf(path));
     }
     if (!isInStore(path))
-        throw BadStorePath("path '%1%' is not in the Nix store", path);
+        throw NotInStore("path '%1%' is not in the Nix store", path);
     return path;
 }
 
 
 StorePath Store::followLinksToStorePath(std::string_view path) const
 {
-    return toStorePath(followLinksToStore(path)).first;
+    return parseStorePath(toStorePath(followLinksToStore(path)));
 }
 
 
@@ -222,40 +221,6 @@ StorePath Store::computeStorePathForText(const string & name, const string & s,
 }
 
 
-ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath,
-    FileIngestionMethod method, HashType hashAlgo,
-    std::optional<Hash> expectedCAHash)
-{
-    /* FIXME: inefficient: we're reading/hashing 'tmpFile' three
-       times. */
-
-    auto [narHash, narSize] = hashPath(htSHA256, srcPath);
-
-    auto hash = method == FileIngestionMethod::Recursive
-        ? hashAlgo == htSHA256
-          ? narHash
-          : hashPath(hashAlgo, srcPath).first
-        : hashFile(hashAlgo, srcPath);
-
-    if (expectedCAHash && expectedCAHash != hash)
-        throw Error("hash mismatch for '%s'", srcPath);
-
-    ValidPathInfo info(makeFixedOutputPath(method, hash, name));
-    info.narHash = narHash;
-    info.narSize = narSize;
-    info.ca = FixedOutputHash { .method = method, .hash = hash };
-
-    if (!isValidPath(info.path)) {
-        auto source = sinkToSource([&](Sink & sink) {
-            dumpPath(srcPath, sink);
-        });
-        addToStore(info, *source);
-    }
-
-    return info;
-}
-
-
 Store::Store(const Params & params)
     : Config(params)
     , state({(size_t) pathInfoCacheSize})
@@ -277,16 +242,6 @@ bool Store::PathInfoCacheValue::isKnownNow()
     return std::chrono::steady_clock::now() < time_point + ttl;
 }
 
-StorePathSet Store::queryDerivationOutputs(const StorePath & path)
-{
-    auto outputMap = this->queryDerivationOutputMap(path);
-    StorePathSet outputPaths;
-    for (auto & i: outputMap) {
-        outputPaths.emplace(std::move(i.second));
-    }
-    return outputPaths;
-}
-
 bool Store::isValidPath(const StorePath & storePath)
 {
     std::string hashPart(storePath.hashPart());
@@ -351,14 +306,6 @@ ref<const ValidPathInfo> Store::queryPathInfo(const StorePath & storePath)
 }
 
 
-static bool goodStorePath(const StorePath & expected, const StorePath & actual)
-{
-    return
-        expected.hashPart() == actual.hashPart()
-        && (expected.name() == Store::MissingName || expected.name() == actual.name());
-}
-
-
 void Store::queryPathInfo(const StorePath & storePath,
     Callback<ref<const ValidPathInfo>> callback) noexcept
 {
@@ -386,7 +333,7 @@ void Store::queryPathInfo(const StorePath & storePath,
                     state_->pathInfoCache.upsert(hashPart,
                         res.first == NarInfoDiskCache::oInvalid ? PathInfoCacheValue{} : PathInfoCacheValue{ .value = res.second });
                     if (res.first == NarInfoDiskCache::oInvalid ||
-                        !goodStorePath(storePath, res.second->path))
+                        res.second->path != storePath)
                         throw InvalidPath("path '%s' is not valid", printStorePath(storePath));
                 }
                 return callback(ref<const ValidPathInfo>(res.second));
@@ -398,7 +345,7 @@ void Store::queryPathInfo(const StorePath & storePath,
     auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback));
 
     queryPathInfoUncached(storePath,
-        {[this, storePathS{printStorePath(storePath)}, hashPart, callbackPtr](std::future<std::shared_ptr<const ValidPathInfo>> fut) {
+        {[this, storePath{printStorePath(storePath)}, hashPart, callbackPtr](std::future<std::shared_ptr<const ValidPathInfo>> fut) {
 
             try {
                 auto info = fut.get();
@@ -411,11 +358,9 @@ void Store::queryPathInfo(const StorePath & storePath,
                     state_->pathInfoCache.upsert(hashPart, PathInfoCacheValue { .value = info });
                 }
 
-                auto storePath = parseStorePath(storePathS);
-
-                if (!info || !goodStorePath(storePath, info->path)) {
+                if (!info || info->path != parseStorePath(storePath)) {
                     stats.narInfoMissing++;
-                    throw InvalidPath("path '%s' is not valid", storePathS);
+                    throw InvalidPath("path '%s' is not valid", storePath);
                 }
 
                 (*callbackPtr)(ref<const ValidPathInfo>(info));
@@ -560,7 +505,7 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & store
                     if (!narInfo->url.empty())
                         jsonPath.attr("url", narInfo->url);
                     if (narInfo->fileHash)
-                        jsonPath.attr("downloadHash", narInfo->fileHash.to_string(hashBase, true));
+                        jsonPath.attr("downloadHash", narInfo->fileHash.to_string(Base32, true));
                     if (narInfo->fileSize)
                         jsonPath.attr("downloadSize", narInfo->fileSize);
                     if (showClosureSize)

src/libstore/store-api.hh

@@ -31,7 +31,7 @@ MakeError(InvalidPath, Error);
 MakeError(Unsupported, Error);
 MakeError(SubstituteGone, Error);
 MakeError(SubstituterDisabled, Error);
-MakeError(BadStorePath, Error);
+MakeError(NotInStore, Error);
 
 
 class FSAccessor;
@@ -317,9 +317,9 @@ public:
        the Nix store. */
     bool isStorePath(std::string_view path) const;
 
-    /* Split a path like /nix/store/<hash>-<name>/<bla> into
-       /nix/store/<hash>-<name> and /<bla>. */
-    std::pair<StorePath, Path> toStorePath(const Path & path) const;
+    /* Chop off the parts after the top-level store name, e.g.,
+       /nix/store/abcd-foo/bar => /nix/store/abcd-foo. */
+    Path toStorePath(const Path & path) const;
 
     /* Follow symlinks until we end up with a path in the Nix store. */
     Path followLinksToStore(std::string_view path) const;
@@ -384,16 +384,13 @@ public:
         SubstituteFlag maybeSubstitute = NoSubstitute);
 
     /* Query the set of all valid paths. Note that for some store
-       backends, the name part of store paths may be replaced by 'x'
-       (i.e. you'll get /nix/store/<hash>-x rather than
+       backends, the name part of store paths may be omitted
+       (i.e. you'll get /nix/store/<hash> rather than
        /nix/store/<hash>-<name>). Use queryPathInfo() to obtain the
-       full store path. FIXME: should return a set of
-       std::variant<StorePath, HashPart> to get rid of this hack. */
+       full store path. */
     virtual StorePathSet queryAllValidPaths()
     { unsupported("queryAllValidPaths"); }
 
-    constexpr static const char * MissingName = "x";
-
     /* Query information about a valid path. It is permitted to omit
        the name part of the store path. */
     ref<const ValidPathInfo> queryPathInfo(const StorePath & path);
@@ -421,11 +418,8 @@ public:
     virtual StorePathSet queryValidDerivers(const StorePath & path) { return {}; };
 
     /* Query the outputs of the derivation denoted by `path'. */
-    virtual StorePathSet queryDerivationOutputs(const StorePath & path);
-
-    /* Query the mapping outputName=>outputPath for the given derivation */
-    virtual OutputPathMap queryDerivationOutputMap(const StorePath & path)
-    { unsupported("queryDerivationOutputMap"); }
+    virtual StorePathSet queryDerivationOutputs(const StorePath & path)
+    { unsupported("queryDerivationOutputs"); }
 
     /* Query the full store path given the hash part of a valid store
        path, or empty if the path doesn't exist. */
@@ -442,7 +436,8 @@ public:
 
     /* Import a path into the store. */
     virtual void addToStore(const ValidPathInfo & info, Source & narSource,
-        RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs) = 0;
+        RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs,
+        std::shared_ptr<FSAccessor> accessor = 0) = 0;
 
     /* Copy the contents of a path to the store and register the
        validity the resulting path.  The resulting path is returned.
@@ -452,13 +447,6 @@ public:
         FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256,
         PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) = 0;
 
-    /* Copy the contents of a path to the store and register the
-       validity the resulting path, using a constant amount of
-       memory. */
-    ValidPathInfo addToStoreSlow(std::string_view name, const Path & srcPath,
-        FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256,
-        std::optional<Hash> expectedCAHash = {});
-
     // FIXME: remove?
     virtual StorePath addToStoreFromDump(const string & dump, const string & name,
         FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair)
@@ -625,7 +613,8 @@ public:
        the Nix store. Optionally, the contents of the NARs are
        preloaded into the specified FS accessor to speed up subsequent
        access. */
-    StorePaths importPaths(Source & source, CheckSigsFlag checkSigs = CheckSigs);
+    StorePaths importPaths(Source & source, std::shared_ptr<FSAccessor> accessor,
+        CheckSigsFlag checkSigs = CheckSigs);
 
     struct Stats
     {

src/libstore/worker-protocol.hh

@@ -6,7 +6,7 @@ namespace nix {
 #define WORKER_MAGIC_1 0x6e697863
 #define WORKER_MAGIC_2 0x6478696f
 
-#define PROTOCOL_VERSION 0x116
+#define PROTOCOL_VERSION 0x115
 #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
 #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
 
@@ -30,7 +30,7 @@ typedef enum {
     wopSetOptions = 19,
     wopCollectGarbage = 20,
     wopQuerySubstitutablePathInfo = 21,
-    wopQueryDerivationOutputs = 22, // obsolete
+    wopQueryDerivationOutputs = 22,
     wopQueryAllValidPaths = 23,
     wopQueryFailedPaths = 24,
     wopClearFailedPaths = 25,
@@ -49,7 +49,6 @@ typedef enum {
     wopNarFromPath = 38,
     wopAddToStoreNar = 39,
     wopQueryMissing = 40,
-    wopQueryDerivationOutputMap = 41,
 } WorkerOp;
 
 
@@ -70,6 +69,5 @@ template<class T> T readStorePaths(const Store & store, Source & from);
 
 void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths);
 
-void writeOutputPathMap(const Store & store, Sink & out, const OutputPathMap & paths);
 
 }

src/libutil/ansicolor.hh

@@ -11,7 +11,7 @@ namespace nix {
 #define ANSI_GREEN "\e[32;1m"
 #define ANSI_YELLOW "\e[33;1m"
 #define ANSI_BLUE "\e[34;1m"
-#define ANSI_MAGENTA "\e[35;1m"
-#define ANSI_CYAN "\e[36;1m"
+#define ANSI_MAGENTA "\e[35m;1m"
+#define ANSI_CYAN "\e[36m;1m"
 
 }

src/libutil/archive.cc

@@ -262,7 +262,7 @@ static void parse(ParseSink & sink, Source & source, const Path & path)
                             names[name] = 0;
                     }
                 } else if (s == "node") {
-                    if (name.empty()) throw badArchive("entry name missing");
+                    if (s.empty()) throw badArchive("entry name missing");
                     parse(sink, source, path + "/" + name);
                 } else
                     throw badArchive("unknown field " + s);

src/libutil/archive.hh

@@ -63,12 +63,11 @@ struct ParseSink
     virtual void createSymlink(const Path & path, const string & target) { };
 };
 
-struct TeeParseSink : ParseSink
+struct TeeSink : ParseSink
 {
-    StringSink saved;
     TeeSource source;
 
-    TeeParseSink(Source & source) : source(source, saved) { }
+    TeeSink(Source & source) : source(source) { }
 };
 
 void parseDump(ParseSink & sink, Source & source);

src/libutil/error.cc

@@ -7,11 +7,14 @@
 
 namespace nix {
 
+
 const std::string nativeSystem = SYSTEM;
 
-BaseError & BaseError::addTrace(std::optional<ErrPos> e, hintformat hint)
+// addPrefix is used for show-trace.  Strings added with addPrefix
+// will print ahead of the error itself.
+BaseError & BaseError::addPrefix(const FormatOrString & fs)
 {
-    err.traces.push_front(Trace { .pos = e, .hint = hint});
+    prefix_ = fs.s + prefix_;
     return *this;
 }
 
@@ -25,7 +28,7 @@ const string& BaseError::calcWhat() const
         err.name = sname();
 
         std::ostringstream oss;
-        showErrorInfo(oss, err, false);
+        oss << err;
         what_ = oss.str();
 
         return *what_;
@@ -53,114 +56,28 @@ string showErrPos(const ErrPos &errPos)
     }
 }
 
-std::optional<LinesOfCode> getCodeLines(const ErrPos &errPos)
-{
-    if (errPos.line <= 0)
-        return std::nullopt;
-
-    if (errPos.origin == foFile) {
-        LinesOfCode loc;
-        try {
-            AutoCloseFD fd = open(errPos.file.c_str(), O_RDONLY | O_CLOEXEC);
-            if (!fd) {
-                logError(SysError("opening file '%1%'", errPos.file).info());
-                return std::nullopt;
-            }
-            else
-            {
-                // count the newlines.
-                int count = 0;
-                string line;
-                int pl = errPos.line - 1;
-                do
-                {
-                    line = readLine(fd.get());
-                    ++count;
-                    if (count < pl)
-                    {
-                        ;
-                    }
-                    else if (count == pl) {
-                        loc.prevLineOfCode = line;
-                    } else if (count == pl + 1) {
-                        loc.errLineOfCode = line;
-                    } else if (count == pl + 2) {
-                        loc.nextLineOfCode = line;
-                        break;
-                    }
-                } while (true);
-                return loc;
-            }
-        }
-        catch (EndOfFile &eof) {
-            if (loc.errLineOfCode.has_value())
-                return loc;
-            else
-                return std::nullopt;
-        }
-        catch (std::exception &e) {
-            printError("error reading nix file: %s\n%s", errPos.file, e.what());
-            return std::nullopt;
-        }
-    } else {
-        std::istringstream iss(errPos.file);
-        // count the newlines.
-        int count = 0;
-        string line;
-        int pl = errPos.line - 1;
-
-        LinesOfCode loc;
-
-        do
-        {
-            std::getline(iss, line);
-            ++count;
-            if (count < pl)
-            {
-                ;
-            }
-            else if (count == pl) {
-                loc.prevLineOfCode = line;
-            } else if (count == pl + 1) {
-                loc.errLineOfCode = line;
-            } else if (count == pl + 2) {
-                loc.nextLineOfCode = line;
-                break;
-            }
-
-            if (!iss.good())
-                break;
-        } while (true);
-
-        return loc;
-    }
-}
-
-// print lines of code to the ostream, indicating the error column.
-void printCodeLines(std::ostream &out,
-    const string &prefix,
-    const ErrPos &errPos,
-    const LinesOfCode &loc)
+// if nixCode contains lines of code, print them to the ostream, indicating the error column.
+void printCodeLines(std::ostream &out, const string &prefix, const NixCode &nixCode)
 {
     // previous line of code.
-    if (loc.prevLineOfCode.has_value()) {
+    if (nixCode.prevLineOfCode.has_value()) {
         out << std::endl
             << fmt("%1% %|2$5d|| %3%",
-            prefix,
-            (errPos.line - 1),
-            *loc.prevLineOfCode);
+                prefix,
+                (nixCode.errPos.line - 1),
+                *nixCode.prevLineOfCode);
     }
 
-    if (loc.errLineOfCode.has_value()) {
+    if (nixCode.errLineOfCode.has_value()) {
         // line of code containing the error.
         out << std::endl
             << fmt("%1% %|2$5d|| %3%",
-            prefix,
-            (errPos.line),
-            *loc.errLineOfCode);
+                prefix,
+                (nixCode.errPos.line),
+                *nixCode.errLineOfCode);
         // error arrows for the column range.
-        if (errPos.column > 0) {
-            int start = errPos.column;
+        if (nixCode.errPos.column > 0) {
+            int start = nixCode.errPos.column;
             std::string spaces;
             for (int i = 0; i < start; ++i) {
                 spaces.append(" ");
@@ -170,49 +87,23 @@ void printCodeLines(std::ostream &out,
 
             out << std::endl
                 << fmt("%1%      |%2%" ANSI_RED "%3%" ANSI_NORMAL,
-                prefix,
-                spaces,
-                arrows);
+                    prefix,
+                    spaces,
+                    arrows);
         }
     }
 
     // next line of code.
-    if (loc.nextLineOfCode.has_value()) {
+    if (nixCode.nextLineOfCode.has_value()) {
         out << std::endl
             << fmt("%1% %|2$5d|| %3%",
-            prefix,
-            (errPos.line + 1),
-            *loc.nextLineOfCode);
+                prefix,
+                (nixCode.errPos.line + 1),
+                *nixCode.nextLineOfCode);
     }
 }
 
-void printAtPos(const string &prefix, const ErrPos &pos, std::ostream &out)
-{
-    if (pos)
-    {
-        switch (pos.origin) {
-            case foFile: {
-                out << prefix << ANSI_BLUE << "at: " << ANSI_YELLOW << showErrPos(pos) <<
-                    ANSI_BLUE << " in file: " << ANSI_NORMAL << pos.file;
-                break;
-            }
-            case foString: {
-                out << prefix << ANSI_BLUE << "at: " << ANSI_YELLOW << showErrPos(pos) <<
-                    ANSI_BLUE << " from string" << ANSI_NORMAL;
-                break;
-            }
-            case foStdin: {
-                out << prefix << ANSI_BLUE << "at: " << ANSI_YELLOW << showErrPos(pos) <<
-                    ANSI_BLUE << " from stdin" << ANSI_NORMAL;
-                break;
-            }
-            default:
-                throw Error("invalid FileOrigin in errPos");
-        }
-    }
-}
-
-std::ostream& showErrorInfo(std::ostream &out, const ErrorInfo &einfo, bool showTrace)
+std::ostream& operator<<(std::ostream &out, const ErrorInfo &einfo)
 {
     auto errwidth = std::max<size_t>(getWindowSize().second, 20);
     string prefix = "";
@@ -267,12 +158,8 @@ std::ostream& showErrorInfo(std::ostream &out, const ErrorInfo &einfo, bool show
         }
     }
 
-    auto ndl = prefix.length()
-        + filterANSIEscapes(levelString, true).length()
-        + 7
-        + einfo.name.length()
-        + einfo.programName.value_or("").length();
-    auto dashwidth = std::max<int>(errwidth - ndl, 3);
+    auto ndl = prefix.length() + levelString.length() + 3 + einfo.name.length() + einfo.programName.value_or("").length();
+    auto dashwidth = ndl > (errwidth - 3) ? 3 : errwidth - ndl;
 
     std::string dashes(dashwidth, '-');
 
@@ -292,9 +179,16 @@ std::ostream& showErrorInfo(std::ostream &out, const ErrorInfo &einfo, bool show
             einfo.programName.value_or(""));
 
     bool nl = false;  // intersperse newline between sections.
-    if (einfo.errPos.has_value() && (*einfo.errPos)) {
-        out << prefix << std::endl;
-        printAtPos(prefix, *einfo.errPos, out);
+    if (einfo.nixCode.has_value()) {
+        if (einfo.nixCode->errPos.file != "") {
+            // filename, line, column.
+            out << std::endl << fmt("%1%in file: " ANSI_BLUE "%2% %3%" ANSI_NORMAL,
+                prefix,
+                einfo.nixCode->errPos.file,
+                showErrPos(einfo.nixCode->errPos));
+        } else {
+            out << std::endl << fmt("%1%from command line argument", prefix);
+        }
         nl = true;
     }
 
@@ -306,16 +200,12 @@ std::ostream& showErrorInfo(std::ostream &out, const ErrorInfo &einfo, bool show
         nl = true;
     }
 
-    if (einfo.errPos.has_value() && (*einfo.errPos)) {
-        auto loc = getCodeLines(*einfo.errPos);
-
-        // lines of code.
-        if (loc.has_value()) {
-            if (nl)
-                out << std::endl << prefix;
-            printCodeLines(out, prefix, *einfo.errPos, *loc);
-            nl = true;
-        }
+    // lines of code.
+    if (einfo.nixCode.has_value() && einfo.nixCode->errLineOfCode.has_value()) {
+        if (nl)
+            out << std::endl << prefix;
+        printCodeLines(out, prefix, *einfo.nixCode);
+        nl = true;
     }
 
     // hint
@@ -326,54 +216,6 @@ std::ostream& showErrorInfo(std::ostream &out, const ErrorInfo &einfo, bool show
         nl = true;
     }
 
-    // traces
-    if (showTrace && !einfo.traces.empty())
-    {
-        const string tracetitle(" show-trace ");
-
-        int fill = errwidth - tracetitle.length();
-        int lw = 0;
-        int rw = 0;
-        const int min_dashes = 3;
-        if (fill > min_dashes * 2) {
-            if (fill % 2 != 0) {
-                lw = fill / 2;
-                rw = lw + 1;
-            }
-            else
-            {
-                lw = rw = fill / 2;
-            }
-        }
-        else
-            lw = rw = min_dashes;
-
-        if (nl)
-            out << std::endl << prefix;
-
-        out << ANSI_BLUE << std::string(lw, '-') << tracetitle << std::string(rw, '-') << ANSI_NORMAL;
-
-        for (auto iter = einfo.traces.rbegin(); iter != einfo.traces.rend(); ++iter)
-        {
-            out << std::endl << prefix;
-            out << ANSI_BLUE << "trace: " << ANSI_NORMAL << iter->hint.str();
-
-            if (iter->pos.has_value() && (*iter->pos)) {
-                auto pos = iter->pos.value();
-                out << std::endl << prefix;
-                printAtPos(prefix, pos, out);
-
-                auto loc = getCodeLines(pos);
-                if (loc.has_value())
-                {
-                    out << std::endl << prefix;
-                    printCodeLines(out, prefix, pos, *loc);
-                    out << std::endl << prefix;
-                }
-            }
-        }
-    }
-
     return out;
 }
 }

src/libutil/error.hh

@@ -1,8 +1,8 @@
 #pragma once
 
+
 #include "ref.hh"
 #include "types.hh"
-#include "fmt.hh"
 
 #include <cstring>
 #include <list>
@@ -10,9 +10,7 @@
 #include <map>
 #include <optional>
 
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <fcntl.h>
+#include "fmt.hh"
 
 /* Before 4.7, gcc's std::exception uses empty throw() specifiers for
  * its (virtual) destructor and what() in c++11 mode, in violation of spec
@@ -27,20 +25,20 @@ namespace nix {
 
 /*
 
-   This file defines two main structs/classes used in nix error handling.
+This file defines two main structs/classes used in nix error handling.
 
-   ErrorInfo provides a standard payload of error information, with conversion to string
-   happening in the logger rather than at the call site.
+ErrorInfo provides a standard payload of error information, with conversion to string
+happening in the logger rather than at the call site.
 
-   BaseError is the ancestor of nix specific exceptions (and Interrupted), and contains
-   an ErrorInfo.
+BaseError is the ancestor of nix specific exceptions (and Interrupted), and contains
+an ErrorInfo.
 
-   ErrorInfo structs are sent to the logger as part of an exception, or directly with the
-   logError or logWarning macros.
+ErrorInfo structs are sent to the logger as part of an exception, or directly with the
+logError or logWarning macros.
 
-   See the error-demo.cc program for usage examples.
+See the error-demo.cc program for usage examples.
 
- */
+*/
 
 typedef enum {
     lvlError = 0,
@@ -52,25 +50,11 @@ typedef enum {
     lvlVomit
 } Verbosity;
 
-typedef enum {
-    foFile,
-    foStdin,
-    foString
-} FileOrigin;
-
-// the lines of code surrounding an error.
-struct LinesOfCode {
-    std::optional<string> prevLineOfCode;
-    std::optional<string> errLineOfCode;
-    std::optional<string> nextLineOfCode;
-};
-
 // ErrPos indicates the location of an error in a nix file.
 struct ErrPos {
     int line = 0;
     int column = 0;
     string file;
-    FileOrigin origin;
 
     operator bool() const
     {
@@ -81,14 +65,9 @@ struct ErrPos {
     template <class P>
     ErrPos& operator=(const P &pos)
     {
-        origin = pos.origin;
         line = pos.line;
         column = pos.column;
-        // is file symbol null?
-        if (pos.file.set())
-            file = pos.file;
-        else
-            file = "";
+        file = pos.file;
         return *this;
     }
 
@@ -99,9 +78,11 @@ struct ErrPos {
     }
 };
 
-struct Trace {
-    std::optional<ErrPos> pos;
-    hintformat hint;
+struct NixCode {
+    ErrPos errPos;
+    std::optional<string> prevLineOfCode;
+    std::optional<string> errLineOfCode;
+    std::optional<string> nextLineOfCode;
 };
 
 struct ErrorInfo {
@@ -109,19 +90,19 @@ struct ErrorInfo {
     string name;
     string description;
     std::optional<hintformat> hint;
-    std::optional<ErrPos> errPos;
-    std::list<Trace> traces;
+    std::optional<NixCode> nixCode;
 
     static std::optional<string> programName;
 };
 
-std::ostream& showErrorInfo(std::ostream &out, const ErrorInfo &einfo, bool showTrace);
+std::ostream& operator<<(std::ostream &out, const ErrorInfo &einfo);
 
 /* BaseError should generally not be caught, as it has Interrupted as
    a subclass. Catch Error instead. */
 class BaseError : public std::exception
 {
 protected:
+    string prefix_; // used for location traces etc.
     mutable ErrorInfo err;
 
     mutable std::optional<string> what_;
@@ -132,23 +113,23 @@ public:
 
     template<typename... Args>
     BaseError(unsigned int status, const Args & ... args)
-        : err {.level = lvlError,
-            .hint = hintfmt(args...)
-            }
+        : err { .level = lvlError,
+                .hint = hintfmt(args...)
+              }
         , status(status)
     { }
 
     template<typename... Args>
     BaseError(const std::string & fs, const Args & ... args)
-        : err {.level = lvlError,
-            .hint = hintfmt(fs, args...)
-            }
+        : err { .level = lvlError,
+                .hint = hintfmt(fs, args...)
+              }
     { }
 
     BaseError(hintformat hint)
-        : err {.level = lvlError,
-            .hint = hint
-            }
+        : err { .level = lvlError,
+                .hint = hint
+              }
     { }
 
     BaseError(ErrorInfo && e)
@@ -169,17 +150,10 @@ public:
 #endif
 
     const string & msg() const { return calcWhat(); }
+    const string & prefix() const { return prefix_; }
+    BaseError & addPrefix(const FormatOrString & fs);
+
     const ErrorInfo & info() { calcWhat(); return err; }
-
-    template<typename... Args>
-    BaseError & addTrace(std::optional<ErrPos> e, const string &fs, const Args & ... args)
-    {
-        return addTrace(e, hintfmt(fs, args...));
-    }
-
-    BaseError & addTrace(std::optional<ErrPos> e, hintformat hint);
-
-    bool hasTrace() const { return !err.traces.empty(); }
 };
 
 #define MakeError(newClass, superClass) \
@@ -199,7 +173,7 @@ public:
 
     template<typename... Args>
     SysError(const Args & ... args)
-        : Error("")
+      :Error("")
     {
         errNo = errno;
         auto hf = hintfmt(args...);

src/libutil/hash.cc

@@ -8,6 +8,7 @@
 #include "hash.hh"
 #include "archive.hh"
 #include "util.hh"
+#include "istringstream_nocopy.hh"
 
 #include <sys/types.h>
 #include <sys/stat.h>
@@ -18,7 +19,7 @@ namespace nix {
 
 void Hash::init()
 {
-    assert(type);
+    if (!type) abort();
     switch (*type) {
     case htMD5: hashSize = md5HashSize; break;
     case htSHA1: hashSize = sha1HashSize; break;
@@ -100,15 +101,15 @@ static string printHash32(const Hash & hash)
 
 string printHash16or32(const Hash & hash)
 {
-    assert(hash.type);
     return hash.to_string(hash.type == htMD5 ? Base16 : Base32, false);
 }
 
 
-HashType assertInitHashType(const Hash & h)
-{
-    assert(h.type);
-    return *h.type;
+HashType assertInitHashType(const Hash & h) {
+    if (h.type)
+        return *h.type;
+    else
+        abort();
 }
 
 std::string Hash::to_string(Base base, bool includeType) const
@@ -362,15 +363,14 @@ HashType parseHashType(const string & s)
 string printHashType(HashType ht)
 {
     switch (ht) {
-    case htMD5: return "md5";
-    case htSHA1: return "sha1";
-    case htSHA256: return "sha256";
-    case htSHA512: return "sha512";
-    default:
-        // illegal hash type enum value internally, as opposed to external input
-        // which should be validated with nice error message.
-        assert(false);
+    case htMD5: return "md5"; break;
+    case htSHA1: return "sha1"; break;
+    case htSHA256: return "sha256"; break;
+    case htSHA512: return "sha512"; break;
     }
+    // illegal hash type enum value internally, as opposed to external input
+    // which should be validated with nice error message.
+    abort();
 }
 
 }

src/libutil/hash.hh

@@ -10,7 +10,7 @@ namespace nix {
 MakeError(BadHash, Error);
 
 
-enum HashType : char { htMD5 = 42, htSHA1, htSHA256, htSHA512 };
+enum HashType : char { htMD5, htSHA1, htSHA256, htSHA512 };
 
 
 const int md5HashSize = 16;

src/libutil/istringstream_nocopy.hh

@@ -0,0 +1,92 @@
+/* This file provides a variant of std::istringstream that doesn't
+   copy its string argument. This is useful for large strings. The
+   caller must ensure that the string object is not destroyed while
+   it's referenced by this object. */
+
+#pragma once
+
+#include <string>
+#include <iostream>
+
+template <class CharT, class Traits = std::char_traits<CharT>, class Allocator = std::allocator<CharT>>
+class basic_istringbuf_nocopy : public std::basic_streambuf<CharT, Traits>
+{
+public:
+    typedef std::basic_string<CharT, Traits, Allocator> string_type;
+
+    typedef typename std::basic_streambuf<CharT, Traits>::off_type off_type;
+
+    typedef typename std::basic_streambuf<CharT, Traits>::pos_type pos_type;
+
+    typedef typename std::basic_streambuf<CharT, Traits>::int_type int_type;
+
+    typedef typename std::basic_streambuf<CharT, Traits>::traits_type traits_type;
+
+private:
+    const string_type & s;
+
+    off_type off;
+
+public:
+    basic_istringbuf_nocopy(const string_type & s) : s{s}, off{0}
+    {
+    }
+
+private:
+    pos_type seekoff(off_type off, std::ios_base::seekdir dir, std::ios_base::openmode which)
+    {
+        if (which & std::ios_base::in) {
+            this->off = dir == std::ios_base::beg
+                ? off
+                : (dir == std::ios_base::end
+                    ? s.size() + off
+                    : this->off + off);
+        }
+        return pos_type(this->off);
+    }
+
+    pos_type seekpos(pos_type pos, std::ios_base::openmode which)
+    {
+        return seekoff(pos, std::ios_base::beg, which);
+    }
+
+    std::streamsize showmanyc()
+    {
+        return s.size() - off;
+    }
+
+    int_type underflow()
+    {
+        if (typename string_type::size_type(off) == s.size())
+            return traits_type::eof();
+        return traits_type::to_int_type(s[off]);
+    }
+
+    int_type uflow()
+    {
+        if (typename string_type::size_type(off) == s.size())
+            return traits_type::eof();
+        return traits_type::to_int_type(s[off++]);
+    }
+
+    int_type pbackfail(int_type ch)
+    {
+        if (off == 0 || (ch != traits_type::eof() && ch != s[off - 1]))
+            return traits_type::eof();
+
+        return traits_type::to_int_type(s[--off]);
+    }
+
+};
+
+template <class CharT, class Traits = std::char_traits<CharT>, class Allocator = std::allocator<CharT>>
+class basic_istringstream_nocopy : public std::basic_iostream<CharT, Traits>
+{
+    typedef basic_istringbuf_nocopy<CharT, Traits, Allocator> buf_type;
+    buf_type buf;
+public:
+    basic_istringstream_nocopy(const typename buf_type::string_type & s) :
+        std::basic_iostream<CharT, Traits>(&buf), buf(s) {};
+};
+
+typedef basic_istringstream_nocopy<char> istringstream_nocopy;

src/libutil/logging.cc

@@ -1,6 +1,5 @@
 #include "logging.hh"
 #include "util.hh"
-#include "config.hh"
 
 #include <atomic>
 #include <nlohmann/json.hpp>
@@ -8,10 +7,6 @@
 
 namespace nix {
 
-LoggerSettings loggerSettings;
-
-static GlobalConfig::Register r1(&loggerSettings);
-
 static thread_local ActivityId curActivity = 0;
 
 ActivityId getCurActivity()
@@ -77,7 +72,7 @@ public:
     void logEI(const ErrorInfo & ei) override
     {
         std::stringstream oss;
-        showErrorInfo(oss, ei, loggerSettings.showTrace.get());
+        oss << ei;
 
         log(ei.level, oss.str());
     }
@@ -178,7 +173,7 @@ struct JSONLogger : Logger {
     void logEI(const ErrorInfo & ei) override
     {
         std::ostringstream oss;
-        showErrorInfo(oss, ei, loggerSettings.showTrace.get());
+        oss << ei;
 
         nlohmann::json json;
         json["action"] = "msg";

src/libutil/logging.hh

@@ -2,7 +2,6 @@
 
 #include "types.hh"
 #include "error.hh"
-#include "config.hh"
 
 namespace nix {
 
@@ -35,16 +34,6 @@ typedef enum {
 
 typedef uint64_t ActivityId;
 
-struct LoggerSettings : Config
-{
-    Setting<bool> showTrace{this,
-        false,
-        "show-trace",
-        "Whether to show a stack trace on evaluation errors."};
-};
-
-extern LoggerSettings loggerSettings;
-
 class Logger
 {
     friend struct Activity;

src/libutil/serialise.hh

@@ -166,30 +166,17 @@ struct StringSource : Source
 };
 
 
-/* A sink that writes all incoming data to two other sinks. */
-struct TeeSink : Sink
-{
-    Sink & sink1, & sink2;
-    TeeSink(Sink & sink1, Sink & sink2) : sink1(sink1), sink2(sink2) { }
-    virtual void operator () (const unsigned char * data, size_t len)
-    {
-        sink1(data, len);
-        sink2(data, len);
-    }
-};
-
-
-/* Adapter class of a Source that saves all data read to a sink. */
+/* Adapter class of a Source that saves all data read to `s'. */
 struct TeeSource : Source
 {
     Source & orig;
-    Sink & sink;
-    TeeSource(Source & orig, Sink & sink)
-        : orig(orig), sink(sink) { }
+    ref<std::string> data;
+    TeeSource(Source & orig)
+        : orig(orig), data(make_ref<std::string>()) { }
     size_t read(unsigned char * data, size_t len)
     {
         size_t n = orig.read(data, len);
-        sink(data, len);
+        this->data->append((const char *) data, n);
         return n;
     }
 };
@@ -349,27 +336,4 @@ Source & operator >> (Source & in, bool & b)
 }
 
 
-/* An adapter that converts a std::basic_istream into a source. */
-struct StreamToSourceAdapter : Source
-{
-    std::shared_ptr<std::basic_istream<char>> istream;
-
-    StreamToSourceAdapter(std::shared_ptr<std::basic_istream<char>> istream)
-        : istream(istream)
-    { }
-
-    size_t read(unsigned char * data, size_t len) override
-    {
-        if (!istream->read((char *) data, len)) {
-            if (istream->eof()) {
-                if (istream->gcount() == 0)
-                    throw EndOfFile("end of file");
-            } else
-                throw Error("I/O error in StreamToSourceAdapter");
-        }
-        return istream->gcount();
-    }
-};
-
-
 }

src/libutil/tests/logging.cc

@@ -11,13 +11,6 @@ namespace nix {
      * logEI
      * --------------------------------------------------------------------------*/
 
-    const char *test_file =
-        "previous line of code\n"
-        "this is the problem line of code\n"
-        "next line of code\n";
-    const char *one_liner =
-        "this is the other problem line of code";
-
     TEST(logEI, catpuresBasicProperties) {
 
         MakeError(TestError, Error);
@@ -144,6 +137,7 @@ namespace nix {
      * logError
      * --------------------------------------------------------------------------*/
 
+
     TEST(logError, logErrorWithoutHintOrCode) {
         testing::internal::CaptureStderr();
 
@@ -158,7 +152,7 @@ namespace nix {
 
     TEST(logError, logErrorWithPreviousAndNextLinesOfCode) {
         SymbolTable testTable;
-        auto problem_file = testTable.create(test_file);
+        auto problem_file = testTable.create("myfile.nix");
 
         testing::internal::CaptureStderr();
 
@@ -168,16 +162,21 @@ namespace nix {
                 .hint = hintfmt("this hint has %1% templated %2%!!",
                     "yellow",
                     "values"),
-                .errPos = Pos(foString, problem_file, 02, 13),
-            });
+                .nixCode = NixCode {
+                    .errPos = Pos(problem_file, 40, 13),
+                    .prevLineOfCode = "previous line of code",
+                    .errLineOfCode = "this is the problem line of code",
+                    .nextLineOfCode = "next line of code",
+                }});
+
 
         auto str = testing::internal::GetCapturedStderr();
-        ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- error name --- error-unit-test\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(2:13)\x1B[34;1m from string\x1B[0m\n\nerror with code lines\n\n     1| previous line of code\n     2| this is the problem line of code\n      |             \x1B[31;1m^\x1B[0m\n     3| next line of code\n\nthis hint has \x1B[33;1myellow\x1B[0m templated \x1B[33;1mvalues\x1B[0m!!\n");
+        ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- error name --- error-unit-test\x1B[0m\nin file: \x1B[34;1mmyfile.nix (40:13)\x1B[0m\n\nerror with code lines\n\n    39| previous line of code\n    40| this is the problem line of code\n      |             \x1B[31;1m^\x1B[0m\n    41| next line of code\n\nthis hint has \x1B[33;1myellow\x1B[0m templated \x1B[33;1mvalues\x1B[0m!!\n");
     }
 
-    TEST(logError, logErrorWithInvalidFile) {
+    TEST(logError, logErrorWithoutLinesOfCode) {
         SymbolTable testTable;
-        auto problem_file = testTable.create("invalid filename");
+        auto problem_file = testTable.create("myfile.nix");
         testing::internal::CaptureStderr();
 
         logError({
@@ -186,23 +185,28 @@ namespace nix {
                 .hint = hintfmt("this hint has %1% templated %2%!!",
                     "yellow",
                     "values"),
-                .errPos = Pos(foFile, problem_file, 02, 13)
-            });
+                .nixCode = NixCode {
+                    .errPos = Pos(problem_file, 40, 13)
+                }});
 
         auto str = testing::internal::GetCapturedStderr();
-        ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- SysError --- error-unit-test\x1B[0m\nopening file '\x1B[33;1minvalid filename\x1B[0m': \x1B[33;1mNo such file or directory\x1B[0m\n\x1B[31;1merror:\x1B[0m\x1B[34;1m --- error name --- error-unit-test\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(2:13)\x1B[34;1m in file: \x1B[0minvalid filename\n\nerror without any code lines.\n\nthis hint has \x1B[33;1myellow\x1B[0m templated \x1B[33;1mvalues\x1B[0m!!\n");
+        ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- error name --- error-unit-test\x1B[0m\nin file: \x1B[34;1mmyfile.nix (40:13)\x1B[0m\n\nerror without any code lines.\n\nthis hint has \x1B[33;1myellow\x1B[0m templated \x1B[33;1mvalues\x1B[0m!!\n");
     }
 
     TEST(logError, logErrorWithOnlyHintAndName) {
+        SymbolTable testTable;
+        auto problem_file = testTable.create("myfile.nix");
         testing::internal::CaptureStderr();
 
         logError({
                 .name = "error name",
                 .hint = hintfmt("hint %1%", "only"),
-            });
+                .nixCode = NixCode {
+                    .errPos = Pos(problem_file, 40, 13)
+                }});
 
         auto str = testing::internal::GetCapturedStderr();
-        ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- error name --- error-unit-test\x1B[0m\nhint \x1B[33;1monly\x1B[0m\n");
+        ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- error name --- error-unit-test\x1B[0m\nin file: \x1B[34;1mmyfile.nix (40:13)\x1B[0m\n\nhint \x1B[33;1monly\x1B[0m\n");
 
     }
 
@@ -215,18 +219,18 @@ namespace nix {
 
         logWarning({
                 .name = "name",
-                .description = "warning description",
+                .description = "error description",
                 .hint = hintfmt("there was a %1%", "warning"),
             });
 
         auto str = testing::internal::GetCapturedStderr();
-        ASSERT_STREQ(str.c_str(), "\x1B[33;1mwarning:\x1B[0m\x1B[34;1m --- name --- error-unit-test\x1B[0m\nwarning description\n\nthere was a \x1B[33;1mwarning\x1B[0m\n");
+        ASSERT_STREQ(str.c_str(), "\x1B[33;1mwarning:\x1B[0m\x1B[34;1m --- name --- error-unit-test\x1B[0m\nerror description\n\nthere was a \x1B[33;1mwarning\x1B[0m\n");
     }
 
     TEST(logWarning, logWarningWithFileLineNumAndCode) {
 
         SymbolTable testTable;
-        auto problem_file = testTable.create(test_file);
+        auto problem_file = testTable.create("myfile.nix");
 
         testing::internal::CaptureStderr();
 
@@ -236,73 +240,18 @@ namespace nix {
                 .hint = hintfmt("this hint has %1% templated %2%!!",
                     "yellow",
                     "values"),
-                .errPos = Pos(foStdin, problem_file, 2, 13),
-            });
+                .nixCode = NixCode {
+                    .errPos = Pos(problem_file, 40, 13),
+                    .prevLineOfCode = std::nullopt,
+                    .errLineOfCode = "this is the problem line of code",
+                    .nextLineOfCode = std::nullopt
+                }});
 
 
         auto str = testing::internal::GetCapturedStderr();
-        ASSERT_STREQ(str.c_str(), "\x1B[33;1mwarning:\x1B[0m\x1B[34;1m --- warning name --- error-unit-test\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(2:13)\x1B[34;1m from stdin\x1B[0m\n\nwarning description\n\n     1| previous line of code\n     2| this is the problem line of code\n      |             \x1B[31;1m^\x1B[0m\n     3| next line of code\n\nthis hint has \x1B[33;1myellow\x1B[0m templated \x1B[33;1mvalues\x1B[0m!!\n");
+        ASSERT_STREQ(str.c_str(), "\x1B[33;1mwarning:\x1B[0m\x1B[34;1m --- warning name --- error-unit-test\x1B[0m\nin file: \x1B[34;1mmyfile.nix (40:13)\x1B[0m\n\nwarning description\n\n    40| this is the problem line of code\n      |             \x1B[31;1m^\x1B[0m\n\nthis hint has \x1B[33;1myellow\x1B[0m templated \x1B[33;1mvalues\x1B[0m!!\n");
     }
 
-    /* ----------------------------------------------------------------------------
-     * traces
-     * --------------------------------------------------------------------------*/
-
-    TEST(addTrace, showTracesWithShowTrace) {
-        SymbolTable testTable;
-        auto problem_file = testTable.create(test_file);
-        auto oneliner_file = testTable.create(one_liner);
-        auto invalidfilename = testTable.create("invalid filename");
-
-        auto e = AssertionError(ErrorInfo {
-                .name = "wat",
-                .description = "show-traces",
-                .hint = hintfmt("it has been %1% days since our last error", "zero"),
-                .errPos = Pos(foString, problem_file, 2, 13),
-            });
-
-        e.addTrace(Pos(foStdin, oneliner_file, 1, 19), "while trying to compute %1%", 42);
-        e.addTrace(std::nullopt, "while doing something without a %1%", "pos");
-        e.addTrace(Pos(foFile, invalidfilename, 100, 1), "missing %s", "nix file");
-
-        testing::internal::CaptureStderr();
-
-        loggerSettings.showTrace.assign(true);
-
-        logError(e.info());
-
-        auto str = testing::internal::GetCapturedStderr();
-        ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- SysError --- error-unit-test\x1B[0m\nopening file '\x1B[33;1minvalid filename\x1B[0m': \x1B[33;1mNo such file or directory\x1B[0m\n\x1B[31;1merror:\x1B[0m\x1B[34;1m --- AssertionError --- error-unit-test\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(2:13)\x1B[34;1m from string\x1B[0m\n\nshow-traces\n\n     1| previous line of code\n     2| this is the problem line of code\n      |             \x1B[31;1m^\x1B[0m\n     3| next line of code\n\nit has been \x1B[33;1mzero\x1B[0m days since our last error\n\x1B[34;1m---- show-trace ----\x1B[0m\n\x1B[34;1mtrace: \x1B[0mwhile trying to compute \x1B[33;1m42\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(1:19)\x1B[34;1m from stdin\x1B[0m\n\n     1| this is the other problem line of code\n      |                   \x1B[31;1m^\x1B[0m\n\n\x1B[34;1mtrace: \x1B[0mwhile doing something without a \x1B[33;1mpos\x1B[0m\n\x1B[34;1mtrace: \x1B[0mmissing \x1B[33;1mnix file\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(100:1)\x1B[34;1m in file: \x1B[0minvalid filename\n");
-    }
-
-    TEST(addTrace, hideTracesWithoutShowTrace) {
-        SymbolTable testTable;
-        auto problem_file = testTable.create(test_file);
-        auto oneliner_file = testTable.create(one_liner);
-        auto invalidfilename = testTable.create("invalid filename");
-
-        auto e = AssertionError(ErrorInfo {
-                .name = "wat",
-                .description = "hide traces",
-                .hint = hintfmt("it has been %1% days since our last error", "zero"),
-                .errPos = Pos(foString, problem_file, 2, 13),
-            });
-
-        e.addTrace(Pos(foStdin, oneliner_file, 1, 19), "while trying to compute %1%", 42);
-        e.addTrace(std::nullopt, "while doing something without a %1%", "pos");
-        e.addTrace(Pos(foFile, invalidfilename, 100, 1), "missing %s", "nix file");
-
-        testing::internal::CaptureStderr();
-
-        loggerSettings.showTrace.assign(false);
-
-        logError(e.info());
-
-        auto str = testing::internal::GetCapturedStderr();
-        ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- AssertionError --- error-unit-test\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(2:13)\x1B[34;1m from string\x1B[0m\n\nhide traces\n\n     1| previous line of code\n     2| this is the problem line of code\n      |             \x1B[31;1m^\x1B[0m\n     3| next line of code\n\nit has been \x1B[33;1mzero\x1B[0m days since our last error\n");
-    }
-
-
     /* ----------------------------------------------------------------------------
      * hintfmt
      * --------------------------------------------------------------------------*/
@@ -340,22 +289,4 @@ namespace nix {
             "what about this " ANSI_YELLOW "%3%" ANSI_NORMAL " " ANSI_YELLOW "one" ANSI_NORMAL);
 
     }
-
-    /* ----------------------------------------------------------------------------
-     * ErrPos
-     * --------------------------------------------------------------------------*/
-
-    TEST(errpos, invalidPos) {
-
-      // contains an invalid symbol, which we should not dereference!
-      Pos invalid;
-
-      // constructing without access violation.
-      ErrPos ep(invalid);
-    
-      // assignment without access violation.
-      ep = invalid;
-
-    }
-
 }

src/nix-env/nix-env.cc

@@ -593,7 +593,7 @@ static void upgradeDerivations(Globals & globals,
                 } else newElems.push_back(i);
 
             } catch (Error & e) {
-                e.addTrace(std::nullopt, "while trying to find an upgrade for '%s'", i.queryName());
+                e.addPrefix(fmt("while trying to find an upgrade for '%s':\n", i.queryName()));
                 throw;
             }
         }
@@ -1185,7 +1185,7 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs)
         } catch (AssertionError & e) {
             printMsg(lvlTalkative, "skipping derivation named '%1%' which gives an assertion failure", i.queryName());
         } catch (Error & e) {
-            e.addTrace(std::nullopt, "while querying the derivation named '%1%'", i.queryName());
+            e.addPrefix(fmt("while querying the derivation named '%1%':\n", i.queryName()));
             throw;
         }
     }

src/nix-prefetch-url/nix-prefetch-url.cc

@@ -153,15 +153,14 @@ static int _main(int argc, char * * argv)
 
         /* If an expected hash is given, the file may already exist in
            the store. */
-        std::optional<Hash> expectedHash;
-        Hash hash;
+        Hash hash, expectedHash(ht);
         std::optional<StorePath> storePath;
         if (args.size() == 2) {
             expectedHash = Hash(args[1], ht);
             const auto recursive = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
-            storePath = store->makeFixedOutputPath(recursive, *expectedHash, name);
+            storePath = store->makeFixedOutputPath(recursive, expectedHash, name);
             if (store->isValidPath(*storePath))
-                hash = *expectedHash;
+                hash = expectedHash;
             else
                 storePath.reset();
         }
@@ -201,12 +200,22 @@ static int _main(int argc, char * * argv)
                     tmpFile = unpacked;
             }
 
-            const auto method = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
+            /* FIXME: inefficient; addToStore() will also hash
+               this. */
+            hash = unpack ? hashPath(ht, tmpFile).first : hashFile(ht, tmpFile);
 
-            auto info = store->addToStoreSlow(name, tmpFile, method, ht, expectedHash);
-            storePath = info.path;
-            assert(info.ca);
-            hash = getContentAddressHash(*info.ca);
+            if (expectedHash != Hash(ht) && expectedHash != hash)
+                throw Error("hash mismatch for '%1%'", uri);
+
+            const auto recursive = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
+
+            /* Copy the file to the Nix store. FIXME: if RemoteStore
+               implemented addToStoreFromDump() and downloadFile()
+               supported a sink, we could stream the download directly
+               into the Nix store. */
+            storePath = store->addToStore(name, tmpFile, recursive, ht);
+
+            assert(*storePath == store->makeFixedOutputPath(recursive, hash, name));
         }
 
         stopProgressBar();

src/nix-store/nix-store.cc

@@ -174,10 +174,10 @@ static void opAdd(Strings opFlags, Strings opArgs)
    store. */
 static void opAddFixed(Strings opFlags, Strings opArgs)
 {
-    auto method = FileIngestionMethod::Flat;
+    auto recursive = FileIngestionMethod::Flat;
 
     for (auto & i : opFlags)
-        if (i == "--recursive") method = FileIngestionMethod::Recursive;
+        if (i == "--recursive") recursive = FileIngestionMethod::Recursive;
         else throw UsageError("unknown flag '%1%'", i);
 
     if (opArgs.empty())
@@ -187,7 +187,7 @@ static void opAddFixed(Strings opFlags, Strings opArgs)
     opArgs.pop_front();
 
     for (auto & i : opArgs)
-        std::cout << fmt("%s\n", store->printStorePath(store->addToStoreSlow(baseNameOf(i), i, method, hashAlgo).path));
+        cout << fmt("%s\n", store->printStorePath(store->addToStore(std::string(baseNameOf(i)), i, recursive, hashAlgo)));
 }
 
 
@@ -671,7 +671,7 @@ static void opImport(Strings opFlags, Strings opArgs)
     if (!opArgs.empty()) throw UsageError("no arguments expected");
 
     FdSource source(STDIN_FILENO);
-    auto paths = store->importPaths(source, NoCheckSigs);
+    auto paths = store->importPaths(source, nullptr, NoCheckSigs);
 
     for (auto & i : paths)
         cout << fmt("%s\n", store->printStorePath(i)) << std::flush;
@@ -878,7 +878,7 @@ static void opServe(Strings opFlags, Strings opArgs)
 
             case cmdImportPaths: {
                 if (!writeAllowed) throw Error("importing paths is not allowed");
-                store->importPaths(in, NoCheckSigs); // FIXME: should we skip sig checking?
+                store->importPaths(in, nullptr, NoCheckSigs); // FIXME: should we skip sig checking?
                 out << 1; // indicate success
                 break;
             }

src/nix/develop.cc

@@ -50,7 +50,7 @@ BuildEnvironment readEnvironment(const Path & path)
         R"re((?:\$?'(?:[^'\\]|\\[abeEfnrtv\\'"?])*'))re";
 
     static std::string indexedArrayRegex =
-        R"re((?:\(( *\[[0-9]+\]="(?:[^"\\]|\\.)*")*\)))re";
+        R"re((?:\(( *\[[0-9]+]="(?:[^"\\]|\\.)*")**\)))re";
 
     static std::regex varRegex(
         "^(" + varNameRegex + ")=(" + simpleStringRegex + "|" + quotedStringRegex + "|" + indexedArrayRegex + ")\n");
@@ -135,7 +135,13 @@ StorePath getDerivationEnvironment(ref<Store> store, const StorePath & drvPath)
     drv.inputSrcs.insert(std::move(getEnvShPath));
     Hash h = hashDerivationModulo(*store, drv, true);
     auto shellOutPath = store->makeOutputPath("out", h, drvName);
-    drv.outputs.insert_or_assign("out", DerivationOutput { .path = shellOutPath });
+    drv.outputs.insert_or_assign("out", DerivationOutput {
+        .path = shellOutPath,
+        .hash = FixedOutputHash {
+            .method = FileIngestionMethod::Flat,
+            .hash = Hash { },
+        },
+    });
     drv.env["out"] = store->printStorePath(shellOutPath);
     auto shellDrvPath2 = writeDerivation(store, drv, drvName);
 

src/nix/diff-closures.cc

@@ -1,134 +0,0 @@
-#include "command.hh"
-#include "shared.hh"
-#include "store-api.hh"
-#include "common-args.hh"
-#include "names.hh"
-
-#include <regex>
-
-using namespace nix;
-
-struct Info
-{
-    std::string outputName;
-};
-
-// name -> version -> store paths
-typedef std::map<std::string, std::map<std::string, std::map<StorePath, Info>>> GroupedPaths;
-
-GroupedPaths getClosureInfo(ref<Store> store, const StorePath & toplevel)
-{
-    StorePathSet closure;
-    store->computeFSClosure({toplevel}, closure);
-
-    GroupedPaths groupedPaths;
-
-    for (auto & path : closure) {
-        /* Strip the output name. Unfortunately this is ambiguous (we
-           can't distinguish between output names like "bin" and
-           version suffixes like "unstable"). */
-        static std::regex regex("(.*)-([a-z]+|lib32|lib64)");
-        std::smatch match;
-        std::string name(path.name());
-        std::string outputName;
-        if (std::regex_match(name, match, regex)) {
-            name = match[1];
-            outputName = match[2];
-        }
-
-        DrvName drvName(name);
-        groupedPaths[drvName.name][drvName.version].emplace(path, Info { .outputName = outputName });
-    }
-
-    return groupedPaths;
-}
-
-std::string showVersions(const std::set<std::string> & versions)
-{
-    if (versions.empty()) return "∅";
-    std::set<std::string> versions2;
-    for (auto & version : versions)
-        versions2.insert(version.empty() ? "ε" : version);
-    return concatStringsSep(", ", versions2);
-}
-
-struct CmdDiffClosures : SourceExprCommand
-{
-    std::string _before, _after;
-
-    CmdDiffClosures()
-    {
-        expectArg("before", &_before);
-        expectArg("after", &_after);
-    }
-
-    std::string description() override
-    {
-        return "show what packages and versions were added and removed between two closures";
-    }
-
-    Category category() override { return catSecondary; }
-
-    Examples examples() override
-    {
-        return {
-            {
-                "To show what got added and removed between two versions of the NixOS system profile:",
-                "nix diff-closures /nix/var/nix/profiles/system-655-link /nix/var/nix/profiles/system-658-link",
-            },
-        };
-    }
-
-    void run(ref<Store> store) override
-    {
-        auto before = parseInstallable(*this, store, _before, false);
-        auto beforePath = toStorePath(store, Build, before);
-        auto after = parseInstallable(*this, store, _after, false);
-        auto afterPath = toStorePath(store, NoBuild, after);
-
-        auto beforeClosure = getClosureInfo(store, beforePath);
-        auto afterClosure = getClosureInfo(store, afterPath);
-
-        std::set<std::string> allNames;
-        for (auto & [name, _] : beforeClosure) allNames.insert(name);
-        for (auto & [name, _] : afterClosure) allNames.insert(name);
-
-        for (auto & name : allNames) {
-            auto & beforeVersions = beforeClosure[name];
-            auto & afterVersions = afterClosure[name];
-
-            auto totalSize = [&](const std::map<std::string, std::map<StorePath, Info>> & versions)
-            {
-                uint64_t sum = 0;
-                for (auto & [_, paths] : versions)
-                    for (auto & [path, _] : paths)
-                        sum += store->queryPathInfo(path)->narSize;
-                return sum;
-            };
-
-            auto beforeSize = totalSize(beforeVersions);
-            auto afterSize = totalSize(afterVersions);
-            auto sizeDelta = (int64_t) afterSize - (int64_t) beforeSize;
-            auto showDelta = abs(sizeDelta) >= 8 * 1024;
-
-            std::set<std::string> removed, unchanged;
-            for (auto & [version, _] : beforeVersions)
-                if (!afterVersions.count(version)) removed.insert(version); else unchanged.insert(version);
-
-            std::set<std::string> added;
-            for (auto & [version, _] : afterVersions)
-                if (!beforeVersions.count(version)) added.insert(version);
-
-            if (showDelta || !removed.empty() || !added.empty()) {
-                std::vector<std::string> items;
-                if (!removed.empty() || !added.empty())
-                    items.push_back(fmt("%s → %s", showVersions(removed), showVersions(added)));
-                if (showDelta)
-                    items.push_back(fmt("%s%+.1f KiB" ANSI_NORMAL, sizeDelta > 0 ? ANSI_RED : ANSI_GREEN, sizeDelta / 1024.0));
-                std::cout << fmt("%s: %s\n", name, concatStringsSep(", ", items));
-            }
-        }
-    }
-};
-
-static auto r1 = registerCommand<CmdDiffClosures>("diff-closures");

src/nix/installables.cc

@@ -94,8 +94,8 @@ struct InstallableStorePath : Installable
     ref<Store> store;
     StorePath storePath;
 
-    InstallableStorePath(ref<Store> store, StorePath && storePath)
-        : store(store), storePath(std::move(storePath)) { }
+    InstallableStorePath(ref<Store> store, const Path & storePath)
+        : store(store), storePath(store->parseStorePath(storePath)) { }
 
     std::string what() override { return store->printStorePath(storePath); }
 
@@ -228,11 +228,11 @@ static std::vector<std::shared_ptr<Installable>> parseInstallables(
             result.push_back(std::make_shared<InstallableExpr>(cmd, s));
 
         else if (s.find("/") != std::string::npos) {
-            try {
-                result.push_back(std::make_shared<InstallableStorePath>(
-                        store,
-                        store->toStorePath(store->followLinksToStore(s)).first));
-            } catch (BadStorePath &) { }
+
+            auto path = store->toStorePath(store->followLinksToStore(s));
+
+            if (store->isStorePath(path))
+                result.push_back(std::make_shared<InstallableStorePath>(store, path));
         }
 
         else if (s == "" || std::regex_match(s, attrPathRegex))

src/nix/repl.cc

@@ -211,12 +211,12 @@ void NixRepl::mainLoop(const std::vector<std::string> & files)
                 // input without clearing the input so far.
                 continue;
             } else {
-              printMsg(lvlError, e.msg());
+              printMsg(lvlError, error + "%1%%2%", (settings.showTrace ? e.prefix() : ""), e.msg());
             }
         } catch (Error & e) {
-          printMsg(lvlError, e.msg());
+            printMsg(lvlError, error + "%1%%2%", (settings.showTrace ? e.prefix() : ""), e.msg());
         } catch (Interrupted & e) {
-          printMsg(lvlError, e.msg());
+            printMsg(lvlError, error + "%1%%2%", (settings.showTrace ? e.prefix() : ""), e.msg());
         }
 
         // We handled the current input fully, so we should clear it

src/nix/search.cc

@@ -216,7 +216,7 @@ struct CmdSearch : SourceExprCommand, MixJSON
             } catch (AssertionError & e) {
             } catch (Error & e) {
                 if (!toplevel) {
-                    e.addTrace(std::nullopt, "While evaluating the attribute '%s'", attrPath);
+                    e.addPrefix(fmt("While evaluating the attribute '%s':\n", attrPath));
                     throw;
                 }
             }

src/nix/verify.cc

@@ -77,16 +77,13 @@ struct CmdVerify : StorePathsCommand
             try {
                 checkInterrupt();
 
+                Activity act2(*logger, lvlInfo, actUnknown, fmt("checking '%s'", storePath));
+
                 MaintainCount<std::atomic<size_t>> mcActive(active);
                 update();
 
                 auto info = store->queryPathInfo(store->parseStorePath(storePath));
 
-                // Note: info->path can be different from storePath
-                // for binary cache stores when using --all (since we
-                // can't enumerate names efficiently).
-                Activity act2(*logger, lvlInfo, actUnknown, fmt("checking '%s'", store->printStorePath(info->path)));
-
                 if (!noContents) {
 
                     std::unique_ptr<AbstractHashSink> hashSink;

tests/binary-cache.sh

@@ -182,56 +182,3 @@ clearCacheCache
 nix-store -r $outPath --substituters "file://$cacheDir2 file://$cacheDir" --trusted-public-keys "$publicKey"
 
 fi # HAVE_LIBSODIUM
-
-
-unset _NIX_FORCE_HTTP
-
-
-# Test 'nix verify --all' on a binary cache.
-nix verify -vvvvv --all --store file://$cacheDir --no-trust
-
-
-# Test local NAR caching.
-narCache=$TEST_ROOT/nar-cache
-rm -rf $narCache
-mkdir $narCache
-
-[[ $(nix cat-store --store "file://$cacheDir?local-nar-cache=$narCache" $outPath/foobar) = FOOBAR ]]
-
-rm -rfv "$cacheDir/nar"
-
-[[ $(nix cat-store --store "file://$cacheDir?local-nar-cache=$narCache" $outPath/foobar) = FOOBAR ]]
-
-(! nix cat-store --store file://$cacheDir $outPath/foobar)
-
-
-# Test NAR listing generation.
-clearCache
-
-outPath=$(nix-build --no-out-link -E '
-  with import ./config.nix;
-  mkDerivation {
-    name = "nar-listing";
-    buildCommand = "mkdir $out; echo foo > $out/bar; ln -s xyzzy $out/link";
-  }
-')
-
-nix copy --to file://$cacheDir?write-nar-listing=1 $outPath
-
-[[ $(cat $cacheDir/$(basename $outPath).ls) = '{"version":1,"root":{"type":"directory","entries":{"bar":{"type":"regular","size":4,"narOffset":232},"link":{"type":"symlink","target":"xyzzy"}}}}' ]]
-
-
-# Test debug info index generation.
-clearCache
-
-outPath=$(nix-build --no-out-link -E '
-  with import ./config.nix;
-  mkDerivation {
-    name = "debug-info";
-    buildCommand = "mkdir -p $out/lib/debug/.build-id/02; echo foo > $out/lib/debug/.build-id/02/623eda209c26a59b1a8638ff7752f6b945c26b.debug";
-  }
-')
-
-nix copy --to "file://$cacheDir?index-debug-info=1&compression=none" $outPath
-
-[[ $(cat $cacheDir/debuginfo/02623eda209c26a59b1a8638ff7752f6b945c26b.debug) = '{"archive":"../nar/100vxs724qr46phz8m24iswmg9p3785hsyagz0kchf6q6gf06sw6.nar","member":"lib/debug/.build-id/02/623eda209c26a59b1a8638ff7752f6b945c26b.debug"}' ]]

tests/build-hook.nix

@@ -1,39 +1,23 @@
-{ busybox }:
-
 with import ./config.nix;
 
 let
 
-  mkDerivation = args:
-    derivation ({
-      inherit system;
-      builder = busybox;
-      args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")];
-    } // removeAttrs args ["builder" "meta"])
-    // { meta = args.meta or {}; };
-
   input1 = mkDerivation {
-    shell = busybox;
-    name = "build-remote-input-1";
-    buildCommand = "echo FOO > $out";
+    name = "build-hook-input-1";
+    buildCommand = "mkdir $out; echo FOO > $out/foo";
     requiredSystemFeatures = ["foo"];
   };
 
   input2 = mkDerivation {
-    shell = busybox;
-    name = "build-remote-input-2";
-    buildCommand = "echo BAR > $out";
+    name = "build-hook-input-2";
+    buildCommand = "mkdir $out; echo BAR > $out/bar";
   };
 
 in
 
   mkDerivation {
-    shell = busybox;
-    name = "build-remote";
-    buildCommand =
-      ''
-        read x < ${input1}
-        read y < ${input2}
-        echo $x$y > $out
-      '';
+    name = "build-hook";
+    builder = ./dependencies.builder0.sh;
+    input1 = " " + input1 + "/.";
+    input2 = " ${input2}/.";
   }

tests/build-remote.sh

@@ -3,29 +3,22 @@ source common.sh
 clearStore
 
 if ! canUseSandbox; then exit; fi
-if ! [[ $busybox =~ busybox ]]; then exit; fi
+if [[ ! $SHELL =~ /nix/store ]]; then exit; fi
 
-chmod -R u+w $TEST_ROOT/machine0 || true
-chmod -R u+w $TEST_ROOT/machine1 || true
-chmod -R u+w $TEST_ROOT/machine2 || true
-rm -rf $TEST_ROOT/machine0 $TEST_ROOT/machine1 $TEST_ROOT/machine2
-rm -f $TEST_ROOT/result
+chmod -R u+w $TEST_ROOT/store0 || true
+chmod -R u+w $TEST_ROOT/store1 || true
+rm -rf $TEST_ROOT/store0 $TEST_ROOT/store1
 
-unset NIX_STORE_DIR
-unset NIX_STATE_DIR
-
-# Note: ssh://localhost bypasses ssh, directly invoking nix-store as a
-# child process. This allows us to test LegacySSHStore::buildDerivation().
-nix build -L -v -f build-hook.nix -o $TEST_ROOT/result --max-jobs 0 \
-  --arg busybox $busybox \
-  --store $TEST_ROOT/machine0 \
-  --builders "ssh://localhost?remote-store=$TEST_ROOT/machine1; $TEST_ROOT/machine2 - - 1 1 foo" \
+nix build -f build-hook.nix -o $TEST_ROOT/result --max-jobs 0 \
+  --sandbox-paths /nix/store --sandbox-build-dir /build-tmp \
+  --builders "$TEST_ROOT/store0; $TEST_ROOT/store1 - - 1 1 foo" \
   --system-features foo
 
-outPath=$(readlink -f $TEST_ROOT/result)
+outPath=$TEST_ROOT/result
 
-cat $TEST_ROOT/machine0/$outPath | grep FOOBAR
+cat $outPath/foobar | grep FOOBAR
 
-# Ensure that input1 was built on store2 due to the required feature.
-(! nix path-info --store $TEST_ROOT/machine1 --all | grep builder-build-remote-input-1.sh)
-nix path-info --store $TEST_ROOT/machine2 --all | grep builder-build-remote-input-1.sh
+# Ensure that input1 was built on store1 due to the required feature.
+p=$(readlink -f $outPath/input-2)
+(! nix path-info --store $TEST_ROOT/store0 --all | grep builder-build-hook-input-1.sh)
+nix path-info --store $TEST_ROOT/store1 --all | grep builder-build-hook-input-1.sh

tests/common.sh.in

@@ -1,6 +1,6 @@
 set -e
 
-export TEST_ROOT=$(realpath ${TMPDIR:-/tmp}/nix-test)/${TEST_NAME:-default}
+export TEST_ROOT=$(realpath ${TMPDIR:-/tmp}/nix-test)
 export NIX_STORE_DIR
 if ! NIX_STORE_DIR=$(readlink -f $TEST_ROOT/store 2> /dev/null); then
     # Maybe the build directory is symlinked.
@@ -11,7 +11,6 @@ export NIX_LOCALSTATE_DIR=$TEST_ROOT/var
 export NIX_LOG_DIR=$TEST_ROOT/var/log/nix
 export NIX_STATE_DIR=$TEST_ROOT/var/nix
 export NIX_CONF_DIR=$TEST_ROOT/etc
-export NIX_DAEMON_SOCKET_PATH=$TEST_ROOT/daemon-socket
 unset NIX_USER_CONF_FILES
 export _NIX_TEST_SHARED=$TEST_ROOT/shared
 if [[ -n $NIX_STORE ]]; then
@@ -36,7 +35,6 @@ export xmllint="@xmllint@"
 export SHELL="@bash@"
 export PAGER=cat
 export HAVE_SODIUM="@HAVE_SODIUM@"
-export busybox="@sandbox_shell@"
 
 export version=@PACKAGE_VERSION@
 export system=@system@
@@ -77,7 +75,7 @@ startDaemon() {
     rm -f $NIX_STATE_DIR/daemon-socket/socket
     nix-daemon &
     for ((i = 0; i < 30; i++)); do
-        if [ -e $NIX_DAEMON_SOCKET_PATH ]; then break; fi
+        if [ -e $NIX_STATE_DIR/daemon-socket/socket ]; then break; fi
         sleep 1
     done
     pidDaemon=$!

tests/gc-auto.sh

@@ -13,32 +13,24 @@ fake_free=$TEST_ROOT/fake-free
 export _NIX_TEST_FREE_SPACE_FILE=$fake_free
 echo 1100 > $fake_free
 
-fifoLock=$TEST_ROOT/fifoLock
-mkfifo "$fifoLock"
-
 expr=$(cat <<EOF
 with import ./config.nix; mkDerivation {
   name = "gc-A";
   buildCommand = ''
     set -x
     [[ \$(ls \$NIX_STORE/*-garbage? | wc -l) = 3 ]]
-
     mkdir \$out
     echo foo > \$out/bar
-
-    # Pretend that we run out of space
-    echo 100 > ${fake_free}.tmp1
+    echo 1...
+    sleep 2
+    echo 200 > ${fake_free}.tmp1
     mv ${fake_free}.tmp1 $fake_free
-
-    # Wait for the GC to run
-    for i in {1..20}; do
-        echo ''\${i}...
-        if [[ \$(ls \$NIX_STORE/*-garbage? | wc -l) = 1 ]]; then
-            exit 0
-        fi
-        sleep 1
-    done
-    exit 1
+    echo 2...
+    sleep 2
+    echo 3...
+    sleep 2
+    echo 4...
+    [[ \$(ls \$NIX_STORE/*-garbage? | wc -l) = 1 ]]
   '';
 }
 EOF
@@ -51,9 +43,15 @@ with import ./config.nix; mkDerivation {
     set -x
     mkdir \$out
     echo foo > \$out/bar
-
-    # Wait for the first build to finish
-    cat "$fifoLock"
+    echo 1...
+    sleep 2
+    echo 200 > ${fake_free}.tmp2
+    mv ${fake_free}.tmp2 $fake_free
+    echo 2...
+    sleep 2
+    echo 3...
+    sleep 2
+    echo 4...
   '';
 }
 EOF
@@ -61,19 +59,12 @@ EOF
 
 nix build -v -o $TEST_ROOT/result-A -L "($expr)" \
     --min-free 1000 --max-free 2000 --min-free-check-interval 1 &
-pid1=$!
+pid=$!
 
 nix build -v -o $TEST_ROOT/result-B -L "($expr2)" \
-    --min-free 1000 --max-free 2000 --min-free-check-interval 1 &
-pid2=$!
+    --min-free 1000 --max-free 2000 --min-free-check-interval 1
 
-# Once the first build is done, unblock the second one.
-# If the first build fails, we need to postpone the failure to still allow
-# the second one to finish
-wait "$pid1" || FIRSTBUILDSTATUS=$?
-echo "unlock" > $fifoLock
-( exit ${FIRSTBUILDSTATUS:-0} )
-wait "$pid2"
+wait "$pid"
 
 [[ foo = $(cat $TEST_ROOT/result-A/bar) ]]
 [[ foo = $(cat $TEST_ROOT/result-B/bar) ]]

tests/gc-concurrent.builder.sh

@@ -1,10 +1,7 @@
-echo "Build started" > "$lockFifo"
-
 mkdir $out
 echo $(cat $input1/foo)$(cat $input2/bar) > $out/foobar
 
-# Wait for someone to write on the fifo
-cat "$lockFifo"
+sleep 10
 
 # $out should not have been GC'ed while we were sleeping, but just in
 # case...

tests/gc-concurrent.nix

@@ -1,7 +1,5 @@
 with import ./config.nix;
 
-{ lockFifo ? null }:
-
 rec {
 
   input1 = mkDerivation {
@@ -18,7 +16,6 @@ rec {
     name = "gc-concurrent";
     builder = ./gc-concurrent.builder.sh;
     inherit input1 input2;
-    inherit lockFifo;
   };
 
   test2 = mkDerivation {

tests/gc-concurrent.sh

@@ -2,10 +2,7 @@ source common.sh
 
 clearStore
 
-lockFifo1=$TEST_ROOT/test1.fifo
-mkfifo "$lockFifo1"
-
-drvPath1=$(nix-instantiate gc-concurrent.nix -A test1 --argstr lockFifo "$lockFifo1")
+drvPath1=$(nix-instantiate gc-concurrent.nix -A test1)
 outPath1=$(nix-store -q $drvPath1)
 
 drvPath2=$(nix-instantiate gc-concurrent.nix -A test2)
@@ -25,16 +22,19 @@ ln -s $outPath3 "$NIX_STATE_DIR"/gcroots/foo2
 nix-store -rvv "$drvPath1" &
 pid1=$!
 
-# Wait for the build of $drvPath1 to start
-cat $lockFifo1
+# Start build #2 in the background after 10 seconds.
+(sleep 10 && nix-store -rvv "$drvPath2") &
+pid2=$!
 
 # Run the garbage collector while the build is running.
+sleep 6
 nix-collect-garbage
 
-# Unlock the build of $drvPath1
-echo "" > $lockFifo1
+# Wait for build #1/#2 to finish.
 echo waiting for pid $pid1 to finish...
 wait $pid1
+echo waiting for pid $pid2 to finish...
+wait $pid2
 
 # Check that the root of build #1 and its dependencies haven't been
 # deleted.  The should not be deleted by the GC because they were
@@ -42,9 +42,8 @@ wait $pid1
 cat $outPath1/foobar
 cat $outPath1/input-2/bar
 
-# Check that the build build $drvPath2 succeeds.
-# It should succeed because the derivation is a GC root.
-nix-store -rvv "$drvPath2"
+# Check that build #2 has succeeded.  It should succeed because the
+# derivation is a GC root.
 cat $outPath2/foobar
 
 rm -f "$NIX_STATE_DIR"/gcroots/foo*

tests/gc-concurrent2.builder.sh

@@ -3,3 +3,5 @@ echo $(cat $input1/foo)$(cat $input2/bar)xyzzy > $out/foobar
 
 # Check that the GC hasn't deleted the lock on our output.
 test -e "$out.lock"
+
+sleep 6

tests/init.sh

@@ -18,7 +18,6 @@ build-users-group =
 keep-derivations = false
 sandbox = false
 experimental-features = nix-command flakes
-gc-reserved-space = 0
 include nix.conf.extra
 EOF
 

tests/local.mk

@@ -40,4 +40,4 @@ tests-environment = NIX_REMOTE= $(bash) -e
 
 clean-files += $(d)/common.sh
 
-test-deps += tests/common.sh tests/config.nix tests/plugins/libplugintest.$(SO_EXT)
+installcheck: $(d)/common.sh $(d)/config.nix $(d)/plugins/libplugintest.$(SO_EXT)

tests/misc.sh

@@ -16,11 +16,6 @@ nix-env --foo 2>&1 | grep "no operation"
 nix-env -q --foo 2>&1 | grep "unknown flag"
 
 # Eval Errors.
-eval_arg_res=$(nix-instantiate --eval -E 'let a = {} // a; in a.foo' 2>&1 || true)
-echo $eval_arg_res | grep "at: (1:15) from string"
-echo $eval_arg_res | grep "infinite recursion encountered"
-
-eval_stdin_res=$(echo 'let a = {} // a; in a.foo' | nix-instantiate --eval -E - 2>&1 || true)
-echo $eval_stdin_res | grep "at: (1:15) from stdin"
-echo $eval_stdin_res | grep "infinite recursion encountered"
-
+eval_res=$(nix-instantiate --eval -E 'let a = {} // a; in a.foo' 2>&1 || true)
+echo $eval_res | grep "(string) (1:15)"
+echo $eval_res | grep "infinite recursion encountered"

tests/nix-shell.sh

@@ -55,10 +55,3 @@ chmod a+rx $TEST_ROOT/shell.shebang.rb
 
 output=$($TEST_ROOT/shell.shebang.rb abc ruby)
 [ "$output" = '-e load("'"$TEST_ROOT"'/shell.shebang.rb") -- abc ruby' ]
-
-# Test 'nix develop'.
-nix develop -f shell.nix shellDrv -c bash -c '[[ -n $stdenv ]]'
-
-# Test 'nix print-dev-env'.
-source <(nix print-dev-env -f shell.nix shellDrv)
-[[ -n $stdenv ]]

tests/post-hook.sh

@@ -2,8 +2,6 @@ source common.sh
 
 clearStore
 
-rm -f $TEST_ROOT/result
-
 export REMOTE_STORE=$TEST_ROOT/remote_store
 
 # Build the dependencies and push them to the remote store

tests/recursive.sh

@@ -5,8 +5,6 @@ if [[ $(uname) != Linux ]]; then exit; fi
 
 clearStore
 
-rm -f $TEST_ROOT/result
-
 export unreachable=$(nix add-to-store ./recursive.sh)
 
 nix --experimental-features 'nix-command recursive-nix' build -o $TEST_ROOT/result -L '(

tests/structured-attrs.sh

@@ -2,8 +2,6 @@ source common.sh
 
 clearStore
 
-rm -f $TEST_ROOT/result
-
 nix-build structured-attrs.nix -A all -o $TEST_ROOT/result
 
 [[ $(cat $TEST_ROOT/result/foo) = bar ]]