Add 'nix diff-closures' command

This command makes it easier to see what changed between two closures,
i.e. what packages/versions got added or removed, and whether there
were any notable changes in path size.

For example:

  $ nix diff-closures /nix/var/nix/profiles/system-655-link /nix/var/nix/profiles/system-658-link
  blender-bin: 2.83.0 → 2.83.2, -294.2 KiB
  curl: 7.68.0 → 7.70.0, +19.1 KiB
  firmware-linux-nonfree: 2020-01-22 → 2020-05-19, +30827.7 KiB
  ibus: -21.8 KiB
  initrd-linux: 5.4.46 → 5.4.51, +16.9 KiB
  libexif: 0.6.21 → 0.6.22, +497.6 KiB
  linux: 5.4.46 → 5.4.51, +13.2 KiB
  mesa: 19.3.3 → 19.3.5, -183.9 KiB
  nix: 2.4pre20200701_6ff9aa8 → 2.4pre20200708_9223603, +9.7 KiB
  nix-bash-completions: 0.6.8 → ∅, -57.6 KiB
  nixos-system-hagbard: 20.03.20200615.a84b797 → 20.03.20200713.add5529
  nvidia-persistenced: 440.82 → 440.100
  nvidia-settings: 440.82 → 440.100
  nvidia-x11: 440.82-5.4.46 → 440.100-5.4.51, +664.7 KiB
  pcre: 8.43 → 8.44
  php: 7.3.16 → 7.3.20, -26.2 KiB
  python3.7-youtube-dl: 2020.06.06 → 2020.06.16.1, +8.4 KiB
  samba: 4.11.5 → 4.11.9, +30.1 KiB
  sane-backends: 1.0.28 → 1.0.30, +680.5 KiB
  source: -182.0 KiB
  zfs-kernel: 0.8.3-5.4.46 → 0.8.4-5.4.51, +9.9 KiB
  zfs-user: 0.8.3 → 0.8.4, +20.1 KiB

.dir-locals.el

@@ -1,6 +1,7 @@
 ((c++-mode . (
   (c-file-style . "k&r")
   (c-basic-offset . 4)
+  (c-block-comment-prefix . "  ")
   (indent-tabs-mode . nil)
   (tab-width . 4)
   (show-trailing-whitespace . t)

.github/dependabot.yml

@@ -0,0 +1,6 @@
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"

.github/workflows/test.yml

@@ -10,5 +10,15 @@ jobs:
     runs-on: ${{ matrix.os }}
     steps:
     - uses: actions/checkout@v2
-    - uses: cachix/install-nix-action@v8
+    - uses: cachix/install-nix-action@v10
+    - run: nix-build release.nix --arg nix '{ outPath = ./.; revCount = 123; shortRev = "abcdefgh"; }' --arg systems '[ builtins.currentSystem ]' -A installerScript -A perlBindings
+  macos_perf_test:
+    runs-on: macos-latest
+    steps:
+    - name: Disable syspolicy assessments
+      run: |
+        spctl --status
+        sudo spctl --master-disable
+    - uses: actions/checkout@v2
+    - uses: cachix/install-nix-action@v10
     - run: nix-build release.nix --arg nix '{ outPath = ./.; revCount = 123; shortRev = "abcdefgh"; }' --arg systems '[ builtins.currentSystem ]' -A installerScript -A perlBindings

maintainers/upload-release.pl

@@ -170,15 +170,5 @@ $channelsBucket->add_key(
 chdir("/home/eelco/Dev/nix-pristine") or die;
 system("git remote update origin") == 0 or die;
 system("git tag --force --sign $version $nixRev -m 'Tagging release $version'") == 0 or die;
-
-# Update the website.
-my $siteDir = "/home/eelco/Dev/nixos-homepage-pristine";
-
-system("cd $siteDir && git pull") == 0 or die;
-
-write_file("$siteDir/nix-release.tt",
-           "[%-\n" .
-           "latestNixVersion = \"$version\"\n" .
-           "-%]\n");
-
-system("cd $siteDir && git commit -a -m 'Nix $version released'") == 0 or die;
+system("git push --tags") == 0 or die;
+system("git push --force-with-lease origin $nixRev:refs/heads/latest-release") == 0 or die;

mk/run_test.sh

@@ -0,0 +1,28 @@
+#!/bin/sh
+
+set -u
+
+red=""
+green=""
+yellow=""
+normal=""
+
+post_run_msg="ran test $1..."
+if [ -t 1 ]; then
+    red=""
+    green=""
+    yellow=""
+    normal=""
+fi
+(cd $(dirname $1) && env ${TESTS_ENVIRONMENT} init.sh 2>/dev/null > /dev/null)
+log="$(cd $(dirname $1) && env ${TESTS_ENVIRONMENT} $(basename $1) 2>&1)"
+status=$?
+if [ $status -eq 0 ]; then
+  echo "$post_run_msg [${green}PASS$normal]"
+elif [ $status -eq 99 ]; then
+  echo "$post_run_msg [${yellow}SKIP$normal]"
+else
+  echo "$post_run_msg [${red}FAIL$normal]"
+  echo "$log" | sed 's/^/    /'
+  exit "$status"
+fi

mk/tests.mk

@@ -1,45 +1,15 @@
 # Run program $1 as part of ‘make installcheck’.
+
+test-deps =
+
 define run-install-test
 
-  installcheck: $1
+  installcheck: $1.test
 
-  _installcheck-list += $1
+  .PHONY: $1.test
+  $1.test: $1 $(test-deps)
+	@env TEST_NAME=$(notdir $(basename $1)) TESTS_ENVIRONMENT="$(tests-environment)" mk/run_test.sh $1
 
 endef
 
-# Color code from https://unix.stackexchange.com/a/10065
-installcheck:
-	@total=0; failed=0; \
-	red=""; \
-	green=""; \
-	yellow=""; \
-	normal=""; \
-	if [ -t 1 ]; then \
-		red=""; \
-		green=""; \
-		yellow=""; \
-		normal=""; \
-	fi; \
-	for i in $(_installcheck-list); do \
-	  total=$$((total + 1)); \
-	  printf "running test $$i..."; \
-	  log="$$(cd $$(dirname $$i) && $(tests-environment) $$(basename $$i) 2>&1)"; \
-	  status=$$?; \
-	  if [ $$status -eq 0 ]; then \
-	    echo " [$${green}PASS$$normal]"; \
-	  elif [ $$status -eq 99 ]; then \
-	    echo " [$${yellow}SKIP$$normal]"; \
-	  else \
-	    echo " [$${red}FAIL$$normal]"; \
-	    echo "$$log" | sed 's/^/    /'; \
-	    failed=$$((failed + 1)); \
-	  fi; \
-	done; \
-	if [ "$$failed" != 0 ]; then \
-	  echo "$${red}$$failed out of $$total tests failed $$normal"; \
-	  exit 1; \
-	else \
-		echo "$${green}All tests succeeded$$normal"; \
-	fi
-
 .PHONY: check installcheck

perl/lib/Nix/Store.xs

@@ -182,7 +182,7 @@ void importPaths(int fd, int dontCheckSigs)
     PPCODE:
         try {
             FdSource source(fd);
-            store()->importPaths(source, nullptr, dontCheckSigs ? NoCheckSigs : CheckSigs);
+            store()->importPaths(source, dontCheckSigs ? NoCheckSigs : CheckSigs);
         } catch (Error & e) {
             croak("%s", e.what());
         }

src/libexpr/attr-path.cc

@@ -130,7 +130,7 @@ Pos findDerivationFilename(EvalState & state, Value & v, std::string what)
 
     Symbol file = state.symbols.create(filename);
 
-    return { file, lineno, 0 };
+    return { foFile, file, lineno, 0 };
 }
 
 

src/libexpr/attr-set.hh

@@ -78,7 +78,7 @@ public:
         if (!a)
             throw Error({
                 .hint = hintfmt("attribute '%s' missing", name),
-                .nixCode = NixCode { .errPos = pos }
+                .errPos = pos
             });
 
         return *a;

src/libexpr/eval-inline.hh

@@ -11,7 +11,7 @@ LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s))
 {
     throw EvalError({
         .hint = hintfmt(s),
-        .nixCode = NixCode { .errPos = pos }
+        .errPos = pos
     });
 }
 
@@ -25,7 +25,7 @@ LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s, const
 {
     throw TypeError({
         .hint = hintfmt(s, showType(v)),
-        .nixCode = NixCode { .errPos = pos }
+        .errPos = pos
     });
 }
 

src/libexpr/eval.cc

@@ -366,7 +366,7 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
 
             if (store->isInStore(r.second)) {
                 StorePathSet closure;
-                store->computeFSClosure(store->parseStorePath(store->toStorePath(r.second)), closure);
+                store->computeFSClosure(store->toStorePath(r.second).first, closure);
                 for (auto & path : closure)
                     allowedPaths->insert(store->printStorePath(path));
             } else
@@ -529,7 +529,7 @@ LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s, const
 {
     throw EvalError({
         .hint = hintfmt(s, s2),
-        .nixCode = NixCode { .errPos = pos }
+        .errPos = pos
     });
 }
 
@@ -542,7 +542,7 @@ LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s, const
 {
     throw EvalError({
         .hint = hintfmt(s, s2, s3),
-        .nixCode = NixCode { .errPos = pos }
+        .errPos = pos
     });
 }
 
@@ -551,7 +551,7 @@ LocalNoInlineNoReturn(void throwEvalError(const Pos & p1, const char * s, const
     // p1 is where the error occurred; p2 is a position mentioned in the message.
     throw EvalError({
         .hint = hintfmt(s, sym, p2),
-        .nixCode = NixCode { .errPos = p1 }
+        .errPos = p1
     });
 }
 
@@ -559,7 +559,7 @@ LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s))
 {
     throw TypeError({
         .hint = hintfmt(s),
-        .nixCode = NixCode { .errPos = pos }
+        .errPos = pos
     });
 }
 
@@ -572,7 +572,7 @@ LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s, const
 {
     throw TypeError({
         .hint = hintfmt(s, fun.showNamePos(), s2),
-        .nixCode = NixCode { .errPos = pos }
+        .errPos = pos
     });
 }
 
@@ -580,7 +580,7 @@ LocalNoInlineNoReturn(void throwAssertionError(const Pos & pos, const char * s,
 {
     throw AssertionError({
         .hint = hintfmt(s, s1),
-        .nixCode = NixCode { .errPos = pos }
+        .errPos = pos
     });
 }
 
@@ -588,23 +588,18 @@ LocalNoInlineNoReturn(void throwUndefinedVarError(const Pos & pos, const char *
 {
     throw UndefinedVarError({
         .hint = hintfmt(s, s1),
-        .nixCode = NixCode { .errPos = pos }
+        .errPos = pos
     });
 }
 
-LocalNoInline(void addErrorPrefix(Error & e, const char * s, const string & s2))
+LocalNoInline(void addErrorTrace(Error & e, const char * s, const string & s2))
 {
-    e.addPrefix(format(s) % s2);
+    e.addTrace(std::nullopt, s, s2);
 }
 
-LocalNoInline(void addErrorPrefix(Error & e, const char * s, const ExprLambda & fun, const Pos & pos))
+LocalNoInline(void addErrorTrace(Error & e, const Pos & pos, const char * s, const string & s2))
 {
-    e.addPrefix(format(s) % fun.showNamePos() % pos);
-}
-
-LocalNoInline(void addErrorPrefix(Error & e, const char * s, const string & s2, const Pos & pos))
-{
-    e.addPrefix(format(s) % s2 % pos);
+    e.addTrace(pos, s, s2);
 }
 
 
@@ -818,7 +813,7 @@ void EvalState::evalFile(const Path & path_, Value & v)
     try {
         eval(e, v);
     } catch (Error & e) {
-        addErrorPrefix(e, "while evaluating the file '%1%':\n", path2);
+        addErrorTrace(e, "while evaluating the file '%1%':", path2);
         throw;
     }
 
@@ -1068,8 +1063,8 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v)
 
     } catch (Error & e) {
         if (pos2 && pos2->file != state.sDerivationNix)
-            addErrorPrefix(e, "while evaluating the attribute '%1%' at %2%:\n",
-                showAttrPath(state, env, attrPath), *pos2);
+            addErrorTrace(e, *pos2, "while evaluating the attribute '%1%'",
+                showAttrPath(state, env, attrPath));
         throw;
     }
 
@@ -1237,11 +1232,15 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po
 
     /* Evaluate the body.  This is conditional on showTrace, because
        catching exceptions makes this function not tail-recursive. */
-    if (settings.showTrace)
+    if (loggerSettings.showTrace.get())
         try {
             lambda.body->eval(*this, env2, v);
         } catch (Error & e) {
-            addErrorPrefix(e, "while evaluating %1%, called from %2%:\n", lambda, pos);
+            addErrorTrace(e, lambda.pos, "while evaluating %s", 
+              (lambda.name.set() 
+                  ? "'" + (string) lambda.name + "'" 
+                  : "anonymous lambdaction"));
+            addErrorTrace(e, pos, "from call site%s", "");
             throw;
         }
     else
@@ -1516,7 +1515,7 @@ void EvalState::forceValueDeep(Value & v)
                 try {
                     recurse(*i.value);
                 } catch (Error & e) {
-                    addErrorPrefix(e, "while evaluating the attribute '%1%' at %2%:\n", i.name, *i.pos);
+                    addErrorTrace(e, *i.pos, "while evaluating the attribute '%1%'", i.name);
                     throw;
                 }
         }
@@ -1936,7 +1935,7 @@ string ExternalValueBase::coerceToString(const Pos & pos, PathSet & context, boo
 {
     throw TypeError({
         .hint = hintfmt("cannot coerce %1% to a string", showType()),
-        .nixCode = NixCode { .errPos = pos }
+        .errPos = pos
     });
 }
 

src/libexpr/eval.hh

@@ -250,7 +250,7 @@ private:
     friend struct ExprAttrs;
     friend struct ExprLet;
 
-    Expr * parse(const char * text, const Path & path,
+    Expr * parse(const char * text, FileOrigin origin, const Path & path,
         const Path & basePath, StaticEnv & staticEnv);
 
 public:

src/libexpr/nixexpr.cc

@@ -197,7 +197,22 @@ std::ostream & operator << (std::ostream & str, const Pos & pos)
     if (!pos)
         str << "undefined position";
     else
-        str << (format(ANSI_BOLD "%1%" ANSI_NORMAL ":%2%:%3%") % (string) pos.file % pos.line % pos.column).str();
+    {
+        auto f = format(ANSI_BOLD "%1%" ANSI_NORMAL ":%2%:%3%");
+        switch (pos.origin) {
+            case foFile:
+                f % (string) pos.file;
+                break;
+            case foStdin:
+            case foString:
+                f % "(string)";
+                break;
+            default:
+                throw Error("unhandled Pos origin!");
+        }
+        str << (f % pos.line % pos.column).str();
+    }
+
     return str;
 }
 
@@ -270,7 +285,7 @@ void ExprVar::bindVars(const StaticEnv & env)
     if (withLevel == -1)
         throw UndefinedVarError({
             .hint = hintfmt("undefined variable '%1%'", name),
-            .nixCode = NixCode { .errPos = pos }
+            .errPos = pos
         });
     fromWith = true;
     this->level = withLevel;

src/libexpr/nixexpr.hh

@@ -24,11 +24,12 @@ MakeError(RestrictedPathError, Error);
 
 struct Pos
 {
+    FileOrigin origin;
     Symbol file;
     unsigned int line, column;
-    Pos() : line(0), column(0) { };
-    Pos(const Symbol & file, unsigned int line, unsigned int column)
-        : file(file), line(line), column(column) { };
+    Pos() : origin(foString), line(0), column(0) { };
+    Pos(FileOrigin origin, const Symbol & file, unsigned int line, unsigned int column)
+        : origin(origin), file(file), line(line), column(column) { };
     operator bool() const
     {
         return line != 0;
@@ -238,7 +239,7 @@ struct ExprLambda : Expr
         if (!arg.empty() && formals && formals->argNames.find(arg) != formals->argNames.end())
             throw ParseError({
                 .hint = hintfmt("duplicate formal function argument '%1%'", arg),
-                .nixCode = NixCode { .errPos = pos }
+                .errPos = pos
             });
     };
     void setName(Symbol & name);

src/libexpr/parser.y

@@ -30,7 +30,8 @@ namespace nix {
         SymbolTable & symbols;
         Expr * result;
         Path basePath;
-        Symbol path;
+        Symbol file;
+        FileOrigin origin;
         ErrorInfo error;
         Symbol sLetBody;
         ParseData(EvalState & state)
@@ -65,18 +66,17 @@ namespace nix {
 static void dupAttr(const AttrPath & attrPath, const Pos & pos, const Pos & prevPos)
 {
     throw ParseError({
-        .hint = hintfmt("attribute '%1%' already defined at %2%",
+         .hint = hintfmt("attribute '%1%' already defined at %2%",
             showAttrPath(attrPath), prevPos),
-        .nixCode = NixCode { .errPos = pos },
+         .errPos = pos
     });
 }
 
-
 static void dupAttr(Symbol attr, const Pos & pos, const Pos & prevPos)
 {
     throw ParseError({
         .hint = hintfmt("attribute '%1%' already defined at %2%", attr, prevPos),
-        .nixCode = NixCode { .errPos = pos },
+        .errPos = pos
     });
 }
 
@@ -148,7 +148,7 @@ static void addFormal(const Pos & pos, Formals * formals, const Formal & formal)
         throw ParseError({
             .hint = hintfmt("duplicate formal function argument '%1%'",
                 formal.name),
-            .nixCode = NixCode { .errPos = pos },
+            .errPos = pos
         });
     formals->formals.push_front(formal);
 }
@@ -246,7 +246,7 @@ static Expr * stripIndentation(const Pos & pos, SymbolTable & symbols, vector<Ex
 
 static inline Pos makeCurPos(const YYLTYPE & loc, ParseData * data)
 {
-    return Pos(data->path, loc.first_line, loc.first_column);
+    return Pos(data->origin, data->file, loc.first_line, loc.first_column);
 }
 
 #define CUR_POS makeCurPos(*yylocp, data)
@@ -259,7 +259,7 @@ void yyerror(YYLTYPE * loc, yyscan_t scanner, ParseData * data, const char * err
 {
     data->error = {
         .hint = hintfmt(error),
-        .nixCode = NixCode { .errPos = makeCurPos(*loc, data) }
+        .errPos = makeCurPos(*loc, data)
     };
 }
 
@@ -339,7 +339,7 @@ expr_function
     { if (!$2->dynamicAttrs.empty())
         throw ParseError({
             .hint = hintfmt("dynamic attributes not allowed in let"),
-            .nixCode = NixCode { .errPos = CUR_POS },
+            .errPos = CUR_POS
         });
       $$ = new ExprLet($2, $4);
     }
@@ -419,7 +419,7 @@ expr_simple
       if (noURLLiterals)
           throw ParseError({
               .hint = hintfmt("URL literals are disabled"),
-              .nixCode = NixCode { .errPos = CUR_POS }
+              .errPos = CUR_POS
           });
       $$ = new ExprString(data->symbols.create($1));
   }
@@ -492,7 +492,7 @@ attrs
       } else
           throw ParseError({
               .hint = hintfmt("dynamic attributes not allowed in inherit"),
-              .nixCode = NixCode { .errPos = makeCurPos(@2, data) },
+              .errPos = makeCurPos(@2, data)
           });
     }
   | { $$ = new AttrPath; }
@@ -569,13 +569,24 @@ formal
 namespace nix {
 
 
-Expr * EvalState::parse(const char * text,
+Expr * EvalState::parse(const char * text, FileOrigin origin,
     const Path & path, const Path & basePath, StaticEnv & staticEnv)
 {
     yyscan_t scanner;
     ParseData data(*this);
+    data.origin = origin;
+    switch (origin) {
+        case foFile: 
+            data.file = data.symbols.create(path);
+            break;
+        case foStdin:
+        case foString:
+            data.file = data.symbols.create(text);
+            break;
+        default:
+            assert(false);
+    }
     data.basePath = basePath;
-    data.path = data.symbols.create(path);
 
     yylex_init(&scanner);
     yy_scan_string(text, scanner);
@@ -625,13 +636,13 @@ Expr * EvalState::parseExprFromFile(const Path & path)
 
 Expr * EvalState::parseExprFromFile(const Path & path, StaticEnv & staticEnv)
 {
-    return parse(readFile(path).c_str(), path, dirOf(path), staticEnv);
+    return parse(readFile(path).c_str(), foFile, path, dirOf(path), staticEnv);
 }
 
 
 Expr * EvalState::parseExprFromString(std::string_view s, const Path & basePath, StaticEnv & staticEnv)
 {
-    return parse(s.data(), "(string)", basePath, staticEnv);
+    return parse(s.data(), foString, "", basePath, staticEnv);
 }
 
 
@@ -644,7 +655,7 @@ Expr * EvalState::parseExprFromString(std::string_view s, const Path & basePath)
 Expr * EvalState::parseStdin()
 {
     //Activity act(*logger, lvlTalkative, format("parsing standard input"));
-    return parseExprFromString(drainFD(0), absPath("."));
+    return parse(drainFD(0).data(), foStdin, "", absPath("."), staticBaseEnv);
 }
 
 
@@ -693,7 +704,7 @@ Path EvalState::findFile(SearchPath & searchPath, const string & path, const Pos
             ? "cannot look up '<%s>' in pure evaluation mode (use '--impure' to override)"
             : "file '%s' was not found in the Nix search path (add it using $NIX_PATH or -I)",
             path),
-        .nixCode = NixCode { .errPos = pos }
+        .errPos = pos
     });
 }
 

src/libexpr/primops.cc

@@ -96,7 +96,7 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args
     } catch (InvalidPathError & e) {
         throw EvalError({
             .hint = hintfmt("cannot import '%1%', since path '%2%' is not valid", path, e.path),
-            .nixCode = NixCode { .errPos = pos }
+            .errPos = pos
         });
     }
 
@@ -177,7 +177,7 @@ void prim_importNative(EvalState & state, const Pos & pos, Value * * args, Value
             .hint = hintfmt(
                 "cannot import '%1%', since path '%2%' is not valid",
                 path, e.path),
-            .nixCode = NixCode { .errPos = pos }
+            .errPos = pos
         });
     }
 
@@ -215,7 +215,7 @@ void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v)
     if (count == 0) {
         throw EvalError({
             .hint = hintfmt("at least one argument to 'exec' required"),
-            .nixCode = NixCode { .errPos = pos }
+            .errPos = pos
         });
     }
     PathSet context;
@@ -230,7 +230,7 @@ void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v)
         throw EvalError({
             .hint = hintfmt("cannot execute '%1%', since path '%2%' is not valid",
                 program, e.path),
-            .nixCode = NixCode { .errPos = pos }
+            .errPos = pos
         });
     }
 
@@ -239,13 +239,13 @@ void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v)
     try {
         parsed = state.parseExprFromString(output, pos.file);
     } catch (Error & e) {
-        e.addPrefix(fmt("While parsing the output from '%1%', at %2%\n", program, pos));
+        e.addTrace(pos, "While parsing the output from '%1%'", program);
         throw;
     }
     try {
         state.eval(parsed, v);
     } catch (Error & e) {
-        e.addPrefix(fmt("While evaluating the output from '%1%', at %2%\n", program, pos));
+        e.addTrace(pos, "While evaluating the output from '%1%'", program);
         throw;
     }
 }
@@ -385,7 +385,7 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar
     if (startSet == args[0]->attrs->end())
         throw EvalError({
             .hint = hintfmt("attribute 'startSet' required"),
-            .nixCode = NixCode { .errPos = pos }
+            .errPos = pos
         });
     state.forceList(*startSet->value, pos);
 
@@ -399,7 +399,7 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar
     if (op == args[0]->attrs->end())
         throw EvalError({
             .hint = hintfmt("attribute 'operator' required"),
-            .nixCode = NixCode { .errPos = pos }
+            .errPos = pos
         });
     state.forceValue(*op->value, pos);
 
@@ -421,7 +421,7 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar
         if (key == e->attrs->end())
             throw EvalError({
                 .hint = hintfmt("attribute 'key' required"),
-                .nixCode = NixCode { .errPos = pos }
+                .errPos = pos
             });
         state.forceValue(*key->value, pos);
 
@@ -471,7 +471,7 @@ static void prim_addErrorContext(EvalState & state, const Pos & pos, Value * * a
         v = *args[1];
     } catch (Error & e) {
         PathSet context;
-        e.addPrefix(format("%1%\n") % state.coerceToString(pos, *args[0], context));
+        e.addTrace(std::nullopt, state.coerceToString(pos, *args[0], context));
         throw;
     }
 }
@@ -556,14 +556,14 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
     if (attr == args[0]->attrs->end())
         throw EvalError({
             .hint = hintfmt("required attribute 'name' missing"),
-            .nixCode = NixCode { .errPos = pos }
+            .errPos = pos
         });
     string drvName;
     Pos & posDrvName(*attr->pos);
     try {
         drvName = state.forceStringNoCtx(*attr->value, pos);
     } catch (Error & e) {
-        e.addPrefix(fmt("while evaluating the derivation attribute 'name' at %1%:\n", posDrvName));
+        e.addTrace(posDrvName, "while evaluating the derivation attribute 'name'");
         throw;
     }
 
@@ -603,7 +603,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
             else
                 throw EvalError({
                     .hint = hintfmt("invalid value '%s' for 'outputHashMode' attribute", s),
-                    .nixCode = NixCode { .errPos = posDrvName }
+                    .errPos = posDrvName
                 });
         };
 
@@ -613,7 +613,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
                 if (outputs.find(j) != outputs.end())
                     throw EvalError({
                         .hint = hintfmt("duplicate derivation output '%1%'", j),
-                        .nixCode = NixCode { .errPos = posDrvName }
+                        .errPos = posDrvName
                     });
                 /* !!! Check whether j is a valid attribute
                    name. */
@@ -623,14 +623,14 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
                 if (j == "drv")
                     throw EvalError({
                         .hint = hintfmt("invalid derivation output name 'drv'" ),
-                        .nixCode = NixCode { .errPos = posDrvName }
+                        .errPos = posDrvName
                     });
                 outputs.insert(j);
             }
             if (outputs.empty())
                 throw EvalError({
                     .hint = hintfmt("derivation cannot have an empty set of outputs"),
-                    .nixCode = NixCode { .errPos = posDrvName }
+                    .errPos = posDrvName
                 });
         };
 
@@ -696,8 +696,9 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
             }
 
         } catch (Error & e) {
-            e.addPrefix(format("while evaluating the attribute '%1%' of the derivation '%2%' at %3%:\n")
-                % key % drvName % posDrvName);
+            e.addTrace(posDrvName, 
+                "while evaluating the attribute '%1%' of the derivation '%2%'",
+                key, drvName);
             throw;
         }
     }
@@ -745,20 +746,20 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
     if (drv.builder == "")
         throw EvalError({
             .hint = hintfmt("required attribute 'builder' missing"),
-            .nixCode = NixCode { .errPos = posDrvName }
+            .errPos = posDrvName
         });
 
     if (drv.platform == "")
         throw EvalError({
             .hint = hintfmt("required attribute 'system' missing"),
-            .nixCode = NixCode { .errPos = posDrvName }
+            .errPos = posDrvName
         });
 
     /* Check whether the derivation name is valid. */
     if (isDerivation(drvName))
         throw EvalError({
             .hint = hintfmt("derivation names are not allowed to end in '%s'", drvExtension),
-            .nixCode = NixCode { .errPos = posDrvName }
+            .errPos = posDrvName
         });
 
     if (outputHash) {
@@ -766,7 +767,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
         if (outputs.size() != 1 || *(outputs.begin()) != "out")
             throw Error({
                 .hint = hintfmt("multiple outputs are not supported in fixed-output derivations"),
-                .nixCode = NixCode { .errPos = posDrvName }
+                .errPos = posDrvName
             });
 
         std::optional<HashType> ht = parseHashTypeOpt(outputHashAlgo);
@@ -880,12 +881,12 @@ static void prim_storePath(EvalState & state, const Pos & pos, Value * * args, V
     if (!state.store->isInStore(path))
         throw EvalError({
             .hint = hintfmt("path '%1%' is not in the Nix store", path),
-            .nixCode = NixCode { .errPos = pos }
+            .errPos = pos
         });
-    Path path2 = state.store->toStorePath(path);
+    auto path2 = state.store->toStorePath(path).first;
     if (!settings.readOnlyMode)
-        state.store->ensurePath(state.store->parseStorePath(path2));
-    context.insert(path2);
+        state.store->ensurePath(path2);
+    context.insert(state.store->printStorePath(path2));
     mkString(v, path, context);
 }
 
@@ -901,7 +902,7 @@ static void prim_pathExists(EvalState & state, const Pos & pos, Value * * args,
             .hint = hintfmt(
                 "cannot check the existence of '%1%', since path '%2%' is not valid",
                 path, e.path),
-            .nixCode = NixCode { .errPos = pos }
+            .errPos = pos
         });
     }
 
@@ -947,7 +948,7 @@ static void prim_readFile(EvalState & state, const Pos & pos, Value * * args, Va
     } catch (InvalidPathError & e) {
         throw EvalError({
             .hint = hintfmt("cannot read '%1%', since path '%2%' is not valid", path, e.path),
-            .nixCode = NixCode { .errPos = pos }
+            .errPos = pos
         });
     }
     string s = readFile(state.checkSourcePath(state.toRealPath(path, context)));
@@ -978,7 +979,7 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va
         if (i == v2.attrs->end())
             throw EvalError({
                 .hint = hintfmt("attribute 'path' missing"),
-                .nixCode = NixCode { .errPos = pos }
+                .errPos = pos
             });
 
         PathSet context;
@@ -989,7 +990,7 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va
         } catch (InvalidPathError & e) {
             throw EvalError({
                 .hint = hintfmt("cannot find '%1%', since path '%2%' is not valid", path, e.path),
-                .nixCode = NixCode { .errPos = pos }
+                .errPos = pos
             });
         }
 
@@ -1009,7 +1010,7 @@ static void prim_hashFile(EvalState & state, const Pos & pos, Value * * args, Va
     if (!ht)
       throw Error({
           .hint = hintfmt("unknown hash type '%1%'", type),
-          .nixCode = NixCode { .errPos = pos }
+          .errPos = pos
       });
 
     PathSet context; // discarded
@@ -1028,7 +1029,7 @@ static void prim_readDir(EvalState & state, const Pos & pos, Value * * args, Val
     } catch (InvalidPathError & e) {
         throw EvalError({
             .hint = hintfmt("cannot read '%1%', since path '%2%' is not valid", path, e.path),
-            .nixCode = NixCode { .errPos = pos }
+            .errPos = pos
         });
     }
 
@@ -1104,7 +1105,7 @@ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Valu
                     "in 'toFile': the file named '%1%' must not contain a reference "
                     "to a derivation but contains (%2%)",
                     name, path),
-                .nixCode = NixCode { .errPos = pos }
+                .errPos = pos
             });
         refs.insert(state.store->parseStorePath(path));
     }
@@ -1175,7 +1176,7 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args
     if (!context.empty())
         throw EvalError({
             .hint = hintfmt("string '%1%' cannot refer to other paths", path),
-            .nixCode = NixCode { .errPos = pos }
+            .errPos = pos
         });
 
     state.forceValue(*args[0], pos);
@@ -1184,7 +1185,7 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args
             .hint = hintfmt(
                 "first argument in call to 'filterSource' is not a function but %1%",
                 showType(*args[0])),
-            .nixCode = NixCode { .errPos = pos }
+            .errPos = pos
         });
 
     addPath(state, pos, std::string(baseNameOf(path)), path, args[0], FileIngestionMethod::Recursive, Hash(), v);
@@ -1207,7 +1208,7 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value
             if (!context.empty())
                 throw EvalError({
                     .hint = hintfmt("string '%1%' cannot refer to other paths", path),
-                    .nixCode = NixCode { .errPos = *attr.pos }
+                    .errPos = *attr.pos
                 });
         } else if (attr.name == state.sName)
             name = state.forceStringNoCtx(*attr.value, *attr.pos);
@@ -1221,13 +1222,13 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value
         else
             throw EvalError({
                 .hint = hintfmt("unsupported argument '%1%' to 'addPath'", attr.name),
-                .nixCode = NixCode { .errPos = *attr.pos }
+                .errPos = *attr.pos
             });
     }
     if (path.empty())
         throw EvalError({
             .hint = hintfmt("'path' required"),
-            .nixCode = NixCode { .errPos = pos }
+            .errPos = pos
         });
     if (name.empty())
         name = baseNameOf(path);
@@ -1288,7 +1289,7 @@ void prim_getAttr(EvalState & state, const Pos & pos, Value * * args, Value & v)
     if (i == args[1]->attrs->end())
         throw EvalError({
             .hint = hintfmt("attribute '%1%' missing", attr),
-            .nixCode = NixCode { .errPos = pos }
+            .errPos = pos
         });
     // !!! add to stack trace?
     if (state.countCalls && i->pos) state.attrSelects[*i->pos]++;
@@ -1371,7 +1372,7 @@ static void prim_listToAttrs(EvalState & state, const Pos & pos, Value * * args,
         if (j == v2.attrs->end())
             throw TypeError({
                 .hint = hintfmt("'name' attribute missing in a call to 'listToAttrs'"),
-                .nixCode = NixCode { .errPos = pos }
+                .errPos = pos
             });
         string name = state.forceStringNoCtx(*j->value, pos);
 
@@ -1381,7 +1382,7 @@ static void prim_listToAttrs(EvalState & state, const Pos & pos, Value * * args,
             if (j2 == v2.attrs->end())
                 throw TypeError({
                     .hint = hintfmt("'value' attribute missing in a call to 'listToAttrs'"),
-                    .nixCode = NixCode { .errPos = pos }
+                    .errPos = pos
                 });
             v.attrs->push_back(Attr(sym, j2->value, j2->pos));
         }
@@ -1457,7 +1458,7 @@ static void prim_functionArgs(EvalState & state, const Pos & pos, Value * * args
     if (args[0]->type != tLambda)
         throw TypeError({
             .hint = hintfmt("'functionArgs' requires a function"),
-            .nixCode = NixCode { .errPos = pos }
+            .errPos = pos
         });
 
     if (!args[0]->lambda.fun->matchAttrs) {
@@ -1513,7 +1514,7 @@ static void elemAt(EvalState & state, const Pos & pos, Value & list, int n, Valu
     if (n < 0 || (unsigned int) n >= list.listSize())
         throw Error({
             .hint = hintfmt("list index %1% is out of bounds", n),
-            .nixCode = NixCode { .errPos = pos }
+            .errPos = pos
         });
     state.forceValue(*list.listElems()[n], pos);
     v = *list.listElems()[n];
@@ -1543,7 +1544,7 @@ static void prim_tail(EvalState & state, const Pos & pos, Value * * args, Value
     if (args[0]->listSize() == 0)
         throw Error({
             .hint = hintfmt("'tail' called on an empty list"),
-            .nixCode = NixCode { .errPos = pos }
+            .errPos = pos
         });
 
     state.mkList(v, args[0]->listSize() - 1);
@@ -1688,7 +1689,7 @@ static void prim_genList(EvalState & state, const Pos & pos, Value * * args, Val
     if (len < 0)
         throw EvalError({
             .hint = hintfmt("cannot create list of size %1%", len),
-            .nixCode = NixCode { .errPos = pos }
+            .errPos = pos
         });
 
     state.mkList(v, len);
@@ -1850,7 +1851,7 @@ static void prim_div(EvalState & state, const Pos & pos, Value * * args, Value &
     if (f2 == 0)
         throw EvalError({
             .hint = hintfmt("division by zero"),
-            .nixCode = NixCode { .errPos = pos }
+            .errPos = pos
         });
 
     if (args[0]->type == tFloat || args[1]->type == tFloat) {
@@ -1862,7 +1863,7 @@ static void prim_div(EvalState & state, const Pos & pos, Value * * args, Value &
         if (i1 == std::numeric_limits<NixInt>::min() && i2 == -1)
             throw EvalError({
                 .hint = hintfmt("overflow in integer division"),
-                .nixCode = NixCode { .errPos = pos }
+                .errPos = pos
             });
 
         mkInt(v, i1 / i2);
@@ -1923,7 +1924,7 @@ static void prim_substring(EvalState & state, const Pos & pos, Value * * args, V
     if (start < 0)
         throw EvalError({
             .hint = hintfmt("negative start position in 'substring'"),
-            .nixCode = NixCode { .errPos = pos }
+            .errPos = pos
         });
 
     mkString(v, (unsigned int) start >= s.size() ? "" : string(s, start, len), context);
@@ -1946,7 +1947,7 @@ static void prim_hashString(EvalState & state, const Pos & pos, Value * * args,
     if (!ht)
         throw Error({
             .hint = hintfmt("unknown hash type '%1%'", type),
-            .nixCode = NixCode { .errPos = pos }
+            .errPos = pos
         });
 
     PathSet context; // discarded
@@ -1992,12 +1993,12 @@ void prim_match(EvalState & state, const Pos & pos, Value * * args, Value & v)
             // limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++
             throw EvalError({
                 .hint = hintfmt("memory limit exceeded by regular expression '%s'", re),
-                .nixCode = NixCode { .errPos = pos }
+                .errPos = pos
             });
         } else {
             throw EvalError({
                 .hint = hintfmt("invalid regular expression '%s'", re),
-                .nixCode = NixCode { .errPos = pos }
+                .errPos = pos
             });
         }
     }
@@ -2065,12 +2066,12 @@ static void prim_split(EvalState & state, const Pos & pos, Value * * args, Value
             // limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++
             throw EvalError({
                 .hint = hintfmt("memory limit exceeded by regular expression '%s'", re),
-                .nixCode = NixCode { .errPos = pos }
+                .errPos = pos
             });
         } else {
             throw EvalError({
                 .hint = hintfmt("invalid regular expression '%s'", re),
-                .nixCode = NixCode { .errPos = pos }
+                .errPos = pos
             });
         }
     }
@@ -2104,7 +2105,7 @@ static void prim_replaceStrings(EvalState & state, const Pos & pos, Value * * ar
     if (args[0]->listSize() != args[1]->listSize())
         throw EvalError({
             .hint = hintfmt("'from' and 'to' arguments to 'replaceStrings' have different lengths"),
-            .nixCode = NixCode { .errPos = pos }
+            .errPos = pos
         });
 
     vector<string> from;

src/libexpr/primops/context.cc

@@ -148,7 +148,7 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg
         if (!state.store->isStorePath(i.name))
             throw EvalError({
                 .hint = hintfmt("Context key '%s' is not a store path", i.name),
-                .nixCode = NixCode { .errPos = *i.pos }
+                .errPos = *i.pos
             });
         if (!settings.readOnlyMode)
             state.store->ensurePath(state.store->parseStorePath(i.name));
@@ -165,7 +165,7 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg
                 if (!isDerivation(i.name)) {
                     throw EvalError({
                         .hint = hintfmt("Tried to add all-outputs context of %s, which is not a derivation, to a string", i.name),
-                        .nixCode = NixCode { .errPos = *i.pos }
+                        .errPos = *i.pos
                     });
                 }
                 context.insert("=" + string(i.name));
@@ -178,7 +178,7 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg
             if (iter->value->listSize() && !isDerivation(i.name)) {
                 throw EvalError({
                     .hint = hintfmt("Tried to add derivation output context of %s, which is not a derivation, to a string", i.name),
-                    .nixCode = NixCode { .errPos = *i.pos }
+                    .errPos = *i.pos
                 });
             }
             for (unsigned int n = 0; n < iter->value->listSize(); ++n) {

src/libexpr/primops/fetchGit.cc

@@ -37,14 +37,14 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va
             else
                 throw EvalError({
                     .hint = hintfmt("unsupported argument '%s' to 'fetchGit'", attr.name),
-                    .nixCode = NixCode { .errPos = *attr.pos }
+                    .errPos = *attr.pos
                 });
         }
 
         if (url.empty())
             throw EvalError({
                 .hint = hintfmt("'url' argument required"),
-                .nixCode = NixCode { .errPos = pos }
+                .errPos = pos
             });
 
     } else

src/libexpr/primops/fetchMercurial.cc

@@ -40,14 +40,14 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar
             else
                 throw EvalError({
                     .hint = hintfmt("unsupported argument '%s' to 'fetchMercurial'", attr.name),
-                    .nixCode = NixCode { .errPos = *attr.pos }
+                    .errPos = *attr.pos
                 });
         }
 
         if (url.empty())
             throw EvalError({
                 .hint = hintfmt("'url' argument required"),
-                .nixCode = NixCode { .errPos = pos }
+                .errPos = pos
             });
 
     } else

src/libexpr/primops/fetchTree.cc

@@ -68,7 +68,7 @@ static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, V
         if (!attrs.count("type"))
             throw Error({
                 .hint = hintfmt("attribute 'type' is missing in call to 'fetchTree'"),
-                .nixCode = NixCode { .errPos = pos }
+                .errPos = pos
             });
 
         input = fetchers::inputFromAttrs(attrs);
@@ -112,14 +112,14 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
             else
                 throw EvalError({
                     .hint = hintfmt("unsupported argument '%s' to '%s'", attr.name, who),
-                    .nixCode = NixCode { .errPos = *attr.pos }
+                    .errPos = *attr.pos
                 });
             }
 
         if (!url)
             throw EvalError({
                 .hint = hintfmt("'url' argument required"),
-                .nixCode = NixCode { .errPos = pos }
+                .errPos = pos
             });
     } else
         url = state.forceStringNoCtx(*args[0], pos);

src/libexpr/primops/fromTOML.cc

@@ -83,7 +83,7 @@ static void prim_fromTOML(EvalState & state, const Pos & pos, Value * * args, Va
     } catch (std::runtime_error & e) {
         throw EvalError({
             .hint = hintfmt("while parsing a TOML string: %s", e.what()),
-            .nixCode = NixCode { .errPos = pos }
+            .errPos = pos
         });
     }
 }

src/libfetchers/cache.hh

@@ -6,6 +6,8 @@ namespace nix::fetchers {
 
 struct Cache
 {
+    virtual ~Cache() { }
+
     virtual void add(
         ref<Store> store,
         const Attrs & inAttrs,

src/libmain/loggers.cc

@@ -26,7 +26,7 @@ Logger * makeDefaultLogger() {
     case LogFormat::rawWithLogs:
         return makeSimpleLogger(true);
     case LogFormat::internalJson:
-        return makeJSONLogger(*makeSimpleLogger());
+        return makeJSONLogger(*makeSimpleLogger(true));
     case LogFormat::bar:
         return makeProgressBar();
     case LogFormat::barWithLogs:

src/libmain/progress-bar.cc

@@ -131,7 +131,7 @@ public:
         auto state(state_.lock());
 
         std::stringstream oss;
-        oss << ei;
+        showErrorInfo(oss, ei, loggerSettings.showTrace.get());
 
         log(*state, ei.level, oss.str());
     }

src/libmain/shared.cc

@@ -323,10 +323,8 @@ int handleExceptions(const string & programName, std::function<void()> fun)
         printError("Try '%1% --help' for more information.", programName);
         return 1;
     } catch (BaseError & e) {
-        if (settings.showTrace && e.prefix() != "")
-            printError(e.prefix());
         logError(e.info());
-        if (e.prefix() != "" && !settings.showTrace)
+        if (e.hasTrace() && !loggerSettings.showTrace.get())
             printError("(use '--show-trace' to show detailed location information)");
         return e.status;
     } catch (std::bad_alloc & e) {

src/libstore/binary-cache-store.cc

@@ -15,6 +15,7 @@
 #include <chrono>
 #include <future>
 #include <regex>
+#include <fstream>
 
 #include <nlohmann/json.hpp>
 
@@ -57,6 +58,13 @@ void BinaryCacheStore::init()
     }
 }
 
+void BinaryCacheStore::upsertFile(const std::string & path,
+    std::string && data,
+    const std::string & mimeType)
+{
+    upsertFile(path, std::make_shared<std::stringstream>(std::move(data)), mimeType);
+}
+
 void BinaryCacheStore::getFile(const std::string & path,
     Callback<std::shared_ptr<std::string>> callback) noexcept
 {
@@ -113,13 +121,74 @@ void BinaryCacheStore::writeNarInfo(ref<NarInfo> narInfo)
         diskCache->upsertNarInfo(getUri(), hashPart, std::shared_ptr<NarInfo>(narInfo));
 }
 
-void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource,
-    RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor)
+AutoCloseFD openFile(const Path & path)
 {
-    // FIXME: See if we can use the original source to reduce memory usage.
-    auto nar = make_ref<std::string>(narSource.drain());
+    auto fd = open(path.c_str(), O_RDONLY | O_CLOEXEC);
+    if (!fd)
+        throw SysError("opening file '%1%'", path);
+    return fd;
+}
 
-    if (!repair && isValidPath(info.path)) return;
+struct FileSource : FdSource
+{
+    AutoCloseFD fd2;
+
+    FileSource(const Path & path)
+        : fd2(openFile(path))
+    {
+        fd = fd2.get();
+    }
+};
+
+void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource,
+    RepairFlag repair, CheckSigsFlag checkSigs)
+{
+    assert(info.narHash && info.narSize);
+
+    if (!repair && isValidPath(info.path)) {
+        // FIXME: copyNAR -> null sink
+        narSource.drain();
+        return;
+    }
+
+    auto [fdTemp, fnTemp] = createTempFile();
+
+    auto now1 = std::chrono::steady_clock::now();
+
+    /* Read the NAR simultaneously into a CompressionSink+FileSink (to
+       write the compressed NAR to disk), into a HashSink (to get the
+       NAR hash), and into a NarAccessor (to get the NAR listing). */
+    HashSink fileHashSink(htSHA256);
+    std::shared_ptr<FSAccessor> narAccessor;
+    {
+    FdSink fileSink(fdTemp.get());
+    TeeSink teeSink(fileSink, fileHashSink);
+    auto compressionSink = makeCompressionSink(compression, teeSink);
+    TeeSource teeSource(narSource, *compressionSink);
+    narAccessor = makeNarAccessor(teeSource);
+    compressionSink->finish();
+    }
+
+    auto now2 = std::chrono::steady_clock::now();
+
+    auto narInfo = make_ref<NarInfo>(info);
+    narInfo->narSize = info.narSize;
+    narInfo->narHash = info.narHash;
+    narInfo->compression = compression;
+    auto [fileHash, fileSize] = fileHashSink.finish();
+    narInfo->fileHash = fileHash;
+    narInfo->fileSize = fileSize;
+    narInfo->url = "nar/" + narInfo->fileHash.to_string(Base32, false) + ".nar"
+        + (compression == "xz" ? ".xz" :
+           compression == "bzip2" ? ".bz2" :
+           compression == "br" ? ".br" :
+           "");
+
+    auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
+    printMsg(lvlTalkative, "copying path '%1%' (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache",
+        printStorePath(narInfo->path), info.narSize,
+        ((1.0 - (double) fileSize / info.narSize) * 100.0),
+        duration);
 
     /* Verify that all references are valid. This may do some .narinfo
        reads, but typically they'll already be cached. */
@@ -132,23 +201,6 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
                 printStorePath(info.path), printStorePath(ref));
         }
 
-    assert(nar->compare(0, narMagic.size(), narMagic) == 0);
-
-    auto narInfo = make_ref<NarInfo>(info);
-
-    narInfo->narSize = nar->size();
-    narInfo->narHash = hashString(htSHA256, *nar);
-
-    if (info.narHash && info.narHash != narInfo->narHash)
-        throw Error("refusing to copy corrupted path '%1%' to binary cache", printStorePath(info.path));
-
-    auto accessor_ = std::dynamic_pointer_cast<RemoteFSAccessor>(accessor);
-
-    auto narAccessor = makeNarAccessor(nar);
-
-    if (accessor_)
-        accessor_->addToCache(printStorePath(info.path), *nar, narAccessor);
-
     /* Optionally write a JSON file containing a listing of the
        contents of the NAR. */
     if (writeNARListing) {
@@ -160,33 +212,13 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
 
             {
                 auto res = jsonRoot.placeholder("root");
-                listNar(res, narAccessor, "", true);
+                listNar(res, ref<FSAccessor>(narAccessor), "", true);
             }
         }
 
         upsertFile(std::string(info.path.to_string()) + ".ls", jsonOut.str(), "application/json");
     }
 
-    /* Compress the NAR. */
-    narInfo->compression = compression;
-    auto now1 = std::chrono::steady_clock::now();
-    auto narCompressed = compress(compression, *nar, parallelCompression);
-    auto now2 = std::chrono::steady_clock::now();
-    narInfo->fileHash = hashString(htSHA256, *narCompressed);
-    narInfo->fileSize = narCompressed->size();
-
-    auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
-    printMsg(lvlTalkative, "copying path '%1%' (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache",
-        printStorePath(narInfo->path), narInfo->narSize,
-        ((1.0 - (double) narCompressed->size() / nar->size()) * 100.0),
-        duration);
-
-    narInfo->url = "nar/" + narInfo->fileHash.to_string(Base32, false) + ".nar"
-        + (compression == "xz" ? ".xz" :
-           compression == "bzip2" ? ".bz2" :
-           compression == "br" ? ".br" :
-           "");
-
     /* Optionally maintain an index of DWARF debug info files
        consisting of JSON files named 'debuginfo/<build-id>' that
        specify the NAR file and member containing the debug info. */
@@ -247,12 +279,14 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
     /* Atomically write the NAR file. */
     if (repair || !fileExists(narInfo->url)) {
         stats.narWrite++;
-        upsertFile(narInfo->url, *narCompressed, "application/x-nix-nar");
+        upsertFile(narInfo->url,
+            std::make_shared<std::fstream>(fnTemp, std::ios_base::in),
+            "application/x-nix-nar");
     } else
         stats.narWriteAverted++;
 
-    stats.narWriteBytes += nar->size();
-    stats.narWriteCompressedBytes += narCompressed->size();
+    stats.narWriteBytes += info.narSize;
+    stats.narWriteCompressedBytes += fileSize;
     stats.narWriteCompressionTimeMs += duration;
 
     /* Atomically write the NAR info file.*/
@@ -351,7 +385,7 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath
     ValidPathInfo info(makeFixedOutputPath(method, h, name));
 
     auto source = StringSource { *sink.s };
-    addToStore(info, source, repair, CheckSigs, nullptr);
+    addToStore(info, source, repair, CheckSigs);
 
     return std::move(info.path);
 }
@@ -366,7 +400,7 @@ StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s
         StringSink sink;
         dumpString(s, sink);
         auto source = StringSource { *sink.s };
-        addToStore(info, source, repair, CheckSigs, nullptr);
+        addToStore(info, source, repair, CheckSigs);
     }
 
     return std::move(info.path);

src/libstore/binary-cache-store.hh

@@ -36,9 +36,13 @@ public:
     virtual bool fileExists(const std::string & path) = 0;
 
     virtual void upsertFile(const std::string & path,
-        const std::string & data,
+        std::shared_ptr<std::basic_iostream<char>> istream,
         const std::string & mimeType) = 0;
 
+    void upsertFile(const std::string & path,
+        std::string && data,
+        const std::string & mimeType);
+
     /* Note: subclasses must implement at least one of the two
        following getFile() methods. */
 
@@ -75,8 +79,7 @@ public:
     { unsupported("queryPathFromHashPart"); }
 
     void addToStore(const ValidPathInfo & info, Source & narSource,
-        RepairFlag repair, CheckSigsFlag checkSigs,
-        std::shared_ptr<FSAccessor> accessor) override;
+        RepairFlag repair, CheckSigsFlag checkSigs) override;
 
     StorePath addToStore(const string & name, const Path & srcPath,
         FileIngestionMethod method, HashType hashAlgo,

src/libstore/build.cc

@@ -1950,8 +1950,11 @@ void linkOrCopy(const Path & from, const Path & to)
         /* Hard-linking fails if we exceed the maximum link count on a
            file (e.g. 32000 of ext3), which is quite possible after a
            'nix-store --optimise'. FIXME: actually, why don't we just
-           bind-mount in this case? */
-        if (errno != EMLINK)
+           bind-mount in this case?
+           
+           It can also fail with EPERM in BeegFS v7 and earlier versions
+           which don't allow hard-links to other directories */
+        if (errno != EMLINK && errno != EPERM)
             throw SysError("linking '%s' to '%s'", to, from);
         copyPath(from, to);
     }
@@ -2038,7 +2041,10 @@ void DerivationGoal::startBuilder()
             if (!std::regex_match(fileName, regex))
                 throw Error("invalid file name '%s' in 'exportReferencesGraph'", fileName);
 
-            auto storePath = worker.store.parseStorePath(*i++);
+            auto storePathS = *i++;
+            if (!worker.store.isInStore(storePathS))
+                throw BuildError("'exportReferencesGraph' contains a non-store path '%1%'", storePathS);
+            auto storePath = worker.store.toStorePath(storePathS).first;
 
             /* Write closure info to <fileName>. */
             writeFile(tmpDir + "/" + fileName,
@@ -2077,7 +2083,7 @@ void DerivationGoal::startBuilder()
         for (auto & i : dirsInChroot)
             try {
                 if (worker.store.isInStore(i.second.source))
-                    worker.store.computeFSClosure(worker.store.parseStorePath(worker.store.toStorePath(i.second.source)), closure);
+                    worker.store.computeFSClosure(worker.store.toStorePath(i.second.source).first, closure);
             } catch (InvalidPath & e) {
             } catch (Error & e) {
                 throw Error("while processing 'sandbox-paths': %s", e.what());
@@ -2750,8 +2756,8 @@ struct RestrictedStore : public LocalFSStore
     void queryReferrers(const StorePath & path, StorePathSet & referrers) override
     { }
 
-    StorePathSet queryDerivationOutputs(const StorePath & path) override
-    { throw Error("queryDerivationOutputs"); }
+    OutputPathMap queryDerivationOutputMap(const StorePath & path) override
+    { throw Error("queryDerivationOutputMap"); }
 
     std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override
     { throw Error("queryPathFromHashPart"); }
@@ -2762,10 +2768,9 @@ struct RestrictedStore : public LocalFSStore
     { throw Error("addToStore"); }
 
     void addToStore(const ValidPathInfo & info, Source & narSource,
-        RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs,
-        std::shared_ptr<FSAccessor> accessor = 0) override
+        RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs) override
     {
-        next->addToStore(info, narSource, repair, checkSigs, accessor);
+        next->addToStore(info, narSource, repair, checkSigs);
         goal.addDependency(info.path);
     }
 

src/libstore/builtins/fetchurl.cc

@@ -58,13 +58,16 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
         }
     };
 
+    /* We always have one output, and if it's a fixed-output derivation (as
+       checked below) it must be the only output */
+    auto & output = drv.outputs.begin()->second;
+
     /* Try the hashed mirrors first. */
-    if (getAttr("outputHashMode") == "flat")
+    if (output.hash && output.hash->method == FileIngestionMethod::Flat)
         for (auto hashedMirror : settings.hashedMirrors.get())
             try {
                 if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/';
-                auto ht = parseHashTypeOpt(getAttr("outputHashAlgo"));
-                auto h = Hash(getAttr("outputHash"), ht);
+                auto & h = output.hash->hash;
                 fetch(hashedMirror + printHashType(*h.type) + "/" + h.to_string(Base16, false));
                 return;
             } catch (Error & e) {

src/libstore/content-address.cc

@@ -82,4 +82,16 @@ std::string renderContentAddress(std::optional<ContentAddress> ca) {
     return ca ? renderContentAddress(*ca) : "";
 }
 
+Hash getContentAddressHash(const ContentAddress & ca)
+{
+    return std::visit(overloaded {
+        [](TextHash th) {
+            return th.hash;
+        },
+        [](FixedOutputHash fsh) {
+            return fsh.hash;
+        }
+    }, ca);
+}
+
 }

src/libstore/content-address.hh

@@ -53,4 +53,6 @@ ContentAddress parseContentAddress(std::string_view rawCa);
 
 std::optional<ContentAddress> parseContentAddressOpt(std::string_view rawCaOpt);
 
+Hash getContentAddressHash(const ContentAddress & ca);
+
 }

src/libstore/daemon.cc

@@ -78,10 +78,10 @@ struct TunnelLogger : public Logger
         if (ei.level > verbosity) return;
 
         std::stringstream oss;
-        oss << ei;
+        showErrorInfo(oss, ei, false);
 
         StringSink buf;
-        buf << STDERR_NEXT << oss.str() << "\n"; // (fs.s + "\n");
+        buf << STDERR_NEXT << oss.str();
         enqueueMsg(*buf.s);
     }
 
@@ -347,6 +347,15 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         break;
     }
 
+    case wopQueryDerivationOutputMap: {
+        auto path = store->parseStorePath(readString(from));
+        logger->startWork();
+        OutputPathMap outputs = store->queryDerivationOutputMap(path);
+        logger->stopWork();
+        writeOutputPathMap(*store, to, outputs);
+        break;
+    }
+
     case wopQueryDeriver: {
         auto path = store->parseStorePath(readString(from));
         logger->startWork();
@@ -382,7 +391,8 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         }
         HashType hashAlgo = parseHashType(s);
 
-        TeeSource savedNAR(from);
+        StringSink savedNAR;
+        TeeSource savedNARSource(from, savedNAR);
         RetrieveRegularNARSink savedRegular;
 
         if (method == FileIngestionMethod::Recursive) {
@@ -390,7 +400,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
                a string so that we can pass it to
                addToStoreFromDump(). */
             ParseSink sink; /* null sink; just parse the NAR */
-            parseDump(sink, savedNAR);
+            parseDump(sink, savedNARSource);
         } else
             parseDump(savedRegular, from);
 
@@ -398,7 +408,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         if (!savedRegular.regular) throw Error("regular file expected");
 
         auto path = store->addToStoreFromDump(
-            method == FileIngestionMethod::Recursive ? *savedNAR.data : savedRegular.s,
+            method == FileIngestionMethod::Recursive ? *savedNAR.s : savedRegular.s,
             baseName,
             method,
             hashAlgo);
@@ -433,7 +443,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     case wopImportPaths: {
         logger->startWork();
         TunnelSource source(from, to);
-        auto paths = store->importPaths(source, nullptr,
+        auto paths = store->importPaths(source,
             trusted ? NoCheckSigs : CheckSigs);
         logger->stopWork();
         Strings paths2;
@@ -722,9 +732,9 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         if (GET_PROTOCOL_MINOR(clientVersion) >= 21)
             source = std::make_unique<TunnelSource>(from, to);
         else {
-            TeeSink tee(from);
+            TeeParseSink tee(from);
             parseDump(tee, tee.source);
-            saved = std::move(*tee.source.data);
+            saved = std::move(*tee.saved.s);
             source = std::make_unique<StringSource>(saved);
         }
 
@@ -732,7 +742,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
 
         // FIXME: race if addToStore doesn't read source?
         store->addToStore(info, *source, (RepairFlag) repair,
-            dontCheckSigs ? NoCheckSigs : CheckSigs, nullptr);
+            dontCheckSigs ? NoCheckSigs : CheckSigs);
 
         logger->stopWork();
         break;

src/libstore/derivations.cc

@@ -4,7 +4,6 @@
 #include "util.hh"
 #include "worker-protocol.hh"
 #include "fs-accessor.hh"
-#include "istringstream_nocopy.hh"
 
 namespace nix {
 
@@ -101,7 +100,7 @@ static StringSet parseStrings(std::istream & str, bool arePaths)
 }
 
 
-static DerivationOutput parseDerivationOutput(const Store & store, istringstream_nocopy & str)
+static DerivationOutput parseDerivationOutput(const Store & store, std::istringstream & str)
 {
     expect(str, ","); auto path = store.parseStorePath(parsePath(str));
     expect(str, ","); auto hashAlgo = parseString(str);
@@ -129,10 +128,10 @@ static DerivationOutput parseDerivationOutput(const Store & store, istringstream
 }
 
 
-static Derivation parseDerivation(const Store & store, const string & s)
+static Derivation parseDerivation(const Store & store, std::string && s)
 {
     Derivation drv;
-    istringstream_nocopy str(s);
+    std::istringstream str(std::move(s));
     expect(str, "Derive([");
 
     /* Parse the list of outputs. */

src/libstore/export-import.cc

@@ -7,24 +7,6 @@
 
 namespace nix {
 
-struct HashAndWriteSink : Sink
-{
-    Sink & writeSink;
-    HashSink hashSink;
-    HashAndWriteSink(Sink & writeSink) : writeSink(writeSink), hashSink(htSHA256)
-    {
-    }
-    virtual void operator () (const unsigned char * data, size_t len)
-    {
-        writeSink(data, len);
-        hashSink(data, len);
-    }
-    Hash currentHash()
-    {
-        return hashSink.currentHash().first;
-    }
-};
-
 void Store::exportPaths(const StorePathSet & paths, Sink & sink)
 {
     auto sorted = topoSortPaths(paths);
@@ -47,28 +29,29 @@ void Store::exportPath(const StorePath & path, Sink & sink)
 {
     auto info = queryPathInfo(path);
 
-    HashAndWriteSink hashAndWriteSink(sink);
+    HashSink hashSink(htSHA256);
+    TeeSink teeSink(sink, hashSink);
 
-    narFromPath(path, hashAndWriteSink);
+    narFromPath(path, teeSink);
 
     /* Refuse to export paths that have changed.  This prevents
        filesystem corruption from spreading to other machines.
        Don't complain if the stored hash is zero (unknown). */
-    Hash hash = hashAndWriteSink.currentHash();
+    Hash hash = hashSink.currentHash().first;
     if (hash != info->narHash && info->narHash != Hash(*info->narHash.type))
         throw Error("hash of path '%s' has changed from '%s' to '%s'!",
             printStorePath(path), info->narHash.to_string(Base32, true), hash.to_string(Base32, true));
 
-    hashAndWriteSink
+    teeSink
         << exportMagic
         << printStorePath(path);
-    writeStorePaths(*this, hashAndWriteSink, info->references);
-    hashAndWriteSink
+    writeStorePaths(*this, teeSink, info->references);
+    teeSink
         << (info->deriver ? printStorePath(*info->deriver) : "")
         << 0;
 }
 
-StorePaths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> accessor, CheckSigsFlag checkSigs)
+StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs)
 {
     StorePaths res;
     while (true) {
@@ -77,7 +60,7 @@ StorePaths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> acces
         if (n != 1) throw Error("input doesn't look like something created by 'nix-store --export'");
 
         /* Extract the NAR from the source. */
-        TeeSink tee(source);
+        TeeParseSink tee(source);
         parseDump(tee, tee.source);
 
         uint32_t magic = readInt(source);
@@ -94,16 +77,16 @@ StorePaths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> acces
         if (deriver != "")
             info.deriver = parseStorePath(deriver);
 
-        info.narHash = hashString(htSHA256, *tee.source.data);
-        info.narSize = tee.source.data->size();
+        info.narHash = hashString(htSHA256, *tee.saved.s);
+        info.narSize = tee.saved.s->size();
 
         // Ignore optional legacy signature.
         if (readInt(source) == 1)
             readString(source);
 
         // Can't use underlying source, which would have been exhausted
-        auto source = StringSource { *tee.source.data };
-        addToStore(info, source, NoRepair, checkSigs, accessor);
+        auto source = StringSource { *tee.saved.s };
+        addToStore(info, source, NoRepair, checkSigs);
 
         res.push_back(info.path);
     }

src/libstore/gc.cc

@@ -262,11 +262,13 @@ void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor)
 void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots)
 {
     auto foundRoot = [&](const Path & path, const Path & target) {
-        auto storePath = maybeParseStorePath(toStorePath(target));
-        if (storePath && isValidPath(*storePath))
-            roots[std::move(*storePath)].emplace(path);
-        else
-            printInfo("skipping invalid root from '%1%' to '%2%'", path, target);
+        try {
+            auto storePath = toStorePath(target).first;
+            if (isValidPath(storePath))
+                roots[std::move(storePath)].emplace(path);
+            else
+                printInfo("skipping invalid root from '%1%' to '%2%'", path, target);
+        } catch (BadStorePath &) { }
     };
 
     try {
@@ -472,15 +474,15 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
 
     for (auto & [target, links] : unchecked) {
         if (!isInStore(target)) continue;
-        Path pathS = toStorePath(target);
-        if (!isStorePath(pathS)) continue;
-        auto path = parseStorePath(pathS);
-        if (!isValidPath(path)) continue;
-        debug("got additional root '%1%'", pathS);
-        if (censor)
-            roots[path].insert(censored);
-        else
-            roots[path].insert(links.begin(), links.end());
+        try {
+            auto path = toStorePath(target).first;
+            if (!isValidPath(path)) continue;
+            debug("got additional root '%1%'", printStorePath(path));
+            if (censor)
+                roots[path].insert(censored);
+            else
+                roots[path].insert(links.begin(), links.end());
+        } catch (BadStorePath &) { }
     }
 }
 

src/libstore/globals.cc

@@ -35,7 +35,7 @@ Settings::Settings()
     , nixLibexecDir(canonPath(getEnv("NIX_LIBEXEC_DIR").value_or(NIX_LIBEXEC_DIR)))
     , nixBinDir(canonPath(getEnv("NIX_BIN_DIR").value_or(NIX_BIN_DIR)))
     , nixManDir(canonPath(NIX_MAN_DIR))
-    , nixDaemonSocketFile(canonPath(nixStateDir + DEFAULT_SOCKET_PATH))
+    , nixDaemonSocketFile(canonPath(getEnv("NIX_DAEMON_SOCKET_PATH").value_or(nixStateDir + DEFAULT_SOCKET_PATH)))
 {
     buildUsersGroup = getuid() == 0 ? "nixbld" : "";
     lockCPU = getEnv("NIX_AFFINITY_HACK") == "1";

src/libstore/globals.hh

@@ -196,10 +196,6 @@ public:
     /* Whether to lock the Nix client and worker to the same CPU. */
     bool lockCPU;
 
-    /* Whether to show a stack trace if Nix evaluation fails. */
-    Setting<bool> showTrace{this, false, "show-trace",
-        "Whether to show a stack trace on evaluation errors."};
-
     Setting<SandboxMode> sandboxMode{this,
         #if __linux__
           smEnabled
@@ -369,6 +365,9 @@ public:
 
     Setting<bool> warnDirty{this, true, "warn-dirty",
         "Whether to warn about dirty Git/Mercurial trees."};
+
+    Setting<size_t> narBufferSize{this, 32 * 1024 * 1024, "nar-buffer-size",
+        "Maximum size of NARs before spilling them to disk."};
 };
 
 

src/libstore/http-binary-cache-store.cc

@@ -100,11 +100,11 @@ protected:
     }
 
     void upsertFile(const std::string & path,
-        const std::string & data,
+        std::shared_ptr<std::basic_iostream<char>> istream,
         const std::string & mimeType) override
     {
         auto req = FileTransferRequest(cacheUri + "/" + path);
-        req.data = std::make_shared<string>(data); // FIXME: inefficient
+        req.data = std::make_shared<string>(StreamToSourceAdapter(istream).drain());
         req.mimeType = mimeType;
         try {
             getFileTransfer()->upload(req);

src/libstore/legacy-ssh-store.cc

@@ -126,8 +126,7 @@ struct LegacySSHStore : public Store
     }
 
     void addToStore(const ValidPathInfo & info, Source & source,
-        RepairFlag repair, CheckSigsFlag checkSigs,
-        std::shared_ptr<FSAccessor> accessor) override
+        RepairFlag repair, CheckSigsFlag checkSigs) override
     {
         debug("adding path '%s' to remote host '%s'", printStorePath(info.path), host);
 

src/libstore/local-binary-cache-store.cc

@@ -31,8 +31,18 @@ protected:
     bool fileExists(const std::string & path) override;
 
     void upsertFile(const std::string & path,
-        const std::string & data,
-        const std::string & mimeType) override;
+        std::shared_ptr<std::basic_iostream<char>> istream,
+        const std::string & mimeType) override
+    {
+        auto path2 = binaryCacheDir + "/" + path;
+        Path tmp = path2 + ".tmp." + std::to_string(getpid());
+        AutoDelete del(tmp, false);
+        StreamToSourceAdapter source(istream);
+        writeFile(tmp, source);
+        if (rename(tmp.c_str(), path2.c_str()))
+            throw SysError("renaming '%1%' to '%2%'", tmp, path2);
+        del.cancel();
+    }
 
     void getFile(const std::string & path, Sink & sink) override
     {
@@ -52,7 +62,9 @@ protected:
             if (entry.name.size() != 40 ||
                 !hasSuffix(entry.name, ".narinfo"))
                 continue;
-            paths.insert(parseStorePath(storeDir + "/" + entry.name.substr(0, entry.name.size() - 8)));
+            paths.insert(parseStorePath(
+                    storeDir + "/" + entry.name.substr(0, entry.name.size() - 8)
+                    + "-" + MissingName));
         }
 
         return paths;
@@ -68,28 +80,11 @@ void LocalBinaryCacheStore::init()
     BinaryCacheStore::init();
 }
 
-static void atomicWrite(const Path & path, const std::string & s)
-{
-    Path tmp = path + ".tmp." + std::to_string(getpid());
-    AutoDelete del(tmp, false);
-    writeFile(tmp, s);
-    if (rename(tmp.c_str(), path.c_str()))
-        throw SysError("renaming '%1%' to '%2%'", tmp, path);
-    del.cancel();
-}
-
 bool LocalBinaryCacheStore::fileExists(const std::string & path)
 {
     return pathExists(binaryCacheDir + "/" + path);
 }
 
-void LocalBinaryCacheStore::upsertFile(const std::string & path,
-    const std::string & data,
-    const std::string & mimeType)
-{
-    atomicWrite(binaryCacheDir + "/" + path, data);
-}
-
 static RegisterStoreImplementation regStore([](
     const std::string & uri, const Store::Params & params)
     -> std::shared_ptr<Store>

src/libstore/local-fs-store.cc

@@ -20,9 +20,9 @@ struct LocalStoreAccessor : public FSAccessor
 
     Path toRealPath(const Path & path)
     {
-        Path storePath = store->toStorePath(path);
-        if (!store->isValidPath(store->parseStorePath(storePath)))
-            throw InvalidPath("path '%1%' is not a valid store path", storePath);
+        auto storePath = store->toStorePath(path).first;
+        if (!store->isValidPath(storePath))
+            throw InvalidPath("path '%1%' is not a valid store path", store->printStorePath(storePath));
         return store->getRealStoreDir() + std::string(path, store->storeDir.size());
     }
 

src/libstore/local-store.cc

@@ -774,17 +774,20 @@ StorePathSet LocalStore::queryValidDerivers(const StorePath & path)
 }
 
 
-StorePathSet LocalStore::queryDerivationOutputs(const StorePath & path)
+OutputPathMap LocalStore::queryDerivationOutputMap(const StorePath & path)
 {
-    return retrySQLite<StorePathSet>([&]() {
+    return retrySQLite<OutputPathMap>([&]() {
         auto state(_state.lock());
 
         auto useQueryDerivationOutputs(state->stmtQueryDerivationOutputs.use()
             (queryValidPathId(*state, path)));
 
-        StorePathSet outputs;
+        OutputPathMap outputs;
         while (useQueryDerivationOutputs.next())
-            outputs.insert(parseStorePath(useQueryDerivationOutputs.getStr(1)));
+            outputs.emplace(
+                useQueryDerivationOutputs.getStr(0),
+                parseStorePath(useQueryDerivationOutputs.getStr(1))
+            );
 
         return outputs;
     });
@@ -959,7 +962,7 @@ const PublicKeys & LocalStore::getPublicKeys()
 
 
 void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
-    RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor)
+    RepairFlag repair, CheckSigsFlag checkSigs)
 {
     if (!info.narHash)
         throw Error("cannot add path '%s' because it lacks a hash", printStorePath(info.path));
@@ -973,7 +976,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
 
         PathLocks outputLock;
 
-        Path realPath = realStoreDir + "/" + std::string(info.path.to_string());
+        auto realPath = Store::toRealPath(info.path);
 
         /* Lock the output path.  But don't lock if we're being called
            from a build hook (whose parent process already acquired a
@@ -1044,8 +1047,7 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam
         /* The first check above is an optimisation to prevent
            unnecessary lock acquisition. */
 
-        Path realPath = realStoreDir + "/";
-        realPath += dstPath.to_string();
+        auto realPath = Store::toRealPath(dstPath);
 
         PathLocks outputLock({realPath});
 
@@ -1095,16 +1097,119 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath,
 {
     Path srcPath(absPath(_srcPath));
 
-    /* Read the whole path into memory. This is not a very scalable
-       method for very large paths, but `copyPath' is mainly used for
-       small files. */
-    StringSink sink;
-    if (method == FileIngestionMethod::Recursive)
-        dumpPath(srcPath, sink, filter);
-    else
-        sink.s = make_ref<std::string>(readFile(srcPath));
+    if (method != FileIngestionMethod::Recursive)
+        return addToStoreFromDump(readFile(srcPath), name, method, hashAlgo, repair);
 
-    return addToStoreFromDump(*sink.s, name, method, hashAlgo, repair);
+    /* For computing the NAR hash. */
+    auto sha256Sink = std::make_unique<HashSink>(htSHA256);
+
+    /* For computing the store path. In recursive SHA-256 mode, this
+       is the same as the NAR hash, so no need to do it again. */
+    std::unique_ptr<HashSink> hashSink =
+        hashAlgo == htSHA256
+        ? nullptr
+        : std::make_unique<HashSink>(hashAlgo);
+
+    /* Read the source path into memory, but only if it's up to
+       narBufferSize bytes. If it's larger, write it to a temporary
+       location in the Nix store. If the subsequently computed
+       destination store path is already valid, we just delete the
+       temporary path. Otherwise, we move it to the destination store
+       path. */
+    bool inMemory = true;
+    std::string nar;
+
+    auto source = sinkToSource([&](Sink & sink) {
+
+        LambdaSink sink2([&](const unsigned char * buf, size_t len) {
+            (*sha256Sink)(buf, len);
+            if (hashSink) (*hashSink)(buf, len);
+
+            if (inMemory) {
+                if (nar.size() + len > settings.narBufferSize) {
+                    inMemory = false;
+                    sink << 1;
+                    sink((const unsigned char *) nar.data(), nar.size());
+                    nar.clear();
+                } else {
+                    nar.append((const char *) buf, len);
+                }
+            }
+
+            if (!inMemory) sink(buf, len);
+        });
+
+        dumpPath(srcPath, sink2, filter);
+    });
+
+    std::unique_ptr<AutoDelete> delTempDir;
+    Path tempPath;
+
+    try {
+        /* Wait for the source coroutine to give us some dummy
+           data. This is so that we don't create the temporary
+           directory if the NAR fits in memory. */
+        readInt(*source);
+
+        auto tempDir = createTempDir(realStoreDir, "add");
+        delTempDir = std::make_unique<AutoDelete>(tempDir);
+        tempPath = tempDir + "/x";
+
+        restorePath(tempPath, *source);
+
+    } catch (EndOfFile &) {
+        if (!inMemory) throw;
+        /* The NAR fits in memory, so we didn't do restorePath(). */
+    }
+
+    auto sha256 = sha256Sink->finish();
+
+    Hash hash = hashSink ? hashSink->finish().first : sha256.first;
+
+    auto dstPath = makeFixedOutputPath(method, hash, name);
+
+    addTempRoot(dstPath);
+
+    if (repair || !isValidPath(dstPath)) {
+
+        /* The first check above is an optimisation to prevent
+           unnecessary lock acquisition. */
+
+        auto realPath = Store::toRealPath(dstPath);
+
+        PathLocks outputLock({realPath});
+
+        if (repair || !isValidPath(dstPath)) {
+
+            deletePath(realPath);
+
+            autoGC();
+
+            if (inMemory) {
+                /* Restore from the NAR in memory. */
+                StringSource source(nar);
+                restorePath(realPath, source);
+            } else {
+                /* Move the temporary path we restored above. */
+                if (rename(tempPath.c_str(), realPath.c_str()))
+                    throw Error("renaming '%s' to '%s'", tempPath, realPath);
+            }
+
+            canonicalisePathMetaData(realPath, -1); // FIXME: merge into restorePath
+
+            optimisePath(realPath);
+
+            ValidPathInfo info(dstPath);
+            info.narHash = sha256.first;
+            info.narSize = sha256.second;
+            info.ca = FixedOutputHash { .method = method, .hash = hash };
+            registerValidPath(info);
+        }
+
+        outputLock.setDeletion(true);
+    }
+
+    return dstPath;
 }
 
 
@@ -1118,8 +1223,7 @@ StorePath LocalStore::addTextToStore(const string & name, const string & s,
 
     if (repair || !isValidPath(dstPath)) {
 
-        Path realPath = realStoreDir + "/";
-        realPath += dstPath.to_string();
+        auto realPath = Store::toRealPath(dstPath);
 
         PathLocks outputLock({realPath});
 

src/libstore/local-store.hh

@@ -133,7 +133,7 @@ public:
 
     StorePathSet queryValidDerivers(const StorePath & path) override;
 
-    StorePathSet queryDerivationOutputs(const StorePath & path) override;
+    OutputPathMap queryDerivationOutputMap(const StorePath & path) override;
 
     std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override;
 
@@ -143,8 +143,7 @@ public:
         SubstitutablePathInfos & infos) override;
 
     void addToStore(const ValidPathInfo & info, Source & source,
-        RepairFlag repair, CheckSigsFlag checkSigs,
-        std::shared_ptr<FSAccessor> accessor) override;
+        RepairFlag repair, CheckSigsFlag checkSigs) override;
 
     StorePath addToStore(const string & name, const Path & srcPath,
         FileIngestionMethod method, HashType hashAlgo,

src/libstore/nar-accessor.cc

@@ -18,7 +18,7 @@ struct NarMember
 
     /* If this is a regular file, position of the contents of this
        file in the NAR. */
-    size_t start = 0, size = 0;
+    uint64_t start = 0, size = 0;
 
     std::string target;
 
@@ -34,17 +34,19 @@ struct NarAccessor : public FSAccessor
 
     NarMember root;
 
-    struct NarIndexer : ParseSink, StringSource
+    struct NarIndexer : ParseSink, Source
     {
         NarAccessor & acc;
+        Source & source;
 
         std::stack<NarMember *> parents;
 
-        std::string currentStart;
         bool isExec = false;
 
-        NarIndexer(NarAccessor & acc, const std::string & nar)
-            : StringSource(nar), acc(acc)
+        uint64_t pos = 0;
+
+        NarIndexer(NarAccessor & acc, Source & source)
+            : acc(acc), source(source)
         { }
 
         void createMember(const Path & path, NarMember member) {
@@ -79,31 +81,38 @@ struct NarAccessor : public FSAccessor
 
         void preallocateContents(unsigned long long size) override
         {
-            currentStart = string(s, pos, 16);
-            assert(size <= std::numeric_limits<size_t>::max());
-            parents.top()->size = (size_t)size;
+            assert(size <= std::numeric_limits<uint64_t>::max());
+            parents.top()->size = (uint64_t) size;
             parents.top()->start = pos;
         }
 
         void receiveContents(unsigned char * data, unsigned int len) override
-        {
-            // Sanity check
-            if (!currentStart.empty()) {
-                assert(len < 16 || currentStart == string((char *) data, 16));
-                currentStart.clear();
-            }
-        }
+        { }
 
         void createSymlink(const Path & path, const string & target) override
         {
             createMember(path,
                 NarMember{FSAccessor::Type::tSymlink, false, 0, 0, target});
         }
+
+        size_t read(unsigned char * data, size_t len) override
+        {
+            auto n = source.read(data, len);
+            pos += n;
+            return n;
+        }
     };
 
     NarAccessor(ref<const std::string> nar) : nar(nar)
     {
-        NarIndexer indexer(*this, *nar);
+        StringSource source(*nar);
+        NarIndexer indexer(*this, source);
+        parseDump(indexer, indexer);
+    }
+
+    NarAccessor(Source & source)
+    {
+        NarIndexer indexer(*this, source);
         parseDump(indexer, indexer);
     }
 
@@ -219,6 +228,11 @@ ref<FSAccessor> makeNarAccessor(ref<const std::string> nar)
     return make_ref<NarAccessor>(nar);
 }
 
+ref<FSAccessor> makeNarAccessor(Source & source)
+{
+    return make_ref<NarAccessor>(source);
+}
+
 ref<FSAccessor> makeLazyNarAccessor(const std::string & listing,
     GetNarBytes getNarBytes)
 {

src/libstore/nar-accessor.hh

@@ -6,10 +6,14 @@
 
 namespace nix {
 
+struct Source;
+
 /* Return an object that provides access to the contents of a NAR
    file. */
 ref<FSAccessor> makeNarAccessor(ref<const std::string> nar);
 
+ref<FSAccessor> makeNarAccessor(Source & source);
+
 /* Create a NAR accessor from a NAR listing (in the format produced by
    listNar()). The callback getNarBytes(offset, length) is used by the
    readFile() method of the accessor to get the contents of files

src/libstore/path.cc

@@ -2,8 +2,6 @@
 
 namespace nix {
 
-MakeError(BadStorePath, Error);
-
 static void checkName(std::string_view path, std::string_view name)
 {
     if (name.empty())

src/libstore/path.hh

@@ -62,6 +62,7 @@ public:
 
 typedef std::set<StorePath> StorePathSet;
 typedef std::vector<StorePath> StorePaths;
+typedef std::map<string, StorePath> OutputPathMap;
 
 /* Extension of derivations in the Nix store. */
 const std::string drvExtension = ".drv";

src/libstore/remote-fs-accessor.cc

@@ -16,26 +16,26 @@ RemoteFSAccessor::RemoteFSAccessor(ref<Store> store, const Path & cacheDir)
         createDirs(cacheDir);
 }
 
-Path RemoteFSAccessor::makeCacheFile(const Path & storePath, const std::string & ext)
+Path RemoteFSAccessor::makeCacheFile(std::string_view hashPart, const std::string & ext)
 {
     assert(cacheDir != "");
-    return fmt("%s/%s.%s", cacheDir, store->parseStorePath(storePath).hashPart(), ext);
+    return fmt("%s/%s.%s", cacheDir, hashPart, ext);
 }
 
-void RemoteFSAccessor::addToCache(const Path & storePath, const std::string & nar,
+void RemoteFSAccessor::addToCache(std::string_view hashPart, const std::string & nar,
     ref<FSAccessor> narAccessor)
 {
-    nars.emplace(storePath, narAccessor);
+    nars.emplace(hashPart, narAccessor);
 
     if (cacheDir != "") {
         try {
             std::ostringstream str;
             JSONPlaceholder jsonRoot(str);
             listNar(jsonRoot, narAccessor, "", true);
-            writeFile(makeCacheFile(storePath, "ls"), str.str());
+            writeFile(makeCacheFile(hashPart, "ls"), str.str());
 
             /* FIXME: do this asynchronously. */
-            writeFile(makeCacheFile(storePath, "nar"), nar);
+            writeFile(makeCacheFile(hashPart, "nar"), nar);
 
         } catch (...) {
             ignoreException();
@@ -47,23 +47,22 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_)
 {
     auto path = canonPath(path_);
 
-    auto storePath = store->toStorePath(path);
-    std::string restPath = std::string(path, storePath.size());
+    auto [storePath, restPath] = store->toStorePath(path);
 
-    if (!store->isValidPath(store->parseStorePath(storePath)))
-        throw InvalidPath("path '%1%' is not a valid store path", storePath);
+    if (!store->isValidPath(storePath))
+        throw InvalidPath("path '%1%' is not a valid store path", store->printStorePath(storePath));
 
-    auto i = nars.find(storePath);
+    auto i = nars.find(std::string(storePath.hashPart()));
     if (i != nars.end()) return {i->second, restPath};
 
     StringSink sink;
     std::string listing;
     Path cacheFile;
 
-    if (cacheDir != "" && pathExists(cacheFile = makeCacheFile(storePath, "nar"))) {
+    if (cacheDir != "" && pathExists(cacheFile = makeCacheFile(storePath.hashPart(), "nar"))) {
 
         try {
-            listing = nix::readFile(makeCacheFile(storePath, "ls"));
+            listing = nix::readFile(makeCacheFile(storePath.hashPart(), "ls"));
 
             auto narAccessor = makeLazyNarAccessor(listing,
                 [cacheFile](uint64_t offset, uint64_t length) {
@@ -81,7 +80,7 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_)
                     return buf;
                 });
 
-            nars.emplace(storePath, narAccessor);
+            nars.emplace(storePath.hashPart(), narAccessor);
             return {narAccessor, restPath};
 
         } catch (SysError &) { }
@@ -90,15 +89,15 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_)
             *sink.s = nix::readFile(cacheFile);
 
             auto narAccessor = makeNarAccessor(sink.s);
-            nars.emplace(storePath, narAccessor);
+            nars.emplace(storePath.hashPart(), narAccessor);
             return {narAccessor, restPath};
 
         } catch (SysError &) { }
     }
 
-    store->narFromPath(store->parseStorePath(storePath), sink);
+    store->narFromPath(storePath, sink);
     auto narAccessor = makeNarAccessor(sink.s);
-    addToCache(storePath, *sink.s, narAccessor);
+    addToCache(storePath.hashPart(), *sink.s, narAccessor);
     return {narAccessor, restPath};
 }
 

src/libstore/remote-fs-accessor.hh

@@ -10,7 +10,7 @@ class RemoteFSAccessor : public FSAccessor
 {
     ref<Store> store;
 
-    std::map<Path, ref<FSAccessor>> nars;
+    std::map<std::string, ref<FSAccessor>> nars;
 
     Path cacheDir;
 
@@ -18,9 +18,9 @@ class RemoteFSAccessor : public FSAccessor
 
     friend class BinaryCacheStore;
 
-    Path makeCacheFile(const Path & storePath, const std::string & ext);
+    Path makeCacheFile(std::string_view hashPart, const std::string & ext);
 
-    void addToCache(const Path & storePath, const std::string & nar,
+    void addToCache(std::string_view hashPart, const std::string & nar,
         ref<FSAccessor> narAccessor);
 
 public:

src/libstore/remote-store.cc

@@ -8,6 +8,7 @@
 #include "derivations.hh"
 #include "pool.hh"
 #include "finally.hh"
+#include "logging.hh"
 
 #include <sys/types.h>
 #include <sys/stat.h>
@@ -38,6 +39,29 @@ void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths
         out << store.printStorePath(i);
 }
 
+std::map<string, StorePath> readOutputPathMap(const Store & store, Source & from)
+{
+    std::map<string, StorePath> pathMap;
+    auto rawInput = readStrings<Strings>(from);
+    if (rawInput.size() % 2)
+        throw Error("got an odd number of elements from the daemon when trying to read a output path map");
+    auto curInput = rawInput.begin();
+    while (curInput != rawInput.end()) {
+        auto thisKey = *curInput++;
+        auto thisValue = *curInput++;
+        pathMap.emplace(thisKey, store.parseStorePath(thisValue));
+    }
+    return pathMap;
+}
+
+void writeOutputPathMap(const Store & store, Sink & out, const std::map<string, StorePath> & pathMap)
+{
+    out << 2*pathMap.size();
+    for (auto & i : pathMap) {
+        out << i.first;
+        out << store.printStorePath(i.second);
+    }
+}
 
 /* TODO: Separate these store impls into different files, give them better names */
 RemoteStore::RemoteStore(const Params & params)
@@ -197,7 +221,7 @@ void RemoteStore::setOptions(Connection & conn)
         overrides.erase(settings.maxSilentTime.name);
         overrides.erase(settings.buildCores.name);
         overrides.erase(settings.useSubstitutes.name);
-        overrides.erase(settings.showTrace.name);
+        overrides.erase(loggerSettings.showTrace.name);
         conn.to << overrides.size();
         for (auto & i : overrides)
             conn.to << i.first << i.second.value;
@@ -412,12 +436,24 @@ StorePathSet RemoteStore::queryValidDerivers(const StorePath & path)
 StorePathSet RemoteStore::queryDerivationOutputs(const StorePath & path)
 {
     auto conn(getConnection());
+    if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 0x16) {
+        return Store::queryDerivationOutputs(path);
+    }
     conn->to << wopQueryDerivationOutputs << printStorePath(path);
     conn.processStderr();
     return readStorePaths<StorePathSet>(*this, conn->from);
 }
 
 
+OutputPathMap RemoteStore::queryDerivationOutputMap(const StorePath & path)
+{
+    auto conn(getConnection());
+    conn->to << wopQueryDerivationOutputMap << printStorePath(path);
+    conn.processStderr();
+    return readOutputPathMap(*this, conn->from);
+
+}
+
 std::optional<StorePath> RemoteStore::queryPathFromHashPart(const std::string & hashPart)
 {
     auto conn(getConnection());
@@ -430,7 +466,7 @@ std::optional<StorePath> RemoteStore::queryPathFromHashPart(const std::string &
 
 
 void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
-    RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor)
+    RepairFlag repair, CheckSigsFlag checkSigs)
 {
     auto conn(getConnection());
 

src/libstore/remote-store.hh

@@ -51,6 +51,7 @@ public:
 
     StorePathSet queryDerivationOutputs(const StorePath & path) override;
 
+    OutputPathMap queryDerivationOutputMap(const StorePath & path) override;
     std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override;
 
     StorePathSet querySubstitutablePaths(const StorePathSet & paths) override;
@@ -59,8 +60,7 @@ public:
         SubstitutablePathInfos & infos) override;
 
     void addToStore(const ValidPathInfo & info, Source & nar,
-        RepairFlag repair, CheckSigsFlag checkSigs,
-        std::shared_ptr<FSAccessor> accessor) override;
+        RepairFlag repair, CheckSigsFlag checkSigs) override;
 
     StorePath addToStore(const string & name, const Path & srcPath,
         FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256,

src/libstore/s3-binary-cache-store.cc

@@ -7,7 +7,6 @@
 #include "globals.hh"
 #include "compression.hh"
 #include "filetransfer.hh"
-#include "istringstream_nocopy.hh"
 
 #include <aws/core/Aws.h>
 #include <aws/core/VersionConfig.h>
@@ -262,12 +261,11 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
     std::shared_ptr<TransferManager> transferManager;
     std::once_flag transferManagerCreated;
 
-    void uploadFile(const std::string & path, const std::string & data,
+    void uploadFile(const std::string & path,
+        std::shared_ptr<std::basic_iostream<char>> istream,
         const std::string & mimeType,
         const std::string & contentEncoding)
     {
-        auto stream = std::make_shared<istringstream_nocopy>(data);
-
         auto maxThreads = std::thread::hardware_concurrency();
 
         static std::shared_ptr<Aws::Utils::Threading::PooledThreadExecutor>
@@ -307,7 +305,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
 
             std::shared_ptr<TransferHandle> transferHandle =
                 transferManager->UploadFile(
-                    stream, bucketName, path, mimeType,
+                    istream, bucketName, path, mimeType,
                     Aws::Map<Aws::String, Aws::String>(),
                     nullptr /*, contentEncoding */);
 
@@ -333,9 +331,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
             if (contentEncoding != "")
                 request.SetContentEncoding(contentEncoding);
 
-            auto stream = std::make_shared<istringstream_nocopy>(data);
-
-            request.SetBody(stream);
+            request.SetBody(istream);
 
             auto result = checkAws(fmt("AWS error uploading '%s'", path),
                 s3Helper.client->PutObject(request));
@@ -347,25 +343,34 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
             std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1)
                 .count();
 
-        printInfo(format("uploaded 's3://%1%/%2%' (%3% bytes) in %4% ms") %
-                  bucketName % path % data.size() % duration);
+        auto size = istream->tellg();
+
+        printInfo("uploaded 's3://%s/%s' (%d bytes) in %d ms",
+            bucketName, path, size, duration);
 
         stats.putTimeMs += duration;
-        stats.putBytes += data.size();
+        stats.putBytes += size;
         stats.put++;
     }
 
-    void upsertFile(const std::string & path, const std::string & data,
+    void upsertFile(const std::string & path,
+        std::shared_ptr<std::basic_iostream<char>> istream,
         const std::string & mimeType) override
     {
+        auto compress = [&](std::string compression)
+        {
+            auto compressed = nix::compress(compression, StreamToSourceAdapter(istream).drain());
+            return std::make_shared<std::stringstream>(std::move(*compressed));
+        };
+
         if (narinfoCompression != "" && hasSuffix(path, ".narinfo"))
-            uploadFile(path, *compress(narinfoCompression, data), mimeType, narinfoCompression);
+            uploadFile(path, compress(narinfoCompression), mimeType, narinfoCompression);
         else if (lsCompression != "" && hasSuffix(path, ".ls"))
-            uploadFile(path, *compress(lsCompression, data), mimeType, lsCompression);
+            uploadFile(path, compress(lsCompression), mimeType, lsCompression);
         else if (logCompression != "" && hasPrefix(path, "log/"))
-            uploadFile(path, *compress(logCompression, data), mimeType, logCompression);
+            uploadFile(path, compress(logCompression), mimeType, logCompression);
         else
-            uploadFile(path, data, mimeType, "");
+            uploadFile(path, istream, mimeType, "");
     }
 
     void getFile(const std::string & path, Sink & sink) override
@@ -410,7 +415,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
             for (auto object : contents) {
                 auto & key = object.GetKey();
                 if (key.size() != 40 || !hasSuffix(key, ".narinfo")) continue;
-                paths.insert(parseStorePath(storeDir + "/" + key.substr(0, key.size() - 8) + "-unknown"));
+                paths.insert(parseStorePath(storeDir + "/" + key.substr(0, key.size() - 8) + "-" + MissingName));
             }
 
             marker = res.GetNextMarker();

src/libstore/store-api.cc

@@ -7,6 +7,7 @@
 #include "json.hh"
 #include "derivations.hh"
 #include "url.hh"
+#include "archive.hh"
 
 #include <future>
 
@@ -20,15 +21,15 @@ bool Store::isInStore(const Path & path) const
 }
 
 
-Path Store::toStorePath(const Path & path) const
+std::pair<StorePath, Path> Store::toStorePath(const Path & path) const
 {
     if (!isInStore(path))
         throw Error("path '%1%' is not in the Nix store", path);
     Path::size_type slash = path.find('/', storeDir.size() + 1);
     if (slash == Path::npos)
-        return path;
+        return {parseStorePath(path), ""};
     else
-        return Path(path, 0, slash);
+        return {parseStorePath(std::string_view(path).substr(0, slash)), path.substr(slash)};
 }
 
 
@@ -41,14 +42,14 @@ Path Store::followLinksToStore(std::string_view _path) const
         path = absPath(target, dirOf(path));
     }
     if (!isInStore(path))
-        throw NotInStore("path '%1%' is not in the Nix store", path);
+        throw BadStorePath("path '%1%' is not in the Nix store", path);
     return path;
 }
 
 
 StorePath Store::followLinksToStorePath(std::string_view path) const
 {
-    return parseStorePath(toStorePath(followLinksToStore(path)));
+    return toStorePath(followLinksToStore(path)).first;
 }
 
 
@@ -221,6 +222,40 @@ StorePath Store::computeStorePathForText(const string & name, const string & s,
 }
 
 
+ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath,
+    FileIngestionMethod method, HashType hashAlgo,
+    std::optional<Hash> expectedCAHash)
+{
+    /* FIXME: inefficient: we're reading/hashing 'tmpFile' three
+       times. */
+
+    auto [narHash, narSize] = hashPath(htSHA256, srcPath);
+
+    auto hash = method == FileIngestionMethod::Recursive
+        ? hashAlgo == htSHA256
+          ? narHash
+          : hashPath(hashAlgo, srcPath).first
+        : hashFile(hashAlgo, srcPath);
+
+    if (expectedCAHash && expectedCAHash != hash)
+        throw Error("hash mismatch for '%s'", srcPath);
+
+    ValidPathInfo info(makeFixedOutputPath(method, hash, name));
+    info.narHash = narHash;
+    info.narSize = narSize;
+    info.ca = FixedOutputHash { .method = method, .hash = hash };
+
+    if (!isValidPath(info.path)) {
+        auto source = sinkToSource([&](Sink & sink) {
+            dumpPath(srcPath, sink);
+        });
+        addToStore(info, *source);
+    }
+
+    return info;
+}
+
+
 Store::Store(const Params & params)
     : Config(params)
     , state({(size_t) pathInfoCacheSize})
@@ -242,6 +277,16 @@ bool Store::PathInfoCacheValue::isKnownNow()
     return std::chrono::steady_clock::now() < time_point + ttl;
 }
 
+StorePathSet Store::queryDerivationOutputs(const StorePath & path)
+{
+    auto outputMap = this->queryDerivationOutputMap(path);
+    StorePathSet outputPaths;
+    for (auto & i: outputMap) {
+        outputPaths.emplace(std::move(i.second));
+    }
+    return outputPaths;
+}
+
 bool Store::isValidPath(const StorePath & storePath)
 {
     std::string hashPart(storePath.hashPart());
@@ -306,6 +351,14 @@ ref<const ValidPathInfo> Store::queryPathInfo(const StorePath & storePath)
 }
 
 
+static bool goodStorePath(const StorePath & expected, const StorePath & actual)
+{
+    return
+        expected.hashPart() == actual.hashPart()
+        && (expected.name() == Store::MissingName || expected.name() == actual.name());
+}
+
+
 void Store::queryPathInfo(const StorePath & storePath,
     Callback<ref<const ValidPathInfo>> callback) noexcept
 {
@@ -333,7 +386,7 @@ void Store::queryPathInfo(const StorePath & storePath,
                     state_->pathInfoCache.upsert(hashPart,
                         res.first == NarInfoDiskCache::oInvalid ? PathInfoCacheValue{} : PathInfoCacheValue{ .value = res.second });
                     if (res.first == NarInfoDiskCache::oInvalid ||
-                        res.second->path != storePath)
+                        !goodStorePath(storePath, res.second->path))
                         throw InvalidPath("path '%s' is not valid", printStorePath(storePath));
                 }
                 return callback(ref<const ValidPathInfo>(res.second));
@@ -345,7 +398,7 @@ void Store::queryPathInfo(const StorePath & storePath,
     auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback));
 
     queryPathInfoUncached(storePath,
-        {[this, storePath{printStorePath(storePath)}, hashPart, callbackPtr](std::future<std::shared_ptr<const ValidPathInfo>> fut) {
+        {[this, storePathS{printStorePath(storePath)}, hashPart, callbackPtr](std::future<std::shared_ptr<const ValidPathInfo>> fut) {
 
             try {
                 auto info = fut.get();
@@ -358,9 +411,11 @@ void Store::queryPathInfo(const StorePath & storePath,
                     state_->pathInfoCache.upsert(hashPart, PathInfoCacheValue { .value = info });
                 }
 
-                if (!info || info->path != parseStorePath(storePath)) {
+                auto storePath = parseStorePath(storePathS);
+
+                if (!info || !goodStorePath(storePath, info->path)) {
                     stats.narInfoMissing++;
-                    throw InvalidPath("path '%s' is not valid", storePath);
+                    throw InvalidPath("path '%s' is not valid", storePathS);
                 }
 
                 (*callbackPtr)(ref<const ValidPathInfo>(info));
@@ -505,7 +560,7 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & store
                     if (!narInfo->url.empty())
                         jsonPath.attr("url", narInfo->url);
                     if (narInfo->fileHash)
-                        jsonPath.attr("downloadHash", narInfo->fileHash.to_string(Base32, true));
+                        jsonPath.attr("downloadHash", narInfo->fileHash.to_string(hashBase, true));
                     if (narInfo->fileSize)
                         jsonPath.attr("downloadSize", narInfo->fileSize);
                     if (showClosureSize)

src/libstore/store-api.hh

@@ -31,7 +31,7 @@ MakeError(InvalidPath, Error);
 MakeError(Unsupported, Error);
 MakeError(SubstituteGone, Error);
 MakeError(SubstituterDisabled, Error);
-MakeError(NotInStore, Error);
+MakeError(BadStorePath, Error);
 
 
 class FSAccessor;
@@ -317,9 +317,9 @@ public:
        the Nix store. */
     bool isStorePath(std::string_view path) const;
 
-    /* Chop off the parts after the top-level store name, e.g.,
-       /nix/store/abcd-foo/bar => /nix/store/abcd-foo. */
-    Path toStorePath(const Path & path) const;
+    /* Split a path like /nix/store/<hash>-<name>/<bla> into
+       /nix/store/<hash>-<name> and /<bla>. */
+    std::pair<StorePath, Path> toStorePath(const Path & path) const;
 
     /* Follow symlinks until we end up with a path in the Nix store. */
     Path followLinksToStore(std::string_view path) const;
@@ -384,13 +384,16 @@ public:
         SubstituteFlag maybeSubstitute = NoSubstitute);
 
     /* Query the set of all valid paths. Note that for some store
-       backends, the name part of store paths may be omitted
-       (i.e. you'll get /nix/store/<hash> rather than
+       backends, the name part of store paths may be replaced by 'x'
+       (i.e. you'll get /nix/store/<hash>-x rather than
        /nix/store/<hash>-<name>). Use queryPathInfo() to obtain the
-       full store path. */
+       full store path. FIXME: should return a set of
+       std::variant<StorePath, HashPart> to get rid of this hack. */
     virtual StorePathSet queryAllValidPaths()
     { unsupported("queryAllValidPaths"); }
 
+    constexpr static const char * MissingName = "x";
+
     /* Query information about a valid path. It is permitted to omit
        the name part of the store path. */
     ref<const ValidPathInfo> queryPathInfo(const StorePath & path);
@@ -418,8 +421,11 @@ public:
     virtual StorePathSet queryValidDerivers(const StorePath & path) { return {}; };
 
     /* Query the outputs of the derivation denoted by `path'. */
-    virtual StorePathSet queryDerivationOutputs(const StorePath & path)
-    { unsupported("queryDerivationOutputs"); }
+    virtual StorePathSet queryDerivationOutputs(const StorePath & path);
+
+    /* Query the mapping outputName=>outputPath for the given derivation */
+    virtual OutputPathMap queryDerivationOutputMap(const StorePath & path)
+    { unsupported("queryDerivationOutputMap"); }
 
     /* Query the full store path given the hash part of a valid store
        path, or empty if the path doesn't exist. */
@@ -436,8 +442,7 @@ public:
 
     /* Import a path into the store. */
     virtual void addToStore(const ValidPathInfo & info, Source & narSource,
-        RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs,
-        std::shared_ptr<FSAccessor> accessor = 0) = 0;
+        RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs) = 0;
 
     /* Copy the contents of a path to the store and register the
        validity the resulting path.  The resulting path is returned.
@@ -447,6 +452,13 @@ public:
         FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256,
         PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) = 0;
 
+    /* Copy the contents of a path to the store and register the
+       validity the resulting path, using a constant amount of
+       memory. */
+    ValidPathInfo addToStoreSlow(std::string_view name, const Path & srcPath,
+        FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256,
+        std::optional<Hash> expectedCAHash = {});
+
     // FIXME: remove?
     virtual StorePath addToStoreFromDump(const string & dump, const string & name,
         FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair)
@@ -613,8 +625,7 @@ public:
        the Nix store. Optionally, the contents of the NARs are
        preloaded into the specified FS accessor to speed up subsequent
        access. */
-    StorePaths importPaths(Source & source, std::shared_ptr<FSAccessor> accessor,
-        CheckSigsFlag checkSigs = CheckSigs);
+    StorePaths importPaths(Source & source, CheckSigsFlag checkSigs = CheckSigs);
 
     struct Stats
     {

src/libstore/worker-protocol.hh

@@ -6,7 +6,7 @@ namespace nix {
 #define WORKER_MAGIC_1 0x6e697863
 #define WORKER_MAGIC_2 0x6478696f
 
-#define PROTOCOL_VERSION 0x115
+#define PROTOCOL_VERSION 0x116
 #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
 #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
 
@@ -30,7 +30,7 @@ typedef enum {
     wopSetOptions = 19,
     wopCollectGarbage = 20,
     wopQuerySubstitutablePathInfo = 21,
-    wopQueryDerivationOutputs = 22,
+    wopQueryDerivationOutputs = 22, // obsolete
     wopQueryAllValidPaths = 23,
     wopQueryFailedPaths = 24,
     wopClearFailedPaths = 25,
@@ -49,6 +49,7 @@ typedef enum {
     wopNarFromPath = 38,
     wopAddToStoreNar = 39,
     wopQueryMissing = 40,
+    wopQueryDerivationOutputMap = 41,
 } WorkerOp;
 
 
@@ -69,5 +70,6 @@ template<class T> T readStorePaths(const Store & store, Source & from);
 
 void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths);
 
+void writeOutputPathMap(const Store & store, Sink & out, const OutputPathMap & paths);
 
 }

src/libutil/ansicolor.hh

@@ -11,7 +11,7 @@ namespace nix {
 #define ANSI_GREEN "\e[32;1m"
 #define ANSI_YELLOW "\e[33;1m"
 #define ANSI_BLUE "\e[34;1m"
-#define ANSI_MAGENTA "\e[35m;1m"
-#define ANSI_CYAN "\e[36m;1m"
+#define ANSI_MAGENTA "\e[35;1m"
+#define ANSI_CYAN "\e[36;1m"
 
 }

src/libutil/archive.cc

@@ -262,7 +262,7 @@ static void parse(ParseSink & sink, Source & source, const Path & path)
                             names[name] = 0;
                     }
                 } else if (s == "node") {
-                    if (s.empty()) throw badArchive("entry name missing");
+                    if (name.empty()) throw badArchive("entry name missing");
                     parse(sink, source, path + "/" + name);
                 } else
                     throw badArchive("unknown field " + s);

src/libutil/archive.hh

@@ -63,11 +63,12 @@ struct ParseSink
     virtual void createSymlink(const Path & path, const string & target) { };
 };
 
-struct TeeSink : ParseSink
+struct TeeParseSink : ParseSink
 {
+    StringSink saved;
     TeeSource source;
 
-    TeeSink(Source & source) : source(source) { }
+    TeeParseSink(Source & source) : source(source, saved) { }
 };
 
 void parseDump(ParseSink & sink, Source & source);

src/libutil/error.cc

@@ -7,14 +7,11 @@
 
 namespace nix {
 
-
 const std::string nativeSystem = SYSTEM;
 
-// addPrefix is used for show-trace.  Strings added with addPrefix
-// will print ahead of the error itself.
-BaseError & BaseError::addPrefix(const FormatOrString & fs)
+BaseError & BaseError::addTrace(std::optional<ErrPos> e, hintformat hint)
 {
-    prefix_ = fs.s + prefix_;
+    err.traces.push_front(Trace { .pos = e, .hint = hint});
     return *this;
 }
 
@@ -28,7 +25,7 @@ const string& BaseError::calcWhat() const
         err.name = sname();
 
         std::ostringstream oss;
-        oss << err;
+        showErrorInfo(oss, err, false);
         what_ = oss.str();
 
         return *what_;
@@ -56,28 +53,114 @@ string showErrPos(const ErrPos &errPos)
     }
 }
 
-// if nixCode contains lines of code, print them to the ostream, indicating the error column.
-void printCodeLines(std::ostream &out, const string &prefix, const NixCode &nixCode)
+std::optional<LinesOfCode> getCodeLines(const ErrPos &errPos)
+{
+    if (errPos.line <= 0)
+        return std::nullopt;
+
+    if (errPos.origin == foFile) {
+        LinesOfCode loc;
+        try {
+            AutoCloseFD fd = open(errPos.file.c_str(), O_RDONLY | O_CLOEXEC);
+            if (!fd) {
+                logError(SysError("opening file '%1%'", errPos.file).info());
+                return std::nullopt;
+            }
+            else
+            {
+                // count the newlines.
+                int count = 0;
+                string line;
+                int pl = errPos.line - 1;
+                do
+                {
+                    line = readLine(fd.get());
+                    ++count;
+                    if (count < pl)
+                    {
+                        ;
+                    }
+                    else if (count == pl) {
+                        loc.prevLineOfCode = line;
+                    } else if (count == pl + 1) {
+                        loc.errLineOfCode = line;
+                    } else if (count == pl + 2) {
+                        loc.nextLineOfCode = line;
+                        break;
+                    }
+                } while (true);
+                return loc;
+            }
+        }
+        catch (EndOfFile &eof) {
+            if (loc.errLineOfCode.has_value())
+                return loc;
+            else
+                return std::nullopt;
+        }
+        catch (std::exception &e) {
+            printError("error reading nix file: %s\n%s", errPos.file, e.what());
+            return std::nullopt;
+        }
+    } else {
+        std::istringstream iss(errPos.file);
+        // count the newlines.
+        int count = 0;
+        string line;
+        int pl = errPos.line - 1;
+
+        LinesOfCode loc;
+
+        do
+        {
+            std::getline(iss, line);
+            ++count;
+            if (count < pl)
+            {
+                ;
+            }
+            else if (count == pl) {
+                loc.prevLineOfCode = line;
+            } else if (count == pl + 1) {
+                loc.errLineOfCode = line;
+            } else if (count == pl + 2) {
+                loc.nextLineOfCode = line;
+                break;
+            }
+
+            if (!iss.good())
+                break;
+        } while (true);
+
+        return loc;
+    }
+}
+
+// print lines of code to the ostream, indicating the error column.
+void printCodeLines(std::ostream &out,
+    const string &prefix,
+    const ErrPos &errPos,
+    const LinesOfCode &loc)
 {
     // previous line of code.
-    if (nixCode.prevLineOfCode.has_value()) {
+    if (loc.prevLineOfCode.has_value()) {
         out << std::endl
             << fmt("%1% %|2$5d|| %3%",
-                prefix,
-                (nixCode.errPos.line - 1),
-                *nixCode.prevLineOfCode);
+            prefix,
+            (errPos.line - 1),
+            *loc.prevLineOfCode);
     }
 
-    if (nixCode.errLineOfCode.has_value()) {
+    if (loc.errLineOfCode.has_value()) {
         // line of code containing the error.
         out << std::endl
             << fmt("%1% %|2$5d|| %3%",
-                prefix,
-                (nixCode.errPos.line),
-                *nixCode.errLineOfCode);
+            prefix,
+            (errPos.line),
+            *loc.errLineOfCode);
         // error arrows for the column range.
-        if (nixCode.errPos.column > 0) {
-            int start = nixCode.errPos.column;
+        if (errPos.column > 0) {
+            int start = errPos.column;
             std::string spaces;
             for (int i = 0; i < start; ++i) {
                 spaces.append(" ");
@@ -87,23 +170,49 @@ void printCodeLines(std::ostream &out, const string &prefix, const NixCode &nixC
 
             out << std::endl
                 << fmt("%1%      |%2%" ANSI_RED "%3%" ANSI_NORMAL,
-                    prefix,
-                    spaces,
-                    arrows);
+                prefix,
+                spaces,
+                arrows);
         }
     }
 
     // next line of code.
-    if (nixCode.nextLineOfCode.has_value()) {
+    if (loc.nextLineOfCode.has_value()) {
         out << std::endl
             << fmt("%1% %|2$5d|| %3%",
-                prefix,
-                (nixCode.errPos.line + 1),
-                *nixCode.nextLineOfCode);
+            prefix,
+            (errPos.line + 1),
+            *loc.nextLineOfCode);
     }
 }
 
-std::ostream& operator<<(std::ostream &out, const ErrorInfo &einfo)
+void printAtPos(const string &prefix, const ErrPos &pos, std::ostream &out)
+{
+    if (pos)
+    {
+        switch (pos.origin) {
+            case foFile: {
+                out << prefix << ANSI_BLUE << "at: " << ANSI_YELLOW << showErrPos(pos) <<
+                    ANSI_BLUE << " in file: " << ANSI_NORMAL << pos.file;
+                break;
+            }
+            case foString: {
+                out << prefix << ANSI_BLUE << "at: " << ANSI_YELLOW << showErrPos(pos) <<
+                    ANSI_BLUE << " from string" << ANSI_NORMAL;
+                break;
+            }
+            case foStdin: {
+                out << prefix << ANSI_BLUE << "at: " << ANSI_YELLOW << showErrPos(pos) <<
+                    ANSI_BLUE << " from stdin" << ANSI_NORMAL;
+                break;
+            }
+            default:
+                throw Error("invalid FileOrigin in errPos");
+        }
+    }
+}
+
+std::ostream& showErrorInfo(std::ostream &out, const ErrorInfo &einfo, bool showTrace)
 {
     auto errwidth = std::max<size_t>(getWindowSize().second, 20);
     string prefix = "";
@@ -158,8 +267,12 @@ std::ostream& operator<<(std::ostream &out, const ErrorInfo &einfo)
         }
     }
 
-    auto ndl = prefix.length() + levelString.length() + 3 + einfo.name.length() + einfo.programName.value_or("").length();
-    auto dashwidth = ndl > (errwidth - 3) ? 3 : errwidth - ndl;
+    auto ndl = prefix.length()
+        + filterANSIEscapes(levelString, true).length()
+        + 7
+        + einfo.name.length()
+        + einfo.programName.value_or("").length();
+    auto dashwidth = std::max<int>(errwidth - ndl, 3);
 
     std::string dashes(dashwidth, '-');
 
@@ -179,16 +292,9 @@ std::ostream& operator<<(std::ostream &out, const ErrorInfo &einfo)
             einfo.programName.value_or(""));
 
     bool nl = false;  // intersperse newline between sections.
-    if (einfo.nixCode.has_value()) {
-        if (einfo.nixCode->errPos.file != "") {
-            // filename, line, column.
-            out << std::endl << fmt("%1%in file: " ANSI_BLUE "%2% %3%" ANSI_NORMAL,
-                prefix,
-                einfo.nixCode->errPos.file,
-                showErrPos(einfo.nixCode->errPos));
-        } else {
-            out << std::endl << fmt("%1%from command line argument", prefix);
-        }
+    if (einfo.errPos.has_value() && (*einfo.errPos)) {
+        out << prefix << std::endl;
+        printAtPos(prefix, *einfo.errPos, out);
         nl = true;
     }
 
@@ -200,12 +306,16 @@ std::ostream& operator<<(std::ostream &out, const ErrorInfo &einfo)
         nl = true;
     }
 
-    // lines of code.
-    if (einfo.nixCode.has_value() && einfo.nixCode->errLineOfCode.has_value()) {
-        if (nl)
-            out << std::endl << prefix;
-        printCodeLines(out, prefix, *einfo.nixCode);
-        nl = true;
+    if (einfo.errPos.has_value() && (*einfo.errPos)) {
+        auto loc = getCodeLines(*einfo.errPos);
+
+        // lines of code.
+        if (loc.has_value()) {
+            if (nl)
+                out << std::endl << prefix;
+            printCodeLines(out, prefix, *einfo.errPos, *loc);
+            nl = true;
+        }
     }
 
     // hint
@@ -216,6 +326,54 @@ std::ostream& operator<<(std::ostream &out, const ErrorInfo &einfo)
         nl = true;
     }
 
+    // traces
+    if (showTrace && !einfo.traces.empty())
+    {
+        const string tracetitle(" show-trace ");
+
+        int fill = errwidth - tracetitle.length();
+        int lw = 0;
+        int rw = 0;
+        const int min_dashes = 3;
+        if (fill > min_dashes * 2) {
+            if (fill % 2 != 0) {
+                lw = fill / 2;
+                rw = lw + 1;
+            }
+            else
+            {
+                lw = rw = fill / 2;
+            }
+        }
+        else
+            lw = rw = min_dashes;
+
+        if (nl)
+            out << std::endl << prefix;
+
+        out << ANSI_BLUE << std::string(lw, '-') << tracetitle << std::string(rw, '-') << ANSI_NORMAL;
+
+        for (auto iter = einfo.traces.rbegin(); iter != einfo.traces.rend(); ++iter)
+        {
+            out << std::endl << prefix;
+            out << ANSI_BLUE << "trace: " << ANSI_NORMAL << iter->hint.str();
+
+            if (iter->pos.has_value() && (*iter->pos)) {
+                auto pos = iter->pos.value();
+                out << std::endl << prefix;
+                printAtPos(prefix, pos, out);
+
+                auto loc = getCodeLines(pos);
+                if (loc.has_value())
+                {
+                    out << std::endl << prefix;
+                    printCodeLines(out, prefix, pos, *loc);
+                    out << std::endl << prefix;
+                }
+            }
+        }
+    }
+
     return out;
 }
 }

src/libutil/error.hh

@@ -1,8 +1,8 @@
 #pragma once
 
-
 #include "ref.hh"
 #include "types.hh"
+#include "fmt.hh"
 
 #include <cstring>
 #include <list>
@@ -10,7 +10,9 @@
 #include <map>
 #include <optional>
 
-#include "fmt.hh"
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
 
 /* Before 4.7, gcc's std::exception uses empty throw() specifiers for
  * its (virtual) destructor and what() in c++11 mode, in violation of spec
@@ -25,20 +27,20 @@ namespace nix {
 
 /*
 
-This file defines two main structs/classes used in nix error handling.
+   This file defines two main structs/classes used in nix error handling.
 
-ErrorInfo provides a standard payload of error information, with conversion to string
-happening in the logger rather than at the call site.
+   ErrorInfo provides a standard payload of error information, with conversion to string
+   happening in the logger rather than at the call site.
 
-BaseError is the ancestor of nix specific exceptions (and Interrupted), and contains
-an ErrorInfo.
+   BaseError is the ancestor of nix specific exceptions (and Interrupted), and contains
+   an ErrorInfo.
 
-ErrorInfo structs are sent to the logger as part of an exception, or directly with the
-logError or logWarning macros.
+   ErrorInfo structs are sent to the logger as part of an exception, or directly with the
+   logError or logWarning macros.
 
-See the error-demo.cc program for usage examples.
+   See the error-demo.cc program for usage examples.
 
-*/
+ */
 
 typedef enum {
     lvlError = 0,
@@ -50,11 +52,25 @@ typedef enum {
     lvlVomit
 } Verbosity;
 
+typedef enum {
+    foFile,
+    foStdin,
+    foString
+} FileOrigin;
+
+// the lines of code surrounding an error.
+struct LinesOfCode {
+    std::optional<string> prevLineOfCode;
+    std::optional<string> errLineOfCode;
+    std::optional<string> nextLineOfCode;
+};
+
 // ErrPos indicates the location of an error in a nix file.
 struct ErrPos {
     int line = 0;
     int column = 0;
     string file;
+    FileOrigin origin;
 
     operator bool() const
     {
@@ -65,9 +81,14 @@ struct ErrPos {
     template <class P>
     ErrPos& operator=(const P &pos)
     {
+        origin = pos.origin;
         line = pos.line;
         column = pos.column;
-        file = pos.file;
+        // is file symbol null?
+        if (pos.file.set())
+            file = pos.file;
+        else
+            file = "";
         return *this;
     }
 
@@ -78,11 +99,9 @@ struct ErrPos {
     }
 };
 
-struct NixCode {
-    ErrPos errPos;
-    std::optional<string> prevLineOfCode;
-    std::optional<string> errLineOfCode;
-    std::optional<string> nextLineOfCode;
+struct Trace {
+    std::optional<ErrPos> pos;
+    hintformat hint;
 };
 
 struct ErrorInfo {
@@ -90,19 +109,19 @@ struct ErrorInfo {
     string name;
     string description;
     std::optional<hintformat> hint;
-    std::optional<NixCode> nixCode;
+    std::optional<ErrPos> errPos;
+    std::list<Trace> traces;
 
     static std::optional<string> programName;
 };
 
-std::ostream& operator<<(std::ostream &out, const ErrorInfo &einfo);
+std::ostream& showErrorInfo(std::ostream &out, const ErrorInfo &einfo, bool showTrace);
 
 /* BaseError should generally not be caught, as it has Interrupted as
    a subclass. Catch Error instead. */
 class BaseError : public std::exception
 {
 protected:
-    string prefix_; // used for location traces etc.
     mutable ErrorInfo err;
 
     mutable std::optional<string> what_;
@@ -113,23 +132,23 @@ public:
 
     template<typename... Args>
     BaseError(unsigned int status, const Args & ... args)
-        : err { .level = lvlError,
-                .hint = hintfmt(args...)
-              }
+        : err {.level = lvlError,
+            .hint = hintfmt(args...)
+            }
         , status(status)
     { }
 
     template<typename... Args>
     BaseError(const std::string & fs, const Args & ... args)
-        : err { .level = lvlError,
-                .hint = hintfmt(fs, args...)
-              }
+        : err {.level = lvlError,
+            .hint = hintfmt(fs, args...)
+            }
     { }
 
     BaseError(hintformat hint)
-        : err { .level = lvlError,
-                .hint = hint
-              }
+        : err {.level = lvlError,
+            .hint = hint
+            }
     { }
 
     BaseError(ErrorInfo && e)
@@ -150,10 +169,17 @@ public:
 #endif
 
     const string & msg() const { return calcWhat(); }
-    const string & prefix() const { return prefix_; }
-    BaseError & addPrefix(const FormatOrString & fs);
-
     const ErrorInfo & info() { calcWhat(); return err; }
+
+    template<typename... Args>
+    BaseError & addTrace(std::optional<ErrPos> e, const string &fs, const Args & ... args)
+    {
+        return addTrace(e, hintfmt(fs, args...));
+    }
+
+    BaseError & addTrace(std::optional<ErrPos> e, hintformat hint);
+
+    bool hasTrace() const { return !err.traces.empty(); }
 };
 
 #define MakeError(newClass, superClass) \
@@ -173,7 +199,7 @@ public:
 
     template<typename... Args>
     SysError(const Args & ... args)
-      :Error("")
+        : Error("")
     {
         errNo = errno;
         auto hf = hintfmt(args...);

src/libutil/hash.cc

@@ -8,7 +8,6 @@
 #include "hash.hh"
 #include "archive.hh"
 #include "util.hh"
-#include "istringstream_nocopy.hh"
 
 #include <sys/types.h>
 #include <sys/stat.h>
@@ -370,7 +369,7 @@ string printHashType(HashType ht)
     default:
         // illegal hash type enum value internally, as opposed to external input
         // which should be validated with nice error message.
-        abort();
+        assert(false);
     }
 }
 

src/libutil/istringstream_nocopy.hh

@@ -1,92 +0,0 @@
-/* This file provides a variant of std::istringstream that doesn't
-   copy its string argument. This is useful for large strings. The
-   caller must ensure that the string object is not destroyed while
-   it's referenced by this object. */
-
-#pragma once
-
-#include <string>
-#include <iostream>
-
-template <class CharT, class Traits = std::char_traits<CharT>, class Allocator = std::allocator<CharT>>
-class basic_istringbuf_nocopy : public std::basic_streambuf<CharT, Traits>
-{
-public:
-    typedef std::basic_string<CharT, Traits, Allocator> string_type;
-
-    typedef typename std::basic_streambuf<CharT, Traits>::off_type off_type;
-
-    typedef typename std::basic_streambuf<CharT, Traits>::pos_type pos_type;
-
-    typedef typename std::basic_streambuf<CharT, Traits>::int_type int_type;
-
-    typedef typename std::basic_streambuf<CharT, Traits>::traits_type traits_type;
-
-private:
-    const string_type & s;
-
-    off_type off;
-
-public:
-    basic_istringbuf_nocopy(const string_type & s) : s{s}, off{0}
-    {
-    }
-
-private:
-    pos_type seekoff(off_type off, std::ios_base::seekdir dir, std::ios_base::openmode which)
-    {
-        if (which & std::ios_base::in) {
-            this->off = dir == std::ios_base::beg
-                ? off
-                : (dir == std::ios_base::end
-                    ? s.size() + off
-                    : this->off + off);
-        }
-        return pos_type(this->off);
-    }
-
-    pos_type seekpos(pos_type pos, std::ios_base::openmode which)
-    {
-        return seekoff(pos, std::ios_base::beg, which);
-    }
-
-    std::streamsize showmanyc()
-    {
-        return s.size() - off;
-    }
-
-    int_type underflow()
-    {
-        if (typename string_type::size_type(off) == s.size())
-            return traits_type::eof();
-        return traits_type::to_int_type(s[off]);
-    }
-
-    int_type uflow()
-    {
-        if (typename string_type::size_type(off) == s.size())
-            return traits_type::eof();
-        return traits_type::to_int_type(s[off++]);
-    }
-
-    int_type pbackfail(int_type ch)
-    {
-        if (off == 0 || (ch != traits_type::eof() && ch != s[off - 1]))
-            return traits_type::eof();
-
-        return traits_type::to_int_type(s[--off]);
-    }
-
-};
-
-template <class CharT, class Traits = std::char_traits<CharT>, class Allocator = std::allocator<CharT>>
-class basic_istringstream_nocopy : public std::basic_iostream<CharT, Traits>
-{
-    typedef basic_istringbuf_nocopy<CharT, Traits, Allocator> buf_type;
-    buf_type buf;
-public:
-    basic_istringstream_nocopy(const typename buf_type::string_type & s) :
-        std::basic_iostream<CharT, Traits>(&buf), buf(s) {};
-};
-
-typedef basic_istringstream_nocopy<char> istringstream_nocopy;

src/libutil/logging.cc

@@ -1,5 +1,6 @@
 #include "logging.hh"
 #include "util.hh"
+#include "config.hh"
 
 #include <atomic>
 #include <nlohmann/json.hpp>
@@ -7,6 +8,10 @@
 
 namespace nix {
 
+LoggerSettings loggerSettings;
+
+static GlobalConfig::Register r1(&loggerSettings);
+
 static thread_local ActivityId curActivity = 0;
 
 ActivityId getCurActivity()
@@ -72,7 +77,7 @@ public:
     void logEI(const ErrorInfo & ei) override
     {
         std::stringstream oss;
-        oss << ei;
+        showErrorInfo(oss, ei, loggerSettings.showTrace.get());
 
         log(ei.level, oss.str());
     }
@@ -173,7 +178,7 @@ struct JSONLogger : Logger {
     void logEI(const ErrorInfo & ei) override
     {
         std::ostringstream oss;
-        oss << ei;
+        showErrorInfo(oss, ei, loggerSettings.showTrace.get());
 
         nlohmann::json json;
         json["action"] = "msg";

src/libutil/logging.hh

@@ -2,6 +2,7 @@
 
 #include "types.hh"
 #include "error.hh"
+#include "config.hh"
 
 namespace nix {
 
@@ -34,6 +35,16 @@ typedef enum {
 
 typedef uint64_t ActivityId;
 
+struct LoggerSettings : Config
+{
+    Setting<bool> showTrace{this,
+        false,
+        "show-trace",
+        "Whether to show a stack trace on evaluation errors."};
+};
+
+extern LoggerSettings loggerSettings;
+
 class Logger
 {
     friend struct Activity;

src/libutil/serialise.hh

@@ -166,17 +166,30 @@ struct StringSource : Source
 };
 
 
-/* Adapter class of a Source that saves all data read to `s'. */
+/* A sink that writes all incoming data to two other sinks. */
+struct TeeSink : Sink
+{
+    Sink & sink1, & sink2;
+    TeeSink(Sink & sink1, Sink & sink2) : sink1(sink1), sink2(sink2) { }
+    virtual void operator () (const unsigned char * data, size_t len)
+    {
+        sink1(data, len);
+        sink2(data, len);
+    }
+};
+
+
+/* Adapter class of a Source that saves all data read to a sink. */
 struct TeeSource : Source
 {
     Source & orig;
-    ref<std::string> data;
-    TeeSource(Source & orig)
-        : orig(orig), data(make_ref<std::string>()) { }
+    Sink & sink;
+    TeeSource(Source & orig, Sink & sink)
+        : orig(orig), sink(sink) { }
     size_t read(unsigned char * data, size_t len)
     {
         size_t n = orig.read(data, len);
-        this->data->append((const char *) data, n);
+        sink(data, len);
         return n;
     }
 };
@@ -336,4 +349,27 @@ Source & operator >> (Source & in, bool & b)
 }
 
 
+/* An adapter that converts a std::basic_istream into a source. */
+struct StreamToSourceAdapter : Source
+{
+    std::shared_ptr<std::basic_istream<char>> istream;
+
+    StreamToSourceAdapter(std::shared_ptr<std::basic_istream<char>> istream)
+        : istream(istream)
+    { }
+
+    size_t read(unsigned char * data, size_t len) override
+    {
+        if (!istream->read((char *) data, len)) {
+            if (istream->eof()) {
+                if (istream->gcount() == 0)
+                    throw EndOfFile("end of file");
+            } else
+                throw Error("I/O error in StreamToSourceAdapter");
+        }
+        return istream->gcount();
+    }
+};
+
+
 }

src/libutil/tests/logging.cc

@@ -11,6 +11,13 @@ namespace nix {
      * logEI
      * --------------------------------------------------------------------------*/
 
+    const char *test_file =
+        "previous line of code\n"
+        "this is the problem line of code\n"
+        "next line of code\n";
+    const char *one_liner =
+        "this is the other problem line of code";
+
     TEST(logEI, catpuresBasicProperties) {
 
         MakeError(TestError, Error);
@@ -137,7 +144,6 @@ namespace nix {
      * logError
      * --------------------------------------------------------------------------*/
 
-
     TEST(logError, logErrorWithoutHintOrCode) {
         testing::internal::CaptureStderr();
 
@@ -152,7 +158,7 @@ namespace nix {
 
     TEST(logError, logErrorWithPreviousAndNextLinesOfCode) {
         SymbolTable testTable;
-        auto problem_file = testTable.create("myfile.nix");
+        auto problem_file = testTable.create(test_file);
 
         testing::internal::CaptureStderr();
 
@@ -162,21 +168,16 @@ namespace nix {
                 .hint = hintfmt("this hint has %1% templated %2%!!",
                     "yellow",
                     "values"),
-                .nixCode = NixCode {
-                    .errPos = Pos(problem_file, 40, 13),
-                    .prevLineOfCode = "previous line of code",
-                    .errLineOfCode = "this is the problem line of code",
-                    .nextLineOfCode = "next line of code",
-                }});
-
+                .errPos = Pos(foString, problem_file, 02, 13),
+            });
 
         auto str = testing::internal::GetCapturedStderr();
-        ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- error name --- error-unit-test\x1B[0m\nin file: \x1B[34;1mmyfile.nix (40:13)\x1B[0m\n\nerror with code lines\n\n    39| previous line of code\n    40| this is the problem line of code\n      |             \x1B[31;1m^\x1B[0m\n    41| next line of code\n\nthis hint has \x1B[33;1myellow\x1B[0m templated \x1B[33;1mvalues\x1B[0m!!\n");
+        ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- error name --- error-unit-test\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(2:13)\x1B[34;1m from string\x1B[0m\n\nerror with code lines\n\n     1| previous line of code\n     2| this is the problem line of code\n      |             \x1B[31;1m^\x1B[0m\n     3| next line of code\n\nthis hint has \x1B[33;1myellow\x1B[0m templated \x1B[33;1mvalues\x1B[0m!!\n");
     }
 
-    TEST(logError, logErrorWithoutLinesOfCode) {
+    TEST(logError, logErrorWithInvalidFile) {
         SymbolTable testTable;
-        auto problem_file = testTable.create("myfile.nix");
+        auto problem_file = testTable.create("invalid filename");
         testing::internal::CaptureStderr();
 
         logError({
@@ -185,28 +186,23 @@ namespace nix {
                 .hint = hintfmt("this hint has %1% templated %2%!!",
                     "yellow",
                     "values"),
-                .nixCode = NixCode {
-                    .errPos = Pos(problem_file, 40, 13)
-                }});
+                .errPos = Pos(foFile, problem_file, 02, 13)
+            });
 
         auto str = testing::internal::GetCapturedStderr();
-        ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- error name --- error-unit-test\x1B[0m\nin file: \x1B[34;1mmyfile.nix (40:13)\x1B[0m\n\nerror without any code lines.\n\nthis hint has \x1B[33;1myellow\x1B[0m templated \x1B[33;1mvalues\x1B[0m!!\n");
+        ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- SysError --- error-unit-test\x1B[0m\nopening file '\x1B[33;1minvalid filename\x1B[0m': \x1B[33;1mNo such file or directory\x1B[0m\n\x1B[31;1merror:\x1B[0m\x1B[34;1m --- error name --- error-unit-test\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(2:13)\x1B[34;1m in file: \x1B[0minvalid filename\n\nerror without any code lines.\n\nthis hint has \x1B[33;1myellow\x1B[0m templated \x1B[33;1mvalues\x1B[0m!!\n");
     }
 
     TEST(logError, logErrorWithOnlyHintAndName) {
-        SymbolTable testTable;
-        auto problem_file = testTable.create("myfile.nix");
         testing::internal::CaptureStderr();
 
         logError({
                 .name = "error name",
                 .hint = hintfmt("hint %1%", "only"),
-                .nixCode = NixCode {
-                    .errPos = Pos(problem_file, 40, 13)
-                }});
+            });
 
         auto str = testing::internal::GetCapturedStderr();
-        ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- error name --- error-unit-test\x1B[0m\nin file: \x1B[34;1mmyfile.nix (40:13)\x1B[0m\n\nhint \x1B[33;1monly\x1B[0m\n");
+        ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- error name --- error-unit-test\x1B[0m\nhint \x1B[33;1monly\x1B[0m\n");
 
     }
 
@@ -219,18 +215,18 @@ namespace nix {
 
         logWarning({
                 .name = "name",
-                .description = "error description",
+                .description = "warning description",
                 .hint = hintfmt("there was a %1%", "warning"),
             });
 
         auto str = testing::internal::GetCapturedStderr();
-        ASSERT_STREQ(str.c_str(), "\x1B[33;1mwarning:\x1B[0m\x1B[34;1m --- name --- error-unit-test\x1B[0m\nerror description\n\nthere was a \x1B[33;1mwarning\x1B[0m\n");
+        ASSERT_STREQ(str.c_str(), "\x1B[33;1mwarning:\x1B[0m\x1B[34;1m --- name --- error-unit-test\x1B[0m\nwarning description\n\nthere was a \x1B[33;1mwarning\x1B[0m\n");
     }
 
     TEST(logWarning, logWarningWithFileLineNumAndCode) {
 
         SymbolTable testTable;
-        auto problem_file = testTable.create("myfile.nix");
+        auto problem_file = testTable.create(test_file);
 
         testing::internal::CaptureStderr();
 
@@ -240,18 +236,73 @@ namespace nix {
                 .hint = hintfmt("this hint has %1% templated %2%!!",
                     "yellow",
                     "values"),
-                .nixCode = NixCode {
-                    .errPos = Pos(problem_file, 40, 13),
-                    .prevLineOfCode = std::nullopt,
-                    .errLineOfCode = "this is the problem line of code",
-                    .nextLineOfCode = std::nullopt
-                }});
+                .errPos = Pos(foStdin, problem_file, 2, 13),
+            });
 
 
         auto str = testing::internal::GetCapturedStderr();
-        ASSERT_STREQ(str.c_str(), "\x1B[33;1mwarning:\x1B[0m\x1B[34;1m --- warning name --- error-unit-test\x1B[0m\nin file: \x1B[34;1mmyfile.nix (40:13)\x1B[0m\n\nwarning description\n\n    40| this is the problem line of code\n      |             \x1B[31;1m^\x1B[0m\n\nthis hint has \x1B[33;1myellow\x1B[0m templated \x1B[33;1mvalues\x1B[0m!!\n");
+        ASSERT_STREQ(str.c_str(), "\x1B[33;1mwarning:\x1B[0m\x1B[34;1m --- warning name --- error-unit-test\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(2:13)\x1B[34;1m from stdin\x1B[0m\n\nwarning description\n\n     1| previous line of code\n     2| this is the problem line of code\n      |             \x1B[31;1m^\x1B[0m\n     3| next line of code\n\nthis hint has \x1B[33;1myellow\x1B[0m templated \x1B[33;1mvalues\x1B[0m!!\n");
     }
 
+    /* ----------------------------------------------------------------------------
+     * traces
+     * --------------------------------------------------------------------------*/
+
+    TEST(addTrace, showTracesWithShowTrace) {
+        SymbolTable testTable;
+        auto problem_file = testTable.create(test_file);
+        auto oneliner_file = testTable.create(one_liner);
+        auto invalidfilename = testTable.create("invalid filename");
+
+        auto e = AssertionError(ErrorInfo {
+                .name = "wat",
+                .description = "show-traces",
+                .hint = hintfmt("it has been %1% days since our last error", "zero"),
+                .errPos = Pos(foString, problem_file, 2, 13),
+            });
+
+        e.addTrace(Pos(foStdin, oneliner_file, 1, 19), "while trying to compute %1%", 42);
+        e.addTrace(std::nullopt, "while doing something without a %1%", "pos");
+        e.addTrace(Pos(foFile, invalidfilename, 100, 1), "missing %s", "nix file");
+
+        testing::internal::CaptureStderr();
+
+        loggerSettings.showTrace.assign(true);
+
+        logError(e.info());
+
+        auto str = testing::internal::GetCapturedStderr();
+        ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- SysError --- error-unit-test\x1B[0m\nopening file '\x1B[33;1minvalid filename\x1B[0m': \x1B[33;1mNo such file or directory\x1B[0m\n\x1B[31;1merror:\x1B[0m\x1B[34;1m --- AssertionError --- error-unit-test\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(2:13)\x1B[34;1m from string\x1B[0m\n\nshow-traces\n\n     1| previous line of code\n     2| this is the problem line of code\n      |             \x1B[31;1m^\x1B[0m\n     3| next line of code\n\nit has been \x1B[33;1mzero\x1B[0m days since our last error\n\x1B[34;1m---- show-trace ----\x1B[0m\n\x1B[34;1mtrace: \x1B[0mwhile trying to compute \x1B[33;1m42\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(1:19)\x1B[34;1m from stdin\x1B[0m\n\n     1| this is the other problem line of code\n      |                   \x1B[31;1m^\x1B[0m\n\n\x1B[34;1mtrace: \x1B[0mwhile doing something without a \x1B[33;1mpos\x1B[0m\n\x1B[34;1mtrace: \x1B[0mmissing \x1B[33;1mnix file\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(100:1)\x1B[34;1m in file: \x1B[0minvalid filename\n");
+    }
+
+    TEST(addTrace, hideTracesWithoutShowTrace) {
+        SymbolTable testTable;
+        auto problem_file = testTable.create(test_file);
+        auto oneliner_file = testTable.create(one_liner);
+        auto invalidfilename = testTable.create("invalid filename");
+
+        auto e = AssertionError(ErrorInfo {
+                .name = "wat",
+                .description = "hide traces",
+                .hint = hintfmt("it has been %1% days since our last error", "zero"),
+                .errPos = Pos(foString, problem_file, 2, 13),
+            });
+
+        e.addTrace(Pos(foStdin, oneliner_file, 1, 19), "while trying to compute %1%", 42);
+        e.addTrace(std::nullopt, "while doing something without a %1%", "pos");
+        e.addTrace(Pos(foFile, invalidfilename, 100, 1), "missing %s", "nix file");
+
+        testing::internal::CaptureStderr();
+
+        loggerSettings.showTrace.assign(false);
+
+        logError(e.info());
+
+        auto str = testing::internal::GetCapturedStderr();
+        ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- AssertionError --- error-unit-test\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(2:13)\x1B[34;1m from string\x1B[0m\n\nhide traces\n\n     1| previous line of code\n     2| this is the problem line of code\n      |             \x1B[31;1m^\x1B[0m\n     3| next line of code\n\nit has been \x1B[33;1mzero\x1B[0m days since our last error\n");
+    }
+
+
     /* ----------------------------------------------------------------------------
      * hintfmt
      * --------------------------------------------------------------------------*/
@@ -289,4 +340,22 @@ namespace nix {
             "what about this " ANSI_YELLOW "%3%" ANSI_NORMAL " " ANSI_YELLOW "one" ANSI_NORMAL);
 
     }
+
+    /* ----------------------------------------------------------------------------
+     * ErrPos
+     * --------------------------------------------------------------------------*/
+
+    TEST(errpos, invalidPos) {
+
+      // contains an invalid symbol, which we should not dereference!
+      Pos invalid;
+
+      // constructing without access violation.
+      ErrPos ep(invalid);
+    
+      // assignment without access violation.
+      ep = invalid;
+
+    }
+
 }

src/nix-env/nix-env.cc

@@ -593,7 +593,7 @@ static void upgradeDerivations(Globals & globals,
                 } else newElems.push_back(i);
 
             } catch (Error & e) {
-                e.addPrefix(fmt("while trying to find an upgrade for '%s':\n", i.queryName()));
+                e.addTrace(std::nullopt, "while trying to find an upgrade for '%s'", i.queryName());
                 throw;
             }
         }
@@ -1185,7 +1185,7 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs)
         } catch (AssertionError & e) {
             printMsg(lvlTalkative, "skipping derivation named '%1%' which gives an assertion failure", i.queryName());
         } catch (Error & e) {
-            e.addPrefix(fmt("while querying the derivation named '%1%':\n", i.queryName()));
+            e.addTrace(std::nullopt, "while querying the derivation named '%1%'", i.queryName());
             throw;
         }
     }

src/nix-prefetch-url/nix-prefetch-url.cc

@@ -153,14 +153,15 @@ static int _main(int argc, char * * argv)
 
         /* If an expected hash is given, the file may already exist in
            the store. */
-        Hash hash, expectedHash(ht);
+        std::optional<Hash> expectedHash;
+        Hash hash;
         std::optional<StorePath> storePath;
         if (args.size() == 2) {
             expectedHash = Hash(args[1], ht);
             const auto recursive = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
-            storePath = store->makeFixedOutputPath(recursive, expectedHash, name);
+            storePath = store->makeFixedOutputPath(recursive, *expectedHash, name);
             if (store->isValidPath(*storePath))
-                hash = expectedHash;
+                hash = *expectedHash;
             else
                 storePath.reset();
         }
@@ -200,22 +201,12 @@ static int _main(int argc, char * * argv)
                     tmpFile = unpacked;
             }
 
-            /* FIXME: inefficient; addToStore() will also hash
-               this. */
-            hash = unpack ? hashPath(ht, tmpFile).first : hashFile(ht, tmpFile);
+            const auto method = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
 
-            if (expectedHash != Hash(ht) && expectedHash != hash)
-                throw Error("hash mismatch for '%1%'", uri);
-
-            const auto recursive = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
-
-            /* Copy the file to the Nix store. FIXME: if RemoteStore
-               implemented addToStoreFromDump() and downloadFile()
-               supported a sink, we could stream the download directly
-               into the Nix store. */
-            storePath = store->addToStore(name, tmpFile, recursive, ht);
-
-            assert(*storePath == store->makeFixedOutputPath(recursive, hash, name));
+            auto info = store->addToStoreSlow(name, tmpFile, method, ht, expectedHash);
+            storePath = info.path;
+            assert(info.ca);
+            hash = getContentAddressHash(*info.ca);
         }
 
         stopProgressBar();

src/nix-store/nix-store.cc

@@ -174,10 +174,10 @@ static void opAdd(Strings opFlags, Strings opArgs)
    store. */
 static void opAddFixed(Strings opFlags, Strings opArgs)
 {
-    auto recursive = FileIngestionMethod::Flat;
+    auto method = FileIngestionMethod::Flat;
 
     for (auto & i : opFlags)
-        if (i == "--recursive") recursive = FileIngestionMethod::Recursive;
+        if (i == "--recursive") method = FileIngestionMethod::Recursive;
         else throw UsageError("unknown flag '%1%'", i);
 
     if (opArgs.empty())
@@ -187,7 +187,7 @@ static void opAddFixed(Strings opFlags, Strings opArgs)
     opArgs.pop_front();
 
     for (auto & i : opArgs)
-        cout << fmt("%s\n", store->printStorePath(store->addToStore(std::string(baseNameOf(i)), i, recursive, hashAlgo)));
+        std::cout << fmt("%s\n", store->printStorePath(store->addToStoreSlow(baseNameOf(i), i, method, hashAlgo).path));
 }
 
 
@@ -671,7 +671,7 @@ static void opImport(Strings opFlags, Strings opArgs)
     if (!opArgs.empty()) throw UsageError("no arguments expected");
 
     FdSource source(STDIN_FILENO);
-    auto paths = store->importPaths(source, nullptr, NoCheckSigs);
+    auto paths = store->importPaths(source, NoCheckSigs);
 
     for (auto & i : paths)
         cout << fmt("%s\n", store->printStorePath(i)) << std::flush;
@@ -878,7 +878,7 @@ static void opServe(Strings opFlags, Strings opArgs)
 
             case cmdImportPaths: {
                 if (!writeAllowed) throw Error("importing paths is not allowed");
-                store->importPaths(in, nullptr, NoCheckSigs); // FIXME: should we skip sig checking?
+                store->importPaths(in, NoCheckSigs); // FIXME: should we skip sig checking?
                 out << 1; // indicate success
                 break;
             }

src/nix/develop.cc

@@ -50,7 +50,7 @@ BuildEnvironment readEnvironment(const Path & path)
         R"re((?:\$?'(?:[^'\\]|\\[abeEfnrtv\\'"?])*'))re";
 
     static std::string indexedArrayRegex =
-        R"re((?:\(( *\[[0-9]+]="(?:[^"\\]|\\.)*")**\)))re";
+        R"re((?:\(( *\[[0-9]+\]="(?:[^"\\]|\\.)*")*\)))re";
 
     static std::regex varRegex(
         "^(" + varNameRegex + ")=(" + simpleStringRegex + "|" + quotedStringRegex + "|" + indexedArrayRegex + ")\n");
@@ -135,13 +135,7 @@ StorePath getDerivationEnvironment(ref<Store> store, const StorePath & drvPath)
     drv.inputSrcs.insert(std::move(getEnvShPath));
     Hash h = hashDerivationModulo(*store, drv, true);
     auto shellOutPath = store->makeOutputPath("out", h, drvName);
-    drv.outputs.insert_or_assign("out", DerivationOutput {
-        .path = shellOutPath,
-        .hash = FixedOutputHash {
-            .method = FileIngestionMethod::Flat,
-            .hash = Hash { },
-        },
-    });
+    drv.outputs.insert_or_assign("out", DerivationOutput { .path = shellOutPath });
     drv.env["out"] = store->printStorePath(shellOutPath);
     auto shellDrvPath2 = writeDerivation(store, drv, drvName);
 

src/nix/diff-closures.cc

@@ -0,0 +1,134 @@
+#include "command.hh"
+#include "shared.hh"
+#include "store-api.hh"
+#include "common-args.hh"
+#include "names.hh"
+
+#include <regex>
+
+using namespace nix;
+
+struct Info
+{
+    std::string outputName;
+};
+
+// name -> version -> store paths
+typedef std::map<std::string, std::map<std::string, std::map<StorePath, Info>>> GroupedPaths;
+
+GroupedPaths getClosureInfo(ref<Store> store, const StorePath & toplevel)
+{
+    StorePathSet closure;
+    store->computeFSClosure({toplevel}, closure);
+
+    GroupedPaths groupedPaths;
+
+    for (auto & path : closure) {
+        /* Strip the output name. Unfortunately this is ambiguous (we
+           can't distinguish between output names like "bin" and
+           version suffixes like "unstable"). */
+        static std::regex regex("(.*)-([a-z]+|lib32|lib64)");
+        std::smatch match;
+        std::string name(path.name());
+        std::string outputName;
+        if (std::regex_match(name, match, regex)) {
+            name = match[1];
+            outputName = match[2];
+        }
+
+        DrvName drvName(name);
+        groupedPaths[drvName.name][drvName.version].emplace(path, Info { .outputName = outputName });
+    }
+
+    return groupedPaths;
+}
+
+std::string showVersions(const std::set<std::string> & versions)
+{
+    if (versions.empty()) return "∅";
+    std::set<std::string> versions2;
+    for (auto & version : versions)
+        versions2.insert(version.empty() ? "ε" : version);
+    return concatStringsSep(", ", versions2);
+}
+
+struct CmdDiffClosures : SourceExprCommand
+{
+    std::string _before, _after;
+
+    CmdDiffClosures()
+    {
+        expectArg("before", &_before);
+        expectArg("after", &_after);
+    }
+
+    std::string description() override
+    {
+        return "show what packages and versions were added and removed between two closures";
+    }
+
+    Category category() override { return catSecondary; }
+
+    Examples examples() override
+    {
+        return {
+            {
+                "To show what got added and removed between two versions of the NixOS system profile:",
+                "nix diff-closures /nix/var/nix/profiles/system-655-link /nix/var/nix/profiles/system-658-link",
+            },
+        };
+    }
+
+    void run(ref<Store> store) override
+    {
+        auto before = parseInstallable(*this, store, _before, false);
+        auto beforePath = toStorePath(store, Build, before);
+        auto after = parseInstallable(*this, store, _after, false);
+        auto afterPath = toStorePath(store, NoBuild, after);
+
+        auto beforeClosure = getClosureInfo(store, beforePath);
+        auto afterClosure = getClosureInfo(store, afterPath);
+
+        std::set<std::string> allNames;
+        for (auto & [name, _] : beforeClosure) allNames.insert(name);
+        for (auto & [name, _] : afterClosure) allNames.insert(name);
+
+        for (auto & name : allNames) {
+            auto & beforeVersions = beforeClosure[name];
+            auto & afterVersions = afterClosure[name];
+
+            auto totalSize = [&](const std::map<std::string, std::map<StorePath, Info>> & versions)
+            {
+                uint64_t sum = 0;
+                for (auto & [_, paths] : versions)
+                    for (auto & [path, _] : paths)
+                        sum += store->queryPathInfo(path)->narSize;
+                return sum;
+            };
+
+            auto beforeSize = totalSize(beforeVersions);
+            auto afterSize = totalSize(afterVersions);
+            auto sizeDelta = (int64_t) afterSize - (int64_t) beforeSize;
+            auto showDelta = abs(sizeDelta) >= 8 * 1024;
+
+            std::set<std::string> removed, unchanged;
+            for (auto & [version, _] : beforeVersions)
+                if (!afterVersions.count(version)) removed.insert(version); else unchanged.insert(version);
+
+            std::set<std::string> added;
+            for (auto & [version, _] : afterVersions)
+                if (!beforeVersions.count(version)) added.insert(version);
+
+            if (showDelta || !removed.empty() || !added.empty()) {
+                std::vector<std::string> items;
+                if (!removed.empty() || !added.empty())
+                    items.push_back(fmt("%s → %s", showVersions(removed), showVersions(added)));
+                if (showDelta)
+                    items.push_back(fmt("%s%+.1f KiB" ANSI_NORMAL, sizeDelta > 0 ? ANSI_RED : ANSI_GREEN, sizeDelta / 1024.0));
+                std::cout << fmt("%s: %s\n", name, concatStringsSep(", ", items));
+            }
+        }
+    }
+};
+
+static auto r1 = registerCommand<CmdDiffClosures>("diff-closures");

src/nix/installables.cc

@@ -94,8 +94,8 @@ struct InstallableStorePath : Installable
     ref<Store> store;
     StorePath storePath;
 
-    InstallableStorePath(ref<Store> store, const Path & storePath)
-        : store(store), storePath(store->parseStorePath(storePath)) { }
+    InstallableStorePath(ref<Store> store, StorePath && storePath)
+        : store(store), storePath(std::move(storePath)) { }
 
     std::string what() override { return store->printStorePath(storePath); }
 
@@ -228,11 +228,11 @@ static std::vector<std::shared_ptr<Installable>> parseInstallables(
             result.push_back(std::make_shared<InstallableExpr>(cmd, s));
 
         else if (s.find("/") != std::string::npos) {
-
-            auto path = store->toStorePath(store->followLinksToStore(s));
-
-            if (store->isStorePath(path))
-                result.push_back(std::make_shared<InstallableStorePath>(store, path));
+            try {
+                result.push_back(std::make_shared<InstallableStorePath>(
+                        store,
+                        store->toStorePath(store->followLinksToStore(s)).first));
+            } catch (BadStorePath &) { }
         }
 
         else if (s == "" || std::regex_match(s, attrPathRegex))

src/nix/repl.cc

@@ -211,12 +211,12 @@ void NixRepl::mainLoop(const std::vector<std::string> & files)
                 // input without clearing the input so far.
                 continue;
             } else {
-              printMsg(lvlError, error + "%1%%2%", (settings.showTrace ? e.prefix() : ""), e.msg());
+              printMsg(lvlError, e.msg());
             }
         } catch (Error & e) {
-            printMsg(lvlError, error + "%1%%2%", (settings.showTrace ? e.prefix() : ""), e.msg());
+          printMsg(lvlError, e.msg());
         } catch (Interrupted & e) {
-            printMsg(lvlError, error + "%1%%2%", (settings.showTrace ? e.prefix() : ""), e.msg());
+          printMsg(lvlError, e.msg());
         }
 
         // We handled the current input fully, so we should clear it

src/nix/search.cc

@@ -216,7 +216,7 @@ struct CmdSearch : SourceExprCommand, MixJSON
             } catch (AssertionError & e) {
             } catch (Error & e) {
                 if (!toplevel) {
-                    e.addPrefix(fmt("While evaluating the attribute '%s':\n", attrPath));
+                    e.addTrace(std::nullopt, "While evaluating the attribute '%s'", attrPath);
                     throw;
                 }
             }

src/nix/verify.cc

@@ -77,13 +77,16 @@ struct CmdVerify : StorePathsCommand
             try {
                 checkInterrupt();
 
-                Activity act2(*logger, lvlInfo, actUnknown, fmt("checking '%s'", storePath));
-
                 MaintainCount<std::atomic<size_t>> mcActive(active);
                 update();
 
                 auto info = store->queryPathInfo(store->parseStorePath(storePath));
 
+                // Note: info->path can be different from storePath
+                // for binary cache stores when using --all (since we
+                // can't enumerate names efficiently).
+                Activity act2(*logger, lvlInfo, actUnknown, fmt("checking '%s'", store->printStorePath(info->path)));
+
                 if (!noContents) {
 
                     std::unique_ptr<AbstractHashSink> hashSink;

tests/binary-cache.sh

@@ -182,3 +182,56 @@ clearCacheCache
 nix-store -r $outPath --substituters "file://$cacheDir2 file://$cacheDir" --trusted-public-keys "$publicKey"
 
 fi # HAVE_LIBSODIUM
+
+
+unset _NIX_FORCE_HTTP
+
+
+# Test 'nix verify --all' on a binary cache.
+nix verify -vvvvv --all --store file://$cacheDir --no-trust
+
+
+# Test local NAR caching.
+narCache=$TEST_ROOT/nar-cache
+rm -rf $narCache
+mkdir $narCache
+
+[[ $(nix cat-store --store "file://$cacheDir?local-nar-cache=$narCache" $outPath/foobar) = FOOBAR ]]
+
+rm -rfv "$cacheDir/nar"
+
+[[ $(nix cat-store --store "file://$cacheDir?local-nar-cache=$narCache" $outPath/foobar) = FOOBAR ]]
+
+(! nix cat-store --store file://$cacheDir $outPath/foobar)
+
+
+# Test NAR listing generation.
+clearCache
+
+outPath=$(nix-build --no-out-link -E '
+  with import ./config.nix;
+  mkDerivation {
+    name = "nar-listing";
+    buildCommand = "mkdir $out; echo foo > $out/bar; ln -s xyzzy $out/link";
+  }
+')
+
+nix copy --to file://$cacheDir?write-nar-listing=1 $outPath
+
+[[ $(cat $cacheDir/$(basename $outPath).ls) = '{"version":1,"root":{"type":"directory","entries":{"bar":{"type":"regular","size":4,"narOffset":232},"link":{"type":"symlink","target":"xyzzy"}}}}' ]]
+
+
+# Test debug info index generation.
+clearCache
+
+outPath=$(nix-build --no-out-link -E '
+  with import ./config.nix;
+  mkDerivation {
+    name = "debug-info";
+    buildCommand = "mkdir -p $out/lib/debug/.build-id/02; echo foo > $out/lib/debug/.build-id/02/623eda209c26a59b1a8638ff7752f6b945c26b.debug";
+  }
+')
+
+nix copy --to "file://$cacheDir?index-debug-info=1&compression=none" $outPath
+
+[[ $(cat $cacheDir/debuginfo/02623eda209c26a59b1a8638ff7752f6b945c26b.debug) = '{"archive":"../nar/100vxs724qr46phz8m24iswmg9p3785hsyagz0kchf6q6gf06sw6.nar","member":"lib/debug/.build-id/02/623eda209c26a59b1a8638ff7752f6b945c26b.debug"}' ]]

tests/common.sh.in

@@ -1,6 +1,6 @@
 set -e
 
-export TEST_ROOT=$(realpath ${TMPDIR:-/tmp}/nix-test)
+export TEST_ROOT=$(realpath ${TMPDIR:-/tmp}/nix-test)/${TEST_NAME:-default}
 export NIX_STORE_DIR
 if ! NIX_STORE_DIR=$(readlink -f $TEST_ROOT/store 2> /dev/null); then
     # Maybe the build directory is symlinked.
@@ -11,6 +11,7 @@ export NIX_LOCALSTATE_DIR=$TEST_ROOT/var
 export NIX_LOG_DIR=$TEST_ROOT/var/log/nix
 export NIX_STATE_DIR=$TEST_ROOT/var/nix
 export NIX_CONF_DIR=$TEST_ROOT/etc
+export NIX_DAEMON_SOCKET_PATH=$TEST_ROOT/daemon-socket
 unset NIX_USER_CONF_FILES
 export _NIX_TEST_SHARED=$TEST_ROOT/shared
 if [[ -n $NIX_STORE ]]; then
@@ -76,7 +77,7 @@ startDaemon() {
     rm -f $NIX_STATE_DIR/daemon-socket/socket
     nix-daemon &
     for ((i = 0; i < 30; i++)); do
-        if [ -e $NIX_STATE_DIR/daemon-socket/socket ]; then break; fi
+        if [ -e $NIX_DAEMON_SOCKET_PATH ]; then break; fi
         sleep 1
     done
     pidDaemon=$!

tests/gc-auto.sh

@@ -13,24 +13,32 @@ fake_free=$TEST_ROOT/fake-free
 export _NIX_TEST_FREE_SPACE_FILE=$fake_free
 echo 1100 > $fake_free
 
+fifoLock=$TEST_ROOT/fifoLock
+mkfifo "$fifoLock"
+
 expr=$(cat <<EOF
 with import ./config.nix; mkDerivation {
   name = "gc-A";
   buildCommand = ''
     set -x
     [[ \$(ls \$NIX_STORE/*-garbage? | wc -l) = 3 ]]
+
     mkdir \$out
     echo foo > \$out/bar
-    echo 1...
-    sleep 2
-    echo 200 > ${fake_free}.tmp1
+
+    # Pretend that we run out of space
+    echo 100 > ${fake_free}.tmp1
     mv ${fake_free}.tmp1 $fake_free
-    echo 2...
-    sleep 2
-    echo 3...
-    sleep 2
-    echo 4...
-    [[ \$(ls \$NIX_STORE/*-garbage? | wc -l) = 1 ]]
+
+    # Wait for the GC to run
+    for i in {1..20}; do
+        echo ''\${i}...
+        if [[ \$(ls \$NIX_STORE/*-garbage? | wc -l) = 1 ]]; then
+            exit 0
+        fi
+        sleep 1
+    done
+    exit 1
   '';
 }
 EOF
@@ -43,15 +51,9 @@ with import ./config.nix; mkDerivation {
     set -x
     mkdir \$out
     echo foo > \$out/bar
-    echo 1...
-    sleep 2
-    echo 200 > ${fake_free}.tmp2
-    mv ${fake_free}.tmp2 $fake_free
-    echo 2...
-    sleep 2
-    echo 3...
-    sleep 2
-    echo 4...
+
+    # Wait for the first build to finish
+    cat "$fifoLock"
   '';
 }
 EOF
@@ -59,12 +61,19 @@ EOF
 
 nix build -v -o $TEST_ROOT/result-A -L "($expr)" \
     --min-free 1000 --max-free 2000 --min-free-check-interval 1 &
-pid=$!
+pid1=$!
 
 nix build -v -o $TEST_ROOT/result-B -L "($expr2)" \
-    --min-free 1000 --max-free 2000 --min-free-check-interval 1
+    --min-free 1000 --max-free 2000 --min-free-check-interval 1 &
+pid2=$!
 
-wait "$pid"
+# Once the first build is done, unblock the second one.
+# If the first build fails, we need to postpone the failure to still allow
+# the second one to finish
+wait "$pid1" || FIRSTBUILDSTATUS=$?
+echo "unlock" > $fifoLock
+( exit ${FIRSTBUILDSTATUS:-0} )
+wait "$pid2"
 
 [[ foo = $(cat $TEST_ROOT/result-A/bar) ]]
 [[ foo = $(cat $TEST_ROOT/result-B/bar) ]]

tests/gc-concurrent.builder.sh

@@ -1,7 +1,10 @@
+echo "Build started" > "$lockFifo"
+
 mkdir $out
 echo $(cat $input1/foo)$(cat $input2/bar) > $out/foobar
 
-sleep 10
+# Wait for someone to write on the fifo
+cat "$lockFifo"
 
 # $out should not have been GC'ed while we were sleeping, but just in
 # case...

tests/gc-concurrent.nix

@@ -1,5 +1,7 @@
 with import ./config.nix;
 
+{ lockFifo ? null }:
+
 rec {
 
   input1 = mkDerivation {
@@ -16,6 +18,7 @@ rec {
     name = "gc-concurrent";
     builder = ./gc-concurrent.builder.sh;
     inherit input1 input2;
+    inherit lockFifo;
   };
 
   test2 = mkDerivation {

tests/gc-concurrent.sh

@@ -2,7 +2,10 @@ source common.sh
 
 clearStore
 
-drvPath1=$(nix-instantiate gc-concurrent.nix -A test1)
+lockFifo1=$TEST_ROOT/test1.fifo
+mkfifo "$lockFifo1"
+
+drvPath1=$(nix-instantiate gc-concurrent.nix -A test1 --argstr lockFifo "$lockFifo1")
 outPath1=$(nix-store -q $drvPath1)
 
 drvPath2=$(nix-instantiate gc-concurrent.nix -A test2)
@@ -22,19 +25,16 @@ ln -s $outPath3 "$NIX_STATE_DIR"/gcroots/foo2
 nix-store -rvv "$drvPath1" &
 pid1=$!
 
-# Start build #2 in the background after 10 seconds.
-(sleep 10 && nix-store -rvv "$drvPath2") &
-pid2=$!
+# Wait for the build of $drvPath1 to start
+cat $lockFifo1
 
 # Run the garbage collector while the build is running.
-sleep 6
 nix-collect-garbage
 
-# Wait for build #1/#2 to finish.
+# Unlock the build of $drvPath1
+echo "" > $lockFifo1
 echo waiting for pid $pid1 to finish...
 wait $pid1
-echo waiting for pid $pid2 to finish...
-wait $pid2
 
 # Check that the root of build #1 and its dependencies haven't been
 # deleted.  The should not be deleted by the GC because they were
@@ -42,8 +42,9 @@ wait $pid2
 cat $outPath1/foobar
 cat $outPath1/input-2/bar
 
-# Check that build #2 has succeeded.  It should succeed because the
-# derivation is a GC root.
+# Check that the build build $drvPath2 succeeds.
+# It should succeed because the derivation is a GC root.
+nix-store -rvv "$drvPath2"
 cat $outPath2/foobar
 
 rm -f "$NIX_STATE_DIR"/gcroots/foo*

tests/gc-concurrent2.builder.sh

@@ -3,5 +3,3 @@ echo $(cat $input1/foo)$(cat $input2/bar)xyzzy > $out/foobar
 
 # Check that the GC hasn't deleted the lock on our output.
 test -e "$out.lock"
-
-sleep 6

tests/init.sh

@@ -18,6 +18,7 @@ build-users-group =
 keep-derivations = false
 sandbox = false
 experimental-features = nix-command flakes
+gc-reserved-space = 0
 include nix.conf.extra
 EOF
 

tests/local.mk

@@ -40,4 +40,4 @@ tests-environment = NIX_REMOTE= $(bash) -e
 
 clean-files += $(d)/common.sh
 
-installcheck: $(d)/common.sh $(d)/config.nix $(d)/plugins/libplugintest.$(SO_EXT)
+test-deps += tests/common.sh tests/config.nix tests/plugins/libplugintest.$(SO_EXT)

tests/misc.sh

@@ -16,6 +16,11 @@ nix-env --foo 2>&1 | grep "no operation"
 nix-env -q --foo 2>&1 | grep "unknown flag"
 
 # Eval Errors.
-eval_res=$(nix-instantiate --eval -E 'let a = {} // a; in a.foo' 2>&1 || true)
-echo $eval_res | grep "(string) (1:15)"
-echo $eval_res | grep "infinite recursion encountered"
+eval_arg_res=$(nix-instantiate --eval -E 'let a = {} // a; in a.foo' 2>&1 || true)
+echo $eval_arg_res | grep "at: (1:15) from string"
+echo $eval_arg_res | grep "infinite recursion encountered"
+
+eval_stdin_res=$(echo 'let a = {} // a; in a.foo' | nix-instantiate --eval -E - 2>&1 || true)
+echo $eval_stdin_res | grep "at: (1:15) from stdin"
+echo $eval_stdin_res | grep "infinite recursion encountered"
+

tests/nix-shell.sh

@@ -55,3 +55,10 @@ chmod a+rx $TEST_ROOT/shell.shebang.rb
 
 output=$($TEST_ROOT/shell.shebang.rb abc ruby)
 [ "$output" = '-e load("'"$TEST_ROOT"'/shell.shebang.rb") -- abc ruby' ]
+
+# Test 'nix develop'.
+nix develop -f shell.nix shellDrv -c bash -c '[[ -n $stdenv ]]'
+
+# Test 'nix print-dev-env'.
+source <(nix print-dev-env -f shell.nix shellDrv)
+[[ -n $stdenv ]]