Compare commits

..
Author SHA1 Message Date
Eelco Dolstra 131d063ea1 Warn when experimental features are used
If you use an experimental feature not listed in the
'experimental-features' setting, you now get a warning rather than a
fatal error. This makes things like the 'nix' command a bit less
obnoxious.

However, if you set 'allow-experimental-features = false', it's still
a fatal error. This is primarily for CI systems where use of
experimental features in the daemon could go unnoticed.
2020-06-24 23:09:12 +02:00
92 changed files with 784 additions and 1531 deletions

No files matched your search

-1
View File
@@ -1,7 +1,6 @@
((c++-mode . ( ((c++-mode . (
(c-file-style . "k&r") (c-file-style . "k&r")
(c-basic-offset . 4) (c-basic-offset . 4)
(c-block-comment-prefix . " ")
(indent-tabs-mode . nil) (indent-tabs-mode . nil)
(tab-width . 4) (tab-width . 4)
(show-trailing-whitespace . t) (show-trailing-whitespace . t)
-6
View File
@@ -1,6 +0,0 @@
version: 2
updates:
- package-ecosystem: "github-actions"
directory: "/"
schedule:
interval: "weekly"
+1 -11
View File
@@ -10,15 +10,5 @@ jobs:
runs-on: ${{ matrix.os }} runs-on: ${{ matrix.os }}
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v2
- uses: cachix/install-nix-action@v10 - uses: cachix/install-nix-action@v8
- run: nix-build release.nix --arg nix '{ outPath = ./.; revCount = 123; shortRev = "abcdefgh"; }' --arg systems '[ builtins.currentSystem ]' -A installerScript -A perlBindings
macos_perf_test:
runs-on: macos-latest
steps:
- name: Disable syspolicy assessments
run: |
spctl --status
sudo spctl --master-disable
- uses: actions/checkout@v2
- uses: cachix/install-nix-action@v10
- run: nix-build release.nix --arg nix '{ outPath = ./.; revCount = 123; shortRev = "abcdefgh"; }' --arg systems '[ builtins.currentSystem ]' -A installerScript -A perlBindings - run: nix-build release.nix --arg nix '{ outPath = ./.; revCount = 123; shortRev = "abcdefgh"; }' --arg systems '[ builtins.currentSystem ]' -A installerScript -A perlBindings
+12 -2
View File
@@ -170,5 +170,15 @@ $channelsBucket->add_key(
chdir("/home/eelco/Dev/nix-pristine") or die; chdir("/home/eelco/Dev/nix-pristine") or die;
system("git remote update origin") == 0 or die; system("git remote update origin") == 0 or die;
system("git tag --force --sign $version $nixRev -m 'Tagging release $version'") == 0 or die; system("git tag --force --sign $version $nixRev -m 'Tagging release $version'") == 0 or die;
system("git push --tags") == 0 or die;
system("git push --force-with-lease origin $nixRev:refs/heads/latest-release") == 0 or die; # Update the website.
my $siteDir = "/home/eelco/Dev/nixos-homepage-pristine";
system("cd $siteDir && git pull") == 0 or die;
write_file("$siteDir/nix-release.tt",
"[%-\n" .
"latestNixVersion = \"$version\"\n" .
"-%]\n");
system("cd $siteDir && git commit -a -m 'Nix $version released'") == 0 or die;
-28
View File
@@ -1,28 +0,0 @@
#!/bin/sh
set -u
red=""
green=""
yellow=""
normal=""
post_run_msg="ran test $1..."
if [ -t 1 ]; then
red=""
green=""
yellow=""
normal=""
fi
(cd $(dirname $1) && env ${TESTS_ENVIRONMENT} init.sh 2>/dev/null > /dev/null)
log="$(cd $(dirname $1) && env ${TESTS_ENVIRONMENT} $(basename $1) 2>&1)"
status=$?
if [ $status -eq 0 ]; then
echo "$post_run_msg [${green}PASS$normal]"
elif [ $status -eq 99 ]; then
echo "$post_run_msg [${yellow}SKIP$normal]"
else
echo "$post_run_msg [${red}FAIL$normal]"
echo "$log" | sed 's/^/ /'
exit "$status"
fi
+37 -7
View File
@@ -1,15 +1,45 @@
# Run program $1 as part of ‘make installcheck’. # Run program $1 as part of ‘make installcheck’.
test-deps =
define run-install-test define run-install-test
installcheck: $1.test installcheck: $1
.PHONY: $1.test _installcheck-list += $1
$1.test: $1 $(test-deps)
@env TEST_NAME=$(notdir $(basename $1)) TESTS_ENVIRONMENT="$(tests-environment)" mk/run_test.sh $1
endef endef
# Color code from https://unix.stackexchange.com/a/10065
installcheck:
@total=0; failed=0; \
red=""; \
green=""; \
yellow=""; \
normal=""; \
if [ -t 1 ]; then \
red=""; \
green=""; \
yellow=""; \
normal=""; \
fi; \
for i in $(_installcheck-list); do \
total=$$((total + 1)); \
printf "running test $$i..."; \
log="$$(cd $$(dirname $$i) && $(tests-environment) $$(basename $$i) 2>&1)"; \
status=$$?; \
if [ $$status -eq 0 ]; then \
echo " [$${green}PASS$$normal]"; \
elif [ $$status -eq 99 ]; then \
echo " [$${yellow}SKIP$$normal]"; \
else \
echo " [$${red}FAIL$$normal]"; \
echo "$$log" | sed 's/^/ /'; \
failed=$$((failed + 1)); \
fi; \
done; \
if [ "$$failed" != 0 ]; then \
echo "$${red}$$failed out of $$total tests failed $$normal"; \
exit 1; \
else \
echo "$${green}All tests succeeded$$normal"; \
fi
.PHONY: check installcheck .PHONY: check installcheck
+1 -1
View File
@@ -182,7 +182,7 @@ void importPaths(int fd, int dontCheckSigs)
PPCODE: PPCODE:
try { try {
FdSource source(fd); FdSource source(fd);
store()->importPaths(source, dontCheckSigs ? NoCheckSigs : CheckSigs); store()->importPaths(source, nullptr, dontCheckSigs ? NoCheckSigs : CheckSigs);
} catch (Error & e) { } catch (Error & e) {
croak("%s", e.what()); croak("%s", e.what());
} }
+1 -1
View File
@@ -130,7 +130,7 @@ Pos findDerivationFilename(EvalState & state, Value & v, std::string what)
Symbol file = state.symbols.create(filename); Symbol file = state.symbols.create(filename);
return { foFile, file, lineno, 0 }; return { file, lineno, 0 };
} }
+1 -1
View File
@@ -78,7 +78,7 @@ public:
if (!a) if (!a)
throw Error({ throw Error({
.hint = hintfmt("attribute '%s' missing", name), .hint = hintfmt("attribute '%s' missing", name),
.errPos = pos .nixCode = NixCode { .errPos = pos }
}); });
return *a; return *a;
+2 -2
View File
@@ -11,7 +11,7 @@ LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s))
{ {
throw EvalError({ throw EvalError({
.hint = hintfmt(s), .hint = hintfmt(s),
.errPos = pos .nixCode = NixCode { .errPos = pos }
}); });
} }
@@ -25,7 +25,7 @@ LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s, const
{ {
throw TypeError({ throw TypeError({
.hint = hintfmt(s, showType(v)), .hint = hintfmt(s, showType(v)),
.errPos = pos .nixCode = NixCode { .errPos = pos }
}); });
} }
+24 -23
View File
@@ -366,7 +366,7 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
if (store->isInStore(r.second)) { if (store->isInStore(r.second)) {
StorePathSet closure; StorePathSet closure;
store->computeFSClosure(store->toStorePath(r.second).first, closure); store->computeFSClosure(store->parseStorePath(store->toStorePath(r.second)), closure);
for (auto & path : closure) for (auto & path : closure)
allowedPaths->insert(store->printStorePath(path)); allowedPaths->insert(store->printStorePath(path));
} else } else
@@ -529,7 +529,7 @@ LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s, const
{ {
throw EvalError({ throw EvalError({
.hint = hintfmt(s, s2), .hint = hintfmt(s, s2),
.errPos = pos .nixCode = NixCode { .errPos = pos }
}); });
} }
@@ -542,7 +542,7 @@ LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s, const
{ {
throw EvalError({ throw EvalError({
.hint = hintfmt(s, s2, s3), .hint = hintfmt(s, s2, s3),
.errPos = pos .nixCode = NixCode { .errPos = pos }
}); });
} }
@@ -551,7 +551,7 @@ LocalNoInlineNoReturn(void throwEvalError(const Pos & p1, const char * s, const
// p1 is where the error occurred; p2 is a position mentioned in the message. // p1 is where the error occurred; p2 is a position mentioned in the message.
throw EvalError({ throw EvalError({
.hint = hintfmt(s, sym, p2), .hint = hintfmt(s, sym, p2),
.errPos = p1 .nixCode = NixCode { .errPos = p1 }
}); });
} }
@@ -559,7 +559,7 @@ LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s))
{ {
throw TypeError({ throw TypeError({
.hint = hintfmt(s), .hint = hintfmt(s),
.errPos = pos .nixCode = NixCode { .errPos = pos }
}); });
} }
@@ -572,7 +572,7 @@ LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s, const
{ {
throw TypeError({ throw TypeError({
.hint = hintfmt(s, fun.showNamePos(), s2), .hint = hintfmt(s, fun.showNamePos(), s2),
.errPos = pos .nixCode = NixCode { .errPos = pos }
}); });
} }
@@ -580,7 +580,7 @@ LocalNoInlineNoReturn(void throwAssertionError(const Pos & pos, const char * s,
{ {
throw AssertionError({ throw AssertionError({
.hint = hintfmt(s, s1), .hint = hintfmt(s, s1),
.errPos = pos .nixCode = NixCode { .errPos = pos }
}); });
} }
@@ -588,18 +588,23 @@ LocalNoInlineNoReturn(void throwUndefinedVarError(const Pos & pos, const char *
{ {
throw UndefinedVarError({ throw UndefinedVarError({
.hint = hintfmt(s, s1), .hint = hintfmt(s, s1),
.errPos = pos .nixCode = NixCode { .errPos = pos }
}); });
} }
LocalNoInline(void addErrorTrace(Error & e, const char * s, const string & s2)) LocalNoInline(void addErrorPrefix(Error & e, const char * s, const string & s2))
{ {
e.addTrace(std::nullopt, s, s2); e.addPrefix(format(s) % s2);
} }
LocalNoInline(void addErrorTrace(Error & e, const Pos & pos, const char * s, const string & s2)) LocalNoInline(void addErrorPrefix(Error & e, const char * s, const ExprLambda & fun, const Pos & pos))
{ {
e.addTrace(pos, s, s2); e.addPrefix(format(s) % fun.showNamePos() % pos);
}
LocalNoInline(void addErrorPrefix(Error & e, const char * s, const string & s2, const Pos & pos))
{
e.addPrefix(format(s) % s2 % pos);
} }
@@ -813,7 +818,7 @@ void EvalState::evalFile(const Path & path_, Value & v)
try { try {
eval(e, v); eval(e, v);
} catch (Error & e) { } catch (Error & e) {
addErrorTrace(e, "while evaluating the file '%1%':", path2); addErrorPrefix(e, "while evaluating the file '%1%':\n", path2);
throw; throw;
} }
@@ -1063,8 +1068,8 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v)
} catch (Error & e) { } catch (Error & e) {
if (pos2 && pos2->file != state.sDerivationNix) if (pos2 && pos2->file != state.sDerivationNix)
addErrorTrace(e, *pos2, "while evaluating the attribute '%1%'", addErrorPrefix(e, "while evaluating the attribute '%1%' at %2%:\n",
showAttrPath(state, env, attrPath)); showAttrPath(state, env, attrPath), *pos2);
throw; throw;
} }
@@ -1232,15 +1237,11 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po
/* Evaluate the body. This is conditional on showTrace, because /* Evaluate the body. This is conditional on showTrace, because
catching exceptions makes this function not tail-recursive. */ catching exceptions makes this function not tail-recursive. */
if (loggerSettings.showTrace.get()) if (settings.showTrace)
try { try {
lambda.body->eval(*this, env2, v); lambda.body->eval(*this, env2, v);
} catch (Error & e) { } catch (Error & e) {
addErrorTrace(e, lambda.pos, "while evaluating %s", addErrorPrefix(e, "while evaluating %1%, called from %2%:\n", lambda, pos);
(lambda.name.set()
? "'" + (string) lambda.name + "'"
: "anonymous lambdaction"));
addErrorTrace(e, pos, "from call site%s", "");
throw; throw;
} }
else else
@@ -1515,7 +1516,7 @@ void EvalState::forceValueDeep(Value & v)
try { try {
recurse(*i.value); recurse(*i.value);
} catch (Error & e) { } catch (Error & e) {
addErrorTrace(e, *i.pos, "while evaluating the attribute '%1%'", i.name); addErrorPrefix(e, "while evaluating the attribute '%1%' at %2%:\n", i.name, *i.pos);
throw; throw;
} }
} }
@@ -1935,7 +1936,7 @@ string ExternalValueBase::coerceToString(const Pos & pos, PathSet & context, boo
{ {
throw TypeError({ throw TypeError({
.hint = hintfmt("cannot coerce %1% to a string", showType()), .hint = hintfmt("cannot coerce %1% to a string", showType()),
.errPos = pos .nixCode = NixCode { .errPos = pos }
}); });
} }
+1 -1
View File
@@ -250,7 +250,7 @@ private:
friend struct ExprAttrs; friend struct ExprAttrs;
friend struct ExprLet; friend struct ExprLet;
Expr * parse(const char * text, FileOrigin origin, const Path & path, Expr * parse(const char * text, const Path & path,
const Path & basePath, StaticEnv & staticEnv); const Path & basePath, StaticEnv & staticEnv);
public: public:
+2 -17
View File
@@ -197,22 +197,7 @@ std::ostream & operator << (std::ostream & str, const Pos & pos)
if (!pos) if (!pos)
str << "undefined position"; str << "undefined position";
else else
{ str << (format(ANSI_BOLD "%1%" ANSI_NORMAL ":%2%:%3%") % (string) pos.file % pos.line % pos.column).str();
auto f = format(ANSI_BOLD "%1%" ANSI_NORMAL ":%2%:%3%");
switch (pos.origin) {
case foFile:
f % (string) pos.file;
break;
case foStdin:
case foString:
f % "(string)";
break;
default:
throw Error("unhandled Pos origin!");
}
str << (f % pos.line % pos.column).str();
}
return str; return str;
} }
@@ -285,7 +270,7 @@ void ExprVar::bindVars(const StaticEnv & env)
if (withLevel == -1) if (withLevel == -1)
throw UndefinedVarError({ throw UndefinedVarError({
.hint = hintfmt("undefined variable '%1%'", name), .hint = hintfmt("undefined variable '%1%'", name),
.errPos = pos .nixCode = NixCode { .errPos = pos }
}); });
fromWith = true; fromWith = true;
this->level = withLevel; this->level = withLevel;
+4 -5
View File
@@ -24,12 +24,11 @@ MakeError(RestrictedPathError, Error);
struct Pos struct Pos
{ {
FileOrigin origin;
Symbol file; Symbol file;
unsigned int line, column; unsigned int line, column;
Pos() : origin(foString), line(0), column(0) { }; Pos() : line(0), column(0) { };
Pos(FileOrigin origin, const Symbol & file, unsigned int line, unsigned int column) Pos(const Symbol & file, unsigned int line, unsigned int column)
: origin(origin), file(file), line(line), column(column) { }; : file(file), line(line), column(column) { };
operator bool() const operator bool() const
{ {
return line != 0; return line != 0;
@@ -239,7 +238,7 @@ struct ExprLambda : Expr
if (!arg.empty() && formals && formals->argNames.find(arg) != formals->argNames.end()) if (!arg.empty() && formals && formals->argNames.find(arg) != formals->argNames.end())
throw ParseError({ throw ParseError({
.hint = hintfmt("duplicate formal function argument '%1%'", arg), .hint = hintfmt("duplicate formal function argument '%1%'", arg),
.errPos = pos .nixCode = NixCode { .errPos = pos }
}); });
}; };
void setName(Symbol & name); void setName(Symbol & name);
+17 -28
View File
@@ -30,8 +30,7 @@ namespace nix {
SymbolTable & symbols; SymbolTable & symbols;
Expr * result; Expr * result;
Path basePath; Path basePath;
Symbol file; Symbol path;
FileOrigin origin;
ErrorInfo error; ErrorInfo error;
Symbol sLetBody; Symbol sLetBody;
ParseData(EvalState & state) ParseData(EvalState & state)
@@ -66,17 +65,18 @@ namespace nix {
static void dupAttr(const AttrPath & attrPath, const Pos & pos, const Pos & prevPos) static void dupAttr(const AttrPath & attrPath, const Pos & pos, const Pos & prevPos)
{ {
throw ParseError({ throw ParseError({
.hint = hintfmt("attribute '%1%' already defined at %2%", .hint = hintfmt("attribute '%1%' already defined at %2%",
showAttrPath(attrPath), prevPos), showAttrPath(attrPath), prevPos),
.errPos = pos .nixCode = NixCode { .errPos = pos },
}); });
} }
static void dupAttr(Symbol attr, const Pos & pos, const Pos & prevPos) static void dupAttr(Symbol attr, const Pos & pos, const Pos & prevPos)
{ {
throw ParseError({ throw ParseError({
.hint = hintfmt("attribute '%1%' already defined at %2%", attr, prevPos), .hint = hintfmt("attribute '%1%' already defined at %2%", attr, prevPos),
.errPos = pos .nixCode = NixCode { .errPos = pos },
}); });
} }
@@ -148,7 +148,7 @@ static void addFormal(const Pos & pos, Formals * formals, const Formal & formal)
throw ParseError({ throw ParseError({
.hint = hintfmt("duplicate formal function argument '%1%'", .hint = hintfmt("duplicate formal function argument '%1%'",
formal.name), formal.name),
.errPos = pos .nixCode = NixCode { .errPos = pos },
}); });
formals->formals.push_front(formal); formals->formals.push_front(formal);
} }
@@ -246,7 +246,7 @@ static Expr * stripIndentation(const Pos & pos, SymbolTable & symbols, vector<Ex
static inline Pos makeCurPos(const YYLTYPE & loc, ParseData * data) static inline Pos makeCurPos(const YYLTYPE & loc, ParseData * data)
{ {
return Pos(data->origin, data->file, loc.first_line, loc.first_column); return Pos(data->path, loc.first_line, loc.first_column);
} }
#define CUR_POS makeCurPos(*yylocp, data) #define CUR_POS makeCurPos(*yylocp, data)
@@ -259,7 +259,7 @@ void yyerror(YYLTYPE * loc, yyscan_t scanner, ParseData * data, const char * err
{ {
data->error = { data->error = {
.hint = hintfmt(error), .hint = hintfmt(error),
.errPos = makeCurPos(*loc, data) .nixCode = NixCode { .errPos = makeCurPos(*loc, data) }
}; };
} }
@@ -339,7 +339,7 @@ expr_function
{ if (!$2->dynamicAttrs.empty()) { if (!$2->dynamicAttrs.empty())
throw ParseError({ throw ParseError({
.hint = hintfmt("dynamic attributes not allowed in let"), .hint = hintfmt("dynamic attributes not allowed in let"),
.errPos = CUR_POS .nixCode = NixCode { .errPos = CUR_POS },
}); });
$$ = new ExprLet($2, $4); $$ = new ExprLet($2, $4);
} }
@@ -419,7 +419,7 @@ expr_simple
if (noURLLiterals) if (noURLLiterals)
throw ParseError({ throw ParseError({
.hint = hintfmt("URL literals are disabled"), .hint = hintfmt("URL literals are disabled"),
.errPos = CUR_POS .nixCode = NixCode { .errPos = CUR_POS }
}); });
$$ = new ExprString(data->symbols.create($1)); $$ = new ExprString(data->symbols.create($1));
} }
@@ -492,7 +492,7 @@ attrs
} else } else
throw ParseError({ throw ParseError({
.hint = hintfmt("dynamic attributes not allowed in inherit"), .hint = hintfmt("dynamic attributes not allowed in inherit"),
.errPos = makeCurPos(@2, data) .nixCode = NixCode { .errPos = makeCurPos(@2, data) },
}); });
} }
| { $$ = new AttrPath; } | { $$ = new AttrPath; }
@@ -569,24 +569,13 @@ formal
namespace nix { namespace nix {
Expr * EvalState::parse(const char * text, FileOrigin origin, Expr * EvalState::parse(const char * text,
const Path & path, const Path & basePath, StaticEnv & staticEnv) const Path & path, const Path & basePath, StaticEnv & staticEnv)
{ {
yyscan_t scanner; yyscan_t scanner;
ParseData data(*this); ParseData data(*this);
data.origin = origin;
switch (origin) {
case foFile:
data.file = data.symbols.create(path);
break;
case foStdin:
case foString:
data.file = data.symbols.create(text);
break;
default:
assert(false);
}
data.basePath = basePath; data.basePath = basePath;
data.path = data.symbols.create(path);
yylex_init(&scanner); yylex_init(&scanner);
yy_scan_string(text, scanner); yy_scan_string(text, scanner);
@@ -636,13 +625,13 @@ Expr * EvalState::parseExprFromFile(const Path & path)
Expr * EvalState::parseExprFromFile(const Path & path, StaticEnv & staticEnv) Expr * EvalState::parseExprFromFile(const Path & path, StaticEnv & staticEnv)
{ {
return parse(readFile(path).c_str(), foFile, path, dirOf(path), staticEnv); return parse(readFile(path).c_str(), path, dirOf(path), staticEnv);
} }
Expr * EvalState::parseExprFromString(std::string_view s, const Path & basePath, StaticEnv & staticEnv) Expr * EvalState::parseExprFromString(std::string_view s, const Path & basePath, StaticEnv & staticEnv)
{ {
return parse(s.data(), foString, "", basePath, staticEnv); return parse(s.data(), "(string)", basePath, staticEnv);
} }
@@ -655,7 +644,7 @@ Expr * EvalState::parseExprFromString(std::string_view s, const Path & basePath)
Expr * EvalState::parseStdin() Expr * EvalState::parseStdin()
{ {
//Activity act(*logger, lvlTalkative, format("parsing standard input")); //Activity act(*logger, lvlTalkative, format("parsing standard input"));
return parse(drainFD(0).data(), foStdin, "", absPath("."), staticBaseEnv); return parseExprFromString(drainFD(0), absPath("."));
} }
@@ -704,7 +693,7 @@ Path EvalState::findFile(SearchPath & searchPath, const string & path, const Pos
? "cannot look up '<%s>' in pure evaluation mode (use '--impure' to override)" ? "cannot look up '<%s>' in pure evaluation mode (use '--impure' to override)"
: "file '%s' was not found in the Nix search path (add it using $NIX_PATH or -I)", : "file '%s' was not found in the Nix search path (add it using $NIX_PATH or -I)",
path), path),
.errPos = pos .nixCode = NixCode { .errPos = pos }
}); });
} }
+54 -55
View File
@@ -96,7 +96,7 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args
} catch (InvalidPathError & e) { } catch (InvalidPathError & e) {
throw EvalError({ throw EvalError({
.hint = hintfmt("cannot import '%1%', since path '%2%' is not valid", path, e.path), .hint = hintfmt("cannot import '%1%', since path '%2%' is not valid", path, e.path),
.errPos = pos .nixCode = NixCode { .errPos = pos }
}); });
} }
@@ -177,7 +177,7 @@ void prim_importNative(EvalState & state, const Pos & pos, Value * * args, Value
.hint = hintfmt( .hint = hintfmt(
"cannot import '%1%', since path '%2%' is not valid", "cannot import '%1%', since path '%2%' is not valid",
path, e.path), path, e.path),
.errPos = pos .nixCode = NixCode { .errPos = pos }
}); });
} }
@@ -215,7 +215,7 @@ void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v)
if (count == 0) { if (count == 0) {
throw EvalError({ throw EvalError({
.hint = hintfmt("at least one argument to 'exec' required"), .hint = hintfmt("at least one argument to 'exec' required"),
.errPos = pos .nixCode = NixCode { .errPos = pos }
}); });
} }
PathSet context; PathSet context;
@@ -230,7 +230,7 @@ void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v)
throw EvalError({ throw EvalError({
.hint = hintfmt("cannot execute '%1%', since path '%2%' is not valid", .hint = hintfmt("cannot execute '%1%', since path '%2%' is not valid",
program, e.path), program, e.path),
.errPos = pos .nixCode = NixCode { .errPos = pos }
}); });
} }
@@ -239,13 +239,13 @@ void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v)
try { try {
parsed = state.parseExprFromString(output, pos.file); parsed = state.parseExprFromString(output, pos.file);
} catch (Error & e) { } catch (Error & e) {
e.addTrace(pos, "While parsing the output from '%1%'", program); e.addPrefix(fmt("While parsing the output from '%1%', at %2%\n", program, pos));
throw; throw;
} }
try { try {
state.eval(parsed, v); state.eval(parsed, v);
} catch (Error & e) { } catch (Error & e) {
e.addTrace(pos, "While evaluating the output from '%1%'", program); e.addPrefix(fmt("While evaluating the output from '%1%', at %2%\n", program, pos));
throw; throw;
} }
} }
@@ -385,7 +385,7 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar
if (startSet == args[0]->attrs->end()) if (startSet == args[0]->attrs->end())
throw EvalError({ throw EvalError({
.hint = hintfmt("attribute 'startSet' required"), .hint = hintfmt("attribute 'startSet' required"),
.errPos = pos .nixCode = NixCode { .errPos = pos }
}); });
state.forceList(*startSet->value, pos); state.forceList(*startSet->value, pos);
@@ -399,7 +399,7 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar
if (op == args[0]->attrs->end()) if (op == args[0]->attrs->end())
throw EvalError({ throw EvalError({
.hint = hintfmt("attribute 'operator' required"), .hint = hintfmt("attribute 'operator' required"),
.errPos = pos .nixCode = NixCode { .errPos = pos }
}); });
state.forceValue(*op->value, pos); state.forceValue(*op->value, pos);
@@ -421,7 +421,7 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar
if (key == e->attrs->end()) if (key == e->attrs->end())
throw EvalError({ throw EvalError({
.hint = hintfmt("attribute 'key' required"), .hint = hintfmt("attribute 'key' required"),
.errPos = pos .nixCode = NixCode { .errPos = pos }
}); });
state.forceValue(*key->value, pos); state.forceValue(*key->value, pos);
@@ -471,7 +471,7 @@ static void prim_addErrorContext(EvalState & state, const Pos & pos, Value * * a
v = *args[1]; v = *args[1];
} catch (Error & e) { } catch (Error & e) {
PathSet context; PathSet context;
e.addTrace(std::nullopt, state.coerceToString(pos, *args[0], context)); e.addPrefix(format("%1%\n") % state.coerceToString(pos, *args[0], context));
throw; throw;
} }
} }
@@ -556,14 +556,14 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
if (attr == args[0]->attrs->end()) if (attr == args[0]->attrs->end())
throw EvalError({ throw EvalError({
.hint = hintfmt("required attribute 'name' missing"), .hint = hintfmt("required attribute 'name' missing"),
.errPos = pos .nixCode = NixCode { .errPos = pos }
}); });
string drvName; string drvName;
Pos & posDrvName(*attr->pos); Pos & posDrvName(*attr->pos);
try { try {
drvName = state.forceStringNoCtx(*attr->value, pos); drvName = state.forceStringNoCtx(*attr->value, pos);
} catch (Error & e) { } catch (Error & e) {
e.addTrace(posDrvName, "while evaluating the derivation attribute 'name'"); e.addPrefix(fmt("while evaluating the derivation attribute 'name' at %1%:\n", posDrvName));
throw; throw;
} }
@@ -603,7 +603,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
else else
throw EvalError({ throw EvalError({
.hint = hintfmt("invalid value '%s' for 'outputHashMode' attribute", s), .hint = hintfmt("invalid value '%s' for 'outputHashMode' attribute", s),
.errPos = posDrvName .nixCode = NixCode { .errPos = posDrvName }
}); });
}; };
@@ -613,7 +613,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
if (outputs.find(j) != outputs.end()) if (outputs.find(j) != outputs.end())
throw EvalError({ throw EvalError({
.hint = hintfmt("duplicate derivation output '%1%'", j), .hint = hintfmt("duplicate derivation output '%1%'", j),
.errPos = posDrvName .nixCode = NixCode { .errPos = posDrvName }
}); });
/* !!! Check whether j is a valid attribute /* !!! Check whether j is a valid attribute
name. */ name. */
@@ -623,14 +623,14 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
if (j == "drv") if (j == "drv")
throw EvalError({ throw EvalError({
.hint = hintfmt("invalid derivation output name 'drv'" ), .hint = hintfmt("invalid derivation output name 'drv'" ),
.errPos = posDrvName .nixCode = NixCode { .errPos = posDrvName }
}); });
outputs.insert(j); outputs.insert(j);
} }
if (outputs.empty()) if (outputs.empty())
throw EvalError({ throw EvalError({
.hint = hintfmt("derivation cannot have an empty set of outputs"), .hint = hintfmt("derivation cannot have an empty set of outputs"),
.errPos = posDrvName .nixCode = NixCode { .errPos = posDrvName }
}); });
}; };
@@ -696,9 +696,8 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
} }
} catch (Error & e) { } catch (Error & e) {
e.addTrace(posDrvName, e.addPrefix(format("while evaluating the attribute '%1%' of the derivation '%2%' at %3%:\n")
"while evaluating the attribute '%1%' of the derivation '%2%'", % key % drvName % posDrvName);
key, drvName);
throw; throw;
} }
} }
@@ -746,20 +745,20 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
if (drv.builder == "") if (drv.builder == "")
throw EvalError({ throw EvalError({
.hint = hintfmt("required attribute 'builder' missing"), .hint = hintfmt("required attribute 'builder' missing"),
.errPos = posDrvName .nixCode = NixCode { .errPos = posDrvName }
}); });
if (drv.platform == "") if (drv.platform == "")
throw EvalError({ throw EvalError({
.hint = hintfmt("required attribute 'system' missing"), .hint = hintfmt("required attribute 'system' missing"),
.errPos = posDrvName .nixCode = NixCode { .errPos = posDrvName }
}); });
/* Check whether the derivation name is valid. */ /* Check whether the derivation name is valid. */
if (isDerivation(drvName)) if (isDerivation(drvName))
throw EvalError({ throw EvalError({
.hint = hintfmt("derivation names are not allowed to end in '%s'", drvExtension), .hint = hintfmt("derivation names are not allowed to end in '%s'", drvExtension),
.errPos = posDrvName .nixCode = NixCode { .errPos = posDrvName }
}); });
if (outputHash) { if (outputHash) {
@@ -767,7 +766,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
if (outputs.size() != 1 || *(outputs.begin()) != "out") if (outputs.size() != 1 || *(outputs.begin()) != "out")
throw Error({ throw Error({
.hint = hintfmt("multiple outputs are not supported in fixed-output derivations"), .hint = hintfmt("multiple outputs are not supported in fixed-output derivations"),
.errPos = posDrvName .nixCode = NixCode { .errPos = posDrvName }
}); });
std::optional<HashType> ht = parseHashTypeOpt(outputHashAlgo); std::optional<HashType> ht = parseHashTypeOpt(outputHashAlgo);
@@ -881,12 +880,12 @@ static void prim_storePath(EvalState & state, const Pos & pos, Value * * args, V
if (!state.store->isInStore(path)) if (!state.store->isInStore(path))
throw EvalError({ throw EvalError({
.hint = hintfmt("path '%1%' is not in the Nix store", path), .hint = hintfmt("path '%1%' is not in the Nix store", path),
.errPos = pos .nixCode = NixCode { .errPos = pos }
}); });
auto path2 = state.store->toStorePath(path).first; Path path2 = state.store->toStorePath(path);
if (!settings.readOnlyMode) if (!settings.readOnlyMode)
state.store->ensurePath(path2); state.store->ensurePath(state.store->parseStorePath(path2));
context.insert(state.store->printStorePath(path2)); context.insert(path2);
mkString(v, path, context); mkString(v, path, context);
} }
@@ -902,7 +901,7 @@ static void prim_pathExists(EvalState & state, const Pos & pos, Value * * args,
.hint = hintfmt( .hint = hintfmt(
"cannot check the existence of '%1%', since path '%2%' is not valid", "cannot check the existence of '%1%', since path '%2%' is not valid",
path, e.path), path, e.path),
.errPos = pos .nixCode = NixCode { .errPos = pos }
}); });
} }
@@ -948,7 +947,7 @@ static void prim_readFile(EvalState & state, const Pos & pos, Value * * args, Va
} catch (InvalidPathError & e) { } catch (InvalidPathError & e) {
throw EvalError({ throw EvalError({
.hint = hintfmt("cannot read '%1%', since path '%2%' is not valid", path, e.path), .hint = hintfmt("cannot read '%1%', since path '%2%' is not valid", path, e.path),
.errPos = pos .nixCode = NixCode { .errPos = pos }
}); });
} }
string s = readFile(state.checkSourcePath(state.toRealPath(path, context))); string s = readFile(state.checkSourcePath(state.toRealPath(path, context)));
@@ -979,7 +978,7 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va
if (i == v2.attrs->end()) if (i == v2.attrs->end())
throw EvalError({ throw EvalError({
.hint = hintfmt("attribute 'path' missing"), .hint = hintfmt("attribute 'path' missing"),
.errPos = pos .nixCode = NixCode { .errPos = pos }
}); });
PathSet context; PathSet context;
@@ -990,7 +989,7 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va
} catch (InvalidPathError & e) { } catch (InvalidPathError & e) {
throw EvalError({ throw EvalError({
.hint = hintfmt("cannot find '%1%', since path '%2%' is not valid", path, e.path), .hint = hintfmt("cannot find '%1%', since path '%2%' is not valid", path, e.path),
.errPos = pos .nixCode = NixCode { .errPos = pos }
}); });
} }
@@ -1010,7 +1009,7 @@ static void prim_hashFile(EvalState & state, const Pos & pos, Value * * args, Va
if (!ht) if (!ht)
throw Error({ throw Error({
.hint = hintfmt("unknown hash type '%1%'", type), .hint = hintfmt("unknown hash type '%1%'", type),
.errPos = pos .nixCode = NixCode { .errPos = pos }
}); });
PathSet context; // discarded PathSet context; // discarded
@@ -1029,7 +1028,7 @@ static void prim_readDir(EvalState & state, const Pos & pos, Value * * args, Val
} catch (InvalidPathError & e) { } catch (InvalidPathError & e) {
throw EvalError({ throw EvalError({
.hint = hintfmt("cannot read '%1%', since path '%2%' is not valid", path, e.path), .hint = hintfmt("cannot read '%1%', since path '%2%' is not valid", path, e.path),
.errPos = pos .nixCode = NixCode { .errPos = pos }
}); });
} }
@@ -1105,7 +1104,7 @@ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Valu
"in 'toFile': the file named '%1%' must not contain a reference " "in 'toFile': the file named '%1%' must not contain a reference "
"to a derivation but contains (%2%)", "to a derivation but contains (%2%)",
name, path), name, path),
.errPos = pos .nixCode = NixCode { .errPos = pos }
}); });
refs.insert(state.store->parseStorePath(path)); refs.insert(state.store->parseStorePath(path));
} }
@@ -1176,7 +1175,7 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args
if (!context.empty()) if (!context.empty())
throw EvalError({ throw EvalError({
.hint = hintfmt("string '%1%' cannot refer to other paths", path), .hint = hintfmt("string '%1%' cannot refer to other paths", path),
.errPos = pos .nixCode = NixCode { .errPos = pos }
}); });
state.forceValue(*args[0], pos); state.forceValue(*args[0], pos);
@@ -1185,7 +1184,7 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args
.hint = hintfmt( .hint = hintfmt(
"first argument in call to 'filterSource' is not a function but %1%", "first argument in call to 'filterSource' is not a function but %1%",
showType(*args[0])), showType(*args[0])),
.errPos = pos .nixCode = NixCode { .errPos = pos }
}); });
addPath(state, pos, std::string(baseNameOf(path)), path, args[0], FileIngestionMethod::Recursive, Hash(), v); addPath(state, pos, std::string(baseNameOf(path)), path, args[0], FileIngestionMethod::Recursive, Hash(), v);
@@ -1208,7 +1207,7 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value
if (!context.empty()) if (!context.empty())
throw EvalError({ throw EvalError({
.hint = hintfmt("string '%1%' cannot refer to other paths", path), .hint = hintfmt("string '%1%' cannot refer to other paths", path),
.errPos = *attr.pos .nixCode = NixCode { .errPos = *attr.pos }
}); });
} else if (attr.name == state.sName) } else if (attr.name == state.sName)
name = state.forceStringNoCtx(*attr.value, *attr.pos); name = state.forceStringNoCtx(*attr.value, *attr.pos);
@@ -1222,13 +1221,13 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value
else else
throw EvalError({ throw EvalError({
.hint = hintfmt("unsupported argument '%1%' to 'addPath'", attr.name), .hint = hintfmt("unsupported argument '%1%' to 'addPath'", attr.name),
.errPos = *attr.pos .nixCode = NixCode { .errPos = *attr.pos }
}); });
} }
if (path.empty()) if (path.empty())
throw EvalError({ throw EvalError({
.hint = hintfmt("'path' required"), .hint = hintfmt("'path' required"),
.errPos = pos .nixCode = NixCode { .errPos = pos }
}); });
if (name.empty()) if (name.empty())
name = baseNameOf(path); name = baseNameOf(path);
@@ -1289,7 +1288,7 @@ void prim_getAttr(EvalState & state, const Pos & pos, Value * * args, Value & v)
if (i == args[1]->attrs->end()) if (i == args[1]->attrs->end())
throw EvalError({ throw EvalError({
.hint = hintfmt("attribute '%1%' missing", attr), .hint = hintfmt("attribute '%1%' missing", attr),
.errPos = pos .nixCode = NixCode { .errPos = pos }
}); });
// !!! add to stack trace? // !!! add to stack trace?
if (state.countCalls && i->pos) state.attrSelects[*i->pos]++; if (state.countCalls && i->pos) state.attrSelects[*i->pos]++;
@@ -1372,7 +1371,7 @@ static void prim_listToAttrs(EvalState & state, const Pos & pos, Value * * args,
if (j == v2.attrs->end()) if (j == v2.attrs->end())
throw TypeError({ throw TypeError({
.hint = hintfmt("'name' attribute missing in a call to 'listToAttrs'"), .hint = hintfmt("'name' attribute missing in a call to 'listToAttrs'"),
.errPos = pos .nixCode = NixCode { .errPos = pos }
}); });
string name = state.forceStringNoCtx(*j->value, pos); string name = state.forceStringNoCtx(*j->value, pos);
@@ -1382,7 +1381,7 @@ static void prim_listToAttrs(EvalState & state, const Pos & pos, Value * * args,
if (j2 == v2.attrs->end()) if (j2 == v2.attrs->end())
throw TypeError({ throw TypeError({
.hint = hintfmt("'value' attribute missing in a call to 'listToAttrs'"), .hint = hintfmt("'value' attribute missing in a call to 'listToAttrs'"),
.errPos = pos .nixCode = NixCode { .errPos = pos }
}); });
v.attrs->push_back(Attr(sym, j2->value, j2->pos)); v.attrs->push_back(Attr(sym, j2->value, j2->pos));
} }
@@ -1458,7 +1457,7 @@ static void prim_functionArgs(EvalState & state, const Pos & pos, Value * * args
if (args[0]->type != tLambda) if (args[0]->type != tLambda)
throw TypeError({ throw TypeError({
.hint = hintfmt("'functionArgs' requires a function"), .hint = hintfmt("'functionArgs' requires a function"),
.errPos = pos .nixCode = NixCode { .errPos = pos }
}); });
if (!args[0]->lambda.fun->matchAttrs) { if (!args[0]->lambda.fun->matchAttrs) {
@@ -1514,7 +1513,7 @@ static void elemAt(EvalState & state, const Pos & pos, Value & list, int n, Valu
if (n < 0 || (unsigned int) n >= list.listSize()) if (n < 0 || (unsigned int) n >= list.listSize())
throw Error({ throw Error({
.hint = hintfmt("list index %1% is out of bounds", n), .hint = hintfmt("list index %1% is out of bounds", n),
.errPos = pos .nixCode = NixCode { .errPos = pos }
}); });
state.forceValue(*list.listElems()[n], pos); state.forceValue(*list.listElems()[n], pos);
v = *list.listElems()[n]; v = *list.listElems()[n];
@@ -1544,7 +1543,7 @@ static void prim_tail(EvalState & state, const Pos & pos, Value * * args, Value
if (args[0]->listSize() == 0) if (args[0]->listSize() == 0)
throw Error({ throw Error({
.hint = hintfmt("'tail' called on an empty list"), .hint = hintfmt("'tail' called on an empty list"),
.errPos = pos .nixCode = NixCode { .errPos = pos }
}); });
state.mkList(v, args[0]->listSize() - 1); state.mkList(v, args[0]->listSize() - 1);
@@ -1689,7 +1688,7 @@ static void prim_genList(EvalState & state, const Pos & pos, Value * * args, Val
if (len < 0) if (len < 0)
throw EvalError({ throw EvalError({
.hint = hintfmt("cannot create list of size %1%", len), .hint = hintfmt("cannot create list of size %1%", len),
.errPos = pos .nixCode = NixCode { .errPos = pos }
}); });
state.mkList(v, len); state.mkList(v, len);
@@ -1851,7 +1850,7 @@ static void prim_div(EvalState & state, const Pos & pos, Value * * args, Value &
if (f2 == 0) if (f2 == 0)
throw EvalError({ throw EvalError({
.hint = hintfmt("division by zero"), .hint = hintfmt("division by zero"),
.errPos = pos .nixCode = NixCode { .errPos = pos }
}); });
if (args[0]->type == tFloat || args[1]->type == tFloat) { if (args[0]->type == tFloat || args[1]->type == tFloat) {
@@ -1863,7 +1862,7 @@ static void prim_div(EvalState & state, const Pos & pos, Value * * args, Value &
if (i1 == std::numeric_limits<NixInt>::min() && i2 == -1) if (i1 == std::numeric_limits<NixInt>::min() && i2 == -1)
throw EvalError({ throw EvalError({
.hint = hintfmt("overflow in integer division"), .hint = hintfmt("overflow in integer division"),
.errPos = pos .nixCode = NixCode { .errPos = pos }
}); });
mkInt(v, i1 / i2); mkInt(v, i1 / i2);
@@ -1924,7 +1923,7 @@ static void prim_substring(EvalState & state, const Pos & pos, Value * * args, V
if (start < 0) if (start < 0)
throw EvalError({ throw EvalError({
.hint = hintfmt("negative start position in 'substring'"), .hint = hintfmt("negative start position in 'substring'"),
.errPos = pos .nixCode = NixCode { .errPos = pos }
}); });
mkString(v, (unsigned int) start >= s.size() ? "" : string(s, start, len), context); mkString(v, (unsigned int) start >= s.size() ? "" : string(s, start, len), context);
@@ -1947,7 +1946,7 @@ static void prim_hashString(EvalState & state, const Pos & pos, Value * * args,
if (!ht) if (!ht)
throw Error({ throw Error({
.hint = hintfmt("unknown hash type '%1%'", type), .hint = hintfmt("unknown hash type '%1%'", type),
.errPos = pos .nixCode = NixCode { .errPos = pos }
}); });
PathSet context; // discarded PathSet context; // discarded
@@ -1993,12 +1992,12 @@ void prim_match(EvalState & state, const Pos & pos, Value * * args, Value & v)
// limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++ // limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++
throw EvalError({ throw EvalError({
.hint = hintfmt("memory limit exceeded by regular expression '%s'", re), .hint = hintfmt("memory limit exceeded by regular expression '%s'", re),
.errPos = pos .nixCode = NixCode { .errPos = pos }
}); });
} else { } else {
throw EvalError({ throw EvalError({
.hint = hintfmt("invalid regular expression '%s'", re), .hint = hintfmt("invalid regular expression '%s'", re),
.errPos = pos .nixCode = NixCode { .errPos = pos }
}); });
} }
} }
@@ -2066,12 +2065,12 @@ static void prim_split(EvalState & state, const Pos & pos, Value * * args, Value
// limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++ // limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++
throw EvalError({ throw EvalError({
.hint = hintfmt("memory limit exceeded by regular expression '%s'", re), .hint = hintfmt("memory limit exceeded by regular expression '%s'", re),
.errPos = pos .nixCode = NixCode { .errPos = pos }
}); });
} else { } else {
throw EvalError({ throw EvalError({
.hint = hintfmt("invalid regular expression '%s'", re), .hint = hintfmt("invalid regular expression '%s'", re),
.errPos = pos .nixCode = NixCode { .errPos = pos }
}); });
} }
} }
@@ -2105,7 +2104,7 @@ static void prim_replaceStrings(EvalState & state, const Pos & pos, Value * * ar
if (args[0]->listSize() != args[1]->listSize()) if (args[0]->listSize() != args[1]->listSize())
throw EvalError({ throw EvalError({
.hint = hintfmt("'from' and 'to' arguments to 'replaceStrings' have different lengths"), .hint = hintfmt("'from' and 'to' arguments to 'replaceStrings' have different lengths"),
.errPos = pos .nixCode = NixCode { .errPos = pos }
}); });
vector<string> from; vector<string> from;
+3 -3
View File
@@ -148,7 +148,7 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg
if (!state.store->isStorePath(i.name)) if (!state.store->isStorePath(i.name))
throw EvalError({ throw EvalError({
.hint = hintfmt("Context key '%s' is not a store path", i.name), .hint = hintfmt("Context key '%s' is not a store path", i.name),
.errPos = *i.pos .nixCode = NixCode { .errPos = *i.pos }
}); });
if (!settings.readOnlyMode) if (!settings.readOnlyMode)
state.store->ensurePath(state.store->parseStorePath(i.name)); state.store->ensurePath(state.store->parseStorePath(i.name));
@@ -165,7 +165,7 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg
if (!isDerivation(i.name)) { if (!isDerivation(i.name)) {
throw EvalError({ throw EvalError({
.hint = hintfmt("Tried to add all-outputs context of %s, which is not a derivation, to a string", i.name), .hint = hintfmt("Tried to add all-outputs context of %s, which is not a derivation, to a string", i.name),
.errPos = *i.pos .nixCode = NixCode { .errPos = *i.pos }
}); });
} }
context.insert("=" + string(i.name)); context.insert("=" + string(i.name));
@@ -178,7 +178,7 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg
if (iter->value->listSize() && !isDerivation(i.name)) { if (iter->value->listSize() && !isDerivation(i.name)) {
throw EvalError({ throw EvalError({
.hint = hintfmt("Tried to add derivation output context of %s, which is not a derivation, to a string", i.name), .hint = hintfmt("Tried to add derivation output context of %s, which is not a derivation, to a string", i.name),
.errPos = *i.pos .nixCode = NixCode { .errPos = *i.pos }
}); });
} }
for (unsigned int n = 0; n < iter->value->listSize(); ++n) { for (unsigned int n = 0; n < iter->value->listSize(); ++n) {
+2 -2
View File
@@ -37,14 +37,14 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va
else else
throw EvalError({ throw EvalError({
.hint = hintfmt("unsupported argument '%s' to 'fetchGit'", attr.name), .hint = hintfmt("unsupported argument '%s' to 'fetchGit'", attr.name),
.errPos = *attr.pos .nixCode = NixCode { .errPos = *attr.pos }
}); });
} }
if (url.empty()) if (url.empty())
throw EvalError({ throw EvalError({
.hint = hintfmt("'url' argument required"), .hint = hintfmt("'url' argument required"),
.errPos = pos .nixCode = NixCode { .errPos = pos }
}); });
} else } else
+2 -2
View File
@@ -40,14 +40,14 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar
else else
throw EvalError({ throw EvalError({
.hint = hintfmt("unsupported argument '%s' to 'fetchMercurial'", attr.name), .hint = hintfmt("unsupported argument '%s' to 'fetchMercurial'", attr.name),
.errPos = *attr.pos .nixCode = NixCode { .errPos = *attr.pos }
}); });
} }
if (url.empty()) if (url.empty())
throw EvalError({ throw EvalError({
.hint = hintfmt("'url' argument required"), .hint = hintfmt("'url' argument required"),
.errPos = pos .nixCode = NixCode { .errPos = pos }
}); });
} else } else
+3 -3
View File
@@ -68,7 +68,7 @@ static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, V
if (!attrs.count("type")) if (!attrs.count("type"))
throw Error({ throw Error({
.hint = hintfmt("attribute 'type' is missing in call to 'fetchTree'"), .hint = hintfmt("attribute 'type' is missing in call to 'fetchTree'"),
.errPos = pos .nixCode = NixCode { .errPos = pos }
}); });
input = fetchers::inputFromAttrs(attrs); input = fetchers::inputFromAttrs(attrs);
@@ -112,14 +112,14 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
else else
throw EvalError({ throw EvalError({
.hint = hintfmt("unsupported argument '%s' to '%s'", attr.name, who), .hint = hintfmt("unsupported argument '%s' to '%s'", attr.name, who),
.errPos = *attr.pos .nixCode = NixCode { .errPos = *attr.pos }
}); });
} }
if (!url) if (!url)
throw EvalError({ throw EvalError({
.hint = hintfmt("'url' argument required"), .hint = hintfmt("'url' argument required"),
.errPos = pos .nixCode = NixCode { .errPos = pos }
}); });
} else } else
url = state.forceStringNoCtx(*args[0], pos); url = state.forceStringNoCtx(*args[0], pos);
+1 -1
View File
@@ -83,7 +83,7 @@ static void prim_fromTOML(EvalState & state, const Pos & pos, Value * * args, Va
} catch (std::runtime_error & e) { } catch (std::runtime_error & e) {
throw EvalError({ throw EvalError({
.hint = hintfmt("while parsing a TOML string: %s", e.what()), .hint = hintfmt("while parsing a TOML string: %s", e.what()),
.errPos = pos .nixCode = NixCode { .errPos = pos }
}); });
} }
} }
-2
View File
@@ -6,8 +6,6 @@ namespace nix::fetchers {
struct Cache struct Cache
{ {
virtual ~Cache() { }
virtual void add( virtual void add(
ref<Store> store, ref<Store> store,
const Attrs & inAttrs, const Attrs & inAttrs,
+1 -1
View File
@@ -26,7 +26,7 @@ Logger * makeDefaultLogger() {
case LogFormat::rawWithLogs: case LogFormat::rawWithLogs:
return makeSimpleLogger(true); return makeSimpleLogger(true);
case LogFormat::internalJson: case LogFormat::internalJson:
return makeJSONLogger(*makeSimpleLogger(true)); return makeJSONLogger(*makeSimpleLogger());
case LogFormat::bar: case LogFormat::bar:
return makeProgressBar(); return makeProgressBar();
case LogFormat::barWithLogs: case LogFormat::barWithLogs:
+1 -1
View File
@@ -131,7 +131,7 @@ public:
auto state(state_.lock()); auto state(state_.lock());
std::stringstream oss; std::stringstream oss;
showErrorInfo(oss, ei, loggerSettings.showTrace.get()); oss << ei;
log(*state, ei.level, oss.str()); log(*state, ei.level, oss.str());
} }
+3 -1
View File
@@ -323,8 +323,10 @@ int handleExceptions(const string & programName, std::function<void()> fun)
printError("Try '%1% --help' for more information.", programName); printError("Try '%1% --help' for more information.", programName);
return 1; return 1;
} catch (BaseError & e) { } catch (BaseError & e) {
if (settings.showTrace && e.prefix() != "")
printError(e.prefix());
logError(e.info()); logError(e.info());
if (e.hasTrace() && !loggerSettings.showTrace.get()) if (e.prefix() != "" && !settings.showTrace)
printError("(use '--show-trace' to show detailed location information)"); printError("(use '--show-trace' to show detailed location information)");
return e.status; return e.status;
} catch (std::bad_alloc & e) { } catch (std::bad_alloc & e) {
+47 -81
View File
@@ -15,7 +15,6 @@
#include <chrono> #include <chrono>
#include <future> #include <future>
#include <regex> #include <regex>
#include <fstream>
#include <nlohmann/json.hpp> #include <nlohmann/json.hpp>
@@ -58,13 +57,6 @@ void BinaryCacheStore::init()
} }
} }
void BinaryCacheStore::upsertFile(const std::string & path,
std::string && data,
const std::string & mimeType)
{
upsertFile(path, std::make_shared<std::stringstream>(std::move(data)), mimeType);
}
void BinaryCacheStore::getFile(const std::string & path, void BinaryCacheStore::getFile(const std::string & path,
Callback<std::shared_ptr<std::string>> callback) noexcept Callback<std::shared_ptr<std::string>> callback) noexcept
{ {
@@ -121,74 +113,13 @@ void BinaryCacheStore::writeNarInfo(ref<NarInfo> narInfo)
diskCache->upsertNarInfo(getUri(), hashPart, std::shared_ptr<NarInfo>(narInfo)); diskCache->upsertNarInfo(getUri(), hashPart, std::shared_ptr<NarInfo>(narInfo));
} }
AutoCloseFD openFile(const Path & path)
{
auto fd = open(path.c_str(), O_RDONLY | O_CLOEXEC);
if (!fd)
throw SysError("opening file '%1%'", path);
return fd;
}
struct FileSource : FdSource
{
AutoCloseFD fd2;
FileSource(const Path & path)
: fd2(openFile(path))
{
fd = fd2.get();
}
};
void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource, void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource,
RepairFlag repair, CheckSigsFlag checkSigs) RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor)
{ {
assert(info.narHash && info.narSize); // FIXME: See if we can use the original source to reduce memory usage.
auto nar = make_ref<std::string>(narSource.drain());
if (!repair && isValidPath(info.path)) { if (!repair && isValidPath(info.path)) return;
// FIXME: copyNAR -> null sink
narSource.drain();
return;
}
auto [fdTemp, fnTemp] = createTempFile();
auto now1 = std::chrono::steady_clock::now();
/* Read the NAR simultaneously into a CompressionSink+FileSink (to
write the compressed NAR to disk), into a HashSink (to get the
NAR hash), and into a NarAccessor (to get the NAR listing). */
HashSink fileHashSink(htSHA256);
std::shared_ptr<FSAccessor> narAccessor;
{
FdSink fileSink(fdTemp.get());
TeeSink teeSink(fileSink, fileHashSink);
auto compressionSink = makeCompressionSink(compression, teeSink);
TeeSource teeSource(narSource, *compressionSink);
narAccessor = makeNarAccessor(teeSource);
compressionSink->finish();
}
auto now2 = std::chrono::steady_clock::now();
auto narInfo = make_ref<NarInfo>(info);
narInfo->narSize = info.narSize;
narInfo->narHash = info.narHash;
narInfo->compression = compression;
auto [fileHash, fileSize] = fileHashSink.finish();
narInfo->fileHash = fileHash;
narInfo->fileSize = fileSize;
narInfo->url = "nar/" + narInfo->fileHash.to_string(Base32, false) + ".nar"
+ (compression == "xz" ? ".xz" :
compression == "bzip2" ? ".bz2" :
compression == "br" ? ".br" :
"");
auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
printMsg(lvlTalkative, "copying path '%1%' (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache",
printStorePath(narInfo->path), info.narSize,
((1.0 - (double) fileSize / info.narSize) * 100.0),
duration);
/* Verify that all references are valid. This may do some .narinfo /* Verify that all references are valid. This may do some .narinfo
reads, but typically they'll already be cached. */ reads, but typically they'll already be cached. */
@@ -201,6 +132,23 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
printStorePath(info.path), printStorePath(ref)); printStorePath(info.path), printStorePath(ref));
} }
assert(nar->compare(0, narMagic.size(), narMagic) == 0);
auto narInfo = make_ref<NarInfo>(info);
narInfo->narSize = nar->size();
narInfo->narHash = hashString(htSHA256, *nar);
if (info.narHash && info.narHash != narInfo->narHash)
throw Error("refusing to copy corrupted path '%1%' to binary cache", printStorePath(info.path));
auto accessor_ = std::dynamic_pointer_cast<RemoteFSAccessor>(accessor);
auto narAccessor = makeNarAccessor(nar);
if (accessor_)
accessor_->addToCache(printStorePath(info.path), *nar, narAccessor);
/* Optionally write a JSON file containing a listing of the /* Optionally write a JSON file containing a listing of the
contents of the NAR. */ contents of the NAR. */
if (writeNARListing) { if (writeNARListing) {
@@ -212,13 +160,33 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
{ {
auto res = jsonRoot.placeholder("root"); auto res = jsonRoot.placeholder("root");
listNar(res, ref<FSAccessor>(narAccessor), "", true); listNar(res, narAccessor, "", true);
} }
} }
upsertFile(std::string(info.path.to_string()) + ".ls", jsonOut.str(), "application/json"); upsertFile(std::string(info.path.to_string()) + ".ls", jsonOut.str(), "application/json");
} }
/* Compress the NAR. */
narInfo->compression = compression;
auto now1 = std::chrono::steady_clock::now();
auto narCompressed = compress(compression, *nar, parallelCompression);
auto now2 = std::chrono::steady_clock::now();
narInfo->fileHash = hashString(htSHA256, *narCompressed);
narInfo->fileSize = narCompressed->size();
auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
printMsg(lvlTalkative, "copying path '%1%' (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache",
printStorePath(narInfo->path), narInfo->narSize,
((1.0 - (double) narCompressed->size() / nar->size()) * 100.0),
duration);
narInfo->url = "nar/" + narInfo->fileHash.to_string(Base32, false) + ".nar"
+ (compression == "xz" ? ".xz" :
compression == "bzip2" ? ".bz2" :
compression == "br" ? ".br" :
"");
/* Optionally maintain an index of DWARF debug info files /* Optionally maintain an index of DWARF debug info files
consisting of JSON files named 'debuginfo/<build-id>' that consisting of JSON files named 'debuginfo/<build-id>' that
specify the NAR file and member containing the debug info. */ specify the NAR file and member containing the debug info. */
@@ -279,14 +247,12 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
/* Atomically write the NAR file. */ /* Atomically write the NAR file. */
if (repair || !fileExists(narInfo->url)) { if (repair || !fileExists(narInfo->url)) {
stats.narWrite++; stats.narWrite++;
upsertFile(narInfo->url, upsertFile(narInfo->url, *narCompressed, "application/x-nix-nar");
std::make_shared<std::fstream>(fnTemp, std::ios_base::in),
"application/x-nix-nar");
} else } else
stats.narWriteAverted++; stats.narWriteAverted++;
stats.narWriteBytes += info.narSize; stats.narWriteBytes += nar->size();
stats.narWriteCompressedBytes += fileSize; stats.narWriteCompressedBytes += narCompressed->size();
stats.narWriteCompressionTimeMs += duration; stats.narWriteCompressionTimeMs += duration;
/* Atomically write the NAR info file.*/ /* Atomically write the NAR info file.*/
@@ -385,7 +351,7 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath
ValidPathInfo info(makeFixedOutputPath(method, h, name)); ValidPathInfo info(makeFixedOutputPath(method, h, name));
auto source = StringSource { *sink.s }; auto source = StringSource { *sink.s };
addToStore(info, source, repair, CheckSigs); addToStore(info, source, repair, CheckSigs, nullptr);
return std::move(info.path); return std::move(info.path);
} }
@@ -400,7 +366,7 @@ StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s
StringSink sink; StringSink sink;
dumpString(s, sink); dumpString(s, sink);
auto source = StringSource { *sink.s }; auto source = StringSource { *sink.s };
addToStore(info, source, repair, CheckSigs); addToStore(info, source, repair, CheckSigs, nullptr);
} }
return std::move(info.path); return std::move(info.path);
+3 -6
View File
@@ -36,13 +36,9 @@ public:
virtual bool fileExists(const std::string & path) = 0; virtual bool fileExists(const std::string & path) = 0;
virtual void upsertFile(const std::string & path, virtual void upsertFile(const std::string & path,
std::shared_ptr<std::basic_iostream<char>> istream, const std::string & data,
const std::string & mimeType) = 0; const std::string & mimeType) = 0;
void upsertFile(const std::string & path,
std::string && data,
const std::string & mimeType);
/* Note: subclasses must implement at least one of the two /* Note: subclasses must implement at least one of the two
following getFile() methods. */ following getFile() methods. */
@@ -79,7 +75,8 @@ public:
{ unsupported("queryPathFromHashPart"); } { unsupported("queryPathFromHashPart"); }
void addToStore(const ValidPathInfo & info, Source & narSource, void addToStore(const ValidPathInfo & info, Source & narSource,
RepairFlag repair, CheckSigsFlag checkSigs) override; RepairFlag repair, CheckSigsFlag checkSigs,
std::shared_ptr<FSAccessor> accessor) override;
StorePath addToStore(const string & name, const Path & srcPath, StorePath addToStore(const string & name, const Path & srcPath,
FileIngestionMethod method, HashType hashAlgo, FileIngestionMethod method, HashType hashAlgo,
+9 -14
View File
@@ -1950,11 +1950,8 @@ void linkOrCopy(const Path & from, const Path & to)
/* Hard-linking fails if we exceed the maximum link count on a /* Hard-linking fails if we exceed the maximum link count on a
file (e.g. 32000 of ext3), which is quite possible after a file (e.g. 32000 of ext3), which is quite possible after a
'nix-store --optimise'. FIXME: actually, why don't we just 'nix-store --optimise'. FIXME: actually, why don't we just
bind-mount in this case? bind-mount in this case? */
if (errno != EMLINK)
It can also fail with EPERM in BeegFS v7 and earlier versions
which don't allow hard-links to other directories */
if (errno != EMLINK && errno != EPERM)
throw SysError("linking '%s' to '%s'", to, from); throw SysError("linking '%s' to '%s'", to, from);
copyPath(from, to); copyPath(from, to);
} }
@@ -2041,10 +2038,7 @@ void DerivationGoal::startBuilder()
if (!std::regex_match(fileName, regex)) if (!std::regex_match(fileName, regex))
throw Error("invalid file name '%s' in 'exportReferencesGraph'", fileName); throw Error("invalid file name '%s' in 'exportReferencesGraph'", fileName);
auto storePathS = *i++; auto storePath = worker.store.parseStorePath(*i++);
if (!worker.store.isInStore(storePathS))
throw BuildError("'exportReferencesGraph' contains a non-store path '%1%'", storePathS);
auto storePath = worker.store.toStorePath(storePathS).first;
/* Write closure info to <fileName>. */ /* Write closure info to <fileName>. */
writeFile(tmpDir + "/" + fileName, writeFile(tmpDir + "/" + fileName,
@@ -2083,7 +2077,7 @@ void DerivationGoal::startBuilder()
for (auto & i : dirsInChroot) for (auto & i : dirsInChroot)
try { try {
if (worker.store.isInStore(i.second.source)) if (worker.store.isInStore(i.second.source))
worker.store.computeFSClosure(worker.store.toStorePath(i.second.source).first, closure); worker.store.computeFSClosure(worker.store.parseStorePath(worker.store.toStorePath(i.second.source)), closure);
} catch (InvalidPath & e) { } catch (InvalidPath & e) {
} catch (Error & e) { } catch (Error & e) {
throw Error("while processing 'sandbox-paths': %s", e.what()); throw Error("while processing 'sandbox-paths': %s", e.what());
@@ -2756,8 +2750,8 @@ struct RestrictedStore : public LocalFSStore
void queryReferrers(const StorePath & path, StorePathSet & referrers) override void queryReferrers(const StorePath & path, StorePathSet & referrers) override
{ } { }
OutputPathMap queryDerivationOutputMap(const StorePath & path) override StorePathSet queryDerivationOutputs(const StorePath & path) override
{ throw Error("queryDerivationOutputMap"); } { throw Error("queryDerivationOutputs"); }
std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override
{ throw Error("queryPathFromHashPart"); } { throw Error("queryPathFromHashPart"); }
@@ -2768,9 +2762,10 @@ struct RestrictedStore : public LocalFSStore
{ throw Error("addToStore"); } { throw Error("addToStore"); }
void addToStore(const ValidPathInfo & info, Source & narSource, void addToStore(const ValidPathInfo & info, Source & narSource,
RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs) override RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs,
std::shared_ptr<FSAccessor> accessor = 0) override
{ {
next->addToStore(info, narSource, repair, checkSigs); next->addToStore(info, narSource, repair, checkSigs, accessor);
goal.addDependency(info.path); goal.addDependency(info.path);
} }
+3 -6
View File
@@ -58,16 +58,13 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
} }
}; };
/* We always have one output, and if it's a fixed-output derivation (as
checked below) it must be the only output */
auto & output = drv.outputs.begin()->second;
/* Try the hashed mirrors first. */ /* Try the hashed mirrors first. */
if (output.hash && output.hash->method == FileIngestionMethod::Flat) if (getAttr("outputHashMode") == "flat")
for (auto hashedMirror : settings.hashedMirrors.get()) for (auto hashedMirror : settings.hashedMirrors.get())
try { try {
if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/'; if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/';
auto & h = output.hash->hash; auto ht = parseHashTypeOpt(getAttr("outputHashAlgo"));
auto h = Hash(getAttr("outputHash"), ht);
fetch(hashedMirror + printHashType(*h.type) + "/" + h.to_string(Base16, false)); fetch(hashedMirror + printHashType(*h.type) + "/" + h.to_string(Base16, false));
return; return;
} catch (Error & e) { } catch (Error & e) {
-12
View File
@@ -82,16 +82,4 @@ std::string renderContentAddress(std::optional<ContentAddress> ca) {
return ca ? renderContentAddress(*ca) : ""; return ca ? renderContentAddress(*ca) : "";
} }
Hash getContentAddressHash(const ContentAddress & ca)
{
return std::visit(overloaded {
[](TextHash th) {
return th.hash;
},
[](FixedOutputHash fsh) {
return fsh.hash;
}
}, ca);
}
} }
-2
View File
@@ -53,6 +53,4 @@ ContentAddress parseContentAddress(std::string_view rawCa);
std::optional<ContentAddress> parseContentAddressOpt(std::string_view rawCaOpt); std::optional<ContentAddress> parseContentAddressOpt(std::string_view rawCaOpt);
Hash getContentAddressHash(const ContentAddress & ca);
} }
+9 -19
View File
@@ -78,10 +78,10 @@ struct TunnelLogger : public Logger
if (ei.level > verbosity) return; if (ei.level > verbosity) return;
std::stringstream oss; std::stringstream oss;
showErrorInfo(oss, ei, false); oss << ei;
StringSink buf; StringSink buf;
buf << STDERR_NEXT << oss.str(); buf << STDERR_NEXT << oss.str() << "\n"; // (fs.s + "\n");
enqueueMsg(*buf.s); enqueueMsg(*buf.s);
} }
@@ -347,15 +347,6 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
break; break;
} }
case wopQueryDerivationOutputMap: {
auto path = store->parseStorePath(readString(from));
logger->startWork();
OutputPathMap outputs = store->queryDerivationOutputMap(path);
logger->stopWork();
writeOutputPathMap(*store, to, outputs);
break;
}
case wopQueryDeriver: { case wopQueryDeriver: {
auto path = store->parseStorePath(readString(from)); auto path = store->parseStorePath(readString(from));
logger->startWork(); logger->startWork();
@@ -391,8 +382,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
} }
HashType hashAlgo = parseHashType(s); HashType hashAlgo = parseHashType(s);
StringSink savedNAR; TeeSource savedNAR(from);
TeeSource savedNARSource(from, savedNAR);
RetrieveRegularNARSink savedRegular; RetrieveRegularNARSink savedRegular;
if (method == FileIngestionMethod::Recursive) { if (method == FileIngestionMethod::Recursive) {
@@ -400,7 +390,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
a string so that we can pass it to a string so that we can pass it to
addToStoreFromDump(). */ addToStoreFromDump(). */
ParseSink sink; /* null sink; just parse the NAR */ ParseSink sink; /* null sink; just parse the NAR */
parseDump(sink, savedNARSource); parseDump(sink, savedNAR);
} else } else
parseDump(savedRegular, from); parseDump(savedRegular, from);
@@ -408,7 +398,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
if (!savedRegular.regular) throw Error("regular file expected"); if (!savedRegular.regular) throw Error("regular file expected");
auto path = store->addToStoreFromDump( auto path = store->addToStoreFromDump(
method == FileIngestionMethod::Recursive ? *savedNAR.s : savedRegular.s, method == FileIngestionMethod::Recursive ? *savedNAR.data : savedRegular.s,
baseName, baseName,
method, method,
hashAlgo); hashAlgo);
@@ -443,7 +433,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
case wopImportPaths: { case wopImportPaths: {
logger->startWork(); logger->startWork();
TunnelSource source(from, to); TunnelSource source(from, to);
auto paths = store->importPaths(source, auto paths = store->importPaths(source, nullptr,
trusted ? NoCheckSigs : CheckSigs); trusted ? NoCheckSigs : CheckSigs);
logger->stopWork(); logger->stopWork();
Strings paths2; Strings paths2;
@@ -732,9 +722,9 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
if (GET_PROTOCOL_MINOR(clientVersion) >= 21) if (GET_PROTOCOL_MINOR(clientVersion) >= 21)
source = std::make_unique<TunnelSource>(from, to); source = std::make_unique<TunnelSource>(from, to);
else { else {
TeeParseSink tee(from); TeeSink tee(from);
parseDump(tee, tee.source); parseDump(tee, tee.source);
saved = std::move(*tee.saved.s); saved = std::move(*tee.source.data);
source = std::make_unique<StringSource>(saved); source = std::make_unique<StringSource>(saved);
} }
@@ -742,7 +732,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
// FIXME: race if addToStore doesn't read source? // FIXME: race if addToStore doesn't read source?
store->addToStore(info, *source, (RepairFlag) repair, store->addToStore(info, *source, (RepairFlag) repair,
dontCheckSigs ? NoCheckSigs : CheckSigs); dontCheckSigs ? NoCheckSigs : CheckSigs, nullptr);
logger->stopWork(); logger->stopWork();
break; break;
+10 -14
View File
@@ -4,6 +4,7 @@
#include "util.hh" #include "util.hh"
#include "worker-protocol.hh" #include "worker-protocol.hh"
#include "fs-accessor.hh" #include "fs-accessor.hh"
#include "istringstream_nocopy.hh"
namespace nix { namespace nix {
@@ -100,7 +101,7 @@ static StringSet parseStrings(std::istream & str, bool arePaths)
} }
static DerivationOutput parseDerivationOutput(const Store & store, std::istringstream & str) static DerivationOutput parseDerivationOutput(const Store & store, istringstream_nocopy & str)
{ {
expect(str, ","); auto path = store.parseStorePath(parsePath(str)); expect(str, ","); auto path = store.parseStorePath(parsePath(str));
expect(str, ","); auto hashAlgo = parseString(str); expect(str, ","); auto hashAlgo = parseString(str);
@@ -128,10 +129,10 @@ static DerivationOutput parseDerivationOutput(const Store & store, std::istrings
} }
static Derivation parseDerivation(const Store & store, std::string && s) static Derivation parseDerivation(const Store & store, const string & s)
{ {
Derivation drv; Derivation drv;
std::istringstream str(std::move(s)); istringstream_nocopy str(s);
expect(str, "Derive(["); expect(str, "Derive([");
/* Parse the list of outputs. */ /* Parse the list of outputs. */
@@ -403,7 +404,7 @@ static DerivationOutput readDerivationOutput(Source & in, const Store & store)
{ {
auto path = store.parseStorePath(readString(in)); auto path = store.parseStorePath(readString(in));
auto hashAlgo = readString(in); auto hashAlgo = readString(in);
auto hash = readString(in); const auto hash = readString(in);
std::optional<FixedOutputHash> fsh; std::optional<FixedOutputHash> fsh;
if (hashAlgo != "") { if (hashAlgo != "") {
@@ -412,7 +413,7 @@ static DerivationOutput readDerivationOutput(Source & in, const Store & store)
method = FileIngestionMethod::Recursive; method = FileIngestionMethod::Recursive;
hashAlgo = string(hashAlgo, 2); hashAlgo = string(hashAlgo, 2);
} }
auto hashType = parseHashType(hashAlgo); const HashType hashType = parseHashType(hashAlgo);
fsh = FixedOutputHash { fsh = FixedOutputHash {
.method = std::move(method), .method = std::move(method),
.hash = Hash(hash, hashType), .hash = Hash(hash, hashType),
@@ -462,16 +463,11 @@ Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv)
void writeDerivation(Sink & out, const Store & store, const BasicDerivation & drv) void writeDerivation(Sink & out, const Store & store, const BasicDerivation & drv)
{ {
out << drv.outputs.size(); out << drv.outputs.size();
for (auto & i : drv.outputs) { for (auto & i : drv.outputs)
out << i.first out << i.first
<< store.printStorePath(i.second.path); << store.printStorePath(i.second.path)
if (i.second.hash) { << i.second.hash->printMethodAlgo()
out << i.second.hash->printMethodAlgo() << i.second.hash->hash.to_string(Base16, false);
<< i.second.hash->hash.to_string(Base16, false);
} else {
out << "" << "";
}
}
writeStorePaths(store, out, drv.inputSrcs); writeStorePaths(store, out, drv.inputSrcs);
out << drv.platform << drv.builder << drv.args; out << drv.platform << drv.builder << drv.args;
out << drv.env.size(); out << drv.env.size();
+30 -13
View File
@@ -7,6 +7,24 @@
namespace nix { namespace nix {
struct HashAndWriteSink : Sink
{
Sink & writeSink;
HashSink hashSink;
HashAndWriteSink(Sink & writeSink) : writeSink(writeSink), hashSink(htSHA256)
{
}
virtual void operator () (const unsigned char * data, size_t len)
{
writeSink(data, len);
hashSink(data, len);
}
Hash currentHash()
{
return hashSink.currentHash().first;
}
};
void Store::exportPaths(const StorePathSet & paths, Sink & sink) void Store::exportPaths(const StorePathSet & paths, Sink & sink)
{ {
auto sorted = topoSortPaths(paths); auto sorted = topoSortPaths(paths);
@@ -29,29 +47,28 @@ void Store::exportPath(const StorePath & path, Sink & sink)
{ {
auto info = queryPathInfo(path); auto info = queryPathInfo(path);
HashSink hashSink(htSHA256); HashAndWriteSink hashAndWriteSink(sink);
TeeSink teeSink(sink, hashSink);
narFromPath(path, teeSink); narFromPath(path, hashAndWriteSink);
/* Refuse to export paths that have changed. This prevents /* Refuse to export paths that have changed. This prevents
filesystem corruption from spreading to other machines. filesystem corruption from spreading to other machines.
Don't complain if the stored hash is zero (unknown). */ Don't complain if the stored hash is zero (unknown). */
Hash hash = hashSink.currentHash().first; Hash hash = hashAndWriteSink.currentHash();
if (hash != info->narHash && info->narHash != Hash(*info->narHash.type)) if (hash != info->narHash && info->narHash != Hash(*info->narHash.type))
throw Error("hash of path '%s' has changed from '%s' to '%s'!", throw Error("hash of path '%s' has changed from '%s' to '%s'!",
printStorePath(path), info->narHash.to_string(Base32, true), hash.to_string(Base32, true)); printStorePath(path), info->narHash.to_string(Base32, true), hash.to_string(Base32, true));
teeSink hashAndWriteSink
<< exportMagic << exportMagic
<< printStorePath(path); << printStorePath(path);
writeStorePaths(*this, teeSink, info->references); writeStorePaths(*this, hashAndWriteSink, info->references);
teeSink hashAndWriteSink
<< (info->deriver ? printStorePath(*info->deriver) : "") << (info->deriver ? printStorePath(*info->deriver) : "")
<< 0; << 0;
} }
StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs) StorePaths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> accessor, CheckSigsFlag checkSigs)
{ {
StorePaths res; StorePaths res;
while (true) { while (true) {
@@ -60,7 +77,7 @@ StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs)
if (n != 1) throw Error("input doesn't look like something created by 'nix-store --export'"); if (n != 1) throw Error("input doesn't look like something created by 'nix-store --export'");
/* Extract the NAR from the source. */ /* Extract the NAR from the source. */
TeeParseSink tee(source); TeeSink tee(source);
parseDump(tee, tee.source); parseDump(tee, tee.source);
uint32_t magic = readInt(source); uint32_t magic = readInt(source);
@@ -77,16 +94,16 @@ StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs)
if (deriver != "") if (deriver != "")
info.deriver = parseStorePath(deriver); info.deriver = parseStorePath(deriver);
info.narHash = hashString(htSHA256, *tee.saved.s); info.narHash = hashString(htSHA256, *tee.source.data);
info.narSize = tee.saved.s->size(); info.narSize = tee.source.data->size();
// Ignore optional legacy signature. // Ignore optional legacy signature.
if (readInt(source) == 1) if (readInt(source) == 1)
readString(source); readString(source);
// Can't use underlying source, which would have been exhausted // Can't use underlying source, which would have been exhausted
auto source = StringSource { *tee.saved.s }; auto source = StringSource { *tee.source.data };
addToStore(info, source, NoRepair, checkSigs); addToStore(info, source, NoRepair, checkSigs, accessor);
res.push_back(info.path); res.push_back(info.path);
} }
+14 -16
View File
@@ -262,13 +262,11 @@ void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor)
void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots) void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots)
{ {
auto foundRoot = [&](const Path & path, const Path & target) { auto foundRoot = [&](const Path & path, const Path & target) {
try { auto storePath = maybeParseStorePath(toStorePath(target));
auto storePath = toStorePath(target).first; if (storePath && isValidPath(*storePath))
if (isValidPath(storePath)) roots[std::move(*storePath)].emplace(path);
roots[std::move(storePath)].emplace(path); else
else printInfo("skipping invalid root from '%1%' to '%2%'", path, target);
printInfo("skipping invalid root from '%1%' to '%2%'", path, target);
} catch (BadStorePath &) { }
}; };
try { try {
@@ -474,15 +472,15 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
for (auto & [target, links] : unchecked) { for (auto & [target, links] : unchecked) {
if (!isInStore(target)) continue; if (!isInStore(target)) continue;
try { Path pathS = toStorePath(target);
auto path = toStorePath(target).first; if (!isStorePath(pathS)) continue;
if (!isValidPath(path)) continue; auto path = parseStorePath(pathS);
debug("got additional root '%1%'", printStorePath(path)); if (!isValidPath(path)) continue;
if (censor) debug("got additional root '%1%'", pathS);
roots[path].insert(censored); if (censor)
else roots[path].insert(censored);
roots[path].insert(links.begin(), links.end()); else
} catch (BadStorePath &) { } roots[path].insert(links.begin(), links.end());
} }
} }
+10 -3
View File
@@ -2,6 +2,7 @@
#include "util.hh" #include "util.hh"
#include "archive.hh" #include "archive.hh"
#include "args.hh" #include "args.hh"
#include "sync.hh"
#include <algorithm> #include <algorithm>
#include <map> #include <map>
@@ -35,7 +36,7 @@ Settings::Settings()
, nixLibexecDir(canonPath(getEnv("NIX_LIBEXEC_DIR").value_or(NIX_LIBEXEC_DIR))) , nixLibexecDir(canonPath(getEnv("NIX_LIBEXEC_DIR").value_or(NIX_LIBEXEC_DIR)))
, nixBinDir(canonPath(getEnv("NIX_BIN_DIR").value_or(NIX_BIN_DIR))) , nixBinDir(canonPath(getEnv("NIX_BIN_DIR").value_or(NIX_BIN_DIR)))
, nixManDir(canonPath(NIX_MAN_DIR)) , nixManDir(canonPath(NIX_MAN_DIR))
, nixDaemonSocketFile(canonPath(getEnv("NIX_DAEMON_SOCKET_PATH").value_or(nixStateDir + DEFAULT_SOCKET_PATH))) , nixDaemonSocketFile(canonPath(nixStateDir + DEFAULT_SOCKET_PATH))
{ {
buildUsersGroup = getuid() == 0 ? "nixbld" : ""; buildUsersGroup = getuid() == 0 ? "nixbld" : "";
lockCPU = getEnv("NIX_AFFINITY_HACK") == "1"; lockCPU = getEnv("NIX_AFFINITY_HACK") == "1";
@@ -129,8 +130,14 @@ bool Settings::isExperimentalFeatureEnabled(const std::string & name)
void Settings::requireExperimentalFeature(const std::string & name) void Settings::requireExperimentalFeature(const std::string & name)
{ {
if (!isExperimentalFeatureEnabled(name)) if (!isExperimentalFeatureEnabled(name)) {
throw Error("experimental Nix feature '%1%' is disabled; use '--experimental-features %1%' to override", name); if (allowExperimentalFeatures) {
static Sync<std::unordered_set<std::string>> warned;
if (warned.lock()->insert(name).second)
warn("feature '%s' is experimental", name);
} else
throw Error("experimental Nix feature '%1%' is disabled; use '--experimental-features %1%' to override", name);
}
} }
bool Settings::isWSL1() bool Settings::isWSL1()
+8 -3
View File
@@ -196,6 +196,10 @@ public:
/* Whether to lock the Nix client and worker to the same CPU. */ /* Whether to lock the Nix client and worker to the same CPU. */
bool lockCPU; bool lockCPU;
/* Whether to show a stack trace if Nix evaluation fails. */
Setting<bool> showTrace{this, false, "show-trace",
"Whether to show a stack trace on evaluation errors."};
Setting<SandboxMode> sandboxMode{this, Setting<SandboxMode> sandboxMode{this,
#if __linux__ #if __linux__
smEnabled smEnabled
@@ -353,6 +357,10 @@ public:
Setting<std::string> githubAccessToken{this, "", "github-access-token", Setting<std::string> githubAccessToken{this, "", "github-access-token",
"GitHub access token to get access to GitHub data through the GitHub API for github:<..> flakes."}; "GitHub access token to get access to GitHub data through the GitHub API for github:<..> flakes."};
Setting<bool> allowExperimentalFeatures{this, true, "allow-experimental-features",
"Whether the use of experimental features other than those listed in "
"the option 'experimental-features' gives a warning rather than fatal error."};
Setting<Strings> experimentalFeatures{this, {}, "experimental-features", Setting<Strings> experimentalFeatures{this, {}, "experimental-features",
"Experimental Nix features to enable."}; "Experimental Nix features to enable."};
@@ -365,9 +373,6 @@ public:
Setting<bool> warnDirty{this, true, "warn-dirty", Setting<bool> warnDirty{this, true, "warn-dirty",
"Whether to warn about dirty Git/Mercurial trees."}; "Whether to warn about dirty Git/Mercurial trees."};
Setting<size_t> narBufferSize{this, 32 * 1024 * 1024, "nar-buffer-size",
"Maximum size of NARs before spilling them to disk."};
}; };
+2 -2
View File
@@ -100,11 +100,11 @@ protected:
} }
void upsertFile(const std::string & path, void upsertFile(const std::string & path,
std::shared_ptr<std::basic_iostream<char>> istream, const std::string & data,
const std::string & mimeType) override const std::string & mimeType) override
{ {
auto req = FileTransferRequest(cacheUri + "/" + path); auto req = FileTransferRequest(cacheUri + "/" + path);
req.data = std::make_shared<string>(StreamToSourceAdapter(istream).drain()); req.data = std::make_shared<string>(data); // FIXME: inefficient
req.mimeType = mimeType; req.mimeType = mimeType;
try { try {
getFileTransfer()->upload(req); getFileTransfer()->upload(req);
+2 -1
View File
@@ -126,7 +126,8 @@ struct LegacySSHStore : public Store
} }
void addToStore(const ValidPathInfo & info, Source & source, void addToStore(const ValidPathInfo & info, Source & source,
RepairFlag repair, CheckSigsFlag checkSigs) override RepairFlag repair, CheckSigsFlag checkSigs,
std::shared_ptr<FSAccessor> accessor) override
{ {
debug("adding path '%s' to remote host '%s'", printStorePath(info.path), host); debug("adding path '%s' to remote host '%s'", printStorePath(info.path), host);
+20 -15
View File
@@ -31,18 +31,8 @@ protected:
bool fileExists(const std::string & path) override; bool fileExists(const std::string & path) override;
void upsertFile(const std::string & path, void upsertFile(const std::string & path,
std::shared_ptr<std::basic_iostream<char>> istream, const std::string & data,
const std::string & mimeType) override const std::string & mimeType) override;
{
auto path2 = binaryCacheDir + "/" + path;
Path tmp = path2 + ".tmp." + std::to_string(getpid());
AutoDelete del(tmp, false);
StreamToSourceAdapter source(istream);
writeFile(tmp, source);
if (rename(tmp.c_str(), path2.c_str()))
throw SysError("renaming '%1%' to '%2%'", tmp, path2);
del.cancel();
}
void getFile(const std::string & path, Sink & sink) override void getFile(const std::string & path, Sink & sink) override
{ {
@@ -62,9 +52,7 @@ protected:
if (entry.name.size() != 40 || if (entry.name.size() != 40 ||
!hasSuffix(entry.name, ".narinfo")) !hasSuffix(entry.name, ".narinfo"))
continue; continue;
paths.insert(parseStorePath( paths.insert(parseStorePath(storeDir + "/" + entry.name.substr(0, entry.name.size() - 8)));
storeDir + "/" + entry.name.substr(0, entry.name.size() - 8)
+ "-" + MissingName));
} }
return paths; return paths;
@@ -80,11 +68,28 @@ void LocalBinaryCacheStore::init()
BinaryCacheStore::init(); BinaryCacheStore::init();
} }
static void atomicWrite(const Path & path, const std::string & s)
{
Path tmp = path + ".tmp." + std::to_string(getpid());
AutoDelete del(tmp, false);
writeFile(tmp, s);
if (rename(tmp.c_str(), path.c_str()))
throw SysError("renaming '%1%' to '%2%'", tmp, path);
del.cancel();
}
bool LocalBinaryCacheStore::fileExists(const std::string & path) bool LocalBinaryCacheStore::fileExists(const std::string & path)
{ {
return pathExists(binaryCacheDir + "/" + path); return pathExists(binaryCacheDir + "/" + path);
} }
void LocalBinaryCacheStore::upsertFile(const std::string & path,
const std::string & data,
const std::string & mimeType)
{
atomicWrite(binaryCacheDir + "/" + path, data);
}
static RegisterStoreImplementation regStore([]( static RegisterStoreImplementation regStore([](
const std::string & uri, const Store::Params & params) const std::string & uri, const Store::Params & params)
-> std::shared_ptr<Store> -> std::shared_ptr<Store>
+3 -3
View File
@@ -20,9 +20,9 @@ struct LocalStoreAccessor : public FSAccessor
Path toRealPath(const Path & path) Path toRealPath(const Path & path)
{ {
auto storePath = store->toStorePath(path).first; Path storePath = store->toStorePath(path);
if (!store->isValidPath(storePath)) if (!store->isValidPath(store->parseStorePath(storePath)))
throw InvalidPath("path '%1%' is not a valid store path", store->printStorePath(storePath)); throw InvalidPath("path '%1%' is not a valid store path", storePath);
return store->getRealStoreDir() + std::string(path, store->storeDir.size()); return store->getRealStoreDir() + std::string(path, store->storeDir.size());
} }
+19 -123
View File
@@ -774,20 +774,17 @@ StorePathSet LocalStore::queryValidDerivers(const StorePath & path)
} }
OutputPathMap LocalStore::queryDerivationOutputMap(const StorePath & path) StorePathSet LocalStore::queryDerivationOutputs(const StorePath & path)
{ {
return retrySQLite<OutputPathMap>([&]() { return retrySQLite<StorePathSet>([&]() {
auto state(_state.lock()); auto state(_state.lock());
auto useQueryDerivationOutputs(state->stmtQueryDerivationOutputs.use() auto useQueryDerivationOutputs(state->stmtQueryDerivationOutputs.use()
(queryValidPathId(*state, path))); (queryValidPathId(*state, path)));
OutputPathMap outputs; StorePathSet outputs;
while (useQueryDerivationOutputs.next()) while (useQueryDerivationOutputs.next())
outputs.emplace( outputs.insert(parseStorePath(useQueryDerivationOutputs.getStr(1)));
useQueryDerivationOutputs.getStr(0),
parseStorePath(useQueryDerivationOutputs.getStr(1))
);
return outputs; return outputs;
}); });
@@ -962,7 +959,7 @@ const PublicKeys & LocalStore::getPublicKeys()
void LocalStore::addToStore(const ValidPathInfo & info, Source & source, void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
RepairFlag repair, CheckSigsFlag checkSigs) RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor)
{ {
if (!info.narHash) if (!info.narHash)
throw Error("cannot add path '%s' because it lacks a hash", printStorePath(info.path)); throw Error("cannot add path '%s' because it lacks a hash", printStorePath(info.path));
@@ -976,7 +973,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
PathLocks outputLock; PathLocks outputLock;
auto realPath = Store::toRealPath(info.path); Path realPath = realStoreDir + "/" + std::string(info.path.to_string());
/* Lock the output path. But don't lock if we're being called /* Lock the output path. But don't lock if we're being called
from a build hook (whose parent process already acquired a from a build hook (whose parent process already acquired a
@@ -1047,7 +1044,8 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam
/* The first check above is an optimisation to prevent /* The first check above is an optimisation to prevent
unnecessary lock acquisition. */ unnecessary lock acquisition. */
auto realPath = Store::toRealPath(dstPath); Path realPath = realStoreDir + "/";
realPath += dstPath.to_string();
PathLocks outputLock({realPath}); PathLocks outputLock({realPath});
@@ -1097,119 +1095,16 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath,
{ {
Path srcPath(absPath(_srcPath)); Path srcPath(absPath(_srcPath));
if (method != FileIngestionMethod::Recursive) /* Read the whole path into memory. This is not a very scalable
return addToStoreFromDump(readFile(srcPath), name, method, hashAlgo, repair); method for very large paths, but `copyPath' is mainly used for
small files. */
StringSink sink;
if (method == FileIngestionMethod::Recursive)
dumpPath(srcPath, sink, filter);
else
sink.s = make_ref<std::string>(readFile(srcPath));
/* For computing the NAR hash. */ return addToStoreFromDump(*sink.s, name, method, hashAlgo, repair);
auto sha256Sink = std::make_unique<HashSink>(htSHA256);
/* For computing the store path. In recursive SHA-256 mode, this
is the same as the NAR hash, so no need to do it again. */
std::unique_ptr<HashSink> hashSink =
hashAlgo == htSHA256
? nullptr
: std::make_unique<HashSink>(hashAlgo);
/* Read the source path into memory, but only if it's up to
narBufferSize bytes. If it's larger, write it to a temporary
location in the Nix store. If the subsequently computed
destination store path is already valid, we just delete the
temporary path. Otherwise, we move it to the destination store
path. */
bool inMemory = true;
std::string nar;
auto source = sinkToSource([&](Sink & sink) {
LambdaSink sink2([&](const unsigned char * buf, size_t len) {
(*sha256Sink)(buf, len);
if (hashSink) (*hashSink)(buf, len);
if (inMemory) {
if (nar.size() + len > settings.narBufferSize) {
inMemory = false;
sink << 1;
sink((const unsigned char *) nar.data(), nar.size());
nar.clear();
} else {
nar.append((const char *) buf, len);
}
}
if (!inMemory) sink(buf, len);
});
dumpPath(srcPath, sink2, filter);
});
std::unique_ptr<AutoDelete> delTempDir;
Path tempPath;
try {
/* Wait for the source coroutine to give us some dummy
data. This is so that we don't create the temporary
directory if the NAR fits in memory. */
readInt(*source);
auto tempDir = createTempDir(realStoreDir, "add");
delTempDir = std::make_unique<AutoDelete>(tempDir);
tempPath = tempDir + "/x";
restorePath(tempPath, *source);
} catch (EndOfFile &) {
if (!inMemory) throw;
/* The NAR fits in memory, so we didn't do restorePath(). */
}
auto sha256 = sha256Sink->finish();
Hash hash = hashSink ? hashSink->finish().first : sha256.first;
auto dstPath = makeFixedOutputPath(method, hash, name);
addTempRoot(dstPath);
if (repair || !isValidPath(dstPath)) {
/* The first check above is an optimisation to prevent
unnecessary lock acquisition. */
auto realPath = Store::toRealPath(dstPath);
PathLocks outputLock({realPath});
if (repair || !isValidPath(dstPath)) {
deletePath(realPath);
autoGC();
if (inMemory) {
/* Restore from the NAR in memory. */
StringSource source(nar);
restorePath(realPath, source);
} else {
/* Move the temporary path we restored above. */
if (rename(tempPath.c_str(), realPath.c_str()))
throw Error("renaming '%s' to '%s'", tempPath, realPath);
}
canonicalisePathMetaData(realPath, -1); // FIXME: merge into restorePath
optimisePath(realPath);
ValidPathInfo info(dstPath);
info.narHash = sha256.first;
info.narSize = sha256.second;
info.ca = FixedOutputHash { .method = method, .hash = hash };
registerValidPath(info);
}
outputLock.setDeletion(true);
}
return dstPath;
} }
@@ -1223,7 +1118,8 @@ StorePath LocalStore::addTextToStore(const string & name, const string & s,
if (repair || !isValidPath(dstPath)) { if (repair || !isValidPath(dstPath)) {
auto realPath = Store::toRealPath(dstPath); Path realPath = realStoreDir + "/";
realPath += dstPath.to_string();
PathLocks outputLock({realPath}); PathLocks outputLock({realPath});
+3 -2
View File
@@ -133,7 +133,7 @@ public:
StorePathSet queryValidDerivers(const StorePath & path) override; StorePathSet queryValidDerivers(const StorePath & path) override;
OutputPathMap queryDerivationOutputMap(const StorePath & path) override; StorePathSet queryDerivationOutputs(const StorePath & path) override;
std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override; std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override;
@@ -143,7 +143,8 @@ public:
SubstitutablePathInfos & infos) override; SubstitutablePathInfos & infos) override;
void addToStore(const ValidPathInfo & info, Source & source, void addToStore(const ValidPathInfo & info, Source & source,
RepairFlag repair, CheckSigsFlag checkSigs) override; RepairFlag repair, CheckSigsFlag checkSigs,
std::shared_ptr<FSAccessor> accessor) override;
StorePath addToStore(const string & name, const Path & srcPath, StorePath addToStore(const string & name, const Path & srcPath,
FileIngestionMethod method, HashType hashAlgo, FileIngestionMethod method, HashType hashAlgo,
+16 -30
View File
@@ -18,7 +18,7 @@ struct NarMember
/* If this is a regular file, position of the contents of this /* If this is a regular file, position of the contents of this
file in the NAR. */ file in the NAR. */
uint64_t start = 0, size = 0; size_t start = 0, size = 0;
std::string target; std::string target;
@@ -34,19 +34,17 @@ struct NarAccessor : public FSAccessor
NarMember root; NarMember root;
struct NarIndexer : ParseSink, Source struct NarIndexer : ParseSink, StringSource
{ {
NarAccessor & acc; NarAccessor & acc;
Source & source;
std::stack<NarMember *> parents; std::stack<NarMember *> parents;
std::string currentStart;
bool isExec = false; bool isExec = false;
uint64_t pos = 0; NarIndexer(NarAccessor & acc, const std::string & nar)
: StringSource(nar), acc(acc)
NarIndexer(NarAccessor & acc, Source & source)
: acc(acc), source(source)
{ } { }
void createMember(const Path & path, NarMember member) { void createMember(const Path & path, NarMember member) {
@@ -81,38 +79,31 @@ struct NarAccessor : public FSAccessor
void preallocateContents(unsigned long long size) override void preallocateContents(unsigned long long size) override
{ {
assert(size <= std::numeric_limits<uint64_t>::max()); currentStart = string(s, pos, 16);
parents.top()->size = (uint64_t) size; assert(size <= std::numeric_limits<size_t>::max());
parents.top()->size = (size_t)size;
parents.top()->start = pos; parents.top()->start = pos;
} }
void receiveContents(unsigned char * data, unsigned int len) override void receiveContents(unsigned char * data, unsigned int len) override
{ } {
// Sanity check
if (!currentStart.empty()) {
assert(len < 16 || currentStart == string((char *) data, 16));
currentStart.clear();
}
}
void createSymlink(const Path & path, const string & target) override void createSymlink(const Path & path, const string & target) override
{ {
createMember(path, createMember(path,
NarMember{FSAccessor::Type::tSymlink, false, 0, 0, target}); NarMember{FSAccessor::Type::tSymlink, false, 0, 0, target});
} }
size_t read(unsigned char * data, size_t len) override
{
auto n = source.read(data, len);
pos += n;
return n;
}
}; };
NarAccessor(ref<const std::string> nar) : nar(nar) NarAccessor(ref<const std::string> nar) : nar(nar)
{ {
StringSource source(*nar); NarIndexer indexer(*this, *nar);
NarIndexer indexer(*this, source);
parseDump(indexer, indexer);
}
NarAccessor(Source & source)
{
NarIndexer indexer(*this, source);
parseDump(indexer, indexer); parseDump(indexer, indexer);
} }
@@ -228,11 +219,6 @@ ref<FSAccessor> makeNarAccessor(ref<const std::string> nar)
return make_ref<NarAccessor>(nar); return make_ref<NarAccessor>(nar);
} }
ref<FSAccessor> makeNarAccessor(Source & source)
{
return make_ref<NarAccessor>(source);
}
ref<FSAccessor> makeLazyNarAccessor(const std::string & listing, ref<FSAccessor> makeLazyNarAccessor(const std::string & listing,
GetNarBytes getNarBytes) GetNarBytes getNarBytes)
{ {
-4
View File
@@ -6,14 +6,10 @@
namespace nix { namespace nix {
struct Source;
/* Return an object that provides access to the contents of a NAR /* Return an object that provides access to the contents of a NAR
file. */ file. */
ref<FSAccessor> makeNarAccessor(ref<const std::string> nar); ref<FSAccessor> makeNarAccessor(ref<const std::string> nar);
ref<FSAccessor> makeNarAccessor(Source & source);
/* Create a NAR accessor from a NAR listing (in the format produced by /* Create a NAR accessor from a NAR listing (in the format produced by
listNar()). The callback getNarBytes(offset, length) is used by the listNar()). The callback getNarBytes(offset, length) is used by the
readFile() method of the accessor to get the contents of files readFile() method of the accessor to get the contents of files
+2
View File
@@ -2,6 +2,8 @@
namespace nix { namespace nix {
MakeError(BadStorePath, Error);
static void checkName(std::string_view path, std::string_view name) static void checkName(std::string_view path, std::string_view name)
{ {
if (name.empty()) if (name.empty())
-1
View File
@@ -62,7 +62,6 @@ public:
typedef std::set<StorePath> StorePathSet; typedef std::set<StorePath> StorePathSet;
typedef std::vector<StorePath> StorePaths; typedef std::vector<StorePath> StorePaths;
typedef std::map<string, StorePath> OutputPathMap;
/* Extension of derivations in the Nix store. */ /* Extension of derivations in the Nix store. */
const std::string drvExtension = ".drv"; const std::string drvExtension = ".drv";
+17 -16
View File
@@ -16,26 +16,26 @@ RemoteFSAccessor::RemoteFSAccessor(ref<Store> store, const Path & cacheDir)
createDirs(cacheDir); createDirs(cacheDir);
} }
Path RemoteFSAccessor::makeCacheFile(std::string_view hashPart, const std::string & ext) Path RemoteFSAccessor::makeCacheFile(const Path & storePath, const std::string & ext)
{ {
assert(cacheDir != ""); assert(cacheDir != "");
return fmt("%s/%s.%s", cacheDir, hashPart, ext); return fmt("%s/%s.%s", cacheDir, store->parseStorePath(storePath).hashPart(), ext);
} }
void RemoteFSAccessor::addToCache(std::string_view hashPart, const std::string & nar, void RemoteFSAccessor::addToCache(const Path & storePath, const std::string & nar,
ref<FSAccessor> narAccessor) ref<FSAccessor> narAccessor)
{ {
nars.emplace(hashPart, narAccessor); nars.emplace(storePath, narAccessor);
if (cacheDir != "") { if (cacheDir != "") {
try { try {
std::ostringstream str; std::ostringstream str;
JSONPlaceholder jsonRoot(str); JSONPlaceholder jsonRoot(str);
listNar(jsonRoot, narAccessor, "", true); listNar(jsonRoot, narAccessor, "", true);
writeFile(makeCacheFile(hashPart, "ls"), str.str()); writeFile(makeCacheFile(storePath, "ls"), str.str());
/* FIXME: do this asynchronously. */ /* FIXME: do this asynchronously. */
writeFile(makeCacheFile(hashPart, "nar"), nar); writeFile(makeCacheFile(storePath, "nar"), nar);
} catch (...) { } catch (...) {
ignoreException(); ignoreException();
@@ -47,22 +47,23 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_)
{ {
auto path = canonPath(path_); auto path = canonPath(path_);
auto [storePath, restPath] = store->toStorePath(path); auto storePath = store->toStorePath(path);
std::string restPath = std::string(path, storePath.size());
if (!store->isValidPath(storePath)) if (!store->isValidPath(store->parseStorePath(storePath)))
throw InvalidPath("path '%1%' is not a valid store path", store->printStorePath(storePath)); throw InvalidPath("path '%1%' is not a valid store path", storePath);
auto i = nars.find(std::string(storePath.hashPart())); auto i = nars.find(storePath);
if (i != nars.end()) return {i->second, restPath}; if (i != nars.end()) return {i->second, restPath};
StringSink sink; StringSink sink;
std::string listing; std::string listing;
Path cacheFile; Path cacheFile;
if (cacheDir != "" && pathExists(cacheFile = makeCacheFile(storePath.hashPart(), "nar"))) { if (cacheDir != "" && pathExists(cacheFile = makeCacheFile(storePath, "nar"))) {
try { try {
listing = nix::readFile(makeCacheFile(storePath.hashPart(), "ls")); listing = nix::readFile(makeCacheFile(storePath, "ls"));
auto narAccessor = makeLazyNarAccessor(listing, auto narAccessor = makeLazyNarAccessor(listing,
[cacheFile](uint64_t offset, uint64_t length) { [cacheFile](uint64_t offset, uint64_t length) {
@@ -80,7 +81,7 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_)
return buf; return buf;
}); });
nars.emplace(storePath.hashPart(), narAccessor); nars.emplace(storePath, narAccessor);
return {narAccessor, restPath}; return {narAccessor, restPath};
} catch (SysError &) { } } catch (SysError &) { }
@@ -89,15 +90,15 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_)
*sink.s = nix::readFile(cacheFile); *sink.s = nix::readFile(cacheFile);
auto narAccessor = makeNarAccessor(sink.s); auto narAccessor = makeNarAccessor(sink.s);
nars.emplace(storePath.hashPart(), narAccessor); nars.emplace(storePath, narAccessor);
return {narAccessor, restPath}; return {narAccessor, restPath};
} catch (SysError &) { } } catch (SysError &) { }
} }
store->narFromPath(storePath, sink); store->narFromPath(store->parseStorePath(storePath), sink);
auto narAccessor = makeNarAccessor(sink.s); auto narAccessor = makeNarAccessor(sink.s);
addToCache(storePath.hashPart(), *sink.s, narAccessor); addToCache(storePath, *sink.s, narAccessor);
return {narAccessor, restPath}; return {narAccessor, restPath};
} }
+3 -3
View File
@@ -10,7 +10,7 @@ class RemoteFSAccessor : public FSAccessor
{ {
ref<Store> store; ref<Store> store;
std::map<std::string, ref<FSAccessor>> nars; std::map<Path, ref<FSAccessor>> nars;
Path cacheDir; Path cacheDir;
@@ -18,9 +18,9 @@ class RemoteFSAccessor : public FSAccessor
friend class BinaryCacheStore; friend class BinaryCacheStore;
Path makeCacheFile(std::string_view hashPart, const std::string & ext); Path makeCacheFile(const Path & storePath, const std::string & ext);
void addToCache(std::string_view hashPart, const std::string & nar, void addToCache(const Path & storePath, const std::string & nar,
ref<FSAccessor> narAccessor); ref<FSAccessor> narAccessor);
public: public:
+2 -38
View File
@@ -8,7 +8,6 @@
#include "derivations.hh" #include "derivations.hh"
#include "pool.hh" #include "pool.hh"
#include "finally.hh" #include "finally.hh"
#include "logging.hh"
#include <sys/types.h> #include <sys/types.h>
#include <sys/stat.h> #include <sys/stat.h>
@@ -39,29 +38,6 @@ void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths
out << store.printStorePath(i); out << store.printStorePath(i);
} }
std::map<string, StorePath> readOutputPathMap(const Store & store, Source & from)
{
std::map<string, StorePath> pathMap;
auto rawInput = readStrings<Strings>(from);
if (rawInput.size() % 2)
throw Error("got an odd number of elements from the daemon when trying to read a output path map");
auto curInput = rawInput.begin();
while (curInput != rawInput.end()) {
auto thisKey = *curInput++;
auto thisValue = *curInput++;
pathMap.emplace(thisKey, store.parseStorePath(thisValue));
}
return pathMap;
}
void writeOutputPathMap(const Store & store, Sink & out, const std::map<string, StorePath> & pathMap)
{
out << 2*pathMap.size();
for (auto & i : pathMap) {
out << i.first;
out << store.printStorePath(i.second);
}
}
/* TODO: Separate these store impls into different files, give them better names */ /* TODO: Separate these store impls into different files, give them better names */
RemoteStore::RemoteStore(const Params & params) RemoteStore::RemoteStore(const Params & params)
@@ -221,7 +197,7 @@ void RemoteStore::setOptions(Connection & conn)
overrides.erase(settings.maxSilentTime.name); overrides.erase(settings.maxSilentTime.name);
overrides.erase(settings.buildCores.name); overrides.erase(settings.buildCores.name);
overrides.erase(settings.useSubstitutes.name); overrides.erase(settings.useSubstitutes.name);
overrides.erase(loggerSettings.showTrace.name); overrides.erase(settings.showTrace.name);
conn.to << overrides.size(); conn.to << overrides.size();
for (auto & i : overrides) for (auto & i : overrides)
conn.to << i.first << i.second.value; conn.to << i.first << i.second.value;
@@ -436,24 +412,12 @@ StorePathSet RemoteStore::queryValidDerivers(const StorePath & path)
StorePathSet RemoteStore::queryDerivationOutputs(const StorePath & path) StorePathSet RemoteStore::queryDerivationOutputs(const StorePath & path)
{ {
auto conn(getConnection()); auto conn(getConnection());
if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 0x16) {
return Store::queryDerivationOutputs(path);
}
conn->to << wopQueryDerivationOutputs << printStorePath(path); conn->to << wopQueryDerivationOutputs << printStorePath(path);
conn.processStderr(); conn.processStderr();
return readStorePaths<StorePathSet>(*this, conn->from); return readStorePaths<StorePathSet>(*this, conn->from);
} }
OutputPathMap RemoteStore::queryDerivationOutputMap(const StorePath & path)
{
auto conn(getConnection());
conn->to << wopQueryDerivationOutputMap << printStorePath(path);
conn.processStderr();
return readOutputPathMap(*this, conn->from);
}
std::optional<StorePath> RemoteStore::queryPathFromHashPart(const std::string & hashPart) std::optional<StorePath> RemoteStore::queryPathFromHashPart(const std::string & hashPart)
{ {
auto conn(getConnection()); auto conn(getConnection());
@@ -466,7 +430,7 @@ std::optional<StorePath> RemoteStore::queryPathFromHashPart(const std::string &
void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
RepairFlag repair, CheckSigsFlag checkSigs) RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor)
{ {
auto conn(getConnection()); auto conn(getConnection());
+2 -2
View File
@@ -51,7 +51,6 @@ public:
StorePathSet queryDerivationOutputs(const StorePath & path) override; StorePathSet queryDerivationOutputs(const StorePath & path) override;
OutputPathMap queryDerivationOutputMap(const StorePath & path) override;
std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override; std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override;
StorePathSet querySubstitutablePaths(const StorePathSet & paths) override; StorePathSet querySubstitutablePaths(const StorePathSet & paths) override;
@@ -60,7 +59,8 @@ public:
SubstitutablePathInfos & infos) override; SubstitutablePathInfos & infos) override;
void addToStore(const ValidPathInfo & info, Source & nar, void addToStore(const ValidPathInfo & info, Source & nar,
RepairFlag repair, CheckSigsFlag checkSigs) override; RepairFlag repair, CheckSigsFlag checkSigs,
std::shared_ptr<FSAccessor> accessor) override;
StorePath addToStore(const string & name, const Path & srcPath, StorePath addToStore(const string & name, const Path & srcPath,
FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256,
+17 -22
View File
@@ -7,6 +7,7 @@
#include "globals.hh" #include "globals.hh"
#include "compression.hh" #include "compression.hh"
#include "filetransfer.hh" #include "filetransfer.hh"
#include "istringstream_nocopy.hh"
#include <aws/core/Aws.h> #include <aws/core/Aws.h>
#include <aws/core/VersionConfig.h> #include <aws/core/VersionConfig.h>
@@ -261,11 +262,12 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
std::shared_ptr<TransferManager> transferManager; std::shared_ptr<TransferManager> transferManager;
std::once_flag transferManagerCreated; std::once_flag transferManagerCreated;
void uploadFile(const std::string & path, void uploadFile(const std::string & path, const std::string & data,
std::shared_ptr<std::basic_iostream<char>> istream,
const std::string & mimeType, const std::string & mimeType,
const std::string & contentEncoding) const std::string & contentEncoding)
{ {
auto stream = std::make_shared<istringstream_nocopy>(data);
auto maxThreads = std::thread::hardware_concurrency(); auto maxThreads = std::thread::hardware_concurrency();
static std::shared_ptr<Aws::Utils::Threading::PooledThreadExecutor> static std::shared_ptr<Aws::Utils::Threading::PooledThreadExecutor>
@@ -305,7 +307,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
std::shared_ptr<TransferHandle> transferHandle = std::shared_ptr<TransferHandle> transferHandle =
transferManager->UploadFile( transferManager->UploadFile(
istream, bucketName, path, mimeType, stream, bucketName, path, mimeType,
Aws::Map<Aws::String, Aws::String>(), Aws::Map<Aws::String, Aws::String>(),
nullptr /*, contentEncoding */); nullptr /*, contentEncoding */);
@@ -331,7 +333,9 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
if (contentEncoding != "") if (contentEncoding != "")
request.SetContentEncoding(contentEncoding); request.SetContentEncoding(contentEncoding);
request.SetBody(istream); auto stream = std::make_shared<istringstream_nocopy>(data);
request.SetBody(stream);
auto result = checkAws(fmt("AWS error uploading '%s'", path), auto result = checkAws(fmt("AWS error uploading '%s'", path),
s3Helper.client->PutObject(request)); s3Helper.client->PutObject(request));
@@ -343,34 +347,25 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1) std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1)
.count(); .count();
auto size = istream->tellg(); printInfo(format("uploaded 's3://%1%/%2%' (%3% bytes) in %4% ms") %
bucketName % path % data.size() % duration);
printInfo("uploaded 's3://%s/%s' (%d bytes) in %d ms",
bucketName, path, size, duration);
stats.putTimeMs += duration; stats.putTimeMs += duration;
stats.putBytes += size; stats.putBytes += data.size();
stats.put++; stats.put++;
} }
void upsertFile(const std::string & path, void upsertFile(const std::string & path, const std::string & data,
std::shared_ptr<std::basic_iostream<char>> istream,
const std::string & mimeType) override const std::string & mimeType) override
{ {
auto compress = [&](std::string compression)
{
auto compressed = nix::compress(compression, StreamToSourceAdapter(istream).drain());
return std::make_shared<std::stringstream>(std::move(*compressed));
};
if (narinfoCompression != "" && hasSuffix(path, ".narinfo")) if (narinfoCompression != "" && hasSuffix(path, ".narinfo"))
uploadFile(path, compress(narinfoCompression), mimeType, narinfoCompression); uploadFile(path, *compress(narinfoCompression, data), mimeType, narinfoCompression);
else if (lsCompression != "" && hasSuffix(path, ".ls")) else if (lsCompression != "" && hasSuffix(path, ".ls"))
uploadFile(path, compress(lsCompression), mimeType, lsCompression); uploadFile(path, *compress(lsCompression, data), mimeType, lsCompression);
else if (logCompression != "" && hasPrefix(path, "log/")) else if (logCompression != "" && hasPrefix(path, "log/"))
uploadFile(path, compress(logCompression), mimeType, logCompression); uploadFile(path, *compress(logCompression, data), mimeType, logCompression);
else else
uploadFile(path, istream, mimeType, ""); uploadFile(path, data, mimeType, "");
} }
void getFile(const std::string & path, Sink & sink) override void getFile(const std::string & path, Sink & sink) override
@@ -415,7 +410,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
for (auto object : contents) { for (auto object : contents) {
auto & key = object.GetKey(); auto & key = object.GetKey();
if (key.size() != 40 || !hasSuffix(key, ".narinfo")) continue; if (key.size() != 40 || !hasSuffix(key, ".narinfo")) continue;
paths.insert(parseStorePath(storeDir + "/" + key.substr(0, key.size() - 8) + "-" + MissingName)); paths.insert(parseStorePath(storeDir + "/" + key.substr(0, key.size() - 8) + "-unknown"));
} }
marker = res.GetNextMarker(); marker = res.GetNextMarker();
+10 -65
View File
@@ -7,7 +7,6 @@
#include "json.hh" #include "json.hh"
#include "derivations.hh" #include "derivations.hh"
#include "url.hh" #include "url.hh"
#include "archive.hh"
#include <future> #include <future>
@@ -21,15 +20,15 @@ bool Store::isInStore(const Path & path) const
} }
std::pair<StorePath, Path> Store::toStorePath(const Path & path) const Path Store::toStorePath(const Path & path) const
{ {
if (!isInStore(path)) if (!isInStore(path))
throw Error("path '%1%' is not in the Nix store", path); throw Error("path '%1%' is not in the Nix store", path);
Path::size_type slash = path.find('/', storeDir.size() + 1); Path::size_type slash = path.find('/', storeDir.size() + 1);
if (slash == Path::npos) if (slash == Path::npos)
return {parseStorePath(path), ""}; return path;
else else
return {parseStorePath(std::string_view(path).substr(0, slash)), path.substr(slash)}; return Path(path, 0, slash);
} }
@@ -42,14 +41,14 @@ Path Store::followLinksToStore(std::string_view _path) const
path = absPath(target, dirOf(path)); path = absPath(target, dirOf(path));
} }
if (!isInStore(path)) if (!isInStore(path))
throw BadStorePath("path '%1%' is not in the Nix store", path); throw NotInStore("path '%1%' is not in the Nix store", path);
return path; return path;
} }
StorePath Store::followLinksToStorePath(std::string_view path) const StorePath Store::followLinksToStorePath(std::string_view path) const
{ {
return toStorePath(followLinksToStore(path)).first; return parseStorePath(toStorePath(followLinksToStore(path)));
} }
@@ -222,40 +221,6 @@ StorePath Store::computeStorePathForText(const string & name, const string & s,
} }
ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath,
FileIngestionMethod method, HashType hashAlgo,
std::optional<Hash> expectedCAHash)
{
/* FIXME: inefficient: we're reading/hashing 'tmpFile' three
times. */
auto [narHash, narSize] = hashPath(htSHA256, srcPath);
auto hash = method == FileIngestionMethod::Recursive
? hashAlgo == htSHA256
? narHash
: hashPath(hashAlgo, srcPath).first
: hashFile(hashAlgo, srcPath);
if (expectedCAHash && expectedCAHash != hash)
throw Error("hash mismatch for '%s'", srcPath);
ValidPathInfo info(makeFixedOutputPath(method, hash, name));
info.narHash = narHash;
info.narSize = narSize;
info.ca = FixedOutputHash { .method = method, .hash = hash };
if (!isValidPath(info.path)) {
auto source = sinkToSource([&](Sink & sink) {
dumpPath(srcPath, sink);
});
addToStore(info, *source);
}
return info;
}
Store::Store(const Params & params) Store::Store(const Params & params)
: Config(params) : Config(params)
, state({(size_t) pathInfoCacheSize}) , state({(size_t) pathInfoCacheSize})
@@ -277,16 +242,6 @@ bool Store::PathInfoCacheValue::isKnownNow()
return std::chrono::steady_clock::now() < time_point + ttl; return std::chrono::steady_clock::now() < time_point + ttl;
} }
StorePathSet Store::queryDerivationOutputs(const StorePath & path)
{
auto outputMap = this->queryDerivationOutputMap(path);
StorePathSet outputPaths;
for (auto & i: outputMap) {
outputPaths.emplace(std::move(i.second));
}
return outputPaths;
}
bool Store::isValidPath(const StorePath & storePath) bool Store::isValidPath(const StorePath & storePath)
{ {
std::string hashPart(storePath.hashPart()); std::string hashPart(storePath.hashPart());
@@ -351,14 +306,6 @@ ref<const ValidPathInfo> Store::queryPathInfo(const StorePath & storePath)
} }
static bool goodStorePath(const StorePath & expected, const StorePath & actual)
{
return
expected.hashPart() == actual.hashPart()
&& (expected.name() == Store::MissingName || expected.name() == actual.name());
}
void Store::queryPathInfo(const StorePath & storePath, void Store::queryPathInfo(const StorePath & storePath,
Callback<ref<const ValidPathInfo>> callback) noexcept Callback<ref<const ValidPathInfo>> callback) noexcept
{ {
@@ -386,7 +333,7 @@ void Store::queryPathInfo(const StorePath & storePath,
state_->pathInfoCache.upsert(hashPart, state_->pathInfoCache.upsert(hashPart,
res.first == NarInfoDiskCache::oInvalid ? PathInfoCacheValue{} : PathInfoCacheValue{ .value = res.second }); res.first == NarInfoDiskCache::oInvalid ? PathInfoCacheValue{} : PathInfoCacheValue{ .value = res.second });
if (res.first == NarInfoDiskCache::oInvalid || if (res.first == NarInfoDiskCache::oInvalid ||
!goodStorePath(storePath, res.second->path)) res.second->path != storePath)
throw InvalidPath("path '%s' is not valid", printStorePath(storePath)); throw InvalidPath("path '%s' is not valid", printStorePath(storePath));
} }
return callback(ref<const ValidPathInfo>(res.second)); return callback(ref<const ValidPathInfo>(res.second));
@@ -398,7 +345,7 @@ void Store::queryPathInfo(const StorePath & storePath,
auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback)); auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback));
queryPathInfoUncached(storePath, queryPathInfoUncached(storePath,
{[this, storePathS{printStorePath(storePath)}, hashPart, callbackPtr](std::future<std::shared_ptr<const ValidPathInfo>> fut) { {[this, storePath{printStorePath(storePath)}, hashPart, callbackPtr](std::future<std::shared_ptr<const ValidPathInfo>> fut) {
try { try {
auto info = fut.get(); auto info = fut.get();
@@ -411,11 +358,9 @@ void Store::queryPathInfo(const StorePath & storePath,
state_->pathInfoCache.upsert(hashPart, PathInfoCacheValue { .value = info }); state_->pathInfoCache.upsert(hashPart, PathInfoCacheValue { .value = info });
} }
auto storePath = parseStorePath(storePathS); if (!info || info->path != parseStorePath(storePath)) {
if (!info || !goodStorePath(storePath, info->path)) {
stats.narInfoMissing++; stats.narInfoMissing++;
throw InvalidPath("path '%s' is not valid", storePathS); throw InvalidPath("path '%s' is not valid", storePath);
} }
(*callbackPtr)(ref<const ValidPathInfo>(info)); (*callbackPtr)(ref<const ValidPathInfo>(info));
@@ -560,7 +505,7 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & store
if (!narInfo->url.empty()) if (!narInfo->url.empty())
jsonPath.attr("url", narInfo->url); jsonPath.attr("url", narInfo->url);
if (narInfo->fileHash) if (narInfo->fileHash)
jsonPath.attr("downloadHash", narInfo->fileHash.to_string(hashBase, true)); jsonPath.attr("downloadHash", narInfo->fileHash.to_string(Base32, true));
if (narInfo->fileSize) if (narInfo->fileSize)
jsonPath.attr("downloadSize", narInfo->fileSize); jsonPath.attr("downloadSize", narInfo->fileSize);
if (showClosureSize) if (showClosureSize)
+13 -24
View File
@@ -31,7 +31,7 @@ MakeError(InvalidPath, Error);
MakeError(Unsupported, Error); MakeError(Unsupported, Error);
MakeError(SubstituteGone, Error); MakeError(SubstituteGone, Error);
MakeError(SubstituterDisabled, Error); MakeError(SubstituterDisabled, Error);
MakeError(BadStorePath, Error); MakeError(NotInStore, Error);
class FSAccessor; class FSAccessor;
@@ -317,9 +317,9 @@ public:
the Nix store. */ the Nix store. */
bool isStorePath(std::string_view path) const; bool isStorePath(std::string_view path) const;
/* Split a path like /nix/store/<hash>-<name>/<bla> into /* Chop off the parts after the top-level store name, e.g.,
/nix/store/<hash>-<name> and /<bla>. */ /nix/store/abcd-foo/bar => /nix/store/abcd-foo. */
std::pair<StorePath, Path> toStorePath(const Path & path) const; Path toStorePath(const Path & path) const;
/* Follow symlinks until we end up with a path in the Nix store. */ /* Follow symlinks until we end up with a path in the Nix store. */
Path followLinksToStore(std::string_view path) const; Path followLinksToStore(std::string_view path) const;
@@ -384,16 +384,13 @@ public:
SubstituteFlag maybeSubstitute = NoSubstitute); SubstituteFlag maybeSubstitute = NoSubstitute);
/* Query the set of all valid paths. Note that for some store /* Query the set of all valid paths. Note that for some store
backends, the name part of store paths may be replaced by 'x' backends, the name part of store paths may be omitted
(i.e. you'll get /nix/store/<hash>-x rather than (i.e. you'll get /nix/store/<hash> rather than
/nix/store/<hash>-<name>). Use queryPathInfo() to obtain the /nix/store/<hash>-<name>). Use queryPathInfo() to obtain the
full store path. FIXME: should return a set of full store path. */
std::variant<StorePath, HashPart> to get rid of this hack. */
virtual StorePathSet queryAllValidPaths() virtual StorePathSet queryAllValidPaths()
{ unsupported("queryAllValidPaths"); } { unsupported("queryAllValidPaths"); }
constexpr static const char * MissingName = "x";
/* Query information about a valid path. It is permitted to omit /* Query information about a valid path. It is permitted to omit
the name part of the store path. */ the name part of the store path. */
ref<const ValidPathInfo> queryPathInfo(const StorePath & path); ref<const ValidPathInfo> queryPathInfo(const StorePath & path);
@@ -421,11 +418,8 @@ public:
virtual StorePathSet queryValidDerivers(const StorePath & path) { return {}; }; virtual StorePathSet queryValidDerivers(const StorePath & path) { return {}; };
/* Query the outputs of the derivation denoted by `path'. */ /* Query the outputs of the derivation denoted by `path'. */
virtual StorePathSet queryDerivationOutputs(const StorePath & path); virtual StorePathSet queryDerivationOutputs(const StorePath & path)
{ unsupported("queryDerivationOutputs"); }
/* Query the mapping outputName=>outputPath for the given derivation */
virtual OutputPathMap queryDerivationOutputMap(const StorePath & path)
{ unsupported("queryDerivationOutputMap"); }
/* Query the full store path given the hash part of a valid store /* Query the full store path given the hash part of a valid store
path, or empty if the path doesn't exist. */ path, or empty if the path doesn't exist. */
@@ -442,7 +436,8 @@ public:
/* Import a path into the store. */ /* Import a path into the store. */
virtual void addToStore(const ValidPathInfo & info, Source & narSource, virtual void addToStore(const ValidPathInfo & info, Source & narSource,
RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs) = 0; RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs,
std::shared_ptr<FSAccessor> accessor = 0) = 0;
/* Copy the contents of a path to the store and register the /* Copy the contents of a path to the store and register the
validity the resulting path. The resulting path is returned. validity the resulting path. The resulting path is returned.
@@ -452,13 +447,6 @@ public:
FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256,
PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) = 0; PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) = 0;
/* Copy the contents of a path to the store and register the
validity the resulting path, using a constant amount of
memory. */
ValidPathInfo addToStoreSlow(std::string_view name, const Path & srcPath,
FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256,
std::optional<Hash> expectedCAHash = {});
// FIXME: remove? // FIXME: remove?
virtual StorePath addToStoreFromDump(const string & dump, const string & name, virtual StorePath addToStoreFromDump(const string & dump, const string & name,
FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair)
@@ -625,7 +613,8 @@ public:
the Nix store. Optionally, the contents of the NARs are the Nix store. Optionally, the contents of the NARs are
preloaded into the specified FS accessor to speed up subsequent preloaded into the specified FS accessor to speed up subsequent
access. */ access. */
StorePaths importPaths(Source & source, CheckSigsFlag checkSigs = CheckSigs); StorePaths importPaths(Source & source, std::shared_ptr<FSAccessor> accessor,
CheckSigsFlag checkSigs = CheckSigs);
struct Stats struct Stats
{ {
+2 -4
View File
@@ -6,7 +6,7 @@ namespace nix {
#define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_1 0x6e697863
#define WORKER_MAGIC_2 0x6478696f #define WORKER_MAGIC_2 0x6478696f
#define PROTOCOL_VERSION 0x116 #define PROTOCOL_VERSION 0x115
#define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
#define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
@@ -30,7 +30,7 @@ typedef enum {
wopSetOptions = 19, wopSetOptions = 19,
wopCollectGarbage = 20, wopCollectGarbage = 20,
wopQuerySubstitutablePathInfo = 21, wopQuerySubstitutablePathInfo = 21,
wopQueryDerivationOutputs = 22, // obsolete wopQueryDerivationOutputs = 22,
wopQueryAllValidPaths = 23, wopQueryAllValidPaths = 23,
wopQueryFailedPaths = 24, wopQueryFailedPaths = 24,
wopClearFailedPaths = 25, wopClearFailedPaths = 25,
@@ -49,7 +49,6 @@ typedef enum {
wopNarFromPath = 38, wopNarFromPath = 38,
wopAddToStoreNar = 39, wopAddToStoreNar = 39,
wopQueryMissing = 40, wopQueryMissing = 40,
wopQueryDerivationOutputMap = 41,
} WorkerOp; } WorkerOp;
@@ -70,6 +69,5 @@ template<class T> T readStorePaths(const Store & store, Source & from);
void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths); void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths);
void writeOutputPathMap(const Store & store, Sink & out, const OutputPathMap & paths);
} }
+2 -2
View File
@@ -11,7 +11,7 @@ namespace nix {
#define ANSI_GREEN "\e[32;1m" #define ANSI_GREEN "\e[32;1m"
#define ANSI_YELLOW "\e[33;1m" #define ANSI_YELLOW "\e[33;1m"
#define ANSI_BLUE "\e[34;1m" #define ANSI_BLUE "\e[34;1m"
#define ANSI_MAGENTA "\e[35;1m" #define ANSI_MAGENTA "\e[35m;1m"
#define ANSI_CYAN "\e[36;1m" #define ANSI_CYAN "\e[36m;1m"
} }
+1 -1
View File
@@ -262,7 +262,7 @@ static void parse(ParseSink & sink, Source & source, const Path & path)
names[name] = 0; names[name] = 0;
} }
} else if (s == "node") { } else if (s == "node") {
if (name.empty()) throw badArchive("entry name missing"); if (s.empty()) throw badArchive("entry name missing");
parse(sink, source, path + "/" + name); parse(sink, source, path + "/" + name);
} else } else
throw badArchive("unknown field " + s); throw badArchive("unknown field " + s);
+2 -3
View File
@@ -63,12 +63,11 @@ struct ParseSink
virtual void createSymlink(const Path & path, const string & target) { }; virtual void createSymlink(const Path & path, const string & target) { };
}; };
struct TeeParseSink : ParseSink struct TeeSink : ParseSink
{ {
StringSink saved;
TeeSource source; TeeSource source;
TeeParseSink(Source & source) : source(source, saved) { } TeeSink(Source & source) : source(source) { }
}; };
void parseDump(ParseSink & sink, Source & source); void parseDump(ParseSink & sink, Source & source);
+44 -202
View File
@@ -7,11 +7,14 @@
namespace nix { namespace nix {
const std::string nativeSystem = SYSTEM; const std::string nativeSystem = SYSTEM;
BaseError & BaseError::addTrace(std::optional<ErrPos> e, hintformat hint) // addPrefix is used for show-trace. Strings added with addPrefix
// will print ahead of the error itself.
BaseError & BaseError::addPrefix(const FormatOrString & fs)
{ {
err.traces.push_front(Trace { .pos = e, .hint = hint}); prefix_ = fs.s + prefix_;
return *this; return *this;
} }
@@ -25,7 +28,7 @@ const string& BaseError::calcWhat() const
err.name = sname(); err.name = sname();
std::ostringstream oss; std::ostringstream oss;
showErrorInfo(oss, err, false); oss << err;
what_ = oss.str(); what_ = oss.str();
return *what_; return *what_;
@@ -53,114 +56,28 @@ string showErrPos(const ErrPos &errPos)
} }
} }
std::optional<LinesOfCode> getCodeLines(const ErrPos &errPos) // if nixCode contains lines of code, print them to the ostream, indicating the error column.
{ void printCodeLines(std::ostream &out, const string &prefix, const NixCode &nixCode)
if (errPos.line <= 0)
return std::nullopt;
if (errPos.origin == foFile) {
LinesOfCode loc;
try {
AutoCloseFD fd = open(errPos.file.c_str(), O_RDONLY | O_CLOEXEC);
if (!fd) {
logError(SysError("opening file '%1%'", errPos.file).info());
return std::nullopt;
}
else
{
// count the newlines.
int count = 0;
string line;
int pl = errPos.line - 1;
do
{
line = readLine(fd.get());
++count;
if (count < pl)
{
;
}
else if (count == pl) {
loc.prevLineOfCode = line;
} else if (count == pl + 1) {
loc.errLineOfCode = line;
} else if (count == pl + 2) {
loc.nextLineOfCode = line;
break;
}
} while (true);
return loc;
}
}
catch (EndOfFile &eof) {
if (loc.errLineOfCode.has_value())
return loc;
else
return std::nullopt;
}
catch (std::exception &e) {
printError("error reading nix file: %s\n%s", errPos.file, e.what());
return std::nullopt;
}
} else {
std::istringstream iss(errPos.file);
// count the newlines.
int count = 0;
string line;
int pl = errPos.line - 1;
LinesOfCode loc;
do
{
std::getline(iss, line);
++count;
if (count < pl)
{
;
}
else if (count == pl) {
loc.prevLineOfCode = line;
} else if (count == pl + 1) {
loc.errLineOfCode = line;
} else if (count == pl + 2) {
loc.nextLineOfCode = line;
break;
}
if (!iss.good())
break;
} while (true);
return loc;
}
}
// print lines of code to the ostream, indicating the error column.
void printCodeLines(std::ostream &out,
const string &prefix,
const ErrPos &errPos,
const LinesOfCode &loc)
{ {
// previous line of code. // previous line of code.
if (loc.prevLineOfCode.has_value()) { if (nixCode.prevLineOfCode.has_value()) {
out << std::endl out << std::endl
<< fmt("%1% %|2$5d|| %3%", << fmt("%1% %|2$5d|| %3%",
prefix, prefix,
(errPos.line - 1), (nixCode.errPos.line - 1),
*loc.prevLineOfCode); *nixCode.prevLineOfCode);
} }
if (loc.errLineOfCode.has_value()) { if (nixCode.errLineOfCode.has_value()) {
// line of code containing the error. // line of code containing the error.
out << std::endl out << std::endl
<< fmt("%1% %|2$5d|| %3%", << fmt("%1% %|2$5d|| %3%",
prefix, prefix,
(errPos.line), (nixCode.errPos.line),
*loc.errLineOfCode); *nixCode.errLineOfCode);
// error arrows for the column range. // error arrows for the column range.
if (errPos.column > 0) { if (nixCode.errPos.column > 0) {
int start = errPos.column; int start = nixCode.errPos.column;
std::string spaces; std::string spaces;
for (int i = 0; i < start; ++i) { for (int i = 0; i < start; ++i) {
spaces.append(" "); spaces.append(" ");
@@ -170,49 +87,23 @@ void printCodeLines(std::ostream &out,
out << std::endl out << std::endl
<< fmt("%1% |%2%" ANSI_RED "%3%" ANSI_NORMAL, << fmt("%1% |%2%" ANSI_RED "%3%" ANSI_NORMAL,
prefix, prefix,
spaces, spaces,
arrows); arrows);
} }
} }
// next line of code. // next line of code.
if (loc.nextLineOfCode.has_value()) { if (nixCode.nextLineOfCode.has_value()) {
out << std::endl out << std::endl
<< fmt("%1% %|2$5d|| %3%", << fmt("%1% %|2$5d|| %3%",
prefix, prefix,
(errPos.line + 1), (nixCode.errPos.line + 1),
*loc.nextLineOfCode); *nixCode.nextLineOfCode);
} }
} }
void printAtPos(const string &prefix, const ErrPos &pos, std::ostream &out) std::ostream& operator<<(std::ostream &out, const ErrorInfo &einfo)
{
if (pos)
{
switch (pos.origin) {
case foFile: {
out << prefix << ANSI_BLUE << "at: " << ANSI_YELLOW << showErrPos(pos) <<
ANSI_BLUE << " in file: " << ANSI_NORMAL << pos.file;
break;
}
case foString: {
out << prefix << ANSI_BLUE << "at: " << ANSI_YELLOW << showErrPos(pos) <<
ANSI_BLUE << " from string" << ANSI_NORMAL;
break;
}
case foStdin: {
out << prefix << ANSI_BLUE << "at: " << ANSI_YELLOW << showErrPos(pos) <<
ANSI_BLUE << " from stdin" << ANSI_NORMAL;
break;
}
default:
throw Error("invalid FileOrigin in errPos");
}
}
}
std::ostream& showErrorInfo(std::ostream &out, const ErrorInfo &einfo, bool showTrace)
{ {
auto errwidth = std::max<size_t>(getWindowSize().second, 20); auto errwidth = std::max<size_t>(getWindowSize().second, 20);
string prefix = ""; string prefix = "";
@@ -267,12 +158,8 @@ std::ostream& showErrorInfo(std::ostream &out, const ErrorInfo &einfo, bool show
} }
} }
auto ndl = prefix.length() auto ndl = prefix.length() + levelString.length() + 3 + einfo.name.length() + einfo.programName.value_or("").length();
+ filterANSIEscapes(levelString, true).length() auto dashwidth = ndl > (errwidth - 3) ? 3 : errwidth - ndl;
+ 7
+ einfo.name.length()
+ einfo.programName.value_or("").length();
auto dashwidth = std::max<int>(errwidth - ndl, 3);
std::string dashes(dashwidth, '-'); std::string dashes(dashwidth, '-');
@@ -292,9 +179,16 @@ std::ostream& showErrorInfo(std::ostream &out, const ErrorInfo &einfo, bool show
einfo.programName.value_or("")); einfo.programName.value_or(""));
bool nl = false; // intersperse newline between sections. bool nl = false; // intersperse newline between sections.
if (einfo.errPos.has_value() && (*einfo.errPos)) { if (einfo.nixCode.has_value()) {
out << prefix << std::endl; if (einfo.nixCode->errPos.file != "") {
printAtPos(prefix, *einfo.errPos, out); // filename, line, column.
out << std::endl << fmt("%1%in file: " ANSI_BLUE "%2% %3%" ANSI_NORMAL,
prefix,
einfo.nixCode->errPos.file,
showErrPos(einfo.nixCode->errPos));
} else {
out << std::endl << fmt("%1%from command line argument", prefix);
}
nl = true; nl = true;
} }
@@ -306,16 +200,12 @@ std::ostream& showErrorInfo(std::ostream &out, const ErrorInfo &einfo, bool show
nl = true; nl = true;
} }
if (einfo.errPos.has_value() && (*einfo.errPos)) { // lines of code.
auto loc = getCodeLines(*einfo.errPos); if (einfo.nixCode.has_value() && einfo.nixCode->errLineOfCode.has_value()) {
if (nl)
// lines of code. out << std::endl << prefix;
if (loc.has_value()) { printCodeLines(out, prefix, *einfo.nixCode);
if (nl) nl = true;
out << std::endl << prefix;
printCodeLines(out, prefix, *einfo.errPos, *loc);
nl = true;
}
} }
// hint // hint
@@ -326,54 +216,6 @@ std::ostream& showErrorInfo(std::ostream &out, const ErrorInfo &einfo, bool show
nl = true; nl = true;
} }
// traces
if (showTrace && !einfo.traces.empty())
{
const string tracetitle(" show-trace ");
int fill = errwidth - tracetitle.length();
int lw = 0;
int rw = 0;
const int min_dashes = 3;
if (fill > min_dashes * 2) {
if (fill % 2 != 0) {
lw = fill / 2;
rw = lw + 1;
}
else
{
lw = rw = fill / 2;
}
}
else
lw = rw = min_dashes;
if (nl)
out << std::endl << prefix;
out << ANSI_BLUE << std::string(lw, '-') << tracetitle << std::string(rw, '-') << ANSI_NORMAL;
for (auto iter = einfo.traces.rbegin(); iter != einfo.traces.rend(); ++iter)
{
out << std::endl << prefix;
out << ANSI_BLUE << "trace: " << ANSI_NORMAL << iter->hint.str();
if (iter->pos.has_value() && (*iter->pos)) {
auto pos = iter->pos.value();
out << std::endl << prefix;
printAtPos(prefix, pos, out);
auto loc = getCodeLines(pos);
if (loc.has_value())
{
out << std::endl << prefix;
printCodeLines(out, prefix, pos, *loc);
out << std::endl << prefix;
}
}
}
}
return out; return out;
} }
} }
+33 -59
View File
@@ -1,8 +1,8 @@
#pragma once #pragma once
#include "ref.hh" #include "ref.hh"
#include "types.hh" #include "types.hh"
#include "fmt.hh"
#include <cstring> #include <cstring>
#include <list> #include <list>
@@ -10,9 +10,7 @@
#include <map> #include <map>
#include <optional> #include <optional>
#include <sys/types.h> #include "fmt.hh"
#include <sys/stat.h>
#include <fcntl.h>
/* Before 4.7, gcc's std::exception uses empty throw() specifiers for /* Before 4.7, gcc's std::exception uses empty throw() specifiers for
* its (virtual) destructor and what() in c++11 mode, in violation of spec * its (virtual) destructor and what() in c++11 mode, in violation of spec
@@ -27,20 +25,20 @@ namespace nix {
/* /*
This file defines two main structs/classes used in nix error handling. This file defines two main structs/classes used in nix error handling.
ErrorInfo provides a standard payload of error information, with conversion to string ErrorInfo provides a standard payload of error information, with conversion to string
happening in the logger rather than at the call site. happening in the logger rather than at the call site.
BaseError is the ancestor of nix specific exceptions (and Interrupted), and contains BaseError is the ancestor of nix specific exceptions (and Interrupted), and contains
an ErrorInfo. an ErrorInfo.
ErrorInfo structs are sent to the logger as part of an exception, or directly with the ErrorInfo structs are sent to the logger as part of an exception, or directly with the
logError or logWarning macros. logError or logWarning macros.
See the error-demo.cc program for usage examples. See the error-demo.cc program for usage examples.
*/ */
typedef enum { typedef enum {
lvlError = 0, lvlError = 0,
@@ -52,25 +50,11 @@ typedef enum {
lvlVomit lvlVomit
} Verbosity; } Verbosity;
typedef enum {
foFile,
foStdin,
foString
} FileOrigin;
// the lines of code surrounding an error.
struct LinesOfCode {
std::optional<string> prevLineOfCode;
std::optional<string> errLineOfCode;
std::optional<string> nextLineOfCode;
};
// ErrPos indicates the location of an error in a nix file. // ErrPos indicates the location of an error in a nix file.
struct ErrPos { struct ErrPos {
int line = 0; int line = 0;
int column = 0; int column = 0;
string file; string file;
FileOrigin origin;
operator bool() const operator bool() const
{ {
@@ -81,14 +65,9 @@ struct ErrPos {
template <class P> template <class P>
ErrPos& operator=(const P &pos) ErrPos& operator=(const P &pos)
{ {
origin = pos.origin;
line = pos.line; line = pos.line;
column = pos.column; column = pos.column;
// is file symbol null? file = pos.file;
if (pos.file.set())
file = pos.file;
else
file = "";
return *this; return *this;
} }
@@ -99,9 +78,11 @@ struct ErrPos {
} }
}; };
struct Trace { struct NixCode {
std::optional<ErrPos> pos; ErrPos errPos;
hintformat hint; std::optional<string> prevLineOfCode;
std::optional<string> errLineOfCode;
std::optional<string> nextLineOfCode;
}; };
struct ErrorInfo { struct ErrorInfo {
@@ -109,19 +90,19 @@ struct ErrorInfo {
string name; string name;
string description; string description;
std::optional<hintformat> hint; std::optional<hintformat> hint;
std::optional<ErrPos> errPos; std::optional<NixCode> nixCode;
std::list<Trace> traces;
static std::optional<string> programName; static std::optional<string> programName;
}; };
std::ostream& showErrorInfo(std::ostream &out, const ErrorInfo &einfo, bool showTrace); std::ostream& operator<<(std::ostream &out, const ErrorInfo &einfo);
/* BaseError should generally not be caught, as it has Interrupted as /* BaseError should generally not be caught, as it has Interrupted as
a subclass. Catch Error instead. */ a subclass. Catch Error instead. */
class BaseError : public std::exception class BaseError : public std::exception
{ {
protected: protected:
string prefix_; // used for location traces etc.
mutable ErrorInfo err; mutable ErrorInfo err;
mutable std::optional<string> what_; mutable std::optional<string> what_;
@@ -132,23 +113,23 @@ public:
template<typename... Args> template<typename... Args>
BaseError(unsigned int status, const Args & ... args) BaseError(unsigned int status, const Args & ... args)
: err {.level = lvlError, : err { .level = lvlError,
.hint = hintfmt(args...) .hint = hintfmt(args...)
} }
, status(status) , status(status)
{ } { }
template<typename... Args> template<typename... Args>
BaseError(const std::string & fs, const Args & ... args) BaseError(const std::string & fs, const Args & ... args)
: err {.level = lvlError, : err { .level = lvlError,
.hint = hintfmt(fs, args...) .hint = hintfmt(fs, args...)
} }
{ } { }
BaseError(hintformat hint) BaseError(hintformat hint)
: err {.level = lvlError, : err { .level = lvlError,
.hint = hint .hint = hint
} }
{ } { }
BaseError(ErrorInfo && e) BaseError(ErrorInfo && e)
@@ -169,17 +150,10 @@ public:
#endif #endif
const string & msg() const { return calcWhat(); } const string & msg() const { return calcWhat(); }
const string & prefix() const { return prefix_; }
BaseError & addPrefix(const FormatOrString & fs);
const ErrorInfo & info() { calcWhat(); return err; } const ErrorInfo & info() { calcWhat(); return err; }
template<typename... Args>
BaseError & addTrace(std::optional<ErrPos> e, const string &fs, const Args & ... args)
{
return addTrace(e, hintfmt(fs, args...));
}
BaseError & addTrace(std::optional<ErrPos> e, hintformat hint);
bool hasTrace() const { return !err.traces.empty(); }
}; };
#define MakeError(newClass, superClass) \ #define MakeError(newClass, superClass) \
@@ -199,7 +173,7 @@ public:
template<typename... Args> template<typename... Args>
SysError(const Args & ... args) SysError(const Args & ... args)
: Error("") :Error("")
{ {
errNo = errno; errNo = errno;
auto hf = hintfmt(args...); auto hf = hintfmt(args...);
+14 -14
View File
@@ -8,6 +8,7 @@
#include "hash.hh" #include "hash.hh"
#include "archive.hh" #include "archive.hh"
#include "util.hh" #include "util.hh"
#include "istringstream_nocopy.hh"
#include <sys/types.h> #include <sys/types.h>
#include <sys/stat.h> #include <sys/stat.h>
@@ -18,7 +19,7 @@ namespace nix {
void Hash::init() void Hash::init()
{ {
assert(type); if (!type) abort();
switch (*type) { switch (*type) {
case htMD5: hashSize = md5HashSize; break; case htMD5: hashSize = md5HashSize; break;
case htSHA1: hashSize = sha1HashSize; break; case htSHA1: hashSize = sha1HashSize; break;
@@ -100,15 +101,15 @@ static string printHash32(const Hash & hash)
string printHash16or32(const Hash & hash) string printHash16or32(const Hash & hash)
{ {
assert(hash.type);
return hash.to_string(hash.type == htMD5 ? Base16 : Base32, false); return hash.to_string(hash.type == htMD5 ? Base16 : Base32, false);
} }
HashType assertInitHashType(const Hash & h) HashType assertInitHashType(const Hash & h) {
{ if (h.type)
assert(h.type); return *h.type;
return *h.type; else
abort();
} }
std::string Hash::to_string(Base base, bool includeType) const std::string Hash::to_string(Base base, bool includeType) const
@@ -362,15 +363,14 @@ HashType parseHashType(const string & s)
string printHashType(HashType ht) string printHashType(HashType ht)
{ {
switch (ht) { switch (ht) {
case htMD5: return "md5"; case htMD5: return "md5"; break;
case htSHA1: return "sha1"; case htSHA1: return "sha1"; break;
case htSHA256: return "sha256"; case htSHA256: return "sha256"; break;
case htSHA512: return "sha512"; case htSHA512: return "sha512"; break;
default:
// illegal hash type enum value internally, as opposed to external input
// which should be validated with nice error message.
assert(false);
} }
// illegal hash type enum value internally, as opposed to external input
// which should be validated with nice error message.
abort();
} }
} }
+1 -1
View File
@@ -10,7 +10,7 @@ namespace nix {
MakeError(BadHash, Error); MakeError(BadHash, Error);
enum HashType : char { htMD5 = 42, htSHA1, htSHA256, htSHA512 }; enum HashType : char { htMD5, htSHA1, htSHA256, htSHA512 };
const int md5HashSize = 16; const int md5HashSize = 16;
+92
View File
@@ -0,0 +1,92 @@
/* This file provides a variant of std::istringstream that doesn't
copy its string argument. This is useful for large strings. The
caller must ensure that the string object is not destroyed while
it's referenced by this object. */
#pragma once
#include <string>
#include <iostream>
template <class CharT, class Traits = std::char_traits<CharT>, class Allocator = std::allocator<CharT>>
class basic_istringbuf_nocopy : public std::basic_streambuf<CharT, Traits>
{
public:
typedef std::basic_string<CharT, Traits, Allocator> string_type;
typedef typename std::basic_streambuf<CharT, Traits>::off_type off_type;
typedef typename std::basic_streambuf<CharT, Traits>::pos_type pos_type;
typedef typename std::basic_streambuf<CharT, Traits>::int_type int_type;
typedef typename std::basic_streambuf<CharT, Traits>::traits_type traits_type;
private:
const string_type & s;
off_type off;
public:
basic_istringbuf_nocopy(const string_type & s) : s{s}, off{0}
{
}
private:
pos_type seekoff(off_type off, std::ios_base::seekdir dir, std::ios_base::openmode which)
{
if (which & std::ios_base::in) {
this->off = dir == std::ios_base::beg
? off
: (dir == std::ios_base::end
? s.size() + off
: this->off + off);
}
return pos_type(this->off);
}
pos_type seekpos(pos_type pos, std::ios_base::openmode which)
{
return seekoff(pos, std::ios_base::beg, which);
}
std::streamsize showmanyc()
{
return s.size() - off;
}
int_type underflow()
{
if (typename string_type::size_type(off) == s.size())
return traits_type::eof();
return traits_type::to_int_type(s[off]);
}
int_type uflow()
{
if (typename string_type::size_type(off) == s.size())
return traits_type::eof();
return traits_type::to_int_type(s[off++]);
}
int_type pbackfail(int_type ch)
{
if (off == 0 || (ch != traits_type::eof() && ch != s[off - 1]))
return traits_type::eof();
return traits_type::to_int_type(s[--off]);
}
};
template <class CharT, class Traits = std::char_traits<CharT>, class Allocator = std::allocator<CharT>>
class basic_istringstream_nocopy : public std::basic_iostream<CharT, Traits>
{
typedef basic_istringbuf_nocopy<CharT, Traits, Allocator> buf_type;
buf_type buf;
public:
basic_istringstream_nocopy(const typename buf_type::string_type & s) :
std::basic_iostream<CharT, Traits>(&buf), buf(s) {};
};
typedef basic_istringstream_nocopy<char> istringstream_nocopy;
+2 -7
View File
@@ -1,6 +1,5 @@
#include "logging.hh" #include "logging.hh"
#include "util.hh" #include "util.hh"
#include "config.hh"
#include <atomic> #include <atomic>
#include <nlohmann/json.hpp> #include <nlohmann/json.hpp>
@@ -8,10 +7,6 @@
namespace nix { namespace nix {
LoggerSettings loggerSettings;
static GlobalConfig::Register r1(&loggerSettings);
static thread_local ActivityId curActivity = 0; static thread_local ActivityId curActivity = 0;
ActivityId getCurActivity() ActivityId getCurActivity()
@@ -77,7 +72,7 @@ public:
void logEI(const ErrorInfo & ei) override void logEI(const ErrorInfo & ei) override
{ {
std::stringstream oss; std::stringstream oss;
showErrorInfo(oss, ei, loggerSettings.showTrace.get()); oss << ei;
log(ei.level, oss.str()); log(ei.level, oss.str());
} }
@@ -178,7 +173,7 @@ struct JSONLogger : Logger {
void logEI(const ErrorInfo & ei) override void logEI(const ErrorInfo & ei) override
{ {
std::ostringstream oss; std::ostringstream oss;
showErrorInfo(oss, ei, loggerSettings.showTrace.get()); oss << ei;
nlohmann::json json; nlohmann::json json;
json["action"] = "msg"; json["action"] = "msg";
-11
View File
@@ -2,7 +2,6 @@
#include "types.hh" #include "types.hh"
#include "error.hh" #include "error.hh"
#include "config.hh"
namespace nix { namespace nix {
@@ -35,16 +34,6 @@ typedef enum {
typedef uint64_t ActivityId; typedef uint64_t ActivityId;
struct LoggerSettings : Config
{
Setting<bool> showTrace{this,
false,
"show-trace",
"Whether to show a stack trace on evaluation errors."};
};
extern LoggerSettings loggerSettings;
class Logger class Logger
{ {
friend struct Activity; friend struct Activity;
+5 -41
View File
@@ -166,30 +166,17 @@ struct StringSource : Source
}; };
/* A sink that writes all incoming data to two other sinks. */ /* Adapter class of a Source that saves all data read to `s'. */
struct TeeSink : Sink
{
Sink & sink1, & sink2;
TeeSink(Sink & sink1, Sink & sink2) : sink1(sink1), sink2(sink2) { }
virtual void operator () (const unsigned char * data, size_t len)
{
sink1(data, len);
sink2(data, len);
}
};
/* Adapter class of a Source that saves all data read to a sink. */
struct TeeSource : Source struct TeeSource : Source
{ {
Source & orig; Source & orig;
Sink & sink; ref<std::string> data;
TeeSource(Source & orig, Sink & sink) TeeSource(Source & orig)
: orig(orig), sink(sink) { } : orig(orig), data(make_ref<std::string>()) { }
size_t read(unsigned char * data, size_t len) size_t read(unsigned char * data, size_t len)
{ {
size_t n = orig.read(data, len); size_t n = orig.read(data, len);
sink(data, len); this->data->append((const char *) data, n);
return n; return n;
} }
}; };
@@ -349,27 +336,4 @@ Source & operator >> (Source & in, bool & b)
} }
/* An adapter that converts a std::basic_istream into a source. */
struct StreamToSourceAdapter : Source
{
std::shared_ptr<std::basic_istream<char>> istream;
StreamToSourceAdapter(std::shared_ptr<std::basic_istream<char>> istream)
: istream(istream)
{ }
size_t read(unsigned char * data, size_t len) override
{
if (!istream->read((char *) data, len)) {
if (istream->eof()) {
if (istream->gcount() == 0)
throw EndOfFile("end of file");
} else
throw Error("I/O error in StreamToSourceAdapter");
}
return istream->gcount();
}
};
} }
+32 -101
View File
@@ -11,13 +11,6 @@ namespace nix {
* logEI * logEI
* --------------------------------------------------------------------------*/ * --------------------------------------------------------------------------*/
const char *test_file =
"previous line of code\n"
"this is the problem line of code\n"
"next line of code\n";
const char *one_liner =
"this is the other problem line of code";
TEST(logEI, catpuresBasicProperties) { TEST(logEI, catpuresBasicProperties) {
MakeError(TestError, Error); MakeError(TestError, Error);
@@ -144,6 +137,7 @@ namespace nix {
* logError * logError
* --------------------------------------------------------------------------*/ * --------------------------------------------------------------------------*/
TEST(logError, logErrorWithoutHintOrCode) { TEST(logError, logErrorWithoutHintOrCode) {
testing::internal::CaptureStderr(); testing::internal::CaptureStderr();
@@ -158,7 +152,7 @@ namespace nix {
TEST(logError, logErrorWithPreviousAndNextLinesOfCode) { TEST(logError, logErrorWithPreviousAndNextLinesOfCode) {
SymbolTable testTable; SymbolTable testTable;
auto problem_file = testTable.create(test_file); auto problem_file = testTable.create("myfile.nix");
testing::internal::CaptureStderr(); testing::internal::CaptureStderr();
@@ -168,16 +162,21 @@ namespace nix {
.hint = hintfmt("this hint has %1% templated %2%!!", .hint = hintfmt("this hint has %1% templated %2%!!",
"yellow", "yellow",
"values"), "values"),
.errPos = Pos(foString, problem_file, 02, 13), .nixCode = NixCode {
}); .errPos = Pos(problem_file, 40, 13),
.prevLineOfCode = "previous line of code",
.errLineOfCode = "this is the problem line of code",
.nextLineOfCode = "next line of code",
}});
auto str = testing::internal::GetCapturedStderr(); auto str = testing::internal::GetCapturedStderr();
ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- error name --- error-unit-test\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(2:13)\x1B[34;1m from string\x1B[0m\n\nerror with code lines\n\n 1| previous line of code\n 2| this is the problem line of code\n | \x1B[31;1m^\x1B[0m\n 3| next line of code\n\nthis hint has \x1B[33;1myellow\x1B[0m templated \x1B[33;1mvalues\x1B[0m!!\n"); ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- error name --- error-unit-test\x1B[0m\nin file: \x1B[34;1mmyfile.nix (40:13)\x1B[0m\n\nerror with code lines\n\n 39| previous line of code\n 40| this is the problem line of code\n | \x1B[31;1m^\x1B[0m\n 41| next line of code\n\nthis hint has \x1B[33;1myellow\x1B[0m templated \x1B[33;1mvalues\x1B[0m!!\n");
} }
TEST(logError, logErrorWithInvalidFile) { TEST(logError, logErrorWithoutLinesOfCode) {
SymbolTable testTable; SymbolTable testTable;
auto problem_file = testTable.create("invalid filename"); auto problem_file = testTable.create("myfile.nix");
testing::internal::CaptureStderr(); testing::internal::CaptureStderr();
logError({ logError({
@@ -186,23 +185,28 @@ namespace nix {
.hint = hintfmt("this hint has %1% templated %2%!!", .hint = hintfmt("this hint has %1% templated %2%!!",
"yellow", "yellow",
"values"), "values"),
.errPos = Pos(foFile, problem_file, 02, 13) .nixCode = NixCode {
}); .errPos = Pos(problem_file, 40, 13)
}});
auto str = testing::internal::GetCapturedStderr(); auto str = testing::internal::GetCapturedStderr();
ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- SysError --- error-unit-test\x1B[0m\nopening file '\x1B[33;1minvalid filename\x1B[0m': \x1B[33;1mNo such file or directory\x1B[0m\n\x1B[31;1merror:\x1B[0m\x1B[34;1m --- error name --- error-unit-test\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(2:13)\x1B[34;1m in file: \x1B[0minvalid filename\n\nerror without any code lines.\n\nthis hint has \x1B[33;1myellow\x1B[0m templated \x1B[33;1mvalues\x1B[0m!!\n"); ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- error name --- error-unit-test\x1B[0m\nin file: \x1B[34;1mmyfile.nix (40:13)\x1B[0m\n\nerror without any code lines.\n\nthis hint has \x1B[33;1myellow\x1B[0m templated \x1B[33;1mvalues\x1B[0m!!\n");
} }
TEST(logError, logErrorWithOnlyHintAndName) { TEST(logError, logErrorWithOnlyHintAndName) {
SymbolTable testTable;
auto problem_file = testTable.create("myfile.nix");
testing::internal::CaptureStderr(); testing::internal::CaptureStderr();
logError({ logError({
.name = "error name", .name = "error name",
.hint = hintfmt("hint %1%", "only"), .hint = hintfmt("hint %1%", "only"),
}); .nixCode = NixCode {
.errPos = Pos(problem_file, 40, 13)
}});
auto str = testing::internal::GetCapturedStderr(); auto str = testing::internal::GetCapturedStderr();
ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- error name --- error-unit-test\x1B[0m\nhint \x1B[33;1monly\x1B[0m\n"); ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- error name --- error-unit-test\x1B[0m\nin file: \x1B[34;1mmyfile.nix (40:13)\x1B[0m\n\nhint \x1B[33;1monly\x1B[0m\n");
} }
@@ -215,18 +219,18 @@ namespace nix {
logWarning({ logWarning({
.name = "name", .name = "name",
.description = "warning description", .description = "error description",
.hint = hintfmt("there was a %1%", "warning"), .hint = hintfmt("there was a %1%", "warning"),
}); });
auto str = testing::internal::GetCapturedStderr(); auto str = testing::internal::GetCapturedStderr();
ASSERT_STREQ(str.c_str(), "\x1B[33;1mwarning:\x1B[0m\x1B[34;1m --- name --- error-unit-test\x1B[0m\nwarning description\n\nthere was a \x1B[33;1mwarning\x1B[0m\n"); ASSERT_STREQ(str.c_str(), "\x1B[33;1mwarning:\x1B[0m\x1B[34;1m --- name --- error-unit-test\x1B[0m\nerror description\n\nthere was a \x1B[33;1mwarning\x1B[0m\n");
} }
TEST(logWarning, logWarningWithFileLineNumAndCode) { TEST(logWarning, logWarningWithFileLineNumAndCode) {
SymbolTable testTable; SymbolTable testTable;
auto problem_file = testTable.create(test_file); auto problem_file = testTable.create("myfile.nix");
testing::internal::CaptureStderr(); testing::internal::CaptureStderr();
@@ -236,73 +240,18 @@ namespace nix {
.hint = hintfmt("this hint has %1% templated %2%!!", .hint = hintfmt("this hint has %1% templated %2%!!",
"yellow", "yellow",
"values"), "values"),
.errPos = Pos(foStdin, problem_file, 2, 13), .nixCode = NixCode {
}); .errPos = Pos(problem_file, 40, 13),
.prevLineOfCode = std::nullopt,
.errLineOfCode = "this is the problem line of code",
.nextLineOfCode = std::nullopt
}});
auto str = testing::internal::GetCapturedStderr(); auto str = testing::internal::GetCapturedStderr();
ASSERT_STREQ(str.c_str(), "\x1B[33;1mwarning:\x1B[0m\x1B[34;1m --- warning name --- error-unit-test\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(2:13)\x1B[34;1m from stdin\x1B[0m\n\nwarning description\n\n 1| previous line of code\n 2| this is the problem line of code\n | \x1B[31;1m^\x1B[0m\n 3| next line of code\n\nthis hint has \x1B[33;1myellow\x1B[0m templated \x1B[33;1mvalues\x1B[0m!!\n"); ASSERT_STREQ(str.c_str(), "\x1B[33;1mwarning:\x1B[0m\x1B[34;1m --- warning name --- error-unit-test\x1B[0m\nin file: \x1B[34;1mmyfile.nix (40:13)\x1B[0m\n\nwarning description\n\n 40| this is the problem line of code\n | \x1B[31;1m^\x1B[0m\n\nthis hint has \x1B[33;1myellow\x1B[0m templated \x1B[33;1mvalues\x1B[0m!!\n");
} }
/* ----------------------------------------------------------------------------
* traces
* --------------------------------------------------------------------------*/
TEST(addTrace, showTracesWithShowTrace) {
SymbolTable testTable;
auto problem_file = testTable.create(test_file);
auto oneliner_file = testTable.create(one_liner);
auto invalidfilename = testTable.create("invalid filename");
auto e = AssertionError(ErrorInfo {
.name = "wat",
.description = "show-traces",
.hint = hintfmt("it has been %1% days since our last error", "zero"),
.errPos = Pos(foString, problem_file, 2, 13),
});
e.addTrace(Pos(foStdin, oneliner_file, 1, 19), "while trying to compute %1%", 42);
e.addTrace(std::nullopt, "while doing something without a %1%", "pos");
e.addTrace(Pos(foFile, invalidfilename, 100, 1), "missing %s", "nix file");
testing::internal::CaptureStderr();
loggerSettings.showTrace.assign(true);
logError(e.info());
auto str = testing::internal::GetCapturedStderr();
ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- SysError --- error-unit-test\x1B[0m\nopening file '\x1B[33;1minvalid filename\x1B[0m': \x1B[33;1mNo such file or directory\x1B[0m\n\x1B[31;1merror:\x1B[0m\x1B[34;1m --- AssertionError --- error-unit-test\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(2:13)\x1B[34;1m from string\x1B[0m\n\nshow-traces\n\n 1| previous line of code\n 2| this is the problem line of code\n | \x1B[31;1m^\x1B[0m\n 3| next line of code\n\nit has been \x1B[33;1mzero\x1B[0m days since our last error\n\x1B[34;1m---- show-trace ----\x1B[0m\n\x1B[34;1mtrace: \x1B[0mwhile trying to compute \x1B[33;1m42\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(1:19)\x1B[34;1m from stdin\x1B[0m\n\n 1| this is the other problem line of code\n | \x1B[31;1m^\x1B[0m\n\n\x1B[34;1mtrace: \x1B[0mwhile doing something without a \x1B[33;1mpos\x1B[0m\n\x1B[34;1mtrace: \x1B[0mmissing \x1B[33;1mnix file\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(100:1)\x1B[34;1m in file: \x1B[0minvalid filename\n");
}
TEST(addTrace, hideTracesWithoutShowTrace) {
SymbolTable testTable;
auto problem_file = testTable.create(test_file);
auto oneliner_file = testTable.create(one_liner);
auto invalidfilename = testTable.create("invalid filename");
auto e = AssertionError(ErrorInfo {
.name = "wat",
.description = "hide traces",
.hint = hintfmt("it has been %1% days since our last error", "zero"),
.errPos = Pos(foString, problem_file, 2, 13),
});
e.addTrace(Pos(foStdin, oneliner_file, 1, 19), "while trying to compute %1%", 42);
e.addTrace(std::nullopt, "while doing something without a %1%", "pos");
e.addTrace(Pos(foFile, invalidfilename, 100, 1), "missing %s", "nix file");
testing::internal::CaptureStderr();
loggerSettings.showTrace.assign(false);
logError(e.info());
auto str = testing::internal::GetCapturedStderr();
ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- AssertionError --- error-unit-test\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(2:13)\x1B[34;1m from string\x1B[0m\n\nhide traces\n\n 1| previous line of code\n 2| this is the problem line of code\n | \x1B[31;1m^\x1B[0m\n 3| next line of code\n\nit has been \x1B[33;1mzero\x1B[0m days since our last error\n");
}
/* ---------------------------------------------------------------------------- /* ----------------------------------------------------------------------------
* hintfmt * hintfmt
* --------------------------------------------------------------------------*/ * --------------------------------------------------------------------------*/
@@ -340,22 +289,4 @@ namespace nix {
"what about this " ANSI_YELLOW "%3%" ANSI_NORMAL " " ANSI_YELLOW "one" ANSI_NORMAL); "what about this " ANSI_YELLOW "%3%" ANSI_NORMAL " " ANSI_YELLOW "one" ANSI_NORMAL);
} }
/* ----------------------------------------------------------------------------
* ErrPos
* --------------------------------------------------------------------------*/
TEST(errpos, invalidPos) {
// contains an invalid symbol, which we should not dereference!
Pos invalid;
// constructing without access violation.
ErrPos ep(invalid);
// assignment without access violation.
ep = invalid;
}
} }
+2 -2
View File
@@ -593,7 +593,7 @@ static void upgradeDerivations(Globals & globals,
} else newElems.push_back(i); } else newElems.push_back(i);
} catch (Error & e) { } catch (Error & e) {
e.addTrace(std::nullopt, "while trying to find an upgrade for '%s'", i.queryName()); e.addPrefix(fmt("while trying to find an upgrade for '%s':\n", i.queryName()));
throw; throw;
} }
} }
@@ -1185,7 +1185,7 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs)
} catch (AssertionError & e) { } catch (AssertionError & e) {
printMsg(lvlTalkative, "skipping derivation named '%1%' which gives an assertion failure", i.queryName()); printMsg(lvlTalkative, "skipping derivation named '%1%' which gives an assertion failure", i.queryName());
} catch (Error & e) { } catch (Error & e) {
e.addTrace(std::nullopt, "while querying the derivation named '%1%'", i.queryName()); e.addPrefix(fmt("while querying the derivation named '%1%':\n", i.queryName()));
throw; throw;
} }
} }
+18 -9
View File
@@ -153,15 +153,14 @@ static int _main(int argc, char * * argv)
/* If an expected hash is given, the file may already exist in /* If an expected hash is given, the file may already exist in
the store. */ the store. */
std::optional<Hash> expectedHash; Hash hash, expectedHash(ht);
Hash hash;
std::optional<StorePath> storePath; std::optional<StorePath> storePath;
if (args.size() == 2) { if (args.size() == 2) {
expectedHash = Hash(args[1], ht); expectedHash = Hash(args[1], ht);
const auto recursive = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; const auto recursive = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
storePath = store->makeFixedOutputPath(recursive, *expectedHash, name); storePath = store->makeFixedOutputPath(recursive, expectedHash, name);
if (store->isValidPath(*storePath)) if (store->isValidPath(*storePath))
hash = *expectedHash; hash = expectedHash;
else else
storePath.reset(); storePath.reset();
} }
@@ -201,12 +200,22 @@ static int _main(int argc, char * * argv)
tmpFile = unpacked; tmpFile = unpacked;
} }
const auto method = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; /* FIXME: inefficient; addToStore() will also hash
this. */
hash = unpack ? hashPath(ht, tmpFile).first : hashFile(ht, tmpFile);
auto info = store->addToStoreSlow(name, tmpFile, method, ht, expectedHash); if (expectedHash != Hash(ht) && expectedHash != hash)
storePath = info.path; throw Error("hash mismatch for '%1%'", uri);
assert(info.ca);
hash = getContentAddressHash(*info.ca); const auto recursive = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
/* Copy the file to the Nix store. FIXME: if RemoteStore
implemented addToStoreFromDump() and downloadFile()
supported a sink, we could stream the download directly
into the Nix store. */
storePath = store->addToStore(name, tmpFile, recursive, ht);
assert(*storePath == store->makeFixedOutputPath(recursive, hash, name));
} }
stopProgressBar(); stopProgressBar();
+5 -5
View File
@@ -174,10 +174,10 @@ static void opAdd(Strings opFlags, Strings opArgs)
store. */ store. */
static void opAddFixed(Strings opFlags, Strings opArgs) static void opAddFixed(Strings opFlags, Strings opArgs)
{ {
auto method = FileIngestionMethod::Flat; auto recursive = FileIngestionMethod::Flat;
for (auto & i : opFlags) for (auto & i : opFlags)
if (i == "--recursive") method = FileIngestionMethod::Recursive; if (i == "--recursive") recursive = FileIngestionMethod::Recursive;
else throw UsageError("unknown flag '%1%'", i); else throw UsageError("unknown flag '%1%'", i);
if (opArgs.empty()) if (opArgs.empty())
@@ -187,7 +187,7 @@ static void opAddFixed(Strings opFlags, Strings opArgs)
opArgs.pop_front(); opArgs.pop_front();
for (auto & i : opArgs) for (auto & i : opArgs)
std::cout << fmt("%s\n", store->printStorePath(store->addToStoreSlow(baseNameOf(i), i, method, hashAlgo).path)); cout << fmt("%s\n", store->printStorePath(store->addToStore(std::string(baseNameOf(i)), i, recursive, hashAlgo)));
} }
@@ -671,7 +671,7 @@ static void opImport(Strings opFlags, Strings opArgs)
if (!opArgs.empty()) throw UsageError("no arguments expected"); if (!opArgs.empty()) throw UsageError("no arguments expected");
FdSource source(STDIN_FILENO); FdSource source(STDIN_FILENO);
auto paths = store->importPaths(source, NoCheckSigs); auto paths = store->importPaths(source, nullptr, NoCheckSigs);
for (auto & i : paths) for (auto & i : paths)
cout << fmt("%s\n", store->printStorePath(i)) << std::flush; cout << fmt("%s\n", store->printStorePath(i)) << std::flush;
@@ -878,7 +878,7 @@ static void opServe(Strings opFlags, Strings opArgs)
case cmdImportPaths: { case cmdImportPaths: {
if (!writeAllowed) throw Error("importing paths is not allowed"); if (!writeAllowed) throw Error("importing paths is not allowed");
store->importPaths(in, NoCheckSigs); // FIXME: should we skip sig checking? store->importPaths(in, nullptr, NoCheckSigs); // FIXME: should we skip sig checking?
out << 1; // indicate success out << 1; // indicate success
break; break;
} }
+8 -2
View File
@@ -50,7 +50,7 @@ BuildEnvironment readEnvironment(const Path & path)
R"re((?:\$?'(?:[^'\\]|\\[abeEfnrtv\\'"?])*'))re"; R"re((?:\$?'(?:[^'\\]|\\[abeEfnrtv\\'"?])*'))re";
static std::string indexedArrayRegex = static std::string indexedArrayRegex =
R"re((?:\(( *\[[0-9]+\]="(?:[^"\\]|\\.)*")*\)))re"; R"re((?:\(( *\[[0-9]+]="(?:[^"\\]|\\.)*")**\)))re";
static std::regex varRegex( static std::regex varRegex(
"^(" + varNameRegex + ")=(" + simpleStringRegex + "|" + quotedStringRegex + "|" + indexedArrayRegex + ")\n"); "^(" + varNameRegex + ")=(" + simpleStringRegex + "|" + quotedStringRegex + "|" + indexedArrayRegex + ")\n");
@@ -135,7 +135,13 @@ StorePath getDerivationEnvironment(ref<Store> store, const StorePath & drvPath)
drv.inputSrcs.insert(std::move(getEnvShPath)); drv.inputSrcs.insert(std::move(getEnvShPath));
Hash h = hashDerivationModulo(*store, drv, true); Hash h = hashDerivationModulo(*store, drv, true);
auto shellOutPath = store->makeOutputPath("out", h, drvName); auto shellOutPath = store->makeOutputPath("out", h, drvName);
drv.outputs.insert_or_assign("out", DerivationOutput { .path = shellOutPath }); drv.outputs.insert_or_assign("out", DerivationOutput {
.path = shellOutPath,
.hash = FixedOutputHash {
.method = FileIngestionMethod::Flat,
.hash = Hash { },
},
});
drv.env["out"] = store->printStorePath(shellOutPath); drv.env["out"] = store->printStorePath(shellOutPath);
auto shellDrvPath2 = writeDerivation(store, drv, drvName); auto shellDrvPath2 = writeDerivation(store, drv, drvName);
-134
View File
@@ -1,134 +0,0 @@
#include "command.hh"
#include "shared.hh"
#include "store-api.hh"
#include "common-args.hh"
#include "names.hh"
#include <regex>
using namespace nix;
struct Info
{
std::string outputName;
};
// name -> version -> store paths
typedef std::map<std::string, std::map<std::string, std::map<StorePath, Info>>> GroupedPaths;
GroupedPaths getClosureInfo(ref<Store> store, const StorePath & toplevel)
{
StorePathSet closure;
store->computeFSClosure({toplevel}, closure);
GroupedPaths groupedPaths;
for (auto & path : closure) {
/* Strip the output name. Unfortunately this is ambiguous (we
can't distinguish between output names like "bin" and
version suffixes like "unstable"). */
static std::regex regex("(.*)-([a-z]+|lib32|lib64)");
std::smatch match;
std::string name(path.name());
std::string outputName;
if (std::regex_match(name, match, regex)) {
name = match[1];
outputName = match[2];
}
DrvName drvName(name);
groupedPaths[drvName.name][drvName.version].emplace(path, Info { .outputName = outputName });
}
return groupedPaths;
}
std::string showVersions(const std::set<std::string> & versions)
{
if (versions.empty()) return "∅";
std::set<std::string> versions2;
for (auto & version : versions)
versions2.insert(version.empty() ? "ε" : version);
return concatStringsSep(", ", versions2);
}
struct CmdDiffClosures : SourceExprCommand
{
std::string _before, _after;
CmdDiffClosures()
{
expectArg("before", &_before);
expectArg("after", &_after);
}
std::string description() override
{
return "show what packages and versions were added and removed between two closures";
}
Category category() override { return catSecondary; }
Examples examples() override
{
return {
{
"To show what got added and removed between two versions of the NixOS system profile:",
"nix diff-closures /nix/var/nix/profiles/system-655-link /nix/var/nix/profiles/system-658-link",
},
};
}
void run(ref<Store> store) override
{
auto before = parseInstallable(*this, store, _before, false);
auto beforePath = toStorePath(store, Build, before);
auto after = parseInstallable(*this, store, _after, false);
auto afterPath = toStorePath(store, NoBuild, after);
auto beforeClosure = getClosureInfo(store, beforePath);
auto afterClosure = getClosureInfo(store, afterPath);
std::set<std::string> allNames;
for (auto & [name, _] : beforeClosure) allNames.insert(name);
for (auto & [name, _] : afterClosure) allNames.insert(name);
for (auto & name : allNames) {
auto & beforeVersions = beforeClosure[name];
auto & afterVersions = afterClosure[name];
auto totalSize = [&](const std::map<std::string, std::map<StorePath, Info>> & versions)
{
uint64_t sum = 0;
for (auto & [_, paths] : versions)
for (auto & [path, _] : paths)
sum += store->queryPathInfo(path)->narSize;
return sum;
};
auto beforeSize = totalSize(beforeVersions);
auto afterSize = totalSize(afterVersions);
auto sizeDelta = (int64_t) afterSize - (int64_t) beforeSize;
auto showDelta = abs(sizeDelta) >= 8 * 1024;
std::set<std::string> removed, unchanged;
for (auto & [version, _] : beforeVersions)
if (!afterVersions.count(version)) removed.insert(version); else unchanged.insert(version);
std::set<std::string> added;
for (auto & [version, _] : afterVersions)
if (!beforeVersions.count(version)) added.insert(version);
if (showDelta || !removed.empty() || !added.empty()) {
std::vector<std::string> items;
if (!removed.empty() || !added.empty())
items.push_back(fmt("%s → %s", showVersions(removed), showVersions(added)));
if (showDelta)
items.push_back(fmt("%s%+.1f KiB" ANSI_NORMAL, sizeDelta > 0 ? ANSI_RED : ANSI_GREEN, sizeDelta / 1024.0));
std::cout << fmt("%s: %s\n", name, concatStringsSep(", ", items));
}
}
}
};
static auto r1 = registerCommand<CmdDiffClosures>("diff-closures");
+7 -7
View File
@@ -94,8 +94,8 @@ struct InstallableStorePath : Installable
ref<Store> store; ref<Store> store;
StorePath storePath; StorePath storePath;
InstallableStorePath(ref<Store> store, StorePath && storePath) InstallableStorePath(ref<Store> store, const Path & storePath)
: store(store), storePath(std::move(storePath)) { } : store(store), storePath(store->parseStorePath(storePath)) { }
std::string what() override { return store->printStorePath(storePath); } std::string what() override { return store->printStorePath(storePath); }
@@ -228,11 +228,11 @@ static std::vector<std::shared_ptr<Installable>> parseInstallables(
result.push_back(std::make_shared<InstallableExpr>(cmd, s)); result.push_back(std::make_shared<InstallableExpr>(cmd, s));
else if (s.find("/") != std::string::npos) { else if (s.find("/") != std::string::npos) {
try {
result.push_back(std::make_shared<InstallableStorePath>( auto path = store->toStorePath(store->followLinksToStore(s));
store,
store->toStorePath(store->followLinksToStore(s)).first)); if (store->isStorePath(path))
} catch (BadStorePath &) { } result.push_back(std::make_shared<InstallableStorePath>(store, path));
} }
else if (s == "" || std::regex_match(s, attrPathRegex)) else if (s == "" || std::regex_match(s, attrPathRegex))
+3 -3
View File
@@ -211,12 +211,12 @@ void NixRepl::mainLoop(const std::vector<std::string> & files)
// input without clearing the input so far. // input without clearing the input so far.
continue; continue;
} else { } else {
printMsg(lvlError, e.msg()); printMsg(lvlError, error + "%1%%2%", (settings.showTrace ? e.prefix() : ""), e.msg());
} }
} catch (Error & e) { } catch (Error & e) {
printMsg(lvlError, e.msg()); printMsg(lvlError, error + "%1%%2%", (settings.showTrace ? e.prefix() : ""), e.msg());
} catch (Interrupted & e) { } catch (Interrupted & e) {
printMsg(lvlError, e.msg()); printMsg(lvlError, error + "%1%%2%", (settings.showTrace ? e.prefix() : ""), e.msg());
} }
// We handled the current input fully, so we should clear it // We handled the current input fully, so we should clear it
+1 -1
View File
@@ -216,7 +216,7 @@ struct CmdSearch : SourceExprCommand, MixJSON
} catch (AssertionError & e) { } catch (AssertionError & e) {
} catch (Error & e) { } catch (Error & e) {
if (!toplevel) { if (!toplevel) {
e.addTrace(std::nullopt, "While evaluating the attribute '%s'", attrPath); e.addPrefix(fmt("While evaluating the attribute '%s':\n", attrPath));
throw; throw;
} }
} }
+2 -5
View File
@@ -77,16 +77,13 @@ struct CmdVerify : StorePathsCommand
try { try {
checkInterrupt(); checkInterrupt();
Activity act2(*logger, lvlInfo, actUnknown, fmt("checking '%s'", storePath));
MaintainCount<std::atomic<size_t>> mcActive(active); MaintainCount<std::atomic<size_t>> mcActive(active);
update(); update();
auto info = store->queryPathInfo(store->parseStorePath(storePath)); auto info = store->queryPathInfo(store->parseStorePath(storePath));
// Note: info->path can be different from storePath
// for binary cache stores when using --all (since we
// can't enumerate names efficiently).
Activity act2(*logger, lvlInfo, actUnknown, fmt("checking '%s'", store->printStorePath(info->path)));
if (!noContents) { if (!noContents) {
std::unique_ptr<AbstractHashSink> hashSink; std::unique_ptr<AbstractHashSink> hashSink;
-53
View File
@@ -182,56 +182,3 @@ clearCacheCache
nix-store -r $outPath --substituters "file://$cacheDir2 file://$cacheDir" --trusted-public-keys "$publicKey" nix-store -r $outPath --substituters "file://$cacheDir2 file://$cacheDir" --trusted-public-keys "$publicKey"
fi # HAVE_LIBSODIUM fi # HAVE_LIBSODIUM
unset _NIX_FORCE_HTTP
# Test 'nix verify --all' on a binary cache.
nix verify -vvvvv --all --store file://$cacheDir --no-trust
# Test local NAR caching.
narCache=$TEST_ROOT/nar-cache
rm -rf $narCache
mkdir $narCache
[[ $(nix cat-store --store "file://$cacheDir?local-nar-cache=$narCache" $outPath/foobar) = FOOBAR ]]
rm -rfv "$cacheDir/nar"
[[ $(nix cat-store --store "file://$cacheDir?local-nar-cache=$narCache" $outPath/foobar) = FOOBAR ]]
(! nix cat-store --store file://$cacheDir $outPath/foobar)
# Test NAR listing generation.
clearCache
outPath=$(nix-build --no-out-link -E '
with import ./config.nix;
mkDerivation {
name = "nar-listing";
buildCommand = "mkdir $out; echo foo > $out/bar; ln -s xyzzy $out/link";
}
')
nix copy --to file://$cacheDir?write-nar-listing=1 $outPath
[[ $(cat $cacheDir/$(basename $outPath).ls) = '{"version":1,"root":{"type":"directory","entries":{"bar":{"type":"regular","size":4,"narOffset":232},"link":{"type":"symlink","target":"xyzzy"}}}}' ]]
# Test debug info index generation.
clearCache
outPath=$(nix-build --no-out-link -E '
with import ./config.nix;
mkDerivation {
name = "debug-info";
buildCommand = "mkdir -p $out/lib/debug/.build-id/02; echo foo > $out/lib/debug/.build-id/02/623eda209c26a59b1a8638ff7752f6b945c26b.debug";
}
')
nix copy --to "file://$cacheDir?index-debug-info=1&compression=none" $outPath
[[ $(cat $cacheDir/debuginfo/02623eda209c26a59b1a8638ff7752f6b945c26b.debug) = '{"archive":"../nar/100vxs724qr46phz8m24iswmg9p3785hsyagz0kchf6q6gf06sw6.nar","member":"lib/debug/.build-id/02/623eda209c26a59b1a8638ff7752f6b945c26b.debug"}' ]]
+8 -24
View File
@@ -1,39 +1,23 @@
{ busybox }:
with import ./config.nix; with import ./config.nix;
let let
mkDerivation = args:
derivation ({
inherit system;
builder = busybox;
args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")];
} // removeAttrs args ["builder" "meta"])
// { meta = args.meta or {}; };
input1 = mkDerivation { input1 = mkDerivation {
shell = busybox; name = "build-hook-input-1";
name = "build-remote-input-1"; buildCommand = "mkdir $out; echo FOO > $out/foo";
buildCommand = "echo FOO > $out";
requiredSystemFeatures = ["foo"]; requiredSystemFeatures = ["foo"];
}; };
input2 = mkDerivation { input2 = mkDerivation {
shell = busybox; name = "build-hook-input-2";
name = "build-remote-input-2"; buildCommand = "mkdir $out; echo BAR > $out/bar";
buildCommand = "echo BAR > $out";
}; };
in in
mkDerivation { mkDerivation {
shell = busybox; name = "build-hook";
name = "build-remote"; builder = ./dependencies.builder0.sh;
buildCommand = input1 = " " + input1 + "/.";
'' input2 = " ${input2}/.";
read x < ${input1}
read y < ${input2}
echo $x$y > $out
'';
} }
+13 -20
View File
@@ -3,29 +3,22 @@ source common.sh
clearStore clearStore
if ! canUseSandbox; then exit; fi if ! canUseSandbox; then exit; fi
if ! [[ $busybox =~ busybox ]]; then exit; fi if [[ ! $SHELL =~ /nix/store ]]; then exit; fi
chmod -R u+w $TEST_ROOT/machine0 || true chmod -R u+w $TEST_ROOT/store0 || true
chmod -R u+w $TEST_ROOT/machine1 || true chmod -R u+w $TEST_ROOT/store1 || true
chmod -R u+w $TEST_ROOT/machine2 || true rm -rf $TEST_ROOT/store0 $TEST_ROOT/store1
rm -rf $TEST_ROOT/machine0 $TEST_ROOT/machine1 $TEST_ROOT/machine2
rm -f $TEST_ROOT/result
unset NIX_STORE_DIR nix build -f build-hook.nix -o $TEST_ROOT/result --max-jobs 0 \
unset NIX_STATE_DIR --sandbox-paths /nix/store --sandbox-build-dir /build-tmp \
--builders "$TEST_ROOT/store0; $TEST_ROOT/store1 - - 1 1 foo" \
# Note: ssh://localhost bypasses ssh, directly invoking nix-store as a
# child process. This allows us to test LegacySSHStore::buildDerivation().
nix build -L -v -f build-hook.nix -o $TEST_ROOT/result --max-jobs 0 \
--arg busybox $busybox \
--store $TEST_ROOT/machine0 \
--builders "ssh://localhost?remote-store=$TEST_ROOT/machine1; $TEST_ROOT/machine2 - - 1 1 foo" \
--system-features foo --system-features foo
outPath=$(readlink -f $TEST_ROOT/result) outPath=$TEST_ROOT/result
cat $TEST_ROOT/machine0/$outPath | grep FOOBAR cat $outPath/foobar | grep FOOBAR
# Ensure that input1 was built on store2 due to the required feature. # Ensure that input1 was built on store1 due to the required feature.
(! nix path-info --store $TEST_ROOT/machine1 --all | grep builder-build-remote-input-1.sh) p=$(readlink -f $outPath/input-2)
nix path-info --store $TEST_ROOT/machine2 --all | grep builder-build-remote-input-1.sh (! nix path-info --store $TEST_ROOT/store0 --all | grep builder-build-hook-input-1.sh)
nix path-info --store $TEST_ROOT/store1 --all | grep builder-build-hook-input-1.sh
+2 -4
View File
@@ -1,6 +1,6 @@
set -e set -e
export TEST_ROOT=$(realpath ${TMPDIR:-/tmp}/nix-test)/${TEST_NAME:-default} export TEST_ROOT=$(realpath ${TMPDIR:-/tmp}/nix-test)
export NIX_STORE_DIR export NIX_STORE_DIR
if ! NIX_STORE_DIR=$(readlink -f $TEST_ROOT/store 2> /dev/null); then if ! NIX_STORE_DIR=$(readlink -f $TEST_ROOT/store 2> /dev/null); then
# Maybe the build directory is symlinked. # Maybe the build directory is symlinked.
@@ -11,7 +11,6 @@ export NIX_LOCALSTATE_DIR=$TEST_ROOT/var
export NIX_LOG_DIR=$TEST_ROOT/var/log/nix export NIX_LOG_DIR=$TEST_ROOT/var/log/nix
export NIX_STATE_DIR=$TEST_ROOT/var/nix export NIX_STATE_DIR=$TEST_ROOT/var/nix
export NIX_CONF_DIR=$TEST_ROOT/etc export NIX_CONF_DIR=$TEST_ROOT/etc
export NIX_DAEMON_SOCKET_PATH=$TEST_ROOT/daemon-socket
unset NIX_USER_CONF_FILES unset NIX_USER_CONF_FILES
export _NIX_TEST_SHARED=$TEST_ROOT/shared export _NIX_TEST_SHARED=$TEST_ROOT/shared
if [[ -n $NIX_STORE ]]; then if [[ -n $NIX_STORE ]]; then
@@ -36,7 +35,6 @@ export xmllint="@xmllint@"
export SHELL="@bash@" export SHELL="@bash@"
export PAGER=cat export PAGER=cat
export HAVE_SODIUM="@HAVE_SODIUM@" export HAVE_SODIUM="@HAVE_SODIUM@"
export busybox="@sandbox_shell@"
export version=@PACKAGE_VERSION@ export version=@PACKAGE_VERSION@
export system=@system@ export system=@system@
@@ -77,7 +75,7 @@ startDaemon() {
rm -f $NIX_STATE_DIR/daemon-socket/socket rm -f $NIX_STATE_DIR/daemon-socket/socket
nix-daemon & nix-daemon &
for ((i = 0; i < 30; i++)); do for ((i = 0; i < 30; i++)); do
if [ -e $NIX_DAEMON_SOCKET_PATH ]; then break; fi if [ -e $NIX_STATE_DIR/daemon-socket/socket ]; then break; fi
sleep 1 sleep 1
done done
pidDaemon=$! pidDaemon=$!
+21 -30
View File
@@ -13,32 +13,24 @@ fake_free=$TEST_ROOT/fake-free
export _NIX_TEST_FREE_SPACE_FILE=$fake_free export _NIX_TEST_FREE_SPACE_FILE=$fake_free
echo 1100 > $fake_free echo 1100 > $fake_free
fifoLock=$TEST_ROOT/fifoLock
mkfifo "$fifoLock"
expr=$(cat <<EOF expr=$(cat <<EOF
with import ./config.nix; mkDerivation { with import ./config.nix; mkDerivation {
name = "gc-A"; name = "gc-A";
buildCommand = '' buildCommand = ''
set -x set -x
[[ \$(ls \$NIX_STORE/*-garbage? | wc -l) = 3 ]] [[ \$(ls \$NIX_STORE/*-garbage? | wc -l) = 3 ]]
mkdir \$out mkdir \$out
echo foo > \$out/bar echo foo > \$out/bar
echo 1...
# Pretend that we run out of space sleep 2
echo 100 > ${fake_free}.tmp1 echo 200 > ${fake_free}.tmp1
mv ${fake_free}.tmp1 $fake_free mv ${fake_free}.tmp1 $fake_free
echo 2...
# Wait for the GC to run sleep 2
for i in {1..20}; do echo 3...
echo ''\${i}... sleep 2
if [[ \$(ls \$NIX_STORE/*-garbage? | wc -l) = 1 ]]; then echo 4...
exit 0 [[ \$(ls \$NIX_STORE/*-garbage? | wc -l) = 1 ]]
fi
sleep 1
done
exit 1
''; '';
} }
EOF EOF
@@ -51,9 +43,15 @@ with import ./config.nix; mkDerivation {
set -x set -x
mkdir \$out mkdir \$out
echo foo > \$out/bar echo foo > \$out/bar
echo 1...
# Wait for the first build to finish sleep 2
cat "$fifoLock" echo 200 > ${fake_free}.tmp2
mv ${fake_free}.tmp2 $fake_free
echo 2...
sleep 2
echo 3...
sleep 2
echo 4...
''; '';
} }
EOF EOF
@@ -61,19 +59,12 @@ EOF
nix build -v -o $TEST_ROOT/result-A -L "($expr)" \ nix build -v -o $TEST_ROOT/result-A -L "($expr)" \
--min-free 1000 --max-free 2000 --min-free-check-interval 1 & --min-free 1000 --max-free 2000 --min-free-check-interval 1 &
pid1=$! pid=$!
nix build -v -o $TEST_ROOT/result-B -L "($expr2)" \ nix build -v -o $TEST_ROOT/result-B -L "($expr2)" \
--min-free 1000 --max-free 2000 --min-free-check-interval 1 & --min-free 1000 --max-free 2000 --min-free-check-interval 1
pid2=$!
# Once the first build is done, unblock the second one. wait "$pid"
# If the first build fails, we need to postpone the failure to still allow
# the second one to finish
wait "$pid1" || FIRSTBUILDSTATUS=$?
echo "unlock" > $fifoLock
( exit ${FIRSTBUILDSTATUS:-0} )
wait "$pid2"
[[ foo = $(cat $TEST_ROOT/result-A/bar) ]] [[ foo = $(cat $TEST_ROOT/result-A/bar) ]]
[[ foo = $(cat $TEST_ROOT/result-B/bar) ]] [[ foo = $(cat $TEST_ROOT/result-B/bar) ]]
+1 -4
View File
@@ -1,10 +1,7 @@
echo "Build started" > "$lockFifo"
mkdir $out mkdir $out
echo $(cat $input1/foo)$(cat $input2/bar) > $out/foobar echo $(cat $input1/foo)$(cat $input2/bar) > $out/foobar
# Wait for someone to write on the fifo sleep 10
cat "$lockFifo"
# $out should not have been GC'ed while we were sleeping, but just in # $out should not have been GC'ed while we were sleeping, but just in
# case... # case...
-3
View File
@@ -1,7 +1,5 @@
with import ./config.nix; with import ./config.nix;
{ lockFifo ? null }:
rec { rec {
input1 = mkDerivation { input1 = mkDerivation {
@@ -18,7 +16,6 @@ rec {
name = "gc-concurrent"; name = "gc-concurrent";
builder = ./gc-concurrent.builder.sh; builder = ./gc-concurrent.builder.sh;
inherit input1 input2; inherit input1 input2;
inherit lockFifo;
}; };
test2 = mkDerivation { test2 = mkDerivation {
+10 -11
View File
@@ -2,10 +2,7 @@ source common.sh
clearStore clearStore
lockFifo1=$TEST_ROOT/test1.fifo drvPath1=$(nix-instantiate gc-concurrent.nix -A test1)
mkfifo "$lockFifo1"
drvPath1=$(nix-instantiate gc-concurrent.nix -A test1 --argstr lockFifo "$lockFifo1")
outPath1=$(nix-store -q $drvPath1) outPath1=$(nix-store -q $drvPath1)
drvPath2=$(nix-instantiate gc-concurrent.nix -A test2) drvPath2=$(nix-instantiate gc-concurrent.nix -A test2)
@@ -25,16 +22,19 @@ ln -s $outPath3 "$NIX_STATE_DIR"/gcroots/foo2
nix-store -rvv "$drvPath1" & nix-store -rvv "$drvPath1" &
pid1=$! pid1=$!
# Wait for the build of $drvPath1 to start # Start build #2 in the background after 10 seconds.
cat $lockFifo1 (sleep 10 && nix-store -rvv "$drvPath2") &
pid2=$!
# Run the garbage collector while the build is running. # Run the garbage collector while the build is running.
sleep 6
nix-collect-garbage nix-collect-garbage
# Unlock the build of $drvPath1 # Wait for build #1/#2 to finish.
echo "" > $lockFifo1
echo waiting for pid $pid1 to finish... echo waiting for pid $pid1 to finish...
wait $pid1 wait $pid1
echo waiting for pid $pid2 to finish...
wait $pid2
# Check that the root of build #1 and its dependencies haven't been # Check that the root of build #1 and its dependencies haven't been
# deleted. The should not be deleted by the GC because they were # deleted. The should not be deleted by the GC because they were
@@ -42,9 +42,8 @@ wait $pid1
cat $outPath1/foobar cat $outPath1/foobar
cat $outPath1/input-2/bar cat $outPath1/input-2/bar
# Check that the build build $drvPath2 succeeds. # Check that build #2 has succeeded. It should succeed because the
# It should succeed because the derivation is a GC root. # derivation is a GC root.
nix-store -rvv "$drvPath2"
cat $outPath2/foobar cat $outPath2/foobar
rm -f "$NIX_STATE_DIR"/gcroots/foo* rm -f "$NIX_STATE_DIR"/gcroots/foo*
+2
View File
@@ -3,3 +3,5 @@ echo $(cat $input1/foo)$(cat $input2/bar)xyzzy > $out/foobar
# Check that the GC hasn't deleted the lock on our output. # Check that the GC hasn't deleted the lock on our output.
test -e "$out.lock" test -e "$out.lock"
sleep 6
-1
View File
@@ -18,7 +18,6 @@ build-users-group =
keep-derivations = false keep-derivations = false
sandbox = false sandbox = false
experimental-features = nix-command flakes experimental-features = nix-command flakes
gc-reserved-space = 0
include nix.conf.extra include nix.conf.extra
EOF EOF
+1 -1
View File
@@ -40,4 +40,4 @@ tests-environment = NIX_REMOTE= $(bash) -e
clean-files += $(d)/common.sh clean-files += $(d)/common.sh
test-deps += tests/common.sh tests/config.nix tests/plugins/libplugintest.$(SO_EXT) installcheck: $(d)/common.sh $(d)/config.nix $(d)/plugins/libplugintest.$(SO_EXT)
+3 -8
View File
@@ -16,11 +16,6 @@ nix-env --foo 2>&1 | grep "no operation"
nix-env -q --foo 2>&1 | grep "unknown flag" nix-env -q --foo 2>&1 | grep "unknown flag"
# Eval Errors. # Eval Errors.
eval_arg_res=$(nix-instantiate --eval -E 'let a = {} // a; in a.foo' 2>&1 || true) eval_res=$(nix-instantiate --eval -E 'let a = {} // a; in a.foo' 2>&1 || true)
echo $eval_arg_res | grep "at: (1:15) from string" echo $eval_res | grep "(string) (1:15)"
echo $eval_arg_res | grep "infinite recursion encountered" echo $eval_res | grep "infinite recursion encountered"
eval_stdin_res=$(echo 'let a = {} // a; in a.foo' | nix-instantiate --eval -E - 2>&1 || true)
echo $eval_stdin_res | grep "at: (1:15) from stdin"
echo $eval_stdin_res | grep "infinite recursion encountered"
-7
View File
@@ -55,10 +55,3 @@ chmod a+rx $TEST_ROOT/shell.shebang.rb
output=$($TEST_ROOT/shell.shebang.rb abc ruby) output=$($TEST_ROOT/shell.shebang.rb abc ruby)
[ "$output" = '-e load("'"$TEST_ROOT"'/shell.shebang.rb") -- abc ruby' ] [ "$output" = '-e load("'"$TEST_ROOT"'/shell.shebang.rb") -- abc ruby' ]
# Test 'nix develop'.
nix develop -f shell.nix shellDrv -c bash -c '[[ -n $stdenv ]]'
# Test 'nix print-dev-env'.
source <(nix print-dev-env -f shell.nix shellDrv)
[[ -n $stdenv ]]
-2
View File
@@ -2,8 +2,6 @@ source common.sh
clearStore clearStore
rm -f $TEST_ROOT/result
export REMOTE_STORE=$TEST_ROOT/remote_store export REMOTE_STORE=$TEST_ROOT/remote_store
# Build the dependencies and push them to the remote store # Build the dependencies and push them to the remote store
-2
View File
@@ -5,8 +5,6 @@ if [[ $(uname) != Linux ]]; then exit; fi
clearStore clearStore
rm -f $TEST_ROOT/result
export unreachable=$(nix add-to-store ./recursive.sh) export unreachable=$(nix add-to-store ./recursive.sh)
nix --experimental-features 'nix-command recursive-nix' build -o $TEST_ROOT/result -L '( nix --experimental-features 'nix-command recursive-nix' build -o $TEST_ROOT/result -L '(
-2
View File
@@ -2,8 +2,6 @@ source common.sh
clearStore clearStore
rm -f $TEST_ROOT/result
nix-build structured-attrs.nix -A all -o $TEST_ROOT/result nix-build structured-attrs.nix -A all -o $TEST_ROOT/result
[[ $(cat $TEST_ROOT/result/foo) = bar ]] [[ $(cat $TEST_ROOT/result/foo) = bar ]]