Compare commits

...
Author SHA1 Message Date
Robert Hensing 359d124bbf fixup: Re-enable import .drv code 2024-08-26 12:46:01 +02:00
Robert Hensing 68a5cfd30e tests: nix:derivation-internal.nix renders with a scheme now 2024-08-26 10:37:42 +02:00
Robert Hensing 1008847630 fix findFile assertion failure
A string is only allowed to create one path component; containing
no slashes.
2024-08-24 21:35:01 +02:00
Robert Hensing 52c0ef24c5 WIP fix baseNameOf (needs test maybe)
https://github.com/NixOS/nix/pull/10252#issuecomment-2275035429
2024-08-24 00:54:31 -04:00
Robert Hensing f13dc79904 Fix nix flake init eval for path value 2024-08-24 00:53:06 -04:00
Robert Hensing f3d2a16eff Fix evalState::rootFS paths' to_string()
This showPath is getting a little too ad hoc, but it works for now.
2024-08-24 00:51:46 -04:00
Robert Hensing 5def18e53b SourceAccessor: insert colon after prefix
This allows clever editors/IDEs to discern the path more easily
for Ctrl+Click navigate to functionality, e.g. when building
.?ref=HEAD
2024-08-24 00:51:10 -04:00
Robert Hensing ee6a764988 Make path values lazy
This fixes the double copy problem and improves performance
for expressions that don't force the whole source to be added to the
store.

Rules for fast expressions:

- Use path literals where possible
   - import ./foo.nix
- Use + operator with slash in string
   - src = fetchTree foo + "/src";
- Use source filtering, lib.fileset

- AVOID toString
- If possible, AVOID interpolations ("${./.}")
- If possible, move slashes into the interpolation to add less to the store
   - "${./src}/foo" -> "${./src/foo}"

toString may be improved later as part of lazy-trees, so these
recommendations are a snapshot. Path values are quite nice though.
2024-08-23 23:53:55 -04:00
Tom Bereknyei b79c918e09 fixup: remove FlakeCache 2024-08-23 23:53:52 -04:00
50b00b0194 fetchTree: Return a path instead of a store path
Co-authored-by: Eelco Dolstra <edolstra@gmail.com>
Co-authored-by: Robert Hensing <robert@roberthensing.nl>
2024-08-23 23:52:21 -04:00
21 changed files with 227 additions and 163 deletions

No files matched your search

+5 -6
View File
@@ -32,8 +32,7 @@ EvalSettings evalSettings {
// FIXME `parseFlakeRef` should take a `std::string_view`. // FIXME `parseFlakeRef` should take a `std::string_view`.
auto flakeRef = parseFlakeRef(fetchSettings, std::string { rest }, {}, true, false); auto flakeRef = parseFlakeRef(fetchSettings, std::string { rest }, {}, true, false);
debug("fetching flake search path element '%s''", rest); debug("fetching flake search path element '%s''", rest);
auto storePath = flakeRef.resolve(store).fetchTree(store).first; return flakeRef.resolve(store).lazyFetch(store).first;
return store->toRealPath(storePath);
}, },
}, },
}, },
@@ -174,15 +173,15 @@ SourcePath lookupFileArg(EvalState & state, std::string_view s, const Path * bas
state.store, state.store,
state.fetchSettings, state.fetchSettings,
EvalSettings::resolvePseudoUrl(s)); EvalSettings::resolvePseudoUrl(s));
auto storePath = fetchToStore(*state.store, SourcePath(accessor), FetchMode::Copy); state.registerAccessor(accessor);
return state.rootPath(CanonPath(state.store->toRealPath(storePath))); return SourcePath(accessor);
} }
else if (hasPrefix(s, "flake:")) { else if (hasPrefix(s, "flake:")) {
experimentalFeatureSettings.require(Xp::Flakes); experimentalFeatureSettings.require(Xp::Flakes);
auto flakeRef = parseFlakeRef(fetchSettings, std::string(s.substr(6)), {}, true, false); auto flakeRef = parseFlakeRef(fetchSettings, std::string(s.substr(6)), {}, true, false);
auto storePath = flakeRef.resolve(state.store).fetchTree(state.store).first; auto accessor = flakeRef.resolve(state.store).lazyFetch(state.store).first;
return state.rootPath(CanonPath(state.store->toRealPath(storePath))); return SourcePath(accessor);
} }
else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') { else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') {
+2 -1
View File
@@ -7,6 +7,7 @@
namespace nix { namespace nix {
class Store; class Store;
struct SourcePath;
struct EvalSettings : Config struct EvalSettings : Config
{ {
@@ -22,7 +23,7 @@ struct EvalSettings : Config
* @todo Return (`std::optional` of) `SourceAccssor` or something * @todo Return (`std::optional` of) `SourceAccssor` or something
* more structured instead of mere `std::string`? * more structured instead of mere `std::string`?
*/ */
using LookupPathHook = std::optional<std::string>(ref<Store> store, std::string_view); using LookupPathHook = std::optional<SourcePath>(ref<Store> store, std::string_view);
/** /**
* Map from "scheme" to a `LookupPathHook`. * Map from "scheme" to a `LookupPathHook`.
+50 -16
View File
@@ -315,6 +315,7 @@ EvalState::EvalState(
{ {
corepkgsFS->setPathDisplay("<nix", ">"); corepkgsFS->setPathDisplay("<nix", ">");
internalFS->setPathDisplay("«nix-internal»", ""); internalFS->setPathDisplay("«nix-internal»", "");
rootFS->setPathDisplay("/", "");
countCalls = getEnv("NIX_COUNT_CALLS").value_or("0") != "0"; countCalls = getEnv("NIX_COUNT_CALLS").value_or("0") != "0";
@@ -859,6 +860,11 @@ void Value::mkPath(const SourcePath & path)
mkPath(&*path.accessor, makeImmutableString(path.path.abs())); mkPath(&*path.accessor, makeImmutableString(path.path.abs()));
} }
void EvalState::registerAccessor(const ref<SourceAccessor> accessor)
{
sourceAccessors.push_back(accessor);
}
inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval) inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval)
{ {
@@ -1973,10 +1979,22 @@ void EvalState::concatLists(Value & v, size_t nrLists, Value * const * lists, co
v.mkList(list); v.mkList(list);
} }
// FIXME limit recursion
Value * resolveOutPath(EvalState & state, Value * v, const PosIdx pos)
{
state.forceValue(*v, pos);
if (v->type() != nAttrs)
return v;
auto found = v->attrs()->find(state.sOutPath);
if (found != v->attrs()->end())
return resolveOutPath(state, found->value, pos);
return v;
}
void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v) void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
{ {
NixStringContext context; NixStringContext context;
std::shared_ptr<SourceAccessor> accessor;
std::vector<BackedStringView> s; std::vector<BackedStringView> s;
size_t sSize = 0; size_t sSize = 0;
NixInt n{0}; NixInt n{0};
@@ -2010,8 +2028,9 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
Value * vTmpP = values.data(); Value * vTmpP = values.data();
for (auto & [i_pos, i] : *es) { for (auto & [i_pos, i] : *es) {
Value & vTmp = *vTmpP++; Value & vTmp0 = *vTmpP++;
i->eval(state, env, vTmp); i->eval(state, env, vTmp0);
Value & vTmp = *resolveOutPath(state, &vTmp0, i_pos);
/* If the first element is a path, then the result will also /* If the first element is a path, then the result will also
be a path, we don't copy anything (yet - that's done later, be a path, we don't copy anything (yet - that's done later,
@@ -2019,6 +2038,9 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
and none of the strings are allowed to have contexts. */ and none of the strings are allowed to have contexts. */
if (first) { if (first) {
firstType = vTmp.type(); firstType = vTmp.type();
if (firstType == nPath) {
accessor = vTmp.path().accessor;
}
} }
if (firstType == nInt) { if (firstType == nInt) {
@@ -2065,7 +2087,7 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
else if (firstType == nPath) { else if (firstType == nPath) {
if (!context.empty()) if (!context.empty())
state.error<EvalError>("a string that refers to a store path cannot be appended to a path").atPos(pos).withFrame(env, *this).debugThrow(); state.error<EvalError>("a string that refers to a store path cannot be appended to a path").atPos(pos).withFrame(env, *this).debugThrow();
v.mkPath(state.rootPath(CanonPath(canonPath(str())))); v.mkPath({ref(accessor), CanonPath(str())});
} else } else
v.mkStringMove(c_str(), context); v.mkStringMove(c_str(), context);
} }
@@ -2315,6 +2337,8 @@ BackedStringView EvalState::coerceToString(
v.payload.path.path v.payload.path.path
: copyToStore : copyToStore
? store->printStorePath(copyPathToStore(context, v.path())) ? store->printStorePath(copyPathToStore(context, v.path()))
: v.path().accessor->toStringReturnsStorePath()
? store->printStorePath(copyPathToStore(context, SourcePath(v.path().accessor, CanonPath::root))) + v.path().path.absOrEmpty()
: std::string(v.path().path.abs()); : std::string(v.path().path.abs());
} }
@@ -2427,10 +2451,14 @@ SourcePath EvalState::coerceToPath(const PosIdx pos, Value & v, NixStringContext
if (v.type() == nPath) if (v.type() == nPath)
return v.path(); return v.path();
/* Similarly, handle __toString where the result may be a path /* Similarly, handle outPath and __toString where the result may be a path
value. */ value. */
if (v.type() == nAttrs) { if (v.type() == nAttrs) {
auto i = v.attrs()->find(sToString); auto i = v.attrs()->find(sOutPath);
if (i != v.attrs()->end()) {
return coerceToPath(pos, *i->value, context, errorCtx);
}
i = v.attrs()->find(sToString);
if (i != v.attrs()->end()) { if (i != v.attrs()->end()) {
Value v1; Value v1;
callFunction(*i->value, v, v1, pos); callFunction(*i->value, v, v1, pos);
@@ -3053,12 +3081,14 @@ SourcePath EvalState::findFile(const LookupPath & lookupPath, const std::string_
if (!suffixOpt) continue; if (!suffixOpt) continue;
auto suffix = *suffixOpt; auto suffix = *suffixOpt;
auto rOpt = resolveLookupPathPath(i.path); auto rOpt = resolveLookupPathPath(
i.path,
true);
if (!rOpt) continue; if (!rOpt) continue;
auto r = *rOpt; auto r = *rOpt;
Path res = suffix == "" ? r : concatStrings(r, "/", suffix); auto res = r / CanonPath(suffix);
if (pathExists(res)) return rootPath(CanonPath(canonPath(res))); if (res.pathExists()) return res;
} }
if (hasPrefix(path, "nix/")) if (hasPrefix(path, "nix/"))
@@ -3073,26 +3103,30 @@ SourcePath EvalState::findFile(const LookupPath & lookupPath, const std::string_
} }
std::optional<std::string> EvalState::resolveLookupPathPath(const LookupPath::Path & value0, bool initAccessControl) std::optional<SourcePath> EvalState::resolveLookupPathPath(const LookupPath::Path & value0, bool initAccessControl)
{ {
auto & value = value0.s; auto & value = value0.s;
auto i = lookupPathResolved.find(value); auto i = lookupPathResolved.find(value);
if (i != lookupPathResolved.end()) return i->second; if (i != lookupPathResolved.end()) return i->second;
auto finish = [&](std::string res) { auto finish = [&](SourcePath res) {
debug("resolved search path element '%s' to '%s'", value, res); debug("resolved search path element '%s' to '%s'", value, res);
lookupPathResolved.emplace(value, res); lookupPathResolved.emplace(value, res);
return res; return res;
}; };
std::optional<SourcePath> res;
if (EvalSettings::isPseudoUrl(value)) { if (EvalSettings::isPseudoUrl(value)) {
try { try {
auto accessor = fetchers::downloadTarball( auto accessor = fetchers::downloadTarball(
store, store,
fetchSettings, fetchSettings,
EvalSettings::resolvePseudoUrl(value)); EvalSettings::resolvePseudoUrl(value));
// Traditional search path lookups use the absolute path space for
// historical consistency.
auto storePath = fetchToStore(*store, SourcePath(accessor), FetchMode::Copy); auto storePath = fetchToStore(*store, SourcePath(accessor), FetchMode::Copy);
return finish(store->toRealPath(storePath)); res.emplace(rootPath(CanonPath(store->toRealPath(storePath))));
} catch (Error & e) { } catch (Error & e) {
logWarning({ logWarning({
.msg = HintFmt("Nix search path entry '%1%' cannot be downloaded, ignoring", value) .msg = HintFmt("Nix search path entry '%1%' cannot be downloaded, ignoring", value)
@@ -3111,22 +3145,22 @@ std::optional<std::string> EvalState::resolveLookupPathPath(const LookupPath::Pa
} }
{ {
auto path = absPath(value); auto path = rootPath(value);
/* Allow access to paths in the search path. */ /* Allow access to paths in the search path. */
if (initAccessControl) { if (initAccessControl) {
allowPath(path); allowPath(path.path.abs());
if (store->isInStore(path)) { if (store->isInStore(path.path.abs())) {
try { try {
StorePathSet closure; StorePathSet closure;
store->computeFSClosure(store->toStorePath(path).first, closure); store->computeFSClosure(store->toStorePath(path.path.abs()).first, closure);
for (auto & p : closure) for (auto & p : closure)
allowPath(p); allowPath(p);
} catch (InvalidPath &) { } } catch (InvalidPath &) { }
} }
} }
if (pathExists(path)) if (path.pathExists())
return finish(std::move(path)); return finish(std::move(path));
else { else {
logWarning({ logWarning({
+7 -2
View File
@@ -249,6 +249,9 @@ public:
const SourcePath callFlakeInternal; const SourcePath callFlakeInternal;
/* A collection of InputAccessors, just to keep them alive. */
std::list<ref<SourceAccessor>> sourceAccessors;
/** /**
* Store used to materialise .drv files. * Store used to materialise .drv files.
*/ */
@@ -339,7 +342,7 @@ private:
LookupPath lookupPath; LookupPath lookupPath;
std::map<std::string, std::optional<std::string>> lookupPathResolved; std::map<std::string, std::optional<SourcePath>> lookupPathResolved;
/** /**
* Cache used by prim_match(). * Cache used by prim_match().
@@ -381,6 +384,8 @@ public:
*/ */
SourcePath rootPath(PathView path); SourcePath rootPath(PathView path);
void registerAccessor(ref<SourceAccessor> accessor);
/** /**
* Allow access to a path. * Allow access to a path.
*/ */
@@ -446,7 +451,7 @@ public:
* *
* If it is not found, return `std::nullopt` * If it is not found, return `std::nullopt`
*/ */
std::optional<std::string> resolveLookupPathPath( std::optional<SourcePath> resolveLookupPathPath(
const LookupPath::Path & elem, const LookupPath::Path & elem,
bool initAccessControl = false); bool initAccessControl = false);
+1 -1
View File
@@ -45,7 +45,7 @@ void ExprString::show(const SymbolTable & symbols, std::ostream & str) const
void ExprPath::show(const SymbolTable & symbols, std::ostream & str) const void ExprPath::show(const SymbolTable & symbols, std::ostream & str) const
{ {
str << s; str << path;
} }
void ExprVar::show(const SymbolTable & symbols, std::ostream & str) const void ExprVar::show(const SymbolTable & symbols, std::ostream & str) const
+7 -4
View File
@@ -131,12 +131,15 @@ struct ExprString : Expr
struct ExprPath : Expr struct ExprPath : Expr
{ {
ref<SourceAccessor> accessor; const SourcePath path;
std::string s;
Value v; Value v;
ExprPath(ref<SourceAccessor> accessor, std::string s) : accessor(accessor), s(std::move(s)) ExprPath(SourcePath && path)
: path(path)
{ {
v.mkPath(&*accessor, this->s.c_str()); v.mkPath(
&*path.accessor,
// TODO: GC_STRDUP
strdup(path.path.abs().c_str()));
} }
Value * maybeThunk(EvalState & state, Env & env) override; Value * maybeThunk(EvalState & state, Env & env) override;
COMMON_METHODS COMMON_METHODS
+23 -10
View File
@@ -136,6 +136,10 @@ static Expr * makeCall(PosIdx pos, Expr * fn, Expr * arg) {
std::vector<std::pair<nix::AttrName, nix::PosIdx>> * inheritAttrs; std::vector<std::pair<nix::AttrName, nix::PosIdx>> * inheritAttrs;
std::vector<std::pair<nix::PosIdx, nix::Expr *>> * string_parts; std::vector<std::pair<nix::PosIdx, nix::Expr *>> * string_parts;
std::vector<std::pair<nix::PosIdx, std::variant<nix::Expr *, nix::StringToken>>> * ind_string_parts; std::vector<std::pair<nix::PosIdx, std::variant<nix::Expr *, nix::StringToken>>> * ind_string_parts;
struct {
nix::Expr * e;
bool appendSlash;
} pathStart;
} }
%type <e> start expr expr_function expr_if expr_op %type <e> start expr expr_function expr_if expr_op
@@ -149,7 +153,8 @@ static Expr * makeCall(PosIdx pos, Expr * fn, Expr * arg) {
%type <inheritAttrs> attrs %type <inheritAttrs> attrs
%type <string_parts> string_parts_interpolated %type <string_parts> string_parts_interpolated
%type <ind_string_parts> ind_string_parts %type <ind_string_parts> ind_string_parts
%type <e> path_start string_parts string_attr %type <pathStart> path_start
%type <e> string_parts string_attr
%type <id> attr %type <id> attr
%token <id> ID %token <id> ID
%token <str> STR IND_STR %token <str> STR IND_STR
@@ -295,9 +300,11 @@ expr_simple
$$ = state->stripIndentation(CUR_POS, std::move(*$2)); $$ = state->stripIndentation(CUR_POS, std::move(*$2));
delete $2; delete $2;
} }
| path_start PATH_END | path_start PATH_END { $$ = $1.e; }
| path_start string_parts_interpolated PATH_END { | path_start string_parts_interpolated PATH_END {
$2->insert($2->begin(), {state->at(@1), $1}); if ($1.appendSlash)
$2->insert($2->begin(), {noPos, new ExprString("/")});
$2->insert($2->begin(), {state->at(@1), $1.e});
$$ = new ExprConcatStrings(CUR_POS, false, $2); $$ = new ExprConcatStrings(CUR_POS, false, $2);
} }
| SPATH { | SPATH {
@@ -350,11 +357,17 @@ string_parts_interpolated
path_start path_start
: PATH { : PATH {
Path path(absPath({$1.p, $1.l}, state->basePath.path.abs())); std::string_view path({$1.p, $1.l});
/* add back in the trailing '/' to the first segment */ $$ = {
if ($1.p[$1.l-1] == '/' && $1.l > 1) .e = new ExprPath(
path += "/"; /* Absolute paths are always interpreted relative to the
$$ = new ExprPath(ref<SourceAccessor>(state->rootFS), std::move(path)); root filesystem accessor, rather than the accessor of the
current Nix expression. */
hasPrefix(path, "/")
? SourcePath{state->rootFS, CanonPath(path)}
: SourcePath{state->basePath.accessor, CanonPath(path, state->basePath.path)}),
.appendSlash = hasSuffix(path, "/")
};
} }
| HPATH { | HPATH {
if (state->settings.pureEval) { if (state->settings.pureEval) {
@@ -363,8 +376,8 @@ path_start
std::string_view($1.p, $1.l) std::string_view($1.p, $1.l)
); );
} }
Path path(getHome() + std::string($1.p + 1, $1.l - 1)); CanonPath path(getHome() + std::string($1.p + 1, $1.l - 1));
$$ = new ExprPath(ref<SourceAccessor>(state->rootFS), std::move(path)); $$ = {.e = new ExprPath(SourcePath{state->rootFS, std::move(path)}), .appendSlash = true};
} }
; ;
+5 -5
View File
@@ -178,7 +178,6 @@ static void import(EvalState & state, const PosIdx pos, Value & vPath, Value * v
auto path = realisePath(state, pos, vPath, std::nullopt); auto path = realisePath(state, pos, vPath, std::nullopt);
auto path2 = path.path.abs(); auto path2 = path.path.abs();
// FIXME
auto isValidDerivationInStore = [&]() -> std::optional<StorePath> { auto isValidDerivationInStore = [&]() -> std::optional<StorePath> {
if (!state.store->isStorePath(path2)) if (!state.store->isStorePath(path2))
return std::nullopt; return std::nullopt;
@@ -216,9 +215,7 @@ static void import(EvalState & state, const PosIdx pos, Value & vPath, Value * v
state.forceFunction(**state.vImportedDrvToDerivation, pos, "while evaluating imported-drv-to-derivation.nix.gen.hh"); state.forceFunction(**state.vImportedDrvToDerivation, pos, "while evaluating imported-drv-to-derivation.nix.gen.hh");
v.mkApp(*state.vImportedDrvToDerivation, w); v.mkApp(*state.vImportedDrvToDerivation, w);
state.forceAttrs(v, pos, "while calling imported-drv-to-derivation.nix.gen.hh"); state.forceAttrs(v, pos, "while calling imported-drv-to-derivation.nix.gen.hh");
} } else {
else {
if (!vScope) if (!vScope)
state.evalFile(path, v); state.evalFile(path, v);
else { else {
@@ -1700,7 +1697,10 @@ static std::string_view legacyBaseNameOf(std::string_view path)
static void prim_baseNameOf(EvalState & state, const PosIdx pos, Value * * args, Value & v) static void prim_baseNameOf(EvalState & state, const PosIdx pos, Value * * args, Value & v)
{ {
NixStringContext context; NixStringContext context;
v.mkString(legacyBaseNameOf(*state.coerceToString(pos, *args[0], context, if (v.type() == nPath)
v.mkString(baseNameOf(v.path().path.abs()), context);
else
v.mkString(legacyBaseNameOf(*state.coerceToString(pos, *args[0], context,
"while evaluating the first argument passed to builtins.baseNameOf", "while evaluating the first argument passed to builtins.baseNameOf",
false, false)), context); false, false)), context);
} }
+49 -9
View File
@@ -19,21 +19,20 @@ namespace nix {
void emitTreeAttrs( void emitTreeAttrs(
EvalState & state, EvalState & state,
const StorePath & storePath,
const fetchers::Input & input, const fetchers::Input & input,
Value & v, Value & v,
std::function<void(Value &)> setOutPath,
bool emptyRevFallback, bool emptyRevFallback,
bool forceDirty) bool forceDirty)
{ {
auto attrs = state.buildBindings(100); auto attrs = state.buildBindings(100);
state.mkStorePathString(storePath, attrs.alloc(state.sOutPath)); setOutPath(attrs.alloc(state.sOutPath));
// FIXME: support arbitrary input attributes. // FIXME: support arbitrary input attributes.
auto narHash = input.getNarHash(); if (auto narHash = input.getNarHash())
assert(narHash); attrs.alloc("narHash").mkString(narHash->to_string(HashFormat::SRI, true));
attrs.alloc("narHash").mkString(narHash->to_string(HashFormat::SRI, true));
if (input.getType() == "git") if (input.getType() == "git")
attrs.alloc("submodules").mkBool( attrs.alloc("submodules").mkBool(
@@ -72,10 +71,28 @@ void emitTreeAttrs(
v.mkAttrs(attrs); v.mkAttrs(attrs);
} }
void emitTreeAttrs(
EvalState & state,
const SourcePath & storePath,
const fetchers::Input & input,
Value & v,
bool emptyRevFallback,
bool forceDirty)
{
emitTreeAttrs(state, input, v,
[&](Value & vOutPath) {
state.registerAccessor(storePath.accessor);
vOutPath.mkPath(storePath);
},
emptyRevFallback,
forceDirty);
}
struct FetchTreeParams { struct FetchTreeParams {
bool emptyRevFallback = false; bool emptyRevFallback = false;
bool allowNameArgument = false; bool allowNameArgument = false;
bool isFetchGit = false; bool isFetchGit = false;
bool returnPath = true; // whether to return a SourcePath instead of a StorePath
}; };
static void fetchTree( static void fetchTree(
@@ -112,7 +129,9 @@ static void fetchTree(
for (auto & attr : *args[0]->attrs()) { for (auto & attr : *args[0]->attrs()) {
if (attr.name == state.sType) continue; if (attr.name == state.sType) continue;
state.forceValue(*attr.value, attr.pos); state.forceValue(*attr.value, attr.pos);
if (attr.value->type() == nPath || attr.value->type() == nString) { if (attr.value->type() == nPath || attr.value->type() == nString) {
auto s = state.coerceToString(attr.pos, *attr.value, context, "", false, false).toOwned(); auto s = state.coerceToString(attr.pos, *attr.value, context, "", false, false).toOwned();
attrs.emplace(state.symbols[attr.name], attrs.emplace(state.symbols[attr.name],
@@ -193,11 +212,31 @@ static void fetchTree(
state.checkURI(input.toURLString()); state.checkURI(input.toURLString());
auto [storePath, input2] = input.fetchToStore(state.store); if (params.returnPath) {
// Clang16+: change to `auto [accessor, input2] =`
auto pair = input.getAccessor(state.store);
auto & accessor = pair.first;
auto & input2 = pair.second;
state.allowPath(storePath); emitTreeAttrs(state, input2, v,
[&](Value & vOutPath) {
state.registerAccessor(accessor);
vOutPath.mkPath(SourcePath { accessor, CanonPath::root });
},
params.emptyRevFallback, false);
} else {
auto pair = input.fetchToStore(state.store);
auto & storePath = pair.first;
auto & input2 = pair.second;
emitTreeAttrs(state, storePath, input2, v, params.emptyRevFallback, false); state.allowPath(storePath);
emitTreeAttrs(state, input2, v,
[&](Value & vOutPath) {
state.mkStorePathString(storePath, vOutPath);
},
params.emptyRevFallback, false);
}
} }
static void prim_fetchTree(EvalState & state, const PosIdx pos, Value * * args, Value & v) static void prim_fetchTree(EvalState & state, const PosIdx pos, Value * * args, Value & v)
@@ -614,7 +653,8 @@ static void prim_fetchGit(EvalState & state, const PosIdx pos, Value * * args, V
FetchTreeParams { FetchTreeParams {
.emptyRevFallback = true, .emptyRevFallback = true,
.allowNameArgument = true, .allowNameArgument = true,
.isFetchGit = true .isFetchGit = true,
.returnPath = false,
}); });
} }
+6 -2
View File
@@ -167,6 +167,8 @@ bool Input::contains(const Input & other) const
std::pair<StorePath, Input> Input::fetchToStore(ref<Store> store) const std::pair<StorePath, Input> Input::fetchToStore(ref<Store> store) const
{ {
// TODO: lazy-trees gets rid of this. Why?
#if 0
if (!scheme) if (!scheme)
throw Error("cannot fetch unsupported input '%s'", attrsToJSON(toAttrs())); throw Error("cannot fetch unsupported input '%s'", attrsToJSON(toAttrs()));
@@ -187,6 +189,7 @@ std::pair<StorePath, Input> Input::fetchToStore(ref<Store> store) const
debug("substitution of input '%s' failed: %s", to_string(), e.what()); debug("substitution of input '%s' failed: %s", to_string(), e.what());
} }
} }
#endif
auto [storePath, input] = [&]() -> std::pair<StorePath, Input> { auto [storePath, input] = [&]() -> std::pair<StorePath, Input> {
try { try {
@@ -194,8 +197,9 @@ std::pair<StorePath, Input> Input::fetchToStore(ref<Store> store) const
auto storePath = nix::fetchToStore(*store, SourcePath(accessor), FetchMode::Copy, final.getName()); auto storePath = nix::fetchToStore(*store, SourcePath(accessor), FetchMode::Copy, final.getName());
auto narHash = store->queryPathInfo(storePath)->narHash; // TODO: do we really want to throw this out?
final.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true)); // auto narHash = store->queryPathInfo(storePath)->narHash;
// final.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true));
scheme->checkLocks(*this, final); scheme->checkLocks(*this, final);
+35 -81
View File
@@ -19,62 +19,17 @@ using namespace flake;
namespace flake { namespace flake {
typedef std::pair<StorePath, FlakeRef> FetchedFlake; static FlakeRef maybeResolve(
typedef std::vector<std::pair<FlakeRef, FetchedFlake>> FlakeCache;
static std::optional<FetchedFlake> lookupInFlakeCache(
const FlakeCache & flakeCache,
const FlakeRef & flakeRef)
{
// FIXME: inefficient.
for (auto & i : flakeCache) {
if (flakeRef == i.first) {
debug("mapping '%s' to previously seen input '%s' -> '%s",
flakeRef, i.first, i.second.second);
return i.second;
}
}
return std::nullopt;
}
static std::tuple<StorePath, FlakeRef, FlakeRef> fetchOrSubstituteTree(
EvalState & state, EvalState & state,
const FlakeRef & originalRef, const FlakeRef & originalRef,
bool allowLookup, bool useRegistries)
FlakeCache & flakeCache)
{ {
auto fetched = lookupInFlakeCache(flakeCache, originalRef);
FlakeRef resolvedRef = originalRef;
if (!fetched) {
if (originalRef.input.isDirect()) { if (originalRef.input.isDirect()) {
fetched.emplace(originalRef.fetchTree(state.store)); if (!useRegistries)
} else {
if (allowLookup) {
resolvedRef = originalRef.resolve(state.store);
auto fetchedResolved = lookupInFlakeCache(flakeCache, originalRef);
if (!fetchedResolved) fetchedResolved.emplace(resolvedRef.fetchTree(state.store));
flakeCache.push_back({resolvedRef, *fetchedResolved});
fetched.emplace(*fetchedResolved);
}
else {
throw Error("'%s' is an indirect flake reference, but registry lookups are not allowed", originalRef); throw Error("'%s' is an indirect flake reference, but registry lookups are not allowed", originalRef);
} return originalRef.resolve(state.store);
} } else
flakeCache.push_back({originalRef, *fetched}); return originalRef;
}
auto [storePath, lockedRef] = *fetched;
debug("got tree '%s' from '%s'",
state.store->printStorePath(storePath), lockedRef);
state.allowPath(storePath);
assert(!originalRef.input.getNarHash() || storePath == originalRef.input.computeStorePath(*state.store));
return {std::move(storePath), resolvedRef, lockedRef};
} }
static void forceTrivialValue(EvalState & state, Value & value, const PosIdx pos) static void forceTrivialValue(EvalState & state, Value & value, const PosIdx pos)
@@ -316,24 +271,19 @@ static Flake getFlake(
EvalState & state, EvalState & state,
const FlakeRef & originalRef, const FlakeRef & originalRef,
bool allowLookup, bool allowLookup,
FlakeCache & flakeCache,
InputPath lockRootPath) InputPath lockRootPath)
{ {
auto [storePath, resolvedRef, lockedRef] = fetchOrSubstituteTree( auto resolvedRef = maybeResolve(state, originalRef, allowLookup);
state, originalRef, allowLookup, flakeCache); auto [accessor, lockedRef] = resolvedRef.lazyFetch(state.store);
return readFlake(state, originalRef, resolvedRef, lockedRef, state.rootPath(state.store->toRealPath(storePath)), lockRootPath); state.registerAccessor(accessor);
}
Flake getFlake(EvalState & state, const FlakeRef & originalRef, bool allowLookup, FlakeCache & flakeCache) return readFlake(state, originalRef, resolvedRef, lockedRef, SourcePath {accessor, CanonPath::root}, lockRootPath);
{
return getFlake(state, originalRef, allowLookup, flakeCache, {});
} }
Flake getFlake(EvalState & state, const FlakeRef & originalRef, bool allowLookup) Flake getFlake(EvalState & state, const FlakeRef & originalRef, bool allowLookup)
{ {
FlakeCache flakeCache; return getFlake(state, originalRef, allowLookup, {});
return getFlake(state, originalRef, allowLookup, flakeCache);
} }
static LockFile readLockFile( static LockFile readLockFile(
@@ -355,11 +305,9 @@ LockedFlake lockFlake(
{ {
experimentalFeatureSettings.require(Xp::Flakes); experimentalFeatureSettings.require(Xp::Flakes);
FlakeCache flakeCache;
auto useRegistries = lockFlags.useRegistries.value_or(settings.useRegistries); auto useRegistries = lockFlags.useRegistries.value_or(settings.useRegistries);
auto flake = getFlake(state, topRef, useRegistries, flakeCache); auto flake = getFlake(state, topRef, useRegistries);
if (lockFlags.applyNixConfig) { if (lockFlags.applyNixConfig) {
flake.config.apply(settings); flake.config.apply(settings);
@@ -553,7 +501,7 @@ LockedFlake lockFlake(
} }
if (mustRefetch) { if (mustRefetch) {
auto inputFlake = getFlake(state, oldLock->lockedRef, false, flakeCache, inputPath); auto inputFlake = getFlake(state, oldLock->lockedRef, false, inputPath);
nodePaths.emplace(childNode, inputFlake.path.parent()); nodePaths.emplace(childNode, inputFlake.path.parent());
computeLocks(inputFlake.inputs, childNode, inputPath, oldLock, lockRootPath, parentPath, false); computeLocks(inputFlake.inputs, childNode, inputPath, oldLock, lockRootPath, parentPath, false);
} else { } else {
@@ -586,7 +534,7 @@ LockedFlake lockFlake(
if (localRef.input.getType() == "path") if (localRef.input.getType() == "path")
localPath = absPath(*input.ref->input.getSourcePath(), parentPath); localPath = absPath(*input.ref->input.getSourcePath(), parentPath);
auto inputFlake = getFlake(state, localRef, useRegistries, flakeCache, inputPath); auto inputFlake = getFlake(state, localRef, useRegistries, inputPath);
auto childNode = make_ref<LockedNode>(inputFlake.lockedRef, ref); auto childNode = make_ref<LockedNode>(inputFlake.lockedRef, ref);
@@ -615,12 +563,17 @@ LockedFlake lockFlake(
} }
else { else {
auto [storePath, resolvedRef, lockedRef] = fetchOrSubstituteTree( auto [path, lockedRef] = [&]() -> std::tuple<SourcePath, FlakeRef>
state, *input.ref, useRegistries, flakeCache); {
auto resolvedRef = maybeResolve(state, *input.ref, useRegistries);
auto [accessor, lockedRef] = resolvedRef.lazyFetch(state.store);
state.registerAccessor(accessor);
return {SourcePath(accessor), lockedRef};
}();
auto childNode = make_ref<LockedNode>(lockedRef, ref, false); auto childNode = make_ref<LockedNode>(lockedRef, ref, false);
nodePaths.emplace(childNode, state.rootPath(state.store->toRealPath(storePath))); nodePaths.emplace(childNode, path);
node->inputs.insert_or_assign(id, childNode); node->inputs.insert_or_assign(id, childNode);
} }
@@ -723,8 +676,7 @@ LockedFlake lockFlake(
repo, so we should re-read it. FIXME: we could repo, so we should re-read it. FIXME: we could
also just clear the 'rev' field... */ also just clear the 'rev' field... */
auto prevLockedRef = flake.lockedRef; auto prevLockedRef = flake.lockedRef;
FlakeCache dummyCache; flake = getFlake(state, topRef, useRegistries);
flake = getFlake(state, topRef, useRegistries, dummyCache);
if (lockFlags.commitLockFile && if (lockFlags.commitLockFile &&
flake.lockedRef.input.getRev() && flake.lockedRef.input.getRev() &&
@@ -768,21 +720,23 @@ void callFlake(EvalState & state,
auto lockedNode = node.dynamic_pointer_cast<const LockedNode>(); auto lockedNode = node.dynamic_pointer_cast<const LockedNode>();
/*
// FIXME: This is a hack to support chroot stores. Remove this // FIXME: This is a hack to support chroot stores. Remove this
// once we can pass a sourcePath rather than a storePath to // once we can pass a sourcePath rather than a storePath to
// call-flake.nix. // call-flake.nix.
auto path = sourcePath.path.abs(); // auto path = sourcePath.path.abs();
if (auto store = state.store.dynamic_pointer_cast<LocalFSStore>()) { // if (auto store = state.store.dynamic_pointer_cast<LocalFSStore>()) {
auto realStoreDir = store->getRealStoreDir(); // auto realStoreDir = store->getRealStoreDir();
if (isInDir(path, realStoreDir)) // if (isInDir(path, realStoreDir))
path = store->storeDir + path.substr(realStoreDir.size()); // path = store->storeDir + path.substr(realStoreDir.size());
} // }
//
auto [storePath, subdir] = state.store->toStorePath(path); // auto [storePath, subdir] = state.store->toStorePath(path);
*/
emitTreeAttrs( emitTreeAttrs(
state, state,
storePath, SourcePath(sourcePath.accessor),
lockedNode ? lockedNode->lockedRef.input : lockedFlake.flake.lockedRef.input, lockedNode ? lockedNode->lockedRef.input : lockedFlake.flake.lockedRef.input,
vSourceInfo, vSourceInfo,
false, false,
@@ -793,7 +747,7 @@ void callFlake(EvalState & state,
override override
.alloc(state.symbols.create("dir")) .alloc(state.symbols.create("dir"))
.mkString(CanonPath(subdir).rel()); .mkString(sourcePath.path.rel());
overrides.alloc(state.symbols.create(key->second)).mkAttrs(override); overrides.alloc(state.symbols.create(key->second)).mkAttrs(override);
} }
+1 -1
View File
@@ -218,7 +218,7 @@ void callFlake(
void emitTreeAttrs( void emitTreeAttrs(
EvalState & state, EvalState & state,
const StorePath & storePath, const SourcePath & storePath,
const fetchers::Input & input, const fetchers::Input & input,
Value & v, Value & v,
bool emptyRevFallback = false, bool emptyRevFallback = false,
+3 -3
View File
@@ -287,10 +287,10 @@ FlakeRef FlakeRef::fromAttrs(
fetchers::maybeGetStrAttr(attrs, "dir").value_or("")); fetchers::maybeGetStrAttr(attrs, "dir").value_or(""));
} }
std::pair<StorePath, FlakeRef> FlakeRef::fetchTree(ref<Store> store) const std::pair<ref<SourceAccessor>, FlakeRef> FlakeRef::lazyFetch(ref<Store> store) const
{ {
auto [storePath, lockedInput] = input.fetchToStore(store); auto [accessor, lockedInput] = input.getAccessor(store);
return {std::move(storePath), FlakeRef(std::move(lockedInput), subdir)}; return {accessor, FlakeRef(std::move(lockedInput), subdir)};
} }
std::tuple<FlakeRef, std::string, ExtendedOutputsSpec> parseFlakeRefWithFragmentAndExtendedOutputsSpec( std::tuple<FlakeRef, std::string, ExtendedOutputsSpec> parseFlakeRefWithFragmentAndExtendedOutputsSpec(
+1 -1
View File
@@ -63,7 +63,7 @@ struct FlakeRef
const fetchers::Settings & fetchSettings, const fetchers::Settings & fetchSettings,
const fetchers::Attrs & attrs); const fetchers::Attrs & attrs);
std::pair<StorePath, FlakeRef> fetchTree(ref<Store> store) const; std::pair<ref<SourceAccessor>, FlakeRef> lazyFetch(ref<Store> store) const;
}; };
std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef); std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef);
+5
View File
@@ -195,4 +195,9 @@ ref<SourceAccessor> makeFSSourceAccessor(std::filesystem::path root)
{ {
return make_ref<PosixSourceAccessor>(std::move(root)); return make_ref<PosixSourceAccessor>(std::move(root));
} }
bool PosixSourceAccessor::toStringReturnsStorePath() const {
return false;
}
} }
+2
View File
@@ -57,6 +57,8 @@ struct PosixSourceAccessor : virtual SourceAccessor
*/ */
static SourcePath createAtRoot(const std::filesystem::path & path); static SourcePath createAtRoot(const std::filesystem::path & path);
virtual bool toStringReturnsStorePath() const override;
private: private:
/** /**
+7 -1
View File
@@ -64,7 +64,9 @@ void SourceAccessor::setPathDisplay(std::string displayPrefix, std::string displ
std::string SourceAccessor::showPath(const CanonPath & path) std::string SourceAccessor::showPath(const CanonPath & path)
{ {
return displayPrefix + path.abs() + displaySuffix; return displayPrefix
+ ((displayPrefix.empty() || displayPrefix.ends_with("/")) ? "" : ":") + path.rel()
+ displaySuffix;
} }
CanonPath SourceAccessor::resolveSymlinks( CanonPath SourceAccessor::resolveSymlinks(
@@ -105,4 +107,8 @@ CanonPath SourceAccessor::resolveSymlinks(
return res; return res;
} }
bool SourceAccessor::toStringReturnsStorePath() const {
return true;
}
} }
+9
View File
@@ -161,6 +161,15 @@ struct SourceAccessor : std::enable_shared_from_this<SourceAccessor>
virtual std::string showPath(const CanonPath & path); virtual std::string showPath(const CanonPath & path);
/**
* System paths: `toString /foo/bar = "/foo/bar"`
* Virtual paths: fetched to the store
*
* In both cases, the returned string functionally identifies the path,
* and can still be read.
*/
virtual bool toStringReturnsStorePath() const;
/** /**
* Resolve any symlinks in `path` according to the given * Resolve any symlinks in `path` according to the given
* resolution mode. * resolution mode.
+6 -16
View File
@@ -18,6 +18,7 @@
#include "markdown.hh" #include "markdown.hh"
#include "users.hh" #include "users.hh"
#include "terminal.hh" #include "terminal.hh"
#include "value/context.hh"
#include <filesystem> #include <filesystem>
#include <nlohmann/json.hpp> #include <nlohmann/json.hpp>
@@ -213,9 +214,6 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON
auto lockedFlake = lockFlake(); auto lockedFlake = lockFlake();
auto & flake = lockedFlake.flake; auto & flake = lockedFlake.flake;
// Currently, all flakes are in the Nix store via the rootFS accessor.
auto storePath = store->printStorePath(store->toStorePath(flake.path.path.abs()).first);
if (json) { if (json) {
nlohmann::json j; nlohmann::json j;
if (flake.description) if (flake.description)
@@ -236,7 +234,6 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON
j["revCount"] = *revCount; j["revCount"] = *revCount;
if (auto lastModified = flake.lockedRef.input.getLastModified()) if (auto lastModified = flake.lockedRef.input.getLastModified())
j["lastModified"] = *lastModified; j["lastModified"] = *lastModified;
j["path"] = storePath;
j["locks"] = lockedFlake.lockFile.toJSON().first; j["locks"] = lockedFlake.lockFile.toJSON().first;
if (auto fingerprint = lockedFlake.getFingerprint(store)) if (auto fingerprint = lockedFlake.getFingerprint(store))
j["fingerprint"] = fingerprint->to_string(HashFormat::Base16, false); j["fingerprint"] = fingerprint->to_string(HashFormat::Base16, false);
@@ -253,9 +250,6 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON
logger->cout( logger->cout(
ANSI_BOLD "Description:" ANSI_NORMAL " %s", ANSI_BOLD "Description:" ANSI_NORMAL " %s",
*flake.description); *flake.description);
logger->cout(
ANSI_BOLD "Path:" ANSI_NORMAL " %s",
storePath);
if (auto rev = flake.lockedRef.input.getRev()) if (auto rev = flake.lockedRef.input.getRev())
logger->cout( logger->cout(
ANSI_BOLD "Revision:" ANSI_NORMAL " %s", ANSI_BOLD "Revision:" ANSI_NORMAL " %s",
@@ -891,7 +885,9 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand
auto cursor = installable.getCursor(*evalState); auto cursor = installable.getCursor(*evalState);
auto templateDirAttr = cursor->getAttr("path"); auto templateDirAttr = cursor->getAttr("path");
auto templateDir = templateDirAttr->getString(); auto & v = templateDirAttr->forceValue();
NixStringContext ctx;
auto templateDir = evalState->coerceToString(noPos, v, ctx, "while casting the template value to a path", false, true).toOwned();
if (!store->isInStore(templateDir)) if (!store->isInStore(templateDir))
evalState->error<TypeError>( evalState->error<TypeError>(
@@ -1523,21 +1519,15 @@ struct CmdFlakePrefetch : FlakeCommand, MixJSON
{ {
auto originalRef = getFlakeRef(); auto originalRef = getFlakeRef();
auto resolvedRef = originalRef.resolve(store); auto resolvedRef = originalRef.resolve(store);
auto [storePath, lockedRef] = resolvedRef.fetchTree(store); auto [accessor, lockedRef] = resolvedRef.lazyFetch(store);
auto hash = store->queryPathInfo(storePath)->narHash;
if (json) { if (json) {
auto res = nlohmann::json::object(); auto res = nlohmann::json::object();
res["storePath"] = store->printStorePath(storePath);
res["hash"] = hash.to_string(HashFormat::SRI, true);
res["original"] = fetchers::attrsToJSON(resolvedRef.toAttrs()); res["original"] = fetchers::attrsToJSON(resolvedRef.toAttrs());
res["locked"] = fetchers::attrsToJSON(lockedRef.toAttrs()); res["locked"] = fetchers::attrsToJSON(lockedRef.toAttrs());
logger->cout(res.dump()); logger->cout(res.dump());
} else { } else {
notice("Downloaded '%s' to '%s' (hash '%s').", notice("Fetched '%s'.", lockedRef.to_string());
lockedRef.to_string(),
store->printStorePath(storePath),
hash.to_string(HashFormat::SRI, true));
} }
} }
}; };
-1
View File
@@ -192,7 +192,6 @@ nix flake metadata "$flake1Dir" | grepQuiet 'URL:.*flake1.*'
# Test 'nix flake metadata --json'. # Test 'nix flake metadata --json'.
json=$(nix flake metadata flake1 --json | jq .) json=$(nix flake metadata flake1 --json | jq .)
[[ $(echo "$json" | jq -r .description) = 'Bla bla' ]] [[ $(echo "$json" | jq -r .description) = 'Bla bla' ]]
[[ -d $(echo "$json" | jq -r .path) ]]
[[ $(echo "$json" | jq -r .lastModified) = $(git -C "$flake1Dir" log -n1 --format=%ct) ]] [[ $(echo "$json" | jq -r .lastModified) = $(git -C "$flake1Dir" log -n1 --format=%ct) ]]
hash1=$(echo "$json" | jq -r .revision) hash1=$(echo "$json" | jq -r .revision)
[[ -n $(echo "$json" | jq -r .fingerprint) ]] [[ -n $(echo "$json" | jq -r .fingerprint) ]]
@@ -1,20 +1,20 @@
error: error:
… while evaluating the attribute 'outPath' … while evaluating the attribute 'outPath'
at <nix/derivation-internal.nix>:<number>:<number>: at <nix:derivation-internal.nix>:<number>:<number>:
<number>| value = commonAttrs // { <number>| value = commonAttrs // {
<number>| outPath = builtins.getAttr outputName strict; <number>| outPath = builtins.getAttr outputName strict;
| ^ | ^
<number>| drvPath = strict.drvPath; <number>| drvPath = strict.drvPath;
… while calling the 'getAttr' builtin … while calling the 'getAttr' builtin
at <nix/derivation-internal.nix>:<number>:<number>: at <nix:derivation-internal.nix>:<number>:<number>:
<number>| value = commonAttrs // { <number>| value = commonAttrs // {
<number>| outPath = builtins.getAttr outputName strict; <number>| outPath = builtins.getAttr outputName strict;
| ^ | ^
<number>| drvPath = strict.drvPath; <number>| drvPath = strict.drvPath;
… while calling the 'derivationStrict' builtin … while calling the 'derivationStrict' builtin
at <nix/derivation-internal.nix>:<number>:<number>: at <nix:derivation-internal.nix>:<number>:<number>:
<number>| <number>|
<number>| strict = derivationStrict drvAttrs; <number>| strict = derivationStrict drvAttrs;
| ^ | ^