Compare commits

...
Author SHA1 Message Date
regnat ea792bcdc8 Make nix search lazier
Only 4 evals left \o/
2021-06-10 12:26:32 +02:00
regnat 71650c83c6 Make the error cleaner when getFields is misused 2021-06-10 12:26:32 +02:00
regnat d51deeac1c Remove the obsolete CmdSearch::visit function 2021-06-10 12:26:32 +02:00
regnat 1f8541258c Make getFields just return the name of the fields
Makes it much easier to deal with non-evaluating stuff
2021-06-10 12:26:32 +02:00
regnat 3a9753132e Make the derivation check more lazy 2021-06-10 08:09:25 +02:00
regnat 2324ee4891 Add a ugly hack to delay errors in getFields 2021-06-10 08:09:25 +02:00
regnat 28c1f8800b Don’t list known absent attributes in listChildren 2021-06-10 08:09:25 +02:00
regnat af775bdcf9 Make lazyGetAttrField return something more informative 2021-06-10 08:09:25 +02:00
regnat 8d2be51d19 Cache the evaluation errors
Doesn’t seem to make much of a difference on `nix search`, but we’ll
need it at some point
2021-06-10 08:09:25 +02:00
regnat 9102508f33 Use the cache in nix search
Not optimal atm, possibly because we don’t cache the evaluation failures
2021-06-10 08:09:25 +02:00
regnat a6aaf81103 [TMP]: Disable the tests
To allow the benchmarks to run even when something’s broken
2021-06-10 08:09:25 +02:00
regnat c7a232e200 Be even lazier for the build evaluation
Now we can not evaluate anything \o/
2021-06-08 16:05:51 +02:00
regnat 30d14b772f Evaluate more lazily in findAlongAttrPath 2021-06-04 16:53:59 +02:00
regnat fbaee9b8fb Add a “cached thunk” value type 2021-06-04 16:45:22 +02:00
regnat 753730c410 Optimize the nix build caching 2021-06-04 15:21:03 +02:00
regnat c116e6e837 Another attempt at caching nix build
A different tradeof set probably. I guess we could unify both
2021-06-04 15:16:28 +02:00
regnat 3e261410bc First attempt at caching the evaluation in nix build 2021-06-04 11:39:29 +02:00
regnat ffec547ebc Add some benchmarks for the actual caching 2021-06-03 16:41:39 +02:00
regnat 512afd8b7a Use the cache in the outer shell
Make sure that we don’t discard it before entering the evaluator proper
2021-06-03 15:29:43 +02:00
regnat 69505c84e1 Commit the cache at the end of the evaluation
Otherwise the db is never properly filled
2021-06-03 15:29:12 +02:00
regnat af5c323e93 Make the root symbol more telling in the DB
I spent a few minutes trying to understand why I had a field with an
empty symbol, until I realise that it’s because that was the symbol for
the root element.
Now that shouldn’t happen anymore
2021-06-03 15:28:04 +02:00
regnat 9053ac0693 use the setEvalCache function at the flake root
Probably doesn’t change much, but much cleaner (and robust)
2021-06-03 15:27:33 +02:00
regnat b39ab10749 Properly fill the cache in case of a miss/forward 2021-06-03 15:27:03 +02:00
regnat 8787218c7c (maybe) Actually cache things 2021-06-03 13:06:01 +02:00
regnat 89951cf7fb Extract a Value method to set the eval cache 2021-06-03 12:26:22 +02:00
regnat 45a28ed36f Extract a Value method to get the eval cache 2021-06-03 12:26:02 +02:00
regnat 6ec852e7f0 Simplify the forcing of nested records 2021-06-03 12:11:51 +02:00
regnat 7c718646cb Add some stats for the evaluation caching 2021-06-03 12:11:51 +02:00
regnat 8d95e1f299 Set the cache when opening a flake 2021-06-03 12:11:51 +02:00
regnat 6396416dfa Query the eval cache
(It’s still always empty)
2021-06-03 12:11:51 +02:00
regnat 44f390ed48 Re-split the function, but without the exception 2021-06-03 12:11:51 +02:00
regnat 4ca1a0b864 Try removing the exception 2021-06-03 12:11:51 +02:00
regnat 2021b1d8d1 [TMP] Try inlining the getAttrField function
See what’s making things slow
2021-06-03 12:11:51 +02:00
regnat b012852cb2 Split a standalone function for accessing an attribute set field 2021-06-03 12:11:51 +02:00
regnat 074e0678bd Add an (always empty) eval cache to the attr sets 2021-06-03 12:11:51 +02:00
regnat 891390d76f [TMP]: Add some benchmarking tools
Not intended to be merged (at least definitely not as it is)
2021-06-03 12:11:51 +02:00
20 changed files with 1300 additions and 171 deletions

No files matched your search

Executable
+53
View File
@@ -0,0 +1,53 @@
#!/bin/sh
set -euo pipefail
set -x
callNix () {
nix \
--experimental-features "nix-command flakes" \
--store /tmp/nix \
"$@"
}
callBuild () {
callNix \
eval --impure --file "$THINGTOBENCH" drvPath \
"$@"
}
getCompletions () {
NIX_GET_COMPLETIONS=6 callNix build "github:NixOS/nixpkgs?rev=ad0d20345219790533ebe06571f82ed6b034db31#firef" "$@"
}
runSearch () {
callNix search "github:NixOS/nixpkgs?rev=ad0d20345219790533ebe06571f82ed6b034db31" firefox "$@"
}
noCache () {
"$@" --option eval-cache false
}
coldCache () {
if [[ -e ~/.cache/nix/eval-cache-v2 || -e ~/.cache/nix/eval-cache-v3 ]]; then
echo "Error: The cache should be clean"
exit 1
fi
"$@"
}
run_all () {
mkdir -p "$out"
NIX_SHOW_STATS=1 NIX_SHOW_STATS_PATH=$out/eval-stats.json bash $0 noCache callBuild
hyperfine \
--warmup 2 \
--export-csv "$out/result.csv" \
--export-json "$out/result.json" \
--export-markdown "$out/result.md" \
--style basic \
--prepare '' "bash $0 noCache callBuild" \
--prepare 'rm -rf ~/.cache/nix/' "bash $0 coldCache callBuild" \
--prepare '' "bash $0 callBuild"
}
"$@"
+9 -2
View File
@@ -229,7 +229,7 @@
echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products
''; '';
doInstallCheck = true; doInstallCheck = false;
installCheckFlags = "sysconfdir=$(out)/etc"; installCheckFlags = "sysconfdir=$(out)/etc";
separateDebugInfo = true; separateDebugInfo = true;
@@ -294,6 +294,13 @@
''; '';
}; };
nix-benchmarks = prev.runCommandNoCC "nix-benchmarks" {
buildInputs = [ final.nix prev.hyperfine ];
THINGTOBENCH = ./thingToBench.nix;
NIX_PATH="nixpkgs=${prev.path}";
} ''
bash ${./benchmark.sh} run_all
'';
}; };
hydraJobs = { hydraJobs = {
@@ -485,7 +492,7 @@
}); });
packages = forAllSystems (system: { packages = forAllSystems (system: {
inherit (nixpkgsFor.${system}) nix; inherit (nixpkgsFor.${system}) nix nix-benchmarks;
} // nixpkgs.lib.optionalAttrs (builtins.elem system linux64BitSystems) { } // nixpkgs.lib.optionalAttrs (builtins.elem system linux64BitSystems) {
nix-static = let nix-static = let
nixpkgs = nixpkgsFor.${system}.pkgsStatic; nixpkgs = nixpkgsFor.${system}.pkgsStatic;
+48 -36
View File
@@ -302,6 +302,14 @@ Installable::getCursors(EvalState & state)
return {{evalCache->getRoot(), ""}}; return {{evalCache->getRoot(), ""}};
} }
std::pair<Value*, Pos> Installable::toValue(EvalState & state)
{
auto values = toValues(state);
if (values.empty())
throw Error("cannot find flake attribute '%s'", what());
return {std::get<0>(values[0]), std::get<1>(values[0])};
}
std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string> std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>
Installable::getCursor(EvalState & state) Installable::getCursor(EvalState & state)
{ {
@@ -378,11 +386,11 @@ struct InstallableAttrPath : InstallableValue
std::string what() override { return attrPath; } std::string what() override { return attrPath; }
std::pair<Value *, Pos> toValue(EvalState & state) override std::vector<std::tuple<Value *, Pos, std::string>> toValues(EvalState & state) override
{ {
auto [vRes, pos] = findAlongAttrPath(state, attrPath, *cmd.getAutoArgs(state), **v); auto [vRes, pos] = findAlongAttrPath(state, attrPath, *cmd.getAutoArgs(state), **v);
state.forceValue(*vRes); state.forceValue(*vRes);
return {vRes, pos}; return {{vRes, pos, attrPath}};
} }
virtual std::vector<InstallableValue::DerivationInfo> toDerivations() override; virtual std::vector<InstallableValue::DerivationInfo> toDerivations() override;
@@ -428,12 +436,10 @@ Value * InstallableFlake::getFlakeOutputs(EvalState & state, const flake::Locked
callFlake(state, lockedFlake, *vFlake); callFlake(state, lockedFlake, *vFlake);
auto aOutputs = vFlake->attrs->get(state.symbols.create("outputs")); auto vRes = state.allocValue();
assert(aOutputs); auto gotField = state.lazyGetAttrField(*vFlake, {state.symbols.create("outputs")}, noPos, *vRes);
assert(gotField != EvalState::LazyValueType::Missing);
state.forceValue(*aOutputs->value); return vRes;
return aOutputs->value;
} }
ref<eval_cache::EvalCache> openEvalCache( ref<eval_cache::EvalCache> openEvalCache(
@@ -500,25 +506,34 @@ std::tuple<std::string, FlakeRef, InstallableValue::DerivationInfo> InstallableF
auto root = cache->getRoot(); auto root = cache->getRoot();
for (auto & attrPath : getActualAttrPaths()) { for (auto & attrPath : getActualAttrPaths()) {
auto attr = root->findAlongAttrPath( auto emptyArgs = state->allocBindings(0);
parseAttrPath(*state, attrPath), try {
true auto [drvValue, pos] = findAlongAttrPath(
); *state,
attrPath,
*emptyArgs,
*getFlakeOutputs(*state, *lockedFlake)
);
Value * v = state->allocValue();
if (!state->getAttrField(*drvValue, {state->sDrvPath}, pos, *v))
break;
auto drvPath = state->forceString(*v);
if (!state->getAttrField(*drvValue, {state->sOutPath}, pos, *v))
break;
auto outPath = state->forceString(*v);
if (!state->getAttrField(*drvValue, {state->sOutputName}, pos, *v))
break;
auto outputName = state->forceString(*v);
if (!attr) continue; auto drvInfo = DerivationInfo{
state->store->parseStorePath(drvPath),
state->store->maybeParseStorePath(outPath),
outputName
};
if (!attr->isDerivation()) return {attrPath, lockedFlake->flake.lockedRef, std::move(drvInfo)};
throw Error("flake output attribute '%s' is not a derivation", attrPath); } catch (AttrPathNotFound & e) {
}
auto drvPath = attr->forceDerivation();
auto drvInfo = DerivationInfo{
std::move(drvPath),
state->store->maybeParseStorePath(attr->getAttr(state->sOutPath)->getString()),
attr->getAttr(state->sOutputName)->getString()
};
return {attrPath, lockedFlake->flake.lockedRef, std::move(drvInfo)};
} }
throw Error("flake '%s' does not provide attribute %s", throw Error("flake '%s' does not provide attribute %s",
@@ -532,25 +547,22 @@ std::vector<InstallableValue::DerivationInfo> InstallableFlake::toDerivations()
return res; return res;
} }
std::pair<Value *, Pos> InstallableFlake::toValue(EvalState & state) std::vector<std::tuple<Value *, Pos, std::string>>
InstallableFlake::toValues(EvalState & state)
{ {
auto lockedFlake = getLockedFlake(); auto vOutputs = getFlakeOutputs(state, *getLockedFlake());
auto vOutputs = getFlakeOutputs(state, *lockedFlake);
auto emptyArgs = state.allocBindings(0); auto emptyArgs = state.allocBindings(0);
std::vector<std::tuple<Value *, Pos, std::string>> res;
for (auto & attrPath : getActualAttrPaths()) { for (auto & attrPath : getActualAttrPaths()) {
try { try {
auto [v, pos] = findAlongAttrPath(state, attrPath, *emptyArgs, *vOutputs); auto [v, pos] = findAlongAttrPath(state, attrPath, *emptyArgs, *vOutputs);
state.forceValue(*v); res.push_back({v, pos, attrPath});
return {v, pos}; } catch (Error &) {}
} catch (AttrPathNotFound & e) {
}
} }
throw Error("flake '%s' does not provide attribute %s", return res;
flakeRef, showAttrPaths(getActualAttrPaths()));
} }
std::vector<std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>> std::vector<std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>>
+3 -2
View File
@@ -41,10 +41,11 @@ struct Installable
UnresolvedApp toApp(EvalState & state); UnresolvedApp toApp(EvalState & state);
virtual std::pair<Value *, Pos> toValue(EvalState & state) virtual std::vector<std::tuple<Value *, Pos, std::string>> toValues(EvalState & state)
{ {
throw Error("argument '%s' cannot be evaluated", what()); throw Error("argument '%s' cannot be evaluated", what());
} }
std::pair<Value *, Pos> toValue(EvalState & state);
/* Return a value only if this installable is a store path or a /* Return a value only if this installable is a store path or a
symlink to it. */ symlink to it. */
@@ -109,7 +110,7 @@ struct InstallableFlake : InstallableValue
std::vector<DerivationInfo> toDerivations() override; std::vector<DerivationInfo> toDerivations() override;
std::pair<Value *, Pos> toValue(EvalState & state) override; std::vector<std::tuple<Value *, Pos, std::string>> toValues(EvalState & state) override;
std::vector<std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>> std::vector<std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>>
getCursors(EvalState & state) override; getCursors(EvalState & state) override;
+7 -10
View File
@@ -58,26 +58,23 @@ std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attr
Value * vNew = state.allocValue(); Value * vNew = state.allocValue();
state.autoCallFunction(autoArgs, *v, *vNew); state.autoCallFunction(autoArgs, *v, *vNew);
v = vNew; v = vNew;
state.forceValue(*v);
/* It should evaluate to either a set or an expression, /* It should evaluate to either a set or an expression,
according to what is specified in the attrPath. */ according to what is specified in the attrPath. */
if (!attrIndex) { if (!attrIndex) {
if (v->type() != nAttrs)
throw TypeError(
"the expression selected by the selection path '%1%' should be a set but is %2%",
attrPath,
showType(*v));
if (attr.empty()) if (attr.empty())
throw Error("empty attribute name in selection path '%1%'", attrPath); throw Error("empty attribute name in selection path '%1%'", attrPath);
Bindings::iterator a = v->attrs->find(state.symbols.create(attr)); auto v2 = state.allocValue();
if (a == v->attrs->end()) auto gotField = state.lazyGetAttrField(*v, {state.symbols.create(attr)}, pos, *v2) != EvalState::LazyValueType::Missing;
if (!gotField)
throw AttrPathNotFound("attribute '%1%' in selection path '%2%' not found", attr, attrPath); throw AttrPathNotFound("attribute '%1%' in selection path '%2%' not found", attr, attrPath);
v = &*a->value; v = v2;
pos = *a->pos; /* Bindings::iterator a = v->attrs->find(state.symbols.create(attr)); */
/* v = &*a->value; */
/* pos = *a->pos; */
} }
else { else {
+2
View File
@@ -2,6 +2,7 @@
#include "nixexpr.hh" #include "nixexpr.hh"
#include "symbol-table.hh" #include "symbol-table.hh"
#include "value-cache.hh"
#include <algorithm> #include <algorithm>
#include <optional> #include <optional>
@@ -36,6 +37,7 @@ class Bindings
public: public:
typedef uint32_t size_t; typedef uint32_t size_t;
Pos *pos; Pos *pos;
ValueCache eval_cache;
private: private:
size_t size_, capacity_; size_t size_, capacity_;
+21
View File
@@ -0,0 +1,21 @@
#include "context.hh"
namespace nix {
/* Decode a context string ‘!<name>!<path>’ into a pair <path,
name>. */
std::pair<string, string> decodeContext(std::string_view s)
{
if (s.at(0) == '!') {
size_t index = s.find("!", 1);
return {std::string(s.substr(index + 1)), std::string(s.substr(1, index - 1))};
} else
return {s.at(0) == '/' ? std::string(s) : std::string(s.substr(1)), ""};
}
std::string encodeContext(std::string_view name, std::string_view path)
{
return "!" + std::string(name) + "!" + std::string(path);
}
}
+11
View File
@@ -0,0 +1,11 @@
#include "util.hh"
namespace nix {
/* Decode a context string ‘!<name>!<path>’ into a pair <path,
name>. */
std::pair<string, string> decodeContext(std::string_view s);
std::string encodeContext(std::string_view name, std::string_view path);
}
+6
View File
@@ -46,6 +46,12 @@ void EvalState::forceValue(Value & v, const Pos & pos)
} }
else if (v.isApp()) else if (v.isApp())
callFunction(*v.app.left, *v.app.right, v, noPos); callFunction(*v.app.left, *v.app.right, v, noPos);
else if (v.isCachedThunk()) {
auto evalCache = v.getEvalCache();
v.mkThunk(v.cachedThunk.thunk->env, v.cachedThunk.thunk->expr);
forceValue(v, pos);
v.setEvalCache(evalCache);
}
else if (v.isBlackhole()) else if (v.isBlackhole())
throwEvalError(pos, "infinite recursion encountered"); throwEvalError(pos, "infinite recursion encountered");
} }
+368 -60
View File
@@ -1,4 +1,5 @@
#include "eval.hh" #include "eval.hh"
#include "value-cache.hh"
#include "hash.hh" #include "hash.hh"
#include "util.hh" #include "util.hh"
#include "store-api.hh" #include "store-api.hh"
@@ -8,6 +9,7 @@
#include "filetransfer.hh" #include "filetransfer.hh"
#include "json.hh" #include "json.hh"
#include "function-trace.hh" #include "function-trace.hh"
#include "value-cache.hh"
#include <algorithm> #include <algorithm>
#include <chrono> #include <chrono>
@@ -122,6 +124,7 @@ void printValue(std::ostream & str, std::set<const Value *> & active, const Valu
str << "]"; str << "]";
break; break;
case tThunk: case tThunk:
case tCachedThunk:
case tApp: case tApp:
str << "<CODE>"; str << "<CODE>";
break; break;
@@ -193,7 +196,7 @@ string showType(const Value & v)
case tPrimOpApp: case tPrimOpApp:
return fmt("the partially applied built-in function '%s'", string(getPrimOp(v)->primOp->name)); return fmt("the partially applied built-in function '%s'", string(getPrimOp(v)->primOp->name));
case tExternal: return v.external->showType(); case tExternal: return v.external->showType();
case tThunk: return "a thunk"; case tThunk: case tCachedThunk: return "a thunk";
case tApp: return "a function application"; case tApp: return "a function application";
case tBlackhole: return "a black hole"; case tBlackhole: return "a black hole";
default: default:
@@ -216,7 +219,7 @@ bool Value::isTrivial() const
return return
internalType != tApp internalType != tApp
&& internalType != tPrimOpApp && internalType != tPrimOpApp
&& (internalType != tThunk && ((internalType != tThunk && internalType != tCachedThunk)
|| (dynamic_cast<ExprAttrs *>(thunk.expr) || (dynamic_cast<ExprAttrs *>(thunk.expr)
&& ((ExprAttrs *) thunk.expr)->dynamicAttrs.empty()) && ((ExprAttrs *) thunk.expr)->dynamicAttrs.empty())
|| dynamic_cast<ExprLambda *>(thunk.expr) || dynamic_cast<ExprLambda *>(thunk.expr)
@@ -442,6 +445,9 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
EvalState::~EvalState() EvalState::~EvalState()
{ {
for (auto [_, cache] : evalCache) {
cache->commit();
}
} }
@@ -1107,68 +1113,235 @@ void ExprVar::eval(EvalState & state, Env & env, Value & v)
v = *v2; v = *v2;
} }
unsigned long nrLookups = 0;
static string showAttrPath(EvalState & state, Env & env, const AttrPath & attrPath) std::pair<ValueCache::CacheResult, ValueCache> ValueCache::getValue(EvalState & state, const std::vector<Symbol> & selector, Value & dest)
{ {
std::ostringstream out; if (!rawCache)
bool first = true; return { {NoCacheKey}, ValueCache(nullptr) };
for (auto & i : attrPath) { auto resultingCursor = rawCache->findAlongAttrPath(selector);
if (!first) out << '.'; else first = false; if (!resultingCursor)
try { return { {CacheMiss}, ValueCache(nullptr) };
out << getName(i, state, env);
} catch (Error & e) { auto cachedValue = resultingCursor->getCachedValue();
assert(!i.symbol.set()); auto cacheResult = std::visit(
out << "\"${" << *i.expr << "}\""; overloaded{
} [&](tree_cache::attributeSet_t) { return ValueCache::CacheResult{ Forward }; },
} [&](tree_cache::unknown_t) { return ValueCache::CacheResult{ UnCacheable }; },
return out.str(); [&](tree_cache::thunk_t) { return ValueCache::CacheResult{ CacheMiss }; },
[&](tree_cache::failed_t x) -> ValueCache::CacheResult {throw EvalError(x.error); },
[&](tree_cache::missing_t x) {
return ValueCache::CacheResult{
.returnCode = CacheHit,
.lastQueriedSymbolIfMissing = x.attrName
};
},
[&](tree_cache::string_t s) {
PathSet context;
for (auto& [pathName, outputName] : s.second) {
// If the cached value depends on some non-existent
// path, we need to discard it and force the evaluation
// to bring back the context in the store
if (!state.store->isValidPath(
state.store->parseStorePath(pathName)))
return ValueCache::CacheResult{UnCacheable};
context.insert("!" + outputName + "!" + pathName);
}
mkString(dest, s.first, context);
return ValueCache::CacheResult{CacheHit};
},
[&](tree_cache::wrapped_basetype<bool> b) {
dest.mkBool(b.value);
return ValueCache::CacheResult{CacheHit};
},
[&](tree_cache::wrapped_basetype<int64_t> i) {
dest.mkInt(i.value);
return ValueCache::CacheResult{CacheHit};
},
[&](tree_cache::wrapped_basetype<double> d) {
dest.mkFloat(d.value);
return ValueCache::CacheResult{CacheHit};
},
},
cachedValue);
return { cacheResult, ValueCache(resultingCursor) };
} }
void EvalState::updateCacheStats(ValueCache::CacheResult cacheResult)
{
switch (cacheResult.returnCode) {
case ValueCache::CacheHit:
nrCacheHits++;
break;
case ValueCache::CacheMiss:
nrCacheMisses++;
break;
case ValueCache::UnCacheable:
nrUncacheable++;
break;
case ValueCache::NoCacheKey:
nrUncached++;
break;
case ValueCache::Forward:
nrCacheHits++;
break;
};
}
unsigned long nrLookups = 0; EvalState::LazyValueType EvalState::lazyGetAttrField(Value & attrs, const std::vector<Symbol> & selector, const Pos & pos, Value & dest)
{
auto eval_cache = attrs.getEvalCache();
if (eval_cache.isEmpty()) {
forceValue(attrs, pos);
eval_cache = attrs.getEvalCache();
}
auto [ cacheResult, resultingCursor ] = eval_cache.getValue(*this, selector, dest);
updateCacheStats(cacheResult);
auto delayValue = [&](ValueCache & cursor) {
auto recordAsVar = new ExprCastedVar(&attrs);
auto accessExpr = new ExprSelect(pos, recordAsVar, selector);
auto thunk = (Thunk*)allocBytes(sizeof(Thunk));
thunk->expr = accessExpr;
thunk->env = &baseEnv;
dest.mkCachedThunk(
thunk,
new ValueCache(cursor)
);
};
switch (cacheResult.returnCode) {
case ValueCache::CacheHit:
if (cacheResult.lastQueriedSymbolIfMissing)
return LazyValueType::Missing;
return LazyValueType::PlainValue;
case ValueCache::UnCacheable:
delayValue(resultingCursor);
return LazyValueType::DelayedUncacheable;
case ValueCache::Forward:
delayValue(resultingCursor);
return LazyValueType::DelayedAttr;
default:
if(getAttrField(attrs, selector, pos, dest))
return LazyValueType::PlainValue;
else
return LazyValueType::Missing;
}
}
bool EvalState::getAttrField(Value & attrs, const std::vector<Symbol> & selector, const Pos & pos, Value & dest)
{
Pos * pos2 = 0;
auto eval_cache = attrs.getEvalCache();
if (eval_cache.isEmpty()) {
forceValue(attrs, pos);
eval_cache = attrs.getEvalCache();
}
auto [ cacheResult, resultingCursor ] = eval_cache.getValue(*this, selector, dest);
updateCacheStats(cacheResult);
switch (cacheResult.returnCode) {
case ValueCache::CacheHit:
if (cacheResult.lastQueriedSymbolIfMissing)
return false;
return true;
case ValueCache::CacheMiss:
resultingCursor = eval_cache;
break;
case ValueCache::Forward: // FIXME: Handle properly
case ValueCache::NoCacheKey:
case ValueCache::UnCacheable:
;
}
forceValue(attrs, pos);
Value * vAttrs = &attrs;
try {
for (auto & name : selector) {
nrLookups++;
Bindings::iterator j;
if (vAttrs->type() != nAttrs ||
(j = vAttrs->attrs->find(name)) == vAttrs->attrs->end()) {
return false;
}
vAttrs = j->value;
pos2 = j->pos;
try {
forceValue(*vAttrs, pos2 != NULL ? *pos2 : pos );
} catch (EvalError & e) {
resultingCursor.addFailedChild(name, e);
throw;
}
if (cacheResult.returnCode == ValueCache::CacheMiss) {
resultingCursor = resultingCursor.addChild(name, *vAttrs);
vAttrs->setEvalCache(resultingCursor);
}
if (countCalls && pos2) attrSelects[*pos2]++;
}
} catch (Error & e) {
if (pos2 && pos2->file != sDerivationNix) {
vector<string> strSelector;
addErrorTrace(e, *pos2, "while evaluating the attribute '%1%'", concatStringsSep(".", selector));
}
throw;
}
if (cacheResult.returnCode == ValueCache::Forward)
vAttrs->setEvalCache(resultingCursor);
dest = *vAttrs;
return true;
}
void EvalState::getAttrFieldThrow(Value & attrs, const std::vector<Symbol> & selector, const Pos & pos, Value & dest)
{
if (!getAttrField(attrs, selector, pos, dest))
throw Error("Missing attribute path '%s'", "ImTooLazyToImplementThisRightNow");
}
std::vector<Symbol> EvalState::getFields(Value & attrs, const Pos & pos)
{
auto eval_cache = attrs.getEvalCache();
if (eval_cache.isEmpty()) {
forceValue(attrs, pos);
eval_cache = attrs.getEvalCache();
}
if (auto attrNames = eval_cache.listChildren(symbols)) {
return *attrNames;
}
forceAttrs(attrs);
std::vector<Symbol> res;
for (auto & attr : *attrs.attrs)
res.push_back(attr.name);
return res;
}
void ExprSelect::eval(EvalState & state, Env & env, Value & v) void ExprSelect::eval(EvalState & state, Env & env, Value & v)
{ {
Value vTmp; Value vTmp;
Pos * pos2 = 0;
Value * vAttrs = &vTmp;
e->eval(state, env, vTmp); e->eval(state, env, vTmp);
try { std::vector<Symbol> selector;
for (auto & i : attrPath)
selector.push_back(getName(i, state, env));
for (auto & i : attrPath) { bool gotField = state.getAttrField(vTmp, selector, pos, v);
nrLookups++; if (!gotField) {
Bindings::iterator j; if (def) {
Symbol name = getName(i, state, env); def->eval(state, env, v);
if (def) { return;
state.forceValue(*vAttrs, pos); } else {
if (vAttrs->type() != nAttrs || throwEvalError(pos, "Missing field");
(j = vAttrs->attrs->find(name)) == vAttrs->attrs->end())
{
def->eval(state, env, v);
return;
}
} else {
state.forceAttrs(*vAttrs, pos);
if ((j = vAttrs->attrs->find(name)) == vAttrs->attrs->end())
throwEvalError(pos, "attribute '%1%' missing", name);
}
vAttrs = j->value;
pos2 = j->pos;
if (state.countCalls && pos2) state.attrSelects[*pos2]++;
} }
state.forceValue(*vAttrs, ( pos2 != NULL ? *pos2 : this->pos ) );
} catch (Error & e) {
if (pos2 && pos2->file != state.sDerivationNix)
addErrorTrace(e, *pos2, "while evaluating the attribute '%1%'",
showAttrPath(state, env, attrPath));
throw;
} }
v = *vAttrs;
} }
@@ -1351,9 +1524,38 @@ void EvalState::incrFunctionCall(ExprLambda * fun)
functionCalls[fun]++; functionCalls[fun]++;
} }
std::optional<tree_cache::AttrValue> ValueCache::getRawValue()
{
if (!rawCache)
return std::nullopt;
return rawCache->getCachedValue();
}
std::optional<std::vector<Symbol>> ValueCache::listChildren(SymbolTable& symbols)
{
auto ret = std::vector<Symbol>();
if (rawCache) {
auto cachedValue = rawCache->getCachedValue();
if (std::get_if<tree_cache::attributeSet_t>(&cachedValue)) {
for (auto & fieldStr : rawCache->getChildren())
ret.push_back(symbols.create(fieldStr));
}
return ret;
}
return std::nullopt;
}
void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res) void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res)
{ {
if (auto evalCache = fun.getEvalCache(); !evalCache.isEmpty()) {
if (auto cacheValue = evalCache.getRawValue()) {
if (std::holds_alternative<tree_cache::attributeSet_t>(*cacheValue)) {
res = fun;
return;
}
}
}
forceValue(fun); forceValue(fun);
if (fun.type() == nAttrs) { if (fun.type() == nAttrs) {
@@ -1699,18 +1901,6 @@ string EvalState::forceString(Value & v, const Pos & pos)
} }
/* Decode a context string ‘!<name>!<path>’ into a pair <path,
name>. */
std::pair<string, string> decodeContext(std::string_view s)
{
if (s.at(0) == '!') {
size_t index = s.find("!", 1);
return {std::string(s.substr(index + 1)), std::string(s.substr(1, index - 1))};
} else
return {s.at(0) == '/' ? std::string(s) : std::string(s.substr(1)), ""};
}
void copyContext(const Value & v, PathSet & context) void copyContext(const Value & v, PathSet & context)
{ {
if (v.string.context) if (v.string.context)
@@ -1729,6 +1919,102 @@ std::vector<std::pair<Path, std::string>> Value::getContext()
return res; return res;
} }
ValueCache & Value::getEvalCache()
{
if (internalType == tAttrs) {
return attrs->eval_cache;
} else if (internalType == tCachedThunk) {
return *cachedThunk.cache;
} else {
return ValueCache::empty;
}
}
ValueCache ValueCache::empty = ValueCache(nullptr);
tree_cache::AttrValue cachedValueFor(Value& v)
{
tree_cache::AttrValue valueToCache;
switch (v.type()) {
case nThunk:
valueToCache = tree_cache::thunk_t{};
break;
case nNull:
case nList:
case nFunction:
case nExternal:
valueToCache = tree_cache::unknown_t{};
break;
case nBool:
valueToCache = tree_cache::wrapped_basetype<bool>{v.boolean};
break;
case nString:
valueToCache = tree_cache::string_t{
v.string.s,
v.getContext()
};
break;
case nPath:
valueToCache = tree_cache::string_t{
v.path,
{}
};
break;
case nAttrs:
valueToCache = tree_cache::attributeSet_t{};
break;
case nInt:
valueToCache = tree_cache::wrapped_basetype<int64_t>{v.integer};
break;
case nFloat:
valueToCache = tree_cache::wrapped_basetype<double>{v.fpoint};
break;
};
return valueToCache;
}
ValueCache ValueCache::addChild(const Symbol& name, Value& value)
{
if (!rawCache)
return ValueCache::empty;
auto cachedValue = cachedValueFor(value);
auto ret = ValueCache(rawCache->addChild(name, cachedValue));
if (value.type() == nAttrs)
ret.addAttrSetChilds(*value.attrs);
return ret;
}
ValueCache ValueCache::addFailedChild(const Symbol& name, const Error & error)
{
if (!rawCache) return ValueCache::empty;
return ValueCache(rawCache->addChild(name, tree_cache::failed_t{ .error = error.msg() }));
}
void ValueCache::addAttrSetChilds(Bindings & children)
{
if (!rawCache) return;
for (auto & attr : children) {
// We could in theory directly store the value, but that would cause
// an infinite recursion in case of a cyclic attrset.
// So in a first time, just store a thunk
if (attr.value->type() == nAttrs)
rawCache->addChild(attr.name, tree_cache::thunk_t{});
else
addChild(attr.name, *attr.value);
}
}
void Value::setEvalCache(ValueCache & newCache)
{
if (internalType == tAttrs) {
attrs->eval_cache = newCache;
} else if (internalType == tCachedThunk) {
cachedThunk.cache = &newCache;
}
}
string EvalState::forceString(Value & v, PathSet & context, const Pos & pos) string EvalState::forceString(Value & v, PathSet & context, const Pos & pos)
{ {
@@ -2011,6 +2297,13 @@ void EvalState::printStats()
topObj.attr("nrLookups", nrLookups); topObj.attr("nrLookups", nrLookups);
topObj.attr("nrPrimOpCalls", nrPrimOpCalls); topObj.attr("nrPrimOpCalls", nrPrimOpCalls);
topObj.attr("nrFunctionCalls", nrFunctionCalls); topObj.attr("nrFunctionCalls", nrFunctionCalls);
{
auto cache = topObj.object("evalCache");
cache.attr("nrCacheMisses", nrCacheMisses);
cache.attr("nrCacheHits", nrCacheHits);
cache.attr("nrUncached", nrUncached);
cache.attr("nrUncacheable", nrUncacheable);
}
#if HAVE_BOEHMGC #if HAVE_BOEHMGC
{ {
auto gc = topObj.object("gc"); auto gc = topObj.object("gc");
@@ -2111,6 +2404,21 @@ Strings EvalSettings::getDefaultNixPath()
return res; return res;
} }
std::shared_ptr<tree_cache::Cache> EvalState::openTreeCache(Hash cacheKey)
{
if (auto iter = evalCache.find(cacheKey); iter != evalCache.end())
return iter->second;
if (!(evalSettings.useEvalCache && evalSettings.pureEval))
return nullptr;
auto thisCache = tree_cache::Cache::tryCreate(
cacheKey,
symbols
);
evalCache.insert({cacheKey, thisCache});
return thisCache;
}
EvalSettings evalSettings; EvalSettings evalSettings;
static GlobalConfig::Register rEvalSettings(&evalSettings); static GlobalConfig::Register rEvalSettings(&evalSettings);
+31 -4
View File
@@ -1,6 +1,7 @@
#pragma once #pragma once
#include "attr-set.hh" #include "attr-set.hh"
#include "context.hh"
#include "value.hh" #include "value.hh"
#include "nixexpr.hh" #include "nixexpr.hh"
#include "symbol-table.hh" #include "symbol-table.hh"
@@ -115,6 +116,7 @@ private:
typedef std::map<Path, Value> FileEvalCache; typedef std::map<Path, Value> FileEvalCache;
#endif #endif
FileEvalCache fileEvalCache; FileEvalCache fileEvalCache;
std::map<Hash, std::shared_ptr<tree_cache::Cache>> evalCache;
SearchPath searchPath; SearchPath searchPath;
@@ -131,6 +133,8 @@ public:
EvalState(const Strings & _searchPath, ref<Store> store); EvalState(const Strings & _searchPath, ref<Store> store);
~EvalState(); ~EvalState();
std::shared_ptr<tree_cache::Cache> openTreeCache(Hash);
void addToSearchPath(const string & s); void addToSearchPath(const string & s);
SearchPath getSearchPath() { return searchPath; } SearchPath getSearchPath() { return searchPath; }
@@ -310,6 +314,8 @@ public:
void realiseContext(const PathSet & context); void realiseContext(const PathSet & context);
void updateCacheStats(ValueCache::CacheResult);
private: private:
unsigned long nrEnvs = 0; unsigned long nrEnvs = 0;
@@ -323,6 +329,10 @@ private:
unsigned long nrListConcats = 0; unsigned long nrListConcats = 0;
unsigned long nrPrimOpCalls = 0; unsigned long nrPrimOpCalls = 0;
unsigned long nrFunctionCalls = 0; unsigned long nrFunctionCalls = 0;
unsigned long nrCacheHits = 0;
unsigned long nrCacheMisses = 0;
unsigned long nrUncacheable = 0;
unsigned long nrUncached = 0;
bool countCalls; bool countCalls;
@@ -342,6 +352,27 @@ private:
friend struct ExprSelect; friend struct ExprSelect;
friend void prim_getAttr(EvalState & state, const Pos & pos, Value * * args, Value & v); friend void prim_getAttr(EvalState & state, const Pos & pos, Value * * args, Value & v);
friend void prim_match(EvalState & state, const Pos & pos, Value * * args, Value & v); friend void prim_match(EvalState & state, const Pos & pos, Value * * args, Value & v);
public:
bool getAttrField(Value & attrs, const std::vector<Symbol> & selector, const Pos & pos, Value & dest);
enum struct LazyValueType {
PlainValue,
DelayedUncacheable,
DelayedAttr,
Missing,
};
// Similar to `getAttrField`, but if the cache says that the result is an
// attribute set, just return a thunk to it rather than forcing it.
LazyValueType lazyGetAttrField(Value & attrs, const std::vector<Symbol> & selector, const Pos & pos, Value & dest);
// Similar to `getAttrField`, but throws an `Error` if the field can’t be
// found
void getAttrFieldThrow(Value & attrs, const std::vector<Symbol> & selector, const Pos & pos, Value & dest);
std::vector<Symbol> getFields(Value & attrs, const Pos & pos);
}; };
@@ -349,10 +380,6 @@ private:
string showType(ValueType type); string showType(ValueType type);
string showType(const Value & v); string showType(const Value & v);
/* Decode a context string ‘!<name>!<path>’ into a pair <path,
name>. */
std::pair<string, string> decodeContext(std::string_view s);
/* If `path' refers to a directory, then append "/default.nix". */ /* If `path' refers to a directory, then append "/default.nix". */
Path resolveExprPath(Path path); Path resolveExprPath(Path path);
+32 -18
View File
@@ -596,30 +596,44 @@ void callFlake(EvalState & state,
const LockedFlake & lockedFlake, const LockedFlake & lockedFlake,
Value & vRes) Value & vRes)
{ {
auto vLocks = state.allocValue(); auto lockExpr = new ExprString(
state.symbols.create(lockedFlake.lockFile.to_string())
);
auto subdirExpr = new ExprString(
state.symbols.create(lockedFlake.flake.lockedRef.subdir)
);
auto vRootSrc = state.allocValue(); auto vRootSrc = state.allocValue();
auto vRootSubdir = state.allocValue();
auto vTmp1 = state.allocValue();
auto vTmp2 = state.allocValue();
mkString(*vLocks, lockedFlake.lockFile.to_string());
emitTreeAttrs(state, *lockedFlake.flake.sourceInfo, lockedFlake.flake.lockedRef.input, *vRootSrc); emitTreeAttrs(state, *lockedFlake.flake.sourceInfo, lockedFlake.flake.lockedRef.input, *vRootSrc);
mkString(*vRootSubdir, lockedFlake.flake.lockedRef.subdir); static Expr * callFlakeExpr = nullptr;
if (!callFlakeExpr) {
static RootValue vCallFlake = nullptr; callFlakeExpr = state.parseExprFromString(
if (!vCallFlake) {
vCallFlake = allocRootValue(state.allocValue());
state.eval(state.parseExprFromString(
#include "call-flake.nix.gen.hh" #include "call-flake.nix.gen.hh"
, "/"), **vCallFlake); , "/");
} }
state.callFunction(**vCallFlake, *vLocks, *vTmp1, noPos); auto resExpr = new ExprApp(
state.callFunction(*vTmp1, *vRootSrc, *vTmp2, noPos); new ExprApp(
state.callFunction(*vTmp2, *vRootSubdir, vRes, noPos); new ExprApp(
callFlakeExpr,
lockExpr
),
new ExprCastedVar(vRootSrc)
),
subdirExpr
);
auto thunk = (Thunk*)allocBytes(sizeof(Thunk));
thunk->expr = resExpr;
thunk->env = &state.baseEnv;
auto fingerprint = lockedFlake.getFingerprint();
auto treeCache = state.openTreeCache(fingerprint);
auto cacheRoot = treeCache ? treeCache->getRoot() : nullptr;
auto evalCache = new ValueCache(cacheRoot);
vRes.mkCachedThunk(
thunk,
evalCache
);
} }
static void prim_getFlake(EvalState & state, const Pos & pos, Value * * args, Value & v) static void prim_getFlake(EvalState & state, const Pos & pos, Value * * args, Value & v)
+12
View File
@@ -440,6 +440,18 @@ string ExprLambda::showNamePos() const
return (format("%1% at %2%") % (name.set() ? "'" + (string) name + "'" : "anonymous function") % pos).str(); return (format("%1% at %2%") % (name.set() ? "'" + (string) name + "'" : "anonymous function") % pos).str();
} }
void ExprCastedVar::show(std::ostream & str) const {
std::set<const Value*> active;
printValue(str, active, *v);
}
void ExprCastedVar::bindVars(const StaticEnv & env) {}
void ExprCastedVar::eval(EvalState & state, Env & env, Value & v) {
v = std::move(*this->v);
}
Value * ExprCastedVar::maybeThunk(EvalState & state, Env & env) {
return v;
}
/* Symbol table. */ /* Symbol table. */
+14 -1
View File
@@ -165,7 +165,12 @@ struct ExprSelect : Expr
Expr * e, * def; Expr * e, * def;
AttrPath attrPath; AttrPath attrPath;
ExprSelect(const Pos & pos, Expr * e, const AttrPath & attrPath, Expr * def) : pos(pos), e(e), def(def), attrPath(attrPath) { }; ExprSelect(const Pos & pos, Expr * e, const AttrPath & attrPath, Expr * def) : pos(pos), e(e), def(def), attrPath(attrPath) { };
ExprSelect(const Pos & pos, Expr * e, const Symbol & name) : pos(pos), e(e), def(0) { attrPath.push_back(AttrName(name)); }; ExprSelect(const Pos & pos, Expr * e, const Symbol & name)
: ExprSelect(pos, e, std::vector{name}) {};
ExprSelect(const Pos & pos, Expr * e, const std::vector<Symbol> & symbolicAttrPath) : pos(pos), e(e), def(0) {
for (auto & name : symbolicAttrPath)
attrPath.push_back(AttrName(name));
};
COMMON_METHODS COMMON_METHODS
}; };
@@ -334,6 +339,14 @@ struct ExprPos : Expr
COMMON_METHODS COMMON_METHODS
}; };
struct ExprCastedVar : Expr
{
Value * v;
ExprCastedVar(Value * v) : v(v) {};
Value * maybeThunk(EvalState & state, Env & env);
COMMON_METHODS
};
/* Static environments are used to map variable names onto (level, /* Static environments are used to map variable names onto (level,
displacement) pairs used to obtain the value of the variable at displacement) pairs used to obtain the value of the variable at
+392
View File
@@ -0,0 +1,392 @@
#include "tree-cache.hh"
#include "sqlite.hh"
#include "store-api.hh"
#include "context.hh"
namespace nix::tree_cache {
static const char * schema = R"sql(
create table if not exists Attributes (
id integer primary key autoincrement not null,
parent integer not null,
name text,
type integer not null,
value text,
context text,
unique (parent, name)
);
create index if not exists IndexByParent on Attributes(parent, name);
)sql";
struct AttrDb
{
std::atomic_bool failed{false};
struct State
{
SQLite db;
SQLiteStmt insertAttribute;
SQLiteStmt updateAttribute;
SQLiteStmt insertAttributeWithContext;
SQLiteStmt queryAttribute;
SQLiteStmt queryAttributes;
std::unique_ptr<SQLiteTxn> txn;
};
std::unique_ptr<Sync<State>> _state;
AttrDb(const Hash & fingerprint)
: _state(std::make_unique<Sync<State>>())
{
auto state(_state->lock());
Path cacheDir = getCacheDir() + "/nix/eval-cache-v3";
createDirs(cacheDir);
Path dbPath = cacheDir + "/" + fingerprint.to_string(Base16, false) + ".sqlite";
state->db = SQLite(dbPath);
state->db.isCache();
state->db.exec(schema);
state->insertAttribute.create(state->db,
"insert into Attributes(parent, name, type, value) values (?, ?, ?, ?)");
state->updateAttribute.create(state->db,
"update Attributes set type = ?, value = ?, context = ? where id = ?");
state->insertAttributeWithContext.create(state->db,
"insert into Attributes(parent, name, type, value, context) values (?, ?, ?, ?, ?)");
state->queryAttribute.create(state->db,
"select id, type, value, context from Attributes where parent = ? and name = ?");
state->queryAttributes.create(state->db,
"select name, type from Attributes where parent = ?");
state->txn = std::make_unique<SQLiteTxn>(state->db);
}
~AttrDb()
{
try {
auto state(_state->lock());
if (!failed)
state->txn->commit();
state->txn.reset();
} catch (...) {
ignoreException();
}
}
template<typename F>
AttrId doSQLite(F && fun)
{
if (failed) return 0;
try {
return fun();
} catch (SQLiteError &) {
ignoreException();
failed = true;
return 0;
}
}
/**
* Store a leaf of the tree in the db
*/
AttrId addEntry(
const AttrKey & key,
const AttrValue & value)
{
return doSQLite([&]()
{
auto state(_state->lock());
auto rawValue = RawValue::fromVariant(value);
state->insertAttributeWithContext.use()
(key.first)
(key.second)
(rawValue.type)
(rawValue.value.value_or(""), rawValue.value.has_value())
(rawValue.serializeContext())
.exec();
AttrId rowId = state->db.getLastInsertedRowId();
assert(rowId);
return rowId;
});
}
std::optional<AttrId> getId(const AttrKey& key)
{
auto state(_state->lock());
auto queryAttribute(state->queryAttribute.use()(key.first)(key.second));
if (!queryAttribute.next()) return std::nullopt;
return (AttrType) queryAttribute.getInt(0);
}
AttrId setOrUpdate(const AttrKey& key, const AttrValue& value)
{
debug("cache: miss for the attribute %s", key.second);
if (auto existingId = getId(key)) {
setValue(*existingId, value);
return *existingId;
}
return addEntry(key, value);
}
void setValue(const AttrId & id, const AttrValue & value)
{
auto state(_state->lock());
auto rawValue = RawValue::fromVariant(value);
state->updateAttribute.use()
(rawValue.type)
(rawValue.value.value_or(""), rawValue.value.has_value())
(rawValue.serializeContext())
(id)
.exec();
}
std::optional<std::pair<AttrId, AttrValue>> getValue(AttrKey key)
{
auto state(_state->lock());
auto queryAttribute(state->queryAttribute.use()(key.first)(key.second));
if (!queryAttribute.next()) return {};
auto rowId = (AttrType) queryAttribute.getInt(0);
auto type = (AttrType) queryAttribute.getInt(1);
switch (type) {
case AttrType::Attrs: {
return {{rowId, attributeSet_t()}};
}
case AttrType::String: {
std::vector<std::pair<Path, std::string>> context;
if (!queryAttribute.isNull(3))
for (auto & s : tokenizeString<std::vector<std::string>>(queryAttribute.getStr(3), ";"))
context.push_back(decodeContext(s));
return {{rowId, string_t{queryAttribute.getStr(2), context}}};
}
case AttrType::Bool:
return {{rowId, wrapped_basetype<bool>{queryAttribute.getInt(2) != 0}}};
case AttrType::Int:
return {{rowId, wrapped_basetype<int64_t>{queryAttribute.getInt(2)}}};
case AttrType::Double:
return {{rowId, wrapped_basetype<double>{(double)queryAttribute.getInt(2)}}};
case AttrType::Unknown:
return {{rowId, unknown_t{}}};
case AttrType::Thunk:
return {{rowId, thunk_t{}}};
case AttrType::Missing:
return {{rowId, missing_t{key.second}}};
case AttrType::Failed:
return {{rowId, failed_t{queryAttribute.getStr(2)}}};
default:
throw Error("unexpected type in evaluation cache");
}
}
std::vector<std::string> getChildren(AttrId parentId)
{
std::vector<std::string> res;
auto state(_state->lock());
auto queryAttributes(state->queryAttributes.use()(parentId));
while (queryAttributes.next()) {
if (queryAttributes.getInt(1) != AttrType::Missing)
res.push_back(queryAttributes.getStr(0));
}
return res;
}
};
Cache::Cache(const Hash & useCache,
SymbolTable & symbols)
: db(std::make_shared<AttrDb>(useCache))
, symbols(symbols)
, rootSymbol(symbols.create("%root%"))
{
}
std::shared_ptr<Cache> Cache::tryCreate(const Hash & useCache, SymbolTable & symbols)
{
try {
return std::make_shared<Cache>(useCache, symbols);
} catch (SQLiteError &) {
ignoreException();
return nullptr;
}
}
void Cache::commit()
{
if (db) {
debug("Saving the cache");
auto state(db->_state->lock());
if (state->txn->active) {
state->txn->commit();
state->txn.reset();
state->txn = std::make_unique<SQLiteTxn>(state->db);
}
}
}
Cursor::Ref Cache::getRoot()
{
return new Cursor(ref(shared_from_this()), std::nullopt, thunk_t{});
}
Cursor::Cursor(
ref<Cache> root,
const Parent & parent,
const AttrValue& value
)
: root(root)
, parentId(parent ? std::optional{parent->first.cachedValue.first} : std::nullopt)
, label(parent ? parent->second : root->rootSymbol)
, cachedValue({root->db->setOrUpdate(getKey(), value), value})
{
}
Cursor::Cursor(
ref<Cache> root,
const Parent & parent,
const AttrId & id,
const AttrValue & value
)
: root(root)
, parentId(parent ? std::optional{parent->first.cachedValue.first} : std::nullopt)
, label(parent ? parent->second : root->rootSymbol)
, cachedValue({id, value})
{
}
AttrKey Cursor::getKey()
{
if (!parentId)
return {0, root->rootSymbol};
return {*parentId, label};
}
AttrValue Cursor::getCachedValue()
{
return cachedValue.second;
}
void Cursor::setValue(const AttrValue & v)
{
root->db->setValue(cachedValue.first, v);
cachedValue.second = v;
}
Cursor::Ref Cursor::addChild(const Symbol & attrPath, const AttrValue & v)
{
Parent parent = {{*this, attrPath}};
auto childCursor = new Cursor(
root,
parent,
v
);
return childCursor;
}
std::vector<std::string> Cursor::getChildren()
{
return root->db->getChildren(cachedValue.first);
}
std::optional<std::vector<std::string>> Cursor::getChildrenAtPath(const std::vector<Symbol> & attrPath)
{
auto cursorAtPath = findAlongAttrPath(attrPath);
if (cursorAtPath)
return cursorAtPath->getChildren();
return std::nullopt;
}
Cursor::Ref Cursor::maybeGetAttr(const Symbol & name)
{
auto rawAttr = root->db->getValue({cachedValue.first, name});
if (rawAttr) {
Parent parent = {{*this, name}};
debug("cache: hit for the attribute %s", cachedValue.first);
return new Cursor (
root, parent, rawAttr->first,
rawAttr->second);
}
if (std::holds_alternative<attributeSet_t>(cachedValue.second)) {
// If the parent is an attribute set but we're not present in the db,
// then we're not a member of this attribute set. So mark as missing
return addChild(name, missing_t{name});
}
return nullptr;
}
Cursor::Ref Cursor::findAlongAttrPath(const std::vector<Symbol> & attrPath)
{
auto currentCursor = this;
for (auto & currentAccessor : attrPath) {
currentCursor = currentCursor->maybeGetAttr(currentAccessor);
if (!currentCursor)
break;
if (std::holds_alternative<missing_t>(currentCursor->cachedValue.second))
break;
if (std::holds_alternative<failed_t>(currentCursor->cachedValue.second))
break;
}
return currentCursor;
}
const RawValue RawValue::fromVariant(const AttrValue & value)
{
RawValue res;
std::visit(overloaded{
[&](attributeSet_t x) { res.type = AttrType::Attrs; },
[&](string_t x) {
res.type = AttrType::String;
res.value = x.first;
res.context = x.second;
},
[&](wrapped_basetype<bool> x) {
res.type = AttrType::Bool;
res.value = x.value ? "1" : "0";
},
[&](wrapped_basetype<int64_t> x) {
res.type = AttrType::Int;
res.value = std::to_string(x.value);
},
[&](wrapped_basetype<double> x) {
res.type = AttrType::Double;
res.value = std::to_string(x.value);
},
[&](unknown_t x) { res.type = AttrType::Unknown; },
[&](missing_t x) { res.type = AttrType::Missing; },
[&](thunk_t x) { res.type = AttrType::Thunk; },
[&](failed_t x) {
res.type = AttrType::Failed;
res.value = x.error;
}
}, value);
return res;
}
std::string RawValue::serializeContext() const
{
std::string res;
for (auto & elt : context) {
res.append(encodeContext(elt.second, elt.first));
res.push_back(' ');
}
if (!res.empty())
res.pop_back(); // Remove the trailing space
return res;
}
}
+156
View File
@@ -0,0 +1,156 @@
/**
* caching for a tree-like data structure (like Nix values)
*
* The cache is an sqlite db whose rows are the nodes of the tree, with a
* pointer to their parent (except for the root of course)
*/
#pragma once
#include "sync.hh"
#include "hash.hh"
#include "symbol-table.hh"
#include <functional>
#include <variant>
namespace nix::tree_cache {
struct AttrDb;
class Cursor;
class Cache : public std::enable_shared_from_this<Cache>
{
private:
friend class Cursor;
/**
* The database holding the cache
*/
std::shared_ptr<AttrDb> db;
SymbolTable & symbols;
/**
* Distinguished symbol indicating the root of the tree
*/
const Symbol rootSymbol;
public:
Cache(
const Hash & useCache,
SymbolTable & symbols
);
static std::shared_ptr<Cache> tryCreate(const Hash & useCache, SymbolTable & symbols);
Cursor * getRoot();
/**
* Flush the cache to disk
*/
void commit();
};
enum AttrType {
Unknown = 0,
Attrs = 1,
String = 2,
Bool = 3,
Int = 4,
Double = 5,
Thunk = 6,
Missing = 7, // Missing fields of attribute sets
Failed = 8,
};
struct attributeSet_t {};
struct unknown_t {};
struct thunk_t {};
struct failed_t { string error; };
struct missing_t { Symbol attrName; };
// Putting several different primitive types in an `std::variant` partially
// breaks the `std::visit(overloaded{...` hackery because of the implicit cast
// from one to another which breaks the exhaustiveness check.
// So we wrap them in a trivial class just to force the disambiguation
template<typename T>
struct wrapped_basetype{ T value; };
typedef uint64_t AttrId;
typedef std::pair<AttrId, Symbol> AttrKey;
typedef std::pair<std::string, std::vector<std::pair<Path, std::string>>> string_t;
typedef std::variant<
attributeSet_t,
string_t,
unknown_t,
thunk_t,
missing_t,
failed_t,
wrapped_basetype<bool>,
wrapped_basetype<int64_t>,
wrapped_basetype<double>
> AttrValue;
struct RawValue {
AttrType type;
std::optional<std::string> value;
std::vector<std::pair<Path, std::string>> context;
std::string serializeContext() const;
static const RawValue fromVariant(const AttrValue&);
AttrValue toVariant() const;
};
/**
* View inside the cache.
*
* A `Cursor` represents a node in the cached tree (be it a leaf or not)
*/
class Cursor : public std::enable_shared_from_this<Cursor>
{
/**
* The overall cache of which this cursor is a view
*/
ref<Cache> root;
typedef std::optional<std::pair<Cursor&, Symbol>> Parent;
std::optional<AttrId> parentId;
Symbol label;
std::pair<AttrId, AttrValue> cachedValue;
/**
* Get the identifier for this node in the database
*/
AttrKey getKey();
public:
using Ref = Cursor*;
// Create a new cache entry
Cursor(ref<Cache> root, const Parent & parent, const AttrValue&);
// Build a cursor from an existing cache entry
Cursor(ref<Cache> root, const Parent & parent, const AttrId& id, const AttrValue&);
AttrValue getCachedValue();
void setValue(const AttrValue & v);
Ref addChild(const Symbol & attrPath, const AttrValue & v);
Ref findAlongAttrPath(const std::vector<Symbol> & attrPath);
Ref maybeGetAttr(const Symbol & attrPath);
std::vector<std::string> getChildren();
std::optional<std::vector<std::string>> getChildrenAtPath(const std::vector<Symbol> & attrPath);
};
}
+51
View File
@@ -0,0 +1,51 @@
#pragma once
#include "tree-cache.hh"
namespace nix {
struct Value;
class EvalState;
class Bindings;
class ValueCache {
tree_cache::Cursor::Ref rawCache;
public:
ValueCache(tree_cache::Cursor::Ref rawCache) : rawCache(rawCache) {}
static ValueCache empty;
bool isEmpty () { return rawCache == nullptr; }
enum ReturnCode {
// The cache result was an attribute set, so we forward it later in the
// chain
Forward,
CacheMiss,
CacheHit,
UnCacheable,
NoCacheKey,
};
struct CacheResult {
ReturnCode returnCode;
// In case the query returns a `missing_t`, the symbol that's missing
std::optional<Symbol> lastQueriedSymbolIfMissing;
};
std::pair<CacheResult, ValueCache> getValue(EvalState & state, const std::vector<Symbol> & selector, Value & dest);
ValueCache addChild(const Symbol & attrName, Value & value);
ValueCache addFailedChild(const Symbol & attrName, const Error & error);
ValueCache addNumChild(SymbolTable & symbols, int idx, const Value & value);
void addAttrSetChilds(Bindings & children);
void addListChilds(SymbolTable & symbols, Value** elems, int listSize);
std::optional<std::vector<Symbol>> listChildren(SymbolTable&);
std::optional<std::vector<Symbol>> listChildrenAtPath(SymbolTable&, const std::vector<Symbol> & attrPath);
std::optional<tree_cache::AttrValue> getRawValue();
ValueCache() : rawCache(nullptr) {}
};
}
+24 -4
View File
@@ -23,6 +23,7 @@ typedef enum {
tApp, tApp,
tLambda, tLambda,
tBlackhole, tBlackhole,
tCachedThunk,
tPrimOp, tPrimOp,
tPrimOpApp, tPrimOpApp,
tExternal, tExternal,
@@ -56,6 +57,7 @@ struct Pos;
class EvalState; class EvalState;
class XMLWriter; class XMLWriter;
class JSONPlaceholder; class JSONPlaceholder;
class ValueCache;
typedef int64_t NixInt; typedef int64_t NixInt;
@@ -103,6 +105,10 @@ class ExternalValueBase
std::ostream & operator << (std::ostream & str, const ExternalValueBase & v); std::ostream & operator << (std::ostream & str, const ExternalValueBase & v);
struct Thunk {
Env * env;
Expr * expr;
};
struct Value struct Value
{ {
@@ -122,6 +128,7 @@ public:
inline bool isThunk() const { return internalType == tThunk; }; inline bool isThunk() const { return internalType == tThunk; };
inline bool isApp() const { return internalType == tApp; }; inline bool isApp() const { return internalType == tApp; };
inline bool isBlackhole() const { return internalType == tBlackhole; }; inline bool isBlackhole() const { return internalType == tBlackhole; };
inline bool isCachedThunk() const { return internalType == tCachedThunk; };
// type() == nFunction // type() == nFunction
inline bool isLambda() const { return internalType == tLambda; }; inline bool isLambda() const { return internalType == tLambda; };
@@ -165,10 +172,13 @@ public:
Value * * elems; Value * * elems;
} bigList; } bigList;
Value * smallList[2]; Value * smallList[2];
Thunk thunk;
struct { struct {
Env * env; Thunk * thunk;
Expr * expr; // This is a pointer only to prevent a recursive import as
} thunk; // `EvalCache` is already a pointer so would fit very nicely here.
ValueCache * cache;
} cachedThunk;
struct { struct {
Value * left, * right; Value * left, * right;
} app; } app;
@@ -199,7 +209,7 @@ public:
case tLambda: case tPrimOp: case tPrimOpApp: return nFunction; case tLambda: case tPrimOp: case tPrimOpApp: return nFunction;
case tExternal: return nExternal; case tExternal: return nExternal;
case tFloat: return nFloat; case tFloat: return nFloat;
case tThunk: case tApp: case tBlackhole: return nThunk; case tThunk: case tApp: case tBlackhole: case tCachedThunk: return nThunk;
} }
abort(); abort();
} }
@@ -272,6 +282,13 @@ public:
thunk.expr = ex; thunk.expr = ex;
} }
inline void mkCachedThunk(Thunk * t, ValueCache * cache)
{
internalType = tCachedThunk;
cachedThunk.thunk = t;
cachedThunk.cache = cache;
}
inline void mkApp(Value * l, Value * r) inline void mkApp(Value * l, Value * r)
{ {
internalType = tApp; internalType = tApp;
@@ -349,6 +366,9 @@ public:
bool isTrivial() const; bool isTrivial() const;
std::vector<std::pair<Path, std::string>> getContext(); std::vector<std::pair<Path, std::string>> getContext();
ValueCache & getEvalCache();
void setEvalCache(ValueCache &);
}; };
+48 -34
View File
@@ -9,6 +9,7 @@
#include "shared.hh" #include "shared.hh"
#include "eval-cache.hh" #include "eval-cache.hh"
#include "attr-path.hh" #include "attr-path.hh"
#include "value.hh"
#include <regex> #include <regex>
#include <fstream> #include <fstream>
@@ -81,31 +82,46 @@ struct CmdSearch : InstallableCommand, MixJSON
uint64_t results = 0; uint64_t results = 0;
std::function<void(eval_cache::AttrCursor & cursor, const std::vector<Symbol> & attrPath, bool initialRecurse)> visit; std::function<void(Value & current, const std::vector<Symbol> & attrPath, bool initialRecurse)> visit;
visit = [&](eval_cache::AttrCursor & cursor, const std::vector<Symbol> & attrPath, bool initialRecurse) Value * vTmp = state->allocValue();
visit = [&](Value & current, const std::vector<Symbol> & attrPath, bool initialRecurse)
{ {
Activity act(*logger, lvlInfo, actUnknown, Activity act(*logger, lvlInfo, actUnknown,
fmt("evaluating '%s'", concatStringsSep(".", attrPath))); fmt("evaluating '%s'", concatStringsSep(".", attrPath)));
try { auto recurse = [&]()
auto recurse = [&]() {
{ for (auto & attrName : state->getFields(current, noPos)) {
for (const auto & attr : cursor.getAttrs()) { try {
auto cursor2 = cursor.getAttr(attr); auto attrPath2(attrPath);
auto attrPath2(attrPath); attrPath2.push_back(attrName);
attrPath2.push_back(attr); auto attrValue = state->allocValue();
visit(*cursor2, attrPath2, false); auto value_ = allocRootValue(attrValue);
} auto childField = state->lazyGetAttrField(current, {attrName}, noPos,
}; *attrValue);
if (childField != EvalState::LazyValueType::DelayedUncacheable)
visit(*attrValue, attrPath2, false);
} catch (EvalError &e) {
if (!(attrPath.size() > 0 &&
attrPath[0] == "legacyPackages"))
throw;
}
}
};
if (cursor.isDerivation()) { try {
auto maybeTypeField = state->lazyGetAttrField(current, {state->sType}, noPos, *vTmp);
if (maybeTypeField == EvalState::LazyValueType::PlainValue
&& vTmp->type() == nix::nString
&& strcmp(vTmp->string.s, "derivation") == 0) {
size_t found = 0; size_t found = 0;
DrvName name(cursor.getAttr("name")->getString()); state->getAttrFieldThrow(current, {state->sName}, noPos, *vTmp);
DrvName name(state->forceString(*vTmp));
auto aMeta = cursor.maybeGetAttr("meta"); auto hasDescription = state->getAttrField(current, {state->sMeta, state->sDescription}, noPos, *vTmp);
auto aDescription = aMeta ? aMeta->maybeGetAttr("description") : nullptr; auto description = hasDescription ? state->forceString(*vTmp) : "";
auto description = aDescription ? aDescription->getString() : "";
std::replace(description.begin(), description.end(), '\n', ' '); std::replace(description.begin(), description.end(), '\n', ' ');
auto attrPath2 = concatStringsSep(".", attrPath); auto attrPath2 = concatStringsSep(".", attrPath);
@@ -118,8 +134,8 @@ struct CmdSearch : InstallableCommand, MixJSON
std::regex_search(name.name, nameMatch, regex); std::regex_search(name.name, nameMatch, regex);
std::regex_search(description, descriptionMatch, regex); std::regex_search(description, descriptionMatch, regex);
if (!attrPathMatch.empty() if (!attrPathMatch.empty()
|| !nameMatch.empty() || !nameMatch.empty()
|| !descriptionMatch.empty()) || !descriptionMatch.empty())
found++; found++;
} }
@@ -134,39 +150,37 @@ struct CmdSearch : InstallableCommand, MixJSON
auto name2 = hilite(name.name, nameMatch, "\e[0;2m"); auto name2 = hilite(name.name, nameMatch, "\e[0;2m");
if (results > 1) logger->cout(""); if (results > 1) logger->cout("");
logger->cout( logger->cout(
"* %s%s", "* %s%s",
wrap("\e[0;1m", hilite(attrPath2, attrPathMatch, "\e[0;1m")), wrap("\e[0;1m", hilite(attrPath2, attrPathMatch, "\e[0;1m")),
name.version != "" ? " (" + name.version + ")" : ""); name.version != "" ? " (" + name.version + ")" : "");
if (description != "") if (description != "")
logger->cout( logger->cout(
" %s", hilite(description, descriptionMatch, ANSI_NORMAL)); " %s", hilite(description, descriptionMatch, ANSI_NORMAL));
} }
} }
} } else if (
attrPath.size() == 0
else if ( || (attrPath[0] == "legacyPackages" && attrPath.size() <= 2)
attrPath.size() == 0 || (attrPath[0] == "packages" && attrPath.size() <= 2))
|| (attrPath[0] == "legacyPackages" && attrPath.size() <= 2)
|| (attrPath[0] == "packages" && attrPath.size() <= 2))
recurse(); recurse();
else if (initialRecurse) else if (initialRecurse)
recurse(); recurse();
else if (attrPath[0] == "legacyPackages" && attrPath.size() > 2) { else if (attrPath[0] == "legacyPackages" && attrPath.size() > 2) {
auto attr = cursor.maybeGetAttr(state->sRecurseForDerivations); auto recurseFieldInfo = state->lazyGetAttrField(current, {state->sRecurseForDerivations}, noPos, *vTmp);
if (attr && attr->getBool()) auto hasRecurse = recurseFieldInfo == EvalState::LazyValueType::PlainValue;
if (hasRecurse && state->forceBool(*vTmp, noPos))
recurse(); recurse();
} }
} catch (EvalError & e) { } catch (EvalError & e) {
if (!(attrPath.size() > 0 && attrPath[0] == "legacyPackages")) if (!(attrPath.size() > 0 && attrPath[0] == "legacyPackages"))
throw; throw;
} }
}; };
for (auto & [cursor, prefix] : installable->getCursors(*state)) for (auto & [value, pos, prefix] : installable->toValues(*state))
visit(*cursor, parseAttrPath(*state, prefix), true); visit(*value, parseAttrPath(*state, prefix), true);
if (!json && !results) if (!json && !results)
throw Error("no results for the given search term(s)!"); throw Error("no results for the given search term(s)!");
+12
View File
@@ -0,0 +1,12 @@
let
# pkgs = (builtins.getFlake "github:NixOS/nixpkgs?rev=ad0d20345219790533ebe06571f82ed6b034db31").legacyPackages.x86_64-linux;
pkgs = import <nixpkgs> {};
in
pkgs.runCommandNoCC "foo" {
buildInputs = [
pkgs.firefox
pkgs.pandoc
# pkgs.nixosTests.xfce
];
}
"echo bar > $out"