Compare commits

..
Author SHA1 Message Date
Eelco Dolstra 1cd5e62402 BuildResult: Add a "Cancelled" status
This denotes the result of a build that didn't succeed or fail, but
was cancelled because some other goal failed and --keep-going was not
enabled.
2025-11-13 17:15:17 +01:00
109 changed files with 1176 additions and 2321 deletions

No files matched your search

+1 -1
View File
@@ -174,7 +174,7 @@ jobs:
echo "installer-url=file://$GITHUB_WORKSPACE/out" >> "$GITHUB_OUTPUT" echo "installer-url=file://$GITHUB_WORKSPACE/out" >> "$GITHUB_OUTPUT"
TARBALL_PATH="$(find "$GITHUB_WORKSPACE/out" -name 'nix*.tar.xz' -print | head -n 1)" TARBALL_PATH="$(find "$GITHUB_WORKSPACE/out" -name 'nix*.tar.xz' -print | head -n 1)"
echo "tarball-path=file://$TARBALL_PATH" >> "$GITHUB_OUTPUT" echo "tarball-path=file://$TARBALL_PATH" >> "$GITHUB_OUTPUT"
- uses: cachix/install-nix-action@0b0e072294b088b73964f1d72dfdac0951439dbd # v31.8.4 - uses: cachix/install-nix-action@456688f15bc354bef6d396e4a35f4f89d40bf2b7 # v31.8.2
if: ${{ !matrix.experimental-installer }} if: ${{ !matrix.experimental-installer }}
with: with:
install_url: ${{ format('{0}/install', steps.installer-tarball-url.outputs.installer-url) }} install_url: ${{ format('{0}/install', steps.installer-tarball-url.outputs.installer-url) }}
+3 -3
View File
@@ -36,7 +36,7 @@ to a temporary location. The tarball must include a single top-level
directory containing at least a file named `default.nix`. directory containing at least a file named `default.nix`.
`nix-build` is essentially a wrapper around `nix-build` is essentially a wrapper around
[`nix-instantiate`](./nix-instantiate.md) (to translate a high-level Nix [`nix-instantiate`](nix-instantiate.md) (to translate a high-level Nix
expression to a low-level [store derivation]) and [`nix-store expression to a low-level [store derivation]) and [`nix-store
--realise`](@docroot@/command-ref/nix-store/realise.md) (to build the store --realise`](@docroot@/command-ref/nix-store/realise.md) (to build the store
derivation). derivation).
@@ -52,8 +52,8 @@ derivation).
# Options # Options
All options not listed here are passed to All options not listed here are passed to
[`nix-store --realise`](./nix-store/realise.md), [`nix-store --realise`](nix-store/realise.md),
except for `--arg` and `--attr` / `-A` which are passed to [`nix-instantiate`](./nix-instantiate.md). except for `--arg` and `--attr` / `-A` which are passed to [`nix-instantiate`](nix-instantiate.md).
- <span id="opt-no-out-link">[`--no-out-link`](#opt-no-out-link)<span> - <span id="opt-no-out-link">[`--no-out-link`](#opt-no-out-link)<span>
@@ -32,7 +32,7 @@ standard input.
- `--add-root` *path* - `--add-root` *path*
See the [corresponding option](./nix-store.md) in `nix-store`. See the [corresponding option](nix-store.md) in `nix-store`.
- `--parse` - `--parse`
+16 -22
View File
@@ -130,19 +130,15 @@ pkgs.nixComponents2.nix-util.overrideAttrs (
havePerl = stdenv.buildPlatform == stdenv.hostPlatform && stdenv.hostPlatform.isUnix; havePerl = stdenv.buildPlatform == stdenv.hostPlatform && stdenv.hostPlatform.isUnix;
ignoreCrossFile = flags: builtins.filter (flag: !(lib.strings.hasInfix "cross-file" flag)) flags; ignoreCrossFile = flags: builtins.filter (flag: !(lib.strings.hasInfix "cross-file" flag)) flags;
availableComponents = lib.filterAttrs (
k: v: lib.meta.availableOn pkgs.hostPlatform v
) allComponents;
activeComponents = buildInputsClosureCond isInternal ( activeComponents = buildInputsClosureCond isInternal (
lib.attrValues (finalAttrs.passthru.config.getComponents availableComponents) lib.attrValues (finalAttrs.passthru.config.getComponents allComponents)
); );
allComponents = lib.filterAttrs (k: v: lib.isDerivation v) pkgs.nixComponents2; allComponents = lib.filterAttrs (k: v: lib.isDerivation v) pkgs.nixComponents2;
internalDrvs = byDrvPath ( internalDrvs = byDrvPath (
# Drop the attr names (not present in buildInputs anyway) # Drop the attr names (not present in buildInputs anyway)
lib.attrValues availableComponents lib.attrValues allComponents
++ lib.concatMap (c: lib.attrValues c.tests or { }) (lib.attrValues availableComponents) ++ lib.concatMap (c: lib.attrValues c.tests or { }) (lib.attrValues allComponents)
); );
isInternal = isInternal =
@@ -191,19 +187,19 @@ pkgs.nixComponents2.nix-util.overrideAttrs (
); );
small = small =
(finalAttrs.finalPackage.withActiveComponents ( (finalAttrs.finalPackage.withActiveComponents (c: {
c: inherit (c)
lib.intersectAttrs (lib.genAttrs [ nix-cli
"nix-cli" nix-util-tests
"nix-util-tests" nix-store-tests
"nix-store-tests" nix-expr-tests
"nix-expr-tests" nix-fetchers-tests
"nix-fetchers-tests" nix-flake-tests
"nix-flake-tests" nix-functional-tests
"nix-functional-tests" # Currently required
"nix-perl-bindings" nix-perl-bindings
] (_: null)) c ;
)).overrideAttrs })).overrideAttrs
(o: { (o: {
mesonFlags = o.mesonFlags ++ [ mesonFlags = o.mesonFlags ++ [
# TODO: infer from activeComponents or vice versa # TODO: infer from activeComponents or vice versa
@@ -273,8 +269,6 @@ pkgs.nixComponents2.nix-util.overrideAttrs (
CXX_LD = "mold"; CXX_LD = "mold";
}; };
dontUseCmakeConfigure = true;
mesonFlags = mesonFlags =
map (transformFlag "libutil") (ignoreCrossFile pkgs.nixComponents2.nix-util.mesonFlags) map (transformFlag "libutil") (ignoreCrossFile pkgs.nixComponents2.nix-util.mesonFlags)
++ map (transformFlag "libstore") (ignoreCrossFile pkgs.nixComponents2.nix-store.mesonFlags) ++ map (transformFlag "libstore") (ignoreCrossFile pkgs.nixComponents2.nix-store.mesonFlags)
+1 -1
View File
@@ -65,6 +65,6 @@ mkMesonDerivation (finalAttrs: {
''; '';
meta = { meta = {
platforms = lib.platforms.unix; platforms = lib.platforms.all;
}; };
}) })
+3 -5
View File
@@ -33,8 +33,7 @@ EvalSettings evalSettings{
// FIXME `parseFlakeRef` should take a `std::string_view`. // FIXME `parseFlakeRef` should take a `std::string_view`.
auto flakeRef = parseFlakeRef(fetchSettings, std::string{rest}, {}, true, false); auto flakeRef = parseFlakeRef(fetchSettings, std::string{rest}, {}, true, false);
debug("fetching flake search path element '%s''", rest); debug("fetching flake search path element '%s''", rest);
auto [accessor, lockedRef] = auto [accessor, lockedRef] = flakeRef.resolve(state.store).lazyFetch(state.store);
flakeRef.resolve(fetchSettings, *state.store).lazyFetch(fetchSettings, *state.store);
auto storePath = nix::fetchToStore( auto storePath = nix::fetchToStore(
state.fetchSettings, state.fetchSettings,
*state.store, *state.store,
@@ -180,7 +179,7 @@ Bindings * MixEvalArgs::getAutoArgs(EvalState & state)
SourcePath lookupFileArg(EvalState & state, std::string_view s, const Path * baseDir) SourcePath lookupFileArg(EvalState & state, std::string_view s, const Path * baseDir)
{ {
if (EvalSettings::isPseudoUrl(s)) { if (EvalSettings::isPseudoUrl(s)) {
auto accessor = fetchers::downloadTarball(*state.store, state.fetchSettings, EvalSettings::resolvePseudoUrl(s)); auto accessor = fetchers::downloadTarball(state.store, state.fetchSettings, EvalSettings::resolvePseudoUrl(s));
auto storePath = fetchToStore(state.fetchSettings, *state.store, SourcePath(accessor), FetchMode::Copy); auto storePath = fetchToStore(state.fetchSettings, *state.store, SourcePath(accessor), FetchMode::Copy);
return state.storePath(storePath); return state.storePath(storePath);
} }
@@ -188,8 +187,7 @@ SourcePath lookupFileArg(EvalState & state, std::string_view s, const Path * bas
else if (hasPrefix(s, "flake:")) { else if (hasPrefix(s, "flake:")) {
experimentalFeatureSettings.require(Xp::Flakes); experimentalFeatureSettings.require(Xp::Flakes);
auto flakeRef = parseFlakeRef(fetchSettings, std::string(s.substr(6)), {}, true, false); auto flakeRef = parseFlakeRef(fetchSettings, std::string(s.substr(6)), {}, true, false);
auto [accessor, lockedRef] = auto [accessor, lockedRef] = flakeRef.resolve(state.store).lazyFetch(state.store);
flakeRef.resolve(fetchSettings, *state.store).lazyFetch(fetchSettings, *state.store);
auto storePath = nix::fetchToStore( auto storePath = nix::fetchToStore(
state.fetchSettings, *state.store, SourcePath(accessor), FetchMode::Copy, lockedRef.input.getName()); state.fetchSettings, *state.store, SourcePath(accessor), FetchMode::Copy, lockedRef.input.getName());
state.allowPath(storePath); state.allowPath(storePath);
+1 -1
View File
@@ -409,7 +409,7 @@ void completeFlakeRef(AddCompletions & completions, ref<Store> store, std::strin
Args::completeDir(completions, 0, prefix); Args::completeDir(completions, 0, prefix);
/* Look for registry entries that match the prefix. */ /* Look for registry entries that match the prefix. */
for (auto & registry : fetchers::getRegistries(fetchSettings, *store)) { for (auto & registry : fetchers::getRegistries(fetchSettings, store)) {
for (auto & entry : registry->entries) { for (auto & entry : registry->entries) {
auto from = entry.from.to_string(); auto from = entry.from.to_string();
if (!hasPrefix(prefix, "flake:") && hasPrefix(from, "flake:")) { if (!hasPrefix(prefix, "flake:") && hasPrefix(from, "flake:")) {
+2 -2
View File
@@ -738,8 +738,8 @@ void NixRepl::loadFlake(const std::string & flakeRefS)
} }
auto flakeRef = parseFlakeRef(fetchSettings, flakeRefS, cwd.string(), true); auto flakeRef = parseFlakeRef(fetchSettings, flakeRefS, cwd.string(), true);
if (evalSettings.pureEval && !flakeRef.input.isLocked(fetchSettings)) if (evalSettings.pureEval && !flakeRef.input.isLocked())
throw Error("cannot use ':load-flake' on unlocked flake reference '%s' (use --impure to override)", flakeRefS); throw Error("cannot use ':load-flake' on locked flake reference '%s' (use --impure to override)", flakeRefS);
Value v; Value v;
-4
View File
@@ -137,8 +137,6 @@ nix_eval_state_builder * nix_eval_state_builder_new(nix_c_context * context, Sto
void nix_eval_state_builder_free(nix_eval_state_builder * builder) void nix_eval_state_builder_free(nix_eval_state_builder * builder)
{ {
if (builder)
builder->~nix_eval_state_builder();
operator delete(builder, static_cast<std::align_val_t>(alignof(nix_eval_state_builder))); operator delete(builder, static_cast<std::align_val_t>(alignof(nix_eval_state_builder)));
} }
@@ -205,8 +203,6 @@ EvalState * nix_state_create(nix_c_context * context, const char ** lookupPath_c
void nix_state_free(EvalState * state) void nix_state_free(EvalState * state)
{ {
if (state)
state->~EvalState();
operator delete(state, static_cast<std::align_val_t>(alignof(EvalState))); operator delete(state, static_cast<std::align_val_t>(alignof(EvalState)));
} }
+1 -5
View File
@@ -14,7 +14,7 @@
namespace nixC { namespace nixC {
TEST_F(nix_api_expr_test, nix_eval_state_lookup_path) TEST_F(nix_api_store_test, nix_eval_state_lookup_path)
{ {
auto tmpDir = nix::createTempDir(); auto tmpDir = nix::createTempDir();
auto delTmpDir = std::make_unique<nix::AutoDelete>(tmpDir, true); auto delTmpDir = std::make_unique<nix::AutoDelete>(tmpDir, true);
@@ -42,16 +42,12 @@ TEST_F(nix_api_expr_test, nix_eval_state_lookup_path)
nix_expr_eval_from_string(ctx, state, "builtins.seq <nixos-config> <nixpkgs>", ".", value); nix_expr_eval_from_string(ctx, state, "builtins.seq <nixos-config> <nixpkgs>", ".", value);
assert_ctx_ok(); assert_ctx_ok();
nix_state_free(state);
ASSERT_EQ(nix_get_type(ctx, value), NIX_TYPE_PATH); ASSERT_EQ(nix_get_type(ctx, value), NIX_TYPE_PATH);
assert_ctx_ok(); assert_ctx_ok();
auto pathStr = nix_get_path_string(ctx, value); auto pathStr = nix_get_path_string(ctx, value);
assert_ctx_ok(); assert_ctx_ok();
ASSERT_EQ(0, strcmp(pathStr, nixpkgs.c_str())); ASSERT_EQ(0, strcmp(pathStr, nixpkgs.c_str()));
nix_gc_decref(nullptr, value);
} }
TEST_F(nix_api_expr_test, nix_expr_eval_from_string) TEST_F(nix_api_expr_test, nix_expr_eval_from_string)
+2 -8
View File
@@ -661,14 +661,8 @@ INSTANTIATE_TEST_SUITE_P(
CASE(R"(null)", ""), CASE(R"(null)", ""),
CASE(R"({ v = "bar"; __toString = self: self.v; })", "bar"), CASE(R"({ v = "bar"; __toString = self: self.v; })", "bar"),
CASE(R"({ v = "bar"; __toString = self: self.v; outPath = "foo"; })", "bar"), CASE(R"({ v = "bar"; __toString = self: self.v; outPath = "foo"; })", "bar"),
CASE(R"({ outPath = "foo"; })", "foo") CASE(R"({ outPath = "foo"; })", "foo"),
// this is broken on cygwin because canonPath("//./test", false) returns //./test CASE(R"(./test)", "/test")));
// FIXME: don't use canonPath
#ifndef __CYGWIN__
,
CASE(R"(./test)", "/test")
#endif
));
#undef CASE #undef CASE
TEST_F(PrimOpTest, substring) TEST_F(PrimOpTest, substring)
+21 -23
View File
@@ -205,7 +205,7 @@ bool Value::isTrivial() const
{ {
return !isa<tApp, tPrimOpApp>() return !isa<tApp, tPrimOpApp>()
&& (!isa<tThunk>() && (!isa<tThunk>()
|| (dynamic_cast<ExprAttrs *>(thunk().expr) && ((ExprAttrs *) thunk().expr)->dynamicAttrs->empty()) || (dynamic_cast<ExprAttrs *>(thunk().expr) && ((ExprAttrs *) thunk().expr)->dynamicAttrs.empty())
|| dynamic_cast<ExprLambda *>(thunk().expr) || dynamic_cast<ExprList *>(thunk().expr)); || dynamic_cast<ExprLambda *>(thunk().expr) || dynamic_cast<ExprList *>(thunk().expr));
} }
@@ -517,16 +517,15 @@ Value * EvalState::addPrimOp(PrimOp && primOp)
if (primOp.arity == 0) { if (primOp.arity == 0) {
primOp.arity = 1; primOp.arity = 1;
auto vPrimOp = allocValue(); auto vPrimOp = allocValue();
vPrimOp->mkPrimOp(new PrimOp(std::move(primOp))); vPrimOp->mkPrimOp(new PrimOp(primOp));
Value v; Value v;
v.mkApp(vPrimOp, vPrimOp); v.mkApp(vPrimOp, vPrimOp);
auto & primOp1 = *vPrimOp->primOp();
return addConstant( return addConstant(
primOp1.name, primOp.name,
v, v,
{ {
.type = nThunk, // FIXME .type = nThunk, // FIXME
.doc = primOp1.doc ? primOp1.doc->c_str() : nullptr, .doc = primOp.doc,
}); });
} }
@@ -566,14 +565,13 @@ std::optional<EvalState::Doc> EvalState::getDoc(Value & v)
{ {
if (v.isPrimOp()) { if (v.isPrimOp()) {
auto v2 = &v; auto v2 = &v;
auto & primOp = *v2->primOp(); if (auto * doc = v2->primOp()->doc)
if (primOp.doc)
return Doc{ return Doc{
.pos = {}, .pos = {},
.name = primOp.name, .name = v2->primOp()->name,
.arity = primOp.arity, .arity = v2->primOp()->arity,
.args = primOp.args, .args = v2->primOp()->args,
.doc = primOp.doc->c_str(), .doc = doc,
}; };
} }
if (v.isLambda()) { if (v.isLambda()) {
@@ -1226,26 +1224,26 @@ Env * ExprAttrs::buildInheritFromEnv(EvalState & state, Env & up)
void ExprAttrs::eval(EvalState & state, Env & env, Value & v) void ExprAttrs::eval(EvalState & state, Env & env, Value & v)
{ {
auto bindings = state.buildBindings(attrs->size() + dynamicAttrs->size()); auto bindings = state.buildBindings(attrs.size() + dynamicAttrs.size());
auto dynamicEnv = &env; auto dynamicEnv = &env;
bool sort = false; bool sort = false;
if (recursive) { if (recursive) {
/* Create a new environment that contains the attributes in /* Create a new environment that contains the attributes in
this `rec'. */ this `rec'. */
Env & env2(state.mem.allocEnv(attrs->size())); Env & env2(state.mem.allocEnv(attrs.size()));
env2.up = &env; env2.up = &env;
dynamicEnv = &env2; dynamicEnv = &env2;
Env * inheritEnv = inheritFromExprs ? buildInheritFromEnv(state, env2) : nullptr; Env * inheritEnv = inheritFromExprs ? buildInheritFromEnv(state, env2) : nullptr;
AttrDefs::iterator overrides = attrs->find(state.s.overrides); AttrDefs::iterator overrides = attrs.find(state.s.overrides);
bool hasOverrides = overrides != attrs->end(); bool hasOverrides = overrides != attrs.end();
/* The recursive attributes are evaluated in the new /* The recursive attributes are evaluated in the new
environment, while the inherited attributes are evaluated environment, while the inherited attributes are evaluated
in the original environment. */ in the original environment. */
Displacement displ = 0; Displacement displ = 0;
for (auto & i : *attrs) { for (auto & i : attrs) {
Value * vAttr; Value * vAttr;
if (hasOverrides && i.second.kind != AttrDef::Kind::Inherited) { if (hasOverrides && i.second.kind != AttrDef::Kind::Inherited) {
vAttr = state.allocValue(); vAttr = state.allocValue();
@@ -1272,8 +1270,8 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v)
"while evaluating the `__overrides` attribute"); "while evaluating the `__overrides` attribute");
bindings.grow(state.buildBindings(bindings.capacity() + vOverrides->attrs()->size())); bindings.grow(state.buildBindings(bindings.capacity() + vOverrides->attrs()->size()));
for (auto & i : *vOverrides->attrs()) { for (auto & i : *vOverrides->attrs()) {
AttrDefs::iterator j = attrs->find(i.name); AttrDefs::iterator j = attrs.find(i.name);
if (j != attrs->end()) { if (j != attrs.end()) {
(*bindings.bindings)[j->second.displ] = i; (*bindings.bindings)[j->second.displ] = i;
env2.values[j->second.displ] = i.value; env2.values[j->second.displ] = i.value;
} else } else
@@ -1285,13 +1283,13 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v)
else { else {
Env * inheritEnv = inheritFromExprs ? buildInheritFromEnv(state, env) : nullptr; Env * inheritEnv = inheritFromExprs ? buildInheritFromEnv(state, env) : nullptr;
for (auto & i : *attrs) for (auto & i : attrs)
bindings.insert( bindings.insert(
i.first, i.second.e->maybeThunk(state, *i.second.chooseByKind(&env, &env, inheritEnv)), i.second.pos); i.first, i.second.e->maybeThunk(state, *i.second.chooseByKind(&env, &env, inheritEnv)), i.second.pos);
} }
/* Dynamic attrs apply *after* rec and __overrides. */ /* Dynamic attrs apply *after* rec and __overrides. */
for (auto & i : *dynamicAttrs) { for (auto & i : dynamicAttrs) {
Value nameVal; Value nameVal;
i.nameExpr->eval(state, *dynamicEnv, nameVal); i.nameExpr->eval(state, *dynamicEnv, nameVal);
state.forceValue(nameVal, i.pos); state.forceValue(nameVal, i.pos);
@@ -1325,7 +1323,7 @@ void ExprLet::eval(EvalState & state, Env & env, Value & v)
{ {
/* Create a new environment that contains the attributes in this /* Create a new environment that contains the attributes in this
`let'. */ `let'. */
Env & env2(state.mem.allocEnv(attrs->attrs->size())); Env & env2(state.mem.allocEnv(attrs->attrs.size()));
env2.up = &env; env2.up = &env;
Env * inheritEnv = attrs->inheritFromExprs ? attrs->buildInheritFromEnv(state, env2) : nullptr; Env * inheritEnv = attrs->inheritFromExprs ? attrs->buildInheritFromEnv(state, env2) : nullptr;
@@ -1334,7 +1332,7 @@ void ExprLet::eval(EvalState & state, Env & env, Value & v)
while the inherited attributes are evaluated in the original while the inherited attributes are evaluated in the original
environment. */ environment. */
Displacement displ = 0; Displacement displ = 0;
for (auto & i : *attrs->attrs) { for (auto & i : attrs->attrs) {
env2.values[displ++] = i.second.e->maybeThunk(state, *i.second.chooseByKind(&env2, &env, inheritEnv)); env2.values[displ++] = i.second.e->maybeThunk(state, *i.second.chooseByKind(&env2, &env, inheritEnv));
} }
@@ -3190,7 +3188,7 @@ std::optional<SourcePath> EvalState::resolveLookupPathPath(const LookupPath::Pat
if (EvalSettings::isPseudoUrl(value)) { if (EvalSettings::isPseudoUrl(value)) {
try { try {
auto accessor = fetchers::downloadTarball(*store, fetchSettings, EvalSettings::resolvePseudoUrl(value)); auto accessor = fetchers::downloadTarball(store, fetchSettings, EvalSettings::resolvePseudoUrl(value));
auto storePath = fetchToStore(fetchSettings, *store, SourcePath(accessor), FetchMode::Copy); auto storePath = fetchToStore(fetchSettings, *store, SourcePath(accessor), FetchMode::Copy);
return finish(this->storePath(storePath)); return finish(this->storePath(storePath));
} catch (Error & e) { } catch (Error & e) {
+1 -1
View File
@@ -108,7 +108,7 @@ struct PrimOp
/** /**
* Optional free-form documentation about the primop. * Optional free-form documentation about the primop.
*/ */
std::optional<std::string> doc; const char * doc = nullptr;
/** /**
* Add a trace item, while calling the `<name>` builtin. * Add a trace item, while calling the `<name>` builtin.
+7 -18
View File
@@ -395,13 +395,9 @@ struct ExprAttrs : Expr
} }
}; };
typedef std::pmr::map<Symbol, AttrDef> AttrDefs; typedef std::map<Symbol, AttrDef> AttrDefs;
/** AttrDefs attrs;
* attrs will never be null. we use std::optional so that we can call emplace() to re-initialize the value with a std::unique_ptr<std::vector<Expr *>> inheritFromExprs;
* new pmr::map using a different allocator (move assignment will copy into the old allocator)
*/
std::optional<AttrDefs> attrs;
std::unique_ptr<std::pmr::vector<Expr *>> inheritFromExprs;
struct DynamicAttrDef struct DynamicAttrDef
{ {
@@ -413,20 +409,13 @@ struct ExprAttrs : Expr
, pos(pos) {}; , pos(pos) {};
}; };
typedef std::pmr::vector<DynamicAttrDef> DynamicAttrDefs; typedef std::vector<DynamicAttrDef> DynamicAttrDefs;
/** DynamicAttrDefs dynamicAttrs;
* dynamicAttrs will never be null. See comment on AttrDefs above.
*/
std::optional<DynamicAttrDefs> dynamicAttrs;
ExprAttrs(const PosIdx & pos) ExprAttrs(const PosIdx & pos)
: recursive(false) : recursive(false)
, pos(pos) , pos(pos) {};
, attrs(AttrDefs{})
, dynamicAttrs(DynamicAttrDefs{}) {};
ExprAttrs() ExprAttrs()
: recursive(false) : recursive(false) {};
, attrs(AttrDefs{})
, dynamicAttrs(DynamicAttrDefs{}) {};
PosIdx getPos() const override PosIdx getPos() const override
{ {
+16 -16
View File
@@ -126,8 +126,8 @@ inline void ParserState::addAttr(
for (i = attrPath.begin(); i + 1 < attrPath.end(); i++) { for (i = attrPath.begin(); i + 1 < attrPath.end(); i++) {
ExprAttrs * nested; ExprAttrs * nested;
if (i->symbol) { if (i->symbol) {
ExprAttrs::AttrDefs::iterator j = attrs->attrs->find(i->symbol); ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(i->symbol);
if (j != attrs->attrs->end()) { if (j != attrs->attrs.end()) {
nested = dynamic_cast<ExprAttrs *>(j->second.e); nested = dynamic_cast<ExprAttrs *>(j->second.e);
if (!nested) { if (!nested) {
attrPath.erase(i + 1, attrPath.end()); attrPath.erase(i + 1, attrPath.end());
@@ -135,11 +135,11 @@ inline void ParserState::addAttr(
} }
} else { } else {
nested = exprs.add<ExprAttrs>(); nested = exprs.add<ExprAttrs>();
(*attrs->attrs)[i->symbol] = ExprAttrs::AttrDef(nested, pos); attrs->attrs[i->symbol] = ExprAttrs::AttrDef(nested, pos);
} }
} else { } else {
nested = exprs.add<ExprAttrs>(); nested = exprs.add<ExprAttrs>();
attrs->dynamicAttrs->push_back(ExprAttrs::DynamicAttrDef(i->expr, nested, pos)); attrs->dynamicAttrs.push_back(ExprAttrs::DynamicAttrDef(i->expr, nested, pos));
} }
attrs = nested; attrs = nested;
} }
@@ -148,7 +148,7 @@ inline void ParserState::addAttr(
if (i->symbol) { if (i->symbol) {
addAttr(attrs, attrPath, i->symbol, ExprAttrs::AttrDef(e, pos)); addAttr(attrs, attrPath, i->symbol, ExprAttrs::AttrDef(e, pos));
} else { } else {
attrs->dynamicAttrs->push_back(ExprAttrs::DynamicAttrDef(i->expr, e, pos)); attrs->dynamicAttrs.push_back(ExprAttrs::DynamicAttrDef(i->expr, e, pos));
} }
auto it = lexerState.positionToDocComment.find(pos); auto it = lexerState.positionToDocComment.find(pos);
@@ -165,8 +165,8 @@ inline void ParserState::addAttr(
inline void inline void
ParserState::addAttr(ExprAttrs * attrs, AttrPath & attrPath, const Symbol & symbol, ExprAttrs::AttrDef && def) ParserState::addAttr(ExprAttrs * attrs, AttrPath & attrPath, const Symbol & symbol, ExprAttrs::AttrDef && def)
{ {
ExprAttrs::AttrDefs::iterator j = attrs->attrs->find(symbol); ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(symbol);
if (j != attrs->attrs->end()) { if (j != attrs->attrs.end()) {
// This attr path is already defined. However, if both // This attr path is already defined. However, if both
// e and the expr pointed by the attr path are two attribute sets, // e and the expr pointed by the attr path are two attribute sets,
// we want to merge them. // we want to merge them.
@@ -181,8 +181,8 @@ ParserState::addAttr(ExprAttrs * attrs, AttrPath & attrPath, const Symbol & symb
// See https://github.com/NixOS/nix/issues/9020. // See https://github.com/NixOS/nix/issues/9020.
if (jAttrs && ae) { if (jAttrs && ae) {
if (ae->inheritFromExprs && !jAttrs->inheritFromExprs) if (ae->inheritFromExprs && !jAttrs->inheritFromExprs)
jAttrs->inheritFromExprs = std::make_unique<std::pmr::vector<Expr *>>(); jAttrs->inheritFromExprs = std::make_unique<std::vector<Expr *>>();
for (auto & ad : *ae->attrs) { for (auto & ad : ae->attrs) {
if (ad.second.kind == ExprAttrs::AttrDef::Kind::InheritedFrom) { if (ad.second.kind == ExprAttrs::AttrDef::Kind::InheritedFrom) {
auto & sel = dynamic_cast<ExprSelect &>(*ad.second.e); auto & sel = dynamic_cast<ExprSelect &>(*ad.second.e);
auto & from = dynamic_cast<ExprInheritFrom &>(*sel.e); auto & from = dynamic_cast<ExprInheritFrom &>(*sel.e);
@@ -192,12 +192,12 @@ ParserState::addAttr(ExprAttrs * attrs, AttrPath & attrPath, const Symbol & symb
addAttr(jAttrs, attrPath, ad.first, std::move(ad.second)); addAttr(jAttrs, attrPath, ad.first, std::move(ad.second));
attrPath.pop_back(); attrPath.pop_back();
} }
ae->attrs->clear(); ae->attrs.clear();
jAttrs->dynamicAttrs->insert( jAttrs->dynamicAttrs.insert(
jAttrs->dynamicAttrs->end(), jAttrs->dynamicAttrs.end(),
std::make_move_iterator(ae->dynamicAttrs->begin()), std::make_move_iterator(ae->dynamicAttrs.begin()),
std::make_move_iterator(ae->dynamicAttrs->end())); std::make_move_iterator(ae->dynamicAttrs.end()));
ae->dynamicAttrs->clear(); ae->dynamicAttrs.clear();
if (ae->inheritFromExprs) { if (ae->inheritFromExprs) {
jAttrs->inheritFromExprs->insert( jAttrs->inheritFromExprs->insert(
jAttrs->inheritFromExprs->end(), jAttrs->inheritFromExprs->end(),
@@ -210,7 +210,7 @@ ParserState::addAttr(ExprAttrs * attrs, AttrPath & attrPath, const Symbol & symb
} }
} else { } else {
// This attr path is not defined. Let's create it. // This attr path is not defined. Let's create it.
attrs->attrs->emplace(symbol, def); attrs->attrs.emplace(symbol, def);
def.e->setName(symbol); def.e->setName(symbol);
} }
} }
+12 -23
View File
@@ -74,9 +74,9 @@ void ExprOpHasAttr::show(const SymbolTable & symbols, std::ostream & str) const
void ExprAttrs::showBindings(const SymbolTable & symbols, std::ostream & str) const void ExprAttrs::showBindings(const SymbolTable & symbols, std::ostream & str) const
{ {
typedef const AttrDefs::value_type * Attr; typedef const decltype(attrs)::value_type * Attr;
std::vector<Attr> sorted; std::vector<Attr> sorted;
for (auto & i : *attrs) for (auto & i : attrs)
sorted.push_back(&i); sorted.push_back(&i);
std::sort(sorted.begin(), sorted.end(), [&](Attr a, Attr b) { std::sort(sorted.begin(), sorted.end(), [&](Attr a, Attr b) {
std::string_view sa = symbols[a->first], sb = symbols[b->first]; std::string_view sa = symbols[a->first], sb = symbols[b->first];
@@ -122,7 +122,7 @@ void ExprAttrs::showBindings(const SymbolTable & symbols, std::ostream & str) co
str << "; "; str << "; ";
} }
} }
for (auto & i : *dynamicAttrs) { for (auto & i : dynamicAttrs) {
str << "\"${"; str << "\"${";
i.nameExpr->show(symbols, str); i.nameExpr->show(symbols, str);
str << "}\" = "; str << "}\" = ";
@@ -401,26 +401,15 @@ ExprAttrs::bindInheritSources(EvalState & es, const std::shared_ptr<const Static
void ExprAttrs::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> & env) void ExprAttrs::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> & env)
{ {
// Move storage into the Exprs arena
{
auto arena = es.mem.exprs.alloc;
AttrDefs newAttrs{std::move(*attrs), arena};
attrs.emplace(std::move(newAttrs), arena);
DynamicAttrDefs newDynamicAttrs{std::move(*dynamicAttrs), arena};
dynamicAttrs.emplace(std::move(newDynamicAttrs), arena);
if (inheritFromExprs)
inheritFromExprs = std::make_unique<std::pmr::vector<Expr *>>(std::move(*inheritFromExprs), arena);
}
if (es.debugRepl) if (es.debugRepl)
es.exprEnvs.insert(std::make_pair(this, env)); es.exprEnvs.insert(std::make_pair(this, env));
if (recursive) { if (recursive) {
auto newEnv = [&]() -> std::shared_ptr<const StaticEnv> { auto newEnv = [&]() -> std::shared_ptr<const StaticEnv> {
auto newEnv = std::make_shared<StaticEnv>(nullptr, env, attrs->size()); auto newEnv = std::make_shared<StaticEnv>(nullptr, env, attrs.size());
Displacement displ = 0; Displacement displ = 0;
for (auto & i : *attrs) for (auto & i : attrs)
newEnv->vars.emplace_back(i.first, i.second.displ = displ++); newEnv->vars.emplace_back(i.first, i.second.displ = displ++);
return newEnv; return newEnv;
}(); }();
@@ -428,20 +417,20 @@ void ExprAttrs::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv>
// No need to sort newEnv since attrs is in sorted order. // No need to sort newEnv since attrs is in sorted order.
auto inheritFromEnv = bindInheritSources(es, newEnv); auto inheritFromEnv = bindInheritSources(es, newEnv);
for (auto & i : *attrs) for (auto & i : attrs)
i.second.e->bindVars(es, i.second.chooseByKind(newEnv, env, inheritFromEnv)); i.second.e->bindVars(es, i.second.chooseByKind(newEnv, env, inheritFromEnv));
for (auto & i : *dynamicAttrs) { for (auto & i : dynamicAttrs) {
i.nameExpr->bindVars(es, newEnv); i.nameExpr->bindVars(es, newEnv);
i.valueExpr->bindVars(es, newEnv); i.valueExpr->bindVars(es, newEnv);
} }
} else { } else {
auto inheritFromEnv = bindInheritSources(es, env); auto inheritFromEnv = bindInheritSources(es, env);
for (auto & i : *attrs) for (auto & i : attrs)
i.second.e->bindVars(es, i.second.chooseByKind(env, env, inheritFromEnv)); i.second.e->bindVars(es, i.second.chooseByKind(env, env, inheritFromEnv));
for (auto & i : *dynamicAttrs) { for (auto & i : dynamicAttrs) {
i.nameExpr->bindVars(es, env); i.nameExpr->bindVars(es, env);
i.valueExpr->bindVars(es, env); i.valueExpr->bindVars(es, env);
} }
@@ -497,10 +486,10 @@ void ExprCall::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> &
void ExprLet::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> & env) void ExprLet::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> & env)
{ {
auto newEnv = [&]() -> std::shared_ptr<const StaticEnv> { auto newEnv = [&]() -> std::shared_ptr<const StaticEnv> {
auto newEnv = std::make_shared<StaticEnv>(nullptr, env, attrs->attrs->size()); auto newEnv = std::make_shared<StaticEnv>(nullptr, env, attrs->attrs.size());
Displacement displ = 0; Displacement displ = 0;
for (auto & i : *attrs->attrs) for (auto & i : attrs->attrs)
newEnv->vars.emplace_back(i.first, i.second.displ = displ++); newEnv->vars.emplace_back(i.first, i.second.displ = displ++);
return newEnv; return newEnv;
}(); }();
@@ -508,7 +497,7 @@ void ExprLet::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> &
// No need to sort newEnv since attrs->attrs is in sorted order. // No need to sort newEnv since attrs->attrs is in sorted order.
auto inheritFromEnv = attrs->bindInheritSources(es, newEnv); auto inheritFromEnv = attrs->bindInheritSources(es, newEnv);
for (auto & i : *attrs->attrs) for (auto & i : attrs->attrs)
i.second.e->bindVars(es, i.second.chooseByKind(newEnv, env, inheritFromEnv)); i.second.e->bindVars(es, i.second.chooseByKind(newEnv, env, inheritFromEnv));
if (es.debugRepl) if (es.debugRepl)
+8 -8
View File
@@ -211,7 +211,7 @@ expr_function
| WITH expr ';' expr_function | WITH expr ';' expr_function
{ $$ = state->exprs.add<ExprWith>(CUR_POS, $2, $4); } { $$ = state->exprs.add<ExprWith>(CUR_POS, $2, $4); }
| LET binds IN_KW expr_function | LET binds IN_KW expr_function
{ if (!$2->dynamicAttrs->empty()) { if (!$2->dynamicAttrs.empty())
throw ParseError({ throw ParseError({
.msg = HintFmt("dynamic attributes not allowed in let"), .msg = HintFmt("dynamic attributes not allowed in let"),
.pos = state->positions[CUR_POS] .pos = state->positions[CUR_POS]
@@ -413,9 +413,9 @@ binds1
| binds[accum] INHERIT attrs ';' | binds[accum] INHERIT attrs ';'
{ $$ = $accum; { $$ = $accum;
for (auto & [i, iPos] : $attrs) { for (auto & [i, iPos] : $attrs) {
if ($accum->attrs->find(i.symbol) != $accum->attrs->end()) if ($accum->attrs.find(i.symbol) != $accum->attrs.end())
state->dupAttr(i.symbol, iPos, (*$accum->attrs)[i.symbol].pos); state->dupAttr(i.symbol, iPos, $accum->attrs[i.symbol].pos);
$accum->attrs->emplace( $accum->attrs.emplace(
i.symbol, i.symbol,
ExprAttrs::AttrDef(state->exprs.add<ExprVar>(iPos, i.symbol), iPos, ExprAttrs::AttrDef::Kind::Inherited)); ExprAttrs::AttrDef(state->exprs.add<ExprVar>(iPos, i.symbol), iPos, ExprAttrs::AttrDef::Kind::Inherited));
} }
@@ -423,13 +423,13 @@ binds1
| binds[accum] INHERIT '(' expr ')' attrs ';' | binds[accum] INHERIT '(' expr ')' attrs ';'
{ $$ = $accum; { $$ = $accum;
if (!$accum->inheritFromExprs) if (!$accum->inheritFromExprs)
$accum->inheritFromExprs = std::make_unique<std::pmr::vector<Expr *>>(); $accum->inheritFromExprs = std::make_unique<std::vector<Expr *>>();
$accum->inheritFromExprs->push_back($expr); $accum->inheritFromExprs->push_back($expr);
auto from = state->exprs.add<ExprInheritFrom>(state->at(@expr), $accum->inheritFromExprs->size() - 1); auto from = state->exprs.add<ExprInheritFrom>(state->at(@expr), $accum->inheritFromExprs->size() - 1);
for (auto & [i, iPos] : $attrs) { for (auto & [i, iPos] : $attrs) {
if ($accum->attrs->find(i.symbol) != $accum->attrs->end()) if ($accum->attrs.find(i.symbol) != $accum->attrs.end())
state->dupAttr(i.symbol, iPos, (*$accum->attrs)[i.symbol].pos); state->dupAttr(i.symbol, iPos, $accum->attrs[i.symbol].pos);
$accum->attrs->emplace( $accum->attrs.emplace(
i.symbol, i.symbol,
ExprAttrs::AttrDef( ExprAttrs::AttrDef(
state->exprs.add<ExprSelect>(state->exprs.alloc, iPos, from, i.symbol), state->exprs.add<ExprSelect>(state->exprs.alloc, iPos, from, i.symbol),
+1 -1
View File
@@ -81,7 +81,7 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value ** ar
attrs.insert_or_assign("rev", rev->gitRev()); attrs.insert_or_assign("rev", rev->gitRev());
auto input = fetchers::Input::fromAttrs(state.fetchSettings, std::move(attrs)); auto input = fetchers::Input::fromAttrs(state.fetchSettings, std::move(attrs));
auto [storePath, input2] = input.fetchToStore(state.fetchSettings, *state.store); auto [storePath, input2] = input.fetchToStore(state.store);
auto attrs2 = state.buildBindings(8); auto attrs2 = state.buildBindings(8);
state.mkStorePathString(storePath, attrs2.alloc(state.s.outPath)); state.mkStorePathString(storePath, attrs2.alloc(state.s.outPath));
+200 -99
View File
@@ -82,7 +82,7 @@ struct FetchTreeParams
static void fetchTree( static void fetchTree(
EvalState & state, const PosIdx pos, Value ** args, Value & v, const FetchTreeParams & params = FetchTreeParams{}) EvalState & state, const PosIdx pos, Value ** args, Value & v, const FetchTreeParams & params = FetchTreeParams{})
{ {
fetchers::Input input{}; fetchers::Input input{state.fetchSettings};
NixStringContext context; NixStringContext context;
std::optional<std::string> type; std::optional<std::string> type;
auto fetcher = params.isFetchGit ? "fetchGit" : "fetchTree"; auto fetcher = params.isFetchGit ? "fetchGit" : "fetchTree";
@@ -194,9 +194,9 @@ static void fetchTree(
} }
if (!state.settings.pureEval && !input.isDirect() && experimentalFeatureSettings.isEnabled(Xp::Flakes)) if (!state.settings.pureEval && !input.isDirect() && experimentalFeatureSettings.isEnabled(Xp::Flakes))
input = lookupInRegistries(state.fetchSettings, *state.store, input, fetchers::UseRegistries::Limited).first; input = lookupInRegistries(state.store, input, fetchers::UseRegistries::Limited).first;
if (state.settings.pureEval && !input.isLocked(state.fetchSettings)) { if (state.settings.pureEval && !input.isLocked()) {
if (input.getNarHash()) if (input.getNarHash())
warn( warn(
"Input '%s' is unlocked (e.g. lacks a Git revision) but is checked by NAR hash. " "Input '%s' is unlocked (e.g. lacks a Git revision) but is checked by NAR hash. "
@@ -219,8 +219,7 @@ static void fetchTree(
throw Error("input '%s' is not allowed to use the '__final' attribute", input.to_string()); throw Error("input '%s' is not allowed to use the '__final' attribute", input.to_string());
} }
auto cachedInput = auto cachedInput = state.inputCache->getAccessor(state.store, input, fetchers::UseRegistries::No);
state.inputCache->getAccessor(state.fetchSettings, *state.store, input, fetchers::UseRegistries::No);
auto storePath = state.mountInput(cachedInput.lockedInput, input, cachedInput.accessor); auto storePath = state.mountInput(cachedInput.lockedInput, input, cachedInput.accessor);
@@ -235,127 +234,229 @@ static void prim_fetchTree(EvalState & state, const PosIdx pos, Value ** args, V
static RegisterPrimOp primop_fetchTree({ static RegisterPrimOp primop_fetchTree({
.name = "fetchTree", .name = "fetchTree",
.args = {"input"}, .args = {"input"},
.doc = []() -> std::string { .doc = R"(
std::string doc = stripIndentation(R"( Fetch a file system tree or a plain file using one of the supported backends and return an attribute set with:
Fetch a file system tree or a plain file using one of the supported backends and return an attribute set with:
- the resulting fixed-output [store path](@docroot@/store/store-path.md) - the resulting fixed-output [store path](@docroot@/store/store-path.md)
- the corresponding [NAR](@docroot@/store/file-system-object/content-address.md#serial-nix-archive) hash - the corresponding [NAR](@docroot@/store/file-system-object/content-address.md#serial-nix-archive) hash
- backend-specific metadata (currently not documented). <!-- TODO: document output attributes --> - backend-specific metadata (currently not documented). <!-- TODO: document output attributes -->
*input* must be an attribute set with the following attributes: *input* must be an attribute set with the following attributes:
- `type` (String, required) - `type` (String, required)
One of the [supported source types](#source-types). One of the [supported source types](#source-types).
This determines other required and allowed input attributes. This determines other required and allowed input attributes.
- `narHash` (String, optional) - `narHash` (String, optional)
The `narHash` parameter can be used to substitute the source of the tree. The `narHash` parameter can be used to substitute the source of the tree.
It also allows for verification of tree contents that may not be provided by the underlying transfer mechanism. It also allows for verification of tree contents that may not be provided by the underlying transfer mechanism.
If `narHash` is set, the source is first looked up is the Nix store and [substituters](@docroot@/command-ref/conf-file.md#conf-substituters), and only fetched if not available. If `narHash` is set, the source is first looked up is the Nix store and [substituters](@docroot@/command-ref/conf-file.md#conf-substituters), and only fetched if not available.
A subset of the output attributes of `fetchTree` can be re-used for subsequent calls to `fetchTree` to produce the same result again. A subset of the output attributes of `fetchTree` can be re-used for subsequent calls to `fetchTree` to produce the same result again.
That is, `fetchTree` is idempotent. That is, `fetchTree` is idempotent.
Downloads are cached in `$XDG_CACHE_HOME/nix`. Downloads are cached in `$XDG_CACHE_HOME/nix`.
The remote source is fetched from the network if both are true: The remote source is fetched from the network if both are true:
- A NAR hash is supplied and the corresponding store path is not [valid](@docroot@/glossary.md#gloss-validity), that is, not available in the store - A NAR hash is supplied and the corresponding store path is not [valid](@docroot@/glossary.md#gloss-validity), that is, not available in the store
> **Note** > **Note**
>
> [Substituters](@docroot@/command-ref/conf-file.md#conf-substituters) are not used in fetching.
- There is no cache entry or the cache entry is older than [`tarball-ttl`](@docroot@/command-ref/conf-file.md#conf-tarball-ttl)
## Source types
The following source types and associated input attributes are supported.
<!-- TODO: It would be soooo much more predictable to work with (and
document) if `fetchTree` was a curried call with the first parameter for
`type` or an attribute like `builtins.fetchTree.git`! -->
- `"file"`
Place a plain file into the Nix store.
This is similar to [`builtins.fetchurl`](@docroot@/language/builtins.md#builtins-fetchurl)
- `url` (String, required)
Supported protocols:
- `https`
> **Example**
> >
> [Substituters](@docroot@/command-ref/conf-file.md#conf-substituters) are not used in fetching. > ```nix
> fetchTree {
> type = "file";
> url = "https://example.com/index.html";
> }
> ```
- There is no cache entry or the cache entry is older than [`tarball-ttl`](@docroot@/command-ref/conf-file.md#conf-tarball-ttl) - `http`
## Source types Insecure HTTP transfer for legacy sources.
The following source types and associated input attributes are supported. > **Warning**
>
> HTTP performs no encryption or authentication.
> Use a `narHash` known in advance to ensure the output has expected contents.
<!-- TODO: It would be soooo much more predictable to work with (and - `file`
document) if `fetchTree` was a curried call with the first parameter for
`type` or an attribute like `builtins.fetchTree.git`! -->
)");
auto indentString = [](std::string const & str, std::string const & indent) { A file on the local file system.
std::string result;
std::istringstream stream(str);
std::string line;
bool first = true;
while (std::getline(stream, line)) {
if (!first)
result += "\n";
result += indent + line;
first = false;
}
return result;
};
for (const auto & [schemeName, scheme] : fetchers::getAllInputSchemes()) { > **Example**
doc += "\n- `" + quoteString(schemeName, '"') + "`\n\n"; >
doc += indentString(scheme->schemeDescription(), " "); > ```nix
if (!doc.empty() && doc.back() != '\n') > fetchTree {
doc += "\n"; > type = "file";
> url = "file:///home/eelco/nix/README.md";
> }
> ```
for (const auto & [attrName, attribute] : scheme->allowedAttrs()) { - `"git"`
doc += "\n - `" + attrName + "` (" + attribute.type + ", "
+ (attribute.required ? "required" : "optional") + ")\n\n";
doc += indentString(stripIndentation(attribute.doc), " ");
if (!doc.empty() && doc.back() != '\n')
doc += "\n";
}
}
doc += "\n" + stripIndentation(R"( Fetch a Git tree and copy it to the Nix store.
The following input types are still subject to change: This is similar to [`builtins.fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit).
- `"path"` - `allRefs` (Bool, optional)
- `"github"`
- `"gitlab"`
- `"sourcehut"`
- `"mercurial"`
*input* can also be a [URL-like reference](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references). By default, this has no effect. This becomes relevant only once `shallow` cloning is disabled.
The additional input types and the URL-like syntax requires the [`flakes` experimental feature](@docroot@/development/experimental-features.md#xp-feature-flakes) to be enabled.
Whether to fetch all references (eg. branches and tags) of the repository.
With this argument being true, it's possible to load a `rev` from *any* `ref`.
(Without setting this option, only `rev`s from the specified `ref` are supported).
Default: `false`
- `lastModified` (Integer, optional)
Unix timestamp of the fetched commit.
If set, pass through the value to the output attribute set.
Otherwise, generated from the fetched Git tree.
- `lfs` (Bool, optional)
Fetch any [Git LFS](https://git-lfs.com/) files.
Default: `false`
- `ref` (String, optional)
By default, this has no effect. This becomes relevant only once `shallow` cloning is disabled.
A [Git reference](https://git-scm.com/book/en/v2/Git-Internals-Git-References), such as a branch or tag name.
Default: `"HEAD"`
- `rev` (String, optional)
A Git revision; a commit hash.
Default: the tip of `ref`
- `revCount` (Integer, optional)
Number of revisions in the history of the Git repository before the fetched commit.
If set, pass through the value to the output attribute set.
Otherwise, generated from the fetched Git tree.
- `shallow` (Bool, optional)
Make a shallow clone when fetching the Git tree.
When this is enabled, the options `ref` and `allRefs` have no effect anymore.
Default: `true`
- `submodules` (Bool, optional)
Also fetch submodules if available.
Default: `false`
- `url` (String, required)
The URL formats supported are the same as for Git itself.
> **Example** > **Example**
> >
> Fetch a GitHub repository using the attribute set representation:
>
> ```nix > ```nix
> builtins.fetchTree { > fetchTree {
> type = "github"; > type = "git";
> owner = "NixOS"; > url = "git@github.com:NixOS/nixpkgs.git";
> repo = "nixpkgs";
> rev = "ae2e6b3958682513d28f7d633734571fb18285dd";
> }
> ```
>
> This evaluates to the following attribute set:
>
> ```nix
> {
> lastModified = 1686503798;
> lastModifiedDate = "20230611171638";
> narHash = "sha256-rA9RqKP9OlBrgGCPvfd5HVAXDOy8k2SmPtB/ijShNXc=";
> outPath = "/nix/store/l5m6qlvfs9sdw14ja3qbzpglcjlb6j1x-source";
> rev = "ae2e6b3958682513d28f7d633734571fb18285dd";
> shortRev = "ae2e6b3";
> } > }
> ``` > ```
> **Example** > **Note**
> >
> Fetch the same GitHub repository using the URL-like syntax: > If the URL points to a local directory, and no `ref` or `rev` is given, Nix only considers files added to the Git index, as listed by `git ls-files` but use the *current file contents* of the Git working directory.
>
> ```nix
> builtins.fetchTree "github:NixOS/nixpkgs/ae2e6b3958682513d28f7d633734571fb18285dd"
> ```
)");
return doc; - `"tarball"`
}(),
Download a tar archive and extract it into the Nix store.
This has the same underlying implementation as [`builtins.fetchTarball`](@docroot@/language/builtins.md#builtins-fetchTarball)
- `url` (String, required)
> **Example**
>
> ```nix
> fetchTree {
> type = "tarball";
> url = "https://github.com/NixOS/nixpkgs/tarball/nixpkgs-23.11";
> }
> ```
The following input types are still subject to change:
- `"path"`
- `"github"`
- `"gitlab"`
- `"sourcehut"`
- `"mercurial"`
*input* can also be a [URL-like reference](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references).
The additional input types and the URL-like syntax requires the [`flakes` experimental feature](@docroot@/development/experimental-features.md#xp-feature-flakes) to be enabled.
> **Example**
>
> Fetch a GitHub repository using the attribute set representation:
>
> ```nix
> builtins.fetchTree {
> type = "github";
> owner = "NixOS";
> repo = "nixpkgs";
> rev = "ae2e6b3958682513d28f7d633734571fb18285dd";
> }
> ```
>
> This evaluates to the following attribute set:
>
> ```nix
> {
> lastModified = 1686503798;
> lastModifiedDate = "20230611171638";
> narHash = "sha256-rA9RqKP9OlBrgGCPvfd5HVAXDOy8k2SmPtB/ijShNXc=";
> outPath = "/nix/store/l5m6qlvfs9sdw14ja3qbzpglcjlb6j1x-source";
> rev = "ae2e6b3958682513d28f7d633734571fb18285dd";
> shortRev = "ae2e6b3";
> }
> ```
> **Example**
>
> Fetch the same GitHub repository using the URL-like syntax:
>
> ```nix
> builtins.fetchTree "github:NixOS/nixpkgs/ae2e6b3958682513d28f7d633734571fb18285dd"
> ```
)",
.fun = prim_fetchTree, .fun = prim_fetchTree,
.experimentalFeature = Xp::FetchTree, .experimentalFeature = Xp::FetchTree,
}); });
@@ -480,10 +581,10 @@ static void fetch(
auto storePath = unpack ? fetchToStore( auto storePath = unpack ? fetchToStore(
state.fetchSettings, state.fetchSettings,
*state.store, *state.store,
fetchers::downloadTarball(*state.store, state.fetchSettings, *url), fetchers::downloadTarball(state.store, state.fetchSettings, *url),
FetchMode::Copy, FetchMode::Copy,
name) name)
: fetchers::downloadFile(*state.store, state.fetchSettings, *url, name).storePath; : fetchers::downloadFile(state.store, state.fetchSettings, *url, name).storePath;
if (expectedHash) { if (expectedHash) {
auto hash = unpack ? state.store->queryPathInfo(storePath)->narHash auto hash = unpack ? state.store->queryPathInfo(storePath)->narHash
+1 -1
View File
@@ -196,7 +196,7 @@ TEST_F(GitTest, submodulePeriodSupport)
{"ref", "main"}, {"ref", "main"},
}); });
auto [accessor, i] = input.getAccessor(settings, *store); auto [accessor, i] = input.getAccessor(store);
ASSERT_EQ(accessor->readFile(CanonPath("deps/sub/lib.txt")), "hello from submodule\n"); ASSERT_EQ(accessor->readFile(CanonPath("deps/sub/lib.txt")), "hello from submodule\n");
} }
+32 -34
View File
@@ -6,7 +6,6 @@
#include "nix/fetchers/fetch-settings.hh" #include "nix/fetchers/fetch-settings.hh"
#include "nix/fetchers/fetch-to-store.hh" #include "nix/fetchers/fetch-to-store.hh"
#include "nix/util/url.hh" #include "nix/util/url.hh"
#include "nix/util/archive.hh"
#include <nlohmann/json.hpp> #include <nlohmann/json.hpp>
@@ -27,9 +26,18 @@ void registerInputScheme(std::shared_ptr<InputScheme> && inputScheme)
throw Error("Input scheme with name %s already registered", schemeName); throw Error("Input scheme with name %s already registered", schemeName);
} }
const InputSchemeMap & getAllInputSchemes() nlohmann::json dumpRegisterInputSchemeInfo()
{ {
return inputSchemes(); using nlohmann::json;
auto res = json::object();
for (auto & [name, scheme] : inputSchemes()) {
auto & r = res[name] = json::object();
r["allowedAttrs"] = scheme->allowedAttrs();
}
return res;
} }
Input Input::fromURL(const Settings & settings, const std::string & url, bool requireTree) Input Input::fromURL(const Settings & settings, const std::string & url, bool requireTree)
@@ -81,7 +89,7 @@ Input Input::fromAttrs(const Settings & settings, Attrs && attrs)
// but not all of them. Doing this is to support those other // but not all of them. Doing this is to support those other
// operations which are supposed to be robust on // operations which are supposed to be robust on
// unknown/uninterpretable inputs. // unknown/uninterpretable inputs.
Input input; Input input{settings};
input.attrs = attrs; input.attrs = attrs;
fixupInput(input); fixupInput(input);
return input; return input;
@@ -111,7 +119,7 @@ Input Input::fromAttrs(const Settings & settings, Attrs && attrs)
return std::move(*res); return std::move(*res);
} }
std::optional<std::string> Input::getFingerprint(Store & store) const std::optional<std::string> Input::getFingerprint(ref<Store> store) const
{ {
if (!scheme) if (!scheme)
return std::nullopt; return std::nullopt;
@@ -151,9 +159,9 @@ bool Input::isDirect() const
return !scheme || scheme->isDirect(*this); return !scheme || scheme->isDirect(*this);
} }
bool Input::isLocked(const Settings & settings) const bool Input::isLocked() const
{ {
return scheme && scheme->isLocked(settings, *this); return scheme && scheme->isLocked(*this);
} }
bool Input::isFinal() const bool Input::isFinal() const
@@ -190,19 +198,19 @@ bool Input::contains(const Input & other) const
} }
// FIXME: remove // FIXME: remove
std::pair<StorePath, Input> Input::fetchToStore(const Settings & settings, Store & store) const std::pair<StorePath, Input> Input::fetchToStore(ref<Store> store) const
{ {
if (!scheme) if (!scheme)
throw Error("cannot fetch unsupported input '%s'", attrsToJSON(toAttrs())); throw Error("cannot fetch unsupported input '%s'", attrsToJSON(toAttrs()));
auto [storePath, input] = [&]() -> std::pair<StorePath, Input> { auto [storePath, input] = [&]() -> std::pair<StorePath, Input> {
try { try {
auto [accessor, result] = getAccessorUnchecked(settings, store); auto [accessor, result] = getAccessorUnchecked(store);
auto storePath = auto storePath =
nix::fetchToStore(settings, store, SourcePath(accessor), FetchMode::Copy, result.getName()); nix::fetchToStore(*settings, *store, SourcePath(accessor), FetchMode::Copy, result.getName());
auto narHash = store.queryPathInfo(storePath)->narHash; auto narHash = store->queryPathInfo(storePath)->narHash;
result.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true)); result.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true));
result.attrs.insert_or_assign("__final", Explicit<bool>(true)); result.attrs.insert_or_assign("__final", Explicit<bool>(true));
@@ -289,10 +297,10 @@ void Input::checkLocks(Input specified, Input & result)
} }
} }
std::pair<ref<SourceAccessor>, Input> Input::getAccessor(const Settings & settings, Store & store) const std::pair<ref<SourceAccessor>, Input> Input::getAccessor(ref<Store> store) const
{ {
try { try {
auto [accessor, result] = getAccessorUnchecked(settings, store); auto [accessor, result] = getAccessorUnchecked(store);
result.attrs.insert_or_assign("__final", Explicit<bool>(true)); result.attrs.insert_or_assign("__final", Explicit<bool>(true));
@@ -305,7 +313,7 @@ std::pair<ref<SourceAccessor>, Input> Input::getAccessor(const Settings & settin
} }
} }
std::pair<ref<SourceAccessor>, Input> Input::getAccessorUnchecked(const Settings & settings, Store & store) const std::pair<ref<SourceAccessor>, Input> Input::getAccessorUnchecked(ref<Store> store) const
{ {
// FIXME: cache the accessor // FIXME: cache the accessor
@@ -325,13 +333,13 @@ std::pair<ref<SourceAccessor>, Input> Input::getAccessorUnchecked(const Settings
*/ */
if (isFinal() && getNarHash()) { if (isFinal() && getNarHash()) {
try { try {
auto storePath = computeStorePath(store); auto storePath = computeStorePath(*store);
store.ensurePath(storePath); store->ensurePath(storePath);
debug("using substituted/cached input '%s' in '%s'", to_string(), store.printStorePath(storePath)); debug("using substituted/cached input '%s' in '%s'", to_string(), store->printStorePath(storePath));
auto accessor = store.requireStoreObjectAccessor(storePath); auto accessor = store->requireStoreObjectAccessor(storePath);
accessor->fingerprint = getFingerprint(store); accessor->fingerprint = getFingerprint(store);
@@ -341,7 +349,7 @@ std::pair<ref<SourceAccessor>, Input> Input::getAccessorUnchecked(const Settings
if (accessor->fingerprint) { if (accessor->fingerprint) {
ContentAddressMethod method = ContentAddressMethod::Raw::NixArchive; ContentAddressMethod method = ContentAddressMethod::Raw::NixArchive;
auto cacheKey = makeFetchToStoreCacheKey(getName(), *accessor->fingerprint, method, "/"); auto cacheKey = makeFetchToStoreCacheKey(getName(), *accessor->fingerprint, method, "/");
settings.getCache()->upsert(cacheKey, store, {}, storePath); settings->getCache()->upsert(cacheKey, *store, {}, storePath);
} }
accessor->setPathDisplay("«" + to_string() + "»"); accessor->setPathDisplay("«" + to_string() + "»");
@@ -352,7 +360,7 @@ std::pair<ref<SourceAccessor>, Input> Input::getAccessorUnchecked(const Settings
} }
} }
auto [accessor, result] = scheme->getAccessor(settings, store, *this); auto [accessor, result] = scheme->getAccessor(store, *this);
if (!accessor->fingerprint) if (!accessor->fingerprint)
accessor->fingerprint = result.getFingerprint(store); accessor->fingerprint = result.getFingerprint(store);
@@ -369,10 +377,10 @@ Input Input::applyOverrides(std::optional<std::string> ref, std::optional<Hash>
return scheme->applyOverrides(*this, ref, rev); return scheme->applyOverrides(*this, ref, rev);
} }
void Input::clone(const Settings & settings, Store & store, const std::filesystem::path & destDir) const void Input::clone(const Path & destDir) const
{ {
assert(scheme); assert(scheme);
scheme->clone(settings, store, *this, destDir); scheme->clone(*this, destDir);
} }
std::optional<std::filesystem::path> Input::getSourcePath() const std::optional<std::filesystem::path> Input::getSourcePath() const
@@ -485,19 +493,9 @@ void InputScheme::putFile(
throw Error("input '%s' does not support modifying file '%s'", input.to_string(), path); throw Error("input '%s' does not support modifying file '%s'", input.to_string(), path);
} }
void InputScheme::clone( void InputScheme::clone(const Input & input, const Path & destDir) const
const Settings & settings, Store & store, const Input & input, const std::filesystem::path & destDir) const
{ {
if (std::filesystem::exists(destDir)) throw Error("do not know how to clone input '%s'", input.to_string());
throw Error("cannot clone into existing path %s", destDir);
auto [accessor, input2] = getAccessor(settings, store, input);
Activity act(*logger, lvlTalkative, actUnknown, fmt("copying '%s' to %s...", input2.to_string(), destDir));
auto source = sinkToSource([&](Sink & sink) { accessor->dumpPath(CanonPath::root, sink); });
restorePath(destDir, *source);
} }
std::optional<ExperimentalFeature> InputScheme::experimentalFeature() const std::optional<ExperimentalFeature> InputScheme::experimentalFeature() const
+1 -1
View File
@@ -209,7 +209,7 @@ std::vector<nlohmann::json> Fetch::fetchUrls(const std::vector<Pointer> & pointe
auto url = api.endpoint + "/objects/batch"; auto url = api.endpoint + "/objects/batch";
const auto & authHeader = api.authHeader; const auto & authHeader = api.authHeader;
FileTransferRequest request(parseURL(url)); FileTransferRequest request(parseURL(url));
request.method = HttpMethod::Post; request.method = HttpMethod::POST;
Headers headers; Headers headers;
if (authHeader.has_value()) if (authHeader.has_value())
headers.push_back({"Authorization", *authHeader}); headers.push_back({"Authorization", *authHeader});
+40 -196
View File
@@ -194,183 +194,28 @@ struct GitInputScheme : InputScheme
return "git"; return "git";
} }
std::string schemeDescription() const override StringSet allowedAttrs() const override
{ {
return stripIndentation(R"( return {
Fetch a Git tree and copy it to the Nix store. "url",
This is similar to [`builtins.fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit). "ref",
)"); "rev",
} "shallow",
"submodules",
const std::map<std::string, AttributeInfo> & allowedAttrs() const override "lfs",
{ "exportIgnore",
static const std::map<std::string, AttributeInfo> attrs = { "lastModified",
{ "revCount",
"url", "narHash",
{ "allRefs",
.type = "String", "name",
.required = true, "dirtyRev",
.doc = R"( "dirtyShortRev",
The URL formats supported are the same as for Git itself. "verifyCommit",
"keytype",
> **Example** "publicKey",
> "publicKeys",
> ```nix
> fetchTree {
> type = "git";
> url = "git@github.com:NixOS/nixpkgs.git";
> }
> ```
> **Note**
>
> If the URL points to a local directory, and no `ref` or `rev` is given, Nix only considers files added to the Git index, as listed by `git ls-files` but uses the *current file contents* of the Git working directory.
)",
},
},
{
"ref",
{
.type = "String",
.required = false,
.doc = R"(
By default, this has no effect. This becomes relevant only once `shallow` cloning is disabled.
A [Git reference](https://git-scm.com/book/en/v2/Git-Internals-Git-References), such as a branch or tag name.
Default: `"HEAD"`
)",
},
},
{
"rev",
{
.type = "String",
.required = false,
.doc = R"(
A Git revision; a commit hash.
Default: the tip of `ref`
)",
},
},
{
"shallow",
{
.type = "Bool",
.required = false,
.doc = R"(
Make a shallow clone when fetching the Git tree.
When this is enabled, the options `ref` and `allRefs` have no effect anymore.
Default: `true`
)",
},
},
{
"submodules",
{
.type = "Bool",
.required = false,
.doc = R"(
Also fetch submodules if available.
Default: `false`
)",
},
},
{
"lfs",
{
.type = "Bool",
.required = false,
.doc = R"(
Fetch any [Git LFS](https://git-lfs.com/) files.
Default: `false`
)",
},
},
{
"exportIgnore",
{},
},
{
"lastModified",
{
.type = "Integer",
.required = false,
.doc = R"(
Unix timestamp of the fetched commit.
If set, pass through the value to the output attribute set.
Otherwise, generated from the fetched Git tree.
)",
},
},
{
"revCount",
{
.type = "Integer",
.required = false,
.doc = R"(
Number of revisions in the history of the Git repository before the fetched commit.
If set, pass through the value to the output attribute set.
Otherwise, generated from the fetched Git tree.
)",
},
},
{
"narHash",
{},
},
{
"allRefs",
{
.type = "Bool",
.required = false,
.doc = R"(
By default, this has no effect. This becomes relevant only once `shallow` cloning is disabled.
Whether to fetch all references (eg. branches and tags) of the repository.
With this argument being true, it's possible to load a `rev` from *any* `ref`.
(Without setting this option, only `rev`s from the specified `ref` are supported).
Default: `false`
)",
},
},
{
"name",
{},
},
{
"dirtyRev",
{},
},
{
"dirtyShortRev",
{},
},
{
"verifyCommit",
{},
},
{
"keytype",
{},
},
{
"publicKey",
{},
},
{
"publicKeys",
{},
},
}; };
return attrs;
} }
std::optional<Input> inputFromAttrs(const Settings & settings, const Attrs & attrs) const override std::optional<Input> inputFromAttrs(const Settings & settings, const Attrs & attrs) const override
@@ -384,7 +229,7 @@ struct GitInputScheme : InputScheme
if (auto ref = maybeGetStrAttr(attrs, "ref"); ref && !isLegalRefName(*ref)) if (auto ref = maybeGetStrAttr(attrs, "ref"); ref && !isLegalRefName(*ref))
throw BadURL("invalid Git branch/tag name '%s'", *ref); throw BadURL("invalid Git branch/tag name '%s'", *ref);
Input input{}; Input input{settings};
input.attrs = attrs; input.attrs = attrs;
input.attrs["url"] = fixGitURL(getStrAttr(attrs, "url")).to_string(); input.attrs["url"] = fixGitURL(getStrAttr(attrs, "url")).to_string();
getShallowAttr(input); getShallowAttr(input);
@@ -433,8 +278,7 @@ struct GitInputScheme : InputScheme
return res; return res;
} }
void clone(const Settings & settings, Store & store, const Input & input, const std::filesystem::path & destDir) void clone(const Input & input, const Path & destDir) const override
const override
{ {
auto repoInfo = getRepoInfo(input); auto repoInfo = getRepoInfo(input);
@@ -779,7 +623,7 @@ struct GitInputScheme : InputScheme
} }
std::pair<ref<SourceAccessor>, Input> std::pair<ref<SourceAccessor>, Input>
getAccessorFromCommit(const Settings & settings, Store & store, RepoInfo & repoInfo, Input && input) const getAccessorFromCommit(ref<Store> store, RepoInfo & repoInfo, Input && input) const
{ {
assert(!repoInfo.workdirInfo.isDirty); assert(!repoInfo.workdirInfo.isDirty);
@@ -889,10 +733,10 @@ struct GitInputScheme : InputScheme
auto rev = *input.getRev(); auto rev = *input.getRev();
input.attrs.insert_or_assign("lastModified", getLastModified(settings, repoInfo, repoDir, rev)); input.attrs.insert_or_assign("lastModified", getLastModified(*input.settings, repoInfo, repoDir, rev));
if (!getShallowAttr(input)) if (!getShallowAttr(input))
input.attrs.insert_or_assign("revCount", getRevCount(settings, repoInfo, repoDir, rev)); input.attrs.insert_or_assign("revCount", getRevCount(*input.settings, repoInfo, repoDir, rev));
printTalkative("using revision %s of repo '%s'", rev.gitRev(), repoInfo.locationToArg()); printTalkative("using revision %s of repo '%s'", rev.gitRev(), repoInfo.locationToArg());
@@ -935,8 +779,8 @@ struct GitInputScheme : InputScheme
attrs.insert_or_assign("submodules", Explicit<bool>{true}); attrs.insert_or_assign("submodules", Explicit<bool>{true});
attrs.insert_or_assign("lfs", Explicit<bool>{smudgeLfs}); attrs.insert_or_assign("lfs", Explicit<bool>{smudgeLfs});
attrs.insert_or_assign("allRefs", Explicit<bool>{true}); attrs.insert_or_assign("allRefs", Explicit<bool>{true});
auto submoduleInput = fetchers::Input::fromAttrs(settings, std::move(attrs)); auto submoduleInput = fetchers::Input::fromAttrs(*input.settings, std::move(attrs));
auto [submoduleAccessor, submoduleInput2] = submoduleInput.getAccessor(settings, store); auto [submoduleAccessor, submoduleInput2] = submoduleInput.getAccessor(store);
submoduleAccessor->setPathDisplay("«" + submoduleInput.to_string() + "»"); submoduleAccessor->setPathDisplay("«" + submoduleInput.to_string() + "»");
mounts.insert_or_assign(submodule.path, submoduleAccessor); mounts.insert_or_assign(submodule.path, submoduleAccessor);
} }
@@ -953,7 +797,7 @@ struct GitInputScheme : InputScheme
} }
std::pair<ref<SourceAccessor>, Input> std::pair<ref<SourceAccessor>, Input>
getAccessorFromWorkdir(const Settings & settings, Store & store, RepoInfo & repoInfo, Input && input) const getAccessorFromWorkdir(ref<Store> store, RepoInfo & repoInfo, Input && input) const
{ {
auto repoPath = repoInfo.getPath().value(); auto repoPath = repoInfo.getPath().value();
@@ -985,8 +829,8 @@ struct GitInputScheme : InputScheme
// TODO: fall back to getAccessorFromCommit-like fetch when submodules aren't checked out // TODO: fall back to getAccessorFromCommit-like fetch when submodules aren't checked out
// attrs.insert_or_assign("allRefs", Explicit<bool>{ true }); // attrs.insert_or_assign("allRefs", Explicit<bool>{ true });
auto submoduleInput = fetchers::Input::fromAttrs(settings, std::move(attrs)); auto submoduleInput = fetchers::Input::fromAttrs(*input.settings, std::move(attrs));
auto [submoduleAccessor, submoduleInput2] = submoduleInput.getAccessor(settings, store); auto [submoduleAccessor, submoduleInput2] = submoduleInput.getAccessor(store);
submoduleAccessor->setPathDisplay("«" + submoduleInput.to_string() + "»"); submoduleAccessor->setPathDisplay("«" + submoduleInput.to_string() + "»");
/* If the submodule is dirty, mark this repo dirty as /* If the submodule is dirty, mark this repo dirty as
@@ -1013,12 +857,12 @@ struct GitInputScheme : InputScheme
input.attrs.insert_or_assign("rev", rev.gitRev()); input.attrs.insert_or_assign("rev", rev.gitRev());
if (!getShallowAttr(input)) { if (!getShallowAttr(input)) {
input.attrs.insert_or_assign( input.attrs.insert_or_assign(
"revCount", rev == nullRev ? 0 : getRevCount(settings, repoInfo, repoPath, rev)); "revCount", rev == nullRev ? 0 : getRevCount(*input.settings, repoInfo, repoPath, rev));
} }
verifyCommit(input, repo); verifyCommit(input, repo);
} else { } else {
repoInfo.warnDirty(settings); repoInfo.warnDirty(*input.settings);
if (repoInfo.workdirInfo.headRev) { if (repoInfo.workdirInfo.headRev) {
input.attrs.insert_or_assign("dirtyRev", repoInfo.workdirInfo.headRev->gitRev() + "-dirty"); input.attrs.insert_or_assign("dirtyRev", repoInfo.workdirInfo.headRev->gitRev() + "-dirty");
@@ -1030,14 +874,14 @@ struct GitInputScheme : InputScheme
input.attrs.insert_or_assign( input.attrs.insert_or_assign(
"lastModified", "lastModified",
repoInfo.workdirInfo.headRev ? getLastModified(settings, repoInfo, repoPath, *repoInfo.workdirInfo.headRev) repoInfo.workdirInfo.headRev
: 0); ? getLastModified(*input.settings, repoInfo, repoPath, *repoInfo.workdirInfo.headRev)
: 0);
return {accessor, std::move(input)}; return {accessor, std::move(input)};
} }
std::pair<ref<SourceAccessor>, Input> std::pair<ref<SourceAccessor>, Input> getAccessor(ref<Store> store, const Input & _input) const override
getAccessor(const Settings & settings, Store & store, const Input & _input) const override
{ {
Input input(_input); Input input(_input);
@@ -1053,13 +897,13 @@ struct GitInputScheme : InputScheme
} }
auto [accessor, final] = input.getRef() || input.getRev() || !repoInfo.getPath() auto [accessor, final] = input.getRef() || input.getRev() || !repoInfo.getPath()
? getAccessorFromCommit(settings, store, repoInfo, std::move(input)) ? getAccessorFromCommit(store, repoInfo, std::move(input))
: getAccessorFromWorkdir(settings, store, repoInfo, std::move(input)); : getAccessorFromWorkdir(store, repoInfo, std::move(input));
return {accessor, std::move(final)}; return {accessor, std::move(final)};
} }
std::optional<std::string> getFingerprint(Store & store, const Input & input) const override std::optional<std::string> getFingerprint(ref<Store> store, const Input & input) const override
{ {
auto makeFingerprint = [&](const Hash & rev) { auto makeFingerprint = [&](const Hash & rev) {
return rev.gitRev() + (getSubmodulesAttr(input) ? ";s" : "") + (getExportIgnoreAttr(input) ? ";e" : "") return rev.gitRev() + (getSubmodulesAttr(input) ? ";s" : "") + (getExportIgnoreAttr(input) ? ";e" : "")
@@ -1090,7 +934,7 @@ struct GitInputScheme : InputScheme
} }
} }
bool isLocked(const Settings & settings, const Input & input) const override bool isLocked(const Input & input) const override
{ {
auto rev = input.getRev(); auto rev = input.getRev();
return rev && rev != nullRev; return rev && rev != nullRev;
+57 -102
View File
@@ -92,7 +92,7 @@ struct GitArchiveInputScheme : InputScheme
if (ref && rev) if (ref && rev)
throw BadURL("URL '%s' contains both a commit hash and a branch/tag name %s %s", url, *ref, rev->gitRev()); throw BadURL("URL '%s' contains both a commit hash and a branch/tag name %s %s", url, *ref, rev->gitRev());
Input input{}; Input input{settings};
input.attrs.insert_or_assign("type", std::string{schemeName()}); input.attrs.insert_or_assign("type", std::string{schemeName()});
input.attrs.insert_or_assign("owner", path[0]); input.attrs.insert_or_assign("owner", path[0]);
input.attrs.insert_or_assign("repo", path[1]); input.attrs.insert_or_assign("repo", path[1]);
@@ -110,43 +110,18 @@ struct GitArchiveInputScheme : InputScheme
return input; return input;
} }
const std::map<std::string, AttributeInfo> & allowedAttrs() const override StringSet allowedAttrs() const override
{ {
static const std::map<std::string, AttributeInfo> attrs = { return {
{ "owner",
"owner", "repo",
{}, "ref",
}, "rev",
{ "narHash",
"repo", "lastModified",
{}, "host",
}, "treeHash",
{
"ref",
{},
},
{
"rev",
{},
},
{
"narHash",
{},
},
{
"lastModified",
{},
},
{
"host",
{},
},
{
"treeHash",
{},
},
}; };
return attrs;
} }
std::optional<Input> inputFromAttrs(const fetchers::Settings & settings, const Attrs & attrs) const override std::optional<Input> inputFromAttrs(const fetchers::Settings & settings, const Attrs & attrs) const override
@@ -154,7 +129,7 @@ struct GitArchiveInputScheme : InputScheme
getStrAttr(attrs, "owner"); getStrAttr(attrs, "owner");
getStrAttr(attrs, "repo"); getStrAttr(attrs, "repo");
Input input{}; Input input{settings};
input.attrs = attrs; input.attrs = attrs;
return input; return input;
} }
@@ -258,9 +233,9 @@ struct GitArchiveInputScheme : InputScheme
std::optional<Hash> treeHash; std::optional<Hash> treeHash;
}; };
virtual RefInfo getRevFromRef(const Settings & settings, nix::Store & store, const Input & input) const = 0; virtual RefInfo getRevFromRef(nix::ref<Store> store, const Input & input) const = 0;
virtual DownloadUrl getDownloadUrl(const Settings & settings, const Input & input) const = 0; virtual DownloadUrl getDownloadUrl(const Input & input) const = 0;
struct TarballInfo struct TarballInfo
{ {
@@ -268,7 +243,7 @@ struct GitArchiveInputScheme : InputScheme
time_t lastModified; time_t lastModified;
}; };
std::pair<Input, TarballInfo> downloadArchive(const Settings & settings, Store & store, Input input) const std::pair<Input, TarballInfo> downloadArchive(ref<Store> store, Input input) const
{ {
if (!maybeGetStrAttr(input.attrs, "ref")) if (!maybeGetStrAttr(input.attrs, "ref"))
input.attrs.insert_or_assign("ref", "HEAD"); input.attrs.insert_or_assign("ref", "HEAD");
@@ -277,7 +252,7 @@ struct GitArchiveInputScheme : InputScheme
auto rev = input.getRev(); auto rev = input.getRev();
if (!rev) { if (!rev) {
auto refInfo = getRevFromRef(settings, store, input); auto refInfo = getRevFromRef(store, input);
rev = refInfo.rev; rev = refInfo.rev;
upstreamTreeHash = refInfo.treeHash; upstreamTreeHash = refInfo.treeHash;
debug("HEAD revision for '%s' is %s", input.to_string(), refInfo.rev.gitRev()); debug("HEAD revision for '%s' is %s", input.to_string(), refInfo.rev.gitRev());
@@ -286,7 +261,7 @@ struct GitArchiveInputScheme : InputScheme
input.attrs.erase("ref"); input.attrs.erase("ref");
input.attrs.insert_or_assign("rev", rev->gitRev()); input.attrs.insert_or_assign("rev", rev->gitRev());
auto cache = settings.getCache(); auto cache = input.settings->getCache();
Cache::Key treeHashKey{"gitRevToTreeHash", {{"rev", rev->gitRev()}}}; Cache::Key treeHashKey{"gitRevToTreeHash", {{"rev", rev->gitRev()}}};
Cache::Key lastModifiedKey{"gitRevToLastModified", {{"rev", rev->gitRev()}}}; Cache::Key lastModifiedKey{"gitRevToLastModified", {{"rev", rev->gitRev()}}};
@@ -295,7 +270,7 @@ struct GitArchiveInputScheme : InputScheme
if (auto lastModifiedAttrs = cache->lookup(lastModifiedKey)) { if (auto lastModifiedAttrs = cache->lookup(lastModifiedKey)) {
auto treeHash = getRevAttr(*treeHashAttrs, "treeHash"); auto treeHash = getRevAttr(*treeHashAttrs, "treeHash");
auto lastModified = getIntAttr(*lastModifiedAttrs, "lastModified"); auto lastModified = getIntAttr(*lastModifiedAttrs, "lastModified");
if (settings.getTarballCache()->hasObject(treeHash)) if (input.settings->getTarballCache()->hasObject(treeHash))
return {std::move(input), TarballInfo{.treeHash = treeHash, .lastModified = (time_t) lastModified}}; return {std::move(input), TarballInfo{.treeHash = treeHash, .lastModified = (time_t) lastModified}};
else else
debug("Git tree with hash '%s' has disappeared from the cache, refetching...", treeHash.gitRev()); debug("Git tree with hash '%s' has disappeared from the cache, refetching...", treeHash.gitRev());
@@ -303,7 +278,7 @@ struct GitArchiveInputScheme : InputScheme
} }
/* Stream the tarball into the tarball cache. */ /* Stream the tarball into the tarball cache. */
auto url = getDownloadUrl(settings, input); auto url = getDownloadUrl(input);
auto source = sinkToSource([&](Sink & sink) { auto source = sinkToSource([&](Sink & sink) {
FileTransferRequest req(url.url); FileTransferRequest req(url.url);
@@ -315,7 +290,7 @@ struct GitArchiveInputScheme : InputScheme
*logger, lvlInfo, actUnknown, fmt("unpacking '%s' into the Git cache", input.to_string())); *logger, lvlInfo, actUnknown, fmt("unpacking '%s' into the Git cache", input.to_string()));
TarArchive archive{*source}; TarArchive archive{*source};
auto tarballCache = settings.getTarballCache(); auto tarballCache = input.settings->getTarballCache();
auto parseSink = tarballCache->getFileSystemObjectSink(); auto parseSink = tarballCache->getFileSystemObjectSink();
auto lastModified = unpackTarfileToSink(archive, *parseSink); auto lastModified = unpackTarfileToSink(archive, *parseSink);
auto tree = parseSink->flush(); auto tree = parseSink->flush();
@@ -340,10 +315,9 @@ struct GitArchiveInputScheme : InputScheme
return {std::move(input), tarballInfo}; return {std::move(input), tarballInfo};
} }
std::pair<ref<SourceAccessor>, Input> std::pair<ref<SourceAccessor>, Input> getAccessor(ref<Store> store, const Input & _input) const override
getAccessor(const Settings & settings, Store & store, const Input & _input) const override
{ {
auto [input, tarballInfo] = downloadArchive(settings, store, _input); auto [input, tarballInfo] = downloadArchive(store, _input);
#if 0 #if 0
input.attrs.insert_or_assign("treeHash", tarballInfo.treeHash.gitRev()); input.attrs.insert_or_assign("treeHash", tarballInfo.treeHash.gitRev());
@@ -351,18 +325,19 @@ struct GitArchiveInputScheme : InputScheme
input.attrs.insert_or_assign("lastModified", uint64_t(tarballInfo.lastModified)); input.attrs.insert_or_assign("lastModified", uint64_t(tarballInfo.lastModified));
auto accessor = auto accessor =
settings.getTarballCache()->getAccessor(tarballInfo.treeHash, false, "«" + input.to_string() + "»"); input.settings->getTarballCache()->getAccessor(tarballInfo.treeHash, false, "«" + input.to_string() + "»");
return {accessor, input}; return {accessor, input};
} }
bool isLocked(const Settings & settings, const Input & input) const override bool isLocked(const Input & input) const override
{ {
/* Since we can't verify the integrity of the tarball from the /* Since we can't verify the integrity of the tarball from the
Git revision alone, we also require a NAR hash for Git revision alone, we also require a NAR hash for
locking. FIXME: in the future, we may want to require a Git locking. FIXME: in the future, we may want to require a Git
tree hash instead of a NAR hash. */ tree hash instead of a NAR hash. */
return input.getRev().has_value() && (settings.trustTarballsFromGitForges || input.getNarHash().has_value()); return input.getRev().has_value()
&& (input.settings->trustTarballsFromGitForges || input.getNarHash().has_value());
} }
std::optional<ExperimentalFeature> experimentalFeature() const override std::optional<ExperimentalFeature> experimentalFeature() const override
@@ -370,7 +345,7 @@ struct GitArchiveInputScheme : InputScheme
return Xp::Flakes; return Xp::Flakes;
} }
std::optional<std::string> getFingerprint(Store & store, const Input & input) const override std::optional<std::string> getFingerprint(ref<Store> store, const Input & input) const override
{ {
if (auto rev = input.getRev()) if (auto rev = input.getRev())
return rev->gitRev(); return rev->gitRev();
@@ -386,12 +361,6 @@ struct GitHubInputScheme : GitArchiveInputScheme
return "github"; return "github";
} }
std::string schemeDescription() const override
{
// TODO
return "";
}
std::optional<std::pair<std::string, std::string>> accessHeaderFromToken(const std::string & token) const override std::optional<std::pair<std::string, std::string>> accessHeaderFromToken(const std::string & token) const override
{ {
// Github supports PAT/OAuth2 tokens and HTTP Basic // Github supports PAT/OAuth2 tokens and HTTP Basic
@@ -418,7 +387,7 @@ struct GitHubInputScheme : GitArchiveInputScheme
return getStrAttr(input.attrs, "repo"); return getStrAttr(input.attrs, "repo");
} }
RefInfo getRevFromRef(const Settings & settings, nix::Store & store, const Input & input) const override RefInfo getRevFromRef(nix::ref<Store> store, const Input & input) const override
{ {
auto host = getHost(input); auto host = getHost(input);
auto url = fmt( auto url = fmt(
@@ -428,22 +397,22 @@ struct GitHubInputScheme : GitArchiveInputScheme
getRepo(input), getRepo(input),
*input.getRef()); *input.getRef());
Headers headers = makeHeadersWithAuthTokens(settings, host, input); Headers headers = makeHeadersWithAuthTokens(*input.settings, host, input);
auto downloadResult = downloadFile(store, settings, url, "source", headers); auto downloadResult = downloadFile(store, *input.settings, url, "source", headers);
auto json = nlohmann::json::parse( auto json = nlohmann::json::parse(
store.requireStoreObjectAccessor(downloadResult.storePath)->readFile(CanonPath::root)); store->requireStoreObjectAccessor(downloadResult.storePath)->readFile(CanonPath::root));
return RefInfo{ return RefInfo{
.rev = Hash::parseAny(std::string{json["sha"]}, HashAlgorithm::SHA1), .rev = Hash::parseAny(std::string{json["sha"]}, HashAlgorithm::SHA1),
.treeHash = Hash::parseAny(std::string{json["commit"]["tree"]["sha"]}, HashAlgorithm::SHA1)}; .treeHash = Hash::parseAny(std::string{json["commit"]["tree"]["sha"]}, HashAlgorithm::SHA1)};
} }
DownloadUrl getDownloadUrl(const Settings & settings, const Input & input) const override DownloadUrl getDownloadUrl(const Input & input) const override
{ {
auto host = getHost(input); auto host = getHost(input);
Headers headers = makeHeadersWithAuthTokens(settings, host, input); Headers headers = makeHeadersWithAuthTokens(*input.settings, host, input);
// If we have no auth headers then we default to the public archive // If we have no auth headers then we default to the public archive
// urls so we do not run into rate limits. // urls so we do not run into rate limits.
@@ -457,13 +426,12 @@ struct GitHubInputScheme : GitArchiveInputScheme
return DownloadUrl{parseURL(url), headers}; return DownloadUrl{parseURL(url), headers};
} }
void clone(const Settings & settings, Store & store, const Input & input, const std::filesystem::path & destDir) void clone(const Input & input, const Path & destDir) const override
const override
{ {
auto host = getHost(input); auto host = getHost(input);
Input::fromURL(settings, fmt("git+https://%s/%s/%s.git", host, getOwner(input), getRepo(input))) Input::fromURL(*input.settings, fmt("git+https://%s/%s/%s.git", host, getOwner(input), getRepo(input)))
.applyOverrides(input.getRef(), input.getRev()) .applyOverrides(input.getRef(), input.getRev())
.clone(settings, store, destDir); .clone(destDir);
} }
}; };
@@ -474,12 +442,6 @@ struct GitLabInputScheme : GitArchiveInputScheme
return "gitlab"; return "gitlab";
} }
std::string schemeDescription() const override
{
// TODO
return "";
}
std::optional<std::pair<std::string, std::string>> accessHeaderFromToken(const std::string & token) const override std::optional<std::pair<std::string, std::string>> accessHeaderFromToken(const std::string & token) const override
{ {
// Gitlab supports 4 kinds of authorization, two of which are // Gitlab supports 4 kinds of authorization, two of which are
@@ -499,7 +461,7 @@ struct GitLabInputScheme : GitArchiveInputScheme
return std::make_pair(token.substr(0, fldsplit), token.substr(fldsplit + 1)); return std::make_pair(token.substr(0, fldsplit), token.substr(fldsplit + 1));
} }
RefInfo getRevFromRef(const Settings & settings, nix::Store & store, const Input & input) const override RefInfo getRevFromRef(nix::ref<Store> store, const Input & input) const override
{ {
auto host = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com"); auto host = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com");
// See rate limiting note below // See rate limiting note below
@@ -510,11 +472,11 @@ struct GitLabInputScheme : GitArchiveInputScheme
getStrAttr(input.attrs, "repo"), getStrAttr(input.attrs, "repo"),
*input.getRef()); *input.getRef());
Headers headers = makeHeadersWithAuthTokens(settings, host, input); Headers headers = makeHeadersWithAuthTokens(*input.settings, host, input);
auto downloadResult = downloadFile(store, settings, url, "source", headers); auto downloadResult = downloadFile(store, *input.settings, url, "source", headers);
auto json = nlohmann::json::parse( auto json = nlohmann::json::parse(
store.requireStoreObjectAccessor(downloadResult.storePath)->readFile(CanonPath::root)); store->requireStoreObjectAccessor(downloadResult.storePath)->readFile(CanonPath::root));
if (json.is_array() && json.size() >= 1 && json[0]["id"] != nullptr) { if (json.is_array() && json.size() >= 1 && json[0]["id"] != nullptr) {
return RefInfo{.rev = Hash::parseAny(std::string(json[0]["id"]), HashAlgorithm::SHA1)}; return RefInfo{.rev = Hash::parseAny(std::string(json[0]["id"]), HashAlgorithm::SHA1)};
@@ -526,7 +488,7 @@ struct GitLabInputScheme : GitArchiveInputScheme
} }
} }
DownloadUrl getDownloadUrl(const Settings & settings, const Input & input) const override DownloadUrl getDownloadUrl(const Input & input) const override
{ {
// This endpoint has a rate limit threshold that may be // This endpoint has a rate limit threshold that may be
// server-specific and vary based whether the user is // server-specific and vary based whether the user is
@@ -541,20 +503,19 @@ struct GitLabInputScheme : GitArchiveInputScheme
getStrAttr(input.attrs, "repo"), getStrAttr(input.attrs, "repo"),
input.getRev()->to_string(HashFormat::Base16, false)); input.getRev()->to_string(HashFormat::Base16, false));
Headers headers = makeHeadersWithAuthTokens(settings, host, input); Headers headers = makeHeadersWithAuthTokens(*input.settings, host, input);
return DownloadUrl{parseURL(url), headers}; return DownloadUrl{parseURL(url), headers};
} }
void clone(const Settings & settings, Store & store, const Input & input, const std::filesystem::path & destDir) void clone(const Input & input, const Path & destDir) const override
const override
{ {
auto host = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com"); auto host = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com");
// FIXME: get username somewhere // FIXME: get username somewhere
Input::fromURL( Input::fromURL(
settings, *input.settings,
fmt("git+https://%s/%s/%s.git", host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"))) fmt("git+https://%s/%s/%s.git", host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo")))
.applyOverrides(input.getRef(), input.getRev()) .applyOverrides(input.getRef(), input.getRev())
.clone(settings, store, destDir); .clone(destDir);
} }
}; };
@@ -565,12 +526,6 @@ struct SourceHutInputScheme : GitArchiveInputScheme
return "sourcehut"; return "sourcehut";
} }
std::string schemeDescription() const override
{
// TODO
return "";
}
std::optional<std::pair<std::string, std::string>> accessHeaderFromToken(const std::string & token) const override std::optional<std::pair<std::string, std::string>> accessHeaderFromToken(const std::string & token) const override
{ {
// SourceHut supports both PAT and OAuth2. See // SourceHut supports both PAT and OAuth2. See
@@ -581,7 +536,7 @@ struct SourceHutInputScheme : GitArchiveInputScheme
// Once it is implemented, however, should work as expected. // Once it is implemented, however, should work as expected.
} }
RefInfo getRevFromRef(const Settings & settings, nix::Store & store, const Input & input) const override RefInfo getRevFromRef(nix::ref<Store> store, const Input & input) const override
{ {
// TODO: In the future, when the sourcehut graphql API is implemented for mercurial // TODO: In the future, when the sourcehut graphql API is implemented for mercurial
// and with anonymous access, this method should use it instead. // and with anonymous access, this method should use it instead.
@@ -592,12 +547,12 @@ struct SourceHutInputScheme : GitArchiveInputScheme
auto base_url = auto base_url =
fmt("https://%s/%s/%s", host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo")); fmt("https://%s/%s/%s", host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"));
Headers headers = makeHeadersWithAuthTokens(settings, host, input); Headers headers = makeHeadersWithAuthTokens(*input.settings, host, input);
std::string refUri; std::string refUri;
if (ref == "HEAD") { if (ref == "HEAD") {
auto downloadFileResult = downloadFile(store, settings, fmt("%s/HEAD", base_url), "source", headers); auto downloadFileResult = downloadFile(store, *input.settings, fmt("%s/HEAD", base_url), "source", headers);
auto contents = store.requireStoreObjectAccessor(downloadFileResult.storePath)->readFile(CanonPath::root); auto contents = store->requireStoreObjectAccessor(downloadFileResult.storePath)->readFile(CanonPath::root);
auto remoteLine = git::parseLsRemoteLine(getLine(contents).first); auto remoteLine = git::parseLsRemoteLine(getLine(contents).first);
if (!remoteLine) { if (!remoteLine) {
@@ -609,8 +564,9 @@ struct SourceHutInputScheme : GitArchiveInputScheme
} }
std::regex refRegex(refUri); std::regex refRegex(refUri);
auto downloadFileResult = downloadFile(store, settings, fmt("%s/info/refs", base_url), "source", headers); auto downloadFileResult =
auto contents = store.requireStoreObjectAccessor(downloadFileResult.storePath)->readFile(CanonPath::root); downloadFile(store, *input.settings, fmt("%s/info/refs", base_url), "source", headers);
auto contents = store->requireStoreObjectAccessor(downloadFileResult.storePath)->readFile(CanonPath::root);
std::istringstream is(contents); std::istringstream is(contents);
std::string line; std::string line;
@@ -627,7 +583,7 @@ struct SourceHutInputScheme : GitArchiveInputScheme
return RefInfo{.rev = Hash::parseAny(*id, HashAlgorithm::SHA1)}; return RefInfo{.rev = Hash::parseAny(*id, HashAlgorithm::SHA1)};
} }
DownloadUrl getDownloadUrl(const Settings & settings, const Input & input) const override DownloadUrl getDownloadUrl(const Input & input) const override
{ {
auto host = maybeGetStrAttr(input.attrs, "host").value_or("git.sr.ht"); auto host = maybeGetStrAttr(input.attrs, "host").value_or("git.sr.ht");
auto url = auto url =
@@ -637,19 +593,18 @@ struct SourceHutInputScheme : GitArchiveInputScheme
getStrAttr(input.attrs, "repo"), getStrAttr(input.attrs, "repo"),
input.getRev()->to_string(HashFormat::Base16, false)); input.getRev()->to_string(HashFormat::Base16, false));
Headers headers = makeHeadersWithAuthTokens(settings, host, input); Headers headers = makeHeadersWithAuthTokens(*input.settings, host, input);
return DownloadUrl{parseURL(url), headers}; return DownloadUrl{parseURL(url), headers};
} }
void clone(const Settings & settings, Store & store, const Input & input, const std::filesystem::path & destDir) void clone(const Input & input, const Path & destDir) const override
const override
{ {
auto host = maybeGetStrAttr(input.attrs, "host").value_or("git.sr.ht"); auto host = maybeGetStrAttr(input.attrs, "host").value_or("git.sr.ht");
Input::fromURL( Input::fromURL(
settings, *input.settings,
fmt("git+https://%s/%s/%s", host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"))) fmt("git+https://%s/%s/%s", host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo")))
.applyOverrides(input.getRef(), input.getRev()) .applyOverrides(input.getRef(), input.getRev())
.clone(settings, store, destDir); .clone(destDir);
} }
}; };
@@ -36,6 +36,13 @@ struct Input
{ {
friend struct InputScheme; friend struct InputScheme;
const Settings * settings;
Input(const Settings & settings)
: settings{&settings}
{
}
std::shared_ptr<InputScheme> scheme; // note: can be null std::shared_ptr<InputScheme> scheme; // note: can be null
Attrs attrs; Attrs attrs;
@@ -80,7 +87,7 @@ public:
* attributes like a Git revision or NAR hash that uniquely * attributes like a Git revision or NAR hash that uniquely
* identify its contents. * identify its contents.
*/ */
bool isLocked(const Settings & settings) const; bool isLocked() const;
/** /**
* Only for relative path flakes, i.e. 'path:./foo', returns the * Only for relative path flakes, i.e. 'path:./foo', returns the
@@ -113,7 +120,7 @@ public:
* Fetch the entire input into the Nix store, returning the * Fetch the entire input into the Nix store, returning the
* location in the Nix store and the locked input. * location in the Nix store and the locked input.
*/ */
std::pair<StorePath, Input> fetchToStore(const Settings & settings, Store & store) const; std::pair<StorePath, Input> fetchToStore(ref<Store> store) const;
/** /**
* Check the locking attributes in `result` against * Check the locking attributes in `result` against
@@ -133,17 +140,17 @@ public:
* input without copying it to the store. Also return a possibly * input without copying it to the store. Also return a possibly
* unlocked input. * unlocked input.
*/ */
std::pair<ref<SourceAccessor>, Input> getAccessor(const Settings & settings, Store & store) const; std::pair<ref<SourceAccessor>, Input> getAccessor(ref<Store> store) const;
private: private:
std::pair<ref<SourceAccessor>, Input> getAccessorUnchecked(const Settings & settings, Store & store) const; std::pair<ref<SourceAccessor>, Input> getAccessorUnchecked(ref<Store> store) const;
public: public:
Input applyOverrides(std::optional<std::string> ref, std::optional<Hash> rev) const; Input applyOverrides(std::optional<std::string> ref, std::optional<Hash> rev) const;
void clone(const Settings & settings, Store & store, const std::filesystem::path & destDir) const; void clone(const Path & destDir) const;
std::optional<std::filesystem::path> getSourcePath() const; std::optional<std::filesystem::path> getSourcePath() const;
@@ -173,7 +180,7 @@ public:
* *
* This is not a stable identifier between Nix versions, but not guaranteed to change either. * This is not a stable identifier between Nix versions, but not guaranteed to change either.
*/ */
std::optional<std::string> getFingerprint(Store & store) const; std::optional<std::string> getFingerprint(ref<Store> store) const;
}; };
/** /**
@@ -203,34 +210,20 @@ struct InputScheme
*/ */
virtual std::string_view schemeName() const = 0; virtual std::string_view schemeName() const = 0;
/**
* Longform description of this scheme, for documentation purposes.
*/
virtual std::string schemeDescription() const = 0;
// TODO remove these defaults
struct AttributeInfo
{
const char * type = "String";
bool required = true;
const char * doc = "";
};
/** /**
* Allowed attributes in an attribute set that is converted to an * Allowed attributes in an attribute set that is converted to an
* input, and documentation for each attribute. * input.
* *
* `type` is not included from this map, because the `type` field is * `type` is not included from this set, because the `type` field is
parsed first to choose which scheme; `type` is always required. parsed first to choose which scheme; `type` is always required.
*/ */
virtual const std::map<std::string, AttributeInfo> & allowedAttrs() const = 0; virtual StringSet allowedAttrs() const = 0;
virtual ParsedURL toURL(const Input & input) const; virtual ParsedURL toURL(const Input & input) const;
virtual Input applyOverrides(const Input & input, std::optional<std::string> ref, std::optional<Hash> rev) const; virtual Input applyOverrides(const Input & input, std::optional<std::string> ref, std::optional<Hash> rev) const;
virtual void virtual void clone(const Input & input, const Path & destDir) const;
clone(const Settings & settings, Store & store, const Input & input, const std::filesystem::path & destDir) const;
virtual std::optional<std::filesystem::path> getSourcePath(const Input & input) const; virtual std::optional<std::filesystem::path> getSourcePath(const Input & input) const;
@@ -240,8 +233,7 @@ struct InputScheme
std::string_view contents, std::string_view contents,
std::optional<std::string> commitMsg) const; std::optional<std::string> commitMsg) const;
virtual std::pair<ref<SourceAccessor>, Input> virtual std::pair<ref<SourceAccessor>, Input> getAccessor(ref<Store> store, const Input & input) const = 0;
getAccessor(const Settings & settings, Store & store, const Input & input) const = 0;
/** /**
* Is this `InputScheme` part of an experimental feature? * Is this `InputScheme` part of an experimental feature?
@@ -253,12 +245,12 @@ struct InputScheme
return true; return true;
} }
virtual std::optional<std::string> getFingerprint(Store & store, const Input & input) const virtual std::optional<std::string> getFingerprint(ref<Store> store, const Input & input) const
{ {
return std::nullopt; return std::nullopt;
} }
virtual bool isLocked(const Settings & settings, const Input & input) const virtual bool isLocked(const Input & input) const
{ {
return false; return false;
} }
@@ -277,12 +269,7 @@ struct InputScheme
void registerInputScheme(std::shared_ptr<InputScheme> && fetcher); void registerInputScheme(std::shared_ptr<InputScheme> && fetcher);
using InputSchemeMap = std::map<std::string_view, std::shared_ptr<InputScheme>>; nlohmann::json dumpRegisterInputSchemeInfo();
/**
* Use this for docs, not for finding a specific scheme
*/
const InputSchemeMap & getAllInputSchemes();
struct PublicKey struct PublicKey
{ {
@@ -3,7 +3,6 @@
namespace nix::fetchers { namespace nix::fetchers {
enum class UseRegistries : int; enum class UseRegistries : int;
struct Settings;
struct InputCache struct InputCache
{ {
@@ -15,8 +14,7 @@ struct InputCache
Attrs extraAttrs; Attrs extraAttrs;
}; };
CachedResult CachedResult getAccessor(ref<Store> store, const Input & originalInput, UseRegistries useRegistries);
getAccessor(const Settings & settings, Store & store, const Input & originalInput, UseRegistries useRegistries);
struct CachedInput struct CachedInput
{ {
@@ -13,6 +13,8 @@ namespace nix::fetchers {
struct Registry struct Registry
{ {
const Settings & settings;
enum RegistryType { enum RegistryType {
Flag = 0, Flag = 0,
User = 1, User = 1,
@@ -32,8 +34,9 @@ struct Registry
std::vector<Entry> entries; std::vector<Entry> entries;
Registry(RegistryType type) Registry(const Settings & settings, RegistryType type)
: type{type} : settings{settings}
, type{type}
{ {
} }
@@ -54,7 +57,7 @@ std::shared_ptr<Registry> getCustomRegistry(const Settings & settings, const Pat
Path getUserRegistryPath(); Path getUserRegistryPath();
Registries getRegistries(const Settings & settings, Store & store); Registries getRegistries(const Settings & settings, ref<Store> store);
void overrideRegistry(const Input & from, const Input & to, const Attrs & extraAttrs); void overrideRegistry(const Input & from, const Input & to, const Attrs & extraAttrs);
@@ -68,7 +71,6 @@ enum class UseRegistries : int {
* Rewrite a flakeref using the registries. If `filter` is set, only * Rewrite a flakeref using the registries. If `filter` is set, only
* use the registries for which the filter function returns true. * use the registries for which the filter function returns true.
*/ */
std::pair<Input, Attrs> std::pair<Input, Attrs> lookupInRegistries(ref<Store> store, const Input & input, UseRegistries useRegistries);
lookupInRegistries(const Settings & settings, Store & store, const Input & input, UseRegistries useRegistries);
} // namespace nix::fetchers } // namespace nix::fetchers
@@ -25,7 +25,7 @@ struct DownloadFileResult
}; };
DownloadFileResult downloadFile( DownloadFileResult downloadFile(
Store & store, ref<Store> store,
const Settings & settings, const Settings & settings,
const std::string & url, const std::string & url,
const std::string & name, const std::string & name,
@@ -43,6 +43,6 @@ struct DownloadTarballResult
* Download and import a tarball into the Git cache. The result is the * Download and import a tarball into the Git cache. The result is the
* Git tree hash of the root directory. * Git tree hash of the root directory.
*/ */
ref<SourceAccessor> downloadTarball(Store & store, const Settings & settings, const std::string & url); ref<SourceAccessor> downloadTarball(ref<Store> store, const Settings & settings, const std::string & url);
} // namespace nix::fetchers } // namespace nix::fetchers
+9 -29
View File
@@ -44,7 +44,7 @@ struct IndirectInputScheme : InputScheme
// FIXME: forbid query params? // FIXME: forbid query params?
Input input{}; Input input{settings};
input.attrs.insert_or_assign("type", "indirect"); input.attrs.insert_or_assign("type", "indirect");
input.attrs.insert_or_assign("id", id); input.attrs.insert_or_assign("id", id);
if (rev) if (rev)
@@ -60,33 +60,14 @@ struct IndirectInputScheme : InputScheme
return "indirect"; return "indirect";
} }
std::string schemeDescription() const override StringSet allowedAttrs() const override
{ {
// TODO return {
return ""; "id",
} "ref",
"rev",
const std::map<std::string, AttributeInfo> & allowedAttrs() const override "narHash",
{
static const std::map<std::string, AttributeInfo> attrs = {
{
"id",
{},
},
{
"ref",
{},
},
{
"rev",
{},
},
{
"narHash",
{},
},
}; };
return attrs;
} }
std::optional<Input> inputFromAttrs(const Settings & settings, const Attrs & attrs) const override std::optional<Input> inputFromAttrs(const Settings & settings, const Attrs & attrs) const override
@@ -95,7 +76,7 @@ struct IndirectInputScheme : InputScheme
if (!std::regex_match(id, flakeRegex)) if (!std::regex_match(id, flakeRegex))
throw BadURL("'%s' is not a valid flake ID", id); throw BadURL("'%s' is not a valid flake ID", id);
Input input{}; Input input{settings};
input.attrs = attrs; input.attrs = attrs;
return input; return input;
} }
@@ -125,8 +106,7 @@ struct IndirectInputScheme : InputScheme
return input; return input;
} }
std::pair<ref<SourceAccessor>, Input> std::pair<ref<SourceAccessor>, Input> getAccessor(ref<Store> store, const Input & input) const override
getAccessor(const Settings & settings, Store & store, const Input & input) const override
{ {
throw Error("indirect input '%s' cannot be fetched directly", input.to_string()); throw Error("indirect input '%s' cannot be fetched directly", input.to_string());
} }
+5 -5
View File
@@ -5,23 +5,23 @@
namespace nix::fetchers { namespace nix::fetchers {
InputCache::CachedResult InputCache::getAccessor( InputCache::CachedResult
const Settings & settings, Store & store, const Input & originalInput, UseRegistries useRegistries) InputCache::getAccessor(ref<Store> store, const Input & originalInput, UseRegistries useRegistries)
{ {
auto fetched = lookup(originalInput); auto fetched = lookup(originalInput);
Input resolvedInput = originalInput; Input resolvedInput = originalInput;
if (!fetched) { if (!fetched) {
if (originalInput.isDirect()) { if (originalInput.isDirect()) {
auto [accessor, lockedInput] = originalInput.getAccessor(settings, store); auto [accessor, lockedInput] = originalInput.getAccessor(store);
fetched.emplace(CachedInput{.lockedInput = lockedInput, .accessor = accessor}); fetched.emplace(CachedInput{.lockedInput = lockedInput, .accessor = accessor});
} else { } else {
if (useRegistries != UseRegistries::No) { if (useRegistries != UseRegistries::No) {
auto [res, extraAttrs] = lookupInRegistries(settings, store, originalInput, useRegistries); auto [res, extraAttrs] = lookupInRegistries(store, originalInput, useRegistries);
resolvedInput = std::move(res); resolvedInput = std::move(res);
fetched = lookup(resolvedInput); fetched = lookup(resolvedInput);
if (!fetched) { if (!fetched) {
auto [accessor, lockedInput] = resolvedInput.getAccessor(settings, store); auto [accessor, lockedInput] = resolvedInput.getAccessor(store);
fetched.emplace( fetched.emplace(
CachedInput{.lockedInput = lockedInput, .accessor = accessor, .extraAttrs = extraAttrs}); CachedInput{.lockedInput = lockedInput, .accessor = accessor, .extraAttrs = extraAttrs});
} }
+25 -51
View File
@@ -68,41 +68,16 @@ struct MercurialInputScheme : InputScheme
return "hg"; return "hg";
} }
std::string schemeDescription() const override StringSet allowedAttrs() const override
{ {
// TODO return {
return ""; "url",
} "ref",
"rev",
const std::map<std::string, AttributeInfo> & allowedAttrs() const override "revCount",
{ "narHash",
static const std::map<std::string, AttributeInfo> attrs = { "name",
{
"url",
{},
},
{
"ref",
{},
},
{
"rev",
{},
},
{
"revCount",
{},
},
{
"narHash",
{},
},
{
"name",
{},
},
}; };
return attrs;
} }
std::optional<Input> inputFromAttrs(const Settings & settings, const Attrs & attrs) const override std::optional<Input> inputFromAttrs(const Settings & settings, const Attrs & attrs) const override
@@ -114,7 +89,7 @@ struct MercurialInputScheme : InputScheme
throw BadURL("invalid Mercurial branch/tag name '%s'", *ref); throw BadURL("invalid Mercurial branch/tag name '%s'", *ref);
} }
Input input{}; Input input{settings};
input.attrs = attrs; input.attrs = attrs;
return input; return input;
} }
@@ -179,7 +154,7 @@ struct MercurialInputScheme : InputScheme
return {isLocal, isLocal ? renderUrlPathEnsureLegal(url.path) : url.to_string()}; return {isLocal, isLocal ? renderUrlPathEnsureLegal(url.path) : url.to_string()};
} }
StorePath fetchToStore(const Settings & settings, Store & store, Input & input) const StorePath fetchToStore(ref<Store> store, Input & input) const
{ {
auto origRev = input.getRev(); auto origRev = input.getRev();
@@ -201,10 +176,10 @@ struct MercurialInputScheme : InputScheme
/* This is an unclean working tree. So copy all tracked /* This is an unclean working tree. So copy all tracked
files. */ files. */
if (!settings.allowDirty) if (!input.settings->allowDirty)
throw Error("Mercurial tree '%s' is unclean", actualUrl); throw Error("Mercurial tree '%s' is unclean", actualUrl);
if (settings.warnDirty) if (input.settings->warnDirty)
warn("Mercurial tree '%s' is unclean", actualUrl); warn("Mercurial tree '%s' is unclean", actualUrl);
input.attrs.insert_or_assign("ref", chomp(runHg({"branch", "-R", actualUrl}))); input.attrs.insert_or_assign("ref", chomp(runHg({"branch", "-R", actualUrl})));
@@ -230,7 +205,7 @@ struct MercurialInputScheme : InputScheme
return files.count(file); return files.count(file);
}; };
auto storePath = store.addToStore( auto storePath = store->addToStore(
input.getName(), input.getName(),
{getFSSourceAccessor(), CanonPath(actualPath)}, {getFSSourceAccessor(), CanonPath(actualPath)},
ContentAddressMethod::Raw::NixArchive, ContentAddressMethod::Raw::NixArchive,
@@ -251,7 +226,7 @@ struct MercurialInputScheme : InputScheme
"Hash '%s' is not supported by Mercurial. Only sha1 is supported.", "Hash '%s' is not supported by Mercurial. Only sha1 is supported.",
rev.to_string(HashFormat::Base16, true)); rev.to_string(HashFormat::Base16, true));
return Cache::Key{"hgRev", {{"store", store.storeDir}, {"name", name}, {"rev", input.getRev()->gitRev()}}}; return Cache::Key{"hgRev", {{"store", store->storeDir}, {"name", name}, {"rev", input.getRev()->gitRev()}}};
}; };
auto makeResult = [&](const Attrs & infoAttrs, const StorePath & storePath) -> StorePath { auto makeResult = [&](const Attrs & infoAttrs, const StorePath & storePath) -> StorePath {
@@ -265,13 +240,13 @@ struct MercurialInputScheme : InputScheme
Cache::Key refToRevKey{"hgRefToRev", {{"url", actualUrl}, {"ref", *input.getRef()}}}; Cache::Key refToRevKey{"hgRefToRev", {{"url", actualUrl}, {"ref", *input.getRef()}}};
if (!input.getRev()) { if (!input.getRev()) {
if (auto res = settings.getCache()->lookupWithTTL(refToRevKey)) if (auto res = input.settings->getCache()->lookupWithTTL(refToRevKey))
input.attrs.insert_or_assign("rev", getRevAttr(*res, "rev").gitRev()); input.attrs.insert_or_assign("rev", getRevAttr(*res, "rev").gitRev());
} }
/* If we have a rev, check if we have a cached store path. */ /* If we have a rev, check if we have a cached store path. */
if (auto rev = input.getRev()) { if (auto rev = input.getRev()) {
if (auto res = settings.getCache()->lookupStorePath(revInfoKey(*rev), store)) if (auto res = input.settings->getCache()->lookupStorePath(revInfoKey(*rev), *store))
return makeResult(res->value, res->storePath); return makeResult(res->value, res->storePath);
} }
@@ -325,7 +300,7 @@ struct MercurialInputScheme : InputScheme
/* Now that we have the rev, check the cache again for a /* Now that we have the rev, check the cache again for a
cached store path. */ cached store path. */
if (auto res = settings.getCache()->lookupStorePath(revInfoKey(rev), store)) if (auto res = input.settings->getCache()->lookupStorePath(revInfoKey(rev), *store))
return makeResult(res->value, res->storePath); return makeResult(res->value, res->storePath);
Path tmpDir = createTempDir(); Path tmpDir = createTempDir();
@@ -335,39 +310,38 @@ struct MercurialInputScheme : InputScheme
deletePath(tmpDir + "/.hg_archival.txt"); deletePath(tmpDir + "/.hg_archival.txt");
auto storePath = store.addToStore(name, {getFSSourceAccessor(), CanonPath(tmpDir)}); auto storePath = store->addToStore(name, {getFSSourceAccessor(), CanonPath(tmpDir)});
Attrs infoAttrs({ Attrs infoAttrs({
{"revCount", (uint64_t) revCount}, {"revCount", (uint64_t) revCount},
}); });
if (!origRev) if (!origRev)
settings.getCache()->upsert(refToRevKey, {{"rev", rev.gitRev()}}); input.settings->getCache()->upsert(refToRevKey, {{"rev", rev.gitRev()}});
settings.getCache()->upsert(revInfoKey(rev), store, infoAttrs, storePath); input.settings->getCache()->upsert(revInfoKey(rev), *store, infoAttrs, storePath);
return makeResult(infoAttrs, std::move(storePath)); return makeResult(infoAttrs, std::move(storePath));
} }
std::pair<ref<SourceAccessor>, Input> std::pair<ref<SourceAccessor>, Input> getAccessor(ref<Store> store, const Input & _input) const override
getAccessor(const Settings & settings, Store & store, const Input & _input) const override
{ {
Input input(_input); Input input(_input);
auto storePath = fetchToStore(settings, store, input); auto storePath = fetchToStore(store, input);
auto accessor = store.requireStoreObjectAccessor(storePath); auto accessor = store->requireStoreObjectAccessor(storePath);
accessor->setPathDisplay("«" + input.to_string() + "»"); accessor->setPathDisplay("«" + input.to_string() + "»");
return {accessor, input}; return {accessor, input};
} }
bool isLocked(const Settings & settings, const Input & input) const override bool isLocked(const Input & input) const override
{ {
return (bool) input.getRev(); return (bool) input.getRev();
} }
std::optional<std::string> getFingerprint(Store & store, const Input & input) const override std::optional<std::string> getFingerprint(ref<Store> store, const Input & input) const override
{ {
if (auto rev = input.getRev()) if (auto rev = input.getRev())
return rev->gitRev(); return rev->gitRev();
+20 -43
View File
@@ -17,7 +17,7 @@ struct PathInputScheme : InputScheme
if (url.authority && url.authority->host.size()) if (url.authority && url.authority->host.size())
throw Error("path URL '%s' should not have an authority ('%s')", url, *url.authority); throw Error("path URL '%s' should not have an authority ('%s')", url, *url.authority);
Input input{}; Input input{settings};
input.attrs.insert_or_assign("type", "path"); input.attrs.insert_or_assign("type", "path");
input.attrs.insert_or_assign("path", renderUrlPathEnsureLegal(url.path)); input.attrs.insert_or_assign("path", renderUrlPathEnsureLegal(url.path));
@@ -40,49 +40,27 @@ struct PathInputScheme : InputScheme
return "path"; return "path";
} }
std::string schemeDescription() const override StringSet allowedAttrs() const override
{ {
// TODO return {
return ""; "path",
}
const std::map<std::string, AttributeInfo> & allowedAttrs() const override
{
static const std::map<std::string, AttributeInfo> attrs = {
{
"path",
{},
},
/* Allow the user to pass in "fake" tree info /* Allow the user to pass in "fake" tree info
attributes. This is useful for making a pinned tree work attributes. This is useful for making a pinned tree work
the same as the repository from which is exported (e.g. the same as the repository from which is exported (e.g.
path:/nix/store/...-source?lastModified=1585388205&rev=b0c285...). path:/nix/store/...-source?lastModified=1585388205&rev=b0c285...).
*/ */
{ "rev",
"rev", "revCount",
{}, "lastModified",
}, "narHash",
{
"revCount",
{},
},
{
"lastModified",
{},
},
{
"narHash",
{},
},
}; };
return attrs;
} }
std::optional<Input> inputFromAttrs(const Settings & settings, const Attrs & attrs) const override std::optional<Input> inputFromAttrs(const Settings & settings, const Attrs & attrs) const override
{ {
getStrAttr(attrs, "path"); getStrAttr(attrs, "path");
Input input{}; Input input{settings};
input.attrs = attrs; input.attrs = attrs;
return input; return input;
} }
@@ -123,7 +101,7 @@ struct PathInputScheme : InputScheme
return path; return path;
} }
bool isLocked(const Settings & settings, const Input & input) const override bool isLocked(const Input & input) const override
{ {
return (bool) input.getNarHash(); return (bool) input.getNarHash();
} }
@@ -138,8 +116,7 @@ struct PathInputScheme : InputScheme
throw Error("cannot fetch input '%s' because it uses a relative path", input.to_string()); throw Error("cannot fetch input '%s' because it uses a relative path", input.to_string());
} }
std::pair<ref<SourceAccessor>, Input> std::pair<ref<SourceAccessor>, Input> getAccessor(ref<Store> store, const Input & _input) const override
getAccessor(const Settings & settings, Store & store, const Input & _input) const override
{ {
Input input(_input); Input input(_input);
auto path = getStrAttr(input.attrs, "path"); auto path = getStrAttr(input.attrs, "path");
@@ -147,31 +124,31 @@ struct PathInputScheme : InputScheme
auto absPath = getAbsPath(input); auto absPath = getAbsPath(input);
// FIXME: check whether access to 'path' is allowed. // FIXME: check whether access to 'path' is allowed.
auto storePath = store.maybeParseStorePath(absPath.string()); auto storePath = store->maybeParseStorePath(absPath.string());
if (storePath) if (storePath)
store.addTempRoot(*storePath); store->addTempRoot(*storePath);
time_t mtime = 0; time_t mtime = 0;
if (!storePath || storePath->name() != "source" || !store.isValidPath(*storePath)) { if (!storePath || storePath->name() != "source" || !store->isValidPath(*storePath)) {
Activity act(*logger, lvlTalkative, actUnknown, fmt("copying %s to the store", absPath)); Activity act(*logger, lvlTalkative, actUnknown, fmt("copying %s to the store", absPath));
// FIXME: try to substitute storePath. // FIXME: try to substitute storePath.
auto src = sinkToSource( auto src = sinkToSource(
[&](Sink & sink) { mtime = dumpPathAndGetMtime(absPath.string(), sink, defaultPathFilter); }); [&](Sink & sink) { mtime = dumpPathAndGetMtime(absPath.string(), sink, defaultPathFilter); });
storePath = store.addToStoreFromDump(*src, "source"); storePath = store->addToStoreFromDump(*src, "source");
} }
auto accessor = store.requireStoreObjectAccessor(*storePath); auto accessor = store->requireStoreObjectAccessor(*storePath);
// To prevent `fetchToStore()` copying the path again to Nix // To prevent `fetchToStore()` copying the path again to Nix
// store, pre-create an entry in the fetcher cache. // store, pre-create an entry in the fetcher cache.
auto info = store.queryPathInfo(*storePath); auto info = store->queryPathInfo(*storePath);
accessor->fingerprint = accessor->fingerprint =
fmt("path:%s", store.queryPathInfo(*storePath)->narHash.to_string(HashFormat::SRI, true)); fmt("path:%s", store->queryPathInfo(*storePath)->narHash.to_string(HashFormat::SRI, true));
settings.getCache()->upsert( input.settings->getCache()->upsert(
makeFetchToStoreCacheKey( makeFetchToStoreCacheKey(
input.getName(), *accessor->fingerprint, ContentAddressMethod::Raw::NixArchive, "/"), input.getName(), *accessor->fingerprint, ContentAddressMethod::Raw::NixArchive, "/"),
store, *store,
{}, {},
*storePath); *storePath);
+13 -14
View File
@@ -14,10 +14,10 @@ std::shared_ptr<Registry> Registry::read(const Settings & settings, const Source
{ {
debug("reading registry '%s'", path); debug("reading registry '%s'", path);
auto registry = std::make_shared<Registry>(type); auto registry = std::make_shared<Registry>(settings, type);
if (!path.pathExists()) if (!path.pathExists())
return std::make_shared<Registry>(type); return std::make_shared<Registry>(settings, type);
try { try {
@@ -125,23 +125,23 @@ std::shared_ptr<Registry> getCustomRegistry(const Settings & settings, const Pat
return customRegistry; return customRegistry;
} }
std::shared_ptr<Registry> getFlagRegistry() std::shared_ptr<Registry> getFlagRegistry(const Settings & settings)
{ {
static auto flagRegistry = std::make_shared<Registry>(Registry::Flag); static auto flagRegistry = std::make_shared<Registry>(settings, Registry::Flag);
return flagRegistry; return flagRegistry;
} }
void overrideRegistry(const Input & from, const Input & to, const Attrs & extraAttrs) void overrideRegistry(const Input & from, const Input & to, const Attrs & extraAttrs)
{ {
getFlagRegistry()->add(from, to, extraAttrs); getFlagRegistry(*from.settings)->add(from, to, extraAttrs);
} }
static std::shared_ptr<Registry> getGlobalRegistry(const Settings & settings, Store & store) static std::shared_ptr<Registry> getGlobalRegistry(const Settings & settings, ref<Store> store)
{ {
static auto reg = [&]() { static auto reg = [&]() {
auto path = settings.flakeRegistry.get(); auto path = settings.flakeRegistry.get();
if (path == "") { if (path == "") {
return std::make_shared<Registry>(Registry::Global); // empty registry return std::make_shared<Registry>(settings, Registry::Global); // empty registry
} }
return Registry::read( return Registry::read(
@@ -149,9 +149,9 @@ static std::shared_ptr<Registry> getGlobalRegistry(const Settings & settings, St
[&] -> SourcePath { [&] -> SourcePath {
if (!isAbsolute(path)) { if (!isAbsolute(path)) {
auto storePath = downloadFile(store, settings, path, "flake-registry.json").storePath; auto storePath = downloadFile(store, settings, path, "flake-registry.json").storePath;
if (auto store2 = dynamic_cast<LocalFSStore *>(&store)) if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>())
store2->addPermRoot(storePath, getCacheDir() + "/flake-registry.json"); store2->addPermRoot(storePath, getCacheDir() + "/flake-registry.json");
return {store.requireStoreObjectAccessor(storePath)}; return {store->requireStoreObjectAccessor(storePath)};
} else { } else {
return SourcePath{getFSSourceAccessor(), CanonPath{path}}.resolveSymlinks(); return SourcePath{getFSSourceAccessor(), CanonPath{path}}.resolveSymlinks();
} }
@@ -162,18 +162,17 @@ static std::shared_ptr<Registry> getGlobalRegistry(const Settings & settings, St
return reg; return reg;
} }
Registries getRegistries(const Settings & settings, Store & store) Registries getRegistries(const Settings & settings, ref<Store> store)
{ {
Registries registries; Registries registries;
registries.push_back(getFlagRegistry()); registries.push_back(getFlagRegistry(settings));
registries.push_back(getUserRegistry(settings)); registries.push_back(getUserRegistry(settings));
registries.push_back(getSystemRegistry(settings)); registries.push_back(getSystemRegistry(settings));
registries.push_back(getGlobalRegistry(settings, store)); registries.push_back(getGlobalRegistry(settings, store));
return registries; return registries;
} }
std::pair<Input, Attrs> std::pair<Input, Attrs> lookupInRegistries(ref<Store> store, const Input & _input, UseRegistries useRegistries)
lookupInRegistries(const Settings & settings, Store & store, const Input & _input, UseRegistries useRegistries)
{ {
Attrs extraAttrs; Attrs extraAttrs;
int n = 0; int n = 0;
@@ -188,7 +187,7 @@ restart:
if (n > 100) if (n > 100)
throw Error("cycle detected in flake registry for '%s'", input.to_string()); throw Error("cycle detected in flake registry for '%s'", input.to_string());
for (auto & registry : getRegistries(settings, store)) { for (auto & registry : getRegistries(*input.settings, store)) {
if (useRegistries == UseRegistries::Limited if (useRegistries == UseRegistries::Limited
&& !(registry->type == fetchers::Registry::Flag || registry->type == fetchers::Registry::Global)) && !(registry->type == fetchers::Registry::Flag || registry->type == fetchers::Registry::Global))
continue; continue;
+33 -133
View File
@@ -13,7 +13,7 @@
namespace nix::fetchers { namespace nix::fetchers {
DownloadFileResult downloadFile( DownloadFileResult downloadFile(
Store & store, ref<Store> store,
const Settings & settings, const Settings & settings,
const std::string & url, const std::string & url,
const std::string & name, const std::string & name,
@@ -28,7 +28,7 @@ DownloadFileResult downloadFile(
{"name", name}, {"name", name},
}}}; }}};
auto cached = settings.getCache()->lookupStorePath(key, store); auto cached = settings.getCache()->lookupStorePath(key, *store);
auto useCached = [&]() -> DownloadFileResult { auto useCached = [&]() -> DownloadFileResult {
return { return {
@@ -74,7 +74,7 @@ DownloadFileResult downloadFile(
dumpString(res.data, sink); dumpString(res.data, sink);
auto hash = hashString(HashAlgorithm::SHA256, res.data); auto hash = hashString(HashAlgorithm::SHA256, res.data);
auto info = ValidPathInfo::makeFromCA( auto info = ValidPathInfo::makeFromCA(
store, *store,
name, name,
FixedOutputInfo{ FixedOutputInfo{
.method = FileIngestionMethod::Flat, .method = FileIngestionMethod::Flat,
@@ -84,7 +84,7 @@ DownloadFileResult downloadFile(
hashString(HashAlgorithm::SHA256, sink.s)); hashString(HashAlgorithm::SHA256, sink.s));
info.narSize = sink.s.size(); info.narSize = sink.s.size();
auto source = StringSource{sink.s}; auto source = StringSource{sink.s};
store.addToStore(info, source, NoRepair, NoCheckSigs); store->addToStore(info, source, NoRepair, NoCheckSigs);
storePath = std::move(info.path); storePath = std::move(info.path);
} }
@@ -93,7 +93,7 @@ DownloadFileResult downloadFile(
key.second.insert_or_assign("url", url); key.second.insert_or_assign("url", url);
assert(!res.urls.empty()); assert(!res.urls.empty());
infoAttrs.insert_or_assign("url", *res.urls.rbegin()); infoAttrs.insert_or_assign("url", *res.urls.rbegin());
settings.getCache()->upsert(key, store, infoAttrs, *storePath); settings.getCache()->upsert(key, *store, infoAttrs, *storePath);
} }
return { return {
@@ -214,7 +214,7 @@ static DownloadTarballResult downloadTarball_(
return attrsToResult(infoAttrs); return attrsToResult(infoAttrs);
} }
ref<SourceAccessor> downloadTarball(Store & store, const Settings & settings, const std::string & url) ref<SourceAccessor> downloadTarball(ref<Store> store, const Settings & settings, const std::string & url)
{ {
/* Go through Input::getAccessor() to ensure that the resulting /* Go through Input::getAccessor() to ensure that the resulting
accessor has a fingerprint. */ accessor has a fingerprint. */
@@ -224,7 +224,7 @@ ref<SourceAccessor> downloadTarball(Store & store, const Settings & settings, co
auto input = Input::fromAttrs(settings, std::move(attrs)); auto input = Input::fromAttrs(settings, std::move(attrs));
return input.getAccessor(settings, store).first; return input.getAccessor(store).first;
} }
// An input scheme corresponding to a curl-downloadable resource. // An input scheme corresponding to a curl-downloadable resource.
@@ -252,7 +252,7 @@ struct CurlInputScheme : InputScheme
if (!isValidURL(_url, requireTree)) if (!isValidURL(_url, requireTree))
return std::nullopt; return std::nullopt;
Input input{}; Input input{settings};
auto url = _url; auto url = _url;
@@ -278,7 +278,7 @@ struct CurlInputScheme : InputScheme
HTTP request. Now that we've processed the Nix-specific HTTP request. Now that we've processed the Nix-specific
attributes above, remove them so we don't also send them as attributes above, remove them so we don't also send them as
part of the HTTP request. */ part of the HTTP request. */
for (auto & [param, _] : allowedAttrs()) for (auto & param : allowedAttrs())
url.query.erase(param); url.query.erase(param);
input.attrs.insert_or_assign("type", std::string{schemeName()}); input.attrs.insert_or_assign("type", std::string{schemeName()});
@@ -286,88 +286,23 @@ struct CurlInputScheme : InputScheme
return input; return input;
} }
static const std::map<std::string, AttributeInfo> & allowedAttrsImpl() StringSet allowedAttrs() const override
{ {
static const std::map<std::string, AttributeInfo> attrs = { return {
{ "type",
"url", "url",
{ "narHash",
.type = "String", "name",
.required = true, "unpack",
.doc = R"( "rev",
Supported protocols: "revCount",
"lastModified",
- `https`
> **Example**
>
> ```nix
> fetchTree {
> type = "file";
> url = "https://example.com/index.html";
> }
> ```
- `http`
Insecure HTTP transfer for legacy sources.
> **Warning**
>
> HTTP performs no encryption or authentication.
> Use a `narHash` known in advance to ensure the output has expected contents.
- `file`
A file on the local file system.
> **Example**
>
> ```nix
> fetchTree {
> type = "file";
> url = "file:///home/eelco/nix/README.md";
> }
> ```
)",
},
},
{
"narHash",
{},
},
{
"name",
{},
},
{
"unpack",
{},
},
{
"rev",
{},
},
{
"revCount",
{},
},
{
"lastModified",
{},
},
}; };
return attrs;
}
const std::map<std::string, AttributeInfo> & allowedAttrs() const override
{
return allowedAttrsImpl();
} }
std::optional<Input> inputFromAttrs(const Settings & settings, const Attrs & attrs) const override std::optional<Input> inputFromAttrs(const Settings & settings, const Attrs & attrs) const override
{ {
Input input{}; Input input{settings};
input.attrs = attrs; input.attrs = attrs;
// input.locked = (bool) maybeGetStrAttr(input.attrs, "hash"); // input.locked = (bool) maybeGetStrAttr(input.attrs, "hash");
@@ -384,7 +319,7 @@ struct CurlInputScheme : InputScheme
return url; return url;
} }
bool isLocked(const Settings & settings, const Input & input) const override bool isLocked(const Input & input) const override
{ {
return (bool) input.getNarHash(); return (bool) input.getNarHash();
} }
@@ -397,14 +332,6 @@ struct FileInputScheme : CurlInputScheme
return "file"; return "file";
} }
std::string schemeDescription() const override
{
return stripIndentation(R"(
Place a plain file into the Nix store.
This is similar to [`builtins.fetchurl`](@docroot@/language/builtins.md#builtins-fetchurl)
)");
}
bool isValidURL(const ParsedURL & url, bool requireTree) const override bool isValidURL(const ParsedURL & url, bool requireTree) const override
{ {
auto parsedUrlScheme = parseUrlScheme(url.scheme); auto parsedUrlScheme = parseUrlScheme(url.scheme);
@@ -413,8 +340,7 @@ struct FileInputScheme : CurlInputScheme
: (!requireTree && !hasTarballExtension(url))); : (!requireTree && !hasTarballExtension(url)));
} }
std::pair<ref<SourceAccessor>, Input> std::pair<ref<SourceAccessor>, Input> getAccessor(ref<Store> store, const Input & _input) const override
getAccessor(const Settings & settings, Store & store, const Input & _input) const override
{ {
auto input(_input); auto input(_input);
@@ -422,12 +348,12 @@ struct FileInputScheme : CurlInputScheme
the Nix store directly, since there is little deduplication the Nix store directly, since there is little deduplication
benefit in using the Git cache for single big files like benefit in using the Git cache for single big files like
tarballs. */ tarballs. */
auto file = downloadFile(store, settings, getStrAttr(input.attrs, "url"), input.getName()); auto file = downloadFile(store, *input.settings, getStrAttr(input.attrs, "url"), input.getName());
auto narHash = store.queryPathInfo(file.storePath)->narHash; auto narHash = store->queryPathInfo(file.storePath)->narHash;
input.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true)); input.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true));
auto accessor = ref{store.getFSAccessor(file.storePath)}; auto accessor = ref{store->getFSAccessor(file.storePath)};
accessor->setPathDisplay("«" + input.to_string() + "»"); accessor->setPathDisplay("«" + input.to_string() + "»");
@@ -442,34 +368,6 @@ struct TarballInputScheme : CurlInputScheme
return "tarball"; return "tarball";
} }
std::string schemeDescription() const override
{
return stripIndentation(R"(
Download a tar archive and extract it into the Nix store.
This has the same underlying implementation as [`builtins.fetchTarball`](@docroot@/language/builtins.md#builtins-fetchTarball)
)");
}
const std::map<std::string, AttributeInfo> & allowedAttrs() const override
{
static const std::map<std::string, AttributeInfo> attrs = [] {
auto attrs = CurlInputScheme::allowedAttrsImpl();
// Override the "url" attribute to add tarball-specific example
attrs["url"].doc = R"(
> **Example**
>
> ```nix
> fetchTree {
> type = "tarball";
> url = "https://github.com/NixOS/nixpkgs/tarball/nixpkgs-23.11";
> }
> ```
)";
return attrs;
}();
return attrs;
}
bool isValidURL(const ParsedURL & url, bool requireTree) const override bool isValidURL(const ParsedURL & url, bool requireTree) const override
{ {
auto parsedUrlScheme = parseUrlScheme(url.scheme); auto parsedUrlScheme = parseUrlScheme(url.scheme);
@@ -479,15 +377,15 @@ struct TarballInputScheme : CurlInputScheme
: (requireTree || hasTarballExtension(url))); : (requireTree || hasTarballExtension(url)));
} }
std::pair<ref<SourceAccessor>, Input> std::pair<ref<SourceAccessor>, Input> getAccessor(ref<Store> store, const Input & _input) const override
getAccessor(const Settings & settings, Store & store, const Input & _input) const override
{ {
auto input(_input); auto input(_input);
auto result = downloadTarball_(settings, getStrAttr(input.attrs, "url"), {}, "«" + input.to_string() + "»"); auto result =
downloadTarball_(*input.settings, getStrAttr(input.attrs, "url"), {}, "«" + input.to_string() + "»");
if (result.immutableUrl) { if (result.immutableUrl) {
auto immutableInput = Input::fromURL(settings, *result.immutableUrl); auto immutableInput = Input::fromURL(*input.settings, *result.immutableUrl);
// FIXME: would be nice to support arbitrary flakerefs // FIXME: would be nice to support arbitrary flakerefs
// here, e.g. git flakes. // here, e.g. git flakes.
if (immutableInput.getType() != "tarball") if (immutableInput.getType() != "tarball")
@@ -500,12 +398,14 @@ struct TarballInputScheme : CurlInputScheme
input.attrs.insert_or_assign( input.attrs.insert_or_assign(
"narHash", "narHash",
settings.getTarballCache()->treeHashToNarHash(settings, result.treeHash).to_string(HashFormat::SRI, true)); input.settings->getTarballCache()
->treeHashToNarHash(*input.settings, result.treeHash)
.to_string(HashFormat::SRI, true));
return {result.accessor, input}; return {result.accessor, input};
} }
std::optional<std::string> getFingerprint(Store & store, const Input & input) const override std::optional<std::string> getFingerprint(ref<Store> store, const Input & input) const override
{ {
if (auto narHash = input.getNarHash()) if (auto narHash = input.getNarHash())
return narHash->to_string(HashFormat::SRI, true); return narHash->to_string(HashFormat::SRI, true);
+1 -1
View File
@@ -38,7 +38,7 @@ PrimOp getFlake(const Settings & settings)
std::string flakeRefS( std::string flakeRefS(
state.forceStringNoCtx(*args[0], pos, "while evaluating the argument passed to builtins.getFlake")); state.forceStringNoCtx(*args[0], pos, "while evaluating the argument passed to builtins.getFlake"));
auto flakeRef = nix::parseFlakeRef(state.fetchSettings, flakeRefS, {}, true); auto flakeRef = nix::parseFlakeRef(state.fetchSettings, flakeRefS, {}, true);
if (state.settings.pureEval && !flakeRef.input.isLocked(state.fetchSettings)) if (state.settings.pureEval && !flakeRef.input.isLocked())
throw Error( throw Error(
"cannot call 'getFlake' on unlocked flake reference '%s', at %s (use --impure to override)", "cannot call 'getFlake' on unlocked flake reference '%s', at %s (use --impure to override)",
flakeRefS, flakeRefS,
+6 -9
View File
@@ -372,8 +372,7 @@ static Flake getFlake(
const InputAttrPath & lockRootAttrPath) const InputAttrPath & lockRootAttrPath)
{ {
// Fetch a lazy tree first. // Fetch a lazy tree first.
auto cachedInput = auto cachedInput = state.inputCache->getAccessor(state.store, originalRef.input, useRegistries);
state.inputCache->getAccessor(state.fetchSettings, *state.store, originalRef.input, useRegistries);
auto subdir = fetchers::maybeGetStrAttr(cachedInput.extraAttrs, "dir").value_or(originalRef.subdir); auto subdir = fetchers::maybeGetStrAttr(cachedInput.extraAttrs, "dir").value_or(originalRef.subdir);
auto resolvedRef = FlakeRef(std::move(cachedInput.resolvedInput), subdir); auto resolvedRef = FlakeRef(std::move(cachedInput.resolvedInput), subdir);
@@ -389,8 +388,7 @@ static Flake getFlake(
debug("refetching input '%s' due to self attribute", newLockedRef); debug("refetching input '%s' due to self attribute", newLockedRef);
// FIXME: need to remove attrs that are invalidated by the changed input attrs, such as 'narHash'. // FIXME: need to remove attrs that are invalidated by the changed input attrs, such as 'narHash'.
newLockedRef.input.attrs.erase("narHash"); newLockedRef.input.attrs.erase("narHash");
auto cachedInput2 = state.inputCache->getAccessor( auto cachedInput2 = state.inputCache->getAccessor(state.store, newLockedRef.input, fetchers::UseRegistries::No);
state.fetchSettings, *state.store, newLockedRef.input, fetchers::UseRegistries::No);
cachedInput.accessor = cachedInput2.accessor; cachedInput.accessor = cachedInput2.accessor;
lockedRef = FlakeRef(std::move(cachedInput2.lockedInput), newLockedRef.subdir); lockedRef = FlakeRef(std::move(cachedInput2.lockedInput), newLockedRef.subdir);
} }
@@ -706,8 +704,7 @@ lockFlake(const Settings & settings, EvalState & state, const FlakeRef & topRef,
this input. */ this input. */
debug("creating new input '%s'", inputAttrPathS); debug("creating new input '%s'", inputAttrPathS);
if (!lockFlags.allowUnlocked && !input.ref->input.isLocked(state.fetchSettings) if (!lockFlags.allowUnlocked && !input.ref->input.isLocked() && !input.ref->input.isRelative())
&& !input.ref->input.isRelative())
throw Error("cannot update unlocked flake input '%s' in pure mode", inputAttrPathS); throw Error("cannot update unlocked flake input '%s' in pure mode", inputAttrPathS);
/* Note: in case of an --override-input, we use /* Note: in case of an --override-input, we use
@@ -756,7 +753,7 @@ lockFlake(const Settings & settings, EvalState & state, const FlakeRef & topRef,
return {*resolvedPath, *input.ref}; return {*resolvedPath, *input.ref};
} else { } else {
auto cachedInput = state.inputCache->getAccessor( auto cachedInput = state.inputCache->getAccessor(
state.fetchSettings, *state.store, input.ref->input, useRegistriesInputs); state.store, input.ref->input, useRegistriesInputs);
auto lockedRef = FlakeRef(std::move(cachedInput.lockedInput), input.ref->subdir); auto lockedRef = FlakeRef(std::move(cachedInput.lockedInput), input.ref->subdir);
@@ -975,7 +972,7 @@ void callFlake(EvalState & state, const LockedFlake & lockedFlake, Value & vRes)
state.callFunction(*vCallFlake, args, vRes, noPos); state.callFunction(*vCallFlake, args, vRes, noPos);
} }
std::optional<Fingerprint> LockedFlake::getFingerprint(Store & store, const fetchers::Settings & fetchSettings) const std::optional<Fingerprint> LockedFlake::getFingerprint(ref<Store> store, const fetchers::Settings & fetchSettings) const
{ {
if (lockFile.isUnlocked(fetchSettings)) if (lockFile.isUnlocked(fetchSettings))
return std::nullopt; return std::nullopt;
@@ -1005,7 +1002,7 @@ Flake::~Flake() {}
ref<eval_cache::EvalCache> openEvalCache(EvalState & state, ref<const LockedFlake> lockedFlake) ref<eval_cache::EvalCache> openEvalCache(EvalState & state, ref<const LockedFlake> lockedFlake)
{ {
auto fingerprint = state.settings.useEvalCache && state.settings.pureEval auto fingerprint = state.settings.useEvalCache && state.settings.pureEval
? lockedFlake->getFingerprint(*state.store, state.fetchSettings) ? lockedFlake->getFingerprint(state.store, state.fetchSettings)
: std::nullopt; : std::nullopt;
auto rootLoader = [&state, lockedFlake]() { auto rootLoader = [&state, lockedFlake]() {
/* For testing whether the evaluation cache is /* For testing whether the evaluation cache is
+4 -6
View File
@@ -64,10 +64,9 @@ std::ostream & operator<<(std::ostream & str, const FlakeRef & flakeRef)
return str; return str;
} }
FlakeRef FlakeRef FlakeRef::resolve(ref<Store> store, fetchers::UseRegistries useRegistries) const
FlakeRef::resolve(const fetchers::Settings & fetchSettings, Store & store, fetchers::UseRegistries useRegistries) const
{ {
auto [input2, extraAttrs] = lookupInRegistries(fetchSettings, store, input, useRegistries); auto [input2, extraAttrs] = lookupInRegistries(store, input, useRegistries);
return FlakeRef(std::move(input2), fetchers::maybeGetStrAttr(extraAttrs, "dir").value_or(subdir)); return FlakeRef(std::move(input2), fetchers::maybeGetStrAttr(extraAttrs, "dir").value_or(subdir));
} }
@@ -288,10 +287,9 @@ FlakeRef FlakeRef::fromAttrs(const fetchers::Settings & fetchSettings, const fet
fetchers::maybeGetStrAttr(attrs, "dir").value_or("")); fetchers::maybeGetStrAttr(attrs, "dir").value_or(""));
} }
std::pair<ref<SourceAccessor>, FlakeRef> std::pair<ref<SourceAccessor>, FlakeRef> FlakeRef::lazyFetch(ref<Store> store) const
FlakeRef::lazyFetch(const fetchers::Settings & fetchSettings, Store & store) const
{ {
auto [accessor, lockedInput] = input.getAccessor(fetchSettings, store); auto [accessor, lockedInput] = input.getAccessor(store);
return {accessor, FlakeRef(std::move(lockedInput), subdir)}; return {accessor, FlakeRef(std::move(lockedInput), subdir)};
} }
+1 -1
View File
@@ -135,7 +135,7 @@ struct LockedFlake
*/ */
std::map<ref<Node>, SourcePath> nodePaths; std::map<ref<Node>, SourcePath> nodePaths;
std::optional<Fingerprint> getFingerprint(Store & store, const fetchers::Settings & fetchSettings) const; std::optional<Fingerprint> getFingerprint(ref<Store> store, const fetchers::Settings & fetchSettings) const;
}; };
struct LockFlags struct LockFlags
+2 -5
View File
@@ -71,14 +71,11 @@ struct FlakeRef
fetchers::Attrs toAttrs() const; fetchers::Attrs toAttrs() const;
FlakeRef resolve( FlakeRef resolve(ref<Store> store, fetchers::UseRegistries useRegistries = fetchers::UseRegistries::All) const;
const fetchers::Settings & fetchSettings,
Store & store,
fetchers::UseRegistries useRegistries = fetchers::UseRegistries::All) const;
static FlakeRef fromAttrs(const fetchers::Settings & fetchSettings, const fetchers::Attrs & attrs); static FlakeRef fromAttrs(const fetchers::Settings & fetchSettings, const fetchers::Attrs & attrs);
std::pair<ref<SourceAccessor>, FlakeRef> lazyFetch(const fetchers::Settings & fetchSettings, Store & store) const; std::pair<ref<SourceAccessor>, FlakeRef> lazyFetch(ref<Store> store) const;
/** /**
* Canonicalize a flakeref for the purpose of comparing "old" and * Canonicalize a flakeref for the purpose of comparing "old" and
+2 -2
View File
@@ -74,7 +74,7 @@ LockedNode::LockedNode(const fetchers::Settings & fetchSettings, const nlohmann:
, parentInputAttrPath( , parentInputAttrPath(
json.find("parent") != json.end() ? (std::optional<InputAttrPath>) json["parent"] : std::nullopt) json.find("parent") != json.end() ? (std::optional<InputAttrPath>) json["parent"] : std::nullopt)
{ {
if (!lockedRef.input.isLocked(fetchSettings) && !lockedRef.input.isRelative()) { if (!lockedRef.input.isLocked() && !lockedRef.input.isRelative()) {
if (lockedRef.input.getNarHash()) if (lockedRef.input.getNarHash())
warn( warn(
"Lock file entry '%s' is unlocked (e.g. lacks a Git revision) but is checked by NAR hash. " "Lock file entry '%s' is unlocked (e.g. lacks a Git revision) but is checked by NAR hash. "
@@ -282,7 +282,7 @@ std::optional<FlakeRef> LockFile::isUnlocked(const fetchers::Settings & fetchSet
latter case, we can verify the input but we may not be able to latter case, we can verify the input but we may not be able to
fetch it from anywhere. */ fetch it from anywhere. */
auto isConsideredLocked = [&](const fetchers::Input & input) { auto isConsideredLocked = [&](const fetchers::Input & input) {
return input.isLocked(fetchSettings) || (fetchSettings.allowDirtyLocks && input.getNarHash()); return input.isLocked() || (fetchSettings.allowDirtyLocks && input.getNarHash());
}; };
for (auto & i : nodes) { for (auto & i : nodes) {
-2
View File
@@ -35,8 +35,6 @@ include_dirs = [ include_directories('.') ]
headers = files( headers = files(
'nix_api_store.h', 'nix_api_store.h',
'nix_api_store/derivation.h',
'nix_api_store/store_path.h',
) )
# TODO don't install this once tests don't use it and/or move the header into `libstore`, non-`c` # TODO don't install this once tests don't use it and/or move the header into `libstore`, non-`c`
+36 -2
View File
@@ -12,8 +12,6 @@
*/ */
#include "nix_api_util.h" #include "nix_api_util.h"
#include "nix_api_store/store_path.h"
#include "nix_api_store/derivation.h"
#include <stdbool.h> #include <stdbool.h>
#ifdef __cplusplus #ifdef __cplusplus
@@ -23,6 +21,10 @@ extern "C" {
/** @brief Reference to a Nix store */ /** @brief Reference to a Nix store */
typedef struct Store Store; typedef struct Store Store;
/** @brief Nix store path */
typedef struct StorePath StorePath;
/** @brief Nix Derivation */
typedef struct nix_derivation nix_derivation;
/** /**
* @brief Initializes the Nix store library * @brief Initializes the Nix store library
@@ -116,6 +118,30 @@ nix_store_get_storedir(nix_c_context * context, Store * store, nix_get_string_ca
*/ */
StorePath * nix_store_parse_path(nix_c_context * context, Store * store, const char * path); StorePath * nix_store_parse_path(nix_c_context * context, Store * store, const char * path);
/**
* @brief Get the path name (e.g. "name" in /nix/store/...-name)
*
* @param[in] store_path the path to get the name from
* @param[in] callback called with the name
* @param[in] user_data arbitrary data, passed to the callback when it's called.
*/
void nix_store_path_name(const StorePath * store_path, nix_get_string_callback callback, void * user_data);
/**
* @brief Copy a StorePath
*
* @param[in] p the path to copy
* @return a new StorePath
*/
StorePath * nix_store_path_clone(const StorePath * p);
/** @brief Deallocate a StorePath
*
* Does not fail.
* @param[in] p the path to free
*/
void nix_store_path_free(StorePath * p);
/** /**
* @brief Check if a StorePath is valid (i.e. that corresponding store object and its closure of references exists in * @brief Check if a StorePath is valid (i.e. that corresponding store object and its closure of references exists in
* the store) * the store)
@@ -203,6 +229,14 @@ nix_derivation * nix_derivation_from_json(nix_c_context * context, Store * store
*/ */
StorePath * nix_add_derivation(nix_c_context * context, Store * store, nix_derivation * derivation); StorePath * nix_add_derivation(nix_c_context * context, Store * store, nix_derivation * derivation);
/**
* @brief Deallocate a `nix_derivation`
*
* Does not fail.
* @param[in] drv the derivation to free
*/
void nix_derivation_free(nix_derivation * drv);
/** /**
* @brief Copy the closure of `path` from `srcStore` to `dstStore`. * @brief Copy the closure of `path` from `srcStore` to `dstStore`.
* *
-38
View File
@@ -1,38 +0,0 @@
#ifndef NIX_API_STORE_DERIVATION_H
#define NIX_API_STORE_DERIVATION_H
/**
* @defgroup libstore_derivation Derivation
* @ingroup libstore
* @brief Derivation operations that don't require a Store
* @{
*/
/** @file
* @brief Derivation operations
*/
#include "nix_api_util.h"
#ifdef __cplusplus
extern "C" {
#endif
// cffi start
/** @brief Nix Derivation */
typedef struct nix_derivation nix_derivation;
/**
* @brief Deallocate a `nix_derivation`
*
* Does not fail.
* @param[in] drv the derivation to free
*/
void nix_derivation_free(nix_derivation * drv);
// cffi end
#ifdef __cplusplus
}
#endif
/**
* @}
*/
#endif // NIX_API_STORE_DERIVATION_H
-54
View File
@@ -1,54 +0,0 @@
#ifndef NIX_API_STORE_STORE_PATH_H
#define NIX_API_STORE_STORE_PATH_H
/**
* @defgroup libstore_storepath StorePath
* @ingroup libstore
* @brief Store path operations that don't require a Store
* @{
*/
/** @file
* @brief Store path operations
*/
#include "nix_api_util.h"
#ifdef __cplusplus
extern "C" {
#endif
// cffi start
/** @brief Nix store path */
typedef struct StorePath StorePath;
/**
* @brief Copy a StorePath
*
* @param[in] p the path to copy
* @return a new StorePath
*/
StorePath * nix_store_path_clone(const StorePath * p);
/** @brief Deallocate a StorePath
*
* Does not fail.
* @param[in] p the path to free
*/
void nix_store_path_free(StorePath * p);
/**
* @brief Get the path name (e.g. "<name>" in /nix/store/<hash>-<name>)
*
* @param[in] store_path the path to get the name from
* @param[in] callback called with the name
* @param[in] user_data arbitrary data, passed to the callback when it's called.
*/
void nix_store_path_name(const StorePath * store_path, nix_get_string_callback callback, void * user_data);
// cffi end
#ifdef __cplusplus
}
#endif
/**
* @}
*/
#endif // NIX_API_STORE_STORE_PATH_H
@@ -4,13 +4,10 @@
"allowSubstitutes": false, "allowSubstitutes": false,
"exportReferencesGraph": { "exportReferencesGraph": {
"refs1": [ "refs1": [
{ "/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9"
"drvPath": "j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv",
"output": "out"
}
], ],
"refs2": [ "refs2": [
"qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv" "/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv"
] ]
}, },
"impureEnvVars": [ "impureEnvVars": [
@@ -23,36 +20,18 @@
"outputChecks": { "outputChecks": {
"forAllOutputs": { "forAllOutputs": {
"allowedReferences": [ "allowedReferences": [
{ "/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9"
"drvPath": "j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv",
"output": "out"
}
], ],
"allowedRequisites": [ "allowedRequisites": [
{ "/0nr45p69vn6izw9446wsh9bng9nndhvn19kpsm4n96a5mycw0s4z",
"drvPath": "self", "bin"
"output": "bin"
},
{
"drvPath": "j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv",
"output": "dev"
}
], ],
"disallowedReferences": [ "disallowedReferences": [
{ "/0nyw57wm2iicnm9rglvjmbci3ikmcp823czdqdzdcgsnnwqps71g",
"drvPath": "self", "dev"
"output": "dev"
},
{
"drvPath": "qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv",
"output": "out"
}
], ],
"disallowedRequisites": [ "disallowedRequisites": [
{ "/07f301yqyz8c6wf6bbbavb2q39j4n8kmcly1s09xadyhgy6x2wr8"
"drvPath": "qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv",
"output": "dev"
}
], ],
"ignoreSelfRefs": true, "ignoreSelfRefs": true,
"maxClosureSize": null, "maxClosureSize": null,
@@ -4,13 +4,10 @@
"allowSubstitutes": false, "allowSubstitutes": false,
"exportReferencesGraph": { "exportReferencesGraph": {
"refs1": [ "refs1": [
{ "/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9"
"drvPath": "j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv",
"output": "out"
}
], ],
"refs2": [ "refs2": [
"qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv" "/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv"
] ]
}, },
"impureEnvVars": [ "impureEnvVars": [
@@ -26,20 +23,11 @@
"allowedReferences": null, "allowedReferences": null,
"allowedRequisites": null, "allowedRequisites": null,
"disallowedReferences": [ "disallowedReferences": [
{ "/0nyw57wm2iicnm9rglvjmbci3ikmcp823czdqdzdcgsnnwqps71g",
"drvPath": "self", "dev"
"output": "dev"
},
{
"drvPath": "qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv",
"output": "out"
}
], ],
"disallowedRequisites": [ "disallowedRequisites": [
{ "/07f301yqyz8c6wf6bbbavb2q39j4n8kmcly1s09xadyhgy6x2wr8"
"drvPath": "qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv",
"output": "dev"
}
], ],
"ignoreSelfRefs": false, "ignoreSelfRefs": false,
"maxClosureSize": null, "maxClosureSize": null,
@@ -56,20 +44,11 @@
}, },
"out": { "out": {
"allowedReferences": [ "allowedReferences": [
{ "/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9"
"drvPath": "j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv",
"output": "out"
}
], ],
"allowedRequisites": [ "allowedRequisites": [
{ "/0nr45p69vn6izw9446wsh9bng9nndhvn19kpsm4n96a5mycw0s4z",
"drvPath": "self", "bin"
"output": "bin"
},
{
"drvPath": "j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv",
"output": "dev"
}
], ],
"disallowedReferences": [], "disallowedReferences": [],
"disallowedRequisites": [], "disallowedRequisites": [],
@@ -4,10 +4,10 @@
"allowSubstitutes": false, "allowSubstitutes": false,
"exportReferencesGraph": { "exportReferencesGraph": {
"refs1": [ "refs1": [
"p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo" "/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo"
], ],
"refs2": [ "refs2": [
"vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv" "/nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv"
] ]
}, },
"impureEnvVars": [ "impureEnvVars": [
@@ -20,24 +20,18 @@
"outputChecks": { "outputChecks": {
"forAllOutputs": { "forAllOutputs": {
"allowedReferences": [ "allowedReferences": [
"p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo" "/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo"
], ],
"allowedRequisites": [ "allowedRequisites": [
{ "/nix/store/z0rjzy29v9k5qa4nqpykrbzirj7sd43v-foo-dev",
"drvPath": "self", "bin"
"output": "bin"
},
"z0rjzy29v9k5qa4nqpykrbzirj7sd43v-foo-dev"
], ],
"disallowedReferences": [ "disallowedReferences": [
{ "/nix/store/r5cff30838majxk5mp3ip2diffi8vpaj-bar",
"drvPath": "self", "dev"
"output": "dev"
},
"r5cff30838majxk5mp3ip2diffi8vpaj-bar"
], ],
"disallowedRequisites": [ "disallowedRequisites": [
"9b61w26b4avv870dw0ymb6rw4r1hzpws-bar-dev" "/nix/store/9b61w26b4avv870dw0ymb6rw4r1hzpws-bar-dev"
], ],
"ignoreSelfRefs": true, "ignoreSelfRefs": true,
"maxClosureSize": null, "maxClosureSize": null,
@@ -4,10 +4,10 @@
"allowSubstitutes": false, "allowSubstitutes": false,
"exportReferencesGraph": { "exportReferencesGraph": {
"refs1": [ "refs1": [
"p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo" "/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo"
], ],
"refs2": [ "refs2": [
"vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv" "/nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv"
] ]
}, },
"impureEnvVars": [ "impureEnvVars": [
@@ -23,14 +23,11 @@
"allowedReferences": null, "allowedReferences": null,
"allowedRequisites": null, "allowedRequisites": null,
"disallowedReferences": [ "disallowedReferences": [
{ "/nix/store/r5cff30838majxk5mp3ip2diffi8vpaj-bar",
"drvPath": "self", "dev"
"output": "dev"
},
"r5cff30838majxk5mp3ip2diffi8vpaj-bar"
], ],
"disallowedRequisites": [ "disallowedRequisites": [
"9b61w26b4avv870dw0ymb6rw4r1hzpws-bar-dev" "/nix/store/9b61w26b4avv870dw0ymb6rw4r1hzpws-bar-dev"
], ],
"ignoreSelfRefs": false, "ignoreSelfRefs": false,
"maxClosureSize": null, "maxClosureSize": null,
@@ -47,14 +44,11 @@
}, },
"out": { "out": {
"allowedReferences": [ "allowedReferences": [
"p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo" "/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo"
], ],
"allowedRequisites": [ "allowedRequisites": [
{ "/nix/store/z0rjzy29v9k5qa4nqpykrbzirj7sd43v-foo-dev",
"drvPath": "self", "bin"
"output": "bin"
},
"z0rjzy29v9k5qa4nqpykrbzirj7sd43v-foo-dev"
], ],
"disallowedReferences": [], "disallowedReferences": [],
"disallowedRequisites": [], "disallowedRequisites": [],
+69 -96
View File
@@ -3,7 +3,7 @@
#include "nix/util/experimental-features.hh" #include "nix/util/experimental-features.hh"
#include "nix/store/derivations.hh" #include "nix/store/derivations.hh"
#include "nix/store/derived-path.hh" #include "nix/store/derivations.hh"
#include "nix/store/derivation-options.hh" #include "nix/store/derivation-options.hh"
#include "nix/store/parsed-derivations.hh" #include "nix/store/parsed-derivations.hh"
#include "nix/util/types.hh" #include "nix/util/types.hh"
@@ -17,7 +17,7 @@ namespace nix {
using namespace nlohmann; using namespace nlohmann;
class DerivationAdvancedAttrsTest : public JsonCharacterizationTest<Derivation>, class DerivationAdvancedAttrsTest : public JsonCharacterizationTest<Derivation>,
public JsonCharacterizationTest<DerivationOptions<SingleDerivedPath>>, public JsonCharacterizationTest<DerivationOptions>,
public LibStoreTest public LibStoreTest
{ {
protected: protected:
@@ -42,8 +42,7 @@ public:
{ {
this->readTest(fileName, [&](auto encoded) { this->readTest(fileName, [&](auto encoded) {
auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings); auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings);
auto options = derivationOptionsFromStructuredAttrs( DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, got.structuredAttrs);
*this->store, got.inputDrvs, got.env, get(got.structuredAttrs), true, this->mockXpSettings);
EXPECT_EQ(options.getRequiredSystemFeatures(got), expectedFeatures); EXPECT_EQ(options.getRequiredSystemFeatures(got), expectedFeatures);
}); });
} }
@@ -52,14 +51,11 @@ public:
* Helper function to test DerivationOptions parsing and comparison * Helper function to test DerivationOptions parsing and comparison
*/ */
void testDerivationOptions( void testDerivationOptions(
const std::string & fileName, const std::string & fileName, const DerivationOptions & expected, const StringSet & expectedSystemFeatures)
const DerivationOptions<SingleDerivedPath> & expected,
const StringSet & expectedSystemFeatures)
{ {
this->readTest(fileName, [&](auto encoded) { this->readTest(fileName, [&](auto encoded) {
auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings); auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings);
auto options = derivationOptionsFromStructuredAttrs( DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, got.structuredAttrs);
*this->store, got.inputDrvs, got.env, get(got.structuredAttrs), true, this->mockXpSettings);
EXPECT_EQ(options, expected); EXPECT_EQ(options, expected);
EXPECT_EQ(options.getRequiredSystemFeatures(got), expectedSystemFeatures); EXPECT_EQ(options.getRequiredSystemFeatures(got), expectedSystemFeatures);
@@ -135,38 +131,22 @@ TEST_ATERM_JSON(advancedAttributes_structuredAttrs_defaults, "advanced-attribute
* Since these are both repeated and sensative opaque values, it makes * Since these are both repeated and sensative opaque values, it makes
* sense to give them names in this file. * sense to give them names in this file.
*/ */
static SingleDerivedPath static std::string pathFoo = "/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo",
pathFoo = SingleDerivedPath::Opaque{StorePath{"p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo"}}, pathFooDev = "/nix/store/z0rjzy29v9k5qa4nqpykrbzirj7sd43v-foo-dev",
pathFooDev = SingleDerivedPath::Opaque{StorePath{"z0rjzy29v9k5qa4nqpykrbzirj7sd43v-foo-dev"}}, pathBar = "/nix/store/r5cff30838majxk5mp3ip2diffi8vpaj-bar",
pathBar = SingleDerivedPath::Opaque{StorePath{"r5cff30838majxk5mp3ip2diffi8vpaj-bar"}}, pathBarDev = "/nix/store/9b61w26b4avv870dw0ymb6rw4r1hzpws-bar-dev",
pathBarDev = SingleDerivedPath::Opaque{StorePath{"9b61w26b4avv870dw0ymb6rw4r1hzpws-bar-dev"}}, pathBarDrvIA = "/nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv",
pathBarDrvIA = SingleDerivedPath::Opaque{StorePath{"vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv"}}, pathBarDrvCA = "/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv",
pathBarDrvCA = SingleDerivedPath::Opaque{StorePath{"qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv"}}, placeholderFoo = "/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9",
placeholderFoo = placeholderFooDev = "/0nr45p69vn6izw9446wsh9bng9nndhvn19kpsm4n96a5mycw0s4z",
SingleDerivedPath::Built{ placeholderBar = "/0nyw57wm2iicnm9rglvjmbci3ikmcp823czdqdzdcgsnnwqps71g",
.drvPath = makeConstantStorePathRef(StorePath{"j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv"}), placeholderBarDev = "/07f301yqyz8c6wf6bbbavb2q39j4n8kmcly1s09xadyhgy6x2wr8";
.output = "out",
},
placeholderFooDev =
SingleDerivedPath::Built{
.drvPath = makeConstantStorePathRef(StorePath{"j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv"}),
.output = "dev",
},
placeholderBar =
SingleDerivedPath::Built{
.drvPath = makeConstantStorePathRef(StorePath{"qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv"}),
.output = "out",
},
placeholderBarDev = SingleDerivedPath::Built{
.drvPath = makeConstantStorePathRef(StorePath{"qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv"}),
.output = "dev",
};
using ExportReferencesMap = decltype(DerivationOptions<SingleDerivedPath>::exportReferencesGraph); using ExportReferencesMap = decltype(DerivationOptions::exportReferencesGraph);
static const DerivationOptions<SingleDerivedPath> advancedAttributes_defaults = { static const DerivationOptions advancedAttributes_defaults = {
.outputChecks = .outputChecks =
DerivationOptions<SingleDerivedPath>::OutputChecks{ DerivationOptions::OutputChecks{
.ignoreSelfRefs = true, .ignoreSelfRefs = true,
}, },
.unsafeDiscardReferences = {}, .unsafeDiscardReferences = {},
@@ -187,8 +167,7 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_defaults)
this->readTest("advanced-attributes-defaults.drv", [&](auto encoded) { this->readTest("advanced-attributes-defaults.drv", [&](auto encoded) {
auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings); auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings);
auto options = derivationOptionsFromStructuredAttrs( DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, got.structuredAttrs);
*this->store, got.inputDrvs, got.env, get(got.structuredAttrs), true, this->mockXpSettings);
EXPECT_TRUE(!got.structuredAttrs); EXPECT_TRUE(!got.structuredAttrs);
@@ -213,9 +192,9 @@ TEST_F(CaDerivationAdvancedAttrsTest, advancedAttributes_defaults)
TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes) TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes)
{ {
DerivationOptions<SingleDerivedPath> expected = { DerivationOptions expected = {
.outputChecks = .outputChecks =
DerivationOptions<SingleDerivedPath>::OutputChecks{ DerivationOptions::OutputChecks{
.ignoreSelfRefs = true, .ignoreSelfRefs = true,
}, },
.unsafeDiscardReferences = {}, .unsafeDiscardReferences = {},
@@ -233,13 +212,12 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes)
this->readTest("advanced-attributes.drv", [&](auto encoded) { this->readTest("advanced-attributes.drv", [&](auto encoded) {
auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings); auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings);
auto options = derivationOptionsFromStructuredAttrs( DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, got.structuredAttrs);
*this->store, got.inputDrvs, got.env, get(got.structuredAttrs), true, this->mockXpSettings);
EXPECT_TRUE(!got.structuredAttrs); EXPECT_TRUE(!got.structuredAttrs);
// Reset fields that vary between test cases to enable whole-object comparison // Reset fields that vary between test cases to enable whole-object comparison
options.outputChecks = DerivationOptions<SingleDerivedPath>::OutputChecks{.ignoreSelfRefs = true}; options.outputChecks = DerivationOptions::OutputChecks{.ignoreSelfRefs = true};
options.exportReferencesGraph = {}; options.exportReferencesGraph = {};
EXPECT_EQ(options, expected); EXPECT_EQ(options, expected);
@@ -249,14 +227,14 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes)
}); });
}; };
DerivationOptions<SingleDerivedPath> advancedAttributes_ia = { DerivationOptions advancedAttributes_ia = {
.outputChecks = .outputChecks =
DerivationOptions<SingleDerivedPath>::OutputChecks{ DerivationOptions::OutputChecks{
.ignoreSelfRefs = true, .ignoreSelfRefs = true,
.allowedReferences = std::set<DrvRef<SingleDerivedPath>>{pathFoo}, .allowedReferences = StringSet{pathFoo},
.disallowedReferences = std::set<DrvRef<SingleDerivedPath>>{pathBar, OutputName{"dev"}}, .disallowedReferences = StringSet{pathBar, "dev"},
.allowedRequisites = std::set<DrvRef<SingleDerivedPath>>{pathFooDev, OutputName{"bin"}}, .allowedRequisites = StringSet{pathFooDev, "bin"},
.disallowedRequisites = std::set<DrvRef<SingleDerivedPath>>{pathBarDev}, .disallowedRequisites = StringSet{pathBarDev},
}, },
.unsafeDiscardReferences = {}, .unsafeDiscardReferences = {},
.passAsFile = {}, .passAsFile = {},
@@ -279,14 +257,14 @@ TEST_F(DerivationAdvancedAttrsTest, advancedAttributes_ia)
testDerivationOptions("advanced-attributes.drv", advancedAttributes_ia, {"rainbow", "uid-range"}); testDerivationOptions("advanced-attributes.drv", advancedAttributes_ia, {"rainbow", "uid-range"});
}; };
DerivationOptions<SingleDerivedPath> advancedAttributes_ca = { DerivationOptions advancedAttributes_ca = {
.outputChecks = .outputChecks =
DerivationOptions<SingleDerivedPath>::OutputChecks{ DerivationOptions::OutputChecks{
.ignoreSelfRefs = true, .ignoreSelfRefs = true,
.allowedReferences = std::set<DrvRef<SingleDerivedPath>>{placeholderFoo}, .allowedReferences = StringSet{placeholderFoo},
.disallowedReferences = std::set<DrvRef<SingleDerivedPath>>{placeholderBar, OutputName{"dev"}}, .disallowedReferences = StringSet{placeholderBar, "dev"},
.allowedRequisites = std::set<DrvRef<SingleDerivedPath>>{placeholderFooDev, OutputName{"bin"}}, .allowedRequisites = StringSet{placeholderFooDev, "bin"},
.disallowedRequisites = std::set<DrvRef<SingleDerivedPath>>{placeholderBarDev}, .disallowedRequisites = StringSet{placeholderBarDev},
}, },
.unsafeDiscardReferences = {}, .unsafeDiscardReferences = {},
.passAsFile = {}, .passAsFile = {},
@@ -309,8 +287,8 @@ TEST_F(CaDerivationAdvancedAttrsTest, advancedAttributes)
testDerivationOptions("advanced-attributes.drv", advancedAttributes_ca, {"rainbow", "uid-range", "ca-derivations"}); testDerivationOptions("advanced-attributes.drv", advancedAttributes_ca, {"rainbow", "uid-range", "ca-derivations"});
}; };
DerivationOptions<SingleDerivedPath> advancedAttributes_structuredAttrs_defaults = { DerivationOptions advancedAttributes_structuredAttrs_defaults = {
.outputChecks = std::map<std::string, DerivationOptions<SingleDerivedPath>::OutputChecks>{}, .outputChecks = std::map<std::string, DerivationOptions::OutputChecks>{},
.unsafeDiscardReferences = {}, .unsafeDiscardReferences = {},
.passAsFile = {}, .passAsFile = {},
.exportReferencesGraph = {}, .exportReferencesGraph = {},
@@ -329,8 +307,7 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_structuredAttrs_d
this->readTest("advanced-attributes-structured-attrs-defaults.drv", [&](auto encoded) { this->readTest("advanced-attributes-structured-attrs-defaults.drv", [&](auto encoded) {
auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings); auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings);
auto options = derivationOptionsFromStructuredAttrs( DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, got.structuredAttrs);
*this->store, got.inputDrvs, got.env, get(got.structuredAttrs), true, this->mockXpSettings);
EXPECT_TRUE(got.structuredAttrs); EXPECT_TRUE(got.structuredAttrs);
@@ -355,11 +332,11 @@ TEST_F(CaDerivationAdvancedAttrsTest, advancedAttributes_structuredAttrs_default
TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_structuredAttrs) TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_structuredAttrs)
{ {
DerivationOptions<SingleDerivedPath> expected = { DerivationOptions expected = {
.outputChecks = .outputChecks =
std::map<std::string, DerivationOptions<SingleDerivedPath>::OutputChecks>{ std::map<std::string, DerivationOptions::OutputChecks>{
{"dev", {"dev",
DerivationOptions<SingleDerivedPath>::OutputChecks{ DerivationOptions::OutputChecks{
.maxSize = 789, .maxSize = 789,
.maxClosureSize = 5909, .maxClosureSize = 5909,
}}, }},
@@ -380,8 +357,7 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_structuredAttrs)
this->readTest("advanced-attributes-structured-attrs.drv", [&](auto encoded) { this->readTest("advanced-attributes-structured-attrs.drv", [&](auto encoded) {
auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings); auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings);
auto options = derivationOptionsFromStructuredAttrs( DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, got.structuredAttrs);
*this->store, got.inputDrvs, got.env, get(got.structuredAttrs), true, this->mockXpSettings);
EXPECT_TRUE(got.structuredAttrs); EXPECT_TRUE(got.structuredAttrs);
@@ -389,8 +365,7 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_structuredAttrs)
{ {
// Delete all keys but "dev" in options.outputChecks // Delete all keys but "dev" in options.outputChecks
auto * outputChecksMapP = auto * outputChecksMapP =
std::get_if<std::map<std::string, DerivationOptions<SingleDerivedPath>::OutputChecks>>( std::get_if<std::map<std::string, DerivationOptions::OutputChecks>>(&options.outputChecks);
&options.outputChecks);
ASSERT_TRUE(outputChecksMapP); ASSERT_TRUE(outputChecksMapP);
auto & outputChecksMap = *outputChecksMapP; auto & outputChecksMap = *outputChecksMapP;
auto devEntry = outputChecksMap.find("dev"); auto devEntry = outputChecksMap.find("dev");
@@ -410,21 +385,21 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_structuredAttrs)
}); });
}; };
DerivationOptions<SingleDerivedPath> advancedAttributes_structuredAttrs_ia = { DerivationOptions advancedAttributes_structuredAttrs_ia = {
.outputChecks = .outputChecks =
std::map<std::string, DerivationOptions<SingleDerivedPath>::OutputChecks>{ std::map<std::string, DerivationOptions::OutputChecks>{
{"out", {"out",
DerivationOptions<SingleDerivedPath>::OutputChecks{ DerivationOptions::OutputChecks{
.allowedReferences = std::set<DrvRef<SingleDerivedPath>>{pathFoo}, .allowedReferences = StringSet{pathFoo},
.allowedRequisites = std::set<DrvRef<SingleDerivedPath>>{pathFooDev, OutputName{"bin"}}, .allowedRequisites = StringSet{pathFooDev, "bin"},
}}, }},
{"bin", {"bin",
DerivationOptions<SingleDerivedPath>::OutputChecks{ DerivationOptions::OutputChecks{
.disallowedReferences = std::set<DrvRef<SingleDerivedPath>>{pathBar, OutputName{"dev"}}, .disallowedReferences = StringSet{pathBar, "dev"},
.disallowedRequisites = std::set<DrvRef<SingleDerivedPath>>{pathBarDev}, .disallowedRequisites = StringSet{pathBarDev},
}}, }},
{"dev", {"dev",
DerivationOptions<SingleDerivedPath>::OutputChecks{ DerivationOptions::OutputChecks{
.maxSize = 789, .maxSize = 789,
.maxClosureSize = 5909, .maxClosureSize = 5909,
}}, }},
@@ -452,21 +427,21 @@ TEST_F(DerivationAdvancedAttrsTest, advancedAttributes_structuredAttrs)
"advanced-attributes-structured-attrs.drv", advancedAttributes_structuredAttrs_ia, {"rainbow", "uid-range"}); "advanced-attributes-structured-attrs.drv", advancedAttributes_structuredAttrs_ia, {"rainbow", "uid-range"});
}; };
DerivationOptions<SingleDerivedPath> advancedAttributes_structuredAttrs_ca = { DerivationOptions advancedAttributes_structuredAttrs_ca = {
.outputChecks = .outputChecks =
std::map<std::string, DerivationOptions<SingleDerivedPath>::OutputChecks>{ std::map<std::string, DerivationOptions::OutputChecks>{
{"out", {"out",
DerivationOptions<SingleDerivedPath>::OutputChecks{ DerivationOptions::OutputChecks{
.allowedReferences = std::set<DrvRef<SingleDerivedPath>>{placeholderFoo}, .allowedReferences = StringSet{placeholderFoo},
.allowedRequisites = std::set<DrvRef<SingleDerivedPath>>{placeholderFooDev, OutputName{"bin"}}, .allowedRequisites = StringSet{placeholderFooDev, "bin"},
}}, }},
{"bin", {"bin",
DerivationOptions<SingleDerivedPath>::OutputChecks{ DerivationOptions::OutputChecks{
.disallowedReferences = std::set<DrvRef<SingleDerivedPath>>{placeholderBar, OutputName{"dev"}}, .disallowedReferences = StringSet{placeholderBar, "dev"},
.disallowedRequisites = std::set<DrvRef<SingleDerivedPath>>{placeholderBarDev}, .disallowedRequisites = StringSet{placeholderBarDev},
}}, }},
{"dev", {"dev",
DerivationOptions<SingleDerivedPath>::OutputChecks{ DerivationOptions::OutputChecks{
.maxSize = 789, .maxSize = 789,
.maxClosureSize = 5909, .maxClosureSize = 5909,
}}, }},
@@ -496,16 +471,14 @@ TEST_F(CaDerivationAdvancedAttrsTest, advancedAttributes_structuredAttrs)
{"rainbow", "uid-range", "ca-derivations"}); {"rainbow", "uid-range", "ca-derivations"});
}; };
#define TEST_JSON_OPTIONS(FIXUTURE, VAR, VAR2) \ #define TEST_JSON_OPTIONS(FIXUTURE, VAR, VAR2) \
TEST_F(FIXUTURE, DerivationOptions_##VAR##_from_json) \ TEST_F(FIXUTURE, DerivationOptions_##VAR##_from_json) \
{ \ { \
this->JsonCharacterizationTest<DerivationOptions<SingleDerivedPath>>::readJsonTest( \ this->JsonCharacterizationTest<DerivationOptions>::readJsonTest(#VAR, advancedAttributes_##VAR2); \
#VAR, advancedAttributes_##VAR2); \ } \
} \ TEST_F(FIXUTURE, DerivationOptions_##VAR##_to_json) \
TEST_F(FIXUTURE, DerivationOptions_##VAR##_to_json) \ { \
{ \ this->JsonCharacterizationTest<DerivationOptions>::writeJsonTest(#VAR, advancedAttributes_##VAR2); \
this->JsonCharacterizationTest<DerivationOptions<SingleDerivedPath>>::writeJsonTest( \
#VAR, advancedAttributes_##VAR2); \
} }
TEST_JSON_OPTIONS(DerivationAdvancedAttrsTest, defaults, defaults) TEST_JSON_OPTIONS(DerivationAdvancedAttrsTest, defaults, defaults)
+2 -1
View File
@@ -40,7 +40,7 @@ static BuildResult::Success::Status successStatusFromString(std::string_view str
throw Error("unknown built result success status '%s'", str); throw Error("unknown built result success status '%s'", str);
} }
static constexpr std::array<std::pair<BuildResult::Failure::Status, std::string_view>, 12> failureStatusStrings{{ static constexpr std::array<std::pair<BuildResult::Failure::Status, std::string_view>, 13> failureStatusStrings{{
#define ENUM_ENTRY(e) {BuildResult::Failure::e, #e} #define ENUM_ENTRY(e) {BuildResult::Failure::e, #e}
ENUM_ENTRY(PermanentFailure), ENUM_ENTRY(PermanentFailure),
ENUM_ENTRY(InputRejected), ENUM_ENTRY(InputRejected),
@@ -54,6 +54,7 @@ static constexpr std::array<std::pair<BuildResult::Failure::Status, std::string_
ENUM_ENTRY(NotDeterministic), ENUM_ENTRY(NotDeterministic),
ENUM_ENTRY(NoSubstituters), ENUM_ENTRY(NoSubstituters),
ENUM_ENTRY(HashMismatch), ENUM_ENTRY(HashMismatch),
ENUM_ENTRY(Cancelled),
#undef ENUM_ENTRY #undef ENUM_ENTRY
}}; }};
+16 -41
View File
@@ -32,6 +32,14 @@ DerivationBuildingGoal::DerivationBuildingGoal(
, drv{std::make_unique<Derivation>(drv)} , drv{std::make_unique<Derivation>(drv)}
, buildMode(buildMode) , buildMode(buildMode)
{ {
try {
drvOptions =
std::make_unique<DerivationOptions>(DerivationOptions::fromStructuredAttrs(drv.env, drv.structuredAttrs));
} catch (Error & e) {
e.addTrace({}, "while parsing derivation '%s'", worker.store.printStorePath(drvPath));
throw;
}
name = fmt("building derivation '%s'", worker.store.printStorePath(drvPath)); name = fmt("building derivation '%s'", worker.store.printStorePath(drvPath));
trace("created"); trace("created");
@@ -198,38 +206,6 @@ Goal::Co DerivationBuildingGoal::gaveUpOnSubstitution(bool storeDerivation)
Goal::Co DerivationBuildingGoal::tryToBuild() Goal::Co DerivationBuildingGoal::tryToBuild()
{ {
auto drvOptions = [&] {
DerivationOptions<SingleDerivedPath> temp;
try {
temp =
derivationOptionsFromStructuredAttrs(worker.store, drv->inputDrvs, drv->env, get(drv->structuredAttrs));
} catch (Error & e) {
e.addTrace({}, "while parsing derivation '%s'", worker.store.printStorePath(drvPath));
throw;
}
auto res = tryResolve(
temp,
[&](ref<const SingleDerivedPath> drvPath, const std::string & outputName) -> std::optional<StorePath> {
try {
return resolveDerivedPath(
worker.store, SingleDerivedPath::Built{drvPath, outputName}, &worker.evalStore);
} catch (Error &) {
return std::nullopt;
}
});
/* The derivation must have all of its inputs gotten this point,
so the resolution will surely succeed.
(Actually, we shouldn't even enter this goal until we have a
resolved derivation, or derivation with only input addressed
transitive inputs, so this should be a no-opt anyways.)
*/
assert(res);
return *res;
}();
std::map<std::string, InitialOutput> initialOutputs; std::map<std::string, InitialOutput> initialOutputs;
/* Recheck at this point. In particular, whereas before we were /* Recheck at this point. In particular, whereas before we were
@@ -368,13 +344,13 @@ Goal::Co DerivationBuildingGoal::tryToBuild()
/* Don't do a remote build if the derivation has the attribute /* Don't do a remote build if the derivation has the attribute
`preferLocalBuild' set. Also, check and repair modes are only `preferLocalBuild' set. Also, check and repair modes are only
supported for local builds. */ supported for local builds. */
bool buildLocally = (buildMode != bmNormal || drvOptions.willBuildLocally(worker.store, *drv)) bool buildLocally = (buildMode != bmNormal || drvOptions->willBuildLocally(worker.store, *drv))
&& settings.maxBuildJobs.get() != 0; && settings.maxBuildJobs.get() != 0;
if (buildLocally) { if (buildLocally) {
useHook = false; useHook = false;
} else { } else {
switch (tryBuildHook(initialOutputs, drvOptions)) { switch (tryBuildHook(initialOutputs)) {
case rpAccept: case rpAccept:
/* Yes, it has started doing so. Wait until we get /* Yes, it has started doing so. Wait until we get
EOF from the hook. */ EOF from the hook. */
@@ -403,7 +379,7 @@ Goal::Co DerivationBuildingGoal::tryToBuild()
externalBuilder = settings.findExternalDerivationBuilderIfSupported(*drv); externalBuilder = settings.findExternalDerivationBuilderIfSupported(*drv);
if (!externalBuilder && !drvOptions.canBuildLocally(worker.store, *drv)) { if (!externalBuilder && !drvOptions->canBuildLocally(worker.store, *drv)) {
auto msg = auto msg =
fmt("Cannot build '%s'.\n" fmt("Cannot build '%s'.\n"
"Reason: " ANSI_RED "required system or feature not available" ANSI_NORMAL "Reason: " ANSI_RED "required system or feature not available" ANSI_NORMAL
@@ -412,7 +388,7 @@ Goal::Co DerivationBuildingGoal::tryToBuild()
"Current system: '%s' with features {%s}", "Current system: '%s' with features {%s}",
Magenta(worker.store.printStorePath(drvPath)), Magenta(worker.store.printStorePath(drvPath)),
Magenta(drv->platform), Magenta(drv->platform),
concatStringsSep(", ", drvOptions.getRequiredSystemFeatures(*drv)), concatStringsSep(", ", drvOptions->getRequiredSystemFeatures(*drv)),
Magenta(settings.thisSystem), Magenta(settings.thisSystem),
concatStringsSep<StringSet>(", ", worker.store.Store::config.systemFeatures)); concatStringsSep<StringSet>(", ", worker.store.Store::config.systemFeatures));
@@ -610,7 +586,7 @@ Goal::Co DerivationBuildingGoal::tryToBuild()
} }
try { try {
desugaredEnv = DesugaredEnv::create(worker.store, *drv, drvOptions, inputPaths); desugaredEnv = DesugaredEnv::create(worker.store, *drv, *drvOptions, inputPaths);
} catch (BuildError & e) { } catch (BuildError & e) {
outputLocks.unlock(); outputLocks.unlock();
worker.permanentFailure = true; worker.permanentFailure = true;
@@ -621,7 +597,7 @@ Goal::Co DerivationBuildingGoal::tryToBuild()
.drvPath = drvPath, .drvPath = drvPath,
.buildResult = buildResult, .buildResult = buildResult,
.drv = *drv, .drv = *drv,
.drvOptions = drvOptions, .drvOptions = *drvOptions,
.inputPaths = inputPaths, .inputPaths = inputPaths,
.initialOutputs = initialOutputs, .initialOutputs = initialOutputs,
.buildMode = buildMode, .buildMode = buildMode,
@@ -827,8 +803,7 @@ BuildError DerivationBuildingGoal::fixupBuilderFailureErrorMessage(BuilderFailur
return BuildError{e.status, msg}; return BuildError{e.status, msg};
} }
HookReply DerivationBuildingGoal::tryBuildHook( HookReply DerivationBuildingGoal::tryBuildHook(const std::map<std::string, InitialOutput> & initialOutputs)
const std::map<std::string, InitialOutput> & initialOutputs, const DerivationOptions<StorePath> & drvOptions)
{ {
#ifdef _WIN32 // TODO enable build hook on Windows #ifdef _WIN32 // TODO enable build hook on Windows
return rpDecline; return rpDecline;
@@ -845,7 +820,7 @@ HookReply DerivationBuildingGoal::tryBuildHook(
/* Send the request to the hook. */ /* Send the request to the hook. */
worker.hook->sink << "try" << (worker.getNrLocalBuilds() < settings.maxBuildJobs ? 1 : 0) << drv->platform worker.hook->sink << "try" << (worker.getNrLocalBuilds() < settings.maxBuildJobs ? 1 : 0) << drv->platform
<< worker.store.printStorePath(drvPath) << drvOptions.getRequiredSystemFeatures(*drv); << worker.store.printStorePath(drvPath) << drvOptions->getRequiredSystemFeatures(*drv);
worker.hook->sink.flush(); worker.hook->sink.flush();
/* Read the first line of input, which should be a word indicating /* Read the first line of input, which should be a word indicating
+21 -26
View File
@@ -11,7 +11,7 @@ void checkOutputs(
Store & store, Store & store,
const StorePath & drvPath, const StorePath & drvPath,
const decltype(Derivation::outputs) & drvOutputs, const decltype(Derivation::outputs) & drvOutputs,
const decltype(DerivationOptions<StorePath>::outputChecks) & outputChecks, const decltype(DerivationOptions::outputChecks) & outputChecks,
const std::map<std::string, ValidPathInfo> & outputs) const std::map<std::string, ValidPathInfo> & outputs)
{ {
std::map<Path, const ValidPathInfo &> outputsByPath; std::map<Path, const ValidPathInfo &> outputsByPath;
@@ -85,7 +85,7 @@ void checkOutputs(
return std::make_pair(std::move(pathsDone), closureSize); return std::make_pair(std::move(pathsDone), closureSize);
}; };
auto applyChecks = [&](const DerivationOptions<StorePath>::OutputChecks & checks) { auto applyChecks = [&](const DerivationOptions::OutputChecks & checks) {
if (checks.maxSize && info.narSize > *checks.maxSize) if (checks.maxSize && info.narSize > *checks.maxSize)
throw BuildError( throw BuildError(
BuildResult::Failure::OutputRejected, BuildResult::Failure::OutputRejected,
@@ -105,33 +105,28 @@ void checkOutputs(
*checks.maxClosureSize); *checks.maxClosureSize);
} }
auto checkRefs = [&](const std::set<DrvRef<StorePath>> & value, bool allowed, bool recursive) { auto checkRefs = [&](const StringSet & value, bool allowed, bool recursive) {
/* Parse a list of reference specifiers. Each element must /* Parse a list of reference specifiers. Each element must
either be a store path, or the symbolic name of the output either be a store path, or the symbolic name of the output
of the derivation (such as `out'). */ of the derivation (such as `out'). */
StorePathSet spec; StorePathSet spec;
for (auto & i : value) { for (auto & i : value) {
std::visit( if (store.isStorePath(i))
overloaded{ spec.insert(store.parseStorePath(i));
[&](const StorePath & path) { spec.insert(path); }, else if (auto output = get(outputs, i))
[&](const OutputName & refOutputName) { spec.insert(output->path);
if (auto output = get(outputs, refOutputName)) else {
spec.insert(output->path); std::string outputsListing =
else { concatMapStringsSep(", ", outputs, [](auto & o) { return o.first; });
std::string outputsListing = throw BuildError(
concatMapStringsSep(", ", outputs, [](auto & o) { return o.first; }); BuildResult::Failure::OutputRejected,
throw BuildError( "derivation '%s' output check for '%s' contains an illegal reference specifier '%s',"
BuildResult::Failure::OutputRejected, " expected store path or output name (one of [%s])",
"derivation '%s' output check for '%s' contains output name '%s'," store.printStorePath(drvPath),
" but this is not a valid output of this derivation." outputName,
" (Valid outputs are [%s].)", i,
store.printStorePath(drvPath), outputsListing);
outputName, }
refOutputName,
outputsListing);
}
}},
i);
} }
auto used = recursive ? getClosure(info.path).first : info.references; auto used = recursive ? getClosure(info.path).first : info.references;
@@ -185,8 +180,8 @@ void checkOutputs(
std::visit( std::visit(
overloaded{ overloaded{
[&](const DerivationOptions<StorePath>::OutputChecks & checks) { applyChecks(checks); }, [&](const DerivationOptions::OutputChecks & checks) { applyChecks(checks); },
[&](const std::map<std::string, DerivationOptions<StorePath>::OutputChecks> & checksPerOutput) { [&](const std::map<std::string, DerivationOptions::OutputChecks> & checksPerOutput) {
if (auto outputChecks = get(checksPerOutput, outputName)) if (auto outputChecks = get(checksPerOutput, outputName))
applyChecks(*outputChecks); applyChecks(*outputChecks);
+1 -1
View File
@@ -21,7 +21,7 @@ void checkOutputs(
Store & store, Store & store,
const StorePath & drvPath, const StorePath & drvPath,
const decltype(Derivation::outputs) & drvOutputs, const decltype(Derivation::outputs) & drvOutputs,
const decltype(DerivationOptions<StorePath>::outputChecks) & drvOptions, const decltype(DerivationOptions::outputChecks) & drvOptions,
const std::map<std::string, ValidPathInfo> & outputs); const std::map<std::string, ValidPathInfo> & outputs);
} // namespace nix } // namespace nix
+2 -5
View File
@@ -18,10 +18,7 @@ std::string & DesugaredEnv::atFileEnvPair(std::string_view name, std::string fil
} }
DesugaredEnv DesugaredEnv::create( DesugaredEnv DesugaredEnv::create(
Store & store, Store & store, const Derivation & drv, const DerivationOptions & drvOptions, const StorePathSet & inputPaths)
const Derivation & drv,
const DerivationOptions<StorePath> & drvOptions,
const StorePathSet & inputPaths)
{ {
DesugaredEnv res; DesugaredEnv res;
@@ -49,7 +46,7 @@ DesugaredEnv DesugaredEnv::create(
} }
/* Handle exportReferencesGraph(), if set. */ /* Handle exportReferencesGraph(), if set. */
for (auto & [fileName, storePaths] : drvOptions.exportReferencesGraph) { for (auto & [fileName, storePaths] : drvOptions.getParsedExportReferencesGraph(store)) {
/* Write closure info to <fileName>. */ /* Write closure info to <fileName>. */
res.extraFiles.insert_or_assign( res.extraFiles.insert_or_assign(
fileName, store.makeValidityRegistration(store.exportReferences(storePaths, inputPaths), false, false)); fileName, store.makeValidityRegistration(store.exportReferences(storePaths, inputPaths), false, false));
+2 -3
View File
@@ -64,10 +64,9 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation)
{ {
trace("have derivation"); trace("have derivation");
auto drvOptions = [&]() -> DerivationOptions<SingleDerivedPath> { auto drvOptions = [&]() -> DerivationOptions {
try { try {
return derivationOptionsFromStructuredAttrs( return DerivationOptions::fromStructuredAttrs(drv->env, drv->structuredAttrs);
worker.store, drv->inputDrvs, drv->env, get(drv->structuredAttrs));
} catch (Error & e) { } catch (Error & e) {
e.addTrace({}, "while parsing derivation '%s'", worker.store.printStorePath(drvPath)); e.addTrace({}, "while parsing derivation '%s'", worker.store.printStorePath(drvPath));
throw; throw;
+67 -298
View File
@@ -2,18 +2,15 @@
#include "nix/util/json-utils.hh" #include "nix/util/json-utils.hh"
#include "nix/store/parsed-derivations.hh" #include "nix/store/parsed-derivations.hh"
#include "nix/store/derivations.hh" #include "nix/store/derivations.hh"
#include "nix/store/derived-path.hh"
#include "nix/store/store-api.hh" #include "nix/store/store-api.hh"
#include "nix/util/types.hh" #include "nix/util/types.hh"
#include "nix/util/util.hh" #include "nix/util/util.hh"
#include "nix/store/globals.hh" #include "nix/store/globals.hh"
#include "nix/util/variant-wrapper.hh"
#include <optional> #include <optional>
#include <string> #include <string>
#include <variant> #include <variant>
#include <regex> #include <regex>
#include <ranges>
namespace nix { namespace nix {
@@ -93,38 +90,14 @@ getStringSetAttr(const StringMap & env, const StructuredAttrs * parsed, const st
return ss ? (std::optional{StringSet{ss->begin(), ss->end()}}) : (std::optional<StringSet>{}); return ss ? (std::optional{StringSet{ss->begin(), ss->end()}}) : (std::optional<StringSet>{});
} }
template<typename Inputs> using OutputChecks = DerivationOptions::OutputChecks;
using OutputChecks = DerivationOptions<Inputs>::OutputChecks;
template<typename Inputs> using OutputChecksVariant = std::variant<OutputChecks, std::map<std::string, OutputChecks>>;
using OutputChecksVariant = std::variant<OutputChecks<Inputs>, std::map<std::string, OutputChecks<Inputs>>>;
DerivationOptions<StorePath> derivationOptionsFromStructuredAttrs( DerivationOptions DerivationOptions::fromStructuredAttrs(
const StoreDirConfig & store, const StringMap & env, const std::optional<StructuredAttrs> & parsed, bool shouldWarn)
const StringMap & env,
const StructuredAttrs * parsed,
bool shouldWarn,
const ExperimentalFeatureSettings & mockXpSettings)
{ {
/* Use the SingleDerivedPath version with empty inputDrvs, then return fromStructuredAttrs(env, parsed ? &*parsed : nullptr);
resolve. */
DerivedPathMap<StringSet> emptyInputDrvs{};
auto singleDerivedPathOptions =
derivationOptionsFromStructuredAttrs(store, emptyInputDrvs, env, parsed, shouldWarn, mockXpSettings);
/* "Resolve" all SingleDerivedPath inputs to StorePath. */
auto resolved = tryResolve(
singleDerivedPathOptions,
[&](ref<const SingleDerivedPath> drvPath, const std::string & outputName) -> std::optional<StorePath> {
// there should be nothing to resolve
assert(false);
});
/* Since we should never need to call the call back, there should be
no way it fails. */
assert(resolved);
return *resolved;
} }
static void flatten(const nlohmann::json & value, StringSet & res) static void flatten(const nlohmann::json & value, StringSet & res)
@@ -138,63 +111,10 @@ static void flatten(const nlohmann::json & value, StringSet & res)
throw Error("'exportReferencesGraph' value is not an array or a string"); throw Error("'exportReferencesGraph' value is not an array or a string");
} }
DerivationOptions<SingleDerivedPath> derivationOptionsFromStructuredAttrs( DerivationOptions
const StoreDirConfig & store, DerivationOptions::fromStructuredAttrs(const StringMap & env, const StructuredAttrs * parsed, bool shouldWarn)
const DerivedPathMap<StringSet> & inputDrvs,
const StringMap & env,
const StructuredAttrs * parsed,
bool shouldWarn,
const ExperimentalFeatureSettings & mockXpSettings)
{ {
DerivationOptions<SingleDerivedPath> defaults = {}; DerivationOptions defaults = {};
std::map<std::string, SingleDerivedPath::Built> placeholders;
if (mockXpSettings.isEnabled(Xp::CaDerivations)) {
/* Initialize placeholder map from inputDrvs */
auto initPlaceholders = [&](this const auto & initPlaceholders,
ref<const SingleDerivedPath> basePath,
const DerivedPathMap<StringSet>::ChildNode & node) -> void {
for (const auto & outputName : node.value) {
auto built = SingleDerivedPath::Built{
.drvPath = basePath,
.output = outputName,
};
placeholders.insert_or_assign(
DownstreamPlaceholder::fromSingleDerivedPathBuilt(built, mockXpSettings).render(),
std::move(built));
}
for (const auto & [outputName, childNode] : node.childMap) {
initPlaceholders(
make_ref<const SingleDerivedPath>(SingleDerivedPath::Built{
.drvPath = basePath,
.output = outputName,
}),
childNode);
}
};
for (const auto & [drvPath, outputs] : inputDrvs.map) {
auto basePath = make_ref<const SingleDerivedPath>(SingleDerivedPath::Opaque{drvPath});
initPlaceholders(basePath, outputs);
}
}
auto parseSingleDerivedPath = [&](const std::string & pathS) -> SingleDerivedPath {
if (auto it = placeholders.find(pathS); it != placeholders.end())
return it->second;
else
return SingleDerivedPath::Opaque{store.toStorePath(pathS).first};
};
auto parseRef = [&](const std::string & pathS) -> DrvRef<SingleDerivedPath> {
if (auto it = placeholders.find(pathS); it != placeholders.end())
return it->second;
if (store.isStorePath(pathS))
return SingleDerivedPath::Opaque{store.toStorePath(pathS).first};
else
return pathS;
};
if (shouldWarn && parsed) { if (shouldWarn && parsed) {
auto & structuredAttrs = parsed->structuredAttrs; auto & structuredAttrs = parsed->structuredAttrs;
@@ -226,14 +146,14 @@ DerivationOptions<SingleDerivedPath> derivationOptionsFromStructuredAttrs(
} }
return { return {
.outputChecks = [&]() -> OutputChecksVariant<SingleDerivedPath> { .outputChecks = [&]() -> OutputChecksVariant {
if (parsed) { if (parsed) {
auto & structuredAttrs = parsed->structuredAttrs; auto & structuredAttrs = parsed->structuredAttrs;
std::map<std::string, OutputChecks<SingleDerivedPath>> res; std::map<std::string, OutputChecks> res;
if (auto * outputChecks = get(structuredAttrs, "outputChecks")) { if (auto * outputChecks = get(structuredAttrs, "outputChecks")) {
for (auto & [outputName, output_] : getObject(*outputChecks)) { for (auto & [outputName, output_] : getObject(*outputChecks)) {
OutputChecks<SingleDerivedPath> checks; OutputChecks checks;
auto & output = getObject(output_); auto & output = getObject(output_);
@@ -243,14 +163,13 @@ DerivationOptions<SingleDerivedPath> derivationOptionsFromStructuredAttrs(
if (auto maxClosureSize = get(output, "maxClosureSize")) if (auto maxClosureSize = get(output, "maxClosureSize"))
checks.maxClosureSize = maxClosureSize->get<uint64_t>(); checks.maxClosureSize = maxClosureSize->get<uint64_t>();
auto get_ = auto get_ = [&output = output](const std::string & name) -> std::optional<StringSet> {
[&](const std::string & name) -> std::optional<std::set<DrvRef<SingleDerivedPath>>> {
if (auto i = get(output, name)) { if (auto i = get(output, name)) {
std::set<DrvRef<SingleDerivedPath>> res; StringSet res;
for (auto j = i->begin(); j != i->end(); ++j) { for (auto j = i->begin(); j != i->end(); ++j) {
if (!j->is_string()) if (!j->is_string())
throw Error("attribute '%s' must be a list of strings", name); throw Error("attribute '%s' must be a list of strings", name);
res.insert(parseRef(j->get<std::string>())); res.insert(j->get<std::string>());
} }
return res; return res;
} }
@@ -259,7 +178,7 @@ DerivationOptions<SingleDerivedPath> derivationOptionsFromStructuredAttrs(
res.insert_or_assign( res.insert_or_assign(
outputName, outputName,
OutputChecks<SingleDerivedPath>{ OutputChecks{
.maxSize = [&]() -> std::optional<uint64_t> { .maxSize = [&]() -> std::optional<uint64_t> {
if (auto maxSize = get(output, "maxSize")) if (auto maxSize = get(output, "maxSize"))
return maxSize->get<uint64_t>(); return maxSize->get<uint64_t>();
@@ -273,32 +192,21 @@ DerivationOptions<SingleDerivedPath> derivationOptionsFromStructuredAttrs(
return std::nullopt; return std::nullopt;
}(), }(),
.allowedReferences = get_("allowedReferences"), .allowedReferences = get_("allowedReferences"),
.disallowedReferences = .disallowedReferences = get_("disallowedReferences").value_or(StringSet{}),
get_("disallowedReferences").value_or(std::set<DrvRef<SingleDerivedPath>>{}),
.allowedRequisites = get_("allowedRequisites"), .allowedRequisites = get_("allowedRequisites"),
.disallowedRequisites = .disallowedRequisites = get_("disallowedRequisites").value_or(StringSet{}),
get_("disallowedRequisites").value_or(std::set<DrvRef<SingleDerivedPath>>{}),
}); });
} }
} }
return res; return res;
} else { } else {
auto parseRefSet = [&](const std::optional<StringSet> optionalStringSet) return OutputChecks{
-> std::optional<std::set<DrvRef<SingleDerivedPath>>> {
if (!optionalStringSet)
return std::nullopt;
auto range = *optionalStringSet | std::views::transform(parseRef);
return std::set<DrvRef<SingleDerivedPath>>(range.begin(), range.end());
};
return OutputChecks<SingleDerivedPath>{
// legacy non-structured-attributes case // legacy non-structured-attributes case
.ignoreSelfRefs = true, .ignoreSelfRefs = true,
.allowedReferences = parseRefSet(getStringSetAttr(env, parsed, "allowedReferences")), .allowedReferences = getStringSetAttr(env, parsed, "allowedReferences"),
.disallowedReferences = parseRefSet(getStringSetAttr(env, parsed, "disallowedReferences")) .disallowedReferences = getStringSetAttr(env, parsed, "disallowedReferences").value_or(StringSet{}),
.value_or(std::set<DrvRef<SingleDerivedPath>>{}), .allowedRequisites = getStringSetAttr(env, parsed, "allowedRequisites"),
.allowedRequisites = parseRefSet(getStringSetAttr(env, parsed, "allowedRequisites")), .disallowedRequisites = getStringSetAttr(env, parsed, "disallowedRequisites").value_or(StringSet{}),
.disallowedRequisites = parseRefSet(getStringSetAttr(env, parsed, "disallowedRequisites"))
.value_or(std::set<DrvRef<SingleDerivedPath>>{}),
}; };
} }
}(), }(),
@@ -337,19 +245,16 @@ DerivationOptions<SingleDerivedPath> derivationOptionsFromStructuredAttrs(
}(), }(),
.exportReferencesGraph = .exportReferencesGraph =
[&] { [&] {
std::map<std::string, std::set<SingleDerivedPath>> ret; std::map<std::string, StringSet> ret;
if (parsed) { if (parsed) {
auto * e = optionalValueAt(parsed->structuredAttrs, "exportReferencesGraph"); auto * e = optionalValueAt(parsed->structuredAttrs, "exportReferencesGraph");
if (!e || !e->is_object()) if (!e || !e->is_object())
return ret; return ret;
for (auto & [key, storePathsJson] : getObject(*e)) { for (auto & [key, value] : getObject(*e)) {
StringSet ss; StringSet ss;
flatten(storePathsJson, ss); flatten(value, ss);
std::set<SingleDerivedPath> storePaths; ret.insert_or_assign(key, std::move(ss));
for (auto & s : ss)
storePaths.insert(parseSingleDerivedPath(s));
ret.insert_or_assign(key, std::move(storePaths));
} }
} else { } else {
auto s = getOr(env, "exportReferencesGraph", ""); auto s = getOr(env, "exportReferencesGraph", "");
@@ -363,7 +268,7 @@ DerivationOptions<SingleDerivedPath> derivationOptionsFromStructuredAttrs(
throw Error("invalid file name '%s' in 'exportReferencesGraph'", fileName); throw Error("invalid file name '%s' in 'exportReferencesGraph'", fileName);
auto & storePathS = *i++; auto & storePathS = *i++;
ret.insert_or_assign(std::move(fileName), std::set{parseSingleDerivedPath(storePathS)}); ret.insert_or_assign(std::move(fileName), StringSet{storePathS});
} }
} }
return ret; return ret;
@@ -381,8 +286,28 @@ DerivationOptions<SingleDerivedPath> derivationOptionsFromStructuredAttrs(
}; };
} }
template<typename Input> std::map<std::string, StorePathSet>
StringSet DerivationOptions<Input>::getRequiredSystemFeatures(const BasicDerivation & drv) const DerivationOptions::getParsedExportReferencesGraph(const StoreDirConfig & store) const
{
std::map<std::string, StorePathSet> res;
for (auto & [fileName, ss] : exportReferencesGraph) {
StorePathSet storePaths;
for (auto & storePathS : ss) {
if (!store.isInStore(storePathS))
throw BuildError(
BuildResult::Failure::InputRejected,
"'exportReferencesGraph' contains a non-store path '%1%'",
storePathS);
storePaths.insert(store.toStorePath(storePathS).first);
}
res.insert_or_assign(fileName, storePaths);
}
return res;
}
StringSet DerivationOptions::getRequiredSystemFeatures(const BasicDerivation & drv) const
{ {
// FIXME: cache this? // FIXME: cache this?
StringSet res; StringSet res;
@@ -393,8 +318,7 @@ StringSet DerivationOptions<Input>::getRequiredSystemFeatures(const BasicDerivat
return res; return res;
} }
template<typename Input> bool DerivationOptions::canBuildLocally(Store & localStore, const BasicDerivation & drv) const
bool DerivationOptions<Input>::canBuildLocally(Store & localStore, const BasicDerivation & drv) const
{ {
if (drv.platform != settings.thisSystem.get() && !settings.extraPlatforms.get().count(drv.platform) if (drv.platform != settings.thisSystem.get() && !settings.extraPlatforms.get().count(drv.platform)
&& !drv.isBuiltin()) && !drv.isBuiltin())
@@ -410,194 +334,42 @@ bool DerivationOptions<Input>::canBuildLocally(Store & localStore, const BasicDe
return true; return true;
} }
template<typename Input> bool DerivationOptions::willBuildLocally(Store & localStore, const BasicDerivation & drv) const
bool DerivationOptions<Input>::willBuildLocally(Store & localStore, const BasicDerivation & drv) const
{ {
return preferLocalBuild && canBuildLocally(localStore, drv); return preferLocalBuild && canBuildLocally(localStore, drv);
} }
template<typename Input> bool DerivationOptions::substitutesAllowed() const
bool DerivationOptions<Input>::substitutesAllowed() const
{ {
return settings.alwaysAllowSubstitutes ? true : allowSubstitutes; return settings.alwaysAllowSubstitutes ? true : allowSubstitutes;
} }
template<typename Input> bool DerivationOptions::useUidRange(const BasicDerivation & drv) const
bool DerivationOptions<Input>::useUidRange(const BasicDerivation & drv) const
{ {
return getRequiredSystemFeatures(drv).count("uid-range"); return getRequiredSystemFeatures(drv).count("uid-range");
} }
std::optional<DerivationOptions<StorePath>> tryResolve(
const DerivationOptions<SingleDerivedPath> & drvOptions,
std::function<std::optional<StorePath>(ref<const SingleDerivedPath> drvPath, const std::string & outputName)>
queryResolutionChain)
{
auto tryResolvePath = [&](const SingleDerivedPath & input) -> std::optional<StorePath> {
return std::visit(
overloaded{
[](const SingleDerivedPath::Opaque & p) -> std::optional<StorePath> { return p.path; },
[&](const SingleDerivedPath::Built & p) -> std::optional<StorePath> {
return queryResolutionChain(p.drvPath, p.output);
}},
input.raw());
};
auto tryResolveRef = [&](const DrvRef<SingleDerivedPath> & ref) -> std::optional<DrvRef<StorePath>> {
return std::visit(
overloaded{
[](const OutputName & outputName) -> std::optional<DrvRef<StorePath>> { return outputName; },
[&](const SingleDerivedPath & input) -> std::optional<DrvRef<StorePath>> {
return tryResolvePath(input);
}},
ref);
};
auto tryResolveRefSet =
[&](const std::set<DrvRef<SingleDerivedPath>> & refSet) -> std::optional<std::set<DrvRef<StorePath>>> {
std::set<DrvRef<StorePath>> resolvedSet;
for (const auto & ref : refSet) {
auto resolvedRef = tryResolveRef(ref);
if (!resolvedRef)
return std::nullopt;
resolvedSet.insert(*resolvedRef);
}
return resolvedSet;
};
// Helper function to try resolving OutputChecks using functional style
auto tryResolveOutputChecks = [&](const DerivationOptions<SingleDerivedPath>::OutputChecks & checks)
-> std::optional<DerivationOptions<StorePath>::OutputChecks> {
std::optional<std::set<DrvRef<StorePath>>> resolvedAllowedReferences;
if (checks.allowedReferences) {
resolvedAllowedReferences = tryResolveRefSet(*checks.allowedReferences);
if (!resolvedAllowedReferences)
return std::nullopt;
}
std::optional<std::set<DrvRef<StorePath>>> resolvedAllowedRequisites;
if (checks.allowedRequisites) {
resolvedAllowedRequisites = tryResolveRefSet(*checks.allowedRequisites);
if (!resolvedAllowedRequisites)
return std::nullopt;
}
auto resolvedDisallowedReferences = tryResolveRefSet(checks.disallowedReferences);
if (!resolvedDisallowedReferences)
return std::nullopt;
auto resolvedDisallowedRequisites = tryResolveRefSet(checks.disallowedRequisites);
if (!resolvedDisallowedRequisites)
return std::nullopt;
return DerivationOptions<StorePath>::OutputChecks{
.ignoreSelfRefs = checks.ignoreSelfRefs,
.maxSize = checks.maxSize,
.maxClosureSize = checks.maxClosureSize,
.allowedReferences = resolvedAllowedReferences,
.disallowedReferences = *resolvedDisallowedReferences,
.allowedRequisites = resolvedAllowedRequisites,
.disallowedRequisites = *resolvedDisallowedRequisites,
};
};
// Helper function to resolve exportReferencesGraph using functional style
auto tryResolveExportReferencesGraph = [&](const std::map<std::string, std::set<SingleDerivedPath>> & exportGraph)
-> std::optional<std::map<std::string, std::set<StorePath>>> {
std::map<std::string, std::set<StorePath>> resolved;
for (const auto & [name, inputPaths] : exportGraph) {
std::set<StorePath> resolvedPaths;
for (const auto & inputPath : inputPaths) {
auto resolvedPath = tryResolvePath(inputPath);
if (!resolvedPath)
return std::nullopt;
resolvedPaths.insert(*resolvedPath);
}
resolved.emplace(name, std::move(resolvedPaths));
}
return resolved;
};
// Resolve outputChecks using functional style with std::visit
auto resolvedOutputChecks = std::visit(
overloaded{
[&](const DerivationOptions<SingleDerivedPath>::OutputChecks & checks)
-> std::optional<std::variant<
DerivationOptions<StorePath>::OutputChecks,
std::map<std::string, DerivationOptions<StorePath>::OutputChecks>>> {
auto resolved = tryResolveOutputChecks(checks);
if (!resolved)
return std::nullopt;
return std::variant<
DerivationOptions<StorePath>::OutputChecks,
std::map<std::string, DerivationOptions<StorePath>::OutputChecks>>(*resolved);
},
[&](const std::map<std::string, DerivationOptions<SingleDerivedPath>::OutputChecks> & checksMap)
-> std::optional<std::variant<
DerivationOptions<StorePath>::OutputChecks,
std::map<std::string, DerivationOptions<StorePath>::OutputChecks>>> {
std::map<std::string, DerivationOptions<StorePath>::OutputChecks> resolvedMap;
for (const auto & [outputName, checks] : checksMap) {
auto resolved = tryResolveOutputChecks(checks);
if (!resolved)
return std::nullopt;
resolvedMap.emplace(outputName, *resolved);
}
return std::variant<
DerivationOptions<StorePath>::OutputChecks,
std::map<std::string, DerivationOptions<StorePath>::OutputChecks>>(resolvedMap);
}},
drvOptions.outputChecks);
if (!resolvedOutputChecks)
return std::nullopt;
// Resolve exportReferencesGraph
auto resolvedExportGraph = tryResolveExportReferencesGraph(drvOptions.exportReferencesGraph);
if (!resolvedExportGraph)
return std::nullopt;
// Return resolved DerivationOptions using designated initializers
return DerivationOptions<StorePath>{
.outputChecks = *resolvedOutputChecks,
.unsafeDiscardReferences = drvOptions.unsafeDiscardReferences,
.passAsFile = drvOptions.passAsFile,
.exportReferencesGraph = *resolvedExportGraph,
.additionalSandboxProfile = drvOptions.additionalSandboxProfile,
.noChroot = drvOptions.noChroot,
.impureHostDeps = drvOptions.impureHostDeps,
.impureEnvVars = drvOptions.impureEnvVars,
.allowLocalNetworking = drvOptions.allowLocalNetworking,
.requiredSystemFeatures = drvOptions.requiredSystemFeatures,
.preferLocalBuild = drvOptions.preferLocalBuild,
.allowSubstitutes = drvOptions.allowSubstitutes,
};
}
template struct DerivationOptions<StorePath>;
template struct DerivationOptions<SingleDerivedPath>;
} // namespace nix } // namespace nix
namespace nlohmann { namespace nlohmann {
using namespace nix; using namespace nix;
DerivationOptions<SingleDerivedPath> adl_serializer<DerivationOptions<SingleDerivedPath>>::from_json(const json & json_) DerivationOptions adl_serializer<DerivationOptions>::from_json(const json & json_)
{ {
auto & json = getObject(json_); auto & json = getObject(json_);
return { return {
.outputChecks = [&]() -> OutputChecksVariant<SingleDerivedPath> { .outputChecks = [&]() -> OutputChecksVariant {
auto outputChecks = getObject(valueAt(json, "outputChecks")); auto outputChecks = getObject(valueAt(json, "outputChecks"));
auto forAllOutputsOpt = optionalValueAt(outputChecks, "forAllOutputs"); auto forAllOutputsOpt = optionalValueAt(outputChecks, "forAllOutputs");
auto perOutputOpt = optionalValueAt(outputChecks, "perOutput"); auto perOutputOpt = optionalValueAt(outputChecks, "perOutput");
if (forAllOutputsOpt && !perOutputOpt) { if (forAllOutputsOpt && !perOutputOpt) {
return static_cast<OutputChecks<SingleDerivedPath>>(*forAllOutputsOpt); return static_cast<OutputChecks>(*forAllOutputsOpt);
} else if (perOutputOpt && !forAllOutputsOpt) { } else if (perOutputOpt && !forAllOutputsOpt) {
return static_cast<std::map<std::string, OutputChecks<SingleDerivedPath>>>(*perOutputOpt); return static_cast<std::map<std::string, OutputChecks>>(*perOutputOpt);
} else { } else {
throw Error("Exactly one of 'perOutput' or 'forAllOutputs' is required"); throw Error("Exactly one of 'perOutput' or 'forAllOutputs' is required");
} }
@@ -605,7 +377,7 @@ DerivationOptions<SingleDerivedPath> adl_serializer<DerivationOptions<SingleDeri
.unsafeDiscardReferences = valueAt(json, "unsafeDiscardReferences"), .unsafeDiscardReferences = valueAt(json, "unsafeDiscardReferences"),
.passAsFile = getStringSet(valueAt(json, "passAsFile")), .passAsFile = getStringSet(valueAt(json, "passAsFile")),
.exportReferencesGraph = valueAt(json, "exportReferencesGraph"), .exportReferencesGraph = getMap<StringSet>(getObject(valueAt(json, "exportReferencesGraph")), getStringSet),
.additionalSandboxProfile = getString(valueAt(json, "additionalSandboxProfile")), .additionalSandboxProfile = getString(valueAt(json, "additionalSandboxProfile")),
.noChroot = getBoolean(valueAt(json, "noChroot")), .noChroot = getBoolean(valueAt(json, "noChroot")),
@@ -619,17 +391,16 @@ DerivationOptions<SingleDerivedPath> adl_serializer<DerivationOptions<SingleDeri
}; };
} }
void adl_serializer<DerivationOptions<SingleDerivedPath>>::to_json( void adl_serializer<DerivationOptions>::to_json(json & json, const DerivationOptions & o)
json & json, const DerivationOptions<SingleDerivedPath> & o)
{ {
json["outputChecks"] = std::visit( json["outputChecks"] = std::visit(
overloaded{ overloaded{
[&](const OutputChecks<SingleDerivedPath> & checks) { [&](const OutputChecks & checks) {
nlohmann::json outputChecks; nlohmann::json outputChecks;
outputChecks["forAllOutputs"] = checks; outputChecks["forAllOutputs"] = checks;
return outputChecks; return outputChecks;
}, },
[&](const std::map<std::string, OutputChecks<SingleDerivedPath>> & checksPerOutput) { [&](const std::map<std::string, OutputChecks> & checksPerOutput) {
nlohmann::json outputChecks; nlohmann::json outputChecks;
outputChecks["perOutput"] = checksPerOutput; outputChecks["perOutput"] = checksPerOutput;
return outputChecks; return outputChecks;
@@ -652,7 +423,7 @@ void adl_serializer<DerivationOptions<SingleDerivedPath>>::to_json(
json["allowSubstitutes"] = o.allowSubstitutes; json["allowSubstitutes"] = o.allowSubstitutes;
} }
OutputChecks<SingleDerivedPath> adl_serializer<OutputChecks<SingleDerivedPath>>::from_json(const json & json_) DerivationOptions::OutputChecks adl_serializer<DerivationOptions::OutputChecks>::from_json(const json & json_)
{ {
auto & json = getObject(json_); auto & json = getObject(json_);
@@ -660,16 +431,14 @@ OutputChecks<SingleDerivedPath> adl_serializer<OutputChecks<SingleDerivedPath>>:
.ignoreSelfRefs = getBoolean(valueAt(json, "ignoreSelfRefs")), .ignoreSelfRefs = getBoolean(valueAt(json, "ignoreSelfRefs")),
.maxSize = ptrToOwned<uint64_t>(getNullable(valueAt(json, "maxSize"))), .maxSize = ptrToOwned<uint64_t>(getNullable(valueAt(json, "maxSize"))),
.maxClosureSize = ptrToOwned<uint64_t>(getNullable(valueAt(json, "maxClosureSize"))), .maxClosureSize = ptrToOwned<uint64_t>(getNullable(valueAt(json, "maxClosureSize"))),
.allowedReferences = .allowedReferences = ptrToOwned<StringSet>(getNullable(valueAt(json, "allowedReferences"))),
ptrToOwned<std::set<DrvRef<SingleDerivedPath>>>(getNullable(valueAt(json, "allowedReferences"))), .disallowedReferences = getStringSet(valueAt(json, "disallowedReferences")),
.disallowedReferences = valueAt(json, "disallowedReferences"), .allowedRequisites = ptrToOwned<StringSet>(getNullable(valueAt(json, "allowedRequisites"))),
.allowedRequisites = .disallowedRequisites = getStringSet(valueAt(json, "disallowedRequisites")),
ptrToOwned<std::set<DrvRef<SingleDerivedPath>>>(getNullable(valueAt(json, "allowedRequisites"))),
.disallowedRequisites = valueAt(json, "disallowedRequisites"),
}; };
} }
void adl_serializer<OutputChecks<SingleDerivedPath>>::to_json(json & json, const OutputChecks<SingleDerivedPath> & c) void adl_serializer<DerivationOptions::OutputChecks>::to_json(json & json, const DerivationOptions::OutputChecks & c)
{ {
json["ignoreSelfRefs"] = c.ignoreSelfRefs; json["ignoreSelfRefs"] = c.ignoreSelfRefs;
json["maxSize"] = c.maxSize; json["maxSize"] = c.maxSize;
+51 -44
View File
@@ -13,7 +13,6 @@
#include <boost/container/small_vector.hpp> #include <boost/container/small_vector.hpp>
#include <boost/unordered/concurrent_flat_map.hpp> #include <boost/unordered/concurrent_flat_map.hpp>
#include <nlohmann/json.hpp> #include <nlohmann/json.hpp>
#include <optional>
namespace nix { namespace nix {
@@ -792,63 +791,71 @@ std::string outputPathName(std::string_view drvName, OutputNameView outputName)
DerivationType BasicDerivation::type() const DerivationType BasicDerivation::type() const
{ {
std::set<std::string_view> inputAddressedOutputs, fixedCAOutputs, floatingCAOutputs, deferredIAOutputs,
impureOutputs;
std::optional<HashAlgorithm> floatingHashAlgo; std::optional<HashAlgorithm> floatingHashAlgo;
std::optional<DerivationType> ty;
auto decide = [&](DerivationType newTy) {
if (!ty)
ty = newTy;
else if (ty.value() != newTy)
throw Error("can't mix derivation output types");
else if (ty.value() == DerivationType::ContentAddressed{.sandboxed = false, .fixed = true})
// FIXME: Experimental feature?
throw Error("only one fixed output is allowed for now");
};
for (auto & i : outputs) { for (auto & i : outputs) {
std::visit( std::visit(
overloaded{ overloaded{
[&](const DerivationOutput::InputAddressed &) { [&](const DerivationOutput::InputAddressed &) { inputAddressedOutputs.insert(i.first); },
decide( [&](const DerivationOutput::CAFixed &) { fixedCAOutputs.insert(i.first); },
DerivationType::InputAddressed{
.deferred = false,
});
},
[&](const DerivationOutput::CAFixed &) {
decide(
DerivationType::ContentAddressed{
.sandboxed = false,
.fixed = true,
});
if (i.first != "out"sv)
throw Error("single fixed output must be named \"out\"");
},
[&](const DerivationOutput::CAFloating & dof) { [&](const DerivationOutput::CAFloating & dof) {
decide( floatingCAOutputs.insert(i.first);
DerivationType::ContentAddressed{ if (!floatingHashAlgo) {
.sandboxed = true,
.fixed = false,
});
if (!floatingHashAlgo)
floatingHashAlgo = dof.hashAlgo; floatingHashAlgo = dof.hashAlgo;
else if (*floatingHashAlgo != dof.hashAlgo) } else {
throw Error("all floating outputs must use the same hash algorithm"); if (*floatingHashAlgo != dof.hashAlgo)
throw Error("all floating outputs must use the same hash algorithm");
}
}, },
[&](const DerivationOutput::Deferred &) { [&](const DerivationOutput::Deferred &) { deferredIAOutputs.insert(i.first); },
decide( [&](const DerivationOutput::Impure &) { impureOutputs.insert(i.first); },
DerivationType::InputAddressed{
.deferred = true,
});
},
[&](const DerivationOutput::Impure &) { decide(DerivationType::Impure{}); },
}, },
i.second.raw); i.second.raw);
} }
if (!ty) if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty()
&& deferredIAOutputs.empty() && impureOutputs.empty())
throw Error("must have at least one output"); throw Error("must have at least one output");
return ty.value(); if (!inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty()
&& deferredIAOutputs.empty() && impureOutputs.empty())
return DerivationType::InputAddressed{
.deferred = false,
};
if (inputAddressedOutputs.empty() && !fixedCAOutputs.empty() && floatingCAOutputs.empty()
&& deferredIAOutputs.empty() && impureOutputs.empty()) {
if (fixedCAOutputs.size() > 1)
// FIXME: Experimental feature?
throw Error("only one fixed output is allowed for now");
if (*fixedCAOutputs.begin() != "out"sv)
throw Error("single fixed output must be named \"out\"");
return DerivationType::ContentAddressed{
.sandboxed = false,
.fixed = true,
};
}
if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && !floatingCAOutputs.empty()
&& deferredIAOutputs.empty() && impureOutputs.empty())
return DerivationType::ContentAddressed{
.sandboxed = true,
.fixed = false,
};
if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty()
&& !deferredIAOutputs.empty() && impureOutputs.empty())
return DerivationType::InputAddressed{
.deferred = true,
};
if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty()
&& deferredIAOutputs.empty() && !impureOutputs.empty())
return DerivationType::Impure{};
throw Error("can't mix derivation output types");
} }
DrvHashes drvHashes; DrvHashes drvHashes;
-43
View File
@@ -1,6 +1,5 @@
#include "nix/store/downstream-placeholder.hh" #include "nix/store/downstream-placeholder.hh"
#include "nix/store/derivations.hh" #include "nix/store/derivations.hh"
#include "nix/util/json-utils.hh"
namespace nix { namespace nix {
@@ -50,45 +49,3 @@ DownstreamPlaceholder DownstreamPlaceholder::fromSingleDerivedPathBuilt(
} }
} // namespace nix } // namespace nix
namespace nlohmann {
using namespace nix;
template<typename Item>
DrvRef<Item> adl_serializer<DrvRef<Item>>::from_json(const json & json)
{
// OutputName case: { "drvPath": "self", "output": <output> }
if (json.type() == nlohmann::json::value_t::object) {
auto & obj = getObject(json);
if (auto * drvPath_ = get(obj, "drvPath")) {
auto & drvPath = *drvPath_;
if (drvPath.type() == nlohmann::json::value_t::string && getString(drvPath) == "self") {
return getString(valueAt(obj, "output"));
}
}
}
// Input case
return adl_serializer<Item>::from_json(json);
}
template<typename Item>
void adl_serializer<DrvRef<Item>>::to_json(json & json, const DrvRef<Item> & ref)
{
std::visit(
overloaded{
[&](const OutputName & outputName) {
json = nlohmann::json::object();
json["drvPath"] = "self";
json["output"] = outputName;
},
[&](const Item & item) { json = item; },
},
ref);
}
template struct adl_serializer<nix::DrvRef<StorePath>>;
template struct adl_serializer<nix::DrvRef<SingleDerivedPath>>;
} // namespace nlohmann
+42 -65
View File
@@ -151,23 +151,15 @@ struct curlFileTransfer : public FileTransfer
} }
} }
void failEx(std::exception_ptr ex) noexcept void failEx(std::exception_ptr ex)
{ {
assert(!done); assert(!done);
done = true; done = true;
try {
std::rethrow_exception(ex);
} catch (nix::Error & e) {
/* Add more context to the error message. */
e.addTrace({}, "during %s of '%s'", Uncolored(request.verb()), request.uri.to_string());
} catch (...) {
/* Can't add more context to the error. */
}
callback.rethrow(ex); callback.rethrow(ex);
} }
template<class T> template<class T>
void fail(T && e) noexcept void fail(T && e)
{ {
failEx(std::make_exception_ptr(std::forward<T>(e))); failEx(std::make_exception_ptr(std::forward<T>(e)));
} }
@@ -176,30 +168,32 @@ struct curlFileTransfer : public FileTransfer
std::shared_ptr<FinishSink> decompressionSink; std::shared_ptr<FinishSink> decompressionSink;
std::optional<StringSink> errorSink; std::optional<StringSink> errorSink;
std::exception_ptr callbackException; std::exception_ptr writeException;
size_t writeCallback(void * contents, size_t size, size_t nmemb) noexcept size_t writeCallback(void * contents, size_t size, size_t nmemb)
try { {
size_t realSize = size * nmemb; try {
result.bodySize += realSize; size_t realSize = size * nmemb;
result.bodySize += realSize;
if (!decompressionSink) { if (!decompressionSink) {
decompressionSink = makeDecompressionSink(encoding, finalSink); decompressionSink = makeDecompressionSink(encoding, finalSink);
if (!successfulStatuses.count(getHTTPStatus())) { if (!successfulStatuses.count(getHTTPStatus())) {
// In this case we want to construct a TeeSink, to keep // In this case we want to construct a TeeSink, to keep
// the response around (which we figure won't be big // the response around (which we figure won't be big
// like an actual download should be) to improve error // like an actual download should be) to improve error
// messages. // messages.
errorSink = StringSink{}; errorSink = StringSink{};
}
} }
(*decompressionSink)({(char *) contents, realSize});
return realSize;
} catch (...) {
writeException = std::current_exception();
return 0;
} }
(*decompressionSink)({(char *) contents, realSize});
return realSize;
} catch (...) {
callbackException = std::current_exception();
return 0;
} }
static size_t writeCallbackWrapper(void * contents, size_t size, size_t nmemb, void * userp) static size_t writeCallbackWrapper(void * contents, size_t size, size_t nmemb, void * userp)
@@ -215,8 +209,8 @@ struct curlFileTransfer : public FileTransfer
result.urls.push_back(effectiveUriCStr); result.urls.push_back(effectiveUriCStr);
} }
size_t headerCallback(void * contents, size_t size, size_t nmemb) noexcept size_t headerCallback(void * contents, size_t size, size_t nmemb)
try { {
size_t realSize = size * nmemb; size_t realSize = size * nmemb;
std::string line((char *) contents, realSize); std::string line((char *) contents, realSize);
printMsg(lvlVomit, "got header for '%s': %s", request.uri, trim(line)); printMsg(lvlVomit, "got header for '%s': %s", request.uri, trim(line));
@@ -269,15 +263,6 @@ struct curlFileTransfer : public FileTransfer
} }
} }
return realSize; return realSize;
} catch (...) {
#if LIBCURL_VERSION_NUM >= 0x075700
/* https://curl.se/libcurl/c/CURLOPT_HEADERFUNCTION.html:
You can also abort the transfer by returning CURL_WRITEFUNC_ERROR. */
callbackException = std::current_exception();
return CURL_WRITEFUNC_ERROR;
#else
return realSize;
#endif
} }
static size_t headerCallbackWrapper(void * contents, size_t size, size_t nmemb, void * userp) static size_t headerCallbackWrapper(void * contents, size_t size, size_t nmemb, void * userp)
@@ -285,17 +270,14 @@ struct curlFileTransfer : public FileTransfer
return ((TransferItem *) userp)->headerCallback(contents, size, nmemb); return ((TransferItem *) userp)->headerCallback(contents, size, nmemb);
} }
int progressCallback(curl_off_t dltotal, curl_off_t dlnow) noexcept int progressCallback(curl_off_t dltotal, curl_off_t dlnow)
try { {
act.progress(dlnow, dltotal); try {
act.progress(dlnow, dltotal);
} catch (nix::Interrupted &) {
assert(getInterrupted());
}
return getInterrupted(); return getInterrupted();
} catch (nix::Interrupted &) {
assert(getInterrupted());
return 1;
} catch (...) {
/* Something unexpected has happened like logger throwing an exception. */
callbackException = std::current_exception();
return 1;
} }
static int progressCallbackWrapper( static int progressCallbackWrapper(
@@ -306,14 +288,11 @@ struct curlFileTransfer : public FileTransfer
return item.progressCallback(isUpload ? ultotal : dltotal, isUpload ? ulnow : dlnow); return item.progressCallback(isUpload ? ultotal : dltotal, isUpload ? ulnow : dlnow);
} }
static int debugCallback(CURL * handle, curl_infotype type, char * data, size_t size, void * userptr) noexcept static int debugCallback(CURL * handle, curl_infotype type, char * data, size_t size, void * userptr)
try { {
if (type == CURLINFO_TEXT) if (type == CURLINFO_TEXT)
vomit("curl: %s", chomp(std::string(data, size))); vomit("curl: %s", chomp(std::string(data, size)));
return 0; return 0;
} catch (...) {
/* Swallow the exception. Nothing left to do. */
return 0;
} }
size_t readCallback(char * buffer, size_t size, size_t nitems) noexcept size_t readCallback(char * buffer, size_t size, size_t nitems) noexcept
@@ -323,7 +302,6 @@ struct curlFileTransfer : public FileTransfer
} catch (EndOfFile &) { } catch (EndOfFile &) {
return 0; return 0;
} catch (...) { } catch (...) {
callbackException = std::current_exception();
return CURL_READFUNC_ABORT; return CURL_READFUNC_ABORT;
} }
@@ -355,7 +333,6 @@ struct curlFileTransfer : public FileTransfer
} }
return CURL_SEEKFUNC_OK; return CURL_SEEKFUNC_OK;
} catch (...) { } catch (...) {
callbackException = std::current_exception();
return CURL_SEEKFUNC_FAIL; return CURL_SEEKFUNC_FAIL;
} }
@@ -409,17 +386,17 @@ struct curlFileTransfer : public FileTransfer
if (settings.downloadSpeed.get() > 0) if (settings.downloadSpeed.get() > 0)
curl_easy_setopt(req, CURLOPT_MAX_RECV_SPEED_LARGE, (curl_off_t) (settings.downloadSpeed.get() * 1024)); curl_easy_setopt(req, CURLOPT_MAX_RECV_SPEED_LARGE, (curl_off_t) (settings.downloadSpeed.get() * 1024));
if (request.method == HttpMethod::Head) if (request.method == HttpMethod::HEAD)
curl_easy_setopt(req, CURLOPT_NOBODY, 1); curl_easy_setopt(req, CURLOPT_NOBODY, 1);
if (request.method == HttpMethod::Delete) if (request.method == HttpMethod::DELETE)
curl_easy_setopt(req, CURLOPT_CUSTOMREQUEST, "DELETE"); curl_easy_setopt(req, CURLOPT_CUSTOMREQUEST, "DELETE");
if (request.data) { if (request.data) {
if (request.method == HttpMethod::Post) { if (request.method == HttpMethod::POST) {
curl_easy_setopt(req, CURLOPT_POST, 1L); curl_easy_setopt(req, CURLOPT_POST, 1L);
curl_easy_setopt(req, CURLOPT_POSTFIELDSIZE_LARGE, (curl_off_t) request.data->sizeHint); curl_easy_setopt(req, CURLOPT_POSTFIELDSIZE_LARGE, (curl_off_t) request.data->sizeHint);
} else if (request.method == HttpMethod::Put) { } else if (request.method == HttpMethod::PUT) {
curl_easy_setopt(req, CURLOPT_UPLOAD, 1L); curl_easy_setopt(req, CURLOPT_UPLOAD, 1L);
curl_easy_setopt(req, CURLOPT_INFILESIZE_LARGE, (curl_off_t) request.data->sizeHint); curl_easy_setopt(req, CURLOPT_INFILESIZE_LARGE, (curl_off_t) request.data->sizeHint);
} else { } else {
@@ -499,7 +476,7 @@ struct curlFileTransfer : public FileTransfer
try { try {
decompressionSink->finish(); decompressionSink->finish();
} catch (...) { } catch (...) {
callbackException = std::current_exception(); writeException = std::current_exception();
} }
} }
@@ -508,8 +485,8 @@ struct curlFileTransfer : public FileTransfer
httpStatus = 304; httpStatus = 304;
} }
if (callbackException) if (writeException)
failEx(callbackException); failEx(writeException);
else if (code == CURLE_OK && successfulStatuses.count(httpStatus)) { else if (code == CURLE_OK && successfulStatuses.count(httpStatus)) {
result.cached = httpStatus == 304; result.cached = httpStatus == 304;
+2 -2
View File
@@ -120,7 +120,7 @@ bool HttpBinaryCacheStore::fileExists(const std::string & path)
try { try {
FileTransferRequest request(makeRequest(path)); FileTransferRequest request(makeRequest(path));
request.method = HttpMethod::Head; request.method = HttpMethod::HEAD;
getFileTransfer()->download(request); getFileTransfer()->download(request);
return true; return true;
} catch (FileTransferError & e) { } catch (FileTransferError & e) {
@@ -141,7 +141,7 @@ void HttpBinaryCacheStore::upload(
std::optional<Headers> headers) std::optional<Headers> headers)
{ {
auto req = makeRequest(path); auto req = makeRequest(path);
req.method = HttpMethod::Put; req.method = HttpMethod::PUT;
if (headers) { if (headers) {
req.headers.reserve(req.headers.size() + headers->size()); req.headers.reserve(req.headers.size() + headers->size());
@@ -74,6 +74,7 @@ struct BuildResult
/// know about this one, so change it back to `OutputRejected` /// know about this one, so change it back to `OutputRejected`
/// before serialization. /// before serialization.
HashMismatch = 15, HashMismatch = 15,
Cancelled = 16,
} status = MiscFailure; } status = MiscFailure;
/** /**
@@ -69,7 +69,7 @@ struct DerivationBuilderParams
* *
* @todo this should be part of `Derivation`. * @todo this should be part of `Derivation`.
*/ */
const DerivationOptions<StorePath> & drvOptions; const DerivationOptions & drvOptions;
// The remainder is state held during the build. // The remainder is state held during the build.
@@ -52,6 +52,8 @@ private:
*/ */
std::unique_ptr<Derivation> drv; std::unique_ptr<Derivation> drv;
std::unique_ptr<DerivationOptions> drvOptions;
/** /**
* The remainder is state held during the build. * The remainder is state held during the build.
*/ */
@@ -113,8 +115,7 @@ private:
/** /**
* Is the build hook willing to perform the build? * Is the build hook willing to perform the build?
*/ */
HookReply tryBuildHook( HookReply tryBuildHook(const std::map<std::string, InitialOutput> & initialOutputs);
const std::map<std::string, InitialOutput> & initialOutputs, const DerivationOptions<StorePath> & drvOptions);
/** /**
* Open a log file and a pipe to it. * Open a log file and a pipe to it.
@@ -8,7 +8,6 @@ namespace nix {
class Store; class Store;
struct Derivation; struct Derivation;
template<typename Input>
struct DerivationOptions; struct DerivationOptions;
/** /**
@@ -78,10 +77,7 @@ struct DesugaredEnv
* just part of `Derivation`. * just part of `Derivation`.
*/ */
static DesugaredEnv create( static DesugaredEnv create(
Store & store, Store & store, const Derivation & drv, const DerivationOptions & drvOptions, const StorePathSet & inputPaths);
const Derivation & drv,
const DerivationOptions<StorePath> & drvOptions,
const StorePathSet & inputPaths);
}; };
} // namespace nix } // namespace nix
+1 -1
View File
@@ -109,7 +109,7 @@ public:
/** /**
* Build result. * Build result.
*/ */
BuildResult buildResult; BuildResult buildResult = {.inner = BuildResult::Failure{.status = BuildResult::Failure::Cancelled}};
/** /**
* Suspend our goal and wait until we get `work`-ed again. * Suspend our goal and wait until we get `work`-ed again.
@@ -8,8 +8,7 @@
#include "nix/util/types.hh" #include "nix/util/types.hh"
#include "nix/util/json-impls.hh" #include "nix/util/json-impls.hh"
#include "nix/store/store-dir-config.hh" #include "nix/store/path.hh"
#include "nix/store/downstream-placeholder.hh"
namespace nix { namespace nix {
@@ -18,9 +17,6 @@ struct StoreDirConfig;
struct BasicDerivation; struct BasicDerivation;
struct StructuredAttrs; struct StructuredAttrs;
template<typename V>
struct DerivedPathMap;
/** /**
* This represents all the special options on a `Derivation`. * This represents all the special options on a `Derivation`.
* *
@@ -38,7 +34,6 @@ struct DerivedPathMap;
* separately. That would be nice to separate concerns, and not make any * separately. That would be nice to separate concerns, and not make any
* environment variable names magical. * environment variable names magical.
*/ */
template<typename Input>
struct DerivationOptions struct DerivationOptions
{ {
struct OutputChecks struct OutputChecks
@@ -46,15 +41,13 @@ struct DerivationOptions
bool ignoreSelfRefs = false; bool ignoreSelfRefs = false;
std::optional<uint64_t> maxSize, maxClosureSize; std::optional<uint64_t> maxSize, maxClosureSize;
using DrvRef = nix::DrvRef<Input>;
/** /**
* env: allowedReferences * env: allowedReferences
* *
* A value of `nullopt` indicates that the check is skipped. * A value of `nullopt` indicates that the check is skipped.
* This means that all references are allowed. * This means that all references are allowed.
*/ */
std::optional<std::set<DrvRef>> allowedReferences; std::optional<StringSet> allowedReferences;
/** /**
* env: disallowedReferences * env: disallowedReferences
@@ -62,21 +55,21 @@ struct DerivationOptions
* No needed for `std::optional`, because skipping the check is * No needed for `std::optional`, because skipping the check is
* the same as disallowing the references. * the same as disallowing the references.
*/ */
std::set<DrvRef> disallowedReferences; StringSet disallowedReferences;
/** /**
* env: allowedRequisites * env: allowedRequisites
* *
* See `allowedReferences` * See `allowedReferences`
*/ */
std::optional<std::set<DrvRef>> allowedRequisites; std::optional<StringSet> allowedRequisites;
/** /**
* env: disallowedRequisites * env: disallowedRequisites
* *
* See `disallowedReferences` * See `disallowedReferences`
*/ */
std::set<DrvRef> disallowedRequisites; StringSet disallowedRequisites;
bool operator==(const OutputChecks &) const = default; bool operator==(const OutputChecks &) const = default;
}; };
@@ -123,7 +116,23 @@ struct DerivationOptions
* attributes give to the builder. The set of paths in the original JSON * attributes give to the builder. The set of paths in the original JSON
* is replaced with a list of `PathInfo` in JSON format. * is replaced with a list of `PathInfo` in JSON format.
*/ */
std::map<std::string, std::set<Input>> exportReferencesGraph; std::map<std::string, StringSet> exportReferencesGraph;
/**
* Once a derivations is resolved, the strings in in
* `exportReferencesGraph` should all be store paths (with possible
* suffix paths, but those are discarded).
*
* @return The parsed path set for for each key in the map.
*
* @todo Ideally, `exportReferencesGraph` would just store
* `StorePath`s for this, but we can't just do that, because for CA
* derivations they is actually in general `DerivedPath`s (via
* placeholder strings) until the derivation is resolved and exact
* inputs store paths are known. We can use better types for that
* too, but that is a longer project.
*/
std::map<std::string, StorePathSet> getParsedExportReferencesGraph(const StoreDirConfig & store) const;
/** /**
* env: __sandboxProfile * env: __sandboxProfile
@@ -176,6 +185,18 @@ struct DerivationOptions
bool operator==(const DerivationOptions &) const = default; bool operator==(const DerivationOptions &) const = default;
/**
* Parse this information from its legacy encoding as part of the
* environment. This should not be used with nice greenfield formats
* (e.g. JSON) but is necessary for supporting old formats (e.g.
* ATerm).
*/
static DerivationOptions
fromStructuredAttrs(const StringMap & env, const StructuredAttrs * parsed, bool shouldWarn = true);
static DerivationOptions
fromStructuredAttrs(const StringMap & env, const std::optional<StructuredAttrs> & parsed, bool shouldWarn = true);
/** /**
* @param drv Must be the same derivation we parsed this from. In * @param drv Must be the same derivation we parsed this from. In
* the future we'll flip things around so a `BasicDerivation` has * the future we'll flip things around so a `BasicDerivation` has
@@ -201,49 +222,7 @@ struct DerivationOptions
bool useUidRange(const BasicDerivation & drv) const; bool useUidRange(const BasicDerivation & drv) const;
}; };
extern template struct DerivationOptions<StorePath>;
extern template struct DerivationOptions<SingleDerivedPath>;
struct DerivationOutput;
/**
* Parse this information from its legacy encoding as part of the
* environment. This should not be used with nice greenfield formats
* (e.g. JSON) but is necessary for supporting old formats (e.g.
* ATerm).
*/
DerivationOptions<SingleDerivedPath> derivationOptionsFromStructuredAttrs(
const StoreDirConfig & store,
const DerivedPathMap<StringSet> & inputDrvs,
const StringMap & env,
const StructuredAttrs * parsed,
bool shouldWarn = true,
const ExperimentalFeatureSettings & mockXpSettings = experimentalFeatureSettings);
DerivationOptions<StorePath> derivationOptionsFromStructuredAttrs(
const StoreDirConfig & store,
const StringMap & env,
const StructuredAttrs * parsed,
bool shouldWarn = true,
const ExperimentalFeatureSettings & mockXpSettings = experimentalFeatureSettings);
/**
* This is the counterpart of `Derivation::tryResolve`. In particular,
* it takes the same sort of callback, which is used to reolve
* non-constant deriving paths.
*
* We need this function when resolving a derivation, and we will use
* this as part of that if/when `Derivation` includes
* `DerivationOptions`
*/
std::optional<DerivationOptions<StorePath>> tryResolve(
const DerivationOptions<SingleDerivedPath> & drvOptions,
std::function<std::optional<StorePath>(ref<const SingleDerivedPath> drvPath, const std::string & outputName)>
queryResolutionChain);
}; // namespace nix }; // namespace nix
JSON_IMPL(nix::DerivationOptions<nix::StorePath>); JSON_IMPL(DerivationOptions);
JSON_IMPL(nix::DerivationOptions<nix::SingleDerivedPath>); JSON_IMPL(DerivationOptions::OutputChecks)
JSON_IMPL(nix::DerivationOptions<nix::StorePath>::OutputChecks)
JSON_IMPL(nix::DerivationOptions<nix::SingleDerivedPath>::OutputChecks)
@@ -2,23 +2,11 @@
///@file ///@file
#include "nix/util/hash.hh" #include "nix/util/hash.hh"
#include "nix/util/json-impls.hh"
#include "nix/store/path.hh" #include "nix/store/path.hh"
#include "nix/store/derived-path.hh" #include "nix/store/derived-path.hh"
namespace nix { namespace nix {
/**
* A reference is either to a to-be-registered output (by name),
* or to an already-registered store object (by `Input`).
*
* `Ref<SingleDerivedPath` is a representation of something that can be
* turned into a placeholder. (Regular own-output placeholder in the
* first case, `DownstreamPlaceholder` in the second case.)
*/
template<typename Input>
using DrvRef = std::variant<OutputName, Input>;
/** /**
* Downstream Placeholders are opaque and almost certainly unique values * Downstream Placeholders are opaque and almost certainly unique values
* used to allow derivations to refer to store objects which are yet to * used to allow derivations to refer to store objects which are yet to
@@ -104,17 +92,3 @@ public:
}; };
} // namespace nix } // namespace nix
namespace nlohmann {
template<typename Item>
struct adl_serializer<nix::DrvRef<Item>>
{
static nix::DrvRef<Item> from_json(const json & json);
static void to_json(json & json, const nix::DrvRef<Item> & t);
};
extern template struct adl_serializer<nix::DrvRef<nix::StorePath>>;
extern template struct adl_serializer<nix::DrvRef<nix::SingleDerivedPath>>;
} // namespace nlohmann
+11 -11
View File
@@ -87,11 +87,11 @@ extern const unsigned int RETRY_TIME_MS_DEFAULT;
* HTTP methods supported by FileTransfer. * HTTP methods supported by FileTransfer.
*/ */
enum struct HttpMethod { enum struct HttpMethod {
Get, GET,
Put, PUT,
Head, HEAD,
Post, POST,
Delete, DELETE,
}; };
/** /**
@@ -110,7 +110,7 @@ struct FileTransferRequest
VerbatimURL uri; VerbatimURL uri;
Headers headers; Headers headers;
std::string expectedETag; std::string expectedETag;
HttpMethod method = HttpMethod::Get; HttpMethod method = HttpMethod::GET;
size_t tries = fileTransferSettings.tries; size_t tries = fileTransferSettings.tries;
unsigned int baseRetryTimeMs = RETRY_TIME_MS_DEFAULT; unsigned int baseRetryTimeMs = RETRY_TIME_MS_DEFAULT;
ActivityId parentAct; ActivityId parentAct;
@@ -164,14 +164,14 @@ struct FileTransferRequest
std::string verb() const std::string verb() const
{ {
switch (method) { switch (method) {
case HttpMethod::Head: case HttpMethod::HEAD:
case HttpMethod::Get: case HttpMethod::GET:
return "download"; return "download";
case HttpMethod::Put: case HttpMethod::PUT:
case HttpMethod::Post: case HttpMethod::POST:
assert(data); assert(data);
return "upload"; return "upload";
case HttpMethod::Delete: case HttpMethod::DELETE:
return "delet"; return "delet";
} }
unreachable(); unreachable();
@@ -9,7 +9,6 @@
namespace nix { namespace nix {
class Store; class Store;
template<typename Input>
struct DerivationOptions; struct DerivationOptions;
struct DerivationOutput; struct DerivationOutput;
@@ -48,7 +47,7 @@ struct StructuredAttrs
nlohmann::json::object_t prepareStructuredAttrs( nlohmann::json::object_t prepareStructuredAttrs(
Store & store, Store & store,
const DerivationOptions<StorePath> & drvOptions, const DerivationOptions & drvOptions,
const StorePathSet & inputPaths, const StorePathSet & inputPaths,
const DerivationOutputs & outputs) const; const DerivationOutputs & outputs) const;
+10 -5
View File
@@ -1246,8 +1246,10 @@ StorePath LocalStore::addToStoreFromDump(
auto desc = ContentAddressWithReferences::fromParts( auto desc = ContentAddressWithReferences::fromParts(
hashMethod, hashMethod,
methodsMatch ? dumpHash methodsMatch
: hashPath(makeFSSourceAccessor(tempPath), hashMethod.getFileIngestionMethod(), hashAlgo).first, ? dumpHash
: hashPath(PosixSourceAccessor::createAtRoot(tempPath), hashMethod.getFileIngestionMethod(), hashAlgo)
.first,
{ {
.others = references, .others = references,
// caller is not capable of creating a self-reference, because this is content-addressed without modulus // caller is not capable of creating a self-reference, because this is content-addressed without modulus
@@ -1383,9 +1385,12 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
checkInterrupt(); checkInterrupt();
auto name = link.path().filename(); auto name = link.path().filename();
printMsg(lvlTalkative, "checking contents of %s", name); printMsg(lvlTalkative, "checking contents of %s", name);
std::string hash = PosixSourceAccessor accessor;
hashPath(makeFSSourceAccessor(link.path()), FileIngestionMethod::NixArchive, HashAlgorithm::SHA256) std::string hash = hashPath(
.first.to_string(HashFormat::Nix32, false); PosixSourceAccessor::createAtRoot(link.path()),
FileIngestionMethod::NixArchive,
HashAlgorithm::SHA256)
.first.to_string(HashFormat::Nix32, false);
if (hash != name.string()) { if (hash != name.string()) {
printError("link %s was modified! expected hash %s, got '%s'", link.path(), name, hash); printError("link %s was modified! expected hash %s, got '%s'", link.path(), name, hash);
if (repair) { if (repair) {
+2 -3
View File
@@ -225,12 +225,11 @@ MissingPaths Store::queryMissing(const std::vector<DerivedPath> & targets)
return; return;
auto drv = make_ref<Derivation>(derivationFromPath(drvPath)); auto drv = make_ref<Derivation>(derivationFromPath(drvPath));
DerivationOptions<SingleDerivedPath> drvOptions; DerivationOptions drvOptions;
try { try {
// FIXME: this is a lot of work just to get the value // FIXME: this is a lot of work just to get the value
// of `allowSubstitutes`. // of `allowSubstitutes`.
drvOptions = derivationOptionsFromStructuredAttrs( drvOptions = DerivationOptions::fromStructuredAttrs(drv->env, drv->structuredAttrs);
*this, drv->inputDrvs, drv->env, get(drv->structuredAttrs));
} catch (Error & e) { } catch (Error & e) {
e.addTrace({}, "while parsing derivation '%s'", printStorePath(drvPath)); e.addTrace({}, "while parsing derivation '%s'", printStorePath(drvPath));
throw; throw;
+1 -1
View File
@@ -172,7 +172,7 @@ void LocalStore::optimisePath_(
auto stLink = lstat(linkPath.string()); auto stLink = lstat(linkPath.string());
if (st.st_size != stLink.st_size || (repair && hash != ({ if (st.st_size != stLink.st_size || (repair && hash != ({
hashPath( hashPath(
makeFSSourceAccessor(linkPath), PosixSourceAccessor::createAtRoot(linkPath),
FileSerialisationMethod::NixArchive, FileSerialisationMethod::NixArchive,
HashAlgorithm::SHA256) HashAlgorithm::SHA256)
.hash; .hash;
+3 -3
View File
@@ -100,7 +100,7 @@ static nlohmann::json pathInfoToJSON(Store & store, const StorePathSet & storePa
nlohmann::json::object_t StructuredAttrs::prepareStructuredAttrs( nlohmann::json::object_t StructuredAttrs::prepareStructuredAttrs(
Store & store, Store & store,
const DerivationOptions<StorePath> & drvOptions, const DerivationOptions & drvOptions,
const StorePathSet & inputPaths, const StorePathSet & inputPaths,
const DerivationOutputs & outputs) const const DerivationOutputs & outputs) const
{ {
@@ -114,8 +114,8 @@ nlohmann::json::object_t StructuredAttrs::prepareStructuredAttrs(
json["outputs"] = std::move(outputsJson); json["outputs"] = std::move(outputsJson);
/* Handle exportReferencesGraph. */ /* Handle exportReferencesGraph. */
for (auto & [key, storePaths] : drvOptions.exportReferencesGraph) { for (auto & [key, storePaths] : drvOptions.getParsedExportReferencesGraph(store)) {
json[key] = pathInfoToJSON(store, store.exportReferences(storePaths, inputPaths)); json[key] = pathInfoToJSON(store, store.exportReferences(storePaths, storePaths));
} }
return json; return json;
+4 -4
View File
@@ -295,7 +295,7 @@ std::string S3BinaryCacheStore::createMultipartUpload(
url.query["uploads"] = ""; url.query["uploads"] = "";
req.uri = VerbatimURL(url); req.uri = VerbatimURL(url);
req.method = HttpMethod::Post; req.method = HttpMethod::POST;
StringSource payload{std::string_view("")}; StringSource payload{std::string_view("")};
req.data = {payload}; req.data = {payload};
req.mimeType = mimeType; req.mimeType = mimeType;
@@ -325,7 +325,7 @@ S3BinaryCacheStore::uploadPart(std::string_view key, std::string_view uploadId,
} }
auto req = makeRequest(key); auto req = makeRequest(key);
req.method = HttpMethod::Put; req.method = HttpMethod::PUT;
req.setupForS3(); req.setupForS3();
auto url = req.uri.parsed(); auto url = req.uri.parsed();
@@ -355,7 +355,7 @@ void S3BinaryCacheStore::abortMultipartUpload(std::string_view key, std::string_
auto url = req.uri.parsed(); auto url = req.uri.parsed();
url.query["uploadId"] = uploadId; url.query["uploadId"] = uploadId;
req.uri = VerbatimURL(url); req.uri = VerbatimURL(url);
req.method = HttpMethod::Delete; req.method = HttpMethod::DELETE;
getFileTransfer()->enqueueFileTransfer(req).get(); getFileTransfer()->enqueueFileTransfer(req).get();
} catch (...) { } catch (...) {
@@ -372,7 +372,7 @@ void S3BinaryCacheStore::completeMultipartUpload(
auto url = req.uri.parsed(); auto url = req.uri.parsed();
url.query["uploadId"] = uploadId; url.query["uploadId"] = uploadId;
req.uri = VerbatimURL(url); req.uri = VerbatimURL(url);
req.method = HttpMethod::Post; req.method = HttpMethod::POST;
std::string xml = "<CompleteMultipartUpload>"; std::string xml = "<CompleteMultipartUpload>";
for (const auto & [idx, etag] : enumerate(partEtags)) { for (const auto & [idx, etag] : enumerate(partEtags)) {
+1 -1
View File
@@ -36,7 +36,7 @@ nix_err nix_context_error(nix_c_context * context)
const char * demangled = abi::__cxa_demangle(typeid(e).name(), 0, 0, &status); const char * demangled = abi::__cxa_demangle(typeid(e).name(), 0, 0, &status);
if (demangled) { if (demangled) {
context->name = demangled; context->name = demangled;
free((void *) demangled); // todo: free(demangled);
} else { } else {
context->name = typeid(e).name(); context->name = typeid(e).name();
} }
+1 -2
View File
@@ -1,5 +1,4 @@
// TODO: investigate why this is hanging on cygwin #ifndef _WIN32
#if !defined(_WIN32) && !defined(__CYGWIN__)
# include "nix/util/util.hh" # include "nix/util/util.hh"
# include "nix/util/monitor-fd.hh" # include "nix/util/monitor-fd.hh"
+41 -41
View File
@@ -103,9 +103,9 @@ void SourceAccessor::dumpPath(const CanonPath & path, Sink & sink, PathFilter &
time_t dumpPathAndGetMtime(const Path & path, Sink & sink, PathFilter & filter) time_t dumpPathAndGetMtime(const Path & path, Sink & sink, PathFilter & filter)
{ {
auto path2 = PosixSourceAccessor::createAtRoot(path, /*trackLastModified=*/true); auto path2 = PosixSourceAccessor::createAtRoot(path);
path2.dumpPath(sink, filter); path2.dumpPath(sink, filter);
return path2.accessor->getLastModified().value(); return path2.accessor.dynamic_pointer_cast<PosixSourceAccessor>()->mtime;
} }
void dumpPath(const Path & path, Sink & sink, PathFilter & filter) void dumpPath(const Path & path, Sink & sink, PathFilter & filter)
@@ -200,54 +200,54 @@ static void parse(FileSystemObjectSink & sink, Source & source, const CanonPath
} }
else if (type == "directory") { else if (type == "directory") {
sink.createDirectory(path, [&](FileSystemObjectSink & dirSink, const CanonPath & relDirPath) { sink.createDirectory(path);
std::map<Path, int, CaseInsensitiveCompare> names;
std::string prevName; std::map<Path, int, CaseInsensitiveCompare> names;
while (1) { std::string prevName;
auto tag = getString();
if (tag == ")") while (1) {
break; auto tag = getString();
if (tag != "entry") if (tag == ")")
throw badArchive("expected tag 'entry' or ')', got '%s'", tag); break;
expectTag("("); if (tag != "entry")
throw badArchive("expected tag 'entry' or ')', got '%s'", tag);
expectTag("name"); expectTag("(");
auto name = getString(); expectTag("name");
if (name.empty() || name == "." || name == ".." || name.find('/') != std::string::npos
|| name.find((char) 0) != std::string::npos)
throw badArchive("NAR contains invalid file name '%1%'", name);
if (name <= prevName)
throw badArchive("NAR directory is not sorted");
prevName = name;
if (archiveSettings.useCaseHack) {
auto i = names.find(name);
if (i != names.end()) {
debug("case collision between '%1%' and '%2%'", i->first, name);
name += caseHackSuffix;
name += std::to_string(++i->second);
auto j = names.find(name);
if (j != names.end())
throw badArchive(
"NAR contains file name '%s' that collides with case-hacked file name '%s'",
prevName,
j->first);
} else
names[name] = 0;
}
expectTag("node"); auto name = getString();
if (name.empty() || name == "." || name == ".." || name.find('/') != std::string::npos
parse(dirSink, source, relDirPath / name); || name.find((char) 0) != std::string::npos)
throw badArchive("NAR contains invalid file name '%1%'", name);
expectTag(")"); if (name <= prevName)
throw badArchive("NAR directory is not sorted");
prevName = name;
if (archiveSettings.useCaseHack) {
auto i = names.find(name);
if (i != names.end()) {
debug("case collision between '%1%' and '%2%'", i->first, name);
name += caseHackSuffix;
name += std::to_string(++i->second);
auto j = names.find(name);
if (j != names.end())
throw badArchive(
"NAR contains file name '%s' that collides with case-hacked file name '%s'",
prevName,
j->first);
} else
names[name] = 0;
} }
});
expectTag("node");
parse(sink, source, path / name);
expectTag(")");
}
} }
else if (type == "symlink") { else if (type == "symlink") {
+39 -64
View File
@@ -14,6 +14,44 @@
namespace nix { namespace nix {
void copyRecursive(SourceAccessor & accessor, const CanonPath & from, FileSystemObjectSink & sink, const CanonPath & to)
{
auto stat = accessor.lstat(from);
switch (stat.type) {
case SourceAccessor::tSymlink: {
sink.createSymlink(to, accessor.readLink(from));
break;
}
case SourceAccessor::tRegular: {
sink.createRegularFile(to, [&](CreateRegularFileSink & crf) {
if (stat.isExecutable)
crf.isExecutable();
accessor.readFile(from, crf, [&](uint64_t size) { crf.preallocateContents(size); });
});
break;
}
case SourceAccessor::tDirectory: {
sink.createDirectory(to);
for (auto & [name, _] : accessor.readDirectory(from)) {
copyRecursive(accessor, from / name, sink, to / name);
break;
}
break;
}
case SourceAccessor::tChar:
case SourceAccessor::tBlock:
case SourceAccessor::tSocket:
case SourceAccessor::tFifo:
case SourceAccessor::tUnknown:
default:
throw Error("file '%1%' has an unsupported type of %2%", from, stat.typeString());
}
}
struct RestoreSinkSettings : Config struct RestoreSinkSettings : Config
{ {
Setting<bool> preallocateContents{ Setting<bool> preallocateContents{
@@ -32,60 +70,11 @@ static std::filesystem::path append(const std::filesystem::path & src, const Can
return dst; return dst;
} }
#ifndef _WIN32
void RestoreSink::createDirectory(const CanonPath & path, DirectoryCreatedCallback callback)
{
if (path.isRoot()) {
createDirectory(path);
callback(*this, path);
return;
}
createDirectory(path);
assert(dirFd); // If that's not true the above call must have thrown an exception.
RestoreSink dirSink{startFsync};
dirSink.dstPath = append(dstPath, path);
dirSink.dirFd = ::openat(dirFd.get(), path.rel_c_str(), O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
if (!dirSink.dirFd)
throw SysError("opening directory '%s'", dirSink.dstPath.string());
callback(dirSink, CanonPath::root);
}
#endif
void RestoreSink::createDirectory(const CanonPath & path) void RestoreSink::createDirectory(const CanonPath & path)
{ {
auto p = append(dstPath, path); auto p = append(dstPath, path);
#ifndef _WIN32
if (dirFd) {
if (path.isRoot())
/* Trying to create a directory that we already have a file descriptor for. */
throw Error("path '%s' already exists", p.string());
if (::mkdirat(dirFd.get(), path.rel_c_str(), 0777) == -1)
throw SysError("creating directory '%s'", p.string());
return;
}
#endif
if (!std::filesystem::create_directory(p)) if (!std::filesystem::create_directory(p))
throw Error("path '%s' already exists", p.string()); throw Error("path '%s' already exists", p.string());
#ifndef _WIN32
if (path.isRoot()) {
assert(!dirFd); // Handled above
/* Open directory for further *at operations relative to the sink root
directory. */
dirFd = open(p.c_str(), O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
if (!dirFd)
throw SysError("creating directory '%1%'", p.string());
}
#endif
}; };
struct RestoreRegularFile : CreateRegularFileSink struct RestoreRegularFile : CreateRegularFileSink
@@ -125,14 +114,7 @@ void RestoreSink::createRegularFile(const CanonPath & path, std::function<void(C
FILE_ATTRIBUTE_NORMAL, FILE_ATTRIBUTE_NORMAL,
NULL) NULL)
#else #else
[&]() { open(p.c_str(), O_CREAT | O_EXCL | O_WRONLY | O_CLOEXEC, 0666)
/* O_EXCL together with O_CREAT ensures symbolic links in the last
component are not followed. */
constexpr int flags = O_CREAT | O_EXCL | O_WRONLY | O_CLOEXEC;
if (!dirFd)
return ::open(p.c_str(), flags, 0666);
return ::openat(dirFd.get(), path.rel_c_str(), flags, 0666);
}();
#endif #endif
; ;
if (!crf.fd) if (!crf.fd)
@@ -179,13 +161,6 @@ void RestoreRegularFile::operator()(std::string_view data)
void RestoreSink::createSymlink(const CanonPath & path, const std::string & target) void RestoreSink::createSymlink(const CanonPath & path, const std::string & target)
{ {
auto p = append(dstPath, path); auto p = append(dstPath, path);
#ifndef _WIN32
if (dirFd) {
if (::symlinkat(requireCString(target), dirFd.get(), path.rel_c_str()) == -1)
throw SysError("creating symlink from '%1%' -> '%2%'", p.string(), target);
return;
}
#endif
nix::createSymlink(target, p.string()); nix::createSymlink(target, p.string());
} }
+23
View File
@@ -217,6 +217,29 @@ std::optional<Mode> convertMode(SourceAccessor::Type type)
} }
} }
void restore(FileSystemObjectSink & sink, Source & source, HashAlgorithm hashAlgo, std::function<RestoreHook> hook)
{
parse(sink, CanonPath::root, source, BlobMode::Regular, hashAlgo, [&](CanonPath name, TreeEntry entry) {
auto [accessor, from] = hook(entry.hash);
auto stat = accessor->lstat(from);
auto gotOpt = convertMode(stat.type);
if (!gotOpt)
throw Error(
"file '%s' (git hash %s) has an unsupported type",
from,
entry.hash.to_string(HashFormat::Base16, false));
auto & got = *gotOpt;
if (got != entry.mode)
throw Error(
"git mode of file '%s' (git hash %s) is %o but expected %o",
from,
entry.hash.to_string(HashFormat::Base16, false),
(RawMode) got,
(RawMode) entry.mode);
copyRecursive(*accessor, from, sink, name);
});
}
void dumpBlobPrefix(uint64_t size, Sink & sink, const ExperimentalFeatureSettings & xpSettings) void dumpBlobPrefix(uint64_t size, Sink & sink, const ExperimentalFeatureSettings & xpSettings)
{ {
xpSettings.require(Xp::GitHashing); xpSettings.require(Xp::GitHashing);
+10 -60
View File
@@ -8,7 +8,6 @@
#include <iostream> #include <iostream>
#include <set> #include <set>
#include <vector> #include <vector>
#include <ranges>
#include <boost/container_hash/hash.hpp> #include <boost/container_hash/hash.hpp>
@@ -123,70 +122,33 @@ public:
return &cs[1]; return &cs[1];
} }
class Iterator struct Iterator
{ {
/**
* Helper class with overloaded operator-> for "drill-down" behavior.
* This was a "temporary" string_view doesn't have to be stored anywhere.
*/
class PointerProxy
{
std::string_view segment;
public:
PointerProxy(std::string_view segment_)
: segment(segment_)
{
}
const std::string_view * operator->() const
{
return &segment;
}
};
public:
using value_type = std::string_view;
using reference_type = const std::string_view;
using pointer_type = PointerProxy;
using difference_type = std::ptrdiff_t;
using iterator_category = std::forward_iterator_tag;
std::string_view remaining; std::string_view remaining;
size_t slash; size_t slash;
/**
* Dummy default constructor required for forward iterators. Doesn't return
* a usable iterator.
*/
Iterator()
: remaining()
, slash(0)
{
}
Iterator(std::string_view remaining) Iterator(std::string_view remaining)
: remaining(remaining) : remaining(remaining)
, slash(remaining.find('/')) , slash(remaining.find('/'))
{ {
} }
bool operator==(const Iterator & x) const bool operator!=(const Iterator & x) const
{ {
return remaining.data() == x.remaining.data(); return remaining.data() != x.remaining.data();
} }
reference_type operator*() const bool operator==(const Iterator & x) const
{
return !(*this != x);
}
const std::string_view operator*() const
{ {
return remaining.substr(0, slash); return remaining.substr(0, slash);
} }
pointer_type operator->() const void operator++()
{
return PointerProxy(**this);
}
Iterator & operator++()
{ {
if (slash == remaining.npos) if (slash == remaining.npos)
remaining = remaining.substr(remaining.size()); remaining = remaining.substr(remaining.size());
@@ -194,19 +156,9 @@ public:
remaining = remaining.substr(slash + 1); remaining = remaining.substr(slash + 1);
slash = remaining.find('/'); slash = remaining.find('/');
} }
return *this;
}
Iterator operator++(int)
{
auto tmp = *this;
++*this;
return tmp;
} }
}; };
static_assert(std::forward_iterator<Iterator>);
Iterator begin() const Iterator begin() const
{ {
return Iterator(rel()); return Iterator(rel());
@@ -313,8 +265,6 @@ public:
friend std::size_t hash_value(const CanonPath &); friend std::size_t hash_value(const CanonPath &);
}; };
static_assert(std::ranges::forward_range<CanonPath>);
std::ostream & operator<<(std::ostream & stream, const CanonPath & path); std::ostream & operator<<(std::ostream & stream, const CanonPath & path);
inline std::size_t hash_value(const CanonPath & path) inline std::size_t hash_value(const CanonPath & path)
-33
View File
@@ -36,23 +36,6 @@ struct FileSystemObjectSink
virtual void createDirectory(const CanonPath & path) = 0; virtual void createDirectory(const CanonPath & path) = 0;
using DirectoryCreatedCallback = std::function<void(FileSystemObjectSink & dirSink, const CanonPath & dirRelPath)>;
/**
* Create a directory and invoke a callback with a pair of sink + CanonPath
* of the created subdirectory relative to dirSink.
*
* @note This allows for UNIX RestoreSink implementations to implement
* *at-style accessors that always keep an open file descriptor for the
* freshly created directory. Use this when it's important to disallow any
* intermediate path components from being symlinks.
*/
virtual void createDirectory(const CanonPath & path, DirectoryCreatedCallback callback)
{
createDirectory(path);
callback(*this, path);
}
/** /**
* This function in general is no re-entrant. Only one file can be * This function in general is no re-entrant. Only one file can be
* written at a time. * written at a time.
@@ -99,18 +82,6 @@ struct NullFileSystemObjectSink : FileSystemObjectSink
struct RestoreSink : FileSystemObjectSink struct RestoreSink : FileSystemObjectSink
{ {
std::filesystem::path dstPath; std::filesystem::path dstPath;
#ifndef _WIN32
/**
* File descriptor for the directory located at dstPath. Used for *at
* operations relative to this file descriptor. This sink must *never*
* follow intermediate symlinks (starting from dstPath) in case a file
* collision is encountered for various reasons like case-insensitivity or
* other types on normalization. using appropriate *at system calls and traversing
* only one path component at a time ensures that writing is race-free and is
* is not susceptible to symlink replacement.
*/
AutoCloseFD dirFd;
#endif
bool startFsync = false; bool startFsync = false;
explicit RestoreSink(bool startFsync) explicit RestoreSink(bool startFsync)
@@ -120,10 +91,6 @@ struct RestoreSink : FileSystemObjectSink
void createDirectory(const CanonPath & path) override; void createDirectory(const CanonPath & path) override;
#ifndef _WIN32
void createDirectory(const CanonPath & path, DirectoryCreatedCallback callback) override;
#endif
void createRegularFile(const CanonPath & path, std::function<void(CreateRegularFileSink &)>) override; void createRegularFile(const CanonPath & path, std::function<void(CreateRegularFileSink &)>) override;
void createSymlink(const CanonPath & path, const std::string & target) override; void createSymlink(const CanonPath & path, const std::string & target) override;
+7
View File
@@ -136,6 +136,13 @@ std::optional<Mode> convertMode(SourceAccessor::Type type);
*/ */
using RestoreHook = SourcePath(Hash); using RestoreHook = SourcePath(Hash);
/**
* Wrapper around `parse` and `RestoreSink`
*
* @param hashAlgo must be `HashAlgo::SHA1` or `HashAlgo::SHA256` for now.
*/
void restore(FileSystemObjectSink & sink, Source & source, HashAlgorithm hashAlgo, std::function<RestoreHook> hook);
/** /**
* Dumps a single file to a sink * Dumps a single file to a sink
* *
@@ -9,7 +9,7 @@ struct SourcePath;
/** /**
* A source accessor that uses the Unix filesystem. * A source accessor that uses the Unix filesystem.
*/ */
class PosixSourceAccessor : virtual public SourceAccessor struct PosixSourceAccessor : virtual SourceAccessor
{ {
/** /**
* Optional root path to prefix all operations into the native file * Optional root path to prefix all operations into the native file
@@ -18,12 +18,8 @@ class PosixSourceAccessor : virtual public SourceAccessor
*/ */
const std::filesystem::path root; const std::filesystem::path root;
const bool trackLastModified = false;
public:
PosixSourceAccessor(); PosixSourceAccessor();
PosixSourceAccessor(std::filesystem::path && root, bool trackLastModified = false); PosixSourceAccessor(std::filesystem::path && root);
/** /**
* The most recent mtime seen by lstat(). This is a hack to * The most recent mtime seen by lstat(). This is a hack to
@@ -47,9 +43,6 @@ public:
* Create a `PosixSourceAccessor` and `SourcePath` corresponding to * Create a `PosixSourceAccessor` and `SourcePath` corresponding to
* some native path. * some native path.
* *
* @param Whether the accessor should return a non-null getLastModified.
* When true the accessor must be used only by a single thread.
*
* The `PosixSourceAccessor` is rooted as far up the tree as * The `PosixSourceAccessor` is rooted as far up the tree as
* possible, (e.g. on Windows it could scoped to a drive like * possible, (e.g. on Windows it could scoped to a drive like
* `C:\`). This allows more `..` parent accessing to work. * `C:\`). This allows more `..` parent accessing to work.
@@ -71,12 +64,7 @@ public:
* and * and
* [`std::filesystem::path::relative_path`](https://en.cppreference.com/w/cpp/filesystem/path/relative_path). * [`std::filesystem::path::relative_path`](https://en.cppreference.com/w/cpp/filesystem/path/relative_path).
*/ */
static SourcePath createAtRoot(const std::filesystem::path & path, bool trackLastModified = false); static SourcePath createAtRoot(const std::filesystem::path & path);
std::optional<std::time_t> getLastModified() override
{
return trackLastModified ? std::optional{mtime} : std::nullopt;
}
private: private:
-6
View File
@@ -166,10 +166,4 @@ public:
} }
}; };
/**
* Check that the string does not contain any NUL bytes and return c_str().
* @throws Error if str contains '\0' bytes.
*/
const char * requireCString(const std::string & str);
} // namespace nix } // namespace nix
-8
View File
@@ -4,15 +4,7 @@
#include <limits> #include <limits>
#include <string> #include <string>
#include "nix/util/file-descriptor.hh"
namespace nix { namespace nix {
/**
* Determine whether \param fd is a terminal.
*/
bool isTTY(Descriptor fd);
/** /**
* Determine whether ANSI escape sequences are appropriate for the * Determine whether ANSI escape sequences are appropriate for the
* present output. * present output.
+2 -37
View File
@@ -33,28 +33,15 @@ auto concatStrings(Parts &&... parts)
return concatStringsSep({}, views); return concatStringsSep({}, views);
} }
/**
* Add quotes around a string.
*/
inline std::string quoteString(std::string_view s, char quote = '\'')
{
std::string result;
result.reserve(s.size() + 2);
result += quote;
result += s;
result += quote;
return result;
}
/** /**
* Add quotes around a collection of strings. * Add quotes around a collection of strings.
*/ */
template<class C> template<class C>
Strings quoteStrings(const C & c, char quote = '\'') Strings quoteStrings(const C & c)
{ {
Strings res; Strings res;
for (auto & s : c) for (auto & s : c)
res.push_back(quoteString(s, quote)); res.push_back("'" + s + "'");
return res; return res;
} }
@@ -242,28 +229,6 @@ std::string stripIndentation(std::string_view s);
*/ */
std::pair<std::string_view, std::string_view> getLine(std::string_view s); std::pair<std::string_view, std::string_view> getLine(std::string_view s);
/**
* Get a pointer to the contents of a `std::optional` if it is set, or a
* null pointer otherise.
*
* Const version.
*/
template<class T>
const T * get(const std::optional<T> & opt)
{
return opt ? &*opt : nullptr;
}
/**
* Non-const counterpart of `const T * get(const std::optional<T>)`.
* Takes a mutable reference, but returns a mutable pointer.
*/
template<class T>
T * get(std::optional<T> & opt)
{
return opt ? &*opt : nullptr;
}
/** /**
* Get a value for the specified key from an associate container. * Get a value for the specified key from an associate container.
*/ */
+6 -10
View File
@@ -7,9 +7,8 @@
namespace nix { namespace nix {
PosixSourceAccessor::PosixSourceAccessor(std::filesystem::path && argRoot, bool trackLastModified) PosixSourceAccessor::PosixSourceAccessor(std::filesystem::path && argRoot)
: root(std::move(argRoot)) : root(std::move(argRoot))
, trackLastModified(trackLastModified)
{ {
assert(root.empty() || root.is_absolute()); assert(root.empty() || root.is_absolute());
displayPrefix = root.string(); displayPrefix = root.string();
@@ -20,11 +19,11 @@ PosixSourceAccessor::PosixSourceAccessor()
{ {
} }
SourcePath PosixSourceAccessor::createAtRoot(const std::filesystem::path & path, bool trackLastModified) SourcePath PosixSourceAccessor::createAtRoot(const std::filesystem::path & path)
{ {
std::filesystem::path path2 = absPath(path); std::filesystem::path path2 = absPath(path);
return { return {
make_ref<PosixSourceAccessor>(path2.root_path(), trackLastModified), make_ref<PosixSourceAccessor>(path2.root_path()),
CanonPath{path2.relative_path().string()}, CanonPath{path2.relative_path().string()},
}; };
} }
@@ -115,12 +114,9 @@ std::optional<SourceAccessor::Stat> PosixSourceAccessor::maybeLstat(const CanonP
auto st = cachedLstat(path); auto st = cachedLstat(path);
if (!st) if (!st)
return std::nullopt; return std::nullopt;
// This makes the accessor thread-unsafe, but we only seem to use the actual value in a single threaded context in
/* The contract is that trackLastModified implies that the caller uses the accessor // `src/libfetchers/path.cc`.
from a single thread. Thus this is not a CAS loop. */ mtime = std::max(mtime, st->st_mtime);
if (trackLastModified)
mtime = std::max(mtime, st->st_mtime);
return Stat{ return Stat{
.type = S_ISREG(st->st_mode) ? tRegular .type = S_ISREG(st->st_mode) ? tRegular
: S_ISDIR(st->st_mode) ? tDirectory : S_ISDIR(st->st_mode) ? tDirectory
-11
View File
@@ -5,7 +5,6 @@
#include "nix/util/strings-inline.hh" #include "nix/util/strings-inline.hh"
#include "nix/util/os-string.hh" #include "nix/util/os-string.hh"
#include "nix/util/error.hh" #include "nix/util/error.hh"
#include "nix/util/util.hh"
namespace nix { namespace nix {
@@ -153,14 +152,4 @@ std::string optionalBracket(std::string_view prefix, std::string_view content, s
return result; return result;
} }
const char * requireCString(const std::string & s)
{
if (std::memchr(s.data(), '\0', s.size())) [[unlikely]] {
using namespace std::string_view_literals;
auto str = replaceStrings(s, "\0"sv, "␀"sv);
throw Error("string '%s' with null (\\0) bytes used where it's not allowed", str);
}
return s.c_str();
}
} // namespace nix } // namespace nix
-10
View File
@@ -64,16 +64,6 @@ inline std::pair<int, size_t> charWidthUTF8Helper(std::string_view s)
namespace nix { namespace nix {
bool isTTY(Descriptor fd)
{
#ifndef _WIN32
return isatty(fd);
#else
DWORD mode;
return GetConsoleMode(fd, &mode);
#endif
}
bool isTTY() bool isTTY()
{ {
static const bool tty = isatty(STDERR_FILENO) && getEnv("TERM").value_or("dumb") != "dumb" static const bool tty = isatty(STDERR_FILENO) && getEnv("TERM").value_or("dumb") != "dumb"
+2 -5
View File
@@ -327,11 +327,8 @@ Path renderUrlPathEnsureLegal(const std::vector<std::string> & urlPath)
/* This is only really valid for UNIX. Windows has more restrictions. */ /* This is only really valid for UNIX. Windows has more restrictions. */
if (comp.contains('/')) if (comp.contains('/'))
throw BadURL("URL path component '%s' contains '/', which is not allowed in file names", comp); throw BadURL("URL path component '%s' contains '/', which is not allowed in file names", comp);
if (comp.contains(char(0))) { if (comp.contains(char(0)))
using namespace std::string_view_literals; throw BadURL("URL path component '%s' contains NUL byte which is not allowed", comp);
auto str = replaceStrings(comp, "\0"sv, "␀"sv);
throw BadURL("URL path component '%s' contains NUL byte which is not allowed", str);
}
} }
return concatStringsSep("/", urlPath); return concatStringsSep("/", urlPath);
+1 -1
View File
@@ -75,7 +75,7 @@ struct CmdCatNar : StoreCommand, MixCat
void run(ref<Store> store) override void run(ref<Store> store) override
{ {
AutoCloseFD fd = toDescriptor(open(narPath.c_str(), O_RDONLY)); AutoCloseFD fd = open(narPath.c_str(), O_RDONLY);
if (!fd) if (!fd)
throw SysError("opening NAR file '%s'", narPath); throw SysError("opening NAR file '%s'", narPath);
auto source = FdSource{fd.get()}; auto source = FdSource{fd.get()};
+1 -2
View File
@@ -1,14 +1,13 @@
#include "nix/cmd/command.hh" #include "nix/cmd/command.hh"
#include "nix/store/store-api.hh" #include "nix/store/store-api.hh"
#include "nix/util/archive.hh" #include "nix/util/archive.hh"
#include "nix/util/terminal.hh"
using namespace nix; using namespace nix;
static FdSink getNarSink() static FdSink getNarSink()
{ {
auto fd = getStandardOutput(); auto fd = getStandardOutput();
if (isTTY(fd)) if (isatty(fd))
throw UsageError("refusing to write NAR to a terminal"); throw UsageError("refusing to write NAR to a terminal");
return FdSink(std::move(fd)); return FdSink(std::move(fd));
} }
+1 -1
View File
@@ -45,7 +45,7 @@ struct CmdFlakePrefetchInputs : FlakeCommand
if (auto lockedNode = dynamic_cast<const LockedNode *>(&node)) { if (auto lockedNode = dynamic_cast<const LockedNode *>(&node)) {
try { try {
Activity act(*logger, lvlInfo, actUnknown, fmt("fetching '%s'", lockedNode->lockedRef)); Activity act(*logger, lvlInfo, actUnknown, fmt("fetching '%s'", lockedNode->lockedRef));
auto accessor = lockedNode->lockedRef.input.getAccessor(fetchSettings, *store).first; auto accessor = lockedNode->lockedRef.input.getAccessor(store).first;
fetchToStore( fetchToStore(
fetchSettings, *store, accessor, FetchMode::Copy, lockedNode->lockedRef.input.getName()); fetchSettings, *store, accessor, FetchMode::Copy, lockedNode->lockedRef.input.getName());
} catch (Error & e) { } catch (Error & e) {
+10 -12
View File
@@ -240,12 +240,12 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON
j["lastModified"] = *lastModified; j["lastModified"] = *lastModified;
j["path"] = storePath; j["path"] = storePath;
j["locks"] = lockedFlake.lockFile.toJSON().first; j["locks"] = lockedFlake.lockFile.toJSON().first;
if (auto fingerprint = lockedFlake.getFingerprint(*store, fetchSettings)) if (auto fingerprint = lockedFlake.getFingerprint(store, fetchSettings))
j["fingerprint"] = fingerprint->to_string(HashFormat::Base16, false); j["fingerprint"] = fingerprint->to_string(HashFormat::Base16, false);
printJSON(j); printJSON(j);
} else { } else {
logger->cout(ANSI_BOLD "Resolved URL:" ANSI_NORMAL " %s", flake.resolvedRef.to_string()); logger->cout(ANSI_BOLD "Resolved URL:" ANSI_NORMAL " %s", flake.resolvedRef.to_string());
if (flake.lockedRef.input.isLocked(fetchSettings)) if (flake.lockedRef.input.isLocked())
logger->cout(ANSI_BOLD "Locked URL:" ANSI_NORMAL " %s", flake.lockedRef.to_string()); logger->cout(ANSI_BOLD "Locked URL:" ANSI_NORMAL " %s", flake.lockedRef.to_string());
if (flake.description) if (flake.description)
logger->cout(ANSI_BOLD "Description:" ANSI_NORMAL " %s", *flake.description); logger->cout(ANSI_BOLD "Description:" ANSI_NORMAL " %s", *flake.description);
@@ -260,7 +260,7 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON
logger->cout( logger->cout(
ANSI_BOLD "Last modified:" ANSI_NORMAL " %s", ANSI_BOLD "Last modified:" ANSI_NORMAL " %s",
std::put_time(std::localtime(&*lastModified), "%F %T")); std::put_time(std::localtime(&*lastModified), "%F %T"));
if (auto fingerprint = lockedFlake.getFingerprint(*store, fetchSettings)) if (auto fingerprint = lockedFlake.getFingerprint(store, fetchSettings))
logger->cout( logger->cout(
ANSI_BOLD "Fingerprint:" ANSI_NORMAL " %s", fingerprint->to_string(HashFormat::Base16, false)); ANSI_BOLD "Fingerprint:" ANSI_NORMAL " %s", fingerprint->to_string(HashFormat::Base16, false));
@@ -1019,7 +1019,7 @@ struct CmdFlakeNew : CmdFlakeInitCommon
struct CmdFlakeClone : FlakeCommand struct CmdFlakeClone : FlakeCommand
{ {
std::filesystem::path destDir; Path destDir;
std::string description() override std::string description() override
{ {
@@ -1049,7 +1049,7 @@ struct CmdFlakeClone : FlakeCommand
if (destDir.empty()) if (destDir.empty())
throw Error("missing flag '--dest'"); throw Error("missing flag '--dest'");
getFlakeRef().resolve(fetchSettings, *store).input.clone(fetchSettings, *store, destDir); getFlakeRef().resolve(store).input.clone(destDir);
} }
}; };
@@ -1100,7 +1100,7 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun, MixNoCheckSigs
std::optional<StorePath> storePath; std::optional<StorePath> storePath;
if (!(*inputNode)->lockedRef.input.isRelative()) { if (!(*inputNode)->lockedRef.input.isRelative()) {
storePath = dryRun ? (*inputNode)->lockedRef.input.computeStorePath(*store) storePath = dryRun ? (*inputNode)->lockedRef.input.computeStorePath(*store)
: (*inputNode)->lockedRef.input.fetchToStore(fetchSettings, *store).first; : (*inputNode)->lockedRef.input.fetchToStore(store).first;
sources.insert(*storePath); sources.insert(*storePath);
} }
if (json) { if (json) {
@@ -1324,10 +1324,8 @@ struct CmdFlakeShow : FlakeCommand, MixJSON
try { try {
if (visitor.isDerivation()) if (visitor.isDerivation())
showDerivation(); showDerivation();
else { else
auto name = visitor.getAttrPathStr(state->s.name); throw Error("expected a derivation");
logger->warn(fmt("%s is not a derivation", name));
}
} catch (IFDError & e) { } catch (IFDError & e) {
if (!json) { if (!json) {
logger->cout( logger->cout(
@@ -1498,8 +1496,8 @@ struct CmdFlakePrefetch : FlakeCommand, MixJSON
void run(ref<Store> store) override void run(ref<Store> store) override
{ {
auto originalRef = getFlakeRef(); auto originalRef = getFlakeRef();
auto resolvedRef = originalRef.resolve(fetchSettings, *store); auto resolvedRef = originalRef.resolve(store);
auto [accessor, lockedRef] = resolvedRef.lazyFetch(getEvalState()->fetchSettings, *store); auto [accessor, lockedRef] = resolvedRef.lazyFetch(store);
auto storePath = auto storePath =
fetchToStore(getEvalState()->fetchSettings, *store, accessor, FetchMode::Copy, lockedRef.input.getName()); fetchToStore(getEvalState()->fetchSettings, *store, accessor, FetchMode::Copy, lockedRef.input.getName());
auto hash = store->queryPathInfo(storePath)->narHash; auto hash = store->queryPathInfo(storePath)->narHash;
+1 -1
View File
@@ -145,7 +145,7 @@ struct CmdLsNar : Command, MixLs
void run() override void run() override
{ {
AutoCloseFD fd = toDescriptor(open(narPath.c_str(), O_RDONLY)); AutoCloseFD fd = open(narPath.c_str(), O_RDONLY);
if (!fd) if (!fd)
throw SysError("opening NAR file '%s'", narPath); throw SysError("opening NAR file '%s'", narPath);
auto source = FdSource{fd.get()}; auto source = FdSource{fd.get()};
+11 -29
View File
@@ -193,38 +193,20 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs, virtual RootArgs
std::string dumpCli() std::string dumpCli()
{ {
using nlohmann::json; auto res = nlohmann::json::object();
auto res = json::object();
res["args"] = toJSON(); res["args"] = toJSON();
{ auto stores = nlohmann::json::object();
auto & stores = res["stores"] = json::object(); for (auto & [storeName, implem] : Implementations::registered()) {
for (auto & [storeName, implem] : Implementations::registered()) { auto & j = stores[storeName];
auto & j = stores[storeName]; j["doc"] = implem.doc;
j["doc"] = implem.doc; j["uri-schemes"] = implem.uriSchemes;
j["uri-schemes"] = implem.uriSchemes; j["settings"] = implem.getConfig()->toJSON();
j["settings"] = implem.getConfig()->toJSON(); j["experimentalFeature"] = implem.experimentalFeature;
j["experimentalFeature"] = implem.experimentalFeature;
}
} }
res["stores"] = std::move(stores);
{ res["fetchers"] = fetchers::dumpRegisterInputSchemeInfo();
auto & fetchers = res["fetchers"] = json::object();
for (const auto & [schemeName, scheme] : fetchers::getAllInputSchemes()) {
auto & s = fetchers[schemeName] = json::object();
s["description"] = scheme->schemeDescription();
auto & attrs = s["allowedAttrs"] = json::object();
for (auto & [fieldName, field] : scheme->allowedAttrs()) {
auto & f = attrs[fieldName] = json::object();
f["type"] = field.type;
f["required"] = field.required;
f["doc"] = stripIndentation(field.doc);
}
}
};
return res.dump(); return res.dump();
} }
@@ -458,7 +440,7 @@ void mainWrapped(int argc, char ** argv)
if (!primOp->doc) if (!primOp->doc)
continue; continue;
b["args"] = primOp->args; b["args"] = primOp->args;
b["doc"] = trim(stripIndentation(*primOp->doc)); b["doc"] = trim(stripIndentation(primOp->doc));
if (primOp->experimentalFeature) if (primOp->experimentalFeature)
b["experimental-feature"] = primOp->experimentalFeature; b["experimental-feature"] = primOp->experimentalFeature;
builtinsJson.emplace(state.symbols[builtin.name], std::move(b)); builtinsJson.emplace(state.symbols[builtin.name], std::move(b));
+2 -2
View File
@@ -554,9 +554,9 @@ static void main_nix_build(int argc, char ** argv)
env["NIX_STORE"] = store->storeDir; env["NIX_STORE"] = store->storeDir;
env["NIX_BUILD_CORES"] = fmt("%d", settings.buildCores ? settings.buildCores : settings.getDefaultCores()); env["NIX_BUILD_CORES"] = fmt("%d", settings.buildCores ? settings.buildCores : settings.getDefaultCores());
DerivationOptions<StorePath> drvOptions; DerivationOptions drvOptions;
try { try {
drvOptions = derivationOptionsFromStructuredAttrs(*store, drv.env, get(drv.structuredAttrs)); drvOptions = DerivationOptions::fromStructuredAttrs(drv.env, drv.structuredAttrs);
} catch (Error & e) { } catch (Error & e) {
e.addTrace({}, "while parsing derivation '%s'", store->printStorePath(packageInfo.requireDrvPath())); e.addTrace({}, "while parsing derivation '%s'", store->printStorePath(packageInfo.requireDrvPath()));
throw; throw;
+3 -3
View File
@@ -121,7 +121,7 @@ static void update(const StringSet & channelNames)
// We want to download the url to a file to see if it's a tarball while also checking if we // We want to download the url to a file to see if it's a tarball while also checking if we
// got redirected in the process, so that we can grab the various parts of a nix channel // got redirected in the process, so that we can grab the various parts of a nix channel
// definition from a consistent location if the redirect changes mid-download. // definition from a consistent location if the redirect changes mid-download.
auto result = fetchers::downloadFile(*store, fetchSettings, url, std::string(baseNameOf(url))); auto result = fetchers::downloadFile(store, fetchSettings, url, std::string(baseNameOf(url)));
url = result.effectiveUrl; url = result.effectiveUrl;
bool unpacked = false; bool unpacked = false;
@@ -139,10 +139,10 @@ static void update(const StringSet & channelNames)
if (!unpacked) { if (!unpacked) {
// Download the channel tarball. // Download the channel tarball.
try { try {
result = fetchers::downloadFile(*store, fetchSettings, url + "/nixexprs.tar.xz", "nixexprs.tar.xz"); result = fetchers::downloadFile(store, fetchSettings, url + "/nixexprs.tar.xz", "nixexprs.tar.xz");
} catch (FileTransferError & e) { } catch (FileTransferError & e) {
result = result =
fetchers::downloadFile(*store, fetchSettings, url + "/nixexprs.tar.bz2", "nixexprs.tar.bz2"); fetchers::downloadFile(store, fetchSettings, url + "/nixexprs.tar.bz2", "nixexprs.tar.bz2");
} }
} }
// Regardless of where it came from, add the expression representing this channel to accumulated expression // Regardless of where it came from, add the expression representing this channel to accumulated expression
+2 -1
View File
@@ -138,7 +138,8 @@ std::tuple<StorePath, Hash> prefetchFile(
Activity act(*logger, lvlChatty, actUnknown, fmt("adding '%s' to the store", url.to_string())); Activity act(*logger, lvlChatty, actUnknown, fmt("adding '%s' to the store", url.to_string()));
auto info = store->addToStoreSlow(*name, makeFSSourceAccessor(tmpFile), method, hashAlgo, {}, expectedHash); auto info = store->addToStoreSlow(
*name, PosixSourceAccessor::createAtRoot(tmpFile), method, hashAlgo, {}, expectedHash);
storePath = info.path; storePath = info.path;
assert(info.ca); assert(info.ca);
hash = info.ca->hash; hash = info.ca->hash;
+1 -1
View File
@@ -23,6 +23,6 @@ R""(
This command shows the difference between the closures of subsequent This command shows the difference between the closures of subsequent
versions of a profile. See [`nix store versions of a profile. See [`nix store
diff-closures`](./nix3-store-diff-closures.md) for details. diff-closures`](nix3-store-diff-closures.md) for details.
)"" )""
Loaded 100 of 109 files, more files were not shown because too many files have changed in this diff. Show more