Compare commits
45
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c98026e982 | ||
|
|
8c10104e9e | ||
|
|
f18af849fe | ||
|
|
1290b7e53d | ||
|
|
e7078d4de1 | ||
|
|
638b7ec6c5 | ||
|
|
f2671886f5 | ||
|
|
fa6f69f9c5 | ||
|
|
6b4a86a9e3 | ||
|
|
2922e3082e | ||
|
|
72232bc28a | ||
|
|
eaee0b4740 | ||
|
|
46beb9af76 | ||
|
|
efcb9e36a9 | ||
|
|
7d89e46f65 | ||
|
|
e5e5c819dd | ||
|
|
147a34c573 | ||
|
|
39f9f5d1b6 | ||
|
|
d692729759 | ||
|
|
041d2374dd | ||
|
|
4c50cf798e | ||
|
|
ea5302c4a2 | ||
|
|
cb16cd707c | ||
|
|
b21fc05047 | ||
|
|
5ea81f5b8f | ||
|
|
4711720efe | ||
|
|
dc1a5135ee | ||
|
|
67535263a5 | ||
|
|
7bd9eef772 | ||
|
|
0d3750e902 | ||
|
|
d626348f42 | ||
|
|
ea09198f93 | ||
|
|
0b46025f7a | ||
|
|
2dacdd110c | ||
|
|
3ba49d7ec2 | ||
|
|
2dd2142754 | ||
|
|
3285b68044 | ||
|
|
bc85e20fb9 | ||
|
|
c3e7e88889 | ||
|
|
99cb85cd37 | ||
|
|
95e1f463c8 | ||
|
|
6591a4119c | ||
|
|
18a5589f9a | ||
|
|
f3090ef703 | ||
|
|
3a1301cd6d |
No files matched your search
@@ -139,3 +139,14 @@ pull_request_rules:
|
||||
labels:
|
||||
- automatic backport
|
||||
- merge-queue
|
||||
|
||||
- name: backport patches to 2.29
|
||||
conditions:
|
||||
- label=backport 2.29-maintenance
|
||||
actions:
|
||||
backport:
|
||||
branches:
|
||||
- "2.29-maintenance"
|
||||
labels:
|
||||
- automatic backport
|
||||
- merge-queue
|
||||
@@ -0,0 +1,17 @@
|
||||
---
|
||||
synopsis: Non-flake inputs now contain a `sourceInfo` attribute
|
||||
issues: 13164
|
||||
prs: 13170
|
||||
---
|
||||
|
||||
Flakes have always a `sourceInfo` attribute which describes the source of the flake.
|
||||
The `sourceInfo.outPath` is often identical to the flake's `outPath`, however it can differ when the flake is located in a subdirectory of its source.
|
||||
|
||||
Non-flake inputs (i.e. inputs with `flake = false`) can also be located at some path _within_ a wider source.
|
||||
This usually happens when defining a relative path input within the same source as the parent flake, e.g. `inputs.foo.url = ./some-file.nix`.
|
||||
Such relative inputs will now inherit their parent's `sourceInfo`.
|
||||
|
||||
This also means it is now possible to use `?dir=subdir` on non-flake inputs.
|
||||
|
||||
This iterates on the work done in 2.26 to improve relative path support ([#10089](https://github.com/NixOS/nix/pull/10089)),
|
||||
and resolves a regression introduced in 2.28 relating to nested relative path inputs ([#13164](https://github.com/NixOS/nix/issues/13164)).
|
||||
@@ -0,0 +1,17 @@
|
||||
---
|
||||
synopsis: Revert incomplete closure mixed download and build feature
|
||||
issues: [77, 12628]
|
||||
prs: [13176]
|
||||
---
|
||||
|
||||
Since Nix 1.3 (299141ecbd08bae17013226dbeae71e842b4fdd7 in 2013) Nix has attempted to mix together upstream fresh builds and downstream substitutions when remote substuters contain an "incomplete closure" (have some store objects, but not the store objects they reference).
|
||||
This feature is now removed.
|
||||
|
||||
Worst case, removing this feature could cause more building downstream, but it should not cause outright failures, since this is not happening for opaque store objects that we don't know how to build if we decide not to substitute.
|
||||
In practice, however, we doubt even the more building is very likely to happen.
|
||||
Remote stores that are missing dependencies in arbitrary ways (e.g. corruption) don't seem to be very common.
|
||||
|
||||
On the contrary, when remote stores fail to implement the [closure property](@docroot@/store/store-object.md#closure-property), it is usually an *intentional* choice on the part of the remote store, because it wishes to serve as an "overlay" store over another store, such as `https://cache.nixos.org`.
|
||||
If an "incomplete closure" is encountered in that situation, the right fix is not to do some sort of "franken-building" as this feature implemented, but instead to make sure both substituters are enabled in the settings.
|
||||
|
||||
(In the future, we should make it easier for remote stores to indicate this to clients, to catch settings that won't work in general before a missing dependency is actually encountered.)
|
||||
@@ -20,8 +20,9 @@ prs: 1238
|
||||
Here's one or more paragraphs that describe the change.
|
||||
|
||||
- It's markdown
|
||||
- Add references to the manual using @docroot@
|
||||
- Add references to the manual using [links like this](@_at_docroot@/example.md)
|
||||
```
|
||||
<!-- for the raw markdown readers: that means using @docroot@ -->
|
||||
|
||||
Significant changes should add the following header, which moves them to the top.
|
||||
|
||||
|
||||
@@ -73,7 +73,7 @@ Derivations can declare some infrequently used optional attributes.
|
||||
|
||||
> **Warning**
|
||||
>
|
||||
> If set to `true`, other advanced attributes such as [`allowedReferences`](#adv-attr-allowedReferences), [`allowedReferences`](#adv-attr-allowedReferences), [`allowedRequisites`](#adv-attr-allowedRequisites),
|
||||
> If set to `true`, other advanced attributes such as [`allowedReferences`](#adv-attr-allowedReferences), [`allowedRequisites`](#adv-attr-allowedRequisites),
|
||||
[`disallowedReferences`](#adv-attr-disallowedReferences) and [`disallowedRequisites`](#adv-attr-disallowedRequisites), maxSize, and maxClosureSize.
|
||||
will have no effect.
|
||||
|
||||
|
||||
@@ -282,7 +282,7 @@ type DerivingPath = ConstantPath | OutputPath;
|
||||
|
||||
Under this extended model, `DerivingPath`s are thus inductively built up from a root `ConstantPath`, wrapped with zero or more outer `OutputPath`s.
|
||||
|
||||
### Encoding {#deriving-path-encoding}
|
||||
### Encoding {#deriving-path-encoding-higher-order}
|
||||
|
||||
The encoding is adjusted in the natural way, encoding the `drv` field recursively using the same deriving path encoding.
|
||||
The result of this is that it is possible to have a chain of `^<output-name>` at the end of the final string, as opposed to just a single one.
|
||||
|
||||
@@ -23,7 +23,7 @@ The output spec for an output with a fixed content addresses additionally contai
|
||||
> **Design note**
|
||||
>
|
||||
> In principle, the output spec could also specify the references the store object should have, since the references and file system objects are equally parts of a content-addressed store object proper that contribute to its content-addressed.
|
||||
> However, at this time, the references are not not done because all fixed content-addressed outputs are required to have no references (including no self-reference).
|
||||
> However, at this time, the references are not done because all fixed content-addressed outputs are required to have no references (including no self-reference).
|
||||
>
|
||||
> Also in principle, rather than specifying the references and file system object data with separate hashes, a single hash that constraints both could be used.
|
||||
> This could be done with the final store path's digest, or better yet, the hash that will become the store path's digest before it is truncated.
|
||||
@@ -116,7 +116,7 @@ Because the derivation output is not fixed (just like with [input addressing]),
|
||||
> (The "environment", in this case, consists of attributes such as the Operating System Nix runs atop, along with the operating-system-specific privileges that Nix has been granted.
|
||||
> Because of how conventional operating systems like macos, Linux, etc. work, granting builders *fewer* privileges may ironically require that Nix be run with *more* privileges.)
|
||||
|
||||
That said, derivations producing floating content-addressed outputs may declare their builders as impure (like the builders of derivations producing producing fixed outputs).
|
||||
That said, derivations producing floating content-addressed outputs may declare their builders as impure (like the builders of derivations producing fixed outputs).
|
||||
This is provisionally supported as part of the [`impure-derivations`][xp-feature-impure-derivations] experimental feature.
|
||||
|
||||
### Compatibility negotiation
|
||||
@@ -144,7 +144,7 @@ A *deterministic* content-addressing derivation should produce outputs with the
|
||||
The choice of provisional store path can be thought of as an impurity, since it is an arbitrary choice.
|
||||
|
||||
If provisional outputs paths are deterministically chosen, we are in the first branch of part (1).
|
||||
The builder the data it produces based on it in arbitrary ways, but this gets us closer to to [input addressing].
|
||||
The builder the data it produces based on it in arbitrary ways, but this gets us closer to [input addressing].
|
||||
Deterministically choosing the provisional path may be considered "complete sandboxing" by removing an impurity, but this is unsatisfactory
|
||||
|
||||
<!--
|
||||
|
||||
@@ -83,7 +83,7 @@ The rules for this are fairly concise:
|
||||
|
||||
- A content-addressing derivation may be pure or impure
|
||||
|
||||
- If it is impure, it may be be fixed (typical), or it may be floating if the additional [`impure-derivations`][xp-feature-impure-derivations] experimental feature is enabled.
|
||||
- If it is impure, it may be fixed (typical), or it may be floating if the additional [`impure-derivations`][xp-feature-impure-derivations] experimental feature is enabled.
|
||||
|
||||
- If it is pure, it must be floating.
|
||||
|
||||
|
||||
@@ -57,6 +57,9 @@ def recursive_replace(data: dict[str, t.Any], book_root: Path, search_path: Path
|
||||
).replace(
|
||||
'@docroot@',
|
||||
("../" * len(path_to_chapter.parent.parts) or "./")[:-1]
|
||||
).replace(
|
||||
'@_at_',
|
||||
'@'
|
||||
),
|
||||
sub_items = [
|
||||
recursive_replace(sub_item, book_root, search_path)
|
||||
|
||||
Generated
+3
-3
@@ -63,11 +63,11 @@
|
||||
},
|
||||
"nixpkgs": {
|
||||
"locked": {
|
||||
"lastModified": 1746141548,
|
||||
"narHash": "sha256-IgBWhX7A2oJmZFIrpRuMnw5RAufVnfvOgHWgIdds+hc=",
|
||||
"lastModified": 1747179050,
|
||||
"narHash": "sha256-qhFMmDkeJX9KJwr5H32f1r7Prs7XbQWtO0h3V0a0rFY=",
|
||||
"owner": "NixOS",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "f02fddb8acef29a8b32f10a335d44828d7825b78",
|
||||
"rev": "adaa24fbf46737f3f1b5497bf64bae750f82942e",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
|
||||
@@ -63,6 +63,7 @@ scope: {
|
||||
"--with-coroutine"
|
||||
"--with-iostreams"
|
||||
];
|
||||
enableIcu = false;
|
||||
}).overrideAttrs
|
||||
(old: {
|
||||
# Need to remove `--with-*` to use `--with-libraries=...`
|
||||
|
||||
@@ -119,9 +119,6 @@ pkgs.nixComponents2.nix-util.overrideAttrs (
|
||||
(pkgs.writeScriptBin "pre-commit-hooks-install" modular.pre-commit.settings.installationScript)
|
||||
pkgs.buildPackages.nixfmt-rfc-style
|
||||
]
|
||||
# TODO: Remove the darwin check once
|
||||
# https://github.com/NixOS/nixpkgs/pull/291814 is available
|
||||
++ lib.optional (stdenv.cc.isClang && !stdenv.buildPlatform.isDarwin) pkgs.buildPackages.bear
|
||||
++ lib.optional (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform) (
|
||||
lib.hiPrio pkgs.buildPackages.clang-tools
|
||||
)
|
||||
|
||||
@@ -34,7 +34,12 @@ EvalSettings evalSettings {
|
||||
auto flakeRef = parseFlakeRef(fetchSettings, std::string { rest }, {}, true, false);
|
||||
debug("fetching flake search path element '%s''", rest);
|
||||
auto [accessor, lockedRef] = flakeRef.resolve(state.store).lazyFetch(state.store);
|
||||
auto storePath = nix::fetchToStore(*state.store, SourcePath(accessor), FetchMode::Copy, lockedRef.input.getName());
|
||||
auto storePath = nix::fetchToStore(
|
||||
state.fetchSettings,
|
||||
*state.store,
|
||||
SourcePath(accessor),
|
||||
FetchMode::Copy,
|
||||
lockedRef.input.getName());
|
||||
state.allowPath(storePath);
|
||||
return state.storePath(storePath);
|
||||
},
|
||||
@@ -177,7 +182,11 @@ SourcePath lookupFileArg(EvalState & state, std::string_view s, const Path * bas
|
||||
state.store,
|
||||
state.fetchSettings,
|
||||
EvalSettings::resolvePseudoUrl(s));
|
||||
auto storePath = fetchToStore(*state.store, SourcePath(accessor), FetchMode::Copy);
|
||||
auto storePath = fetchToStore(
|
||||
state.fetchSettings,
|
||||
*state.store,
|
||||
SourcePath(accessor),
|
||||
FetchMode::Copy);
|
||||
return state.storePath(storePath);
|
||||
}
|
||||
|
||||
@@ -185,7 +194,12 @@ SourcePath lookupFileArg(EvalState & state, std::string_view s, const Path * bas
|
||||
experimentalFeatureSettings.require(Xp::Flakes);
|
||||
auto flakeRef = parseFlakeRef(fetchSettings, std::string(s.substr(6)), {}, true, false);
|
||||
auto [accessor, lockedRef] = flakeRef.resolve(state.store).lazyFetch(state.store);
|
||||
auto storePath = nix::fetchToStore(*state.store, SourcePath(accessor), FetchMode::Copy, lockedRef.input.getName());
|
||||
auto storePath = nix::fetchToStore(
|
||||
state.fetchSettings,
|
||||
*state.store,
|
||||
SourcePath(accessor),
|
||||
FetchMode::Copy,
|
||||
lockedRef.input.getName());
|
||||
state.allowPath(storePath);
|
||||
return state.storePath(storePath);
|
||||
}
|
||||
|
||||
@@ -45,7 +45,7 @@ ref<InstallableValue> InstallableValue::require(ref<Installable> installable)
|
||||
std::optional<DerivedPathWithInfo> InstallableValue::trySinglePathToDerivedPaths(Value & v, const PosIdx pos, std::string_view errorCtx)
|
||||
{
|
||||
if (v.type() == nPath) {
|
||||
auto storePath = fetchToStore(*state->store, v.path(), FetchMode::Copy);
|
||||
auto storePath = fetchToStore(state->fetchSettings, *state->store, v.path(), FetchMode::Copy);
|
||||
return {{
|
||||
.path = DerivedPath::Opaque {
|
||||
.path = std::move(storePath),
|
||||
@@ -57,7 +57,8 @@ std::optional<DerivedPathWithInfo> InstallableValue::trySinglePathToDerivedPaths
|
||||
else if (v.type() == nString) {
|
||||
return {{
|
||||
.path = DerivedPath::fromSingle(
|
||||
state->coerceToSingleDerivedPath(pos, v, errorCtx)),
|
||||
state->devirtualize(
|
||||
state->coerceToSingleDerivedPath(pos, v, errorCtx))),
|
||||
.info = make_ref<ExtraPathInfo>(),
|
||||
}};
|
||||
}
|
||||
|
||||
@@ -23,6 +23,11 @@ struct Arbitrary<NixStringContextElem::DrvDeep> {
|
||||
static Gen<NixStringContextElem::DrvDeep> arbitrary();
|
||||
};
|
||||
|
||||
template<>
|
||||
struct Arbitrary<NixStringContextElem::Path> {
|
||||
static Gen<NixStringContextElem::Path> arbitrary();
|
||||
};
|
||||
|
||||
template<>
|
||||
struct Arbitrary<NixStringContextElem> {
|
||||
static Gen<NixStringContextElem> arbitrary();
|
||||
|
||||
@@ -15,6 +15,15 @@ Gen<NixStringContextElem::DrvDeep> Arbitrary<NixStringContextElem::DrvDeep>::arb
|
||||
});
|
||||
}
|
||||
|
||||
Gen<NixStringContextElem::Path> Arbitrary<NixStringContextElem::Path>::arbitrary()
|
||||
{
|
||||
return gen::map(gen::arbitrary<StorePath>(), [](StorePath storePath) {
|
||||
return NixStringContextElem::Path{
|
||||
.storePath = storePath,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
Gen<NixStringContextElem> Arbitrary<NixStringContextElem>::arbitrary()
|
||||
{
|
||||
return gen::mapcat(
|
||||
@@ -30,6 +39,9 @@ Gen<NixStringContextElem> Arbitrary<NixStringContextElem>::arbitrary()
|
||||
case 2:
|
||||
return gen::map(
|
||||
gen::arbitrary<NixStringContextElem::Built>(), [](NixStringContextElem a) { return a; });
|
||||
case 3:
|
||||
return gen::map(
|
||||
gen::arbitrary<NixStringContextElem::Path>(), [](NixStringContextElem a) { return a; });
|
||||
default:
|
||||
assert(false);
|
||||
}
|
||||
|
||||
@@ -618,18 +618,21 @@ string_t AttrCursor::getStringWithContext()
|
||||
if (auto s = std::get_if<string_t>(&cachedValue->second)) {
|
||||
bool valid = true;
|
||||
for (auto & c : s->second) {
|
||||
const StorePath & path = std::visit(overloaded {
|
||||
[&](const NixStringContextElem::DrvDeep & d) -> const StorePath & {
|
||||
return d.drvPath;
|
||||
const StorePath * path = std::visit(overloaded {
|
||||
[&](const NixStringContextElem::DrvDeep & d) -> const StorePath * {
|
||||
return &d.drvPath;
|
||||
},
|
||||
[&](const NixStringContextElem::Built & b) -> const StorePath & {
|
||||
return b.drvPath->getBaseStorePath();
|
||||
[&](const NixStringContextElem::Built & b) -> const StorePath * {
|
||||
return &b.drvPath->getBaseStorePath();
|
||||
},
|
||||
[&](const NixStringContextElem::Opaque & o) -> const StorePath & {
|
||||
return o.path;
|
||||
[&](const NixStringContextElem::Opaque & o) -> const StorePath * {
|
||||
return &o.path;
|
||||
},
|
||||
[&](const NixStringContextElem::Path & p) -> const StorePath * {
|
||||
return nullptr;
|
||||
},
|
||||
}, c.raw);
|
||||
if (!root->state.store->isValidPath(path)) {
|
||||
if (!path || !root->state.store->isValidPath(*path)) {
|
||||
valid = false;
|
||||
break;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
#include "nix/expr/eval-profiler.hh"
|
||||
#include "nix/expr/nixexpr.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
void EvalProfiler::preFunctionCallHook(
|
||||
const EvalState & state, const Value & v, std::span<Value *> args, const PosIdx pos)
|
||||
{
|
||||
}
|
||||
|
||||
void EvalProfiler::postFunctionCallHook(
|
||||
const EvalState & state, const Value & v, std::span<Value *> args, const PosIdx pos)
|
||||
{
|
||||
}
|
||||
|
||||
void MultiEvalProfiler::preFunctionCallHook(
|
||||
const EvalState & state, const Value & v, std::span<Value *> args, const PosIdx pos)
|
||||
{
|
||||
for (auto & profiler : profilers) {
|
||||
if (profiler->getNeededHooks().test(Hook::preFunctionCall))
|
||||
profiler->preFunctionCallHook(state, v, args, pos);
|
||||
}
|
||||
}
|
||||
|
||||
void MultiEvalProfiler::postFunctionCallHook(
|
||||
const EvalState & state, const Value & v, std::span<Value *> args, const PosIdx pos)
|
||||
{
|
||||
for (auto & profiler : profilers) {
|
||||
if (profiler->getNeededHooks().test(Hook::postFunctionCall))
|
||||
profiler->postFunctionCallHook(state, v, args, pos);
|
||||
}
|
||||
}
|
||||
|
||||
EvalProfiler::Hooks MultiEvalProfiler::getNeededHooksImpl() const
|
||||
{
|
||||
Hooks hooks;
|
||||
for (auto & p : profilers)
|
||||
hooks |= p->getNeededHooks();
|
||||
return hooks;
|
||||
}
|
||||
|
||||
void MultiEvalProfiler::addProfiler(ref<EvalProfiler> profiler)
|
||||
{
|
||||
profilers.push_back(profiler);
|
||||
invalidateNeededHooks();
|
||||
}
|
||||
|
||||
}
|
||||
+64
-17
@@ -15,6 +15,7 @@
|
||||
#include "nix/expr/print.hh"
|
||||
#include "nix/fetchers/filtering-source-accessor.hh"
|
||||
#include "nix/util/memory-source-accessor.hh"
|
||||
#include "nix/util/mounted-source-accessor.hh"
|
||||
#include "nix/expr/gc-small-vector.hh"
|
||||
#include "nix/util/url.hh"
|
||||
#include "nix/fetchers/fetch-to-store.hh"
|
||||
@@ -269,7 +270,7 @@ EvalState::EvalState(
|
||||
exception, and make union source accessor
|
||||
catch it, so we don't need to do this hack.
|
||||
*/
|
||||
{CanonPath(store->storeDir), store->getFSAccessor(settings.pureEval)},
|
||||
{CanonPath(store->storeDir), makeFSSourceAccessor(dirOf(store->toRealPath(StorePath::dummy)))}
|
||||
}))
|
||||
, rootFS(
|
||||
({
|
||||
@@ -284,12 +285,9 @@ EvalState::EvalState(
|
||||
/nix/store while using a chroot store. */
|
||||
auto accessor = getFSSourceAccessor();
|
||||
|
||||
auto realStoreDir = dirOf(store->toRealPath(StorePath::dummy));
|
||||
if (settings.pureEval || store->storeDir != realStoreDir) {
|
||||
accessor = settings.pureEval
|
||||
? storeFS
|
||||
: makeUnionSourceAccessor({accessor, storeFS});
|
||||
}
|
||||
accessor = settings.pureEval
|
||||
? storeFS.cast<SourceAccessor>()
|
||||
: makeUnionSourceAccessor({accessor, storeFS});
|
||||
|
||||
/* Apply access control if needed. */
|
||||
if (settings.restrictEval || settings.pureEval)
|
||||
@@ -372,6 +370,10 @@ EvalState::EvalState(
|
||||
);
|
||||
|
||||
createBaseEnv(settings);
|
||||
|
||||
/* Register function call tracer. */
|
||||
if (settings.traceFunctionCalls)
|
||||
profiler.addProfiler(make_ref<FunctionCallTrace>());
|
||||
}
|
||||
|
||||
|
||||
@@ -968,7 +970,16 @@ void EvalState::mkPos(Value & v, PosIdx p)
|
||||
auto origin = positions.originOf(p);
|
||||
if (auto path = std::get_if<SourcePath>(&origin)) {
|
||||
auto attrs = buildBindings(3);
|
||||
attrs.alloc(sFile).mkString(path->path.abs());
|
||||
if (path->accessor == rootFS && store->isInStore(path->path.abs()))
|
||||
// FIXME: only do this for virtual store paths?
|
||||
attrs.alloc(sFile).mkString(path->path.abs(),
|
||||
{
|
||||
NixStringContextElem::Path{
|
||||
.storePath = store->toStorePath(path->path.abs()).first
|
||||
}
|
||||
});
|
||||
else
|
||||
attrs.alloc(sFile).mkString(path->path.abs());
|
||||
makePositionThunks(*this, p, attrs.alloc(sLine), attrs.alloc(sColumn));
|
||||
v.mkAttrs(attrs);
|
||||
} else
|
||||
@@ -1526,9 +1537,14 @@ void EvalState::callFunction(Value & fun, std::span<Value *> args, Value & vRes,
|
||||
{
|
||||
auto _level = addCallDepth(pos);
|
||||
|
||||
auto trace = settings.traceFunctionCalls
|
||||
? std::make_unique<FunctionCallTrace>(positions[pos])
|
||||
: nullptr;
|
||||
auto neededHooks = profiler.getNeededHooks();
|
||||
if (neededHooks.test(EvalProfiler::preFunctionCall)) [[unlikely]]
|
||||
profiler.preFunctionCallHook(*this, fun, args, pos);
|
||||
|
||||
Finally traceExit_{[&](){
|
||||
if (profiler.getNeededHooks().test(EvalProfiler::postFunctionCall)) [[unlikely]]
|
||||
profiler.postFunctionCallHook(*this, fun, args, pos);
|
||||
}};
|
||||
|
||||
forceValue(fun, pos);
|
||||
|
||||
@@ -2089,7 +2105,7 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
|
||||
else if (firstType == nFloat)
|
||||
v.mkFloat(nf);
|
||||
else if (firstType == nPath) {
|
||||
if (!context.empty())
|
||||
if (hasContext(context))
|
||||
state.error<EvalError>("a string that refers to a store path cannot be appended to a path").atPos(pos).withFrame(env, *this).debugThrow();
|
||||
v.mkPath(state.rootPath(CanonPath(str())));
|
||||
} else
|
||||
@@ -2288,7 +2304,10 @@ std::string_view EvalState::forceStringNoCtx(Value & v, const PosIdx pos, std::s
|
||||
{
|
||||
auto s = forceString(v, pos, errorCtx);
|
||||
if (v.context()) {
|
||||
error<EvalError>("the string '%1%' is not allowed to refer to a store path (such as '%2%')", v.string_view(), v.context()[0]).withTrace(pos, errorCtx).debugThrow();
|
||||
NixStringContext context;
|
||||
copyContext(v, context);
|
||||
if (hasContext(context))
|
||||
error<EvalError>("the string '%1%' is not allowed to refer to a store path (such as '%2%')", v.string_view(), v.context()[0]).withTrace(pos, errorCtx).debugThrow();
|
||||
}
|
||||
return s;
|
||||
}
|
||||
@@ -2337,6 +2356,9 @@ BackedStringView EvalState::coerceToString(
|
||||
}
|
||||
|
||||
if (v.type() == nPath) {
|
||||
// FIXME: instead of copying the path to the store, we could
|
||||
// return a virtual store path that lazily copies the path to
|
||||
// the store in devirtualize().
|
||||
return
|
||||
!canonicalizePath && !copyToStore
|
||||
? // FIXME: hack to preserve path literals that end in a
|
||||
@@ -2344,7 +2366,16 @@ BackedStringView EvalState::coerceToString(
|
||||
v.payload.path.path
|
||||
: copyToStore
|
||||
? store->printStorePath(copyPathToStore(context, v.path()))
|
||||
: std::string(v.path().path.abs());
|
||||
: ({
|
||||
auto path = v.path();
|
||||
if (path.accessor == rootFS && store->isInStore(path.path.abs())) {
|
||||
context.insert(
|
||||
NixStringContextElem::Path{
|
||||
.storePath = store->toStorePath(path.path.abs()).first
|
||||
});
|
||||
}
|
||||
std::string(path.path.abs());
|
||||
});
|
||||
}
|
||||
|
||||
if (v.type() == nAttrs) {
|
||||
@@ -2423,10 +2454,11 @@ StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePat
|
||||
? *dstPathCached
|
||||
: [&]() {
|
||||
auto dstPath = fetchToStore(
|
||||
fetchSettings,
|
||||
*store,
|
||||
path.resolveSymlinks(SymlinkResolution::Ancestors),
|
||||
settings.readOnlyMode ? FetchMode::DryRun : FetchMode::Copy,
|
||||
path.baseName(),
|
||||
computeBaseName(path),
|
||||
ContentAddressMethod::Raw::NixArchive,
|
||||
nullptr,
|
||||
repair);
|
||||
@@ -2481,7 +2513,7 @@ StorePath EvalState::coerceToStorePath(const PosIdx pos, Value & v, NixStringCon
|
||||
auto path = coerceToString(pos, v, context, errorCtx, false, false, true).toOwned();
|
||||
if (auto storePath = store->maybeParseStorePath(path))
|
||||
return *storePath;
|
||||
error<EvalError>("path '%1%' is not in the Nix store", path).withTrace(pos, errorCtx).debugThrow();
|
||||
error<EvalError>("cannot coerce '%s' to a store path because it is not a subpath of the Nix store", path).withTrace(pos, errorCtx).debugThrow();
|
||||
}
|
||||
|
||||
|
||||
@@ -2507,6 +2539,11 @@ std::pair<SingleDerivedPath, std::string_view> EvalState::coerceToSingleDerivedP
|
||||
[&](NixStringContextElem::Built && b) -> SingleDerivedPath {
|
||||
return std::move(b);
|
||||
},
|
||||
[&](NixStringContextElem::Path && p) -> SingleDerivedPath {
|
||||
error<EvalError>(
|
||||
"string '%s' has no context",
|
||||
s).withTrace(pos, errorCtx).debugThrow();
|
||||
},
|
||||
}, ((NixStringContextElem &&) *context.begin()).raw);
|
||||
return {
|
||||
std::move(derivedPath),
|
||||
@@ -3090,6 +3127,11 @@ SourcePath EvalState::findFile(const LookupPath & lookupPath, const std::string_
|
||||
|
||||
auto res = (r / CanonPath(suffix)).resolveSymlinks();
|
||||
if (res.pathExists()) return res;
|
||||
|
||||
// Backward compatibility hack: throw an exception if access
|
||||
// to this path is not allowed.
|
||||
if (auto accessor = res.accessor.dynamic_pointer_cast<FilteringSourceAccessor>())
|
||||
accessor->checkAccess(res.path);
|
||||
}
|
||||
|
||||
if (hasPrefix(path, "nix/"))
|
||||
@@ -3125,7 +3167,7 @@ std::optional<SourcePath> EvalState::resolveLookupPathPath(const LookupPath::Pat
|
||||
store,
|
||||
fetchSettings,
|
||||
EvalSettings::resolvePseudoUrl(value));
|
||||
auto storePath = fetchToStore(*store, SourcePath(accessor), FetchMode::Copy);
|
||||
auto storePath = fetchToStore(fetchSettings, *store, SourcePath(accessor), FetchMode::Copy);
|
||||
return finish(this->storePath(storePath));
|
||||
} catch (Error & e) {
|
||||
logWarning({
|
||||
@@ -3160,6 +3202,11 @@ std::optional<SourcePath> EvalState::resolveLookupPathPath(const LookupPath::Pat
|
||||
if (path.resolveSymlinks().pathExists())
|
||||
return finish(std::move(path));
|
||||
else {
|
||||
// Backward compatibility hack: throw an exception if access
|
||||
// to this path is not allowed.
|
||||
if (auto accessor = path.accessor.dynamic_pointer_cast<FilteringSourceAccessor>())
|
||||
accessor->checkAccess(path.path);
|
||||
|
||||
logWarning({
|
||||
.msg = HintFmt("Nix search path entry '%1%' does not exist, ignoring", value)
|
||||
});
|
||||
|
||||
@@ -3,16 +3,20 @@
|
||||
|
||||
namespace nix {
|
||||
|
||||
FunctionCallTrace::FunctionCallTrace(const Pos & pos) : pos(pos) {
|
||||
void FunctionCallTrace::preFunctionCallHook(
|
||||
const EvalState & state, const Value & v, std::span<Value *> args, const PosIdx pos)
|
||||
{
|
||||
auto duration = std::chrono::high_resolution_clock::now().time_since_epoch();
|
||||
auto ns = std::chrono::duration_cast<std::chrono::nanoseconds>(duration);
|
||||
printMsg(lvlInfo, "function-trace entered %1% at %2%", pos, ns.count());
|
||||
printMsg(lvlInfo, "function-trace entered %1% at %2%", state.positions[pos], ns.count());
|
||||
}
|
||||
|
||||
FunctionCallTrace::~FunctionCallTrace() {
|
||||
void FunctionCallTrace::postFunctionCallHook(
|
||||
const EvalState & state, const Value & v, std::span<Value *> args, const PosIdx pos)
|
||||
{
|
||||
auto duration = std::chrono::high_resolution_clock::now().time_since_epoch();
|
||||
auto ns = std::chrono::duration_cast<std::chrono::nanoseconds>(duration);
|
||||
printMsg(lvlInfo, "function-trace exited %1% at %2%", pos, ns.count());
|
||||
printMsg(lvlInfo, "function-trace exited %1% at %2%", state.positions[pos], ns.count());
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
#pragma once
|
||||
/**
|
||||
* @file
|
||||
*
|
||||
* Evaluation profiler interface definitions and builtin implementations.
|
||||
*/
|
||||
|
||||
#include "nix/util/ref.hh"
|
||||
|
||||
#include <vector>
|
||||
#include <span>
|
||||
#include <bitset>
|
||||
#include <optional>
|
||||
|
||||
namespace nix {
|
||||
|
||||
class EvalState;
|
||||
class PosIdx;
|
||||
struct Value;
|
||||
|
||||
class EvalProfiler
|
||||
{
|
||||
public:
|
||||
enum Hook {
|
||||
preFunctionCall,
|
||||
postFunctionCall,
|
||||
};
|
||||
|
||||
static constexpr std::size_t numHooks = Hook::postFunctionCall + 1;
|
||||
using Hooks = std::bitset<numHooks>;
|
||||
|
||||
private:
|
||||
std::optional<Hooks> neededHooks;
|
||||
|
||||
protected:
|
||||
/** Invalidate the cached neededHooks. */
|
||||
void invalidateNeededHooks()
|
||||
{
|
||||
neededHooks = std::nullopt;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get which hooks need to be called.
|
||||
*
|
||||
* This is the actual implementation which has to be defined by subclasses.
|
||||
* Public API goes through the needsHooks, which is a
|
||||
* non-virtual interface (NVI) which caches the return value.
|
||||
*/
|
||||
virtual Hooks getNeededHooksImpl() const
|
||||
{
|
||||
return Hooks{};
|
||||
}
|
||||
|
||||
public:
|
||||
/**
|
||||
* Hook called in the EvalState::callFunction preamble.
|
||||
* Gets called only if (getNeededHooks().test(Hook::preFunctionCall)) is true.
|
||||
*
|
||||
* @param state Evaluator state.
|
||||
* @param v Function being invoked.
|
||||
* @param args Function arguments.
|
||||
* @param pos Function position.
|
||||
*/
|
||||
virtual void
|
||||
preFunctionCallHook(const EvalState & state, const Value & v, std::span<Value *> args, const PosIdx pos);
|
||||
|
||||
/**
|
||||
* Hook called on EvalState::callFunction exit.
|
||||
* Gets called only if (getNeededHooks().test(Hook::postFunctionCall)) is true.
|
||||
*
|
||||
* @param state Evaluator state.
|
||||
* @param v Function being invoked.
|
||||
* @param args Function arguments.
|
||||
* @param pos Function position.
|
||||
*/
|
||||
virtual void
|
||||
postFunctionCallHook(const EvalState & state, const Value & v, std::span<Value *> args, const PosIdx pos);
|
||||
|
||||
virtual ~EvalProfiler() = default;
|
||||
|
||||
/**
|
||||
* Get which hooks need to be invoked for this EvalProfiler instance.
|
||||
*/
|
||||
Hooks getNeededHooks()
|
||||
{
|
||||
if (neededHooks.has_value())
|
||||
return *neededHooks;
|
||||
return *(neededHooks = getNeededHooksImpl());
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Profiler that invokes multiple profilers at once.
|
||||
*/
|
||||
class MultiEvalProfiler : public EvalProfiler
|
||||
{
|
||||
std::vector<ref<EvalProfiler>> profilers;
|
||||
|
||||
[[gnu::noinline]] Hooks getNeededHooksImpl() const override;
|
||||
|
||||
public:
|
||||
MultiEvalProfiler() = default;
|
||||
|
||||
/** Register a profiler instance. */
|
||||
void addProfiler(ref<EvalProfiler> profiler);
|
||||
|
||||
[[gnu::noinline]] void
|
||||
preFunctionCallHook(const EvalState & state, const Value & v, std::span<Value *> args, const PosIdx pos) override;
|
||||
[[gnu::noinline]] void
|
||||
postFunctionCallHook(const EvalState & state, const Value & v, std::span<Value *> args, const PosIdx pos) override;
|
||||
};
|
||||
|
||||
}
|
||||
@@ -247,6 +247,11 @@ struct EvalSettings : Config
|
||||
|
||||
This option can be enabled by setting `NIX_ABORT_ON_WARN=1` in the environment.
|
||||
)"};
|
||||
|
||||
Setting<bool> lazyTrees{this, false, "lazy-trees",
|
||||
R"(
|
||||
If set to true, flakes and trees fetched by [`builtins.fetchTree`](@docroot@/language/builtins.md#builtins-fetchTree) are only copied to the Nix store when they're used as a dependency of a derivation. This avoids copying (potentially large) source trees unnecessarily.
|
||||
)"};
|
||||
};
|
||||
|
||||
/**
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
|
||||
#include "nix/expr/attr-set.hh"
|
||||
#include "nix/expr/eval-error.hh"
|
||||
#include "nix/expr/eval-profiler.hh"
|
||||
#include "nix/util/types.hh"
|
||||
#include "nix/expr/value.hh"
|
||||
#include "nix/expr/nixexpr.hh"
|
||||
@@ -36,6 +37,7 @@ class Store;
|
||||
namespace fetchers {
|
||||
struct Settings;
|
||||
struct InputCache;
|
||||
struct Input;
|
||||
}
|
||||
struct EvalSettings;
|
||||
class EvalState;
|
||||
@@ -43,6 +45,7 @@ class StorePath;
|
||||
struct SingleDerivedPath;
|
||||
enum RepairFlag : bool;
|
||||
struct MemorySourceAccessor;
|
||||
struct MountedSourceAccessor;
|
||||
namespace eval_cache {
|
||||
class EvalCache;
|
||||
}
|
||||
@@ -271,7 +274,7 @@ public:
|
||||
/**
|
||||
* The accessor corresponding to `store`.
|
||||
*/
|
||||
const ref<SourceAccessor> storeFS;
|
||||
const ref<MountedSourceAccessor> storeFS;
|
||||
|
||||
/**
|
||||
* The accessor for the root filesystem.
|
||||
@@ -449,6 +452,15 @@ public:
|
||||
|
||||
void checkURI(const std::string & uri);
|
||||
|
||||
/**
|
||||
* Mount an input on the Nix store.
|
||||
*/
|
||||
StorePath mountInput(
|
||||
fetchers::Input & input,
|
||||
const fetchers::Input & originalInput,
|
||||
ref<SourceAccessor> accessor,
|
||||
bool requireLockable);
|
||||
|
||||
/**
|
||||
* Parse a Nix expression from the specified file.
|
||||
*/
|
||||
@@ -558,6 +570,18 @@ public:
|
||||
std::optional<std::string> tryAttrsToString(const PosIdx pos, Value & v,
|
||||
NixStringContext & context, bool coerceMore = false, bool copyToStore = true);
|
||||
|
||||
StorePath devirtualize(
|
||||
const StorePath & path,
|
||||
StringMap * rewrites = nullptr);
|
||||
|
||||
SingleDerivedPath devirtualize(
|
||||
const SingleDerivedPath & path,
|
||||
StringMap * rewrites = nullptr);
|
||||
|
||||
std::string devirtualize(
|
||||
std::string_view s,
|
||||
const NixStringContext & context);
|
||||
|
||||
/**
|
||||
* String coercion.
|
||||
*
|
||||
@@ -573,6 +597,19 @@ public:
|
||||
|
||||
StorePath copyPathToStore(NixStringContext & context, const SourcePath & path);
|
||||
|
||||
|
||||
/**
|
||||
* Compute the base name for a `SourcePath`. For non-store paths,
|
||||
* this is just `SourcePath::baseName()`. But for store paths, for
|
||||
* backwards compatibility, it needs to be `<hash>-source`,
|
||||
* i.e. as if the path were copied to the Nix store. This results
|
||||
* in a "double-copied" store path like
|
||||
* `/nix/store/<hash1>-<hash2>-source`. We don't need to
|
||||
* materialize /nix/store/<hash2>-source though. Still, this
|
||||
* requires reading/hashing the path twice.
|
||||
*/
|
||||
std::string computeBaseName(const SourcePath & path);
|
||||
|
||||
/**
|
||||
* Path coercion.
|
||||
*
|
||||
@@ -903,6 +940,9 @@ private:
|
||||
typedef std::map<ExprLambda *, size_t> FunctionCalls;
|
||||
FunctionCalls functionCalls;
|
||||
|
||||
/** Evaluation/call profiler. */
|
||||
MultiEvalProfiler profiler;
|
||||
|
||||
void incrFunctionCall(ExprLambda * fun);
|
||||
|
||||
typedef std::map<PosIdx, size_t> AttrSelects;
|
||||
|
||||
@@ -2,15 +2,24 @@
|
||||
///@file
|
||||
|
||||
#include "nix/expr/eval.hh"
|
||||
|
||||
#include <chrono>
|
||||
#include "nix/expr/eval-profiler.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
struct FunctionCallTrace
|
||||
class FunctionCallTrace : public EvalProfiler
|
||||
{
|
||||
const Pos pos;
|
||||
FunctionCallTrace(const Pos & pos);
|
||||
~FunctionCallTrace();
|
||||
Hooks getNeededHooksImpl() const override
|
||||
{
|
||||
return Hooks().set(preFunctionCall).set(postFunctionCall);
|
||||
}
|
||||
|
||||
public:
|
||||
FunctionCallTrace() = default;
|
||||
|
||||
[[gnu::noinline]] void
|
||||
preFunctionCallHook(const EvalState & state, const Value & v, std::span<Value *> args, const PosIdx pos) override;
|
||||
[[gnu::noinline]] void
|
||||
postFunctionCallHook(const EvalState & state, const Value & v, std::span<Value *> args, const PosIdx pos) override;
|
||||
};
|
||||
|
||||
}
|
||||
@@ -14,6 +14,7 @@ headers = [config_pub_h] + files(
|
||||
'eval-error.hh',
|
||||
'eval-gc.hh',
|
||||
'eval-inline.hh',
|
||||
'eval-profiler.hh',
|
||||
'eval-settings.hh',
|
||||
'eval.hh',
|
||||
'function-trace.hh',
|
||||
|
||||
@@ -15,10 +15,10 @@ namespace nix {
|
||||
* See: https://github.com/NixOS/nix/issues/9730
|
||||
*/
|
||||
void printAmbiguous(
|
||||
Value &v,
|
||||
const SymbolTable &symbols,
|
||||
std::ostream &str,
|
||||
std::set<const void *> *seen,
|
||||
EvalState & state,
|
||||
Value & v,
|
||||
std::ostream & str,
|
||||
std::set<const void *> * seen,
|
||||
int depth);
|
||||
|
||||
}
|
||||
@@ -54,10 +54,35 @@ struct NixStringContextElem {
|
||||
*/
|
||||
using Built = SingleDerivedPath::Built;
|
||||
|
||||
/**
|
||||
* A store path that will not result in a store reference when
|
||||
* used in a derivation or toFile.
|
||||
*
|
||||
* When you apply `builtins.toString` to a path value representing
|
||||
* a path in the Nix store (as is the case with flake inputs),
|
||||
* historically you got a string without context
|
||||
* (e.g. `/nix/store/...-source`). This is broken, since it allows
|
||||
* you to pass a store path to a derivation/toFile without a
|
||||
* proper store reference. This is especially a problem with lazy
|
||||
* trees, since the store path is a virtual path that doesn't
|
||||
* exist.
|
||||
*
|
||||
* For backwards compatibility, and to warn users about this
|
||||
* unsafe use of `toString`, we keep track of such strings as a
|
||||
* special type of context.
|
||||
*/
|
||||
struct Path
|
||||
{
|
||||
StorePath storePath;
|
||||
|
||||
GENERATE_CMP(Path, me->storePath);
|
||||
};
|
||||
|
||||
using Raw = std::variant<
|
||||
Opaque,
|
||||
DrvDeep,
|
||||
Built
|
||||
Built,
|
||||
Path
|
||||
>;
|
||||
|
||||
Raw raw;
|
||||
@@ -82,4 +107,10 @@ struct NixStringContextElem {
|
||||
|
||||
typedef std::set<NixStringContextElem> NixStringContext;
|
||||
|
||||
/**
|
||||
* Returns false if `context` has no elements other than
|
||||
* `NixStringContextElem::Path`.
|
||||
*/
|
||||
bool hasContext(const NixStringContext & context);
|
||||
|
||||
}
|
||||
@@ -140,6 +140,7 @@ sources = files(
|
||||
'eval-cache.cc',
|
||||
'eval-error.cc',
|
||||
'eval-gc.cc',
|
||||
'eval-profiler.cc',
|
||||
'eval-settings.cc',
|
||||
'eval.cc',
|
||||
'function-trace.cc',
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
#include "nix/store/store-api.hh"
|
||||
#include "nix/expr/eval.hh"
|
||||
#include "nix/util/mounted-source-accessor.hh"
|
||||
#include "nix/fetchers/fetch-to-store.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
@@ -18,4 +20,87 @@ SourcePath EvalState::storePath(const StorePath & path)
|
||||
return {rootFS, CanonPath{store->printStorePath(path)}};
|
||||
}
|
||||
|
||||
StorePath EvalState::devirtualize(const StorePath & path, StringMap * rewrites)
|
||||
{
|
||||
if (auto mount = storeFS->getMount(CanonPath(store->printStorePath(path)))) {
|
||||
auto storePath = fetchToStore(
|
||||
fetchSettings,
|
||||
*store,
|
||||
SourcePath{ref(mount)},
|
||||
settings.readOnlyMode ? FetchMode::DryRun : FetchMode::Copy,
|
||||
path.name());
|
||||
assert(storePath.name() == path.name());
|
||||
if (rewrites)
|
||||
rewrites->emplace(path.hashPart(), storePath.hashPart());
|
||||
return storePath;
|
||||
} else
|
||||
return path;
|
||||
}
|
||||
|
||||
SingleDerivedPath EvalState::devirtualize(const SingleDerivedPath & path, StringMap * rewrites)
|
||||
{
|
||||
if (auto o = std::get_if<SingleDerivedPath::Opaque>(&path.raw()))
|
||||
return SingleDerivedPath::Opaque{devirtualize(o->path, rewrites)};
|
||||
else
|
||||
return path;
|
||||
}
|
||||
|
||||
std::string EvalState::devirtualize(std::string_view s, const NixStringContext & context)
|
||||
{
|
||||
StringMap rewrites;
|
||||
|
||||
for (auto & c : context)
|
||||
if (auto o = std::get_if<NixStringContextElem::Opaque>(&c.raw))
|
||||
devirtualize(o->path, &rewrites);
|
||||
|
||||
return rewriteStrings(std::string(s), rewrites);
|
||||
}
|
||||
|
||||
std::string EvalState::computeBaseName(const SourcePath & path)
|
||||
{
|
||||
if (path.accessor == rootFS) {
|
||||
if (auto storePath = store->maybeParseStorePath(path.path.abs())) {
|
||||
warn(
|
||||
"Performing inefficient double copy of path '%s' to the store. "
|
||||
"This can typically be avoided by rewriting an attribute like `src = ./.` "
|
||||
"to `src = builtins.path { path = ./.; name = \"source\"; }`.",
|
||||
path);
|
||||
return std::string(
|
||||
fetchToStore(fetchSettings, *store, path, FetchMode::DryRun, storePath->name()).to_string());
|
||||
}
|
||||
}
|
||||
return std::string(path.baseName());
|
||||
}
|
||||
|
||||
StorePath EvalState::mountInput(
|
||||
fetchers::Input & input, const fetchers::Input & originalInput, ref<SourceAccessor> accessor, bool requireLockable)
|
||||
{
|
||||
auto storePath = settings.lazyTrees
|
||||
? StorePath::random(input.getName())
|
||||
: fetchToStore(fetchSettings, *store, accessor, FetchMode::Copy, input.getName());
|
||||
|
||||
allowPath(storePath); // FIXME: should just whitelist the entire virtual store
|
||||
|
||||
storeFS->mount(CanonPath(store->printStorePath(storePath)), accessor);
|
||||
|
||||
if (requireLockable && (!settings.lazyTrees || !input.isLocked()) && !input.getNarHash()) {
|
||||
auto narHash = accessor->hashPath(CanonPath::root);
|
||||
input.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true));
|
||||
}
|
||||
|
||||
// FIXME: what to do with the NAR hash in lazy mode?
|
||||
if (!settings.lazyTrees && originalInput.getNarHash()) {
|
||||
auto expected = originalInput.computeStorePath(*store);
|
||||
if (storePath != expected)
|
||||
throw Error(
|
||||
(unsigned int) 102,
|
||||
"NAR hash mismatch in input '%s', expected '%s' but got '%s'",
|
||||
originalInput.to_string(),
|
||||
store->printStorePath(storePath),
|
||||
store->printStorePath(expected));
|
||||
}
|
||||
|
||||
return storePath;
|
||||
}
|
||||
|
||||
}
|
||||
+42
-3
@@ -14,6 +14,7 @@
|
||||
#include "nix/expr/value-to-xml.hh"
|
||||
#include "nix/expr/primops.hh"
|
||||
#include "nix/fetchers/fetch-to-store.hh"
|
||||
#include "nix/util/mounted-source-accessor.hh"
|
||||
|
||||
#include <boost/container/small_vector.hpp>
|
||||
#include <nlohmann/json.hpp>
|
||||
@@ -75,7 +76,10 @@ StringMap EvalState::realiseContext(const NixStringContext & context, StorePathS
|
||||
ensureValid(b.drvPath->getBaseStorePath());
|
||||
},
|
||||
[&](const NixStringContextElem::Opaque & o) {
|
||||
ensureValid(o.path);
|
||||
// We consider virtual store paths valid here. They'll
|
||||
// be devirtualized if needed elsewhere.
|
||||
if (!storeFS->getMount(CanonPath(store->printStorePath(o.path))))
|
||||
ensureValid(o.path);
|
||||
if (maybePathsOut)
|
||||
maybePathsOut->emplace(o.path);
|
||||
},
|
||||
@@ -85,6 +89,9 @@ StringMap EvalState::realiseContext(const NixStringContext & context, StorePathS
|
||||
if (maybePathsOut)
|
||||
maybePathsOut->emplace(d.drvPath);
|
||||
},
|
||||
[&](const NixStringContextElem::Path & p) {
|
||||
// FIXME: do something?
|
||||
},
|
||||
}, c.raw);
|
||||
}
|
||||
|
||||
@@ -1452,6 +1459,10 @@ static void derivationStrictInternal(
|
||||
/* Everything in the context of the strings in the derivation
|
||||
attributes should be added as dependencies of the resulting
|
||||
derivation. */
|
||||
StringMap rewrites;
|
||||
|
||||
std::optional<std::string> drvS;
|
||||
|
||||
for (auto & c : context) {
|
||||
std::visit(overloaded {
|
||||
/* Since this allows the builder to gain access to every
|
||||
@@ -1474,11 +1485,24 @@ static void derivationStrictInternal(
|
||||
drv.inputDrvs.ensureSlot(*b.drvPath).value.insert(b.output);
|
||||
},
|
||||
[&](const NixStringContextElem::Opaque & o) {
|
||||
drv.inputSrcs.insert(o.path);
|
||||
drv.inputSrcs.insert(state.devirtualize(o.path, &rewrites));
|
||||
},
|
||||
[&](const NixStringContextElem::Path & p) {
|
||||
if (!drvS) drvS = drv.unparse(*state.store, true);
|
||||
if (drvS->find(p.storePath.to_string()) != drvS->npos) {
|
||||
auto devirtualized = state.devirtualize(p.storePath, &rewrites);
|
||||
warn(
|
||||
"Using 'builtins.derivation' to create a derivation named '%s' that references the store path '%s' without a proper context. "
|
||||
"The resulting derivation will not have a correct store reference, so this is unreliable and may stop working in the future.",
|
||||
drvName,
|
||||
state.store->printStorePath(devirtualized));
|
||||
}
|
||||
},
|
||||
}, c.raw);
|
||||
}
|
||||
|
||||
drv.applyRewrites(rewrites);
|
||||
|
||||
/* Do we have all required attributes? */
|
||||
if (drv.builder == "")
|
||||
state.error<EvalError>("required attribute 'builder' missing")
|
||||
@@ -2382,10 +2406,21 @@ static void prim_toFile(EvalState & state, const PosIdx pos, Value * * args, Val
|
||||
std::string contents(state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.toFile"));
|
||||
|
||||
StorePathSet refs;
|
||||
StringMap rewrites;
|
||||
|
||||
for (auto c : context) {
|
||||
if (auto p = std::get_if<NixStringContextElem::Opaque>(&c.raw))
|
||||
refs.insert(p->path);
|
||||
else if (auto p = std::get_if<NixStringContextElem::Path>(&c.raw)) {
|
||||
if (contents.find(p->storePath.to_string()) != contents.npos) {
|
||||
auto devirtualized = state.devirtualize(p->storePath, &rewrites);
|
||||
warn(
|
||||
"Using 'builtins.toFile' to create a file named '%s' that references the store path '%s' without a proper context. "
|
||||
"The resulting file will not have a correct store reference, so this is unreliable and may stop working in the future.",
|
||||
name,
|
||||
state.store->printStorePath(devirtualized));
|
||||
}
|
||||
}
|
||||
else
|
||||
state.error<EvalError>(
|
||||
"files created by %1% may not reference derivations, but %2% references %3%",
|
||||
@@ -2395,6 +2430,8 @@ static void prim_toFile(EvalState & state, const PosIdx pos, Value * * args, Val
|
||||
).atPos(pos).debugThrow();
|
||||
}
|
||||
|
||||
contents = rewriteStrings(contents, rewrites);
|
||||
|
||||
auto storePath = settings.readOnlyMode
|
||||
? state.store->makeFixedOutputPathFromCA(name, TextInfo {
|
||||
.hash = hashString(HashAlgorithm::SHA256, contents),
|
||||
@@ -2544,7 +2581,9 @@ static void addPath(
|
||||
{}));
|
||||
|
||||
if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) {
|
||||
// FIXME: make this lazy?
|
||||
auto dstPath = fetchToStore(
|
||||
state.fetchSettings,
|
||||
*state.store,
|
||||
path.resolveSymlinks(),
|
||||
settings.readOnlyMode ? FetchMode::DryRun : FetchMode::Copy,
|
||||
@@ -2574,7 +2613,7 @@ static void prim_filterSource(EvalState & state, const PosIdx pos, Value * * arg
|
||||
"while evaluating the second argument (the path to filter) passed to 'builtins.filterSource'");
|
||||
state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.filterSource");
|
||||
|
||||
addPath(state, pos, path.baseName(), path, args[0], ContentAddressMethod::Raw::NixArchive, std::nullopt, v, context);
|
||||
addPath(state, pos, state.computeBaseName(path), path, args[0], ContentAddressMethod::Raw::NixArchive, std::nullopt, v, context);
|
||||
}
|
||||
|
||||
static RegisterPrimOp primop_filterSource({
|
||||
|
||||
@@ -7,9 +7,15 @@ namespace nix {
|
||||
|
||||
static void prim_unsafeDiscardStringContext(EvalState & state, const PosIdx pos, Value * * args, Value & v)
|
||||
{
|
||||
NixStringContext context;
|
||||
NixStringContext context, filtered;
|
||||
|
||||
auto s = state.coerceToString(pos, *args[0], context, "while evaluating the argument passed to builtins.unsafeDiscardStringContext");
|
||||
v.mkString(*s);
|
||||
|
||||
for (auto & c : context)
|
||||
if (auto * p = std::get_if<NixStringContextElem::Path>(&c.raw))
|
||||
filtered.insert(*p);
|
||||
|
||||
v.mkString(*s, filtered);
|
||||
}
|
||||
|
||||
static RegisterPrimOp primop_unsafeDiscardStringContext({
|
||||
@@ -21,12 +27,19 @@ static RegisterPrimOp primop_unsafeDiscardStringContext({
|
||||
.fun = prim_unsafeDiscardStringContext,
|
||||
});
|
||||
|
||||
bool hasContext(const NixStringContext & context)
|
||||
{
|
||||
for (auto & c : context)
|
||||
if (!std::get_if<NixStringContextElem::Path>(&c.raw))
|
||||
return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
static void prim_hasContext(EvalState & state, const PosIdx pos, Value * * args, Value & v)
|
||||
{
|
||||
NixStringContext context;
|
||||
state.forceString(*args[0], context, pos, "while evaluating the argument passed to builtins.hasContext");
|
||||
v.mkBool(!context.empty());
|
||||
v.mkBool(hasContext(context));
|
||||
}
|
||||
|
||||
static RegisterPrimOp primop_hasContext({
|
||||
@@ -103,7 +116,7 @@ static void prim_addDrvOutputDependencies(EvalState & state, const PosIdx pos, V
|
||||
NixStringContext context;
|
||||
auto s = state.coerceToString(pos, *args[0], context, "while evaluating the argument passed to builtins.addDrvOutputDependencies");
|
||||
|
||||
auto contextSize = context.size();
|
||||
auto contextSize = context.size();
|
||||
if (contextSize != 1) {
|
||||
state.error<EvalError>(
|
||||
"context of string '%s' must have exactly one element, but has %d",
|
||||
@@ -136,6 +149,11 @@ static void prim_addDrvOutputDependencies(EvalState & state, const PosIdx pos, V
|
||||
above does not make much sense. */
|
||||
return std::move(c);
|
||||
},
|
||||
[&](const NixStringContextElem::Path & p) -> NixStringContextElem::DrvDeep {
|
||||
state.error<EvalError>(
|
||||
"`addDrvOutputDependencies` does not work on a string without context"
|
||||
).atPos(pos).debugThrow();
|
||||
},
|
||||
}, context.begin()->raw) }),
|
||||
};
|
||||
|
||||
@@ -206,6 +224,8 @@ static void prim_getContext(EvalState & state, const PosIdx pos, Value * * args,
|
||||
[&](NixStringContextElem::Opaque && o) {
|
||||
contextInfos[std::move(o.path)].path = true;
|
||||
},
|
||||
[&](NixStringContextElem::Path && p) {
|
||||
},
|
||||
}, ((NixStringContextElem &&) i).raw);
|
||||
}
|
||||
|
||||
|
||||
@@ -64,7 +64,7 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value * * a
|
||||
if (rev) attrs.insert_or_assign("rev", rev->gitRev());
|
||||
auto input = fetchers::Input::fromAttrs(state.fetchSettings, std::move(attrs));
|
||||
|
||||
auto [storePath, input2] = input.fetchToStore(state.store);
|
||||
auto [storePath, accessor, input2] = input.fetchToStore(state.store);
|
||||
|
||||
auto attrs2 = state.buildBindings(8);
|
||||
state.mkStorePathString(storePath, attrs2.alloc(state.sOutPath));
|
||||
|
||||
@@ -10,6 +10,8 @@
|
||||
#include "nix/util/url.hh"
|
||||
#include "nix/expr/value-to-json.hh"
|
||||
#include "nix/fetchers/fetch-to-store.hh"
|
||||
#include "nix/fetchers/input-cache.hh"
|
||||
#include "nix/util/mounted-source-accessor.hh"
|
||||
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
@@ -204,11 +206,11 @@ static void fetchTree(
|
||||
throw Error("input '%s' is not allowed to use the '__final' attribute", input.to_string());
|
||||
}
|
||||
|
||||
auto [storePath, input2] = input.fetchToStore(state.store);
|
||||
auto cachedInput = state.inputCache->getAccessor(state.store, input, fetchers::UseRegistries::No);
|
||||
|
||||
state.allowPath(storePath);
|
||||
auto storePath = state.mountInput(cachedInput.lockedInput, input, cachedInput.accessor, true);
|
||||
|
||||
emitTreeAttrs(state, storePath, input2, v, params.emptyRevFallback, false);
|
||||
emitTreeAttrs(state, storePath, cachedInput.lockedInput, v, params.emptyRevFallback, false);
|
||||
}
|
||||
|
||||
static void prim_fetchTree(EvalState & state, const PosIdx pos, Value * * args, Value & v)
|
||||
@@ -537,11 +539,12 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v
|
||||
auto storePath =
|
||||
unpack
|
||||
? fetchToStore(
|
||||
state.fetchSettings,
|
||||
*state.store,
|
||||
fetchers::downloadTarball(state.store, state.fetchSettings, *url),
|
||||
FetchMode::Copy,
|
||||
name)
|
||||
: fetchers::downloadFile(state.store, *url, name).storePath;
|
||||
: fetchers::downloadFile(state.store, state.fetchSettings, *url, name).storePath;
|
||||
|
||||
if (expectedHash) {
|
||||
auto hash = unpack
|
||||
|
||||
@@ -7,10 +7,10 @@ namespace nix {
|
||||
|
||||
// See: https://github.com/NixOS/nix/issues/9730
|
||||
void printAmbiguous(
|
||||
Value &v,
|
||||
const SymbolTable &symbols,
|
||||
std::ostream &str,
|
||||
std::set<const void *> *seen,
|
||||
EvalState & state,
|
||||
Value & v,
|
||||
std::ostream & str,
|
||||
std::set<const void *> * seen,
|
||||
int depth)
|
||||
{
|
||||
checkInterrupt();
|
||||
@@ -26,9 +26,13 @@ void printAmbiguous(
|
||||
case nBool:
|
||||
printLiteralBool(str, v.boolean());
|
||||
break;
|
||||
case nString:
|
||||
printLiteralString(str, v.string_view());
|
||||
case nString: {
|
||||
NixStringContext context;
|
||||
copyContext(v, context);
|
||||
// FIXME: make devirtualization configurable?
|
||||
printLiteralString(str, state.devirtualize(v.string_view(), context));
|
||||
break;
|
||||
}
|
||||
case nPath:
|
||||
str << v.path().to_string(); // !!! escaping?
|
||||
break;
|
||||
@@ -40,9 +44,9 @@ void printAmbiguous(
|
||||
str << "«repeated»";
|
||||
else {
|
||||
str << "{ ";
|
||||
for (auto & i : v.attrs()->lexicographicOrder(symbols)) {
|
||||
str << symbols[i->name] << " = ";
|
||||
printAmbiguous(*i->value, symbols, str, seen, depth - 1);
|
||||
for (auto & i : v.attrs()->lexicographicOrder(state.symbols)) {
|
||||
str << state.symbols[i->name] << " = ";
|
||||
printAmbiguous(state, *i->value, str, seen, depth - 1);
|
||||
str << "; ";
|
||||
}
|
||||
str << "}";
|
||||
@@ -56,7 +60,7 @@ void printAmbiguous(
|
||||
str << "[ ";
|
||||
for (auto v2 : v.listItems()) {
|
||||
if (v2)
|
||||
printAmbiguous(*v2, symbols, str, seen, depth - 1);
|
||||
printAmbiguous(state, *v2, str, seen, depth - 1);
|
||||
else
|
||||
str << "(nullptr)";
|
||||
str << " ";
|
||||
|
||||
@@ -249,7 +249,11 @@ private:
|
||||
|
||||
void printString(Value & v)
|
||||
{
|
||||
printLiteralString(output, v.string_view(), options.maxStringLength, options.ansiColors);
|
||||
NixStringContext context;
|
||||
copyContext(v, context);
|
||||
std::ostringstream s;
|
||||
printLiteralString(s, v.string_view(), options.maxStringLength, options.ansiColors);
|
||||
output << state.devirtualize(s.str(), context);
|
||||
}
|
||||
|
||||
void printPath(Value & v)
|
||||
|
||||
@@ -7,9 +7,10 @@
|
||||
#include <iomanip>
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
|
||||
namespace nix {
|
||||
|
||||
using json = nlohmann::json;
|
||||
|
||||
// TODO: rename. It doesn't print.
|
||||
json printValueAsJSON(EvalState & state, bool strict,
|
||||
Value & v, const PosIdx pos, NixStringContext & context, bool copyToStore)
|
||||
|
||||
@@ -57,6 +57,11 @@ NixStringContextElem NixStringContextElem::parse(
|
||||
.drvPath = StorePath { s.substr(1) },
|
||||
};
|
||||
}
|
||||
case '@': {
|
||||
return NixStringContextElem::Path {
|
||||
.storePath = StorePath { s.substr(1) },
|
||||
};
|
||||
}
|
||||
default: {
|
||||
// Ensure no '!'
|
||||
if (s.find("!") != std::string_view::npos) {
|
||||
@@ -100,6 +105,10 @@ std::string NixStringContextElem::to_string() const
|
||||
res += '=';
|
||||
res += d.drvPath.to_string();
|
||||
},
|
||||
[&](const NixStringContextElem::Path & p) {
|
||||
res += '@';
|
||||
res += p.storePath.to_string();
|
||||
},
|
||||
}, raw);
|
||||
|
||||
return res;
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
#include "nix/fetchers/cache.hh"
|
||||
#include "nix/fetchers/fetch-settings.hh"
|
||||
#include "nix/util/users.hh"
|
||||
#include "nix/store/sqlite.hh"
|
||||
#include "nix/util/sync.hh"
|
||||
@@ -162,10 +163,12 @@ struct CacheImpl : Cache
|
||||
}
|
||||
};
|
||||
|
||||
ref<Cache> getCache()
|
||||
ref<Cache> Settings::getCache() const
|
||||
{
|
||||
static auto cache = std::make_shared<CacheImpl>();
|
||||
return ref<Cache>(cache);
|
||||
auto cache(_cache.lock());
|
||||
if (!*cache)
|
||||
*cache = std::make_shared<CacheImpl>();
|
||||
return ref<Cache>(*cache);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,13 +1,14 @@
|
||||
#include "nix/fetchers/fetch-to-store.hh"
|
||||
#include "nix/fetchers/fetchers.hh"
|
||||
#include "nix/fetchers/fetch-settings.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
fetchers::Cache::Key makeFetchToStoreCacheKey(
|
||||
const std::string &name,
|
||||
const std::string &fingerprint,
|
||||
const std::string & name,
|
||||
const std::string & fingerprint,
|
||||
ContentAddressMethod method,
|
||||
const std::string &path)
|
||||
const std::string & path)
|
||||
{
|
||||
return fetchers::Cache::Key{"fetchToStore", {
|
||||
{"name", name},
|
||||
@@ -19,6 +20,7 @@ fetchers::Cache::Key makeFetchToStoreCacheKey(
|
||||
}
|
||||
|
||||
StorePath fetchToStore(
|
||||
const fetchers::Settings & settings,
|
||||
Store & store,
|
||||
const SourcePath & path,
|
||||
FetchMode mode,
|
||||
@@ -34,7 +36,7 @@ StorePath fetchToStore(
|
||||
|
||||
if (!filter && path.accessor->fingerprint) {
|
||||
cacheKey = makeFetchToStoreCacheKey(std::string{name}, *path.accessor->fingerprint, method, path.path.abs());
|
||||
if (auto res = fetchers::getCache()->lookupStorePath(*cacheKey, store)) {
|
||||
if (auto res = settings.getCache()->lookupStorePath(*cacheKey, store)) {
|
||||
debug("store path cache hit for '%s'", path);
|
||||
return res->storePath;
|
||||
}
|
||||
@@ -56,7 +58,7 @@ StorePath fetchToStore(
|
||||
debug(mode == FetchMode::DryRun ? "hashed '%s'" : "copied '%s' to '%s'", path, store.printStorePath(storePath));
|
||||
|
||||
if (cacheKey && mode == FetchMode::Copy)
|
||||
fetchers::getCache()->upsert(*cacheKey, store, {}, storePath);
|
||||
settings.getCache()->upsert(*cacheKey, store, {}, storePath);
|
||||
|
||||
return storePath;
|
||||
}
|
||||
|
||||
+17
-18
@@ -189,34 +189,30 @@ bool Input::contains(const Input & other) const
|
||||
}
|
||||
|
||||
// FIXME: remove
|
||||
std::pair<StorePath, Input> Input::fetchToStore(ref<Store> store) const
|
||||
std::tuple<StorePath, ref<SourceAccessor>, Input> Input::fetchToStore(ref<Store> store) const
|
||||
{
|
||||
if (!scheme)
|
||||
throw Error("cannot fetch unsupported input '%s'", attrsToJSON(toAttrs()));
|
||||
|
||||
auto [storePath, input] = [&]() -> std::pair<StorePath, Input> {
|
||||
try {
|
||||
auto [accessor, result] = getAccessorUnchecked(store);
|
||||
try {
|
||||
auto [accessor, result] = getAccessorUnchecked(store);
|
||||
|
||||
auto storePath = nix::fetchToStore(*store, SourcePath(accessor), FetchMode::Copy, result.getName());
|
||||
auto storePath = nix::fetchToStore(*settings, *store, SourcePath(accessor), FetchMode::Copy, result.getName());
|
||||
|
||||
auto narHash = store->queryPathInfo(storePath)->narHash;
|
||||
result.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true));
|
||||
auto narHash = store->queryPathInfo(storePath)->narHash;
|
||||
result.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true));
|
||||
|
||||
result.attrs.insert_or_assign("__final", Explicit<bool>(true));
|
||||
result.attrs.insert_or_assign("__final", Explicit<bool>(true));
|
||||
|
||||
assert(result.isFinal());
|
||||
assert(result.isFinal());
|
||||
|
||||
checkLocks(*this, result);
|
||||
checkLocks(*this, result);
|
||||
|
||||
return {storePath, result};
|
||||
} catch (Error & e) {
|
||||
e.addTrace({}, "while fetching the input '%s'", to_string());
|
||||
throw;
|
||||
}
|
||||
}();
|
||||
|
||||
return {std::move(storePath), input};
|
||||
return {std::move(storePath), accessor, result};
|
||||
} catch (Error & e) {
|
||||
e.addTrace({}, "while fetching the input '%s'", to_string());
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
void Input::checkLocks(Input specified, Input & result)
|
||||
@@ -234,6 +230,9 @@ void Input::checkLocks(Input specified, Input & result)
|
||||
if (auto prevNarHash = specified.getNarHash())
|
||||
specified.attrs.insert_or_assign("narHash", prevNarHash->to_string(HashFormat::SRI, true));
|
||||
|
||||
if (auto narHash = result.getNarHash())
|
||||
result.attrs.insert_or_assign("narHash", narHash->to_string(HashFormat::SRI, true));
|
||||
|
||||
for (auto & field : specified.attrs) {
|
||||
auto field2 = result.attrs.find(field.first);
|
||||
if (field2 != result.attrs.end() && field.second != field2->second)
|
||||
|
||||
@@ -20,9 +20,14 @@ bool FilteringSourceAccessor::pathExists(const CanonPath & path)
|
||||
}
|
||||
|
||||
std::optional<SourceAccessor::Stat> FilteringSourceAccessor::maybeLstat(const CanonPath & path)
|
||||
{
|
||||
return isAllowed(path) ? next->maybeLstat(prefix / path) : std::nullopt;
|
||||
}
|
||||
|
||||
SourceAccessor::Stat FilteringSourceAccessor::lstat(const CanonPath & path)
|
||||
{
|
||||
checkAccess(path);
|
||||
return next->maybeLstat(prefix / path);
|
||||
return next->lstat(prefix / path);
|
||||
}
|
||||
|
||||
SourceAccessor::DirEntries FilteringSourceAccessor::readDirectory(const CanonPath & path)
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
#include "nix/fetchers/git-utils.hh"
|
||||
#include "nix/fetchers/git-lfs-fetch.hh"
|
||||
#include "nix/fetchers/cache.hh"
|
||||
#include "nix/fetchers/fetch-settings.hh"
|
||||
#include "nix/util/finally.hh"
|
||||
#include "nix/util/processes.hh"
|
||||
#include "nix/util/signals.hh"
|
||||
@@ -610,18 +611,18 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
|
||||
throw Error("Commit signature verification on commit %s failed: %s", rev.gitRev(), output);
|
||||
}
|
||||
|
||||
Hash treeHashToNarHash(const Hash & treeHash) override
|
||||
Hash treeHashToNarHash(const fetchers::Settings & settings, const Hash & treeHash) override
|
||||
{
|
||||
auto accessor = getAccessor(treeHash, false, "");
|
||||
|
||||
fetchers::Cache::Key cacheKey{"treeHashToNarHash", {{"treeHash", treeHash.gitRev()}}};
|
||||
|
||||
if (auto res = fetchers::getCache()->lookup(cacheKey))
|
||||
if (auto res = settings.getCache()->lookup(cacheKey))
|
||||
return Hash::parseAny(fetchers::getStrAttr(*res, "narHash"), HashAlgorithm::SHA256);
|
||||
|
||||
auto narHash = accessor->hashPath(CanonPath::root);
|
||||
|
||||
fetchers::getCache()->upsert(cacheKey, fetchers::Attrs({{"narHash", narHash.to_string(HashFormat::SRI, true)}}));
|
||||
settings.getCache()->upsert(cacheKey, fetchers::Attrs({{"narHash", narHash.to_string(HashFormat::SRI, true)}}));
|
||||
|
||||
return narHash;
|
||||
}
|
||||
|
||||
@@ -15,6 +15,7 @@
|
||||
#include "nix/fetchers/fetch-settings.hh"
|
||||
#include "nix/util/json-utils.hh"
|
||||
#include "nix/util/archive.hh"
|
||||
#include "nix/util/mounted-source-accessor.hh"
|
||||
|
||||
#include <regex>
|
||||
#include <string.h>
|
||||
@@ -480,11 +481,11 @@ struct GitInputScheme : InputScheme
|
||||
return repoInfo;
|
||||
}
|
||||
|
||||
uint64_t getLastModified(const RepoInfo & repoInfo, const std::filesystem::path & repoDir, const Hash & rev) const
|
||||
uint64_t getLastModified(const Settings & settings, const RepoInfo & repoInfo, const std::filesystem::path & repoDir, const Hash & rev) const
|
||||
{
|
||||
Cache::Key key{"gitLastModified", {{"rev", rev.gitRev()}}};
|
||||
|
||||
auto cache = getCache();
|
||||
auto cache = settings.getCache();
|
||||
|
||||
if (auto res = cache->lookup(key))
|
||||
return getIntAttr(*res, "lastModified");
|
||||
@@ -496,11 +497,11 @@ struct GitInputScheme : InputScheme
|
||||
return lastModified;
|
||||
}
|
||||
|
||||
uint64_t getRevCount(const RepoInfo & repoInfo, const std::filesystem::path & repoDir, const Hash & rev) const
|
||||
uint64_t getRevCount(const Settings & settings, const RepoInfo & repoInfo, const std::filesystem::path & repoDir, const Hash & rev) const
|
||||
{
|
||||
Cache::Key key{"gitRevCount", {{"rev", rev.gitRev()}}};
|
||||
|
||||
auto cache = getCache();
|
||||
auto cache = settings.getCache();
|
||||
|
||||
if (auto revCountAttrs = cache->lookup(key))
|
||||
return getIntAttr(*revCountAttrs, "revCount");
|
||||
@@ -678,12 +679,12 @@ struct GitInputScheme : InputScheme
|
||||
|
||||
Attrs infoAttrs({
|
||||
{"rev", rev.gitRev()},
|
||||
{"lastModified", getLastModified(repoInfo, repoDir, rev)},
|
||||
{"lastModified", getLastModified(*input.settings, repoInfo, repoDir, rev)},
|
||||
});
|
||||
|
||||
if (!getShallowAttr(input))
|
||||
infoAttrs.insert_or_assign("revCount",
|
||||
getRevCount(repoInfo, repoDir, rev));
|
||||
getRevCount(*input.settings, repoInfo, repoDir, rev));
|
||||
|
||||
printTalkative("using revision %s of repo '%s'", rev.gitRev(), repoInfo.locationToArg());
|
||||
|
||||
@@ -799,7 +800,7 @@ struct GitInputScheme : InputScheme
|
||||
|
||||
input.attrs.insert_or_assign("rev", rev.gitRev());
|
||||
input.attrs.insert_or_assign("revCount",
|
||||
rev == nullRev ? 0 : getRevCount(repoInfo, repoPath, rev));
|
||||
rev == nullRev ? 0 : getRevCount(*input.settings, repoInfo, repoPath, rev));
|
||||
|
||||
verifyCommit(input, repo);
|
||||
} else {
|
||||
@@ -818,7 +819,7 @@ struct GitInputScheme : InputScheme
|
||||
input.attrs.insert_or_assign(
|
||||
"lastModified",
|
||||
repoInfo.workdirInfo.headRev
|
||||
? getLastModified(repoInfo, repoPath, *repoInfo.workdirInfo.headRev)
|
||||
? getLastModified(*input.settings, repoInfo, repoPath, *repoInfo.workdirInfo.headRev)
|
||||
: 0);
|
||||
|
||||
return {accessor, std::move(input)};
|
||||
|
||||
@@ -265,7 +265,7 @@ struct GitArchiveInputScheme : InputScheme
|
||||
input.attrs.erase("ref");
|
||||
input.attrs.insert_or_assign("rev", rev->gitRev());
|
||||
|
||||
auto cache = getCache();
|
||||
auto cache = input.settings->getCache();
|
||||
|
||||
Cache::Key treeHashKey{"gitRevToTreeHash", {{"rev", rev->gitRev()}}};
|
||||
Cache::Key lastModifiedKey{"gitRevToLastModified", {{"rev", rev->gitRev()}}};
|
||||
@@ -335,6 +335,13 @@ struct GitArchiveInputScheme : InputScheme
|
||||
false,
|
||||
"«" + input.to_string() + "»");
|
||||
|
||||
if (!input.settings->trustTarballsFromGitForges)
|
||||
// FIXME: computing the NAR hash here is wasteful if
|
||||
// copyInputToStore() is just going to hash/copy it as
|
||||
// well.
|
||||
input.attrs.insert_or_assign("narHash",
|
||||
accessor->hashPath(CanonPath::root).to_string(HashFormat::SRI, true));
|
||||
|
||||
return {accessor, input};
|
||||
}
|
||||
|
||||
@@ -407,7 +414,7 @@ struct GitHubInputScheme : GitArchiveInputScheme
|
||||
auto json = nlohmann::json::parse(
|
||||
readFile(
|
||||
store->toRealPath(
|
||||
downloadFile(store, url, "source", headers).storePath)));
|
||||
downloadFile(store, *input.settings, url, "source", headers).storePath)));
|
||||
|
||||
return RefInfo {
|
||||
.rev = Hash::parseAny(std::string { json["sha"] }, HashAlgorithm::SHA1),
|
||||
@@ -481,7 +488,7 @@ struct GitLabInputScheme : GitArchiveInputScheme
|
||||
auto json = nlohmann::json::parse(
|
||||
readFile(
|
||||
store->toRealPath(
|
||||
downloadFile(store, url, "source", headers).storePath)));
|
||||
downloadFile(store, *input.settings, url, "source", headers).storePath)));
|
||||
|
||||
if (json.is_array() && json.size() >= 1 && json[0]["id"] != nullptr) {
|
||||
return RefInfo {
|
||||
@@ -551,7 +558,7 @@ struct SourceHutInputScheme : GitArchiveInputScheme
|
||||
std::string refUri;
|
||||
if (ref == "HEAD") {
|
||||
auto file = store->toRealPath(
|
||||
downloadFile(store, fmt("%s/HEAD", base_url), "source", headers).storePath);
|
||||
downloadFile(store, *input.settings, fmt("%s/HEAD", base_url), "source", headers).storePath);
|
||||
std::ifstream is(file);
|
||||
std::string line;
|
||||
getline(is, line);
|
||||
@@ -567,7 +574,7 @@ struct SourceHutInputScheme : GitArchiveInputScheme
|
||||
std::regex refRegex(refUri);
|
||||
|
||||
auto file = store->toRealPath(
|
||||
downloadFile(store, fmt("%s/info/refs", base_url), "source", headers).storePath);
|
||||
downloadFile(store, *input.settings, fmt("%s/info/refs", base_url), "source", headers).storePath);
|
||||
std::ifstream is(file);
|
||||
|
||||
std::string line;
|
||||
|
||||
@@ -91,6 +91,4 @@ struct Cache
|
||||
Store & store) = 0;
|
||||
};
|
||||
|
||||
ref<Cache> getCache();
|
||||
|
||||
}
|
||||
@@ -3,6 +3,8 @@
|
||||
|
||||
#include "nix/util/types.hh"
|
||||
#include "nix/util/configuration.hh"
|
||||
#include "nix/util/ref.hh"
|
||||
#include "nix/util/sync.hh"
|
||||
|
||||
#include <map>
|
||||
#include <limits>
|
||||
@@ -11,6 +13,8 @@
|
||||
|
||||
namespace nix::fetchers {
|
||||
|
||||
struct Cache;
|
||||
|
||||
struct Settings : public Config
|
||||
{
|
||||
Settings();
|
||||
@@ -110,6 +114,11 @@ struct Settings : public Config
|
||||
When empty, disables the global flake registry.
|
||||
)",
|
||||
{}, true, Xp::Flakes};
|
||||
|
||||
ref<Cache> getCache() const;
|
||||
|
||||
private:
|
||||
mutable Sync<std::shared_ptr<Cache>> _cache;
|
||||
};
|
||||
|
||||
}
|
||||
@@ -15,6 +15,7 @@ enum struct FetchMode { DryRun, Copy };
|
||||
* Copy the `path` to the Nix store.
|
||||
*/
|
||||
StorePath fetchToStore(
|
||||
const fetchers::Settings & settings,
|
||||
Store & store,
|
||||
const SourcePath & path,
|
||||
FetchMode mode,
|
||||
|
||||
@@ -121,7 +121,7 @@ public:
|
||||
* Fetch the entire input into the Nix store, returning the
|
||||
* location in the Nix store and the locked input.
|
||||
*/
|
||||
std::pair<StorePath, Input> fetchToStore(ref<Store> store) const;
|
||||
std::tuple<StorePath, ref<SourceAccessor>, Input> fetchToStore(ref<Store> store) const;
|
||||
|
||||
/**
|
||||
* Check the locking attributes in `result` against
|
||||
|
||||
@@ -38,6 +38,8 @@ struct FilteringSourceAccessor : SourceAccessor
|
||||
|
||||
bool pathExists(const CanonPath & path) override;
|
||||
|
||||
Stat lstat(const CanonPath & path) override;
|
||||
|
||||
std::optional<Stat> maybeLstat(const CanonPath & path) override;
|
||||
|
||||
DirEntries readDirectory(const CanonPath & path) override;
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
|
||||
namespace nix {
|
||||
|
||||
namespace fetchers { struct PublicKey; }
|
||||
namespace fetchers { struct PublicKey; struct Settings; }
|
||||
|
||||
/**
|
||||
* A sink that writes into a Git repository. Note that nothing may be written
|
||||
@@ -115,7 +115,7 @@ struct GitRepo
|
||||
* Given a Git tree hash, compute the hash of its NAR
|
||||
* serialisation. This is memoised on-disk.
|
||||
*/
|
||||
virtual Hash treeHashToNarHash(const Hash & treeHash) = 0;
|
||||
virtual Hash treeHashToNarHash(const fetchers::Settings & settings, const Hash & treeHash) = 0;
|
||||
|
||||
/**
|
||||
* If the specified Git object is a directory with a single entry
|
||||
|
||||
@@ -26,6 +26,7 @@ struct DownloadFileResult
|
||||
|
||||
DownloadFileResult downloadFile(
|
||||
ref<Store> store,
|
||||
const Settings & settings,
|
||||
const std::string & url,
|
||||
const std::string & name,
|
||||
const Headers & headers = {});
|
||||
|
||||
@@ -253,13 +253,13 @@ struct MercurialInputScheme : InputScheme
|
||||
}};
|
||||
|
||||
if (!input.getRev()) {
|
||||
if (auto res = getCache()->lookupWithTTL(refToRevKey))
|
||||
if (auto res = input.settings->getCache()->lookupWithTTL(refToRevKey))
|
||||
input.attrs.insert_or_assign("rev", getRevAttr(*res, "rev").gitRev());
|
||||
}
|
||||
|
||||
/* If we have a rev, check if we have a cached store path. */
|
||||
if (auto rev = input.getRev()) {
|
||||
if (auto res = getCache()->lookupStorePath(revInfoKey(*rev), *store))
|
||||
if (auto res = input.settings->getCache()->lookupStorePath(revInfoKey(*rev), *store))
|
||||
return makeResult(res->value, res->storePath);
|
||||
}
|
||||
|
||||
@@ -309,7 +309,7 @@ struct MercurialInputScheme : InputScheme
|
||||
|
||||
/* Now that we have the rev, check the cache again for a
|
||||
cached store path. */
|
||||
if (auto res = getCache()->lookupStorePath(revInfoKey(rev), *store))
|
||||
if (auto res = input.settings->getCache()->lookupStorePath(revInfoKey(rev), *store))
|
||||
return makeResult(res->value, res->storePath);
|
||||
|
||||
Path tmpDir = createTempDir();
|
||||
@@ -326,9 +326,9 @@ struct MercurialInputScheme : InputScheme
|
||||
});
|
||||
|
||||
if (!origRev)
|
||||
getCache()->upsert(refToRevKey, {{"rev", rev.gitRev()}});
|
||||
input.settings->getCache()->upsert(refToRevKey, {{"rev", rev.gitRev()}});
|
||||
|
||||
getCache()->upsert(revInfoKey(rev), *store, infoAttrs, storePath);
|
||||
input.settings->getCache()->upsert(revInfoKey(rev), *store, infoAttrs, storePath);
|
||||
|
||||
return makeResult(infoAttrs, std::move(storePath));
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
#include "nix/fetchers/store-path-accessor.hh"
|
||||
#include "nix/fetchers/cache.hh"
|
||||
#include "nix/fetchers/fetch-to-store.hh"
|
||||
#include "nix/fetchers/fetch-settings.hh"
|
||||
|
||||
namespace nix::fetchers {
|
||||
|
||||
@@ -149,7 +150,7 @@ struct PathInputScheme : InputScheme
|
||||
auto fp = getFingerprint(store, input);
|
||||
if (fp) {
|
||||
auto cacheKey = makeFetchToStoreCacheKey(input.getName(), *fp, method, "/");
|
||||
fetchers::getCache()->upsert(cacheKey, *store, {}, *storePath);
|
||||
input.settings->getCache()->upsert(cacheKey, *store, {}, *storePath);
|
||||
}
|
||||
|
||||
/* Trust the lastModified value supplied by the user, if
|
||||
|
||||
@@ -156,7 +156,7 @@ static std::shared_ptr<Registry> getGlobalRegistry(const Settings & settings, re
|
||||
}
|
||||
|
||||
if (!isAbsolute(path)) {
|
||||
auto storePath = downloadFile(store, path, "flake-registry.json").storePath;
|
||||
auto storePath = downloadFile(store, settings, path, "flake-registry.json").storePath;
|
||||
if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>())
|
||||
store2->addPermRoot(storePath, getCacheDir() + "/flake-registry.json");
|
||||
path = store->toRealPath(storePath);
|
||||
|
||||
@@ -9,11 +9,13 @@
|
||||
#include "nix/fetchers/store-path-accessor.hh"
|
||||
#include "nix/store/store-api.hh"
|
||||
#include "nix/fetchers/git-utils.hh"
|
||||
#include "nix/fetchers/fetch-settings.hh"
|
||||
|
||||
namespace nix::fetchers {
|
||||
|
||||
DownloadFileResult downloadFile(
|
||||
ref<Store> store,
|
||||
const Settings & settings,
|
||||
const std::string & url,
|
||||
const std::string & name,
|
||||
const Headers & headers)
|
||||
@@ -25,7 +27,7 @@ DownloadFileResult downloadFile(
|
||||
{"name", name},
|
||||
}}};
|
||||
|
||||
auto cached = getCache()->lookupStorePath(key, *store);
|
||||
auto cached = settings.getCache()->lookupStorePath(key, *store);
|
||||
|
||||
auto useCached = [&]() -> DownloadFileResult
|
||||
{
|
||||
@@ -92,7 +94,7 @@ DownloadFileResult downloadFile(
|
||||
key.second.insert_or_assign("url", url);
|
||||
assert(!res.urls.empty());
|
||||
infoAttrs.insert_or_assign("url", *res.urls.rbegin());
|
||||
getCache()->upsert(key, *store, infoAttrs, *storePath);
|
||||
settings.getCache()->upsert(key, *store, infoAttrs, *storePath);
|
||||
}
|
||||
|
||||
return {
|
||||
@@ -104,13 +106,14 @@ DownloadFileResult downloadFile(
|
||||
}
|
||||
|
||||
static DownloadTarballResult downloadTarball_(
|
||||
const Settings & settings,
|
||||
const std::string & url,
|
||||
const Headers & headers,
|
||||
const std::string & displayPrefix)
|
||||
{
|
||||
Cache::Key cacheKey{"tarball", {{"url", url}}};
|
||||
|
||||
auto cached = getCache()->lookupExpired(cacheKey);
|
||||
auto cached = settings.getCache()->lookupExpired(cacheKey);
|
||||
|
||||
auto attrsToResult = [&](const Attrs & infoAttrs)
|
||||
{
|
||||
@@ -196,7 +199,7 @@ static DownloadTarballResult downloadTarball_(
|
||||
/* Insert a cache entry for every URL in the redirect chain. */
|
||||
for (auto & url : res->urls) {
|
||||
cacheKey.second.insert_or_assign("url", url);
|
||||
getCache()->upsert(cacheKey, infoAttrs);
|
||||
settings.getCache()->upsert(cacheKey, infoAttrs);
|
||||
}
|
||||
|
||||
// FIXME: add a cache entry for immutableUrl? That could allow
|
||||
@@ -341,7 +344,7 @@ struct FileInputScheme : CurlInputScheme
|
||||
the Nix store directly, since there is little deduplication
|
||||
benefit in using the Git cache for single big files like
|
||||
tarballs. */
|
||||
auto file = downloadFile(store, getStrAttr(input.attrs, "url"), input.getName());
|
||||
auto file = downloadFile(store, *input.settings, getStrAttr(input.attrs, "url"), input.getName());
|
||||
|
||||
auto narHash = store->queryPathInfo(file.storePath)->narHash;
|
||||
input.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true));
|
||||
@@ -373,6 +376,7 @@ struct TarballInputScheme : CurlInputScheme
|
||||
auto input(_input);
|
||||
|
||||
auto result = downloadTarball_(
|
||||
*input.settings,
|
||||
getStrAttr(input.attrs, "url"),
|
||||
{},
|
||||
"«" + input.to_string() + "»");
|
||||
@@ -390,7 +394,7 @@ struct TarballInputScheme : CurlInputScheme
|
||||
input.attrs.insert_or_assign("lastModified", uint64_t(result.lastModified));
|
||||
|
||||
input.attrs.insert_or_assign("narHash",
|
||||
getTarballCache()->treeHashToNarHash(result.treeHash).to_string(HashFormat::SRI, true));
|
||||
getTarballCache()->treeHashToNarHash(*input.settings, result.treeHash).to_string(HashFormat::SRI, true));
|
||||
|
||||
return {result.accessor, input};
|
||||
}
|
||||
|
||||
+11
-15
@@ -39,24 +39,16 @@ let
|
||||
allNodes = mapAttrs (
|
||||
key: node:
|
||||
let
|
||||
hasOverride = overrides ? ${key};
|
||||
isRelative = node.locked.type or null == "path" && builtins.substring 0 1 node.locked.path != "/";
|
||||
|
||||
parentNode = allNodes.${getInputByPath lockFile.root node.parent};
|
||||
|
||||
flakeDir =
|
||||
let
|
||||
dir = overrides.${key}.dir or node.locked.path or "";
|
||||
parentDir = parentNode.flakeDir;
|
||||
in
|
||||
if node ? parent then parentDir + ("/" + dir) else dir;
|
||||
|
||||
sourceInfo =
|
||||
if overrides ? ${key} then
|
||||
if hasOverride then
|
||||
overrides.${key}.sourceInfo
|
||||
else if node.locked.type == "path" && builtins.substring 0 1 node.locked.path != "/" then
|
||||
else if isRelative then
|
||||
parentNode.sourceInfo
|
||||
// {
|
||||
outPath = parentNode.sourceInfo.outPath + ("/" + flakeDir);
|
||||
}
|
||||
else
|
||||
# FIXME: remove obsolete node.info.
|
||||
# Note: lock file entries are always final.
|
||||
@@ -64,7 +56,11 @@ let
|
||||
|
||||
subdir = overrides.${key}.dir or node.locked.dir or "";
|
||||
|
||||
outPath = sourceInfo + ((if subdir == "" then "" else "/") + subdir);
|
||||
outPath =
|
||||
if !hasOverride && isRelative then
|
||||
parentNode.outPath + (if node.locked.path == "" then "" else "/" + node.locked.path)
|
||||
else
|
||||
sourceInfo.outPath + (if subdir == "" then "" else "/" + subdir);
|
||||
|
||||
flake = import (outPath + "/flake.nix");
|
||||
|
||||
@@ -99,9 +95,9 @@ let
|
||||
assert builtins.isFunction flake.outputs;
|
||||
result
|
||||
else
|
||||
sourceInfo;
|
||||
sourceInfo // { inherit sourceInfo outPath; };
|
||||
|
||||
inherit flakeDir sourceInfo;
|
||||
inherit outPath sourceInfo;
|
||||
}
|
||||
) lockFile.nodes;
|
||||
|
||||
|
||||
+24
-38
@@ -14,6 +14,7 @@
|
||||
#include "nix/store/local-fs-store.hh"
|
||||
#include "nix/fetchers/fetch-to-store.hh"
|
||||
#include "nix/util/memory-source-accessor.hh"
|
||||
#include "nix/util/mounted-source-accessor.hh"
|
||||
#include "nix/fetchers/input-cache.hh"
|
||||
|
||||
#include <nlohmann/json.hpp>
|
||||
@@ -21,27 +22,10 @@
|
||||
namespace nix {
|
||||
|
||||
using namespace flake;
|
||||
using namespace fetchers;
|
||||
|
||||
namespace flake {
|
||||
|
||||
static StorePath copyInputToStore(
|
||||
EvalState & state,
|
||||
fetchers::Input & input,
|
||||
const fetchers::Input & originalInput,
|
||||
ref<SourceAccessor> accessor)
|
||||
{
|
||||
auto storePath = fetchToStore(*state.store, accessor, FetchMode::Copy, input.getName());
|
||||
|
||||
state.allowPath(storePath);
|
||||
|
||||
auto narHash = state.store->queryPathInfo(storePath)->narHash;
|
||||
input.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true));
|
||||
|
||||
assert(!originalInput.getNarHash() || storePath == originalInput.computeStorePath(*state.store));
|
||||
|
||||
return storePath;
|
||||
}
|
||||
|
||||
static void forceTrivialValue(EvalState & state, Value & value, const PosIdx pos)
|
||||
{
|
||||
if (value.isThunk() && value.isTrivial())
|
||||
@@ -67,7 +51,7 @@ static std::pair<std::map<FlakeId, FlakeInput>, fetchers::Attrs> parseFlakeInput
|
||||
|
||||
static void parseFlakeInputAttr(
|
||||
EvalState & state,
|
||||
const Attr & attr,
|
||||
const nix::Attr & attr,
|
||||
fetchers::Attrs & attrs)
|
||||
{
|
||||
// Allow selecting a subset of enum values
|
||||
@@ -276,7 +260,7 @@ static Flake readFlake(
|
||||
state.symbols[setting.name],
|
||||
std::string(state.forceStringNoCtx(*setting.value, setting.pos, "")));
|
||||
else if (setting.value->type() == nPath) {
|
||||
auto storePath = fetchToStore(*state.store, setting.value->path(), FetchMode::Copy);
|
||||
auto storePath = fetchToStore(state.fetchSettings, *state.store, setting.value->path(), FetchMode::Copy);
|
||||
flake.config.settings.emplace(
|
||||
state.symbols[setting.name],
|
||||
state.store->printStorePath(storePath));
|
||||
@@ -338,7 +322,8 @@ static Flake getFlake(
|
||||
EvalState & state,
|
||||
const FlakeRef & originalRef,
|
||||
fetchers::UseRegistries useRegistries,
|
||||
const InputAttrPath & lockRootAttrPath)
|
||||
const InputAttrPath & lockRootAttrPath,
|
||||
bool requireLockable)
|
||||
{
|
||||
// Fetch a lazy tree first.
|
||||
auto cachedInput = state.inputCache->getAccessor(state.store, originalRef.input, useRegistries);
|
||||
@@ -361,16 +346,16 @@ static Flake getFlake(
|
||||
lockedRef = FlakeRef(std::move(cachedInput2.lockedInput), newLockedRef.subdir);
|
||||
}
|
||||
|
||||
// Copy the tree to the store.
|
||||
auto storePath = copyInputToStore(state, lockedRef.input, originalRef.input, cachedInput.accessor);
|
||||
|
||||
// Re-parse flake.nix from the store.
|
||||
return readFlake(state, originalRef, resolvedRef, lockedRef, state.storePath(storePath), lockRootAttrPath);
|
||||
return readFlake(
|
||||
state, originalRef, resolvedRef, lockedRef,
|
||||
state.storePath(state.mountInput(lockedRef.input, originalRef.input, cachedInput.accessor, requireLockable)),
|
||||
lockRootAttrPath);
|
||||
}
|
||||
|
||||
Flake getFlake(EvalState & state, const FlakeRef & originalRef, fetchers::UseRegistries useRegistries)
|
||||
Flake getFlake(EvalState & state, const FlakeRef & originalRef, fetchers::UseRegistries useRegistries, bool requireLockable)
|
||||
{
|
||||
return getFlake(state, originalRef, useRegistries, {});
|
||||
return getFlake(state, originalRef, useRegistries, {}, requireLockable);
|
||||
}
|
||||
|
||||
static LockFile readLockFile(
|
||||
@@ -400,7 +385,8 @@ LockedFlake lockFlake(
|
||||
state,
|
||||
topRef,
|
||||
useRegistriesTop,
|
||||
{});
|
||||
{},
|
||||
lockFlags.requireLockable);
|
||||
|
||||
if (lockFlags.applyNixConfig) {
|
||||
flake.config.apply(settings);
|
||||
@@ -579,7 +565,8 @@ LockedFlake lockFlake(
|
||||
state,
|
||||
ref,
|
||||
useRegistriesInputs,
|
||||
inputAttrPath);
|
||||
inputAttrPath,
|
||||
true);
|
||||
}
|
||||
};
|
||||
|
||||
@@ -727,17 +714,15 @@ LockedFlake lockFlake(
|
||||
if (auto resolvedPath = resolveRelativePath()) {
|
||||
return {*resolvedPath, *input.ref};
|
||||
} else {
|
||||
auto cachedInput = state.inputCache->getAccessor(
|
||||
state.store,
|
||||
input.ref->input,
|
||||
useRegistriesInputs);
|
||||
auto cachedInput = state.inputCache->getAccessor(state.store, input.ref->input, useRegistriesTop);
|
||||
|
||||
auto resolvedRef = FlakeRef(std::move(cachedInput.resolvedInput), input.ref->subdir);
|
||||
auto lockedRef = FlakeRef(std::move(cachedInput.lockedInput), input.ref->subdir);
|
||||
|
||||
// FIXME: allow input to be lazy.
|
||||
auto storePath = copyInputToStore(state, lockedRef.input, input.ref->input, cachedInput.accessor);
|
||||
|
||||
return {state.storePath(storePath), lockedRef};
|
||||
return {
|
||||
state.storePath(state.mountInput(lockedRef.input, input.ref->input, cachedInput.accessor, true)),
|
||||
lockedRef
|
||||
};
|
||||
}
|
||||
}();
|
||||
|
||||
@@ -850,7 +835,8 @@ LockedFlake lockFlake(
|
||||
flake = getFlake(
|
||||
state,
|
||||
topRef,
|
||||
useRegistriesTop);
|
||||
useRegistriesTop,
|
||||
lockFlags.requireLockable);
|
||||
|
||||
if (lockFlags.commitLockFile &&
|
||||
flake.lockedRef.input.getRev() &&
|
||||
|
||||
@@ -115,7 +115,11 @@ struct Flake
|
||||
}
|
||||
};
|
||||
|
||||
Flake getFlake(EvalState & state, const FlakeRef & flakeRef, fetchers::UseRegistries useRegistries);
|
||||
Flake getFlake(
|
||||
EvalState & state,
|
||||
const FlakeRef & flakeRef,
|
||||
fetchers::UseRegistries useRegistries,
|
||||
bool requireLockable = true);
|
||||
|
||||
/**
|
||||
* Fingerprint of a locked flake; used as a cache key.
|
||||
@@ -213,6 +217,11 @@ struct LockFlags
|
||||
* for those inputs will be ignored.
|
||||
*/
|
||||
std::set<InputAttrPath> inputUpdates;
|
||||
|
||||
/**
|
||||
* Whether to require a locked input.
|
||||
*/
|
||||
bool requireLockable = true;
|
||||
};
|
||||
|
||||
LockedFlake lockFlake(
|
||||
|
||||
@@ -285,40 +285,13 @@ Goal::Co DerivationGoal::haveDerivation()
|
||||
|
||||
assert(!drv->type().isImpure());
|
||||
|
||||
if (nrFailed > 0 && nrFailed > nrNoSubstituters + nrIncompleteClosure && !settings.tryFallback) {
|
||||
if (nrFailed > 0 && nrFailed > nrNoSubstituters && !settings.tryFallback) {
|
||||
co_return done(BuildResult::TransientFailure, {},
|
||||
Error("some substitutes for the outputs of derivation '%s' failed (usually happens due to networking issues); try '--fallback' to build derivation from source ",
|
||||
worker.store.printStorePath(drvPath)));
|
||||
}
|
||||
|
||||
/* If the substitutes form an incomplete closure, then we should
|
||||
build the dependencies of this derivation, but after that, we
|
||||
can still use the substitutes for this derivation itself.
|
||||
|
||||
If the nrIncompleteClosure != nrFailed, we have another issue as well.
|
||||
In particular, it may be the case that the hole in the closure is
|
||||
an output of the current derivation, which causes a loop if retried.
|
||||
*/
|
||||
{
|
||||
bool substitutionFailed =
|
||||
nrIncompleteClosure > 0 &&
|
||||
nrIncompleteClosure == nrFailed;
|
||||
switch (retrySubstitution) {
|
||||
case RetrySubstitution::NoNeed:
|
||||
if (substitutionFailed)
|
||||
retrySubstitution = RetrySubstitution::YesNeed;
|
||||
break;
|
||||
case RetrySubstitution::YesNeed:
|
||||
// Should not be able to reach this state from here.
|
||||
assert(false);
|
||||
break;
|
||||
case RetrySubstitution::AlreadyRetried:
|
||||
debug("substitution failed again, but we already retried once. Not retrying again.");
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
nrFailed = nrNoSubstituters = nrIncompleteClosure = 0;
|
||||
nrFailed = nrNoSubstituters = 0;
|
||||
|
||||
if (needRestart == NeedRestartForMoreOutputs::OutputsAddedDoNeed) {
|
||||
needRestart = NeedRestartForMoreOutputs::OutputsUnmodifedDontNeed;
|
||||
@@ -456,11 +429,6 @@ Goal::Co DerivationGoal::gaveUpOnSubstitution()
|
||||
co_return done(BuildResult::DependencyFailed, {}, Error(msg));
|
||||
}
|
||||
|
||||
if (retrySubstitution == RetrySubstitution::YesNeed) {
|
||||
retrySubstitution = RetrySubstitution::AlreadyRetried;
|
||||
co_return haveDerivation();
|
||||
}
|
||||
|
||||
/* Gather information necessary for computing the closure and/or
|
||||
running the build hook. */
|
||||
|
||||
|
||||
@@ -139,7 +139,7 @@ Goal::Co DrvOutputSubstitutionGoal::realisationFetched(Goals waitees, std::share
|
||||
|
||||
if (nrFailed > 0) {
|
||||
debug("The output path of the derivation output '%s' could not be substituted", id.to_string());
|
||||
co_return amDone(nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed);
|
||||
co_return amDone(nrNoSubstituters > 0 ? ecNoSubstituters : ecFailed);
|
||||
}
|
||||
|
||||
worker.store.registerDrvOutput(*outputInfo);
|
||||
|
||||
@@ -151,7 +151,7 @@ Goal::Done Goal::amDone(ExitCode result, std::optional<Error> ex)
|
||||
trace("done");
|
||||
assert(top_co);
|
||||
assert(exitCode == ecBusy);
|
||||
assert(result == ecSuccess || result == ecFailed || result == ecNoSubstituters || result == ecIncompleteClosure);
|
||||
assert(result == ecSuccess || result == ecFailed || result == ecNoSubstituters);
|
||||
exitCode = result;
|
||||
|
||||
if (ex) {
|
||||
@@ -170,12 +170,10 @@ Goal::Done Goal::amDone(ExitCode result, std::optional<Error> ex)
|
||||
|
||||
goal->trace(fmt("waitee '%s' done; %d left", name, goal->waitees.size()));
|
||||
|
||||
if (result == ecFailed || result == ecNoSubstituters || result == ecIncompleteClosure) ++goal->nrFailed;
|
||||
if (result == ecFailed || result == ecNoSubstituters) ++goal->nrFailed;
|
||||
|
||||
if (result == ecNoSubstituters) ++goal->nrNoSubstituters;
|
||||
|
||||
if (result == ecIncompleteClosure) ++goal->nrIncompleteClosure;
|
||||
|
||||
if (goal->waitees.empty()) {
|
||||
worker.wakeUp(goal);
|
||||
} else if (result == ecFailed && !settings.keepGoing) {
|
||||
|
||||
@@ -169,7 +169,7 @@ Goal::Co PathSubstitutionGoal::tryToRun(StorePath subPath, nix::ref<Store> sub,
|
||||
|
||||
if (nrFailed > 0) {
|
||||
co_return done(
|
||||
nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed,
|
||||
nrNoSubstituters > 0 ? ecNoSubstituters : ecFailed,
|
||||
BuildResult::DependencyFailed,
|
||||
fmt("some references of path '%s' could not be realised", worker.store.printStorePath(storePath)));
|
||||
}
|
||||
|
||||
@@ -41,6 +41,16 @@ Worker::~Worker()
|
||||
assert(expectedNarSize == 0);
|
||||
}
|
||||
|
||||
template<class G, typename... Args>
|
||||
std::shared_ptr<G> Worker::initGoalIfNeeded(std::weak_ptr<G> & goal_weak, Args && ...args)
|
||||
{
|
||||
if (auto goal = goal_weak.lock()) return goal;
|
||||
|
||||
auto goal = std::make_shared<G>(args...);
|
||||
goal_weak = goal;
|
||||
wakeUp(goal);
|
||||
return goal;
|
||||
}
|
||||
|
||||
std::shared_ptr<DerivationGoal> Worker::makeDerivationGoalCommon(
|
||||
const StorePath & drvPath,
|
||||
@@ -79,27 +89,13 @@ std::shared_ptr<DerivationGoal> Worker::makeBasicDerivationGoal(const StorePath
|
||||
|
||||
std::shared_ptr<PathSubstitutionGoal> Worker::makePathSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional<ContentAddress> ca)
|
||||
{
|
||||
std::weak_ptr<PathSubstitutionGoal> & goal_weak = substitutionGoals[path];
|
||||
auto goal = goal_weak.lock(); // FIXME
|
||||
if (!goal) {
|
||||
goal = std::make_shared<PathSubstitutionGoal>(path, *this, repair, ca);
|
||||
goal_weak = goal;
|
||||
wakeUp(goal);
|
||||
}
|
||||
return goal;
|
||||
return initGoalIfNeeded(substitutionGoals[path], path, *this, repair, ca);
|
||||
}
|
||||
|
||||
|
||||
std::shared_ptr<DrvOutputSubstitutionGoal> Worker::makeDrvOutputSubstitutionGoal(const DrvOutput& id, RepairFlag repair, std::optional<ContentAddress> ca)
|
||||
{
|
||||
std::weak_ptr<DrvOutputSubstitutionGoal> & goal_weak = drvOutputSubstitutionGoals[id];
|
||||
auto goal = goal_weak.lock(); // FIXME
|
||||
if (!goal) {
|
||||
goal = std::make_shared<DrvOutputSubstitutionGoal>(id, *this, repair, ca);
|
||||
goal_weak = goal;
|
||||
wakeUp(goal);
|
||||
}
|
||||
return goal;
|
||||
return initGoalIfNeeded(drvOutputSubstitutionGoals[id], id, *this, repair, ca);
|
||||
}
|
||||
|
||||
|
||||
|
||||
+15
-14
@@ -13,10 +13,11 @@
|
||||
# include "nix/util/processes.hh"
|
||||
#endif
|
||||
|
||||
#include <boost/regex.hpp>
|
||||
|
||||
#include <functional>
|
||||
#include <queue>
|
||||
#include <algorithm>
|
||||
#include <regex>
|
||||
#include <random>
|
||||
|
||||
#include <climits>
|
||||
@@ -331,8 +332,8 @@ static void readProcLink(const std::filesystem::path & file, UncheckedRoots & ro
|
||||
|
||||
static std::string quoteRegexChars(const std::string & raw)
|
||||
{
|
||||
static auto specialRegex = std::regex(R"([.^$\\*+?()\[\]{}|])");
|
||||
return std::regex_replace(raw, specialRegex, R"(\$&)");
|
||||
static auto specialRegex = boost::regex(R"([.^$\\*+?()\[\]{}|])");
|
||||
return boost::regex_replace(raw, specialRegex, R"(\$&)");
|
||||
}
|
||||
|
||||
#ifdef __linux__
|
||||
@@ -354,12 +355,12 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
|
||||
auto procDir = AutoCloseDir{opendir("/proc")};
|
||||
if (procDir) {
|
||||
struct dirent * ent;
|
||||
auto digitsRegex = std::regex(R"(^\d+$)");
|
||||
auto mapRegex = std::regex(R"(^\s*\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+(/\S+)\s*$)");
|
||||
auto storePathRegex = std::regex(quoteRegexChars(storeDir) + R"(/[0-9a-z]+[0-9a-zA-Z\+\-\._\?=]*)");
|
||||
static const auto digitsRegex = boost::regex(R"(^\d+$)");
|
||||
static const auto mapRegex = boost::regex(R"(^\s*\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+(/\S+)\s*$)");
|
||||
auto storePathRegex = boost::regex(quoteRegexChars(storeDir) + R"(/[0-9a-z]+[0-9a-zA-Z\+\-\._\?=]*)");
|
||||
while (errno = 0, ent = readdir(procDir.get())) {
|
||||
checkInterrupt();
|
||||
if (std::regex_match(ent->d_name, digitsRegex)) {
|
||||
if (boost::regex_match(ent->d_name, digitsRegex)) {
|
||||
try {
|
||||
readProcLink(fmt("/proc/%s/exe" ,ent->d_name), unchecked);
|
||||
readProcLink(fmt("/proc/%s/cwd", ent->d_name), unchecked);
|
||||
@@ -386,15 +387,15 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
|
||||
std::filesystem::path mapFile = fmt("/proc/%s/maps", ent->d_name);
|
||||
auto mapLines = tokenizeString<std::vector<std::string>>(readFile(mapFile.string()), "\n");
|
||||
for (const auto & line : mapLines) {
|
||||
auto match = std::smatch{};
|
||||
if (std::regex_match(line, match, mapRegex))
|
||||
auto match = boost::smatch{};
|
||||
if (boost::regex_match(line, match, mapRegex))
|
||||
unchecked[match[1]].emplace(mapFile.string());
|
||||
}
|
||||
|
||||
auto envFile = fmt("/proc/%s/environ", ent->d_name);
|
||||
auto envString = readFile(envFile);
|
||||
auto env_end = std::sregex_iterator{};
|
||||
for (auto i = std::sregex_iterator{envString.begin(), envString.end(), storePathRegex}; i != env_end; ++i)
|
||||
auto env_end = boost::sregex_iterator{};
|
||||
for (auto i = boost::sregex_iterator{envString.begin(), envString.end(), storePathRegex}; i != env_end; ++i)
|
||||
unchecked[i->str()].emplace(envFile);
|
||||
} catch (SystemError & e) {
|
||||
if (errno == ENOENT || errno == EACCES || errno == ESRCH)
|
||||
@@ -413,12 +414,12 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
|
||||
// Because of this we disable lsof when running the tests.
|
||||
if (getEnv("_NIX_TEST_NO_LSOF") != "1") {
|
||||
try {
|
||||
std::regex lsofRegex(R"(^n(/.*)$)");
|
||||
boost::regex lsofRegex(R"(^n(/.*)$)");
|
||||
auto lsofLines =
|
||||
tokenizeString<std::vector<std::string>>(runProgram(LSOF, true, { "-n", "-w", "-F", "n" }), "\n");
|
||||
for (const auto & line : lsofLines) {
|
||||
std::smatch match;
|
||||
if (std::regex_match(line, match, lsofRegex))
|
||||
boost::smatch match;
|
||||
if (boost::regex_match(line, match, lsofRegex))
|
||||
unchecked[match[1].str()].emplace("{lsof}");
|
||||
}
|
||||
} catch (ExecError & e) {
|
||||
|
||||
@@ -73,32 +73,6 @@ struct DerivationGoal : public Goal
|
||||
*/
|
||||
NeedRestartForMoreOutputs needRestart = NeedRestartForMoreOutputs::OutputsUnmodifedDontNeed;
|
||||
|
||||
/**
|
||||
* See `retrySubstitution`; just for that field.
|
||||
*/
|
||||
enum RetrySubstitution {
|
||||
/**
|
||||
* No issues have yet arose, no need to restart.
|
||||
*/
|
||||
NoNeed,
|
||||
/**
|
||||
* Something failed and there is an incomplete closure. Let's retry
|
||||
* substituting.
|
||||
*/
|
||||
YesNeed,
|
||||
/**
|
||||
* We are current or have already retried substitution, and whether or
|
||||
* not something goes wrong we will not retry again.
|
||||
*/
|
||||
AlreadyRetried,
|
||||
};
|
||||
|
||||
/**
|
||||
* Whether to retry substituting the outputs after building the
|
||||
* inputs. This is done in case of an incomplete closure.
|
||||
*/
|
||||
RetrySubstitution retrySubstitution = RetrySubstitution::NoNeed;
|
||||
|
||||
/**
|
||||
* The derivation stored at drvPath.
|
||||
*/
|
||||
|
||||
@@ -61,7 +61,7 @@ private:
|
||||
Goals waitees;
|
||||
|
||||
public:
|
||||
typedef enum {ecBusy, ecSuccess, ecFailed, ecNoSubstituters, ecIncompleteClosure} ExitCode;
|
||||
typedef enum {ecBusy, ecSuccess, ecFailed, ecNoSubstituters} ExitCode;
|
||||
|
||||
/**
|
||||
* Backlink to the worker.
|
||||
@@ -85,12 +85,6 @@ public:
|
||||
*/
|
||||
size_t nrNoSubstituters = 0;
|
||||
|
||||
/**
|
||||
* Number of substitution goals we are/were waiting for that
|
||||
* failed because they had unsubstitutable references.
|
||||
*/
|
||||
size_t nrIncompleteClosure = 0;
|
||||
|
||||
/**
|
||||
* Name of this goal for debugging purposes.
|
||||
*/
|
||||
|
||||
@@ -196,6 +196,9 @@ public:
|
||||
* @ref DerivationGoal "derivation goal"
|
||||
*/
|
||||
private:
|
||||
template<class G, typename... Args>
|
||||
std::shared_ptr<G> initGoalIfNeeded(std::weak_ptr<G> & goal_weak, Args && ...args);
|
||||
|
||||
std::shared_ptr<DerivationGoal> makeDerivationGoalCommon(
|
||||
const StorePath & drvPath, const OutputsSpec & wantedOutputs,
|
||||
std::function<std::shared_ptr<DerivationGoal>()> mkDrvGoal);
|
||||
|
||||
@@ -104,7 +104,7 @@ struct MixStoreDirMethods
|
||||
* Need to make this a separate class so I can get the right
|
||||
* initialization order in the constructor for `StoreDirConfig`.
|
||||
*/
|
||||
struct StoreDirConfigItself : Config
|
||||
struct StoreDirConfigBase : Config
|
||||
{
|
||||
using Config::Config;
|
||||
|
||||
@@ -118,12 +118,12 @@ struct StoreDirConfigItself : Config
|
||||
};
|
||||
|
||||
/**
|
||||
* The order of `StoreDirConfigItself` and then `MixStoreDirMethods` is
|
||||
* very important. This ensures that `StoreDirConfigItself::storeDir_`
|
||||
* The order of `StoreDirConfigBase` and then `MixStoreDirMethods` is
|
||||
* very important. This ensures that `StoreDirConfigBase::storeDir_`
|
||||
* is initialized before we have our one chance (because references are
|
||||
* immutable) to initialize `MixStoreDirMethods::storeDir`.
|
||||
*/
|
||||
struct StoreDirConfig : StoreDirConfigItself, MixStoreDirMethods
|
||||
struct StoreDirConfig : StoreDirConfigBase, MixStoreDirMethods
|
||||
{
|
||||
using Params = std::map<std::string, std::string>;
|
||||
|
||||
|
||||
@@ -94,7 +94,7 @@ subdir('nix-meson-build-support/libatomic')
|
||||
|
||||
boost = dependency(
|
||||
'boost',
|
||||
modules : ['container'],
|
||||
modules : ['container', 'regex'],
|
||||
include_type: 'system',
|
||||
)
|
||||
# boost is a public dependency, but not a pkg-config dependency unfortunately, so we
|
||||
|
||||
@@ -215,8 +215,12 @@ StorePath Store::addToStore(
|
||||
auto sink = sourceToSink([&](Source & source) {
|
||||
LengthSource lengthSource(source);
|
||||
storePath = addToStoreFromDump(lengthSource, name, fsm, method, hashAlgo, references, repair);
|
||||
if (settings.warnLargePathThreshold && lengthSource.total >= settings.warnLargePathThreshold)
|
||||
warn("copied large path '%s' to the store (%s)", path, renderSize(lengthSource.total));
|
||||
if (settings.warnLargePathThreshold && lengthSource.total >= settings.warnLargePathThreshold) {
|
||||
static bool failOnLargePath = getEnv("_NIX_TEST_FAIL_ON_LARGE_PATH").value_or("") == "1";
|
||||
if (failOnLargePath)
|
||||
throw Error("won't copy large path '%s' to the store (%d)", path, renderSize(lengthSource.total));
|
||||
warn("copied large path '%s' to the store (%d)", path, renderSize(lengthSource.total));
|
||||
}
|
||||
});
|
||||
dumpPath(path, *sink, fsm, filter);
|
||||
sink->finish();
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
namespace nix {
|
||||
|
||||
StoreDirConfig::StoreDirConfig(const Params & params)
|
||||
: StoreDirConfigItself(params)
|
||||
: StoreDirConfigBase(params)
|
||||
, MixStoreDirMethods{storeDir_}
|
||||
{
|
||||
}
|
||||
|
||||
@@ -131,7 +131,7 @@ TEST(getString, wrongAssertions) {
|
||||
TEST(getIntegralNumber, rightAssertions) {
|
||||
auto simple = R"({ "int": 0, "signed": -1 })"_json;
|
||||
|
||||
ASSERT_EQ(getUnsigned(valueAt(getObject(simple), "int")), 0);
|
||||
ASSERT_EQ(getUnsigned(valueAt(getObject(simple), "int")), 0u);
|
||||
ASSERT_EQ(getInteger<int8_t>(valueAt(getObject(simple), "int")), 0);
|
||||
ASSERT_EQ(getInteger<int8_t>(valueAt(getObject(simple), "signed")), -1);
|
||||
}
|
||||
|
||||
@@ -33,6 +33,18 @@ private:
|
||||
Data data;
|
||||
LRU lru;
|
||||
|
||||
/**
|
||||
* Move this item to the back of the LRU list.
|
||||
*/
|
||||
void promote(LRU::iterator it)
|
||||
{
|
||||
/* Think of std::list iterators as stable pointers to the list node,
|
||||
* which never get invalidated. Thus, we can reuse the same lru list
|
||||
* element and just splice it to the back of the list without the need
|
||||
* to update its value in the key -> list iterator map. */
|
||||
lru.splice(/*pos=*/lru.end(), /*other=*/lru, it);
|
||||
}
|
||||
|
||||
public:
|
||||
|
||||
LRUCache(size_t capacity)
|
||||
@@ -83,7 +95,9 @@ public:
|
||||
/**
|
||||
* Look up an item in the cache. If it exists, it becomes the most
|
||||
* recently used item.
|
||||
* */
|
||||
*
|
||||
* @returns corresponding cache entry, std::nullopt if it's not in the cache
|
||||
*/
|
||||
template<typename K>
|
||||
std::optional<Value> get(const K & key)
|
||||
{
|
||||
@@ -91,20 +105,30 @@ public:
|
||||
if (i == data.end())
|
||||
return {};
|
||||
|
||||
/**
|
||||
* Move this item to the back of the LRU list.
|
||||
*
|
||||
* Think of std::list iterators as stable pointers to the list node,
|
||||
* which never get invalidated. Thus, we can reuse the same lru list
|
||||
* element and just splice it to the back of the list without the need
|
||||
* to update its value in the key -> list iterator map.
|
||||
*/
|
||||
auto & [it, value] = i->second;
|
||||
lru.splice(/*pos=*/lru.end(), /*other=*/lru, it.it);
|
||||
|
||||
promote(it.it);
|
||||
return value;
|
||||
}
|
||||
|
||||
/**
|
||||
* Look up an item in the cache. If it exists, it becomes the most
|
||||
* recently used item.
|
||||
*
|
||||
* @returns mutable pointer to the corresponding cache entry, nullptr if
|
||||
* it's not in the cache
|
||||
*/
|
||||
template<typename K>
|
||||
Value * getOrNullptr(const K & key)
|
||||
{
|
||||
auto i = data.find(key);
|
||||
if (i == data.end())
|
||||
return nullptr;
|
||||
|
||||
auto & [it, value] = i->second;
|
||||
promote(it.it);
|
||||
return &value;
|
||||
}
|
||||
|
||||
size_t size() const noexcept
|
||||
{
|
||||
return data.size();
|
||||
|
||||
@@ -43,6 +43,7 @@ headers = files(
|
||||
'logging.hh',
|
||||
'lru-cache.hh',
|
||||
'memory-source-accessor.hh',
|
||||
'mounted-source-accessor.hh',
|
||||
'muxable-pipe.hh',
|
||||
'os-string.hh',
|
||||
'pool.hh',
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
#pragma once
|
||||
|
||||
#include "source-accessor.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
struct MountedSourceAccessor : SourceAccessor
|
||||
{
|
||||
virtual void mount(CanonPath mountPoint, ref<SourceAccessor> accessor) = 0;
|
||||
|
||||
/**
|
||||
* Return the accessor mounted on `mountPoint`, or `nullptr` if
|
||||
* there is no such mount point.
|
||||
*/
|
||||
virtual std::shared_ptr<SourceAccessor> getMount(CanonPath mountPoint) = 0;
|
||||
};
|
||||
|
||||
ref<MountedSourceAccessor> makeMountedSourceAccessor(std::map<CanonPath, ref<SourceAccessor>> mounts);
|
||||
|
||||
}
|
||||
@@ -4,6 +4,7 @@
|
||||
#include <cstdint>
|
||||
#include <vector>
|
||||
|
||||
#include "nix/util/lru-cache.hh"
|
||||
#include "nix/util/pos-idx.hh"
|
||||
#include "nix/util/position.hh"
|
||||
#include "nix/util/sync.hh"
|
||||
@@ -37,10 +38,20 @@ public:
|
||||
};
|
||||
|
||||
private:
|
||||
/**
|
||||
* Vector of byte offsets (in the virtual input buffer) of initial line character's position.
|
||||
* Sorted by construction. Binary search over it allows for efficient translation of arbitrary
|
||||
* byte offsets in the virtual input buffer to its line + column position.
|
||||
*/
|
||||
using Lines = std::vector<uint32_t>;
|
||||
/**
|
||||
* Cache from byte offset in the virtual buffer of Origins -> @ref Lines in that origin.
|
||||
*/
|
||||
using LinesCache = LRUCache<uint32_t, Lines>;
|
||||
|
||||
std::map<uint32_t, Origin> origins;
|
||||
mutable Sync<std::map<uint32_t, Lines>> lines;
|
||||
|
||||
mutable Sync<LinesCache> linesCache;
|
||||
|
||||
const Origin * resolve(PosIdx p) const
|
||||
{
|
||||
@@ -56,6 +67,11 @@ private:
|
||||
}
|
||||
|
||||
public:
|
||||
PosTable(std::size_t linesCacheCapacity = 65536)
|
||||
: linesCache(linesCacheCapacity)
|
||||
{
|
||||
}
|
||||
|
||||
Origin addOrigin(Pos::Origin origin, size_t size)
|
||||
{
|
||||
uint32_t offset = 0;
|
||||
|
||||
@@ -118,7 +118,7 @@ struct SourceAccessor : std::enable_shared_from_this<SourceAccessor>
|
||||
std::string typeString();
|
||||
};
|
||||
|
||||
Stat lstat(const CanonPath & path);
|
||||
virtual Stat lstat(const CanonPath & path);
|
||||
|
||||
virtual std::optional<Stat> maybeLstat(const CanonPath & path) = 0;
|
||||
|
||||
@@ -214,8 +214,6 @@ ref<SourceAccessor> getFSSourceAccessor();
|
||||
*/
|
||||
ref<SourceAccessor> makeFSSourceAccessor(std::filesystem::path root);
|
||||
|
||||
ref<SourceAccessor> makeMountedSourceAccessor(std::map<CanonPath, ref<SourceAccessor>> mounts);
|
||||
|
||||
/**
|
||||
* Construct an accessor that presents a "union" view of a vector of
|
||||
* underlying accessors. Earlier accessors take precedence over later.
|
||||
|
||||
@@ -39,6 +39,7 @@ public:
|
||||
SyncBase() { }
|
||||
SyncBase(const T & data) : data(data) { }
|
||||
SyncBase(T && data) noexcept : data(std::move(data)) { }
|
||||
SyncBase(SyncBase && other) noexcept : data(std::move(*other.lock())) { }
|
||||
|
||||
template<class L>
|
||||
class Lock
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
#include "nix/util/source-accessor.hh"
|
||||
#include "nix/util/mounted-source-accessor.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
struct MountedSourceAccessor : SourceAccessor
|
||||
struct MountedSourceAccessorImpl : MountedSourceAccessor
|
||||
{
|
||||
std::map<CanonPath, ref<SourceAccessor>> mounts;
|
||||
|
||||
MountedSourceAccessor(std::map<CanonPath, ref<SourceAccessor>> _mounts)
|
||||
MountedSourceAccessorImpl(std::map<CanonPath, ref<SourceAccessor>> _mounts)
|
||||
: mounts(std::move(_mounts))
|
||||
{
|
||||
displayPrefix.clear();
|
||||
@@ -23,6 +23,12 @@ struct MountedSourceAccessor : SourceAccessor
|
||||
return accessor->readFile(subpath);
|
||||
}
|
||||
|
||||
Stat lstat(const CanonPath & path) override
|
||||
{
|
||||
auto [accessor, subpath] = resolve(path);
|
||||
return accessor->lstat(subpath);
|
||||
}
|
||||
|
||||
std::optional<Stat> maybeLstat(const CanonPath & path) override
|
||||
{
|
||||
auto [accessor, subpath] = resolve(path);
|
||||
@@ -69,11 +75,26 @@ struct MountedSourceAccessor : SourceAccessor
|
||||
auto [accessor, subpath] = resolve(path);
|
||||
return accessor->getPhysicalPath(subpath);
|
||||
}
|
||||
|
||||
void mount(CanonPath mountPoint, ref<SourceAccessor> accessor) override
|
||||
{
|
||||
// FIXME: thread-safety
|
||||
mounts.insert_or_assign(std::move(mountPoint), accessor);
|
||||
}
|
||||
|
||||
std::shared_ptr<SourceAccessor> getMount(CanonPath mountPoint) override
|
||||
{
|
||||
auto i = mounts.find(mountPoint);
|
||||
if (i != mounts.end())
|
||||
return i->second;
|
||||
else
|
||||
return nullptr;
|
||||
}
|
||||
};
|
||||
|
||||
ref<SourceAccessor> makeMountedSourceAccessor(std::map<CanonPath, ref<SourceAccessor>> mounts)
|
||||
ref<MountedSourceAccessor> makeMountedSourceAccessor(std::map<CanonPath, ref<SourceAccessor>> mounts)
|
||||
{
|
||||
return make_ref<MountedSourceAccessor>(std::move(mounts));
|
||||
return make_ref<MountedSourceAccessorImpl>(std::move(mounts));
|
||||
}
|
||||
|
||||
}
|
||||
+26
-12
@@ -15,21 +15,35 @@ Pos PosTable::operator[](PosIdx p) const
|
||||
const auto offset = origin->offsetOf(p);
|
||||
|
||||
Pos result{0, 0, origin->origin};
|
||||
auto lines = this->lines.lock();
|
||||
auto linesForInput = (*lines)[origin->offset];
|
||||
auto linesCache = this->linesCache.lock();
|
||||
|
||||
if (linesForInput.empty()) {
|
||||
auto source = result.getSource().value_or("");
|
||||
const char * begin = source.data();
|
||||
for (Pos::LinesIterator it(source), end; it != end; it++)
|
||||
linesForInput.push_back(it->data() - begin);
|
||||
if (linesForInput.empty())
|
||||
linesForInput.push_back(0);
|
||||
/* Try the origin's line cache */
|
||||
const auto * linesForInput = linesCache->getOrNullptr(origin->offset);
|
||||
|
||||
auto fillCacheForOrigin = [](std::string_view content) {
|
||||
auto contentLines = Lines();
|
||||
|
||||
const char * begin = content.data();
|
||||
for (Pos::LinesIterator it(content), end; it != end; it++)
|
||||
contentLines.push_back(it->data() - begin);
|
||||
if (contentLines.empty())
|
||||
contentLines.push_back(0);
|
||||
|
||||
return contentLines;
|
||||
};
|
||||
|
||||
/* Calculate line offsets and fill the cache */
|
||||
if (!linesForInput) {
|
||||
auto originContent = result.getSource().value_or("");
|
||||
linesCache->upsert(origin->offset, fillCacheForOrigin(originContent));
|
||||
linesForInput = linesCache->getOrNullptr(origin->offset);
|
||||
}
|
||||
// as above: the first line starts at byte 0 and is always present
|
||||
auto lineStartOffset = std::prev(std::upper_bound(linesForInput.begin(), linesForInput.end(), offset));
|
||||
|
||||
result.line = 1 + (lineStartOffset - linesForInput.begin());
|
||||
assert(linesForInput);
|
||||
|
||||
// as above: the first line starts at byte 0 and is always present
|
||||
auto lineStartOffset = std::prev(std::upper_bound(linesForInput->begin(), linesForInput->end(), offset));
|
||||
result.line = 1 + (lineStartOffset - linesForInput->begin());
|
||||
result.column = 1 + (offset - *lineStartOffset);
|
||||
return result;
|
||||
}
|
||||
|
||||
@@ -4,9 +4,11 @@
|
||||
#include "nix/store/filetransfer.hh"
|
||||
#include "nix/store/store-open.hh"
|
||||
#include "nix/cmd/legacy.hh"
|
||||
#include "nix/cmd/common-eval-args.hh"
|
||||
#include "nix/expr/eval-settings.hh" // for defexpr
|
||||
#include "nix/util/users.hh"
|
||||
#include "nix/fetchers/tarball.hh"
|
||||
#include "nix/fetchers/fetch-settings.hh"
|
||||
#include "self-exe.hh"
|
||||
#include "man-pages.hh"
|
||||
|
||||
@@ -114,7 +116,7 @@ static void update(const StringSet & channelNames)
|
||||
// We want to download the url to a file to see if it's a tarball while also checking if we
|
||||
// got redirected in the process, so that we can grab the various parts of a nix channel
|
||||
// definition from a consistent location if the redirect changes mid-download.
|
||||
auto result = fetchers::downloadFile(store, url, std::string(baseNameOf(url)));
|
||||
auto result = fetchers::downloadFile(store, fetchSettings, url, std::string(baseNameOf(url)));
|
||||
auto filename = store->toRealPath(result.storePath);
|
||||
url = result.effectiveUrl;
|
||||
|
||||
@@ -128,9 +130,9 @@ static void update(const StringSet & channelNames)
|
||||
if (!unpacked) {
|
||||
// Download the channel tarball.
|
||||
try {
|
||||
filename = store->toRealPath(fetchers::downloadFile(store, url + "/nixexprs.tar.xz", "nixexprs.tar.xz").storePath);
|
||||
filename = store->toRealPath(fetchers::downloadFile(store, fetchSettings, url + "/nixexprs.tar.xz", "nixexprs.tar.xz").storePath);
|
||||
} catch (FileTransferError & e) {
|
||||
filename = store->toRealPath(fetchers::downloadFile(store, url + "/nixexprs.tar.bz2", "nixexprs.tar.bz2").storePath);
|
||||
filename = store->toRealPath(fetchers::downloadFile(store, fetchSettings, url + "/nixexprs.tar.bz2", "nixexprs.tar.bz2").storePath);
|
||||
}
|
||||
}
|
||||
// Regardless of where it came from, add the expression representing this channel to accumulated expression
|
||||
|
||||
@@ -110,7 +110,7 @@ bool createUserEnv(EvalState & state, PackageInfos & elems,
|
||||
environment. */
|
||||
auto manifestFile = ({
|
||||
std::ostringstream str;
|
||||
printAmbiguous(manifest, state.symbols, str, nullptr, std::numeric_limits<int>::max());
|
||||
printAmbiguous(state, manifest, str, nullptr, std::numeric_limits<int>::max());
|
||||
StringSource source { toView(str) };
|
||||
state.store->addToStoreFromDump(
|
||||
source, "env-manifest.nix", FileSerialisationMethod::Flat, ContentAddressMethod::Raw::Text, HashAlgorithm::SHA256, references);
|
||||
|
||||
@@ -52,7 +52,10 @@ void processExpr(EvalState & state, const Strings & attrPaths,
|
||||
else
|
||||
state.autoCallFunction(autoArgs, v, vRes);
|
||||
if (output == okRaw)
|
||||
std::cout << *state.coerceToString(noPos, vRes, context, "while generating the nix-instantiate output");
|
||||
std::cout <<
|
||||
state.devirtualize(
|
||||
*state.coerceToString(noPos, vRes, context, "while generating the nix-instantiate output"),
|
||||
context);
|
||||
// We intentionally don't output a newline here. The default PS1 for Bash in NixOS starts with a newline
|
||||
// and other interactive shells like Zsh are smart enough to print a missing newline before the prompt.
|
||||
else if (output == okXML)
|
||||
@@ -63,7 +66,7 @@ void processExpr(EvalState & state, const Strings & attrPaths,
|
||||
} else {
|
||||
if (strict) state.forceValueDeep(vRes);
|
||||
std::set<const void *> seen;
|
||||
printAmbiguous(vRes, state.symbols, std::cout, &seen, std::numeric_limits<int>::max());
|
||||
printAmbiguous(state, vRes, std::cout, &seen, std::numeric_limits<int>::max());
|
||||
std::cout << std::endl;
|
||||
}
|
||||
} else {
|
||||
|
||||
@@ -92,6 +92,9 @@ UnresolvedApp InstallableValue::toApp(EvalState & state)
|
||||
.path = o.path,
|
||||
};
|
||||
},
|
||||
[&](const NixStringContextElem::Path & p) -> DerivedPath {
|
||||
throw Error("'program' attribute of an 'app' output cannot have no context");
|
||||
},
|
||||
}, c.raw));
|
||||
}
|
||||
|
||||
|
||||
@@ -6,6 +6,7 @@
|
||||
#include "run.hh"
|
||||
#include "nix/util/strings.hh"
|
||||
#include "nix/util/executable-path.hh"
|
||||
#include "nix/util/mounted-source-accessor.hh"
|
||||
|
||||
using namespace nix;
|
||||
|
||||
|
||||
+23
-10
@@ -134,6 +134,7 @@ public:
|
||||
lockFlags.recreateLockFile = updateAll;
|
||||
lockFlags.writeLockFile = true;
|
||||
lockFlags.applyNixConfig = true;
|
||||
lockFlags.requireLockable = false;
|
||||
|
||||
lockFlake();
|
||||
}
|
||||
@@ -166,6 +167,7 @@ struct CmdFlakeLock : FlakeCommand
|
||||
lockFlags.writeLockFile = true;
|
||||
lockFlags.failOnUnlocked = true;
|
||||
lockFlags.applyNixConfig = true;
|
||||
lockFlags.requireLockable = false;
|
||||
|
||||
lockFlake();
|
||||
}
|
||||
@@ -212,11 +214,17 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON
|
||||
|
||||
void run(nix::ref<nix::Store> store) override
|
||||
{
|
||||
lockFlags.requireLockable = false;
|
||||
auto lockedFlake = lockFlake();
|
||||
auto & flake = lockedFlake.flake;
|
||||
|
||||
// Currently, all flakes are in the Nix store via the rootFS accessor.
|
||||
auto storePath = store->printStorePath(store->toStorePath(flake.path.path.abs()).first);
|
||||
/* Hack to show the store path if available. */
|
||||
std::optional<StorePath> storePath;
|
||||
if (store->isInStore(flake.path.path.abs())) {
|
||||
auto path = store->toStorePath(flake.path.path.abs()).first;
|
||||
if (store->isValidPath(path))
|
||||
storePath = path;
|
||||
}
|
||||
|
||||
if (json) {
|
||||
nlohmann::json j;
|
||||
@@ -238,7 +246,8 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON
|
||||
j["revCount"] = *revCount;
|
||||
if (auto lastModified = flake.lockedRef.input.getLastModified())
|
||||
j["lastModified"] = *lastModified;
|
||||
j["path"] = storePath;
|
||||
if (storePath)
|
||||
j["path"] = store->printStorePath(*storePath);
|
||||
j["locks"] = lockedFlake.lockFile.toJSON().first;
|
||||
if (auto fingerprint = lockedFlake.getFingerprint(store, fetchSettings))
|
||||
j["fingerprint"] = fingerprint->to_string(HashFormat::Base16, false);
|
||||
@@ -255,9 +264,10 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON
|
||||
logger->cout(
|
||||
ANSI_BOLD "Description:" ANSI_NORMAL " %s",
|
||||
*flake.description);
|
||||
logger->cout(
|
||||
ANSI_BOLD "Path:" ANSI_NORMAL " %s",
|
||||
storePath);
|
||||
if (storePath)
|
||||
logger->cout(
|
||||
ANSI_BOLD "Path:" ANSI_NORMAL " %s",
|
||||
store->printStorePath(*storePath));
|
||||
if (auto rev = flake.lockedRef.input.getRev())
|
||||
logger->cout(
|
||||
ANSI_BOLD "Revision:" ANSI_NORMAL " %s",
|
||||
@@ -637,7 +647,7 @@ struct CmdFlakeCheck : FlakeCommand
|
||||
if (name == "checks") {
|
||||
state->forceAttrs(vOutput, pos, "");
|
||||
for (auto & attr : *vOutput.attrs()) {
|
||||
std::string_view attr_name = state->symbols[attr.name];
|
||||
const auto & attr_name = state->symbols[attr.name];
|
||||
checkSystemName(attr_name, attr.pos);
|
||||
if (checkSystemType(attr_name, attr.pos)) {
|
||||
state->forceAttrs(*attr.value, attr.pos, "");
|
||||
@@ -1079,7 +1089,10 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun
|
||||
|
||||
StorePathSet sources;
|
||||
|
||||
auto storePath = store->toStorePath(flake.flake.path.path.abs()).first;
|
||||
auto storePath =
|
||||
dryRun
|
||||
? flake.flake.lockedRef.input.computeStorePath(*store)
|
||||
: std::get<StorePath>(flake.flake.lockedRef.input.fetchToStore(store));
|
||||
|
||||
sources.insert(storePath);
|
||||
|
||||
@@ -1095,7 +1108,7 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun
|
||||
storePath =
|
||||
dryRun
|
||||
? (*inputNode)->lockedRef.input.computeStorePath(*store)
|
||||
: (*inputNode)->lockedRef.input.fetchToStore(store).first;
|
||||
: std::get<StorePath>((*inputNode)->lockedRef.input.fetchToStore(store));
|
||||
sources.insert(*storePath);
|
||||
}
|
||||
if (json) {
|
||||
@@ -1488,7 +1501,7 @@ struct CmdFlakePrefetch : FlakeCommand, MixJSON
|
||||
auto originalRef = getFlakeRef();
|
||||
auto resolvedRef = originalRef.resolve(store);
|
||||
auto [accessor, lockedRef] = resolvedRef.lazyFetch(store);
|
||||
auto storePath = fetchToStore(*store, accessor, FetchMode::Copy, lockedRef.input.getName());
|
||||
auto storePath = fetchToStore(getEvalState()->fetchSettings, *store, accessor, FetchMode::Copy, lockedRef.input.getName());
|
||||
auto hash = store->queryPathInfo(storePath)->narHash;
|
||||
|
||||
if (json) {
|
||||
|
||||
@@ -151,8 +151,11 @@ nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -
|
||||
grepQuiet "don't know how to build" "$TEST_ROOT/log"
|
||||
grepQuiet "building.*input-1" "$TEST_ROOT/log"
|
||||
grepQuiet "building.*input-2" "$TEST_ROOT/log"
|
||||
grepQuiet "copying path.*input-0" "$TEST_ROOT/log"
|
||||
grepQuiet "copying path.*top" "$TEST_ROOT/log"
|
||||
|
||||
# Removed for now since 299141ecbd08bae17013226dbeae71e842b4fdd7 / issue #77 is reverted
|
||||
|
||||
#grepQuiet "copying path.*input-0" "$TEST_ROOT/log"
|
||||
#grepQuiet "copying path.*top" "$TEST_ROOT/log"
|
||||
|
||||
|
||||
# Create a signed binary cache.
|
||||
|
||||
@@ -77,6 +77,7 @@ hash1=$(echo "$json" | jq -r .revision)
|
||||
echo foo > "$flake1Dir/foo"
|
||||
git -C "$flake1Dir" add $flake1Dir/foo
|
||||
[[ $(nix flake metadata flake1 --json --refresh | jq -r .dirtyRevision) == "$hash1-dirty" ]]
|
||||
[[ $(_NIX_TEST_FAIL_ON_LARGE_PATH=1 nix flake metadata flake1 --json --refresh --warn-large-path-threshold 1 --lazy-trees | jq -r .dirtyRevision) == "$hash1-dirty" ]]
|
||||
[[ "$(nix flake metadata flake1 --json | jq -r .fingerprint)" != null ]]
|
||||
|
||||
echo -n '# foo' >> "$flake1Dir/flake.nix"
|
||||
|
||||
@@ -37,11 +37,20 @@ cat > "$flake3Dir/flake.nix" <<EOF
|
||||
url = "$nonFlakeDir/README.md";
|
||||
flake = false;
|
||||
};
|
||||
nonFlakeFile3 = {
|
||||
url = "$nonFlakeDir?dir=README.md";
|
||||
flake = false;
|
||||
};
|
||||
relativeNonFlakeFile = {
|
||||
url = ./config.nix;
|
||||
flake = false;
|
||||
};
|
||||
};
|
||||
|
||||
description = "Fnord";
|
||||
|
||||
outputs = inputs: rec {
|
||||
inherit inputs;
|
||||
packages.$system.xyzzy = inputs.flake2.packages.$system.bar;
|
||||
packages.$system.sth = inputs.flake1.packages.$system.foo;
|
||||
packages.$system.fnord =
|
||||
@@ -88,6 +97,43 @@ mv "$flake2Dir.tmp" "$flake2Dir"
|
||||
mv "$nonFlakeDir.tmp" "$nonFlakeDir"
|
||||
nix build -o "$TEST_ROOT/result" flake3#xyzzy flake3#fnord
|
||||
|
||||
# Check non-flake inputs have a sourceInfo and an outPath
|
||||
#
|
||||
# This may look redundant, but the other checks below happen in a command
|
||||
# substitution subshell, so failures there will not exit this shell
|
||||
nix eval --raw flake3#inputs.nonFlake.outPath
|
||||
nix eval --raw flake3#inputs.nonFlake.sourceInfo.outPath
|
||||
nix eval --raw flake3#inputs.nonFlakeFile.outPath
|
||||
nix eval --raw flake3#inputs.nonFlakeFile.sourceInfo.outPath
|
||||
nix eval --raw flake3#inputs.nonFlakeFile2.outPath
|
||||
nix eval --raw flake3#inputs.nonFlakeFile2.sourceInfo.outPath
|
||||
nix eval --raw flake3#inputs.nonFlakeFile3.outPath
|
||||
nix eval --raw flake3#inputs.nonFlakeFile3.sourceInfo.outPath
|
||||
nix eval --raw flake3#inputs.relativeNonFlakeFile.outPath
|
||||
nix eval --raw flake3#inputs.relativeNonFlakeFile.sourceInfo.outPath
|
||||
|
||||
# Check non-flake file inputs have the expected outPaths
|
||||
[[
|
||||
$(nix eval --raw flake3#inputs.nonFlake.outPath) \
|
||||
= $(nix eval --raw flake3#inputs.nonFlake.sourceInfo.outPath)
|
||||
]]
|
||||
[[
|
||||
$(nix eval --raw flake3#inputs.nonFlakeFile.outPath) \
|
||||
= $(nix eval --raw flake3#inputs.nonFlakeFile.sourceInfo.outPath)
|
||||
]]
|
||||
[[
|
||||
$(nix eval --raw flake3#inputs.nonFlakeFile2.outPath) \
|
||||
= $(nix eval --raw flake3#inputs.nonFlakeFile2.sourceInfo.outPath)
|
||||
]]
|
||||
[[
|
||||
$(nix eval --raw flake3#inputs.nonFlakeFile3.outPath) \
|
||||
= $(nix eval --raw flake3#inputs.nonFlakeFile3.sourceInfo.outPath)/README.md
|
||||
]]
|
||||
[[
|
||||
$(nix eval --raw flake3#inputs.relativeNonFlakeFile.outPath) \
|
||||
= $(nix eval --raw flake3#inputs.relativeNonFlakeFile.sourceInfo.outPath)/config.nix
|
||||
]]
|
||||
|
||||
# Make branch "removeXyzzy" where flake3 doesn't have xyzzy anymore
|
||||
git -C "$flake3Dir" checkout -b removeXyzzy
|
||||
rm "$flake3Dir/flake.nix"
|
||||
|
||||
@@ -131,3 +131,46 @@ EOF
|
||||
# would fail:
|
||||
nix eval .#ok
|
||||
)
|
||||
|
||||
# https://github.com/NixOS/nix/issues/13164
|
||||
mkdir -p "$TEST_ROOT/issue-13164/nested-flake1/nested-flake2"
|
||||
(
|
||||
cd "$TEST_ROOT/issue-13164"
|
||||
git init
|
||||
git config --global user.email "you@example.com"
|
||||
git config --global user.name "Your Name"
|
||||
cat >flake.nix <<EOF
|
||||
{
|
||||
inputs.nestedFlake1.url = "path:./nested-flake1";
|
||||
outputs = { self, nestedFlake1 }: {
|
||||
inherit nestedFlake1;
|
||||
};
|
||||
}
|
||||
EOF
|
||||
|
||||
cat >nested-flake1/flake.nix <<EOF
|
||||
{
|
||||
inputs.nestedFlake2.url = "path:./nested-flake2";
|
||||
|
||||
outputs = { self, nestedFlake2 }: {
|
||||
name = "nestedFlake1";
|
||||
inherit nestedFlake2;
|
||||
};
|
||||
}
|
||||
EOF
|
||||
|
||||
cat >nested-flake1/nested-flake2/flake.nix <<EOF
|
||||
{
|
||||
outputs = { self }: {
|
||||
name = "nestedFlake2";
|
||||
};
|
||||
}
|
||||
EOF
|
||||
|
||||
git add .
|
||||
git commit -m "Initial commit"
|
||||
|
||||
# I don't understand why two calls are necessary to reproduce the issue.
|
||||
nix eval --json .#nestedFlake1.nestedFlake2 --no-eval-cache
|
||||
nix eval --json .#nestedFlake1.nestedFlake2 --no-eval-cache
|
||||
)
|
||||
@@ -12,6 +12,10 @@ cat > "$repo/flake.nix" <<EOF
|
||||
{
|
||||
outputs = { ... }: {
|
||||
x = 1;
|
||||
y = assert false; 1;
|
||||
z = builtins.readFile ./foo;
|
||||
a = import ./foo;
|
||||
b = import ./dir;
|
||||
};
|
||||
}
|
||||
EOF
|
||||
@@ -21,3 +25,33 @@ expectStderr 1 nix eval "$repo#x" | grepQuiet "error: Path 'flake.nix' in the re
|
||||
git -C "$repo" add flake.nix
|
||||
|
||||
[[ $(nix eval "$repo#x") = 1 ]]
|
||||
|
||||
expectStderr 1 nix eval "$repo#y" | grepQuiet "at $repo/flake.nix:"
|
||||
|
||||
git -C "$repo" commit -a -m foo
|
||||
|
||||
expectStderr 1 nix eval "git+file://$repo?ref=master#y" | grepQuiet "at «git+file://$repo?ref=master&rev=.*»/flake.nix:"
|
||||
|
||||
expectStderr 1 nix eval "$repo#z" | grepQuiet "error: Path 'foo' does not exist in Git repository \"$repo\"."
|
||||
expectStderr 1 nix eval "git+file://$repo?ref=master#z" | grepQuiet "error: '«git+file://$repo?ref=master&rev=.*»/foo' does not exist"
|
||||
expectStderr 1 nix eval "$repo#a" | grepQuiet "error: Path 'foo' does not exist in Git repository \"$repo\"."
|
||||
|
||||
echo 123 > "$repo/foo"
|
||||
|
||||
expectStderr 1 nix eval "$repo#z" | grepQuiet "error: Path 'foo' in the repository \"$repo\" is not tracked by Git."
|
||||
expectStderr 1 nix eval "$repo#a" | grepQuiet "error: Path 'foo' in the repository \"$repo\" is not tracked by Git."
|
||||
|
||||
git -C "$repo" add "$repo/foo"
|
||||
|
||||
[[ $(nix eval --raw "$repo#z") = 123 ]]
|
||||
|
||||
expectStderr 1 nix eval "$repo#b" | grepQuiet "error: Path 'dir' does not exist in Git repository \"$repo\"."
|
||||
|
||||
mkdir -p "$repo/dir"
|
||||
echo 456 > "$repo/dir/default.nix"
|
||||
|
||||
expectStderr 1 nix eval "$repo#b" | grepQuiet "error: Path 'dir' in the repository \"$repo\" is not tracked by Git."
|
||||
|
||||
git -C "$repo" add "$repo/dir/default.nix"
|
||||
|
||||
[[ $(nix eval "$repo#b") = 456 ]]
|
||||
@@ -36,6 +36,7 @@ expectStderr 1 nix flake lock "$flake2Dir" --override-input flake1 "$TEST_ROOT/f
|
||||
grepQuiet "Will not write lock file.*because it has an unlocked input"
|
||||
|
||||
nix flake lock "$flake2Dir" --override-input flake1 "$TEST_ROOT/flake1" --allow-dirty-locks
|
||||
_NIX_TEST_FAIL_ON_LARGE_PATH=1 nix flake lock "$flake2Dir" --override-input flake1 "$TEST_ROOT/flake1" --allow-dirty-locks --warn-large-path-threshold 1 --lazy-trees
|
||||
|
||||
# Using a lock file with a dirty lock does not require --allow-dirty-locks, but should print a warning.
|
||||
expectStderr 0 nix eval "$flake2Dir#x" |
|
||||
|
||||
@@ -10,4 +10,4 @@ error:
|
||||
|
||||
… while calling the 'hashFile' builtin
|
||||
|
||||
error: opening file '/pwd/lang/this-file-is-definitely-not-there-7392097': No such file or directory
|
||||
error: path '/pwd/lang/this-file-is-definitely-not-there-7392097' does not exist
|
||||
@@ -94,13 +94,6 @@ let
|
||||
);
|
||||
};
|
||||
|
||||
otherNixes.nix_2_18.setNixPackage =
|
||||
{ lib, pkgs, ... }:
|
||||
{
|
||||
imports = [ checkOverrideNixVersion ];
|
||||
nix.package = lib.mkForce pkgs.nixVersions.nix_2_18;
|
||||
};
|
||||
|
||||
in
|
||||
|
||||
{
|
||||
|
||||
@@ -205,7 +205,7 @@ in
|
||||
cat_log()
|
||||
|
||||
# ... otherwise it should use the API
|
||||
out = client.succeed("nix flake metadata private-flake --json --access-tokens github.com=ghp_000000000000000000000000000000000000 --tarball-ttl 0")
|
||||
out = client.succeed("nix flake metadata private-flake --json --access-tokens github.com=ghp_000000000000000000000000000000000000 --tarball-ttl 0 --no-trust-tarballs-from-git-forges")
|
||||
print(out)
|
||||
info = json.loads(out)
|
||||
assert info["revision"] == "${private-flake-rev}", f"revision mismatch: {info['revision']} != ${private-flake-rev}"
|
||||
@@ -224,6 +224,10 @@ in
|
||||
hash = client.succeed(f"nix eval --no-trust-tarballs-from-git-forges --raw --expr '(fetchTree {info['url']}).narHash'")
|
||||
assert hash == info['locked']['narHash']
|
||||
|
||||
# Fetching with an incorrect NAR hash should fail.
|
||||
out = client.fail(f"nix eval --no-trust-tarballs-from-git-forges --raw --expr '(fetchTree \"github:fancy-enterprise/private-flake/{info['revision']}?narHash=sha256-HsrRFZYg69qaVe/wDyWBYLeS6ca7ACEJg2Z%2BGpEFw4A%3D\").narHash' 2>&1")
|
||||
assert "NAR hash mismatch in input" in out, "NAR hash check did not fail with the expected error"
|
||||
|
||||
# Fetching without a narHash should succeed if trust-github is set and fail otherwise.
|
||||
client.succeed(f"nix eval --raw --expr 'builtins.fetchTree github:github:fancy-enterprise/private-flake/{info['revision']}'")
|
||||
out = client.fail(f"nix eval --no-trust-tarballs-from-git-forges --raw --expr 'builtins.fetchTree github:github:fancy-enterprise/private-flake/{info['revision']}' 2>&1")
|
||||
|
||||
@@ -61,12 +61,10 @@ in
|
||||
"${pkgs.openssh}/bin/ssh-keygen", "-t", "ed25519", "-f", "key", "-N", ""
|
||||
], capture_output=True, check=True)
|
||||
|
||||
client.succeed("mkdir -m 700 /root/.ssh")
|
||||
client.copy_from_host("key", "/root/.ssh/id_ed25519")
|
||||
client.succeed("chmod 600 /root/.ssh/id_ed25519")
|
||||
|
||||
# Install the SSH key on the server.
|
||||
server.succeed("mkdir -m 700 /root/.ssh")
|
||||
server.copy_from_host("key.pub", "/root/.ssh/authorized_keys")
|
||||
server.wait_for_unit("sshd")
|
||||
server.wait_for_unit("multi-user.target")
|
||||
|
||||
Reference in new issue
Block a user