Compare commits
62
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
3f53a435cb | ||
|
|
0a632cbc3a | ||
|
|
8c2021989e | ||
|
|
92344a31fa | ||
|
|
ee64ffcd75 | ||
|
|
8012b584c3 | ||
|
|
609a4e999f | ||
|
|
6fd36553b9 | ||
|
|
ce28cb32e9 | ||
|
|
3d18d73003 | ||
|
|
8089af3bb0 | ||
|
|
b49ea8b246 | ||
|
|
af821ba647 | ||
|
|
1b1c949d0c | ||
|
|
fb70a45b9e | ||
|
|
53e942bfcc | ||
|
|
4f5172ba21 | ||
|
|
1bddbff3a6 | ||
|
|
4991defc44 | ||
|
|
8f829e478f | ||
|
|
7ac5a61208 | ||
|
|
2af5792cc2 | ||
|
|
e251ffdd46 | ||
|
|
a8087ebf52 | ||
|
|
66fefcd795 | ||
|
|
1ea3a841bb | ||
|
|
fd0bcd97e8 | ||
|
|
b19bfc6373 | ||
|
|
3b028edbf7 | ||
|
|
23a7178eb4 | ||
|
|
252aff5c8f | ||
|
|
920c5ceb0c | ||
|
|
de76cb681d | ||
|
|
d19bfb0045 | ||
|
|
160822858a | ||
|
|
25998fcd1e | ||
|
|
7ba09399cc | ||
|
|
f8a92564f7 | ||
|
|
2ae4121c1d | ||
|
|
38c755f168 | ||
|
|
08887caa1a | ||
|
|
8d588ad471 | ||
|
|
6970efe2e1 | ||
|
|
3b95b7c9aa | ||
|
|
4289e2f9e6 | ||
|
|
ccdd1f1c65 | ||
|
|
b7fd471f84 | ||
|
|
4ab2cdacfc | ||
|
|
1807dc78d6 | ||
|
|
cf5644ce99 | ||
|
|
47ccf201fb | ||
|
|
ec65132ab9 | ||
|
|
bde0b22dc8 | ||
|
|
a8e8614fd3 | ||
|
|
6a32b754a9 | ||
|
|
43a472631b | ||
|
|
eff403b5ab | ||
|
|
6e6dda9f67 | ||
|
|
ee1383f75f | ||
|
|
5be07abf6d | ||
|
|
469212bd38 | ||
|
|
4a1a562461 |
No files matched your search
Generated
+5
-5
@@ -3,15 +3,15 @@
|
||||
"flake-compat": {
|
||||
"flake": false,
|
||||
"locked": {
|
||||
"lastModified": 1733328505,
|
||||
"narHash": "sha256-NeCCThCEP3eCl2l/+27kNNK7QrwZB1IJCrXfrbv5oqU=",
|
||||
"owner": "edolstra",
|
||||
"lastModified": 1767039857,
|
||||
"narHash": "sha256-vNpUSpF5Nuw8xvDLj2KCwwksIbjua2LZCqhV1LNRDns=",
|
||||
"owner": "NixOS",
|
||||
"repo": "flake-compat",
|
||||
"rev": "ff81ac966bb2cae68946d5ed5fc4994f96d0ffec",
|
||||
"rev": "5edf11c44bc78a0d334f6334cdaf7d60d732daab",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "edolstra",
|
||||
"owner": "NixOS",
|
||||
"repo": "flake-compat",
|
||||
"type": "github"
|
||||
}
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2";
|
||||
inputs.nixpkgs-23-11.url = "github:NixOS/nixpkgs/a62e6edd6d5e1fa0329b8653c801147986f8d446";
|
||||
inputs.flake-compat = {
|
||||
url = "github:edolstra/flake-compat";
|
||||
url = "github:NixOS/flake-compat";
|
||||
flake = false;
|
||||
};
|
||||
|
||||
|
||||
@@ -157,7 +157,7 @@ MixFlakeOptions::MixFlakeOptions()
|
||||
.category = category,
|
||||
.labels = {"flake-lock-path"},
|
||||
.handler = {[&](std::string lockFilePath) {
|
||||
lockFlags.referenceLockFilePath = {makeFSSourceAccessor(absPath(lockFilePath)), CanonPath::root};
|
||||
lockFlags.referenceLockFilePath = {getFSSourceAccessor(), CanonPath(absPath(lockFilePath))};
|
||||
}},
|
||||
.completer = completePath,
|
||||
});
|
||||
|
||||
@@ -213,24 +213,26 @@ struct MercurialInputScheme : InputScheme
|
||||
runHg({"status", "-R", actualUrl, "--clean", "--modified", "--added", "--no-status", "--print0"}),
|
||||
"\0"s);
|
||||
|
||||
auto accessor = makeFSSourceAccessor(absPath(actualUrl));
|
||||
std::filesystem::path actualPath(absPath(actualUrl));
|
||||
|
||||
PathFilter filter = [&](const Path & p) -> bool {
|
||||
auto cp = CanonPath(p);
|
||||
auto st = accessor->lstat(cp);
|
||||
assert(hasPrefix(p, actualPath.string()));
|
||||
std::string file(p, actualPath.string().size() + 1);
|
||||
|
||||
if (st.type == SourceAccessor::tDirectory) {
|
||||
auto prefix = cp.rel() + "/";
|
||||
auto st = lstat(p);
|
||||
|
||||
if (S_ISDIR(st.st_mode)) {
|
||||
auto prefix = file + "/";
|
||||
auto i = files.lower_bound(prefix);
|
||||
return i != files.end() && hasPrefix(*i, prefix);
|
||||
}
|
||||
|
||||
return files.count(cp.rel());
|
||||
return files.count(file);
|
||||
};
|
||||
|
||||
auto storePath = store.addToStore(
|
||||
input.getName(),
|
||||
{accessor, CanonPath::root},
|
||||
{getFSSourceAccessor(), CanonPath(actualPath.string())},
|
||||
ContentAddressMethod::Raw::NixArchive,
|
||||
HashAlgorithm::SHA256,
|
||||
{},
|
||||
@@ -332,7 +334,7 @@ struct MercurialInputScheme : InputScheme
|
||||
|
||||
deletePath(tmpDir / ".hg_archival.txt");
|
||||
|
||||
auto storePath = store.addToStore(name, {makeFSSourceAccessor(tmpDir), CanonPath::root});
|
||||
auto storePath = store.addToStore(name, {getFSSourceAccessor(), CanonPath(tmpDir.string())});
|
||||
|
||||
Attrs infoAttrs({
|
||||
{"revCount", (uint64_t) revCount},
|
||||
|
||||
@@ -114,6 +114,12 @@ static DownloadTarballResult downloadTarball_(
|
||||
// it is not in fact a tarball.
|
||||
if (url.scheme == "file") {
|
||||
std::filesystem::path localPath = renderUrlPathEnsureLegal(url.path);
|
||||
if (!localPath.is_absolute()) {
|
||||
throw Error(
|
||||
"tarball '%s' must use an absolute path. "
|
||||
"The 'file' scheme does not support relative paths.",
|
||||
url);
|
||||
}
|
||||
if (!exists(localPath)) {
|
||||
throw Error("tarball '%s' does not exist.", localPath);
|
||||
}
|
||||
|
||||
@@ -377,6 +377,7 @@ TEST_F(nix_api_store_test, nix_api_load_flake_with_flags)
|
||||
assert_ctx_ok();
|
||||
ASSERT_EQ("Claire", helloStr);
|
||||
|
||||
nix_locked_flake_free(lockedFlake);
|
||||
nix_flake_reference_parse_flags_free(parseFlags);
|
||||
nix_flake_lock_flags_free(lockFlags);
|
||||
nix_flake_reference_free(flakeReference);
|
||||
|
||||
@@ -13,6 +13,7 @@
|
||||
#include "nix/util/callback.hh"
|
||||
#include "nix/util/signals.hh"
|
||||
#include "nix/util/archive.hh"
|
||||
#include "nix/store/nar-cache.hh"
|
||||
|
||||
#include <chrono>
|
||||
#include <future>
|
||||
@@ -26,6 +27,7 @@ namespace nix {
|
||||
|
||||
BinaryCacheStore::BinaryCacheStore(Config & config)
|
||||
: config{config}
|
||||
, narCache{config.localNarCache.get() ? std::make_shared<NarCache>(*config.localNarCache.get()) : nullptr}
|
||||
{
|
||||
if (config.secretKeyFile != "")
|
||||
signers.push_back(std::make_unique<LocalSigner>(SecretKey{readFile(config.secretKeyFile)}));
|
||||
@@ -147,7 +149,7 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
|
||||
write the compressed NAR to disk), into a HashSink (to get the
|
||||
NAR hash), and into a NarAccessor (to get the NAR listing). */
|
||||
HashSink fileHashSink{HashAlgorithm::SHA256};
|
||||
std::shared_ptr<SourceAccessor> narAccessor;
|
||||
std::shared_ptr<NarAccessor> narAccessor;
|
||||
HashSink narHashSink{HashAlgorithm::SHA256};
|
||||
{
|
||||
FdSink fileSink(fdTemp.get());
|
||||
@@ -205,7 +207,7 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
|
||||
if (config.writeNARListing) {
|
||||
nlohmann::json j = {
|
||||
{"version", 1},
|
||||
{"root", listNarDeep(*narAccessor, CanonPath::root)},
|
||||
{"root", narAccessor->getListing()},
|
||||
};
|
||||
|
||||
upsertFile(std::string(info.path.hashPart()) + ".ls", j.dump(), "application/json");
|
||||
@@ -552,7 +554,7 @@ void BinaryCacheStore::registerDrvOutput(const Realisation & info)
|
||||
|
||||
ref<RemoteFSAccessor> BinaryCacheStore::getRemoteFSAccessor(bool requireValidPath)
|
||||
{
|
||||
return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this()), requireValidPath, config.localNarCache);
|
||||
return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this()), requireValidPath, narCache);
|
||||
}
|
||||
|
||||
ref<SourceAccessor> BinaryCacheStore::getFSAccessor(bool requireValidPath)
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
#include "nix/store/build/build-log.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
BuildLog::BuildLog(size_t maxTailLines, std::unique_ptr<Activity> act)
|
||||
: maxTailLines(maxTailLines)
|
||||
, act(std::move(act))
|
||||
{
|
||||
}
|
||||
|
||||
void BuildLog::operator()(std::string_view data)
|
||||
{
|
||||
for (auto c : data)
|
||||
if (c == '\r')
|
||||
currentLogLinePos = 0;
|
||||
else if (c == '\n')
|
||||
flushLine();
|
||||
else {
|
||||
if (currentLogLinePos >= currentLogLine.size())
|
||||
currentLogLine.resize(currentLogLinePos + 1);
|
||||
currentLogLine[currentLogLinePos++] = c;
|
||||
}
|
||||
}
|
||||
|
||||
void BuildLog::flush()
|
||||
{
|
||||
if (!currentLogLine.empty())
|
||||
flushLine();
|
||||
}
|
||||
|
||||
void BuildLog::flushLine()
|
||||
{
|
||||
// Truncate to actual content (currentLogLinePos may be less than size due to \r)
|
||||
currentLogLine.resize(currentLogLinePos);
|
||||
|
||||
if (!handleJSONLogMessage(currentLogLine, *act, builderActivities, "the derivation builder", false)) {
|
||||
// Line was not handled as JSON, emit and add to tail
|
||||
act->result(resBuildLogLine, currentLogLine);
|
||||
logTail.push_back(currentLogLine);
|
||||
if (logTail.size() > maxTailLines)
|
||||
logTail.pop_front();
|
||||
}
|
||||
|
||||
currentLogLine.clear();
|
||||
currentLogLinePos = 0;
|
||||
}
|
||||
|
||||
} // namespace nix
|
||||
@@ -40,37 +40,13 @@ DerivationBuildingGoal::DerivationBuildingGoal(
|
||||
worker.store.addTempRoot(this->drvPath);
|
||||
}
|
||||
|
||||
DerivationBuildingGoal::~DerivationBuildingGoal()
|
||||
{
|
||||
/* Careful: we should never ever throw an exception from a
|
||||
destructor. */
|
||||
#ifndef _WIN32 // TODO enable `DerivationBuilder` on Windows
|
||||
if (builder)
|
||||
builder.reset();
|
||||
#endif
|
||||
try {
|
||||
closeLogFile();
|
||||
} catch (...) {
|
||||
ignoreExceptionInDestructor();
|
||||
}
|
||||
}
|
||||
DerivationBuildingGoal::~DerivationBuildingGoal() = default;
|
||||
|
||||
std::string DerivationBuildingGoal::key()
|
||||
{
|
||||
return "dd$" + std::string(drvPath.name()) + "$" + worker.store.printStorePath(drvPath);
|
||||
}
|
||||
|
||||
void DerivationBuildingGoal::killChild()
|
||||
{
|
||||
#ifndef _WIN32 // TODO enable build hook on Windows
|
||||
hook.reset();
|
||||
#endif
|
||||
#ifndef _WIN32 // TODO enable `DerivationBuilder` on Windows
|
||||
if (builder && builder->killChild())
|
||||
worker.childTerminated(this);
|
||||
#endif
|
||||
}
|
||||
|
||||
std::string showKnownOutputs(const StoreDirConfig & store, const Derivation & drv)
|
||||
{
|
||||
std::string msg;
|
||||
@@ -146,6 +122,8 @@ Goal::Co DerivationBuildingGoal::gaveUpOnSubstitution(bool storeDerivation)
|
||||
writeDerivation(worker.store, *drv);
|
||||
}
|
||||
|
||||
StorePathSet inputPaths;
|
||||
|
||||
{
|
||||
/* If we get this far, we know no dynamic drvs inputs */
|
||||
|
||||
@@ -185,10 +163,23 @@ Goal::Co DerivationBuildingGoal::gaveUpOnSubstitution(bool storeDerivation)
|
||||
slot to become available, since we don't need one if there is a
|
||||
build hook. */
|
||||
co_await yield();
|
||||
co_return tryToBuild();
|
||||
co_return tryToBuild(std::move(inputPaths));
|
||||
}
|
||||
|
||||
Goal::Co DerivationBuildingGoal::tryToBuild()
|
||||
/**
|
||||
* RAII wrapper for build log file.
|
||||
* Constructor opens the log file, destructor closes it.
|
||||
*/
|
||||
struct LogFile
|
||||
{
|
||||
AutoCloseFD fd;
|
||||
std::shared_ptr<BufferedSink> fileSink, sink;
|
||||
|
||||
LogFile(Store & store, const StorePath & drvPath);
|
||||
~LogFile();
|
||||
};
|
||||
|
||||
Goal::Co DerivationBuildingGoal::tryToBuild(StorePathSet inputPaths)
|
||||
{
|
||||
auto drvOptions = [&] {
|
||||
DerivationOptions<SingleDerivedPath> temp;
|
||||
@@ -249,33 +240,6 @@ Goal::Co DerivationBuildingGoal::tryToBuild()
|
||||
}
|
||||
checkPathValidity(initialOutputs);
|
||||
|
||||
auto started = [&]() {
|
||||
auto msg =
|
||||
fmt(buildMode == bmRepair ? "repairing outputs of '%s'"
|
||||
: buildMode == bmCheck ? "checking outputs of '%s'"
|
||||
: "building '%s'",
|
||||
worker.store.printStorePath(drvPath));
|
||||
#ifndef _WIN32 // TODO enable build hook on Windows
|
||||
if (hook)
|
||||
msg += fmt(" on '%s'", hook->machineName);
|
||||
#endif
|
||||
act = std::make_unique<Activity>(
|
||||
*logger,
|
||||
lvlInfo,
|
||||
actBuild,
|
||||
msg,
|
||||
Logger::Fields{
|
||||
worker.store.printStorePath(drvPath),
|
||||
#ifndef _WIN32 // TODO enable build hook on Windows
|
||||
hook ? hook->machineName :
|
||||
#endif
|
||||
"",
|
||||
1,
|
||||
1});
|
||||
mcRunningBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.runningBuilds);
|
||||
worker.updateProgress();
|
||||
};
|
||||
|
||||
/**
|
||||
* Activity that denotes waiting for a lock.
|
||||
*/
|
||||
@@ -366,7 +330,7 @@ Goal::Co DerivationBuildingGoal::tryToBuild()
|
||||
if (buildLocally) {
|
||||
useHook = false;
|
||||
} else {
|
||||
switch (tryBuildHook(initialOutputs, drvOptions)) {
|
||||
switch (tryBuildHook(drvOptions)) {
|
||||
case rpAccept:
|
||||
/* Yes, it has started doing so. Wait until we get
|
||||
EOF from the hook. */
|
||||
@@ -416,9 +380,6 @@ Goal::Co DerivationBuildingGoal::tryToBuild()
|
||||
Magenta(
|
||||
"/usr/sbin/softwareupdate --install-rosetta && launchctl stop org.nixos.nix-daemon"));
|
||||
|
||||
#ifndef _WIN32 // TODO enable `DerivationBuilder` on Windows
|
||||
builder.reset();
|
||||
#endif
|
||||
outputLocks.unlock();
|
||||
worker.permanentFailure = true;
|
||||
co_return doneFailure({BuildResult::Failure::InputRejected, std::move(msg)});
|
||||
@@ -433,103 +394,229 @@ Goal::Co DerivationBuildingGoal::tryToBuild()
|
||||
actLock.reset();
|
||||
|
||||
if (useHook) {
|
||||
buildResult.startTime = time(0); // inexact
|
||||
started();
|
||||
co_return buildWithHook(
|
||||
std::move(inputPaths), std::move(initialOutputs), std::move(drvOptions), std::move(outputLocks));
|
||||
} else {
|
||||
co_return buildLocally(
|
||||
std::move(inputPaths),
|
||||
std::move(initialOutputs),
|
||||
std::move(drvOptions),
|
||||
std::move(outputLocks),
|
||||
externalBuilder);
|
||||
}
|
||||
}
|
||||
|
||||
while (true) {
|
||||
auto event = co_await WaitForChildEvent{};
|
||||
if (auto * output = std::get_if<ChildOutput>(&event)) {
|
||||
co_await processChildOutput(output->fd, output->data);
|
||||
} else if (std::get_if<ChildEOF>(&event)) {
|
||||
if (!currentLogLine.empty())
|
||||
flushLine();
|
||||
break;
|
||||
} else if (auto * timeout = std::get_if<TimedOut>(&event)) {
|
||||
killChild();
|
||||
co_return doneFailure(std::move(*timeout));
|
||||
}
|
||||
}
|
||||
|
||||
#ifndef _WIN32
|
||||
assert(hook);
|
||||
#endif
|
||||
|
||||
trace("hook build done");
|
||||
|
||||
/* Since we got an EOF on the logger pipe, the builder is presumed
|
||||
to have terminated. In fact, the builder could also have
|
||||
simply have closed its end of the pipe, so just to be sure,
|
||||
kill it. */
|
||||
int status =
|
||||
#ifndef _WIN32 // TODO enable build hook on Windows
|
||||
hook->pid.kill();
|
||||
Goal::Co DerivationBuildingGoal::buildWithHook(
|
||||
StorePathSet inputPaths,
|
||||
std::map<std::string, InitialOutput> initialOutputs,
|
||||
DerivationOptions<StorePath> drvOptions,
|
||||
PathLocks outputLocks)
|
||||
{
|
||||
#ifdef _WIN32 // TODO enable build hook on Windows
|
||||
unreachable();
|
||||
#else
|
||||
0;
|
||||
#endif
|
||||
std::unique_ptr<HookInstance> hook = std::move(worker.hook);
|
||||
|
||||
debug("build hook for '%s' finished", worker.store.printStorePath(drvPath));
|
||||
/* Set up callback so childTerminated is called if the hook is
|
||||
destroyed (e.g., during failure cascades). */
|
||||
hook->onKillChild = [this]() { worker.childTerminated(this, JobCategory::Build); };
|
||||
|
||||
buildResult.timesBuilt++;
|
||||
buildResult.stopTime = time(0);
|
||||
|
||||
/* So the child is gone now. */
|
||||
worker.childTerminated(this);
|
||||
|
||||
/* Close the read side of the logger pipe. */
|
||||
#ifndef _WIN32 // TODO enable build hook on Windows
|
||||
hook->builderOut.readSide.close();
|
||||
hook->fromHook.readSide.close();
|
||||
#endif
|
||||
|
||||
/* Close the log file. */
|
||||
closeLogFile();
|
||||
|
||||
/* Check the exit status. */
|
||||
if (!statusOk(status)) {
|
||||
auto e = fixupBuilderFailureErrorMessage({BuildResult::Failure::MiscFailure, status, ""});
|
||||
|
||||
outputLocks.unlock();
|
||||
|
||||
/* TODO (once again) support fine-grained error codes, see issue #12641. */
|
||||
|
||||
co_return doneFailure(std::move(e));
|
||||
}
|
||||
|
||||
/* Compute the FS closure of the outputs and register them as
|
||||
being valid. */
|
||||
auto builtOutputs =
|
||||
/* When using a build hook, the build hook can register the output
|
||||
as valid (by doing `nix-store --import'). If so we don't have
|
||||
to do anything here.
|
||||
|
||||
We can only early return when the outputs are known a priori. For
|
||||
floating content-addressing derivations this isn't the case.
|
||||
|
||||
Aborts if any output is not valid or corrupt, and otherwise
|
||||
returns a 'SingleDrvOutputs' structure containing all outputs.
|
||||
*/
|
||||
[&] {
|
||||
auto [allValid, validOutputs] = checkPathValidity(initialOutputs);
|
||||
if (!allValid)
|
||||
throw Error("some outputs are unexpectedly invalid");
|
||||
return validOutputs;
|
||||
}();
|
||||
|
||||
StorePathSet outputPaths;
|
||||
for (auto & [_, output] : builtOutputs)
|
||||
outputPaths.insert(output.outPath);
|
||||
runPostBuildHook(worker.store, *logger, drvPath, outputPaths);
|
||||
|
||||
/* It is now safe to delete the lock files, since all future
|
||||
lockers will see that the output paths are valid; they will
|
||||
not create new lock files with the same names as the old
|
||||
(unlinked) lock files. */
|
||||
outputLocks.setDeletion(true);
|
||||
outputLocks.unlock();
|
||||
|
||||
co_return doneSuccess(BuildResult::Success::Built, std::move(builtOutputs));
|
||||
try {
|
||||
hook->machineName = readLine(hook->fromHook.readSide.get());
|
||||
} catch (Error & e) {
|
||||
e.addTrace({}, "while reading the machine name from the build hook");
|
||||
throw;
|
||||
}
|
||||
|
||||
CommonProto::WriteConn conn{hook->sink};
|
||||
|
||||
/* Tell the hook all the inputs that have to be copied to the
|
||||
remote system. */
|
||||
CommonProto::write(worker.store, conn, inputPaths);
|
||||
|
||||
/* Tell the hooks the missing outputs that have to be copied back
|
||||
from the remote system. */
|
||||
{
|
||||
StringSet missingOutputs;
|
||||
for (auto & [outputName, status] : initialOutputs) {
|
||||
// XXX: Does this include known CA outputs?
|
||||
if (buildMode != bmCheck && status.known && status.known->isValid())
|
||||
continue;
|
||||
missingOutputs.insert(outputName);
|
||||
}
|
||||
CommonProto::write(worker.store, conn, missingOutputs);
|
||||
}
|
||||
|
||||
hook->sink = FdSink();
|
||||
hook->toHook.writeSide.close();
|
||||
|
||||
/* Create the log file and pipe. */
|
||||
std::unique_ptr<LogFile> logFile = std::make_unique<LogFile>(worker.store, drvPath);
|
||||
|
||||
std::set<MuxablePipePollState::CommChannel> fds;
|
||||
fds.insert(hook->fromHook.readSide.get());
|
||||
fds.insert(hook->builderOut.readSide.get());
|
||||
worker.childStarted(shared_from_this(), fds, false, false);
|
||||
|
||||
buildResult.startTime = time(0); // inexact
|
||||
|
||||
auto msg =
|
||||
fmt(buildMode == bmRepair ? "repairing outputs of '%s'"
|
||||
: buildMode == bmCheck ? "checking outputs of '%s'"
|
||||
: "building '%s'",
|
||||
worker.store.printStorePath(drvPath));
|
||||
msg += fmt(" on '%s'", hook->machineName);
|
||||
|
||||
std::unique_ptr<BuildLog> buildLog = std::make_unique<BuildLog>(
|
||||
settings.logLines,
|
||||
std::make_unique<Activity>(
|
||||
*logger,
|
||||
lvlInfo,
|
||||
actBuild,
|
||||
msg,
|
||||
Logger::Fields{worker.store.printStorePath(drvPath), hook->machineName, 1, 1}));
|
||||
mcRunningBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.runningBuilds);
|
||||
worker.updateProgress();
|
||||
|
||||
std::string currentHookLine;
|
||||
uint64_t logSize = 0;
|
||||
|
||||
while (true) {
|
||||
auto event = co_await WaitForChildEvent{};
|
||||
if (auto * output = std::get_if<ChildOutput>(&event)) {
|
||||
auto & fd = output->fd;
|
||||
auto & data = output->data;
|
||||
if (fd == hook->builderOut.readSide.get()) {
|
||||
logSize += data.size();
|
||||
if (settings.maxLogSize && logSize > settings.maxLogSize) {
|
||||
hook.reset();
|
||||
co_return doneFailureLogTooLong(*buildLog);
|
||||
}
|
||||
(*buildLog)(data);
|
||||
if (logFile->sink)
|
||||
(*logFile->sink)(data);
|
||||
} else if (fd == hook->fromHook.readSide.get()) {
|
||||
for (auto c : data)
|
||||
if (c == '\n') {
|
||||
auto json = parseJSONMessage(currentHookLine, "the derivation builder");
|
||||
if (json) {
|
||||
auto s = handleJSONLogMessage(
|
||||
*json, worker.act, hook->activities, "the derivation builder", true);
|
||||
// ensure that logs from a builder using `ssh-ng://` as protocol
|
||||
// are also available to `nix log`.
|
||||
if (s && logFile->sink) {
|
||||
const auto type = (*json)["type"];
|
||||
const auto fields = (*json)["fields"];
|
||||
if (type == resBuildLogLine) {
|
||||
(*logFile->sink)((fields.size() > 0 ? fields[0].get<std::string>() : "") + "\n");
|
||||
} else if (type == resSetPhase && !fields.is_null()) {
|
||||
const auto phase = fields[0];
|
||||
if (!phase.is_null()) {
|
||||
// nixpkgs' stdenv produces lines in the log to signal
|
||||
// phase changes.
|
||||
// We want to get the same lines in case of remote builds.
|
||||
// The format is:
|
||||
// @nix { "action": "setPhase", "phase": "$curPhase" }
|
||||
const auto logLine =
|
||||
nlohmann::json::object({{"action", "setPhase"}, {"phase", phase}});
|
||||
(*logFile->sink)(
|
||||
"@nix "
|
||||
+ logLine.dump(-1, ' ', false, nlohmann::json::error_handler_t::replace)
|
||||
+ "\n");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
currentHookLine.clear();
|
||||
} else
|
||||
currentHookLine += c;
|
||||
}
|
||||
} else if (std::get_if<ChildEOF>(&event)) {
|
||||
buildLog->flush();
|
||||
break;
|
||||
} else if (auto * timeout = std::get_if<TimedOut>(&event)) {
|
||||
hook.reset();
|
||||
co_return doneFailure(std::move(*timeout));
|
||||
}
|
||||
}
|
||||
|
||||
trace("hook build done");
|
||||
|
||||
/* Since we got an EOF on the logger pipe, the builder is presumed
|
||||
to have terminated. In fact, the builder could also have
|
||||
simply have closed its end of the pipe, so just to be sure,
|
||||
kill it. */
|
||||
int status = hook->pid.kill();
|
||||
|
||||
debug("build hook for '%s' finished", worker.store.printStorePath(drvPath));
|
||||
|
||||
buildResult.timesBuilt++;
|
||||
buildResult.stopTime = time(0);
|
||||
|
||||
/* So the child is gone now. */
|
||||
worker.childTerminated(this);
|
||||
|
||||
/* Close the read side of the logger pipe. */
|
||||
hook->builderOut.readSide.close();
|
||||
hook->fromHook.readSide.close();
|
||||
|
||||
/* Close the log file. */
|
||||
logFile.reset();
|
||||
|
||||
/* Check the exit status. */
|
||||
if (!statusOk(status)) {
|
||||
auto e = fixupBuilderFailureErrorMessage({BuildResult::Failure::MiscFailure, status, ""}, *buildLog);
|
||||
|
||||
outputLocks.unlock();
|
||||
|
||||
/* TODO (once again) support fine-grained error codes, see issue #12641. */
|
||||
|
||||
co_return doneFailure(std::move(e));
|
||||
}
|
||||
|
||||
/* Compute the FS closure of the outputs and register them as
|
||||
being valid. */
|
||||
auto builtOutputs =
|
||||
/* When using a build hook, the build hook can register the output
|
||||
as valid (by doing `nix-store --import'). If so we don't have
|
||||
to do anything here.
|
||||
|
||||
We can only early return when the outputs are known a priori. For
|
||||
floating content-addressing derivations this isn't the case.
|
||||
|
||||
Aborts if any output is not valid or corrupt, and otherwise
|
||||
returns a 'SingleDrvOutputs' structure containing all outputs.
|
||||
*/
|
||||
[&] {
|
||||
auto [allValid, validOutputs] = checkPathValidity(initialOutputs);
|
||||
if (!allValid)
|
||||
throw Error("some outputs are unexpectedly invalid");
|
||||
return validOutputs;
|
||||
}();
|
||||
|
||||
StorePathSet outputPaths;
|
||||
for (auto & [_, output] : builtOutputs)
|
||||
outputPaths.insert(output.outPath);
|
||||
runPostBuildHook(worker.store, *logger, drvPath, outputPaths);
|
||||
|
||||
/* It is now safe to delete the lock files, since all future
|
||||
lockers will see that the output paths are valid; they will
|
||||
not create new lock files with the same names as the old
|
||||
(unlinked) lock files. */
|
||||
outputLocks.setDeletion(true);
|
||||
outputLocks.unlock();
|
||||
|
||||
co_return doneSuccess(BuildResult::Success::Built, std::move(builtOutputs));
|
||||
#endif
|
||||
}
|
||||
|
||||
Goal::Co DerivationBuildingGoal::buildLocally(
|
||||
StorePathSet inputPaths,
|
||||
std::map<std::string, InitialOutput> initialOutputs,
|
||||
DerivationOptions<StorePath> drvOptions,
|
||||
PathLocks outputLocks,
|
||||
const ExternalBuilder * externalBuilder)
|
||||
{
|
||||
co_await yield();
|
||||
|
||||
if (!dynamic_cast<LocalStore *>(&worker.store)) {
|
||||
@@ -545,8 +632,29 @@ Goal::Co DerivationBuildingGoal::tryToBuild()
|
||||
#ifdef _WIN32 // TODO enable `DerivationBuilder` on Windows
|
||||
throw UnimplementedError("building derivations is not yet implemented on Windows");
|
||||
#else
|
||||
assert(!hook);
|
||||
std::unique_ptr<BuildLog> buildLog;
|
||||
std::unique_ptr<LogFile> logFile;
|
||||
|
||||
auto openLogFile = [&]() { logFile = std::make_unique<LogFile>(worker.store, drvPath); };
|
||||
|
||||
auto closeLogFile = [&]() { logFile.reset(); };
|
||||
|
||||
auto started = [&]() {
|
||||
auto msg =
|
||||
fmt(buildMode == bmRepair ? "repairing outputs of '%s'"
|
||||
: buildMode == bmCheck ? "checking outputs of '%s'"
|
||||
: "building '%s'",
|
||||
worker.store.printStorePath(drvPath));
|
||||
buildLog = std::make_unique<BuildLog>(
|
||||
settings.logLines,
|
||||
std::make_unique<Activity>(
|
||||
*logger, lvlInfo, actBuild, msg, Logger::Fields{worker.store.printStorePath(drvPath), "", 1, 1}));
|
||||
mcRunningBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.runningBuilds);
|
||||
worker.updateProgress();
|
||||
};
|
||||
|
||||
std::unique_ptr<Activity> actLock;
|
||||
std::unique_ptr<DerivationBuilder> builder;
|
||||
Descriptor builderOut;
|
||||
|
||||
// Will continue here while waiting for a build user below
|
||||
@@ -556,7 +664,7 @@ Goal::Co DerivationBuildingGoal::tryToBuild()
|
||||
if (curBuilds >= settings.maxBuildJobs) {
|
||||
outputLocks.unlock();
|
||||
co_await waitForBuildSlot();
|
||||
co_return tryToBuild();
|
||||
co_return tryToBuild(std::move(inputPaths));
|
||||
}
|
||||
|
||||
if (!builder) {
|
||||
@@ -567,10 +675,16 @@ Goal::Co DerivationBuildingGoal::tryToBuild()
|
||||
struct DerivationBuildingGoalCallbacks : DerivationBuilderCallbacks
|
||||
{
|
||||
DerivationBuildingGoal & goal;
|
||||
std::function<void()> openLogFileFn;
|
||||
std::function<void()> closeLogFileFn;
|
||||
|
||||
DerivationBuildingGoalCallbacks(
|
||||
DerivationBuildingGoal & goal, std::unique_ptr<DerivationBuilder> & builder)
|
||||
DerivationBuildingGoal & goal,
|
||||
std::function<void()> openLogFileFn,
|
||||
std::function<void()> closeLogFileFn)
|
||||
: goal{goal}
|
||||
, openLogFileFn{std::move(openLogFileFn)}
|
||||
, closeLogFileFn{std::move(closeLogFileFn)}
|
||||
{
|
||||
}
|
||||
|
||||
@@ -578,17 +692,17 @@ Goal::Co DerivationBuildingGoal::tryToBuild()
|
||||
|
||||
void childTerminated() override
|
||||
{
|
||||
goal.worker.childTerminated(&goal);
|
||||
goal.worker.childTerminated(&goal, JobCategory::Build);
|
||||
}
|
||||
|
||||
std::filesystem::path openLogFile() override
|
||||
void openLogFile() override
|
||||
{
|
||||
return goal.openLogFile();
|
||||
openLogFileFn();
|
||||
}
|
||||
|
||||
void closeLogFile() override
|
||||
{
|
||||
goal.closeLogFile();
|
||||
closeLogFileFn();
|
||||
}
|
||||
};
|
||||
|
||||
@@ -638,15 +752,16 @@ Goal::Co DerivationBuildingGoal::tryToBuild()
|
||||
|
||||
/* If we have to wait and retry (see below), then `builder` will
|
||||
already be created, so we don't need to create it again. */
|
||||
builder = externalBuilder ? makeExternalDerivationBuilder(
|
||||
*localStoreP,
|
||||
std::make_unique<DerivationBuildingGoalCallbacks>(*this, builder),
|
||||
std::move(params),
|
||||
*externalBuilder)
|
||||
: makeDerivationBuilder(
|
||||
*localStoreP,
|
||||
std::make_unique<DerivationBuildingGoalCallbacks>(*this, builder),
|
||||
std::move(params));
|
||||
builder = externalBuilder
|
||||
? makeExternalDerivationBuilder(
|
||||
*localStoreP,
|
||||
std::make_unique<DerivationBuildingGoalCallbacks>(*this, openLogFile, closeLogFile),
|
||||
std::move(params),
|
||||
*externalBuilder)
|
||||
: makeDerivationBuilder(
|
||||
*localStoreP,
|
||||
std::make_unique<DerivationBuildingGoalCallbacks>(*this, openLogFile, closeLogFile),
|
||||
std::move(params));
|
||||
}
|
||||
|
||||
if (auto builderOutOpt = builder->startBuild()) {
|
||||
@@ -671,16 +786,26 @@ Goal::Co DerivationBuildingGoal::tryToBuild()
|
||||
|
||||
started();
|
||||
|
||||
uint64_t logSize = 0;
|
||||
|
||||
while (true) {
|
||||
auto event = co_await WaitForChildEvent{};
|
||||
if (auto * output = std::get_if<ChildOutput>(&event)) {
|
||||
co_await processChildOutput(output->fd, output->data);
|
||||
if (output->fd == builder->builderOut.get()) {
|
||||
logSize += output->data.size();
|
||||
if (settings.maxLogSize && logSize > settings.maxLogSize) {
|
||||
builder->killChild();
|
||||
co_return doneFailureLogTooLong(*buildLog);
|
||||
}
|
||||
(*buildLog)(output->data);
|
||||
if (logFile->sink)
|
||||
(*logFile->sink)(output->data);
|
||||
}
|
||||
} else if (std::get_if<ChildEOF>(&event)) {
|
||||
if (!currentLogLine.empty())
|
||||
flushLine();
|
||||
buildLog->flush();
|
||||
break;
|
||||
} else if (auto * timeout = std::get_if<TimedOut>(&event)) {
|
||||
killChild();
|
||||
builder->killChild();
|
||||
co_return doneFailure(std::move(*timeout));
|
||||
}
|
||||
}
|
||||
@@ -693,7 +818,7 @@ Goal::Co DerivationBuildingGoal::tryToBuild()
|
||||
} catch (BuilderFailureError & e) {
|
||||
builder.reset();
|
||||
outputLocks.unlock();
|
||||
co_return doneFailure(fixupBuilderFailureErrorMessage(std::move(e)));
|
||||
co_return doneFailure(fixupBuilderFailureErrorMessage(std::move(e), *buildLog));
|
||||
} catch (BuildError & e) {
|
||||
builder.reset();
|
||||
outputLocks.unlock();
|
||||
@@ -814,7 +939,7 @@ static void runPostBuildHook(
|
||||
});
|
||||
}
|
||||
|
||||
BuildError DerivationBuildingGoal::fixupBuilderFailureErrorMessage(BuilderFailureError e)
|
||||
BuildError DerivationBuildingGoal::fixupBuilderFailureErrorMessage(BuilderFailureError e, BuildLog & buildLog)
|
||||
{
|
||||
auto msg =
|
||||
fmt("Cannot build '%s'.\n"
|
||||
@@ -824,6 +949,7 @@ BuildError DerivationBuildingGoal::fixupBuilderFailureErrorMessage(BuilderFailur
|
||||
|
||||
msg += showKnownOutputs(worker.store, *drv);
|
||||
|
||||
auto & logTail = buildLog.getTail();
|
||||
if (!logger->isVerbose() && !logTail.empty()) {
|
||||
msg += fmt("\nLast %d log lines:\n", logTail.size());
|
||||
for (auto & line : logTail) {
|
||||
@@ -846,8 +972,7 @@ BuildError DerivationBuildingGoal::fixupBuilderFailureErrorMessage(BuilderFailur
|
||||
return BuildError{e.status, msg};
|
||||
}
|
||||
|
||||
HookReply DerivationBuildingGoal::tryBuildHook(
|
||||
const std::map<std::string, InitialOutput> & initialOutputs, const DerivationOptions<StorePath> & drvOptions)
|
||||
HookReply DerivationBuildingGoal::tryBuildHook(const DerivationOptions<StorePath> & drvOptions)
|
||||
{
|
||||
#ifdef _WIN32 // TODO enable build hook on Windows
|
||||
return rpDecline;
|
||||
@@ -912,61 +1037,19 @@ HookReply DerivationBuildingGoal::tryBuildHook(
|
||||
throw;
|
||||
}
|
||||
|
||||
hook = std::move(worker.hook);
|
||||
|
||||
try {
|
||||
hook->machineName = readLine(hook->fromHook.readSide.get());
|
||||
} catch (Error & e) {
|
||||
e.addTrace({}, "while reading the machine name from the build hook");
|
||||
throw;
|
||||
}
|
||||
|
||||
CommonProto::WriteConn conn{hook->sink};
|
||||
|
||||
/* Tell the hook all the inputs that have to be copied to the
|
||||
remote system. */
|
||||
CommonProto::write(worker.store, conn, inputPaths);
|
||||
|
||||
/* Tell the hooks the missing outputs that have to be copied back
|
||||
from the remote system. */
|
||||
{
|
||||
StringSet missingOutputs;
|
||||
for (auto & [outputName, status] : initialOutputs) {
|
||||
// XXX: Does this include known CA outputs?
|
||||
if (buildMode != bmCheck && status.known && status.known->isValid())
|
||||
continue;
|
||||
missingOutputs.insert(outputName);
|
||||
}
|
||||
CommonProto::write(worker.store, conn, missingOutputs);
|
||||
}
|
||||
|
||||
hook->sink = FdSink();
|
||||
hook->toHook.writeSide.close();
|
||||
|
||||
/* Create the log file and pipe. */
|
||||
[[maybe_unused]] Path logFile = openLogFile();
|
||||
|
||||
std::set<MuxablePipePollState::CommChannel> fds;
|
||||
fds.insert(hook->fromHook.readSide.get());
|
||||
fds.insert(hook->builderOut.readSide.get());
|
||||
worker.childStarted(shared_from_this(), fds, false, false);
|
||||
|
||||
return rpAccept;
|
||||
#endif
|
||||
}
|
||||
|
||||
Path DerivationBuildingGoal::openLogFile()
|
||||
LogFile::LogFile(Store & store, const StorePath & drvPath)
|
||||
{
|
||||
logSize = 0;
|
||||
|
||||
if (!settings.keepLog)
|
||||
return "";
|
||||
return;
|
||||
|
||||
auto baseName = std::string(baseNameOf(worker.store.printStorePath(drvPath)));
|
||||
auto baseName = std::string(baseNameOf(store.printStorePath(drvPath)));
|
||||
|
||||
/* Create a log file. */
|
||||
Path logDir;
|
||||
if (auto localStore = dynamic_cast<LocalStore *>(&worker.store))
|
||||
if (auto localStore = dynamic_cast<LocalStore *>(&store))
|
||||
logDir = localStore->config->logDir;
|
||||
else
|
||||
logDir = settings.nixLogDir;
|
||||
@@ -975,7 +1058,7 @@ Path DerivationBuildingGoal::openLogFile()
|
||||
|
||||
Path logFileName = fmt("%s/%s%s", dir, baseName.substr(2), settings.compressLog ? ".bz2" : "");
|
||||
|
||||
fdLogFile = toDescriptor(open(
|
||||
fd = toDescriptor(open(
|
||||
logFileName.c_str(),
|
||||
O_CREAT | O_WRONLY | O_TRUNC
|
||||
#ifndef _WIN32
|
||||
@@ -983,122 +1066,37 @@ Path DerivationBuildingGoal::openLogFile()
|
||||
#endif
|
||||
,
|
||||
0666));
|
||||
if (!fdLogFile)
|
||||
if (!fd)
|
||||
throw SysError("creating log file '%1%'", logFileName);
|
||||
|
||||
logFileSink = std::make_shared<FdSink>(fdLogFile.get());
|
||||
fileSink = std::make_shared<FdSink>(fd.get());
|
||||
|
||||
if (settings.compressLog)
|
||||
logSink = std::shared_ptr<CompressionSink>(makeCompressionSink(CompressionAlgo::bzip2, *logFileSink));
|
||||
sink = std::shared_ptr<CompressionSink>(makeCompressionSink(CompressionAlgo::bzip2, *fileSink));
|
||||
else
|
||||
logSink = logFileSink;
|
||||
|
||||
return logFileName;
|
||||
sink = fileSink;
|
||||
}
|
||||
|
||||
void DerivationBuildingGoal::closeLogFile()
|
||||
LogFile::~LogFile()
|
||||
{
|
||||
auto logSink2 = std::dynamic_pointer_cast<CompressionSink>(logSink);
|
||||
if (logSink2)
|
||||
logSink2->finish();
|
||||
if (logFileSink)
|
||||
logFileSink->flush();
|
||||
logSink = logFileSink = 0;
|
||||
fdLogFile.close();
|
||||
}
|
||||
|
||||
bool DerivationBuildingGoal::isReadDesc(Descriptor fd)
|
||||
{
|
||||
#ifdef _WIN32 // TODO enable build hook on Windows
|
||||
return false;
|
||||
#else
|
||||
return (hook && fd == hook->builderOut.readSide.get()) || (builder && fd == builder->builderOut.get());
|
||||
#endif
|
||||
}
|
||||
|
||||
Goal::Co DerivationBuildingGoal::processChildOutput(Descriptor fd, std::string_view data)
|
||||
{
|
||||
// local & `ssh://`-builds are dealt with here.
|
||||
auto isWrittenToLog = isReadDesc(fd);
|
||||
if (isWrittenToLog) {
|
||||
logSize += data.size();
|
||||
if (settings.maxLogSize && logSize > settings.maxLogSize) {
|
||||
killChild();
|
||||
co_return doneFailure(BuildError(
|
||||
BuildResult::Failure::LogLimitExceeded,
|
||||
"%s killed after writing more than %d bytes of log output",
|
||||
getName(),
|
||||
settings.maxLogSize));
|
||||
}
|
||||
|
||||
for (auto c : data)
|
||||
if (c == '\r')
|
||||
currentLogLinePos = 0;
|
||||
else if (c == '\n')
|
||||
flushLine();
|
||||
else {
|
||||
if (currentLogLinePos >= currentLogLine.size())
|
||||
currentLogLine.resize(currentLogLinePos + 1);
|
||||
currentLogLine[currentLogLinePos++] = c;
|
||||
}
|
||||
|
||||
if (logSink)
|
||||
(*logSink)(data);
|
||||
try {
|
||||
auto sink2 = std::dynamic_pointer_cast<CompressionSink>(sink);
|
||||
if (sink2)
|
||||
sink2->finish();
|
||||
if (fileSink)
|
||||
fileSink->flush();
|
||||
} catch (...) {
|
||||
ignoreExceptionInDestructor();
|
||||
}
|
||||
|
||||
#ifndef _WIN32 // TODO enable build hook on Windows
|
||||
if (hook && fd == hook->fromHook.readSide.get()) {
|
||||
for (auto c : data)
|
||||
if (c == '\n') {
|
||||
auto json = parseJSONMessage(currentHookLine, "the derivation builder");
|
||||
if (json) {
|
||||
auto s = handleJSONLogMessage(*json, worker.act, hook->activities, "the derivation builder", true);
|
||||
// ensure that logs from a builder using `ssh-ng://` as protocol
|
||||
// are also available to `nix log`.
|
||||
if (s && !isWrittenToLog && logSink) {
|
||||
const auto type = (*json)["type"];
|
||||
const auto fields = (*json)["fields"];
|
||||
if (type == resBuildLogLine) {
|
||||
(*logSink)((fields.size() > 0 ? fields[0].get<std::string>() : "") + "\n");
|
||||
} else if (type == resSetPhase && !fields.is_null()) {
|
||||
const auto phase = fields[0];
|
||||
if (!phase.is_null()) {
|
||||
// nixpkgs' stdenv produces lines in the log to signal
|
||||
// phase changes.
|
||||
// We want to get the same lines in case of remote builds.
|
||||
// The format is:
|
||||
// @nix { "action": "setPhase", "phase": "$curPhase" }
|
||||
const auto logLine = nlohmann::json::object({{"action", "setPhase"}, {"phase", phase}});
|
||||
(*logSink)(
|
||||
"@nix " + logLine.dump(-1, ' ', false, nlohmann::json::error_handler_t::replace)
|
||||
+ "\n");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
currentHookLine.clear();
|
||||
} else
|
||||
currentHookLine += c;
|
||||
}
|
||||
#endif
|
||||
co_return Return{};
|
||||
}
|
||||
|
||||
void DerivationBuildingGoal::flushLine()
|
||||
Goal::Done DerivationBuildingGoal::doneFailureLogTooLong(BuildLog & buildLog)
|
||||
{
|
||||
if (handleJSONLogMessage(currentLogLine, *act, builderActivities, "the derivation builder", false))
|
||||
;
|
||||
|
||||
else {
|
||||
logTail.push_back(currentLogLine);
|
||||
if (logTail.size() > settings.logLines)
|
||||
logTail.pop_front();
|
||||
|
||||
act->result(resBuildLogLine, currentLogLine);
|
||||
}
|
||||
|
||||
currentLogLine = "";
|
||||
currentLogLinePos = 0;
|
||||
return doneFailure(BuildError(
|
||||
BuildResult::Failure::LogLimitExceeded,
|
||||
"%s killed after writing more than %d bytes of log output",
|
||||
getName(),
|
||||
settings.maxLogSize));
|
||||
}
|
||||
|
||||
std::map<std::string, std::optional<StorePath>> DerivationBuildingGoal::queryPartialDerivationOutputMap()
|
||||
|
||||
@@ -324,7 +324,7 @@ void PathSubstitutionGoal::cleanup()
|
||||
if (thr.joinable()) {
|
||||
// FIXME: signal worker thread to quit.
|
||||
thr.join();
|
||||
worker.childTerminated(this);
|
||||
worker.childTerminated(this, JobCategory::Substitution);
|
||||
}
|
||||
|
||||
outPipe.close();
|
||||
|
||||
@@ -162,7 +162,9 @@ template<typename G>
|
||||
static bool
|
||||
removeGoal(std::shared_ptr<G> goal, typename DerivedPathMap<std::map<OutputsSpec, std::weak_ptr<G>>>::ChildNode & node)
|
||||
{
|
||||
return removeGoal(goal, node.value) || removeGoal(goal, node.childMap);
|
||||
bool valueKeep = removeGoal(goal, node.value);
|
||||
bool childMapKeep = removeGoal(goal, node.childMap);
|
||||
return valueKeep || childMapKeep;
|
||||
}
|
||||
|
||||
void Worker::removeGoal(GoalPtr goal)
|
||||
@@ -245,13 +247,18 @@ void Worker::childStarted(
|
||||
}
|
||||
|
||||
void Worker::childTerminated(Goal * goal, bool wakeSleepers)
|
||||
{
|
||||
childTerminated(goal, goal->jobCategory(), wakeSleepers);
|
||||
}
|
||||
|
||||
void Worker::childTerminated(Goal * goal, JobCategory jobCategory, bool wakeSleepers)
|
||||
{
|
||||
auto i = std::find_if(children.begin(), children.end(), [&](const Child & child) { return child.goal2 == goal; });
|
||||
if (i == children.end())
|
||||
return;
|
||||
|
||||
if (i->inBuildSlot) {
|
||||
switch (goal->jobCategory()) {
|
||||
switch (jobCategory) {
|
||||
case JobCategory::Substitution:
|
||||
assert(nrSubstitutions > 0);
|
||||
nrSubstitutions--;
|
||||
|
||||
@@ -339,17 +339,6 @@ static void performOp(
|
||||
break;
|
||||
}
|
||||
|
||||
case WorkerProto::Op::HasSubstitutes: {
|
||||
auto path = WorkerProto::Serialise<StorePath>::read(*store, rconn);
|
||||
logger->startWork();
|
||||
StorePathSet paths; // FIXME
|
||||
paths.insert(path);
|
||||
auto res = store->querySubstitutablePaths(paths);
|
||||
logger->stopWork();
|
||||
conn.to << (res.count(path) != 0);
|
||||
break;
|
||||
}
|
||||
|
||||
case WorkerProto::Op::QuerySubstitutablePaths: {
|
||||
auto paths = WorkerProto::Serialise<StorePathSet>::read(*store, rconn);
|
||||
logger->startWork();
|
||||
|
||||
@@ -69,7 +69,7 @@ struct curlFileTransfer : public FileTransfer
|
||||
curlFileTransfer & fileTransfer;
|
||||
FileTransferRequest request;
|
||||
FileTransferResult result;
|
||||
Activity act;
|
||||
std::unique_ptr<Activity> _act;
|
||||
bool done = false; // whether either the success or failure function has been called
|
||||
Callback<FileTransferResult> callback;
|
||||
CURL * req = 0;
|
||||
@@ -124,12 +124,6 @@ struct curlFileTransfer : public FileTransfer
|
||||
Callback<FileTransferResult> && callback)
|
||||
: fileTransfer(fileTransfer)
|
||||
, request(request)
|
||||
, act(*logger,
|
||||
lvlTalkative,
|
||||
actFileTransfer,
|
||||
fmt("%s '%s'", request.verb(/*continuous=*/true), request.uri),
|
||||
{request.uri.to_string()},
|
||||
request.parentAct)
|
||||
, callback(std::move(callback))
|
||||
, finalSink([this](std::string_view data) {
|
||||
if (errorSink) {
|
||||
@@ -325,9 +319,29 @@ struct curlFileTransfer : public FileTransfer
|
||||
return ((TransferItem *) userp)->headerCallback(contents, size, nmemb);
|
||||
}
|
||||
|
||||
/**
|
||||
* Lazily start an `Activity`. We don't do this in the `TransferItem` constructor to avoid showing downloads
|
||||
* that are only enqueued but not actually started.
|
||||
*/
|
||||
Activity & act()
|
||||
{
|
||||
if (!_act) {
|
||||
_act = std::make_unique<Activity>(
|
||||
*logger,
|
||||
lvlTalkative,
|
||||
actFileTransfer,
|
||||
fmt("%s '%s'", request.verb(/*continuous=*/true), request.uri),
|
||||
Logger::Fields{request.uri.to_string()},
|
||||
request.parentAct);
|
||||
// Reset the start time to when we actually started the download.
|
||||
startTime = std::chrono::steady_clock::now();
|
||||
}
|
||||
return *_act;
|
||||
}
|
||||
|
||||
int progressCallback(curl_off_t dltotal, curl_off_t dlnow) noexcept
|
||||
try {
|
||||
act.progress(dlnow, dltotal);
|
||||
act().progress(dlnow, dltotal);
|
||||
return getInterrupted();
|
||||
} catch (nix::Interrupted &) {
|
||||
assert(getInterrupted());
|
||||
@@ -404,6 +418,13 @@ struct curlFileTransfer : public FileTransfer
|
||||
return ((TransferItem *) clientp)->seekCallback(offset, origin);
|
||||
}
|
||||
|
||||
static int resolverCallbackWrapper(void *, void *, void * clientp) noexcept
|
||||
{
|
||||
// Create the `Activity` associated with this download.
|
||||
((TransferItem *) clientp)->act();
|
||||
return 0;
|
||||
}
|
||||
|
||||
void unpause()
|
||||
{
|
||||
/* Unpausing an already unpaused transfer is a no-op. */
|
||||
@@ -516,6 +537,11 @@ struct curlFileTransfer : public FileTransfer
|
||||
}
|
||||
#endif
|
||||
|
||||
// This seems to be the earliest libcurl callback that signals that the download is happening, so we can
|
||||
// call act().
|
||||
curl_easy_setopt(req, CURLOPT_RESOLVER_START_FUNCTION, resolverCallbackWrapper);
|
||||
curl_easy_setopt(req, CURLOPT_RESOLVER_START_DATA, this);
|
||||
|
||||
result.data.clear();
|
||||
result.bodySize = 0;
|
||||
}
|
||||
@@ -564,7 +590,7 @@ struct curlFileTransfer : public FileTransfer
|
||||
if (httpStatus == 304 && result.etag == "")
|
||||
result.etag = request.expectedETag;
|
||||
|
||||
act.progress(result.bodySize, result.bodySize);
|
||||
act().progress(result.bodySize, result.bodySize);
|
||||
done = true;
|
||||
callback(std::move(result));
|
||||
}
|
||||
@@ -715,6 +741,8 @@ struct curlFileTransfer : public FileTransfer
|
||||
|
||||
std::thread workerThread;
|
||||
|
||||
const size_t maxQueueSize = fileTransferSettings.httpConnections.get() * 5;
|
||||
|
||||
curlFileTransfer()
|
||||
: mt19937(rd())
|
||||
{
|
||||
@@ -820,6 +848,13 @@ struct curlFileTransfer : public FileTransfer
|
||||
{
|
||||
auto state(state_.lock());
|
||||
while (!state->incoming.empty()) {
|
||||
/* Limit the number of active curl handles, since curl doesn't scale well. */
|
||||
if (items.size() + incoming.size() >= maxQueueSize) {
|
||||
auto t = now + std::chrono::milliseconds(100);
|
||||
if (nextWakeup == std::chrono::steady_clock::time_point() || t < nextWakeup)
|
||||
nextWakeup = t;
|
||||
break;
|
||||
}
|
||||
auto item = state->incoming.top();
|
||||
if (item->embargo <= now) {
|
||||
incoming.push_back(item);
|
||||
|
||||
@@ -12,6 +12,7 @@
|
||||
namespace nix {
|
||||
|
||||
struct NarInfo;
|
||||
class NarCache;
|
||||
class RemoteFSAccessor;
|
||||
|
||||
struct BinaryCacheStoreConfig : virtual StoreConfig
|
||||
@@ -89,6 +90,8 @@ protected:
|
||||
|
||||
constexpr const static std::string cacheInfoFile = "nix-cache-info";
|
||||
|
||||
std::shared_ptr<NarCache> narCache;
|
||||
|
||||
BinaryCacheStore(Config &);
|
||||
|
||||
/**
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
#pragma once
|
||||
///@file
|
||||
|
||||
#include "nix/util/logging.hh"
|
||||
#include "nix/util/serialise.hh"
|
||||
|
||||
#include <list>
|
||||
#include <map>
|
||||
#include <string>
|
||||
|
||||
namespace nix {
|
||||
|
||||
/**
|
||||
* Line buffering and log tracking for build output.
|
||||
*
|
||||
* This class handles:
|
||||
* - Owning the build Activity for logging
|
||||
* - Buffering partial lines (handling \r and \n)
|
||||
* - Maintaining a tail of recent log lines (for error messages)
|
||||
* - Processing JSON log messages via handleJSONLogMessage
|
||||
*
|
||||
* Implements Sink so it can be used as a data destination.
|
||||
* I/O is handled separately by the caller.
|
||||
*/
|
||||
struct BuildLog : Sink
|
||||
{
|
||||
private:
|
||||
size_t maxTailLines;
|
||||
|
||||
std::list<std::string> logTail;
|
||||
std::string currentLogLine;
|
||||
size_t currentLogLinePos = 0; // to handle carriage return
|
||||
|
||||
void flushLine();
|
||||
|
||||
public:
|
||||
/**
|
||||
* The build activity. Owned by BuildLog.
|
||||
*/
|
||||
std::unique_ptr<Activity> act;
|
||||
|
||||
/**
|
||||
* Map for tracking nested activities from JSON messages.
|
||||
*/
|
||||
std::map<ActivityId, Activity> builderActivities;
|
||||
|
||||
/**
|
||||
* @param maxTailLines Maximum number of tail lines to keep
|
||||
* @param act Activity for this build
|
||||
*/
|
||||
BuildLog(size_t maxTailLines, std::unique_ptr<Activity> act);
|
||||
|
||||
/**
|
||||
* Process output data from child process.
|
||||
* Handles JSON log messages and emits regular lines to activity.
|
||||
* @param data Raw output data from child
|
||||
*/
|
||||
void operator()(std::string_view data) override;
|
||||
|
||||
/**
|
||||
* Flush any remaining partial line.
|
||||
* Call this when the child process exits.
|
||||
*/
|
||||
void flush();
|
||||
|
||||
/**
|
||||
* Get the most recent log lines.
|
||||
* Used for including in error messages.
|
||||
*/
|
||||
const std::list<std::string> & getTail() const
|
||||
{
|
||||
return logTail;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if there's an incomplete line buffered.
|
||||
*/
|
||||
bool hasPartialLine() const
|
||||
{
|
||||
return !currentLogLine.empty();
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace nix
|
||||
@@ -111,7 +111,7 @@ struct DerivationBuilderCallbacks
|
||||
/**
|
||||
* Open a log file and a pipe to it.
|
||||
*/
|
||||
virtual std::filesystem::path openLogFile() = 0;
|
||||
virtual void openLogFile() = 0;
|
||||
|
||||
/**
|
||||
* Close the log file.
|
||||
|
||||
@@ -9,12 +9,14 @@
|
||||
#include "nix/store/store-api.hh"
|
||||
#include "nix/store/pathlocks.hh"
|
||||
#include "nix/store/build/goal.hh"
|
||||
#include "nix/store/build/build-log.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
using std::map;
|
||||
|
||||
struct BuilderFailureError;
|
||||
struct ExternalBuilder;
|
||||
#ifndef _WIN32 // TODO enable build hook on Windows
|
||||
struct HookInstance;
|
||||
struct DerivationBuilder;
|
||||
@@ -45,93 +47,46 @@ struct DerivationBuildingGoal : public Goal
|
||||
private:
|
||||
|
||||
/** The path of the derivation. */
|
||||
StorePath drvPath;
|
||||
const StorePath drvPath;
|
||||
|
||||
/**
|
||||
* The derivation stored at drvPath.
|
||||
*/
|
||||
std::unique_ptr<Derivation> drv;
|
||||
const std::unique_ptr<Derivation> drv;
|
||||
|
||||
/**
|
||||
* The remainder is state held during the build.
|
||||
*/
|
||||
|
||||
/**
|
||||
* All input paths (that is, the union of FS closures of the
|
||||
* immediate input paths).
|
||||
*/
|
||||
StorePathSet inputPaths;
|
||||
|
||||
/**
|
||||
* File descriptor for the log file.
|
||||
*/
|
||||
AutoCloseFD fdLogFile;
|
||||
std::shared_ptr<BufferedSink> logFileSink, logSink;
|
||||
|
||||
/**
|
||||
* Number of bytes received from the builder's stdout/stderr.
|
||||
*/
|
||||
unsigned long logSize;
|
||||
|
||||
/**
|
||||
* The most recent log lines.
|
||||
*/
|
||||
std::list<std::string> logTail;
|
||||
|
||||
std::string currentLogLine;
|
||||
size_t currentLogLinePos = 0; // to handle carriage return
|
||||
|
||||
std::string currentHookLine;
|
||||
|
||||
#ifndef _WIN32 // TODO enable build hook on Windows
|
||||
/**
|
||||
* The build hook.
|
||||
*/
|
||||
std::unique_ptr<HookInstance> hook;
|
||||
|
||||
std::unique_ptr<DerivationBuilder> builder;
|
||||
#endif
|
||||
|
||||
BuildMode buildMode;
|
||||
const BuildMode buildMode;
|
||||
|
||||
std::unique_ptr<MaintainCount<uint64_t>> mcRunningBuilds;
|
||||
|
||||
std::unique_ptr<Activity> act;
|
||||
|
||||
std::map<ActivityId, Activity> builderActivities;
|
||||
|
||||
std::string key() override;
|
||||
|
||||
/**
|
||||
* The states.
|
||||
*/
|
||||
Co gaveUpOnSubstitution(bool storeDerivation);
|
||||
Co tryToBuild();
|
||||
Co tryToBuild(StorePathSet inputPaths);
|
||||
Co buildWithHook(
|
||||
StorePathSet inputPaths,
|
||||
std::map<std::string, InitialOutput> initialOutputs,
|
||||
DerivationOptions<StorePath> drvOptions,
|
||||
PathLocks outputLocks);
|
||||
Co buildLocally(
|
||||
StorePathSet inputPaths,
|
||||
std::map<std::string, InitialOutput> initialOutputs,
|
||||
DerivationOptions<StorePath> drvOptions,
|
||||
PathLocks outputLocks,
|
||||
const ExternalBuilder * externalBuilder);
|
||||
|
||||
/**
|
||||
* Is the build hook willing to perform the build?
|
||||
*/
|
||||
HookReply tryBuildHook(
|
||||
const std::map<std::string, InitialOutput> & initialOutputs, const DerivationOptions<StorePath> & drvOptions);
|
||||
HookReply tryBuildHook(const DerivationOptions<StorePath> & drvOptions);
|
||||
|
||||
/**
|
||||
* Open a log file and a pipe to it.
|
||||
*/
|
||||
Path openLogFile();
|
||||
|
||||
/**
|
||||
* Close the log file.
|
||||
*/
|
||||
void closeLogFile();
|
||||
|
||||
bool isReadDesc(Descriptor fd);
|
||||
|
||||
/**
|
||||
* Process output from a child process.
|
||||
*/
|
||||
Co processChildOutput(Descriptor fd, std::string_view data);
|
||||
|
||||
void flushLine();
|
||||
Done doneFailureLogTooLong(BuildLog & buildLog);
|
||||
|
||||
/**
|
||||
* Wrappers around the corresponding Store methods that first consult the
|
||||
@@ -148,16 +103,11 @@ private:
|
||||
*/
|
||||
std::pair<bool, SingleDrvOutputs> checkPathValidity(std::map<std::string, InitialOutput> & initialOutputs);
|
||||
|
||||
/**
|
||||
* Forcibly kill the child process, if any.
|
||||
*/
|
||||
void killChild();
|
||||
|
||||
Done doneSuccess(BuildResult::Success::Status status, SingleDrvOutputs builtOutputs);
|
||||
|
||||
Done doneFailure(BuildError ex);
|
||||
|
||||
BuildError fixupBuilderFailureErrorMessage(BuilderFailureError msg);
|
||||
BuildError fixupBuilderFailureErrorMessage(BuilderFailureError msg, BuildLog & buildLog);
|
||||
|
||||
JobCategory jobCategory() const override
|
||||
{
|
||||
|
||||
@@ -295,9 +295,21 @@ public:
|
||||
* false if there is no sense in waking up goals that are sleeping
|
||||
* because they can't run yet (e.g., there is no free build slot,
|
||||
* or the hook would still say `postpone`).
|
||||
*
|
||||
* This overload requires `goal` to point to a fully constructed,
|
||||
* valid goal object, as it calls `goal->jobCategory()`.
|
||||
*/
|
||||
void childTerminated(Goal * goal, bool wakeSleepers = true);
|
||||
|
||||
/**
|
||||
* Unregisters a running child process, like the other overload.
|
||||
*
|
||||
* This overload only uses `goal` as a pointer for comparison with
|
||||
* weak goal references, so it is safe to call from destructors
|
||||
* where the goal object may be partially destroyed.
|
||||
*/
|
||||
void childTerminated(Goal * goal, JobCategory jobCategory, bool wakeSleepers = true);
|
||||
|
||||
/**
|
||||
* Put `goal` to sleep until a build slot becomes available (which
|
||||
* might be right away).
|
||||
|
||||
@@ -214,6 +214,12 @@ public:
|
||||
{
|
||||
unsupported("queryRealisation");
|
||||
}
|
||||
|
||||
StorePathSet querySubstitutablePaths(const StorePathSet & paths) override
|
||||
{
|
||||
// not supported
|
||||
return {};
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace nix
|
||||
@@ -232,8 +232,6 @@ public:
|
||||
|
||||
std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override;
|
||||
|
||||
StorePathSet querySubstitutablePaths(const StorePathSet & paths) override;
|
||||
|
||||
bool pathInfoIsUntrusted(const ValidPathInfo &) override;
|
||||
bool realisationIsUntrusted(const Realisation &) override;
|
||||
|
||||
|
||||
@@ -13,6 +13,7 @@ headers = [ config_pub_h ] + files(
|
||||
'aws-creds.hh',
|
||||
'binary-cache-store.hh',
|
||||
'build-result.hh',
|
||||
'build/build-log.hh',
|
||||
'build/derivation-builder.hh',
|
||||
'build/derivation-building-goal.hh',
|
||||
'build/derivation-building-misc.hh',
|
||||
@@ -55,6 +56,7 @@ headers = [ config_pub_h ] + files(
|
||||
'machines.hh',
|
||||
'make-content-addressed.hh',
|
||||
'names.hh',
|
||||
'nar-cache.hh',
|
||||
'nar-info-disk-cache.hh',
|
||||
'nar-info.hh',
|
||||
'outputs-spec.hh',
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
#pragma once
|
||||
|
||||
#include "nix/util/hash.hh"
|
||||
#include "nix/util/nar-accessor.hh"
|
||||
|
||||
#include <filesystem>
|
||||
|
||||
namespace nix {
|
||||
|
||||
class NarCache
|
||||
{
|
||||
const std::filesystem::path cacheDir;
|
||||
|
||||
std::filesystem::path makeCacheFile(const Hash & narHash, const std::string & ext);
|
||||
|
||||
public:
|
||||
|
||||
NarCache(std::filesystem::path cacheDir);
|
||||
|
||||
void upsertNar(const Hash & narHash, Source & source);
|
||||
|
||||
void upsertNarListing(const Hash & narHash, std::string_view narListingData);
|
||||
|
||||
// FIXME: use a sink.
|
||||
std::optional<std::string> getNar(const Hash & narHash);
|
||||
|
||||
// FIXME: use a sink.
|
||||
GetNarBytes getNarBytes(const Hash & narHash);
|
||||
|
||||
std::optional<std::string> getNarListing(const Hash & narHash);
|
||||
};
|
||||
|
||||
} // namespace nix
|
||||
@@ -7,24 +7,32 @@
|
||||
|
||||
namespace nix {
|
||||
|
||||
struct NarCache;
|
||||
|
||||
class RemoteFSAccessor : public SourceAccessor
|
||||
{
|
||||
ref<Store> store;
|
||||
|
||||
std::map<std::string, ref<SourceAccessor>> nars;
|
||||
/**
|
||||
* Map from store path hash part to NAR hash. Used to then look up
|
||||
* in `nars`. The indirection allows avoiding opening multiple
|
||||
* redundant NAR accessors for the same NAR.
|
||||
*/
|
||||
std::map<std::string, Hash, std::less<>> narHashes;
|
||||
|
||||
/**
|
||||
* Map from NAR hash to NAR accessor.
|
||||
*/
|
||||
std::map<Hash, ref<SourceAccessor>> nars;
|
||||
|
||||
bool requireValidPath;
|
||||
|
||||
std::optional<std::filesystem::path> cacheDir;
|
||||
std::shared_ptr<NarCache> narCache;
|
||||
|
||||
std::pair<ref<SourceAccessor>, CanonPath> fetch(const CanonPath & path);
|
||||
|
||||
friend struct BinaryCacheStore;
|
||||
|
||||
std::filesystem::path makeCacheFile(std::string_view hashPart, const std::string & ext);
|
||||
|
||||
ref<SourceAccessor> addToCache(std::string_view hashPart, std::string && nar);
|
||||
|
||||
public:
|
||||
|
||||
/**
|
||||
@@ -32,8 +40,7 @@ public:
|
||||
*/
|
||||
std::shared_ptr<SourceAccessor> accessObject(const StorePath & path);
|
||||
|
||||
RemoteFSAccessor(
|
||||
ref<Store> store, bool requireValidPath = true, std::optional<std::filesystem::path> cacheDir = {});
|
||||
RemoteFSAccessor(ref<Store> store, bool requireValidPath = true, std::shared_ptr<NarCache> narCache = {});
|
||||
|
||||
std::optional<Stat> maybeLstat(const CanonPath & path) override;
|
||||
|
||||
|
||||
@@ -497,10 +497,7 @@ public:
|
||||
/**
|
||||
* Query which of the given paths have substitutes.
|
||||
*/
|
||||
virtual StorePathSet querySubstitutablePaths(const StorePathSet & paths)
|
||||
{
|
||||
return {};
|
||||
};
|
||||
virtual StorePathSet querySubstitutablePaths(const StorePathSet & paths);
|
||||
|
||||
/**
|
||||
* Query substitute info (i.e. references, derivers and download
|
||||
|
||||
@@ -142,7 +142,7 @@ struct WorkerProto
|
||||
|
||||
enum struct WorkerProto::Op : uint64_t {
|
||||
IsValidPath = 1,
|
||||
HasSubstitutes = 3,
|
||||
// HasSubstitutes = 3, // removed
|
||||
QueryPathHash = 4, // obsolete
|
||||
QueryReferences = 5, // obsolete
|
||||
QueryReferrers = 6,
|
||||
|
||||
@@ -71,8 +71,8 @@ protected:
|
||||
{
|
||||
try {
|
||||
readFile(config->binaryCacheDir + "/" + path, sink);
|
||||
} catch (SysError & e) {
|
||||
if (e.errNo == ENOENT)
|
||||
} catch (SystemError & e) {
|
||||
if (e.is(std::errc::no_such_file_or_directory))
|
||||
throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache", path);
|
||||
throw;
|
||||
}
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
#include "nix/util/archive.hh"
|
||||
#include "nix/util/posix-source-accessor.hh"
|
||||
#include "nix/store/store-api.hh"
|
||||
#include "nix/store/local-fs-store.hh"
|
||||
#include "nix/store/globals.hh"
|
||||
@@ -38,14 +40,13 @@ LocalFSStore::LocalFSStore(const Config & config)
|
||||
{
|
||||
}
|
||||
|
||||
struct LocalStoreAccessor : SourceAccessor
|
||||
struct LocalStoreAccessor : PosixSourceAccessor
|
||||
{
|
||||
ref<SourceAccessor> accessor;
|
||||
ref<LocalFSStore> store;
|
||||
bool requireValidPath;
|
||||
|
||||
LocalStoreAccessor(ref<LocalFSStore> store, bool requireValidPath)
|
||||
: accessor(makeFSSourceAccessor(std::filesystem::path{store->config.realStoreDir.get()}))
|
||||
: PosixSourceAccessor(std::filesystem::path{store->config.realStoreDir.get()})
|
||||
, store(store)
|
||||
, requireValidPath(requireValidPath)
|
||||
{
|
||||
@@ -66,67 +67,25 @@ struct LocalStoreAccessor : SourceAccessor
|
||||
return Stat{.type = tDirectory};
|
||||
|
||||
requireStoreObject(path);
|
||||
return accessor->maybeLstat(path);
|
||||
}
|
||||
|
||||
Stat lstat(const CanonPath & path) override
|
||||
{
|
||||
/* Also allow `path` to point to the entire store, which is
|
||||
needed for resolving symlinks. */
|
||||
if (path.isRoot())
|
||||
return Stat{.type = tDirectory};
|
||||
|
||||
requireStoreObject(path);
|
||||
return accessor->lstat(path);
|
||||
return PosixSourceAccessor::maybeLstat(path);
|
||||
}
|
||||
|
||||
DirEntries readDirectory(const CanonPath & path) override
|
||||
{
|
||||
requireStoreObject(path);
|
||||
return accessor->readDirectory(path);
|
||||
}
|
||||
|
||||
std::string readFile(const CanonPath & path) override
|
||||
{
|
||||
requireStoreObject(path);
|
||||
return accessor->readFile(path);
|
||||
return PosixSourceAccessor::readDirectory(path);
|
||||
}
|
||||
|
||||
void readFile(const CanonPath & path, Sink & sink, std::function<void(uint64_t)> sizeCallback) override
|
||||
{
|
||||
requireStoreObject(path);
|
||||
return accessor->readFile(path, sink, sizeCallback);
|
||||
return PosixSourceAccessor::readFile(path, sink, sizeCallback);
|
||||
}
|
||||
|
||||
std::string readLink(const CanonPath & path) override
|
||||
{
|
||||
requireStoreObject(path);
|
||||
return accessor->readLink(path);
|
||||
}
|
||||
|
||||
std::string showPath(const CanonPath & path) override
|
||||
{
|
||||
return accessor->showPath(path);
|
||||
}
|
||||
|
||||
std::optional<std::filesystem::path> getPhysicalPath(const CanonPath & path) override
|
||||
{
|
||||
return accessor->getPhysicalPath(path);
|
||||
}
|
||||
|
||||
std::pair<CanonPath, std::optional<std::string>> getFingerprint(const CanonPath & path) override
|
||||
{
|
||||
return accessor->getFingerprint(path);
|
||||
}
|
||||
|
||||
std::optional<time_t> getLastModified() override
|
||||
{
|
||||
return accessor->getLastModified();
|
||||
}
|
||||
|
||||
bool pathExists(const CanonPath & path) override
|
||||
{
|
||||
return accessor->pathExists(path);
|
||||
return PosixSourceAccessor::readLink(path);
|
||||
}
|
||||
};
|
||||
|
||||
@@ -150,7 +109,7 @@ std::shared_ptr<SourceAccessor> LocalFSStore::getFSAccessor(const StorePath & pa
|
||||
if (!pathExists(absPath))
|
||||
return nullptr;
|
||||
}
|
||||
return makeFSSourceAccessor(std::move(absPath));
|
||||
return std::make_shared<PosixSourceAccessor>(std::move(absPath));
|
||||
}
|
||||
|
||||
const std::string LocalFSStore::drvsLogDir = "drvs";
|
||||
|
||||
@@ -867,40 +867,6 @@ std::optional<StorePath> LocalStore::queryPathFromHashPart(const std::string & h
|
||||
});
|
||||
}
|
||||
|
||||
StorePathSet LocalStore::querySubstitutablePaths(const StorePathSet & paths)
|
||||
{
|
||||
if (!settings.useSubstitutes)
|
||||
return StorePathSet();
|
||||
|
||||
StorePathSet remaining;
|
||||
for (auto & i : paths)
|
||||
remaining.insert(i);
|
||||
|
||||
StorePathSet res;
|
||||
|
||||
for (auto & sub : getDefaultSubstituters()) {
|
||||
if (remaining.empty())
|
||||
break;
|
||||
if (sub->storeDir != storeDir)
|
||||
continue;
|
||||
if (!sub->config.wantMassQuery)
|
||||
continue;
|
||||
|
||||
auto valid = sub->queryValidPaths(remaining);
|
||||
|
||||
StorePathSet remaining2;
|
||||
for (auto & path : remaining)
|
||||
if (valid.count(path))
|
||||
res.insert(path);
|
||||
else
|
||||
remaining2.insert(path);
|
||||
|
||||
std::swap(remaining, remaining2);
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
void LocalStore::registerValidPath(const ValidPathInfo & info)
|
||||
{
|
||||
registerValidPaths({{info.path, info}});
|
||||
@@ -1059,7 +1025,8 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, RepairF
|
||||
if (info.ca) {
|
||||
auto & specified = *info.ca;
|
||||
auto actualHash = ({
|
||||
SourcePath sourcePath = requireStoreObjectAccessor(info.path, /*requireValidPath=*/false);
|
||||
auto accessor = getFSAccessor(false);
|
||||
CanonPath path{info.path.to_string()};
|
||||
Hash h{HashAlgorithm::SHA256}; // throwaway def to appease C++
|
||||
auto fim = specified.method.getFileIngestionMethod();
|
||||
switch (fim) {
|
||||
@@ -1069,12 +1036,12 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, RepairF
|
||||
specified.hash.algo,
|
||||
std::string{info.path.hashPart()},
|
||||
};
|
||||
dumpPath(sourcePath, caSink, (FileSerialisationMethod) fim);
|
||||
dumpPath({accessor, path}, caSink, (FileSerialisationMethod) fim);
|
||||
h = caSink.finish().hash;
|
||||
break;
|
||||
}
|
||||
case FileIngestionMethod::Git:
|
||||
h = git::dumpHash(specified.hash.algo, sourcePath).hash;
|
||||
h = git::dumpHash(specified.hash.algo, {accessor, path}).hash;
|
||||
break;
|
||||
}
|
||||
ContentAddress{
|
||||
|
||||
@@ -275,6 +275,7 @@ subdir('nix-meson-build-support/common')
|
||||
sources = files(
|
||||
'binary-cache-store.cc',
|
||||
'build-result.cc',
|
||||
'build/build-log.cc',
|
||||
'build/derivation-builder.cc',
|
||||
'build/derivation-building-goal.cc',
|
||||
'build/derivation-check.cc',
|
||||
@@ -317,6 +318,7 @@ sources = files(
|
||||
'make-content-addressed.cc',
|
||||
'misc.cc',
|
||||
'names.cc',
|
||||
'nar-cache.cc',
|
||||
'nar-info-disk-cache.cc',
|
||||
'nar-info.cc',
|
||||
'optimise-store.cc',
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
#include "nix/store/nar-cache.hh"
|
||||
#include "nix/util/file-system.hh"
|
||||
|
||||
#include <sys/types.h>
|
||||
#include <sys/stat.h>
|
||||
#include <fcntl.h>
|
||||
|
||||
namespace nix {
|
||||
|
||||
NarCache::NarCache(std::filesystem::path cacheDir_)
|
||||
: cacheDir(std::move(cacheDir_))
|
||||
{
|
||||
assert(!cacheDir.empty());
|
||||
createDirs(cacheDir);
|
||||
}
|
||||
|
||||
std::filesystem::path NarCache::makeCacheFile(const Hash & narHash, const std::string & ext)
|
||||
{
|
||||
return (cacheDir / narHash.to_string(HashFormat::Nix32, false)) + "." + ext;
|
||||
}
|
||||
|
||||
void NarCache::upsertNar(const Hash & narHash, Source & source)
|
||||
{
|
||||
try {
|
||||
/* FIXME: do this asynchronously. */
|
||||
writeFile(makeCacheFile(narHash, "nar"), source);
|
||||
} catch (SystemError &) {
|
||||
ignoreExceptionExceptInterrupt();
|
||||
}
|
||||
}
|
||||
|
||||
void NarCache::upsertNarListing(const Hash & narHash, std::string_view narListingData)
|
||||
{
|
||||
try {
|
||||
writeFile(makeCacheFile(narHash, "ls"), narListingData);
|
||||
} catch (SystemError &) {
|
||||
ignoreExceptionExceptInterrupt();
|
||||
}
|
||||
}
|
||||
|
||||
std::optional<std::string> NarCache::getNar(const Hash & narHash)
|
||||
{
|
||||
try {
|
||||
return nix::readFile(makeCacheFile(narHash, "nar"));
|
||||
} catch (SystemError &) {
|
||||
return std::nullopt;
|
||||
}
|
||||
}
|
||||
|
||||
GetNarBytes NarCache::getNarBytes(const Hash & narHash)
|
||||
{
|
||||
return seekableGetNarBytes(makeCacheFile(narHash, "nar"));
|
||||
}
|
||||
|
||||
std::optional<std::string> NarCache::getNarListing(const Hash & narHash)
|
||||
{
|
||||
try {
|
||||
return nix::readFile(makeCacheFile(narHash, "ls"));
|
||||
} catch (SystemError &) {
|
||||
return std::nullopt;
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace nix
|
||||
@@ -1,55 +1,14 @@
|
||||
#include <nlohmann/json.hpp>
|
||||
#include "nix/store/remote-fs-accessor.hh"
|
||||
#include "nix/util/nar-accessor.hh"
|
||||
|
||||
#include <sys/types.h>
|
||||
#include <sys/stat.h>
|
||||
#include <fcntl.h>
|
||||
#include "nix/store/nar-cache.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
RemoteFSAccessor::RemoteFSAccessor(
|
||||
ref<Store> store, bool requireValidPath, std::optional<std::filesystem::path> cacheDir_)
|
||||
RemoteFSAccessor::RemoteFSAccessor(ref<Store> store, bool requireValidPath, std::shared_ptr<NarCache> narCache)
|
||||
: store(store)
|
||||
, requireValidPath(requireValidPath)
|
||||
, cacheDir(std::move(cacheDir_))
|
||||
, narCache(std::move(narCache))
|
||||
{
|
||||
if (cacheDir)
|
||||
createDirs(*cacheDir);
|
||||
}
|
||||
|
||||
std::filesystem::path RemoteFSAccessor::makeCacheFile(std::string_view hashPart, const std::string & ext)
|
||||
{
|
||||
assert(cacheDir);
|
||||
auto res = (*cacheDir / hashPart);
|
||||
res.concat(concatStrings(".", ext));
|
||||
return res;
|
||||
}
|
||||
|
||||
ref<SourceAccessor> RemoteFSAccessor::addToCache(std::string_view hashPart, std::string && nar)
|
||||
{
|
||||
if (cacheDir) {
|
||||
try {
|
||||
/* FIXME: do this asynchronously. */
|
||||
writeFile(makeCacheFile(hashPart, "nar"), nar);
|
||||
} catch (...) {
|
||||
ignoreExceptionExceptInterrupt();
|
||||
}
|
||||
}
|
||||
|
||||
auto narAccessor = makeNarAccessor(std::move(nar));
|
||||
nars.emplace(hashPart, narAccessor);
|
||||
|
||||
if (cacheDir) {
|
||||
try {
|
||||
nlohmann::json j = listNarDeep(*narAccessor, CanonPath::root);
|
||||
writeFile(makeCacheFile(hashPart, "ls"), j.dump());
|
||||
} catch (...) {
|
||||
ignoreExceptionExceptInterrupt();
|
||||
}
|
||||
}
|
||||
|
||||
return narAccessor;
|
||||
}
|
||||
|
||||
std::pair<ref<SourceAccessor>, CanonPath> RemoteFSAccessor::fetch(const CanonPath & path)
|
||||
@@ -62,37 +21,46 @@ std::pair<ref<SourceAccessor>, CanonPath> RemoteFSAccessor::fetch(const CanonPat
|
||||
|
||||
std::shared_ptr<SourceAccessor> RemoteFSAccessor::accessObject(const StorePath & storePath)
|
||||
{
|
||||
auto i = nars.find(std::string(storePath.hashPart()));
|
||||
if (i != nars.end())
|
||||
return i->second;
|
||||
|
||||
std::string listing;
|
||||
std::filesystem::path cacheFile;
|
||||
|
||||
if (cacheDir && nix::pathExists(cacheFile = makeCacheFile(storePath.hashPart(), "nar"))) {
|
||||
|
||||
try {
|
||||
listing = nix::readFile(makeCacheFile(storePath.hashPart(), "ls"));
|
||||
auto listingJson = nlohmann::json::parse(listing);
|
||||
auto narAccessor = makeLazyNarAccessor(listingJson, seekableGetNarBytes(cacheFile));
|
||||
|
||||
nars.emplace(storePath.hashPart(), narAccessor);
|
||||
return narAccessor;
|
||||
|
||||
} catch (SystemError &) {
|
||||
}
|
||||
|
||||
try {
|
||||
auto narAccessor = makeNarAccessor(nix::readFile(cacheFile));
|
||||
nars.emplace(storePath.hashPart(), narAccessor);
|
||||
return narAccessor;
|
||||
} catch (SystemError &) {
|
||||
}
|
||||
if (auto * narHash = get(narHashes, storePath.hashPart())) {
|
||||
if (auto * accessor = get(nars, *narHash))
|
||||
return *accessor;
|
||||
}
|
||||
|
||||
StringSink sink;
|
||||
store->narFromPath(storePath, sink);
|
||||
return addToCache(storePath.hashPart(), std::move(sink.s));
|
||||
auto info = store->queryPathInfo(storePath);
|
||||
|
||||
auto cacheAccessor = [&](ref<SourceAccessor> accessor) {
|
||||
narHashes.emplace(storePath.hashPart(), info->narHash);
|
||||
nars.emplace(info->narHash, accessor);
|
||||
return accessor;
|
||||
};
|
||||
|
||||
auto getNar = [&]() {
|
||||
StringSink sink;
|
||||
store->narFromPath(storePath, sink);
|
||||
return std::move(sink.s);
|
||||
};
|
||||
|
||||
if (narCache) {
|
||||
if (auto listingData = narCache->getNarListing(info->narHash))
|
||||
return cacheAccessor(makeLazyNarAccessor(
|
||||
nlohmann::json::parse(*listingData).template get<NarListing>(), narCache->getNarBytes(info->narHash)));
|
||||
|
||||
if (auto nar = narCache->getNar(info->narHash))
|
||||
return cacheAccessor(makeNarAccessor(std::move(*nar)));
|
||||
|
||||
auto nar = getNar();
|
||||
|
||||
StringSource source{nar};
|
||||
narCache->upsertNar(info->narHash, source);
|
||||
|
||||
auto narAccessor = makeNarAccessor(std::move(nar));
|
||||
|
||||
narCache->upsertNarListing(info->narHash, nlohmann::json(narAccessor->getListing()).dump());
|
||||
|
||||
return cacheAccessor(narAccessor);
|
||||
}
|
||||
|
||||
return cacheAccessor(makeNarAccessor(getNar()));
|
||||
}
|
||||
|
||||
std::optional<SourceAccessor::Stat> RemoteFSAccessor::maybeLstat(const CanonPath & path)
|
||||
|
||||
@@ -471,6 +471,40 @@ void Store::querySubstitutablePathInfos(const StorePathCAMap & paths, Substituta
|
||||
}
|
||||
}
|
||||
|
||||
StorePathSet Store::querySubstitutablePaths(const StorePathSet & paths)
|
||||
{
|
||||
if (!settings.useSubstitutes)
|
||||
return StorePathSet();
|
||||
|
||||
StorePathSet remaining;
|
||||
for (auto & i : paths)
|
||||
remaining.insert(i);
|
||||
|
||||
StorePathSet res;
|
||||
|
||||
for (auto & sub : getDefaultSubstituters()) {
|
||||
if (remaining.empty())
|
||||
break;
|
||||
if (sub->storeDir != storeDir)
|
||||
continue;
|
||||
if (!sub->config.wantMassQuery)
|
||||
continue;
|
||||
|
||||
auto valid = sub->queryValidPaths(remaining);
|
||||
|
||||
StorePathSet remaining2;
|
||||
for (auto & path : remaining)
|
||||
if (valid.count(path))
|
||||
res.insert(path);
|
||||
else
|
||||
remaining2.insert(path);
|
||||
|
||||
std::swap(remaining, remaining2);
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
bool Store::isValidPath(const StorePath & storePath)
|
||||
{
|
||||
auto res = pathInfoCache->lock()->get(storePath);
|
||||
|
||||
@@ -495,6 +495,8 @@ bool DerivationBuilderImpl::killChild()
|
||||
killSandbox(true);
|
||||
|
||||
pid.wait();
|
||||
|
||||
miscMethods->childTerminated();
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
@@ -1638,11 +1640,12 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs()
|
||||
HashModuloSink caSink{outputHash.hashAlgo, oldHashPart};
|
||||
auto fim = outputHash.method.getFileIngestionMethod();
|
||||
dumpPath(
|
||||
{makeFSSourceAccessor(actualPath), CanonPath::root}, caSink, (FileSerialisationMethod) fim);
|
||||
{getFSSourceAccessor(), CanonPath(actualPath.native())}, caSink, (FileSerialisationMethod) fim);
|
||||
return caSink.finish().hash;
|
||||
}
|
||||
case FileIngestionMethod::Git: {
|
||||
return git::dumpHash(outputHash.hashAlgo, {makeFSSourceAccessor(actualPath), CanonPath::root}).hash;
|
||||
return git::dumpHash(outputHash.hashAlgo, {getFSSourceAccessor(), CanonPath(actualPath.native())})
|
||||
.hash;
|
||||
}
|
||||
}
|
||||
assert(false);
|
||||
@@ -1664,7 +1667,7 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs()
|
||||
|
||||
{
|
||||
HashResult narHashAndSize = hashPath(
|
||||
{makeFSSourceAccessor(actualPath), CanonPath::root},
|
||||
{getFSSourceAccessor(), CanonPath(actualPath.native())},
|
||||
FileSerialisationMethod::NixArchive,
|
||||
HashAlgorithm::SHA256);
|
||||
newInfo0.narHash = narHashAndSize.hash;
|
||||
@@ -1688,7 +1691,7 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs()
|
||||
std::string{scratchPath->hashPart()}, std::string{requiredFinalPath.hashPart()});
|
||||
rewriteOutput(outputRewrites);
|
||||
HashResult narHashAndSize = hashPath(
|
||||
{makeFSSourceAccessor(actualPath), CanonPath::root},
|
||||
{getFSSourceAccessor(), CanonPath(actualPath.native())},
|
||||
FileSerialisationMethod::NixArchive,
|
||||
HashAlgorithm::SHA256);
|
||||
ValidPathInfo newInfo0{requiredFinalPath, {store, narHashAndSize.hash}};
|
||||
@@ -1703,18 +1706,10 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs()
|
||||
[&](const DerivationOutput::CAFixed & dof) {
|
||||
auto & wanted = dof.ca.hash;
|
||||
|
||||
/* Replace the output by a fresh copy of itself to make sure
|
||||
that there's no stale file descriptor pointing to it.
|
||||
IMPORTANT: Copying and deletion must be done in a race-free manner, thus
|
||||
we are using the source accessor and sink here, since they are implemented
|
||||
the most robustly. DO NOT USE copyFile here. */
|
||||
|
||||
// Replace the output by a fresh copy of itself to make sure
|
||||
// that there's no stale file descriptor pointing to it
|
||||
std::filesystem::path tmpOutput = actualPath.native() + ".tmp";
|
||||
auto accessor = makeFSSourceAccessor(actualPath);
|
||||
auto copySink = RestoreSink(/*startFsync=*/settings.fsyncStorePaths);
|
||||
copySink.dstPath = tmpOutput;
|
||||
copyRecursive(*accessor, /*sourcePath=*/CanonPath::root, copySink, /*destPath=*/CanonPath::root);
|
||||
deletePath(actualPath);
|
||||
copyFile(actualPath, tmpOutput, true);
|
||||
|
||||
std::filesystem::rename(tmpOutput, actualPath);
|
||||
|
||||
|
||||
@@ -88,8 +88,11 @@ HookInstance::~HookInstance()
|
||||
{
|
||||
try {
|
||||
toHook.writeSide = -1;
|
||||
if (pid != -1)
|
||||
if (pid != -1) {
|
||||
pid.kill();
|
||||
if (onKillChild)
|
||||
onKillChild();
|
||||
}
|
||||
} catch (...) {
|
||||
ignoreExceptionInDestructor();
|
||||
}
|
||||
|
||||
@@ -5,6 +5,8 @@
|
||||
#include "nix/util/serialise.hh"
|
||||
#include "nix/util/processes.hh"
|
||||
|
||||
#include <functional>
|
||||
|
||||
namespace nix {
|
||||
|
||||
/**
|
||||
@@ -50,6 +52,12 @@ struct HookInstance
|
||||
|
||||
std::map<ActivityId, Activity> activities;
|
||||
|
||||
/**
|
||||
* Callback to run when the hook process is killed in the destructor.
|
||||
* Used to call `Worker::childTerminated`.
|
||||
*/
|
||||
std::function<void()> onKillChild;
|
||||
|
||||
HookInstance();
|
||||
|
||||
~HookInstance();
|
||||
|
||||
@@ -119,20 +119,15 @@ TEST_F(FSSourceAccessorTest, works)
|
||||
|
||||
EXPECT_THROW(accessor->readFile(CanonPath("a/dirlink/file2")), SymlinkNotAllowed);
|
||||
EXPECT_THROW(accessor->maybeLstat(CanonPath("a/dirlink/file2")), SymlinkNotAllowed);
|
||||
EXPECT_THROW(accessor->readDirectory(CanonPath("a/dirlink")), NotADirectory);
|
||||
EXPECT_THROW(accessor->readDirectory(CanonPath("a/dirlink")), SymlinkNotAllowed);
|
||||
EXPECT_THROW(accessor->pathExists(CanonPath("a/dirlink/file2")), SymlinkNotAllowed);
|
||||
}
|
||||
|
||||
#ifndef _WIN32
|
||||
EXPECT_THAT(
|
||||
[this]() { makeFSSourceAccessor(tmpDir / "nonexistent"); },
|
||||
::testing::Throws<SysError>(::testing::Field(&SysError::errNo, ENOENT)));
|
||||
EXPECT_THAT(
|
||||
[this]() { makeFSSourceAccessor(tmpDir / "nonexistent" / "file"); },
|
||||
::testing::Throws<SysError>(::testing::Field(&SysError::errNo, ENOENT)));
|
||||
#endif
|
||||
|
||||
EXPECT_THAT(makeFSSourceAccessor(tmpDir / "a" / "dirlink"), HasSymlink(CanonPath::root, "../subdir"));
|
||||
{
|
||||
auto accessor = makeFSSourceAccessor(tmpDir / "nonexistent");
|
||||
EXPECT_FALSE(accessor->maybeLstat(CanonPath::root));
|
||||
EXPECT_THROW(accessor->readFile(CanonPath::root), SystemError);
|
||||
}
|
||||
|
||||
{
|
||||
auto accessor = makeFSSourceAccessor(tmpDir, true);
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
#include <gtest/gtest.h>
|
||||
#include <gmock/gmock.h>
|
||||
|
||||
#include "nix/util/file-descriptor.hh"
|
||||
#include "nix/util/file-system.hh"
|
||||
@@ -195,57 +194,4 @@ TEST(fchmodatTryNoFollow, fallbackWithoutProc)
|
||||
}
|
||||
#endif
|
||||
|
||||
/* ----------------------------------------------------------------------------
|
||||
* readLinkAt
|
||||
* --------------------------------------------------------------------------*/
|
||||
|
||||
TEST(readLinkAt, works)
|
||||
{
|
||||
std::filesystem::path tmpDir = nix::createTempDir();
|
||||
nix::AutoDelete delTmpDir(tmpDir, /*recursive=*/true);
|
||||
|
||||
std::string mediumTarget(PATH_MAX / 2, 'x');
|
||||
std::string longTarget(PATH_MAX - 1, 'y');
|
||||
|
||||
{
|
||||
RestoreSink sink(/*startFsync=*/false);
|
||||
sink.dstPath = tmpDir;
|
||||
sink.dirFd = openDirectory(tmpDir);
|
||||
sink.createSymlink(CanonPath("link"), "target");
|
||||
sink.createSymlink(CanonPath("relative"), "../relative/path");
|
||||
sink.createSymlink(CanonPath("absolute"), "/absolute/path");
|
||||
sink.createSymlink(CanonPath("medium"), mediumTarget);
|
||||
sink.createSymlink(CanonPath("long"), longTarget);
|
||||
sink.createDirectory(CanonPath("a"));
|
||||
sink.createDirectory(CanonPath("a/b"));
|
||||
sink.createSymlink(CanonPath("a/b/link"), "nested_target");
|
||||
sink.createRegularFile(CanonPath("regular"), [](CreateRegularFileSink &) {});
|
||||
sink.createDirectory(CanonPath("dir"));
|
||||
}
|
||||
|
||||
AutoCloseFD dirFd = openDirectory(tmpDir);
|
||||
|
||||
EXPECT_EQ(readLinkAt(dirFd.get(), CanonPath("link")), "target");
|
||||
EXPECT_EQ(readLinkAt(dirFd.get(), CanonPath("relative")), "../relative/path");
|
||||
EXPECT_EQ(readLinkAt(dirFd.get(), CanonPath("absolute")), "/absolute/path");
|
||||
EXPECT_EQ(readLinkAt(dirFd.get(), CanonPath("medium")), mediumTarget);
|
||||
EXPECT_EQ(readLinkAt(dirFd.get(), CanonPath("long")), longTarget);
|
||||
EXPECT_EQ(readLinkAt(dirFd.get(), CanonPath("a/b/link")), "nested_target");
|
||||
|
||||
AutoCloseFD subDirFd = openDirectory(tmpDir / "a");
|
||||
EXPECT_EQ(readLinkAt(subDirFd.get(), CanonPath("b/link")), "nested_target");
|
||||
|
||||
EXPECT_THAT(
|
||||
[&] { readLinkAt(dirFd.get(), CanonPath("regular")); },
|
||||
Throws<SysError>(::testing::Field(&SysError::errNo, EINVAL)));
|
||||
|
||||
EXPECT_THAT(
|
||||
[&] { readLinkAt(dirFd.get(), CanonPath("dir")); },
|
||||
Throws<SysError>(::testing::Field(&SysError::errNo, EINVAL)));
|
||||
|
||||
EXPECT_THAT(
|
||||
[&] { readLinkAt(dirFd.get(), CanonPath("nonexistent")); },
|
||||
Throws<SysError>(::testing::Field(&SysError::errNo, ENOENT)));
|
||||
}
|
||||
|
||||
} // namespace nix
|
||||
+17
-36
@@ -112,19 +112,13 @@ TEST_P(FixGitURLTestSuite, parsesVariedGitUrls)
|
||||
EXPECT_EQ(actual.to_string(), p.expected);
|
||||
}
|
||||
|
||||
TEST(FixGitURLTestSuite, scpLikeNoUserParsesPoorly)
|
||||
TEST(FixGitURLTestSuite, rejectScpLikeNoUser)
|
||||
{
|
||||
// SCP-like URL (no user)
|
||||
|
||||
// Cannot "to_string" this because has illegal path not starting
|
||||
// with `/`.
|
||||
EXPECT_EQ(
|
||||
fixGitURL("github.com:owner/repo.git"),
|
||||
(ParsedURL{
|
||||
.scheme = "file",
|
||||
.authority = ParsedURL::Authority{},
|
||||
.path = {"github.com:owner", "repo.git"},
|
||||
}));
|
||||
// SCP-like URL without user. Proper support can be implemented, but this is
|
||||
// a deceptively deep feature - study existing implementations carefully.
|
||||
EXPECT_THAT(
|
||||
[]() { fixGitURL("github.com:owner/repo.git"); },
|
||||
::testing::ThrowsMessage<BadURL>(testing::HasSubstrIgnoreANSIMatcher("SCP-like URL")));
|
||||
}
|
||||
|
||||
TEST(FixGitURLTestSuite, properlyRejectFileURLWithAuthority)
|
||||
@@ -136,37 +130,24 @@ TEST(FixGitURLTestSuite, properlyRejectFileURLWithAuthority)
|
||||
testing::HasSubstrIgnoreANSIMatcher("file:// URL 'file://var/repos/x' has unexpected authority 'var'")));
|
||||
}
|
||||
|
||||
TEST(FixGitURLTestSuite, scpLikePathLeadingSlashParsesPoorly)
|
||||
TEST(FixGitURLTestSuite, rejectScpLikeNoUserLeadingSlash)
|
||||
{
|
||||
// SCP-like URL (no user)
|
||||
|
||||
// Cannot "to_string" this because has illegal path not starting
|
||||
// with `/`.
|
||||
EXPECT_EQ(
|
||||
fixGitURL("github.com:/owner/repo.git"),
|
||||
(ParsedURL{
|
||||
.scheme = "file",
|
||||
.authority = ParsedURL::Authority{},
|
||||
.path = {"github.com:", "owner", "repo.git"},
|
||||
}));
|
||||
EXPECT_THAT(
|
||||
[]() { fixGitURL("github.com:/owner/repo.git"); },
|
||||
::testing::ThrowsMessage<BadURL>(testing::HasSubstrIgnoreANSIMatcher("SCP-like URL")));
|
||||
}
|
||||
|
||||
TEST(FixGitURLTestSuite, relativePathParsesPoorly)
|
||||
TEST(FixGitURLTestSuite, relativePath)
|
||||
{
|
||||
// Relative path (becomes file:// absolute)
|
||||
|
||||
// Cannot "to_string" this because has illegal path not starting
|
||||
// with `/`.
|
||||
// Relative path - parsed as file path without authority
|
||||
auto parsed = fixGitURL("relative/repo");
|
||||
EXPECT_EQ(
|
||||
fixGitURL("relative/repo"),
|
||||
parsed,
|
||||
(ParsedURL{
|
||||
.scheme = "file",
|
||||
.authority =
|
||||
ParsedURL::Authority{
|
||||
.hostType = ParsedURL::Authority::HostType::Name,
|
||||
.host = "",
|
||||
},
|
||||
.path = {"relative", "repo"}}));
|
||||
.path = {"relative", "repo"},
|
||||
}));
|
||||
EXPECT_EQ(parsed.to_string(), "file:relative/repo");
|
||||
}
|
||||
|
||||
struct ParseURLSuccessCase
|
||||
|
||||
+16
-19
@@ -36,10 +36,10 @@ PathFilter defaultPathFilter = [](const Path &) { return true; };
|
||||
|
||||
void SourceAccessor::dumpPath(const CanonPath & path, Sink & sink, PathFilter & filter)
|
||||
{
|
||||
auto dumpContents = [&sink](SourceAccessor & accessor, const CanonPath & path) {
|
||||
auto dumpContents = [&](const CanonPath & path) {
|
||||
sink << "contents";
|
||||
std::optional<uint64_t> size;
|
||||
accessor.readFile(path, sink, [&](uint64_t _size) {
|
||||
readFile(path, sink, [&](uint64_t _size) {
|
||||
size = _size;
|
||||
sink << _size;
|
||||
});
|
||||
@@ -49,10 +49,10 @@ void SourceAccessor::dumpPath(const CanonPath & path, Sink & sink, PathFilter &
|
||||
|
||||
sink << narVersionMagic1;
|
||||
|
||||
[&sink, &filter, &dumpContents](this const auto & dump, SourceAccessor & accessor, const CanonPath & path) -> void {
|
||||
[&, &this_(*this)](this const auto & dump, const CanonPath & path) -> void {
|
||||
checkInterrupt();
|
||||
|
||||
auto st = accessor.lstat(path);
|
||||
auto st = this_.lstat(path);
|
||||
|
||||
sink << "(";
|
||||
|
||||
@@ -60,7 +60,7 @@ void SourceAccessor::dumpPath(const CanonPath & path, Sink & sink, PathFilter &
|
||||
sink << "type" << "regular";
|
||||
if (st.isExecutable)
|
||||
sink << "executable" << "";
|
||||
dumpContents(accessor, path);
|
||||
dumpContents(path);
|
||||
}
|
||||
|
||||
else if (st.type == tDirectory) {
|
||||
@@ -69,7 +69,7 @@ void SourceAccessor::dumpPath(const CanonPath & path, Sink & sink, PathFilter &
|
||||
/* If we're on a case-insensitive system like macOS, undo
|
||||
the case hack applied by restorePath(). */
|
||||
StringMap unhacked;
|
||||
for (auto & i : accessor.readDirectory(path))
|
||||
for (auto & i : this_.readDirectory(path))
|
||||
if (archiveSettings.useCaseHack) {
|
||||
std::string name(i.first);
|
||||
size_t pos = i.first.find(caseHackSuffix);
|
||||
@@ -83,37 +83,34 @@ void SourceAccessor::dumpPath(const CanonPath & path, Sink & sink, PathFilter &
|
||||
} else
|
||||
unhacked.emplace(i.first, i.first);
|
||||
|
||||
accessor.readDirectory(path, [&](SourceAccessor & subdirAccessor, const CanonPath & subdirRelPath) {
|
||||
for (auto & i : unhacked)
|
||||
if (filter((path / i.first).abs())) {
|
||||
sink << "entry" << "(" << "name" << i.first << "node";
|
||||
dump(subdirAccessor, subdirRelPath / i.second);
|
||||
sink << ")";
|
||||
}
|
||||
});
|
||||
for (auto & i : unhacked)
|
||||
if (filter((path / i.first).abs())) {
|
||||
sink << "entry" << "(" << "name" << i.first << "node";
|
||||
dump(path / i.second);
|
||||
sink << ")";
|
||||
}
|
||||
}
|
||||
|
||||
else if (st.type == tSymlink)
|
||||
sink << "type" << "symlink" << "target" << accessor.readLink(path);
|
||||
sink << "type" << "symlink" << "target" << this_.readLink(path);
|
||||
|
||||
else
|
||||
throw Error("file '%s' has an unsupported type", path);
|
||||
|
||||
sink << ")";
|
||||
}(*this, path);
|
||||
}(path);
|
||||
}
|
||||
|
||||
time_t dumpPathAndGetMtime(const Path & path, Sink & sink, PathFilter & filter)
|
||||
{
|
||||
SourcePath path2 = makeFSSourceAccessor(absPath(path), /*trackLastModified=*/true);
|
||||
auto path2 = PosixSourceAccessor::createAtRoot(path, /*trackLastModified=*/true);
|
||||
path2.dumpPath(sink, filter);
|
||||
return path2.accessor->getLastModified().value();
|
||||
}
|
||||
|
||||
void dumpPath(const Path & path, Sink & sink, PathFilter & filter)
|
||||
{
|
||||
SourcePath path2 = makeFSSourceAccessor(absPath(path), /*trackLastModified=*/false);
|
||||
path2.dumpPath(sink, filter);
|
||||
dumpPathAndGetMtime(path, sink, filter);
|
||||
}
|
||||
|
||||
void dumpString(std::string_view s, Sink & sink)
|
||||
|
||||
+24
-23
@@ -252,21 +252,15 @@ Path readLink(const Path & path)
|
||||
|
||||
std::string readFile(const Path & path)
|
||||
{
|
||||
AutoCloseFD fd = toDescriptor(open(
|
||||
path.c_str(),
|
||||
O_RDONLY
|
||||
#ifdef O_CLOEXEC
|
||||
| O_CLOEXEC
|
||||
#endif
|
||||
));
|
||||
if (!fd)
|
||||
throw SysError("opening file '%1%'", path);
|
||||
return readFile(fd.get());
|
||||
return readFile(std::filesystem::path(path));
|
||||
}
|
||||
|
||||
std::string readFile(const std::filesystem::path & path)
|
||||
{
|
||||
return readFile(os_string_to_string(PathViewNG{path}));
|
||||
AutoCloseFD fd = openFileReadonly(path);
|
||||
if (!fd)
|
||||
throw NativeSysError("opening file %1%", path);
|
||||
return readFile(fd.get());
|
||||
}
|
||||
|
||||
void readFile(const Path & path, Sink & sink, bool memory_map)
|
||||
@@ -285,15 +279,9 @@ void readFile(const Path & path, Sink & sink, bool memory_map)
|
||||
}
|
||||
|
||||
// Stream the file instead if memory-mapping fails or is disabled.
|
||||
AutoCloseFD fd = toDescriptor(open(
|
||||
path.c_str(),
|
||||
O_RDONLY
|
||||
#ifdef O_CLOEXEC
|
||||
| O_CLOEXEC
|
||||
#endif
|
||||
));
|
||||
AutoCloseFD fd = openFileReadonly(std::filesystem::path(path));
|
||||
if (!fd)
|
||||
throw SysError("opening file '%s'", path);
|
||||
throw NativeSysError("opening file %s", path);
|
||||
drainFD(fd.get(), sink);
|
||||
}
|
||||
|
||||
@@ -552,7 +540,21 @@ std::filesystem::path createTempDir(const std::filesystem::path & tmpRoot, const
|
||||
AutoCloseFD createAnonymousTempFile()
|
||||
{
|
||||
AutoCloseFD fd;
|
||||
#ifdef O_TMPFILE
|
||||
|
||||
#ifdef _WIN32
|
||||
auto path = makeTempPath(defaultTempDir(), "nix-anonymous");
|
||||
fd = CreateFileW(
|
||||
path.c_str(),
|
||||
GENERIC_READ | GENERIC_WRITE,
|
||||
/*dwShareMode=*/0,
|
||||
/*lpSecurityAttributes=*/nullptr,
|
||||
CREATE_NEW,
|
||||
FILE_ATTRIBUTE_TEMPORARY | FILE_FLAG_DELETE_ON_CLOSE,
|
||||
/*hTemplateFile=*/nullptr);
|
||||
if (!fd)
|
||||
throw windows::WinError("creating temporary file %1%", path);
|
||||
#else
|
||||
# ifdef O_TMPFILE
|
||||
static std::atomic_flag tmpfileUnsupported{};
|
||||
if (!tmpfileUnsupported.test()) /* Try with O_TMPFILE first. */ {
|
||||
/* Use O_EXCL, because the file is never supposed to be linked into filesystem. */
|
||||
@@ -567,14 +569,14 @@ AutoCloseFD createAnonymousTempFile()
|
||||
return fd; /* Successfully created. */
|
||||
}
|
||||
}
|
||||
#endif
|
||||
# endif
|
||||
auto [fd2, path] = createTempFile("nix-anonymous");
|
||||
if (!fd2)
|
||||
throw SysError("creating temporary file '%s'", path);
|
||||
fd = std::move(fd2);
|
||||
#ifndef _WIN32
|
||||
unlink(requireCString(path)); /* We only care about the file descriptor. */
|
||||
#endif
|
||||
|
||||
return fd;
|
||||
}
|
||||
|
||||
@@ -583,7 +585,6 @@ std::pair<AutoCloseFD, Path> createTempFile(const Path & prefix)
|
||||
Path tmpl(defaultTempDir().string() + "/" + prefix + ".XXXXXX");
|
||||
// Strictly speaking, this is UB, but who cares...
|
||||
// FIXME: use O_TMPFILE.
|
||||
// FIXME: Windows should use FILE_ATTRIBUTE_TEMPORARY | FILE_FLAG_DELETE_ON_CLOSE
|
||||
AutoCloseFD fd = toDescriptor(mkstemp((char *) tmpl.c_str()));
|
||||
if (!fd)
|
||||
throw SysError("creating temporary file '%s'", tmpl);
|
||||
|
||||
+18
-19
@@ -33,12 +33,10 @@ void copyRecursive(SourceAccessor & accessor, const CanonPath & from, FileSystem
|
||||
}
|
||||
|
||||
case SourceAccessor::tDirectory: {
|
||||
sink.createDirectory(to, [&](FileSystemObjectSink & dirSink, const CanonPath & relDirPathTo) {
|
||||
accessor.readDirectory(from, [&](SourceAccessor & subdirAccessor, const CanonPath & relDirPathFrom) {
|
||||
for (auto & [name, _] : accessor.readDirectory(from)) {
|
||||
copyRecursive(subdirAccessor, relDirPathFrom / name, dirSink, relDirPathTo / name);
|
||||
}
|
||||
});
|
||||
sink.createDirectory(to, [&](FileSystemObjectSink & dirSink, const CanonPath & relDirPath) {
|
||||
for (auto & [name, _] : accessor.readDirectory(from)) {
|
||||
copyRecursive(accessor, from / name, dirSink, relDirPath / name);
|
||||
}
|
||||
});
|
||||
break;
|
||||
}
|
||||
@@ -128,11 +126,18 @@ void RestoreSink::createDirectory(const CanonPath & path)
|
||||
#endif
|
||||
};
|
||||
|
||||
struct RestoreRegularFile : CreateRegularFileSink
|
||||
struct RestoreRegularFile : CreateRegularFileSink, FdSink
|
||||
{
|
||||
AutoCloseFD fd;
|
||||
bool startFsync = false;
|
||||
|
||||
RestoreRegularFile(bool startFSync_, AutoCloseFD fd_)
|
||||
: FdSink(fd_.get())
|
||||
, fd(std::move(fd_))
|
||||
, startFsync(startFSync_)
|
||||
{
|
||||
}
|
||||
|
||||
~RestoreRegularFile()
|
||||
{
|
||||
/* Initiate an fsync operation without waiting for the
|
||||
@@ -143,7 +148,6 @@ struct RestoreRegularFile : CreateRegularFileSink
|
||||
fd.startFsync();
|
||||
}
|
||||
|
||||
void operator()(std::string_view data) override;
|
||||
void isExecutable() override;
|
||||
void preallocateContents(uint64_t size) override;
|
||||
};
|
||||
@@ -152,9 +156,8 @@ void RestoreSink::createRegularFile(const CanonPath & path, std::function<void(C
|
||||
{
|
||||
auto p = append(dstPath, path);
|
||||
|
||||
RestoreRegularFile crf;
|
||||
crf.startFsync = startFsync;
|
||||
crf.fd =
|
||||
auto crf = RestoreRegularFile(
|
||||
startFsync,
|
||||
#ifdef _WIN32
|
||||
CreateFileW(
|
||||
p.c_str(),
|
||||
@@ -167,17 +170,18 @@ void RestoreSink::createRegularFile(const CanonPath & path, std::function<void(C
|
||||
#else
|
||||
[&]() {
|
||||
/* O_EXCL together with O_CREAT ensures symbolic links in the last
|
||||
component are not followed. */
|
||||
component are not followed. */
|
||||
constexpr int flags = O_CREAT | O_EXCL | O_WRONLY | O_CLOEXEC;
|
||||
if (!dirFd)
|
||||
return ::open(p.c_str(), flags, 0666);
|
||||
return unix::openFileEnsureBeneathNoSymlinks(dirFd.get(), path, flags, 0666);
|
||||
}();
|
||||
}()
|
||||
#endif
|
||||
;
|
||||
);
|
||||
if (!crf.fd)
|
||||
throw NativeSysError("creating file '%1%'", p);
|
||||
func(crf);
|
||||
crf.flush();
|
||||
}
|
||||
|
||||
void RestoreRegularFile::isExecutable()
|
||||
@@ -211,11 +215,6 @@ void RestoreRegularFile::preallocateContents(uint64_t len)
|
||||
#endif
|
||||
}
|
||||
|
||||
void RestoreRegularFile::operator()(std::string_view data)
|
||||
{
|
||||
writeFull(fd.get(), data);
|
||||
}
|
||||
|
||||
void RestoreSink::createSymlink(const CanonPath & path, const std::string & target)
|
||||
{
|
||||
auto p = append(dstPath, path);
|
||||
|
||||
@@ -240,9 +240,39 @@ MakeError(UsageError, Error);
|
||||
MakeError(UnimplementedError, Error);
|
||||
|
||||
/**
|
||||
* To use in catch-blocks.
|
||||
* To use in catch-blocks. Provides a convenience method to get the portable
|
||||
* std::error_code. Use when you want to catch and check an error condition like
|
||||
* no_such_file_or_directory (ENOENT) without ifdefs.
|
||||
*/
|
||||
MakeError(SystemError, Error);
|
||||
class SystemError : public Error
|
||||
{
|
||||
std::error_code errorCode;
|
||||
|
||||
public:
|
||||
template<typename... Args>
|
||||
SystemError(std::errc posixErrNo, Args &&... args)
|
||||
: Error(std::forward<Args>(args)...)
|
||||
, errorCode(std::make_error_code(posixErrNo))
|
||||
{
|
||||
}
|
||||
|
||||
template<typename... Args>
|
||||
SystemError(std::error_code errorCode, Args &&... args)
|
||||
: Error(std::forward<Args>(args)...)
|
||||
, errorCode(errorCode)
|
||||
{
|
||||
}
|
||||
|
||||
const std::error_code ec() const &
|
||||
{
|
||||
return errorCode;
|
||||
}
|
||||
|
||||
bool is(std::errc e) const
|
||||
{
|
||||
return errorCode == e;
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* POSIX system error, created using `errno`, `strerror` friends.
|
||||
@@ -271,7 +301,7 @@ public:
|
||||
*/
|
||||
template<typename... Args>
|
||||
SysError(int errNo, const Args &... args)
|
||||
: SystemError("")
|
||||
: SystemError(static_cast<std::errc>(errNo), "")
|
||||
, errNo(errNo)
|
||||
{
|
||||
auto hf = HintFmt(args...);
|
||||
@@ -332,7 +362,7 @@ public:
|
||||
*/
|
||||
template<typename... Args>
|
||||
WinError(DWORD lastError, const Args &... args)
|
||||
: SystemError("")
|
||||
: SystemError(std::error_code(lastError, std::system_category()), "")
|
||||
, lastError(lastError)
|
||||
{
|
||||
auto hf = HintFmt(args...);
|
||||
|
||||
@@ -250,18 +250,12 @@ namespace unix {
|
||||
*
|
||||
* @param flags O_* flags
|
||||
* @param mode Mode for O_{CREAT,TMPFILE}
|
||||
* @param dirFdCallback Callback invoked that gets the ownership of an intermediate directory fd.
|
||||
*
|
||||
* @pre path.isRoot() is false
|
||||
*
|
||||
* @throws SymlinkNotAllowed if any path components
|
||||
*/
|
||||
Descriptor openFileEnsureBeneathNoSymlinks(
|
||||
Descriptor dirFd,
|
||||
const CanonPath & path,
|
||||
int flags,
|
||||
mode_t mode = 0,
|
||||
std::function<void(AutoCloseFD dirFd, CanonPath relPath)> dirFdCallback = nullptr);
|
||||
Descriptor openFileEnsureBeneathNoSymlinks(Descriptor dirFd, const CanonPath & path, int flags, mode_t mode = 0);
|
||||
|
||||
/**
|
||||
* Try to change the mode of file named by \ref path relative to the parent directory denoted by \ref dirFd.
|
||||
@@ -274,11 +268,6 @@ Descriptor openFileEnsureBeneathNoSymlinks(
|
||||
*/
|
||||
void fchmodatTryNoFollow(Descriptor dirFd, const CanonPath & path, mode_t mode);
|
||||
|
||||
/*
|
||||
* Read a symlink relative to a directory file descriptor.
|
||||
*/
|
||||
std::string readLinkAt(Descriptor dirFd, const CanonPath & path);
|
||||
|
||||
} // namespace unix
|
||||
#endif
|
||||
|
||||
|
||||
@@ -51,6 +51,7 @@ headers = files(
|
||||
'mounted-source-accessor.hh',
|
||||
'muxable-pipe.hh',
|
||||
'nar-accessor.hh',
|
||||
'nar-listing.hh',
|
||||
'os-string.hh',
|
||||
'pool.hh',
|
||||
'pos-idx.hh',
|
||||
|
||||
@@ -1,24 +1,33 @@
|
||||
#pragma once
|
||||
///@file
|
||||
|
||||
#include "nix/util/memory-source-accessor.hh"
|
||||
#include "nix/util/nar-listing.hh"
|
||||
|
||||
#include <functional>
|
||||
#include <filesystem>
|
||||
|
||||
#include <nlohmann/json_fwd.hpp>
|
||||
|
||||
namespace nix {
|
||||
|
||||
struct Source;
|
||||
|
||||
/**
|
||||
* A SourceAccessor for NAR files that provides access to the listing structure.
|
||||
*/
|
||||
struct NarAccessor : SourceAccessor
|
||||
{
|
||||
/**
|
||||
* Get the NAR listing structure.
|
||||
*/
|
||||
virtual const NarListing & getListing() const = 0;
|
||||
};
|
||||
|
||||
/**
|
||||
* Return an object that provides access to the contents of a NAR
|
||||
* file.
|
||||
*/
|
||||
ref<SourceAccessor> makeNarAccessor(std::string && nar);
|
||||
ref<NarAccessor> makeNarAccessor(std::string && nar);
|
||||
|
||||
ref<SourceAccessor> makeNarAccessor(Source & source);
|
||||
ref<NarAccessor> makeNarAccessor(Source & source);
|
||||
|
||||
/**
|
||||
* Create a NAR accessor from a NAR listing (in the format produced by
|
||||
@@ -35,55 +44,12 @@ GetNarBytes seekableGetNarBytes(const std::filesystem::path & path);
|
||||
|
||||
GetNarBytes seekableGetNarBytes(Descriptor fd);
|
||||
|
||||
ref<SourceAccessor> makeLazyNarAccessor(const nlohmann::json & listing, GetNarBytes getNarBytes);
|
||||
ref<NarAccessor> makeLazyNarAccessor(NarListing listing, GetNarBytes getNarBytes);
|
||||
|
||||
/**
|
||||
* Creates a NAR accessor from a given stream and a GetNarBytes getter.
|
||||
* @param source Consumed eagerly. References to it are not persisted in the resulting SourceAccessor.
|
||||
*/
|
||||
ref<SourceAccessor> makeLazyNarAccessor(Source & source, GetNarBytes getNarBytes);
|
||||
|
||||
struct NarListingRegularFile
|
||||
{
|
||||
/**
|
||||
* @see `SourceAccessor::Stat::fileSize`
|
||||
*/
|
||||
std::optional<uint64_t> fileSize;
|
||||
|
||||
/**
|
||||
* @see `SourceAccessor::Stat::narOffset`
|
||||
*
|
||||
* We only set to non-`std::nullopt` if it is also non-zero.
|
||||
*/
|
||||
std::optional<uint64_t> narOffset;
|
||||
|
||||
auto operator<=>(const NarListingRegularFile &) const = default;
|
||||
};
|
||||
|
||||
/**
|
||||
* Abstract syntax for a "NAR listing".
|
||||
*/
|
||||
using NarListing = fso::VariantT<NarListingRegularFile, true>;
|
||||
|
||||
/**
|
||||
* Shallow NAR listing where directory children are not recursively expanded.
|
||||
* Uses a variant that can hold Regular/Symlink fully, but Directory children
|
||||
* are just unit types indicating presence without content.
|
||||
*/
|
||||
using ShallowNarListing = fso::VariantT<NarListingRegularFile, false>;
|
||||
|
||||
/**
|
||||
* Return a deep structured representation of the contents of a NAR (except file
|
||||
* contents), recursively listing all children.
|
||||
*/
|
||||
NarListing listNarDeep(SourceAccessor & accessor, const CanonPath & path);
|
||||
|
||||
/**
|
||||
* Return a shallow structured representation of the contents of a NAR (except file
|
||||
* contents), only listing immediate children without recursing.
|
||||
*/
|
||||
ShallowNarListing listNarShallow(SourceAccessor & accessor, const CanonPath & path);
|
||||
|
||||
// All json_avoids_null and JSON_IMPL covered by generic templates in memory-source-accessor.hh
|
||||
ref<NarAccessor> makeLazyNarAccessor(Source & source, GetNarBytes getNarBytes);
|
||||
|
||||
} // namespace nix
|
||||
@@ -0,0 +1,56 @@
|
||||
#pragma once
|
||||
///@file
|
||||
|
||||
#include "nix/util/memory-source-accessor.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
struct NarListingRegularFile
|
||||
{
|
||||
/**
|
||||
* @see `SourceAccessor::Stat::fileSize`
|
||||
*/
|
||||
std::optional<uint64_t> fileSize;
|
||||
|
||||
/**
|
||||
* @see `SourceAccessor::Stat::narOffset`
|
||||
*
|
||||
* We only set to non-`std::nullopt` if it is also non-zero.
|
||||
*/
|
||||
std::optional<uint64_t> narOffset;
|
||||
|
||||
auto operator<=>(const NarListingRegularFile &) const = default;
|
||||
};
|
||||
|
||||
/**
|
||||
* Abstract syntax for a "NAR listing".
|
||||
*/
|
||||
using NarListing = fso::VariantT<NarListingRegularFile, true>;
|
||||
|
||||
/**
|
||||
* Shallow NAR listing where directory children are not recursively expanded.
|
||||
* Uses a variant that can hold Regular/Symlink fully, but Directory children
|
||||
* are just unit types indicating presence without content.
|
||||
*/
|
||||
using ShallowNarListing = fso::VariantT<NarListingRegularFile, false>;
|
||||
|
||||
/**
|
||||
* Parse a NAR from a Source and return its listing structure.
|
||||
*/
|
||||
NarListing parseNarListing(Source & source);
|
||||
|
||||
/**
|
||||
* Return a deep structured representation of the contents of a NAR (except file
|
||||
* contents), recursively listing all children.
|
||||
*/
|
||||
NarListing listNarDeep(SourceAccessor & accessor, const CanonPath & path);
|
||||
|
||||
/**
|
||||
* Return a shallow structured representation of the contents of a NAR (except file
|
||||
* contents), only listing immediate children without recursing.
|
||||
*/
|
||||
ShallowNarListing listNarShallow(SourceAccessor & accessor, const CanonPath & path);
|
||||
|
||||
// All json_avoids_null and JSON_IMPL covered by generic templates in memory-source-accessor.hh
|
||||
|
||||
} // namespace nix
|
||||
@@ -43,6 +43,36 @@ public:
|
||||
|
||||
std::optional<std::filesystem::path> getPhysicalPath(const CanonPath & path) override;
|
||||
|
||||
/**
|
||||
* Create a `PosixSourceAccessor` and `SourcePath` corresponding to
|
||||
* some native path.
|
||||
*
|
||||
* @param Whether the accessor should return a non-null getLastModified.
|
||||
* When true the accessor must be used only by a single thread.
|
||||
*
|
||||
* The `PosixSourceAccessor` is rooted as far up the tree as
|
||||
* possible, (e.g. on Windows it could scoped to a drive like
|
||||
* `C:\`). This allows more `..` parent accessing to work.
|
||||
*
|
||||
* @note When `path` is trusted user input, canonicalize it using
|
||||
* `std::filesystem::canonical`, `makeParentCanonical`, `std::filesystem::weakly_canonical`, etc,
|
||||
* as appropriate for the use case. At least weak canonicalization is
|
||||
* required for the `SourcePath` to do anything useful at the location it
|
||||
* points to.
|
||||
*
|
||||
* @note A canonicalizing behavior is not built in `createAtRoot` so that
|
||||
* callers do not accidentally introduce symlink-related security vulnerabilities.
|
||||
* Furthermore, `createAtRoot` does not know whether the file pointed to by
|
||||
* `path` should be resolved if it is itself a symlink. In other words,
|
||||
* `createAtRoot` can not decide between aforementioned `canonical`, `makeParentCanonical`, etc. for its callers.
|
||||
*
|
||||
* See
|
||||
* [`std::filesystem::path::root_path`](https://en.cppreference.com/w/cpp/filesystem/path/root_path)
|
||||
* and
|
||||
* [`std::filesystem::path::relative_path`](https://en.cppreference.com/w/cpp/filesystem/path/relative_path).
|
||||
*/
|
||||
static SourcePath createAtRoot(const std::filesystem::path & path, bool trackLastModified = false);
|
||||
|
||||
std::optional<std::time_t> getLastModified() override
|
||||
{
|
||||
return trackLastModified ? std::optional{mtime} : std::nullopt;
|
||||
|
||||
@@ -138,22 +138,6 @@ struct SourceAccessor : std::enable_shared_from_this<SourceAccessor>
|
||||
*/
|
||||
virtual DirEntries readDirectory(const CanonPath & path) = 0;
|
||||
|
||||
/**
|
||||
* Variation of readDirectory that receives a SourceAccessor possibly scoped to \ref dirPath.
|
||||
* Primary meant for recursive traversal functions that would benefit from *at-style syscalls
|
||||
* relative to a particular directory.
|
||||
*
|
||||
* @note Like `readFile`, this method should *not* follow symlinks.
|
||||
* @param callback Caller-provided function invoked with a maximally deeply scoped SourceAccessor and the path that
|
||||
* would have to be prepended to each path relative to dirPath to access a particular file with it.
|
||||
*/
|
||||
virtual void readDirectory(
|
||||
const CanonPath & dirPath,
|
||||
std::function<void(SourceAccessor & subdirAccessor, const CanonPath & subdirRelPath)> callback)
|
||||
{
|
||||
callback(*this, dirPath);
|
||||
}
|
||||
|
||||
virtual std::string readLink(const CanonPath & path) = 0;
|
||||
|
||||
virtual void dumpPath(const CanonPath & path, Sink & sink, PathFilter & filter = defaultPathFilter);
|
||||
@@ -236,11 +220,6 @@ struct SourceAccessor : std::enable_shared_from_this<SourceAccessor>
|
||||
*/
|
||||
ref<SourceAccessor> makeEmptySourceAccessor();
|
||||
|
||||
/**
|
||||
* Helper function that's shared between PosixSourceAccessor and UNIX accessors.
|
||||
*/
|
||||
SourceAccessor::Stat posixStatToAccessorStat(const struct ::stat & st);
|
||||
|
||||
/**
|
||||
* Exception thrown when accessing a filtered path (see
|
||||
* `FilteringSourceAccessor`).
|
||||
@@ -275,18 +254,6 @@ ref<SourceAccessor> getFSSourceAccessor();
|
||||
* that it is not possible to escape `root` by appending `..` path
|
||||
* elements, and that absolute symlinks are resolved relative to
|
||||
* `root`.
|
||||
*
|
||||
* @param root Path to the root of the accessor. Must exist. Symlinks in non-last
|
||||
* components are followed. If \ref root names a symlink an accessor for that symlink
|
||||
* is returned.
|
||||
*
|
||||
* @note Once created the accessor cannot be used if the root of the accessor is modified.
|
||||
* For example, if \ref root names a directory it will be possible to call readDirectory(CanonPath:root).
|
||||
* Similarly with symlinks and regular files.
|
||||
*
|
||||
* @todo Use file HANDLEs on windows.
|
||||
*
|
||||
* @throws SysError if path doesn't exist.
|
||||
*/
|
||||
ref<SourceAccessor> makeFSSourceAccessor(std::filesystem::path root, bool trackLastModified = false);
|
||||
|
||||
|
||||
@@ -149,6 +149,7 @@ sources = [ config_priv_h ] + files(
|
||||
'memory-source-accessor/json.cc',
|
||||
'mounted-source-accessor.cc',
|
||||
'nar-accessor.cc',
|
||||
'nar-listing.cc',
|
||||
'pos-table.cc',
|
||||
'position.cc',
|
||||
'posix-source-accessor.cc',
|
||||
|
||||
+75
-239
@@ -1,198 +1,65 @@
|
||||
#include "nix/util/nar-accessor.hh"
|
||||
#include "nix/util/file-descriptor.hh"
|
||||
#include "nix/util/archive.hh"
|
||||
#include "nix/util/error.hh"
|
||||
|
||||
#include <map>
|
||||
#include <stack>
|
||||
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
namespace nix {
|
||||
|
||||
struct NarMember
|
||||
struct NarAccessorImpl : NarAccessor
|
||||
{
|
||||
SourceAccessor::Stat stat;
|
||||
|
||||
std::string target;
|
||||
|
||||
/* If this is a directory, all the children of the directory. */
|
||||
std::map<std::string, NarMember> children;
|
||||
};
|
||||
|
||||
struct NarMemberConstructor : CreateRegularFileSink
|
||||
{
|
||||
private:
|
||||
|
||||
NarMember & narMember;
|
||||
|
||||
uint64_t & pos;
|
||||
|
||||
public:
|
||||
|
||||
NarMemberConstructor(NarMember & nm, uint64_t & pos)
|
||||
: narMember(nm)
|
||||
, pos(pos)
|
||||
{
|
||||
}
|
||||
|
||||
void isExecutable() override
|
||||
{
|
||||
narMember.stat.isExecutable = true;
|
||||
}
|
||||
|
||||
void preallocateContents(uint64_t size) override
|
||||
{
|
||||
narMember.stat.fileSize = size;
|
||||
narMember.stat.narOffset = pos;
|
||||
}
|
||||
|
||||
void operator()(std::string_view data) override {}
|
||||
};
|
||||
|
||||
struct NarAccessor : public SourceAccessor
|
||||
{
|
||||
std::optional<const std::string> nar;
|
||||
NarListing root;
|
||||
|
||||
GetNarBytes getNarBytes;
|
||||
|
||||
NarMember root;
|
||||
|
||||
struct NarIndexer : FileSystemObjectSink, Source
|
||||
const NarListing & getListing() const override
|
||||
{
|
||||
NarAccessor & acc;
|
||||
Source & source;
|
||||
|
||||
std::stack<NarMember *> parents;
|
||||
|
||||
bool isExec = false;
|
||||
|
||||
uint64_t pos = 0;
|
||||
|
||||
NarIndexer(NarAccessor & acc, Source & source)
|
||||
: acc(acc)
|
||||
, source(source)
|
||||
{
|
||||
}
|
||||
|
||||
NarMember & createMember(const CanonPath & path, NarMember member)
|
||||
{
|
||||
size_t level = 0;
|
||||
for (auto _ : path) {
|
||||
(void) _;
|
||||
++level;
|
||||
}
|
||||
|
||||
while (parents.size() > level)
|
||||
parents.pop();
|
||||
|
||||
if (parents.empty()) {
|
||||
acc.root = std::move(member);
|
||||
parents.push(&acc.root);
|
||||
return acc.root;
|
||||
} else {
|
||||
if (parents.top()->stat.type != Type::tDirectory)
|
||||
throw Error("NAR file missing parent directory of path '%s'", path);
|
||||
auto result = parents.top()->children.emplace(*path.baseName(), std::move(member));
|
||||
auto & ref = result.first->second;
|
||||
parents.push(&ref);
|
||||
return ref;
|
||||
}
|
||||
}
|
||||
|
||||
void createDirectory(const CanonPath & path) override
|
||||
{
|
||||
createMember(
|
||||
path,
|
||||
NarMember{.stat = {.type = Type::tDirectory, .fileSize = 0, .isExecutable = false, .narOffset = 0}});
|
||||
}
|
||||
|
||||
void createRegularFile(const CanonPath & path, std::function<void(CreateRegularFileSink &)> func) override
|
||||
{
|
||||
auto & nm = createMember(
|
||||
path,
|
||||
NarMember{.stat = {.type = Type::tRegular, .fileSize = 0, .isExecutable = false, .narOffset = 0}});
|
||||
NarMemberConstructor nmc{nm, pos};
|
||||
nmc.skipContents = true; /* Don't care about contents. */
|
||||
func(nmc);
|
||||
}
|
||||
|
||||
void createSymlink(const CanonPath & path, const std::string & target) override
|
||||
{
|
||||
createMember(path, NarMember{.stat = {.type = Type::tSymlink}, .target = target});
|
||||
}
|
||||
|
||||
size_t read(char * data, size_t len) override
|
||||
{
|
||||
auto n = source.read(data, len);
|
||||
pos += n;
|
||||
return n;
|
||||
}
|
||||
};
|
||||
|
||||
NarAccessor(std::string && _nar)
|
||||
: nar(_nar)
|
||||
{
|
||||
StringSource source(*nar);
|
||||
NarIndexer indexer(*this, source);
|
||||
parseDump(indexer, indexer);
|
||||
return root;
|
||||
}
|
||||
|
||||
NarAccessor(Source & source)
|
||||
NarAccessorImpl(std::string && nar)
|
||||
: root{[&nar]() {
|
||||
StringSource source(nar);
|
||||
return parseNarListing(source);
|
||||
}()}
|
||||
, getNarBytes{
|
||||
[nar = std::move(nar)](uint64_t offset, uint64_t length) { return std::string{nar, offset, length}; }}
|
||||
{
|
||||
NarIndexer indexer(*this, source);
|
||||
parseDump(indexer, indexer);
|
||||
}
|
||||
|
||||
NarAccessor(Source & source, GetNarBytes getNarBytes)
|
||||
: getNarBytes(std::move(getNarBytes))
|
||||
NarAccessorImpl(Source & source)
|
||||
: root{parseNarListing(source)}
|
||||
{
|
||||
NarIndexer indexer(*this, source);
|
||||
parseDump(indexer, indexer);
|
||||
}
|
||||
|
||||
NarAccessor(const nlohmann::json & listing, GetNarBytes getNarBytes)
|
||||
: getNarBytes(getNarBytes)
|
||||
NarAccessorImpl(Source & source, GetNarBytes getNarBytes)
|
||||
: root{parseNarListing(source)}
|
||||
, getNarBytes{std::move(getNarBytes)}
|
||||
{
|
||||
[&](this const auto & recurse, NarMember & member, const nlohmann::json & v) -> void {
|
||||
std::string type = v["type"];
|
||||
|
||||
if (type == "directory") {
|
||||
member.stat = {.type = Type::tDirectory};
|
||||
for (const auto & [name, function] : v["entries"].items()) {
|
||||
recurse(member.children[name], function);
|
||||
}
|
||||
} else if (type == "regular") {
|
||||
member.stat = {
|
||||
.type = Type::tRegular,
|
||||
.fileSize = v["size"],
|
||||
.isExecutable = v.value("executable", false),
|
||||
.narOffset = v["narOffset"]};
|
||||
} else if (type == "symlink") {
|
||||
member.stat = {.type = Type::tSymlink};
|
||||
member.target = v.value("target", "");
|
||||
} else
|
||||
return;
|
||||
}(root, listing);
|
||||
}
|
||||
|
||||
NarMember * find(const CanonPath & path)
|
||||
NarAccessorImpl(NarListing && listing, GetNarBytes getNarBytes)
|
||||
: root{std::move(listing)}
|
||||
, getNarBytes{std::move(getNarBytes)}
|
||||
{
|
||||
NarMember * current = &root;
|
||||
}
|
||||
|
||||
NarListing * find(const CanonPath & path)
|
||||
{
|
||||
NarListing * current = &root;
|
||||
|
||||
for (const auto & i : path) {
|
||||
if (current->stat.type != Type::tDirectory)
|
||||
auto * dir = std::get_if<NarListing::Directory>(¤t->raw);
|
||||
if (!dir)
|
||||
return nullptr;
|
||||
auto child = current->children.find(std::string(i));
|
||||
if (child == current->children.end())
|
||||
auto * child = nix::get(dir->entries, i);
|
||||
if (!child)
|
||||
return nullptr;
|
||||
current = &child->second;
|
||||
current = child;
|
||||
}
|
||||
|
||||
return current;
|
||||
}
|
||||
|
||||
NarMember & get(const CanonPath & path)
|
||||
NarListing & get(const CanonPath & path)
|
||||
{
|
||||
auto result = find(path);
|
||||
if (!result)
|
||||
@@ -205,70 +72,91 @@ struct NarAccessor : public SourceAccessor
|
||||
auto i = find(path);
|
||||
if (!i)
|
||||
return std::nullopt;
|
||||
return i->stat;
|
||||
return std::visit(
|
||||
overloaded{
|
||||
[](const NarListing::Regular & r) -> Stat {
|
||||
return {
|
||||
.type = Type::tRegular,
|
||||
.fileSize = r.contents.fileSize,
|
||||
.isExecutable = r.executable,
|
||||
.narOffset = r.contents.narOffset,
|
||||
};
|
||||
},
|
||||
[](const NarListing::Directory &) -> Stat {
|
||||
return {
|
||||
.type = Type::tDirectory,
|
||||
};
|
||||
},
|
||||
[](const NarListing::Symlink &) -> Stat {
|
||||
return {
|
||||
.type = Type::tSymlink,
|
||||
};
|
||||
},
|
||||
},
|
||||
i->raw);
|
||||
}
|
||||
|
||||
DirEntries readDirectory(const CanonPath & path) override
|
||||
{
|
||||
auto i = get(path);
|
||||
auto & i = get(path);
|
||||
|
||||
if (i.stat.type != Type::tDirectory)
|
||||
auto * dir = std::get_if<NarListing::Directory>(&i.raw);
|
||||
if (!dir)
|
||||
throw Error("path '%1%' inside NAR file is not a directory", path);
|
||||
|
||||
DirEntries res;
|
||||
for (const auto & child : i.children)
|
||||
res.insert_or_assign(child.first, std::nullopt);
|
||||
for (const auto & [name, child] : dir->entries)
|
||||
res.insert_or_assign(name, std::nullopt);
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
std::string readFile(const CanonPath & path) override
|
||||
{
|
||||
auto i = get(path);
|
||||
if (i.stat.type != Type::tRegular)
|
||||
auto & i = get(path);
|
||||
auto * reg = std::get_if<NarListing::Regular>(&i.raw);
|
||||
if (!reg)
|
||||
throw Error("path '%1%' inside NAR file is not a regular file", path);
|
||||
|
||||
if (getNarBytes)
|
||||
return getNarBytes(*i.stat.narOffset, *i.stat.fileSize);
|
||||
|
||||
assert(nar);
|
||||
return std::string(*nar, *i.stat.narOffset, *i.stat.fileSize);
|
||||
assert(getNarBytes);
|
||||
return getNarBytes(*reg->contents.narOffset, *reg->contents.fileSize);
|
||||
}
|
||||
|
||||
std::string readLink(const CanonPath & path) override
|
||||
{
|
||||
auto i = get(path);
|
||||
if (i.stat.type != Type::tSymlink)
|
||||
auto & i = get(path);
|
||||
auto * sym = std::get_if<NarListing::Symlink>(&i.raw);
|
||||
if (!sym)
|
||||
throw Error("path '%1%' inside NAR file is not a symlink", path);
|
||||
return i.target;
|
||||
return sym->target;
|
||||
}
|
||||
};
|
||||
|
||||
ref<SourceAccessor> makeNarAccessor(std::string && nar)
|
||||
ref<NarAccessor> makeNarAccessor(std::string && nar)
|
||||
{
|
||||
return make_ref<NarAccessor>(std::move(nar));
|
||||
return make_ref<NarAccessorImpl>(std::move(nar));
|
||||
}
|
||||
|
||||
ref<SourceAccessor> makeNarAccessor(Source & source)
|
||||
ref<NarAccessor> makeNarAccessor(Source & source)
|
||||
{
|
||||
return make_ref<NarAccessor>(source);
|
||||
return make_ref<NarAccessorImpl>(source);
|
||||
}
|
||||
|
||||
ref<SourceAccessor> makeLazyNarAccessor(const nlohmann::json & listing, GetNarBytes getNarBytes)
|
||||
ref<NarAccessor> makeLazyNarAccessor(NarListing listing, GetNarBytes getNarBytes)
|
||||
{
|
||||
return make_ref<NarAccessor>(listing, getNarBytes);
|
||||
return make_ref<NarAccessorImpl>(std::move(listing), getNarBytes);
|
||||
}
|
||||
|
||||
ref<SourceAccessor> makeLazyNarAccessor(Source & source, GetNarBytes getNarBytes)
|
||||
ref<NarAccessor> makeLazyNarAccessor(Source & source, GetNarBytes getNarBytes)
|
||||
{
|
||||
return make_ref<NarAccessor>(source, getNarBytes);
|
||||
return make_ref<NarAccessorImpl>(source, getNarBytes);
|
||||
}
|
||||
|
||||
GetNarBytes seekableGetNarBytes(const std::filesystem::path & path)
|
||||
{
|
||||
AutoCloseFD fd = openFileReadonly(path);
|
||||
if (!fd)
|
||||
throw NativeSysError("opening NAR cache file '%s'", path);
|
||||
throw NativeSysError("opening NAR cache file %s", path);
|
||||
|
||||
return [inner = seekableGetNarBytes(fd.get()), fd = make_ref<AutoCloseFD>(std::move(fd))](
|
||||
uint64_t offset, uint64_t length) { return inner(offset, length); };
|
||||
@@ -287,56 +175,4 @@ GetNarBytes seekableGetNarBytes(Descriptor fd)
|
||||
};
|
||||
}
|
||||
|
||||
template<bool deep>
|
||||
using ListNarResult = std::conditional_t<deep, NarListing, ShallowNarListing>;
|
||||
|
||||
template<bool deep>
|
||||
static ListNarResult<deep> listNarImpl(SourceAccessor & accessor, const CanonPath & path)
|
||||
{
|
||||
auto st = accessor.lstat(path);
|
||||
|
||||
switch (st.type) {
|
||||
case SourceAccessor::Type::tRegular:
|
||||
return typename ListNarResult<deep>::Regular{
|
||||
.executable = st.isExecutable,
|
||||
.contents =
|
||||
NarListingRegularFile{
|
||||
.fileSize = st.fileSize,
|
||||
.narOffset = st.narOffset && *st.narOffset ? st.narOffset : std::nullopt,
|
||||
},
|
||||
};
|
||||
case SourceAccessor::Type::tDirectory: {
|
||||
typename ListNarResult<deep>::Directory dir;
|
||||
for (const auto & [name, type] : accessor.readDirectory(path)) {
|
||||
if constexpr (deep) {
|
||||
dir.entries.emplace(name, listNarImpl<true>(accessor, path / name));
|
||||
} else {
|
||||
dir.entries.emplace(name, fso::Opaque{});
|
||||
}
|
||||
}
|
||||
return dir;
|
||||
}
|
||||
case SourceAccessor::Type::tSymlink:
|
||||
return typename ListNarResult<deep>::Symlink{
|
||||
.target = accessor.readLink(path),
|
||||
};
|
||||
case SourceAccessor::Type::tBlock:
|
||||
case SourceAccessor::Type::tChar:
|
||||
case SourceAccessor::Type::tSocket:
|
||||
case SourceAccessor::Type::tFifo:
|
||||
case SourceAccessor::Type::tUnknown:
|
||||
assert(false); // cannot happen for NARs
|
||||
}
|
||||
}
|
||||
|
||||
NarListing listNarDeep(SourceAccessor & accessor, const CanonPath & path)
|
||||
{
|
||||
return listNarImpl<true>(accessor, path);
|
||||
}
|
||||
|
||||
ShallowNarListing listNarShallow(SourceAccessor & accessor, const CanonPath & path)
|
||||
{
|
||||
return listNarImpl<false>(accessor, path);
|
||||
}
|
||||
|
||||
} // namespace nix
|
||||
@@ -0,0 +1,174 @@
|
||||
#include "nix/util/nar-listing.hh"
|
||||
#include "nix/util/archive.hh"
|
||||
#include "nix/util/error.hh"
|
||||
|
||||
#include <stack>
|
||||
|
||||
namespace nix {
|
||||
|
||||
NarListing parseNarListing(Source & source)
|
||||
{
|
||||
struct NarMemberConstructor : CreateRegularFileSink
|
||||
{
|
||||
private:
|
||||
|
||||
NarListing::Regular & regular;
|
||||
|
||||
uint64_t & pos;
|
||||
|
||||
public:
|
||||
|
||||
NarMemberConstructor(NarListing::Regular & reg, uint64_t & pos)
|
||||
: regular(reg)
|
||||
, pos(pos)
|
||||
{
|
||||
}
|
||||
|
||||
void isExecutable() override
|
||||
{
|
||||
regular.executable = true;
|
||||
}
|
||||
|
||||
void preallocateContents(uint64_t size) override
|
||||
{
|
||||
regular.contents.fileSize = size;
|
||||
regular.contents.narOffset = pos;
|
||||
}
|
||||
|
||||
void operator()(std::string_view data) override {}
|
||||
};
|
||||
|
||||
struct NarIndexer : FileSystemObjectSink, Source
|
||||
{
|
||||
std::optional<NarListing> root;
|
||||
Source & source;
|
||||
|
||||
std::stack<NarListing *> parents;
|
||||
|
||||
uint64_t pos = 0;
|
||||
|
||||
NarIndexer(Source & source)
|
||||
: source(source)
|
||||
{
|
||||
}
|
||||
|
||||
NarListing & createMember(const CanonPath & path, NarListing member)
|
||||
{
|
||||
size_t level = 0;
|
||||
for (auto _ : path) {
|
||||
(void) _;
|
||||
++level;
|
||||
}
|
||||
|
||||
while (parents.size() > level)
|
||||
parents.pop();
|
||||
|
||||
if (parents.empty()) {
|
||||
root = std::move(member);
|
||||
parents.push(&*root);
|
||||
return *root;
|
||||
} else {
|
||||
auto * parentDir = std::get_if<NarListing::Directory>(&parents.top()->raw);
|
||||
if (!parentDir)
|
||||
throw Error("NAR file missing parent directory of path '%s'", path);
|
||||
auto result = parentDir->entries.emplace(*path.baseName(), std::move(member));
|
||||
parents.push(&result.first->second);
|
||||
return result.first->second;
|
||||
}
|
||||
}
|
||||
|
||||
void createDirectory(const CanonPath & path) override
|
||||
{
|
||||
createMember(path, NarListing::Directory{});
|
||||
}
|
||||
|
||||
void createRegularFile(const CanonPath & path, std::function<void(CreateRegularFileSink &)> func) override
|
||||
{
|
||||
auto & nm = createMember(
|
||||
path,
|
||||
NarListing::Regular{
|
||||
.executable = false,
|
||||
.contents =
|
||||
NarListingRegularFile{
|
||||
.fileSize = 0,
|
||||
.narOffset = pos,
|
||||
},
|
||||
});
|
||||
/* We know the downcast will succeed because we just added this */
|
||||
auto & reg = std::get<NarListing::Regular>(nm.raw);
|
||||
NarMemberConstructor nmc{reg, pos};
|
||||
nmc.skipContents = true; /* Don't care about contents. */
|
||||
func(nmc);
|
||||
}
|
||||
|
||||
void createSymlink(const CanonPath & path, const std::string & target) override
|
||||
{
|
||||
createMember(path, NarListing::Symlink{.target = target});
|
||||
}
|
||||
|
||||
size_t read(char * data, size_t len) override
|
||||
{
|
||||
auto n = source.read(data, len);
|
||||
pos += n;
|
||||
return n;
|
||||
}
|
||||
};
|
||||
|
||||
NarIndexer indexer(source);
|
||||
parseDump(indexer, indexer);
|
||||
return std::move(*indexer.root);
|
||||
}
|
||||
|
||||
template<bool deep>
|
||||
using ListNarResult = std::conditional_t<deep, NarListing, ShallowNarListing>;
|
||||
|
||||
template<bool deep>
|
||||
static ListNarResult<deep> listNarImpl(SourceAccessor & accessor, const CanonPath & path)
|
||||
{
|
||||
auto st = accessor.lstat(path);
|
||||
|
||||
switch (st.type) {
|
||||
case SourceAccessor::Type::tRegular:
|
||||
return typename ListNarResult<deep>::Regular{
|
||||
.executable = st.isExecutable,
|
||||
.contents =
|
||||
NarListingRegularFile{
|
||||
.fileSize = st.fileSize,
|
||||
.narOffset = st.narOffset && *st.narOffset ? st.narOffset : std::nullopt,
|
||||
},
|
||||
};
|
||||
case SourceAccessor::Type::tDirectory: {
|
||||
typename ListNarResult<deep>::Directory dir;
|
||||
for (const auto & [name, type] : accessor.readDirectory(path)) {
|
||||
if constexpr (deep) {
|
||||
dir.entries.emplace(name, listNarImpl<true>(accessor, path / name));
|
||||
} else {
|
||||
dir.entries.emplace(name, fso::Opaque{});
|
||||
}
|
||||
}
|
||||
return dir;
|
||||
}
|
||||
case SourceAccessor::Type::tSymlink:
|
||||
return typename ListNarResult<deep>::Symlink{
|
||||
.target = accessor.readLink(path),
|
||||
};
|
||||
case SourceAccessor::Type::tBlock:
|
||||
case SourceAccessor::Type::tChar:
|
||||
case SourceAccessor::Type::tSocket:
|
||||
case SourceAccessor::Type::tFifo:
|
||||
case SourceAccessor::Type::tUnknown:
|
||||
assert(false); // cannot happen for NARs
|
||||
}
|
||||
}
|
||||
|
||||
NarListing listNarDeep(SourceAccessor & accessor, const CanonPath & path)
|
||||
{
|
||||
return listNarImpl<true>(accessor, path);
|
||||
}
|
||||
|
||||
ShallowNarListing listNarShallow(SourceAccessor & accessor, const CanonPath & path)
|
||||
{
|
||||
return listNarImpl<false>(accessor, path);
|
||||
}
|
||||
|
||||
} // namespace nix
|
||||
@@ -1,5 +1,7 @@
|
||||
#include "nix/util/posix-source-accessor.hh"
|
||||
#include "nix/util/source-path.hh"
|
||||
#include "nix/util/signals.hh"
|
||||
#include "nix/util/sync.hh"
|
||||
|
||||
#include <boost/unordered/concurrent_flat_map.hpp>
|
||||
|
||||
@@ -18,6 +20,15 @@ PosixSourceAccessor::PosixSourceAccessor()
|
||||
{
|
||||
}
|
||||
|
||||
SourcePath PosixSourceAccessor::createAtRoot(const std::filesystem::path & path, bool trackLastModified)
|
||||
{
|
||||
std::filesystem::path path2 = absPath(path);
|
||||
return {
|
||||
make_ref<PosixSourceAccessor>(path2.root_path(), trackLastModified),
|
||||
CanonPath{path2.relative_path().string()},
|
||||
};
|
||||
}
|
||||
|
||||
std::filesystem::path PosixSourceAccessor::makeAbsPath(const CanonPath & path)
|
||||
{
|
||||
return root.empty() ? (std::filesystem::path{path.abs()})
|
||||
@@ -110,7 +121,22 @@ std::optional<SourceAccessor::Stat> PosixSourceAccessor::maybeLstat(const CanonP
|
||||
if (trackLastModified)
|
||||
mtime = std::max(mtime, st->st_mtime);
|
||||
|
||||
return posixStatToAccessorStat(*st);
|
||||
return Stat{
|
||||
.type = S_ISREG(st->st_mode) ? tRegular
|
||||
: S_ISDIR(st->st_mode) ? tDirectory
|
||||
: S_ISLNK(st->st_mode) ? tSymlink
|
||||
: S_ISCHR(st->st_mode) ? tChar
|
||||
: S_ISBLK(st->st_mode) ? tBlock
|
||||
:
|
||||
#ifdef S_ISSOCK
|
||||
S_ISSOCK(st->st_mode) ? tSocket
|
||||
:
|
||||
#endif
|
||||
S_ISFIFO(st->st_mode) ? tFifo
|
||||
: tUnknown,
|
||||
.fileSize = S_ISREG(st->st_mode) ? std::optional<uint64_t>(st->st_size) : std::nullopt,
|
||||
.isExecutable = S_ISREG(st->st_mode) && st->st_mode & S_IXUSR,
|
||||
};
|
||||
}
|
||||
|
||||
SourceAccessor::DirEntries PosixSourceAccessor::readDirectory(const CanonPath & path)
|
||||
@@ -187,7 +213,6 @@ void PosixSourceAccessor::assertNoSymlinks(CanonPath path)
|
||||
}
|
||||
}
|
||||
|
||||
#ifdef _WIN32
|
||||
ref<SourceAccessor> getFSSourceAccessor()
|
||||
{
|
||||
static auto rootFS = make_ref<PosixSourceAccessor>();
|
||||
@@ -198,6 +223,4 @@ ref<SourceAccessor> makeFSSourceAccessor(std::filesystem::path root, bool trackL
|
||||
{
|
||||
return make_ref<PosixSourceAccessor>(std::move(root), trackLastModified);
|
||||
}
|
||||
#endif
|
||||
|
||||
} // namespace nix
|
||||
@@ -76,27 +76,6 @@ SourceAccessor::Stat SourceAccessor::lstat(const CanonPath & path)
|
||||
throw FileNotFound("path '%s' does not exist", showPath(path));
|
||||
}
|
||||
|
||||
SourceAccessor::Stat posixStatToAccessorStat(const struct ::stat & st)
|
||||
{
|
||||
using enum SourceAccessor::Type;
|
||||
return SourceAccessor::Stat{
|
||||
.type = S_ISREG(st.st_mode) ? tRegular
|
||||
: S_ISDIR(st.st_mode) ? tDirectory
|
||||
: S_ISLNK(st.st_mode) ? tSymlink
|
||||
: S_ISCHR(st.st_mode) ? tChar
|
||||
: S_ISBLK(st.st_mode) ? tBlock
|
||||
:
|
||||
#ifdef S_ISSOCK
|
||||
S_ISSOCK(st.st_mode) ? tSocket
|
||||
:
|
||||
#endif
|
||||
S_ISFIFO(st.st_mode) ? tFifo
|
||||
: tUnknown,
|
||||
.fileSize = S_ISREG(st.st_mode) ? std::optional<uint64_t>(st.st_size) : std::nullopt,
|
||||
.isExecutable = S_ISREG(st.st_mode) && st.st_mode & S_IXUSR,
|
||||
};
|
||||
}
|
||||
|
||||
void SourceAccessor::setPathDisplay(std::string displayPrefix, std::string displaySuffix)
|
||||
{
|
||||
this->displayPrefix = std::move(displayPrefix);
|
||||
|
||||
@@ -340,27 +340,20 @@ void unix::fchmodatTryNoFollow(Descriptor dirFd, const CanonPath & path, mode_t
|
||||
throw SysError("fchmodat '%s' relative to parent directory", path.rel());
|
||||
}
|
||||
|
||||
static Descriptor openFileEnsureBeneathNoSymlinksIterative(
|
||||
Descriptor dirFd,
|
||||
const CanonPath & path,
|
||||
int flags,
|
||||
mode_t mode,
|
||||
std::function<void(AutoCloseFD dirFd, CanonPath relPath)> dirFdCallback)
|
||||
static Descriptor
|
||||
openFileEnsureBeneathNoSymlinksIterative(Descriptor dirFd, const CanonPath & path, int flags, mode_t mode)
|
||||
{
|
||||
AutoCloseFD parentFd;
|
||||
auto nrComponents = std::ranges::distance(path);
|
||||
assert(nrComponents >= 1);
|
||||
auto components = std::views::take(path, nrComponents - 1); /* Everything but last component */
|
||||
auto getParentFd = [&]() { return parentFd ? parentFd.get() : dirFd; };
|
||||
auto currentRelPath = CanonPath::root;
|
||||
|
||||
/* This rather convoluted loop is necessary to avoid TOCTOU when validating that
|
||||
no inner path component is a symlink. */
|
||||
for (auto it = components.begin(); it != components.end(); ++it) {
|
||||
auto component = std::string(*it); /* Copy into a string to make NUL terminated. */
|
||||
assert(component != ".." && !component.starts_with('/')); /* In case invariant is broken somehow.. */
|
||||
auto prevRelPath = currentRelPath;
|
||||
currentRelPath = currentRelPath / *it;
|
||||
|
||||
AutoCloseFD parentFd2 = ::openat(
|
||||
getParentFd(), /* First iteration uses dirFd. */
|
||||
@@ -393,9 +386,6 @@ static Descriptor openFileEnsureBeneathNoSymlinksIterative(
|
||||
return INVALID_DESCRIPTOR;
|
||||
}
|
||||
|
||||
if (dirFdCallback && parentFd)
|
||||
dirFdCallback(std::move(parentFd), std::move(prevRelPath));
|
||||
|
||||
parentFd = std::move(parentFd2);
|
||||
}
|
||||
|
||||
@@ -405,12 +395,7 @@ static Descriptor openFileEnsureBeneathNoSymlinksIterative(
|
||||
return res;
|
||||
}
|
||||
|
||||
Descriptor unix::openFileEnsureBeneathNoSymlinks(
|
||||
Descriptor dirFd,
|
||||
const CanonPath & path,
|
||||
int flags,
|
||||
mode_t mode,
|
||||
std::function<void(AutoCloseFD dirFd, CanonPath relPath)> dirFdCallback)
|
||||
Descriptor unix::openFileEnsureBeneathNoSymlinks(Descriptor dirFd, const CanonPath & path, int flags, mode_t mode)
|
||||
{
|
||||
assert(!path.rel().starts_with('/')); /* Just in case the invariant is somehow broken. */
|
||||
assert(!path.isRoot());
|
||||
@@ -423,23 +408,7 @@ Descriptor unix::openFileEnsureBeneathNoSymlinks(
|
||||
return *maybeFd;
|
||||
}
|
||||
#endif
|
||||
return openFileEnsureBeneathNoSymlinksIterative(dirFd, path, flags, mode, dirFdCallback);
|
||||
}
|
||||
|
||||
std::string unix::readLinkAt(Descriptor dirFd, const CanonPath & path)
|
||||
{
|
||||
assert(!path.isRoot());
|
||||
assert(!path.rel().starts_with('/')); /* Just in case the invariant is somehow broken. */
|
||||
std::vector<char> buf;
|
||||
for (ssize_t bufSize = PATH_MAX / 4; true; bufSize += bufSize / 2) {
|
||||
checkInterrupt();
|
||||
buf.resize(bufSize);
|
||||
ssize_t rlSize = ::readlinkat(dirFd, path.rel_c_str(), buf.data(), bufSize);
|
||||
if (rlSize == -1)
|
||||
throw SysError("reading symbolic link '%1%'", path);
|
||||
else if (rlSize < bufSize)
|
||||
return {buf.data(), static_cast<std::size_t>(rlSize)};
|
||||
}
|
||||
return openFileEnsureBeneathNoSymlinksIterative(dirFd, path, flags, mode);
|
||||
}
|
||||
|
||||
} // namespace nix
|
||||
@@ -1,165 +0,0 @@
|
||||
#pragma once
|
||||
///@file
|
||||
|
||||
#include <cassert>
|
||||
#include <mutex>
|
||||
|
||||
#include "nix/util/lru-cache.hh"
|
||||
#include "nix/util/signals.hh"
|
||||
#include "nix/util/source-accessor.hh"
|
||||
#include "nix/util/file-descriptor.hh"
|
||||
#include "nix/util/sync.hh"
|
||||
|
||||
namespace nix {
|
||||
namespace unix {
|
||||
|
||||
/* The accessors for file/directory access are different, because we want them
|
||||
all to work with file descriptors. Technically that could be done on Linux using
|
||||
O_PATH descriptors, but that wouldn't work on Darwin. */
|
||||
|
||||
class UnixSourceAccessorBase : public SourceAccessor
|
||||
{
|
||||
protected:
|
||||
bool trackLastModified;
|
||||
/**
|
||||
* The most recent mtime seen by fstat(). This is a hack to
|
||||
* support dumpPathAndGetMtime(). Should remove this eventually.
|
||||
*/
|
||||
std::time_t mtime = 0;
|
||||
|
||||
UnixSourceAccessorBase(bool trackLastModified)
|
||||
: trackLastModified(trackLastModified)
|
||||
{
|
||||
}
|
||||
|
||||
void updateMtime(std::time_t newMtime)
|
||||
{
|
||||
/* The contract is that trackLastModified implies that the caller uses the accessor
|
||||
from a single thread. Thus this is not a CAS loop. */
|
||||
if (trackLastModified)
|
||||
mtime = std::max(mtime, newMtime);
|
||||
}
|
||||
|
||||
public:
|
||||
std::optional<std::time_t> getLastModified() override
|
||||
{
|
||||
return trackLastModified ? std::optional{mtime} : std::nullopt;
|
||||
}
|
||||
};
|
||||
|
||||
class UnixFileSourceAccessor : public UnixSourceAccessorBase
|
||||
{
|
||||
AutoCloseFD fd;
|
||||
CanonPath rootPath;
|
||||
mutable std::once_flag statFlag;
|
||||
mutable struct ::stat cachedStat;
|
||||
|
||||
public:
|
||||
|
||||
UnixFileSourceAccessor(AutoCloseFD fd_, CanonPath rootPath_, bool trackLastModified, struct ::stat * st = nullptr)
|
||||
: UnixSourceAccessorBase(trackLastModified)
|
||||
, fd(std::move(fd_))
|
||||
, rootPath(std::move(rootPath_))
|
||||
{
|
||||
displayPrefix = rootPath.abs();
|
||||
if (st) {
|
||||
std::call_once(statFlag, [this, st] {
|
||||
cachedStat = *st;
|
||||
updateMtime(cachedStat.st_mtime);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
std::string showPath(const CanonPath & path) override
|
||||
{
|
||||
if (path.isRoot())
|
||||
return displayPrefix; /* No trailing slash, we know it's not a directory. */
|
||||
return SourceAccessor::showPath(path);
|
||||
}
|
||||
|
||||
DirEntries readDirectory(const CanonPath & path) override
|
||||
{
|
||||
if (!path.isRoot())
|
||||
throw FileNotFound("path '%s' does not exist", showPath(path));
|
||||
throw NotADirectory("path '%s' is not a directory", showPath(path));
|
||||
}
|
||||
|
||||
std::string readLink(const CanonPath & path) override
|
||||
{
|
||||
if (!path.isRoot())
|
||||
throw FileNotFound("path '%s' does not exist", showPath(path));
|
||||
throw NotASymlink("path '%s' is not a symlink", showPath(path));
|
||||
}
|
||||
|
||||
bool pathExists(const CanonPath & path) override
|
||||
{
|
||||
return path.isRoot(); /* We know that we are accessing a regular file and not a directory. */
|
||||
}
|
||||
|
||||
std::optional<std::filesystem::path> getPhysicalPath(const CanonPath & path) override
|
||||
{
|
||||
if (path.isRoot())
|
||||
return std::filesystem::path(rootPath.abs());
|
||||
/* Slightly different than what PosixSourceAccessor used to do, but we know that this is not a directory. */
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
std::optional<Stat> maybeLstat(const CanonPath & path) override;
|
||||
|
||||
void readFile(const CanonPath & path, Sink & sink, std::function<void(uint64_t)> sizeCallback) override;
|
||||
};
|
||||
|
||||
class UnixDirectorySourceAccessor : public UnixSourceAccessorBase
|
||||
{
|
||||
AutoCloseFD fd;
|
||||
CanonPath rootPath;
|
||||
std::unique_ptr<Sync<LRUCache<CanonPath, ref<AutoCloseFD>>>> dirFdCache;
|
||||
|
||||
/**
|
||||
* Get the parent directory of path. The second pair element might be an owning file descriptor
|
||||
* if path.parent().isRoot() is false.
|
||||
*/
|
||||
std::pair<Descriptor, std::shared_ptr<AutoCloseFD>> openParent(const CanonPath & path);
|
||||
|
||||
std::function<void(AutoCloseFD, CanonPath)> makeDirFdCallback();
|
||||
|
||||
AutoCloseFD openSubdirectory(const CanonPath & path);
|
||||
|
||||
public:
|
||||
UnixDirectorySourceAccessor(
|
||||
AutoCloseFD fd_, CanonPath rootPath_, bool trackLastModified, unsigned dirFdCacheSize = 128)
|
||||
: UnixSourceAccessorBase(trackLastModified)
|
||||
, fd(std::move(fd_))
|
||||
, rootPath(std::move(rootPath_))
|
||||
{
|
||||
if (rootPath.isRoot())
|
||||
displayPrefix.clear(); /* To avoid the double slash. */
|
||||
else
|
||||
displayPrefix = rootPath.abs();
|
||||
|
||||
if (dirFdCacheSize)
|
||||
dirFdCache = std::make_unique<Sync<LRUCache<CanonPath, ref<AutoCloseFD>>>>(dirFdCacheSize);
|
||||
}
|
||||
|
||||
std::optional<std::filesystem::path> getPhysicalPath(const CanonPath & path) override
|
||||
{
|
||||
if (path.isRoot())
|
||||
return std::filesystem::path(rootPath.abs());
|
||||
return std::filesystem::path(rootPath.abs()) / path.rel(); /* RHS *must* be a relative path. */
|
||||
}
|
||||
|
||||
std::optional<Stat> maybeLstat(const CanonPath & path) override;
|
||||
|
||||
void readFile(const CanonPath & path, Sink & sink, std::function<void(uint64_t)> sizeCallback) override;
|
||||
|
||||
DirEntries readDirectory(const CanonPath & path) override;
|
||||
|
||||
void readDirectory(
|
||||
const CanonPath & dirPath,
|
||||
std::function<void(SourceAccessor & subdirAccessor, const CanonPath & subdirRelPath)> callback) override;
|
||||
|
||||
std::string readLink(const CanonPath & path) override;
|
||||
};
|
||||
|
||||
} // namespace unix
|
||||
} // namespace nix
|
||||
@@ -58,7 +58,6 @@ sources += files(
|
||||
'os-string.cc',
|
||||
'processes.cc',
|
||||
'signals.cc',
|
||||
'unix-source-accessor.cc',
|
||||
'users.cc',
|
||||
)
|
||||
|
||||
|
||||
@@ -1,358 +0,0 @@
|
||||
#include "nix/util/memory-source-accessor.hh"
|
||||
#include "nix/util/source-accessor.hh"
|
||||
#include "nix/util/unix-source-accessor.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
using namespace nix::unix;
|
||||
|
||||
std::optional<SourceAccessor::Stat> UnixFileSourceAccessor::maybeLstat(const CanonPath & path)
|
||||
{
|
||||
if (!path.isRoot())
|
||||
/* This is not a directory. Nothing can be beneath it. */
|
||||
return std::nullopt;
|
||||
|
||||
std::call_once(statFlag, [this] {
|
||||
if (::fstat(fd.get(), &cachedStat) == -1)
|
||||
throw SysError("statting file '%s'", displayPrefix);
|
||||
updateMtime(cachedStat.st_mtime);
|
||||
});
|
||||
|
||||
return posixStatToAccessorStat(cachedStat);
|
||||
}
|
||||
|
||||
void UnixFileSourceAccessor::readFile(const CanonPath & path, Sink & sink, std::function<void(uint64_t)> sizeCallback)
|
||||
{
|
||||
if (!path.isRoot())
|
||||
throw FileNotFound("path '%s' does not exist", showPath(path));
|
||||
|
||||
struct ::stat st;
|
||||
/* Fresh fstat. TODO: Maybe reuse the cached stat? There are some nuances when it comes
|
||||
to non-regular file handling (e.g. /dev/stdin) that is also system dependent.
|
||||
See https://github.com/NixOS/nix/issues/9330. We should probably ban non-regular files
|
||||
completely. */
|
||||
if (::fstat(fd.get(), &st) == -1)
|
||||
throw SysError("getting status of '%s'", showPath(path));
|
||||
|
||||
off_t left = st.st_size;
|
||||
off_t offset = 0;
|
||||
/* Currently trusts st_size to be correct, errors out if EOF is reached before reading st_size bytes:
|
||||
https://github.com/NixOS/nix/issues/10667. */
|
||||
sizeCallback(left);
|
||||
|
||||
/* TODO: Optimise for the case when Sink is an FdSink and call sendfile. Can
|
||||
also use copy_file_range to leverage reflinking if the destination is a
|
||||
regular file and not a socket. */
|
||||
|
||||
std::array<unsigned char, 64 * 1024> buf;
|
||||
while (left) {
|
||||
checkInterrupt();
|
||||
/* N.B. Using pread for thread-safety. File pointer must not be modified. */
|
||||
ssize_t rd = ::pread(fd.get(), buf.data(), std::min<std::size_t>(left, buf.size()), offset);
|
||||
if (rd == -1) {
|
||||
if (errno != EINTR)
|
||||
throw SysError("reading from file '%s'", showPath(path));
|
||||
} else if (rd == 0)
|
||||
throw SysError("unexpected end-of-file reading '%s'", showPath(path));
|
||||
else {
|
||||
assert(rd <= left);
|
||||
sink({reinterpret_cast<char *>(buf.data()), static_cast<std::size_t>(rd)});
|
||||
left -= rd;
|
||||
offset += rd;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
std::function<void(AutoCloseFD, CanonPath)> UnixDirectorySourceAccessor::makeDirFdCallback()
|
||||
{
|
||||
if (!dirFdCache)
|
||||
return nullptr;
|
||||
|
||||
return [this](AutoCloseFD fd, CanonPath key) {
|
||||
auto cache(dirFdCache->lock());
|
||||
assert(fd);
|
||||
cache->upsert(key, make_ref<AutoCloseFD>(std::move(fd)));
|
||||
};
|
||||
}
|
||||
|
||||
std::pair<Descriptor, std::shared_ptr<AutoCloseFD>> UnixDirectorySourceAccessor::openParent(const CanonPath & path)
|
||||
{
|
||||
assert(!path.isRoot());
|
||||
auto parent = path.parent().value();
|
||||
if (parent.isRoot())
|
||||
return {fd.get(), nullptr};
|
||||
|
||||
if (dirFdCache) {
|
||||
if (auto cachedFd = dirFdCache->lock()->get(parent)) {
|
||||
assert((*cachedFd)->get());
|
||||
return {(*cachedFd)->get(), *cachedFd};
|
||||
}
|
||||
}
|
||||
|
||||
AutoCloseFD parentFdOwning = openFileEnsureBeneathNoSymlinks(
|
||||
fd.get(), parent, O_DIRECTORY | O_RDONLY | O_NOFOLLOW | O_CLOEXEC, 0, makeDirFdCallback());
|
||||
if (!parentFdOwning && (errno == ELOOP || errno == ENOTDIR))
|
||||
throw SymlinkNotAllowed(parent);
|
||||
return {parentFdOwning.get(), make_ref<AutoCloseFD>(std::move(parentFdOwning))};
|
||||
}
|
||||
|
||||
std::optional<SourceAccessor::Stat> UnixDirectorySourceAccessor::maybeLstat(const CanonPath & path)
|
||||
try {
|
||||
struct ::stat st;
|
||||
|
||||
if (path.isRoot()) {
|
||||
/* This error is unexpected. Would only happen if the directory fd is messed up. */
|
||||
if (::fstat(fd.get(), &st) == -1)
|
||||
throw SysError("getting status of '%s'", showPath(path));
|
||||
} else {
|
||||
auto [parentFd, parentFdOwning] = openParent(path);
|
||||
if (parentFd == INVALID_DESCRIPTOR)
|
||||
return std::nullopt;
|
||||
if (::fstatat(parentFd, std::string(path.baseName().value()).c_str(), &st, AT_SYMLINK_NOFOLLOW) == -1)
|
||||
return std::nullopt;
|
||||
|
||||
if (dirFdCache && parentFdOwning) {
|
||||
assert(*parentFdOwning);
|
||||
dirFdCache->lock()->upsert(path.parent().value(), ref<AutoCloseFD>(parentFdOwning));
|
||||
}
|
||||
}
|
||||
|
||||
updateMtime(st.st_mtime);
|
||||
return posixStatToAccessorStat(st);
|
||||
} catch (SymlinkNotAllowed & e) {
|
||||
throw SymlinkNotAllowed(e.path, "path '%s' is a symlink", showPath(e.path));
|
||||
}
|
||||
|
||||
void UnixDirectorySourceAccessor::readFile(
|
||||
const CanonPath & path, Sink & sink, std::function<void(uint64_t)> sizeCallback)
|
||||
try {
|
||||
if (path.isRoot())
|
||||
throw NotARegularFile("'%s' is not a regular file", showPath(path));
|
||||
|
||||
AutoCloseFD fileFd = openFileEnsureBeneathNoSymlinks(fd.get(), path, O_RDONLY | O_NOFOLLOW | O_CLOEXEC);
|
||||
if (!fileFd) {
|
||||
if (errno == ELOOP) /* The last component is a symlink. */
|
||||
throw NotARegularFile("'%s' is a symlink, not a regular file", showPath(path));
|
||||
if (errno == ENOENT || errno == ENOTDIR) /* Intermediate component might not exist. */
|
||||
throw FileNotFound("file '%s' does not exist", showPath(path));
|
||||
throw SysError("opening '%s'", showPath(path));
|
||||
}
|
||||
|
||||
UnixFileSourceAccessor fileAccessor(std::move(fileFd), rootPath / path, trackLastModified);
|
||||
fileAccessor.readFile(CanonPath::root, sink, sizeCallback);
|
||||
|
||||
if (auto fileMtime = fileAccessor.getLastModified())
|
||||
mtime = std::max(mtime, *fileMtime);
|
||||
} catch (SymlinkNotAllowed & e) {
|
||||
throw SymlinkNotAllowed(e.path, "path '%s' is a symlink", showPath(e.path));
|
||||
}
|
||||
|
||||
AutoCloseFD UnixDirectorySourceAccessor::openSubdirectory(const CanonPath & path)
|
||||
try {
|
||||
AutoCloseFD dirFdOwning;
|
||||
|
||||
if (path.isRoot()) {
|
||||
/* Get a fresh file descriptor for thread-safety. */
|
||||
dirFdOwning = ::openat(fd.get(), ".", O_DIRECTORY | O_RDONLY | O_CLOEXEC);
|
||||
if (!dirFdOwning)
|
||||
throw SysError("opening directory '%s'", showPath(path));
|
||||
} else {
|
||||
dirFdOwning = openFileEnsureBeneathNoSymlinks(
|
||||
fd.get(), path, O_DIRECTORY | O_RDONLY | O_NOFOLLOW | O_CLOEXEC, 0, makeDirFdCallback());
|
||||
if (!dirFdOwning) {
|
||||
if (errno == ENOTDIR)
|
||||
throw NotADirectory("'%s' is not a directory", showPath(path));
|
||||
throw SysError("opening directory '%s'", showPath(path));
|
||||
}
|
||||
}
|
||||
|
||||
return dirFdOwning;
|
||||
} catch (SymlinkNotAllowed & e) {
|
||||
throw SymlinkNotAllowed(e.path, "path '%s' is a symlink", showPath(e.path));
|
||||
}
|
||||
|
||||
SourceAccessor::DirEntries UnixDirectorySourceAccessor::readDirectory(const CanonPath & path)
|
||||
try {
|
||||
AutoCloseFD dirFdOwning = openSubdirectory(path);
|
||||
AutoCloseDir dir(::fdopendir(dirFdOwning.get()));
|
||||
if (!dir)
|
||||
throw SysError("opening directory '%s'", showPath(path));
|
||||
dirFdOwning.release();
|
||||
|
||||
DirEntries entries;
|
||||
const ::dirent * dirent = nullptr;
|
||||
|
||||
while (errno = 0, dirent = ::readdir(dir.get())) {
|
||||
checkInterrupt();
|
||||
std::string_view name(dirent->d_name);
|
||||
if (name == "." || name == "..")
|
||||
continue;
|
||||
|
||||
std::optional<Type> type;
|
||||
switch (dirent->d_type) {
|
||||
case DT_REG:
|
||||
type = tRegular;
|
||||
break;
|
||||
case DT_DIR:
|
||||
type = tDirectory;
|
||||
break;
|
||||
case DT_LNK:
|
||||
type = tSymlink;
|
||||
break;
|
||||
case DT_CHR:
|
||||
type = tChar;
|
||||
break;
|
||||
case DT_BLK:
|
||||
type = tBlock;
|
||||
break;
|
||||
case DT_FIFO:
|
||||
type = tFifo;
|
||||
break;
|
||||
case DT_SOCK:
|
||||
type = tSocket;
|
||||
break;
|
||||
default:
|
||||
type = std::nullopt;
|
||||
break;
|
||||
}
|
||||
entries.emplace(name, type);
|
||||
}
|
||||
|
||||
if (errno)
|
||||
throw SysError("reading directory %1%", path);
|
||||
|
||||
return entries;
|
||||
} catch (SymlinkNotAllowed & e) {
|
||||
throw SymlinkNotAllowed(e.path, "path '%s' is a symlink", showPath(e.path));
|
||||
}
|
||||
|
||||
void UnixDirectorySourceAccessor::readDirectory(
|
||||
const CanonPath & dirPath,
|
||||
std::function<void(SourceAccessor & subdirAccessor, const CanonPath & subdirRelPath)> callback)
|
||||
{
|
||||
auto fd = openSubdirectory(dirPath);
|
||||
UnixDirectorySourceAccessor accessor{std::move(fd), rootPath / dirPath, trackLastModified, /*dirFdCacheSize=*/0};
|
||||
callback(accessor, CanonPath::root);
|
||||
}
|
||||
|
||||
std::string UnixDirectorySourceAccessor::readLink(const CanonPath & path)
|
||||
try {
|
||||
if (path.isRoot())
|
||||
throw NotASymlink("file '%s' is not a symlink", showPath(path));
|
||||
|
||||
auto [parentFd, parentFdOwning] = openParent(path);
|
||||
if (parentFd == INVALID_DESCRIPTOR)
|
||||
throw FileNotFound("file '%s' does not exist", showPath(path));
|
||||
|
||||
if (dirFdCache && parentFdOwning) {
|
||||
assert(*parentFdOwning);
|
||||
dirFdCache->lock()->upsert(path.parent().value(), ref<AutoCloseFD>(std::move(parentFdOwning)));
|
||||
}
|
||||
|
||||
try {
|
||||
return readLinkAt(parentFd, CanonPath(path.baseName().value()));
|
||||
} catch (SysError & e) {
|
||||
if (e.errNo == EINVAL)
|
||||
throw NotASymlink("file '%s' is not a symlink", showPath(path));
|
||||
throw;
|
||||
}
|
||||
} catch (SymlinkNotAllowed & e) {
|
||||
throw SymlinkNotAllowed(e.path, "path '%s' is a symlink", showPath(e.path));
|
||||
}
|
||||
|
||||
namespace {
|
||||
|
||||
class SymlinkSourceAccessor : public MemorySourceAccessor
|
||||
{
|
||||
bool trackLastModified;
|
||||
std::time_t mtime;
|
||||
CanonPath rootPath;
|
||||
|
||||
public:
|
||||
SymlinkSourceAccessor(std::string target, CanonPath rootPath_, bool trackLastModified, std::time_t mtime)
|
||||
: trackLastModified(trackLastModified)
|
||||
, mtime(mtime)
|
||||
, rootPath(std::move(rootPath_))
|
||||
{
|
||||
MemorySink sink{*this};
|
||||
sink.createSymlink(CanonPath::root, target);
|
||||
displayPrefix = rootPath.abs();
|
||||
}
|
||||
|
||||
std::optional<std::time_t> getLastModified() override
|
||||
{
|
||||
return trackLastModified ? std::optional{mtime} : std::nullopt;
|
||||
}
|
||||
|
||||
std::optional<std::filesystem::path> getPhysicalPath(const CanonPath & path) override
|
||||
{
|
||||
auto fsRootPath = std::filesystem::path(rootPath.abs());
|
||||
if (path.isRoot())
|
||||
return fsRootPath;
|
||||
return fsRootPath / path.rel(); /* RHS must be a relative path. */
|
||||
}
|
||||
|
||||
std::string showPath(const CanonPath & path) override
|
||||
{
|
||||
/* When rendering the file itself omit the trailing slash. */
|
||||
return path.isRoot() ? displayPrefix : SourceAccessor::showPath(path);
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace
|
||||
|
||||
ref<SourceAccessor> getFSSourceAccessor()
|
||||
{
|
||||
static auto rootFS =
|
||||
make_ref<UnixDirectorySourceAccessor>(openDirectory("/"), CanonPath("/"), /*trackLastModified=*/false);
|
||||
return rootFS;
|
||||
}
|
||||
|
||||
ref<SourceAccessor> makeFSSourceAccessor(std::filesystem::path root, bool trackLastModified)
|
||||
{
|
||||
using namespace unix;
|
||||
|
||||
if (root.empty())
|
||||
return getFSSourceAccessor();
|
||||
|
||||
assert(root.is_absolute());
|
||||
auto rootPath = CanonPath(root.native());
|
||||
assert(rootPath.abs().starts_with("/")); /* In case the invariant is broken somehow. */
|
||||
|
||||
/* Any symlinks get resolved eagerly here. Unlike with SourceAccessor semantics that requires that
|
||||
all links get resolved manually, the root can be resolved eagerly. */
|
||||
AutoCloseFD fd(::open(rootPath.c_str(), O_RDONLY | O_CLOEXEC | O_NOFOLLOW));
|
||||
|
||||
if (!fd) {
|
||||
if (errno == ELOOP) /* Opening a symlink, can read it straight into memory source accessor. */ {
|
||||
auto parent = rootPath.parent().value(); /* Always present, isRoot is handled above. */
|
||||
auto name = std::string(rootPath.baseName().value());
|
||||
AutoCloseFD parentFd(::open(parent.c_str(), O_RDONLY | O_DIRECTORY | O_CLOEXEC));
|
||||
if (!parentFd)
|
||||
throw SysError("opening '%s'", parent.abs());
|
||||
|
||||
struct ::stat st;
|
||||
if (::fstatat(parentFd.get(), name.c_str(), &st, AT_SYMLINK_NOFOLLOW) == -1)
|
||||
throw SysError("statting '%s' relative to parent directory '%s'", name, parent.abs());
|
||||
|
||||
auto target = readLinkAt(parentFd.get(), CanonPath(name));
|
||||
return make_ref<SymlinkSourceAccessor>(
|
||||
std::move(target), std::move(rootPath), trackLastModified, st.st_mtime);
|
||||
}
|
||||
|
||||
throw SysError("opening '%s'", rootPath.abs());
|
||||
}
|
||||
|
||||
struct ::stat st;
|
||||
if (::fstat(fd.get(), &st) == -1)
|
||||
throw SysError("statting '%s'", rootPath.abs());
|
||||
|
||||
if (S_ISDIR(st.st_mode))
|
||||
return make_ref<UnixDirectorySourceAccessor>(std::move(fd), std::move(rootPath), trackLastModified, /*dirFdCacheSize=*/0);
|
||||
|
||||
/* TODO: Ban non-regular files that cannot file represented by the FSO
|
||||
semantics. See comment in UnixFileSourceAccessor::readFile. */
|
||||
return make_ref<UnixFileSourceAccessor>(std::move(fd), std::move(rootPath), trackLastModified, &st);
|
||||
}
|
||||
|
||||
} // namespace nix
|
||||
+18
-6
@@ -417,12 +417,24 @@ ParsedURL fixGitURL(std::string url)
|
||||
std::regex scpRegex("([^/]*)@(.*):(.*)");
|
||||
if (!hasPrefix(url, "/") && std::regex_match(url, scpRegex))
|
||||
url = std::regex_replace(url, scpRegex, "ssh://$1@$2/$3");
|
||||
if (!hasPrefix(url, "file:") && !hasPrefix(url, "git+file:") && url.find("://") == std::string::npos)
|
||||
return ParsedURL{
|
||||
.scheme = "file",
|
||||
.authority = ParsedURL::Authority{},
|
||||
.path = splitString<std::vector<std::string>>(url, "/"),
|
||||
};
|
||||
if (!hasPrefix(url, "file:") && !hasPrefix(url, "git+file:") && url.find("://") == std::string::npos) {
|
||||
auto path = splitString<std::vector<std::string>>(url, "/");
|
||||
// Reject SCP-like URLs without user (e.g., "github.com:path") - colon in first component
|
||||
if (!path.empty() && path[0].find(':') != std::string::npos)
|
||||
throw BadURL("SCP-like URL '%s' is not supported; use SSH URL syntax instead (ssh://...)", url);
|
||||
// Absolute paths get an empty authority (file:///path), relative paths get none (file:path)
|
||||
if (hasPrefix(url, "/"))
|
||||
return ParsedURL{
|
||||
.scheme = "file",
|
||||
.authority = ParsedURL::Authority{},
|
||||
.path = path,
|
||||
};
|
||||
else
|
||||
return ParsedURL{
|
||||
.scheme = "file",
|
||||
.path = path,
|
||||
};
|
||||
}
|
||||
auto parsed = parseURL(url);
|
||||
// Drop the superfluous "git+" from the scheme.
|
||||
auto scheme = parseUrlScheme(parsed.scheme);
|
||||
|
||||
@@ -38,7 +38,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand
|
||||
if (!namePart)
|
||||
namePart = baseNameOf(path);
|
||||
|
||||
auto sourcePath = makeFSSourceAccessor(makeParentCanonical(path));
|
||||
auto sourcePath = PosixSourceAccessor::createAtRoot(makeParentCanonical(path));
|
||||
|
||||
auto storePath = dryRun ? store->computeStorePath(*namePart, sourcePath, caMethod, hashAlgo, {}).first
|
||||
: store->addToStoreSlow(*namePart, sourcePath, caMethod, hashAlgo, {}).path;
|
||||
|
||||
+3
-3
@@ -52,7 +52,7 @@ struct CmdCatStore : StoreCommand, MixCat
|
||||
|
||||
struct CmdCatNar : StoreCommand, MixCat
|
||||
{
|
||||
Path narPath;
|
||||
std::filesystem::path narPath;
|
||||
|
||||
std::string path;
|
||||
|
||||
@@ -76,9 +76,9 @@ struct CmdCatNar : StoreCommand, MixCat
|
||||
|
||||
void run(ref<Store> store) override
|
||||
{
|
||||
AutoCloseFD fd = toDescriptor(open(narPath.c_str(), O_RDONLY));
|
||||
AutoCloseFD fd = openFileReadonly(narPath);
|
||||
if (!fd)
|
||||
throw SysError("opening NAR file '%s'", narPath);
|
||||
throw NativeSysError("opening NAR file %s", narPath);
|
||||
auto source = FdSource{fd.get()};
|
||||
|
||||
struct CatRegularFileSink : NullFileSystemObjectSink
|
||||
|
||||
+3
-1
@@ -85,7 +85,9 @@ struct CmdHashBase : Command
|
||||
return std::make_unique<HashSink>(hashAlgo);
|
||||
};
|
||||
|
||||
auto makeSourcePath = [&]() -> SourcePath { return makeFSSourceAccessor(makeParentCanonical(path)); };
|
||||
auto makeSourcePath = [&]() -> SourcePath {
|
||||
return PosixSourceAccessor::createAtRoot(makeParentCanonical(path));
|
||||
};
|
||||
|
||||
Hash h{HashAlgorithm::SHA256}; // throwaway def to appease C++
|
||||
switch (mode) {
|
||||
|
||||
+3
-3
@@ -126,7 +126,7 @@ struct CmdLsStore : StoreCommand, MixLs
|
||||
|
||||
struct CmdLsNar : Command, MixLs
|
||||
{
|
||||
Path narPath;
|
||||
std::filesystem::path narPath;
|
||||
|
||||
std::string path;
|
||||
|
||||
@@ -150,9 +150,9 @@ struct CmdLsNar : Command, MixLs
|
||||
|
||||
void run() override
|
||||
{
|
||||
AutoCloseFD fd = toDescriptor(open(narPath.c_str(), O_RDONLY));
|
||||
AutoCloseFD fd = openFileReadonly(narPath);
|
||||
if (!fd)
|
||||
throw SysError("opening NAR file '%s'", narPath);
|
||||
throw NativeSysError("opening NAR file %s", narPath);
|
||||
auto source = FdSource{fd.get()};
|
||||
list(makeLazyNarAccessor(source, seekableGetNarBytes(fd.get())), CanonPath{path});
|
||||
}
|
||||
|
||||
@@ -191,7 +191,7 @@ static void opAdd(Strings opFlags, Strings opArgs)
|
||||
throw UsageError("unknown flag");
|
||||
|
||||
for (auto & i : opArgs) {
|
||||
auto sourcePath = makeFSSourceAccessor(makeParentCanonical(i));
|
||||
auto sourcePath = PosixSourceAccessor::createAtRoot(makeParentCanonical(i));
|
||||
cout << fmt("%s\n", store->printStorePath(store->addToStore(std::string(baseNameOf(i)), sourcePath)));
|
||||
}
|
||||
}
|
||||
@@ -215,7 +215,7 @@ static void opAddFixed(Strings opFlags, Strings opArgs)
|
||||
opArgs.pop_front();
|
||||
|
||||
for (auto & i : opArgs) {
|
||||
auto sourcePath = makeFSSourceAccessor(makeParentCanonical(i));
|
||||
auto sourcePath = PosixSourceAccessor::createAtRoot(makeParentCanonical(i));
|
||||
std::cout << fmt(
|
||||
"%s\n", store->printStorePath(store->addToStoreSlow(baseNameOf(i), sourcePath, method, hashAlgo).path));
|
||||
}
|
||||
|
||||
@@ -256,7 +256,7 @@ hashPath(char * algo, int base32, char * path)
|
||||
PPCODE:
|
||||
try {
|
||||
Hash h = hashPath(
|
||||
makeFSSourceAccessor(absPath(Path(path))),
|
||||
PosixSourceAccessor::createAtRoot(path),
|
||||
FileIngestionMethod::NixArchive, parseHashAlgo(algo)).first;
|
||||
auto s = h.to_string(base32 ? HashFormat::Nix32 : HashFormat::Base16, false);
|
||||
XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
|
||||
@@ -336,7 +336,7 @@ StoreWrapper::addToStore(char * srcPath, int recursive, char * algo)
|
||||
auto method = recursive ? ContentAddressMethod::Raw::NixArchive : ContentAddressMethod::Raw::Flat;
|
||||
auto path = THIS->store->addToStore(
|
||||
std::string(baseNameOf(srcPath)),
|
||||
makeFSSourceAccessor(absPath(Path(srcPath))),
|
||||
PosixSourceAccessor::createAtRoot(srcPath),
|
||||
method, parseHashAlgo(algo));
|
||||
XPUSHs(sv_2mortal(newSVpv(THIS->store->printStorePath(path).c_str(), 0)));
|
||||
} catch (Error & e) {
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
error:
|
||||
… while calling the 'fetchTree' builtin
|
||||
at /pwd/lang/eval-fail-fetchTree-relative-path.nix:1:1:
|
||||
1| builtins.fetchTree "file:./relative-path.tar.gz"
|
||||
| ^
|
||||
2|
|
||||
|
||||
… while fetching the input 'file:./relative-path.tar.gz'
|
||||
|
||||
error: tarball 'file:./relative-path.tar.gz' must use an absolute path. The 'file' scheme does not support relative paths.
|
||||
@@ -0,0 +1 @@
|
||||
builtins.fetchTree "file:./relative-path.tar.gz"
|
||||
@@ -13,4 +13,4 @@ error:
|
||||
| ^
|
||||
3| }
|
||||
|
||||
error: '/pwd/lang/readDir/bar' is not a directory
|
||||
error: cannot read directory "/pwd/lang/readDir/bar": Not a directory
|
||||
@@ -13,4 +13,4 @@ error:
|
||||
| ^
|
||||
3| }
|
||||
|
||||
error: '/pwd/lang/readDir/foo/git-hates-directories' is not a directory
|
||||
error: cannot read directory "/pwd/lang/readDir/foo/git-hates-directories": Not a directory
|
||||
Reference in new issue
Block a user