Befator-Inc-Firmen-Netzwerk/nix
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #15211 from obsidiansystems/worker-settings

Browse Source 
libstore: extract `WorkerSettings` from `Settings`
This commit is contained in:
John Ericson
2026-02-12 21:15:43 +00:00
committed by GitHub
parent eae7e0151c d3388d3d81
commit a3d51172e9
36 changed files with 605 additions and 526 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
src/libfetchers/cache.cc
View File

@@ -34,14 +34,20 @@ struct CacheImpl : Cache
Sync<State> _state;
CacheImpl()
/**
* This is a back-reference to the `Settings` that owns us.
*/
const Settings & settings;
CacheImpl(const Settings & _settings)
: settings(_settings)
{
auto state(_state.lock());
auto dbPath = (getCacheDir() / "fetcher-cache-v4.sqlite").string();
createDirs(dirOf(dbPath));
state->db = SQLite(dbPath, {.useWAL = settings.useSQLiteWAL});
state->db = SQLite(dbPath, {.useWAL = nix::settings.useSQLiteWAL});
state->db.isCache();
state->db.exec(schema);
@@ -154,7 +160,7 @@ ref<Cache> Settings::getCache() const
{
auto cache(_cache.lock());
if (!*cache)
*cache = std::make_shared<CacheImpl>();
*cache = std::make_shared<CacheImpl>(*this);
return ref<Cache>(*cache);
}

14
src/libfetchers/git.cc
View File

@@ -37,7 +37,7 @@ namespace {
// old version of git, which will ignore unrecognized `-c` options.
const std::string gitInitialBranch = "__nix_dummy_branch";
bool isCacheFileWithinTtl(time_t now, const PosixStat & st)
static bool isCacheFileWithinTtl(const Settings & settings, time_t now, const PosixStat & st)
{
return st.st_mtime + static_cast<time_t>(settings.tarballTtl) > now;
}
@@ -105,7 +105,7 @@ bool storeCachedHead(const std::string & actualUrl, bool shallow, const std::str
return true;
}
std::optional<std::string> readHeadCached(const std::string & actualUrl, bool shallow)
static std::optional<std::string> readHeadCached(const Settings & settings, const std::string & actualUrl, bool shallow)
{
// Create a cache path to store the branch of the HEAD ref. Append something
// in front of the URL to prevent collision with the repository itself.
@@ -117,7 +117,7 @@ std::optional<std::string> readHeadCached(const std::string & actualUrl, bool sh
std::optional<std::string> cachedRef;
if (st) {
cachedRef = readHead(cacheDir);
if (cachedRef != std::nullopt && *cachedRef != gitInitialBranch && isCacheFileWithinTtl(now, *st)) {
if (cachedRef != std::nullopt && *cachedRef != gitInitialBranch && isCacheFileWithinTtl(settings, now, *st)) {
debug("using cached HEAD ref '%s' for repo '%s'", *cachedRef, actualUrl);
return cachedRef;
}
@@ -728,12 +728,12 @@ struct GitInputScheme : InputScheme
return revCount;
}
std::string getDefaultRef(const RepoInfo & repoInfo, bool shallow) const
std::string getDefaultRef(const Settings & settings, const RepoInfo & repoInfo, bool shallow) const
{
auto head = std::visit(
overloaded{
[&](const std::filesystem::path & path) { return GitRepo::openRepo(path, {})->getWorkdirRef(); },
[&](const ParsedURL & url) { return readHeadCached(url.to_string(), shallow); }},
[&](const ParsedURL & url) { return readHeadCached(settings, url.to_string(), shallow); }},
repoInfo.location);
if (!head) {
warn("could not read HEAD ref from repo at '%s', using 'master'", repoInfo.locationToArg());
@@ -783,7 +783,7 @@ struct GitInputScheme : InputScheme
auto originalRef = input.getRef();
bool shallow = getShallowAttr(input);
auto ref = originalRef ? *originalRef : getDefaultRef(repoInfo, shallow);
auto ref = originalRef ? *originalRef : getDefaultRef(settings, repoInfo, shallow);
input.attrs.insert_or_assign("ref", ref);
std::filesystem::path repoDir;
@@ -822,7 +822,7 @@ struct GitInputScheme : InputScheme
/* If the local ref is older than 'tarball-ttl' seconds, do a
git fetch to update the local ref to the remote ref. */
auto st = maybeStat(localRefFile);
doFetch = !st || !isCacheFileWithinTtl(now, *st);
doFetch = !st || !isCacheFileWithinTtl(settings, now, *st);
}
}

19
src/libfetchers/include/nix/fetchers/fetch-settings.hh
View File

@@ -129,6 +129,25 @@ struct Settings : public Config
true,
Xp::Flakes};
Setting<unsigned int> tarballTtl{
this,
60 * 60,
"tarball-ttl",
R"(
The number of seconds a downloaded tarball is considered fresh. If
the cached tarball is stale, Nix checks whether it is still up
to date using the ETag header. Nix downloads a new version if
the ETag header is unsupported, or the cached ETag doesn't match.
Setting the TTL to `0` forces Nix to always check if the tarball is
up to date.
Nix caches tarballs in `$XDG_CACHE_HOME/nix/tarballs`.
Files fetched via `NIX_PATH`, `fetchGit`, `fetchMercurial`,
`fetchTarball`, and `fetchurl` respect this TTL.
)"};
ref<Cache> getCache() const;
ref<GitRepo> getTarballCache() const;

4
src/libmain/shared.cc
View File

@@ -233,13 +233,13 @@ LegacyArgs::LegacyArgs(
.longName = "keep-going",
.shortName = 'k',
.description = "Keep going after a build fails.",
.handler = {&(bool &) settings.keepGoing, true},
.handler = {&(bool &) settings.getWorkerSettings().keepGoing, true},
});
addFlag({
.longName = "fallback",
.description = "Build from source if substitution fails.",
.handler = {&(bool &) settings.tryFallback, true},
.handler = {&(bool &) settings.getWorkerSettings().tryFallback, true},
});
auto intSettingAlias =

4
src/libstore-test-support/test-main.cc
View File

@@ -15,10 +15,10 @@ int testMainForBuidingPre(int argc, char ** argv)
}
// Disable build hook. We won't be testing remote builds in these unit tests. If we do, fix the above build hook.
settings.buildHook = {};
settings.getWorkerSettings().buildHook = {};
// No substituters, unless a test specifically requests.
settings.substituters = {};
settings.getWorkerSettings().substituters = {};
#ifdef __linux__ // should match the conditional around sandboxBuildDir declaration.

9
src/libstore-tests/derivation-advanced-attrs.cc
View File

@@ -5,6 +5,7 @@
#include "nix/store/derivations.hh"
#include "nix/store/derived-path.hh"
#include "nix/store/derivation-options.hh"
#include "nix/store/globals.hh"
#include "nix/store/parsed-derivations.hh"
#include "nix/util/types.hh"
#include "nix/util/json-utils.hh"
@@ -194,7 +195,7 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_defaults)
EXPECT_EQ(options.canBuildLocally(*this->store, got), false);
EXPECT_EQ(options.willBuildLocally(*this->store, got), false);
EXPECT_EQ(options.substitutesAllowed(), true);
EXPECT_EQ(options.substitutesAllowed(settings.getWorkerSettings()), true);
EXPECT_EQ(options.useUidRange(got), false);
});
};
@@ -242,7 +243,7 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes)
EXPECT_EQ(options, expected);
EXPECT_EQ(options.substitutesAllowed(), false);
EXPECT_EQ(options.substitutesAllowed(settings.getWorkerSettings()), false);
EXPECT_EQ(options.useUidRange(got), true);
});
};
@@ -336,7 +337,7 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_structuredAttrs_d
EXPECT_EQ(options.canBuildLocally(*this->store, got), false);
EXPECT_EQ(options.willBuildLocally(*this->store, got), false);
EXPECT_EQ(options.substitutesAllowed(), true);
EXPECT_EQ(options.substitutesAllowed(settings.getWorkerSettings()), true);
EXPECT_EQ(options.useUidRange(got), false);
});
};
@@ -403,7 +404,7 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_structuredAttrs)
EXPECT_EQ(options.canBuildLocally(*this->store, got), false);
EXPECT_EQ(options.willBuildLocally(*this->store, got), false);
EXPECT_EQ(options.substitutesAllowed(), false);
EXPECT_EQ(options.substitutesAllowed(settings.getWorkerSettings()), false);
EXPECT_EQ(options.useUidRange(got), true);
});
};

12
src/libstore-tests/nix_api_store.cc
View File

@@ -299,7 +299,7 @@ public:
nix_api_store_test_base::SetUp();
nix::experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations");
nix::settings.substituters = {};
nix::settings.getWorkerSettings().substituters = {};
store = open_local_store();
@@ -354,7 +354,7 @@ TEST_F(nix_api_store_test_base, build_from_json)
{
// FIXME get rid of these
nix::experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations");
nix::settings.substituters = {};
nix::settings.getWorkerSettings().substituters = {};
auto * store = open_local_store();
@@ -401,7 +401,7 @@ TEST_F(nix_api_store_test_base, nix_store_realise_invalid_system)
{
// Test that nix_store_realise properly reports errors when the system is invalid
nix::experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations");
nix::settings.substituters = {};
nix::settings.getWorkerSettings().substituters = {};
auto * store = open_local_store();
@@ -446,7 +446,7 @@ TEST_F(nix_api_store_test_base, nix_store_realise_builder_fails)
{
// Test that nix_store_realise properly reports errors when the builder fails
nix::experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations");
nix::settings.substituters = {};
nix::settings.getWorkerSettings().substituters = {};
auto * store = open_local_store();
@@ -491,7 +491,7 @@ TEST_F(nix_api_store_test_base, nix_store_realise_builder_no_output)
{
// Test that nix_store_realise properly reports errors when builder succeeds but produces no output
nix::experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations");
nix::settings.substituters = {};
nix::settings.getWorkerSettings().substituters = {};
auto * store = open_local_store();
@@ -687,7 +687,7 @@ TEST_F(NixApiStoreTestWithRealisedPath, nix_store_realise_output_ordering)
// This test uses a CA derivation with 10 outputs in randomized input order
// to verify that the callback order is deterministic and alphabetical.
nix::experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations");
nix::settings.substituters = {};
nix::settings.getWorkerSettings().substituters = {};
auto * store = open_local_store();

47
src/libstore/build/derivation-building-goal.cc
View File

@@ -63,7 +63,11 @@ std::string showKnownOutputs(const StoreDirConfig & store, const Derivation & dr
}
static void runPostBuildHook(
const StoreDirConfig & store, Logger & logger, const StorePath & drvPath, const StorePathSet & outputPaths);
const WorkerSettings & workerSettings,
const StoreDirConfig & store,
Logger & logger,
const StorePath & drvPath,
const StorePathSet & outputPaths);
/* At least one of the output paths could not be
produced using a substitute. So we have to build instead. */
@@ -87,7 +91,7 @@ Goal::Co DerivationBuildingGoal::gaveUpOnSubstitution(bool storeDerivation)
for (auto & i : drv->inputSrcs) {
if (worker.store.isValidPath(i))
continue;
if (!settings.useSubstitutes)
if (!worker.settings.useSubstitutes)
throw Error(
"dependency '%s' of '%s' does not exist, and substitution is disabled",
worker.store.printStorePath(i),
@@ -325,7 +329,7 @@ Goal::Co DerivationBuildingGoal::tryToBuild(StorePathSet inputPaths)
`preferLocalBuild' set. Also, check and repair modes are only
supported for local builds. */
bool buildLocally = (buildMode != bmNormal || drvOptions.willBuildLocally(worker.store, *drv))
&& settings.maxBuildJobs.get() != 0;
&& worker.settings.maxBuildJobs.get() != 0;
if (buildLocally) {
useHook = false;
@@ -476,7 +480,7 @@ Goal::Co DerivationBuildingGoal::buildWithHook(
msg += fmt(" on '%s'", hook->machineName);
std::unique_ptr<BuildLog> buildLog = std::make_unique<BuildLog>(
settings.logLines,
worker.settings.logLines,
std::make_unique<Activity>(
*logger,
lvlInfo,
@@ -496,7 +500,7 @@ Goal::Co DerivationBuildingGoal::buildWithHook(
auto & data = output->data;
if (fd == hook->builderOut.readSide.get()) {
logSize += data.size();
if (settings.maxLogSize && logSize > settings.maxLogSize) {
if (worker.settings.maxLogSize && logSize > worker.settings.maxLogSize) {
hook.reset();
co_return doneFailureLogTooLong(*buildLog);
}
@@ -605,7 +609,7 @@ Goal::Co DerivationBuildingGoal::buildWithHook(
StorePathSet outputPaths;
for (auto & [_, output] : builtOutputs)
outputPaths.insert(output.outPath);
runPostBuildHook(worker.store, *logger, drvPath, outputPaths);
runPostBuildHook(worker.settings, worker.store, *logger, drvPath, outputPaths);
/* It is now safe to delete the lock files, since all future
lockers will see that the output paths are valid; they will
@@ -647,7 +651,7 @@ Goal::Co DerivationBuildingGoal::buildLocally(
: "building '%s'",
worker.store.printStorePath(drvPath));
buildLog = std::make_unique<BuildLog>(
settings.logLines,
worker.settings.logLines,
std::make_unique<Activity>(
*logger, lvlInfo, actBuild, msg, Logger::Fields{worker.store.printStorePath(drvPath), "", 1, 1}));
mcRunningBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.runningBuilds);
@@ -662,7 +666,7 @@ Goal::Co DerivationBuildingGoal::buildLocally(
while (true) {
unsigned int curBuilds = worker.getNrLocalBuilds();
if (curBuilds >= settings.maxBuildJobs) {
if (curBuilds >= worker.settings.maxBuildJobs) {
outputLocks.unlock();
co_await waitForBuildSlot();
co_return tryToBuild(std::move(inputPaths));
@@ -791,7 +795,7 @@ Goal::Co DerivationBuildingGoal::buildLocally(
if (auto * output = std::get_if<ChildOutput>(&event)) {
if (output->fd == builder->builderOut.get()) {
logSize += output->data.size();
if (settings.maxLogSize && logSize > settings.maxLogSize) {
if (worker.settings.maxLogSize && logSize > worker.settings.maxLogSize) {
builder->killChild();
co_return doneFailureLogTooLong(*buildLog);
}
@@ -838,7 +842,7 @@ Goal::Co DerivationBuildingGoal::buildLocally(
worker.markContentsGood(output.outPath);
outputPaths.insert(output.outPath);
}
runPostBuildHook(worker.store, *logger, drvPath, outputPaths);
runPostBuildHook(worker.settings, worker.store, *logger, drvPath, outputPaths);
/* It is now safe to delete the lock files, since all future
lockers will see that the output paths are valid; they will
@@ -852,9 +856,13 @@ Goal::Co DerivationBuildingGoal::buildLocally(
}
static void runPostBuildHook(
const StoreDirConfig & store, Logger & logger, const StorePath & drvPath, const StorePathSet & outputPaths)
const WorkerSettings & workerSettings,
const StoreDirConfig & store,
Logger & logger,
const StorePath & drvPath,
const StorePathSet & outputPaths)
{
auto hook = settings.postBuildHook;
auto hook = workerSettings.postBuildHook;
if (hook == "")
return;
@@ -862,7 +870,7 @@ static void runPostBuildHook(
logger,
lvlTalkative,
actPostBuildHook,
fmt("running post-build-hook '%s'", settings.postBuildHook),
fmt("running post-build-hook '%s'", workerSettings.postBuildHook),
Logger::Fields{store.printStorePath(drvPath)});
PushActivity pact(act.id);
StringMap hookEnvironment = getEnv();
@@ -910,7 +918,7 @@ static void runPostBuildHook(
LogSink sink(act);
runProgram2({
.program = settings.postBuildHook,
.program = workerSettings.postBuildHook,
.environment = hookEnvironment,
.standardOut = &sink,
.mergeStderrToStdout = true,
@@ -957,17 +965,18 @@ HookReply DerivationBuildingGoal::tryBuildHook(const DerivationOptions<StorePath
#else
/* This should use `worker.evalStore`, but per #13179 the build hook
doesn't work with eval store anyways. */
if (settings.buildHook.get().empty() || !worker.tryBuildHook || !worker.store.isValidPath(drvPath))
if (worker.settings.buildHook.get().empty() || !worker.tryBuildHook || !worker.store.isValidPath(drvPath))
return rpDecline;
if (!worker.hook)
worker.hook = std::make_unique<HookInstance>();
worker.hook = std::make_unique<HookInstance>(worker.settings.buildHook);
try {
/* Send the request to the hook. */
worker.hook->sink << "try" << (worker.getNrLocalBuilds() < settings.maxBuildJobs ? 1 : 0) << drv->platform
<< worker.store.printStorePath(drvPath) << drvOptions.getRequiredSystemFeatures(*drv);
worker.hook->sink << "try" << (worker.getNrLocalBuilds() < worker.settings.maxBuildJobs ? 1 : 0)
<< drv->platform << worker.store.printStorePath(drvPath)
<< drvOptions.getRequiredSystemFeatures(*drv);
worker.hook->sink.flush();
/* Read the first line of input, which should be a word indicating
@@ -1073,7 +1082,7 @@ Goal::Done DerivationBuildingGoal::doneFailureLogTooLong(BuildLog & buildLog)
BuildResult::Failure::LogLimitExceeded,
"%s killed after writing more than %d bytes of log output",
getName(),
settings.maxLogSize));
worker.settings.maxLogSize));
}
std::map<std::string, std::optional<StorePath>> DerivationBuildingGoal::queryPartialDerivationOutputMap()

4
src/libstore/build/derivation-goal.cc
View File

@@ -100,7 +100,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation)
/* We are first going to try to create the invalid output paths
through substitutes. If that doesn't work, we'll build
them. */
if (settings.useSubstitutes && drvOptions.substitutesAllowed()) {
if (worker.settings.useSubstitutes && drvOptions.substitutesAllowed(worker.settings)) {
if (!checkResult) {
DrvOutput id{outputHash, wantedOutput};
auto g = worker.makeDrvOutputSubstitutionGoal(id);
@@ -133,7 +133,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation)
assert(!drv->type().isImpure());
if (nrFailed > 0 && nrFailed > nrNoSubstituters && !settings.tryFallback) {
if (nrFailed > 0 && nrFailed > nrNoSubstituters && !worker.settings.tryFallback) {
co_return doneFailure(BuildError(
BuildResult::Failure::TransientFailure,
"some substitutes for the outputs of derivation '%s' failed (usually happens due to networking issues); try '--fallback' to build derivation from source ",

2
src/libstore/build/goal.cc
View File

@@ -232,7 +232,7 @@ Goal::Done Goal::amDone(ExitCode result)
if (goal->waitees.empty()) {
worker.wakeUp(goal);
} else if (result == ecFailed && !settings.keepGoing) {
} else if (result == ecFailed && !worker.settings.keepGoing) {
/* If we failed and keepGoing is not set, we remove all
remaining waitees. */
for (auto & g : goal->waitees) {

4
src/libstore/build/substitution-goal.cc
View File

@@ -145,7 +145,7 @@ Goal::Co PathSubstitutionGoal::init()
worker.updateProgress();
}
if (lastStoresException.has_value()) {
if (!settings.tryFallback) {
if (!worker.settings.tryFallback) {
throw *lastStoresException;
} else
logError(lastStoresException->info());
@@ -197,7 +197,7 @@ Goal::Co PathSubstitutionGoal::tryToRun(
/* Make sure that we are allowed to start a substitution. Note that even
if maxSubstitutionJobs == 0, we still allow a substituter to run. This
prevents infinite waiting. */
while (worker.getNrSubstitutions() >= std::max(1U, (unsigned int) settings.maxSubstitutionJobs)) {
while (worker.getNrSubstitutions() >= std::max(1U, (unsigned int) worker.settings.maxSubstitutionJobs)) {
co_await waitForBuildSlot();
}

5
src/libstore/build/worker.cc
View File

@@ -22,7 +22,10 @@ Worker::Worker(Store & store, Store & evalStore)
, actSubstitutions(*logger, actCopyPaths)
, store(store)
, evalStore(evalStore)
, getSubstituters{[] { return settings.useSubstitutes ? getDefaultSubstituters() : std::list<ref<Store>>{}; }}
, settings(nix::settings.getWorkerSettings())
, getSubstituters{[] {
return nix::settings.getWorkerSettings().useSubstitutes ? getDefaultSubstituters() : std::list<ref<Store>>{};
}}
{
nrLocalBuilds = 0;
nrSubstitutions = 0;

21
src/libstore/daemon.cc
View File

@@ -233,14 +233,14 @@ struct ClientSettings
void apply(TrustedFlag trusted)
{
settings.keepFailed = keepFailed;
settings.keepGoing = keepGoing;
settings.tryFallback = tryFallback;
settings.getWorkerSettings().keepGoing = keepGoing;
settings.getWorkerSettings().tryFallback = tryFallback;
nix::verbosity = verbosity;
settings.maxBuildJobs.assign(maxBuildJobs);
settings.maxSilentTime = maxSilentTime;
settings.getWorkerSettings().maxBuildJobs.assign(maxBuildJobs);
settings.getWorkerSettings().maxSilentTime = maxSilentTime;
settings.verboseBuild = verboseBuild;
settings.buildCores = buildCores;
settings.useSubstitutes = useSubstitutes;
settings.getLocalSettings().buildCores = buildCores;
settings.getWorkerSettings().useSubstitutes = useSubstitutes;
for (auto & i : overrides) {
auto & name(i.first);
@@ -250,7 +250,7 @@ struct ClientSettings
if (name != res.name && res.aliases.count(name) == 0)
return false;
std::set<StoreReference> trusted = settings.trustedSubstituters;
for (auto & ref : settings.substituters.get())
for (auto & ref : settings.getWorkerSettings().substituters.get())
trusted.insert(ref);
std::vector<StoreReference> subs;
auto ss = tokenizeString<Strings>(value);
@@ -293,11 +293,12 @@ struct ClientSettings
"Ignoring the client-specified plugin-files.\n"
"The client specifying plugins to the daemon never made sense, and was removed in Nix >=2.14.");
} else if (
trusted || name == settings.buildTimeout.name || name == settings.maxSilentTime.name
|| name == settings.pollInterval.name || name == "connect-timeout"
trusted || name == settings.getWorkerSettings().buildTimeout.name
|| name == settings.getWorkerSettings().maxSilentTime.name
|| name == settings.getWorkerSettings().pollInterval.name || name == "connect-timeout"
|| (name == "builders" && value == ""))
settings.set(name, value);
else if (setSubstituters(settings.substituters))
else if (setSubstituters(settings.getWorkerSettings().substituters))
;
else
warn(

6
src/libstore/derivation-options.cc
View File

@@ -366,7 +366,7 @@ bool DerivationOptions<Input>::canBuildLocally(Store & localStore, const BasicDe
&& !drv.isBuiltin())
return false;
if (settings.maxBuildJobs.get() == 0 && !drv.isBuiltin())
if (settings.getWorkerSettings().maxBuildJobs.get() == 0 && !drv.isBuiltin())
return false;
for (auto & feature : getRequiredSystemFeatures(drv))
@@ -383,9 +383,9 @@ bool DerivationOptions<Input>::willBuildLocally(Store & localStore, const BasicD
}
template<typename Input>
bool DerivationOptions<Input>::substitutesAllowed() const
bool DerivationOptions<Input>::substitutesAllowed(const WorkerSettings & workerSettings) const
{
return settings.alwaysAllowSubstitutes ? true : allowSubstitutes;
return workerSettings.alwaysAllowSubstitutes ? true : allowSubstitutes;
}
template<typename Input>

2
src/libstore/http-binary-cache-store.cc
View File

@@ -102,7 +102,7 @@ std::optional<CompressionAlgo> HttpBinaryCacheStore::getCompressionMethod(const
void HttpBinaryCacheStore::maybeDisable()
{
auto state(_state.lock());
if (state->enabled && settings.tryFallback) {
if (state->enabled && settings.getWorkerSettings().tryFallback) {
int t = 60;
printError("disabling binary cache '%s' for %s seconds", config->getHumanReadableURI(), t);
state->enabled = false;

2
src/libstore/include/nix/store/build/worker.hh
View File

@@ -16,6 +16,7 @@
namespace nix {
/* Forward definition. */
struct WorkerSettings;
struct DerivationTrampolineGoal;
struct DerivationGoal;
struct DerivationResolutionGoal;
@@ -156,6 +157,7 @@ public:
Store & store;
Store & evalStore;
const WorkerSettings & settings;
/**
* Function to get the substituters to use for path substitution.

3
src/libstore/include/nix/store/derivation-options.hh
View File

@@ -10,6 +10,7 @@
#include "nix/util/json-impls.hh"
#include "nix/store/store-dir-config.hh"
#include "nix/store/downstream-placeholder.hh"
#include "nix/store/worker-settings.hh"
namespace nix {
@@ -193,7 +194,7 @@ struct DerivationOptions
*/
bool willBuildLocally(Store & localStore, const BasicDerivation & drv) const;
bool substitutesAllowed() const;
bool substitutesAllowed(const WorkerSettings & workerSettings) const;
/**
* @param drv See note on `getRequiredSystemFeatures`

33
src/libstore/include/nix/store/global-paths.hh Normal file
View File

@@ -0,0 +1,33 @@
#pragma once
///@file
#include <filesystem>
#include <vector>
namespace nix {
/**
* The directory where system configuration files are stored.
*
* This is needed very early during initialization, before a main
* `Settings` object can be constructed.
*/
const std::filesystem::path & nixConfDir();
/**
* The path to the system configuration file (`nix.conf`).
*/
static inline std::filesystem::path nixConfFile()
{
return nixConfDir() / "nix.conf";
}
/**
* A list of user configuration files to load.
*
* This is needed very early during initialization, before a main
* `Settings` object can be constructed.
*/
const std::vector<std::filesystem::path> & nixUserConfFiles();
} // namespace nix

435
src/libstore/include/nix/store/globals.hh
View File

@@ -9,51 +9,12 @@
#include "nix/store/build/derivation-builder.hh"
#include "nix/store/local-settings.hh"
#include "nix/store/store-reference.hh"
#include "nix/store/worker-settings.hh"
#include "nix/store/config.hh"
namespace nix {
struct MaxBuildJobsSetting : public BaseSetting<unsigned int>
{
MaxBuildJobsSetting(
Config * options,
unsigned int def,
const std::string & name,
const std::string & description,
const StringSet & aliases = {})
: BaseSetting<unsigned int>(def, true, name, description, aliases)
{
options->addSetting(this);
}
unsigned int parse(const std::string & str) const override;
};
/**
* The directory where system configuration files are stored.
*
* This is needed very early during initialization, before a main
* `Settings` object can be constructed.
*/
const std::filesystem::path & nixConfDir();
/**
* The path to the system configuration file (`nix.conf`).
*/
static inline std::filesystem::path nixConfFile()
{
return nixConfDir() / "nix.conf";
}
/**
* A list of user configuration files to load.
*
* This is needed very early during initialization, before a main
* `Settings` object can be constructed.
*/
const std::vector<std::filesystem::path> & nixUserConfFiles();
struct LogFileSettings : public virtual Config
{
/**
@@ -89,7 +50,7 @@ public:
{"build-compress-log"}};
};
class Settings : public virtual Config, private LocalSettings, private LogFileSettings
class Settings : public virtual Config, private LocalSettings, private LogFileSettings, private WorkerSettings
{
StringSet getDefaultSystemFeatures();
@@ -129,6 +90,19 @@ public:
return *this;
}
/**
* Get the worker settings.
*/
WorkerSettings & getWorkerSettings()
{
return *this;
}
const WorkerSettings & getWorkerSettings() const
{
return *this;
}
static unsigned int getDefaultCores();
/**
@@ -162,86 +136,11 @@ public:
Setting<bool> keepFailed{this, false, "keep-failed", "Whether to keep temporary directories of failed builds."};
Setting<bool> keepGoing{
this, false, "keep-going", "Whether to keep building derivations when another build fails."};
Setting<bool> tryFallback{
this,
false,
"fallback",
R"(
If set to `true`, Nix falls back to building from source if a
binary substitute fails. This is equivalent to the `--fallback`
flag. The default is `false`.
)",
{"build-fallback"}};
/**
* Whether to show build log output in real time.
*/
bool verboseBuild = true;
Setting<size_t> logLines{
this,
25,
"log-lines",
"The number of lines of the tail of "
"the log to show if a build fails."};
MaxBuildJobsSetting maxBuildJobs{
this,
1,
"max-jobs",
R"(
Maximum number of jobs that Nix tries to build locally in parallel.
The special value `auto` causes Nix to use the number of CPUs in your system.
Use `0` to disable local builds and directly use the remote machines specified in [`builders`](#conf-builders).
This doesn't affect derivations that have [`preferLocalBuild = true`](@docroot@/language/advanced-attributes.md#adv-attr-preferLocalBuild), which are always built locally.
> **Note**
>
> The number of CPU cores to use for each build job is independently determined by the [`cores`](#conf-cores) setting.
<!-- TODO(@fricklerhandwerk): would be good to have those shorthands for common options as part of the specification -->
The setting can be overridden using the `--max-jobs` (`-j`) command line switch.
)",
{"build-max-jobs"}};
Setting<unsigned int> maxSubstitutionJobs{
this,
16,
"max-substitution-jobs",
R"(
This option defines the maximum number of substitution jobs that Nix
tries to run in parallel. The default is `16`. The minimum value
one can choose is `1` and lower values are interpreted as `1`.
)",
{"substitution-max-jobs"}};
Setting<unsigned int> buildCores{
this,
0,
"cores",
R"(
Sets the value of the `NIX_BUILD_CORES` environment variable in the [invocation of the `builder` executable](@docroot@/store/building.md#builder-execution) of a derivation.
The `builder` executable can use this variable to control its own maximum amount of parallelism.
<!--
FIXME(@fricklerhandwerk): I don't think this should even be mentioned here.
A very generic example using `derivation` and `xargs` may be more appropriate to explain the mechanism.
Using `mkDerivation` as an example requires being aware of that there are multiple independent layers that are completely opaque here.
-->
For instance, in Nixpkgs, if the attribute `enableParallelBuilding` for the `mkDerivation` build helper is set to `true`, it passes the `-j${NIX_BUILD_CORES}` flag to GNU Make.
If set to `0`, nix will detect the number of CPU cores and pass this number via `NIX_BUILD_CORES`.
> **Note**
>
> The number of parallel local Nix build jobs is independently controlled with the [`max-jobs`](#conf-max-jobs) setting.
)",
{"build-cores"}};
/**
* Read-only mode. Don't copy stuff to the store, don't change
* the database.
@@ -277,219 +176,6 @@ public:
configuration option is set as the empty string.
)"};
Setting<time_t> maxSilentTime{
this,
0,
"max-silent-time",
R"(
This option defines the maximum number of seconds that a builder can
go without producing any data on standard output or standard error.
This is useful (for instance in an automated build system) to catch
builds that are stuck in an infinite loop, or to catch remote builds
that are hanging due to network problems. It can be overridden using
the `--max-silent-time` command line switch.
The value `0` means that there is no timeout. This is also the
default.
)",
{"build-max-silent-time"}};
Setting<time_t> buildTimeout{
this,
0,
"timeout",
R"(
This option defines the maximum number of seconds that a builder can
run. This is useful (for instance in an automated build system) to
catch builds that are stuck in an infinite loop but keep writing to
their standard output or standard error. It can be overridden using
the `--timeout` command line switch.
The value `0` means that there is no timeout. This is also the
default.
)",
{"build-timeout"}};
Setting<Strings> buildHook{
this,
{"nix", "__build-remote"},
"build-hook",
R"(
The path to the helper program that executes remote builds.
Nix communicates with the build hook over `stdio` using a custom protocol to request builds that cannot be performed directly by the Nix daemon.
The default value is the internal Nix binary that implements remote building.
> **Important**
>
> Change this setting only if you really know what you’re doing.
)"};
Setting<std::string> builders{
this,
"@" + (nixConfDir() / "machines").string(),
"builders",
R"(
A semicolon- or newline-separated list of build machines.
In addition to the [usual ways of setting configuration options](@docroot@/command-ref/conf-file.md), the value can be read from a file by prefixing its absolute path with `@`.
> **Example**
>
> This is the default setting:
>
> ```
> builders = @/etc/nix/machines
> ```
Each machine specification consists of the following elements, separated by spaces.
Only the first element is required.
To leave a field at its default, set it to `-`.
1. The URI of the remote store in the format `ssh://[username@]hostname[:port]`.
> **Example**
>
> `ssh://nix@mac`
For backward compatibility, `ssh://` may be omitted.
The hostname may be an alias defined in `~/.ssh/config`.
2. A comma-separated list of [Nix system types](@docroot@/development/building.md#system-type).
If omitted, this defaults to the local platform type.
> **Example**
>
> `aarch64-darwin`
It is possible for a machine to support multiple platform types.
> **Example**
>
> `i686-linux,x86_64-linux`
3. The SSH identity file to be used to log in to the remote machine.
If omitted, SSH uses its regular identities.
> **Example**
>
> `/home/user/.ssh/id_mac`
4. The maximum number of builds that Nix executes in parallel on the machine.
Typically this should be equal to the number of CPU cores.
5. The “speed factor”, indicating the relative speed of the machine as a positive integer.
If there are multiple machines of the right type, Nix prefers the fastest, taking load into account.
6. A comma-separated list of supported [system features](#conf-system-features).
A machine is only used to build a derivation if all the features in the derivation's [`requiredSystemFeatures`](@docroot@/language/advanced-attributes.html#adv-attr-requiredSystemFeatures) attribute are supported by that machine.
7. A comma-separated list of required [system features](#conf-system-features).
A machine is only used to build a derivation if all of the machine’s required features appear in the derivation’s [`requiredSystemFeatures`](@docroot@/language/advanced-attributes.html#adv-attr-requiredSystemFeatures) attribute.
8. The (base64-encoded) public host key of the remote machine.
If omitted, SSH uses its regular `known_hosts` file.
The value for this field can be obtained via `base64 -w0`.
> **Example**
>
> Multiple builders specified on the command line:
>
> ```console
> --builders 'ssh://mac x86_64-darwin ; ssh://beastie x86_64-freebsd'
> ```
> **Example**
>
> This specifies several machines that can perform `i686-linux` builds:
>
> ```
> nix@scratchy.labs.cs.uu.nl i686-linux /home/nix/.ssh/id_scratchy 8 1 kvm
> nix@itchy.labs.cs.uu.nl i686-linux /home/nix/.ssh/id_scratchy 8 2
> nix@poochie.labs.cs.uu.nl i686-linux /home/nix/.ssh/id_scratchy 1 2 kvm benchmark
> ```
>
> However, `poochie` only builds derivations that have the attribute
>
> ```nix
> requiredSystemFeatures = [ "benchmark" ];
> ```
>
> or
>
> ```nix
> requiredSystemFeatures = [ "benchmark" "kvm" ];
> ```
>
> `itchy` cannot do builds that require `kvm`, but `scratchy` does support such builds.
> For regular builds, `itchy` is preferred over `scratchy` because it has a higher speed factor.
For Nix to use substituters, the calling user must be in the [`trusted-users`](#conf-trusted-users) list.
> **Note**
>
> A build machine must be accessible via SSH and have Nix installed.
> `nix` must be available in `$PATH` for the user connecting over SSH.
> **Warning**
>
> If you are building via the Nix daemon (default), the Nix daemon user account on the local machine (that is, `root`) requires access to a user account on the remote machine (not necessarily `root`).
>
> If you can’t or don’t want to configure `root` to be able to access the remote machine, set [`store`](#conf-store) to any [local store](@docroot@/store/types/local-store.html), e.g. by passing `--store /tmp` to the command on the local machine.
To build only on remote machines and disable local builds, set [`max-jobs`](#conf-max-jobs) to 0.
If you want the remote machines to use substituters, set [`builders-use-substitutes`](#conf-builders-use-substitutes) to `true`.
)",
{},
false};
Setting<bool> alwaysAllowSubstitutes{
this,
false,
"always-allow-substitutes",
R"(
If set to `true`, Nix ignores the [`allowSubstitutes`](@docroot@/language/advanced-attributes.md) attribute in derivations and always attempt to use [available substituters](#conf-substituters).
)"};
Setting<bool> buildersUseSubstitutes{
this,
false,
"builders-use-substitutes",
R"(
If set to `true`, Nix instructs [remote build machines](#conf-builders) to use their own [`substituters`](#conf-substituters) if available.
It means that remote build hosts fetch as many dependencies as possible from their own substituters (e.g, from `cache.nixos.org`) instead of waiting for the local machine to upload them all.
This can drastically reduce build times if the network connection between the local machine and the remote build host is slow.
)"};
Setting<bool> useSubstitutes{
this,
true,
"substitute",
R"(
If set to `true` (default), Nix uses binary substitutes if
available. This option can be disabled to force building from
source.
)",
{"build-use-substitutes"}};
Setting<unsigned long> maxLogSize{
this,
0,
"max-build-log-size",
R"(
This option defines the maximum number of bytes that a builder can
write to its stdout/stderr. If the builder exceeds this limit, it’s
killed. A value of `0` (the default) means that there is no limit.
)",
{"build-max-log-size"}};
Setting<unsigned int> pollInterval{this, 5, "build-poll-interval", "How often (in seconds) to poll for locks."};
Setting<Strings> trustedPublicKeys{
this,
{"cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="},
@@ -520,25 +206,6 @@ public:
can add it to `trusted-public-keys` in their `nix.conf`.
)"};
Setting<unsigned int> tarballTtl{
this,
60 * 60,
"tarball-ttl",
R"(
The number of seconds a downloaded tarball is considered fresh. If
the cached tarball is stale, Nix checks whether it is still up
to date using the ETag header. Nix downloads a new version if
the ETag header is unsupported, or the cached ETag doesn't match.
Setting the TTL to `0` forces Nix to always check if the tarball is
up to date.
Nix caches tarballs in `$XDG_CACHE_HOME/nix/tarballs`.
Files fetched via `NIX_PATH`, `fetchGit`, `fetchMercurial`,
`fetchTarball`, and `fetchurl` respect this TTL.
)"};
Setting<bool> requireSigs{
this,
true,
@@ -629,27 +296,9 @@ public:
// Don't document the machine-specific default value
false};
Setting<std::vector<StoreReference>> substituters{
this,
std::vector<StoreReference>{StoreReference::parse("https://cache.nixos.org/")},
"substituters",
R"(
A list of [URLs of Nix stores](@docroot@/store/types/index.md#store-url-format) to be used as substituters, separated by whitespace.
A substituter is an additional [store](@docroot@/glossary.md#gloss-store) from which Nix can obtain [store objects](@docroot@/store/store-object.md) instead of building them.
Substituters are tried based on their priority value, which each substituter can set independently.
Lower value means higher priority.
The default is `https://cache.nixos.org`, which has a priority of 40.
At least one of the following conditions must be met for Nix to use a substituter:
- The substituter is in the [`trusted-substituters`](#conf-trusted-substituters) list
- The user calling Nix is in the [`trusted-users`](#conf-trusted-users) list
In addition, each store path should be trusted as described in [`trusted-public-keys`](#conf-trusted-public-keys)
)",
{"binary-caches"}};
// move to daemonsettings in another pass
//
// we'll add a another parameter to processConnection to thread it through
Setting<std::set<StoreReference>> trustedSubstituters{
this,
{},
@@ -695,56 +344,10 @@ public:
mismatch if the build isn't reproducible.
)"};
// move it out in the 2nd pass
Setting<bool> printMissing{
this, true, "print-missing", "Whether to print what paths need to be built or downloaded."};
Setting<std::string> postBuildHook{
this,
"",
"post-build-hook",
R"(
Optional. The path to a program to execute after each build.
This option is only settable in the global `nix.conf`, or on the
command line by trusted users.
When using the nix-daemon, the daemon executes the hook as `root`.
If the nix-daemon is not involved, the hook runs as the user
executing the nix-build.
- The hook executes after an evaluation-time build.
- The hook does not execute on substituted paths.
- The hook's output always goes to the user's terminal.
- If the hook fails, the build succeeds but no further builds
execute.
- The hook executes synchronously, and blocks other builds from
progressing while it runs.
The program executes with no arguments. The program's environment
contains the following environment variables:
- `DRV_PATH`
The derivation for the built paths.
Example:
`/nix/store/5nihn1a7pa8b25l9zafqaqibznlvvp3f-bash-4.4-p23.drv`
- `OUT_PATHS`
Output paths of the built derivation, separated by a space
character.
Example:
`/nix/store/l88brggg9hpy96ijds34dlq4n8fan63g-bash-4.4-p23-dev
/nix/store/vch71bhyi5akr5zs40k8h2wqxx69j80l-bash-4.4-p23-doc
/nix/store/c5cxjywi66iwn9dcx5yvwjkvl559ay6p-bash-4.4-p23-info
/nix/store/scz72lskj03ihkcn42ias5mlp4i4gr1k-bash-4.4-p23-man
/nix/store/a724znygmd1cac856j3gfsyvih3lw07j-bash-4.4-p23`.
)"};
Setting<bool> useXDGBaseDirectories{
this,
false,

23
src/libstore/include/nix/store/local-settings.hh
View File

@@ -185,6 +185,29 @@ struct LocalSettings : public virtual Config, public GCSettings, public AutoAllo
return autoAllocateUids ? this : nullptr;
}
Setting<unsigned int> buildCores{
this,
0,
"cores",
R"(
Sets the value of the `NIX_BUILD_CORES` environment variable in the [invocation of the `builder` executable](@docroot@/store/building.md#builder-execution) of a derivation.
The `builder` executable can use this variable to control its own maximum amount of parallelism.
<!--
FIXME(@fricklerhandwerk): I don't think this should even be mentioned here.
A very generic example using `derivation` and `xargs` may be more appropriate to explain the mechanism.
Using `mkDerivation` as an example requires being aware of that there are multiple independent layers that are completely opaque here.
-->
For instance, in Nixpkgs, if the attribute `enableParallelBuilding` for the `mkDerivation` build helper is set to `true`, it passes the `-j${NIX_BUILD_CORES}` flag to GNU Make.
If set to `0`, nix will detect the number of CPU cores and pass this number via `NIX_BUILD_CORES`.
> **Note**
>
> The number of parallel local Nix build jobs is independently controlled with the [`max-jobs`](#conf-max-jobs) setting.
)",
{"build-cores"}};
Setting<bool> fsyncMetadata{
this,
true,

2
src/libstore/include/nix/store/meson.build
View File

@@ -42,6 +42,7 @@ headers = [ config_pub_h ] + files(
'export-import.hh',
'filetransfer.hh',
'gc-store.hh',
'global-paths.hh',
'globals.hh',
'http-binary-cache-store.hh',
'indirect-root-store.hh',
@@ -94,4 +95,5 @@ headers = [ config_pub_h ] + files(
'worker-protocol-connection.hh',
'worker-protocol-impl.hh',
'worker-protocol.hh',
'worker-settings.hh',
)

366
src/libstore/include/nix/store/worker-settings.hh Normal file
View File

@@ -0,0 +1,366 @@
#pragma once
///@file
#include "nix/util/configuration.hh"
#include "nix/store/global-paths.hh"
#include "nix/store/store-reference.hh"
namespace nix {
struct MaxBuildJobsSetting : public BaseSetting<unsigned int>
{
MaxBuildJobsSetting(
Config * options,
unsigned int def,
const std::string & name,
const std::string & description,
const StringSet & aliases = {})
: BaseSetting<unsigned int>(def, true, name, description, aliases)
{
options->addSetting(this);
}
unsigned int parse(const std::string & str) const override;
};
struct WorkerSettings : public virtual Config
{
protected:
WorkerSettings() = default;
public:
Setting<bool> keepGoing{
this, false, "keep-going", "Whether to keep building derivations when another build fails."};
Setting<bool> tryFallback{
this,
false,
"fallback",
R"(
If set to `true`, Nix falls back to building from source if a
binary substitute fails. This is equivalent to the `--fallback`
flag. The default is `false`.
)",
{"build-fallback"}};
Setting<size_t> logLines{
this,
25,
"log-lines",
"The number of lines of the tail of "
"the log to show if a build fails."};
MaxBuildJobsSetting maxBuildJobs{
this,
1,
"max-jobs",
R"(
Maximum number of jobs that Nix tries to build locally in parallel.
The special value `auto` causes Nix to use the number of CPUs in your system.
Use `0` to disable local builds and directly use the remote machines specified in [`builders`](#conf-builders).
This doesn't affect derivations that have [`preferLocalBuild = true`](@docroot@/language/advanced-attributes.md#adv-attr-preferLocalBuild), which are always built locally.
> **Note**
>
> The number of CPU cores to use for each build job is independently determined by the [`cores`](#conf-cores) setting.
<!-- TODO(@fricklerhandwerk): would be good to have those shorthands for common options as part of the specification -->
The setting can be overridden using the `--max-jobs` (`-j`) command line switch.
)",
{"build-max-jobs"}};
Setting<unsigned int> maxSubstitutionJobs{
this,
16,
"max-substitution-jobs",
R"(
This option defines the maximum number of substitution jobs that Nix
tries to run in parallel. The default is `16`. The minimum value
one can choose is `1` and lower values are interpreted as `1`.
)",
{"substitution-max-jobs"}};
Setting<time_t> maxSilentTime{
this,
0,
"max-silent-time",
R"(
This option defines the maximum number of seconds that a builder can
go without producing any data on standard output or standard error.
This is useful (for instance in an automated build system) to catch
builds that are stuck in an infinite loop, or to catch remote builds
that are hanging due to network problems. It can be overridden using
the `--max-silent-time` command line switch.
The value `0` means that there is no timeout. This is also the
default.
)",
{"build-max-silent-time"}};
Setting<time_t> buildTimeout{
this,
0,
"timeout",
R"(
This option defines the maximum number of seconds that a builder can
run. This is useful (for instance in an automated build system) to
catch builds that are stuck in an infinite loop but keep writing to
their standard output or standard error. It can be overridden using
the `--timeout` command line switch.
The value `0` means that there is no timeout. This is also the
default.
)",
{"build-timeout"}};
Setting<Strings> buildHook{
this,
{"nix", "__build-remote"},
"build-hook",
R"(
The path to the helper program that executes remote builds.
Nix communicates with the build hook over `stdio` using a custom protocol to request builds that cannot be performed directly by the Nix daemon.
The default value is the internal Nix binary that implements remote building.
> **Important**
>
> Change this setting only if you really know what you’re doing.
)"};
Setting<std::string> builders{
this,
"@" + (nixConfDir() / "machines").string(),
"builders",
R"(
A semicolon- or newline-separated list of build machines.
In addition to the [usual ways of setting configuration options](@docroot@/command-ref/conf-file.md), the value can be read from a file by prefixing its absolute path with `@`.
> **Example**
>
> This is the default setting:
>
> ```
> builders = @/etc/nix/machines
> ```
Each machine specification consists of the following elements, separated by spaces.
Only the first element is required.
To leave a field at its default, set it to `-`.
1. The URI of the remote store in the format `ssh://[username@]hostname[:port]`.
> **Example**
>
> `ssh://nix@mac`
For backward compatibility, `ssh://` may be omitted.
The hostname may be an alias defined in `~/.ssh/config`.
2. A comma-separated list of [Nix system types](@docroot@/development/building.md#system-type).
If omitted, this defaults to the local platform type.
> **Example**
>
> `aarch64-darwin`
It is possible for a machine to support multiple platform types.
> **Example**
>
> `i686-linux,x86_64-linux`
3. The SSH identity file to be used to log in to the remote machine.
If omitted, SSH uses its regular identities.
> **Example**
>
> `/home/user/.ssh/id_mac`
4. The maximum number of builds that Nix executes in parallel on the machine.
Typically this should be equal to the number of CPU cores.
5. The “speed factor”, indicating the relative speed of the machine as a positive integer.
If there are multiple machines of the right type, Nix prefers the fastest, taking load into account.
6. A comma-separated list of supported [system features](#conf-system-features).
A machine is only used to build a derivation if all the features in the derivation's [`requiredSystemFeatures`](@docroot@/language/advanced-attributes.html#adv-attr-requiredSystemFeatures) attribute are supported by that machine.
7. A comma-separated list of required [system features](#conf-system-features).
A machine is only used to build a derivation if all of the machine’s required features appear in the derivation’s [`requiredSystemFeatures`](@docroot@/language/advanced-attributes.html#adv-attr-requiredSystemFeatures) attribute.
8. The (base64-encoded) public host key of the remote machine.
If omitted, SSH uses its regular `known_hosts` file.
The value for this field can be obtained via `base64 -w0`.
> **Example**
>
> Multiple builders specified on the command line:
>
> ```console
> --builders 'ssh://mac x86_64-darwin ; ssh://beastie x86_64-freebsd'
> ```
> **Example**
>
> This specifies several machines that can perform `i686-linux` builds:
>
> ```
> nix@scratchy.labs.cs.uu.nl i686-linux /home/nix/.ssh/id_scratchy 8 1 kvm
> nix@itchy.labs.cs.uu.nl i686-linux /home/nix/.ssh/id_scratchy 8 2
> nix@poochie.labs.cs.uu.nl i686-linux /home/nix/.ssh/id_scratchy 1 2 kvm benchmark
> ```
>
> However, `poochie` only builds derivations that have the attribute
>
> ```nix
> requiredSystemFeatures = [ "benchmark" ];
> ```
>
> or
>
> ```nix
> requiredSystemFeatures = [ "benchmark" "kvm" ];
> ```
>
> `itchy` cannot do builds that require `kvm`, but `scratchy` does support such builds.
> For regular builds, `itchy` is preferred over `scratchy` because it has a higher speed factor.
For Nix to use substituters, the calling user must be in the [`trusted-users`](#conf-trusted-users) list.
> **Note**
>
> A build machine must be accessible via SSH and have Nix installed.
> `nix` must be available in `$PATH` for the user connecting over SSH.
> **Warning**
>
> If you are building via the Nix daemon (default), the Nix daemon user account on the local machine (that is, `root`) requires access to a user account on the remote machine (not necessarily `root`).
>
> If you can’t or don’t want to configure `root` to be able to access the remote machine, set [`store`](#conf-store) to any [local store](@docroot@/store/types/local-store.html), e.g. by passing `--store /tmp` to the command on the local machine.
To build only on remote machines and disable local builds, set [`max-jobs`](#conf-max-jobs) to 0.
If you want the remote machines to use substituters, set [`builders-use-substitutes`](#conf-builders-use-substitutes) to `true`.
)",
{},
false};
Setting<bool> alwaysAllowSubstitutes{
this,
false,
"always-allow-substitutes",
R"(
If set to `true`, Nix ignores the [`allowSubstitutes`](@docroot@/language/advanced-attributes.md) attribute in derivations and always attempt to use [available substituters](#conf-substituters).
)"};
Setting<bool> buildersUseSubstitutes{
this,
false,
"builders-use-substitutes",
R"(
If set to `true`, Nix instructs [remote build machines](#conf-builders) to use their own [`substituters`](#conf-substituters) if available.
It means that remote build hosts fetch as many dependencies as possible from their own substituters (e.g, from `cache.nixos.org`) instead of waiting for the local machine to upload them all.
This can drastically reduce build times if the network connection between the local machine and the remote build host is slow.
)"};
Setting<bool> useSubstitutes{
this,
true,
"substitute",
R"(
If set to `true` (default), Nix uses binary substitutes if
available. This option can be disabled to force building from
source.
)",
{"build-use-substitutes"}};
Setting<std::vector<StoreReference>> substituters{
this,
std::vector<StoreReference>{StoreReference::parse("https://cache.nixos.org/")},
"substituters",
R"(
A list of [URLs of Nix stores](@docroot@/store/types/index.md#store-url-format) to be used as substituters, separated by whitespace.
A substituter is an additional [store](@docroot@/glossary.md#gloss-store) from which Nix can obtain [store objects](@docroot@/store/store-object.md) instead of building them.
Substituters are tried based on their priority value, which each substituter can set independently.
Lower value means higher priority.
The default is `https://cache.nixos.org`, which has a priority of 40.
At least one of the following conditions must be met for Nix to use a substituter:
- The substituter is in the [`trusted-substituters`](#conf-trusted-substituters) list
- The user calling Nix is in the [`trusted-users`](#conf-trusted-users) list
In addition, each store path should be trusted as described in [`trusted-public-keys`](#conf-trusted-public-keys)
)",
{"binary-caches"}};
Setting<unsigned long> maxLogSize{
this,
0,
"max-build-log-size",
R"(
This option defines the maximum number of bytes that a builder can
write to its stdout/stderr. If the builder exceeds this limit, it's
killed. A value of `0` (the default) means that there is no limit.
)",
{"build-max-log-size"}};
Setting<unsigned int> pollInterval{this, 5, "build-poll-interval", "How often (in seconds) to poll for locks."};
Setting<std::string> postBuildHook{
this,
"",
"post-build-hook",
R"(
Optional. The path to a program to execute after each build.
This option is only settable in the global `nix.conf`, or on the
command line by trusted users.
When using the nix-daemon, the daemon executes the hook as `root`.
If the nix-daemon is not involved, the hook runs as the user
executing the nix-build.
- The hook executes after an evaluation-time build.
- The hook does not execute on substituted paths.
- The hook's output always goes to the user's terminal.
- If the hook fails, the build succeeds but no further builds
execute.
- The hook executes synchronously, and blocks other builds from
progressing while it runs.
The program executes with no arguments. The program's environment
contains the following environment variables:
- `DRV_PATH`
The derivation for the built paths.
Example:
`/nix/store/5nihn1a7pa8b25l9zafqaqibznlvvp3f-bash-4.4-p23.drv`
- `OUT_PATHS`
Output paths of the built derivation, separated by a space
character.
Example:
`/nix/store/l88brggg9hpy96ijds34dlq4n8fan63g-bash-4.4-p23-dev
/nix/store/vch71bhyi5akr5zs40k8h2wqxx69j80l-bash-4.4-p23-doc
/nix/store/c5cxjywi66iwn9dcx5yvwjkvl559ay6p-bash-4.4-p23-info
/nix/store/scz72lskj03ihkcn42ias5mlp4i4gr1k-bash-4.4-p23-man
/nix/store/a724znygmd1cac856j3gfsyvih3lw07j-bash-4.4-p23`.
)"};
};
} // namespace nix

6
src/libstore/legacy-ssh-store.cc
View File

@@ -183,9 +183,9 @@ void LegacySSHStore::narFromPath(const StorePath & path, std::function<void(Sour
static ServeProto::BuildOptions buildSettings()
{
return {
.maxSilentTime = settings.maxSilentTime,
.buildTimeout = settings.buildTimeout,
.maxLogSize = settings.maxLogSize,
.maxSilentTime = settings.getWorkerSettings().maxSilentTime,
.buildTimeout = settings.getWorkerSettings().buildTimeout,
.maxLogSize = settings.getWorkerSettings().maxLogSize,
.nrRepeats = 0, // buildRepeat hasn't worked for ages anyway
.enforceDeterminism = 0,
.keepFailed = settings.keepFailed,

2
src/libstore/machines.cc
View File

@@ -209,7 +209,7 @@ Machines Machine::parseConfig(const StringSet & defaultSystems, const std::strin
Machines getMachines()
{
return Machine::parseConfig({settings.thisSystem}, settings.builders);
return Machine::parseConfig({settings.thisSystem}, settings.getWorkerSettings().builders);
}
} // namespace nix

6
src/libstore/misc.cc
View File

@@ -236,7 +236,8 @@ MissingPaths Store::queryMissing(const std::vector<DerivedPath> & targets)
throw;
}
if (!knownOutputPaths && settings.useSubstitutes && drvOptions.substitutesAllowed()) {
if (!knownOutputPaths && settings.getWorkerSettings().useSubstitutes
&& drvOptions.substitutesAllowed(settings.getWorkerSettings())) {
experimentalFeatureSettings.require(Xp::CaDerivations);
// If there are unknown output paths, attempt to find if the
@@ -266,7 +267,8 @@ MissingPaths Store::queryMissing(const std::vector<DerivedPath> & targets)
}
}
if (knownOutputPaths && settings.useSubstitutes && drvOptions.substitutesAllowed()) {
if (knownOutputPaths && settings.getWorkerSettings().useSubstitutes
&& drvOptions.substitutesAllowed(settings.getWorkerSettings())) {
auto drvState = make_ref<Sync<DrvState>>(DrvState(invalid.size()));
for (auto & output : invalid)
pool.enqueue(std::bind(checkOutput, drvPath, drv, output, drvState));

23
src/libstore/remote-store.cc
View File

@@ -114,22 +114,23 @@ void RemoteStore::initConnection(Connection & conn)
void RemoteStore::setOptions(Connection & conn)
{
conn.to << WorkerProto::Op::SetOptions << settings.keepFailed << settings.keepGoing << settings.tryFallback
<< verbosity << settings.maxBuildJobs << settings.maxSilentTime << true
<< (settings.verboseBuild ? lvlError : lvlVomit) << 0 // obsolete log type
<< 0 /* obsolete print build trace */
<< settings.buildCores << settings.useSubstitutes;
conn.to << WorkerProto::Op::SetOptions << settings.keepFailed << settings.getWorkerSettings().keepGoing
<< settings.getWorkerSettings().tryFallback << verbosity << settings.getWorkerSettings().maxBuildJobs
<< settings.getWorkerSettings().maxSilentTime << true << (settings.verboseBuild ? lvlError : lvlVomit)
<< 0 // obsolete log type
<< 0 /* obsolete print build trace */
<< settings.getLocalSettings().buildCores << settings.getWorkerSettings().useSubstitutes;
std::map<std::string, nix::Config::SettingInfo> overrides;
settings.getSettings(overrides, true); // libstore settings
fileTransferSettings.getSettings(overrides, true);
overrides.erase(settings.keepFailed.name);
overrides.erase(settings.keepGoing.name);
overrides.erase(settings.tryFallback.name);
overrides.erase(settings.maxBuildJobs.name);
overrides.erase(settings.maxSilentTime.name);
overrides.erase(settings.buildCores.name);
overrides.erase(settings.useSubstitutes.name);
overrides.erase(settings.getWorkerSettings().keepGoing.name);
overrides.erase(settings.getWorkerSettings().tryFallback.name);
overrides.erase(settings.getWorkerSettings().maxBuildJobs.name);
overrides.erase(settings.getWorkerSettings().maxSilentTime.name);
overrides.erase(settings.getLocalSettings().buildCores.name);
overrides.erase(settings.getWorkerSettings().useSubstitutes.name);
overrides.erase(loggerSettings.showTrace.name);
overrides.erase(experimentalFeatureSettings.experimentalFeatures.name);
overrides.erase("plugin-files");

8
src/libstore/store-api.cc
View File

@@ -176,7 +176,7 @@ void Store::addMultipleToStore(PathsSource && pathsToCopy, Activity & act, Repai
addToStore(info, *source, repair, checkSigs);
} catch (Error & e) {
nrFailed++;
if (!settings.keepGoing)
if (!settings.getWorkerSettings().keepGoing)
throw e;
printMsg(lvlError, "could not copy %s: %s", printStorePath(path), e.what());
showProgress();
@@ -407,7 +407,7 @@ StorePathSet Store::queryDerivationOutputs(const StorePath & path)
void Store::querySubstitutablePathInfos(const StorePathCAMap & paths, SubstitutablePathInfos & infos)
{
if (!settings.useSubstitutes)
if (!settings.getWorkerSettings().useSubstitutes)
return;
for (auto & path : paths) {
@@ -463,7 +463,7 @@ void Store::querySubstitutablePathInfos(const StorePathCAMap & paths, Substituta
}
}
if (lastStoresException.has_value()) {
if (!settings.tryFallback) {
if (!settings.getWorkerSettings().tryFallback) {
throw *lastStoresException;
} else
logError(lastStoresException->info());
@@ -473,7 +473,7 @@ void Store::querySubstitutablePathInfos(const StorePathCAMap & paths, Substituta
StorePathSet Store::querySubstitutablePaths(const StorePathSet & paths)
{
if (!settings.useSubstitutes)
if (!settings.getWorkerSettings().useSubstitutes)
return StorePathSet();
StorePathSet remaining;

2
src/libstore/store-registration.cc
View File

@@ -91,7 +91,7 @@ std::list<ref<Store>> getDefaultSubstituters()
}
};
for (const auto & ref : settings.substituters.get())
for (const auto & ref : settings.getWorkerSettings().substituters.get())
addStore(ref);
stores.sort([](ref<Store> & a, ref<Store> & b) { return a->config.priority < b->config.priority; });

4
src/libstore/unix/build/derivation-builder.cc
View File

@@ -1076,7 +1076,9 @@ void DerivationBuilderImpl::initEnv()
env["NIX_STORE"] = store.storeDir;
/* The maximum number of cores to utilize for parallel building. */
env["NIX_BUILD_CORES"] = fmt("%d", settings.buildCores ? settings.buildCores : settings.getDefaultCores());
env["NIX_BUILD_CORES"] = fmt(
"%d",
settings.getLocalSettings().buildCores ? settings.getLocalSettings().buildCores : settings.getDefaultCores());
/* Write the final environment. Note that this is intentionally
*not* `drv.env`, because we've desugared things like like

8
src/libstore/unix/build/hook-instance.cc
View File

@@ -1,18 +1,16 @@
#include "nix/store/globals.hh"
#include "nix/util/config-global.hh"
#include "nix/store/build/hook-instance.hh"
#include "nix/util/file-system.hh"
#include "nix/store/build/child.hh"
#include "nix/util/strings.hh"
#include "nix/util/executable-path.hh"
namespace nix {
HookInstance::HookInstance()
HookInstance::HookInstance(const Strings & _buildHook)
{
debug("starting build hook '%s'", concatStringsSep(" ", settings.buildHook.get()));
debug("starting build hook '%s'", concatStringsSep(" ", _buildHook));
auto buildHookArgs = settings.buildHook.get();
auto buildHookArgs = _buildHook;
if (buildHookArgs.empty())
throw Error("'build-hook' setting is empty");

2
src/libstore/unix/include/nix/store/build/hook-instance.hh
View File

@@ -58,7 +58,7 @@ struct HookInstance
*/
std::function<void()> onKillChild;
HookInstance();
HookInstance(const Strings & buildHook);
~HookInstance();
};

6
src/nix/build-remote/build-remote.cc
View File

@@ -76,8 +76,8 @@ static int main_build_remote(int argc, char ** argv)
settings.set(name, value);
}
auto maxBuildJobs = settings.maxBuildJobs;
settings.maxBuildJobs.set("1"); // hack to make tests with local?root= work
auto maxBuildJobs = settings.getWorkerSettings().maxBuildJobs;
settings.getWorkerSettings().maxBuildJobs.set("1"); // hack to make tests with local?root= work
initPlugins();
@@ -284,7 +284,7 @@ static int main_build_remote(int argc, char ** argv)
signal(SIGALRM, old);
}
auto substitute = settings.buildersUseSubstitutes ? Substitute : NoSubstitute;
auto substitute = settings.getWorkerSettings().buildersUseSubstitutes ? Substitute : NoSubstitute;
{
Activity act(*logger, lvlTalkative, actUnknown, fmt("copying dependencies to '%s'", storeUri));

8
src/nix/flake.cc
View File

@@ -1,3 +1,4 @@
#include "nix/cmd/common-eval-args.hh"
#include "nix/main/common-args.hh"
#include "nix/main/shared.hh"
#include "nix/expr/eval.hh"
@@ -9,6 +10,7 @@
#include "nix/store/derivations.hh"
#include "nix/store/outputs-spec.hh"
#include "nix/expr/attr-path.hh"
#include "nix/fetchers/fetch-settings.hh"
#include "nix/fetchers/fetchers.hh"
#include "nix/fetchers/registry.hh"
#include "nix/expr/eval-cache.hh"
@@ -129,7 +131,7 @@ public:
void run(nix::ref<nix::Store> store) override
{
settings.tarballTtl = 0;
fetchSettings.tarballTtl = 0;
auto updateAll = lockFlags.inputUpdates.empty();
lockFlags.recreateLockFile = updateAll;
@@ -162,7 +164,7 @@ struct CmdFlakeLock : FlakeCommand
void run(nix::ref<nix::Store> store) override
{
settings.tarballTtl = 0;
fetchSettings.tarballTtl = 0;
lockFlags.writeLockFile = true;
lockFlags.failOnUnlocked = true;
@@ -364,7 +366,7 @@ struct CmdFlakeCheck : FlakeCommand
} catch (Interrupted & e) {
throw;
} catch (Error & e) {
if (settings.keepGoing) {
if (settings.getWorkerSettings().keepGoing) {
logError(e.info());
hasErrors = true;
} else

14
src/nix/main.cc
View File

@@ -1,3 +1,5 @@
#include "nix/cmd/common-eval-args.hh"
#include "nix/fetchers/fetch-settings.hh"
#include "nix/util/args/root.hh"
#include "nix/util/current-process.hh"
#include "nix/cmd/command.hh"
@@ -384,7 +386,7 @@ void mainWrapped(int argc, char ** argv)
self-aware. That is, it has to know where it is installed. We
don't think it's sentient.
*/
settings.buildHook.setDefault(
settings.getWorkerSettings().buildHook.setDefault(
Strings{
getNixBin({}).string(),
"__build-remote",
@@ -552,10 +554,10 @@ void mainWrapped(int argc, char ** argv)
if (!args.useNet) {
// FIXME: should check for command line overrides only.
if (!settings.useSubstitutes.overridden)
settings.useSubstitutes = false;
if (!settings.tarballTtl.overridden)
settings.tarballTtl = std::numeric_limits<unsigned int>::max();
if (!settings.getWorkerSettings().useSubstitutes.overridden)
settings.getWorkerSettings().useSubstitutes = false;
if (!fetchSettings.tarballTtl.overridden)
fetchSettings.tarballTtl = std::numeric_limits<unsigned int>::max();
if (!fileTransferSettings.tries.overridden)
fileTransferSettings.tries = 0;
if (!fileTransferSettings.connectTimeout.overridden)
@@ -563,7 +565,7 @@ void mainWrapped(int argc, char ** argv)
}
if (args.refresh) {
settings.tarballTtl = 0;
fetchSettings.tarballTtl = 0;
settings.ttlNegativeNarInfoCache = 0;
settings.ttlPositiveNarInfoCache = 0;
}

5
src/nix/nix-build/nix-build.cc
View File

@@ -552,7 +552,10 @@ static void main_nix_build(int argc, char ** argv)
env["NIX_BUILD_TOP"] = env["TMPDIR"] = env["TEMPDIR"] = env["TMP"] = env["TEMP"] = tmpDir.path().string();
env["NIX_STORE"] = store->storeDir;
env["NIX_BUILD_CORES"] = fmt("%d", settings.buildCores ? settings.buildCores : settings.getDefaultCores());
env["NIX_BUILD_CORES"] =
fmt("%d",
settings.getLocalSettings().buildCores ? settings.getLocalSettings().buildCores
: settings.getDefaultCores());
DerivationOptions<StorePath> drvOptions;
try {

8
src/nix/nix-store/nix-store.cc
View File

@@ -898,7 +898,7 @@ static void opServe(Strings opFlags, Strings opArgs)
// building through the daemon.
verbosity = lvlError;
settings.getLogFileSettings().keepLog = false;
settings.useSubstitutes = false;
settings.getWorkerSettings().useSubstitutes = false;
auto options = ServeProto::Serialise<ServeProto::BuildOptions>::read(*store, rconn);
@@ -907,10 +907,10 @@ static void opServe(Strings opFlags, Strings opArgs)
// See how the serialization logic in
// `ServeProto::Serialise<ServeProto::BuildOptions>` matches
// these conditions.
settings.maxSilentTime = options.maxSilentTime;
settings.buildTimeout = options.buildTimeout;
settings.getWorkerSettings().maxSilentTime = options.maxSilentTime;
settings.getWorkerSettings().buildTimeout = options.buildTimeout;
if (clientVersion >= ServeProto::Version{2, 2})
settings.maxLogSize = options.maxLogSize;
settings.getWorkerSettings().maxLogSize = options.maxLogSize;
if (clientVersion >= ServeProto::Version{2, 3}) {
if (options.nrRepeats != 0) {
throw Error("client requested repeating builds, but this is not currently implemented");