Compare commits

..
Author SHA1 Message Date
Jörg Thalheim 0bc58d5501 ignore-gc-delete-failure: add release note 2026-01-23 19:49:04 +01:00
58 changed files with 361 additions and 1150 deletions

No files matched your search

@@ -3,8 +3,8 @@ synopsis: "New setting `ignore-gc-delete-failure` for local stores"
prs: [15054]
---
A new local store setting [`ignore-gc-delete-failure`](@docroot@/store/types/local-store.md#store-local-store-ignore-gc-delete-failure) has been added.
A new local store setting `ignore-gc-delete-failure` has been added.
When enabled, garbage collection will log warnings instead of failing when it cannot delete store paths.
This is useful when running Nix as an unprivileged user that may not have write access to all paths in the store.
This setting is experimental and requires the [`local-overlay-store`](@docroot@/development/experimental-features.md#xp-feature-local-overlay-store) experimental feature.
This setting is experimental and requires `--extra-experimental-features local-overlay-store`.
-15
View File
@@ -1,15 +0,0 @@
---
synopsis: Support HTTPS binary caches using mTLS (client certificate) authentication
issues: [13002]
prs: [13030]
---
Added support for `tls-certificate` and `tls-private-key` options in substituter URLs.
Example:
```
https://substituter.invalid?tls-certificate=/path/to/cert.pem&tls-private-key=/path/to/key.pem
```
When these options are configured, Nix will use this certificate/private key pair to authenticate to the server.
+3 -3
View File
@@ -131,7 +131,7 @@ pkgs.nixComponents2.nix-util.overrideAttrs (
ignoreCrossFile = flags: builtins.filter (flag: !(lib.strings.hasInfix "cross-file" flag)) flags;
availableComponents = lib.filterAttrs (
k: v: lib.meta.availableOn pkgs.stdenv.hostPlatform v
k: v: lib.meta.availableOn pkgs.hostPlatform v
) allComponents;
activeComponents = buildInputsClosureCond isInternal (
@@ -323,7 +323,7 @@ pkgs.nixComponents2.nix-util.overrideAttrs (
pkgs.buildPackages.shellcheck
pkgs.buildPackages.include-what-you-use
]
++ lib.optional stdenv.hostPlatform.isUnix pkgs.buildPackages.gdb
++ lib.optional pkgs.hostPlatform.isUnix pkgs.buildPackages.gdb
++ lib.optional (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform) (
lib.hiPrio pkgs.buildPackages.clang-tools
)
@@ -341,7 +341,7 @@ pkgs.nixComponents2.nix-util.overrideAttrs (
buildInputs =
# TODO change Nixpkgs to mark gbenchmark as building on Windows
lib.optional stdenv.hostPlatform.isUnix pkgs.gbenchmark
lib.optional pkgs.hostPlatform.isUnix pkgs.gbenchmark
++ dedupByString (v: "${v}") (
lib.filter (x: !isInternal x) (lib.lists.concatMap (c: c.buildInputs) activeComponents)
)
-125
View File
@@ -1,125 +0,0 @@
#include "nix/store/tests/https-store.hh"
#include <thread>
namespace nix::testing {
void TestHttpBinaryCacheStore::init()
{
BinaryCacheStore::init();
}
ref<TestHttpBinaryCacheStore> TestHttpBinaryCacheStoreConfig::openTestStore() const
{
auto store = make_ref<TestHttpBinaryCacheStore>(
ref{// FIXME we shouldn't actually need a mutable config
std::const_pointer_cast<HttpBinaryCacheStore::Config>(shared_from_this())});
store->init();
return store;
}
void HttpsBinaryCacheStoreTest::openssl(Strings args)
{
runProgram("openssl", /*lookupPath=*/true, args);
}
void HttpsBinaryCacheStoreTest::SetUp()
{
LibStoreNetworkTest::SetUp();
#ifdef _WIN32
GTEST_SKIP() << "HTTPS store tests are not supported on Windows";
#endif
tmpDir = createTempDir();
cacheDir = tmpDir / "cache";
delTmpDir = std::make_unique<AutoDelete>(tmpDir);
localCacheStore =
make_ref<LocalBinaryCacheStoreConfig>("file", cacheDir.string(), LocalBinaryCacheStoreConfig::Params{})
->openStore();
caCert = tmpDir / "ca.crt";
caKey = tmpDir / "ca.key";
serverCert = tmpDir / "server.crt";
serverKey = tmpDir / "server.key";
clientCert = tmpDir / "client.crt";
clientKey = tmpDir / "client.key";
// clang-format off
openssl({"ecparam", "-genkey", "-name", "prime256v1", "-out", caKey.string()});
openssl({"req", "-new", "-x509", "-days", "1", "-key", caKey.string(), "-out", caCert.string(), "-subj", "/CN=TestCA"});
openssl({"ecparam", "-genkey", "-name", "prime256v1", "-out", serverKey.string()});
openssl({"req", "-new", "-key", serverKey.string(), "-out", (tmpDir / "server.csr").string(), "-subj", "/CN=localhost"});
openssl({"x509", "-req", "-in", (tmpDir / "server.csr").string(), "-CA", caCert.string(), "-CAkey", caKey.string(), "-CAcreateserial", "-out", serverCert.string(), "-days", "1"});
openssl({"ecparam", "-genkey", "-name", "prime256v1", "-out", clientKey.string()});
openssl({"req", "-new", "-key", clientKey.string(), "-out", (tmpDir / "client.csr").string(), "-subj", "/CN=TestClient"});
openssl({"x509", "-req", "-in", (tmpDir / "client.csr").string(), "-CA", caCert.string(), "-CAkey", caKey.string(), "-CAcreateserial", "-out", clientCert.string(), "-days", "1"});
// clang-format on
#ifndef _WIN32 /* FIXME: Can't yet start processes on windows */
auto args = serverArgs();
serverPid = startProcess(
[&] {
if (chdir(cacheDir.c_str()) == -1)
_exit(1);
std::vector<char *> argv;
argv.push_back(const_cast<char *>("openssl"));
for (auto & a : args)
argv.push_back(const_cast<char *>(a.c_str()));
argv.push_back(nullptr);
execvp("openssl", argv.data());
_exit(1);
},
{.dieWithParent = true});
#endif
/* As an optimization, sleep for a bit to allow the server to come up to avoid retrying when connecting.
This won't make the tests fail, but does make them run faster. We don't need to overcomplicate by waiting
for the port explicitly - this is enough. */
std::this_thread::sleep_for(std::chrono::milliseconds(50));
/* FIXME: Don't use global settings. Tests are not run concurrently, so this is fine for now. */
oldCaCert = settings.caFile;
settings.caFile = caCert.string();
}
void HttpsBinaryCacheStoreTest::TearDown()
{
settings.caFile = oldCaCert;
serverPid.kill();
delTmpDir.reset();
}
std::vector<std::string> HttpsBinaryCacheStoreTest::serverArgs()
{
return {
"s_server",
"-accept",
std::to_string(port),
"-cert",
serverCert.string(),
"-key",
serverKey.string(),
"-WWW", /* Serve from current directory. */
"-quiet",
};
}
std::vector<std::string> HttpsBinaryCacheStoreMtlsTest::serverArgs()
{
auto args = HttpsBinaryCacheStoreTest::serverArgs();
/* With the -Verify option the client must supply a certificate or an error occurs, which is not the
case with -verify. */
args.insert(args.end(), {"-CAfile", caCert.string(), "-Verify", "1", "-verify_return_error"});
return args;
}
ref<TestHttpBinaryCacheStoreConfig> HttpsBinaryCacheStoreTest::makeConfig(BinaryCacheStoreConfig::Params params)
{
auto res = make_ref<TestHttpBinaryCacheStoreConfig>("https", fmt("localhost:%d", port), std::move(params));
res->pathInfoCacheSize = 0; /* We don't want any caching in tests. */
return res;
}
} // namespace nix::testing
@@ -1,93 +0,0 @@
#pragma once
///@file
#include <gtest/gtest.h>
#include <gmock/gmock.h>
#include "nix/store/tests/libstore-network.hh"
#include "nix/store/http-binary-cache-store.hh"
#include "nix/store/store-api.hh"
#include "nix/store/globals.hh"
#include "nix/store/local-binary-cache-store.hh"
#include "nix/util/file-system.hh"
#include "nix/util/processes.hh"
namespace nix::testing {
class TestHttpBinaryCacheStoreConfig;
/**
* Test shim for testing. We don't want to use the on-disk narinfo cache in unit
* tests.
*/
class TestHttpBinaryCacheStore : public HttpBinaryCacheStore
{
public:
TestHttpBinaryCacheStore(const TestHttpBinaryCacheStore &) = delete;
TestHttpBinaryCacheStore(TestHttpBinaryCacheStore &&) = delete;
TestHttpBinaryCacheStore & operator=(const TestHttpBinaryCacheStore &) = delete;
TestHttpBinaryCacheStore & operator=(TestHttpBinaryCacheStore &&) = delete;
TestHttpBinaryCacheStore(ref<HttpBinaryCacheStoreConfig> config)
: Store{*config}
, BinaryCacheStore{*config}
, HttpBinaryCacheStore(config)
{
diskCache = nullptr; /* Disable caching, we'll be creating a new binary cache for each test. */
}
void init() override;
};
class TestHttpBinaryCacheStoreConfig : public HttpBinaryCacheStoreConfig
{
public:
TestHttpBinaryCacheStoreConfig(
std::string_view scheme, std::string_view cacheUri, const Store::Config::Params & params)
: StoreConfig(params)
, HttpBinaryCacheStoreConfig(scheme, cacheUri, params)
{
}
ref<TestHttpBinaryCacheStore> openTestStore() const;
ref<Store> openStore() const override
{
return openTestStore();
}
};
class HttpsBinaryCacheStoreTest : public virtual LibStoreNetworkTest
{
std::unique_ptr<AutoDelete> delTmpDir;
public:
static void SetUpTestSuite()
{
initLibStore(/*loadConfig=*/false);
}
protected:
std::filesystem::path tmpDir, cacheDir;
std::filesystem::path caCert, caKey, serverCert, serverKey;
std::filesystem::path clientCert, clientKey;
std::string oldCaCert;
Pid serverPid;
uint16_t port = 8443;
std::shared_ptr<Store> localCacheStore;
static void openssl(Strings args);
void SetUp() override;
void TearDown() override;
virtual std::vector<std::string> serverArgs();
ref<TestHttpBinaryCacheStoreConfig> makeConfig(BinaryCacheStoreConfig::Params params);
};
class HttpsBinaryCacheStoreMtlsTest : public HttpsBinaryCacheStoreTest
{
protected:
std::vector<std::string> serverArgs() override;
};
} // namespace nix::testing
@@ -1,39 +0,0 @@
#pragma once
/// @file
#include <gtest/gtest.h>
namespace nix::testing {
/**
* Whether to run network tests. This is global so that the test harness can
* enable this by default if we can run tests in isolation.
*/
extern bool networkTestsAvailable;
/**
* Set up network tests and, if on linux, create a new network namespace for
* tests with a loopback interface. This is to avoid binding to ports in the
* host's namespace.
*/
void setupNetworkTests();
class LibStoreNetworkTest : public virtual ::testing::Test
{
protected:
void SetUp() override
{
if (networkTestsAvailable)
return;
static bool warned = false;
if (!warned) {
warned = true;
GTEST_SKIP()
<< "Network tests not enabled by default without user namespaces, use NIX_TEST_FORCE_NETWORK_TESTS=1 to override";
} else {
GTEST_SKIP();
}
}
};
} // namespace nix::testing
@@ -4,8 +4,6 @@ include_dirs = [ include_directories('../../..') ]
headers = files(
'derived-path.hh',
'https-store.hh',
'libstore-network.hh',
'libstore.hh',
'nix_api_store.hh',
'outputs-spec.hh',
@@ -1,60 +0,0 @@
#include "nix/store/tests/libstore-network.hh"
#include "nix/util/error.hh"
#include "nix/util/environment-variables.hh"
#ifdef __linux__
# include "nix/util/file-system.hh"
# include "nix/util/linux-namespaces.hh"
# include <sched.h>
# include <sys/ioctl.h>
# include <net/if.h>
# include <netinet/in.h>
#endif
namespace nix::testing {
bool networkTestsAvailable = false;
#ifdef __linux__
static void enterNetworkNamespace()
{
auto uid = ::getuid();
auto gid = ::getgid();
if (::unshare(CLONE_NEWUSER | CLONE_NEWNET) == -1)
throw SysError("setting up a private network namespace for tests");
std::filesystem::path procSelf = "/proc/self";
writeFile(procSelf / "setgroups", "deny");
writeFile(procSelf / "uid_map", fmt("%d %d 1", uid, uid));
writeFile(procSelf / "gid_map", fmt("%d %d 1", gid, gid));
AutoCloseFD fd(::socket(PF_INET, SOCK_DGRAM, IPPROTO_IP));
if (!fd)
throw SysError("cannot open IP socket for loopback interface");
struct ::ifreq ifr = {};
strcpy(ifr.ifr_name, "lo");
ifr.ifr_flags = IFF_UP | IFF_LOOPBACK | IFF_RUNNING;
if (::ioctl(fd.get(), SIOCSIFFLAGS, &ifr) == -1)
throw SysError("cannot set loopback interface flags");
}
#endif
void setupNetworkTests()
try {
networkTestsAvailable = getEnvOs(OS_STR("NIX_TEST_FORCE_NETWORK_TESTS")).has_value();
#ifdef __linux__
if (!networkTestsAvailable && userNamespacesSupported()) {
enterNetworkNamespace();
networkTestsAvailable = true;
}
#endif
} catch (SystemError & e) {
/* Ignore any set up errors. */
}
} // namespace nix::testing
-5
View File
@@ -28,15 +28,10 @@ subdir('nix-meson-build-support/subprojects')
rapidcheck = dependency('rapidcheck')
deps_public += rapidcheck
gtest = dependency('gtest')
deps_public += gtest
subdir('nix-meson-build-support/common')
sources = files(
'derived-path.cc',
'https-store.cc',
'libstore-network.cc',
'outputs-spec.cc',
'path.cc',
'test-main.cc',
-2
View File
@@ -7,7 +7,6 @@
nix-store-c,
rapidcheck,
gtest,
# Configuration Options
@@ -40,7 +39,6 @@ mkMesonLibrary (finalAttrs: {
nix-store
nix-store-c
rapidcheck
gtest
];
mesonFlags = [
@@ -1,9 +1,6 @@
#include <gtest/gtest.h>
#include <regex>
#include "nix/store/http-binary-cache-store.hh"
#include "nix/store/tests/https-store.hh"
#include "nix/util/fs-sink.hh"
namespace nix {
@@ -37,88 +34,4 @@ TEST(HttpBinaryCacheStore, constructConfigWithParamsAndUrlWithParams)
EXPECT_EQ(config.getReference().params, params);
}
using testing::HttpsBinaryCacheStoreMtlsTest;
using testing::HttpsBinaryCacheStoreTest;
using namespace std::string_view_literals;
using namespace std::string_literals;
TEST_F(HttpsBinaryCacheStoreTest, queryPathInfo)
{
auto config = makeConfig({});
auto store = config->openStore();
StringSource dump{"test"sv};
auto path = localCacheStore->addToStoreFromDump(dump, "test-name", FileSerialisationMethod::Flat);
EXPECT_NO_THROW(store->queryPathInfo(path));
}
auto withNoRetries()
{
auto oldTries = fileTransferSettings.tries.get();
Finally restoreTries = [=]() { fileTransferSettings.tries = oldTries; };
fileTransferSettings.tries = 1; /* FIXME: Don't use global settings. */
return restoreTries;
}
TEST_F(HttpsBinaryCacheStoreMtlsTest, queryPathInfo)
{
auto config = makeConfig({
{"tls-certificate"s, clientCert.string()},
{"tls-private-key"s, clientKey.string()},
});
auto store = config->openStore();
StringSource dump{"test"sv};
auto path = localCacheStore->addToStoreFromDump(dump, "test-name", FileSerialisationMethod::Flat);
EXPECT_NO_THROW(store->queryPathInfo(path));
}
TEST_F(HttpsBinaryCacheStoreMtlsTest, rejectsWithoutClientCert)
{
auto restoreTries = withNoRetries();
auto config = makeConfig({});
EXPECT_THROW(config->openStore(), Error);
}
TEST_F(HttpsBinaryCacheStoreMtlsTest, rejectsWrongClientCert)
{
auto wrongKey = tmpDir / "wrong.key";
auto wrongCert = tmpDir / "wrong.crt";
// clang-format off
openssl({"ecparam", "-genkey", "-name", "prime256v1", "-out", wrongKey.string()});
openssl({"req", "-new", "-x509", "-days", "1", "-key", wrongKey.string(), "-out", wrongCert.string(), "-subj", "/CN=WrongClient"});
// clang-format on
auto config = makeConfig({
{"tls-certificate"s, wrongCert.string()},
{"tls-private-key"s, wrongKey.string()},
});
auto restoreTries = withNoRetries();
EXPECT_THROW(config->openStore(), Error);
}
TEST_F(HttpsBinaryCacheStoreMtlsTest, doesNotSendCertOnRedirectToDifferentAuthority)
{
StringSource dump{"test"sv};
auto path = localCacheStore->addToStoreFromDump(dump, "test-name", FileSerialisationMethod::Flat);
for (auto & entry : DirectoryIterator{cacheDir})
if (entry.path().extension() == ".narinfo") {
auto content = readFile(entry.path());
content = std::regex_replace(content, std::regex("URL: nar/"), fmt("URL: https://127.0.0.1:%d/nar/", port));
writeFile(entry.path(), content);
}
auto config = makeConfig({
{"tls-certificate"s, clientCert.string()},
{"tls-private-key"s, clientKey.string()},
});
auto store = config->openStore();
auto restoreTries = withNoRetries();
auto info = store->queryPathInfo(path);
NullSink null;
EXPECT_THROW(store->narFromPath(path, null), Error);
}
} // namespace nix
-2
View File
@@ -1,7 +1,6 @@
#include <gtest/gtest.h>
#include "nix/store/tests/test-main.hh"
#include "nix/store/tests/libstore-network.hh"
using namespace nix;
@@ -11,7 +10,6 @@ int main(int argc, char ** argv)
if (res)
return res;
nix::testing::setupNetworkTests();
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}
+1 -5
View File
@@ -9,7 +9,6 @@
nix-store-c,
nix-store-test-support,
sqlite,
openssl,
rapidcheck,
gtest,
@@ -76,10 +75,7 @@ mkMesonExecutable (finalAttrs: {
runCommand "${finalAttrs.pname}-run"
{
meta.broken = !stdenv.hostPlatform.emulatorAvailable buildPackages;
nativeBuildInputs = [
writableTmpDirAsHomeHook
openssl
];
buildInputs = [ writableTmpDirAsHomeHook ];
}
(
''
+12 -103
View File
@@ -13,11 +13,6 @@
// C library headers for SSO provider support
# include <aws/auth/credentials.h>
// C library headers for custom logging
# include <aws/common/logging.h>
# include <cstdarg>
# include <boost/unordered/concurrent_flat_map.hpp>
# include <chrono>
@@ -35,101 +30,6 @@ AwsAuthError::AwsAuthError(int errorCode)
namespace {
/**
* Map AWS log level to Nix verbosity.
* AWS levels: AWS_LL_NONE=0, AWS_LL_FATAL=1, AWS_LL_ERROR=2, AWS_LL_WARN=3,
* AWS_LL_INFO=4, AWS_LL_DEBUG=5, AWS_LL_TRACE=6
*
* We map very conservatively because the AWS SDK is extremely noisy. What AWS
* considers "info" includes low-level details like "Initializing epoll" and
* "Starting event-loop thread". What it considers "errors" includes expected
* conditions like missing ~/.aws/config or IMDS being unavailable on non-EC2.
*
* To avoid spamming users, we only show FATAL at default verbosity. Everything
* else requires -vvvvv (lvlDebug) or higher to see.
*/
static Verbosity awsLogLevelToVerbosity(enum aws_log_level level)
{
switch (level) {
case AWS_LL_FATAL:
return lvlError;
case AWS_LL_ERROR:
case AWS_LL_WARN:
case AWS_LL_INFO:
return lvlDebug;
case AWS_LL_DEBUG:
case AWS_LL_TRACE:
return lvlVomit;
// AWS_LL_NONE and AWS_LL_COUNT are enum sentinels, not real log levels
case AWS_LL_NONE:
case AWS_LL_COUNT:
return lvlDebug;
}
unreachable();
}
/**
* Custom AWS logger that routes logs through Nix's logging infrastructure.
*
* The AWS CRT C++ wrapper (ApiHandle::InitializeLogging) only supports FILE*
* or filename-based logging. The underlying C library supports custom loggers
* via aws_logger struct with a vtable containing callback functions.
*/
static int nixAwsLoggerLog(
struct aws_logger * logger, enum aws_log_level logLevel, aws_log_subject_t subject, const char * format, ...)
{
Verbosity nixLevel = awsLogLevelToVerbosity(logLevel);
if (nixLevel > verbosity)
return AWS_OP_SUCCESS; /* Bail out early to avoid formatting the message unnecessarily. */
va_list args;
va_start(args, format);
std::array<char, 4096> buffer{};
auto res = vsnprintf(buffer.data(), buffer.size(), format, args);
va_end(args);
if (res < 0) /* Skip garbage debug messages in case the SDK is busted. */
return AWS_OP_SUCCESS;
const char * subjectName = aws_log_subject_name(subject);
printMsgUsing(nix::logger, nixLevel, "(aws:%s) %s", subjectName ? subjectName : "unknown", chomp(buffer.data()));
return AWS_OP_SUCCESS;
}
/**
* Get current log level for a subject - determines which messages will be logged.
* Must be consistent with awsLogLevelToVerbosity mapping.
*/
static aws_log_level nixAwsLoggerGetLevel(struct aws_logger * logger, aws_log_subject_t subject)
{
// Map Nix verbosity back to AWS log level (inverse of awsLogLevelToVerbosity)
if (verbosity >= lvlVomit)
return AWS_LL_TRACE;
if (verbosity >= lvlDebug)
return AWS_LL_INFO; // error/warn/info are all mapped to lvlDebug
return AWS_LL_FATAL;
}
static void initialiseAwsLogger()
{
static std::once_flag initialised; /* aws_logger_set must only be called once */
std::call_once(initialised, []() {
static aws_logger_vtable nixAwsLoggerVtable = {
.log = nixAwsLoggerLog,
.get_log_level = nixAwsLoggerGetLevel,
.clean_up = [](struct aws_logger *) {}, // No resources to clean up
.set_log_level = nullptr,
};
static aws_logger nixAwsLogger = {
.vtable = &nixAwsLoggerVtable,
.allocator = nullptr,
.p_impl = nullptr,
};
aws_logger_set(&nixAwsLogger);
});
}
/**
* Helper function to wrap a C credentials provider in the C++ interface.
* This replicates the static s_CreateWrappedProvider from aws-crt-cpp.
@@ -219,9 +119,18 @@ class AwsCredentialProviderImpl : public AwsCredentialProvider
public:
AwsCredentialProviderImpl()
{
// Install custom logger that routes AWS CRT logs through Nix's logging infrastructure.
// This ensures AWS logs respect Nix's verbosity settings and are formatted consistently.
initialiseAwsLogger();
// Map Nix's verbosity to AWS CRT log level
Aws::Crt::LogLevel logLevel;
if (verbosity >= lvlVomit) {
logLevel = Aws::Crt::LogLevel::Trace;
} else if (verbosity >= lvlDebug) {
logLevel = Aws::Crt::LogLevel::Debug;
} else if (verbosity >= lvlChatty) {
logLevel = Aws::Crt::LogLevel::Info;
} else {
logLevel = Aws::Crt::LogLevel::Warn;
}
apiHandle.InitializeLogging(logLevel, stderr);
// Create a shared TLS context for SSO (required for HTTPS connections)
auto allocator = Aws::Crt::ApiAllocator();
+2 -2
View File
@@ -156,7 +156,7 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
config.compression, teeSinkCompressed, config.parallelCompression, config.compressionLevel);
TeeSink teeSinkUncompressed{*compressionSink, narHashSink};
TeeSource teeSource{narSource, teeSinkUncompressed};
narAccessor = makeNarAccessor(parseNarListing(teeSource));
narAccessor = makeNarAccessor(teeSource);
compressionSink->finish();
fileSink.flush();
}
@@ -562,7 +562,7 @@ ref<SourceAccessor> BinaryCacheStore::getFSAccessor(bool requireValidPath)
std::shared_ptr<SourceAccessor> BinaryCacheStore::getFSAccessor(const StorePath & storePath, bool requireValidPath)
{
return static_cast<ref<SourceAccessor>>(getRemoteFSAccessor(requireValidPath)->accessObject(storePath));
return getRemoteFSAccessor(requireValidPath)->accessObject(storePath);
}
void BinaryCacheStore::addSignatures(const StorePath & storePath, const std::set<Signature> & sigs)
+9 -11
View File
@@ -175,7 +175,7 @@ struct LogFile
AutoCloseFD fd;
std::shared_ptr<BufferedSink> fileSink, sink;
LogFile(Store & store, const StorePath & drvPath, const LogFileSettings & logSettings);
LogFile(Store & store, const StorePath & drvPath);
~LogFile();
};
@@ -451,7 +451,7 @@ Goal::Co DerivationBuildingGoal::buildWithHook(
hook->toHook.writeSide.close();
/* Create the log file and pipe. */
std::unique_ptr<LogFile> logFile = std::make_unique<LogFile>(worker.store, drvPath, settings.getLogFileSettings());
std::unique_ptr<LogFile> logFile = std::make_unique<LogFile>(worker.store, drvPath);
std::set<MuxablePipePollState::CommChannel> fds;
fds.insert(hook->fromHook.readSide.get());
@@ -635,9 +635,7 @@ Goal::Co DerivationBuildingGoal::buildLocally(
std::unique_ptr<BuildLog> buildLog;
std::unique_ptr<LogFile> logFile;
auto openLogFile = [&]() {
logFile = std::make_unique<LogFile>(worker.store, drvPath, settings.getLogFileSettings());
};
auto openLogFile = [&]() { logFile = std::make_unique<LogFile>(worker.store, drvPath); };
auto closeLogFile = [&]() { logFile.reset(); };
@@ -656,7 +654,7 @@ Goal::Co DerivationBuildingGoal::buildLocally(
};
std::unique_ptr<Activity> actLock;
DerivationBuilderUnique builder;
std::unique_ptr<DerivationBuilder> builder;
Descriptor builderOut;
// Will continue here while waiting for a build user below
@@ -1043,9 +1041,9 @@ HookReply DerivationBuildingGoal::tryBuildHook(const DerivationOptions<StorePath
#endif
}
LogFile::LogFile(Store & store, const StorePath & drvPath, const LogFileSettings & logSettings)
LogFile::LogFile(Store & store, const StorePath & drvPath)
{
if (!logSettings.keepLog)
if (!settings.keepLog)
return;
auto baseName = std::string(baseNameOf(store.printStorePath(drvPath)));
@@ -1054,11 +1052,11 @@ LogFile::LogFile(Store & store, const StorePath & drvPath, const LogFileSettings
if (auto localStore = dynamic_cast<LocalStore *>(&store))
logDir = localStore->config->logDir;
else
logDir = logSettings.nixLogDir;
logDir = settings.nixLogDir;
Path dir = fmt("%s/%s/%s/", logDir, LocalFSStore::drvsLogDir, baseName.substr(0, 2));
createDirs(dir);
Path logFileName = fmt("%s/%s%s", dir, baseName.substr(2), logSettings.compressLog ? ".bz2" : "");
Path logFileName = fmt("%s/%s%s", dir, baseName.substr(2), settings.compressLog ? ".bz2" : "");
fd = toDescriptor(open(
logFileName.c_str(),
@@ -1073,7 +1071,7 @@ LogFile::LogFile(Store & store, const StorePath & drvPath, const LogFileSettings
fileSink = std::make_shared<FdSink>(fd.get());
if (logSettings.compressLog)
if (settings.compressLog)
sink = std::shared_ptr<CompressionSink>(makeCompressionSink(CompressionAlgo::bzip2, *fileSink));
else
sink = fileSink;
+2 -2
View File
@@ -4,7 +4,6 @@
#include "nix/store/globals.hh"
#include "nix/util/archive.hh"
#include "nix/util/compression.hh"
#include "nix/util/file-system.hh"
namespace nix {
@@ -66,7 +65,8 @@ static void builtinFetchurl(const BuiltinBuilderContext & ctx)
auto executable = ctx.drv.env.find("executable");
if (executable != ctx.drv.env.end() && executable->second == "1") {
chmod(storePath, 0755);
if (chmod(storePath.c_str(), 0755) == -1)
throw SysError("making '%1%' executable", storePath);
}
};
-15
View File
@@ -531,21 +531,6 @@ struct curlFileTransfer : public FileTransfer
if (writtenToSink)
curl_easy_setopt(req, CURLOPT_RESUME_FROM_LARGE, writtenToSink);
/* Note that the underlying strings get copied by libcurl, so the path -> string conversion is ok:
> The application does not have to keep the string around after setting this option.
https://curl.se/libcurl/c/CURLOPT_SSLKEY.html
https://curl.se/libcurl/c/CURLOPT_SSLCERT.html */
if (request.tlsCert) {
curl_easy_setopt(req, CURLOPT_SSLCERTTYPE, "PEM");
curl_easy_setopt(req, CURLOPT_SSLCERT, request.tlsCert->string().c_str());
}
if (request.tlsKey) {
curl_easy_setopt(req, CURLOPT_SSLKEYTYPE, "PEM");
curl_easy_setopt(req, CURLOPT_SSLKEY, request.tlsKey->string().c_str());
}
curl_easy_setopt(req, CURLOPT_ERRORBUFFER, errbuf);
errbuf[0] = 0;
+1 -5
View File
@@ -69,11 +69,6 @@ static std::filesystem::path resolveNixConfDir()
#endif
}
LogFileSettings::LogFileSettings()
: nixLogDir(canonPath(getEnvNonEmpty("NIX_LOG_DIR").value_or(NIX_LOG_DIR)))
{
}
Settings settings;
static GlobalConfig::Register rSettings(&settings);
@@ -86,6 +81,7 @@ Settings::Settings()
canonPath
#endif
(getEnvNonEmpty("NIX_STORE_DIR").value_or(getEnvNonEmpty("NIX_STORE").value_or(NIX_STORE_DIR))))
, nixLogDir(canonPath(getEnvNonEmpty("NIX_LOG_DIR").value_or(NIX_LOG_DIR)))
, nixStateDir(canonPath(getEnvNonEmpty("NIX_STATE_DIR").value_or(NIX_STATE_DIR)))
, nixConfDir(canonPath(getEnvOsNonEmpty(OS_STR("NIX_CONF_DIR"))
.transform([](auto && s) { return std::filesystem::path(s); })
+1 -17
View File
@@ -195,23 +195,7 @@ FileTransferRequest HttpBinaryCacheStore::makeRequest(std::string_view path)
result.query = config->cacheUri.query;
}
FileTransferRequest request(result);
/* Only use the specified SSL certificate and private key if the resolved URL names the same
authority and uses the same protocol. */
if (result.scheme == config->cacheUri.scheme && result.authority == config->cacheUri.authority) {
if (const auto & cert = config->tlsCert.get()) {
debug("using TLS client certificate %s for '%s'", PathFmt(*cert), request.uri);
request.tlsCert = *cert;
}
if (const auto & key = config->tlsKey.get()) {
debug("using TLS client key '%s' for '%s'", PathFmt(*key), request.uri);
request.tlsKey = *key;
}
}
return request;
return FileTransferRequest(result);
}
void HttpBinaryCacheStore::getFile(const std::string & path, Sink & sink)
@@ -190,22 +190,15 @@ struct ExternalBuilder
std::vector<std::string> args;
};
struct DerivationBuilderDeleter
{
void operator()(DerivationBuilder * builder) noexcept;
};
using DerivationBuilderUnique = std::unique_ptr<DerivationBuilder, DerivationBuilderDeleter>;
#ifndef _WIN32 // TODO enable `DerivationBuilder` on Windows
DerivationBuilderUnique makeDerivationBuilder(
std::unique_ptr<DerivationBuilder> makeDerivationBuilder(
LocalStore & store, std::unique_ptr<DerivationBuilderCallbacks> miscMethods, DerivationBuilderParams params);
/**
* @param handler Must be chosen such that it supports the given
* derivation.
*/
DerivationBuilderUnique makeExternalDerivationBuilder(
std::unique_ptr<DerivationBuilder> makeExternalDerivationBuilder(
LocalStore & store,
std::unique_ptr<DerivationBuilderCallbacks> miscMethods,
DerivationBuilderParams params,
@@ -121,16 +121,6 @@ struct FileTransferRequest
ActivityId parentAct;
bool decompress = true;
/**
* Optional path to the client certificate in "PEM" format. Only used for TLS-based protocols.
*/
std::optional<std::filesystem::path> tlsCert;
/**
* Optional path to the client private key in "PEM" format. Only used for TLS-based protocols.
*/
std::optional<std::filesystem::path> tlsKey;
struct UploadData
{
UploadData(StringSource & s)
+49 -80
View File
@@ -143,66 +143,9 @@ struct GCSettings : public virtual Config
};
};
struct LogFileSettings : public virtual Config
class Settings : public virtual Config, private GCSettings
{
/**
* The directory where we log various operations.
*/
const Path nixLogDir;
protected:
LogFileSettings();
public:
Setting<bool> keepLog{
this,
true,
"keep-build-log",
R"(
If set to `true` (the default), Nix writes the build log of a
derivation (i.e. the standard output and error of its builder) to
the directory `/nix/var/log/nix/drvs`. The build log can be
retrieved using the command `nix-store -l path`.
)",
{"build-keep-log"}};
Setting<bool> compressLog{
this,
true,
"compress-build-log",
R"(
If set to `true` (the default), build logs written to
`/nix/var/log/nix/drvs` are compressed on the fly using bzip2.
Otherwise, they are not compressed.
)",
{"build-compress-log"}};
};
struct AutoAllocateUidSettings : public virtual Config
{
Setting<uint32_t> startId{
this,
#ifdef __linux__
0x34000000,
#else
56930,
#endif
"start-id",
"The first UID and GID to use for dynamic ID allocation."};
Setting<uint32_t> uidCount{
this,
#ifdef __linux__
maxIdsPerBuild * 128,
#else
128,
#endif
"id-count",
"The number of UIDs/GIDs to use for dynamic ID allocation."};
};
class Settings : public virtual Config, private AutoAllocateUidSettings, private GCSettings, private LogFileSettings
{
StringSet getDefaultSystemFeatures();
StringSet getDefaultExtraPlatforms();
@@ -228,28 +171,6 @@ public:
return *this;
}
/**
* Get the log file settings.
*/
LogFileSettings & getLogFileSettings()
{
return *this;
}
const LogFileSettings & getLogFileSettings() const
{
return *this;
}
/**
* Get AutoAllocateUidSettings if auto-allocate-uids is enabled.
* @return Pointer to settings if enabled, nullptr otherwise.
*/
const AutoAllocateUidSettings * getAutoAllocateUidSettings() const
{
return autoAllocateUids ? this : nullptr;
}
static unsigned int getDefaultCores();
/**
@@ -257,6 +178,11 @@ public:
*/
Path nixStore;
/**
* The directory where we log various operations.
*/
Path nixLogDir;
/**
* The directory where state is stored.
*/
@@ -689,6 +615,26 @@ public:
true,
Xp::AutoAllocateUids};
Setting<uint32_t> startId{
this,
#ifdef __linux__
0x34000000,
#else
56930,
#endif
"start-id",
"The first UID and GID to use for dynamic ID allocation."};
Setting<uint32_t> uidCount{
this,
#ifdef __linux__
maxIdsPerBuild * 128,
#else
128,
#endif
"id-count",
"The number of UIDs/GIDs to use for dynamic ID allocation."};
#ifdef __linux__
Setting<bool> useCgroups{
this,
@@ -710,6 +656,29 @@ public:
"Whether to impersonate a Linux 2.6 machine on newer kernels.",
{"build-impersonate-linux-26"}};
Setting<bool> keepLog{
this,
true,
"keep-build-log",
R"(
If set to `true` (the default), Nix writes the build log of a
derivation (i.e. the standard output and error of its builder) to
the directory `/nix/var/log/nix/drvs`. The build log can be
retrieved using the command `nix-store -l path`.
)",
{"build-keep-log"}};
Setting<bool> compressLog{
this,
true,
"compress-build-log",
R"(
If set to `true` (the default), build logs written to
`/nix/var/log/nix/drvs` are compressed on the fly using bzip2.
Otherwise, they are not compressed.
)",
{"build-compress-log"}};
Setting<unsigned long> maxLogSize{
this,
0,
@@ -37,12 +37,6 @@ struct HttpBinaryCacheStoreConfig : std::enable_shared_from_this<HttpBinaryCache
(e.g. `brotli`).
)"};
const Setting<std::optional<std::filesystem::path>> tlsCert{
this, std::nullopt, "tls-certificate", "Path to an optional TLS client certificate in PEM format."};
const Setting<std::optional<std::filesystem::path>> tlsKey{
this, std::nullopt, "tls-private-key", "Path to an optional TLS client certificate private key in PEM format."};
static const std::string name()
{
return "HTTP Binary Cache Store";
@@ -1,20 +0,0 @@
#pragma once
///@file
#include "nix/util/nar-cache.hh"
#include <filesystem>
#include <memory>
namespace nix {
/**
* Create a NAR cache with local disk storage.
*
* Uses file locks to ensure only one process downloads a NAR at a time.
*
* @param cacheDir Directory to store cached NAR files
*/
std::unique_ptr<NarCache> makeLocalNarCache(std::filesystem::path cacheDir);
} // namespace nix
@@ -50,7 +50,6 @@ headers = [ config_pub_h ] + files(
'length-prefixed-protocol-helper.hh',
'local-binary-cache-store.hh',
'local-fs-store.hh',
'local-nar-cache.hh',
'local-overlay-store.hh',
'local-store.hh',
'log-store.hh',
@@ -3,7 +3,6 @@
#include "nix/util/source-accessor.hh"
#include "nix/util/ref.hh"
#include "nix/util/nar-cache.hh"
#include "nix/store/store-api.hh"
namespace nix {
@@ -14,15 +13,20 @@ class RemoteFSAccessor : public SourceAccessor
/**
* Map from store path hash part to NAR hash. Used to then look up
* in the NAR cache. The indirection allows avoiding opening multiple
* in `nars`. The indirection allows avoiding opening multiple
* redundant NAR accessors for the same NAR.
*/
std::map<std::string, Hash, std::less<>> narHashes;
std::unique_ptr<NarCache> narCache;
/**
* Map from NAR hash to NAR accessor.
*/
std::map<Hash, ref<SourceAccessor>> nars;
bool requireValidPath;
std::optional<std::filesystem::path> cacheDir;
std::pair<ref<SourceAccessor>, CanonPath> fetch(const CanonPath & path);
friend struct BinaryCacheStore;
@@ -32,7 +36,7 @@ public:
/**
* @return nullptr if the store does not contain any object at that path.
*/
ref<NarAccessor> accessObject(const StorePath & path);
std::shared_ptr<SourceAccessor> accessObject(const StorePath & path);
RemoteFSAccessor(
ref<Store> store, bool requireValidPath = true, std::optional<std::filesystem::path> cacheDir = {});
+1 -1
View File
@@ -15,7 +15,7 @@ Path LocalFSStoreConfig::getDefaultStateDir()
Path LocalFSStoreConfig::getDefaultLogDir()
{
return settings.getLogFileSettings().nixLogDir;
return settings.nixLogDir;
}
LocalFSStoreConfig::LocalFSStoreConfig(PathView rootDir, const Params & params)
-134
View File
@@ -1,134 +0,0 @@
#include "nix/util/nar-cache.hh"
#include "nix/util/file-system.hh"
#include "nix/util/file-descriptor.hh"
#include "nix/util/fs-sink.hh"
#include "nix/store/pathlocks.hh"
#include <nlohmann/json.hpp>
#include <optional>
namespace nix {
namespace {
/**
* NAR cache with local disk storage (private implementation).
*
* Uses file locks to ensure only one process downloads a NAR at a time.
*/
class LocalNarCache : public NarCache
{
RestoreSink cacheSink;
public:
LocalNarCache(std::filesystem::path cacheDir)
: cacheSink(false)
{
createDirs(cacheDir);
cacheSink.dstPath = std::move(cacheDir);
}
ref<NarAccessor> getOrInsert(const Hash & narHash, std::function<void(Sink &)> populate) override;
};
} // anonymous namespace
ref<NarAccessor> LocalNarCache::getOrInsert(const Hash & narHash, std::function<void(Sink &)> populate)
{
// Check in-memory cache first
if (auto * accessor = get(nars, narHash))
return *accessor;
auto cacheAccessor = [&](ref<NarAccessor> accessor) {
nars.emplace(narHash, accessor);
return accessor;
};
auto makeCacheFile = [&](const std::string & ext) -> CanonPath {
return {narHash.to_string(HashFormat::Nix32, false) + "." + ext};
};
auto cacheFile = makeCacheFile("nar");
auto listingFile = makeCacheFile("ls");
auto lockFile = makeCacheFile("lock");
auto cacheFilePath = cacheSink.dstPath / cacheFile.rel();
auto lockFilePath = cacheSink.dstPath / lockFile.rel();
auto listingFilePath = cacheSink.dstPath / listingFile.rel();
// Helper to try loading from cache files using FD operations to avoid race conditions
auto tryLoadFromCache = [&]() -> std::optional<ref<NarAccessor>> {
try {
// Try to open cache file - will throw if doesn't exist
AutoCloseFD cacheFD = openFileReadonly(cacheFilePath);
// Try lazy accessor with listing file first
try {
AutoCloseFD listingFD = openFileReadonly(listingFilePath);
auto listingContent = readFile(listingFD.get());
return cacheAccessor(makeLazyNarAccessor(
nlohmann::json::parse(listingContent).template get<NarListing>(),
seekableGetNarBytes(cacheFilePath)));
} catch (SystemError &) {
// Listing file missing or invalid, fall back to full NAR
}
// Fall back to reading full NAR
auto narContent = readFile(cacheFD.get());
return cacheAccessor(makeNarAccessor(std::move(narContent)));
} catch (SystemError &) {
// Cache file doesn't exist or can't be opened
return std::nullopt;
}
};
// Check if already cached (before acquiring lock)
if (auto accessor = tryLoadFromCache())
return *accessor;
// Acquire lock to ensure only one process downloads this NAR
AutoCloseFD lockFD = openLockFile(lockFilePath, true);
FdLock lock(lockFD.get(), ltWrite, true, "waiting for NAR cache lock");
// Check again after acquiring lock (another process might have just finished)
if (auto accessor = tryLoadFromCache())
return *accessor;
// Download and cache the NAR
NarListing listing;
try {
/* FIXME: do this asynchronously. */
cacheSink.createRegularFile(cacheFile, [&](CreateRegularFileSink & fileSink) {
auto source = sinkToSource([&](Sink & parseSink) {
TeeSink teeSink{fileSink, parseSink};
populate(teeSink);
});
listing = parseNarListing(*source);
});
} catch (...) {
ignoreExceptionExceptInterrupt();
StringSink narSink;
populate(narSink);
return cacheAccessor(makeNarAccessor(std::move(narSink.s)));
}
try {
cacheSink.createRegularFile(listingFile, [&](CreateRegularFileSink & sink) {
auto s = nlohmann::json(listing).dump();
StringSource source{s};
source.drainInto(sink);
});
} catch (...) {
ignoreExceptionExceptInterrupt();
}
return cacheAccessor(makeLazyNarAccessor(std::move(listing), seekableGetNarBytes(cacheFilePath)));
}
std::unique_ptr<NarCache> makeLocalNarCache(std::filesystem::path cacheDir)
{
return std::make_unique<LocalNarCache>(std::move(cacheDir));
}
} // namespace nix
+2 -1
View File
@@ -174,7 +174,8 @@ LocalStore::LocalStore(ref<const Config> config)
if (st.st_uid != 0 || st.st_gid != gr->gr_gid || (st.st_mode & ~S_IFMT) != perm) {
if (chown(config->realStoreDir.get().c_str(), 0, gr->gr_gid) == -1)
throw SysError("changing ownership of path '%1%'", config->realStoreDir);
chmod(config->realStoreDir.get(), perm);
if (chmod(config->realStoreDir.get().c_str(), perm) == -1)
throw SysError("changing permissions on path '%1%'", config->realStoreDir);
}
}
}
+5 -7
View File
@@ -158,15 +158,14 @@ sqlite = dependency('sqlite3', 'sqlite', version : '>=3.6.19')
deps_private += sqlite
s3_aws_auth = get_option('s3-aws-auth')
aws_crt_cpp = dependency(
'aws-crt-cpp',
required : s3_aws_auth,
method : 'cmake',
modules : [ 'AWS::aws-crt-cpp' ],
)
aws_crt_cpp = cxx.find_library('aws-crt-cpp', required : s3_aws_auth)
if s3_aws_auth.enabled()
deps_other += aws_crt_cpp
aws_c_common = cxx.find_library('aws-c-common', required : true)
deps_other += aws_c_common
aws_c_auth = cxx.find_library('aws-c-auth', required : true)
deps_other += aws_c_auth
endif
configdata_pub.set('NIX_WITH_AWS_AUTH', s3_aws_auth.enabled().to_int())
@@ -315,7 +314,6 @@ sources = files(
'legacy-ssh-store.cc',
'local-binary-cache-store.cc',
'local-fs-store.cc',
'local-nar-cache.cc',
'local-overlay-store.cc',
'local-store.cc',
'log-store.cc',
+2 -2
View File
@@ -3,7 +3,6 @@
#include "nix/util/signals.hh"
#include "nix/store/posix-fs-canonicalise.hh"
#include "nix/util/posix-source-accessor.hh"
#include "nix/util/file-system.hh"
#include <cstdlib>
#include <cstring>
@@ -21,7 +20,8 @@ namespace nix {
static void makeWritable(const Path & path)
{
auto st = lstat(path);
chmod(path, st.st_mode | S_IWUSR);
if (chmod(path.c_str(), st.st_mode | S_IWUSR) == -1)
throw SysError("changing writability of '%1%'", path);
}
struct MakeReadOnly
+2 -4
View File
@@ -14,7 +14,6 @@
libseccomp,
nlohmann_json,
sqlite,
cmake, # for resolving aws-crt-cpp dep
busybox-sandbox-shell ? null,
@@ -26,7 +25,7 @@
withAWS ?
# Default is this way because there have been issues building this dependency
(lib.meta.availableOn stdenv.hostPlatform aws-c-common),
lib.meta.availableOn stdenv.hostPlatform aws-c-common,
}:
let
@@ -58,8 +57,7 @@ mkMesonLibrary (finalAttrs: {
(fileset.fileFilter (file: file.hasExt "sql") ./.)
];
nativeBuildInputs =
lib.optional withAWS cmake ++ lib.optional embeddedSandboxShell unixtools.hexdump;
nativeBuildInputs = lib.optional embeddedSandboxShell unixtools.hexdump;
buildInputs = [
boost
+2 -1
View File
@@ -24,7 +24,8 @@ static void canonicaliseTimestampAndPermissions(const Path & path, const struct
bool isDir = S_ISDIR(st.st_mode);
if ((mode != 0444 || isDir) && mode != 0555) {
mode = (st.st_mode & S_IFMT) | 0444 | (st.st_mode & S_IXUSR || isDir ? 0111 : 0);
chmod(path, mode);
if (chmod(path.c_str(), mode) == -1)
throw SysError("changing mode of '%1%' to %2$o", path, mode);
}
}
+73 -12
View File
@@ -1,14 +1,21 @@
#include <nlohmann/json.hpp>
#include "nix/store/remote-fs-accessor.hh"
#include "nix/store/local-nar-cache.hh"
#include "nix/util/nar-accessor.hh"
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
namespace nix {
RemoteFSAccessor::RemoteFSAccessor(
ref<Store> store, bool requireValidPath, std::optional<std::filesystem::path> cacheDir)
ref<Store> store, bool requireValidPath, std::optional<std::filesystem::path> cacheDir_)
: store(store)
, narCache(cacheDir ? makeLocalNarCache(*cacheDir) : makeMemoryNarCache())
, requireValidPath(requireValidPath)
, cacheDir(std::move(cacheDir_))
{
if (cacheDir)
createDirs(*cacheDir);
}
std::pair<ref<SourceAccessor>, CanonPath> RemoteFSAccessor::fetch(const CanonPath & path)
@@ -19,20 +26,74 @@ std::pair<ref<SourceAccessor>, CanonPath> RemoteFSAccessor::fetch(const CanonPat
return {ref{accessObject(storePath)}, CanonPath{restPath}};
}
ref<NarAccessor> RemoteFSAccessor::accessObject(const StorePath & storePath)
std::shared_ptr<SourceAccessor> RemoteFSAccessor::accessObject(const StorePath & storePath)
{
// Check if we already have the NAR hash for this store path
if (auto * narHash = get(narHashes, storePath.hashPart()))
return narCache->getOrInsert(*narHash, [&](Sink & sink) { store->narFromPath(storePath, sink); });
if (auto * narHash = get(narHashes, storePath.hashPart())) {
if (auto * accessor = get(nars, *narHash))
return *accessor;
}
// Query the path info to get the NAR hash
auto info = store->queryPathInfo(storePath);
// Cache the mapping from store path to NAR hash
narHashes.emplace(storePath.hashPart(), info->narHash);
auto cacheAccessor = [&](ref<SourceAccessor> accessor) {
narHashes.emplace(storePath.hashPart(), info->narHash);
nars.emplace(info->narHash, accessor);
return accessor;
};
// Get or create the NAR accessor
return narCache->getOrInsert(info->narHash, [&](Sink & sink) { store->narFromPath(storePath, sink); });
auto getNar = [&]() {
StringSink sink;
store->narFromPath(storePath, sink);
return std::move(sink.s);
};
if (cacheDir) {
auto makeCacheFile = [&](const std::string & ext) {
auto res = *cacheDir / info->narHash.to_string(HashFormat::Nix32, false);
res += ".";
res += ext;
return res;
};
auto cacheFile = makeCacheFile("nar");
auto listingFile = makeCacheFile("ls");
if (nix::pathExists(cacheFile)) {
try {
return cacheAccessor(makeLazyNarAccessor(
nlohmann::json::parse(nix::readFile(listingFile)).template get<NarListing>(),
seekableGetNarBytes(cacheFile)));
} catch (SystemError &) {
}
try {
return cacheAccessor(makeNarAccessor(nix::readFile(cacheFile)));
} catch (SystemError &) {
}
}
auto nar = getNar();
try {
/* FIXME: do this asynchronously. */
writeFile(cacheFile, nar);
} catch (...) {
ignoreExceptionExceptInterrupt();
}
auto narAccessor = makeNarAccessor(std::move(nar));
try {
nlohmann::json j = narAccessor->getListing();
writeFile(listingFile, j.dump());
} catch (...) {
ignoreExceptionExceptInterrupt();
}
return cacheAccessor(narAccessor);
}
return cacheAccessor(makeNarAccessor(getNar()));
}
std::optional<SourceAccessor::Stat> RemoteFSAccessor::maybeLstat(const CanonPath & path)
+1 -1
View File
@@ -841,7 +841,7 @@ ref<SourceAccessor> RemoteStore::getFSAccessor(bool requireValidPath)
std::shared_ptr<SourceAccessor> RemoteStore::getFSAccessor(const StorePath & path, bool requireValidPath)
{
return static_cast<ref<SourceAccessor>>(getRemoteFSAccessor(requireValidPath)->accessObject(path));
return getRemoteFSAccessor(requireValidPath)->accessObject(path);
}
void RemoteStore::ConnectionHandle::withFramedSink(std::function<void(Sink & sink)> fun)
@@ -85,7 +85,7 @@ struct ChrootDerivationBuilder : virtual DerivationBuilderImpl
instead.) */
std::filesystem::path chrootTmpDir = chrootRootDir / "tmp";
createDirs(chrootTmpDir);
chmod(chrootTmpDir, 01777);
chmod_(chrootTmpDir, 01777);
/* Create a /etc/passwd with entries for the build user and the
nobody account. The latter is kind of a hack to support
@@ -119,7 +119,7 @@ struct ChrootDerivationBuilder : virtual DerivationBuilderImpl
build user. */
std::filesystem::path chrootStoreDir = chrootRootDir / std::filesystem::path(store.storeDir).relative_path();
createDirs(chrootStoreDir);
chmod(chrootStoreDir, 01775);
chmod_(chrootStoreDir, 01775);
if (buildUser && chown(chrootStoreDir.c_str(), 0, buildUser->getGID()) == -1)
throw SysError("cannot change ownership of %1%", PathFmt(chrootStoreDir));
+21 -32
View File
@@ -98,13 +98,10 @@ public:
{
}
/**
* Cleanup code to run when destroying any DerivationBuilderImpl implementation.
*/
void cleanupOnDestruction() noexcept
~DerivationBuilderImpl()
{
/* Careful: we should never ever throw an exception from a
noexcept function. */
destructor. */
try {
killChild();
} catch (...) {
@@ -235,7 +232,7 @@ protected:
*/
virtual std::unique_ptr<UserLock> getBuildUser()
{
return acquireUserLock(settings.buildUsersGroup, 1, false);
return acquireUserLock(1, false);
}
/**
@@ -570,6 +567,12 @@ SingleDrvOutputs DerivationBuilderImpl::unprepareBuild()
return builtOutputs;
}
static void chmod_(const std::filesystem::path & path, mode_t mode)
{
if (chmod(path.c_str(), mode) == -1)
throw SysError("setting permissions on %s", PathFmt(path));
}
/* Move/rename path 'src' to 'dst'. Temporarily make 'src' writable if
it's a directory and we're not root (to be able to update the
directory's parent link ".."). */
@@ -580,12 +583,12 @@ static void movePath(const std::filesystem::path & src, const std::filesystem::p
bool changePerm = (geteuid() && S_ISDIR(st.st_mode) && !(st.st_mode & S_IWUSR));
if (changePerm)
chmod(src, st.st_mode | S_IWUSR);
chmod_(src, st.st_mode | S_IWUSR);
std::filesystem::rename(src, dst);
if (changePerm)
chmod(dst, st.st_mode);
chmod_(dst, st.st_mode);
}
static void replaceValidPath(const std::filesystem::path & storePath, const std::filesystem::path & tmpPath)
@@ -818,7 +821,8 @@ std::optional<Descriptor> DerivationBuilderImpl::startBuild()
std::string slaveName = getPtsName(builderOut.get());
if (buildUser) {
chmod(slaveName, 0600);
if (chmod(slaveName.c_str(), 0600))
throw SysError("changing mode of pseudoterminal slave");
if (chown(slaveName.c_str(), buildUser->getUID(), 0))
throw SysError("changing owner of pseudoterminal slave");
@@ -1919,14 +1923,14 @@ void DerivationBuilderImpl::cleanupBuild(bool force)
* This hardens against an attack which smuggles a file descriptor
* to make use of the temporary directory.
*/
chmod(topTmpDir, 0000);
chmod(topTmpDir.c_str(), 0000);
/* Don't keep temporary directories for builtins because they
might have privileged stuff (like a copy of netrc). */
if (settings.keepFailed && !force && !drv.isBuiltin()) {
printError("note: keeping build directory %s", PathFmt(tmpDir));
chmod(topTmpDir, 0755);
chmod(tmpDir, 0755);
chmod(topTmpDir.c_str(), 0755);
chmod(tmpDir.c_str(), 0755);
} else
deletePath(topTmpDir);
topTmpDir = "";
@@ -1970,20 +1974,7 @@ StorePath DerivationBuilderImpl::makeFallbackPath(const StorePath & path)
namespace nix {
void DerivationBuilderDeleter::operator()(DerivationBuilder * builder) noexcept
{
if (!builder) /* Idempotent and handles nullptr as any deleter must. */
return;
if (auto builderImpl = dynamic_cast<DerivationBuilderImpl *>(builder))
/* Note that this might call into virtual functions, which we can't do in a destructor of
the DerivationBuilderImpl itself. */
builderImpl->cleanupOnDestruction();
delete builder;
}
std::unique_ptr<DerivationBuilder, DerivationBuilderDeleter> makeDerivationBuilder(
std::unique_ptr<DerivationBuilder> makeDerivationBuilder(
LocalStore & store, std::unique_ptr<DerivationBuilderCallbacks> miscMethods, DerivationBuilderParams params)
{
bool useSandbox = false;
@@ -2034,19 +2025,17 @@ std::unique_ptr<DerivationBuilder, DerivationBuilderDeleter> makeDerivationBuild
throw Error("feature 'uid-range' is only supported in sandboxed builds");
#ifdef __APPLE__
return DerivationBuilderUnique(
new DarwinDerivationBuilder(store, std::move(miscMethods), std::move(params), useSandbox));
return std::make_unique<DarwinDerivationBuilder>(store, std::move(miscMethods), std::move(params), useSandbox);
#elif defined(__linux__)
if (useSandbox)
return DerivationBuilderUnique(
new ChrootLinuxDerivationBuilder(store, std::move(miscMethods), std::move(params)));
return std::make_unique<ChrootLinuxDerivationBuilder>(store, std::move(miscMethods), std::move(params));
return DerivationBuilderUnique(new LinuxDerivationBuilder(store, std::move(miscMethods), std::move(params)));
return std::make_unique<LinuxDerivationBuilder>(store, std::move(miscMethods), std::move(params));
#else
if (useSandbox)
throw Error("sandboxing builds is not supported on this platform");
return DerivationBuilderUnique(new DerivationBuilderImpl(store, std::move(miscMethods), std::move(params)));
return std::make_unique<DerivationBuilderImpl>(store, std::move(miscMethods), std::move(params));
#endif
}
@@ -106,14 +106,13 @@ struct ExternalDerivationBuilder : DerivationBuilderImpl
}
};
DerivationBuilderUnique makeExternalDerivationBuilder(
std::unique_ptr<DerivationBuilder> makeExternalDerivationBuilder(
LocalStore & store,
std::unique_ptr<DerivationBuilderCallbacks> miscMethods,
DerivationBuilderParams params,
const ExternalBuilder & handler)
{
return DerivationBuilderUnique(
new ExternalDerivationBuilder(store, std::move(miscMethods), std::move(params), handler));
return std::make_unique<ExternalDerivationBuilder>(store, std::move(miscMethods), std::move(params), handler);
}
} // namespace nix
@@ -215,7 +215,7 @@ struct ChrootLinuxDerivationBuilder : ChrootDerivationBuilder, LinuxDerivationBu
std::unique_ptr<UserLock> getBuildUser() override
{
return acquireUserLock(settings.buildUsersGroup, drvOptions.useUidRange(drv) ? 65536 : 1, true);
return acquireUserLock(drvOptions.useUidRange(drv) ? 65536 : 1, true);
}
void prepareUser() override
@@ -586,7 +586,7 @@ struct ChrootLinuxDerivationBuilder : ChrootDerivationBuilder, LinuxDerivationBu
auto dst = chrootRootDir / i.first.relative_path();
createDirs(dst.parent_path());
writeFile(dst, std::string_view((const char *) sh, sizeof(sh)));
chmod(dst, 0555);
chmod_(dst, 0555);
} else
# endif
{
@@ -629,7 +629,7 @@ struct ChrootLinuxDerivationBuilder : ChrootDerivationBuilder, LinuxDerivationBu
/* Make sure /dev/pts/ptmx is world-writable. With some
Linux versions, it is created with permissions 0. */
chmod(chrootRootDir / "dev" / "pts" / "ptmx", 0666);
chmod_(chrootRootDir / "dev" / "pts" / "ptmx", 0666);
} else {
if (errno != EINVAL)
throw SysError("mounting /dev/pts");
@@ -640,7 +640,7 @@ struct ChrootLinuxDerivationBuilder : ChrootDerivationBuilder, LinuxDerivationBu
/* Make /etc unwritable */
if (!drvOptions.useUidRange(drv))
chmod(chrootRootDir / "etc", 0555);
chmod_(chrootRootDir / "etc", 0555);
/* Unshare this mount namespace. This is necessary because
pivot_root() below changes the root of the mount
@@ -36,7 +36,7 @@ struct UserLock
* Acquire a user lock for a UID range of size `nrIds`. Note that this
* may return nullptr if no user is available.
*/
std::unique_ptr<UserLock> acquireUserLock(const std::string & userGroup, uid_t nrIds, bool useUserNamespace);
std::unique_ptr<UserLock> acquireUserLock(uid_t nrIds, bool useUserNamespace);
bool useBuildUsers();
+30 -35
View File
@@ -64,15 +64,15 @@ struct SimpleUserLock : UserLock
return supplementaryGIDs;
}
static std::unique_ptr<UserLock>
acquire(const std::filesystem::path & userPoolDir, const std::string & buildUsersGroup)
static std::unique_ptr<UserLock> acquire()
{
assert(buildUsersGroup != "");
assert(settings.buildUsersGroup != "");
createDirs(settings.nixStateDir + "/userpool");
/* Get the members of the build-users-group. */
struct group * gr = getgrnam(buildUsersGroup.c_str());
struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str());
if (!gr)
throw Error("the group '%s' specified in 'build-users-group' does not exist", buildUsersGroup);
throw Error("the group '%s' specified in 'build-users-group' does not exist", settings.buildUsersGroup);
/* Copy the result of getgrnam. */
Strings users;
@@ -82,7 +82,7 @@ struct SimpleUserLock : UserLock
}
if (users.empty())
throw Error("the build users group '%s' has no members", buildUsersGroup);
throw Error("the build users group '%s' has no members", settings.buildUsersGroup);
/* Find a user account that isn't currently in use for another
build. */
@@ -91,13 +91,13 @@ struct SimpleUserLock : UserLock
struct passwd * pw = getpwnam(i.c_str());
if (!pw)
throw Error("the user '%s' in the group '%s' does not exist", i, buildUsersGroup);
throw Error("the user '%s' in the group '%s' does not exist", i, settings.buildUsersGroup);
auto fnUserLock = userPoolDir / std::to_string(pw->pw_uid);
auto fnUserLock = fmt("%s/userpool/%s", settings.nixStateDir, pw->pw_uid);
AutoCloseFD fd = open(fnUserLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600);
if (!fd)
throw SysError("opening user lock %s", PathFmt(fnUserLock));
throw SysError("opening user lock '%s'", fnUserLock);
if (lockFile(fd.get(), ltWrite, false)) {
auto lock = std::make_unique<SimpleUserLock>();
@@ -108,7 +108,7 @@ struct SimpleUserLock : UserLock
/* Sanity check... */
if (lock->uid == getuid() || lock->uid == geteuid())
throw Error("the Nix user should not be a member of '%s'", buildUsersGroup);
throw Error("the Nix user should not be a member of '%s'", settings.buildUsersGroup);
#ifdef __linux__
/* Get the list of supplementary groups of this user. This is
@@ -158,37 +158,36 @@ struct AutoUserLock : UserLock
return {};
}
static std::unique_ptr<UserLock> acquire(
const std::filesystem::path & userPoolDir,
const std::string & buildUsersGroup,
uid_t nrIds,
bool useUserNamespace,
const AutoAllocateUidSettings & uidSettings)
static std::unique_ptr<UserLock> acquire(uid_t nrIds, bool useUserNamespace)
{
#if !defined(__linux__)
useUserNamespace = false;
#endif
experimentalFeatureSettings.require(Xp::AutoAllocateUids);
assert(uidSettings.startId > 0);
assert(uidSettings.uidCount % maxIdsPerBuild == 0);
assert((uint64_t) uidSettings.startId + (uint64_t) uidSettings.uidCount <= std::numeric_limits<uid_t>::max());
assert(settings.startId > 0);
assert(settings.uidCount % maxIdsPerBuild == 0);
assert((uint64_t) settings.startId + (uint64_t) settings.uidCount <= std::numeric_limits<uid_t>::max());
assert(nrIds <= maxIdsPerBuild);
size_t nrSlots = uidSettings.uidCount / maxIdsPerBuild;
createDirs(settings.nixStateDir + "/userpool2");
size_t nrSlots = settings.uidCount / maxIdsPerBuild;
for (size_t i = 0; i < nrSlots; i++) {
debug("trying user slot '%d'", i);
auto fnUserLock = userPoolDir / fmt("slot-%d", i);
createDirs(settings.nixStateDir + "/userpool2");
auto fnUserLock = fmt("%s/userpool2/slot-%d", settings.nixStateDir, i);
AutoCloseFD fd = open(fnUserLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600);
if (!fd)
throw SysError("opening user lock %s", PathFmt(fnUserLock));
throw SysError("opening user lock '%s'", fnUserLock);
if (lockFile(fd.get(), ltWrite, false)) {
auto firstUid = uidSettings.startId + i * maxIdsPerBuild;
auto firstUid = settings.startId + i * maxIdsPerBuild;
auto pw = getpwuid(firstUid);
if (pw)
@@ -200,9 +199,10 @@ struct AutoUserLock : UserLock
if (useUserNamespace)
lock->firstGid = firstUid;
else {
struct group * gr = getgrnam(buildUsersGroup.c_str());
struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str());
if (!gr)
throw Error("the group '%s' specified in 'build-users-group' does not exist", buildUsersGroup);
throw Error(
"the group '%s' specified in 'build-users-group' does not exist", settings.buildUsersGroup);
lock->firstGid = gr->gr_gid;
}
lock->nrIds = nrIds;
@@ -214,17 +214,12 @@ struct AutoUserLock : UserLock
}
};
std::unique_ptr<UserLock> acquireUserLock(const std::string & userGroup, uid_t nrIds, bool useUserNamespace)
std::unique_ptr<UserLock> acquireUserLock(uid_t nrIds, bool useUserNamespace)
{
if (auto * uidSettings = settings.getAutoAllocateUidSettings()) {
auto userPoolDir = std::filesystem::path{settings.nixStateDir} / "userpool2";
createDirs(userPoolDir);
return AutoUserLock::acquire(userPoolDir, userGroup, nrIds, useUserNamespace, *uidSettings);
} else {
auto userPoolDir = std::filesystem::path{settings.nixStateDir} / "userpool";
createDirs(userPoolDir);
return SimpleUserLock::acquire(userPoolDir, userGroup);
}
if (settings.autoAllocateUids)
return AutoUserLock::acquire(nrIds, useUserNamespace);
else
return SimpleUserLock::acquire();
}
bool useBuildUsers()
+3 -3
View File
@@ -97,8 +97,8 @@ TEST(fchmodatTryNoFollow, works)
sink.createSymlink(CanonPath("dirlink"), "dir");
}
ASSERT_NO_THROW(chmod(tmpDir / "file", 0644));
ASSERT_NO_THROW(chmod(tmpDir / "dir", 0755));
ASSERT_EQ(chmod((tmpDir / "file").c_str(), 0644), 0);
ASSERT_EQ(chmod((tmpDir / "dir").c_str(), 0755), 0);
AutoCloseFD dirFd = openDirectory(tmpDir);
ASSERT_TRUE(dirFd);
@@ -152,7 +152,7 @@ TEST(fchmodatTryNoFollow, fallbackWithoutProc)
sink.createSymlink(CanonPath("link"), "file");
}
ASSERT_NO_THROW(chmod(tmpDir / "file", 0644));
ASSERT_EQ(chmod((tmpDir / "file").c_str(), 0644), 0);
Pid pid = startProcess(
[&] {
+3 -14
View File
@@ -739,19 +739,6 @@ std::filesystem::path makeParentCanonical(const std::filesystem::path & rawPath)
}
}
void chmod(const std::filesystem::path & path, mode_t mode)
{
if (
#ifdef _WIN32
::_wchmod
#else
::chmod
#endif
(path.c_str(), mode)
== -1)
throw SysError("setting permissions on %s", PathFmt(path));
}
bool chmodIfNeeded(const std::filesystem::path & path, mode_t mode, mode_t mask)
{
auto pathString = path.string();
@@ -760,7 +747,9 @@ bool chmodIfNeeded(const std::filesystem::path & path, mode_t mode, mode_t mask)
if (((prevMode ^ mode) & mask) == 0)
return false;
chmod(path, mode);
if (chmod(pathString.c_str(), mode) != 0)
throw SysError("could not set permissions on '%s' to %o", pathString, mode);
return true;
}
@@ -418,17 +418,6 @@ extern PathFilter defaultPathFilter;
*/
bool chmodIfNeeded(const std::filesystem::path & path, mode_t mode, mode_t mask = S_IRWXU | S_IRWXG | S_IRWXO);
/**
* Set permissions on a path, throwing an exception on error.
*
* @param path Path to the file to change the permissions for.
* @param mode New file mode.
*
* @todo stop using this and start using `fchmodatTryNoFollow` (or a different
* wrapper) to avoid TOCTOU issues.
*/
void chmod(const std::filesystem::path & path, mode_t mode);
/**
* @brief A directory iterator that can be used to iterate over the
* contents of a directory. It is similar to std::filesystem::directory_iterator
-1
View File
@@ -53,7 +53,6 @@ headers = files(
'mounted-source-accessor.hh',
'muxable-pipe.hh',
'nar-accessor.hh',
'nar-cache.hh',
'nar-listing.hh',
'os-string.hh',
'pool.hh',
+7 -9
View File
@@ -27,12 +27,7 @@ struct NarAccessor : SourceAccessor
*/
ref<NarAccessor> makeNarAccessor(std::string && nar);
/**
* This NAR accessor doesn't actually access a NAR, and thus cannot read
* the contents of files. It just conveys the information which is
* gotten from `listing`.
*/
ref<NarAccessor> makeNarAccessor(NarListing listing);
ref<NarAccessor> makeNarAccessor(Source & source);
/**
* Create a NAR accessor from a NAR listing (in the format produced by
@@ -49,9 +44,12 @@ GetNarBytes seekableGetNarBytes(const std::filesystem::path & path);
GetNarBytes seekableGetNarBytes(Descriptor fd);
/**
* Creates a NAR accessor from a given listing and a `GetNarBytes` getter.
*/
ref<NarAccessor> makeLazyNarAccessor(NarListing listing, GetNarBytes getNarBytes);
/**
* Creates a NAR accessor from a given stream and a GetNarBytes getter.
* @param source Consumed eagerly. References to it are not persisted in the resulting SourceAccessor.
*/
ref<NarAccessor> makeLazyNarAccessor(Source & source, GetNarBytes getNarBytes);
} // namespace nix
-44
View File
@@ -1,44 +0,0 @@
#pragma once
#include "nix/util/hash.hh"
#include "nix/util/nar-accessor.hh"
#include "nix/util/ref.hh"
#include <filesystem>
#include <functional>
#include <map>
#include <memory>
namespace nix {
/**
* Abstract cache for NAR accessors.
*/
class NarCache
{
protected:
/**
* Map from NAR hash to NAR accessor.
*/
std::map<Hash, ref<NarAccessor>> nars;
public:
virtual ~NarCache() = default;
/**
* Lookup or create a NAR accessor.
*
* @param narHash The NAR hash to use as cache key
* @param populate Function called with a Sink to populate the NAR if not cached
* @return The cached or newly created accessor
*/
virtual ref<NarAccessor> getOrInsert(const Hash & narHash, std::function<void(Sink &)> populate) = 0;
};
/**
* Create an in-memory only NAR cache.
*/
std::unique_ptr<NarCache> makeMemoryNarCache();
} // namespace nix
-1
View File
@@ -151,7 +151,6 @@ sources = [ config_priv_h ] + files(
'memory-source-accessor/json.cc',
'mounted-source-accessor.cc',
'nar-accessor.cc',
'nar-cache.cc',
'nar-listing.cc',
'pos-table.cc',
'position.cc',
+15 -4
View File
@@ -34,8 +34,14 @@ struct NarAccessorImpl : NarAccessor
{
}
NarAccessorImpl(NarListing && listing)
: root{std::move(listing)}
NarAccessorImpl(Source & source)
: root{parseNarListing(source)}
{
}
NarAccessorImpl(Source & source, GetNarBytes getNarBytes)
: root{parseNarListing(source)}
, getNarBytes{std::move(getNarBytes)}
{
}
@@ -141,9 +147,9 @@ ref<NarAccessor> makeNarAccessor(std::string && nar)
return make_ref<NarAccessorImpl>(std::move(nar));
}
ref<NarAccessor> makeNarAccessor(NarListing listing)
ref<NarAccessor> makeNarAccessor(Source & source)
{
return make_ref<NarAccessorImpl>(std::move(listing));
return make_ref<NarAccessorImpl>(source);
}
ref<NarAccessor> makeLazyNarAccessor(NarListing listing, GetNarBytes getNarBytes)
@@ -151,6 +157,11 @@ ref<NarAccessor> makeLazyNarAccessor(NarListing listing, GetNarBytes getNarBytes
return make_ref<NarAccessorImpl>(std::move(listing), getNarBytes);
}
ref<NarAccessor> makeLazyNarAccessor(Source & source, GetNarBytes getNarBytes)
{
return make_ref<NarAccessorImpl>(source, getNarBytes);
}
GetNarBytes seekableGetNarBytes(const std::filesystem::path & path)
{
AutoCloseFD fd = openFileReadonly(path);
-36
View File
@@ -1,36 +0,0 @@
#include "nix/util/nar-cache.hh"
namespace nix {
namespace {
/**
* In-memory only NAR cache (private implementation).
*/
class MemoryNarCache : public NarCache
{
public:
ref<NarAccessor> getOrInsert(const Hash & narHash, std::function<void(Sink &)> populate) override;
};
} // anonymous namespace
ref<NarAccessor> MemoryNarCache::getOrInsert(const Hash & narHash, std::function<void(Sink &)> populate)
{
// Check in-memory cache first
if (auto * accessor = get(nars, narHash))
return *accessor;
StringSink sink;
populate(sink);
auto accessor = makeNarAccessor(std::move(sink.s));
nars.emplace(narHash, accessor);
return accessor;
}
std::unique_ptr<NarCache> makeMemoryNarCache()
{
return std::make_unique<MemoryNarCache>();
}
} // namespace nix
+2 -1
View File
@@ -38,7 +38,8 @@ AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode)
bind(fdSocket.get(), path);
chmod(path, mode);
if (chmod(path.c_str(), mode) == -1)
throw SysError("changing permissions on '%1%'", path);
if (listen(toSocket(fdSocket.get()), 100) == -1)
throw SysError("cannot listen on socket '%1%'", path);
+1 -1
View File
@@ -154,7 +154,7 @@ struct CmdLsNar : Command, MixLs
if (!fd)
throw NativeSysError("opening NAR file %s", PathFmt(narPath));
auto source = FdSource{fd.get()};
list(makeLazyNarAccessor(parseNarListing(source), seekableGetNarBytes(fd.get())), CanonPath{path});
list(makeLazyNarAccessor(source, seekableGetNarBytes(fd.get())), CanonPath{path});
}
};
+1 -1
View File
@@ -910,7 +910,7 @@ static void opServe(Strings opFlags, Strings opArgs)
// FIXME: changing options here doesn't work if we're
// building through the daemon.
verbosity = lvlError;
settings.getLogFileSettings().keepLog = false;
settings.keepLog = false;
settings.useSubstitutes = false;
auto options = ServeProto::Serialise<ServeProto::BuildOptions>::read(*store, rconn);
+2 -14
View File
@@ -224,17 +224,9 @@ if isDaemonNewer "2.34pre" && canUseSandbox; then
mkdir -p "$fifoDir"
mkfifo "$fifoDir/fifo"
chmod a+rw "$fifoDir/fifo"
# When using a separate test store, we need sandbox-paths to access
# the system store (where bash/coreutils live). On NixOS, the test
# uses the system store directly, so this isn't needed (and would
# conflict with input paths).
sandboxPathsArg=()
if ! isTestOnNixOS; then
sandboxPathsArg=(--option sandbox-paths "/nix/store")
fi
out="$(nix flake check ./cancelled-builds --impure -L -j2 \
--option sandbox true \
"${sandboxPathsArg[@]}" \
--option sandbox-paths "${NIX_STORE:-/nix/store}" \
--option sandbox-build-dir /build-tmp \
--option extra-sandbox-paths "/cancelled-builds-fifo=$fifoDir" \
2>&1)" && status=0 || status=$?
@@ -253,14 +245,10 @@ if isDaemonNewer "2.34pre" && canUseSandbox; then
mkdir -p "$fifoDir"
mkfifo "$fifoDir/fifo"
chmod a+rw "$fifoDir/fifo"
sandboxPathsArg=()
if ! isTestOnNixOS; then
sandboxPathsArg=(--option sandbox-paths "/nix/store")
fi
system=$(nix eval --raw --impure --expr builtins.currentSystem)
out="$(nix build --impure -L -j2 \
--option sandbox true \
"${sandboxPathsArg[@]}" \
--option sandbox-paths "${NIX_STORE:-/nix/store}" \
--option sandbox-build-dir /build-tmp \
--option extra-sandbox-paths "/cancelled-builds-fifo=$fifoDir" \
"./cancelled-builds#checks.$system.slow" \
+15 -8
View File
@@ -75,6 +75,21 @@ let
];
};
otherNixes.nix_2_3.setNixPackage =
{ lib, pkgs, ... }:
{
imports = [ checkOverrideNixVersion ];
nix.package = lib.mkForce (
pkgs.nixVersions.nix_2_3.overrideAttrs (o: {
meta = o.meta // {
# This version shouldn't be used by end-users, but we run tests against
# it to ensure we don't break protocol compatibility.
knownVulnerabilities = [ ];
};
})
);
};
otherNixes.nix_2_13.setNixPackage =
{ lib, pkgs, ... }:
{
@@ -91,14 +106,6 @@ let
);
};
otherNixes.nix_2_18.setNixPackage =
{ lib, pkgs, ... }:
{
imports = [ checkOverrideNixVersion ];
nix.package = lib.mkForce (
nixpkgs-23-11.legacyPackages.${pkgs.stdenv.hostPlatform.system}.nixVersions.nix_2_18
);
};
in
{
+2 -2
View File
@@ -40,7 +40,7 @@ in
for fs in ("ext4", "btrfs", "xfs"):
machine.succeed("mkfs.{} {} /dev/vdb".format(fs, "-F" if fs == "ext4" else "-f"))
machine.succeed("mkdir -p /mnt")
machine.succeed("mount -t {} /dev/vdb /mnt".format(fs))
machine.succeed("mount /dev/vdb /mnt")
machine.succeed("sync")
machine.succeed("nix copy --offline ${pkg1} --to /mnt")
machine.crash()
@@ -48,7 +48,7 @@ in
machine.start()
machine.wait_for_unit("multi-user.target")
machine.succeed("mkdir -p /mnt")
machine.succeed("mount -t {} /dev/vdb /mnt".format(fs))
machine.succeed("mount /dev/vdb /mnt")
machine.succeed("nix path-info --offline --store /mnt ${pkg1}")
machine.succeed("nix store verify --all --store /mnt --no-trust")
+70 -35
View File
@@ -227,40 +227,7 @@ in
"Credential provider caching failed"
)
@setup_s3()
def test_aws_log_integration(bucket):
"""Test that AWS SDK logs are properly routed through Nix logger"""
print("\n=== Testing AWS Log Integration ===")
store_url = make_s3_url(bucket)
# With default verbosity, AWS noise should NOT appear
# All AWS messages are demoted to lvlDebug or lvlVomit
output_default = server.succeed(
f"{ENV_WITH_CREDS} nix copy --to '{store_url}' {PKGS['A']} 2>&1"
)
if "(aws:" in output_default:
print("Output at default verbosity:")
print(output_default)
raise Exception("Found AWS noise at default verbosity")
# With --debug (lvlDebug), we should see AWS messages with (aws:subject) prefix
output_debug = server.succeed(
f"{ENV_WITH_CREDS} nix copy --debug --to '{store_url}' {PKGS['B']} 2>&1"
)
# Check for the (aws:subject) prefix format
if "(aws:" not in output_debug:
print("Output at --debug verbosity:")
print(output_debug)
raise Exception("Expected to see (aws:subject) prefix in debug output")
# Should also see Nix's own credential provider creation message
if "creating new AWS credential provider" not in output_debug:
print("Debug output:")
print(output_debug)
raise Exception("Expected to see credential provider creation at debug level")
print("✓ Credential provider created once and cached")
@setup_s3(populate_bucket=[PKGS['A']])
def test_fetchurl_basic(bucket):
@@ -276,6 +243,8 @@ in
f"'builtins.fetchurl {{ name = \"foo\"; url = \"{cache_info_url}\"; }}'"
)
print("✓ builtins.fetchurl works with s3:// URLs")
@setup_s3()
def test_error_message_formatting(bucket):
"""Verify error messages display URLs correctly"""
@@ -294,6 +263,8 @@ in
print(error_msg)
raise Exception("Error message formatting failed - should show actual URL, not %s placeholder")
print("✓ Error messages format URLs correctly")
@setup_s3(populate_bucket=[PKGS['A']])
def test_fork_credential_preresolution(bucket):
"""Test credential pre-resolution in forked processes"""
@@ -333,6 +304,8 @@ in
print(output)
raise Exception("Expected to find FileTransfer creation in forked process")
print(" ✓ Forked process creates fresh FileTransfer")
# Verify pre-resolution in parent
required_messages = [
"Pre-resolving AWS credentials for S3 URL in builtin:fetchurl",
@@ -345,6 +318,8 @@ in
print(output)
raise Exception(f"Missing expected message: {msg}")
print(" ✓ Parent pre-resolves credentials")
# Verify child uses pre-resolved credentials
if "Using pre-resolved AWS credentials from parent process" not in output:
print("Debug output:")
@@ -368,6 +343,8 @@ in
print(output)
raise Exception(f"Child process (pid={child_pid}) should NOT create new credential providers")
print(" ✓ Child uses pre-resolved credentials (no new providers)")
@setup_s3(populate_bucket=[PKGS['A'], PKGS['B'], PKGS['C']])
def test_store_operations(bucket):
"""Test nix store info and copy operations"""
@@ -385,6 +362,8 @@ in
if not store_info.get("url"):
raise Exception("Store should have a URL")
print(f" ✓ Store URL: {store_info['url']}")
# Test copy from store
verify_packages_in_store(client, PKGS['A'], should_exist=False)
@@ -402,6 +381,9 @@ in
verify_packages_in_store(client, [PKGS['A'], PKGS['B'], PKGS['C']])
print(" ✓ nix copy works")
print(" ✓ Credentials cached on client")
@setup_s3(populate_bucket=[PKGS['A'], PKGS['B']], public=True)
def test_public_bucket_operations(bucket):
"""Test store operations on public bucket without credentials"""
@@ -411,6 +393,7 @@ in
# Verify store info works without credentials
client.succeed(f"nix store info --store '{store_url}' >&2")
print(" ✓ nix store info works without credentials")
# Get and validate store info JSON
info_json = client.succeed(f"nix store info --json --store '{store_url}'")
@@ -419,6 +402,8 @@ in
if not store_info.get("url"):
raise Exception("Store should have a URL")
print(f" ✓ Store URL: {store_info['url']}")
# Verify packages are not yet in client store
verify_packages_in_store(client, [PKGS['A'], PKGS['B']], should_exist=False)
@@ -431,6 +416,8 @@ in
# Verify packages were copied successfully
verify_packages_in_store(client, [PKGS['A'], PKGS['B']])
print(" ✓ nix copy from public bucket works without credentials")
@setup_s3(populate_bucket=[PKGS['A']])
def test_url_format_variations(bucket):
"""Test different S3 URL parameter combinations"""
@@ -439,10 +426,12 @@ in
# Test parameter order variation (region before endpoint)
url1 = f"s3://{bucket}?region={REGION}&endpoint={ENDPOINT}"
client.succeed(f"{ENV_WITH_CREDS} nix store info --store '{url1}' >&2")
print(" ✓ Parameter order: region before endpoint works")
# Test parameter order variation (endpoint before region)
url2 = f"s3://{bucket}?endpoint={ENDPOINT}&region={REGION}"
client.succeed(f"{ENV_WITH_CREDS} nix store info --store '{url2}' >&2")
print(" ✓ Parameter order: endpoint before region works")
@setup_s3(populate_bucket=[PKGS['A']])
def test_concurrent_fetches(bucket):
@@ -496,6 +485,9 @@ in
providers_created = output.count("creating new AWS credential provider")
transfers_created = output.count("builtin:fetchurl creating fresh FileTransfer instance")
print(f" ✓ {providers_created} credential providers created")
print(f" ✓ {transfers_created} FileTransfer instances created")
if transfers_created != 5:
print("Debug output:")
print(output)
@@ -521,10 +513,14 @@ in
pkg_hash = get_package_hash(PKGS['B'])
verify_content_encoding(server, bucket, f"{pkg_hash}.narinfo", "gzip")
print(" ✓ .narinfo has Content-Encoding: gzip")
# Verify client can download and decompress
client.succeed(f"{ENV_WITH_CREDS} nix copy --from '{store_url}' --no-check-sigs {PKGS['B']}")
verify_packages_in_store(client, PKGS['B'])
print(" ✓ Client decompressed .narinfo successfully")
@setup_s3()
def test_compression_mixed(bucket):
"""Test mixed compression (narinfo=xz, ls=gzip)"""
@@ -541,14 +537,18 @@ in
# Verify .narinfo has xz compression
verify_content_encoding(server, bucket, f"{pkg_hash}.narinfo", "xz")
print(" ✓ .narinfo has Content-Encoding: xz")
# Verify .ls has gzip compression
verify_content_encoding(server, bucket, f"{pkg_hash}.ls", "gzip")
print(" ✓ .ls has Content-Encoding: gzip")
# Verify client can download with mixed compression
client.succeed(f"{ENV_WITH_CREDS} nix copy --from '{store_url}' --no-check-sigs {PKGS['C']}")
verify_packages_in_store(client, PKGS['C'])
print(" ✓ Client downloaded package with mixed compression")
@setup_s3()
def test_compression_disabled(bucket):
"""Verify no compression by default"""
@@ -560,6 +560,8 @@ in
pkg_hash = get_package_hash(PKGS['A'])
verify_no_compression(server, bucket, f"{pkg_hash}.narinfo")
print(" ✓ No compression applied by default")
@setup_s3()
def test_nix_prefetch_url(bucket):
"""Test that nix-prefetch-url retrieves actual file content from S3, not empty files (issue #8862)"""
@@ -579,6 +581,8 @@ in
"nix hash file --type sha256 --base32 /tmp/test-file.txt"
).strip()
print(f" ✓ Uploaded test file to S3 ({test_file_size} bytes)")
# Use nix-prefetch-url to download from S3
s3_url = make_s3_url(bucket, path="/test-file.txt")
@@ -598,6 +602,8 @@ in
f"Hash mismatch: expected {expected_hash}, got {prefetch_hash}"
)
print(" ✓ nix-prefetch-url completed with correct hash")
# Verify the downloaded file is NOT empty (the bug in #8862)
file_size = int(client.succeed(f"stat -c %s {store_path}").strip())
@@ -609,12 +615,16 @@ in
f"File size mismatch: expected {test_file_size}, got {file_size}"
)
print(f" ✓ File has correct size ({file_size} bytes, not empty)")
# Verify actual content matches by comparing hashes instead of printing entire file
downloaded_hash = client.succeed(f"nix hash file --type sha256 --base32 {store_path}").strip()
if downloaded_hash != expected_hash:
raise Exception(f"Content hash mismatch: expected {expected_hash}, got {downloaded_hash}")
print(" ✓ File content verified correct (hash matches)")
@setup_s3(populate_bucket=[PKGS['A']], versioned=True)
def test_versioned_urls(bucket):
"""Test that versionId parameter is accepted in S3 URLs"""
@@ -628,6 +638,7 @@ in
f"{ENV_WITH_CREDS} nix eval --impure --expr "
f"'builtins.fetchurl {{ name = \"cache-info\"; url = \"{cache_info_url}\"; }}'"
)
print(" ✓ Fetch without versionId works")
# List versions to get a version ID
# MinIO output format: [timestamp] size tier versionId versionNumber method filename
@@ -641,6 +652,8 @@ in
raise Exception("Could not extract version ID from MinIO output")
version_id = version_match.group(1)
print(f" ✓ Found version ID: {version_id}")
# Version ID should not be "null" since versioning was enabled before upload
if version_id == "null":
raise Exception("Version ID is 'null' - versioning may not be working correctly")
@@ -651,6 +664,7 @@ in
f"{ENV_WITH_CREDS} nix eval --impure --expr "
f"'builtins.fetchurl {{ name = \"cache-info-versioned\"; url = \"{versioned_url}\"; }}'"
)
print(" ✓ Fetch with versionId parameter works")
@setup_s3()
def test_multipart_upload_basic(bucket):
@@ -686,9 +700,13 @@ in
print(output)
raise Exception(f"Expected '{expected_msg}' in output")
print(f" ✓ Multipart upload used with {expected_parts} parts")
client.succeed(f"{ENV_WITH_CREDS} nix copy --from '{store_url}' {large_pkg} --no-check-sigs")
verify_packages_in_store(client, large_pkg, should_exist=True)
print(" ✓ Large file downloaded and verified")
@setup_s3()
def test_multipart_threshold(bucket):
"""Test that files below threshold use regular upload"""
@@ -711,9 +729,13 @@ in
if "using S3 regular upload" not in output:
raise Exception("Expected regular upload to be used")
print(" ✓ Regular upload used for file below threshold")
client.succeed(f"{ENV_WITH_CREDS} nix copy --no-check-sigs --from '{store_url}' {PKGS['A']}")
verify_packages_in_store(client, PKGS['A'], should_exist=True)
print(" ✓ Small file uploaded and verified")
@setup_s3()
def test_multipart_with_log_compression(bucket):
"""Test multipart upload with compressed build logs"""
@@ -765,6 +787,8 @@ in
print(output)
raise Exception("Expected multipart completion message")
print(" ✓ Compressed log uploaded with multipart")
@setup_s3(
populate_bucket=[PKGS['A']],
profiles={
@@ -780,11 +804,13 @@ in
# Verify store info works with profile credentials (no env vars)
client.succeed(f"HOME=/root nix store info --store '{store_url}' >&2")
print(" ✓ nix store info works with profile credentials")
# Verify we can copy from the store using profile
verify_packages_in_store(client, PKGS['A'], should_exist=False)
client.succeed(f"HOME=/root nix copy --no-check-sigs --from '{store_url}' {PKGS['A']}")
verify_packages_in_store(client, PKGS['A'])
print(" ✓ nix copy works with profile credentials")
# Clean up the package we just copied so we can test invalid profile
client.succeed(f"nix store delete --ignore-liveness {PKGS['A']}")
@@ -793,6 +819,7 @@ in
# Verify invalid profile fails when trying to copy
invalid_url = make_s3_url(bucket, profile="invalid")
client.fail(f"HOME=/root nix copy --no-check-sigs --from '{invalid_url}' {PKGS['A']} 2>&1")
print(" ✓ Invalid profile credentials correctly rejected")
@setup_s3(
populate_bucket=[PKGS['A']],
@@ -814,12 +841,14 @@ in
output = client.succeed(
f"HOME=/root {ENV_WITH_CREDS} nix copy --no-check-sigs --debug --from '{store_url}' {PKGS['A']} 2>&1"
)
print(" ✓ nix copy succeeded with env vars overriding wrong profile")
# Verify the credential chain shows Environment provider was added
if "Added AWS Environment Credential Provider" not in output:
print("Debug output:")
print(output)
raise Exception("Expected Environment provider to be added to chain")
print(" ✓ Environment provider added to credential chain")
# Clean up the package so we can test again without env vars
client.succeed(f"nix store delete --ignore-liveness {PKGS['A']}")
@@ -827,6 +856,7 @@ in
# Without env vars, same URL should fail (proving profile creds are actually wrong)
client.fail(f"HOME=/root nix copy --no-check-sigs --from '{store_url}' {PKGS['A']} 2>&1")
print(" ✓ Without env vars, wrong profile credentials correctly fail")
@setup_s3(
populate_bucket=[PKGS['A']],
@@ -852,12 +882,14 @@ in
print("Debug output:")
print(output)
raise Exception(f"Expected to find: {msg}")
print(f" ✓ {provider} provider added to chain")
# SSO should be skipped (no SSO config for this profile)
if "Skipped AWS SSO Credential Provider for profile 'testprofile'" not in output:
print("Debug output:")
print(output)
raise Exception("Expected SSO provider to be skipped")
print(" ✓ SSO provider correctly skipped (not configured)")
# ============================================================================
# Main Test Execution
@@ -877,7 +909,6 @@ in
# Run tests (each gets isolated bucket via decorator)
test_credential_caching()
test_aws_log_integration()
test_fetchurl_basic()
test_error_message_formatting()
test_fork_credential_preresolution()
@@ -896,5 +927,9 @@ in
test_profile_credentials()
test_env_vars_precedence()
test_credential_provider_chain()
print("\n" + "="*80)
print("✓ All S3 Binary Cache Store Tests Passed!")
print("="*80)
'';
}