Compare commits

..
Author SHA1 Message Date
John Ericson 97d29276b9 WIP 2026-01-28 18:06:36 -05:00
John Ericson 2545b6716a EvalState::coerceToSingleDerivedPath also alow taking "packages"
This relaxes `builtins.outputOf` and also allows it to be used for
`builtins.derivationOf`
2026-01-28 17:47:13 -05:00
296 changed files with 4704 additions and 8895 deletions

No files matched your search

+2 -13
View File
@@ -39,24 +39,13 @@ jobs:
role-to-assume: "arn:aws:iam::080433136561:role/nix-release"
role-session-name: nix-release-oidc-${{ github.run_id }}
aws-region: eu-west-1
- name: Disable containerd image store
run: |
# Docker 28+ defaults to the containerd image store, which
# pushes layers uncompressed instead of gzip. OCI clients
# that only support gzip (e.g. go-containerregistry) fail
# with "gzip: invalid header". Disabling the containerd
# snapshotter restores the classic storage driver, which
# preserves gzip-compressed layers through the
# `docker load` / `docker push` pipeline.
echo '{"features":{"containerd-snapshotter":false}}' | sudo tee /etc/docker/daemon.json > /dev/null
sudo systemctl restart docker
- name: Login to Docker Hub
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Login to GitHub Container Registry
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
with:
registry: ghcr.io
username: ${{ github.actor }}
-29
View File
@@ -1,29 +0,0 @@
---
synopsis: "Rust nix-installer in beta"
prs: []
---
The Rust-based rewrite of the Nix installer is now in beta.
We'd love help testing it out!
To test out the new installer, run:
```
curl -sSfL https://artifacts.nixos.org/nix-installer | sh -s -- install
```
This installer can be run even when you have an existing, script-based Nix installation without any adjustments.
This new installer also comes with the ability to uninstall your Nix installation; run:
```
/nix/nix-installer uninstall
```
This will get rid of your entire Nix installation (even if you installed over an existing, script-based installation).
This installer is a modified version of the [Determinate Nix Installer](https://github.com/DeterminateSystems/nix-installer) by Determinate Systems.
Thanks to Determinate Systems for all the investment they've put into the installer.
Source for the installer is in https://github.com/NixOS/nix-installer.
Report any issues in that repo.
For CI usage, a GitHub Action to install Nix using this installer is available at https://github.com/NixOS/nix-installer-action.
@@ -6,4 +6,3 @@ prs: [14766]
The C API now includes additional methods:
- `nix_store_query_path_from_hash_part()` - Get the full store path given its hash part
- `nix_store_copy_path()` - Copy a single store path between two stores, allows repairs and configuring signature checking
-11
View File
@@ -1,11 +0,0 @@
---
synopsis: New command `nix store roots-daemon` for serving GC roots
prs: [15143]
---
New command [`nix store roots-daemon`](@docroot@/command-ref/new-cli/nix3-store-roots-daemon.md) runs a daemon that serves garbage collector roots over a Unix domain socket.
It enables the garbage collector to discover runtime roots when the main Nix daemon doesn't have `CAP_SYS_PTRACE` capability and therefore cannot scan `/proc`.
The garbage collector can be configured to use this daemon via the [`use-roots-daemon`](@docroot@/store/types/local-store.md#store-experimental-option-use-roots-daemon) store setting.
This feature requires the [`local-overlay-store` experimental feature](@docroot@/development/experimental-features.md#xp-feature-local-overlay-store).
@@ -1,32 +0,0 @@
---
synopsis: S3 binary caches now use virtual-hosted-style addressing by default
issues: [15208]
---
S3 binary caches now use virtual-hosted-style URLs
(`https://bucket.s3.region.amazonaws.com/key`) instead of path-style URLs
(`https://s3.region.amazonaws.com/bucket/key`) when connecting to standard AWS
S3 endpoints. This enables HTTP/2 multiplexing and fixes TCP connection
exhaustion (TIME_WAIT socket accumulation) under high-concurrency workloads.
A new `addressing-style` store option controls this behavior:
- `auto` (default): virtual-hosted-style for standard AWS endpoints, path-style
for custom endpoints.
- `path`: forces path-style addressing (deprecated by AWS).
- `virtual`: forces virtual-hosted-style addressing (bucket names must not
contain dots).
Bucket names containing dots (e.g., `my.bucket.name`) automatically fall back
to path-style addressing in `auto` mode, because dotted names create
multi-level subdomains that break TLS wildcard certificate validation.
Example using path-style for backwards compatibility:
```
s3://my-bucket/key?region=us-east-1&addressing-style=path
```
Additionally, TCP keep-alive is now enabled on all HTTP connections, preventing
idle connections from being silently dropped by intermediate network devices
(NATs, firewalls, load balancers).
Generated
+4 -4
View File
@@ -63,11 +63,11 @@
},
"nixpkgs": {
"locked": {
"lastModified": 1771043024,
"narHash": "sha256-WoiezqWJQ3OHILah+p6rzNXdJceEAmAhyDFZFZ6pZzY=",
"rev": "3aadb7ca9eac2891d52a9dec199d9580a6e2bf44",
"lastModified": 1769089682,
"narHash": "sha256-Xu+7iYcAuOvsI2wdkUcIEmkqEJbvvE6n7qR9QNjJyP4=",
"rev": "078d69f03934859a181e81ba987c2bb033eebfc5",
"type": "tarball",
"url": "https://releases.nixos.org/nixos/25.11/nixos-25.11.5960.3aadb7ca9eac/nixexprs.tar.xz"
"url": "https://releases.nixos.org/nixos/25.11/nixos-25.11.4506.078d69f03934/nixexprs.tar.xz"
},
"original": {
"type": "tarball",
-5
View File
@@ -115,9 +115,6 @@
}
// lib.optionalAttrs (crossSystem == "x86_64-unknown-freebsd13") {
useLLVM = true;
}
// lib.optionalAttrs (crossSystem == "x86_64-w64-mingw32") {
emulator = pkgs: "${pkgs.buildPackages.wineWow64Packages.stable_11}/bin/wine";
};
overlays = [
(overlayFor (pkgs: pkgs.${stdenv}))
@@ -409,8 +406,6 @@
"nix-cmd" = { };
"nix-nswrapper" = { };
"nix-cli" = { };
"nix-everything" = { };
-4
View File
@@ -24,10 +24,6 @@ subproject('libcmd')
# Executables
subproject('nix')
if host_machine.system() == 'linux'
subproject('nswrapper')
endif
# Docs
if get_option('doc-gen')
subproject('internal-api-docs')
@@ -22,8 +22,6 @@ add_project_arguments(
'-Werror=undef',
'-Werror=unused-result',
'-Werror=sign-compare',
'-Werror=return-type',
'-Werror=non-virtual-dtor',
'-Wignored-qualifiers',
'-Wimplicit-fallthrough',
'-Wno-deprecated-declarations',
@@ -33,13 +31,6 @@ add_project_arguments(
# GCC doesn't benefit much from precompiled headers.
do_pch = cxx.get_id() == 'clang'
if cxx.get_id() == 'gcc'
add_project_arguments(
'-Wno-interference-size', # Used for C++ ABI only. We don't provide any guarantees about different march tunings.
language : 'cpp',
)
endif
# This is a clang-only option for improving build times.
# It forces the instantiation of templates in the PCH itself and
# not every translation unit it's included in.
@@ -49,11 +40,6 @@ endif
# instantiations in libutil and libstore.
if cxx.get_id() == 'clang'
add_project_arguments('-fpch-instantiate-templates', language : 'cpp')
# Catch brace elision bugs: when WorkerProto::Version changed from `unsigned int`
# to `struct { unsigned int major; uint8_t minor; }`, `.version = 16` silently
# became `.version = {16, 0}` instead of failing, breaking protocol compatibility
# in a subtle way
add_project_arguments('-Werror=c99-designator', language : 'cpp')
endif
# Detect if we're using libstdc++ (GCC's standard library)
-3
View File
@@ -4,7 +4,6 @@
buildPackages,
cacert,
nix,
nixComponents2,
}:
let
@@ -12,7 +11,6 @@ let
installerClosureInfo = buildPackages.closureInfo {
rootPaths = [
nix
nixComponents2.nix-manual.man
cacert
];
};
@@ -44,7 +42,6 @@ runCommand "nix-binary-tarball-${version}" env ''
--subst-var-by cacert ${cacert}
substitute ${../scripts/install-multi-user.sh} $TMPDIR/install-multi-user \
--subst-var-by nix ${nix} \
--subst-var-by nix-manual ${nixComponents2.nix-manual.man} \
--subst-var-by cacert ${cacert}
if type -p shellcheck; then
-2
View File
@@ -418,8 +418,6 @@ in
nix-cmd = callPackage ../src/libcmd/package.nix { };
nix-nswrapper = callPackage ../src/nswrapper/package.nix { };
/**
The Nix command line interface. Note that this does not include its tests, whereas `nix-everything` does.
*/
-5
View File
@@ -31,8 +31,6 @@
nix-cmd,
nix-nswrapper,
nix-cli,
nix-functional-tests,
@@ -173,9 +171,6 @@ stdenv.mkDerivation (finalAttrs: {
# Forwarded outputs
ln -sT ${nix-manual} $doc
ln -sT ${nix-manual.man} $man
''
+ lib.optionalString stdenv.isLinux ''
lndir ${nix-nswrapper} $out
'';
passthru = {
-1
View File
@@ -57,7 +57,6 @@ let
"nix-flake"
"nix-flake-c"
"nix-flake-tests"
"nix-nswrapper"
"nix-main"
"nix-main-c"
"nix-cmd"
-3
View File
@@ -52,7 +52,6 @@ readonly PROFILE_FISH_PREFIXES=(
readonly PROFILE_NIX_FILE_FISH="$NIX_ROOT/var/nix/profiles/default/etc/profile.d/nix-daemon.fish"
readonly NIX_INSTALLED_NIX="@nix@"
readonly NIX_INSTALLED_NIX_MAN="@nix-manual@"
readonly NIX_INSTALLED_CACERT="@cacert@"
#readonly NIX_INSTALLED_NIX="/nix/store/byi37zv50wnfrpp4d81z3spswd5zva37-nix-2.3.6"
#readonly NIX_INSTALLED_CACERT="/nix/store/7pi45g541xa8ahwgpbpy7ggsl0xj1jj6-nss-cacert-3.49.2"
@@ -970,8 +969,6 @@ setup_default_profile() {
task "Setting up the default profile"
_sudo "to install a bootstrapping Nix in to the default profile" \
HOME="$ROOT_HOME" "$NIX_INSTALLED_NIX/bin/nix-env" -i "$NIX_INSTALLED_NIX"
_sudo "to install Nix man pages in to the default profile" \
HOME="$ROOT_HOME" "$NIX_INSTALLED_NIX/bin/nix-env" -i "$NIX_INSTALLED_NIX_MAN"
if [ -z "${NIX_SSL_CERT_FILE:-}" ] || ! [ -f "${NIX_SSL_CERT_FILE:-}" ] || cert_in_store; then
_sudo "to install a bootstrapping SSL certificate just for Nix in to the default profile" \
+15 -43
View File
@@ -4,7 +4,6 @@
#include "nix/cmd/command.hh"
#include "nix/cmd/legacy.hh"
#include "nix/cmd/markdown.hh"
#include "nix/store/globals.hh"
#include "nix/store/store-open.hh"
#include "nix/store/local-fs-store.hh"
#include "nix/store/derivations.hh"
@@ -64,25 +63,6 @@ void NixMultiCommand::run()
command->second->run();
}
StoreConfigCommand::StoreConfigCommand() {}
ref<StoreConfig> StoreConfigCommand::getStoreConfig()
{
if (!_storeConfig)
_storeConfig = createStoreConfig();
return ref<StoreConfig>(_storeConfig);
}
ref<StoreConfig> StoreConfigCommand::createStoreConfig()
{
return resolveStoreConfig(StoreReference{settings.storeUri.get()});
}
void StoreConfigCommand::run()
{
run(getStoreConfig());
}
StoreCommand::StoreCommand() {}
ref<Store> StoreCommand::getStore()
@@ -94,20 +74,12 @@ ref<Store> StoreCommand::getStore()
ref<Store> StoreCommand::createStore()
{
auto store = getStoreConfig()->openStore();
store->init();
return store;
return openStore();
}
void StoreCommand::run(ref<StoreConfig> storeConfig)
void StoreCommand::run()
{
// We can either efficiently implement getStore/createStore with memoization,
// or use the StoreConfig passed in run.
// It's more efficient to memoize, especially since there are some direct users
// of getStore. The StoreConfig in both cases should be the same, though.
auto store = getStore();
assert(&*storeConfig == &store->config);
run(std::move(store));
run(getStore());
}
CopyCommand::CopyCommand()
@@ -116,28 +88,28 @@ CopyCommand::CopyCommand()
.longName = "from",
.description = "URL of the source Nix store.",
.labels = {"store-uri"},
.handler = {[this](std::string s) { srcUri = StoreReference::parse(s); }},
.handler = {&srcUri},
});
addFlag({
.longName = "to",
.description = "URL of the destination Nix store.",
.labels = {"store-uri"},
.handler = {[this](std::string s) { dstUri = StoreReference::parse(s); }},
.handler = {&dstUri},
});
}
ref<StoreConfig> CopyCommand::createStoreConfig()
ref<Store> CopyCommand::createStore()
{
return !srcUri ? StoreCommand::createStoreConfig() : resolveStoreConfig(StoreReference{*srcUri});
return srcUri.empty() ? StoreCommand::createStore() : openStore(srcUri);
}
ref<Store> CopyCommand::getDstStore()
{
if (!srcUri && !dstUri)
if (srcUri.empty() && dstUri.empty())
throw UsageError("you must pass '--from' and/or '--to'");
return !dstUri ? openStore() : openStore(StoreReference{*dstUri});
return dstUri.empty() ? openStore() : openStore(dstUri);
}
EvalCommand::EvalCommand()
@@ -159,7 +131,7 @@ EvalCommand::~EvalCommand()
ref<Store> EvalCommand::getEvalStore()
{
if (!evalStore)
evalStore = evalStoreUrl ? openStore(StoreReference{*evalStoreUrl}) : getStore();
evalStore = evalStoreUrl ? openStore(*evalStoreUrl) : getStore();
return ref<Store>(evalStore);
}
@@ -285,18 +257,18 @@ MixProfile::MixProfile()
});
}
void MixProfile::updateProfile(Store & store_, const StorePath & storePath)
void MixProfile::updateProfile(const StorePath & storePath)
{
if (!profile)
return;
auto * store = dynamic_cast<LocalFSStore *>(&store_);
auto store = getDstStore().dynamic_pointer_cast<LocalFSStore>();
if (!store)
throw Error("'--profile' is not supported for this Nix store");
auto profile2 = absPath(*profile);
switchLink(profile2, createGeneration(*store, profile2, storePath));
}
void MixProfile::updateProfile(Store & store, const BuiltPaths & buildables)
void MixProfile::updateProfile(const BuiltPaths & buildables)
{
if (!profile)
return;
@@ -320,12 +292,12 @@ void MixProfile::updateProfile(Store & store, const BuiltPaths & buildables)
throw UsageError(
"'--profile' requires that the arguments produce a single store path, but there are %d", result.size());
updateProfile(store, result[0]);
updateProfile(result[0]);
}
MixDefaultProfile::MixDefaultProfile()
{
profile = getDefaultProfile(settings.getProfileDirsOptions()).string();
profile = getDefaultProfile().string();
}
static constexpr auto environmentVariablesCategory = "Options that change environment variables";
+1 -1
View File
@@ -148,7 +148,7 @@ MixEvalArgs::MixEvalArgs()
)",
.category = category,
.labels = {"store-url"},
.handler = {[this](std::string s) { evalStoreUrl = StoreReference::parse(s); }},
.handler = {&evalStoreUrl},
});
}
-31
View File
@@ -1,31 +0,0 @@
#include "nix/cmd/get-build-log.hh"
#include "nix/store/log-store.hh"
#include "nix/store/store-open.hh"
namespace nix {
std::string fetchBuildLog(ref<Store> store, const StorePath & path, std::string_view what)
{
auto subs = getDefaultSubstituters();
subs.push_front(store);
for (auto & sub : subs) {
auto * logSubP = dynamic_cast<LogStore *>(&*sub);
if (!logSubP) {
printInfo("Skipped '%s' which does not support retrieving build logs", sub->config.getHumanReadableURI());
continue;
}
auto & logSub = *logSubP;
auto log = logSub.getBuildLog(path);
if (!log)
continue;
printInfo("got build log for '%s' from '%s'", what, logSub.config.getHumanReadableURI());
return *log;
}
throw Error("build log of '%s' is not available", what);
}
} // namespace nix
+16 -32
View File
@@ -5,7 +5,6 @@
#include "nix/util/args.hh"
#include "nix/cmd/common-eval-args.hh"
#include "nix/store/path.hh"
#include "nix/store/store-reference.hh"
#include "nix/flake/lockfile.hh"
#include <optional>
@@ -41,43 +40,28 @@ struct NixMultiCommand : MultiCommand, virtual Command
// For the overloaded run methods
#pragma GCC diagnostic ignored "-Woverloaded-virtual"
/**
* A command that requires a \ref StoreConfig store configuration.
*/
struct StoreConfigCommand : virtual Command
{
StoreConfigCommand();
void run() override;
/**
* Return the default Nix store configuration.
*/
ref<StoreConfig> getStoreConfig();
virtual ref<StoreConfig> createStoreConfig();
/**
* Main entry point, with a `StoreConfig` provided
*/
virtual void run(ref<StoreConfig>) = 0;
private:
std::shared_ptr<StoreConfig> _storeConfig;
};
/**
* A command that requires a \ref Store "Nix store".
*/
struct StoreCommand : virtual StoreConfigCommand
struct StoreCommand : virtual Command
{
StoreCommand();
void run(ref<StoreConfig>) override;
void run() override;
/**
* Return the default Nix store.
*/
ref<Store> getStore();
ref<Store> createStore();
/**
* Return the destination Nix store.
*/
virtual ref<Store> getDstStore()
{
return getStore();
}
virtual ref<Store> createStore();
/**
* Main entry point, with a `Store` provided
*/
@@ -93,13 +77,13 @@ private:
*/
struct CopyCommand : virtual StoreCommand
{
std::optional<StoreReference> srcUri, dstUri;
std::string srcUri, dstUri;
CopyCommand();
ref<StoreConfig> createStoreConfig() override;
ref<Store> createStore() override;
ref<Store> getDstStore();
ref<Store> getDstStore() override;
};
/**
@@ -331,11 +315,11 @@ struct MixProfile : virtual StoreCommand
MixProfile();
/* If 'profile' is set, make it point at 'storePath'. */
void updateProfile(Store & store, const StorePath & storePath);
void updateProfile(const StorePath & storePath);
/* If 'profile' is set, make it point at the store path produced
by 'buildables'. */
void updateProfile(Store & store, const BuiltPaths & buildables);
void updateProfile(const BuiltPaths & buildables);
};
struct MixDefaultProfile : MixProfile
@@ -6,7 +6,6 @@
#include "nix/main/common-args.hh"
#include "nix/expr/search-path.hh"
#include "nix/expr/eval-settings.hh"
#include "nix/store/store-reference.hh"
#include <filesystem>
@@ -56,7 +55,7 @@ struct MixEvalArgs : virtual Args, virtual MixRepair
LookupPath lookupPath;
std::optional<StoreReference> evalStoreUrl;
std::optional<std::string> evalStoreUrl;
private:
struct AutoArgExpr
@@ -1,23 +0,0 @@
#pragma once
///@file
#include "nix/store/store-api.hh"
#include <string>
#include <string_view>
namespace nix {
/**
* Fetch the build log for a store path, searching the store and its
* substituters.
*
* @param store The store to search (and its substituters).
* @param path The store path to get the build log for.
* @param what A description of what we're fetching the log for (used in messages).
* @return The build log content.
* @throws Error if the build log is not available.
*/
std::string fetchBuildLog(ref<Store> store, const StorePath & path, std::string_view what);
} // namespace nix
-2
View File
@@ -9,7 +9,6 @@ headers = files(
'common-eval-args.hh',
'compatibility-settings.hh',
'editor-for.hh',
'get-build-log.hh',
'installable-attr-path.hh',
'installable-derived-path.hh',
'installable-flake.hh',
@@ -21,5 +20,4 @@ headers = files(
'network-proxy.hh',
'repl-interacter.hh',
'repl.hh',
'unix-socket-server.hh',
)
@@ -14,7 +14,6 @@ namespace detail {
struct ReplCompleterMixin
{
virtual StringSet completePrefix(const std::string & prefix) = 0;
virtual ~ReplCompleterMixin() = default;
};
}; // namespace detail
+2 -1
View File
@@ -25,7 +25,7 @@ struct AbstractNixRepl
* @todo this is a layer violation
*
* @param programName Name of the command, e.g. `nix` or `nix-env`.
* @param args arguments to the command.
* @param args aguments to the command.
*/
using RunNix =
void(const std::string & programName, const Strings & args, const std::optional<std::string> & input);
@@ -37,6 +37,7 @@ struct AbstractNixRepl
*/
static std::unique_ptr<AbstractNixRepl> create(
const LookupPath & lookupPath,
nix::ref<Store> store,
ref<EvalState> state,
std::function<AnnotatedValues()> getValues,
RunNix * runNix = nullptr);
@@ -1,79 +0,0 @@
#pragma once
///@file
#include "nix/util/file-descriptor.hh"
#include <filesystem>
#include <functional>
#include <optional>
#include <sys/types.h>
namespace nix::unix {
/**
* Information about the identity of the peer on a Unix domain socket connection.
*/
struct PeerInfo
{
std::optional<pid_t> pid;
std::optional<uid_t> uid;
std::optional<gid_t> gid;
};
/**
* Get the identity of the caller, if possible.
*/
PeerInfo getPeerInfo(Descriptor remote);
/**
* Callback type for handling new connections.
*
* The callback receives ownership of the connection and is responsible
* for handling it (e.g., forking a child process, spawning a thread, etc.).
*
* @param socket The accepted connection file descriptor.
* @param closeListeners A callback to close the listening sockets.
* Useful in forked child processes to release the bound sockets.
*/
using UnixSocketHandler = std::function<void(AutoCloseFD socket, std::function<void()> closeListeners)>;
/**
* Options for the serve loop.
*
* Only used if no systemd socket activation is detected.
*/
struct ServeUnixSocketOptions
{
/**
* The Unix domain socket path to create and listen on.
*/
std::filesystem::path socketPath;
/**
* Mode for the created socket file.
*/
mode_t socketMode = 0666;
};
/**
* Run a server loop that accepts connections and calls the handler for each.
*
* This function handles:
* - systemd socket activation (via LISTEN_FDS environment variable)
* - Creating and binding a Unix domain socket if no activation is detected
* - Polling for incoming connections
* - Accepting connections
*
* For each accepted connection, the handler is called with the connection
* file descriptor. The handler takes ownership of the file descriptor and
* is responsible for closing it when done.
*
* This function never returns normally. It runs until interrupted
* (e.g., via SIGINT), at which point it throws `Interrupted`.
*
* @param options Configuration for the server.
* @param handler Callback invoked for each accepted connection.
*/
[[noreturn]] void serveUnixSocket(const ServeUnixSocketOptions & options, UnixSocketHandler handler);
} // namespace nix::unix
+2 -4
View File
@@ -203,10 +203,8 @@ FlakeRef InstallableFlake::nixpkgsFlakeRef() const
if (auto nixpkgsInput = lockedFlake->lockFile.findInput({"nixpkgs"})) {
if (auto lockedNode = std::dynamic_pointer_cast<const flake::LockedNode>(nixpkgsInput)) {
if (lockedNode->isFlake) {
debug("using nixpkgs flake '%s'", lockedNode->lockedRef);
return std::move(lockedNode->lockedRef);
}
debug("using nixpkgs flake '%s'", lockedNode->lockedRef);
return std::move(lockedNode->lockedRef);
}
}
+7 -3
View File
@@ -583,12 +583,16 @@ static void throwBuildErrors(std::vector<KeyedBuildResult> & buildResults, const
auto failedResult = failed.begin();
if (failedResult != failed.end()) {
if (failed.size() == 1) {
throw *failedResult->second;
failedResult->second->rethrow();
} else {
StringSet failedPaths;
for (; failedResult != failed.end(); failedResult++) {
if (!failedResult->second->message().empty()) {
logError(failedResult->second->info());
if (!failedResult->second->errorMsg.empty()) {
logError(
ErrorInfo{
.level = lvlError,
.msg = failedResult->second->errorMsg,
});
}
failedPaths.insert(failedResult->first->path.to_string(store));
}
-7
View File
@@ -74,7 +74,6 @@ sources = files(
'command.cc',
'common-eval-args.cc',
'editor-for.cc',
'get-build-log.cc',
'installable-attr-path.cc',
'installable-derived-path.cc',
'installable-flake.cc',
@@ -87,12 +86,6 @@ sources = files(
'repl.cc',
)
if host_machine.system() != 'windows'
sources += files(
'unix/unix-socket-server.cc',
)
endif
subdir('include/nix/cmd')
subdir('nix-meson-build-support/export-all-symbols')
+4 -9
View File
@@ -40,8 +40,8 @@ void sigintHandler(int signo)
static detail::ReplCompleterMixin * curRepl; // ugly
#if !USE_READLINE
static char * completionCallback(char * s, int * match) noexcept
try {
static char * completionCallback(char * s, int * match)
{
auto possible = curRepl->completePrefix(s);
if (possible.size() == 1) {
*match = 1;
@@ -73,12 +73,10 @@ try {
*match = 0;
return nullptr;
} catch (...) {
return nullptr;
}
static int listPossibleCallback(char * s, char *** avp) noexcept
try {
static int listPossibleCallback(char * s, char *** avp)
{
auto possible = curRepl->completePrefix(s);
if (possible.size() > (std::numeric_limits<int>::max() / sizeof(char *)))
@@ -107,9 +105,6 @@ try {
*avp = vp;
return ac;
} catch (...) {
*avp = nullptr;
return 0;
}
#endif
+41 -19
View File
@@ -12,8 +12,9 @@
#include "nix/expr/eval-settings.hh"
#include "nix/expr/attr-path.hh"
#include "nix/util/signals.hh"
#include "nix/store/store-open.hh"
#include "nix/store/log-store.hh"
#include "nix/cmd/common-eval-args.hh"
#include "nix/cmd/get-build-log.hh"
#include "nix/expr/get-drvs.hh"
#include "nix/store/derivations.hh"
#include "nix/store/globals.hh"
@@ -64,7 +65,7 @@ struct NixRepl : AbstractNixRepl, detail::ReplCompleterMixin, gc
const static int envSize = 32768;
std::shared_ptr<StaticEnv> staticEnv;
std::optional<Value> lastLoaded;
Value lastLoaded;
Env * env;
int displ;
StringSet varNames;
@@ -77,6 +78,7 @@ struct NixRepl : AbstractNixRepl, detail::ReplCompleterMixin, gc
NixRepl(
const LookupPath & lookupPath,
nix::ref<Store> store,
ref<EvalState> state,
std::function<AnnotatedValues()> getValues,
RunNix * runNix);
@@ -131,6 +133,7 @@ std::string removeWhitespace(std::string s)
NixRepl::NixRepl(
const LookupPath & lookupPath,
nix::ref<Store> store,
ref<EvalState> state,
std::function<NixRepl::AnnotatedValues()> getValues,
RunNix * runNix)
@@ -563,9 +566,31 @@ ProcessLineResult NixRepl::processLine(std::string line)
} else if (command == ":log") {
settings.readOnlyMode = true;
Finally roModeReset([&]() { settings.readOnlyMode = false; });
auto subs = getDefaultSubstituters();
subs.push_front(state->store);
bool foundLog = false;
RunPager pager;
auto log = fetchBuildLog(state->store, drvPath, drvPathRaw);
logger->writeToStdout(log);
for (auto & sub : subs) {
auto * logSubP = dynamic_cast<LogStore *>(&*sub);
if (!logSubP) {
printInfo(
"Skipped '%s' which does not support retrieving build logs", sub->config.getHumanReadableURI());
continue;
}
auto & logSub = *logSubP;
auto log = logSub.getBuildLog(drvPath);
if (log) {
printInfo("got build log for '%s' from '%s'", drvPathRaw, logSub.config.getHumanReadableURI());
logger->writeToStdout(*log);
foundLog = true;
break;
}
}
if (!foundLog)
throw Error("build log of '%s' is not available", drvPathRaw);
} else {
runNix("nix-shell", {drvPathRaw});
}
@@ -710,7 +735,7 @@ void NixRepl::loadFlake(const std::string & flakeRefS)
try {
cwd = std::filesystem::current_path();
} catch (std::filesystem::filesystem_error & e) {
throw SystemError(e.code(), "cannot determine current working directory");
throw SysError("cannot determine current working directory");
}
auto flakeRef = parseFlakeRef(fetchSettings, flakeRefS, cwd.string(), true);
@@ -748,19 +773,11 @@ void NixRepl::initEnv()
void NixRepl::showLastLoaded()
{
if (!lastLoaded)
throw Error("nothing has been loaded yet");
RunPager pager;
try {
for (auto & i : *lastLoaded->attrs()) {
std::string_view name = state->symbols[i.name];
logger->cout(name);
}
} catch (SystemError & e) {
/* Ignore broken pipes when the pager gets interrupted. */
if (!e.is(std::errc::broken_pipe))
throw;
for (auto & i : *lastLoaded.attrs()) {
std::string_view name = state->symbols[i.name];
logger->cout(name);
}
}
@@ -883,9 +900,13 @@ void NixRepl::runNix(const std::string & program, const Strings & args, const st
}
std::unique_ptr<AbstractNixRepl> AbstractNixRepl::create(
const LookupPath & lookupPath, ref<EvalState> state, std::function<AnnotatedValues()> getValues, RunNix * runNix)
const LookupPath & lookupPath,
nix::ref<Store> store,
ref<EvalState> state,
std::function<AnnotatedValues()> getValues,
RunNix * runNix)
{
return std::make_unique<NixRepl>(lookupPath, state, getValues, runNix);
return std::make_unique<NixRepl>(lookupPath, std::move(store), state, getValues, runNix);
}
ReplExitStatus AbstractNixRepl::runSimple(ref<EvalState> evalState, const ValMap & extraEnv)
@@ -898,6 +919,7 @@ ReplExitStatus AbstractNixRepl::runSimple(ref<EvalState> evalState, const ValMap
// NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDelete)
auto repl = std::make_unique<NixRepl>(
lookupPath,
openStore(),
evalState,
getValues,
/*runNix=*/nullptr);
-126
View File
@@ -1,126 +0,0 @@
///@file
#include "nix/cmd/unix-socket-server.hh"
#include "nix/util/environment-variables.hh"
#include "nix/util/file-system.hh"
#include "nix/util/logging.hh"
#include "nix/util/signals.hh"
#include "nix/util/strings.hh"
#include "nix/util/unix-domain-socket.hh"
#include "nix/util/util.hh"
#include <sys/socket.h>
#include <sys/un.h>
#include <poll.h>
#if defined(__APPLE__) || defined(__FreeBSD__)
# include <sys/ucred.h>
#endif
namespace nix::unix {
PeerInfo getPeerInfo(Descriptor remote)
{
PeerInfo peer;
#if defined(SO_PEERCRED)
# if defined(__OpenBSD__)
struct sockpeercred cred;
# else
ucred cred;
# endif
socklen_t credLen = sizeof(cred);
if (getsockopt(remote, SOL_SOCKET, SO_PEERCRED, &cred, &credLen) == 0) {
peer.pid = cred.pid;
peer.uid = cred.uid;
peer.gid = cred.gid;
}
#elif defined(LOCAL_PEERCRED)
# if !defined(SOL_LOCAL)
# define SOL_LOCAL 0
# endif
xucred cred;
socklen_t credLen = sizeof(cred);
if (getsockopt(remote, SOL_LOCAL, LOCAL_PEERCRED, &cred, &credLen) == 0)
peer.uid = cred.cr_uid;
#endif
return peer;
}
[[noreturn]] void serveUnixSocket(const ServeUnixSocketOptions & options, UnixSocketHandler handler)
{
std::vector<AutoCloseFD> listeningSockets;
static constexpr int SD_LISTEN_FDS_START = 3;
// Handle socket-based activation by systemd.
auto listenFds = getEnv("LISTEN_FDS");
if (listenFds) {
if (getEnv("LISTEN_PID") != std::to_string(getpid()))
throw Error("unexpected systemd environment variables");
auto count = string2Int<unsigned int>(*listenFds);
assert(count);
for (unsigned int i = 0; i < count; ++i) {
AutoCloseFD fdSocket(SD_LISTEN_FDS_START + i);
closeOnExec(fdSocket.get());
listeningSockets.push_back(std::move(fdSocket));
}
}
// Otherwise, create and bind to a Unix domain socket.
else {
createDirs(options.socketPath.parent_path());
listeningSockets.push_back(createUnixDomainSocket(options.socketPath.string(), options.socketMode));
}
std::vector<struct pollfd> fds;
for (auto & i : listeningSockets)
fds.push_back({.fd = i.get(), .events = POLLIN});
// Loop accepting connections.
while (1) {
try {
checkInterrupt();
auto count = poll(fds.data(), fds.size(), -1);
if (count == -1) {
if (errno == EINTR)
continue;
throw SysError("polling for incoming connections");
}
for (auto & fd : fds) {
if (!fd.revents)
continue;
// Accept a connection.
struct sockaddr_un remoteAddr;
socklen_t remoteAddrLen = sizeof(remoteAddr);
AutoCloseFD remote = accept(fd.fd, (struct sockaddr *) &remoteAddr, &remoteAddrLen);
checkInterrupt();
if (!remote) {
if (errno == EINTR)
continue;
throw SysError("accepting connection");
}
handler(std::move(remote), [&]() { listeningSockets.clear(); });
}
} catch (Error & error) {
auto ei = error.info();
// FIXME: add to trace?
ei.msg = HintFmt("while processing connection: %1%", ei.msg.str());
logError(ei);
}
}
}
} // namespace nix::unix
+2 -2
View File
@@ -43,7 +43,7 @@ RC_GTEST_FIXTURE_PROP(
auto * v = state.allocValue();
state.mkOutputString(*v, b, std::nullopt, mockXpSettings);
auto [d, _] = state.coerceToSingleDerivedPathUnchecked(noPos, *v, "", mockXpSettings);
auto d = state.coerceToSingleDerivedPath(noPos, *v, "", mockXpSettings, true);
RC_ASSERT(SingleDerivedPath{b} == d);
}
@@ -57,7 +57,7 @@ RC_GTEST_FIXTURE_PROP(
auto * v = state.allocValue();
state.mkOutputString(*v, b, outPath, mockXpSettings);
auto [d, _] = state.coerceToSingleDerivedPathUnchecked(noPos, *v, "", mockXpSettings);
auto d = state.coerceToSingleDerivedPath(noPos, *v, "", mockXpSettings, true);
RC_ASSERT(SingleDerivedPath{b} == d);
}
+1 -1
View File
@@ -756,7 +756,7 @@ TEST_F(PrimOpTest, langVersion)
TEST_F(PrimOpTest, storeDir)
{
auto v = eval("builtins.storeDir");
ASSERT_THAT(v, IsStringEq(state.store->storeDir));
ASSERT_THAT(v, IsStringEq(settings.nixStore));
}
TEST_F(PrimOpTest, nixVersion)
+2 -1
View File
@@ -13,7 +13,8 @@ TEST_F(ValueTest, unsetValue)
{
Value unsetValue;
ASSERT_EQ(false, unsetValue.isValid());
ASSERT_EQ(nThunk, unsetValue.type</*invalidIsThunk=*/true>());
ASSERT_EQ(nThunk, unsetValue.type(true));
ASSERT_DEATH(unsetValue.type(), "");
}
TEST_F(ValueTest, vInt)
+2 -8
View File
@@ -133,17 +133,11 @@ class SampleStack : public EvalProfiler
FrameInfo getPrimOpFrameInfo(const PrimOp & primOp, std::span<Value *> args, PosIdx pos);
public:
SampleStack(EvalState & state, const std::filesystem::path & profileFile, std::chrono::nanoseconds period)
SampleStack(EvalState & state, std::filesystem::path profileFile, std::chrono::nanoseconds period)
: state(state)
, sampleInterval(period)
, profileFd([&]() {
AutoCloseFD fd = openNewFileForWrite(
profileFile,
0660,
{
.truncateExisting = true,
.followSymlinksOnTruncate = true, /* FIXME: Probably shouldn't follow symlinks. */
});
AutoCloseFD fd = toDescriptor(open(profileFile.string().c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0660));
if (!fd)
throw SysError("opening file %s", PathFmt(profileFile));
return fd;
+2 -3
View File
@@ -71,9 +71,8 @@ Strings EvalSettings::getDefaultNixPath()
};
add(std::filesystem::path{getNixDefExpr()} / "channels");
auto profilesDirOpts = settings.getProfileDirsOptions();
add(rootChannelsDir(profilesDirOpts) / "nixpkgs", "nixpkgs");
add(rootChannelsDir(profilesDirOpts));
add(rootChannelsDir() / "nixpkgs", "nixpkgs");
add(rootChannelsDir());
return res;
}
+144 -59
View File
@@ -318,6 +318,7 @@ EvalState::EvalState(
countCalls = getEnv("NIX_COUNT_CALLS").value_or("0") != "0";
static_assert(sizeof(Env) <= 16, "environment must be <= 16 bytes");
static_assert(sizeof(Counter) == 64, "counters must be 64 bytes");
/* Construct the Nix expression search path. */
assert(lookupPath.elements.empty());
@@ -466,7 +467,7 @@ void EvalState::addConstant(const std::string & name, Value * v, Constant info)
We might know the type of a thunk in advance, so be allowed
to just write it down in that case. */
if (auto gotType = v->type</*invalidIsThunk=*/true>(); gotType != nThunk)
if (auto gotType = v->type(true); gotType != nThunk)
assert(info.type == gotType);
/* Install value the base environment. */
@@ -885,7 +886,7 @@ void Value::mkPath(const SourcePath & path, EvalMemory & mem)
mkPath(&*path.accessor, StringData::make(mem, path.path.abs()));
}
[[gnu::always_inline]] inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval)
inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval)
{
for (auto l = var.level; l; --l, env = env->up)
;
@@ -903,11 +904,11 @@ void Value::mkPath(const SourcePath & path, EvalMemory & mem)
while (1) {
forceAttrs(*env->values[0], fromWith->pos, "while evaluating the first subexpression of a with expression");
if (auto j = env->values[0]->attrs()->get(var.name)) {
if (countCalls) [[unlikely]]
if (countCalls)
attrSelects[j->pos]++;
return j->value;
}
if (!fromWith->parentWith) [[unlikely]]
if (!fromWith->parentWith)
error<UndefinedVarError>("undefined variable '%1%'", symbols[var.name])
.atPos(var.pos)
.withFrame(*env, var)
@@ -2405,7 +2406,8 @@ BackedStringView EvalState::coerceToString(
std::string_view errorCtx,
bool coerceMore,
bool copyToStore,
bool canonicalizePath)
bool canonicalizePath,
bool strict)
{
auto _level = addCallDepth(pos);
@@ -2432,14 +2434,28 @@ BackedStringView EvalState::coerceToString(
auto maybeString = tryAttrsToString(pos, v, context, coerceMore, copyToStore);
if (maybeString)
return std::move(*maybeString);
auto i = v.attrs()->get(s.outPath);
if (!i) {
auto outPathAttr = v.attrs()->get(s.outPath);
if (!outPathAttr) {
error<TypeError>(
"cannot coerce %1% to a string: %2%", showType(v), ValuePrinter(*this, v, errorPrintOptions))
.withTrace(pos, errorCtx)
.debugThrow();
}
return coerceToString(pos, *i->value, context, errorCtx, coerceMore, copyToStore, canonicalizePath);
try {
auto derivedPath = coerceToSingleDerivedPath(pos, v, errorCtx);
auto result = mkSingleDerivedPathStringRaw(derivedPath);
context.insert(
std::visit([](auto && o) -> NixStringContextElem { return std::move(o); }, std::move(derivedPath)));
return result;
} catch (Error & e) {
if (strict)
throw;
auto info = e.info();
info.msg = HintFmt("in a future version of Nix this will be an error: %s", Uncolored(info.msg.str()));
logWarning(info);
return coerceToString(
pos, *outPathAttr->value, context, errorCtx, coerceMore, copyToStore, canonicalizePath, strict);
}
}
if (v.type() == nExternal) {
@@ -2477,7 +2493,8 @@ BackedStringView EvalState::coerceToString(
"while evaluating one element of the list",
coerceMore,
copyToStore,
canonicalizePath);
canonicalizePath,
strict);
} catch (Error & e) {
e.addTrace(positions[pos], errorCtx);
throw;
@@ -2564,63 +2581,131 @@ EvalState::coerceToStorePath(const PosIdx pos, Value & v, NixStringContext & con
error<EvalError>("path '%1%' is not in the Nix store", path).withTrace(pos, errorCtx).debugThrow();
}
std::pair<SingleDerivedPath, std::string_view> EvalState::coerceToSingleDerivedPathUnchecked(
const PosIdx pos, Value & v, std::string_view errorCtx, const ExperimentalFeatureSettings & xpSettings)
SingleDerivedPath EvalState::coerceToSingleDerivedPath(
const PosIdx pos,
Value & v,
std::string_view errorCtx,
const ExperimentalFeatureSettings & xpSettings,
bool skipStringValidation)
{
NixStringContext context;
auto s = forceString(v, context, pos, errorCtx, xpSettings);
auto csize = context.size();
if (csize != 1)
error<EvalError>("string '%s' has %d entries in its context. It should only have exactly one entry", s, csize)
.withTrace(pos, errorCtx)
.debugThrow();
auto derivedPath = std::visit(
overloaded{
[&](NixStringContextElem::Opaque && o) -> SingleDerivedPath { return std::move(o); },
[&](NixStringContextElem::DrvDeep &&) -> SingleDerivedPath {
error<EvalError>(
"string '%s' has a context which refers to a complete source and binary closure. This is not supported at this time",
s)
.withTrace(pos, errorCtx)
.debugThrow();
},
[&](NixStringContextElem::Built && b) -> SingleDerivedPath { return std::move(b); },
},
((NixStringContextElem &&) *context.begin()).raw);
return {
std::move(derivedPath),
std::move(s),
};
}
forceValue(v, pos);
SingleDerivedPath EvalState::coerceToSingleDerivedPath(const PosIdx pos, Value & v, std::string_view errorCtx)
{
auto [derivedPath, s_] = coerceToSingleDerivedPathUnchecked(pos, v, errorCtx);
auto s = s_;
auto sExpected = mkSingleDerivedPathStringRaw(derivedPath);
if (s != sExpected) {
/* `std::visit` is used here just to provide a more precise
error message. */
std::visit(
switch (v.type()) {
case nAttrs: {
if (!isDerivation(v)) {
error<TypeError>(
"cannot coerce a set that is not a package to a single derived path: %1%",
ValuePrinter(*this, v, errorPrintOptions))
.withTrace(pos, errorCtx)
.debugThrow();
}
auto outPathAttr = v.attrs()->get(s.outPath);
if (!outPathAttr) {
error<TypeError>(
"cannot coerce %1% to a single derived path: no 'outPath' attribute: %2%",
showType(v),
ValuePrinter(*this, v, errorPrintOptions))
.withTrace(pos, errorCtx)
.debugThrow();
}
auto outPathDerivedPath =
coerceToSingleDerivedPath(pos, *outPathAttr->value, errorCtx, xpSettings, skipStringValidation);
auto drvPathAttr = v.attrs()->get(s.drvPath);
if (drvPathAttr) {
auto drvPathDerivedPath =
coerceToSingleDerivedPath(pos, *drvPathAttr->value, errorCtx, xpSettings, skipStringValidation);
std::visit(
overloaded{
[&](const SingleDerivedPath::Built & built) {
if (*built.drvPath != drvPathDerivedPath) {
error<EvalError>(
"relative outPath '%s' does not refer to drvPath '%s'",
outPathDerivedPath.to_string(*store),
drvPathDerivedPath.to_string(*store))
.withTrace(pos, errorCtx)
.debugThrow();
}
},
[&](const SingleDerivedPath::Opaque &) {
error<EvalError>(
"relative outPath '%s' does not refer to drvPath '%s'",
outPathDerivedPath.to_string(*store),
drvPathDerivedPath.to_string(*store))
.withTrace(pos, errorCtx)
.debugThrow();
},
},
outPathDerivedPath.raw());
}
return outPathDerivedPath;
}
case nString: {
auto s = v.string_view();
NixStringContext context;
copyContext(v, context, xpSettings);
auto csize = context.size();
if (csize != 1)
error<EvalError>(
"string '%s' has %d entries in its context. It should only have exactly one entry", s, csize)
.withTrace(pos, errorCtx)
.debugThrow();
auto derivedPath = std::visit(
overloaded{
[&](const SingleDerivedPath::Opaque & o) {
error<EvalError>("path string '%s' has context with the different path '%s'", s, sExpected)
[&](NixStringContextElem::Opaque && o) -> SingleDerivedPath { return std::move(o); },
[&](NixStringContextElem::DrvDeep &&) -> SingleDerivedPath {
error<EvalError>(
"string '%s' has a context which refers to a complete source and binary closure. This is not "
"supported at this time",
s)
.withTrace(pos, errorCtx)
.debugThrow();
},
[&](const SingleDerivedPath::Built & b) {
error<EvalError>(
"string '%s' has context with the output '%s' from derivation '%s', but the string is not the right placeholder for this derivation output. It should be '%s'",
s,
b.output,
b.drvPath->to_string(*store),
sExpected)
.withTrace(pos, errorCtx)
.debugThrow();
}},
derivedPath.raw());
[&](NixStringContextElem::Built && b) -> SingleDerivedPath { return std::move(b); },
},
((NixStringContextElem &&) *context.begin()).raw);
if (!skipStringValidation) {
auto sExpected = mkSingleDerivedPathStringRaw(derivedPath);
if (s != sExpected) {
/* `std::visit` is used here just to provide a more precise
error message. */
std::visit(
overloaded{
[&](const SingleDerivedPath::Opaque & o) {
error<EvalError>("path string '%s' has context with the different path '%s'", s, sExpected)
.withTrace(pos, errorCtx)
.debugThrow();
},
[&](const SingleDerivedPath::Built & b) {
error<EvalError>(
"string '%s' has context with the output '%s' from derivation '%s', but the string is "
"not the right placeholder for this derivation output. It should be '%s'",
s,
b.output,
b.drvPath->to_string(*store),
sExpected)
.withTrace(pos, errorCtx)
.debugThrow();
}},
derivedPath.raw());
}
}
return derivedPath;
}
default:
error<TypeError>(
"cannot coerce %1% to a single derived path: %2%", showType(v), ValuePrinter(*this, v, errorPrintOptions))
.withTrace(pos, errorCtx)
.debugThrow();
}
return derivedPath;
}
// NOTE: This implementation must match eqValues!
+97 -23
View File
@@ -1,5 +1,6 @@
#include "nix/expr/get-drvs.hh"
#include "nix/expr/eval-inline.hh"
#include "nix/store/derived-path.hh"
#include "nix/store/derivations.hh"
#include "nix/store/store-api.hh"
#include "nix/store/path-with-outputs.hh"
@@ -21,13 +22,13 @@ PackageInfo::PackageInfo(EvalState & state, ref<Store> store, const std::string
, attrs(nullptr)
, attrPath("")
{
auto [drvPath, selectedOutputs] = parsePathWithOutputs(*store, drvPathWithOutputs);
auto [drvStorePath, selectedOutputs] = parsePathWithOutputs(*store, drvPathWithOutputs);
this->drvPath = drvPath;
this->drvPath = {{Path{SingleDerivedPath::Opaque{drvStorePath}, drvStorePath}}};
auto drv = store->derivationFromPath(drvPath);
auto drv = store->derivationFromPath(drvStorePath);
name = drvPath.name();
name = drvStorePath.name();
if (selectedOutputs.size() > 1)
throw Error("building more than one derivation output is not supported, in '%s'", drvPathWithOutputs);
@@ -36,10 +37,18 @@ PackageInfo::PackageInfo(EvalState & state, ref<Store> store, const std::string
auto i = drv.outputs.find(outputName);
if (i == drv.outputs.end())
throw Error("derivation '%s' does not have output '%s'", store->printStorePath(drvPath), outputName);
auto & [outputName, output] = *i;
throw Error("derivation '%s' does not have output '%s'", store->printStorePath(drvStorePath), outputName);
auto & [outputName_, output] = *i;
outPath = {output.path(*store, drv.name, outputName)};
auto outStorePath = output.path(*store, drv.name, outputName_);
if (outStorePath) {
outPath = Path{SingleDerivedPath::Opaque{*outStorePath}, *outStorePath};
} else {
// CA derivation with unknown output path
outPath = Path{
SingleDerivedPath::Built{makeConstantStorePathRef(drvStorePath), outputName_},
std::nullopt};
}
}
std::string PackageInfo::queryName() const
@@ -64,26 +73,60 @@ std::string PackageInfo::querySystem() const
return system;
}
std::optional<StorePath> PackageInfo::queryDrvPath() const
std::optional<PackageInfo::Path> PackageInfo::queryDrvPathFlexible() const
{
if (!drvPath && attrs) {
if (auto i = attrs->get(state->s.drvPath)) {
NixStringContext context;
auto found = state->coerceToStorePath(
i->pos, *i->value, context, "while evaluating the 'drvPath' attribute of a derivation");
auto i = attrs->get(state->s.drvPath);
if (i) {
Value v;
v.mkAttrs(const_cast<Bindings *>(attrs));
std::optional<SingleDerivedPath> outPathDerivedPath;
try {
found.requireDerivation();
// Validate derivation structure
outPathDerivedPath = state->coerceToSingleDerivedPath(
noPos, v, "while evaluating the derivation");
} catch (Error & e) {
e.addTrace(state->positions[i->pos], "while evaluating the 'drvPath' attribute of a derivation");
throw;
auto info = e.info();
info.msg = HintFmt("in a future version of Nix this will be an error: %s", Uncolored(info.msg.str()));
logWarning(info);
}
if (outPathDerivedPath) {
// Validation passed. Now get drvPath.
auto drvPathDerivedPath = state->coerceToSingleDerivedPath(
i->pos, *i->value, "while evaluating the 'drvPath' attribute of a derivation");
if (auto * opaque = std::get_if<SingleDerivedPath::Opaque>(&drvPathDerivedPath.raw())) {
opaque->path.requireDerivation();
drvPath = {{Path{std::move(drvPathDerivedPath), opaque->path}}};
} else {
// Dynamic derivation - drvPath is itself a derivation output
drvPath = {{Path{std::move(drvPathDerivedPath), std::nullopt}}};
}
} else {
// Fall back to old behavior
NixStringContext context;
auto found = state->coerceToStorePath(
i->pos, *i->value, context, "while evaluating the 'drvPath' attribute of a derivation");
try {
found.requireDerivation();
} catch (Error & e) {
e.addTrace(state->positions[i->pos], "while evaluating the 'drvPath' attribute of a derivation");
throw;
}
drvPath = {{Path{SingleDerivedPath::Opaque{found}, std::move(found)}}};
}
drvPath = {std::move(found)};
} else
drvPath = {std::nullopt};
}
return drvPath.value_or(std::nullopt);
}
std::optional<StorePath> PackageInfo::queryDrvPath() const
{
if (auto path = queryDrvPathFlexible())
return path->storePath;
return std::nullopt;
}
StorePath PackageInfo::requireDrvPath() const
{
if (auto drvPath = queryDrvPath())
@@ -91,20 +134,51 @@ StorePath PackageInfo::requireDrvPath() const
throw Error("derivation does not contain a 'drvPath' attribute");
}
StorePath PackageInfo::queryOutPath() const
PackageInfo::Path PackageInfo::queryOutPathFlexible() const
{
if (!outPath && attrs) {
auto i = attrs->get(state->s.outPath);
NixStringContext context;
if (i)
outPath = state->coerceToStorePath(
i->pos, *i->value, context, "while evaluating the output path of a derivation");
Value v;
v.mkAttrs(const_cast<Bindings *>(attrs));
std::optional<SingleDerivedPath> derivedPath;
try {
derivedPath = state->coerceToSingleDerivedPath(
noPos, v, "while evaluating the derivation");
} catch (Error & e) {
auto info = e.info();
info.msg = HintFmt("in a future version of Nix this will be an error: %s", Uncolored(info.msg.str()));
logWarning(info);
}
if (derivedPath) {
if (auto * opaque = std::get_if<SingleDerivedPath::Opaque>(&derivedPath->raw())) {
outPath = Path{std::move(*derivedPath), opaque->path};
} else {
// Built path - no concrete output path available (placeholder)
outPath = Path{std::move(*derivedPath), std::nullopt};
}
} else {
// Fall back to old behavior
auto i = attrs->get(state->s.outPath);
NixStringContext context;
if (i) {
auto path = state->coerceToStorePath(
i->pos, *i->value, context, "while evaluating the output path of a derivation");
outPath = Path{SingleDerivedPath::Opaque{path}, std::move(path)};
}
}
}
if (!outPath)
throw UnimplementedError("CA derivations are not yet supported");
throw Error("derivation does not have an 'outPath' attribute");
return *outPath;
}
StorePath PackageInfo::queryOutPath() const
{
auto path = queryOutPathFlexible();
if (!path.storePath)
throw UnimplementedError("CA derivations are not yet supported");
return *path.storePath;
}
PackageInfo::Outputs PackageInfo::queryOutputs(bool withPaths, bool onlyOutputsToInstall)
{
if (outputs.empty()) {
+1 -1
View File
@@ -11,7 +11,7 @@ namespace nix {
* variable is set. This is to prevent contention on these counters
* when multi-threaded evaluation is enabled.
*/
struct alignas(std::hardware_destructive_interference_size) Counter
struct alignas(64) Counter
{
using value_type = uint64_t;
+28 -21
View File
@@ -715,6 +715,9 @@ public:
* string. If `coerceMore` is set, also converts nulls, integers,
* booleans and lists to a string. If `copyToStore` is set,
* referenced paths are copied to the Nix store as a side effect.
* If `strict` is set, ensures that when coercing an attrset with
* both `outPath` and `drvPath`, the string context for `outPath`
* agrees with the `drvPath` attribute.
*/
BackedStringView coerceToString(
const PosIdx pos,
@@ -723,7 +726,8 @@ public:
std::string_view errorCtx,
bool coerceMore = false,
bool copyToStore = true,
bool canonicalizePath = true);
bool canonicalizePath = true,
bool strict = false);
StorePath copyPathToStore(NixStringContext & context, const SourcePath & path);
@@ -742,30 +746,33 @@ public:
StorePath coerceToStorePath(const PosIdx pos, Value & v, NixStringContext & context, std::string_view errorCtx);
/**
* Part of `coerceToSingleDerivedPath()` without any store IO which is exposed for unit testing only.
* Coerce to `SingleDerivedPath`.
*
* For strings: must be either a literal store path or a
* placeholder (see `DownstreamPlaceholder`). The string context
* must be exactly one element, which is either a
* `NixStringContextElem::Opaque` or `NixStringContextElem::Built`.
* (`NixStringContextElem::DrvDeep` is not permitted). The string
* is parsed based on the context --- the context is the source of
* truth, and ultimately tells us what we want, and then we ensure
* the string corresponds to it.
*
* For attrsets: must have an `outPath` attribute which is
* recursively coerced. If a `drvPath` attribute is present, it is
* also coerced and validated to agree with the `outPath` context.
*
* @param xpSettings Stop-gap to avoid globals during unit tests.
*
* @param skipStringValidation Skip validating that the string
* matches the expected placeholder for the context. This is
* exposed for unit testing only to avoid store IO.
*/
std::pair<SingleDerivedPath, std::string_view> coerceToSingleDerivedPathUnchecked(
SingleDerivedPath coerceToSingleDerivedPath(
const PosIdx pos,
Value & v,
std::string_view errorCtx,
const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
/**
* Coerce to `SingleDerivedPath`.
*
* Must be a string which is either a literal store path or a
* "placeholder (see `DownstreamPlaceholder`).
*
* Even more importantly, the string context must be exactly one
* element, which is either a `NixStringContextElem::Opaque` or
* `NixStringContextElem::Built`. (`NixStringContextEleme::DrvDeep`
* is not permitted).
*
* The string is parsed based on the context --- the context is the
* source of truth, and ultimately tells us what we want, and then
* we ensure the string corresponds to it.
*/
SingleDerivedPath coerceToSingleDerivedPath(const PosIdx pos, Value & v, std::string_view errorCtx);
const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings,
bool skipStringValidation = false);
#if NIX_USE_BOEHMGC
/** A GC root for the baseEnv reference. */
+20 -4
View File
@@ -2,6 +2,7 @@
///@file
#include "nix/expr/eval.hh"
#include "nix/store/derived-path.hh"
#include "nix/store/path.hh"
#include <string>
@@ -20,10 +21,23 @@ public:
private:
EvalState * state;
struct Path {
/**
* Underlying semantic path, valid in CA and IA cases.
*/
SingleDerivedPath derivedPath;
/**
* If we don't need a placeholder, we'll have a real store path, which we can remember here.
* For CA derivations using placeholders, this will be std::nullopt.
*/
std::optional<StorePath> storePath;
};
mutable std::string name;
mutable std::string system;
mutable std::optional<std::optional<StorePath>> drvPath;
mutable std::optional<StorePath> outPath;
mutable std::optional<std::optional<Path>> drvPath;
mutable std::optional<Path> outPath;
mutable std::string outputName;
Outputs outputs;
@@ -52,8 +66,10 @@ public:
std::string queryName() const;
std::string querySystem() const;
std::optional<StorePath> queryDrvPath() const;
std::optional<Path> queryDrvPathFlexible() const;
StorePath requireDrvPath() const;
StorePath queryOutPath() const;
Path queryOutPathFlexible() const;
std::string queryOutputName() const;
/**
* Return the unordered map of output names to (optional) output paths.
@@ -81,12 +97,12 @@ public:
void setDrvPath(StorePath path)
{
drvPath = {{std::move(path)}};
drvPath = {{Path{SingleDerivedPath::Opaque{path}, std::move(path)}}};
}
void setOutPath(StorePath path)
{
outPath = {{std::move(path)}};
outPath = Path{SingleDerivedPath::Opaque{path}, std::move(path)};
}
void setFailed()
+47 -50
View File
@@ -35,7 +35,7 @@ class BindingsBuilder;
* about how this is mapped into the alignment bits to save significant memory.
* This also restricts the number of internal types represented with distinct memory layouts.
*/
enum InternalType {
typedef enum {
tUninitialized = 0,
/* layout: Single/zero field payload */
tInt = 1,
@@ -46,20 +46,16 @@ enum InternalType {
tPrimOp,
tAttrs,
/* layout: Pair of pointers payload */
tFirstPairOfPointers,
tListSmall = tFirstPairOfPointers,
tListSmall,
tPrimOpApp,
tApp,
tThunk,
tLambda,
tLastPairOfPointers = tLambda,
/* layout: Single untaggable field */
tFirstSingleUntaggable,
tListN = tFirstSingleUntaggable,
tListN,
tString,
tPath,
tNumberOfInternalTypes, // Must be last
};
} InternalType;
/**
* This type abstracts over all actual value types in the language,
@@ -138,7 +134,7 @@ public:
virtual bool operator==(const ExternalValueBase & b) const noexcept;
/**
* Print the value as JSON. Defaults to unconvertible, i.e. throws an error
* Print the value as JSON. Defaults to unconvertable, i.e. throws an error
*/
virtual nlohmann::json
printValueAsJSON(EvalState & state, bool strict, NixStringContext & context, bool copyToStore = true) const;
@@ -591,7 +587,7 @@ class alignas(16)
enum PrimaryDiscriminator : int {
pdUninitialized = 0,
pdSingleDWord, //< layout: Single/zero field payload
/* The order of these enumerations must be the same as in InternalType. */
/* The order of these enumations must be the same as in InternalType. */
pdListN, //< layout: Single untaggable field.
pdString,
pdPath,
@@ -637,7 +633,7 @@ class alignas(16)
template<InternalType type, typename T, typename U>
void setPairOfPointersPayload(T * firstPtrField, U * secondPtrField) noexcept
{
static_assert(type >= tFirstPairOfPointers && type <= tLastPairOfPointers);
static_assert(type >= tListSmall && type <= tLambda);
{
auto firstFieldPayload = std::bit_cast<PackedPointer>(firstPtrField);
assertAligned(firstFieldPayload);
@@ -646,7 +642,7 @@ class alignas(16)
{
auto secondFieldPayload = std::bit_cast<PackedPointer>(secondPtrField);
assertAligned(secondFieldPayload);
payload[1] = (type - tFirstPairOfPointers) | secondFieldPayload;
payload[1] = (type - tListSmall) | secondFieldPayload;
}
}
@@ -674,11 +670,11 @@ protected:
case pdListN:
case pdString:
case pdPath:
return static_cast<InternalType>(tFirstSingleUntaggable + (pd - pdListN));
return static_cast<InternalType>(tListN + (pd - pdListN));
case pdPairOfPointers:
return static_cast<InternalType>(tFirstPairOfPointers + (payload[1] & discriminatorMask));
return static_cast<InternalType>(tListSmall + (payload[1] & discriminatorMask));
[[unlikely]] default:
nixUnreachableWhenHardened();
unreachable();
}
}
@@ -1031,7 +1027,7 @@ private:
T getStorage() const noexcept
{
if (getInternalType() != detail::payloadTypeToInternalType<T>) [[unlikely]]
nixUnreachableWhenHardened();
unreachable();
T out;
ValueStorage::getStorage(out);
return out;
@@ -1083,44 +1079,45 @@ public:
* Returns the normal type of a Value. This only returns nThunk if
* the Value hasn't been forceValue'd
*
* @param invalidIsThunk Instead of UB an an invalid (probably
* @param invalidIsThunk Instead of aborting an an invalid (probably
* 0, so uninitialized) internal type, return `nThunk`.
*/
template<bool invalidIsThunk = false>
inline ValueType type() const
inline ValueType type(bool invalidIsThunk = false) const
{
/* Explicit lookup table. switch() might compile down (and it does at least with GCC 14)
to a jump table. Let's help the compiler a bit here. */
static constexpr auto table = [] {
std::array<ValueType, tNumberOfInternalTypes> t{};
t[tUninitialized] = nThunk;
t[tInt] = nInt;
t[tBool] = nBool;
t[tNull] = nNull;
t[tFloat] = nFloat;
t[tExternal] = nExternal;
t[tAttrs] = nAttrs;
t[tPrimOp] = nFunction;
t[tLambda] = nFunction;
t[tPrimOpApp] = nFunction;
t[tApp] = nThunk;
t[tThunk] = nThunk;
t[tListSmall] = nList;
t[tListN] = nList;
t[tString] = nString;
t[tPath] = nPath;
return t;
}();
auto it = getInternalType();
if (it == tUninitialized || it >= tNumberOfInternalTypes) [[unlikely]] {
if constexpr (invalidIsThunk)
return nThunk;
else
nixUnreachableWhenHardened();
switch (getInternalType()) {
case tUninitialized:
break;
case tInt:
return nInt;
case tBool:
return nBool;
case tString:
return nString;
case tPath:
return nPath;
case tNull:
return nNull;
case tAttrs:
return nAttrs;
case tListSmall:
case tListN:
return nList;
case tLambda:
case tPrimOp:
case tPrimOpApp:
return nFunction;
case tExternal:
return nExternal;
case tFloat:
return nFloat;
case tThunk:
case tApp:
return nThunk;
}
return table[it];
if (invalidIsThunk)
return nThunk;
else
unreachable();
}
/**
+1 -1
View File
@@ -28,7 +28,7 @@ deps_public_maybe_subproject = [
subdir('nix-meson-build-support/subprojects')
subdir('nix-meson-build-support/big-objs')
# Check for each of these functions, and create a define like `#define HAVE_SYSCONF 1`.
# Check for each of these functions, and create a define like `#define HAVE_LCHOWN 1`.
check_funcs = [
'sysconf',
]
+2 -1
View File
@@ -23,12 +23,13 @@ SourcePath EvalState::storePath(const StorePath & path)
StorePath
EvalState::mountInput(fetchers::Input & input, const fetchers::Input & originalInput, ref<SourceAccessor> accessor)
{
auto [storePath, narHash] = fetchToStore2(fetchSettings, *store, accessor, FetchMode::Copy, input.getName());
auto storePath = fetchToStore(fetchSettings, *store, accessor, FetchMode::Copy, input.getName());
allowPath(storePath); // FIXME: should just whitelist the entire virtual store
storeFS->mount(CanonPath(store->printStorePath(storePath)), accessor);
auto narHash = store->queryPathInfo(storePath)->narHash;
input.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true));
if (originalInput.getNarHash() && narHash != *originalInput.getNarHash())
+2 -7
View File
@@ -1814,13 +1814,8 @@ static void derivationStrictInternal(EvalState & state, std::string_view drvName
drv.fillInOutputPaths(*state.store);
}
/* Write the resulting term into the Nix store directory.
Unless we are in read-only mode, that is, in which case we do not
write anything. Users commonly do this to speed up evaluation in
contexts where they don't actually want to build anything. */
auto drvPath =
settings.readOnlyMode ? computeStorePath(*state.store, drv) : state.store->writeDerivation(drv, state.repair);
/* Write the resulting term into the Nix store directory. */
auto drvPath = writeDerivation(*state.store, drv, state.repair);
auto drvPathS = state.store->printStorePath(drvPath);
printMsg(lvlChatty, "instantiated '%1%' -> '%2%'", drvName, drvPathS);
+7 -13
View File
@@ -191,25 +191,19 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value ** args
{.msg = HintFmt("attribute '%s' is missing in call to 'fetchClosure'", "fromStore"),
.pos = state.positions[pos]});
auto storeRef = StoreReference::parse(*fromStoreUrl);
auto parsedURL = parseURL(*fromStoreUrl, /*lenient=*/true);
if ([&] {
auto * specified = std::get_if<StoreReference::Specified>(&storeRef.variant);
return !specified
|| (specified->scheme != "http" && specified->scheme != "https"
&& !(getEnv("_NIX_IN_TEST").has_value() && specified->scheme == "file"));
}())
throw Error({
.msg = HintFmt("'fetchClosure' only supports http:// and https:// stores"),
.pos = state.positions[pos],
});
if (parsedURL.scheme != "http" && parsedURL.scheme != "https"
&& !(getEnv("_NIX_IN_TEST").has_value() && parsedURL.scheme == "file"))
throw Error(
{.msg = HintFmt("'fetchClosure' only supports http:// and https:// stores"), .pos = state.positions[pos]});
if (!storeRef.params.empty())
if (!parsedURL.query.empty())
throw Error(
{.msg = HintFmt("'fetchClosure' does not support URL query parameters (in '%s')", *fromStoreUrl),
.pos = state.positions[pos]});
auto fromStore = openStore(std::move(storeRef));
auto fromStore = openStore(parsedURL.to_string());
if (toPath)
runFetchClosureWithRewrite(state, pos, *fromStore, *fromPath, *toPath, v);
+3 -9
View File
@@ -34,20 +34,14 @@ struct CacheImpl : Cache
Sync<State> _state;
/**
* This is a back-reference to the `Settings` that owns us.
*/
const Settings & settings;
CacheImpl(const Settings & _settings)
: settings(_settings)
CacheImpl()
{
auto state(_state.lock());
auto dbPath = (getCacheDir() / "fetcher-cache-v4.sqlite").string();
createDirs(dirOf(dbPath));
state->db = SQLite(dbPath, {.useWAL = nix::settings.useSQLiteWAL});
state->db = SQLite(dbPath, {.useWAL = settings.useSQLiteWAL});
state->db.isCache();
state->db.exec(schema);
@@ -160,7 +154,7 @@ ref<Cache> Settings::getCache() const
{
auto cache(_cache.lock());
if (!*cache)
*cache = std::make_shared<CacheImpl>(*this);
*cache = std::make_shared<CacheImpl>();
return ref<Cache>(*cache);
}
+18 -69
View File
@@ -5,12 +5,12 @@
namespace nix {
fetchers::Cache::Key
makeSourcePathToHashCacheKey(std::string_view fingerprint, ContentAddressMethod method, const CanonPath & path)
fetchers::Cache::Key makeFetchToStoreCacheKey(
const std::string & name, const std::string & fingerprint, ContentAddressMethod method, const std::string & path)
{
return fetchers::Cache::Key{
"sourcePathToHash",
{{"fingerprint", std::string(fingerprint)}, {"method", std::string{method.render()}}, {"path", path.abs()}}};
"fetchToStore",
{{"name", name}, {"fingerprint", fingerprint}, {"method", std::string{method.render()}}, {"path", path}}};
}
StorePath fetchToStore(
@@ -23,45 +23,19 @@ StorePath fetchToStore(
PathFilter * filter,
RepairFlag repair)
{
return fetchToStore2(settings, store, path, mode, name, method, filter, repair).first;
}
// FIXME: add an optimisation for the case where the accessor is
// a `PosixSourceAccessor` pointing to a store path.
std::pair<StorePath, Hash> fetchToStore2(
const fetchers::Settings & settings,
Store & store,
const SourcePath & path,
FetchMode mode,
std::string_view name,
ContentAddressMethod method,
PathFilter * filter,
RepairFlag repair)
{
std::optional<fetchers::Cache::Key> cacheKey;
auto [subpath, fingerprint] = filter ? std::pair<CanonPath, std::optional<std::string>>{path.path, std::nullopt}
: path.accessor->getFingerprint(path.path);
if (fingerprint) {
cacheKey = makeSourcePathToHashCacheKey(*fingerprint, method, subpath);
if (auto res = settings.getCache()->lookup(*cacheKey)) {
auto hash = Hash::parseSRI(fetchers::getStrAttr(*res, "hash"));
auto storePath =
store.makeFixedOutputPathFromCA(name, ContentAddressWithReferences::fromParts(method, hash, {}));
/* Add a temproot before the call to isValidPath to prevent accidental GC in case the
input is cached. Note that this must be done before to avoid races. */
if (mode != FetchMode::DryRun)
store.addTempRoot(storePath);
if (mode == FetchMode::DryRun || store.isValidPath(storePath)) {
debug(
"source path '%s' cache hit in '%s' (hash '%s')",
path,
store.printStorePath(storePath),
hash.to_string(HashFormat::SRI, true));
return {storePath, hash};
}
debug("source path '%s' not in store", path);
cacheKey = makeFetchToStoreCacheKey(std::string{name}, *fingerprint, method, subpath.abs());
if (auto res = settings.getCache()->lookupStorePath(*cacheKey, store)) {
debug("store path cache hit for '%s'", path);
return res->storePath;
}
} else {
static auto barf = getEnv("_NIX_TEST_BARF_ON_UNCACHEABLE").value_or("") == "1";
@@ -79,41 +53,16 @@ std::pair<StorePath, Hash> fetchToStore2(
auto filter2 = filter ? *filter : defaultPathFilter;
auto [storePath, hash] =
mode == FetchMode::DryRun
? [&]() {
auto [storePath, hash] =
store.computeStorePath(name, path, method, HashAlgorithm::SHA256, {}, filter2);
debug(
"hashed '%s' to '%s' (hash '%s')",
path,
store.printStorePath(storePath),
hash.to_string(HashFormat::SRI, true));
return std::make_pair(storePath, hash);
}()
: [&]() {
// FIXME: ideally addToStore() would return the hash
// right away (like computeStorePath()).
auto storePath = store.addToStore(name, path, method, HashAlgorithm::SHA256, {}, filter2, repair);
auto info = store.queryPathInfo(storePath);
assert(info->references.empty());
auto hash = method == ContentAddressMethod::Raw::NixArchive ? info->narHash : ({
if (!info->ca || info->ca->method != method)
throw Error("path '%s' lacks a CA field", store.printStorePath(storePath));
info->ca->hash;
});
debug(
"copied '%s' to '%s' (hash '%s')",
path,
store.printStorePath(storePath),
hash.to_string(HashFormat::SRI, true));
return std::make_pair(storePath, hash);
}();
auto storePath = mode == FetchMode::DryRun
? store.computeStorePath(name, path, method, HashAlgorithm::SHA256, {}, filter2).first
: store.addToStore(name, path, method, HashAlgorithm::SHA256, {}, filter2, repair);
if (cacheKey)
settings.getCache()->upsert(*cacheKey, {{"hash", hash.to_string(HashFormat::SRI, true)}});
debug(mode == FetchMode::DryRun ? "hashed '%s'" : "copied '%s' to '%s'", path, store.printStorePath(storePath));
return {storePath, hash};
if (cacheKey && mode == FetchMode::Copy)
settings.getCache()->upsert(*cacheKey, store, {}, storePath);
return storePath;
}
} // namespace nix
+3 -4
View File
@@ -339,10 +339,9 @@ std::pair<ref<SourceAccessor>, Input> Input::getAccessorUnchecked(const Settings
// can reuse the existing nar instead of copying the unpacked
// input back into the store on every evaluation.
if (accessor->fingerprint) {
settings.getCache()->upsert(
makeSourcePathToHashCacheKey(
*accessor->fingerprint, ContentAddressMethod::Raw::NixArchive, CanonPath::root),
{{"hash", store.queryPathInfo(storePath)->narHash.to_string(HashFormat::SRI, true)}});
ContentAddressMethod method = ContentAddressMethod::Raw::NixArchive;
auto cacheKey = makeFetchToStoreCacheKey(getName(), *accessor->fingerprint, method, "/");
settings.getCache()->upsert(cacheKey, store, {}, storePath);
}
accessor->setPathDisplay("«" + to_string() + "»");
@@ -61,11 +61,6 @@ std::pair<CanonPath, std::optional<std::string>> FilteringSourceAccessor::getFin
return next->getFingerprint(prefix / path);
}
void FilteringSourceAccessor::invalidateCache(const CanonPath & path)
{
next->invalidateCache(prefix / path);
}
void FilteringSourceAccessor::checkAccess(const CanonPath & path)
{
if (!isAllowed(path))
+46 -65
View File
@@ -74,29 +74,6 @@ namespace nix {
struct GitSourceAccessor;
struct GitError : public Error
{
template<typename... Ts>
GitError(const git_error & error, Ts &&... args)
: Error("")
{
auto hf = HintFmt(std::forward<Ts>(args)...);
err.msg = HintFmt("%1%: %2% (libgit2 error code = %3%)", Uncolored(hf.str()), error.message, error.klass);
}
template<typename... Ts>
GitError(Ts &&... args)
: GitError(
[]() -> const git_error & {
const git_error * p = git_error_last();
assert(p && "git_error_last() is unexpectedly null");
return *p;
}(),
std::forward<Ts>(args)...)
{
}
};
typedef std::unique_ptr<git_repository, Deleter<git_repository_free>> Repository;
typedef std::unique_ptr<git_tree_entry, Deleter<git_tree_entry_free>> TreeEntry;
typedef std::unique_ptr<git_tree, Deleter<git_tree_free>> Tree;
@@ -129,7 +106,7 @@ static void initLibGit2()
static std::once_flag initialized;
std::call_once(initialized, []() {
if (git_libgit2_init() < 0)
throw GitError("initialising libgit2");
throw Error("initialising libgit2: %s", git_error_last()->message);
});
}
@@ -137,7 +114,7 @@ static git_oid hashToOID(const Hash & hash)
{
git_oid oid;
if (git_oid_fromstr(&oid, hash.gitRev().c_str()))
throw GitError("cannot convert '%s' to a Git OID", hash.gitRev());
throw Error("cannot convert '%s' to a Git OID", hash.gitRev());
return oid;
}
@@ -145,7 +122,8 @@ static Object lookupObject(git_repository * repo, const git_oid & oid, git_objec
{
Object obj;
if (git_object_lookup(Setter(obj), repo, &oid, type)) {
throw GitError("getting Git object '%s'", oid);
auto err = git_error_last();
throw Error("getting Git object '%s': %s", oid, err->message);
}
return obj;
}
@@ -155,7 +133,8 @@ static T peelObject(git_object * obj, git_object_t type)
{
T obj2;
if (git_object_peel((git_object **) (typename T::pointer *) Setter(obj2), obj, type)) {
throw Error("peeling Git object '%s'", *git_object_id(obj));
auto err = git_error_last();
throw Error("peeling Git object '%s': %s", *git_object_id(obj), err->message);
}
return obj2;
}
@@ -165,7 +144,7 @@ static T dupObject(typename T::pointer obj)
{
T obj2;
if (git_object_dup((git_object **) (typename T::pointer *) Setter(obj2), (git_object *) obj))
throw GitError("duplicating object '%s'", *git_object_id((git_object *) obj));
throw Error("duplicating object '%s': %s", *git_object_id((git_object *) obj), git_error_last()->message);
return obj2;
}
@@ -237,7 +216,7 @@ static void initRepoAtomically(std::filesystem::path & path, GitRepo::Options op
Repository tmpRepo;
if (git_repository_init(Setter(tmpRepo), tmpDir.string().c_str(), options.bare))
throw GitError("creating Git repository %s", PathFmt(path));
throw Error("creating Git repository %s: %s", PathFmt(path), git_error_last()->message);
try {
std::filesystem::rename(tmpDir, path);
} catch (std::filesystem::filesystem_error & e) {
@@ -247,8 +226,7 @@ static void initRepoAtomically(std::filesystem::path & path, GitRepo::Options op
|| e.code() == std::errc::directory_not_empty) {
return;
} else
throw SystemError(
e.code(), "moving temporary git repository from %s to %s", PathFmt(tmpDir), PathFmt(path));
throw SysError("moving temporary git repository from %s to %s", PathFmt(tmpDir), PathFmt(path));
}
// we successfully moved the repository, so the temporary directory no longer exists.
delTmpDir.cancel();
@@ -288,7 +266,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
initRepoAtomically(path, options);
if (git_repository_open(Setter(repo), path.string().c_str()))
throw GitError("opening Git repository %s", PathFmt(path));
throw Error("opening Git repository %s: %s", PathFmt(path), git_error_last()->message);
ObjectDb odb;
if (options.packfilesOnly) {
@@ -301,28 +279,28 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
*/
if (git_odb_new(Setter(odb)))
throw GitError("creating Git object database");
throw Error("creating Git object database: %s", git_error_last()->message);
if (git_odb_backend_pack(&packBackend, (path / "objects").string().c_str()))
throw GitError("creating pack backend");
throw Error("creating pack backend: %s", git_error_last()->message);
if (git_odb_add_backend(odb.get(), packBackend, 1))
throw GitError("adding pack backend to Git object database");
throw Error("adding pack backend to Git object database: %s", git_error_last()->message);
} else {
if (git_repository_odb(Setter(odb), repo.get()))
throw GitError("getting Git object database");
throw Error("getting Git object database: %s", git_error_last()->message);
}
// mempack_backend will be owned by the repository, so we are not expected to free it ourselves.
if (git_mempack_new(&mempackBackend))
throw GitError("creating mempack backend");
throw Error("creating mempack backend: %s", git_error_last()->message);
if (git_odb_add_backend(odb.get(), mempackBackend, 999))
throw GitError("adding mempack backend to Git object database");
throw Error("adding mempack backend to Git object database: %s", git_error_last()->message);
if (options.packfilesOnly) {
if (git_repository_set_odb(repo.get(), odb.get()))
throw GitError("setting Git object database");
throw Error("setting Git object database: %s", git_error_last()->message);
}
}
@@ -362,7 +340,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
Indexer indexer;
git_indexer_progress stats;
if (git_indexer_new(Setter(indexer), pack_dir_path.c_str(), 0, nullptr, nullptr))
throw GitError("creating git packfile indexer");
throw Error("creating git packfile indexer: %s", git_error_last()->message);
// TODO: provide index callback for checkInterrupt() termination
// though this is about an order of magnitude faster than the packbuilder
@@ -370,15 +348,15 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
constexpr size_t chunkSize = 128 * 1024;
for (size_t offset = 0; offset < buf.size; offset += chunkSize) {
if (git_indexer_append(indexer.get(), buf.ptr + offset, std::min(chunkSize, buf.size - offset), &stats))
throw GitError("appending to git packfile index");
throw Error("appending to git packfile index: %s", git_error_last()->message);
checkInterrupt();
}
if (git_indexer_commit(indexer.get(), &stats))
throw GitError("committing git packfile index");
throw Error("committing git packfile index: %s", git_error_last()->message);
if (git_mempack_reset(mempackBackend))
throw GitError("resetting git mempack backend");
throw Error("resetting git mempack backend: %s", git_error_last()->message);
checkInterrupt();
}
@@ -471,7 +449,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
void setRemote(const std::string & name, const std::string & url) override
{
if (git_remote_set_url(*this, name.c_str(), url.c_str()))
throw GitError("setting remote '%s' URL to '%s'", name, url);
throw Error("setting remote '%s' URL to '%s': %s", name, url, git_error_last()->message);
}
Hash resolveRef(std::string ref) override
@@ -484,7 +462,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
// an object_id.
std::string peeledRef = ref + "^{commit}";
if (git_revparse_single(Setter(object), *this, peeledRef.c_str()))
throw GitError("resolving Git reference '%s'", ref);
throw Error("resolving Git reference '%s': %s", ref, git_error_last()->message);
auto oid = git_object_id(object.get());
return toHash(*oid);
}
@@ -493,11 +471,11 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
{
GitConfig config;
if (git_config_open_ondisk(Setter(config), configFile.string().c_str()))
throw GitError("parsing .gitmodules file");
throw Error("parsing .gitmodules file: %s", git_error_last()->message);
ConfigIterator it;
if (git_config_iterator_glob_new(Setter(it), config.get(), "^submodule\\..*\\.(path|url|branch)$"))
throw GitError("iterating over .gitmodules");
throw Error("iterating over .gitmodules: %s", git_error_last()->message);
StringMap entries;
@@ -506,7 +484,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
if (auto err = git_config_next(&entry, it.get())) {
if (err == GIT_ITEROVER)
break;
throw GitError("iterating over .gitmodules");
throw Error("iterating over .gitmodules: %s", git_error_last()->message);
}
entries.emplace(entry->name + 10, entry->value);
}
@@ -543,7 +521,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
git_oid headRev;
if (auto err = git_reference_name_to_id(&headRev, *this, "HEAD")) {
if (err != GIT_ENOTFOUND)
throw GitError("resolving HEAD");
throw Error("resolving HEAD: %s", git_error_last()->message);
} else
info.headRev = toHash(headRev);
@@ -566,7 +544,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
options.flags |= GIT_STATUS_OPT_INCLUDE_UNMODIFIED;
options.flags |= GIT_STATUS_OPT_EXCLUDE_SUBMODULES;
if (git_status_foreach_ext(*this, &options, &statusCallbackTrampoline, &statusCallback))
throw GitError("getting working directory status");
throw Error("getting working directory status: %s", git_error_last()->message);
/* Get submodule info. */
auto modulesFile = path / ".gitmodules";
@@ -609,7 +587,8 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
if (auto errCode = git_object_lookup(Setter(obj), *this, &oid, GIT_OBJECT_ANY)) {
if (errCode == GIT_ENOTFOUND)
return false;
throw GitError("getting Git object '%s'", oid);
auto err = git_error_last();
throw Error("getting Git object '%s': %s", oid, err->message);
}
return true;
@@ -939,7 +918,7 @@ struct GitSourceAccessor : SourceAccessor
TreeEntry copy;
if (git_tree_entry_dup(Setter(copy), entry))
throw GitError("dupping tree entry");
throw Error("dupping tree entry: %s", git_error_last()->message);
auto entryName = std::string_view(git_tree_entry_name(entry));
@@ -969,7 +948,7 @@ struct GitSourceAccessor : SourceAccessor
Tree tree;
if (git_tree_entry_to_object((git_object **) (git_tree **) Setter(tree), *state.repo, entry))
throw GitError("looking up directory '%s'", showPath(path));
throw Error("looking up directory '%s': %s", showPath(path), git_error_last()->message);
return tree;
}
@@ -1004,7 +983,7 @@ struct GitSourceAccessor : SourceAccessor
Tree tree;
if (git_tree_entry_to_object((git_object **) (git_tree **) Setter(tree), *state.repo, entry))
throw GitError("looking up directory '%s'", showPath(path));
throw Error("looking up directory '%s': %s", showPath(path), git_error_last()->message);
return tree;
}
@@ -1037,7 +1016,7 @@ struct GitSourceAccessor : SourceAccessor
Blob blob;
if (git_tree_entry_to_object((git_object **) (git_blob **) Setter(blob), *state.repo, entry))
throw GitError("looking up file '%s'", showPath(path));
throw Error("looking up file '%s': %s", showPath(path), git_error_last()->message);
return blob;
}
@@ -1089,7 +1068,7 @@ struct GitExportIgnoreSourceAccessor : CachingFilteringSourceAccessor
if (git_error_last()->klass == GIT_ENOTFOUND)
return false;
else
throw GitError("looking up '%s'", showPath(path));
throw Error("looking up '%s': %s", showPath(path), git_error_last()->message);
} else {
// Official git will silently reject export-ignore lines that have
// values. We do the same.
@@ -1245,17 +1224,17 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink
repo.emplace(parent.repoPool.get());
if (git_blob_create_from_stream(Setter(stream), **repo, nullptr))
throw GitError("creating a blob stream object");
throw Error("creating a blob stream object: %s", git_error_last()->message);
if (stream->write(stream.get(), contents.data(), contents.size()))
throw GitError("writing a blob for tarball member '%s'", path);
throw Error("writing a blob for tarball member '%s': %s", path, git_error_last()->message);
parent.totalBufSize -= contents.size();
contents.clear();
}
} else {
if (stream->write(stream.get(), data.data(), data.size()))
throw GitError("writing a blob for tarball member '%s'", path);
throw Error("writing a blob for tarball member '%s': %s", path, git_error_last()->message);
}
}
@@ -1277,7 +1256,7 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink
acquires ownership and frees the stream. */
git_oid oid;
if (git_blob_create_from_stream_commit(&oid, crf->stream.release()))
throw GitError("creating a blob object for '%s'", path);
throw Error("creating a blob object for '%s': %s", path, git_error_last()->message);
addNode(
*_state.lock(),
crf->path,
@@ -1291,7 +1270,8 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink
git_oid oid;
if (git_blob_create_from_buffer(&oid, *repo, crf->contents.data(), crf->contents.size()))
throw GitError("creating a blob object for '%s' from in-memory buffer", crf->path);
throw Error(
"creating a blob object for '%s' from in-memory buffer: %s", crf->path, git_error_last()->message);
addNode(
*_state.lock(),
@@ -1315,7 +1295,8 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink
git_oid oid;
if (git_blob_create_from_buffer(&oid, *repo, target.c_str(), target.size()))
throw GitError("creating a blob object for tarball symlink member '%s'", path);
throw Error(
"creating a blob object for tarball symlink member '%s': %s", path, git_error_last()->message);
auto state(_state.lock());
addNode(*state, path, Child{GIT_FILEMODE_LINK, oid});
@@ -1376,19 +1357,19 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink
// Write this directory.
git_treebuilder * b;
if (git_treebuilder_new(&b, *repo, nullptr))
throw GitError("creating a tree builder");
throw Error("creating a tree builder: %s", git_error_last()->message);
TreeBuilder builder(b);
for (auto & [name, child] : node.children) {
auto oid_p = std::get_if<git_oid>(&child.file);
auto oid = oid_p ? *oid_p : std::get<Directory>(child.file).oid.value();
if (git_treebuilder_insert(nullptr, builder.get(), name.c_str(), &oid, child.mode))
throw GitError("adding a file to a tree builder");
throw Error("adding a file to a tree builder: %s", git_error_last()->message);
}
git_oid oid;
if (git_treebuilder_write(&oid, builder.get()))
throw GitError("creating a tree object");
throw Error("creating a tree object: %s", git_error_last()->message);
node.oid = oid;
}(_state.lock()->root);
+7 -7
View File
@@ -37,7 +37,7 @@ namespace {
// old version of git, which will ignore unrecognized `-c` options.
const std::string gitInitialBranch = "__nix_dummy_branch";
static bool isCacheFileWithinTtl(const Settings & settings, time_t now, const PosixStat & st)
bool isCacheFileWithinTtl(time_t now, const PosixStat & st)
{
return st.st_mtime + static_cast<time_t>(settings.tarballTtl) > now;
}
@@ -105,7 +105,7 @@ bool storeCachedHead(const std::string & actualUrl, bool shallow, const std::str
return true;
}
static std::optional<std::string> readHeadCached(const Settings & settings, const std::string & actualUrl, bool shallow)
std::optional<std::string> readHeadCached(const std::string & actualUrl, bool shallow)
{
// Create a cache path to store the branch of the HEAD ref. Append something
// in front of the URL to prevent collision with the repository itself.
@@ -117,7 +117,7 @@ static std::optional<std::string> readHeadCached(const Settings & settings, cons
std::optional<std::string> cachedRef;
if (st) {
cachedRef = readHead(cacheDir);
if (cachedRef != std::nullopt && *cachedRef != gitInitialBranch && isCacheFileWithinTtl(settings, now, *st)) {
if (cachedRef != std::nullopt && *cachedRef != gitInitialBranch && isCacheFileWithinTtl(now, *st)) {
debug("using cached HEAD ref '%s' for repo '%s'", *cachedRef, actualUrl);
return cachedRef;
}
@@ -728,12 +728,12 @@ struct GitInputScheme : InputScheme
return revCount;
}
std::string getDefaultRef(const Settings & settings, const RepoInfo & repoInfo, bool shallow) const
std::string getDefaultRef(const RepoInfo & repoInfo, bool shallow) const
{
auto head = std::visit(
overloaded{
[&](const std::filesystem::path & path) { return GitRepo::openRepo(path, {})->getWorkdirRef(); },
[&](const ParsedURL & url) { return readHeadCached(settings, url.to_string(), shallow); }},
[&](const ParsedURL & url) { return readHeadCached(url.to_string(), shallow); }},
repoInfo.location);
if (!head) {
warn("could not read HEAD ref from repo at '%s', using 'master'", repoInfo.locationToArg());
@@ -783,7 +783,7 @@ struct GitInputScheme : InputScheme
auto originalRef = input.getRef();
bool shallow = getShallowAttr(input);
auto ref = originalRef ? *originalRef : getDefaultRef(settings, repoInfo, shallow);
auto ref = originalRef ? *originalRef : getDefaultRef(repoInfo, shallow);
input.attrs.insert_or_assign("ref", ref);
std::filesystem::path repoDir;
@@ -822,7 +822,7 @@ struct GitInputScheme : InputScheme
/* If the local ref is older than 'tarball-ttl' seconds, do a
git fetch to update the local ref to the remote ref. */
auto st = maybeStat(localRefFile);
doFetch = !st || !isCacheFileWithinTtl(settings, now, *st);
doFetch = !st || !isCacheFileWithinTtl(now, *st);
}
}
@@ -129,25 +129,6 @@ struct Settings : public Config
true,
Xp::Flakes};
Setting<unsigned int> tarballTtl{
this,
60 * 60,
"tarball-ttl",
R"(
The number of seconds a downloaded tarball is considered fresh. If
the cached tarball is stale, Nix checks whether it is still up
to date using the ETag header. Nix downloads a new version if
the ETag header is unsupported, or the cached ETag doesn't match.
Setting the TTL to `0` forces Nix to always check if the tarball is
up to date.
Nix caches tarballs in `$XDG_CACHE_HOME/nix/tarballs`.
Files fetched via `NIX_PATH`, `fetchGit`, `fetchMercurial`,
`fetchTarball`, and `fetchurl` respect this TTL.
)"};
ref<Cache> getCache() const;
ref<GitRepo> getTarballCache() const;
@@ -24,17 +24,7 @@ StorePath fetchToStore(
PathFilter * filter = nullptr,
RepairFlag repair = NoRepair);
std::pair<StorePath, Hash> fetchToStore2(
const fetchers::Settings & settings,
Store & store,
const SourcePath & path,
FetchMode mode,
std::string_view name = "source",
ContentAddressMethod method = ContentAddressMethod::Raw::NixArchive,
PathFilter * filter = nullptr,
RepairFlag repair = NoRepair);
fetchers::Cache::Key
makeSourcePathToHashCacheKey(std::string_view fingerprint, ContentAddressMethod method, const CanonPath & path);
fetchers::Cache::Key makeFetchToStoreCacheKey(
const std::string & name, const std::string & fingerprint, ContentAddressMethod method, const std::string & path);
} // namespace nix
@@ -52,8 +52,6 @@ struct FilteringSourceAccessor : SourceAccessor
std::pair<CanonPath, std::optional<std::string>> getFingerprint(const CanonPath & path) override;
void invalidateCache(const CanonPath & path) override;
/**
* Call `makeNotAllowedError` to throw a `RestrictedPathError`
* exception if `isAllowed()` returns `false` for `path`.
@@ -1,5 +1,3 @@
#pragma once
#include "nix/fetchers/fetchers.hh"
namespace nix::fetchers {
+2 -3
View File
@@ -1,6 +1,5 @@
#include "nix/fetchers/fetchers.hh"
#include "nix/util/processes.hh"
#include "nix/util/environment-variables.hh"
#include "nix/util/users.hh"
#include "nix/fetchers/cache.hh"
#include "nix/store/globals.hh"
@@ -17,9 +16,9 @@ namespace nix::fetchers {
static RunOptions hgOptions(const Strings & args)
{
auto env = getEnvOs();
auto env = getEnv();
// Set HGPLAIN: this means we get consistent output from hg and avoids leakage from a user or system .hgrc.
env[OS_STR("HGPLAIN")] = OS_STR("");
env["HGPLAIN"] = "";
return {.program = "hg", .lookupPath = true, .args = args, .environment = env};
}
+7 -5
View File
@@ -165,12 +165,14 @@ struct PathInputScheme : InputScheme
// To prevent `fetchToStore()` copying the path again to Nix
// store, pre-create an entry in the fetcher cache.
auto narHash = store.queryPathInfo(*storePath)->narHash.to_string(HashFormat::SRI, true);
accessor->fingerprint = fmt("path:%s", narHash);
accessor->fingerprint =
fmt("path:%s", store.queryPathInfo(*storePath)->narHash.to_string(HashFormat::SRI, true));
settings.getCache()->upsert(
makeSourcePathToHashCacheKey(
*accessor->fingerprint, ContentAddressMethod::Raw::NixArchive, CanonPath::root),
{{"hash", narHash}});
makeFetchToStoreCacheKey(
input.getName(), *accessor->fingerprint, ContentAddressMethod::Raw::NixArchive, "/"),
store,
{},
*storePath);
/* Trust the lastModified value supplied by the user, if
any. It's not a "secure" attribute so we don't care. */
+1 -1
View File
@@ -92,7 +92,7 @@ void Registry::remove(const Input & input)
static std::filesystem::path getSystemRegistryPath()
{
return nixConfDir() / "registry.json";
return settings.nixConfDir / "registry.json";
}
static std::shared_ptr<Registry> getSystemRegistry(const Settings & settings)
-14
View File
@@ -283,18 +283,4 @@ TEST(to_string, doesntReencodeUrl)
ASSERT_EQ(unparsed, expected);
}
TEST(parseFlakeRef, malformedGithubUrlDoesNotCrash)
{
experimentalFeatureSettings.experimentalFeatures.get().insert(Xp::Flakes);
fetchers::Settings fetchSettings;
// Using ref= instead of rev= with a github: URL should produce an
// error, not an assertion failure in renderAuthorityAndPath
// (https://github.com/NixOS/nix/issues/15196).
EXPECT_THROW(
parseFlakeRef(fetchSettings, "github:nixos/nixpkgs/nixpkgs.git?ref=aead170c1a49253ebfa5027010dfd89a77b73ca4"),
Error);
}
} // namespace nix
-1
View File
@@ -128,7 +128,6 @@ static void prim_flakeRefToString(EvalState & state, const PosIdx pos, Value **
state.forceAttrs(*args[0], noPos, "while evaluating the argument passed to builtins.flakeRefToString");
fetchers::Attrs attrs;
for (const auto & attr : *args[0]->attrs()) {
state.forceValue(*attr.value, attr.pos);
auto t = attr.value->type();
if (t == nInt) {
auto intValue = attr.value->integer().value;
-2
View File
@@ -876,8 +876,6 @@ lockFlake(const Settings & settings, EvalState & state, const FlakeRef & topRef,
CanonPath((topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock"),
newLockFileS,
commitMessage);
flake.lockFilePath().invalidateCache();
}
/* Rewriting the lockfile changed the top-level
+1 -1
View File
@@ -205,7 +205,7 @@ std::pair<FlakeRef, std::string> parsePathFlakeRefWithFragment(
fetchSettings,
{
.scheme = "path",
.authority = isAbsolute(path) ? std::optional{ParsedURL::Authority{}} : std::nullopt,
.authority = ParsedURL::Authority{},
.path = splitString<std::vector<std::string>>(path, "/"),
.query = query,
.fragment = fragment,
+15 -1
View File
@@ -13,6 +13,8 @@
namespace nix {
int handleExceptions(const std::string & programName, std::function<void()> fun);
/**
* Don't forget to call initPlugins() after settings are initialized!
* @param loadConfig Whether to load configuration from `nix.conf`, `NIX_CONFIG`, etc. May be disabled for unit tests.
@@ -89,7 +91,19 @@ extern volatile ::sig_atomic_t blockInt;
struct GCResults;
void printFreed(bool dryRun, const GCResults & results);
struct PrintFreed
{
bool show;
const GCResults & results;
PrintFreed(bool show, const GCResults & results)
: show(show)
, results(results)
{
}
~PrintFreed();
};
#ifndef _WIN32
/**
+37 -41
View File
@@ -1,9 +1,7 @@
#include "nix/store/globals.hh"
#include "nix/util/current-process.hh"
#include "nix/util/executable-path.hh"
#include "nix/main/shared.hh"
#include "nix/store/store-api.hh"
#include "nix/store/store-open.hh"
#include "nix/store/gc-store.hh"
#include "nix/main/loggers.hh"
#include "nix/main/progress-bar.hh"
@@ -19,10 +17,6 @@
#include <sys/stat.h>
#include <unistd.h>
#include <signal.h>
#ifndef _WIN32
# include <sys/resource.h>
#endif
#ifdef __linux__
# include <features.h>
#endif
@@ -122,26 +116,6 @@ std::string getArg(const std::string & opt, Strings::iterator & i, const Strings
static void sigHandler(int signo) {}
#endif
/**
* Increase the open file soft limit to the hard limit. On some
* platforms (macOS), the default soft limit is very low, but the hard
* limit is high. So let's just raise it the maximum permitted.
*/
void bumpFileLimit()
{
#ifndef _WIN32
struct rlimit limit;
if (getrlimit(RLIMIT_NOFILE, &limit) != 0)
return;
if (limit.rlim_cur < limit.rlim_max) {
limit.rlim_cur = limit.rlim_max;
// Ignore errors, this is best effort.
setrlimit(RLIMIT_NOFILE, &limit);
}
#endif
}
void initNix(bool loadConfig)
{
/* Turn on buffering for cerr. */
@@ -209,8 +183,6 @@ void initNix(bool loadConfig)
now. In particular, store objects should be readable by
everybody. */
umask(0022);
bumpFileLimit();
}
LegacyArgs::LegacyArgs(
@@ -237,13 +209,13 @@ LegacyArgs::LegacyArgs(
.longName = "keep-going",
.shortName = 'k',
.description = "Keep going after a build fails.",
.handler = {&(bool &) settings.getWorkerSettings().keepGoing, true},
.handler = {&(bool &) settings.keepGoing, true},
});
addFlag({
.longName = "fallback",
.description = "Build from source if substitution fails.",
.handler = {&(bool &) settings.getWorkerSettings().tryFallback, true},
.handler = {&(bool &) settings.tryFallback, true},
});
auto intSettingAlias =
@@ -280,7 +252,7 @@ LegacyArgs::LegacyArgs(
.longName = "store",
.description = "The URL of the Nix store to use.",
.labels = {"store-uri"},
.handler = {[](std::string s) { settings.storeUri = StoreReference::parse(s); }},
.handler = {&(std::string &) settings.storeUri},
});
}
@@ -332,16 +304,43 @@ void printVersion(const std::string & programName)
std::cout << "System type: " << settings.thisSystem << "\n";
std::cout << "Additional system types: " << concatStringsSep(", ", settings.extraPlatforms.get()) << "\n";
std::cout << "Features: " << concatStringsSep(", ", cfg) << "\n";
std::cout << "System configuration file: " << nixConfFile() << "\n";
std::cout << "User configuration files: "
<< os_string_to_string(ExecutablePath{.directories = nixUserConfFiles()}.render()) << "\n";
std::cout << "Store directory: " << resolveStoreConfig(StoreReference{settings.storeUri.get()})->storeDir
<< "\n";
std::cout << "System configuration file: " << (settings.nixConfDir / "nix.conf").string() << "\n";
std::cout << "User configuration files: " << concatStringsSep(":", settings.nixUserConfFiles) << "\n";
std::cout << "Store directory: " << settings.nixStore << "\n";
std::cout << "State directory: " << settings.nixStateDir << "\n";
}
throw Exit();
}
int handleExceptions(const std::string & programName, std::function<void()> fun)
{
ReceiveInterrupts receiveInterrupts; // FIXME: need better place for this
ErrorInfo::programName = baseNameOf(programName);
std::string error = ANSI_RED "error:" ANSI_NORMAL " ";
try {
fun();
} catch (Exit & e) {
return e.status;
} catch (UsageError & e) {
logError(e.info());
printError("Try '%1% --help' for more information.", programName);
return 1;
} catch (BaseError & e) {
logError(e.info());
return e.info().status;
} catch (std::bad_alloc & e) {
printError(error + "out of memory");
return 1;
} catch (std::exception & e) {
printError(error + e.what());
return 1;
}
return 0;
}
RunPager::RunPager()
{
if (!isatty(STDOUT_FILENO))
@@ -396,12 +395,9 @@ RunPager::~RunPager()
}
}
void printFreed(bool dryRun, const GCResults & results)
PrintFreed::~PrintFreed()
{
/* bytesFreed cannot be reliably computed without actually deleting store paths because of hardlinking. */
if (dryRun)
std::cout << fmt("%d store paths would be deleted\n", results.paths.size());
else
if (show)
std::cout << fmt("%d store paths deleted, %s freed\n", results.paths.size(), renderSize(results.bytesFreed));
}
+11 -37
View File
@@ -9,7 +9,6 @@
#include "nix/store/path.hh"
#include "nix/store/store-api.hh"
#include "nix/store/store-open.hh"
#include "nix/store/store-reference.hh"
#include "nix/store/build-result.hh"
#include "nix/store/local-fs-store.hh"
#include "nix/util/base-nix-32.hh"
@@ -48,14 +47,14 @@ Store * nix_store_open(nix_c_context * context, const char * uri, const char ***
if (uri_str.empty())
return new Store{nix::openStore()};
auto storeRef = nix::StoreReference::parse(uri_str);
if (!params)
return new Store{nix::openStore(uri_str)};
if (params) {
for (size_t i = 0; params[i] != nullptr; i++) {
storeRef.params[params[i][0]] = params[i][1];
}
nix::Store::Config::Params params_map;
for (size_t i = 0; params[i] != nullptr; i++) {
params_map[params[i][0]] = params[i][1];
}
return new Store{nix::openStore(std::move(storeRef))};
return new Store{nix::openStore(uri_str, params_map)};
}
NIXC_CATCH_ERRS_NULL
}
@@ -183,8 +182,10 @@ nix_err nix_store_realise(
assert(results.size() == 1);
// Check if any builds failed
for (auto & result : results)
result.tryThrowBuildError();
for (auto & result : results) {
if (auto * failureP = result.tryGetFailure())
failureP->rethrow();
}
if (callback) {
for (const auto & result : results) {
@@ -308,12 +309,7 @@ StorePath * nix_add_derivation(nix_c_context * context, Store * store, nix_deriv
if (context)
context->last_err_code = NIX_OK;
try {
/* Quite dubious that users would want this to silently suceed
without actually writing the derivation if this setting is
set, but it was that way already, so we are doing this for
back-compat for now. */
auto ret = nix::settings.readOnlyMode ? nix::computeStorePath(*store->ptr, derivation->drv)
: store->ptr->writeDerivation(derivation->drv, nix::NoRepair);
auto ret = nix::writeDerivation(*store->ptr, derivation->drv, nix::NoRepair);
return new StorePath{ret};
}
@@ -358,26 +354,4 @@ StorePath * nix_store_query_path_from_hash_part(nix_c_context * context, Store *
NIXC_CATCH_ERRS_NULL
}
nix_err nix_store_copy_path(
nix_c_context * context, Store * srcStore, Store * dstStore, const StorePath * path, bool repair, bool checkSigs)
{
if (context)
context->last_err_code = NIX_OK;
try {
if (srcStore == nullptr)
return nix_set_err_msg(context, NIX_ERR_UNKNOWN, "Source store is null");
if (dstStore == nullptr)
return nix_set_err_msg(context, NIX_ERR_UNKNOWN, "Destination store is null");
if (path == nullptr)
return nix_set_err_msg(context, NIX_ERR_UNKNOWN, "Store path is null");
auto repairFlag = repair ? nix::RepairFlag::Repair : nix::RepairFlag::NoRepair;
auto checkSigsFlag = checkSigs ? nix::CheckSigsFlag::CheckSigs : nix::CheckSigsFlag::NoCheckSigs;
nix::copyStorePath(*srcStore->ptr, *dstStore->ptr, path->path, repairFlag, checkSigsFlag);
}
NIXC_CATCH_ERRS
}
} // extern "C"
-13
View File
@@ -270,19 +270,6 @@ nix_derivation * nix_store_drv_from_store_path(nix_c_context * context, Store *
*/
StorePath * nix_store_query_path_from_hash_part(nix_c_context * context, Store * store, const char * hash);
/**
* @brief Copy a path from one store to another.
*
* @param[out] context Optional, stores error information
* @param[in] srcStore nix source store reference
* @param[in] dstStore nix destination store reference
* @param[in] path The path to copy
* @param[in] repair Whether to repair the path
* @param[in] checkSigs Whether to check path signatures are trusted before copying
*/
nix_err nix_store_copy_path(
nix_c_context * context, Store * srcStore, Store * dstStore, const StorePath * path, bool repair, bool checkSigs);
// cffi end
#ifdef __cplusplus
}
+10 -29
View File
@@ -9,12 +9,11 @@ void TestHttpBinaryCacheStore::init()
BinaryCacheStore::init();
}
ref<TestHttpBinaryCacheStore> TestHttpBinaryCacheStoreConfig::openTestStore(ref<FileTransfer> fileTransfer) const
ref<TestHttpBinaryCacheStore> TestHttpBinaryCacheStoreConfig::openTestStore() const
{
auto store = make_ref<TestHttpBinaryCacheStore>(
ref{// FIXME we shouldn't actually need a mutable config
std::const_pointer_cast<HttpBinaryCacheStore::Config>(shared_from_this())},
fileTransfer);
std::const_pointer_cast<HttpBinaryCacheStore::Config>(shared_from_this())});
store->init();
return store;
}
@@ -50,11 +49,9 @@ void HttpsBinaryCacheStoreTest::SetUp()
// clang-format off
openssl({"ecparam", "-genkey", "-name", "prime256v1", "-out", caKey.string()});
openssl({"req", "-new", "-x509", "-days", "1", "-key", caKey.string(), "-out", caCert.string(), "-subj", "/CN=TestCA"});
auto serverExtFile = tmpDir / "server.ext";
writeFile(serverExtFile, "subjectAltName=DNS:localhost,IP:127.0.0.1");
openssl({"ecparam", "-genkey", "-name", "prime256v1", "-out", serverKey.string()});
openssl({"req", "-new", "-key", serverKey.string(), "-out", (tmpDir / "server.csr").string(), "-subj", "/CN=localhost", "-addext", "subjectAltName=DNS:localhost,IP:127.0.0.1"});
openssl({"x509", "-req", "-in", (tmpDir / "server.csr").string(), "-CA", caCert.string(), "-CAkey", caKey.string(), "-CAcreateserial", "-out", serverCert.string(), "-days", "1", "-extfile", serverExtFile.string()});
openssl({"req", "-new", "-key", serverKey.string(), "-out", (tmpDir / "server.csr").string(), "-subj", "/CN=localhost"});
openssl({"x509", "-req", "-in", (tmpDir / "server.csr").string(), "-CA", caCert.string(), "-CAkey", caKey.string(), "-CAcreateserial", "-out", serverCert.string(), "-days", "1"});
openssl({"ecparam", "-genkey", "-name", "prime256v1", "-out", clientKey.string()});
openssl({"req", "-new", "-key", clientKey.string(), "-out", (tmpDir / "client.csr").string(), "-subj", "/CN=TestClient"});
openssl({"x509", "-req", "-in", (tmpDir / "client.csr").string(), "-CA", caCert.string(), "-CAkey", caKey.string(), "-CAcreateserial", "-out", clientCert.string(), "-days", "1"});
@@ -82,18 +79,16 @@ void HttpsBinaryCacheStoreTest::SetUp()
for the port explicitly - this is enough. */
std::this_thread::sleep_for(std::chrono::milliseconds(50));
/* Create custom FileTransferSettings with our test CA certificate.
This avoids mutating global settings. */
testFileTransferSettings = std::make_unique<FileTransferSettings>();
testFileTransferSettings->caFile = caCert;
testFileTransfer = makeFileTransfer(*testFileTransferSettings);
/* FIXME: Don't use global settings. Tests are not run concurrently, so this is fine for now. */
oldCaCert = settings.caFile;
settings.caFile = caCert.string();
}
void HttpsBinaryCacheStoreTest::TearDown()
{
settings.caFile = oldCaCert;
serverPid.kill();
delTmpDir.reset();
testFileTransferSettings.reset();
}
std::vector<std::string> HttpsBinaryCacheStoreTest::serverArgs()
@@ -120,25 +115,11 @@ std::vector<std::string> HttpsBinaryCacheStoreMtlsTest::serverArgs()
return args;
}
ref<TestHttpBinaryCacheStoreConfig> HttpsBinaryCacheStoreTest::makeConfig()
ref<TestHttpBinaryCacheStoreConfig> HttpsBinaryCacheStoreTest::makeConfig(BinaryCacheStoreConfig::Params params)
{
auto res = make_ref<TestHttpBinaryCacheStoreConfig>(
ParsedURL{
.scheme = "https",
.authority =
ParsedURL::Authority{
.host = "localhost",
.port = port,
},
},
TestHttpBinaryCacheStoreConfig::Params{});
auto res = make_ref<TestHttpBinaryCacheStoreConfig>("https", fmt("localhost:%d", port), std::move(params));
res->pathInfoCacheSize = 0; /* We don't want any caching in tests. */
return res;
}
ref<TestHttpBinaryCacheStore> HttpsBinaryCacheStoreTest::openStore(ref<TestHttpBinaryCacheStoreConfig> config)
{
return config->openTestStore(ref<FileTransfer>{testFileTransfer});
}
} // namespace nix::testing
@@ -28,10 +28,10 @@ public:
TestHttpBinaryCacheStore & operator=(const TestHttpBinaryCacheStore &) = delete;
TestHttpBinaryCacheStore & operator=(TestHttpBinaryCacheStore &&) = delete;
TestHttpBinaryCacheStore(ref<HttpBinaryCacheStoreConfig> config, ref<FileTransfer> fileTransfer)
TestHttpBinaryCacheStore(ref<HttpBinaryCacheStoreConfig> config)
: Store{*config}
, BinaryCacheStore{*config}
, HttpBinaryCacheStore(config, fileTransfer)
, HttpBinaryCacheStore(config)
{
diskCache = nullptr; /* Disable caching, we'll be creating a new binary cache for each test. */
}
@@ -42,13 +42,19 @@ public:
class TestHttpBinaryCacheStoreConfig : public HttpBinaryCacheStoreConfig
{
public:
TestHttpBinaryCacheStoreConfig(ParsedURL url, const Store::Config::Params & params)
TestHttpBinaryCacheStoreConfig(
std::string_view scheme, std::string_view cacheUri, const Store::Config::Params & params)
: StoreConfig(params)
, HttpBinaryCacheStoreConfig(url, params)
, HttpBinaryCacheStoreConfig(scheme, cacheUri, params)
{
}
ref<TestHttpBinaryCacheStore> openTestStore(ref<FileTransfer> fileTransfer) const;
ref<TestHttpBinaryCacheStore> openTestStore() const;
ref<Store> openStore() const override
{
return openTestStore();
}
};
class HttpsBinaryCacheStoreTest : public virtual LibStoreNetworkTest
@@ -65,29 +71,17 @@ protected:
std::filesystem::path tmpDir, cacheDir;
std::filesystem::path caCert, caKey, serverCert, serverKey;
std::filesystem::path clientCert, clientKey;
std::string oldCaCert;
Pid serverPid;
uint16_t port = 8443;
std::shared_ptr<Store> localCacheStore;
/**
* Custom FileTransferSettings with the test CA certificate.
* This is used instead of modifying global settings.
*/
std::unique_ptr<FileTransferSettings> testFileTransferSettings;
/**
* FileTransfer instance using our test settings.
* Initialized in SetUp().
*/
std::shared_ptr<FileTransfer> testFileTransfer;
static void openssl(Strings args);
void SetUp() override;
void TearDown() override;
virtual std::vector<std::string> serverArgs();
ref<TestHttpBinaryCacheStoreConfig> makeConfig();
ref<TestHttpBinaryCacheStore> openStore(ref<TestHttpBinaryCacheStoreConfig> config);
ref<TestHttpBinaryCacheStoreConfig> makeConfig(BinaryCacheStoreConfig::Params params);
};
class HttpsBinaryCacheStoreMtlsTest : public HttpsBinaryCacheStoreTest
@@ -100,26 +100,26 @@ public:
writeProtoTest(STEM, VERSION, VALUE); \
}
#define VERSIONED_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE) \
VERSIONED_READ_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, (VERSION), VALUE) \
VERSIONED_WRITE_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, (VERSION), VALUE)
#define VERSIONED_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE) \
VERSIONED_READ_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE) \
VERSIONED_WRITE_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE)
#define VERSIONED_READ_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE) \
VERSIONED_READ_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, (VERSION), VALUE) \
TEST_F(FIXTURE, NAME##_json_read) \
{ \
readJsonTest(STEM, VALUE); \
#define VERSIONED_READ_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE) \
VERSIONED_READ_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE) \
TEST_F(FIXTURE, NAME##_json_read) \
{ \
readJsonTest(STEM, VALUE); \
}
#define VERSIONED_WRITE_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE) \
VERSIONED_WRITE_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, (VERSION), VALUE) \
TEST_F(FIXTURE, NAME##_json_write) \
{ \
writeJsonTest(STEM, VALUE); \
#define VERSIONED_WRITE_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE) \
VERSIONED_WRITE_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE) \
TEST_F(FIXTURE, NAME##_json_write) \
{ \
writeJsonTest(STEM, VALUE); \
}
#define VERSIONED_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE) \
VERSIONED_READ_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, (VERSION), VALUE) \
VERSIONED_WRITE_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, (VERSION), VALUE)
#define VERSIONED_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE) \
VERSIONED_READ_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE) \
VERSIONED_WRITE_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE)
} // namespace nix
+4 -4
View File
@@ -15,10 +15,10 @@ int testMainForBuidingPre(int argc, char ** argv)
}
// Disable build hook. We won't be testing remote builds in these unit tests. If we do, fix the above build hook.
settings.getWorkerSettings().buildHook = {};
settings.buildHook = {};
// No substituters, unless a test specifically requests.
settings.getWorkerSettings().substituters = {};
settings.substituters = {};
#ifdef __linux__ // should match the conditional around sandboxBuildDir declaration.
@@ -31,13 +31,13 @@ int testMainForBuidingPre(int argc, char ** argv)
// sandboxBuildDir = /build
// However, we have a rule that the store dir must not be inside the storeDir, so we need to pick a different
// sandboxBuildDir.
settings.getLocalSettings().sandboxBuildDir = "/test-build-dir-instead-of-usual-build-dir";
settings.sandboxBuildDir = "/test-build-dir-instead-of-usual-build-dir";
#endif
#ifdef __APPLE__
// Avoid this error, when already running in a sandbox:
// sandbox-exec: sandbox_apply: Operation not permitted
settings.getLocalSettings().sandboxMode = smDisabled;
settings.sandboxMode = smDisabled;
setEnv("_NIX_TEST_NO_SANDBOX", "1");
#endif
+6 -7
View File
@@ -1,7 +1,6 @@
#include <gtest/gtest.h>
#include "nix/store/build-result.hh"
#include "nix/util/tests/characterization.hh"
#include "nix/util/tests/json-characterization.hh"
namespace nix {
@@ -45,22 +44,22 @@ INSTANTIATE_TEST_SUITE_P(
std::pair{
"not-deterministic",
BuildResult{
.inner{BuildResult::Failure{{
.inner{BuildResult::Failure{
.status = BuildResult::Failure::NotDeterministic,
.msg = HintFmt("no idea why"),
.errorMsg = "no idea why",
.isNonDeterministic = false, // Note: This field is separate from the status
}}},
}},
.timesBuilt = 1,
},
},
std::pair{
"output-rejected",
BuildResult{
.inner{BuildResult::Failure{{
.inner{BuildResult::Failure{
.status = BuildResult::Failure::OutputRejected,
.msg = HintFmt("no idea why"),
.errorMsg = "no idea why",
.isNonDeterministic = false,
}}},
}},
.timesBuilt = 3,
.startTime = 30,
.stopTime = 50,
@@ -5,7 +5,6 @@
#include "nix/store/derivations.hh"
#include "nix/store/derived-path.hh"
#include "nix/store/derivation-options.hh"
#include "nix/store/globals.hh"
#include "nix/store/parsed-derivations.hh"
#include "nix/util/types.hh"
#include "nix/util/json-utils.hh"
@@ -193,7 +192,9 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_defaults)
EXPECT_EQ(options, advancedAttributes_defaults);
EXPECT_EQ(options.substitutesAllowed(settings.getWorkerSettings()), true);
EXPECT_EQ(options.canBuildLocally(*this->store, got), false);
EXPECT_EQ(options.willBuildLocally(*this->store, got), false);
EXPECT_EQ(options.substitutesAllowed(), true);
EXPECT_EQ(options.useUidRange(got), false);
});
};
@@ -241,7 +242,7 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes)
EXPECT_EQ(options, expected);
EXPECT_EQ(options.substitutesAllowed(settings.getWorkerSettings()), false);
EXPECT_EQ(options.substitutesAllowed(), false);
EXPECT_EQ(options.useUidRange(got), true);
});
};
@@ -333,7 +334,9 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_structuredAttrs_d
EXPECT_EQ(options, advancedAttributes_structuredAttrs_defaults);
EXPECT_EQ(options.substitutesAllowed(settings.getWorkerSettings()), true);
EXPECT_EQ(options.canBuildLocally(*this->store, got), false);
EXPECT_EQ(options.willBuildLocally(*this->store, got), false);
EXPECT_EQ(options.substitutesAllowed(), true);
EXPECT_EQ(options.useUidRange(got), false);
});
};
@@ -398,7 +401,9 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_structuredAttrs)
EXPECT_EQ(options, expected);
EXPECT_EQ(options.substitutesAllowed(settings.getWorkerSettings()), false);
EXPECT_EQ(options.canBuildLocally(*this->store, got), false);
EXPECT_EQ(options.willBuildLocally(*this->store, got), false);
EXPECT_EQ(options.substitutesAllowed(), false);
EXPECT_EQ(options.useUidRange(got), true);
});
};
+1 -1
View File
@@ -51,7 +51,7 @@ protected:
depDrv.fillInOutputPaths(*store);
// Write the dependency to the store
return store->writeDerivation(depDrv, NoRepair);
return writeDerivation(*store, depDrv, NoRepair);
}
public:
+31 -17
View File
@@ -45,18 +45,28 @@ using namespace std::string_literals;
TEST_F(HttpsBinaryCacheStoreTest, queryPathInfo)
{
auto store = openStore(makeConfig());
auto config = makeConfig({});
auto store = config->openStore();
StringSource dump{"test"sv};
auto path = localCacheStore->addToStoreFromDump(dump, "test-name", FileSerialisationMethod::Flat);
EXPECT_NO_THROW(store->queryPathInfo(path));
}
auto withNoRetries()
{
auto oldTries = fileTransferSettings.tries.get();
Finally restoreTries = [=]() { fileTransferSettings.tries = oldTries; };
fileTransferSettings.tries = 1; /* FIXME: Don't use global settings. */
return restoreTries;
}
TEST_F(HttpsBinaryCacheStoreMtlsTest, queryPathInfo)
{
auto config = makeConfig();
config->tlsCert = clientCert;
config->tlsKey = clientKey;
auto store = openStore(config);
auto config = makeConfig({
{"tls-certificate"s, clientCert.string()},
{"tls-private-key"s, clientKey.string()},
});
auto store = config->openStore();
StringSource dump{"test"sv};
auto path = localCacheStore->addToStoreFromDump(dump, "test-name", FileSerialisationMethod::Flat);
EXPECT_NO_THROW(store->queryPathInfo(path));
@@ -64,8 +74,9 @@ TEST_F(HttpsBinaryCacheStoreMtlsTest, queryPathInfo)
TEST_F(HttpsBinaryCacheStoreMtlsTest, rejectsWithoutClientCert)
{
testFileTransferSettings->tries = 1;
EXPECT_THROW(openStore(makeConfig()), Error);
auto restoreTries = withNoRetries();
auto config = makeConfig({});
EXPECT_THROW(config->openStore(), Error);
}
TEST_F(HttpsBinaryCacheStoreMtlsTest, rejectsWrongClientCert)
@@ -78,11 +89,12 @@ TEST_F(HttpsBinaryCacheStoreMtlsTest, rejectsWrongClientCert)
openssl({"req", "-new", "-x509", "-days", "1", "-key", wrongKey.string(), "-out", wrongCert.string(), "-subj", "/CN=WrongClient"});
// clang-format on
auto config = makeConfig();
config->tlsCert = wrongCert;
config->tlsKey = wrongKey;
testFileTransferSettings->tries = 1;
EXPECT_THROW(openStore(config), Error);
auto config = makeConfig({
{"tls-certificate"s, wrongCert.string()},
{"tls-private-key"s, wrongKey.string()},
});
auto restoreTries = withNoRetries();
EXPECT_THROW(config->openStore(), Error);
}
TEST_F(HttpsBinaryCacheStoreMtlsTest, doesNotSendCertOnRedirectToDifferentAuthority)
@@ -97,11 +109,13 @@ TEST_F(HttpsBinaryCacheStoreMtlsTest, doesNotSendCertOnRedirectToDifferentAuthor
writeFile(entry.path(), content);
}
auto config = makeConfig();
config->tlsCert = clientCert;
config->tlsKey = clientKey;
testFileTransferSettings->tries = 1;
auto store = openStore(config);
auto config = makeConfig({
{"tls-certificate"s, clientCert.string()},
{"tls-private-key"s, clientKey.string()},
});
auto store = config->openStore();
auto restoreTries = withNoRetries();
auto info = store->queryPathInfo(path);
NullSink null;
EXPECT_THROW(store->narFromPath(path, null), Error);
+2 -4
View File
@@ -25,8 +25,7 @@ TEST(NarInfoDiskCacheImpl, create_and_read)
SQLiteStmt getIds;
{
auto cache = NarInfoDiskCache::getTest(
settings.getNarInfoDiskCacheSettings(), {.useWAL = settings.useSQLiteWAL}, dbPath.string());
auto cache = getTestNarInfoDiskCache(dbPath.string());
// Set up "background noise" and check that different caches receive different ids
{
@@ -75,8 +74,7 @@ TEST(NarInfoDiskCacheImpl, create_and_read)
{
// We can't clear the in-memory cache, so we use a new cache object. This is
// more realistic anyway.
auto cache2 = NarInfoDiskCache::getTest(
settings.getNarInfoDiskCacheSettings(), {.useWAL = settings.useSQLiteWAL}, dbPath.string());
auto cache2 = getTestNarInfoDiskCache(dbPath.string());
{
auto r = cache2->upToDateCacheExists("http://foo");
+6 -6
View File
@@ -299,7 +299,7 @@ public:
nix_api_store_test_base::SetUp();
nix::experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations");
nix::settings.getWorkerSettings().substituters = {};
nix::settings.substituters = {};
store = open_local_store();
@@ -354,7 +354,7 @@ TEST_F(nix_api_store_test_base, build_from_json)
{
// FIXME get rid of these
nix::experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations");
nix::settings.getWorkerSettings().substituters = {};
nix::settings.substituters = {};
auto * store = open_local_store();
@@ -401,7 +401,7 @@ TEST_F(nix_api_store_test_base, nix_store_realise_invalid_system)
{
// Test that nix_store_realise properly reports errors when the system is invalid
nix::experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations");
nix::settings.getWorkerSettings().substituters = {};
nix::settings.substituters = {};
auto * store = open_local_store();
@@ -446,7 +446,7 @@ TEST_F(nix_api_store_test_base, nix_store_realise_builder_fails)
{
// Test that nix_store_realise properly reports errors when the builder fails
nix::experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations");
nix::settings.getWorkerSettings().substituters = {};
nix::settings.substituters = {};
auto * store = open_local_store();
@@ -491,7 +491,7 @@ TEST_F(nix_api_store_test_base, nix_store_realise_builder_no_output)
{
// Test that nix_store_realise properly reports errors when builder succeeds but produces no output
nix::experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations");
nix::settings.getWorkerSettings().substituters = {};
nix::settings.substituters = {};
auto * store = open_local_store();
@@ -687,7 +687,7 @@ TEST_F(NixApiStoreTestWithRealisedPath, nix_store_realise_output_ordering)
// This test uses a CA derivation with 10 outputs in randomized input order
// to verify that the callback order is deterministic and alphabetical.
nix::experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations");
nix::settings.getWorkerSettings().substituters = {};
nix::settings.substituters = {};
auto * store = open_local_store();
+15 -237
View File
@@ -104,33 +104,6 @@ INSTANTIATE_TEST_SUITE_P(
},
},
"with_absolute_endpoint_uri",
},
ParsedS3URLTestCase{
"s3://bucket/key?addressing-style=virtual",
{
.bucket = "bucket",
.key = {"key"},
.addressingStyle = S3AddressingStyle::Virtual,
},
"with_addressing_style_virtual",
},
ParsedS3URLTestCase{
"s3://bucket/key?addressing-style=path",
{
.bucket = "bucket",
.key = {"key"},
.addressingStyle = S3AddressingStyle::Path,
},
"with_addressing_style_path",
},
ParsedS3URLTestCase{
"s3://bucket/key?addressing-style=auto",
{
.bucket = "bucket",
.key = {"key"},
.addressingStyle = S3AddressingStyle::Auto,
},
"with_addressing_style_auto",
}),
[](const ::testing::TestParamInfo<ParsedS3URLTestCase> & info) { return info.param.description; });
@@ -165,26 +138,6 @@ INSTANTIATE_TEST_SUITE_P(
InvalidS3URLTestCase{"s3://bucket", "error: URI has a missing or invalid key", "missing_key"}),
[](const ::testing::TestParamInfo<InvalidS3URLTestCase> & info) { return info.param.description; });
TEST(ParsedS3URLTest, invalidAddressingStyleThrows)
{
ASSERT_THROW(ParsedS3URL::parse(parseURL("s3://bucket/key?addressing-style=bogus")), InvalidS3AddressingStyle);
}
TEST(ParsedS3URLTest, virtualStyleWithAuthoritylessEndpointThrows)
{
ParsedS3URL input{
.bucket = "bucket",
.key = {"key"},
.addressingStyle = S3AddressingStyle::Virtual,
.endpoint =
ParsedURL{
.scheme = "file",
.path = {"", "some", "path"},
},
};
ASSERT_THROW(input.toHttpsUrl(), nix::Error);
}
// =============================================================================
// S3 URL to HTTPS Conversion Tests
// =============================================================================
@@ -213,7 +166,6 @@ INSTANTIATE_TEST_SUITE_P(
S3ToHttpsConversion,
S3ToHttpsConversionTest,
::testing::Values(
// Default (auto) addressing style: virtual-hosted for standard AWS endpoints
S3ToHttpsConversionTestCase{
ParsedS3URL{
.bucket = "my-bucket",
@@ -221,10 +173,10 @@ INSTANTIATE_TEST_SUITE_P(
},
ParsedURL{
.scheme = "https",
.authority = ParsedURL::Authority{.host = "my-bucket.s3.us-east-1.amazonaws.com"},
.path = {"", "my-key.txt"},
.authority = ParsedURL::Authority{.host = "s3.us-east-1.amazonaws.com"},
.path = {"", "my-bucket", "my-key.txt"},
},
"https://my-bucket.s3.us-east-1.amazonaws.com/my-key.txt",
"https://s3.us-east-1.amazonaws.com/my-bucket/my-key.txt",
"basic_s3_default_region",
},
S3ToHttpsConversionTestCase{
@@ -235,13 +187,12 @@ INSTANTIATE_TEST_SUITE_P(
},
ParsedURL{
.scheme = "https",
.authority = ParsedURL::Authority{.host = "prod-cache.s3.eu-west-1.amazonaws.com"},
.path = {"", "nix", "store", "abc123.nar.xz"},
.authority = ParsedURL::Authority{.host = "s3.eu-west-1.amazonaws.com"},
.path = {"", "prod-cache", "nix", "store", "abc123.nar.xz"},
},
"https://prod-cache.s3.eu-west-1.amazonaws.com/nix/store/abc123.nar.xz",
"https://s3.eu-west-1.amazonaws.com/prod-cache/nix/store/abc123.nar.xz",
"with_eu_west_1_region",
},
// Custom endpoint authority: path-style by default
S3ToHttpsConversionTestCase{
ParsedS3URL{
.bucket = "bucket",
@@ -257,7 +208,6 @@ INSTANTIATE_TEST_SUITE_P(
"http://custom.s3.com/bucket/key",
"custom_endpoint_authority",
},
// Custom endpoint URL: path-style by default
S3ToHttpsConversionTestCase{
ParsedS3URL{
.bucket = "bucket",
@@ -286,10 +236,10 @@ INSTANTIATE_TEST_SUITE_P(
},
ParsedURL{
.scheme = "https",
.authority = ParsedURL::Authority{.host = "bucket.s3.ap-southeast-2.amazonaws.com"},
.path = {"", "path", "to", "file.txt"},
.authority = ParsedURL::Authority{.host = "s3.ap-southeast-2.amazonaws.com"},
.path = {"", "bucket", "path", "to", "file.txt"},
},
"https://bucket.s3.ap-southeast-2.amazonaws.com/path/to/file.txt",
"https://s3.ap-southeast-2.amazonaws.com/bucket/path/to/file.txt",
"complex_path_and_region",
},
S3ToHttpsConversionTestCase{
@@ -300,11 +250,11 @@ INSTANTIATE_TEST_SUITE_P(
},
ParsedURL{
.scheme = "https",
.authority = ParsedURL::Authority{.host = "my-bucket.s3.us-east-1.amazonaws.com"},
.path = {"", "my-key.txt"},
.authority = ParsedURL::Authority{.host = "s3.us-east-1.amazonaws.com"},
.path = {"", "my-bucket", "my-key.txt"},
.query = {{"versionId", "abc123xyz"}},
},
"https://my-bucket.s3.us-east-1.amazonaws.com/my-key.txt?versionId=abc123xyz",
"https://s3.us-east-1.amazonaws.com/my-bucket/my-key.txt?versionId=abc123xyz",
"with_versionId",
},
S3ToHttpsConversionTestCase{
@@ -316,185 +266,13 @@ INSTANTIATE_TEST_SUITE_P(
},
ParsedURL{
.scheme = "https",
.authority = ParsedURL::Authority{.host = "versioned-bucket.s3.eu-west-1.amazonaws.com"},
.path = {"", "path", "to", "object"},
.authority = ParsedURL::Authority{.host = "s3.eu-west-1.amazonaws.com"},
.path = {"", "versioned-bucket", "path", "to", "object"},
.query = {{"versionId", "version456"}},
},
"https://versioned-bucket.s3.eu-west-1.amazonaws.com/path/to/object?versionId=version456",
"https://s3.eu-west-1.amazonaws.com/versioned-bucket/path/to/object?versionId=version456",
"with_region_and_versionId",
},
// Explicit addressing-style=path forces path-style on standard AWS endpoints
S3ToHttpsConversionTestCase{
ParsedS3URL{
.bucket = "my-bucket",
.key = {"my-key.txt"},
.region = "us-west-2",
.addressingStyle = S3AddressingStyle::Path,
},
ParsedURL{
.scheme = "https",
.authority = ParsedURL::Authority{.host = "s3.us-west-2.amazonaws.com"},
.path = {"", "my-bucket", "my-key.txt"},
},
"https://s3.us-west-2.amazonaws.com/my-bucket/my-key.txt",
"explicit_path_style",
},
// Explicit addressing-style=virtual forces virtual-hosted-style on custom endpoints
S3ToHttpsConversionTestCase{
ParsedS3URL{
.bucket = "bucket",
.key = {"key"},
.scheme = "http",
.addressingStyle = S3AddressingStyle::Virtual,
.endpoint = ParsedURL::Authority{.host = "custom.s3.com"},
},
ParsedURL{
.scheme = "http",
.authority = ParsedURL::Authority{.host = "bucket.custom.s3.com"},
.path = {"", "key"},
},
"http://bucket.custom.s3.com/key",
"explicit_virtual_style_custom_endpoint",
},
// Explicit addressing-style=virtual with full endpoint URL
S3ToHttpsConversionTestCase{
ParsedS3URL{
.bucket = "bucket",
.key = {"key"},
.addressingStyle = S3AddressingStyle::Virtual,
.endpoint =
ParsedURL{
.scheme = "http",
.authority = ParsedURL::Authority{.host = "server", .port = 9000},
.path = {""},
},
},
ParsedURL{
.scheme = "http",
.authority = ParsedURL::Authority{.host = "bucket.server", .port = 9000},
.path = {"", "key"},
},
"http://bucket.server:9000/key",
"explicit_virtual_style_full_endpoint_url",
},
// Dotted bucket names work normally with explicit path-style
S3ToHttpsConversionTestCase{
ParsedS3URL{
.bucket = "my.bucket",
.key = {"key"},
.addressingStyle = S3AddressingStyle::Path,
},
ParsedURL{
.scheme = "https",
.authority = ParsedURL::Authority{.host = "s3.us-east-1.amazonaws.com"},
.path = {"", "my.bucket", "key"},
},
"https://s3.us-east-1.amazonaws.com/my.bucket/key",
"dotted_bucket_with_path_style",
},
// Dotted bucket names fall back to path-style with auto on standard AWS endpoints
S3ToHttpsConversionTestCase{
ParsedS3URL{
.bucket = "my.bucket.name",
.key = {"key"},
},
ParsedURL{
.scheme = "https",
.authority = ParsedURL::Authority{.host = "s3.us-east-1.amazonaws.com"},
.path = {"", "my.bucket.name", "key"},
},
"https://s3.us-east-1.amazonaws.com/my.bucket.name/key",
"dotted_bucket_with_auto_style_on_aws",
},
// Dotted bucket names work with auto style on custom endpoints (auto = path-style)
S3ToHttpsConversionTestCase{
ParsedS3URL{
.bucket = "my.bucket",
.key = {"key"},
.endpoint = ParsedURL::Authority{.host = "minio.local"},
},
ParsedURL{
.scheme = "https",
.authority = ParsedURL::Authority{.host = "minio.local"},
.path = {"", "my.bucket", "key"},
},
"https://minio.local/my.bucket/key",
"dotted_bucket_with_auto_style_custom_endpoint",
}),
[](const ::testing::TestParamInfo<S3ToHttpsConversionTestCase> & info) { return info.param.description; });
// =============================================================================
// S3 URL to HTTPS Conversion Error Tests
// =============================================================================
struct S3ToHttpsConversionErrorTestCase
{
ParsedS3URL input;
std::string description;
};
class S3ToHttpsConversionErrorTest : public ::testing::WithParamInterface<S3ToHttpsConversionErrorTestCase>,
public ::testing::Test
{};
TEST_P(S3ToHttpsConversionErrorTest, ThrowsOnConversion)
{
auto & [input, description] = GetParam();
ASSERT_THROW(input.toHttpsUrl(), nix::Error);
}
INSTANTIATE_TEST_SUITE_P(
S3ToHttpsConversionErrors,
S3ToHttpsConversionErrorTest,
::testing::Values(
S3ToHttpsConversionErrorTestCase{
ParsedS3URL{
.bucket = "bucket",
.key = {"key"},
.addressingStyle = S3AddressingStyle::Virtual,
.endpoint = ParsedURL::Authority{.host = ""},
},
"virtual_style_with_empty_host_authority",
},
S3ToHttpsConversionErrorTestCase{
ParsedS3URL{
.bucket = "my.bucket",
.key = {"key"},
.addressingStyle = S3AddressingStyle::Virtual,
},
"dotted_bucket_with_explicit_virtual_style",
},
S3ToHttpsConversionErrorTestCase{
ParsedS3URL{
.bucket = "my.bucket.name",
.key = {"key"},
.addressingStyle = S3AddressingStyle::Virtual,
},
"dotted_bucket_with_explicit_virtual_style_multi_dot",
},
S3ToHttpsConversionErrorTestCase{
ParsedS3URL{
.bucket = "my.bucket",
.key = {"key"},
.addressingStyle = S3AddressingStyle::Virtual,
.endpoint = ParsedURL::Authority{.host = "minio.local"},
},
"dotted_bucket_with_explicit_virtual_style_custom_authority",
},
S3ToHttpsConversionErrorTestCase{
ParsedS3URL{
.bucket = "my.bucket",
.key = {"key"},
.addressingStyle = S3AddressingStyle::Virtual,
.endpoint =
ParsedURL{
.scheme = "http",
.authority = ParsedURL::Authority{.host = "minio.local", .port = 9000},
.path = {""},
},
},
"dotted_bucket_with_explicit_virtual_style_full_endpoint_url",
}),
[](const ::testing::TestParamInfo<S3ToHttpsConversionErrorTestCase> & info) { return info.param.description; });
} // namespace nix
+58 -102
View File
@@ -25,10 +25,7 @@ struct ServeProtoTest : VersionedProtoTest<ServeProto, serveProtoDir>
* For serializers that don't care about the minimum version, we
* used the oldest one: 2.5.
*/
ServeProto::Version defaultVersion = {
.major = 2,
.minor = 5,
};
ServeProto::Version defaultVersion = 2 << 8 | 5;
};
VERSIONED_CHARACTERIZATION_TEST(
@@ -147,89 +144,66 @@ VERSIONED_READ_CHARACTERIZATION_TEST(
},
}))
VERSIONED_CHARACTERIZATION_TEST(ServeProtoTest, buildResult_2_2, "build-result-2.2", 2 << 8 | 2, ({
using namespace std::literals::chrono_literals;
std::tuple<BuildResult, BuildResult, BuildResult> t{
BuildResult{.inner{BuildResult::Failure{
.status = BuildResult::Failure::OutputRejected,
.errorMsg = "no idea why",
}}},
BuildResult{.inner{BuildResult::Failure{
.status = BuildResult::Failure::NotDeterministic,
.errorMsg = "no idea why",
}}},
BuildResult{.inner{BuildResult::Success{
.status = BuildResult::Success::Built,
}}},
};
t;
}))
VERSIONED_CHARACTERIZATION_TEST(ServeProtoTest, buildResult_2_3, "build-result-2.3", 2 << 8 | 3, ({
using namespace std::literals::chrono_literals;
std::tuple<BuildResult, BuildResult, BuildResult> t{
BuildResult{.inner{BuildResult::Failure{
.status = BuildResult::Failure::OutputRejected,
.errorMsg = "no idea why",
}}},
BuildResult{
.inner{BuildResult::Failure{
.status = BuildResult::Failure::NotDeterministic,
.errorMsg = "no idea why",
.isNonDeterministic = true,
}},
.timesBuilt = 3,
.startTime = 30,
.stopTime = 50,
},
BuildResult{
.inner{BuildResult::Success{
.status = BuildResult::Success::Built,
}},
.startTime = 30,
.stopTime = 50,
},
};
t;
}))
VERSIONED_CHARACTERIZATION_TEST(
ServeProtoTest,
buildResult_2_2,
"build-result-2.2",
(ServeProto::Version{
.major = 2,
.minor = 2,
}),
({
ServeProtoTest, buildResult_2_6, "build-result-2.6", 2 << 8 | 6, ({
using namespace std::literals::chrono_literals;
std::tuple<BuildResult, BuildResult, BuildResult> t{
BuildResult{.inner{BuildResult::Failure{{
BuildResult{.inner{BuildResult::Failure{
.status = BuildResult::Failure::OutputRejected,
.msg = HintFmt("no idea why"),
}}}},
BuildResult{.inner{BuildResult::Failure{{
.status = BuildResult::Failure::NotDeterministic,
.msg = HintFmt("no idea why"),
}}}},
BuildResult{.inner{BuildResult::Success{
.status = BuildResult::Success::Built,
.errorMsg = "no idea why",
}}},
};
t;
}))
VERSIONED_CHARACTERIZATION_TEST(
ServeProtoTest,
buildResult_2_3,
"build-result-2.3",
(ServeProto::Version{
.major = 2,
.minor = 3,
}),
({
using namespace std::literals::chrono_literals;
std::tuple<BuildResult, BuildResult, BuildResult> t{
BuildResult{.inner{BuildResult::Failure{{
.status = BuildResult::Failure::OutputRejected,
.msg = HintFmt("no idea why"),
}}}},
BuildResult{
.inner{BuildResult::Failure{{
.inner{BuildResult::Failure{
.status = BuildResult::Failure::NotDeterministic,
.msg = HintFmt("no idea why"),
.errorMsg = "no idea why",
.isNonDeterministic = true,
}}},
.timesBuilt = 3,
.startTime = 30,
.stopTime = 50,
},
BuildResult{
.inner{BuildResult::Success{
.status = BuildResult::Success::Built,
}},
.startTime = 30,
.stopTime = 50,
},
};
t;
}))
VERSIONED_CHARACTERIZATION_TEST(
ServeProtoTest,
buildResult_2_6,
"build-result-2.6",
(ServeProto::Version{
.major = 2,
.minor = 6,
}),
({
using namespace std::literals::chrono_literals;
std::tuple<BuildResult, BuildResult, BuildResult> t{
BuildResult{.inner{BuildResult::Failure{{
.status = BuildResult::Failure::OutputRejected,
.msg = HintFmt("no idea why"),
}}}},
BuildResult{
.inner{BuildResult::Failure{{
.status = BuildResult::Failure::NotDeterministic,
.msg = HintFmt("no idea why"),
.isNonDeterministic = true,
}}},
.timesBuilt = 3,
.startTime = 30,
.stopTime = 50,
@@ -286,10 +260,7 @@ VERSIONED_CHARACTERIZATION_TEST(
ServeProtoTest,
unkeyedValidPathInfo_2_3,
"unkeyed-valid-path-info-2.3",
(ServeProto::Version{
.major = 2,
.minor = 3,
}),
2 << 8 | 3,
(std::tuple<UnkeyedValidPathInfo, UnkeyedValidPathInfo>{
({
UnkeyedValidPathInfo info{std::string{defaultStoreDir}, Hash::dummy};
@@ -315,10 +286,7 @@ VERSIONED_CHARACTERIZATION_TEST(
ServeProtoTest,
unkeyedValidPathInfo_2_4,
"unkeyed-valid-path-info-2.4",
(ServeProto::Version{
.major = 2,
.minor = 4,
}),
2 << 8 | 4,
(std::tuple<UnkeyedValidPathInfo, UnkeyedValidPathInfo>{
({
UnkeyedValidPathInfo info{
@@ -372,10 +340,7 @@ VERSIONED_CHARACTERIZATION_TEST_NO_JSON(
ServeProtoTest,
build_options_2_1,
"build-options-2.1",
(ServeProto::Version{
.major = 2,
.minor = 1,
}),
2 << 8 | 1,
(ServeProto::BuildOptions{
.maxSilentTime = 5,
.buildTimeout = 6,
@@ -385,10 +350,7 @@ VERSIONED_CHARACTERIZATION_TEST_NO_JSON(
ServeProtoTest,
build_options_2_2,
"build-options-2.2",
(ServeProto::Version{
.major = 2,
.minor = 2,
}),
2 << 8 | 2,
(ServeProto::BuildOptions{
.maxSilentTime = 5,
.buildTimeout = 6,
@@ -399,10 +361,7 @@ VERSIONED_CHARACTERIZATION_TEST_NO_JSON(
ServeProtoTest,
build_options_2_3,
"build-options-2.3",
(ServeProto::Version{
.major = 2,
.minor = 3,
}),
2 << 8 | 3,
(ServeProto::BuildOptions{
.maxSilentTime = 5,
.buildTimeout = 6,
@@ -415,10 +374,7 @@ VERSIONED_CHARACTERIZATION_TEST_NO_JSON(
ServeProtoTest,
build_options_2_7,
"build-options-2.7",
(ServeProto::Version{
.major = 2,
.minor = 7,
}),
2 << 8 | 7,
(ServeProto::BuildOptions{
.maxSilentTime = 5,
.buildTimeout = 6,
+99 -296
View File
@@ -15,80 +15,6 @@
namespace nix {
TEST(WorkerProtoVersionNumber, ordering)
{
using Number = WorkerProto::Version::Number;
EXPECT_LT((Number{1, 10}), (Number{1, 20}));
EXPECT_GT((Number{1, 30}), (Number{1, 20}));
EXPECT_EQ((Number{1, 10}), (Number{1, 10}));
EXPECT_LT((Number{0, 255}), (Number{1, 0}));
}
TEST(WorkerProtoVersion, partialOrderingSameFeatures)
{
using V = WorkerProto::Version;
V v1{.number = {1, 20}, .features = {"a", "b"}};
V v2{.number = {1, 30}, .features = {"a", "b"}};
EXPECT_TRUE(v1 < v2);
EXPECT_TRUE(v2 > v1);
EXPECT_TRUE(v1 <= v2);
EXPECT_TRUE(v2 >= v1);
EXPECT_FALSE(v1 == v2);
}
TEST(WorkerProtoVersion, partialOrderingSubsetFeatures)
{
using V = WorkerProto::Version;
V fewer{.number = {1, 30}, .features = {"a"}};
V more{.number = {1, 30}, .features = {"a", "b"}};
// fewer <= more: JUST the features are a subset
EXPECT_TRUE(fewer < more);
EXPECT_TRUE(fewer <= more);
EXPECT_FALSE(fewer > more);
EXPECT_TRUE(fewer != more);
}
TEST(WorkerProtoVersion, partialOrderingUnordered)
{
using V = WorkerProto::Version;
// Same number but incomparable features
V v1{.number = {1, 20}, .features = {"a", "c"}};
V v2{.number = {1, 20}, .features = {"a", "b"}};
EXPECT_FALSE(v1 < v2);
EXPECT_FALSE(v1 > v2);
EXPECT_FALSE(v1 <= v2);
EXPECT_FALSE(v1 >= v2);
EXPECT_FALSE(v1 == v2);
EXPECT_TRUE(v1 != v2);
}
TEST(WorkerProtoVersion, partialOrderingHigherNumberFewerFeatures)
{
using V = WorkerProto::Version;
// Higher number but fewer features — unordered
V v1{.number = {1, 30}, .features = {"a"}};
V v2{.number = {1, 20}, .features = {"a", "b"}};
EXPECT_FALSE(v1 < v2);
EXPECT_FALSE(v1 > v2);
EXPECT_FALSE(v1 == v2);
}
TEST(WorkerProtoVersion, partialOrderingEmptyFeatures)
{
using V = WorkerProto::Version;
V empty{.number = {1, 20}, .features = {}};
V some{.number = {1, 30}, .features = {"a"}};
// empty features is a subset of everything
EXPECT_TRUE(empty < some);
EXPECT_TRUE(empty <= some);
EXPECT_TRUE(empty != some);
}
const char workerProtoDir[] = "worker-protocol";
static constexpr std::string_view defaultStoreDir = "/nix/store";
@@ -99,13 +25,7 @@ struct WorkerProtoTest : VersionedProtoTest<WorkerProto, workerProtoDir>
* For serializers that don't care about the minimum version, we
* used the oldest one: 1.10.
*/
WorkerProto::Version defaultVersion = {
.number =
{
.major = 1,
.minor = 10,
},
};
WorkerProto::Version defaultVersion = 1 << 8 | 10;
};
VERSIONED_CHARACTERIZATION_TEST(
@@ -159,13 +79,7 @@ VERSIONED_CHARACTERIZATION_TEST(
WorkerProtoTest,
derivedPath_1_29,
"derived-path-1.29",
(WorkerProto::Version{
.number =
{
.major = 1,
.minor = 29,
},
}),
1 << 8 | 29,
(std::tuple<DerivedPath, DerivedPath, DerivedPath>{
DerivedPath::Opaque{
.path = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
@@ -190,13 +104,7 @@ VERSIONED_CHARACTERIZATION_TEST(
WorkerProtoTest,
derivedPath_1_30,
"derived-path-1.30",
(WorkerProto::Version{
.number =
{
.major = 1,
.minor = 30,
},
}),
1 << 8 | 30,
(std::tuple<DerivedPath, DerivedPath, DerivedPath, DerivedPath>{
DerivedPath::Opaque{
.path = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
@@ -289,57 +197,36 @@ VERSIONED_READ_CHARACTERIZATION_TEST(
},
}))
VERSIONED_CHARACTERIZATION_TEST(
WorkerProtoTest,
buildResult_1_27,
"build-result-1.27",
(WorkerProto::Version{
.number =
{
.major = 1,
.minor = 27,
},
}),
({
using namespace std::literals::chrono_literals;
std::tuple<BuildResult, BuildResult, BuildResult> t{
BuildResult{.inner{BuildResult::Failure{{
.status = BuildResult::Failure::OutputRejected,
.msg = HintFmt("no idea why"),
}}}},
BuildResult{.inner{BuildResult::Failure{{
.status = BuildResult::Failure::NotDeterministic,
.msg = HintFmt("no idea why"),
}}}},
BuildResult{.inner{BuildResult::Success{
.status = BuildResult::Success::Built,
}}},
};
t;
}))
VERSIONED_CHARACTERIZATION_TEST(WorkerProtoTest, buildResult_1_27, "build-result-1.27", 1 << 8 | 27, ({
using namespace std::literals::chrono_literals;
std::tuple<BuildResult, BuildResult, BuildResult> t{
BuildResult{.inner{BuildResult::Failure{
.status = BuildResult::Failure::OutputRejected,
.errorMsg = "no idea why",
}}},
BuildResult{.inner{BuildResult::Failure{
.status = BuildResult::Failure::NotDeterministic,
.errorMsg = "no idea why",
}}},
BuildResult{.inner{BuildResult::Success{
.status = BuildResult::Success::Built,
}}},
};
t;
}))
VERSIONED_CHARACTERIZATION_TEST(
WorkerProtoTest,
buildResult_1_28,
"build-result-1.28",
(WorkerProto::Version{
.number =
{
.major = 1,
.minor = 28,
},
}),
({
WorkerProtoTest, buildResult_1_28, "build-result-1.28", 1 << 8 | 28, ({
using namespace std::literals::chrono_literals;
std::tuple<BuildResult, BuildResult, BuildResult> t{
BuildResult{.inner{BuildResult::Failure{{
BuildResult{.inner{BuildResult::Failure{
.status = BuildResult::Failure::OutputRejected,
.msg = HintFmt("no idea why"),
}}}},
BuildResult{.inner{BuildResult::Failure{{
.errorMsg = "no idea why",
}}},
BuildResult{.inner{BuildResult::Failure{
.status = BuildResult::Failure::NotDeterministic,
.msg = HintFmt("no idea why"),
}}}},
.errorMsg = "no idea why",
}}},
BuildResult{.inner{BuildResult::Success{
.status = BuildResult::Success::Built,
.builtOutputs =
@@ -375,29 +262,19 @@ VERSIONED_CHARACTERIZATION_TEST(
}))
VERSIONED_CHARACTERIZATION_TEST(
WorkerProtoTest,
buildResult_1_29,
"build-result-1.29",
(WorkerProto::Version{
.number =
{
.major = 1,
.minor = 29,
},
}),
({
WorkerProtoTest, buildResult_1_29, "build-result-1.29", 1 << 8 | 29, ({
using namespace std::literals::chrono_literals;
std::tuple<BuildResult, BuildResult, BuildResult> t{
BuildResult{.inner{BuildResult::Failure{{
BuildResult{.inner{BuildResult::Failure{
.status = BuildResult::Failure::OutputRejected,
.msg = HintFmt("no idea why"),
}}}},
.errorMsg = "no idea why",
}}},
BuildResult{
.inner{BuildResult::Failure{{
.inner{BuildResult::Failure{
.status = BuildResult::Failure::NotDeterministic,
.msg = HintFmt("no idea why"),
.errorMsg = "no idea why",
.isNonDeterministic = true,
}}},
}},
.timesBuilt = 3,
.startTime = 30,
.stopTime = 50,
@@ -444,29 +321,19 @@ VERSIONED_CHARACTERIZATION_TEST(
}))
VERSIONED_CHARACTERIZATION_TEST(
WorkerProtoTest,
buildResult_1_37,
"build-result-1.37",
(WorkerProto::Version{
.number =
{
.major = 1,
.minor = 37,
},
}),
({
WorkerProtoTest, buildResult_1_37, "build-result-1.37", 1 << 8 | 37, ({
using namespace std::literals::chrono_literals;
std::tuple<BuildResult, BuildResult, BuildResult> t{
BuildResult{.inner{BuildResult::Failure{{
BuildResult{.inner{BuildResult::Failure{
.status = BuildResult::Failure::OutputRejected,
.msg = HintFmt("no idea why"),
}}}},
.errorMsg = "no idea why",
}}},
BuildResult{
.inner{BuildResult::Failure{{
.inner{BuildResult::Failure{
.status = BuildResult::Failure::NotDeterministic,
.msg = HintFmt("no idea why"),
.errorMsg = "no idea why",
.isNonDeterministic = true,
}}},
}},
.timesBuilt = 3,
.startTime = 30,
.stopTime = 50,
@@ -514,65 +381,48 @@ VERSIONED_CHARACTERIZATION_TEST(
t;
}))
VERSIONED_CHARACTERIZATION_TEST(
WorkerProtoTest,
keyedBuildResult_1_29,
"keyed-build-result-1.29",
(WorkerProto::Version{
.number =
{
.major = 1,
.minor = 29,
},
}),
({
using namespace std::literals::chrono_literals;
std::tuple<KeyedBuildResult, KeyedBuildResult /*, KeyedBuildResult*/> t{
KeyedBuildResult{
BuildResult{.inner{KeyedBuildResult::Failure{{
.status = KeyedBuildResult::Failure::OutputRejected,
.msg = HintFmt("no idea why"),
}}}},
/* .path = */
DerivedPath::Opaque{
StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-xxx"},
},
},
KeyedBuildResult{
BuildResult{
.inner{KeyedBuildResult::Failure{{
.status = KeyedBuildResult::Failure::NotDeterministic,
.msg = HintFmt("no idea why"),
.isNonDeterministic = true,
}}},
.timesBuilt = 3,
.startTime = 30,
.stopTime = 50,
},
/* .path = */
DerivedPath::Built{
.drvPath = makeConstantStorePathRef(
StorePath{
"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv",
}),
.outputs = OutputsSpec::Names{"out"},
},
},
};
t;
}))
VERSIONED_CHARACTERIZATION_TEST(WorkerProtoTest, keyedBuildResult_1_29, "keyed-build-result-1.29", 1 << 8 | 29, ({
using namespace std::literals::chrono_literals;
std::tuple<KeyedBuildResult, KeyedBuildResult /*, KeyedBuildResult*/> t{
KeyedBuildResult{
{.inner{BuildResult::Failure{
.status = KeyedBuildResult::Failure::OutputRejected,
.errorMsg = "no idea why",
}}},
/* .path = */
DerivedPath::Opaque{
StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-xxx"},
},
},
KeyedBuildResult{
{
.inner{BuildResult::Failure{
.status = KeyedBuildResult::Failure::NotDeterministic,
.errorMsg = "no idea why",
.isNonDeterministic = true,
}},
.timesBuilt = 3,
.startTime = 30,
.stopTime = 50,
},
/* .path = */
DerivedPath::Built{
.drvPath = makeConstantStorePathRef(
StorePath{
"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv",
}),
.outputs = OutputsSpec::Names{"out"},
},
},
};
t;
}))
VERSIONED_CHARACTERIZATION_TEST(
WorkerProtoTest,
unkeyedValidPathInfo_1_15,
"unkeyed-valid-path-info-1.15",
(WorkerProto::Version{
.number =
{
.major = 1,
.minor = 15,
},
}),
1 << 8 | 15,
(std::tuple<UnkeyedValidPathInfo, UnkeyedValidPathInfo>{
({
UnkeyedValidPathInfo info{
@@ -606,13 +456,7 @@ VERSIONED_CHARACTERIZATION_TEST(
WorkerProtoTest,
validPathInfo_1_15,
"valid-path-info-1.15",
(WorkerProto::Version{
.number =
{
.major = 1,
.minor = 15,
},
}),
1 << 8 | 15,
(std::tuple<ValidPathInfo, ValidPathInfo>{
({
ValidPathInfo info{
@@ -661,13 +505,7 @@ VERSIONED_CHARACTERIZATION_TEST(
WorkerProtoTest,
validPathInfo_1_16,
"valid-path-info-1.16",
(WorkerProto::Version{
.number =
{
.major = 1,
.minor = 16,
},
}),
1 << 8 | 16,
(std::tuple<ValidPathInfo, ValidPathInfo, ValidPathInfo>{
({
ValidPathInfo info{
@@ -822,13 +660,7 @@ VERSIONED_CHARACTERIZATION_TEST_NO_JSON(
WorkerProtoTest,
clientHandshakeInfo_1_30,
"client-handshake-info_1_30",
(WorkerProto::Version{
.number =
{
.major = 1,
.minor = 30,
},
}),
1 << 8 | 30,
(std::tuple<WorkerProto::ClientHandshakeInfo>{
{},
}))
@@ -837,13 +669,7 @@ VERSIONED_CHARACTERIZATION_TEST_NO_JSON(
WorkerProtoTest,
clientHandshakeInfo_1_33,
"client-handshake-info_1_33",
(WorkerProto::Version{
.number =
{
.major = 1,
.minor = 33,
},
}),
1 << 8 | 33,
(std::tuple<WorkerProto::ClientHandshakeInfo, WorkerProto::ClientHandshakeInfo>{
{
.daemonNixVersion = std::optional{"foo"},
@@ -857,13 +683,7 @@ VERSIONED_CHARACTERIZATION_TEST_NO_JSON(
WorkerProtoTest,
clientHandshakeInfo_1_35,
"client-handshake-info_1_35",
(WorkerProto::Version{
.number =
{
.major = 1,
.minor = 35,
},
}),
1 << 8 | 35,
(std::tuple<WorkerProto::ClientHandshakeInfo, WorkerProto::ClientHandshakeInfo>{
{
.daemonNixVersion = std::optional{"foo"},
@@ -890,13 +710,13 @@ TEST_F(WorkerProtoTest, handshake_log)
FdSink out{toServer.writeSide.get()};
FdSource in0{toClient.readSide.get()};
TeeSource in{in0, toClientLog};
clientResult = WorkerProto::BasicClientConnection::handshake(out, in, defaultVersion);
clientResult = std::get<0>(WorkerProto::BasicClientConnection::handshake(out, in, defaultVersion, {}));
});
{
FdSink out{toClient.writeSide.get()};
FdSource in{toServer.readSide.get()};
WorkerProto::BasicServerConnection::handshake(out, in, defaultVersion);
WorkerProto::BasicServerConnection::handshake(out, in, defaultVersion, {});
};
thread.join();
@@ -911,43 +731,23 @@ TEST_F(WorkerProtoTest, handshake_features)
toClient.create();
toServer.create();
WorkerProto::Version clientResult;
std::tuple<WorkerProto::Version, WorkerProto::FeatureSet> clientResult;
auto clientThread = std::thread([&]() {
FdSink out{toServer.writeSide.get()};
FdSource in{toClient.readSide.get()};
clientResult = WorkerProto::BasicClientConnection::handshake(
out,
in,
WorkerProto::Version{
.number = {.major = 1, .minor = 123},
.features = {"bar", "aap", "mies", "xyzzy"},
});
clientResult = WorkerProto::BasicClientConnection::handshake(out, in, 123, {"bar", "aap", "mies", "xyzzy"});
});
FdSink out{toClient.writeSide.get()};
FdSource in{toServer.readSide.get()};
auto daemonResult = WorkerProto::BasicServerConnection::handshake(
out,
in,
WorkerProto::Version{
.number = {.major = 1, .minor = 200},
.features = {"foo", "bar", "xyzzy"},
});
auto daemonResult = WorkerProto::BasicServerConnection::handshake(out, in, 456, {"foo", "bar", "xyzzy"});
clientThread.join();
EXPECT_EQ(clientResult, daemonResult);
EXPECT_EQ(
clientResult,
(WorkerProto::Version{
.number =
{
.major = 1,
.minor = 123,
},
.features = {"bar", "xyzzy"},
}));
EXPECT_EQ(std::get<0>(clientResult), 123u);
EXPECT_EQ(std::get<1>(clientResult), WorkerProto::FeatureSet({"bar", "xyzzy"}));
}
/// Has to be a `BufferedSink` for handshake.
@@ -962,7 +762,8 @@ TEST_F(WorkerProtoTest, handshake_client_replay)
NullBufferedSink nullSink;
StringSource in{toClientLog};
auto clientResult = WorkerProto::BasicClientConnection::handshake(nullSink, in, defaultVersion);
auto clientResult =
std::get<0>(WorkerProto::BasicClientConnection::handshake(nullSink, in, defaultVersion, {}));
EXPECT_EQ(clientResult, defaultVersion);
});
@@ -976,10 +777,11 @@ TEST_F(WorkerProtoTest, handshake_client_truncated_replay_throws)
auto substring = toClientLog.substr(0, len);
StringSource in{substring};
if (len < 8) {
EXPECT_THROW(WorkerProto::BasicClientConnection::handshake(nullSink, in, defaultVersion), EndOfFile);
EXPECT_THROW(
WorkerProto::BasicClientConnection::handshake(nullSink, in, defaultVersion, {}), EndOfFile);
} else {
// Not sure why cannot keep on checking for `EndOfFile`.
EXPECT_THROW(WorkerProto::BasicClientConnection::handshake(nullSink, in, defaultVersion), Error);
EXPECT_THROW(WorkerProto::BasicClientConnection::handshake(nullSink, in, defaultVersion, {}), Error);
}
}
});
@@ -999,13 +801,14 @@ TEST_F(WorkerProtoTest, handshake_client_corrupted_throws)
if (idx < 4 || idx == 9) {
// magic bytes don't match
EXPECT_THROW(WorkerProto::BasicClientConnection::handshake(nullSink, in, defaultVersion), Error);
EXPECT_THROW(WorkerProto::BasicClientConnection::handshake(nullSink, in, defaultVersion, {}), Error);
} else if (idx < 8 || idx >= 12) {
// Number out of bounds
EXPECT_THROW(
WorkerProto::BasicClientConnection::handshake(nullSink, in, defaultVersion), SerialisationError);
WorkerProto::BasicClientConnection::handshake(nullSink, in, defaultVersion, {}),
SerialisationError);
} else {
auto ver = WorkerProto::BasicClientConnection::handshake(nullSink, in, defaultVersion);
auto ver = std::get<0>(WorkerProto::BasicClientConnection::handshake(nullSink, in, defaultVersion, {}));
// `std::min` of this and the other version saves us
EXPECT_EQ(ver, defaultVersion);
}
+3 -3
View File
@@ -193,7 +193,7 @@ TEST_F(WorkerSubstitutionTest, floatingDerivationOutput)
};
// Write the derivation to the destination store
auto drvPath = dummyStore->writeDerivation(drv);
auto drvPath = writeDerivation(*dummyStore, drv);
// Snapshot the destination store before
checkpointJson("ca-drv/store-before", dummyStore);
@@ -297,7 +297,7 @@ TEST_F(WorkerSubstitutionTest, floatingDerivationOutputWithDepDrv)
};
// Write the dependency derivation to the destination store
auto depDrvPath = dummyStore->writeDerivation(depDrv);
auto depDrvPath = writeDerivation(*dummyStore, depDrv);
// Compute the hash modulo for the dependency derivation
auto depHashModulo = hashDerivationModulo(*dummyStore, depDrv, true);
@@ -348,7 +348,7 @@ TEST_F(WorkerSubstitutionTest, floatingDerivationOutputWithDepDrv)
rootDrv.inputDrvs = {.map = {{depDrvPath, {.value = {"out"}}}}};
// Write the root derivation to the destination store
auto rootDrvPath = dummyStore->writeDerivation(rootDrv);
auto rootDrvPath = writeDerivation(*dummyStore, rootDrv);
// Snapshot the destination store before
checkpointJson("issue-11928/store-before", dummyStore);
+3 -3
View File
@@ -44,12 +44,12 @@ TEST_F(WriteDerivationTest, addToStoreFromDumpCalledOnce)
{
auto drv = makeSimpleDrv();
auto path1 = store->writeDerivation(drv, NoRepair);
auto path1 = writeDerivation(*store, drv, NoRepair);
config->readOnly = true;
auto path2 = computeStorePath(*store, drv);
auto path2 = writeDerivation(*store, drv, NoRepair);
EXPECT_EQ(path1, path2);
EXPECT_THAT(
[&] { store->writeDerivation(drv, Repair); },
[&] { writeDerivation(*store, drv, Repair); },
::testing::ThrowsMessage<Error>(
testing::HasSubstrIgnoreANSIMatcher("operation 'writeDerivation' is not supported by store 'dummy://'")));
}
+25 -29
View File
@@ -435,41 +435,37 @@ void BinaryCacheStore::narFromPath(const StorePath & storePath, Sink & sink)
void BinaryCacheStore::queryPathInfoUncached(
const StorePath & storePath, Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept
{
auto uri = config.getReference().render(/*FIXME withParams=*/false);
auto storePathS = printStorePath(storePath);
auto act = std::make_shared<Activity>(
*logger,
lvlTalkative,
actQueryPathInfo,
fmt("querying info about '%s' on '%s'", storePathS, uri),
Logger::Fields{storePathS, uri});
PushActivity pact(act->id);
auto narInfoFile = narInfoFileFor(storePath);
auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback));
try {
auto uri = config.getReference().render(/*FIXME withParams=*/false);
auto storePathS = printStorePath(storePath);
auto act = std::make_shared<Activity>(
*logger,
lvlTalkative,
actQueryPathInfo,
fmt("querying info about '%s' on '%s'", storePathS, uri),
Logger::Fields{storePathS, uri});
PushActivity pact(act->id);
getFile(narInfoFile, {[=, this](std::future<std::optional<std::string>> fut) {
try {
auto data = fut.get();
auto narInfoFile = narInfoFileFor(storePath);
if (!data)
return (*callbackPtr)({});
getFile(narInfoFile, {[=, this](std::future<std::optional<std::string>> fut) {
try {
auto data = fut.get();
stats.narInfoRead++;
if (!data)
return (*callbackPtr)({});
(*callbackPtr)(
(std::shared_ptr<ValidPathInfo>) std::make_shared<NarInfo>(*this, *data, narInfoFile));
stats.narInfoRead++;
(*callbackPtr)(
(std::shared_ptr<ValidPathInfo>) std::make_shared<NarInfo>(*this, *data, narInfoFile));
(void) act; // force Activity into this lambda to ensure it stays alive
} catch (...) {
callbackPtr->rethrow();
}
}});
} catch (...) {
callbackPtr->rethrow();
}
(void) act; // force Activity into this lambda to ensure it stays alive
} catch (...) {
callbackPtr->rethrow();
}
}});
}
StorePath BinaryCacheStore::addToStore(
+11 -70
View File
@@ -4,65 +4,15 @@
namespace nix {
void ExitStatusFlags::updateFromStatus(BuildResult::Failure::Status status)
{
// Allow selecting a subset of enum values
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wswitch-enum"
switch (status) {
case BuildResult::Failure::TimedOut:
timedOut = true;
break;
case BuildResult::Failure::HashMismatch:
hashMismatch = true;
break;
case BuildResult::Failure::NotDeterministic:
checkMismatch = true;
break;
case BuildResult::Failure::PermanentFailure:
// Also considered a permenant failure, it seems
case BuildResult::Failure::InputRejected:
permanentFailure = true;
break;
default:
break;
}
#pragma GCC diagnostic pop
}
unsigned int ExitStatusFlags::failingExitStatus() const
{
bool buildFailure = permanentFailure || timedOut || hashMismatch;
/* Any of the 4 booleans we track */
bool problemWithSpecialExitCode = checkMismatch || buildFailure;
unsigned int mask = 0;
if (problemWithSpecialExitCode) {
mask |= 0b1100000;
if (buildFailure) {
mask |= 0b0100; // 100
if (timedOut)
mask |= 0b0001; // 101
if (hashMismatch)
mask |= 0b0010; // 102
}
if (checkMismatch)
mask |= 0b1000; // 104
}
/* We still (per the function docs) only call this function in the
failure case, so the default should not be 0, but 1, indicating
"some other kind of error. */
return mask ? mask : 1;
}
bool BuildResult::operator==(const BuildResult &) const noexcept = default;
std::strong_ordering BuildResult::operator<=>(const BuildResult &) const noexcept = default;
bool BuildResult::Success::operator==(const BuildResult::Success &) const noexcept = default;
std::strong_ordering BuildResult::Success::operator<=>(const BuildResult::Success &) const noexcept = default;
bool BuildResult::Failure::operator==(const BuildResult::Failure &) const noexcept = default;
std::strong_ordering BuildResult::Failure::operator<=>(const BuildResult::Failure &) const noexcept = default;
static constexpr std::array<std::pair<BuildResult::Success::Status, std::string_view>, 4> successStatusStrings{{
#define ENUM_ENTRY(e) {BuildResult::Success::e, #e}
ENUM_ENTRY(Built),
@@ -125,18 +75,9 @@ static BuildResult::Failure::Status failureStatusFromString(std::string_view str
throw Error("unknown built result failure status '%s'", str);
}
bool BuildError::operator==(const BuildError & other) const noexcept
[[noreturn]] void BuildResult::Failure::rethrow() const
{
return status == other.status && isNonDeterministic == other.isNonDeterministic && message() == other.message();
}
std::strong_ordering BuildError::operator<=>(const BuildError & other) const noexcept
{
if (auto cmp = status <=> other.status; cmp != 0)
return cmp;
if (auto cmp = isNonDeterministic <=> other.isNonDeterministic; cmp != 0)
return cmp;
return message() <=> other.message();
throw BuildError(status, "%s", errorMsg);
}
} // namespace nix
@@ -172,7 +113,7 @@ void adl_serializer<BuildResult>::to_json(json & res, const BuildResult & br)
[&](const BuildResult::Failure & failure) {
res["success"] = false;
res["status"] = failureStatusToString(failure.status);
res["errorMsg"] = failure.message();
res["errorMsg"] = failure.errorMsg;
res["isNonDeterministic"] = failure.isNonDeterministic;
},
},
@@ -207,11 +148,11 @@ BuildResult adl_serializer<BuildResult>::from_json(const json & _json)
s.builtOutputs = valueAt(json, "builtOutputs");
br.inner = std::move(s);
} else {
br.inner = BuildResult::Failure{{
.status = failureStatusFromString(statusStr),
.msg = HintFmt(getString(valueAt(json, "errorMsg"))),
.isNonDeterministic = getBoolean(valueAt(json, "isNonDeterministic")),
}};
BuildResult::Failure f;
f.status = failureStatusFromString(statusStr);
f.errorMsg = getString(valueAt(json, "errorMsg"));
f.isNonDeterministic = getBoolean(valueAt(json, "isNonDeterministic"));
br.inner = std::move(f);
}
return br;
+165 -286
View File
@@ -5,7 +5,6 @@
# include "nix/store/build/derivation-builder.hh"
#endif
#include "nix/util/processes.hh"
#include "nix/util/environment-variables.hh"
#include "nix/util/config-global.hh"
#include "nix/store/build/worker.hh"
#include "nix/util/util.hh"
@@ -15,7 +14,6 @@
#include "nix/store/local-store.hh" // TODO remove, along with remaining downcasts
#include "nix/store/globals.hh"
#include <algorithm>
#include <fstream>
#include <sys/types.h>
#include <fcntl.h>
@@ -65,11 +63,7 @@ std::string showKnownOutputs(const StoreDirConfig & store, const Derivation & dr
}
static void runPostBuildHook(
const WorkerSettings & workerSettings,
const StoreDirConfig & store,
Logger & logger,
const StorePath & drvPath,
const StorePathSet & outputPaths);
const StoreDirConfig & store, Logger & logger, const StorePath & drvPath, const StorePathSet & outputPaths);
/* At least one of the output paths could not be
produced using a substitute. So we have to build instead. */
@@ -93,7 +87,7 @@ Goal::Co DerivationBuildingGoal::gaveUpOnSubstitution(bool storeDerivation)
for (auto & i : drv->inputSrcs) {
if (worker.store.isValidPath(i))
continue;
if (!worker.settings.useSubstitutes)
if (!settings.useSubstitutes)
throw Error(
"dependency '%s' of '%s' does not exist, and substitution is disabled",
worker.store.printStorePath(i),
@@ -125,7 +119,7 @@ Goal::Co DerivationBuildingGoal::gaveUpOnSubstitution(bool storeDerivation)
assert(drv->inputDrvs.map.empty());
/* Store the resolved derivation, as part of the record of
what we're actually building */
worker.store.writeDerivation(*drv);
writeDerivation(worker.store, *drv);
}
StorePathSet inputPaths;
@@ -185,82 +179,6 @@ struct LogFile
~LogFile();
};
struct LocalBuildRejection
{
bool maxJobsZero = false;
struct NoLocalStore
{};
/**
* We have a local store, but we don't have an external derivation builder (which is fine), if we did, it'd be
* fine because we would not care about platforms and features then. Since we don't, we either have the wrong
* platform, or we are missing some system features.
*/
struct WrongLocalStore
{
template<typename T>
struct Pair
{
T derivation;
T localStore;
};
std::optional<Pair<std::string>> badPlatform;
std::optional<Pair<StringSet>> missingFeatures;
};
std::variant<NoLocalStore, WrongLocalStore> rejection;
};
static BuildError reject(const LocalBuildRejection & rejection, std::string_view thingCannotBuild)
{
if (std::get_if<LocalBuildRejection::NoLocalStore>(&rejection.rejection))
return BuildError(
BuildResult::Failure::InputRejected,
"Unable to build with a primary store that isn't a local store; "
"either pass a different '--store' or enable remote builds.\n\n"
"For more information check 'man nix.conf' and search for '/machines'.");
auto & wrongStore = std::get<LocalBuildRejection::WrongLocalStore>(rejection.rejection);
std::string msg = fmt("Cannot build '%s'.", Magenta(thingCannotBuild));
if (rejection.maxJobsZero)
msg += "\nReason: " ANSI_RED "local builds are disabled" ANSI_NORMAL
" (max-jobs = 0)"
"\nHint: set 'max-jobs' to a non-zero value to enable local builds, "
"or configure remote builders via 'builders'";
if (wrongStore.badPlatform)
msg +=
fmt("\nReason: " ANSI_RED "platform mismatch" ANSI_NORMAL
"\nRequired system: '%s'"
"\nCurrent system: '%s'",
Magenta(wrongStore.badPlatform->derivation),
Magenta(wrongStore.badPlatform->localStore));
if (wrongStore.missingFeatures)
msg +=
fmt("\nReason: " ANSI_RED "missing system features" ANSI_NORMAL
"\nRequired features: {%s}"
"\nAvailable features: {%s}",
concatStringsSep(", ", wrongStore.missingFeatures->derivation),
concatStringsSep<StringSet>(", ", wrongStore.missingFeatures->localStore));
if (wrongStore.badPlatform || wrongStore.missingFeatures) {
// since aarch64-darwin has Rosetta 2, this user can actually run x86_64-darwin on their
// hardware - we should tell them to run the command to install Rosetta
if (wrongStore.badPlatform && wrongStore.badPlatform->derivation == "x86_64-darwin"
&& wrongStore.badPlatform->localStore == "aarch64-darwin")
msg +=
fmt("\nNote: run `%s` to run programs for x86_64-darwin",
Magenta("/usr/sbin/softwareupdate --install-rosetta && launchctl stop org.nixos.nix-daemon"));
}
return BuildError(BuildResult::Failure::InputRejected, std::move(msg));
}
Goal::Co DerivationBuildingGoal::tryToBuild(StorePathSet inputPaths)
{
auto drvOptions = [&] {
@@ -322,57 +240,29 @@ Goal::Co DerivationBuildingGoal::tryToBuild(StorePathSet inputPaths)
}
checkPathValidity(initialOutputs);
auto localBuildResult = [&]() -> std::variant<LocalBuildCapability, LocalBuildRejection> {
bool maxJobsZero = worker.settings.maxBuildJobs.get() == 0;
/**
* Activity that denotes waiting for a lock.
*/
std::unique_ptr<Activity> actLock;
auto * localStoreP = dynamic_cast<LocalStore *>(&worker.store);
if (!localStoreP)
return LocalBuildRejection{.maxJobsZero = maxJobsZero, .rejection = LocalBuildRejection::NoLocalStore{}};
/**
* Locks on (fixed) output paths.
*/
PathLocks outputLocks;
/**
* Now that we've decided we can't / won't do a remote build, check
* that we can in fact build locally. First see if there is an
* external builder for a "semi-local build". If there is, prefer to
* use that. If there is not, then check if we can do a "true" local
* build.
*/
auto * ext = settings.getLocalSettings().findExternalDerivationBuilderIfSupported(*drv);
bool useHook;
if (ext)
return LocalBuildCapability{*localStoreP, ext};
const ExternalBuilder * externalBuilder = nullptr;
using WrongLocalStore = LocalBuildRejection::WrongLocalStore;
WrongLocalStore wrongStore;
if (drv->platform != settings.thisSystem.get() && !settings.extraPlatforms.get().count(drv->platform)
&& !drv->isBuiltin())
wrongStore.badPlatform = WrongLocalStore::Pair<std::string>{drv->platform, settings.thisSystem.get()};
{
auto required = drvOptions.getRequiredSystemFeatures(*drv);
auto & available = worker.store.config.systemFeatures.get();
if (std::ranges::any_of(required, [&](const std::string & f) { return !available.count(f); }))
wrongStore.missingFeatures = WrongLocalStore::Pair<StringSet>{required, available};
}
if (maxJobsZero || wrongStore.badPlatform || wrongStore.missingFeatures)
return LocalBuildRejection{.maxJobsZero = maxJobsZero, .rejection = std::move(wrongStore)};
return LocalBuildCapability{*localStoreP, ext};
}();
auto acquireResources = [&](bool & done, PathLocks & outputLocks) -> Goal::Co {
while (true) {
trace("trying to build");
/**
* Output paths to acquire locks on, if known a priori.
*
* The locks are automatically released when the caller's `PathLocks` goes
* out of scope, including on exception unwinding. If we can't acquire the lock, then
* continue; hopefully some other goal can start a build, and if not, the
* main loop will sleep a few seconds and then retry this goal.
*/
/* Obtain locks on all output paths, if the paths are known a priori.
The locks are automatically released when we exit this function or Nix
crashes. If we can't acquire the lock, then continue; hopefully some
other goal can start a build, and if not, the main loop will sleep a few
seconds and then retry this goal. */
std::set<std::filesystem::path> lockFiles;
/* FIXME: Should lock something like the drv itself so we don't build same
CA drv concurrently */
@@ -416,8 +306,7 @@ Goal::Co DerivationBuildingGoal::tryToBuild(StorePathSet inputPaths)
debug("skipping build of derivation '%s', someone beat us to it", worker.store.printStorePath(drvPath));
outputLocks.setDeletion(true);
outputLocks.unlock();
done = true;
co_return Return{};
co_return doneSuccess(BuildResult::Success::AlreadyValid, std::move(validOutputs));
}
/* If any of the outputs already exist but are not valid, delete
@@ -432,133 +321,98 @@ Goal::Co DerivationBuildingGoal::tryToBuild(StorePathSet inputPaths)
}
}
co_return Return{};
};
auto tryHookLoop = [&](bool & valid) -> Goal::Co {
{
PathLocks outputLocks;
co_await acquireResources(valid, outputLocks);
if (valid)
co_return doneSuccess(BuildResult::Success::AlreadyValid, checkPathValidity(initialOutputs).second);
/* Don't do a remote build if the derivation has the attribute
`preferLocalBuild' set. Also, check and repair modes are only
supported for local builds. */
bool buildLocally = (buildMode != bmNormal || drvOptions.willBuildLocally(worker.store, *drv))
&& settings.maxBuildJobs.get() != 0;
if (buildLocally) {
useHook = false;
} else {
switch (tryBuildHook(drvOptions)) {
case rpAccept:
/* Yes, it has started doing so. Wait until we get
EOF from the hook. */
valid = true;
co_return buildWithHook(
std::move(inputPaths), std::move(initialOutputs), std::move(drvOptions), std::move(outputLocks));
case rpDecline:
// We should do it ourselves.
co_return Return{};
useHook = true;
break;
case rpPostpone:
/* Not now; wait until at least one child finishes or
the wake-up timeout expires. */
break;
}
}
PathLocks outputLocks;
{
// First attempt was postponed. Retry in a loop with an activity
// that lives until accept or decline.
Activity act(
*logger,
lvlWarn,
actBuildWaiting,
fmt("waiting for a machine to build '%s'", Magenta(worker.store.printStorePath(drvPath))));
while (true) {
if (!actLock)
actLock = std::make_unique<Activity>(
*logger,
lvlWarn,
actBuildWaiting,
fmt("waiting for a machine to build '%s'", Magenta(worker.store.printStorePath(drvPath))));
outputLocks.unlock();
co_await waitForAWhile();
co_await acquireResources(valid, outputLocks);
if (valid)
break;
continue;
case rpDecline:
/* We should do it ourselves.
Now that we've decided we can't / won't do a remote build, check
that we can in fact build locally. First see if there is an
external builder for a "semi-local build". If there is, prefer to
use that. If there is not, then check if we can do a "true" local
build. */
externalBuilder = settings.findExternalDerivationBuilderIfSupported(*drv);
if (!externalBuilder && !drvOptions.canBuildLocally(worker.store, *drv)) {
auto msg =
fmt("Cannot build '%s'.\n"
"Reason: " ANSI_RED "required system or feature not available" ANSI_NORMAL
"\n"
"Required system: '%s' with features {%s}\n"
"Current system: '%s' with features {%s}",
Magenta(worker.store.printStorePath(drvPath)),
Magenta(drv->platform),
concatStringsSep(", ", drvOptions.getRequiredSystemFeatures(*drv)),
Magenta(settings.thisSystem),
concatStringsSep<StringSet>(", ", worker.store.Store::config.systemFeatures));
// since aarch64-darwin has Rosetta 2, this user can actually run x86_64-darwin on their hardware -
// we should tell them to run the command to install Darwin 2
if (drv->platform == "x86_64-darwin" && settings.thisSystem == "aarch64-darwin")
msg += fmt(
"\nNote: run `%s` to run programs for x86_64-darwin",
Magenta(
"/usr/sbin/softwareupdate --install-rosetta && launchctl stop org.nixos.nix-daemon"));
switch (tryBuildHook(drvOptions)) {
case rpAccept:
/* Yes, it has started doing so. Wait until we get
EOF from the hook. */
break;
case rpPostpone:
/* Not now; wait until at least one child finishes or
the wake-up timeout expires. */
outputLocks.unlock();
continue;
case rpDecline:
// We should do it ourselves.
co_return Return{};
worker.permanentFailure = true;
co_return doneFailure({BuildResult::Failure::InputRejected, std::move(msg)});
}
useHook = false;
break;
}
}
if (valid) {
co_return doneSuccess(BuildResult::Success::AlreadyValid, checkPathValidity(initialOutputs).second);
} else {
co_return buildWithHook(
std::move(inputPaths), std::move(initialOutputs), std::move(drvOptions), std::move(outputLocks));
}
};
auto tryBuildLocally = [&](bool & valid) -> Goal::Co {
if (auto * cap = std::get_if<LocalBuildCapability>(&localBuildResult)) {
PathLocks outputLocks;
co_await acquireResources(valid, outputLocks);
if (valid)
co_return doneSuccess(BuildResult::Success::AlreadyValid, checkPathValidity(initialOutputs).second);
valid = true;
co_return buildLocally(
*cap, std::move(inputPaths), std::move(initialOutputs), std::move(drvOptions), std::move(outputLocks));
}
co_return Return{};
};
if (buildMode != bmNormal) {
// Check and repair modes operate on the state of this store specifically,
// so they must always build locally.
bool valid = false;
co_await tryBuildLocally(valid);
if (valid)
co_return Return{};
} else if (drvOptions.preferLocalBuild) {
// Local is preferred, so try it first. If it's not available, fall back to the hook.
{
bool valid = false;
co_await tryBuildLocally(valid);
if (valid)
co_return Return{};
}
{
bool valid = false;
co_await tryHookLoop(valid);
if (valid)
co_return Return{};
}
} else {
// Default preference is a remote build: they tend to be faster and preserve local
// resources for other tasks. Fall back to local if no remote is available.
{
bool valid = false;
co_await tryHookLoop(valid);
if (valid)
co_return Return{};
}
{
bool valid = false;
co_await tryBuildLocally(valid);
if (valid)
co_return Return{};
}
break;
}
std::string storePath = worker.store.printStorePath(drvPath);
auto * rejection = std::get_if<LocalBuildRejection>(&localBuildResult);
assert(rejection);
co_return doneFailure(reject(*rejection, storePath));
actLock.reset();
if (useHook) {
co_return buildWithHook(
std::move(inputPaths), std::move(initialOutputs), std::move(drvOptions), std::move(outputLocks));
} else if (auto * localStoreP = dynamic_cast<LocalStore *>(&worker.store)) {
co_return buildLocally(
*localStoreP,
std::move(inputPaths),
std::move(initialOutputs),
std::move(drvOptions),
std::move(outputLocks),
externalBuilder);
} else {
throw Error(
R"(
Unable to build with a primary store that isn't a local store;
either pass a different '--store' or enable remote builds.
For more information check 'man nix.conf' and search for '/machines'.
)");
}
}
Goal::Co DerivationBuildingGoal::buildWithHook(
@@ -623,7 +477,7 @@ Goal::Co DerivationBuildingGoal::buildWithHook(
msg += fmt(" on '%s'", hook->machineName);
std::unique_ptr<BuildLog> buildLog = std::make_unique<BuildLog>(
worker.settings.logLines,
settings.logLines,
std::make_unique<Activity>(
*logger,
lvlInfo,
@@ -643,7 +497,7 @@ Goal::Co DerivationBuildingGoal::buildWithHook(
auto & data = output->data;
if (fd == hook->builderOut.readSide.get()) {
logSize += data.size();
if (worker.settings.maxLogSize && logSize > worker.settings.maxLogSize) {
if (settings.maxLogSize && logSize > settings.maxLogSize) {
hook.reset();
co_return doneFailureLogTooLong(*buildLog);
}
@@ -752,7 +606,7 @@ Goal::Co DerivationBuildingGoal::buildWithHook(
StorePathSet outputPaths;
for (auto & [_, output] : builtOutputs)
outputPaths.insert(output.outPath);
runPostBuildHook(worker.settings, worker.store, *logger, drvPath, outputPaths);
runPostBuildHook(worker.store, *logger, drvPath, outputPaths);
/* It is now safe to delete the lock files, since all future
lockers will see that the output paths are valid; they will
@@ -766,11 +620,12 @@ Goal::Co DerivationBuildingGoal::buildWithHook(
}
Goal::Co DerivationBuildingGoal::buildLocally(
LocalBuildCapability localBuildCap,
LocalStore & localStore,
StorePathSet inputPaths,
std::map<std::string, InitialOutput> initialOutputs,
DerivationOptions<StorePath> drvOptions,
PathLocks outputLocks)
PathLocks outputLocks,
const ExternalBuilder * externalBuilder)
{
co_await yield();
@@ -793,7 +648,7 @@ Goal::Co DerivationBuildingGoal::buildLocally(
: "building '%s'",
worker.store.printStorePath(drvPath));
buildLog = std::make_unique<BuildLog>(
worker.settings.logLines,
settings.logLines,
std::make_unique<Activity>(
*logger, lvlInfo, actBuild, msg, Logger::Fields{worker.store.printStorePath(drvPath), "", 1, 1}));
mcRunningBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.runningBuilds);
@@ -808,7 +663,7 @@ Goal::Co DerivationBuildingGoal::buildLocally(
while (true) {
unsigned int curBuilds = worker.getNrLocalBuilds();
if (curBuilds >= worker.settings.maxBuildJobs) {
if (curBuilds >= settings.maxBuildJobs) {
outputLocks.unlock();
co_await waitForBuildSlot();
co_return tryToBuild(std::move(inputPaths));
@@ -853,8 +708,7 @@ Goal::Co DerivationBuildingGoal::buildLocally(
}
};
decltype(DerivationBuilderParams::defaultPathsInChroot) defaultPathsInChroot =
localBuildCap.localStore.config->getLocalSettings().sandboxPaths.get();
decltype(DerivationBuilderParams::defaultPathsInChroot) defaultPathsInChroot = settings.sandboxPaths.get();
DesugaredEnv desugaredEnv;
/* Add the closure of store paths to the chroot. */
@@ -878,6 +732,7 @@ Goal::Co DerivationBuildingGoal::buildLocally(
desugaredEnv = DesugaredEnv::create(worker.store, *drv, drvOptions, inputPaths);
} catch (BuildError & e) {
outputLocks.unlock();
worker.permanentFailure = true;
co_return doneFailure(std::move(e));
}
@@ -896,14 +751,14 @@ Goal::Co DerivationBuildingGoal::buildLocally(
/* If we have to wait and retry (see below), then `builder` will
already be created, so we don't need to create it again. */
builder = localBuildCap.externalBuilder
builder = externalBuilder
? makeExternalDerivationBuilder(
localBuildCap.localStore,
localStore,
std::make_unique<DerivationBuildingGoalCallbacks>(*this, openLogFile, closeLogFile),
std::move(params),
*localBuildCap.externalBuilder)
*externalBuilder)
: makeDerivationBuilder(
localBuildCap.localStore,
localStore,
std::make_unique<DerivationBuildingGoalCallbacks>(*this, openLogFile, closeLogFile),
std::move(params));
}
@@ -937,7 +792,7 @@ Goal::Co DerivationBuildingGoal::buildLocally(
if (auto * output = std::get_if<ChildOutput>(&event)) {
if (output->fd == builder->builderOut.get()) {
logSize += output->data.size();
if (worker.settings.maxLogSize && logSize > worker.settings.maxLogSize) {
if (settings.maxLogSize && logSize > settings.maxLogSize) {
builder->killChild();
co_return doneFailureLogTooLong(*buildLog);
}
@@ -966,6 +821,26 @@ Goal::Co DerivationBuildingGoal::buildLocally(
} catch (BuildError & e) {
builder.reset();
outputLocks.unlock();
// Allow selecting a subset of enum values
# pragma GCC diagnostic push
# pragma GCC diagnostic ignored "-Wswitch-enum"
switch (e.status) {
case BuildResult::Failure::HashMismatch:
worker.hashMismatch = true;
/* See header, the protocols don't know about `HashMismatch`
yet, so change it to `OutputRejected`, which they expect
for this case (hash mismatch is a type of output
rejection). */
e.status = BuildResult::Failure::OutputRejected;
break;
case BuildResult::Failure::NotDeterministic:
worker.checkMismatch = true;
break;
default:
/* Other statuses need no adjusting */
break;
}
# pragma GCC diagnostic pop
co_return doneFailure(std::move(e));
}
{
@@ -984,7 +859,7 @@ Goal::Co DerivationBuildingGoal::buildLocally(
worker.markContentsGood(output.outPath);
outputPaths.insert(output.outPath);
}
runPostBuildHook(worker.settings, worker.store, *logger, drvPath, outputPaths);
runPostBuildHook(worker.store, *logger, drvPath, outputPaths);
/* It is now safe to delete the lock files, since all future
lockers will see that the output paths are valid; they will
@@ -998,13 +873,9 @@ Goal::Co DerivationBuildingGoal::buildLocally(
}
static void runPostBuildHook(
const WorkerSettings & workerSettings,
const StoreDirConfig & store,
Logger & logger,
const StorePath & drvPath,
const StorePathSet & outputPaths)
const StoreDirConfig & store, Logger & logger, const StorePath & drvPath, const StorePathSet & outputPaths)
{
auto hook = workerSettings.postBuildHook;
auto hook = settings.postBuildHook;
if (hook == "")
return;
@@ -1012,15 +883,14 @@ static void runPostBuildHook(
logger,
lvlTalkative,
actPostBuildHook,
fmt("running post-build-hook '%s'", workerSettings.postBuildHook),
fmt("running post-build-hook '%s'", settings.postBuildHook),
Logger::Fields{store.printStorePath(drvPath)});
PushActivity pact(act.id);
OsStringMap hookEnvironment = getEnvOs();
StringMap hookEnvironment = getEnv();
hookEnvironment.emplace(OS_STR("DRV_PATH"), string_to_os_string(store.printStorePath(drvPath)));
hookEnvironment.emplace(
OS_STR("OUT_PATHS"), string_to_os_string(chomp(concatStringsSep(" ", store.printStorePathSet(outputPaths)))));
hookEnvironment.emplace(OS_STR("NIX_CONFIG"), string_to_os_string(globalConfig.toKeyValue()));
hookEnvironment.emplace("DRV_PATH", store.printStorePath(drvPath));
hookEnvironment.emplace("OUT_PATHS", chomp(concatStringsSep(" ", store.printStorePathSet(outputPaths))));
hookEnvironment.emplace("NIX_CONFIG", globalConfig.toKeyValue());
struct LogSink : Sink
{
@@ -1061,7 +931,7 @@ static void runPostBuildHook(
LogSink sink(act);
runProgram2({
.program = workerSettings.postBuildHook,
.program = settings.postBuildHook,
.environment = hookEnvironment,
.standardOut = &sink,
.mergeStderrToStdout = true,
@@ -1108,18 +978,17 @@ HookReply DerivationBuildingGoal::tryBuildHook(const DerivationOptions<StorePath
#else
/* This should use `worker.evalStore`, but per #13179 the build hook
doesn't work with eval store anyways. */
if (worker.settings.buildHook.get().empty() || !worker.tryBuildHook || !worker.store.isValidPath(drvPath))
if (settings.buildHook.get().empty() || !worker.tryBuildHook || !worker.store.isValidPath(drvPath))
return rpDecline;
if (!worker.hook)
worker.hook = std::make_unique<HookInstance>(worker.settings.buildHook);
worker.hook = std::make_unique<HookInstance>();
try {
/* Send the request to the hook. */
worker.hook->sink << "try" << (worker.getNrLocalBuilds() < worker.settings.maxBuildJobs ? 1 : 0)
<< drv->platform << worker.store.printStorePath(drvPath)
<< drvOptions.getRequiredSystemFeatures(*drv);
worker.hook->sink << "try" << (worker.getNrLocalBuilds() < settings.maxBuildJobs ? 1 : 0) << drv->platform
<< worker.store.printStorePath(drvPath) << drvOptions.getRequiredSystemFeatures(*drv);
worker.hook->sink.flush();
/* Read the first line of input, which should be a word indicating
@@ -1182,19 +1051,20 @@ LogFile::LogFile(Store & store, const StorePath & drvPath, const LogFileSettings
if (auto localStore = dynamic_cast<LocalStore *>(&store))
logDir = localStore->config->logDir;
else
logDir = logSettings.nixLogDir.string();
logDir = logSettings.nixLogDir;
Path dir = fmt("%s/%s/%s/", logDir, LocalFSStore::drvsLogDir, baseName.substr(0, 2));
createDirs(dir);
Path logFileName = fmt("%s/%s%s", dir, baseName.substr(2), logSettings.compressLog ? ".bz2" : "");
fd = openNewFileForWrite(
logFileName,
0666,
{
.truncateExisting = true,
.followSymlinksOnTruncate = true, /* FIXME: Probably shouldn't follow symlinks. */
});
fd = toDescriptor(open(
logFileName.c_str(),
O_CREAT | O_WRONLY | O_TRUNC
#ifndef _WIN32
| O_CLOEXEC
#endif
,
0666));
if (!fd)
throw SysError("creating log file '%1%'", logFileName);
@@ -1225,7 +1095,7 @@ Goal::Done DerivationBuildingGoal::doneFailureLogTooLong(BuildLog & buildLog)
BuildResult::Failure::LogLimitExceeded,
"%s killed after writing more than %d bytes of log output",
getName(),
worker.settings.maxLogSize));
settings.maxLogSize));
}
std::map<std::string, std::optional<StorePath>> DerivationBuildingGoal::queryPartialDerivationOutputMap()
@@ -1331,13 +1201,22 @@ Goal::Done DerivationBuildingGoal::doneFailure(BuildError ex)
{
mcRunningBuilds.reset();
worker.exitStatusFlags.updateFromStatus(ex.status);
if (ex.status == BuildResult::Failure::TimedOut)
worker.timedOut = true;
if (ex.status == BuildResult::Failure::PermanentFailure)
worker.permanentFailure = true;
if (ex.status != BuildResult::Failure::DependencyFailed)
worker.failedBuilds++;
worker.updateProgress();
return Goal::doneFailure(ecFailed, std::move(ex));
return Goal::doneFailure(
ecFailed,
BuildResult::Failure{
.status = ex.status,
.errorMsg = fmt("%s", Uncolored(ex.info().msg)),
},
std::move(ex));
}
} // namespace nix
+15 -6
View File
@@ -100,7 +100,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation)
/* We are first going to try to create the invalid output paths
through substitutes. If that doesn't work, we'll build
them. */
if (worker.settings.useSubstitutes && drvOptions.substitutesAllowed(worker.settings)) {
if (settings.useSubstitutes && drvOptions.substitutesAllowed()) {
if (!checkResult) {
DrvOutput id{outputHash, wantedOutput};
auto g = worker.makeDrvOutputSubstitutionGoal(id);
@@ -133,7 +133,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation)
assert(!drv->type().isImpure());
if (nrFailed > 0 && nrFailed > nrNoSubstituters && !worker.settings.tryFallback) {
if (nrFailed > 0 && nrFailed > nrNoSubstituters && !settings.tryFallback) {
co_return doneFailure(BuildError(
BuildResult::Failure::TransientFailure,
"some substitutes for the outputs of derivation '%s' failed (usually happens due to networking issues); try '--fallback' to build derivation from source ",
@@ -252,7 +252,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation)
auto g = worker.makeDerivationBuildingGoal(drvPath, *drv, buildMode, storeDerivation);
/* We will finish with it ourselves, as if we were the derivational goal. */
g->preserveFailure = true;
g->preserveException = true;
{
Goals waitees;
@@ -312,7 +312,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation)
}
}
co_return amDone(g->exitCode);
co_return amDone(g->exitCode, g->ex);
}
Goal::Co DerivationGoal::repairClosure()
@@ -492,13 +492,22 @@ Goal::Done DerivationGoal::doneFailure(BuildError ex)
{
mcExpectedBuilds.reset();
worker.exitStatusFlags.updateFromStatus(ex.status);
if (ex.status == BuildResult::Failure::TimedOut)
worker.timedOut = true;
if (ex.status == BuildResult::Failure::PermanentFailure)
worker.permanentFailure = true;
if (ex.status != BuildResult::Failure::DependencyFailed)
worker.failedBuilds++;
worker.updateProgress();
return Goal::doneFailure(ecFailed, std::move(ex));
return Goal::doneFailure(
ecFailed,
BuildResult::Failure{
.status = ex.status,
.errorMsg = fmt("%s", Uncolored(ex.info().msg)),
},
std::move(ex));
}
} // namespace nix
@@ -90,12 +90,7 @@ Goal::Co DerivationResolutionGoal::resolveDerivation()
nrFailed,
nrFailed == 1 ? "dependency" : "dependencies");
msg += showKnownOutputs(worker.store, *drv);
co_return doneFailure(
ecFailed,
BuildResult::Failure{{
.status = BuildResult::Failure::DependencyFailed,
.msg = HintFmt(msg),
}});
co_return amDone(ecFailed, {BuildError(BuildResult::Failure::DependencyFailed, msg)});
}
/* Gather information necessary for computing the closure and/or
@@ -107,7 +102,32 @@ Goal::Co DerivationResolutionGoal::resolveDerivation()
{
auto & fullDrv = *drv;
if (fullDrv.shouldResolve()) {
auto drvType = fullDrv.type();
bool resolveDrv =
std::visit(
overloaded{
[&](const DerivationType::InputAddressed & ia) {
/* must resolve if deferred. */
return ia.deferred;
},
[&](const DerivationType::ContentAddressed & ca) {
return !fullDrv.inputDrvs.map.empty()
&& (ca.fixed
/* Can optionally resolve if fixed, which is good
for avoiding unnecessary rebuilds. */
? experimentalFeatureSettings.isEnabled(Xp::CaDerivations)
/* Must resolve if floating and there are any inputs
drvs. */
: true);
},
[&](const DerivationType::Impure &) { return true; }},
drvType.raw)
/* no inputs are outputs of dynamic derivations */
|| std::ranges::any_of(fullDrv.inputDrvs.map.begin(), fullDrv.inputDrvs.map.end(), [](auto & pair) {
return !pair.second.childMap.empty();
});
if (resolveDrv && !fullDrv.inputDrvs.map.empty()) {
experimentalFeatureSettings.require(Xp::CaDerivations);
/* We are be able to resolve this derivation based on the
@@ -144,7 +164,7 @@ Goal::Co DerivationResolutionGoal::resolveDerivation()
}
assert(attempt);
auto pathResolved = computeStorePath(worker.store, Derivation{*attempt});
auto pathResolved = writeDerivation(worker.store, *attempt, NoRepair, /*readOnly =*/true);
auto msg =
fmt("resolved derivation: '%s' -> '%s'",
@@ -165,7 +185,7 @@ Goal::Co DerivationResolutionGoal::resolveDerivation()
}
}
co_return amDone(ecSuccess);
co_return amDone(ecSuccess, std::nullopt);
}
} // namespace nix
@@ -100,10 +100,11 @@ Goal::Co DerivationTrampolineGoal::init()
if (nrFailed != 0) {
co_return doneFailure(
ecFailed,
BuildResult::Failure{{
BuildResult::Failure{
.status = BuildResult::Failure::DependencyFailed,
.msg = HintFmt("failed to obtain derivation of '%s'", drvReq->to_string(worker.store)),
}});
.errorMsg = fmt("failed to obtain derivation of '%s'", drvReq->to_string(worker.store)),
},
Error("failed to obtain derivation of '%s'", drvReq->to_string(worker.store)));
}
StorePath drvPath = resolveDerivedPath(worker.store, *drvReq);
@@ -151,7 +152,7 @@ Goal::Co DerivationTrampolineGoal::haveDerivation(StorePath drvPath, Derivation
for (auto & output : resolvedWantedOutputs) {
auto g = upcast_goal(worker.makeDerivationGoal(drvPath, drv, output, buildMode, false));
g->preserveFailure = true;
g->preserveException = true;
/* We will finish with it ourselves, as if we were the derivational goal. */
concreteDrvGoals.insert(std::move(g));
}
@@ -169,7 +170,7 @@ Goal::Co DerivationTrampolineGoal::haveDerivation(StorePath drvPath, Derivation
for (auto && [x, y] : successP2->builtOutputs)
successP->builtOutputs.insert_or_assign(x, y);
co_return amDone(g->exitCode);
co_return amDone(g->exitCode, g->ex);
}
} // namespace nix
+22 -21
View File
@@ -18,13 +18,13 @@ void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMod
worker.run(goals);
StringSet failed;
BuildResult::Failure * failure = nullptr;
std::optional<Error> ex;
for (auto & i : goals) {
if (auto * f = i->buildResult.tryGetFailure()) {
if (failure)
logError(f->info());
if (i->ex) {
if (ex)
logError(i->ex->info());
else
failure = f;
ex = std::move(i->ex);
}
if (i->exitCode != Goal::ecSuccess) {
if (auto i2 = dynamic_cast<DerivationTrampolineGoal *>(i.get()))
@@ -34,14 +34,13 @@ void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMod
}
}
if (failed.size() == 1 && failure) {
failure->withExitStatus(worker.exitStatusFlags.failingExitStatus());
throw *failure;
if (failed.size() == 1 && ex) {
ex->withExitStatus(worker.failingExitStatus());
throw std::move(*ex);
} else if (!failed.empty()) {
auto exitStatus = worker.exitStatusFlags.failingExitStatus();
if (failure)
logError(failure->info());
throw Error(exitStatus, "build of %s failed", concatStringsSep(", ", quoteStrings(failed)));
if (ex)
logError(ex->info());
throw Error(worker.failingExitStatus(), "build of %s failed", concatStringsSep(", ", quoteStrings(failed)));
}
}
@@ -89,11 +88,10 @@ BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivat
worker.run(Goals{goal});
return goal->buildResult;
} catch (Error & e) {
return BuildResult{
.inner = BuildResult::Failure{{
.status = BuildResult::Failure::MiscFailure,
.msg = e.msg(),
}}};
return BuildResult{.inner{BuildResult::Failure{
.status = BuildResult::Failure::MiscFailure,
.errorMsg = e.msg(),
}}};
};
}
@@ -110,9 +108,12 @@ void Store::ensurePath(const StorePath & path)
worker.run(goals);
if (goal->exitCode != Goal::ecSuccess) {
auto exitStatus = worker.exitStatusFlags.failingExitStatus();
goal->buildResult.tryThrowBuildError(exitStatus);
throw Error(exitStatus, "path '%s' does not exist and cannot be created", printStorePath(path));
if (goal->ex) {
goal->ex->withExitStatus(worker.failingExitStatus());
throw std::move(*goal->ex);
} else
throw Error(
worker.failingExitStatus(), "path '%s' does not exist and cannot be created", printStorePath(path));
}
}
@@ -139,7 +140,7 @@ void Store::repairPath(const StorePath & path)
bmRepair));
worker.run(goals);
} else
throw Error(worker.exitStatusFlags.failingExitStatus(), "cannot repair path '%s'", printStorePath(path));
throw Error(worker.failingExitStatus(), "cannot repair path '%s'", printStorePath(path));
}
}
+9 -13
View File
@@ -189,14 +189,14 @@ Goal::Done Goal::doneSuccess(BuildResult::Success success)
return amDone(ecSuccess);
}
Goal::Done Goal::doneFailure(ExitCode result, BuildResult::Failure failure)
Goal::Done Goal::doneFailure(ExitCode result, BuildResult::Failure failure, std::optional<Error> ex)
{
assert(result == ecFailed || result == ecNoSubstituters);
buildResult.inner = std::move(failure);
return amDone(result);
return amDone(result, std::move(ex));
}
Goal::Done Goal::amDone(ExitCode result)
Goal::Done Goal::amDone(ExitCode result, std::optional<Error> ex)
{
trace("done");
assert(top_co);
@@ -204,15 +204,11 @@ Goal::Done Goal::amDone(ExitCode result)
assert(result == ecSuccess || result == ecFailed || result == ecNoSubstituters);
exitCode = result;
// Log the failure if we have one and shouldn't preserve it.
// Only log for actual failures (ecFailed), not for ecNoSubstituters
// which indicates "couldn't substitute, will try building" - that's
// expected behavior, not an error.
if (result == ecFailed) {
if (auto * failure = buildResult.tryGetFailure()) {
if (!preserveFailure && !waiters.empty())
logError(failure->info());
}
if (ex) {
if (!preserveException && !waiters.empty())
logError(ex->info());
else
this->ex = std::move(*ex);
}
for (auto & i : waiters) {
@@ -232,7 +228,7 @@ Goal::Done Goal::amDone(ExitCode result)
if (goal->waitees.empty()) {
worker.wakeUp(goal);
} else if (result == ecFailed && !worker.settings.keepGoing) {
} else if (result == ecFailed && !settings.keepGoing) {
/* If we failed and keepGoing is not set, we remove all
remaining waitees. */
for (auto & g : goal->waitees) {
+29 -16
View File
@@ -26,6 +26,25 @@ PathSubstitutionGoal::~PathSubstitutionGoal()
cleanup();
}
Goal::Done PathSubstitutionGoal::doneSuccess(BuildResult::Success::Status status)
{
return Goal::doneSuccess(
BuildResult::Success{
.status = status,
});
}
Goal::Done PathSubstitutionGoal::doneFailure(ExitCode result, BuildResult::Failure::Status status, std::string errorMsg)
{
debug(errorMsg);
return Goal::doneFailure(
result,
BuildResult::Failure{
.status = status,
.errorMsg = std::move(errorMsg),
});
}
Goal::Co PathSubstitutionGoal::init()
{
trace("init");
@@ -34,10 +53,10 @@ Goal::Co PathSubstitutionGoal::init()
/* If the path already exists we're done. */
if (!repair && worker.store.isValidPath(storePath)) {
co_return doneSuccess(BuildResult::Success{.status = BuildResult::Success::AlreadyValid});
co_return doneSuccess(BuildResult::Success::AlreadyValid);
}
if (worker.store.config.getReadOnly())
if (settings.readOnlyMode)
throw Error(
"cannot substitute path '%s' - no write access to the Nix store", worker.store.printStorePath(storePath));
@@ -145,7 +164,7 @@ Goal::Co PathSubstitutionGoal::init()
worker.updateProgress();
}
if (lastStoresException.has_value()) {
if (!worker.settings.tryFallback) {
if (!settings.tryFallback) {
throw *lastStoresException;
} else
logError(lastStoresException->info());
@@ -156,12 +175,9 @@ Goal::Co PathSubstitutionGoal::init()
build. */
co_return doneFailure(
substituterFailed ? ecFailed : ecNoSubstituters,
BuildResult::Failure{{
.status = BuildResult::Failure::NoSubstituters,
.msg = HintFmt(
"path '%s' is required, but there is no substituter that can build it",
worker.store.printStorePath(storePath)),
}});
BuildResult::Failure::NoSubstituters,
fmt("path '%s' is required, but there is no substituter that can build it",
worker.store.printStorePath(storePath)));
}
Goal::Co PathSubstitutionGoal::tryToRun(
@@ -172,11 +188,8 @@ Goal::Co PathSubstitutionGoal::tryToRun(
if (nrFailed > 0) {
co_return doneFailure(
nrNoSubstituters > 0 ? ecNoSubstituters : ecFailed,
BuildResult::Failure{{
.status = BuildResult::Failure::DependencyFailed,
.msg = HintFmt(
"some references of path '%s' could not be realised", worker.store.printStorePath(storePath)),
}});
BuildResult::Failure::DependencyFailed,
fmt("some references of path '%s' could not be realised", worker.store.printStorePath(storePath)));
}
for (auto & i : info->references)
@@ -197,7 +210,7 @@ Goal::Co PathSubstitutionGoal::tryToRun(
/* Make sure that we are allowed to start a substitution. Note that even
if maxSubstitutionJobs == 0, we still allow a substituter to run. This
prevents infinite waiting. */
while (worker.getNrSubstitutions() >= std::max(1U, (unsigned int) worker.settings.maxSubstitutionJobs)) {
while (worker.getNrSubstitutions() >= std::max(1U, (unsigned int) settings.maxSubstitutionJobs)) {
co_await waitForBuildSlot();
}
@@ -303,7 +316,7 @@ Goal::Co PathSubstitutionGoal::tryToRun(
worker.updateProgress();
co_return doneSuccess(BuildResult::Success{.status = BuildResult::Success::Substituted});
co_return doneSuccess(BuildResult::Success::Substituted);
}
void PathSubstitutionGoal::cleanup()
+27 -6
View File
@@ -22,14 +22,15 @@ Worker::Worker(Store & store, Store & evalStore)
, actSubstitutions(*logger, actCopyPaths)
, store(store)
, evalStore(evalStore)
, settings(nix::settings.getWorkerSettings())
, getSubstituters{[] {
return nix::settings.getWorkerSettings().useSubstitutes ? getDefaultSubstituters() : std::list<ref<Store>>{};
}}
, getSubstituters{[] { return settings.useSubstitutes ? getDefaultSubstituters() : std::list<ref<Store>>{}; }}
{
nrLocalBuilds = 0;
nrSubstitutions = 0;
lastWokenUp = steady_time_point::min();
permanentFailure = false;
timedOut = false;
hashMismatch = false;
checkMismatch = false;
}
Worker::~Worker()
@@ -367,7 +368,7 @@ void Worker::run(const Goals & _topGoals)
if (!children.empty() || !waitingForAWhile.empty())
waitForInput();
else if (awake.empty() && 0U == settings.maxBuildJobs) {
if (Machine::parseConfig({nix::settings.thisSystem}, nix::settings.getWorkerSettings().builders).empty())
if (getMachines().empty())
throw Error(
"Unable to start any build; either increase '--max-jobs' or enable remote builds.\n"
"\n"
@@ -409,7 +410,7 @@ void Worker::waitForInput()
auto nearest = steady_time_point::max(); // nearest deadline
auto localStore = dynamic_cast<LocalStore *>(&store);
if (localStore && localStore->config->getLocalSettings().getGCSettings().minFree.get() != 0)
if (localStore && localStore->config->getGCSettings().minFree.get() != 0)
// If we have a local store (and thus are capable of automatically collecting garbage) and configured to do so,
// periodically wake up to see if we need to run the garbage collector. (See the `autoGC` call site above in
// this file, also gated on having a local store. when we wake up, we intended to reach that call site.)
@@ -512,6 +513,26 @@ void Worker::waitForInput()
}
}
unsigned int Worker::failingExitStatus()
{
// See API docs in header for explanation
unsigned int mask = 0;
bool buildFailure = permanentFailure || timedOut || hashMismatch;
if (buildFailure)
mask |= 0x04; // 100
if (timedOut)
mask |= 0x01; // 101
if (hashMismatch)
mask |= 0x02; // 102
if (checkMismatch) {
mask |= 0x08; // 104
}
if (mask)
mask |= 0x60;
return mask ? mask : 1;
}
bool Worker::pathContentsGood(const StorePath & path)
{
auto i = pathContentsGoodCache.find(path);
+5 -5
View File
@@ -14,12 +14,12 @@ static void builtinFetchurl(const BuiltinBuilderContext & ctx)
this to be stored in a file. It would be nice if we could just
pass a pointer to the data. */
if (ctx.netrcData != "") {
fileTransferSettings.netrcFile = "netrc";
writeFile(fileTransferSettings.netrcFile, ctx.netrcData, 0600);
settings.netrcFile = "netrc";
writeFile(settings.netrcFile, ctx.netrcData, 0600);
}
fileTransferSettings.caFile = "ca-certificates.crt";
writeFile(*fileTransferSettings.caFile.get(), ctx.caFileData, 0600);
settings.caFile = "ca-certificates.crt";
writeFile(settings.caFile, ctx.caFileData, 0600);
auto out = get(ctx.drv.outputs, "out");
if (!out)
@@ -73,7 +73,7 @@ static void builtinFetchurl(const BuiltinBuilderContext & ctx)
/* Try the hashed mirrors first. */
auto dof = std::get_if<DerivationOutput::CAFixed>(&out->raw);
if (dof && dof->ca.method.getFileIngestionMethod() == FileIngestionMethod::Flat)
for (auto hashedMirror : ctx.hashedMirrors)
for (auto hashedMirror : settings.hashedMirrors.get())
try {
if (!hasSuffix(hashedMirror, "/"))
hashedMirror += '/';
+2 -2
View File
@@ -36,8 +36,8 @@ static void builtinUnpackChannel(const BuiltinBuilderContext & ctx)
auto target = out / channelName;
try {
std::filesystem::rename(fileName, target);
} catch (std::filesystem::filesystem_error & e) {
throw SystemError(e.code(), "failed to rename %1% to %2%", fileName, target.string());
} catch (std::filesystem::filesystem_error &) {
throw SysError("failed to rename %1% to %2%", fileName, target.string());
}
}
-7
View File
@@ -150,13 +150,6 @@ CommonProto::Serialise<BuildResultStatus>::read(const StoreDirConfig & store, Co
void CommonProto::Serialise<BuildResultStatus>::write(
const StoreDirConfig & store, CommonProto::WriteConn conn, const BuildResultStatus & status)
{
/* See definition, the protocols don't know about `HashMismatch`
yet, so change it to `OutputRejected`, which they expect
for this case (hash mismatch is a type of output
rejection). */
if (status == BuildResultStatus{BuildResultFailureStatus::HashMismatch}) {
return write(store, conn, BuildResultFailureStatus::OutputRejected);
}
for (auto && [wire, val] : enumerate(buildResultStatusTable))
if (val == status) {
conn.to << uint8_t(wire);
+40 -52
View File
@@ -133,7 +133,7 @@ struct TunnelLogger : public Logger
if (!ex)
to << STDERR_LAST;
else {
if (clientVersion >= WorkerProto::Version{.number = {1, 26}}) {
if (GET_PROTOCOL_MINOR(clientVersion) >= 26) {
to << STDERR_ERROR << *ex;
} else {
to << STDERR_ERROR << ex->what() << ex->info().status;
@@ -149,7 +149,7 @@ struct TunnelLogger : public Logger
const Fields & fields,
ActivityId parent) override
{
if (clientVersion.number < WorkerProto::Version::Number{1, 20}) {
if (GET_PROTOCOL_MINOR(clientVersion) < 20) {
if (!s.empty())
log(lvl, s + "...");
return;
@@ -162,7 +162,7 @@ struct TunnelLogger : public Logger
void stopActivity(ActivityId act) override
{
if (clientVersion.number < WorkerProto::Version::Number{1, 20})
if (GET_PROTOCOL_MINOR(clientVersion) < 20)
return;
StringSink buf;
buf << STDERR_STOP_ACTIVITY << act;
@@ -171,7 +171,7 @@ struct TunnelLogger : public Logger
void result(ActivityId act, ResultType type, const Fields & fields) override
{
if (clientVersion.number < WorkerProto::Version::Number{1, 20})
if (GET_PROTOCOL_MINOR(clientVersion) < 20)
return;
StringSink buf;
buf << STDERR_RESULT << act << type << fields;
@@ -233,49 +233,38 @@ struct ClientSettings
void apply(TrustedFlag trusted)
{
settings.keepFailed = keepFailed;
settings.getWorkerSettings().keepGoing = keepGoing;
settings.getWorkerSettings().tryFallback = tryFallback;
settings.keepGoing = keepGoing;
settings.tryFallback = tryFallback;
nix::verbosity = verbosity;
settings.getWorkerSettings().maxBuildJobs.assign(maxBuildJobs);
settings.getWorkerSettings().maxSilentTime = maxSilentTime;
settings.maxBuildJobs.assign(maxBuildJobs);
settings.maxSilentTime = maxSilentTime;
settings.verboseBuild = verboseBuild;
settings.getLocalSettings().buildCores = buildCores;
settings.getWorkerSettings().useSubstitutes = useSubstitutes;
settings.buildCores = buildCores;
settings.useSubstitutes = useSubstitutes;
for (auto & i : overrides) {
auto & name(i.first);
auto & value(i.second);
auto setSubstituters = [&](Setting<std::vector<StoreReference>> & res) {
auto setSubstituters = [&](Setting<Strings> & res) {
if (name != res.name && res.aliases.count(name) == 0)
return false;
std::set<StoreReference> trusted = settings.trustedSubstituters;
for (auto & ref : settings.getWorkerSettings().substituters.get())
trusted.insert(ref);
std::vector<StoreReference> subs;
StringSet trusted = settings.trustedSubstituters;
for (auto & s : settings.substituters.get())
trusted.insert(s);
Strings subs;
auto ss = tokenizeString<Strings>(value);
for (auto & s : ss) {
auto ref = StoreReference::parse(s);
auto tryTrust = [&] {
if (trusted.count(ref))
return true;
if (auto * specified = std::get_if<StoreReference::Specified>(&ref.variant);
specified && !hasSuffix(specified->authority, "/")) {
specified->authority += "/";
if (trusted.count(ref))
return true;
}
return false;
};
if (tryTrust())
subs.push_back(std::move(ref));
for (auto & s : ss)
if (trusted.count(s))
subs.push_back(s);
else if (!hasSuffix(s, "/") && trusted.count(s + "/"))
subs.push_back(s + "/");
else
warn(
"ignoring untrusted substituter '%s', you are not a trusted user.\n"
"Run `man nix.conf` for more information on the `substituters` configuration option.",
s);
}
res = std::move(subs);
res = subs;
return true;
};
@@ -293,12 +282,11 @@ struct ClientSettings
"Ignoring the client-specified plugin-files.\n"
"The client specifying plugins to the daemon never made sense, and was removed in Nix >=2.14.");
} else if (
trusted || name == settings.getWorkerSettings().buildTimeout.name
|| name == settings.getWorkerSettings().maxSilentTime.name
|| name == settings.getWorkerSettings().pollInterval.name || name == "connect-timeout"
trusted || name == settings.buildTimeout.name || name == settings.maxSilentTime.name
|| name == settings.pollInterval.name || name == "connect-timeout"
|| (name == "builders" && value == ""))
settings.set(name, value);
else if (setSubstituters(settings.getWorkerSettings().substituters))
else if (setSubstituters(settings.substituters))
;
else
warn(
@@ -337,7 +325,7 @@ static void performOp(
auto paths = WorkerProto::Serialise<StorePathSet>::read(*store, rconn);
SubstituteFlag substitute = NoSubstitute;
if (conn.protoVersion >= WorkerProto::Version{.number = {1, 27}}) {
if (GET_PROTOCOL_MINOR(conn.protoVersion) >= 27) {
substitute = readInt(conn.from) ? Substitute : NoSubstitute;
}
@@ -414,7 +402,7 @@ static void performOp(
}
case WorkerProto::Op::AddToStore: {
if (conn.protoVersion >= WorkerProto::Version{.number = {1, 25}}) {
if (GET_PROTOCOL_MINOR(conn.protoVersion) >= 25) {
auto name = readString(conn.from);
auto camStr = readString(conn.from);
auto refs = WorkerProto::Serialise<StorePathSet>::read(*store, rconn);
@@ -647,7 +635,7 @@ static void performOp(
Derivation drv2;
static_cast<BasicDerivation &>(drv2) = drv;
drvPath = store->writeDerivation(Derivation{drv2});
drvPath = writeDerivation(*store, Derivation{drv2});
}
auto res = store->buildDerivation(drvPath, drv, buildMode);
@@ -810,7 +798,7 @@ static void performOp(
case WorkerProto::Op::QuerySubstitutablePathInfos: {
SubstitutablePathInfos infos;
StorePathCAMap pathsMap = {};
if (conn.protoVersion.number < WorkerProto::Version::Number{1, 22}) {
if (GET_PROTOCOL_MINOR(conn.protoVersion) < 22) {
auto paths = WorkerProto::Serialise<StorePathSet>::read(*store, rconn);
for (auto & path : paths)
pathsMap.emplace(path, std::nullopt);
@@ -841,10 +829,7 @@ static void performOp(
auto path = WorkerProto::Serialise<StorePath>::read(*store, rconn);
std::shared_ptr<const ValidPathInfo> info;
logger->startWork();
try {
info = store->queryPathInfo(path);
} catch (InvalidPath &) {
}
info = store->queryPathInfo(path);
logger->stopWork();
if (info) {
conn.to << 1;
@@ -909,7 +894,7 @@ static void performOp(
if (!trusted)
info.ultimate = false;
if (conn.protoVersion >= WorkerProto::Version{.number = {1, 23}}) {
if (GET_PROTOCOL_MINOR(conn.protoVersion) >= 23) {
logger->startWork();
{
FramedSource source(conn.from);
@@ -921,7 +906,7 @@ static void performOp(
else {
std::unique_ptr<Source> source;
StringSink saved;
if (conn.protoVersion >= WorkerProto::Version{.number = {1, 21}})
if (GET_PROTOCOL_MINOR(conn.protoVersion) >= 21)
source = std::make_unique<TunnelSource>(conn.from, conn.to);
else {
TeeSource tee{conn.from, saved};
@@ -955,7 +940,7 @@ static void performOp(
case WorkerProto::Op::RegisterDrvOutput: {
logger->startWork();
if (conn.protoVersion.number < WorkerProto::Version::Number{1, 31}) {
if (GET_PROTOCOL_MINOR(conn.protoVersion) < 31) {
auto outputId = WorkerProto::Serialise<DrvOutput>::read(*store, rconn);
auto outputPath = StorePath(readString(conn.from));
store->registerDrvOutput(Realisation{{.outPath = outputPath}, outputId});
@@ -972,7 +957,7 @@ static void performOp(
auto outputId = WorkerProto::Serialise<DrvOutput>::read(*store, rconn);
auto info = store->queryRealisation(outputId);
logger->stopWork();
if (conn.protoVersion.number < WorkerProto::Version::Number{1, 31}) {
if (GET_PROTOCOL_MINOR(conn.protoVersion) < 31) {
std::set<StorePath> outPaths;
if (info)
outPaths.insert(info->outPath);
@@ -1027,16 +1012,19 @@ void processConnection(ref<Store> store, FdSource && from, FdSink && to, Trusted
#endif
/* Exchange the greeting. */
WorkerProto::BasicServerConnection conn;
conn.protoVersion = WorkerProto::BasicServerConnection::handshake(to, from, WorkerProto::latest);
auto [protoVersion, features] =
WorkerProto::BasicServerConnection::handshake(to, from, PROTOCOL_VERSION, WorkerProto::allFeatures);
if (conn.protoVersion.number < WorkerProto::minimum.number)
if (protoVersion < MINIMUM_PROTOCOL_VERSION)
throw Error("the Nix client version is too old");
WorkerProto::BasicServerConnection conn;
conn.to = std::move(to);
conn.from = std::move(from);
conn.protoVersion = protoVersion;
conn.features = features;
auto tunnelLogger_ = std::make_unique<TunnelLogger>(conn.to, conn.protoVersion);
auto tunnelLogger_ = std::make_unique<TunnelLogger>(conn.to, protoVersion);
auto tunnelLogger = tunnelLogger_.get();
std::unique_ptr<Logger> prevLogger_;
auto prevLogger = logger.get();
+26 -2
View File
@@ -6,6 +6,7 @@
#include "nix/store/store-api.hh"
#include "nix/util/types.hh"
#include "nix/util/util.hh"
#include "nix/store/globals.hh"
#include "nix/util/variant-wrapper.hh"
#include <optional>
@@ -359,9 +360,32 @@ StringSet DerivationOptions<Input>::getRequiredSystemFeatures(const BasicDerivat
}
template<typename Input>
bool DerivationOptions<Input>::substitutesAllowed(const WorkerSettings & workerSettings) const
bool DerivationOptions<Input>::canBuildLocally(Store & localStore, const BasicDerivation & drv) const
{
return workerSettings.alwaysAllowSubstitutes ? true : allowSubstitutes;
if (drv.platform != settings.thisSystem.get() && !settings.extraPlatforms.get().count(drv.platform)
&& !drv.isBuiltin())
return false;
if (settings.maxBuildJobs.get() == 0 && !drv.isBuiltin())
return false;
for (auto & feature : getRequiredSystemFeatures(drv))
if (!localStore.config.systemFeatures.get().count(feature))
return false;
return true;
}
template<typename Input>
bool DerivationOptions<Input>::willBuildLocally(Store & localStore, const BasicDerivation & drv) const
{
return preferLocalBuild && canBuildLocally(localStore, drv);
}
template<typename Input>
bool DerivationOptions<Input>::substitutesAllowed() const
{
return settings.alwaysAllowSubstitutes ? true : allowSubstitutes;
}
template<typename Input>
+7 -43
View File
@@ -106,7 +106,7 @@ bool BasicDerivation::isBuiltin() const
return builder.substr(0, 8) == "builtin:";
}
static auto infoForDerivation(const StoreDirConfig & store, const Derivation & drv)
static auto infoForDerivation(Store & store, const Derivation & drv)
{
auto references = drv.inputSrcs;
for (auto & i : drv.inputDrvs.map)
@@ -126,22 +126,19 @@ static auto infoForDerivation(const StoreDirConfig & store, const Derivation & d
};
}
StorePath computeStorePath(const StoreDirConfig & store, const Derivation & drv)
StorePath writeDerivation(Store & store, const Derivation & drv, RepairFlag repair, bool readOnly)
{
auto [_suffix, _contents, _references, path] = infoForDerivation(store, drv);
return path;
if (readOnly || settings.readOnlyMode) {
auto [_x, _y, _z, path] = infoForDerivation(store, drv);
return path;
} else
return store.writeDerivation(drv, repair);
}
StorePath Store::writeDerivation(const Derivation & drv, RepairFlag repair)
{
auto [suffix, contents, references, path] = infoForDerivation(*this, drv);
/* In case the derivation is already valid, we bail out early since that's
faster. But we need to make sure that the derivation has a corresponding
temproot. It is added by the remote in addToStoreFromDump, but we'd like
to avoid sending a lot of drv contents to the daemon. */
addTempRoot(path);
if (isValidPath(path) && !repair)
return path;
@@ -1122,39 +1119,6 @@ static void rewriteDerivation(Store & store, BasicDerivation & drv, const String
}
}
bool Derivation::shouldResolve() const
{
/* No input drvs means nothing to resolve. */
if (inputDrvs.map.empty())
return false;
auto drvType = type();
bool typeNeedsResolve = std::visit(
overloaded{
[&](const DerivationType::InputAddressed & ia) {
/* Must resolve if deferred. */
return ia.deferred;
},
[&](const DerivationType::ContentAddressed & ca) {
return ca.fixed
/* Can optionally resolve if fixed, which is good
for avoiding unnecessary rebuilds. */
? experimentalFeatureSettings.isEnabled(Xp::CaDerivations)
/* Must resolve if floating. */
: true;
},
[&](const DerivationType::Impure &) { return true; },
},
drvType.raw);
/* Also need to resolve if any inputs are outputs of dynamic derivations. */
bool hasDynamicInputs = std::ranges::any_of(
inputDrvs.map.begin(), inputDrvs.map.end(), [](auto & pair) { return !pair.second.childMap.empty(); });
return typeNeedsResolve || hasDynamicInputs;
}
std::optional<BasicDerivation> Derivation::tryResolve(Store & store, Store * evalStore) const
{
return tryResolve(
+1 -6
View File
@@ -119,11 +119,6 @@ ref<Store> DummyStoreConfig::openStore() const
return openDummyStore();
}
bool DummyStoreConfig::getReadOnly() const
{
return readOnly.get() || StoreConfig::getReadOnly();
}
struct DummyStoreImpl : DummyStore
{
using Config = DummyStoreConfig;
@@ -301,7 +296,7 @@ struct DummyStoreImpl : DummyStore
StorePath writeDerivation(const Derivation & drv, RepairFlag repair = NoRepair) override
{
auto drvPath = nix::computeStorePath(*this, drv);
auto drvPath = ::nix::writeDerivation(*this, drv, repair, /*readonly=*/true);
if (!derivations.contains(drvPath) || repair) {
if (config->readOnly)
+18 -46
View File
@@ -35,22 +35,6 @@ namespace nix {
const unsigned int RETRY_TIME_MS_DEFAULT = 250;
const unsigned int RETRY_TIME_MS_TOO_MANY_REQUESTS = 60000;
std::filesystem::path FileTransferSettings::getDefaultSSLCertFile()
{
for (auto & fn :
{"/etc/ssl/certs/ca-certificates.crt", "/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt"})
if (pathAccessible(fn))
return fn;
return "";
}
FileTransferSettings::FileTransferSettings()
{
auto sslOverride = getEnv("NIX_SSL_CERT_FILE").value_or(getEnv("SSL_CERT_FILE").value_or(""));
if (sslOverride != "")
caFile = sslOverride;
}
FileTransferSettings fileTransferSettings;
static GlobalConfig::Register rFileTransferSettings(&fileTransferSettings);
@@ -75,8 +59,6 @@ struct curlMultiError : Error
struct curlFileTransfer : public FileTransfer
{
const FileTransferSettings & settings;
curlMulti curlm;
std::random_device rd;
@@ -482,11 +464,10 @@ struct curlFileTransfer : public FileTransfer
req,
CURLOPT_USERAGENT,
("curl/" LIBCURL_VERSION " Nix/" + nixVersion
+ (fileTransfer.settings.userAgentSuffix != "" ? " " + fileTransfer.settings.userAgentSuffix.get()
: ""))
+ (fileTransferSettings.userAgentSuffix != "" ? " " + fileTransferSettings.userAgentSuffix.get() : ""))
.c_str());
curl_easy_setopt(req, CURLOPT_PIPEWAIT, 1);
if (fileTransfer.settings.enableHttp2)
if (fileTransferSettings.enableHttp2)
curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2TLS);
else
curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1);
@@ -501,9 +482,8 @@ struct curlFileTransfer : public FileTransfer
curl_easy_setopt(req, CURLOPT_HTTPHEADER, requestHeaders.get());
if (fileTransfer.settings.downloadSpeed.get() > 0)
curl_easy_setopt(
req, CURLOPT_MAX_RECV_SPEED_LARGE, (curl_off_t) (fileTransfer.settings.downloadSpeed.get() * 1024));
if (settings.downloadSpeed.get() > 0)
curl_easy_setopt(req, CURLOPT_MAX_RECV_SPEED_LARGE, (curl_off_t) (settings.downloadSpeed.get() * 1024));
if (request.method == HttpMethod::Head)
curl_easy_setopt(req, CURLOPT_NOBODY, 1);
@@ -532,26 +512,20 @@ struct curlFileTransfer : public FileTransfer
curl_easy_setopt(req, CURLOPT_SEEKDATA, this);
}
if (auto & caFile = fileTransfer.settings.caFile.get())
curl_easy_setopt(req, CURLOPT_CAINFO, caFile->c_str());
if (settings.caFile != "")
curl_easy_setopt(req, CURLOPT_CAINFO, settings.caFile.get().c_str());
#if !defined(_WIN32)
curl_easy_setopt(req, CURLOPT_SOCKOPTFUNCTION, cloexec_callback);
#endif
curl_easy_setopt(req, CURLOPT_CONNECTTIMEOUT, fileTransfer.settings.connectTimeout.get());
// Enable TCP keep-alive so that idle connections in curl's reuse pool
// are not silently dropped by NATs, firewalls, or load balancers.
curl_easy_setopt(req, CURLOPT_TCP_KEEPALIVE, 1L);
curl_easy_setopt(req, CURLOPT_TCP_KEEPIDLE, 60L);
curl_easy_setopt(req, CURLOPT_TCP_KEEPINTVL, 60L);
curl_easy_setopt(req, CURLOPT_CONNECTTIMEOUT, fileTransferSettings.connectTimeout.get());
curl_easy_setopt(req, CURLOPT_LOW_SPEED_LIMIT, 1L);
curl_easy_setopt(req, CURLOPT_LOW_SPEED_TIME, fileTransfer.settings.stalledDownloadTimeout.get());
curl_easy_setopt(req, CURLOPT_LOW_SPEED_TIME, fileTransferSettings.stalledDownloadTimeout.get());
/* If no file exist in the specified path, curl continues to work
anyway as if netrc support was disabled. */
curl_easy_setopt(req, CURLOPT_NETRC_FILE, fileTransfer.settings.netrcFile.get().c_str());
curl_easy_setopt(req, CURLOPT_NETRC_FILE, settings.netrcFile.get().c_str());
curl_easy_setopt(req, CURLOPT_NETRC, CURL_NETRC_OPTIONAL);
if (writtenToSink)
@@ -737,7 +711,7 @@ struct curlFileTransfer : public FileTransfer
download after a while. If we're writing to a
sink, we can only retry if the server supports
ranged requests. */
if (err == Transient && attempt < fileTransfer.settings.tries
if (err == Transient && attempt < request.tries
&& (!this->request.dataCallback || writtenToSink == 0 || (acceptRanges && encoding.empty()))) {
int ms = retryTimeMs
* std::pow(
@@ -797,12 +771,10 @@ struct curlFileTransfer : public FileTransfer
std::thread workerThread;
const size_t maxQueueSize;
const size_t maxQueueSize = fileTransferSettings.httpConnections.get() * 5;
curlFileTransfer(const FileTransferSettings & settings)
: settings(settings)
, mt19937(rd())
, maxQueueSize(settings.httpConnections.get() * 5)
curlFileTransfer()
: mt19937(rd())
{
static std::once_flag globalInit;
std::call_once(globalInit, curl_global_init, CURL_GLOBAL_ALL);
@@ -810,7 +782,7 @@ struct curlFileTransfer : public FileTransfer
curlm = curlMulti(curl_multi_init());
curl_multi_setopt(curlm.get(), CURLMOPT_PIPELINING, CURLPIPE_MULTIPLEX);
curl_multi_setopt(curlm.get(), CURLMOPT_MAX_TOTAL_CONNECTIONS, settings.httpConnections.get());
curl_multi_setopt(curlm.get(), CURLMOPT_MAX_TOTAL_CONNECTIONS, fileTransferSettings.httpConnections.get());
workerThread = std::thread([&]() { workerThreadEntry(); });
}
@@ -1011,9 +983,9 @@ struct curlFileTransfer : public FileTransfer
}
};
ref<curlFileTransfer> makeCurlFileTransfer(const FileTransferSettings & settings = fileTransferSettings)
ref<curlFileTransfer> makeCurlFileTransfer()
{
return make_ref<curlFileTransfer>(settings);
return make_ref<curlFileTransfer>();
}
ref<FileTransfer> getFileTransfer()
@@ -1026,9 +998,9 @@ ref<FileTransfer> getFileTransfer()
return fileTransfer;
}
ref<FileTransfer> makeFileTransfer(const FileTransferSettings & settings)
ref<FileTransfer> makeFileTransfer()
{
return makeCurlFileTransfer(settings);
return makeCurlFileTransfer();
}
void FileTransferRequest::setupForS3()
+148 -37
View File
@@ -1,9 +1,7 @@
#include "nix/store/derivations.hh"
#include "nix/store/globals.hh"
#include "nix/store/local-gc.hh"
#include "nix/store/local-store.hh"
#include "nix/store/path.hh"
#include "nix/util/configuration.hh"
#include "nix/util/finally.hh"
#include "nix/util/unix-domain-socket.hh"
#include "nix/util/signals.hh"
@@ -14,6 +12,11 @@
#include "store-config-private.hh"
#if !defined(__linux__)
// For shelling out to lsof
# include "nix/util/processes.hh"
#endif
#include <boost/unordered/unordered_flat_map.hpp>
#include <boost/unordered/unordered_flat_set.hpp>
#include <boost/regex.hpp>
@@ -66,7 +69,8 @@ void LocalStore::createTempRootsFile()
/* Check whether the garbage collector didn't get in our
way. */
if (getFileSize(fdTempRoots->get()) == 0)
auto st = nix::fstat(fromDescriptorReadOnly(fdTempRoots->get()));
if (st.st_size == 0)
break;
/* The garbage collector deleted this file before we could get
@@ -272,7 +276,7 @@ void LocalStore::findRoots(const Path & path, std::filesystem::file_type type, R
|| e.code() == std::errc::not_a_directory)
printInfo("cannot read potential root '%1%'", path);
else
throw SystemError(e.code(), "finding GC roots in '%1%'", path);
throw;
}
catch (SystemError & e) {
@@ -307,42 +311,151 @@ Roots LocalStore::findRoots(bool censor)
return roots;
}
static Roots requestRuntimeRoots(const LocalStoreConfig & config, const std::filesystem::path & socketPath)
/**
* Key is a mere string because cannot has path with macOS's libc++
*/
typedef boost::unordered_flat_map<
std::string,
boost::unordered_flat_set<std::string, StringViewHash, std::equal_to<>>,
StringViewHash,
std::equal_to<>>
UncheckedRoots;
static void readProcLink(const std::filesystem::path & file, UncheckedRoots & roots)
{
Roots roots;
auto socket = connect(socketPath);
auto socketSource = FdSource(socket.get());
while (1) {
auto line = socketSource.readLine(true, '\0');
if (line == "")
break;
roots[config.parseStorePath(line)].insert(censored);
};
return roots;
std::filesystem::path buf;
try {
buf = std::filesystem::read_symlink(file);
} catch (std::filesystem::filesystem_error & e) {
if (e.code() == std::errc::no_such_file_or_directory || e.code() == std::errc::permission_denied
|| e.code() == std::errc::no_such_process)
return;
throw;
}
if (buf.is_absolute())
roots[buf.string()].emplace(file.string());
}
static std::string quoteRegexChars(const std::string & raw)
{
static auto specialRegex = boost::regex(R"([.^$\\*+?()\[\]{}|])");
return boost::regex_replace(raw, specialRegex, R"(\$&)");
}
#ifdef __linux__
static void readFileRoots(const std::filesystem::path & path, UncheckedRoots & roots)
{
try {
roots[readFile(path)].emplace(path.string());
} catch (SystemError & e) {
if (!e.is(std::errc::no_such_file_or_directory) && !e.is(std::errc::permission_denied))
throw;
}
}
#endif
void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
{
Roots unchecked;
UncheckedRoots unchecked;
if (config->useRootsDaemon) {
experimentalFeatureSettings.require(Xp::LocalOverlayStore);
unchecked = requestRuntimeRoots(*config, config->getRootsSocketPath());
} else {
unchecked = findRuntimeRootsUnchecked(*config);
auto procDir = AutoCloseDir{opendir("/proc")};
if (procDir) {
struct dirent * ent;
static const auto digitsRegex = boost::regex(R"(^\d+$)");
static const auto mapRegex = boost::regex(R"(^\s*\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+(/\S+)\s*$)");
auto storePathRegex = boost::regex(quoteRegexChars(storeDir) + R"(/[0-9a-z]+[0-9a-zA-Z\+\-\._\?=]*)");
while (errno = 0, ent = readdir(procDir.get())) {
checkInterrupt();
if (boost::regex_match(ent->d_name, digitsRegex)) {
try {
readProcLink(fmt("/proc/%s/exe", ent->d_name), unchecked);
readProcLink(fmt("/proc/%s/cwd", ent->d_name), unchecked);
auto fdStr = fmt("/proc/%s/fd", ent->d_name);
auto fdDir = AutoCloseDir(opendir(fdStr.c_str()));
if (!fdDir) {
if (errno == ENOENT || errno == EACCES)
continue;
throw SysError("opening %1%", fdStr);
}
struct dirent * fd_ent;
while (errno = 0, fd_ent = readdir(fdDir.get())) {
if (fd_ent->d_name[0] != '.')
readProcLink(fmt("%s/%s", fdStr, fd_ent->d_name), unchecked);
}
if (errno) {
if (errno == ESRCH)
continue;
throw SysError("iterating /proc/%1%/fd", ent->d_name);
}
fdDir.reset();
std::filesystem::path mapFile = fmt("/proc/%s/maps", ent->d_name);
auto mapLines = tokenizeString<std::vector<std::string>>(readFile(mapFile.string()), "\n");
for (const auto & line : mapLines) {
auto match = boost::smatch{};
if (boost::regex_match(line, match, mapRegex))
unchecked[match[1]].emplace(mapFile.string());
}
auto envFile = fmt("/proc/%s/environ", ent->d_name);
auto envString = readFile(envFile);
auto env_end = boost::sregex_iterator{};
for (auto i = boost::sregex_iterator{envString.begin(), envString.end(), storePathRegex};
i != env_end;
++i)
unchecked[i->str()].emplace(envFile);
} catch (SystemError & e) {
if (errno == ENOENT || errno == EACCES || errno == ESRCH)
continue;
throw;
}
}
}
if (errno)
throw SysError("iterating /proc");
}
for (auto & [path, links] : unchecked) {
if (!isValidPath(path))
#if !defined(__linux__)
// lsof is really slow on OS X. This actually causes the gc-concurrent.sh test to fail.
// See: https://github.com/NixOS/nix/issues/3011
// Because of this we disable lsof when running the tests.
if (getEnv("_NIX_TEST_NO_LSOF") != "1") {
try {
boost::regex lsofRegex(R"(^n(/.*)$)");
auto lsofLines =
tokenizeString<std::vector<std::string>>(runProgram(LSOF, true, {"-n", "-w", "-F", "n"}), "\n");
for (const auto & line : lsofLines) {
boost::smatch match;
if (boost::regex_match(line, match, lsofRegex))
unchecked[match[1].str()].emplace("{lsof}");
}
} catch (ExecError & e) {
/* lsof not installed, lsof failed */
}
}
#endif
#ifdef __linux__
readFileRoots("/proc/sys/kernel/modprobe", unchecked);
readFileRoots("/proc/sys/kernel/fbsplash", unchecked);
readFileRoots("/proc/sys/kernel/poweroff_cmd", unchecked);
#endif
for (auto & [target, links] : unchecked) {
if (!isInStore(target))
continue;
debug("got additional root '%1%'", printStorePath(path));
if (censor)
roots[path].insert(censored);
else
roots[path].insert(links.begin(), links.end());
try {
auto path = toStorePath(target).first;
if (!isValidPath(path))
continue;
debug("got additional root '%1%'", printStorePath(path));
if (censor)
roots[path].insert(censored);
else
roots[path].insert(links.begin(), links.end());
} catch (BadStorePath &) {
}
}
}
@@ -351,7 +464,7 @@ struct GCLimitReached
void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
{
const auto & gcSettings = config->getLocalSettings().getGCSettings();
const auto & gcSettings = config->getGCSettings();
bool shouldDelete = options.action == GCOptions::gcDeleteDead || options.action == GCOptions::gcDeleteSpecific;
bool keepOutputs = gcSettings.keepOutputs;
@@ -787,14 +900,12 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
accounting. */
}
auto st = stat(linksDir);
int64_t overhead =
#ifdef _WIN32
0
#else
[&] {
auto st = stat(linksDir);
return st.st_blocks * 512ULL;
}()
st.st_blocks * 512ULL
#endif
;
@@ -808,7 +919,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
void LocalStore::autoGC(bool sync)
{
#if HAVE_STATVFS
const auto & gcSettings = config->getLocalSettings().getGCSettings();
const auto & gcSettings = config->getGCSettings();
static auto fakeFreeSpaceFile = getEnv("_NIX_TEST_FREE_SPACE_FILE");
Loaded 100 of 296 files, more files were not shown because too many files have changed in this diff. Show more