Compare commits

..
Author SHA1 Message Date
John Ericson bec4da9e26 NarCache improvements
- Separate implementation, abstract implements

- `NarCache` work in constant memory

  This is using `RestoreSink` to dedup some write-side IO, but I am not
  so sure that is a good idea. We'll continue reworking it.

- Use pathlocks to avoid download storms

- Use `Descriptor`-based logic, not `pathExists` to avoid symlink races
  if anything is deleted.
2026-01-26 15:30:09 -05:00
394 changed files with 4748 additions and 7852 deletions

No files matched your search

+2 -2
View File
@@ -40,12 +40,12 @@ jobs:
role-session-name: nix-release-oidc-${{ github.run_id }}
aws-region: eu-west-1
- name: Login to Docker Hub
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Login to GitHub Container Registry
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
with:
registry: ghcr.io
username: ${{ github.actor }}
-2
View File
@@ -1,7 +1,5 @@
# Default meson build dir
/build
# Meson creates this file too
src/.wraplock
# /tests/functional/
/tests/functional/common/subst-vars.sh
-29
View File
@@ -1,29 +0,0 @@
---
synopsis: "Rust nix-installer in beta"
prs: []
---
The Rust-based rewrite of the Nix installer is now in beta.
We'd love help testing it out!
To test out the new installer, run:
```
curl -sSfL https://artifacts.nixos.org/nix-installer | sh -s -- install
```
This installer can be run even when you have an existing, script-based Nix installation without any adjustments.
This new installer also comes with the ability to uninstall your Nix installation; run:
```
/nix/nix-installer uninstall
```
This will get rid of your entire Nix installation (even if you installed over an existing, script-based installation).
This installer is a modified version of the [Determinate Nix Installer](https://github.com/DeterminateSystems/nix-installer) by Determinate Systems.
Thanks to Determinate Systems for all the investment they've put into the installer.
Source for the installer is in https://github.com/NixOS/nix-installer.
Report any issues in that repo.
For CI usage, a GitHub Action to install Nix using this installer is available at https://github.com/NixOS/nix-installer-action.
@@ -6,4 +6,3 @@ prs: [14766]
The C API now includes additional methods:
- `nix_store_query_path_from_hash_part()` - Get the full store path given its hash part
- `nix_store_copy_path()` - Copy a single store path between two stores, allows repairs and configuring signature checking
@@ -1,32 +0,0 @@
---
synopsis: S3 binary caches now use virtual-hosted-style addressing by default
issues: [15208]
---
S3 binary caches now use virtual-hosted-style URLs
(`https://bucket.s3.region.amazonaws.com/key`) instead of path-style URLs
(`https://s3.region.amazonaws.com/bucket/key`) when connecting to standard AWS
S3 endpoints. This enables HTTP/2 multiplexing and fixes TCP connection
exhaustion (TIME_WAIT socket accumulation) under high-concurrency workloads.
A new `addressing-style` store option controls this behavior:
- `auto` (default): virtual-hosted-style for standard AWS endpoints, path-style
for custom endpoints.
- `path`: forces path-style addressing (deprecated by AWS).
- `virtual`: forces virtual-hosted-style addressing (bucket names must not
contain dots).
Bucket names containing dots (e.g., `my.bucket.name`) automatically fall back
to path-style addressing in `auto` mode, because dotted names create
multi-level subdomains that break TLS wildcard certificate validation.
Example using path-style for backwards compatibility:
```
s3://my-bucket/key?region=us-east-1&addressing-style=path
```
Additionally, TCP keep-alive is now enabled on all HTTP connections, preventing
idle connections from being silently dropped by intermediate network devices
(NATs, firewalls, load balancers).
Generated
+5 -5
View File
@@ -63,15 +63,15 @@
},
"nixpkgs": {
"locked": {
"lastModified": 1771043024,
"narHash": "sha256-WoiezqWJQ3OHILah+p6rzNXdJceEAmAhyDFZFZ6pZzY=",
"rev": "3aadb7ca9eac2891d52a9dec199d9580a6e2bf44",
"lastModified": 1763948260,
"narHash": "sha256-zZk7fn2ARAqmLwaYTpxBJmj81KIdz11NiWt7ydHHD/M=",
"rev": "1c8ba8d3f7634acac4a2094eef7c32ad9106532c",
"type": "tarball",
"url": "https://releases.nixos.org/nixos/25.11/nixos-25.11.5960.3aadb7ca9eac/nixexprs.tar.xz"
"url": "https://releases.nixos.org/nixos/25.05/nixos-25.05.813095.1c8ba8d3f763/nixexprs.tar.xz"
},
"original": {
"type": "tarball",
"url": "https://channels.nixos.org/nixos-25.11/nixexprs.tar.xz"
"url": "https://channels.nixos.org/nixos-25.05/nixexprs.tar.xz"
}
},
"nixpkgs-23-11": {
+1 -6
View File
@@ -1,7 +1,7 @@
{
description = "The purely functional package manager";
inputs.nixpkgs.url = "https://channels.nixos.org/nixos-25.11/nixexprs.tar.xz";
inputs.nixpkgs.url = "https://channels.nixos.org/nixos-25.05/nixexprs.tar.xz";
inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2";
inputs.nixpkgs-23-11.url = "github:NixOS/nixpkgs/a62e6edd6d5e1fa0329b8653c801147986f8d446";
@@ -115,9 +115,6 @@
}
// lib.optionalAttrs (crossSystem == "x86_64-unknown-freebsd13") {
useLLVM = true;
}
// lib.optionalAttrs (crossSystem == "x86_64-w64-mingw32") {
emulator = pkgs: "${pkgs.buildPackages.wineWow64Packages.stable_11}/bin/wine";
};
overlays = [
(overlayFor (pkgs: pkgs.${stdenv}))
@@ -409,8 +406,6 @@
"nix-cmd" = { };
"nix-nswrapper" = { };
"nix-cli" = { };
"nix-everything" = { };
+2 -9
View File
@@ -88,23 +88,16 @@
''^tests/functional/lang/eval-fail-path-slash\.nix$''
''^tests/functional/lang/eval-fail-toJSON-non-utf-8\.nix$''
''^tests/functional/lang/eval-fail-set\.nix$''
# Language tests, don't churn the formatting of strings
''^tests/functional/lang/eval-fail-fromTOML-overflow\.nix$''
''^tests/functional/lang/eval-fail-fromTOML-underflow\.nix$''
''^tests/functional/lang/eval-fail-bad-string-interpolation-3\.nix$''
''^tests/functional/lang/eval-fail-bad-string-interpolation-4\.nix$''
''^tests/functional/lang/eval-okay-regex-match2\.nix$''
];
};
clang-format = {
enable = true;
# https://github.com/cachix/git-hooks.nix/pull/532
package = pkgs.llvmPackages_21.clang-tools;
package = pkgs.llvmPackages_latest.clang-tools;
excludes = [
# We don't want to format test data
# ''tests/(?!nixos/).*\.nix''
"^src/[^/]*-tests/data/.*$"
''^src/[^/]*-tests/data/.*$''
# Don't format vendored code
''^doc/manual/redirects\.js$''
-4
View File
@@ -24,10 +24,6 @@ subproject('libcmd')
# Executables
subproject('nix')
if host_machine.system() == 'linux'
subproject('nswrapper')
endif
# Docs
if get_option('doc-gen')
subproject('internal-api-docs')
@@ -1,7 +1,7 @@
# Clang gets grumpy about missing libasan symbols if -shared-libasan is not
# passed when building shared libs, at least on Linux
if cxx.get_id() == 'clang' and ('address' in get_option('b_sanitize') or 'undefined' in get_option(
'b_sanitize',
'b_sanitize',
))
add_project_link_arguments('-shared-libasan', language : 'cpp')
endif
@@ -22,8 +22,6 @@ add_project_arguments(
'-Werror=undef',
'-Werror=unused-result',
'-Werror=sign-compare',
'-Werror=return-type',
'-Werror=non-virtual-dtor',
'-Wignored-qualifiers',
'-Wimplicit-fallthrough',
'-Wno-deprecated-declarations',
@@ -33,13 +31,6 @@ add_project_arguments(
# GCC doesn't benefit much from precompiled headers.
do_pch = cxx.get_id() == 'clang'
if cxx.get_id() == 'gcc'
add_project_arguments(
'-Wno-interference-size', # Used for C++ ABI only. We don't provide any guarantees about different march tunings.
language : 'cpp',
)
endif
# This is a clang-only option for improving build times.
# It forces the instantiation of templates in the PCH itself and
# not every translation unit it's included in.
@@ -49,11 +40,6 @@ endif
# instantiations in libutil and libstore.
if cxx.get_id() == 'clang'
add_project_arguments('-fpch-instantiate-templates', language : 'cpp')
# Catch brace elision bugs: when WorkerProto::Version changed from `unsigned int`
# to `struct { unsigned int major; uint8_t minor; }`, `.version = 16` silently
# became `.version = {16, 0}` instead of failing, breaking protocol compatibility
# in a subtle way
add_project_arguments('-Werror=c99-designator', language : 'cpp')
endif
# Detect if we're using libstdc++ (GCC's standard library)
-3
View File
@@ -4,7 +4,6 @@
buildPackages,
cacert,
nix,
nixComponents2,
}:
let
@@ -12,7 +11,6 @@ let
installerClosureInfo = buildPackages.closureInfo {
rootPaths = [
nix
nixComponents2.nix-manual.man
cacert
];
};
@@ -44,7 +42,6 @@ runCommand "nix-binary-tarball-${version}" env ''
--subst-var-by cacert ${cacert}
substitute ${../scripts/install-multi-user.sh} $TMPDIR/install-multi-user \
--subst-var-by nix ${nix} \
--subst-var-by nix-manual ${nixComponents2.nix-manual.man} \
--subst-var-by cacert ${cacert}
if type -p shellcheck; then
+2 -6
View File
@@ -155,14 +155,12 @@ let
];
};
mesonBuildLayer = finalAttrs: prevAttrs: rec {
mesonBuildLayer = finalAttrs: prevAttrs: {
nativeBuildInputs = prevAttrs.nativeBuildInputs or [ ] ++ [
pkg-config
];
separateDebugInfo = !stdenv.hostPlatform.isStatic;
# needed by separateDebugInfo
# SEE: https://github.com/NixOS/nixpkgs/pull/394674/commits/a4d355342976e9e9823fb94f133bc43ebec9da5b
__structuredAttrs = separateDebugInfo;
hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie";
};
mesonLibraryLayer = finalAttrs: prevAttrs: {
@@ -418,8 +416,6 @@ in
nix-cmd = callPackage ../src/libcmd/package.nix { };
nix-nswrapper = callPackage ../src/nswrapper/package.nix { };
/**
The Nix command line interface. Note that this does not include its tests, whereas `nix-everything` does.
*/
+26 -6
View File
@@ -30,13 +30,33 @@ scope: {
NIX_CFLAGS_COMPILE = "-DINITIAL_MARK_STACK_SIZE=1048576";
});
curl = pkgs.curl.override {
http3Support = !pkgs.stdenv.hostPlatform.isWindows;
};
lowdown = pkgs.lowdown.overrideAttrs (prevAttrs: rec {
version = "2.0.2";
src = pkgs.fetchurl {
url = "https://kristaps.bsd.lv/lowdown/snapshots/lowdown-${version}.tar.gz";
hash = "sha512-cfzhuF4EnGmLJf5EGSIbWqJItY3npbRSALm+GarZ7SMU7Hr1xw0gtBFMpOdi5PBar4TgtvbnG4oRPh+COINGlA==";
};
nativeBuildInputs = prevAttrs.nativeBuildInputs ++ [ pkgs.buildPackages.bmake ];
postInstall =
lib.replaceStrings [ "lowdown.so.1" "lowdown.1.dylib" ] [ "lowdown.so.2" "lowdown.2.dylib" ]
(prevAttrs.postInstall or "");
patches = [ ];
});
libblake3 = pkgs.libblake3.override {
useTBB = !(stdenv.hostPlatform.isWindows || stdenv.hostPlatform.isStatic);
};
# TODO: Remove this when https://github.com/NixOS/nixpkgs/pull/442682 is included in a stable release
toml11 =
if lib.versionAtLeast pkgs.toml11.version "4.4.0" then
pkgs.toml11
else
pkgs.toml11.overrideAttrs rec {
version = "4.4.0";
src = pkgs.fetchFromGitHub {
owner = "ToruNiina";
repo = "toml11";
tag = "v${version}";
hash = "sha256-sgWKYxNT22nw376ttGsTdg0AMzOwp8QH3E8mx0BZJTQ=";
};
};
# TODO Hack until https://github.com/NixOS/nixpkgs/issues/45462 is fixed.
boost =
-5
View File
@@ -31,8 +31,6 @@
nix-cmd,
nix-nswrapper,
nix-cli,
nix-functional-tests,
@@ -173,9 +171,6 @@ stdenv.mkDerivation (finalAttrs: {
# Forwarded outputs
ln -sT ${nix-manual} $doc
ln -sT ${nix-manual.man} $man
''
+ lib.optionalString stdenv.isLinux ''
lndir ${nix-nswrapper} $out
'';
passthru = {
-1
View File
@@ -57,7 +57,6 @@ let
"nix-flake"
"nix-flake-c"
"nix-flake-tests"
"nix-nswrapper"
"nix-main"
"nix-main-c"
"nix-cmd"
-3
View File
@@ -52,7 +52,6 @@ readonly PROFILE_FISH_PREFIXES=(
readonly PROFILE_NIX_FILE_FISH="$NIX_ROOT/var/nix/profiles/default/etc/profile.d/nix-daemon.fish"
readonly NIX_INSTALLED_NIX="@nix@"
readonly NIX_INSTALLED_NIX_MAN="@nix-manual@"
readonly NIX_INSTALLED_CACERT="@cacert@"
#readonly NIX_INSTALLED_NIX="/nix/store/byi37zv50wnfrpp4d81z3spswd5zva37-nix-2.3.6"
#readonly NIX_INSTALLED_CACERT="/nix/store/7pi45g541xa8ahwgpbpy7ggsl0xj1jj6-nss-cacert-3.49.2"
@@ -970,8 +969,6 @@ setup_default_profile() {
task "Setting up the default profile"
_sudo "to install a bootstrapping Nix in to the default profile" \
HOME="$ROOT_HOME" "$NIX_INSTALLED_NIX/bin/nix-env" -i "$NIX_INSTALLED_NIX"
_sudo "to install Nix man pages in to the default profile" \
HOME="$ROOT_HOME" "$NIX_INSTALLED_NIX/bin/nix-env" -i "$NIX_INSTALLED_NIX_MAN"
if [ -z "${NIX_SSL_CERT_FILE:-}" ] || ! [ -f "${NIX_SSL_CERT_FILE:-}" ] || cert_in_store; then
_sudo "to install a bootstrapping SSL certificate just for Nix in to the default profile" \
-1
View File
@@ -38,7 +38,6 @@ escape_systemd_env() {
create_systemd_proxy_env() {
vars="http_proxy https_proxy ftp_proxy all_proxy no_proxy HTTP_PROXY HTTPS_PROXY FTP_PROXY ALL_PROXY NO_PROXY"
for v in $vars; do
# shellcheck disable=SC2268
if [ "x${!v:-}" != "x" ]; then
echo "Environment=${v}=$(escape_systemd_env "${!v}")"
fi
+15 -43
View File
@@ -4,7 +4,6 @@
#include "nix/cmd/command.hh"
#include "nix/cmd/legacy.hh"
#include "nix/cmd/markdown.hh"
#include "nix/store/globals.hh"
#include "nix/store/store-open.hh"
#include "nix/store/local-fs-store.hh"
#include "nix/store/derivations.hh"
@@ -64,25 +63,6 @@ void NixMultiCommand::run()
command->second->run();
}
StoreConfigCommand::StoreConfigCommand() {}
ref<StoreConfig> StoreConfigCommand::getStoreConfig()
{
if (!_storeConfig)
_storeConfig = createStoreConfig();
return ref<StoreConfig>(_storeConfig);
}
ref<StoreConfig> StoreConfigCommand::createStoreConfig()
{
return resolveStoreConfig(StoreReference{settings.storeUri.get()});
}
void StoreConfigCommand::run()
{
run(getStoreConfig());
}
StoreCommand::StoreCommand() {}
ref<Store> StoreCommand::getStore()
@@ -94,20 +74,12 @@ ref<Store> StoreCommand::getStore()
ref<Store> StoreCommand::createStore()
{
auto store = getStoreConfig()->openStore();
store->init();
return store;
return openStore();
}
void StoreCommand::run(ref<StoreConfig> storeConfig)
void StoreCommand::run()
{
// We can either efficiently implement getStore/createStore with memoization,
// or use the StoreConfig passed in run.
// It's more efficient to memoize, especially since there are some direct users
// of getStore. The StoreConfig in both cases should be the same, though.
auto store = getStore();
assert(&*storeConfig == &store->config);
run(std::move(store));
run(getStore());
}
CopyCommand::CopyCommand()
@@ -116,28 +88,28 @@ CopyCommand::CopyCommand()
.longName = "from",
.description = "URL of the source Nix store.",
.labels = {"store-uri"},
.handler = {[this](std::string s) { srcUri = StoreReference::parse(s); }},
.handler = {&srcUri},
});
addFlag({
.longName = "to",
.description = "URL of the destination Nix store.",
.labels = {"store-uri"},
.handler = {[this](std::string s) { dstUri = StoreReference::parse(s); }},
.handler = {&dstUri},
});
}
ref<StoreConfig> CopyCommand::createStoreConfig()
ref<Store> CopyCommand::createStore()
{
return !srcUri ? StoreCommand::createStoreConfig() : resolveStoreConfig(StoreReference{*srcUri});
return srcUri.empty() ? StoreCommand::createStore() : openStore(srcUri);
}
ref<Store> CopyCommand::getDstStore()
{
if (!srcUri && !dstUri)
if (srcUri.empty() && dstUri.empty())
throw UsageError("you must pass '--from' and/or '--to'");
return !dstUri ? openStore() : openStore(StoreReference{*dstUri});
return dstUri.empty() ? openStore() : openStore(dstUri);
}
EvalCommand::EvalCommand()
@@ -159,7 +131,7 @@ EvalCommand::~EvalCommand()
ref<Store> EvalCommand::getEvalStore()
{
if (!evalStore)
evalStore = evalStoreUrl ? openStore(StoreReference{*evalStoreUrl}) : getStore();
evalStore = evalStoreUrl ? openStore(*evalStoreUrl) : getStore();
return ref<Store>(evalStore);
}
@@ -285,18 +257,18 @@ MixProfile::MixProfile()
});
}
void MixProfile::updateProfile(Store & store_, const StorePath & storePath)
void MixProfile::updateProfile(const StorePath & storePath)
{
if (!profile)
return;
auto * store = dynamic_cast<LocalFSStore *>(&store_);
auto store = getDstStore().dynamic_pointer_cast<LocalFSStore>();
if (!store)
throw Error("'--profile' is not supported for this Nix store");
auto profile2 = absPath(*profile);
switchLink(profile2, createGeneration(*store, profile2, storePath));
}
void MixProfile::updateProfile(Store & store, const BuiltPaths & buildables)
void MixProfile::updateProfile(const BuiltPaths & buildables)
{
if (!profile)
return;
@@ -320,12 +292,12 @@ void MixProfile::updateProfile(Store & store, const BuiltPaths & buildables)
throw UsageError(
"'--profile' requires that the arguments produce a single store path, but there are %d", result.size());
updateProfile(store, result[0]);
updateProfile(result[0]);
}
MixDefaultProfile::MixDefaultProfile()
{
profile = getDefaultProfile(settings.getProfileDirsOptions()).string();
profile = getDefaultProfile().string();
}
static constexpr auto environmentVariablesCategory = "Options that change environment variables";
+1 -1
View File
@@ -148,7 +148,7 @@ MixEvalArgs::MixEvalArgs()
)",
.category = category,
.labels = {"store-url"},
.handler = {[this](std::string s) { evalStoreUrl = StoreReference::parse(s); }},
.handler = {&evalStoreUrl},
});
}
-31
View File
@@ -1,31 +0,0 @@
#include "nix/cmd/get-build-log.hh"
#include "nix/store/log-store.hh"
#include "nix/store/store-open.hh"
namespace nix {
std::string fetchBuildLog(ref<Store> store, const StorePath & path, std::string_view what)
{
auto subs = getDefaultSubstituters();
subs.push_front(store);
for (auto & sub : subs) {
auto * logSubP = dynamic_cast<LogStore *>(&*sub);
if (!logSubP) {
printInfo("Skipped '%s' which does not support retrieving build logs", sub->config.getHumanReadableURI());
continue;
}
auto & logSub = *logSubP;
auto log = logSub.getBuildLog(path);
if (!log)
continue;
printInfo("got build log for '%s' from '%s'", what, logSub.config.getHumanReadableURI());
return *log;
}
throw Error("build log of '%s' is not available", what);
}
} // namespace nix
+18 -34
View File
@@ -5,7 +5,6 @@
#include "nix/util/args.hh"
#include "nix/cmd/common-eval-args.hh"
#include "nix/store/path.hh"
#include "nix/store/store-reference.hh"
#include "nix/flake/lockfile.hh"
#include <optional>
@@ -41,43 +40,28 @@ struct NixMultiCommand : MultiCommand, virtual Command
// For the overloaded run methods
#pragma GCC diagnostic ignored "-Woverloaded-virtual"
/**
* A command that requires a \ref StoreConfig store configuration.
*/
struct StoreConfigCommand : virtual Command
{
StoreConfigCommand();
void run() override;
/**
* Return the default Nix store configuration.
*/
ref<StoreConfig> getStoreConfig();
virtual ref<StoreConfig> createStoreConfig();
/**
* Main entry point, with a `StoreConfig` provided
*/
virtual void run(ref<StoreConfig>) = 0;
private:
std::shared_ptr<StoreConfig> _storeConfig;
};
/**
* A command that requires a \ref Store "Nix store".
*/
struct StoreCommand : virtual StoreConfigCommand
struct StoreCommand : virtual Command
{
StoreCommand();
void run(ref<StoreConfig>) override;
void run() override;
/**
* Return the default Nix store.
*/
ref<Store> getStore();
ref<Store> createStore();
/**
* Return the destination Nix store.
*/
virtual ref<Store> getDstStore()
{
return getStore();
}
virtual ref<Store> createStore();
/**
* Main entry point, with a `Store` provided
*/
@@ -93,13 +77,13 @@ private:
*/
struct CopyCommand : virtual StoreCommand
{
std::optional<StoreReference> srcUri, dstUri;
std::string srcUri, dstUri;
CopyCommand();
ref<StoreConfig> createStoreConfig() override;
ref<Store> createStore() override;
ref<Store> getDstStore();
ref<Store> getDstStore() override;
};
/**
@@ -331,11 +315,11 @@ struct MixProfile : virtual StoreCommand
MixProfile();
/* If 'profile' is set, make it point at 'storePath'. */
void updateProfile(Store & store, const StorePath & storePath);
void updateProfile(const StorePath & storePath);
/* If 'profile' is set, make it point at the store path produced
by 'buildables'. */
void updateProfile(Store & store, const BuiltPaths & buildables);
void updateProfile(const BuiltPaths & buildables);
};
struct MixDefaultProfile : MixProfile
@@ -407,7 +391,7 @@ void createOutLinks(const std::filesystem::path & outLink, const BuiltPaths & bu
struct MixOutLinkBase : virtual Args
{
/** Prefix for any output symlinks. Empty means do not write an output symlink. */
std::filesystem::path outLink;
Path outLink;
MixOutLinkBase(const std::string & defaultOutLink)
: outLink(defaultOutLink)
@@ -435,7 +419,7 @@ struct MixOutLinkByDefault : MixOutLinkBase, virtual Args
addFlag({
.longName = "no-link",
.description = "Do not create symlinks to the build results.",
.handler = {&outLink, std::filesystem::path{}},
.handler = {&outLink, Path("")},
});
}
};
@@ -6,7 +6,6 @@
#include "nix/main/common-args.hh"
#include "nix/expr/search-path.hh"
#include "nix/expr/eval-settings.hh"
#include "nix/store/store-reference.hh"
#include <filesystem>
@@ -56,7 +55,7 @@ struct MixEvalArgs : virtual Args, virtual MixRepair
LookupPath lookupPath;
std::optional<StoreReference> evalStoreUrl;
std::optional<std::string> evalStoreUrl;
private:
struct AutoArgExpr
@@ -1,23 +0,0 @@
#pragma once
///@file
#include "nix/store/store-api.hh"
#include <string>
#include <string_view>
namespace nix {
/**
* Fetch the build log for a store path, searching the store and its
* substituters.
*
* @param store The store to search (and its substituters).
* @param path The store path to get the build log for.
* @param what A description of what we're fetching the log for (used in messages).
* @return The build log content.
* @throws Error if the build log is not available.
*/
std::string fetchBuildLog(ref<Store> store, const StorePath & path, std::string_view what);
} // namespace nix
-2
View File
@@ -9,7 +9,6 @@ headers = files(
'common-eval-args.hh',
'compatibility-settings.hh',
'editor-for.hh',
'get-build-log.hh',
'installable-attr-path.hh',
'installable-derived-path.hh',
'installable-flake.hh',
@@ -21,5 +20,4 @@ headers = files(
'network-proxy.hh',
'repl-interacter.hh',
'repl.hh',
'unix-socket-server.hh',
)
@@ -14,7 +14,6 @@ namespace detail {
struct ReplCompleterMixin
{
virtual StringSet completePrefix(const std::string & prefix) = 0;
virtual ~ReplCompleterMixin() = default;
};
}; // namespace detail
+2 -1
View File
@@ -25,7 +25,7 @@ struct AbstractNixRepl
* @todo this is a layer violation
*
* @param programName Name of the command, e.g. `nix` or `nix-env`.
* @param args arguments to the command.
* @param args aguments to the command.
*/
using RunNix =
void(const std::string & programName, const Strings & args, const std::optional<std::string> & input);
@@ -37,6 +37,7 @@ struct AbstractNixRepl
*/
static std::unique_ptr<AbstractNixRepl> create(
const LookupPath & lookupPath,
nix::ref<Store> store,
ref<EvalState> state,
std::function<AnnotatedValues()> getValues,
RunNix * runNix = nullptr);
@@ -1,79 +0,0 @@
#pragma once
///@file
#include "nix/util/file-descriptor.hh"
#include <filesystem>
#include <functional>
#include <optional>
#include <sys/types.h>
namespace nix::unix {
/**
* Information about the identity of the peer on a Unix domain socket connection.
*/
struct PeerInfo
{
std::optional<pid_t> pid;
std::optional<uid_t> uid;
std::optional<gid_t> gid;
};
/**
* Get the identity of the caller, if possible.
*/
PeerInfo getPeerInfo(Descriptor remote);
/**
* Callback type for handling new connections.
*
* The callback receives ownership of the connection and is responsible
* for handling it (e.g., forking a child process, spawning a thread, etc.).
*
* @param socket The accepted connection file descriptor.
* @param closeListeners A callback to close the listening sockets.
* Useful in forked child processes to release the bound sockets.
*/
using UnixSocketHandler = std::function<void(AutoCloseFD socket, std::function<void()> closeListeners)>;
/**
* Options for the serve loop.
*
* Only used if no systemd socket activation is detected.
*/
struct ServeUnixSocketOptions
{
/**
* The Unix domain socket path to create and listen on.
*/
std::filesystem::path socketPath;
/**
* Mode for the created socket file.
*/
mode_t socketMode = 0666;
};
/**
* Run a server loop that accepts connections and calls the handler for each.
*
* This function handles:
* - systemd socket activation (via LISTEN_FDS environment variable)
* - Creating and binding a Unix domain socket if no activation is detected
* - Polling for incoming connections
* - Accepting connections
*
* For each accepted connection, the handler is called with the connection
* file descriptor. The handler takes ownership of the file descriptor and
* is responsible for closing it when done.
*
* This function never returns normally. It runs until interrupted
* (e.g., via SIGINT), at which point it throws `Interrupted`.
*
* @param options Configuration for the server.
* @param handler Callback invoked for each accepted connection.
*/
[[noreturn]] void serveUnixSocket(const ServeUnixSocketOptions & options, UnixSocketHandler handler);
} // namespace nix::unix
+2 -4
View File
@@ -203,10 +203,8 @@ FlakeRef InstallableFlake::nixpkgsFlakeRef() const
if (auto nixpkgsInput = lockedFlake->lockFile.findInput({"nixpkgs"})) {
if (auto lockedNode = std::dynamic_pointer_cast<const flake::LockedNode>(nixpkgsInput)) {
if (lockedNode->isFlake) {
debug("using nixpkgs flake '%s'", lockedNode->lockedRef);
return std::move(lockedNode->lockedRef);
}
debug("using nixpkgs flake '%s'", lockedNode->lockedRef);
return std::move(lockedNode->lockedRef);
}
}
+8 -4
View File
@@ -157,7 +157,7 @@ MixFlakeOptions::MixFlakeOptions()
.category = category,
.labels = {"flake-lock-path"},
.handler = {[&](std::string lockFilePath) {
lockFlags.referenceLockFilePath = {getFSSourceAccessor(), CanonPath(absPath(lockFilePath).string())};
lockFlags.referenceLockFilePath = {getFSSourceAccessor(), CanonPath(absPath(lockFilePath))};
}},
.completer = completePath,
});
@@ -583,12 +583,16 @@ static void throwBuildErrors(std::vector<KeyedBuildResult> & buildResults, const
auto failedResult = failed.begin();
if (failedResult != failed.end()) {
if (failed.size() == 1) {
throw *failedResult->second;
failedResult->second->rethrow();
} else {
StringSet failedPaths;
for (; failedResult != failed.end(); failedResult++) {
if (!failedResult->second->message().empty()) {
logError(failedResult->second->info());
if (!failedResult->second->errorMsg.empty()) {
logError(
ErrorInfo{
.level = lvlError,
.msg = failedResult->second->errorMsg,
});
}
failedPaths.insert(failedResult->first->path.to_string(store));
}
-7
View File
@@ -74,7 +74,6 @@ sources = files(
'command.cc',
'common-eval-args.cc',
'editor-for.cc',
'get-build-log.cc',
'installable-attr-path.cc',
'installable-derived-path.cc',
'installable-flake.cc',
@@ -87,12 +86,6 @@ sources = files(
'repl.cc',
)
if host_machine.system() != 'windows'
sources += files(
'unix/unix-socket-server.cc',
)
endif
subdir('include/nix/cmd')
subdir('nix-meson-build-support/export-all-symbols')
+1 -1
View File
@@ -113,7 +113,7 @@ ReadlineLikeInteracter::Guard ReadlineLikeInteracter::init(detail::ReplCompleter
// Allow nix-repl specific settings in .inputrc
rl_readline_name = "nix-repl";
try {
createDirs(std::filesystem::path(historyFile).parent_path());
createDirs(dirOf(historyFile));
} catch (SystemError & e) {
logWarning(e.info());
}
+40 -18
View File
@@ -12,8 +12,9 @@
#include "nix/expr/eval-settings.hh"
#include "nix/expr/attr-path.hh"
#include "nix/util/signals.hh"
#include "nix/store/store-open.hh"
#include "nix/store/log-store.hh"
#include "nix/cmd/common-eval-args.hh"
#include "nix/cmd/get-build-log.hh"
#include "nix/expr/get-drvs.hh"
#include "nix/store/derivations.hh"
#include "nix/store/globals.hh"
@@ -64,7 +65,7 @@ struct NixRepl : AbstractNixRepl, detail::ReplCompleterMixin, gc
const static int envSize = 32768;
std::shared_ptr<StaticEnv> staticEnv;
std::optional<Value> lastLoaded;
Value lastLoaded;
Env * env;
int displ;
StringSet varNames;
@@ -77,6 +78,7 @@ struct NixRepl : AbstractNixRepl, detail::ReplCompleterMixin, gc
NixRepl(
const LookupPath & lookupPath,
nix::ref<Store> store,
ref<EvalState> state,
std::function<AnnotatedValues()> getValues,
RunNix * runNix);
@@ -131,6 +133,7 @@ std::string removeWhitespace(std::string s)
NixRepl::NixRepl(
const LookupPath & lookupPath,
nix::ref<Store> store,
ref<EvalState> state,
std::function<NixRepl::AnnotatedValues()> getValues,
RunNix * runNix)
@@ -563,9 +566,31 @@ ProcessLineResult NixRepl::processLine(std::string line)
} else if (command == ":log") {
settings.readOnlyMode = true;
Finally roModeReset([&]() { settings.readOnlyMode = false; });
auto subs = getDefaultSubstituters();
subs.push_front(state->store);
bool foundLog = false;
RunPager pager;
auto log = fetchBuildLog(state->store, drvPath, drvPathRaw);
logger->writeToStdout(log);
for (auto & sub : subs) {
auto * logSubP = dynamic_cast<LogStore *>(&*sub);
if (!logSubP) {
printInfo(
"Skipped '%s' which does not support retrieving build logs", sub->config.getHumanReadableURI());
continue;
}
auto & logSub = *logSubP;
auto log = logSub.getBuildLog(drvPath);
if (log) {
printInfo("got build log for '%s' from '%s'", drvPathRaw, logSub.config.getHumanReadableURI());
logger->writeToStdout(*log);
foundLog = true;
break;
}
}
if (!foundLog)
throw Error("build log of '%s' is not available", drvPathRaw);
} else {
runNix("nix-shell", {drvPathRaw});
}
@@ -748,19 +773,11 @@ void NixRepl::initEnv()
void NixRepl::showLastLoaded()
{
if (!lastLoaded)
throw Error("nothing has been loaded yet");
RunPager pager;
try {
for (auto & i : *lastLoaded->attrs()) {
std::string_view name = state->symbols[i.name];
logger->cout(name);
}
} catch (SystemError & e) {
/* Ignore broken pipes when the pager gets interrupted. */
if (!e.is(std::errc::broken_pipe))
throw;
for (auto & i : *lastLoaded.attrs()) {
std::string_view name = state->symbols[i.name];
logger->cout(name);
}
}
@@ -883,9 +900,13 @@ void NixRepl::runNix(const std::string & program, const Strings & args, const st
}
std::unique_ptr<AbstractNixRepl> AbstractNixRepl::create(
const LookupPath & lookupPath, ref<EvalState> state, std::function<AnnotatedValues()> getValues, RunNix * runNix)
const LookupPath & lookupPath,
nix::ref<Store> store,
ref<EvalState> state,
std::function<AnnotatedValues()> getValues,
RunNix * runNix)
{
return std::make_unique<NixRepl>(lookupPath, state, getValues, runNix);
return std::make_unique<NixRepl>(lookupPath, std::move(store), state, getValues, runNix);
}
ReplExitStatus AbstractNixRepl::runSimple(ref<EvalState> evalState, const ValMap & extraEnv)
@@ -898,6 +919,7 @@ ReplExitStatus AbstractNixRepl::runSimple(ref<EvalState> evalState, const ValMap
// NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDelete)
auto repl = std::make_unique<NixRepl>(
lookupPath,
openStore(),
evalState,
getValues,
/*runNix=*/nullptr);
-126
View File
@@ -1,126 +0,0 @@
///@file
#include "nix/cmd/unix-socket-server.hh"
#include "nix/util/environment-variables.hh"
#include "nix/util/file-system.hh"
#include "nix/util/logging.hh"
#include "nix/util/signals.hh"
#include "nix/util/strings.hh"
#include "nix/util/unix-domain-socket.hh"
#include "nix/util/util.hh"
#include <sys/socket.h>
#include <sys/un.h>
#include <poll.h>
#if defined(__APPLE__) || defined(__FreeBSD__)
# include <sys/ucred.h>
#endif
namespace nix::unix {
PeerInfo getPeerInfo(Descriptor remote)
{
PeerInfo peer;
#if defined(SO_PEERCRED)
# if defined(__OpenBSD__)
struct sockpeercred cred;
# else
ucred cred;
# endif
socklen_t credLen = sizeof(cred);
if (getsockopt(remote, SOL_SOCKET, SO_PEERCRED, &cred, &credLen) == 0) {
peer.pid = cred.pid;
peer.uid = cred.uid;
peer.gid = cred.gid;
}
#elif defined(LOCAL_PEERCRED)
# if !defined(SOL_LOCAL)
# define SOL_LOCAL 0
# endif
xucred cred;
socklen_t credLen = sizeof(cred);
if (getsockopt(remote, SOL_LOCAL, LOCAL_PEERCRED, &cred, &credLen) == 0)
peer.uid = cred.cr_uid;
#endif
return peer;
}
[[noreturn]] void serveUnixSocket(const ServeUnixSocketOptions & options, UnixSocketHandler handler)
{
std::vector<AutoCloseFD> listeningSockets;
static constexpr int SD_LISTEN_FDS_START = 3;
// Handle socket-based activation by systemd.
auto listenFds = getEnv("LISTEN_FDS");
if (listenFds) {
if (getEnv("LISTEN_PID") != std::to_string(getpid()))
throw Error("unexpected systemd environment variables");
auto count = string2Int<unsigned int>(*listenFds);
assert(count);
for (unsigned int i = 0; i < count; ++i) {
AutoCloseFD fdSocket(SD_LISTEN_FDS_START + i);
closeOnExec(fdSocket.get());
listeningSockets.push_back(std::move(fdSocket));
}
}
// Otherwise, create and bind to a Unix domain socket.
else {
createDirs(options.socketPath.parent_path());
listeningSockets.push_back(createUnixDomainSocket(options.socketPath.string(), options.socketMode));
}
std::vector<struct pollfd> fds;
for (auto & i : listeningSockets)
fds.push_back({.fd = i.get(), .events = POLLIN});
// Loop accepting connections.
while (1) {
try {
checkInterrupt();
auto count = poll(fds.data(), fds.size(), -1);
if (count == -1) {
if (errno == EINTR)
continue;
throw SysError("polling for incoming connections");
}
for (auto & fd : fds) {
if (!fd.revents)
continue;
// Accept a connection.
struct sockaddr_un remoteAddr;
socklen_t remoteAddrLen = sizeof(remoteAddr);
AutoCloseFD remote = accept(fd.fd, (struct sockaddr *) &remoteAddr, &remoteAddrLen);
checkInterrupt();
if (!remote) {
if (errno == EINTR)
continue;
throw SysError("accepting connection");
}
handler(std::move(remote), [&]() { listeningSockets.clear(); });
}
} catch (Error & error) {
auto ei = error.info();
// FIXME: add to trace?
ei.msg = HintFmt("while processing connection: %1%", ei.msg.str());
logError(ei);
}
}
}
} // namespace nix::unix
+1 -1
View File
@@ -756,7 +756,7 @@ TEST_F(PrimOpTest, langVersion)
TEST_F(PrimOpTest, storeDir)
{
auto v = eval("builtins.storeDir");
ASSERT_THAT(v, IsStringEq(state.store->storeDir));
ASSERT_THAT(v, IsStringEq(settings.nixStore));
}
TEST_F(PrimOpTest, nixVersion)
+1 -1
View File
@@ -70,7 +70,7 @@ struct AttrDb
{
auto state(_state->lock());
auto cacheDir = getCacheDir() / "eval-cache-v6";
auto cacheDir = std::filesystem::path(getCacheDir()) / "eval-cache-v6";
createDirs(cacheDir);
auto dbPath = cacheDir / (fingerprint.to_string(HashFormat::Base16, false) + ".sqlite");
-7
View File
@@ -1,16 +1,9 @@
#include "nix/expr/eval-error.hh"
#include "nix/expr/eval.hh"
#include "nix/expr/value.hh"
#include "nix/store/store-api.hh"
namespace nix {
InvalidPathError::InvalidPathError(EvalState & state, const StorePath & path)
: EvalError(state, "path '%s' is not valid", state.store->printStorePath(path))
, path{path}
{
}
template<class T>
EvalErrorBuilder<T> & EvalErrorBuilder<T>::withExitStatus(unsigned int exitStatus)
{
+2 -8
View File
@@ -133,17 +133,11 @@ class SampleStack : public EvalProfiler
FrameInfo getPrimOpFrameInfo(const PrimOp & primOp, std::span<Value *> args, PosIdx pos);
public:
SampleStack(EvalState & state, const std::filesystem::path & profileFile, std::chrono::nanoseconds period)
SampleStack(EvalState & state, std::filesystem::path profileFile, std::chrono::nanoseconds period)
: state(state)
, sampleInterval(period)
, profileFd([&]() {
AutoCloseFD fd = openNewFileForWrite(
profileFile,
0660,
{
.truncateExisting = true,
.followSymlinksOnTruncate = true, /* FIXME: Probably shouldn't follow symlinks. */
});
AutoCloseFD fd = toDescriptor(open(profileFile.string().c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0660));
if (!fd)
throw SysError("opening file %s", PathFmt(profileFile));
return fd;
+3 -4
View File
@@ -70,10 +70,9 @@ Strings EvalSettings::getDefaultNixPath()
}
};
add(getNixDefExpr() / "channels");
auto profilesDirOpts = settings.getProfileDirsOptions();
add(rootChannelsDir(profilesDirOpts) / "nixpkgs", "nixpkgs");
add(rootChannelsDir(profilesDirOpts));
add(std::filesystem::path{getNixDefExpr()} / "channels");
add(rootChannelsDir() / "nixpkgs", "nixpkgs");
add(rootChannelsDir());
return res;
}
+2 -1
View File
@@ -318,6 +318,7 @@ EvalState::EvalState(
countCalls = getEnv("NIX_COUNT_CALLS").value_or("0") != "0";
static_assert(sizeof(Env) <= 16, "environment must be <= 16 bytes");
static_assert(sizeof(Counter) == 64, "counters must be 64 bytes");
/* Construct the Nix expression search path. */
assert(lookupPath.elements.empty());
@@ -432,7 +433,7 @@ void EvalState::checkURI(const std::string & uri)
/* If the URI is a path, then check it against allowedPaths as
well. */
if (std::filesystem::path(uri).is_absolute()) {
if (isAbsolute(uri)) {
if (auto rootFS2 = rootFS.dynamic_pointer_cast<AllowListSourceAccessor>())
rootFS2->checkAccess(CanonPath(uri));
return;
+1 -1
View File
@@ -11,7 +11,7 @@ namespace nix {
* variable is set. This is to prevent contention on these counters
* when multi-threaded evaluation is enabled.
*/
struct alignas(std::hardware_destructive_interference_size) Counter
struct alignas(64) Counter
{
using value_type = uint64_t;
+5 -2
View File
@@ -73,9 +73,12 @@ MakeError(IFDError, EvalBaseError);
struct InvalidPathError : public EvalError
{
public:
StorePath path;
Path path;
InvalidPathError(EvalState & state, const StorePath & path);
InvalidPathError(EvalState & state, const Path & path)
: EvalError(state, "path '%s' is not valid", path)
{
}
};
/**
+4 -4
View File
@@ -134,7 +134,7 @@ public:
virtual bool operator==(const ExternalValueBase & b) const noexcept;
/**
* Print the value as JSON. Defaults to unconvertible, i.e. throws an error
* Print the value as JSON. Defaults to unconvertable, i.e. throws an error
*/
virtual nlohmann::json
printValueAsJSON(EvalState & state, bool strict, NixStringContext & context, bool copyToStore = true) const;
@@ -533,8 +533,8 @@ inline constexpr bool useBitPackedValueStorage = (ptrSize == 8) && (__STDCPP_DEF
* Packs discriminator bits into the pointer alignment niches.
*/
template<std::size_t ptrSize>
class alignas(16)
ValueStorage<ptrSize, std::enable_if_t<detail::useBitPackedValueStorage<ptrSize>>> : public detail::ValueBase
class alignas(16) ValueStorage<ptrSize, std::enable_if_t<detail::useBitPackedValueStorage<ptrSize>>>
: public detail::ValueBase
{
/* Needs a dependent type name in order for member functions (and
* potentially ill-formed bit casts) to be SFINAE'd out.
@@ -587,7 +587,7 @@ class alignas(16)
enum PrimaryDiscriminator : int {
pdUninitialized = 0,
pdSingleDWord, //< layout: Single/zero field payload
/* The order of these enumerations must be the same as in InternalType. */
/* The order of these enumations must be the same as in InternalType. */
pdListN, //< layout: Single untaggable field.
pdString,
pdPath,
+14 -38
View File
@@ -3,7 +3,7 @@
%define api.namespace { ::nix::parser }
%define api.parser.class { BisonParser }
%locations
%define parse.error detailed
%define parse.error verbose
%defines
/* %no-lines */
%parse-param { void * scanner }
@@ -140,41 +140,18 @@ static Expr * makeCall(Exprs & exprs, PosIdx pos, Expr * fn, Expr * arg) {
%type <Expr *> path_start
%type <ToBeStringyExpr> string_parts string_attr
%type <StringToken> attr
%token <StringToken> ID "identifier"
%token <StringToken> STR "string"
%token <StringToken> IND_STR "indented string"
%token <NixInt> INT_LIT "integer"
%token <NixFloat> FLOAT_LIT "floating-point literal"
%token <StringToken> PATH "path"
%token <StringToken> HPATH "'~/…' path"
%token <StringToken> SPATH "'<…>' path"
%token <StringToken> PATH_END "end of path"
%token <StringToken> URI "URI"
%token IF "'if'"
%token THEN "'then'"
%token ELSE "'else'"
%token ASSERT "'assert'"
%token WITH "'with'"
%token LET "'let'"
%token IN_KW "'in'"
%token REC "'rec'"
%token INHERIT "'inherit'"
%token EQ "'=='"
%token NEQ "'!='"
%token LEQ "'<='"
%token GEQ "'>='"
%token UPDATE "'//'"
%token CONCAT "'++'"
%token AND "'&&'"
%token OR "'||'"
%token IMPL "'->'"
%token OR_KW "'or'"
%token PIPE_FROM "'<|'"
%token PIPE_INTO "'|>'"
%token DOLLAR_CURLY "'${'"
%token IND_STRING_OPEN "start of an indented string"
%token IND_STRING_CLOSE "end of an indented string"
%token ELLIPSIS "'...'"
%token <StringToken> ID
%token <StringToken> STR IND_STR
%token <NixInt> INT_LIT
%token <NixFloat> FLOAT_LIT
%token <StringToken> PATH HPATH SPATH PATH_END
%token <StringToken> URI
%token IF THEN ELSE ASSERT WITH LET IN_KW REC INHERIT EQ NEQ AND OR IMPL OR_KW
%token PIPE_FROM PIPE_INTO /* <| and |> */
%token DOLLAR_CURLY /* == ${ */
%token IND_STRING_OPEN IND_STRING_CLOSE
%token ELLIPSIS
%right IMPL
%left OR
@@ -388,8 +365,7 @@ path_start
});
}
auto basePath = std::filesystem::path(state->basePath.path.abs());
Path path(absPath(literal, &basePath).string());
Path path(absPath(literal, state->basePath.path.abs()));
/* add back in the trailing '/' to the first segment */
if (literal.size() > 1 && literal.back() == '/')
path += '/';
+3 -2
View File
@@ -12,7 +12,7 @@ SourcePath EvalState::rootPath(CanonPath path)
SourcePath EvalState::rootPath(PathView path)
{
return {rootFS, CanonPath(absPath(std::filesystem::path{path}).string())};
return {rootFS, CanonPath(absPath(path))};
}
SourcePath EvalState::storePath(const StorePath & path)
@@ -23,12 +23,13 @@ SourcePath EvalState::storePath(const StorePath & path)
StorePath
EvalState::mountInput(fetchers::Input & input, const fetchers::Input & originalInput, ref<SourceAccessor> accessor)
{
auto [storePath, narHash] = fetchToStore2(fetchSettings, *store, accessor, FetchMode::Copy, input.getName());
auto storePath = fetchToStore(fetchSettings, *store, accessor, FetchMode::Copy, input.getName());
allowPath(storePath); // FIXME: should just whitelist the entire virtual store
storeFS->mount(CanonPath(store->printStorePath(storePath)), accessor);
auto narHash = store->queryPathInfo(storePath)->narHash;
input.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true));
if (originalInput.getNarHash() && narHash != *originalInput.getNarHash())
+6 -11
View File
@@ -74,7 +74,7 @@ StringMap EvalState::realiseContext(const NixStringContext & context, StorePathS
for (auto & c : context) {
auto ensureValid = [&](const StorePath & p) {
if (!store->isValidPath(p))
error<InvalidPathError>(p).debugThrow();
error<InvalidPathError>(store->printStorePath(p)).debugThrow();
};
std::visit(
overloaded{
@@ -503,7 +503,7 @@ void prim_exec(EvalState & state, const PosIdx pos, Value ** args, Value & v)
try {
auto _ = state.realiseContext(context); // FIXME: Handle CA derivations
} catch (InvalidPathError & e) {
state.error<EvalError>("cannot execute '%1%', since path '%2%' is not valid", program, e.path.to_string())
state.error<EvalError>("cannot execute '%1%', since path '%2%' is not valid", program, e.path)
.atPos(pos)
.debugThrow();
}
@@ -1814,13 +1814,8 @@ static void derivationStrictInternal(EvalState & state, std::string_view drvName
drv.fillInOutputPaths(*state.store);
}
/* Write the resulting term into the Nix store directory.
Unless we are in read-only mode, that is, in which case we do not
write anything. Users commonly do this to speed up evaluation in
contexts where they don't actually want to build anything. */
auto drvPath =
settings.readOnlyMode ? computeStorePath(*state.store, drv) : state.store->writeDerivation(drv, state.repair);
/* Write the resulting term into the Nix store directory. */
auto drvPath = writeDerivation(*state.store, drv, state.repair);
auto drvPathS = state.store->printStorePath(drvPath);
printMsg(lvlChatty, "instantiated '%1%' -> '%2%'", drvName, drvPathS);
@@ -1931,7 +1926,7 @@ static void prim_storePath(EvalState & state, const PosIdx pos, Value ** args, V
directly in the store. The latter condition is necessary so
e.g. nix-push does the right thing. */
if (!state.store->isStorePath(path.abs()))
path = CanonPath(canonPath(path.abs(), true).string());
path = CanonPath(canonPath(path.abs(), true));
if (!state.store->isInStore(path.abs()))
state.error<EvalError>("path '%1%' is not in the Nix store", path).atPos(pos).debugThrow();
auto path2 = state.store->toStorePath(path.abs()).first;
@@ -2152,7 +2147,7 @@ static void prim_findFile(EvalState & state, const PosIdx pos, Value ** args, Va
auto rewrites = state.realiseContext(context);
path = rewriteStrings(std::move(path), rewrites);
} catch (InvalidPathError & e) {
state.error<EvalError>("cannot find '%1%', since path '%2%' is not valid", path, state.store->printStorePath(e.path))
state.error<EvalError>("cannot find '%1%', since path '%2%' is not valid", path, e.path)
.atPos(pos)
.debugThrow();
}
+7 -13
View File
@@ -191,25 +191,19 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value ** args
{.msg = HintFmt("attribute '%s' is missing in call to 'fetchClosure'", "fromStore"),
.pos = state.positions[pos]});
auto storeRef = StoreReference::parse(*fromStoreUrl);
auto parsedURL = parseURL(*fromStoreUrl, /*lenient=*/true);
if ([&] {
auto * specified = std::get_if<StoreReference::Specified>(&storeRef.variant);
return !specified
|| (specified->scheme != "http" && specified->scheme != "https"
&& !(getEnv("_NIX_IN_TEST").has_value() && specified->scheme == "file"));
}())
throw Error({
.msg = HintFmt("'fetchClosure' only supports http:// and https:// stores"),
.pos = state.positions[pos],
});
if (parsedURL.scheme != "http" && parsedURL.scheme != "https"
&& !(getEnv("_NIX_IN_TEST").has_value() && parsedURL.scheme == "file"))
throw Error(
{.msg = HintFmt("'fetchClosure' only supports http:// and https:// stores"), .pos = state.positions[pos]});
if (!storeRef.params.empty())
if (!parsedURL.query.empty())
throw Error(
{.msg = HintFmt("'fetchClosure' does not support URL query parameters (in '%s')", *fromStoreUrl),
.pos = state.positions[pos]});
auto fromStore = openStore(std::move(storeRef));
auto fromStore = openStore(parsedURL.to_string());
if (toPath)
runFetchClosureWithRewrite(state, pos, *fromStore, *fromPath, *toPath, v);
+5 -11
View File
@@ -34,20 +34,14 @@ struct CacheImpl : Cache
Sync<State> _state;
/**
* This is a back-reference to the `Settings` that owns us.
*/
const Settings & settings;
CacheImpl(const Settings & _settings)
: settings(_settings)
CacheImpl()
{
auto state(_state.lock());
auto dbPath = getCacheDir() / "fetcher-cache-v4.sqlite";
createDirs(dbPath.parent_path());
auto dbPath = (getCacheDir() / "fetcher-cache-v4.sqlite").string();
createDirs(dirOf(dbPath));
state->db = SQLite(dbPath, {.useWAL = nix::settings.useSQLiteWAL});
state->db = SQLite(dbPath, {.useWAL = settings.useSQLiteWAL});
state->db.isCache();
state->db.exec(schema);
@@ -160,7 +154,7 @@ ref<Cache> Settings::getCache() const
{
auto cache(_cache.lock());
if (!*cache)
*cache = std::make_shared<CacheImpl>(*this);
*cache = std::make_shared<CacheImpl>();
return ref<Cache>(*cache);
}
+18 -63
View File
@@ -5,12 +5,12 @@
namespace nix {
fetchers::Cache::Key
makeSourcePathToHashCacheKey(std::string_view fingerprint, ContentAddressMethod method, const CanonPath & path)
fetchers::Cache::Key makeFetchToStoreCacheKey(
const std::string & name, const std::string & fingerprint, ContentAddressMethod method, const std::string & path)
{
return fetchers::Cache::Key{
"sourcePathToHash",
{{"fingerprint", std::string(fingerprint)}, {"method", std::string{method.render()}}, {"path", path.abs()}}};
"fetchToStore",
{{"name", name}, {"fingerprint", fingerprint}, {"method", std::string{method.render()}}, {"path", path}}};
}
StorePath fetchToStore(
@@ -23,39 +23,19 @@ StorePath fetchToStore(
PathFilter * filter,
RepairFlag repair)
{
return fetchToStore2(settings, store, path, mode, name, method, filter, repair).first;
}
// FIXME: add an optimisation for the case where the accessor is
// a `PosixSourceAccessor` pointing to a store path.
std::pair<StorePath, Hash> fetchToStore2(
const fetchers::Settings & settings,
Store & store,
const SourcePath & path,
FetchMode mode,
std::string_view name,
ContentAddressMethod method,
PathFilter * filter,
RepairFlag repair)
{
std::optional<fetchers::Cache::Key> cacheKey;
auto [subpath, fingerprint] = filter ? std::pair<CanonPath, std::optional<std::string>>{path.path, std::nullopt}
: path.accessor->getFingerprint(path.path);
if (fingerprint) {
cacheKey = makeSourcePathToHashCacheKey(*fingerprint, method, subpath);
if (auto res = settings.getCache()->lookup(*cacheKey)) {
auto hash = Hash::parseSRI(fetchers::getStrAttr(*res, "hash"));
auto storePath =
store.makeFixedOutputPathFromCA(name, ContentAddressWithReferences::fromParts(method, hash, {}));
if (mode == FetchMode::DryRun || store.isValidPath(storePath)) {
debug(
"source path '%s' cache hit in '%s' (hash '%s')",
path,
store.printStorePath(storePath),
hash.to_string(HashFormat::SRI, true));
return {storePath, hash};
}
debug("source path '%s' not in store", path);
cacheKey = makeFetchToStoreCacheKey(std::string{name}, *fingerprint, method, subpath.abs());
if (auto res = settings.getCache()->lookupStorePath(*cacheKey, store)) {
debug("store path cache hit for '%s'", path);
return res->storePath;
}
} else {
static auto barf = getEnv("_NIX_TEST_BARF_ON_UNCACHEABLE").value_or("") == "1";
@@ -73,41 +53,16 @@ std::pair<StorePath, Hash> fetchToStore2(
auto filter2 = filter ? *filter : defaultPathFilter;
auto [storePath, hash] =
mode == FetchMode::DryRun
? [&]() {
auto [storePath, hash] =
store.computeStorePath(name, path, method, HashAlgorithm::SHA256, {}, filter2);
debug(
"hashed '%s' to '%s' (hash '%s')",
path,
store.printStorePath(storePath),
hash.to_string(HashFormat::SRI, true));
return std::make_pair(storePath, hash);
}()
: [&]() {
// FIXME: ideally addToStore() would return the hash
// right away (like computeStorePath()).
auto storePath = store.addToStore(name, path, method, HashAlgorithm::SHA256, {}, filter2, repair);
auto info = store.queryPathInfo(storePath);
assert(info->references.empty());
auto hash = method == ContentAddressMethod::Raw::NixArchive ? info->narHash : ({
if (!info->ca || info->ca->method != method)
throw Error("path '%s' lacks a CA field", store.printStorePath(storePath));
info->ca->hash;
});
debug(
"copied '%s' to '%s' (hash '%s')",
path,
store.printStorePath(storePath),
hash.to_string(HashFormat::SRI, true));
return std::make_pair(storePath, hash);
}();
auto storePath = mode == FetchMode::DryRun
? store.computeStorePath(name, path, method, HashAlgorithm::SHA256, {}, filter2).first
: store.addToStore(name, path, method, HashAlgorithm::SHA256, {}, filter2, repair);
if (cacheKey)
settings.getCache()->upsert(*cacheKey, {{"hash", hash.to_string(HashFormat::SRI, true)}});
debug(mode == FetchMode::DryRun ? "hashed '%s'" : "copied '%s' to '%s'", path, store.printStorePath(storePath));
return {storePath, hash};
if (cacheKey && mode == FetchMode::Copy)
settings.getCache()->upsert(*cacheKey, store, {}, storePath);
return storePath;
}
} // namespace nix
+3 -4
View File
@@ -339,10 +339,9 @@ std::pair<ref<SourceAccessor>, Input> Input::getAccessorUnchecked(const Settings
// can reuse the existing nar instead of copying the unpacked
// input back into the store on every evaluation.
if (accessor->fingerprint) {
settings.getCache()->upsert(
makeSourcePathToHashCacheKey(
*accessor->fingerprint, ContentAddressMethod::Raw::NixArchive, CanonPath::root),
{{"hash", store.queryPathInfo(storePath)->narHash.to_string(HashFormat::SRI, true)}});
ContentAddressMethod method = ContentAddressMethod::Raw::NixArchive;
auto cacheKey = makeFetchToStoreCacheKey(getName(), *accessor->fingerprint, method, "/");
settings.getCache()->upsert(cacheKey, store, {}, storePath);
}
accessor->setPathDisplay("«" + to_string() + "»");
@@ -61,11 +61,6 @@ std::pair<CanonPath, std::optional<std::string>> FilteringSourceAccessor::getFin
return next->getFingerprint(prefix / path);
}
void FilteringSourceAccessor::invalidateCache(const CanonPath & path)
{
next->invalidateCache(prefix / path);
}
void FilteringSourceAccessor::checkAccess(const CanonPath & path)
{
if (!isAllowed(path))
+2 -2
View File
@@ -268,10 +268,10 @@ void Fetch::fetch(
return;
}
auto cacheDir = getCacheDir() / "git-lfs";
std::filesystem::path cacheDir = getCacheDir() / "git-lfs";
std::string key = hashString(HashAlgorithm::SHA256, pointerFilePath.rel()).to_string(HashFormat::Base16, false)
+ "/" + pointer->oid;
auto cachePath = cacheDir / key;
std::filesystem::path cachePath = cacheDir / key;
if (pathExists(cachePath)) {
debug("using cache entry %s -> %s", key, PathFmt(cachePath));
sink(readFile(cachePath));
+1 -1
View File
@@ -666,7 +666,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
.program = "git",
.args =
{"-c",
"gpg.ssh.allowedSignersFile=" + allowedSignersFile.string(),
"gpg.ssh.allowedSignersFile=" + allowedSignersFile,
"-C",
path.string(),
"verify-commit",
+14 -15
View File
@@ -37,16 +37,15 @@ namespace {
// old version of git, which will ignore unrecognized `-c` options.
const std::string gitInitialBranch = "__nix_dummy_branch";
static bool isCacheFileWithinTtl(const Settings & settings, time_t now, const PosixStat & st)
bool isCacheFileWithinTtl(time_t now, const struct stat & st)
{
return st.st_mtime + static_cast<time_t>(settings.tarballTtl) > now;
}
std::filesystem::path getCachePath(std::string_view key, bool shallow)
{
auto name =
hashString(HashAlgorithm::SHA256, key).to_string(HashFormat::Nix32, false) + (shallow ? "-shallow" : "");
return getCacheDir() / "gitv3" / std::move(name);
return getCacheDir() / "gitv3"
/ (hashString(HashAlgorithm::SHA256, key).to_string(HashFormat::Nix32, false) + (shallow ? "-shallow" : ""));
}
// Returns the name of the HEAD branch.
@@ -106,7 +105,7 @@ bool storeCachedHead(const std::string & actualUrl, bool shallow, const std::str
return true;
}
static std::optional<std::string> readHeadCached(const Settings & settings, const std::string & actualUrl, bool shallow)
std::optional<std::string> readHeadCached(const std::string & actualUrl, bool shallow)
{
// Create a cache path to store the branch of the HEAD ref. Append something
// in front of the URL to prevent collision with the repository itself.
@@ -114,11 +113,11 @@ static std::optional<std::string> readHeadCached(const Settings & settings, cons
std::filesystem::path headRefFile = cacheDir / "HEAD";
time_t now = time(0);
auto st = maybeStat(headRefFile);
struct stat st;
std::optional<std::string> cachedRef;
if (st) {
if (stat(headRefFile.string().c_str(), &st) == 0) {
cachedRef = readHead(cacheDir);
if (cachedRef != std::nullopt && *cachedRef != gitInitialBranch && isCacheFileWithinTtl(settings, now, *st)) {
if (cachedRef != std::nullopt && *cachedRef != gitInitialBranch && isCacheFileWithinTtl(now, st)) {
debug("using cached HEAD ref '%s' for repo '%s'", *cachedRef, actualUrl);
return cachedRef;
}
@@ -630,7 +629,7 @@ struct GitInputScheme : InputScheme
if (url.scheme == "file" && !forceHttp && !isBareRepository(renderUrlPathEnsureLegal(url.path))) {
auto path = renderUrlPathEnsureLegal(url.path);
if (!std::filesystem::path(path).is_absolute()) {
if (!isAbsolute(path)) {
warn(
"Fetching Git repository '%s', which uses a path relative to the current directory. "
"This is not supported and will stop working in a future release. "
@@ -729,12 +728,12 @@ struct GitInputScheme : InputScheme
return revCount;
}
std::string getDefaultRef(const Settings & settings, const RepoInfo & repoInfo, bool shallow) const
std::string getDefaultRef(const RepoInfo & repoInfo, bool shallow) const
{
auto head = std::visit(
overloaded{
[&](const std::filesystem::path & path) { return GitRepo::openRepo(path, {})->getWorkdirRef(); },
[&](const ParsedURL & url) { return readHeadCached(settings, url.to_string(), shallow); }},
[&](const ParsedURL & url) { return readHeadCached(url.to_string(), shallow); }},
repoInfo.location);
if (!head) {
warn("could not read HEAD ref from repo at '%s', using 'master'", repoInfo.locationToArg());
@@ -784,7 +783,7 @@ struct GitInputScheme : InputScheme
auto originalRef = input.getRef();
bool shallow = getShallowAttr(input);
auto ref = originalRef ? *originalRef : getDefaultRef(settings, repoInfo, shallow);
auto ref = originalRef ? *originalRef : getDefaultRef(repoInfo, shallow);
input.attrs.insert_or_assign("ref", ref);
std::filesystem::path repoDir;
@@ -820,10 +819,10 @@ struct GitInputScheme : InputScheme
if (getAllRefsAttr(input)) {
doFetch = true;
} else {
/* If the local ref is older than 'tarball-ttl' seconds, do a
/* If the local ref is older than ‘tarball-ttl’ seconds, do a
git fetch to update the local ref to the remote ref. */
auto st = maybeStat(localRefFile);
doFetch = !st || !isCacheFileWithinTtl(settings, now, *st);
struct stat st;
doFetch = stat(localRefFile.string().c_str(), &st) != 0 || !isCacheFileWithinTtl(now, st);
}
}
@@ -129,25 +129,6 @@ struct Settings : public Config
true,
Xp::Flakes};
Setting<unsigned int> tarballTtl{
this,
60 * 60,
"tarball-ttl",
R"(
The number of seconds a downloaded tarball is considered fresh. If
the cached tarball is stale, Nix checks whether it is still up
to date using the ETag header. Nix downloads a new version if
the ETag header is unsupported, or the cached ETag doesn't match.
Setting the TTL to `0` forces Nix to always check if the tarball is
up to date.
Nix caches tarballs in `$XDG_CACHE_HOME/nix/tarballs`.
Files fetched via `NIX_PATH`, `fetchGit`, `fetchMercurial`,
`fetchTarball`, and `fetchurl` respect this TTL.
)"};
ref<Cache> getCache() const;
ref<GitRepo> getTarballCache() const;
@@ -24,17 +24,7 @@ StorePath fetchToStore(
PathFilter * filter = nullptr,
RepairFlag repair = NoRepair);
std::pair<StorePath, Hash> fetchToStore2(
const fetchers::Settings & settings,
Store & store,
const SourcePath & path,
FetchMode mode,
std::string_view name = "source",
ContentAddressMethod method = ContentAddressMethod::Raw::NixArchive,
PathFilter * filter = nullptr,
RepairFlag repair = NoRepair);
fetchers::Cache::Key
makeSourcePathToHashCacheKey(std::string_view fingerprint, ContentAddressMethod method, const CanonPath & path);
fetchers::Cache::Key makeFetchToStoreCacheKey(
const std::string & name, const std::string & fingerprint, ContentAddressMethod method, const std::string & path);
} // namespace nix
@@ -52,8 +52,6 @@ struct FilteringSourceAccessor : SourceAccessor
std::pair<CanonPath, std::optional<std::string>> getFingerprint(const CanonPath & path) override;
void invalidateCache(const CanonPath & path) override;
/**
* Call `makeNotAllowedError` to throw a `RestrictedPathError`
* exception if `isAllowed()` returns `false` for `path`.
@@ -1,5 +1,3 @@
#pragma once
#include "nix/fetchers/fetchers.hh"
namespace nix::fetchers {
+3 -4
View File
@@ -1,6 +1,5 @@
#include "nix/fetchers/fetchers.hh"
#include "nix/util/processes.hh"
#include "nix/util/environment-variables.hh"
#include "nix/util/users.hh"
#include "nix/fetchers/cache.hh"
#include "nix/store/globals.hh"
@@ -17,9 +16,9 @@ namespace nix::fetchers {
static RunOptions hgOptions(const Strings & args)
{
auto env = getEnvOs();
auto env = getEnv();
// Set HGPLAIN: this means we get consistent output from hg and avoids leakage from a user or system .hgrc.
env[OS_STR("HGPLAIN")] = OS_STR("");
env["HGPLAIN"] = "";
return {.program = "hg", .lookupPath = true, .args = args, .environment = env};
}
@@ -302,7 +301,7 @@ struct MercurialInputScheme : InputScheme
}
}
} else {
createDirs(cacheDir.parent_path());
createDirs(dirOf(cacheDir.string()));
runHg({"clone", "--noupdate", "--", actualUrl, cacheDir.string()});
}
}
+9 -7
View File
@@ -117,7 +117,7 @@ struct PathInputScheme : InputScheme
std::optional<std::string> isRelative(const Input & input) const override
{
auto path = getStrAttr(input.attrs, "path");
if (std::filesystem::path(path).is_absolute())
if (isAbsolute(path))
return std::nullopt;
else
return path;
@@ -132,7 +132,7 @@ struct PathInputScheme : InputScheme
{
auto path = getStrAttr(input.attrs, "path");
if (std::filesystem::path(path).is_absolute())
if (isAbsolute(path))
return canonPath(path);
throw Error("cannot fetch input '%s' because it uses a relative path", input.to_string());
@@ -165,12 +165,14 @@ struct PathInputScheme : InputScheme
// To prevent `fetchToStore()` copying the path again to Nix
// store, pre-create an entry in the fetcher cache.
auto narHash = store.queryPathInfo(*storePath)->narHash.to_string(HashFormat::SRI, true);
accessor->fingerprint = fmt("path:%s", narHash);
accessor->fingerprint =
fmt("path:%s", store.queryPathInfo(*storePath)->narHash.to_string(HashFormat::SRI, true));
settings.getCache()->upsert(
makeSourcePathToHashCacheKey(
*accessor->fingerprint, ContentAddressMethod::Raw::NixArchive, CanonPath::root),
{{"hash", narHash}});
makeFetchToStoreCacheKey(
input.getName(), *accessor->fingerprint, ContentAddressMethod::Raw::NixArchive, "/"),
store,
{},
*storePath);
/* Trust the lastModified value supplied by the user, if
any. It's not a "secure" attribute so we don't care. */
+3 -3
View File
@@ -92,7 +92,7 @@ void Registry::remove(const Input & input)
static std::filesystem::path getSystemRegistryPath()
{
return nixConfDir() / "registry.json";
return settings.nixConfDir / "registry.json";
}
static std::shared_ptr<Registry> getSystemRegistry(const Settings & settings)
@@ -147,13 +147,13 @@ static std::shared_ptr<Registry> getGlobalRegistry(const Settings & settings, St
return Registry::read(
settings,
[&] -> SourcePath {
if (!std::filesystem::path(path).is_absolute()) {
if (!isAbsolute(path)) {
auto storePath = downloadFile(store, settings, path, "flake-registry.json").storePath;
if (auto store2 = dynamic_cast<LocalFSStore *>(&store))
store2->addPermRoot(storePath, (getCacheDir() / "flake-registry.json").string());
return {store.requireStoreObjectAccessor(storePath)};
} else {
return SourcePath{makeFSSourceAccessor(path)}.resolveSymlinks();
return SourcePath{getFSSourceAccessor(), CanonPath{path}}.resolveSymlinks();
}
}(),
Registry::Global);
+2 -3
View File
@@ -177,9 +177,8 @@ static DownloadTarballResult downloadTarball_(
the entire file to disk so libarchive can access it
in random-access mode. */
auto [fdTemp, path] = createTempFile("nix-zipfile");
cleanupTemp.cancel();
cleanupTemp = {path};
debug("downloading '%s' into %s...", url, PathFmt(path));
cleanupTemp.reset(path);
debug("downloading '%s' into '%s'...", url, path);
{
FdSink sink(fdTemp.get());
source->drainInto(sink);
+1 -1
View File
@@ -13,7 +13,7 @@ struct nix_flake_settings
struct nix_flake_reference_parse_flags
{
std::optional<std::filesystem::path> baseDirectory;
std::optional<nix::Path> baseDirectory;
};
struct nix_flake_reference
+1 -1
View File
@@ -31,7 +31,7 @@ namespace nix::flake {
// setting name -> setting value -> allow or ignore.
typedef std::map<std::string, std::map<std::string, bool>> TrustedList;
static std::filesystem::path trustedListPath()
std::filesystem::path trustedListPath()
{
return getDataDir() / "trusted-settings.json";
}
-1
View File
@@ -128,7 +128,6 @@ static void prim_flakeRefToString(EvalState & state, const PosIdx pos, Value **
state.forceAttrs(*args[0], noPos, "while evaluating the argument passed to builtins.flakeRefToString");
fetchers::Attrs attrs;
for (const auto & attr : *args[0]->attrs()) {
state.forceValue(*attr.value, attr.pos);
auto t = attr.value->type();
if (t == nInt) {
auto intValue = attr.value->integer().value;
-2
View File
@@ -876,8 +876,6 @@ lockFlake(const Settings & settings, EvalState & state, const FlakeRef & topRef,
CanonPath((topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock"),
newLockFileS,
commitMessage);
flake.lockFilePath().invalidateCache();
}
/* Rewriting the lockfile changed the top-level
+8 -14
View File
@@ -121,23 +121,19 @@ std::pair<FlakeRef, std::string> parsePathFlakeRefWithFragment(
to 'baseDir'). If so, search upward to the root of the
repo (i.e. the directory containing .git). */
{
auto baseDirPath = *baseDir;
path = absPath(path, &baseDirPath, true).string();
}
path = absPath(path, baseDir->string(), true);
if (isFlake) {
if (!S_ISDIR(lstat(path).st_mode)) {
if (baseNameOf(path) == "flake.nix") {
// Be gentle with people who accidentally write `/foo/bar/flake.nix` instead of `/foo/bar`
auto parentPath = std::filesystem::path(path).parent_path().string();
warn(
"Path '%s' should point at the directory containing the 'flake.nix' file, not the file itself. "
"Pretending that you meant '%s'",
path,
parentPath);
path = parentPath;
dirOf(path));
path = dirOf(path);
} else {
throw BadURL("path '%s' is not a flake (because it's not a directory)", path);
}
@@ -161,7 +157,7 @@ std::pair<FlakeRef, std::string> parsePathFlakeRefWithFragment(
if (lstat(path).st_dev != device)
throw Error("unable to find a flake before encountering filesystem boundary at '%s'", path);
}
path = std::filesystem::path(path).parent_path().string();
path = dirOf(path);
}
if (!found)
throw BadURL("could not find a flake.nix file");
@@ -196,12 +192,12 @@ std::pair<FlakeRef, std::string> parsePathFlakeRefWithFragment(
}
subdir = std::string(baseNameOf(flakeRoot)) + (subdir.empty() ? "" : "/" + subdir);
flakeRoot = std::filesystem::path(flakeRoot).parent_path().string();
flakeRoot = dirOf(flakeRoot);
}
}
} else {
if (!preserveRelativePaths && !std::filesystem::path(path).is_absolute())
if (!preserveRelativePaths && !isAbsolute(path))
throw BadURL("flake reference '%s' is not an absolute path", url);
}
@@ -255,10 +251,8 @@ std::optional<std::pair<FlakeRef, std::string>> parseURLFlakeRef(
if (baseDir && (parsed.scheme == "path" || parsed.scheme == "git+file")) {
/* Here we know that the path must not contain encoded '/' or NUL bytes. */
auto path = renderUrlPathEnsureLegal(parsed.path);
if (!std::filesystem::path(path).is_absolute()) {
auto baseDirPath = *baseDir;
parsed.path = splitString<std::vector<std::string>>(absPath(path, &baseDirPath).string(), "/");
}
if (!isAbsolute(path))
parsed.path = splitString<std::vector<std::string>>(absPath(path, baseDir->string()), "/");
}
return fromParsedURL(fetchSettings, std::move(parsed), isFlake);
} catch (BadURL &) {
+15 -1
View File
@@ -13,6 +13,8 @@
namespace nix {
int handleExceptions(const std::string & programName, std::function<void()> fun);
/**
* Don't forget to call initPlugins() after settings are initialized!
* @param loadConfig Whether to load configuration from `nix.conf`, `NIX_CONFIG`, etc. May be disabled for unit tests.
@@ -89,7 +91,19 @@ extern volatile ::sig_atomic_t blockInt;
struct GCResults;
void printFreed(bool dryRun, const GCResults & results);
struct PrintFreed
{
bool show;
const GCResults & results;
PrintFreed(bool show, const GCResults & results)
: show(show)
, results(results)
{
}
~PrintFreed();
};
#ifndef _WIN32
/**
+37 -41
View File
@@ -1,9 +1,7 @@
#include "nix/store/globals.hh"
#include "nix/util/current-process.hh"
#include "nix/util/executable-path.hh"
#include "nix/main/shared.hh"
#include "nix/store/store-api.hh"
#include "nix/store/store-open.hh"
#include "nix/store/gc-store.hh"
#include "nix/main/loggers.hh"
#include "nix/main/progress-bar.hh"
@@ -19,10 +17,6 @@
#include <sys/stat.h>
#include <unistd.h>
#include <signal.h>
#ifndef _WIN32
# include <sys/resource.h>
#endif
#ifdef __linux__
# include <features.h>
#endif
@@ -122,26 +116,6 @@ std::string getArg(const std::string & opt, Strings::iterator & i, const Strings
static void sigHandler(int signo) {}
#endif
/**
* Increase the open file soft limit to the hard limit. On some
* platforms (macOS), the default soft limit is very low, but the hard
* limit is high. So let's just raise it the maximum permitted.
*/
void bumpFileLimit()
{
#ifndef _WIN32
struct rlimit limit;
if (getrlimit(RLIMIT_NOFILE, &limit) != 0)
return;
if (limit.rlim_cur < limit.rlim_max) {
limit.rlim_cur = limit.rlim_max;
// Ignore errors, this is best effort.
setrlimit(RLIMIT_NOFILE, &limit);
}
#endif
}
void initNix(bool loadConfig)
{
/* Turn on buffering for cerr. */
@@ -209,8 +183,6 @@ void initNix(bool loadConfig)
now. In particular, store objects should be readable by
everybody. */
umask(0022);
bumpFileLimit();
}
LegacyArgs::LegacyArgs(
@@ -237,13 +209,13 @@ LegacyArgs::LegacyArgs(
.longName = "keep-going",
.shortName = 'k',
.description = "Keep going after a build fails.",
.handler = {&(bool &) settings.getWorkerSettings().keepGoing, true},
.handler = {&(bool &) settings.keepGoing, true},
});
addFlag({
.longName = "fallback",
.description = "Build from source if substitution fails.",
.handler = {&(bool &) settings.getWorkerSettings().tryFallback, true},
.handler = {&(bool &) settings.tryFallback, true},
});
auto intSettingAlias =
@@ -280,7 +252,7 @@ LegacyArgs::LegacyArgs(
.longName = "store",
.description = "The URL of the Nix store to use.",
.labels = {"store-uri"},
.handler = {[](std::string s) { settings.storeUri = StoreReference::parse(s); }},
.handler = {&(std::string &) settings.storeUri},
});
}
@@ -332,16 +304,43 @@ void printVersion(const std::string & programName)
std::cout << "System type: " << settings.thisSystem << "\n";
std::cout << "Additional system types: " << concatStringsSep(", ", settings.extraPlatforms.get()) << "\n";
std::cout << "Features: " << concatStringsSep(", ", cfg) << "\n";
std::cout << "System configuration file: " << nixConfFile() << "\n";
std::cout << "User configuration files: "
<< os_string_to_string(ExecutablePath{.directories = nixUserConfFiles()}.render()) << "\n";
std::cout << "Store directory: " << resolveStoreConfig(StoreReference{settings.storeUri.get()})->storeDir
<< "\n";
std::cout << "System configuration file: " << (settings.nixConfDir / "nix.conf").string() << "\n";
std::cout << "User configuration files: " << concatStringsSep(":", settings.nixUserConfFiles) << "\n";
std::cout << "Store directory: " << settings.nixStore << "\n";
std::cout << "State directory: " << settings.nixStateDir << "\n";
}
throw Exit();
}
int handleExceptions(const std::string & programName, std::function<void()> fun)
{
ReceiveInterrupts receiveInterrupts; // FIXME: need better place for this
ErrorInfo::programName = baseNameOf(programName);
std::string error = ANSI_RED "error:" ANSI_NORMAL " ";
try {
fun();
} catch (Exit & e) {
return e.status;
} catch (UsageError & e) {
logError(e.info());
printError("Try '%1% --help' for more information.", programName);
return 1;
} catch (BaseError & e) {
logError(e.info());
return e.info().status;
} catch (std::bad_alloc & e) {
printError(error + "out of memory");
return 1;
} catch (std::exception & e) {
printError(error + e.what());
return 1;
}
return 0;
}
RunPager::RunPager()
{
if (!isatty(STDOUT_FILENO))
@@ -396,12 +395,9 @@ RunPager::~RunPager()
}
}
void printFreed(bool dryRun, const GCResults & results)
PrintFreed::~PrintFreed()
{
/* bytesFreed cannot be reliably computed without actually deleting store paths because of hardlinking. */
if (dryRun)
std::cout << fmt("%d store paths would be deleted\n", results.paths.size());
else
if (show)
std::cout << fmt("%d store paths deleted, %s freed\n", results.paths.size(), renderSize(results.bytesFreed));
}
+11 -37
View File
@@ -9,7 +9,6 @@
#include "nix/store/path.hh"
#include "nix/store/store-api.hh"
#include "nix/store/store-open.hh"
#include "nix/store/store-reference.hh"
#include "nix/store/build-result.hh"
#include "nix/store/local-fs-store.hh"
#include "nix/util/base-nix-32.hh"
@@ -48,14 +47,14 @@ Store * nix_store_open(nix_c_context * context, const char * uri, const char ***
if (uri_str.empty())
return new Store{nix::openStore()};
auto storeRef = nix::StoreReference::parse(uri_str);
if (!params)
return new Store{nix::openStore(uri_str)};
if (params) {
for (size_t i = 0; params[i] != nullptr; i++) {
storeRef.params[params[i][0]] = params[i][1];
}
nix::Store::Config::Params params_map;
for (size_t i = 0; params[i] != nullptr; i++) {
params_map[params[i][0]] = params[i][1];
}
return new Store{nix::openStore(std::move(storeRef))};
return new Store{nix::openStore(uri_str, params_map)};
}
NIXC_CATCH_ERRS_NULL
}
@@ -183,8 +182,10 @@ nix_err nix_store_realise(
assert(results.size() == 1);
// Check if any builds failed
for (auto & result : results)
result.tryThrowBuildError();
for (auto & result : results) {
if (auto * failureP = result.tryGetFailure())
failureP->rethrow();
}
if (callback) {
for (const auto & result : results) {
@@ -308,12 +309,7 @@ StorePath * nix_add_derivation(nix_c_context * context, Store * store, nix_deriv
if (context)
context->last_err_code = NIX_OK;
try {
/* Quite dubious that users would want this to silently suceed
without actually writing the derivation if this setting is
set, but it was that way already, so we are doing this for
back-compat for now. */
auto ret = nix::settings.readOnlyMode ? nix::computeStorePath(*store->ptr, derivation->drv)
: store->ptr->writeDerivation(derivation->drv, nix::NoRepair);
auto ret = nix::writeDerivation(*store->ptr, derivation->drv, nix::NoRepair);
return new StorePath{ret};
}
@@ -358,26 +354,4 @@ StorePath * nix_store_query_path_from_hash_part(nix_c_context * context, Store *
NIXC_CATCH_ERRS_NULL
}
nix_err nix_store_copy_path(
nix_c_context * context, Store * srcStore, Store * dstStore, const StorePath * path, bool repair, bool checkSigs)
{
if (context)
context->last_err_code = NIX_OK;
try {
if (srcStore == nullptr)
return nix_set_err_msg(context, NIX_ERR_UNKNOWN, "Source store is null");
if (dstStore == nullptr)
return nix_set_err_msg(context, NIX_ERR_UNKNOWN, "Destination store is null");
if (path == nullptr)
return nix_set_err_msg(context, NIX_ERR_UNKNOWN, "Store path is null");
auto repairFlag = repair ? nix::RepairFlag::Repair : nix::RepairFlag::NoRepair;
auto checkSigsFlag = checkSigs ? nix::CheckSigsFlag::CheckSigs : nix::CheckSigsFlag::NoCheckSigs;
nix::copyStorePath(*srcStore->ptr, *dstStore->ptr, path->path, repairFlag, checkSigsFlag);
}
NIXC_CATCH_ERRS
}
} // extern "C"
-13
View File
@@ -270,19 +270,6 @@ nix_derivation * nix_store_drv_from_store_path(nix_c_context * context, Store *
*/
StorePath * nix_store_query_path_from_hash_part(nix_c_context * context, Store * store, const char * hash);
/**
* @brief Copy a path from one store to another.
*
* @param[out] context Optional, stores error information
* @param[in] srcStore nix source store reference
* @param[in] dstStore nix destination store reference
* @param[in] path The path to copy
* @param[in] repair Whether to repair the path
* @param[in] checkSigs Whether to check path signatures are trusted before copying
*/
nix_err nix_store_copy_path(
nix_c_context * context, Store * srcStore, Store * dstStore, const StorePath * path, bool repair, bool checkSigs);
// cffi end
#ifdef __cplusplus
}
+10 -29
View File
@@ -9,12 +9,11 @@ void TestHttpBinaryCacheStore::init()
BinaryCacheStore::init();
}
ref<TestHttpBinaryCacheStore> TestHttpBinaryCacheStoreConfig::openTestStore(ref<FileTransfer> fileTransfer) const
ref<TestHttpBinaryCacheStore> TestHttpBinaryCacheStoreConfig::openTestStore() const
{
auto store = make_ref<TestHttpBinaryCacheStore>(
ref{// FIXME we shouldn't actually need a mutable config
std::const_pointer_cast<HttpBinaryCacheStore::Config>(shared_from_this())},
fileTransfer);
std::const_pointer_cast<HttpBinaryCacheStore::Config>(shared_from_this())});
store->init();
return store;
}
@@ -50,11 +49,9 @@ void HttpsBinaryCacheStoreTest::SetUp()
// clang-format off
openssl({"ecparam", "-genkey", "-name", "prime256v1", "-out", caKey.string()});
openssl({"req", "-new", "-x509", "-days", "1", "-key", caKey.string(), "-out", caCert.string(), "-subj", "/CN=TestCA"});
auto serverExtFile = tmpDir / "server.ext";
writeFile(serverExtFile, "subjectAltName=DNS:localhost,IP:127.0.0.1");
openssl({"ecparam", "-genkey", "-name", "prime256v1", "-out", serverKey.string()});
openssl({"req", "-new", "-key", serverKey.string(), "-out", (tmpDir / "server.csr").string(), "-subj", "/CN=localhost", "-addext", "subjectAltName=DNS:localhost,IP:127.0.0.1"});
openssl({"x509", "-req", "-in", (tmpDir / "server.csr").string(), "-CA", caCert.string(), "-CAkey", caKey.string(), "-CAcreateserial", "-out", serverCert.string(), "-days", "1", "-extfile", serverExtFile.string()});
openssl({"req", "-new", "-key", serverKey.string(), "-out", (tmpDir / "server.csr").string(), "-subj", "/CN=localhost"});
openssl({"x509", "-req", "-in", (tmpDir / "server.csr").string(), "-CA", caCert.string(), "-CAkey", caKey.string(), "-CAcreateserial", "-out", serverCert.string(), "-days", "1"});
openssl({"ecparam", "-genkey", "-name", "prime256v1", "-out", clientKey.string()});
openssl({"req", "-new", "-key", clientKey.string(), "-out", (tmpDir / "client.csr").string(), "-subj", "/CN=TestClient"});
openssl({"x509", "-req", "-in", (tmpDir / "client.csr").string(), "-CA", caCert.string(), "-CAkey", caKey.string(), "-CAcreateserial", "-out", clientCert.string(), "-days", "1"});
@@ -82,18 +79,16 @@ void HttpsBinaryCacheStoreTest::SetUp()
for the port explicitly - this is enough. */
std::this_thread::sleep_for(std::chrono::milliseconds(50));
/* Create custom FileTransferSettings with our test CA certificate.
This avoids mutating global settings. */
testFileTransferSettings = std::make_unique<FileTransferSettings>();
testFileTransferSettings->caFile = caCert;
testFileTransfer = makeFileTransfer(*testFileTransferSettings);
/* FIXME: Don't use global settings. Tests are not run concurrently, so this is fine for now. */
oldCaCert = settings.caFile;
settings.caFile = caCert.string();
}
void HttpsBinaryCacheStoreTest::TearDown()
{
settings.caFile = oldCaCert;
serverPid.kill();
delTmpDir.reset();
testFileTransferSettings.reset();
}
std::vector<std::string> HttpsBinaryCacheStoreTest::serverArgs()
@@ -120,25 +115,11 @@ std::vector<std::string> HttpsBinaryCacheStoreMtlsTest::serverArgs()
return args;
}
ref<TestHttpBinaryCacheStoreConfig> HttpsBinaryCacheStoreTest::makeConfig()
ref<TestHttpBinaryCacheStoreConfig> HttpsBinaryCacheStoreTest::makeConfig(BinaryCacheStoreConfig::Params params)
{
auto res = make_ref<TestHttpBinaryCacheStoreConfig>(
ParsedURL{
.scheme = "https",
.authority =
ParsedURL::Authority{
.host = "localhost",
.port = port,
},
},
TestHttpBinaryCacheStoreConfig::Params{});
auto res = make_ref<TestHttpBinaryCacheStoreConfig>("https", fmt("localhost:%d", port), std::move(params));
res->pathInfoCacheSize = 0; /* We don't want any caching in tests. */
return res;
}
ref<TestHttpBinaryCacheStore> HttpsBinaryCacheStoreTest::openStore(ref<TestHttpBinaryCacheStoreConfig> config)
{
return config->openTestStore(ref<FileTransfer>{testFileTransfer});
}
} // namespace nix::testing
@@ -28,10 +28,10 @@ public:
TestHttpBinaryCacheStore & operator=(const TestHttpBinaryCacheStore &) = delete;
TestHttpBinaryCacheStore & operator=(TestHttpBinaryCacheStore &&) = delete;
TestHttpBinaryCacheStore(ref<HttpBinaryCacheStoreConfig> config, ref<FileTransfer> fileTransfer)
TestHttpBinaryCacheStore(ref<HttpBinaryCacheStoreConfig> config)
: Store{*config}
, BinaryCacheStore{*config}
, HttpBinaryCacheStore(config, fileTransfer)
, HttpBinaryCacheStore(config)
{
diskCache = nullptr; /* Disable caching, we'll be creating a new binary cache for each test. */
}
@@ -42,13 +42,19 @@ public:
class TestHttpBinaryCacheStoreConfig : public HttpBinaryCacheStoreConfig
{
public:
TestHttpBinaryCacheStoreConfig(ParsedURL url, const Store::Config::Params & params)
TestHttpBinaryCacheStoreConfig(
std::string_view scheme, std::string_view cacheUri, const Store::Config::Params & params)
: StoreConfig(params)
, HttpBinaryCacheStoreConfig(url, params)
, HttpBinaryCacheStoreConfig(scheme, cacheUri, params)
{
}
ref<TestHttpBinaryCacheStore> openTestStore(ref<FileTransfer> fileTransfer) const;
ref<TestHttpBinaryCacheStore> openTestStore() const;
ref<Store> openStore() const override
{
return openTestStore();
}
};
class HttpsBinaryCacheStoreTest : public virtual LibStoreNetworkTest
@@ -65,29 +71,17 @@ protected:
std::filesystem::path tmpDir, cacheDir;
std::filesystem::path caCert, caKey, serverCert, serverKey;
std::filesystem::path clientCert, clientKey;
std::string oldCaCert;
Pid serverPid;
uint16_t port = 8443;
std::shared_ptr<Store> localCacheStore;
/**
* Custom FileTransferSettings with the test CA certificate.
* This is used instead of modifying global settings.
*/
std::unique_ptr<FileTransferSettings> testFileTransferSettings;
/**
* FileTransfer instance using our test settings.
* Initialized in SetUp().
*/
std::shared_ptr<FileTransfer> testFileTransfer;
static void openssl(Strings args);
void SetUp() override;
void TearDown() override;
virtual std::vector<std::string> serverArgs();
ref<TestHttpBinaryCacheStoreConfig> makeConfig();
ref<TestHttpBinaryCacheStore> openStore(ref<TestHttpBinaryCacheStoreConfig> config);
ref<TestHttpBinaryCacheStoreConfig> makeConfig(BinaryCacheStoreConfig::Params params);
};
class HttpsBinaryCacheStoreMtlsTest : public HttpsBinaryCacheStoreTest
@@ -51,7 +51,7 @@ protected:
// resolve any symlinks in i.e. on macOS /tmp -> /private/tmp
// because this is not allowed for a nix store.
auto tmpl =
nix::absPath(std::filesystem::path(nix::defaultTempDir()) / "tests_nix-store.XXXXXX", nullptr, true);
nix::absPath(std::filesystem::path(nix::defaultTempDir()) / "tests_nix-store.XXXXXX", std::nullopt, true);
nixDir = mkdtemp((char *) tmpl.c_str());
#endif
@@ -100,26 +100,26 @@ public:
writeProtoTest(STEM, VERSION, VALUE); \
}
#define VERSIONED_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE) \
VERSIONED_READ_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, (VERSION), VALUE) \
VERSIONED_WRITE_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, (VERSION), VALUE)
#define VERSIONED_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE) \
VERSIONED_READ_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE) \
VERSIONED_WRITE_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE)
#define VERSIONED_READ_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE) \
VERSIONED_READ_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, (VERSION), VALUE) \
TEST_F(FIXTURE, NAME##_json_read) \
{ \
readJsonTest(STEM, VALUE); \
#define VERSIONED_READ_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE) \
VERSIONED_READ_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE) \
TEST_F(FIXTURE, NAME##_json_read) \
{ \
readJsonTest(STEM, VALUE); \
}
#define VERSIONED_WRITE_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE) \
VERSIONED_WRITE_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, (VERSION), VALUE) \
TEST_F(FIXTURE, NAME##_json_write) \
{ \
writeJsonTest(STEM, VALUE); \
#define VERSIONED_WRITE_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE) \
VERSIONED_WRITE_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE) \
TEST_F(FIXTURE, NAME##_json_write) \
{ \
writeJsonTest(STEM, VALUE); \
}
#define VERSIONED_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE) \
VERSIONED_READ_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, (VERSION), VALUE) \
VERSIONED_WRITE_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, (VERSION), VALUE)
#define VERSIONED_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE) \
VERSIONED_READ_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE) \
VERSIONED_WRITE_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE)
} // namespace nix
+4 -4
View File
@@ -15,10 +15,10 @@ int testMainForBuidingPre(int argc, char ** argv)
}
// Disable build hook. We won't be testing remote builds in these unit tests. If we do, fix the above build hook.
settings.getWorkerSettings().buildHook = {};
settings.buildHook = {};
// No substituters, unless a test specifically requests.
settings.getWorkerSettings().substituters = {};
settings.substituters = {};
#ifdef __linux__ // should match the conditional around sandboxBuildDir declaration.
@@ -31,13 +31,13 @@ int testMainForBuidingPre(int argc, char ** argv)
// sandboxBuildDir = /build
// However, we have a rule that the store dir must not be inside the storeDir, so we need to pick a different
// sandboxBuildDir.
settings.getLocalSettings().sandboxBuildDir = "/test-build-dir-instead-of-usual-build-dir";
settings.sandboxBuildDir = "/test-build-dir-instead-of-usual-build-dir";
#endif
#ifdef __APPLE__
// Avoid this error, when already running in a sandbox:
// sandbox-exec: sandbox_apply: Operation not permitted
settings.getLocalSettings().sandboxMode = smDisabled;
settings.sandboxMode = smDisabled;
setEnv("_NIX_TEST_NO_SANDBOX", "1");
#endif
+6 -7
View File
@@ -1,7 +1,6 @@
#include <gtest/gtest.h>
#include "nix/store/build-result.hh"
#include "nix/util/tests/characterization.hh"
#include "nix/util/tests/json-characterization.hh"
namespace nix {
@@ -45,22 +44,22 @@ INSTANTIATE_TEST_SUITE_P(
std::pair{
"not-deterministic",
BuildResult{
.inner{BuildResult::Failure{{
.inner{BuildResult::Failure{
.status = BuildResult::Failure::NotDeterministic,
.msg = HintFmt("no idea why"),
.errorMsg = "no idea why",
.isNonDeterministic = false, // Note: This field is separate from the status
}}},
}},
.timesBuilt = 1,
},
},
std::pair{
"output-rejected",
BuildResult{
.inner{BuildResult::Failure{{
.inner{BuildResult::Failure{
.status = BuildResult::Failure::OutputRejected,
.msg = HintFmt("no idea why"),
.errorMsg = "no idea why",
.isNonDeterministic = false,
}}},
}},
.timesBuilt = 3,
.startTime = 30,
.stopTime = 50,
@@ -5,7 +5,6 @@
#include "nix/store/derivations.hh"
#include "nix/store/derived-path.hh"
#include "nix/store/derivation-options.hh"
#include "nix/store/globals.hh"
#include "nix/store/parsed-derivations.hh"
#include "nix/util/types.hh"
#include "nix/util/json-utils.hh"
@@ -193,7 +192,9 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_defaults)
EXPECT_EQ(options, advancedAttributes_defaults);
EXPECT_EQ(options.substitutesAllowed(settings.getWorkerSettings()), true);
EXPECT_EQ(options.canBuildLocally(*this->store, got), false);
EXPECT_EQ(options.willBuildLocally(*this->store, got), false);
EXPECT_EQ(options.substitutesAllowed(), true);
EXPECT_EQ(options.useUidRange(got), false);
});
};
@@ -241,7 +242,7 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes)
EXPECT_EQ(options, expected);
EXPECT_EQ(options.substitutesAllowed(settings.getWorkerSettings()), false);
EXPECT_EQ(options.substitutesAllowed(), false);
EXPECT_EQ(options.useUidRange(got), true);
});
};
@@ -333,7 +334,9 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_structuredAttrs_d
EXPECT_EQ(options, advancedAttributes_structuredAttrs_defaults);
EXPECT_EQ(options.substitutesAllowed(settings.getWorkerSettings()), true);
EXPECT_EQ(options.canBuildLocally(*this->store, got), false);
EXPECT_EQ(options.willBuildLocally(*this->store, got), false);
EXPECT_EQ(options.substitutesAllowed(), true);
EXPECT_EQ(options.useUidRange(got), false);
});
};
@@ -398,7 +401,9 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_structuredAttrs)
EXPECT_EQ(options, expected);
EXPECT_EQ(options.substitutesAllowed(settings.getWorkerSettings()), false);
EXPECT_EQ(options.canBuildLocally(*this->store, got), false);
EXPECT_EQ(options.willBuildLocally(*this->store, got), false);
EXPECT_EQ(options.substitutesAllowed(), false);
EXPECT_EQ(options.useUidRange(got), true);
});
};
+1 -1
View File
@@ -51,7 +51,7 @@ protected:
depDrv.fillInOutputPaths(*store);
// Write the dependency to the store
return store->writeDerivation(depDrv, NoRepair);
return writeDerivation(*store, depDrv, NoRepair);
}
public:
+31 -17
View File
@@ -45,18 +45,28 @@ using namespace std::string_literals;
TEST_F(HttpsBinaryCacheStoreTest, queryPathInfo)
{
auto store = openStore(makeConfig());
auto config = makeConfig({});
auto store = config->openStore();
StringSource dump{"test"sv};
auto path = localCacheStore->addToStoreFromDump(dump, "test-name", FileSerialisationMethod::Flat);
EXPECT_NO_THROW(store->queryPathInfo(path));
}
auto withNoRetries()
{
auto oldTries = fileTransferSettings.tries.get();
Finally restoreTries = [=]() { fileTransferSettings.tries = oldTries; };
fileTransferSettings.tries = 1; /* FIXME: Don't use global settings. */
return restoreTries;
}
TEST_F(HttpsBinaryCacheStoreMtlsTest, queryPathInfo)
{
auto config = makeConfig();
config->tlsCert = clientCert;
config->tlsKey = clientKey;
auto store = openStore(config);
auto config = makeConfig({
{"tls-certificate"s, clientCert.string()},
{"tls-private-key"s, clientKey.string()},
});
auto store = config->openStore();
StringSource dump{"test"sv};
auto path = localCacheStore->addToStoreFromDump(dump, "test-name", FileSerialisationMethod::Flat);
EXPECT_NO_THROW(store->queryPathInfo(path));
@@ -64,8 +74,9 @@ TEST_F(HttpsBinaryCacheStoreMtlsTest, queryPathInfo)
TEST_F(HttpsBinaryCacheStoreMtlsTest, rejectsWithoutClientCert)
{
testFileTransferSettings->tries = 1;
EXPECT_THROW(openStore(makeConfig()), Error);
auto restoreTries = withNoRetries();
auto config = makeConfig({});
EXPECT_THROW(config->openStore(), Error);
}
TEST_F(HttpsBinaryCacheStoreMtlsTest, rejectsWrongClientCert)
@@ -78,11 +89,12 @@ TEST_F(HttpsBinaryCacheStoreMtlsTest, rejectsWrongClientCert)
openssl({"req", "-new", "-x509", "-days", "1", "-key", wrongKey.string(), "-out", wrongCert.string(), "-subj", "/CN=WrongClient"});
// clang-format on
auto config = makeConfig();
config->tlsCert = wrongCert;
config->tlsKey = wrongKey;
testFileTransferSettings->tries = 1;
EXPECT_THROW(openStore(config), Error);
auto config = makeConfig({
{"tls-certificate"s, wrongCert.string()},
{"tls-private-key"s, wrongKey.string()},
});
auto restoreTries = withNoRetries();
EXPECT_THROW(config->openStore(), Error);
}
TEST_F(HttpsBinaryCacheStoreMtlsTest, doesNotSendCertOnRedirectToDifferentAuthority)
@@ -97,11 +109,13 @@ TEST_F(HttpsBinaryCacheStoreMtlsTest, doesNotSendCertOnRedirectToDifferentAuthor
writeFile(entry.path(), content);
}
auto config = makeConfig();
config->tlsCert = clientCert;
config->tlsKey = clientKey;
testFileTransferSettings->tries = 1;
auto store = openStore(config);
auto config = makeConfig({
{"tls-certificate"s, clientCert.string()},
{"tls-private-key"s, clientKey.string()},
});
auto store = config->openStore();
auto restoreTries = withNoRetries();
auto info = store->queryPathInfo(path);
NullSink null;
EXPECT_THROW(store->narFromPath(path, null), Error);
+4
View File
@@ -13,6 +13,10 @@ using testing::Eq;
using testing::Field;
using testing::SizeIs;
namespace nix::fs {
using namespace std::filesystem;
}
using namespace nix;
TEST(machines, getMachinesWithEmptyBuilders)
+2 -4
View File
@@ -25,8 +25,7 @@ TEST(NarInfoDiskCacheImpl, create_and_read)
SQLiteStmt getIds;
{
auto cache = NarInfoDiskCache::getTest(
settings.getNarInfoDiskCacheSettings(), {.useWAL = settings.useSQLiteWAL}, dbPath.string());
auto cache = getTestNarInfoDiskCache(dbPath.string());
// Set up "background noise" and check that different caches receive different ids
{
@@ -75,8 +74,7 @@ TEST(NarInfoDiskCacheImpl, create_and_read)
{
// We can't clear the in-memory cache, so we use a new cache object. This is
// more realistic anyway.
auto cache2 = NarInfoDiskCache::getTest(
settings.getNarInfoDiskCacheSettings(), {.useWAL = settings.useSQLiteWAL}, dbPath.string());
auto cache2 = getTestNarInfoDiskCache(dbPath.string());
{
auto r = cache2->upToDateCacheExists("http://foo");
+6 -6
View File
@@ -299,7 +299,7 @@ public:
nix_api_store_test_base::SetUp();
nix::experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations");
nix::settings.getWorkerSettings().substituters = {};
nix::settings.substituters = {};
store = open_local_store();
@@ -354,7 +354,7 @@ TEST_F(nix_api_store_test_base, build_from_json)
{
// FIXME get rid of these
nix::experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations");
nix::settings.getWorkerSettings().substituters = {};
nix::settings.substituters = {};
auto * store = open_local_store();
@@ -401,7 +401,7 @@ TEST_F(nix_api_store_test_base, nix_store_realise_invalid_system)
{
// Test that nix_store_realise properly reports errors when the system is invalid
nix::experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations");
nix::settings.getWorkerSettings().substituters = {};
nix::settings.substituters = {};
auto * store = open_local_store();
@@ -446,7 +446,7 @@ TEST_F(nix_api_store_test_base, nix_store_realise_builder_fails)
{
// Test that nix_store_realise properly reports errors when the builder fails
nix::experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations");
nix::settings.getWorkerSettings().substituters = {};
nix::settings.substituters = {};
auto * store = open_local_store();
@@ -491,7 +491,7 @@ TEST_F(nix_api_store_test_base, nix_store_realise_builder_no_output)
{
// Test that nix_store_realise properly reports errors when builder succeeds but produces no output
nix::experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations");
nix::settings.getWorkerSettings().substituters = {};
nix::settings.substituters = {};
auto * store = open_local_store();
@@ -687,7 +687,7 @@ TEST_F(NixApiStoreTestWithRealisedPath, nix_store_realise_output_ordering)
// This test uses a CA derivation with 10 outputs in randomized input order
// to verify that the callback order is deterministic and alphabetical.
nix::experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations");
nix::settings.getWorkerSettings().substituters = {};
nix::settings.substituters = {};
auto * store = open_local_store();
+15 -237
View File
@@ -104,33 +104,6 @@ INSTANTIATE_TEST_SUITE_P(
},
},
"with_absolute_endpoint_uri",
},
ParsedS3URLTestCase{
"s3://bucket/key?addressing-style=virtual",
{
.bucket = "bucket",
.key = {"key"},
.addressingStyle = S3AddressingStyle::Virtual,
},
"with_addressing_style_virtual",
},
ParsedS3URLTestCase{
"s3://bucket/key?addressing-style=path",
{
.bucket = "bucket",
.key = {"key"},
.addressingStyle = S3AddressingStyle::Path,
},
"with_addressing_style_path",
},
ParsedS3URLTestCase{
"s3://bucket/key?addressing-style=auto",
{
.bucket = "bucket",
.key = {"key"},
.addressingStyle = S3AddressingStyle::Auto,
},
"with_addressing_style_auto",
}),
[](const ::testing::TestParamInfo<ParsedS3URLTestCase> & info) { return info.param.description; });
@@ -165,26 +138,6 @@ INSTANTIATE_TEST_SUITE_P(
InvalidS3URLTestCase{"s3://bucket", "error: URI has a missing or invalid key", "missing_key"}),
[](const ::testing::TestParamInfo<InvalidS3URLTestCase> & info) { return info.param.description; });
TEST(ParsedS3URLTest, invalidAddressingStyleThrows)
{
ASSERT_THROW(ParsedS3URL::parse(parseURL("s3://bucket/key?addressing-style=bogus")), InvalidS3AddressingStyle);
}
TEST(ParsedS3URLTest, virtualStyleWithAuthoritylessEndpointThrows)
{
ParsedS3URL input{
.bucket = "bucket",
.key = {"key"},
.addressingStyle = S3AddressingStyle::Virtual,
.endpoint =
ParsedURL{
.scheme = "file",
.path = {"", "some", "path"},
},
};
ASSERT_THROW(input.toHttpsUrl(), nix::Error);
}
// =============================================================================
// S3 URL to HTTPS Conversion Tests
// =============================================================================
@@ -213,7 +166,6 @@ INSTANTIATE_TEST_SUITE_P(
S3ToHttpsConversion,
S3ToHttpsConversionTest,
::testing::Values(
// Default (auto) addressing style: virtual-hosted for standard AWS endpoints
S3ToHttpsConversionTestCase{
ParsedS3URL{
.bucket = "my-bucket",
@@ -221,10 +173,10 @@ INSTANTIATE_TEST_SUITE_P(
},
ParsedURL{
.scheme = "https",
.authority = ParsedURL::Authority{.host = "my-bucket.s3.us-east-1.amazonaws.com"},
.path = {"", "my-key.txt"},
.authority = ParsedURL::Authority{.host = "s3.us-east-1.amazonaws.com"},
.path = {"", "my-bucket", "my-key.txt"},
},
"https://my-bucket.s3.us-east-1.amazonaws.com/my-key.txt",
"https://s3.us-east-1.amazonaws.com/my-bucket/my-key.txt",
"basic_s3_default_region",
},
S3ToHttpsConversionTestCase{
@@ -235,13 +187,12 @@ INSTANTIATE_TEST_SUITE_P(
},
ParsedURL{
.scheme = "https",
.authority = ParsedURL::Authority{.host = "prod-cache.s3.eu-west-1.amazonaws.com"},
.path = {"", "nix", "store", "abc123.nar.xz"},
.authority = ParsedURL::Authority{.host = "s3.eu-west-1.amazonaws.com"},
.path = {"", "prod-cache", "nix", "store", "abc123.nar.xz"},
},
"https://prod-cache.s3.eu-west-1.amazonaws.com/nix/store/abc123.nar.xz",
"https://s3.eu-west-1.amazonaws.com/prod-cache/nix/store/abc123.nar.xz",
"with_eu_west_1_region",
},
// Custom endpoint authority: path-style by default
S3ToHttpsConversionTestCase{
ParsedS3URL{
.bucket = "bucket",
@@ -257,7 +208,6 @@ INSTANTIATE_TEST_SUITE_P(
"http://custom.s3.com/bucket/key",
"custom_endpoint_authority",
},
// Custom endpoint URL: path-style by default
S3ToHttpsConversionTestCase{
ParsedS3URL{
.bucket = "bucket",
@@ -286,10 +236,10 @@ INSTANTIATE_TEST_SUITE_P(
},
ParsedURL{
.scheme = "https",
.authority = ParsedURL::Authority{.host = "bucket.s3.ap-southeast-2.amazonaws.com"},
.path = {"", "path", "to", "file.txt"},
.authority = ParsedURL::Authority{.host = "s3.ap-southeast-2.amazonaws.com"},
.path = {"", "bucket", "path", "to", "file.txt"},
},
"https://bucket.s3.ap-southeast-2.amazonaws.com/path/to/file.txt",
"https://s3.ap-southeast-2.amazonaws.com/bucket/path/to/file.txt",
"complex_path_and_region",
},
S3ToHttpsConversionTestCase{
@@ -300,11 +250,11 @@ INSTANTIATE_TEST_SUITE_P(
},
ParsedURL{
.scheme = "https",
.authority = ParsedURL::Authority{.host = "my-bucket.s3.us-east-1.amazonaws.com"},
.path = {"", "my-key.txt"},
.authority = ParsedURL::Authority{.host = "s3.us-east-1.amazonaws.com"},
.path = {"", "my-bucket", "my-key.txt"},
.query = {{"versionId", "abc123xyz"}},
},
"https://my-bucket.s3.us-east-1.amazonaws.com/my-key.txt?versionId=abc123xyz",
"https://s3.us-east-1.amazonaws.com/my-bucket/my-key.txt?versionId=abc123xyz",
"with_versionId",
},
S3ToHttpsConversionTestCase{
@@ -316,185 +266,13 @@ INSTANTIATE_TEST_SUITE_P(
},
ParsedURL{
.scheme = "https",
.authority = ParsedURL::Authority{.host = "versioned-bucket.s3.eu-west-1.amazonaws.com"},
.path = {"", "path", "to", "object"},
.authority = ParsedURL::Authority{.host = "s3.eu-west-1.amazonaws.com"},
.path = {"", "versioned-bucket", "path", "to", "object"},
.query = {{"versionId", "version456"}},
},
"https://versioned-bucket.s3.eu-west-1.amazonaws.com/path/to/object?versionId=version456",
"https://s3.eu-west-1.amazonaws.com/versioned-bucket/path/to/object?versionId=version456",
"with_region_and_versionId",
},
// Explicit addressing-style=path forces path-style on standard AWS endpoints
S3ToHttpsConversionTestCase{
ParsedS3URL{
.bucket = "my-bucket",
.key = {"my-key.txt"},
.region = "us-west-2",
.addressingStyle = S3AddressingStyle::Path,
},
ParsedURL{
.scheme = "https",
.authority = ParsedURL::Authority{.host = "s3.us-west-2.amazonaws.com"},
.path = {"", "my-bucket", "my-key.txt"},
},
"https://s3.us-west-2.amazonaws.com/my-bucket/my-key.txt",
"explicit_path_style",
},
// Explicit addressing-style=virtual forces virtual-hosted-style on custom endpoints
S3ToHttpsConversionTestCase{
ParsedS3URL{
.bucket = "bucket",
.key = {"key"},
.scheme = "http",
.addressingStyle = S3AddressingStyle::Virtual,
.endpoint = ParsedURL::Authority{.host = "custom.s3.com"},
},
ParsedURL{
.scheme = "http",
.authority = ParsedURL::Authority{.host = "bucket.custom.s3.com"},
.path = {"", "key"},
},
"http://bucket.custom.s3.com/key",
"explicit_virtual_style_custom_endpoint",
},
// Explicit addressing-style=virtual with full endpoint URL
S3ToHttpsConversionTestCase{
ParsedS3URL{
.bucket = "bucket",
.key = {"key"},
.addressingStyle = S3AddressingStyle::Virtual,
.endpoint =
ParsedURL{
.scheme = "http",
.authority = ParsedURL::Authority{.host = "server", .port = 9000},
.path = {""},
},
},
ParsedURL{
.scheme = "http",
.authority = ParsedURL::Authority{.host = "bucket.server", .port = 9000},
.path = {"", "key"},
},
"http://bucket.server:9000/key",
"explicit_virtual_style_full_endpoint_url",
},
// Dotted bucket names work normally with explicit path-style
S3ToHttpsConversionTestCase{
ParsedS3URL{
.bucket = "my.bucket",
.key = {"key"},
.addressingStyle = S3AddressingStyle::Path,
},
ParsedURL{
.scheme = "https",
.authority = ParsedURL::Authority{.host = "s3.us-east-1.amazonaws.com"},
.path = {"", "my.bucket", "key"},
},
"https://s3.us-east-1.amazonaws.com/my.bucket/key",
"dotted_bucket_with_path_style",
},
// Dotted bucket names fall back to path-style with auto on standard AWS endpoints
S3ToHttpsConversionTestCase{
ParsedS3URL{
.bucket = "my.bucket.name",
.key = {"key"},
},
ParsedURL{
.scheme = "https",
.authority = ParsedURL::Authority{.host = "s3.us-east-1.amazonaws.com"},
.path = {"", "my.bucket.name", "key"},
},
"https://s3.us-east-1.amazonaws.com/my.bucket.name/key",
"dotted_bucket_with_auto_style_on_aws",
},
// Dotted bucket names work with auto style on custom endpoints (auto = path-style)
S3ToHttpsConversionTestCase{
ParsedS3URL{
.bucket = "my.bucket",
.key = {"key"},
.endpoint = ParsedURL::Authority{.host = "minio.local"},
},
ParsedURL{
.scheme = "https",
.authority = ParsedURL::Authority{.host = "minio.local"},
.path = {"", "my.bucket", "key"},
},
"https://minio.local/my.bucket/key",
"dotted_bucket_with_auto_style_custom_endpoint",
}),
[](const ::testing::TestParamInfo<S3ToHttpsConversionTestCase> & info) { return info.param.description; });
// =============================================================================
// S3 URL to HTTPS Conversion Error Tests
// =============================================================================
struct S3ToHttpsConversionErrorTestCase
{
ParsedS3URL input;
std::string description;
};
class S3ToHttpsConversionErrorTest : public ::testing::WithParamInterface<S3ToHttpsConversionErrorTestCase>,
public ::testing::Test
{};
TEST_P(S3ToHttpsConversionErrorTest, ThrowsOnConversion)
{
auto & [input, description] = GetParam();
ASSERT_THROW(input.toHttpsUrl(), nix::Error);
}
INSTANTIATE_TEST_SUITE_P(
S3ToHttpsConversionErrors,
S3ToHttpsConversionErrorTest,
::testing::Values(
S3ToHttpsConversionErrorTestCase{
ParsedS3URL{
.bucket = "bucket",
.key = {"key"},
.addressingStyle = S3AddressingStyle::Virtual,
.endpoint = ParsedURL::Authority{.host = ""},
},
"virtual_style_with_empty_host_authority",
},
S3ToHttpsConversionErrorTestCase{
ParsedS3URL{
.bucket = "my.bucket",
.key = {"key"},
.addressingStyle = S3AddressingStyle::Virtual,
},
"dotted_bucket_with_explicit_virtual_style",
},
S3ToHttpsConversionErrorTestCase{
ParsedS3URL{
.bucket = "my.bucket.name",
.key = {"key"},
.addressingStyle = S3AddressingStyle::Virtual,
},
"dotted_bucket_with_explicit_virtual_style_multi_dot",
},
S3ToHttpsConversionErrorTestCase{
ParsedS3URL{
.bucket = "my.bucket",
.key = {"key"},
.addressingStyle = S3AddressingStyle::Virtual,
.endpoint = ParsedURL::Authority{.host = "minio.local"},
},
"dotted_bucket_with_explicit_virtual_style_custom_authority",
},
S3ToHttpsConversionErrorTestCase{
ParsedS3URL{
.bucket = "my.bucket",
.key = {"key"},
.addressingStyle = S3AddressingStyle::Virtual,
.endpoint =
ParsedURL{
.scheme = "http",
.authority = ParsedURL::Authority{.host = "minio.local", .port = 9000},
.path = {""},
},
},
"dotted_bucket_with_explicit_virtual_style_full_endpoint_url",
}),
[](const ::testing::TestParamInfo<S3ToHttpsConversionErrorTestCase> & info) { return info.param.description; });
} // namespace nix
+58 -102
View File
@@ -25,10 +25,7 @@ struct ServeProtoTest : VersionedProtoTest<ServeProto, serveProtoDir>
* For serializers that don't care about the minimum version, we
* used the oldest one: 2.5.
*/
ServeProto::Version defaultVersion = {
.major = 2,
.minor = 5,
};
ServeProto::Version defaultVersion = 2 << 8 | 5;
};
VERSIONED_CHARACTERIZATION_TEST(
@@ -147,89 +144,66 @@ VERSIONED_READ_CHARACTERIZATION_TEST(
},
}))
VERSIONED_CHARACTERIZATION_TEST(ServeProtoTest, buildResult_2_2, "build-result-2.2", 2 << 8 | 2, ({
using namespace std::literals::chrono_literals;
std::tuple<BuildResult, BuildResult, BuildResult> t{
BuildResult{.inner{BuildResult::Failure{
.status = BuildResult::Failure::OutputRejected,
.errorMsg = "no idea why",
}}},
BuildResult{.inner{BuildResult::Failure{
.status = BuildResult::Failure::NotDeterministic,
.errorMsg = "no idea why",
}}},
BuildResult{.inner{BuildResult::Success{
.status = BuildResult::Success::Built,
}}},
};
t;
}))
VERSIONED_CHARACTERIZATION_TEST(ServeProtoTest, buildResult_2_3, "build-result-2.3", 2 << 8 | 3, ({
using namespace std::literals::chrono_literals;
std::tuple<BuildResult, BuildResult, BuildResult> t{
BuildResult{.inner{BuildResult::Failure{
.status = BuildResult::Failure::OutputRejected,
.errorMsg = "no idea why",
}}},
BuildResult{
.inner{BuildResult::Failure{
.status = BuildResult::Failure::NotDeterministic,
.errorMsg = "no idea why",
.isNonDeterministic = true,
}},
.timesBuilt = 3,
.startTime = 30,
.stopTime = 50,
},
BuildResult{
.inner{BuildResult::Success{
.status = BuildResult::Success::Built,
}},
.startTime = 30,
.stopTime = 50,
},
};
t;
}))
VERSIONED_CHARACTERIZATION_TEST(
ServeProtoTest,
buildResult_2_2,
"build-result-2.2",
(ServeProto::Version{
.major = 2,
.minor = 2,
}),
({
ServeProtoTest, buildResult_2_6, "build-result-2.6", 2 << 8 | 6, ({
using namespace std::literals::chrono_literals;
std::tuple<BuildResult, BuildResult, BuildResult> t{
BuildResult{.inner{BuildResult::Failure{{
BuildResult{.inner{BuildResult::Failure{
.status = BuildResult::Failure::OutputRejected,
.msg = HintFmt("no idea why"),
}}}},
BuildResult{.inner{BuildResult::Failure{{
.status = BuildResult::Failure::NotDeterministic,
.msg = HintFmt("no idea why"),
}}}},
BuildResult{.inner{BuildResult::Success{
.status = BuildResult::Success::Built,
.errorMsg = "no idea why",
}}},
};
t;
}))
VERSIONED_CHARACTERIZATION_TEST(
ServeProtoTest,
buildResult_2_3,
"build-result-2.3",
(ServeProto::Version{
.major = 2,
.minor = 3,
}),
({
using namespace std::literals::chrono_literals;
std::tuple<BuildResult, BuildResult, BuildResult> t{
BuildResult{.inner{BuildResult::Failure{{
.status = BuildResult::Failure::OutputRejected,
.msg = HintFmt("no idea why"),
}}}},
BuildResult{
.inner{BuildResult::Failure{{
.inner{BuildResult::Failure{
.status = BuildResult::Failure::NotDeterministic,
.msg = HintFmt("no idea why"),
.errorMsg = "no idea why",
.isNonDeterministic = true,
}}},
.timesBuilt = 3,
.startTime = 30,
.stopTime = 50,
},
BuildResult{
.inner{BuildResult::Success{
.status = BuildResult::Success::Built,
}},
.startTime = 30,
.stopTime = 50,
},
};
t;
}))
VERSIONED_CHARACTERIZATION_TEST(
ServeProtoTest,
buildResult_2_6,
"build-result-2.6",
(ServeProto::Version{
.major = 2,
.minor = 6,
}),
({
using namespace std::literals::chrono_literals;
std::tuple<BuildResult, BuildResult, BuildResult> t{
BuildResult{.inner{BuildResult::Failure{{
.status = BuildResult::Failure::OutputRejected,
.msg = HintFmt("no idea why"),
}}}},
BuildResult{
.inner{BuildResult::Failure{{
.status = BuildResult::Failure::NotDeterministic,
.msg = HintFmt("no idea why"),
.isNonDeterministic = true,
}}},
.timesBuilt = 3,
.startTime = 30,
.stopTime = 50,
@@ -286,10 +260,7 @@ VERSIONED_CHARACTERIZATION_TEST(
ServeProtoTest,
unkeyedValidPathInfo_2_3,
"unkeyed-valid-path-info-2.3",
(ServeProto::Version{
.major = 2,
.minor = 3,
}),
2 << 8 | 3,
(std::tuple<UnkeyedValidPathInfo, UnkeyedValidPathInfo>{
({
UnkeyedValidPathInfo info{std::string{defaultStoreDir}, Hash::dummy};
@@ -315,10 +286,7 @@ VERSIONED_CHARACTERIZATION_TEST(
ServeProtoTest,
unkeyedValidPathInfo_2_4,
"unkeyed-valid-path-info-2.4",
(ServeProto::Version{
.major = 2,
.minor = 4,
}),
2 << 8 | 4,
(std::tuple<UnkeyedValidPathInfo, UnkeyedValidPathInfo>{
({
UnkeyedValidPathInfo info{
@@ -372,10 +340,7 @@ VERSIONED_CHARACTERIZATION_TEST_NO_JSON(
ServeProtoTest,
build_options_2_1,
"build-options-2.1",
(ServeProto::Version{
.major = 2,
.minor = 1,
}),
2 << 8 | 1,
(ServeProto::BuildOptions{
.maxSilentTime = 5,
.buildTimeout = 6,
@@ -385,10 +350,7 @@ VERSIONED_CHARACTERIZATION_TEST_NO_JSON(
ServeProtoTest,
build_options_2_2,
"build-options-2.2",
(ServeProto::Version{
.major = 2,
.minor = 2,
}),
2 << 8 | 2,
(ServeProto::BuildOptions{
.maxSilentTime = 5,
.buildTimeout = 6,
@@ -399,10 +361,7 @@ VERSIONED_CHARACTERIZATION_TEST_NO_JSON(
ServeProtoTest,
build_options_2_3,
"build-options-2.3",
(ServeProto::Version{
.major = 2,
.minor = 3,
}),
2 << 8 | 3,
(ServeProto::BuildOptions{
.maxSilentTime = 5,
.buildTimeout = 6,
@@ -415,10 +374,7 @@ VERSIONED_CHARACTERIZATION_TEST_NO_JSON(
ServeProtoTest,
build_options_2_7,
"build-options-2.7",
(ServeProto::Version{
.major = 2,
.minor = 7,
}),
2 << 8 | 7,
(ServeProto::BuildOptions{
.maxSilentTime = 5,
.buildTimeout = 6,
+99 -296
View File
@@ -15,80 +15,6 @@
namespace nix {
TEST(WorkerProtoVersionNumber, ordering)
{
using Number = WorkerProto::Version::Number;
EXPECT_LT((Number{1, 10}), (Number{1, 20}));
EXPECT_GT((Number{1, 30}), (Number{1, 20}));
EXPECT_EQ((Number{1, 10}), (Number{1, 10}));
EXPECT_LT((Number{0, 255}), (Number{1, 0}));
}
TEST(WorkerProtoVersion, partialOrderingSameFeatures)
{
using V = WorkerProto::Version;
V v1{.number = {1, 20}, .features = {"a", "b"}};
V v2{.number = {1, 30}, .features = {"a", "b"}};
EXPECT_TRUE(v1 < v2);
EXPECT_TRUE(v2 > v1);
EXPECT_TRUE(v1 <= v2);
EXPECT_TRUE(v2 >= v1);
EXPECT_FALSE(v1 == v2);
}
TEST(WorkerProtoVersion, partialOrderingSubsetFeatures)
{
using V = WorkerProto::Version;
V fewer{.number = {1, 30}, .features = {"a"}};
V more{.number = {1, 30}, .features = {"a", "b"}};
// fewer <= more: JUST the features are a subset
EXPECT_TRUE(fewer < more);
EXPECT_TRUE(fewer <= more);
EXPECT_FALSE(fewer > more);
EXPECT_TRUE(fewer != more);
}
TEST(WorkerProtoVersion, partialOrderingUnordered)
{
using V = WorkerProto::Version;
// Same number but incomparable features
V v1{.number = {1, 20}, .features = {"a", "c"}};
V v2{.number = {1, 20}, .features = {"a", "b"}};
EXPECT_FALSE(v1 < v2);
EXPECT_FALSE(v1 > v2);
EXPECT_FALSE(v1 <= v2);
EXPECT_FALSE(v1 >= v2);
EXPECT_FALSE(v1 == v2);
EXPECT_TRUE(v1 != v2);
}
TEST(WorkerProtoVersion, partialOrderingHigherNumberFewerFeatures)
{
using V = WorkerProto::Version;
// Higher number but fewer features — unordered
V v1{.number = {1, 30}, .features = {"a"}};
V v2{.number = {1, 20}, .features = {"a", "b"}};
EXPECT_FALSE(v1 < v2);
EXPECT_FALSE(v1 > v2);
EXPECT_FALSE(v1 == v2);
}
TEST(WorkerProtoVersion, partialOrderingEmptyFeatures)
{
using V = WorkerProto::Version;
V empty{.number = {1, 20}, .features = {}};
V some{.number = {1, 30}, .features = {"a"}};
// empty features is a subset of everything
EXPECT_TRUE(empty < some);
EXPECT_TRUE(empty <= some);
EXPECT_TRUE(empty != some);
}
const char workerProtoDir[] = "worker-protocol";
static constexpr std::string_view defaultStoreDir = "/nix/store";
@@ -99,13 +25,7 @@ struct WorkerProtoTest : VersionedProtoTest<WorkerProto, workerProtoDir>
* For serializers that don't care about the minimum version, we
* used the oldest one: 1.10.
*/
WorkerProto::Version defaultVersion = {
.number =
{
.major = 1,
.minor = 10,
},
};
WorkerProto::Version defaultVersion = 1 << 8 | 10;
};
VERSIONED_CHARACTERIZATION_TEST(
@@ -159,13 +79,7 @@ VERSIONED_CHARACTERIZATION_TEST(
WorkerProtoTest,
derivedPath_1_29,
"derived-path-1.29",
(WorkerProto::Version{
.number =
{
.major = 1,
.minor = 29,
},
}),
1 << 8 | 29,
(std::tuple<DerivedPath, DerivedPath, DerivedPath>{
DerivedPath::Opaque{
.path = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
@@ -190,13 +104,7 @@ VERSIONED_CHARACTERIZATION_TEST(
WorkerProtoTest,
derivedPath_1_30,
"derived-path-1.30",
(WorkerProto::Version{
.number =
{
.major = 1,
.minor = 30,
},
}),
1 << 8 | 30,
(std::tuple<DerivedPath, DerivedPath, DerivedPath, DerivedPath>{
DerivedPath::Opaque{
.path = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
@@ -289,57 +197,36 @@ VERSIONED_READ_CHARACTERIZATION_TEST(
},
}))
VERSIONED_CHARACTERIZATION_TEST(
WorkerProtoTest,
buildResult_1_27,
"build-result-1.27",
(WorkerProto::Version{
.number =
{
.major = 1,
.minor = 27,
},
}),
({
using namespace std::literals::chrono_literals;
std::tuple<BuildResult, BuildResult, BuildResult> t{
BuildResult{.inner{BuildResult::Failure{{
.status = BuildResult::Failure::OutputRejected,
.msg = HintFmt("no idea why"),
}}}},
BuildResult{.inner{BuildResult::Failure{{
.status = BuildResult::Failure::NotDeterministic,
.msg = HintFmt("no idea why"),
}}}},
BuildResult{.inner{BuildResult::Success{
.status = BuildResult::Success::Built,
}}},
};
t;
}))
VERSIONED_CHARACTERIZATION_TEST(WorkerProtoTest, buildResult_1_27, "build-result-1.27", 1 << 8 | 27, ({
using namespace std::literals::chrono_literals;
std::tuple<BuildResult, BuildResult, BuildResult> t{
BuildResult{.inner{BuildResult::Failure{
.status = BuildResult::Failure::OutputRejected,
.errorMsg = "no idea why",
}}},
BuildResult{.inner{BuildResult::Failure{
.status = BuildResult::Failure::NotDeterministic,
.errorMsg = "no idea why",
}}},
BuildResult{.inner{BuildResult::Success{
.status = BuildResult::Success::Built,
}}},
};
t;
}))
VERSIONED_CHARACTERIZATION_TEST(
WorkerProtoTest,
buildResult_1_28,
"build-result-1.28",
(WorkerProto::Version{
.number =
{
.major = 1,
.minor = 28,
},
}),
({
WorkerProtoTest, buildResult_1_28, "build-result-1.28", 1 << 8 | 28, ({
using namespace std::literals::chrono_literals;
std::tuple<BuildResult, BuildResult, BuildResult> t{
BuildResult{.inner{BuildResult::Failure{{
BuildResult{.inner{BuildResult::Failure{
.status = BuildResult::Failure::OutputRejected,
.msg = HintFmt("no idea why"),
}}}},
BuildResult{.inner{BuildResult::Failure{{
.errorMsg = "no idea why",
}}},
BuildResult{.inner{BuildResult::Failure{
.status = BuildResult::Failure::NotDeterministic,
.msg = HintFmt("no idea why"),
}}}},
.errorMsg = "no idea why",
}}},
BuildResult{.inner{BuildResult::Success{
.status = BuildResult::Success::Built,
.builtOutputs =
@@ -375,29 +262,19 @@ VERSIONED_CHARACTERIZATION_TEST(
}))
VERSIONED_CHARACTERIZATION_TEST(
WorkerProtoTest,
buildResult_1_29,
"build-result-1.29",
(WorkerProto::Version{
.number =
{
.major = 1,
.minor = 29,
},
}),
({
WorkerProtoTest, buildResult_1_29, "build-result-1.29", 1 << 8 | 29, ({
using namespace std::literals::chrono_literals;
std::tuple<BuildResult, BuildResult, BuildResult> t{
BuildResult{.inner{BuildResult::Failure{{
BuildResult{.inner{BuildResult::Failure{
.status = BuildResult::Failure::OutputRejected,
.msg = HintFmt("no idea why"),
}}}},
.errorMsg = "no idea why",
}}},
BuildResult{
.inner{BuildResult::Failure{{
.inner{BuildResult::Failure{
.status = BuildResult::Failure::NotDeterministic,
.msg = HintFmt("no idea why"),
.errorMsg = "no idea why",
.isNonDeterministic = true,
}}},
}},
.timesBuilt = 3,
.startTime = 30,
.stopTime = 50,
@@ -444,29 +321,19 @@ VERSIONED_CHARACTERIZATION_TEST(
}))
VERSIONED_CHARACTERIZATION_TEST(
WorkerProtoTest,
buildResult_1_37,
"build-result-1.37",
(WorkerProto::Version{
.number =
{
.major = 1,
.minor = 37,
},
}),
({
WorkerProtoTest, buildResult_1_37, "build-result-1.37", 1 << 8 | 37, ({
using namespace std::literals::chrono_literals;
std::tuple<BuildResult, BuildResult, BuildResult> t{
BuildResult{.inner{BuildResult::Failure{{
BuildResult{.inner{BuildResult::Failure{
.status = BuildResult::Failure::OutputRejected,
.msg = HintFmt("no idea why"),
}}}},
.errorMsg = "no idea why",
}}},
BuildResult{
.inner{BuildResult::Failure{{
.inner{BuildResult::Failure{
.status = BuildResult::Failure::NotDeterministic,
.msg = HintFmt("no idea why"),
.errorMsg = "no idea why",
.isNonDeterministic = true,
}}},
}},
.timesBuilt = 3,
.startTime = 30,
.stopTime = 50,
@@ -514,65 +381,48 @@ VERSIONED_CHARACTERIZATION_TEST(
t;
}))
VERSIONED_CHARACTERIZATION_TEST(
WorkerProtoTest,
keyedBuildResult_1_29,
"keyed-build-result-1.29",
(WorkerProto::Version{
.number =
{
.major = 1,
.minor = 29,
},
}),
({
using namespace std::literals::chrono_literals;
std::tuple<KeyedBuildResult, KeyedBuildResult /*, KeyedBuildResult*/> t{
KeyedBuildResult{
BuildResult{.inner{KeyedBuildResult::Failure{{
.status = KeyedBuildResult::Failure::OutputRejected,
.msg = HintFmt("no idea why"),
}}}},
/* .path = */
DerivedPath::Opaque{
StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-xxx"},
},
},
KeyedBuildResult{
BuildResult{
.inner{KeyedBuildResult::Failure{{
.status = KeyedBuildResult::Failure::NotDeterministic,
.msg = HintFmt("no idea why"),
.isNonDeterministic = true,
}}},
.timesBuilt = 3,
.startTime = 30,
.stopTime = 50,
},
/* .path = */
DerivedPath::Built{
.drvPath = makeConstantStorePathRef(
StorePath{
"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv",
}),
.outputs = OutputsSpec::Names{"out"},
},
},
};
t;
}))
VERSIONED_CHARACTERIZATION_TEST(WorkerProtoTest, keyedBuildResult_1_29, "keyed-build-result-1.29", 1 << 8 | 29, ({
using namespace std::literals::chrono_literals;
std::tuple<KeyedBuildResult, KeyedBuildResult /*, KeyedBuildResult*/> t{
KeyedBuildResult{
{.inner{BuildResult::Failure{
.status = KeyedBuildResult::Failure::OutputRejected,
.errorMsg = "no idea why",
}}},
/* .path = */
DerivedPath::Opaque{
StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-xxx"},
},
},
KeyedBuildResult{
{
.inner{BuildResult::Failure{
.status = KeyedBuildResult::Failure::NotDeterministic,
.errorMsg = "no idea why",
.isNonDeterministic = true,
}},
.timesBuilt = 3,
.startTime = 30,
.stopTime = 50,
},
/* .path = */
DerivedPath::Built{
.drvPath = makeConstantStorePathRef(
StorePath{
"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv",
}),
.outputs = OutputsSpec::Names{"out"},
},
},
};
t;
}))
VERSIONED_CHARACTERIZATION_TEST(
WorkerProtoTest,
unkeyedValidPathInfo_1_15,
"unkeyed-valid-path-info-1.15",
(WorkerProto::Version{
.number =
{
.major = 1,
.minor = 15,
},
}),
1 << 8 | 15,
(std::tuple<UnkeyedValidPathInfo, UnkeyedValidPathInfo>{
({
UnkeyedValidPathInfo info{
@@ -606,13 +456,7 @@ VERSIONED_CHARACTERIZATION_TEST(
WorkerProtoTest,
validPathInfo_1_15,
"valid-path-info-1.15",
(WorkerProto::Version{
.number =
{
.major = 1,
.minor = 15,
},
}),
1 << 8 | 15,
(std::tuple<ValidPathInfo, ValidPathInfo>{
({
ValidPathInfo info{
@@ -661,13 +505,7 @@ VERSIONED_CHARACTERIZATION_TEST(
WorkerProtoTest,
validPathInfo_1_16,
"valid-path-info-1.16",
(WorkerProto::Version{
.number =
{
.major = 1,
.minor = 16,
},
}),
1 << 8 | 16,
(std::tuple<ValidPathInfo, ValidPathInfo, ValidPathInfo>{
({
ValidPathInfo info{
@@ -822,13 +660,7 @@ VERSIONED_CHARACTERIZATION_TEST_NO_JSON(
WorkerProtoTest,
clientHandshakeInfo_1_30,
"client-handshake-info_1_30",
(WorkerProto::Version{
.number =
{
.major = 1,
.minor = 30,
},
}),
1 << 8 | 30,
(std::tuple<WorkerProto::ClientHandshakeInfo>{
{},
}))
@@ -837,13 +669,7 @@ VERSIONED_CHARACTERIZATION_TEST_NO_JSON(
WorkerProtoTest,
clientHandshakeInfo_1_33,
"client-handshake-info_1_33",
(WorkerProto::Version{
.number =
{
.major = 1,
.minor = 33,
},
}),
1 << 8 | 33,
(std::tuple<WorkerProto::ClientHandshakeInfo, WorkerProto::ClientHandshakeInfo>{
{
.daemonNixVersion = std::optional{"foo"},
@@ -857,13 +683,7 @@ VERSIONED_CHARACTERIZATION_TEST_NO_JSON(
WorkerProtoTest,
clientHandshakeInfo_1_35,
"client-handshake-info_1_35",
(WorkerProto::Version{
.number =
{
.major = 1,
.minor = 35,
},
}),
1 << 8 | 35,
(std::tuple<WorkerProto::ClientHandshakeInfo, WorkerProto::ClientHandshakeInfo>{
{
.daemonNixVersion = std::optional{"foo"},
@@ -890,13 +710,13 @@ TEST_F(WorkerProtoTest, handshake_log)
FdSink out{toServer.writeSide.get()};
FdSource in0{toClient.readSide.get()};
TeeSource in{in0, toClientLog};
clientResult = WorkerProto::BasicClientConnection::handshake(out, in, defaultVersion);
clientResult = std::get<0>(WorkerProto::BasicClientConnection::handshake(out, in, defaultVersion, {}));
});
{
FdSink out{toClient.writeSide.get()};
FdSource in{toServer.readSide.get()};
WorkerProto::BasicServerConnection::handshake(out, in, defaultVersion);
WorkerProto::BasicServerConnection::handshake(out, in, defaultVersion, {});
};
thread.join();
@@ -911,43 +731,23 @@ TEST_F(WorkerProtoTest, handshake_features)
toClient.create();
toServer.create();
WorkerProto::Version clientResult;
std::tuple<WorkerProto::Version, WorkerProto::FeatureSet> clientResult;
auto clientThread = std::thread([&]() {
FdSink out{toServer.writeSide.get()};
FdSource in{toClient.readSide.get()};
clientResult = WorkerProto::BasicClientConnection::handshake(
out,
in,
WorkerProto::Version{
.number = {.major = 1, .minor = 123},
.features = {"bar", "aap", "mies", "xyzzy"},
});
clientResult = WorkerProto::BasicClientConnection::handshake(out, in, 123, {"bar", "aap", "mies", "xyzzy"});
});
FdSink out{toClient.writeSide.get()};
FdSource in{toServer.readSide.get()};
auto daemonResult = WorkerProto::BasicServerConnection::handshake(
out,
in,
WorkerProto::Version{
.number = {.major = 1, .minor = 200},
.features = {"foo", "bar", "xyzzy"},
});
auto daemonResult = WorkerProto::BasicServerConnection::handshake(out, in, 456, {"foo", "bar", "xyzzy"});
clientThread.join();
EXPECT_EQ(clientResult, daemonResult);
EXPECT_EQ(
clientResult,
(WorkerProto::Version{
.number =
{
.major = 1,
.minor = 123,
},
.features = {"bar", "xyzzy"},
}));
EXPECT_EQ(std::get<0>(clientResult), 123u);
EXPECT_EQ(std::get<1>(clientResult), WorkerProto::FeatureSet({"bar", "xyzzy"}));
}
/// Has to be a `BufferedSink` for handshake.
@@ -962,7 +762,8 @@ TEST_F(WorkerProtoTest, handshake_client_replay)
NullBufferedSink nullSink;
StringSource in{toClientLog};
auto clientResult = WorkerProto::BasicClientConnection::handshake(nullSink, in, defaultVersion);
auto clientResult =
std::get<0>(WorkerProto::BasicClientConnection::handshake(nullSink, in, defaultVersion, {}));
EXPECT_EQ(clientResult, defaultVersion);
});
@@ -976,10 +777,11 @@ TEST_F(WorkerProtoTest, handshake_client_truncated_replay_throws)
auto substring = toClientLog.substr(0, len);
StringSource in{substring};
if (len < 8) {
EXPECT_THROW(WorkerProto::BasicClientConnection::handshake(nullSink, in, defaultVersion), EndOfFile);
EXPECT_THROW(
WorkerProto::BasicClientConnection::handshake(nullSink, in, defaultVersion, {}), EndOfFile);
} else {
// Not sure why cannot keep on checking for `EndOfFile`.
EXPECT_THROW(WorkerProto::BasicClientConnection::handshake(nullSink, in, defaultVersion), Error);
EXPECT_THROW(WorkerProto::BasicClientConnection::handshake(nullSink, in, defaultVersion, {}), Error);
}
}
});
@@ -999,13 +801,14 @@ TEST_F(WorkerProtoTest, handshake_client_corrupted_throws)
if (idx < 4 || idx == 9) {
// magic bytes don't match
EXPECT_THROW(WorkerProto::BasicClientConnection::handshake(nullSink, in, defaultVersion), Error);
EXPECT_THROW(WorkerProto::BasicClientConnection::handshake(nullSink, in, defaultVersion, {}), Error);
} else if (idx < 8 || idx >= 12) {
// Number out of bounds
EXPECT_THROW(
WorkerProto::BasicClientConnection::handshake(nullSink, in, defaultVersion), SerialisationError);
WorkerProto::BasicClientConnection::handshake(nullSink, in, defaultVersion, {}),
SerialisationError);
} else {
auto ver = WorkerProto::BasicClientConnection::handshake(nullSink, in, defaultVersion);
auto ver = std::get<0>(WorkerProto::BasicClientConnection::handshake(nullSink, in, defaultVersion, {}));
// `std::min` of this and the other version saves us
EXPECT_EQ(ver, defaultVersion);
}
+3 -3
View File
@@ -193,7 +193,7 @@ TEST_F(WorkerSubstitutionTest, floatingDerivationOutput)
};
// Write the derivation to the destination store
auto drvPath = dummyStore->writeDerivation(drv);
auto drvPath = writeDerivation(*dummyStore, drv);
// Snapshot the destination store before
checkpointJson("ca-drv/store-before", dummyStore);
@@ -297,7 +297,7 @@ TEST_F(WorkerSubstitutionTest, floatingDerivationOutputWithDepDrv)
};
// Write the dependency derivation to the destination store
auto depDrvPath = dummyStore->writeDerivation(depDrv);
auto depDrvPath = writeDerivation(*dummyStore, depDrv);
// Compute the hash modulo for the dependency derivation
auto depHashModulo = hashDerivationModulo(*dummyStore, depDrv, true);
@@ -348,7 +348,7 @@ TEST_F(WorkerSubstitutionTest, floatingDerivationOutputWithDepDrv)
rootDrv.inputDrvs = {.map = {{depDrvPath, {.value = {"out"}}}}};
// Write the root derivation to the destination store
auto rootDrvPath = dummyStore->writeDerivation(rootDrv);
auto rootDrvPath = writeDerivation(*dummyStore, rootDrv);
// Snapshot the destination store before
checkpointJson("issue-11928/store-before", dummyStore);
+3 -3
View File
@@ -44,12 +44,12 @@ TEST_F(WriteDerivationTest, addToStoreFromDumpCalledOnce)
{
auto drv = makeSimpleDrv();
auto path1 = store->writeDerivation(drv, NoRepair);
auto path1 = writeDerivation(*store, drv, NoRepair);
config->readOnly = true;
auto path2 = computeStorePath(*store, drv);
auto path2 = writeDerivation(*store, drv, NoRepair);
EXPECT_EQ(path1, path2);
EXPECT_THAT(
[&] { store->writeDerivation(drv, Repair); },
[&] { writeDerivation(*store, drv, Repair); },
::testing::ThrowsMessage<Error>(
testing::HasSubstrIgnoreANSIMatcher("operation 'writeDerivation' is not supported by store 'dummy://'")));
}
+27 -31
View File
@@ -28,7 +28,7 @@ BinaryCacheStore::BinaryCacheStore(Config & config)
: config{config}
{
if (config.secretKeyFile != "")
signers.push_back(std::make_unique<LocalSigner>(SecretKey{readFile(config.secretKeyFile.get())}));
signers.push_back(std::make_unique<LocalSigner>(SecretKey{readFile(config.secretKeyFile)}));
if (config.secretKeyFiles != "") {
std::stringstream ss(config.secretKeyFiles);
@@ -435,41 +435,37 @@ void BinaryCacheStore::narFromPath(const StorePath & storePath, Sink & sink)
void BinaryCacheStore::queryPathInfoUncached(
const StorePath & storePath, Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept
{
auto uri = config.getReference().render(/*FIXME withParams=*/false);
auto storePathS = printStorePath(storePath);
auto act = std::make_shared<Activity>(
*logger,
lvlTalkative,
actQueryPathInfo,
fmt("querying info about '%s' on '%s'", storePathS, uri),
Logger::Fields{storePathS, uri});
PushActivity pact(act->id);
auto narInfoFile = narInfoFileFor(storePath);
auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback));
try {
auto uri = config.getReference().render(/*FIXME withParams=*/false);
auto storePathS = printStorePath(storePath);
auto act = std::make_shared<Activity>(
*logger,
lvlTalkative,
actQueryPathInfo,
fmt("querying info about '%s' on '%s'", storePathS, uri),
Logger::Fields{storePathS, uri});
PushActivity pact(act->id);
getFile(narInfoFile, {[=, this](std::future<std::optional<std::string>> fut) {
try {
auto data = fut.get();
auto narInfoFile = narInfoFileFor(storePath);
if (!data)
return (*callbackPtr)({});
getFile(narInfoFile, {[=, this](std::future<std::optional<std::string>> fut) {
try {
auto data = fut.get();
stats.narInfoRead++;
if (!data)
return (*callbackPtr)({});
(*callbackPtr)(
(std::shared_ptr<ValidPathInfo>) std::make_shared<NarInfo>(*this, *data, narInfoFile));
stats.narInfoRead++;
(*callbackPtr)(
(std::shared_ptr<ValidPathInfo>) std::make_shared<NarInfo>(*this, *data, narInfoFile));
(void) act; // force Activity into this lambda to ensure it stays alive
} catch (...) {
callbackPtr->rethrow();
}
}});
} catch (...) {
callbackPtr->rethrow();
}
(void) act; // force Activity into this lambda to ensure it stays alive
} catch (...) {
callbackPtr->rethrow();
}
}});
}
StorePath BinaryCacheStore::addToStore(
@@ -566,7 +562,7 @@ ref<SourceAccessor> BinaryCacheStore::getFSAccessor(bool requireValidPath)
std::shared_ptr<SourceAccessor> BinaryCacheStore::getFSAccessor(const StorePath & storePath, bool requireValidPath)
{
return getRemoteFSAccessor(requireValidPath)->accessObject(storePath);
return static_cast<ref<SourceAccessor>>(getRemoteFSAccessor(requireValidPath)->accessObject(storePath));
}
void BinaryCacheStore::addSignatures(const StorePath & storePath, const std::set<Signature> & sigs)
+11 -70
View File
@@ -4,65 +4,15 @@
namespace nix {
void ExitStatusFlags::updateFromStatus(BuildResult::Failure::Status status)
{
// Allow selecting a subset of enum values
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wswitch-enum"
switch (status) {
case BuildResult::Failure::TimedOut:
timedOut = true;
break;
case BuildResult::Failure::HashMismatch:
hashMismatch = true;
break;
case BuildResult::Failure::NotDeterministic:
checkMismatch = true;
break;
case BuildResult::Failure::PermanentFailure:
// Also considered a permenant failure, it seems
case BuildResult::Failure::InputRejected:
permanentFailure = true;
break;
default:
break;
}
#pragma GCC diagnostic pop
}
unsigned int ExitStatusFlags::failingExitStatus() const
{
bool buildFailure = permanentFailure || timedOut || hashMismatch;
/* Any of the 4 booleans we track */
bool problemWithSpecialExitCode = checkMismatch || buildFailure;
unsigned int mask = 0;
if (problemWithSpecialExitCode) {
mask |= 0b1100000;
if (buildFailure) {
mask |= 0b0100; // 100
if (timedOut)
mask |= 0b0001; // 101
if (hashMismatch)
mask |= 0b0010; // 102
}
if (checkMismatch)
mask |= 0b1000; // 104
}
/* We still (per the function docs) only call this function in the
failure case, so the default should not be 0, but 1, indicating
"some other kind of error. */
return mask ? mask : 1;
}
bool BuildResult::operator==(const BuildResult &) const noexcept = default;
std::strong_ordering BuildResult::operator<=>(const BuildResult &) const noexcept = default;
bool BuildResult::Success::operator==(const BuildResult::Success &) const noexcept = default;
std::strong_ordering BuildResult::Success::operator<=>(const BuildResult::Success &) const noexcept = default;
bool BuildResult::Failure::operator==(const BuildResult::Failure &) const noexcept = default;
std::strong_ordering BuildResult::Failure::operator<=>(const BuildResult::Failure &) const noexcept = default;
static constexpr std::array<std::pair<BuildResult::Success::Status, std::string_view>, 4> successStatusStrings{{
#define ENUM_ENTRY(e) {BuildResult::Success::e, #e}
ENUM_ENTRY(Built),
@@ -125,18 +75,9 @@ static BuildResult::Failure::Status failureStatusFromString(std::string_view str
throw Error("unknown built result failure status '%s'", str);
}
bool BuildError::operator==(const BuildError & other) const noexcept
[[noreturn]] void BuildResult::Failure::rethrow() const
{
return status == other.status && isNonDeterministic == other.isNonDeterministic && message() == other.message();
}
std::strong_ordering BuildError::operator<=>(const BuildError & other) const noexcept
{
if (auto cmp = status <=> other.status; cmp != 0)
return cmp;
if (auto cmp = isNonDeterministic <=> other.isNonDeterministic; cmp != 0)
return cmp;
return message() <=> other.message();
throw BuildError(status, "%s", errorMsg);
}
} // namespace nix
@@ -172,7 +113,7 @@ void adl_serializer<BuildResult>::to_json(json & res, const BuildResult & br)
[&](const BuildResult::Failure & failure) {
res["success"] = false;
res["status"] = failureStatusToString(failure.status);
res["errorMsg"] = failure.message();
res["errorMsg"] = failure.errorMsg;
res["isNonDeterministic"] = failure.isNonDeterministic;
},
},
@@ -207,11 +148,11 @@ BuildResult adl_serializer<BuildResult>::from_json(const json & _json)
s.builtOutputs = valueAt(json, "builtOutputs");
br.inner = std::move(s);
} else {
br.inner = BuildResult::Failure{{
.status = failureStatusFromString(statusStr),
.msg = HintFmt(getString(valueAt(json, "errorMsg"))),
.isNonDeterministic = getBoolean(valueAt(json, "isNonDeterministic")),
}};
BuildResult::Failure f;
f.status = failureStatusFromString(statusStr);
f.errorMsg = getString(valueAt(json, "errorMsg"));
f.isNonDeterministic = getBoolean(valueAt(json, "isNonDeterministic"));
br.inner = std::move(f);
}
return br;
+89 -81
View File
@@ -5,7 +5,6 @@
# include "nix/store/build/derivation-builder.hh"
#endif
#include "nix/util/processes.hh"
#include "nix/util/environment-variables.hh"
#include "nix/util/config-global.hh"
#include "nix/store/build/worker.hh"
#include "nix/util/util.hh"
@@ -64,11 +63,7 @@ std::string showKnownOutputs(const StoreDirConfig & store, const Derivation & dr
}
static void runPostBuildHook(
const WorkerSettings & workerSettings,
const StoreDirConfig & store,
Logger & logger,
const StorePath & drvPath,
const StorePathSet & outputPaths);
const StoreDirConfig & store, Logger & logger, const StorePath & drvPath, const StorePathSet & outputPaths);
/* At least one of the output paths could not be
produced using a substitute. So we have to build instead. */
@@ -92,7 +87,7 @@ Goal::Co DerivationBuildingGoal::gaveUpOnSubstitution(bool storeDerivation)
for (auto & i : drv->inputSrcs) {
if (worker.store.isValidPath(i))
continue;
if (!worker.settings.useSubstitutes)
if (!settings.useSubstitutes)
throw Error(
"dependency '%s' of '%s' does not exist, and substitution is disabled",
worker.store.printStorePath(i),
@@ -124,7 +119,7 @@ Goal::Co DerivationBuildingGoal::gaveUpOnSubstitution(bool storeDerivation)
assert(drv->inputDrvs.map.empty());
/* Store the resolved derivation, as part of the record of
what we're actually building */
worker.store.writeDerivation(*drv);
writeDerivation(worker.store, *drv);
}
StorePathSet inputPaths;
@@ -326,26 +321,11 @@ Goal::Co DerivationBuildingGoal::tryToBuild(StorePathSet inputPaths)
}
}
bool canBuildLocally = [&] {
if (drv->platform != settings.thisSystem.get() && !settings.extraPlatforms.get().count(drv->platform)
&& !drv->isBuiltin())
return false;
if (worker.settings.maxBuildJobs.get() == 0 && !drv->isBuiltin())
return false;
for (auto & feature : drvOptions.getRequiredSystemFeatures(*drv))
if (!worker.store.config.systemFeatures.get().count(feature))
return false;
return true;
}();
/* Don't do a remote build if the derivation has the attribute
`preferLocalBuild' set. Also, check and repair modes are only
supported for local builds. */
bool buildLocally = (buildMode != bmNormal || (drvOptions.preferLocalBuild && canBuildLocally))
&& worker.settings.maxBuildJobs.get() != 0;
bool buildLocally = (buildMode != bmNormal || drvOptions.willBuildLocally(worker.store, *drv))
&& settings.maxBuildJobs.get() != 0;
if (buildLocally) {
useHook = false;
@@ -379,7 +359,7 @@ Goal::Co DerivationBuildingGoal::tryToBuild(StorePathSet inputPaths)
externalBuilder = settings.findExternalDerivationBuilderIfSupported(*drv);
if (!externalBuilder && !canBuildLocally) {
if (!externalBuilder && !drvOptions.canBuildLocally(worker.store, *drv)) {
auto msg =
fmt("Cannot build '%s'.\n"
"Reason: " ANSI_RED "required system or feature not available" ANSI_NORMAL
@@ -401,6 +381,7 @@ Goal::Co DerivationBuildingGoal::tryToBuild(StorePathSet inputPaths)
"/usr/sbin/softwareupdate --install-rosetta && launchctl stop org.nixos.nix-daemon"));
outputLocks.unlock();
worker.permanentFailure = true;
co_return doneFailure({BuildResult::Failure::InputRejected, std::move(msg)});
}
useHook = false;
@@ -415,22 +396,13 @@ Goal::Co DerivationBuildingGoal::tryToBuild(StorePathSet inputPaths)
if (useHook) {
co_return buildWithHook(
std::move(inputPaths), std::move(initialOutputs), std::move(drvOptions), std::move(outputLocks));
} else if (auto * localStoreP = dynamic_cast<LocalStore *>(&worker.store)) {
} else {
co_return buildLocally(
*localStoreP,
std::move(inputPaths),
std::move(initialOutputs),
std::move(drvOptions),
std::move(outputLocks),
externalBuilder);
} else {
throw Error(
R"(
Unable to build with a primary store that isn't a local store;
either pass a different '--store' or enable remote builds.
For more information check 'man nix.conf' and search for '/machines'.
)");
}
}
@@ -496,7 +468,7 @@ Goal::Co DerivationBuildingGoal::buildWithHook(
msg += fmt(" on '%s'", hook->machineName);
std::unique_ptr<BuildLog> buildLog = std::make_unique<BuildLog>(
worker.settings.logLines,
settings.logLines,
std::make_unique<Activity>(
*logger,
lvlInfo,
@@ -516,7 +488,7 @@ Goal::Co DerivationBuildingGoal::buildWithHook(
auto & data = output->data;
if (fd == hook->builderOut.readSide.get()) {
logSize += data.size();
if (worker.settings.maxLogSize && logSize > worker.settings.maxLogSize) {
if (settings.maxLogSize && logSize > settings.maxLogSize) {
hook.reset();
co_return doneFailureLogTooLong(*buildLog);
}
@@ -625,7 +597,7 @@ Goal::Co DerivationBuildingGoal::buildWithHook(
StorePathSet outputPaths;
for (auto & [_, output] : builtOutputs)
outputPaths.insert(output.outPath);
runPostBuildHook(worker.settings, worker.store, *logger, drvPath, outputPaths);
runPostBuildHook(worker.store, *logger, drvPath, outputPaths);
/* It is now safe to delete the lock files, since all future
lockers will see that the output paths are valid; they will
@@ -639,7 +611,6 @@ Goal::Co DerivationBuildingGoal::buildWithHook(
}
Goal::Co DerivationBuildingGoal::buildLocally(
LocalStore & localStore,
StorePathSet inputPaths,
std::map<std::string, InitialOutput> initialOutputs,
DerivationOptions<StorePath> drvOptions,
@@ -648,6 +619,16 @@ Goal::Co DerivationBuildingGoal::buildLocally(
{
co_await yield();
if (!dynamic_cast<LocalStore *>(&worker.store)) {
throw Error(
R"(
Unable to build with a primary store that isn't a local store;
either pass a different '--store' or enable remote builds.
For more information check 'man nix.conf' and search for '/machines'.
)");
}
#ifdef _WIN32 // TODO enable `DerivationBuilder` on Windows
throw UnimplementedError("building derivations is not yet implemented on Windows");
#else
@@ -667,7 +648,7 @@ Goal::Co DerivationBuildingGoal::buildLocally(
: "building '%s'",
worker.store.printStorePath(drvPath));
buildLog = std::make_unique<BuildLog>(
worker.settings.logLines,
settings.logLines,
std::make_unique<Activity>(
*logger, lvlInfo, actBuild, msg, Logger::Fields{worker.store.printStorePath(drvPath), "", 1, 1}));
mcRunningBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.runningBuilds);
@@ -682,7 +663,7 @@ Goal::Co DerivationBuildingGoal::buildLocally(
while (true) {
unsigned int curBuilds = worker.getNrLocalBuilds();
if (curBuilds >= worker.settings.maxBuildJobs) {
if (curBuilds >= settings.maxBuildJobs) {
outputLocks.unlock();
co_await waitForBuildSlot();
co_return tryToBuild(std::move(inputPaths));
@@ -727,8 +708,10 @@ Goal::Co DerivationBuildingGoal::buildLocally(
}
};
decltype(DerivationBuilderParams::defaultPathsInChroot) defaultPathsInChroot =
localStore.config->getLocalSettings().sandboxPaths.get();
auto * localStoreP = dynamic_cast<LocalStore *>(&worker.store);
assert(localStoreP);
decltype(DerivationBuilderParams::defaultPathsInChroot) defaultPathsInChroot = settings.sandboxPaths.get();
DesugaredEnv desugaredEnv;
/* Add the closure of store paths to the chroot. */
@@ -752,6 +735,7 @@ Goal::Co DerivationBuildingGoal::buildLocally(
desugaredEnv = DesugaredEnv::create(worker.store, *drv, drvOptions, inputPaths);
} catch (BuildError & e) {
outputLocks.unlock();
worker.permanentFailure = true;
co_return doneFailure(std::move(e));
}
@@ -772,12 +756,12 @@ Goal::Co DerivationBuildingGoal::buildLocally(
already be created, so we don't need to create it again. */
builder = externalBuilder
? makeExternalDerivationBuilder(
localStore,
*localStoreP,
std::make_unique<DerivationBuildingGoalCallbacks>(*this, openLogFile, closeLogFile),
std::move(params),
*externalBuilder)
: makeDerivationBuilder(
localStore,
*localStoreP,
std::make_unique<DerivationBuildingGoalCallbacks>(*this, openLogFile, closeLogFile),
std::move(params));
}
@@ -811,7 +795,7 @@ Goal::Co DerivationBuildingGoal::buildLocally(
if (auto * output = std::get_if<ChildOutput>(&event)) {
if (output->fd == builder->builderOut.get()) {
logSize += output->data.size();
if (worker.settings.maxLogSize && logSize > worker.settings.maxLogSize) {
if (settings.maxLogSize && logSize > settings.maxLogSize) {
builder->killChild();
co_return doneFailureLogTooLong(*buildLog);
}
@@ -840,6 +824,26 @@ Goal::Co DerivationBuildingGoal::buildLocally(
} catch (BuildError & e) {
builder.reset();
outputLocks.unlock();
// Allow selecting a subset of enum values
# pragma GCC diagnostic push
# pragma GCC diagnostic ignored "-Wswitch-enum"
switch (e.status) {
case BuildResult::Failure::HashMismatch:
worker.hashMismatch = true;
/* See header, the protocols don't know about `HashMismatch`
yet, so change it to `OutputRejected`, which they expect
for this case (hash mismatch is a type of output
rejection). */
e.status = BuildResult::Failure::OutputRejected;
break;
case BuildResult::Failure::NotDeterministic:
worker.checkMismatch = true;
break;
default:
/* Other statuses need no adjusting */
break;
}
# pragma GCC diagnostic pop
co_return doneFailure(std::move(e));
}
{
@@ -858,7 +862,7 @@ Goal::Co DerivationBuildingGoal::buildLocally(
worker.markContentsGood(output.outPath);
outputPaths.insert(output.outPath);
}
runPostBuildHook(worker.settings, worker.store, *logger, drvPath, outputPaths);
runPostBuildHook(worker.store, *logger, drvPath, outputPaths);
/* It is now safe to delete the lock files, since all future
lockers will see that the output paths are valid; they will
@@ -872,13 +876,9 @@ Goal::Co DerivationBuildingGoal::buildLocally(
}
static void runPostBuildHook(
const WorkerSettings & workerSettings,
const StoreDirConfig & store,
Logger & logger,
const StorePath & drvPath,
const StorePathSet & outputPaths)
const StoreDirConfig & store, Logger & logger, const StorePath & drvPath, const StorePathSet & outputPaths)
{
auto hook = workerSettings.postBuildHook;
auto hook = settings.postBuildHook;
if (hook == "")
return;
@@ -886,15 +886,14 @@ static void runPostBuildHook(
logger,
lvlTalkative,
actPostBuildHook,
fmt("running post-build-hook '%s'", workerSettings.postBuildHook),
fmt("running post-build-hook '%s'", settings.postBuildHook),
Logger::Fields{store.printStorePath(drvPath)});
PushActivity pact(act.id);
OsStringMap hookEnvironment = getEnvOs();
StringMap hookEnvironment = getEnv();
hookEnvironment.emplace(OS_STR("DRV_PATH"), string_to_os_string(store.printStorePath(drvPath)));
hookEnvironment.emplace(
OS_STR("OUT_PATHS"), string_to_os_string(chomp(concatStringsSep(" ", store.printStorePathSet(outputPaths)))));
hookEnvironment.emplace(OS_STR("NIX_CONFIG"), string_to_os_string(globalConfig.toKeyValue()));
hookEnvironment.emplace("DRV_PATH", store.printStorePath(drvPath));
hookEnvironment.emplace("OUT_PATHS", chomp(concatStringsSep(" ", store.printStorePathSet(outputPaths))));
hookEnvironment.emplace("NIX_CONFIG", globalConfig.toKeyValue());
struct LogSink : Sink
{
@@ -935,7 +934,7 @@ static void runPostBuildHook(
LogSink sink(act);
runProgram2({
.program = workerSettings.postBuildHook,
.program = settings.postBuildHook,
.environment = hookEnvironment,
.standardOut = &sink,
.mergeStderrToStdout = true,
@@ -982,18 +981,17 @@ HookReply DerivationBuildingGoal::tryBuildHook(const DerivationOptions<StorePath
#else
/* This should use `worker.evalStore`, but per #13179 the build hook
doesn't work with eval store anyways. */
if (worker.settings.buildHook.get().empty() || !worker.tryBuildHook || !worker.store.isValidPath(drvPath))
if (settings.buildHook.get().empty() || !worker.tryBuildHook || !worker.store.isValidPath(drvPath))
return rpDecline;
if (!worker.hook)
worker.hook = std::make_unique<HookInstance>(worker.settings.buildHook);
worker.hook = std::make_unique<HookInstance>();
try {
/* Send the request to the hook. */
worker.hook->sink << "try" << (worker.getNrLocalBuilds() < worker.settings.maxBuildJobs ? 1 : 0)
<< drv->platform << worker.store.printStorePath(drvPath)
<< drvOptions.getRequiredSystemFeatures(*drv);
worker.hook->sink << "try" << (worker.getNrLocalBuilds() < settings.maxBuildJobs ? 1 : 0) << drv->platform
<< worker.store.printStorePath(drvPath) << drvOptions.getRequiredSystemFeatures(*drv);
worker.hook->sink.flush();
/* Read the first line of input, which should be a word indicating
@@ -1052,25 +1050,26 @@ LogFile::LogFile(Store & store, const StorePath & drvPath, const LogFileSettings
auto baseName = std::string(baseNameOf(store.printStorePath(drvPath)));
std::filesystem::path logDir;
Path logDir;
if (auto localStore = dynamic_cast<LocalStore *>(&store))
logDir = localStore->config->logDir.get();
logDir = localStore->config->logDir;
else
logDir = logSettings.nixLogDir;
auto dir = logDir / LocalFSStore::drvsLogDir / baseName.substr(0, 2);
Path dir = fmt("%s/%s/%s/", logDir, LocalFSStore::drvsLogDir, baseName.substr(0, 2));
createDirs(dir);
auto logFileName = dir / (baseName.substr(2) + (logSettings.compressLog ? ".bz2" : ""));
Path logFileName = fmt("%s/%s%s", dir, baseName.substr(2), logSettings.compressLog ? ".bz2" : "");
fd = openNewFileForWrite(
logFileName,
0666,
{
.truncateExisting = true,
.followSymlinksOnTruncate = true, /* FIXME: Probably shouldn't follow symlinks. */
});
fd = toDescriptor(open(
logFileName.c_str(),
O_CREAT | O_WRONLY | O_TRUNC
#ifndef _WIN32
| O_CLOEXEC
#endif
,
0666));
if (!fd)
throw SysError("creating log file %1%", PathFmt(logFileName));
throw SysError("creating log file '%1%'", logFileName);
fileSink = std::make_shared<FdSink>(fd.get());
@@ -1099,7 +1098,7 @@ Goal::Done DerivationBuildingGoal::doneFailureLogTooLong(BuildLog & buildLog)
BuildResult::Failure::LogLimitExceeded,
"%s killed after writing more than %d bytes of log output",
getName(),
worker.settings.maxLogSize));
settings.maxLogSize));
}
std::map<std::string, std::optional<StorePath>> DerivationBuildingGoal::queryPartialDerivationOutputMap()
@@ -1205,13 +1204,22 @@ Goal::Done DerivationBuildingGoal::doneFailure(BuildError ex)
{
mcRunningBuilds.reset();
worker.exitStatusFlags.updateFromStatus(ex.status);
if (ex.status == BuildResult::Failure::TimedOut)
worker.timedOut = true;
if (ex.status == BuildResult::Failure::PermanentFailure)
worker.permanentFailure = true;
if (ex.status != BuildResult::Failure::DependencyFailed)
worker.failedBuilds++;
worker.updateProgress();
return Goal::doneFailure(ecFailed, std::move(ex));
return Goal::doneFailure(
ecFailed,
BuildResult::Failure{
.status = ex.status,
.errorMsg = fmt("%s", Uncolored(ex.info().msg)),
},
std::move(ex));
}
} // namespace nix
+1 -1
View File
@@ -14,7 +14,7 @@ void checkOutputs(
const decltype(DerivationOptions<StorePath>::outputChecks) & outputChecks,
const std::map<std::string, ValidPathInfo> & outputs)
{
std::map<std::filesystem::path, const ValidPathInfo &> outputsByPath;
std::map<Path, const ValidPathInfo &> outputsByPath;
for (auto & output : outputs)
outputsByPath.emplace(store.printStorePath(output.second.path), output.second);
+15 -6
View File
@@ -100,7 +100,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation)
/* We are first going to try to create the invalid output paths
through substitutes. If that doesn't work, we'll build
them. */
if (worker.settings.useSubstitutes && drvOptions.substitutesAllowed(worker.settings)) {
if (settings.useSubstitutes && drvOptions.substitutesAllowed()) {
if (!checkResult) {
DrvOutput id{outputHash, wantedOutput};
auto g = worker.makeDrvOutputSubstitutionGoal(id);
@@ -133,7 +133,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation)
assert(!drv->type().isImpure());
if (nrFailed > 0 && nrFailed > nrNoSubstituters && !worker.settings.tryFallback) {
if (nrFailed > 0 && nrFailed > nrNoSubstituters && !settings.tryFallback) {
co_return doneFailure(BuildError(
BuildResult::Failure::TransientFailure,
"some substitutes for the outputs of derivation '%s' failed (usually happens due to networking issues); try '--fallback' to build derivation from source ",
@@ -252,7 +252,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation)
auto g = worker.makeDerivationBuildingGoal(drvPath, *drv, buildMode, storeDerivation);
/* We will finish with it ourselves, as if we were the derivational goal. */
g->preserveFailure = true;
g->preserveException = true;
{
Goals waitees;
@@ -312,7 +312,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation)
}
}
co_return amDone(g->exitCode);
co_return amDone(g->exitCode, g->ex);
}
Goal::Co DerivationGoal::repairClosure()
@@ -492,13 +492,22 @@ Goal::Done DerivationGoal::doneFailure(BuildError ex)
{
mcExpectedBuilds.reset();
worker.exitStatusFlags.updateFromStatus(ex.status);
if (ex.status == BuildResult::Failure::TimedOut)
worker.timedOut = true;
if (ex.status == BuildResult::Failure::PermanentFailure)
worker.permanentFailure = true;
if (ex.status != BuildResult::Failure::DependencyFailed)
worker.failedBuilds++;
worker.updateProgress();
return Goal::doneFailure(ecFailed, std::move(ex));
return Goal::doneFailure(
ecFailed,
BuildResult::Failure{
.status = ex.status,
.errorMsg = fmt("%s", Uncolored(ex.info().msg)),
},
std::move(ex));
}
} // namespace nix
@@ -90,12 +90,7 @@ Goal::Co DerivationResolutionGoal::resolveDerivation()
nrFailed,
nrFailed == 1 ? "dependency" : "dependencies");
msg += showKnownOutputs(worker.store, *drv);
co_return doneFailure(
ecFailed,
BuildResult::Failure{{
.status = BuildResult::Failure::DependencyFailed,
.msg = HintFmt(msg),
}});
co_return amDone(ecFailed, {BuildError(BuildResult::Failure::DependencyFailed, msg)});
}
/* Gather information necessary for computing the closure and/or
@@ -107,7 +102,32 @@ Goal::Co DerivationResolutionGoal::resolveDerivation()
{
auto & fullDrv = *drv;
if (fullDrv.shouldResolve()) {
auto drvType = fullDrv.type();
bool resolveDrv =
std::visit(
overloaded{
[&](const DerivationType::InputAddressed & ia) {
/* must resolve if deferred. */
return ia.deferred;
},
[&](const DerivationType::ContentAddressed & ca) {
return !fullDrv.inputDrvs.map.empty()
&& (ca.fixed
/* Can optionally resolve if fixed, which is good
for avoiding unnecessary rebuilds. */
? experimentalFeatureSettings.isEnabled(Xp::CaDerivations)
/* Must resolve if floating and there are any inputs
drvs. */
: true);
},
[&](const DerivationType::Impure &) { return true; }},
drvType.raw)
/* no inputs are outputs of dynamic derivations */
|| std::ranges::any_of(fullDrv.inputDrvs.map.begin(), fullDrv.inputDrvs.map.end(), [](auto & pair) {
return !pair.second.childMap.empty();
});
if (resolveDrv && !fullDrv.inputDrvs.map.empty()) {
experimentalFeatureSettings.require(Xp::CaDerivations);
/* We are be able to resolve this derivation based on the
@@ -144,7 +164,7 @@ Goal::Co DerivationResolutionGoal::resolveDerivation()
}
assert(attempt);
auto pathResolved = computeStorePath(worker.store, Derivation{*attempt});
auto pathResolved = writeDerivation(worker.store, *attempt, NoRepair, /*readOnly =*/true);
auto msg =
fmt("resolved derivation: '%s' -> '%s'",
@@ -165,7 +185,7 @@ Goal::Co DerivationResolutionGoal::resolveDerivation()
}
}
co_return amDone(ecSuccess);
co_return amDone(ecSuccess, std::nullopt);
}
} // namespace nix
@@ -100,10 +100,11 @@ Goal::Co DerivationTrampolineGoal::init()
if (nrFailed != 0) {
co_return doneFailure(
ecFailed,
BuildResult::Failure{{
BuildResult::Failure{
.status = BuildResult::Failure::DependencyFailed,
.msg = HintFmt("failed to obtain derivation of '%s'", drvReq->to_string(worker.store)),
}});
.errorMsg = fmt("failed to obtain derivation of '%s'", drvReq->to_string(worker.store)),
},
Error("failed to obtain derivation of '%s'", drvReq->to_string(worker.store)));
}
StorePath drvPath = resolveDerivedPath(worker.store, *drvReq);
@@ -151,7 +152,7 @@ Goal::Co DerivationTrampolineGoal::haveDerivation(StorePath drvPath, Derivation
for (auto & output : resolvedWantedOutputs) {
auto g = upcast_goal(worker.makeDerivationGoal(drvPath, drv, output, buildMode, false));
g->preserveFailure = true;
g->preserveException = true;
/* We will finish with it ourselves, as if we were the derivational goal. */
concreteDrvGoals.insert(std::move(g));
}
@@ -169,7 +170,7 @@ Goal::Co DerivationTrampolineGoal::haveDerivation(StorePath drvPath, Derivation
for (auto && [x, y] : successP2->builtOutputs)
successP->builtOutputs.insert_or_assign(x, y);
co_return amDone(g->exitCode);
co_return amDone(g->exitCode, g->ex);
}
} // namespace nix
+22 -21
View File
@@ -18,13 +18,13 @@ void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMod
worker.run(goals);
StringSet failed;
BuildResult::Failure * failure = nullptr;
std::optional<Error> ex;
for (auto & i : goals) {
if (auto * f = i->buildResult.tryGetFailure()) {
if (failure)
logError(f->info());
if (i->ex) {
if (ex)
logError(i->ex->info());
else
failure = f;
ex = std::move(i->ex);
}
if (i->exitCode != Goal::ecSuccess) {
if (auto i2 = dynamic_cast<DerivationTrampolineGoal *>(i.get()))
@@ -34,14 +34,13 @@ void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMod
}
}
if (failed.size() == 1 && failure) {
failure->withExitStatus(worker.exitStatusFlags.failingExitStatus());
throw *failure;
if (failed.size() == 1 && ex) {
ex->withExitStatus(worker.failingExitStatus());
throw std::move(*ex);
} else if (!failed.empty()) {
auto exitStatus = worker.exitStatusFlags.failingExitStatus();
if (failure)
logError(failure->info());
throw Error(exitStatus, "build of %s failed", concatStringsSep(", ", quoteStrings(failed)));
if (ex)
logError(ex->info());
throw Error(worker.failingExitStatus(), "build of %s failed", concatStringsSep(", ", quoteStrings(failed)));
}
}
@@ -89,11 +88,10 @@ BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivat
worker.run(Goals{goal});
return goal->buildResult;
} catch (Error & e) {
return BuildResult{
.inner = BuildResult::Failure{{
.status = BuildResult::Failure::MiscFailure,
.msg = e.msg(),
}}};
return BuildResult{.inner{BuildResult::Failure{
.status = BuildResult::Failure::MiscFailure,
.errorMsg = e.msg(),
}}};
};
}
@@ -110,9 +108,12 @@ void Store::ensurePath(const StorePath & path)
worker.run(goals);
if (goal->exitCode != Goal::ecSuccess) {
auto exitStatus = worker.exitStatusFlags.failingExitStatus();
goal->buildResult.tryThrowBuildError(exitStatus);
throw Error(exitStatus, "path '%s' does not exist and cannot be created", printStorePath(path));
if (goal->ex) {
goal->ex->withExitStatus(worker.failingExitStatus());
throw std::move(*goal->ex);
} else
throw Error(
worker.failingExitStatus(), "path '%s' does not exist and cannot be created", printStorePath(path));
}
}
@@ -139,7 +140,7 @@ void Store::repairPath(const StorePath & path)
bmRepair));
worker.run(goals);
} else
throw Error(worker.exitStatusFlags.failingExitStatus(), "cannot repair path '%s'", printStorePath(path));
throw Error(worker.failingExitStatus(), "cannot repair path '%s'", printStorePath(path));
}
}
+9 -13
View File
@@ -189,14 +189,14 @@ Goal::Done Goal::doneSuccess(BuildResult::Success success)
return amDone(ecSuccess);
}
Goal::Done Goal::doneFailure(ExitCode result, BuildResult::Failure failure)
Goal::Done Goal::doneFailure(ExitCode result, BuildResult::Failure failure, std::optional<Error> ex)
{
assert(result == ecFailed || result == ecNoSubstituters);
buildResult.inner = std::move(failure);
return amDone(result);
return amDone(result, std::move(ex));
}
Goal::Done Goal::amDone(ExitCode result)
Goal::Done Goal::amDone(ExitCode result, std::optional<Error> ex)
{
trace("done");
assert(top_co);
@@ -204,15 +204,11 @@ Goal::Done Goal::amDone(ExitCode result)
assert(result == ecSuccess || result == ecFailed || result == ecNoSubstituters);
exitCode = result;
// Log the failure if we have one and shouldn't preserve it.
// Only log for actual failures (ecFailed), not for ecNoSubstituters
// which indicates "couldn't substitute, will try building" - that's
// expected behavior, not an error.
if (result == ecFailed) {
if (auto * failure = buildResult.tryGetFailure()) {
if (!preserveFailure && !waiters.empty())
logError(failure->info());
}
if (ex) {
if (!preserveException && !waiters.empty())
logError(ex->info());
else
this->ex = std::move(*ex);
}
for (auto & i : waiters) {
@@ -232,7 +228,7 @@ Goal::Done Goal::amDone(ExitCode result)
if (goal->waitees.empty()) {
worker.wakeUp(goal);
} else if (result == ecFailed && !worker.settings.keepGoing) {
} else if (result == ecFailed && !settings.keepGoing) {
/* If we failed and keepGoing is not set, we remove all
remaining waitees. */
for (auto & g : goal->waitees) {
+29 -16
View File
@@ -26,6 +26,25 @@ PathSubstitutionGoal::~PathSubstitutionGoal()
cleanup();
}
Goal::Done PathSubstitutionGoal::doneSuccess(BuildResult::Success::Status status)
{
return Goal::doneSuccess(
BuildResult::Success{
.status = status,
});
}
Goal::Done PathSubstitutionGoal::doneFailure(ExitCode result, BuildResult::Failure::Status status, std::string errorMsg)
{
debug(errorMsg);
return Goal::doneFailure(
result,
BuildResult::Failure{
.status = status,
.errorMsg = std::move(errorMsg),
});
}
Goal::Co PathSubstitutionGoal::init()
{
trace("init");
@@ -34,10 +53,10 @@ Goal::Co PathSubstitutionGoal::init()
/* If the path already exists we're done. */
if (!repair && worker.store.isValidPath(storePath)) {
co_return doneSuccess(BuildResult::Success{.status = BuildResult::Success::AlreadyValid});
co_return doneSuccess(BuildResult::Success::AlreadyValid);
}
if (worker.store.config.getReadOnly())
if (settings.readOnlyMode)
throw Error(
"cannot substitute path '%s' - no write access to the Nix store", worker.store.printStorePath(storePath));
@@ -145,7 +164,7 @@ Goal::Co PathSubstitutionGoal::init()
worker.updateProgress();
}
if (lastStoresException.has_value()) {
if (!worker.settings.tryFallback) {
if (!settings.tryFallback) {
throw *lastStoresException;
} else
logError(lastStoresException->info());
@@ -156,12 +175,9 @@ Goal::Co PathSubstitutionGoal::init()
build. */
co_return doneFailure(
substituterFailed ? ecFailed : ecNoSubstituters,
BuildResult::Failure{{
.status = BuildResult::Failure::NoSubstituters,
.msg = HintFmt(
"path '%s' is required, but there is no substituter that can build it",
worker.store.printStorePath(storePath)),
}});
BuildResult::Failure::NoSubstituters,
fmt("path '%s' is required, but there is no substituter that can build it",
worker.store.printStorePath(storePath)));
}
Goal::Co PathSubstitutionGoal::tryToRun(
@@ -172,11 +188,8 @@ Goal::Co PathSubstitutionGoal::tryToRun(
if (nrFailed > 0) {
co_return doneFailure(
nrNoSubstituters > 0 ? ecNoSubstituters : ecFailed,
BuildResult::Failure{{
.status = BuildResult::Failure::DependencyFailed,
.msg = HintFmt(
"some references of path '%s' could not be realised", worker.store.printStorePath(storePath)),
}});
BuildResult::Failure::DependencyFailed,
fmt("some references of path '%s' could not be realised", worker.store.printStorePath(storePath)));
}
for (auto & i : info->references)
@@ -197,7 +210,7 @@ Goal::Co PathSubstitutionGoal::tryToRun(
/* Make sure that we are allowed to start a substitution. Note that even
if maxSubstitutionJobs == 0, we still allow a substituter to run. This
prevents infinite waiting. */
while (worker.getNrSubstitutions() >= std::max(1U, (unsigned int) worker.settings.maxSubstitutionJobs)) {
while (worker.getNrSubstitutions() >= std::max(1U, (unsigned int) settings.maxSubstitutionJobs)) {
co_await waitForBuildSlot();
}
@@ -303,7 +316,7 @@ Goal::Co PathSubstitutionGoal::tryToRun(
worker.updateProgress();
co_return doneSuccess(BuildResult::Success{.status = BuildResult::Success::Substituted});
co_return doneSuccess(BuildResult::Success::Substituted);
}
void PathSubstitutionGoal::cleanup()
+27 -6
View File
@@ -22,14 +22,15 @@ Worker::Worker(Store & store, Store & evalStore)
, actSubstitutions(*logger, actCopyPaths)
, store(store)
, evalStore(evalStore)
, settings(nix::settings.getWorkerSettings())
, getSubstituters{[] {
return nix::settings.getWorkerSettings().useSubstitutes ? getDefaultSubstituters() : std::list<ref<Store>>{};
}}
, getSubstituters{[] { return settings.useSubstitutes ? getDefaultSubstituters() : std::list<ref<Store>>{}; }}
{
nrLocalBuilds = 0;
nrSubstitutions = 0;
lastWokenUp = steady_time_point::min();
permanentFailure = false;
timedOut = false;
hashMismatch = false;
checkMismatch = false;
}
Worker::~Worker()
@@ -367,7 +368,7 @@ void Worker::run(const Goals & _topGoals)
if (!children.empty() || !waitingForAWhile.empty())
waitForInput();
else if (awake.empty() && 0U == settings.maxBuildJobs) {
if (Machine::parseConfig({nix::settings.thisSystem}, nix::settings.getWorkerSettings().builders).empty())
if (getMachines().empty())
throw Error(
"Unable to start any build; either increase '--max-jobs' or enable remote builds.\n"
"\n"
@@ -409,7 +410,7 @@ void Worker::waitForInput()
auto nearest = steady_time_point::max(); // nearest deadline
auto localStore = dynamic_cast<LocalStore *>(&store);
if (localStore && localStore->config->getLocalSettings().getGCSettings().minFree.get() != 0)
if (localStore && localStore->config->getGCSettings().minFree.get() != 0)
// If we have a local store (and thus are capable of automatically collecting garbage) and configured to do so,
// periodically wake up to see if we need to run the garbage collector. (See the `autoGC` call site above in
// this file, also gated on having a local store. when we wake up, we intended to reach that call site.)
@@ -512,6 +513,26 @@ void Worker::waitForInput()
}
}
unsigned int Worker::failingExitStatus()
{
// See API docs in header for explanation
unsigned int mask = 0;
bool buildFailure = permanentFailure || timedOut || hashMismatch;
if (buildFailure)
mask |= 0x04; // 100
if (timedOut)
mask |= 0x01; // 101
if (hashMismatch)
mask |= 0x02; // 102
if (checkMismatch) {
mask |= 0x08; // 104
}
if (mask)
mask |= 0x60;
return mask ? mask : 1;
}
bool Worker::pathContentsGood(const StorePath & path)
{
auto i = pathContentsGoodCache.find(path);
+18 -13
View File
@@ -20,14 +20,14 @@ namespace {
struct State
{
std::map<std::filesystem::path, int> priorities;
std::map<Path, int> priorities;
unsigned long symlinks = 0;
};
} // namespace
/* For each activated package, create symlinks */
static void createLinks(State & state, const std::filesystem::path & srcDir, const std::filesystem::path & dstDir, int priority)
static void createLinks(State & state, const Path & srcDir, const Path & dstDir, int priority)
{
DirectoryIterator srcFiles;
@@ -35,7 +35,7 @@ static void createLinks(State & state, const std::filesystem::path & srcDir, con
srcFiles = DirectoryIterator{srcDir};
} catch (SystemError & e) {
if (e.is(std::errc::not_a_directory)) {
warn("not including %s in the user environment because it's not a directory", PathFmt(srcDir));
warn("not including '%s' in the user environment because it's not a directory", srcDir);
return;
}
throw;
@@ -50,12 +50,17 @@ static void createLinks(State & state, const std::filesystem::path & srcDir, con
auto srcFile = (std::filesystem::path{srcDir} / name).string();
auto dstFile = (std::filesystem::path{dstDir} / name).string();
auto srcStOpt = maybeStat(srcFile.c_str());
if (!srcStOpt) {
warn("skipping dangling symlink '%s'", dstFile);
continue;
struct stat srcSt;
try {
if (stat(srcFile.c_str(), &srcSt) == -1)
throw SysError("getting status of '%1%'", srcFile);
} catch (SystemError & e) {
if (e.is(std::errc::no_such_file_or_directory) || e.is(std::errc::not_a_directory)) {
warn("skipping dangling symlink '%s'", dstFile);
continue;
}
throw;
}
auto & srcSt = *srcStOpt;
/* The files below are special-cased to that they don't show
* up in user profiles, either because they are useless, or
@@ -78,9 +83,9 @@ static void createLinks(State & state, const std::filesystem::path & srcDir, con
} else if (S_ISLNK(dstSt.st_mode)) {
auto target = canonPath(dstFile, true);
if (!S_ISDIR(lstat(target).st_mode))
throw Error("collision between %1% and non-directory %2%", PathFmt(srcFile), PathFmt(target));
throw Error("collision between '%1%' and non-directory '%2%'", srcFile, target);
if (unlink(dstFile.c_str()) == -1)
throw SysError("unlinking %1%", PathFmt(dstFile));
throw SysError("unlinking '%1%'", dstFile);
if (mkdir(
dstFile.c_str()
#ifndef _WIN32 // TODO abstract mkdir perms for Windows
@@ -108,7 +113,7 @@ static void createLinks(State & state, const std::filesystem::path & srcDir, con
if (prevPriority < priority)
continue;
if (unlink(dstFile.c_str()) == -1)
throw SysError("unlinking %1%", PathFmt(dstFile));
throw SysError("unlinking '%1%'", dstFile);
} else if (S_ISDIR(dstSt.st_mode))
throw Error("collision between non-directory '%1%' and directory '%2%'", srcFile, dstFile);
}
@@ -120,13 +125,13 @@ static void createLinks(State & state, const std::filesystem::path & srcDir, con
}
}
void buildProfile(const std::filesystem::path & out, Packages && pkgs)
void buildProfile(const Path & out, Packages && pkgs)
{
State state;
PathSet done, postponed;
auto addPkg = [&](const std::filesystem::path & pkgDir, int priority) {
auto addPkg = [&](const Path & pkgDir, int priority) {
if (!done.insert(pkgDir).second)
return;
createLinks(state, pkgDir, out, priority);
Loaded 100 of 394 files, more files were not shown because too many files have changed in this diff. Show more