Compare commits

..
Author SHA1 Message Date
Jörg Thalheim 0bc58d5501 ignore-gc-delete-failure: add release note 2026-01-23 19:49:04 +01:00
411 changed files with 4810 additions and 8703 deletions

No files matched your search

+2 -2
View File
@@ -40,12 +40,12 @@ jobs:
role-session-name: nix-release-oidc-${{ github.run_id }}
aws-region: eu-west-1
- name: Login to Docker Hub
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Login to GitHub Container Registry
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
with:
registry: ghcr.io
username: ${{ github.actor }}
-2
View File
@@ -1,7 +1,5 @@
# Default meson build dir
/build
# Meson creates this file too
src/.wraplock
# /tests/functional/
/tests/functional/common/subst-vars.sh
-29
View File
@@ -1,29 +0,0 @@
---
synopsis: "Rust nix-installer in beta"
prs: []
---
The Rust-based rewrite of the Nix installer is now in beta.
We'd love help testing it out!
To test out the new installer, run:
```
curl -sSfL https://artifacts.nixos.org/nix-installer | sh -s -- install
```
This installer can be run even when you have an existing, script-based Nix installation without any adjustments.
This new installer also comes with the ability to uninstall your Nix installation; run:
```
/nix/nix-installer uninstall
```
This will get rid of your entire Nix installation (even if you installed over an existing, script-based installation).
This installer is a modified version of the [Determinate Nix Installer](https://github.com/DeterminateSystems/nix-installer) by Determinate Systems.
Thanks to Determinate Systems for all the investment they've put into the installer.
Source for the installer is in https://github.com/NixOS/nix-installer.
Report any issues in that repo.
For CI usage, a GitHub Action to install Nix using this installer is available at https://github.com/NixOS/nix-installer-action.
@@ -6,4 +6,3 @@ prs: [14766]
The C API now includes additional methods:
- `nix_store_query_path_from_hash_part()` - Get the full store path given its hash part
- `nix_store_copy_path()` - Copy a single store path between two stores, allows repairs and configuring signature checking
@@ -3,8 +3,8 @@ synopsis: "New setting `ignore-gc-delete-failure` for local stores"
prs: [15054]
---
A new local store setting [`ignore-gc-delete-failure`](@docroot@/store/types/local-store.md#store-local-store-ignore-gc-delete-failure) has been added.
A new local store setting `ignore-gc-delete-failure` has been added.
When enabled, garbage collection will log warnings instead of failing when it cannot delete store paths.
This is useful when running Nix as an unprivileged user that may not have write access to all paths in the store.
This setting is experimental and requires the [`local-overlay-store`](@docroot@/development/experimental-features.md#xp-feature-local-overlay-store) experimental feature.
This setting is experimental and requires `--extra-experimental-features local-overlay-store`.
-15
View File
@@ -1,15 +0,0 @@
---
synopsis: Support HTTPS binary caches using mTLS (client certificate) authentication
issues: [13002]
prs: [13030]
---
Added support for `tls-certificate` and `tls-private-key` options in substituter URLs.
Example:
```
https://substituter.invalid?tls-certificate=/path/to/cert.pem&tls-private-key=/path/to/key.pem
```
When these options are configured, Nix will use this certificate/private key pair to authenticate to the server.
@@ -1,32 +0,0 @@
---
synopsis: S3 binary caches now use virtual-hosted-style addressing by default
issues: [15208]
---
S3 binary caches now use virtual-hosted-style URLs
(`https://bucket.s3.region.amazonaws.com/key`) instead of path-style URLs
(`https://s3.region.amazonaws.com/bucket/key`) when connecting to standard AWS
S3 endpoints. This enables HTTP/2 multiplexing and fixes TCP connection
exhaustion (TIME_WAIT socket accumulation) under high-concurrency workloads.
A new `addressing-style` store option controls this behavior:
- `auto` (default): virtual-hosted-style for standard AWS endpoints, path-style
for custom endpoints.
- `path`: forces path-style addressing (deprecated by AWS).
- `virtual`: forces virtual-hosted-style addressing (bucket names must not
contain dots).
Bucket names containing dots (e.g., `my.bucket.name`) automatically fall back
to path-style addressing in `auto` mode, because dotted names create
multi-level subdomains that break TLS wildcard certificate validation.
Example using path-style for backwards compatibility:
```
s3://my-bucket/key?region=us-east-1&addressing-style=path
```
Additionally, TCP keep-alive is now enabled on all HTTP connections, preventing
idle connections from being silently dropped by intermediate network devices
(NATs, firewalls, load balancers).
Generated
+5 -5
View File
@@ -63,15 +63,15 @@
},
"nixpkgs": {
"locked": {
"lastModified": 1771043024,
"narHash": "sha256-WoiezqWJQ3OHILah+p6rzNXdJceEAmAhyDFZFZ6pZzY=",
"rev": "3aadb7ca9eac2891d52a9dec199d9580a6e2bf44",
"lastModified": 1763948260,
"narHash": "sha256-zZk7fn2ARAqmLwaYTpxBJmj81KIdz11NiWt7ydHHD/M=",
"rev": "1c8ba8d3f7634acac4a2094eef7c32ad9106532c",
"type": "tarball",
"url": "https://releases.nixos.org/nixos/25.11/nixos-25.11.5960.3aadb7ca9eac/nixexprs.tar.xz"
"url": "https://releases.nixos.org/nixos/25.05/nixos-25.05.813095.1c8ba8d3f763/nixexprs.tar.xz"
},
"original": {
"type": "tarball",
"url": "https://channels.nixos.org/nixos-25.11/nixexprs.tar.xz"
"url": "https://channels.nixos.org/nixos-25.05/nixexprs.tar.xz"
}
},
"nixpkgs-23-11": {
+1 -6
View File
@@ -1,7 +1,7 @@
{
description = "The purely functional package manager";
inputs.nixpkgs.url = "https://channels.nixos.org/nixos-25.11/nixexprs.tar.xz";
inputs.nixpkgs.url = "https://channels.nixos.org/nixos-25.05/nixexprs.tar.xz";
inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2";
inputs.nixpkgs-23-11.url = "github:NixOS/nixpkgs/a62e6edd6d5e1fa0329b8653c801147986f8d446";
@@ -115,9 +115,6 @@
}
// lib.optionalAttrs (crossSystem == "x86_64-unknown-freebsd13") {
useLLVM = true;
}
// lib.optionalAttrs (crossSystem == "x86_64-w64-mingw32") {
emulator = pkgs: "${pkgs.buildPackages.wineWow64Packages.stable_11}/bin/wine";
};
overlays = [
(overlayFor (pkgs: pkgs.${stdenv}))
@@ -409,8 +406,6 @@
"nix-cmd" = { };
"nix-nswrapper" = { };
"nix-cli" = { };
"nix-everything" = { };
+2 -9
View File
@@ -88,23 +88,16 @@
''^tests/functional/lang/eval-fail-path-slash\.nix$''
''^tests/functional/lang/eval-fail-toJSON-non-utf-8\.nix$''
''^tests/functional/lang/eval-fail-set\.nix$''
# Language tests, don't churn the formatting of strings
''^tests/functional/lang/eval-fail-fromTOML-overflow\.nix$''
''^tests/functional/lang/eval-fail-fromTOML-underflow\.nix$''
''^tests/functional/lang/eval-fail-bad-string-interpolation-3\.nix$''
''^tests/functional/lang/eval-fail-bad-string-interpolation-4\.nix$''
''^tests/functional/lang/eval-okay-regex-match2\.nix$''
];
};
clang-format = {
enable = true;
# https://github.com/cachix/git-hooks.nix/pull/532
package = pkgs.llvmPackages_21.clang-tools;
package = pkgs.llvmPackages_latest.clang-tools;
excludes = [
# We don't want to format test data
# ''tests/(?!nixos/).*\.nix''
"^src/[^/]*-tests/data/.*$"
''^src/[^/]*-tests/data/.*$''
# Don't format vendored code
''^doc/manual/redirects\.js$''
-4
View File
@@ -24,10 +24,6 @@ subproject('libcmd')
# Executables
subproject('nix')
if host_machine.system() == 'linux'
subproject('nswrapper')
endif
# Docs
if get_option('doc-gen')
subproject('internal-api-docs')
@@ -1,7 +1,7 @@
# Clang gets grumpy about missing libasan symbols if -shared-libasan is not
# passed when building shared libs, at least on Linux
if cxx.get_id() == 'clang' and ('address' in get_option('b_sanitize') or 'undefined' in get_option(
'b_sanitize',
'b_sanitize',
))
add_project_link_arguments('-shared-libasan', language : 'cpp')
endif
@@ -22,8 +22,6 @@ add_project_arguments(
'-Werror=undef',
'-Werror=unused-result',
'-Werror=sign-compare',
'-Werror=return-type',
'-Werror=non-virtual-dtor',
'-Wignored-qualifiers',
'-Wimplicit-fallthrough',
'-Wno-deprecated-declarations',
@@ -33,13 +31,6 @@ add_project_arguments(
# GCC doesn't benefit much from precompiled headers.
do_pch = cxx.get_id() == 'clang'
if cxx.get_id() == 'gcc'
add_project_arguments(
'-Wno-interference-size', # Used for C++ ABI only. We don't provide any guarantees about different march tunings.
language : 'cpp',
)
endif
# This is a clang-only option for improving build times.
# It forces the instantiation of templates in the PCH itself and
# not every translation unit it's included in.
@@ -49,11 +40,6 @@ endif
# instantiations in libutil and libstore.
if cxx.get_id() == 'clang'
add_project_arguments('-fpch-instantiate-templates', language : 'cpp')
# Catch brace elision bugs: when WorkerProto::Version changed from `unsigned int`
# to `struct { unsigned int major; uint8_t minor; }`, `.version = 16` silently
# became `.version = {16, 0}` instead of failing, breaking protocol compatibility
# in a subtle way
add_project_arguments('-Werror=c99-designator', language : 'cpp')
endif
# Detect if we're using libstdc++ (GCC's standard library)
-3
View File
@@ -4,7 +4,6 @@
buildPackages,
cacert,
nix,
nixComponents2,
}:
let
@@ -12,7 +11,6 @@ let
installerClosureInfo = buildPackages.closureInfo {
rootPaths = [
nix
nixComponents2.nix-manual.man
cacert
];
};
@@ -44,7 +42,6 @@ runCommand "nix-binary-tarball-${version}" env ''
--subst-var-by cacert ${cacert}
substitute ${../scripts/install-multi-user.sh} $TMPDIR/install-multi-user \
--subst-var-by nix ${nix} \
--subst-var-by nix-manual ${nixComponents2.nix-manual.man} \
--subst-var-by cacert ${cacert}
if type -p shellcheck; then
+2 -6
View File
@@ -155,14 +155,12 @@ let
];
};
mesonBuildLayer = finalAttrs: prevAttrs: rec {
mesonBuildLayer = finalAttrs: prevAttrs: {
nativeBuildInputs = prevAttrs.nativeBuildInputs or [ ] ++ [
pkg-config
];
separateDebugInfo = !stdenv.hostPlatform.isStatic;
# needed by separateDebugInfo
# SEE: https://github.com/NixOS/nixpkgs/pull/394674/commits/a4d355342976e9e9823fb94f133bc43ebec9da5b
__structuredAttrs = separateDebugInfo;
hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie";
};
mesonLibraryLayer = finalAttrs: prevAttrs: {
@@ -418,8 +416,6 @@ in
nix-cmd = callPackage ../src/libcmd/package.nix { };
nix-nswrapper = callPackage ../src/nswrapper/package.nix { };
/**
The Nix command line interface. Note that this does not include its tests, whereas `nix-everything` does.
*/
+26 -6
View File
@@ -30,13 +30,33 @@ scope: {
NIX_CFLAGS_COMPILE = "-DINITIAL_MARK_STACK_SIZE=1048576";
});
curl = pkgs.curl.override {
http3Support = !pkgs.stdenv.hostPlatform.isWindows;
};
lowdown = pkgs.lowdown.overrideAttrs (prevAttrs: rec {
version = "2.0.2";
src = pkgs.fetchurl {
url = "https://kristaps.bsd.lv/lowdown/snapshots/lowdown-${version}.tar.gz";
hash = "sha512-cfzhuF4EnGmLJf5EGSIbWqJItY3npbRSALm+GarZ7SMU7Hr1xw0gtBFMpOdi5PBar4TgtvbnG4oRPh+COINGlA==";
};
nativeBuildInputs = prevAttrs.nativeBuildInputs ++ [ pkgs.buildPackages.bmake ];
postInstall =
lib.replaceStrings [ "lowdown.so.1" "lowdown.1.dylib" ] [ "lowdown.so.2" "lowdown.2.dylib" ]
(prevAttrs.postInstall or "");
patches = [ ];
});
libblake3 = pkgs.libblake3.override {
useTBB = !(stdenv.hostPlatform.isWindows || stdenv.hostPlatform.isStatic);
};
# TODO: Remove this when https://github.com/NixOS/nixpkgs/pull/442682 is included in a stable release
toml11 =
if lib.versionAtLeast pkgs.toml11.version "4.4.0" then
pkgs.toml11
else
pkgs.toml11.overrideAttrs rec {
version = "4.4.0";
src = pkgs.fetchFromGitHub {
owner = "ToruNiina";
repo = "toml11";
tag = "v${version}";
hash = "sha256-sgWKYxNT22nw376ttGsTdg0AMzOwp8QH3E8mx0BZJTQ=";
};
};
# TODO Hack until https://github.com/NixOS/nixpkgs/issues/45462 is fixed.
boost =
+3 -3
View File
@@ -131,7 +131,7 @@ pkgs.nixComponents2.nix-util.overrideAttrs (
ignoreCrossFile = flags: builtins.filter (flag: !(lib.strings.hasInfix "cross-file" flag)) flags;
availableComponents = lib.filterAttrs (
k: v: lib.meta.availableOn pkgs.stdenv.hostPlatform v
k: v: lib.meta.availableOn pkgs.hostPlatform v
) allComponents;
activeComponents = buildInputsClosureCond isInternal (
@@ -323,7 +323,7 @@ pkgs.nixComponents2.nix-util.overrideAttrs (
pkgs.buildPackages.shellcheck
pkgs.buildPackages.include-what-you-use
]
++ lib.optional stdenv.hostPlatform.isUnix pkgs.buildPackages.gdb
++ lib.optional pkgs.hostPlatform.isUnix pkgs.buildPackages.gdb
++ lib.optional (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform) (
lib.hiPrio pkgs.buildPackages.clang-tools
)
@@ -341,7 +341,7 @@ pkgs.nixComponents2.nix-util.overrideAttrs (
buildInputs =
# TODO change Nixpkgs to mark gbenchmark as building on Windows
lib.optional stdenv.hostPlatform.isUnix pkgs.gbenchmark
lib.optional pkgs.hostPlatform.isUnix pkgs.gbenchmark
++ dedupByString (v: "${v}") (
lib.filter (x: !isInternal x) (lib.lists.concatMap (c: c.buildInputs) activeComponents)
)
-5
View File
@@ -31,8 +31,6 @@
nix-cmd,
nix-nswrapper,
nix-cli,
nix-functional-tests,
@@ -173,9 +171,6 @@ stdenv.mkDerivation (finalAttrs: {
# Forwarded outputs
ln -sT ${nix-manual} $doc
ln -sT ${nix-manual.man} $man
''
+ lib.optionalString stdenv.isLinux ''
lndir ${nix-nswrapper} $out
'';
passthru = {
-1
View File
@@ -57,7 +57,6 @@ let
"nix-flake"
"nix-flake-c"
"nix-flake-tests"
"nix-nswrapper"
"nix-main"
"nix-main-c"
"nix-cmd"
-3
View File
@@ -52,7 +52,6 @@ readonly PROFILE_FISH_PREFIXES=(
readonly PROFILE_NIX_FILE_FISH="$NIX_ROOT/var/nix/profiles/default/etc/profile.d/nix-daemon.fish"
readonly NIX_INSTALLED_NIX="@nix@"
readonly NIX_INSTALLED_NIX_MAN="@nix-manual@"
readonly NIX_INSTALLED_CACERT="@cacert@"
#readonly NIX_INSTALLED_NIX="/nix/store/byi37zv50wnfrpp4d81z3spswd5zva37-nix-2.3.6"
#readonly NIX_INSTALLED_CACERT="/nix/store/7pi45g541xa8ahwgpbpy7ggsl0xj1jj6-nss-cacert-3.49.2"
@@ -970,8 +969,6 @@ setup_default_profile() {
task "Setting up the default profile"
_sudo "to install a bootstrapping Nix in to the default profile" \
HOME="$ROOT_HOME" "$NIX_INSTALLED_NIX/bin/nix-env" -i "$NIX_INSTALLED_NIX"
_sudo "to install Nix man pages in to the default profile" \
HOME="$ROOT_HOME" "$NIX_INSTALLED_NIX/bin/nix-env" -i "$NIX_INSTALLED_NIX_MAN"
if [ -z "${NIX_SSL_CERT_FILE:-}" ] || ! [ -f "${NIX_SSL_CERT_FILE:-}" ] || cert_in_store; then
_sudo "to install a bootstrapping SSL certificate just for Nix in to the default profile" \
-1
View File
@@ -38,7 +38,6 @@ escape_systemd_env() {
create_systemd_proxy_env() {
vars="http_proxy https_proxy ftp_proxy all_proxy no_proxy HTTP_PROXY HTTPS_PROXY FTP_PROXY ALL_PROXY NO_PROXY"
for v in $vars; do
# shellcheck disable=SC2268
if [ "x${!v:-}" != "x" ]; then
echo "Environment=${v}=$(escape_systemd_env "${!v}")"
fi
+15 -43
View File
@@ -4,7 +4,6 @@
#include "nix/cmd/command.hh"
#include "nix/cmd/legacy.hh"
#include "nix/cmd/markdown.hh"
#include "nix/store/globals.hh"
#include "nix/store/store-open.hh"
#include "nix/store/local-fs-store.hh"
#include "nix/store/derivations.hh"
@@ -64,25 +63,6 @@ void NixMultiCommand::run()
command->second->run();
}
StoreConfigCommand::StoreConfigCommand() {}
ref<StoreConfig> StoreConfigCommand::getStoreConfig()
{
if (!_storeConfig)
_storeConfig = createStoreConfig();
return ref<StoreConfig>(_storeConfig);
}
ref<StoreConfig> StoreConfigCommand::createStoreConfig()
{
return resolveStoreConfig(StoreReference{settings.storeUri.get()});
}
void StoreConfigCommand::run()
{
run(getStoreConfig());
}
StoreCommand::StoreCommand() {}
ref<Store> StoreCommand::getStore()
@@ -94,20 +74,12 @@ ref<Store> StoreCommand::getStore()
ref<Store> StoreCommand::createStore()
{
auto store = getStoreConfig()->openStore();
store->init();
return store;
return openStore();
}
void StoreCommand::run(ref<StoreConfig> storeConfig)
void StoreCommand::run()
{
// We can either efficiently implement getStore/createStore with memoization,
// or use the StoreConfig passed in run.
// It's more efficient to memoize, especially since there are some direct users
// of getStore. The StoreConfig in both cases should be the same, though.
auto store = getStore();
assert(&*storeConfig == &store->config);
run(std::move(store));
run(getStore());
}
CopyCommand::CopyCommand()
@@ -116,28 +88,28 @@ CopyCommand::CopyCommand()
.longName = "from",
.description = "URL of the source Nix store.",
.labels = {"store-uri"},
.handler = {[this](std::string s) { srcUri = StoreReference::parse(s); }},
.handler = {&srcUri},
});
addFlag({
.longName = "to",
.description = "URL of the destination Nix store.",
.labels = {"store-uri"},
.handler = {[this](std::string s) { dstUri = StoreReference::parse(s); }},
.handler = {&dstUri},
});
}
ref<StoreConfig> CopyCommand::createStoreConfig()
ref<Store> CopyCommand::createStore()
{
return !srcUri ? StoreCommand::createStoreConfig() : resolveStoreConfig(StoreReference{*srcUri});
return srcUri.empty() ? StoreCommand::createStore() : openStore(srcUri);
}
ref<Store> CopyCommand::getDstStore()
{
if (!srcUri && !dstUri)
if (srcUri.empty() && dstUri.empty())
throw UsageError("you must pass '--from' and/or '--to'");
return !dstUri ? openStore() : openStore(StoreReference{*dstUri});
return dstUri.empty() ? openStore() : openStore(dstUri);
}
EvalCommand::EvalCommand()
@@ -159,7 +131,7 @@ EvalCommand::~EvalCommand()
ref<Store> EvalCommand::getEvalStore()
{
if (!evalStore)
evalStore = evalStoreUrl ? openStore(StoreReference{*evalStoreUrl}) : getStore();
evalStore = evalStoreUrl ? openStore(*evalStoreUrl) : getStore();
return ref<Store>(evalStore);
}
@@ -285,18 +257,18 @@ MixProfile::MixProfile()
});
}
void MixProfile::updateProfile(Store & store_, const StorePath & storePath)
void MixProfile::updateProfile(const StorePath & storePath)
{
if (!profile)
return;
auto * store = dynamic_cast<LocalFSStore *>(&store_);
auto store = getDstStore().dynamic_pointer_cast<LocalFSStore>();
if (!store)
throw Error("'--profile' is not supported for this Nix store");
auto profile2 = absPath(*profile);
switchLink(profile2, createGeneration(*store, profile2, storePath));
}
void MixProfile::updateProfile(Store & store, const BuiltPaths & buildables)
void MixProfile::updateProfile(const BuiltPaths & buildables)
{
if (!profile)
return;
@@ -320,12 +292,12 @@ void MixProfile::updateProfile(Store & store, const BuiltPaths & buildables)
throw UsageError(
"'--profile' requires that the arguments produce a single store path, but there are %d", result.size());
updateProfile(store, result[0]);
updateProfile(result[0]);
}
MixDefaultProfile::MixDefaultProfile()
{
profile = getDefaultProfile(settings.getProfileDirsOptions()).string();
profile = getDefaultProfile().string();
}
static constexpr auto environmentVariablesCategory = "Options that change environment variables";
+1 -1
View File
@@ -148,7 +148,7 @@ MixEvalArgs::MixEvalArgs()
)",
.category = category,
.labels = {"store-url"},
.handler = {[this](std::string s) { evalStoreUrl = StoreReference::parse(s); }},
.handler = {&evalStoreUrl},
});
}
-31
View File
@@ -1,31 +0,0 @@
#include "nix/cmd/get-build-log.hh"
#include "nix/store/log-store.hh"
#include "nix/store/store-open.hh"
namespace nix {
std::string fetchBuildLog(ref<Store> store, const StorePath & path, std::string_view what)
{
auto subs = getDefaultSubstituters();
subs.push_front(store);
for (auto & sub : subs) {
auto * logSubP = dynamic_cast<LogStore *>(&*sub);
if (!logSubP) {
printInfo("Skipped '%s' which does not support retrieving build logs", sub->config.getHumanReadableURI());
continue;
}
auto & logSub = *logSubP;
auto log = logSub.getBuildLog(path);
if (!log)
continue;
printInfo("got build log for '%s' from '%s'", what, logSub.config.getHumanReadableURI());
return *log;
}
throw Error("build log of '%s' is not available", what);
}
} // namespace nix
+18 -34
View File
@@ -5,7 +5,6 @@
#include "nix/util/args.hh"
#include "nix/cmd/common-eval-args.hh"
#include "nix/store/path.hh"
#include "nix/store/store-reference.hh"
#include "nix/flake/lockfile.hh"
#include <optional>
@@ -41,43 +40,28 @@ struct NixMultiCommand : MultiCommand, virtual Command
// For the overloaded run methods
#pragma GCC diagnostic ignored "-Woverloaded-virtual"
/**
* A command that requires a \ref StoreConfig store configuration.
*/
struct StoreConfigCommand : virtual Command
{
StoreConfigCommand();
void run() override;
/**
* Return the default Nix store configuration.
*/
ref<StoreConfig> getStoreConfig();
virtual ref<StoreConfig> createStoreConfig();
/**
* Main entry point, with a `StoreConfig` provided
*/
virtual void run(ref<StoreConfig>) = 0;
private:
std::shared_ptr<StoreConfig> _storeConfig;
};
/**
* A command that requires a \ref Store "Nix store".
*/
struct StoreCommand : virtual StoreConfigCommand
struct StoreCommand : virtual Command
{
StoreCommand();
void run(ref<StoreConfig>) override;
void run() override;
/**
* Return the default Nix store.
*/
ref<Store> getStore();
ref<Store> createStore();
/**
* Return the destination Nix store.
*/
virtual ref<Store> getDstStore()
{
return getStore();
}
virtual ref<Store> createStore();
/**
* Main entry point, with a `Store` provided
*/
@@ -93,13 +77,13 @@ private:
*/
struct CopyCommand : virtual StoreCommand
{
std::optional<StoreReference> srcUri, dstUri;
std::string srcUri, dstUri;
CopyCommand();
ref<StoreConfig> createStoreConfig() override;
ref<Store> createStore() override;
ref<Store> getDstStore();
ref<Store> getDstStore() override;
};
/**
@@ -331,11 +315,11 @@ struct MixProfile : virtual StoreCommand
MixProfile();
/* If 'profile' is set, make it point at 'storePath'. */
void updateProfile(Store & store, const StorePath & storePath);
void updateProfile(const StorePath & storePath);
/* If 'profile' is set, make it point at the store path produced
by 'buildables'. */
void updateProfile(Store & store, const BuiltPaths & buildables);
void updateProfile(const BuiltPaths & buildables);
};
struct MixDefaultProfile : MixProfile
@@ -407,7 +391,7 @@ void createOutLinks(const std::filesystem::path & outLink, const BuiltPaths & bu
struct MixOutLinkBase : virtual Args
{
/** Prefix for any output symlinks. Empty means do not write an output symlink. */
std::filesystem::path outLink;
Path outLink;
MixOutLinkBase(const std::string & defaultOutLink)
: outLink(defaultOutLink)
@@ -435,7 +419,7 @@ struct MixOutLinkByDefault : MixOutLinkBase, virtual Args
addFlag({
.longName = "no-link",
.description = "Do not create symlinks to the build results.",
.handler = {&outLink, std::filesystem::path{}},
.handler = {&outLink, Path("")},
});
}
};
@@ -6,7 +6,6 @@
#include "nix/main/common-args.hh"
#include "nix/expr/search-path.hh"
#include "nix/expr/eval-settings.hh"
#include "nix/store/store-reference.hh"
#include <filesystem>
@@ -56,7 +55,7 @@ struct MixEvalArgs : virtual Args, virtual MixRepair
LookupPath lookupPath;
std::optional<StoreReference> evalStoreUrl;
std::optional<std::string> evalStoreUrl;
private:
struct AutoArgExpr
@@ -1,23 +0,0 @@
#pragma once
///@file
#include "nix/store/store-api.hh"
#include <string>
#include <string_view>
namespace nix {
/**
* Fetch the build log for a store path, searching the store and its
* substituters.
*
* @param store The store to search (and its substituters).
* @param path The store path to get the build log for.
* @param what A description of what we're fetching the log for (used in messages).
* @return The build log content.
* @throws Error if the build log is not available.
*/
std::string fetchBuildLog(ref<Store> store, const StorePath & path, std::string_view what);
} // namespace nix
-2
View File
@@ -9,7 +9,6 @@ headers = files(
'common-eval-args.hh',
'compatibility-settings.hh',
'editor-for.hh',
'get-build-log.hh',
'installable-attr-path.hh',
'installable-derived-path.hh',
'installable-flake.hh',
@@ -21,5 +20,4 @@ headers = files(
'network-proxy.hh',
'repl-interacter.hh',
'repl.hh',
'unix-socket-server.hh',
)
@@ -14,7 +14,6 @@ namespace detail {
struct ReplCompleterMixin
{
virtual StringSet completePrefix(const std::string & prefix) = 0;
virtual ~ReplCompleterMixin() = default;
};
}; // namespace detail
+2 -1
View File
@@ -25,7 +25,7 @@ struct AbstractNixRepl
* @todo this is a layer violation
*
* @param programName Name of the command, e.g. `nix` or `nix-env`.
* @param args arguments to the command.
* @param args aguments to the command.
*/
using RunNix =
void(const std::string & programName, const Strings & args, const std::optional<std::string> & input);
@@ -37,6 +37,7 @@ struct AbstractNixRepl
*/
static std::unique_ptr<AbstractNixRepl> create(
const LookupPath & lookupPath,
nix::ref<Store> store,
ref<EvalState> state,
std::function<AnnotatedValues()> getValues,
RunNix * runNix = nullptr);
@@ -1,79 +0,0 @@
#pragma once
///@file
#include "nix/util/file-descriptor.hh"
#include <filesystem>
#include <functional>
#include <optional>
#include <sys/types.h>
namespace nix::unix {
/**
* Information about the identity of the peer on a Unix domain socket connection.
*/
struct PeerInfo
{
std::optional<pid_t> pid;
std::optional<uid_t> uid;
std::optional<gid_t> gid;
};
/**
* Get the identity of the caller, if possible.
*/
PeerInfo getPeerInfo(Descriptor remote);
/**
* Callback type for handling new connections.
*
* The callback receives ownership of the connection and is responsible
* for handling it (e.g., forking a child process, spawning a thread, etc.).
*
* @param socket The accepted connection file descriptor.
* @param closeListeners A callback to close the listening sockets.
* Useful in forked child processes to release the bound sockets.
*/
using UnixSocketHandler = std::function<void(AutoCloseFD socket, std::function<void()> closeListeners)>;
/**
* Options for the serve loop.
*
* Only used if no systemd socket activation is detected.
*/
struct ServeUnixSocketOptions
{
/**
* The Unix domain socket path to create and listen on.
*/
std::filesystem::path socketPath;
/**
* Mode for the created socket file.
*/
mode_t socketMode = 0666;
};
/**
* Run a server loop that accepts connections and calls the handler for each.
*
* This function handles:
* - systemd socket activation (via LISTEN_FDS environment variable)
* - Creating and binding a Unix domain socket if no activation is detected
* - Polling for incoming connections
* - Accepting connections
*
* For each accepted connection, the handler is called with the connection
* file descriptor. The handler takes ownership of the file descriptor and
* is responsible for closing it when done.
*
* This function never returns normally. It runs until interrupted
* (e.g., via SIGINT), at which point it throws `Interrupted`.
*
* @param options Configuration for the server.
* @param handler Callback invoked for each accepted connection.
*/
[[noreturn]] void serveUnixSocket(const ServeUnixSocketOptions & options, UnixSocketHandler handler);
} // namespace nix::unix
+2 -4
View File
@@ -203,10 +203,8 @@ FlakeRef InstallableFlake::nixpkgsFlakeRef() const
if (auto nixpkgsInput = lockedFlake->lockFile.findInput({"nixpkgs"})) {
if (auto lockedNode = std::dynamic_pointer_cast<const flake::LockedNode>(nixpkgsInput)) {
if (lockedNode->isFlake) {
debug("using nixpkgs flake '%s'", lockedNode->lockedRef);
return std::move(lockedNode->lockedRef);
}
debug("using nixpkgs flake '%s'", lockedNode->lockedRef);
return std::move(lockedNode->lockedRef);
}
}
+8 -4
View File
@@ -157,7 +157,7 @@ MixFlakeOptions::MixFlakeOptions()
.category = category,
.labels = {"flake-lock-path"},
.handler = {[&](std::string lockFilePath) {
lockFlags.referenceLockFilePath = {getFSSourceAccessor(), CanonPath(absPath(lockFilePath).string())};
lockFlags.referenceLockFilePath = {getFSSourceAccessor(), CanonPath(absPath(lockFilePath))};
}},
.completer = completePath,
});
@@ -583,12 +583,16 @@ static void throwBuildErrors(std::vector<KeyedBuildResult> & buildResults, const
auto failedResult = failed.begin();
if (failedResult != failed.end()) {
if (failed.size() == 1) {
throw *failedResult->second;
failedResult->second->rethrow();
} else {
StringSet failedPaths;
for (; failedResult != failed.end(); failedResult++) {
if (!failedResult->second->message().empty()) {
logError(failedResult->second->info());
if (!failedResult->second->errorMsg.empty()) {
logError(
ErrorInfo{
.level = lvlError,
.msg = failedResult->second->errorMsg,
});
}
failedPaths.insert(failedResult->first->path.to_string(store));
}
-7
View File
@@ -74,7 +74,6 @@ sources = files(
'command.cc',
'common-eval-args.cc',
'editor-for.cc',
'get-build-log.cc',
'installable-attr-path.cc',
'installable-derived-path.cc',
'installable-flake.cc',
@@ -87,12 +86,6 @@ sources = files(
'repl.cc',
)
if host_machine.system() != 'windows'
sources += files(
'unix/unix-socket-server.cc',
)
endif
subdir('include/nix/cmd')
subdir('nix-meson-build-support/export-all-symbols')
+1 -1
View File
@@ -113,7 +113,7 @@ ReadlineLikeInteracter::Guard ReadlineLikeInteracter::init(detail::ReplCompleter
// Allow nix-repl specific settings in .inputrc
rl_readline_name = "nix-repl";
try {
createDirs(std::filesystem::path(historyFile).parent_path());
createDirs(dirOf(historyFile));
} catch (SystemError & e) {
logWarning(e.info());
}
+40 -18
View File
@@ -12,8 +12,9 @@
#include "nix/expr/eval-settings.hh"
#include "nix/expr/attr-path.hh"
#include "nix/util/signals.hh"
#include "nix/store/store-open.hh"
#include "nix/store/log-store.hh"
#include "nix/cmd/common-eval-args.hh"
#include "nix/cmd/get-build-log.hh"
#include "nix/expr/get-drvs.hh"
#include "nix/store/derivations.hh"
#include "nix/store/globals.hh"
@@ -64,7 +65,7 @@ struct NixRepl : AbstractNixRepl, detail::ReplCompleterMixin, gc
const static int envSize = 32768;
std::shared_ptr<StaticEnv> staticEnv;
std::optional<Value> lastLoaded;
Value lastLoaded;
Env * env;
int displ;
StringSet varNames;
@@ -77,6 +78,7 @@ struct NixRepl : AbstractNixRepl, detail::ReplCompleterMixin, gc
NixRepl(
const LookupPath & lookupPath,
nix::ref<Store> store,
ref<EvalState> state,
std::function<AnnotatedValues()> getValues,
RunNix * runNix);
@@ -131,6 +133,7 @@ std::string removeWhitespace(std::string s)
NixRepl::NixRepl(
const LookupPath & lookupPath,
nix::ref<Store> store,
ref<EvalState> state,
std::function<NixRepl::AnnotatedValues()> getValues,
RunNix * runNix)
@@ -563,9 +566,31 @@ ProcessLineResult NixRepl::processLine(std::string line)
} else if (command == ":log") {
settings.readOnlyMode = true;
Finally roModeReset([&]() { settings.readOnlyMode = false; });
auto subs = getDefaultSubstituters();
subs.push_front(state->store);
bool foundLog = false;
RunPager pager;
auto log = fetchBuildLog(state->store, drvPath, drvPathRaw);
logger->writeToStdout(log);
for (auto & sub : subs) {
auto * logSubP = dynamic_cast<LogStore *>(&*sub);
if (!logSubP) {
printInfo(
"Skipped '%s' which does not support retrieving build logs", sub->config.getHumanReadableURI());
continue;
}
auto & logSub = *logSubP;
auto log = logSub.getBuildLog(drvPath);
if (log) {
printInfo("got build log for '%s' from '%s'", drvPathRaw, logSub.config.getHumanReadableURI());
logger->writeToStdout(*log);
foundLog = true;
break;
}
}
if (!foundLog)
throw Error("build log of '%s' is not available", drvPathRaw);
} else {
runNix("nix-shell", {drvPathRaw});
}
@@ -748,19 +773,11 @@ void NixRepl::initEnv()
void NixRepl::showLastLoaded()
{
if (!lastLoaded)
throw Error("nothing has been loaded yet");
RunPager pager;
try {
for (auto & i : *lastLoaded->attrs()) {
std::string_view name = state->symbols[i.name];
logger->cout(name);
}
} catch (SystemError & e) {
/* Ignore broken pipes when the pager gets interrupted. */
if (!e.is(std::errc::broken_pipe))
throw;
for (auto & i : *lastLoaded.attrs()) {
std::string_view name = state->symbols[i.name];
logger->cout(name);
}
}
@@ -883,9 +900,13 @@ void NixRepl::runNix(const std::string & program, const Strings & args, const st
}
std::unique_ptr<AbstractNixRepl> AbstractNixRepl::create(
const LookupPath & lookupPath, ref<EvalState> state, std::function<AnnotatedValues()> getValues, RunNix * runNix)
const LookupPath & lookupPath,
nix::ref<Store> store,
ref<EvalState> state,
std::function<AnnotatedValues()> getValues,
RunNix * runNix)
{
return std::make_unique<NixRepl>(lookupPath, state, getValues, runNix);
return std::make_unique<NixRepl>(lookupPath, std::move(store), state, getValues, runNix);
}
ReplExitStatus AbstractNixRepl::runSimple(ref<EvalState> evalState, const ValMap & extraEnv)
@@ -898,6 +919,7 @@ ReplExitStatus AbstractNixRepl::runSimple(ref<EvalState> evalState, const ValMap
// NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDelete)
auto repl = std::make_unique<NixRepl>(
lookupPath,
openStore(),
evalState,
getValues,
/*runNix=*/nullptr);
-126
View File
@@ -1,126 +0,0 @@
///@file
#include "nix/cmd/unix-socket-server.hh"
#include "nix/util/environment-variables.hh"
#include "nix/util/file-system.hh"
#include "nix/util/logging.hh"
#include "nix/util/signals.hh"
#include "nix/util/strings.hh"
#include "nix/util/unix-domain-socket.hh"
#include "nix/util/util.hh"
#include <sys/socket.h>
#include <sys/un.h>
#include <poll.h>
#if defined(__APPLE__) || defined(__FreeBSD__)
# include <sys/ucred.h>
#endif
namespace nix::unix {
PeerInfo getPeerInfo(Descriptor remote)
{
PeerInfo peer;
#if defined(SO_PEERCRED)
# if defined(__OpenBSD__)
struct sockpeercred cred;
# else
ucred cred;
# endif
socklen_t credLen = sizeof(cred);
if (getsockopt(remote, SOL_SOCKET, SO_PEERCRED, &cred, &credLen) == 0) {
peer.pid = cred.pid;
peer.uid = cred.uid;
peer.gid = cred.gid;
}
#elif defined(LOCAL_PEERCRED)
# if !defined(SOL_LOCAL)
# define SOL_LOCAL 0
# endif
xucred cred;
socklen_t credLen = sizeof(cred);
if (getsockopt(remote, SOL_LOCAL, LOCAL_PEERCRED, &cred, &credLen) == 0)
peer.uid = cred.cr_uid;
#endif
return peer;
}
[[noreturn]] void serveUnixSocket(const ServeUnixSocketOptions & options, UnixSocketHandler handler)
{
std::vector<AutoCloseFD> listeningSockets;
static constexpr int SD_LISTEN_FDS_START = 3;
// Handle socket-based activation by systemd.
auto listenFds = getEnv("LISTEN_FDS");
if (listenFds) {
if (getEnv("LISTEN_PID") != std::to_string(getpid()))
throw Error("unexpected systemd environment variables");
auto count = string2Int<unsigned int>(*listenFds);
assert(count);
for (unsigned int i = 0; i < count; ++i) {
AutoCloseFD fdSocket(SD_LISTEN_FDS_START + i);
closeOnExec(fdSocket.get());
listeningSockets.push_back(std::move(fdSocket));
}
}
// Otherwise, create and bind to a Unix domain socket.
else {
createDirs(options.socketPath.parent_path());
listeningSockets.push_back(createUnixDomainSocket(options.socketPath.string(), options.socketMode));
}
std::vector<struct pollfd> fds;
for (auto & i : listeningSockets)
fds.push_back({.fd = i.get(), .events = POLLIN});
// Loop accepting connections.
while (1) {
try {
checkInterrupt();
auto count = poll(fds.data(), fds.size(), -1);
if (count == -1) {
if (errno == EINTR)
continue;
throw SysError("polling for incoming connections");
}
for (auto & fd : fds) {
if (!fd.revents)
continue;
// Accept a connection.
struct sockaddr_un remoteAddr;
socklen_t remoteAddrLen = sizeof(remoteAddr);
AutoCloseFD remote = accept(fd.fd, (struct sockaddr *) &remoteAddr, &remoteAddrLen);
checkInterrupt();
if (!remote) {
if (errno == EINTR)
continue;
throw SysError("accepting connection");
}
handler(std::move(remote), [&]() { listeningSockets.clear(); });
}
} catch (Error & error) {
auto ei = error.info();
// FIXME: add to trace?
ei.msg = HintFmt("while processing connection: %1%", ei.msg.str());
logError(ei);
}
}
}
} // namespace nix::unix
+1 -1
View File
@@ -756,7 +756,7 @@ TEST_F(PrimOpTest, langVersion)
TEST_F(PrimOpTest, storeDir)
{
auto v = eval("builtins.storeDir");
ASSERT_THAT(v, IsStringEq(state.store->storeDir));
ASSERT_THAT(v, IsStringEq(settings.nixStore));
}
TEST_F(PrimOpTest, nixVersion)
+1 -1
View File
@@ -70,7 +70,7 @@ struct AttrDb
{
auto state(_state->lock());
auto cacheDir = getCacheDir() / "eval-cache-v6";
auto cacheDir = std::filesystem::path(getCacheDir()) / "eval-cache-v6";
createDirs(cacheDir);
auto dbPath = cacheDir / (fingerprint.to_string(HashFormat::Base16, false) + ".sqlite");
-7
View File
@@ -1,16 +1,9 @@
#include "nix/expr/eval-error.hh"
#include "nix/expr/eval.hh"
#include "nix/expr/value.hh"
#include "nix/store/store-api.hh"
namespace nix {
InvalidPathError::InvalidPathError(EvalState & state, const StorePath & path)
: EvalError(state, "path '%s' is not valid", state.store->printStorePath(path))
, path{path}
{
}
template<class T>
EvalErrorBuilder<T> & EvalErrorBuilder<T>::withExitStatus(unsigned int exitStatus)
{
+2 -8
View File
@@ -133,17 +133,11 @@ class SampleStack : public EvalProfiler
FrameInfo getPrimOpFrameInfo(const PrimOp & primOp, std::span<Value *> args, PosIdx pos);
public:
SampleStack(EvalState & state, const std::filesystem::path & profileFile, std::chrono::nanoseconds period)
SampleStack(EvalState & state, std::filesystem::path profileFile, std::chrono::nanoseconds period)
: state(state)
, sampleInterval(period)
, profileFd([&]() {
AutoCloseFD fd = openNewFileForWrite(
profileFile,
0660,
{
.truncateExisting = true,
.followSymlinksOnTruncate = true, /* FIXME: Probably shouldn't follow symlinks. */
});
AutoCloseFD fd = toDescriptor(open(profileFile.string().c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0660));
if (!fd)
throw SysError("opening file %s", PathFmt(profileFile));
return fd;
+3 -4
View File
@@ -70,10 +70,9 @@ Strings EvalSettings::getDefaultNixPath()
}
};
add(getNixDefExpr() / "channels");
auto profilesDirOpts = settings.getProfileDirsOptions();
add(rootChannelsDir(profilesDirOpts) / "nixpkgs", "nixpkgs");
add(rootChannelsDir(profilesDirOpts));
add(std::filesystem::path{getNixDefExpr()} / "channels");
add(rootChannelsDir() / "nixpkgs", "nixpkgs");
add(rootChannelsDir());
return res;
}
+2 -1
View File
@@ -318,6 +318,7 @@ EvalState::EvalState(
countCalls = getEnv("NIX_COUNT_CALLS").value_or("0") != "0";
static_assert(sizeof(Env) <= 16, "environment must be <= 16 bytes");
static_assert(sizeof(Counter) == 64, "counters must be 64 bytes");
/* Construct the Nix expression search path. */
assert(lookupPath.elements.empty());
@@ -432,7 +433,7 @@ void EvalState::checkURI(const std::string & uri)
/* If the URI is a path, then check it against allowedPaths as
well. */
if (std::filesystem::path(uri).is_absolute()) {
if (isAbsolute(uri)) {
if (auto rootFS2 = rootFS.dynamic_pointer_cast<AllowListSourceAccessor>())
rootFS2->checkAccess(CanonPath(uri));
return;
+1 -1
View File
@@ -11,7 +11,7 @@ namespace nix {
* variable is set. This is to prevent contention on these counters
* when multi-threaded evaluation is enabled.
*/
struct alignas(std::hardware_destructive_interference_size) Counter
struct alignas(64) Counter
{
using value_type = uint64_t;
+5 -2
View File
@@ -73,9 +73,12 @@ MakeError(IFDError, EvalBaseError);
struct InvalidPathError : public EvalError
{
public:
StorePath path;
Path path;
InvalidPathError(EvalState & state, const StorePath & path);
InvalidPathError(EvalState & state, const Path & path)
: EvalError(state, "path '%s' is not valid", path)
{
}
};
/**
+4 -4
View File
@@ -134,7 +134,7 @@ public:
virtual bool operator==(const ExternalValueBase & b) const noexcept;
/**
* Print the value as JSON. Defaults to unconvertible, i.e. throws an error
* Print the value as JSON. Defaults to unconvertable, i.e. throws an error
*/
virtual nlohmann::json
printValueAsJSON(EvalState & state, bool strict, NixStringContext & context, bool copyToStore = true) const;
@@ -533,8 +533,8 @@ inline constexpr bool useBitPackedValueStorage = (ptrSize == 8) && (__STDCPP_DEF
* Packs discriminator bits into the pointer alignment niches.
*/
template<std::size_t ptrSize>
class alignas(16)
ValueStorage<ptrSize, std::enable_if_t<detail::useBitPackedValueStorage<ptrSize>>> : public detail::ValueBase
class alignas(16) ValueStorage<ptrSize, std::enable_if_t<detail::useBitPackedValueStorage<ptrSize>>>
: public detail::ValueBase
{
/* Needs a dependent type name in order for member functions (and
* potentially ill-formed bit casts) to be SFINAE'd out.
@@ -587,7 +587,7 @@ class alignas(16)
enum PrimaryDiscriminator : int {
pdUninitialized = 0,
pdSingleDWord, //< layout: Single/zero field payload
/* The order of these enumerations must be the same as in InternalType. */
/* The order of these enumations must be the same as in InternalType. */
pdListN, //< layout: Single untaggable field.
pdString,
pdPath,
+14 -38
View File
@@ -3,7 +3,7 @@
%define api.namespace { ::nix::parser }
%define api.parser.class { BisonParser }
%locations
%define parse.error detailed
%define parse.error verbose
%defines
/* %no-lines */
%parse-param { void * scanner }
@@ -140,41 +140,18 @@ static Expr * makeCall(Exprs & exprs, PosIdx pos, Expr * fn, Expr * arg) {
%type <Expr *> path_start
%type <ToBeStringyExpr> string_parts string_attr
%type <StringToken> attr
%token <StringToken> ID "identifier"
%token <StringToken> STR "string"
%token <StringToken> IND_STR "indented string"
%token <NixInt> INT_LIT "integer"
%token <NixFloat> FLOAT_LIT "floating-point literal"
%token <StringToken> PATH "path"
%token <StringToken> HPATH "'~/…' path"
%token <StringToken> SPATH "'<…>' path"
%token <StringToken> PATH_END "end of path"
%token <StringToken> URI "URI"
%token IF "'if'"
%token THEN "'then'"
%token ELSE "'else'"
%token ASSERT "'assert'"
%token WITH "'with'"
%token LET "'let'"
%token IN_KW "'in'"
%token REC "'rec'"
%token INHERIT "'inherit'"
%token EQ "'=='"
%token NEQ "'!='"
%token LEQ "'<='"
%token GEQ "'>='"
%token UPDATE "'//'"
%token CONCAT "'++'"
%token AND "'&&'"
%token OR "'||'"
%token IMPL "'->'"
%token OR_KW "'or'"
%token PIPE_FROM "'<|'"
%token PIPE_INTO "'|>'"
%token DOLLAR_CURLY "'${'"
%token IND_STRING_OPEN "start of an indented string"
%token IND_STRING_CLOSE "end of an indented string"
%token ELLIPSIS "'...'"
%token <StringToken> ID
%token <StringToken> STR IND_STR
%token <NixInt> INT_LIT
%token <NixFloat> FLOAT_LIT
%token <StringToken> PATH HPATH SPATH PATH_END
%token <StringToken> URI
%token IF THEN ELSE ASSERT WITH LET IN_KW REC INHERIT EQ NEQ AND OR IMPL OR_KW
%token PIPE_FROM PIPE_INTO /* <| and |> */
%token DOLLAR_CURLY /* == ${ */
%token IND_STRING_OPEN IND_STRING_CLOSE
%token ELLIPSIS
%right IMPL
%left OR
@@ -388,8 +365,7 @@ path_start
});
}
auto basePath = std::filesystem::path(state->basePath.path.abs());
Path path(absPath(literal, &basePath).string());
Path path(absPath(literal, state->basePath.path.abs()));
/* add back in the trailing '/' to the first segment */
if (literal.size() > 1 && literal.back() == '/')
path += '/';
+3 -2
View File
@@ -12,7 +12,7 @@ SourcePath EvalState::rootPath(CanonPath path)
SourcePath EvalState::rootPath(PathView path)
{
return {rootFS, CanonPath(absPath(std::filesystem::path{path}).string())};
return {rootFS, CanonPath(absPath(path))};
}
SourcePath EvalState::storePath(const StorePath & path)
@@ -23,12 +23,13 @@ SourcePath EvalState::storePath(const StorePath & path)
StorePath
EvalState::mountInput(fetchers::Input & input, const fetchers::Input & originalInput, ref<SourceAccessor> accessor)
{
auto [storePath, narHash] = fetchToStore2(fetchSettings, *store, accessor, FetchMode::Copy, input.getName());
auto storePath = fetchToStore(fetchSettings, *store, accessor, FetchMode::Copy, input.getName());
allowPath(storePath); // FIXME: should just whitelist the entire virtual store
storeFS->mount(CanonPath(store->printStorePath(storePath)), accessor);
auto narHash = store->queryPathInfo(storePath)->narHash;
input.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true));
if (originalInput.getNarHash() && narHash != *originalInput.getNarHash())
+6 -11
View File
@@ -74,7 +74,7 @@ StringMap EvalState::realiseContext(const NixStringContext & context, StorePathS
for (auto & c : context) {
auto ensureValid = [&](const StorePath & p) {
if (!store->isValidPath(p))
error<InvalidPathError>(p).debugThrow();
error<InvalidPathError>(store->printStorePath(p)).debugThrow();
};
std::visit(
overloaded{
@@ -503,7 +503,7 @@ void prim_exec(EvalState & state, const PosIdx pos, Value ** args, Value & v)
try {
auto _ = state.realiseContext(context); // FIXME: Handle CA derivations
} catch (InvalidPathError & e) {
state.error<EvalError>("cannot execute '%1%', since path '%2%' is not valid", program, e.path.to_string())
state.error<EvalError>("cannot execute '%1%', since path '%2%' is not valid", program, e.path)
.atPos(pos)
.debugThrow();
}
@@ -1814,13 +1814,8 @@ static void derivationStrictInternal(EvalState & state, std::string_view drvName
drv.fillInOutputPaths(*state.store);
}
/* Write the resulting term into the Nix store directory.
Unless we are in read-only mode, that is, in which case we do not
write anything. Users commonly do this to speed up evaluation in
contexts where they don't actually want to build anything. */
auto drvPath =
settings.readOnlyMode ? computeStorePath(*state.store, drv) : state.store->writeDerivation(drv, state.repair);
/* Write the resulting term into the Nix store directory. */
auto drvPath = writeDerivation(*state.store, drv, state.repair);
auto drvPathS = state.store->printStorePath(drvPath);
printMsg(lvlChatty, "instantiated '%1%' -> '%2%'", drvName, drvPathS);
@@ -1931,7 +1926,7 @@ static void prim_storePath(EvalState & state, const PosIdx pos, Value ** args, V
directly in the store. The latter condition is necessary so
e.g. nix-push does the right thing. */
if (!state.store->isStorePath(path.abs()))
path = CanonPath(canonPath(path.abs(), true).string());
path = CanonPath(canonPath(path.abs(), true));
if (!state.store->isInStore(path.abs()))
state.error<EvalError>("path '%1%' is not in the Nix store", path).atPos(pos).debugThrow();
auto path2 = state.store->toStorePath(path.abs()).first;
@@ -2152,7 +2147,7 @@ static void prim_findFile(EvalState & state, const PosIdx pos, Value ** args, Va
auto rewrites = state.realiseContext(context);
path = rewriteStrings(std::move(path), rewrites);
} catch (InvalidPathError & e) {
state.error<EvalError>("cannot find '%1%', since path '%2%' is not valid", path, state.store->printStorePath(e.path))
state.error<EvalError>("cannot find '%1%', since path '%2%' is not valid", path, e.path)
.atPos(pos)
.debugThrow();
}
+7 -13
View File
@@ -191,25 +191,19 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value ** args
{.msg = HintFmt("attribute '%s' is missing in call to 'fetchClosure'", "fromStore"),
.pos = state.positions[pos]});
auto storeRef = StoreReference::parse(*fromStoreUrl);
auto parsedURL = parseURL(*fromStoreUrl, /*lenient=*/true);
if ([&] {
auto * specified = std::get_if<StoreReference::Specified>(&storeRef.variant);
return !specified
|| (specified->scheme != "http" && specified->scheme != "https"
&& !(getEnv("_NIX_IN_TEST").has_value() && specified->scheme == "file"));
}())
throw Error({
.msg = HintFmt("'fetchClosure' only supports http:// and https:// stores"),
.pos = state.positions[pos],
});
if (parsedURL.scheme != "http" && parsedURL.scheme != "https"
&& !(getEnv("_NIX_IN_TEST").has_value() && parsedURL.scheme == "file"))
throw Error(
{.msg = HintFmt("'fetchClosure' only supports http:// and https:// stores"), .pos = state.positions[pos]});
if (!storeRef.params.empty())
if (!parsedURL.query.empty())
throw Error(
{.msg = HintFmt("'fetchClosure' does not support URL query parameters (in '%s')", *fromStoreUrl),
.pos = state.positions[pos]});
auto fromStore = openStore(std::move(storeRef));
auto fromStore = openStore(parsedURL.to_string());
if (toPath)
runFetchClosureWithRewrite(state, pos, *fromStore, *fromPath, *toPath, v);
+5 -11
View File
@@ -34,20 +34,14 @@ struct CacheImpl : Cache
Sync<State> _state;
/**
* This is a back-reference to the `Settings` that owns us.
*/
const Settings & settings;
CacheImpl(const Settings & _settings)
: settings(_settings)
CacheImpl()
{
auto state(_state.lock());
auto dbPath = getCacheDir() / "fetcher-cache-v4.sqlite";
createDirs(dbPath.parent_path());
auto dbPath = (getCacheDir() / "fetcher-cache-v4.sqlite").string();
createDirs(dirOf(dbPath));
state->db = SQLite(dbPath, {.useWAL = nix::settings.useSQLiteWAL});
state->db = SQLite(dbPath, {.useWAL = settings.useSQLiteWAL});
state->db.isCache();
state->db.exec(schema);
@@ -160,7 +154,7 @@ ref<Cache> Settings::getCache() const
{
auto cache(_cache.lock());
if (!*cache)
*cache = std::make_shared<CacheImpl>(*this);
*cache = std::make_shared<CacheImpl>();
return ref<Cache>(*cache);
}
+18 -63
View File
@@ -5,12 +5,12 @@
namespace nix {
fetchers::Cache::Key
makeSourcePathToHashCacheKey(std::string_view fingerprint, ContentAddressMethod method, const CanonPath & path)
fetchers::Cache::Key makeFetchToStoreCacheKey(
const std::string & name, const std::string & fingerprint, ContentAddressMethod method, const std::string & path)
{
return fetchers::Cache::Key{
"sourcePathToHash",
{{"fingerprint", std::string(fingerprint)}, {"method", std::string{method.render()}}, {"path", path.abs()}}};
"fetchToStore",
{{"name", name}, {"fingerprint", fingerprint}, {"method", std::string{method.render()}}, {"path", path}}};
}
StorePath fetchToStore(
@@ -23,39 +23,19 @@ StorePath fetchToStore(
PathFilter * filter,
RepairFlag repair)
{
return fetchToStore2(settings, store, path, mode, name, method, filter, repair).first;
}
// FIXME: add an optimisation for the case where the accessor is
// a `PosixSourceAccessor` pointing to a store path.
std::pair<StorePath, Hash> fetchToStore2(
const fetchers::Settings & settings,
Store & store,
const SourcePath & path,
FetchMode mode,
std::string_view name,
ContentAddressMethod method,
PathFilter * filter,
RepairFlag repair)
{
std::optional<fetchers::Cache::Key> cacheKey;
auto [subpath, fingerprint] = filter ? std::pair<CanonPath, std::optional<std::string>>{path.path, std::nullopt}
: path.accessor->getFingerprint(path.path);
if (fingerprint) {
cacheKey = makeSourcePathToHashCacheKey(*fingerprint, method, subpath);
if (auto res = settings.getCache()->lookup(*cacheKey)) {
auto hash = Hash::parseSRI(fetchers::getStrAttr(*res, "hash"));
auto storePath =
store.makeFixedOutputPathFromCA(name, ContentAddressWithReferences::fromParts(method, hash, {}));
if (mode == FetchMode::DryRun || store.isValidPath(storePath)) {
debug(
"source path '%s' cache hit in '%s' (hash '%s')",
path,
store.printStorePath(storePath),
hash.to_string(HashFormat::SRI, true));
return {storePath, hash};
}
debug("source path '%s' not in store", path);
cacheKey = makeFetchToStoreCacheKey(std::string{name}, *fingerprint, method, subpath.abs());
if (auto res = settings.getCache()->lookupStorePath(*cacheKey, store)) {
debug("store path cache hit for '%s'", path);
return res->storePath;
}
} else {
static auto barf = getEnv("_NIX_TEST_BARF_ON_UNCACHEABLE").value_or("") == "1";
@@ -73,41 +53,16 @@ std::pair<StorePath, Hash> fetchToStore2(
auto filter2 = filter ? *filter : defaultPathFilter;
auto [storePath, hash] =
mode == FetchMode::DryRun
? [&]() {
auto [storePath, hash] =
store.computeStorePath(name, path, method, HashAlgorithm::SHA256, {}, filter2);
debug(
"hashed '%s' to '%s' (hash '%s')",
path,
store.printStorePath(storePath),
hash.to_string(HashFormat::SRI, true));
return std::make_pair(storePath, hash);
}()
: [&]() {
// FIXME: ideally addToStore() would return the hash
// right away (like computeStorePath()).
auto storePath = store.addToStore(name, path, method, HashAlgorithm::SHA256, {}, filter2, repair);
auto info = store.queryPathInfo(storePath);
assert(info->references.empty());
auto hash = method == ContentAddressMethod::Raw::NixArchive ? info->narHash : ({
if (!info->ca || info->ca->method != method)
throw Error("path '%s' lacks a CA field", store.printStorePath(storePath));
info->ca->hash;
});
debug(
"copied '%s' to '%s' (hash '%s')",
path,
store.printStorePath(storePath),
hash.to_string(HashFormat::SRI, true));
return std::make_pair(storePath, hash);
}();
auto storePath = mode == FetchMode::DryRun
? store.computeStorePath(name, path, method, HashAlgorithm::SHA256, {}, filter2).first
: store.addToStore(name, path, method, HashAlgorithm::SHA256, {}, filter2, repair);
if (cacheKey)
settings.getCache()->upsert(*cacheKey, {{"hash", hash.to_string(HashFormat::SRI, true)}});
debug(mode == FetchMode::DryRun ? "hashed '%s'" : "copied '%s' to '%s'", path, store.printStorePath(storePath));
return {storePath, hash};
if (cacheKey && mode == FetchMode::Copy)
settings.getCache()->upsert(*cacheKey, store, {}, storePath);
return storePath;
}
} // namespace nix
+3 -4
View File
@@ -339,10 +339,9 @@ std::pair<ref<SourceAccessor>, Input> Input::getAccessorUnchecked(const Settings
// can reuse the existing nar instead of copying the unpacked
// input back into the store on every evaluation.
if (accessor->fingerprint) {
settings.getCache()->upsert(
makeSourcePathToHashCacheKey(
*accessor->fingerprint, ContentAddressMethod::Raw::NixArchive, CanonPath::root),
{{"hash", store.queryPathInfo(storePath)->narHash.to_string(HashFormat::SRI, true)}});
ContentAddressMethod method = ContentAddressMethod::Raw::NixArchive;
auto cacheKey = makeFetchToStoreCacheKey(getName(), *accessor->fingerprint, method, "/");
settings.getCache()->upsert(cacheKey, store, {}, storePath);
}
accessor->setPathDisplay("«" + to_string() + "»");
@@ -61,11 +61,6 @@ std::pair<CanonPath, std::optional<std::string>> FilteringSourceAccessor::getFin
return next->getFingerprint(prefix / path);
}
void FilteringSourceAccessor::invalidateCache(const CanonPath & path)
{
next->invalidateCache(prefix / path);
}
void FilteringSourceAccessor::checkAccess(const CanonPath & path)
{
if (!isAllowed(path))
+2 -2
View File
@@ -268,10 +268,10 @@ void Fetch::fetch(
return;
}
auto cacheDir = getCacheDir() / "git-lfs";
std::filesystem::path cacheDir = getCacheDir() / "git-lfs";
std::string key = hashString(HashAlgorithm::SHA256, pointerFilePath.rel()).to_string(HashFormat::Base16, false)
+ "/" + pointer->oid;
auto cachePath = cacheDir / key;
std::filesystem::path cachePath = cacheDir / key;
if (pathExists(cachePath)) {
debug("using cache entry %s -> %s", key, PathFmt(cachePath));
sink(readFile(cachePath));
+1 -1
View File
@@ -666,7 +666,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
.program = "git",
.args =
{"-c",
"gpg.ssh.allowedSignersFile=" + allowedSignersFile.string(),
"gpg.ssh.allowedSignersFile=" + allowedSignersFile,
"-C",
path.string(),
"verify-commit",
+14 -15
View File
@@ -37,16 +37,15 @@ namespace {
// old version of git, which will ignore unrecognized `-c` options.
const std::string gitInitialBranch = "__nix_dummy_branch";
static bool isCacheFileWithinTtl(const Settings & settings, time_t now, const PosixStat & st)
bool isCacheFileWithinTtl(time_t now, const struct stat & st)
{
return st.st_mtime + static_cast<time_t>(settings.tarballTtl) > now;
}
std::filesystem::path getCachePath(std::string_view key, bool shallow)
{
auto name =
hashString(HashAlgorithm::SHA256, key).to_string(HashFormat::Nix32, false) + (shallow ? "-shallow" : "");
return getCacheDir() / "gitv3" / std::move(name);
return getCacheDir() / "gitv3"
/ (hashString(HashAlgorithm::SHA256, key).to_string(HashFormat::Nix32, false) + (shallow ? "-shallow" : ""));
}
// Returns the name of the HEAD branch.
@@ -106,7 +105,7 @@ bool storeCachedHead(const std::string & actualUrl, bool shallow, const std::str
return true;
}
static std::optional<std::string> readHeadCached(const Settings & settings, const std::string & actualUrl, bool shallow)
std::optional<std::string> readHeadCached(const std::string & actualUrl, bool shallow)
{
// Create a cache path to store the branch of the HEAD ref. Append something
// in front of the URL to prevent collision with the repository itself.
@@ -114,11 +113,11 @@ static std::optional<std::string> readHeadCached(const Settings & settings, cons
std::filesystem::path headRefFile = cacheDir / "HEAD";
time_t now = time(0);
auto st = maybeStat(headRefFile);
struct stat st;
std::optional<std::string> cachedRef;
if (st) {
if (stat(headRefFile.string().c_str(), &st) == 0) {
cachedRef = readHead(cacheDir);
if (cachedRef != std::nullopt && *cachedRef != gitInitialBranch && isCacheFileWithinTtl(settings, now, *st)) {
if (cachedRef != std::nullopt && *cachedRef != gitInitialBranch && isCacheFileWithinTtl(now, st)) {
debug("using cached HEAD ref '%s' for repo '%s'", *cachedRef, actualUrl);
return cachedRef;
}
@@ -630,7 +629,7 @@ struct GitInputScheme : InputScheme
if (url.scheme == "file" && !forceHttp && !isBareRepository(renderUrlPathEnsureLegal(url.path))) {
auto path = renderUrlPathEnsureLegal(url.path);
if (!std::filesystem::path(path).is_absolute()) {
if (!isAbsolute(path)) {
warn(
"Fetching Git repository '%s', which uses a path relative to the current directory. "
"This is not supported and will stop working in a future release. "
@@ -729,12 +728,12 @@ struct GitInputScheme : InputScheme
return revCount;
}
std::string getDefaultRef(const Settings & settings, const RepoInfo & repoInfo, bool shallow) const
std::string getDefaultRef(const RepoInfo & repoInfo, bool shallow) const
{
auto head = std::visit(
overloaded{
[&](const std::filesystem::path & path) { return GitRepo::openRepo(path, {})->getWorkdirRef(); },
[&](const ParsedURL & url) { return readHeadCached(settings, url.to_string(), shallow); }},
[&](const ParsedURL & url) { return readHeadCached(url.to_string(), shallow); }},
repoInfo.location);
if (!head) {
warn("could not read HEAD ref from repo at '%s', using 'master'", repoInfo.locationToArg());
@@ -784,7 +783,7 @@ struct GitInputScheme : InputScheme
auto originalRef = input.getRef();
bool shallow = getShallowAttr(input);
auto ref = originalRef ? *originalRef : getDefaultRef(settings, repoInfo, shallow);
auto ref = originalRef ? *originalRef : getDefaultRef(repoInfo, shallow);
input.attrs.insert_or_assign("ref", ref);
std::filesystem::path repoDir;
@@ -820,10 +819,10 @@ struct GitInputScheme : InputScheme
if (getAllRefsAttr(input)) {
doFetch = true;
} else {
/* If the local ref is older than 'tarball-ttl' seconds, do a
/* If the local ref is older than ‘tarball-ttl’ seconds, do a
git fetch to update the local ref to the remote ref. */
auto st = maybeStat(localRefFile);
doFetch = !st || !isCacheFileWithinTtl(settings, now, *st);
struct stat st;
doFetch = stat(localRefFile.string().c_str(), &st) != 0 || !isCacheFileWithinTtl(now, st);
}
}
@@ -129,25 +129,6 @@ struct Settings : public Config
true,
Xp::Flakes};
Setting<unsigned int> tarballTtl{
this,
60 * 60,
"tarball-ttl",
R"(
The number of seconds a downloaded tarball is considered fresh. If
the cached tarball is stale, Nix checks whether it is still up
to date using the ETag header. Nix downloads a new version if
the ETag header is unsupported, or the cached ETag doesn't match.
Setting the TTL to `0` forces Nix to always check if the tarball is
up to date.
Nix caches tarballs in `$XDG_CACHE_HOME/nix/tarballs`.
Files fetched via `NIX_PATH`, `fetchGit`, `fetchMercurial`,
`fetchTarball`, and `fetchurl` respect this TTL.
)"};
ref<Cache> getCache() const;
ref<GitRepo> getTarballCache() const;
@@ -24,17 +24,7 @@ StorePath fetchToStore(
PathFilter * filter = nullptr,
RepairFlag repair = NoRepair);
std::pair<StorePath, Hash> fetchToStore2(
const fetchers::Settings & settings,
Store & store,
const SourcePath & path,
FetchMode mode,
std::string_view name = "source",
ContentAddressMethod method = ContentAddressMethod::Raw::NixArchive,
PathFilter * filter = nullptr,
RepairFlag repair = NoRepair);
fetchers::Cache::Key
makeSourcePathToHashCacheKey(std::string_view fingerprint, ContentAddressMethod method, const CanonPath & path);
fetchers::Cache::Key makeFetchToStoreCacheKey(
const std::string & name, const std::string & fingerprint, ContentAddressMethod method, const std::string & path);
} // namespace nix
@@ -52,8 +52,6 @@ struct FilteringSourceAccessor : SourceAccessor
std::pair<CanonPath, std::optional<std::string>> getFingerprint(const CanonPath & path) override;
void invalidateCache(const CanonPath & path) override;
/**
* Call `makeNotAllowedError` to throw a `RestrictedPathError`
* exception if `isAllowed()` returns `false` for `path`.
@@ -1,5 +1,3 @@
#pragma once
#include "nix/fetchers/fetchers.hh"
namespace nix::fetchers {
+3 -4
View File
@@ -1,6 +1,5 @@
#include "nix/fetchers/fetchers.hh"
#include "nix/util/processes.hh"
#include "nix/util/environment-variables.hh"
#include "nix/util/users.hh"
#include "nix/fetchers/cache.hh"
#include "nix/store/globals.hh"
@@ -17,9 +16,9 @@ namespace nix::fetchers {
static RunOptions hgOptions(const Strings & args)
{
auto env = getEnvOs();
auto env = getEnv();
// Set HGPLAIN: this means we get consistent output from hg and avoids leakage from a user or system .hgrc.
env[OS_STR("HGPLAIN")] = OS_STR("");
env["HGPLAIN"] = "";
return {.program = "hg", .lookupPath = true, .args = args, .environment = env};
}
@@ -302,7 +301,7 @@ struct MercurialInputScheme : InputScheme
}
}
} else {
createDirs(cacheDir.parent_path());
createDirs(dirOf(cacheDir.string()));
runHg({"clone", "--noupdate", "--", actualUrl, cacheDir.string()});
}
}
+9 -7
View File
@@ -117,7 +117,7 @@ struct PathInputScheme : InputScheme
std::optional<std::string> isRelative(const Input & input) const override
{
auto path = getStrAttr(input.attrs, "path");
if (std::filesystem::path(path).is_absolute())
if (isAbsolute(path))
return std::nullopt;
else
return path;
@@ -132,7 +132,7 @@ struct PathInputScheme : InputScheme
{
auto path = getStrAttr(input.attrs, "path");
if (std::filesystem::path(path).is_absolute())
if (isAbsolute(path))
return canonPath(path);
throw Error("cannot fetch input '%s' because it uses a relative path", input.to_string());
@@ -165,12 +165,14 @@ struct PathInputScheme : InputScheme
// To prevent `fetchToStore()` copying the path again to Nix
// store, pre-create an entry in the fetcher cache.
auto narHash = store.queryPathInfo(*storePath)->narHash.to_string(HashFormat::SRI, true);
accessor->fingerprint = fmt("path:%s", narHash);
accessor->fingerprint =
fmt("path:%s", store.queryPathInfo(*storePath)->narHash.to_string(HashFormat::SRI, true));
settings.getCache()->upsert(
makeSourcePathToHashCacheKey(
*accessor->fingerprint, ContentAddressMethod::Raw::NixArchive, CanonPath::root),
{{"hash", narHash}});
makeFetchToStoreCacheKey(
input.getName(), *accessor->fingerprint, ContentAddressMethod::Raw::NixArchive, "/"),
store,
{},
*storePath);
/* Trust the lastModified value supplied by the user, if
any. It's not a "secure" attribute so we don't care. */
+3 -3
View File
@@ -92,7 +92,7 @@ void Registry::remove(const Input & input)
static std::filesystem::path getSystemRegistryPath()
{
return nixConfDir() / "registry.json";
return settings.nixConfDir / "registry.json";
}
static std::shared_ptr<Registry> getSystemRegistry(const Settings & settings)
@@ -147,13 +147,13 @@ static std::shared_ptr<Registry> getGlobalRegistry(const Settings & settings, St
return Registry::read(
settings,
[&] -> SourcePath {
if (!std::filesystem::path(path).is_absolute()) {
if (!isAbsolute(path)) {
auto storePath = downloadFile(store, settings, path, "flake-registry.json").storePath;
if (auto store2 = dynamic_cast<LocalFSStore *>(&store))
store2->addPermRoot(storePath, (getCacheDir() / "flake-registry.json").string());
return {store.requireStoreObjectAccessor(storePath)};
} else {
return SourcePath{makeFSSourceAccessor(path)}.resolveSymlinks();
return SourcePath{getFSSourceAccessor(), CanonPath{path}}.resolveSymlinks();
}
}(),
Registry::Global);
+2 -3
View File
@@ -177,9 +177,8 @@ static DownloadTarballResult downloadTarball_(
the entire file to disk so libarchive can access it
in random-access mode. */
auto [fdTemp, path] = createTempFile("nix-zipfile");
cleanupTemp.cancel();
cleanupTemp = {path};
debug("downloading '%s' into %s...", url, PathFmt(path));
cleanupTemp.reset(path);
debug("downloading '%s' into '%s'...", url, path);
{
FdSink sink(fdTemp.get());
source->drainInto(sink);
+1 -1
View File
@@ -13,7 +13,7 @@ struct nix_flake_settings
struct nix_flake_reference_parse_flags
{
std::optional<std::filesystem::path> baseDirectory;
std::optional<nix::Path> baseDirectory;
};
struct nix_flake_reference
+1 -1
View File
@@ -31,7 +31,7 @@ namespace nix::flake {
// setting name -> setting value -> allow or ignore.
typedef std::map<std::string, std::map<std::string, bool>> TrustedList;
static std::filesystem::path trustedListPath()
std::filesystem::path trustedListPath()
{
return getDataDir() / "trusted-settings.json";
}
-1
View File
@@ -128,7 +128,6 @@ static void prim_flakeRefToString(EvalState & state, const PosIdx pos, Value **
state.forceAttrs(*args[0], noPos, "while evaluating the argument passed to builtins.flakeRefToString");
fetchers::Attrs attrs;
for (const auto & attr : *args[0]->attrs()) {
state.forceValue(*attr.value, attr.pos);
auto t = attr.value->type();
if (t == nInt) {
auto intValue = attr.value->integer().value;
-2
View File
@@ -876,8 +876,6 @@ lockFlake(const Settings & settings, EvalState & state, const FlakeRef & topRef,
CanonPath((topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock"),
newLockFileS,
commitMessage);
flake.lockFilePath().invalidateCache();
}
/* Rewriting the lockfile changed the top-level
+8 -14
View File
@@ -121,23 +121,19 @@ std::pair<FlakeRef, std::string> parsePathFlakeRefWithFragment(
to 'baseDir'). If so, search upward to the root of the
repo (i.e. the directory containing .git). */
{
auto baseDirPath = *baseDir;
path = absPath(path, &baseDirPath, true).string();
}
path = absPath(path, baseDir->string(), true);
if (isFlake) {
if (!S_ISDIR(lstat(path).st_mode)) {
if (baseNameOf(path) == "flake.nix") {
// Be gentle with people who accidentally write `/foo/bar/flake.nix` instead of `/foo/bar`
auto parentPath = std::filesystem::path(path).parent_path().string();
warn(
"Path '%s' should point at the directory containing the 'flake.nix' file, not the file itself. "
"Pretending that you meant '%s'",
path,
parentPath);
path = parentPath;
dirOf(path));
path = dirOf(path);
} else {
throw BadURL("path '%s' is not a flake (because it's not a directory)", path);
}
@@ -161,7 +157,7 @@ std::pair<FlakeRef, std::string> parsePathFlakeRefWithFragment(
if (lstat(path).st_dev != device)
throw Error("unable to find a flake before encountering filesystem boundary at '%s'", path);
}
path = std::filesystem::path(path).parent_path().string();
path = dirOf(path);
}
if (!found)
throw BadURL("could not find a flake.nix file");
@@ -196,12 +192,12 @@ std::pair<FlakeRef, std::string> parsePathFlakeRefWithFragment(
}
subdir = std::string(baseNameOf(flakeRoot)) + (subdir.empty() ? "" : "/" + subdir);
flakeRoot = std::filesystem::path(flakeRoot).parent_path().string();
flakeRoot = dirOf(flakeRoot);
}
}
} else {
if (!preserveRelativePaths && !std::filesystem::path(path).is_absolute())
if (!preserveRelativePaths && !isAbsolute(path))
throw BadURL("flake reference '%s' is not an absolute path", url);
}
@@ -255,10 +251,8 @@ std::optional<std::pair<FlakeRef, std::string>> parseURLFlakeRef(
if (baseDir && (parsed.scheme == "path" || parsed.scheme == "git+file")) {
/* Here we know that the path must not contain encoded '/' or NUL bytes. */
auto path = renderUrlPathEnsureLegal(parsed.path);
if (!std::filesystem::path(path).is_absolute()) {
auto baseDirPath = *baseDir;
parsed.path = splitString<std::vector<std::string>>(absPath(path, &baseDirPath).string(), "/");
}
if (!isAbsolute(path))
parsed.path = splitString<std::vector<std::string>>(absPath(path, baseDir->string()), "/");
}
return fromParsedURL(fetchSettings, std::move(parsed), isFlake);
} catch (BadURL &) {
+15 -1
View File
@@ -13,6 +13,8 @@
namespace nix {
int handleExceptions(const std::string & programName, std::function<void()> fun);
/**
* Don't forget to call initPlugins() after settings are initialized!
* @param loadConfig Whether to load configuration from `nix.conf`, `NIX_CONFIG`, etc. May be disabled for unit tests.
@@ -89,7 +91,19 @@ extern volatile ::sig_atomic_t blockInt;
struct GCResults;
void printFreed(bool dryRun, const GCResults & results);
struct PrintFreed
{
bool show;
const GCResults & results;
PrintFreed(bool show, const GCResults & results)
: show(show)
, results(results)
{
}
~PrintFreed();
};
#ifndef _WIN32
/**
+37 -41
View File
@@ -1,9 +1,7 @@
#include "nix/store/globals.hh"
#include "nix/util/current-process.hh"
#include "nix/util/executable-path.hh"
#include "nix/main/shared.hh"
#include "nix/store/store-api.hh"
#include "nix/store/store-open.hh"
#include "nix/store/gc-store.hh"
#include "nix/main/loggers.hh"
#include "nix/main/progress-bar.hh"
@@ -19,10 +17,6 @@
#include <sys/stat.h>
#include <unistd.h>
#include <signal.h>
#ifndef _WIN32
# include <sys/resource.h>
#endif
#ifdef __linux__
# include <features.h>
#endif
@@ -122,26 +116,6 @@ std::string getArg(const std::string & opt, Strings::iterator & i, const Strings
static void sigHandler(int signo) {}
#endif
/**
* Increase the open file soft limit to the hard limit. On some
* platforms (macOS), the default soft limit is very low, but the hard
* limit is high. So let's just raise it the maximum permitted.
*/
void bumpFileLimit()
{
#ifndef _WIN32
struct rlimit limit;
if (getrlimit(RLIMIT_NOFILE, &limit) != 0)
return;
if (limit.rlim_cur < limit.rlim_max) {
limit.rlim_cur = limit.rlim_max;
// Ignore errors, this is best effort.
setrlimit(RLIMIT_NOFILE, &limit);
}
#endif
}
void initNix(bool loadConfig)
{
/* Turn on buffering for cerr. */
@@ -209,8 +183,6 @@ void initNix(bool loadConfig)
now. In particular, store objects should be readable by
everybody. */
umask(0022);
bumpFileLimit();
}
LegacyArgs::LegacyArgs(
@@ -237,13 +209,13 @@ LegacyArgs::LegacyArgs(
.longName = "keep-going",
.shortName = 'k',
.description = "Keep going after a build fails.",
.handler = {&(bool &) settings.getWorkerSettings().keepGoing, true},
.handler = {&(bool &) settings.keepGoing, true},
});
addFlag({
.longName = "fallback",
.description = "Build from source if substitution fails.",
.handler = {&(bool &) settings.getWorkerSettings().tryFallback, true},
.handler = {&(bool &) settings.tryFallback, true},
});
auto intSettingAlias =
@@ -280,7 +252,7 @@ LegacyArgs::LegacyArgs(
.longName = "store",
.description = "The URL of the Nix store to use.",
.labels = {"store-uri"},
.handler = {[](std::string s) { settings.storeUri = StoreReference::parse(s); }},
.handler = {&(std::string &) settings.storeUri},
});
}
@@ -332,16 +304,43 @@ void printVersion(const std::string & programName)
std::cout << "System type: " << settings.thisSystem << "\n";
std::cout << "Additional system types: " << concatStringsSep(", ", settings.extraPlatforms.get()) << "\n";
std::cout << "Features: " << concatStringsSep(", ", cfg) << "\n";
std::cout << "System configuration file: " << nixConfFile() << "\n";
std::cout << "User configuration files: "
<< os_string_to_string(ExecutablePath{.directories = nixUserConfFiles()}.render()) << "\n";
std::cout << "Store directory: " << resolveStoreConfig(StoreReference{settings.storeUri.get()})->storeDir
<< "\n";
std::cout << "System configuration file: " << (settings.nixConfDir / "nix.conf").string() << "\n";
std::cout << "User configuration files: " << concatStringsSep(":", settings.nixUserConfFiles) << "\n";
std::cout << "Store directory: " << settings.nixStore << "\n";
std::cout << "State directory: " << settings.nixStateDir << "\n";
}
throw Exit();
}
int handleExceptions(const std::string & programName, std::function<void()> fun)
{
ReceiveInterrupts receiveInterrupts; // FIXME: need better place for this
ErrorInfo::programName = baseNameOf(programName);
std::string error = ANSI_RED "error:" ANSI_NORMAL " ";
try {
fun();
} catch (Exit & e) {
return e.status;
} catch (UsageError & e) {
logError(e.info());
printError("Try '%1% --help' for more information.", programName);
return 1;
} catch (BaseError & e) {
logError(e.info());
return e.info().status;
} catch (std::bad_alloc & e) {
printError(error + "out of memory");
return 1;
} catch (std::exception & e) {
printError(error + e.what());
return 1;
}
return 0;
}
RunPager::RunPager()
{
if (!isatty(STDOUT_FILENO))
@@ -396,12 +395,9 @@ RunPager::~RunPager()
}
}
void printFreed(bool dryRun, const GCResults & results)
PrintFreed::~PrintFreed()
{
/* bytesFreed cannot be reliably computed without actually deleting store paths because of hardlinking. */
if (dryRun)
std::cout << fmt("%d store paths would be deleted\n", results.paths.size());
else
if (show)
std::cout << fmt("%d store paths deleted, %s freed\n", results.paths.size(), renderSize(results.bytesFreed));
}
+11 -37
View File
@@ -9,7 +9,6 @@
#include "nix/store/path.hh"
#include "nix/store/store-api.hh"
#include "nix/store/store-open.hh"
#include "nix/store/store-reference.hh"
#include "nix/store/build-result.hh"
#include "nix/store/local-fs-store.hh"
#include "nix/util/base-nix-32.hh"
@@ -48,14 +47,14 @@ Store * nix_store_open(nix_c_context * context, const char * uri, const char ***
if (uri_str.empty())
return new Store{nix::openStore()};
auto storeRef = nix::StoreReference::parse(uri_str);
if (!params)
return new Store{nix::openStore(uri_str)};
if (params) {
for (size_t i = 0; params[i] != nullptr; i++) {
storeRef.params[params[i][0]] = params[i][1];
}
nix::Store::Config::Params params_map;
for (size_t i = 0; params[i] != nullptr; i++) {
params_map[params[i][0]] = params[i][1];
}
return new Store{nix::openStore(std::move(storeRef))};
return new Store{nix::openStore(uri_str, params_map)};
}
NIXC_CATCH_ERRS_NULL
}
@@ -183,8 +182,10 @@ nix_err nix_store_realise(
assert(results.size() == 1);
// Check if any builds failed
for (auto & result : results)
result.tryThrowBuildError();
for (auto & result : results) {
if (auto * failureP = result.tryGetFailure())
failureP->rethrow();
}
if (callback) {
for (const auto & result : results) {
@@ -308,12 +309,7 @@ StorePath * nix_add_derivation(nix_c_context * context, Store * store, nix_deriv
if (context)
context->last_err_code = NIX_OK;
try {
/* Quite dubious that users would want this to silently suceed
without actually writing the derivation if this setting is
set, but it was that way already, so we are doing this for
back-compat for now. */
auto ret = nix::settings.readOnlyMode ? nix::computeStorePath(*store->ptr, derivation->drv)
: store->ptr->writeDerivation(derivation->drv, nix::NoRepair);
auto ret = nix::writeDerivation(*store->ptr, derivation->drv, nix::NoRepair);
return new StorePath{ret};
}
@@ -358,26 +354,4 @@ StorePath * nix_store_query_path_from_hash_part(nix_c_context * context, Store *
NIXC_CATCH_ERRS_NULL
}
nix_err nix_store_copy_path(
nix_c_context * context, Store * srcStore, Store * dstStore, const StorePath * path, bool repair, bool checkSigs)
{
if (context)
context->last_err_code = NIX_OK;
try {
if (srcStore == nullptr)
return nix_set_err_msg(context, NIX_ERR_UNKNOWN, "Source store is null");
if (dstStore == nullptr)
return nix_set_err_msg(context, NIX_ERR_UNKNOWN, "Destination store is null");
if (path == nullptr)
return nix_set_err_msg(context, NIX_ERR_UNKNOWN, "Store path is null");
auto repairFlag = repair ? nix::RepairFlag::Repair : nix::RepairFlag::NoRepair;
auto checkSigsFlag = checkSigs ? nix::CheckSigsFlag::CheckSigs : nix::CheckSigsFlag::NoCheckSigs;
nix::copyStorePath(*srcStore->ptr, *dstStore->ptr, path->path, repairFlag, checkSigsFlag);
}
NIXC_CATCH_ERRS
}
} // extern "C"
-13
View File
@@ -270,19 +270,6 @@ nix_derivation * nix_store_drv_from_store_path(nix_c_context * context, Store *
*/
StorePath * nix_store_query_path_from_hash_part(nix_c_context * context, Store * store, const char * hash);
/**
* @brief Copy a path from one store to another.
*
* @param[out] context Optional, stores error information
* @param[in] srcStore nix source store reference
* @param[in] dstStore nix destination store reference
* @param[in] path The path to copy
* @param[in] repair Whether to repair the path
* @param[in] checkSigs Whether to check path signatures are trusted before copying
*/
nix_err nix_store_copy_path(
nix_c_context * context, Store * srcStore, Store * dstStore, const StorePath * path, bool repair, bool checkSigs);
// cffi end
#ifdef __cplusplus
}
-144
View File
@@ -1,144 +0,0 @@
#include "nix/store/tests/https-store.hh"
#include <thread>
namespace nix::testing {
void TestHttpBinaryCacheStore::init()
{
BinaryCacheStore::init();
}
ref<TestHttpBinaryCacheStore> TestHttpBinaryCacheStoreConfig::openTestStore(ref<FileTransfer> fileTransfer) const
{
auto store = make_ref<TestHttpBinaryCacheStore>(
ref{// FIXME we shouldn't actually need a mutable config
std::const_pointer_cast<HttpBinaryCacheStore::Config>(shared_from_this())},
fileTransfer);
store->init();
return store;
}
void HttpsBinaryCacheStoreTest::openssl(Strings args)
{
runProgram("openssl", /*lookupPath=*/true, args);
}
void HttpsBinaryCacheStoreTest::SetUp()
{
LibStoreNetworkTest::SetUp();
#ifdef _WIN32
GTEST_SKIP() << "HTTPS store tests are not supported on Windows";
#endif
tmpDir = createTempDir();
cacheDir = tmpDir / "cache";
delTmpDir = std::make_unique<AutoDelete>(tmpDir);
localCacheStore =
make_ref<LocalBinaryCacheStoreConfig>("file", cacheDir.string(), LocalBinaryCacheStoreConfig::Params{})
->openStore();
caCert = tmpDir / "ca.crt";
caKey = tmpDir / "ca.key";
serverCert = tmpDir / "server.crt";
serverKey = tmpDir / "server.key";
clientCert = tmpDir / "client.crt";
clientKey = tmpDir / "client.key";
// clang-format off
openssl({"ecparam", "-genkey", "-name", "prime256v1", "-out", caKey.string()});
openssl({"req", "-new", "-x509", "-days", "1", "-key", caKey.string(), "-out", caCert.string(), "-subj", "/CN=TestCA"});
auto serverExtFile = tmpDir / "server.ext";
writeFile(serverExtFile, "subjectAltName=DNS:localhost,IP:127.0.0.1");
openssl({"ecparam", "-genkey", "-name", "prime256v1", "-out", serverKey.string()});
openssl({"req", "-new", "-key", serverKey.string(), "-out", (tmpDir / "server.csr").string(), "-subj", "/CN=localhost", "-addext", "subjectAltName=DNS:localhost,IP:127.0.0.1"});
openssl({"x509", "-req", "-in", (tmpDir / "server.csr").string(), "-CA", caCert.string(), "-CAkey", caKey.string(), "-CAcreateserial", "-out", serverCert.string(), "-days", "1", "-extfile", serverExtFile.string()});
openssl({"ecparam", "-genkey", "-name", "prime256v1", "-out", clientKey.string()});
openssl({"req", "-new", "-key", clientKey.string(), "-out", (tmpDir / "client.csr").string(), "-subj", "/CN=TestClient"});
openssl({"x509", "-req", "-in", (tmpDir / "client.csr").string(), "-CA", caCert.string(), "-CAkey", caKey.string(), "-CAcreateserial", "-out", clientCert.string(), "-days", "1"});
// clang-format on
#ifndef _WIN32 /* FIXME: Can't yet start processes on windows */
auto args = serverArgs();
serverPid = startProcess(
[&] {
if (chdir(cacheDir.c_str()) == -1)
_exit(1);
std::vector<char *> argv;
argv.push_back(const_cast<char *>("openssl"));
for (auto & a : args)
argv.push_back(const_cast<char *>(a.c_str()));
argv.push_back(nullptr);
execvp("openssl", argv.data());
_exit(1);
},
{.dieWithParent = true});
#endif
/* As an optimization, sleep for a bit to allow the server to come up to avoid retrying when connecting.
This won't make the tests fail, but does make them run faster. We don't need to overcomplicate by waiting
for the port explicitly - this is enough. */
std::this_thread::sleep_for(std::chrono::milliseconds(50));
/* Create custom FileTransferSettings with our test CA certificate.
This avoids mutating global settings. */
testFileTransferSettings = std::make_unique<FileTransferSettings>();
testFileTransferSettings->caFile = caCert;
testFileTransfer = makeFileTransfer(*testFileTransferSettings);
}
void HttpsBinaryCacheStoreTest::TearDown()
{
serverPid.kill();
delTmpDir.reset();
testFileTransferSettings.reset();
}
std::vector<std::string> HttpsBinaryCacheStoreTest::serverArgs()
{
return {
"s_server",
"-accept",
std::to_string(port),
"-cert",
serverCert.string(),
"-key",
serverKey.string(),
"-WWW", /* Serve from current directory. */
"-quiet",
};
}
std::vector<std::string> HttpsBinaryCacheStoreMtlsTest::serverArgs()
{
auto args = HttpsBinaryCacheStoreTest::serverArgs();
/* With the -Verify option the client must supply a certificate or an error occurs, which is not the
case with -verify. */
args.insert(args.end(), {"-CAfile", caCert.string(), "-Verify", "1", "-verify_return_error"});
return args;
}
ref<TestHttpBinaryCacheStoreConfig> HttpsBinaryCacheStoreTest::makeConfig()
{
auto res = make_ref<TestHttpBinaryCacheStoreConfig>(
ParsedURL{
.scheme = "https",
.authority =
ParsedURL::Authority{
.host = "localhost",
.port = port,
},
},
TestHttpBinaryCacheStoreConfig::Params{});
res->pathInfoCacheSize = 0; /* We don't want any caching in tests. */
return res;
}
ref<TestHttpBinaryCacheStore> HttpsBinaryCacheStoreTest::openStore(ref<TestHttpBinaryCacheStoreConfig> config)
{
return config->openTestStore(ref<FileTransfer>{testFileTransfer});
}
} // namespace nix::testing
@@ -1,99 +0,0 @@
#pragma once
///@file
#include <gtest/gtest.h>
#include <gmock/gmock.h>
#include "nix/store/tests/libstore-network.hh"
#include "nix/store/http-binary-cache-store.hh"
#include "nix/store/store-api.hh"
#include "nix/store/globals.hh"
#include "nix/store/local-binary-cache-store.hh"
#include "nix/util/file-system.hh"
#include "nix/util/processes.hh"
namespace nix::testing {
class TestHttpBinaryCacheStoreConfig;
/**
* Test shim for testing. We don't want to use the on-disk narinfo cache in unit
* tests.
*/
class TestHttpBinaryCacheStore : public HttpBinaryCacheStore
{
public:
TestHttpBinaryCacheStore(const TestHttpBinaryCacheStore &) = delete;
TestHttpBinaryCacheStore(TestHttpBinaryCacheStore &&) = delete;
TestHttpBinaryCacheStore & operator=(const TestHttpBinaryCacheStore &) = delete;
TestHttpBinaryCacheStore & operator=(TestHttpBinaryCacheStore &&) = delete;
TestHttpBinaryCacheStore(ref<HttpBinaryCacheStoreConfig> config, ref<FileTransfer> fileTransfer)
: Store{*config}
, BinaryCacheStore{*config}
, HttpBinaryCacheStore(config, fileTransfer)
{
diskCache = nullptr; /* Disable caching, we'll be creating a new binary cache for each test. */
}
void init() override;
};
class TestHttpBinaryCacheStoreConfig : public HttpBinaryCacheStoreConfig
{
public:
TestHttpBinaryCacheStoreConfig(ParsedURL url, const Store::Config::Params & params)
: StoreConfig(params)
, HttpBinaryCacheStoreConfig(url, params)
{
}
ref<TestHttpBinaryCacheStore> openTestStore(ref<FileTransfer> fileTransfer) const;
};
class HttpsBinaryCacheStoreTest : public virtual LibStoreNetworkTest
{
std::unique_ptr<AutoDelete> delTmpDir;
public:
static void SetUpTestSuite()
{
initLibStore(/*loadConfig=*/false);
}
protected:
std::filesystem::path tmpDir, cacheDir;
std::filesystem::path caCert, caKey, serverCert, serverKey;
std::filesystem::path clientCert, clientKey;
Pid serverPid;
uint16_t port = 8443;
std::shared_ptr<Store> localCacheStore;
/**
* Custom FileTransferSettings with the test CA certificate.
* This is used instead of modifying global settings.
*/
std::unique_ptr<FileTransferSettings> testFileTransferSettings;
/**
* FileTransfer instance using our test settings.
* Initialized in SetUp().
*/
std::shared_ptr<FileTransfer> testFileTransfer;
static void openssl(Strings args);
void SetUp() override;
void TearDown() override;
virtual std::vector<std::string> serverArgs();
ref<TestHttpBinaryCacheStoreConfig> makeConfig();
ref<TestHttpBinaryCacheStore> openStore(ref<TestHttpBinaryCacheStoreConfig> config);
};
class HttpsBinaryCacheStoreMtlsTest : public HttpsBinaryCacheStoreTest
{
protected:
std::vector<std::string> serverArgs() override;
};
} // namespace nix::testing
@@ -1,39 +0,0 @@
#pragma once
/// @file
#include <gtest/gtest.h>
namespace nix::testing {
/**
* Whether to run network tests. This is global so that the test harness can
* enable this by default if we can run tests in isolation.
*/
extern bool networkTestsAvailable;
/**
* Set up network tests and, if on linux, create a new network namespace for
* tests with a loopback interface. This is to avoid binding to ports in the
* host's namespace.
*/
void setupNetworkTests();
class LibStoreNetworkTest : public virtual ::testing::Test
{
protected:
void SetUp() override
{
if (networkTestsAvailable)
return;
static bool warned = false;
if (!warned) {
warned = true;
GTEST_SKIP()
<< "Network tests not enabled by default without user namespaces, use NIX_TEST_FORCE_NETWORK_TESTS=1 to override";
} else {
GTEST_SKIP();
}
}
};
} // namespace nix::testing
@@ -4,8 +4,6 @@ include_dirs = [ include_directories('../../..') ]
headers = files(
'derived-path.hh',
'https-store.hh',
'libstore-network.hh',
'libstore.hh',
'nix_api_store.hh',
'outputs-spec.hh',
@@ -51,7 +51,7 @@ protected:
// resolve any symlinks in i.e. on macOS /tmp -> /private/tmp
// because this is not allowed for a nix store.
auto tmpl =
nix::absPath(std::filesystem::path(nix::defaultTempDir()) / "tests_nix-store.XXXXXX", nullptr, true);
nix::absPath(std::filesystem::path(nix::defaultTempDir()) / "tests_nix-store.XXXXXX", std::nullopt, true);
nixDir = mkdtemp((char *) tmpl.c_str());
#endif
@@ -100,26 +100,26 @@ public:
writeProtoTest(STEM, VERSION, VALUE); \
}
#define VERSIONED_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE) \
VERSIONED_READ_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, (VERSION), VALUE) \
VERSIONED_WRITE_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, (VERSION), VALUE)
#define VERSIONED_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE) \
VERSIONED_READ_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE) \
VERSIONED_WRITE_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE)
#define VERSIONED_READ_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE) \
VERSIONED_READ_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, (VERSION), VALUE) \
TEST_F(FIXTURE, NAME##_json_read) \
{ \
readJsonTest(STEM, VALUE); \
#define VERSIONED_READ_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE) \
VERSIONED_READ_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE) \
TEST_F(FIXTURE, NAME##_json_read) \
{ \
readJsonTest(STEM, VALUE); \
}
#define VERSIONED_WRITE_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE) \
VERSIONED_WRITE_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, (VERSION), VALUE) \
TEST_F(FIXTURE, NAME##_json_write) \
{ \
writeJsonTest(STEM, VALUE); \
#define VERSIONED_WRITE_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE) \
VERSIONED_WRITE_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE) \
TEST_F(FIXTURE, NAME##_json_write) \
{ \
writeJsonTest(STEM, VALUE); \
}
#define VERSIONED_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE) \
VERSIONED_READ_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, (VERSION), VALUE) \
VERSIONED_WRITE_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, (VERSION), VALUE)
#define VERSIONED_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE) \
VERSIONED_READ_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE) \
VERSIONED_WRITE_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE)
} // namespace nix
@@ -1,60 +0,0 @@
#include "nix/store/tests/libstore-network.hh"
#include "nix/util/error.hh"
#include "nix/util/environment-variables.hh"
#ifdef __linux__
# include "nix/util/file-system.hh"
# include "nix/util/linux-namespaces.hh"
# include <sched.h>
# include <sys/ioctl.h>
# include <net/if.h>
# include <netinet/in.h>
#endif
namespace nix::testing {
bool networkTestsAvailable = false;
#ifdef __linux__
static void enterNetworkNamespace()
{
auto uid = ::getuid();
auto gid = ::getgid();
if (::unshare(CLONE_NEWUSER | CLONE_NEWNET) == -1)
throw SysError("setting up a private network namespace for tests");
std::filesystem::path procSelf = "/proc/self";
writeFile(procSelf / "setgroups", "deny");
writeFile(procSelf / "uid_map", fmt("%d %d 1", uid, uid));
writeFile(procSelf / "gid_map", fmt("%d %d 1", gid, gid));
AutoCloseFD fd(::socket(PF_INET, SOCK_DGRAM, IPPROTO_IP));
if (!fd)
throw SysError("cannot open IP socket for loopback interface");
struct ::ifreq ifr = {};
strcpy(ifr.ifr_name, "lo");
ifr.ifr_flags = IFF_UP | IFF_LOOPBACK | IFF_RUNNING;
if (::ioctl(fd.get(), SIOCSIFFLAGS, &ifr) == -1)
throw SysError("cannot set loopback interface flags");
}
#endif
void setupNetworkTests()
try {
networkTestsAvailable = getEnvOs(OS_STR("NIX_TEST_FORCE_NETWORK_TESTS")).has_value();
#ifdef __linux__
if (!networkTestsAvailable && userNamespacesSupported()) {
enterNetworkNamespace();
networkTestsAvailable = true;
}
#endif
} catch (SystemError & e) {
/* Ignore any set up errors. */
}
} // namespace nix::testing
-5
View File
@@ -28,15 +28,10 @@ subdir('nix-meson-build-support/subprojects')
rapidcheck = dependency('rapidcheck')
deps_public += rapidcheck
gtest = dependency('gtest')
deps_public += gtest
subdir('nix-meson-build-support/common')
sources = files(
'derived-path.cc',
'https-store.cc',
'libstore-network.cc',
'outputs-spec.cc',
'path.cc',
'test-main.cc',
-2
View File
@@ -7,7 +7,6 @@
nix-store-c,
rapidcheck,
gtest,
# Configuration Options
@@ -40,7 +39,6 @@ mkMesonLibrary (finalAttrs: {
nix-store
nix-store-c
rapidcheck
gtest
];
mesonFlags = [
+4 -4
View File
@@ -15,10 +15,10 @@ int testMainForBuidingPre(int argc, char ** argv)
}
// Disable build hook. We won't be testing remote builds in these unit tests. If we do, fix the above build hook.
settings.getWorkerSettings().buildHook = {};
settings.buildHook = {};
// No substituters, unless a test specifically requests.
settings.getWorkerSettings().substituters = {};
settings.substituters = {};
#ifdef __linux__ // should match the conditional around sandboxBuildDir declaration.
@@ -31,13 +31,13 @@ int testMainForBuidingPre(int argc, char ** argv)
// sandboxBuildDir = /build
// However, we have a rule that the store dir must not be inside the storeDir, so we need to pick a different
// sandboxBuildDir.
settings.getLocalSettings().sandboxBuildDir = "/test-build-dir-instead-of-usual-build-dir";
settings.sandboxBuildDir = "/test-build-dir-instead-of-usual-build-dir";
#endif
#ifdef __APPLE__
// Avoid this error, when already running in a sandbox:
// sandbox-exec: sandbox_apply: Operation not permitted
settings.getLocalSettings().sandboxMode = smDisabled;
settings.sandboxMode = smDisabled;
setEnv("_NIX_TEST_NO_SANDBOX", "1");
#endif
+6 -7
View File
@@ -1,7 +1,6 @@
#include <gtest/gtest.h>
#include "nix/store/build-result.hh"
#include "nix/util/tests/characterization.hh"
#include "nix/util/tests/json-characterization.hh"
namespace nix {
@@ -45,22 +44,22 @@ INSTANTIATE_TEST_SUITE_P(
std::pair{
"not-deterministic",
BuildResult{
.inner{BuildResult::Failure{{
.inner{BuildResult::Failure{
.status = BuildResult::Failure::NotDeterministic,
.msg = HintFmt("no idea why"),
.errorMsg = "no idea why",
.isNonDeterministic = false, // Note: This field is separate from the status
}}},
}},
.timesBuilt = 1,
},
},
std::pair{
"output-rejected",
BuildResult{
.inner{BuildResult::Failure{{
.inner{BuildResult::Failure{
.status = BuildResult::Failure::OutputRejected,
.msg = HintFmt("no idea why"),
.errorMsg = "no idea why",
.isNonDeterministic = false,
}}},
}},
.timesBuilt = 3,
.startTime = 30,
.stopTime = 50,
@@ -5,7 +5,6 @@
#include "nix/store/derivations.hh"
#include "nix/store/derived-path.hh"
#include "nix/store/derivation-options.hh"
#include "nix/store/globals.hh"
#include "nix/store/parsed-derivations.hh"
#include "nix/util/types.hh"
#include "nix/util/json-utils.hh"
@@ -193,7 +192,9 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_defaults)
EXPECT_EQ(options, advancedAttributes_defaults);
EXPECT_EQ(options.substitutesAllowed(settings.getWorkerSettings()), true);
EXPECT_EQ(options.canBuildLocally(*this->store, got), false);
EXPECT_EQ(options.willBuildLocally(*this->store, got), false);
EXPECT_EQ(options.substitutesAllowed(), true);
EXPECT_EQ(options.useUidRange(got), false);
});
};
@@ -241,7 +242,7 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes)
EXPECT_EQ(options, expected);
EXPECT_EQ(options.substitutesAllowed(settings.getWorkerSettings()), false);
EXPECT_EQ(options.substitutesAllowed(), false);
EXPECT_EQ(options.useUidRange(got), true);
});
};
@@ -333,7 +334,9 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_structuredAttrs_d
EXPECT_EQ(options, advancedAttributes_structuredAttrs_defaults);
EXPECT_EQ(options.substitutesAllowed(settings.getWorkerSettings()), true);
EXPECT_EQ(options.canBuildLocally(*this->store, got), false);
EXPECT_EQ(options.willBuildLocally(*this->store, got), false);
EXPECT_EQ(options.substitutesAllowed(), true);
EXPECT_EQ(options.useUidRange(got), false);
});
};
@@ -398,7 +401,9 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_structuredAttrs)
EXPECT_EQ(options, expected);
EXPECT_EQ(options.substitutesAllowed(settings.getWorkerSettings()), false);
EXPECT_EQ(options.canBuildLocally(*this->store, got), false);
EXPECT_EQ(options.willBuildLocally(*this->store, got), false);
EXPECT_EQ(options.substitutesAllowed(), false);
EXPECT_EQ(options.useUidRange(got), true);
});
};
+1 -1
View File
@@ -51,7 +51,7 @@ protected:
depDrv.fillInOutputPaths(*store);
// Write the dependency to the store
return store->writeDerivation(depDrv, NoRepair);
return writeDerivation(*store, depDrv, NoRepair);
}
public:
@@ -1,9 +1,6 @@
#include <gtest/gtest.h>
#include <regex>
#include "nix/store/http-binary-cache-store.hh"
#include "nix/store/tests/https-store.hh"
#include "nix/util/fs-sink.hh"
namespace nix {
@@ -37,74 +34,4 @@ TEST(HttpBinaryCacheStore, constructConfigWithParamsAndUrlWithParams)
EXPECT_EQ(config.getReference().params, params);
}
using testing::HttpsBinaryCacheStoreMtlsTest;
using testing::HttpsBinaryCacheStoreTest;
using namespace std::string_view_literals;
using namespace std::string_literals;
TEST_F(HttpsBinaryCacheStoreTest, queryPathInfo)
{
auto store = openStore(makeConfig());
StringSource dump{"test"sv};
auto path = localCacheStore->addToStoreFromDump(dump, "test-name", FileSerialisationMethod::Flat);
EXPECT_NO_THROW(store->queryPathInfo(path));
}
TEST_F(HttpsBinaryCacheStoreMtlsTest, queryPathInfo)
{
auto config = makeConfig();
config->tlsCert = clientCert;
config->tlsKey = clientKey;
auto store = openStore(config);
StringSource dump{"test"sv};
auto path = localCacheStore->addToStoreFromDump(dump, "test-name", FileSerialisationMethod::Flat);
EXPECT_NO_THROW(store->queryPathInfo(path));
}
TEST_F(HttpsBinaryCacheStoreMtlsTest, rejectsWithoutClientCert)
{
testFileTransferSettings->tries = 1;
EXPECT_THROW(openStore(makeConfig()), Error);
}
TEST_F(HttpsBinaryCacheStoreMtlsTest, rejectsWrongClientCert)
{
auto wrongKey = tmpDir / "wrong.key";
auto wrongCert = tmpDir / "wrong.crt";
// clang-format off
openssl({"ecparam", "-genkey", "-name", "prime256v1", "-out", wrongKey.string()});
openssl({"req", "-new", "-x509", "-days", "1", "-key", wrongKey.string(), "-out", wrongCert.string(), "-subj", "/CN=WrongClient"});
// clang-format on
auto config = makeConfig();
config->tlsCert = wrongCert;
config->tlsKey = wrongKey;
testFileTransferSettings->tries = 1;
EXPECT_THROW(openStore(config), Error);
}
TEST_F(HttpsBinaryCacheStoreMtlsTest, doesNotSendCertOnRedirectToDifferentAuthority)
{
StringSource dump{"test"sv};
auto path = localCacheStore->addToStoreFromDump(dump, "test-name", FileSerialisationMethod::Flat);
for (auto & entry : DirectoryIterator{cacheDir})
if (entry.path().extension() == ".narinfo") {
auto content = readFile(entry.path());
content = std::regex_replace(content, std::regex("URL: nar/"), fmt("URL: https://127.0.0.1:%d/nar/", port));
writeFile(entry.path(), content);
}
auto config = makeConfig();
config->tlsCert = clientCert;
config->tlsKey = clientKey;
testFileTransferSettings->tries = 1;
auto store = openStore(config);
auto info = store->queryPathInfo(path);
NullSink null;
EXPECT_THROW(store->narFromPath(path, null), Error);
}
} // namespace nix
+4
View File
@@ -13,6 +13,10 @@ using testing::Eq;
using testing::Field;
using testing::SizeIs;
namespace nix::fs {
using namespace std::filesystem;
}
using namespace nix;
TEST(machines, getMachinesWithEmptyBuilders)
-2
View File
@@ -1,7 +1,6 @@
#include <gtest/gtest.h>
#include "nix/store/tests/test-main.hh"
#include "nix/store/tests/libstore-network.hh"
using namespace nix;
@@ -11,7 +10,6 @@ int main(int argc, char ** argv)
if (res)
return res;
nix::testing::setupNetworkTests();
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}
+2 -4
View File
@@ -25,8 +25,7 @@ TEST(NarInfoDiskCacheImpl, create_and_read)
SQLiteStmt getIds;
{
auto cache = NarInfoDiskCache::getTest(
settings.getNarInfoDiskCacheSettings(), {.useWAL = settings.useSQLiteWAL}, dbPath.string());
auto cache = getTestNarInfoDiskCache(dbPath.string());
// Set up "background noise" and check that different caches receive different ids
{
@@ -75,8 +74,7 @@ TEST(NarInfoDiskCacheImpl, create_and_read)
{
// We can't clear the in-memory cache, so we use a new cache object. This is
// more realistic anyway.
auto cache2 = NarInfoDiskCache::getTest(
settings.getNarInfoDiskCacheSettings(), {.useWAL = settings.useSQLiteWAL}, dbPath.string());
auto cache2 = getTestNarInfoDiskCache(dbPath.string());
{
auto r = cache2->upToDateCacheExists("http://foo");
+6 -6
View File
@@ -299,7 +299,7 @@ public:
nix_api_store_test_base::SetUp();
nix::experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations");
nix::settings.getWorkerSettings().substituters = {};
nix::settings.substituters = {};
store = open_local_store();
@@ -354,7 +354,7 @@ TEST_F(nix_api_store_test_base, build_from_json)
{
// FIXME get rid of these
nix::experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations");
nix::settings.getWorkerSettings().substituters = {};
nix::settings.substituters = {};
auto * store = open_local_store();
@@ -401,7 +401,7 @@ TEST_F(nix_api_store_test_base, nix_store_realise_invalid_system)
{
// Test that nix_store_realise properly reports errors when the system is invalid
nix::experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations");
nix::settings.getWorkerSettings().substituters = {};
nix::settings.substituters = {};
auto * store = open_local_store();
@@ -446,7 +446,7 @@ TEST_F(nix_api_store_test_base, nix_store_realise_builder_fails)
{
// Test that nix_store_realise properly reports errors when the builder fails
nix::experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations");
nix::settings.getWorkerSettings().substituters = {};
nix::settings.substituters = {};
auto * store = open_local_store();
@@ -491,7 +491,7 @@ TEST_F(nix_api_store_test_base, nix_store_realise_builder_no_output)
{
// Test that nix_store_realise properly reports errors when builder succeeds but produces no output
nix::experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations");
nix::settings.getWorkerSettings().substituters = {};
nix::settings.substituters = {};
auto * store = open_local_store();
@@ -687,7 +687,7 @@ TEST_F(NixApiStoreTestWithRealisedPath, nix_store_realise_output_ordering)
// This test uses a CA derivation with 10 outputs in randomized input order
// to verify that the callback order is deterministic and alphabetical.
nix::experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations");
nix::settings.getWorkerSettings().substituters = {};
nix::settings.substituters = {};
auto * store = open_local_store();
+1 -5
View File
@@ -9,7 +9,6 @@
nix-store-c,
nix-store-test-support,
sqlite,
openssl,
rapidcheck,
gtest,
@@ -76,10 +75,7 @@ mkMesonExecutable (finalAttrs: {
runCommand "${finalAttrs.pname}-run"
{
meta.broken = !stdenv.hostPlatform.emulatorAvailable buildPackages;
nativeBuildInputs = [
writableTmpDirAsHomeHook
openssl
];
buildInputs = [ writableTmpDirAsHomeHook ];
}
(
''
+15 -237
View File
@@ -104,33 +104,6 @@ INSTANTIATE_TEST_SUITE_P(
},
},
"with_absolute_endpoint_uri",
},
ParsedS3URLTestCase{
"s3://bucket/key?addressing-style=virtual",
{
.bucket = "bucket",
.key = {"key"},
.addressingStyle = S3AddressingStyle::Virtual,
},
"with_addressing_style_virtual",
},
ParsedS3URLTestCase{
"s3://bucket/key?addressing-style=path",
{
.bucket = "bucket",
.key = {"key"},
.addressingStyle = S3AddressingStyle::Path,
},
"with_addressing_style_path",
},
ParsedS3URLTestCase{
"s3://bucket/key?addressing-style=auto",
{
.bucket = "bucket",
.key = {"key"},
.addressingStyle = S3AddressingStyle::Auto,
},
"with_addressing_style_auto",
}),
[](const ::testing::TestParamInfo<ParsedS3URLTestCase> & info) { return info.param.description; });
@@ -165,26 +138,6 @@ INSTANTIATE_TEST_SUITE_P(
InvalidS3URLTestCase{"s3://bucket", "error: URI has a missing or invalid key", "missing_key"}),
[](const ::testing::TestParamInfo<InvalidS3URLTestCase> & info) { return info.param.description; });
TEST(ParsedS3URLTest, invalidAddressingStyleThrows)
{
ASSERT_THROW(ParsedS3URL::parse(parseURL("s3://bucket/key?addressing-style=bogus")), InvalidS3AddressingStyle);
}
TEST(ParsedS3URLTest, virtualStyleWithAuthoritylessEndpointThrows)
{
ParsedS3URL input{
.bucket = "bucket",
.key = {"key"},
.addressingStyle = S3AddressingStyle::Virtual,
.endpoint =
ParsedURL{
.scheme = "file",
.path = {"", "some", "path"},
},
};
ASSERT_THROW(input.toHttpsUrl(), nix::Error);
}
// =============================================================================
// S3 URL to HTTPS Conversion Tests
// =============================================================================
@@ -213,7 +166,6 @@ INSTANTIATE_TEST_SUITE_P(
S3ToHttpsConversion,
S3ToHttpsConversionTest,
::testing::Values(
// Default (auto) addressing style: virtual-hosted for standard AWS endpoints
S3ToHttpsConversionTestCase{
ParsedS3URL{
.bucket = "my-bucket",
@@ -221,10 +173,10 @@ INSTANTIATE_TEST_SUITE_P(
},
ParsedURL{
.scheme = "https",
.authority = ParsedURL::Authority{.host = "my-bucket.s3.us-east-1.amazonaws.com"},
.path = {"", "my-key.txt"},
.authority = ParsedURL::Authority{.host = "s3.us-east-1.amazonaws.com"},
.path = {"", "my-bucket", "my-key.txt"},
},
"https://my-bucket.s3.us-east-1.amazonaws.com/my-key.txt",
"https://s3.us-east-1.amazonaws.com/my-bucket/my-key.txt",
"basic_s3_default_region",
},
S3ToHttpsConversionTestCase{
@@ -235,13 +187,12 @@ INSTANTIATE_TEST_SUITE_P(
},
ParsedURL{
.scheme = "https",
.authority = ParsedURL::Authority{.host = "prod-cache.s3.eu-west-1.amazonaws.com"},
.path = {"", "nix", "store", "abc123.nar.xz"},
.authority = ParsedURL::Authority{.host = "s3.eu-west-1.amazonaws.com"},
.path = {"", "prod-cache", "nix", "store", "abc123.nar.xz"},
},
"https://prod-cache.s3.eu-west-1.amazonaws.com/nix/store/abc123.nar.xz",
"https://s3.eu-west-1.amazonaws.com/prod-cache/nix/store/abc123.nar.xz",
"with_eu_west_1_region",
},
// Custom endpoint authority: path-style by default
S3ToHttpsConversionTestCase{
ParsedS3URL{
.bucket = "bucket",
@@ -257,7 +208,6 @@ INSTANTIATE_TEST_SUITE_P(
"http://custom.s3.com/bucket/key",
"custom_endpoint_authority",
},
// Custom endpoint URL: path-style by default
S3ToHttpsConversionTestCase{
ParsedS3URL{
.bucket = "bucket",
@@ -286,10 +236,10 @@ INSTANTIATE_TEST_SUITE_P(
},
ParsedURL{
.scheme = "https",
.authority = ParsedURL::Authority{.host = "bucket.s3.ap-southeast-2.amazonaws.com"},
.path = {"", "path", "to", "file.txt"},
.authority = ParsedURL::Authority{.host = "s3.ap-southeast-2.amazonaws.com"},
.path = {"", "bucket", "path", "to", "file.txt"},
},
"https://bucket.s3.ap-southeast-2.amazonaws.com/path/to/file.txt",
"https://s3.ap-southeast-2.amazonaws.com/bucket/path/to/file.txt",
"complex_path_and_region",
},
S3ToHttpsConversionTestCase{
@@ -300,11 +250,11 @@ INSTANTIATE_TEST_SUITE_P(
},
ParsedURL{
.scheme = "https",
.authority = ParsedURL::Authority{.host = "my-bucket.s3.us-east-1.amazonaws.com"},
.path = {"", "my-key.txt"},
.authority = ParsedURL::Authority{.host = "s3.us-east-1.amazonaws.com"},
.path = {"", "my-bucket", "my-key.txt"},
.query = {{"versionId", "abc123xyz"}},
},
"https://my-bucket.s3.us-east-1.amazonaws.com/my-key.txt?versionId=abc123xyz",
"https://s3.us-east-1.amazonaws.com/my-bucket/my-key.txt?versionId=abc123xyz",
"with_versionId",
},
S3ToHttpsConversionTestCase{
@@ -316,185 +266,13 @@ INSTANTIATE_TEST_SUITE_P(
},
ParsedURL{
.scheme = "https",
.authority = ParsedURL::Authority{.host = "versioned-bucket.s3.eu-west-1.amazonaws.com"},
.path = {"", "path", "to", "object"},
.authority = ParsedURL::Authority{.host = "s3.eu-west-1.amazonaws.com"},
.path = {"", "versioned-bucket", "path", "to", "object"},
.query = {{"versionId", "version456"}},
},
"https://versioned-bucket.s3.eu-west-1.amazonaws.com/path/to/object?versionId=version456",
"https://s3.eu-west-1.amazonaws.com/versioned-bucket/path/to/object?versionId=version456",
"with_region_and_versionId",
},
// Explicit addressing-style=path forces path-style on standard AWS endpoints
S3ToHttpsConversionTestCase{
ParsedS3URL{
.bucket = "my-bucket",
.key = {"my-key.txt"},
.region = "us-west-2",
.addressingStyle = S3AddressingStyle::Path,
},
ParsedURL{
.scheme = "https",
.authority = ParsedURL::Authority{.host = "s3.us-west-2.amazonaws.com"},
.path = {"", "my-bucket", "my-key.txt"},
},
"https://s3.us-west-2.amazonaws.com/my-bucket/my-key.txt",
"explicit_path_style",
},
// Explicit addressing-style=virtual forces virtual-hosted-style on custom endpoints
S3ToHttpsConversionTestCase{
ParsedS3URL{
.bucket = "bucket",
.key = {"key"},
.scheme = "http",
.addressingStyle = S3AddressingStyle::Virtual,
.endpoint = ParsedURL::Authority{.host = "custom.s3.com"},
},
ParsedURL{
.scheme = "http",
.authority = ParsedURL::Authority{.host = "bucket.custom.s3.com"},
.path = {"", "key"},
},
"http://bucket.custom.s3.com/key",
"explicit_virtual_style_custom_endpoint",
},
// Explicit addressing-style=virtual with full endpoint URL
S3ToHttpsConversionTestCase{
ParsedS3URL{
.bucket = "bucket",
.key = {"key"},
.addressingStyle = S3AddressingStyle::Virtual,
.endpoint =
ParsedURL{
.scheme = "http",
.authority = ParsedURL::Authority{.host = "server", .port = 9000},
.path = {""},
},
},
ParsedURL{
.scheme = "http",
.authority = ParsedURL::Authority{.host = "bucket.server", .port = 9000},
.path = {"", "key"},
},
"http://bucket.server:9000/key",
"explicit_virtual_style_full_endpoint_url",
},
// Dotted bucket names work normally with explicit path-style
S3ToHttpsConversionTestCase{
ParsedS3URL{
.bucket = "my.bucket",
.key = {"key"},
.addressingStyle = S3AddressingStyle::Path,
},
ParsedURL{
.scheme = "https",
.authority = ParsedURL::Authority{.host = "s3.us-east-1.amazonaws.com"},
.path = {"", "my.bucket", "key"},
},
"https://s3.us-east-1.amazonaws.com/my.bucket/key",
"dotted_bucket_with_path_style",
},
// Dotted bucket names fall back to path-style with auto on standard AWS endpoints
S3ToHttpsConversionTestCase{
ParsedS3URL{
.bucket = "my.bucket.name",
.key = {"key"},
},
ParsedURL{
.scheme = "https",
.authority = ParsedURL::Authority{.host = "s3.us-east-1.amazonaws.com"},
.path = {"", "my.bucket.name", "key"},
},
"https://s3.us-east-1.amazonaws.com/my.bucket.name/key",
"dotted_bucket_with_auto_style_on_aws",
},
// Dotted bucket names work with auto style on custom endpoints (auto = path-style)
S3ToHttpsConversionTestCase{
ParsedS3URL{
.bucket = "my.bucket",
.key = {"key"},
.endpoint = ParsedURL::Authority{.host = "minio.local"},
},
ParsedURL{
.scheme = "https",
.authority = ParsedURL::Authority{.host = "minio.local"},
.path = {"", "my.bucket", "key"},
},
"https://minio.local/my.bucket/key",
"dotted_bucket_with_auto_style_custom_endpoint",
}),
[](const ::testing::TestParamInfo<S3ToHttpsConversionTestCase> & info) { return info.param.description; });
// =============================================================================
// S3 URL to HTTPS Conversion Error Tests
// =============================================================================
struct S3ToHttpsConversionErrorTestCase
{
ParsedS3URL input;
std::string description;
};
class S3ToHttpsConversionErrorTest : public ::testing::WithParamInterface<S3ToHttpsConversionErrorTestCase>,
public ::testing::Test
{};
TEST_P(S3ToHttpsConversionErrorTest, ThrowsOnConversion)
{
auto & [input, description] = GetParam();
ASSERT_THROW(input.toHttpsUrl(), nix::Error);
}
INSTANTIATE_TEST_SUITE_P(
S3ToHttpsConversionErrors,
S3ToHttpsConversionErrorTest,
::testing::Values(
S3ToHttpsConversionErrorTestCase{
ParsedS3URL{
.bucket = "bucket",
.key = {"key"},
.addressingStyle = S3AddressingStyle::Virtual,
.endpoint = ParsedURL::Authority{.host = ""},
},
"virtual_style_with_empty_host_authority",
},
S3ToHttpsConversionErrorTestCase{
ParsedS3URL{
.bucket = "my.bucket",
.key = {"key"},
.addressingStyle = S3AddressingStyle::Virtual,
},
"dotted_bucket_with_explicit_virtual_style",
},
S3ToHttpsConversionErrorTestCase{
ParsedS3URL{
.bucket = "my.bucket.name",
.key = {"key"},
.addressingStyle = S3AddressingStyle::Virtual,
},
"dotted_bucket_with_explicit_virtual_style_multi_dot",
},
S3ToHttpsConversionErrorTestCase{
ParsedS3URL{
.bucket = "my.bucket",
.key = {"key"},
.addressingStyle = S3AddressingStyle::Virtual,
.endpoint = ParsedURL::Authority{.host = "minio.local"},
},
"dotted_bucket_with_explicit_virtual_style_custom_authority",
},
S3ToHttpsConversionErrorTestCase{
ParsedS3URL{
.bucket = "my.bucket",
.key = {"key"},
.addressingStyle = S3AddressingStyle::Virtual,
.endpoint =
ParsedURL{
.scheme = "http",
.authority = ParsedURL::Authority{.host = "minio.local", .port = 9000},
.path = {""},
},
},
"dotted_bucket_with_explicit_virtual_style_full_endpoint_url",
}),
[](const ::testing::TestParamInfo<S3ToHttpsConversionErrorTestCase> & info) { return info.param.description; });
} // namespace nix
+58 -102
View File
@@ -25,10 +25,7 @@ struct ServeProtoTest : VersionedProtoTest<ServeProto, serveProtoDir>
* For serializers that don't care about the minimum version, we
* used the oldest one: 2.5.
*/
ServeProto::Version defaultVersion = {
.major = 2,
.minor = 5,
};
ServeProto::Version defaultVersion = 2 << 8 | 5;
};
VERSIONED_CHARACTERIZATION_TEST(
@@ -147,89 +144,66 @@ VERSIONED_READ_CHARACTERIZATION_TEST(
},
}))
VERSIONED_CHARACTERIZATION_TEST(ServeProtoTest, buildResult_2_2, "build-result-2.2", 2 << 8 | 2, ({
using namespace std::literals::chrono_literals;
std::tuple<BuildResult, BuildResult, BuildResult> t{
BuildResult{.inner{BuildResult::Failure{
.status = BuildResult::Failure::OutputRejected,
.errorMsg = "no idea why",
}}},
BuildResult{.inner{BuildResult::Failure{
.status = BuildResult::Failure::NotDeterministic,
.errorMsg = "no idea why",
}}},
BuildResult{.inner{BuildResult::Success{
.status = BuildResult::Success::Built,
}}},
};
t;
}))
VERSIONED_CHARACTERIZATION_TEST(ServeProtoTest, buildResult_2_3, "build-result-2.3", 2 << 8 | 3, ({
using namespace std::literals::chrono_literals;
std::tuple<BuildResult, BuildResult, BuildResult> t{
BuildResult{.inner{BuildResult::Failure{
.status = BuildResult::Failure::OutputRejected,
.errorMsg = "no idea why",
}}},
BuildResult{
.inner{BuildResult::Failure{
.status = BuildResult::Failure::NotDeterministic,
.errorMsg = "no idea why",
.isNonDeterministic = true,
}},
.timesBuilt = 3,
.startTime = 30,
.stopTime = 50,
},
BuildResult{
.inner{BuildResult::Success{
.status = BuildResult::Success::Built,
}},
.startTime = 30,
.stopTime = 50,
},
};
t;
}))
VERSIONED_CHARACTERIZATION_TEST(
ServeProtoTest,
buildResult_2_2,
"build-result-2.2",
(ServeProto::Version{
.major = 2,
.minor = 2,
}),
({
ServeProtoTest, buildResult_2_6, "build-result-2.6", 2 << 8 | 6, ({
using namespace std::literals::chrono_literals;
std::tuple<BuildResult, BuildResult, BuildResult> t{
BuildResult{.inner{BuildResult::Failure{{
BuildResult{.inner{BuildResult::Failure{
.status = BuildResult::Failure::OutputRejected,
.msg = HintFmt("no idea why"),
}}}},
BuildResult{.inner{BuildResult::Failure{{
.status = BuildResult::Failure::NotDeterministic,
.msg = HintFmt("no idea why"),
}}}},
BuildResult{.inner{BuildResult::Success{
.status = BuildResult::Success::Built,
.errorMsg = "no idea why",
}}},
};
t;
}))
VERSIONED_CHARACTERIZATION_TEST(
ServeProtoTest,
buildResult_2_3,
"build-result-2.3",
(ServeProto::Version{
.major = 2,
.minor = 3,
}),
({
using namespace std::literals::chrono_literals;
std::tuple<BuildResult, BuildResult, BuildResult> t{
BuildResult{.inner{BuildResult::Failure{{
.status = BuildResult::Failure::OutputRejected,
.msg = HintFmt("no idea why"),
}}}},
BuildResult{
.inner{BuildResult::Failure{{
.inner{BuildResult::Failure{
.status = BuildResult::Failure::NotDeterministic,
.msg = HintFmt("no idea why"),
.errorMsg = "no idea why",
.isNonDeterministic = true,
}}},
.timesBuilt = 3,
.startTime = 30,
.stopTime = 50,
},
BuildResult{
.inner{BuildResult::Success{
.status = BuildResult::Success::Built,
}},
.startTime = 30,
.stopTime = 50,
},
};
t;
}))
VERSIONED_CHARACTERIZATION_TEST(
ServeProtoTest,
buildResult_2_6,
"build-result-2.6",
(ServeProto::Version{
.major = 2,
.minor = 6,
}),
({
using namespace std::literals::chrono_literals;
std::tuple<BuildResult, BuildResult, BuildResult> t{
BuildResult{.inner{BuildResult::Failure{{
.status = BuildResult::Failure::OutputRejected,
.msg = HintFmt("no idea why"),
}}}},
BuildResult{
.inner{BuildResult::Failure{{
.status = BuildResult::Failure::NotDeterministic,
.msg = HintFmt("no idea why"),
.isNonDeterministic = true,
}}},
.timesBuilt = 3,
.startTime = 30,
.stopTime = 50,
@@ -286,10 +260,7 @@ VERSIONED_CHARACTERIZATION_TEST(
ServeProtoTest,
unkeyedValidPathInfo_2_3,
"unkeyed-valid-path-info-2.3",
(ServeProto::Version{
.major = 2,
.minor = 3,
}),
2 << 8 | 3,
(std::tuple<UnkeyedValidPathInfo, UnkeyedValidPathInfo>{
({
UnkeyedValidPathInfo info{std::string{defaultStoreDir}, Hash::dummy};
@@ -315,10 +286,7 @@ VERSIONED_CHARACTERIZATION_TEST(
ServeProtoTest,
unkeyedValidPathInfo_2_4,
"unkeyed-valid-path-info-2.4",
(ServeProto::Version{
.major = 2,
.minor = 4,
}),
2 << 8 | 4,
(std::tuple<UnkeyedValidPathInfo, UnkeyedValidPathInfo>{
({
UnkeyedValidPathInfo info{
@@ -372,10 +340,7 @@ VERSIONED_CHARACTERIZATION_TEST_NO_JSON(
ServeProtoTest,
build_options_2_1,
"build-options-2.1",
(ServeProto::Version{
.major = 2,
.minor = 1,
}),
2 << 8 | 1,
(ServeProto::BuildOptions{
.maxSilentTime = 5,
.buildTimeout = 6,
@@ -385,10 +350,7 @@ VERSIONED_CHARACTERIZATION_TEST_NO_JSON(
ServeProtoTest,
build_options_2_2,
"build-options-2.2",
(ServeProto::Version{
.major = 2,
.minor = 2,
}),
2 << 8 | 2,
(ServeProto::BuildOptions{
.maxSilentTime = 5,
.buildTimeout = 6,
@@ -399,10 +361,7 @@ VERSIONED_CHARACTERIZATION_TEST_NO_JSON(
ServeProtoTest,
build_options_2_3,
"build-options-2.3",
(ServeProto::Version{
.major = 2,
.minor = 3,
}),
2 << 8 | 3,
(ServeProto::BuildOptions{
.maxSilentTime = 5,
.buildTimeout = 6,
@@ -415,10 +374,7 @@ VERSIONED_CHARACTERIZATION_TEST_NO_JSON(
ServeProtoTest,
build_options_2_7,
"build-options-2.7",
(ServeProto::Version{
.major = 2,
.minor = 7,
}),
2 << 8 | 7,
(ServeProto::BuildOptions{
.maxSilentTime = 5,
.buildTimeout = 6,
+99 -296
View File
@@ -15,80 +15,6 @@
namespace nix {
TEST(WorkerProtoVersionNumber, ordering)
{
using Number = WorkerProto::Version::Number;
EXPECT_LT((Number{1, 10}), (Number{1, 20}));
EXPECT_GT((Number{1, 30}), (Number{1, 20}));
EXPECT_EQ((Number{1, 10}), (Number{1, 10}));
EXPECT_LT((Number{0, 255}), (Number{1, 0}));
}
TEST(WorkerProtoVersion, partialOrderingSameFeatures)
{
using V = WorkerProto::Version;
V v1{.number = {1, 20}, .features = {"a", "b"}};
V v2{.number = {1, 30}, .features = {"a", "b"}};
EXPECT_TRUE(v1 < v2);
EXPECT_TRUE(v2 > v1);
EXPECT_TRUE(v1 <= v2);
EXPECT_TRUE(v2 >= v1);
EXPECT_FALSE(v1 == v2);
}
TEST(WorkerProtoVersion, partialOrderingSubsetFeatures)
{
using V = WorkerProto::Version;
V fewer{.number = {1, 30}, .features = {"a"}};
V more{.number = {1, 30}, .features = {"a", "b"}};
// fewer <= more: JUST the features are a subset
EXPECT_TRUE(fewer < more);
EXPECT_TRUE(fewer <= more);
EXPECT_FALSE(fewer > more);
EXPECT_TRUE(fewer != more);
}
TEST(WorkerProtoVersion, partialOrderingUnordered)
{
using V = WorkerProto::Version;
// Same number but incomparable features
V v1{.number = {1, 20}, .features = {"a", "c"}};
V v2{.number = {1, 20}, .features = {"a", "b"}};
EXPECT_FALSE(v1 < v2);
EXPECT_FALSE(v1 > v2);
EXPECT_FALSE(v1 <= v2);
EXPECT_FALSE(v1 >= v2);
EXPECT_FALSE(v1 == v2);
EXPECT_TRUE(v1 != v2);
}
TEST(WorkerProtoVersion, partialOrderingHigherNumberFewerFeatures)
{
using V = WorkerProto::Version;
// Higher number but fewer features — unordered
V v1{.number = {1, 30}, .features = {"a"}};
V v2{.number = {1, 20}, .features = {"a", "b"}};
EXPECT_FALSE(v1 < v2);
EXPECT_FALSE(v1 > v2);
EXPECT_FALSE(v1 == v2);
}
TEST(WorkerProtoVersion, partialOrderingEmptyFeatures)
{
using V = WorkerProto::Version;
V empty{.number = {1, 20}, .features = {}};
V some{.number = {1, 30}, .features = {"a"}};
// empty features is a subset of everything
EXPECT_TRUE(empty < some);
EXPECT_TRUE(empty <= some);
EXPECT_TRUE(empty != some);
}
const char workerProtoDir[] = "worker-protocol";
static constexpr std::string_view defaultStoreDir = "/nix/store";
@@ -99,13 +25,7 @@ struct WorkerProtoTest : VersionedProtoTest<WorkerProto, workerProtoDir>
* For serializers that don't care about the minimum version, we
* used the oldest one: 1.10.
*/
WorkerProto::Version defaultVersion = {
.number =
{
.major = 1,
.minor = 10,
},
};
WorkerProto::Version defaultVersion = 1 << 8 | 10;
};
VERSIONED_CHARACTERIZATION_TEST(
@@ -159,13 +79,7 @@ VERSIONED_CHARACTERIZATION_TEST(
WorkerProtoTest,
derivedPath_1_29,
"derived-path-1.29",
(WorkerProto::Version{
.number =
{
.major = 1,
.minor = 29,
},
}),
1 << 8 | 29,
(std::tuple<DerivedPath, DerivedPath, DerivedPath>{
DerivedPath::Opaque{
.path = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
@@ -190,13 +104,7 @@ VERSIONED_CHARACTERIZATION_TEST(
WorkerProtoTest,
derivedPath_1_30,
"derived-path-1.30",
(WorkerProto::Version{
.number =
{
.major = 1,
.minor = 30,
},
}),
1 << 8 | 30,
(std::tuple<DerivedPath, DerivedPath, DerivedPath, DerivedPath>{
DerivedPath::Opaque{
.path = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
@@ -289,57 +197,36 @@ VERSIONED_READ_CHARACTERIZATION_TEST(
},
}))
VERSIONED_CHARACTERIZATION_TEST(
WorkerProtoTest,
buildResult_1_27,
"build-result-1.27",
(WorkerProto::Version{
.number =
{
.major = 1,
.minor = 27,
},
}),
({
using namespace std::literals::chrono_literals;
std::tuple<BuildResult, BuildResult, BuildResult> t{
BuildResult{.inner{BuildResult::Failure{{
.status = BuildResult::Failure::OutputRejected,
.msg = HintFmt("no idea why"),
}}}},
BuildResult{.inner{BuildResult::Failure{{
.status = BuildResult::Failure::NotDeterministic,
.msg = HintFmt("no idea why"),
}}}},
BuildResult{.inner{BuildResult::Success{
.status = BuildResult::Success::Built,
}}},
};
t;
}))
VERSIONED_CHARACTERIZATION_TEST(WorkerProtoTest, buildResult_1_27, "build-result-1.27", 1 << 8 | 27, ({
using namespace std::literals::chrono_literals;
std::tuple<BuildResult, BuildResult, BuildResult> t{
BuildResult{.inner{BuildResult::Failure{
.status = BuildResult::Failure::OutputRejected,
.errorMsg = "no idea why",
}}},
BuildResult{.inner{BuildResult::Failure{
.status = BuildResult::Failure::NotDeterministic,
.errorMsg = "no idea why",
}}},
BuildResult{.inner{BuildResult::Success{
.status = BuildResult::Success::Built,
}}},
};
t;
}))
VERSIONED_CHARACTERIZATION_TEST(
WorkerProtoTest,
buildResult_1_28,
"build-result-1.28",
(WorkerProto::Version{
.number =
{
.major = 1,
.minor = 28,
},
}),
({
WorkerProtoTest, buildResult_1_28, "build-result-1.28", 1 << 8 | 28, ({
using namespace std::literals::chrono_literals;
std::tuple<BuildResult, BuildResult, BuildResult> t{
BuildResult{.inner{BuildResult::Failure{{
BuildResult{.inner{BuildResult::Failure{
.status = BuildResult::Failure::OutputRejected,
.msg = HintFmt("no idea why"),
}}}},
BuildResult{.inner{BuildResult::Failure{{
.errorMsg = "no idea why",
}}},
BuildResult{.inner{BuildResult::Failure{
.status = BuildResult::Failure::NotDeterministic,
.msg = HintFmt("no idea why"),
}}}},
.errorMsg = "no idea why",
}}},
BuildResult{.inner{BuildResult::Success{
.status = BuildResult::Success::Built,
.builtOutputs =
@@ -375,29 +262,19 @@ VERSIONED_CHARACTERIZATION_TEST(
}))
VERSIONED_CHARACTERIZATION_TEST(
WorkerProtoTest,
buildResult_1_29,
"build-result-1.29",
(WorkerProto::Version{
.number =
{
.major = 1,
.minor = 29,
},
}),
({
WorkerProtoTest, buildResult_1_29, "build-result-1.29", 1 << 8 | 29, ({
using namespace std::literals::chrono_literals;
std::tuple<BuildResult, BuildResult, BuildResult> t{
BuildResult{.inner{BuildResult::Failure{{
BuildResult{.inner{BuildResult::Failure{
.status = BuildResult::Failure::OutputRejected,
.msg = HintFmt("no idea why"),
}}}},
.errorMsg = "no idea why",
}}},
BuildResult{
.inner{BuildResult::Failure{{
.inner{BuildResult::Failure{
.status = BuildResult::Failure::NotDeterministic,
.msg = HintFmt("no idea why"),
.errorMsg = "no idea why",
.isNonDeterministic = true,
}}},
}},
.timesBuilt = 3,
.startTime = 30,
.stopTime = 50,
@@ -444,29 +321,19 @@ VERSIONED_CHARACTERIZATION_TEST(
}))
VERSIONED_CHARACTERIZATION_TEST(
WorkerProtoTest,
buildResult_1_37,
"build-result-1.37",
(WorkerProto::Version{
.number =
{
.major = 1,
.minor = 37,
},
}),
({
WorkerProtoTest, buildResult_1_37, "build-result-1.37", 1 << 8 | 37, ({
using namespace std::literals::chrono_literals;
std::tuple<BuildResult, BuildResult, BuildResult> t{
BuildResult{.inner{BuildResult::Failure{{
BuildResult{.inner{BuildResult::Failure{
.status = BuildResult::Failure::OutputRejected,
.msg = HintFmt("no idea why"),
}}}},
.errorMsg = "no idea why",
}}},
BuildResult{
.inner{BuildResult::Failure{{
.inner{BuildResult::Failure{
.status = BuildResult::Failure::NotDeterministic,
.msg = HintFmt("no idea why"),
.errorMsg = "no idea why",
.isNonDeterministic = true,
}}},
}},
.timesBuilt = 3,
.startTime = 30,
.stopTime = 50,
@@ -514,65 +381,48 @@ VERSIONED_CHARACTERIZATION_TEST(
t;
}))
VERSIONED_CHARACTERIZATION_TEST(
WorkerProtoTest,
keyedBuildResult_1_29,
"keyed-build-result-1.29",
(WorkerProto::Version{
.number =
{
.major = 1,
.minor = 29,
},
}),
({
using namespace std::literals::chrono_literals;
std::tuple<KeyedBuildResult, KeyedBuildResult /*, KeyedBuildResult*/> t{
KeyedBuildResult{
BuildResult{.inner{KeyedBuildResult::Failure{{
.status = KeyedBuildResult::Failure::OutputRejected,
.msg = HintFmt("no idea why"),
}}}},
/* .path = */
DerivedPath::Opaque{
StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-xxx"},
},
},
KeyedBuildResult{
BuildResult{
.inner{KeyedBuildResult::Failure{{
.status = KeyedBuildResult::Failure::NotDeterministic,
.msg = HintFmt("no idea why"),
.isNonDeterministic = true,
}}},
.timesBuilt = 3,
.startTime = 30,
.stopTime = 50,
},
/* .path = */
DerivedPath::Built{
.drvPath = makeConstantStorePathRef(
StorePath{
"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv",
}),
.outputs = OutputsSpec::Names{"out"},
},
},
};
t;
}))
VERSIONED_CHARACTERIZATION_TEST(WorkerProtoTest, keyedBuildResult_1_29, "keyed-build-result-1.29", 1 << 8 | 29, ({
using namespace std::literals::chrono_literals;
std::tuple<KeyedBuildResult, KeyedBuildResult /*, KeyedBuildResult*/> t{
KeyedBuildResult{
{.inner{BuildResult::Failure{
.status = KeyedBuildResult::Failure::OutputRejected,
.errorMsg = "no idea why",
}}},
/* .path = */
DerivedPath::Opaque{
StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-xxx"},
},
},
KeyedBuildResult{
{
.inner{BuildResult::Failure{
.status = KeyedBuildResult::Failure::NotDeterministic,
.errorMsg = "no idea why",
.isNonDeterministic = true,
}},
.timesBuilt = 3,
.startTime = 30,
.stopTime = 50,
},
/* .path = */
DerivedPath::Built{
.drvPath = makeConstantStorePathRef(
StorePath{
"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv",
}),
.outputs = OutputsSpec::Names{"out"},
},
},
};
t;
}))
VERSIONED_CHARACTERIZATION_TEST(
WorkerProtoTest,
unkeyedValidPathInfo_1_15,
"unkeyed-valid-path-info-1.15",
(WorkerProto::Version{
.number =
{
.major = 1,
.minor = 15,
},
}),
1 << 8 | 15,
(std::tuple<UnkeyedValidPathInfo, UnkeyedValidPathInfo>{
({
UnkeyedValidPathInfo info{
@@ -606,13 +456,7 @@ VERSIONED_CHARACTERIZATION_TEST(
WorkerProtoTest,
validPathInfo_1_15,
"valid-path-info-1.15",
(WorkerProto::Version{
.number =
{
.major = 1,
.minor = 15,
},
}),
1 << 8 | 15,
(std::tuple<ValidPathInfo, ValidPathInfo>{
({
ValidPathInfo info{
@@ -661,13 +505,7 @@ VERSIONED_CHARACTERIZATION_TEST(
WorkerProtoTest,
validPathInfo_1_16,
"valid-path-info-1.16",
(WorkerProto::Version{
.number =
{
.major = 1,
.minor = 16,
},
}),
1 << 8 | 16,
(std::tuple<ValidPathInfo, ValidPathInfo, ValidPathInfo>{
({
ValidPathInfo info{
@@ -822,13 +660,7 @@ VERSIONED_CHARACTERIZATION_TEST_NO_JSON(
WorkerProtoTest,
clientHandshakeInfo_1_30,
"client-handshake-info_1_30",
(WorkerProto::Version{
.number =
{
.major = 1,
.minor = 30,
},
}),
1 << 8 | 30,
(std::tuple<WorkerProto::ClientHandshakeInfo>{
{},
}))
@@ -837,13 +669,7 @@ VERSIONED_CHARACTERIZATION_TEST_NO_JSON(
WorkerProtoTest,
clientHandshakeInfo_1_33,
"client-handshake-info_1_33",
(WorkerProto::Version{
.number =
{
.major = 1,
.minor = 33,
},
}),
1 << 8 | 33,
(std::tuple<WorkerProto::ClientHandshakeInfo, WorkerProto::ClientHandshakeInfo>{
{
.daemonNixVersion = std::optional{"foo"},
@@ -857,13 +683,7 @@ VERSIONED_CHARACTERIZATION_TEST_NO_JSON(
WorkerProtoTest,
clientHandshakeInfo_1_35,
"client-handshake-info_1_35",
(WorkerProto::Version{
.number =
{
.major = 1,
.minor = 35,
},
}),
1 << 8 | 35,
(std::tuple<WorkerProto::ClientHandshakeInfo, WorkerProto::ClientHandshakeInfo>{
{
.daemonNixVersion = std::optional{"foo"},
@@ -890,13 +710,13 @@ TEST_F(WorkerProtoTest, handshake_log)
FdSink out{toServer.writeSide.get()};
FdSource in0{toClient.readSide.get()};
TeeSource in{in0, toClientLog};
clientResult = WorkerProto::BasicClientConnection::handshake(out, in, defaultVersion);
clientResult = std::get<0>(WorkerProto::BasicClientConnection::handshake(out, in, defaultVersion, {}));
});
{
FdSink out{toClient.writeSide.get()};
FdSource in{toServer.readSide.get()};
WorkerProto::BasicServerConnection::handshake(out, in, defaultVersion);
WorkerProto::BasicServerConnection::handshake(out, in, defaultVersion, {});
};
thread.join();
@@ -911,43 +731,23 @@ TEST_F(WorkerProtoTest, handshake_features)
toClient.create();
toServer.create();
WorkerProto::Version clientResult;
std::tuple<WorkerProto::Version, WorkerProto::FeatureSet> clientResult;
auto clientThread = std::thread([&]() {
FdSink out{toServer.writeSide.get()};
FdSource in{toClient.readSide.get()};
clientResult = WorkerProto::BasicClientConnection::handshake(
out,
in,
WorkerProto::Version{
.number = {.major = 1, .minor = 123},
.features = {"bar", "aap", "mies", "xyzzy"},
});
clientResult = WorkerProto::BasicClientConnection::handshake(out, in, 123, {"bar", "aap", "mies", "xyzzy"});
});
FdSink out{toClient.writeSide.get()};
FdSource in{toServer.readSide.get()};
auto daemonResult = WorkerProto::BasicServerConnection::handshake(
out,
in,
WorkerProto::Version{
.number = {.major = 1, .minor = 200},
.features = {"foo", "bar", "xyzzy"},
});
auto daemonResult = WorkerProto::BasicServerConnection::handshake(out, in, 456, {"foo", "bar", "xyzzy"});
clientThread.join();
EXPECT_EQ(clientResult, daemonResult);
EXPECT_EQ(
clientResult,
(WorkerProto::Version{
.number =
{
.major = 1,
.minor = 123,
},
.features = {"bar", "xyzzy"},
}));
EXPECT_EQ(std::get<0>(clientResult), 123u);
EXPECT_EQ(std::get<1>(clientResult), WorkerProto::FeatureSet({"bar", "xyzzy"}));
}
/// Has to be a `BufferedSink` for handshake.
@@ -962,7 +762,8 @@ TEST_F(WorkerProtoTest, handshake_client_replay)
NullBufferedSink nullSink;
StringSource in{toClientLog};
auto clientResult = WorkerProto::BasicClientConnection::handshake(nullSink, in, defaultVersion);
auto clientResult =
std::get<0>(WorkerProto::BasicClientConnection::handshake(nullSink, in, defaultVersion, {}));
EXPECT_EQ(clientResult, defaultVersion);
});
@@ -976,10 +777,11 @@ TEST_F(WorkerProtoTest, handshake_client_truncated_replay_throws)
auto substring = toClientLog.substr(0, len);
StringSource in{substring};
if (len < 8) {
EXPECT_THROW(WorkerProto::BasicClientConnection::handshake(nullSink, in, defaultVersion), EndOfFile);
EXPECT_THROW(
WorkerProto::BasicClientConnection::handshake(nullSink, in, defaultVersion, {}), EndOfFile);
} else {
// Not sure why cannot keep on checking for `EndOfFile`.
EXPECT_THROW(WorkerProto::BasicClientConnection::handshake(nullSink, in, defaultVersion), Error);
EXPECT_THROW(WorkerProto::BasicClientConnection::handshake(nullSink, in, defaultVersion, {}), Error);
}
}
});
@@ -999,13 +801,14 @@ TEST_F(WorkerProtoTest, handshake_client_corrupted_throws)
if (idx < 4 || idx == 9) {
// magic bytes don't match
EXPECT_THROW(WorkerProto::BasicClientConnection::handshake(nullSink, in, defaultVersion), Error);
EXPECT_THROW(WorkerProto::BasicClientConnection::handshake(nullSink, in, defaultVersion, {}), Error);
} else if (idx < 8 || idx >= 12) {
// Number out of bounds
EXPECT_THROW(
WorkerProto::BasicClientConnection::handshake(nullSink, in, defaultVersion), SerialisationError);
WorkerProto::BasicClientConnection::handshake(nullSink, in, defaultVersion, {}),
SerialisationError);
} else {
auto ver = WorkerProto::BasicClientConnection::handshake(nullSink, in, defaultVersion);
auto ver = std::get<0>(WorkerProto::BasicClientConnection::handshake(nullSink, in, defaultVersion, {}));
// `std::min` of this and the other version saves us
EXPECT_EQ(ver, defaultVersion);
}
+3 -3
View File
@@ -193,7 +193,7 @@ TEST_F(WorkerSubstitutionTest, floatingDerivationOutput)
};
// Write the derivation to the destination store
auto drvPath = dummyStore->writeDerivation(drv);
auto drvPath = writeDerivation(*dummyStore, drv);
// Snapshot the destination store before
checkpointJson("ca-drv/store-before", dummyStore);
@@ -297,7 +297,7 @@ TEST_F(WorkerSubstitutionTest, floatingDerivationOutputWithDepDrv)
};
// Write the dependency derivation to the destination store
auto depDrvPath = dummyStore->writeDerivation(depDrv);
auto depDrvPath = writeDerivation(*dummyStore, depDrv);
// Compute the hash modulo for the dependency derivation
auto depHashModulo = hashDerivationModulo(*dummyStore, depDrv, true);
@@ -348,7 +348,7 @@ TEST_F(WorkerSubstitutionTest, floatingDerivationOutputWithDepDrv)
rootDrv.inputDrvs = {.map = {{depDrvPath, {.value = {"out"}}}}};
// Write the root derivation to the destination store
auto rootDrvPath = dummyStore->writeDerivation(rootDrv);
auto rootDrvPath = writeDerivation(*dummyStore, rootDrv);
// Snapshot the destination store before
checkpointJson("issue-11928/store-before", dummyStore);
+3 -3
View File
@@ -44,12 +44,12 @@ TEST_F(WriteDerivationTest, addToStoreFromDumpCalledOnce)
{
auto drv = makeSimpleDrv();
auto path1 = store->writeDerivation(drv, NoRepair);
auto path1 = writeDerivation(*store, drv, NoRepair);
config->readOnly = true;
auto path2 = computeStorePath(*store, drv);
auto path2 = writeDerivation(*store, drv, NoRepair);
EXPECT_EQ(path1, path2);
EXPECT_THAT(
[&] { store->writeDerivation(drv, Repair); },
[&] { writeDerivation(*store, drv, Repair); },
::testing::ThrowsMessage<Error>(
testing::HasSubstrIgnoreANSIMatcher("operation 'writeDerivation' is not supported by store 'dummy://'")));
}
+12 -103
View File
@@ -13,11 +13,6 @@
// C library headers for SSO provider support
# include <aws/auth/credentials.h>
// C library headers for custom logging
# include <aws/common/logging.h>
# include <cstdarg>
# include <boost/unordered/concurrent_flat_map.hpp>
# include <chrono>
@@ -35,101 +30,6 @@ AwsAuthError::AwsAuthError(int errorCode)
namespace {
/**
* Map AWS log level to Nix verbosity.
* AWS levels: AWS_LL_NONE=0, AWS_LL_FATAL=1, AWS_LL_ERROR=2, AWS_LL_WARN=3,
* AWS_LL_INFO=4, AWS_LL_DEBUG=5, AWS_LL_TRACE=6
*
* We map very conservatively because the AWS SDK is extremely noisy. What AWS
* considers "info" includes low-level details like "Initializing epoll" and
* "Starting event-loop thread". What it considers "errors" includes expected
* conditions like missing ~/.aws/config or IMDS being unavailable on non-EC2.
*
* To avoid spamming users, we only show FATAL at default verbosity. Everything
* else requires -vvvvv (lvlDebug) or higher to see.
*/
static Verbosity awsLogLevelToVerbosity(enum aws_log_level level)
{
switch (level) {
case AWS_LL_FATAL:
return lvlError;
case AWS_LL_ERROR:
case AWS_LL_WARN:
case AWS_LL_INFO:
return lvlDebug;
case AWS_LL_DEBUG:
case AWS_LL_TRACE:
return lvlVomit;
// AWS_LL_NONE and AWS_LL_COUNT are enum sentinels, not real log levels
case AWS_LL_NONE:
case AWS_LL_COUNT:
return lvlDebug;
}
unreachable();
}
/**
* Custom AWS logger that routes logs through Nix's logging infrastructure.
*
* The AWS CRT C++ wrapper (ApiHandle::InitializeLogging) only supports FILE*
* or filename-based logging. The underlying C library supports custom loggers
* via aws_logger struct with a vtable containing callback functions.
*/
static int nixAwsLoggerLog(
struct aws_logger * logger, enum aws_log_level logLevel, aws_log_subject_t subject, const char * format, ...)
{
Verbosity nixLevel = awsLogLevelToVerbosity(logLevel);
if (nixLevel > verbosity)
return AWS_OP_SUCCESS; /* Bail out early to avoid formatting the message unnecessarily. */
va_list args;
va_start(args, format);
std::array<char, 4096> buffer{};
auto res = vsnprintf(buffer.data(), buffer.size(), format, args);
va_end(args);
if (res < 0) /* Skip garbage debug messages in case the SDK is busted. */
return AWS_OP_SUCCESS;
const char * subjectName = aws_log_subject_name(subject);
printMsgUsing(nix::logger, nixLevel, "(aws:%s) %s", subjectName ? subjectName : "unknown", chomp(buffer.data()));
return AWS_OP_SUCCESS;
}
/**
* Get current log level for a subject - determines which messages will be logged.
* Must be consistent with awsLogLevelToVerbosity mapping.
*/
static aws_log_level nixAwsLoggerGetLevel(struct aws_logger * logger, aws_log_subject_t subject)
{
// Map Nix verbosity back to AWS log level (inverse of awsLogLevelToVerbosity)
if (verbosity >= lvlVomit)
return AWS_LL_TRACE;
if (verbosity >= lvlDebug)
return AWS_LL_INFO; // error/warn/info are all mapped to lvlDebug
return AWS_LL_FATAL;
}
static void initialiseAwsLogger()
{
static std::once_flag initialised; /* aws_logger_set must only be called once */
std::call_once(initialised, []() {
static aws_logger_vtable nixAwsLoggerVtable = {
.log = nixAwsLoggerLog,
.get_log_level = nixAwsLoggerGetLevel,
.clean_up = [](struct aws_logger *) {}, // No resources to clean up
.set_log_level = nullptr,
};
static aws_logger nixAwsLogger = {
.vtable = &nixAwsLoggerVtable,
.allocator = nullptr,
.p_impl = nullptr,
};
aws_logger_set(&nixAwsLogger);
});
}
/**
* Helper function to wrap a C credentials provider in the C++ interface.
* This replicates the static s_CreateWrappedProvider from aws-crt-cpp.
@@ -219,9 +119,18 @@ class AwsCredentialProviderImpl : public AwsCredentialProvider
public:
AwsCredentialProviderImpl()
{
// Install custom logger that routes AWS CRT logs through Nix's logging infrastructure.
// This ensures AWS logs respect Nix's verbosity settings and are formatted consistently.
initialiseAwsLogger();
// Map Nix's verbosity to AWS CRT log level
Aws::Crt::LogLevel logLevel;
if (verbosity >= lvlVomit) {
logLevel = Aws::Crt::LogLevel::Trace;
} else if (verbosity >= lvlDebug) {
logLevel = Aws::Crt::LogLevel::Debug;
} else if (verbosity >= lvlChatty) {
logLevel = Aws::Crt::LogLevel::Info;
} else {
logLevel = Aws::Crt::LogLevel::Warn;
}
apiHandle.InitializeLogging(logLevel, stderr);
// Create a shared TLS context for SSO (required for HTTPS connections)
auto allocator = Aws::Crt::ApiAllocator();
+27 -31
View File
@@ -28,7 +28,7 @@ BinaryCacheStore::BinaryCacheStore(Config & config)
: config{config}
{
if (config.secretKeyFile != "")
signers.push_back(std::make_unique<LocalSigner>(SecretKey{readFile(config.secretKeyFile.get())}));
signers.push_back(std::make_unique<LocalSigner>(SecretKey{readFile(config.secretKeyFile)}));
if (config.secretKeyFiles != "") {
std::stringstream ss(config.secretKeyFiles);
@@ -156,7 +156,7 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
config.compression, teeSinkCompressed, config.parallelCompression, config.compressionLevel);
TeeSink teeSinkUncompressed{*compressionSink, narHashSink};
TeeSource teeSource{narSource, teeSinkUncompressed};
narAccessor = makeNarAccessor(parseNarListing(teeSource));
narAccessor = makeNarAccessor(teeSource);
compressionSink->finish();
fileSink.flush();
}
@@ -435,41 +435,37 @@ void BinaryCacheStore::narFromPath(const StorePath & storePath, Sink & sink)
void BinaryCacheStore::queryPathInfoUncached(
const StorePath & storePath, Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept
{
auto uri = config.getReference().render(/*FIXME withParams=*/false);
auto storePathS = printStorePath(storePath);
auto act = std::make_shared<Activity>(
*logger,
lvlTalkative,
actQueryPathInfo,
fmt("querying info about '%s' on '%s'", storePathS, uri),
Logger::Fields{storePathS, uri});
PushActivity pact(act->id);
auto narInfoFile = narInfoFileFor(storePath);
auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback));
try {
auto uri = config.getReference().render(/*FIXME withParams=*/false);
auto storePathS = printStorePath(storePath);
auto act = std::make_shared<Activity>(
*logger,
lvlTalkative,
actQueryPathInfo,
fmt("querying info about '%s' on '%s'", storePathS, uri),
Logger::Fields{storePathS, uri});
PushActivity pact(act->id);
getFile(narInfoFile, {[=, this](std::future<std::optional<std::string>> fut) {
try {
auto data = fut.get();
auto narInfoFile = narInfoFileFor(storePath);
if (!data)
return (*callbackPtr)({});
getFile(narInfoFile, {[=, this](std::future<std::optional<std::string>> fut) {
try {
auto data = fut.get();
stats.narInfoRead++;
if (!data)
return (*callbackPtr)({});
(*callbackPtr)(
(std::shared_ptr<ValidPathInfo>) std::make_shared<NarInfo>(*this, *data, narInfoFile));
stats.narInfoRead++;
(*callbackPtr)(
(std::shared_ptr<ValidPathInfo>) std::make_shared<NarInfo>(*this, *data, narInfoFile));
(void) act; // force Activity into this lambda to ensure it stays alive
} catch (...) {
callbackPtr->rethrow();
}
}});
} catch (...) {
callbackPtr->rethrow();
}
(void) act; // force Activity into this lambda to ensure it stays alive
} catch (...) {
callbackPtr->rethrow();
}
}});
}
StorePath BinaryCacheStore::addToStore(
Loaded 100 of 411 files, more files were not shown because too many files have changed in this diff. Show more