Compare commits
52
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0769726d44 | ||
|
|
296b8d7bb9 | ||
|
|
a35595c6de | ||
|
|
e258c32fa6 | ||
|
|
e2af95376a | ||
|
|
b42319aa85 | ||
|
|
6e88d0cf6c | ||
|
|
eef8a1a5ff | ||
|
|
377fc95bce | ||
|
|
48bbd96d2d | ||
|
|
695501815b | ||
|
|
5ac10221a6 | ||
|
|
e9d77856f5 | ||
|
|
0c774b7ca4 | ||
|
|
70e39c2721 | ||
|
|
18f8b70cd9 | ||
|
|
4aa2942014 | ||
|
|
f28e495294 | ||
|
|
0142a88b8f | ||
|
|
9bf642b88d | ||
|
|
e96a3f7a73 | ||
|
|
d297aeca2c | ||
|
|
534de0df4e | ||
|
|
65c7ec71ce | ||
|
|
50aecffbf9 | ||
|
|
2d85a6ba32 | ||
|
|
73a0311ef8 | ||
|
|
cb1c413db8 | ||
|
|
8a4d8aabb2 | ||
|
|
344a0eaed1 | ||
|
|
3034589047 | ||
|
|
a2b044dc8d | ||
|
|
70ecd8c8a9 | ||
|
|
a7276a24b9 | ||
|
|
48bf9a8e50 | ||
|
|
2d879efee0 | ||
|
|
2ec62f1974 | ||
|
|
09884e2c1a | ||
|
|
753bf479f9 | ||
|
|
a6c201e039 | ||
|
|
1d39afac38 | ||
|
|
ba42159b63 | ||
|
|
f42de47a40 | ||
|
|
a569ebca7e | ||
|
|
7808b682bb | ||
|
|
7672220083 | ||
|
|
ed28ceb12f | ||
|
|
f99073b646 | ||
|
|
fbffd5683e | ||
|
|
ad5cd0b9f3 | ||
|
|
c0c13d7323 | ||
|
|
c272697224 |
No files matched your search
@@ -24,8 +24,8 @@ inputs:
|
||||
description: "Github token"
|
||||
required: true
|
||||
use_cache:
|
||||
description: "Whether to setup magic-nix-cache"
|
||||
default: true
|
||||
description: "Whether to setup github actions cache (not implemented currently)"
|
||||
default: false
|
||||
required: false
|
||||
runs:
|
||||
using: "composite"
|
||||
@@ -122,10 +122,3 @@ runs:
|
||||
source-url: ${{ inputs.experimental-installer-version != 'latest' && 'https://artifacts.nixos.org/experimental-installer/tag/${{ inputs.experimental-installer-version }}/${{ env.EXPERIMENTAL_INSTALLER_ARTIFACT }}' || '' }}
|
||||
nix-package-url: ${{ inputs.dogfood == 'true' && steps.download-nix-installer.outputs.tarball-path || (inputs.tarball_url || '') }}
|
||||
extra-conf: ${{ inputs.extra_nix_config }}
|
||||
- uses: DeterminateSystems/magic-nix-cache-action@565684385bcd71bad329742eefe8d12f2e765b39 # v13
|
||||
if: ${{ inputs.use_cache == 'true' }}
|
||||
with:
|
||||
diagnostic-endpoint: ''
|
||||
use-flakehub: false
|
||||
use-gha-cache: true
|
||||
source-revision: 92d9581367be2233c2d5714a2640e1339f4087d8 # main
|
||||
@@ -39,6 +39,17 @@ jobs:
|
||||
role-to-assume: "arn:aws:iam::080433136561:role/nix-release"
|
||||
role-session-name: nix-release-oidc-${{ github.run_id }}
|
||||
aws-region: eu-west-1
|
||||
- name: Disable containerd image store
|
||||
run: |
|
||||
# Docker 28+ defaults to the containerd image store, which
|
||||
# pushes layers uncompressed instead of gzip. OCI clients
|
||||
# that only support gzip (e.g. go-containerregistry) fail
|
||||
# with "gzip: invalid header". Disabling the containerd
|
||||
# snapshotter restores the classic storage driver, which
|
||||
# preserves gzip-compressed layers through the
|
||||
# `docker load` / `docker push` pipeline.
|
||||
echo '{"features":{"containerd-snapshotter":false}}' | sudo tee /etc/docker/daemon.json > /dev/null
|
||||
sudo systemctl restart docker
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
|
||||
with:
|
||||
|
||||
@@ -279,3 +279,35 @@ This release was made possible by the following 33 contributors:
|
||||
- Henry [**(@cootshk)**](https://github.com/cootshk)
|
||||
- Martin Joerg [**(@mjoerg)**](https://github.com/mjoerg)
|
||||
- Farid Zakaria [**(@fzakaria)**](https://github.com/fzakaria)
|
||||
# Release 2.33.3 (2026-02-13)
|
||||
|
||||
- S3 binary caches now use virtual-hosted-style addressing by default [#15208](https://github.com/NixOS/nix/issues/15208)
|
||||
|
||||
S3 binary caches now use virtual-hosted-style URLs
|
||||
(`https://bucket.s3.region.amazonaws.com/key`) instead of path-style URLs
|
||||
(`https://s3.region.amazonaws.com/bucket/key`) when connecting to standard AWS
|
||||
S3 endpoints. This enables HTTP/2 multiplexing and fixes TCP connection
|
||||
exhaustion (TIME_WAIT socket accumulation) under high-concurrency workloads.
|
||||
|
||||
A new `addressing-style` store option controls this behavior:
|
||||
|
||||
- `auto` (default): virtual-hosted-style for standard AWS endpoints, path-style
|
||||
for custom endpoints.
|
||||
- `path`: forces path-style addressing (deprecated by AWS).
|
||||
- `virtual`: forces virtual-hosted-style addressing (bucket names must not
|
||||
contain dots).
|
||||
|
||||
Bucket names containing dots (e.g., `my.bucket.name`) automatically fall back
|
||||
to path-style addressing in `auto` mode, because dotted names create
|
||||
multi-level subdomains that break TLS wildcard certificate validation.
|
||||
|
||||
Example using path-style for backwards compatibility:
|
||||
|
||||
```
|
||||
s3://my-bucket/key?region=us-east-1&addressing-style=path
|
||||
```
|
||||
|
||||
Additionally, TCP keep-alive is now enabled on all HTTP connections, preventing
|
||||
idle connections from being silently dropped by intermediate network devices
|
||||
(NATs, firewalls, load balancers).
|
||||
|
||||
@@ -236,6 +236,9 @@ void Input::checkLocks(Input specified, Input & result)
|
||||
if (auto prevNarHash = specified.getNarHash())
|
||||
specified.attrs.insert_or_assign("narHash", prevNarHash->to_string(HashFormat::SRI, true));
|
||||
|
||||
if (auto narHash = result.getNarHash())
|
||||
result.attrs.insert_or_assign("narHash", narHash->to_string(HashFormat::SRI, true));
|
||||
|
||||
for (auto & field : specified.attrs) {
|
||||
auto field2 = result.attrs.find(field.first);
|
||||
if (field2 != result.attrs.end() && field.second != field2->second)
|
||||
|
||||
@@ -128,6 +128,7 @@ static void prim_flakeRefToString(EvalState & state, const PosIdx pos, Value **
|
||||
state.forceAttrs(*args[0], noPos, "while evaluating the argument passed to builtins.flakeRefToString");
|
||||
fetchers::Attrs attrs;
|
||||
for (const auto & attr : *args[0]->attrs()) {
|
||||
state.forceValue(*attr.value, attr.pos);
|
||||
auto t = attr.value->type();
|
||||
if (t == nInt) {
|
||||
auto intValue = attr.value->integer().value;
|
||||
|
||||
+237
-15
@@ -104,6 +104,33 @@ INSTANTIATE_TEST_SUITE_P(
|
||||
},
|
||||
},
|
||||
"with_absolute_endpoint_uri",
|
||||
},
|
||||
ParsedS3URLTestCase{
|
||||
"s3://bucket/key?addressing-style=virtual",
|
||||
{
|
||||
.bucket = "bucket",
|
||||
.key = {"key"},
|
||||
.addressingStyle = S3AddressingStyle::Virtual,
|
||||
},
|
||||
"with_addressing_style_virtual",
|
||||
},
|
||||
ParsedS3URLTestCase{
|
||||
"s3://bucket/key?addressing-style=path",
|
||||
{
|
||||
.bucket = "bucket",
|
||||
.key = {"key"},
|
||||
.addressingStyle = S3AddressingStyle::Path,
|
||||
},
|
||||
"with_addressing_style_path",
|
||||
},
|
||||
ParsedS3URLTestCase{
|
||||
"s3://bucket/key?addressing-style=auto",
|
||||
{
|
||||
.bucket = "bucket",
|
||||
.key = {"key"},
|
||||
.addressingStyle = S3AddressingStyle::Auto,
|
||||
},
|
||||
"with_addressing_style_auto",
|
||||
}),
|
||||
[](const ::testing::TestParamInfo<ParsedS3URLTestCase> & info) { return info.param.description; });
|
||||
|
||||
@@ -138,6 +165,26 @@ INSTANTIATE_TEST_SUITE_P(
|
||||
InvalidS3URLTestCase{"s3://bucket", "error: URI has a missing or invalid key", "missing_key"}),
|
||||
[](const ::testing::TestParamInfo<InvalidS3URLTestCase> & info) { return info.param.description; });
|
||||
|
||||
TEST(ParsedS3URLTest, invalidAddressingStyleThrows)
|
||||
{
|
||||
ASSERT_THROW(ParsedS3URL::parse(parseURL("s3://bucket/key?addressing-style=bogus")), InvalidS3AddressingStyle);
|
||||
}
|
||||
|
||||
TEST(ParsedS3URLTest, virtualStyleWithAuthoritylessEndpointThrows)
|
||||
{
|
||||
ParsedS3URL input{
|
||||
.bucket = "bucket",
|
||||
.key = {"key"},
|
||||
.addressingStyle = S3AddressingStyle::Virtual,
|
||||
.endpoint =
|
||||
ParsedURL{
|
||||
.scheme = "file",
|
||||
.path = {"", "some", "path"},
|
||||
},
|
||||
};
|
||||
ASSERT_THROW(input.toHttpsUrl(), nix::Error);
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// S3 URL to HTTPS Conversion Tests
|
||||
// =============================================================================
|
||||
@@ -166,6 +213,7 @@ INSTANTIATE_TEST_SUITE_P(
|
||||
S3ToHttpsConversion,
|
||||
S3ToHttpsConversionTest,
|
||||
::testing::Values(
|
||||
// Default (auto) addressing style: virtual-hosted for standard AWS endpoints
|
||||
S3ToHttpsConversionTestCase{
|
||||
ParsedS3URL{
|
||||
.bucket = "my-bucket",
|
||||
@@ -173,10 +221,10 @@ INSTANTIATE_TEST_SUITE_P(
|
||||
},
|
||||
ParsedURL{
|
||||
.scheme = "https",
|
||||
.authority = ParsedURL::Authority{.host = "s3.us-east-1.amazonaws.com"},
|
||||
.path = {"", "my-bucket", "my-key.txt"},
|
||||
.authority = ParsedURL::Authority{.host = "my-bucket.s3.us-east-1.amazonaws.com"},
|
||||
.path = {"", "my-key.txt"},
|
||||
},
|
||||
"https://s3.us-east-1.amazonaws.com/my-bucket/my-key.txt",
|
||||
"https://my-bucket.s3.us-east-1.amazonaws.com/my-key.txt",
|
||||
"basic_s3_default_region",
|
||||
},
|
||||
S3ToHttpsConversionTestCase{
|
||||
@@ -187,12 +235,13 @@ INSTANTIATE_TEST_SUITE_P(
|
||||
},
|
||||
ParsedURL{
|
||||
.scheme = "https",
|
||||
.authority = ParsedURL::Authority{.host = "s3.eu-west-1.amazonaws.com"},
|
||||
.path = {"", "prod-cache", "nix", "store", "abc123.nar.xz"},
|
||||
.authority = ParsedURL::Authority{.host = "prod-cache.s3.eu-west-1.amazonaws.com"},
|
||||
.path = {"", "nix", "store", "abc123.nar.xz"},
|
||||
},
|
||||
"https://s3.eu-west-1.amazonaws.com/prod-cache/nix/store/abc123.nar.xz",
|
||||
"https://prod-cache.s3.eu-west-1.amazonaws.com/nix/store/abc123.nar.xz",
|
||||
"with_eu_west_1_region",
|
||||
},
|
||||
// Custom endpoint authority: path-style by default
|
||||
S3ToHttpsConversionTestCase{
|
||||
ParsedS3URL{
|
||||
.bucket = "bucket",
|
||||
@@ -208,6 +257,7 @@ INSTANTIATE_TEST_SUITE_P(
|
||||
"http://custom.s3.com/bucket/key",
|
||||
"custom_endpoint_authority",
|
||||
},
|
||||
// Custom endpoint URL: path-style by default
|
||||
S3ToHttpsConversionTestCase{
|
||||
ParsedS3URL{
|
||||
.bucket = "bucket",
|
||||
@@ -236,10 +286,10 @@ INSTANTIATE_TEST_SUITE_P(
|
||||
},
|
||||
ParsedURL{
|
||||
.scheme = "https",
|
||||
.authority = ParsedURL::Authority{.host = "s3.ap-southeast-2.amazonaws.com"},
|
||||
.path = {"", "bucket", "path", "to", "file.txt"},
|
||||
.authority = ParsedURL::Authority{.host = "bucket.s3.ap-southeast-2.amazonaws.com"},
|
||||
.path = {"", "path", "to", "file.txt"},
|
||||
},
|
||||
"https://s3.ap-southeast-2.amazonaws.com/bucket/path/to/file.txt",
|
||||
"https://bucket.s3.ap-southeast-2.amazonaws.com/path/to/file.txt",
|
||||
"complex_path_and_region",
|
||||
},
|
||||
S3ToHttpsConversionTestCase{
|
||||
@@ -250,11 +300,11 @@ INSTANTIATE_TEST_SUITE_P(
|
||||
},
|
||||
ParsedURL{
|
||||
.scheme = "https",
|
||||
.authority = ParsedURL::Authority{.host = "s3.us-east-1.amazonaws.com"},
|
||||
.path = {"", "my-bucket", "my-key.txt"},
|
||||
.authority = ParsedURL::Authority{.host = "my-bucket.s3.us-east-1.amazonaws.com"},
|
||||
.path = {"", "my-key.txt"},
|
||||
.query = {{"versionId", "abc123xyz"}},
|
||||
},
|
||||
"https://s3.us-east-1.amazonaws.com/my-bucket/my-key.txt?versionId=abc123xyz",
|
||||
"https://my-bucket.s3.us-east-1.amazonaws.com/my-key.txt?versionId=abc123xyz",
|
||||
"with_versionId",
|
||||
},
|
||||
S3ToHttpsConversionTestCase{
|
||||
@@ -266,13 +316,185 @@ INSTANTIATE_TEST_SUITE_P(
|
||||
},
|
||||
ParsedURL{
|
||||
.scheme = "https",
|
||||
.authority = ParsedURL::Authority{.host = "s3.eu-west-1.amazonaws.com"},
|
||||
.path = {"", "versioned-bucket", "path", "to", "object"},
|
||||
.authority = ParsedURL::Authority{.host = "versioned-bucket.s3.eu-west-1.amazonaws.com"},
|
||||
.path = {"", "path", "to", "object"},
|
||||
.query = {{"versionId", "version456"}},
|
||||
},
|
||||
"https://s3.eu-west-1.amazonaws.com/versioned-bucket/path/to/object?versionId=version456",
|
||||
"https://versioned-bucket.s3.eu-west-1.amazonaws.com/path/to/object?versionId=version456",
|
||||
"with_region_and_versionId",
|
||||
},
|
||||
// Explicit addressing-style=path forces path-style on standard AWS endpoints
|
||||
S3ToHttpsConversionTestCase{
|
||||
ParsedS3URL{
|
||||
.bucket = "my-bucket",
|
||||
.key = {"my-key.txt"},
|
||||
.region = "us-west-2",
|
||||
.addressingStyle = S3AddressingStyle::Path,
|
||||
},
|
||||
ParsedURL{
|
||||
.scheme = "https",
|
||||
.authority = ParsedURL::Authority{.host = "s3.us-west-2.amazonaws.com"},
|
||||
.path = {"", "my-bucket", "my-key.txt"},
|
||||
},
|
||||
"https://s3.us-west-2.amazonaws.com/my-bucket/my-key.txt",
|
||||
"explicit_path_style",
|
||||
},
|
||||
// Explicit addressing-style=virtual forces virtual-hosted-style on custom endpoints
|
||||
S3ToHttpsConversionTestCase{
|
||||
ParsedS3URL{
|
||||
.bucket = "bucket",
|
||||
.key = {"key"},
|
||||
.scheme = "http",
|
||||
.addressingStyle = S3AddressingStyle::Virtual,
|
||||
.endpoint = ParsedURL::Authority{.host = "custom.s3.com"},
|
||||
},
|
||||
ParsedURL{
|
||||
.scheme = "http",
|
||||
.authority = ParsedURL::Authority{.host = "bucket.custom.s3.com"},
|
||||
.path = {"", "key"},
|
||||
},
|
||||
"http://bucket.custom.s3.com/key",
|
||||
"explicit_virtual_style_custom_endpoint",
|
||||
},
|
||||
// Explicit addressing-style=virtual with full endpoint URL
|
||||
S3ToHttpsConversionTestCase{
|
||||
ParsedS3URL{
|
||||
.bucket = "bucket",
|
||||
.key = {"key"},
|
||||
.addressingStyle = S3AddressingStyle::Virtual,
|
||||
.endpoint =
|
||||
ParsedURL{
|
||||
.scheme = "http",
|
||||
.authority = ParsedURL::Authority{.host = "server", .port = 9000},
|
||||
.path = {""},
|
||||
},
|
||||
},
|
||||
ParsedURL{
|
||||
.scheme = "http",
|
||||
.authority = ParsedURL::Authority{.host = "bucket.server", .port = 9000},
|
||||
.path = {"", "key"},
|
||||
},
|
||||
"http://bucket.server:9000/key",
|
||||
"explicit_virtual_style_full_endpoint_url",
|
||||
},
|
||||
// Dotted bucket names work normally with explicit path-style
|
||||
S3ToHttpsConversionTestCase{
|
||||
ParsedS3URL{
|
||||
.bucket = "my.bucket",
|
||||
.key = {"key"},
|
||||
.addressingStyle = S3AddressingStyle::Path,
|
||||
},
|
||||
ParsedURL{
|
||||
.scheme = "https",
|
||||
.authority = ParsedURL::Authority{.host = "s3.us-east-1.amazonaws.com"},
|
||||
.path = {"", "my.bucket", "key"},
|
||||
},
|
||||
"https://s3.us-east-1.amazonaws.com/my.bucket/key",
|
||||
"dotted_bucket_with_path_style",
|
||||
},
|
||||
// Dotted bucket names fall back to path-style with auto on standard AWS endpoints
|
||||
S3ToHttpsConversionTestCase{
|
||||
ParsedS3URL{
|
||||
.bucket = "my.bucket.name",
|
||||
.key = {"key"},
|
||||
},
|
||||
ParsedURL{
|
||||
.scheme = "https",
|
||||
.authority = ParsedURL::Authority{.host = "s3.us-east-1.amazonaws.com"},
|
||||
.path = {"", "my.bucket.name", "key"},
|
||||
},
|
||||
"https://s3.us-east-1.amazonaws.com/my.bucket.name/key",
|
||||
"dotted_bucket_with_auto_style_on_aws",
|
||||
},
|
||||
// Dotted bucket names work with auto style on custom endpoints (auto = path-style)
|
||||
S3ToHttpsConversionTestCase{
|
||||
ParsedS3URL{
|
||||
.bucket = "my.bucket",
|
||||
.key = {"key"},
|
||||
.endpoint = ParsedURL::Authority{.host = "minio.local"},
|
||||
},
|
||||
ParsedURL{
|
||||
.scheme = "https",
|
||||
.authority = ParsedURL::Authority{.host = "minio.local"},
|
||||
.path = {"", "my.bucket", "key"},
|
||||
},
|
||||
"https://minio.local/my.bucket/key",
|
||||
"dotted_bucket_with_auto_style_custom_endpoint",
|
||||
}),
|
||||
[](const ::testing::TestParamInfo<S3ToHttpsConversionTestCase> & info) { return info.param.description; });
|
||||
|
||||
// =============================================================================
|
||||
// S3 URL to HTTPS Conversion Error Tests
|
||||
// =============================================================================
|
||||
|
||||
struct S3ToHttpsConversionErrorTestCase
|
||||
{
|
||||
ParsedS3URL input;
|
||||
std::string description;
|
||||
};
|
||||
|
||||
class S3ToHttpsConversionErrorTest : public ::testing::WithParamInterface<S3ToHttpsConversionErrorTestCase>,
|
||||
public ::testing::Test
|
||||
{};
|
||||
|
||||
TEST_P(S3ToHttpsConversionErrorTest, ThrowsOnConversion)
|
||||
{
|
||||
auto & [input, description] = GetParam();
|
||||
ASSERT_THROW(input.toHttpsUrl(), nix::Error);
|
||||
}
|
||||
|
||||
INSTANTIATE_TEST_SUITE_P(
|
||||
S3ToHttpsConversionErrors,
|
||||
S3ToHttpsConversionErrorTest,
|
||||
::testing::Values(
|
||||
S3ToHttpsConversionErrorTestCase{
|
||||
ParsedS3URL{
|
||||
.bucket = "bucket",
|
||||
.key = {"key"},
|
||||
.addressingStyle = S3AddressingStyle::Virtual,
|
||||
.endpoint = ParsedURL::Authority{.host = ""},
|
||||
},
|
||||
"virtual_style_with_empty_host_authority",
|
||||
},
|
||||
S3ToHttpsConversionErrorTestCase{
|
||||
ParsedS3URL{
|
||||
.bucket = "my.bucket",
|
||||
.key = {"key"},
|
||||
.addressingStyle = S3AddressingStyle::Virtual,
|
||||
},
|
||||
"dotted_bucket_with_explicit_virtual_style",
|
||||
},
|
||||
S3ToHttpsConversionErrorTestCase{
|
||||
ParsedS3URL{
|
||||
.bucket = "my.bucket.name",
|
||||
.key = {"key"},
|
||||
.addressingStyle = S3AddressingStyle::Virtual,
|
||||
},
|
||||
"dotted_bucket_with_explicit_virtual_style_multi_dot",
|
||||
},
|
||||
S3ToHttpsConversionErrorTestCase{
|
||||
ParsedS3URL{
|
||||
.bucket = "my.bucket",
|
||||
.key = {"key"},
|
||||
.addressingStyle = S3AddressingStyle::Virtual,
|
||||
.endpoint = ParsedURL::Authority{.host = "minio.local"},
|
||||
},
|
||||
"dotted_bucket_with_explicit_virtual_style_custom_authority",
|
||||
},
|
||||
S3ToHttpsConversionErrorTestCase{
|
||||
ParsedS3URL{
|
||||
.bucket = "my.bucket",
|
||||
.key = {"key"},
|
||||
.addressingStyle = S3AddressingStyle::Virtual,
|
||||
.endpoint =
|
||||
ParsedURL{
|
||||
.scheme = "http",
|
||||
.authority = ParsedURL::Authority{.host = "minio.local", .port = 9000},
|
||||
.path = {""},
|
||||
},
|
||||
},
|
||||
"dotted_bucket_with_explicit_virtual_style_full_endpoint_url",
|
||||
}),
|
||||
[](const ::testing::TestParamInfo<S3ToHttpsConversionErrorTestCase> & info) { return info.param.description; });
|
||||
|
||||
} // namespace nix
|
||||
+212
-28
@@ -4,15 +4,20 @@
|
||||
|
||||
# include <aws/crt/Types.h>
|
||||
# include "nix/store/s3-url.hh"
|
||||
# include "nix/util/finally.hh"
|
||||
# include "nix/util/logging.hh"
|
||||
# include "nix/util/url.hh"
|
||||
# include "nix/util/util.hh"
|
||||
|
||||
# include <aws/crt/Api.h>
|
||||
# include <aws/crt/auth/Credentials.h>
|
||||
# include <aws/crt/io/Bootstrap.h>
|
||||
|
||||
// C library headers for SSO provider support
|
||||
# include <aws/auth/credentials.h>
|
||||
|
||||
// C library headers for custom logging
|
||||
# include <aws/common/logging.h>
|
||||
|
||||
# include <cstdarg>
|
||||
|
||||
# include <boost/unordered/concurrent_flat_map.hpp>
|
||||
|
||||
# include <chrono>
|
||||
@@ -30,6 +35,141 @@ AwsAuthError::AwsAuthError(int errorCode)
|
||||
|
||||
namespace {
|
||||
|
||||
/**
|
||||
* Map AWS log level to Nix verbosity.
|
||||
* AWS levels: AWS_LL_NONE=0, AWS_LL_FATAL=1, AWS_LL_ERROR=2, AWS_LL_WARN=3,
|
||||
* AWS_LL_INFO=4, AWS_LL_DEBUG=5, AWS_LL_TRACE=6
|
||||
*
|
||||
* We map very conservatively because the AWS SDK is extremely noisy. What AWS
|
||||
* considers "info" includes low-level details like "Initializing epoll" and
|
||||
* "Starting event-loop thread". What it considers "errors" includes expected
|
||||
* conditions like missing ~/.aws/config or IMDS being unavailable on non-EC2.
|
||||
*
|
||||
* To avoid spamming users, we only show FATAL at default verbosity. Everything
|
||||
* else requires -vvvvv (lvlDebug) or higher to see.
|
||||
*/
|
||||
static Verbosity awsLogLevelToVerbosity(enum aws_log_level level)
|
||||
{
|
||||
switch (level) {
|
||||
case AWS_LL_FATAL:
|
||||
return lvlError;
|
||||
case AWS_LL_ERROR:
|
||||
case AWS_LL_WARN:
|
||||
case AWS_LL_INFO:
|
||||
return lvlDebug;
|
||||
case AWS_LL_DEBUG:
|
||||
case AWS_LL_TRACE:
|
||||
return lvlVomit;
|
||||
// AWS_LL_NONE and AWS_LL_COUNT are enum sentinels, not real log levels
|
||||
case AWS_LL_NONE:
|
||||
case AWS_LL_COUNT:
|
||||
return lvlDebug;
|
||||
}
|
||||
unreachable();
|
||||
}
|
||||
|
||||
/**
|
||||
* Custom AWS logger that routes logs through Nix's logging infrastructure.
|
||||
*
|
||||
* The AWS CRT C++ wrapper (ApiHandle::InitializeLogging) only supports FILE*
|
||||
* or filename-based logging. The underlying C library supports custom loggers
|
||||
* via aws_logger struct with a vtable containing callback functions.
|
||||
*/
|
||||
static int nixAwsLoggerLog(
|
||||
struct aws_logger * logger, enum aws_log_level logLevel, aws_log_subject_t subject, const char * format, ...)
|
||||
{
|
||||
Verbosity nixLevel = awsLogLevelToVerbosity(logLevel);
|
||||
if (nixLevel > verbosity)
|
||||
return AWS_OP_SUCCESS; /* Bail out early to avoid formatting the message unnecessarily. */
|
||||
|
||||
va_list args;
|
||||
va_start(args, format);
|
||||
std::array<char, 4096> buffer{};
|
||||
auto res = vsnprintf(buffer.data(), buffer.size(), format, args);
|
||||
va_end(args);
|
||||
if (res < 0) /* Skip garbage debug messages in case the SDK is busted. */
|
||||
return AWS_OP_SUCCESS;
|
||||
|
||||
const char * subjectName = aws_log_subject_name(subject);
|
||||
printMsgUsing(nix::logger, nixLevel, "(aws:%s) %s", subjectName ? subjectName : "unknown", chomp(buffer.data()));
|
||||
return AWS_OP_SUCCESS;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get current log level for a subject - determines which messages will be logged.
|
||||
* Must be consistent with awsLogLevelToVerbosity mapping.
|
||||
*/
|
||||
static aws_log_level nixAwsLoggerGetLevel(struct aws_logger * logger, aws_log_subject_t subject)
|
||||
{
|
||||
// Map Nix verbosity back to AWS log level (inverse of awsLogLevelToVerbosity)
|
||||
if (verbosity >= lvlVomit)
|
||||
return AWS_LL_TRACE;
|
||||
if (verbosity >= lvlDebug)
|
||||
return AWS_LL_INFO; // error/warn/info are all mapped to lvlDebug
|
||||
return AWS_LL_FATAL;
|
||||
}
|
||||
|
||||
static void initialiseAwsLogger()
|
||||
{
|
||||
static std::once_flag initialised; /* aws_logger_set must only be called once */
|
||||
std::call_once(initialised, []() {
|
||||
static aws_logger_vtable nixAwsLoggerVtable = {
|
||||
.log = nixAwsLoggerLog,
|
||||
.get_log_level = nixAwsLoggerGetLevel,
|
||||
.clean_up = [](struct aws_logger *) {}, // No resources to clean up
|
||||
.set_log_level = nullptr,
|
||||
};
|
||||
|
||||
static aws_logger nixAwsLogger = {
|
||||
.vtable = &nixAwsLoggerVtable,
|
||||
.allocator = nullptr,
|
||||
.p_impl = nullptr,
|
||||
};
|
||||
|
||||
aws_logger_set(&nixAwsLogger);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper function to wrap a C credentials provider in the C++ interface.
|
||||
* This replicates the static s_CreateWrappedProvider from aws-crt-cpp.
|
||||
*/
|
||||
static std::shared_ptr<Aws::Crt::Auth::ICredentialsProvider> createWrappedProvider(
|
||||
aws_credentials_provider * rawProvider, Aws::Crt::Allocator * allocator = Aws::Crt::ApiAllocator())
|
||||
{
|
||||
if (rawProvider == nullptr) {
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
auto provider = Aws::Crt::MakeShared<Aws::Crt::Auth::CredentialsProvider>(allocator, rawProvider, allocator);
|
||||
return std::static_pointer_cast<Aws::Crt::Auth::ICredentialsProvider>(provider);
|
||||
}
|
||||
|
||||
/**
|
||||
* Create an SSO credentials provider using the C library directly.
|
||||
* The C++ wrapper doesn't expose SSO, so we call the C library and wrap the result.
|
||||
* Returns nullptr if SSO provider creation fails (e.g., profile doesn't have SSO config).
|
||||
*/
|
||||
static std::shared_ptr<Aws::Crt::Auth::ICredentialsProvider> createSSOProvider(
|
||||
const std::string & profileName,
|
||||
Aws::Crt::Io::ClientBootstrap * bootstrap,
|
||||
Aws::Crt::Io::TlsContext * tlsContext,
|
||||
Aws::Crt::Allocator * allocator = Aws::Crt::ApiAllocator())
|
||||
{
|
||||
aws_credentials_provider_sso_options options;
|
||||
AWS_ZERO_STRUCT(options);
|
||||
|
||||
options.bootstrap = bootstrap->GetUnderlyingHandle();
|
||||
options.tls_ctx = tlsContext ? tlsContext->GetUnderlyingHandle() : nullptr;
|
||||
if (!profileName.empty()) {
|
||||
options.profile_name_override = aws_byte_cursor_from_c_str(profileName.c_str());
|
||||
}
|
||||
|
||||
// Create the SSO provider - will return nullptr if SSO isn't configured for this profile
|
||||
// createWrappedProvider handles nullptr gracefully
|
||||
return createWrappedProvider(aws_credentials_provider_new_sso(allocator, &options), allocator);
|
||||
}
|
||||
|
||||
static AwsCredentials getCredentialsFromProvider(std::shared_ptr<Aws::Crt::Auth::ICredentialsProvider> provider)
|
||||
{
|
||||
if (!provider || !provider->IsValid()) {
|
||||
@@ -79,18 +219,25 @@ class AwsCredentialProviderImpl : public AwsCredentialProvider
|
||||
public:
|
||||
AwsCredentialProviderImpl()
|
||||
{
|
||||
// Map Nix's verbosity to AWS CRT log level
|
||||
Aws::Crt::LogLevel logLevel;
|
||||
if (verbosity >= lvlVomit) {
|
||||
logLevel = Aws::Crt::LogLevel::Trace;
|
||||
} else if (verbosity >= lvlDebug) {
|
||||
logLevel = Aws::Crt::LogLevel::Debug;
|
||||
} else if (verbosity >= lvlChatty) {
|
||||
logLevel = Aws::Crt::LogLevel::Info;
|
||||
} else {
|
||||
logLevel = Aws::Crt::LogLevel::Warn;
|
||||
// Install custom logger that routes AWS CRT logs through Nix's logging infrastructure.
|
||||
// This ensures AWS logs respect Nix's verbosity settings and are formatted consistently.
|
||||
initialiseAwsLogger();
|
||||
|
||||
// Create a shared TLS context for SSO (required for HTTPS connections)
|
||||
auto allocator = Aws::Crt::ApiAllocator();
|
||||
auto tlsCtxOptions = Aws::Crt::Io::TlsContextOptions::InitDefaultClient(allocator);
|
||||
tlsContext =
|
||||
std::make_shared<Aws::Crt::Io::TlsContext>(tlsCtxOptions, Aws::Crt::Io::TlsMode::CLIENT, allocator);
|
||||
if (!tlsContext || !*tlsContext) {
|
||||
warn("failed to create TLS context for AWS SSO; SSO authentication will be unavailable");
|
||||
tlsContext = nullptr;
|
||||
}
|
||||
|
||||
// Get bootstrap (lives as long as apiHandle)
|
||||
bootstrap = Aws::Crt::ApiHandle::GetOrCreateStaticDefaultClientBootstrap();
|
||||
if (!bootstrap) {
|
||||
throw AwsAuthError("failed to create AWS client bootstrap");
|
||||
}
|
||||
apiHandle.InitializeLogging(logLevel, stderr);
|
||||
}
|
||||
|
||||
AwsCredentials getCredentialsRaw(const std::string & profile);
|
||||
@@ -111,6 +258,8 @@ public:
|
||||
|
||||
private:
|
||||
Aws::Crt::ApiHandle apiHandle;
|
||||
std::shared_ptr<Aws::Crt::Io::TlsContext> tlsContext;
|
||||
Aws::Crt::Io::ClientBootstrap * bootstrap;
|
||||
boost::concurrent_flat_map<std::string, std::shared_ptr<Aws::Crt::Auth::ICredentialsProvider>>
|
||||
credentialProviderCache;
|
||||
};
|
||||
@@ -118,23 +267,58 @@ private:
|
||||
std::shared_ptr<Aws::Crt::Auth::ICredentialsProvider>
|
||||
AwsCredentialProviderImpl::createProviderForProfile(const std::string & profile)
|
||||
{
|
||||
debug(
|
||||
"[pid=%d] creating new AWS credential provider for profile '%s'",
|
||||
getpid(),
|
||||
profile.empty() ? "(default)" : profile.c_str());
|
||||
// profileDisplayName is only used for debug logging - SDK uses its default profile
|
||||
// when ProfileNameOverride is not set
|
||||
const char * profileDisplayName = profile.empty() ? "(default)" : profile.c_str();
|
||||
|
||||
if (profile.empty()) {
|
||||
Aws::Crt::Auth::CredentialsProviderChainDefaultConfig config;
|
||||
config.Bootstrap = Aws::Crt::ApiHandle::GetOrCreateStaticDefaultClientBootstrap();
|
||||
return Aws::Crt::Auth::CredentialsProvider::CreateCredentialsProviderChainDefault(config);
|
||||
debug("[pid=%d] creating new AWS credential provider for profile '%s'", getpid(), profileDisplayName);
|
||||
|
||||
// Build a custom credential chain: Environment → SSO → Profile → IMDS
|
||||
// This works for both default and named profiles, ensuring consistent behavior
|
||||
// including SSO support and proper TLS context for STS-based role assumption.
|
||||
Aws::Crt::Auth::CredentialsProviderChainConfig chainConfig;
|
||||
auto allocator = Aws::Crt::ApiAllocator();
|
||||
|
||||
auto addProviderToChain = [&](std::string_view name, auto createProvider) {
|
||||
if (auto provider = createProvider()) {
|
||||
chainConfig.Providers.push_back(provider);
|
||||
debug("Added AWS %s Credential Provider to chain for profile '%s'", name, profileDisplayName);
|
||||
} else {
|
||||
debug("Skipped AWS %s Credential Provider for profile '%s'", name, profileDisplayName);
|
||||
}
|
||||
};
|
||||
|
||||
// 1. Environment variables (highest priority)
|
||||
addProviderToChain("Environment", [&]() {
|
||||
return Aws::Crt::Auth::CredentialsProvider::CreateCredentialsProviderEnvironment(allocator);
|
||||
});
|
||||
|
||||
// 2. SSO provider (try it, will fail gracefully if not configured)
|
||||
if (tlsContext) {
|
||||
addProviderToChain("SSO", [&]() { return createSSOProvider(profile, bootstrap, tlsContext.get(), allocator); });
|
||||
} else {
|
||||
debug("Skipped AWS SSO Credential Provider for profile '%s': TLS context unavailable", profileDisplayName);
|
||||
}
|
||||
|
||||
Aws::Crt::Auth::CredentialsProviderProfileConfig config;
|
||||
config.Bootstrap = Aws::Crt::ApiHandle::GetOrCreateStaticDefaultClientBootstrap();
|
||||
// This is safe because the underlying C library will copy this string
|
||||
// c.f. https://github.com/awslabs/aws-c-auth/blob/main/source/credentials_provider_profile.c#L220
|
||||
config.ProfileNameOverride = Aws::Crt::ByteCursorFromCString(profile.c_str());
|
||||
return Aws::Crt::Auth::CredentialsProvider::CreateCredentialsProviderProfile(config);
|
||||
// 3. Profile provider (for static credentials and role_arn/source_profile with STS)
|
||||
addProviderToChain("Profile", [&]() {
|
||||
Aws::Crt::Auth::CredentialsProviderProfileConfig profileConfig;
|
||||
profileConfig.Bootstrap = bootstrap;
|
||||
profileConfig.TlsContext = tlsContext.get();
|
||||
if (!profile.empty()) {
|
||||
profileConfig.ProfileNameOverride = Aws::Crt::ByteCursorFromCString(profile.c_str());
|
||||
}
|
||||
return Aws::Crt::Auth::CredentialsProvider::CreateCredentialsProviderProfile(profileConfig, allocator);
|
||||
});
|
||||
|
||||
// 4. IMDS provider (for EC2 instances, lowest priority)
|
||||
addProviderToChain("IMDS", [&]() {
|
||||
Aws::Crt::Auth::CredentialsProviderImdsConfig imdsConfig;
|
||||
imdsConfig.Bootstrap = bootstrap;
|
||||
return Aws::Crt::Auth::CredentialsProvider::CreateCredentialsProviderImds(imdsConfig, allocator);
|
||||
});
|
||||
|
||||
return Aws::Crt::Auth::CredentialsProvider::CreateCredentialsProviderChain(chainConfig, allocator);
|
||||
}
|
||||
|
||||
AwsCredentials AwsCredentialProviderImpl::getCredentialsRaw(const std::string & profile)
|
||||
|
||||
@@ -563,8 +563,7 @@ Goal::Co DerivationBuildingGoal::tryToBuild()
|
||||
{
|
||||
DerivationBuildingGoal & goal;
|
||||
|
||||
DerivationBuildingGoalCallbacks(
|
||||
DerivationBuildingGoal & goal, std::unique_ptr<DerivationBuilder> & builder)
|
||||
DerivationBuildingGoalCallbacks(DerivationBuildingGoal & goal)
|
||||
: goal{goal}
|
||||
{
|
||||
}
|
||||
@@ -632,15 +631,15 @@ Goal::Co DerivationBuildingGoal::tryToBuild()
|
||||
|
||||
/* If we have to wait and retry (see below), then `builder` will
|
||||
already be created, so we don't need to create it again. */
|
||||
builder = externalBuilder ? makeExternalDerivationBuilder(
|
||||
*localStoreP,
|
||||
std::make_unique<DerivationBuildingGoalCallbacks>(*this, builder),
|
||||
std::move(params),
|
||||
*externalBuilder)
|
||||
: makeDerivationBuilder(
|
||||
*localStoreP,
|
||||
std::make_unique<DerivationBuildingGoalCallbacks>(*this, builder),
|
||||
std::move(params));
|
||||
builder =
|
||||
externalBuilder
|
||||
? makeExternalDerivationBuilder(
|
||||
*localStoreP,
|
||||
std::make_unique<DerivationBuildingGoalCallbacks>(*this),
|
||||
std::move(params),
|
||||
*externalBuilder)
|
||||
: makeDerivationBuilder(
|
||||
*localStoreP, std::make_unique<DerivationBuildingGoalCallbacks>(*this), std::move(params));
|
||||
}
|
||||
|
||||
if (auto builderOutOpt = builder->startBuild()) {
|
||||
@@ -1176,11 +1175,6 @@ DerivationBuildingGoal::checkPathValidity(std::map<std::string, InitialOutput> &
|
||||
|
||||
Goal::Done DerivationBuildingGoal::doneSuccess(BuildResult::Success::Status status, SingleDrvOutputs builtOutputs)
|
||||
{
|
||||
buildResult.inner = BuildResult::Success{
|
||||
.status = status,
|
||||
.builtOutputs = std::move(builtOutputs),
|
||||
};
|
||||
|
||||
mcRunningBuilds.reset();
|
||||
|
||||
if (status == BuildResult::Success::Built)
|
||||
@@ -1188,16 +1182,15 @@ Goal::Done DerivationBuildingGoal::doneSuccess(BuildResult::Success::Status stat
|
||||
|
||||
worker.updateProgress();
|
||||
|
||||
return amDone(ecSuccess, std::nullopt);
|
||||
return Goal::doneSuccess(
|
||||
BuildResult::Success{
|
||||
.status = status,
|
||||
.builtOutputs = std::move(builtOutputs),
|
||||
});
|
||||
}
|
||||
|
||||
Goal::Done DerivationBuildingGoal::doneFailure(BuildError ex)
|
||||
{
|
||||
buildResult.inner = BuildResult::Failure{
|
||||
.status = ex.status,
|
||||
.errorMsg = fmt("%s", Uncolored(ex.info().msg)),
|
||||
};
|
||||
|
||||
mcRunningBuilds.reset();
|
||||
|
||||
if (ex.status == BuildResult::Failure::TimedOut)
|
||||
@@ -1209,7 +1202,13 @@ Goal::Done DerivationBuildingGoal::doneFailure(BuildError ex)
|
||||
|
||||
worker.updateProgress();
|
||||
|
||||
return amDone(ecFailed, {std::move(ex)});
|
||||
return Goal::doneFailure(
|
||||
ecFailed,
|
||||
BuildResult::Failure{
|
||||
.status = ex.status,
|
||||
.errorMsg = fmt("%s", Uncolored(ex.info().msg)),
|
||||
},
|
||||
std::move(ex));
|
||||
}
|
||||
|
||||
} // namespace nix
|
||||
@@ -452,20 +452,6 @@ UnkeyedRealisation DerivationGoal::assertPathValidity()
|
||||
|
||||
Goal::Done DerivationGoal::doneSuccess(BuildResult::Success::Status status, UnkeyedRealisation builtOutput)
|
||||
{
|
||||
buildResult.inner = BuildResult::Success{
|
||||
.status = status,
|
||||
.builtOutputs = {{
|
||||
wantedOutput,
|
||||
{
|
||||
std::move(builtOutput),
|
||||
DrvOutput{
|
||||
.drvHash = outputHash,
|
||||
.outputName = wantedOutput,
|
||||
},
|
||||
},
|
||||
}},
|
||||
};
|
||||
|
||||
mcExpectedBuilds.reset();
|
||||
|
||||
if (status == BuildResult::Success::Built)
|
||||
@@ -473,16 +459,24 @@ Goal::Done DerivationGoal::doneSuccess(BuildResult::Success::Status status, Unke
|
||||
|
||||
worker.updateProgress();
|
||||
|
||||
return amDone(ecSuccess, std::nullopt);
|
||||
return Goal::doneSuccess(
|
||||
BuildResult::Success{
|
||||
.status = status,
|
||||
.builtOutputs = {{
|
||||
wantedOutput,
|
||||
{
|
||||
std::move(builtOutput),
|
||||
DrvOutput{
|
||||
.drvHash = outputHash,
|
||||
.outputName = wantedOutput,
|
||||
},
|
||||
},
|
||||
}},
|
||||
});
|
||||
}
|
||||
|
||||
Goal::Done DerivationGoal::doneFailure(BuildError ex)
|
||||
{
|
||||
buildResult.inner = BuildResult::Failure{
|
||||
.status = ex.status,
|
||||
.errorMsg = fmt("%s", Uncolored(ex.info().msg)),
|
||||
};
|
||||
|
||||
mcExpectedBuilds.reset();
|
||||
|
||||
if (ex.status == BuildResult::Failure::TimedOut)
|
||||
@@ -494,7 +488,13 @@ Goal::Done DerivationGoal::doneFailure(BuildError ex)
|
||||
|
||||
worker.updateProgress();
|
||||
|
||||
return amDone(ecFailed, {std::move(ex)});
|
||||
return Goal::doneFailure(
|
||||
ecFailed,
|
||||
BuildResult::Failure{
|
||||
.status = ex.status,
|
||||
.errorMsg = fmt("%s", Uncolored(ex.info().msg)),
|
||||
},
|
||||
std::move(ex));
|
||||
}
|
||||
|
||||
} // namespace nix
|
||||
@@ -98,7 +98,13 @@ Goal::Co DerivationTrampolineGoal::init()
|
||||
trace("outer load and build derivation");
|
||||
|
||||
if (nrFailed != 0) {
|
||||
co_return amDone(ecFailed, Error("cannot build missing derivation '%s'", drvReq->to_string(worker.store)));
|
||||
co_return doneFailure(
|
||||
ecFailed,
|
||||
BuildResult::Failure{
|
||||
.status = BuildResult::Failure::DependencyFailed,
|
||||
.errorMsg = fmt("failed to obtain derivation of '%s'", drvReq->to_string(worker.store)),
|
||||
},
|
||||
Error("failed to obtain derivation of '%s'", drvReq->to_string(worker.store)));
|
||||
}
|
||||
|
||||
StorePath drvPath = resolveDerivedPath(worker.store, *drvReq);
|
||||
|
||||
@@ -63,12 +63,18 @@ std::vector<KeyedBuildResult> Store::buildPathsWithResults(
|
||||
std::vector<KeyedBuildResult> results;
|
||||
results.reserve(state.size());
|
||||
|
||||
for (auto & [req, goalPtr] : state)
|
||||
for (auto & [req, goalPtr] : state) {
|
||||
/* Goals that were never started or were cancelled have exitCode
|
||||
ecBusy and a default buildResult with empty errorMsg. Skip them
|
||||
to avoid reporting spurious failures with empty messages. */
|
||||
if (goalPtr->exitCode == Goal::ecBusy)
|
||||
continue;
|
||||
results.emplace_back(
|
||||
KeyedBuildResult{
|
||||
goalPtr->buildResult,
|
||||
/* .path = */ req,
|
||||
});
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
@@ -133,6 +133,19 @@ Co Goal::await(Goals new_waitees)
|
||||
co_return Return{};
|
||||
}
|
||||
|
||||
Goal::Done Goal::doneSuccess(BuildResult::Success success)
|
||||
{
|
||||
buildResult.inner = std::move(success);
|
||||
return amDone(ecSuccess);
|
||||
}
|
||||
|
||||
Goal::Done Goal::doneFailure(ExitCode result, BuildResult::Failure failure, std::optional<Error> ex)
|
||||
{
|
||||
assert(result == ecFailed || result == ecNoSubstituters);
|
||||
buildResult.inner = std::move(failure);
|
||||
return amDone(result, std::move(ex));
|
||||
}
|
||||
|
||||
Goal::Done Goal::amDone(ExitCode result, std::optional<Error> ex)
|
||||
{
|
||||
trace("done");
|
||||
|
||||
@@ -29,20 +29,21 @@ PathSubstitutionGoal::~PathSubstitutionGoal()
|
||||
|
||||
Goal::Done PathSubstitutionGoal::doneSuccess(BuildResult::Success::Status status)
|
||||
{
|
||||
buildResult.inner = BuildResult::Success{
|
||||
.status = status,
|
||||
};
|
||||
return amDone(ecSuccess);
|
||||
return Goal::doneSuccess(
|
||||
BuildResult::Success{
|
||||
.status = status,
|
||||
});
|
||||
}
|
||||
|
||||
Goal::Done PathSubstitutionGoal::doneFailure(ExitCode result, BuildResult::Failure::Status status, std::string errorMsg)
|
||||
{
|
||||
debug(errorMsg);
|
||||
buildResult.inner = BuildResult::Failure{
|
||||
.status = status,
|
||||
.errorMsg = std::move(errorMsg),
|
||||
};
|
||||
return amDone(result);
|
||||
return Goal::doneFailure(
|
||||
result,
|
||||
BuildResult::Failure{
|
||||
.status = status,
|
||||
.errorMsg = std::move(errorMsg),
|
||||
});
|
||||
}
|
||||
|
||||
Goal::Co PathSubstitutionGoal::init()
|
||||
|
||||
@@ -852,7 +852,10 @@ static void performOp(
|
||||
auto path = WorkerProto::Serialise<StorePath>::read(*store, rconn);
|
||||
std::shared_ptr<const ValidPathInfo> info;
|
||||
logger->startWork();
|
||||
info = store->queryPathInfo(path);
|
||||
try {
|
||||
info = store->queryPathInfo(path);
|
||||
} catch (InvalidPath &) {
|
||||
}
|
||||
logger->stopWork();
|
||||
if (info) {
|
||||
conn.to << 1;
|
||||
|
||||
@@ -131,7 +131,16 @@ struct curlFileTransfer : public FileTransfer
|
||||
{
|
||||
result.urls.push_back(request.uri.to_string());
|
||||
|
||||
requestHeaders = curl_slist_append(requestHeaders, "Accept-Encoding: zstd, br, gzip, deflate, bzip2, xz");
|
||||
/* Don't set Accept-Encoding for S3 requests that use AWS SigV4 signing.
|
||||
curl's SigV4 implementation signs all headers including Accept-Encoding,
|
||||
but some S3-compatible services (like GCS) modify this header in transit,
|
||||
causing signature verification to fail.
|
||||
See https://github.com/NixOS/nix/issues/15019 */
|
||||
#if NIX_WITH_AWS_AUTH
|
||||
if (!request.awsSigV4Provider)
|
||||
#endif
|
||||
requestHeaders =
|
||||
curl_slist_append(requestHeaders, "Accept-Encoding: zstd, br, gzip, deflate, bzip2, xz");
|
||||
if (!request.expectedETag.empty())
|
||||
requestHeaders = curl_slist_append(requestHeaders, ("If-None-Match: " + request.expectedETag).c_str());
|
||||
if (!request.mimeType.empty())
|
||||
@@ -441,6 +450,11 @@ struct curlFileTransfer : public FileTransfer
|
||||
curl_easy_setopt(req, CURLOPT_CUSTOMREQUEST, "DELETE");
|
||||
|
||||
if (request.data) {
|
||||
// Restart the source to ensure it's at the beginning.
|
||||
// This is necessary for retries, where the source was
|
||||
// already consumed by a previous attempt.
|
||||
request.data->source->restart();
|
||||
|
||||
if (request.method == HttpMethod::Post) {
|
||||
curl_easy_setopt(req, CURLOPT_POST, 1L);
|
||||
curl_easy_setopt(req, CURLOPT_POSTFIELDSIZE_LARGE, (curl_off_t) request.data->sizeHint);
|
||||
@@ -465,6 +479,12 @@ struct curlFileTransfer : public FileTransfer
|
||||
|
||||
curl_easy_setopt(req, CURLOPT_CONNECTTIMEOUT, fileTransferSettings.connectTimeout.get());
|
||||
|
||||
// Enable TCP keep-alive so that idle connections in curl's reuse pool
|
||||
// are not silently dropped by NATs, firewalls, or load balancers.
|
||||
curl_easy_setopt(req, CURLOPT_TCP_KEEPALIVE, 1L);
|
||||
curl_easy_setopt(req, CURLOPT_TCP_KEEPIDLE, 60L);
|
||||
curl_easy_setopt(req, CURLOPT_TCP_KEEPINTVL, 60L);
|
||||
|
||||
curl_easy_setopt(req, CURLOPT_LOW_SPEED_LIMIT, 1L);
|
||||
curl_easy_setopt(req, CURLOPT_LOW_SPEED_TIME, fileTransferSettings.stalledDownloadTimeout.get());
|
||||
|
||||
|
||||
@@ -189,15 +189,22 @@ struct ExternalBuilder
|
||||
std::vector<std::string> args;
|
||||
};
|
||||
|
||||
struct DerivationBuilderDeleter
|
||||
{
|
||||
void operator()(DerivationBuilder * builder) noexcept;
|
||||
};
|
||||
|
||||
using DerivationBuilderUnique = std::unique_ptr<DerivationBuilder, DerivationBuilderDeleter>;
|
||||
|
||||
#ifndef _WIN32 // TODO enable `DerivationBuilder` on Windows
|
||||
std::unique_ptr<DerivationBuilder> makeDerivationBuilder(
|
||||
DerivationBuilderUnique makeDerivationBuilder(
|
||||
LocalStore & store, std::unique_ptr<DerivationBuilderCallbacks> miscMethods, DerivationBuilderParams params);
|
||||
|
||||
/**
|
||||
* @param handler Must be chosen such that it supports the given
|
||||
* derivation.
|
||||
*/
|
||||
std::unique_ptr<DerivationBuilder> makeExternalDerivationBuilder(
|
||||
DerivationBuilderUnique makeExternalDerivationBuilder(
|
||||
LocalStore & store,
|
||||
std::unique_ptr<DerivationBuilderCallbacks> miscMethods,
|
||||
DerivationBuilderParams params,
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
#include "nix/store/parsed-derivations.hh"
|
||||
#include "nix/store/derivation-options.hh"
|
||||
#include "nix/store/build/derivation-building-misc.hh"
|
||||
#include "nix/store/build/derivation-builder.hh"
|
||||
#include "nix/store/outputs-spec.hh"
|
||||
#include "nix/store/store-api.hh"
|
||||
#include "nix/store/pathlocks.hh"
|
||||
@@ -89,7 +90,7 @@ private:
|
||||
*/
|
||||
std::unique_ptr<HookInstance> hook;
|
||||
|
||||
std::unique_ptr<DerivationBuilder> builder;
|
||||
DerivationBuilderUnique builder;
|
||||
#endif
|
||||
|
||||
BuildMode buildMode;
|
||||
|
||||
@@ -393,9 +393,28 @@ protected:
|
||||
* Signals that the goal is done.
|
||||
* `co_return` the result. If you're not inside a coroutine, you can ignore
|
||||
* the return value safely.
|
||||
*
|
||||
* Prefer using `doneSuccess` or `doneFailure` instead, which ensure
|
||||
* `buildResult` is set correctly.
|
||||
*/
|
||||
Done amDone(ExitCode result, std::optional<Error> ex = {});
|
||||
|
||||
/**
|
||||
* Signals successful completion of the goal.
|
||||
* Sets `buildResult` and calls `amDone`.
|
||||
*/
|
||||
Done doneSuccess(BuildResult::Success success);
|
||||
|
||||
/**
|
||||
* Signals failed completion of the goal.
|
||||
* Sets `buildResult` and calls `amDone`.
|
||||
*
|
||||
* @param result The exit code (ecFailed or ecNoSubstituters)
|
||||
* @param failure The failure details including status and error message
|
||||
* @param ex Optional exception to store/log
|
||||
*/
|
||||
Done doneFailure(ExitCode result, BuildResult::Failure failure, std::optional<Error> ex = {});
|
||||
|
||||
public:
|
||||
virtual void cleanup() {}
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
|
||||
#include "nix/store/config.hh"
|
||||
#include "nix/store/http-binary-cache-store.hh"
|
||||
#include "nix/store/s3-url.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
@@ -52,13 +53,22 @@ struct S3BinaryCacheStoreConfig : HttpBinaryCacheStoreConfig
|
||||
"endpoint",
|
||||
R"(
|
||||
The S3 endpoint to use. When empty (default), uses AWS S3 with
|
||||
region-specific endpoints (e.g., s3.us-east-1.amazonaws.com).
|
||||
For S3-compatible services such as MinIO, set this to your service's endpoint.
|
||||
region-specific endpoints. For S3-compatible services such as
|
||||
MinIO, set this to your service's endpoint.
|
||||
)"};
|
||||
|
||||
> **Note**
|
||||
>
|
||||
> Custom endpoints must support HTTPS and use path-based
|
||||
> addressing instead of virtual host based addressing.
|
||||
Setting<S3AddressingStyle> addressingStyle{
|
||||
this,
|
||||
S3AddressingStyle::Auto,
|
||||
"addressing-style",
|
||||
R"(
|
||||
The S3 addressing style to use. `auto` (default) uses
|
||||
virtual-hosted-style for standard AWS endpoints and path-style
|
||||
for custom endpoints; bucket names containing dots automatically
|
||||
fall back to path-style to avoid TLS certificate errors. `path`
|
||||
forces path-style addressing (deprecated by AWS). `virtual`
|
||||
forces virtual-hosted-style addressing (bucket names must not
|
||||
contain dots).
|
||||
)"};
|
||||
|
||||
const Setting<bool> multipartUpload{
|
||||
@@ -117,7 +127,7 @@ struct S3BinaryCacheStoreConfig : HttpBinaryCacheStoreConfig
|
||||
* Set of settings that are part of the S3 URI itself.
|
||||
* These are needed for region specification and other S3-specific settings.
|
||||
*/
|
||||
const std::set<const AbstractSetting *> s3UriSettings = {&profile, ®ion, &scheme, &endpoint};
|
||||
const std::set<const AbstractSetting *> s3UriSettings = {&profile, ®ion, &scheme, &endpoint, &addressingStyle};
|
||||
|
||||
static const std::string name()
|
||||
{
|
||||
|
||||
@@ -1,16 +1,41 @@
|
||||
#pragma once
|
||||
///@file
|
||||
#include "nix/store/config.hh"
|
||||
#include "nix/util/error.hh"
|
||||
#include "nix/util/url.hh"
|
||||
#include "nix/util/util.hh"
|
||||
|
||||
#include <optional>
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
#include <variant>
|
||||
#include <vector>
|
||||
|
||||
namespace nix {
|
||||
|
||||
/**
|
||||
* S3 addressing style for bucket access.
|
||||
* - Auto: virtual-hosted-style for standard AWS endpoints, path-style for custom endpoints.
|
||||
* - Path: always use path-style (bucket in URL path).
|
||||
* - Virtual: always use virtual-hosted-style (bucket as hostname prefix; bucket name must not contain dots).
|
||||
*/
|
||||
enum class S3AddressingStyle {
|
||||
Auto,
|
||||
Path,
|
||||
Virtual,
|
||||
};
|
||||
|
||||
MakeError(InvalidS3AddressingStyle, Error);
|
||||
|
||||
S3AddressingStyle parseS3AddressingStyle(std::string_view style);
|
||||
std::string_view showS3AddressingStyle(S3AddressingStyle style);
|
||||
|
||||
template<>
|
||||
S3AddressingStyle BaseSetting<S3AddressingStyle>::parse(const std::string & str) const;
|
||||
|
||||
template<>
|
||||
std::string BaseSetting<S3AddressingStyle>::to_string() const;
|
||||
|
||||
/**
|
||||
* Parsed S3 URL.
|
||||
*/
|
||||
@@ -27,6 +52,7 @@ struct ParsedS3URL
|
||||
std::optional<std::string> region;
|
||||
std::optional<std::string> scheme;
|
||||
std::optional<std::string> versionId;
|
||||
std::optional<S3AddressingStyle> addressingStyle;
|
||||
/**
|
||||
* The endpoint can be either missing, be an absolute URI (with a scheme like `http:`)
|
||||
* or an authority (so an IP address or a registered name).
|
||||
@@ -46,7 +72,8 @@ struct ParsedS3URL
|
||||
static ParsedS3URL parse(const ParsedURL & uri);
|
||||
|
||||
/**
|
||||
* Convert this ParsedS3URL to HTTPS ParsedURL for use with curl's AWS SigV4 authentication
|
||||
* Convert this ParsedS3URL to an HTTP(S) ParsedURL for use with curl's AWS SigV4 authentication.
|
||||
* The scheme defaults to HTTPS but respects the 'scheme' setting and custom endpoint schemes.
|
||||
*/
|
||||
ParsedURL toHttpsUrl() const;
|
||||
|
||||
|
||||
@@ -160,6 +160,8 @@ if s3_aws_auth.enabled()
|
||||
deps_other += aws_crt_cpp
|
||||
aws_c_common = cxx.find_library('aws-c-common', required : true)
|
||||
deps_other += aws_c_common
|
||||
aws_c_auth = cxx.find_library('aws-c-auth', required : true)
|
||||
deps_other += aws_c_auth
|
||||
endif
|
||||
|
||||
configdata_pub.set('NIX_WITH_AWS_AUTH', s3_aws_auth.enabled().to_int())
|
||||
|
||||
+137
-9
@@ -1,8 +1,14 @@
|
||||
#include "nix/store/s3-url.hh"
|
||||
#include "nix/util/abstract-setting-to-json.hh"
|
||||
#include "nix/util/config-impl.hh"
|
||||
#include "nix/util/error.hh"
|
||||
#include "nix/util/logging.hh"
|
||||
#include "nix/util/json-impls.hh"
|
||||
#include "nix/util/split.hh"
|
||||
#include "nix/util/strings-inline.hh"
|
||||
|
||||
#include <atomic>
|
||||
#include <nlohmann/json.hpp>
|
||||
#include <ranges>
|
||||
#include <string_view>
|
||||
|
||||
@@ -10,6 +16,30 @@ using namespace std::string_view_literals;
|
||||
|
||||
namespace nix {
|
||||
|
||||
S3AddressingStyle parseS3AddressingStyle(std::string_view style)
|
||||
{
|
||||
if (style == "auto")
|
||||
return S3AddressingStyle::Auto;
|
||||
if (style == "path")
|
||||
return S3AddressingStyle::Path;
|
||||
if (style == "virtual")
|
||||
return S3AddressingStyle::Virtual;
|
||||
throw InvalidS3AddressingStyle("unknown S3 addressing style '%s', expected 'auto', 'path', or 'virtual'", style);
|
||||
}
|
||||
|
||||
std::string_view showS3AddressingStyle(S3AddressingStyle style)
|
||||
{
|
||||
switch (style) {
|
||||
case S3AddressingStyle::Auto:
|
||||
return "auto";
|
||||
case S3AddressingStyle::Path:
|
||||
return "path";
|
||||
case S3AddressingStyle::Virtual:
|
||||
return "virtual";
|
||||
}
|
||||
unreachable();
|
||||
}
|
||||
|
||||
ParsedS3URL ParsedS3URL::parse(const ParsedURL & parsed)
|
||||
try {
|
||||
if (parsed.scheme != "s3"sv)
|
||||
@@ -49,6 +79,9 @@ try {
|
||||
.region = getOptionalParam("region"),
|
||||
.scheme = getOptionalParam("scheme"),
|
||||
.versionId = getOptionalParam("versionId"),
|
||||
.addressingStyle = getOptionalParam("addressing-style").transform([](const std::string & s) {
|
||||
return parseS3AddressingStyle(s);
|
||||
}),
|
||||
.endpoint = [&]() -> decltype(ParsedS3URL::endpoint) {
|
||||
if (!endpoint)
|
||||
return std::monostate();
|
||||
@@ -65,6 +98,9 @@ try {
|
||||
} catch (BadURL & e) {
|
||||
e.addTrace({}, "while parsing S3 URI: '%s'", parsed.to_string());
|
||||
throw;
|
||||
} catch (InvalidS3AddressingStyle & e) {
|
||||
e.addTrace({}, "while parsing S3 URI: '%s'", parsed.to_string());
|
||||
throw;
|
||||
}
|
||||
|
||||
ParsedURL ParsedS3URL::toHttpsUrl() const
|
||||
@@ -80,41 +116,95 @@ ParsedURL ParsedS3URL::toHttpsUrl() const
|
||||
queryParams["versionId"] = *versionId;
|
||||
}
|
||||
|
||||
auto style = addressingStyle.value_or(S3AddressingStyle::Auto);
|
||||
|
||||
// Virtual-hosted-style prepends the bucket name to the hostname, so bucket
|
||||
// names containing dots produce multi-level subdomains (e.g.
|
||||
// my.bucket.s3.amazonaws.com) that break TLS wildcard certificate validation.
|
||||
// In auto mode, fall back to path-style; only error on explicit virtual.
|
||||
auto hasDottedBucket = bucket.find('.') != std::string::npos;
|
||||
auto useVirtualForEndpoint = [&](bool defaultVirtual) {
|
||||
auto useVirtual = defaultVirtual ? style != S3AddressingStyle::Path : style == S3AddressingStyle::Virtual;
|
||||
if (useVirtual && hasDottedBucket) {
|
||||
if (style == S3AddressingStyle::Virtual)
|
||||
throw Error(
|
||||
"bucket name '%s' contains a dot, which is incompatible with "
|
||||
"virtual-hosted-style addressing (causes TLS certificate errors); "
|
||||
"use 'addressing-style=path' or 'addressing-style=auto' instead",
|
||||
bucket);
|
||||
static std::atomic<bool> warnedDottedBucket{false};
|
||||
warnOnce(
|
||||
warnedDottedBucket,
|
||||
"bucket name '%s' contains a dot; falling back to path-style addressing "
|
||||
"(virtual-hosted-style requires non-dotted bucket names for TLS certificate validity); "
|
||||
"set 'addressing-style=path' to silence this warning",
|
||||
bucket);
|
||||
return false;
|
||||
}
|
||||
return useVirtual;
|
||||
};
|
||||
|
||||
// Handle endpoint configuration using std::visit
|
||||
return std::visit(
|
||||
overloaded{
|
||||
[&](const std::monostate &) {
|
||||
// No custom endpoint, use standard AWS S3 endpoint
|
||||
// No custom endpoint: use virtual-hosted-style by default (auto),
|
||||
// path-style when explicitly requested or for dotted bucket names.
|
||||
auto useVirtual = useVirtualForEndpoint(/* defaultVirtual = */ true);
|
||||
std::vector<std::string> path{""};
|
||||
path.push_back(bucket);
|
||||
if (!useVirtual)
|
||||
path.push_back(bucket);
|
||||
path.insert(path.end(), key.begin(), key.end());
|
||||
return ParsedURL{
|
||||
.scheme = std::string{schemeStr},
|
||||
.authority = ParsedURL::Authority{.host = "s3." + regionStr + ".amazonaws.com"},
|
||||
.authority =
|
||||
ParsedURL::Authority{
|
||||
.host = useVirtual ? bucket + ".s3." + regionStr + ".amazonaws.com"
|
||||
: "s3." + regionStr + ".amazonaws.com"},
|
||||
.path = std::move(path),
|
||||
.query = std::move(queryParams),
|
||||
};
|
||||
},
|
||||
[&](const ParsedURL::Authority & auth) {
|
||||
// Endpoint is just an authority (hostname/port)
|
||||
// Custom endpoint authority: use path-style by default (auto),
|
||||
// virtual-hosted-style only when explicitly requested (not for dotted buckets).
|
||||
auto useVirtual = useVirtualForEndpoint(/* defaultVirtual = */ false);
|
||||
if (useVirtual && auth.host.empty())
|
||||
throw Error(
|
||||
"cannot use virtual-hosted-style addressing with endpoint '%s' "
|
||||
"because it has no hostname; use 'addressing-style=path' instead",
|
||||
auth.to_string());
|
||||
std::vector<std::string> path{""};
|
||||
path.push_back(bucket);
|
||||
if (!useVirtual)
|
||||
path.push_back(bucket);
|
||||
path.insert(path.end(), key.begin(), key.end());
|
||||
return ParsedURL{
|
||||
.scheme = std::string{schemeStr},
|
||||
.authority = auth,
|
||||
.authority =
|
||||
useVirtual ? ParsedURL::Authority{.host = bucket + "." + auth.host, .port = auth.port} : auth,
|
||||
.path = std::move(path),
|
||||
.query = std::move(queryParams),
|
||||
};
|
||||
},
|
||||
[&](const ParsedURL & endpointUrl) {
|
||||
// Endpoint is already a ParsedURL (e.g., http://server:9000)
|
||||
// Full endpoint URL: use path-style by default (auto),
|
||||
// virtual-hosted-style only when explicitly requested (not for dotted buckets).
|
||||
auto useVirtual = useVirtualForEndpoint(/* defaultVirtual = */ false);
|
||||
if (useVirtual && (!endpointUrl.authority || endpointUrl.authority->host.empty()))
|
||||
throw Error(
|
||||
"cannot use virtual-hosted-style addressing with endpoint '%s' "
|
||||
"because it has no authority (hostname)",
|
||||
endpointUrl.to_string());
|
||||
auto path = endpointUrl.path;
|
||||
path.push_back(bucket);
|
||||
if (!useVirtual)
|
||||
path.push_back(bucket);
|
||||
path.insert(path.end(), key.begin(), key.end());
|
||||
return ParsedURL{
|
||||
.scheme = endpointUrl.scheme,
|
||||
.authority = endpointUrl.authority,
|
||||
.authority = useVirtual ? std::optional{ParsedURL::Authority{
|
||||
.host = bucket + "." + endpointUrl.authority->host,
|
||||
.port = endpointUrl.authority->port}}
|
||||
: endpointUrl.authority,
|
||||
.path = std::move(path),
|
||||
.query = std::move(queryParams),
|
||||
};
|
||||
@@ -123,4 +213,42 @@ ParsedURL ParsedS3URL::toHttpsUrl() const
|
||||
endpoint);
|
||||
}
|
||||
|
||||
void to_json(nlohmann::json & j, const S3AddressingStyle & e)
|
||||
{
|
||||
j = std::string{showS3AddressingStyle(e)};
|
||||
}
|
||||
|
||||
void from_json(const nlohmann::json & j, S3AddressingStyle & e)
|
||||
{
|
||||
e = parseS3AddressingStyle(j.get<std::string>());
|
||||
}
|
||||
|
||||
template<>
|
||||
struct json_avoids_null<S3AddressingStyle> : std::true_type
|
||||
{};
|
||||
|
||||
template<>
|
||||
S3AddressingStyle BaseSetting<S3AddressingStyle>::parse(const std::string & str) const
|
||||
{
|
||||
try {
|
||||
return parseS3AddressingStyle(str);
|
||||
} catch (InvalidS3AddressingStyle &) {
|
||||
throw UsageError("option '%s' has invalid value '%s', expected 'auto', 'path', or 'virtual'", name, str);
|
||||
}
|
||||
}
|
||||
|
||||
template<>
|
||||
std::string BaseSetting<S3AddressingStyle>::to_string() const
|
||||
{
|
||||
return std::string{showS3AddressingStyle(value)};
|
||||
}
|
||||
|
||||
template<>
|
||||
struct BaseSetting<S3AddressingStyle>::trait
|
||||
{
|
||||
static constexpr bool appendable = false;
|
||||
};
|
||||
|
||||
template class BaseSetting<S3AddressingStyle>;
|
||||
|
||||
} // namespace nix
|
||||
@@ -98,10 +98,13 @@ public:
|
||||
{
|
||||
}
|
||||
|
||||
~DerivationBuilderImpl()
|
||||
/**
|
||||
* Cleanup code to run when destroying any DerivationBuilderImpl implementation.
|
||||
*/
|
||||
void cleanupOnDestruction() noexcept
|
||||
{
|
||||
/* Careful: we should never ever throw an exception from a
|
||||
destructor. */
|
||||
noexcept function. */
|
||||
try {
|
||||
killChild();
|
||||
} catch (...) {
|
||||
@@ -1962,7 +1965,20 @@ StorePath DerivationBuilderImpl::makeFallbackPath(const StorePath & path)
|
||||
|
||||
namespace nix {
|
||||
|
||||
std::unique_ptr<DerivationBuilder> makeDerivationBuilder(
|
||||
void DerivationBuilderDeleter::operator()(DerivationBuilder * builder) noexcept
|
||||
{
|
||||
if (!builder) /* Idempotent and handles nullptr as any deleter must. */
|
||||
return;
|
||||
|
||||
if (auto builderImpl = dynamic_cast<DerivationBuilderImpl *>(builder))
|
||||
/* Note that this might call into virtual functions, which we can't do in a destructor of
|
||||
the DerivationBuilderImpl itself. */
|
||||
builderImpl->cleanupOnDestruction();
|
||||
|
||||
delete builder;
|
||||
}
|
||||
|
||||
std::unique_ptr<DerivationBuilder, DerivationBuilderDeleter> makeDerivationBuilder(
|
||||
LocalStore & store, std::unique_ptr<DerivationBuilderCallbacks> miscMethods, DerivationBuilderParams params)
|
||||
{
|
||||
bool useSandbox = false;
|
||||
@@ -2013,17 +2029,19 @@ std::unique_ptr<DerivationBuilder> makeDerivationBuilder(
|
||||
throw Error("feature 'uid-range' is only supported in sandboxed builds");
|
||||
|
||||
#ifdef __APPLE__
|
||||
return std::make_unique<DarwinDerivationBuilder>(store, std::move(miscMethods), std::move(params), useSandbox);
|
||||
return DerivationBuilderUnique(
|
||||
new DarwinDerivationBuilder(store, std::move(miscMethods), std::move(params), useSandbox));
|
||||
#elif defined(__linux__)
|
||||
if (useSandbox)
|
||||
return std::make_unique<ChrootLinuxDerivationBuilder>(store, std::move(miscMethods), std::move(params));
|
||||
return DerivationBuilderUnique(
|
||||
new ChrootLinuxDerivationBuilder(store, std::move(miscMethods), std::move(params)));
|
||||
|
||||
return std::make_unique<LinuxDerivationBuilder>(store, std::move(miscMethods), std::move(params));
|
||||
return DerivationBuilderUnique(new LinuxDerivationBuilder(store, std::move(miscMethods), std::move(params)));
|
||||
#else
|
||||
if (useSandbox)
|
||||
throw Error("sandboxing builds is not supported on this platform");
|
||||
|
||||
return std::make_unique<DerivationBuilderImpl>(store, std::move(miscMethods), std::move(params));
|
||||
return DerivationBuilderUnique(new DerivationBuilderImpl(store, std::move(miscMethods), std::move(params)));
|
||||
#endif
|
||||
}
|
||||
|
||||
|
||||
@@ -106,13 +106,14 @@ struct ExternalDerivationBuilder : DerivationBuilderImpl
|
||||
}
|
||||
};
|
||||
|
||||
std::unique_ptr<DerivationBuilder> makeExternalDerivationBuilder(
|
||||
DerivationBuilderUnique makeExternalDerivationBuilder(
|
||||
LocalStore & store,
|
||||
std::unique_ptr<DerivationBuilderCallbacks> miscMethods,
|
||||
DerivationBuilderParams params,
|
||||
const ExternalBuilder & handler)
|
||||
{
|
||||
return std::make_unique<ExternalDerivationBuilder>(store, std::move(miscMethods), std::move(params), handler);
|
||||
return DerivationBuilderUnique(
|
||||
new ExternalDerivationBuilder(store, std::move(miscMethods), std::move(params), handler));
|
||||
}
|
||||
|
||||
} // namespace nix
|
||||
@@ -15,4 +15,32 @@ TEST(alignUp, notAPowerOf2)
|
||||
ASSERT_DEATH({ alignUp(1u, 42); }, "alignment must be a power of 2");
|
||||
}
|
||||
|
||||
template<typename T>
|
||||
class alignUpOverflowTest : public ::testing::Test
|
||||
{};
|
||||
|
||||
using UnsignedTypes = ::testing::Types<uint8_t, uint16_t, uint32_t, uint64_t>;
|
||||
TYPED_TEST_SUITE(alignUpOverflowTest, UnsignedTypes);
|
||||
|
||||
TYPED_TEST(alignUpOverflowTest, lastSafeValue)
|
||||
{
|
||||
constexpr auto max = std::numeric_limits<TypeParam>::max();
|
||||
ASSERT_EQ(alignUp<TypeParam>(max - 15, 16), (max - 15) & ~TypeParam{15});
|
||||
ASSERT_NO_THROW(alignUp<TypeParam>(max - 15, 16));
|
||||
}
|
||||
|
||||
TYPED_TEST(alignUpOverflowTest, overflowThrows)
|
||||
{
|
||||
constexpr auto max = std::numeric_limits<TypeParam>::max();
|
||||
ASSERT_THROW(alignUp<TypeParam>(max - 14, 16), Error);
|
||||
ASSERT_THROW(alignUp<TypeParam>(max, 16), Error);
|
||||
ASSERT_THROW(alignUp<TypeParam>(max, 2), Error);
|
||||
}
|
||||
|
||||
TYPED_TEST(alignUpOverflowTest, alignmentOneNeverOverflows)
|
||||
{
|
||||
constexpr auto max = std::numeric_limits<TypeParam>::max();
|
||||
ASSERT_EQ(alignUp<TypeParam>(max, 1), max);
|
||||
}
|
||||
|
||||
} // namespace nix
|
||||
@@ -1,9 +1,11 @@
|
||||
#pragma once
|
||||
///@file
|
||||
|
||||
#include "nix/util/error.hh"
|
||||
|
||||
#include <cassert>
|
||||
#include <limits>
|
||||
#include <type_traits>
|
||||
#include <cstdint>
|
||||
#include <bit>
|
||||
|
||||
namespace nix {
|
||||
@@ -16,7 +18,10 @@ template<typename T>
|
||||
constexpr T alignUp(T val, unsigned alignment)
|
||||
{
|
||||
assert(std::has_single_bit(alignment) && "alignment must be a power of 2");
|
||||
T mask = ~(T{alignment} - 1u);
|
||||
assert(alignment <= std::numeric_limits<T>::max());
|
||||
T mask = ~(static_cast<T>(alignment) - 1u);
|
||||
if (val > std::numeric_limits<T>::max() - (alignment - 1)) /* Overflow check. */
|
||||
throw Error("can't align %d to %d: value is too large", val, alignment);
|
||||
return (val + alignment - 1) & mask;
|
||||
}
|
||||
|
||||
|
||||
+14
-9
@@ -150,18 +150,23 @@ public:
|
||||
|
||||
Verbosity verbosity = lvlInfo;
|
||||
|
||||
void writeToStderr(std::string_view s)
|
||||
static void writeFullLogging(Descriptor fd, std::string_view s)
|
||||
{
|
||||
try {
|
||||
writeFull(getStandardError(), s, false);
|
||||
writeFull(fd, s, false);
|
||||
} catch (SystemError & e) {
|
||||
/* Ignore failing writes to stderr. We need to ignore write
|
||||
errors to ensure that cleanup code that logs to stderr runs
|
||||
to completion if the other side of stderr has been closed
|
||||
unexpectedly. */
|
||||
/* Ignore failing logging writes. We need to ignore write
|
||||
errors to ensure that cleanup code that writes logs runs
|
||||
to completion if the other side of the logging fd has
|
||||
been closed unexpectedly. */
|
||||
}
|
||||
}
|
||||
|
||||
void writeToStderr(std::string_view s)
|
||||
{
|
||||
writeFullLogging(getStandardError(), s);
|
||||
}
|
||||
|
||||
std::unique_ptr<Logger> makeSimpleLogger(bool printBuildLogs)
|
||||
{
|
||||
return std::make_unique<SimpleLogger>(printBuildLogs);
|
||||
@@ -245,15 +250,15 @@ struct JSONLogger : Logger
|
||||
|
||||
void write(const nlohmann::json & json)
|
||||
{
|
||||
auto line =
|
||||
(includeNixPrefix ? "@nix " : "") + json.dump(-1, ' ', false, nlohmann::json::error_handler_t::replace);
|
||||
auto line = (includeNixPrefix ? "@nix " : "")
|
||||
+ json.dump(-1, ' ', false, nlohmann::json::error_handler_t::replace) + "\n";
|
||||
|
||||
/* Acquire a lock to prevent log messages from clobbering each
|
||||
other. */
|
||||
try {
|
||||
auto state(_state.lock());
|
||||
if (state->enabled)
|
||||
writeLine(fd, line);
|
||||
writeFullLogging(fd, line);
|
||||
} catch (...) {
|
||||
bool enabled = false;
|
||||
std::swap(_state.lock()->enabled, enabled);
|
||||
|
||||
@@ -8,9 +8,12 @@
|
||||
#include <unistd.h>
|
||||
#include <poll.h>
|
||||
|
||||
#if defined(__linux__)
|
||||
# include <sys/syscall.h> /* pull __NR_* definitions */
|
||||
#endif
|
||||
|
||||
#if defined(__linux__) && defined(__NR_openat2)
|
||||
# define HAVE_OPENAT2 1
|
||||
# include <sys/syscall.h>
|
||||
# include <linux/openat2.h>
|
||||
#else
|
||||
# define HAVE_OPENAT2 0
|
||||
@@ -323,7 +326,7 @@ Descriptor unix::openFileEnsureBeneathNoSymlinks(Descriptor dirFd, const CanonPa
|
||||
{
|
||||
assert(!path.rel().starts_with('/')); /* Just in case the invariant is somehow broken. */
|
||||
assert(!path.isRoot());
|
||||
#ifdef __linux__
|
||||
#if HAVE_OPENAT2
|
||||
auto maybeFd = linux::openat2(
|
||||
dirFd, path.rel_c_str(), flags, static_cast<uint64_t>(mode), RESOLVE_BENEATH | RESOLVE_NO_SYMLINKS);
|
||||
if (maybeFd) {
|
||||
|
||||
@@ -158,14 +158,24 @@ printf "" | nix build --no-link --stdin --json | jq --exit-status '. == []'
|
||||
printf "%s\n" "$drv^*" | nix build --no-link --stdin --json | jq --exit-status '.[0]|has("drvPath")'
|
||||
|
||||
# --keep-going and FOD
|
||||
out="$(nix build -f fod-failing.nix -L 2>&1)" && status=0 || status=$?
|
||||
test "$status" = 1
|
||||
# one "hash mismatch" error, one "build of ... failed"
|
||||
test "$(<<<"$out" grep -cE '^error:')" = 2
|
||||
if isDaemonNewer "2.34pre"; then
|
||||
# With the fix, cancelled goals are not reported as failures.
|
||||
# Use -j1 so only x1 starts and fails; x2, x3, x4 are cancelled.
|
||||
out="$(nix build -f fod-failing.nix -j1 -L 2>&1)" && status=0 || status=$?
|
||||
test "$status" = 1
|
||||
# Only the hash mismatch error for x1. Cancelled goals not reported.
|
||||
test "$(<<<"$out" grep -cE '^error:')" = 1
|
||||
# Regression test: error messages should not be empty (end with just "failed:")
|
||||
<<<"$out" grepQuietInverse -E "^error:.*failed: *$"
|
||||
else
|
||||
out="$(nix build -f fod-failing.nix -L 2>&1)" && status=0 || status=$?
|
||||
test "$status" = 1
|
||||
# At minimum, check that x1 is reported as failing
|
||||
<<<"$out" grepQuiet -E "error:.*-x1"
|
||||
fi
|
||||
<<<"$out" grepQuiet -E "hash mismatch in fixed-output derivation '.*-x1\\.drv'"
|
||||
<<<"$out" grepQuiet -vE "hash mismatch in fixed-output derivation '.*-x3\\.drv'"
|
||||
<<<"$out" grepQuiet -vE "hash mismatch in fixed-output derivation '.*-x2\\.drv'"
|
||||
<<<"$out" grepQuiet -E "error: build of '.*-x[1-4]\\.drv\\^out', '.*-x[1-4]\\.drv\\^out', '.*-x[1-4]\\.drv\\^out', '.*-x[1-4]\\.drv\\^out' failed"
|
||||
|
||||
out="$(nix build -f fod-failing.nix -L x1 x2 x3 --keep-going 2>&1)" && status=0 || status=$?
|
||||
test "$status" = 1
|
||||
@@ -203,3 +213,69 @@ else
|
||||
fi
|
||||
<<<"$out" grepQuiet -vE "hash mismatch in fixed-output derivation '.*-x3\\.drv'"
|
||||
<<<"$out" grepQuiet -vE "hash mismatch in fixed-output derivation '.*-x2\\.drv'"
|
||||
|
||||
# Regression test: cancelled builds should not be reported as failures
|
||||
# When fast-fail fails, slow and depends-on-slow are cancelled (not failed).
|
||||
# Only fast-fail should be reported as a failure.
|
||||
# Uses fifo for synchronization to ensure deterministic behavior.
|
||||
# Requires -j2 so slow and fast-fail run concurrently (fifo deadlocks if serialized).
|
||||
if isDaemonNewer "2.34pre" && canUseSandbox; then
|
||||
fifoDir="$TEST_ROOT/cancelled-builds-fifo"
|
||||
mkdir -p "$fifoDir"
|
||||
mkfifo "$fifoDir/fifo"
|
||||
chmod a+rw "$fifoDir/fifo"
|
||||
# When using a separate test store, we need sandbox-paths to access
|
||||
# the system store (where bash/coreutils live). On NixOS, the test
|
||||
# uses the system store directly, so this isn't needed (and would
|
||||
# conflict with input paths).
|
||||
sandboxPathsArg=()
|
||||
if ! isTestOnNixOS; then
|
||||
sandboxPathsArg=(--option sandbox-paths "/nix/store")
|
||||
fi
|
||||
out="$(nix flake check ./cancelled-builds --impure -L -j2 \
|
||||
--option sandbox true \
|
||||
"${sandboxPathsArg[@]}" \
|
||||
--option sandbox-build-dir /build-tmp \
|
||||
--option extra-sandbox-paths "/cancelled-builds-fifo=$fifoDir" \
|
||||
2>&1)" && status=0 || status=$?
|
||||
rm -rf "$fifoDir"
|
||||
test "$status" = 1
|
||||
# The error should be for fast-fail, not for cancelled goals
|
||||
<<<"$out" grepQuiet -E "Cannot build.*fast-fail"
|
||||
# Cancelled goals should NOT appear in error messages (but may appear in "will be built" list)
|
||||
<<<"$out" grepQuietInverse -E "^error:.*slow"
|
||||
<<<"$out" grepQuietInverse -E "^error:.*depends-on-slow"
|
||||
<<<"$out" grepQuietInverse -E "^error:.*depends-on-fail"
|
||||
# Error messages should not be empty (end with just "failed:")
|
||||
<<<"$out" grepQuietInverse -E "^error:.*failed: *$"
|
||||
|
||||
# Test that nix build follows the same rules (uses a slightly different code path)
|
||||
mkdir -p "$fifoDir"
|
||||
mkfifo "$fifoDir/fifo"
|
||||
chmod a+rw "$fifoDir/fifo"
|
||||
sandboxPathsArg=()
|
||||
if ! isTestOnNixOS; then
|
||||
sandboxPathsArg=(--option sandbox-paths "/nix/store")
|
||||
fi
|
||||
system=$(nix eval --raw --impure --expr builtins.currentSystem)
|
||||
out="$(nix build --impure -L -j2 \
|
||||
--option sandbox true \
|
||||
"${sandboxPathsArg[@]}" \
|
||||
--option sandbox-build-dir /build-tmp \
|
||||
--option extra-sandbox-paths "/cancelled-builds-fifo=$fifoDir" \
|
||||
"./cancelled-builds#checks.$system.slow" \
|
||||
"./cancelled-builds#checks.$system.depends-on-slow" \
|
||||
"./cancelled-builds#checks.$system.fast-fail" \
|
||||
"./cancelled-builds#checks.$system.depends-on-fail" \
|
||||
2>&1)" && status=0 || status=$?
|
||||
rm -rf "$fifoDir"
|
||||
test "$status" = 1
|
||||
# The error should be for fast-fail, not for cancelled goals
|
||||
<<<"$out" grepQuiet -E "Cannot build.*fast-fail"
|
||||
# Cancelled goals should NOT appear in error messages
|
||||
<<<"$out" grepQuietInverse -E "^error:.*slow"
|
||||
<<<"$out" grepQuietInverse -E "^error:.*depends-on-slow"
|
||||
<<<"$out" grepQuietInverse -E "^error:.*depends-on-fail"
|
||||
# Error messages should not be empty (end with just "failed:")
|
||||
<<<"$out" grepQuietInverse -E "^error:.*failed: *$"
|
||||
fi
|
||||
@@ -0,0 +1,64 @@
|
||||
# Regression test for cancelled builds not being reported as failures.
|
||||
#
|
||||
# Scenario: When a build fails while other builds are running, those other
|
||||
# builds (and their dependents) get cancelled. Previously, cancelled builds
|
||||
# were incorrectly reported as failures with empty error messages.
|
||||
#
|
||||
# Uses a fifo for synchronization: fast-fail waits for slow to start before
|
||||
# failing, ensuring slow is actually running when it gets cancelled.
|
||||
#
|
||||
# See: tests/functional/build.sh (search for "cancelled-builds")
|
||||
{
|
||||
outputs =
|
||||
{ self }:
|
||||
let
|
||||
config = import "${builtins.getEnv "_NIX_TEST_BUILD_DIR"}/config.nix";
|
||||
in
|
||||
with config;
|
||||
{
|
||||
checks.${system} = {
|
||||
# A derivation that signals it started via fifo, then waits
|
||||
slow = mkDerivation {
|
||||
name = "slow";
|
||||
buildCommand = ''
|
||||
echo "slow: started, signaling via fifo"
|
||||
echo started > /cancelled-builds-fifo/fifo
|
||||
echo "slow: waiting..."
|
||||
sleep 10
|
||||
touch $out
|
||||
'';
|
||||
};
|
||||
|
||||
# Depends on slow - will be cancelled when fast-fail fails
|
||||
depends-on-slow = mkDerivation {
|
||||
name = "depends-on-slow";
|
||||
slow = self.checks.${system}.slow;
|
||||
buildCommand = ''
|
||||
echo "depends-on-slow: slow finished at $slow"
|
||||
touch $out
|
||||
'';
|
||||
};
|
||||
|
||||
# Waits for slow to start via fifo, then fails
|
||||
fast-fail = mkDerivation {
|
||||
name = "fast-fail";
|
||||
buildCommand = ''
|
||||
echo "fast-fail: waiting for slow to start..."
|
||||
read line < /cancelled-builds-fifo/fifo
|
||||
echo "fast-fail: slow started, now failing" >&2
|
||||
exit 1
|
||||
'';
|
||||
};
|
||||
|
||||
# Depends on fast-fail - will fail with DependencyFailed
|
||||
depends-on-fail = mkDerivation {
|
||||
name = "depends-on-fail";
|
||||
fast-fail = self.checks.${system}.fast-fail;
|
||||
buildCommand = ''
|
||||
echo "depends-on-fail: fast-fail finished (should never get here)"
|
||||
touch $out
|
||||
'';
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -10,3 +10,36 @@ if [[ -v NIX_DAEMON_PACKAGE ]]; then expected=1; fi # work around the daemon not
|
||||
|
||||
expectStderr "$expected" nix-build ./text-hashed-output.nix -A failingWrapper --no-out-link \
|
||||
| grepQuiet "build of resolved derivation '.*use-dynamic-drv-in-non-dynamic-drv-wrong.drv' failed"
|
||||
|
||||
# Test that error messages are not empty when a producer derivation fails.
|
||||
# This exercises the nrFailed path in DerivationTrampolineGoal::init().
|
||||
#
|
||||
# Using `nix build --expr` with builtins.outputOf creates a top-level
|
||||
# DerivationTrampolineGoal that goes through buildPathsWithResults.
|
||||
# When the producer fails, the nrFailed path must use doneFailure (not amDone)
|
||||
# to set buildResult.inner with a proper error message.
|
||||
#
|
||||
# Without the fix, the error message would be empty because amDone doesn't
|
||||
# set buildResult.inner, so rethrow() throws Error("") - an empty message.
|
||||
|
||||
out=$(nix build --impure --no-link --expr '
|
||||
let
|
||||
config = import (builtins.getEnv "_NIX_TEST_BUILD_DIR" + "/config.nix");
|
||||
inherit (config) mkDerivation;
|
||||
|
||||
# A CA derivation that fails before producing a .drv
|
||||
failingProducer = mkDerivation {
|
||||
name = "failing-producer";
|
||||
buildCommand = "echo This producer fails; exit 1";
|
||||
__contentAddressed = true;
|
||||
outputHashMode = "text";
|
||||
outputHashAlgo = "sha256";
|
||||
};
|
||||
in
|
||||
# Build the dynamic derivation output directly - this creates a top-level
|
||||
# DerivationTrampolineGoal, not a nested one inside a DerivationGoal
|
||||
builtins.outputOf failingProducer.outPath "out"
|
||||
' 2>&1) || true
|
||||
|
||||
# The error message must NOT be empty - it should mention the failed derivation
|
||||
echo "$out" | grepQuiet "failed to obtain derivation of"
|
||||
@@ -48,9 +48,14 @@ in
|
||||
rootCredentialsFile = pkgs.writeText "minio-credentials-full" ''
|
||||
MINIO_ROOT_USER=${accessKey}
|
||||
MINIO_ROOT_PASSWORD=${secretKey}
|
||||
MINIO_DOMAIN=minio.local
|
||||
'';
|
||||
};
|
||||
networking.firewall.allowedTCPPorts = [ 9000 ];
|
||||
# Static hosts for virtual-hosted-style S3 tests.
|
||||
# MinIO with MINIO_DOMAIN=minio.local accepts virtual-hosted requests
|
||||
# where the bucket name is a hostname prefix.
|
||||
networking.extraHosts = "127.0.0.1 vhost-test.minio.local minio.local";
|
||||
};
|
||||
|
||||
client =
|
||||
@@ -62,6 +67,7 @@ in
|
||||
experimental-features = nix-command
|
||||
substituters =
|
||||
'';
|
||||
networking.extraHosts = "192.168.1.2 vhost-test.minio.local minio.local";
|
||||
};
|
||||
};
|
||||
|
||||
@@ -83,6 +89,10 @@ in
|
||||
ENDPOINT = 'http://server:9000'
|
||||
REGION = 'eu-west-1'
|
||||
|
||||
# Virtual-hosted-style configuration (requires MINIO_DOMAIN and static host entries)
|
||||
VHOST_DOMAIN = 'minio.local'
|
||||
VHOST_ENDPOINT = f'http://{VHOST_DOMAIN}:9000'
|
||||
|
||||
PKGS = {
|
||||
'A': '${pkgA}',
|
||||
'B': '${pkgB}',
|
||||
@@ -147,7 +157,7 @@ in
|
||||
else:
|
||||
machine.fail(f"nix path-info {pkg}")
|
||||
|
||||
def setup_s3(populate_bucket=[], public=False, versioned=False):
|
||||
def setup_s3(populate_bucket=[], public=False, versioned=False, profiles=None):
|
||||
"""
|
||||
Decorator that creates/destroys a unique bucket for each test.
|
||||
Optionally pre-populates bucket with specified packages.
|
||||
@@ -157,9 +167,22 @@ in
|
||||
populate_bucket: List of packages to upload before test runs
|
||||
public: If True, make the bucket publicly accessible
|
||||
versioned: If True, enable versioning on the bucket before populating
|
||||
profiles: Dict of AWS profiles to create, e.g.:
|
||||
{"valid": {"access_key": "...", "secret_key": "..."},
|
||||
"invalid": {"access_key": "WRONG", "secret_key": "WRONG"}}
|
||||
Profiles are created on the client machine at /root/.aws/credentials
|
||||
"""
|
||||
def decorator(test_func):
|
||||
def wrapper():
|
||||
# Restart nix-daemon on both machines to clear the credential provider cache.
|
||||
# The AwsCredentialProviderImpl singleton persists in the daemon process,
|
||||
# and its cache can cause credentials from previous tests to be reused.
|
||||
# We reset-failed first to avoid systemd's start rate limiting.
|
||||
server.succeed("systemctl reset-failed nix-daemon.service nix-daemon.socket")
|
||||
server.succeed("systemctl restart nix-daemon")
|
||||
client.succeed("systemctl reset-failed nix-daemon.service nix-daemon.socket")
|
||||
client.succeed("systemctl restart nix-daemon")
|
||||
|
||||
bucket = str(uuid.uuid4())
|
||||
server.succeed(f"mc mb minio/{bucket}")
|
||||
try:
|
||||
@@ -167,6 +190,15 @@ in
|
||||
server.succeed(f"mc anonymous set download minio/{bucket}")
|
||||
if versioned:
|
||||
server.succeed(f"mc version enable minio/{bucket}")
|
||||
if profiles:
|
||||
# Build credentials file content
|
||||
creds_content = ""
|
||||
for name, creds in profiles.items():
|
||||
creds_content += f"[{name}]\n"
|
||||
creds_content += f"aws_access_key_id = {creds['access_key']}\n"
|
||||
creds_content += f"aws_secret_access_key = {creds['secret_key']}\n\n"
|
||||
client.succeed("mkdir -p /root/.aws")
|
||||
client.succeed(f"cat > /root/.aws/credentials << 'AWSCREDS'\n{creds_content}AWSCREDS")
|
||||
if populate_bucket:
|
||||
store_url = make_s3_url(bucket)
|
||||
for pkg in populate_bucket:
|
||||
@@ -174,6 +206,9 @@ in
|
||||
test_func(bucket)
|
||||
finally:
|
||||
server.succeed(f"mc rb --force minio/{bucket}")
|
||||
# Clean up AWS profiles if created
|
||||
if profiles:
|
||||
client.succeed("rm -rf /root/.aws")
|
||||
# Clean up client store - only delete if path exists
|
||||
for pkg in PKGS.values():
|
||||
client.succeed(f"[ ! -e {pkg} ] || nix store delete --ignore-liveness {pkg}")
|
||||
@@ -202,7 +237,40 @@ in
|
||||
"Credential provider caching failed"
|
||||
)
|
||||
|
||||
print("✓ Credential provider created once and cached")
|
||||
@setup_s3()
|
||||
def test_aws_log_integration(bucket):
|
||||
"""Test that AWS SDK logs are properly routed through Nix logger"""
|
||||
print("\n=== Testing AWS Log Integration ===")
|
||||
|
||||
store_url = make_s3_url(bucket)
|
||||
|
||||
# With default verbosity, AWS noise should NOT appear
|
||||
# All AWS messages are demoted to lvlDebug or lvlVomit
|
||||
output_default = server.succeed(
|
||||
f"{ENV_WITH_CREDS} nix copy --to '{store_url}' {PKGS['A']} 2>&1"
|
||||
)
|
||||
|
||||
if "(aws:" in output_default:
|
||||
print("Output at default verbosity:")
|
||||
print(output_default)
|
||||
raise Exception("Found AWS noise at default verbosity")
|
||||
|
||||
# With --debug (lvlDebug), we should see AWS messages with (aws:subject) prefix
|
||||
output_debug = server.succeed(
|
||||
f"{ENV_WITH_CREDS} nix copy --debug --to '{store_url}' {PKGS['B']} 2>&1"
|
||||
)
|
||||
|
||||
# Check for the (aws:subject) prefix format
|
||||
if "(aws:" not in output_debug:
|
||||
print("Output at --debug verbosity:")
|
||||
print(output_debug)
|
||||
raise Exception("Expected to see (aws:subject) prefix in debug output")
|
||||
|
||||
# Should also see Nix's own credential provider creation message
|
||||
if "creating new AWS credential provider" not in output_debug:
|
||||
print("Debug output:")
|
||||
print(output_debug)
|
||||
raise Exception("Expected to see credential provider creation at debug level")
|
||||
|
||||
@setup_s3(populate_bucket=[PKGS['A']])
|
||||
def test_fetchurl_basic(bucket):
|
||||
@@ -218,8 +286,6 @@ in
|
||||
f"'builtins.fetchurl {{ name = \"foo\"; url = \"{cache_info_url}\"; }}'"
|
||||
)
|
||||
|
||||
print("✓ builtins.fetchurl works with s3:// URLs")
|
||||
|
||||
@setup_s3()
|
||||
def test_error_message_formatting(bucket):
|
||||
"""Verify error messages display URLs correctly"""
|
||||
@@ -238,8 +304,6 @@ in
|
||||
print(error_msg)
|
||||
raise Exception("Error message formatting failed - should show actual URL, not %s placeholder")
|
||||
|
||||
print("✓ Error messages format URLs correctly")
|
||||
|
||||
@setup_s3(populate_bucket=[PKGS['A']])
|
||||
def test_fork_credential_preresolution(bucket):
|
||||
"""Test credential pre-resolution in forked processes"""
|
||||
@@ -279,8 +343,6 @@ in
|
||||
print(output)
|
||||
raise Exception("Expected to find FileTransfer creation in forked process")
|
||||
|
||||
print(" ✓ Forked process creates fresh FileTransfer")
|
||||
|
||||
# Verify pre-resolution in parent
|
||||
required_messages = [
|
||||
"Pre-resolving AWS credentials for S3 URL in builtin:fetchurl",
|
||||
@@ -293,8 +355,6 @@ in
|
||||
print(output)
|
||||
raise Exception(f"Missing expected message: {msg}")
|
||||
|
||||
print(" ✓ Parent pre-resolves credentials")
|
||||
|
||||
# Verify child uses pre-resolved credentials
|
||||
if "Using pre-resolved AWS credentials from parent process" not in output:
|
||||
print("Debug output:")
|
||||
@@ -318,8 +378,6 @@ in
|
||||
print(output)
|
||||
raise Exception(f"Child process (pid={child_pid}) should NOT create new credential providers")
|
||||
|
||||
print(" ✓ Child uses pre-resolved credentials (no new providers)")
|
||||
|
||||
@setup_s3(populate_bucket=[PKGS['A'], PKGS['B'], PKGS['C']])
|
||||
def test_store_operations(bucket):
|
||||
"""Test nix store info and copy operations"""
|
||||
@@ -337,8 +395,6 @@ in
|
||||
if not store_info.get("url"):
|
||||
raise Exception("Store should have a URL")
|
||||
|
||||
print(f" ✓ Store URL: {store_info['url']}")
|
||||
|
||||
# Test copy from store
|
||||
verify_packages_in_store(client, PKGS['A'], should_exist=False)
|
||||
|
||||
@@ -356,9 +412,6 @@ in
|
||||
|
||||
verify_packages_in_store(client, [PKGS['A'], PKGS['B'], PKGS['C']])
|
||||
|
||||
print(" ✓ nix copy works")
|
||||
print(" ✓ Credentials cached on client")
|
||||
|
||||
@setup_s3(populate_bucket=[PKGS['A'], PKGS['B']], public=True)
|
||||
def test_public_bucket_operations(bucket):
|
||||
"""Test store operations on public bucket without credentials"""
|
||||
@@ -368,7 +421,6 @@ in
|
||||
|
||||
# Verify store info works without credentials
|
||||
client.succeed(f"nix store info --store '{store_url}' >&2")
|
||||
print(" ✓ nix store info works without credentials")
|
||||
|
||||
# Get and validate store info JSON
|
||||
info_json = client.succeed(f"nix store info --json --store '{store_url}'")
|
||||
@@ -377,8 +429,6 @@ in
|
||||
if not store_info.get("url"):
|
||||
raise Exception("Store should have a URL")
|
||||
|
||||
print(f" ✓ Store URL: {store_info['url']}")
|
||||
|
||||
# Verify packages are not yet in client store
|
||||
verify_packages_in_store(client, [PKGS['A'], PKGS['B']], should_exist=False)
|
||||
|
||||
@@ -391,8 +441,6 @@ in
|
||||
# Verify packages were copied successfully
|
||||
verify_packages_in_store(client, [PKGS['A'], PKGS['B']])
|
||||
|
||||
print(" ✓ nix copy from public bucket works without credentials")
|
||||
|
||||
@setup_s3(populate_bucket=[PKGS['A']])
|
||||
def test_url_format_variations(bucket):
|
||||
"""Test different S3 URL parameter combinations"""
|
||||
@@ -401,12 +449,10 @@ in
|
||||
# Test parameter order variation (region before endpoint)
|
||||
url1 = f"s3://{bucket}?region={REGION}&endpoint={ENDPOINT}"
|
||||
client.succeed(f"{ENV_WITH_CREDS} nix store info --store '{url1}' >&2")
|
||||
print(" ✓ Parameter order: region before endpoint works")
|
||||
|
||||
# Test parameter order variation (endpoint before region)
|
||||
url2 = f"s3://{bucket}?endpoint={ENDPOINT}®ion={REGION}"
|
||||
client.succeed(f"{ENV_WITH_CREDS} nix store info --store '{url2}' >&2")
|
||||
print(" ✓ Parameter order: endpoint before region works")
|
||||
|
||||
@setup_s3(populate_bucket=[PKGS['A']])
|
||||
def test_concurrent_fetches(bucket):
|
||||
@@ -460,9 +506,6 @@ in
|
||||
providers_created = output.count("creating new AWS credential provider")
|
||||
transfers_created = output.count("builtin:fetchurl creating fresh FileTransfer instance")
|
||||
|
||||
print(f" ✓ {providers_created} credential providers created")
|
||||
print(f" ✓ {transfers_created} FileTransfer instances created")
|
||||
|
||||
if transfers_created != 5:
|
||||
print("Debug output:")
|
||||
print(output)
|
||||
@@ -488,14 +531,10 @@ in
|
||||
pkg_hash = get_package_hash(PKGS['B'])
|
||||
verify_content_encoding(server, bucket, f"{pkg_hash}.narinfo", "gzip")
|
||||
|
||||
print(" ✓ .narinfo has Content-Encoding: gzip")
|
||||
|
||||
# Verify client can download and decompress
|
||||
client.succeed(f"{ENV_WITH_CREDS} nix copy --from '{store_url}' --no-check-sigs {PKGS['B']}")
|
||||
verify_packages_in_store(client, PKGS['B'])
|
||||
|
||||
print(" ✓ Client decompressed .narinfo successfully")
|
||||
|
||||
@setup_s3()
|
||||
def test_compression_mixed(bucket):
|
||||
"""Test mixed compression (narinfo=xz, ls=gzip)"""
|
||||
@@ -512,18 +551,14 @@ in
|
||||
|
||||
# Verify .narinfo has xz compression
|
||||
verify_content_encoding(server, bucket, f"{pkg_hash}.narinfo", "xz")
|
||||
print(" ✓ .narinfo has Content-Encoding: xz")
|
||||
|
||||
# Verify .ls has gzip compression
|
||||
verify_content_encoding(server, bucket, f"{pkg_hash}.ls", "gzip")
|
||||
print(" ✓ .ls has Content-Encoding: gzip")
|
||||
|
||||
# Verify client can download with mixed compression
|
||||
client.succeed(f"{ENV_WITH_CREDS} nix copy --from '{store_url}' --no-check-sigs {PKGS['C']}")
|
||||
verify_packages_in_store(client, PKGS['C'])
|
||||
|
||||
print(" ✓ Client downloaded package with mixed compression")
|
||||
|
||||
@setup_s3()
|
||||
def test_compression_disabled(bucket):
|
||||
"""Verify no compression by default"""
|
||||
@@ -535,8 +570,6 @@ in
|
||||
pkg_hash = get_package_hash(PKGS['A'])
|
||||
verify_no_compression(server, bucket, f"{pkg_hash}.narinfo")
|
||||
|
||||
print(" ✓ No compression applied by default")
|
||||
|
||||
@setup_s3()
|
||||
def test_nix_prefetch_url(bucket):
|
||||
"""Test that nix-prefetch-url retrieves actual file content from S3, not empty files (issue #8862)"""
|
||||
@@ -556,8 +589,6 @@ in
|
||||
"nix hash file --type sha256 --base32 /tmp/test-file.txt"
|
||||
).strip()
|
||||
|
||||
print(f" ✓ Uploaded test file to S3 ({test_file_size} bytes)")
|
||||
|
||||
# Use nix-prefetch-url to download from S3
|
||||
s3_url = make_s3_url(bucket, path="/test-file.txt")
|
||||
|
||||
@@ -577,8 +608,6 @@ in
|
||||
f"Hash mismatch: expected {expected_hash}, got {prefetch_hash}"
|
||||
)
|
||||
|
||||
print(" ✓ nix-prefetch-url completed with correct hash")
|
||||
|
||||
# Verify the downloaded file is NOT empty (the bug in #8862)
|
||||
file_size = int(client.succeed(f"stat -c %s {store_path}").strip())
|
||||
|
||||
@@ -590,16 +619,12 @@ in
|
||||
f"File size mismatch: expected {test_file_size}, got {file_size}"
|
||||
)
|
||||
|
||||
print(f" ✓ File has correct size ({file_size} bytes, not empty)")
|
||||
|
||||
# Verify actual content matches by comparing hashes instead of printing entire file
|
||||
downloaded_hash = client.succeed(f"nix hash file --type sha256 --base32 {store_path}").strip()
|
||||
|
||||
if downloaded_hash != expected_hash:
|
||||
raise Exception(f"Content hash mismatch: expected {expected_hash}, got {downloaded_hash}")
|
||||
|
||||
print(" ✓ File content verified correct (hash matches)")
|
||||
|
||||
@setup_s3(populate_bucket=[PKGS['A']], versioned=True)
|
||||
def test_versioned_urls(bucket):
|
||||
"""Test that versionId parameter is accepted in S3 URLs"""
|
||||
@@ -613,7 +638,6 @@ in
|
||||
f"{ENV_WITH_CREDS} nix eval --impure --expr "
|
||||
f"'builtins.fetchurl {{ name = \"cache-info\"; url = \"{cache_info_url}\"; }}'"
|
||||
)
|
||||
print(" ✓ Fetch without versionId works")
|
||||
|
||||
# List versions to get a version ID
|
||||
# MinIO output format: [timestamp] size tier versionId versionNumber method filename
|
||||
@@ -627,8 +651,6 @@ in
|
||||
raise Exception("Could not extract version ID from MinIO output")
|
||||
|
||||
version_id = version_match.group(1)
|
||||
print(f" ✓ Found version ID: {version_id}")
|
||||
|
||||
# Version ID should not be "null" since versioning was enabled before upload
|
||||
if version_id == "null":
|
||||
raise Exception("Version ID is 'null' - versioning may not be working correctly")
|
||||
@@ -639,7 +661,6 @@ in
|
||||
f"{ENV_WITH_CREDS} nix eval --impure --expr "
|
||||
f"'builtins.fetchurl {{ name = \"cache-info-versioned\"; url = \"{versioned_url}\"; }}'"
|
||||
)
|
||||
print(" ✓ Fetch with versionId parameter works")
|
||||
|
||||
@setup_s3()
|
||||
def test_multipart_upload_basic(bucket):
|
||||
@@ -675,13 +696,9 @@ in
|
||||
print(output)
|
||||
raise Exception(f"Expected '{expected_msg}' in output")
|
||||
|
||||
print(f" ✓ Multipart upload used with {expected_parts} parts")
|
||||
|
||||
client.succeed(f"{ENV_WITH_CREDS} nix copy --from '{store_url}' {large_pkg} --no-check-sigs")
|
||||
verify_packages_in_store(client, large_pkg, should_exist=True)
|
||||
|
||||
print(" ✓ Large file downloaded and verified")
|
||||
|
||||
@setup_s3()
|
||||
def test_multipart_threshold(bucket):
|
||||
"""Test that files below threshold use regular upload"""
|
||||
@@ -704,13 +721,9 @@ in
|
||||
if "using S3 regular upload" not in output:
|
||||
raise Exception("Expected regular upload to be used")
|
||||
|
||||
print(" ✓ Regular upload used for file below threshold")
|
||||
|
||||
client.succeed(f"{ENV_WITH_CREDS} nix copy --no-check-sigs --from '{store_url}' {PKGS['A']}")
|
||||
verify_packages_in_store(client, PKGS['A'], should_exist=True)
|
||||
|
||||
print(" ✓ Small file uploaded and verified")
|
||||
|
||||
@setup_s3()
|
||||
def test_multipart_with_log_compression(bucket):
|
||||
"""Test multipart upload with compressed build logs"""
|
||||
@@ -762,7 +775,185 @@ in
|
||||
print(output)
|
||||
raise Exception("Expected multipart completion message")
|
||||
|
||||
print(" ✓ Compressed log uploaded with multipart")
|
||||
@setup_s3(
|
||||
populate_bucket=[PKGS['A']],
|
||||
profiles={
|
||||
"valid": {"access_key": ACCESS_KEY, "secret_key": SECRET_KEY},
|
||||
"invalid": {"access_key": "INVALIDKEY", "secret_key": "INVALIDSECRET"},
|
||||
}
|
||||
)
|
||||
def test_profile_credentials(bucket):
|
||||
"""Test that profile-based credentials work without environment variables"""
|
||||
print("\n=== Testing Profile-Based Credentials ===")
|
||||
|
||||
store_url = make_s3_url(bucket, profile="valid")
|
||||
|
||||
# Verify store info works with profile credentials (no env vars)
|
||||
client.succeed(f"HOME=/root nix store info --store '{store_url}' >&2")
|
||||
|
||||
# Verify we can copy from the store using profile
|
||||
verify_packages_in_store(client, PKGS['A'], should_exist=False)
|
||||
client.succeed(f"HOME=/root nix copy --no-check-sigs --from '{store_url}' {PKGS['A']}")
|
||||
verify_packages_in_store(client, PKGS['A'])
|
||||
|
||||
# Clean up the package we just copied so we can test invalid profile
|
||||
client.succeed(f"nix store delete --ignore-liveness {PKGS['A']}")
|
||||
verify_packages_in_store(client, PKGS['A'], should_exist=False)
|
||||
|
||||
# Verify invalid profile fails when trying to copy
|
||||
invalid_url = make_s3_url(bucket, profile="invalid")
|
||||
client.fail(f"HOME=/root nix copy --no-check-sigs --from '{invalid_url}' {PKGS['A']} 2>&1")
|
||||
|
||||
@setup_s3(
|
||||
populate_bucket=[PKGS['A']],
|
||||
profiles={
|
||||
"wrong": {"access_key": "WRONGKEY", "secret_key": "WRONGSECRET"},
|
||||
}
|
||||
)
|
||||
def test_env_vars_precedence(bucket):
|
||||
"""Test that environment variables take precedence over profile credentials"""
|
||||
print("\n=== Testing Environment Variables Precedence ===")
|
||||
|
||||
# Use profile with wrong credentials, but provide correct creds via env vars
|
||||
store_url = make_s3_url(bucket, profile="wrong")
|
||||
|
||||
# Ensure package is not in client store
|
||||
verify_packages_in_store(client, PKGS['A'], should_exist=False)
|
||||
|
||||
# This should succeed because env vars (correct) override profile (wrong)
|
||||
output = client.succeed(
|
||||
f"HOME=/root {ENV_WITH_CREDS} nix copy --no-check-sigs --debug --from '{store_url}' {PKGS['A']} 2>&1"
|
||||
)
|
||||
|
||||
# Verify the credential chain shows Environment provider was added
|
||||
if "Added AWS Environment Credential Provider" not in output:
|
||||
print("Debug output:")
|
||||
print(output)
|
||||
raise Exception("Expected Environment provider to be added to chain")
|
||||
|
||||
# Clean up the package so we can test again without env vars
|
||||
client.succeed(f"nix store delete --ignore-liveness {PKGS['A']}")
|
||||
verify_packages_in_store(client, PKGS['A'], should_exist=False)
|
||||
|
||||
# Without env vars, same URL should fail (proving profile creds are actually wrong)
|
||||
client.fail(f"HOME=/root nix copy --no-check-sigs --from '{store_url}' {PKGS['A']} 2>&1")
|
||||
|
||||
@setup_s3(
|
||||
populate_bucket=[PKGS['A']],
|
||||
profiles={
|
||||
"testprofile": {"access_key": ACCESS_KEY, "secret_key": SECRET_KEY},
|
||||
}
|
||||
)
|
||||
def test_credential_provider_chain(bucket):
|
||||
"""Test that debug logging shows which providers are added to the chain"""
|
||||
print("\n=== Testing Credential Provider Chain Logging ===")
|
||||
|
||||
store_url = make_s3_url(bucket, profile="testprofile")
|
||||
|
||||
output = client.succeed(
|
||||
f"HOME=/root nix store info --debug --store '{store_url}' 2>&1"
|
||||
)
|
||||
|
||||
# For a named profile, we expect to see these providers in the chain
|
||||
expected_providers = ["Environment", "Profile", "IMDS"]
|
||||
for provider in expected_providers:
|
||||
msg = f"Added AWS {provider} Credential Provider to chain for profile 'testprofile'"
|
||||
if msg not in output:
|
||||
print("Debug output:")
|
||||
print(output)
|
||||
raise Exception(f"Expected to find: {msg}")
|
||||
|
||||
# SSO should be skipped (no SSO config for this profile)
|
||||
if "Skipped AWS SSO Credential Provider for profile 'testprofile'" not in output:
|
||||
print("Debug output:")
|
||||
print(output)
|
||||
raise Exception("Expected SSO provider to be skipped")
|
||||
|
||||
def test_virtual_hosted_copy():
|
||||
"""Test nix copy with virtual-hosted-style addressing on custom endpoint"""
|
||||
print("\n=== Testing Virtual-Hosted-Style Addressing ===")
|
||||
|
||||
# Use a fixed bucket name matching the static /etc/hosts entries
|
||||
bucket = 'vhost-test'
|
||||
server.succeed(f"mc mb minio/{bucket}")
|
||||
try:
|
||||
store_url = make_s3_url(
|
||||
bucket,
|
||||
endpoint=VHOST_ENDPOINT,
|
||||
**{'addressing-style': 'virtual'}
|
||||
)
|
||||
|
||||
# Upload with virtual-hosted-style, capture debug output
|
||||
output = server.succeed(
|
||||
f"{ENV_WITH_CREDS} nix copy --debug --to '{store_url}' {PKGS['A']} 2>&1"
|
||||
)
|
||||
|
||||
# Verify virtual-hosted-style URL was used (bucket in hostname)
|
||||
vhost_url_prefix = f"http://{bucket}.{VHOST_DOMAIN}:9000/"
|
||||
if vhost_url_prefix not in output:
|
||||
print("Debug output:")
|
||||
print(output)
|
||||
raise Exception(
|
||||
f"Expected virtual-hosted-style URL containing '{vhost_url_prefix}'"
|
||||
)
|
||||
|
||||
# Verify path-style URL was NOT used (bucket should not be in the path)
|
||||
path_style_pattern = f"{VHOST_ENDPOINT}/{bucket}/"
|
||||
if path_style_pattern in output:
|
||||
print("Debug output:")
|
||||
print(output)
|
||||
raise Exception("Found path-style URL when virtual-hosted-style was expected")
|
||||
|
||||
# Download with virtual-hosted-style
|
||||
verify_packages_in_store(client, PKGS['A'], should_exist=False)
|
||||
output = client.succeed(
|
||||
f"{ENV_WITH_CREDS} nix copy --debug --no-check-sigs "
|
||||
f"--from '{store_url}' {PKGS['A']} 2>&1"
|
||||
)
|
||||
|
||||
if vhost_url_prefix not in output:
|
||||
print("Debug output:")
|
||||
print(output)
|
||||
raise Exception(
|
||||
f"Expected virtual-hosted-style URL in download containing '{vhost_url_prefix}'"
|
||||
)
|
||||
|
||||
verify_packages_in_store(client, PKGS['A'])
|
||||
finally:
|
||||
server.succeed(f"mc rb --force minio/{bucket}")
|
||||
for pkg in PKGS.values():
|
||||
client.succeed(f"[ ! -e {pkg} ] || nix store delete --ignore-liveness {pkg}")
|
||||
|
||||
@setup_s3()
|
||||
def test_explicit_path_style(bucket):
|
||||
"""Test that addressing-style=path works as backwards-compatible fallback"""
|
||||
print("\n=== Testing Explicit Path-Style Addressing ===")
|
||||
|
||||
store_url = make_s3_url(
|
||||
bucket,
|
||||
**{'addressing-style': 'path'}
|
||||
)
|
||||
|
||||
# Upload with explicit path-style
|
||||
output = server.succeed(
|
||||
f"{ENV_WITH_CREDS} nix copy --debug --to '{store_url}' {PKGS['A']} 2>&1"
|
||||
)
|
||||
|
||||
# Verify path-style URL was used (bucket in path, not hostname)
|
||||
path_style_pattern = f"{ENDPOINT}/{bucket}/"
|
||||
if path_style_pattern not in output:
|
||||
print("Debug output:")
|
||||
print(output)
|
||||
raise Exception(
|
||||
f"Expected path-style URL containing '{path_style_pattern}'"
|
||||
)
|
||||
|
||||
# Download
|
||||
verify_packages_in_store(client, PKGS['A'], should_exist=False)
|
||||
client.succeed(
|
||||
f"{ENV_WITH_CREDS} nix copy --no-check-sigs --from '{store_url}' {PKGS['A']}"
|
||||
)
|
||||
verify_packages_in_store(client, PKGS['A'])
|
||||
|
||||
# ============================================================================
|
||||
# Main Test Execution
|
||||
@@ -782,6 +973,7 @@ in
|
||||
|
||||
# Run tests (each gets isolated bucket via decorator)
|
||||
test_credential_caching()
|
||||
test_aws_log_integration()
|
||||
test_fetchurl_basic()
|
||||
test_error_message_formatting()
|
||||
test_fork_credential_preresolution()
|
||||
@@ -797,9 +989,10 @@ in
|
||||
test_multipart_upload_basic()
|
||||
test_multipart_threshold()
|
||||
test_multipart_with_log_compression()
|
||||
|
||||
print("\n" + "="*80)
|
||||
print("✓ All S3 Binary Cache Store Tests Passed!")
|
||||
print("="*80)
|
||||
test_profile_credentials()
|
||||
test_env_vars_precedence()
|
||||
test_credential_provider_chain()
|
||||
test_virtual_hosted_copy()
|
||||
test_explicit_path_style()
|
||||
'';
|
||||
}
|
||||
Reference in new issue
Block a user