Befator-Inc-Firmen-Netzwerk/nix
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #15220 from NixOS/backport-15216-to-2.33-maintenance

Browse Source 
[Backport 2.33-maintenance] fix: #15208
This commit is contained in:
internal-nix-ci[bot]
2026-02-13 02:24:55 +00:00
committed by GitHub
parent e9d77856f5 695501815b
commit 48bbd96d2d
7 changed files with 555 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
doc/manual/rl-next/s3-virtual-hosted-style.md Normal file
View File

@@ -0,0 +1,32 @@
---
synopsis: S3 binary caches now use virtual-hosted-style addressing by default
issues: [15208]
---
S3 binary caches now use virtual-hosted-style URLs
(`https://bucket.s3.region.amazonaws.com/key`) instead of path-style URLs
(`https://s3.region.amazonaws.com/bucket/key`) when connecting to standard AWS
S3 endpoints. This enables HTTP/2 multiplexing and fixes TCP connection
exhaustion (TIME_WAIT socket accumulation) under high-concurrency workloads.
A new `addressing-style` store option controls this behavior:
- `auto` (default): virtual-hosted-style for standard AWS endpoints, path-style
for custom endpoints.
- `path`: forces path-style addressing (deprecated by AWS).
- `virtual`: forces virtual-hosted-style addressing (bucket names must not
contain dots).
Bucket names containing dots (e.g., `my.bucket.name`) automatically fall back
to path-style addressing in `auto` mode, because dotted names create
multi-level subdomains that break TLS wildcard certificate validation.
Example using path-style for backwards compatibility:
```
s3://my-bucket/key?region=us-east-1&addressing-style=path
```
Additionally, TCP keep-alive is now enabled on all HTTP connections, preventing
idle connections from being silently dropped by intermediate network devices
(NATs, firewalls, load balancers).

252
src/libstore-tests/s3-url.cc
View File

@@ -104,6 +104,33 @@ INSTANTIATE_TEST_SUITE_P(
},
},
"with_absolute_endpoint_uri",
},
ParsedS3URLTestCase{
"s3://bucket/key?addressing-style=virtual",
{
.bucket = "bucket",
.key = {"key"},
.addressingStyle = S3AddressingStyle::Virtual,
},
"with_addressing_style_virtual",
},
ParsedS3URLTestCase{
"s3://bucket/key?addressing-style=path",
{
.bucket = "bucket",
.key = {"key"},
.addressingStyle = S3AddressingStyle::Path,
},
"with_addressing_style_path",
},
ParsedS3URLTestCase{
"s3://bucket/key?addressing-style=auto",
{
.bucket = "bucket",
.key = {"key"},
.addressingStyle = S3AddressingStyle::Auto,
},
"with_addressing_style_auto",
}),
[](const ::testing::TestParamInfo<ParsedS3URLTestCase> & info) { return info.param.description; });
@@ -138,6 +165,26 @@ INSTANTIATE_TEST_SUITE_P(
InvalidS3URLTestCase{"s3://bucket", "error: URI has a missing or invalid key", "missing_key"}),
[](const ::testing::TestParamInfo<InvalidS3URLTestCase> & info) { return info.param.description; });
TEST(ParsedS3URLTest, invalidAddressingStyleThrows)
{
ASSERT_THROW(ParsedS3URL::parse(parseURL("s3://bucket/key?addressing-style=bogus")), InvalidS3AddressingStyle);
}
TEST(ParsedS3URLTest, virtualStyleWithAuthoritylessEndpointThrows)
{
ParsedS3URL input{
.bucket = "bucket",
.key = {"key"},
.addressingStyle = S3AddressingStyle::Virtual,
.endpoint =
ParsedURL{
.scheme = "file",
.path = {"", "some", "path"},
},
};
ASSERT_THROW(input.toHttpsUrl(), nix::Error);
}
// =============================================================================
// S3 URL to HTTPS Conversion Tests
// =============================================================================
@@ -166,6 +213,7 @@ INSTANTIATE_TEST_SUITE_P(
S3ToHttpsConversion,
S3ToHttpsConversionTest,
::testing::Values(
// Default (auto) addressing style: virtual-hosted for standard AWS endpoints
S3ToHttpsConversionTestCase{
ParsedS3URL{
.bucket = "my-bucket",
@@ -173,10 +221,10 @@ INSTANTIATE_TEST_SUITE_P(
},
ParsedURL{
.scheme = "https",
.authority = ParsedURL::Authority{.host = "s3.us-east-1.amazonaws.com"},
.path = {"", "my-bucket", "my-key.txt"},
.authority = ParsedURL::Authority{.host = "my-bucket.s3.us-east-1.amazonaws.com"},
.path = {"", "my-key.txt"},
},
"https://s3.us-east-1.amazonaws.com/my-bucket/my-key.txt",
"https://my-bucket.s3.us-east-1.amazonaws.com/my-key.txt",
"basic_s3_default_region",
},
S3ToHttpsConversionTestCase{
@@ -187,12 +235,13 @@ INSTANTIATE_TEST_SUITE_P(
},
ParsedURL{
.scheme = "https",
.authority = ParsedURL::Authority{.host = "s3.eu-west-1.amazonaws.com"},
.path = {"", "prod-cache", "nix", "store", "abc123.nar.xz"},
.authority = ParsedURL::Authority{.host = "prod-cache.s3.eu-west-1.amazonaws.com"},
.path = {"", "nix", "store", "abc123.nar.xz"},
},
"https://s3.eu-west-1.amazonaws.com/prod-cache/nix/store/abc123.nar.xz",
"https://prod-cache.s3.eu-west-1.amazonaws.com/nix/store/abc123.nar.xz",
"with_eu_west_1_region",
},
// Custom endpoint authority: path-style by default
S3ToHttpsConversionTestCase{
ParsedS3URL{
.bucket = "bucket",
@@ -208,6 +257,7 @@ INSTANTIATE_TEST_SUITE_P(
"http://custom.s3.com/bucket/key",
"custom_endpoint_authority",
},
// Custom endpoint URL: path-style by default
S3ToHttpsConversionTestCase{
ParsedS3URL{
.bucket = "bucket",
@@ -236,10 +286,10 @@ INSTANTIATE_TEST_SUITE_P(
},
ParsedURL{
.scheme = "https",
.authority = ParsedURL::Authority{.host = "s3.ap-southeast-2.amazonaws.com"},
.path = {"", "bucket", "path", "to", "file.txt"},
.authority = ParsedURL::Authority{.host = "bucket.s3.ap-southeast-2.amazonaws.com"},
.path = {"", "path", "to", "file.txt"},
},
"https://s3.ap-southeast-2.amazonaws.com/bucket/path/to/file.txt",
"https://bucket.s3.ap-southeast-2.amazonaws.com/path/to/file.txt",
"complex_path_and_region",
},
S3ToHttpsConversionTestCase{
@@ -250,11 +300,11 @@ INSTANTIATE_TEST_SUITE_P(
},
ParsedURL{
.scheme = "https",
.authority = ParsedURL::Authority{.host = "s3.us-east-1.amazonaws.com"},
.path = {"", "my-bucket", "my-key.txt"},
.authority = ParsedURL::Authority{.host = "my-bucket.s3.us-east-1.amazonaws.com"},
.path = {"", "my-key.txt"},
.query = {{"versionId", "abc123xyz"}},
},
"https://s3.us-east-1.amazonaws.com/my-bucket/my-key.txt?versionId=abc123xyz",
"https://my-bucket.s3.us-east-1.amazonaws.com/my-key.txt?versionId=abc123xyz",
"with_versionId",
},
S3ToHttpsConversionTestCase{
@@ -266,13 +316,185 @@ INSTANTIATE_TEST_SUITE_P(
},
ParsedURL{
.scheme = "https",
.authority = ParsedURL::Authority{.host = "s3.eu-west-1.amazonaws.com"},
.path = {"", "versioned-bucket", "path", "to", "object"},
.authority = ParsedURL::Authority{.host = "versioned-bucket.s3.eu-west-1.amazonaws.com"},
.path = {"", "path", "to", "object"},
.query = {{"versionId", "version456"}},
},
"https://s3.eu-west-1.amazonaws.com/versioned-bucket/path/to/object?versionId=version456",
"https://versioned-bucket.s3.eu-west-1.amazonaws.com/path/to/object?versionId=version456",
"with_region_and_versionId",
},
// Explicit addressing-style=path forces path-style on standard AWS endpoints
S3ToHttpsConversionTestCase{
ParsedS3URL{
.bucket = "my-bucket",
.key = {"my-key.txt"},
.region = "us-west-2",
.addressingStyle = S3AddressingStyle::Path,
},
ParsedURL{
.scheme = "https",
.authority = ParsedURL::Authority{.host = "s3.us-west-2.amazonaws.com"},
.path = {"", "my-bucket", "my-key.txt"},
},
"https://s3.us-west-2.amazonaws.com/my-bucket/my-key.txt",
"explicit_path_style",
},
// Explicit addressing-style=virtual forces virtual-hosted-style on custom endpoints
S3ToHttpsConversionTestCase{
ParsedS3URL{
.bucket = "bucket",
.key = {"key"},
.scheme = "http",
.addressingStyle = S3AddressingStyle::Virtual,
.endpoint = ParsedURL::Authority{.host = "custom.s3.com"},
},
ParsedURL{
.scheme = "http",
.authority = ParsedURL::Authority{.host = "bucket.custom.s3.com"},
.path = {"", "key"},
},
"http://bucket.custom.s3.com/key",
"explicit_virtual_style_custom_endpoint",
},
// Explicit addressing-style=virtual with full endpoint URL
S3ToHttpsConversionTestCase{
ParsedS3URL{
.bucket = "bucket",
.key = {"key"},
.addressingStyle = S3AddressingStyle::Virtual,
.endpoint =
ParsedURL{
.scheme = "http",
.authority = ParsedURL::Authority{.host = "server", .port = 9000},
.path = {""},
},
},
ParsedURL{
.scheme = "http",
.authority = ParsedURL::Authority{.host = "bucket.server", .port = 9000},
.path = {"", "key"},
},
"http://bucket.server:9000/key",
"explicit_virtual_style_full_endpoint_url",
},
// Dotted bucket names work normally with explicit path-style
S3ToHttpsConversionTestCase{
ParsedS3URL{
.bucket = "my.bucket",
.key = {"key"},
.addressingStyle = S3AddressingStyle::Path,
},
ParsedURL{
.scheme = "https",
.authority = ParsedURL::Authority{.host = "s3.us-east-1.amazonaws.com"},
.path = {"", "my.bucket", "key"},
},
"https://s3.us-east-1.amazonaws.com/my.bucket/key",
"dotted_bucket_with_path_style",
},
// Dotted bucket names fall back to path-style with auto on standard AWS endpoints
S3ToHttpsConversionTestCase{
ParsedS3URL{
.bucket = "my.bucket.name",
.key = {"key"},
},
ParsedURL{
.scheme = "https",
.authority = ParsedURL::Authority{.host = "s3.us-east-1.amazonaws.com"},
.path = {"", "my.bucket.name", "key"},
},
"https://s3.us-east-1.amazonaws.com/my.bucket.name/key",
"dotted_bucket_with_auto_style_on_aws",
},
// Dotted bucket names work with auto style on custom endpoints (auto = path-style)
S3ToHttpsConversionTestCase{
ParsedS3URL{
.bucket = "my.bucket",
.key = {"key"},
.endpoint = ParsedURL::Authority{.host = "minio.local"},
},
ParsedURL{
.scheme = "https",
.authority = ParsedURL::Authority{.host = "minio.local"},
.path = {"", "my.bucket", "key"},
},
"https://minio.local/my.bucket/key",
"dotted_bucket_with_auto_style_custom_endpoint",
}),
[](const ::testing::TestParamInfo<S3ToHttpsConversionTestCase> & info) { return info.param.description; });
// =============================================================================
// S3 URL to HTTPS Conversion Error Tests
// =============================================================================
struct S3ToHttpsConversionErrorTestCase
{
ParsedS3URL input;
std::string description;
};
class S3ToHttpsConversionErrorTest : public ::testing::WithParamInterface<S3ToHttpsConversionErrorTestCase>,
public ::testing::Test
{};
TEST_P(S3ToHttpsConversionErrorTest, ThrowsOnConversion)
{
auto & [input, description] = GetParam();
ASSERT_THROW(input.toHttpsUrl(), nix::Error);
}
INSTANTIATE_TEST_SUITE_P(
S3ToHttpsConversionErrors,
S3ToHttpsConversionErrorTest,
::testing::Values(
S3ToHttpsConversionErrorTestCase{
ParsedS3URL{
.bucket = "bucket",
.key = {"key"},
.addressingStyle = S3AddressingStyle::Virtual,
.endpoint = ParsedURL::Authority{.host = ""},
},
"virtual_style_with_empty_host_authority",
},
S3ToHttpsConversionErrorTestCase{
ParsedS3URL{
.bucket = "my.bucket",
.key = {"key"},
.addressingStyle = S3AddressingStyle::Virtual,
},
"dotted_bucket_with_explicit_virtual_style",
},
S3ToHttpsConversionErrorTestCase{
ParsedS3URL{
.bucket = "my.bucket.name",
.key = {"key"},
.addressingStyle = S3AddressingStyle::Virtual,
},
"dotted_bucket_with_explicit_virtual_style_multi_dot",
},
S3ToHttpsConversionErrorTestCase{
ParsedS3URL{
.bucket = "my.bucket",
.key = {"key"},
.addressingStyle = S3AddressingStyle::Virtual,
.endpoint = ParsedURL::Authority{.host = "minio.local"},
},
"dotted_bucket_with_explicit_virtual_style_custom_authority",
},
S3ToHttpsConversionErrorTestCase{
ParsedS3URL{
.bucket = "my.bucket",
.key = {"key"},
.addressingStyle = S3AddressingStyle::Virtual,
.endpoint =
ParsedURL{
.scheme = "http",
.authority = ParsedURL::Authority{.host = "minio.local", .port = 9000},
.path = {""},
},
},
"dotted_bucket_with_explicit_virtual_style_full_endpoint_url",
}),
[](const ::testing::TestParamInfo<S3ToHttpsConversionErrorTestCase> & info) { return info.param.description; });
} // namespace nix

6
src/libstore/filetransfer.cc
View File

@@ -479,6 +479,12 @@ struct curlFileTransfer : public FileTransfer
curl_easy_setopt(req, CURLOPT_CONNECTTIMEOUT, fileTransferSettings.connectTimeout.get());
// Enable TCP keep-alive so that idle connections in curl's reuse pool
// are not silently dropped by NATs, firewalls, or load balancers.
curl_easy_setopt(req, CURLOPT_TCP_KEEPALIVE, 1L);
curl_easy_setopt(req, CURLOPT_TCP_KEEPIDLE, 60L);
curl_easy_setopt(req, CURLOPT_TCP_KEEPINTVL, 60L);
curl_easy_setopt(req, CURLOPT_LOW_SPEED_LIMIT, 1L);
curl_easy_setopt(req, CURLOPT_LOW_SPEED_TIME, fileTransferSettings.stalledDownloadTimeout.get());

24
src/libstore/include/nix/store/s3-binary-cache-store.hh
View File

@@ -3,6 +3,7 @@
#include "nix/store/config.hh"
#include "nix/store/http-binary-cache-store.hh"
#include "nix/store/s3-url.hh"
namespace nix {
@@ -52,13 +53,22 @@ struct S3BinaryCacheStoreConfig : HttpBinaryCacheStoreConfig
"endpoint",
R"(
The S3 endpoint to use. When empty (default), uses AWS S3 with
region-specific endpoints (e.g., s3.us-east-1.amazonaws.com).
For S3-compatible services such as MinIO, set this to your service's endpoint.
region-specific endpoints. For S3-compatible services such as
MinIO, set this to your service's endpoint.
)"};
> **Note**
>
> Custom endpoints must support HTTPS and use path-based
> addressing instead of virtual host based addressing.
Setting<S3AddressingStyle> addressingStyle{
this,
S3AddressingStyle::Auto,
"addressing-style",
R"(
The S3 addressing style to use. `auto` (default) uses
virtual-hosted-style for standard AWS endpoints and path-style
for custom endpoints; bucket names containing dots automatically
fall back to path-style to avoid TLS certificate errors. `path`
forces path-style addressing (deprecated by AWS). `virtual`
forces virtual-hosted-style addressing (bucket names must not
contain dots).
)"};
const Setting<bool> multipartUpload{
@@ -117,7 +127,7 @@ struct S3BinaryCacheStoreConfig : HttpBinaryCacheStoreConfig
* Set of settings that are part of the S3 URI itself.
* These are needed for region specification and other S3-specific settings.
*/
const std::set<const AbstractSetting *> s3UriSettings = {&profile, &region, &scheme, &endpoint};
const std::set<const AbstractSetting *> s3UriSettings = {&profile, &region, &scheme, &endpoint, &addressingStyle};
static const std::string name()
{

29
src/libstore/include/nix/store/s3-url.hh
View File

@@ -1,16 +1,41 @@
#pragma once
///@file
#include "nix/store/config.hh"
#include "nix/util/error.hh"
#include "nix/util/url.hh"
#include "nix/util/util.hh"
#include <optional>
#include <string>
#include <string_view>
#include <variant>
#include <vector>
namespace nix {
/**
* S3 addressing style for bucket access.
* - Auto: virtual-hosted-style for standard AWS endpoints, path-style for custom endpoints.
* - Path: always use path-style (bucket in URL path).
* - Virtual: always use virtual-hosted-style (bucket as hostname prefix; bucket name must not contain dots).
*/
enum class S3AddressingStyle {
Auto,
Path,
Virtual,
};
MakeError(InvalidS3AddressingStyle, Error);
S3AddressingStyle parseS3AddressingStyle(std::string_view style);
std::string_view showS3AddressingStyle(S3AddressingStyle style);
template<>
S3AddressingStyle BaseSetting<S3AddressingStyle>::parse(const std::string & str) const;
template<>
std::string BaseSetting<S3AddressingStyle>::to_string() const;
/**
* Parsed S3 URL.
*/
@@ -27,6 +52,7 @@ struct ParsedS3URL
std::optional<std::string> region;
std::optional<std::string> scheme;
std::optional<std::string> versionId;
std::optional<S3AddressingStyle> addressingStyle;
/**
* The endpoint can be either missing, be an absolute URI (with a scheme like `http:`)
* or an authority (so an IP address or a registered name).
@@ -46,7 +72,8 @@ struct ParsedS3URL
static ParsedS3URL parse(const ParsedURL & uri);
/**
* Convert this ParsedS3URL to HTTPS ParsedURL for use with curl's AWS SigV4 authentication
* Convert this ParsedS3URL to an HTTP(S) ParsedURL for use with curl's AWS SigV4 authentication.
* The scheme defaults to HTTPS but respects the 'scheme' setting and custom endpoint schemes.
*/
ParsedURL toHttpsUrl() const;

146
src/libstore/s3-url.cc
View File

@@ -1,8 +1,14 @@
#include "nix/store/s3-url.hh"
#include "nix/util/abstract-setting-to-json.hh"
#include "nix/util/config-impl.hh"
#include "nix/util/error.hh"
#include "nix/util/logging.hh"
#include "nix/util/json-impls.hh"
#include "nix/util/split.hh"
#include "nix/util/strings-inline.hh"
#include <atomic>
#include <nlohmann/json.hpp>
#include <ranges>
#include <string_view>
@@ -10,6 +16,30 @@ using namespace std::string_view_literals;
namespace nix {
S3AddressingStyle parseS3AddressingStyle(std::string_view style)
{
if (style == "auto")
return S3AddressingStyle::Auto;
if (style == "path")
return S3AddressingStyle::Path;
if (style == "virtual")
return S3AddressingStyle::Virtual;
throw InvalidS3AddressingStyle("unknown S3 addressing style '%s', expected 'auto', 'path', or 'virtual'", style);
}
std::string_view showS3AddressingStyle(S3AddressingStyle style)
{
switch (style) {
case S3AddressingStyle::Auto:
return "auto";
case S3AddressingStyle::Path:
return "path";
case S3AddressingStyle::Virtual:
return "virtual";
}
unreachable();
}
ParsedS3URL ParsedS3URL::parse(const ParsedURL & parsed)
try {
if (parsed.scheme != "s3"sv)
@@ -49,6 +79,9 @@ try {
.region = getOptionalParam("region"),
.scheme = getOptionalParam("scheme"),
.versionId = getOptionalParam("versionId"),
.addressingStyle = getOptionalParam("addressing-style").transform([](const std::string & s) {
return parseS3AddressingStyle(s);
}),
.endpoint = [&]() -> decltype(ParsedS3URL::endpoint) {
if (!endpoint)
return std::monostate();
@@ -65,6 +98,9 @@ try {
} catch (BadURL & e) {
e.addTrace({}, "while parsing S3 URI: '%s'", parsed.to_string());
throw;
} catch (InvalidS3AddressingStyle & e) {
e.addTrace({}, "while parsing S3 URI: '%s'", parsed.to_string());
throw;
}
ParsedURL ParsedS3URL::toHttpsUrl() const
@@ -80,41 +116,95 @@ ParsedURL ParsedS3URL::toHttpsUrl() const
queryParams["versionId"] = *versionId;
}
auto style = addressingStyle.value_or(S3AddressingStyle::Auto);
// Virtual-hosted-style prepends the bucket name to the hostname, so bucket
// names containing dots produce multi-level subdomains (e.g.
// my.bucket.s3.amazonaws.com) that break TLS wildcard certificate validation.
// In auto mode, fall back to path-style; only error on explicit virtual.
auto hasDottedBucket = bucket.find('.') != std::string::npos;
auto useVirtualForEndpoint = [&](bool defaultVirtual) {
auto useVirtual = defaultVirtual ? style != S3AddressingStyle::Path : style == S3AddressingStyle::Virtual;
if (useVirtual && hasDottedBucket) {
if (style == S3AddressingStyle::Virtual)
throw Error(
"bucket name '%s' contains a dot, which is incompatible with "
"virtual-hosted-style addressing (causes TLS certificate errors); "
"use 'addressing-style=path' or 'addressing-style=auto' instead",
bucket);
static std::atomic<bool> warnedDottedBucket{false};
warnOnce(
warnedDottedBucket,
"bucket name '%s' contains a dot; falling back to path-style addressing "
"(virtual-hosted-style requires non-dotted bucket names for TLS certificate validity); "
"set 'addressing-style=path' to silence this warning",
bucket);
return false;
}
return useVirtual;
};
// Handle endpoint configuration using std::visit
return std::visit(
overloaded{
[&](const std::monostate &) {
// No custom endpoint, use standard AWS S3 endpoint
// No custom endpoint: use virtual-hosted-style by default (auto),
// path-style when explicitly requested or for dotted bucket names.
auto useVirtual = useVirtualForEndpoint(/* defaultVirtual = */ true);
std::vector<std::string> path{""};
path.push_back(bucket);
if (!useVirtual)
path.push_back(bucket);
path.insert(path.end(), key.begin(), key.end());
return ParsedURL{
.scheme = std::string{schemeStr},
.authority = ParsedURL::Authority{.host = "s3." + regionStr + ".amazonaws.com"},
.authority =
ParsedURL::Authority{
.host = useVirtual ? bucket + ".s3." + regionStr + ".amazonaws.com"
: "s3." + regionStr + ".amazonaws.com"},
.path = std::move(path),
.query = std::move(queryParams),
};
},
[&](const ParsedURL::Authority & auth) {
// Endpoint is just an authority (hostname/port)
// Custom endpoint authority: use path-style by default (auto),
// virtual-hosted-style only when explicitly requested (not for dotted buckets).
auto useVirtual = useVirtualForEndpoint(/* defaultVirtual = */ false);
if (useVirtual && auth.host.empty())
throw Error(
"cannot use virtual-hosted-style addressing with endpoint '%s' "
"because it has no hostname; use 'addressing-style=path' instead",
auth.to_string());
std::vector<std::string> path{""};
path.push_back(bucket);
if (!useVirtual)
path.push_back(bucket);
path.insert(path.end(), key.begin(), key.end());
return ParsedURL{
.scheme = std::string{schemeStr},
.authority = auth,
.authority =
useVirtual ? ParsedURL::Authority{.host = bucket + "." + auth.host, .port = auth.port} : auth,
.path = std::move(path),
.query = std::move(queryParams),
};
},
[&](const ParsedURL & endpointUrl) {
// Endpoint is already a ParsedURL (e.g., http://server:9000)
// Full endpoint URL: use path-style by default (auto),
// virtual-hosted-style only when explicitly requested (not for dotted buckets).
auto useVirtual = useVirtualForEndpoint(/* defaultVirtual = */ false);
if (useVirtual && (!endpointUrl.authority || endpointUrl.authority->host.empty()))
throw Error(
"cannot use virtual-hosted-style addressing with endpoint '%s' "
"because it has no authority (hostname)",
endpointUrl.to_string());
auto path = endpointUrl.path;
path.push_back(bucket);
if (!useVirtual)
path.push_back(bucket);
path.insert(path.end(), key.begin(), key.end());
return ParsedURL{
.scheme = endpointUrl.scheme,
.authority = endpointUrl.authority,
.authority = useVirtual ? std::optional{ParsedURL::Authority{
.host = bucket + "." + endpointUrl.authority->host,
.port = endpointUrl.authority->port}}
: endpointUrl.authority,
.path = std::move(path),
.query = std::move(queryParams),
};
@@ -123,4 +213,42 @@ ParsedURL ParsedS3URL::toHttpsUrl() const
endpoint);
}
void to_json(nlohmann::json & j, const S3AddressingStyle & e)
{
j = std::string{showS3AddressingStyle(e)};
}
void from_json(const nlohmann::json & j, S3AddressingStyle & e)
{
e = parseS3AddressingStyle(j.get<std::string>());
}
template<>
struct json_avoids_null<S3AddressingStyle> : std::true_type
{};
template<>
S3AddressingStyle BaseSetting<S3AddressingStyle>::parse(const std::string & str) const
{
try {
return parseS3AddressingStyle(str);
} catch (InvalidS3AddressingStyle &) {
throw UsageError("option '%s' has invalid value '%s', expected 'auto', 'path', or 'virtual'", name, str);
}
}
template<>
std::string BaseSetting<S3AddressingStyle>::to_string() const
{
return std::string{showS3AddressingStyle(value)};
}
template<>
struct BaseSetting<S3AddressingStyle>::trait
{
static constexpr bool appendable = false;
};
template class BaseSetting<S3AddressingStyle>;
} // namespace nix

98
tests/nixos/s3-binary-cache-store.nix
View File

@@ -48,9 +48,14 @@ in
rootCredentialsFile = pkgs.writeText "minio-credentials-full" ''
MINIO_ROOT_USER=${accessKey}
MINIO_ROOT_PASSWORD=${secretKey}
MINIO_DOMAIN=minio.local
'';
};
networking.firewall.allowedTCPPorts = [ 9000 ];
# Static hosts for virtual-hosted-style S3 tests.
# MinIO with MINIO_DOMAIN=minio.local accepts virtual-hosted requests
# where the bucket name is a hostname prefix.
networking.extraHosts = "127.0.0.1 vhost-test.minio.local minio.local";
};
client =
@@ -62,6 +67,7 @@ in
experimental-features = nix-command
substituters =
'';
networking.extraHosts = "192.168.1.2 vhost-test.minio.local minio.local";
};
};
@@ -83,6 +89,10 @@ in
ENDPOINT = 'http://server:9000'
REGION = 'eu-west-1'
# Virtual-hosted-style configuration (requires MINIO_DOMAIN and static host entries)
VHOST_DOMAIN = 'minio.local'
VHOST_ENDPOINT = f'http://{VHOST_DOMAIN}:9000'
PKGS = {
'A': '${pkgA}',
'B': '${pkgB}',
@@ -859,6 +869,92 @@ in
print(output)
raise Exception("Expected SSO provider to be skipped")
def test_virtual_hosted_copy():
"""Test nix copy with virtual-hosted-style addressing on custom endpoint"""
print("\n=== Testing Virtual-Hosted-Style Addressing ===")
# Use a fixed bucket name matching the static /etc/hosts entries
bucket = 'vhost-test'
server.succeed(f"mc mb minio/{bucket}")
try:
store_url = make_s3_url(
bucket,
endpoint=VHOST_ENDPOINT,
**{'addressing-style': 'virtual'}
)
# Upload with virtual-hosted-style, capture debug output
output = server.succeed(
f"{ENV_WITH_CREDS} nix copy --debug --to '{store_url}' {PKGS['A']} 2>&1"
)
# Verify virtual-hosted-style URL was used (bucket in hostname)
vhost_url_prefix = f"http://{bucket}.{VHOST_DOMAIN}:9000/"
if vhost_url_prefix not in output:
print("Debug output:")
print(output)
raise Exception(
f"Expected virtual-hosted-style URL containing '{vhost_url_prefix}'"
)
# Verify path-style URL was NOT used (bucket should not be in the path)
path_style_pattern = f"{VHOST_ENDPOINT}/{bucket}/"
if path_style_pattern in output:
print("Debug output:")
print(output)
raise Exception("Found path-style URL when virtual-hosted-style was expected")
# Download with virtual-hosted-style
verify_packages_in_store(client, PKGS['A'], should_exist=False)
output = client.succeed(
f"{ENV_WITH_CREDS} nix copy --debug --no-check-sigs "
f"--from '{store_url}' {PKGS['A']} 2>&1"
)
if vhost_url_prefix not in output:
print("Debug output:")
print(output)
raise Exception(
f"Expected virtual-hosted-style URL in download containing '{vhost_url_prefix}'"
)
verify_packages_in_store(client, PKGS['A'])
finally:
server.succeed(f"mc rb --force minio/{bucket}")
for pkg in PKGS.values():
client.succeed(f"[ ! -e {pkg} ] || nix store delete --ignore-liveness {pkg}")
@setup_s3()
def test_explicit_path_style(bucket):
"""Test that addressing-style=path works as backwards-compatible fallback"""
print("\n=== Testing Explicit Path-Style Addressing ===")
store_url = make_s3_url(
bucket,
**{'addressing-style': 'path'}
)
# Upload with explicit path-style
output = server.succeed(
f"{ENV_WITH_CREDS} nix copy --debug --to '{store_url}' {PKGS['A']} 2>&1"
)
# Verify path-style URL was used (bucket in path, not hostname)
path_style_pattern = f"{ENDPOINT}/{bucket}/"
if path_style_pattern not in output:
print("Debug output:")
print(output)
raise Exception(
f"Expected path-style URL containing '{path_style_pattern}'"
)
# Download
verify_packages_in_store(client, PKGS['A'], should_exist=False)
client.succeed(
f"{ENV_WITH_CREDS} nix copy --no-check-sigs --from '{store_url}' {PKGS['A']}"
)
verify_packages_in_store(client, PKGS['A'])
# ============================================================================
# Main Test Execution
# ============================================================================
@@ -896,5 +992,7 @@ in
test_profile_credentials()
test_env_vars_precedence()
test_credential_provider_chain()
test_virtual_hosted_copy()
test_explicit_path_style()
'';
}