Compare commits
19
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0769726d44 | ||
|
|
296b8d7bb9 | ||
|
|
a35595c6de | ||
|
|
e258c32fa6 | ||
|
|
e2af95376a | ||
|
|
b42319aa85 | ||
|
|
6e88d0cf6c | ||
|
|
eef8a1a5ff | ||
|
|
377fc95bce | ||
|
|
48bbd96d2d | ||
|
|
695501815b | ||
|
|
5ac10221a6 | ||
|
|
e9d77856f5 | ||
|
|
0c774b7ca4 | ||
|
|
70e39c2721 | ||
|
|
18f8b70cd9 | ||
|
|
4aa2942014 | ||
|
|
f28e495294 | ||
|
|
0142a88b8f |
No files matched your search
@@ -39,6 +39,17 @@ jobs:
|
||||
role-to-assume: "arn:aws:iam::080433136561:role/nix-release"
|
||||
role-session-name: nix-release-oidc-${{ github.run_id }}
|
||||
aws-region: eu-west-1
|
||||
- name: Disable containerd image store
|
||||
run: |
|
||||
# Docker 28+ defaults to the containerd image store, which
|
||||
# pushes layers uncompressed instead of gzip. OCI clients
|
||||
# that only support gzip (e.g. go-containerregistry) fail
|
||||
# with "gzip: invalid header". Disabling the containerd
|
||||
# snapshotter restores the classic storage driver, which
|
||||
# preserves gzip-compressed layers through the
|
||||
# `docker load` / `docker push` pipeline.
|
||||
echo '{"features":{"containerd-snapshotter":false}}' | sudo tee /etc/docker/daemon.json > /dev/null
|
||||
sudo systemctl restart docker
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
|
||||
with:
|
||||
|
||||
@@ -279,3 +279,35 @@ This release was made possible by the following 33 contributors:
|
||||
- Henry [**(@cootshk)**](https://github.com/cootshk)
|
||||
- Martin Joerg [**(@mjoerg)**](https://github.com/mjoerg)
|
||||
- Farid Zakaria [**(@fzakaria)**](https://github.com/fzakaria)
|
||||
# Release 2.33.3 (2026-02-13)
|
||||
|
||||
- S3 binary caches now use virtual-hosted-style addressing by default [#15208](https://github.com/NixOS/nix/issues/15208)
|
||||
|
||||
S3 binary caches now use virtual-hosted-style URLs
|
||||
(`https://bucket.s3.region.amazonaws.com/key`) instead of path-style URLs
|
||||
(`https://s3.region.amazonaws.com/bucket/key`) when connecting to standard AWS
|
||||
S3 endpoints. This enables HTTP/2 multiplexing and fixes TCP connection
|
||||
exhaustion (TIME_WAIT socket accumulation) under high-concurrency workloads.
|
||||
|
||||
A new `addressing-style` store option controls this behavior:
|
||||
|
||||
- `auto` (default): virtual-hosted-style for standard AWS endpoints, path-style
|
||||
for custom endpoints.
|
||||
- `path`: forces path-style addressing (deprecated by AWS).
|
||||
- `virtual`: forces virtual-hosted-style addressing (bucket names must not
|
||||
contain dots).
|
||||
|
||||
Bucket names containing dots (e.g., `my.bucket.name`) automatically fall back
|
||||
to path-style addressing in `auto` mode, because dotted names create
|
||||
multi-level subdomains that break TLS wildcard certificate validation.
|
||||
|
||||
Example using path-style for backwards compatibility:
|
||||
|
||||
```
|
||||
s3://my-bucket/key?region=us-east-1&addressing-style=path
|
||||
```
|
||||
|
||||
Additionally, TCP keep-alive is now enabled on all HTTP connections, preventing
|
||||
idle connections from being silently dropped by intermediate network devices
|
||||
(NATs, firewalls, load balancers).
|
||||
|
||||
@@ -236,6 +236,9 @@ void Input::checkLocks(Input specified, Input & result)
|
||||
if (auto prevNarHash = specified.getNarHash())
|
||||
specified.attrs.insert_or_assign("narHash", prevNarHash->to_string(HashFormat::SRI, true));
|
||||
|
||||
if (auto narHash = result.getNarHash())
|
||||
result.attrs.insert_or_assign("narHash", narHash->to_string(HashFormat::SRI, true));
|
||||
|
||||
for (auto & field : specified.attrs) {
|
||||
auto field2 = result.attrs.find(field.first);
|
||||
if (field2 != result.attrs.end() && field.second != field2->second)
|
||||
|
||||
@@ -128,6 +128,7 @@ static void prim_flakeRefToString(EvalState & state, const PosIdx pos, Value **
|
||||
state.forceAttrs(*args[0], noPos, "while evaluating the argument passed to builtins.flakeRefToString");
|
||||
fetchers::Attrs attrs;
|
||||
for (const auto & attr : *args[0]->attrs()) {
|
||||
state.forceValue(*attr.value, attr.pos);
|
||||
auto t = attr.value->type();
|
||||
if (t == nInt) {
|
||||
auto intValue = attr.value->integer().value;
|
||||
|
||||
+237
-15
@@ -104,6 +104,33 @@ INSTANTIATE_TEST_SUITE_P(
|
||||
},
|
||||
},
|
||||
"with_absolute_endpoint_uri",
|
||||
},
|
||||
ParsedS3URLTestCase{
|
||||
"s3://bucket/key?addressing-style=virtual",
|
||||
{
|
||||
.bucket = "bucket",
|
||||
.key = {"key"},
|
||||
.addressingStyle = S3AddressingStyle::Virtual,
|
||||
},
|
||||
"with_addressing_style_virtual",
|
||||
},
|
||||
ParsedS3URLTestCase{
|
||||
"s3://bucket/key?addressing-style=path",
|
||||
{
|
||||
.bucket = "bucket",
|
||||
.key = {"key"},
|
||||
.addressingStyle = S3AddressingStyle::Path,
|
||||
},
|
||||
"with_addressing_style_path",
|
||||
},
|
||||
ParsedS3URLTestCase{
|
||||
"s3://bucket/key?addressing-style=auto",
|
||||
{
|
||||
.bucket = "bucket",
|
||||
.key = {"key"},
|
||||
.addressingStyle = S3AddressingStyle::Auto,
|
||||
},
|
||||
"with_addressing_style_auto",
|
||||
}),
|
||||
[](const ::testing::TestParamInfo<ParsedS3URLTestCase> & info) { return info.param.description; });
|
||||
|
||||
@@ -138,6 +165,26 @@ INSTANTIATE_TEST_SUITE_P(
|
||||
InvalidS3URLTestCase{"s3://bucket", "error: URI has a missing or invalid key", "missing_key"}),
|
||||
[](const ::testing::TestParamInfo<InvalidS3URLTestCase> & info) { return info.param.description; });
|
||||
|
||||
TEST(ParsedS3URLTest, invalidAddressingStyleThrows)
|
||||
{
|
||||
ASSERT_THROW(ParsedS3URL::parse(parseURL("s3://bucket/key?addressing-style=bogus")), InvalidS3AddressingStyle);
|
||||
}
|
||||
|
||||
TEST(ParsedS3URLTest, virtualStyleWithAuthoritylessEndpointThrows)
|
||||
{
|
||||
ParsedS3URL input{
|
||||
.bucket = "bucket",
|
||||
.key = {"key"},
|
||||
.addressingStyle = S3AddressingStyle::Virtual,
|
||||
.endpoint =
|
||||
ParsedURL{
|
||||
.scheme = "file",
|
||||
.path = {"", "some", "path"},
|
||||
},
|
||||
};
|
||||
ASSERT_THROW(input.toHttpsUrl(), nix::Error);
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// S3 URL to HTTPS Conversion Tests
|
||||
// =============================================================================
|
||||
@@ -166,6 +213,7 @@ INSTANTIATE_TEST_SUITE_P(
|
||||
S3ToHttpsConversion,
|
||||
S3ToHttpsConversionTest,
|
||||
::testing::Values(
|
||||
// Default (auto) addressing style: virtual-hosted for standard AWS endpoints
|
||||
S3ToHttpsConversionTestCase{
|
||||
ParsedS3URL{
|
||||
.bucket = "my-bucket",
|
||||
@@ -173,10 +221,10 @@ INSTANTIATE_TEST_SUITE_P(
|
||||
},
|
||||
ParsedURL{
|
||||
.scheme = "https",
|
||||
.authority = ParsedURL::Authority{.host = "s3.us-east-1.amazonaws.com"},
|
||||
.path = {"", "my-bucket", "my-key.txt"},
|
||||
.authority = ParsedURL::Authority{.host = "my-bucket.s3.us-east-1.amazonaws.com"},
|
||||
.path = {"", "my-key.txt"},
|
||||
},
|
||||
"https://s3.us-east-1.amazonaws.com/my-bucket/my-key.txt",
|
||||
"https://my-bucket.s3.us-east-1.amazonaws.com/my-key.txt",
|
||||
"basic_s3_default_region",
|
||||
},
|
||||
S3ToHttpsConversionTestCase{
|
||||
@@ -187,12 +235,13 @@ INSTANTIATE_TEST_SUITE_P(
|
||||
},
|
||||
ParsedURL{
|
||||
.scheme = "https",
|
||||
.authority = ParsedURL::Authority{.host = "s3.eu-west-1.amazonaws.com"},
|
||||
.path = {"", "prod-cache", "nix", "store", "abc123.nar.xz"},
|
||||
.authority = ParsedURL::Authority{.host = "prod-cache.s3.eu-west-1.amazonaws.com"},
|
||||
.path = {"", "nix", "store", "abc123.nar.xz"},
|
||||
},
|
||||
"https://s3.eu-west-1.amazonaws.com/prod-cache/nix/store/abc123.nar.xz",
|
||||
"https://prod-cache.s3.eu-west-1.amazonaws.com/nix/store/abc123.nar.xz",
|
||||
"with_eu_west_1_region",
|
||||
},
|
||||
// Custom endpoint authority: path-style by default
|
||||
S3ToHttpsConversionTestCase{
|
||||
ParsedS3URL{
|
||||
.bucket = "bucket",
|
||||
@@ -208,6 +257,7 @@ INSTANTIATE_TEST_SUITE_P(
|
||||
"http://custom.s3.com/bucket/key",
|
||||
"custom_endpoint_authority",
|
||||
},
|
||||
// Custom endpoint URL: path-style by default
|
||||
S3ToHttpsConversionTestCase{
|
||||
ParsedS3URL{
|
||||
.bucket = "bucket",
|
||||
@@ -236,10 +286,10 @@ INSTANTIATE_TEST_SUITE_P(
|
||||
},
|
||||
ParsedURL{
|
||||
.scheme = "https",
|
||||
.authority = ParsedURL::Authority{.host = "s3.ap-southeast-2.amazonaws.com"},
|
||||
.path = {"", "bucket", "path", "to", "file.txt"},
|
||||
.authority = ParsedURL::Authority{.host = "bucket.s3.ap-southeast-2.amazonaws.com"},
|
||||
.path = {"", "path", "to", "file.txt"},
|
||||
},
|
||||
"https://s3.ap-southeast-2.amazonaws.com/bucket/path/to/file.txt",
|
||||
"https://bucket.s3.ap-southeast-2.amazonaws.com/path/to/file.txt",
|
||||
"complex_path_and_region",
|
||||
},
|
||||
S3ToHttpsConversionTestCase{
|
||||
@@ -250,11 +300,11 @@ INSTANTIATE_TEST_SUITE_P(
|
||||
},
|
||||
ParsedURL{
|
||||
.scheme = "https",
|
||||
.authority = ParsedURL::Authority{.host = "s3.us-east-1.amazonaws.com"},
|
||||
.path = {"", "my-bucket", "my-key.txt"},
|
||||
.authority = ParsedURL::Authority{.host = "my-bucket.s3.us-east-1.amazonaws.com"},
|
||||
.path = {"", "my-key.txt"},
|
||||
.query = {{"versionId", "abc123xyz"}},
|
||||
},
|
||||
"https://s3.us-east-1.amazonaws.com/my-bucket/my-key.txt?versionId=abc123xyz",
|
||||
"https://my-bucket.s3.us-east-1.amazonaws.com/my-key.txt?versionId=abc123xyz",
|
||||
"with_versionId",
|
||||
},
|
||||
S3ToHttpsConversionTestCase{
|
||||
@@ -266,13 +316,185 @@ INSTANTIATE_TEST_SUITE_P(
|
||||
},
|
||||
ParsedURL{
|
||||
.scheme = "https",
|
||||
.authority = ParsedURL::Authority{.host = "s3.eu-west-1.amazonaws.com"},
|
||||
.path = {"", "versioned-bucket", "path", "to", "object"},
|
||||
.authority = ParsedURL::Authority{.host = "versioned-bucket.s3.eu-west-1.amazonaws.com"},
|
||||
.path = {"", "path", "to", "object"},
|
||||
.query = {{"versionId", "version456"}},
|
||||
},
|
||||
"https://s3.eu-west-1.amazonaws.com/versioned-bucket/path/to/object?versionId=version456",
|
||||
"https://versioned-bucket.s3.eu-west-1.amazonaws.com/path/to/object?versionId=version456",
|
||||
"with_region_and_versionId",
|
||||
},
|
||||
// Explicit addressing-style=path forces path-style on standard AWS endpoints
|
||||
S3ToHttpsConversionTestCase{
|
||||
ParsedS3URL{
|
||||
.bucket = "my-bucket",
|
||||
.key = {"my-key.txt"},
|
||||
.region = "us-west-2",
|
||||
.addressingStyle = S3AddressingStyle::Path,
|
||||
},
|
||||
ParsedURL{
|
||||
.scheme = "https",
|
||||
.authority = ParsedURL::Authority{.host = "s3.us-west-2.amazonaws.com"},
|
||||
.path = {"", "my-bucket", "my-key.txt"},
|
||||
},
|
||||
"https://s3.us-west-2.amazonaws.com/my-bucket/my-key.txt",
|
||||
"explicit_path_style",
|
||||
},
|
||||
// Explicit addressing-style=virtual forces virtual-hosted-style on custom endpoints
|
||||
S3ToHttpsConversionTestCase{
|
||||
ParsedS3URL{
|
||||
.bucket = "bucket",
|
||||
.key = {"key"},
|
||||
.scheme = "http",
|
||||
.addressingStyle = S3AddressingStyle::Virtual,
|
||||
.endpoint = ParsedURL::Authority{.host = "custom.s3.com"},
|
||||
},
|
||||
ParsedURL{
|
||||
.scheme = "http",
|
||||
.authority = ParsedURL::Authority{.host = "bucket.custom.s3.com"},
|
||||
.path = {"", "key"},
|
||||
},
|
||||
"http://bucket.custom.s3.com/key",
|
||||
"explicit_virtual_style_custom_endpoint",
|
||||
},
|
||||
// Explicit addressing-style=virtual with full endpoint URL
|
||||
S3ToHttpsConversionTestCase{
|
||||
ParsedS3URL{
|
||||
.bucket = "bucket",
|
||||
.key = {"key"},
|
||||
.addressingStyle = S3AddressingStyle::Virtual,
|
||||
.endpoint =
|
||||
ParsedURL{
|
||||
.scheme = "http",
|
||||
.authority = ParsedURL::Authority{.host = "server", .port = 9000},
|
||||
.path = {""},
|
||||
},
|
||||
},
|
||||
ParsedURL{
|
||||
.scheme = "http",
|
||||
.authority = ParsedURL::Authority{.host = "bucket.server", .port = 9000},
|
||||
.path = {"", "key"},
|
||||
},
|
||||
"http://bucket.server:9000/key",
|
||||
"explicit_virtual_style_full_endpoint_url",
|
||||
},
|
||||
// Dotted bucket names work normally with explicit path-style
|
||||
S3ToHttpsConversionTestCase{
|
||||
ParsedS3URL{
|
||||
.bucket = "my.bucket",
|
||||
.key = {"key"},
|
||||
.addressingStyle = S3AddressingStyle::Path,
|
||||
},
|
||||
ParsedURL{
|
||||
.scheme = "https",
|
||||
.authority = ParsedURL::Authority{.host = "s3.us-east-1.amazonaws.com"},
|
||||
.path = {"", "my.bucket", "key"},
|
||||
},
|
||||
"https://s3.us-east-1.amazonaws.com/my.bucket/key",
|
||||
"dotted_bucket_with_path_style",
|
||||
},
|
||||
// Dotted bucket names fall back to path-style with auto on standard AWS endpoints
|
||||
S3ToHttpsConversionTestCase{
|
||||
ParsedS3URL{
|
||||
.bucket = "my.bucket.name",
|
||||
.key = {"key"},
|
||||
},
|
||||
ParsedURL{
|
||||
.scheme = "https",
|
||||
.authority = ParsedURL::Authority{.host = "s3.us-east-1.amazonaws.com"},
|
||||
.path = {"", "my.bucket.name", "key"},
|
||||
},
|
||||
"https://s3.us-east-1.amazonaws.com/my.bucket.name/key",
|
||||
"dotted_bucket_with_auto_style_on_aws",
|
||||
},
|
||||
// Dotted bucket names work with auto style on custom endpoints (auto = path-style)
|
||||
S3ToHttpsConversionTestCase{
|
||||
ParsedS3URL{
|
||||
.bucket = "my.bucket",
|
||||
.key = {"key"},
|
||||
.endpoint = ParsedURL::Authority{.host = "minio.local"},
|
||||
},
|
||||
ParsedURL{
|
||||
.scheme = "https",
|
||||
.authority = ParsedURL::Authority{.host = "minio.local"},
|
||||
.path = {"", "my.bucket", "key"},
|
||||
},
|
||||
"https://minio.local/my.bucket/key",
|
||||
"dotted_bucket_with_auto_style_custom_endpoint",
|
||||
}),
|
||||
[](const ::testing::TestParamInfo<S3ToHttpsConversionTestCase> & info) { return info.param.description; });
|
||||
|
||||
// =============================================================================
|
||||
// S3 URL to HTTPS Conversion Error Tests
|
||||
// =============================================================================
|
||||
|
||||
struct S3ToHttpsConversionErrorTestCase
|
||||
{
|
||||
ParsedS3URL input;
|
||||
std::string description;
|
||||
};
|
||||
|
||||
class S3ToHttpsConversionErrorTest : public ::testing::WithParamInterface<S3ToHttpsConversionErrorTestCase>,
|
||||
public ::testing::Test
|
||||
{};
|
||||
|
||||
TEST_P(S3ToHttpsConversionErrorTest, ThrowsOnConversion)
|
||||
{
|
||||
auto & [input, description] = GetParam();
|
||||
ASSERT_THROW(input.toHttpsUrl(), nix::Error);
|
||||
}
|
||||
|
||||
INSTANTIATE_TEST_SUITE_P(
|
||||
S3ToHttpsConversionErrors,
|
||||
S3ToHttpsConversionErrorTest,
|
||||
::testing::Values(
|
||||
S3ToHttpsConversionErrorTestCase{
|
||||
ParsedS3URL{
|
||||
.bucket = "bucket",
|
||||
.key = {"key"},
|
||||
.addressingStyle = S3AddressingStyle::Virtual,
|
||||
.endpoint = ParsedURL::Authority{.host = ""},
|
||||
},
|
||||
"virtual_style_with_empty_host_authority",
|
||||
},
|
||||
S3ToHttpsConversionErrorTestCase{
|
||||
ParsedS3URL{
|
||||
.bucket = "my.bucket",
|
||||
.key = {"key"},
|
||||
.addressingStyle = S3AddressingStyle::Virtual,
|
||||
},
|
||||
"dotted_bucket_with_explicit_virtual_style",
|
||||
},
|
||||
S3ToHttpsConversionErrorTestCase{
|
||||
ParsedS3URL{
|
||||
.bucket = "my.bucket.name",
|
||||
.key = {"key"},
|
||||
.addressingStyle = S3AddressingStyle::Virtual,
|
||||
},
|
||||
"dotted_bucket_with_explicit_virtual_style_multi_dot",
|
||||
},
|
||||
S3ToHttpsConversionErrorTestCase{
|
||||
ParsedS3URL{
|
||||
.bucket = "my.bucket",
|
||||
.key = {"key"},
|
||||
.addressingStyle = S3AddressingStyle::Virtual,
|
||||
.endpoint = ParsedURL::Authority{.host = "minio.local"},
|
||||
},
|
||||
"dotted_bucket_with_explicit_virtual_style_custom_authority",
|
||||
},
|
||||
S3ToHttpsConversionErrorTestCase{
|
||||
ParsedS3URL{
|
||||
.bucket = "my.bucket",
|
||||
.key = {"key"},
|
||||
.addressingStyle = S3AddressingStyle::Virtual,
|
||||
.endpoint =
|
||||
ParsedURL{
|
||||
.scheme = "http",
|
||||
.authority = ParsedURL::Authority{.host = "minio.local", .port = 9000},
|
||||
.path = {""},
|
||||
},
|
||||
},
|
||||
"dotted_bucket_with_explicit_virtual_style_full_endpoint_url",
|
||||
}),
|
||||
[](const ::testing::TestParamInfo<S3ToHttpsConversionErrorTestCase> & info) { return info.param.description; });
|
||||
|
||||
} // namespace nix
|
||||
@@ -852,7 +852,10 @@ static void performOp(
|
||||
auto path = WorkerProto::Serialise<StorePath>::read(*store, rconn);
|
||||
std::shared_ptr<const ValidPathInfo> info;
|
||||
logger->startWork();
|
||||
info = store->queryPathInfo(path);
|
||||
try {
|
||||
info = store->queryPathInfo(path);
|
||||
} catch (InvalidPath &) {
|
||||
}
|
||||
logger->stopWork();
|
||||
if (info) {
|
||||
conn.to << 1;
|
||||
|
||||
@@ -479,6 +479,12 @@ struct curlFileTransfer : public FileTransfer
|
||||
|
||||
curl_easy_setopt(req, CURLOPT_CONNECTTIMEOUT, fileTransferSettings.connectTimeout.get());
|
||||
|
||||
// Enable TCP keep-alive so that idle connections in curl's reuse pool
|
||||
// are not silently dropped by NATs, firewalls, or load balancers.
|
||||
curl_easy_setopt(req, CURLOPT_TCP_KEEPALIVE, 1L);
|
||||
curl_easy_setopt(req, CURLOPT_TCP_KEEPIDLE, 60L);
|
||||
curl_easy_setopt(req, CURLOPT_TCP_KEEPINTVL, 60L);
|
||||
|
||||
curl_easy_setopt(req, CURLOPT_LOW_SPEED_LIMIT, 1L);
|
||||
curl_easy_setopt(req, CURLOPT_LOW_SPEED_TIME, fileTransferSettings.stalledDownloadTimeout.get());
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
|
||||
#include "nix/store/config.hh"
|
||||
#include "nix/store/http-binary-cache-store.hh"
|
||||
#include "nix/store/s3-url.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
@@ -52,13 +53,22 @@ struct S3BinaryCacheStoreConfig : HttpBinaryCacheStoreConfig
|
||||
"endpoint",
|
||||
R"(
|
||||
The S3 endpoint to use. When empty (default), uses AWS S3 with
|
||||
region-specific endpoints (e.g., s3.us-east-1.amazonaws.com).
|
||||
For S3-compatible services such as MinIO, set this to your service's endpoint.
|
||||
region-specific endpoints. For S3-compatible services such as
|
||||
MinIO, set this to your service's endpoint.
|
||||
)"};
|
||||
|
||||
> **Note**
|
||||
>
|
||||
> Custom endpoints must support HTTPS and use path-based
|
||||
> addressing instead of virtual host based addressing.
|
||||
Setting<S3AddressingStyle> addressingStyle{
|
||||
this,
|
||||
S3AddressingStyle::Auto,
|
||||
"addressing-style",
|
||||
R"(
|
||||
The S3 addressing style to use. `auto` (default) uses
|
||||
virtual-hosted-style for standard AWS endpoints and path-style
|
||||
for custom endpoints; bucket names containing dots automatically
|
||||
fall back to path-style to avoid TLS certificate errors. `path`
|
||||
forces path-style addressing (deprecated by AWS). `virtual`
|
||||
forces virtual-hosted-style addressing (bucket names must not
|
||||
contain dots).
|
||||
)"};
|
||||
|
||||
const Setting<bool> multipartUpload{
|
||||
@@ -117,7 +127,7 @@ struct S3BinaryCacheStoreConfig : HttpBinaryCacheStoreConfig
|
||||
* Set of settings that are part of the S3 URI itself.
|
||||
* These are needed for region specification and other S3-specific settings.
|
||||
*/
|
||||
const std::set<const AbstractSetting *> s3UriSettings = {&profile, ®ion, &scheme, &endpoint};
|
||||
const std::set<const AbstractSetting *> s3UriSettings = {&profile, ®ion, &scheme, &endpoint, &addressingStyle};
|
||||
|
||||
static const std::string name()
|
||||
{
|
||||
|
||||
@@ -1,16 +1,41 @@
|
||||
#pragma once
|
||||
///@file
|
||||
#include "nix/store/config.hh"
|
||||
#include "nix/util/error.hh"
|
||||
#include "nix/util/url.hh"
|
||||
#include "nix/util/util.hh"
|
||||
|
||||
#include <optional>
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
#include <variant>
|
||||
#include <vector>
|
||||
|
||||
namespace nix {
|
||||
|
||||
/**
|
||||
* S3 addressing style for bucket access.
|
||||
* - Auto: virtual-hosted-style for standard AWS endpoints, path-style for custom endpoints.
|
||||
* - Path: always use path-style (bucket in URL path).
|
||||
* - Virtual: always use virtual-hosted-style (bucket as hostname prefix; bucket name must not contain dots).
|
||||
*/
|
||||
enum class S3AddressingStyle {
|
||||
Auto,
|
||||
Path,
|
||||
Virtual,
|
||||
};
|
||||
|
||||
MakeError(InvalidS3AddressingStyle, Error);
|
||||
|
||||
S3AddressingStyle parseS3AddressingStyle(std::string_view style);
|
||||
std::string_view showS3AddressingStyle(S3AddressingStyle style);
|
||||
|
||||
template<>
|
||||
S3AddressingStyle BaseSetting<S3AddressingStyle>::parse(const std::string & str) const;
|
||||
|
||||
template<>
|
||||
std::string BaseSetting<S3AddressingStyle>::to_string() const;
|
||||
|
||||
/**
|
||||
* Parsed S3 URL.
|
||||
*/
|
||||
@@ -27,6 +52,7 @@ struct ParsedS3URL
|
||||
std::optional<std::string> region;
|
||||
std::optional<std::string> scheme;
|
||||
std::optional<std::string> versionId;
|
||||
std::optional<S3AddressingStyle> addressingStyle;
|
||||
/**
|
||||
* The endpoint can be either missing, be an absolute URI (with a scheme like `http:`)
|
||||
* or an authority (so an IP address or a registered name).
|
||||
@@ -46,7 +72,8 @@ struct ParsedS3URL
|
||||
static ParsedS3URL parse(const ParsedURL & uri);
|
||||
|
||||
/**
|
||||
* Convert this ParsedS3URL to HTTPS ParsedURL for use with curl's AWS SigV4 authentication
|
||||
* Convert this ParsedS3URL to an HTTP(S) ParsedURL for use with curl's AWS SigV4 authentication.
|
||||
* The scheme defaults to HTTPS but respects the 'scheme' setting and custom endpoint schemes.
|
||||
*/
|
||||
ParsedURL toHttpsUrl() const;
|
||||
|
||||
|
||||
+137
-9
@@ -1,8 +1,14 @@
|
||||
#include "nix/store/s3-url.hh"
|
||||
#include "nix/util/abstract-setting-to-json.hh"
|
||||
#include "nix/util/config-impl.hh"
|
||||
#include "nix/util/error.hh"
|
||||
#include "nix/util/logging.hh"
|
||||
#include "nix/util/json-impls.hh"
|
||||
#include "nix/util/split.hh"
|
||||
#include "nix/util/strings-inline.hh"
|
||||
|
||||
#include <atomic>
|
||||
#include <nlohmann/json.hpp>
|
||||
#include <ranges>
|
||||
#include <string_view>
|
||||
|
||||
@@ -10,6 +16,30 @@ using namespace std::string_view_literals;
|
||||
|
||||
namespace nix {
|
||||
|
||||
S3AddressingStyle parseS3AddressingStyle(std::string_view style)
|
||||
{
|
||||
if (style == "auto")
|
||||
return S3AddressingStyle::Auto;
|
||||
if (style == "path")
|
||||
return S3AddressingStyle::Path;
|
||||
if (style == "virtual")
|
||||
return S3AddressingStyle::Virtual;
|
||||
throw InvalidS3AddressingStyle("unknown S3 addressing style '%s', expected 'auto', 'path', or 'virtual'", style);
|
||||
}
|
||||
|
||||
std::string_view showS3AddressingStyle(S3AddressingStyle style)
|
||||
{
|
||||
switch (style) {
|
||||
case S3AddressingStyle::Auto:
|
||||
return "auto";
|
||||
case S3AddressingStyle::Path:
|
||||
return "path";
|
||||
case S3AddressingStyle::Virtual:
|
||||
return "virtual";
|
||||
}
|
||||
unreachable();
|
||||
}
|
||||
|
||||
ParsedS3URL ParsedS3URL::parse(const ParsedURL & parsed)
|
||||
try {
|
||||
if (parsed.scheme != "s3"sv)
|
||||
@@ -49,6 +79,9 @@ try {
|
||||
.region = getOptionalParam("region"),
|
||||
.scheme = getOptionalParam("scheme"),
|
||||
.versionId = getOptionalParam("versionId"),
|
||||
.addressingStyle = getOptionalParam("addressing-style").transform([](const std::string & s) {
|
||||
return parseS3AddressingStyle(s);
|
||||
}),
|
||||
.endpoint = [&]() -> decltype(ParsedS3URL::endpoint) {
|
||||
if (!endpoint)
|
||||
return std::monostate();
|
||||
@@ -65,6 +98,9 @@ try {
|
||||
} catch (BadURL & e) {
|
||||
e.addTrace({}, "while parsing S3 URI: '%s'", parsed.to_string());
|
||||
throw;
|
||||
} catch (InvalidS3AddressingStyle & e) {
|
||||
e.addTrace({}, "while parsing S3 URI: '%s'", parsed.to_string());
|
||||
throw;
|
||||
}
|
||||
|
||||
ParsedURL ParsedS3URL::toHttpsUrl() const
|
||||
@@ -80,41 +116,95 @@ ParsedURL ParsedS3URL::toHttpsUrl() const
|
||||
queryParams["versionId"] = *versionId;
|
||||
}
|
||||
|
||||
auto style = addressingStyle.value_or(S3AddressingStyle::Auto);
|
||||
|
||||
// Virtual-hosted-style prepends the bucket name to the hostname, so bucket
|
||||
// names containing dots produce multi-level subdomains (e.g.
|
||||
// my.bucket.s3.amazonaws.com) that break TLS wildcard certificate validation.
|
||||
// In auto mode, fall back to path-style; only error on explicit virtual.
|
||||
auto hasDottedBucket = bucket.find('.') != std::string::npos;
|
||||
auto useVirtualForEndpoint = [&](bool defaultVirtual) {
|
||||
auto useVirtual = defaultVirtual ? style != S3AddressingStyle::Path : style == S3AddressingStyle::Virtual;
|
||||
if (useVirtual && hasDottedBucket) {
|
||||
if (style == S3AddressingStyle::Virtual)
|
||||
throw Error(
|
||||
"bucket name '%s' contains a dot, which is incompatible with "
|
||||
"virtual-hosted-style addressing (causes TLS certificate errors); "
|
||||
"use 'addressing-style=path' or 'addressing-style=auto' instead",
|
||||
bucket);
|
||||
static std::atomic<bool> warnedDottedBucket{false};
|
||||
warnOnce(
|
||||
warnedDottedBucket,
|
||||
"bucket name '%s' contains a dot; falling back to path-style addressing "
|
||||
"(virtual-hosted-style requires non-dotted bucket names for TLS certificate validity); "
|
||||
"set 'addressing-style=path' to silence this warning",
|
||||
bucket);
|
||||
return false;
|
||||
}
|
||||
return useVirtual;
|
||||
};
|
||||
|
||||
// Handle endpoint configuration using std::visit
|
||||
return std::visit(
|
||||
overloaded{
|
||||
[&](const std::monostate &) {
|
||||
// No custom endpoint, use standard AWS S3 endpoint
|
||||
// No custom endpoint: use virtual-hosted-style by default (auto),
|
||||
// path-style when explicitly requested or for dotted bucket names.
|
||||
auto useVirtual = useVirtualForEndpoint(/* defaultVirtual = */ true);
|
||||
std::vector<std::string> path{""};
|
||||
path.push_back(bucket);
|
||||
if (!useVirtual)
|
||||
path.push_back(bucket);
|
||||
path.insert(path.end(), key.begin(), key.end());
|
||||
return ParsedURL{
|
||||
.scheme = std::string{schemeStr},
|
||||
.authority = ParsedURL::Authority{.host = "s3." + regionStr + ".amazonaws.com"},
|
||||
.authority =
|
||||
ParsedURL::Authority{
|
||||
.host = useVirtual ? bucket + ".s3." + regionStr + ".amazonaws.com"
|
||||
: "s3." + regionStr + ".amazonaws.com"},
|
||||
.path = std::move(path),
|
||||
.query = std::move(queryParams),
|
||||
};
|
||||
},
|
||||
[&](const ParsedURL::Authority & auth) {
|
||||
// Endpoint is just an authority (hostname/port)
|
||||
// Custom endpoint authority: use path-style by default (auto),
|
||||
// virtual-hosted-style only when explicitly requested (not for dotted buckets).
|
||||
auto useVirtual = useVirtualForEndpoint(/* defaultVirtual = */ false);
|
||||
if (useVirtual && auth.host.empty())
|
||||
throw Error(
|
||||
"cannot use virtual-hosted-style addressing with endpoint '%s' "
|
||||
"because it has no hostname; use 'addressing-style=path' instead",
|
||||
auth.to_string());
|
||||
std::vector<std::string> path{""};
|
||||
path.push_back(bucket);
|
||||
if (!useVirtual)
|
||||
path.push_back(bucket);
|
||||
path.insert(path.end(), key.begin(), key.end());
|
||||
return ParsedURL{
|
||||
.scheme = std::string{schemeStr},
|
||||
.authority = auth,
|
||||
.authority =
|
||||
useVirtual ? ParsedURL::Authority{.host = bucket + "." + auth.host, .port = auth.port} : auth,
|
||||
.path = std::move(path),
|
||||
.query = std::move(queryParams),
|
||||
};
|
||||
},
|
||||
[&](const ParsedURL & endpointUrl) {
|
||||
// Endpoint is already a ParsedURL (e.g., http://server:9000)
|
||||
// Full endpoint URL: use path-style by default (auto),
|
||||
// virtual-hosted-style only when explicitly requested (not for dotted buckets).
|
||||
auto useVirtual = useVirtualForEndpoint(/* defaultVirtual = */ false);
|
||||
if (useVirtual && (!endpointUrl.authority || endpointUrl.authority->host.empty()))
|
||||
throw Error(
|
||||
"cannot use virtual-hosted-style addressing with endpoint '%s' "
|
||||
"because it has no authority (hostname)",
|
||||
endpointUrl.to_string());
|
||||
auto path = endpointUrl.path;
|
||||
path.push_back(bucket);
|
||||
if (!useVirtual)
|
||||
path.push_back(bucket);
|
||||
path.insert(path.end(), key.begin(), key.end());
|
||||
return ParsedURL{
|
||||
.scheme = endpointUrl.scheme,
|
||||
.authority = endpointUrl.authority,
|
||||
.authority = useVirtual ? std::optional{ParsedURL::Authority{
|
||||
.host = bucket + "." + endpointUrl.authority->host,
|
||||
.port = endpointUrl.authority->port}}
|
||||
: endpointUrl.authority,
|
||||
.path = std::move(path),
|
||||
.query = std::move(queryParams),
|
||||
};
|
||||
@@ -123,4 +213,42 @@ ParsedURL ParsedS3URL::toHttpsUrl() const
|
||||
endpoint);
|
||||
}
|
||||
|
||||
void to_json(nlohmann::json & j, const S3AddressingStyle & e)
|
||||
{
|
||||
j = std::string{showS3AddressingStyle(e)};
|
||||
}
|
||||
|
||||
void from_json(const nlohmann::json & j, S3AddressingStyle & e)
|
||||
{
|
||||
e = parseS3AddressingStyle(j.get<std::string>());
|
||||
}
|
||||
|
||||
template<>
|
||||
struct json_avoids_null<S3AddressingStyle> : std::true_type
|
||||
{};
|
||||
|
||||
template<>
|
||||
S3AddressingStyle BaseSetting<S3AddressingStyle>::parse(const std::string & str) const
|
||||
{
|
||||
try {
|
||||
return parseS3AddressingStyle(str);
|
||||
} catch (InvalidS3AddressingStyle &) {
|
||||
throw UsageError("option '%s' has invalid value '%s', expected 'auto', 'path', or 'virtual'", name, str);
|
||||
}
|
||||
}
|
||||
|
||||
template<>
|
||||
std::string BaseSetting<S3AddressingStyle>::to_string() const
|
||||
{
|
||||
return std::string{showS3AddressingStyle(value)};
|
||||
}
|
||||
|
||||
template<>
|
||||
struct BaseSetting<S3AddressingStyle>::trait
|
||||
{
|
||||
static constexpr bool appendable = false;
|
||||
};
|
||||
|
||||
template class BaseSetting<S3AddressingStyle>;
|
||||
|
||||
} // namespace nix
|
||||
@@ -15,4 +15,32 @@ TEST(alignUp, notAPowerOf2)
|
||||
ASSERT_DEATH({ alignUp(1u, 42); }, "alignment must be a power of 2");
|
||||
}
|
||||
|
||||
template<typename T>
|
||||
class alignUpOverflowTest : public ::testing::Test
|
||||
{};
|
||||
|
||||
using UnsignedTypes = ::testing::Types<uint8_t, uint16_t, uint32_t, uint64_t>;
|
||||
TYPED_TEST_SUITE(alignUpOverflowTest, UnsignedTypes);
|
||||
|
||||
TYPED_TEST(alignUpOverflowTest, lastSafeValue)
|
||||
{
|
||||
constexpr auto max = std::numeric_limits<TypeParam>::max();
|
||||
ASSERT_EQ(alignUp<TypeParam>(max - 15, 16), (max - 15) & ~TypeParam{15});
|
||||
ASSERT_NO_THROW(alignUp<TypeParam>(max - 15, 16));
|
||||
}
|
||||
|
||||
TYPED_TEST(alignUpOverflowTest, overflowThrows)
|
||||
{
|
||||
constexpr auto max = std::numeric_limits<TypeParam>::max();
|
||||
ASSERT_THROW(alignUp<TypeParam>(max - 14, 16), Error);
|
||||
ASSERT_THROW(alignUp<TypeParam>(max, 16), Error);
|
||||
ASSERT_THROW(alignUp<TypeParam>(max, 2), Error);
|
||||
}
|
||||
|
||||
TYPED_TEST(alignUpOverflowTest, alignmentOneNeverOverflows)
|
||||
{
|
||||
constexpr auto max = std::numeric_limits<TypeParam>::max();
|
||||
ASSERT_EQ(alignUp<TypeParam>(max, 1), max);
|
||||
}
|
||||
|
||||
} // namespace nix
|
||||
@@ -1,9 +1,11 @@
|
||||
#pragma once
|
||||
///@file
|
||||
|
||||
#include "nix/util/error.hh"
|
||||
|
||||
#include <cassert>
|
||||
#include <limits>
|
||||
#include <type_traits>
|
||||
#include <cstdint>
|
||||
#include <bit>
|
||||
|
||||
namespace nix {
|
||||
@@ -16,7 +18,10 @@ template<typename T>
|
||||
constexpr T alignUp(T val, unsigned alignment)
|
||||
{
|
||||
assert(std::has_single_bit(alignment) && "alignment must be a power of 2");
|
||||
T mask = ~(T{alignment} - 1u);
|
||||
assert(alignment <= std::numeric_limits<T>::max());
|
||||
T mask = ~(static_cast<T>(alignment) - 1u);
|
||||
if (val > std::numeric_limits<T>::max() - (alignment - 1)) /* Overflow check. */
|
||||
throw Error("can't align %d to %d: value is too large", val, alignment);
|
||||
return (val + alignment - 1) & mask;
|
||||
}
|
||||
|
||||
|
||||
+14
-9
@@ -150,18 +150,23 @@ public:
|
||||
|
||||
Verbosity verbosity = lvlInfo;
|
||||
|
||||
void writeToStderr(std::string_view s)
|
||||
static void writeFullLogging(Descriptor fd, std::string_view s)
|
||||
{
|
||||
try {
|
||||
writeFull(getStandardError(), s, false);
|
||||
writeFull(fd, s, false);
|
||||
} catch (SystemError & e) {
|
||||
/* Ignore failing writes to stderr. We need to ignore write
|
||||
errors to ensure that cleanup code that logs to stderr runs
|
||||
to completion if the other side of stderr has been closed
|
||||
unexpectedly. */
|
||||
/* Ignore failing logging writes. We need to ignore write
|
||||
errors to ensure that cleanup code that writes logs runs
|
||||
to completion if the other side of the logging fd has
|
||||
been closed unexpectedly. */
|
||||
}
|
||||
}
|
||||
|
||||
void writeToStderr(std::string_view s)
|
||||
{
|
||||
writeFullLogging(getStandardError(), s);
|
||||
}
|
||||
|
||||
std::unique_ptr<Logger> makeSimpleLogger(bool printBuildLogs)
|
||||
{
|
||||
return std::make_unique<SimpleLogger>(printBuildLogs);
|
||||
@@ -245,15 +250,15 @@ struct JSONLogger : Logger
|
||||
|
||||
void write(const nlohmann::json & json)
|
||||
{
|
||||
auto line =
|
||||
(includeNixPrefix ? "@nix " : "") + json.dump(-1, ' ', false, nlohmann::json::error_handler_t::replace);
|
||||
auto line = (includeNixPrefix ? "@nix " : "")
|
||||
+ json.dump(-1, ' ', false, nlohmann::json::error_handler_t::replace) + "\n";
|
||||
|
||||
/* Acquire a lock to prevent log messages from clobbering each
|
||||
other. */
|
||||
try {
|
||||
auto state(_state.lock());
|
||||
if (state->enabled)
|
||||
writeLine(fd, line);
|
||||
writeFullLogging(fd, line);
|
||||
} catch (...) {
|
||||
bool enabled = false;
|
||||
std::swap(_state.lock()->enabled, enabled);
|
||||
|
||||
@@ -48,9 +48,14 @@ in
|
||||
rootCredentialsFile = pkgs.writeText "minio-credentials-full" ''
|
||||
MINIO_ROOT_USER=${accessKey}
|
||||
MINIO_ROOT_PASSWORD=${secretKey}
|
||||
MINIO_DOMAIN=minio.local
|
||||
'';
|
||||
};
|
||||
networking.firewall.allowedTCPPorts = [ 9000 ];
|
||||
# Static hosts for virtual-hosted-style S3 tests.
|
||||
# MinIO with MINIO_DOMAIN=minio.local accepts virtual-hosted requests
|
||||
# where the bucket name is a hostname prefix.
|
||||
networking.extraHosts = "127.0.0.1 vhost-test.minio.local minio.local";
|
||||
};
|
||||
|
||||
client =
|
||||
@@ -62,6 +67,7 @@ in
|
||||
experimental-features = nix-command
|
||||
substituters =
|
||||
'';
|
||||
networking.extraHosts = "192.168.1.2 vhost-test.minio.local minio.local";
|
||||
};
|
||||
};
|
||||
|
||||
@@ -83,6 +89,10 @@ in
|
||||
ENDPOINT = 'http://server:9000'
|
||||
REGION = 'eu-west-1'
|
||||
|
||||
# Virtual-hosted-style configuration (requires MINIO_DOMAIN and static host entries)
|
||||
VHOST_DOMAIN = 'minio.local'
|
||||
VHOST_ENDPOINT = f'http://{VHOST_DOMAIN}:9000'
|
||||
|
||||
PKGS = {
|
||||
'A': '${pkgA}',
|
||||
'B': '${pkgB}',
|
||||
@@ -859,6 +869,92 @@ in
|
||||
print(output)
|
||||
raise Exception("Expected SSO provider to be skipped")
|
||||
|
||||
def test_virtual_hosted_copy():
|
||||
"""Test nix copy with virtual-hosted-style addressing on custom endpoint"""
|
||||
print("\n=== Testing Virtual-Hosted-Style Addressing ===")
|
||||
|
||||
# Use a fixed bucket name matching the static /etc/hosts entries
|
||||
bucket = 'vhost-test'
|
||||
server.succeed(f"mc mb minio/{bucket}")
|
||||
try:
|
||||
store_url = make_s3_url(
|
||||
bucket,
|
||||
endpoint=VHOST_ENDPOINT,
|
||||
**{'addressing-style': 'virtual'}
|
||||
)
|
||||
|
||||
# Upload with virtual-hosted-style, capture debug output
|
||||
output = server.succeed(
|
||||
f"{ENV_WITH_CREDS} nix copy --debug --to '{store_url}' {PKGS['A']} 2>&1"
|
||||
)
|
||||
|
||||
# Verify virtual-hosted-style URL was used (bucket in hostname)
|
||||
vhost_url_prefix = f"http://{bucket}.{VHOST_DOMAIN}:9000/"
|
||||
if vhost_url_prefix not in output:
|
||||
print("Debug output:")
|
||||
print(output)
|
||||
raise Exception(
|
||||
f"Expected virtual-hosted-style URL containing '{vhost_url_prefix}'"
|
||||
)
|
||||
|
||||
# Verify path-style URL was NOT used (bucket should not be in the path)
|
||||
path_style_pattern = f"{VHOST_ENDPOINT}/{bucket}/"
|
||||
if path_style_pattern in output:
|
||||
print("Debug output:")
|
||||
print(output)
|
||||
raise Exception("Found path-style URL when virtual-hosted-style was expected")
|
||||
|
||||
# Download with virtual-hosted-style
|
||||
verify_packages_in_store(client, PKGS['A'], should_exist=False)
|
||||
output = client.succeed(
|
||||
f"{ENV_WITH_CREDS} nix copy --debug --no-check-sigs "
|
||||
f"--from '{store_url}' {PKGS['A']} 2>&1"
|
||||
)
|
||||
|
||||
if vhost_url_prefix not in output:
|
||||
print("Debug output:")
|
||||
print(output)
|
||||
raise Exception(
|
||||
f"Expected virtual-hosted-style URL in download containing '{vhost_url_prefix}'"
|
||||
)
|
||||
|
||||
verify_packages_in_store(client, PKGS['A'])
|
||||
finally:
|
||||
server.succeed(f"mc rb --force minio/{bucket}")
|
||||
for pkg in PKGS.values():
|
||||
client.succeed(f"[ ! -e {pkg} ] || nix store delete --ignore-liveness {pkg}")
|
||||
|
||||
@setup_s3()
|
||||
def test_explicit_path_style(bucket):
|
||||
"""Test that addressing-style=path works as backwards-compatible fallback"""
|
||||
print("\n=== Testing Explicit Path-Style Addressing ===")
|
||||
|
||||
store_url = make_s3_url(
|
||||
bucket,
|
||||
**{'addressing-style': 'path'}
|
||||
)
|
||||
|
||||
# Upload with explicit path-style
|
||||
output = server.succeed(
|
||||
f"{ENV_WITH_CREDS} nix copy --debug --to '{store_url}' {PKGS['A']} 2>&1"
|
||||
)
|
||||
|
||||
# Verify path-style URL was used (bucket in path, not hostname)
|
||||
path_style_pattern = f"{ENDPOINT}/{bucket}/"
|
||||
if path_style_pattern not in output:
|
||||
print("Debug output:")
|
||||
print(output)
|
||||
raise Exception(
|
||||
f"Expected path-style URL containing '{path_style_pattern}'"
|
||||
)
|
||||
|
||||
# Download
|
||||
verify_packages_in_store(client, PKGS['A'], should_exist=False)
|
||||
client.succeed(
|
||||
f"{ENV_WITH_CREDS} nix copy --no-check-sigs --from '{store_url}' {PKGS['A']}"
|
||||
)
|
||||
verify_packages_in_store(client, PKGS['A'])
|
||||
|
||||
# ============================================================================
|
||||
# Main Test Execution
|
||||
# ============================================================================
|
||||
@@ -896,5 +992,7 @@ in
|
||||
test_profile_credentials()
|
||||
test_env_vars_precedence()
|
||||
test_credential_provider_chain()
|
||||
test_virtual_hosted_copy()
|
||||
test_explicit_path_style()
|
||||
'';
|
||||
}
|
||||
Reference in new issue
Block a user