Compare commits
32
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
4816b640a2 | ||
|
|
79211b6110 | ||
|
|
f5f9e32f54 | ||
|
|
564593bcb9 | ||
|
|
8ee74792fe | ||
|
|
e82210b3b2 | ||
|
|
625477a7df | ||
|
|
231f3af535 | ||
|
|
cc4aa70e6e | ||
|
|
0bd9d6a28e | ||
|
|
7989e3192d | ||
|
|
1e16a54ee5 | ||
|
|
afade27123 | ||
|
|
c1e2396d58 | ||
|
|
ca94905593 | ||
|
|
e492c64c8e | ||
|
|
e4e8a615fa | ||
|
|
fac34ad20f | ||
|
|
024d3954af | ||
|
|
f0e4af4365 | ||
|
|
5985d67906 | ||
|
|
9bee0fa6ac | ||
|
|
adec28bf85 | ||
|
|
f5e09d9b58 | ||
|
|
f67daa4a87 | ||
|
|
c9211b0b2d | ||
|
|
2fa2c0b09f | ||
|
|
ebf1cf5227 | ||
|
|
3e86d75c9d | ||
|
|
72a548ed6a | ||
|
|
4083eff0c0 | ||
|
|
a1b3934a78 |
No files matched your search
@@ -161,3 +161,14 @@ pull_request_rules:
|
||||
labels:
|
||||
- automatic backport
|
||||
- merge-queue
|
||||
|
||||
- name: backport patches to 2.31
|
||||
conditions:
|
||||
- label=backport 2.31-maintenance
|
||||
actions:
|
||||
backport:
|
||||
branches:
|
||||
- "2.31-maintenance"
|
||||
labels:
|
||||
- automatic backport
|
||||
- merge-queue
|
||||
Generated
+3
-3
@@ -63,11 +63,11 @@
|
||||
},
|
||||
"nixpkgs": {
|
||||
"locked": {
|
||||
"lastModified": 1755442223,
|
||||
"narHash": "sha256-VtMQg02B3kt1oejwwrGn50U9Xbjgzfbb5TV5Wtx8dKI=",
|
||||
"lastModified": 1756178832,
|
||||
"narHash": "sha256-O2CIn7HjZwEGqBrwu9EU76zlmA5dbmna7jL1XUmAId8=",
|
||||
"owner": "NixOS",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "cd32a774ac52caaa03bcfc9e7591ac8c18617ced",
|
||||
"rev": "d98ce345cdab58477ca61855540999c86577d19d",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
|
||||
@@ -46,7 +46,7 @@ The team meets twice a week (times are denoted in the [Europe/Amsterdam](https:/
|
||||
- mark it as draft if it is blocked on the contributor
|
||||
- escalate it back to the team by moving it to To discuss, and leaving a comment as to why the issue needs to be discussed again.
|
||||
|
||||
- Work meeting: Mondays 14:00-16:00 Europe/Amsterdam see [calendar](https://calendar.google.com/calendar/u/0/embed?src=b9o52fobqjak8oq8lfkhg3t0qg@group.calendar.google.com).
|
||||
- Work meeting: Mondays 18:00-20:00 Europe/Amsterdam; see [calendar](https://calendar.google.com/calendar/u/0/embed?src=b9o52fobqjak8oq8lfkhg3t0qg@group.calendar.google.com).
|
||||
|
||||
1. Code review on pull requests from [In review](#in-review).
|
||||
2. Other chores and tasks.
|
||||
|
||||
Executable
+58
@@ -0,0 +1,58 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
# debug:
|
||||
# set -x
|
||||
|
||||
START_REF="${1}"
|
||||
END_REF="${2:-upstream/master}"
|
||||
|
||||
# Get the merge base
|
||||
MERGE_BASE=$(git merge-base "$START_REF" "$END_REF")
|
||||
unset START_REF
|
||||
|
||||
# Get date range
|
||||
START_DATE=$(git show -s --format=%cI "$MERGE_BASE")
|
||||
END_DATE=$(git show -s --format=%cI "$END_REF")
|
||||
|
||||
echo "Checking PRs merged between $START_DATE and $END_DATE" >&2
|
||||
|
||||
# Get all commits between merge base and HEAD
|
||||
COMMITS=$(git rev-list "$MERGE_BASE..$END_REF")
|
||||
|
||||
# Convert to set for fast lookup
|
||||
declare -A commit_set
|
||||
for commit in $COMMITS; do
|
||||
commit_set["$commit"]=1
|
||||
done
|
||||
|
||||
# Get the current changelog
|
||||
LOG_DONE="$(changelog-d doc/manual/rl-next)"
|
||||
is_done(){
|
||||
local nr="$1"
|
||||
echo "$LOG_DONE" | grep -E "^- .*/pull/$nr)"
|
||||
}
|
||||
|
||||
# Query merged PRs in date range
|
||||
gh pr list \
|
||||
--repo NixOS/nix \
|
||||
--state merged \
|
||||
--limit 1000 \
|
||||
--json number,title,author,mergeCommit \
|
||||
--search "merged:$START_DATE..$END_DATE" | \
|
||||
jq -r '.[] | [.number, .mergeCommit.oid, .title, .author.login] | @tsv' | \
|
||||
while IFS=$'\t' read -r pr_num merge_commit _title author; do
|
||||
# Check if this PR's merge commit is in our branch
|
||||
if [[ -n "${commit_set[$merge_commit]:-}" ]]; then
|
||||
# Full detail, not suitable for comment due to mass ping and duplicate title
|
||||
# echo "- #$pr_num $_title (@$author)"
|
||||
echo "- #$pr_num ($author)"
|
||||
if is_done "$pr_num"
|
||||
then
|
||||
echo " - [x] has note"
|
||||
else
|
||||
echo " - [ ] has note"
|
||||
fi
|
||||
echo " - [ ] skip"
|
||||
fi
|
||||
done
|
||||
@@ -24,6 +24,12 @@ release:
|
||||
* In a checkout of the Nix repo, make sure you're on `master` and run
|
||||
`git pull`.
|
||||
|
||||
* Compile a release notes to-do list by running
|
||||
|
||||
```console
|
||||
$ ./maintainers/release-notes-todo PREV_RELEASE HEAD
|
||||
```
|
||||
|
||||
* Compile the release notes by running
|
||||
|
||||
```console
|
||||
@@ -127,6 +133,8 @@ release:
|
||||
|
||||
Commit and push this to the maintenance branch.
|
||||
|
||||
* Create a backport label.
|
||||
|
||||
* Bump the version of `master`:
|
||||
|
||||
```console
|
||||
@@ -134,6 +142,7 @@ release:
|
||||
$ git pull
|
||||
$ NEW_VERSION=2.13.0
|
||||
$ echo $NEW_VERSION > .version
|
||||
$ ... edit .mergify.yml to add the previous version ...
|
||||
$ git checkout -b bump-$NEW_VERSION
|
||||
$ git commit -a -m 'Bump version'
|
||||
$ git push --set-upstream origin bump-$NEW_VERSION
|
||||
@@ -141,10 +150,6 @@ release:
|
||||
|
||||
Make a pull request and auto-merge it.
|
||||
|
||||
* Create a backport label.
|
||||
|
||||
* Add the new backport label to `.mergify.yml`.
|
||||
|
||||
* Post an [announcement on Discourse](https://discourse.nixos.org/c/announcements/8), including the contents of
|
||||
`rl-$VERSION.md`.
|
||||
|
||||
|
||||
@@ -595,12 +595,17 @@ struct ExprOpNot : Expr
|
||||
{ \
|
||||
return pos; \
|
||||
} \
|
||||
};
|
||||
}
|
||||
|
||||
MakeBinOp(ExprOpEq, "==") MakeBinOp(ExprOpNEq, "!=") MakeBinOp(ExprOpAnd, "&&") MakeBinOp(ExprOpOr, "||")
|
||||
MakeBinOp(ExprOpImpl, "->") MakeBinOp(ExprOpUpdate, "//") MakeBinOp(ExprOpConcatLists, "++")
|
||||
MakeBinOp(ExprOpEq, "==");
|
||||
MakeBinOp(ExprOpNEq, "!=");
|
||||
MakeBinOp(ExprOpAnd, "&&");
|
||||
MakeBinOp(ExprOpOr, "||");
|
||||
MakeBinOp(ExprOpImpl, "->");
|
||||
MakeBinOp(ExprOpUpdate, "//");
|
||||
MakeBinOp(ExprOpConcatLists, "++");
|
||||
|
||||
struct ExprConcatStrings : Expr
|
||||
struct ExprConcatStrings : Expr
|
||||
{
|
||||
PosIdx pos;
|
||||
bool forceString;
|
||||
|
||||
@@ -185,7 +185,7 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value ** args
|
||||
{.msg = HintFmt("attribute '%s' is missing in call to 'fetchClosure'", "fromStore"),
|
||||
.pos = state.positions[pos]});
|
||||
|
||||
auto parsedURL = parseURL(*fromStoreUrl);
|
||||
auto parsedURL = parseURL(*fromStoreUrl, /*lenient=*/true);
|
||||
|
||||
if (parsedURL.scheme != "http" && parsedURL.scheme != "https"
|
||||
&& !(getEnv("_NIX_IN_TEST").has_value() && parsedURL.scheme == "file"))
|
||||
|
||||
@@ -113,7 +113,7 @@ static void fetchTree(
|
||||
auto s = state.coerceToString(attr.pos, *attr.value, context, "", false, false).toOwned();
|
||||
attrs.emplace(
|
||||
state.symbols[attr.name],
|
||||
params.isFetchGit && state.symbols[attr.name] == "url" ? fixGitURL(s) : s);
|
||||
params.isFetchGit && state.symbols[attr.name] == "url" ? fixGitURL(s).to_string() : s);
|
||||
} else if (attr.value->type() == nBool)
|
||||
attrs.emplace(state.symbols[attr.name], Explicit<bool>{attr.value->boolean()});
|
||||
else if (attr.value->type() == nInt) {
|
||||
@@ -175,7 +175,7 @@ static void fetchTree(
|
||||
if (params.isFetchGit) {
|
||||
fetchers::Attrs attrs;
|
||||
attrs.emplace("type", "git");
|
||||
attrs.emplace("url", fixGitURL(url));
|
||||
attrs.emplace("url", fixGitURL(url).to_string());
|
||||
if (!attrs.contains("exportIgnore")
|
||||
&& (!attrs.contains("submodules") || !*fetchers::maybeGetBoolAttr(attrs, "submodules"))) {
|
||||
attrs.emplace("exportIgnore", Explicit<bool>{true});
|
||||
|
||||
@@ -25,7 +25,7 @@ static void downloadToSink(
|
||||
std::string sha256Expected,
|
||||
size_t sizeExpected)
|
||||
{
|
||||
FileTransferRequest request(url);
|
||||
FileTransferRequest request(parseURL(url));
|
||||
Headers headers;
|
||||
if (authHeader.has_value())
|
||||
headers.push_back({"Authorization", *authHeader});
|
||||
@@ -179,7 +179,7 @@ Fetch::Fetch(git_repository * repo, git_oid rev)
|
||||
|
||||
const auto remoteUrl = lfs::getLfsEndpointUrl(repo);
|
||||
|
||||
this->url = nix::parseURL(nix::fixGitURL(remoteUrl)).canonicalise();
|
||||
this->url = nix::fixGitURL(remoteUrl).canonicalise();
|
||||
}
|
||||
|
||||
bool Fetch::shouldFetch(const CanonPath & path) const
|
||||
@@ -207,7 +207,7 @@ std::vector<nlohmann::json> Fetch::fetchUrls(const std::vector<Pointer> & pointe
|
||||
auto api = lfs::getLfsApi(this->url);
|
||||
auto url = api.endpoint + "/objects/batch";
|
||||
const auto & authHeader = api.authHeader;
|
||||
FileTransferRequest request(url);
|
||||
FileTransferRequest request(parseURL(url));
|
||||
request.post = true;
|
||||
Headers headers;
|
||||
if (authHeader.has_value())
|
||||
|
||||
@@ -233,9 +233,7 @@ struct GitInputScheme : InputScheme
|
||||
|
||||
Input input{settings};
|
||||
input.attrs = attrs;
|
||||
auto url = fixGitURL(getStrAttr(attrs, "url"));
|
||||
parseURL(url);
|
||||
input.attrs["url"] = url;
|
||||
input.attrs["url"] = fixGitURL(getStrAttr(attrs, "url")).to_string();
|
||||
getShallowAttr(input);
|
||||
getSubmodulesAttr(input);
|
||||
getAllRefsAttr(input);
|
||||
|
||||
@@ -19,7 +19,7 @@ namespace nix::fetchers {
|
||||
|
||||
struct DownloadUrl
|
||||
{
|
||||
std::string url;
|
||||
ParsedURL url;
|
||||
Headers headers;
|
||||
};
|
||||
|
||||
@@ -420,7 +420,7 @@ struct GitHubInputScheme : GitArchiveInputScheme
|
||||
const auto url =
|
||||
fmt(urlFmt, host, getOwner(input), getRepo(input), input.getRev()->to_string(HashFormat::Base16, false));
|
||||
|
||||
return DownloadUrl{url, headers};
|
||||
return DownloadUrl{parseURL(url), headers};
|
||||
}
|
||||
|
||||
void clone(const Input & input, const Path & destDir) const override
|
||||
@@ -500,7 +500,7 @@ struct GitLabInputScheme : GitArchiveInputScheme
|
||||
input.getRev()->to_string(HashFormat::Base16, false));
|
||||
|
||||
Headers headers = makeHeadersWithAuthTokens(*input.settings, host, input);
|
||||
return DownloadUrl{url, headers};
|
||||
return DownloadUrl{parseURL(url), headers};
|
||||
}
|
||||
|
||||
void clone(const Input & input, const Path & destDir) const override
|
||||
@@ -592,7 +592,7 @@ struct SourceHutInputScheme : GitArchiveInputScheme
|
||||
input.getRev()->to_string(HashFormat::Base16, false));
|
||||
|
||||
Headers headers = makeHeadersWithAuthTokens(*input.settings, host, input);
|
||||
return DownloadUrl{url, headers};
|
||||
return DownloadUrl{parseURL(url), headers};
|
||||
}
|
||||
|
||||
void clone(const Input & input, const Path & destDir) const override
|
||||
|
||||
@@ -43,7 +43,7 @@ DownloadFileResult downloadFile(
|
||||
if (cached && !cached->expired)
|
||||
return useCached();
|
||||
|
||||
FileTransferRequest request(url);
|
||||
FileTransferRequest request(parseURL(url));
|
||||
request.headers = headers;
|
||||
if (cached)
|
||||
request.expectedETag = getStrAttr(cached->value, "etag");
|
||||
@@ -153,7 +153,7 @@ static DownloadTarballResult downloadTarball_(
|
||||
auto _res = std::make_shared<Sync<FileTransferResult>>();
|
||||
|
||||
auto source = sinkToSource([&](Sink & sink) {
|
||||
FileTransferRequest req(url);
|
||||
FileTransferRequest req(parseURL(url));
|
||||
req.expectedETag = cached ? getStrAttr(cached->value, "etag") : "";
|
||||
getFileTransfer()->download(std::move(req), sink, [_res](FileTransferResult r) { *_res->lock() = r; });
|
||||
});
|
||||
|
||||
@@ -13,8 +13,9 @@ TEST(getNameFromURL, getNameFromURL)
|
||||
ASSERT_EQ(getNameFromURL(parseURL("path:~/repos/nixpkgs#packages.x86_64-linux.Hello")), "Hello");
|
||||
ASSERT_EQ(getNameFromURL(parseURL("path:.#nonStandardAttr.mylaptop")), "mylaptop");
|
||||
ASSERT_EQ(getNameFromURL(parseURL("path:./repos/myflake#nonStandardAttr.mylaptop")), "mylaptop");
|
||||
ASSERT_EQ(getNameFromURL(parseURL("path:./nixpkgs#packages.x86_64-linux.complex^bin,man")), "complex");
|
||||
ASSERT_EQ(getNameFromURL(parseURL("path:./myproj#packages.x86_64-linux.default^*")), "myproj");
|
||||
ASSERT_EQ(
|
||||
getNameFromURL(parseURL("path:./nixpkgs#packages.x86_64-linux.complex^bin,man", /*lenient=*/true)), "complex");
|
||||
ASSERT_EQ(getNameFromURL(parseURL("path:./myproj#packages.x86_64-linux.default^*", /*lenient=*/true)), "myproj");
|
||||
ASSERT_EQ(getNameFromURL(parseURL("path:./myproj#defaultPackage.x86_64-linux")), "myproj");
|
||||
|
||||
ASSERT_EQ(getNameFromURL(parseURL("github:NixOS/nixpkgs#packages.x86_64-linux.hello")), "hello");
|
||||
@@ -80,6 +81,6 @@ TEST(getNameFromURL, getNameFromURL)
|
||||
ASSERT_EQ(getNameFromURL(parseURL("path:.")), std::nullopt);
|
||||
ASSERT_EQ(getNameFromURL(parseURL("file:.#")), std::nullopt);
|
||||
ASSERT_EQ(getNameFromURL(parseURL("path:.#packages.x86_64-linux.default")), std::nullopt);
|
||||
ASSERT_EQ(getNameFromURL(parseURL("path:.#packages.x86_64-linux.default^*")), std::nullopt);
|
||||
ASSERT_EQ(getNameFromURL(parseURL("path:.#packages.x86_64-linux.default^*", /*lenient=*/true)), std::nullopt);
|
||||
}
|
||||
} // namespace nix
|
||||
@@ -82,7 +82,7 @@ std::pair<FlakeRef, std::string> parsePathFlakeRefWithFragment(
|
||||
auto succeeds = std::regex_match(url, match, pathFlakeRegex);
|
||||
assert(succeeds);
|
||||
auto path = match[1].str();
|
||||
auto query = decodeQuery(match[3]);
|
||||
auto query = decodeQuery(match[3].str(), /*lenient=*/true);
|
||||
auto fragment = percentDecode(match[5].str());
|
||||
|
||||
if (baseDir) {
|
||||
@@ -210,7 +210,7 @@ std::optional<std::pair<FlakeRef, std::string>> parseURLFlakeRef(
|
||||
bool isFlake)
|
||||
{
|
||||
try {
|
||||
auto parsed = parseURL(url);
|
||||
auto parsed = parseURL(url, /*lenient=*/true);
|
||||
if (baseDir && (parsed.scheme == "path" || parsed.scheme == "git+file") && !isAbsolute(parsed.path))
|
||||
parsed.path = absPath(parsed.path, *baseDir);
|
||||
return fromParsedURL(fetchSettings, std::move(parsed), isFlake);
|
||||
@@ -289,7 +289,7 @@ FlakeRef FlakeRef::canonicalize() const
|
||||
filtering the `dir` query parameter from the URL. */
|
||||
if (auto url = fetchers::maybeGetStrAttr(flakeRef.input.attrs, "url")) {
|
||||
try {
|
||||
auto parsed = parseURL(*url);
|
||||
auto parsed = parseURL(*url, /*lenient=*/true);
|
||||
if (auto dir2 = get(parsed.query, "dir")) {
|
||||
if (flakeRef.subdir != "" && flakeRef.subdir == *dir2)
|
||||
parsed.query.erase("dir");
|
||||
|
||||
+108
-8
@@ -21,7 +21,7 @@ class ParsedS3URLTest : public ::testing::WithParamInterface<ParsedS3URLTestCase
|
||||
TEST_P(ParsedS3URLTest, parseS3URLSuccessfully)
|
||||
{
|
||||
const auto & testCase = GetParam();
|
||||
auto parsed = ParsedS3URL::parse(testCase.url);
|
||||
auto parsed = ParsedS3URL::parse(parseURL(testCase.url));
|
||||
ASSERT_EQ(parsed, testCase.expected);
|
||||
}
|
||||
|
||||
@@ -35,7 +35,8 @@ INSTANTIATE_TEST_SUITE_P(
|
||||
.bucket = "my-bucket",
|
||||
.key = "my-key.txt",
|
||||
},
|
||||
"basic_s3_bucket"},
|
||||
"basic_s3_bucket",
|
||||
},
|
||||
ParsedS3URLTestCase{
|
||||
"s3://prod-cache/nix/store/abc123.nar.xz?region=eu-west-1",
|
||||
{
|
||||
@@ -43,7 +44,8 @@ INSTANTIATE_TEST_SUITE_P(
|
||||
.key = "nix/store/abc123.nar.xz",
|
||||
.region = "eu-west-1",
|
||||
},
|
||||
"with_region"},
|
||||
"with_region",
|
||||
},
|
||||
ParsedS3URLTestCase{
|
||||
"s3://bucket/key?region=us-west-2&profile=prod&endpoint=custom.s3.com&scheme=https®ion=us-east-1",
|
||||
{
|
||||
@@ -54,7 +56,8 @@ INSTANTIATE_TEST_SUITE_P(
|
||||
.scheme = "https",
|
||||
.endpoint = ParsedURL::Authority{.host = "custom.s3.com"},
|
||||
},
|
||||
"complex"},
|
||||
"complex",
|
||||
},
|
||||
ParsedS3URLTestCase{
|
||||
"s3://cache/file.txt?profile=production®ion=ap-southeast-2",
|
||||
{
|
||||
@@ -63,7 +66,8 @@ INSTANTIATE_TEST_SUITE_P(
|
||||
.profile = "production",
|
||||
.region = "ap-southeast-2",
|
||||
},
|
||||
"with_profile_and_region"},
|
||||
"with_profile_and_region",
|
||||
},
|
||||
ParsedS3URLTestCase{
|
||||
"s3://bucket/key?endpoint=https://minio.local&scheme=http",
|
||||
{
|
||||
@@ -77,7 +81,8 @@ INSTANTIATE_TEST_SUITE_P(
|
||||
.authority = ParsedURL::Authority{.host = "minio.local"},
|
||||
},
|
||||
},
|
||||
"with_absolute_endpoint_uri"}),
|
||||
"with_absolute_endpoint_uri",
|
||||
}),
|
||||
[](const ::testing::TestParamInfo<ParsedS3URLTestCase> & info) { return info.param.description; });
|
||||
|
||||
TEST(InvalidParsedS3URLTest, parseS3URLErrors)
|
||||
@@ -86,11 +91,106 @@ TEST(InvalidParsedS3URLTest, parseS3URLErrors)
|
||||
testing::HasSubstrIgnoreANSIMatcher("error: URI has a missing or invalid bucket name"));
|
||||
|
||||
/* Empty bucket (authority) */
|
||||
ASSERT_THAT([]() { ParsedS3URL::parse("s3:///key"); }, invalidBucketMatcher);
|
||||
ASSERT_THAT([]() { ParsedS3URL::parse(parseURL("s3:///key")); }, invalidBucketMatcher);
|
||||
/* Invalid bucket name */
|
||||
ASSERT_THAT([]() { ParsedS3URL::parse("s3://127.0.0.1"); }, invalidBucketMatcher);
|
||||
ASSERT_THAT([]() { ParsedS3URL::parse(parseURL("s3://127.0.0.1")); }, invalidBucketMatcher);
|
||||
}
|
||||
|
||||
// Parameterized test for s3ToHttpsUrl conversion
|
||||
struct S3ToHttpsConversionTestCase
|
||||
{
|
||||
ParsedS3URL input;
|
||||
ParsedURL expected;
|
||||
std::string description;
|
||||
};
|
||||
|
||||
class S3ToHttpsConversionTest : public ::testing::WithParamInterface<S3ToHttpsConversionTestCase>,
|
||||
public ::testing::Test
|
||||
{};
|
||||
|
||||
TEST_P(S3ToHttpsConversionTest, ConvertsCorrectly)
|
||||
{
|
||||
const auto & testCase = GetParam();
|
||||
auto result = testCase.input.toHttpsUrl();
|
||||
EXPECT_EQ(result, testCase.expected) << "Failed for: " << testCase.description;
|
||||
}
|
||||
|
||||
INSTANTIATE_TEST_SUITE_P(
|
||||
S3ToHttpsConversion,
|
||||
S3ToHttpsConversionTest,
|
||||
::testing::Values(
|
||||
S3ToHttpsConversionTestCase{
|
||||
ParsedS3URL{
|
||||
.bucket = "my-bucket",
|
||||
.key = "my-key.txt",
|
||||
},
|
||||
ParsedURL{
|
||||
.scheme = "https",
|
||||
.authority = ParsedURL::Authority{.host = "s3.us-east-1.amazonaws.com"},
|
||||
.path = "/my-bucket/my-key.txt",
|
||||
},
|
||||
"basic_s3_default_region",
|
||||
},
|
||||
S3ToHttpsConversionTestCase{
|
||||
ParsedS3URL{
|
||||
.bucket = "prod-cache",
|
||||
.key = "nix/store/abc123.nar.xz",
|
||||
.region = "eu-west-1",
|
||||
},
|
||||
ParsedURL{
|
||||
.scheme = "https",
|
||||
.authority = ParsedURL::Authority{.host = "s3.eu-west-1.amazonaws.com"},
|
||||
.path = "/prod-cache/nix/store/abc123.nar.xz",
|
||||
},
|
||||
"with_eu_west_1_region",
|
||||
},
|
||||
S3ToHttpsConversionTestCase{
|
||||
ParsedS3URL{
|
||||
.bucket = "bucket",
|
||||
.key = "key",
|
||||
.scheme = "http",
|
||||
.endpoint = ParsedURL::Authority{.host = "custom.s3.com"},
|
||||
},
|
||||
ParsedURL{
|
||||
.scheme = "http",
|
||||
.authority = ParsedURL::Authority{.host = "custom.s3.com"},
|
||||
.path = "/bucket/key",
|
||||
},
|
||||
"custom_endpoint_authority",
|
||||
},
|
||||
S3ToHttpsConversionTestCase{
|
||||
ParsedS3URL{
|
||||
.bucket = "bucket",
|
||||
.key = "key",
|
||||
.endpoint =
|
||||
ParsedURL{
|
||||
.scheme = "http",
|
||||
.authority = ParsedURL::Authority{.host = "server", .port = 9000},
|
||||
},
|
||||
},
|
||||
ParsedURL{
|
||||
.scheme = "http",
|
||||
.authority = ParsedURL::Authority{.host = "server", .port = 9000},
|
||||
.path = "/bucket/key",
|
||||
},
|
||||
"custom_endpoint_with_port",
|
||||
},
|
||||
S3ToHttpsConversionTestCase{
|
||||
ParsedS3URL{
|
||||
.bucket = "bucket",
|
||||
.key = "path/to/file.txt",
|
||||
.region = "ap-southeast-2",
|
||||
.scheme = "https",
|
||||
},
|
||||
ParsedURL{
|
||||
.scheme = "https",
|
||||
.authority = ParsedURL::Authority{.host = "s3.ap-southeast-2.amazonaws.com"},
|
||||
.path = "/bucket/path/to/file.txt",
|
||||
},
|
||||
"complex_path_and_region",
|
||||
}),
|
||||
[](const ::testing::TestParamInfo<S3ToHttpsConversionTestCase> & info) { return info.param.description; });
|
||||
|
||||
} // namespace nix
|
||||
|
||||
#endif
|
||||
@@ -758,16 +758,16 @@ Goal::Co DerivationBuildingGoal::tryToBuild()
|
||||
*localStoreP,
|
||||
std::make_unique<DerivationBuildingGoalCallbacks>(*this, builder),
|
||||
DerivationBuilderParams{
|
||||
drvPath,
|
||||
buildMode,
|
||||
buildResult,
|
||||
*drv,
|
||||
*drvOptions,
|
||||
inputPaths,
|
||||
initialOutputs,
|
||||
std::move(defaultPathsInChroot),
|
||||
std::move(finalEnv),
|
||||
std::move(extraFiles),
|
||||
.drvPath = drvPath,
|
||||
.buildResult = buildResult,
|
||||
.drv = *drv,
|
||||
.drvOptions = *drvOptions,
|
||||
.inputPaths = inputPaths,
|
||||
.initialOutputs = initialOutputs,
|
||||
.buildMode = buildMode,
|
||||
.defaultPathsInChroot = std::move(defaultPathsInChroot),
|
||||
.finalEnv = std::move(finalEnv),
|
||||
.extraFiles = std::move(extraFiles),
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -37,7 +37,7 @@ static void builtinFetchurl(const BuiltinBuilderContext & ctx)
|
||||
|
||||
auto fetch = [&](const std::string & url) {
|
||||
auto source = sinkToSource([&](Sink & sink) {
|
||||
FileTransferRequest request(url);
|
||||
FileTransferRequest request(parseURL(url));
|
||||
request.decompress = false;
|
||||
|
||||
auto decompressor = makeDecompressionSink(unpack && hasSuffix(mainUrl, ".xz") ? "xz" : "none", sink);
|
||||
|
||||
@@ -100,7 +100,7 @@ struct curlFileTransfer : public FileTransfer
|
||||
lvlTalkative,
|
||||
actFileTransfer,
|
||||
fmt("%sing '%s'", request.verb(), request.uri),
|
||||
{request.uri},
|
||||
{request.uri.to_string()},
|
||||
request.parentAct)
|
||||
, callback(std::move(callback))
|
||||
, finalSink([this](std::string_view data) {
|
||||
@@ -121,7 +121,7 @@ struct curlFileTransfer : public FileTransfer
|
||||
this->result.data.append(data);
|
||||
})
|
||||
{
|
||||
result.urls.push_back(request.uri);
|
||||
result.urls.push_back(request.uri.to_string());
|
||||
|
||||
requestHeaders = curl_slist_append(requestHeaders, "Accept-Encoding: zstd, br, gzip, deflate, bzip2, xz");
|
||||
if (!request.expectedETag.empty())
|
||||
@@ -350,7 +350,7 @@ struct curlFileTransfer : public FileTransfer
|
||||
curl_easy_setopt(req, CURLOPT_DEBUGFUNCTION, TransferItem::debugCallback);
|
||||
}
|
||||
|
||||
curl_easy_setopt(req, CURLOPT_URL, request.uri.c_str());
|
||||
curl_easy_setopt(req, CURLOPT_URL, request.uri.to_string().c_str());
|
||||
curl_easy_setopt(req, CURLOPT_FOLLOWLOCATION, 1L);
|
||||
curl_easy_setopt(req, CURLOPT_MAXREDIRS, 10);
|
||||
curl_easy_setopt(req, CURLOPT_NOSIGNAL, 1);
|
||||
@@ -784,8 +784,8 @@ struct curlFileTransfer : public FileTransfer
|
||||
|
||||
void enqueueItem(std::shared_ptr<TransferItem> item)
|
||||
{
|
||||
if (item->request.data && !hasPrefix(item->request.uri, "http://") && !hasPrefix(item->request.uri, "https://"))
|
||||
throw nix::Error("uploading to '%s' is not supported", item->request.uri);
|
||||
if (item->request.data && item->request.uri.scheme != "http" && item->request.uri.scheme != "https")
|
||||
throw nix::Error("uploading to '%s' is not supported", item->request.uri.to_string());
|
||||
|
||||
{
|
||||
auto state(state_.lock());
|
||||
@@ -801,7 +801,7 @@ struct curlFileTransfer : public FileTransfer
|
||||
void enqueueFileTransfer(const FileTransferRequest & request, Callback<FileTransferResult> callback) override
|
||||
{
|
||||
/* Ugly hack to support s3:// URIs. */
|
||||
if (hasPrefix(request.uri, "s3://")) {
|
||||
if (request.uri.scheme == "s3") {
|
||||
// FIXME: do this on a worker thread
|
||||
try {
|
||||
#if NIX_WITH_S3_SUPPORT
|
||||
@@ -820,10 +820,11 @@ struct curlFileTransfer : public FileTransfer
|
||||
if (!s3Res.data)
|
||||
throw FileTransferError(NotFound, {}, "S3 object '%s' does not exist", request.uri);
|
||||
res.data = std::move(*s3Res.data);
|
||||
res.urls.push_back(request.uri);
|
||||
res.urls.push_back(request.uri.to_string());
|
||||
callback(std::move(res));
|
||||
#else
|
||||
throw nix::Error("cannot download '%s' because Nix is not built with S3 support", request.uri);
|
||||
throw nix::Error(
|
||||
"cannot download '%s' because Nix is not built with S3 support", request.uri.to_string());
|
||||
#endif
|
||||
} catch (...) {
|
||||
callback.rethrow();
|
||||
|
||||
@@ -154,22 +154,17 @@ protected:
|
||||
|
||||
FileTransferRequest makeRequest(const std::string & path)
|
||||
{
|
||||
/* FIXME path is not a path, but a full relative or absolute
|
||||
/* Otherwise the last path fragment will get discarded. */
|
||||
auto cacheUriWithTrailingSlash = config->cacheUri;
|
||||
if (!cacheUriWithTrailingSlash.path.empty())
|
||||
cacheUriWithTrailingSlash.path += "/";
|
||||
|
||||
/* path is not a path, but a full relative or absolute
|
||||
URL, e.g. we've seen in the wild NARINFO files have a URL
|
||||
field which is
|
||||
`nar/15f99rdaf26k39knmzry4xd0d97wp6yfpnfk1z9avakis7ipb9yg.nar?hash=zphkqn2wg8mnvbkixnl2aadkbn0rcnfj`
|
||||
(note the query param) and that gets passed here.
|
||||
|
||||
What should actually happen is that we have two parsed URLs
|
||||
(if we support relative URLs), and then we combined them with
|
||||
a URL `operator/` which would be like
|
||||
`std::filesystem::path`'s equivalent operator, which properly
|
||||
combines the the URLs, whether the right is relative or
|
||||
absolute. */
|
||||
return FileTransferRequest(
|
||||
hasPrefix(path, "https://") || hasPrefix(path, "http://") || hasPrefix(path, "file://")
|
||||
? path
|
||||
: config->cacheUri.to_string() + "/" + path);
|
||||
(note the query param) and that gets passed here. */
|
||||
return FileTransferRequest(parseURLRelative(path, cacheUriWithTrailingSlash));
|
||||
}
|
||||
|
||||
void getFile(const std::string & path, Sink & sink) override
|
||||
|
||||
@@ -93,32 +93,6 @@ struct DerivationBuilderParams
|
||||
* `EnvEntry::nameOfPassAsFile` above.
|
||||
*/
|
||||
StringMap extraFiles;
|
||||
|
||||
DerivationBuilderParams(
|
||||
const StorePath & drvPath,
|
||||
const BuildMode & buildMode,
|
||||
BuildResult & buildResult,
|
||||
const Derivation & drv,
|
||||
const DerivationOptions & drvOptions,
|
||||
const StorePathSet & inputPaths,
|
||||
std::map<std::string, InitialOutput> & initialOutputs,
|
||||
PathsInChroot defaultPathsInChroot,
|
||||
std::map<std::string, EnvEntry, std::less<>> finalEnv,
|
||||
StringMap extraFiles)
|
||||
: drvPath{drvPath}
|
||||
, buildResult{buildResult}
|
||||
, drv{drv}
|
||||
, drvOptions{drvOptions}
|
||||
, inputPaths{inputPaths}
|
||||
, initialOutputs{initialOutputs}
|
||||
, buildMode{buildMode}
|
||||
, defaultPathsInChroot{std::move(defaultPathsInChroot)}
|
||||
, finalEnv{std::move(finalEnv)}
|
||||
, extraFiles{std::move(extraFiles)}
|
||||
{
|
||||
}
|
||||
|
||||
DerivationBuilderParams(DerivationBuilderParams &&) = default;
|
||||
};
|
||||
|
||||
/**
|
||||
|
||||
@@ -9,6 +9,7 @@
|
||||
#include "nix/util/ref.hh"
|
||||
#include "nix/util/configuration.hh"
|
||||
#include "nix/util/serialise.hh"
|
||||
#include "nix/util/url.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
@@ -70,7 +71,7 @@ extern const unsigned int RETRY_TIME_MS_DEFAULT;
|
||||
|
||||
struct FileTransferRequest
|
||||
{
|
||||
std::string uri;
|
||||
ParsedURL uri;
|
||||
Headers headers;
|
||||
std::string expectedETag;
|
||||
bool verifyTLS = true;
|
||||
@@ -84,7 +85,7 @@ struct FileTransferRequest
|
||||
std::string mimeType;
|
||||
std::function<void(std::string_view data)> dataCallback;
|
||||
|
||||
FileTransferRequest(std::string_view uri)
|
||||
FileTransferRequest(ParsedURL uri)
|
||||
: uri(uri)
|
||||
, parentAct(getCurActivity())
|
||||
{
|
||||
@@ -111,6 +112,9 @@ struct FileTransferResult
|
||||
|
||||
/**
|
||||
* All URLs visited in the redirect chain.
|
||||
*
|
||||
* @note Intentionally strings and not `ParsedURL`s so we faithfully
|
||||
* return what cURL gave us.
|
||||
*/
|
||||
std::vector<std::string> urls;
|
||||
|
||||
|
||||
@@ -74,7 +74,13 @@ struct ParsedS3URL
|
||||
endpoint);
|
||||
}
|
||||
|
||||
static ParsedS3URL parse(std::string_view uri);
|
||||
static ParsedS3URL parse(const ParsedURL & uri);
|
||||
|
||||
/**
|
||||
* Convert this ParsedS3URL to HTTPS ParsedURL for use with curl's AWS SigV4 authentication
|
||||
*/
|
||||
ParsedURL toHttpsUrl() const;
|
||||
|
||||
auto operator<=>(const ParsedS3URL & other) const = default;
|
||||
};
|
||||
|
||||
|
||||
+43
-5
@@ -1,6 +1,8 @@
|
||||
#include "nix/store/s3.hh"
|
||||
#include "nix/util/split.hh"
|
||||
#include "nix/util/url.hh"
|
||||
#include "nix/util/util.hh"
|
||||
#include "nix/util/canon-path.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
@@ -8,10 +10,8 @@ using namespace std::string_view_literals;
|
||||
|
||||
#if NIX_WITH_S3_SUPPORT
|
||||
|
||||
ParsedS3URL ParsedS3URL::parse(std::string_view uri)
|
||||
ParsedS3URL ParsedS3URL::parse(const ParsedURL & parsed)
|
||||
try {
|
||||
auto parsed = parseURL(uri);
|
||||
|
||||
if (parsed.scheme != "s3"sv)
|
||||
throw BadURL("URI scheme '%s' is not 's3'", parsed.scheme);
|
||||
|
||||
@@ -43,7 +43,7 @@ try {
|
||||
auto endpoint = getOptionalParam("endpoint");
|
||||
|
||||
return ParsedS3URL{
|
||||
.bucket = std::move(parsed.authority->host),
|
||||
.bucket = parsed.authority->host,
|
||||
.key = std::string{key},
|
||||
.profile = getOptionalParam("profile"),
|
||||
.region = getOptionalParam("region"),
|
||||
@@ -62,10 +62,48 @@ try {
|
||||
}(),
|
||||
};
|
||||
} catch (BadURL & e) {
|
||||
e.addTrace({}, "while parsing S3 URI: '%s'", uri);
|
||||
e.addTrace({}, "while parsing S3 URI: '%s'", parsed.to_string());
|
||||
throw;
|
||||
}
|
||||
|
||||
ParsedURL ParsedS3URL::toHttpsUrl() const
|
||||
{
|
||||
auto toView = [](const auto & x) { return std::string_view{x}; };
|
||||
|
||||
auto regionStr = region.transform(toView).value_or("us-east-1");
|
||||
auto schemeStr = scheme.transform(toView).value_or("https");
|
||||
|
||||
// Handle endpoint configuration using std::visit
|
||||
return std::visit(
|
||||
overloaded{
|
||||
[&](const std::monostate &) {
|
||||
// No custom endpoint, use standard AWS S3 endpoint
|
||||
return ParsedURL{
|
||||
.scheme = std::string{schemeStr},
|
||||
.authority = ParsedURL::Authority{.host = "s3." + regionStr + ".amazonaws.com"},
|
||||
.path = (CanonPath::root / bucket / CanonPath(key)).abs(),
|
||||
};
|
||||
},
|
||||
[&](const ParsedURL::Authority & auth) {
|
||||
// Endpoint is just an authority (hostname/port)
|
||||
return ParsedURL{
|
||||
.scheme = std::string{schemeStr},
|
||||
.authority = auth,
|
||||
.path = (CanonPath::root / bucket / CanonPath(key)).abs(),
|
||||
};
|
||||
},
|
||||
[&](const ParsedURL & endpointUrl) {
|
||||
// Endpoint is already a ParsedURL (e.g., http://server:9000)
|
||||
return ParsedURL{
|
||||
.scheme = endpointUrl.scheme,
|
||||
.authority = endpointUrl.authority,
|
||||
.path = (CanonPath(endpointUrl.path) / bucket / CanonPath(key)).abs(),
|
||||
};
|
||||
},
|
||||
},
|
||||
endpoint);
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
} // namespace nix
|
||||
@@ -4,6 +4,10 @@
|
||||
#include "nix/util/url.hh"
|
||||
#include "nix/util/signals.hh"
|
||||
|
||||
#ifdef __linux__
|
||||
# include <sys/vfs.h>
|
||||
#endif
|
||||
|
||||
#include <sqlite3.h>
|
||||
|
||||
#include <atomic>
|
||||
@@ -60,6 +64,28 @@ static void traceSQL(void * x, const char * sql)
|
||||
|
||||
SQLite::SQLite(const std::filesystem::path & path, SQLiteOpenMode mode)
|
||||
{
|
||||
// Work around a ZFS issue where SQLite's truncate() call on
|
||||
// db.sqlite-shm can randomly take up to a few seconds. See
|
||||
// https://github.com/openzfs/zfs/issues/14290#issuecomment-3074672917.
|
||||
// Remove this workaround when a fix is widely installed, perhaps 2027? Candidate:
|
||||
// https://github.com/search?q=repo%3Aopenzfs%2Fzfs+%22Linux%3A+zfs_putpage%3A+complete+async+page+writeback+immediately%22&type=commits
|
||||
#ifdef __linux__
|
||||
try {
|
||||
auto shmFile = path;
|
||||
shmFile += "-shm";
|
||||
AutoCloseFD fd = open(shmFile.string().c_str(), O_RDWR | O_CLOEXEC);
|
||||
if (fd) {
|
||||
struct statfs fs;
|
||||
if (fstatfs(fd.get(), &fs))
|
||||
throw SysError("statfs() on '%s'", shmFile);
|
||||
if (fs.f_type == /* ZFS_SUPER_MAGIC */ 801189825 && fdatasync(fd.get()) != 0)
|
||||
throw SysError("fsync() on '%s'", shmFile);
|
||||
}
|
||||
} catch (...) {
|
||||
throw;
|
||||
}
|
||||
#endif
|
||||
|
||||
// useSQLiteWAL also indicates what virtual file system we need. Using
|
||||
// `unix-dotfile` is needed on NFS file systems and on Windows' Subsystem
|
||||
// for Linux (WSL) where useSQLiteWAL should be false by default.
|
||||
|
||||
@@ -45,7 +45,7 @@ StoreReference StoreReference::parse(const std::string & uri, const StoreReferen
|
||||
{
|
||||
auto params = extraParams;
|
||||
try {
|
||||
auto parsedUri = parseURL(uri);
|
||||
auto parsedUri = parseURL(uri, /*lenient=*/true);
|
||||
params.insert(parsedUri.query.begin(), parsedUri.query.end());
|
||||
|
||||
auto baseURI = parsedUri.authority.value_or(ParsedURL::Authority{}).to_string() + parsedUri.path;
|
||||
@@ -107,7 +107,7 @@ std::pair<std::string, StoreReference::Params> splitUriAndParams(const std::stri
|
||||
StoreReference::Params params;
|
||||
auto q = uri.find('?');
|
||||
if (q != std::string::npos) {
|
||||
params = decodeQuery(uri.substr(q + 1));
|
||||
params = decodeQuery(uri.substr(q + 1), /*lenient=*/true);
|
||||
uri = uri_.substr(0, q);
|
||||
}
|
||||
return {uri, params};
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
#include <rapidcheck/gtest.h>
|
||||
|
||||
#include "nix/util/strings.hh"
|
||||
#include "nix/util/strings-inline.hh"
|
||||
#include "nix/util/error.hh"
|
||||
|
||||
namespace nix {
|
||||
@@ -271,113 +272,122 @@ TEST(tokenizeString, tokenizeSepEmpty)
|
||||
* splitString
|
||||
* --------------------------------------------------------------------------*/
|
||||
|
||||
TEST(splitString, empty)
|
||||
{
|
||||
Strings expected = {""};
|
||||
using SplitStringTestContainerTypes = ::testing::
|
||||
Types<std::vector<std::string>, std::vector<std::string_view>, std::list<std::string>, std::list<std::string_view>>;
|
||||
|
||||
ASSERT_EQ(splitString<Strings>("", " \t\n\r"), expected);
|
||||
template<typename T>
|
||||
class splitStringTest : public ::testing::Test
|
||||
{};
|
||||
|
||||
TYPED_TEST_SUITE(splitStringTest, SplitStringTestContainerTypes);
|
||||
|
||||
TYPED_TEST(splitStringTest, empty)
|
||||
{
|
||||
TypeParam expected = {""};
|
||||
|
||||
EXPECT_EQ(splitString<TypeParam>("", " \t\n\r"), expected);
|
||||
}
|
||||
|
||||
TEST(splitString, oneSep)
|
||||
TYPED_TEST(splitStringTest, oneSep)
|
||||
{
|
||||
Strings expected = {"", ""};
|
||||
TypeParam expected = {"", ""};
|
||||
|
||||
ASSERT_EQ(splitString<Strings>(" ", " \t\n\r"), expected);
|
||||
EXPECT_EQ(splitString<TypeParam>(" ", " \t\n\r"), expected);
|
||||
}
|
||||
|
||||
TEST(splitString, twoSep)
|
||||
TYPED_TEST(splitStringTest, twoSep)
|
||||
{
|
||||
Strings expected = {"", "", ""};
|
||||
TypeParam expected = {"", "", ""};
|
||||
|
||||
ASSERT_EQ(splitString<Strings>(" \n", " \t\n\r"), expected);
|
||||
EXPECT_EQ(splitString<TypeParam>(" \n", " \t\n\r"), expected);
|
||||
}
|
||||
|
||||
TEST(splitString, tokenizeSpacesWithSpaces)
|
||||
TYPED_TEST(splitStringTest, tokenizeSpacesWithSpaces)
|
||||
{
|
||||
auto s = "foo bar baz";
|
||||
Strings expected = {"foo", "bar", "baz"};
|
||||
TypeParam expected = {"foo", "bar", "baz"};
|
||||
|
||||
ASSERT_EQ(splitString<Strings>(s, " \t\n\r"), expected);
|
||||
EXPECT_EQ(splitString<TypeParam>(s, " \t\n\r"), expected);
|
||||
}
|
||||
|
||||
TEST(splitString, tokenizeTabsWithDefaults)
|
||||
TYPED_TEST(splitStringTest, tokenizeTabsWithDefaults)
|
||||
{
|
||||
auto s = "foo\tbar\tbaz";
|
||||
// Using it like this is weird, but shows the difference with tokenizeString, which also has this test
|
||||
Strings expected = {"foo", "bar", "baz"};
|
||||
TypeParam expected = {"foo", "bar", "baz"};
|
||||
|
||||
ASSERT_EQ(splitString<Strings>(s, " \t\n\r"), expected);
|
||||
EXPECT_EQ(splitString<TypeParam>(s, " \t\n\r"), expected);
|
||||
}
|
||||
|
||||
TEST(splitString, tokenizeTabsSpacesWithDefaults)
|
||||
TYPED_TEST(splitStringTest, tokenizeTabsSpacesWithDefaults)
|
||||
{
|
||||
auto s = "foo\t bar\t baz";
|
||||
// Using it like this is weird, but shows the difference with tokenizeString, which also has this test
|
||||
Strings expected = {"foo", "", "bar", "", "baz"};
|
||||
TypeParam expected = {"foo", "", "bar", "", "baz"};
|
||||
|
||||
ASSERT_EQ(splitString<Strings>(s, " \t\n\r"), expected);
|
||||
EXPECT_EQ(splitString<TypeParam>(s, " \t\n\r"), expected);
|
||||
}
|
||||
|
||||
TEST(splitString, tokenizeTabsSpacesNewlineWithDefaults)
|
||||
TYPED_TEST(splitStringTest, tokenizeTabsSpacesNewlineWithDefaults)
|
||||
{
|
||||
auto s = "foo\t\n bar\t\n baz";
|
||||
// Using it like this is weird, but shows the difference with tokenizeString, which also has this test
|
||||
Strings expected = {"foo", "", "", "bar", "", "", "baz"};
|
||||
TypeParam expected = {"foo", "", "", "bar", "", "", "baz"};
|
||||
|
||||
ASSERT_EQ(splitString<Strings>(s, " \t\n\r"), expected);
|
||||
EXPECT_EQ(splitString<TypeParam>(s, " \t\n\r"), expected);
|
||||
}
|
||||
|
||||
TEST(splitString, tokenizeTabsSpacesNewlineRetWithDefaults)
|
||||
TYPED_TEST(splitStringTest, tokenizeTabsSpacesNewlineRetWithDefaults)
|
||||
{
|
||||
auto s = "foo\t\n\r bar\t\n\r baz";
|
||||
// Using it like this is weird, but shows the difference with tokenizeString, which also has this test
|
||||
Strings expected = {"foo", "", "", "", "bar", "", "", "", "baz"};
|
||||
TypeParam expected = {"foo", "", "", "", "bar", "", "", "", "baz"};
|
||||
|
||||
ASSERT_EQ(splitString<Strings>(s, " \t\n\r"), expected);
|
||||
EXPECT_EQ(splitString<TypeParam>(s, " \t\n\r"), expected);
|
||||
|
||||
auto s2 = "foo \t\n\r bar \t\n\r baz";
|
||||
Strings expected2 = {"foo", "", "", "", "", "bar", "", "", "", "", "baz"};
|
||||
TypeParam expected2 = {"foo", "", "", "", "", "bar", "", "", "", "", "baz"};
|
||||
|
||||
ASSERT_EQ(splitString<Strings>(s2, " \t\n\r"), expected2);
|
||||
EXPECT_EQ(splitString<TypeParam>(s2, " \t\n\r"), expected2);
|
||||
}
|
||||
|
||||
TEST(splitString, tokenizeWithCustomSep)
|
||||
TYPED_TEST(splitStringTest, tokenizeWithCustomSep)
|
||||
{
|
||||
auto s = "foo\n,bar\n,baz\n";
|
||||
Strings expected = {"foo\n", "bar\n", "baz\n"};
|
||||
TypeParam expected = {"foo\n", "bar\n", "baz\n"};
|
||||
|
||||
ASSERT_EQ(splitString<Strings>(s, ","), expected);
|
||||
EXPECT_EQ(splitString<TypeParam>(s, ","), expected);
|
||||
}
|
||||
|
||||
TEST(splitString, tokenizeSepAtStart)
|
||||
TYPED_TEST(splitStringTest, tokenizeSepAtStart)
|
||||
{
|
||||
auto s = ",foo,bar,baz";
|
||||
Strings expected = {"", "foo", "bar", "baz"};
|
||||
TypeParam expected = {"", "foo", "bar", "baz"};
|
||||
|
||||
ASSERT_EQ(splitString<Strings>(s, ","), expected);
|
||||
EXPECT_EQ(splitString<TypeParam>(s, ","), expected);
|
||||
}
|
||||
|
||||
TEST(splitString, tokenizeSepAtEnd)
|
||||
TYPED_TEST(splitStringTest, tokenizeSepAtEnd)
|
||||
{
|
||||
auto s = "foo,bar,baz,";
|
||||
Strings expected = {"foo", "bar", "baz", ""};
|
||||
TypeParam expected = {"foo", "bar", "baz", ""};
|
||||
|
||||
ASSERT_EQ(splitString<Strings>(s, ","), expected);
|
||||
EXPECT_EQ(splitString<TypeParam>(s, ","), expected);
|
||||
}
|
||||
|
||||
TEST(splitString, tokenizeSepEmpty)
|
||||
TYPED_TEST(splitStringTest, tokenizeSepEmpty)
|
||||
{
|
||||
auto s = "foo,,baz";
|
||||
Strings expected = {"foo", "", "baz"};
|
||||
TypeParam expected = {"foo", "", "baz"};
|
||||
|
||||
ASSERT_EQ(splitString<Strings>(s, ","), expected);
|
||||
EXPECT_EQ(splitString<TypeParam>(s, ","), expected);
|
||||
}
|
||||
|
||||
// concatStringsSep sep . splitString sep = id if sep is 1 char
|
||||
RC_GTEST_PROP(splitString, recoveredByConcatStringsSep, (const std::string & s))
|
||||
RC_GTEST_TYPED_FIXTURE_PROP(splitStringTest, recoveredByConcatStringsSep, (const std::string & s))
|
||||
{
|
||||
RC_ASSERT(concatStringsSep("/", splitString<Strings>(s, "/")) == s);
|
||||
RC_ASSERT(concatStringsSep("a", splitString<Strings>(s, "a")) == s);
|
||||
RC_ASSERT(concatStringsSep("/", splitString<TypeParam>(s, "/")) == s);
|
||||
RC_ASSERT(concatStringsSep("a", splitString<TypeParam>(s, "a")) == s);
|
||||
}
|
||||
|
||||
/* ----------------------------------------------------------------------------
|
||||
|
||||
+213
-3
@@ -221,15 +221,20 @@ TEST(parseURL, parsedUrlsWithUnescapedChars)
|
||||
* 2. Unescaped spaces and quotes in query.
|
||||
*/
|
||||
auto s = "http://www.example.org/file.tar.gz?query \"= 123\"#shevron^quote\"space ";
|
||||
auto url = parseURL(s);
|
||||
|
||||
ASSERT_EQ(url.fragment, "shevron^quote\"space ");
|
||||
/* Without leniency for back compat, this should throw. */
|
||||
EXPECT_THROW(parseURL(s), Error);
|
||||
|
||||
/* With leniency for back compat, this should parse. */
|
||||
auto url = parseURL(s, /*lenient=*/true);
|
||||
|
||||
EXPECT_EQ(url.fragment, "shevron^quote\"space ");
|
||||
|
||||
auto query = StringMap{
|
||||
{"query \"", " 123\""},
|
||||
};
|
||||
|
||||
ASSERT_EQ(url.query, query);
|
||||
EXPECT_EQ(url.query, query);
|
||||
}
|
||||
|
||||
TEST(parseURL, parseFTPUrl)
|
||||
@@ -268,6 +273,211 @@ TEST(parseURL, emptyStringIsInvalidURL)
|
||||
ASSERT_THROW(parseURL(""), Error);
|
||||
}
|
||||
|
||||
TEST(parseURL, parsesHttpUrlWithEmptyPort)
|
||||
{
|
||||
auto s = "http://www.example.org:/file.tar.gz?foo=bar";
|
||||
auto parsed = parseURL(s);
|
||||
|
||||
ParsedURL expected{
|
||||
.scheme = "http",
|
||||
.authority = Authority{.hostType = HostType::Name, .host = "www.example.org"},
|
||||
.path = "/file.tar.gz",
|
||||
.query = (StringMap) {{"foo", "bar"}},
|
||||
.fragment = "",
|
||||
};
|
||||
|
||||
ASSERT_EQ(parsed, expected);
|
||||
ASSERT_EQ("http://www.example.org/file.tar.gz?foo=bar", parsed.to_string());
|
||||
}
|
||||
|
||||
/* ----------------------------------------------------------------------------
|
||||
* parseURLRelative
|
||||
* --------------------------------------------------------------------------*/
|
||||
|
||||
TEST(parseURLRelative, resolvesRelativePath)
|
||||
{
|
||||
ParsedURL base = parseURL("http://example.org/dir/page.html");
|
||||
auto parsed = parseURLRelative("subdir/file.txt", base);
|
||||
ParsedURL expected{
|
||||
.scheme = "http",
|
||||
.authority = ParsedURL::Authority{.hostType = HostType::Name, .host = "example.org"},
|
||||
.path = "/dir/subdir/file.txt",
|
||||
.query = {},
|
||||
.fragment = "",
|
||||
};
|
||||
ASSERT_EQ(parsed, expected);
|
||||
}
|
||||
|
||||
TEST(parseURLRelative, baseUrlIpv6AddressWithoutZoneId)
|
||||
{
|
||||
ParsedURL base = parseURL("http://[fe80::818c:da4d:8975:415c]/dir/page.html");
|
||||
auto parsed = parseURLRelative("subdir/file.txt", base);
|
||||
ParsedURL expected{
|
||||
.scheme = "http",
|
||||
.authority = ParsedURL::Authority{.hostType = HostType::IPv6, .host = "fe80::818c:da4d:8975:415c"},
|
||||
.path = "/dir/subdir/file.txt",
|
||||
.query = {},
|
||||
.fragment = "",
|
||||
};
|
||||
ASSERT_EQ(parsed, expected);
|
||||
}
|
||||
|
||||
TEST(parseURLRelative, resolvesRelativePathIpv6AddressWithZoneId)
|
||||
{
|
||||
ParsedURL base = parseURL("http://[fe80::818c:da4d:8975:415c\%25enp0s25]:8080/dir/page.html");
|
||||
auto parsed = parseURLRelative("subdir/file2.txt", base);
|
||||
ParsedURL expected{
|
||||
.scheme = "http",
|
||||
.authority = Authority{.hostType = HostType::IPv6, .host = "fe80::818c:da4d:8975:415c\%enp0s25", .port = 8080},
|
||||
.path = "/dir/subdir/file2.txt",
|
||||
.query = {},
|
||||
.fragment = "",
|
||||
};
|
||||
|
||||
ASSERT_EQ(parsed, expected);
|
||||
}
|
||||
|
||||
TEST(parseURLRelative, resolvesRelativePathWithDot)
|
||||
{
|
||||
ParsedURL base = parseURL("http://example.org/dir/page.html");
|
||||
auto parsed = parseURLRelative("./subdir/file.txt", base);
|
||||
ParsedURL expected{
|
||||
.scheme = "http",
|
||||
.authority = ParsedURL::Authority{.hostType = HostType::Name, .host = "example.org"},
|
||||
.path = "/dir/subdir/file.txt",
|
||||
.query = {},
|
||||
.fragment = "",
|
||||
};
|
||||
ASSERT_EQ(parsed, expected);
|
||||
}
|
||||
|
||||
TEST(parseURLRelative, resolvesParentDirectory)
|
||||
{
|
||||
ParsedURL base = parseURL("http://example.org:234/dir/page.html");
|
||||
auto parsed = parseURLRelative("../up.txt", base);
|
||||
ParsedURL expected{
|
||||
.scheme = "http",
|
||||
.authority = ParsedURL::Authority{.hostType = HostType::Name, .host = "example.org", .port = 234},
|
||||
.path = "/up.txt",
|
||||
.query = {},
|
||||
.fragment = "",
|
||||
};
|
||||
ASSERT_EQ(parsed, expected);
|
||||
}
|
||||
|
||||
TEST(parseURLRelative, replacesPathWithAbsoluteRelative)
|
||||
{
|
||||
ParsedURL base = parseURL("http://example.org/dir/page.html");
|
||||
auto parsed = parseURLRelative("/rooted.txt", base);
|
||||
ParsedURL expected{
|
||||
.scheme = "http",
|
||||
.authority = ParsedURL::Authority{.hostType = HostType::Name, .host = "example.org"},
|
||||
.path = "/rooted.txt",
|
||||
.query = {},
|
||||
.fragment = "",
|
||||
};
|
||||
ASSERT_EQ(parsed, expected);
|
||||
}
|
||||
|
||||
TEST(parseURLRelative, keepsQueryAndFragmentFromRelative)
|
||||
{
|
||||
// But discard query params on base URL
|
||||
ParsedURL base = parseURL("https://www.example.org/path/index.html?z=3");
|
||||
auto parsed = parseURLRelative("other.html?x=1&y=2#frag", base);
|
||||
ParsedURL expected{
|
||||
.scheme = "https",
|
||||
.authority = ParsedURL::Authority{.hostType = HostType::Name, .host = "www.example.org"},
|
||||
.path = "/path/other.html",
|
||||
.query = {{"x", "1"}, {"y", "2"}},
|
||||
.fragment = "frag",
|
||||
};
|
||||
ASSERT_EQ(parsed, expected);
|
||||
}
|
||||
|
||||
TEST(parseURLRelative, absOverride)
|
||||
{
|
||||
ParsedURL base = parseURL("http://example.org/path/page.html");
|
||||
std::string_view abs = "https://127.0.0.1.org/secure";
|
||||
auto parsed = parseURLRelative(abs, base);
|
||||
auto parsedAbs = parseURL(abs);
|
||||
ASSERT_EQ(parsed, parsedAbs);
|
||||
}
|
||||
|
||||
TEST(parseURLRelative, absOverrideWithZoneId)
|
||||
{
|
||||
ParsedURL base = parseURL("http://example.org/path/page.html");
|
||||
std::string_view abs = "https://[fe80::818c:da4d:8975:415c\%25enp0s25]/secure?foo=bar";
|
||||
auto parsed = parseURLRelative(abs, base);
|
||||
auto parsedAbs = parseURL(abs);
|
||||
ASSERT_EQ(parsed, parsedAbs);
|
||||
}
|
||||
|
||||
TEST(parseURLRelative, bothWithoutAuthority)
|
||||
{
|
||||
ParsedURL base = parseURL("mailto:mail-base@bar.baz?bcc=alice@asdf.com");
|
||||
std::string_view over = "mailto:mail-override@foo.bar?subject=url-testing";
|
||||
auto parsed = parseURLRelative(over, base);
|
||||
auto parsedOverride = parseURL(over);
|
||||
ASSERT_EQ(parsed, parsedOverride);
|
||||
}
|
||||
|
||||
TEST(parseURLRelative, emptyRelative)
|
||||
{
|
||||
ParsedURL base = parseURL("https://www.example.org/path/index.html?a\%20b=5\%206&x\%20y=34#frag");
|
||||
auto parsed = parseURLRelative("", base);
|
||||
ParsedURL expected{
|
||||
.scheme = "https",
|
||||
.authority = ParsedURL::Authority{.hostType = HostType::Name, .host = "www.example.org"},
|
||||
.path = "/path/index.html",
|
||||
.query = {{"a b", "5 6"}, {"x y", "34"}},
|
||||
.fragment = "",
|
||||
};
|
||||
EXPECT_EQ(base.fragment, "frag");
|
||||
EXPECT_EQ(parsed, expected);
|
||||
}
|
||||
|
||||
TEST(parseURLRelative, fragmentRelative)
|
||||
{
|
||||
ParsedURL base = parseURL("https://www.example.org/path/index.html?a\%20b=5\%206&x\%20y=34#frag");
|
||||
auto parsed = parseURLRelative("#frag2", base);
|
||||
ParsedURL expected{
|
||||
.scheme = "https",
|
||||
.authority = ParsedURL::Authority{.hostType = HostType::Name, .host = "www.example.org"},
|
||||
.path = "/path/index.html",
|
||||
.query = {{"a b", "5 6"}, {"x y", "34"}},
|
||||
.fragment = "frag2",
|
||||
};
|
||||
EXPECT_EQ(parsed, expected);
|
||||
}
|
||||
|
||||
TEST(parseURLRelative, queryRelative)
|
||||
{
|
||||
ParsedURL base = parseURL("https://www.example.org/path/index.html?a\%20b=5\%206&x\%20y=34#frag");
|
||||
auto parsed = parseURLRelative("?asdf\%20qwer=1\%202\%203", base);
|
||||
ParsedURL expected{
|
||||
.scheme = "https",
|
||||
.authority = ParsedURL::Authority{.hostType = HostType::Name, .host = "www.example.org"},
|
||||
.path = "/path/index.html",
|
||||
.query = {{"asdf qwer", "1 2 3"}},
|
||||
.fragment = "",
|
||||
};
|
||||
EXPECT_EQ(parsed, expected);
|
||||
}
|
||||
|
||||
TEST(parseURLRelative, queryFragmentRelative)
|
||||
{
|
||||
ParsedURL base = parseURL("https://www.example.org/path/index.html?a\%20b=5\%206&x\%20y=34#frag");
|
||||
auto parsed = parseURLRelative("?asdf\%20qwer=1\%202\%203#frag2", base);
|
||||
ParsedURL expected{
|
||||
.scheme = "https",
|
||||
.authority = ParsedURL::Authority{.hostType = HostType::Name, .host = "www.example.org"},
|
||||
.path = "/path/index.html",
|
||||
.query = {{"asdf qwer", "1 2 3"}},
|
||||
.fragment = "frag2",
|
||||
};
|
||||
EXPECT_EQ(parsed, expected);
|
||||
}
|
||||
|
||||
/* ----------------------------------------------------------------------------
|
||||
* decodeQuery
|
||||
* --------------------------------------------------------------------------*/
|
||||
|
||||
@@ -26,18 +26,29 @@ C tokenizeString(std::string_view s, std::string_view separators)
|
||||
}
|
||||
|
||||
template<class C, class CharT>
|
||||
C basicSplitString(std::basic_string_view<CharT> s, std::basic_string_view<CharT> separators)
|
||||
void basicSplitStringInto(C & accum, std::basic_string_view<CharT> s, std::basic_string_view<CharT> separators)
|
||||
{
|
||||
C result;
|
||||
size_t pos = 0;
|
||||
while (pos <= s.size()) {
|
||||
auto end = s.find_first_of(separators, pos);
|
||||
if (end == s.npos)
|
||||
end = s.size();
|
||||
result.insert(result.end(), std::basic_string<CharT>(s, pos, end - pos));
|
||||
accum.insert(accum.end(), typename C::value_type{s.substr(pos, end - pos)});
|
||||
pos = end + 1;
|
||||
}
|
||||
}
|
||||
|
||||
template<typename C>
|
||||
void splitStringInto(C & accum, std::string_view s, std::string_view separators)
|
||||
{
|
||||
basicSplitStringInto<C, char>(accum, s, separators);
|
||||
}
|
||||
|
||||
template<class C, class CharT>
|
||||
C basicSplitString(std::basic_string_view<CharT> s, std::basic_string_view<CharT> separators)
|
||||
{
|
||||
C result;
|
||||
basicSplitStringInto(result, s, separators);
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
@@ -96,14 +96,18 @@ MakeError(BadURL, Error);
|
||||
std::string percentDecode(std::string_view in);
|
||||
std::string percentEncode(std::string_view s, std::string_view keep = "");
|
||||
|
||||
StringMap decodeQuery(const std::string & query);
|
||||
/**
|
||||
* @param lenient @see parseURL
|
||||
*/
|
||||
StringMap decodeQuery(std::string_view query, bool lenient = false);
|
||||
|
||||
std::string encodeQuery(const StringMap & query);
|
||||
|
||||
/**
|
||||
* Parse a Nix URL into a ParsedURL.
|
||||
* Parse a URL into a ParsedURL.
|
||||
*
|
||||
* Nix URI is mostly compliant with RFC3986, but with some deviations:
|
||||
* @parm lenient Also allow some long-supported Nix URIs that are not quite compliant with RFC3986.
|
||||
* Here are the deviations:
|
||||
* - Fragments can contain unescaped (not URL encoded) '^', '"' or space literals.
|
||||
* - Queries may contain unescaped '"' or spaces.
|
||||
*
|
||||
@@ -111,7 +115,17 @@ std::string encodeQuery(const StringMap & query);
|
||||
*
|
||||
* @throws BadURL
|
||||
*/
|
||||
ParsedURL parseURL(std::string_view url);
|
||||
ParsedURL parseURL(std::string_view url, bool lenient = false);
|
||||
|
||||
/**
|
||||
* Like `parseURL`, but also accepts relative URLs, which are resolved
|
||||
* against the given base URL.
|
||||
*
|
||||
* This is specified in [IETF RFC 3986, section 5](https://datatracker.ietf.org/doc/html/rfc3986#section-5)
|
||||
*
|
||||
* Behavior should also match the `new URL(url, base)` JavaScript constructor.
|
||||
*/
|
||||
ParsedURL parseURLRelative(std::string_view url, const ParsedURL & base);
|
||||
|
||||
/**
|
||||
* Although that’s not really standardized anywhere, an number of tools
|
||||
@@ -132,7 +146,7 @@ ParsedUrlScheme parseUrlScheme(std::string_view scheme);
|
||||
/* Detects scp-style uris (e.g. git@github.com:NixOS/nix) and fixes
|
||||
them by removing the `:` and assuming a scheme of `ssh://`. Also
|
||||
changes absolute paths into file:// URLs. */
|
||||
std::string fixGitURL(const std::string & url);
|
||||
ParsedURL fixGitURL(const std::string & url);
|
||||
|
||||
/**
|
||||
* Whether a string is valid as RFC 3986 scheme name.
|
||||
|
||||
@@ -55,9 +55,16 @@ public:
|
||||
// https://github.com/apple-oss-distributions/xnu/commit/e13b1fa57645afc8a7b2e7d868fe9845c6b08c40#diff-a5aa0b0e7f4d866ca417f60702689fc797e9cdfe33b601b05ccf43086c35d395R1468
|
||||
// That means added in 2007 or earlier. Should be good enough
|
||||
// for us.
|
||||
//
|
||||
// Update: as of macOS 15.4, passing 0 or POLLHUP
|
||||
// doesn't seem to work at all for sockets any more
|
||||
// (though it does work for `notifyPipe`). As a
|
||||
// workaround, also pass POLLIN. That does cause us to
|
||||
// receive a bunch of POLLIN events we don't care
|
||||
// about, so we sleep for a bit when receiving POLLIN.
|
||||
short hangup_events =
|
||||
#ifdef __APPLE__
|
||||
POLLHUP
|
||||
POLLIN | POLLHUP
|
||||
#else
|
||||
0
|
||||
#endif
|
||||
@@ -98,6 +105,12 @@ public:
|
||||
if (fds[1].revents & POLLHUP) {
|
||||
break;
|
||||
}
|
||||
if (fds[0].revents & POLLIN) {
|
||||
/* macOS only: we have to pass POLLIN to receive
|
||||
POLLHUP, but we don't care about POLLIN. To
|
||||
avoid a lot of wakeups, sleep for a bit. */
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(100));
|
||||
}
|
||||
// On macOS, (jade thinks that) it is possible (although not
|
||||
// observed on macOS 14.5) that in some limited cases on buggy
|
||||
// kernel versions, all the non-POLLHUP events for the socket
|
||||
|
||||
+106
-39
@@ -33,7 +33,7 @@ ParsedURL::Authority ParsedURL::Authority::parse(std::string_view encodedAuthori
|
||||
}();
|
||||
|
||||
auto port = [&]() -> std::optional<uint16_t> {
|
||||
if (!parsed->has_port())
|
||||
if (!parsed->has_port() || parsed->port() == "")
|
||||
return std::nullopt;
|
||||
/* If the port number is non-zero and representable. */
|
||||
if (auto portNumber = parsed->port_number())
|
||||
@@ -108,44 +108,58 @@ static std::string percentEncodeCharSet(std::string_view s, auto charSet)
|
||||
return res;
|
||||
}
|
||||
|
||||
ParsedURL parseURL(std::string_view url)
|
||||
static ParsedURL fromBoostUrlView(boost::urls::url_view url, bool lenient);
|
||||
|
||||
ParsedURL parseURL(std::string_view url, bool lenient)
|
||||
try {
|
||||
/* Account for several non-standard properties of nix urls (for back-compat):
|
||||
* - Allow unescaped spaces ' ' and '"' characters in queries.
|
||||
* - Allow '"', ' ' and '^' characters in the fragment component.
|
||||
* We could write our own grammar for this, but fixing it up here seems
|
||||
* more concise, since the deviation is rather minor.
|
||||
*
|
||||
* If `!lenient` don't bother initializing, because we can just
|
||||
* parse `url` directly`.
|
||||
*/
|
||||
std::string fixedEncodedUrl = [&]() {
|
||||
std::string fixed;
|
||||
std::string_view view = url;
|
||||
std::string fixedEncodedUrl;
|
||||
|
||||
if (auto beforeQuery = splitPrefixTo(view, '?')) {
|
||||
fixed += *beforeQuery;
|
||||
fixed += '?';
|
||||
auto fragmentStart = view.find('#');
|
||||
auto queryView = view.substr(0, fragmentStart);
|
||||
auto fixedQuery = percentEncodeCharSet(queryView, extraAllowedCharsInQuery);
|
||||
fixed += fixedQuery;
|
||||
view.remove_prefix(std::min(fragmentStart, view.size()));
|
||||
}
|
||||
if (lenient) {
|
||||
fixedEncodedUrl = [&] {
|
||||
std::string fixed;
|
||||
std::string_view view = url;
|
||||
|
||||
if (auto beforeFragment = splitPrefixTo(view, '#')) {
|
||||
fixed += *beforeFragment;
|
||||
fixed += '#';
|
||||
auto fixedFragment = percentEncodeCharSet(view, extraAllowedCharsInFragment);
|
||||
fixed += fixedFragment;
|
||||
if (auto beforeQuery = splitPrefixTo(view, '?')) {
|
||||
fixed += *beforeQuery;
|
||||
fixed += '?';
|
||||
auto fragmentStart = view.find('#');
|
||||
auto queryView = view.substr(0, fragmentStart);
|
||||
auto fixedQuery = percentEncodeCharSet(queryView, extraAllowedCharsInQuery);
|
||||
fixed += fixedQuery;
|
||||
view.remove_prefix(std::min(fragmentStart, view.size()));
|
||||
}
|
||||
|
||||
if (auto beforeFragment = splitPrefixTo(view, '#')) {
|
||||
fixed += *beforeFragment;
|
||||
fixed += '#';
|
||||
auto fixedFragment = percentEncodeCharSet(view, extraAllowedCharsInFragment);
|
||||
fixed += fixedFragment;
|
||||
return fixed;
|
||||
}
|
||||
|
||||
fixed += view;
|
||||
return fixed;
|
||||
}
|
||||
}();
|
||||
}
|
||||
|
||||
fixed += view;
|
||||
return fixed;
|
||||
}();
|
||||
|
||||
auto urlView = boost::urls::url_view(fixedEncodedUrl);
|
||||
return fromBoostUrlView(boost::urls::url_view(lenient ? fixedEncodedUrl : url), lenient);
|
||||
} catch (boost::system::system_error & e) {
|
||||
throw BadURL("'%s' is not a valid URL: %s", url, e.code().message());
|
||||
}
|
||||
|
||||
static ParsedURL fromBoostUrlView(boost::urls::url_view urlView, bool lenient)
|
||||
{
|
||||
if (!urlView.has_scheme())
|
||||
throw BadURL("'%s' doesn't have a scheme", url);
|
||||
throw BadURL("'%s' doesn't have a scheme", urlView.buffer());
|
||||
|
||||
auto scheme = urlView.scheme();
|
||||
auto authority = [&]() -> std::optional<ParsedURL::Authority> {
|
||||
@@ -163,7 +177,7 @@ try {
|
||||
* scheme considers a missing authority or empty host invalid. */
|
||||
auto transportIsFile = parseUrlScheme(scheme).transport == "file";
|
||||
if (authority && authority->host.size() && transportIsFile)
|
||||
throw BadURL("file:// URL '%s' has unexpected authority '%s'", url, *authority);
|
||||
throw BadURL("file:// URL '%s' has unexpected authority '%s'", urlView.buffer(), *authority);
|
||||
|
||||
auto path = urlView.path(); /* Does pct-decoding */
|
||||
auto fragment = urlView.fragment(); /* Does pct-decoding */
|
||||
@@ -179,11 +193,61 @@ try {
|
||||
.scheme = scheme,
|
||||
.authority = authority,
|
||||
.path = path,
|
||||
.query = decodeQuery(std::string(query)),
|
||||
.query = decodeQuery(query, lenient),
|
||||
.fragment = fragment,
|
||||
};
|
||||
} catch (boost::system::system_error & e) {
|
||||
throw BadURL("'%s' is not a valid URL: %s", url, e.code().message());
|
||||
}
|
||||
|
||||
ParsedURL parseURLRelative(std::string_view urlS, const ParsedURL & base)
|
||||
try {
|
||||
|
||||
boost::urls::url resolved;
|
||||
|
||||
try {
|
||||
resolved.set_scheme(base.scheme);
|
||||
if (base.authority) {
|
||||
auto & authority = *base.authority;
|
||||
resolved.set_host_address(authority.host);
|
||||
if (authority.user)
|
||||
resolved.set_user(*authority.user);
|
||||
if (authority.password)
|
||||
resolved.set_password(*authority.password);
|
||||
if (authority.port)
|
||||
resolved.set_port_number(*authority.port);
|
||||
}
|
||||
resolved.set_path(base.path);
|
||||
resolved.set_encoded_query(encodeQuery(base.query));
|
||||
resolved.set_fragment(base.fragment);
|
||||
} catch (boost::system::system_error & e) {
|
||||
throw BadURL("'%s' is not a valid URL: %s", base.to_string(), e.code().message());
|
||||
}
|
||||
|
||||
boost::urls::url_view url;
|
||||
try {
|
||||
url = urlS;
|
||||
resolved.resolve(url).value();
|
||||
} catch (boost::system::system_error & e) {
|
||||
throw BadURL("'%s' is not a valid URL: %s", urlS, e.code().message());
|
||||
}
|
||||
|
||||
auto ret = fromBoostUrlView(resolved, /*lenient=*/false);
|
||||
|
||||
/* Hack: Boost `url_view` supports Zone IDs, but `url` does not.
|
||||
Just manually take the authority from the original URL to work
|
||||
around it. See https://github.com/boostorg/url/issues/919 for
|
||||
details. */
|
||||
if (!url.has_authority()) {
|
||||
ret.authority = base.authority;
|
||||
}
|
||||
|
||||
/* Hack, work around fragment of base URL improperly being preserved
|
||||
https://github.com/boostorg/url/issues/920 */
|
||||
ret.fragment = url.has_fragment() ? std::string{url.fragment()} : "";
|
||||
|
||||
return ret;
|
||||
} catch (BadURL & e) {
|
||||
e.addTrace({}, "while resolving possibly-relative url '%s' against base URL '%s'", urlS, base);
|
||||
throw;
|
||||
}
|
||||
|
||||
std::string percentDecode(std::string_view in)
|
||||
@@ -201,14 +265,17 @@ std::string percentEncode(std::string_view s, std::string_view keep)
|
||||
s, [keep](char c) { return boost::urls::unreserved_chars(c) || keep.find(c) != keep.npos; });
|
||||
}
|
||||
|
||||
StringMap decodeQuery(const std::string & query)
|
||||
StringMap decodeQuery(std::string_view query, bool lenient)
|
||||
try {
|
||||
/* For back-compat unescaped characters are allowed. */
|
||||
auto fixedEncodedQuery = percentEncodeCharSet(query, extraAllowedCharsInQuery);
|
||||
/* When `lenient = true`, for back-compat unescaped characters are allowed. */
|
||||
std::string fixedEncodedQuery;
|
||||
if (lenient) {
|
||||
fixedEncodedQuery = percentEncodeCharSet(query, extraAllowedCharsInQuery);
|
||||
}
|
||||
|
||||
StringMap result;
|
||||
|
||||
auto encodedQuery = boost::urls::params_encoded_view(fixedEncodedQuery);
|
||||
auto encodedQuery = boost::urls::params_encoded_view(lenient ? fixedEncodedQuery : query);
|
||||
for (auto && [key, value, value_specified] : encodedQuery) {
|
||||
if (!value_specified) {
|
||||
warn("dubious URI query '%s' is missing equal sign '%s', ignoring", std::string_view(key), "=");
|
||||
@@ -277,17 +344,17 @@ ParsedUrlScheme parseUrlScheme(std::string_view scheme)
|
||||
};
|
||||
}
|
||||
|
||||
std::string fixGitURL(const std::string & url)
|
||||
ParsedURL fixGitURL(const std::string & url)
|
||||
{
|
||||
std::regex scpRegex("([^/]*)@(.*):(.*)");
|
||||
if (!hasPrefix(url, "/") && std::regex_match(url, scpRegex))
|
||||
return std::regex_replace(url, scpRegex, "ssh://$1@$2/$3");
|
||||
return parseURL(std::regex_replace(url, scpRegex, "ssh://$1@$2/$3"));
|
||||
if (hasPrefix(url, "file:"))
|
||||
return url;
|
||||
return parseURL(url);
|
||||
if (url.find("://") == std::string::npos) {
|
||||
return (ParsedURL{.scheme = "file", .authority = ParsedURL::Authority{}, .path = url}).to_string();
|
||||
return (ParsedURL{.scheme = "file", .authority = ParsedURL::Authority{}, .path = url});
|
||||
}
|
||||
return url;
|
||||
return parseURL(url);
|
||||
}
|
||||
|
||||
// https://www.rfc-editor.org/rfc/rfc3986#section-3.1
|
||||
|
||||
+1
-1
@@ -105,7 +105,7 @@ std::tuple<StorePath, Hash> prefetchFile(
|
||||
|
||||
FdSink sink(fd.get());
|
||||
|
||||
FileTransferRequest req(url);
|
||||
FileTransferRequest req(parseURL(url));
|
||||
req.decompress = false;
|
||||
getFileTransfer()->download(std::move(req), sink);
|
||||
}
|
||||
|
||||
+11
-6
@@ -105,7 +105,8 @@ std::string getNameFromElement(const ProfileElement & element)
|
||||
{
|
||||
std::optional<std::string> result = std::nullopt;
|
||||
if (element.source) {
|
||||
result = getNameFromURL(parseURL(element.source->to_string()));
|
||||
// Seems to be for Flake URLs
|
||||
result = getNameFromURL(parseURL(element.source->to_string(), /*lenient=*/true));
|
||||
}
|
||||
return result.value_or(element.identifier());
|
||||
}
|
||||
@@ -160,11 +161,15 @@ struct ProfileManifest
|
||||
e["outputs"].get<ExtendedOutputsSpec>()};
|
||||
}
|
||||
|
||||
std::string name =
|
||||
elems.is_object() ? elem.key()
|
||||
: element.source
|
||||
? getNameFromURL(parseURL(element.source->to_string())).value_or(element.identifier())
|
||||
: element.identifier();
|
||||
std::string name = [&] {
|
||||
if (elems.is_object())
|
||||
return elem.key();
|
||||
if (element.source) {
|
||||
if (auto optName = getNameFromURL(parseURL(element.source->to_string(), /*lenient=*/true)))
|
||||
return *optName;
|
||||
}
|
||||
return element.identifier();
|
||||
}();
|
||||
|
||||
addElement(name, std::move(element));
|
||||
}
|
||||
|
||||
@@ -156,7 +156,7 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand
|
||||
Activity act(*logger, lvlInfo, actUnknown, "querying latest Nix version");
|
||||
|
||||
// FIXME: use nixos.org?
|
||||
auto req = FileTransferRequest((std::string &) settings.upgradeNixStorePathUrl);
|
||||
auto req = FileTransferRequest(parseURL(settings.upgradeNixStorePathUrl.get()));
|
||||
auto res = getFileTransfer()->download(req);
|
||||
|
||||
auto state = std::make_unique<EvalState>(LookupPath{}, store, fetchSettings, evalSettings);
|
||||
|
||||
Reference in new issue
Block a user