Compare commits

...
Author SHA1 Message Date
John Ericson 6a839b5f30 Introduce ParsedRelativeUrl
For NAR URLs in libstore, we will want a proper notion of a relative
URL. As such, we are making this type to complement `ParsedURL`.

`parseURLRelative` used to be a relatively black-box function using
Boost, but now that we have this separate type, we can properly
decompose the parsing and the resolving parts into separate functions,
leaving this function a thin wrapper.

The relative URL resolution I tried to continue doing with Boost, but it
prooved difficult --- we were already working around two bugs, and the
rountrip encodings were janks, so I just ended up reimplementing it from
scratch (with the LLM). Since the inputs are both fully decoded abstract
syntax, this ended up making nice readable code.

The existing `parseURLRelative` unit test data is very good, so we just
extended that test suite with new functions to test the round trips on
the constituent parts.
2026-02-26 14:47:15 -05:00
John Ericson 95e647df89 Add a test case for relative ? empty explicit params vs none at all
Unlike with absolute (complete) URLs, with relative URLs, this does make
a difference -- explicit empty overrides, implicit empty does not.
2026-02-26 14:46:15 -05:00
John Ericson 371f017afa Convert parseURLRelative tests to parameters test suite
This will allow extending it further in the next commit
2026-02-26 14:37:27 -05:00
3 changed files with 602 additions and 276 deletions

No files matched your search

+319 -196
View File
@@ -168,7 +168,11 @@ INSTANTIATE_TEST_SUITE_P(
.expected =
ParsedURL{
.scheme = "http",
.authority = Authority{.hostType = HostType::Name, .host = "www.example.org"},
.authority =
Authority{
.hostType = HostType::Name,
.host = "www.example.org",
},
.path = {"", "file.tar.gz"},
.query = (StringMap) {},
.fragment = "",
@@ -179,7 +183,11 @@ INSTANTIATE_TEST_SUITE_P(
.expected =
ParsedURL{
.scheme = "https",
.authority = Authority{.hostType = HostType::Name, .host = "www.example.org"},
.authority =
Authority{
.hostType = HostType::Name,
.host = "www.example.org",
},
.path = {"", "file.tar.gz"},
.query = (StringMap) {},
.fragment = "",
@@ -190,7 +198,11 @@ INSTANTIATE_TEST_SUITE_P(
.expected =
ParsedURL{
.scheme = "https",
.authority = Authority{.hostType = HostType::Name, .host = "www.example.org"},
.authority =
Authority{
.hostType = HostType::Name,
.host = "www.example.org",
},
.path = {"", "file.tar.gz"},
.query = (StringMap) {{"download", "fast"}, {"when", "now"}},
.fragment = "hello",
@@ -201,7 +213,11 @@ INSTANTIATE_TEST_SUITE_P(
.expected =
ParsedURL{
.scheme = "file+https",
.authority = Authority{.hostType = HostType::Name, .host = "www.example.org"},
.authority =
Authority{
.hostType = HostType::Name,
.host = "www.example.org",
},
.path = {"", "video.mp4"},
.query = (StringMap) {},
.fragment = "",
@@ -212,7 +228,12 @@ INSTANTIATE_TEST_SUITE_P(
.expected =
ParsedURL{
.scheme = "http",
.authority = Authority{.hostType = HostType::IPv4, .host = "127.0.0.1", .port = 8080},
.authority =
Authority{
.hostType = HostType::IPv4,
.host = "127.0.0.1",
.port = 8080,
},
.path = {"", "file.tar.gz"},
.query = (StringMap) {{"download", "fast"}, {"when", "now"}},
.fragment = "hello",
@@ -225,7 +246,10 @@ INSTANTIATE_TEST_SUITE_P(
.scheme = "http",
.authority =
Authority{
.hostType = HostType::IPv6, .host = "fe80::818c:da4d:8975:415c\%enp0s25", .port = 8080},
.hostType = HostType::IPv6,
.host = "fe80::818c:da4d:8975:415c\%enp0s25",
.port = 8080,
},
.path = {""},
.query = (StringMap) {},
.fragment = "",
@@ -277,7 +301,11 @@ TEST(parseURL, parsesSimpleHttpUrlWithComplexFragment)
ParsedURL expected{
.scheme = "http",
.authority = Authority{.hostType = HostType::Name, .host = "www.example.org"},
.authority =
Authority{
.hostType = HostType::Name,
.host = "www.example.org",
},
.path = {"", "file.tar.gz"},
.query = (StringMap) {{"field", "value"}},
.fragment = "?foo=bar#",
@@ -484,7 +512,11 @@ TEST(parseURL, parsesHttpUrlWithEmptyPort)
ParsedURL expected{
.scheme = "http",
.authority = Authority{.hostType = HostType::Name, .host = "www.example.org"},
.authority =
Authority{
.hostType = HostType::Name,
.host = "www.example.org",
},
.path = {"", "file.tar.gz"},
.query = (StringMap) {{"foo", "bar"}},
.fragment = "",
@@ -498,203 +530,294 @@ TEST(parseURL, parsesHttpUrlWithEmptyPort)
* parseURLRelative
* --------------------------------------------------------------------------*/
TEST(parseURLRelative, resolvesRelativePath)
struct ParseURLRelativeParam
{
ParsedURL base = parseURL("http://example.org/dir/page.html");
auto parsed = parseURLRelative("subdir/file.txt", base);
ParsedURL expected{
.scheme = "http",
.authority = ParsedURL::Authority{.hostType = HostType::Name, .host = "example.org"},
.path = {"", "dir", "subdir", "file.txt"},
.query = {},
.fragment = "",
};
ASSERT_EQ(parsed, expected);
std::string_view base;
std::string_view relative;
ParsedURL expected;
std::string description;
};
class ParseURLRelativeTestSuite : public ::testing::TestWithParam<ParseURLRelativeParam>
{};
TEST_P(ParseURLRelativeTestSuite, baseRoundTrips)
{
auto & p = GetParam();
auto base = parseURL(p.base);
EXPECT_EQ(base.to_string(), p.base);
}
TEST(parseURLRelative, baseUrlIpv6AddressWithoutZoneId)
TEST_P(ParseURLRelativeTestSuite, relativeRoundTrips)
{
ParsedURL base = parseURL("http://[fe80::818c:da4d:8975:415c]/dir/page.html");
auto parsed = parseURLRelative("subdir/file.txt", base);
ParsedURL expected{
.scheme = "http",
.authority = ParsedURL::Authority{.hostType = HostType::IPv6, .host = "fe80::818c:da4d:8975:415c"},
.path = {"", "dir", "subdir", "file.txt"},
.query = {},
.fragment = "",
};
ASSERT_EQ(parsed, expected);
auto & p = GetParam();
auto parsed = parsePossiblyRelativeURL(p.relative);
auto str = std::visit([](auto & url) { return url.to_string(); }, parsed);
EXPECT_EQ(str, p.relative);
}
TEST(parseURLRelative, resolvesRelativePathIpv6AddressWithZoneId)
TEST_P(ParseURLRelativeTestSuite, resolve)
{
ParsedURL base = parseURL("http://[fe80::818c:da4d:8975:415c\%25enp0s25]:8080/dir/page.html");
auto parsed = parseURLRelative("subdir/file2.txt", base);
ParsedURL expected{
.scheme = "http",
.authority = Authority{.hostType = HostType::IPv6, .host = "fe80::818c:da4d:8975:415c\%enp0s25", .port = 8080},
.path = {"", "dir", "subdir", "file2.txt"},
.query = {},
.fragment = "",
};
ASSERT_EQ(parsed, expected);
auto & p = GetParam();
auto base = parseURL(p.base);
auto parsed = parseURLRelative(p.relative, base);
EXPECT_EQ(parsed, p.expected);
}
TEST(parseURLRelative, resolvesRelativePathWithDot)
{
ParsedURL base = parseURL("http://example.org/dir/page.html");
auto parsed = parseURLRelative("./subdir/file.txt", base);
ParsedURL expected{
.scheme = "http",
.authority = ParsedURL::Authority{.hostType = HostType::Name, .host = "example.org"},
.path = {"", "dir", "subdir", "file.txt"},
.query = {},
.fragment = "",
};
ASSERT_EQ(parsed, expected);
}
TEST(parseURLRelative, resolvesParentDirectory)
{
ParsedURL base = parseURL("http://example.org:234/dir/page.html");
auto parsed = parseURLRelative("../up.txt", base);
ParsedURL expected{
.scheme = "http",
.authority = ParsedURL::Authority{.hostType = HostType::Name, .host = "example.org", .port = 234},
.path = {"", "up.txt"},
.query = {},
.fragment = "",
};
ASSERT_EQ(parsed, expected);
}
TEST(parseURLRelative, resolvesParentDirectoryNotTrickedByEscapedSlash)
{
ParsedURL base = parseURL("http://example.org:234/dir\%2Ffirst-trick/another-dir\%2Fsecond-trick/page.html");
auto parsed = parseURLRelative("../up.txt", base);
ParsedURL expected{
.scheme = "http",
.authority = ParsedURL::Authority{.hostType = HostType::Name, .host = "example.org", .port = 234},
.path = {"", "dir/first-trick", "up.txt"},
.query = {},
.fragment = "",
};
ASSERT_EQ(parsed, expected);
}
TEST(parseURLRelative, replacesPathWithAbsoluteRelative)
{
ParsedURL base = parseURL("http://example.org/dir/page.html");
auto parsed = parseURLRelative("/rooted.txt", base);
ParsedURL expected{
.scheme = "http",
.authority = ParsedURL::Authority{.hostType = HostType::Name, .host = "example.org"},
.path = {"", "rooted.txt"},
.query = {},
.fragment = "",
};
ASSERT_EQ(parsed, expected);
}
TEST(parseURLRelative, keepsQueryAndFragmentFromRelative)
{
// But discard query params on base URL
ParsedURL base = parseURL("https://www.example.org/path/index.html?z=3");
auto parsed = parseURLRelative("other.html?x=1&y=2#frag", base);
ParsedURL expected{
.scheme = "https",
.authority = ParsedURL::Authority{.hostType = HostType::Name, .host = "www.example.org"},
.path = {"", "path", "other.html"},
.query = {{"x", "1"}, {"y", "2"}},
.fragment = "frag",
};
ASSERT_EQ(parsed, expected);
}
TEST(parseURLRelative, absOverride)
{
ParsedURL base = parseURL("http://example.org/path/page.html");
std::string_view abs = "https://127.0.0.1.org/secure";
auto parsed = parseURLRelative(abs, base);
auto parsedAbs = parseURL(abs);
ASSERT_EQ(parsed, parsedAbs);
}
TEST(parseURLRelative, absOverrideWithZoneId)
{
ParsedURL base = parseURL("http://example.org/path/page.html");
std::string_view abs = "https://[fe80::818c:da4d:8975:415c\%25enp0s25]/secure?foo=bar";
auto parsed = parseURLRelative(abs, base);
auto parsedAbs = parseURL(abs);
ASSERT_EQ(parsed, parsedAbs);
}
TEST(parseURLRelative, bothWithoutAuthority)
{
ParsedURL base = parseURL("mailto:mail-base@bar.baz?bcc=alice@asdf.com");
std::string_view over = "mailto:mail-override@foo.bar?subject=url-testing";
auto parsed = parseURLRelative(over, base);
auto parsedOverride = parseURL(over);
ASSERT_EQ(parsed, parsedOverride);
}
TEST(parseURLRelative, emptyRelative)
{
ParsedURL base = parseURL("https://www.example.org/path/index.html?a\%20b=5\%206&x\%20y=34#frag");
auto parsed = parseURLRelative("", base);
ParsedURL expected{
.scheme = "https",
.authority = ParsedURL::Authority{.hostType = HostType::Name, .host = "www.example.org"},
.path = {"", "path", "index.html"},
.query = {{"a b", "5 6"}, {"x y", "34"}},
.fragment = "",
};
EXPECT_EQ(base.fragment, "frag");
EXPECT_EQ(parsed, expected);
}
TEST(parseURLRelative, fragmentRelative)
{
ParsedURL base = parseURL("https://www.example.org/path/index.html?a\%20b=5\%206&x\%20y=34#frag");
auto parsed = parseURLRelative("#frag2", base);
ParsedURL expected{
.scheme = "https",
.authority = ParsedURL::Authority{.hostType = HostType::Name, .host = "www.example.org"},
.path = {"", "path", "index.html"},
.query = {{"a b", "5 6"}, {"x y", "34"}},
.fragment = "frag2",
};
EXPECT_EQ(parsed, expected);
}
TEST(parseURLRelative, queryRelative)
{
ParsedURL base = parseURL("https://www.example.org/path/index.html?a\%20b=5\%206&x\%20y=34#frag");
auto parsed = parseURLRelative("?asdf\%20qwer=1\%202\%203", base);
ParsedURL expected{
.scheme = "https",
.authority = ParsedURL::Authority{.hostType = HostType::Name, .host = "www.example.org"},
.path = {"", "path", "index.html"},
.query = {{"asdf qwer", "1 2 3"}},
.fragment = "",
};
EXPECT_EQ(parsed, expected);
}
TEST(parseURLRelative, queryFragmentRelative)
{
ParsedURL base = parseURL("https://www.example.org/path/index.html?a\%20b=5\%206&x\%20y=34#frag");
auto parsed = parseURLRelative("?asdf\%20qwer=1\%202\%203#frag2", base);
ParsedURL expected{
.scheme = "https",
.authority = ParsedURL::Authority{.hostType = HostType::Name, .host = "www.example.org"},
.path = {"", "path", "index.html"},
.query = {{"asdf qwer", "1 2 3"}},
.fragment = "frag2",
};
EXPECT_EQ(parsed, expected);
}
INSTANTIATE_TEST_SUITE_P(
parseURLRelative,
ParseURLRelativeTestSuite,
::testing::Values(
ParseURLRelativeParam{
.base = "http://example.org/dir/page.html",
.relative = "subdir/file.txt",
.expected =
ParsedURL{
.scheme = "http",
.authority =
Authority{
.hostType = HostType::Name,
.host = "example.org",
},
.path = {"", "dir", "subdir", "file.txt"},
},
.description = "resolvesRelativePath",
},
ParseURLRelativeParam{
.base = "http://[fe80::818c:da4d:8975:415c]/dir/page.html",
.relative = "subdir/file.txt",
.expected =
ParsedURL{
.scheme = "http",
.authority =
Authority{
.hostType = HostType::IPv6,
.host = "fe80::818c:da4d:8975:415c",
},
.path = {"", "dir", "subdir", "file.txt"},
},
.description = "baseUrlIpv6AddressWithoutZoneId",
},
ParseURLRelativeParam{
.base = "http://[fe80::818c:da4d:8975:415c\%25enp0s25]:8080/dir/page.html",
.relative = "subdir/file2.txt",
.expected =
ParsedURL{
.scheme = "http",
.authority =
Authority{
.hostType = HostType::IPv6,
.host = "fe80::818c:da4d:8975:415c%enp0s25",
.port = 8080,
},
.path = {"", "dir", "subdir", "file2.txt"},
},
.description = "resolvesRelativePathIpv6AddressWithZoneId",
},
ParseURLRelativeParam{
.base = "http://example.org/dir/page.html",
.relative = "./subdir/file.txt",
.expected =
ParsedURL{
.scheme = "http",
.authority =
Authority{
.hostType = HostType::Name,
.host = "example.org",
},
.path = {"", "dir", "subdir", "file.txt"},
},
.description = "resolvesRelativePathWithDot",
},
ParseURLRelativeParam{
.base = "http://example.org:234/dir/page.html",
.relative = "../up.txt",
.expected =
ParsedURL{
.scheme = "http",
.authority =
Authority{
.hostType = HostType::Name,
.host = "example.org",
.port = 234,
},
.path = {"", "up.txt"},
},
.description = "resolvesParentDirectory",
},
ParseURLRelativeParam{
.base = "http://example.org:234/dir\%2Ffirst-trick/another-dir\%2Fsecond-trick/page.html",
.relative = "../up.txt",
.expected =
ParsedURL{
.scheme = "http",
.authority =
Authority{
.hostType = HostType::Name,
.host = "example.org",
.port = 234,
},
.path = {"", "dir/first-trick", "up.txt"},
},
.description = "resolvesParentDirectoryNotTrickedByEscapedSlash",
},
ParseURLRelativeParam{
.base = "http://example.org/dir/page.html",
.relative = "/rooted.txt",
.expected =
ParsedURL{
.scheme = "http",
.authority =
Authority{
.hostType = HostType::Name,
.host = "example.org",
},
.path = {"", "rooted.txt"},
},
.description = "replacesPathWithAbsoluteRelative",
},
ParseURLRelativeParam{
.base = "https://www.example.org/path/index.html?z=3",
.relative = "other.html?x=1&y=2#frag",
.expected =
ParsedURL{
.scheme = "https",
.authority =
Authority{
.hostType = HostType::Name,
.host = "www.example.org",
},
.path = {"", "path", "other.html"},
.query = {{"x", "1"}, {"y", "2"}},
.fragment = "frag",
},
.description = "keepsQueryAndFragmentFromRelative",
},
ParseURLRelativeParam{
.base = "http://example.org/path/page.html",
.relative = "https://127.0.0.1.org/secure",
.expected =
ParsedURL{
.scheme = "https",
.authority =
Authority{
.hostType = HostType::Name,
.host = "127.0.0.1.org",
},
.path = {"", "secure"},
},
.description = "absOverride",
},
ParseURLRelativeParam{
.base = "http://example.org/path/page.html",
.relative = "https://[fe80::818c:da4d:8975:415c\%25enp0s25]/secure?foo=bar",
.expected =
ParsedURL{
.scheme = "https",
.authority =
Authority{
.hostType = HostType::IPv6,
.host = "fe80::818c:da4d:8975:415c%enp0s25",
},
.path = {"", "secure"},
.query = {{"foo", "bar"}},
},
.description = "absOverrideWithZoneId",
},
ParseURLRelativeParam{
.base = "mailto:mail-base@bar.baz?bcc=alice@asdf.com",
.relative = "mailto:mail-override@foo.bar?subject=url-testing",
.expected =
ParsedURL{
.scheme = "mailto",
.path = {"mail-override@foo.bar"},
.query = {{"subject", "url-testing"}},
},
.description = "bothWithoutAuthority",
},
ParseURLRelativeParam{
.base = "https://www.example.org/path/index.html?a\%20b=5\%206&x\%20y=34#frag",
.relative = "",
.expected =
ParsedURL{
.scheme = "https",
.authority =
Authority{
.hostType = HostType::Name,
.host = "www.example.org",
},
.path = {"", "path", "index.html"},
.query = {{"a b", "5 6"}, {"x y", "34"}},
},
.description = "emptyRelative",
},
ParseURLRelativeParam{
.base = "https://www.example.org/path/index.html?a\%20b=5\%206&x\%20y=34#frag",
.relative = "#frag2",
.expected =
ParsedURL{
.scheme = "https",
.authority =
Authority{
.hostType = HostType::Name,
.host = "www.example.org",
},
.path = {"", "path", "index.html"},
.query = {{"a b", "5 6"}, {"x y", "34"}},
.fragment = "frag2",
},
.description = "fragmentRelative",
},
ParseURLRelativeParam{
.base = "https://www.example.org/path/index.html?a\%20b=5\%206&x\%20y=34#frag",
.relative = "?",
.expected =
ParsedURL{
.scheme = "https",
.authority =
Authority{
.hostType = HostType::Name,
.host = "www.example.org",
},
.path = {"", "path", "index.html"},
.query = {},
},
.description = "emptyQueryRelative",
},
ParseURLRelativeParam{
.base = "https://www.example.org/path/index.html?a\%20b=5\%206&x\%20y=34#frag",
.relative = "?asdf\%20qwer=1\%202\%203",
.expected =
ParsedURL{
.scheme = "https",
.authority =
Authority{
.hostType = HostType::Name,
.host = "www.example.org",
},
.path = {"", "path", "index.html"},
.query = {{"asdf qwer", "1 2 3"}},
},
.description = "queryRelative",
},
ParseURLRelativeParam{
.base = "https://www.example.org/path/index.html?a\%20b=5\%206&x\%20y=34#frag",
.relative = "?asdf\%20qwer=1\%202\%203#frag2",
.expected =
ParsedURL{
.scheme = "https",
.authority =
Authority{
.hostType = HostType::Name,
.host = "www.example.org",
},
.path = {"", "path", "index.html"},
.query = {{"asdf qwer", "1 2 3"}},
.fragment = "frag2",
},
.description = "queryFragmentRelative",
}),
[](const auto & info) { return info.param.description; });
/* ----------------------------------------------------------------------------
* decodeQuery
+81 -2
View File
@@ -253,6 +253,54 @@ struct ParsedURL
std::ostream & operator<<(std::ostream & os, const ParsedURL & url);
/**
* A relative URL (no scheme or authority) with path, query parameters, and fragment.
*
* This represents a URL reference relative to some base, such as:
* - A path within a binary cache: "nar/xxx.nar.xz?sig=abc"
* - An absolute path reference: "/foo/bar?x=1#section"
* - A relative path reference: "foo/bar?x=1#section"
*
* The path is represented the same way as `ParsedURL::path`:
* a vector of percent-decoded path segments split on '/'.
*
* @see ParsedURL::path for documentation on the path representation.
*/
struct ParsedRelativeUrl
{
/**
* Path segments, same representation as `ParsedURL::path`.
*
* An absolute path starts with an empty segment (leading slash),
* a relative path does not.
*/
std::vector<std::string> path;
/**
* Query parameters. `std::nullopt` means no query component (preserves base query
* during resolution), empty map means empty query (e.g., `?` with no params).
*/
std::optional<StringMap> query;
std::string fragment;
static ParsedRelativeUrl parse(std::string_view raw, bool lenient = false);
/**
* Render to a string, encoding the path, query parameters, and fragment.
*/
std::string to_string() const;
/**
* Render the path to a string.
*
* @param encode Whether to percent encode path segments.
*/
std::string renderPath(bool encode = false) const;
auto operator<=>(const ParsedRelativeUrl &) const = default;
};
std::ostream & operator<<(std::ostream & os, const ParsedRelativeUrl & url);
MakeError(BadURL, Error);
std::string percentDecode(std::string_view in);
@@ -298,11 +346,39 @@ std::string encodeQuery(const StringMap & query);
ParsedURL parseURL(std::string_view url, bool lenient = false);
/**
* Like `parseURL`, but also accepts relative URLs, which are resolved
* against the given base URL.
* Exactly what is says.
*
* It is important that we don't use one type for both of these, because there
* are many cases where we must know whether we have one or the other.
*/
using ParsedMaybeRelativeURL = std::variant<ParsedRelativeUrl, ParsedURL>;
/**
* Parse a URL that may be either absolute or relative.
*
* @return `ParsedURL` if the URL has a scheme, `ParsedRelativeUrl` otherwise.
*
* @throws BadURL
*/
ParsedMaybeRelativeURL parsePossiblyRelativeURL(std::string_view url);
/**
* Resolve a relative URL against a base URL.
*
* This is specified in [IETF RFC 3986, section 5](https://datatracker.ietf.org/doc/html/rfc3986#section-5)
*
* @param url The relative URL to resolve
* @param base The base URL to resolve against
* @return The resolved absolute URL
*
* @throws BadURL
*/
ParsedURL resolveParsedRelativeUrl(const ParsedRelativeUrl & url, const ParsedURL & base);
/**
* A small wrapper around @ref parsePossiblyRelativeURL and @ref
* resolveParsedRelativeUrl.
*
* @throws BadURL
*
* Behavior should also match the `new URL(url, base)` JavaScript
@@ -371,6 +447,9 @@ std::filesystem::path urlPathToPath(std::span<const std::string> urlPath);
* not strictly RFC3986 compliant. We must preserve that information verbatim.
*
* Though we perform parsing and validation for internal needs.
*
* There is intentionally no `operator==` on `VerbatimURL` because
* equality on string-based ad-hoc URLs is not well-defined.
*/
struct VerbatimURL
{
+202 -78
View File
@@ -111,6 +111,7 @@ static std::string percentEncodeCharSet(std::string_view s, auto charSet)
}
static ParsedURL fromBoostUrlView(boost::urls::url_view url, bool lenient);
static ParsedRelativeUrl fromBoostUrlViewRelative(boost::urls::url_view urlView, bool lenient);
ParsedURL parseURL(std::string_view url, bool lenient)
try {
@@ -158,6 +159,38 @@ try {
throw BadURL("'%s' is not a valid URL: %s", url, e.code().message());
}
ParsedRelativeUrl ParsedRelativeUrl::parse(std::string_view raw, bool lenient)
{
auto parsed = boost::urls::parse_relative_ref(raw);
if (!parsed)
throw BadURL("invalid relative URL '%s': %s", raw, parsed.error().message());
return fromBoostUrlViewRelative(*parsed, lenient);
}
/**
* Decode a percent-encoded URL path into path segments.
*/
static std::vector<std::string> decodeUrlPath(std::string_view encodedPath)
{
return std::views::transform(splitString<std::vector<std::string_view>>(encodedPath, "/"), percentDecode)
| std::ranges::to<std::vector<std::string>>();
}
/**
* Extract path, query, and fragment from a boost url_view into a ParsedRelativeUrl.
*/
static ParsedRelativeUrl fromBoostUrlViewRelative(boost::urls::url_view urlView, bool lenient)
{
return ParsedRelativeUrl{
.path = decodeUrlPath(urlView.encoded_path()),
/* Get the raw query. Store URI supports smuggling doubly nested queries, where
the inner &/? are pct-encoded. */
.query = urlView.has_query() ? std::optional{decodeQuery(urlView.encoded_query(), lenient)} : std::nullopt,
.fragment = urlView.has_fragment() ? std::string{urlView.fragment()} : std::string{},
};
}
static ParsedURL fromBoostUrlView(boost::urls::url_view urlView, bool lenient)
{
if (!urlView.has_scheme())
@@ -181,78 +214,136 @@ static ParsedURL fromBoostUrlView(boost::urls::url_view urlView, bool lenient)
if (authority && authority->host.size() && transportIsFile)
throw BadURL("file:// URL '%s' has unexpected authority '%s'", urlView.buffer(), *authority);
auto fragment = urlView.fragment(); /* Does pct-decoding */
ParsedRelativeUrl relative = fromBoostUrlViewRelative(urlView, lenient);
boost::core::string_view encodedPath = urlView.encoded_path();
if (transportIsFile && encodedPath.empty())
encodedPath = "/";
auto path = std::views::transform(splitString<std::vector<std::string_view>>(encodedPath, "/"), percentDecode)
| std::ranges::to<std::vector<std::string>>();
/* Get the raw query. Store URI supports smuggling doubly nested queries, where
the inner &/? are pct-encoded. */
auto query = std::string_view(urlView.encoded_query());
// Handle empty path for file:// URLs
if (transportIsFile && relative.path.empty())
relative.path = {""};
return ParsedURL{
.scheme = scheme,
.authority = authority,
.path = std::move(path),
.query = decodeQuery(query, lenient),
.fragment = fragment,
.path = std::move(relative.path),
.query = std::move(relative.query).value_or(StringMap{}),
.fragment = std::move(relative.fragment),
};
}
std::variant<ParsedRelativeUrl, ParsedURL> parsePossiblyRelativeURL(std::string_view url)
{
auto parsed = boost::urls::parse_uri_reference(url);
if (!parsed)
throw BadURL("'%s' is not a valid URL: %s", url, parsed.error().message());
if (parsed->has_scheme())
return fromBoostUrlView(*parsed, /*lenient=*/false);
else
return fromBoostUrlViewRelative(*parsed, /*lenient=*/false);
}
/**
* Check if a URL path is empty (no path or just a single empty segment).
*/
static bool isEmptyPath(const std::vector<std::string> & path)
{
return path.empty() || (path.size() == 1 && path[0].empty());
}
/**
* Remove dot segments from a path per RFC 3986 Section 5.2.4.
*/
static std::vector<std::string> canonicalizeDotSegments(std::vector<std::string> path)
{
std::vector<std::string> output;
for (auto & segment : path) {
if (segment == ".") {
// Skip "." segments, but preserve trailing slash
if (&segment == &path.back())
output.push_back("");
} else if (segment == "..") {
// Go up one level: remove last non-empty segment
if (output.size() > 1)
output.pop_back();
// Preserve trailing slash
if (&segment == &path.back() && !output.empty())
output.push_back("");
} else {
output.push_back(std::move(segment));
}
}
return output;
}
/**
* Merge a relative path with a base path per RFC 3986 Section 5.2.3.
*/
static std::vector<std::string>
mergePaths(const std::vector<std::string> & base, const std::vector<std::string> & ref, bool baseHasAuthority)
{
// If base has authority and empty path, result is "/" + ref
if (baseHasAuthority && isEmptyPath(base)) {
std::vector<std::string> result = {""};
result.insert(result.end(), ref.begin(), ref.end());
return result;
}
// Remove everything after last "/" in base (i.e., remove last segment) and append ref
std::vector<std::string> result = base;
if (!result.empty())
result.pop_back();
result.insert(result.end(), ref.begin(), ref.end());
return result;
}
ParsedURL resolveParsedRelativeUrl(const ParsedRelativeUrl & ref, const ParsedURL & base)
{
// RFC 3986 Section 5.2.2 - Transform References
// Since we only handle relative references here (no scheme), we follow the R.scheme undefined branch
std::vector<std::string> targetPath;
StringMap targetQuery;
// Path representations:
// - "" parses to [""] (one empty segment)
// - "/" parses to ["", ""] (two empty segments)
// - "/foo" parses to ["", "foo"]
// - "foo" parses to ["foo"]
bool hasEmptyPath = isEmptyPath(ref.path);
bool hasAbsolutePath = ref.path.size() >= 2 && ref.path[0].empty();
if (hasEmptyPath) {
// Empty path: use base path
targetPath = base.path;
// If ref has query, use it; otherwise use base query
targetQuery = ref.query.value_or(base.query);
} else if (hasAbsolutePath) {
// Absolute path (starts with "/"): use ref path directly
targetPath = canonicalizeDotSegments(ref.path);
targetQuery = ref.query.value_or(StringMap{});
} else {
// Relative path: merge with base
targetPath = canonicalizeDotSegments(mergePaths(base.path, ref.path, base.authority.has_value()));
targetQuery = ref.query.value_or(StringMap{});
}
return ParsedURL{
.scheme = base.scheme,
.authority = base.authority,
.path = targetPath,
.query = targetQuery,
.fragment = ref.fragment,
};
}
ParsedURL parseURLRelative(std::string_view urlS, const ParsedURL & base)
try {
boost::urls::url resolved;
try {
resolved.set_scheme(base.scheme);
if (base.authority) {
auto & authority = *base.authority;
resolved.set_host_address(authority.host);
if (authority.user)
resolved.set_user(*authority.user);
if (authority.password)
resolved.set_password(*authority.password);
if (authority.port)
resolved.set_port_number(*authority.port);
}
resolved.set_encoded_path(encodeUrlPath(base.path));
resolved.set_encoded_query(encodeQuery(base.query));
resolved.set_fragment(base.fragment);
} catch (boost::system::system_error & e) {
throw BadURL("'%s' is not a valid URL: %s", base.to_string(), e.code().message());
}
boost::urls::url_view url;
try {
url = urlS;
resolved.resolve(url).value();
} catch (boost::system::system_error & e) {
throw BadURL("'%s' is not a valid URL: %s", urlS, e.code().message());
}
auto ret = fromBoostUrlView(resolved, /*lenient=*/false);
/* Hack: Boost `url_view` supports Zone IDs, but `url` does not.
Just manually take the authority from the original URL to work
around it. See https://github.com/boostorg/url/issues/919 for
details. */
if (!url.has_authority()) {
ret.authority = base.authority;
}
/* Hack, work around fragment of base URL improperly being preserved
https://github.com/boostorg/url/issues/920 */
ret.fragment = url.has_fragment() ? std::string{url.fragment()} : "";
return ret;
} catch (BadURL & e) {
e.addTrace({}, "while resolving possibly-relative url '%s' against base URL '%s'", urlS, base);
throw;
{
auto parsed = parsePossiblyRelativeURL(urlS);
return std::visit(
overloaded{
[&](ParsedRelativeUrl & relative) { return resolveParsedRelativeUrl(relative, base); },
[&](ParsedURL & absolute) { return std::move(absolute); },
},
parsed);
}
std::string percentDecode(std::string_view in)
@@ -333,25 +424,54 @@ std::string ParsedURL::renderPath(bool encode) const
return renderUrlPathNoPctEncoding(path);
}
std::string ParsedURL::renderAuthorityAndPath() const
std::string ParsedRelativeUrl::renderPath(bool encode) const
{
if (encode)
return encodeUrlPath(path);
return renderUrlPathNoPctEncoding(path);
}
/**
* Render the authority component, validating path constraints per RFC 3986 Section 3.3.
*
* URL parser will never violate these properties, but hand-constructed ParsedURLs might.
*/
static std::string
renderAuthority(const std::optional<ParsedURL::Authority> & authority, const std::vector<std::string> & path)
{
std::string res;
/* The following assertions correspond to 3.3. Path [rfc3986]. URL parser
will never violate these properties, but hand-constructed ParsedURLs might. */
if (authority.has_value()) {
/* If a URI contains an authority component, then the path component
must either be empty or begin with a slash ("/") character. */
assert(path.empty() || path.front().empty());
res += authority->to_string();
return authority->to_string();
} else if (std::ranges::equal(std::views::take(path, 3), std::views::repeat("", 3))) {
/* If a URI does not contain an authority component, then the path cannot begin
with two slash characters ("//") */
unreachable();
}
res += encodeUrlPath(path);
return "";
}
static std::string
renderPathQueryFragment(const std::vector<std::string> & path, const StringMap * query, const std::string & fragment)
{
std::string res = encodeUrlPath(path);
if (query) {
res += '?';
res += encodeQuery(*query);
}
if (!fragment.empty()) {
res += '#';
res += percentEncode(fragment);
}
return res;
}
std::string ParsedURL::renderAuthorityAndPath() const
{
return renderAuthority(authority, path) + encodeUrlPath(path);
}
std::string ParsedURL::to_string() const
{
std::string res;
@@ -359,24 +479,28 @@ std::string ParsedURL::to_string() const
res += ":";
if (authority.has_value())
res += "//";
res += renderAuthorityAndPath();
if (!query.empty()) {
res += "?";
res += encodeQuery(query);
}
if (!fragment.empty()) {
res += "#";
res += percentEncode(fragment);
}
res += renderAuthority(authority, path);
res += renderPathQueryFragment(path, query.empty() ? nullptr : &query, fragment);
return res;
}
std::string ParsedRelativeUrl::to_string() const
{
return renderPathQueryFragment(path, get(query), fragment);
}
std::ostream & operator<<(std::ostream & os, const ParsedURL & url)
{
os << url.to_string();
return os;
}
std::ostream & operator<<(std::ostream & os, const ParsedRelativeUrl & url)
{
os << url.to_string();
return os;
}
ParsedURL ParsedURL::canonicalise()
{
ParsedURL res(*this);