Jorijn van der Graaf
7ee938c98e
One request to an http(s) URL with redirects followed by RFC 9110 §15.4: 301, 302, 303, 307 and 308 for every method; a 303 makes the request a GET without its body and content headers, a 301 or 302 does so for a POST alone, a 307 or 308 keeps the method and the body. The Location is resolved against the sending URL by RFC 3986, a hop between http and https is allowed, Authorization, Proxy-Authorization and Cookie are dropped on a change of origin, and each hop is a client of its own, since one ClientHTTP1 is one origin. A redirect without a usable Location, to another scheme, back to a URL already visited or past maxRedirects comes back as the response, with `stop` saying why. The new :URL partition is the URL type behind it, exported: Parse, Resolve (RFC 3986 §5.2; its §5.4 examples are the test), ToString, and the pieces a request is built from. This replaces the FetchHTTP1 free function, which followed redirects the way Python's urllib did (its repeat limit, its quoting of the Location, a POST turned back on a 307) for a port that no longer needs the fidelity. Breaking: FetchHTTP1, FetchOptionsHTTP1 and FetchResultHTTP1 are gone; use ClientHTTP1::Fetch, FetchOptions and FetchResult, whose `url` is a URL and whose `limited` became `stop`.
271 lines
11 KiB
C++
271 lines
11 KiB
C++
//SPDX-License-Identifier: LGPL-3.0-only
|
|
//SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts®
|
|
|
|
export module Crafter.Network:URL;
|
|
import std;
|
|
|
|
#ifndef CRAFTER_NETWORK_BROWSER
|
|
namespace Crafter {
|
|
namespace URLDetail {
|
|
// The components RFC 3986 appendix B cuts a URI reference into; a
|
|
// reference may lack the scheme and the authority, a URL may not.
|
|
struct Parts {
|
|
std::optional<std::string> scheme;
|
|
std::optional<std::string> authority;
|
|
std::string path;
|
|
std::optional<std::string> query;
|
|
std::optional<std::string> fragment;
|
|
};
|
|
|
|
inline bool IsAsciiAlpha(char c) {
|
|
return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z');
|
|
}
|
|
|
|
inline bool IsSchemeChar(char c) {
|
|
return IsAsciiAlpha(c) || (c >= '0' && c <= '9') || c == '+' || c == '-' || c == '.';
|
|
}
|
|
|
|
inline std::string Lowered(std::string_view text) {
|
|
std::string lower(text);
|
|
for (char& c : lower) {
|
|
if (c >= 'A' && c <= 'Z') c = static_cast<char>(c - 'A' + 'a');
|
|
}
|
|
return lower;
|
|
}
|
|
|
|
// ^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?, with the
|
|
// scheme taken only when it is one by RFC 3986 §3.1.
|
|
inline Parts Split(std::string_view text) {
|
|
Parts parts;
|
|
const std::size_t mark = text.find_first_of(":/?#");
|
|
if (mark != std::string_view::npos && mark > 0 && text[mark] == ':' && IsAsciiAlpha(text[0])
|
|
&& std::ranges::all_of(text.substr(0, mark), IsSchemeChar)) {
|
|
parts.scheme = Lowered(text.substr(0, mark));
|
|
text.remove_prefix(mark + 1);
|
|
}
|
|
if (text.starts_with("//")) {
|
|
text.remove_prefix(2);
|
|
std::size_t end = text.find_first_of("/?#");
|
|
if (end == std::string_view::npos) end = text.size();
|
|
parts.authority = std::string(text.substr(0, end));
|
|
text.remove_prefix(end);
|
|
}
|
|
std::size_t end = text.find_first_of("?#");
|
|
if (end == std::string_view::npos) end = text.size();
|
|
parts.path = std::string(text.substr(0, end));
|
|
text.remove_prefix(end);
|
|
if (text.starts_with('?')) {
|
|
text.remove_prefix(1);
|
|
std::size_t hash = text.find('#');
|
|
if (hash == std::string_view::npos) hash = text.size();
|
|
parts.query = std::string(text.substr(0, hash));
|
|
text.remove_prefix(hash);
|
|
}
|
|
if (text.starts_with('#')) parts.fragment = std::string(text.substr(1));
|
|
return parts;
|
|
}
|
|
|
|
inline void RemoveLastSegment(std::string& path) {
|
|
const std::size_t slash = path.rfind('/');
|
|
path.erase(slash == std::string::npos ? 0 : slash);
|
|
}
|
|
|
|
// RFC 3986 §5.2.4.
|
|
inline std::string RemoveDotSegments(std::string_view path) {
|
|
std::string in(path);
|
|
std::string out;
|
|
while (!in.empty()) {
|
|
if (in.starts_with("../")) {
|
|
in.erase(0, 3);
|
|
} else if (in.starts_with("./")) {
|
|
in.erase(0, 2);
|
|
} else if (in.starts_with("/./")) {
|
|
in.erase(0, 2);
|
|
} else if (in == "/.") {
|
|
in = "/";
|
|
} else if (in.starts_with("/../")) {
|
|
in.erase(0, 3);
|
|
RemoveLastSegment(out);
|
|
} else if (in == "/..") {
|
|
in = "/";
|
|
RemoveLastSegment(out);
|
|
} else if (in == "." || in == "..") {
|
|
in.clear();
|
|
} else {
|
|
std::size_t next = in.find('/', in.starts_with('/') ? 1 : 0);
|
|
if (next == std::string::npos) next = in.size();
|
|
out += in.substr(0, next);
|
|
in.erase(0, next);
|
|
}
|
|
}
|
|
return out;
|
|
}
|
|
|
|
// RFC 3986 §5.2.3.
|
|
inline std::string Merge(bool baseHasAuthority, std::string_view basePath, std::string_view referencePath) {
|
|
if (baseHasAuthority && basePath.empty()) return "/" + std::string(referencePath);
|
|
const std::size_t slash = basePath.rfind('/');
|
|
if (slash == std::string_view::npos) return std::string(referencePath);
|
|
return std::string(basePath.substr(0, slash + 1)) + std::string(referencePath);
|
|
}
|
|
}
|
|
|
|
// An absolute URL, as RFC 3986 reads it:
|
|
// `scheme://[userinfo@]host[:port][/path][?query][#fragment]`. What an
|
|
// HTTP client needs of URLs: the parse, the resolution of the reference
|
|
// a redirect carries against the URL that sent it, the recomposition,
|
|
// and the pieces a request is built from.
|
|
export struct URL {
|
|
// Lowercased.
|
|
std::string scheme;
|
|
std::string userinfo;
|
|
// Lowercased; an IPv6 literal without its brackets.
|
|
std::string host;
|
|
// None when the URL names no port: the scheme's default.
|
|
std::optional<std::uint16_t> port;
|
|
// Empty, or starting with '/'.
|
|
std::string path;
|
|
// Without the '?'; none when the URL has no query (an empty query,
|
|
// `?` and nothing after it, is "").
|
|
std::optional<std::string> query;
|
|
// Without the '#'.
|
|
std::optional<std::string> fragment;
|
|
|
|
// Throws std::invalid_argument for text without a scheme or a host,
|
|
// a port that is not a number or is above 65535, and unbalanced
|
|
// brackets around an IPv6 host.
|
|
static URL Parse(std::string_view text);
|
|
|
|
// RFC 3986 §5.2 resolution of a reference against this URL as the
|
|
// base: an absolute reference stands as it is, a relative one takes
|
|
// the parts it lacks from the base, and dot segments are removed.
|
|
// Throws std::invalid_argument when the result is no URL (a
|
|
// reference such as `mailto:x` has no host).
|
|
URL Resolve(std::string_view reference) const;
|
|
|
|
std::string ToString() const;
|
|
|
|
bool Secure() const {
|
|
return scheme == "https";
|
|
}
|
|
|
|
// The port, or the scheme's default: 443 for https, 80 for http, 0
|
|
// for any other scheme.
|
|
std::uint16_t PortOrDefault() const {
|
|
if (port) return *port;
|
|
return scheme == "https" ? 443 : scheme == "http" ? 80 : 0;
|
|
}
|
|
|
|
// host[:port] as the Host header carries it: brackets around an IPv6
|
|
// literal, the port only when the URL names one.
|
|
std::string Authority() const {
|
|
std::string authority = host.contains(':') ? "[" + host + "]" : host;
|
|
if (port) authority += ":" + std::to_string(*port);
|
|
return authority;
|
|
}
|
|
|
|
// The request target: the path, "/" when there is none, and the
|
|
// query.
|
|
std::string Target() const {
|
|
std::string target = path.empty() ? std::string("/") : path;
|
|
if (query) target += "?" + *query;
|
|
return target;
|
|
}
|
|
|
|
// The same scheme, host and effective port.
|
|
bool SameOrigin(const URL& other) const {
|
|
return scheme == other.scheme && host == other.host && PortOrDefault() == other.PortOrDefault();
|
|
}
|
|
};
|
|
|
|
namespace URLDetail {
|
|
inline URL FromParts(const Parts& parts, std::string_view shown) {
|
|
if (!parts.scheme) throw std::invalid_argument("URL: no scheme in " + std::string(shown));
|
|
if (!parts.authority) throw std::invalid_argument("URL: no host in " + std::string(shown));
|
|
URL url;
|
|
url.scheme = *parts.scheme;
|
|
std::string hostport = *parts.authority;
|
|
if (const std::size_t at = hostport.rfind('@'); at != std::string::npos) {
|
|
url.userinfo = hostport.substr(0, at);
|
|
hostport.erase(0, at + 1);
|
|
}
|
|
std::string port;
|
|
if (hostport.starts_with('[')) {
|
|
const std::size_t close = hostport.find(']');
|
|
if (close == std::string::npos) throw std::invalid_argument("URL: unbalanced brackets in the host of " + std::string(shown));
|
|
url.host = Lowered(std::string_view(hostport).substr(1, close - 1));
|
|
const std::string rest = hostport.substr(close + 1);
|
|
if (rest.starts_with(':')) {
|
|
port = rest.substr(1);
|
|
} else if (!rest.empty()) {
|
|
throw std::invalid_argument("URL: unexpected text after the IPv6 host in " + std::string(shown));
|
|
}
|
|
} else {
|
|
if (hostport.contains(']')) throw std::invalid_argument("URL: unbalanced brackets in the host of " + std::string(shown));
|
|
const std::size_t colon = hostport.rfind(':');
|
|
url.host = Lowered(std::string_view(hostport).substr(0, colon));
|
|
if (colon != std::string::npos) port = hostport.substr(colon + 1);
|
|
}
|
|
if (url.host.empty()) throw std::invalid_argument("URL: no host in " + std::string(shown));
|
|
if (!port.empty()) {
|
|
std::uint32_t value = 0;
|
|
const auto [end, error] = std::from_chars(port.data(), port.data() + port.size(), value);
|
|
if (error != std::errc() || end != port.data() + port.size() || value > 65535) {
|
|
throw std::invalid_argument("URL: bad port '" + port + "' in " + std::string(shown));
|
|
}
|
|
url.port = static_cast<std::uint16_t>(value);
|
|
}
|
|
url.path = parts.path;
|
|
url.query = parts.query;
|
|
url.fragment = parts.fragment;
|
|
return url;
|
|
}
|
|
}
|
|
|
|
inline URL URL::Parse(std::string_view text) {
|
|
return URLDetail::FromParts(URLDetail::Split(text), text);
|
|
}
|
|
|
|
inline URL URL::Resolve(std::string_view reference) const {
|
|
using namespace URLDetail;
|
|
const Parts ref = Split(reference);
|
|
Parts target;
|
|
if (ref.scheme) {
|
|
target = ref;
|
|
target.path = RemoveDotSegments(ref.path);
|
|
} else {
|
|
if (ref.authority) {
|
|
target.authority = ref.authority;
|
|
target.path = RemoveDotSegments(ref.path);
|
|
target.query = ref.query;
|
|
} else {
|
|
if (ref.path.empty()) {
|
|
target.path = path;
|
|
target.query = ref.query ? ref.query : query;
|
|
} else {
|
|
target.path = RemoveDotSegments(ref.path.starts_with('/') ? ref.path : Merge(true, path, ref.path));
|
|
target.query = ref.query;
|
|
}
|
|
// The base's authority, as it was written.
|
|
std::string authority;
|
|
if (!userinfo.empty()) authority += userinfo + "@";
|
|
authority += Authority();
|
|
target.authority = authority;
|
|
}
|
|
target.scheme = scheme;
|
|
target.fragment = ref.fragment;
|
|
}
|
|
return FromParts(target, reference);
|
|
}
|
|
|
|
inline std::string URL::ToString() const {
|
|
std::string text = scheme + "://";
|
|
if (!userinfo.empty()) text += userinfo + "@";
|
|
text += Authority();
|
|
text += path;
|
|
if (query) text += "?" + *query;
|
|
if (fragment) text += "#" + *fragment;
|
|
return text;
|
|
}
|
|
}
|
|
#endif
|