Catcrafts/Crafter.Network
Watch
2
0
Fork
You've already forked Crafter.Network
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(http1): ClientHTTP1::Fetch follows redirects by the RFC, on a URL type

Browse source 
One request to an http(s) URL with redirects followed by RFC 9110 §15.4:
301, 302, 303, 307 and 308 for every method; a 303 makes the request a
GET without its body and content headers, a 301 or 302 does so for a POST
alone, a 307 or 308 keeps the method and the body. The Location is
resolved against the sending URL by RFC 3986, a hop between http and
https is allowed, Authorization, Proxy-Authorization and Cookie are
dropped on a change of origin, and each hop is a client of its own, since
one ClientHTTP1 is one origin. A redirect without a usable Location, to
another scheme, back to a URL already visited or past maxRedirects comes
back as the response, with `stop` saying why.

The new :URL partition is the URL type behind it, exported: Parse,
Resolve (RFC 3986 §5.2; its §5.4 examples are the test), ToString, and
the pieces a request is built from.

This replaces the FetchHTTP1 free function, which followed redirects the
way Python's urllib did (its repeat limit, its quoting of the Location, a
POST turned back on a 307) for a port that no longer needs the fidelity.

Breaking: FetchHTTP1, FetchOptionsHTTP1 and FetchResultHTTP1 are gone;
use ClientHTTP1::Fetch, FetchOptions and FetchResult, whose `url` is a
URL and whose `limited` became `stop`.
This commit is contained in:
Jorijn van der Graaf 2026-10-31 20:39:43 +01:00
commit 7ee938c98e
Signed by: jorijnvdgraaf
GPG key ID: 2937E59CDCC1BCFB
10 changed files with 596 additions and 372 deletions
Download patch file Download diff file Expand all files Collapse all files

331
implementations/Crafter.Network-ClientHTTP1.cpp
View file

@ -11,6 +11,7 @@ import :HTTP;
import :HTTP1;
import :Stream;
import :TLS;
import :URL;
import Crafter.Thread;
import std;
@ -180,254 +181,35 @@ void ClientHTTP1::SendAsync(const HTTPRequest& request,
}
namespace {
// The five parts urllib.parse.urlsplit cuts a URL into. FetchHTTP1 needs
// the splitting, the joining (urljoin) and the reassembly to resolve a
// Location the way urllib does, and nothing else of urllib.parse.
struct SplitURL {
std::string scheme;
std::string netloc;
std::string path;
std::string query;
std::string fragment;
};
bool IsAsciiAlpha(char c) {
return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z');
}
bool IsSchemeChar(char c) {
return IsAsciiAlpha(c) || (c >= '0' && c <= '9') || c == '+' || c == '-' || c == '.';
}
std::string ToLowerAsciiCopy(std::string_view text) {
std::string lower(text);
for (char& c : lower) {
if (c >= 'A' && c <= 'Z') c = static_cast<char>(c - 'A' + 'a');
}
return lower;
}
// str.split('/'): every piece, empty ones included, and [""] for "".
std::vector<std::string> SplitSlashes(std::string_view text) {
std::vector<std::string> pieces;
std::size_t start = 0;
while (true) {
const std::size_t slash = text.find('/', start);
if (slash == std::string_view::npos) {
pieces.emplace_back(text.substr(start));
return pieces;
}
pieces.emplace_back(text.substr(start, slash - start));
start = slash + 1;
}
}
// urlsplit: tabs and newlines dropped anywhere, leading controls and
// spaces stripped, the scheme lowercased.
SplitURL Split(std::string_view input) {
std::string url;
url.reserve(input.size());
for (const char c : input) {
if (c != '\t' && c != '\r' && c != '\n') url.push_back(c);
}
std::size_t start = 0;
while (start < url.size() && static_cast<unsigned char>(url[start]) <= 0x20) ++start;
url.erase(0, start);
SplitURL parts;
const std::size_t colon = url.find(':');
if (colon != std::string::npos && colon > 0 && IsAsciiAlpha(url[0])
&& std::ranges::all_of(std::string_view(url).substr(0, colon), IsSchemeChar)) {
parts.scheme = ToLowerAsciiCopy(std::string_view(url).substr(0, colon));
url.erase(0, colon + 1);
}
if (url.starts_with("//")) {
std::size_t end = url.find_first_of("/?#", 2);
if (end == std::string::npos) end = url.size();
parts.netloc = url.substr(2, end - 2);
url.erase(0, end);
if (parts.netloc.contains('[') != parts.netloc.contains(']')) {
throw std::invalid_argument("FetchHTTP1: unbalanced brackets in the host of " + std::string(input));
}
}
if (const std::size_t hash = url.find('#'); hash != std::string::npos) {
parts.fragment = url.substr(hash + 1);
url.erase(hash);
}
if (const std::size_t question = url.find('?'); question != std::string::npos) {
parts.query = url.substr(question + 1);
url.erase(question);
}
parts.path = std::move(url);
return parts;
}
// urlunsplit, for the http(s) URLs FetchHTTP1 puts back together.
std::string Unsplit(const SplitURL& parts) {
std::string url = parts.path;
if (!parts.netloc.empty()) {
if (!url.empty() && url.front() != '/') url.insert(url.begin(), '/');
url = "//" + parts.netloc + url;
} else if (url.starts_with("//")) {
url = "//" + url;
} else if ((parts.scheme == "http" || parts.scheme == "https") && (url.empty() || url.front() == '/')) {
url = "//" + url;
}
if (!parts.scheme.empty()) url = parts.scheme + ":" + url;
if (!parts.query.empty()) url += "?" + parts.query;
if (!parts.fragment.empty()) url += "#" + parts.fragment;
return url;
}
// urljoin, for a reference found in a redirect: RFC 3986 resolution the
// way Python does it, a `..` that would climb above the root dropped.
std::string Join(std::string_view base, std::string_view reference) {
if (base.empty()) return std::string(reference);
if (reference.empty()) return std::string(base);
const SplitURL from = Split(base);
SplitURL to = Split(reference);
if (to.scheme.empty()) to.scheme = from.scheme;
const bool relative = to.scheme == "http" || to.scheme == "https";
if (to.scheme != from.scheme || !relative) return std::string(reference);
if (!to.netloc.empty()) return Unsplit(to);
to.netloc = from.netloc;
if (to.path.empty()) {
to.path = from.path;
if (to.query.empty()) to.query = from.query;
return Unsplit(to);
}
std::vector<std::string> baseParts = SplitSlashes(from.path);
// The base's last segment is a file, not a directory: it does not
// take part in resolving.
if (!baseParts.back().empty()) baseParts.pop_back();
std::vector<std::string> segments;
if (to.path.starts_with('/')) {
segments = SplitSlashes(to.path);
} else {
segments = std::move(baseParts);
std::vector<std::string> more = SplitSlashes(to.path);
segments.insert(segments.end(), more.begin(), more.end());
// Empty segments between the first and the last would come back
// as doubled slashes.
if (segments.size() > 2) {
std::vector<std::string> kept { segments.front() };
for (std::size_t i = 1; i + 1 < segments.size(); ++i) {
if (!segments[i].empty()) kept.push_back(segments[i]);
}
kept.push_back(segments.back());
segments = std::move(kept);
}
}
std::vector<std::string> resolved;
for (const std::string& segment : segments) {
if (segment == "..") {
if (!resolved.empty()) resolved.pop_back();
} else if (segment != ".") {
resolved.push_back(segment);
}
}
// A reference ending in a directory step keeps its trailing slash.
if (segments.back() == "." || segments.back() == "..") resolved.emplace_back();
std::string path;
for (std::size_t i = 0; i < resolved.size(); ++i) {
if (i > 0) path += '/';
path += resolved[i];
}
to.path = path.empty() ? std::string("/") : path;
return Unsplit(to);
}
// urllib quotes a Location before it follows it, keeping every ASCII
// letter, digit and punctuation mark: what it escapes are spaces,
// controls and bytes outside ASCII.
std::string QuoteLocation(std::string_view location) {
std::string quoted;
for (const char c : location) {
const unsigned char byte = static_cast<unsigned char>(c);
if (byte > 0x20 && byte < 0x7f) {
quoted.push_back(c);
} else {
quoted += std::format("%{:02X}", byte);
}
}
return quoted;
}
struct FetchTarget {
bool secure = false;
// Without brackets: what the socket and TLS verification are given.
std::string host;
std::uint16_t port = 0;
// The URL's host[:port] as written, for the Host header.
std::string authority;
// Path and query: the request target.
std::string target;
};
FetchTarget ParseTarget(std::string_view url) {
const SplitURL parts = Split(url);
if (parts.scheme != "http" && parts.scheme != "https") {
throw std::invalid_argument("FetchHTTP1: not an http:// or https:// URL: " + std::string(url));
}
FetchTarget target;
target.secure = parts.scheme == "https";
std::string hostinfo = parts.netloc;
if (const std::size_t at = hostinfo.rfind('@'); at != std::string::npos) hostinfo.erase(0, at + 1);
std::string port;
if (hostinfo.starts_with('[')) {
const std::size_t close = hostinfo.find(']');
target.host = hostinfo.substr(1, close - 1);
const std::string rest = hostinfo.substr(close + 1);
if (rest.starts_with(':')) {
port = rest.substr(1);
} else if (!rest.empty()) {
throw std::invalid_argument("FetchHTTP1: unexpected text after the IPv6 host in " + std::string(url));
}
} else {
const std::size_t colon = hostinfo.find(':');
target.host = hostinfo.substr(0, colon);
if (colon != std::string::npos) port = hostinfo.substr(colon + 1);
}
if (target.host.empty()) {
throw std::invalid_argument("FetchHTTP1: no host in " + std::string(url));
}
if (port.empty()) {
target.port = target.secure ? 443 : 80;
} else {
std::uint32_t value = 0;
const auto [end, error] = std::from_chars(port.data(), port.data() + port.size(), value);
if (error != std::errc() || end != port.data() + port.size() || value > 65535) {
throw std::invalid_argument("FetchHTTP1: bad port '" + port + "' in " + std::string(url));
}
target.port = static_cast<std::uint16_t>(value);
}
target.authority = hostinfo;
target.target = parts.path.empty() ? std::string("/") : parts.path;
if (!target.target.starts_with('/')) target.target.insert(target.target.begin(), '/');
if (!parts.query.empty()) target.target += "?" + parts.query;
return target;
}
std::int32_t StatusCode(const HTTPResponse& response) {
std::int32_t code = 0;
std::from_chars(response.status.data(), response.status.data() + response.status.size(), code);
return code;
}
bool IsRedirect(std::int32_t code) {
return code == 301 || code == 302 || code == 303 || code == 307 || code == 308;
}
}
FetchResultHTTP1 Crafter::FetchHTTP1(std::string_view url, const FetchOptionsHTTP1& options) {
std::string current(url);
FetchResult ClientHTTP1::Fetch(std::string_view url, const FetchOptions& options) {
URL current = URL::Parse(url);
if (current.scheme != "http" && current.scheme != "https") {
throw std::invalid_argument("ClientHTTP1::Fetch: not an http:// or https:// URL: " + std::string(url));
}
std::string method = options.method.empty() ? std::string("GET") : options.method;
std::unordered_map<std::string, std::string> headers = options.headers;
std::string body = options.body;
// How often the chain has been sent to each URL: urllib's loop check.
std::map<std::string, std::uint32_t> visited;
FetchResultHTTP1 result;
// Every URL the chain has been sent to: a redirect back to one of them
// would loop.
std::set<std::string> visited { current.ToString() };
FetchResult result;
while (true) {
const FetchTarget target = ParseTarget(current);
ClientHTTP1 client = target.secure ? ClientHTTP1(target.host, target.port, options.credentials)
: ClientHTTP1(target.host, target.port);
// A client per hop: one ClientHTTP1 is one origin, and a redirect
// may leave it. Destroying the client closes the hop's connection
// once its response, body included, has been read.
ClientHTTP1 client = current.Secure() ? ClientHTTP1(current.host, current.PortOrDefault(), options.credentials)
: ClientHTTP1(current.host, current.PortOrDefault());
client.connectTimeout = options.connectTimeout;
client.handshakeTimeout = options.handshakeTimeout;
client.timeout = options.timeout;
@ -435,48 +217,59 @@ FetchResultHTTP1 Crafter::FetchHTTP1(std::string_view url, const FetchOptionsHTT
HTTPRequest request;
request.method = method;
request.scheme = target.secure ? "https" : "http";
request.authority = target.authority;
request.path = target.target;
request.scheme = current.scheme;
request.authority = current.Authority();
request.path = current.Target();
request.headers = headers;
request.body = body;
result.response = client.Send(request);
result.url = current;
const std::int32_t code = StatusCode(result.response);
const bool redirect = code == 301 || code == 302 || code == 303 || code == 307 || code == 308;
if (!redirect) return result;
// The methods urllib redirects; any other comes back as the redirect.
const bool followed = method == "GET" || method == "HEAD" || (method == "POST" && code <= 303);
if (!followed) return result;
auto location = result.response.headers.find("location");
if (location == result.response.headers.end()) location = result.response.headers.find("uri");
if (location == result.response.headers.end()) return result;
std::string next = QuoteLocation(location->second);
SplitURL parts = Split(next);
if (!parts.scheme.empty() && parts.scheme != "http" && parts.scheme != "https") return result;
if (parts.path.empty() && !parts.netloc.empty()) {
parts.path = "/";
next = Unsplit(parts);
}
next = Join(current, next);
const auto seen = visited.find(next);
const std::uint32_t times = seen == visited.end() ? 0 : seen->second;
if (times >= options.maxRepeats || visited.size() >= options.maxRedirects) {
result.limited = true;
if (!IsRedirect(code)) return result;
const auto location = result.response.headers.find("location");
if (location == result.response.headers.end()) {
result.stop = FetchStop::NoLocation;
return result;
}
URL next;
try {
next = current.Resolve(location->second);
} catch (const std::invalid_argument&) {
result.stop = FetchStop::NoLocation;
return result;
}
if (next.scheme != "http" && next.scheme != "https") {
result.stop = FetchStop::OtherScheme;
return result;
}
if (result.redirects >= options.maxRedirects) {
result.stop = FetchStop::Limit;
return result;
}
if (!visited.insert(next.ToString()).second) {
result.stop = FetchStop::Loop;
return result;
}
visited[next] = times + 1;
++result.redirects;
// What urllib sends next: a GET, or a HEAD for a HEAD, without the
// body or the headers that described it.
if (method != "HEAD") method = "GET";
body.clear();
headers.erase("content-length");
headers.erase("content-type");
// RFC 9110 §15.4: a 303 asks for a GET of the Location; a 301 or 302
// keeps the method, but a POST becomes a GET, as the user agents the
// sections describe have always done; a 307 or 308 keeps the method
// and the body. A request that turned into a GET loses the body and
// the headers that described it.
const bool toGet = (code == 303 && method != "HEAD") || ((code == 301 || code == 302) && method == "POST");
if (toGet) {
method = "GET";
body.clear();
std::erase_if(headers, [](const auto& header) { return header.first.starts_with("content-"); });
}
// Credentials stay with the origin they were meant for.
if (!next.SameOrigin(current)) {
headers.erase("authorization");
headers.erase("proxy-authorization");
headers.erase("cookie");
}
current = std::move(next);
}
}