Jorijn van der Graaf
7ee938c98e
One request to an http(s) URL with redirects followed by RFC 9110 §15.4: 301, 302, 303, 307 and 308 for every method; a 303 makes the request a GET without its body and content headers, a 301 or 302 does so for a POST alone, a 307 or 308 keeps the method and the body. The Location is resolved against the sending URL by RFC 3986, a hop between http and https is allowed, Authorization, Proxy-Authorization and Cookie are dropped on a change of origin, and each hop is a client of its own, since one ClientHTTP1 is one origin. A redirect without a usable Location, to another scheme, back to a URL already visited or past maxRedirects comes back as the response, with `stop` saying why. The new :URL partition is the URL type behind it, exported: Parse, Resolve (RFC 3986 §5.2; its §5.4 examples are the test), ToString, and the pieces a request is built from. This replaces the FetchHTTP1 free function, which followed redirects the way Python's urllib did (its repeat limit, its quoting of the Location, a POST turned back on a 307) for a port that no longer needs the fidelity. Breaking: FetchHTTP1, FetchOptionsHTTP1 and FetchResultHTTP1 are gone; use ClientHTTP1::Fetch, FetchOptions and FetchResult, whose `url` is a URL and whose `limited` became `stop`.
275 lines
11 KiB
C++
275 lines
11 KiB
C++
//SPDX-License-Identifier: LGPL-3.0-only
|
|
//SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts®
|
|
|
|
module;
|
|
#include <cerrno>
|
|
|
|
module Crafter.Network:ClientHTTP1_impl;
|
|
import :ClientHTTP1;
|
|
import :ClientTCP;
|
|
import :HTTP;
|
|
import :HTTP1;
|
|
import :Stream;
|
|
import :TLS;
|
|
import :URL;
|
|
import Crafter.Thread;
|
|
import std;
|
|
|
|
using namespace Crafter;
|
|
|
|
namespace {
|
|
// Host header value. The port is elided when it is the scheme default,
|
|
// which is what every other HTTP/1.1 client on the wire does and what
|
|
// virtual-host matching on the far side tends to expect.
|
|
std::string DefaultAuthority(const std::string& host, std::uint16_t port, bool secure) {
|
|
// An IPv6 literal goes in brackets, or its colons read as the port's.
|
|
const std::string name = host.contains(':') ? "[" + host + "]" : host;
|
|
if (port == (secure ? 443 : 80)) return name;
|
|
return name + ":" + std::to_string(port);
|
|
}
|
|
}
|
|
|
|
struct ClientHTTP1::Impl {
|
|
std::string host;
|
|
std::uint16_t port;
|
|
// Null for plaintext; shared by every connection this client dials, so the
|
|
// trust store is parsed once rather than per redial.
|
|
std::shared_ptr<TLSContext> tls;
|
|
std::unique_ptr<ClientTCP> tcp;
|
|
// Sits on top of `tcp` and must therefore be destroyed before it.
|
|
std::unique_ptr<ByteStream> stream;
|
|
std::string protocol;
|
|
|
|
void Connect(std::chrono::milliseconds connectTimeout, std::chrono::milliseconds handshakeTimeout) {
|
|
tcp = std::make_unique<ClientTCP>(host, port, connectTimeout);
|
|
try {
|
|
if (tls) {
|
|
stream = TLSStream::Connect(tcp->socketid, tls, host, handshakeTimeout);
|
|
} else {
|
|
stream = std::make_unique<PlainStream>(tcp->socketid);
|
|
}
|
|
} catch (...) {
|
|
// A half-built connection must not be left pooled: the next Send()
|
|
// would treat it as reusable and read from a socket with no
|
|
// session on it.
|
|
tcp.reset();
|
|
throw;
|
|
}
|
|
protocol = std::string(stream->Protocol());
|
|
}
|
|
|
|
void Close() {
|
|
stream.reset();
|
|
tcp.reset();
|
|
}
|
|
|
|
// One request/response exchange on the current connection. `received`
|
|
// is set as soon as any response byte arrives, which is what decides
|
|
// whether a failure is safe to replay on a new connection.
|
|
HTTPResponse Exchange(const std::string& wire, std::string_view method,
|
|
const HTTP1::MessageLimits& limits,
|
|
std::chrono::milliseconds timeout, bool& received) {
|
|
stream->Write(wire.data(), wire.size(), timeout);
|
|
|
|
HTTP1::MessageParser parser(HTTP1::MessageKind::Response, limits);
|
|
parser.SetRequestMethod(method);
|
|
std::vector<char> chunk(16 * 1024);
|
|
while (!parser.Complete()) {
|
|
std::size_t read = 0;
|
|
const StreamStatus status = stream->ReadSome(chunk.data(), chunk.size(),
|
|
timeout, read);
|
|
if (status == StreamStatus::TimedOut) {
|
|
throw std::runtime_error("timed out waiting for the response");
|
|
}
|
|
if (status == StreamStatus::Closed) {
|
|
// Peer closed. Completes a response framed by close;
|
|
// anything else throws out of Finish().
|
|
parser.Finish();
|
|
break;
|
|
}
|
|
received = true;
|
|
parser.Feed(chunk.data(), read);
|
|
}
|
|
if (!parser.Complete()) {
|
|
throw std::runtime_error("connection closed before a complete response arrived");
|
|
}
|
|
HTTPResponse response = parser.TakeResponse();
|
|
if (!parser.KeepAlive()) Close();
|
|
return response;
|
|
}
|
|
};
|
|
|
|
ClientHTTP1::ClientHTTP1(const char* host, std::uint16_t port)
|
|
: host(host), port(port), impl(std::make_unique<Impl>(std::string(host), port)) {}
|
|
|
|
ClientHTTP1::ClientHTTP1(std::string host, std::uint16_t port)
|
|
: ClientHTTP1(host.c_str(), port) {}
|
|
|
|
ClientHTTP1::ClientHTTP1(const char* host, std::uint16_t port, TLSClientCredentials credentials)
|
|
: host(host), port(port), impl(std::make_unique<Impl>(std::string(host), port)) {
|
|
// Built here rather than on first Send() so bad credentials — an
|
|
// unreadable certificate, a trust anchor that is not a certificate —
|
|
// surface at construction, where the caller is still looking.
|
|
impl->tls = TLSContext::Client(credentials);
|
|
}
|
|
|
|
ClientHTTP1::ClientHTTP1(std::string host, std::uint16_t port, TLSClientCredentials credentials)
|
|
: ClientHTTP1(host.c_str(), port, std::move(credentials)) {}
|
|
|
|
ClientHTTP1::ClientHTTP1(ClientHTTP1&&) noexcept = default;
|
|
ClientHTTP1::~ClientHTTP1() = default;
|
|
|
|
bool ClientHTTP1::Connected() const noexcept {
|
|
return impl && impl->tcp != nullptr;
|
|
}
|
|
|
|
bool ClientHTTP1::Secure() const noexcept {
|
|
return impl && impl->tls != nullptr;
|
|
}
|
|
|
|
std::string_view ClientHTTP1::Protocol() const noexcept {
|
|
return impl ? std::string_view(impl->protocol) : std::string_view();
|
|
}
|
|
|
|
void ClientHTTP1::Disconnect() {
|
|
if (impl) impl->Close();
|
|
}
|
|
|
|
HTTPResponse ClientHTTP1::Send(const HTTPRequest& request) {
|
|
HTTPRequest prepared = request;
|
|
if (prepared.authority.empty()) {
|
|
prepared.authority = DefaultAuthority(host, port, Secure());
|
|
}
|
|
const std::string wire = HTTP1::SerializeRequest(prepared);
|
|
const std::string method = prepared.method.empty() ? std::string("GET") : prepared.method;
|
|
|
|
// Two attempts at most, and the second one only for the keep-alive
|
|
// race: the peer may have closed a pooled connection at the same moment
|
|
// we wrote to it, which is indistinguishable from success until the
|
|
// read fails. Nothing is replayed once a response byte has arrived.
|
|
for (int attempt = 0;; ++attempt) {
|
|
const bool reused = impl->tcp != nullptr;
|
|
if (!reused) impl->Connect(connectTimeout, handshakeTimeout);
|
|
|
|
bool received = false;
|
|
try {
|
|
return impl->Exchange(wire, method, limits, timeout, received);
|
|
} catch (const std::exception&) {
|
|
impl->Close();
|
|
if (attempt == 0 && reused && !received) continue;
|
|
throw;
|
|
}
|
|
}
|
|
}
|
|
|
|
void ClientHTTP1::SendAsync(const HTTPRequest& request,
|
|
std::function<void(HTTPResponse)> onSuccess,
|
|
std::function<void(std::string)> onError) {
|
|
HTTPRequest copy = request;
|
|
ThreadPool::Enqueue([this, copy = std::move(copy),
|
|
onSuccess = std::move(onSuccess),
|
|
onError = std::move(onError)]() mutable {
|
|
try {
|
|
HTTPResponse response = this->Send(copy);
|
|
if (onSuccess) onSuccess(std::move(response));
|
|
} catch (const std::exception& e) {
|
|
if (onError) onError(e.what());
|
|
} catch (...) {
|
|
if (onError) onError("unknown error");
|
|
}
|
|
});
|
|
}
|
|
|
|
namespace {
|
|
std::int32_t StatusCode(const HTTPResponse& response) {
|
|
std::int32_t code = 0;
|
|
std::from_chars(response.status.data(), response.status.data() + response.status.size(), code);
|
|
return code;
|
|
}
|
|
|
|
bool IsRedirect(std::int32_t code) {
|
|
return code == 301 || code == 302 || code == 303 || code == 307 || code == 308;
|
|
}
|
|
}
|
|
|
|
FetchResult ClientHTTP1::Fetch(std::string_view url, const FetchOptions& options) {
|
|
URL current = URL::Parse(url);
|
|
if (current.scheme != "http" && current.scheme != "https") {
|
|
throw std::invalid_argument("ClientHTTP1::Fetch: not an http:// or https:// URL: " + std::string(url));
|
|
}
|
|
std::string method = options.method.empty() ? std::string("GET") : options.method;
|
|
std::unordered_map<std::string, std::string> headers = options.headers;
|
|
std::string body = options.body;
|
|
// Every URL the chain has been sent to: a redirect back to one of them
|
|
// would loop.
|
|
std::set<std::string> visited { current.ToString() };
|
|
FetchResult result;
|
|
while (true) {
|
|
// A client per hop: one ClientHTTP1 is one origin, and a redirect
|
|
// may leave it. Destroying the client closes the hop's connection
|
|
// once its response, body included, has been read.
|
|
ClientHTTP1 client = current.Secure() ? ClientHTTP1(current.host, current.PortOrDefault(), options.credentials)
|
|
: ClientHTTP1(current.host, current.PortOrDefault());
|
|
client.connectTimeout = options.connectTimeout;
|
|
client.handshakeTimeout = options.handshakeTimeout;
|
|
client.timeout = options.timeout;
|
|
client.limits = options.limits;
|
|
|
|
HTTPRequest request;
|
|
request.method = method;
|
|
request.scheme = current.scheme;
|
|
request.authority = current.Authority();
|
|
request.path = current.Target();
|
|
request.headers = headers;
|
|
request.body = body;
|
|
result.response = client.Send(request);
|
|
result.url = current;
|
|
|
|
const std::int32_t code = StatusCode(result.response);
|
|
if (!IsRedirect(code)) return result;
|
|
const auto location = result.response.headers.find("location");
|
|
if (location == result.response.headers.end()) {
|
|
result.stop = FetchStop::NoLocation;
|
|
return result;
|
|
}
|
|
URL next;
|
|
try {
|
|
next = current.Resolve(location->second);
|
|
} catch (const std::invalid_argument&) {
|
|
result.stop = FetchStop::NoLocation;
|
|
return result;
|
|
}
|
|
if (next.scheme != "http" && next.scheme != "https") {
|
|
result.stop = FetchStop::OtherScheme;
|
|
return result;
|
|
}
|
|
if (result.redirects >= options.maxRedirects) {
|
|
result.stop = FetchStop::Limit;
|
|
return result;
|
|
}
|
|
if (!visited.insert(next.ToString()).second) {
|
|
result.stop = FetchStop::Loop;
|
|
return result;
|
|
}
|
|
++result.redirects;
|
|
|
|
// RFC 9110 §15.4: a 303 asks for a GET of the Location; a 301 or 302
|
|
// keeps the method, but a POST becomes a GET, as the user agents the
|
|
// sections describe have always done; a 307 or 308 keeps the method
|
|
// and the body. A request that turned into a GET loses the body and
|
|
// the headers that described it.
|
|
const bool toGet = (code == 303 && method != "HEAD") || ((code == 301 || code == 302) && method == "POST");
|
|
if (toGet) {
|
|
method = "GET";
|
|
body.clear();
|
|
std::erase_if(headers, [](const auto& header) { return header.first.starts_with("content-"); });
|
|
}
|
|
// Credentials stay with the origin they were meant for.
|
|
if (!next.SameOrigin(current)) {
|
|
headers.erase("authorization");
|
|
headers.erase("proxy-authorization");
|
|
headers.erase("cookie");
|
|
}
|
|
current = std::move(next);
|
|
}
|
|
}
|