//SPDX-License-Identifier: LGPL-3.0-only //SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts® module; #include module Crafter.Network:ClientHTTP1_impl; import :ClientHTTP1; import :ClientTCP; import :HTTP; import :HTTP1; import :Stream; import :TLS; import Crafter.Thread; import std; using namespace Crafter; namespace { // Host header value. The port is elided when it is the scheme default, // which is what every other HTTP/1.1 client on the wire does and what // virtual-host matching on the far side tends to expect. std::string DefaultAuthority(const std::string& host, std::uint16_t port, bool secure) { // An IPv6 literal goes in brackets, or its colons read as the port's. const std::string name = host.contains(':') ? "[" + host + "]" : host; if (port == (secure ? 443 : 80)) return name; return name + ":" + std::to_string(port); } } struct ClientHTTP1::Impl { std::string host; std::uint16_t port; // Null for plaintext; shared by every connection this client dials, so the // trust store is parsed once rather than per redial. std::shared_ptr tls; std::unique_ptr tcp; // Sits on top of `tcp` and must therefore be destroyed before it. std::unique_ptr stream; std::string protocol; void Connect(std::chrono::milliseconds connectTimeout, std::chrono::milliseconds handshakeTimeout) { tcp = std::make_unique(host, port, connectTimeout); try { if (tls) { stream = TLSStream::Connect(tcp->socketid, tls, host, handshakeTimeout); } else { stream = std::make_unique(tcp->socketid); } } catch (...) { // A half-built connection must not be left pooled: the next Send() // would treat it as reusable and read from a socket with no // session on it. tcp.reset(); throw; } protocol = std::string(stream->Protocol()); } void Close() { stream.reset(); tcp.reset(); } // One request/response exchange on the current connection. `received` // is set as soon as any response byte arrives, which is what decides // whether a failure is safe to replay on a new connection. HTTPResponse Exchange(const std::string& wire, std::string_view method, const HTTP1::MessageLimits& limits, std::chrono::milliseconds timeout, bool& received) { stream->Write(wire.data(), wire.size(), timeout); HTTP1::MessageParser parser(HTTP1::MessageKind::Response, limits); parser.SetRequestMethod(method); std::vector chunk(16 * 1024); while (!parser.Complete()) { std::size_t read = 0; const StreamStatus status = stream->ReadSome(chunk.data(), chunk.size(), timeout, read); if (status == StreamStatus::TimedOut) { throw std::runtime_error("timed out waiting for the response"); } if (status == StreamStatus::Closed) { // Peer closed. Completes a response framed by close; // anything else throws out of Finish(). parser.Finish(); break; } received = true; parser.Feed(chunk.data(), read); } if (!parser.Complete()) { throw std::runtime_error("connection closed before a complete response arrived"); } HTTPResponse response = parser.TakeResponse(); if (!parser.KeepAlive()) Close(); return response; } }; ClientHTTP1::ClientHTTP1(const char* host, std::uint16_t port) : host(host), port(port), impl(std::make_unique(std::string(host), port)) {} ClientHTTP1::ClientHTTP1(std::string host, std::uint16_t port) : ClientHTTP1(host.c_str(), port) {} ClientHTTP1::ClientHTTP1(const char* host, std::uint16_t port, TLSClientCredentials credentials) : host(host), port(port), impl(std::make_unique(std::string(host), port)) { // Built here rather than on first Send() so bad credentials — an // unreadable certificate, a trust anchor that is not a certificate — // surface at construction, where the caller is still looking. impl->tls = TLSContext::Client(credentials); } ClientHTTP1::ClientHTTP1(std::string host, std::uint16_t port, TLSClientCredentials credentials) : ClientHTTP1(host.c_str(), port, std::move(credentials)) {} ClientHTTP1::ClientHTTP1(ClientHTTP1&&) noexcept = default; ClientHTTP1::~ClientHTTP1() = default; bool ClientHTTP1::Connected() const noexcept { return impl && impl->tcp != nullptr; } bool ClientHTTP1::Secure() const noexcept { return impl && impl->tls != nullptr; } std::string_view ClientHTTP1::Protocol() const noexcept { return impl ? std::string_view(impl->protocol) : std::string_view(); } void ClientHTTP1::Disconnect() { if (impl) impl->Close(); } HTTPResponse ClientHTTP1::Send(const HTTPRequest& request) { HTTPRequest prepared = request; if (prepared.authority.empty()) { prepared.authority = DefaultAuthority(host, port, Secure()); } const std::string wire = HTTP1::SerializeRequest(prepared); const std::string method = prepared.method.empty() ? std::string("GET") : prepared.method; // Two attempts at most, and the second one only for the keep-alive // race: the peer may have closed a pooled connection at the same moment // we wrote to it, which is indistinguishable from success until the // read fails. Nothing is replayed once a response byte has arrived. for (int attempt = 0;; ++attempt) { const bool reused = impl->tcp != nullptr; if (!reused) impl->Connect(connectTimeout, handshakeTimeout); bool received = false; try { return impl->Exchange(wire, method, limits, timeout, received); } catch (const std::exception&) { impl->Close(); if (attempt == 0 && reused && !received) continue; throw; } } } void ClientHTTP1::SendAsync(const HTTPRequest& request, std::function onSuccess, std::function onError) { HTTPRequest copy = request; ThreadPool::Enqueue([this, copy = std::move(copy), onSuccess = std::move(onSuccess), onError = std::move(onError)]() mutable { try { HTTPResponse response = this->Send(copy); if (onSuccess) onSuccess(std::move(response)); } catch (const std::exception& e) { if (onError) onError(e.what()); } catch (...) { if (onError) onError("unknown error"); } }); } namespace { // The five parts urllib.parse.urlsplit cuts a URL into. FetchHTTP1 needs // the splitting, the joining (urljoin) and the reassembly to resolve a // Location the way urllib does, and nothing else of urllib.parse. struct SplitURL { std::string scheme; std::string netloc; std::string path; std::string query; std::string fragment; }; bool IsAsciiAlpha(char c) { return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z'); } bool IsSchemeChar(char c) { return IsAsciiAlpha(c) || (c >= '0' && c <= '9') || c == '+' || c == '-' || c == '.'; } std::string ToLowerAsciiCopy(std::string_view text) { std::string lower(text); for (char& c : lower) { if (c >= 'A' && c <= 'Z') c = static_cast(c - 'A' + 'a'); } return lower; } // str.split('/'): every piece, empty ones included, and [""] for "". std::vector SplitSlashes(std::string_view text) { std::vector pieces; std::size_t start = 0; while (true) { const std::size_t slash = text.find('/', start); if (slash == std::string_view::npos) { pieces.emplace_back(text.substr(start)); return pieces; } pieces.emplace_back(text.substr(start, slash - start)); start = slash + 1; } } // urlsplit: tabs and newlines dropped anywhere, leading controls and // spaces stripped, the scheme lowercased. SplitURL Split(std::string_view input) { std::string url; url.reserve(input.size()); for (const char c : input) { if (c != '\t' && c != '\r' && c != '\n') url.push_back(c); } std::size_t start = 0; while (start < url.size() && static_cast(url[start]) <= 0x20) ++start; url.erase(0, start); SplitURL parts; const std::size_t colon = url.find(':'); if (colon != std::string::npos && colon > 0 && IsAsciiAlpha(url[0]) && std::ranges::all_of(std::string_view(url).substr(0, colon), IsSchemeChar)) { parts.scheme = ToLowerAsciiCopy(std::string_view(url).substr(0, colon)); url.erase(0, colon + 1); } if (url.starts_with("//")) { std::size_t end = url.find_first_of("/?#", 2); if (end == std::string::npos) end = url.size(); parts.netloc = url.substr(2, end - 2); url.erase(0, end); if (parts.netloc.contains('[') != parts.netloc.contains(']')) { throw std::invalid_argument("FetchHTTP1: unbalanced brackets in the host of " + std::string(input)); } } if (const std::size_t hash = url.find('#'); hash != std::string::npos) { parts.fragment = url.substr(hash + 1); url.erase(hash); } if (const std::size_t question = url.find('?'); question != std::string::npos) { parts.query = url.substr(question + 1); url.erase(question); } parts.path = std::move(url); return parts; } // urlunsplit, for the http(s) URLs FetchHTTP1 puts back together. std::string Unsplit(const SplitURL& parts) { std::string url = parts.path; if (!parts.netloc.empty()) { if (!url.empty() && url.front() != '/') url.insert(url.begin(), '/'); url = "//" + parts.netloc + url; } else if (url.starts_with("//")) { url = "//" + url; } else if ((parts.scheme == "http" || parts.scheme == "https") && (url.empty() || url.front() == '/')) { url = "//" + url; } if (!parts.scheme.empty()) url = parts.scheme + ":" + url; if (!parts.query.empty()) url += "?" + parts.query; if (!parts.fragment.empty()) url += "#" + parts.fragment; return url; } // urljoin, for a reference found in a redirect: RFC 3986 resolution the // way Python does it, a `..` that would climb above the root dropped. std::string Join(std::string_view base, std::string_view reference) { if (base.empty()) return std::string(reference); if (reference.empty()) return std::string(base); const SplitURL from = Split(base); SplitURL to = Split(reference); if (to.scheme.empty()) to.scheme = from.scheme; const bool relative = to.scheme == "http" || to.scheme == "https"; if (to.scheme != from.scheme || !relative) return std::string(reference); if (!to.netloc.empty()) return Unsplit(to); to.netloc = from.netloc; if (to.path.empty()) { to.path = from.path; if (to.query.empty()) to.query = from.query; return Unsplit(to); } std::vector baseParts = SplitSlashes(from.path); // The base's last segment is a file, not a directory: it does not // take part in resolving. if (!baseParts.back().empty()) baseParts.pop_back(); std::vector segments; if (to.path.starts_with('/')) { segments = SplitSlashes(to.path); } else { segments = std::move(baseParts); std::vector more = SplitSlashes(to.path); segments.insert(segments.end(), more.begin(), more.end()); // Empty segments between the first and the last would come back // as doubled slashes. if (segments.size() > 2) { std::vector kept { segments.front() }; for (std::size_t i = 1; i + 1 < segments.size(); ++i) { if (!segments[i].empty()) kept.push_back(segments[i]); } kept.push_back(segments.back()); segments = std::move(kept); } } std::vector resolved; for (const std::string& segment : segments) { if (segment == "..") { if (!resolved.empty()) resolved.pop_back(); } else if (segment != ".") { resolved.push_back(segment); } } // A reference ending in a directory step keeps its trailing slash. if (segments.back() == "." || segments.back() == "..") resolved.emplace_back(); std::string path; for (std::size_t i = 0; i < resolved.size(); ++i) { if (i > 0) path += '/'; path += resolved[i]; } to.path = path.empty() ? std::string("/") : path; return Unsplit(to); } // urllib quotes a Location before it follows it, keeping every ASCII // letter, digit and punctuation mark: what it escapes are spaces, // controls and bytes outside ASCII. std::string QuoteLocation(std::string_view location) { std::string quoted; for (const char c : location) { const unsigned char byte = static_cast(c); if (byte > 0x20 && byte < 0x7f) { quoted.push_back(c); } else { quoted += std::format("%{:02X}", byte); } } return quoted; } struct FetchTarget { bool secure = false; // Without brackets: what the socket and TLS verification are given. std::string host; std::uint16_t port = 0; // The URL's host[:port] as written, for the Host header. std::string authority; // Path and query: the request target. std::string target; }; FetchTarget ParseTarget(std::string_view url) { const SplitURL parts = Split(url); if (parts.scheme != "http" && parts.scheme != "https") { throw std::invalid_argument("FetchHTTP1: not an http:// or https:// URL: " + std::string(url)); } FetchTarget target; target.secure = parts.scheme == "https"; std::string hostinfo = parts.netloc; if (const std::size_t at = hostinfo.rfind('@'); at != std::string::npos) hostinfo.erase(0, at + 1); std::string port; if (hostinfo.starts_with('[')) { const std::size_t close = hostinfo.find(']'); target.host = hostinfo.substr(1, close - 1); const std::string rest = hostinfo.substr(close + 1); if (rest.starts_with(':')) { port = rest.substr(1); } else if (!rest.empty()) { throw std::invalid_argument("FetchHTTP1: unexpected text after the IPv6 host in " + std::string(url)); } } else { const std::size_t colon = hostinfo.find(':'); target.host = hostinfo.substr(0, colon); if (colon != std::string::npos) port = hostinfo.substr(colon + 1); } if (target.host.empty()) { throw std::invalid_argument("FetchHTTP1: no host in " + std::string(url)); } if (port.empty()) { target.port = target.secure ? 443 : 80; } else { std::uint32_t value = 0; const auto [end, error] = std::from_chars(port.data(), port.data() + port.size(), value); if (error != std::errc() || end != port.data() + port.size() || value > 65535) { throw std::invalid_argument("FetchHTTP1: bad port '" + port + "' in " + std::string(url)); } target.port = static_cast(value); } target.authority = hostinfo; target.target = parts.path.empty() ? std::string("/") : parts.path; if (!target.target.starts_with('/')) target.target.insert(target.target.begin(), '/'); if (!parts.query.empty()) target.target += "?" + parts.query; return target; } std::int32_t StatusCode(const HTTPResponse& response) { std::int32_t code = 0; std::from_chars(response.status.data(), response.status.data() + response.status.size(), code); return code; } } FetchResultHTTP1 Crafter::FetchHTTP1(std::string_view url, const FetchOptionsHTTP1& options) { std::string current(url); std::string method = options.method.empty() ? std::string("GET") : options.method; std::unordered_map headers = options.headers; std::string body = options.body; // How often the chain has been sent to each URL: urllib's loop check. std::map visited; FetchResultHTTP1 result; while (true) { const FetchTarget target = ParseTarget(current); ClientHTTP1 client = target.secure ? ClientHTTP1(target.host, target.port, options.credentials) : ClientHTTP1(target.host, target.port); client.connectTimeout = options.connectTimeout; client.handshakeTimeout = options.handshakeTimeout; client.timeout = options.timeout; client.limits = options.limits; HTTPRequest request; request.method = method; request.scheme = target.secure ? "https" : "http"; request.authority = target.authority; request.path = target.target; request.headers = headers; request.body = body; result.response = client.Send(request); result.url = current; const std::int32_t code = StatusCode(result.response); const bool redirect = code == 301 || code == 302 || code == 303 || code == 307 || code == 308; if (!redirect) return result; // The methods urllib redirects; any other comes back as the redirect. const bool followed = method == "GET" || method == "HEAD" || (method == "POST" && code <= 303); if (!followed) return result; auto location = result.response.headers.find("location"); if (location == result.response.headers.end()) location = result.response.headers.find("uri"); if (location == result.response.headers.end()) return result; std::string next = QuoteLocation(location->second); SplitURL parts = Split(next); if (!parts.scheme.empty() && parts.scheme != "http" && parts.scheme != "https") return result; if (parts.path.empty() && !parts.netloc.empty()) { parts.path = "/"; next = Unsplit(parts); } next = Join(current, next); const auto seen = visited.find(next); const std::uint32_t times = seen == visited.end() ? 0 : seen->second; if (times >= options.maxRepeats || visited.size() >= options.maxRedirects) { result.limited = true; return result; } visited[next] = times + 1; ++result.redirects; // What urllib sends next: a GET, or a HEAD for a HEAD, without the // body or the headers that described it. if (method != "HEAD") method = "GET"; body.clear(); headers.erase("content-length"); headers.erase("content-type"); current = std::move(next); } }