From 78853033e8193dc3840b38aa69e34f2d51eb56bc Mon Sep 17 00:00:00 2001 From: Jorijn van der Graaf Date: Sat, 31 Oct 2026 05:06:41 +0100 Subject: [PATCH 1/6] feat(tcp): connect over IPv4 and IPv6, and say why a connect failed - ClientTCP resolves with AF_UNSPEC and tries every address the name resolves to, in getaddrinfo's order, the way Python's socket.create_connection does. It used to ask for AF_INET only and keep the first answer, so `localhost` could not reach a server listening on ::1 alone, which is what a dev server on a recent Node does. The connect timeout bounds each address on its own; when none connects, what the last one failed with is thrown. - A failed resolve, socket() or connect throws SocketErrorException: the std::runtime_error it always was, with the same message, plus the errno (SocketErrorKind::System) or the getaddrinfo code (SocketErrorKind::Resolver). A caller that reports the cause in its own words, such as a Python-style "[Errno 111] Connection refused", no longer has to parse it back out of the text. - The move constructor carries the resolved addresses along. New: ShouldConnectDualStack, ShouldReportConnectErrors. Co-Authored-By: Claude Opus 5.5 --- README.md | 14 ++ implementations/Crafter.Network-ClientTCP.cpp | 132 +++++++++------ interfaces/Crafter.Network-ClientTCP.cppm | 42 ++++- project.cpp | 2 + tests/ShouldConnectDualStack/main.cpp | 151 ++++++++++++++++++ tests/ShouldReportConnectErrors/main.cpp | 93 +++++++++++ 6 files changed, 383 insertions(+), 51 deletions(-) create mode 100644 tests/ShouldConnectDualStack/main.cpp create mode 100644 tests/ShouldReportConnectErrors/main.cpp diff --git a/README.md b/README.md index a6e4c19..ca19e69 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,18 @@ client.Send("Hello World", 11); std::vector data = client.RecieveSync(); ``` +The host is resolved for IPv4 and IPv6 alike and every address it resolves to is tried in turn, as Python's `socket.create_connection` does, so `localhost` reaches a server that listens on `::1` only. The optional third argument is a connect timeout, applied to each address on its own. A failed resolve or connect throws `SocketErrorException`: still a `std::runtime_error` with the same message, but carrying a `kind` (`SocketErrorKind::System` for an errno such as `ECONNREFUSED`, `SocketErrorKind::Resolver` for a getaddrinfo result such as `EAI_NONAME`) and the `code`, so a caller can report the cause without parsing the text. + +```cpp +try { + Crafter::ClientTCP client("localhost", 5432, std::chrono::seconds(5)); +} catch (const Crafter::SocketErrorException& e) { + std::println("[Errno {}] {}", e.code, std::strerror(e.code)); // [Errno 111] Connection refused +} catch (const Crafter::SocketTimeoutException&) { + std::println("timed out"); +} +``` + #### ListenerTCP ```cpp // Create a TCP listener @@ -287,6 +299,8 @@ The library includes tests covering: - HTTPS round-trip (`ShouldSendRecieveHTTPS1`) — the plaintext round-trip replayed over TLS, so a transport regression surfaces as an HTTP failure, plus what only exists under TLS: ALPN, `scheme=https` reaching handlers, a body spanning many records, verification failing for an untrusted certificate *and* for a trusted certificate presented under the wrong name, and a plaintext peer on the TLS port being counted and shrugged off - HTTPS interop (`ShouldInteropCurlHTTPS1`) — `curl` verifying our certificate with `--cacert` (not `--insecure`): keep-alive reuse, POST, HEAD, negotiated version, and an h2-only client being refused rather than mis-served; then `ClientHTTP1` against python3's `http.server` behind `ssl.wrap_socket`, whose HTTP/1.0 `Connection: close` frames the body by `close_notify` - Mutual TLS and the raw stream (`ShouldRequireClientCertificateHTTPS1`) — a client certificate the listener's CA vouches for is served and one absent is refused; `TLSStream` is also driven directly with hand-written HTTP/1.1 to keep `:TLS` usable without `:ClientHTTP1` on top +- Dual-stack connect (`ShouldConnectDualStack`) — `localhost` reaches a server on `::1` only and one on `127.0.0.1` only, an IPv6 literal connects over IPv6, and the last failure is what is thrown when no address answers; skips on a host without IPv6 loopback +- Connect errors (`ShouldReportConnectErrors`) — a refused connect and an unresolvable name throw `SocketErrorException` with their errno or getaddrinfo code and the message ClientTCP always threw, and still reach a `std::runtime_error` handler The external-interop test requires outbound UDP/443; if your network blocks it the test will fail. `ShouldInteropCurlHTTP1` and `ShouldInteropCurlHTTPS1` skip whichever half is unavailable when `curl` or `python3` is not installed, and the mutual-TLS half of `ShouldRequireClientCertificateHTTPS1` skips without the `openssl` CLI (a *client* certificate needs the `clientAuth` extended key usage, which the built-in development certificate does not carry) — so all three pass on a bare machine. Install `curl`, `python3` and `openssl` to actually exercise them. diff --git a/implementations/Crafter.Network-ClientTCP.cpp b/implementations/Crafter.Network-ClientTCP.cpp index e4c791c..defeae6 100755 --- a/implementations/Crafter.Network-ClientTCP.cpp +++ b/implementations/Crafter.Network-ClientTCP.cpp @@ -43,6 +43,60 @@ namespace { if (errno == EAGAIN || errno == EWOULDBLOCK) throw SocketTimeoutException(); throw std::runtime_error(std::strerror(errno)); } + + // One connect attempt to one address: a connected, blocking socket, or a + // throw that leaves nothing open. + int ConnectTo(const sockaddr* address, socklen_t length, std::chrono::milliseconds timeout) { + const int fd = socket(address->sa_family, SOCK_STREAM, 0); + if (fd == -1) { + // An address family the host cannot open — IPv6 switched off, + // say — fails this address alone; the caller moves on. + const int failure = errno; + throw SocketErrorException(SocketErrorKind::System, failure, std::string("Could not open socket: ") + std::strerror(failure)); + } + + // With a timeout the connect runs non-blocking, so poll can bound the + // wait; the socket is made blocking again once it is connected. + const bool bounded = timeout > std::chrono::milliseconds::zero(); + const int flags = fcntl(fd, F_GETFL, 0); + if (bounded) { + fcntl(fd, F_SETFL, flags | O_NONBLOCK); + } + int failure = 0; + if (connect(fd, address, length) == -1) { + failure = errno; + if (failure == EINPROGRESS) { + pollfd connecting { .fd = fd, .events = POLLOUT, .revents = 0 }; + const int waitMs = static_cast(std::min(timeout.count(), std::numeric_limits::max())); + int ready; + do { + ready = poll(&connecting, 1, waitMs); + } while (ready == -1 && errno == EINTR); + if (ready == 0) { + close(fd); + throw SocketTimeoutException(); + } + if (ready == -1) { + failure = errno; + } else { + // The connect's own result. + socklen_t size = sizeof(failure); + getsockopt(fd, SOL_SOCKET, SO_ERROR, &failure, &size); + } + } + } + if (failure != 0) { + // Report the failure instead of handing back a socket that is not + // connected to anything — every later send/recv on it would fail + // with a far less obvious error. + close(fd); + throw SocketErrorException(SocketErrorKind::System, failure, std::string("Could not connect to server: ") + std::strerror(failure)); + } + if (bounded) { + fcntl(fd, F_SETFL, flags); + } + return fd; + } } ClientTCP::ClientTCP(int socketid) : socketid(socketid) @@ -55,17 +109,28 @@ ClientTCP::ClientTCP(const char* hostName, std::uint16_t port, std::chrono::mill // getaddrinfo rather than gethostbyname: the latter is not thread safe, // and it signals failure by returning null — which was then dereferenced // straight into a crash on any unresolvable host. + // + // AF_UNSPEC, and every result kept: `localhost` resolves to ::1 and + // 127.0.0.1, and a server may listen on either one alone. Asking for + // AF_INET only, as this used to, made a server bound to ::1 unreachable + // by name. No AI_ADDRCONFIG, as Python's create_connection passes none + // either: it hides ::1 on a host whose only IPv6 address is loopback. addrinfo hints{}; - hints.ai_family = AF_INET; + hints.ai_family = AF_UNSPEC; hints.ai_socktype = SOCK_STREAM; addrinfo* resolved = nullptr; const std::string service = std::to_string(port); const int status = getaddrinfo(hostName, service.c_str(), &hints, &resolved); if (status != 0 || resolved == nullptr) { - throw std::runtime_error(std::string("Could not resolve host '") + hostName + "': " - + gai_strerror(status)); + throw SocketErrorException(SocketErrorKind::Resolver, status, + std::string("Could not resolve host '") + hostName + "': " + gai_strerror(status)); + } + for (const addrinfo* entry = resolved; entry != nullptr; entry = entry->ai_next) { + Address address {}; + std::memcpy(&address.storage, entry->ai_addr, entry->ai_addrlen); + address.length = entry->ai_addrlen; + addresses.push_back(address); } - serv_addr = *reinterpret_cast(resolved->ai_addr); freeaddrinfo(resolved); Connect(connectTimeout); @@ -79,7 +144,7 @@ ClientTCP::ClientTCP(std::string hostName, std::uint16_t port, std::chrono::mill // The moved-from object gives up ownership; the socket itself stays open — // closing it here (as this used to) meant moving a ClientTCP silently // dropped the connection it was carrying. -ClientTCP::ClientTCP(ClientTCP&& other) noexcept : socketid(other.socketid) { +ClientTCP::ClientTCP(ClientTCP&& other) noexcept : socketid(other.socketid), addresses(std::move(other.addresses)) { other.socketid = -1; } @@ -92,52 +157,23 @@ ClientTCP::~ClientTCP() } void ClientTCP::Connect(std::chrono::milliseconds timeout) { - if((socketid = socket(AF_INET, SOCK_STREAM, 0)) == -1){ - throw std::runtime_error(std::string("Could not open socket: ") + std::strerror(errno)); - } - - // With a timeout the connect runs non-blocking, so poll can bound the - // wait; the socket is made blocking again once it is connected. - const bool bounded = timeout > std::chrono::milliseconds::zero(); - const int flags = fcntl(socketid, F_GETFL, 0); - if (bounded) { - fcntl(socketid, F_SETFL, flags | O_NONBLOCK); + if (addresses.empty()) { + throw std::logic_error("ClientTCP::Connect: no address to connect to; a socket handed over by accept() cannot connect again"); } - int failure = 0; - if(connect(socketid,(sockaddr*)&serv_addr, sizeof(sockaddr)) == -1){ - failure = errno; - if (failure == EINPROGRESS) { - pollfd connecting { .fd = socketid, .events = POLLOUT, .revents = 0 }; - const int waitMs = static_cast(std::min(timeout.count(), std::numeric_limits::max())); - int ready; - do { - ready = poll(&connecting, 1, waitMs); - } while (ready == -1 && errno == EINTR); - if (ready == 0) { - close(socketid); - socketid = -1; - throw SocketTimeoutException(); - } - if (ready == -1) { - failure = errno; - } else { - // The connect's own result. - socklen_t length = sizeof(failure); - getsockopt(socketid, SOL_SOCKET, SO_ERROR, &failure, &length); - } + // Each address in turn, in the order getaddrinfo ranked them; when none + // connects, the caller sees what the last one failed with, as with + // Python's create_connection. + std::exception_ptr last; + for (const Address& address : addresses) { + try { + socketid = ConnectTo(reinterpret_cast(&address.storage), address.length, timeout); + return; + } catch (...) { + last = std::current_exception(); } - } - if (failure != 0) { - // Report the failure instead of handing back a socket that is not - // connected to anything — every later send/recv on it would fail - // with a far less obvious error. - close(socketid); - socketid = -1; - throw std::runtime_error(std::string("Could not connect to server: ") + std::strerror(failure)); - } - if (bounded) { - fcntl(socketid, F_SETFL, flags); } + socketid = -1; + std::rethrow_exception(last); } void ClientTCP::Stop() { diff --git a/interfaces/Crafter.Network-ClientTCP.cppm b/interfaces/Crafter.Network-ClientTCP.cppm index 181c830..2481c4a 100755 --- a/interfaces/Crafter.Network-ClientTCP.cppm +++ b/interfaces/Crafter.Network-ClientTCP.cppm @@ -37,14 +37,44 @@ namespace Crafter { } }; + // Where the code of a SocketErrorException comes from. + export enum class SocketErrorKind { + // An errno: ECONNREFUSED, ENETUNREACH, EMFILE, ... + System, + // A getaddrinfo result: EAI_NONAME, EAI_AGAIN, ... + Resolver, + }; + + // A resolve, a socket() or a connect that failed, with the code it failed + // with. what() is the text ClientTCP has always thrown, and catching + // std::runtime_error still catches it; the code is for callers that + // report the cause in their own words — a Python-style + // "[Errno 111] Connection refused", say — and would otherwise have to + // parse it back out of the message. + export class SocketErrorException : public std::runtime_error { + public: + SocketErrorException(SocketErrorKind kind, std::int32_t code, const std::string& message) + : std::runtime_error(message), kind(kind), code(code) {} + SocketErrorKind kind; + std::int32_t code; + }; + export class ClientTCP { public: int socketid; ClientTCP(int socket); + // The host is resolved for IPv4 and IPv6 alike and every address it + // resolves to is tried in turn, the way Python's + // socket.create_connection does: `localhost` reaches a server that + // listens on ::1 only, as a dev server on a recent Node does. The + // first address that connects wins; when none does, what the last + // one failed with is thrown — SocketErrorException, or + // SocketTimeoutException for a timeout. + // // A non-zero connectTimeout makes a connect nobody answers throw // SocketTimeoutException after that long, where it would otherwise - // wait out the kernel's retries, about two minutes. A refused connect - // fails at once either way. + // wait out the kernel's retries, about two minutes. It bounds each + // address on its own. A refused connect fails at once either way. ClientTCP(const char* host, std::uint16_t port, std::chrono::milliseconds connectTimeout = std::chrono::milliseconds::zero()); ClientTCP(std::string host, std::uint16_t port, std::chrono::milliseconds connectTimeout = std::chrono::milliseconds::zero()); ~ClientTCP(); @@ -83,7 +113,13 @@ namespace Crafter { // Into `buffer`, which must outlive the receive; the callback gets the count. void RecieveAsync(std::uint32_t bufferSize, std::function recieveCallback, char* buffer, std::function onError) const; private: - sockaddr_in serv_addr; + struct Address { + sockaddr_storage storage; + socklen_t length; + }; + // What the host resolved to, in getaddrinfo's order; empty for a + // socket handed over by accept(). + std::vector
addresses; }; } #endif \ No newline at end of file diff --git a/project.cpp b/project.cpp index 4060c1d..66b1019 100644 --- a/project.cpp +++ b/project.cpp @@ -174,6 +174,8 @@ extern "C" Configuration CrafterBuildProject(std::span a if (cfg.target.find("linux") != std::string::npos) { cfg.AddTest("ShouldConfigureTCP").Dependencies({ &cfg }); cfg.AddTest("ShouldRecieveTCP").Dependencies({ &cfg }); + cfg.AddTest("ShouldConnectDualStack").Dependencies({ &cfg }); + cfg.AddTest("ShouldReportConnectErrors").Dependencies({ &cfg }); cfg.AddTest("ShouldInteropCurlHTTP1").Dependencies({ &cfg }); cfg.AddTest("ShouldInteropCurlHTTPS1").Dependencies({ &cfg }); cfg.AddTest("ShouldParseHTTP1").Dependencies({ &cfg }); diff --git a/tests/ShouldConnectDualStack/main.cpp b/tests/ShouldConnectDualStack/main.cpp new file mode 100644 index 0000000..17707f8 --- /dev/null +++ b/tests/ShouldConnectDualStack/main.cpp @@ -0,0 +1,151 @@ +//SPDX-License-Identifier: LGPL-3.0-only +//SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts® + +// ClientTCP resolves a host for IPv4 and IPv6 alike and tries every address +// the name resolves to, in order, the way Python's socket.create_connection +// does. Before, it asked getaddrinfo for AF_INET only and kept the first +// answer, so a server listening on ::1 alone — a dev server on a recent Node +// binds `localhost` that way — could not be reached by name, while every +// Python client reached it. +// +// The listeners are raw sockets, so nothing here depends on ListenerTCP: one +// IPv6-only on ::1, one on 127.0.0.1. Skips (77) on a host without IPv6 +// loopback. + +#include +#include +#include +#include +#include +#include +#include + +import Crafter.Network; +import std; +using namespace Crafter; +using namespace std::chrono_literals; + +namespace { + +int failures = 0; + +void Check(bool ok, std::string_view what) { + std::println("{} {}", ok ? "PASS" : "FAIL", what); + if (!ok) ++failures; +} + +// A socket listening on the loopback address of `family`, on a port the +// kernel picks; fd stays -1 when the host cannot open one. +struct LoopbackListener { + int fd = -1; + std::uint16_t port = 0; + + explicit LoopbackListener(int family) { + fd = socket(family, SOCK_STREAM, 0); + if (fd == -1) return; + sockaddr_storage address {}; + socklen_t length = 0; + if (family == AF_INET6) { + // v6 only, or the socket would also take IPv4 connects and the + // checks below would not tell the two apart. + int only = 1; + setsockopt(fd, IPPROTO_IPV6, IPV6_V6ONLY, &only, sizeof(only)); + auto* v6 = reinterpret_cast(&address); + v6->sin6_family = AF_INET6; + v6->sin6_addr = in6addr_loopback; + length = sizeof(sockaddr_in6); + } else { + auto* v4 = reinterpret_cast(&address); + v4->sin_family = AF_INET; + v4->sin_addr.s_addr = htonl(INADDR_LOOPBACK); + length = sizeof(sockaddr_in); + } + if (bind(fd, reinterpret_cast(&address), length) != 0 || listen(fd, 8) != 0) { + close(fd); + fd = -1; + return; + } + getsockname(fd, reinterpret_cast(&address), &length); + port = family == AF_INET6 ? ntohs(reinterpret_cast(&address)->sin6_port) + : ntohs(reinterpret_cast(&address)->sin_port); + } + ~LoopbackListener() { + if (fd != -1) close(fd); + } +}; + +// The address family of the peer a connected client reached. +int PeerFamily(const ClientTCP& client) { + sockaddr_storage address {}; + socklen_t length = sizeof(address); + getpeername(client.socketid, reinterpret_cast(&address), &length); + return address.ss_family; +} + +// Whether `localhost` resolves to ::1 here; /etc/hosts decides. +bool LocalhostResolvesToIPv6() { + addrinfo hints {}; + hints.ai_family = AF_INET6; + hints.ai_socktype = SOCK_STREAM; + addrinfo* resolved = nullptr; + if (getaddrinfo("localhost", nullptr, &hints, &resolved) != 0) return false; + freeaddrinfo(resolved); + return true; +} + +} // namespace + +int main() { + LoopbackListener v6(AF_INET6); + if (v6.fd == -1) { + std::println("SKIP no IPv6 loopback on this host"); + return 77; + } + + { + ClientTCP client("::1", v6.port, 2s); + Check(PeerFamily(client) == AF_INET6, "an IPv6 literal connects over IPv6"); + } + + if (LocalhostResolvesToIPv6()) { + try { + ClientTCP client("localhost", v6.port, 2s); + Check(PeerFamily(client) == AF_INET6, "localhost reaches a server that listens on ::1 only"); + } catch (const std::exception& e) { + Check(false, std::format("localhost reaches a server that listens on ::1 only (threw: {})", e.what())); + } + } else { + std::println("SKIP localhost does not resolve to ::1 on this host"); + } + + { + // Whichever address the resolver ranks first, a refused one is not + // the end: the next is tried. + LoopbackListener v4(AF_INET); + try { + ClientTCP client("localhost", v4.port, 2s); + Check(PeerFamily(client) == AF_INET, "localhost reaches a server that listens on 127.0.0.1 only"); + } catch (const std::exception& e) { + Check(false, std::format("localhost reaches a server that listens on 127.0.0.1 only (threw: {})", e.what())); + } + } + + { + std::uint16_t closedPort = 0; + { + LoopbackListener gone(AF_INET); + closedPort = gone.port; + } + bool refused = false; + try { + ClientTCP client("localhost", closedPort, 2s); + } catch (const SocketErrorException& e) { + refused = e.kind == SocketErrorKind::System && e.code == ECONNREFUSED; + } catch (const std::exception&) { + } + Check(refused, "when no address answers, the last failure is thrown, with its errno"); + } + + std::println("{}", failures == 0 ? "all checks passed" : "check(s) failed"); + return failures == 0 ? EXIT_SUCCESS : EXIT_FAILURE; +} diff --git a/tests/ShouldReportConnectErrors/main.cpp b/tests/ShouldReportConnectErrors/main.cpp new file mode 100644 index 0000000..c919907 --- /dev/null +++ b/tests/ShouldReportConnectErrors/main.cpp @@ -0,0 +1,93 @@ +//SPDX-License-Identifier: LGPL-3.0-only +//SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts® + +// A failed resolve or connect throws SocketErrorException, which carries the +// code it failed with: the errno for a connect, the getaddrinfo result for a +// name. Before, both were a bare std::runtime_error, so a caller that reports +// the cause in its own words — a Python-style "[Errno 111] Connection +// refused" in front of a language model reading the output — had to parse the +// message back apart. The message itself stays what it always was, and the +// exception is still a std::runtime_error, so existing handlers keep working. + +#include +#include +#include +#include +#include +#include +#include +#include + +import Crafter.Network; +import std; +using namespace Crafter; +using namespace std::chrono_literals; + +namespace { + +int failures = 0; + +void Check(bool ok, std::string_view what) { + std::println("{} {}", ok ? "PASS" : "FAIL", what); + if (!ok) ++failures; +} + +// A port on 127.0.0.1 nothing listens on: bound by the kernel, then released. +std::uint16_t ClosedPort() { + const int fd = socket(AF_INET, SOCK_STREAM, 0); + sockaddr_in address {}; + address.sin_family = AF_INET; + address.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + bind(fd, reinterpret_cast(&address), sizeof(address)); + socklen_t length = sizeof(address); + getsockname(fd, reinterpret_cast(&address), &length); + close(fd); + return ntohs(address.sin_port); +} + +} // namespace + +int main() { + { + bool thrown = false; + try { + ClientTCP client("127.0.0.1", ClosedPort(), 2s); + } catch (const SocketErrorException& e) { + thrown = true; + Check(e.kind == SocketErrorKind::System, "a refused connect is a system error"); + Check(e.code == ECONNREFUSED, std::format("it carries ECONNREFUSED (got {})", e.code)); + Check(std::string_view(e.what()) == std::string("Could not connect to server: ") + std::strerror(ECONNREFUSED), + std::format("its message is the one ClientTCP always threw (got '{}')", e.what())); + } + Check(thrown, "a refused connect throws SocketErrorException"); + } + + { + bool caught = false; + try { + ClientTCP client("127.0.0.1", ClosedPort(), 2s); + } catch (const std::runtime_error&) { + caught = true; + } + Check(caught, "it is still caught as std::runtime_error"); + } + + { + // .invalid never resolves (RFC 6761); without a network the resolver + // fails differently (EAI_AGAIN), but always as a resolver error. + bool thrown = false; + try { + ClientTCP client("no-such-host.invalid", 80, 2s); + } catch (const SocketErrorException& e) { + thrown = true; + Check(e.kind == SocketErrorKind::Resolver, "a name that does not resolve is a resolver error"); + Check(e.code != 0, std::format("it carries the getaddrinfo code (got {})", e.code)); + Check(std::string_view(e.what()).starts_with("Could not resolve host 'no-such-host.invalid': "), + std::format("its message names the host (got '{}')", e.what())); + } + Check(thrown, "an unresolvable host throws SocketErrorException"); + } + + std::println("{}", failures == 0 ? "all checks passed" : "check(s) failed"); + return failures == 0 ? EXIT_SUCCESS : EXIT_FAILURE; +} From d6f8f4edcb41eba5883c3029bc6446a126bd9fc7 Mon Sep 17 00:00:00 2001 From: Jorijn van der Graaf Date: Sat, 31 Oct 2026 05:06:41 +0100 Subject: [PATCH 2/6] feat(tcp): ListenerTCP binds the address it is given, at port 0 if asked A new constructor takes a numeric bind address (127.0.0.1, ::1, 0.0.0.0, ::) and accepts port 0, and Port() reports the port the listener got. Until now every listener bound 0.0.0.0 at a port the caller chose, so a helper meant for this machine's own processes was reachable from the network, and the caller had to guess a free port. A name such as `localhost` is refused instead of bound to whichever address the resolver ranks first. The port-only constructor still listens on every IPv4 interface. `running` is atomic now: Stop() runs on another thread than the accept loop that reads it. The accept loops take a sockaddr_storage, so an IPv6 peer's address is no longer cut short. New: ShouldBindLoopbackEphemeral. Co-Authored-By: Claude Opus 5.5 --- README.md | 10 ++ .../Crafter.Network-ListenerTCP.cpp | 53 +++++-- interfaces/Crafter.Network-ListenerTCP.cppm | 68 +++++---- project.cpp | 1 + tests/ShouldBindLoopbackEphemeral/main.cpp | 143 ++++++++++++++++++ 5 files changed, 235 insertions(+), 40 deletions(-) create mode 100644 tests/ShouldBindLoopbackEphemeral/main.cpp diff --git a/README.md b/README.md index ca19e69..b47bd56 100644 --- a/README.md +++ b/README.md @@ -75,6 +75,15 @@ Crafter::ListenerTCP listener(8080, callback); listener.ListenSyncSync(); // Synchronous listening ``` +The port-only constructor listens on every IPv4 interface. To keep a listener to this machine, or to let the kernel pick the port, name the address and pass port 0; `Port()` reports the port it got. The address must be numeric (`127.0.0.1`, `::1`, `0.0.0.0`, `::`): a name such as `localhost` is refused rather than bound to whichever address the resolver ranks first. + +```cpp +Crafter::ListenerTCP broker("127.0.0.1", 0, callback); +const std::uint16_t port = broker.Port(); +std::thread([&] { broker.ListenSyncSync(); }).detach(); +// ... broker.Stop() ends the loop from another thread. +``` + ### HTTP/3 Components HTTP/3 runs over QUIC, which always requires TLS. Pass server credentials when constructing the listener (or set `selfSigned = true` for a development-only ephemeral cert) and matching client credentials when constructing the client (`insecureNoServerValidation = true` for self-signed servers). @@ -301,6 +310,7 @@ The library includes tests covering: - Mutual TLS and the raw stream (`ShouldRequireClientCertificateHTTPS1`) — a client certificate the listener's CA vouches for is served and one absent is refused; `TLSStream` is also driven directly with hand-written HTTP/1.1 to keep `:TLS` usable without `:ClientHTTP1` on top - Dual-stack connect (`ShouldConnectDualStack`) — `localhost` reaches a server on `::1` only and one on `127.0.0.1` only, an IPv6 literal connects over IPv6, and the last failure is what is thrown when no address answers; skips on a host without IPv6 loopback - Connect errors (`ShouldReportConnectErrors`) — a refused connect and an unresolvable name throw `SocketErrorException` with their errno or getaddrinfo code and the message ClientTCP always threw, and still reach a `std::runtime_error` handler +- Loopback listeners (`ShouldBindLoopbackEphemeral`) — a listener on `127.0.0.1` at port 0 reports its port, takes loopback clients and refuses them through the host's other address; `::1` works too, a name is refused, and the port-only constructor still listens on every interface The external-interop test requires outbound UDP/443; if your network blocks it the test will fail. `ShouldInteropCurlHTTP1` and `ShouldInteropCurlHTTPS1` skip whichever half is unavailable when `curl` or `python3` is not installed, and the mutual-TLS half of `ShouldRequireClientCertificateHTTPS1` skips without the `openssl` CLI (a *client* certificate needs the `clientAuth` extended key usage, which the built-in development certificate does not carry) — so all three pass on a bare machine. Install `curl`, `python3` and `openssl` to actually exercise them. diff --git a/implementations/Crafter.Network-ListenerTCP.cpp b/implementations/Crafter.Network-ListenerTCP.cpp index 57e956f..29f92ed 100755 --- a/implementations/Crafter.Network-ListenerTCP.cpp +++ b/implementations/Crafter.Network-ListenerTCP.cpp @@ -12,6 +12,7 @@ module; #include #include #include +#include #include module Crafter.Network:ListenerTCP_impl; @@ -21,14 +22,30 @@ import Crafter.Thread; using namespace Crafter; -ListenerTCP::ListenerTCP(std::uint16_t port, std::function connectCallback, std::uint32_t concurrentClientLimit, std::uint32_t totalClientLimit) : connectCallback(connectCallback), concurrentClientLimit(concurrentClientLimit), totalClientLimit(totalClientLimit) { - sockaddr_in servAddr; - bzero((char*)&servAddr, sizeof(servAddr)); - servAddr.sin_family = AF_INET; - servAddr.sin_addr.s_addr = htonl(INADDR_ANY); - servAddr.sin_port = htons(port); +ListenerTCP::ListenerTCP(std::uint16_t port, std::function connectCallback, std::uint32_t concurrentClientLimit, std::uint32_t totalClientLimit) + : ListenerTCP("0.0.0.0", port, std::move(connectCallback), concurrentClientLimit, totalClientLimit) {} - s = socket(AF_INET, SOCK_STREAM, 0); +ListenerTCP::ListenerTCP(std::string_view bindAddress, std::uint16_t port, std::function connectCallback, std::uint32_t concurrentClientLimit, std::uint32_t totalClientLimit) : connectCallback(connectCallback), concurrentClientLimit(concurrentClientLimit), totalClientLimit(totalClientLimit) { + // AI_NUMERICHOST: no lookup, so `localhost` is refused here rather than + // bound to whichever of ::1 and 127.0.0.1 the resolver ranks first. + addrinfo hints{}; + hints.ai_family = AF_UNSPEC; + hints.ai_socktype = SOCK_STREAM; + hints.ai_flags = AI_PASSIVE | AI_NUMERICHOST | AI_NUMERICSERV; + const std::string host(bindAddress); + const std::string service = std::to_string(port); + addrinfo* resolved = nullptr; + const int status = getaddrinfo(host.c_str(), service.c_str(), &hints, &resolved); + if (status != 0 || resolved == nullptr) { + throw std::runtime_error("Could not bind " + host + ": not a numeric IPv4 or IPv6 address (" + gai_strerror(status) + ")"); + } + sockaddr_storage servAddr {}; + std::memcpy(&servAddr, resolved->ai_addr, resolved->ai_addrlen); + const socklen_t servAddrLength = resolved->ai_addrlen; + const int family = resolved->ai_family; + freeaddrinfo(resolved); + + s = socket(family, SOCK_STREAM, 0); if(s < 0) { throw std::runtime_error("Error establishing the server socket"); @@ -39,7 +56,7 @@ ListenerTCP::ListenerTCP(std::uint16_t port, std::function con int reuse = 1; setsockopt(s, SOL_SOCKET, SO_REUSEADDR, &reuse, sizeof(reuse)); - if(bind(s, (struct sockaddr*) &servAddr, sizeof(servAddr)) < 0) { + if(bind(s, reinterpret_cast(&servAddr), servAddrLength) < 0) { // Ignoring this left the listener silently accepting nothing at // all, with no hint as to why. const std::string reason = std::strerror(errno); @@ -55,6 +72,18 @@ ListenerTCP::ListenerTCP(std::uint16_t port, std::function con } } +std::uint16_t ListenerTCP::Port() const { + sockaddr_storage address {}; + socklen_t length = sizeof(address); + if (getsockname(s, reinterpret_cast(&address), &length) == -1) { + throw std::runtime_error(std::string("Could not read the listening port: ") + std::strerror(errno)); + } + if (address.ss_family == AF_INET6) { + return ntohs(reinterpret_cast(&address)->sin6_port); + } + return ntohs(reinterpret_cast(&address)->sin_port); +} + void ListenerTCP::Stop() { running = false; shutdown(s, SHUT_RDWR); @@ -64,7 +93,7 @@ void ListenerTCP::Stop() { void ListenerTCP::ListenSyncSync() { while (running && totalClientCounter < totalClientLimit) { - sockaddr_in newSockAddr; + sockaddr_storage newSockAddr; socklen_t newSockAddrSize = sizeof(newSockAddr); int client = accept(s, (sockaddr*)&newSockAddr, &newSockAddrSize); if (client > 0) { @@ -81,7 +110,7 @@ void ListenerTCP::ListenSyncSync() { void ListenerTCP::ListenSyncAsync() { while (running && totalClientCounter < totalClientLimit) { - sockaddr_in newSockAddr; + sockaddr_storage newSockAddr; socklen_t newSockAddrSize = sizeof(newSockAddr); int client = accept(s, (sockaddr*)&newSockAddr, &newSockAddrSize); if (client > 0) { @@ -97,7 +126,7 @@ void ListenerTCP::ListenSyncAsync() { void ListenerTCP::ListenAsyncSync() { ThreadPool::Enqueue([this]() { while (running && totalClientCounter < totalClientLimit) { - sockaddr_in newSockAddr; + sockaddr_storage newSockAddr; socklen_t newSockAddrSize = sizeof(newSockAddr); int client = accept(s, (sockaddr*)&newSockAddr, &newSockAddrSize); if (client > 0) { @@ -114,7 +143,7 @@ void ListenerTCP::ListenAsyncSync() { void ListenerTCP::ListenAsyncAsync() { ThreadPool::Enqueue([this]() { while (running && totalClientCounter < totalClientLimit) { - sockaddr_in newSockAddr; + sockaddr_storage newSockAddr; socklen_t newSockAddrSize = sizeof(newSockAddr); int client = accept(s, (sockaddr*)&newSockAddr, &newSockAddrSize); if (client > 0) { diff --git a/interfaces/Crafter.Network-ListenerTCP.cppm b/interfaces/Crafter.Network-ListenerTCP.cppm index 7795b67..d41253e 100755 --- a/interfaces/Crafter.Network-ListenerTCP.cppm +++ b/interfaces/Crafter.Network-ListenerTCP.cppm @@ -1,28 +1,40 @@ -//SPDX-License-Identifier: LGPL-3.0-only -//SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts® - -export module Crafter.Network:ListenerTCP; -import std; -import :ClientTCP; - -#ifndef CRAFTER_NETWORK_BROWSER -namespace Crafter { - export class ListenerTCP { - public: - bool running = true; - std::function connectCallback; - std::uint32_t concurrentClientLimit; - std::uint32_t totalClientLimit; - ListenerTCP(std::uint16_t port, std::function connectCallback, std::uint32_t concurrentClientLimit = std::numeric_limits::max(), std::uint32_t totalClientLimit = std::numeric_limits::max()); - ~ListenerTCP(); - void Stop(); - void ListenSyncSync(); - void ListenSyncAsync(); - void ListenAsyncSync(); - void ListenAsyncAsync(); - private: - std::uint32_t totalClientCounter = 0; - int s; - }; -} -#endif \ No newline at end of file +//SPDX-License-Identifier: LGPL-3.0-only +//SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts® + +export module Crafter.Network:ListenerTCP; +import std; +import :ClientTCP; + +#ifndef CRAFTER_NETWORK_BROWSER +namespace Crafter { + export class ListenerTCP { + public: + // Atomic: Stop() is called from another thread than the one running + // an accept loop, which reads this on every turn. + std::atomic running = true; + std::function connectCallback; + std::uint32_t concurrentClientLimit; + std::uint32_t totalClientLimit; + // Every IPv4 interface (0.0.0.0) at `port`. + ListenerTCP(std::uint16_t port, std::function connectCallback, std::uint32_t concurrentClientLimit = std::numeric_limits::max(), std::uint32_t totalClientLimit = std::numeric_limits::max()); + // `bindAddress` is a numeric IPv4 or IPv6 address — 127.0.0.1 or ::1 + // for this machine alone, 0.0.0.0 or :: for every interface — and a + // name such as `localhost` is refused: the address decides who can + // reach the listener, so it is spelled out rather than looked up. A + // `port` of 0 lets the kernel pick a free one; Port() says which. + ListenerTCP(std::string_view bindAddress, std::uint16_t port, std::function connectCallback, std::uint32_t concurrentClientLimit = std::numeric_limits::max(), std::uint32_t totalClientLimit = std::numeric_limits::max()); + ~ListenerTCP(); + // The port the listener is bound to: the one asked for, or the + // kernel's pick when that was 0. + std::uint16_t Port() const; + void Stop(); + void ListenSyncSync(); + void ListenSyncAsync(); + void ListenAsyncSync(); + void ListenAsyncAsync(); + private: + std::uint32_t totalClientCounter = 0; + int s; + }; +} +#endif diff --git a/project.cpp b/project.cpp index 66b1019..b54f9c7 100644 --- a/project.cpp +++ b/project.cpp @@ -176,6 +176,7 @@ extern "C" Configuration CrafterBuildProject(std::span a cfg.AddTest("ShouldRecieveTCP").Dependencies({ &cfg }); cfg.AddTest("ShouldConnectDualStack").Dependencies({ &cfg }); cfg.AddTest("ShouldReportConnectErrors").Dependencies({ &cfg }); + cfg.AddTest("ShouldBindLoopbackEphemeral").Dependencies({ &cfg }); cfg.AddTest("ShouldInteropCurlHTTP1").Dependencies({ &cfg }); cfg.AddTest("ShouldInteropCurlHTTPS1").Dependencies({ &cfg }); cfg.AddTest("ShouldParseHTTP1").Dependencies({ &cfg }); diff --git a/tests/ShouldBindLoopbackEphemeral/main.cpp b/tests/ShouldBindLoopbackEphemeral/main.cpp new file mode 100644 index 0000000..a897e22 --- /dev/null +++ b/tests/ShouldBindLoopbackEphemeral/main.cpp @@ -0,0 +1,143 @@ +//SPDX-License-Identifier: LGPL-3.0-only +//SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts® + +// ListenerTCP can be told the address it binds and take port 0, with Port() +// reporting what the kernel picked. Before, it always bound 0.0.0.0 at a port +// the caller chose: a helper meant for this machine's own processes — a broker +// that hands out database access, say — was reachable from the network, and +// the caller had to guess a free port and hope. +// +// Also: a host name is refused rather than bound to whichever address the +// resolver ranks first, and the port-only constructor keeps listening on every +// IPv4 interface. + +#include +#include +#include +#include +#include +#include +#include + +import Crafter.Network; +import std; +using namespace Crafter; +using namespace std::chrono_literals; + +namespace { + +int failures = 0; + +void Check(bool ok, std::string_view what) { + std::println("{} {}", ok ? "PASS" : "FAIL", what); + if (!ok) ++failures; +} + +// An IPv4 address of this host that is not loopback, if it has one. +std::optional NonLoopbackIPv4() { + ifaddrs* interfaces = nullptr; + if (getifaddrs(&interfaces) != 0) return std::nullopt; + std::optional found; + for (const ifaddrs* entry = interfaces; entry != nullptr && !found; entry = entry->ifa_next) { + if (entry->ifa_addr == nullptr || entry->ifa_addr->sa_family != AF_INET) continue; + const auto* v4 = reinterpret_cast(entry->ifa_addr); + if (ntohl(v4->sin_addr.s_addr) >> 24 == 127) continue; + char text[INET_ADDRSTRLEN] = {}; + inet_ntop(AF_INET, &v4->sin_addr, text, sizeof(text)); + found = std::string(text); + } + freeifaddrs(interfaces); + return found; +} + +// Runs a listener's accept loop on a thread of its own and counts the +// connections it was handed. +struct Serving { + ListenerTCP& listener; + std::atomic accepted = 0; + std::thread loop; + + explicit Serving(ListenerTCP& listener) : listener(listener) { + listener.connectCallback = [this](ClientTCP* client) { + ++accepted; + delete client; + }; + loop = std::thread([this] { this->listener.ListenSyncSync(); }); + } + ~Serving() { + listener.Stop(); + loop.join(); + } + bool WaitFor(int count) { + for (int i = 0; i < 200 && accepted.load() < count; ++i) std::this_thread::sleep_for(10ms); + return accepted.load() >= count; + } +}; + +} // namespace + +int main() { + { + ListenerTCP listener("127.0.0.1", 0, nullptr); + const std::uint16_t port = listener.Port(); + Check(port != 0, std::format("port 0 gets a port from the kernel, which Port() reports ({})", port)); + Serving serving(listener); + { + ClientTCP client("127.0.0.1", port, 2s); + } + Check(serving.WaitFor(1), "a client on loopback reaches it"); + + if (const std::optional outside = NonLoopbackIPv4()) { + bool refused = false; + try { + ClientTCP client(*outside, port, 2s); + } catch (const SocketErrorException& e) { + refused = e.code == ECONNREFUSED; + } catch (const std::exception&) { + } + Check(refused, std::format("through {} it is refused: it listens on 127.0.0.1 alone", *outside)); + } else { + std::println("SKIP no non-loopback IPv4 address to try from"); + } + } + + { + std::optional v6; + try { + v6.emplace("::1", 0, nullptr); + } catch (const std::exception&) { + } + if (v6) { + Serving serving(*v6); + { + ClientTCP client("::1", v6->Port(), 2s); + } + Check(serving.WaitFor(1), "an IPv6 loopback listener takes a client on ::1"); + } else { + std::println("SKIP no IPv6 loopback on this host"); + } + } + + { + bool refused = false; + try { + ListenerTCP named("localhost", 0, nullptr); + } catch (const std::runtime_error&) { + refused = true; + } + Check(refused, "a host name is refused: the address to bind is spelled out"); + } + + { + ListenerTCP any(0, nullptr); + Check(any.Port() != 0, "the port-only constructor takes port 0 as well"); + Serving serving(any); + { + ClientTCP client("127.0.0.1", any.Port(), 2s); + } + Check(serving.WaitFor(1), "and still listens on every IPv4 interface, loopback included"); + } + + std::println("{}", failures == 0 ? "all checks passed" : "check(s) failed"); + return failures == 0 ? EXIT_SUCCESS : EXIT_FAILURE; +} From f9a45c143ec43842f6aa51b3a0349f51b28470e2 Mon Sep 17 00:00:00 2001 From: Jorijn van der Graaf Date: Sat, 31 Oct 2026 05:06:41 +0100 Subject: [PATCH 3/6] feat(http1): bound the connect, and FetchHTTP1 for a URL with redirects - ClientHTTP1 gains connectTimeout (default 15 s), applied to each address the host resolves to. `timeout` only ever bounded the waits for a response, so a host that drops the SYNs held Send() for the kernel's retries, about two minutes, whatever the caller had asked for. - The default Host header puts an IPv6 literal in brackets. - FetchHTTP1(url, options) makes one request to an http:// or https:// URL and follows redirects as Python's urllib does, so urllib code ported over keeps its behaviour: 301/302/303/307/308 for GET and HEAD, 301/302/303 for POST (re-sent as a GET without its body), a relative Location resolved as urljoin resolves it and percent-encoded as urllib quotes it, http and https hops mixed, each redirect's body read before the next request, and the chain stopped by urllib's limits (4 repeats of one URL, 10 different URLs) with the last redirect returned and `limited` set. Proxies are not consulted. New: ShouldTimeoutSilentHTTP1, ShouldFollowRedirectsHTTP1. Co-Authored-By: Claude Opus 5.5 --- README.md | 21 +- .../Crafter.Network-ClientHTTP1.cpp | 314 +++++++++++++++++- interfaces/Crafter.Network-ClientHTTP1.cppm | 56 ++++ project.cpp | 2 + tests/ShouldFollowRedirectsHTTP1/main.cpp | 153 +++++++++ tests/ShouldTimeoutSilentHTTP1/main.cpp | 109 ++++++ 6 files changed, 649 insertions(+), 6 deletions(-) create mode 100644 tests/ShouldFollowRedirectsHTTP1/main.cpp create mode 100644 tests/ShouldTimeoutSilentHTTP1/main.cpp diff --git a/README.md b/README.md index b47bd56..fb69171 100644 --- a/README.md +++ b/README.md @@ -129,7 +129,24 @@ Crafter::HTTPResponse response = client.Send( ); ``` -The connection is persistent: the first `Send()` dials, and later calls reuse the socket unless the peer asked for it to be closed. A pooled connection that the peer closed while it looked idle — the race HTTP/1.1 keep-alive cannot avoid — is redialled once and the request replayed; nothing is replayed once a response byte has arrived, and a freshly dialled connection is never retried on, so a genuinely broken server surfaces as an exception rather than a retry loop. `authority` defaults to the host:port the client was constructed with (the port is elided when it is the scheme default — 443 under TLS, 80 without). +The connection is persistent: the first `Send()` dials, and later calls reuse the socket unless the peer asked for it to be closed. A pooled connection that the peer closed while it looked idle — the race HTTP/1.1 keep-alive cannot avoid — is redialled once and the request replayed; nothing is replayed once a response byte has arrived, and a freshly dialled connection is never retried on, so a genuinely broken server surfaces as an exception rather than a retry loop. `authority` defaults to the host:port the client was constructed with (the port is elided when it is the scheme default — 443 under TLS, 80 without; an IPv6 literal goes in brackets). + +Three timeouts bound a request: `connectTimeout` (default 15 s) for the TCP connect, per address the host resolves to; `handshakeTimeout` (default 15 s) for the TLS handshake; and `timeout` (default 30 s) for each wait on the next piece of a response. + +#### FetchHTTP1 + +`FetchHTTP1(url, options)` is one request to an `http://` or `https://` URL, following redirects the way Python's urllib does, so code ported from urllib keeps its behaviour: + +```cpp +Crafter::FetchOptionsHTTP1 options; +options.timeout = std::chrono::seconds(10); +options.headers["accept-encoding"] = "identity"; +Crafter::FetchResultHTTP1 result = Crafter::FetchHTTP1("http://localhost:8080/", options); +// result.response: the last response; result.url: the URL that sent it; +// result.redirects: how many were followed; result.limited: the limits stopped the chain. +``` + +301, 302, 303, 307 and 308 are followed for a GET or HEAD, and 301/302/303 also for a POST; the redirected request is a GET (a HEAD stays a HEAD) without the body or its content headers. A relative `Location` is resolved against the URL that sent it, as `urljoin` resolves it, and is percent-encoded the way urllib quotes it; a redirect may move between `http://` and `https://` (`options.credentials` applies to every TLS hop). Each redirect's body is read before the next request goes out. A redirect is returned unfollowed when it has no `Location`, points at another scheme, or comes for a method urllib does not redirect. The chain stops, returning the redirect with `limited` set, at the fifth redirect to one URL (`maxRepeats`, 4) or once ten different URLs have been visited (`maxRedirects`, 10). Proxies are not consulted. A URL that is not `http(s)://`, has no host or a bad port throws `std::invalid_argument`. #### ListenerHTTP1 ```cpp @@ -311,6 +328,8 @@ The library includes tests covering: - Dual-stack connect (`ShouldConnectDualStack`) — `localhost` reaches a server on `::1` only and one on `127.0.0.1` only, an IPv6 literal connects over IPv6, and the last failure is what is thrown when no address answers; skips on a host without IPv6 loopback - Connect errors (`ShouldReportConnectErrors`) — a refused connect and an unresolvable name throw `SocketErrorException` with their errno or getaddrinfo code and the message ClientTCP always threw, and still reach a `std::runtime_error` handler - Loopback listeners (`ShouldBindLoopbackEphemeral`) — a listener on `127.0.0.1` at port 0 reports its port, takes loopback clients and refuses them through the host's other address; `::1` works too, a name is refused, and the port-only constructor still listens on every interface +- HTTP/1.1 timeouts (`ShouldTimeoutSilentHTTP1`) — a server that accepts and never answers fails after `timeout`, and a connect nobody answers after `connectTimeout` +- Redirects (`ShouldFollowRedirectsHTTP1`) — relative and `..` Locations, an http-to-https hop, the 4-repeat and 10-URL limits, a redirect without Location, POST after 303 and 307, percent-encoding a Location, and the URLs `FetchHTTP1` refuses The external-interop test requires outbound UDP/443; if your network blocks it the test will fail. `ShouldInteropCurlHTTP1` and `ShouldInteropCurlHTTPS1` skip whichever half is unavailable when `curl` or `python3` is not installed, and the mutual-TLS half of `ShouldRequireClientCertificateHTTPS1` skips without the `openssl` CLI (a *client* certificate needs the `clientAuth` extended key usage, which the built-in development certificate does not carry) — so all three pass on a bare machine. Install `curl`, `python3` and `openssl` to actually exercise them. diff --git a/implementations/Crafter.Network-ClientHTTP1.cpp b/implementations/Crafter.Network-ClientHTTP1.cpp index 134f83d..5e43633 100644 --- a/implementations/Crafter.Network-ClientHTTP1.cpp +++ b/implementations/Crafter.Network-ClientHTTP1.cpp @@ -21,8 +21,10 @@ namespace { // which is what every other HTTP/1.1 client on the wire does and what // virtual-host matching on the far side tends to expect. std::string DefaultAuthority(const std::string& host, std::uint16_t port, bool secure) { - if (port == (secure ? 443 : 80)) return host; - return host + ":" + std::to_string(port); + // An IPv6 literal goes in brackets, or its colons read as the port's. + const std::string name = host.contains(':') ? "[" + host + "]" : host; + if (port == (secure ? 443 : 80)) return name; + return name + ":" + std::to_string(port); } } @@ -37,8 +39,8 @@ struct ClientHTTP1::Impl { std::unique_ptr stream; std::string protocol; - void Connect(std::chrono::milliseconds handshakeTimeout) { - tcp = std::make_unique(host, port); + void Connect(std::chrono::milliseconds connectTimeout, std::chrono::milliseconds handshakeTimeout) { + tcp = std::make_unique(host, port, connectTimeout); try { if (tls) { stream = TLSStream::Connect(tcp->socketid, tls, host, handshakeTimeout); @@ -146,7 +148,7 @@ HTTPResponse ClientHTTP1::Send(const HTTPRequest& request) { // read fails. Nothing is replayed once a response byte has arrived. for (int attempt = 0;; ++attempt) { const bool reused = impl->tcp != nullptr; - if (!reused) impl->Connect(handshakeTimeout); + if (!reused) impl->Connect(connectTimeout, handshakeTimeout); bool received = false; try { @@ -176,3 +178,305 @@ void ClientHTTP1::SendAsync(const HTTPRequest& request, } }); } + +namespace { + // The five parts urllib.parse.urlsplit cuts a URL into. FetchHTTP1 needs + // the splitting, the joining (urljoin) and the reassembly to resolve a + // Location the way urllib does, and nothing else of urllib.parse. + struct SplitURL { + std::string scheme; + std::string netloc; + std::string path; + std::string query; + std::string fragment; + }; + + bool IsAsciiAlpha(char c) { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z'); + } + + bool IsSchemeChar(char c) { + return IsAsciiAlpha(c) || (c >= '0' && c <= '9') || c == '+' || c == '-' || c == '.'; + } + + std::string ToLowerAsciiCopy(std::string_view text) { + std::string lower(text); + for (char& c : lower) { + if (c >= 'A' && c <= 'Z') c = static_cast(c - 'A' + 'a'); + } + return lower; + } + + // str.split('/'): every piece, empty ones included, and [""] for "". + std::vector SplitSlashes(std::string_view text) { + std::vector pieces; + std::size_t start = 0; + while (true) { + const std::size_t slash = text.find('/', start); + if (slash == std::string_view::npos) { + pieces.emplace_back(text.substr(start)); + return pieces; + } + pieces.emplace_back(text.substr(start, slash - start)); + start = slash + 1; + } + } + + // urlsplit: tabs and newlines dropped anywhere, leading controls and + // spaces stripped, the scheme lowercased. + SplitURL Split(std::string_view input) { + std::string url; + url.reserve(input.size()); + for (const char c : input) { + if (c != '\t' && c != '\r' && c != '\n') url.push_back(c); + } + std::size_t start = 0; + while (start < url.size() && static_cast(url[start]) <= 0x20) ++start; + url.erase(0, start); + + SplitURL parts; + const std::size_t colon = url.find(':'); + if (colon != std::string::npos && colon > 0 && IsAsciiAlpha(url[0]) + && std::ranges::all_of(std::string_view(url).substr(0, colon), IsSchemeChar)) { + parts.scheme = ToLowerAsciiCopy(std::string_view(url).substr(0, colon)); + url.erase(0, colon + 1); + } + if (url.starts_with("//")) { + std::size_t end = url.find_first_of("/?#", 2); + if (end == std::string::npos) end = url.size(); + parts.netloc = url.substr(2, end - 2); + url.erase(0, end); + if (parts.netloc.contains('[') != parts.netloc.contains(']')) { + throw std::invalid_argument("FetchHTTP1: unbalanced brackets in the host of " + std::string(input)); + } + } + if (const std::size_t hash = url.find('#'); hash != std::string::npos) { + parts.fragment = url.substr(hash + 1); + url.erase(hash); + } + if (const std::size_t question = url.find('?'); question != std::string::npos) { + parts.query = url.substr(question + 1); + url.erase(question); + } + parts.path = std::move(url); + return parts; + } + + // urlunsplit, for the http(s) URLs FetchHTTP1 puts back together. + std::string Unsplit(const SplitURL& parts) { + std::string url = parts.path; + if (!parts.netloc.empty()) { + if (!url.empty() && url.front() != '/') url.insert(url.begin(), '/'); + url = "//" + parts.netloc + url; + } else if (url.starts_with("//")) { + url = "//" + url; + } else if ((parts.scheme == "http" || parts.scheme == "https") && (url.empty() || url.front() == '/')) { + url = "//" + url; + } + if (!parts.scheme.empty()) url = parts.scheme + ":" + url; + if (!parts.query.empty()) url += "?" + parts.query; + if (!parts.fragment.empty()) url += "#" + parts.fragment; + return url; + } + + // urljoin, for a reference found in a redirect: RFC 3986 resolution the + // way Python does it, a `..` that would climb above the root dropped. + std::string Join(std::string_view base, std::string_view reference) { + if (base.empty()) return std::string(reference); + if (reference.empty()) return std::string(base); + const SplitURL from = Split(base); + SplitURL to = Split(reference); + if (to.scheme.empty()) to.scheme = from.scheme; + const bool relative = to.scheme == "http" || to.scheme == "https"; + if (to.scheme != from.scheme || !relative) return std::string(reference); + if (!to.netloc.empty()) return Unsplit(to); + to.netloc = from.netloc; + if (to.path.empty()) { + to.path = from.path; + if (to.query.empty()) to.query = from.query; + return Unsplit(to); + } + + std::vector baseParts = SplitSlashes(from.path); + // The base's last segment is a file, not a directory: it does not + // take part in resolving. + if (!baseParts.back().empty()) baseParts.pop_back(); + std::vector segments; + if (to.path.starts_with('/')) { + segments = SplitSlashes(to.path); + } else { + segments = std::move(baseParts); + std::vector more = SplitSlashes(to.path); + segments.insert(segments.end(), more.begin(), more.end()); + // Empty segments between the first and the last would come back + // as doubled slashes. + if (segments.size() > 2) { + std::vector kept { segments.front() }; + for (std::size_t i = 1; i + 1 < segments.size(); ++i) { + if (!segments[i].empty()) kept.push_back(segments[i]); + } + kept.push_back(segments.back()); + segments = std::move(kept); + } + } + std::vector resolved; + for (const std::string& segment : segments) { + if (segment == "..") { + if (!resolved.empty()) resolved.pop_back(); + } else if (segment != ".") { + resolved.push_back(segment); + } + } + // A reference ending in a directory step keeps its trailing slash. + if (segments.back() == "." || segments.back() == "..") resolved.emplace_back(); + std::string path; + for (std::size_t i = 0; i < resolved.size(); ++i) { + if (i > 0) path += '/'; + path += resolved[i]; + } + to.path = path.empty() ? std::string("/") : path; + return Unsplit(to); + } + + // urllib quotes a Location before it follows it, keeping every ASCII + // letter, digit and punctuation mark: what it escapes are spaces, + // controls and bytes outside ASCII. + std::string QuoteLocation(std::string_view location) { + std::string quoted; + for (const char c : location) { + const unsigned char byte = static_cast(c); + if (byte > 0x20 && byte < 0x7f) { + quoted.push_back(c); + } else { + quoted += std::format("%{:02X}", byte); + } + } + return quoted; + } + + struct FetchTarget { + bool secure = false; + // Without brackets: what the socket and TLS verification are given. + std::string host; + std::uint16_t port = 0; + // The URL's host[:port] as written, for the Host header. + std::string authority; + // Path and query: the request target. + std::string target; + }; + + FetchTarget ParseTarget(std::string_view url) { + const SplitURL parts = Split(url); + if (parts.scheme != "http" && parts.scheme != "https") { + throw std::invalid_argument("FetchHTTP1: not an http:// or https:// URL: " + std::string(url)); + } + FetchTarget target; + target.secure = parts.scheme == "https"; + std::string hostinfo = parts.netloc; + if (const std::size_t at = hostinfo.rfind('@'); at != std::string::npos) hostinfo.erase(0, at + 1); + std::string port; + if (hostinfo.starts_with('[')) { + const std::size_t close = hostinfo.find(']'); + target.host = hostinfo.substr(1, close - 1); + const std::string rest = hostinfo.substr(close + 1); + if (rest.starts_with(':')) { + port = rest.substr(1); + } else if (!rest.empty()) { + throw std::invalid_argument("FetchHTTP1: unexpected text after the IPv6 host in " + std::string(url)); + } + } else { + const std::size_t colon = hostinfo.find(':'); + target.host = hostinfo.substr(0, colon); + if (colon != std::string::npos) port = hostinfo.substr(colon + 1); + } + if (target.host.empty()) { + throw std::invalid_argument("FetchHTTP1: no host in " + std::string(url)); + } + if (port.empty()) { + target.port = target.secure ? 443 : 80; + } else { + std::uint32_t value = 0; + const auto [end, error] = std::from_chars(port.data(), port.data() + port.size(), value); + if (error != std::errc() || end != port.data() + port.size() || value > 65535) { + throw std::invalid_argument("FetchHTTP1: bad port '" + port + "' in " + std::string(url)); + } + target.port = static_cast(value); + } + target.authority = hostinfo; + target.target = parts.path.empty() ? std::string("/") : parts.path; + if (!target.target.starts_with('/')) target.target.insert(target.target.begin(), '/'); + if (!parts.query.empty()) target.target += "?" + parts.query; + return target; + } + + std::int32_t StatusCode(const HTTPResponse& response) { + std::int32_t code = 0; + std::from_chars(response.status.data(), response.status.data() + response.status.size(), code); + return code; + } +} + +FetchResultHTTP1 Crafter::FetchHTTP1(std::string_view url, const FetchOptionsHTTP1& options) { + std::string current(url); + std::string method = options.method.empty() ? std::string("GET") : options.method; + std::unordered_map headers = options.headers; + std::string body = options.body; + // How often the chain has been sent to each URL: urllib's loop check. + std::map visited; + FetchResultHTTP1 result; + while (true) { + const FetchTarget target = ParseTarget(current); + ClientHTTP1 client = target.secure ? ClientHTTP1(target.host, target.port, options.credentials) + : ClientHTTP1(target.host, target.port); + client.connectTimeout = options.connectTimeout; + client.handshakeTimeout = options.handshakeTimeout; + client.timeout = options.timeout; + client.limits = options.limits; + + HTTPRequest request; + request.method = method; + request.scheme = target.secure ? "https" : "http"; + request.authority = target.authority; + request.path = target.target; + request.headers = headers; + request.body = body; + result.response = client.Send(request); + result.url = current; + + const std::int32_t code = StatusCode(result.response); + const bool redirect = code == 301 || code == 302 || code == 303 || code == 307 || code == 308; + if (!redirect) return result; + // The methods urllib redirects; any other comes back as the redirect. + const bool followed = method == "GET" || method == "HEAD" || (method == "POST" && code <= 303); + if (!followed) return result; + auto location = result.response.headers.find("location"); + if (location == result.response.headers.end()) location = result.response.headers.find("uri"); + if (location == result.response.headers.end()) return result; + + std::string next = QuoteLocation(location->second); + SplitURL parts = Split(next); + if (!parts.scheme.empty() && parts.scheme != "http" && parts.scheme != "https") return result; + if (parts.path.empty() && !parts.netloc.empty()) { + parts.path = "/"; + next = Unsplit(parts); + } + next = Join(current, next); + + const auto seen = visited.find(next); + const std::uint32_t times = seen == visited.end() ? 0 : seen->second; + if (times >= options.maxRepeats || visited.size() >= options.maxRedirects) { + result.limited = true; + return result; + } + visited[next] = times + 1; + ++result.redirects; + + // What urllib sends next: a GET, or a HEAD for a HEAD, without the + // body or the headers that described it. + if (method != "HEAD") method = "GET"; + body.clear(); + headers.erase("content-length"); + headers.erase("content-type"); + current = std::move(next); + } +} diff --git a/interfaces/Crafter.Network-ClientHTTP1.cppm b/interfaces/Crafter.Network-ClientHTTP1.cppm index 87ff2c5..b5cb7b8 100644 --- a/interfaces/Crafter.Network-ClientHTTP1.cppm +++ b/interfaces/Crafter.Network-ClientHTTP1.cppm @@ -87,10 +87,66 @@ namespace Crafter { // from `timeout` because it covers a multi-round-trip exchange before // any request has been written. std::chrono::milliseconds handshakeTimeout{15000}; + // How long the TCP connect may take, for each address the host + // resolves to (see ClientTCP). Without it a host that drops the SYNs + // held Send() for the kernel's retries, about two minutes, whatever + // `timeout` said. + std::chrono::milliseconds connectTimeout{15000}; private: struct Impl; std::unique_ptr impl; }; + + // What FetchHTTP1 needs beyond the URL. + export struct FetchOptionsHTTP1 { + std::string method = "GET"; + // Lowercase names, as on HTTPRequest. Host comes from the URL. + std::unordered_map headers; + std::string body; + // As on ClientHTTP1, for every request of the chain. + std::chrono::milliseconds connectTimeout{15000}; + std::chrono::milliseconds handshakeTimeout{15000}; + std::chrono::milliseconds timeout{30000}; + HTTP1::MessageLimits limits; + // For every https:// request of the chain. + TLSClientCredentials credentials; + // Python urllib's limits: a chain stops at a redirect to a URL it has + // already been sent to `maxRepeats` times, or once it has been sent + // to `maxRedirects` different URLs. + std::uint32_t maxRedirects = 10; + std::uint32_t maxRepeats = 4; + }; + + export struct FetchResultHTTP1 { + // The last response of the chain. A redirect only when it had nothing + // to follow — no Location, a method urllib does not redirect, a + // scheme other than http(s) — or when the limits stopped the chain. + HTTPResponse response; + // The URL that answered with `response`. + std::string url; + // Redirects followed to get there. + std::uint32_t redirects = 0; + // The limits stopped the chain; `response` is the redirect that would + // have gone past them. + bool limited = false; + }; + + // One request to an http:// or https:// URL, following redirects the way + // Python's urllib does, so a port of urllib code keeps its behaviour: + // 301, 302, 303, 307 and 308 are followed for GET and HEAD, the first + // three also for POST; the redirected request is a GET (a HEAD stays a + // HEAD) without the body and its content headers. A relative Location is + // resolved against the URL that sent it, as urljoin resolves it; a + // redirect may switch between http:// and https://. Each redirect's body + // is read before the next request goes out. Proxies are not consulted. + // + // The URL is `scheme://[user@]host[:port][/path][?query][#fragment]`, + // with an IPv6 host in brackets; the fragment is not sent and the user + // part is dropped. Throws std::invalid_argument for a URL that is not + // http(s) or has no host or a bad port, and whatever a request throws + // (SocketErrorException, SocketTimeoutException, TLSException, + // std::runtime_error) otherwise. + export FetchResultHTTP1 FetchHTTP1(std::string_view url, const FetchOptionsHTTP1& options = {}); } #endif diff --git a/project.cpp b/project.cpp index b54f9c7..b4b2d29 100644 --- a/project.cpp +++ b/project.cpp @@ -177,6 +177,8 @@ extern "C" Configuration CrafterBuildProject(std::span a cfg.AddTest("ShouldConnectDualStack").Dependencies({ &cfg }); cfg.AddTest("ShouldReportConnectErrors").Dependencies({ &cfg }); cfg.AddTest("ShouldBindLoopbackEphemeral").Dependencies({ &cfg }); + cfg.AddTest("ShouldTimeoutSilentHTTP1").Dependencies({ &cfg }); + cfg.AddTest("ShouldFollowRedirectsHTTP1").Dependencies({ &cfg }); cfg.AddTest("ShouldInteropCurlHTTP1").Dependencies({ &cfg }); cfg.AddTest("ShouldInteropCurlHTTPS1").Dependencies({ &cfg }); cfg.AddTest("ShouldParseHTTP1").Dependencies({ &cfg }); diff --git a/tests/ShouldFollowRedirectsHTTP1/main.cpp b/tests/ShouldFollowRedirectsHTTP1/main.cpp new file mode 100644 index 0000000..c99d718 --- /dev/null +++ b/tests/ShouldFollowRedirectsHTTP1/main.cpp @@ -0,0 +1,153 @@ +//SPDX-License-Identifier: LGPL-3.0-only +//SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts® + +// FetchHTTP1 follows redirects the way Python's urllib does, so code ported +// from urllib keeps its behaviour: +// - a relative Location is resolved against the URL that sent it (urljoin, +// `..` included), and a redirect may move from http:// to https://; +// - 301/302/303/307/308 are followed for a GET; a POST follows 301/302/303 +// as a GET without its body, and comes back with a 307 or 308 unfollowed; +// - a redirect without a Location comes back as the response; +// - the chain stops at the 5th redirect to one URL (4 repeats) or after 10 +// different URLs, returning that redirect with `limited` set; +// - a Location with a space is percent-encoded, as urllib quotes it; +// - a URL that is not http(s), has no host or has a bad port is refused. +// +// Two listeners: plaintext on 8130, TLS on 8131 with a self-signed certificate +// the client is handed as its trust anchor. + +#include + +import Crafter.Network; +import std; +using namespace Crafter; + +namespace { + +int failures = 0; + +void Check(bool ok, std::string_view what) { + std::println("{} {}", ok ? "PASS" : "FAIL", what); + if (!ok) ++failures; +} + +HTTPResponse Redirect(std::string status, std::string location) { + return CreateResponseHTTP(std::move(status), {{"location", std::move(location)}}, std::string()); +} + +std::unordered_map> Routes() { + std::unordered_map> routes; + routes["/start"] = [](const HTTPRequest&) { return Redirect("302", "next"); }; + routes["/next"] = [](const HTTPRequest&) { return Redirect("301", "/final?x=1"); }; + routes["/a/b/start"] = [](const HTTPRequest&) { return Redirect("302", "../c/d"); }; + routes["/to-https"] = [](const HTTPRequest&) { return Redirect("307", "https://localhost:8131/secure"); }; + routes["/loop"] = [](const HTTPRequest&) { return Redirect("302", "/loop"); }; + routes["/nowhere"] = [](const HTTPRequest&) { return CreateResponseHTTP("302", std::string("no location")); }; + routes["/form"] = [](const HTTPRequest&) { return Redirect("303", "/method"); }; + routes["/keep"] = [](const HTTPRequest&) { return Redirect("307", "/method"); }; + routes["/method"] = [](const HTTPRequest& request) { + return CreateResponseHTTP("200", request.method + " body=" + request.body); + }; + routes["/spaced"] = [](const HTTPRequest&) { return Redirect("302", "/with space"); }; + return routes; +} + +// Everything else echoes its target, and /hop/ redirects to /hop/. +HTTPResponse Fallback(const HTTPRequest& request) { + if (request.path.starts_with("/hop/")) { + const std::int32_t next = std::stoi(request.path.substr(5)) + 1; + return Redirect("302", std::format("/hop/{}", next)); + } + return CreateResponseHTTP("200", request.path); +} + +bool Refused(std::string_view url) { + try { + FetchHTTP1(url); + } catch (const std::invalid_argument&) { + return true; + } catch (const std::exception&) { + } + return false; +} + +} // namespace + +int main() { + try { + ListenerAsyncHTTP1 plain(8130, Routes(), Fallback); + std::unordered_map> secureRoutes; + secureRoutes["/secure"] = [](const HTTPRequest& request) { + return CreateResponseHTTP("200", "secure over " + request.scheme); + }; + ListenerAsyncHTTP1 secure(8131, secureRoutes, TLSServerCredentials { .selfSigned = true }); + + { + const FetchResultHTTP1 result = FetchHTTP1("http://127.0.0.1:8130/start"); + Check(result.response.status == "200" && result.response.body == "/final?x=1", + std::format("a relative chain arrives at /final?x=1 (got {} '{}')", result.response.status, result.response.body)); + Check(result.url == "http://127.0.0.1:8130/final?x=1", std::format("the URL that answered is reported ({})", result.url)); + Check(result.redirects == 2 && !result.limited, std::format("two redirects, not limited (got {})", result.redirects)); + } + + { + const FetchResultHTTP1 result = FetchHTTP1("http://127.0.0.1:8130/a/b/start"); + Check(result.response.body == "/a/c/d", std::format("../c/d from /a/b/start resolves to /a/c/d (got '{}')", result.response.body)); + } + + { + FetchOptionsHTTP1 options; + options.credentials.caPem = GetSelfSignedCertificatePem().certificate; + const FetchResultHTTP1 result = FetchHTTP1("http://127.0.0.1:8130/to-https", options); + Check(result.response.status == "200" && result.response.body == "secure over https", + std::format("a redirect from http:// to https:// is followed (got {} '{}')", result.response.status, result.response.body)); + Check(result.url == "https://localhost:8131/secure", std::format("and lands on the https:// URL ({})", result.url)); + } + + { + const FetchResultHTTP1 result = FetchHTTP1("http://127.0.0.1:8130/loop"); + Check(result.limited && result.response.status == "302" && result.redirects == 4, + std::format("a redirect loop stops after 4 repeats (limited {}, {} redirects)", result.limited, result.redirects)); + } + + { + const FetchResultHTTP1 result = FetchHTTP1("http://127.0.0.1:8130/hop/0"); + Check(result.limited && result.redirects == 10 && result.url == "http://127.0.0.1:8130/hop/10", + std::format("an endless chain stops after 10 different URLs ({} redirects, at {})", result.redirects, result.url)); + } + + { + const FetchResultHTTP1 result = FetchHTTP1("http://127.0.0.1:8130/nowhere"); + Check(result.response.status == "302" && result.redirects == 0 && !result.limited, + "a redirect without a Location comes back as the response"); + } + + { + FetchOptionsHTTP1 post; + post.method = "POST"; + post.body = "payload"; + post.headers["content-type"] = "text/plain"; + const FetchResultHTTP1 seeOther = FetchHTTP1("http://127.0.0.1:8130/form", post); + Check(seeOther.response.body == "GET body=", + std::format("a POST answered with 303 is followed as a GET without its body (got '{}')", seeOther.response.body)); + const FetchResultHTTP1 kept = FetchHTTP1("http://127.0.0.1:8130/keep", post); + Check(kept.response.status == "307" && kept.redirects == 0, "a POST answered with 307 comes back unfollowed"); + } + + { + const FetchResultHTTP1 result = FetchHTTP1("http://127.0.0.1:8130/spaced"); + Check(result.response.body == "/with%20space", + std::format("a Location with a space is percent-encoded before it is followed (got '{}')", result.response.body)); + } + + Check(Refused("ftp://127.0.0.1/file"), "an ftp:// URL is refused"); + Check(Refused("http:///path"), "a URL without a host is refused"); + Check(Refused("http://127.0.0.1:99999/"), "a port above 65535 is refused"); + Check(Refused("http://127.0.0.1:8x/"), "a port that is not a number is refused"); + } catch (const std::exception& e) { + Check(false, std::format("unexpected exception: {}", e.what())); + } + + std::println("{}", failures == 0 ? "all checks passed" : "check(s) failed"); + return failures == 0 ? EXIT_SUCCESS : EXIT_FAILURE; +} diff --git a/tests/ShouldTimeoutSilentHTTP1/main.cpp b/tests/ShouldTimeoutSilentHTTP1/main.cpp new file mode 100644 index 0000000..2643dd5 --- /dev/null +++ b/tests/ShouldTimeoutSilentHTTP1/main.cpp @@ -0,0 +1,109 @@ +//SPDX-License-Identifier: LGPL-3.0-only +//SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts® + +// ClientHTTP1 bounds the TCP connect (connectTimeout) as well as every wait +// for a response (timeout). Before, the connect had no bound of its own: a +// host that drops the SYNs held Send() for the kernel's retries, about two +// minutes, whatever `timeout` said — a readiness probe with a ten-second +// budget could hang its caller for two minutes. +// +// Two servers on loopback. One accepts — the kernel completes the handshake +// from its backlog — and never answers: the read timeout. The other has a +// full accept queue, so Linux drops the SYNs of further connects: the connect +// timeout. + +#include +#include +#include +#include +#include + +import Crafter.Network; +import std; +using namespace Crafter; +using namespace std::chrono_literals; + +namespace { + +int failures = 0; + +void Check(bool ok, std::string_view what) { + std::println("{} {}", ok ? "PASS" : "FAIL", what); + if (!ok) ++failures; +} + +// A socket listening on 127.0.0.1, on a port the kernel picks. +struct LoopbackListener { + int fd = -1; + std::uint16_t port = 0; + + explicit LoopbackListener(int backlog) { + fd = socket(AF_INET, SOCK_STREAM, 0); + sockaddr_in address {}; + address.sin_family = AF_INET; + address.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + bind(fd, reinterpret_cast(&address), sizeof(address)); + listen(fd, backlog); + socklen_t length = sizeof(address); + getsockname(fd, reinterpret_cast(&address), &length); + port = ntohs(address.sin_port); + } + ~LoopbackListener() { close(fd); } +}; + +long long Milliseconds(std::chrono::steady_clock::duration duration) { + return std::chrono::duration_cast(duration).count(); +} + +} // namespace + +int main() { + { + LoopbackListener silent(8); + ClientHTTP1 client("127.0.0.1", silent.port); + client.timeout = 300ms; + const auto start = std::chrono::steady_clock::now(); + bool failed = false; + try { + client.Send(CreateRequestHTTP("GET", "/", "127.0.0.1")); + } catch (const std::exception&) { + failed = true; + } + const auto waited = std::chrono::steady_clock::now() - start; + Check(failed, "a server that accepts and never answers fails the request"); + Check(waited >= 280ms && waited < 5s, std::format("after about `timeout` ({} ms)", Milliseconds(waited))); + } + + { + // Connects queue up until the accept queue is full, which with a + // backlog of 0 is after one or two; the next one goes unanswered. + LoopbackListener full(0); + std::vector queued; + bool queueFull = false; + for (int i = 0; i < 8 && !queueFull; ++i) { + try { + queued.emplace_back("127.0.0.1", full.port, 200ms); + } catch (const SocketTimeoutException&) { + queueFull = true; + } + } + Check(queueFull, std::format("the accept queue fills up (after {} connects)", queued.size())); + + ClientHTTP1 client("127.0.0.1", full.port); + client.connectTimeout = 300ms; + const auto start = std::chrono::steady_clock::now(); + bool timedOut = false; + try { + client.Send(CreateRequestHTTP("GET", "/", "127.0.0.1")); + } catch (const SocketTimeoutException&) { + timedOut = true; + } catch (const std::exception&) { + } + const auto waited = std::chrono::steady_clock::now() - start; + Check(timedOut, "a connect nobody answers throws SocketTimeoutException"); + Check(waited >= 280ms && waited < 5s, std::format("after about `connectTimeout` ({} ms)", Milliseconds(waited))); + } + + std::println("{}", failures == 0 ? "all checks passed" : "check(s) failed"); + return failures == 0 ? EXIT_SUCCESS : EXIT_FAILURE; +} From 16da0a197973f61f005da2deab1a43056130cfe5 Mon Sep 17 00:00:00 2001 From: Jorijn van der Graaf Date: Sat, 31 Oct 2026 07:06:24 +0100 Subject: [PATCH 4/6] feat(tcp): a failed send or receive says which errno it failed with Send() and the receives threw a bare std::runtime_error with the strerror text, so a caller that reports the cause the way Python does - "the connection broke ([Errno 104] Connection reset by peer)", which a language model reads in Crafter.Review's SQL helper output - could not tell the code. They throw SocketErrorException now, as a failed connect already does: kind System, the errno, and the same message, still a std::runtime_error. A receive timeout stays SocketTimeoutException and a closed peer SocketClosedException. ShouldReportIoErrors: a peer that resets the connection (SO_LINGER 0) fails the next receive with ECONNRESET and a send into it with EPIPE or ECONNRESET. Co-Authored-By: Claude Opus 5.5 --- README.md | 3 +- implementations/Crafter.Network-ClientTCP.cpp | 12 +- interfaces/Crafter.Network-ClientTCP.cppm | 4 +- project.cpp | 1 + tests/ShouldReportIoErrors/main.cpp | 111 ++++++++++++++++++ 5 files changed, 124 insertions(+), 7 deletions(-) create mode 100644 tests/ShouldReportIoErrors/main.cpp diff --git a/README.md b/README.md index fb69171..67f747b 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,7 @@ client.Send("Hello World", 11); std::vector data = client.RecieveSync(); ``` -The host is resolved for IPv4 and IPv6 alike and every address it resolves to is tried in turn, as Python's `socket.create_connection` does, so `localhost` reaches a server that listens on `::1` only. The optional third argument is a connect timeout, applied to each address on its own. A failed resolve or connect throws `SocketErrorException`: still a `std::runtime_error` with the same message, but carrying a `kind` (`SocketErrorKind::System` for an errno such as `ECONNREFUSED`, `SocketErrorKind::Resolver` for a getaddrinfo result such as `EAI_NONAME`) and the `code`, so a caller can report the cause without parsing the text. +The host is resolved for IPv4 and IPv6 alike and every address it resolves to is tried in turn, as Python's `socket.create_connection` does, so `localhost` reaches a server that listens on `::1` only. The optional third argument is a connect timeout, applied to each address on its own. A failed resolve, connect, send or receive throws `SocketErrorException`: still a `std::runtime_error` with the same message, but carrying a `kind` (`SocketErrorKind::System` for an errno such as `ECONNREFUSED`, `SocketErrorKind::Resolver` for a getaddrinfo result such as `EAI_NONAME`) and the `code`, so a caller can report the cause without parsing the text. ```cpp try { @@ -327,6 +327,7 @@ The library includes tests covering: - Mutual TLS and the raw stream (`ShouldRequireClientCertificateHTTPS1`) — a client certificate the listener's CA vouches for is served and one absent is refused; `TLSStream` is also driven directly with hand-written HTTP/1.1 to keep `:TLS` usable without `:ClientHTTP1` on top - Dual-stack connect (`ShouldConnectDualStack`) — `localhost` reaches a server on `::1` only and one on `127.0.0.1` only, an IPv6 literal connects over IPv6, and the last failure is what is thrown when no address answers; skips on a host without IPv6 loopback - Connect errors (`ShouldReportConnectErrors`) — a refused connect and an unresolvable name throw `SocketErrorException` with their errno or getaddrinfo code and the message ClientTCP always threw, and still reach a `std::runtime_error` handler +- Send and receive errors (`ShouldReportIoErrors`) — a connection the peer resets fails the next receive, and a send into it, with `SocketErrorException` carrying ECONNRESET or EPIPE under the plain strerror text a receive and a send always threw - Loopback listeners (`ShouldBindLoopbackEphemeral`) — a listener on `127.0.0.1` at port 0 reports its port, takes loopback clients and refuses them through the host's other address; `::1` works too, a name is refused, and the port-only constructor still listens on every interface - HTTP/1.1 timeouts (`ShouldTimeoutSilentHTTP1`) — a server that accepts and never answers fails after `timeout`, and a connect nobody answers after `connectTimeout` - Redirects (`ShouldFollowRedirectsHTTP1`) — relative and `..` Locations, an http-to-https hop, the 4-repeat and 10-URL limits, a redirect without Location, POST after 303 and 307, percent-encoding a Location, and the URLs `FetchHTTP1` refuses diff --git a/implementations/Crafter.Network-ClientTCP.cpp b/implementations/Crafter.Network-ClientTCP.cpp index defeae6..ddab4b3 100755 --- a/implementations/Crafter.Network-ClientTCP.cpp +++ b/implementations/Crafter.Network-ClientTCP.cpp @@ -38,10 +38,13 @@ namespace { return read; } - // A failed recv. EAGAIN is the receive timeout (SO_RCVTIMEO) running out. + // A failed recv. EAGAIN is the receive timeout (SO_RCVTIMEO) running out; + // anything else carries its errno, under the message a receive always + // threw. [[noreturn]] void ThrowRecieveError() { - if (errno == EAGAIN || errno == EWOULDBLOCK) throw SocketTimeoutException(); - throw std::runtime_error(std::strerror(errno)); + const int failure = errno; + if (failure == EAGAIN || failure == EWOULDBLOCK) throw SocketTimeoutException(); + throw SocketErrorException(SocketErrorKind::System, failure, std::strerror(failure)); } // One connect attempt to one address: a connected, blocking socket, or a @@ -211,7 +214,8 @@ void ClientTCP::Send(const void* buffer, std::uint32_t size) const { throw SocketClosedException(); } else if (status < 0) { if (errno == EINTR) continue; - throw std::runtime_error(std::strerror(errno)); + const int failure = errno; + throw SocketErrorException(SocketErrorKind::System, failure, std::strerror(failure)); } sent += static_cast(status); } diff --git a/interfaces/Crafter.Network-ClientTCP.cppm b/interfaces/Crafter.Network-ClientTCP.cppm index 2481c4a..389d7c8 100755 --- a/interfaces/Crafter.Network-ClientTCP.cppm +++ b/interfaces/Crafter.Network-ClientTCP.cppm @@ -45,8 +45,8 @@ namespace Crafter { Resolver, }; - // A resolve, a socket() or a connect that failed, with the code it failed - // with. what() is the text ClientTCP has always thrown, and catching + // A resolve, a socket(), a connect, a send or a receive that failed, with + // the code it failed with. what() is the text ClientTCP has always thrown, and catching // std::runtime_error still catches it; the code is for callers that // report the cause in their own words — a Python-style // "[Errno 111] Connection refused", say — and would otherwise have to diff --git a/project.cpp b/project.cpp index b4b2d29..1030b81 100644 --- a/project.cpp +++ b/project.cpp @@ -176,6 +176,7 @@ extern "C" Configuration CrafterBuildProject(std::span a cfg.AddTest("ShouldRecieveTCP").Dependencies({ &cfg }); cfg.AddTest("ShouldConnectDualStack").Dependencies({ &cfg }); cfg.AddTest("ShouldReportConnectErrors").Dependencies({ &cfg }); + cfg.AddTest("ShouldReportIoErrors").Dependencies({ &cfg }); cfg.AddTest("ShouldBindLoopbackEphemeral").Dependencies({ &cfg }); cfg.AddTest("ShouldTimeoutSilentHTTP1").Dependencies({ &cfg }); cfg.AddTest("ShouldFollowRedirectsHTTP1").Dependencies({ &cfg }); diff --git a/tests/ShouldReportIoErrors/main.cpp b/tests/ShouldReportIoErrors/main.cpp new file mode 100644 index 0000000..52d5004 --- /dev/null +++ b/tests/ShouldReportIoErrors/main.cpp @@ -0,0 +1,111 @@ +//SPDX-License-Identifier: LGPL-3.0-only +//SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts® + +// A send or a receive that fails throws SocketErrorException with its errno, +// as a failed connect does: a caller that reports the cause in its own words +// — "the connection broke ([Errno 104] Connection reset by peer)" — needs the +// code, and the message alone is the strerror text a send and a receive +// always threw. A receive timeout is still SocketTimeoutException and a +// clean close SocketClosedException. + +#include +#include +#include +#include +#include +#include + +import Crafter.Network; +import std; +using namespace Crafter; +using namespace std::chrono_literals; + +namespace { + +int failures = 0; + +void Check(bool ok, std::string_view what) { + std::println("{} {}", ok ? "PASS" : "FAIL", what); + if (!ok) ++failures; +} + +// A listener on 127.0.0.1 that accepts one connection and resets it: SO_LINGER +// with a zero timeout makes close() send an RST instead of a FIN. +struct Resetter { + int listener = -1; + std::uint16_t port = 0; + std::thread thread; + + Resetter() { + listener = socket(AF_INET, SOCK_STREAM, 0); + sockaddr_in address {}; + address.sin_family = AF_INET; + address.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + bind(listener, reinterpret_cast(&address), sizeof(address)); + listen(listener, 1); + socklen_t length = sizeof(address); + getsockname(listener, reinterpret_cast(&address), &length); + port = ntohs(address.sin_port); + thread = std::thread([this] { + const int client = accept(listener, nullptr, nullptr); + linger reset { .l_onoff = 1, .l_linger = 0 }; + setsockopt(client, SOL_SOCKET, SO_LINGER, &reset, sizeof(reset)); + close(client); + }); + } + + ~Resetter() { + if (thread.joinable()) thread.join(); + close(listener); + } +}; + +} // namespace + +int main() { + { + Resetter peer; + ClientTCP client("127.0.0.1", peer.port, 2s); + client.SetReceiveTimeout(2s); + peer.thread.join(); + bool thrown = false; + try { + client.RecieveSync(16); + } catch (const SocketErrorException& e) { + thrown = true; + Check(e.kind == SocketErrorKind::System, "a reset connection fails the receive as a system error"); + Check(e.code == ECONNRESET, std::format("it carries ECONNRESET (got {})", e.code)); + Check(std::string_view(e.what()) == std::strerror(ECONNRESET), std::format("its message is the strerror text (got '{}')", e.what())); + } catch (const std::exception& e) { + Check(false, std::format("the receive threw something else: {}", e.what())); + } + Check(thrown, "a receive on a reset connection throws SocketErrorException"); + } + + { + Resetter peer; + ClientTCP client("127.0.0.1", peer.port, 2s); + peer.thread.join(); + // The first send after the reset may still be accepted locally; the + // kernel answers the next one with EPIPE or ECONNRESET. + std::int32_t code = 0; + bool runtime = false; + const std::string data(4096, 'x'); + for (int i = 0; i < 50 && code == 0; ++i) { + try { + client.Send(data.data(), static_cast(data.size())); + } catch (const SocketErrorException& e) { + code = e.code; + runtime = dynamic_cast(&e) != nullptr; + } catch (const std::exception&) { + break; + } + std::this_thread::sleep_for(10ms); + } + Check(code == EPIPE || code == ECONNRESET, std::format("a send into a reset connection throws SocketErrorException with EPIPE or ECONNRESET (got {})", code)); + Check(runtime, "and it is still a std::runtime_error"); + } + + std::println("{}", failures == 0 ? "all passed" : std::format("{} failed", failures)); + return failures == 0 ? 0 : 1; +} From e86c3b70a0751bb9d2ed2ae463ecbdfe5a7cd4e6 Mon Sep 17 00:00:00 2001 From: Jorijn van der Graaf Date: Sat, 31 Oct 2026 20:33:37 +0100 Subject: [PATCH 5/6] feat(tcp): socket errors as std::system_error, with the phase that failed SocketErrorException is now a std::system_error: code() carries the errno under std::generic_category(), or getaddrinfo's result under the new ResolverCategory(), whose message() is gai_strerror's text; `phase` says which step failed (resolve, open, connect, send, receive). It is still a std::runtime_error with the message ClientTCP always threw, the code's text after it, so existing handlers keep working. Before, the exception carried a `kind` and an int, and a caller that had to print the cause its own way still had to match the message prefix to learn where the failure happened. The standard type ends both: the category knows how to print the number, and the phase is a field. A caller writes "[Errno {}] {}" from code().value() and code().message(), or compares code() with std::errc::connection_refused. Breaking: SocketErrorException's `kind` and `code` members are gone; read code().category() and code().value(). --- AGENTS.consumer.md | 14 +++--- AGENTS.template.md | 5 +- README.md | 5 +- implementations/Crafter.Network-ClientTCP.cpp | 29 ++++++++--- interfaces/Crafter.Network-ClientTCP.cppm | 50 ++++++++++++------- tests/ShouldBindLoopbackEphemeral/main.cpp | 2 +- tests/ShouldConnectDualStack/main.cpp | 2 +- tests/ShouldReportConnectErrors/main.cpp | 41 +++++++++------ tests/ShouldReportIoErrors/main.cpp | 22 ++++---- 9 files changed, 107 insertions(+), 63 deletions(-) diff --git a/AGENTS.consumer.md b/AGENTS.consumer.md index 376a689..0e70b41 100644 --- a/AGENTS.consumer.md +++ b/AGENTS.consumer.md @@ -41,7 +41,7 @@ A route map is `std::unordered_map: Address already in use` on the HTTP/1.1 side and a `ListenerStart failed` `QUICException` on the QUIC side; find the stray process rather than changing the port. - **A warning is printed only when its unit actually recompiles**; a build with nothing to do is silent and says nothing about warnings. -- **`claude/tcp-only` carries four unmerged TCP and HTTP/1.1 commits** (IPv4 and IPv6 connects with the reason a connect failed, which retires the IPv4-only gotcha for the client; a bind address and port 0 for `ListenerTCP`; a bounded connect and `FetchHTTP1` following redirects; errno in the TCP errors), and `bin/` and `build/` keep its six test directories until `clean`. diff --git a/README.md b/README.md index 67f747b..274f1c1 100644 --- a/README.md +++ b/README.md @@ -53,13 +53,14 @@ client.Send("Hello World", 11); std::vector data = client.RecieveSync(); ``` -The host is resolved for IPv4 and IPv6 alike and every address it resolves to is tried in turn, as Python's `socket.create_connection` does, so `localhost` reaches a server that listens on `::1` only. The optional third argument is a connect timeout, applied to each address on its own. A failed resolve, connect, send or receive throws `SocketErrorException`: still a `std::runtime_error` with the same message, but carrying a `kind` (`SocketErrorKind::System` for an errno such as `ECONNREFUSED`, `SocketErrorKind::Resolver` for a getaddrinfo result such as `EAI_NONAME`) and the `code`, so a caller can report the cause without parsing the text. +The host is resolved for IPv4 and IPv6 alike and every address it resolves to is tried in turn, as Python's `socket.create_connection` does, so `localhost` reaches a server that listens on `::1` only. The optional third argument is a connect timeout, applied to each address on its own. A failed resolve, connect, send or receive throws `SocketErrorException`: a `std::system_error`, so still a `std::runtime_error` with the same message, whose `code()` is the errno under `std::generic_category()` or the `getaddrinfo` result under `Crafter::ResolverCategory()`, and whose `phase` names the step (`SocketPhase::Resolve`, `Open`, `Connect`, `Send` or `Receive`), so a caller can report the cause without parsing the text. ```cpp try { Crafter::ClientTCP client("localhost", 5432, std::chrono::seconds(5)); } catch (const Crafter::SocketErrorException& e) { - std::println("[Errno {}] {}", e.code, std::strerror(e.code)); // [Errno 111] Connection refused + if (e.code() == std::errc::connection_refused) std::println("nothing listens there"); + std::println("[Errno {}] {}", e.code().value(), e.code().message()); // [Errno 111] Connection refused } catch (const Crafter::SocketTimeoutException&) { std::println("timed out"); } diff --git a/implementations/Crafter.Network-ClientTCP.cpp b/implementations/Crafter.Network-ClientTCP.cpp index ddab4b3..6f5aee6 100755 --- a/implementations/Crafter.Network-ClientTCP.cpp +++ b/implementations/Crafter.Network-ClientTCP.cpp @@ -44,9 +44,21 @@ namespace { [[noreturn]] void ThrowRecieveError() { const int failure = errno; if (failure == EAGAIN || failure == EWOULDBLOCK) throw SocketTimeoutException(); - throw SocketErrorException(SocketErrorKind::System, failure, std::strerror(failure)); + throw SocketErrorException(SocketPhase::Receive, std::error_code(failure, std::generic_category())); } + // getaddrinfo's results as an error category: the number is one of + // netdb.h's EAI_* codes and the text is what gai_strerror makes of it. + class ResolverErrorCategory : public std::error_category { + public: + const char* name() const noexcept override { + return "resolver"; + } + std::string message(int code) const override { + return gai_strerror(code); + } + }; + // One connect attempt to one address: a connected, blocking socket, or a // throw that leaves nothing open. int ConnectTo(const sockaddr* address, socklen_t length, std::chrono::milliseconds timeout) { @@ -55,7 +67,7 @@ namespace { // An address family the host cannot open — IPv6 switched off, // say — fails this address alone; the caller moves on. const int failure = errno; - throw SocketErrorException(SocketErrorKind::System, failure, std::string("Could not open socket: ") + std::strerror(failure)); + throw SocketErrorException(SocketPhase::Open, std::error_code(failure, std::generic_category()), "Could not open socket"); } // With a timeout the connect runs non-blocking, so poll can bound the @@ -93,7 +105,7 @@ namespace { // connected to anything — every later send/recv on it would fail // with a far less obvious error. close(fd); - throw SocketErrorException(SocketErrorKind::System, failure, std::string("Could not connect to server: ") + std::strerror(failure)); + throw SocketErrorException(SocketPhase::Connect, std::error_code(failure, std::generic_category()), "Could not connect to server"); } if (bounded) { fcntl(fd, F_SETFL, flags); @@ -102,6 +114,11 @@ namespace { } } +const std::error_category& Crafter::ResolverCategory() noexcept { + static const ResolverErrorCategory category; + return category; +} + ClientTCP::ClientTCP(int socketid) : socketid(socketid) { @@ -125,8 +142,8 @@ ClientTCP::ClientTCP(const char* hostName, std::uint16_t port, std::chrono::mill const std::string service = std::to_string(port); const int status = getaddrinfo(hostName, service.c_str(), &hints, &resolved); if (status != 0 || resolved == nullptr) { - throw SocketErrorException(SocketErrorKind::Resolver, status, - std::string("Could not resolve host '") + hostName + "': " + gai_strerror(status)); + throw SocketErrorException(SocketPhase::Resolve, std::error_code(status, ResolverCategory()), + std::string("Could not resolve host '") + hostName + "'"); } for (const addrinfo* entry = resolved; entry != nullptr; entry = entry->ai_next) { Address address {}; @@ -215,7 +232,7 @@ void ClientTCP::Send(const void* buffer, std::uint32_t size) const { } else if (status < 0) { if (errno == EINTR) continue; const int failure = errno; - throw SocketErrorException(SocketErrorKind::System, failure, std::strerror(failure)); + throw SocketErrorException(SocketPhase::Send, std::error_code(failure, std::generic_category())); } sent += static_cast(status); } diff --git a/interfaces/Crafter.Network-ClientTCP.cppm b/interfaces/Crafter.Network-ClientTCP.cppm index 389d7c8..3a527e7 100755 --- a/interfaces/Crafter.Network-ClientTCP.cppm +++ b/interfaces/Crafter.Network-ClientTCP.cppm @@ -37,26 +37,42 @@ namespace Crafter { } }; - // Where the code of a SocketErrorException comes from. - export enum class SocketErrorKind { - // An errno: ECONNREFUSED, ENETUNREACH, EMFILE, ... - System, - // A getaddrinfo result: EAI_NONAME, EAI_AGAIN, ... - Resolver, + // The step of a connection a SocketErrorException failed at. + export enum class SocketPhase { + // getaddrinfo on the host name. + Resolve, + // socket() for an address the host resolved to. + Open, + // connect() to that address. + Connect, + Send, + Receive, }; - // A resolve, a socket(), a connect, a send or a receive that failed, with - // the code it failed with. what() is the text ClientTCP has always thrown, and catching - // std::runtime_error still catches it; the code is for callers that - // report the cause in their own words — a Python-style - // "[Errno 111] Connection refused", say — and would otherwise have to - // parse it back out of the message. - export class SocketErrorException : public std::runtime_error { + // The error category of getaddrinfo's results (EAI_NONAME, EAI_AGAIN, + // ...): a number space of its own that overlaps the errno values, so a + // resolver code is never confused with an errno, and whose message() is + // gai_strerror's text. One instance for the process, like + // std::generic_category(). + export const std::error_category& ResolverCategory() noexcept; + + // A resolve, a socket(), a connect, a send or a receive that failed. A + // std::system_error, so still a std::runtime_error for the handlers that + // catch one: code() is the errno under std::generic_category(), or the + // getaddrinfo result under ResolverCategory(), and what() is the text + // ClientTCP has always thrown, the code's own text after it. `phase` + // says which step failed. A caller that reports the cause in its own + // words — "[Errno 111] Connection refused", say — takes code().value() + // and code().message() instead of parsing what(), and compares code() + // with std::errc::connection_refused instead of the number. + export class SocketErrorException : public std::system_error { public: - SocketErrorException(SocketErrorKind kind, std::int32_t code, const std::string& message) - : std::runtime_error(message), kind(kind), code(code) {} - SocketErrorKind kind; - std::int32_t code; + SocketErrorException(SocketPhase phase, std::error_code code, const std::string& what) + : std::system_error(code, what), phase(phase) {} + // Without a message of its own: what() is the code's text alone. + SocketErrorException(SocketPhase phase, std::error_code code) + : std::system_error(code), phase(phase) {} + SocketPhase phase; }; export class ClientTCP { diff --git a/tests/ShouldBindLoopbackEphemeral/main.cpp b/tests/ShouldBindLoopbackEphemeral/main.cpp index a897e22..1653c9e 100644 --- a/tests/ShouldBindLoopbackEphemeral/main.cpp +++ b/tests/ShouldBindLoopbackEphemeral/main.cpp @@ -92,7 +92,7 @@ int main() { try { ClientTCP client(*outside, port, 2s); } catch (const SocketErrorException& e) { - refused = e.code == ECONNREFUSED; + refused = e.phase == SocketPhase::Connect && e.code() == std::errc::connection_refused; } catch (const std::exception&) { } Check(refused, std::format("through {} it is refused: it listens on 127.0.0.1 alone", *outside)); diff --git a/tests/ShouldConnectDualStack/main.cpp b/tests/ShouldConnectDualStack/main.cpp index 17707f8..2629650 100644 --- a/tests/ShouldConnectDualStack/main.cpp +++ b/tests/ShouldConnectDualStack/main.cpp @@ -140,7 +140,7 @@ int main() { try { ClientTCP client("localhost", closedPort, 2s); } catch (const SocketErrorException& e) { - refused = e.kind == SocketErrorKind::System && e.code == ECONNREFUSED; + refused = e.phase == SocketPhase::Connect && e.code() == std::errc::connection_refused; } catch (const std::exception&) { } Check(refused, "when no address answers, the last failure is thrown, with its errno"); diff --git a/tests/ShouldReportConnectErrors/main.cpp b/tests/ShouldReportConnectErrors/main.cpp index c919907..6d0ecf5 100644 --- a/tests/ShouldReportConnectErrors/main.cpp +++ b/tests/ShouldReportConnectErrors/main.cpp @@ -1,13 +1,15 @@ //SPDX-License-Identifier: LGPL-3.0-only //SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts® -// A failed resolve or connect throws SocketErrorException, which carries the -// code it failed with: the errno for a connect, the getaddrinfo result for a -// name. Before, both were a bare std::runtime_error, so a caller that reports -// the cause in its own words — a Python-style "[Errno 111] Connection -// refused" in front of a language model reading the output — had to parse the -// message back apart. The message itself stays what it always was, and the -// exception is still a std::runtime_error, so existing handlers keep working. +// A failed resolve or connect throws SocketErrorException, a std::system_error +// whose code() carries what it failed with: the errno for a connect, under +// std::generic_category(); the getaddrinfo result for a name, under +// ResolverCategory(), whose text is gai_strerror's. `phase` says which step +// failed. Before, both were a bare std::runtime_error, so a caller that +// reports the cause in its own words — "[Errno 111] Connection refused" in +// front of a language model reading the output — had to parse the message +// back apart. The message itself stays what it always was, and the exception +// is still a std::runtime_error, so existing handlers keep working. #include #include @@ -54,8 +56,11 @@ int main() { ClientTCP client("127.0.0.1", ClosedPort(), 2s); } catch (const SocketErrorException& e) { thrown = true; - Check(e.kind == SocketErrorKind::System, "a refused connect is a system error"); - Check(e.code == ECONNREFUSED, std::format("it carries ECONNREFUSED (got {})", e.code)); + Check(e.phase == SocketPhase::Connect, "a refused connect failed in the connect phase"); + Check(e.code().category() == std::generic_category(), "its code is an errno"); + Check(e.code().value() == ECONNREFUSED, std::format("it carries ECONNREFUSED (got {})", e.code().value())); + Check(e.code() == std::errc::connection_refused, "and compares with std::errc::connection_refused"); + Check(e.code().message() == std::strerror(ECONNREFUSED), std::format("its code's text is strerror's (got '{}')", e.code().message())); Check(std::string_view(e.what()) == std::string("Could not connect to server: ") + std::strerror(ECONNREFUSED), std::format("its message is the one ClientTCP always threw (got '{}')", e.what())); } @@ -64,12 +69,14 @@ int main() { { bool caught = false; + bool system = false; try { ClientTCP client("127.0.0.1", ClosedPort(), 2s); - } catch (const std::runtime_error&) { - caught = true; + } catch (const std::system_error& e) { + system = e.code() == std::errc::connection_refused; + caught = dynamic_cast(&e) != nullptr; } - Check(caught, "it is still caught as std::runtime_error"); + Check(caught && system, "it is caught as std::system_error, and is still a std::runtime_error"); } { @@ -80,10 +87,12 @@ int main() { ClientTCP client("no-such-host.invalid", 80, 2s); } catch (const SocketErrorException& e) { thrown = true; - Check(e.kind == SocketErrorKind::Resolver, "a name that does not resolve is a resolver error"); - Check(e.code != 0, std::format("it carries the getaddrinfo code (got {})", e.code)); - Check(std::string_view(e.what()).starts_with("Could not resolve host 'no-such-host.invalid': "), - std::format("its message names the host (got '{}')", e.what())); + Check(e.phase == SocketPhase::Resolve, "a name that does not resolve failed in the resolve phase"); + Check(e.code().category() == ResolverCategory(), "its code is in the resolver category"); + Check(e.code().value() != 0, std::format("it carries the getaddrinfo code (got {})", e.code().value())); + Check(e.code().message() == gai_strerror(e.code().value()), std::format("the category's text is gai_strerror's (got '{}')", e.code().message())); + Check(std::string_view(e.what()) == std::string("Could not resolve host 'no-such-host.invalid': ") + gai_strerror(e.code().value()), + std::format("its message names the host, the code's text after it (got '{}')", e.what())); } Check(thrown, "an unresolvable host throws SocketErrorException"); } diff --git a/tests/ShouldReportIoErrors/main.cpp b/tests/ShouldReportIoErrors/main.cpp index 52d5004..61efa03 100644 --- a/tests/ShouldReportIoErrors/main.cpp +++ b/tests/ShouldReportIoErrors/main.cpp @@ -1,12 +1,12 @@ //SPDX-License-Identifier: LGPL-3.0-only //SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts® -// A send or a receive that fails throws SocketErrorException with its errno, -// as a failed connect does: a caller that reports the cause in its own words -// — "the connection broke ([Errno 104] Connection reset by peer)" — needs the -// code, and the message alone is the strerror text a send and a receive -// always threw. A receive timeout is still SocketTimeoutException and a -// clean close SocketClosedException. +// A send or a receive that fails throws SocketErrorException with its errno +// in code() and the phase that failed, as a failed connect does: a caller +// that reports the cause in its own words — "the connection broke ([Errno +// 104] Connection reset by peer)" — needs the code, and the message alone is +// the strerror text a send and a receive always threw. A receive timeout is +// still SocketTimeoutException and a clean close SocketClosedException. #include #include @@ -73,8 +73,8 @@ int main() { client.RecieveSync(16); } catch (const SocketErrorException& e) { thrown = true; - Check(e.kind == SocketErrorKind::System, "a reset connection fails the receive as a system error"); - Check(e.code == ECONNRESET, std::format("it carries ECONNRESET (got {})", e.code)); + Check(e.phase == SocketPhase::Receive, "a reset connection fails in the receive phase"); + Check(e.code() == std::errc::connection_reset && e.code().value() == ECONNRESET, std::format("it carries ECONNRESET (got {})", e.code().value())); Check(std::string_view(e.what()) == std::strerror(ECONNRESET), std::format("its message is the strerror text (got '{}')", e.what())); } catch (const std::exception& e) { Check(false, std::format("the receive threw something else: {}", e.what())); @@ -90,12 +90,14 @@ int main() { // kernel answers the next one with EPIPE or ECONNRESET. std::int32_t code = 0; bool runtime = false; + bool sending = false; const std::string data(4096, 'x'); for (int i = 0; i < 50 && code == 0; ++i) { try { client.Send(data.data(), static_cast(data.size())); } catch (const SocketErrorException& e) { - code = e.code; + code = e.code().value(); + sending = e.phase == SocketPhase::Send && e.code().category() == std::generic_category(); runtime = dynamic_cast(&e) != nullptr; } catch (const std::exception&) { break; @@ -103,7 +105,7 @@ int main() { std::this_thread::sleep_for(10ms); } Check(code == EPIPE || code == ECONNRESET, std::format("a send into a reset connection throws SocketErrorException with EPIPE or ECONNRESET (got {})", code)); - Check(runtime, "and it is still a std::runtime_error"); + Check(sending && runtime, "in the send phase, an errno, and still a std::runtime_error"); } std::println("{}", failures == 0 ? "all passed" : std::format("{} failed", failures)); From 7ee938c98e8c76290eef41b03e49ca5eed27a174 Mon Sep 17 00:00:00 2001 From: Jorijn van der Graaf Date: Sat, 31 Oct 2026 20:39:43 +0100 Subject: [PATCH 6/6] feat(http1): ClientHTTP1::Fetch follows redirects by the RFC, on a URL type MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit One request to an http(s) URL with redirects followed by RFC 9110 §15.4: 301, 302, 303, 307 and 308 for every method; a 303 makes the request a GET without its body and content headers, a 301 or 302 does so for a POST alone, a 307 or 308 keeps the method and the body. The Location is resolved against the sending URL by RFC 3986, a hop between http and https is allowed, Authorization, Proxy-Authorization and Cookie are dropped on a change of origin, and each hop is a client of its own, since one ClientHTTP1 is one origin. A redirect without a usable Location, to another scheme, back to a URL already visited or past maxRedirects comes back as the response, with `stop` saying why. The new :URL partition is the URL type behind it, exported: Parse, Resolve (RFC 3986 §5.2; its §5.4 examples are the test), ToString, and the pieces a request is built from. This replaces the FetchHTTP1 free function, which followed redirects the way Python's urllib did (its repeat limit, its quoting of the Location, a POST turned back on a 307) for a port that no longer needs the fidelity. Breaking: FetchHTTP1, FetchOptionsHTTP1 and FetchResultHTTP1 are gone; use ClientHTTP1::Fetch, FetchOptions and FetchResult, whose `url` is a URL and whose `limited` became `stop`. --- AGENTS.consumer.md | 5 +- AGENTS.template.md | 11 +- README.md | 18 +- .../Crafter.Network-ClientHTTP1.cpp | 331 ++++-------------- interfaces/Crafter.Network-ClientHTTP1.cppm | 116 +++--- interfaces/Crafter.Network-URL.cppm | 271 ++++++++++++++ interfaces/Crafter.Network.cppm | 3 + project.cpp | 4 +- tests/ShouldFollowRedirectsHTTP1/main.cpp | 106 ++++-- tests/ShouldParseURLs/main.cpp | 105 ++++++ 10 files changed, 597 insertions(+), 373 deletions(-) create mode 100644 interfaces/Crafter.Network-URL.cppm create mode 100644 tests/ShouldParseURLs/main.cpp diff --git a/AGENTS.consumer.md b/AGENTS.consumer.md index 0e70b41..ae8486e 100644 --- a/AGENTS.consumer.md +++ b/AGENTS.consumer.md @@ -13,7 +13,7 @@ The Crafter engine's networking library: a C++ module, `import Crafter.Network;` |---|---|---|---| | `HTTPRequest`, `HTTPResponse`, `CreateRequestHTTP`, `CreateResponseHTTP`, `PathWithoutQueryHTTP` | yes | yes | yes | | `ClientTCP`, `ListenerTCP` | yes | yes | no | -| `ByteStream`, `PlainStream`, `TLSContext`, `TLSStream`, `HTTP1::*`, `ClientHTTP1`, `ListenerHTTP1` | yes | yes | no | +| `ByteStream`, `PlainStream`, `TLSContext`, `TLSStream`, `HTTP1::*`, `URL`, `ClientHTTP1`, `ListenerHTTP1` | yes | yes | no | | `ClientQUIC`, `QUICStream` | yes | no | `*Async` only, over `WebTransport` | | `ClientHTTP` | yes | no | `SendAsync` only, over `fetch()` | | `ListenerQUIC`, `ListenerHTTP`, `WebTransportSession`, `HTTP3::*` | yes | no | no | @@ -43,6 +43,8 @@ A route map is `std::unordered_map.cppm` | One partition per file. The native-only ones wrap their whole body in `#ifndef CRAFTER_NETWORK_BROWSER`, so the browser build compiles them as empty modules. | -| `implementations/Crafter.Network-.cpp` | The native implementation units, `Crafter.Network:_impl`. `HTTP`, `HTTP1` and `HTTP3` have none: they are inline. | +| `implementations/Crafter.Network-.cpp` | The native implementation units, `Crafter.Network:_impl`. `HTTP`, `HTTP1`, `HTTP3` and `URL` have none: they are inline. | | `implementations/Crafter.Network-ClientHTTP-Browser.cpp`, `-ClientQUIC-Browser.cpp` | The browser implementations of those two partitions, over the JS bridge. | | `additional/network-env.js` | The JS bridge: `fetch()` and `WebTransport` behind the `env` imports the wasm declares, shipped next to a consumer's `.wasm` through `cfg.files`. | | `project.cpp` | The build description: one static library, `crafter-network`, in its three variants, the msquic external, the OpenSSL link flags and the test declarations. | @@ -25,8 +25,9 @@ Partitions and what they own: - `:Stream`: `ByteStream` (`ReadSome` and `Write` with a timeout per call, `Shutdown`, `Descriptor`, `Protocol`, `Secure`) and `PlainStream`, so one HTTP/1.1 implementation serves `http://` and `https://`. `SetNonBlocking`, `PollDescriptor`. - `:TLS`: `TLSContext`, `TLSStream`, both credential structs, `GetSelfSignedCertificatePem`. OpenSSL stays behind the `Impl` pointers; no OpenSSL header reaches an importer. - `:HTTP`: `HTTPRequest`, `HTTPResponse`, the `Create*` helpers, `PathWithoutQueryHTTP`. Shared by both HTTP versions and by every build. +- `:URL`: `URL`, an absolute URL as RFC 3986 reads it (`Parse`), the resolution of a reference against it (`Resolve`, RFC 3986 §5.2, dot segments removed), the recomposition, and the pieces a request is built from. Inline, native only; what `ClientHTTP1::Fetch` follows a Location with. - `:HTTP1`: the RFC 9112 codec, inline: `SerializeRequest`, `SerializeResponse`, `SerializeContinue`, `MessageParser`, `MessageLimits`, `HTTP1ProtocolError`, and the framing it refuses to guess at. -- `:ClientHTTP1`, `:ListenerHTTP1`: HTTP/1.1 over a `ByteStream&`. The listener is a `ListenerTCP` with a thread per accepted connection; the client keeps one pooled connection with a replay-once rule. +- `:ClientHTTP1`, `:ListenerHTTP1`: HTTP/1.1 over a `ByteStream&`. The listener is a `ListenerTCP` with a thread per accepted connection; the client keeps one pooled connection with a replay-once rule, and `ClientHTTP1::Fetch` is the URL fetch that follows redirects by RFC 9110 §15.4 with a client per hop (`FetchOptions`, `FetchResult`, `FetchStop`). - `:ClientQUIC`: `ClientQUIC` and `QUICStream` for both ends of a connection, `QUICSettings`, both QUIC credential structs, `QUICException`, `QUICClosedException`. The native unit holds the process-wide `MsQuicRuntime` (API table and registration, closed by a static destructor at exit), the stream registry with its refcounted handles, and the drain in `~ClientQUIC`. - `:ListenerQUIC`: the msquic listener, its four `Listen*` variants, the self-signed certificate (made by the `openssl` CLI and cached under `/tmp/crafter-network-quic-cert/`), `ComputeCertificateHashSHA256`, `GetSelfSignedCertificatePath`. - `:HTTP3`: QUIC varints, HTTP/3 frames, the QPACK static table, Huffman decoding, the control-stream preludes and the WebTransport stream prefixes. Native only: it throws for protocol errors. @@ -50,9 +51,9 @@ crafter-build # native: TCP, TLS, HTTP/1.1, QUI crafter-build --no-quic # native without QUIC, HTTP/3, WebTransport and msquic crafter-build --local # Crafter.Thread from ../Crafter.Thread crafter-build --target=wasm32-wasip1 # the browser library -crafter-build test # the suite, native: 23 tests +crafter-build test # the suite, native: 30 tests crafter-build test 'Should*HTTP1' # a name glob -crafter-build test --no-quic # the TCP, TLS and HTTP/1.1 tests plus ShouldBuildWithoutQUIC: 12 +crafter-build test --no-quic # the TCP, URL, TLS and HTTP/1.1 tests plus ShouldBuildWithoutQUIC: 19 crafter-build test --list crafter-build test --jobs=1 --timeout=120 crafter-build clean diff --git a/README.md b/README.md index 274f1c1..ece27d0 100644 --- a/README.md +++ b/README.md @@ -134,20 +134,22 @@ The connection is persistent: the first `Send()` dials, and later calls reuse th Three timeouts bound a request: `connectTimeout` (default 15 s) for the TCP connect, per address the host resolves to; `handshakeTimeout` (default 15 s) for the TLS handshake; and `timeout` (default 30 s) for each wait on the next piece of a response. -#### FetchHTTP1 +#### ClientHTTP1::Fetch -`FetchHTTP1(url, options)` is one request to an `http://` or `https://` URL, following redirects the way Python's urllib does, so code ported from urllib keeps its behaviour: +`ClientHTTP1::Fetch(url, options)` is one request to an `http://` or `https://` URL, following redirects by RFC 9110 §15.4: ```cpp -Crafter::FetchOptionsHTTP1 options; +Crafter::FetchOptions options; options.timeout = std::chrono::seconds(10); options.headers["accept-encoding"] = "identity"; -Crafter::FetchResultHTTP1 result = Crafter::FetchHTTP1("http://localhost:8080/", options); -// result.response: the last response; result.url: the URL that sent it; -// result.redirects: how many were followed; result.limited: the limits stopped the chain. +Crafter::FetchResult result = Crafter::ClientHTTP1::Fetch("http://localhost:8080/", options); +// result.response: the last response; result.url: the URL that sent it (a Crafter::URL); +// result.redirects: how many were followed; result.stop: why a redirect was not followed, or None. ``` -301, 302, 303, 307 and 308 are followed for a GET or HEAD, and 301/302/303 also for a POST; the redirected request is a GET (a HEAD stays a HEAD) without the body or its content headers. A relative `Location` is resolved against the URL that sent it, as `urljoin` resolves it, and is percent-encoded the way urllib quotes it; a redirect may move between `http://` and `https://` (`options.credentials` applies to every TLS hop). Each redirect's body is read before the next request goes out. A redirect is returned unfollowed when it has no `Location`, points at another scheme, or comes for a method urllib does not redirect. The chain stops, returning the redirect with `limited` set, at the fifth redirect to one URL (`maxRepeats`, 4) or once ten different URLs have been visited (`maxRedirects`, 10). Proxies are not consulted. A URL that is not `http(s)://`, has no host or a bad port throws `std::invalid_argument`. +301, 302, 303, 307 and 308 are followed for every method: a 303 turns the request into a GET (a HEAD stays a HEAD) without its body and content headers, a 301 or 302 does the same for a POST alone, and a 307 or 308 keeps the method and the body. A relative `Location` is resolved against the URL that sent it (RFC 3986, through `Crafter::URL::Resolve`), and a redirect may move between `http://` and `https://` (`options.credentials` applies to every TLS hop). `Authorization`, `Proxy-Authorization` and `Cookie` are not carried to another origin. Each hop is a client of its own, so a redirect's body is read and its connection closed before the next request goes out. A redirect comes back as the response, with `result.stop` saying why, when it has no usable `Location`, points at another scheme, points back at a URL the chain already visited, or would be the eleventh (`maxRedirects`, 10). Proxies are not consulted. A URL that is not `http(s)://`, has no host or has a bad port throws `std::invalid_argument`. + +`Crafter::URL` is the URL type behind it, exported for callers that build or resolve URLs themselves: `URL::Parse(text)`, `url.Resolve(reference)`, `url.ToString()`, and the pieces a request is built from (`Authority()`, `Target()`, `PortOrDefault()`, `Secure()`, `SameOrigin(other)`). #### ListenerHTTP1 ```cpp @@ -331,7 +333,7 @@ The library includes tests covering: - Send and receive errors (`ShouldReportIoErrors`) — a connection the peer resets fails the next receive, and a send into it, with `SocketErrorException` carrying ECONNRESET or EPIPE under the plain strerror text a receive and a send always threw - Loopback listeners (`ShouldBindLoopbackEphemeral`) — a listener on `127.0.0.1` at port 0 reports its port, takes loopback clients and refuses them through the host's other address; `::1` works too, a name is refused, and the port-only constructor still listens on every interface - HTTP/1.1 timeouts (`ShouldTimeoutSilentHTTP1`) — a server that accepts and never answers fails after `timeout`, and a connect nobody answers after `connectTimeout` -- Redirects (`ShouldFollowRedirectsHTTP1`) — relative and `..` Locations, an http-to-https hop, the 4-repeat and 10-URL limits, a redirect without Location, POST after 303 and 307, percent-encoding a Location, and the URLs `FetchHTTP1` refuses +- Redirects (`ShouldFollowRedirectsHTTP1`) — relative and `..` Locations, an http-to-https hop, a loop and the redirect limit, a redirect without Location or to another scheme, a POST after 303, 302 and 307 and a PUT after 302, credentials dropped across origins, and the URLs `Fetch` refuses; `ShouldParseURLs` runs RFC 3986 §5.4's resolution examples and the parse rules The external-interop test requires outbound UDP/443; if your network blocks it the test will fail. `ShouldInteropCurlHTTP1` and `ShouldInteropCurlHTTPS1` skip whichever half is unavailable when `curl` or `python3` is not installed, and the mutual-TLS half of `ShouldRequireClientCertificateHTTPS1` skips without the `openssl` CLI (a *client* certificate needs the `clientAuth` extended key usage, which the built-in development certificate does not carry) — so all three pass on a bare machine. Install `curl`, `python3` and `openssl` to actually exercise them. diff --git a/implementations/Crafter.Network-ClientHTTP1.cpp b/implementations/Crafter.Network-ClientHTTP1.cpp index 5e43633..2ecb6bb 100644 --- a/implementations/Crafter.Network-ClientHTTP1.cpp +++ b/implementations/Crafter.Network-ClientHTTP1.cpp @@ -11,6 +11,7 @@ import :HTTP; import :HTTP1; import :Stream; import :TLS; +import :URL; import Crafter.Thread; import std; @@ -180,254 +181,35 @@ void ClientHTTP1::SendAsync(const HTTPRequest& request, } namespace { - // The five parts urllib.parse.urlsplit cuts a URL into. FetchHTTP1 needs - // the splitting, the joining (urljoin) and the reassembly to resolve a - // Location the way urllib does, and nothing else of urllib.parse. - struct SplitURL { - std::string scheme; - std::string netloc; - std::string path; - std::string query; - std::string fragment; - }; - - bool IsAsciiAlpha(char c) { - return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z'); - } - - bool IsSchemeChar(char c) { - return IsAsciiAlpha(c) || (c >= '0' && c <= '9') || c == '+' || c == '-' || c == '.'; - } - - std::string ToLowerAsciiCopy(std::string_view text) { - std::string lower(text); - for (char& c : lower) { - if (c >= 'A' && c <= 'Z') c = static_cast(c - 'A' + 'a'); - } - return lower; - } - - // str.split('/'): every piece, empty ones included, and [""] for "". - std::vector SplitSlashes(std::string_view text) { - std::vector pieces; - std::size_t start = 0; - while (true) { - const std::size_t slash = text.find('/', start); - if (slash == std::string_view::npos) { - pieces.emplace_back(text.substr(start)); - return pieces; - } - pieces.emplace_back(text.substr(start, slash - start)); - start = slash + 1; - } - } - - // urlsplit: tabs and newlines dropped anywhere, leading controls and - // spaces stripped, the scheme lowercased. - SplitURL Split(std::string_view input) { - std::string url; - url.reserve(input.size()); - for (const char c : input) { - if (c != '\t' && c != '\r' && c != '\n') url.push_back(c); - } - std::size_t start = 0; - while (start < url.size() && static_cast(url[start]) <= 0x20) ++start; - url.erase(0, start); - - SplitURL parts; - const std::size_t colon = url.find(':'); - if (colon != std::string::npos && colon > 0 && IsAsciiAlpha(url[0]) - && std::ranges::all_of(std::string_view(url).substr(0, colon), IsSchemeChar)) { - parts.scheme = ToLowerAsciiCopy(std::string_view(url).substr(0, colon)); - url.erase(0, colon + 1); - } - if (url.starts_with("//")) { - std::size_t end = url.find_first_of("/?#", 2); - if (end == std::string::npos) end = url.size(); - parts.netloc = url.substr(2, end - 2); - url.erase(0, end); - if (parts.netloc.contains('[') != parts.netloc.contains(']')) { - throw std::invalid_argument("FetchHTTP1: unbalanced brackets in the host of " + std::string(input)); - } - } - if (const std::size_t hash = url.find('#'); hash != std::string::npos) { - parts.fragment = url.substr(hash + 1); - url.erase(hash); - } - if (const std::size_t question = url.find('?'); question != std::string::npos) { - parts.query = url.substr(question + 1); - url.erase(question); - } - parts.path = std::move(url); - return parts; - } - - // urlunsplit, for the http(s) URLs FetchHTTP1 puts back together. - std::string Unsplit(const SplitURL& parts) { - std::string url = parts.path; - if (!parts.netloc.empty()) { - if (!url.empty() && url.front() != '/') url.insert(url.begin(), '/'); - url = "//" + parts.netloc + url; - } else if (url.starts_with("//")) { - url = "//" + url; - } else if ((parts.scheme == "http" || parts.scheme == "https") && (url.empty() || url.front() == '/')) { - url = "//" + url; - } - if (!parts.scheme.empty()) url = parts.scheme + ":" + url; - if (!parts.query.empty()) url += "?" + parts.query; - if (!parts.fragment.empty()) url += "#" + parts.fragment; - return url; - } - - // urljoin, for a reference found in a redirect: RFC 3986 resolution the - // way Python does it, a `..` that would climb above the root dropped. - std::string Join(std::string_view base, std::string_view reference) { - if (base.empty()) return std::string(reference); - if (reference.empty()) return std::string(base); - const SplitURL from = Split(base); - SplitURL to = Split(reference); - if (to.scheme.empty()) to.scheme = from.scheme; - const bool relative = to.scheme == "http" || to.scheme == "https"; - if (to.scheme != from.scheme || !relative) return std::string(reference); - if (!to.netloc.empty()) return Unsplit(to); - to.netloc = from.netloc; - if (to.path.empty()) { - to.path = from.path; - if (to.query.empty()) to.query = from.query; - return Unsplit(to); - } - - std::vector baseParts = SplitSlashes(from.path); - // The base's last segment is a file, not a directory: it does not - // take part in resolving. - if (!baseParts.back().empty()) baseParts.pop_back(); - std::vector segments; - if (to.path.starts_with('/')) { - segments = SplitSlashes(to.path); - } else { - segments = std::move(baseParts); - std::vector more = SplitSlashes(to.path); - segments.insert(segments.end(), more.begin(), more.end()); - // Empty segments between the first and the last would come back - // as doubled slashes. - if (segments.size() > 2) { - std::vector kept { segments.front() }; - for (std::size_t i = 1; i + 1 < segments.size(); ++i) { - if (!segments[i].empty()) kept.push_back(segments[i]); - } - kept.push_back(segments.back()); - segments = std::move(kept); - } - } - std::vector resolved; - for (const std::string& segment : segments) { - if (segment == "..") { - if (!resolved.empty()) resolved.pop_back(); - } else if (segment != ".") { - resolved.push_back(segment); - } - } - // A reference ending in a directory step keeps its trailing slash. - if (segments.back() == "." || segments.back() == "..") resolved.emplace_back(); - std::string path; - for (std::size_t i = 0; i < resolved.size(); ++i) { - if (i > 0) path += '/'; - path += resolved[i]; - } - to.path = path.empty() ? std::string("/") : path; - return Unsplit(to); - } - - // urllib quotes a Location before it follows it, keeping every ASCII - // letter, digit and punctuation mark: what it escapes are spaces, - // controls and bytes outside ASCII. - std::string QuoteLocation(std::string_view location) { - std::string quoted; - for (const char c : location) { - const unsigned char byte = static_cast(c); - if (byte > 0x20 && byte < 0x7f) { - quoted.push_back(c); - } else { - quoted += std::format("%{:02X}", byte); - } - } - return quoted; - } - - struct FetchTarget { - bool secure = false; - // Without brackets: what the socket and TLS verification are given. - std::string host; - std::uint16_t port = 0; - // The URL's host[:port] as written, for the Host header. - std::string authority; - // Path and query: the request target. - std::string target; - }; - - FetchTarget ParseTarget(std::string_view url) { - const SplitURL parts = Split(url); - if (parts.scheme != "http" && parts.scheme != "https") { - throw std::invalid_argument("FetchHTTP1: not an http:// or https:// URL: " + std::string(url)); - } - FetchTarget target; - target.secure = parts.scheme == "https"; - std::string hostinfo = parts.netloc; - if (const std::size_t at = hostinfo.rfind('@'); at != std::string::npos) hostinfo.erase(0, at + 1); - std::string port; - if (hostinfo.starts_with('[')) { - const std::size_t close = hostinfo.find(']'); - target.host = hostinfo.substr(1, close - 1); - const std::string rest = hostinfo.substr(close + 1); - if (rest.starts_with(':')) { - port = rest.substr(1); - } else if (!rest.empty()) { - throw std::invalid_argument("FetchHTTP1: unexpected text after the IPv6 host in " + std::string(url)); - } - } else { - const std::size_t colon = hostinfo.find(':'); - target.host = hostinfo.substr(0, colon); - if (colon != std::string::npos) port = hostinfo.substr(colon + 1); - } - if (target.host.empty()) { - throw std::invalid_argument("FetchHTTP1: no host in " + std::string(url)); - } - if (port.empty()) { - target.port = target.secure ? 443 : 80; - } else { - std::uint32_t value = 0; - const auto [end, error] = std::from_chars(port.data(), port.data() + port.size(), value); - if (error != std::errc() || end != port.data() + port.size() || value > 65535) { - throw std::invalid_argument("FetchHTTP1: bad port '" + port + "' in " + std::string(url)); - } - target.port = static_cast(value); - } - target.authority = hostinfo; - target.target = parts.path.empty() ? std::string("/") : parts.path; - if (!target.target.starts_with('/')) target.target.insert(target.target.begin(), '/'); - if (!parts.query.empty()) target.target += "?" + parts.query; - return target; - } - std::int32_t StatusCode(const HTTPResponse& response) { std::int32_t code = 0; std::from_chars(response.status.data(), response.status.data() + response.status.size(), code); return code; } + + bool IsRedirect(std::int32_t code) { + return code == 301 || code == 302 || code == 303 || code == 307 || code == 308; + } } -FetchResultHTTP1 Crafter::FetchHTTP1(std::string_view url, const FetchOptionsHTTP1& options) { - std::string current(url); +FetchResult ClientHTTP1::Fetch(std::string_view url, const FetchOptions& options) { + URL current = URL::Parse(url); + if (current.scheme != "http" && current.scheme != "https") { + throw std::invalid_argument("ClientHTTP1::Fetch: not an http:// or https:// URL: " + std::string(url)); + } std::string method = options.method.empty() ? std::string("GET") : options.method; std::unordered_map headers = options.headers; std::string body = options.body; - // How often the chain has been sent to each URL: urllib's loop check. - std::map visited; - FetchResultHTTP1 result; + // Every URL the chain has been sent to: a redirect back to one of them + // would loop. + std::set visited { current.ToString() }; + FetchResult result; while (true) { - const FetchTarget target = ParseTarget(current); - ClientHTTP1 client = target.secure ? ClientHTTP1(target.host, target.port, options.credentials) - : ClientHTTP1(target.host, target.port); + // A client per hop: one ClientHTTP1 is one origin, and a redirect + // may leave it. Destroying the client closes the hop's connection + // once its response, body included, has been read. + ClientHTTP1 client = current.Secure() ? ClientHTTP1(current.host, current.PortOrDefault(), options.credentials) + : ClientHTTP1(current.host, current.PortOrDefault()); client.connectTimeout = options.connectTimeout; client.handshakeTimeout = options.handshakeTimeout; client.timeout = options.timeout; @@ -435,48 +217,59 @@ FetchResultHTTP1 Crafter::FetchHTTP1(std::string_view url, const FetchOptionsHTT HTTPRequest request; request.method = method; - request.scheme = target.secure ? "https" : "http"; - request.authority = target.authority; - request.path = target.target; + request.scheme = current.scheme; + request.authority = current.Authority(); + request.path = current.Target(); request.headers = headers; request.body = body; result.response = client.Send(request); result.url = current; const std::int32_t code = StatusCode(result.response); - const bool redirect = code == 301 || code == 302 || code == 303 || code == 307 || code == 308; - if (!redirect) return result; - // The methods urllib redirects; any other comes back as the redirect. - const bool followed = method == "GET" || method == "HEAD" || (method == "POST" && code <= 303); - if (!followed) return result; - auto location = result.response.headers.find("location"); - if (location == result.response.headers.end()) location = result.response.headers.find("uri"); - if (location == result.response.headers.end()) return result; - - std::string next = QuoteLocation(location->second); - SplitURL parts = Split(next); - if (!parts.scheme.empty() && parts.scheme != "http" && parts.scheme != "https") return result; - if (parts.path.empty() && !parts.netloc.empty()) { - parts.path = "/"; - next = Unsplit(parts); - } - next = Join(current, next); - - const auto seen = visited.find(next); - const std::uint32_t times = seen == visited.end() ? 0 : seen->second; - if (times >= options.maxRepeats || visited.size() >= options.maxRedirects) { - result.limited = true; + if (!IsRedirect(code)) return result; + const auto location = result.response.headers.find("location"); + if (location == result.response.headers.end()) { + result.stop = FetchStop::NoLocation; + return result; + } + URL next; + try { + next = current.Resolve(location->second); + } catch (const std::invalid_argument&) { + result.stop = FetchStop::NoLocation; + return result; + } + if (next.scheme != "http" && next.scheme != "https") { + result.stop = FetchStop::OtherScheme; + return result; + } + if (result.redirects >= options.maxRedirects) { + result.stop = FetchStop::Limit; + return result; + } + if (!visited.insert(next.ToString()).second) { + result.stop = FetchStop::Loop; return result; } - visited[next] = times + 1; ++result.redirects; - // What urllib sends next: a GET, or a HEAD for a HEAD, without the - // body or the headers that described it. - if (method != "HEAD") method = "GET"; - body.clear(); - headers.erase("content-length"); - headers.erase("content-type"); + // RFC 9110 §15.4: a 303 asks for a GET of the Location; a 301 or 302 + // keeps the method, but a POST becomes a GET, as the user agents the + // sections describe have always done; a 307 or 308 keeps the method + // and the body. A request that turned into a GET loses the body and + // the headers that described it. + const bool toGet = (code == 303 && method != "HEAD") || ((code == 301 || code == 302) && method == "POST"); + if (toGet) { + method = "GET"; + body.clear(); + std::erase_if(headers, [](const auto& header) { return header.first.starts_with("content-"); }); + } + // Credentials stay with the origin they were meant for. + if (!next.SameOrigin(current)) { + headers.erase("authorization"); + headers.erase("proxy-authorization"); + headers.erase("cookie"); + } current = std::move(next); } } diff --git a/interfaces/Crafter.Network-ClientHTTP1.cppm b/interfaces/Crafter.Network-ClientHTTP1.cppm index b5cb7b8..2f73783 100644 --- a/interfaces/Crafter.Network-ClientHTTP1.cppm +++ b/interfaces/Crafter.Network-ClientHTTP1.cppm @@ -6,9 +6,53 @@ import std; import :HTTP; import :HTTP1; import :TLS; +import :URL; #ifndef CRAFTER_NETWORK_BROWSER namespace Crafter { + // What ClientHTTP1::Fetch needs beyond the URL. + export struct FetchOptions { + std::string method = "GET"; + // Lowercase names, as on HTTPRequest. Host comes from the URL. + std::unordered_map headers; + std::string body; + // As on ClientHTTP1, for every request of the chain. + std::chrono::milliseconds connectTimeout{15000}; + std::chrono::milliseconds handshakeTimeout{15000}; + std::chrono::milliseconds timeout{30000}; + HTTP1::MessageLimits limits; + // For every https:// request of the chain. + TLSClientCredentials credentials; + // How many redirects are followed at most; the one after comes back + // unfollowed. + std::uint32_t maxRedirects = 10; + }; + + // Why a Fetch ended on a redirect it did not follow. + export enum class FetchStop { + // It did not: the chain ended on a final response. + None, + // The redirect carried no Location, or one that is no URL. + NoLocation, + // The Location names a scheme other than http or https. + OtherScheme, + // The Location is a URL the chain has already been sent to. + Loop, + // maxRedirects redirects had been followed already. + Limit, + }; + + export struct FetchResult { + // The last response of the chain: the final response, or the + // redirect that was not followed, with `stop` saying why. + HTTPResponse response; + // The URL that answered with `response`. + URL url; + // Redirects followed to get there. + std::uint32_t redirects = 0; + FetchStop stop = FetchStop::None; + }; + // HTTP/1.1 client over TCP, with or without TLS, for peers that cannot // speak HTTP/3. The request/response types are the ones the HTTP/3 client // uses, so swapping ClientHTTP for ClientHTTP1 is a one-line change at the @@ -78,6 +122,27 @@ namespace Crafter { // Drop the pooled connection; the next Send() dials again. void Disconnect(); + // One request to an http:// or https:// URL, following redirects by + // RFC 9110 §15.4: 301, 302, 303, 307 and 308, for every method. A + // 303 turns the request into a GET (a HEAD stays a HEAD) without its + // body and content headers; 301 and 302 do the same for a POST + // alone; 307 and 308 keep the method and the body. The Location is + // resolved against the URL that sent it (URL::Resolve), and a + // redirect may move between http:// and https://, `credentials` + // serving every https:// hop. Authorization, Proxy-Authorization and + // Cookie are not carried to another origin. Each hop is a client of + // its own, since one ClientHTTP1 is one origin: the redirect's body + // is read and its connection closed before the next request goes + // out. A redirect without a usable Location, to another scheme, to + // a URL already visited, or past maxRedirects comes back as the + // response with `stop` set. Proxies are not consulted. + // + // Throws std::invalid_argument for a URL that is not http(s) or that + // URL::Parse refuses, and whatever a request throws otherwise + // (SocketErrorException, SocketTimeoutException, TLSException, + // HTTP1::HTTP1ProtocolError, std::runtime_error). + static FetchResult Fetch(std::string_view url, const FetchOptions& options = {}); + // Limits applied to responses. Set before the first Send(). HTTP1::MessageLimits limits; // How long to wait for the next piece of a response before giving @@ -97,56 +162,5 @@ namespace Crafter { struct Impl; std::unique_ptr impl; }; - - // What FetchHTTP1 needs beyond the URL. - export struct FetchOptionsHTTP1 { - std::string method = "GET"; - // Lowercase names, as on HTTPRequest. Host comes from the URL. - std::unordered_map headers; - std::string body; - // As on ClientHTTP1, for every request of the chain. - std::chrono::milliseconds connectTimeout{15000}; - std::chrono::milliseconds handshakeTimeout{15000}; - std::chrono::milliseconds timeout{30000}; - HTTP1::MessageLimits limits; - // For every https:// request of the chain. - TLSClientCredentials credentials; - // Python urllib's limits: a chain stops at a redirect to a URL it has - // already been sent to `maxRepeats` times, or once it has been sent - // to `maxRedirects` different URLs. - std::uint32_t maxRedirects = 10; - std::uint32_t maxRepeats = 4; - }; - - export struct FetchResultHTTP1 { - // The last response of the chain. A redirect only when it had nothing - // to follow — no Location, a method urllib does not redirect, a - // scheme other than http(s) — or when the limits stopped the chain. - HTTPResponse response; - // The URL that answered with `response`. - std::string url; - // Redirects followed to get there. - std::uint32_t redirects = 0; - // The limits stopped the chain; `response` is the redirect that would - // have gone past them. - bool limited = false; - }; - - // One request to an http:// or https:// URL, following redirects the way - // Python's urllib does, so a port of urllib code keeps its behaviour: - // 301, 302, 303, 307 and 308 are followed for GET and HEAD, the first - // three also for POST; the redirected request is a GET (a HEAD stays a - // HEAD) without the body and its content headers. A relative Location is - // resolved against the URL that sent it, as urljoin resolves it; a - // redirect may switch between http:// and https://. Each redirect's body - // is read before the next request goes out. Proxies are not consulted. - // - // The URL is `scheme://[user@]host[:port][/path][?query][#fragment]`, - // with an IPv6 host in brackets; the fragment is not sent and the user - // part is dropped. Throws std::invalid_argument for a URL that is not - // http(s) or has no host or a bad port, and whatever a request throws - // (SocketErrorException, SocketTimeoutException, TLSException, - // std::runtime_error) otherwise. - export FetchResultHTTP1 FetchHTTP1(std::string_view url, const FetchOptionsHTTP1& options = {}); } #endif diff --git a/interfaces/Crafter.Network-URL.cppm b/interfaces/Crafter.Network-URL.cppm new file mode 100644 index 0000000..59f6808 --- /dev/null +++ b/interfaces/Crafter.Network-URL.cppm @@ -0,0 +1,271 @@ +//SPDX-License-Identifier: LGPL-3.0-only +//SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts® + +export module Crafter.Network:URL; +import std; + +#ifndef CRAFTER_NETWORK_BROWSER +namespace Crafter { + namespace URLDetail { + // The components RFC 3986 appendix B cuts a URI reference into; a + // reference may lack the scheme and the authority, a URL may not. + struct Parts { + std::optional scheme; + std::optional authority; + std::string path; + std::optional query; + std::optional fragment; + }; + + inline bool IsAsciiAlpha(char c) { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z'); + } + + inline bool IsSchemeChar(char c) { + return IsAsciiAlpha(c) || (c >= '0' && c <= '9') || c == '+' || c == '-' || c == '.'; + } + + inline std::string Lowered(std::string_view text) { + std::string lower(text); + for (char& c : lower) { + if (c >= 'A' && c <= 'Z') c = static_cast(c - 'A' + 'a'); + } + return lower; + } + + // ^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?, with the + // scheme taken only when it is one by RFC 3986 §3.1. + inline Parts Split(std::string_view text) { + Parts parts; + const std::size_t mark = text.find_first_of(":/?#"); + if (mark != std::string_view::npos && mark > 0 && text[mark] == ':' && IsAsciiAlpha(text[0]) + && std::ranges::all_of(text.substr(0, mark), IsSchemeChar)) { + parts.scheme = Lowered(text.substr(0, mark)); + text.remove_prefix(mark + 1); + } + if (text.starts_with("//")) { + text.remove_prefix(2); + std::size_t end = text.find_first_of("/?#"); + if (end == std::string_view::npos) end = text.size(); + parts.authority = std::string(text.substr(0, end)); + text.remove_prefix(end); + } + std::size_t end = text.find_first_of("?#"); + if (end == std::string_view::npos) end = text.size(); + parts.path = std::string(text.substr(0, end)); + text.remove_prefix(end); + if (text.starts_with('?')) { + text.remove_prefix(1); + std::size_t hash = text.find('#'); + if (hash == std::string_view::npos) hash = text.size(); + parts.query = std::string(text.substr(0, hash)); + text.remove_prefix(hash); + } + if (text.starts_with('#')) parts.fragment = std::string(text.substr(1)); + return parts; + } + + inline void RemoveLastSegment(std::string& path) { + const std::size_t slash = path.rfind('/'); + path.erase(slash == std::string::npos ? 0 : slash); + } + + // RFC 3986 §5.2.4. + inline std::string RemoveDotSegments(std::string_view path) { + std::string in(path); + std::string out; + while (!in.empty()) { + if (in.starts_with("../")) { + in.erase(0, 3); + } else if (in.starts_with("./")) { + in.erase(0, 2); + } else if (in.starts_with("/./")) { + in.erase(0, 2); + } else if (in == "/.") { + in = "/"; + } else if (in.starts_with("/../")) { + in.erase(0, 3); + RemoveLastSegment(out); + } else if (in == "/..") { + in = "/"; + RemoveLastSegment(out); + } else if (in == "." || in == "..") { + in.clear(); + } else { + std::size_t next = in.find('/', in.starts_with('/') ? 1 : 0); + if (next == std::string::npos) next = in.size(); + out += in.substr(0, next); + in.erase(0, next); + } + } + return out; + } + + // RFC 3986 §5.2.3. + inline std::string Merge(bool baseHasAuthority, std::string_view basePath, std::string_view referencePath) { + if (baseHasAuthority && basePath.empty()) return "/" + std::string(referencePath); + const std::size_t slash = basePath.rfind('/'); + if (slash == std::string_view::npos) return std::string(referencePath); + return std::string(basePath.substr(0, slash + 1)) + std::string(referencePath); + } + } + + // An absolute URL, as RFC 3986 reads it: + // `scheme://[userinfo@]host[:port][/path][?query][#fragment]`. What an + // HTTP client needs of URLs: the parse, the resolution of the reference + // a redirect carries against the URL that sent it, the recomposition, + // and the pieces a request is built from. + export struct URL { + // Lowercased. + std::string scheme; + std::string userinfo; + // Lowercased; an IPv6 literal without its brackets. + std::string host; + // None when the URL names no port: the scheme's default. + std::optional port; + // Empty, or starting with '/'. + std::string path; + // Without the '?'; none when the URL has no query (an empty query, + // `?` and nothing after it, is ""). + std::optional query; + // Without the '#'. + std::optional fragment; + + // Throws std::invalid_argument for text without a scheme or a host, + // a port that is not a number or is above 65535, and unbalanced + // brackets around an IPv6 host. + static URL Parse(std::string_view text); + + // RFC 3986 §5.2 resolution of a reference against this URL as the + // base: an absolute reference stands as it is, a relative one takes + // the parts it lacks from the base, and dot segments are removed. + // Throws std::invalid_argument when the result is no URL (a + // reference such as `mailto:x` has no host). + URL Resolve(std::string_view reference) const; + + std::string ToString() const; + + bool Secure() const { + return scheme == "https"; + } + + // The port, or the scheme's default: 443 for https, 80 for http, 0 + // for any other scheme. + std::uint16_t PortOrDefault() const { + if (port) return *port; + return scheme == "https" ? 443 : scheme == "http" ? 80 : 0; + } + + // host[:port] as the Host header carries it: brackets around an IPv6 + // literal, the port only when the URL names one. + std::string Authority() const { + std::string authority = host.contains(':') ? "[" + host + "]" : host; + if (port) authority += ":" + std::to_string(*port); + return authority; + } + + // The request target: the path, "/" when there is none, and the + // query. + std::string Target() const { + std::string target = path.empty() ? std::string("/") : path; + if (query) target += "?" + *query; + return target; + } + + // The same scheme, host and effective port. + bool SameOrigin(const URL& other) const { + return scheme == other.scheme && host == other.host && PortOrDefault() == other.PortOrDefault(); + } + }; + + namespace URLDetail { + inline URL FromParts(const Parts& parts, std::string_view shown) { + if (!parts.scheme) throw std::invalid_argument("URL: no scheme in " + std::string(shown)); + if (!parts.authority) throw std::invalid_argument("URL: no host in " + std::string(shown)); + URL url; + url.scheme = *parts.scheme; + std::string hostport = *parts.authority; + if (const std::size_t at = hostport.rfind('@'); at != std::string::npos) { + url.userinfo = hostport.substr(0, at); + hostport.erase(0, at + 1); + } + std::string port; + if (hostport.starts_with('[')) { + const std::size_t close = hostport.find(']'); + if (close == std::string::npos) throw std::invalid_argument("URL: unbalanced brackets in the host of " + std::string(shown)); + url.host = Lowered(std::string_view(hostport).substr(1, close - 1)); + const std::string rest = hostport.substr(close + 1); + if (rest.starts_with(':')) { + port = rest.substr(1); + } else if (!rest.empty()) { + throw std::invalid_argument("URL: unexpected text after the IPv6 host in " + std::string(shown)); + } + } else { + if (hostport.contains(']')) throw std::invalid_argument("URL: unbalanced brackets in the host of " + std::string(shown)); + const std::size_t colon = hostport.rfind(':'); + url.host = Lowered(std::string_view(hostport).substr(0, colon)); + if (colon != std::string::npos) port = hostport.substr(colon + 1); + } + if (url.host.empty()) throw std::invalid_argument("URL: no host in " + std::string(shown)); + if (!port.empty()) { + std::uint32_t value = 0; + const auto [end, error] = std::from_chars(port.data(), port.data() + port.size(), value); + if (error != std::errc() || end != port.data() + port.size() || value > 65535) { + throw std::invalid_argument("URL: bad port '" + port + "' in " + std::string(shown)); + } + url.port = static_cast(value); + } + url.path = parts.path; + url.query = parts.query; + url.fragment = parts.fragment; + return url; + } + } + + inline URL URL::Parse(std::string_view text) { + return URLDetail::FromParts(URLDetail::Split(text), text); + } + + inline URL URL::Resolve(std::string_view reference) const { + using namespace URLDetail; + const Parts ref = Split(reference); + Parts target; + if (ref.scheme) { + target = ref; + target.path = RemoveDotSegments(ref.path); + } else { + if (ref.authority) { + target.authority = ref.authority; + target.path = RemoveDotSegments(ref.path); + target.query = ref.query; + } else { + if (ref.path.empty()) { + target.path = path; + target.query = ref.query ? ref.query : query; + } else { + target.path = RemoveDotSegments(ref.path.starts_with('/') ? ref.path : Merge(true, path, ref.path)); + target.query = ref.query; + } + // The base's authority, as it was written. + std::string authority; + if (!userinfo.empty()) authority += userinfo + "@"; + authority += Authority(); + target.authority = authority; + } + target.scheme = scheme; + target.fragment = ref.fragment; + } + return FromParts(target, reference); + } + + inline std::string URL::ToString() const { + std::string text = scheme + "://"; + if (!userinfo.empty()) text += userinfo + "@"; + text += Authority(); + text += path; + if (query) text += "?" + *query; + if (fragment) text += "#" + *fragment; + return text; + } +} +#endif diff --git a/interfaces/Crafter.Network.cppm b/interfaces/Crafter.Network.cppm index 66ea843..8eabf09 100755 --- a/interfaces/Crafter.Network.cppm +++ b/interfaces/Crafter.Network.cppm @@ -33,6 +33,9 @@ export import :HTTP1; // same way. export import :Stream; export import :TLS; +// The URL type ClientHTTP1::Fetch parses and resolves, for callers that +// build or resolve URLs themselves. +export import :URL; export import :ClientHTTP1; export import :ListenerHTTP1; #endif \ No newline at end of file diff --git a/project.cpp b/project.cpp index 1030b81..76a3c66 100644 --- a/project.cpp +++ b/project.cpp @@ -8,7 +8,7 @@ using namespace Crafter; extern "C" Configuration CrafterBuildProject(std::span args) { // TCP, HTTP/1.1 and TLS: what every native build has. - constexpr std::array networkInterfaces = { + constexpr std::array networkInterfaces = { "interfaces/Crafter.Network", "interfaces/Crafter.Network-ClientTCP", "interfaces/Crafter.Network-ListenerTCP", @@ -16,6 +16,7 @@ extern "C" Configuration CrafterBuildProject(std::span a "interfaces/Crafter.Network-HTTP1", "interfaces/Crafter.Network-Stream", "interfaces/Crafter.Network-TLS", + "interfaces/Crafter.Network-URL", "interfaces/Crafter.Network-ClientHTTP1", "interfaces/Crafter.Network-ListenerHTTP1", }; @@ -180,6 +181,7 @@ extern "C" Configuration CrafterBuildProject(std::span a cfg.AddTest("ShouldBindLoopbackEphemeral").Dependencies({ &cfg }); cfg.AddTest("ShouldTimeoutSilentHTTP1").Dependencies({ &cfg }); cfg.AddTest("ShouldFollowRedirectsHTTP1").Dependencies({ &cfg }); + cfg.AddTest("ShouldParseURLs").Dependencies({ &cfg }); cfg.AddTest("ShouldInteropCurlHTTP1").Dependencies({ &cfg }); cfg.AddTest("ShouldInteropCurlHTTPS1").Dependencies({ &cfg }); cfg.AddTest("ShouldParseHTTP1").Dependencies({ &cfg }); diff --git a/tests/ShouldFollowRedirectsHTTP1/main.cpp b/tests/ShouldFollowRedirectsHTTP1/main.cpp index c99d718..0ba456b 100644 --- a/tests/ShouldFollowRedirectsHTTP1/main.cpp +++ b/tests/ShouldFollowRedirectsHTTP1/main.cpp @@ -1,20 +1,22 @@ //SPDX-License-Identifier: LGPL-3.0-only //SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts® -// FetchHTTP1 follows redirects the way Python's urllib does, so code ported -// from urllib keeps its behaviour: -// - a relative Location is resolved against the URL that sent it (urljoin, -// `..` included), and a redirect may move from http:// to https://; -// - 301/302/303/307/308 are followed for a GET; a POST follows 301/302/303 -// as a GET without its body, and comes back with a 307 or 308 unfollowed; -// - a redirect without a Location comes back as the response; -// - the chain stops at the 5th redirect to one URL (4 repeats) or after 10 -// different URLs, returning that redirect with `limited` set; -// - a Location with a space is percent-encoded, as urllib quotes it; +// ClientHTTP1::Fetch follows redirects by RFC 9110 §15.4: +// - a relative Location is resolved against the URL that sent it (RFC +// 3986, `..` included), and a redirect may move from http:// to https://; +// - 301, 302, 303, 307 and 308 are followed for every method: 303 turns +// the request into a GET without its body, 301 and 302 do so for a POST +// alone, 307 and 308 keep the method and the body; +// - a redirect without a Location, or to another scheme, comes back as the +// response, with `stop` saying why; +// - a chain stops at a URL it has already been sent to, or after +// maxRedirects, returning that redirect; +// - Authorization and Cookie are not carried to another origin; // - a URL that is not http(s), has no host or has a bad port is refused. // // Two listeners: plaintext on 8130, TLS on 8131 with a self-signed certificate -// the client is handed as its trust anchor. +// the client is handed as its trust anchor. The cross-origin case goes from +// 127.0.0.1 to localhost, the same listener under another name. #include @@ -43,12 +45,20 @@ std::unordered_map> routes["/to-https"] = [](const HTTPRequest&) { return Redirect("307", "https://localhost:8131/secure"); }; routes["/loop"] = [](const HTTPRequest&) { return Redirect("302", "/loop"); }; routes["/nowhere"] = [](const HTTPRequest&) { return CreateResponseHTTP("302", std::string("no location")); }; + routes["/elsewhere"] = [](const HTTPRequest&) { return Redirect("302", "ftp://127.0.0.1/file"); }; routes["/form"] = [](const HTTPRequest&) { return Redirect("303", "/method"); }; + routes["/found"] = [](const HTTPRequest&) { return Redirect("302", "/method"); }; routes["/keep"] = [](const HTTPRequest&) { return Redirect("307", "/method"); }; routes["/method"] = [](const HTTPRequest& request) { - return CreateResponseHTTP("200", request.method + " body=" + request.body); + return CreateResponseHTTP("200", request.method + " body=" + request.body + " type=" + (request.headers.contains("content-type") ? request.headers.at("content-type") : "none")); + }; + routes["/auth-away"] = [](const HTTPRequest&) { return Redirect("302", "http://localhost:8130/show-auth"); }; + routes["/auth-home"] = [](const HTTPRequest&) { return Redirect("302", "/show-auth"); }; + routes["/show-auth"] = [](const HTTPRequest& request) { + return CreateResponseHTTP("200", std::string("authorization=") + (request.headers.contains("authorization") ? "yes" : "no") + + " cookie=" + (request.headers.contains("cookie") ? "yes" : "no") + + " accept=" + (request.headers.contains("accept") ? "yes" : "no")); }; - routes["/spaced"] = [](const HTTPRequest&) { return Redirect("302", "/with space"); }; return routes; } @@ -63,7 +73,7 @@ HTTPResponse Fallback(const HTTPRequest& request) { bool Refused(std::string_view url) { try { - FetchHTTP1(url); + ClientHTTP1::Fetch(url); } catch (const std::invalid_argument&) { return true; } catch (const std::exception&) { @@ -83,61 +93,81 @@ int main() { ListenerAsyncHTTP1 secure(8131, secureRoutes, TLSServerCredentials { .selfSigned = true }); { - const FetchResultHTTP1 result = FetchHTTP1("http://127.0.0.1:8130/start"); + const FetchResult result = ClientHTTP1::Fetch("http://127.0.0.1:8130/start"); Check(result.response.status == "200" && result.response.body == "/final?x=1", std::format("a relative chain arrives at /final?x=1 (got {} '{}')", result.response.status, result.response.body)); - Check(result.url == "http://127.0.0.1:8130/final?x=1", std::format("the URL that answered is reported ({})", result.url)); - Check(result.redirects == 2 && !result.limited, std::format("two redirects, not limited (got {})", result.redirects)); + Check(result.url.ToString() == "http://127.0.0.1:8130/final?x=1", std::format("the URL that answered is reported ({})", result.url.ToString())); + Check(result.redirects == 2 && result.stop == FetchStop::None, std::format("two redirects, the chain ended on a final response (got {})", result.redirects)); } { - const FetchResultHTTP1 result = FetchHTTP1("http://127.0.0.1:8130/a/b/start"); + const FetchResult result = ClientHTTP1::Fetch("http://127.0.0.1:8130/a/b/start"); Check(result.response.body == "/a/c/d", std::format("../c/d from /a/b/start resolves to /a/c/d (got '{}')", result.response.body)); } { - FetchOptionsHTTP1 options; + FetchOptions options; options.credentials.caPem = GetSelfSignedCertificatePem().certificate; - const FetchResultHTTP1 result = FetchHTTP1("http://127.0.0.1:8130/to-https", options); + const FetchResult result = ClientHTTP1::Fetch("http://127.0.0.1:8130/to-https", options); Check(result.response.status == "200" && result.response.body == "secure over https", std::format("a redirect from http:// to https:// is followed (got {} '{}')", result.response.status, result.response.body)); - Check(result.url == "https://localhost:8131/secure", std::format("and lands on the https:// URL ({})", result.url)); + Check(result.url.ToString() == "https://localhost:8131/secure" && result.url.Secure(), std::format("and lands on the https:// URL ({})", result.url.ToString())); } { - const FetchResultHTTP1 result = FetchHTTP1("http://127.0.0.1:8130/loop"); - Check(result.limited && result.response.status == "302" && result.redirects == 4, - std::format("a redirect loop stops after 4 repeats (limited {}, {} redirects)", result.limited, result.redirects)); + const FetchResult result = ClientHTTP1::Fetch("http://127.0.0.1:8130/loop"); + Check(result.stop == FetchStop::Loop && result.response.status == "302" && result.redirects == 0, + std::format("a redirect back to a URL already visited stops the chain at once (stop {}, {} redirects)", static_cast(result.stop), result.redirects)); } { - const FetchResultHTTP1 result = FetchHTTP1("http://127.0.0.1:8130/hop/0"); - Check(result.limited && result.redirects == 10 && result.url == "http://127.0.0.1:8130/hop/10", - std::format("an endless chain stops after 10 different URLs ({} redirects, at {})", result.redirects, result.url)); + const FetchResult result = ClientHTTP1::Fetch("http://127.0.0.1:8130/hop/0"); + Check(result.stop == FetchStop::Limit && result.redirects == 10 && result.url.ToString() == "http://127.0.0.1:8130/hop/10", + std::format("an endless chain stops after maxRedirects, 10 ({} redirects, at {})", result.redirects, result.url.ToString())); + FetchOptions two; + two.maxRedirects = 2; + const FetchResult shorter = ClientHTTP1::Fetch("http://127.0.0.1:8130/hop/0", two); + Check(shorter.stop == FetchStop::Limit && shorter.redirects == 2, "maxRedirects is the option's"); } { - const FetchResultHTTP1 result = FetchHTTP1("http://127.0.0.1:8130/nowhere"); - Check(result.response.status == "302" && result.redirects == 0 && !result.limited, + const FetchResult result = ClientHTTP1::Fetch("http://127.0.0.1:8130/nowhere"); + Check(result.stop == FetchStop::NoLocation && result.response.status == "302" && result.redirects == 0, "a redirect without a Location comes back as the response"); + const FetchResult other = ClientHTTP1::Fetch("http://127.0.0.1:8130/elsewhere"); + Check(other.stop == FetchStop::OtherScheme && other.response.status == "302", "a redirect to another scheme comes back as the response"); } { - FetchOptionsHTTP1 post; + FetchOptions post; post.method = "POST"; post.body = "payload"; post.headers["content-type"] = "text/plain"; - const FetchResultHTTP1 seeOther = FetchHTTP1("http://127.0.0.1:8130/form", post); - Check(seeOther.response.body == "GET body=", - std::format("a POST answered with 303 is followed as a GET without its body (got '{}')", seeOther.response.body)); - const FetchResultHTTP1 kept = FetchHTTP1("http://127.0.0.1:8130/keep", post); - Check(kept.response.status == "307" && kept.redirects == 0, "a POST answered with 307 comes back unfollowed"); + const FetchResult seeOther = ClientHTTP1::Fetch("http://127.0.0.1:8130/form", post); + Check(seeOther.response.body == "GET body= type=none", + std::format("a POST answered with 303 is followed as a GET without its body or content headers (got '{}')", seeOther.response.body)); + const FetchResult found = ClientHTTP1::Fetch("http://127.0.0.1:8130/found", post); + Check(found.response.body == "GET body= type=none", std::format("a POST answered with 302 is followed as a GET (got '{}')", found.response.body)); + const FetchResult kept = ClientHTTP1::Fetch("http://127.0.0.1:8130/keep", post); + Check(kept.response.body == "POST body=payload type=text/plain" && kept.redirects == 1, + std::format("a POST answered with 307 is followed with its method, body and content headers (got '{}')", kept.response.body)); + FetchOptions put = post; + put.method = "PUT"; + const FetchResult putFound = ClientHTTP1::Fetch("http://127.0.0.1:8130/found", put); + Check(putFound.response.body == "PUT body=payload type=text/plain", std::format("a PUT answered with 302 keeps its method and body (got '{}')", putFound.response.body)); } { - const FetchResultHTTP1 result = FetchHTTP1("http://127.0.0.1:8130/spaced"); - Check(result.response.body == "/with%20space", - std::format("a Location with a space is percent-encoded before it is followed (got '{}')", result.response.body)); + FetchOptions options; + options.headers["authorization"] = "Bearer secret"; + options.headers["cookie"] = "a=b"; + options.headers["accept"] = "text/plain"; + const FetchResult away = ClientHTTP1::Fetch("http://127.0.0.1:8130/auth-away", options); + Check(away.response.body == "authorization=no cookie=no accept=yes", + std::format("Authorization and Cookie are dropped on a redirect to another origin, other headers kept (got '{}')", away.response.body)); + const FetchResult home = ClientHTTP1::Fetch("http://127.0.0.1:8130/auth-home", options); + Check(home.response.body == "authorization=yes cookie=yes accept=yes", + std::format("and kept on a redirect within the origin (got '{}')", home.response.body)); } Check(Refused("ftp://127.0.0.1/file"), "an ftp:// URL is refused"); diff --git a/tests/ShouldParseURLs/main.cpp b/tests/ShouldParseURLs/main.cpp new file mode 100644 index 0000000..3cbaf73 --- /dev/null +++ b/tests/ShouldParseURLs/main.cpp @@ -0,0 +1,105 @@ +//SPDX-License-Identifier: LGPL-3.0-only +//SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts® + +// URL reads an absolute URL as RFC 3986 does and resolves a reference +// against it as §5.2 says, which is what ClientHTTP1::Fetch leans on to +// follow a Location. The resolution cases are RFC 3986 §5.4's own examples, +// normal and abnormal, against its base `http://a/b/c/d;p?q`; the parse +// cases guard the pieces a request is built from (the default port, the +// IPv6 brackets, the userinfo that must not reach the host) and what is +// refused. + +#include + +import Crafter.Network; +import std; +using namespace Crafter; + +namespace { + +int failures = 0; + +void Check(bool ok, std::string_view what) { + std::println("{} {}", ok ? "PASS" : "FAIL", what); + if (!ok) ++failures; +} + +bool Refused(std::string_view text) { + try { + URL::Parse(text); + } catch (const std::invalid_argument&) { + return true; + } + return false; +} + +} // namespace + +int main() { + { + const URL url = URL::Parse("HTTP://User:pw@Example.COM:8080/a/b?x=1&y#frag"); + Check(url.scheme == "http" && url.host == "example.com", "the scheme and the host are lowercased"); + Check(url.userinfo == "User:pw", "the userinfo is kept apart from the host"); + Check(url.port && *url.port == 8080 && url.PortOrDefault() == 8080, "an explicit port"); + Check(url.path == "/a/b" && url.query && *url.query == "x=1&y" && url.fragment && *url.fragment == "frag", "path, query and fragment"); + Check(url.Authority() == "example.com:8080" && url.Target() == "/a/b?x=1&y", "the Host value and the request target"); + Check(url.ToString() == "http://User:pw@example.com:8080/a/b?x=1&y#frag", std::format("recomposed ({})", url.ToString())); + } + { + const URL plain = URL::Parse("http://example.com"); + const URL secure = URL::Parse("https://example.com/"); + Check(!plain.port && plain.PortOrDefault() == 80 && secure.PortOrDefault() == 443, "the scheme's default port when none is named"); + Check(plain.path.empty() && plain.Target() == "/" && plain.Authority() == "example.com", "no path is the target `/`, and the Host value carries no port"); + Check(!plain.query && URL::Parse("http://example.com/?").query && URL::Parse("http://example.com/?").query->empty(), + "no query is none; `?` alone is an empty one"); + Check(!plain.SameOrigin(secure) && plain.SameOrigin(URL::Parse("http://EXAMPLE.com:80/other")) && !plain.SameOrigin(URL::Parse("http://example.com:8080/")), + "the origin is the scheme, the host and the effective port"); + } + { + const URL v6 = URL::Parse("http://[::1]:8130/x"); + Check(v6.host == "::1" && v6.port && *v6.port == 8130 && v6.Authority() == "[::1]:8130" && v6.ToString() == "http://[::1]:8130/x", + std::format("an IPv6 host loses its brackets in `host` and gets them back in the Host value ({})", v6.ToString())); + } + Check(Refused("example.com/path"), "text without a scheme is refused"); + Check(Refused("http:///path"), "a URL without a host is refused"); + Check(Refused("http://example.com:99999/"), "a port above 65535 is refused"); + Check(Refused("http://example.com:8x/"), "a port that is not a number is refused"); + Check(Refused("http://[::1/"), "unbalanced brackets are refused"); + Check(URL::Parse("http://example.com:/").PortOrDefault() == 80, "an empty port is the default"); + + // RFC 3986 §5.4. + const URL base = URL::Parse("http://a/b/c/d;p?q"); + const std::vector> cases = { + {"g", "http://a/b/c/g"}, {"./g", "http://a/b/c/g"}, {"g/", "http://a/b/c/g/"}, {"/g", "http://a/g"}, {"//g", "http://g"}, + {"?y", "http://a/b/c/d;p?y"}, {"g?y", "http://a/b/c/g?y"}, {"#s", "http://a/b/c/d;p?q#s"}, {"g#s", "http://a/b/c/g#s"}, + {"g?y#s", "http://a/b/c/g?y#s"}, {";x", "http://a/b/c/;x"}, {"g;x", "http://a/b/c/g;x"}, {"g;x?y#s", "http://a/b/c/g;x?y#s"}, + {"", "http://a/b/c/d;p?q"}, {".", "http://a/b/c/"}, {"./", "http://a/b/c/"}, {"..", "http://a/b/"}, {"../", "http://a/b/"}, + {"../g", "http://a/b/g"}, {"../..", "http://a/"}, {"../../", "http://a/"}, {"../../g", "http://a/g"}, + // §5.4.2, the abnormal ones. + {"../../../g", "http://a/g"}, {"../../../../g", "http://a/g"}, {"/./g", "http://a/g"}, {"/../g", "http://a/g"}, + {"g.", "http://a/b/c/g."}, {".g", "http://a/b/c/.g"}, {"g..", "http://a/b/c/g.."}, {"..g", "http://a/b/c/..g"}, + {"./../g", "http://a/b/g"}, {"./g/.", "http://a/b/c/g/"}, {"g/./h", "http://a/b/c/g/h"}, {"g/../h", "http://a/b/c/h"}, + {"g;x=1/./y", "http://a/b/c/g;x=1/y"}, {"g;x=1/../y", "http://a/b/c/y"}, {"g?y/./x", "http://a/b/c/g?y/./x"}, + {"g?y/../x", "http://a/b/c/g?y/../x"}, {"g#s/./x", "http://a/b/c/g#s/./x"}, {"g#s/../x", "http://a/b/c/g#s/../x"}, + {"https://b/x", "https://b/x"}, + }; + for (const auto& [reference, expected] : cases) { + const std::string resolved = base.Resolve(reference).ToString(); + Check(resolved == expected, std::format("'{}' against the base resolves to {} (got {})", reference, expected, resolved)); + } + { + const URL from = URL::Parse("http://127.0.0.1:8130/a/b/start"); + Check(from.Resolve("../c/d").ToString() == "http://127.0.0.1:8130/a/c/d", "a port and a userinfo-free base keep their authority through a resolve"); + Check(URL::Parse("http://u@h:81/p").Resolve("x").ToString() == "http://u@h:81/x", "the base's userinfo stays"); + bool refused = false; + try { + from.Resolve("mailto:someone@example.com"); + } catch (const std::invalid_argument&) { + refused = true; + } + Check(refused, "a reference that resolves to no URL (mailto:) is refused"); + } + + std::println("{}", failures == 0 ? "all checks passed" : "check(s) failed"); + return failures == 0 ? EXIT_SUCCESS : EXIT_FAILURE; +}