feat(http1): bound the connect, and FetchHTTP1 for a URL with redirects
- ClientHTTP1 gains connectTimeout (default 15 s), applied to each address the host resolves to. `timeout` only ever bounded the waits for a response, so a host that drops the SYNs held Send() for the kernel's retries, about two minutes, whatever the caller had asked for. - The default Host header puts an IPv6 literal in brackets. - FetchHTTP1(url, options) makes one request to an http:// or https:// URL and follows redirects as Python's urllib does, so urllib code ported over keeps its behaviour: 301/302/303/307/308 for GET and HEAD, 301/302/303 for POST (re-sent as a GET without its body), a relative Location resolved as urljoin resolves it and percent-encoded as urllib quotes it, http and https hops mixed, each redirect's body read before the next request, and the chain stopped by urllib's limits (4 repeats of one URL, 10 different URLs) with the last redirect returned and `limited` set. Proxies are not consulted. New: ShouldTimeoutSilentHTTP1, ShouldFollowRedirectsHTTP1. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
d6f8f4edcb
commit
f9a45c143e
GPG key ID:
2937E59CDCC1BCFB
6 changed files with 649 additions and 6 deletions
153
tests/ShouldFollowRedirectsHTTP1/main.cpp
Normal file
153
tests/ShouldFollowRedirectsHTTP1/main.cpp
Normal file
|
|
@ -0,0 +1,153 @@
|
|||
//SPDX-License-Identifier: LGPL-3.0-only
|
||||
//SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts®
|
||||
|
||||
// FetchHTTP1 follows redirects the way Python's urllib does, so code ported
|
||||
// from urllib keeps its behaviour:
|
||||
// - a relative Location is resolved against the URL that sent it (urljoin,
|
||||
// `..` included), and a redirect may move from http:// to https://;
|
||||
// - 301/302/303/307/308 are followed for a GET; a POST follows 301/302/303
|
||||
// as a GET without its body, and comes back with a 307 or 308 unfollowed;
|
||||
// - a redirect without a Location comes back as the response;
|
||||
// - the chain stops at the 5th redirect to one URL (4 repeats) or after 10
|
||||
// different URLs, returning that redirect with `limited` set;
|
||||
// - a Location with a space is percent-encoded, as urllib quotes it;
|
||||
// - a URL that is not http(s), has no host or has a bad port is refused.
|
||||
//
|
||||
// Two listeners: plaintext on 8130, TLS on 8131 with a self-signed certificate
|
||||
// the client is handed as its trust anchor.
|
||||
|
||||
#include <cstdlib>
|
||||
|
||||
import Crafter.Network;
|
||||
import std;
|
||||
using namespace Crafter;
|
||||
|
||||
namespace {
|
||||
|
||||
int failures = 0;
|
||||
|
||||
void Check(bool ok, std::string_view what) {
|
||||
std::println("{} {}", ok ? "PASS" : "FAIL", what);
|
||||
if (!ok) ++failures;
|
||||
}
|
||||
|
||||
HTTPResponse Redirect(std::string status, std::string location) {
|
||||
return CreateResponseHTTP(std::move(status), {{"location", std::move(location)}}, std::string());
|
||||
}
|
||||
|
||||
std::unordered_map<std::string, std::function<HTTPResponse(const HTTPRequest&)>> Routes() {
|
||||
std::unordered_map<std::string, std::function<HTTPResponse(const HTTPRequest&)>> routes;
|
||||
routes["/start"] = [](const HTTPRequest&) { return Redirect("302", "next"); };
|
||||
routes["/next"] = [](const HTTPRequest&) { return Redirect("301", "/final?x=1"); };
|
||||
routes["/a/b/start"] = [](const HTTPRequest&) { return Redirect("302", "../c/d"); };
|
||||
routes["/to-https"] = [](const HTTPRequest&) { return Redirect("307", "https://localhost:8131/secure"); };
|
||||
routes["/loop"] = [](const HTTPRequest&) { return Redirect("302", "/loop"); };
|
||||
routes["/nowhere"] = [](const HTTPRequest&) { return CreateResponseHTTP("302", std::string("no location")); };
|
||||
routes["/form"] = [](const HTTPRequest&) { return Redirect("303", "/method"); };
|
||||
routes["/keep"] = [](const HTTPRequest&) { return Redirect("307", "/method"); };
|
||||
routes["/method"] = [](const HTTPRequest& request) {
|
||||
return CreateResponseHTTP("200", request.method + " body=" + request.body);
|
||||
};
|
||||
routes["/spaced"] = [](const HTTPRequest&) { return Redirect("302", "/with space"); };
|
||||
return routes;
|
||||
}
|
||||
|
||||
// Everything else echoes its target, and /hop/<n> redirects to /hop/<n+1>.
|
||||
HTTPResponse Fallback(const HTTPRequest& request) {
|
||||
if (request.path.starts_with("/hop/")) {
|
||||
const std::int32_t next = std::stoi(request.path.substr(5)) + 1;
|
||||
return Redirect("302", std::format("/hop/{}", next));
|
||||
}
|
||||
return CreateResponseHTTP("200", request.path);
|
||||
}
|
||||
|
||||
bool Refused(std::string_view url) {
|
||||
try {
|
||||
FetchHTTP1(url);
|
||||
} catch (const std::invalid_argument&) {
|
||||
return true;
|
||||
} catch (const std::exception&) {
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
int main() {
|
||||
try {
|
||||
ListenerAsyncHTTP1 plain(8130, Routes(), Fallback);
|
||||
std::unordered_map<std::string, std::function<HTTPResponse(const HTTPRequest&)>> secureRoutes;
|
||||
secureRoutes["/secure"] = [](const HTTPRequest& request) {
|
||||
return CreateResponseHTTP("200", "secure over " + request.scheme);
|
||||
};
|
||||
ListenerAsyncHTTP1 secure(8131, secureRoutes, TLSServerCredentials { .selfSigned = true });
|
||||
|
||||
{
|
||||
const FetchResultHTTP1 result = FetchHTTP1("http://127.0.0.1:8130/start");
|
||||
Check(result.response.status == "200" && result.response.body == "/final?x=1",
|
||||
std::format("a relative chain arrives at /final?x=1 (got {} '{}')", result.response.status, result.response.body));
|
||||
Check(result.url == "http://127.0.0.1:8130/final?x=1", std::format("the URL that answered is reported ({})", result.url));
|
||||
Check(result.redirects == 2 && !result.limited, std::format("two redirects, not limited (got {})", result.redirects));
|
||||
}
|
||||
|
||||
{
|
||||
const FetchResultHTTP1 result = FetchHTTP1("http://127.0.0.1:8130/a/b/start");
|
||||
Check(result.response.body == "/a/c/d", std::format("../c/d from /a/b/start resolves to /a/c/d (got '{}')", result.response.body));
|
||||
}
|
||||
|
||||
{
|
||||
FetchOptionsHTTP1 options;
|
||||
options.credentials.caPem = GetSelfSignedCertificatePem().certificate;
|
||||
const FetchResultHTTP1 result = FetchHTTP1("http://127.0.0.1:8130/to-https", options);
|
||||
Check(result.response.status == "200" && result.response.body == "secure over https",
|
||||
std::format("a redirect from http:// to https:// is followed (got {} '{}')", result.response.status, result.response.body));
|
||||
Check(result.url == "https://localhost:8131/secure", std::format("and lands on the https:// URL ({})", result.url));
|
||||
}
|
||||
|
||||
{
|
||||
const FetchResultHTTP1 result = FetchHTTP1("http://127.0.0.1:8130/loop");
|
||||
Check(result.limited && result.response.status == "302" && result.redirects == 4,
|
||||
std::format("a redirect loop stops after 4 repeats (limited {}, {} redirects)", result.limited, result.redirects));
|
||||
}
|
||||
|
||||
{
|
||||
const FetchResultHTTP1 result = FetchHTTP1("http://127.0.0.1:8130/hop/0");
|
||||
Check(result.limited && result.redirects == 10 && result.url == "http://127.0.0.1:8130/hop/10",
|
||||
std::format("an endless chain stops after 10 different URLs ({} redirects, at {})", result.redirects, result.url));
|
||||
}
|
||||
|
||||
{
|
||||
const FetchResultHTTP1 result = FetchHTTP1("http://127.0.0.1:8130/nowhere");
|
||||
Check(result.response.status == "302" && result.redirects == 0 && !result.limited,
|
||||
"a redirect without a Location comes back as the response");
|
||||
}
|
||||
|
||||
{
|
||||
FetchOptionsHTTP1 post;
|
||||
post.method = "POST";
|
||||
post.body = "payload";
|
||||
post.headers["content-type"] = "text/plain";
|
||||
const FetchResultHTTP1 seeOther = FetchHTTP1("http://127.0.0.1:8130/form", post);
|
||||
Check(seeOther.response.body == "GET body=",
|
||||
std::format("a POST answered with 303 is followed as a GET without its body (got '{}')", seeOther.response.body));
|
||||
const FetchResultHTTP1 kept = FetchHTTP1("http://127.0.0.1:8130/keep", post);
|
||||
Check(kept.response.status == "307" && kept.redirects == 0, "a POST answered with 307 comes back unfollowed");
|
||||
}
|
||||
|
||||
{
|
||||
const FetchResultHTTP1 result = FetchHTTP1("http://127.0.0.1:8130/spaced");
|
||||
Check(result.response.body == "/with%20space",
|
||||
std::format("a Location with a space is percent-encoded before it is followed (got '{}')", result.response.body));
|
||||
}
|
||||
|
||||
Check(Refused("ftp://127.0.0.1/file"), "an ftp:// URL is refused");
|
||||
Check(Refused("http:///path"), "a URL without a host is refused");
|
||||
Check(Refused("http://127.0.0.1:99999/"), "a port above 65535 is refused");
|
||||
Check(Refused("http://127.0.0.1:8x/"), "a port that is not a number is refused");
|
||||
} catch (const std::exception& e) {
|
||||
Check(false, std::format("unexpected exception: {}", e.what()));
|
||||
}
|
||||
|
||||
std::println("{}", failures == 0 ? "all checks passed" : "check(s) failed");
|
||||
return failures == 0 ? EXIT_SUCCESS : EXIT_FAILURE;
|
||||
}
|
||||
109
tests/ShouldTimeoutSilentHTTP1/main.cpp
Normal file
109
tests/ShouldTimeoutSilentHTTP1/main.cpp
Normal file
|
|
@ -0,0 +1,109 @@
|
|||
//SPDX-License-Identifier: LGPL-3.0-only
|
||||
//SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts®
|
||||
|
||||
// ClientHTTP1 bounds the TCP connect (connectTimeout) as well as every wait
|
||||
// for a response (timeout). Before, the connect had no bound of its own: a
|
||||
// host that drops the SYNs held Send() for the kernel's retries, about two
|
||||
// minutes, whatever `timeout` said — a readiness probe with a ten-second
|
||||
// budget could hang its caller for two minutes.
|
||||
//
|
||||
// Two servers on loopback. One accepts — the kernel completes the handshake
|
||||
// from its backlog — and never answers: the read timeout. The other has a
|
||||
// full accept queue, so Linux drops the SYNs of further connects: the connect
|
||||
// timeout.
|
||||
|
||||
#include <arpa/inet.h>
|
||||
#include <netinet/in.h>
|
||||
#include <sys/socket.h>
|
||||
#include <unistd.h>
|
||||
#include <cstdlib>
|
||||
|
||||
import Crafter.Network;
|
||||
import std;
|
||||
using namespace Crafter;
|
||||
using namespace std::chrono_literals;
|
||||
|
||||
namespace {
|
||||
|
||||
int failures = 0;
|
||||
|
||||
void Check(bool ok, std::string_view what) {
|
||||
std::println("{} {}", ok ? "PASS" : "FAIL", what);
|
||||
if (!ok) ++failures;
|
||||
}
|
||||
|
||||
// A socket listening on 127.0.0.1, on a port the kernel picks.
|
||||
struct LoopbackListener {
|
||||
int fd = -1;
|
||||
std::uint16_t port = 0;
|
||||
|
||||
explicit LoopbackListener(int backlog) {
|
||||
fd = socket(AF_INET, SOCK_STREAM, 0);
|
||||
sockaddr_in address {};
|
||||
address.sin_family = AF_INET;
|
||||
address.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
|
||||
bind(fd, reinterpret_cast<sockaddr*>(&address), sizeof(address));
|
||||
listen(fd, backlog);
|
||||
socklen_t length = sizeof(address);
|
||||
getsockname(fd, reinterpret_cast<sockaddr*>(&address), &length);
|
||||
port = ntohs(address.sin_port);
|
||||
}
|
||||
~LoopbackListener() { close(fd); }
|
||||
};
|
||||
|
||||
long long Milliseconds(std::chrono::steady_clock::duration duration) {
|
||||
return std::chrono::duration_cast<std::chrono::milliseconds>(duration).count();
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
int main() {
|
||||
{
|
||||
LoopbackListener silent(8);
|
||||
ClientHTTP1 client("127.0.0.1", silent.port);
|
||||
client.timeout = 300ms;
|
||||
const auto start = std::chrono::steady_clock::now();
|
||||
bool failed = false;
|
||||
try {
|
||||
client.Send(CreateRequestHTTP("GET", "/", "127.0.0.1"));
|
||||
} catch (const std::exception&) {
|
||||
failed = true;
|
||||
}
|
||||
const auto waited = std::chrono::steady_clock::now() - start;
|
||||
Check(failed, "a server that accepts and never answers fails the request");
|
||||
Check(waited >= 280ms && waited < 5s, std::format("after about `timeout` ({} ms)", Milliseconds(waited)));
|
||||
}
|
||||
|
||||
{
|
||||
// Connects queue up until the accept queue is full, which with a
|
||||
// backlog of 0 is after one or two; the next one goes unanswered.
|
||||
LoopbackListener full(0);
|
||||
std::vector<ClientTCP> queued;
|
||||
bool queueFull = false;
|
||||
for (int i = 0; i < 8 && !queueFull; ++i) {
|
||||
try {
|
||||
queued.emplace_back("127.0.0.1", full.port, 200ms);
|
||||
} catch (const SocketTimeoutException&) {
|
||||
queueFull = true;
|
||||
}
|
||||
}
|
||||
Check(queueFull, std::format("the accept queue fills up (after {} connects)", queued.size()));
|
||||
|
||||
ClientHTTP1 client("127.0.0.1", full.port);
|
||||
client.connectTimeout = 300ms;
|
||||
const auto start = std::chrono::steady_clock::now();
|
||||
bool timedOut = false;
|
||||
try {
|
||||
client.Send(CreateRequestHTTP("GET", "/", "127.0.0.1"));
|
||||
} catch (const SocketTimeoutException&) {
|
||||
timedOut = true;
|
||||
} catch (const std::exception&) {
|
||||
}
|
||||
const auto waited = std::chrono::steady_clock::now() - start;
|
||||
Check(timedOut, "a connect nobody answers throws SocketTimeoutException");
|
||||
Check(waited >= 280ms && waited < 5s, std::format("after about `connectTimeout` ({} ms)", Milliseconds(waited)));
|
||||
}
|
||||
|
||||
std::println("{}", failures == 0 ? "all checks passed" : "check(s) failed");
|
||||
return failures == 0 ? EXIT_SUCCESS : EXIT_FAILURE;
|
||||
}
|
||||
Loading…
Reference in a new issue