feat(seatpool): the seat pool and the board several drivers share

The seats of .env and the ledger, the probe, placement, settling a
finished pass, the child's credentials, the OpenRouter seats, and the
board every driver on the machine shares, in one shape and under one
lock. A driver is alive while its lifetime lock is held; with no lock file
natively its pid and start time decide, and in the sandbox it is taken for
alive.

:Http is the HTTP the driver needs, on FetchHTTP1: a status outside 2xx is
HttpError, every other failure carries a fixed text. :Platform gains
MakeDirs, with the last directory's own mode, and errno texts for its
writes and locks.

The tests cover the seat pool and several drivers at once, less the checks
that drive the pool's Scheduler, Admission and Stacks.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Jorijn van der Graaf 2026-10-31 08:15:03 +01:00
commit c63c898833
Signed by: jorijnvdgraaf
GPG key ID: 2937E59CDCC1BCFB
10 changed files with 2562 additions and 8 deletions

View file

@ -0,0 +1,108 @@
// SPDX-License-Identifier: AGPL-3.0-only
// SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts®
module;
#include <netdb.h>
module Crafter.Review:Http_impl;
import :Http;
import :Json;
import :Text;
import Crafter.Network;
import std;
namespace Crafter::Review::Http {
namespace {
std::string Wrapped(std::string_view reason) {
return std::format("<urlopen error {}>", reason);
}
// ClientTCP's resolve, socket() and connect failures, which happen
// while the request goes out, apart from a send's or a receive's.
bool WhileSending(const SocketErrorException& failure) {
const std::string_view text = failure.what();
return failure.kind == SocketErrorKind::Resolver || text.starts_with("Could not connect") || text.starts_with("Could not open socket");
}
std::string SocketText(const SocketErrorException& failure) {
if (failure.kind == SocketErrorKind::Resolver) return std::format("[Errno {}] {}", failure.code, gai_strerror(failure.code));
return Dyn::ErrnoText(failure.code);
}
constexpr std::array<std::pair<std::int32_t, std::string_view>, 62> Phrases = {{
{100, "Continue"}, {101, "Switching Protocols"}, {102, "Processing"}, {103, "Early Hints"}, {200, "OK"}, {201, "Created"}, {202, "Accepted"},
{203, "Non-Authoritative Information"}, {204, "No Content"}, {205, "Reset Content"}, {206, "Partial Content"}, {207, "Multi-Status"},
{208, "Already Reported"}, {226, "IM Used"}, {300, "Multiple Choices"}, {301, "Moved Permanently"}, {302, "Found"}, {303, "See Other"},
{304, "Not Modified"}, {305, "Use Proxy"}, {307, "Temporary Redirect"}, {308, "Permanent Redirect"}, {400, "Bad Request"}, {401, "Unauthorized"},
{402, "Payment Required"}, {403, "Forbidden"}, {404, "Not Found"}, {405, "Method Not Allowed"}, {406, "Not Acceptable"},
{407, "Proxy Authentication Required"}, {408, "Request Timeout"}, {409, "Conflict"}, {410, "Gone"}, {411, "Length Required"},
{412, "Precondition Failed"}, {413, "Content Too Large"}, {414, "URI Too Long"}, {415, "Unsupported Media Type"}, {416, "Range Not Satisfiable"},
{417, "Expectation Failed"}, {418, "I'm a Teapot"}, {421, "Misdirected Request"}, {422, "Unprocessable Content"}, {423, "Locked"},
{424, "Failed Dependency"}, {425, "Too Early"}, {426, "Upgrade Required"}, {428, "Precondition Required"}, {429, "Too Many Requests"},
{431, "Request Header Fields Too Large"}, {451, "Unavailable For Legal Reasons"}, {500, "Internal Server Error"}, {501, "Not Implemented"},
{502, "Bad Gateway"}, {503, "Service Unavailable"}, {504, "Gateway Timeout"}, {505, "HTTP Version Not Supported"}, {506, "Variant Also Negotiates"},
{507, "Insufficient Storage"}, {508, "Loop Detected"}, {510, "Not Extended"}, {511, "Network Authentication Required"},
}};
}
HttpError::HttpError(std::int32_t code, const std::string& message) : std::runtime_error(std::format("HTTP Error {}: {}", code, message)), code(code) {}
HttpError::HttpError(std::int32_t code) : HttpError(code, std::string(Reason(code))) {}
std::int32_t HttpError::Code() const {
return code;
}
std::string_view Reason(std::int32_t code) {
for (const auto& [value, phrase] : Phrases) {
if (value == code) return phrase;
}
return "";
}
Response UrlOpen(std::string_view url, const std::map<std::string, std::string>& headers, double timeout) {
FetchOptionsHTTP1 options;
options.headers = {{"accept-encoding", "identity"}, {"connection", "close"}, {"user-agent", "crafter-review"}};
for (const auto& [name, value] : headers) options.headers[Dyn::Lower(name)] = value;
// The timeout bounds each blocking step: the connect, the handshake,
// every read.
const std::chrono::milliseconds budget(static_cast<std::int64_t>(timeout * 1000));
options.connectTimeout = budget;
options.handshakeTimeout = budget;
options.timeout = budget;
FetchResultHTTP1 result;
try {
result = FetchHTTP1(url, options);
} catch (const SocketTimeoutException&) {
throw Error(Wrapped("timed out"));
} catch (const SocketErrorException& failure) {
throw Error(WhileSending(failure) ? Wrapped(SocketText(failure)) : SocketText(failure));
} catch (const SocketClosedException&) {
throw Error("Remote end closed connection without response");
} catch (const TLSException& failure) {
throw Error(Wrapped(failure.what()));
} catch (const std::invalid_argument& failure) {
throw Error(failure.what());
} catch (const std::runtime_error& failure) {
const std::string_view text = failure.what();
if (text.starts_with("timed out")) throw Error("timed out");
if (text.starts_with("connection closed")) throw Error("Remote end closed connection without response");
throw Error(failure.what());
} catch (const std::exception& failure) {
throw Error(failure.what());
}
const std::optional<std::int64_t> status = Dyn::ParseInt(result.response.status);
const auto code = static_cast<std::int32_t>(status.value_or(0));
if (result.limited) {
throw HttpError(code, std::format("The HTTP server returned a redirect error that would lead to an infinite loop.\nThe last 30x error message was:\n{}", Reason(code)));
}
if (code < 200 || code >= 300) throw HttpError(code);
return {code, std::move(result.response.body)};
}
Json JsonBody(const Response& response) {
if (const std::optional<std::string> error = Dyn::Utf8DecodeError(response.body)) throw Dyn::Error("UnicodeDecodeError", *error);
if (const std::optional<std::string> error = Dyn::JsonDecodeError(response.body)) throw Dyn::Error("JSONDecodeError", *error);
return ParseLoose(response.body);
}
}

View file

@ -169,7 +169,7 @@ namespace Crafter::Review::Platform {
if (path.has_parent_path()) fs::create_directories(path.parent_path()); if (path.has_parent_path()) fs::create_directories(path.parent_path());
const fs::path temp = std::format("{}.{}.tmp", path.string(), PublishedPid()); const fs::path temp = std::format("{}.{}.tmp", path.string(), PublishedPid());
const std::int32_t fd = open(temp.c_str(), O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC, static_cast<mode_t>(mode)); const std::int32_t fd = open(temp.c_str(), O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC, static_cast<mode_t>(mode));
if (fd < 0) throw std::runtime_error(std::format("{}: {}", temp.string(), Dyn::ErrnoText(errno))); if (fd < 0) throw std::runtime_error(std::format("{}: {}", Dyn::ErrnoText(errno), Dyn::Repr(temp.string())));
std::size_t written = 0; std::size_t written = 0;
while (written < content.size()) { while (written < content.size()) {
const ssize_t step = write(fd, content.data() + written, content.size() - written); const ssize_t step = write(fd, content.data() + written, content.size() - written);
@ -178,7 +178,7 @@ namespace Crafter::Review::Platform {
const std::int32_t failure = errno; const std::int32_t failure = errno;
close(fd); close(fd);
unlink(temp.c_str()); unlink(temp.c_str());
throw std::runtime_error(std::format("{}: {}", temp.string(), Dyn::ErrnoText(failure))); throw std::runtime_error(Dyn::ErrnoText(failure));
} }
written += static_cast<std::size_t>(step); written += static_cast<std::size_t>(step);
} }
@ -186,7 +186,21 @@ namespace Crafter::Review::Platform {
if (rename(temp.c_str(), path.c_str()) != 0) { if (rename(temp.c_str(), path.c_str()) != 0) {
const std::int32_t failure = errno; const std::int32_t failure = errno;
unlink(temp.c_str()); unlink(temp.c_str());
throw std::runtime_error(std::format("{}: {}", path.string(), Dyn::ErrnoText(failure))); throw std::runtime_error(std::format("{}: {} -> {}", Dyn::ErrnoText(failure), Dyn::Repr(temp.string()), Dyn::Repr(path.string())));
}
}
void MakeDirs(const fs::path& path, std::uint32_t mode) {
const fs::path name = path.filename().empty() && path.has_parent_path() ? path.parent_path() : path;
const fs::path head = name.parent_path();
std::error_code error;
if (!head.empty() && head != name && !fs::exists(head, error)) {
MakeDirs(head, 0777);
if (name.filename() == ".") return;
}
if (mkdir(name.c_str(), static_cast<mode_t>(mode)) != 0) {
const std::int32_t failure = errno;
if (!fs::is_directory(name, error)) throw std::runtime_error(std::format("{}: {}", Dyn::ErrnoText(failure), Dyn::Repr(name.string())));
} }
} }
@ -195,7 +209,7 @@ namespace Crafter::Review::Platform {
if (lockPath.has_parent_path()) fs::create_directories(lockPath.parent_path(), error); if (lockPath.has_parent_path()) fs::create_directories(lockPath.parent_path(), error);
fd = open(lockPath.c_str(), O_WRONLY | O_CREAT | O_APPEND | O_CLOEXEC, 0666); fd = open(lockPath.c_str(), O_WRONLY | O_CREAT | O_APPEND | O_CLOEXEC, 0666);
if (fd < 0) { if (fd < 0) {
problem = Dyn::ErrnoText(errno); problem = std::format("{}: {}", Dyn::ErrnoText(errno), Dyn::Repr(lockPath.string()));
return; return;
} }
while (flock(fd, LOCK_EX) != 0) { while (flock(fd, LOCK_EX) != 0) {
@ -230,7 +244,7 @@ namespace Crafter::Review::Platform {
} }
Json data; Json data;
try { try {
if (const std::optional<std::string> text = ReadFile(path)) data = Json::parse(*text); if (const std::optional<std::string> text = ReadFile(path)) data = ParseLoose(*text);
} catch (const std::exception&) { } catch (const std::exception&) {
data = Json(); data = Json();
} }

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,49 @@
// SPDX-License-Identifier: AGPL-3.0-only
// SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts®
export module Crafter.Review:Http;
import std;
import :Json;
// The HTTP the driver needs: app_ready's preflight, the OpenRouter catalogue
// and key probe. On Crafter.Network's FetchHTTP1, which follows redirects; on
// top of it a status outside 2xx thrown as HttpError, and every other failure
// as Error with a fixed text, because those texts reach the run log and
// app_ready's detail. Proxies are not consulted (AGENTS.md).
export namespace Crafter::Review::Http {
struct Response {
std::int32_t status = 0;
std::string body;
};
// A status outside 2xx. what(): "HTTP Error 404: Not Found", with the
// standard reason phrase rather than the server's.
class HttpError : public std::runtime_error {
public:
HttpError(std::int32_t code, const std::string& message);
explicit HttpError(std::int32_t code);
std::int32_t Code() const;
private:
std::int32_t code;
};
// Every other failure. What fails while the request goes out reads
// "<urlopen error [Errno 111] Connection refused>" or "<urlopen error
// timed out>"; what fails while the response comes in reads as it is:
// "timed out".
class Error : public std::runtime_error {
public:
using std::runtime_error::runtime_error;
};
// A GET of `url`, read whole. Header names in any case; Accept-Encoding:
// identity and Connection: close always go out, and a User-Agent unless
// one is given.
Response UrlOpen(std::string_view url, const std::map<std::string, std::string>& headers, double timeout);
// The body as JSON: Dyn::Error JSONDecodeError (or UnicodeDecodeError)
// with the scanner's message for a body that is not.
Json JsonBody(const Response& response);
// http.HTTPStatus(code).phrase, "" for a code it does not know.
std::string_view Reason(std::int32_t code);
}

View file

@ -74,6 +74,12 @@ export namespace Crafter::Review::Platform {
// Directories are created as needed. // Directories are created as needed.
void WriteAtomic(const std::filesystem::path& path, std::string_view content, std::uint32_t mode = 0644); void WriteAtomic(const std::filesystem::path& path, std::string_view content, std::uint32_t mode = 0644);
// `path` and its missing parents: the parents made as mkdir makes them,
// the last directory with `mode`, both less the umask; a directory that is
// there keeps its mode. Throws with the errno text ("[Errno 13] Permission
// denied: '/x'").
void MakeDirs(const std::filesystem::path& path, std::uint32_t mode = 0777);
// An exclusive flock(2) on `lockPath`, held while the object lives. flock, // An exclusive flock(2) on `lockPath`, held while the object lives. flock,
// not fcntl or OFD locks: every driver on the machine takes flock on the // not fcntl or OFD locks: every driver on the machine takes flock on the
// same side files, and on Linux the two kinds do not exclude each other. A // same side files, and on Linux the two kinds do not exclude each other. A

View file

@ -0,0 +1,208 @@
// SPDX-License-Identifier: AGPL-3.0-only
// SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts®
export module Crafter.Review:SeatPool;
import std;
import :Json;
// The seat pool: which credential each lens process runs on. OAuth seats
// (CLAUDE_CODE_OAUTH_TOKEN1..6), API-credit seats (ANTHROPIC_API_KEY1..6,
// real money, a ledger and a budget), OpenRouter seats
// (OPENROUTER_KEY[1..6], never in the default pool), and `local`, this
// machine's login. Placement: API seats first (their credit expires monthly),
// then OAuth seats by projected utilisation; a process that fails on a seat's
// limit, credit or auth is placed again; the pool aborts only when every seat
// is out. Secrets never reach a log, a dump or a child's environment except
// the one the CLI reads.
//
// Several drivers at once (2026-10-31): every driver on this machine
// registers on the board (~/.cache/review-gates/seat-board.json) what it has
// in flight on each seat, and folds the board into its seats and admission
// before every launch; the pick is registered under the same lock. Every
// driver on the machine shares the file, its shape and its lock. A seat is a
// JSON object: the dump records it (without the secret).
//
// Liveness (README, AGENTS.md): a driver is alive while its lifetime lock
// (<board dir>/drivers/<pid>-<start>.lock) is held; without one, natively
// the pid and start time decide, and in the Flatpak the driver is taken for
// alive, because pids of another namespace say nothing here.
export namespace Crafter::Review::SeatPool {
// A job is re-placed this often after a limit, credit or auth failure.
constexpr std::int32_t MaxAttempts = 4;
// An API seat's pause after a 429.
constexpr double ApiBackoffS = 90;
constexpr double OauthFallbackPauseS = 300;
// The kinds whose processes cost money the ledger counts.
constexpr std::array<std::string_view, 2> Paid = {"api", "openrouter"};
// The kinds whose out/invalid verdict crosses drivers on the board.
constexpr std::array<std::string_view, 2> StateShared = {"oauth", "local"};
constexpr std::string_view BoardFile = "seat-board.json";
// An in-flight entry outlives its pass's time budget x1.5 by this much
// before the sweep takes it.
constexpr double InflightGraceS = 300;
constexpr std::string_view OpenRouterBaseUrl = "https://openrouter.ai/api";
constexpr std::string_view OpenRouterModelsUrl = "https://openrouter.ai/api/v1/models";
constexpr std::string_view OpenRouterKeyUrl = "https://openrouter.ai/api/v1/key";
extern const std::string_view ResumePrompt;
bool IsPaid(const Json& seat);
// The API-seat knobs.
Json ApiConfig(const std::map<std::string, std::string>& env);
// The other drivers' load on a seat as the last board fold left it.
Json Foreign(const Json& seat);
std::filesystem::path LedgerPath(const std::optional<std::filesystem::path>& scratchRoot = std::nullopt);
// The most recent date with that day of the month, as YYYY-MM-DD.
std::string CycleStart(double now, std::int32_t day);
// What the driver spent on each API seat this cycle, and whether the key
// answered the credit error; one file for every driver, each write a delta
// applied under its lock.
class Ledger {
public:
Ledger(std::filesystem::path path, Json apiConfig, Json data = Json::object());
static Ledger Load(const std::filesystem::path& path, const Json& apiConfig);
Json& Entry(const std::string& seatId, std::optional<double> now = std::nullopt);
Json Add(const std::string& seatId, double usd, std::optional<double> now = std::nullopt);
Json MarkOut(const std::string& seatId, std::string_view reason, std::optional<double> now = std::nullopt);
void Save();
const Json& Data() const;
private:
std::filesystem::path path;
Json apiConfig;
Json data;
Json Update(const std::string& seatId, const std::function<void(Json&)>& change, std::optional<double> now);
};
Json NewSeat(const std::string& id, const std::string& kind, const std::string& secret);
// The seats a run may use. spec: auto (every Anthropic seat of the env
// file and the environment, else local), local | api | oauth | openrouter,
// or a list such as api1,1,2,local. std::invalid_argument with a
// ValueError message for a seat that is not configured, or OpenRouter
// mixed with the rest.
std::vector<Json> LoadSeats(const std::map<std::string, std::string>& env, const std::optional<std::filesystem::path>& envFile, std::string_view spec,
Ledger* ledger = nullptr, const Json& apiConfig = nullptr);
using ProbeFunction = std::function<Json(std::int32_t seat, const std::string& token, const std::string& model, double timeout, const std::string& claudeBin, bool local, bool api)>;
// One probe per seat before the run; a seat already out is skipped.
void Probe(std::vector<Json>& seats, const std::string& claudeBin, double timeout, const std::vector<std::string>& models,
const std::function<void(const std::string&)>& log, const ProbeFunction& probe = {});
void ApplyProbe(Json& seat, const Json& result);
std::string Describe(const Json& seat);
// The OAuth placement knobs, with the pool's own per-process default (0.25:
// the 2026-10-15 load test moved a 20x seat's window 0.08 points a pass).
Json PoolConfig(const std::map<std::string, std::string>& env);
double OauthProjected(const Json& seat, const Json& config);
// A window whose reset has passed counts as empty until the next reading.
std::vector<std::string> ExpireWindows(Json& seat, double now);
bool Eligible(Json& seat, double now, const Json& config, const Json& apiConfig);
// Every seat with room for a launch now, best first (pointers into seats).
std::vector<Json*> Ranked(std::vector<Json>& seats, double now, const Json& config, const Json& apiConfig);
Json* Pick(std::vector<Json>& seats, double now, const Json& config, const Json& apiConfig);
std::filesystem::path SeatDir(const std::filesystem::path& scratch, std::string_view seatId);
// A pass interrupted by its seat's limit continues on another seat once
// its transcript is copied into that seat's config dir.
std::optional<std::filesystem::path> CarryTranscript(const std::filesystem::path& scratch, std::string_view fromSeat, std::string_view toSeat,
std::string_view sessionId, const std::function<void(const std::string&)>& log = {});
bool AllOut(const std::vector<Json>& seats);
std::optional<double> EarliestResume(const std::vector<Json>& seats, double now, std::optional<double> recheck = std::nullopt);
// A finished process onto its seat; true when the job must be placed again
// (the seat failed it, not the lens).
bool Settle(Json& seat, Json& record, double now, const Json& config, const Json& apiConfig, Ledger* ledger, const std::function<void(const std::string&)>& log);
double RetryAfter(std::string_view text, double fallback);
std::filesystem::path WriteCredentials(const std::filesystem::path& configDir, std::string_view token);
// The environment of a lens process on this seat.
std::map<std::string, std::string> ChildEnv(const Json& seat, const std::map<std::string, std::string>& base, const std::filesystem::path& scratch);
std::string ModelKey(std::string_view model);
// {catalogue id: prices or null} of the run's models.
Json FetchOpenRouterPrices(const std::vector<std::string>& models, const std::function<void(const std::string&)>& log = {}, double timeout = 20);
double OpenRouterCost(std::string_view model, const Json& usage);
Json ProbeOpenRouter(const std::string& secret, double timeout = 15);
// ---- several drivers at once: the board --------------------------------------
std::filesystem::path BoardPath();
// `<kind>:<12 hex of the secret's sha256>`, `local` for the login.
std::string SeatKey(std::string_view kind, std::string_view secret);
// This driver as the board names it: `<pid>:<start>`, the sandbox
// instance's in the Flatpak (Platform::Self).
std::string MyKey();
// Is the driver `key` alive? Its lifetime lock decides when it has one
// (a free one is taken away when `reap`); else natively its pid and start
// time, and in the sandbox yes.
bool DriverAlive(std::string_view key, const std::filesystem::path& boardDir, bool reap = true);
// One live driver per scratch dir: a `.driver` marker, taken over when its
// driver is dead.
bool ClaimScratch(const std::filesystem::path& path);
// Fold `other`'s readings into `into`, in place; the windows that changed.
std::vector<std::string> MergeWindows(Json& into, const Json& other, double now);
// The other drivers' passes in flight on this machine.
struct Others {
std::int64_t count = 0;
double mb = 0;
std::set<std::string> stacks;
std::int64_t unmeasured = 0;
};
class Board {
public:
explicit Board(std::filesystem::path path, std::string me = {}, std::optional<std::string> boot = std::nullopt,
std::function<void(const std::string&)> log = {});
// Is the driver `key` on the board alive: this driver; else not when
// it joined under another boot; else DriverAlive.
bool Alive(std::string_view key, const Json& drivers, bool reap = true) const;
// Drop what no longer stands: dead drivers with their in-flight
// entries, entries past their deadline, verdicts past their expiry
// or whose driver is gone, seat records with nothing live. (changed,
// the lines that say what went).
std::pair<bool, std::vector<std::string>> Sweep(Json& data, double now, bool reap = true) const;
void Join(const Json& info, double now);
// Everything of this driver off the board; idempotent, never throws.
void Leave();
void Publish(const std::vector<Json>& seats, double now, bool probed = false);
std::vector<std::string> Fold(std::vector<Json>& seats, double now);
static std::int64_t TotalInflight(const Json& data);
// The memory a gate's budget starts from, and how it was had.
std::pair<double, std::string> MemoryBase(std::int64_t free, double now);
using Chooser = std::function<std::optional<std::pair<Json*, Json*>>(const std::vector<Json*>& ranked, const Others& others)>;
struct Placed {
std::vector<Json*> ranked;
std::optional<std::pair<Json*, Json*>> picked;
std::int64_t full = 0;
};
// One locked section: the board folded, the seats ranked, the
// chooser's pick registered before the lock drops.
Placed Place(std::vector<Json>& seats, double now, const Json& config, const Json& apiConfig, const Chooser& chooser, std::optional<std::int64_t> cap = std::nullopt);
void Finish(const Json& seat, std::string_view label, double now);
// The board as it stands, swept in memory, nothing written.
Json View(double now) const;
std::vector<std::string> OthersLines(const std::vector<Json>& seats, double now, const Json& config, const Json& apiConfig, std::optional<std::int64_t> cap = std::nullopt) const;
const Others& OthersNow() const;
const std::string& Me() const;
const std::filesystem::path& Path() const;
const Json& Data() const;
private:
std::filesystem::path path;
std::string me;
std::string boot;
std::function<void(const std::string&)> log;
Json data;
Others others;
bool warned = false;
void Warn(const std::string& message);
Json Update(const std::function<bool(Json&)>& change);
Json& Record(Json& data, const Json& seat) const;
static bool StateHolds(const Json& state, const Json& data, double now);
Others OthersOf(const Json& data) const;
std::pair<bool, std::vector<std::string>> FoldInto(std::vector<Json>& seats, double now, Json& data);
bool PublishSeat(Json& data, const Json& seat, double now, bool probed) const;
};
// The seats as the dump records them: no secrets.
Json Summary(const std::vector<Json>& seats);
std::string Table(const std::vector<Json>& seats, std::optional<double> now = std::nullopt);
}

View file

@ -9,6 +9,7 @@ export import :Text;
export import :Crypto; export import :Crypto;
export import :Platform; export import :Platform;
export import :Process; export import :Process;
export import :Http;
export import :Core; export import :Core;
export import :Schema; export import :Schema;
export import :Tools; export import :Tools;
@ -26,5 +27,6 @@ export import :Config;
export import :Memory; export import :Memory;
export import :Mcp; export import :Mcp;
export import :Seats; export import :Seats;
export import :SeatPool;
export import :Freeze; export import :Freeze;
export import :Driver; export import :Driver;

View file

@ -13,7 +13,7 @@ using namespace Crafter;
namespace { namespace {
// The umbrella, then the partitions in import order: each after the ones // The umbrella, then the partitions in import order: each after the ones
// it imports, which is also the order the module map in AGENTS.md uses. // it imports, which is also the order the module map in AGENTS.md uses.
constexpr std::array<std::string_view, 26> Interfaces = { constexpr std::array<std::string_view, 28> Interfaces = {
"interfaces/Crafter.Review", "interfaces/Crafter.Review",
"interfaces/Crafter.Review-Unicode", "interfaces/Crafter.Review-Unicode",
"interfaces/Crafter.Review-Json", "interfaces/Crafter.Review-Json",
@ -21,6 +21,7 @@ namespace {
"interfaces/Crafter.Review-Crypto", "interfaces/Crafter.Review-Crypto",
"interfaces/Crafter.Review-Platform", "interfaces/Crafter.Review-Platform",
"interfaces/Crafter.Review-Process", "interfaces/Crafter.Review-Process",
"interfaces/Crafter.Review-Http",
"interfaces/Crafter.Review-Core", "interfaces/Crafter.Review-Core",
"interfaces/Crafter.Review-Schema", "interfaces/Crafter.Review-Schema",
"interfaces/Crafter.Review-Tools", "interfaces/Crafter.Review-Tools",
@ -38,14 +39,16 @@ namespace {
"interfaces/Crafter.Review-Memory", "interfaces/Crafter.Review-Memory",
"interfaces/Crafter.Review-Mcp", "interfaces/Crafter.Review-Mcp",
"interfaces/Crafter.Review-Seats", "interfaces/Crafter.Review-Seats",
"interfaces/Crafter.Review-SeatPool",
"interfaces/Crafter.Review-Freeze", "interfaces/Crafter.Review-Freeze",
"interfaces/Crafter.Review-Driver", "interfaces/Crafter.Review-Driver",
}; };
constexpr std::array<std::string_view, 23> Implementations = { constexpr std::array<std::string_view, 25> Implementations = {
"implementations/Crafter.Review-Text", "implementations/Crafter.Review-Text",
"implementations/Crafter.Review-Crypto", "implementations/Crafter.Review-Crypto",
"implementations/Crafter.Review-Platform", "implementations/Crafter.Review-Platform",
"implementations/Crafter.Review-Process", "implementations/Crafter.Review-Process",
"implementations/Crafter.Review-Http",
"implementations/Crafter.Review-Core", "implementations/Crafter.Review-Core",
"implementations/Crafter.Review-Schema", "implementations/Crafter.Review-Schema",
"implementations/Crafter.Review-Tools", "implementations/Crafter.Review-Tools",
@ -63,6 +66,7 @@ namespace {
"implementations/Crafter.Review-Memory", "implementations/Crafter.Review-Memory",
"implementations/Crafter.Review-Mcp", "implementations/Crafter.Review-Mcp",
"implementations/Crafter.Review-Seats", "implementations/Crafter.Review-Seats",
"implementations/Crafter.Review-SeatPool",
"implementations/Crafter.Review-Freeze", "implementations/Crafter.Review-Freeze",
"implementations/Crafter.Review-Driver", "implementations/Crafter.Review-Driver",
}; };
@ -74,7 +78,7 @@ namespace {
// One per directory under tests/, each an ordinary main() that links the // One per directory under tests/, each an ordinary main() that links the
// library: exit 0 passes, nonzero fails, 77 skips. // library: exit 0 passes, nonzero fails, 77 skips.
constexpr std::array<std::string_view, 15> Tests = { constexpr std::array<std::string_view, 17> Tests = {
"Crypto", "Crypto",
"Text", "Text",
"Platform", "Platform",
@ -90,6 +94,8 @@ namespace {
"Freeze", "Freeze",
"PgWire", "PgWire",
"Seats", "Seats",
"SeatPool",
"Drivers",
}; };
// The version the binary reports: CI sets it from the tag, a checkout // The version the binary reports: CI sets it from the tag, a checkout

496
tests/Drivers/main.cpp Normal file
View file

@ -0,0 +1,496 @@
// SPDX-License-Identifier: AGPL-3.0-only
// SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts®
// Several drivers at once: the board (seats, memory, the cap), one full run
// per hash, one driver per scratch dir, the merged unit-state, the locked
// ledger, the shared memory record. The checks that run the Scheduler,
// Admission, Stacks, the stack keys and the run's state store come with the
// pool and the driver.
//
// Another driver is a live process of its own (`sleep 300`), so its entries
// stand until it dies; two drivers recording into one memory file are two
// runs of this binary (`--record`).
#include <signal.h>
#include <stdlib.h>
#include <unistd.h>
import std;
import Crafter.Review;
namespace fs = std::filesystem;
using namespace Crafter::Review;
using namespace std::chrono_literals;
namespace {
std::int32_t Failures = 0;
void Check(std::string_view name, bool cond, std::string_view detail = {}) {
std::println("{}{}{}", cond ? " ok " : " FAIL ", name, !cond && !detail.empty() ? std::format(" ({})", detail) : std::string());
if (!cond) ++Failures;
}
void Require(bool cond, std::string_view what) {
if (!cond) throw std::runtime_error(std::string(what));
}
Json With(Json value, const Json& changes) {
for (auto it = changes.begin(); it != changes.end(); ++it) value[it.key()] = it.value();
return value;
}
void Write(const fs::path& path, std::string_view text) {
fs::create_directories(path.parent_path());
std::ofstream(path, std::ios::binary) << text;
}
// The board file, as JSON.
Json Read(const fs::path& path) {
return Dyn::LoadJsonFile(path);
}
std::set<std::string> Keys(const Json& object) {
std::set<std::string> keys;
for (auto it = object.begin(); it != object.end(); ++it) keys.insert(it.key());
return keys;
}
bool AnyContains(const std::vector<std::string>& lines, std::string_view part) {
return std::ranges::any_of(lines, [&](const std::string& line) { return line.contains(part); });
}
Json& At(std::vector<Json>& seats, std::string_view id) {
for (Json& s : seats) {
if (s.at("id") == id) return s;
}
throw std::out_of_range(std::string(id));
}
// A driver of its own: a live process whose pid and start time name it.
struct Sleeper {
std::int64_t pid = Process::Start({.argv = {"sleep", "300"}}).pid;
std::string key = std::format("{}:{}", pid, Platform::ProcStart(pid).value_or(0));
void Kill() const {
Process::Signal(pid, SIGKILL);
Process::Wait(pid);
}
};
std::int32_t Run() {
std::println("several drivers at once: the board (seats, memory, stacks, the cap), one full run per hash, one driver per scratch dir, the merged unit-state, the locked ledger");
Platform::SetMode(Platform::Mode::Native);
const fs::path td = fs::temp_directory_path() / std::format("crafter-review-drivers-{}", Platform::PublishedPid());
fs::remove_all(td);
fs::create_directories(td);
const fs::path path = td / "seat-board.json";
// The lifetime locks ClaimScratch looks for are beside the board.
setenv("REVIEW_GATES_BOARD", path.c_str(), 1);
std::mutex logLock;
std::vector<std::string> logs;
auto log = [&](const std::string& line) {
const std::lock_guard guard(logLock);
logs.push_back(line);
};
const double now = Platform::Now();
const std::int64_t start = Platform::ProcStart(getpid()).value_or(0);
const std::string mine = std::format("{}:{}", getpid(), start);
SeatPool::Board A(path, {}, std::nullopt, log);
Check("identity: pid + start time from /proc, alive; a nonexistent pid, a recycled pid (another start time) and another boot are not",
A.Me() == mine && A.Alive(A.Me(), Json::object()) && !A.Alive("2147483647:1", Json::object()) && !A.Alive(std::format("{}:{}", getpid(), start + 1), Json::object())
&& !SeatPool::Board(path, "x:y", "bootA").Alive(mine, {{mine, {{"boot", "bootB"}}}}));
Check("seat keys: one token under two .env numbers is one account, the key carries no secret, the login is local, the kinds differ",
SeatPool::NewSeat("1", "oauth", "tok-one").at("boardKey") == SeatPool::NewSeat("5", "oauth", "tok-one").at("boardKey") && !SeatPool::SeatKey("oauth", "tok-one").contains("tok-one")
&& SeatPool::SeatKey("local", "") == "local" && SeatPool::SeatKey("api", "k") != SeatPool::SeatKey("oauth", "k"));
const Json cfg = SeatPool::PoolConfig({});
const Json acfg = SeatPool::ApiConfig({{"REVIEW_GATES_API_CONCURRENCY", "8"}, {"REVIEW_GATES_API_LENS_COST_USD", "1"}});
auto mk = [&](const std::string& id, const std::string& kind, const std::string& secret, std::optional<std::int64_t> fh = std::nullopt, std::optional<std::int64_t> sd = std::nullopt,
const std::string& state = "ok") {
Json s = SeatPool::NewSeat(id, kind, secret);
s["state"] = state;
if (fh) {
s["windows"] = {{"five_hour", {{"utilization", *fh}, {"resetsAt", now + 3600}}}, {"seven_day", {{"utilization", sd.value_or(0)}, {"resetsAt", now + 86400}}}};
const auto [util, binding] = Seats::SeatUtil(s.at("windows"));
s["util"] = util;
s["binding"] = binding;
}
return s;
};
// The jobs the choosers hand out: they must outlive the place that
// registers them.
std::deque<Json> jobs;
std::mutex jobsLock;
auto chooserFor = [&](const std::string& id, const std::string& label, const Json& extra = Json::object()) {
return SeatPool::Board::Chooser([&jobs, &jobsLock, id, label, extra](const std::vector<Json*>& ranked, const SeatPool::Others&) -> std::optional<std::pair<Json*, Json*>> {
for (Json* s : ranked) {
if (s->at("id") == id) {
const std::lock_guard guard(jobsLock);
jobs.push_back(With({{"label", label}, {"timeoutMin", 20}}, extra));
return std::pair(s, &jobs.back());
}
}
return std::nullopt;
});
};
// A call on [seat]: the seat comes back changed.
auto fold1 = [&](SeatPool::Board& board, Json& seat, double at) {
std::vector<Json> one = {seat};
std::vector<std::string> lines = board.Fold(one, at);
seat = one.front();
return lines;
};
auto place1 = [&](SeatPool::Board& board, Json& seat, const Json& c, const SeatPool::Board::Chooser& chooser) {
std::vector<Json> one = {seat};
const bool picked = board.Place(one, now, c, acfg, chooser).picked.has_value();
seat = one.front();
return picked;
};
const Sleeper child;
SeatPool::Board B(path, child.key, std::nullopt, log);
A.Join({{"run", "A"}}, now);
B.Join({{"run", "B"}}, now);
std::vector<Json> sa = {mk("1", "oauth", "t1", 10, 50), mk("2", "oauth", "t2", 10, 50), mk("api1", "api", "k1")};
std::vector<Json> sb = {mk("1", "oauth", "t1", 10, 50), mk("2", "oauth", "t2", 10, 50), mk("api1", "api", "k1")};
for (std::int32_t i = 0; i < 3; ++i) {
const SeatPool::Board::Placed placed = B.Place(sb, now, cfg, acfg, chooserFor("1", std::format("b{}", i)));
Require(placed.picked && placed.picked->first->at("id") == "1", "B places on seat 1");
}
for (std::int32_t i = 0; i < 2; ++i) {
const SeatPool::Board::Placed placed = B.Place(sb, now, cfg, acfg, chooserFor("api1", std::format("ba{}", i)));
Require(placed.picked && placed.picked->first->at("id") == "api1", "B places on api1");
}
// The pool width is the machine's cap: B's five passes count against
// A's.
SeatPool::Board::Placed placed = A.Place(sa, now, cfg, acfg, chooserFor("2", "cap0"), 5);
Check("the cap counts every driver: five in flight on the machine hold a sixth at cap 5, the seats still ranked", !placed.picked && placed.full == 5 && !placed.ranked.empty());
placed = A.Place(sa, now, cfg, acfg, chooserFor("2", "cap0"), 6);
Check("...and a sixth goes at cap 6; it counts from then on", placed.picked && placed.full == 0 && SeatPool::Board::TotalInflight(Read(path)) == 6);
A.Finish(At(sa, "2"), "cap0", now);
A.Fold(sa, now);
const Json f1 = At(sa, "1").at("foreign");
const Json fa = At(sa, "api1").at("foreign");
Check("three passes of B on seat 1 and two on api1 fold into A as foreign, at B's own per-pass figures",
f1.at("count") == 3 && Dyn::Equal(f1.at("cost5h"), 0.75) && Dyn::Equal(f1.at("costUsd"), 0) && fa.at("count") == 2 && Dyn::Equal(fa.at("costUsd"), 2.0)
&& Dyn::Equal(f1.at("drivers"), {{B.Me(), 3}}),
std::format("{} {}", DumpSpaced(f1), DumpSpaced(fa)));
Json s84 = mk("x", "oauth", "tx", 84, 50);
s84["foreign"] = {{"count", 3}, {"cost5h", 1.5}, {"costUsd", 0}};
const bool over = !SeatPool::Eligible(s84, now, cfg, acfg);
Json s84Bare = With(s84, {{"foreign", {{"count", 0}, {"cost5h", 0.0}, {"costUsd", 0.0}}}});
Check("the 5-hour projection counts the foreign passes: 84 + 1.5 + 0.25 is over 85, without them under", over && SeatPool::Eligible(s84Bare, now, cfg, acfg));
Json a8 = mk("api9", "api", "k9");
a8["foreign"] = {{"count", 6}, {"cost5h", 0}, {"costUsd", 6.0}};
a8["running"] = 2;
Json a8One = With(a8, {{"running", 1}});
Check("an API key's 8 slots count every driver: 6 foreign + 2 running is full, 6 + 1 is not", !SeatPool::Eligible(a8, now, cfg, acfg) && SeatPool::Eligible(a8One, now, cfg, acfg));
Check("...and its budget the other drivers' reserve: $6 foreign + $1 running + $1 for the next is over a $7 budget, inside $8",
!SeatPool::Eligible(a8One, now, cfg, With(acfg, {{"budgetUsd", 7}})) && SeatPool::Eligible(a8One, now, cfg, With(acfg, {{"budgetUsd", 8}})));
std::vector<Json> es = {mk("e1", "oauth", "te1", 20, 10), mk("e2", "oauth", "te2", 20, 10)};
es[0]["foreign"] = {{"count", 10}, {"cost5h", 5.0}, {"costUsd", 0}};
std::vector<std::string> order;
for (const Json* s : SeatPool::Ranked(es, now, cfg, With(acfg, {{"budgetUsd", 0}}))) order.push_back(s->at("id").get<std::string>());
Check("ranked: equal readings, the seat with foreign load ranks second", order == std::vector<std::string>{"e2", "e1"}, Dyn::Join(order, ","));
// Readings merge by max per reset epoch, newer epoch wins, expired
// ignored.
Json w = {{"five_hour", {{"utilization", 40}, {"resetsAt", now + 3600}}}};
const std::vector<std::string> ch = SeatPool::MergeWindows(w, {{"five_hour", {{"utilization", 44}, {"resetsAt", now + 3600}}}}, now);
const bool higher = ch == std::vector<std::string>{"five_hour"} && w["five_hour"]["utilization"] == 44;
Check("windows merge: the same epoch takes the higher utilization, a lower one changes nothing",
higher && SeatPool::MergeWindows(w, {{"five_hour", {{"utilization", 30}, {"resetsAt", now + 3600}}}}, now).empty() && w["five_hour"]["utilization"] == 44);
SeatPool::MergeWindows(w, {{"five_hour", {{"utilization", 3}, {"resetsAt", now + 7200}}}}, now);
Check("...a newer epoch replaces an older one, however low", Dyn::Equal(w["five_hour"], {{"utilization", 3}, {"resetsAt", now + 7200}}));
Json expired = {{"five_hour", {{"utilization", 0}, {"resetsAt", 0}}}};
Check("...a reading whose reset has passed is ignored on either side",
SeatPool::MergeWindows(w, {{"five_hour", {{"utilization", 90}, {"resetsAt", now - 100}}}}, now).empty() && w["five_hour"]["utilization"] == 3
&& SeatPool::MergeWindows(expired, {{"five_hour", {{"utilization", 50}, {"resetsAt", now + 100}}}}, now) == std::vector<std::string>{"five_hour"});
Json sx = mk("1", "oauth", "t1");
sx["windows"] = {{"five_hour", {{"utilization", 0}, {"resetsAt", 0}}}};
A.Publish({sx}, now, true);
Check("...and a locally expired window (expire_windows) is never published",
!Dyn::Has(Dyn::Or(Dyn::Get(Read(path).at("seats").at(sx.at("boardKey").get<std::string>()), "windows"), Dyn::EmptyDict()), "five_hour"));
At(sb, "1")["windows"]["five_hour"]["utilization"] = 40;
B.Publish({At(sb, "1")}, now);
fold1(A, At(sa, "1"), now);
Check("a higher reading of the same epoch on the board reaches the other driver's seat, util and binding recomputed",
At(sa, "1")["windows"]["five_hour"]["utilization"] == 40 && At(sa, "1")["util"] == 50 && At(sa, "1")["binding"] == "seven_day");
const std::optional<std::string> before = Platform::ReadFile(path);
SeatPool::Board C(path, A.Me(), std::nullopt, log);
C.Publish({At(sa, "1")}, now);
fold1(C, At(sa, "1"), now);
Check("publish then fold of what the board already holds writes nothing (the merge is idempotent)", Platform::ReadFile(path) == before);
// Verdicts: an OAuth seat's out stands until its window resets,
// invalid while its driver lives; never a paid seat's.
Json weekly = {{"status", "limit"},
{"reason", "weekly"},
{"rateType", "seven_day"},
{"rateLimit", {{"five_hour", {{"utilization", 12}, {"resetsAt", now + 3600}}}, {"seven_day", {{"utilization", 100}, {"resetsAt", now + 500}}}}}};
const bool requeued = SeatPool::Settle(At(sb, "2"), weekly, now, cfg, acfg, nullptr, log);
B.Finish(At(sb, "2"), "none", now);
A.Fold(sa, now);
const std::string key2 = At(sa, "2").at("boardKey").get<std::string>();
Check("an OAuth seat another driver found out (seven-day rejected) is out here, until that window resets",
requeued && At(sa, "2")["state"] == "out" && Dyn::Str(At(sa, "2")["error"]).contains("seen by pid") && Read(path)["seats"][key2]["state"]["until"].get<double>() == now + 530,
Dyn::Str(At(sa, "2")["error"]));
Json sa2 = mk("2", "oauth", "t2", 10, 50);
fold1(A, sa2, now + 600);
const Json record2 = Dyn::Get(Read(path).at("seats"), key2);
Check("...and not once that reset has passed (the record is swept)", sa2["state"] == "ok" && Dyn::Get(Dyn::Or(record2, Dyn::EmptyDict()), "state").is_null());
sa2 = mk("2", "oauth", "t2", 10, 50, "invalid");
sa2["error"] = "mine";
fold1(A, sa2, now);
Check("a local out/invalid is not overwritten by the board's", sa2["state"] == "invalid" && sa2["error"] == "mine");
sa2 = mk("2", "oauth", "t2", 10, 50);
A.Publish({sa2}, now, true);
Check("a seat our own probe reads ok clears the verdict another driver left", Read(path)["seats"][key2]["state"].is_null());
At(sb, "1")["state"] = "invalid";
At(sb, "1")["error"] = "authentication failure (401)";
B.Publish({At(sb, "1")}, now);
Json sa1 = mk("1", "oauth", "t1", 10, 50);
fold1(A, sa1, now);
Check("an invalid token seen by another driver is adopted while that driver lives", sa1["state"] == "invalid" && Dyn::Str(sa1["error"]).contains("authentication"));
const std::string kApi = At(sa, "api1").at("boardKey").get<std::string>();
Json bd = Read(path);
if (!bd["seats"].contains(kApi)) bd["seats"][kApi] = {{"id", "api1"}, {"kind", "api"}, {"windows", Json::object()}, {"pauseUntil", 0}, {"state", nullptr}, {"inflight", Json::object()}};
bd["seats"][kApi]["state"] = {{"state", "out"}, {"error", "budget"}, {"until", now + 9999}, {"by", B.Me()}};
bd["seats"][kApi]["pauseUntil"] = now + 90;
Write(path, DumpSpaced(bd));
Json saApi = mk("api1", "api", "k1");
fold1(A, saApi, now);
Check("a paid seat's verdict never crosses drivers (the ledger carries a credit-out, a budget-out is each driver's own) - its pause does, until the same moment",
saApi["state"] == "ok" && saApi["pauseUntil"].get<double>() == now + 90 && !SeatPool::Eligible(saApi, now, cfg, acfg) && SeatPool::Eligible(saApi, now + 91, cfg, acfg));
child.Kill();
sa1 = mk("1", "oauth", "t1", 10, 50);
const std::vector<std::string> lines = fold1(A, sa1, now);
Check("a driver that died is swept with its in-flight entries and its invalid verdict, and the sweep says so",
sa1["state"] == "ok" && sa1["foreign"]["count"] == 0 && !Read(path).at("drivers").contains(B.Me()) && AnyContains(lines, "(dead)"), Dyn::Join(lines, "; "));
SeatPool::Board Z(path, std::format("{}:{}", getpid(), start + 1), std::nullopt, log);
Z.Join({{"run", "Z"}}, now);
Json z0 = mk("1", "oauth", "t1", 10, 50);
place1(Z, z0, cfg, chooserFor("1", "z0"));
Json sz = mk("1", "oauth", "t1", 10, 50);
fold1(A, sz, now);
Check("a driver record with a live pid but another start time is a recycled pid: swept", sz["foreign"]["count"] == 0 && !Read(path).at("drivers").contains(Z.Me()));
Json data = {{"drivers", {{A.Me(), Json::object()}}},
{"seats",
{{"k",
{{"id", "1"},
{"kind", "oauth"},
{"windows", Json::object()},
{"pauseUntil", 0},
{"state", nullptr},
{"inflight", {{std::format("{}/late", A.Me()), {{"since", now - 9999}, {"deadline", now - 1}}}, {std::format("{}/live", A.Me()), {{"since", now}, {"deadline", now + 100}}}}}}}}}};
const auto [swept, sweptLines] = A.Sweep(data, now);
A.Finish(mk("1", "oauth", "t1", 10, 50), "late", now);
Check("an in-flight entry past its deadline is swept, a live one kept; finishing a pass the sweep already took is harmless",
swept && Keys(data["seats"]["k"]["inflight"]) == std::set<std::string>{std::format("{}/live", A.Me())} && AnyContains(sweptLines, "past their deadline"), Dyn::Join(sweptLines, "; "));
// A second live driver: leave, the atomic place, its own projection.
const Sleeper child2;
SeatPool::Board D(path, child2.key, std::nullopt, log);
D.Join({{"run", "D"}}, now);
Json d0 = mk("1", "oauth", "t1", 10, 50);
place1(D, d0, cfg, chooserFor("1", "d0"));
D.Leave();
D.Leave();
Json sl = mk("1", "oauth", "t1", 10, 50);
fold1(A, sl, now);
Check("leave takes the driver and its in-flight entries off the board; a second leave is harmless", sl["foreign"]["count"] == 0 && !Read(path).at("drivers").contains(D.Me()));
D.Join({{"run", "D"}}, now);
const Json cfg2 = With(cfg, {{"lensCostPct", 2}, {"maxUtil", 85}});
// An account of its own: its finish publishes 82.
Json sa82 = mk("4", "oauth", "t4", 82, 10);
Json sd82 = mk("4", "oauth", "t4", 82, 10);
std::mutex resultsLock;
std::vector<std::pair<bool, bool>> results;
auto race = [&](SeatPool::Board& board, Json& seat) {
std::vector<Json> one = {seat};
Json job = {{"label", "race"}, {"timeoutMin", 20}};
const SeatPool::Board::Placed got = board.Place(one, now, cfg2, acfg, [&job](const std::vector<Json*>& ranked, const SeatPool::Others&) -> std::optional<std::pair<Json*, Json*>> {
std::this_thread::sleep_for(50ms);
if (ranked.empty()) return std::nullopt;
return std::pair(ranked.front(), &job);
});
seat = one.front();
const std::lock_guard guard(resultsLock);
results.emplace_back(!got.ranked.empty(), got.picked.has_value());
};
{
const std::jthread ta(race, std::ref(A), std::ref(sa82));
const std::jthread tb(race, std::ref(D), std::ref(sd82));
}
const auto launched = std::ranges::count_if(results, [](const auto& r) { return r.second; });
const auto roomless = std::ranges::count_if(results, [](const auto& r) { return !r.first; });
Check("place is one locked section: two drivers racing for a seat with room for one pass get exactly one launch",
launched == 1 && roomless == 1 && Read(path)["seats"][sa82.at("boardKey").get<std::string>()]["inflight"].size() == 1, std::format("{} launched, {} without room", launched, roomless));
A.Finish(sa82, "race", now);
D.Finish(sd82, "race", now);
Json overCeiling = mk("o", "oauth", "to", 90, 10);
overCeiling["windows"]["five_hour"]["resetsAt"] = now + 600;
overCeiling["foreign"] = {{"count", 1}, {"cost5h", 0.5}, {"costUsd", 0}};
const Json overBare = With(overCeiling, {{"foreign", {{"count", 0}, {"cost5h", 0.0}, {"costUsd", 0.0}}}});
Check("earliest_resume: with another driver's pass in flight the wait is one re-check, else the window reset",
SeatPool::EarliestResume({overCeiling}, now, 30) == now + 30 && SeatPool::EarliestResume({overCeiling}, now) == now + 630 && SeatPool::EarliestResume({overBare}, now, 30) == now + 630);
D.Leave();
D.Join({{"run", "D"}}, now);
Json sd1 = mk("1", "oauth", "t1", 10, 50);
place1(D, sd1, cfg2, chooserFor("1", "m0"));
place1(D, sd1, cfg2, chooserFor("1", "m1"));
Json sm = mk("1", "oauth", "t1", 10, 50);
fold1(A, sm, now);
// The board's reading of the account (40, B's) folded in before the
// projection.
const Json base = sm["windows"]["five_hour"]["utilization"];
Check("a driver's passes fold at its own projection (2 % each) into another driver's (0.25 %)",
Dyn::Equal(sm["foreign"], {{"count", 2}, {"cost5h", 4.0}, {"costUsd", 0}, {"drivers", {{D.Me(), 2}}}}) && base == 40 && SeatPool::OauthProjected(sm, cfg) == base.get<double>() + 4.0,
std::format("{} {}", DumpSpaced(sm["foreign"]), base.dump()));
const std::optional<std::string> unviewed = Platform::ReadFile(path);
Json v = A.View(now);
const std::string km = sm.at("boardKey").get<std::string>();
Check("view: the drivers and, per account, the passes in flight by driver - nothing written",
v["drivers"].contains(A.Me()) && v["drivers"].contains(D.Me()) && v["seats"][km]["inflight"] == 2 && Dyn::Equal(v["seats"][km]["by"], {{D.Me(), 2}}) && v["drivers"][D.Me()]["inflight"] == 2
&& Platform::ReadFile(path) == unviewed);
D.Leave();
Check("no secret on the board", !DumpSpaced(Read(path).at("seats")).contains("t1") && !Platform::ReadFile(path).value_or("").contains("k1"));
// Memory across drivers (2026-10-31): what the board's passes were
// admitted at and the stacks they hold.
D.Join({{"run", "D"}}, now);
Json sE = mk("1", "oauth", "t1", 10, 50);
place1(D, sE, cfg, chooserFor("1", "e0", {{"_admitted", 840.0}, {"stackKey", "http://127.0.0.1:5173"}}));
place1(D, sE, cfg, chooserFor("1", "e1", {{"_admitted", 210.0}}));
// Another account: memory and stacks are the machine's, not a seat's.
Json sA = mk("5", "oauth", "t5", 10, 50);
fold1(A, sA, now);
Json entry = Read(path)["seats"][sE.at("boardKey").get<std::string>()]["inflight"][std::format("{}/e0", D.Me())];
const SeatPool::Others& others = A.OthersNow();
Check("a pass on the board carries the memory it was admitted at and the stack it holds; another driver's fold sees every pass on the machine, whatever account",
Dyn::Equal(entry["mb"], 840.0) && entry["stack"] == "http://127.0.0.1:5173" && others.count == 2 && others.mb == 1050.0 && others.stacks == std::set<std::string>{"http://127.0.0.1:5173"}
&& others.unmeasured == 0 && sA["foreign"]["count"] == 0,
std::format("{} passes, {} MB", others.count, others.mb));
// The memory base the gates budget from.
D.Leave();
const auto r0 = A.MemoryBase(50'000, now);
D.Join({{"run", "D"}}, now);
place1(D, sE, cfg, chooserFor("1", "f0", {{"_admitted", 840.0}}));
const auto r1 = A.MemoryBase(30'000, now);
// A pass that records no memory (a driver that does not measure it).
place1(D, sE, cfg, chooserFor("1", "f1"));
const auto r2 = A.MemoryBase(30'000, now);
Check("memory base: with no pass in flight on the machine the reading of now, recorded; with other drivers' measured passes in flight the board's reading from before them; with one that records none, the reading of now",
r0 == std::pair<double, std::string>{50'000, "no pass in flight on this machine"} && r1.first == 50'000 && r1.second.contains("the board's reading") && r1.second.contains("840 MB")
&& r2.first == 30'000 && r2.second.contains("record no memory"),
std::format("{} | {} | {}", r0.second, r1.second, r2.second));
D.Leave();
A.Leave();
child2.Kill();
// One live driver per scratch dir.
const fs::path sd = td / "h8-run-2";
const Sleeper child4;
fs::create_directories(sd);
Write(sd / ".driver", child4.key);
const bool held = SeatPool::ClaimScratch(sd);
child4.Kill();
const bool takenOver = SeatPool::ClaimScratch(sd);
Check("a scratch dir a live driver claimed is refused; a dead claim is taken over; a fresh dir is claimed with our pid",
!held && takenOver && Platform::ReadFile(sd / ".driver") == mine && SeatPool::ClaimScratch(td / "h8-run-3") && SeatPool::ClaimScratch(sd));
// One full run per hash.
const fs::path gd = td / "gate-state";
fs::create_directories(gd);
auto f = std::make_unique<State::RunLock>(gd, "commits", "abcd1234");
try {
const State::RunLock second(gd, "commits", "abcd1234");
Check("run lock: a second full run on the hash is refused", false);
} catch (const State::RunLocked& e) {
Check("run lock: a second full run on the hash is refused, the holder not yet identified", std::string_view(e.what()).contains("not identified"), e.what());
}
f->WriteInfo({{"pid", 1}, {"since", "s"}, {"run_n", 1}, {"scratch", "x"}});
f->WriteInfo({{"pid", 1}, {"since", "s"}, {"run_n", 2}, {"scratch", "x"}});
try {
const State::RunLock second(gd, "commits", "abcd1234");
Check("run lock: ...named once it wrote itself", false);
} catch (const State::RunLocked& e) {
const std::optional<Json> holder = State::LockHolder(gd, "commits", "abcd1234");
Check("run lock: ...named once it wrote itself (one JSON document after two writes)",
std::string_view(e.what()).contains("pid 1") && std::string_view(e.what()).contains("run 2") && holder && holder->at("run_n") == 2, e.what());
}
auto takes = [&](std::string_view hash8) {
try {
const State::RunLock taken(gd, "commits", hash8);
return true;
} catch (const State::RunLocked&) {
return false;
}
};
Check("run lock: another hash is free; nobody holds an untaken one", takes("ffff0000") && !State::LockHolder(gd, "commits", "00000000"));
f.reset();
Check("run lock: released on close (the kernel releases it on death the same way)", !State::LockHolder(gd, "commits", "abcd1234") && takes("abcd1234"));
// unit-state merge-on-write
const fs::path sp = td / "unit-state.json";
const Json a1 = {{"unit", "X"}, {"hash", "H1"}};
const Json b1 = {{"unit", "Y"}, {"hash", "H1"}};
const Json read1 = {{"hash", "H1"}, {"round", 1}, {"run", "r1"}};
State::SaveState(sp, {{"ka", a1}}, {}, read1);
Json m = State::SaveState(sp, {{"kb", b1}}, {}, read1);
Check("unit-state merge: two drivers' stores both survive, each with its read",
Keys(m["entries"]) == std::set<std::string>{"ka", "kb"} && Dyn::Equal(m["entries"]["ka"]["reads"], Json::array({read1})) && Keys(State::LoadState(sp)["entries"]) == std::set<std::string>{"ka", "kb"});
const std::vector<std::string> dropKa = {"ka"};
m = State::SaveState(sp, {{"ka", a1}}, dropKa);
Check("...a drop wins over a set in the same store", Keys(m["entries"]) == std::set<std::string>{"kb"});
const Json read2 = {{"hash", "H2"}, {"round", 1}, {"run", "r2"}};
State::SaveState(sp, {{"kb", b1}}, {}, read2);
m = State::SaveState(sp, {{"kb", b1}}, {}, read2);
std::vector<std::string> reads;
for (const Json& x : m["entries"]["kb"]["reads"]) reads.push_back(std::format("{}/{}", x.at("hash").get<std::string>(), x.at("round").dump()));
Check("...a read of another hash is a second read, counted once however often it is stored", reads == std::vector<std::string>{"H1/1", "H2/1"}, Dyn::Join(reads, " "));
Check("...a whole pass carries no count", !State::SaveState(sp, {{"kw", {{"kind", "whole"}}}}, {}, {{"hash", "H"}, {"round", 1}, {"run", "r"}})["entries"]["kw"].contains("reads"));
Write(sp, "garbage");
Check("...garbage on disk is the empty state", Dyn::Equal(State::SaveState(sp, {{"k", {{"unit", "Z"}}}})["entries"], {{"k", {{"unit", "Z"}}}}));
const Json old = {{"version", 1}, {"entries", {{"q", {{"unit", "Q"}}}}}, {"personaPass", nullptr}};
State::SaveWhole(sp, old);
Check("...sets=None is the whole-file write as before", Dyn::Equal(Dyn::LoadJsonFile(sp), old));
// The ledger: two drivers on one file, every write a locked delta.
const fs::path lp = td / "seat-ledger.json";
SeatPool::Ledger la = SeatPool::Ledger::Load(lp, acfg);
SeatPool::Ledger lb = SeatPool::Ledger::Load(lp, acfg);
la.Add("api9", 2.0, now);
lb.Add("api9", 3.0, now);
la.Add("api9", 2.0, now);
lb.MarkOut("api9", "credit balance too low", now);
const Json last = Dyn::LoadJsonFile(lp).at("api9");
Check("two drivers on one ledger file: every add and the out mark survive, the last writer no longer wins",
Dyn::Equal(last.at("spentUsd"), 7.0) && last.at("procs") == 3 && Dyn::Truthy(last.at("outSince")) && Dyn::Equal(la.Entry("api9", now).at("spentUsd"), 7.0), DumpSpaced(last));
SeatPool::Ledger lc(lp, acfg, {{"api8", {{"cycle", "2099-01-01"}, {"spentUsd", 5.0}, {"procs", 1}, {"outSince", nullptr}, {"outReason", nullptr}}}});
Check("...an entry of a later cycle (another driver's cycle day) is not wiped", Dyn::Equal(lc.Entry("api8", now).at("spentUsd"), 5.0));
// The memory record: every driver's peaks in one file.
const fs::path mp = td / "memory.json";
std::vector<std::int64_t> recorders;
for (const std::string_view environment : {"none", "browser"}) {
recorders.push_back(Process::Start({.argv = {Platform::SelfExe().string(), "--record", mp.string(), td.string(), std::string(environment)}}).pid);
}
for (const std::int64_t pid : recorders) Process::Wait(pid);
const Memory::Calibration mc(mp, td, {});
Check("two drivers recording peaks into one memory file at once: every peak of both is kept", mc.Peaks("none").size() == 20 && mc.Peaks("browser").size() == 20,
std::format("{} {}", mc.Peaks("none").size(), mc.Peaks("browser").size()));
fs::remove_all(td);
return Failures == 0 ? 0 : 1;
}
}
int main(int argc, char** argv) {
// A second driver recording its peaks: twenty passes on one environment.
if (argc == 5 && std::string_view(argv[1]) == "--record") {
Memory::Calibration calibration(argv[2], argv[3], {});
for (std::int32_t i = 0; i < 20; ++i) calibration.Record(argv[4], 100.0 + i);
return 0;
}
try {
return Run();
} catch (const std::exception& error) {
std::println(std::cerr, "drivers: {}", error.what());
return 1;
}
}

167
tests/SeatPool/main.cpp Normal file
View file

@ -0,0 +1,167 @@
// SPDX-License-Identifier: AGPL-3.0-only
// SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts®
// The seat pool: seats from .env, placement, settling, credentials, the
// ledger; the checks that drive the Scheduler and the Runner are in the
// Scheduler test. Then reference vectors for the retry-after scan, the cycle
// start, the knobs, the OpenRouter cost and the description.
#include <stdlib.h>
#include <time.h>
import std;
import Crafter.Review;
namespace fs = std::filesystem;
using namespace Crafter::Review;
namespace {
std::int32_t Failures = 0;
void Check(std::string_view name, bool cond, std::string_view detail = {}) {
std::println("{}{}{}", cond ? " ok " : " FAIL ", name, !cond && !detail.empty() ? std::format(" ({})", detail) : std::string());
if (!cond) ++Failures;
}
Json With(Json value, const Json& changes) {
for (auto it = changes.begin(); it != changes.end(); ++it) value[it.key()] = it.value();
return value;
}
void Write(const fs::path& path, std::string_view text) {
fs::create_directories(path.parent_path());
std::ofstream(path, std::ios::binary) << text;
}
std::string Ids(const std::vector<Json>& seats) {
std::vector<std::string> ids;
for (const Json& s : seats) ids.push_back(s.at("id").get<std::string>());
return Dyn::Join(ids, ",");
}
}
int main() {
std::println("seat pool: seats from .env, placement, settling, credentials, the ledger, re-placement");
// The dates below are UTC's; the process zone decides what a timestamp's
// date is, as it does for dt.date.fromtimestamp.
setenv("TZ", "UTC", 1);
tzset();
Platform::SetMode(Platform::Mode::Native);
const fs::path td = fs::temp_directory_path() / std::format("crafter-review-seatpool-{}", Platform::PublishedPid());
fs::remove_all(td);
fs::create_directories(td);
setenv("REVIEW_GATES_BOARD", (td / "seat-board.json").c_str(), 1);
const fs::path envFile = td / ".env";
Write(envFile, "CLAUDE_CODE_OAUTH_TOKEN1=\"tok-one\"\nCLAUDE_CODE_OAUTH_TOKEN2=tok-two\nANTHROPIC_API_KEY1=sk-ant-api-one\nANTHROPIC_API_KEY=sk-ant-bare-never-a-seat\nMAILOSAUR=x\n");
const Json acfg = SeatPool::ApiConfig({{"REVIEW_GATES_API_BUDGET_USD", "10"}, {"REVIEW_GATES_API_LENS_COST_USD", "1"}, {"REVIEW_GATES_API_CONCURRENCY", "2"}});
const Json cfg = Seats::ConfigFromEnv({});
SeatPool::Ledger ledger(td / "ledger.json", acfg);
std::vector<Json> seats = SeatPool::LoadSeats({}, envFile, "", &ledger, acfg);
Check(".env: api seats, oauth seats, never the bare ANTHROPIC_API_KEY, local only when nothing is configured",
Ids(seats) == "api1,1,2" && Ids(SeatPool::LoadSeats({}, std::nullopt, "")) == "local", Ids(seats));
const std::string summary = DumpSpaced(SeatPool::Summary(seats));
Check("no secret in the summary or the table", !(summary + SeatPool::Table(seats)).contains("tok-one") && !summary.contains("sk-ant"));
Check("the pool's default projection is 0.25% of the 5-hour window per process (measured 0.08, 2026-10-15); the env overrides",
Dyn::Equal(SeatPool::PoolConfig({}).at("lensCostPct"), 0.25) && Dyn::Equal(SeatPool::PoolConfig({{"REVIEW_GATES_LENS_COST_PCT", "5"}}).at("lensCostPct"), 5));
const double now = std::floor(Platform::Now());
Json& api1 = seats[0];
Json& s1 = seats[1];
Json& s2 = seats[2];
api1["state"] = "ok";
s1 = With(s1, {{"state", "ok"}, {"util", 60}, {"windows", {{"five_hour", {{"utilization", 60}, {"resetsAt", now + 600}}}, {"seven_day", {{"utilization", 10}, {"resetsAt", now + 86400}}}}}});
s2 = With(s2, {{"state", "ok"}, {"util", 50}, {"windows", {{"five_hour", {{"utilization", 20}, {"resetsAt", now + 900}}}, {"seven_day", {{"utilization", 50}, {"resetsAt", now + 86400}}}}}});
auto pick = [&](const Json& apiConfig) {
const Json* picked = SeatPool::Pick(seats, now, cfg, apiConfig);
return picked ? picked->at("id").get<std::string>() : std::string("None");
};
const std::string first = pick(acfg);
api1["running"] = 2;
Check("api seat first; at its cap the oauth seat with the most 5-hour headroom", first == "api1" && pick(acfg) == "2");
api1["running"] = 0;
api1["spentUsd"] = 9.5;
Check("api seat without budget room: skipped; oauth-first order", pick(acfg) == "2" && pick(With(acfg, {{"order", "oauth-first"}, {"budgetUsd", 100}})) == "2");
std::vector<std::string> logs;
auto log = [&](const std::string& line) { logs.push_back(line); };
s1 = With(s1, {{"state", "ok"}, {"util", 20}, {"running", 1}, {"windows", {{"five_hour", {{"utilization", 20}, {"resetsAt", now + 600}}}}}});
Json limit = {{"status", "limit"}, {"reason", "rate limit"}, {"rateType", "five_hour"}, {"rateLimit", {{"five_hour", {{"utilization", 100}, {"resetsAt", now + 600}}}}}};
bool requeued = SeatPool::Settle(s1, limit, now, cfg, acfg, &ledger, log);
Check("5h limit on an oauth seat: limited, paused until its reset + 30 s, requeued",
requeued && s1.at("state") == "limited" && s1.at("pauseUntil").get<double>() == now + 630 && !SeatPool::Eligible(s1, now, cfg, acfg));
Json spend = {{"status", "ok"}, {"costUsd", 2.5}, {"rateLimit", Json::object()}};
requeued = SeatPool::Settle(api1, spend, now, cfg, acfg, &ledger, log);
Check("api seat: spend accrues in the seat and the ledger, and the budget takes the seat out",
!requeued && api1.at("spentUsd").get<double>() == 12.0 && ledger.Entry("api1").at("spentUsd").get<double>() == 2.5 && api1.at("state") == "out");
Json api2 = SeatPool::NewSeat("api2", "api", "k");
api2["state"] = "ok";
Json credits = {{"status", "credits"}, {"reason", "Your credit balance is too low"}, {"costUsd", 0}, {"rateLimit", Json::object()}};
requeued = SeatPool::Settle(api2, credits, now, cfg, acfg, &ledger, log);
Check("credit error: out, recorded in the ledger, requeued", requeued && api2.at("state") == "out" && Dyn::Truthy(ledger.Entry("api2").at("outSince")));
const fs::path scratch = td / "scratch";
fs::create_directories(scratch);
const std::map<std::string, std::string> baseEnv = {{"PATH", "/bin"}, {"ANTHROPIC_API_KEY", "leak"}, {"CLAUDE_CODE_OAUTH_TOKEN", "leak2"}, {"HOME", td.string()}};
const std::map<std::string, std::string> e1 = SeatPool::ChildEnv(SeatPool::NewSeat("1", "oauth", "tok-one"), baseEnv, scratch);
const fs::path cred = scratch / "seat-1" / ".credentials.json";
Check("oauth seat: its own config dir, the token in .credentials.json (0600), no token variable",
e1.at("CLAUDE_CONFIG_DIR") == (scratch / "seat-1").string() && Dyn::LoadJsonFile(cred).at("claudeAiOauth").at("accessToken") == "tok-one"
&& (fs::status(cred).permissions() & fs::perms::all) == (fs::perms::owner_read | fs::perms::owner_write) && !e1.contains("CLAUDE_CODE_OAUTH_TOKEN") && !e1.contains("ANTHROPIC_API_KEY"));
std::map<std::string, std::string> launcher = baseEnv;
launcher["CLAUDE_CODE_ENTRYPOINT"] = "claude-vscode";
const std::map<std::string, std::string> e2 = SeatPool::ChildEnv(SeatPool::NewSeat("api1", "api", "sk-ant-api-one"), launcher, scratch);
Check("api seat: the key in ANTHROPIC_API_KEY, the launcher's CLAUDE_CODE_ENTRYPOINT dropped, the key approved in its own .claude.json",
e2.at("ANTHROPIC_API_KEY") == "sk-ant-api-one" && !e2.contains("CLAUDE_CODE_ENTRYPOINT")
&& Dyn::Equal(Dyn::LoadJsonFile(scratch / "seat-api1" / ".claude.json").at("customApiKeyResponses").at("approved"), Json::array({"sk-ant-api-one"})));
// ---- reference vectors ------------------------------------------------------------------
const std::vector<std::pair<std::string, double>> retry = {
{"Retry-After: 30", 35},
{"retry after 2000 seconds", 900},
{"RETRY-AFTER:\u0663", 30},
{"please retry after a moment", 90},
{std::format("retry-after:{}45", std::string(14, ' ')), 90},
{std::format("retry-after:{}45", std::string(12, ' ')), 90},
{"x retry_after 5 retry after 7", 30},
{"retry-after: \u0663\u0660 x", 30},
{"Retry after 99999999999999999999999", 900},
{"", 90},
{"retry\u2028after 5", 90},
{"RETRY AFTER\n\n12", 30},
};
std::vector<std::string> missed;
for (const auto& [text, want] : retry) {
if (SeatPool::RetryAfter(text, 90) != want) missed.push_back(std::format("{} -> {}", Dyn::Repr(text), SeatPool::RetryAfter(text, 90)));
}
Check("retry-after: the regex's greedy run, Unicode digits, the 30..900 clamp, the fallback", missed.empty(), Dyn::Join(missed, "; "));
const std::vector<std::tuple<double, std::int32_t, std::string>> cycles = {
{1788775200.0, 1, "2026-09-01"}, {1788775200.0, 7, "2026-09-07"}, {1788775200.0, 8, "2026-08-08"}, {1788775200.0, 28, "2026-08-28"},
{1772323200.0, 28, "2026-02-28"}, {1772323200.0, 15, "2026-02-15"}, {1767225599.0, 5, "2025-12-05"}, {1709164800.0, 28, "2024-02-28"},
};
missed.clear();
for (const auto& [at, day, want] : cycles) {
if (SeatPool::CycleStart(at, day) != want) missed.push_back(std::format("{} {} -> {}", at, day, SeatPool::CycleStart(at, day)));
}
Check("cycle start: this month's billing day once reached, else last month's, clamped to its length", missed.empty(), Dyn::Join(missed, "; "));
const std::string knobs = DumpSpaced(SeatPool::ApiConfig({{"REVIEW_GATES_API_BUDGET_USD", " 1_0 "},
{"REVIEW_GATES_API_CONCURRENCY", "3.9"},
{"REVIEW_GATES_API_CYCLE_DAY", "40"},
{"REVIEW_GATES_SEAT_ORDER", "oauth-first"},
{"REVIEW_GATES_OPENROUTER_BUDGET_USD", "-1"},
{"REVIEW_GATES_API_LENS_COST_USD", "nan"}}));
Check("api config: float() parsing, int() truncation, the cycle day clamped, a negative or NaN value the default",
knobs == R"({"budgetUsd": 10.0, "lensCostUsd": 1.0, "concurrency": 3, "cycleDay": 28, "order": "oauth-first", "orBudgetUsd": 100.0, "orLensCostUsd": 0.1, "orConcurrency": 16})", knobs);
const double cost = SeatPool::OpenRouterCost("openrouter/xiaomi/mimo-v2.6-pro", {{"input", 1234}, {"cacheCreation", 10}, {"cacheRead", 99999}, {"output", 777}});
Check("openrouter cost: tokens x the built-in list prices, an unknown model free", cost == 0.001577 && SeatPool::OpenRouterCost("nope", {{"input", 1}}) == 0.0, Dyn::FloatRepr(cost));
Json described = SeatPool::NewSeat("1", "oauth", "t");
described["state"] = "ok";
described["windows"] = {{"five_hour", {{"utilization", 61.5}, {"resetsAt", 1}}}, {"seven_day", {{"utilization", 18}, {"resetsAt", 2}}}, {"seven_day_overage_included", {{"utilization", 22}, {"resetsAt", 0}}},
{"seven_day_fable", {{"utilization", 3}}}, {"other", {{"utilization", nullptr}}}};
described["binding"] = "five_hour";
Json paid = SeatPool::NewSeat("api1", "api", "k");
paid = With(paid, {{"state", "out"}, {"error", "budget"}, {"spentUsd", 12.345}, {"remainingUsd", 3}});
Check("describe: an OAuth seat's windows by label, a paid seat's spend and what is left on the key",
SeatPool::Describe(described) == "ok: 5h 61.5%, 7d 18%, fable 22%, fable 3% (binding five_hour)" && SeatPool::Describe(paid) == "out (budget), $12.35 spent this cycle, $3.00 left on the key",
std::format("{} | {}", SeatPool::Describe(described), SeatPool::Describe(paid)));
fs::remove_all(td);
return Failures == 0 ? 0 : 1;
}