Catcrafts/Crafter.Review
Watch
2
0
Fork
You've already forked Crafter.Review
 0
Code Issues 7 Pull requests Projects Releases 3 Packages Wiki Activity Actions
Crafter.Review/tests/SeatPool/main.cpp
Jorijn van der Graaf c28307601b
 feat(seats): accounts without a 5-hour limit 
Anthropic is removing the 5-hour limit in stages (from 2026-10-31), and
an account without it reports the weekly window alone. Until now the pool
took such an account for an unprobed seat: always eligible, ranked after
every other, never held to the weekly cutoff, with no cap on its passes.

- The burst window is the 5-hour one, else the weekly one
  (Seats::BurstWindow). An account whose burst window is the weekly one is
  ranked by it and admitted under the 95 % cutoff with the passes in
  flight projected on it, at the measured share per pass (0.04 at xhigh,
  0.125 otherwise; REVIEW_GATES_LENS_COST_7D_PCT), and waits for the weekly
  reset when over it.
- Every OAuth account and the local login takes at most 128 passes at once,
  every driver's counted (REVIEW_GATES_OAUTH_CONCURRENCY): every run
  measured with 149 or more in flight on one account drew usage-limit
  rejections, none of the 119 with 139 or fewer.
- A 'concurrents' rejection pauses the seat for 20 s or its retry-after,
  not until a window's reset (a seat sat two hours at 10 % on one).
- Board entries, the folded foreign record and the board's view carry
  cost7d, so the drivers see each other's weekly load.
- `seats` and the log table say `no limit` in such an account's 5-hour
  column; the board's others-line names its weekly projection.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-31 17:07:17 +01:00

212 lines
17 KiB
C++
Raw Blame History

// SPDX-License-Identifier: AGPL-3.0-only
// SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts®
// The seat pool: seats from .env, placement, settling, credentials, the
// ledger; the checks that drive the Scheduler and the Runner are in the
// Scheduler test. Then reference vectors for the retry-after scan, the cycle
// start, the knobs, the OpenRouter cost and the description.
#include <stdlib.h>
#include <time.h>
import std;
import Crafter.Review;
namespace fs = std::filesystem;
using namespace Crafter::Review;
namespace {
std::int32_t Failures = 0;
void Check(std::string_view name, bool cond, std::string_view detail = {}) {
std::println("{}{}{}", cond ? " ok " : " FAIL ", name, !cond && !detail.empty() ? std::format(" ({})", detail) : std::string());
if (!cond) ++Failures;
}
Json With(Json value, const Json& changes) {
for (auto it = changes.begin(); it != changes.end(); ++it) value[it.key()] = it.value();
return value;
}
void Write(const fs::path& path, std::string_view text) {
fs::create_directories(path.parent_path());
std::ofstream(path, std::ios::binary) << text;
}
std::string Ids(const std::vector<Json>& seats) {
std::vector<std::string> ids;
for (const Json& s : seats) ids.push_back(s.at("id").get<std::string>());
return Dyn::Join(ids, ",");
}
}
int main() {
std::println("seat pool: seats from .env, placement, settling, credentials, the ledger, re-placement");
// The dates below are UTC's; the process zone decides what a timestamp's
// date is, as it does for dt.date.fromtimestamp.
setenv("TZ", "UTC", 1);
tzset();
Platform::SetMode(Platform::Mode::Native);
const fs::path td = fs::temp_directory_path() / std::format("crafter-review-seatpool-{}", Platform::PublishedPid());
fs::remove_all(td);
fs::create_directories(td);
setenv("REVIEW_GATES_BOARD", (td / "seat-board.json").c_str(), 1);
const fs::path envFile = td / ".env";
Write(envFile, "CLAUDE_CODE_OAUTH_TOKEN1=\"tok-one\"\nCLAUDE_CODE_OAUTH_TOKEN2=tok-two\nANTHROPIC_API_KEY1=sk-ant-api-one\nANTHROPIC_API_KEY=sk-ant-bare-never-a-seat\nMAILOSAUR=x\n");
const Json acfg = SeatPool::ApiConfig({{"REVIEW_GATES_API_BUDGET_USD", "10"}, {"REVIEW_GATES_API_LENS_COST_USD", "1"}, {"REVIEW_GATES_API_CONCURRENCY", "2"}});
const Json cfg = Seats::ConfigFromEnv({});
SeatPool::Ledger ledger(td / "ledger.json", acfg);
std::vector<Json> seats = SeatPool::LoadSeats({}, envFile, "", &ledger, acfg);
Check(".env: api seats, oauth seats, never the bare ANTHROPIC_API_KEY, local only when nothing is configured",
Ids(seats) == "api1,1,2" && Ids(SeatPool::LoadSeats({}, std::nullopt, "")) == "local", Ids(seats));
const std::string summary = DumpSpaced(SeatPool::Summary(seats));
Check("no secret in the summary or the table", !(summary + SeatPool::Table(seats)).contains("tok-one") && !summary.contains("sk-ant"));
Check("the pool's default projection is 0.25% of the 5-hour window per process (measured 0.08, 2026-10-15); the env overrides",
Dyn::Equal(SeatPool::PoolConfig({}).at("lensCostPct"), 0.25) && Dyn::Equal(SeatPool::PoolConfig({{"REVIEW_GATES_LENS_COST_PCT", "5"}}).at("lensCostPct"), 5));
const double now = std::floor(Platform::Now());
Json& api1 = seats[0];
Json& s1 = seats[1];
Json& s2 = seats[2];
api1["state"] = "ok";
s1 = With(s1, {{"state", "ok"}, {"util", 60}, {"windows", {{"five_hour", {{"utilization", 60}, {"resetsAt", now + 600}}}, {"seven_day", {{"utilization", 10}, {"resetsAt", now + 86400}}}}}});
s2 = With(s2, {{"state", "ok"}, {"util", 50}, {"windows", {{"five_hour", {{"utilization", 20}, {"resetsAt", now + 900}}}, {"seven_day", {{"utilization", 50}, {"resetsAt", now + 86400}}}}}});
auto pick = [&](const Json& apiConfig) {
const Json* picked = SeatPool::Pick(seats, now, cfg, apiConfig);
return picked ? picked->at("id").get<std::string>() : std::string("None");
};
const std::string first = pick(acfg);
api1["running"] = 2;
Check("api seat first; at its cap the oauth seat with the most 5-hour headroom", first == "api1" && pick(acfg) == "2");
api1["running"] = 0;
api1["spentUsd"] = 9.5;
Check("api seat without budget room: skipped; oauth-first order", pick(acfg) == "2" && pick(With(acfg, {{"order", "oauth-first"}, {"budgetUsd", 100}})) == "2");
std::vector<std::string> logs;
auto log = [&](const std::string& line) { logs.push_back(line); };
s1 = With(s1, {{"state", "ok"}, {"util", 20}, {"running", 1}, {"windows", {{"five_hour", {{"utilization", 20}, {"resetsAt", now + 600}}}}}});
Json limit = {{"status", "limit"}, {"reason", "rate limit"}, {"rateType", "five_hour"}, {"rateLimit", {{"five_hour", {{"utilization", 100}, {"resetsAt", now + 600}}}}}};
bool requeued = SeatPool::Settle(s1, limit, now, cfg, acfg, &ledger, log);
Check("5h limit on an oauth seat: limited, paused until its reset + 30 s, requeued",
requeued && s1.at("state") == "limited" && s1.at("pauseUntil").get<double>() == now + 630 && !SeatPool::Eligible(s1, now, cfg, acfg));
Json spend = {{"status", "ok"}, {"costUsd", 2.5}, {"rateLimit", Json::object()}};
requeued = SeatPool::Settle(api1, spend, now, cfg, acfg, &ledger, log);
Check("api seat: spend accrues in the seat and the ledger, and the budget takes the seat out",
!requeued && api1.at("spentUsd").get<double>() == 12.0 && ledger.Entry("api1").at("spentUsd").get<double>() == 2.5 && api1.at("state") == "out");
Json api2 = SeatPool::NewSeat("api2", "api", "k");
api2["state"] = "ok";
Json credits = {{"status", "credits"}, {"reason", "Your credit balance is too low"}, {"costUsd", 0}, {"rateLimit", Json::object()}};
requeued = SeatPool::Settle(api2, credits, now, cfg, acfg, &ledger, log);
Check("credit error: out, recorded in the ledger, requeued", requeued && api2.at("state") == "out" && Dyn::Truthy(ledger.Entry("api2").at("outSince")));
// 2026-10-31: a 'concurrents' rejection (too many requests at once on the
// account) is a short back-off, not the window's reset.
Json sc = With(SeatPool::NewSeat("c", "oauth", "t"), {{"state", "ok"}, {"util", 10}, {"running", 1}, {"pauseUntil", 0}, {"windows", {{"five_hour", {{"utilization", 10}, {"resetsAt", now + 7000}}}}}});
Json concurrents = {{"status", "limit"}, {"reason", "You've hit your usage limit"}, {"rateType", "concurrents"}, {"rateLimit", {{"five_hour", {{"utilization", 10}, {"resetsAt", now + 7000}}}}}};
requeued = SeatPool::Settle(sc, concurrents, now, cfg, acfg, &ledger, log);
Check("a 'concurrents' rejection on an oauth seat: requeued, paused for seconds, not until the 5h reset",
requeued && sc.at("state") == "limited" && sc.at("pauseUntil").get<double>() == now + SeatPool::ConcurrentBackoffS && !SeatPool::Eligible(sc, now, cfg, acfg));
Json backOk = {{"status", "ok"}, {"rateLimit", {{"five_hour", {{"utilization", 11}, {"resetsAt", now + 7000}}}}}};
requeued = SeatPool::Settle(sc, backOk, now + SeatPool::ConcurrentBackoffS + 1, cfg, acfg, &ledger, log);
Check("...and the seat is back after the back-off", !requeued && sc.at("state") == "ok" && SeatPool::Eligible(sc, now + SeatPool::ConcurrentBackoffS + 1, cfg, acfg));
// 2026-10-31: an account whose 5-hour limit is removed reports the weekly
// window alone; until then the pool took it for an unprobed seat - ranked
// after every other, never held to the weekly cutoff. The cutoff is 95 %.
const Json px = SeatPool::PoolConfig({}, "xhigh");
Json nf = With(SeatPool::NewSeat("n", "oauth", "tn"), {{"state", "ok"}, {"running", 0}, {"pauseUntil", 0}, {"windows", {{"seven_day", {{"utilization", 3}, {"resetsAt", now + 6 * 86400}}}}}});
Json f15 = With(SeatPool::NewSeat("f", "oauth", "tf"), {{"state", "ok"}, {"running", 0}, {"pauseUntil", 0},
{"windows", {{"five_hour", {{"utilization", 15}, {"resetsAt", now + 3600}}}, {"seven_day", {{"utilization", 11}, {"resetsAt", now + 86400}}}}}});
std::vector<Json> pair = {f15, nf};
std::vector<std::string> order;
for (const Json* s : SeatPool::Ranked(pair, now, px, acfg)) order.push_back(s->at("id").get<std::string>());
Check("an account without a 5-hour window is placed by its weekly one: ranked by it, ahead of a seat at 5h 15 %, its running passes projected at the weekly figure",
order == std::vector<std::string>{"n", "f"} && std::abs(SeatPool::OauthProjected(With(nf, {{"running", 100}}), px) - 7.0) < 1e-9, Dyn::Join(order, ","));
Json n93 = With(nf, {{"windows", {{"seven_day", {{"utilization", 93}, {"resetsAt", now + 86400}}}}}});
Json n93r40 = With(n93, {{"running", 40}});
Json n93r60 = With(n93, {{"running", 60}});
Check("...under the weekly cutoff alone, no 5-hour ceiling: at 93 % it takes a pass with 40 running, not with 60 (93 + 61 x 0.04 >= 95)",
SeatPool::Eligible(n93r40, now, px, acfg) && !SeatPool::Eligible(n93r60, now, px, acfg));
Json n95 = With(nf, {{"windows", {{"seven_day", {{"utilization", 95}, {"resetsAt", now + 86400}}}}}});
Check("...and over the cutoff it waits for its weekly reset + 30 s", !SeatPool::Eligible(n95, now, px, acfg) && SeatPool::EarliestResume({n95}, now) == now + 86430);
Json n127 = With(nf, {{"running", 127}});
Json n128 = With(nf, {{"running", 128}});
Json nShared = With(nf, {{"running", 100}, {"foreign", {{"count", 28}, {"cost5h", 0.0}, {"cost7d", 0.0}, {"costUsd", 0.0}}}});
Json f127 = With(f15, {{"running", 127}});
Json f128 = With(f15, {{"running", 128}});
Check("128 passes per OAuth account, every driver counted, whatever its windows; REVIEW_GATES_OAUTH_CONCURRENCY overrides",
SeatPool::Eligible(n127, now, px, acfg) && !SeatPool::Eligible(n128, now, px, acfg) && !SeatPool::Eligible(nShared, now, px, acfg) && SeatPool::Eligible(f127, now, px, acfg)
&& !SeatPool::Eligible(f128, now, px, acfg) && px.at("oauthConcurrency") == 128 && SeatPool::PoolConfig({{"REVIEW_GATES_OAUTH_CONCURRENCY", "64"}}, "xhigh").at("oauthConcurrency") == 64);
Json weekly = {{"status", "ok"}, {"rateLimit", {{"seven_day", {{"utilization", 4}, {"resetsAt", now + 6 * 86400}}}}}};
requeued = SeatPool::Settle(nf, weekly, now, px, acfg, &ledger, log);
Check("...a finished pass's reading keeps it a weekly seat, unpaused; the log table says `no limit` in its 5h column",
!requeued && Seats::BurstWindow(nf.at("windows")) == "seven_day" && nf.at("pauseUntil") == 0 && SeatPool::Table({nf}).contains("| no limit |"), SeatPool::Table({nf}));
Check("the weekly share of a pass follows the run's effort (0.04 at xhigh, 0.125 otherwise); the env overrides it",
Dyn::Equal(px.at("lensCost7dPct"), 0.04) && Dyn::Equal(SeatPool::PoolConfig({}).at("lensCost7dPct"), 0.125) && Dyn::Equal(SeatPool::PoolConfig({}, "max").at("lensCost7dPct"), 0.125)
&& Dyn::Equal(SeatPool::PoolConfig({{"REVIEW_GATES_LENS_COST_7D_PCT", "0.5"}}, "xhigh").at("lensCost7dPct"), 0.5));
const fs::path scratch = td / "scratch";
fs::create_directories(scratch);
const std::map<std::string, std::string> baseEnv = {{"PATH", "/bin"}, {"ANTHROPIC_API_KEY", "leak"}, {"CLAUDE_CODE_OAUTH_TOKEN", "leak2"}, {"HOME", td.string()}};
const std::map<std::string, std::string> e1 = SeatPool::ChildEnv(SeatPool::NewSeat("1", "oauth", "tok-one"), baseEnv, scratch);
const fs::path cred = scratch / "seat-1" / ".credentials.json";
Check("oauth seat: its own config dir, the token in .credentials.json (0600), no token variable",
e1.at("CLAUDE_CONFIG_DIR") == (scratch / "seat-1").string() && Dyn::LoadJsonFile(cred).at("claudeAiOauth").at("accessToken") == "tok-one"
&& (fs::status(cred).permissions() & fs::perms::all) == (fs::perms::owner_read | fs::perms::owner_write) && !e1.contains("CLAUDE_CODE_OAUTH_TOKEN") && !e1.contains("ANTHROPIC_API_KEY"));
std::map<std::string, std::string> launcher = baseEnv;
launcher["CLAUDE_CODE_ENTRYPOINT"] = "claude-vscode";
const std::map<std::string, std::string> e2 = SeatPool::ChildEnv(SeatPool::NewSeat("api1", "api", "sk-ant-api-one"), launcher, scratch);
Check("api seat: the key in ANTHROPIC_API_KEY, the launcher's CLAUDE_CODE_ENTRYPOINT dropped, the key approved in its own .claude.json",
e2.at("ANTHROPIC_API_KEY") == "sk-ant-api-one" && !e2.contains("CLAUDE_CODE_ENTRYPOINT")
&& Dyn::Equal(Dyn::LoadJsonFile(scratch / "seat-api1" / ".claude.json").at("customApiKeyResponses").at("approved"), Json::array({"sk-ant-api-one"})));
// ---- reference vectors ------------------------------------------------------------------
const std::vector<std::pair<std::string, double>> retry = {
{"Retry-After: 30", 35},
{"retry after 2000 seconds", 900},
{"RETRY-AFTER:\u0663", 30},
{"please retry after a moment", 90},
{std::format("retry-after:{}45", std::string(14, ' ')), 90},
{std::format("retry-after:{}45", std::string(12, ' ')), 90},
{"x retry_after 5 retry after 7", 30},
{"retry-after: \u0663\u0660 x", 30},
{"Retry after 99999999999999999999999", 900},
{"", 90},
{"retry\u2028after 5", 90},
{"RETRY AFTER\n\n12", 30},
};
std::vector<std::string> missed;
for (const auto& [text, want] : retry) {
if (SeatPool::RetryAfter(text, 90) != want) missed.push_back(std::format("{} -> {}", Dyn::Repr(text), SeatPool::RetryAfter(text, 90)));
}
Check("retry-after: the regex's greedy run, Unicode digits, the 30..900 clamp, the fallback", missed.empty(), Dyn::Join(missed, "; "));
const std::vector<std::tuple<double, std::int32_t, std::string>> cycles = {
{1788775200.0, 1, "2026-09-01"}, {1788775200.0, 7, "2026-09-07"}, {1788775200.0, 8, "2026-08-08"}, {1788775200.0, 28, "2026-08-28"},
{1772323200.0, 28, "2026-02-28"}, {1772323200.0, 15, "2026-02-15"}, {1767225599.0, 5, "2025-12-05"}, {1709164800.0, 28, "2024-02-28"},
};
missed.clear();
for (const auto& [at, day, want] : cycles) {
if (SeatPool::CycleStart(at, day) != want) missed.push_back(std::format("{} {} -> {}", at, day, SeatPool::CycleStart(at, day)));
}
Check("cycle start: this month's billing day once reached, else last month's, clamped to its length", missed.empty(), Dyn::Join(missed, "; "));
const std::string knobs = DumpSpaced(SeatPool::ApiConfig({{"REVIEW_GATES_API_BUDGET_USD", " 1_0 "},
{"REVIEW_GATES_API_CONCURRENCY", "3.9"},
{"REVIEW_GATES_API_CYCLE_DAY", "40"},
{"REVIEW_GATES_SEAT_ORDER", "oauth-first"},
{"REVIEW_GATES_OPENROUTER_BUDGET_USD", "-1"},
{"REVIEW_GATES_API_LENS_COST_USD", "nan"}}));
Check("api config: float() parsing, int() truncation, the cycle day clamped, a negative or NaN value the default",
knobs == R"({"budgetUsd": 10.0, "lensCostUsd": 1.0, "concurrency": 3, "cycleDay": 28, "order": "oauth-first", "orBudgetUsd": 100.0, "orLensCostUsd": 0.1, "orConcurrency": 16})", knobs);
const double cost = SeatPool::OpenRouterCost("openrouter/xiaomi/mimo-v2.6-pro", {{"input", 1234}, {"cacheCreation", 10}, {"cacheRead", 99999}, {"output", 777}});
Check("openrouter cost: tokens x the built-in list prices, an unknown model free", cost == 0.001577 && SeatPool::OpenRouterCost("nope", {{"input", 1}}) == 0.0, Dyn::FloatRepr(cost));
Json described = SeatPool::NewSeat("1", "oauth", "t");
described["state"] = "ok";
described["windows"] = {{"five_hour", {{"utilization", 61.5}, {"resetsAt", 1}}}, {"seven_day", {{"utilization", 18}, {"resetsAt", 2}}}, {"seven_day_overage_included", {{"utilization", 22}, {"resetsAt", 0}}},
{"seven_day_fable", {{"utilization", 3}}}, {"other", {{"utilization", nullptr}}}};
described["binding"] = "five_hour";
Json paid = SeatPool::NewSeat("api1", "api", "k");
paid = With(paid, {{"state", "out"}, {"error", "budget"}, {"spentUsd", 12.345}, {"remainingUsd", 3}});
Check("describe: an OAuth seat's windows by label, a paid seat's spend and what is left on the key",
SeatPool::Describe(described) == "ok: 5h 61.5%, 7d 18%, fable 22%, fable 3% (binding five_hour)" && SeatPool::Describe(paid) == "out (budget), $12.35 spent this cycle, $3.00 left on the key",
std::format("{} | {}", SeatPool::Describe(described), SeatPool::Describe(paid)));
fs::remove_all(td);
return Failures == 0 ? 0 : 1;
}
Reference in a new issue View git blame Copy permalink