feat(pool): the pool - stacks, admission, the runner and the scheduler
The allowlists and the instrument checks (mcp_connected, db_answered, exercised), the stack keys, Stacks and Admission, the Runner that runs one claude or opencode process per pass, and the Scheduler with its retries, rate-window pauses, the prompt-prefix warm-up, the seats and the board. One thread per pass; each worker runs on its own copy of the job. Every pass runs under the supervisor, so a timeout or an abort takes its whole tree down; a timed-out pass reports exit -9, a SIGKILLed child's code. The helpers the lenses run are this binary's `sql` and `totp`. :Process gains an Append stream for the OpenCode meta line. The tests cover the pool and its share of the commits-mode, seat-pool and several-drivers checks, a fake claude under the supervisor included. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
5253fe3aff
commit
554a5ddd48
GPG key ID:
2937E59CDCC1BCFB
8 changed files with 2132 additions and 4 deletions
1065
implementations/Crafter.Review-Pool.cpp
Normal file
1065
implementations/Crafter.Review-Pool.cpp
Normal file
File diff suppressed because it is too large
Load diff
|
|
@ -97,9 +97,12 @@ namespace Crafter::Review::Process {
|
|||
return -1;
|
||||
case Stream::Kind::Null:
|
||||
return open("/dev/null", (reading ? O_RDONLY : O_WRONLY) | O_CLOEXEC);
|
||||
// Created 0666 less the umask, truncated.
|
||||
case Stream::Kind::File:
|
||||
return reading ? open(stream.file.c_str(), O_RDONLY | O_CLOEXEC)
|
||||
: open(stream.file.c_str(), O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC, 0644);
|
||||
: open(stream.file.c_str(), O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC, 0666);
|
||||
case Stream::Kind::Append:
|
||||
return open(stream.file.c_str(), O_WRONLY | O_CREAT | O_APPEND | O_CLOEXEC, 0666);
|
||||
case Stream::Kind::Pipe: {
|
||||
std::array<std::int32_t, 2> ends {};
|
||||
if (pipe2(ends.data(), O_CLOEXEC) != 0) return -2;
|
||||
|
|
|
|||
269
interfaces/Crafter.Review-Pool.cppm
Normal file
269
interfaces/Crafter.Review-Pool.cppm
Normal file
|
|
@ -0,0 +1,269 @@
|
|||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
// SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts®
|
||||
|
||||
export module Crafter.Review:Pool;
|
||||
import std;
|
||||
import :Json;
|
||||
import :SeatPool;
|
||||
import :Memory;
|
||||
import :PgWire;
|
||||
|
||||
// One `claude --print` process per pass, a pool with gating, retries,
|
||||
// rate-window pauses and progress; with seats every process runs on the seat
|
||||
// picked for it and a seat's limit pauses that seat, not the pool. What else
|
||||
// holds a launch back is the pass's environment, never its lens (Admission):
|
||||
// a pass that drives an application waits for a stack of its own (Stacks),
|
||||
// and every pass waits until it fits in free memory at its environment's
|
||||
// measured cost. A pass with the database alone shares the first stack:
|
||||
// every SQL call is its own transaction, rolled back.
|
||||
//
|
||||
// Every pass runs under the supervisor (`crafter-review __supervise`), so a
|
||||
// timeout, an abort or the driver's death takes its whole tree down - the
|
||||
// MCP server and Chromium with it.
|
||||
export namespace Crafter::Review::Pool {
|
||||
using Log = std::function<void(const std::string&)>;
|
||||
// (status, reason)
|
||||
using Verdict = std::pair<std::string, std::string>;
|
||||
|
||||
// seats.find_claude_bin
|
||||
std::string DefaultBin();
|
||||
// The exact Bash prefix a lens with the database under test runs SQL with:
|
||||
// this binary's `sql`, spelled absolute (allowlisted as a prefix).
|
||||
std::string SqlCommand();
|
||||
// The exact Bash line a lens with an application under test runs for a
|
||||
// one-time code: this binary's `totp <fixture>`; nullopt without one.
|
||||
std::optional<std::string> TotpCommand(const std::optional<std::string>& fixture);
|
||||
// Trees outside the working directory: the CLI must be told (--add-dir)
|
||||
// and Read allowed there, because --setting-sources '' drops the
|
||||
// project's additionalDirectories and a read there would prompt.
|
||||
std::vector<std::string> ExtraDirs(std::span<const std::string> trees);
|
||||
// A pass with the browser that returned ok, fail closed: the Playwright
|
||||
// server connected with its tools (2026-10-15: --safe-mode had dropped it
|
||||
// and four units came back n/a), and the browser answered - an empirical
|
||||
// lens's whole pass must have called it, and no pass may have had every
|
||||
// browser call fail.
|
||||
Verdict McpConnected(const Json& parsed, std::string_view kind = "block", bool empirical = true);
|
||||
// A pass that ran the SQL helper and never got the database's answer had
|
||||
// no database: its verdict is not evidence from one.
|
||||
Verdict DbAnswered(const Json& parsed, std::string_view kind = "block", bool empirical = false);
|
||||
// Every instrument the environment gives must show it was there.
|
||||
Verdict Exercised(const Json& parsed, std::string_view environment, std::string_view kind = "block", bool empirical = false);
|
||||
// A stack's name on the board, the same in every driver: its
|
||||
// application's address, localhost and 127.0.0.1 one host, a default port
|
||||
// spelled out; a stack without a URL is its id.
|
||||
std::string StackKey(const Json& stack);
|
||||
// The read-only tools, git -C on every tree in both spellings, and what
|
||||
// the environment adds: the browser tools and curl, the SQL helper, and
|
||||
// with an application the one-time-code command and the config's extras.
|
||||
std::vector<std::string> AllowedTools(std::string_view lensFile, std::string_view kind, std::span<const std::string> trees, std::span<const std::string> extra = {},
|
||||
const std::optional<std::string>& loginFixture = std::nullopt, std::string_view environment = "none");
|
||||
|
||||
// The stacks a run's passes use (2026-10-22): the one the config names,
|
||||
// or the slots brought up side by side. A pass whose environment drives an
|
||||
// application holds a stack of its own from launch to end, least recently
|
||||
// returned first; another driver's pass holds one too (`held`, the
|
||||
// StackKey of every stack the other drivers on this machine hold).
|
||||
class Stacks {
|
||||
public:
|
||||
explicit Stacks(std::vector<Json> stacks);
|
||||
std::size_t Size() const;
|
||||
// nullptr when there is none.
|
||||
const Json* First() const;
|
||||
const std::vector<Json>& All() const;
|
||||
const std::string& KeyOf(const std::string& id) const;
|
||||
// Is a stack free (prefer: the one a resumed session ran on - only
|
||||
// that one will do)?
|
||||
bool Can(const std::optional<std::string>& prefer = std::nullopt, const std::set<std::string>& held = {}) const;
|
||||
Json Take(const std::optional<std::string>& prefer = std::nullopt, const std::set<std::string>& held = {});
|
||||
void Give(const std::string& id);
|
||||
std::vector<std::string> Free() const;
|
||||
|
||||
private:
|
||||
std::vector<Json> all;
|
||||
std::map<std::string, std::string> keys;
|
||||
std::vector<std::string> free;
|
||||
mutable std::mutex lock;
|
||||
};
|
||||
|
||||
// May the pool launch this pass now? No per-lens number (2026-10-22): an
|
||||
// environment that drives an application needs a stack of its own, and the
|
||||
// pass must fit the memory budget - free memory at the gate's start less
|
||||
// the reserve, less what the passes in flight were admitted at - with the
|
||||
// machine still holding its cost free beyond the reserve right now. With
|
||||
// nothing in flight on the machine a pass is admitted whatever the memory
|
||||
// says, and the log says so once. `others` is what the other drivers'
|
||||
// passes hold, as the board's last fold left it.
|
||||
class Admission {
|
||||
public:
|
||||
using CostOf = std::function<double(const Json& job)>;
|
||||
using NeedsStack = std::function<bool(const Json& job)>;
|
||||
using Available = std::function<std::optional<double>()>;
|
||||
Admission(CostOf costOf, NeedsStack needsStack, Stacks* stacks = nullptr, std::optional<double> budget = std::nullopt, double reserve = 0, Available available = {}, Log log = {});
|
||||
bool Can(const Json& job);
|
||||
// The pass's memory and its stack taken: _admitted, stack, prevStack
|
||||
// and stackKey on the job.
|
||||
void Take(Json& job);
|
||||
void Give(Json& job);
|
||||
void SetOthers(SeatPool::Others others);
|
||||
SeatPool::Others OthersNow() const;
|
||||
std::int64_t Inflight() const;
|
||||
double Held() const;
|
||||
|
||||
private:
|
||||
CostOf costOf;
|
||||
NeedsStack needsStack;
|
||||
Stacks* stacks;
|
||||
std::optional<double> budget;
|
||||
double reserve;
|
||||
Available available;
|
||||
Log log;
|
||||
std::int64_t inflight = 0;
|
||||
double held = 0;
|
||||
bool warned = false;
|
||||
SeatPool::Others others;
|
||||
mutable std::mutex lock;
|
||||
std::optional<std::string> Prefer(const Json& job) const;
|
||||
};
|
||||
|
||||
struct RunnerOptions {
|
||||
std::filesystem::path repo;
|
||||
std::filesystem::path scratch;
|
||||
// "": REVIEW_GATES_CLAUDE_BIN, else DefaultBin().
|
||||
std::string bin;
|
||||
std::string effort = "xhigh";
|
||||
Log log;
|
||||
bool dryRun = false;
|
||||
std::string harness = "claude";
|
||||
std::optional<std::string> variant;
|
||||
std::string opencodeBin;
|
||||
// The config's functional section: the MCP server, the extra tools.
|
||||
Json functional = Json::object();
|
||||
std::optional<std::string> loginFixture;
|
||||
// {lens file: environment}
|
||||
std::map<std::string, std::string> environments;
|
||||
// The first stack's: a pass with the database alone runs there.
|
||||
std::optional<std::string> dbUrl;
|
||||
Memory::Sampler* sampler = nullptr;
|
||||
// The database URLs stay in the driver; a db pass gets the broker's
|
||||
// address and a token.
|
||||
PgWire::Broker* broker = nullptr;
|
||||
};
|
||||
|
||||
class Runner {
|
||||
public:
|
||||
explicit Runner(RunnerOptions options);
|
||||
// Every running pass stopped (its supervisor takes the tree down); how
|
||||
// many.
|
||||
std::int32_t KillChildren();
|
||||
std::string EnvOf(const Json& job) const;
|
||||
// A pass's process environment: the seat's (base), plus - when its
|
||||
// environment reaches the database - the broker's address and the
|
||||
// token of the database of the stack it holds (none for a stack
|
||||
// without one), else the run's first stack's.
|
||||
std::map<std::string, std::string> ProcessEnv(const Json& job, const std::map<std::string, std::string>& base) const;
|
||||
// The login fixture of the stack it holds, else the run's.
|
||||
std::optional<std::string> LoginOf(const Json& job) const;
|
||||
// The --mcp-config of a pass, nullopt for an environment without the
|
||||
// browser.
|
||||
std::optional<std::string> McpConfig(const Json& job) const;
|
||||
std::vector<std::string> Cmd(const Json& job, const std::filesystem::path& systemPath, std::string_view schema, std::span<const std::string> trees, std::int64_t maxTurns, bool persist,
|
||||
const std::string& sessionId, bool resume = false) const;
|
||||
// One pass: (record, the structured output or null).
|
||||
std::pair<Json, Json> Run(const Json& job, const std::string& prompt, const std::filesystem::path& systemPath, std::string_view schema, std::span<const std::string> trees,
|
||||
std::int64_t maxTurns, std::int64_t timeoutMin, bool persist = false);
|
||||
// The seats a job's `seat` names, as they were when the pool began:
|
||||
// only what never changes (id, kind, secret) is read from them.
|
||||
void SetSeats(const std::vector<Json>& seats);
|
||||
const std::string& Bin() const;
|
||||
// The environment every pass starts from.
|
||||
std::map<std::string, std::string>& Env();
|
||||
const std::filesystem::path& Scratch() const;
|
||||
std::filesystem::path opencodeHome;
|
||||
|
||||
private:
|
||||
RunnerOptions options;
|
||||
std::map<std::string, std::string> env;
|
||||
std::map<std::string, Json> seatsById;
|
||||
std::map<std::int64_t, bool> running;
|
||||
mutable std::mutex lock;
|
||||
std::pair<Json, Json> RunOpenCode(const Json& job, const std::string& prompt, const std::filesystem::path& systemPath, std::span<const std::string> trees, std::int64_t timeoutMin,
|
||||
const Json& seat, Json rec, const std::filesystem::path& out, const std::filesystem::path& err, const std::string& sessionId);
|
||||
std::optional<std::int32_t> Wait(std::int64_t pid, std::int64_t timeoutMin, bool& timedOut);
|
||||
};
|
||||
|
||||
struct SchedulerOptions {
|
||||
Runner* runner = nullptr;
|
||||
std::int64_t concurrency = 8;
|
||||
Log log;
|
||||
std::optional<std::filesystem::path> progressPath;
|
||||
double pauseAt = 85;
|
||||
double maxWaitMin = 360;
|
||||
// The key of a pass's prompt prefix (nullopt: none), which the driver
|
||||
// keeps on the job as `cacheKey`; and whether a running pass's first
|
||||
// response is in.
|
||||
std::function<std::optional<std::string>(Json& job)> cacheKey;
|
||||
std::function<bool(const Json& job)> warmProbe;
|
||||
double warmTtl = 240;
|
||||
double warmTimeout = 180;
|
||||
double warmPoll = 5;
|
||||
std::vector<Json>* seats = nullptr;
|
||||
// null: from the environment.
|
||||
Json seatConfig;
|
||||
Json apiConfig;
|
||||
SeatPool::Ledger* ledger = nullptr;
|
||||
std::function<double()> now;
|
||||
Admission* admission = nullptr;
|
||||
SeatPool::Board* board = nullptr;
|
||||
double noRoomWait = 30;
|
||||
// A stop that came before this pool (a signal, `crafter-review stop`):
|
||||
// the pool starts aborted.
|
||||
std::optional<std::string> abort;
|
||||
};
|
||||
|
||||
struct Done {
|
||||
Json job;
|
||||
Json rec;
|
||||
Json raw;
|
||||
};
|
||||
|
||||
// Runs jobs on a pool; onDone(job, rec, raw) -> the follow-up jobs (the
|
||||
// blocks after a passing whole pass). Retries: error/timeout once (x1.5
|
||||
// budget for a timeout); limit -> pause until resetsAt (or a backoff) and
|
||||
// requeue (<= 4); auth or a seven_day rejection -> abort. With seats every
|
||||
// launch is placed (with the board: placed, admitted and registered under
|
||||
// its lock, the pool width the machine's cap), every finished process
|
||||
// settled onto its seat and taken off the board. One thread per pass.
|
||||
class Scheduler {
|
||||
public:
|
||||
using ExecJob = std::function<std::pair<Json, Json>(const Json& job, std::int64_t attempt)>;
|
||||
using OnDone = std::function<std::vector<Json>(const Json& job, const Json& rec, const Json& raw)>;
|
||||
explicit Scheduler(SchedulerOptions options);
|
||||
const std::vector<Done>& Run(std::vector<Json> jobs, const ExecJob& exec, const OnDone& onDone);
|
||||
// From any thread: the pool stops launching and returns once what runs
|
||||
// has ended.
|
||||
void Abort(const std::string& reason);
|
||||
std::optional<std::string> Aborted() const;
|
||||
const std::vector<Done>& DoneList() const;
|
||||
// pause_until: 0 until a pause raised it.
|
||||
Json PauseUntil() const;
|
||||
|
||||
private:
|
||||
SchedulerOptions o;
|
||||
std::vector<Done> done;
|
||||
std::optional<std::string> abort;
|
||||
Json pauseUntil = 0;
|
||||
Json roomWait;
|
||||
double waitedMin = 0;
|
||||
bool heldOff = false;
|
||||
std::map<std::pair<std::string, std::string>, double> warm;
|
||||
std::map<std::pair<std::string, std::string>, std::pair<std::string, double>> warming;
|
||||
mutable std::mutex lock;
|
||||
std::condition_variable wake;
|
||||
bool Capped(const Json& job);
|
||||
bool Launchable(Json& job, const std::vector<Json*>& running, double now, const std::optional<std::string>& seatId);
|
||||
void RaisePause(double until);
|
||||
void Sleep(double seconds);
|
||||
void Progress(const std::vector<std::string>& running, const std::vector<std::string>& queued);
|
||||
void Log(const std::string& line) const;
|
||||
};
|
||||
}
|
||||
|
|
@ -32,6 +32,9 @@ export namespace Crafter::Review::Process {
|
|||
// Opened for reading (stdin) or created and truncated (stdout,
|
||||
// stderr).
|
||||
File,
|
||||
// stdout or stderr after what the file already holds: a line the
|
||||
// driver wrote before the child starts stays first.
|
||||
Append,
|
||||
// A pipe whose other end Start() returns.
|
||||
Pipe,
|
||||
};
|
||||
|
|
|
|||
|
|
@ -29,5 +29,6 @@ export import :Mcp;
|
|||
export import :Seats;
|
||||
export import :SeatPool;
|
||||
export import :OpenCode;
|
||||
export import :Pool;
|
||||
export import :Freeze;
|
||||
export import :Driver;
|
||||
|
|
|
|||
10
project.cpp
10
project.cpp
|
|
@ -13,7 +13,7 @@ using namespace Crafter;
|
|||
namespace {
|
||||
// The umbrella, then the partitions in import order: each after the ones
|
||||
// it imports, which is also the order the module map in AGENTS.md uses.
|
||||
constexpr std::array<std::string_view, 29> Interfaces = {
|
||||
constexpr std::array<std::string_view, 30> Interfaces = {
|
||||
"interfaces/Crafter.Review",
|
||||
"interfaces/Crafter.Review-Unicode",
|
||||
"interfaces/Crafter.Review-Json",
|
||||
|
|
@ -41,10 +41,11 @@ namespace {
|
|||
"interfaces/Crafter.Review-Seats",
|
||||
"interfaces/Crafter.Review-SeatPool",
|
||||
"interfaces/Crafter.Review-OpenCode",
|
||||
"interfaces/Crafter.Review-Pool",
|
||||
"interfaces/Crafter.Review-Freeze",
|
||||
"interfaces/Crafter.Review-Driver",
|
||||
};
|
||||
constexpr std::array<std::string_view, 26> Implementations = {
|
||||
constexpr std::array<std::string_view, 27> Implementations = {
|
||||
"implementations/Crafter.Review-Text",
|
||||
"implementations/Crafter.Review-Crypto",
|
||||
"implementations/Crafter.Review-Platform",
|
||||
|
|
@ -69,6 +70,7 @@ namespace {
|
|||
"implementations/Crafter.Review-Seats",
|
||||
"implementations/Crafter.Review-SeatPool",
|
||||
"implementations/Crafter.Review-OpenCode",
|
||||
"implementations/Crafter.Review-Pool",
|
||||
"implementations/Crafter.Review-Freeze",
|
||||
"implementations/Crafter.Review-Driver",
|
||||
};
|
||||
|
|
@ -80,7 +82,7 @@ namespace {
|
|||
|
||||
// One per directory under tests/, each an ordinary main() that links the
|
||||
// library: exit 0 passes, nonzero fails, 77 skips.
|
||||
constexpr std::array<std::string_view, 18> Tests = {
|
||||
constexpr std::array<std::string_view, 20> Tests = {
|
||||
"Crypto",
|
||||
"Text",
|
||||
"Platform",
|
||||
|
|
@ -99,6 +101,8 @@ namespace {
|
|||
"SeatPool",
|
||||
"Drivers",
|
||||
"OpenCode",
|
||||
"Runner",
|
||||
"Scheduler",
|
||||
};
|
||||
|
||||
// The version the binary reports: CI sets it from the tag, a checkout
|
||||
|
|
|
|||
277
tests/Runner/main.cpp
Normal file
277
tests/Runner/main.cpp
Normal file
|
|
@ -0,0 +1,277 @@
|
|||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
// SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts®
|
||||
|
||||
// The pool's command lines: the allowlists and the deny list, the MCP
|
||||
// config, the command line by environment, the process environment of a
|
||||
// pass with the database, mcp_connected and exercised, and the stream
|
||||
// accounting behind them. The helpers are this binary's `sql` and `totp`.
|
||||
|
||||
#include <stdlib.h>
|
||||
import std;
|
||||
import Crafter.Review;
|
||||
namespace fs = std::filesystem;
|
||||
using namespace Crafter::Review;
|
||||
|
||||
namespace {
|
||||
std::int32_t Failures = 0;
|
||||
const std::string Fn(Plan::FunctionalLens);
|
||||
|
||||
void Check(std::string_view name, bool cond, std::string_view detail = {}) {
|
||||
std::println("{}{}{}", cond ? " ok " : " FAIL ", name, !cond && !detail.empty() ? std::format(" ({})", detail) : std::string());
|
||||
if (!cond) ++Failures;
|
||||
}
|
||||
|
||||
bool Has(const std::vector<std::string>& list, std::string_view item) {
|
||||
return std::ranges::contains(list, item);
|
||||
}
|
||||
|
||||
std::ptrdiff_t Index(const std::vector<std::string>& list, std::string_view item) {
|
||||
const auto at = std::ranges::find(list, item);
|
||||
return at == list.end() ? -1 : at - list.begin();
|
||||
}
|
||||
|
||||
// c[c.index(flag) + 1]
|
||||
std::string After(const std::vector<std::string>& c, std::string_view flag) {
|
||||
const std::ptrdiff_t at = Index(c, flag);
|
||||
return at < 0 || at + 1 >= static_cast<std::ptrdiff_t>(c.size()) ? std::string() : c[static_cast<std::size_t>(at + 1)];
|
||||
}
|
||||
|
||||
Json Job(std::string_view label, std::string_view lensFile) {
|
||||
return {{"label", label}, {"model", "m"}, {"lensFile", lensFile}, {"kind", "block"}};
|
||||
}
|
||||
|
||||
Json With(Json value, const Json& changes) {
|
||||
for (auto it = changes.begin(); it != changes.end(); ++it) value[it.key()] = it.value();
|
||||
return value;
|
||||
}
|
||||
}
|
||||
|
||||
int main() {
|
||||
std::println("commits mode (the pool's part): allowlists, the deny list, the MCP config, the command line, the process environment, mcp_connected, exercised");
|
||||
Platform::SetMode(Platform::Mode::Native);
|
||||
const std::string cwd = fs::current_path().string();
|
||||
const std::vector<std::string> here = {"."};
|
||||
const std::vector<std::string> al = Pool::AllowedTools("repo-bug-prover.md", "block", here);
|
||||
Check("static lens allowlist: git -C . and absolute, python3 -, no browser tool, no curl, no Read(//) rule for the cwd",
|
||||
Has(al, "Bash(git -C . show:*)") && Has(al, std::format("Bash(git -C {} show:*)", cwd)) && Has(al, "Bash(python3 -:*)") && std::ranges::none_of(al, [](const std::string& x) { return x.starts_with("mcp__"); })
|
||||
&& !Has(al, "Bash(curl:*)") && std::ranges::none_of(al, [](const std::string& x) { return x.starts_with("Read("); }));
|
||||
const std::vector<std::string> extras = {"Bash(psql:*)", "mcp__x__y"};
|
||||
const std::string fixture = "cypress/fixtures/created-user.json";
|
||||
const std::vector<std::string> af = Pool::AllowedTools(Fn, "block", here, extras, fixture, "browser");
|
||||
const std::string totp = std::format("Bash({}:*)", Pool::TotpCommand(fixture).value_or(""));
|
||||
Check("functional allowlist: every Playwright tool by name, curl, the TOTP command as a prefix, the extra tools",
|
||||
std::ranges::all_of(Tools::McpToolNames(), [&](const std::string& t) { return Has(af, t); }) && Has(af, "Bash(curl:*)") && Has(af, totp) && Has(af, "Bash(psql:*)") && Has(af, "mcp__x__y")
|
||||
&& Pool::TotpCommand(fixture)->starts_with(std::format("{} totp ", Platform::SelfExe().string())));
|
||||
const std::vector<std::string> psql = {"Bash(psql:*)"};
|
||||
const std::vector<std::string> staticBrowser = Pool::AllowedTools("repo-bug-prover.md", "block", here, {}, std::nullopt, "browser");
|
||||
std::vector<std::string> wanted = Tools::McpToolNames();
|
||||
wanted.emplace_back("Bash(curl:*)");
|
||||
const std::vector<std::string> functionalNone = Pool::AllowedTools(Fn, "block", here, psql, std::string("u.json"), "none");
|
||||
Check("the environment carries the instrument, not the lens name: a static lens with browser gets the browser tools and curl; the functional lens with none gets neither, nor the extras or the TOTP command",
|
||||
std::ranges::all_of(wanted, [&](const std::string& t) { return Has(staticBrowser, t); })
|
||||
&& std::ranges::none_of(functionalNone, [](const std::string& x) { return x.starts_with("mcp__") || x == "Bash(curl:*)" || x == "Bash(psql:*)" || x.contains(" totp "); }));
|
||||
Check("the browser allowlist keeps the order of the functional allowlist it replaces: the browser tools, curl, the TOTP command, the extras",
|
||||
Index(af, "mcp__playwright__browser_navigate") < Index(af, "Bash(curl:*)") && Index(af, "Bash(curl:*)") < Index(af, totp) && Index(af, totp) < Index(af, "Bash(psql:*)")
|
||||
&& af == Pool::AllowedTools(Fn, "block", here, extras, fixture, "browser"));
|
||||
const std::vector<std::string>& deny = Tools::Deny();
|
||||
Check("deny list: run_code_unsafe, package managers, stacks, writes, a database client that commits",
|
||||
Has(deny, "mcp__playwright__browser_run_code_unsafe") && Has(deny, "Bash(npm:*)") && Has(deny, "Bash(supabase:*)") && Has(deny, "Write") && Has(deny, "Bash(psql:*)"));
|
||||
const std::string sq = std::format("Bash({}:*)", Pool::SqlCommand());
|
||||
const std::vector<std::string> redis = {"Bash(redis-cli:*)"};
|
||||
const std::vector<std::string> asb = Pool::AllowedTools("repo-bug-prover.md", "block", here, redis, std::string("u.json"), "supabase");
|
||||
Check("supabase allowlist: the SQL helper as its exact prefix (absolute, this binary's sql); no browser tool, no curl, no TOTP command, no extras",
|
||||
Has(asb, sq) && Pool::SqlCommand() == std::format("{} sql", Platform::SelfExe().string())
|
||||
&& std::ranges::none_of(asb, [](const std::string& x) { return x.starts_with("mcp__") || x == "Bash(curl:*)" || x == "Bash(redis-cli:*)" || x.contains(" totp "); }));
|
||||
const std::vector<std::string> abs = Pool::AllowedTools(Fn, "block", here, redis, std::string("u.json"), "browser+supabase");
|
||||
const std::string totpU = std::format("Bash({}:*)", Pool::TotpCommand("u.json").value_or(""));
|
||||
Check("browser+supabase allowlist: the browser tools, curl, the SQL helper, the TOTP command, the extras - in that order; browser alone has no SQL helper",
|
||||
Index(abs, "mcp__playwright__browser_navigate") < Index(abs, "Bash(curl:*)") && Index(abs, "Bash(curl:*)") < Index(abs, sq) && Index(abs, sq) < Index(abs, totpU) && Index(abs, totpU) < Index(abs, "Bash(redis-cli:*)")
|
||||
&& !Has(Pool::AllowedTools(Fn, "block", here, {}, std::string("u.json"), "browser"), sq) && !Has(Pool::AllowedTools(Fn, "block", here, {}, std::string("u.json"), "none"), sq));
|
||||
const std::vector<std::string> outside = {"/home/x/repo"};
|
||||
const std::vector<std::string> al2 = Pool::AllowedTools("repo-bug-prover.md", "block", outside);
|
||||
Check("allowlist (outside tree): Read(//abs/**)", Has(al2, "Read(//home/x/repo/**)") && Has(al2, "Bash(git -C /home/x/repo show:*)"));
|
||||
const std::vector<std::string> trees = {".", "sub", "/home/x/repo", "/home/x/repo"};
|
||||
Check("extra_dirs: the cwd and paths inside it dropped, outside kept once", Pool::ExtraDirs(trees) == std::vector<std::string>{"/home/x/repo"});
|
||||
|
||||
// The MCP config and the command line.
|
||||
const Json npx = {{"mcp", {{"command", "npx"}, {"args", nullptr}}}};
|
||||
const Json mc = Mcp::PlaywrightConfig(npx, "/tmp/out");
|
||||
Check("mcp config: the pinned server, headless, isolated, the output dir appended",
|
||||
mc.at("mcpServers").at("playwright").at("args") == Json::array({"-y", std::format("@playwright/mcp@{}", Mcp::PlaywrightMcpVersion), "--headless", "--isolated", "--browser", "chromium", "--output-dir", "/tmp/out"})
|
||||
&& mc.at("mcpServers").at("playwright").at("command") == "npx");
|
||||
const Json own = {{"mcp", {{"command", "playwright-mcp"}, {"args", {"--headless", "--output-dir", "/x"}}}}};
|
||||
Check("a consumer's own command and args are kept; an output dir it names is not overridden",
|
||||
Dyn::Equal(Mcp::PlaywrightConfig(own, "/tmp/out").at("mcpServers").at("playwright"), {{"command", "playwright-mcp"}, {"args", {"--headless", "--output-dir", "/x"}}}));
|
||||
const fs::path td = fs::temp_directory_path() / std::format("crafter-review-runner-{}", Platform::PublishedPid());
|
||||
fs::remove_all(td);
|
||||
fs::create_directories(td);
|
||||
const Json functional = {{"mcp", {{"command", "npx"}, {"args", nullptr}}}, {"extraTools", Json::array()}};
|
||||
auto options = [&](std::map<std::string, std::string> environments) {
|
||||
return Pool::RunnerOptions {.repo = cwd, .scratch = td, .bin = "/bin/true", .effort = "xhigh", .log = [](const std::string&) {}, .dryRun = true, .functional = functional,
|
||||
.environments = std::move(environments)};
|
||||
};
|
||||
{
|
||||
const Pool::Runner runner(options({{Fn, "browser"}}));
|
||||
const std::vector<std::string> cf = runner.Cmd(Job("f", Fn), "/dev/null", "{}", here, 20, false, "sid");
|
||||
const std::vector<std::string> cs = runner.Cmd(Job("s", "repo-bug-prover.md"), "/dev/null", "{}", here, 20, false, "sid");
|
||||
Check("cmd: the functional pass gets --mcp-config with its own output dir and the browser tools allowed; a static pass gets neither; both keep --tools Read,Glob,Grep,Bash and --strict-mcp-config",
|
||||
Has(cf, "--mcp-config") && After(cf, "--mcp-config").contains((td / "playwright" / "f").string()) && After(cf, "--allowedTools").contains("mcp__playwright__browser_snapshot")
|
||||
&& !Has(cs, "--mcp-config") && !After(cs, "--allowedTools").contains("mcp__playwright") && After(cf, "--tools") == "Read,Glob,Grep,Bash" && Has(cs, "--strict-mcp-config")
|
||||
&& After(cf, "--disallowedTools").contains("mcp__playwright__browser_run_code_unsafe"));
|
||||
Check("cmd: --safe-mode drops MCP servers, so the functional pass runs without it and the static pass keeps it", !Has(cf, "--safe-mode") && Has(cs, "--safe-mode"));
|
||||
}
|
||||
{
|
||||
const Pool::Runner rb(options({{"repo-bug-prover.md", "browser"}}));
|
||||
const std::vector<std::string> cb = rb.Cmd(Job("b", "repo-bug-prover.md"), "/dev/null", "{}", here, 20, false, "sid");
|
||||
const std::vector<std::string> cfn = rb.Cmd(Job("f", Fn), "/dev/null", "{}", here, 20, false, "sid");
|
||||
Check("cmd by environment: a static lens given the browser gets --mcp-config, the browser tools and no --safe-mode; a lens with no environment entry runs as none",
|
||||
Has(cb, "--mcp-config") && !Has(cb, "--safe-mode") && After(cb, "--allowedTools").contains("mcp__playwright__browser_snapshot") && !Has(cfn, "--mcp-config") && Has(cfn, "--safe-mode")
|
||||
&& !After(cfn, "--allowedTools").contains("mcp__playwright"));
|
||||
}
|
||||
{
|
||||
const std::string url = "postgresql://postgres:hunter2@127.0.0.1:54322/postgres";
|
||||
// An operator's shell values never reach a pass.
|
||||
setenv(std::string(PgWire::UrlEnv).c_str(), "postgresql://leaked@127.0.0.1:1/x", 1);
|
||||
setenv(std::string(PgWire::BrokerEnv).c_str(), "127.0.0.1:1#leaked", 1);
|
||||
PgWire::Broker broker;
|
||||
broker.Start();
|
||||
Pool::RunnerOptions withDb = options({{"repo-bug-prover.md", "supabase"}, {Fn, "browser+supabase"}});
|
||||
withDb.dbUrl = url;
|
||||
withDb.broker = &broker;
|
||||
Pool::Runner rd(withDb);
|
||||
unsetenv(std::string(PgWire::UrlEnv).c_str());
|
||||
unsetenv(std::string(PgWire::BrokerEnv).c_str());
|
||||
const std::vector<std::string> cd = rd.Cmd(Job("d", "repo-bug-prover.md"), "/dev/null", "{}", here, 20, false, "sid");
|
||||
const std::vector<std::string> cbs = rd.Cmd(Job("e", Fn), "/dev/null", "{}", here, 20, false, "sid");
|
||||
Check("cmd: the supabase environment keeps --safe-mode (no MCP server) and gets the SQL helper allowed; browser+supabase gets --mcp-config, the helper and no --safe-mode; the URL is on no command line",
|
||||
Has(cd, "--safe-mode") && !Has(cd, "--mcp-config") && After(cd, "--allowedTools").contains(sq) && Has(cbs, "--mcp-config") && !Has(cbs, "--safe-mode") && After(cbs, "--allowedTools").contains(sq)
|
||||
&& !(Dyn::Join(cd, " ") + Dyn::Join(cbs, " ")).contains("hunter2"));
|
||||
const Json jd = {{"lensFile", "repo-bug-prover.md"}};
|
||||
const Json js = {{"lensFile", "repo-better-solution.md"}};
|
||||
const Json jf = {{"lensFile", Fn}};
|
||||
const std::string brokerEnv(PgWire::BrokerEnv);
|
||||
const std::map<std::string, std::string> ed = rd.ProcessEnv(jd, rd.Env());
|
||||
const std::map<std::string, std::string> ef = rd.ProcessEnv(jf, rd.Env());
|
||||
const std::map<std::string, std::string> es = rd.ProcessEnv(js, rd.Env());
|
||||
auto token = [&](const std::map<std::string, std::string>& e) {
|
||||
const auto found = e.find(brokerEnv);
|
||||
const std::string value = found == e.end() ? std::string("#") : found->second;
|
||||
return broker.UrlOf(value.substr(value.rfind('#') + 1));
|
||||
};
|
||||
auto shaped = [](std::string_view address) {
|
||||
// ^127\.0\.0\.1:\d+#[0-9a-f]{32}$
|
||||
if (!address.starts_with("127.0.0.1:")) return false;
|
||||
const std::size_t hash = address.find('#');
|
||||
if (hash == std::string_view::npos || hash == 10) return false;
|
||||
const std::string_view port = address.substr(10, hash - 10);
|
||||
const std::string_view tok = address.substr(hash + 1);
|
||||
return std::ranges::all_of(port, [](char c) { return c >= '0' && c <= '9'; }) && tok.size() == 32 && std::ranges::all_of(tok, [](char c) { return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f'); });
|
||||
};
|
||||
bool leaked = false;
|
||||
for (const auto* e : {&ed, &ef}) {
|
||||
for (const auto& [k, v] : *e) leaked = leaked || v.contains("hunter2");
|
||||
}
|
||||
Check("process_env: a db pass gets the broker's address and its database's token - never the URL or its password; a static pass gets neither; the operator's shell values are dropped",
|
||||
shaped(ed.at(brokerEnv)) && token(ed) == url && token(ef) == url && !leaked && !ed.contains(std::string(PgWire::UrlEnv)) && !es.contains(brokerEnv) && !rd.Env().contains(std::string(PgWire::UrlEnv))
|
||||
&& !rd.Env().contains(brokerEnv));
|
||||
const std::string ownDb = "postgresql://postgres:postgres@127.0.0.1:20222/postgres";
|
||||
Pool::RunnerOptions noBroker = options({{"repo-bug-prover.md", "supabase"}});
|
||||
noBroker.dbUrl = url;
|
||||
Check("process_env: a pass that holds a stack gets that stack's database and no other - a stack without one gets none, never the first stack's (finding 1)",
|
||||
token(rd.ProcessEnv({{"lensFile", Fn}, {"stack", {{"id", "slot 2"}, {"dbUrl", ownDb}}}}, rd.Env())) == ownDb
|
||||
&& rd.ProcessEnv({{"lensFile", Fn}, {"stack", {{"id", "slot 2"}, {"dbUrl", nullptr}}}}, rd.Env()).at(brokerEnv).empty()
|
||||
&& Pool::Runner(noBroker).ProcessEnv(jd, {}).at(brokerEnv).empty());
|
||||
broker.Close();
|
||||
}
|
||||
|
||||
auto init = [](const Json& servers, const Json& tools, const Json& browser = Json()) {
|
||||
return Json {{"init", {{"mcp_servers", servers}, {"tools", tools}}}, {"browser", browser.is_null() ? Json {{"calls", 2}, {"errors", 0}} : browser}};
|
||||
};
|
||||
const Json conn = Json::array({{{"name", "playwright"}, {"status", "connected"}}});
|
||||
const Json none = Json::array();
|
||||
const Json read = Json::array({"Read"});
|
||||
const Json pw = Json::array({"mcp__playwright__x"});
|
||||
const Json zero = {{"calls", 0}, {"errors", 0}};
|
||||
const Pool::Verdict ok = {"ok", ""};
|
||||
Check("mcp_connected: ok only with a connected playwright server AND its tools in the init line; else an error that says so",
|
||||
Pool::McpConnected(init(conn, Json::array({"Read", "mcp__playwright__browser_snapshot"}))) == ok && Pool::McpConnected(init(none, read)).first == "error"
|
||||
&& Pool::McpConnected(init(none, read)).second.contains("did not connect") && Pool::McpConnected(init(Json::array({{{"name", "playwright"}, {"status", "failed"}}}), none)).first == "error"
|
||||
&& Pool::McpConnected({{"init", nullptr}}).first == "error");
|
||||
Check("mcp_connected: a whole pass with no browser call, or any pass whose every browser call failed, is an error; a block pass with no browser call is allowed; one answered call is enough",
|
||||
Pool::McpConnected(init(conn, pw, zero), "whole").second.contains("no browser call") && Pool::McpConnected(init(conn, pw, zero), "block") == ok
|
||||
&& Pool::McpConnected(init(conn, pw, {{"calls", 3}, {"errors", 3}}), "block").second.contains("every browser call")
|
||||
&& Pool::McpConnected(init(conn, pw, {{"calls", 3}, {"errors", 3}}), "whole").second.contains("every browser call") && Pool::McpConnected(init(conn, pw, {{"calls", 3}, {"errors", 2}}), "whole") == ok);
|
||||
Check("mcp_connected: a static lens given the browser (not empirical) may finish a whole pass without a browser call; a server that did not connect or answered every call with an error still fails it",
|
||||
Pool::McpConnected(init(conn, pw, zero), "whole", false) == ok && Pool::McpConnected(init(conn, pw, {{"calls", 2}, {"errors", 2}}), "whole", false).first == "error"
|
||||
&& Pool::McpConnected(init(none, read), "block", false).first == "error");
|
||||
Check("exercised: 'browser' is mcp_connected with the lens's empirical flag, 'none' is ok as classified",
|
||||
Pool::Exercised(init(none, read), "browser", "whole", true) == Pool::McpConnected(init(none, read), "whole", true) && Pool::Exercised(init(conn, pw, zero), "browser", "whole", true).first == "error"
|
||||
&& Pool::Exercised(init(none, read), "none", "whole", true) == ok);
|
||||
std::vector<std::string> names;
|
||||
for (const Plan::Environment& e : Plan::Environments) names.emplace_back(e.name);
|
||||
const Plan::Environment& browser = *Plan::FindEnvironment("browser");
|
||||
const Plan::Environment& noneEnv = *Plan::FindEnvironment("none");
|
||||
const Plan::Environment& supabase = *Plan::FindEnvironment("supabase");
|
||||
const Plan::Environment& both = *Plan::FindEnvironment("browser+supabase");
|
||||
auto tools = [](const Plan::Environment& e) { return std::vector<std::string>(e.tools.begin(), e.tools.end()); };
|
||||
bool otherEmpirical = false;
|
||||
for (const Plan::Lens& l : Plan::Lenses) otherEmpirical = otherEmpirical || (l.file != Fn && l.empirical);
|
||||
Check("environments: exactly none, browser, supabase, browser+supabase - a new entry first meets the seven-item list in plan.ENVIRONMENTS (the environment in the reuse keys, _resumable and _salvage; its prompt texts; pool.exercised; validate/preflight; DENY; the opencode refusal; a firing proof)",
|
||||
names == std::vector<std::string>{"none", "browser", "supabase", "browser+supabase"} && browser.mcp == "playwright" && tools(browser) == std::vector<std::string>{"Bash(curl:*)"} && browser.app && !browser.db
|
||||
&& browser.memMb == Plan::MemBrowser && noneEnv.mcp.empty() && tools(noneEnv).empty() && !noneEnv.app && !noneEnv.db && noneEnv.memMb == Plan::MemNone && supabase.mcp.empty() && tools(supabase).empty()
|
||||
&& !supabase.app && supabase.db && supabase.memMb == Plan::MemNone && both.mcp == browser.mcp && tools(both) == tools(browser) && both.app && both.db && both.memMb == browser.memMb
|
||||
&& Plan::LensOf(Fn).empirical && !otherEmpirical);
|
||||
|
||||
// The stream accounting behind it: a navigate the server answered with an
|
||||
// error, a snapshot it answered.
|
||||
const std::vector<std::string> ev = {
|
||||
R"({"type":"system","subtype":"init","mcp_servers":[{"name":"playwright","status":"connected"}],"tools":["mcp__playwright__browser_navigate"]})",
|
||||
R"({"type":"assistant","message":{"role":"assistant","content":[{"type":"tool_use","id":"t1","name":"mcp__playwright__browser_navigate","input":{}},{"type":"tool_use","id":"t2","name":"Bash","input":{}}]}})",
|
||||
R"({"type":"user","message":{"role":"user","content":[{"type":"tool_result","tool_use_id":"t1","is_error":true,"content":"Chromium distribution chrome is not found"},{"type":"tool_result","tool_use_id":"t2","content":"ok"}]}})",
|
||||
R"({"type":"assistant","message":{"role":"assistant","content":[{"type":"tool_use","id":"t3","name":"mcp__playwright__browser_snapshot","input":{}}]}})",
|
||||
R"({"type":"user","message":{"role":"user","content":[{"type":"tool_result","tool_use_id":"t3","content":"- page"}]}})",
|
||||
R"({"type":"result","subtype":"success","is_error":false,"structured_output":{"verdict":"SEND-READY","findings":[],"notes":[],"coverage":[]},"usage":{"input_tokens":1,"output_tokens":1},"num_turns":2})",
|
||||
};
|
||||
const Json ps = Stream::ParseStream(Dyn::Join(ev, "\n"));
|
||||
Check("stream: browser calls and their errors are counted by tool_use id (2 calls, 1 error), Bash errors are not browser errors",
|
||||
Dyn::Equal(ps.at("browser"), {{"calls", 2}, {"errors", 1}}) && ps.at("toolErrors") == 1 && Pool::McpConnected(ps, "whole") == ok);
|
||||
const std::vector<std::string> ev2 = {ev[0], ev[1], ev[2], ev[5]};
|
||||
const Json ps2 = Stream::ParseStream(Dyn::Join(ev2, "\n"));
|
||||
Check("...one failed navigate and nothing else -> every browser call failed -> error", Dyn::Equal(ps2.at("browser"), {{"calls", 1}, {"errors", 1}}) && Pool::McpConnected(ps2, "block").first == "error");
|
||||
// The helper as the Flatpak spells it: what the stream recognises is the
|
||||
// program's name, whatever path a build runs it from.
|
||||
const std::string sqlCommand = "/app/bin/crafter-review sql";
|
||||
auto sqlEv = [&](bool answered, const std::optional<std::string>& cmd = std::nullopt, const Json& content = Json()) {
|
||||
const Json result = content.is_null() ? Json(answered ? std::format("{} to postgres on 127.0.0.1:54322 as postgres - ONE transaction\n SELECT 1", PgWire::Answered)
|
||||
: std::format("{} - cannot connect", PgWire::NoDatabase))
|
||||
: content;
|
||||
return std::vector<std::string> {
|
||||
R"({"type":"system","subtype":"init","mcp_servers":[],"tools":["Bash"]})",
|
||||
DumpSpaced({{"type", "assistant"},
|
||||
{"message", {{"role", "assistant"}, {"content", Json::array({{{"type", "tool_use"}, {"id", "s1"}, {"name", "Bash"}, {"input", {{"command", cmd.value_or(std::format("{} <<'SQL'\nselect 1;\nSQL", sqlCommand))}}}}})}}}}),
|
||||
DumpSpaced({{"type", "user"}, {"message", {{"role", "user"}, {"content", Json::array({{{"type", "tool_result"}, {"tool_use_id", "s1"}, {"is_error", !answered}, {"content", result}}})}}}}),
|
||||
ev[5],
|
||||
};
|
||||
};
|
||||
const Json pa = Stream::ParseStream(Dyn::Join(sqlEv(true), "\n"));
|
||||
const Json pn = Stream::ParseStream(Dyn::Join(sqlEv(false), "\n"));
|
||||
Check("stream: a SQL helper call is counted by its command, and answered by the helper's connected line in its result",
|
||||
Dyn::Equal(pa.at("db"), {{"calls", 1}, {"answered", 1}}) && Dyn::Equal(pn.at("db"), {{"calls", 1}, {"answered", 0}}) && Dyn::Equal(pa.at("browser"), zero));
|
||||
Check("stream: the helper inside a compound command counts; the marker in a list-of-blocks result counts; the marker mid-line does not; another python3 script is no SQL call",
|
||||
Dyn::Equal(Stream::ParseStream(Dyn::Join(sqlEv(true, std::format("cd . && {} 'select 1'", sqlCommand)), "\n")).at("db"), {{"calls", 1}, {"answered", 1}})
|
||||
&& Stream::ParseStream(Dyn::Join(sqlEv(true, std::nullopt, Json::array({{{"type", "text"}, {"text", std::format("{} to x", PgWire::Answered)}}})), "\n")).at("db").at("answered") == 1
|
||||
&& Stream::ParseStream(Dyn::Join(sqlEv(true, std::nullopt, Json(std::format("echo says {}", PgWire::Answered))), "\n")).at("db").at("answered") == 0
|
||||
&& Dyn::Equal(Stream::ParseStream(Dyn::Join(sqlEv(true, std::string("python3 - <<EOF\nprint(1)\nEOF")), "\n")).at("db"), {{"calls", 0}, {"answered", 0}}));
|
||||
const std::vector<std::string> noSql = {sqlEv(true)[0], ev[5]};
|
||||
Check("exercised: supabase is db_answered - a pass whose every SQL call missed the database is an error, a pass with no SQL call is ok (a lens may judge by the code)",
|
||||
Pool::Exercised(pa, "supabase", "block") == ok && Pool::Exercised(pn, "supabase", "block").first == "error" && Pool::Exercised(pn, "supabase", "whole", true).second.contains("reached the database")
|
||||
&& Pool::Exercised(Stream::ParseStream(Dyn::Join(noSql, "\n")), "supabase", "whole", true) == ok && Pool::Exercised(pn, "none", "block") == ok);
|
||||
const Json bothOk = With(pa, {{"init", {{"mcp_servers", conn}, {"tools", pw}}}, {"browser", {{"calls", 2}, {"errors", 0}}}});
|
||||
Check("exercised: browser+supabase needs both - the browser (connected, used by the empirical whole pass) and, when SQL ran, the database",
|
||||
Pool::Exercised(bothOk, "browser+supabase", "whole", true) == ok && Pool::Exercised(With(pa, {{"init", {{"mcp_servers", Json::array()}}}}), "browser+supabase", "block").second.contains("did not connect")
|
||||
&& Pool::Exercised(With(bothOk, {{"db", {{"calls", 2}, {"answered", 0}}}}), "browser+supabase", "whole", true).second.contains("reached the database")
|
||||
&& Pool::Exercised(With(bothOk, {{"browser", zero}}), "browser+supabase", "whole", true).second.contains("no browser call"));
|
||||
|
||||
fs::remove_all(td);
|
||||
return Failures == 0 ? 0 : 1;
|
||||
}
|
||||
506
tests/Scheduler/main.cpp
Normal file
506
tests/Scheduler/main.cpp
Normal file
|
|
@ -0,0 +1,506 @@
|
|||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
// SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts®
|
||||
|
||||
// The pool at work: re-placement after a seat's failure, the Runner on a
|
||||
// fake claude (the seat, the MCP check, resume, the database broker), a
|
||||
// no-room wait other drivers hold, the machine-wide cap, the stack keys, the
|
||||
// admission against other drivers' memory and stacks, and the race for the
|
||||
// last stack.
|
||||
//
|
||||
// The test binary is the supervisor every pass runs under (`__supervise`);
|
||||
// another driver is a live `sleep 300`.
|
||||
|
||||
#include <signal.h>
|
||||
#include <stdlib.h>
|
||||
#include <sys/stat.h>
|
||||
#include <unistd.h>
|
||||
import std;
|
||||
import Crafter.Review;
|
||||
namespace fs = std::filesystem;
|
||||
using namespace Crafter::Review;
|
||||
using namespace std::chrono_literals;
|
||||
|
||||
namespace {
|
||||
std::int32_t Failures = 0;
|
||||
const std::string Fn(Plan::FunctionalLens);
|
||||
|
||||
void Check(std::string_view name, bool cond, std::string_view detail = {}) {
|
||||
std::println("{}{}{}", cond ? " ok " : " FAIL ", name, !cond && !detail.empty() ? std::format(" ({})", detail) : std::string());
|
||||
if (!cond) ++Failures;
|
||||
}
|
||||
|
||||
void Write(const fs::path& path, std::string_view text) {
|
||||
fs::create_directories(path.parent_path());
|
||||
std::ofstream(path, std::ios::binary) << text;
|
||||
}
|
||||
|
||||
std::string Read(const fs::path& path) {
|
||||
return Platform::ReadFile(path).value_or("");
|
||||
}
|
||||
|
||||
Json With(Json value, const Json& changes) {
|
||||
for (auto it = changes.begin(); it != changes.end(); ++it) value[it.key()] = it.value();
|
||||
return value;
|
||||
}
|
||||
|
||||
bool AnyContains(const std::vector<std::string>& lines, std::string_view part) {
|
||||
return std::ranges::any_of(lines, [&](const std::string& line) { return line.contains(part); });
|
||||
}
|
||||
|
||||
// The log a pool writes into from its threads.
|
||||
struct Logs {
|
||||
std::mutex lock;
|
||||
std::vector<std::string> lines;
|
||||
Pool::Log Sink() {
|
||||
return [this](const std::string& line) {
|
||||
const std::lock_guard guard(lock);
|
||||
lines.push_back(line);
|
||||
};
|
||||
}
|
||||
std::vector<std::string> Copy() {
|
||||
const std::lock_guard guard(lock);
|
||||
return lines;
|
||||
}
|
||||
void Clear() {
|
||||
const std::lock_guard guard(lock);
|
||||
lines.clear();
|
||||
}
|
||||
};
|
||||
|
||||
struct Sleeper {
|
||||
std::int64_t pid = Process::Start({.argv = {"sleep", "300"}}).pid;
|
||||
std::string key = std::format("{}:{}", pid, Platform::ProcStart(pid).value_or(0));
|
||||
|
||||
void Kill() const {
|
||||
Process::Signal(pid, SIGKILL);
|
||||
Process::Wait(pid);
|
||||
}
|
||||
};
|
||||
|
||||
double Seconds(std::chrono::steady_clock::time_point since) {
|
||||
return std::chrono::duration<double>(std::chrono::steady_clock::now() - since).count();
|
||||
}
|
||||
|
||||
const Pool::Scheduler::OnDone NoFollowUp = [](const Json&, const Json&, const Json&) { return std::vector<Json> {}; };
|
||||
|
||||
void TestPool() {
|
||||
std::println("pool: admission (stacks, memory), a no-room wait ends on the next completion, one pass per cold prefix goes first");
|
||||
std::mutex lock;
|
||||
std::map<std::string, std::int32_t> running = {{"F", 0}, {"S", 0}};
|
||||
std::map<std::string, std::int32_t> peak = {{"F", 0}, {"S", 0}};
|
||||
auto execJob = [&](const Json& job, std::int64_t) {
|
||||
const std::string lens = job.at("lens").get<std::string>();
|
||||
{
|
||||
const std::lock_guard guard(lock);
|
||||
peak[lens] = std::max(peak[lens], ++running[lens]);
|
||||
}
|
||||
std::this_thread::sleep_for(150ms);
|
||||
{
|
||||
const std::lock_guard guard(lock);
|
||||
--running[lens];
|
||||
}
|
||||
return std::pair<Json, Json>({{"label", job.at("label")}, {"status", "ok"}}, nullptr);
|
||||
};
|
||||
std::vector<Json> slots;
|
||||
for (const std::int32_t n : {1, 2}) slots.push_back({{"id", std::format("slot {}", n)}, {"appUrl", std::format("http://localhost:{}", n)}, {"apiUrls", Json::array()}, {"dbUrl", nullptr}, {"login", nullptr}, {"slot", true}});
|
||||
Pool::Stacks stacks(slots);
|
||||
auto plenty = []() -> std::optional<double> { return 1e6; };
|
||||
Pool::Admission adm([](const Json&) { return 100.0; }, [](const Json& j) { return j.at("lens") == "F"; }, &stacks, 10'000.0, 0, plenty);
|
||||
std::vector<Json> jobs;
|
||||
for (std::int32_t i = 0; i < 6; ++i) jobs.push_back({{"label", std::format("f{}", i)}, {"lens", "F"}});
|
||||
for (std::int32_t i = 0; i < 3; ++i) jobs.push_back({{"label", std::format("s{}", i)}, {"lens", "S"}});
|
||||
Pool::Scheduler s({.concurrency = 8, .log = [](const std::string&) {}, .admission = &adm});
|
||||
s.Run(jobs, execJob, NoFollowUp);
|
||||
std::vector<std::string> free = stacks.Free();
|
||||
std::ranges::sort(free);
|
||||
Check("admission: the passes that need a stack never outnumber the stacks (2), the rest fill the pool, everything finishes, every stack is back",
|
||||
s.DoneList().size() == 9 && peak["F"] == 2 && peak["S"] == 3 && free == std::vector<std::string>{"slot 1", "slot 2"} && adm.Inflight() == 0 && adm.Held() == 0,
|
||||
std::format("F {} S {}", peak["F"], peak["S"]));
|
||||
running = {{"F", 0}, {"S", 0}};
|
||||
peak = {{"F", 0}, {"S", 0}};
|
||||
Pool::Admission memory([](const Json&) { return 300.0; }, [](const Json&) { return false; }, nullptr, 1000.0, 0, plenty);
|
||||
jobs.clear();
|
||||
for (std::int32_t i = 0; i < 7; ++i) jobs.push_back({{"label", std::format("s{}", i)}, {"lens", "S"}});
|
||||
Pool::Scheduler m({.concurrency = 8, .log = [](const std::string&) {}, .admission = &memory});
|
||||
m.Run(jobs, execJob, NoFollowUp);
|
||||
Check("admission: memory - passes at 300 MB in a 1000 MB budget run three at once, and all of them finish", m.DoneList().size() == 7 && peak["S"] == 3, std::format("S {}", peak["S"]));
|
||||
|
||||
Logs logs;
|
||||
const double now = Platform::Now();
|
||||
std::vector<Json> seat = {{{"id", "3"}, {"kind", "oauth"}, {"secret", "x"}, {"state", "ok"}, {"util", 80.0}, {"binding", "five_hour"}, {"resetsAt", nullptr},
|
||||
{"windows", {{"five_hour", {{"utilization", 80.0}, {"resetsAt", now + 3600}}}, {"seven_day", {{"utilization", 30.0}, {"resetsAt", now + 86400}}}}},
|
||||
{"pauseUntil", 0.0}, {"running", 0}, {"procs", 0}, {"spentUsd", 0.0}, {"error", nullptr}, {"probedWith", nullptr}}};
|
||||
Json cfg = SeatPool::PoolConfig({});
|
||||
cfg["lensCostPct"] = 2;
|
||||
cfg["maxUtil"] = 85;
|
||||
Pool::Scheduler r({.concurrency = 8, .log = logs.Sink(), .seats = &seat, .seatConfig = cfg, .apiConfig = SeatPool::ApiConfig({})});
|
||||
const auto t0 = std::chrono::steady_clock::now();
|
||||
jobs.clear();
|
||||
for (std::int32_t i = 0; i < 3; ++i) jobs.push_back({{"label", std::format("r{}", i)}});
|
||||
r.Run(jobs, [](const Json& j, std::int64_t) {
|
||||
std::this_thread::sleep_for(200ms);
|
||||
return std::pair<Json, Json>({{"label", j.at("label")}, {"status", "ok"}}, nullptr);
|
||||
}, NoFollowUp);
|
||||
const std::vector<std::string> waited = logs.Copy();
|
||||
Check("a no-room wait ends on the next completion, not at the window reset",
|
||||
r.DoneList().size() == 3 && Seconds(t0) < 5 && AnyContains(waited, "no seat has room") && AnyContains(waited, "seat room is re-checked") && r.PauseUntil() == 0, Dyn::Join(waited, " | "));
|
||||
|
||||
std::map<std::string, double> started;
|
||||
std::map<std::string, double> answered;
|
||||
logs.Clear();
|
||||
Pool::Scheduler w({.concurrency = 8,
|
||||
.log = logs.Sink(),
|
||||
.cacheKey = [](Json& j) -> std::optional<std::string> {
|
||||
const Json& k = Dyn::Get(j, "key");
|
||||
return k.is_null() ? std::nullopt : std::optional<std::string>(k.get<std::string>());
|
||||
},
|
||||
.warmProbe = [&](const Json& j) {
|
||||
const std::lock_guard guard(lock);
|
||||
return answered.contains(j.at("label").get<std::string>());
|
||||
},
|
||||
.warmPoll = 0.02});
|
||||
jobs.clear();
|
||||
for (std::int32_t i = 0; i < 5; ++i) jobs.push_back({{"label", std::format("a{}", i)}, {"key", "A"}, {"respondAfter", i == 0 ? 0.3 : 0.05}});
|
||||
jobs.push_back({{"label", "n0"}, {"key", nullptr}});
|
||||
w.Run(jobs, [&](const Json& j, std::int64_t) {
|
||||
const std::string label = j.at("label").get<std::string>();
|
||||
{
|
||||
const std::lock_guard guard(lock);
|
||||
started[label] = Platform::Now();
|
||||
}
|
||||
std::this_thread::sleep_for(std::chrono::duration<double>(Dyn::Has(j, "respondAfter") ? j.at("respondAfter").get<double>() : 0.05));
|
||||
{
|
||||
const std::lock_guard guard(lock);
|
||||
answered[label] = Platform::Now();
|
||||
}
|
||||
std::this_thread::sleep_for(300ms);
|
||||
return std::pair<Json, Json>({{"label", j.at("label")}, {"status", "ok"}}, nullptr);
|
||||
}, NoFollowUp);
|
||||
const double first = answered.at("a0");
|
||||
bool followed = true;
|
||||
for (std::int32_t i = 1; i < 5; ++i) {
|
||||
const double at = started.at(std::format("a{}", i));
|
||||
followed = followed && first <= at && at < first + 0.3;
|
||||
}
|
||||
Check("warm-up: the first pass of a cold prefix starts alone, the others follow its first response, a pass without a key is never held",
|
||||
w.DoneList().size() == 6 && followed && started.at("n0") < first && AnyContains(logs.Copy(), "a0 goes first"));
|
||||
}
|
||||
|
||||
void TestSeatPool(const fs::path& td) {
|
||||
std::println("seat pool: re-placement on another seat, Runner.run on a fake claude");
|
||||
const Json acfg = SeatPool::ApiConfig({{"REVIEW_GATES_API_BUDGET_USD", "10"}, {"REVIEW_GATES_API_LENS_COST_USD", "1"}, {"REVIEW_GATES_API_CONCURRENCY", "2"}});
|
||||
const Json cfg = Seats::ConfigFromEnv({});
|
||||
Logs logs;
|
||||
std::mutex lock;
|
||||
std::vector<std::pair<std::string, std::string>> order;
|
||||
std::vector<Json> pool = {SeatPool::NewSeat("api1", "api", "k"), SeatPool::NewSeat("1", "oauth", "t1")};
|
||||
for (Json& s : pool) s["state"] = "ok";
|
||||
SeatPool::Ledger ledger(td / "l2.json", acfg);
|
||||
Pool::Scheduler sch({.concurrency = 4, .log = logs.Sink(), .seats = &pool, .seatConfig = cfg, .apiConfig = With(acfg, {{"budgetUsd", 100}}), .ledger = &ledger});
|
||||
sch.Run({{{"label", "j1"}}}, [&](const Json& job, std::int64_t) {
|
||||
{
|
||||
const std::lock_guard guard(lock);
|
||||
order.emplace_back(job.at("label").get<std::string>(), job.at("seat").get<std::string>());
|
||||
}
|
||||
if (job.at("seat") == "api1") return std::pair<Json, Json>({{"label", job.at("label")}, {"status", "credits"}, {"reason", "Your credit balance is too low"}, {"costUsd", 0.0}, {"rateLimit", Json::object()}}, nullptr);
|
||||
return std::pair<Json, Json>({{"label", job.at("label")}, {"status", "ok"}, {"costUsd", 0.5}, {"rateLimit", {{"five_hour", {{"utilization", 30}, {"resetsAt", 9'999'999'999LL}}}}}}, nullptr);
|
||||
}, NoFollowUp);
|
||||
std::map<std::string, std::string> statuses;
|
||||
for (const Pool::Done& d : sch.DoneList()) statuses[d.rec.at("label").get<std::string>()] = d.rec.at("status").get<std::string>();
|
||||
Check("a process that fails on its seat is placed again on another; the first seat is out",
|
||||
statuses == std::map<std::string, std::string>{{"j1", "ok"}} && pool[0].at("state") == "out" && !order.empty() && order.front().second == "api1" && order.back().second == "1");
|
||||
|
||||
const fs::path fake = td / "claude";
|
||||
const fs::path argsf = td / "args.txt";
|
||||
Write(fake, "#!/bin/sh\nprintf \"%s\\n\" \"$@\" > \"$FAKE_ARGS\"\nprintf \"%s\" \"$REVIEW_GATES_DB\" > \"$FAKE_ARGS.db\"\ncat > /dev/null\n"
|
||||
"if [ -n \"$FAKE_MCP\" ]; then echo '{\"type\":\"system\",\"subtype\":\"init\",\"mcp_servers\":[{\"name\":\"playwright\",\"status\":\"connected\"}],\"tools\":[\"Read\",\"mcp__playwright__browser_snapshot\"]}'; "
|
||||
"echo '{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"t1\",\"name\":\"mcp__playwright__browser_snapshot\",\"input\":{}}]}}'; "
|
||||
"echo '{\"type\":\"user\",\"message\":{\"role\":\"user\",\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"t1\",\"content\":\"- page\"}]}}'; "
|
||||
"elif [ -n \"$FAKE_SQL\" ]; then echo '{\"type\":\"system\",\"subtype\":\"init\",\"mcp_servers\":[],\"tools\":[\"Read\",\"Bash\"]}'; printf \"%s\\n\" \"$FAKE_SQL\"; "
|
||||
"else echo '{\"type\":\"system\",\"subtype\":\"init\",\"mcp_servers\":[],\"tools\":[\"Read\"]}'; fi\n"
|
||||
"echo '{\"type\":\"result\",\"subtype\":\"success\",\"is_error\":false,\"result\":\"ok\",\"structured_output\":{\"verdict\":\"SEND-READY\",\"findings\":[],\"notes\":[],\"coverage\":[]},"
|
||||
"\"usage\":{\"input_tokens\":1,\"output_tokens\":1},\"total_cost_usd\":0.01}'\n");
|
||||
fs::permissions(fake, fs::perms::owner_all | fs::perms::group_read | fs::perms::group_exec | fs::perms::others_read | fs::perms::others_exec);
|
||||
setenv("FAKE_ARGS", argsf.c_str(), 1);
|
||||
const fs::path rscratch = td / "rscratch";
|
||||
const std::string cwd = fs::current_path().string();
|
||||
Pool::Runner runner({.repo = cwd, .scratch = rscratch, .bin = fake.string(), .effort = "xhigh", .log = logs.Sink(), .dryRun = false,
|
||||
.functional = {{"mcp", {{"command", "npx"}, {"args", nullptr}}}, {"extraTools", Json::array()}}, .environments = {{Fn, "browser"}}});
|
||||
runner.SetSeats({SeatPool::NewSeat("api1", "api", "k1"), SeatPool::NewSeat("api2", "api", "k2")});
|
||||
const fs::path sysf = td / "sys.md";
|
||||
Write(sysf, "x");
|
||||
const std::vector<std::string> trees = {"."};
|
||||
auto args = [&] { return Dyn::Split(Read(argsf), "\n"); };
|
||||
auto has = [](const std::vector<std::string>& list, std::string_view item) { return std::ranges::contains(list, item); };
|
||||
auto [rec, raw] = runner.Run({{"label", "f1"}, {"model", "claude-opus-5-5"}, {"lensFile", "repo-bug-prover.md"}, {"kind", "block"}, {"seat", "api1"}, {"attempt", 1}}, "the prompt", sysf, "{}", trees, 20, 1);
|
||||
std::vector<std::string> a = args();
|
||||
Check("Runner.run on a fake claude: ok, the seat on the record, persisted, no --mcp-config for a static pass, environment none",
|
||||
rec.at("status") == "ok" && rec.at("seat") == "api1" && !has(a, "--no-session-persistence") && has(a, "--session-id") && !has(a, "--mcp-config") && rec.at("mcp") == false && rec.at("environment") == "none",
|
||||
DumpSpaced(rec));
|
||||
std::tie(rec, raw) = runner.Run({{"label", "f2"}, {"model", "claude-opus-5-5"}, {"lensFile", Fn}, {"kind", "block"}, {"seat", "api2"}, {"attempt", 1}}, "the prompt", sysf, "{}", trees, 20, 1);
|
||||
a = args();
|
||||
Check("...a functional pass whose init line shows no connected server is an error (no browser = no functional evidence), with --mcp-config passed, its playwright output dir made and environment browser",
|
||||
rec.at("mcp") == true && rec.at("environment") == "browser" && rec.at("status") == "error" && rec.at("reason").get<std::string>().contains("did not connect") && has(a, "--mcp-config")
|
||||
&& fs::is_directory(rscratch / "playwright" / "f2"));
|
||||
// The runner captured its environment at construction; the fake reads
|
||||
// this one.
|
||||
runner.Env()["FAKE_MCP"] = "1";
|
||||
std::tie(rec, raw) = runner.Run({{"label", "f2"}, {"model", "claude-opus-5-5"}, {"lensFile", Fn}, {"kind", "block"}, {"seat", "api2"}, {"attempt", 1}}, "the prompt", sysf, "{}", trees, 20, 1);
|
||||
runner.Env().erase("FAKE_MCP");
|
||||
Check("...and ok when the server connected", rec.at("status") == "ok" && !raw.is_null());
|
||||
const std::string sid1 = rec.at("sessionId").get<std::string>();
|
||||
Write(rscratch / "seat-api2" / "projects" / "-home-repo" / (std::format("{}.jsonl", sid1)), "{}\n");
|
||||
std::tie(rec, raw) = runner.Run({{"label", "f2"}, {"model", "claude-opus-5-5"}, {"lensFile", Fn}, {"kind", "block"}, {"seat", "api1"}, {"attempt", 2}, {"resumeSession", sid1}, {"resumeSeat", "api2"}},
|
||||
"the prompt", sysf, "{}", trees, 20, 1);
|
||||
a = args();
|
||||
const auto resumeAt = std::ranges::find(a, "--resume");
|
||||
Check("resumed pass: --resume <id>, the transcript carried to the new seat, the resume prompt",
|
||||
rec.at("sessionId") == sid1 && resumeAt != a.end() && resumeAt + 1 != a.end() && *(resumeAt + 1) == sid1 && fs::exists(rscratch / "seat-api1" / "projects" / "-home-repo" / (std::format("{}.jsonl", sid1)))
|
||||
&& Read(rscratch / "prompts" / "f2.txt") == SeatPool::ResumePrompt);
|
||||
|
||||
// A pass with the database under test: the broker's address and a token
|
||||
// in its process alone, never the URL; and no evidence when no SQL call
|
||||
// reached the database.
|
||||
const std::string durl = "postgresql://postgres:hunter2@127.0.0.1:54322/postgres";
|
||||
PgWire::Broker dbr;
|
||||
dbr.Start();
|
||||
Pool::Runner rdb({.repo = cwd, .scratch = td / "dscratch", .bin = fake.string(), .effort = "xhigh", .log = logs.Sink(), .dryRun = false,
|
||||
.environments = {{"repo-bug-prover.md", "supabase"}}, .dbUrl = durl, .broker = &dbr});
|
||||
rdb.SetSeats({SeatPool::NewSeat("api1", "api", "k1")});
|
||||
auto sqlLines = [](const std::string& content) {
|
||||
return std::format("{}\n{}", DumpSpaced({{"type", "assistant"},
|
||||
{"message", {{"role", "assistant"}, {"content", Json::array({{{"type", "tool_use"}, {"id", "q1"}, {"name", "Bash"}, {"input", {{"command", "/app/bin/crafter-review sql <<'SQL'\nselect 1;\nSQL"}}}}})}}}}),
|
||||
DumpSpaced({{"type", "user"}, {"message", {{"role", "user"}, {"content", Json::array({{{"type", "tool_result"}, {"tool_use_id", "q1"}, {"content", content}}})}}}}));
|
||||
};
|
||||
rdb.Env()["FAKE_SQL"] = sqlLines(std::format("{} - cannot connect to 127.0.0.1:54322", PgWire::NoDatabase));
|
||||
std::tie(rec, raw) = rdb.Run({{"label", "d1"}, {"model", "claude-opus-5-5"}, {"lensFile", "repo-bug-prover.md"}, {"kind", "block"}, {"seat", "api1"}, {"attempt", 1}}, "the prompt", sysf, "{}", trees, 20, 1);
|
||||
const std::string argsText = Read(argsf);
|
||||
const std::string gotEnv = Read(std::format("{}.db", argsf.string()));
|
||||
Check("Runner.run, supabase: the db pass's process gets the broker and its token (no URL, no password, on no command line); a pass whose every SQL call missed the database is an error with no result",
|
||||
gotEnv == dbr.Address(durl) && dbr.UrlOf(gotEnv.substr(gotEnv.rfind('#') + 1)) == durl && !(gotEnv + argsText).contains("hunter2") && has(Dyn::Split(argsText, "\n"), "--safe-mode") && rec.at("status") == "error"
|
||||
&& rec.at("reason").get<std::string>().contains("reached the database") && raw.is_null() && Dyn::Equal(rec.at("db"), {{"calls", 1}, {"answered", 0}}) && rec.at("environment") == "supabase",
|
||||
DumpSpaced(rec));
|
||||
rdb.Env()["FAKE_SQL"] = sqlLines(std::format("{} to postgres on 127.0.0.1:54322 as postgres - ONE transaction\n SELECT 1", PgWire::Answered));
|
||||
std::tie(rec, raw) = rdb.Run({{"label", "d1"}, {"model", "claude-opus-5-5"}, {"lensFile", "repo-bug-prover.md"}, {"kind", "block"}, {"seat", "api1"}, {"attempt", 1}}, "the prompt", sysf, "{}", trees, 20, 1);
|
||||
rdb.Env().erase("FAKE_SQL");
|
||||
Check("...and ok once the database answered", rec.at("status") == "ok" && !raw.is_null() && Dyn::Equal(rec.at("db"), {{"calls", 1}, {"answered", 1}}));
|
||||
std::tie(rec, raw) = rdb.Run({{"label", "d2"}, {"model", "claude-opus-5-5"}, {"lensFile", "repo-better-solution.md"}, {"kind", "block"}, {"seat", "api1"}, {"attempt", 1}}, "the prompt", sysf, "{}", trees, 20, 1);
|
||||
Check("...while a lens without the database gets no broker at all", Read(std::format("{}.db", argsf.string())).empty() && rec.at("status") == "ok" && rec.at("environment") == "none");
|
||||
dbr.Close();
|
||||
unsetenv("FAKE_ARGS");
|
||||
}
|
||||
|
||||
void TestDrivers(const fs::path& td) {
|
||||
std::println("several drivers at once: the pool against the board - a wait other drivers hold, the machine's cap, stacks and memory across drivers");
|
||||
const fs::path path = td / "seat-board.json";
|
||||
Logs logs;
|
||||
const double now = Platform::Now();
|
||||
const Json cfg = SeatPool::PoolConfig({});
|
||||
const Json acfg = SeatPool::ApiConfig({{"REVIEW_GATES_API_CONCURRENCY", "8"}, {"REVIEW_GATES_API_LENS_COST_USD", "1"}});
|
||||
auto mk = [&](const std::string& id, const std::string& secret, std::int64_t fh, std::int64_t sd) {
|
||||
Json s = SeatPool::NewSeat(id, "oauth", secret);
|
||||
s["state"] = "ok";
|
||||
s["windows"] = {{"five_hour", {{"utilization", fh}, {"resetsAt", now + 3600}}}, {"seven_day", {{"utilization", sd}, {"resetsAt", now + 86400}}}};
|
||||
const auto [util, binding] = Seats::SeatUtil(s.at("windows"));
|
||||
s["util"] = util;
|
||||
s["binding"] = binding;
|
||||
return s;
|
||||
};
|
||||
std::deque<Json> jobs;
|
||||
std::mutex jobsLock;
|
||||
auto chooserFor = [&](const std::string& id, const std::string& label, const Json& extra = Json::object()) {
|
||||
return SeatPool::Board::Chooser([&jobs, &jobsLock, id, label, extra](const std::vector<Json*>& ranked, const SeatPool::Others&) -> std::optional<std::pair<Json*, Json*>> {
|
||||
for (Json* s : ranked) {
|
||||
if (s->at("id") == id) {
|
||||
const std::lock_guard guard(jobsLock);
|
||||
jobs.push_back(With({{"label", label}, {"timeoutMin", 20}}, extra));
|
||||
return std::pair(s, &jobs.back());
|
||||
}
|
||||
}
|
||||
return std::nullopt;
|
||||
});
|
||||
};
|
||||
auto place = [&](SeatPool::Board& board, Json& seat, const Json& c, const SeatPool::Board::Chooser& chooser) {
|
||||
std::vector<Json> one = {seat};
|
||||
board.Place(one, now, c, acfg, chooser);
|
||||
seat = one.front();
|
||||
};
|
||||
SeatPool::Board A(path, {}, std::nullopt, logs.Sink());
|
||||
A.Join({{"run", "A"}}, now);
|
||||
const Sleeper child2;
|
||||
SeatPool::Board D(path, child2.key, std::nullopt, logs.Sink());
|
||||
D.Join({{"run", "D"}}, now);
|
||||
|
||||
const Json cfg3 = With(cfg, {{"lensCostPct", 2}, {"maxUtil", 85}});
|
||||
std::vector<Json> seat3 = {mk("3", "t3", 80, 30)};
|
||||
Json sd3 = mk("3", "t3", 80, 30);
|
||||
place(D, sd3, cfg3, chooserFor("3", "h0"));
|
||||
// 80 + 4 held: 80 + 4 + 2 > 85, no room for A until D's passes finish.
|
||||
place(D, sd3, cfg3, chooserFor("3", "h1"));
|
||||
auto execJob = [&](const Json& job, std::int64_t) {
|
||||
std::this_thread::sleep_for(100ms);
|
||||
return std::pair<Json, Json>({{"label", job.at("label")}, {"status", "ok"}, {"rateLimit", {{"five_hour", {{"utilization", 81}, {"resetsAt", now + 3600}}}}}}, nullptr);
|
||||
};
|
||||
{
|
||||
std::jthread freer([&] {
|
||||
std::this_thread::sleep_for(500ms);
|
||||
D.Finish(sd3, "h0", Platform::Now());
|
||||
D.Finish(sd3, "h1", Platform::Now());
|
||||
});
|
||||
logs.Clear();
|
||||
Pool::Scheduler sch({.concurrency = 8, .log = logs.Sink(), .seats = &seat3, .seatConfig = cfg3, .apiConfig = acfg, .board = &A, .noRoomWait = 0.2});
|
||||
const auto t0 = std::chrono::steady_clock::now();
|
||||
sch.Run({{{"label", "a0"}, {"timeoutMin", 20}}}, execJob, NoFollowUp);
|
||||
const double dt = Seconds(t0);
|
||||
const Json bd = Dyn::LoadJsonFile(path);
|
||||
const Json& inflight = bd.at("seats").at(seat3[0].at("boardKey").get<std::string>()).at("inflight");
|
||||
bool mine = false;
|
||||
for (auto it = inflight.begin(); it != inflight.end(); ++it) mine = mine || it.key().starts_with(A.Me());
|
||||
const std::vector<std::string> lines = logs.Copy();
|
||||
Check("a no-room wait held by another driver's passes ends when they finish, not at the window reset",
|
||||
sch.DoneList().size() == 1 && dt < 5 && AnyContains(lines, "hold the room") && !mine, std::format("{:.1f}s {}", dt, Dyn::Join(lines, " | ")));
|
||||
Check("...and the finished pass's reading is on the board", bd.at("seats").at(seat3[0].at("boardKey").get<std::string>()).at("windows").at("five_hour").at("utilization") == 81);
|
||||
}
|
||||
place(D, sd3, cfg3, chooserFor("3", "w0"));
|
||||
// Two of D's on the machine: a width of 2 holds A's pass until one
|
||||
// finishes.
|
||||
place(D, sd3, cfg3, chooserFor("3", "w1"));
|
||||
{
|
||||
std::jthread freer([&] {
|
||||
std::this_thread::sleep_for(500ms);
|
||||
D.Finish(sd3, "w0", Platform::Now());
|
||||
});
|
||||
logs.Clear();
|
||||
std::vector<Json> seat3b = {mk("3", "t3", 10, 30)};
|
||||
Pool::Scheduler sch({.concurrency = 2, .log = logs.Sink(), .seats = &seat3b, .seatConfig = cfg3, .apiConfig = acfg, .board = &A, .noRoomWait = 0.2});
|
||||
const auto t0 = std::chrono::steady_clock::now();
|
||||
sch.Run({{{"label", "a1"}, {"timeoutMin", 20}}}, execJob, NoFollowUp);
|
||||
const double dt = Seconds(t0);
|
||||
const std::vector<std::string> lines = logs.Copy();
|
||||
Check("the machine-wide cap holds a launch while other drivers fill it, and releases it when one of theirs finishes",
|
||||
sch.DoneList().size() == 1 && 0.4 < dt && dt < 5 && AnyContains(lines, "the cap is 2"), std::format("{:.1f}s {}", dt, Dyn::Join(lines, " | ")));
|
||||
}
|
||||
D.Finish(sd3, "w1", Platform::Now());
|
||||
|
||||
// Memory and stacks across drivers (2026-10-31).
|
||||
Check("stack keys: one application whatever the spelling (localhost/127.0.0.1, case, trailing slash, the default port); another port another stack; no URL, its id",
|
||||
Pool::StackKey({{"id", "a"}, {"appUrl", "http://localhost:5173/"}}) == Pool::StackKey({{"id", "b"}, {"appUrl", "HTTP://127.0.0.1:5173"}})
|
||||
&& Pool::StackKey({{"id", "b"}, {"appUrl", "HTTP://127.0.0.1:5173"}}) == "http://127.0.0.1:5173" && Pool::StackKey({{"id", "c"}, {"appUrl", "http://localhost"}}) == "http://127.0.0.1:80"
|
||||
&& Pool::StackKey({{"id", "d"}, {"appUrl", "http://localhost:5273"}}) != Pool::StackKey({{"id", "a"}, {"appUrl", "http://localhost:5173"}}) && Pool::StackKey({{"id", "s1"}}) == "s1");
|
||||
// What the board's fold left in A.others when another driver had two
|
||||
// passes in flight (test_drivers checks the fold itself).
|
||||
SeatPool::Others held;
|
||||
held.count = 2;
|
||||
held.mb = 1050.0;
|
||||
held.stacks = {"http://127.0.0.1:5173"};
|
||||
auto plenty = []() -> std::optional<double> { return 1e6; };
|
||||
Pool::Stacks stx({{{"id", "slot 1"}, {"appUrl", "http://localhost:5173/"}}, {{"id", "slot 2"}, {"appUrl", "http://localhost:5273"}}});
|
||||
Pool::Admission adm([](const Json& j) { return j.at("mb").get<double>(); }, [](const Json& j) { return Dyn::Truthy(Dyn::Get(j, "app")); }, &stx, 2000.0, 0, plenty, logs.Sink());
|
||||
adm.SetOthers(held);
|
||||
Json j1 = {{"label", "x"}, {"mb", 840}, {"app", true}};
|
||||
const bool ok = adm.Can(j1);
|
||||
adm.Take(j1);
|
||||
Check("Admission: a stack another driver holds is not free here - the pass takes the other; their memory counts against the budget beside this driver's (1050 + 840 + 200 > 2000, + 100 fits)",
|
||||
ok && j1.at("stack").at("id") == "slot 2" && j1.at("stackKey") == "http://127.0.0.1:5273" && !adm.Can({{"label", "y"}, {"mb", 840}, {"app", true}}) && !adm.Can({{"label", "z"}, {"mb", 200}})
|
||||
&& adm.Can({{"label", "z"}, {"mb", 100}}),
|
||||
DumpSpaced(Dyn::Get(j1, "stack")));
|
||||
adm.Give(j1);
|
||||
logs.Clear();
|
||||
const Json big = {{"label", "b"}, {"mb", 5000}};
|
||||
const bool waits = !adm.Can(big);
|
||||
adm.SetOthers(SeatPool::Others());
|
||||
Check("Admission: nothing of this driver's in flight but other drivers' passes on the machine - a pass over the budget waits for them instead of running alone; with the machine idle it runs alone",
|
||||
waits && adm.Can(big) && !j1.contains("stackKey") && logs.Copy().empty());
|
||||
|
||||
// The pool: a stack another driver holds - the pass waits (re-checked,
|
||||
// not spun on) and runs when theirs finishes.
|
||||
Json sE = mk("1", "t1", 10, 50);
|
||||
place(D, sE, cfg, chooserFor("1", "hold", {{"_admitted", 840.0}, {"stackKey", "http://127.0.0.1:5173"}}));
|
||||
Pool::Stacks stw({{{"id", "slot 1"}, {"appUrl", "http://localhost:5173"}}});
|
||||
Pool::Admission admw([](const Json&) { return 100.0; }, [](const Json&) { return true; }, &stw, 10'000.0, 0, plenty, logs.Sink());
|
||||
double seenAt = 0;
|
||||
Json seenEntry;
|
||||
const std::string keyE = sE.at("boardKey").get<std::string>();
|
||||
auto execBoard = [&](const Json& job, std::int64_t) {
|
||||
seenAt = Platform::Now();
|
||||
seenEntry = Dyn::Get(Dyn::LoadJsonFile(path).at("seats").at(keyE).at("inflight"), std::format("{}/{}", A.Me(), job.at("label").get<std::string>()));
|
||||
std::this_thread::sleep_for(100ms);
|
||||
return std::pair<Json, Json>({{"label", job.at("label")}, {"status", "ok"}}, nullptr);
|
||||
};
|
||||
{
|
||||
std::jthread freer([&] {
|
||||
std::this_thread::sleep_for(600ms);
|
||||
D.Finish(sE, "hold", Platform::Now());
|
||||
});
|
||||
logs.Clear();
|
||||
std::vector<Json> seats = {mk("1", "t1", 10, 50)};
|
||||
Pool::Scheduler sch({.concurrency = 8, .log = logs.Sink(), .seats = &seats, .seatConfig = cfg, .apiConfig = acfg, .admission = &admw, .board = &A, .noRoomWait = 0.2});
|
||||
const double t0 = Platform::Now();
|
||||
sch.Run({{{"label", "w0"}, {"timeoutMin", 20}}}, execBoard, NoFollowUp);
|
||||
const Json bd = Dyn::LoadJsonFile(path);
|
||||
bool mine = false;
|
||||
for (const Json& s : bd.at("seats")) {
|
||||
for (auto it = Dyn::Or(Dyn::Get(s, "inflight"), Dyn::EmptyDict()).begin(); it != Dyn::Or(Dyn::Get(s, "inflight"), Dyn::EmptyDict()).end(); ++it) mine = mine || it.key().starts_with(A.Me());
|
||||
}
|
||||
const std::vector<std::string> lines = logs.Copy();
|
||||
const auto once = std::ranges::count_if(lines, [](const std::string& l) { return l.contains("nothing can launch"); });
|
||||
Check("the pool: a pass whose only stack another driver holds waits, re-checked every no_room_wait and logged once, and launches when theirs finishes - on the board with its memory and its stack",
|
||||
sch.DoneList().size() == 1 && 0.5 < seenAt - t0 && seenAt - t0 < 5 && once == 1 && AnyContains(lines, "stack(s) http://127.0.0.1:5173") && Dyn::Equal(Dyn::Get(seenEntry, "mb"), 100.0)
|
||||
&& Dyn::Get(seenEntry, "stack") == "http://127.0.0.1:5173" && stw.Can() && !mine,
|
||||
std::format("{:.2f}s {} {}", seenAt - t0, DumpSpaced(seenEntry), Dyn::Join(lines, " | ")));
|
||||
}
|
||||
|
||||
// Two drivers racing for the last stack: one gets it.
|
||||
const Sleeper child3;
|
||||
SeatPool::Board F(path, child3.key, std::nullopt, logs.Sink());
|
||||
F.Join({{"run", "F"}}, now);
|
||||
std::mutex picksLock;
|
||||
std::vector<bool> picks;
|
||||
auto racer = [&](SeatPool::Board& board, const std::string& label) {
|
||||
Pool::Stacks st({{{"id", "slot 1"}, {"appUrl", "http://localhost:5999"}}});
|
||||
Pool::Admission ad([](const Json&) { return 100.0; }, [](const Json&) { return true; }, &st, 10'000.0, 0, plenty);
|
||||
Json job = {{"label", label}, {"timeoutMin", 20}};
|
||||
std::vector<Json> one = {mk("1", "t1", 10, 50)};
|
||||
const SeatPool::Board::Placed got = board.Place(one, now, cfg, acfg, [&](const std::vector<Json*>& cands, const SeatPool::Others& others) -> std::optional<std::pair<Json*, Json*>> {
|
||||
ad.SetOthers(others);
|
||||
std::this_thread::sleep_for(50ms);
|
||||
if (!ad.Can(job)) return std::nullopt;
|
||||
ad.Take(job);
|
||||
return std::pair(cands.front(), &job);
|
||||
});
|
||||
const std::lock_guard guard(picksLock);
|
||||
picks.push_back(got.picked.has_value());
|
||||
};
|
||||
{
|
||||
const std::jthread ta(racer, std::ref(A), std::string("ra"));
|
||||
const std::jthread tf(racer, std::ref(F), std::string("rf"));
|
||||
}
|
||||
Check("two drivers racing for the last stack on the machine: exactly one gets it", std::ranges::count(picks, true) == 1);
|
||||
F.Leave();
|
||||
child3.Kill();
|
||||
A.Leave();
|
||||
child2.Kill();
|
||||
}
|
||||
}
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
const std::vector<std::string_view> args(argv, argv + argc);
|
||||
if (args.size() > 1 && args[1] == Process::SuperviseVerb) return Process::SuperviseMain(std::span(args).subspan(2));
|
||||
Platform::SetMode(Platform::Mode::Native);
|
||||
const fs::path td = fs::temp_directory_path() / std::format("crafter-review-scheduler-{}", Platform::PublishedPid());
|
||||
fs::remove_all(td);
|
||||
fs::create_directories(td);
|
||||
// No check reaches the machine's own board: the drivers running on it are
|
||||
// real.
|
||||
setenv("REVIEW_GATES_BOARD", (td / "seat-board.json").c_str(), 1);
|
||||
try {
|
||||
TestPool();
|
||||
TestSeatPool(td);
|
||||
TestDrivers(td);
|
||||
} catch (const std::exception& error) {
|
||||
std::println(std::cerr, "scheduler: {}", error.what());
|
||||
return 1;
|
||||
}
|
||||
fs::remove_all(td);
|
||||
return Failures == 0 ? 0 : 1;
|
||||
}
|
||||
Loading…
Reference in a new issue