277 lines
25 KiB
C++
277 lines
25 KiB
C++
|
// SPDX-License-Identifier: AGPL-3.0-only
|
||
|
|
// SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts®
|
||
|
|
|
||
|
|
// The pool's command lines: the allowlists and the deny list, the MCP
|
||
|
|
// config, the command line by environment, the process environment of a
|
||
|
|
// pass with the database, mcp_connected and exercised, and the stream
|
||
|
|
// accounting behind them. The helpers are this binary's `sql` and `totp`.
|
||
|
|
|
||
|
|
#include <stdlib.h>
|
||
|
|
import std;
|
||
|
|
import Crafter.Review;
|
||
|
|
namespace fs = std::filesystem;
|
||
|
|
using namespace Crafter::Review;
|
||
|
|
|
||
|
|
namespace {
|
||
|
|
std::int32_t Failures = 0;
|
||
|
|
const std::string Fn(Plan::FunctionalLens);
|
||
|
|
|
||
|
|
void Check(std::string_view name, bool cond, std::string_view detail = {}) {
|
||
|
|
std::println("{}{}{}", cond ? " ok " : " FAIL ", name, !cond && !detail.empty() ? std::format(" ({})", detail) : std::string());
|
||
|
|
if (!cond) ++Failures;
|
||
|
|
}
|
||
|
|
|
||
|
|
bool Has(const std::vector<std::string>& list, std::string_view item) {
|
||
|
|
return std::ranges::contains(list, item);
|
||
|
|
}
|
||
|
|
|
||
|
|
std::ptrdiff_t Index(const std::vector<std::string>& list, std::string_view item) {
|
||
|
|
const auto at = std::ranges::find(list, item);
|
||
|
|
return at == list.end() ? -1 : at - list.begin();
|
||
|
|
}
|
||
|
|
|
||
|
|
// c[c.index(flag) + 1]
|
||
|
|
std::string After(const std::vector<std::string>& c, std::string_view flag) {
|
||
|
|
const std::ptrdiff_t at = Index(c, flag);
|
||
|
|
return at < 0 || at + 1 >= static_cast<std::ptrdiff_t>(c.size()) ? std::string() : c[static_cast<std::size_t>(at + 1)];
|
||
|
|
}
|
||
|
|
|
||
|
|
Json Job(std::string_view label, std::string_view lensFile) {
|
||
|
|
return {{"label", label}, {"model", "m"}, {"lensFile", lensFile}, {"kind", "block"}};
|
||
|
|
}
|
||
|
|
|
||
|
|
Json With(Json value, const Json& changes) {
|
||
|
|
for (auto it = changes.begin(); it != changes.end(); ++it) value[it.key()] = it.value();
|
||
|
|
return value;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
int main() {
|
||
|
|
std::println("commits mode (the pool's part): allowlists, the deny list, the MCP config, the command line, the process environment, mcp_connected, exercised");
|
||
|
|
Platform::SetMode(Platform::Mode::Native);
|
||
|
|
const std::string cwd = fs::current_path().string();
|
||
|
|
const std::vector<std::string> here = {"."};
|
||
|
|
const std::vector<std::string> al = Pool::AllowedTools("repo-bug-prover.md", "block", here);
|
||
|
|
Check("static lens allowlist: git -C . and absolute, python3 -, no browser tool, no curl, no Read(//) rule for the cwd",
|
||
|
|
Has(al, "Bash(git -C . show:*)") && Has(al, std::format("Bash(git -C {} show:*)", cwd)) && Has(al, "Bash(python3 -:*)") && std::ranges::none_of(al, [](const std::string& x) { return x.starts_with("mcp__"); })
|
||
|
|
&& !Has(al, "Bash(curl:*)") && std::ranges::none_of(al, [](const std::string& x) { return x.starts_with("Read("); }));
|
||
|
|
const std::vector<std::string> extras = {"Bash(psql:*)", "mcp__x__y"};
|
||
|
|
const std::string fixture = "cypress/fixtures/created-user.json";
|
||
|
|
const std::vector<std::string> af = Pool::AllowedTools(Fn, "block", here, extras, fixture, "browser");
|
||
|
|
const std::string totp = std::format("Bash({}:*)", Pool::TotpCommand(fixture).value_or(""));
|
||
|
|
Check("functional allowlist: every Playwright tool by name, curl, the TOTP command as a prefix, the extra tools",
|
||
|
|
std::ranges::all_of(Tools::McpToolNames(), [&](const std::string& t) { return Has(af, t); }) && Has(af, "Bash(curl:*)") && Has(af, totp) && Has(af, "Bash(psql:*)") && Has(af, "mcp__x__y")
|
||
|
|
&& Pool::TotpCommand(fixture)->starts_with(std::format("{} totp ", Platform::SelfExe().string())));
|
||
|
|
const std::vector<std::string> psql = {"Bash(psql:*)"};
|
||
|
|
const std::vector<std::string> staticBrowser = Pool::AllowedTools("repo-bug-prover.md", "block", here, {}, std::nullopt, "browser");
|
||
|
|
std::vector<std::string> wanted = Tools::McpToolNames();
|
||
|
|
wanted.emplace_back("Bash(curl:*)");
|
||
|
|
const std::vector<std::string> functionalNone = Pool::AllowedTools(Fn, "block", here, psql, std::string("u.json"), "none");
|
||
|
|
Check("the environment carries the instrument, not the lens name: a static lens with browser gets the browser tools and curl; the functional lens with none gets neither, nor the extras or the TOTP command",
|
||
|
|
std::ranges::all_of(wanted, [&](const std::string& t) { return Has(staticBrowser, t); })
|
||
|
|
&& std::ranges::none_of(functionalNone, [](const std::string& x) { return x.starts_with("mcp__") || x == "Bash(curl:*)" || x == "Bash(psql:*)" || x.contains(" totp "); }));
|
||
|
|
Check("the browser allowlist keeps the order of the functional allowlist it replaces: the browser tools, curl, the TOTP command, the extras",
|
||
|
|
Index(af, "mcp__playwright__browser_navigate") < Index(af, "Bash(curl:*)") && Index(af, "Bash(curl:*)") < Index(af, totp) && Index(af, totp) < Index(af, "Bash(psql:*)")
|
||
|
|
&& af == Pool::AllowedTools(Fn, "block", here, extras, fixture, "browser"));
|
||
|
|
const std::vector<std::string>& deny = Tools::Deny();
|
||
|
|
Check("deny list: run_code_unsafe, package managers, stacks, writes, a database client that commits",
|
||
|
|
Has(deny, "mcp__playwright__browser_run_code_unsafe") && Has(deny, "Bash(npm:*)") && Has(deny, "Bash(supabase:*)") && Has(deny, "Write") && Has(deny, "Bash(psql:*)"));
|
||
|
|
const std::string sq = std::format("Bash({}:*)", Pool::SqlCommand());
|
||
|
|
const std::vector<std::string> redis = {"Bash(redis-cli:*)"};
|
||
|
|
const std::vector<std::string> asb = Pool::AllowedTools("repo-bug-prover.md", "block", here, redis, std::string("u.json"), "supabase");
|
||
|
|
Check("supabase allowlist: the SQL helper as its exact prefix (absolute, this binary's sql); no browser tool, no curl, no TOTP command, no extras",
|
||
|
|
Has(asb, sq) && Pool::SqlCommand() == std::format("{} sql", Platform::SelfExe().string())
|
||
|
|
&& std::ranges::none_of(asb, [](const std::string& x) { return x.starts_with("mcp__") || x == "Bash(curl:*)" || x == "Bash(redis-cli:*)" || x.contains(" totp "); }));
|
||
|
|
const std::vector<std::string> abs = Pool::AllowedTools(Fn, "block", here, redis, std::string("u.json"), "browser+supabase");
|
||
|
|
const std::string totpU = std::format("Bash({}:*)", Pool::TotpCommand("u.json").value_or(""));
|
||
|
|
Check("browser+supabase allowlist: the browser tools, curl, the SQL helper, the TOTP command, the extras - in that order; browser alone has no SQL helper",
|
||
|
|
Index(abs, "mcp__playwright__browser_navigate") < Index(abs, "Bash(curl:*)") && Index(abs, "Bash(curl:*)") < Index(abs, sq) && Index(abs, sq) < Index(abs, totpU) && Index(abs, totpU) < Index(abs, "Bash(redis-cli:*)")
|
||
|
|
&& !Has(Pool::AllowedTools(Fn, "block", here, {}, std::string("u.json"), "browser"), sq) && !Has(Pool::AllowedTools(Fn, "block", here, {}, std::string("u.json"), "none"), sq));
|
||
|
|
const std::vector<std::string> outside = {"/home/x/repo"};
|
||
|
|
const std::vector<std::string> al2 = Pool::AllowedTools("repo-bug-prover.md", "block", outside);
|
||
|
|
Check("allowlist (outside tree): Read(//abs/**)", Has(al2, "Read(//home/x/repo/**)") && Has(al2, "Bash(git -C /home/x/repo show:*)"));
|
||
|
|
const std::vector<std::string> trees = {".", "sub", "/home/x/repo", "/home/x/repo"};
|
||
|
|
Check("extra_dirs: the cwd and paths inside it dropped, outside kept once", Pool::ExtraDirs(trees) == std::vector<std::string>{"/home/x/repo"});
|
||
|
|
|
||
|
|
// The MCP config and the command line.
|
||
|
|
const Json npx = {{"mcp", {{"command", "npx"}, {"args", nullptr}}}};
|
||
|
|
const Json mc = Mcp::PlaywrightConfig(npx, "/tmp/out");
|
||
|
|
Check("mcp config: the pinned server, headless, isolated, the output dir appended",
|
||
|
|
mc.at("mcpServers").at("playwright").at("args") == Json::array({"-y", std::format("@playwright/mcp@{}", Mcp::PlaywrightMcpVersion), "--headless", "--isolated", "--browser", "chromium", "--output-dir", "/tmp/out"})
|
||
|
|
&& mc.at("mcpServers").at("playwright").at("command") == "npx");
|
||
|
|
const Json own = {{"mcp", {{"command", "playwright-mcp"}, {"args", {"--headless", "--output-dir", "/x"}}}}};
|
||
|
|
Check("a consumer's own command and args are kept; an output dir it names is not overridden",
|
||
|
|
Dyn::Equal(Mcp::PlaywrightConfig(own, "/tmp/out").at("mcpServers").at("playwright"), {{"command", "playwright-mcp"}, {"args", {"--headless", "--output-dir", "/x"}}}));
|
||
|
|
const fs::path td = fs::temp_directory_path() / std::format("crafter-review-runner-{}", Platform::PublishedPid());
|
||
|
|
fs::remove_all(td);
|
||
|
|
fs::create_directories(td);
|
||
|
|
const Json functional = {{"mcp", {{"command", "npx"}, {"args", nullptr}}}, {"extraTools", Json::array()}};
|
||
|
|
auto options = [&](std::map<std::string, std::string> environments) {
|
||
|
|
return Pool::RunnerOptions {.repo = cwd, .scratch = td, .bin = "/bin/true", .effort = "xhigh", .log = [](const std::string&) {}, .dryRun = true, .functional = functional,
|
||
|
|
.environments = std::move(environments)};
|
||
|
|
};
|
||
|
|
{
|
||
|
|
const Pool::Runner runner(options({{Fn, "browser"}}));
|
||
|
|
const std::vector<std::string> cf = runner.Cmd(Job("f", Fn), "/dev/null", "{}", here, 20, false, "sid");
|
||
|
|
const std::vector<std::string> cs = runner.Cmd(Job("s", "repo-bug-prover.md"), "/dev/null", "{}", here, 20, false, "sid");
|
||
|
|
Check("cmd: the functional pass gets --mcp-config with its own output dir and the browser tools allowed; a static pass gets neither; both keep --tools Read,Glob,Grep,Bash and --strict-mcp-config",
|
||
|
|
Has(cf, "--mcp-config") && After(cf, "--mcp-config").contains((td / "playwright" / "f").string()) && After(cf, "--allowedTools").contains("mcp__playwright__browser_snapshot")
|
||
|
|
&& !Has(cs, "--mcp-config") && !After(cs, "--allowedTools").contains("mcp__playwright") && After(cf, "--tools") == "Read,Glob,Grep,Bash" && Has(cs, "--strict-mcp-config")
|
||
|
|
&& After(cf, "--disallowedTools").contains("mcp__playwright__browser_run_code_unsafe"));
|
||
|
|
Check("cmd: --safe-mode drops MCP servers, so the functional pass runs without it and the static pass keeps it", !Has(cf, "--safe-mode") && Has(cs, "--safe-mode"));
|
||
|
|
}
|
||
|
|
{
|
||
|
|
const Pool::Runner rb(options({{"repo-bug-prover.md", "browser"}}));
|
||
|
|
const std::vector<std::string> cb = rb.Cmd(Job("b", "repo-bug-prover.md"), "/dev/null", "{}", here, 20, false, "sid");
|
||
|
|
const std::vector<std::string> cfn = rb.Cmd(Job("f", Fn), "/dev/null", "{}", here, 20, false, "sid");
|
||
|
|
Check("cmd by environment: a static lens given the browser gets --mcp-config, the browser tools and no --safe-mode; a lens with no environment entry runs as none",
|
||
|
|
Has(cb, "--mcp-config") && !Has(cb, "--safe-mode") && After(cb, "--allowedTools").contains("mcp__playwright__browser_snapshot") && !Has(cfn, "--mcp-config") && Has(cfn, "--safe-mode")
|
||
|
|
&& !After(cfn, "--allowedTools").contains("mcp__playwright"));
|
||
|
|
}
|
||
|
|
{
|
||
|
|
const std::string url = "postgresql://postgres:hunter2@127.0.0.1:54322/postgres";
|
||
|
|
// An operator's shell values never reach a pass.
|
||
|
|
setenv(std::string(PgWire::UrlEnv).c_str(), "postgresql://leaked@127.0.0.1:1/x", 1);
|
||
|
|
setenv(std::string(PgWire::BrokerEnv).c_str(), "127.0.0.1:1#leaked", 1);
|
||
|
|
PgWire::Broker broker;
|
||
|
|
broker.Start();
|
||
|
|
Pool::RunnerOptions withDb = options({{"repo-bug-prover.md", "supabase"}, {Fn, "browser+supabase"}});
|
||
|
|
withDb.dbUrl = url;
|
||
|
|
withDb.broker = &broker;
|
||
|
|
Pool::Runner rd(withDb);
|
||
|
|
unsetenv(std::string(PgWire::UrlEnv).c_str());
|
||
|
|
unsetenv(std::string(PgWire::BrokerEnv).c_str());
|
||
|
|
const std::vector<std::string> cd = rd.Cmd(Job("d", "repo-bug-prover.md"), "/dev/null", "{}", here, 20, false, "sid");
|
||
|
|
const std::vector<std::string> cbs = rd.Cmd(Job("e", Fn), "/dev/null", "{}", here, 20, false, "sid");
|
||
|
|
Check("cmd: the supabase environment keeps --safe-mode (no MCP server) and gets the SQL helper allowed; browser+supabase gets --mcp-config, the helper and no --safe-mode; the URL is on no command line",
|
||
|
|
Has(cd, "--safe-mode") && !Has(cd, "--mcp-config") && After(cd, "--allowedTools").contains(sq) && Has(cbs, "--mcp-config") && !Has(cbs, "--safe-mode") && After(cbs, "--allowedTools").contains(sq)
|
||
|
|
&& !(Dyn::Join(cd, " ") + Dyn::Join(cbs, " ")).contains("hunter2"));
|
||
|
|
const Json jd = {{"lensFile", "repo-bug-prover.md"}};
|
||
|
|
const Json js = {{"lensFile", "repo-better-solution.md"}};
|
||
|
|
const Json jf = {{"lensFile", Fn}};
|
||
|
|
const std::string brokerEnv(PgWire::BrokerEnv);
|
||
|
|
const std::map<std::string, std::string> ed = rd.ProcessEnv(jd, rd.Env());
|
||
|
|
const std::map<std::string, std::string> ef = rd.ProcessEnv(jf, rd.Env());
|
||
|
|
const std::map<std::string, std::string> es = rd.ProcessEnv(js, rd.Env());
|
||
|
|
auto token = [&](const std::map<std::string, std::string>& e) {
|
||
|
|
const auto found = e.find(brokerEnv);
|
||
|
|
const std::string value = found == e.end() ? std::string("#") : found->second;
|
||
|
|
return broker.UrlOf(value.substr(value.rfind('#') + 1));
|
||
|
|
};
|
||
|
|
auto shaped = [](std::string_view address) {
|
||
|
|
// ^127\.0\.0\.1:\d+#[0-9a-f]{32}$
|
||
|
|
if (!address.starts_with("127.0.0.1:")) return false;
|
||
|
|
const std::size_t hash = address.find('#');
|
||
|
|
if (hash == std::string_view::npos || hash == 10) return false;
|
||
|
|
const std::string_view port = address.substr(10, hash - 10);
|
||
|
|
const std::string_view tok = address.substr(hash + 1);
|
||
|
|
return std::ranges::all_of(port, [](char c) { return c >= '0' && c <= '9'; }) && tok.size() == 32 && std::ranges::all_of(tok, [](char c) { return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f'); });
|
||
|
|
};
|
||
|
|
bool leaked = false;
|
||
|
|
for (const auto* e : {&ed, &ef}) {
|
||
|
|
for (const auto& [k, v] : *e) leaked = leaked || v.contains("hunter2");
|
||
|
|
}
|
||
|
|
Check("process_env: a db pass gets the broker's address and its database's token - never the URL or its password; a static pass gets neither; the operator's shell values are dropped",
|
||
|
|
shaped(ed.at(brokerEnv)) && token(ed) == url && token(ef) == url && !leaked && !ed.contains(std::string(PgWire::UrlEnv)) && !es.contains(brokerEnv) && !rd.Env().contains(std::string(PgWire::UrlEnv))
|
||
|
|
&& !rd.Env().contains(brokerEnv));
|
||
|
|
const std::string ownDb = "postgresql://postgres:postgres@127.0.0.1:20222/postgres";
|
||
|
|
Pool::RunnerOptions noBroker = options({{"repo-bug-prover.md", "supabase"}});
|
||
|
|
noBroker.dbUrl = url;
|
||
|
|
Check("process_env: a pass that holds a stack gets that stack's database and no other - a stack without one gets none, never the first stack's (finding 1)",
|
||
|
|
token(rd.ProcessEnv({{"lensFile", Fn}, {"stack", {{"id", "slot 2"}, {"dbUrl", ownDb}}}}, rd.Env())) == ownDb
|
||
|
|
&& rd.ProcessEnv({{"lensFile", Fn}, {"stack", {{"id", "slot 2"}, {"dbUrl", nullptr}}}}, rd.Env()).at(brokerEnv).empty()
|
||
|
|
&& Pool::Runner(noBroker).ProcessEnv(jd, {}).at(brokerEnv).empty());
|
||
|
|
broker.Close();
|
||
|
|
}
|
||
|
|
|
||
|
|
auto init = [](const Json& servers, const Json& tools, const Json& browser = Json()) {
|
||
|
|
return Json {{"init", {{"mcp_servers", servers}, {"tools", tools}}}, {"browser", browser.is_null() ? Json {{"calls", 2}, {"errors", 0}} : browser}};
|
||
|
|
};
|
||
|
|
const Json conn = Json::array({{{"name", "playwright"}, {"status", "connected"}}});
|
||
|
|
const Json none = Json::array();
|
||
|
|
const Json read = Json::array({"Read"});
|
||
|
|
const Json pw = Json::array({"mcp__playwright__x"});
|
||
|
|
const Json zero = {{"calls", 0}, {"errors", 0}};
|
||
|
|
const Pool::Verdict ok = {"ok", ""};
|
||
|
|
Check("mcp_connected: ok only with a connected playwright server AND its tools in the init line; else an error that says so",
|
||
|
|
Pool::McpConnected(init(conn, Json::array({"Read", "mcp__playwright__browser_snapshot"}))) == ok && Pool::McpConnected(init(none, read)).first == "error"
|
||
|
|
&& Pool::McpConnected(init(none, read)).second.contains("did not connect") && Pool::McpConnected(init(Json::array({{{"name", "playwright"}, {"status", "failed"}}}), none)).first == "error"
|
||
|
|
&& Pool::McpConnected({{"init", nullptr}}).first == "error");
|
||
|
|
Check("mcp_connected: a whole pass with no browser call, or any pass whose every browser call failed, is an error; a block pass with no browser call is allowed; one answered call is enough",
|
||
|
|
Pool::McpConnected(init(conn, pw, zero), "whole").second.contains("no browser call") && Pool::McpConnected(init(conn, pw, zero), "block") == ok
|
||
|
|
&& Pool::McpConnected(init(conn, pw, {{"calls", 3}, {"errors", 3}}), "block").second.contains("every browser call")
|
||
|
|
&& Pool::McpConnected(init(conn, pw, {{"calls", 3}, {"errors", 3}}), "whole").second.contains("every browser call") && Pool::McpConnected(init(conn, pw, {{"calls", 3}, {"errors", 2}}), "whole") == ok);
|
||
|
|
Check("mcp_connected: a static lens given the browser (not empirical) may finish a whole pass without a browser call; a server that did not connect or answered every call with an error still fails it",
|
||
|
|
Pool::McpConnected(init(conn, pw, zero), "whole", false) == ok && Pool::McpConnected(init(conn, pw, {{"calls", 2}, {"errors", 2}}), "whole", false).first == "error"
|
||
|
|
&& Pool::McpConnected(init(none, read), "block", false).first == "error");
|
||
|
|
Check("exercised: 'browser' is mcp_connected with the lens's empirical flag, 'none' is ok as classified",
|
||
|
|
Pool::Exercised(init(none, read), "browser", "whole", true) == Pool::McpConnected(init(none, read), "whole", true) && Pool::Exercised(init(conn, pw, zero), "browser", "whole", true).first == "error"
|
||
|
|
&& Pool::Exercised(init(none, read), "none", "whole", true) == ok);
|
||
|
|
std::vector<std::string> names;
|
||
|
|
for (const Plan::Environment& e : Plan::Environments) names.emplace_back(e.name);
|
||
|
|
const Plan::Environment& browser = *Plan::FindEnvironment("browser");
|
||
|
|
const Plan::Environment& noneEnv = *Plan::FindEnvironment("none");
|
||
|
|
const Plan::Environment& supabase = *Plan::FindEnvironment("supabase");
|
||
|
|
const Plan::Environment& both = *Plan::FindEnvironment("browser+supabase");
|
||
|
|
auto tools = [](const Plan::Environment& e) { return std::vector<std::string>(e.tools.begin(), e.tools.end()); };
|
||
|
|
bool otherEmpirical = false;
|
||
|
|
for (const Plan::Lens& l : Plan::Lenses) otherEmpirical = otherEmpirical || (l.file != Fn && l.empirical);
|
||
|
|
Check("environments: exactly none, browser, supabase, browser+supabase - a new entry first meets the seven-item list in plan.ENVIRONMENTS (the environment in the reuse keys, _resumable and _salvage; its prompt texts; pool.exercised; validate/preflight; DENY; the opencode refusal; a firing proof)",
|
||
|
|
names == std::vector<std::string>{"none", "browser", "supabase", "browser+supabase"} && browser.mcp == "playwright" && tools(browser) == std::vector<std::string>{"Bash(curl:*)"} && browser.app && !browser.db
|
||
|
|
&& browser.memMb == Plan::MemBrowser && noneEnv.mcp.empty() && tools(noneEnv).empty() && !noneEnv.app && !noneEnv.db && noneEnv.memMb == Plan::MemNone && supabase.mcp.empty() && tools(supabase).empty()
|
||
|
|
&& !supabase.app && supabase.db && supabase.memMb == Plan::MemNone && both.mcp == browser.mcp && tools(both) == tools(browser) && both.app && both.db && both.memMb == browser.memMb
|
||
|
|
&& Plan::LensOf(Fn).empirical && !otherEmpirical);
|
||
|
|
|
||
|
|
// The stream accounting behind it: a navigate the server answered with an
|
||
|
|
// error, a snapshot it answered.
|
||
|
|
const std::vector<std::string> ev = {
|
||
|
|
R"({"type":"system","subtype":"init","mcp_servers":[{"name":"playwright","status":"connected"}],"tools":["mcp__playwright__browser_navigate"]})",
|
||
|
|
R"({"type":"assistant","message":{"role":"assistant","content":[{"type":"tool_use","id":"t1","name":"mcp__playwright__browser_navigate","input":{}},{"type":"tool_use","id":"t2","name":"Bash","input":{}}]}})",
|
||
|
|
R"({"type":"user","message":{"role":"user","content":[{"type":"tool_result","tool_use_id":"t1","is_error":true,"content":"Chromium distribution chrome is not found"},{"type":"tool_result","tool_use_id":"t2","content":"ok"}]}})",
|
||
|
|
R"({"type":"assistant","message":{"role":"assistant","content":[{"type":"tool_use","id":"t3","name":"mcp__playwright__browser_snapshot","input":{}}]}})",
|
||
|
|
R"({"type":"user","message":{"role":"user","content":[{"type":"tool_result","tool_use_id":"t3","content":"- page"}]}})",
|
||
|
|
R"({"type":"result","subtype":"success","is_error":false,"structured_output":{"verdict":"SEND-READY","findings":[],"notes":[],"coverage":[]},"usage":{"input_tokens":1,"output_tokens":1},"num_turns":2})",
|
||
|
|
};
|
||
|
|
const Json ps = Stream::ParseStream(Dyn::Join(ev, "\n"));
|
||
|
|
Check("stream: browser calls and their errors are counted by tool_use id (2 calls, 1 error), Bash errors are not browser errors",
|
||
|
|
Dyn::Equal(ps.at("browser"), {{"calls", 2}, {"errors", 1}}) && ps.at("toolErrors") == 1 && Pool::McpConnected(ps, "whole") == ok);
|
||
|
|
const std::vector<std::string> ev2 = {ev[0], ev[1], ev[2], ev[5]};
|
||
|
|
const Json ps2 = Stream::ParseStream(Dyn::Join(ev2, "\n"));
|
||
|
|
Check("...one failed navigate and nothing else -> every browser call failed -> error", Dyn::Equal(ps2.at("browser"), {{"calls", 1}, {"errors", 1}}) && Pool::McpConnected(ps2, "block").first == "error");
|
||
|
|
// The helper as the Flatpak spells it: what the stream recognises is the
|
||
|
|
// program's name, whatever path a build runs it from.
|
||
|
|
const std::string sqlCommand = "/app/bin/crafter-review sql";
|
||
|
|
auto sqlEv = [&](bool answered, const std::optional<std::string>& cmd = std::nullopt, const Json& content = Json()) {
|
||
|
|
const Json result = content.is_null() ? Json(answered ? std::format("{} to postgres on 127.0.0.1:54322 as postgres - ONE transaction\n SELECT 1", PgWire::Answered)
|
||
|
|
: std::format("{} - cannot connect", PgWire::NoDatabase))
|
||
|
|
: content;
|
||
|
|
return std::vector<std::string> {
|
||
|
|
R"({"type":"system","subtype":"init","mcp_servers":[],"tools":["Bash"]})",
|
||
|
|
DumpSpaced({{"type", "assistant"},
|
||
|
|
{"message", {{"role", "assistant"}, {"content", Json::array({{{"type", "tool_use"}, {"id", "s1"}, {"name", "Bash"}, {"input", {{"command", cmd.value_or(std::format("{} <<'SQL'\nselect 1;\nSQL", sqlCommand))}}}}})}}}}),
|
||
|
|
DumpSpaced({{"type", "user"}, {"message", {{"role", "user"}, {"content", Json::array({{{"type", "tool_result"}, {"tool_use_id", "s1"}, {"is_error", !answered}, {"content", result}}})}}}}),
|
||
|
|
ev[5],
|
||
|
|
};
|
||
|
|
};
|
||
|
|
const Json pa = Stream::ParseStream(Dyn::Join(sqlEv(true), "\n"));
|
||
|
|
const Json pn = Stream::ParseStream(Dyn::Join(sqlEv(false), "\n"));
|
||
|
|
Check("stream: a SQL helper call is counted by its command, and answered by the helper's connected line in its result",
|
||
|
|
Dyn::Equal(pa.at("db"), {{"calls", 1}, {"answered", 1}}) && Dyn::Equal(pn.at("db"), {{"calls", 1}, {"answered", 0}}) && Dyn::Equal(pa.at("browser"), zero));
|
||
|
|
Check("stream: the helper inside a compound command counts; the marker in a list-of-blocks result counts; the marker mid-line does not; another python3 script is no SQL call",
|
||
|
|
Dyn::Equal(Stream::ParseStream(Dyn::Join(sqlEv(true, std::format("cd . && {} 'select 1'", sqlCommand)), "\n")).at("db"), {{"calls", 1}, {"answered", 1}})
|
||
|
|
&& Stream::ParseStream(Dyn::Join(sqlEv(true, std::nullopt, Json::array({{{"type", "text"}, {"text", std::format("{} to x", PgWire::Answered)}}})), "\n")).at("db").at("answered") == 1
|
||
|
|
&& Stream::ParseStream(Dyn::Join(sqlEv(true, std::nullopt, Json(std::format("echo says {}", PgWire::Answered))), "\n")).at("db").at("answered") == 0
|
||
|
|
&& Dyn::Equal(Stream::ParseStream(Dyn::Join(sqlEv(true, std::string("python3 - <<EOF\nprint(1)\nEOF")), "\n")).at("db"), {{"calls", 0}, {"answered", 0}}));
|
||
|
|
const std::vector<std::string> noSql = {sqlEv(true)[0], ev[5]};
|
||
|
|
Check("exercised: supabase is db_answered - a pass whose every SQL call missed the database is an error, a pass with no SQL call is ok (a lens may judge by the code)",
|
||
|
|
Pool::Exercised(pa, "supabase", "block") == ok && Pool::Exercised(pn, "supabase", "block").first == "error" && Pool::Exercised(pn, "supabase", "whole", true).second.contains("reached the database")
|
||
|
|
&& Pool::Exercised(Stream::ParseStream(Dyn::Join(noSql, "\n")), "supabase", "whole", true) == ok && Pool::Exercised(pn, "none", "block") == ok);
|
||
|
|
const Json bothOk = With(pa, {{"init", {{"mcp_servers", conn}, {"tools", pw}}}, {"browser", {{"calls", 2}, {"errors", 0}}}});
|
||
|
|
Check("exercised: browser+supabase needs both - the browser (connected, used by the empirical whole pass) and, when SQL ran, the database",
|
||
|
|
Pool::Exercised(bothOk, "browser+supabase", "whole", true) == ok && Pool::Exercised(With(pa, {{"init", {{"mcp_servers", Json::array()}}}}), "browser+supabase", "block").second.contains("did not connect")
|
||
|
|
&& Pool::Exercised(With(bothOk, {{"db", {{"calls", 2}, {"answered", 0}}}}), "browser+supabase", "whole", true).second.contains("reached the database")
|
||
|
|
&& Pool::Exercised(With(bothOk, {{"browser", zero}}), "browser+supabase", "whole", true).second.contains("no browser call"));
|
||
|
|
|
||
|
|
fs::remove_all(td);
|
||
|
|
return Failures == 0 ? 0 : 1;
|
||
|
|
}
|