Catcrafts/Crafter.Review
Watch
2
0
Fork
You've already forked Crafter.Review
 0
Code Issues 7 Pull requests Projects Releases 3 Packages Wiki Activity Actions

test(pipeline): the checks of the pure pipeline

Browse source 
Stream, Normalize, Merge, Dispositions, Config, CodeKeys and the pure half
of CommitsMode, each a test of its own (131 checks). The fixture rewrites
are checked in (code-keys/<name>/, commits3/), so the suite reads fixtures
and edits nothing. The allowlist, command-line and MCP checks of
CommitsMode come with the pool they test.

Among them: this binary's `<exe> sql` spelling counts as a SQL call, and a
result line with a raw U+2028 in a JSON string is still the result line.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Jorijn van der Graaf 2026-10-31 06:56:09 +01:00
commit 71dba5690f
Signed by: jorijnvdgraaf
GPG key ID: 2937E59CDCC1BCFB
12 changed files with 3778 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

271
tests/CommitsMode/main.cpp Normal file
View file

@ -0,0 +1,271 @@
// SPDX-License-Identifier: AGPL-3.0-only
// SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts®
// Commits mode: the inventory, the gates, the plans (the functional lens's
// n/a), the prompts, over the commits fixture. The allowlist, command line
// and MCP checks sit with the pool they test.
import std;
import Crafter.Review;
namespace fs = std::filesystem;
using namespace Crafter::Review;
namespace {
std::int32_t Failures = 0;
const std::string Fn(Plan::FunctionalLens);
void Check(std::string_view name, bool cond, std::string_view detail = {}) {
std::println("{}{}{}", cond ? " ok " : " FAIL ", name, !cond && !detail.empty() ? std::format(" ({})", detail) : std::string());
if (!cond) ++Failures;
}
std::set<std::string> Set(const std::vector<std::string>& values) {
return {values.begin(), values.end()};
}
std::string Show(const std::set<std::string>& values) {
Json out = Json::array();
for (const std::string& value : values) out.push_back(value);
return Dyn::ReprValue(out);
}
const Json& Unit(const Json& units, std::string_view id) {
for (const Json& unit : units) {
if (unit.at("id") == id) return unit;
}
throw std::runtime_error(std::format("no unit {}", id));
}
const Json& Job(const std::vector<Json>& jobs, std::string_view kind, std::string_view unit = {}) {
for (const Json& job : jobs) {
if (job.at("kind") == kind && (unit.empty() || job.at("unit") == unit)) return job;
}
throw std::runtime_error(std::format("no {} job {}", kind, unit));
}
std::vector<std::pair<std::string, std::vector<std::string>>> Gates(const std::vector<Plan::Gate>& gates) {
std::vector<std::pair<std::string, std::vector<std::string>>> out;
for (const Plan::Gate& gate : gates) out.emplace_back(std::string(gate.title), std::vector<std::string>(gate.lenses.begin(), gate.lenses.end()));
return out;
}
std::vector<std::string> Titles(const std::vector<Plan::Gate>& gates) {
std::vector<std::string> out;
for (const Plan::Gate& gate : gates) out.emplace_back(gate.title);
return out;
}
// re.findall(r'(?i)\b(exploit\w*|attack\w*|...)\b', text): the words that
// start with one of the stems.
std::vector<std::string> Offensive(std::string_view text) {
constexpr std::array<std::string_view, 10> Stems = {"exploit", "attack", "payload", "malicious", "weaponi", "hack", "pwn", "backdoor", "intrusion", "penetrat"};
const std::u32string decoded = Dyn::Decode(text);
std::vector<std::string> found;
std::size_t at = 0;
while (at < decoded.size()) {
if (!Dyn::IsWord(decoded[at])) {
++at;
continue;
}
std::size_t end = at;
while (end < decoded.size() && Dyn::IsWord(decoded[end])) ++end;
const std::u32string_view word = std::u32string_view(decoded).substr(at, end - at);
for (const std::string_view stem : Stems) {
if (word.size() < stem.size()) continue;
bool match = true;
for (std::size_t i = 0; i < stem.size(); ++i) match = match && Dyn::IgnoreCaseLetter(word[i]) == stem[i];
if (match) {
found.push_back(Dyn::Encode(word));
break;
}
}
at = end;
}
return found;
}
}
int main() {
std::println("commits mode: inventory, gates, plans (the functional lens's n/a), prompts (fixtures/commits)");
SetHome(fs::current_path());
const fs::path fixture = fs::current_path() / "tests" / "Goldens" / "fixture";
const fs::path d = fixture / "commits";
const Inventory::Built built = Inventory::Build("commits", d.string());
const Json& units = built.out.at("units");
const std::vector<std::string>& lines = built.lines;
std::vector<std::string> ids;
for (const Json& u : units) ids.push_back(u.at("id").get<std::string>());
const std::set<std::string> idSet = Set(ids);
Check("body-less p1: no msg/trailers units, subject/diffstat/hunk present", !idSet.contains("p1:msg") && !idSet.contains("p1:trailers") && idSet.contains("p1:subject")
&& idSet.contains("p1:diffstat") && idSet.contains("p1:hello.py#1"));
Check("p2: msg unit, no trailers, three hunks over two files", idSet.contains("p2:msg") && !idSet.contains("p2:trailers") && idSet.contains("p2:README.md#1")
&& idSet.contains("p2:hello.py#1") && idSet.contains("p2:hello.py#2"));
Check("the three whole slots", idSet.contains("whole:order") && idSet.contains("whole:absent") && idSet.contains("whole:behaviour") && !idSet.contains("whole:recipients"));
Check("body-less commit: diffstat unit starts at the --- line", Plan::UnitText(lines, Unit(units, "p1:diffstat")).starts_with("---"));
std::string all;
for (const std::string& name : Artifact::PatchFiles(d)) all += *Platform::ReadFile(d / name);
Check("artifact_hash = sha256 of the sorted concatenation; patch_head = the From line of the last patch", Artifact::ArtifactHash("commits", d) == Sha256Hex(all)
&& Artifact::PatchHead(d) == std::string(40, '2'));
const auto g = Gates(Plan::GatesFor());
std::vector<std::string> gateNames;
for (const auto& [title, lenses] : g) gateNames.push_back(Dyn::Split(title, " — ")[0]);
Check("gates: 1 approach, 2 functional, 3 proof - the bug-prover and the security lens side by side (2026-10-22), the owner stays Gate 4",
gateNames == std::vector<std::string>{"Gate 1", "Gate 2", "Gate 3"} && g[0].second == std::vector<std::string>{"repo-better-solution.md"} && g[1].second == std::vector<std::string>{Fn}
&& g[2].second == std::vector<std::string>{"repo-bug-prover.md", "repo-security.md"});
const std::vector<std::string> onlyFn = {Fn};
const std::vector<std::string> security = {"repo-security.md"};
std::set<std::string> lensSet;
for (const Plan::Lens& lens : Plan::Lenses) lensSet.emplace(lens.file);
using GateList = std::vector<std::pair<std::string, std::vector<std::string>>>;
Check("mode_lenses = the four; --only keeps, --without drops, an emptied gate disappears; Gate 3 with one of its lenses dropped keeps the other",
Set(Plan::ModeLenses()) == lensSet && Gates(Plan::GatesFor(onlyFn)) == GateList{{"Gate 2 — Functional", {Fn}}}
&& Titles(Plan::GatesFor({}, onlyFn)) == std::vector<std::string>{"Gate 1 — Approach", "Gate 3 — Proof"} && Plan::GatesFor(onlyFn, onlyFn).empty()
&& Gates(Plan::GatesFor({}, security)).back() == std::pair<std::string, std::vector<std::string>>{"Gate 3 — Proof", {"repo-bug-prover.md"}}
&& Gates(Plan::GatesFor(security)) == GateList{{"Gate 3 — Proof", {"repo-security.md"}}});
const std::vector<std::string> threeOff = {"repo-better-solution.md", "repo-bug-prover.md", "repo-security.md"};
const std::vector<std::string> bugOff = {"repo-bug-prover.md"};
Check("lenses off by config: gates_for drops the lens and its gate; --only cannot bring it back", Titles(Plan::GatesFor({}, {}, onlyFn)) == std::vector<std::string>{"Gate 1 — Approach", "Gate 3 — Proof"}
&& Plan::GatesFor(onlyFn, {}, onlyFn).empty() && Gates(Plan::GatesFor({}, {}, threeOff)) == GateList{{"Gate 2 — Functional", {Fn}}}
&& Gates(Plan::GatesFor({}, {}, bugOff)).back() == std::pair<std::string, std::vector<std::string>>{"Gate 3 — Proof", {"repo-security.md"}});
const std::vector<std::string> allLenses(lensSet.begin(), lensSet.end());
const std::set<std::string> nrAll = Set(Plan::NotReviewed(units, lines, Plan::ModeLenses()));
Check("not reviewed (every lens runs): the text units and the documentation units", nrAll == std::set<std::string>{"p1:subject", "p1:diffstat", "p2:subject", "p2:msg", "p2:diffstat", "p2:README.md#hdr", "p2:README.md#1"},
Show(nrAll));
std::vector<std::string> statics;
for (const Plan::Lens& lens : Plan::Lenses) {
if (lens.file != Fn) statics.emplace_back(lens.file);
}
std::set<std::string> plusBehaviour = nrAll;
plusBehaviour.insert("whole:behaviour");
std::set<std::string> plusStatic = nrAll;
plusStatic.insert({"whole:order", "whole:absent", "p1:outside", "p1:absent", "p2:outside", "p2:absent", "p1:hello.py#hdr", "p2:hello.py#hdr"});
Check("not reviewed is derived from the lenses that run: the functional lens off adds whole:behaviour; only the functional lens adds the static slots and the file headers (a mechanical n/a is no reading)",
Set(Plan::NotReviewed(units, lines, statics)) == plusBehaviour && Set(Plan::NotReviewed(units, lines, onlyFn)) == plusStatic, Show(Set(Plan::NotReviewed(units, lines, onlyFn))));
std::set<std::string> codeIds;
for (const std::string& id : ids) {
if (Plan::UnitKindOf(id) == "code" && !id.contains(".md#")) codeIds.insert(id);
}
auto unitsOf = [](const std::vector<Json>& jobs) {
std::set<std::string> out;
for (const Json& job : jobs) out.insert(job.at("unit").get<std::string>());
return out;
};
const Plan::LensPlan approach = Plan::PlanLens("repo-better-solution.md", units, lines);
std::vector<std::string> expected = {approach.whole.at("expectedIds").at(0).get<std::string>(), approach.whole.at("expectedIds").at(1).get<std::string>()};
std::ranges::sort(expected);
Check("repo-better-solution: whole pass over order+absent, every code unit incl. outside/absent, no doc unit", !approach.whole.is_null()
&& expected == std::vector<std::string>{"whole:absent", "whole:order"} && approach.whole.at("expectedIds").size() == 2 && unitsOf(approach.blocks) == codeIds
&& std::ranges::all_of(approach.blocks, [](const Json& j) { return j.at("kind") == "block"; }));
const Plan::LensPlan prover = Plan::PlanLens("repo-bug-prover.md", units, lines);
Check("repo-bug-prover: no whole pass, the same code units", prover.whole.is_null() && unitsOf(prover.blocks) == codeIds && codeIds.contains("p2:hello.py#hdr"));
const Plan::LensPlan sec = Plan::PlanLens("repo-security.md", units, lines);
const Plan::Lens& s = Plan::LensOf("repo-security.md");
const Plan::Lens& b = Plan::LensOf("repo-bug-prover.md");
const bool sameShape = s.gate == b.gate && s.whole == b.whole && std::ranges::equal(s.slots, b.slots) && std::ranges::equal(s.block, b.block) && s.model == b.model && s.reuse == b.reuse
&& s.wholeReuse == b.wholeReuse && s.reads == b.reads && s.key == b.key && s.empirical == b.empirical && s.perCommitSlots == b.perCommitSlots && s.env == b.env;
Check("repo-security: the bug-prover's shape - no whole pass, the same code units, static blocks keyed on the code, three reads, none by default, not empirical",
sec.whole.is_null() && unitsOf(sec.blocks) == codeIds && sameShape && s.gate == 3
&& std::ranges::all_of(sec.blocks, [](const Json& j) { return j.at("maxTurns") == 60 && j.at("timeoutMin") == 40 && j.at("model") == Plan::ModelFor("opus"); }));
const std::vector<std::string> exclude = {"**/*.test.*", "docs/**"};
const Plan::LensPlan functional = Plan::PlanLens(Fn, units, lines, {}, exclude);
std::map<std::string, std::string> kinds;
for (const Json& j : functional.blocks) kinds[j.at("unit").get<std::string>()] = j.at("kind").get<std::string>();
const std::map<std::string, std::string> wantKinds = {{"p1:hello.py#hdr", "auto-na"}, {"p1:hello.py#1", "block"}, {"p2:hello.py#hdr", "auto-na"}, {"p2:hello.py#1", "block"}, {"p2:hello.py#2", "block"}};
Check("repo-functional: whole pass over whole:behaviour only; hunks as blocks; file headers n/a; no per-commit slots; no doc unit", !functional.whole.is_null()
&& functional.whole.at("expectedIds") == Json::array({"whole:behaviour"}) && kinds == wantKinds);
const Json& functionalBlock = Job(functional.blocks, "block");
Check("functional budgets: block 90/60, whole 220/120; static block 60/40, whole 180/105", functionalBlock.at("maxTurns") == 90 && functionalBlock.at("timeoutMin") == 60
&& functional.whole.at("maxTurns") == 220 && functional.whole.at("timeoutMin") == 120 && prover.blocks.at(0).at("maxTurns") == 60 && prover.blocks.at(0).at("timeoutMin") == 40
&& approach.whole.at("maxTurns") == 180 && approach.whole.at("timeoutMin") == 105);
// A third commit touching a test file: excluded for the functional lens,
// a block for the static ones.
const Inventory::Built three = Inventory::Build("commits", (fixture / "commits3").string());
const std::vector<std::string> testsGlob = {"tests/**"};
std::map<std::string, std::pair<std::string, std::string>> f3;
for (const Json& j : Plan::PlanLens(Fn, three.out.at("units"), three.lines, {}, testsGlob).blocks) {
if (j.at("unit").get<std::string>().starts_with("p3:")) f3[j.at("unit").get<std::string>()] = {j.at("kind").get<std::string>(), Dyn::Get(j, "basis").is_string() ? j.at("basis").get<std::string>() : ""};
}
std::map<std::string, std::string> s3;
for (const Json& j : Plan::PlanLens("repo-bug-prover.md", three.out.at("units"), three.lines).blocks) {
if (j.at("unit").get<std::string>().starts_with("p3:")) s3[j.at("unit").get<std::string>()] = j.at("kind").get<std::string>();
}
Check("functional.exclude: the test hunk is a mechanical n/a naming the glob; the static lens still reads it", f3["p3:tests/test_hello.py#1"].first == "auto-na"
&& f3["p3:tests/test_hello.py#1"].second.contains("tests/**") && s3["p3:tests/test_hello.py#1"] == "block" && !f3.contains("p3:outside") && s3.contains("p3:outside"));
const std::vector<std::string> testTs = {"**/*.test.ts"};
const std::vector<std::string> github = {".github/**"};
Check("excluded_by: ** prefixes match at the root too", Plan::ExcludedBy("a.test.ts", testTs) == "**/*.test.ts" && Plan::ExcludedBy("src/a.test.ts", testTs)
&& !Plan::ExcludedBy("src/a.ts", testTs) && Plan::ExcludedBy(".github/x.yml", github));
// Prompts.
const std::string st = Prompt::SystemText();
Check("system text: commit range; the browser tools belong to a lens with an APPLICATION UNDER TEST block, not to the functional lens by name", st.contains("commit range")
&& st.contains("mcp__playwright__") && st.contains("APPLICATION UNDER TEST block") && !st.contains("The functional lens alone") && !st.contains("The functional lens acts"));
const std::string h = Artifact::ArtifactHash("commits", d);
const std::vector<std::string> dot = {"."};
const std::string rl = Prompt::RunLayer(".review-gates/push/x", "abc123", "2026-10-15", dot, h, lines, units, {}, "myrepo");
Check("run layer: INPUTS (tree `.`, base, no review record), the artifact with line numbers, the inventory", rl.contains("myrepo") && rl.contains("`.` is the working directory")
&& !Dyn::Lower(rl).contains("ledger") && rl.contains("\nARTIFACT (frozen, hash ") && rl.contains("\n\nINVENTORY - ") && rl.contains("\n0001| "));
const std::vector<std::string> worktree = {".review-gates/trees/r-1234"};
const std::vector<std::string> captures = {"captures/"};
const std::string rl2 = Prompt::RunLayer("x", "", "2026-10-15", worktree, h, lines, units, captures, "r");
Check("run layer: the context paths and a worktree named", rl2.contains("captures/") && rl2.contains(".review-gates/trees/r-1234") && rl2.contains("base-commit line"));
const std::string totp = "/app/bin/crafter-review totp cypress/fixtures/created-user.json";
const std::string ab = Prompt::AppBlock({{"appUrl", "http://localhost:8080"}, {"apiUrls", {"http://127.0.0.1:54321"}}, {"login", {{"notes", "TOTP enrolled"}}}, {"notes", "leave billing alone"}},
"cypress/fixtures/created-user.json", totp);
Check("app block: this lens drives it (no lens named); the URL, the origins, the fixture path (not its content), the exact TOTP command, the notes, the shared-stack rules",
ab.contains("this lens drives it") && !ab.contains("every other lens") && ab.contains("http://localhost:8080") && ab.contains("127.0.0.1:54321")
&& ab.contains("cypress/fixtures/created-user.json") && ab.contains(std::format("`{}`", totp)) && ab.contains("TOTP enrolled") && ab.contains("leave billing alone")
&& ab.contains("not reset between passes") && ab.contains("browser_take_screenshot") && ab.contains("browser_snapshot before every action"));
const std::string ab2 = Prompt::AppBlock({{"appUrl", "http://localhost:1"}});
Check("app block without a fixture says so", ab2.contains("no fixture was found"));
const std::string tailApproach = Prompt::Tail(approach.whole, units, lines);
const std::string tailFunctional = Prompt::Tail(functional.whole, units, lines);
Check("whole tails: the approach text lists order+absent; the functional text lists behaviour and names the APPLICATION UNDER TEST", tailApproach.contains("for THIS repository")
&& tailApproach.contains("whole:order") && tailFunctional.contains("as a USER") && tailFunctional.contains("whole:behaviour") && !tailFunctional.contains("whole:order"));
Check("block tails: the functional hint on a functional block, none on a static hunk, the repo hints on the slots",
Prompt::Tail(functionalBlock, units, lines).contains("judges ONE hunk in the running application") && !Prompt::Tail(Job(prover.blocks, "block", "p2:hello.py#1"), units, lines).contains("judges ONE hunk")
&& Prompt::Tail(Job(prover.blocks, "block", "p1:absent"), units, lines).contains("body-less commit") && Prompt::Tail(Job(prover.blocks, "block", "p1:outside"), units, lines).contains("users and callers"));
const std::string ll = Prompt::LensLayer(Fn, "CHARTER TEXT", "DISPOSITIONS BLOCK", ab, "SUMMARY");
const std::string sp = Prompt::SystemPrompt(st, rl, ll);
const std::string up = Prompt::UserPrompt(tailFunctional, {{"maxTurns", 220}});
Check("lens layer order: charter, application block, dispositions, the whole summary; system prompt = common + run + lens; user prompt = tail + budget line, no run layer",
ll.find("CHARTER TEXT") < ll.find("APPLICATION UNDER TEST") && ll.find("APPLICATION UNDER TEST") < ll.find("DISPOSITIONS BLOCK") && ll.find("DISPOSITIONS BLOCK") < ll.find("SUMMARY")
&& sp.starts_with(std::format("{}{}{}{}", st, Prompt::SepRun, rl, Prompt::SepLens)) && sp.ends_with(ll) && up.contains("Tool-call budget for this pass: 220")
&& !up.contains(rl.substr(0, 200)));
const std::string sql = "/app/bin/crafter-review sql";
const std::string dbb = Prompt::DbBlock({{"dbUrl", "postgresql://postgres:hunter2@127.0.0.1:54322/postgres"}, {"notes", "the scan tables are seeded"}}, sql, "postgres", false,
"it answers (x); its 3 applied migrations are the range HEAD's", true);
const std::string dba = Prompt::DbBlock({{"notes", ""}}, sql, "postgres", true);
Check("db block: the exact helper command and its heredoc, one transaction rolled back, nothing shared, the refused statements, the API roles, the limits, the evidence, the owner's notes - never the URL or its password",
dbb.starts_with("DATABASE UNDER TEST") && dbb.contains(std::format("`{} <<'SQL'`", sql)) && dbb.contains("rolls back at its end") && dbb.contains("not by the other passes")
&& dbb.contains("COMMIT") && dbb.contains("ROLLBACK TO are fine") && dbb.contains("set local role authenticated") && dbb.contains("request.jwt.claims") && dbb.contains("service_role")
&& dbb.contains("statement_timeout 20 s") && dbb.contains("NO DATABASE") && dbb.contains("the scan tables are seeded") && !dbb.contains("hunter2") && !dbb.contains("54322")
&& !dbb.contains("reaches the application") && dba.contains("Nothing you write here reaches the application")
&& dbb.contains("Checked by the driver before this gate: it answers (x); its 3 applied migrations are the range HEAD's.") && dba.contains("Checked by the driver before this gate: nothing (a dry run).")
&& dbb.contains("the role `postgres`, which bypasses row-level security.") && dba.contains("the role `postgres`.")
&& Prompt::DbBlock(Json::object(), sql, "reviewer", false, std::nullopt, false).contains("the role `reviewer`, which row-level security applies to."));
const std::string lld = Prompt::LensLayer(Fn, "CHARTER TEXT", "DISPOSITIONS BLOCK", ab, "SUMMARY", dbb);
const std::vector<std::string> layered = {std::format("YOUR CHARTER (process/lenses/{}), verbatim:\n\nCHARTER TEXT", Fn), ab, "DISPOSITIONS BLOCK",
"The whole pass of this lens on this hash concluded (the constraints your block pass rests on):\nSUMMARY"};
Check("lens layer with the database: charter, application block, database block, dispositions, summary; without one it is byte for byte what it was",
lld.find("CHARTER TEXT") < lld.find("APPLICATION UNDER TEST") && lld.find("APPLICATION UNDER TEST") < lld.find("DATABASE UNDER TEST")
&& lld.find("DATABASE UNDER TEST") < lld.find("DISPOSITIONS BLOCK") && lld.find("DISPOSITIONS BLOCK") < lld.find("SUMMARY")
&& Prompt::LensLayer(Fn, "CHARTER TEXT", "DISPOSITIONS BLOCK", ab, "SUMMARY", "") == ll && ll == Dyn::Join(layered, "\n\n"));
Check("system text: a lens with a DATABASE UNDER TEST block runs SQL there, rolled back, and has the exact command it prints",
st.contains("DATABASE UNDER TEST block runs SQL in that database, in transactions the pipeline rolls back") && st.contains("exact SQL command that block prints"));
Check("prompt version 1; the charters exist and the common text has the placeholder filled", PromptVersion == 1
&& std::ranges::all_of(Plan::Lenses, [](const Plan::Lens& lens) { return fs::exists(fs::current_path() / "process" / "lenses" / lens.file); }) && !st.contains("{{"));
// The security charter is the owner's defensive review of their own code
// (2026-10-22: a charter that reads as a request to do harm is declined by
// the process, and a declined pass is NOT RETURNED).
const std::string charter = Prompt::Charter("repo-security.md");
const std::vector<std::string> banned = Offensive(charter);
Check("the security charter: no offensive vocabulary; the throwaway local copy, the smallest step, no tooling built, refutations name the control, the per-commit slots, one residual-risk on the null path",
banned.empty() && charter.contains("throwaway local copy") && charter.contains("smallest step") && charter.contains("no tooling built") && charter.contains("Refutations name the control")
&& charter.contains("`pK:absent`") && charter.contains("`pK:outside`") && charter.contains("exactly one\n `residual-risk`") && charter.starts_with("# Lens: repo-security")
&& charter.contains("Calibration (append dated entries at the bottom)"), Dyn::Join(banned, ", "));
Check("the security charter names what the other lenses own and tells its blocks to route it as a note, never to suppress it", charter.contains("`repo-better-solution`")
&& charter.contains("`repo-bug-prover`") && charter.contains("`repo-functional`") && charter.contains("instead of suppressing it"));
return Failures ? 1 : 0;
}