feat(lenses): the commit-truth lens, repo-facts, as Gate 4
Every claim in every commit message, and in the documentation the range adds or changes, proven or gone. The lens's block kinds are text, doc and slot: a message unit is keyed on its text, the base and the code up to its commit (MessageDeps); a documentation unit on its file, its normalised text, the base and the code of the whole range (DocKeyParts); a diffstat on its text alone; whole:order and whole:absent are asked in slot passes of its own, never stored. A documentation file's diff header and a part the commit writes nothing in are a mechanical n/a. Budgets 60/40 for a text or doc block, 50/35 for a slot; one read. Gate 4, after the proofs, so the messages and documents are read once the code they describe has settled; the owner's gate is 5. The goldens that follow from the lens table are derived by tests/Goldens/derive.py, which asserts that the recorded table still reproduces the recorded entries before it writes. Proven, with the opt-in commit after this one: the suite (26 passed); a dry run of a scratch range (2 commits, a README with sections) by the previous binary and this one with the three static lenses on and the rest off, 25 prompts and 4 system files byte for byte; with repo-facts on as well, those files unchanged and 10 passes and 1 system file added. The lens's firing proof in this pipeline is owed (the owner's go).
This commit is contained in:
parent
61ee4602f4
commit
febbfc3fcc
GPG key ID:
2937E59CDCC1BCFB
19 changed files with 2564 additions and 646 deletions
|
|
@ -35,6 +35,25 @@ namespace {
|
|||
return out;
|
||||
}
|
||||
|
||||
// {unit id: reuse key} of every text and doc unit under the commit-truth
|
||||
// lens (a doc file header and a context-only part are auto-n/a: no key).
|
||||
Keys TruthKeys(const fs::path& d, std::string_view env = {}) {
|
||||
const Inventory::Built built = Inventory::Build("commits", d.string());
|
||||
const Plan::CommitCode cc = Plan::CommitCodeOf(Artifact::PatchTexts(d));
|
||||
const Json& units = built.out.at("units");
|
||||
Keys out;
|
||||
for (const Json& job : Plan::PlanLens(Plan::FactsLens, units, built.lines).blocks) {
|
||||
if (job.at("kind") != "block") continue;
|
||||
const std::string id = job.at("unit").get<std::string>();
|
||||
const Json* u = nullptr;
|
||||
for (const Json& unit : units) {
|
||||
if (unit.at("id") == id) u = &unit;
|
||||
}
|
||||
out[id] = Plan::BlockReuseKey(job, Plan::FactsLens, Plan::UnitText(built.lines, *u), cc, {}, "c", 1, env).value_or("");
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
std::optional<std::string> WholeKey(const fs::path& d, const std::string& lens = "repo-better-solution.md", std::string_view env = {}) {
|
||||
return Plan::WholeCodeKey({{"kind", "whole"}, {"unit", nullptr}}, lens, Plan::CommitCodeOf(Artifact::PatchTexts(d)), {}, "c", 1, env);
|
||||
}
|
||||
|
|
@ -136,6 +155,42 @@ int main() {
|
|||
&& WholeKey(rw, "repo-better-solution.md", "supabase") == WholeKey(src, "repo-better-solution.md", "supabase")
|
||||
&& Changed(CodeKeys(code, "repo-bug-prover.md", "supabase"), sb) == expected);
|
||||
|
||||
// The commit-truth lens's keys (2026-10-31, the fp6 driver's doc_reuse_key
|
||||
// and message_deps): a doc unit on its file, its text, the base and the
|
||||
// code of the whole range; a message on its text, the base and the code
|
||||
// up to its commit; a diffstat on its text alone; a slot pass never.
|
||||
const Keys tbase = TruthKeys(src);
|
||||
std::set<std::string> tbaseValues;
|
||||
for (const auto& [id, key] : tbase) tbaseValues.insert(key);
|
||||
std::set<std::string> sharedWithCode;
|
||||
std::ranges::set_intersection(baseValues, tbaseValues, std::inserter(sharedWithCode, sharedWithCode.begin()));
|
||||
const std::set<std::string> truthUnits = {"p1:subject", "p1:diffstat", "p2:subject", "p2:msg", "p2:diffstat", "p2:README.md#1"};
|
||||
Check("fixture: the commit-truth lens keys the five text units and the one doc unit (the README header is n/a), every key its own, none a code key's, no slot key",
|
||||
std::set<std::string>(std::views::keys(tbase).begin(), std::views::keys(tbase).end()) == truthUnits && std::ranges::all_of(tbase, [](const auto& e) { return !e.second.empty(); })
|
||||
&& tbaseValues.size() == tbase.size() && sharedWithCode.empty());
|
||||
const Keys trw = TruthKeys(rw);
|
||||
Check("the rewrite (p2's message reworded, the patch counter bumped): the doc key and the diffstat keys hold, p2's body and both subjects are asked again",
|
||||
trw.at("p2:README.md#1") == tbase.at("p2:README.md#1") && trw.at("p1:diffstat") == tbase.at("p1:diffstat") && trw.at("p2:diffstat") == tbase.at("p2:diffstat")
|
||||
&& trw.at("p2:msg") != tbase.at("p2:msg") && trw.at("p1:subject") != tbase.at("p1:subject") && trw.at("p2:subject") != tbase.at("p2:subject"));
|
||||
const Keys tdocs = TruthKeys(docs);
|
||||
Check("a reworded README sentence: its doc unit is asked again, nothing else of this lens", Changed(tdocs, tbase) == std::set<std::string>{"p2:README.md#1"});
|
||||
const Keys tcode = TruthKeys(code);
|
||||
Check("one changed line in p2's code: the doc unit (the range is in its key) and p2's subject and body (the code up to p2) are asked again; p1's subject and the diffstats hold",
|
||||
Changed(tcode, tbase) == std::set<std::string>{"p2:README.md#1", "p2:subject", "p2:msg"}, Show(Changed(tcode, tbase)));
|
||||
Check("another base: every doc and message key changes; a diffstat depends on no code", Changed(TruthKeys(bse), tbase) == std::set<std::string>{"p2:README.md#1", "p1:subject", "p2:subject", "p2:msg"});
|
||||
Check("p2 drops its README hunk: no doc key is left", !TruthKeys(nodoc).contains("p2:README.md#1") && TruthKeys(nodoc).size() == 5);
|
||||
const Plan::CommitCode c0 = Plan::CommitCodeOf(Artifact::PatchTexts(src));
|
||||
const Plan::CommitCode c1 = Plan::CommitCodeOf(Artifact::PatchTexts(code));
|
||||
Check("doc key: a sentence depends on the code of the whole range - a later commit's code change asks an earlier commit's sentence again; a part is keyed on its file, not on its hunk or part number",
|
||||
Plan::DocKeyParts("p1:README.md#1", c0) != Plan::DocKeyParts("p1:README.md#1", c1) && Plan::DocKeyParts("p2:README.md#1", c0) == Plan::DocKeyParts("p2:README.md#3.12", c0)
|
||||
&& Plan::DocKeyParts("p2:README.md#1", c0).first != Plan::DocKeyParts("p2:AGENTS.md#1", c0).first && Plan::DocKeyParts("p2:README.md#1", c0).first == "doc:README.md"
|
||||
&& c0.docs.at(1).empty() && c0.docs.at(2).size() == 1 && c0.docs.at(2).at(0).starts_with("diff --git a/README.md b/README.md\nnew file mode"));
|
||||
Check("message key: a commit message depends on the code up to its own commit - p2's code change asks p2's subject and body again, not p1's; a diffstat, a slot and a hunk depend on no code",
|
||||
Plan::MessageDeps("p2:msg", c0) != Plan::MessageDeps("p2:msg", c1) && Plan::MessageDeps("p2:subject", c0) != Plan::MessageDeps("p2:subject", c1)
|
||||
&& Plan::MessageDeps("p1:subject", c0) == Plan::MessageDeps("p1:subject", c1) && !Plan::MessageDeps("p1:subject", c0).empty() && Plan::MessageDeps("p2:diffstat", c0).empty()
|
||||
&& Plan::MessageDeps("whole:order", c0).empty() && Plan::MessageDeps("p2:hello.py#1", c0).empty() && Plan::MessageDeps("p2:msg\n", c0) == Plan::MessageDeps("p2:msg", c0));
|
||||
Check("environment: a text or doc key in another environment is its own, and a slot job has no key", TruthKeys(src, "supabase") != tbase && AllChanged(TruthKeys(src, "supabase"), tbase)
|
||||
&& !Plan::BlockReuseKey({{"kind", "slot"}, {"unit", "whole:order"}, {"unitKind", "slot"}}, Plan::FactsLens, "", c0, {}, "c", 1));
|
||||
const Plan::CommitCode cc = Plan::CommitCodeOf(Artifact::PatchTexts(src));
|
||||
Check("whole key: both wholes have keys and differ; a block never; the bug-prover has none", WholeKey(src) && WholeKey(src, Fn) && WholeKey(src) != WholeKey(src, Fn)
|
||||
&& !WholeKey(src, "repo-bug-prover.md") && !Plan::WholeCodeKey({{"kind", "block"}, {"unit", "x"}}, "repo-better-solution.md", cc, {}, "c", 1));
|
||||
|
|
|
|||
|
|
@ -109,37 +109,45 @@ int main() {
|
|||
const auto g = Gates(Plan::GatesFor());
|
||||
std::vector<std::string> gateNames;
|
||||
for (const auto& [title, lenses] : g) gateNames.push_back(Dyn::Split(title, " — ")[0]);
|
||||
Check("gates: 1 approach, 2 functional, 3 proof - the bug-prover and the security lens side by side (2026-10-22), the owner stays Gate 4",
|
||||
gateNames == std::vector<std::string>{"Gate 1", "Gate 2", "Gate 3"} && g[0].second == std::vector<std::string>{"repo-better-solution.md"} && g[1].second == std::vector<std::string>{Fn}
|
||||
&& g[2].second == std::vector<std::string>{"repo-bug-prover.md", "repo-security.md"});
|
||||
const std::string Facts(Plan::FactsLens);
|
||||
Check("gates: 1 approach, 2 functional, 3 proof - the bug-prover and the security lens side by side (2026-10-22) - 4 commit truth (2026-10-31); the owner is Gate 5",
|
||||
gateNames == std::vector<std::string>{"Gate 1", "Gate 2", "Gate 3", "Gate 4"} && g[0].second == std::vector<std::string>{"repo-better-solution.md"} && g[1].second == std::vector<std::string>{Fn}
|
||||
&& g[2].second == std::vector<std::string>{"repo-bug-prover.md", "repo-security.md"} && g[3] == std::pair<std::string, std::vector<std::string>>{"Gate 4 — Commit truth", {Facts}});
|
||||
const std::vector<std::string> onlyFn = {Fn};
|
||||
const std::vector<std::string> security = {"repo-security.md"};
|
||||
std::set<std::string> lensSet;
|
||||
for (const Plan::Lens& lens : Plan::Lenses) lensSet.emplace(lens.file);
|
||||
using GateList = std::vector<std::pair<std::string, std::vector<std::string>>>;
|
||||
Check("mode_lenses = the four; --only keeps, --without drops, an emptied gate disappears; Gate 3 with one of its lenses dropped keeps the other",
|
||||
Check("mode_lenses = the five; --only keeps, --without drops, an emptied gate disappears; Gate 3 with one of its lenses dropped keeps the other",
|
||||
Set(Plan::ModeLenses()) == lensSet && Gates(Plan::GatesFor(onlyFn)) == GateList{{"Gate 2 — Functional", {Fn}}}
|
||||
&& Titles(Plan::GatesFor({}, onlyFn)) == std::vector<std::string>{"Gate 1 — Approach", "Gate 3 — Proof"} && Plan::GatesFor(onlyFn, onlyFn).empty()
|
||||
&& Gates(Plan::GatesFor({}, security)).back() == std::pair<std::string, std::vector<std::string>>{"Gate 3 — Proof", {"repo-bug-prover.md"}}
|
||||
&& Titles(Plan::GatesFor({}, onlyFn)) == std::vector<std::string>{"Gate 1 — Approach", "Gate 3 — Proof", "Gate 4 — Commit truth"} && Plan::GatesFor(onlyFn, onlyFn).empty()
|
||||
&& Gates(Plan::GatesFor({}, security))[2] == std::pair<std::string, std::vector<std::string>>{"Gate 3 — Proof", {"repo-bug-prover.md"}}
|
||||
&& Gates(Plan::GatesFor(security)) == GateList{{"Gate 3 — Proof", {"repo-security.md"}}});
|
||||
const std::vector<std::string> threeOff = {"repo-better-solution.md", "repo-bug-prover.md", "repo-security.md"};
|
||||
const std::vector<std::string> bugOff = {"repo-bug-prover.md"};
|
||||
Check("lenses off by config: gates_for drops the lens and its gate; --only cannot bring it back", Titles(Plan::GatesFor({}, {}, onlyFn)) == std::vector<std::string>{"Gate 1 — Approach", "Gate 3 — Proof"}
|
||||
&& Plan::GatesFor(onlyFn, {}, onlyFn).empty() && Gates(Plan::GatesFor({}, {}, threeOff)) == GateList{{"Gate 2 — Functional", {Fn}}}
|
||||
&& Gates(Plan::GatesFor({}, {}, bugOff)).back() == std::pair<std::string, std::vector<std::string>>{"Gate 3 — Proof", {"repo-security.md"}});
|
||||
const std::vector<std::string> onlyFacts = {Facts};
|
||||
Check("lenses off by config: gates_for drops the lens and its gate; --only cannot bring it back", Titles(Plan::GatesFor({}, {}, onlyFn)) == std::vector<std::string>{"Gate 1 — Approach", "Gate 3 — Proof", "Gate 4 — Commit truth"}
|
||||
&& Plan::GatesFor(onlyFn, {}, onlyFn).empty() && Gates(Plan::GatesFor({}, {}, threeOff)) == GateList{{"Gate 2 — Functional", {Fn}}, {"Gate 4 — Commit truth", {Facts}}}
|
||||
&& Gates(Plan::GatesFor({}, {}, bugOff))[2] == std::pair<std::string, std::vector<std::string>>{"Gate 3 — Proof", {"repo-security.md"}}
|
||||
&& Gates(Plan::GatesFor(onlyFacts)) == GateList{{"Gate 4 — Commit truth", {Facts}}} && Titles(Plan::GatesFor({}, {}, onlyFacts)).size() == 3);
|
||||
const std::vector<std::string> allLenses(lensSet.begin(), lensSet.end());
|
||||
const std::set<std::string> textAndDoc = {"p1:subject", "p1:diffstat", "p2:subject", "p2:msg", "p2:diffstat", "p2:README.md#hdr", "p2:README.md#1"};
|
||||
const std::set<std::string> nrAll = Set(Plan::NotReviewed(units, lines, Plan::ModeLenses()));
|
||||
Check("not reviewed (every lens runs): the text units and the documentation units", nrAll == std::set<std::string>{"p1:subject", "p1:diffstat", "p2:subject", "p2:msg", "p2:diffstat", "p2:README.md#hdr", "p2:README.md#1"},
|
||||
Show(nrAll));
|
||||
std::vector<std::string> withoutFacts;
|
||||
for (const Plan::Lens& lens : Plan::Lenses) {
|
||||
if (lens.file != Facts) withoutFacts.emplace_back(lens.file);
|
||||
}
|
||||
Check("not reviewed (every lens runs): only the documentation file's header, a mechanical n/a of the commit-truth lens; without that lens the text units and the documentation units",
|
||||
nrAll == std::set<std::string>{"p2:README.md#hdr"} && Set(Plan::NotReviewed(units, lines, withoutFacts)) == textAndDoc, Show(nrAll));
|
||||
std::vector<std::string> statics;
|
||||
for (const Plan::Lens& lens : Plan::Lenses) {
|
||||
if (lens.file != Fn) statics.emplace_back(lens.file);
|
||||
}
|
||||
std::set<std::string> plusBehaviour = nrAll;
|
||||
plusBehaviour.insert("whole:behaviour");
|
||||
std::set<std::string> plusStatic = nrAll;
|
||||
std::set<std::string> plusStatic = textAndDoc;
|
||||
plusStatic.insert({"whole:order", "whole:absent", "p1:outside", "p1:absent", "p2:outside", "p2:absent", "p1:hello.py#hdr", "p2:hello.py#hdr"});
|
||||
Check("not reviewed is derived from the lenses that run: the functional lens off adds whole:behaviour; only the functional lens adds the static slots and the file headers (a mechanical n/a is no reading)",
|
||||
Check("not reviewed is derived from the lenses that run: the functional lens off adds whole:behaviour; only the functional lens adds the static slots, the file headers and the prose (a mechanical n/a is no reading)",
|
||||
Set(Plan::NotReviewed(units, lines, statics)) == plusBehaviour && Set(Plan::NotReviewed(units, lines, onlyFn)) == plusStatic, Show(Set(Plan::NotReviewed(units, lines, onlyFn))));
|
||||
std::set<std::string> codeIds;
|
||||
for (const std::string& id : ids) {
|
||||
|
|
@ -166,6 +174,36 @@ int main() {
|
|||
Check("repo-security: the bug-prover's shape - no whole pass, the same code units, static blocks keyed on the code, three reads, none by default, not empirical",
|
||||
sec.whole.is_null() && unitsOf(sec.blocks) == codeIds && sameShape && s.gate == 3
|
||||
&& std::ranges::all_of(sec.blocks, [](const Json& j) { return j.at("maxTurns") == 60 && j.at("timeoutMin") == 40 && j.at("model") == Plan::ModelFor("opus"); }));
|
||||
// The commit-truth lens (2026-10-31): the prose of the range and its two
|
||||
// whole slots, in passes of their own.
|
||||
const Plan::LensPlan truth = Plan::PlanLens(Facts, units, lines);
|
||||
std::map<std::string, std::string> truthKinds;
|
||||
for (const Json& j : truth.blocks) truthKinds[j.at("unit").get<std::string>()] = j.at("kind").get<std::string>();
|
||||
const std::map<std::string, std::string> wantTruth = {{"p1:subject", "block"}, {"p1:diffstat", "block"}, {"p2:subject", "block"}, {"p2:msg", "block"}, {"p2:diffstat", "block"},
|
||||
{"p2:README.md#hdr", "auto-na"}, {"p2:README.md#1", "block"}, {"whole:order", "slot"}, {"whole:absent", "slot"}};
|
||||
const Json& truthMsg = Job(truth.blocks, "block", "p2:msg");
|
||||
const Json& truthDoc = Job(truth.blocks, "block", "p2:README.md#1");
|
||||
const Json& truthOrder = Job(truth.blocks, "slot", "whole:order");
|
||||
const Plan::Lens& t = Plan::LensOf(Facts);
|
||||
Check("repo-facts: no whole pass; the text units, the documentation units and its two whole slots (whole:behaviour is the functional lens's); a doc file header is a mechanical n/a with its own basis; "
|
||||
"text and doc blocks 60/40, a slot 50/35, one read, opus, Gate 4, none by default",
|
||||
truth.whole.is_null() && truthKinds == wantTruth && Job(truth.blocks, "auto-na", "p2:README.md#hdr").at("basis") == "mechanical: diff header of a documentation file"
|
||||
&& truthMsg.at("unitKind") == "text" && truthMsg.at("maxTurns") == 60 && truthMsg.at("timeoutMin") == 40 && truthDoc.at("unitKind") == "doc" && truthDoc.at("maxTurns") == 60
|
||||
&& truthOrder.at("unitKind") == "slot" && truthOrder.at("maxTurns") == 50 && truthOrder.at("timeoutMin") == 35 && truthOrder.at("label") == "repo-facts:slot:whole:order"
|
||||
&& truthOrder.at("expectedIds") == Json::array({"whole:order"}) && !t.reads && t.gate == 4 && t.model == "opus" && t.env == "none" && !t.empirical && t.whole.empty()
|
||||
&& std::ranges::all_of(truth.blocks, [](const Json& j) { return j.at("kind") == "auto-na" || j.at("model") == Plan::ModelFor("opus"); }),
|
||||
Dyn::ReprValue(Json(truthKinds)));
|
||||
const Plan::Budget textElsewhere = Plan::BudgetFor("block", "text", "repo-bug-prover.md");
|
||||
const Plan::Budget docBudget = Plan::BudgetFor("block", "doc", Facts);
|
||||
Check("budgets: a text block of another lens 50/30, a doc block 60/40, a slot 50/35; an unknown unit kind is still a KeyError", textElsewhere.maxTurns == 50 && textElsewhere.timeoutMin == 30
|
||||
&& docBudget.maxTurns == 60 && docBudget.timeoutMin == 40 && Plan::BudgetFor("slot", "slot", Facts).maxTurns == 50 && [] {
|
||||
try {
|
||||
Plan::BudgetFor("block", "nope", "repo-bug-prover.md");
|
||||
return false;
|
||||
} catch (const Dyn::Error&) {
|
||||
return true;
|
||||
}
|
||||
}());
|
||||
const std::vector<std::string> exclude = {"**/*.test.*", "docs/**"};
|
||||
const Plan::LensPlan functional = Plan::PlanLens(Fn, units, lines, {}, exclude);
|
||||
std::map<std::string, std::string> kinds;
|
||||
|
|
@ -223,9 +261,22 @@ int main() {
|
|||
const std::string tailFunctional = Prompt::Tail(functional.whole, units, lines);
|
||||
Check("whole tails: the approach text lists order+absent; the functional text lists behaviour and names the APPLICATION UNDER TEST", tailApproach.contains("for THIS repository")
|
||||
&& tailApproach.contains("whole:order") && tailFunctional.contains("as a USER") && tailFunctional.contains("whole:behaviour") && !tailFunctional.contains("whole:order"));
|
||||
const std::string hunkTail = Prompt::Tail(Job(prover.blocks, "block", "p2:hello.py#1"), units, lines);
|
||||
Check("block tails: the functional hint on a functional block, none on a static hunk, the repo hints on the slots",
|
||||
Prompt::Tail(functionalBlock, units, lines).contains("judges ONE hunk in the running application") && !Prompt::Tail(Job(prover.blocks, "block", "p2:hello.py#1"), units, lines).contains("judges ONE hunk")
|
||||
Prompt::Tail(functionalBlock, units, lines).contains("judges ONE hunk in the running application") && !hunkTail.contains("judges ONE hunk")
|
||||
&& Prompt::Tail(Job(prover.blocks, "block", "p1:absent"), units, lines).contains("body-less commit") && Prompt::Tail(Job(prover.blocks, "block", "p1:outside"), units, lines).contains("users and callers"));
|
||||
const std::string msgTail = Prompt::Tail(truthMsg, units, lines);
|
||||
const std::string docTail = Prompt::Tail(truthDoc, units, lines);
|
||||
const std::string orderTail = Prompt::Tail(truthOrder, units, lines);
|
||||
const std::string absentTail = Prompt::Tail(Job(truth.blocks, "slot", "whole:absent"), units, lines);
|
||||
Check("repo-facts tails: a message block judges a claim at its source with the text hint, a doc block with the documentation hint, each slot pass the slot's question with this lens's duty; "
|
||||
"a code block's tail stays a hunk against the standard, and a slot of another lens would carry no duty",
|
||||
msgTail.contains("a claim at its source, a sentence against its proof obligation") && msgTail.contains("verify every claim against this commit's diff") && !msgTail.contains("a hunk against the standard")
|
||||
&& docTail.contains("a claim at its source") && docTail.contains("This unit is documentation the commit adds or changes") && docTail.contains("UNIT TEXT (verbatim")
|
||||
&& orderTail.starts_with("SCOPE - SLOT PASS, unit \"whole:order\"") && orderTail.contains("Answer it for your lens; for this lens that means each message describes its own commit alone")
|
||||
&& absentTail.contains("documentation the range leaves stale") && absentTail.ends_with(Prompt::GenericVocab)
|
||||
&& hunkTail.contains("a hunk against the standard") && !hunkTail.contains("a claim at its source") && Prompt::SlotDuty("repo-bug-prover.md", "whole:order").empty()
|
||||
&& Prompt::SlotDuty(Facts, "whole:behaviour").empty());
|
||||
const std::string ll = Prompt::LensLayer(Fn, "CHARTER TEXT", "DISPOSITIONS BLOCK", ab, "SUMMARY");
|
||||
const std::string sp = Prompt::SystemPrompt(st, rl, ll);
|
||||
const std::string up = Prompt::UserPrompt(tailFunctional, {{"maxTurns", 220}});
|
||||
|
|
|
|||
|
|
@ -86,8 +86,8 @@ int main() {
|
|||
const Json defaults = Config::Defaults();
|
||||
std::set<std::string> defaultNames;
|
||||
for (auto it = defaults.at("lenses").begin(); it != defaults.at("lenses").end(); ++it) defaultNames.insert(it.key());
|
||||
const Json environments = {{"repo-better-solution.md", "none"}, {fn, "browser"}, {"repo-bug-prover.md", "none"}, {"repo-security.md", "none"}};
|
||||
Check("lenses: the defaults run all four with none/browser/none/none - plan.LENSES' env; the table names every lens of plan.LENSES", loaded.problems.empty() && allRun
|
||||
const Json environments = {{"repo-better-solution.md", "none"}, {fn, "browser"}, {"repo-bug-prover.md", "none"}, {"repo-security.md", "none"}, {Plan::FactsLens, "none"}};
|
||||
Check("lenses: the defaults run all five with none/browser/none/none/none - plan.LENSES' env; the table names every lens of plan.LENSES", loaded.problems.empty() && allRun
|
||||
&& Config::LensesOff(loaded.config).empty() && Dyn::Equal(Config::Environments(loaded.config), environments) && defaultNames == names && defaultsMatch, Show(loaded.problems));
|
||||
put({{"lenses", {{"repo-functional", {{"run", false}}}, {"repo-bug-prover", {{"environment", "browser"}}}}}});
|
||||
loaded = load();
|
||||
|
|
|
|||
166
tests/Goldens/derive.py
Normal file
166
tests/Goldens/derive.py
Normal file
|
|
@ -0,0 +1,166 @@
|
|||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
# SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts®
|
||||
"""Re-derive the goldens that follow from the lens table and the config defaults.
|
||||
|
||||
tests/Goldens/fixture/pipeline.json holds the recorded answers the suite
|
||||
compares against. Where an entry follows from the lens table or the config
|
||||
defaults, this script derives it from the rules instead of a hand edit: the
|
||||
config cases through the binary's own `config` command (what Load, Validate,
|
||||
LensesOff and Environments answer), the commit-truth lens's plans, the
|
||||
not-reviewed lists and the estimates from the recorded units by the lens's
|
||||
rules, and the gates from the table. Before it writes, it asserts that the
|
||||
recorded table still reproduces the recorded entries, so a drift in the
|
||||
derivation itself shows up as a failure here.
|
||||
|
||||
python3 -I tests/Goldens/derive.py <checkout> <crafter-review binary>
|
||||
|
||||
Run from anywhere; review the diff of pipeline.json. The suite never runs it.
|
||||
"""
|
||||
import sys, json, hashlib, subprocess, pathlib, tempfile, os, shutil
|
||||
|
||||
W = pathlib.Path(sys.argv[1]); EXE = sys.argv[2]
|
||||
G = W / 'tests/Goldens/fixture/pipeline.json'
|
||||
g = json.loads(G.read_text())
|
||||
TABLE = [('repo-better-solution.md', 'none'), ('repo-functional.md', 'browser'), ('repo-bug-prover.md', 'none'), ('repo-security.md', 'none'), ('repo-facts.md', 'none')]
|
||||
GATES = [('Gate 1 — Approach', ['repo-better-solution.md']), ('Gate 2 — Functional', ['repo-functional.md']),
|
||||
('Gate 3 — Proof', ['repo-bug-prover.md', 'repo-security.md']), ('Gate 4 — Commit truth', ['repo-facts.md'])]
|
||||
name_of = lambda f: f.replace('.md', '')
|
||||
sha = lambda s: hashlib.sha256(s.encode()).hexdigest()
|
||||
|
||||
# ---- config: every case through the binary --------------------------------------------
|
||||
c = g['config']
|
||||
scratch = pathlib.Path(tempfile.mkdtemp(prefix='goldens-patch-'))
|
||||
def run_config(repo, extra=()):
|
||||
r = subprocess.run([EXE, *extra, 'config'], cwd=repo, capture_output=True, text=True, env={**os.environ, 'GIT_CONFIG_GLOBAL': '/dev/null', 'GIT_CONFIG_SYSTEM': '/dev/null'})
|
||||
lines = r.stdout.split('\n'); body, warnings, problems = [], [], []
|
||||
tail = False
|
||||
for l in lines:
|
||||
if l.startswith('warning: '): warnings.append(l[9:]); tail = True
|
||||
elif l.startswith('problem: '): problems.append(l[9:]); tail = True
|
||||
elif not tail: body.append(l)
|
||||
shown = json.loads('\n'.join(body))
|
||||
return shown, warnings, problems, r.returncode
|
||||
def spelled(text, repo):
|
||||
text = text.replace(str(repo), '<repo>')
|
||||
return text.replace(f'`{EXE} slots`', '`<invocation> slots`')
|
||||
# LensEntry: the config's entry for the lens, else its default one - the
|
||||
# defaults as the binary answers them, so this holds whatever they are.
|
||||
default_dir = scratch / 'defaults'; default_dir.mkdir()
|
||||
DEFAULT_LENSES = run_config(default_dir)[0]['lenses']
|
||||
def entry_of(lenses, name):
|
||||
e = lenses.get(name) if isinstance(lenses, dict) else None
|
||||
return e if isinstance(e, dict) else DEFAULT_LENSES[name]
|
||||
changed_problems = {}
|
||||
for case in c['cases']:
|
||||
repo = scratch / case['name']; repo.mkdir(parents=True)
|
||||
if case['body'] is not None:
|
||||
(repo / '.review-gates').mkdir()
|
||||
(repo / '.review-gates' / 'config.json').write_text(case['body'], encoding='utf-8', newline='')
|
||||
shown, warnings, problems, code = run_config(repo)
|
||||
path = shown.pop('_path')
|
||||
old_problems = case['problems']
|
||||
case['problems'] = [spelled(x, repo) for x in problems]
|
||||
case['warnings'] = [spelled(x, repo) for x in warnings]
|
||||
case['config'] = json.loads(spelled(json.dumps(shown), repo))
|
||||
case['path'] = None if path is None else spelled(path, repo)
|
||||
lenses = shown.get('lenses')
|
||||
case['lensesOff'] = [f for f, _ in TABLE if entry_of(lenses, name_of(f)).get('run') is False]
|
||||
case['environments'] = {f: entry_of(lenses, name_of(f)).get('environment', 'none') for f, _ in TABLE}
|
||||
if old_problems != case['problems']: changed_problems[case['name']] = (old_problems, case['problems'])
|
||||
missing = scratch / 'missing'; missing.mkdir()
|
||||
_, _, mprob, _ = run_config(missing, ['--config', str(missing / 'none.json')])
|
||||
c['missing'] = [spelled(x, missing) for x in mprob]
|
||||
# The defaults as the binary answers them: the 'none' case is Defaults() with
|
||||
# _path null and the database password masked, which the record keeps unmasked.
|
||||
none = next(case for case in c['cases'] if case['name'] == 'none')
|
||||
defaults = json.loads(json.dumps(none['config']))
|
||||
defaults['supabase']['dbUrl'] = c['defaults']['supabase']['dbUrl']
|
||||
c['defaults'] = defaults
|
||||
c['example'] = json.loads((W / 'process/templates/config.json').read_text())
|
||||
ex = scratch / 'example'; ex.mkdir()
|
||||
_, _, eprob, _ = run_config(ex, ['--config', str(W / 'process/templates/config.json')])
|
||||
c['exampleProblems'] = [spelled(x, ex) for x in eprob]
|
||||
shutil.rmtree(scratch)
|
||||
print('config cases with changed problems:', len(changed_problems))
|
||||
for n, (o, new) in changed_problems.items(): print(' ', n, '\n -', o, '\n +', new)
|
||||
|
||||
# ---- series: the facts lens's plans, not_reviewed, the estimate ------------------------
|
||||
OPUS = 'claude-opus-5-5'
|
||||
def facts_plan(entry):
|
||||
inv = entry['inventory']['units']; info = entry['units']
|
||||
blocks = []
|
||||
for u, meta in zip(inv, info):
|
||||
uid = u['id']; kind = meta['kind']
|
||||
if kind == 'code' and meta['doc']: kind = 'doc'
|
||||
common = {'lens': 'repo-facts', 'lensFile': 'repo-facts.md'}
|
||||
if kind == 'slot':
|
||||
if uid not in ('whole:order', 'whole:absent'): continue
|
||||
blocks.append({'label': f'repo-facts:slot:{uid}', **common, 'kind': 'slot', 'unit': uid, 'unitKind': 'slot', 'expectedIds': [uid], 'unitSha': sha(meta['text']),
|
||||
'model': OPUS, 'maxTurns': 50, 'timeoutMin': 35})
|
||||
elif kind == 'doc':
|
||||
if uid.endswith('#hdr'):
|
||||
blocks.append({'label': f'repo-facts:na:{uid}', **common, 'kind': 'auto-na', 'unit': uid, 'expectedIds': [uid], 'unitSha': sha(meta['text']), 'basis': 'mechanical: diff header of a documentation file'})
|
||||
elif not any(l[:1] in ('+', '-') and l[1:].strip() for l in meta['norm'].split('\n')):
|
||||
blocks.append({'label': f'repo-facts:na:{uid}', **common, 'kind': 'auto-na', 'unit': uid, 'expectedIds': [uid], 'unitSha': sha(meta['text']), 'basis': 'mechanical: context only - the commit adds and removes no line in this part'})
|
||||
else:
|
||||
blocks.append({'label': f'repo-facts:block:{uid}', **common, 'kind': 'block', 'unit': uid, 'unitKind': 'doc', 'expectedIds': [uid], 'unitSha': sha(meta['text']), 'model': OPUS, 'maxTurns': 60, 'timeoutMin': 40})
|
||||
elif kind == 'text':
|
||||
blocks.append({'label': f'repo-facts:block:{uid}', **common, 'kind': 'block', 'unit': uid, 'unitKind': 'text', 'expectedIds': [uid], 'unitSha': sha(meta['text']), 'model': OPUS, 'maxTurns': 60, 'timeoutMin': 40})
|
||||
return {'whole': None, 'blocks': blocks}
|
||||
def estimate(jobs, prefix_bytes=123457, lens_layer_bytes={'repo-better-solution': 4100, 'repo-functional': 9000}, tail_bytes=1800, tpb=3.7, sys_tokens=32000, out_per_turn=1200):
|
||||
P = prefix_bytes / tpb + sys_tokens
|
||||
total = {'cacheRead': 0.0, 'cacheWrite': 0.0, 'output': 0.0, 'processes': 0}
|
||||
seen = set()
|
||||
for j in jobs:
|
||||
if j['kind'] == 'auto-na': continue
|
||||
L = lens_layer_bytes.get(j['lens'], 3000) / tpb; T = tail_bytes / tpb
|
||||
k = 40 if j['kind'] == 'whole' else 10 if j['kind'] == 'slot' else 25 if j['lensFile'] == 'repo-functional.md' else 10 if j.get('unitKind') == 'code' else (12 if j['lensFile'] == 'repo-facts.md' else 8)
|
||||
key = (j['lens'], j['kind'])
|
||||
if key not in seen: seen.add(key); total['cacheWrite'] += P + L
|
||||
total['cacheWrite'] += T + k * 3000
|
||||
total['cacheRead'] += sum(P + L + T + 3000 * i for i in range(k))
|
||||
total['output'] += k * out_per_turn
|
||||
total['processes'] += 1
|
||||
total['costUnits'] = 1.25 * total['cacheWrite'] + 0.1 * total['cacheRead'] + 5 * total['output']
|
||||
total['listUsd'] = round((6.25 * total['cacheWrite'] + 0.5 * total['cacheRead'] + 25 * total['output']) / 1e6, 2)
|
||||
return total
|
||||
for name, entry in g['series'].items():
|
||||
if 'plans' not in entry: continue
|
||||
jobs = []
|
||||
recorded = []
|
||||
for f, _ in TABLE:
|
||||
plan = entry['plans'].get(f)
|
||||
if plan is None: continue
|
||||
here = ([plan['whole']] if plan['whole'] is not None else []) + plan['blocks']
|
||||
recorded += here
|
||||
if f != 'repo-facts.md': jobs += here
|
||||
assert estimate(recorded) == entry['estimate'], f'{name}: the estimate formula does not reproduce the golden'
|
||||
fp = facts_plan(entry)
|
||||
entry['plans']['repo-facts.md'] = fp
|
||||
entry['plans']['repo-facts.md:exclude'] = fp
|
||||
covered = {b['unit'] for b in fp['blocks'] if b['kind'] != 'auto-na'}
|
||||
nr = entry['notReviewed']
|
||||
nr['all'] = [i for i in nr['all'] if i not in covered]
|
||||
nr['static'] = [i for i in nr['static'] if i not in covered]
|
||||
entry['estimate'] = estimate(jobs + fp['blocks'])
|
||||
print(f"series {name}: facts plan {len(fp['blocks'])} jobs ({sum(b['kind']=='auto-na' for b in fp['blocks'])} n/a), not reviewed all -> {nr['all']}")
|
||||
|
||||
# ---- gates ---------------------------------------------------------------------------------
|
||||
def gates_for(table, only, without, off):
|
||||
out = []
|
||||
for title, lenses in table:
|
||||
kept = [l for l in lenses if (not only or l in only) and l not in without and l not in off]
|
||||
if kept: out.append([title, kept])
|
||||
return out
|
||||
for args, old in g['gates']:
|
||||
only, without, off = args.get('only', []), args.get('without', []), args.get('off', [])
|
||||
assert old in (gates_for(GATES[:3], only, without, off), gates_for(GATES, only, without, off)), f'gates {args}: neither table reproduces the golden'
|
||||
new_gates = [[args, gates_for(GATES, args.get('only', []), args.get('without', []), args.get('off', []))] for args, _ in g['gates']]
|
||||
print('gates entries changed:', sum(1 for (a, o), (_, n) in zip(g['gates'], new_gates) if o != n), 'of', len(new_gates))
|
||||
g['gates'] = new_gates
|
||||
|
||||
# ---- the charter ----------------------------------------------------------------------------
|
||||
g['prompts']['charterSha']['repo-facts.md'] = sha((W / 'process/lenses/repo-facts.md').read_text())
|
||||
G.write_text(json.dumps(g, indent=1, ensure_ascii=False) + '\n', encoding='utf-8')
|
||||
print('written', G)
|
||||
File diff suppressed because one or more lines are too long
|
|
@ -61,7 +61,7 @@ for a in "$@"; do
|
|||
prev=$a
|
||||
done
|
||||
lens=$(sed -n 's/^# Lens: //p' "$sys" | head -n 1)
|
||||
ids=$(sed -n -e 's/^SCOPE - BLOCK PASS, unit "\([^"]*\)".*/\1/p' -e 's/^- \(whole:[a-z]*\) \[.*/\1/p')
|
||||
ids=$(sed -n -e 's/^SCOPE - BLOCK PASS, unit "\([^"]*\)".*/\1/p' -e 's/^SCOPE - SLOT PASS, unit "\([^"]*\)".*/\1/p' -e 's/^- \(whole:[a-z]*\) \[.*/\1/p')
|
||||
echo "$$ $lens" $ids >> "$FAKE_DIR/calls"
|
||||
echo "{\"type\":\"system\",\"subtype\":\"init\",\"session_id\":\"$sid\",\"model\":\"$model\",\"mcp_servers\":[],\"tools\":[\"Read\"]}"
|
||||
echo '{"type":"assistant","message":{"role":"assistant","content":[{"type":"text","text":"reading"}]}}'
|
||||
|
|
@ -197,13 +197,14 @@ echo "{\"type\":\"result\",\"subtype\":\"success\",\"is_error\":false,\"result\"
|
|||
{"REVIEW_GATES_BOARD", (base / "board.json").string()}, {"FAKE_DIR", fakeDir.string()}});
|
||||
|
||||
// A repository with one commit to review, the functional lens off (it
|
||||
// needs a running application).
|
||||
// needs a running application) and the commit-truth lens off (it gets
|
||||
// the last scenario).
|
||||
fs::create_directories(repo);
|
||||
Git(repo, {"init", "-q", "-b", "main"});
|
||||
Git(repo, {"config", "user.email", "me@example.com"});
|
||||
Git(repo, {"config", "user.name", "Me"});
|
||||
Write(repo / ".gitignore", ".review-gates/push/\n.review-gates/trees/\n");
|
||||
Write(repo / ".review-gates" / "config.json", R"({"lenses": {"repo-functional": {"run": false}}})");
|
||||
Write(repo / ".review-gates" / "config.json", R"({"lenses": {"repo-functional": {"run": false}, "repo-facts": {"run": false}}})");
|
||||
Write(repo / "README.md", "a greeter\n");
|
||||
Git(repo, {"add", "-A"});
|
||||
Git(repo, {"commit", "-q", "-m", "base"});
|
||||
|
|
@ -420,6 +421,32 @@ echo "{\"type\":\"result\",\"subtype\":\"success\",\"is_error\":false,\"result\"
|
|||
Check("an error escaping a full run (a reserve that is no number) ends it with its ValueError, exit 1",
|
||||
badReserve.returnCode == 1 && badReserve.err.contains("crafter-review: ValueError: could not convert string to float: 'x'"), badReserve.err);
|
||||
|
||||
// The commit-truth lens (2026-10-31), alone: a config that turns off
|
||||
// every other lens (through --config: the tracked config file must
|
||||
// stay as committed, or the tree is dirty) runs Gate 4 - its three text
|
||||
// blocks and its two slot passes (the stand-in answers a SLOT PASS
|
||||
// too) - push-ready, with the code units, the per-commit slots and
|
||||
// the functional lens's slot listed as not reviewed.
|
||||
{
|
||||
Write(base / "facts.json", R"({"lenses": {"repo-better-solution": {"run": false}, "repo-functional": {"run": false}, "repo-bug-prover": {"run": false}, "repo-security": {"run": false}}})");
|
||||
const std::int64_t before = calls();
|
||||
std::vector<std::string> truthArgs = runArgs({"--no-reuse", "--no-salvage"});
|
||||
truthArgs.insert(truthArgs.begin() + 1, {"--config", (base / "facts.json").string()});
|
||||
const Process::Captured truth = Process::Run(truthArgs, repo, std::chrono::seconds(60), {}, Env);
|
||||
const Json dumpT = LatestDump(gateDir, h8);
|
||||
const Json& processesT = dumpT.is_object() ? dumpT.at("processes") : Json::array();
|
||||
std::set<std::string> labels;
|
||||
for (const Json& p : processesT) labels.insert(Text(p.at("label")));
|
||||
const Json notReviewedT = dumpT.is_object() ? dumpT.at("coverage").at("notReviewed") : Json::array();
|
||||
auto notReviewed = [&](std::string_view id) { return std::ranges::any_of(notReviewedT, [&](const Json& x) { return x == id; }); };
|
||||
Check("the commit-truth lens alone: Gate 4 runs its three text blocks and its two slot passes (5 calls) - SEND-READY, exit 0; the code units, the per-commit slots and whole:behaviour are not reviewed",
|
||||
truth.returnCode == 0 && calls() - before == 5 && dumpT.is_object() && dumpT.at("overall") == "SEND-READY" && dumpT.at("stoppedAt").is_null()
|
||||
&& labels == std::set<std::string>{"repo-facts:block:p1:subject", "repo-facts:block:p1:msg", "repo-facts:block:p1:diffstat", "repo-facts:slot:whole:order", "repo-facts:slot:whole:absent"}
|
||||
&& Count(processesT, "kind", "slot") == 2 && dumpT.at("lensesOff") == Json::array({"repo-better-solution", "repo-functional", "repo-bug-prover", "repo-security"})
|
||||
&& notReviewed("p1:hello.py#1") && notReviewed("p1:absent") && notReviewed("whole:behaviour") && !notReviewed("whole:order") && !notReviewed("p1:msg"),
|
||||
std::format("exit {}, {} call(s), {}", truth.returnCode, calls() - before, dumpT.is_object() ? Text(dumpT.at("overall")) : truth.out + truth.err));
|
||||
}
|
||||
|
||||
// A stop between two gates: the next gate's pool starts aborted and
|
||||
// runs nothing of it.
|
||||
{
|
||||
|
|
|
|||
Loading…
Reference in a new issue