feat(modes): the kernel modes - a patch series and a mailing-list reply
Beside the push gates, crafter-review runs the pre-send review of a kernel contributor's workbench: --mode series over a format-patch export with its cover letter, --mode reply over a mail draft. The inventory partitions a cover letter, a plain mail and the camera review's shape; ten kernel lenses join the table with their gates (structure, code, artifact truth) and the persona gate - one maintainer-persona instance per --reviewers entry, its whole pass gating its blocks and pruning the units that are not theirs. A series' code units are keyed on the code like the push gates'; every text unit on its text, the references it cites and the anchor when it is dated. The references an artifact cites are prefetched through utilities/lore.py into the gate-state, with a digest in the run layer, and a persona's calibration map is its lore listing. The kernel schema carries the persona vocabulary, the kernel note kinds, the predicted review and the not-mine list; the matrix the persona holes, the thin and the modelled units; gates clean without a persona pass is GATES CLEAN. A reply's gate-state sits beside the draft and is read once; a kernel mode needs --subsystem, defaults the effort to max and the trees to work/linux,work/milos-linux, and excuses no unit from the holes. The kernel lenses and the common text ship verbatim; the table lists them after the push gates' lenses and the config template holds them off, so a repository that never turns them on changes nothing. Prompts, keys and plans were checked byte for byte against the pipeline these modes replace, on its own artifacts: two replies and a series, every system prompt, user prompt, job and reuse key identical.
This commit is contained in:
parent
4fc15fe1c2
commit
339f4b9b40
GPG key ID:
2937E59CDCC1BCFB
46 changed files with 5732 additions and 362 deletions
|
|
@ -173,8 +173,10 @@ namespace Crafter::Review::Run {
|
|||
return {
|
||||
{"config", opt(a.config)},
|
||||
{"mode", a.mode},
|
||||
{"subsystem", opt(a.subsystem)},
|
||||
{"artifact", a.artifact},
|
||||
{"hash", opt(a.hash)},
|
||||
{"reviewers", a.reviewers},
|
||||
{"dispositions", opt(a.dispositions)},
|
||||
{"base", a.base},
|
||||
{"date", a.date},
|
||||
|
|
@ -189,6 +191,7 @@ namespace Crafter::Review::Run {
|
|||
{"reads", a.reads},
|
||||
{"no_reuse", a.noReuse},
|
||||
{"no_salvage", a.noSalvage},
|
||||
{"persona_blocks", a.personaBlocks},
|
||||
{"dry_run", a.dryRun},
|
||||
{"claude_bin", opt(a.claudeBin)},
|
||||
{"models", a.models},
|
||||
|
|
@ -332,6 +335,10 @@ namespace Crafter::Review::Run {
|
|||
environments = Config::Environments(cfg);
|
||||
}
|
||||
|
||||
bool Run::Kernel() const {
|
||||
return Plan::IsKernelMode(a.mode);
|
||||
}
|
||||
|
||||
Run::~Run() {
|
||||
// atexit's board.leave: everything of this driver off the board.
|
||||
if (board) board->Leave();
|
||||
|
|
@ -373,21 +380,29 @@ namespace Crafter::Review::Run {
|
|||
// ---- Gate 0 -------------------------------------------------------------------------
|
||||
void Run::Gate0() {
|
||||
if (!fs::exists(a.artifact)) Die(std::format("artifact {} does not exist", a.artifact));
|
||||
hash = Artifact::ArtifactHash("commits", a.artifact);
|
||||
hash = Artifact::ArtifactHash(a.mode, a.artifact);
|
||||
if (a.hash && !a.hash->empty() && *a.hash != hash) Die(std::format("--hash {} does not match the artifact ({})", Dyn::Slice(*a.hash, 0, 12), hash.substr(0, 12)));
|
||||
h8 = hash.substr(0, 8);
|
||||
const std::optional<std::string> patchHead = Artifact::PatchHead(a.artifact);
|
||||
if (!patchHead || patchHead->empty()) Die("the last patch has no `From <sha>` line - not a git format-patch export (use `freeze`)");
|
||||
head = *patchHead;
|
||||
for (const std::string& t : trees) {
|
||||
if (!Artifact::IsToplevel(t)) Die(std::format("--trees: {} is not the toplevel of a git checkout", t));
|
||||
if (a.mode == "commits") {
|
||||
// The lenses read the live checkout named first in --trees: it
|
||||
// must be the range HEAD the artifact was cut from, and clean -
|
||||
// anything else is evidence the review never saw (re-checked
|
||||
// before every launch).
|
||||
const std::optional<std::string> patchHead = Artifact::PatchHead(a.artifact);
|
||||
if (!patchHead || patchHead->empty()) Die("the last patch has no `From <sha>` line - not a git format-patch export (use `freeze`)");
|
||||
head = *patchHead;
|
||||
for (const std::string& t : trees) {
|
||||
if (!Artifact::IsToplevel(t)) Die(std::format("--trees: {} is not the toplevel of a git checkout", t));
|
||||
}
|
||||
if (const std::optional<std::string> drift = treeDrift()) Die(std::format("{} - re-freeze, or check out the frozen worktree (freeze --worktree)", *drift));
|
||||
}
|
||||
if (const std::optional<std::string> drift = treeDrift()) Die(std::format("{} - re-freeze, or check out the frozen worktree (freeze --worktree)", *drift));
|
||||
gateDir = Dyn::JoinPath(a.artifact, "gate-state");
|
||||
// A reply's gate-state sits beside the draft; a directory artifact's
|
||||
// inside it.
|
||||
gateDir = Dyn::JoinPath(a.mode == "reply" ? Dyn::DirName(a.artifact) : a.artifact, "gate-state");
|
||||
fs::create_directories(gateDir);
|
||||
Inventory::Built built;
|
||||
try {
|
||||
built = Inventory::Build("commits", a.artifact);
|
||||
built = Inventory::Build(a.mode, a.artifact);
|
||||
} catch (const Inventory::InvariantError& e) {
|
||||
Die(e.what());
|
||||
} catch (const Dyn::Error& e) {
|
||||
|
|
@ -418,16 +433,16 @@ namespace Crafter::Review::Run {
|
|||
Json holder;
|
||||
if (!a.dryRun && !testMode) {
|
||||
try {
|
||||
runLock = std::make_unique<State::RunLock>(gateDir, "commits", h8);
|
||||
runLock = std::make_unique<State::RunLock>(gateDir, a.mode, h8);
|
||||
} catch (const State::RunLocked& e) {
|
||||
const std::string how = Platform::InSandbox() ? std::format("wait for it or stop it (`{} stop <its scratch dir>`)", Invocation()) : std::string("wait for it or kill it");
|
||||
Die(std::format("a full run on this hash is already live in {}: {} - {}; a dead holder releases the lock", Dyn::RelPath(gateDir.string(), repo.string()), e.what(), how));
|
||||
}
|
||||
} else {
|
||||
const std::optional<Json> found = State::LockHolder(gateDir, "commits", h8);
|
||||
const std::optional<Json> found = State::LockHolder(gateDir, a.mode, h8);
|
||||
if (found) holder = *found;
|
||||
}
|
||||
std::int64_t runN = State::NextRunNumber(gateDir, "commits", h8);
|
||||
std::int64_t runN = State::NextRunNumber(gateDir, a.mode, h8);
|
||||
if (Dyn::Truthy(holder) && Dyn::Truthy(Dyn::Get(holder, "run_n"))) runN = std::max(runN, Int(holder.at("run_n")) + 1);
|
||||
const fs::path scratchRoot = ScratchRoot(a.scratch);
|
||||
// A run killed before its first checkpoint left no dump but did leave
|
||||
|
|
@ -445,7 +460,7 @@ namespace Crafter::Review::Run {
|
|||
// run that bought nothing (no outputs in its dir) would otherwise have
|
||||
// its number taken again and its dump written over.
|
||||
auto dumped = [&](std::int64_t n) {
|
||||
return fs::exists(gateDir / std::format("commits-{}-run-{}.json", h8, n)) || fs::exists(gateDir / std::format("commits-{}-run-{}-test.json", h8, n));
|
||||
return fs::exists(gateDir / std::format("{}-{}-run-{}.json", a.mode, h8, n)) || fs::exists(gateDir / std::format("{}-{}-run-{}-test.json", a.mode, h8, n));
|
||||
};
|
||||
while (!a.dryRun) {
|
||||
const fs::path candidate = scratchRoot / std::format("{}-run-{}", h8, runN);
|
||||
|
|
@ -473,7 +488,7 @@ namespace Crafter::Review::Run {
|
|||
const std::lock_guard guard(shared);
|
||||
stopFile = scratch / "stop";
|
||||
}
|
||||
dumpPath = a.dryRun ? scratch / "dry-run.json" : gateDir / std::format("commits-{}-run-{}{}.json", h8, runN, testMode ? "-test" : "");
|
||||
dumpPath = a.dryRun ? scratch / "dry-run.json" : gateDir / std::format("{}-{}-run-{}{}.json", a.mode, h8, runN, testMode ? "-test" : "");
|
||||
if (runLock) {
|
||||
runLock->WriteInfo({{"pid", Platform::PublishedPid()},
|
||||
{"since", Platform::LocalTime("%Y-%m-%dT%H:%M:%S")},
|
||||
|
|
@ -482,25 +497,33 @@ namespace Crafter::Review::Run {
|
|||
{"run_n", runN}});
|
||||
}
|
||||
logFile = scratch / "log.txt";
|
||||
log(std::format("crafter-review {} prompt v{}: commits {} @ {}", Version(), PromptVersion, Dyn::RelPath(a.artifact, repo.string()), hash));
|
||||
log(std::format("crafter-review {} prompt v{}: {} {} @ {}", Version(), PromptVersion, a.mode, Dyn::RelPath(a.artifact, repo.string()), hash));
|
||||
std::error_code canonical;
|
||||
if (fs::weakly_canonical(reviewed, canonical) != fs::weakly_canonical(repo, canonical)) log(std::format("repository under review: {} (read at {})", reviewed.string(), trees.front()));
|
||||
if (a.mode == "commits" && fs::weakly_canonical(reviewed, canonical) != fs::weakly_canonical(repo, canonical)) log(std::format("repository under review: {} (read at {})", reviewed.string(), trees.front()));
|
||||
log(std::format("config: {}", Dyn::Truthy(Dyn::Get(cfg, "_path")) ? Dyn::Str(cfg.at("_path")) : std::string("defaults (no .review-gates/config.json)")));
|
||||
for (const Json& w : Dyn::Or(Dyn::Get(cfg, "_warnings"), Dyn::EmptyList())) log(std::format("config: warning: {}", Dyn::Str(w)));
|
||||
// The mode's lenses: the others are another mode's and say nothing
|
||||
// about this run.
|
||||
const std::vector<std::string> modeLenses = Plan::ModeLenses(a.mode);
|
||||
std::vector<std::string> lensParts;
|
||||
for (const Plan::Lens& lens : Plan::Lenses) {
|
||||
const std::string lf(lens.file);
|
||||
for (const std::string& lf : modeLenses) {
|
||||
const std::string name = Plan::LensName(lf);
|
||||
lensParts.push_back(std::ranges::contains(off, lf) ? std::format("{} OFF (lenses.{}.run = false)", name, name) : std::format("{} ({})", name, Dyn::Str(Dyn::At(environments, lf))));
|
||||
}
|
||||
log(std::format("lenses: {}", Dyn::Join(lensParts, ", ")));
|
||||
std::vector<std::string> running;
|
||||
for (const Plan::Lens& lens : Plan::Lenses) {
|
||||
if (!std::ranges::contains(off, std::string(lens.file))) running.emplace_back(lens.file);
|
||||
for (const std::string& lf : modeLenses) {
|
||||
if (!std::ranges::contains(off, lf)) running.push_back(lf);
|
||||
}
|
||||
const std::vector<std::string> exclude = Strings(cfg.at("functional").at("exclude"));
|
||||
notReviewed = Plan::NotReviewed(units, lines, running, exclude);
|
||||
log(std::format("inventory: {} units ({}); {} units no lens that runs reads", units.size(), Dyn::RelPath(invPath.string(), repo.string()), notReviewed.size()));
|
||||
if (Kernel()) {
|
||||
// The kernel modes excuse nothing: a unit no lens owns is a hole.
|
||||
notReviewed.clear();
|
||||
log(std::format("inventory: {} units ({})", units.size(), Dyn::RelPath(invPath.string(), repo.string())));
|
||||
} else {
|
||||
notReviewed = Plan::NotReviewed(units, lines, running, exclude);
|
||||
log(std::format("inventory: {} units ({}); {} units no lens that runs reads", units.size(), Dyn::RelPath(invPath.string(), repo.string()), notReviewed.size()));
|
||||
}
|
||||
if (a.units) {
|
||||
std::set<std::string> known;
|
||||
for (const Json& u : units) known.insert(Dyn::Str(u.at("id")));
|
||||
|
|
@ -511,11 +534,49 @@ namespace Crafter::Review::Run {
|
|||
if (!unknown.empty()) Die(std::format("--units names ids not in the inventory: {}", Dyn::Join(unknown, ", ")));
|
||||
}
|
||||
dispEntries = Dispositions::Load(a.dispositions.value_or(""));
|
||||
dispRows = Dispositions::Classify(dispEntries, hash, a.artifact);
|
||||
dispRows = Dispositions::Classify(dispEntries, hash, a.artifact, a.mode);
|
||||
inForce = Dispositions::InForce(dispEntries, dispRows);
|
||||
std::vector<std::string> rowParts;
|
||||
for (const Json& r : dispRows) rowParts.push_back(std::format("{}:{}", Dyn::Str(r.at("id")), Dyn::Str(r.at("status"))));
|
||||
log(std::format("dispositions: {} in force of {} ({})", inForce.size(), dispEntries.size(), rowParts.empty() ? std::string("none") : Dyn::Join(rowParts, ", ")));
|
||||
if (Kernel()) {
|
||||
// The references the artifact cites, prefetched into the
|
||||
// gate-state (a dry run reads an existing index and fetches
|
||||
// nothing), and each reviewer's calibration map.
|
||||
const std::string text = a.mode == "reply" ? Dyn::ReadText(fs::path(a.artifact)) : Dyn::Join(lines, "\n");
|
||||
refUrls = Refs::ParseRefs(text);
|
||||
const fs::path refsDir = gateDir / "refs";
|
||||
const fs::path indexPath = gateDir / std::format("refs-{}.json", h8);
|
||||
if (a.dryRun && fs::exists(indexPath)) {
|
||||
refIndex = Dyn::LoadJsonFile(indexPath);
|
||||
log(std::format("refs: {} entries loaded from the existing index (dry run, no fetch)", refIndex.size()));
|
||||
} else {
|
||||
refIndex = Refs::Prefetch(repo, refUrls, refsDir, indexPath, log);
|
||||
}
|
||||
refDigest = Dyn::Truthy(refIndex) ? Refs::Digest(refIndex) : std::string("REFERENCES prefetched: (none - the artifact cites no lore URLs)");
|
||||
const std::string since = Refs::SinceOf(a.date);
|
||||
corpora.clear();
|
||||
for (const std::string& name : reviewers) {
|
||||
auto [path, corpus] = Refs::PersonaCorpus(repo, name, refsDir, since, log);
|
||||
if (corpus.empty()) {
|
||||
// A private contributor (no lore record): their mail is
|
||||
// indexed in the ledger's live-threads table.
|
||||
const std::string s = a.subsystem.value_or("");
|
||||
corpus = std::format("lore.py search for mail from {} since {}: NO RESULTS - this person has no mail on lore in the last year. "
|
||||
"Their mail is PRIVATE: the Live threads table in journal/{}/ledger.md indexes every message with its capture path "
|
||||
"under journal/{}/captures/, and their submissions are frozen under journal/{}/patch/. Calibrate on those, verbatim.",
|
||||
name, since, s, s, s);
|
||||
}
|
||||
corpora[name] = corpus;
|
||||
}
|
||||
anchorText.clear();
|
||||
for (const Json& u : units) {
|
||||
if (u.at("id") == "anchor") {
|
||||
anchorText = Plan::UnitText(lines, u);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
statePath = gateDir / "unit-state.json";
|
||||
state = a.noReuse ? State::EmptyState() : State::LoadState(statePath);
|
||||
salvageDirs.clear();
|
||||
|
|
@ -551,7 +612,7 @@ namespace Crafter::Review::Run {
|
|||
round = (Dyn::Truthy(r) ? Int(r) : 1) + (newRound ? 1 : 0);
|
||||
}
|
||||
if (!a.dryRun) WriteText(scratch / "round", std::to_string(round));
|
||||
log(std::format("reads: every code unit of the static code lenses needs {} clean reads, one per round; this is round {} on this hash{}", x, round,
|
||||
log(std::format("reads: every code unit of the {}code lenses needs {} clean reads, one per round; this is round {} on this hash{}", Kernel() ? "" : "static ", x, round,
|
||||
newRound || !sameHash ? std::string() : std::string(" (continuing an unfinished run)")));
|
||||
}
|
||||
for (const Plan::Lens& lens : Plan::Lenses) {
|
||||
|
|
@ -559,17 +620,18 @@ namespace Crafter::Review::Run {
|
|||
charters[lf] = Prompt::Charter(lf);
|
||||
charterSha[lf] = Plan::Sha(charters[lf]);
|
||||
}
|
||||
systemText = Prompt::SystemText();
|
||||
systemText = Prompt::SystemText(a.mode);
|
||||
systemPaths.clear();
|
||||
systemByLens.clear();
|
||||
runLayer = Prompt::RunLayer(Dyn::RelPath(a.artifact, repo.string()), a.base, a.date, trees, hash, lines, units, contextPaths, Basename(reviewed));
|
||||
runLayer = Kernel() ? Prompt::RunLayerKernel(a.mode, Dyn::RelPath(a.artifact, repo.string()), a.subsystem.value_or(""), a.base, reviewers, a.date, trees, hash, lines, units, refDigest, contextPaths)
|
||||
: Prompt::RunLayer(Dyn::RelPath(a.artifact, repo.string()), a.base, a.date, trees, hash, lines, units, contextPaths, Basename(reviewed), a.subsystem.value_or(""));
|
||||
// The application under test: the login fixture, the TOTP helper.
|
||||
const Json& fcfg = cfg.at("functional");
|
||||
loginFixture = Config::LoginFixture(cfg, repo);
|
||||
stacks = LoadStacks();
|
||||
const Json first = *stacks->First();
|
||||
appBlock = Prompt::AppBlock(fcfg, loginFixture, Pool::TotpCommand(loginFixture), slotMode);
|
||||
schemas = {{"block", Schema::SchemaJson("block")}, {"whole", Schema::SchemaJson("whole")}};
|
||||
schemas = {{"block", Schema::SchemaJson("block", a.mode)}, {"whole", Schema::SchemaJson("whole", a.mode)}};
|
||||
// Memory: every pass's tree measured, its peak recorded per repository
|
||||
// and environment; the budget per gate.
|
||||
calibration = std::make_unique<Memory::Calibration>(scratchRoot / "memory.json", reviewed, DefaultCosts());
|
||||
|
|
@ -652,7 +714,12 @@ namespace Crafter::Review::Run {
|
|||
}
|
||||
const Json pc = SeatPool::PoolConfig(processEnv, a.effort);
|
||||
board = std::make_unique<SeatPool::Board>(SeatPool::BoardPath(), std::string(), std::nullopt, log);
|
||||
board->Join({{"run", std::format("commits {} @ {}", Basename(Dyn::RStrip(a.artifact, "/")), h8)},
|
||||
std::vector<std::string> runParts = {a.mode};
|
||||
if (a.subsystem && !a.subsystem->empty()) runParts.push_back(*a.subsystem);
|
||||
runParts.push_back(Basename(Dyn::RStrip(a.artifact, "/")));
|
||||
runParts.emplace_back("@");
|
||||
runParts.push_back(h8);
|
||||
board->Join({{"run", Dyn::Join(runParts, " ")},
|
||||
{"dump", Basename(dumpPath)},
|
||||
{"scratch", scratch.string()},
|
||||
{"effort", a.effort},
|
||||
|
|
@ -1061,7 +1128,10 @@ namespace Crafter::Review::Run {
|
|||
}
|
||||
const Plan::Environment& env = *Plan::FindEnvironment(Dyn::Str(Dyn::At(environments, lensFile)));
|
||||
const std::string db = env.db ? Prompt::DbBlock(cfg.at("supabase"), Pool::SqlCommand(), dbRole, env.app, dbChecked, dbBypass) : std::string();
|
||||
const std::string ll = Prompt::LensLayer(lensFile, charters.at(lensFile), Dispositions::Block(inForce, dispRows, lensFile, hash), env.app ? appBlock : std::string(), summary, db);
|
||||
const std::string persona = Dyn::StrOr(Dyn::Get(job, "persona"), "");
|
||||
const auto corpus = corpora.find(persona);
|
||||
const std::string ll = Prompt::LensLayer(lensFile, charters.at(lensFile), Dispositions::Block(inForce, dispRows, lensFile, hash, a.subsystem.value_or("")), env.app ? appBlock : std::string(), summary, db,
|
||||
persona, corpus == corpora.end() ? std::string() : corpus->second);
|
||||
lensLayersBytes[lens] = static_cast<std::int64_t>(Dyn::Length(ll));
|
||||
const std::string sysText = Prompt::SystemPrompt(systemText, runLayer, ll);
|
||||
const std::string sha = Plan::Sha(sysText);
|
||||
|
|
@ -1084,7 +1154,7 @@ namespace Crafter::Review::Run {
|
|||
const Json& login = Dyn::Get(stack, "login");
|
||||
stackText = Prompt::StackText(stack, Pool::TotpCommand(login.is_null() ? std::nullopt : std::optional<std::string>(Dyn::Str(login))));
|
||||
}
|
||||
return {found->second, Prompt::UserPrompt(Prompt::Tail(job, units, lines, summary), job, stackText)};
|
||||
return {found->second, Prompt::UserPrompt(Prompt::Tail(job, units, lines, summary), job, stackText, Kernel())};
|
||||
}
|
||||
|
||||
std::optional<std::string> Run::CacheKey(Json& job) {
|
||||
|
|
@ -1099,7 +1169,16 @@ namespace Crafter::Review::Run {
|
|||
|
||||
// ---- reuse --------------------------------------------------------------------------
|
||||
const Plan::CommitCode& Run::RangeCode() {
|
||||
if (!commitCode) commitCode = Plan::CommitCodeOf(Artifact::PatchTexts(a.artifact));
|
||||
if (!commitCode) {
|
||||
commitCode = Plan::CommitCodeOf(Artifact::PatchTexts(a.artifact), a.mode == "series");
|
||||
if (a.mode == "series" && !a.base.empty()) {
|
||||
// A kernel export carries a base-commit line only when it was
|
||||
// cut with --base, and then in the cover letter: the
|
||||
// submission base the run was given is part of every code
|
||||
// key as well.
|
||||
commitCode->base = commitCode->base.empty() ? a.base : std::format("{},{}", commitCode->base, a.base);
|
||||
}
|
||||
}
|
||||
return *commitCode;
|
||||
}
|
||||
|
||||
|
|
@ -1108,6 +1187,13 @@ namespace Crafter::Review::Run {
|
|||
// a part, re-normalised against today's dispositions, with its summary
|
||||
// for the block passes. nullopt: the whole pass runs.
|
||||
const std::string lensFile = Dyn::Str(job.at("lensFile"));
|
||||
if (a.mode != "commits") {
|
||||
// Only the push gates reuse a whole pass; a kernel whole pass
|
||||
// runs on every hash.
|
||||
job["reuseKey"] = nullptr;
|
||||
job["otherKeys"] = Json::array();
|
||||
return std::nullopt;
|
||||
}
|
||||
std::vector<std::string> ids;
|
||||
for (const Json& d : inForce) ids.push_back(Dyn::Str(d.at("id")));
|
||||
std::map<std::string, std::optional<std::string>> keys;
|
||||
|
|
@ -1164,6 +1250,14 @@ namespace Crafter::Review::Run {
|
|||
for (const Json& d : inForce) ids.push_back(Dyn::Str(d.at("id")));
|
||||
const Json& configured = Dyn::Get(environments, lensFile);
|
||||
const std::string env = environment && !environment->empty() ? *environment : (configured.is_null() ? std::string() : Dyn::Str(configured));
|
||||
if (Kernel()) {
|
||||
// A series' code unit is keyed on the code (the push gates'
|
||||
// rule, 2026-10-06.6); every text unit on its text and what it
|
||||
// cites: the [n] URLs, the anchor when it is dated, the R-units
|
||||
// it names.
|
||||
if (a.mode == "series" && Dyn::Get(job, "unitKind") == "code") return Plan::CodeReuseKey(job, lensFile, text, RangeCode(), ids, charterSha.at(lensFile), PromptVersion, env);
|
||||
return Plan::ReuseKey(job, lensFile, text, Plan::TextDeps(text, units, lines, refUrls, anchorText), ids, charterSha.at(lensFile), PromptVersion);
|
||||
}
|
||||
return Plan::BlockReuseKey(job, lensFile, text, RangeCode(), ids, charterSha.at(lensFile), PromptVersion, env);
|
||||
}
|
||||
|
||||
|
|
@ -1171,6 +1265,7 @@ namespace Crafter::Review::Run {
|
|||
// The block's keys in every other environment: a defect its pass
|
||||
// returns here drops the clean verdicts those hold for the same code -
|
||||
// a finding is not undone by switching the lens's environment.
|
||||
if (Kernel()) return {};
|
||||
std::set<std::string> out;
|
||||
for (const Plan::Environment& e : Plan::Environments) {
|
||||
const std::optional<std::string> k = ReuseKey(job, std::string(e.name));
|
||||
|
|
@ -1295,6 +1390,7 @@ namespace Crafter::Review::Run {
|
|||
auto stamp = [&](Json& rec) {
|
||||
rec["lens"] = job.at("lens");
|
||||
rec["lensFile"] = job.at("lensFile");
|
||||
rec["persona"] = Dyn::Get(job, "persona");
|
||||
rec["kind"] = job.at("kind");
|
||||
rec["unit"] = Dyn::Get(job, "unit");
|
||||
rec["unitSha"] = Dyn::Get(job, "unitSha");
|
||||
|
|
@ -1319,12 +1415,12 @@ namespace Crafter::Review::Run {
|
|||
const std::lock_guard guard(shared);
|
||||
if (sched) sched->Abort(reason);
|
||||
};
|
||||
const std::string now = Artifact::ArtifactHash("commits", a.artifact);
|
||||
const std::string now = Artifact::ArtifactHash(a.mode, a.artifact);
|
||||
if (now != hash) {
|
||||
abortPool(std::format("artifact changed during the run ({} != {})", now.substr(0, 8), h8));
|
||||
return {{{"label", job.at("label")}, {"status", "error"}, {"reason", "artifact changed during the run - aborted"}}, nullptr};
|
||||
}
|
||||
if (const std::optional<std::string> drift = treeDrift()) {
|
||||
if (const std::optional<std::string> drift = a.mode == "commits" ? treeDrift() : std::nullopt) {
|
||||
abortPool(std::format("evidence drift during the run ({})", *drift));
|
||||
return {{{"label", job.at("label")}, {"status", "error"}, {"reason", std::format("{} - aborted", *drift)}}, nullptr};
|
||||
}
|
||||
|
|
@ -1380,23 +1476,33 @@ namespace Crafter::Review::Run {
|
|||
|
||||
// ---- a gate -----------------------------------------------------------------------------
|
||||
std::vector<Json> Run::RunGate(std::string_view title, std::vector<std::string> lensFiles) {
|
||||
// lensFiles -> the merged lens results, one per lens.
|
||||
std::vector<std::pair<std::string, std::string>> specs;
|
||||
for (const std::string& lf : lensFiles) specs.emplace_back(lf, std::string());
|
||||
return RunGate(title, std::move(specs));
|
||||
}
|
||||
|
||||
std::vector<Json> Run::RunGate(std::string_view title, std::vector<std::pair<std::string, std::string>> specs) {
|
||||
// specs -> the merged lens results, one per lens instance.
|
||||
std::vector<std::string> names;
|
||||
for (const std::string& lf : lensFiles) names.push_back(Plan::LensName(lf));
|
||||
for (const auto& [lf, persona] : specs) names.push_back(Plan::LensName(lf, persona));
|
||||
log(std::format("{}: {}", title, Dyn::Join(names, ", ")));
|
||||
std::vector<Json> resumed;
|
||||
std::vector<std::string> live;
|
||||
for (const std::string& lf : lensFiles) {
|
||||
if (std::optional<Json> r = resumable(Plan::LensName(lf), lf, title)) {
|
||||
std::vector<std::pair<std::string, std::string>> live;
|
||||
for (const auto& [lf, persona] : specs) {
|
||||
if (std::optional<Json> r = resumable(Plan::LensName(lf, persona), lf, title)) {
|
||||
resumed.push_back(std::move(*r));
|
||||
} else {
|
||||
live.push_back(lf);
|
||||
live.emplace_back(lf, persona);
|
||||
}
|
||||
}
|
||||
lensFiles = live;
|
||||
specs = live;
|
||||
// Nothing runs: no stack, database or application is needed for a
|
||||
// result already bought.
|
||||
if (lensFiles.empty()) return resumed;
|
||||
if (specs.empty()) return resumed;
|
||||
std::vector<std::string> lensFiles;
|
||||
for (const auto& [lf, persona] : specs) {
|
||||
if (!std::ranges::contains(lensFiles, lf)) lensFiles.push_back(lf);
|
||||
}
|
||||
instrumentPreflight(lensFiles);
|
||||
std::map<std::string, std::vector<Json>> parts;
|
||||
std::map<std::string, std::set<std::string>> missing;
|
||||
|
|
@ -1406,12 +1512,12 @@ namespace Crafter::Review::Run {
|
|||
const std::vector<std::string> exclude = Strings(cfg.at("functional").at("exclude"));
|
||||
std::vector<std::string> unitFilter;
|
||||
if (a.units) unitFilter.assign(a.units->begin(), a.units->end());
|
||||
for (const std::string& lf : lensFiles) {
|
||||
const std::string name = Plan::LensName(lf);
|
||||
for (const auto& [lf, persona] : specs) {
|
||||
const std::string name = Plan::LensName(lf, persona);
|
||||
parts[name];
|
||||
missing[name];
|
||||
planned[name];
|
||||
Plan::LensPlan plan = Plan::PlanLens(lf, units, lines, unitFilter, exclude);
|
||||
Plan::LensPlan plan = Plan::PlanLens(lf, units, lines, unitFilter, exclude, persona);
|
||||
if (!plan.whole.is_null()) {
|
||||
plan.whole["gate"] = title;
|
||||
for (const Json& id : plan.whole.at("expectedIds")) planned[name].insert(Dyn::Str(id));
|
||||
|
|
@ -1457,9 +1563,38 @@ namespace Crafter::Review::Run {
|
|||
}
|
||||
{
|
||||
const std::lock_guard guard(shared);
|
||||
summaries[name] = Dyn::Slice(Dyn::StrOr(Dyn::Get(part, "summary"), ""), 0, 2500);
|
||||
// A persona's digest is its summary, else the review it
|
||||
// would send.
|
||||
std::string digest = Dyn::StrOr(Dyn::Get(part, "summary"), "");
|
||||
if (digest.empty()) digest = Dyn::StrOr(Dyn::Get(part, "predictedReview"), "");
|
||||
summaries[name] = Dyn::Slice(digest, 0, 2500);
|
||||
}
|
||||
return PrepareBlocks(name, pendingBlocks[name], parts);
|
||||
std::vector<Json> blocks = pendingBlocks[name];
|
||||
if (Dyn::Truthy(Dyn::Get(job, "persona")) && a.personaBlocks != "all") {
|
||||
// The units the persona's whole pass marked not-mine get no
|
||||
// block pass from it: each becomes a pruned part with that
|
||||
// verdict.
|
||||
std::vector<std::string> notMine;
|
||||
for (const Json& id : Dyn::Or(Dyn::Get(part, "notMine"), Dyn::EmptyList())) notMine.push_back(Dyn::Str(id));
|
||||
std::ranges::sort(notMine);
|
||||
notMine.erase(std::ranges::unique(notMine).begin(), notMine.end());
|
||||
if (!notMine.empty()) {
|
||||
const double ratio = static_cast<double>(notMine.size()) / static_cast<double>(std::max<std::size_t>(1, blocks.size()));
|
||||
log(std::format("{}: {} units marked not-mine by the whole pass ({:.0f}%){}; those block passes are pruned", name, notMine.size(), ratio * 100,
|
||||
ratio > 0.5 ? " - WARNING above 50%" : ""));
|
||||
for (const std::string& uid : notMine) {
|
||||
parts[name].push_back({{"partLabel", std::format("{}:not-mine:{}", name, uid)}, {"partKind", "pruned"}, {"partUnit", uid}, {"verdict", "SEND-READY"}, {"findings", Json::array()},
|
||||
{"notes", Json::array()}, {"coverage", Json::array({{{"item", uid}, {"verdict", "not-mine"}, {"basis", "whole pass: not this reviewer's tree, file or decision"}}})},
|
||||
{"coverageMissing", Json::array()}, {"reused", nullptr}});
|
||||
}
|
||||
std::vector<Json> kept;
|
||||
for (const Json& b : blocks) {
|
||||
if (!std::ranges::contains(notMine, Dyn::Str(b.at("unit")))) kept.push_back(b);
|
||||
}
|
||||
blocks = kept;
|
||||
}
|
||||
}
|
||||
return PrepareBlocks(name, blocks, parts);
|
||||
};
|
||||
const bool warm = !a.dryRun && (!testMode || a.warm);
|
||||
std::unique_ptr<Pool::Admission> admission = Admission(lensFiles);
|
||||
|
|
@ -1489,8 +1624,8 @@ namespace Crafter::Review::Run {
|
|||
}
|
||||
if (const std::optional<std::string> aborted = scheduler.Aborted()) abort = aborted;
|
||||
std::vector<Json> out;
|
||||
for (const std::string& lf : lensFiles) {
|
||||
const std::string name = Plan::LensName(lf);
|
||||
for (const auto& [lf, persona] : specs) {
|
||||
const std::string name = Plan::LensName(lf, persona);
|
||||
const bool stoppedEarly = std::ranges::any_of(parts[name], [](const Json& p) { return Dyn::Get(p, "partKind") == "skipped"; });
|
||||
std::vector<Json> real;
|
||||
for (const Json& p : parts[name]) {
|
||||
|
|
@ -1699,7 +1834,13 @@ namespace Crafter::Review::Run {
|
|||
for (const std::string& c : contextPaths) {
|
||||
if (!fs::exists(c)) Die(std::format("--context-paths: {} does not exist", c));
|
||||
}
|
||||
reviewers = CommaList(a.reviewers);
|
||||
if (!reviewers.empty() && std::ranges::contains(off, std::string(Plan::PersonaLens))) {
|
||||
Die(std::format("--reviewers names {} but the config turns off the persona lens (lenses.{}.run = false): turn it on, or drop --reviewers", Dyn::Join(reviewers, ", "),
|
||||
Plan::LensName(Plan::PersonaLens)));
|
||||
}
|
||||
Gate0();
|
||||
const std::vector<std::string> modeLenses = Plan::ModeLenses(a.mode);
|
||||
std::vector<std::string> only;
|
||||
if (a.only && !a.only->empty()) {
|
||||
only = CommaList(*a.only);
|
||||
|
|
@ -1708,6 +1849,11 @@ namespace Crafter::Review::Run {
|
|||
if (!Plan::IsLens(lf)) bad.push_back(lf);
|
||||
}
|
||||
if (!bad.empty()) Die(std::format("--only names unknown lens file(s): {}", Dyn::Join(bad, ", ")));
|
||||
std::vector<std::string> foreign;
|
||||
for (const std::string& lf : only) {
|
||||
if (!std::ranges::contains(modeLenses, lf)) foreign.push_back(lf);
|
||||
}
|
||||
if (!foreign.empty()) Die(std::format("--only names lens file(s) the {} mode never runs (nothing would run): {}", a.mode, Dyn::Join(foreign, ", ")));
|
||||
std::vector<std::string> isOff;
|
||||
for (const std::string& lf : only) {
|
||||
if (std::ranges::contains(off, lf)) isOff.push_back(lf);
|
||||
|
|
@ -1724,16 +1870,25 @@ namespace Crafter::Review::Run {
|
|||
if (!Plan::IsLens(lf)) bad.push_back(lf);
|
||||
}
|
||||
if (!bad.empty()) Die(std::format("--without names unknown lens file(s): {}", Dyn::Join(bad, ", ")));
|
||||
std::vector<std::string> foreign;
|
||||
for (const std::string& lf : without) {
|
||||
if (!std::ranges::contains(modeLenses, lf)) foreign.push_back(lf);
|
||||
}
|
||||
if (!foreign.empty()) Die(std::format("--without names lens file(s) the {} mode never runs: {}", a.mode, Dyn::Join(foreign, ", ")));
|
||||
log(std::format("without: {} - left out of this run by owner ruling (recorded in the dump's args); the run stays a full run", Dyn::Join(without, ", ")));
|
||||
}
|
||||
const std::vector<Plan::Gate> gates = Plan::GatesFor(only, without, off);
|
||||
const std::vector<Plan::Gate> gates = Plan::GatesFor(a.mode, only, without, off);
|
||||
if (gates.empty()) {
|
||||
std::vector<std::string> why;
|
||||
if (off.size() == Plan::Lenses.size()) {
|
||||
std::vector<std::string> offHere;
|
||||
for (const std::string& lf : off) {
|
||||
if (std::ranges::contains(modeLenses, lf)) offHere.push_back(lf);
|
||||
}
|
||||
if (offHere.size() == modeLenses.size()) {
|
||||
why.emplace_back("the config turns on no lens (every lens is off until lenses.<name>.run = true turns it on)");
|
||||
} else if (!off.empty()) {
|
||||
} else if (!offHere.empty()) {
|
||||
std::vector<std::string> names;
|
||||
for (const std::string& lf : off) names.push_back(Plan::LensName(lf));
|
||||
for (const std::string& lf : offHere) names.push_back(Plan::LensName(lf));
|
||||
why.push_back(std::format("the config turns off {}", Dyn::Join(names, ", ")));
|
||||
}
|
||||
if (!without.empty()) why.push_back(std::format("--without drops {}", Dyn::Join(without, ", ")));
|
||||
|
|
@ -1771,6 +1926,32 @@ namespace Crafter::Review::Run {
|
|||
break;
|
||||
}
|
||||
}
|
||||
personaRan = false;
|
||||
const std::string personaFile(Plan::PersonaLens);
|
||||
if (!Dyn::Truthy(stoppedAt) && !reviewers.empty() && (only.empty() || std::ranges::contains(only, personaFile)) && !std::ranges::contains(without, personaFile)) {
|
||||
// The persona gate: one instance of the persona lens per
|
||||
// reviewer, after every other gate passed.
|
||||
std::vector<std::pair<std::string, std::string>> specs;
|
||||
for (const std::string& r : reviewers) specs.emplace_back(personaFile, r);
|
||||
const std::vector<Json> res = RunGate(Plan::PersonaGate, specs);
|
||||
std::ranges::copy(res, std::back_inserter(results));
|
||||
Checkpoint(Plan::PersonaGate);
|
||||
std::vector<std::string> notReturned;
|
||||
for (const Json& r : res) {
|
||||
if (Dyn::Truthy(r.at("coverageMissing"))) notReturned.push_back(Dyn::Str(r.at("lens")));
|
||||
}
|
||||
if (abort) {
|
||||
stoppedAt = std::format("{} - ABORTED ({})", Plan::PersonaGate, *abort);
|
||||
incomplete.push_back({{"gate", Plan::PersonaGate}, {"reason", *abort}});
|
||||
} else if (!notReturned.empty()) {
|
||||
stoppedAt = std::format("{} - INCOMPLETE ({} not returned)", Plan::PersonaGate, Dyn::Join(notReturned, ", "));
|
||||
incomplete.push_back({{"gate", Plan::PersonaGate}, {"notReturned", Array(notReturned)}});
|
||||
} else if (std::ranges::any_of(res, [](const Json& r) { return r.at("verdict") == "FIX FIRST"; })) {
|
||||
stoppedAt = Plan::PersonaGate;
|
||||
} else {
|
||||
personaRan = true;
|
||||
}
|
||||
}
|
||||
const bool anyFailed = std::ranges::any_of(results, [](const Json& r) { return r.at("verdict") == "FIX FIRST"; });
|
||||
if (x > 1 && !a.dryRun && !testMode) {
|
||||
// The file after this run's store: a --no-reuse run counts its own
|
||||
|
|
@ -1814,16 +1995,19 @@ namespace Crafter::Review::Run {
|
|||
owedReads += o.at("reads").get<std::int64_t>();
|
||||
}
|
||||
overall = std::format("CONFIRMING - NOT SEND-READY (gates clean; {} code unit(s) owe {} clean read(s) to reach {}: run the same line again)", owedUnits, owedReads, x);
|
||||
} else {
|
||||
// Gates 1-3 clean = push-ready (Gate 4 is the owner).
|
||||
} else if (personaRan || a.mode == "commits") {
|
||||
// commits: no persona gate - the gates clean = push-ready (the
|
||||
// owner is the last gate).
|
||||
overall = "SEND-READY";
|
||||
} else {
|
||||
overall = "GATES CLEAN - NOT SEND-READY (no reviewers passed, Gate 4 did not run)";
|
||||
}
|
||||
return Finish(overall, stoppedAt, incomplete);
|
||||
}
|
||||
|
||||
std::int32_t Run::Finish(std::string overall, const Json& stoppedAt, Json incomplete) {
|
||||
if (broker) broker->Close();
|
||||
if (Artifact::ArtifactHash("commits", a.artifact) != hash) {
|
||||
if (Artifact::ArtifactHash(a.mode, a.artifact) != hash) {
|
||||
overall = "INCOMPLETE - NOT SEND-READY (artifact changed during the run)";
|
||||
incomplete.push_back({{"gate", "end"}, {"reason", "artifact hash changed during the run"}});
|
||||
}
|
||||
|
|
@ -1924,13 +2108,29 @@ namespace Crafter::Review::Run {
|
|||
const Json& db = Dyn::Get(s, "dbUrl");
|
||||
stackList.push_back({{"id", s.at("id")}, {"appUrl", s.at("appUrl")}, {"dbUrl", Dyn::Truthy(db) ? Json(PgWire::Redact(Dyn::Str(db))) : Json()}, {"login", Dyn::Get(s, "login")}});
|
||||
}
|
||||
const std::vector<std::string> modeLenses = Plan::ModeLenses(a.mode);
|
||||
Json lensesOff = Json::array();
|
||||
for (const std::string& lf : off) lensesOff.push_back(Plan::LensName(lf));
|
||||
for (const std::string& lf : off) {
|
||||
if (std::ranges::contains(modeLenses, lf)) lensesOff.push_back(Plan::LensName(lf));
|
||||
}
|
||||
Json environmentNames = Json::object();
|
||||
for (auto it = environments.begin(); it != environments.end(); ++it) environmentNames[Plan::LensName(it.key())] = it.value();
|
||||
for (auto it = environments.begin(); it != environments.end(); ++it) {
|
||||
if (std::ranges::contains(modeLenses, it.key())) environmentNames[Plan::LensName(it.key())] = it.value();
|
||||
}
|
||||
Json refs;
|
||||
if (Kernel()) {
|
||||
std::int64_t fetched = 0;
|
||||
Json failed = Json::array();
|
||||
for (auto it = refIndex.begin(); it != refIndex.end(); ++it) {
|
||||
const Json& status = Dyn::Get(it.value(), "status");
|
||||
if (status == "cached" || status == "fetched") ++fetched;
|
||||
if (status == "failed") failed.push_back(it.key());
|
||||
}
|
||||
refs = {{"index", std::format("gate-state/refs-{}.json", h8)}, {"fetched", fetched}, {"failed", failed}};
|
||||
}
|
||||
Json dump = {
|
||||
{"driver", DriverLine()},
|
||||
{"mode", "commits"},
|
||||
{"mode", a.mode},
|
||||
{"artifact", Dyn::RelPath(a.artifact, repo.string())},
|
||||
{"hash", hash},
|
||||
{"date", a.date},
|
||||
|
|
@ -1950,6 +2150,7 @@ namespace Crafter::Review::Run {
|
|||
{"results", results},
|
||||
{"processes", recs},
|
||||
{"usage", {{"run", State::UsageTotals(recs)}, {"perLens", perLensUsage}, {"prefixHitRate", firsts ? Json(Dyn::Round(static_cast<double>(hits) / static_cast<double>(firsts), 2)) : Json()}}},
|
||||
{"refs", refs},
|
||||
{"estimate", runJobs.empty() ? Json() : Plan::Estimate(runJobs, static_cast<std::int64_t>(Dyn::Length(runLayer)), lensLayersBytes)},
|
||||
{"wallMin", (Platform::Now() - t0) / 60},
|
||||
{"scratch", scratch.string()},
|
||||
|
|
@ -2166,14 +2367,18 @@ namespace Crafter::Review::Run {
|
|||
}
|
||||
sets[key] = {{"lens", rec.at("lensFile")}, {"unit", rec.at("unit")}, {"unitSha", Dyn::Get(rec, "unitSha")}, {"hash", hash}, {"run", Basename(dumpPath)}, {"date", a.date},
|
||||
{"coverage", cov ? *cov : Json()}, {"findings", citing}, {"notes", Normalize::Dedupe(notes)}};
|
||||
if (Dyn::Truthy(Dyn::Get(rec, "persona"))) sets[key]["persona"] = rec.at("persona");
|
||||
++n;
|
||||
}
|
||||
// One clean read per round: the count comes from the FILE at write
|
||||
// time, so a read another driver stored is kept; a second store of the
|
||||
// same run, or the relaunch of an unfinished run, adds nothing.
|
||||
const Json read = {{"hash", h8}, {"round", round}, {"run", Basename(dumpPath)}};
|
||||
// One clean read per round, not for a reply: the count comes from the
|
||||
// FILE at write time, so a read another driver stored is kept; a
|
||||
// second store of the same run, or the relaunch of an unfinished run,
|
||||
// adds nothing. The persona pass is recorded once the persona gate
|
||||
// passed.
|
||||
const Json read = a.mode == "reply" ? Json() : Json({{"hash", h8}, {"round", round}, {"run", Basename(dumpPath)}});
|
||||
const Json persona = personaRan ? Json({{"hash", hash}, {"reviewers", Array(reviewers)}, {"date", a.date}}) : Json();
|
||||
const std::vector<std::string> drops(dirty.begin(), dirty.end());
|
||||
Json merged = State::SaveState(statePath, sets, drops, read);
|
||||
Json merged = State::SaveState(statePath, sets, drops, read, persona);
|
||||
// A --no-reuse run keeps its empty view for the reuse decisions; its
|
||||
// reads are counted from the file.
|
||||
if (!a.noReuse) state = merged;
|
||||
|
|
|
|||
Loading…
Reference in a new issue