From 952eb13f5c6538365eac44d1488551846dbec3e4 Mon Sep 17 00:00:00 2001 From: Jorijn van der Graaf Date: Sat, 31 Oct 2026 09:59:12 +0100 Subject: [PATCH] test(run): full runs end to end on a claude stand-in The Runs test drives this binary as crafter-review on a scratch repository, with a /bin/sh claude that answers every pass from its system file and prompt: a dry run; a run SIGKILLed in its last gate (every pass goes with it through its supervisor); the next run resuming the first gate from the checkpoint, salvaging the killed run's outputs and buying only the pass that never answered; a run with nothing left to buy; a FIX FIRST with --no-reuse --no-salvage; a stop by the stop file and by SIGTERM, each leaving no process behind; a stale stop file dropped by the run that claims its dir; a stop between gates. Two fixes it led to: - A stop that lands while a gate's pool is set up is no longer lost: the stop is checked under the same lock that publishes the pool. - A run never takes the number of a dump on record. Past a killed run that left no dump, a later run that bought nothing could have its number taken again and its dump written over. Drivers gains the --no-reuse store check. Co-Authored-By: Claude Opus 5.5 --- implementations/Crafter.Review-Run.cpp | 19 +- project.cpp | 3 +- tests/Drivers/main.cpp | 30 +- tests/Runs/main.cpp | 414 +++++++++++++++++++++++++ 4 files changed, 455 insertions(+), 11 deletions(-) create mode 100644 tests/Runs/main.cpp diff --git a/implementations/Crafter.Review-Run.cpp b/implementations/Crafter.Review-Run.cpp index d9ca24f..8353bde 100644 --- a/implementations/Crafter.Review-Run.cpp +++ b/implementations/Crafter.Review-Run.cpp @@ -436,9 +436,15 @@ namespace Crafter::Review::Run { } return false; }; + // Nor the number of a dump on record: past a killed run's gap, a later + // run that bought nothing (no outputs in its dir) would otherwise have + // its number taken again and its dump written over. + auto dumped = [&](std::int64_t n) { + return fs::exists(gateDir / std::format("commits-{}-run-{}.json", h8, n)) || fs::exists(gateDir / std::format("commits-{}-run-{}-test.json", h8, n)); + }; while (!a.dryRun) { const fs::path candidate = scratchRoot / std::format("{}-run-{}", h8, runN); - if (!hasOutputs(candidate) && SeatPool::ClaimScratch(candidate)) break; + if (!dumped(runN) && !hasOutputs(candidate) && SeatPool::ClaimScratch(candidate)) break; ++runN; } scratch = scratchRoot / std::format("{}-run-{}{}", h8, runN, a.dryRun ? "-dry" : ""); @@ -1440,11 +1446,6 @@ namespace Crafter::Review::Run { }; const bool warm = !a.dryRun && (!testMode || a.warm); std::unique_ptr admission = Admission(lensFiles); - std::optional before; - { - const std::lock_guard guard(shared); - before = stopped; - } Pool::Scheduler scheduler({.runner = runner.get(), .concurrency = a.concurrency, .log = log, @@ -1455,10 +1456,12 @@ namespace Crafter::Review::Run { .apiConfig = apiCfg, .ledger = ledger.get(), .admission = admission.get(), - .board = board.get(), - .abort = before}); + .board = board.get()}); { + // A stop before this gate, or while its pool was set up, stays in + // force: the pool starts aborted, so the next gate never runs. const std::lock_guard guard(shared); + if (stopped) scheduler.Abort(*stopped); sched = &scheduler; } scheduler.Run(initial, [this](const Json& job, std::int64_t attempt) { return ExecJob(job, attempt); }, onDone); diff --git a/project.cpp b/project.cpp index 16cf4a9..5433f23 100644 --- a/project.cpp +++ b/project.cpp @@ -84,7 +84,7 @@ namespace { // One per directory under tests/, each an ordinary main() that links the // library: exit 0 passes, nonzero fails, 77 skips. - constexpr std::array Tests = { + constexpr std::array Tests = { "Crypto", "Text", "Platform", @@ -108,6 +108,7 @@ namespace { "StoreRules", "StacksMemory", "ReviewFixes", + "Runs", }; // The version the binary reports: CI sets it from the tag, a checkout diff --git a/tests/Drivers/main.cpp b/tests/Drivers/main.cpp index 939a3f6..5d5257b 100644 --- a/tests/Drivers/main.cpp +++ b/tests/Drivers/main.cpp @@ -4,8 +4,7 @@ // Several drivers at once: the board (seats, memory, the cap), one full run // per hash, one driver per scratch dir, the merged unit-state, the locked // ledger, the shared memory record. The checks that run the Scheduler, -// Admission, Stacks, the stack keys and the run's state store come with the -// pool and the driver. +// Admission, Stacks and the stack keys are in the Scheduler test. // // Another driver is a live process of its own (`sleep 300`), so its entries // stand until it dies; two drivers recording into one memory file are two @@ -449,6 +448,33 @@ namespace { const Json old = {{"version", 1}, {"entries", {{"q", {{"unit", "Q"}}}}}, {"personaPass", nullptr}}; State::SaveWhole(sp, old); Check("...sets=None is the whole-file write as before", Dyn::Equal(Dyn::LoadJsonFile(sp), old)); + { + Run::Run r; + r.records = {{{"label", "L:block:X"}, {"lens", "L"}, {"lensFile", "repo-bug-prover.md"}, {"unit", "X"}, {"kind", "block"}, {"status", "ok"}, {"reuseKey", "kx"}}}; + r.hash = "H"; + r.h8 = "H"; + r.round = 1; + r.dumpPath = td / "run-1.json"; + r.repo = td; + r.results = {{{"lens", "L"}, + {"coverage", Json::array({Json {{"item", "X"}, {"verdict", "clear"}, {"basis", ""}}})}, + {"findings", Json::array()}, + {"notes", Json::array()}, + {"parts", Json::array({Json {{"label", "L:block:X"}, {"kind", "block"}, {"unit", "X"}, {"verdict", "SEND-READY"}, {"findings", 0}}})}}}; + const fs::path sp2 = td / "unit-state-nr.json"; + const Json kx = {{"unit", "X"}, {"hash", "H"}, {"run", "run-0.json"}, {"reads", Json::array({Json {{"hash", "H"}, {"round", 0}, {"run", "run-0.json"}}})}}; + State::SaveWhole(sp2, {{"version", 1}, {"entries", {{"kx", kx}, {"ko", {{"unit", "O"}}}}}, {"personaPass", nullptr}}); + r.state = {{"version", 1}, {"entries", Json::object()}, {"personaPass", nullptr}}; + r.statePath = sp2; + r.a.date = "2026-10-31"; + r.a.noReuse = true; + r.log = [](const std::string&) {}; + const Json merged = r.StoreState(); + Check("a --no-reuse run: the store returns the merged file (its own read on top of the file's, another unit's entry kept) and keeps its empty view for the reuse decisions", + Plan::ReadsOf(merged.at("entries").at("kx")) == 2 && merged.at("entries").contains("ko") && r.state.at("entries").empty() + && Plan::ReadsOf(State::LoadState(sp2).at("entries").at("kx")) == 2, + merged.dump()); + } // The ledger: two drivers on one file, every write a locked delta. const fs::path lp = td / "seat-ledger.json"; diff --git a/tests/Runs/main.cpp b/tests/Runs/main.cpp new file mode 100644 index 0000000..e5eb1a8 --- /dev/null +++ b/tests/Runs/main.cpp @@ -0,0 +1,414 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts® + +// Full runs end to end, at no cost: this binary as crafter-review on a +// scratch repository, and a claude stand-in (/bin/sh) that answers every pass +// from what the driver hands it. A run SIGKILLed in its last gate, then the +// next one resuming its first gate from the checkpoint and salvaging the +// outputs of its last; a run with nothing left to buy; a FIX FIRST; a stop by +// `crafter-review stop`'s file and by SIGTERM; a stop between two gates. No +// pass outlives any of them. + +#include +#include +import std; +import Crafter.Review; +namespace fs = std::filesystem; +using namespace Crafter::Review; + +namespace { + std::int32_t Failures = 0; + + void Check(std::string_view name, bool cond, std::string_view detail = {}) { + std::println("{}{}{}", cond ? " ok " : " FAIL ", name, !cond && !detail.empty() ? std::format(" ({})", detail) : std::string()); + if (!cond) ++Failures; + } + + void Write(const fs::path& path, std::string_view text) { + fs::create_directories(path.parent_path()); + std::ofstream(path, std::ios::binary) << text; + } + + std::string Read(const fs::path& path) { + std::ifstream in(path, std::ios::binary); + return {std::istreambuf_iterator(in), std::istreambuf_iterator()}; + } + + std::vector Lines(const fs::path& path) { + const std::string text = Read(path); + return text.empty() ? std::vector() : Dyn::Split(Dyn::RStrip(text, "\n"), "\n"); + } + + // claude for the runs: the lens from its system file, the units from its + // prompt, every unit clear. Each pass adds a line to $FAKE_DIR/calls (its + // pid, its lens, its units). FAKE_FIX: a lens:unit that comes back a + // defect; FAKE_SLEEP: seconds a pass waits before it answers - every pass, + // or the one FAKE_SLEEP_ON names. It speaks first, so the warm-up lets + // the passes behind it go. + constexpr std::string_view FakeClaude = R"sh(#!/bin/sh +set -f +[ "$1" = --version ] && { echo '2.1.314 (Claude Code)'; exit 0; } +sys= +model= +sid= +prev= +for a in "$@"; do + case $prev in + --append-system-prompt-file) sys=$a ;; + --model) model=$a ;; + --session-id) sid=$a ;; + esac + prev=$a +done +lens=$(sed -n 's/^# Lens: //p' "$sys" | head -n 1) +ids=$(sed -n -e 's/^SCOPE - BLOCK PASS, unit "\([^"]*\)".*/\1/p' -e 's/^- \(whole:[a-z]*\) \[.*/\1/p') +echo "$$ $lens" $ids >> "$FAKE_DIR/calls" +echo "{\"type\":\"system\",\"subtype\":\"init\",\"session_id\":\"$sid\",\"model\":\"$model\",\"mcp_servers\":[],\"tools\":[\"Read\"]}" +echo '{"type":"assistant","message":{"role":"assistant","content":[{"type":"text","text":"reading"}]}}' +pause=$FAKE_SLEEP +if [ -n "$FAKE_SLEEP_ON" ]; then + pause= + for id in $ids; do [ "$lens:$id" = "$FAKE_SLEEP_ON" ] && pause=$FAKE_SLEEP; done +fi +if [ -n "$pause" ]; then + sleep "$pause" & + echo "$!" >> "$FAKE_DIR/sleeps" + wait "$!" +fi +verdict=SEND-READY +findings= +coverage= +summary= +for id in $ids; do + v=clear + if [ "$lens:$id" = "$FAKE_FIX" ]; then + v=defect + verdict='FIX FIRST' + findings="{\"finding\":\"planted\",\"evidence\":\"hello.py:2\",\"proposedAction\":\"fix it\",\"items\":[\"$id\"]}" + fi + coverage="$coverage${coverage:+,}{\"item\":\"$id\",\"verdict\":\"$v\",\"basis\":\"read at HEAD\"}" + case $id in whole:*) summary=',"summary":"the approach holds"' ;; esac +done +echo "{\"type\":\"result\",\"subtype\":\"success\",\"is_error\":false,\"result\":\"ok\",\"structured_output\":{\"verdict\":\"$verdict\",\"findings\":[$findings],\"notes\":[],\"coverage\":[$coverage]$summary},\"usage\":{\"input_tokens\":1,\"output_tokens\":1},\"total_cost_usd\":0.01}" +)sh"; + + // The environment of every child: git with no user config (the user's + // signs commits), this checkout as the home, a board of the test's own, + // and none of the review or seat variables of the shell the test runs in. + std::map Env; + + Process::Captured Git(const fs::path& repo, std::vector args) { + args.insert(args.begin(), {"git", "-C", repo.string()}); + return Process::Run(args, std::nullopt, std::chrono::seconds(60), {}, Env); + } + + std::map With(std::map env, const std::map& changes) { + for (const auto& [k, v] : changes) env[k] = v; + return env; + } + + // The stand-in's processes and their sleeps that still run. + std::vector Leftovers(const fs::path& fakeDir) { + std::vector alive; + auto look = [&](const fs::path& list, std::string_view marker) { + for (const std::string& line : Lines(list)) { + const std::string pid = line.substr(0, line.find(' ')); + const std::string cmdline = Read(std::format("/proc/{}/cmdline", pid)); + if (cmdline.contains(marker)) alive.push_back(std::format("{}: {}", pid, Dyn::Replace(cmdline, std::string(1, '\0'), " "))); + } + }; + look(fakeDir / "calls", (fakeDir / "claude").string()); + look(fakeDir / "sleeps", "sleep"); + return alive; + } + + // Leftovers once the supervisors have had their moment. + std::vector LeftoversAfter(const fs::path& fakeDir) { + for (std::int32_t i = 0; i < 100; ++i) { + if (Leftovers(fakeDir).empty()) break; + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + } + return Leftovers(fakeDir); + } + + bool WaitFor(const std::function& ready, std::chrono::seconds limit) { + const auto end = std::chrono::steady_clock::now() + limit; + while (std::chrono::steady_clock::now() < end) { + if (ready()) return true; + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + } + return ready(); + } + + // The dump of the highest run number on this hash. + Json LatestDump(const fs::path& gateDir, std::string_view h8) { + const std::string prefix = std::format("commits-{}-run-", h8); + std::int64_t best = 0; + fs::path found; + for (const fs::directory_entry& entry : fs::directory_iterator(gateDir)) { + const std::string name = entry.path().filename().string(); + if (!name.starts_with(prefix) || !name.ends_with(".json")) continue; + const std::string number = name.substr(prefix.size(), name.size() - prefix.size() - 5); + if (number.empty() || !std::ranges::all_of(number, [](char c) { return c >= '0' && c <= '9'; })) continue; + if (const std::int64_t n = std::stoll(number); n > best) { + best = n; + found = entry.path(); + } + } + return found.empty() ? Json() : Dyn::LoadJsonFile(found); + } + + // The run number in a dump's own name. + std::int64_t RunNumber(const Json& dump) { + if (!dump.is_object()) return 0; + const std::string path = Dyn::Str(dump.at("dumpPath")); + const std::size_t at = path.rfind("-run-"); + return at == std::string::npos ? 0 : std::stoll(path.substr(at + 5)); + } + + std::int64_t Count(const Json& records, std::string_view key, const Json& value) { + return std::ranges::count_if(records, [&](const Json& r) { return Dyn::Get(r, key) == value; }); + } + + std::string Text(const Json& value) { + return value.is_string() ? value.get() : value.dump(); + } + + std::int32_t Checks() { + std::println("runs: a full run on a claude stand-in - killed and salvaged, resumed, nothing left to buy, FIX FIRST, stopped by the stop file, by SIGTERM and between gates"); + Platform::SetMode(Platform::Mode::Native); + const fs::path checkout = fs::current_path(); + SetHome(checkout); + const fs::path base = fs::temp_directory_path() / std::format("crafter-review-runs-{}", Platform::PublishedPid()); + fs::remove_all(base); + const fs::path repo = base / "repo"; + const fs::path fakeDir = base / "fake"; + const fs::path fake = fakeDir / "claude"; + const fs::path scratchRoot = base / "scratch"; + Write(fake, FakeClaude); + fs::permissions(fake, fs::perms::owner_all | fs::perms::group_read | fs::perms::group_exec | fs::perms::others_read | fs::perms::others_exec); + for (const auto& [k, v] : Platform::Environment()) { + const bool review = k.starts_with("REVIEW_GATES_") || k.starts_with("CRAFTER_REVIEW_") || k == "SEATS_ENV_FILE"; + const bool seat = k.starts_with("CLAUDE_CODE_OAUTH_TOKEN") || k.starts_with("ANTHROPIC_API_KEY") || k.starts_with("OPENROUTER_KEY") || k.starts_with("FAKE_"); + if (!review && !seat) Env[k] = v; + } + Env = With(Env, {{"GIT_CONFIG_GLOBAL", "/dev/null"}, {"GIT_CONFIG_SYSTEM", "/dev/null"}, {"GIT_AUTHOR_NAME", "Me"}, {"GIT_AUTHOR_EMAIL", "me@example.com"}, + {"GIT_COMMITTER_NAME", "Me"}, {"GIT_COMMITTER_EMAIL", "me@example.com"}, {"CRAFTER_REVIEW_HOME", checkout.string()}, + {"REVIEW_GATES_BOARD", (base / "board.json").string()}, {"FAKE_DIR", fakeDir.string()}}); + + // A repository with one commit to review, the functional lens off (it + // needs a running application). + fs::create_directories(repo); + Git(repo, {"init", "-q", "-b", "main"}); + Git(repo, {"config", "user.email", "me@example.com"}); + Git(repo, {"config", "user.name", "Me"}); + Write(repo / ".gitignore", ".review-gates/push/\n.review-gates/trees/\n"); + Write(repo / ".review-gates" / "config.json", R"({"lenses": {"repo-functional": {"run": false}}})"); + Write(repo / "README.md", "a greeter\n"); + Git(repo, {"add", "-A"}); + Git(repo, {"commit", "-q", "-m", "base"}); + const std::string baseSha = Dyn::Strip(Git(repo, {"rev-parse", "HEAD"}).out); + Write(repo / "hello.py", "def hello(name):\n return f'hello {name}'\n"); + Git(repo, {"add", "hello.py"}); + Git(repo, {"commit", "-q", "-m", "add hello\n\nThe greeting the README promises."}); + const Process::Captured frozen = Process::Run({Platform::SelfExe().string(), "freeze", "--slug", "x", "--base", baseSha}, repo, std::chrono::seconds(60), {}, Env); + const fs::path artifact = repo / ".review-gates" / "push" / "x"; + const fs::path gateDir = artifact / "gate-state"; + Json freezeJson; + try { + freezeJson = Dyn::LoadJsonFile(artifact / "freeze.json"); + } catch (const std::exception&) { + Check("freeze: the scratch range freezes", false, frozen.out + frozen.err); + return 1; + } + const std::string hash = freezeJson.at("hash").get(); + const std::string h8 = hash.substr(0, 8); + auto runArgs = [&](std::vector extra) { + std::vector args = {Platform::SelfExe().string(), "run", "--artifact", ".review-gates/push/x", "--hash", hash, "--base", baseSha, "--trees", ".", + "--dispositions", ".review-gates/push/x/gate-state/dispositions.json", "--date", Text(freezeJson.at("date")), + "--claude-bin", fake.string(), "--scratch", scratchRoot.string(), "--seats", "local", "--no-probe", "--reads", "1"}; + std::ranges::move(extra, std::back_inserter(args)); + return args; + }; + auto run = [&](std::vector extra, const std::map& env = {}) { + return Process::Run(runArgs(std::move(extra)), repo, std::chrono::seconds(60), {}, With(Env, env)); + }; + auto start = [&](std::vector extra, const std::map& env, std::string_view name) { + return Process::Start({.argv = runArgs(std::move(extra)), + .environment = With(Env, env), + .cwd = repo, + .input = {.kind = Process::Stream::Kind::Null}, + .output = {.kind = Process::Stream::Kind::File, .file = base / std::format("{}.out", name)}, + .error = {.kind = Process::Stream::Kind::File, .file = base / std::format("{}.err", name)}}); + }; + auto calls = [&] { return static_cast(Lines(fakeDir / "calls").size()); }; + + // The plan, from a dry run: what every full run below buys. + const Process::Captured dry = run({"--dry-run"}); + Json plan; + try { + plan = Dyn::LoadJsonFile(scratchRoot / std::format("{}-run-1-dry", h8) / "dry-run.json"); + } catch (const std::exception&) { + } + const Json& planned = plan.is_object() ? plan.at("processes") : Json::array(); + const auto all = static_cast(planned.size()); + const std::int64_t gate1 = Count(planned, "lens", "repo-better-solution"); + std::string securityUnit; + std::string proverUnit; + for (const Json& p : planned) { + if (p.at("kind") != "block") continue; + if (securityUnit.empty() && p.at("lens") == "repo-security") securityUnit = Text(p.at("unit")); + if (proverUnit.empty() && p.at("lens") == "repo-bug-prover") proverUnit = Text(p.at("unit")); + } + Check("dry run: the range plans the approach lens's whole pass and blocks, then the bug-prover's and the security lens's blocks - and no pass reaches claude", + dry.returnCode == 2 && all > gate1 && gate1 > 1 && !securityUnit.empty() && !proverUnit.empty() && calls() == 0 && Count(planned, "lens", "repo-functional") == 0, + dry.out.substr(dry.out.size() > 2000 ? dry.out.size() - 2000 : 0) + dry.err); + + // Killed in its last gate: one security pass never answers, and the + // driver is SIGKILLed once every other pass has. + const fs::path scratchA = scratchRoot / std::format("{}-run-1", h8); + const Process::Child a = start({}, {{"FAKE_SLEEP", "571"}, {"FAKE_SLEEP_ON", std::format("repo-security:{}", securityUnit)}}, "a"); + auto answered = [&] { + std::int64_t n = 0; + std::error_code error; + for (fs::directory_iterator it(scratchA / "out", error), end; !error && it != end; it.increment(error)) { + if (it->path().extension() == ".jsonl" && Read(it->path()).contains(R"("type":"result")")) ++n; + } + return n; + }; + const bool settled = WaitFor([&] { return calls() == all && answered() == all - 1; }, std::chrono::seconds(30)); + Process::Signal(a.pid, SIGKILL); + const std::optional aCode = Process::Wait(a.pid, std::chrono::seconds(10)); + const std::vector aLeft = LeftoversAfter(fakeDir); + Json resumeA; + try { + resumeA = Dyn::LoadJsonFile(gateDir / std::format("resume-{}.json", h8)); + } catch (const std::exception&) { + } + const Json dumpA = LatestDump(gateDir, h8); + Check("SIGKILL: a driver killed in its last gate takes its passes with it (each pass's supervisor ends its tree); its first gate is on record in the checkpoint and the partial dump", + settled && aCode == -9 && aLeft.empty() && dumpA.is_object() && Text(dumpA.at("overall")).starts_with("IN PROGRESS - after Gate 1 — Approach") && resumeA.is_object() + && resumeA.at("results").size() == 1 && resumeA.at("results").at(0).at("lens") == "repo-better-solution", + std::format("settled {}, exit {}, left {}, {} of {} answered, {}", settled, aCode.value_or(0), Dyn::Join(aLeft, "; "), answered(), all, resumeA.dump().substr(0, 300))); + + // The next run: the first gate resumed, the outputs of the killed run + // salvaged, the one pass that never answered bought again. + std::int64_t before = calls(); + const Process::Captured b = run({}); + const Json dumpB = LatestDump(gateDir, h8); + const Json& processesB = dumpB.is_object() ? dumpB.at("processes") : Json::array(); + Check("the next run resumes the first gate from the checkpoint, salvages every output the killed run finished and buys only the pass that never answered - SEND-READY, exit 0", + b.returnCode == 0 && calls() - before == 1 && dumpB.is_object() && dumpB.at("overall") == "SEND-READY" && dumpB.at("resumedLenses") == Json::array({"repo-better-solution"}) + && std::ranges::count_if(processesB, [](const Json& r) { return Dyn::Truthy(Dyn::Get(r, "salvaged")); }) == all - gate1 - 1 + && Count(processesB, "status", "ok") == all - gate1, + std::format("exit {}, {} call(s), {}", b.returnCode, calls() - before, dumpB.is_object() ? Text(dumpB.at("overall")) : b.out + b.err)); + const Json stored = State::LoadState(gateDir / "unit-state.json").at("entries"); + Check("...and stores the clean block results of the range for reuse", static_cast(stored.size()) >= all - gate1, std::to_string(stored.size())); + + // Nothing left to buy. + before = calls(); + const Process::Captured c = run({}); + const Json dumpC = LatestDump(gateDir, h8); + Check("a run on the same hash has nothing left to buy: every lens resumes, claude is never called, SEND-READY", + c.returnCode == 0 && calls() == before && dumpC.is_object() && dumpC.at("overall") == "SEND-READY" && dumpC.at("resumedLenses").size() == 3, + std::format("exit {}, {} call(s)", c.returnCode, calls() - before)); + + // A defect, on a run that reuses nothing; the scratch dir it claims + // holds a stop file from before, which is not this run's. + const std::string nextRun = std::format("{}-run-{}", h8, RunNumber(dumpC) + 1); + const fs::path stale = scratchRoot / nextRun / "stop"; + Write(stale, ""); + before = calls(); + const Process::Captured d = run({"--no-reuse", "--no-salvage"}, + {{"FAKE_FIX", std::format("repo-bug-prover:{}", proverUnit)}, {"FAKE_SLEEP", "3"}, {"FAKE_SLEEP_ON", "repo-better-solution:whole:order"}}); + const Json dumpD = LatestDump(gateDir, h8); + Check("FIX FIRST: a defect in the last gate stops the run there, exit 1 - every pass bought again with --no-reuse --no-salvage", + d.returnCode == 1 && calls() - before == all && dumpD.is_object() && dumpD.at("overall") == "FIX FIRST" && dumpD.at("stoppedAt") == "Gate 3 — Proof", + std::format("exit {}, {} call(s), {}", d.returnCode, calls() - before, dumpD.is_object() ? Text(dumpD.at("stoppedAt")) : d.out + d.err)); + Check("stop: a stop file left in the scratch dir a run claims is dropped - the run ran its course past the watcher's first look", + !fs::exists(stale) && dumpD.is_object() && Dyn::Str(dumpD.at("scratch")).ends_with(nextRun), nextRun); + + // Stopped by `crafter-review stop`'s file, and by SIGTERM, while the + // first gate's whole pass is under way: the pass is killed, no later + // gate starts, nothing is left running. + for (const bool bySignal : {false, true}) { + before = calls(); + const std::string name = bySignal ? "term" : "file"; + const Process::Child child = start({"--no-reuse", "--no-salvage"}, {{"FAKE_SLEEP", "573"}}, name); + const bool running = WaitFor([&] { return calls() > before; }, std::chrono::seconds(30)); + std::string scratch; + for (const std::string& line : Lines(base / std::format("{}.out", name))) { + if (const std::size_t at = line.rfind("; scratch "); at != std::string::npos) scratch = line.substr(at + 10); + } + if (bySignal) { + Process::Signal(child.pid, SIGTERM); + } else if (!scratch.empty()) { + Write(fs::path(scratch) / "stop", ""); + } + const std::optional code = Process::Wait(child.pid, std::chrono::seconds(30)); + if (!code) Process::Signal(child.pid, SIGKILL); + const std::vector left = LeftoversAfter(fakeDir); + const Json dump = LatestDump(gateDir, h8); + const std::string why = bySignal ? "stopped by signal 15 (1 running processes killed)" : "stopped by crafter-review stop (1 running processes killed)"; + Check(bySignal ? "SIGTERM: the run stops as the stop file stops it - the same abort, the same exit, nothing left running" + : "stop file: `crafter-review stop` stops the run - its running pass killed, no later gate started, INCOMPLETE, exit 2, nothing left running", + running && code == 2 && calls() - before == 1 && left.empty() && dump.is_object() && dump.at("stoppedAt") == std::format("Gate 1 — Approach - ABORTED ({})", why) + && Text(dump.at("overall")).starts_with("INCOMPLETE - NOT SEND-READY"), + std::format("running {}, exit {}, {} call(s), left {}, {}", running, code.value_or(-1), calls() - before, Dyn::Join(left, "; "), + dump.is_object() ? Text(dump.at("stoppedAt")) : Read(base / std::format("{}.err", name)))); + } + + // A stop between two gates: the next gate's pool starts aborted and + // runs nothing of it. + { + const fs::path fx = checkout / "tests" / "Goldens" / "fixture" / "commits"; + Run::Run g; + g.log = [](const std::string&) {}; + g.a.artifact = fx.string(); + g.hash = Artifact::ArtifactHash("commits", fx); + g.testMode = true; + g.cfg = Config::Defaults(); + g.environments = Config::Environments(g.cfg); + for (const Plan::Lens& lens : Plan::Lenses) { + const std::string lf(lens.file); + g.charters[lf] = Prompt::Charter(lf); + g.charterSha[lf] = Plan::Sha(g.charters[lf]); + } + const Inventory::Built built = Inventory::Build("commits", fx.string()); + g.units = built.out.at("units"); + g.lines = built.lines; + g.scratch = base / "between"; + fs::create_directories(g.scratch); + g.calibration = std::make_unique(base / "between" / "memory.json", base, std::map {{"none", 210}}); + std::int32_t ran = 0; + g.runPass = [&](const Json& job, const std::string&, const fs::path&, std::string_view, std::int64_t, std::int64_t, bool) { + ++ran; + return std::pair({{"label", job.at("label")}, {"status", "error"}, {"reason", "ran"}}, nullptr); + }; + g.Stop("crafter-review stop"); + const std::vector res = g.RunGate("Gate 3 — Proof", {"repo-bug-prover.md"}); + Check("stop between gates: the next gate's pool starts aborted - none of its passes runs, and the abort is the run's", + ran == 0 && g.abort && g.abort->starts_with("stopped by crafter-review stop") && res.size() == 1 && !res[0].at("coverageMissing").empty(), g.abort.value_or("None")); + } + fs::remove_all(base); + return Failures ? 1 : 0; + } +} + +int main(int argc, char** argv) { + const std::vector args(argv, argv + argc); + // With arguments this binary is crafter-review itself - the driver, its + // supervisor, its helpers - in native mode whatever builds it. + if (args.size() > 1) { + Platform::SystemPaths paths = Platform::Paths(); + paths.flatpakInfo = "/nonexistent/.flatpak-info"; + Platform::SetPaths(paths); + return Crafter::Review::Main(args); + } + try { + return Checks(); + } catch (const std::exception& error) { + std::println(std::cerr, "runs: {}", error.what()); + return 1; + } +}