Jorijn van der Graaf
0f803d90bd
Run: gate 0 (the artifact, the trees, the inventory, the run lock, the scratch dir, dispositions, reuse state, resume, reads per round, seats, the board), the stack and migration preflights, the prompts, reuse of whole and block verdicts, salvage, a gate on the pool, checkpoints, the dump, the dry-run report and the unit-state store; slots_plan and app_ready beside it. The CLI parses run's 28 options, types and choices included. A full run takes its lifetime lock beside the board before it joins it. A signal - or the stop file `crafter-review stop` writes, looked at every 2 s - stops the run once gate 0 is through, and the stop stays in force for the gates after; before that it ends the process with its descendants. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
229 lines
15 KiB
C++
229 lines
15 KiB
C++
// SPDX-License-Identifier: AGPL-3.0-only
|
|
// SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts®
|
|
|
|
// The OpenCode harness, against the goldens (tests/Goldens/fixture/
|
|
// pipeline.json, under `opencode`): the tool source, the permission rules,
|
|
// the agent config, the commands, the environment, the home, the event
|
|
// parsing, the classification, and a warm-up against a fake opencode. The
|
|
// test binary is its own supervisor (`__supervise`), as the driver is.
|
|
|
|
#include <signal.h>
|
|
#include <sys/stat.h>
|
|
import std;
|
|
import Crafter.Review;
|
|
namespace fs = std::filesystem;
|
|
using namespace Crafter::Review;
|
|
|
|
namespace {
|
|
std::int32_t Failures = 0;
|
|
|
|
void Check(std::string_view name, bool cond, std::string_view detail = {}) {
|
|
std::println("{}{}{}", cond ? " ok " : " FAIL ", name, !cond && !detail.empty() ? std::format(" ({})", detail) : std::string());
|
|
if (!cond) ++Failures;
|
|
}
|
|
|
|
void Write(const fs::path& path, std::string_view text) {
|
|
fs::create_directories(path.parent_path());
|
|
std::ofstream(path, std::ios::binary) << text;
|
|
}
|
|
|
|
std::string Read(const fs::path& path) {
|
|
return Platform::ReadFile(path).value_or("");
|
|
}
|
|
|
|
std::vector<std::string> Strings(const Json& array) {
|
|
std::vector<std::string> out;
|
|
for (const Json& item : array) out.push_back(item.get<std::string>());
|
|
return out;
|
|
}
|
|
|
|
std::vector<std::string> Names(const fs::path& dir) {
|
|
std::vector<std::string> names;
|
|
for (const fs::directory_entry& entry : fs::directory_iterator(dir)) names.push_back(entry.path().filename().string());
|
|
std::ranges::sort(names);
|
|
return names;
|
|
}
|
|
|
|
// What the generator's temp dir is in the goldens.
|
|
std::string Tidy(std::string text, const fs::path& td) {
|
|
return Dyn::Replace(text, td.string(), "<td>");
|
|
}
|
|
|
|
std::int32_t Checks() {
|
|
std::println("opencode harness + OpenRouter seats: loader, environments, permissions, event parsing, the tool schema");
|
|
Platform::SetMode(Platform::Mode::Native);
|
|
// The directory modes below are the generator's, made under 022.
|
|
umask(022);
|
|
const fs::path td = fs::temp_directory_path() / std::format("crafter-review-opencode-{}", Platform::PublishedPid());
|
|
fs::remove_all(td);
|
|
fs::create_directories(td);
|
|
|
|
const fs::path envFile = td / ".env";
|
|
Write(envFile, "CLAUDE_CODE_OAUTH_TOKEN1=tok\nOPENROUTER_KEY=sk-or-one\n");
|
|
auto ids = [](const std::vector<Json>& seats) {
|
|
std::vector<std::string> out;
|
|
for (const Json& s : seats) out.push_back(s.at("id").get<std::string>());
|
|
return out;
|
|
};
|
|
Check("OPENROUTER_KEY is a seat only on request",
|
|
ids(SeatPool::LoadSeats({}, envFile, "")) == std::vector<std::string>{"1"} && ids(SeatPool::LoadSeats({}, envFile, "openrouter")) == std::vector<std::string>{"or1"});
|
|
try {
|
|
SeatPool::LoadSeats({}, envFile, "or1,1");
|
|
Check("an OpenRouter seat never shares a pool with Anthropic seats", false);
|
|
} catch (const std::invalid_argument& e) {
|
|
Check("an OpenRouter seat never shares a pool with Anthropic seats", std::string_view(e.what()).contains("cannot share"), e.what());
|
|
}
|
|
const std::vector<std::string> allow = {"Read", "Glob", "Grep", "Bash(cat:*)", "Bash(git -C . diff:*)", "Bash(python3 -:*)"};
|
|
const Json p = OpenCode::Permissions(allow);
|
|
Check("allowlist -> permission rules", p.at("read") == "allow" && p.at("StructuredOutput") == "allow" && p.at("bash").at("*") == "deny" && p.at("bash").at("git -C . diff*") == "allow");
|
|
const std::vector<Json> ev = {
|
|
{{"type", "step_start"}, {"sessionID", "ses_1"}, {"part", Json::object()}},
|
|
{{"type", "tool_use"},
|
|
{"part",
|
|
{{"type", "tool"},
|
|
{"tool", "StructuredOutput"},
|
|
{"state", {{"status", "completed"}, {"input", {{"verdict", "SEND-READY"}, {"findings", Json::array()}, {"notes", Json::array()}, {"coverage", {{{"item", "u"}, {"verdict", "clear"}, {"basis", "b"}}}}}}}}}}},
|
|
{{"type", "step_finish"}, {"part", {{"type", "step-finish"}, {"reason", "stop"}, {"tokens", {{"input", 20}, {"output", 4}, {"reasoning", 1}, {"cache", {{"read", 150}, {"write", 0}}}}}, {"cost", 0.0005}}}},
|
|
};
|
|
std::vector<std::string> lines;
|
|
for (const Json& e : ev) lines.push_back(DumpSpaced(e));
|
|
const Json ps = OpenCode::ParseEvents(Dyn::Join(lines, "\n"));
|
|
Check("events -> the structured output, usage, classify ok",
|
|
ps.at("structured").at("coverage").at(0).at("item") == "u" && ps.at("usage").at("output") == 5 && OpenCode::Classify(ps, 0, "") == std::pair<std::string, std::string>{"ok", ""});
|
|
const std::string src = OpenCode::ToolSource();
|
|
Check("the tool schema carries the note kinds, the coverage vocabulary and the verdicts of the schema; no persona field",
|
|
std::ranges::all_of(std::array<std::string_view, 5>{"reserve-answer", "unverified", "FIX FIRST", "refutation-verified", "n/a"},
|
|
[&](std::string_view k) { return src.contains(DumpSpaced(Json(std::string(k)))); })
|
|
&& !src.contains("notMine") && !src.contains("wouldComment"));
|
|
const fs::path repo = td / "repo";
|
|
fs::create_directories(repo / "src");
|
|
fs::create_directories(repo / ".git");
|
|
fs::create_directories(repo / "node_modules");
|
|
Write(repo / "package.json", "{}");
|
|
const std::vector<std::string> one = {"openrouter/x/y"};
|
|
const fs::path home = OpenCode::PrepareHome(td / "sc", repo, one);
|
|
Check("prepare_home: the view links every top-level entry but .git and node_modules",
|
|
Names(home / "view") == std::vector<std::string>{"package.json", "src"} && fs::is_symlink(fs::symlink_status(home / "view" / "src")));
|
|
|
|
// ---- against the goldens ---------------------------------------------------------------
|
|
const Json g = Dyn::LoadJsonFile(fs::current_path() / "tests" / "Goldens" / "fixture" / "pipeline.json").at("opencode");
|
|
Check("goldens: the tool source, byte for byte", src == g.at("toolSource").get<std::string>());
|
|
Check("goldens: the provider routing", DumpFile(OpenCode::Routing()) == DumpFile(g.at("routing")));
|
|
std::vector<std::string> missed;
|
|
for (const Json& entry : g.at("modelId")) {
|
|
const auto [prov, id] = OpenCode::ModelId(entry.at(0).get<std::string>());
|
|
if (Json::array({prov, id}) != entry.at(1)) missed.push_back(entry.at(0).get<std::string>());
|
|
}
|
|
Check("goldens: model ids", missed.empty(), Dyn::Join(missed, ", "));
|
|
missed.clear();
|
|
std::size_t index = 0;
|
|
for (const Json& entry : g.at("permissions")) {
|
|
const std::vector<std::string> list = Strings(entry.at(0));
|
|
if (DumpFile(OpenCode::Permissions(list)) != DumpFile(entry.at(1))) missed.push_back(DumpCompact(entry.at(0)));
|
|
const std::map<std::string, std::string> child = OpenCode::ChildEnv({{"PATH", "/bin"}}, "/abs/system é.md", "/abs/so.json", list, td / "data" / std::to_string(index));
|
|
if (child.at("OPENCODE_CONFIG_CONTENT") != g.at("agentConfig").at(index).at(1).get<std::string>()) missed.push_back(std::format("agent config {}", std::to_string(index)));
|
|
if (child.at("REVIEW_GATES_STRUCTURED_OUT") != "/abs/so.json" || child.at("XDG_DATA_HOME") != (td / "data" / std::to_string(index)).string()) missed.push_back(std::format("child env {}", std::to_string(index)));
|
|
++index;
|
|
}
|
|
Check("goldens: permission rules and the inline agent config, key for key", missed.empty(), Dyn::Join(missed, "; "));
|
|
missed.clear();
|
|
for (const Json& entry : g.at("cmd")) {
|
|
const Json& a = entry.at(0);
|
|
const std::vector<std::string> got = OpenCode::Cmd(a.at(0).get<std::string>(), a.at(1).get<std::string>(), a.at(2).get<std::string>(), a.at(3).get<std::string>(),
|
|
a.at(4).is_null() ? std::string() : a.at(4).get<std::string>(), a.at(5).get<std::string>());
|
|
if (got != Strings(entry.at(1))) missed.push_back(DumpCompact(a));
|
|
}
|
|
Check("goldens: the opencode command lines", missed.empty(), Dyn::Join(missed, "; "));
|
|
std::map<std::string, std::string> baseIn;
|
|
for (auto it = g.at("baseEnv").at(0).begin(); it != g.at("baseEnv").at(0).end(); ++it) baseIn[it.key()] = it.value().get<std::string>();
|
|
std::map<std::string, std::string> baseWant;
|
|
for (auto it = g.at("baseEnv").at(1).begin(); it != g.at("baseEnv").at(1).end(); ++it) baseWant[it.key()] = it.value().get<std::string>();
|
|
Check("goldens: the shared environment - its own XDG home and key, no Anthropic or OpenCode variable", OpenCode::BaseEnv("/h", "sk-or-key", baseIn) == baseWant);
|
|
|
|
// The generator's temp dir, which the goldens call <td>.
|
|
const fs::path gd = td / "g";
|
|
const fs::path repo2 = gd / "repo";
|
|
for (const std::string_view d : {".git", "node_modules", "src", "b", "a"}) fs::create_directories(repo2 / d);
|
|
for (const std::string_view name : {"package.json", ".env", "Z"}) Write(repo2 / name, "{}");
|
|
const Json& h = g.at("home");
|
|
const fs::path home2 = OpenCode::PrepareHome(gd / "sc", repo2, Strings(h.at("models")));
|
|
const fs::path config = home2 / "xdg" / "opencode" / "opencode.json";
|
|
const std::string firstConfig = Read(config);
|
|
OpenCode::PrepareHome(gd / "sc", repo2, one);
|
|
Check("goldens: prepare_home's config, byte for byte, and rewritten by the next call", firstConfig == h.at("firstConfig").get<std::string>() && Read(config) == h.at("config").get<std::string>());
|
|
std::vector<std::string> view;
|
|
for (const std::string& name : Names(home2 / "view")) view.push_back(std::format("{} -> {}", name, Tidy(fs::read_symlink(home2 / "view" / name).string(), gd)));
|
|
std::vector<std::string> viewWant;
|
|
for (const Json& entry : h.at("view")) viewWant.push_back(std::format("{} -> {}", entry.at(0).get<std::string>(), entry.at(1).get<std::string>()));
|
|
Check("goldens: the view's links, sorted, dot files included, .git and node_modules left out; idempotent", view == viewWant && Tidy(home2.string(), gd) == h.at("home").get<std::string>(),
|
|
Dyn::Join(view, "; "));
|
|
missed.clear();
|
|
for (auto it = h.at("modes").begin(); it != h.at("modes").end(); ++it) {
|
|
const auto mode = static_cast<std::uint32_t>(fs::status(home2 / it.key()).permissions()) & 0777;
|
|
if (std::format("0o{:o}", mode) != it.value().get<std::string>()) missed.push_back(std::format("{} 0o{:o}", it.key(), mode));
|
|
}
|
|
Check("goldens: os.makedirs' modes - the last directory 0700, its parents as mkdir makes them", missed.empty() && Read(home2 / "xdg" / "opencode" / "tools" / "StructuredOutput.ts") == src,
|
|
Dyn::Join(missed, ", "));
|
|
|
|
missed.clear();
|
|
for (const Json& entry : g.at("events")) {
|
|
if (DumpFile(OpenCode::ParseEvents(entry.at(0).get<std::string>())) != DumpFile(entry.at(1))) missed.push_back(Dyn::Repr(Dyn::Slice(entry.at(0).get<std::string>(), 0, 60)));
|
|
}
|
|
Check("goldens: the event stream - steps, tokens, tool calls and errors, the session, the first turn", missed.empty(), Dyn::Join(missed, "; "));
|
|
missed.clear();
|
|
for (const Json& entry : g.at("classify")) {
|
|
const Json& c = entry.at(0);
|
|
const auto [status, reason] = OpenCode::Classify(c.at(0), c.at(1).get<std::int64_t>(), c.at(2).get<std::string>(), c.at(3).get<bool>());
|
|
if (Json::array({status, reason}) != entry.at(1)) missed.push_back(std::format("{} -> {} {}", Dyn::Repr(Dyn::Slice(c.at(2).get<std::string>(), 0, 40)), status, Dyn::Repr(Dyn::Slice(reason, 0, 60))));
|
|
}
|
|
Check("goldens: classify - the time budget, then credits, auth and limits as the regexes read them (re.I, \\b), then why nothing came back", missed.empty(), Dyn::Join(missed, "; "));
|
|
|
|
const Json& w = g.at("warm");
|
|
const fs::path fake = td / "opencode";
|
|
Write(fake, g.at("fake").get<std::string>());
|
|
fs::permissions(fake, fs::perms::owner_all | fs::perms::group_read | fs::perms::group_exec | fs::perms::others_read | fs::perms::others_exec);
|
|
const std::map<std::string, std::string> env = OpenCode::BaseEnv(home2, "sk-or", std::map<std::string, std::string>{{"PATH", "/usr/bin:/bin"}});
|
|
std::vector<std::string> log;
|
|
const auto warmed = OpenCode::Warm(fake.string(), home2, env, "openrouter/x/y", [&](const std::string& line) { log.push_back(line); });
|
|
const std::string args = Tidy(Read(home2 / "data" / "warm" / "args"), gd);
|
|
Check("goldens: a warm-up that fails - its log line, the tail of its stderr, the command it ran",
|
|
warmed == std::pair<bool, std::string>{w.at("result").at(0).get<bool>(), w.at("result").at(1).get<std::string>()} && log == Strings(w.at("log")) && args == w.at("args").get<std::string>(),
|
|
std::format("{} | {}", Dyn::Join(log, " | "), Dyn::Repr(args)));
|
|
const fs::path slow = td / "slow-opencode";
|
|
Write(slow, "#!/bin/sh\necho $$ > \"$XDG_DATA_HOME/pid\"\nexec sleep 30\n");
|
|
fs::permissions(slow, fs::perms::owner_all);
|
|
log.clear();
|
|
const auto timed = OpenCode::Warm(slow.string(), home2, env, "openrouter/x/y", [&](const std::string& line) { log.push_back(line); }, 1);
|
|
const std::optional<std::int64_t> sleeper = Dyn::ParseInt(Dyn::Strip(Read(home2 / "data" / "warm" / "pid")));
|
|
Check("goldens: a warm-up out of time says so and logs nothing",
|
|
timed == std::pair<bool, std::string>{w.at("timedOut").at(0).get<bool>(), w.at("timedOut").at(1).get<std::string>()} && log.empty() && w.at("timedLog").empty());
|
|
Check("...and leaves no process behind: the supervisor takes it down", sleeper && !Platform::ProcStart(*sleeper));
|
|
|
|
// ---- beyond the goldens -----------------------------------------------------------------
|
|
const fs::path so = td / "so.json";
|
|
Write(so, R"({"verdict": "SEND-READY", "findings": [], "notes": [], "coverage": []})");
|
|
const fs::path bad = td / "bad.json";
|
|
Write(bad, R"({"verdict": "SEND-READY", "findings": "none"})");
|
|
const fs::path garbage = td / "garbage.json";
|
|
Write(garbage, "{");
|
|
const std::string noTool = DumpSpaced({{"type", "step_finish"}, {"part", {{"reason", "stop"}}}});
|
|
Check("the structured-output file stands in when the stream carries no call: one with a findings list, nothing else",
|
|
OpenCode::ParseEvents(noTool, so).at("structured").at("verdict") == "SEND-READY" && OpenCode::ParseEvents(noTool, bad).at("structured").is_null()
|
|
&& OpenCode::ParseEvents(noTool, garbage).at("structured").is_null() && OpenCode::ParseEvents(noTool, td / "missing.json").at("structured").is_null()
|
|
&& OpenCode::ParseEvents(noTool).at("structured").is_null());
|
|
|
|
fs::remove_all(td);
|
|
return Failures == 0 ? 0 : 1;
|
|
}
|
|
}
|
|
|
|
int main(int argc, char** argv) {
|
|
const std::vector<std::string_view> args(argv, argv + argc);
|
|
if (args.size() > 1 && args[1] == Process::SuperviseVerb) return Process::SuperviseMain(std::span(args).subspan(2));
|
|
try {
|
|
return Checks();
|
|
} catch (const std::exception& error) {
|
|
std::println(std::cerr, "opencode: {}", error.what());
|
|
return 1;
|
|
}
|
|
}
|