feat(opencode): the OpenCode harness, checked against its goldens
The tool source generated from the schema, the home with its view, the shared and per-process environments, the permission rules, the commands, the warm-up under the supervisor, the event stream and the classification, its patterns as hand-written scanners. The goldens record the answer for each, a warm-up against a fake opencode included. A timed-out supervised command now gets SIGTERM, so the supervisor takes its tree down (SIGKILL 10 s later); any other child gets SIGKILL. :Json gains DumpCompact. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
c63c898833
commit
5253fe3aff
GPG key ID:
2937E59CDCC1BCFB
10 changed files with 1946 additions and 7 deletions
File diff suppressed because it is too large
Load diff
229
tests/OpenCode/main.cpp
Normal file
229
tests/OpenCode/main.cpp
Normal file
|
|
@ -0,0 +1,229 @@
|
|||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
// SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts®
|
||||
|
||||
// The OpenCode harness, against the goldens (tests/Goldens/fixture/
|
||||
// pipeline.json, under `opencode`): the tool source, the permission rules,
|
||||
// the agent config, the commands, the environment, the home, the event
|
||||
// parsing, the classification, and a warm-up against a fake opencode. The
|
||||
// test binary is its own supervisor (`__supervise`), as the driver is.
|
||||
|
||||
#include <signal.h>
|
||||
#include <sys/stat.h>
|
||||
import std;
|
||||
import Crafter.Review;
|
||||
namespace fs = std::filesystem;
|
||||
using namespace Crafter::Review;
|
||||
|
||||
namespace {
|
||||
std::int32_t Failures = 0;
|
||||
|
||||
void Check(std::string_view name, bool cond, std::string_view detail = {}) {
|
||||
std::println("{}{}{}", cond ? " ok " : " FAIL ", name, !cond && !detail.empty() ? std::format(" ({})", detail) : std::string());
|
||||
if (!cond) ++Failures;
|
||||
}
|
||||
|
||||
void Write(const fs::path& path, std::string_view text) {
|
||||
fs::create_directories(path.parent_path());
|
||||
std::ofstream(path, std::ios::binary) << text;
|
||||
}
|
||||
|
||||
std::string Read(const fs::path& path) {
|
||||
return Platform::ReadFile(path).value_or("");
|
||||
}
|
||||
|
||||
std::vector<std::string> Strings(const Json& array) {
|
||||
std::vector<std::string> out;
|
||||
for (const Json& item : array) out.push_back(item.get<std::string>());
|
||||
return out;
|
||||
}
|
||||
|
||||
std::vector<std::string> Names(const fs::path& dir) {
|
||||
std::vector<std::string> names;
|
||||
for (const fs::directory_entry& entry : fs::directory_iterator(dir)) names.push_back(entry.path().filename().string());
|
||||
std::ranges::sort(names);
|
||||
return names;
|
||||
}
|
||||
|
||||
// What the generator's temp dir is in the goldens.
|
||||
std::string Tidy(std::string text, const fs::path& td) {
|
||||
return Dyn::Replace(text, td.string(), "<td>");
|
||||
}
|
||||
|
||||
std::int32_t Run() {
|
||||
std::println("opencode harness + OpenRouter seats: loader, environments, permissions, event parsing, the tool schema");
|
||||
Platform::SetMode(Platform::Mode::Native);
|
||||
// The directory modes below are the generator's, made under 022.
|
||||
umask(022);
|
||||
const fs::path td = fs::temp_directory_path() / std::format("crafter-review-opencode-{}", Platform::PublishedPid());
|
||||
fs::remove_all(td);
|
||||
fs::create_directories(td);
|
||||
|
||||
const fs::path envFile = td / ".env";
|
||||
Write(envFile, "CLAUDE_CODE_OAUTH_TOKEN1=tok\nOPENROUTER_KEY=sk-or-one\n");
|
||||
auto ids = [](const std::vector<Json>& seats) {
|
||||
std::vector<std::string> out;
|
||||
for (const Json& s : seats) out.push_back(s.at("id").get<std::string>());
|
||||
return out;
|
||||
};
|
||||
Check("OPENROUTER_KEY is a seat only on request",
|
||||
ids(SeatPool::LoadSeats({}, envFile, "")) == std::vector<std::string>{"1"} && ids(SeatPool::LoadSeats({}, envFile, "openrouter")) == std::vector<std::string>{"or1"});
|
||||
try {
|
||||
SeatPool::LoadSeats({}, envFile, "or1,1");
|
||||
Check("an OpenRouter seat never shares a pool with Anthropic seats", false);
|
||||
} catch (const std::invalid_argument& e) {
|
||||
Check("an OpenRouter seat never shares a pool with Anthropic seats", std::string_view(e.what()).contains("cannot share"), e.what());
|
||||
}
|
||||
const std::vector<std::string> allow = {"Read", "Glob", "Grep", "Bash(cat:*)", "Bash(git -C . diff:*)", "Bash(python3 -:*)"};
|
||||
const Json p = OpenCode::Permissions(allow);
|
||||
Check("allowlist -> permission rules", p.at("read") == "allow" && p.at("StructuredOutput") == "allow" && p.at("bash").at("*") == "deny" && p.at("bash").at("git -C . diff*") == "allow");
|
||||
const std::vector<Json> ev = {
|
||||
{{"type", "step_start"}, {"sessionID", "ses_1"}, {"part", Json::object()}},
|
||||
{{"type", "tool_use"},
|
||||
{"part",
|
||||
{{"type", "tool"},
|
||||
{"tool", "StructuredOutput"},
|
||||
{"state", {{"status", "completed"}, {"input", {{"verdict", "SEND-READY"}, {"findings", Json::array()}, {"notes", Json::array()}, {"coverage", {{{"item", "u"}, {"verdict", "clear"}, {"basis", "b"}}}}}}}}}}},
|
||||
{{"type", "step_finish"}, {"part", {{"type", "step-finish"}, {"reason", "stop"}, {"tokens", {{"input", 20}, {"output", 4}, {"reasoning", 1}, {"cache", {{"read", 150}, {"write", 0}}}}}, {"cost", 0.0005}}}},
|
||||
};
|
||||
std::vector<std::string> lines;
|
||||
for (const Json& e : ev) lines.push_back(DumpSpaced(e));
|
||||
const Json ps = OpenCode::ParseEvents(Dyn::Join(lines, "\n"));
|
||||
Check("events -> the structured output, usage, classify ok",
|
||||
ps.at("structured").at("coverage").at(0).at("item") == "u" && ps.at("usage").at("output") == 5 && OpenCode::Classify(ps, 0, "") == std::pair<std::string, std::string>{"ok", ""});
|
||||
const std::string src = OpenCode::ToolSource();
|
||||
Check("the tool schema carries the note kinds, the coverage vocabulary and the verdicts of the schema; no persona field",
|
||||
std::ranges::all_of(std::array<std::string_view, 5>{"reserve-answer", "unverified", "FIX FIRST", "refutation-verified", "n/a"},
|
||||
[&](std::string_view k) { return src.contains(DumpSpaced(Json(std::string(k)))); })
|
||||
&& !src.contains("notMine") && !src.contains("wouldComment"));
|
||||
const fs::path repo = td / "repo";
|
||||
fs::create_directories(repo / "src");
|
||||
fs::create_directories(repo / ".git");
|
||||
fs::create_directories(repo / "node_modules");
|
||||
Write(repo / "package.json", "{}");
|
||||
const std::vector<std::string> one = {"openrouter/x/y"};
|
||||
const fs::path home = OpenCode::PrepareHome(td / "sc", repo, one);
|
||||
Check("prepare_home: the view links every top-level entry but .git and node_modules",
|
||||
Names(home / "view") == std::vector<std::string>{"package.json", "src"} && fs::is_symlink(fs::symlink_status(home / "view" / "src")));
|
||||
|
||||
// ---- against the goldens ---------------------------------------------------------------
|
||||
const Json g = Dyn::LoadJsonFile(fs::current_path() / "tests" / "Goldens" / "fixture" / "pipeline.json").at("opencode");
|
||||
Check("goldens: the tool source, byte for byte", src == g.at("toolSource").get<std::string>());
|
||||
Check("goldens: the provider routing", DumpFile(OpenCode::Routing()) == DumpFile(g.at("routing")));
|
||||
std::vector<std::string> missed;
|
||||
for (const Json& entry : g.at("modelId")) {
|
||||
const auto [prov, id] = OpenCode::ModelId(entry.at(0).get<std::string>());
|
||||
if (Json::array({prov, id}) != entry.at(1)) missed.push_back(entry.at(0).get<std::string>());
|
||||
}
|
||||
Check("goldens: model ids", missed.empty(), Dyn::Join(missed, ", "));
|
||||
missed.clear();
|
||||
std::size_t index = 0;
|
||||
for (const Json& entry : g.at("permissions")) {
|
||||
const std::vector<std::string> list = Strings(entry.at(0));
|
||||
if (DumpFile(OpenCode::Permissions(list)) != DumpFile(entry.at(1))) missed.push_back(DumpCompact(entry.at(0)));
|
||||
const std::map<std::string, std::string> child = OpenCode::ChildEnv({{"PATH", "/bin"}}, "/abs/system é.md", "/abs/so.json", list, td / "data" / std::to_string(index));
|
||||
if (child.at("OPENCODE_CONFIG_CONTENT") != g.at("agentConfig").at(index).at(1).get<std::string>()) missed.push_back(std::format("agent config {}", std::to_string(index)));
|
||||
if (child.at("REVIEW_GATES_STRUCTURED_OUT") != "/abs/so.json" || child.at("XDG_DATA_HOME") != (td / "data" / std::to_string(index)).string()) missed.push_back(std::format("child env {}", std::to_string(index)));
|
||||
++index;
|
||||
}
|
||||
Check("goldens: permission rules and the inline agent config, key for key", missed.empty(), Dyn::Join(missed, "; "));
|
||||
missed.clear();
|
||||
for (const Json& entry : g.at("cmd")) {
|
||||
const Json& a = entry.at(0);
|
||||
const std::vector<std::string> got = OpenCode::Cmd(a.at(0).get<std::string>(), a.at(1).get<std::string>(), a.at(2).get<std::string>(), a.at(3).get<std::string>(),
|
||||
a.at(4).is_null() ? std::string() : a.at(4).get<std::string>(), a.at(5).get<std::string>());
|
||||
if (got != Strings(entry.at(1))) missed.push_back(DumpCompact(a));
|
||||
}
|
||||
Check("goldens: the opencode command lines", missed.empty(), Dyn::Join(missed, "; "));
|
||||
std::map<std::string, std::string> baseIn;
|
||||
for (auto it = g.at("baseEnv").at(0).begin(); it != g.at("baseEnv").at(0).end(); ++it) baseIn[it.key()] = it.value().get<std::string>();
|
||||
std::map<std::string, std::string> baseWant;
|
||||
for (auto it = g.at("baseEnv").at(1).begin(); it != g.at("baseEnv").at(1).end(); ++it) baseWant[it.key()] = it.value().get<std::string>();
|
||||
Check("goldens: the shared environment - its own XDG home and key, no Anthropic or OpenCode variable", OpenCode::BaseEnv("/h", "sk-or-key", baseIn) == baseWant);
|
||||
|
||||
// The generator's temp dir, which the goldens call <td>.
|
||||
const fs::path gd = td / "g";
|
||||
const fs::path repo2 = gd / "repo";
|
||||
for (const std::string_view d : {".git", "node_modules", "src", "b", "a"}) fs::create_directories(repo2 / d);
|
||||
for (const std::string_view name : {"package.json", ".env", "Z"}) Write(repo2 / name, "{}");
|
||||
const Json& h = g.at("home");
|
||||
const fs::path home2 = OpenCode::PrepareHome(gd / "sc", repo2, Strings(h.at("models")));
|
||||
const fs::path config = home2 / "xdg" / "opencode" / "opencode.json";
|
||||
const std::string firstConfig = Read(config);
|
||||
OpenCode::PrepareHome(gd / "sc", repo2, one);
|
||||
Check("goldens: prepare_home's config, byte for byte, and rewritten by the next call", firstConfig == h.at("firstConfig").get<std::string>() && Read(config) == h.at("config").get<std::string>());
|
||||
std::vector<std::string> view;
|
||||
for (const std::string& name : Names(home2 / "view")) view.push_back(std::format("{} -> {}", name, Tidy(fs::read_symlink(home2 / "view" / name).string(), gd)));
|
||||
std::vector<std::string> viewWant;
|
||||
for (const Json& entry : h.at("view")) viewWant.push_back(std::format("{} -> {}", entry.at(0).get<std::string>(), entry.at(1).get<std::string>()));
|
||||
Check("goldens: the view's links, sorted, dot files included, .git and node_modules left out; idempotent", view == viewWant && Tidy(home2.string(), gd) == h.at("home").get<std::string>(),
|
||||
Dyn::Join(view, "; "));
|
||||
missed.clear();
|
||||
for (auto it = h.at("modes").begin(); it != h.at("modes").end(); ++it) {
|
||||
const auto mode = static_cast<std::uint32_t>(fs::status(home2 / it.key()).permissions()) & 0777;
|
||||
if (std::format("0o{:o}", mode) != it.value().get<std::string>()) missed.push_back(std::format("{} 0o{:o}", it.key(), mode));
|
||||
}
|
||||
Check("goldens: os.makedirs' modes - the last directory 0700, its parents as mkdir makes them", missed.empty() && Read(home2 / "xdg" / "opencode" / "tools" / "StructuredOutput.ts") == src,
|
||||
Dyn::Join(missed, ", "));
|
||||
|
||||
missed.clear();
|
||||
for (const Json& entry : g.at("events")) {
|
||||
if (DumpFile(OpenCode::ParseEvents(entry.at(0).get<std::string>())) != DumpFile(entry.at(1))) missed.push_back(Dyn::Repr(Dyn::Slice(entry.at(0).get<std::string>(), 0, 60)));
|
||||
}
|
||||
Check("goldens: the event stream - steps, tokens, tool calls and errors, the session, the first turn", missed.empty(), Dyn::Join(missed, "; "));
|
||||
missed.clear();
|
||||
for (const Json& entry : g.at("classify")) {
|
||||
const Json& c = entry.at(0);
|
||||
const auto [status, reason] = OpenCode::Classify(c.at(0), c.at(1).get<std::int64_t>(), c.at(2).get<std::string>(), c.at(3).get<bool>());
|
||||
if (Json::array({status, reason}) != entry.at(1)) missed.push_back(std::format("{} -> {} {}", Dyn::Repr(Dyn::Slice(c.at(2).get<std::string>(), 0, 40)), status, Dyn::Repr(Dyn::Slice(reason, 0, 60))));
|
||||
}
|
||||
Check("goldens: classify - the time budget, then credits, auth and limits as the regexes read them (re.I, \\b), then why nothing came back", missed.empty(), Dyn::Join(missed, "; "));
|
||||
|
||||
const Json& w = g.at("warm");
|
||||
const fs::path fake = td / "opencode";
|
||||
Write(fake, g.at("fake").get<std::string>());
|
||||
fs::permissions(fake, fs::perms::owner_all | fs::perms::group_read | fs::perms::group_exec | fs::perms::others_read | fs::perms::others_exec);
|
||||
const std::map<std::string, std::string> env = OpenCode::BaseEnv(home2, "sk-or", std::map<std::string, std::string>{{"PATH", "/usr/bin:/bin"}});
|
||||
std::vector<std::string> log;
|
||||
const auto warmed = OpenCode::Warm(fake.string(), home2, env, "openrouter/x/y", [&](const std::string& line) { log.push_back(line); });
|
||||
const std::string args = Tidy(Read(home2 / "data" / "warm" / "args"), gd);
|
||||
Check("goldens: a warm-up that fails - its log line, the tail of its stderr, the command it ran",
|
||||
warmed == std::pair<bool, std::string>{w.at("result").at(0).get<bool>(), w.at("result").at(1).get<std::string>()} && log == Strings(w.at("log")) && args == w.at("args").get<std::string>(),
|
||||
std::format("{} | {}", Dyn::Join(log, " | "), Dyn::Repr(args)));
|
||||
const fs::path slow = td / "slow-opencode";
|
||||
Write(slow, "#!/bin/sh\necho $$ > \"$XDG_DATA_HOME/pid\"\nexec sleep 30\n");
|
||||
fs::permissions(slow, fs::perms::owner_all);
|
||||
log.clear();
|
||||
const auto timed = OpenCode::Warm(slow.string(), home2, env, "openrouter/x/y", [&](const std::string& line) { log.push_back(line); }, 1);
|
||||
const std::optional<std::int64_t> sleeper = Dyn::ParseInt(Dyn::Strip(Read(home2 / "data" / "warm" / "pid")));
|
||||
Check("goldens: a warm-up out of time says so and logs nothing",
|
||||
timed == std::pair<bool, std::string>{w.at("timedOut").at(0).get<bool>(), w.at("timedOut").at(1).get<std::string>()} && log.empty() && w.at("timedLog").empty());
|
||||
Check("...and leaves no process behind: the supervisor takes it down", sleeper && !Platform::ProcStart(*sleeper));
|
||||
|
||||
// ---- beyond the goldens -----------------------------------------------------------------
|
||||
const fs::path so = td / "so.json";
|
||||
Write(so, R"({"verdict": "SEND-READY", "findings": [], "notes": [], "coverage": []})");
|
||||
const fs::path bad = td / "bad.json";
|
||||
Write(bad, R"({"verdict": "SEND-READY", "findings": "none"})");
|
||||
const fs::path garbage = td / "garbage.json";
|
||||
Write(garbage, "{");
|
||||
const std::string noTool = DumpSpaced({{"type", "step_finish"}, {"part", {{"reason", "stop"}}}});
|
||||
Check("the structured-output file stands in when the stream carries no call: one with a findings list, nothing else",
|
||||
OpenCode::ParseEvents(noTool, so).at("structured").at("verdict") == "SEND-READY" && OpenCode::ParseEvents(noTool, bad).at("structured").is_null()
|
||||
&& OpenCode::ParseEvents(noTool, garbage).at("structured").is_null() && OpenCode::ParseEvents(noTool, td / "missing.json").at("structured").is_null()
|
||||
&& OpenCode::ParseEvents(noTool).at("structured").is_null());
|
||||
|
||||
fs::remove_all(td);
|
||||
return Failures == 0 ? 0 : 1;
|
||||
}
|
||||
}
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
const std::vector<std::string_view> args(argv, argv + argc);
|
||||
if (args.size() > 1 && args[1] == Process::SuperviseVerb) return Process::SuperviseMain(std::span(args).subspan(2));
|
||||
try {
|
||||
return Run();
|
||||
} catch (const std::exception& error) {
|
||||
std::println(std::cerr, "opencode: {}", error.what());
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
Loading…
Reference in a new issue