Catcrafts/Crafter.Review
Watch
2
0
Fork
You've already forked Crafter.Review
 0
Code Issues 7 Pull requests Projects Releases 3 Packages Wiki Activity Actions

test(pipeline): the checks of the pure pipeline

Browse source 
Stream, Normalize, Merge, Dispositions, Config, CodeKeys and the pure half
of CommitsMode, each a test of its own (131 checks). The fixture rewrites
are checked in (code-keys/<name>/, commits3/), so the suite reads fixtures
and edits nothing. The allowlist, command-line and MCP checks of
CommitsMode come with the pool they test.

Among them: this binary's `<exe> sql` spelling counts as a SQL call, and a
result line with a raw U+2028 in a JSON string is still the result line.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Jorijn van der Graaf 2026-10-31 06:56:09 +01:00
commit 71dba5690f
Signed by: jorijnvdgraaf
GPG key ID: 2937E59CDCC1BCFB
12 changed files with 3778 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

97
tests/Stream/main.cpp Normal file
View file

@ -0,0 +1,97 @@
// SPDX-License-Identifier: AGPL-3.0-only
// SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts®
// The claude stream: its envelopes.
import std;
import Crafter.Review;
namespace fs = std::filesystem;
using namespace Crafter::Review;
namespace {
std::int32_t Failures = 0;
void Check(std::string_view name, bool cond, std::string_view detail = {}) {
std::println("{}{}{}", cond ? " ok " : " FAIL ", name, !cond && !detail.empty() ? std::format(" ({})", detail) : std::string());
if (!cond) ++Failures;
}
// The result line mutated, or dropped when the mutation returns nullopt.
std::string Variant(const std::string& text, const std::function<std::optional<Json>(Json)>& mutate) {
std::vector<std::string> out;
for (const std::string& line : Dyn::SplitLines(text)) {
if (line.starts_with('{') && Dyn::Replace(line, " ", "").contains("\"type\":\"result\"")) {
const std::optional<Json> changed = mutate(Json::parse(line));
if (!changed) continue;
out.push_back(DumpSpaced(*changed));
} else {
out.push_back(line);
}
}
return Dyn::Join(out, "\n");
}
std::function<std::optional<Json>(Json)> Failed(const Json& update) {
return [update](Json o) -> std::optional<Json> {
o.update(update);
o.erase("structured_output");
return o;
};
}
std::string Status(const std::string& text, std::optional<std::int32_t> code, std::string_view err = {}) {
return Stream::Classify(Stream::ParseStream(text), code, err).first;
}
}
int main() {
std::println("stream envelopes");
const std::string text = Dyn::ReadText(fs::current_path() / "tests" / "Goldens" / "fixture" / "stream-success.jsonl");
const Json p = Stream::ParseStream(text);
Check("success: classify ok", Stream::Classify(p, 0, "") == std::pair<std::string, std::string>{"ok", ""});
const std::optional<Json> so = Stream::StructuredOutput(p.at("result"));
Check("success: structured_output parsed", so && so->at("verdict") == "SEND-READY" && so->at("coverage").at(0).at("item") == "R1");
Check("success: init parsed", Dyn::Truthy(p.at("init")) && Dyn::Get(p.at("init"), "mcp_servers") == Json::array());
Check("success: usage from the result line + firstTurn", p.at("turns") == 2 && p.at("firstTurn").at("cache_creation_input_tokens").get<std::int64_t>() > 0
&& p.at("usage").at("output").get<std::int64_t>() > 0 && !p.at("costUsd").is_null() && p.at("denials") == Json::array());
const Json w = Stream::Windows(p.at("rateLimit"));
Check("success: rate windows as percent", w.contains("five_hour") && w.at("five_hour").at("utilization").get<double>() > 0 && w.at("five_hour").at("utilization").get<double>() < 100
&& Dyn::Truthy(w.at("seven_day").at("resetsAt")), DumpCompactSorted(w));
Check("error_max_turns -> error", Status(Variant(text, Failed({{"subtype", "error_max_turns"}, {"is_error", true}, {"result", "Reached max turns (1)"}})), 1) == "error");
Check("429 -> limit", Status(Variant(text, Failed({{"subtype", "error_during_execution"}, {"is_error", true}, {"api_error_status", 429}, {"result", "API Error: 429 rate_limit_error"}})), 1) == "limit");
Check("401 -> auth", Status(Variant(text, Failed({{"subtype", "error_during_execution"}, {"is_error", true}, {"api_error_status", 401}, {"result", "API Error: 401 authentication_error"}})), 1) == "auth");
const std::string noResult = Variant(text, [](Json) -> std::optional<Json> { return std::nullopt; });
Check("403 in stderr, no result -> auth", Status(noResult, 1, "Error: 403 Forbidden") == "auth");
Check("no result line -> error", Status(noResult, 1) == "error");
Check("no result line + usage limit text -> limit", Status(noResult, 1, "You're out of extra usage") == "limit");
Check("timeout flag / exit 124 -> timeout", Stream::Classify(p, std::nullopt, "", true).first == "timeout" && Stream::Classify(p, 124, "").first == "timeout");
const std::string doubled = Variant(text, [](Json o) -> std::optional<Json> {
o["structured_output"] = DumpSpaced(o.at("structured_output"));
return o;
});
Check("double-encoded structured_output decoded", Stream::StructuredOutput(Stream::ParseStream(doubled).at("result")) == so);
const std::string inResult = Variant(text, [](Json o) -> std::optional<Json> {
const Json inner = o.at("structured_output");
o.erase("structured_output");
o["result"] = DumpSpaced(inner);
return o;
});
Check("structured output inside result text decoded", Stream::StructuredOutput(Stream::ParseStream(inResult).at("result")) == so);
const std::string noFindings = Variant(text, [](Json o) -> std::optional<Json> {
o["structured_output"] = {{"verdict", "SEND-READY"}};
return o;
});
Check("missing findings array -> error (no structured output)", Status(noFindings, 0) == "error");
Check("credit balance too low -> credits", Status(Variant(text, Failed({{"subtype", "error_during_execution"}, {"is_error", true}, {"api_error_status", 400},
{"result", "API Error: 400 Your credit balance is too low to access the Anthropic API."}})), 1) == "credits");
// The helper's spelling is a SQL call wherever it stands, and a stream
// line is cut at \n only: a raw U+2028 in a JSON string stays inside its
// line.
Check("this binary's helper is a SQL call, alone, after a path, in a compound command; another verb is not",
Stream::IsSqlCall("/app/bin/crafter-review sql <<'SQL'\nselect 1;\nSQL") && Stream::IsSqlCall("cd . && crafter-review sql 'select 1'")
&& Stream::IsSqlCall("/x/bin/crafter.review-exe-v/crafter-review sql") && !Stream::IsSqlCall("/app/bin/crafter-review sqlx") && !Stream::IsSqlCall("/app/bin/crafter-review totp f.json"));
const std::string split = "{\"type\":\"result\",\"subtype\":\"success\",\"is_error\":false,\"result\":\"a\u2028b\",\"structured_output\":{\"verdict\":\"SEND-READY\",\"findings\":[],\"notes\":[],\"coverage\":[]}}";
Check("a result line with a raw U+2028 in a string is still the result line", Stream::Classify(Stream::ParseStream(split), 0, "").first == "ok");
return Failures ? 1 : 0;
}