Jorijn van der Graaf
741e3a656d
The modules that turn a frozen range into passes and passes into a verdict: :Schema, :Tools (the DENY, git and base lists, a leaf, so the config and the pool both read them), :Plan, :Artifact, :Inventory, :Dispositions, :State, :Normalize, :Stream, :Prompt, :Config, and the URL half of :PgWire. Units, jobs, results, the config and the claude stream stay JSON from end to end, so every key keeps its place; :Text gives JSON values truthiness, get, ==, membership and Str(), and throws Dyn::Error under a name for each kind of unexpected data. Exactness where the bytes are seen: - the stream classifiers are hand-written scanners with case-insensitive folds (the Kelvin sign is a k), Unicode \b and \s, and `.` stopping at \n; the SQL call detection takes `<exe> sql` with any path before it; - config problems carry the JSON scanner's and the UTF-8 decoder's own texts, positions included; - a float total is Neumaier's compensated sum, and the build passes -ffp-contract=off: -march=native would fuse a*b+c; - `crafter-review inventory` and `config` print what the goldens record, byte for byte (the fixture and 14 configs, broken ones included); a message names this binary as the command to run. tests/Goldens: the recorded answers over the fixtures next to them (the commits fixture, a rich versioned series with CRLF, a folded subject, Markdown headings in and out of fences, the code-key rewrites); the suite compares 2304 results as DumpFile writes them, so a key out of order fails too. All agree. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
680 lines
32 KiB
C++
680 lines
32 KiB
C++
// SPDX-License-Identifier: AGPL-3.0-only
|
|
// SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts®
|
|
|
|
module Crafter.Review:Plan_impl;
|
|
import :Plan;
|
|
import :Json;
|
|
import :Text;
|
|
import :Crypto;
|
|
import std;
|
|
|
|
namespace Crafter::Review::Plan {
|
|
namespace {
|
|
// The bytes of `p\d+:` at the start of a unit id, where \d is any
|
|
// Unicode decimal digit, as re.match reads it.
|
|
std::optional<std::size_t> PkPrefix(std::string_view id) {
|
|
if (!id.starts_with('p')) return std::nullopt;
|
|
const std::size_t digits = Dyn::DecimalRun(id, 1);
|
|
if (digits == 0 || 1 + digits >= id.size() || id[1 + digits] != ':') return std::nullopt;
|
|
return 2 + digits;
|
|
}
|
|
|
|
// group(1) of ^p\d+:(.+)#[^#]+$: the path between the prefix and the
|
|
// last '#', which must have something after it. `.` stops at a
|
|
// newline; [^#] does not.
|
|
std::optional<std::string_view> FileOfUnit(std::string_view id) {
|
|
const std::optional<std::size_t> prefix = PkPrefix(id);
|
|
if (!prefix) return std::nullopt;
|
|
const std::string_view rest = id.substr(*prefix);
|
|
const std::size_t hash = rest.rfind('#');
|
|
if (hash == std::string_view::npos || hash == 0 || hash + 1 >= rest.size()) return std::nullopt;
|
|
const std::string_view path = rest.substr(0, hash);
|
|
if (path.contains('\n')) return std::nullopt;
|
|
return path;
|
|
}
|
|
|
|
bool IsHex(std::string_view text) {
|
|
return !text.empty() && std::ranges::all_of(text, [](char c) { return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f'); });
|
|
}
|
|
|
|
// ^\d+\.\d+
|
|
bool StartsWithVersion(std::string_view line) {
|
|
const std::size_t major = Dyn::DecimalRun(line, 0);
|
|
if (major == 0 || major >= line.size() || line[major] != '.') return false;
|
|
return Dyn::DecimalRun(line, major + 1) > 0;
|
|
}
|
|
|
|
// ^(base-commit|prerequisite-patch-id): [0-9a-f]+$
|
|
bool IsTailLine(std::string_view line) {
|
|
for (const std::string_view head : {std::string_view("base-commit: "), std::string_view("prerequisite-patch-id: ")}) {
|
|
if (line.starts_with(head) && IsHex(line.substr(head.size()))) return true;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
// ^index [0-9a-f]+\.\.[0-9a-f]+( \d+)?$ -> group(1) ("" when absent)
|
|
std::optional<std::string_view> IndexMode(std::string_view line) {
|
|
if (!line.starts_with("index ")) return std::nullopt;
|
|
const std::string_view rest = line.substr(6);
|
|
const std::size_t dots = rest.find("..");
|
|
if (dots == std::string_view::npos || !IsHex(rest.substr(0, dots))) return std::nullopt;
|
|
const std::string_view after = rest.substr(dots + 2);
|
|
const std::size_t space = after.find(' ');
|
|
if (space == std::string_view::npos) {
|
|
if (!IsHex(after)) return std::nullopt;
|
|
return std::string_view();
|
|
}
|
|
if (!IsHex(after.substr(0, space))) return std::nullopt;
|
|
const std::string_view mode = after.substr(space);
|
|
if (mode.size() < 2 || Dyn::DecimalRun(mode, 1) != mode.size() - 1) return std::nullopt;
|
|
return mode;
|
|
}
|
|
|
|
// \d+(?:,\d+)? at `at`: the bytes it takes, 0 for no match.
|
|
std::size_t LineRange(std::string_view text, std::size_t at) {
|
|
const std::size_t start = Dyn::DecimalRun(text, at);
|
|
if (start == 0) return 0;
|
|
std::size_t end = at + start;
|
|
if (end < text.size() && text[end] == ',') {
|
|
const std::size_t count = Dyn::DecimalRun(text, end + 1);
|
|
if (count > 0) end += 1 + count;
|
|
}
|
|
return end - at;
|
|
}
|
|
|
|
// ^@@ -\d+(?:,\d+)? \+\d+(?:,\d+)? @@(.*)$ -> group(1)
|
|
std::optional<std::string_view> HunkTail(std::string_view line) {
|
|
if (!line.starts_with("@@ -")) return std::nullopt;
|
|
std::size_t at = 4;
|
|
const std::size_t old = LineRange(line, at);
|
|
if (old == 0) return std::nullopt;
|
|
at += old;
|
|
if (!line.substr(at).starts_with(" +")) return std::nullopt;
|
|
at += 2;
|
|
const std::size_t now = LineRange(line, at);
|
|
if (now == 0) return std::nullopt;
|
|
at += now;
|
|
if (!line.substr(at).starts_with(" @@")) return std::nullopt;
|
|
return line.substr(at + 3);
|
|
}
|
|
|
|
// Every line start that `head` follows: where the text is cut.
|
|
// Position 0 is one when the text starts with it, which makes the
|
|
// first piece empty.
|
|
std::vector<std::string> SplitBeforeLines(std::string_view text, std::string_view head) {
|
|
std::vector<std::size_t> cuts;
|
|
for (std::size_t at = 0; at <= text.size();) {
|
|
if (text.substr(at).starts_with(head)) cuts.push_back(at);
|
|
const std::size_t newline = text.find('\n', at);
|
|
if (newline == std::string_view::npos) break;
|
|
at = newline + 1;
|
|
}
|
|
std::vector<std::string> pieces;
|
|
std::size_t from = 0;
|
|
for (const std::size_t cut : cuts) {
|
|
pieces.emplace_back(text.substr(from, cut - from));
|
|
from = cut;
|
|
}
|
|
pieces.emplace_back(text.substr(from));
|
|
return pieces;
|
|
}
|
|
|
|
// re.sub(r'^Subject: (\[[^\]]*\]\s*)*', 'Subject: ', line): the
|
|
// `[PATCH n/m]` tags after `Subject: ` dropped, and the space after
|
|
// each.
|
|
std::string StripSubjectTags(std::string_view line) {
|
|
if (!line.starts_with("Subject: ")) return std::string(line);
|
|
std::size_t at = 9;
|
|
while (at < line.size() && line[at] == '[') {
|
|
const std::size_t close = line.find(']', at + 1);
|
|
if (close == std::string_view::npos) break;
|
|
at = close + 1;
|
|
at += Dyn::SpaceRun(line, at);
|
|
}
|
|
return std::format("Subject: {}", line.substr(at));
|
|
}
|
|
|
|
// ^base-commit: [0-9a-f]{40}$
|
|
bool IsBaseCommit(std::string_view line) {
|
|
constexpr std::string_view Head = "base-commit: ";
|
|
return line.size() == Head.size() + 40 && line.starts_with(Head) && IsHex(line.substr(Head.size()));
|
|
}
|
|
|
|
std::int64_t IntItem(const Json& range, std::size_t index) {
|
|
const Json& value = range.at(index);
|
|
if (value.is_boolean()) return value.get<bool>() ? 1 : 0;
|
|
return value.get<std::int64_t>();
|
|
}
|
|
|
|
Json Mechanical(std::string_view name, std::string_view lensFile, const Json& unit, std::string_view text, std::string_view basis) {
|
|
const std::string id = Dyn::At(unit, "id").get<std::string>();
|
|
return {
|
|
{"label", std::format("{}:na:{}", name, id)},
|
|
{"lens", name},
|
|
{"lensFile", lensFile},
|
|
{"kind", "auto-na"},
|
|
{"unit", id},
|
|
{"expectedIds", {id}},
|
|
{"unitSha", Sha(text)},
|
|
{"basis", basis},
|
|
};
|
|
}
|
|
|
|
bool Contains(std::span<const std::string> list, std::string_view item) {
|
|
return std::ranges::contains(list, item);
|
|
}
|
|
}
|
|
|
|
std::vector<ModelTier>& Models() {
|
|
static std::vector<ModelTier> models = {{"opus", "claude-opus-5-5"}, {"fable", "claude-opus-5-5"}};
|
|
return models;
|
|
}
|
|
|
|
const std::string& ModelFor(std::string_view tier) {
|
|
for (const ModelTier& model : Models()) {
|
|
if (model.tier == tier) return model.id;
|
|
}
|
|
throw Dyn::Error("KeyError", Dyn::Repr(tier));
|
|
}
|
|
|
|
bool IsLens(std::string_view file) {
|
|
return std::ranges::any_of(Lenses, [&](const Lens& lens) { return lens.file == file; });
|
|
}
|
|
|
|
const Lens& LensOf(std::string_view file) {
|
|
for (const Lens& lens : Lenses) {
|
|
if (lens.file == file) return lens;
|
|
}
|
|
throw Dyn::Error("KeyError", Dyn::Repr(file));
|
|
}
|
|
|
|
const std::vector<Gate>& Gates() {
|
|
static const std::vector<Gate> gates = {
|
|
{"Gate 1 — Approach", {"repo-better-solution.md"}},
|
|
{"Gate 2 — Functional", {FunctionalLens}},
|
|
{"Gate 3 — Proof", {"repo-bug-prover.md", "repo-security.md"}},
|
|
};
|
|
return gates;
|
|
}
|
|
|
|
const Environment* FindEnvironment(std::string_view name) {
|
|
for (const Environment& environment : Environments) {
|
|
if (environment.name == name) return &environment;
|
|
}
|
|
return nullptr;
|
|
}
|
|
|
|
std::vector<std::string> EnvDeps(std::string_view lensFile, std::string_view environment) {
|
|
const std::string_view fallback = LensOf(lensFile).env;
|
|
const std::string_view used = environment.empty() ? fallback : environment;
|
|
if (used == fallback) return {};
|
|
return {std::format("env={}", used)};
|
|
}
|
|
|
|
Budget BudgetFor(std::string_view kind, std::string_view unitKind, std::string_view lensFile) {
|
|
if (kind == "whole") return lensFile == FunctionalLens ? Budget{220, 120} : Budget{180, 105};
|
|
if (unitKind != "code") throw Dyn::Error("KeyError", Dyn::Repr(std::format("block:{}", unitKind)));
|
|
return lensFile == FunctionalLens ? Budget{90, 60} : Budget{60, 40};
|
|
}
|
|
|
|
std::string_view DefaultEffort() {
|
|
return Effort;
|
|
}
|
|
|
|
std::string_view UnitKindOf(std::string_view unitId) {
|
|
if (unitId.starts_with("whole:")) return "slot";
|
|
const std::optional<std::size_t> prefix = PkPrefix(unitId);
|
|
if (!prefix) return "text";
|
|
// `$` also matches before a final newline.
|
|
std::string_view name = unitId.substr(*prefix);
|
|
if (name.ends_with('\n')) name.remove_suffix(1);
|
|
if (name == "subject" || name == "msg" || name == "trailers" || name == "diffstat") return "text";
|
|
return "code";
|
|
}
|
|
|
|
std::string_view UnitKind(const Json& unit) {
|
|
return UnitKindOf(Dyn::At(unit, "id").get_ref<const std::string&>());
|
|
}
|
|
|
|
bool IsDocPath(std::string_view path) {
|
|
auto endsWithMd = [](std::string_view text) {
|
|
const std::u32string decoded = Dyn::Decode(text);
|
|
if (decoded.size() < 3) return false;
|
|
const std::size_t n = decoded.size();
|
|
return decoded[n - 3] == '.' && Dyn::IgnoreCaseLetter(decoded[n - 2]) == 'm' && Dyn::IgnoreCaseLetter(decoded[n - 1]) == 'd';
|
|
};
|
|
return endsWithMd(path) || (path.ends_with('\n') && endsWithMd(path.substr(0, path.size() - 1)));
|
|
}
|
|
|
|
bool IsDocUnit(std::string_view unitId) {
|
|
const std::optional<std::string_view> path = FileOfUnit(unitId);
|
|
return path && IsDocPath(*path);
|
|
}
|
|
|
|
// The lens is not consulted (yet): no lens of this pipeline reads a
|
|
// documentation file since the facts lens was removed.
|
|
std::string_view KindFor(std::string_view, const Json& unit) {
|
|
const std::string_view kind = UnitKind(unit);
|
|
return kind == "code" && IsDocUnit(Dyn::At(unit, "id").get_ref<const std::string&>()) ? "doc" : kind;
|
|
}
|
|
|
|
std::optional<std::string> UnitPath(std::string_view unitId) {
|
|
const std::optional<std::string_view> path = FileOfUnit(unitId);
|
|
if (!path) return std::nullopt;
|
|
return std::string(*path);
|
|
}
|
|
|
|
std::vector<std::string> NotReviewed(const Json& units, std::span<const std::string> lines, std::span<const std::string> lensFiles,
|
|
std::span<const std::string> exclude) {
|
|
std::set<std::string> covered;
|
|
for (const std::string& lensFile : lensFiles) {
|
|
const LensPlan plan = PlanLens(lensFile, units, lines, {}, exclude);
|
|
if (Dyn::Truthy(plan.whole)) {
|
|
for (const Json& id : Dyn::At(plan.whole, "expectedIds")) covered.insert(id.get<std::string>());
|
|
}
|
|
for (const Json& block : plan.blocks) {
|
|
if (Dyn::At(block, "kind") != "auto-na") covered.insert(Dyn::At(block, "unit").get<std::string>());
|
|
}
|
|
}
|
|
std::vector<std::string> out;
|
|
for (const Json& unit : units) {
|
|
const std::string& id = Dyn::At(unit, "id").get_ref<const std::string&>();
|
|
if (!covered.contains(id)) out.push_back(id);
|
|
}
|
|
return out;
|
|
}
|
|
|
|
std::optional<std::string> ExcludedBy(std::string_view path, std::span<const std::string> globs) {
|
|
for (const std::string& glob : globs) {
|
|
if (Dyn::FnMatch(path, glob) || (glob.starts_with("**/") && Dyn::FnMatch(path, std::string_view(glob).substr(3)))) return glob;
|
|
}
|
|
return std::nullopt;
|
|
}
|
|
|
|
std::string UnitText(std::span<const std::string> lines, const Json& unit) {
|
|
const Json& range = Dyn::At(unit, "lines");
|
|
const std::int64_t a = IntItem(range, 0);
|
|
const std::int64_t b = IntItem(range, 1);
|
|
if (a == 0) return "";
|
|
const auto [from, to] = Dyn::SliceBounds(lines.size(), a - 1, b);
|
|
return Dyn::Join(lines.subspan(from, to - from), "\n");
|
|
}
|
|
|
|
std::string Sha(std::string_view text) {
|
|
return Sha256Hex(text);
|
|
}
|
|
|
|
std::string LensName(std::string_view lensFile) {
|
|
return Dyn::Replace(lensFile, ".md", "");
|
|
}
|
|
|
|
std::vector<std::string> ModeLenses() {
|
|
std::vector<std::string> out;
|
|
for (const Gate& gate : Gates()) {
|
|
for (const std::string_view lens : gate.lenses) out.emplace_back(lens);
|
|
}
|
|
return out;
|
|
}
|
|
|
|
std::vector<Gate> GatesFor(std::span<const std::string> only, std::span<const std::string> without, std::span<const std::string> off) {
|
|
std::vector<Gate> out;
|
|
for (const Gate& gate : Gates()) {
|
|
Gate kept{gate.title, {}};
|
|
for (const std::string_view lens : gate.lenses) {
|
|
if (!only.empty() && !Contains(only, lens)) continue;
|
|
if (Contains(without, lens) || Contains(off, lens)) continue;
|
|
kept.lenses.push_back(lens);
|
|
}
|
|
if (!kept.lenses.empty()) out.push_back(std::move(kept));
|
|
}
|
|
return out;
|
|
}
|
|
|
|
LensPlan PlanLens(std::string_view lensFile, const Json& units, std::span<const std::string> lines, std::span<const std::string> unitsFilter,
|
|
std::span<const std::string> exclude) {
|
|
const Lens& lens = LensOf(lensFile);
|
|
const std::string name = LensName(lensFile);
|
|
LensPlan plan;
|
|
if (!lens.whole.empty()) {
|
|
const Budget budget = BudgetFor("whole", "", lensFile);
|
|
Json slots = Json::array();
|
|
for (const Json& unit : units) {
|
|
const std::string& id = Dyn::At(unit, "id").get_ref<const std::string&>();
|
|
if (UnitKind(unit) == "slot" && std::ranges::contains(lens.slots, std::string_view(id))) slots.push_back(id);
|
|
}
|
|
plan.whole = {
|
|
{"label", std::format("{}:whole", name)},
|
|
{"lens", name},
|
|
{"lensFile", lensFile},
|
|
{"kind", "whole"},
|
|
{"unit", nullptr},
|
|
{"expectedIds", slots},
|
|
{"model", ModelFor(lens.model)},
|
|
{"maxTurns", budget.maxTurns},
|
|
{"timeoutMin", budget.timeoutMin},
|
|
};
|
|
}
|
|
for (const Json& unit : units) {
|
|
const std::string_view kind = KindFor(lensFile, unit);
|
|
if (!std::ranges::contains(lens.block, kind)) continue;
|
|
const std::string& id = Dyn::At(unit, "id").get_ref<const std::string&>();
|
|
if (!unitsFilter.empty() && !Contains(unitsFilter, id)) continue;
|
|
const std::string text = UnitText(lines, unit);
|
|
const std::optional<std::string> path = UnitPath(id);
|
|
if (!lens.perCommitSlots && (id.ends_with(":outside") || id.ends_with(":absent"))) {
|
|
// The functional lens judges adjacent flows and what is missing
|
|
// in its whole pass, not per commit.
|
|
continue;
|
|
}
|
|
if (lensFile == FunctionalLens && path && id.ends_with("#hdr")) {
|
|
plan.blocks.push_back(Mechanical(name, lensFile, unit, text,
|
|
"mechanical: a diff header (mode, rename, new or deleted file) carries no behaviour of its own; the hunks of the file do"));
|
|
continue;
|
|
}
|
|
if (lensFile == FunctionalLens && path) {
|
|
if (const std::optional<std::string> glob = ExcludedBy(*path, exclude)) {
|
|
plan.blocks.push_back(Mechanical(name, lensFile, unit, text,
|
|
std::format("mechanical: no runtime behaviour - the file matches functional.exclude ({})", *glob)));
|
|
continue;
|
|
}
|
|
}
|
|
const Budget budget = BudgetFor("block", kind, lensFile);
|
|
plan.blocks.push_back({
|
|
{"label", std::format("{}:block:{}", name, id)},
|
|
{"lens", name},
|
|
{"lensFile", lensFile},
|
|
{"kind", "block"},
|
|
{"unit", id},
|
|
{"unitKind", kind},
|
|
{"expectedIds", {id}},
|
|
{"unitSha", Sha(text)},
|
|
{"model", ModelFor(lens.model)},
|
|
{"maxTurns", budget.maxTurns},
|
|
{"timeoutMin", budget.timeoutMin},
|
|
});
|
|
}
|
|
return plan;
|
|
}
|
|
|
|
std::optional<std::string> ReuseKey(const Json& job, std::string_view lensFile, std::string_view unitText, std::vector<std::string> deps,
|
|
std::vector<std::string> inForceIds, std::string_view charterSha, std::int32_t promptVersion,
|
|
std::string_view ident) {
|
|
if (!LensOf(lensFile).reuse || Dyn::At(job, "kind") != "block") return std::nullopt;
|
|
std::ranges::sort(deps);
|
|
std::ranges::sort(inForceIds);
|
|
const Json& model = Dyn::Get(job, "model");
|
|
const std::vector<std::string> parts = {
|
|
std::string(lensFile),
|
|
ident.empty() ? Dyn::At(job, "unit").get<std::string>() : std::string(ident),
|
|
Sha(unitText),
|
|
Sha(Dyn::Join(deps, "\n")),
|
|
Dyn::Join(inForceIds, ","),
|
|
std::string(charterSha),
|
|
std::to_string(promptVersion),
|
|
Dyn::Has(job, "model") ? model.get<std::string>() : std::string(),
|
|
std::string(ReuseKeyEffortSlot),
|
|
};
|
|
return Sha(Dyn::Join(parts, "|")).substr(0, 16);
|
|
}
|
|
|
|
std::string CodeNorm(std::string_view text) {
|
|
std::vector<std::string> lines = Dyn::Split(text, "\n");
|
|
while (!lines.empty() && lines.back().empty()) lines.pop_back();
|
|
if (lines.size() >= 2 && lines[lines.size() - 2] == "-- " && StartsWithVersion(lines.back())) {
|
|
lines.resize(lines.size() - 2);
|
|
while (!lines.empty() && (lines.back().empty() || IsTailLine(lines.back()))) lines.pop_back();
|
|
}
|
|
for (std::string& line : lines) {
|
|
if (const std::optional<std::string_view> mode = IndexMode(line)) {
|
|
line = std::format("index{}", *mode);
|
|
} else if (const std::optional<std::string_view> tail = HunkTail(line)) {
|
|
line = std::format("@@{}", *tail);
|
|
}
|
|
}
|
|
return Dyn::Join(lines, "\n");
|
|
}
|
|
|
|
PatchParts PartsOf(std::string_view patchText) {
|
|
const std::vector<std::string> lines = Dyn::Split(patchText, "\n");
|
|
std::optional<std::size_t> subject;
|
|
for (std::size_t i = 0; i < lines.size(); ++i) {
|
|
if (lines[i].starts_with("Subject:")) {
|
|
subject = i;
|
|
break;
|
|
}
|
|
}
|
|
std::size_t body0 = 1;
|
|
if (subject) {
|
|
for (std::size_t i = *subject + 1; i < lines.size(); ++i) {
|
|
if (Dyn::Strip(lines[i]).empty()) {
|
|
body0 = i + 1;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
std::size_t diff0 = lines.size();
|
|
for (std::size_t i = body0; i < lines.size(); ++i) {
|
|
if (lines[i].starts_with("---")) {
|
|
diff0 = i;
|
|
break;
|
|
}
|
|
}
|
|
std::size_t first = lines.size();
|
|
for (std::size_t i = diff0 + 1; i < lines.size(); ++i) {
|
|
if (lines[i].starts_with("diff --git")) {
|
|
first = i;
|
|
break;
|
|
}
|
|
}
|
|
PatchParts parts;
|
|
for (const std::string& line : lines) {
|
|
if (IsBaseCommit(line)) {
|
|
parts.base = Dyn::Split(line)[1];
|
|
break;
|
|
}
|
|
}
|
|
const std::span<const std::string> code(lines.begin() + static_cast<std::ptrdiff_t>(std::min(first, lines.size())), lines.end());
|
|
for (const std::string& section : SplitBeforeLines(CodeNorm(Dyn::Join(code, "\n")), "diff --git ")) {
|
|
const std::string firstLine = Dyn::Split(section, "\n", 1)[0];
|
|
const std::string path = Dyn::Strip(Dyn::Split(firstLine, " b/").back());
|
|
if (section.starts_with("diff --git ") && IsDocPath(path)) {
|
|
// re.split(r'(?m)^@@', section, maxsplit=1)[0].rstrip('\n')
|
|
std::size_t header = section.size();
|
|
for (std::size_t at = 0; at < section.size();) {
|
|
if (std::string_view(section).substr(at).starts_with("@@")) {
|
|
header = at;
|
|
break;
|
|
}
|
|
const std::size_t newline = section.find('\n', at);
|
|
if (newline == std::string::npos) break;
|
|
at = newline + 1;
|
|
}
|
|
parts.docs.push_back(Dyn::RStrip(std::string_view(section).substr(0, header), "\n"));
|
|
} else {
|
|
parts.code += section;
|
|
}
|
|
}
|
|
std::ranges::sort(parts.docs);
|
|
const auto [messageFrom, messageTo] = Dyn::SliceBounds(lines.size(), static_cast<std::int64_t>(subject ? *subject : body0), static_cast<std::int64_t>(diff0));
|
|
std::vector<std::string> message;
|
|
for (std::size_t i = messageFrom; i < messageTo; ++i) message.push_back(StripSubjectTags(lines[i]));
|
|
parts.message = Dyn::Join(message, "\n");
|
|
const auto [bodyFrom, bodyTo] = Dyn::SliceBounds(lines.size(), static_cast<std::int64_t>(body0), static_cast<std::int64_t>(diff0));
|
|
for (std::size_t i = bodyFrom; i < bodyTo; ++i) {
|
|
if (!Dyn::Strip(lines[i]).empty()) {
|
|
parts.hasBody = true;
|
|
break;
|
|
}
|
|
}
|
|
return parts;
|
|
}
|
|
|
|
CommitCode CommitCodeOf(const std::map<std::int64_t, std::string>& patches) {
|
|
CommitCode cc;
|
|
std::set<std::string> bases;
|
|
std::vector<std::string> codes;
|
|
for (const auto& [k, text] : patches) {
|
|
PatchParts parts = PartsOf(text);
|
|
cc.body[k] = parts.hasBody;
|
|
cc.code[k] = Sha(parts.code);
|
|
cc.docs[k] = std::move(parts.docs);
|
|
cc.message[k] = Sha(parts.message);
|
|
if (!parts.base.empty()) bases.insert(parts.base);
|
|
codes.push_back(cc.code[k]);
|
|
cc.upto[k] = Sha(Dyn::Join(codes, "\n"));
|
|
}
|
|
cc.range = Sha(Dyn::Join(codes, "\n"));
|
|
cc.base = Dyn::Join(std::vector<std::string>(bases.begin(), bases.end()), ",");
|
|
return cc;
|
|
}
|
|
|
|
std::pair<std::string, std::vector<std::string>> CodeKeyParts(std::string_view unitId, const CommitCode& cc) {
|
|
const std::size_t colon = unitId.find(':');
|
|
if (colon == std::string_view::npos) throw Dyn::Error("ValueError", "not enough values to unpack (expected 2, got 1)");
|
|
const std::string_view pk = unitId.substr(0, colon);
|
|
const std::string_view rest = unitId.substr(colon + 1);
|
|
const std::string_view number = pk.empty() ? pk : pk.substr(1);
|
|
const std::optional<std::int64_t> k = Dyn::ParseInt(number);
|
|
if (!k) throw Dyn::Error("ValueError", Dyn::IntError(number));
|
|
std::vector<std::string> deps = {std::format("base={}", cc.base)};
|
|
if (rest == "outside" || rest == "absent") {
|
|
const auto code = cc.code.find(*k);
|
|
if (code == cc.code.end()) throw Dyn::Error("KeyError", std::to_string(*k));
|
|
deps.push_back(std::format("commit={}:{}", *k, code->second));
|
|
deps.push_back(std::format("range={}", cc.range));
|
|
deps.push_back(std::format("docs={}", Dyn::Join(cc.docs.at(*k), "|")));
|
|
if (rest == "absent") deps.push_back(std::format("body={}", cc.body.at(*k) ? "yes" : "no"));
|
|
return {std::format("code:{}", rest), deps};
|
|
}
|
|
const std::size_t hash = rest.rfind('#');
|
|
if (hash == std::string_view::npos) throw Dyn::Error("ValueError", "not enough values to unpack (expected 2, got 1)");
|
|
const std::string_view tag = rest.substr(hash + 1);
|
|
return {std::format("code:{}#{}", rest.substr(0, hash), tag == "hdr" ? "hdr" : "hunk"), deps};
|
|
}
|
|
|
|
std::optional<std::string> CodeReuseKey(const Json& job, std::string_view lensFile, std::string_view unitText, const CommitCode& cc,
|
|
std::span<const std::string> inForceIds, std::string_view charterSha, std::int32_t promptVersion,
|
|
std::string_view environment) {
|
|
auto [ident, deps] = CodeKeyParts(Dyn::At(job, "unit").get_ref<const std::string&>(), cc);
|
|
std::ranges::move(EnvDeps(lensFile, environment), std::back_inserter(deps));
|
|
return ReuseKey(job, lensFile, CodeNorm(unitText), deps, {inForceIds.begin(), inForceIds.end()}, charterSha, promptVersion, ident);
|
|
}
|
|
|
|
std::optional<std::string> FunctionalReuseKey(const Json& job, std::string_view lensFile, std::string_view unitText, const CommitCode& cc,
|
|
std::span<const std::string> inForceIds, std::string_view charterSha, std::int32_t promptVersion,
|
|
std::string_view environment) {
|
|
auto [ident, deps] = CodeKeyParts(Dyn::At(job, "unit").get_ref<const std::string&>(), cc);
|
|
deps.push_back(std::format("range={}", cc.range));
|
|
std::ranges::move(EnvDeps(lensFile, environment), std::back_inserter(deps));
|
|
return ReuseKey(job, lensFile, CodeNorm(unitText), deps, {inForceIds.begin(), inForceIds.end()}, charterSha, promptVersion,
|
|
std::format("functional:{}", ident));
|
|
}
|
|
|
|
std::optional<std::string> BlockReuseKey(const Json& job, std::string_view lensFile, std::string_view unitText, const CommitCode& cc,
|
|
std::span<const std::string> inForceIds, std::string_view charterSha, std::int32_t promptVersion,
|
|
std::string_view environment) {
|
|
if (LensOf(lensFile).key == "functional") {
|
|
return FunctionalReuseKey(job, lensFile, unitText, cc, inForceIds, charterSha, promptVersion, environment);
|
|
}
|
|
return CodeReuseKey(job, lensFile, unitText, cc, inForceIds, charterSha, promptVersion, environment);
|
|
}
|
|
|
|
std::optional<std::string> WholeCodeKey(const Json& job, std::string_view lensFile, const CommitCode& cc, std::span<const std::string> inForceIds,
|
|
std::string_view charterSha, std::int32_t promptVersion, std::string_view environment) {
|
|
if (!LensOf(lensFile).wholeReuse || Dyn::At(job, "kind") != "whole") return std::nullopt;
|
|
std::string body;
|
|
for (const auto& [k, has] : cc.body) {
|
|
if (!body.empty()) body.push_back(',');
|
|
body.push_back(has ? 'y' : 'n');
|
|
}
|
|
std::vector<std::string> docs;
|
|
for (const auto& [k, files] : cc.docs) docs.push_back(Dyn::Join(files, "|"));
|
|
std::vector<std::string> deps = {
|
|
std::format("base={}", cc.base),
|
|
std::format("range={}", cc.range),
|
|
std::format("body={}", body),
|
|
std::format("docs={}", Dyn::Join(docs, ";")),
|
|
};
|
|
std::ranges::move(EnvDeps(lensFile, environment), std::back_inserter(deps));
|
|
std::vector<std::string> ids(inForceIds.begin(), inForceIds.end());
|
|
std::ranges::sort(ids);
|
|
const std::vector<std::string> parts = {
|
|
std::string(lensFile),
|
|
"code:whole",
|
|
Sha(Dyn::Join(deps, "\n")),
|
|
Dyn::Join(ids, ","),
|
|
std::string(charterSha),
|
|
std::to_string(promptVersion),
|
|
Dyn::Has(job, "model") ? Dyn::Get(job, "model").get<std::string>() : std::string(),
|
|
std::string(ReuseKeyEffortSlot),
|
|
};
|
|
return Sha(Dyn::Join(parts, "|")).substr(0, 16);
|
|
}
|
|
|
|
std::int64_t ReadsOf(const Json& entry) {
|
|
if (!Dyn::Truthy(entry)) return 0;
|
|
const Json& reads = Dyn::Get(entry, "reads");
|
|
if (!Dyn::Truthy(reads)) return 1;
|
|
if (reads.is_string()) return static_cast<std::int64_t>(Dyn::Length(reads.get_ref<const std::string&>()));
|
|
return static_cast<std::int64_t>(reads.size());
|
|
}
|
|
|
|
bool ReadIn(const Json& entry, std::string_view hash8, std::int64_t round) {
|
|
const Json& reads = Dyn::Or(Dyn::Get(entry, "reads"), Dyn::EmptyList());
|
|
return std::ranges::any_of(reads, [&](const Json& read) {
|
|
return Dyn::Equal(Dyn::Get(read, "hash"), Json(hash8)) && Dyn::Equal(Dyn::Get(read, "round"), Json(round));
|
|
});
|
|
}
|
|
|
|
Json Rebind(const Json& entry, std::string_view unitId) {
|
|
Json out = entry;
|
|
out["unit"] = unitId;
|
|
Json coverage = Dyn::At(out, "coverage");
|
|
if (!coverage.is_object()) throw Dyn::Error("TypeError", std::format("'{}' object is not iterable", Dyn::TypeName(coverage)));
|
|
coverage["item"] = unitId;
|
|
out["coverage"] = std::move(coverage);
|
|
for (const std::string_view field : {std::string_view("findings"), std::string_view("notes")}) {
|
|
if (!Dyn::Has(out, field)) continue;
|
|
for (Json& item : out[std::string(field)]) {
|
|
if (Dyn::Get(item, "items").is_array()) item["items"] = Json::array({unitId});
|
|
}
|
|
}
|
|
return out;
|
|
}
|
|
|
|
Json Estimate(std::span<const Json> jobs, std::int64_t prefixBytes, const std::map<std::string, std::int64_t>& lensLayerBytes,
|
|
std::int64_t tailBytes, double tpb, std::int64_t sysTokens, std::int64_t outPerTurn) {
|
|
const double prefix = static_cast<double>(prefixBytes) / tpb + static_cast<double>(sysTokens);
|
|
double cacheRead = 0.0;
|
|
double cacheWrite = 0.0;
|
|
double output = 0.0;
|
|
std::int64_t processes = 0;
|
|
std::set<std::pair<std::string, std::string>> seen;
|
|
for (const Json& job : jobs) {
|
|
const std::string kind = Dyn::At(job, "kind").get<std::string>();
|
|
if (kind == "auto-na") continue;
|
|
const std::string lens = Dyn::At(job, "lens").get<std::string>();
|
|
const auto layer = lensLayerBytes.find(lens);
|
|
const double lensTokens = static_cast<double>(layer == lensLayerBytes.end() ? 3000 : layer->second) / tpb;
|
|
const double tail = static_cast<double>(tailBytes) / tpb;
|
|
const std::int64_t turns = kind == "whole" ? 40 : (Dyn::At(job, "lensFile") == FunctionalLens ? 25 : 10);
|
|
if (seen.insert({lens, kind}).second) cacheWrite += prefix + lensTokens;
|
|
cacheWrite += tail + static_cast<double>(turns * 3000);
|
|
std::vector<double> reads;
|
|
for (std::int64_t i = 0; i < turns; ++i) reads.push_back(prefix + lensTokens + tail + static_cast<double>(3000 * i));
|
|
cacheRead += Dyn::Sum(reads);
|
|
output += static_cast<double>(turns * outPerTurn);
|
|
++processes;
|
|
}
|
|
const double costUnits = 1.25 * cacheWrite + 0.1 * cacheRead + 5.0 * output;
|
|
const double listUsd = Dyn::Round((6.25 * cacheWrite + 0.5 * cacheRead + 25.0 * output) / 1e6, 2);
|
|
return {
|
|
{"cacheRead", cacheRead},
|
|
{"cacheWrite", cacheWrite},
|
|
{"output", output},
|
|
{"processes", processes},
|
|
{"costUnits", costUnits},
|
|
{"listUsd", listUsd},
|
|
};
|
|
}
|
|
}
|