// SPDX-License-Identifier: AGPL-3.0-only // SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts® module Crafter.Review:Inventory_impl; import :Inventory; import :Json; import :Text; import :Artifact; import :Plan; import std; namespace fs = std::filesystem; namespace Crafter::Review::Inventory { namespace { constexpr std::string_view Usage = R"(Partition a frozen artifact into review units for the coverage contract. The inventory is MECHANICAL: no agent decides what the units are. Every non-blank line of the artifact belongs to exactly one unit, and the tool fails if that invariant does not hold. Fixed slots are added so that what is NOT in the text still has a row in the coverage matrix. crafter-review inventory reply > gate-state/inventory-.json crafter-review inventory series > gate-state/inventory-.json crafter-review inventory commits > gate-state/inventory-.json (own-repo push: git format-patch of a range) Reply mode units: nl:intro, nl:, anchor, RN, :N, :Na, x:N, refs, whole:* - or, for a plain mail without the dated anchor: hdr, quote:N, para:N, trailers, whole:*. Series mode: cover:header, cover:N and the units below (every hunk whole), whole:* from the series slots. Commits mode units (ids): pK:subject commit K's subject pK:msg commit K's message body (before the trailers) - only when non-empty pK:trailers commit K's trailer block - only when non-empty pK:diffstat commit K's diffstat pK:#hdr the diff header of in commit K (mode, rename, blobs) pK:#H hunk H of in commit K; a hunk of a documentation file (DOC_FILE) that holds more than one heading-delimited part is cut into #H.1, #H.2, ... (doc_parts) pK:outside fixed slot: the in-repository users/callers of what commit K changes (outside its diff) pK:absent fixed slot: what a reviewer expects in commit K and does not find whole:* fixed slots (WHOLE_COMMITS) )"; // re.match(r'\s{0,3}(```|~~~)', body): a fence, indented by three // whitespace characters at most. bool OpensFence(std::string_view body) { std::size_t at = 0; for (std::int32_t i = 0; i < 3 && at < body.size(); ++i) { const auto [c, used] = Dyn::CodePointAt(body, at); if (!Dyn::IsSpace(c)) break; at += used; } const std::string_view rest = body.substr(at); return rest.starts_with("```") || rest.starts_with("~~~"); } // re.match(r'#{1,6}\s', body): an ATX heading. bool IsHeading(std::string_view body) { std::size_t hashes = 0; while (hashes < body.size() && body[hashes] == '#') ++hashes; if (hashes == 0 || hashes > 6 || hashes >= body.size()) return false; return Dyn::IsSpace(Dyn::CodePointAt(body, hashes).first); } // re.match(r'^[A-Z][A-Za-z-]+: ', line): a trailer such as // `Signed-off-by: `. bool IsTrailer(std::string_view line) { if (line.empty() || line[0] < 'A' || line[0] > 'Z') return false; std::size_t at = 1; while (at < line.size() && ((line[at] >= 'A' && line[at] <= 'Z') || (line[at] >= 'a' && line[at] <= 'z') || line[at] == '-')) ++at; return at > 1 && line.substr(at).starts_with(": "); } bool AnyNonBlank(std::span lines, std::int64_t from, std::int64_t to) { const auto [a, b] = Dyn::SliceBounds(lines.size(), from, to); return std::any_of(lines.begin() + static_cast(a), lines.begin() + static_cast(b), [](const std::string& line) { return !Dyn::Strip(line).empty(); }); } std::span Lines(std::span lines, std::int64_t from, std::int64_t to) { const auto [a, b] = Dyn::SliceBounds(lines.size(), from, to); return lines.subspan(a, b - a); } Json Unit(std::string id, std::int64_t first, std::int64_t last, std::string excerpt) { return {{"id", std::move(id)}, {"lines", {first, last}}, {"excerpt", std::move(excerpt)}}; } // ^On .+ wrote:$ on a stripped line: an attribution. bool IsAttribution(std::string_view stripped) { return stripped.starts_with("On ") && stripped.ends_with(" wrote:") && stripped.size() > 10 && !stripped.contains('\n'); } // ^[A-Z][A-Za-z-]+: \S : a trailer line of a mail (Tested-by: ...). bool IsMailTrailer(std::string_view line) { if (line.empty() || line[0] < 'A' || line[0] > 'Z') return false; std::size_t at = 1; while (at < line.size() && ((line[at] >= 'A' && line[at] <= 'Z') || (line[at] >= 'a' && line[at] <= 'z') || line[at] == '-')) ++at; if (at < 2 || !line.substr(at).starts_with(": ")) return false; return at + 2 < line.size() && !Dyn::IsSpace(Dyn::CodePointAt(line, at + 2).first); } // ^[\w.-]+/\d{4}-[^\s]+\.patch$ : a bare patch path, the header of a // per-file group of the camera review. bool IsPatchPath(std::string_view text) { std::size_t at = 0; std::size_t head = 0; while (at < text.size()) { const auto [c, used] = Dyn::CodePointAt(text, at); if (!(Dyn::IsWord(c) || c == '.' || c == '-')) break; at += used; ++head; } if (head == 0 || at >= text.size() || text[at] != '/') return false; ++at; if (Dyn::DecimalRun(text, at) != 4) return false; at += 4; if (at >= text.size() || text[at] != '-') return false; ++at; const std::size_t rest = Dyn::NonSpaceRun(text, at); if (rest <= 6 || at + rest != text.size()) return false; return text.ends_with(".patch"); } // ^R\d+\. and ^\d+\. : a reversal or a numbered item; the id is the // text before the dot. std::optional NumberedHead(std::string_view line, bool reversal) { std::size_t at = 0; if (reversal) { if (!line.starts_with('R')) return std::nullopt; at = 1; } const std::size_t digits = Dyn::DecimalRun(line, at); if (digits == 0 || !line.substr(at + digits).starts_with(". ")) return std::nullopt; return std::string(line.substr(0, at + digits)); } // ^\s+[a-z]\. \S : a lettered sub-item; the letter. std::optional LetteredHead(std::string_view line) { const std::size_t spaces = Dyn::SpaceRun(line, 0); if (spaces == 0 || spaces + 3 >= line.size()) return std::nullopt; const char letter = line[spaces]; if (letter < 'a' || letter > 'z' || !line.substr(spaces + 1).starts_with(". ")) return std::nullopt; if (Dyn::IsSpace(Dyn::CodePointAt(line, spaces + 3).first)) return std::nullopt; return letter; } // ^\[1\] https?:// bool IsFirstReference(std::string_view line) { return line.starts_with("[1] http://") || line.starts_with("[1] https://"); } std::span Range(const std::vector& lines, std::size_t from, std::size_t to) { return std::span(lines).subspan(from, to - from); } std::string ReprInts(std::span values) { std::string out = "["; for (std::size_t i = 0; i < values.size(); ++i) out += std::format("{}{}", i ? ", " : "", values[i]); return std::format("{}]", out); } } std::vector> DocParts(std::span lines, std::size_t a, std::size_t b) { std::vector> parts; std::size_t start = a; bool hasBody = false; bool fenced = false; for (std::size_t i = a + 1; i < b; ++i) { const std::string& line = lines[i]; // l[:1] in (' ', '+') and l[1:]: the marker and the line after it. const bool isNew = line.starts_with(' ') || line.starts_with('+'); const std::string_view body = line.empty() ? std::string_view() : std::string_view(line).substr(Dyn::CodePointAt(line, 0).second); if (isNew && OpensFence(body)) fenced = !fenced; if (isNew && !fenced && IsHeading(body)) { if (hasBody) { parts.emplace_back(start, i); start = i; hasBody = false; } } else if (!Dyn::Strip(line).empty() && !Dyn::Strip(body).empty()) { hasBody = true; } } parts.emplace_back(start, b); return parts; } std::string Excerpt(std::span lines) { for (const std::string& line : lines) { const std::string stripped = Dyn::Strip(line); if (!stripped.empty()) return Dyn::Slice(stripped, 0, 90); } return ""; } Series PartitionSeries(const fs::path& dir, bool splitDocs, bool kernel) { Series series; series.units = Json::array(); Json& units = series.units; std::int64_t offset = 0; for (const std::string& name : Artifact::PatchFiles(dir)) { const std::string text = Dyn::ReadText(dir / name); const std::vector lines = Dyn::Split(text, "\n"); const auto count = static_cast(lines.size()); const std::optional number = Artifact::PatchNumber(name); const std::int64_t k = number ? *number : -1; const std::string pid = k == 0 ? std::string("cover") : std::format("p{}", k); const std::int64_t base = offset; std::optional subject; for (std::int64_t i = 0; i < count; ++i) { if (lines[static_cast(i)].starts_with("Subject:")) { subject = i; break; } } std::int64_t body0 = 1; if (subject) { for (std::int64_t i = *subject + 1; i < count; ++i) { if (Dyn::Strip(lines[static_cast(i)]).empty()) { body0 = i + 1; break; } } } // >= : a body-less commit's first body line IS the '---'. std::int64_t diff0 = count; for (std::int64_t i = body0; i < count; ++i) { if (lines[static_cast(i)].starts_with("---")) { diff0 = i; break; } } if (pid == "cover") { // The header as one unit, the paragraphs of the body. units.push_back(Unit("cover:header", base + 1, base + body0, subject ? Dyn::Slice(Dyn::Strip(lines[static_cast(*subject)]), 0, 90) : std::string())); std::int64_t i = body0; std::int64_t pn = 0; while (i < count) { if (Dyn::Strip(lines[static_cast(i)]).empty()) { ++i; continue; } std::int64_t j = i; while (j < count && !Dyn::Strip(lines[static_cast(j)]).empty()) ++j; ++pn; units.push_back(Unit(std::format("cover:{}", pn), base + i + 1, base + j, Excerpt(Lines(lines, i, j)))); i = j; } series.lines.insert(series.lines.end(), lines.begin(), lines.end()); offset += count; continue; } units.push_back(Unit(std::format("{}:subject", pid), base + 1, base + body0, subject ? Dyn::Slice(Dyn::Strip(lines[static_cast(*subject)]), 0, 90) : std::string())); std::int64_t t = diff0; while (t - 1 > body0 && (IsTrailer(lines[static_cast(t - 1)]) || Dyn::Strip(lines[static_cast(t - 1)]).empty())) --t; if (AnyNonBlank(lines, body0, t)) { units.push_back(Unit(std::format("{}:msg", pid), base + body0 + 1, base + t, Excerpt(Lines(lines, body0, t)))); } if (AnyNonBlank(lines, t, diff0)) { units.push_back(Unit(std::format("{}:trailers", pid), base + t + 1, base + diff0, Excerpt(Lines(lines, t, diff0)))); } const std::int64_t statStart = diff0; std::int64_t firstDiff = count; for (std::int64_t i = diff0 + 1; i < count; ++i) { if (lines[static_cast(i)].starts_with("diff --git")) { firstDiff = i; break; } } units.push_back(Unit(std::format("{}:diffstat", pid), base + statStart + 1, base + firstDiff, Excerpt(Lines(lines, statStart, firstDiff)))); std::string currentFile; std::int64_t hunk = 0; std::optional hunkStart; auto closeHunk = [&](std::int64_t upto) { if (!hunkStart) return; std::vector> parts; if (splitDocs && Plan::IsDocPath(currentFile)) { parts = DocParts(lines, static_cast(*hunkStart), static_cast(upto)); } else { parts = {{static_cast(*hunkStart), static_cast(upto)}}; } for (std::size_t n = 0; n < parts.size(); ++n) { const auto [s, e] = parts[n]; const std::string suffix = parts.size() > 1 ? std::format(".{}", n + 1) : std::string(); units.push_back(Unit(std::format("{}:{}#{}{}", pid, currentFile, hunk, suffix), base + static_cast(s) + 1, base + static_cast(e), Excerpt(Lines(lines, static_cast(s), static_cast(s) + 1)))); } }; std::int64_t i = firstDiff; while (i < count) { const std::string& line = lines[static_cast(i)]; if (line.starts_with("diff --git")) { closeHunk(i); hunkStart.reset(); currentFile = Dyn::Strip(Dyn::Split(line, " b/").back()); hunk = 0; const std::int64_t header = i; std::int64_t j = i + 1; while (j < count && !lines[static_cast(j)].starts_with("@@") && !lines[static_cast(j)].starts_with("diff --git")) ++j; units.push_back(Unit(std::format("{}:{}#hdr", pid, currentFile), base + header + 1, base + j, Dyn::Slice(Dyn::Strip(line), 0, 90))); i = j; continue; } if (line.starts_with("@@")) { closeHunk(i); ++hunk; hunkStart = i; } ++i; } closeHunk(count); units.push_back(Unit(std::format("{}:outside", pid), 0, 0, kernel ? "FIXED SLOT: in-tree users/callers of what this patch changes, outside its diff" : "FIXED SLOT: in-repository users/callers of what this commit changes, outside its diff")); units.push_back(Unit(std::format("{}:absent", pid), 0, 0, kernel ? "FIXED SLOT: what a reviewer expects in this patch and does not find" : "FIXED SLOT: what a reviewer expects in this commit and does not find")); series.lines.insert(series.lines.end(), lines.begin(), lines.end()); offset += count; } return series; } Series PartitionMail(std::string_view text) { Series out; out.lines = Dyn::Split(text, "\n"); out.units = Json::array(); const std::vector& lines = out.lines; const std::size_t n = lines.size(); std::size_t hdrEnd = n; for (std::size_t i = 0; i < n; ++i) { if (Dyn::Strip(lines[i]).empty()) { hdrEnd = i; break; } } if (hdrEnd > 0) out.units.push_back(Unit("hdr", 1, static_cast(hdrEnd), Excerpt(Range(lines, 0, hdrEnd)))); std::size_t i = hdrEnd; std::int64_t q = 0; std::int64_t pn = 0; std::int64_t tr = 0; while (i < n) { if (Dyn::Strip(lines[i]).empty()) { ++i; continue; } std::size_t j = i; std::string uid; if (lines[i].starts_with('>') || IsAttribution(Dyn::Strip(lines[i]))) { // A quoted block: the attribution line (if any) and every // following quoted line; a blank line inside the quote is // part of it only when a quoted line follows. while (j < n && (lines[j].starts_with('>') || IsAttribution(Dyn::Strip(lines[j])) || (Dyn::Strip(lines[j]).empty() && j + 1 < n && lines[j + 1].starts_with('>')))) ++j; ++q; uid = std::format("quote:{}", q); } else { while (j < n && !Dyn::Strip(lines[j]).empty() && !lines[j].starts_with('>')) ++j; if (std::all_of(lines.begin() + static_cast(i), lines.begin() + static_cast(j), [](const std::string& l) { return IsMailTrailer(l); })) { ++tr; uid = tr == 1 ? std::string("trailers") : std::format("trailers:{}", tr); } else { ++pn; uid = std::format("para:{}", pn); } } out.units.push_back(Unit(uid, static_cast(i) + 1, static_cast(j), Excerpt(Range(lines, i, j)))); i = j; } return out; } Series PartitionReply(std::string_view text) { Series out; out.lines = Dyn::Split(text, "\n"); out.units = Json::array(); const std::vector& lines = out.lines; const std::size_t n = lines.size(); // The block starts at the dated anchor line; a mail without one is a // plain reply. std::optional anchor; for (std::size_t i = 0; i < n; ++i) { if (lines[i].starts_with("Dates: every")) { anchor = i; break; } } if (!anchor) return PartitionMail(text); const std::size_t blk = *anchor; // The author's part: paragraphs grouped by the patch file they // discuss - a group starts at a paragraph that is a bare path ending // in .patch; everything before the first such header is nl:intro. struct Group { std::string id; std::size_t start; std::size_t end; }; std::vector groups; std::size_t i = 0; while (i < blk) { if (Dyn::Strip(lines[i]).empty()) { ++i; continue; } std::size_t j = i; while (j < blk && !Dyn::Strip(lines[j]).empty()) ++j; const std::string first = Dyn::Strip(lines[i]); if (IsPatchPath(first) && j - i == 1) { groups.push_back({std::format("nl:{}", first), i, j}); } else if (!groups.empty()) { groups.back().end = j; } else { groups.push_back({"nl:intro", i, j}); } i = j; } for (const Group& g : groups) out.units.push_back(Unit(g.id, static_cast(g.start) + 1, static_cast(g.end), Excerpt(Range(lines, g.start, g.end)))); // The anchor paragraph. std::size_t j = blk; while (j < n && !Dyn::Strip(lines[j]).empty()) ++j; out.units.push_back(Unit("anchor", static_cast(blk) + 1, static_cast(j), Excerpt(Range(lines, blk, j)))); // The rest of the block: reversals, sections, items, sub-items, // bullets, refs. std::optional refsStart; for (std::size_t r = 0; r < n; ++r) { if (IsFirstReference(lines[r])) { refsStart = r; break; } } if (!refsStart) throw InvariantError("StopIteration"); static const std::map Sections = {{"camss-milos", "camss"}, {"ov13b10-of", "ov13b10"}, {"milos-dts-camss", "dts"}, {"across the three series", "x"}}; std::optional sec; std::optional item; std::int64_t bullet = 0; std::vector> starts; i = j; while (i < *refsStart) { const std::string& l = lines[i]; const std::string s = Dyn::Strip(l); if (s.empty()) { ++i; continue; } const std::string low = Dyn::Lower(s); const bool indented = l.starts_with(' '); if (Sections.contains(low) && !indented) { sec = Sections.at(low); item.reset(); bullet = 0; starts.emplace_back(i, std::format("{}:header", *sec)); } else if (const std::optional r = NumberedHead(l, true)) { starts.emplace_back(i, *r); } else if (l.starts_with("Reversals of the")) { starts.emplace_back(i, "reversals:header"); } else if (const std::optional num = NumberedHead(l, false); num && sec) { item = *num; starts.emplace_back(i, std::format("{}:{}", *sec, *item)); } else if (const std::optional letter = LetteredHead(l); letter && sec && item) { starts.emplace_back(i, std::format("{}:{}{}", *sec, *item, *letter)); } else if (l.starts_with("- ") && sec == "x") { ++bullet; starts.emplace_back(i, std::format("x:{}", bullet)); } else if (!indented && starts.empty()) { starts.emplace_back(i, std::format("para:{}", i + 1)); } else if (!indented && !NumberedHead(l, false) && !NumberedHead(l, true) && Dyn::Strip(lines[i - 1]).empty() && !sec) { starts.emplace_back(i, std::format("para:{}", i + 1)); } ++i; } for (std::size_t k = 0; k < starts.size(); ++k) { const auto& [a, uid] = starts[k]; std::size_t b = k + 1 < starts.size() ? starts[k + 1].first : *refsStart; while (b > a && Dyn::Strip(lines[b - 1]).empty()) --b; out.units.push_back(Unit(uid, static_cast(a) + 1, static_cast(b), Excerpt(Range(lines, a, b)))); } std::size_t b = n; while (b > *refsStart && Dyn::Strip(lines[b - 1]).empty()) --b; out.units.push_back(Unit("refs", static_cast(*refsStart) + 1, static_cast(b), Excerpt(Range(lines, *refsStart, b)))); return out; } Built Build(std::string_view mode, std::string_view artifact) { if (mode != "reply" && mode != "series" && mode != "commits") throw InvariantError(std::format("inventory: mode {} is not reply, series or commits", Dyn::Repr(mode))); Series series; std::string h8; std::span whole; if (mode == "reply") { series = PartitionReply(Dyn::ReadText(fs::path(artifact))); h8 = Artifact::Sha256File(fs::path(artifact)).substr(0, 8); whole = WholeReply; } else { series = PartitionSeries(fs::path(artifact), mode == "commits", mode == "series"); h8 = Artifact::ArtifactHash("commits", fs::path(artifact)).substr(0, 8); whole = mode == "commits" ? std::span(WholeCommits) : std::span(WholeSeries); } Json& units = series.units; std::map owner; for (const Json& unit : units) { const std::int64_t a = Dyn::At(unit, "lines")[0].get(); const std::int64_t b = Dyn::At(unit, "lines")[1].get(); const std::string& id = Dyn::At(unit, "id").get_ref(); for (std::int64_t line = a; line <= b; ++line) { if (line == 0) continue; if (const auto held = owner.find(line); held != owner.end()) { throw InvariantError(std::format("INVARIANT: line {} in both {} and {}", line, held->second, id)); } owner[line] = id; } } std::vector uncovered; for (std::size_t i = 0; i < series.lines.size(); ++i) { const auto line = static_cast(i) + 1; if (!Dyn::Strip(series.lines[i]).empty() && !owner.contains(line)) uncovered.push_back(line); } if (!uncovered.empty()) { const std::span first(uncovered.data(), std::min(uncovered.size(), 10)); throw InvariantError(std::format("INVARIANT: {} non-blank lines in no unit, first: {}", uncovered.size(), ReprInts(first))); } for (const WholeSlot& slot : whole) units.push_back(Unit(std::string(slot.id), 0, 0, std::format("FIXED SLOT: {}", slot.description))); std::map seen; for (const Json& unit : units) ++seen[Dyn::At(unit, "id").get()]; Json duplicates = Json::array(); for (const auto& [id, times] : seen) { if (times > 1) duplicates.push_back(id); } if (!duplicates.empty()) throw InvariantError(std::format("INVARIANT: duplicate ids {}", Dyn::ReprValue(duplicates))); const std::size_t total = units.size(); Built built; built.out = { {"mode", mode}, {"artifact", artifact}, {"hash8", h8}, {"units", std::move(units)}, {"invariant", "every non-blank line belongs to exactly one unit; fixed slots have lines [0,0]"}, {"count", total}, }; built.lines = std::move(series.lines); return built; } std::int32_t Main(std::span args) { if (args.size() != 2 || (args[0] != "reply" && args[0] != "series" && args[0] != "commits")) { std::print(std::cerr, "{}", Usage); return 1; } try { const Built built = Build(args[0], args[1]); std::println("{}", DumpFile(built.out)); return 0; } catch (const InvariantError& error) { std::println(std::cerr, "{}", error.what()); return 1; } catch (const std::exception& error) { std::println(std::cerr, "inventory: {}", error.what()); return 1; } } }