Crafter.Review/implementations/Crafter.Review-Normalize.cpp

// SPDX-License-Identifier: AGPL-3.0-only
// SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts®

module Crafter.Review:Normalize_impl;
import :Normalize;
import :Json;
import :Text;
import :Schema;
import :Plan;
import std;

namespace Crafter::Review::Normalize {
    namespace {
        // A map key or set member, as text: equal values give equal keys. A
        // list or an object cannot be one (TypeError).
        std::string KeyOf(const Json& value) {
            if (value.is_array() || value.is_object()) throw Dyn::Error("TypeError", std::format("unhashable type: '{}'", Dyn::TypeName(value)));
            return DumpCompactSorted(value);
        }

        // The key's value, or the fallback when the key is absent.
        const Json& GetOr(const Json& dict, std::string_view key, const Json& fallback) {
            return Dyn::Has(dict, key) ? Dyn::At(dict, key) : fallback;
        }

        // `list + list`
        Json Concat(const Json& a, const Json& b) {
            if (!a.is_array() || !b.is_array()) throw Dyn::Error("TypeError", "can only concatenate list (not \"NoneType\") to list");
            Json out = a;
            for (const Json& item : b) out.push_back(item);
            return out;
        }

        // The hashes a disposition is bound to: this run's, its own and the
        // ones it was carried from.
        Json Chain(std::string_view hash, const Json& d) {
            Json chain = Json::array();
            auto add = [&](const Json& h) {
                if (Dyn::Truthy(h)) chain.push_back(h);
            };
            add(Json(hash));
            add(Dyn::Get(d, "hash"));
            for (const Json& carried : GetOr(d, "carried", Dyn::EmptyList())) add(Dyn::Get(carried, "hash"));
            return chain;
        }

        // Reaffirmed by the owner, or a contest of it rejected, on this
        // binding.
        bool Settled(std::string_view hash, const Json& d) {
            const Json chain = Chain(hash, d);
            const Json records = Concat(GetOr(d, "reaffirmed", Dyn::EmptyList()), GetOr(d, "contestRejected", Dyn::EmptyList()));
            return std::ranges::any_of(records, [&](const Json& record) { return Dyn::In(Dyn::Get(record, "hash"), chain); });
        }

        bool In(std::string_view verdict, std::span<const std::string_view> vocabulary) {
            return std::ranges::contains(vocabulary, verdict);
        }

        bool InVocabulary(const Json& verdict, std::span<const std::string_view> vocabulary) {
            return verdict.is_string() && In(verdict.get_ref<const std::string&>(), vocabulary);
        }

        std::string JoinStrings(std::span<const Json> items, std::string_view separator) {
            std::vector<std::string> parts;
            for (const Json& item : items) {
                if (!item.is_string()) throw Dyn::Error("TypeError", std::format("sequence item {}: expected str instance, {} found", parts.size(), Dyn::TypeName(item)));
                parts.push_back(item.get<std::string>());
            }
            return Dyn::Join(parts, separator);
        }

        // An ordered map keyed by value (KeyOf): the order of first insertion,
        // and the last value set.
        struct OrderedMap {
            std::vector<std::pair<Json, Json>> items;
            std::map<std::string, std::size_t> index;

            Json* Find(const Json& key) {
                const auto found = index.find(KeyOf(key));
                return found == index.end() ? nullptr : &items[found->second].second;
            }

            void Set(const Json& key, Json value) {
                if (Json* existing = Find(key)) {
                    *existing = std::move(value);
                    return;
                }
                index[KeyOf(key)] = items.size();
                items.emplace_back(key, std::move(value));
            }
        };
    }

    Json ResolveId(const Json& id, std::span<const std::string> universe) {
        if (id.is_string()) {
            const std::string& text = id.get_ref<const std::string&>();
            if (std::ranges::contains(universe, text)) return id;
            std::vector<const std::string*> matches;
            for (const std::string& candidate : universe) {
                if (candidate.starts_with(text)) matches.push_back(&candidate);
            }
            return matches.size() == 1 ? Json(*matches.front()) : id;
        }
        // `s in universe` is false for anything but a str; u.startswith(s)
        // then raises, unless there is no u to ask.
        if (universe.empty()) return id;
        throw Dyn::Error("TypeError", std::format("startswith first arg must be str or a tuple of str, not {}", Dyn::TypeName(id)));
    }

    Json NormalizeOutput(Json& v, std::span<const Json> inForce, std::string_view lensFile, std::string_view hash, const Json& inventory,
                         std::span<const std::string> allIds) {
        std::map<std::string, const Json*> byId;
        for (const Json& d : inForce) byId[KeyOf(Dyn::At(d, "id"))] = &d;
        auto disposition = [&](const Json& id) -> const Json* {
            const auto found = byId.find(KeyOf(id));
            return found == byId.end() ? nullptr : found->second;
        };
        Json violations = Json::array();
        Json contests = Json::array();
        Json cross = Json::array();
        Json logs = Json::array();
        Json findings = Json::array();
        for (const Json& f : Dyn::Or(Dyn::Get(v, "findings"), Dyn::EmptyList())) {
            if (!f.is_object()) throw Dyn::Error("TypeError", std::format("'{}' object is not iterable", Dyn::TypeName(f)));
            Json out = f;
            bool discharged = false;
            if (Dyn::Truthy(Dyn::Get(f, "dischargedBy"))) {
                const Json* d = disposition(Dyn::At(f, "dischargedBy"));
                if (!d || !Dyn::Truthy(*d)) {
                    logs.push_back(std::format("{}: dischargedBy={} names no disposition in force - treated as undischarged", lensFile, Dyn::Str(Dyn::At(f, "dischargedBy"))));
                    out.erase("dischargedBy");
                } else {
                    discharged = true;
                    const Json& lenses = Dyn::Get(*d, "lenses");
                    if (Dyn::Truthy(lenses) && !Dyn::In(Json(lensFile), Dyn::At(*d, "lenses"))) {
                        cross.push_back({{"id", Dyn::At(*d, "id")}, {"lens", lensFile}, {"dispositionLenses", Dyn::At(*d, "lenses")}});
                    }
                }
            }
            const Json& c = Dyn::Get(f, "contests");
            if (Dyn::Truthy(c) && Dyn::Truthy(Dyn::Get(c, "id"))) {
                const Json* d = disposition(Dyn::At(c, "id"));
                if (!d || !Dyn::Truthy(*d)) {
                    logs.push_back(std::format("{}: contests id={} names no disposition in force - ignored", lensFile, Dyn::Str(Dyn::At(c, "id"))));
                    out.erase("contests");
                } else if (Settled(hash, *d)) {
                    discharged = true;
                    out["dischargedBy"] = Dyn::At(*d, "id");
                    contests.push_back({{"id", Dyn::At(*d, "id")}, {"lens", lensFile}, {"why", Dyn::Get(c, "why")},
                                        {"outcome", "recorded - entry already reaffirmed/contest-rejected on this binding; finding discharged (owner outranks lens; see Gate 4)"}});
                } else {
                    discharged = false;
                    out.erase("dischargedBy");
                    contests.push_back({{"id", Dyn::At(*d, "id")}, {"lens", lensFile}, {"why", Dyn::Get(c, "why")},
                                        {"outcome", "DISPOSITION CONTESTED - blocks until the contest is verified and the entry reaffirmed, rejected, withdrawn or converted"}});
                }
            }
            out["discharged"] = discharged;
            findings.push_back(std::move(out));
        }

        Json coverage;
        Json missing = Json::array();
        if (!inventory.is_null()) {
            std::vector<std::string> ids;
            for (const Json& unit : inventory) ids.push_back(Dyn::At(unit, "id").get<std::string>());
            const std::span<const std::string> universe = allIds.empty() ? std::span<const std::string>(ids) : allIds;
            auto resolveItems = [&](Json& holder) {
                if (!Dyn::Get(holder, "items").is_array()) return;
                Json resolved = Json::array();
                for (const Json& item : holder["items"]) resolved.push_back(ResolveId(item, universe));
                holder["items"] = std::move(resolved);
            };
            for (Json& f : findings) resolveItems(f);
            // v.get('notes') or []: the notes of the output itself, resolved in
            // place.
            Json noNotes = Json::array();
            Json& notes = Dyn::Truthy(Dyn::Get(v, "notes")) ? v["notes"] : noNotes;
            for (Json& n : notes) {
                if (!n.is_object()) throw Dyn::Error("AttributeError", std::format("'{}' object has no attribute 'get'", Dyn::TypeName(n)));
                resolveItems(n);
            }
            OrderedMap byItem;
            for (const Json& entry : Dyn::Or(Dyn::Get(v, "coverage"), Dyn::EmptyList())) {
                if (!Dyn::Truthy(entry) || !Dyn::Truthy(Dyn::Get(entry, "item"))) continue;
                const Json cid = ResolveId(Dyn::At(entry, "item"), universe);
                if (!byItem.Find(cid)) {
                    Json copy = entry;
                    copy["item"] = cid;
                    byItem.Set(cid, std::move(copy));
                }
            }
            std::vector<Json> unknown;
            for (const auto& [key, value] : byItem.items) {
                if (!key.is_string() || !std::ranges::contains(ids, key.get_ref<const std::string&>())) unknown.push_back(key);
            }
            if (!unknown.empty()) {
                const std::span<const Json> shown(unknown.data(), std::min<std::size_t>(unknown.size(), 5));
                logs.push_back(std::format("{}: coverage names {} id(s) outside this pass's units ({}) - ignored", lensFile, unknown.size(), JoinStrings(shown, ", ")));
            }
            for (const std::string& id : ids) {
                if (!byItem.Find(Json(id))) missing.push_back(id);
            }
            std::set<std::string> cited;
            for (const Json& f : findings) {
                for (const Json& item : Dyn::Or(Dyn::Get(f, "items"), Dyn::EmptyList())) cited.insert(KeyOf(item));
            }
            std::set<std::string> noted;
            for (const Json& n : notes) {
                for (const Json& item : Dyn::Or(Dyn::Get(n, "items"), Dyn::EmptyList())) noted.insert(KeyOf(item));
            }
            // A persona speaks its own vocabulary; every other lens the generic one.
            const bool isPersona = lensFile == Plan::PersonaLens;
            const std::span<const std::string_view> allowed = isPersona ? std::span<const std::string_view>(Schema::PersonaVerdicts) : std::span<const std::string_view>(Schema::GenericVerdicts);
            const std::string vocabulary = Dyn::Join(std::vector<std::string>(allowed.begin(), allowed.end()), "/");
            for (const std::string& id : ids) {
                const Json* c = byItem.Find(Json(id));
                if (!c || !Dyn::Truthy(*c)) continue;
                const Json& verdict = Dyn::At(*c, "verdict");
                const std::string key = KeyOf(Json(id));
                if (!InVocabulary(verdict, allowed)) {
                    logs.push_back(std::format("{}: coverage verdict \"{}\" on {} is not in the vocabulary ({}) - kept as given", lensFile, Dyn::Str(verdict), id, vocabulary));
                }
                if (verdict == "immaterial" && !noted.contains(key)) {
                    logs.push_back(std::format("{}: immaterial on {} with no note citing it (items:[]) - the reason is not on record", lensFile, id));
                }
                if (InVocabulary(verdict, Schema::CoverageBlocking) && !cited.contains(key)) {
                    findings.push_back({
                        {"finding", std::format("coverage: verdict {} on unit {} with no finding written for it (basis: {})", Dyn::Str(verdict), id, Dyn::Str(Dyn::Get(*c, "basis")))},
                        {"evidence", std::format("coverage entry for {}", id)},
                        {"proposedAction", "the lens marked this unit defective and did not write the finding - write it or re-run"},
                        {"items", {id}},
                        {"discharged", false},
                        {"synthesized", true},
                    });
                    logs.push_back(std::format("{}: {} on {} with no finding - synthesized an undischarged finding (fail closed)", lensFile, Dyn::Str(verdict), id));
                }
            }
            coverage = Json::array();
            const Json empty = "";
            for (const std::string& id : ids) {
                const Json* c = byItem.Find(Json(id));
                if (!c) {
                    coverage.push_back({{"item", id}, {"verdict", "MISSING"}, {"basis", ""}});
                } else {
                    coverage.push_back({{"item", Dyn::At(*c, "item")}, {"verdict", Dyn::At(*c, "verdict")}, {"basis", GetOr(*c, "basis", empty)}});
                }
            }
            if (!missing.empty()) {
                const std::span<const Json> shown(&*missing.begin(), std::min<std::size_t>(missing.size(), 8));
                violations.push_back({{"lens", lensFile}, {"kind", std::format("coverage incomplete: {} of {} units unaddressed ({}{}) - lens treated as NOT RETURNED", missing.size(),
                                                                                 ids.size(), JoinStrings(shown, ", "), missing.size() > 8 ? ", ..." : "")}});
            }
        }
        const auto undischarged = std::ranges::count_if(findings, [](const Json& f) { return !Dyn::Truthy(Dyn::At(f, "discharged")); });
        const std::string derived = undischarged ? "FIX FIRST" : "SEND-READY";
        const Json& declared = Dyn::Get(v, "verdict");
        if (Dyn::Truthy(declared) && declared != derived) {
            if (declared == "SEND-READY") {
                violations.push_back({{"lens", lensFile}, {"declared", declared}, {"derived", derived},
                                      {"kind", "SEND-READY declared over undischarged finding(s) - failed closed to FIX FIRST"}});
            } else {
                violations.push_back({{"lens", lensFile}, {"declared", declared}, {"derived", derived},
                                      {"kind", "FIX FIRST declared with every finding discharged and no blocking contest (silent veto) - derived pass stands"}});
            }
        }
        Json out = {
            {"verdict", derived},
            {"declaredVerdict", declared},
            {"findings", std::move(findings)},
            {"notes", Dyn::Or(Dyn::Get(v, "notes"), Dyn::EmptyList())},
            {"coverage", std::move(coverage)},
            {"coverageMissing", std::move(missing)},
            {"undischarged", undischarged},
            {"contests", std::move(contests)},
            {"contractViolations", std::move(violations)},
            {"crossLensDischarges", std::move(cross)},
            {"normalizeLogs", std::move(logs)},
            {"summary", Dyn::Get(v, "summary")},
        };
        if (lensFile == Plan::PersonaLens) {
            // The review this person would send, and the units that are not
            // theirs: what the persona's whole pass hands its block passes.
            out["predictedReview"] = Dyn::Get(v, "predictedReview");
            out["notMine"] = Dyn::Get(v, "notMine");
        }
        return out;
    }

    std::vector<Json> Dedupe(std::span<const Json> items) {
        std::set<std::string> seen;
        std::vector<Json> out;
        for (const Json& item : items) {
            if (seen.insert(DumpCompactSorted(item)).second) out.push_back(item);
        }
        return out;
    }

    Json MergeParts(std::string_view lens, std::string_view lensFile, std::span<const Json> parts, std::span<const std::string> missingUnits,
                    const Json& stoppedEarly) {
        // De-duplicated: a reused part carries the notes and findings its
        // unit-state entry stored, and a note citing two units is stored under
        // both keys - concatenating the parts doubled it every round
        // (2026-10-11).
        std::vector<Json> allFindings;
        std::vector<Json> allNotes;
        for (const Json& part : parts) {
            for (const Json& f : GetOr(part, "findings", Dyn::EmptyList())) allFindings.push_back(f);
            for (const Json& n : GetOr(part, "notes", Dyn::EmptyList())) allNotes.push_back(n);
        }
        const std::vector<Json> findings = Dedupe(allFindings);
        const std::vector<Json> notes = Dedupe(allNotes);
        const auto undischarged = std::ranges::count_if(findings, [](const Json& f) { return !Dyn::Truthy(Dyn::Get(f, "discharged")); });
        auto rank = [](const Json& verdict) {
            if (InVocabulary(verdict, Schema::CoverageBlocking)) return 2;
            return verdict == "immaterial" ? 1 : 0;
        };
        OrderedMap merged;
        for (const Json& part : parts) {
            for (const Json& c : Dyn::Or(Dyn::Get(part, "coverage"), Dyn::EmptyList())) {
                const Json& item = Dyn::At(c, "item");
                const Json* previous = merged.Find(item);
                if (!previous || !Dyn::Truthy(*previous) || rank(Dyn::At(c, "verdict")) >= rank(Dyn::At(*previous, "verdict"))) merged.Set(item, c);
            }
        }
        // A persona's blocking verdict is would-object; every other lens's
        // defect.
        const std::string blocking = lensFile == Plan::PersonaLens ? "would-object" : "defect";
        Json crossLogs = Json::array();
        for (const Json& f : findings) {
            if (Dyn::Truthy(Dyn::Get(f, "discharged"))) continue;
            for (const Json& item : Dyn::Or(Dyn::Get(f, "items"), Dyn::EmptyList())) {
                const Json* entry = merged.Find(item);
                if (entry && !InVocabulary(Dyn::At(*entry, "verdict"), Schema::CoverageBlocking)) {
                    const Json& finding = Dyn::Get(f, "finding");
                    const std::string cited = Dyn::Truthy(finding) ? Dyn::Slice(finding.get<std::string>(), 0, 80) : std::string();
                    const std::string was = Dyn::Str(Dyn::At(*entry, "verdict"));
                    merged.Set(item, {{"item", item}, {"verdict", blocking}, {"basis", std::format("cited by finding: {}", cited)}});
                    crossLogs.push_back(std::format("{}: {} raised to {} - cited by an undischarged finding while its own entry was {}", lens, Dyn::Str(item), blocking, was));
                }
            }
        }
        // Deduplicated, in the order of first appearance.
        Json missing = Json::array();
        std::set<std::string> seenMissing;
        auto addMissing = [&](const Json& id) {
            if (seenMissing.insert(KeyOf(id)).second) missing.push_back(id);
        };
        for (const Json& part : parts) {
            for (const Json& id : GetOr(part, "coverageMissing", Dyn::EmptyList())) addMissing(id);
        }
        for (const std::string& id : missingUnits) addMissing(Json(id));
        Json violations = Json::array();
        for (const Json& part : parts) {
            for (const Json& violation : GetOr(part, "contractViolations", Dyn::EmptyList())) violations.push_back(violation);
        }
        if (!missingUnits.empty()) {
            const std::span<const std::string> shown = missingUnits.first(std::min<std::size_t>(missingUnits.size(), 8));
            violations.push_back({{"lens", lensFile}, {"kind", std::format("process(es) did not return for unit(s) {}{} - lens treated as NOT RETURNED", Dyn::Join(shown, ", "),
                                                                             missingUnits.size() > 8 ? ", ..." : "")}});
        }
        Json contests = Json::array();
        Json crossDischarges = Json::array();
        Json normalizeLogs = Json::array();
        Json partRows = Json::array();
        Json summary;
        for (const Json& part : parts) {
            for (const Json& c : GetOr(part, "contests", Dyn::EmptyList())) contests.push_back(c);
            for (const Json& c : GetOr(part, "crossLensDischarges", Dyn::EmptyList())) crossDischarges.push_back(c);
            for (const Json& l : GetOr(part, "normalizeLogs", Dyn::EmptyList())) normalizeLogs.push_back(l);
            const Json ok = "ok";
            partRows.push_back({
                {"label", Dyn::Get(part, "partLabel")},
                {"kind", Dyn::Get(part, "partKind")},
                {"unit", Dyn::Get(part, "partUnit")},
                {"verdict", Dyn::Get(part, "verdict")},
                {"findings", GetOr(part, "findings", Dyn::EmptyList()).size()},
                {"status", GetOr(part, "status", ok)},
                {"reused", Dyn::Get(part, "reused")},
                {"turns", Dyn::Get(part, "turns")},
                {"usage", Dyn::Get(part, "usage")},
            });
            if (summary.is_null() && Dyn::Truthy(Dyn::Get(part, "summary"))) summary = Dyn::At(part, "summary");
        }
        for (const Json& l : crossLogs) normalizeLogs.push_back(l);
        Json coverage = Json::array();
        for (const auto& [key, value] : merged.items) coverage.push_back(value);
        // A lens with unaddressed units is NOT RETURNED (never a pass); real
        // undischarged findings still read FIX FIRST.
        const std::string verdict = undischarged ? "FIX FIRST" : (missing.empty() ? "SEND-READY" : "NOT RETURNED");
        return {
            {"lens", lens},
            {"lensFile", lensFile},
            {"verdict", verdict},
            {"findings", findings},
            {"notes", notes},
            {"coverage", std::move(coverage)},
            {"coverageMissing", std::move(missing)},
            {"undischarged", undischarged},
            {"contests", std::move(contests)},
            {"contractViolations", std::move(violations)},
            {"crossLensDischarges", std::move(crossDischarges)},
            {"normalizeLogs", std::move(normalizeLogs)},
            {"parts", std::move(partRows)},
            {"summary", std::move(summary)},
            {"stoppedEarly", stoppedEarly},
        };
    }

    Json Matrix(const Json& inventory, std::span<const Json> results, std::span<const std::string> notReviewed) {
        auto owned = [](const Json& entry) { return !InVocabulary(Dyn::At(entry, "verdict"), Schema::CoverageUnowned); };
        Json columns = Json::array();
        for (const Json& result : results) {
            if (Dyn::Get(result, "coverage").is_array()) columns.push_back(Dyn::At(result, "lens"));
        }
        Json mx = Json::object();
        const Json empty = "";
        for (const Json& unit : inventory) {
            const std::string id = Dyn::At(unit, "id").get<std::string>();
            Json row = Json::object();
            for (const Json& result : results) {
                if (!Dyn::Get(result, "coverage").is_array()) continue;
                const Json* entry = nullptr;
                for (const Json& x : Dyn::At(result, "coverage")) {
                    if (Dyn::Equal(Dyn::At(x, "item"), Json(id))) {
                        entry = &x;
                        break;
                    }
                }
                const std::string lens = Dyn::At(result, "lens").get<std::string>();
                row[lens] = entry ? Json{{"verdict", Dyn::At(*entry, "verdict")}, {"basis", GetOr(*entry, "basis", empty)}} : Json{{"verdict", "MISSING"}, {"basis", ""}};
            }
            mx[id] = std::move(row);
        }
        const std::set<std::string> excused(notReviewed.begin(), notReviewed.end());
        Json holes = Json::array();
        for (const Json& unit : inventory) {
            const std::string id = Dyn::At(unit, "id").get<std::string>();
            if (excused.contains(id)) continue;
            const Json& row = mx[id];
            if (!std::ranges::any_of(row, owned)) holes.push_back(id);
        }
        // The persona columns: a unit no persona owns is a persona hole, one
        // a single persona owns is thin, and one whose every persona entry
        // rests on "no record" is modelled, not calibrated.
        std::vector<std::string> personaColumns;
        for (const Json& column : columns) {
            if (column.is_string() && column.get_ref<const std::string&>().starts_with("persona:")) personaColumns.push_back(column.get<std::string>());
        }
        Json personaHoles = Json::array();
        Json thin = Json::array();
        Json modelled = Json::array();
        for (const Json& unit : inventory) {
            const std::string id = Dyn::At(unit, "id").get<std::string>();
            std::vector<const Json*> entries;
            for (const std::string& column : personaColumns) {
                const Json& entry = Dyn::Get(mx[id], column);
                if (Dyn::Truthy(entry) && owned(entry)) entries.push_back(&entry);
            }
            if (personaColumns.empty()) continue;
            if (entries.empty()) personaHoles.push_back(id);
            if (entries.size() == 1) thin.push_back(id);
            const bool noRecord = std::ranges::all_of(entries, [](const Json* e) { return Dyn::Lower(Dyn::Strip(Dyn::Str(Dyn::Get(*e, "basis")))) == "no record"; });
            if (!entries.empty() && noRecord) modelled.push_back(id);
        }
        Json defects = Json::object();
        for (const Json& unit : inventory) {
            const std::string id = Dyn::At(unit, "id").get<std::string>();
            Json lenses = Json::array();
            const Json& row = mx[id];
            for (auto it = row.begin(); it != row.end(); ++it) {
                if (InVocabulary(Dyn::At(it.value(), "verdict"), Schema::CoverageBlocking)) lenses.push_back(it.key());
            }
            if (!lenses.empty()) defects[id] = std::move(lenses);
        }
        Json sortedExcused = Json::array();
        for (const std::string& id : excused) sortedExcused.push_back(id);
        return {
            {"units", inventory.size()},
            {"lenses", std::move(columns)},
            {"holes", std::move(holes)},
            {"personaHoles", std::move(personaHoles)},
            {"thin", std::move(thin)},
            {"modelled", std::move(modelled)},
            {"notReviewed", std::move(sortedExcused)},
            {"defects", std::move(defects)},
            {"matrix", std::move(mx)},
        };
    }
}