325 lines
15 KiB
C++
325 lines
15 KiB
C++
|
// SPDX-License-Identifier: AGPL-3.0-only
|
||
|
|
// SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts®
|
||
|
|
|
||
|
|
module Crafter.Review:Refs_impl;
|
||
|
|
import :Refs;
|
||
|
|
import :Json;
|
||
|
|
import :Text;
|
||
|
|
import :Platform;
|
||
|
|
import :Process;
|
||
|
|
import std;
|
||
|
|
|
||
|
|
namespace fs = std::filesystem;
|
||
|
|
|
||
|
|
namespace Crafter::Review::Refs {
|
||
|
|
namespace {
|
||
|
|
constexpr std::string_view LoreHosts[] = {"https://lore.kernel.org/", "http://lore.kernel.org/"};
|
||
|
|
|
||
|
|
bool IsListChar(char c) {
|
||
|
|
return (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || c == '-';
|
||
|
|
}
|
||
|
|
|
||
|
|
// LORE_RE at `at`: https?://lore\.kernel\.org/(?:r|all|[a-z0-9-]+)/([^/\s>]+)/?
|
||
|
|
// -> (the end of the match, the message id), or nullopt.
|
||
|
|
std::optional<std::pair<std::size_t, std::string>> LoreAt(std::string_view text, std::size_t at) {
|
||
|
|
std::size_t after = 0;
|
||
|
|
for (const std::string_view host : LoreHosts) {
|
||
|
|
if (text.substr(at).starts_with(host)) {
|
||
|
|
after = at + host.size();
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if (after == 0) return std::nullopt;
|
||
|
|
std::size_t list = after;
|
||
|
|
while (list < text.size() && IsListChar(text[list])) ++list;
|
||
|
|
if (list == after || list >= text.size() || text[list] != '/') return std::nullopt;
|
||
|
|
std::size_t id = list + 1;
|
||
|
|
std::size_t end = id;
|
||
|
|
while (end < text.size()) {
|
||
|
|
const auto [c, used] = Dyn::CodePointAt(text, end);
|
||
|
|
if (c == '/' || c == '>' || Dyn::IsSpace(c)) break;
|
||
|
|
end += used;
|
||
|
|
}
|
||
|
|
if (end == id) return std::nullopt;
|
||
|
|
const std::string msgid(text.substr(id, end - id));
|
||
|
|
if (end < text.size() && text[end] == '/') ++end;
|
||
|
|
return std::pair(end, msgid);
|
||
|
|
}
|
||
|
|
|
||
|
|
Fetcher& TheFetcher() {
|
||
|
|
static Fetcher fetcher;
|
||
|
|
return fetcher;
|
||
|
|
}
|
||
|
|
|
||
|
|
Searcher& TheSearcher() {
|
||
|
|
static Searcher searcher;
|
||
|
|
return searcher;
|
||
|
|
}
|
||
|
|
|
||
|
|
std::string Tail(std::string_view text, std::size_t n) {
|
||
|
|
return Dyn::Slice(text, -static_cast<std::int64_t>(n));
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
Table ParseRefs(std::string_view text) {
|
||
|
|
Table refs;
|
||
|
|
auto set = [&](const std::string& key, const std::string& url) {
|
||
|
|
for (auto& [k, u] : refs) {
|
||
|
|
if (k == key) {
|
||
|
|
u = url;
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
refs.emplace_back(key, url);
|
||
|
|
};
|
||
|
|
// ^\[(\d+)\] (\S+), multiline.
|
||
|
|
for (const std::string& line : Dyn::Split(text, "\n")) {
|
||
|
|
if (!line.starts_with('[')) continue;
|
||
|
|
const std::size_t digits = Dyn::DecimalRun(line, 1);
|
||
|
|
if (digits == 0 || !std::string_view(line).substr(1 + digits).starts_with("] ")) continue;
|
||
|
|
const std::size_t urlAt = 1 + digits + 2;
|
||
|
|
const std::size_t urlLen = Dyn::NonSpaceRun(line, urlAt);
|
||
|
|
if (urlLen == 0) continue;
|
||
|
|
const std::optional<std::int64_t> n = Dyn::ParseInt(line.substr(1, digits));
|
||
|
|
if (!n) continue;
|
||
|
|
set(std::to_string(*n), line.substr(urlAt, urlLen));
|
||
|
|
}
|
||
|
|
if (!refs.empty()) return refs;
|
||
|
|
std::vector<std::string> seen;
|
||
|
|
for (std::size_t at = 0; at < text.size();) {
|
||
|
|
const std::optional<std::pair<std::size_t, std::string>> found = LoreAt(text, at);
|
||
|
|
if (!found) {
|
||
|
|
++at;
|
||
|
|
continue;
|
||
|
|
}
|
||
|
|
const std::string whole(text.substr(at, found->first - at));
|
||
|
|
if (!std::ranges::contains(seen, whole)) seen.push_back(whole);
|
||
|
|
at = found->first;
|
||
|
|
}
|
||
|
|
for (std::size_t k = 0; k < seen.size(); ++k) refs.emplace_back(std::format("L{}", k + 1), seen[k]);
|
||
|
|
return refs;
|
||
|
|
}
|
||
|
|
|
||
|
|
std::optional<std::string> UrlOf(const Table& refs, std::int64_t n) {
|
||
|
|
const std::string key = std::to_string(n);
|
||
|
|
for (const auto& [k, url] : refs) {
|
||
|
|
if (k == key) return url;
|
||
|
|
}
|
||
|
|
return std::nullopt;
|
||
|
|
}
|
||
|
|
|
||
|
|
std::optional<std::string> MsgidOf(std::string_view url) {
|
||
|
|
for (std::size_t at = 0; at < url.size(); ++at) {
|
||
|
|
if (const std::optional<std::pair<std::size_t, std::string>> found = LoreAt(url, at)) return found->second;
|
||
|
|
}
|
||
|
|
return std::nullopt;
|
||
|
|
}
|
||
|
|
|
||
|
|
std::string SafeName(std::string_view msgid) {
|
||
|
|
std::u32string out;
|
||
|
|
for (const char32_t c : Dyn::Decode(msgid)) {
|
||
|
|
const bool kept = (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || c == '.' || c == '_' || c == '@' || c == '-';
|
||
|
|
out.push_back(kept ? c : U'_');
|
||
|
|
}
|
||
|
|
if (out.size() > 200) out.resize(200);
|
||
|
|
return Dyn::Encode(out);
|
||
|
|
}
|
||
|
|
|
||
|
|
std::map<std::string, std::string> Headers(std::string_view raw) {
|
||
|
|
std::map<std::string, std::string> out;
|
||
|
|
const std::string head = Dyn::Split(raw, "\n\n", 1).front();
|
||
|
|
for (const std::string& line : Dyn::SplitLines(head)) {
|
||
|
|
for (const std::string_view key : {"From", "Date", "Subject"}) {
|
||
|
|
const std::string prefix = std::format("{}:", key);
|
||
|
|
if (line.starts_with(prefix) && !out.contains(std::string(key))) out[std::string(key)] = Dyn::Strip(std::string_view(line).substr(prefix.size()));
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return out;
|
||
|
|
}
|
||
|
|
|
||
|
|
void SetFetcher(Fetcher fetcher) {
|
||
|
|
TheFetcher() = std::move(fetcher);
|
||
|
|
}
|
||
|
|
|
||
|
|
std::pair<std::string, std::string> Fetch(const fs::path& repo, const std::string& url, const fs::path& path) {
|
||
|
|
if (TheFetcher()) return TheFetcher()(repo, url, path);
|
||
|
|
std::string last;
|
||
|
|
for (std::int32_t attempt = 0; attempt < 3; ++attempt) {
|
||
|
|
Process::Captured p;
|
||
|
|
try {
|
||
|
|
p = Process::Run({"python3", "utilities/lore.py", "fetch", std::format("{}/raw", Dyn::RStrip(url, "/"))}, repo, std::chrono::milliseconds(180000));
|
||
|
|
} catch (const std::exception& e) {
|
||
|
|
last = Dyn::Slice(std::format("{}: {}", "RuntimeError", e.what()), 0, 200);
|
||
|
|
std::this_thread::sleep_for(std::chrono::seconds(2));
|
||
|
|
continue;
|
||
|
|
}
|
||
|
|
if (p.timedOut) {
|
||
|
|
last = Dyn::Slice(std::format("TimeoutExpired: Command '{}' timed out after 180 seconds", url), 0, 200);
|
||
|
|
std::this_thread::sleep_for(std::chrono::seconds(2));
|
||
|
|
continue;
|
||
|
|
}
|
||
|
|
if (p.returnCode == 0 && !Dyn::Strip(p.out).empty()) {
|
||
|
|
Platform::WriteAtomic(path, p.out);
|
||
|
|
return {"fetched", ""};
|
||
|
|
}
|
||
|
|
last = Tail(Dyn::DecodeReplace(p.err.empty() ? p.out : p.err), 200);
|
||
|
|
std::this_thread::sleep_for(std::chrono::seconds(2));
|
||
|
|
}
|
||
|
|
return {"failed", last};
|
||
|
|
}
|
||
|
|
|
||
|
|
Json Prefetch(const fs::path& repo, const Table& refs, const fs::path& refsDir, const fs::path& indexPath, const Log& log, std::int32_t workers) {
|
||
|
|
fs::create_directories(refsDir);
|
||
|
|
Json index = Json::object();
|
||
|
|
struct Todo {
|
||
|
|
std::string key;
|
||
|
|
std::string url;
|
||
|
|
fs::path path;
|
||
|
|
};
|
||
|
|
std::vector<Todo> todo;
|
||
|
|
for (const auto& [n, url] : refs) {
|
||
|
|
const std::optional<std::string> mid = MsgidOf(url);
|
||
|
|
if (!mid) {
|
||
|
|
index[n] = {{"url", url}, {"status", "not-prefetched (not a lore URL)"}};
|
||
|
|
continue;
|
||
|
|
}
|
||
|
|
const fs::path path = refsDir / std::format("{}.raw", SafeName(*mid));
|
||
|
|
index[n] = {{"url", url}, {"msgid", *mid}, {"path", Dyn::RelPath(path.string(), repo.string())}};
|
||
|
|
std::error_code error;
|
||
|
|
if (fs::exists(path, error) && fs::file_size(path, error) > 0) {
|
||
|
|
index[n]["status"] = "cached";
|
||
|
|
} else {
|
||
|
|
todo.push_back({n, url, path});
|
||
|
|
}
|
||
|
|
}
|
||
|
|
std::vector<std::pair<std::string, std::string>> results(todo.size());
|
||
|
|
{
|
||
|
|
std::atomic<std::size_t> next = 0;
|
||
|
|
std::vector<std::jthread> pool;
|
||
|
|
const std::size_t width = std::min<std::size_t>(todo.size(), static_cast<std::size_t>(std::max(1, workers)));
|
||
|
|
for (std::size_t w = 0; w < width; ++w) {
|
||
|
|
pool.emplace_back([&] {
|
||
|
|
for (std::size_t i = next.fetch_add(1); i < todo.size(); i = next.fetch_add(1)) results[i] = Fetch(repo, todo[i].url, todo[i].path);
|
||
|
|
});
|
||
|
|
}
|
||
|
|
}
|
||
|
|
for (std::size_t i = 0; i < todo.size(); ++i) {
|
||
|
|
index[todo[i].key]["status"] = results[i].first;
|
||
|
|
if (!results[i].second.empty()) index[todo[i].key]["error"] = results[i].second;
|
||
|
|
}
|
||
|
|
for (auto it = index.begin(); it != index.end(); ++it) {
|
||
|
|
Json& e = it.value();
|
||
|
|
const Json& status = Dyn::Get(e, "status");
|
||
|
|
if (!Dyn::Truthy(Dyn::Get(e, "path")) || !(status == "cached" || status == "fetched")) continue;
|
||
|
|
const std::optional<std::string> bytes = Platform::ReadFile(repo / Dyn::Str(e.at("path")));
|
||
|
|
if (!bytes) {
|
||
|
|
e["status"] = "failed";
|
||
|
|
e["error"] = std::format("[Errno 2] No such file or directory: {}", Dyn::Repr((repo / Dyn::Str(e.at("path"))).string()));
|
||
|
|
continue;
|
||
|
|
}
|
||
|
|
const std::string raw = Dyn::DecodeReplace(*bytes);
|
||
|
|
for (const auto& [k, v] : Headers(raw)) e[k] = v;
|
||
|
|
e["bytes"] = Dyn::Length(raw);
|
||
|
|
}
|
||
|
|
Platform::WriteAtomic(indexPath, DumpFile(index));
|
||
|
|
std::int64_t ok = 0;
|
||
|
|
std::int64_t fetched = 0;
|
||
|
|
for (auto it = index.begin(); it != index.end(); ++it) {
|
||
|
|
const Json& status = Dyn::Get(it.value(), "status");
|
||
|
|
if (status == "cached" || status == "fetched") ++ok;
|
||
|
|
if (status == "fetched") ++fetched;
|
||
|
|
}
|
||
|
|
log(std::format("refs: {}/{} lore references available locally ({} fetched now)", ok, index.size(), fetched));
|
||
|
|
return index;
|
||
|
|
}
|
||
|
|
|
||
|
|
std::string Digest(const Json& index) {
|
||
|
|
std::vector<std::string> lines = {
|
||
|
|
"REFERENCES prefetched (read the raw file instead of fetching; a FETCH FAILED line means the reference must be fetched by you or reported as unverifiable):"};
|
||
|
|
std::vector<std::string> keys;
|
||
|
|
for (auto it = index.begin(); it != index.end(); ++it) keys.push_back(it.key());
|
||
|
|
auto numeric = [](const std::string& k) { return !k.empty() && Dyn::DecimalRun(k, 0) == k.size(); };
|
||
|
|
std::ranges::sort(keys, [&](const std::string& a, const std::string& b) {
|
||
|
|
const bool na = numeric(a);
|
||
|
|
const bool nb = numeric(b);
|
||
|
|
if (na != nb) return na;
|
||
|
|
if (na) return *Dyn::ParseInt(a) < *Dyn::ParseInt(b);
|
||
|
|
return a < b;
|
||
|
|
});
|
||
|
|
const Json unknown = "?";
|
||
|
|
for (const std::string& n : keys) {
|
||
|
|
const Json& e = index.at(n);
|
||
|
|
const Json& status = Dyn::Get(e, "status");
|
||
|
|
if (status == "cached" || status == "fetched") {
|
||
|
|
const Json& subject = Dyn::Has(e, "Subject") ? e.at("Subject") : unknown;
|
||
|
|
lines.push_back(std::format("[{}] {} | From: {} | Date: {} | Subject: {} | {} ({} B)", n, Dyn::Str(Dyn::Get(e, "msgid")), Dyn::Str(Dyn::Has(e, "From") ? e.at("From") : unknown),
|
||
|
|
Dyn::Str(Dyn::Has(e, "Date") ? e.at("Date") : unknown), Dyn::Slice(Dyn::Str(subject), 0, 100), Dyn::Str(e.at("path")),
|
||
|
|
Dyn::Str(Dyn::Has(e, "bytes") ? e.at("bytes") : Json(0))));
|
||
|
|
} else if (status == "failed") {
|
||
|
|
lines.push_back(std::format("[{}] {} | FETCH FAILED | {}", n, Dyn::Str(Dyn::Get(e, "msgid")), Dyn::Str(e.at("url"))));
|
||
|
|
} else {
|
||
|
|
lines.push_back(std::format("[{}] {} | {}", n, Dyn::Str(e.at("url")), Dyn::Str(status)));
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return Dyn::Join(lines, "\n");
|
||
|
|
}
|
||
|
|
|
||
|
|
void SetSearcher(Searcher searcher) {
|
||
|
|
TheSearcher() = std::move(searcher);
|
||
|
|
}
|
||
|
|
|
||
|
|
std::pair<std::optional<fs::path>, std::string> PersonaCorpus(const fs::path& repo, std::string_view name, const fs::path& refsDir, std::string_view since, const Log& log) {
|
||
|
|
// re.sub(r'[^a-z0-9]+', '-', name.lower()).strip('-')
|
||
|
|
std::u32string slug;
|
||
|
|
bool dash = false;
|
||
|
|
for (const char32_t c : Dyn::Decode(Dyn::Lower(name))) {
|
||
|
|
if ((c >= 'a' && c <= 'z') || (c >= '0' && c <= '9')) {
|
||
|
|
slug.push_back(c);
|
||
|
|
dash = false;
|
||
|
|
} else if (!dash) {
|
||
|
|
slug.push_back(U'-');
|
||
|
|
dash = true;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
const std::string slugText = Dyn::Strip(Dyn::Encode(slug), "-");
|
||
|
|
const fs::path path = refsDir / std::format("persona-{}.txt", slugText);
|
||
|
|
std::error_code error;
|
||
|
|
if (fs::exists(path, error) && fs::file_size(path, error) > 0) return {path, Dyn::ReadText(path)};
|
||
|
|
const std::string q = std::format("f:\"{}\" AND d:{}..", name, since);
|
||
|
|
std::string out;
|
||
|
|
std::string err;
|
||
|
|
if (TheSearcher()) {
|
||
|
|
std::tie(out, err) = TheSearcher()(repo, q);
|
||
|
|
} else {
|
||
|
|
try {
|
||
|
|
const Process::Captured p = Process::Run({"python3", "utilities/lore.py", "search", "-n", "100", q}, repo, std::chrono::milliseconds(180000));
|
||
|
|
out = Dyn::DecodeReplace(p.out);
|
||
|
|
err = Dyn::DecodeReplace(p.err);
|
||
|
|
} catch (const std::exception& e) {
|
||
|
|
err = e.what();
|
||
|
|
}
|
||
|
|
}
|
||
|
|
const std::string text = Dyn::Strip(out);
|
||
|
|
if (text.empty() || Dyn::Lower(text).starts_with("no results")) {
|
||
|
|
log(std::format("persona corpus: {} -> NO RESULTS for {} (stderr: {}) - the whole pass calibrates by its own searches", name, Dyn::Repr(q), Dyn::Slice(Dyn::Strip(err), 0, 120)));
|
||
|
|
return {std::nullopt, ""};
|
||
|
|
}
|
||
|
|
fs::create_directories(refsDir);
|
||
|
|
Platform::WriteAtomic(path, std::format("lore.py search {} ({})\n{}\n", Dyn::Repr(q), Platform::LocalTime("%Y-%m-%d"), text));
|
||
|
|
log(std::format("persona corpus: {} -> {} ({} chars)", name, Dyn::RelPath(path.string(), repo.string()), Dyn::Length(text)));
|
||
|
|
return {path, Dyn::ReadText(path)};
|
||
|
|
}
|
||
|
|
|
||
|
|
std::string SinceOf(std::string_view isoDate) {
|
||
|
|
const std::vector<std::string> parts = Dyn::Split(isoDate, "-");
|
||
|
|
if (parts.size() != 3) throw Dyn::Error("ValueError", std::format("Invalid isoformat string: {}", Dyn::Repr(isoDate)));
|
||
|
|
const std::optional<std::int64_t> y = Dyn::ParseInt(parts[0]);
|
||
|
|
const std::optional<std::int64_t> m = Dyn::ParseInt(parts[1]);
|
||
|
|
const std::optional<std::int64_t> d = Dyn::ParseInt(parts[2]);
|
||
|
|
if (!y || !m || !d) throw Dyn::Error("ValueError", std::format("Invalid isoformat string: {}", Dyn::Repr(isoDate)));
|
||
|
|
const std::chrono::year_month_day date{std::chrono::year(static_cast<int>(*y)), std::chrono::month(static_cast<unsigned>(*m)), std::chrono::day(static_cast<unsigned>(*d))};
|
||
|
|
if (!date.ok()) throw Dyn::Error("ValueError", std::format("Invalid isoformat string: {}", Dyn::Repr(isoDate)));
|
||
|
|
const std::chrono::year_month_day since{std::chrono::sys_days(date) - std::chrono::days(365)};
|
||
|
|
return std::format("{:04}{:02}{:02}", static_cast<std::int32_t>(since.year()), static_cast<std::uint32_t>(since.month()), static_cast<std::uint32_t>(since.day()));
|
||
|
|
}
|
||
|
|
}
|