Jorijn van der Graaf
741e3a656d
The modules that turn a frozen range into passes and passes into a verdict: :Schema, :Tools (the DENY, git and base lists, a leaf, so the config and the pool both read them), :Plan, :Artifact, :Inventory, :Dispositions, :State, :Normalize, :Stream, :Prompt, :Config, and the URL half of :PgWire. Units, jobs, results, the config and the claude stream stay JSON from end to end, so every key keeps its place; :Text gives JSON values truthiness, get, ==, membership and Str(), and throws Dyn::Error under a name for each kind of unexpected data. Exactness where the bytes are seen: - the stream classifiers are hand-written scanners with case-insensitive folds (the Kelvin sign is a k), Unicode \b and \s, and `.` stopping at \n; the SQL call detection takes `<exe> sql` with any path before it; - config problems carry the JSON scanner's and the UTF-8 decoder's own texts, positions included; - a float total is Neumaier's compensated sum, and the build passes -ffp-contract=off: -march=native would fuse a*b+c; - `crafter-review inventory` and `config` print what the goldens record, byte for byte (the fixture and 14 configs, broken ones included); a message names this binary as the command to run. tests/Goldens: the recorded answers over the fixtures next to them (the commits fixture, a rich versioned series with CRLF, a folded subject, Markdown headings in and out of fences, the code-key rewrites); the suite compares 2304 results as DumpFile writes them, so a key out of order fails too. All agree. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
245 lines
13 KiB
C++
245 lines
13 KiB
C++
// SPDX-License-Identifier: AGPL-3.0-only
|
||
// SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts®
|
||
|
||
export module Crafter.Review:Text;
|
||
import std;
|
||
import :Json;
|
||
|
||
// The text, number, URL and path semantics every other partition relies on.
|
||
// Where a result reaches a prompt, a reuse key, a shared file or an exit code
|
||
// it is decided by these exact rules — Unicode whitespace in Strip(), U+2028
|
||
// in SplitLines(), code points in Length(), half-even Round(), FnMatch()'s
|
||
// character classes — never by the nearest C++ equivalent. tests/Text checks
|
||
// every function against the reference vectors in tests/Text/fixture.
|
||
//
|
||
// Text is UTF-8 in std::string. Lengths and indices are in code points.
|
||
//
|
||
// Lower() maps Greek capital sigma to σ everywhere: it never writes the
|
||
// final form ς.
|
||
export namespace Crafter::Review::Dyn {
|
||
// ---- UTF-8 -------------------------------------------------------------
|
||
// Each maximal subpart of an ill-formed sequence becomes one U+FFFD, the
|
||
// Unicode Standard's recommended practice.
|
||
std::u32string Decode(std::string_view utf8);
|
||
std::string Encode(std::u32string_view text);
|
||
std::string Encode(char32_t c);
|
||
// Well-formed UTF-8: no overlong forms, no surrogates, nothing above
|
||
// U+10FFFF.
|
||
bool IsUtf8(std::string_view bytes);
|
||
// bytes.decode('utf-8', 'replace').
|
||
std::string DecodeReplace(std::string_view bytes);
|
||
// len()
|
||
std::size_t Length(std::string_view text);
|
||
// The code points from start to end: a negative index counts from the
|
||
// end, and both are clipped to the text.
|
||
std::string Slice(std::string_view text, std::int64_t start, std::optional<std::int64_t> end = std::nullopt);
|
||
// The code point at byte `at` and the bytes it takes: U+FFFD and the
|
||
// length of the invalid sequence where the bytes are no UTF-8.
|
||
std::pair<char32_t, std::size_t> CodePointAt(std::string_view text, std::size_t at);
|
||
// [start, end) of a slice of `size` items: a negative index counts from
|
||
// the end, and both are clipped to the sequence.
|
||
std::pair<std::size_t, std::size_t> SliceBounds(std::size_t size, std::int64_t start, std::optional<std::int64_t> end = std::nullopt);
|
||
|
||
// ---- regex classes, for the hand-written scanners -----------------------
|
||
// The bytes of the run at `at` of regex \d (Unicode decimal digits), \s
|
||
// (Unicode whitespace) or \S characters.
|
||
std::size_t DecimalRun(std::string_view text, std::size_t at);
|
||
std::size_t SpaceRun(std::string_view text, std::size_t at);
|
||
std::size_t NonSpaceRun(std::string_view text, std::size_t at);
|
||
|
||
// ---- characters ----------------------------------------------------------
|
||
bool IsSpace(char32_t c);
|
||
bool IsLineBreak(char32_t c);
|
||
bool IsDecimal(char32_t c);
|
||
// The digit's value, -1 for a character that is not a decimal digit.
|
||
std::int32_t DecimalValue(char32_t c);
|
||
bool IsPrintable(char32_t c);
|
||
// regex \w: what \b takes for a word character.
|
||
bool IsWord(char32_t c);
|
||
// The lowercase ASCII letter `c` matches case-insensitively, or '\0':
|
||
// ASCII letters, and the four code points outside ASCII that fold onto
|
||
// one (İ ı → i, ſ → s, K → k).
|
||
char IgnoreCaseLetter(char32_t c);
|
||
|
||
// ---- str methods ----------------------------------------------------------
|
||
// strip()/lstrip()/rstrip(): Unicode whitespace.
|
||
std::string Strip(std::string_view text);
|
||
std::string LStrip(std::string_view text);
|
||
std::string RStrip(std::string_view text);
|
||
// strip(chars): the code points of `chars`.
|
||
std::string Strip(std::string_view text, std::string_view chars);
|
||
std::string LStrip(std::string_view text, std::string_view chars);
|
||
std::string RStrip(std::string_view text, std::string_view chars);
|
||
// split(): runs of Unicode whitespace, no empty pieces.
|
||
std::vector<std::string> Split(std::string_view text);
|
||
// split(sep, maxsplit) / rsplit(sep, maxsplit): every piece, empty ones
|
||
// included; a negative maxSplit splits at every separator.
|
||
std::vector<std::string> Split(std::string_view text, std::string_view separator, std::int64_t maxSplit = -1);
|
||
std::vector<std::string> RSplit(std::string_view text, std::string_view separator, std::int64_t maxSplit = -1);
|
||
// splitlines(keepends): \n, \r, \r\n, \v, \f, \x1c-\x1e, \x85, U+2028,
|
||
// U+2029; no empty piece after a final line break.
|
||
std::vector<std::string> SplitLines(std::string_view text, bool keepEnds = false);
|
||
std::string Lower(std::string_view text);
|
||
std::string Upper(std::string_view text);
|
||
// count(sub): non-overlapping; an empty `sub` counts len() + 1.
|
||
std::size_t Count(std::string_view text, std::string_view sub);
|
||
std::string Replace(std::string_view text, std::string_view from, std::string_view to);
|
||
std::string Join(std::span<const std::string> parts, std::string_view separator);
|
||
// f'{text:<width}' / text.ljust(width): padded with spaces to `width`
|
||
// code points, where std::format would count display columns.
|
||
std::string LJust(std::string_view text, std::size_t width);
|
||
|
||
// ---- reading text files ----------------------------------------------------
|
||
// A text file, read whole: strict UTF-8 — invalid bytes throw Error
|
||
// ("UnicodeDecodeError") with Utf8DecodeError()'s message — and universal
|
||
// newlines. A file that cannot be opened throws Error ("FileNotFoundError").
|
||
std::string ReadText(const std::filesystem::path& path);
|
||
// \r\n and a lone \r become \n, as text mode reads them.
|
||
std::string UniversalNewlines(std::string_view text);
|
||
// The message for bytes that are not UTF-8 — "'utf-8' codec can't decode
|
||
// byte 0xff in position 3: invalid start byte" — nullopt for valid UTF-8.
|
||
std::optional<std::string> Utf8DecodeError(std::string_view bytes);
|
||
// The message for text that is no JSON document, with the scanner's
|
||
// reason and position ("Expecting ',' delimiter: line 3 column 5 (char
|
||
// 21)"); nullopt for a document LoadJsonFile() takes.
|
||
std::optional<std::string> JsonDecodeError(std::string_view text);
|
||
// A JSON file: read as text, then parsed. Throws Error with the message a
|
||
// "not JSON (...)" problem carries: the decode error, or the JSON error.
|
||
Json LoadJsonFile(const std::filesystem::path& path);
|
||
|
||
// ---- numbers ------------------------------------------------------------------
|
||
// A float as its shortest digits that round-trip; an exponent below 1e-4
|
||
// and from 1e16 up; ".0" on an integral value; inf, nan.
|
||
std::string FloatRepr(double value);
|
||
// value to `digits` decimals: correctly rounded, half to even on exact
|
||
// ties.
|
||
double Round(double value, std::int32_t digits);
|
||
// value to the nearest integer, half to even.
|
||
std::int64_t RoundToInt(double value);
|
||
// An integer or a float from text: surrounding Unicode whitespace ignored,
|
||
// Unicode decimal digits accepted, single underscores between digits
|
||
// allowed. nullopt for anything else (and, for an integer, beyond 64 bits).
|
||
std::optional<std::int64_t> ParseInt(std::string_view text);
|
||
std::optional<double> ParseFloat(std::string_view text);
|
||
// The ValueError messages for those.
|
||
std::string IntError(std::string_view text);
|
||
std::string FloatError(std::string_view text);
|
||
|
||
// ---- repr ----------------------------------------------------------------------
|
||
std::string Repr(std::string_view text);
|
||
// A JSON value printed for a message: objects as {'key': value}, strings
|
||
// quoted, true, false and null as True, False and None.
|
||
std::string ReprValue(const Json& value);
|
||
|
||
// ---- shell patterns ---------------------------------------------------------------
|
||
// A shell-style pattern: case-sensitive, `*` crosses `/`, `[!...]` negates,
|
||
// an unclosed `[` is literal and a reversed range matches nothing.
|
||
bool FnMatch(std::string_view name, std::string_view pattern);
|
||
|
||
// ---- URLs ---------------------------------------------------------------------------
|
||
struct SplitUrl {
|
||
std::string scheme;
|
||
std::string netloc;
|
||
std::string path;
|
||
std::string query;
|
||
std::string fragment;
|
||
// .hostname: lowercased, nullopt when there is none.
|
||
std::optional<std::string> Hostname() const;
|
||
std::optional<std::string> Username() const;
|
||
std::optional<std::string> Password() const;
|
||
// .port: nullopt for none; std::invalid_argument with a ValueError
|
||
// message for one that is not a number or out of range.
|
||
std::optional<std::int32_t> Port() const;
|
||
};
|
||
// A URL's scheme, netloc, path, query and fragment. Throws
|
||
// std::invalid_argument with a ValueError message for unbalanced or
|
||
// invalid brackets.
|
||
SplitUrl UrlSplit(std::string_view url);
|
||
// The five parts joined back into a URL: an empty query or fragment gets
|
||
// no '?' or '#'.
|
||
std::string UrlUnsplit(std::string_view scheme, std::string_view netloc, std::string_view path, std::string_view query, std::string_view fragment);
|
||
// unquote(text): %XX sequences decoded as UTF-8, bad bytes replaced.
|
||
std::string Unquote(std::string_view text);
|
||
|
||
// ---- natural sort ---------------------------------------------------------------------
|
||
// A name cut into digit runs and the text between them, compared element
|
||
// by element, a digit run by its value.
|
||
struct NaturalKey {
|
||
std::vector<std::variant<std::string, std::int64_t>> parts;
|
||
std::strong_ordering operator<=>(const NaturalKey& other) const = default;
|
||
bool operator==(const NaturalKey& other) const = default;
|
||
};
|
||
NaturalKey Natural(std::string_view name);
|
||
|
||
// ---- POSIX paths -----------------------------------------------------------------------
|
||
std::string NormPath(std::string_view path);
|
||
// Against the current directory, lexically: symlinks are not resolved.
|
||
std::string AbsPath(std::string_view path);
|
||
std::string RelPath(std::string_view path, std::string_view start = ".");
|
||
std::string BaseName(std::string_view path);
|
||
std::string DirName(std::string_view path);
|
||
std::string JoinPath(std::string_view a, std::string_view b);
|
||
|
||
// ---- errno texts -------------------------------------------------------------------------
|
||
// "[Errno 111] Connection refused": an errno as a message prints it.
|
||
std::string ErrnoText(std::int32_t code);
|
||
|
||
// ---- exceptions ------------------------------------------------------------------------------
|
||
// Data the code does not expect — a key that is absent, a value of the
|
||
// wrong type, a lookup in something that is no object — throws Error
|
||
// named for the case (KeyError, TypeError, AttributeError): a caller
|
||
// catches the names it expects, and anything else fails loudly.
|
||
class Error : public std::runtime_error {
|
||
public:
|
||
Error(std::string_view kind, const std::string& message);
|
||
const std::string& Kind() const;
|
||
|
||
private:
|
||
std::string kind;
|
||
};
|
||
|
||
// ---- JSON values ---------------------------------------------------------------------------------
|
||
// The driver's data — units, jobs, results, dispositions, the config, the
|
||
// claude stream — stays JSON from end to end, so every key keeps its place
|
||
// and its unknown neighbours. These are the operations on it.
|
||
//
|
||
// Truthiness: null, false, 0, 0.0, "", [] and {} are false.
|
||
bool Truthy(const Json& value);
|
||
// `value or fallback`. A reference to one of the two, so neither may be a
|
||
// temporary: name the fallback, or use EmptyDict()/EmptyList().
|
||
const Json& Or(const Json& value, const Json& fallback);
|
||
const Json& Or(Json&& value, const Json& fallback) = delete;
|
||
const Json& Or(const Json& value, Json&& fallback) = delete;
|
||
// The shared {} and [] that `x or {}` and `x or []` fall back to.
|
||
const Json& EmptyDict();
|
||
const Json& EmptyList();
|
||
// Whether an object has the key; false for anything that is no object.
|
||
bool Has(const Json& dict, std::string_view key);
|
||
// The key's value, null when it is absent; Error (AttributeError) on
|
||
// something that is no object.
|
||
const Json& Get(const Json& dict, std::string_view key);
|
||
// The key's value; Error (KeyError) when it is absent.
|
||
const Json& At(const Json& dict, std::string_view key);
|
||
// The type name messages use: dict, list, str, int, float, bool, NoneType.
|
||
std::string_view TypeName(const Json& value);
|
||
// An integer, a bool included.
|
||
bool IsInt(const Json& value);
|
||
// An integer or a float, a bool included.
|
||
bool IsNumber(const Json& value);
|
||
// A string as it is, anything else as ReprValue() prints it.
|
||
std::string Str(const Json& value);
|
||
// Str() of the value, or of the fallback when the value is falsy.
|
||
std::string StrOr(const Json& value, std::string_view fallback);
|
||
// a == b: numbers by value (True == 1 == 1.0), dicts whatever their key
|
||
// order, a str never equal to a number.
|
||
bool Equal(const Json& a, const Json& b);
|
||
// Membership: an element of a list, a key of an object, a substring of a
|
||
// string (Error when the item is no string there).
|
||
bool In(const Json& item, const Json& container);
|
||
// a + b on numbers: int + int stays an int, a float makes a float, a bool
|
||
// counts as 0 or 1; Error (TypeError) for anything else.
|
||
Json Add(const Json& a, const Json& b);
|
||
// A total of floats by Neumaier's compensated summation, so it comes out
|
||
// to the same last bit as the goldens record it.
|
||
double Sum(std::span<const double> values);
|
||
}
|