Jorijn van der Graaf
36852edf63
What the rest of crafter-review stands on. - project.cpp: a static crafter.review-lib and the crafter-review executable; Crafter.Network without QUIC, pinned, or with --local from the checkout next to this one; Crafter.LintRules' lite rules, pinned; the version from CRAFTER_REVIEW_VERSION, git describe or 0.0.0-dev. - :Core and :Driver: the version, PromptVersion 1, --version. - :Json: nlohmann's ordered_json, which keeps key order and unknown keys, so the files every driver on a machine shares survive a rewrite; dumps in the indented file layout, the spaced one-line form and the compact sorted form the structured-output schema needs byte for byte. - :Crypto: SHA-256/SHA-1/MD5, HMAC, PBKDF2 through libcrypto; base64 and base32 written out, because what they refuse, and with which message, is pinned. - :Unicode, Unicode 16.0.0's tables: whitespace, line breaks, decimal digits, printable characters, case mappings and case-insensitive folds. - :Text: string functions on code points, number parsing, the shortest float form and half-even rounding, printed values, shell patterns, URL splitting and unquoting, POSIX paths, universal newlines and the replacing UTF-8 decoder. Tests: Crypto (published vectors), Text (293 reference vectors). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
155 lines
8 KiB
C++
155 lines
8 KiB
C++
// SPDX-License-Identifier: AGPL-3.0-only
|
||
// SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts®
|
||
|
||
export module Crafter.Review:Text;
|
||
import std;
|
||
import :Json;
|
||
|
||
// The text, number, URL and path semantics every other partition relies on.
|
||
// Where a result reaches a prompt, a reuse key, a shared file or an exit code
|
||
// it is decided by these exact rules — Unicode whitespace in Strip(), U+2028
|
||
// in SplitLines(), code points in Length(), half-even Round(), FnMatch()'s
|
||
// character classes — never by the nearest C++ equivalent. tests/Text checks
|
||
// every function against the reference vectors in tests/Text/fixture.
|
||
//
|
||
// Text is UTF-8 in std::string. Lengths and indices are in code points.
|
||
//
|
||
// Lower() maps Greek capital sigma to σ everywhere: it never writes the
|
||
// final form ς.
|
||
export namespace Crafter::Review::Dyn {
|
||
// ---- UTF-8 -------------------------------------------------------------
|
||
// Each maximal subpart of an ill-formed sequence becomes one U+FFFD, the
|
||
// Unicode Standard's recommended practice.
|
||
std::u32string Decode(std::string_view utf8);
|
||
std::string Encode(std::u32string_view text);
|
||
std::string Encode(char32_t c);
|
||
// Well-formed UTF-8: no overlong forms, no surrogates, nothing above
|
||
// U+10FFFF.
|
||
bool IsUtf8(std::string_view bytes);
|
||
// bytes.decode('utf-8', 'replace').
|
||
std::string DecodeReplace(std::string_view bytes);
|
||
// len()
|
||
std::size_t Length(std::string_view text);
|
||
// The code points from start to end: a negative index counts from the
|
||
// end, and both are clipped to the text.
|
||
std::string Slice(std::string_view text, std::int64_t start, std::optional<std::int64_t> end = std::nullopt);
|
||
|
||
// ---- characters ----------------------------------------------------------
|
||
bool IsSpace(char32_t c);
|
||
bool IsLineBreak(char32_t c);
|
||
bool IsDecimal(char32_t c);
|
||
// The digit's value, -1 for a character that is not a decimal digit.
|
||
std::int32_t DecimalValue(char32_t c);
|
||
bool IsPrintable(char32_t c);
|
||
// The lowercase ASCII letter `c` matches case-insensitively, or '\0':
|
||
// ASCII letters, and the four code points outside ASCII that fold onto
|
||
// one (İ ı → i, ſ → s, K → k).
|
||
char IgnoreCaseLetter(char32_t c);
|
||
|
||
// ---- str methods ----------------------------------------------------------
|
||
// strip()/lstrip()/rstrip(): Unicode whitespace.
|
||
std::string Strip(std::string_view text);
|
||
std::string LStrip(std::string_view text);
|
||
std::string RStrip(std::string_view text);
|
||
// strip(chars): the code points of `chars`.
|
||
std::string Strip(std::string_view text, std::string_view chars);
|
||
std::string LStrip(std::string_view text, std::string_view chars);
|
||
std::string RStrip(std::string_view text, std::string_view chars);
|
||
// split(): runs of Unicode whitespace, no empty pieces.
|
||
std::vector<std::string> Split(std::string_view text);
|
||
// split(sep, maxsplit) / rsplit(sep, maxsplit): every piece, empty ones
|
||
// included; a negative maxSplit splits at every separator.
|
||
std::vector<std::string> Split(std::string_view text, std::string_view separator, std::int64_t maxSplit = -1);
|
||
std::vector<std::string> RSplit(std::string_view text, std::string_view separator, std::int64_t maxSplit = -1);
|
||
// splitlines(keepends): \n, \r, \r\n, \v, \f, \x1c-\x1e, \x85, U+2028,
|
||
// U+2029; no empty piece after a final line break.
|
||
std::vector<std::string> SplitLines(std::string_view text, bool keepEnds = false);
|
||
std::string Lower(std::string_view text);
|
||
std::string Upper(std::string_view text);
|
||
// count(sub): non-overlapping; an empty `sub` counts len() + 1.
|
||
std::size_t Count(std::string_view text, std::string_view sub);
|
||
std::string Replace(std::string_view text, std::string_view from, std::string_view to);
|
||
std::string Join(std::span<const std::string> parts, std::string_view separator);
|
||
|
||
// ---- reading text files ----------------------------------------------------
|
||
// A text file, read whole: strict UTF-8 — an invalid byte throws
|
||
// std::runtime_error naming the file and the offset — and universal
|
||
// newlines.
|
||
std::string ReadText(const std::filesystem::path& path);
|
||
// \r\n and a lone \r become \n, as text mode reads them.
|
||
std::string UniversalNewlines(std::string_view text);
|
||
|
||
// ---- numbers ------------------------------------------------------------------
|
||
// A float as its shortest digits that round-trip; an exponent below 1e-4
|
||
// and from 1e16 up; ".0" on an integral value; inf, nan.
|
||
std::string FloatRepr(double value);
|
||
// value to `digits` decimals: correctly rounded, half to even on exact
|
||
// ties.
|
||
double Round(double value, std::int32_t digits);
|
||
// value to the nearest integer, half to even.
|
||
std::int64_t RoundToInt(double value);
|
||
// An integer or a float from text: surrounding Unicode whitespace ignored,
|
||
// Unicode decimal digits accepted, single underscores between digits
|
||
// allowed. nullopt for anything else (and, for an integer, beyond 64 bits).
|
||
std::optional<std::int64_t> ParseInt(std::string_view text);
|
||
std::optional<double> ParseFloat(std::string_view text);
|
||
// The ValueError messages for those.
|
||
std::string IntError(std::string_view text);
|
||
std::string FloatError(std::string_view text);
|
||
|
||
// ---- repr ----------------------------------------------------------------------
|
||
std::string Repr(std::string_view text);
|
||
// A JSON value printed for a message: objects as {'key': value}, strings
|
||
// quoted, true, false and null as True, False and None.
|
||
std::string ReprValue(const Json& value);
|
||
|
||
// ---- shell patterns ---------------------------------------------------------------
|
||
// A shell-style pattern: case-sensitive, `*` crosses `/`, `[!...]` negates,
|
||
// an unclosed `[` is literal and a reversed range matches nothing.
|
||
bool FnMatch(std::string_view name, std::string_view pattern);
|
||
|
||
// ---- URLs ---------------------------------------------------------------------------
|
||
struct SplitUrl {
|
||
std::string scheme;
|
||
std::string netloc;
|
||
std::string path;
|
||
std::string query;
|
||
std::string fragment;
|
||
// .hostname: lowercased, nullopt when there is none.
|
||
std::optional<std::string> Hostname() const;
|
||
std::optional<std::string> Username() const;
|
||
std::optional<std::string> Password() const;
|
||
// .port: nullopt for none; std::invalid_argument with a ValueError
|
||
// message for one that is not a number or out of range.
|
||
std::optional<std::int32_t> Port() const;
|
||
};
|
||
// A URL's scheme, netloc, path, query and fragment. Throws
|
||
// std::invalid_argument with a ValueError message for unbalanced or
|
||
// invalid brackets.
|
||
SplitUrl UrlSplit(std::string_view url);
|
||
// unquote(text): %XX sequences decoded as UTF-8, bad bytes replaced.
|
||
std::string Unquote(std::string_view text);
|
||
|
||
// ---- natural sort ---------------------------------------------------------------------
|
||
// A name cut into digit runs and the text between them, compared element
|
||
// by element, a digit run by its value.
|
||
struct NaturalKey {
|
||
std::vector<std::variant<std::string, std::int64_t>> parts;
|
||
std::strong_ordering operator<=>(const NaturalKey& other) const = default;
|
||
bool operator==(const NaturalKey& other) const = default;
|
||
};
|
||
NaturalKey Natural(std::string_view name);
|
||
|
||
// ---- POSIX paths -----------------------------------------------------------------------
|
||
std::string NormPath(std::string_view path);
|
||
// Against the current directory, lexically: symlinks are not resolved.
|
||
std::string AbsPath(std::string_view path);
|
||
std::string RelPath(std::string_view path, std::string_view start = ".");
|
||
std::string BaseName(std::string_view path);
|
||
std::string DirName(std::string_view path);
|
||
std::string JoinPath(std::string_view a, std::string_view b);
|
||
|
||
// ---- errno texts -------------------------------------------------------------------------
|
||
// "[Errno 111] Connection refused": an errno as a message prints it.
|
||
std::string ErrnoText(std::int32_t code);
|
||
}
|