Jorijn van der Graaf
36852edf63
What the rest of crafter-review stands on. - project.cpp: a static crafter.review-lib and the crafter-review executable; Crafter.Network without QUIC, pinned, or with --local from the checkout next to this one; Crafter.LintRules' lite rules, pinned; the version from CRAFTER_REVIEW_VERSION, git describe or 0.0.0-dev. - :Core and :Driver: the version, PromptVersion 1, --version. - :Json: nlohmann's ordered_json, which keeps key order and unknown keys, so the files every driver on a machine shares survive a rewrite; dumps in the indented file layout, the spaced one-line form and the compact sorted form the structured-output schema needs byte for byte. - :Crypto: SHA-256/SHA-1/MD5, HMAC, PBKDF2 through libcrypto; base64 and base32 written out, because what they refuse, and with which message, is pinned. - :Unicode, Unicode 16.0.0's tables: whitespace, line breaks, decimal digits, printable characters, case mappings and case-insensitive folds. - :Text: string functions on code points, number parsing, the shortest float form and half-even rounding, printed values, shell patterns, URL splitting and unquoting, POSIX paths, universal newlines and the replacing UTF-8 decoder. Tests: Crypto (published vectors), Text (293 reference vectors). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1061 lines
44 KiB
C++
1061 lines
44 KiB
C++
// SPDX-License-Identifier: AGPL-3.0-only
|
|
// SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts®
|
|
|
|
module;
|
|
#include <arpa/inet.h>
|
|
#include <sys/socket.h>
|
|
|
|
module Crafter.Review:Text_impl;
|
|
import :Text;
|
|
import :Unicode;
|
|
import :Json;
|
|
import std;
|
|
|
|
namespace Crafter::Review::Dyn {
|
|
namespace {
|
|
template <typename Table>
|
|
bool InRuns(const Table& table, char32_t c) {
|
|
const auto found = std::ranges::upper_bound(table, c, {}, [](const Unicode::Run& run) { return run.first; });
|
|
if (found == table.begin()) return false;
|
|
return c <= std::prev(found)->last;
|
|
}
|
|
|
|
template <typename Table>
|
|
std::string_view Mapped(const Table& table, char32_t c) {
|
|
const auto found = std::ranges::lower_bound(table, c, {}, [](const Unicode::Mapping& mapping) { return mapping.from; });
|
|
if (found == table.end() || found->from != c) return {};
|
|
return found->to;
|
|
}
|
|
|
|
void AppendUtf8(std::string& out, char32_t c) {
|
|
if (c < 0x80) {
|
|
out.push_back(static_cast<char>(c));
|
|
} else if (c < 0x800) {
|
|
out.push_back(static_cast<char>(0xC0 | (c >> 6)));
|
|
out.push_back(static_cast<char>(0x80 | (c & 0x3F)));
|
|
} else if (c < 0x10000) {
|
|
out.push_back(static_cast<char>(0xE0 | (c >> 12)));
|
|
out.push_back(static_cast<char>(0x80 | ((c >> 6) & 0x3F)));
|
|
out.push_back(static_cast<char>(0x80 | (c & 0x3F)));
|
|
} else {
|
|
out.push_back(static_cast<char>(0xF0 | (c >> 18)));
|
|
out.push_back(static_cast<char>(0x80 | ((c >> 12) & 0x3F)));
|
|
out.push_back(static_cast<char>(0x80 | ((c >> 6) & 0x3F)));
|
|
out.push_back(static_cast<char>(0x80 | (c & 0x3F)));
|
|
}
|
|
}
|
|
|
|
// One code point from `bytes` at `at`: the code point and the bytes
|
|
// it took, or U+FFFD and the length of the maximal subpart of an
|
|
// ill-formed sequence, which one U+FFFD replaces.
|
|
std::pair<char32_t, std::size_t> Next(std::string_view bytes, std::size_t at) {
|
|
const auto lead = static_cast<std::uint8_t>(bytes[at]);
|
|
if (lead < 0x80) return {lead, 1};
|
|
std::size_t need = 0;
|
|
char32_t value = 0;
|
|
std::uint8_t low = 0x80;
|
|
std::uint8_t high = 0xBF;
|
|
if (lead >= 0xC2 && lead <= 0xDF) {
|
|
need = 1;
|
|
value = lead & 0x1F;
|
|
} else if (lead >= 0xE0 && lead <= 0xEF) {
|
|
need = 2;
|
|
value = lead & 0x0F;
|
|
if (lead == 0xE0) low = 0xA0;
|
|
if (lead == 0xED) high = 0x9F;
|
|
} else if (lead >= 0xF0 && lead <= 0xF4) {
|
|
need = 3;
|
|
value = lead & 0x07;
|
|
if (lead == 0xF0) low = 0x90;
|
|
if (lead == 0xF4) high = 0x8F;
|
|
} else {
|
|
return {0xFFFD, 1};
|
|
}
|
|
std::size_t used = 1;
|
|
for (std::size_t i = 0; i < need; ++i) {
|
|
if (at + used >= bytes.size()) return {0xFFFD, used};
|
|
const auto next = static_cast<std::uint8_t>(bytes[at + used]);
|
|
if (next < low || next > high) return {0xFFFD, used};
|
|
value = (value << 6) | (next & 0x3F);
|
|
++used;
|
|
low = 0x80;
|
|
high = 0xBF;
|
|
}
|
|
return {value, used};
|
|
}
|
|
|
|
bool IsAsciiDigit(char32_t c) {
|
|
return c >= '0' && c <= '9';
|
|
}
|
|
|
|
// ParseInt()'s and ParseFloat()'s first step: Unicode whitespace
|
|
// stripped, then every decimal digit made ASCII and every space a plain
|
|
// one.
|
|
std::optional<std::string> AsciiNumber(std::string_view text) {
|
|
std::u32string decoded = Decode(Strip(text));
|
|
std::string out;
|
|
for (const char32_t c : decoded) {
|
|
if (c < 0x80) {
|
|
out.push_back(static_cast<char>(c));
|
|
} else if (const std::int32_t digit = DecimalValue(c); digit >= 0) {
|
|
out.push_back(static_cast<char>('0' + digit));
|
|
} else if (IsSpace(c)) {
|
|
out.push_back(' ');
|
|
} else {
|
|
return std::nullopt;
|
|
}
|
|
}
|
|
return out;
|
|
}
|
|
|
|
// A run of digits with single underscores between them, as ParseInt()
|
|
// and ParseFloat() take them; the digits alone.
|
|
std::optional<std::string> Digits(std::string_view text) {
|
|
if (text.empty() || text.front() == '_' || text.back() == '_') return std::nullopt;
|
|
std::string digits;
|
|
char previous = '\0';
|
|
for (const char c : text) {
|
|
if (c == '_') {
|
|
if (previous == '_') return std::nullopt;
|
|
} else if (c >= '0' && c <= '9') {
|
|
digits.push_back(c);
|
|
} else {
|
|
return std::nullopt;
|
|
}
|
|
previous = c;
|
|
}
|
|
return digits;
|
|
}
|
|
|
|
std::string ToLowerAscii(std::string_view text) {
|
|
std::string out(text);
|
|
for (char& c : out) {
|
|
if (c >= 'A' && c <= 'Z') c = static_cast<char>(c - 'A' + 'a');
|
|
}
|
|
return out;
|
|
}
|
|
|
|
// NormPath() on a string that is known to be a path.
|
|
std::vector<std::string> PathComponents(std::string_view path) {
|
|
std::vector<std::string> out;
|
|
for (const std::string& piece : Split(path, "/")) {
|
|
if (!piece.empty()) out.push_back(piece);
|
|
}
|
|
return out;
|
|
}
|
|
}
|
|
|
|
std::u32string Decode(std::string_view utf8) {
|
|
std::u32string out;
|
|
out.reserve(utf8.size());
|
|
for (std::size_t at = 0; at < utf8.size();) {
|
|
const auto [c, used] = Next(utf8, at);
|
|
out.push_back(c);
|
|
at += used;
|
|
}
|
|
return out;
|
|
}
|
|
|
|
std::string Encode(std::u32string_view text) {
|
|
std::string out;
|
|
out.reserve(text.size());
|
|
for (const char32_t c : text) AppendUtf8(out, c);
|
|
return out;
|
|
}
|
|
|
|
std::string Encode(char32_t c) {
|
|
std::string out;
|
|
AppendUtf8(out, c);
|
|
return out;
|
|
}
|
|
|
|
bool IsUtf8(std::string_view bytes) {
|
|
for (std::size_t at = 0; at < bytes.size();) {
|
|
const auto [c, used] = Next(bytes, at);
|
|
if (c == 0xFFFD) {
|
|
// A real U+FFFD in the input is three valid bytes.
|
|
if (used != 3 || bytes.substr(at, 3) != "\xEF\xBF\xBD") return false;
|
|
}
|
|
at += used;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
std::string DecodeReplace(std::string_view bytes) {
|
|
if (IsUtf8(bytes)) return std::string(bytes);
|
|
return Encode(Decode(bytes));
|
|
}
|
|
|
|
std::size_t Length(std::string_view text) {
|
|
std::size_t count = 0;
|
|
for (std::size_t at = 0; at < text.size(); ++count) at += Next(text, at).second;
|
|
return count;
|
|
}
|
|
|
|
std::string Slice(std::string_view text, std::int64_t start, std::optional<std::int64_t> end) {
|
|
const std::u32string decoded = Decode(text);
|
|
const auto size = static_cast<std::int64_t>(decoded.size());
|
|
auto clip = [size](std::int64_t index) {
|
|
if (index < 0) index += size;
|
|
return std::clamp<std::int64_t>(index, 0, size);
|
|
};
|
|
const std::int64_t from = clip(start);
|
|
const std::int64_t to = end ? clip(*end) : size;
|
|
if (to <= from) return {};
|
|
return Encode(std::u32string_view(decoded).substr(static_cast<std::size_t>(from), static_cast<std::size_t>(to - from)));
|
|
}
|
|
|
|
bool IsSpace(char32_t c) {
|
|
return std::ranges::binary_search(Unicode::Space, c);
|
|
}
|
|
|
|
bool IsLineBreak(char32_t c) {
|
|
return std::ranges::binary_search(Unicode::LineBreak, c);
|
|
}
|
|
|
|
bool IsDecimal(char32_t c) {
|
|
return DecimalValue(c) >= 0;
|
|
}
|
|
|
|
std::int32_t DecimalValue(char32_t c) {
|
|
if (IsAsciiDigit(c)) return static_cast<std::int32_t>(c - '0');
|
|
const auto found = std::ranges::upper_bound(Unicode::Decimal, c, {}, [](const Unicode::Run& run) { return run.first; });
|
|
if (found == Unicode::Decimal.begin()) return -1;
|
|
const Unicode::Run& run = *std::prev(found);
|
|
if (c > run.last) return -1;
|
|
return static_cast<std::int32_t>((c - run.first) % 10);
|
|
}
|
|
|
|
bool IsPrintable(char32_t c) {
|
|
return InRuns(Unicode::Printable, c);
|
|
}
|
|
|
|
char IgnoreCaseLetter(char32_t c) {
|
|
if (c >= 'a' && c <= 'z') return static_cast<char>(c);
|
|
if (c >= 'A' && c <= 'Z') return static_cast<char>(c - 'A' + 'a');
|
|
const auto found = std::ranges::lower_bound(Unicode::IgnoreCase, c, {}, [](const Unicode::Fold& fold) { return fold.from; });
|
|
if (found == Unicode::IgnoreCase.end() || found->from != c) return '\0';
|
|
return found->letter;
|
|
}
|
|
|
|
std::string LStrip(std::string_view text) {
|
|
std::size_t at = 0;
|
|
while (at < text.size()) {
|
|
const auto [c, used] = Next(text, at);
|
|
if (!IsSpace(c)) break;
|
|
at += used;
|
|
}
|
|
return std::string(text.substr(at));
|
|
}
|
|
|
|
std::string RStrip(std::string_view text) {
|
|
const std::u32string decoded = Decode(text);
|
|
std::size_t end = decoded.size();
|
|
while (end > 0 && IsSpace(decoded[end - 1])) --end;
|
|
if (end == decoded.size()) return std::string(text);
|
|
return Encode(std::u32string_view(decoded).substr(0, end));
|
|
}
|
|
|
|
std::string Strip(std::string_view text) {
|
|
return RStrip(LStrip(text));
|
|
}
|
|
|
|
std::string LStrip(std::string_view text, std::string_view chars) {
|
|
const std::u32string set = Decode(chars);
|
|
const std::u32string decoded = Decode(text);
|
|
std::size_t start = 0;
|
|
while (start < decoded.size() && set.contains(decoded[start])) ++start;
|
|
return Encode(std::u32string_view(decoded).substr(start));
|
|
}
|
|
|
|
std::string RStrip(std::string_view text, std::string_view chars) {
|
|
const std::u32string set = Decode(chars);
|
|
const std::u32string decoded = Decode(text);
|
|
std::size_t end = decoded.size();
|
|
while (end > 0 && set.contains(decoded[end - 1])) --end;
|
|
return Encode(std::u32string_view(decoded).substr(0, end));
|
|
}
|
|
|
|
std::string Strip(std::string_view text, std::string_view chars) {
|
|
return RStrip(LStrip(text, chars), chars);
|
|
}
|
|
|
|
std::vector<std::string> Split(std::string_view text) {
|
|
std::vector<std::string> out;
|
|
std::string current;
|
|
bool inWord = false;
|
|
for (std::size_t at = 0; at < text.size();) {
|
|
const auto [c, used] = Next(text, at);
|
|
if (IsSpace(c)) {
|
|
if (inWord) out.push_back(std::move(current));
|
|
current.clear();
|
|
inWord = false;
|
|
} else {
|
|
current.append(text.substr(at, used));
|
|
inWord = true;
|
|
}
|
|
at += used;
|
|
}
|
|
if (inWord) out.push_back(std::move(current));
|
|
return out;
|
|
}
|
|
|
|
std::vector<std::string> Split(std::string_view text, std::string_view separator, std::int64_t maxSplit) {
|
|
if (separator.empty()) throw std::invalid_argument("empty separator");
|
|
std::vector<std::string> out;
|
|
std::size_t start = 0;
|
|
while (maxSplit < 0 || static_cast<std::int64_t>(out.size()) < maxSplit) {
|
|
const std::size_t found = text.find(separator, start);
|
|
if (found == std::string_view::npos) break;
|
|
out.emplace_back(text.substr(start, found - start));
|
|
start = found + separator.size();
|
|
}
|
|
out.emplace_back(text.substr(start));
|
|
return out;
|
|
}
|
|
|
|
std::vector<std::string> RSplit(std::string_view text, std::string_view separator, std::int64_t maxSplit) {
|
|
if (separator.empty()) throw std::invalid_argument("empty separator");
|
|
std::vector<std::string> out;
|
|
std::size_t end = text.size();
|
|
while (maxSplit < 0 || static_cast<std::int64_t>(out.size()) < maxSplit) {
|
|
if (end < separator.size()) break;
|
|
const std::size_t found = text.rfind(separator, end - separator.size());
|
|
if (found == std::string_view::npos) break;
|
|
out.emplace_back(text.substr(found + separator.size(), end - found - separator.size()));
|
|
end = found;
|
|
}
|
|
out.emplace_back(text.substr(0, end));
|
|
std::ranges::reverse(out);
|
|
return out;
|
|
}
|
|
|
|
std::vector<std::string> SplitLines(std::string_view text, bool keepEnds) {
|
|
std::vector<std::string> out;
|
|
std::size_t start = 0;
|
|
std::size_t at = 0;
|
|
while (at < text.size()) {
|
|
const auto [c, used] = Next(text, at);
|
|
if (!IsLineBreak(c)) {
|
|
at += used;
|
|
continue;
|
|
}
|
|
std::size_t end = at + used;
|
|
if (c == '\r' && end < text.size() && text[end] == '\n') ++end;
|
|
out.emplace_back(text.substr(start, (keepEnds ? end : at) - start));
|
|
start = end;
|
|
at = end;
|
|
}
|
|
if (start < text.size()) out.emplace_back(text.substr(start));
|
|
return out;
|
|
}
|
|
|
|
std::string Lower(std::string_view text) {
|
|
std::string out;
|
|
out.reserve(text.size());
|
|
for (std::size_t at = 0; at < text.size();) {
|
|
const auto [c, used] = Next(text, at);
|
|
if (c < 0x80) {
|
|
out.push_back(static_cast<char>(c >= 'A' && c <= 'Z' ? c - 'A' + 'a' : c));
|
|
} else if (const std::string_view to = Mapped(Unicode::Lower, c); !to.empty()) {
|
|
out.append(to);
|
|
} else {
|
|
AppendUtf8(out, c);
|
|
}
|
|
at += used;
|
|
}
|
|
return out;
|
|
}
|
|
|
|
std::string Upper(std::string_view text) {
|
|
std::string out;
|
|
out.reserve(text.size());
|
|
for (std::size_t at = 0; at < text.size();) {
|
|
const auto [c, used] = Next(text, at);
|
|
if (c < 0x80) {
|
|
out.push_back(static_cast<char>(c >= 'a' && c <= 'z' ? c - 'a' + 'A' : c));
|
|
} else if (const std::string_view to = Mapped(Unicode::Upper, c); !to.empty()) {
|
|
out.append(to);
|
|
} else {
|
|
AppendUtf8(out, c);
|
|
}
|
|
at += used;
|
|
}
|
|
return out;
|
|
}
|
|
|
|
std::size_t Count(std::string_view text, std::string_view sub) {
|
|
if (sub.empty()) return Length(text) + 1;
|
|
std::size_t count = 0;
|
|
for (std::size_t at = text.find(sub); at != std::string_view::npos; at = text.find(sub, at + sub.size())) ++count;
|
|
return count;
|
|
}
|
|
|
|
std::string Replace(std::string_view text, std::string_view from, std::string_view to) {
|
|
if (from.empty()) return std::string(text);
|
|
std::string out;
|
|
std::size_t start = 0;
|
|
for (std::size_t at = text.find(from); at != std::string_view::npos; at = text.find(from, start)) {
|
|
out.append(text.substr(start, at - start));
|
|
out.append(to);
|
|
start = at + from.size();
|
|
}
|
|
out.append(text.substr(start));
|
|
return out;
|
|
}
|
|
|
|
std::string Join(std::span<const std::string> parts, std::string_view separator) {
|
|
std::string out;
|
|
for (std::size_t i = 0; i < parts.size(); ++i) {
|
|
if (i > 0) out.append(separator);
|
|
out.append(parts[i]);
|
|
}
|
|
return out;
|
|
}
|
|
|
|
std::string ReadText(const std::filesystem::path& path) {
|
|
std::ifstream in(path, std::ios::binary);
|
|
if (!in) throw std::runtime_error(std::format("[Errno 2] No such file or directory: {}", Repr(std::string_view(path.string()))));
|
|
const std::string bytes((std::istreambuf_iterator<char>(in)), std::istreambuf_iterator<char>());
|
|
for (std::size_t at = 0; at < bytes.size();) {
|
|
const auto [c, used] = Next(bytes, at);
|
|
if (c == 0xFFFD && (used != 3 || std::string_view(bytes).substr(at, 3) != "\xEF\xBF\xBD")) {
|
|
throw std::runtime_error(std::format("'utf-8' codec can't decode byte 0x{:02x} in position {}: invalid utf-8 ({})",
|
|
static_cast<std::uint8_t>(bytes[at]), at, path.string()));
|
|
}
|
|
at += used;
|
|
}
|
|
return UniversalNewlines(bytes);
|
|
}
|
|
|
|
std::string UniversalNewlines(std::string_view text) {
|
|
if (!text.contains('\r')) return std::string(text);
|
|
std::string out;
|
|
out.reserve(text.size());
|
|
for (std::size_t i = 0; i < text.size(); ++i) {
|
|
if (text[i] == '\r') {
|
|
out.push_back('\n');
|
|
if (i + 1 < text.size() && text[i + 1] == '\n') ++i;
|
|
} else {
|
|
out.push_back(text[i]);
|
|
}
|
|
}
|
|
return out;
|
|
}
|
|
|
|
std::string FloatRepr(double value) {
|
|
if (std::isnan(value)) return "nan";
|
|
if (std::isinf(value)) return value > 0 ? "inf" : "-inf";
|
|
// The shortest round-trip digits, in to_chars' scientific form, then
|
|
// laid out: fixed from 1e-4 up to 1e16, with an exponent otherwise.
|
|
std::array<char, 64> buffer {};
|
|
const auto [end, error] = std::to_chars(buffer.data(), buffer.data() + buffer.size(), value, std::chars_format::scientific);
|
|
const std::string_view scientific(buffer.data(), end);
|
|
const bool negative = scientific.starts_with('-');
|
|
const std::string_view body = negative ? scientific.substr(1) : scientific;
|
|
const std::size_t e = body.find('e');
|
|
std::string digits;
|
|
for (const char c : body.substr(0, e)) {
|
|
if (c != '.') digits.push_back(c);
|
|
}
|
|
std::int32_t exponent = 0;
|
|
std::from_chars(body.data() + e + 1 + (body[e + 1] == '+' ? 1 : 0), body.data() + body.size(), exponent);
|
|
while (digits.size() > 1 && digits.back() == '0') digits.pop_back();
|
|
const std::int32_t decimalPoint = exponent + 1;
|
|
std::string out = negative ? "-" : "";
|
|
if (decimalPoint <= -4 || decimalPoint > 16) {
|
|
out += digits.substr(0, 1);
|
|
if (digits.size() > 1) out += std::format(".{}", digits.substr(1));
|
|
const std::int32_t shown = decimalPoint - 1;
|
|
out += std::format("e{}{:02}", shown < 0 ? '-' : '+', shown < 0 ? -shown : shown);
|
|
return out;
|
|
}
|
|
if (decimalPoint <= 0) {
|
|
out += std::format("0.{}{}", std::string(static_cast<std::size_t>(-decimalPoint), '0'), digits);
|
|
} else if (static_cast<std::size_t>(decimalPoint) >= digits.size()) {
|
|
out += std::format("{}{}.0", digits, std::string(static_cast<std::size_t>(decimalPoint) - digits.size(), '0'));
|
|
} else {
|
|
out += std::format("{}.{}", digits.substr(0, static_cast<std::size_t>(decimalPoint)), digits.substr(static_cast<std::size_t>(decimalPoint)));
|
|
}
|
|
return out;
|
|
}
|
|
|
|
double Round(double value, std::int32_t digits) {
|
|
if (!std::isfinite(value)) return value;
|
|
// to_chars with a precision rounds the exact binary value correctly,
|
|
// half to even on an exact tie.
|
|
std::array<char, 400> buffer {};
|
|
const auto [end, error] = std::to_chars(buffer.data(), buffer.data() + buffer.size(), value, std::chars_format::fixed, digits);
|
|
if (error != std::errc()) return value;
|
|
double rounded = value;
|
|
std::from_chars(buffer.data(), end, rounded);
|
|
return rounded;
|
|
}
|
|
|
|
std::int64_t RoundToInt(double value) {
|
|
return static_cast<std::int64_t>(std::nearbyint(value));
|
|
}
|
|
|
|
std::optional<std::int64_t> ParseInt(std::string_view text) {
|
|
const std::optional<std::string> ascii = AsciiNumber(text);
|
|
if (!ascii || ascii->empty()) return std::nullopt;
|
|
std::string_view body = *ascii;
|
|
bool negative = false;
|
|
if (body.front() == '+' || body.front() == '-') {
|
|
negative = body.front() == '-';
|
|
body.remove_prefix(1);
|
|
}
|
|
const std::optional<std::string> digits = Digits(body);
|
|
if (!digits) return std::nullopt;
|
|
std::int64_t value = 0;
|
|
for (const char c : *digits) {
|
|
if (value > (std::numeric_limits<std::int64_t>::max() - (c - '0')) / 10) return std::nullopt;
|
|
value = value * 10 + (c - '0');
|
|
}
|
|
return negative ? -value : value;
|
|
}
|
|
|
|
std::optional<double> ParseFloat(std::string_view text) {
|
|
const std::optional<std::string> ascii = AsciiNumber(text);
|
|
if (!ascii || ascii->empty()) return std::nullopt;
|
|
std::string_view body = *ascii;
|
|
bool negative = false;
|
|
if (body.front() == '+' || body.front() == '-') {
|
|
negative = body.front() == '-';
|
|
body.remove_prefix(1);
|
|
}
|
|
const std::string word = ToLowerAscii(body);
|
|
if (word == "inf" || word == "infinity") return negative ? -std::numeric_limits<double>::infinity() : std::numeric_limits<double>::infinity();
|
|
if (word == "nan") return std::numeric_limits<double>::quiet_NaN();
|
|
// digits [. digits] [e [sign] digits], one of the two digit runs
|
|
// present, underscores only between digits.
|
|
const std::size_t exponentAt = body.find_first_of("eE");
|
|
const std::string_view mantissa = body.substr(0, exponentAt);
|
|
const std::size_t dot = mantissa.find('.');
|
|
const std::string_view whole = mantissa.substr(0, dot);
|
|
const std::string_view fraction = dot == std::string_view::npos ? std::string_view() : mantissa.substr(dot + 1);
|
|
if (whole.empty() && fraction.empty()) return std::nullopt;
|
|
std::string normal = negative ? "-" : "";
|
|
if (!whole.empty()) {
|
|
const std::optional<std::string> digits = Digits(whole);
|
|
if (!digits) return std::nullopt;
|
|
normal += *digits;
|
|
}
|
|
if (dot != std::string_view::npos) {
|
|
normal += '.';
|
|
if (!fraction.empty()) {
|
|
const std::optional<std::string> digits = Digits(fraction);
|
|
if (!digits) return std::nullopt;
|
|
normal += *digits;
|
|
}
|
|
}
|
|
if (exponentAt != std::string_view::npos) {
|
|
std::string_view exponent = body.substr(exponentAt + 1);
|
|
normal += 'e';
|
|
if (!exponent.empty() && (exponent.front() == '+' || exponent.front() == '-')) {
|
|
normal += exponent.front();
|
|
exponent.remove_prefix(1);
|
|
}
|
|
const std::optional<std::string> digits = Digits(exponent);
|
|
if (!digits) return std::nullopt;
|
|
normal += *digits;
|
|
}
|
|
// strtod, not from_chars: an exponent out of range is inf or 0.0, not
|
|
// an error. The process runs in the "C" locale.
|
|
return std::strtod(normal.c_str(), nullptr);
|
|
}
|
|
|
|
std::string IntError(std::string_view text) {
|
|
return std::format("invalid literal for int() with base 10: {}", Repr(text));
|
|
}
|
|
|
|
std::string FloatError(std::string_view text) {
|
|
return std::format("could not convert string to float: {}", Repr(text));
|
|
}
|
|
|
|
std::string Repr(std::string_view text) {
|
|
const char quote = text.contains('\'') && !text.contains('"') ? '"' : '\'';
|
|
std::string out(1, quote);
|
|
for (std::size_t at = 0; at < text.size();) {
|
|
const auto [c, used] = Next(text, at);
|
|
if (c == static_cast<char32_t>(quote) || c == '\\') {
|
|
out.push_back('\\');
|
|
out.push_back(static_cast<char>(c));
|
|
} else if (c == '\t') {
|
|
out += "\\t";
|
|
} else if (c == '\n') {
|
|
out += "\\n";
|
|
} else if (c == '\r') {
|
|
out += "\\r";
|
|
} else if (c < 0x20 || c == 0x7F) {
|
|
out += std::format("\\x{:02x}", static_cast<std::uint32_t>(c));
|
|
} else if (c < 0x7F) {
|
|
out.push_back(static_cast<char>(c));
|
|
} else if (IsPrintable(c)) {
|
|
AppendUtf8(out, c);
|
|
} else if (c <= 0xFF) {
|
|
out += std::format("\\x{:02x}", static_cast<std::uint32_t>(c));
|
|
} else if (c <= 0xFFFF) {
|
|
out += std::format("\\u{:04x}", static_cast<std::uint32_t>(c));
|
|
} else {
|
|
out += std::format("\\U{:08x}", static_cast<std::uint32_t>(c));
|
|
}
|
|
at += used;
|
|
}
|
|
out.push_back(quote);
|
|
return out;
|
|
}
|
|
|
|
std::string ReprValue(const Json& value) {
|
|
if (value.is_null()) return "None";
|
|
if (value.is_boolean()) return value.get<bool>() ? "True" : "False";
|
|
if (value.is_number_float()) return FloatRepr(value.get<double>());
|
|
if (value.is_number_unsigned()) return std::to_string(value.get<std::uint64_t>());
|
|
if (value.is_number_integer()) return std::to_string(value.get<std::int64_t>());
|
|
if (value.is_string()) return Repr(std::string_view(value.get_ref<const std::string&>()));
|
|
if (value.is_array()) {
|
|
std::string out = "[";
|
|
bool first = true;
|
|
for (const Json& item : value) {
|
|
if (!first) out += ", ";
|
|
first = false;
|
|
out += ReprValue(item);
|
|
}
|
|
return std::format("{}]", out);
|
|
}
|
|
std::string out = "{";
|
|
bool first = true;
|
|
for (auto it = value.begin(); it != value.end(); ++it) {
|
|
if (!first) out += ", ";
|
|
first = false;
|
|
out += std::format("{}: {}", Repr(std::string_view(it.key())), ReprValue(it.value()));
|
|
}
|
|
return std::format("{}}}", out);
|
|
}
|
|
|
|
namespace {
|
|
// One element of a translated shell pattern.
|
|
struct GlobToken {
|
|
enum class Kind { Star, Any, Literal, Class, Never };
|
|
Kind kind;
|
|
char32_t literal = 0;
|
|
bool negate = false;
|
|
std::vector<std::pair<char32_t, char32_t>> ranges;
|
|
};
|
|
|
|
// A bracket expression, from the text between [ and ] — escapes and
|
|
// all — read as a regular expression reads a character class.
|
|
GlobToken ClassFrom(const std::u32string& content) {
|
|
GlobToken token { .kind = GlobToken::Kind::Class };
|
|
std::u32string_view rest(content);
|
|
if (rest.starts_with(U'!')) {
|
|
token.negate = true;
|
|
rest.remove_prefix(1);
|
|
}
|
|
auto item = [&rest]() {
|
|
char32_t c = rest.front();
|
|
rest.remove_prefix(1);
|
|
if (c == U'\\' && !rest.empty()) {
|
|
c = rest.front();
|
|
rest.remove_prefix(1);
|
|
}
|
|
return c;
|
|
};
|
|
while (!rest.empty()) {
|
|
const char32_t first = item();
|
|
if (rest.size() >= 2 && rest.front() == U'-') {
|
|
rest.remove_prefix(1);
|
|
const char32_t last = item();
|
|
token.ranges.emplace_back(first, last);
|
|
} else {
|
|
token.ranges.emplace_back(first, first);
|
|
}
|
|
}
|
|
return token;
|
|
}
|
|
|
|
std::u32string EscapeClassChunk(std::u32string_view chunk, bool hyphens) {
|
|
std::u32string out;
|
|
for (const char32_t c : chunk) {
|
|
if (c == U'\\' || (hyphens && c == U'-')) out.push_back(U'\\');
|
|
out.push_back(c);
|
|
}
|
|
return out;
|
|
}
|
|
|
|
// A shell pattern, translated token by token.
|
|
std::vector<GlobToken> Translate(const std::u32string& pattern) {
|
|
std::vector<GlobToken> tokens;
|
|
const std::size_t n = pattern.size();
|
|
std::size_t i = 0;
|
|
while (i < n) {
|
|
const char32_t c = pattern[i++];
|
|
if (c == U'*') {
|
|
if (tokens.empty() || tokens.back().kind != GlobToken::Kind::Star) tokens.push_back({ .kind = GlobToken::Kind::Star });
|
|
while (i < n && pattern[i] == U'*') ++i;
|
|
} else if (c == U'?') {
|
|
tokens.push_back({ .kind = GlobToken::Kind::Any });
|
|
} else if (c == U'[') {
|
|
std::size_t j = i;
|
|
if (j < n && pattern[j] == U'!') ++j;
|
|
if (j < n && pattern[j] == U']') ++j;
|
|
while (j < n && pattern[j] != U']') ++j;
|
|
if (j >= n) {
|
|
tokens.push_back({ .kind = GlobToken::Kind::Literal, .literal = U'[' });
|
|
continue;
|
|
}
|
|
std::u32string stuff = pattern.substr(i, j - i);
|
|
if (!stuff.contains(U'-')) {
|
|
stuff = EscapeClassChunk(stuff, false);
|
|
} else {
|
|
std::vector<std::u32string> chunks;
|
|
std::size_t start = i;
|
|
std::size_t k = pattern[i] == U'!' ? i + 2 : i + 1;
|
|
while (true) {
|
|
const std::size_t found = k < j ? pattern.find(U'-', k) : std::u32string::npos;
|
|
if (found == std::u32string::npos || found >= j) break;
|
|
chunks.push_back(pattern.substr(start, found - start));
|
|
start = found + 1;
|
|
k = found + 3;
|
|
}
|
|
const std::u32string chunk = pattern.substr(start, j - start);
|
|
if (!chunk.empty()) {
|
|
chunks.push_back(chunk);
|
|
} else {
|
|
chunks.back() += U'-';
|
|
}
|
|
// A reversed range is dropped: it matches nothing.
|
|
for (std::size_t m = chunks.size() - 1; m > 0; --m) {
|
|
if (chunks[m - 1].back() > chunks[m].front()) {
|
|
chunks[m - 1] = chunks[m - 1].substr(0, chunks[m - 1].size() - 1) + chunks[m].substr(1);
|
|
chunks.erase(chunks.begin() + static_cast<std::ptrdiff_t>(m));
|
|
}
|
|
}
|
|
stuff.clear();
|
|
for (std::size_t m = 0; m < chunks.size(); ++m) {
|
|
if (m > 0) stuff += U'-';
|
|
stuff += EscapeClassChunk(chunks[m], true);
|
|
}
|
|
}
|
|
i = j + 1;
|
|
if (stuff.empty()) {
|
|
tokens.push_back({ .kind = GlobToken::Kind::Never });
|
|
} else if (stuff == U"!") {
|
|
tokens.push_back({ .kind = GlobToken::Kind::Any });
|
|
} else {
|
|
tokens.push_back(ClassFrom(stuff));
|
|
}
|
|
} else {
|
|
tokens.push_back({ .kind = GlobToken::Kind::Literal, .literal = c });
|
|
}
|
|
}
|
|
return tokens;
|
|
}
|
|
|
|
bool Matches(const GlobToken& token, char32_t c) {
|
|
switch (token.kind) {
|
|
case GlobToken::Kind::Any:
|
|
return true;
|
|
case GlobToken::Kind::Literal:
|
|
return token.literal == c;
|
|
case GlobToken::Kind::Class: {
|
|
const bool in = std::ranges::any_of(token.ranges, [c](const auto& range) { return c >= range.first && c <= range.second; });
|
|
return in != token.negate;
|
|
}
|
|
default:
|
|
return false;
|
|
}
|
|
}
|
|
}
|
|
|
|
bool FnMatch(std::string_view name, std::string_view pattern) {
|
|
const std::vector<GlobToken> tokens = Translate(Decode(pattern));
|
|
const std::u32string text = Decode(name);
|
|
// The classic two-pointer match: every token but * consumes exactly
|
|
// one character, so backtracking only ever returns to the last star.
|
|
std::size_t t = 0;
|
|
std::size_t s = 0;
|
|
std::optional<std::size_t> starToken;
|
|
std::size_t starText = 0;
|
|
while (s < text.size()) {
|
|
if (t < tokens.size() && tokens[t].kind == GlobToken::Kind::Star) {
|
|
starToken = t++;
|
|
starText = s;
|
|
} else if (t < tokens.size() && Matches(tokens[t], text[s])) {
|
|
++t;
|
|
++s;
|
|
} else if (starToken) {
|
|
t = *starToken + 1;
|
|
s = ++starText;
|
|
} else {
|
|
return false;
|
|
}
|
|
}
|
|
while (t < tokens.size() && tokens[t].kind == GlobToken::Kind::Star) ++t;
|
|
return t == tokens.size();
|
|
}
|
|
|
|
std::optional<std::string> SplitUrl::Hostname() const {
|
|
std::string hostinfo = netloc.substr(netloc.rfind('@') == std::string::npos ? 0 : netloc.rfind('@') + 1);
|
|
std::string hostname;
|
|
if (const std::size_t open = hostinfo.find('['); open != std::string::npos) {
|
|
const std::string bracketed = hostinfo.substr(open + 1);
|
|
hostname = bracketed.substr(0, bracketed.find(']'));
|
|
} else {
|
|
hostname = hostinfo.substr(0, hostinfo.find(':'));
|
|
}
|
|
if (hostname.empty()) return std::nullopt;
|
|
// A scoped IPv6 address's zone is left as written.
|
|
const std::size_t percent = hostname.find('%');
|
|
if (percent == std::string::npos) return Lower(hostname);
|
|
return Lower(hostname.substr(0, percent)) + hostname.substr(percent);
|
|
}
|
|
|
|
std::optional<std::string> SplitUrl::Username() const {
|
|
const std::size_t at = netloc.rfind('@');
|
|
if (at == std::string::npos) return std::nullopt;
|
|
const std::string userinfo = netloc.substr(0, at);
|
|
return userinfo.substr(0, userinfo.find(':'));
|
|
}
|
|
|
|
std::optional<std::string> SplitUrl::Password() const {
|
|
const std::size_t at = netloc.rfind('@');
|
|
if (at == std::string::npos) return std::nullopt;
|
|
const std::string userinfo = netloc.substr(0, at);
|
|
const std::size_t colon = userinfo.find(':');
|
|
if (colon == std::string::npos) return std::nullopt;
|
|
return userinfo.substr(colon + 1);
|
|
}
|
|
|
|
std::optional<std::int32_t> SplitUrl::Port() const {
|
|
std::string hostinfo = netloc.substr(netloc.rfind('@') == std::string::npos ? 0 : netloc.rfind('@') + 1);
|
|
std::string port;
|
|
if (const std::size_t open = hostinfo.find('['); open != std::string::npos) {
|
|
const std::string bracketed = hostinfo.substr(open + 1);
|
|
const std::size_t close = bracketed.find(']');
|
|
if (close != std::string::npos) {
|
|
const std::string after = bracketed.substr(close + 1);
|
|
if (const std::size_t colon = after.find(':'); colon != std::string::npos) port = after.substr(colon + 1);
|
|
}
|
|
} else if (const std::size_t colon = hostinfo.find(':'); colon != std::string::npos) {
|
|
port = hostinfo.substr(colon + 1);
|
|
}
|
|
if (port.empty()) return std::nullopt;
|
|
if (!std::ranges::all_of(port, [](char c) { return c >= '0' && c <= '9'; })) {
|
|
throw std::invalid_argument(std::format("Port could not be cast to integer value as {}", Repr(port)));
|
|
}
|
|
std::int64_t value = 0;
|
|
for (const char c : port) {
|
|
value = value * 10 + (c - '0');
|
|
if (value > 65535) break;
|
|
}
|
|
if (value > 65535) throw std::invalid_argument("Port out of range 0-65535");
|
|
return static_cast<std::int32_t>(value);
|
|
}
|
|
|
|
namespace {
|
|
// ipaddress.ip_address's verdict on what sits between brackets, as
|
|
// far as urlsplit needs one: an IPv4 address is refused, anything that
|
|
// is not an IPv6 address is invalid.
|
|
void CheckBracketedHost(const std::string& host) {
|
|
if (host.starts_with('v')) {
|
|
// \Av[a-fA-F0-9]+\..+\z
|
|
std::size_t i = 1;
|
|
while (i < host.size() && std::isxdigit(static_cast<std::uint8_t>(host[i]))) ++i;
|
|
if (i == 1 || i >= host.size() || host[i] != '.' || i + 1 >= host.size()) {
|
|
throw std::invalid_argument("IPvFuture address is invalid");
|
|
}
|
|
return;
|
|
}
|
|
std::array<unsigned char, 16> address {};
|
|
std::string withoutZone = host.substr(0, host.find('%'));
|
|
if (inet_pton(AF_INET, withoutZone.c_str(), address.data()) == 1) {
|
|
throw std::invalid_argument("An IPv4 address cannot be in brackets");
|
|
}
|
|
if (inet_pton(AF_INET6, withoutZone.c_str(), address.data()) != 1) {
|
|
throw std::invalid_argument(std::format("{} does not appear to be an IPv4 or IPv6 address", Repr(host)));
|
|
}
|
|
}
|
|
}
|
|
|
|
SplitUrl UrlSplit(std::string_view input) {
|
|
std::string url;
|
|
for (const char c : input) {
|
|
if (c != '\t' && c != '\r' && c != '\n') url.push_back(c);
|
|
}
|
|
std::size_t lead = 0;
|
|
while (lead < url.size() && static_cast<std::uint8_t>(url[lead]) <= 0x20) ++lead;
|
|
url.erase(0, lead);
|
|
|
|
SplitUrl parts;
|
|
const std::size_t colon = url.find(':');
|
|
auto isAlpha = [](char c) { return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z'); };
|
|
auto isSchemeChar = [&](char c) { return isAlpha(c) || (c >= '0' && c <= '9') || c == '+' || c == '-' || c == '.'; };
|
|
if (colon != std::string::npos && colon > 0 && isAlpha(url[0])
|
|
&& std::ranges::all_of(std::string_view(url).substr(0, colon), isSchemeChar)) {
|
|
parts.scheme = ToLowerAscii(std::string_view(url).substr(0, colon));
|
|
url.erase(0, colon + 1);
|
|
}
|
|
if (url.starts_with("//")) {
|
|
std::size_t end = url.find_first_of("/?#", 2);
|
|
if (end == std::string::npos) end = url.size();
|
|
parts.netloc = url.substr(2, end - 2);
|
|
url.erase(0, end);
|
|
const bool open = parts.netloc.contains('[');
|
|
const bool close = parts.netloc.contains(']');
|
|
if (open != close) throw std::invalid_argument("Invalid IPv6 URL");
|
|
if (open && close) {
|
|
// _check_bracketed_netloc: the brackets may sit in the user
|
|
// part, and then the host is checked as an address all the same.
|
|
const std::string hostAndPort = parts.netloc.substr(parts.netloc.rfind('@') == std::string::npos ? 0 : parts.netloc.rfind('@') + 1);
|
|
const std::size_t bracket = hostAndPort.find('[');
|
|
if (bracket == std::string::npos) {
|
|
CheckBracketedHost(hostAndPort.substr(0, hostAndPort.find(':')));
|
|
} else {
|
|
if (bracket != 0) throw std::invalid_argument("Invalid IPv6 URL");
|
|
const std::string bracketed = hostAndPort.substr(1);
|
|
const std::size_t closing = bracketed.find(']');
|
|
const std::string after = closing == std::string::npos ? std::string() : bracketed.substr(closing + 1);
|
|
if (!after.empty() && !after.starts_with(':')) throw std::invalid_argument("Invalid IPv6 URL");
|
|
CheckBracketedHost(bracketed.substr(0, closing));
|
|
}
|
|
}
|
|
}
|
|
if (const std::size_t hash = url.find('#'); hash != std::string::npos) {
|
|
parts.fragment = url.substr(hash + 1);
|
|
url.erase(hash);
|
|
}
|
|
if (const std::size_t question = url.find('?'); question != std::string::npos) {
|
|
parts.query = url.substr(question + 1);
|
|
url.erase(question);
|
|
}
|
|
parts.path = std::move(url);
|
|
return parts;
|
|
}
|
|
|
|
std::string Unquote(std::string_view text) {
|
|
if (!text.contains('%')) return std::string(text);
|
|
// Percent sequences become bytes; runs of bytes are decoded as UTF-8
|
|
// with bad bytes replaced; everything else is kept as it is.
|
|
std::string out;
|
|
std::string bytes;
|
|
auto hexValue = [](char c) -> std::int32_t {
|
|
if (c >= '0' && c <= '9') return c - '0';
|
|
if (c >= 'a' && c <= 'f') return c - 'a' + 10;
|
|
if (c >= 'A' && c <= 'F') return c - 'A' + 10;
|
|
return -1;
|
|
};
|
|
for (std::size_t i = 0; i < text.size();) {
|
|
if (text[i] == '%' && i + 2 < text.size() && hexValue(text[i + 1]) >= 0 && hexValue(text[i + 2]) >= 0) {
|
|
bytes.push_back(static_cast<char>(hexValue(text[i + 1]) * 16 + hexValue(text[i + 2])));
|
|
i += 3;
|
|
continue;
|
|
}
|
|
if (!bytes.empty()) {
|
|
out += DecodeReplace(bytes);
|
|
bytes.clear();
|
|
}
|
|
out.push_back(text[i]);
|
|
++i;
|
|
}
|
|
if (!bytes.empty()) out += DecodeReplace(bytes);
|
|
return out;
|
|
}
|
|
|
|
NaturalKey Natural(std::string_view name) {
|
|
NaturalKey key;
|
|
std::string text;
|
|
std::int64_t number = 0;
|
|
bool inDigits = false;
|
|
for (std::size_t at = 0; at < name.size();) {
|
|
const auto [c, used] = Next(name, at);
|
|
const std::int32_t digit = DecimalValue(c);
|
|
if (digit >= 0) {
|
|
if (!inDigits) {
|
|
key.parts.emplace_back(text);
|
|
text.clear();
|
|
number = 0;
|
|
inDigits = true;
|
|
}
|
|
number = number * 10 + digit;
|
|
} else {
|
|
if (inDigits) {
|
|
key.parts.emplace_back(number);
|
|
inDigits = false;
|
|
}
|
|
text.append(name.substr(at, used));
|
|
}
|
|
at += used;
|
|
}
|
|
if (inDigits) {
|
|
key.parts.emplace_back(number);
|
|
text.clear();
|
|
}
|
|
key.parts.emplace_back(text);
|
|
return key;
|
|
}
|
|
|
|
std::string NormPath(std::string_view path) {
|
|
if (path.empty()) return ".";
|
|
std::size_t leading = 0;
|
|
while (leading < path.size() && path[leading] == '/') ++leading;
|
|
// POSIX gives exactly two leading slashes a meaning of their own.
|
|
const std::string prefix = leading == 0 ? "" : (leading == 2 ? "//" : "/");
|
|
std::vector<std::string> kept;
|
|
for (const std::string& component : PathComponents(path)) {
|
|
if (component == ".") continue;
|
|
if (component == "..") {
|
|
if (!kept.empty() && kept.back() != "..") {
|
|
kept.pop_back();
|
|
} else if (prefix.empty()) {
|
|
kept.push_back(component);
|
|
}
|
|
continue;
|
|
}
|
|
kept.push_back(component);
|
|
}
|
|
std::string out = prefix + Join(kept, "/");
|
|
return out.empty() ? std::string(".") : out;
|
|
}
|
|
|
|
std::string AbsPath(std::string_view path) {
|
|
if (path.starts_with('/')) return NormPath(path);
|
|
return NormPath(JoinPath(std::filesystem::current_path().string(), path));
|
|
}
|
|
|
|
std::string RelPath(std::string_view path, std::string_view start) {
|
|
if (path.empty()) throw std::invalid_argument("no path specified");
|
|
const std::vector<std::string> from = PathComponents(AbsPath(start.empty() ? "." : start));
|
|
const std::vector<std::string> to = PathComponents(AbsPath(path));
|
|
std::size_t common = 0;
|
|
while (common < from.size() && common < to.size() && from[common] == to[common]) ++common;
|
|
std::vector<std::string> relative(from.size() - common, "..");
|
|
relative.insert(relative.end(), to.begin() + static_cast<std::ptrdiff_t>(common), to.end());
|
|
if (relative.empty()) return ".";
|
|
return Join(relative, "/");
|
|
}
|
|
|
|
std::string BaseName(std::string_view path) {
|
|
return std::string(path.substr(path.rfind('/') == std::string_view::npos ? 0 : path.rfind('/') + 1));
|
|
}
|
|
|
|
std::string DirName(std::string_view path) {
|
|
const std::size_t slash = path.rfind('/');
|
|
if (slash == std::string_view::npos) return "";
|
|
std::string head(path.substr(0, slash + 1));
|
|
if (!std::ranges::all_of(head, [](char c) { return c == '/'; })) {
|
|
while (!head.empty() && head.back() == '/') head.pop_back();
|
|
}
|
|
return head;
|
|
}
|
|
|
|
std::string JoinPath(std::string_view a, std::string_view b) {
|
|
if (b.starts_with('/')) return std::string(b);
|
|
if (a.empty() || a.ends_with('/')) return std::string(a) + std::string(b);
|
|
return std::format("{}/{}", std::string(a), std::string(b));
|
|
}
|
|
|
|
std::string ErrnoText(std::int32_t code) {
|
|
return std::format("[Errno {}] {}", code, std::strerror(code));
|
|
}
|
|
}
|