diff --git a/implementations/Crafter.Build-Lint.cpp b/implementations/Crafter.Build-Lint.cpp index 93756f6..b010b8a 100644 --- a/implementations/Crafter.Build-Lint.cpp +++ b/implementations/Crafter.Build-Lint.cpp @@ -215,72 +215,45 @@ namespace { return tokens; } - // Blank //-comments, /*...*/ comments and string/char literal bodies to - // spaces while copying '\n' through, so byte offsets and line numbers in - // the result match the original text. Raw string literals are not - // recognized (documented v1 limitation) — their bodies pass through as - // ordinary string content until the first '"'. - std::string StripComments(std::string_view src) { - enum class State { Code, LineComment, BlockComment, String, Char }; - std::string out; - out.reserve(src.size()); - State state = State::Code; - for (std::size_t i = 0; i < src.size(); ++i) { - char c = src[i]; - char next = i + 1 < src.size() ? src[i + 1] : '\0'; - switch (state) { - case State::Code: - if (c == '/' && next == '/') { - state = State::LineComment; - out += " "; - ++i; - } else if (c == '/' && next == '*') { - state = State::BlockComment; - out += " "; - ++i; - } else if (c == '"') { - state = State::String; - out += c; // keep the delimiter so quoting stays visible - } else if (c == '\'') { - state = State::Char; - out += c; - } else { - out += c; - } - break; - case State::LineComment: - if (c == '\n') { - state = State::Code; - out += c; - } else { - out += ' '; - } - break; - case State::BlockComment: - if (c == '*' && next == '/') { - state = State::Code; - out += " "; - ++i; - } else { - out += c == '\n' ? '\n' : ' '; - } - break; - case State::String: - case State::Char: { - char delim = state == State::String ? '"' : '\''; - if (c == '\\' && next != '\0') { - out += " "; - ++i; - if (next == '\n') out.back() = '\n'; - } else if (c == delim) { - state = State::Code; - out += c; - } else { - out += c == '\n' ? '\n' : ' '; - } - break; - } + // Blank comments and the bodies of string/character literals to spaces, + // copying '\n' through so byte offsets and line numbers in the result + // match the original text exactly. + // + // Derived from the token stream rather than scanned character by + // character, which is what makes raw strings, escapes, encoding prefixes + // and a literal like '"' come out right. Editing a copy of the buffer in + // place — rather than appending to a fresh string — makes the + // length-preserving property structural instead of something every branch + // has to remember. + std::string StripLiterals(const std::string& content, std::span tokens) { + std::string out = content; + auto blank = [&out](std::size_t from, std::size_t to) { + for (std::size_t i = from; i < to && i < out.size(); ++i) { + if (out[i] != '\n') out[i] = ' '; } + }; + for (const LintToken& token : tokens) { + std::size_t begin = token.offset; + std::size_t end = token.offset + token.length; + if (token.kind == LintTokenKind::Comment) { + blank(begin, end); + continue; + } + if (token.kind != LintTokenKind::Literal || token.length < 2) continue; + std::string_view text(content.data() + begin, token.length); + // Numeric literals are code and stay; only string and character + // literals have a body to hide. A digit separator makes 1'000 look + // quote-ish, so require everything before the quote to be an + // encoding prefix (L, u, U, u8, R and their combinations). + std::size_t quote = text.find_first_of("\"'"); + if (quote == std::string_view::npos) continue; + std::string_view prefix = text.substr(0, quote); + if (!std::ranges::all_of(prefix, [](char c) { return c == 'L' || c == 'u' || c == 'U' || c == '8' || c == 'R'; })) continue; + // Keep the opening quote and the closing one, blank everything + // between. For a raw string that also blanks the R"delim( and + // )delim" scaffolding, leaving exactly two quotes — which is what + // rules counting quotes to find a literal's extent rely on. + blank(begin + quote + 1, end - 1); } return out; } @@ -362,7 +335,7 @@ std::string_view LintContext::Line(std::size_t n) const { const std::string& LintContext::CommentStripped() { if (!commentStrippedCache) { - commentStrippedCache = StripComments(content); + commentStrippedCache = StripLiterals(content, Tokens()); } return *commentStrippedCache; } diff --git a/interfaces/Crafter.Build-Clang.cppm b/interfaces/Crafter.Build-Clang.cppm index ac9b81d..e5a1f1f 100644 --- a/interfaces/Crafter.Build-Clang.cppm +++ b/interfaces/Crafter.Build-Clang.cppm @@ -161,10 +161,16 @@ export namespace Crafter { CRAFTER_API std::string Extension() const; // ".cppm", ".cpp", ".h", ... CRAFTER_API std::string_view Line(std::size_t n) const; // 1-based; empty if out of range - // `content` with //-comments, /*...*/ comments and string/char literal - // bodies blanked to spaces, newlines preserved — offsets and line - // numbers stay valid. Built on first call, cached per file. Raw string - // literals are not recognized (v1 limitation). + // `content` with comments and string/char literal bodies blanked to + // spaces, newlines preserved — offsets and line numbers stay valid. + // Built from Tokens() on first call, cached per file, so raw strings, + // escapes, encoding prefixes and literals like '"' all come out right. + // A raw string is reduced to R"…" with the body and the delimiter + // scaffolding blanked, leaving exactly two quote characters. + // + // Convenient for a quick scan, but Tokens() is the better tool for + // anything structural: this view cannot tell an identifier from a + // keyword, and it has already thrown away where the literals were. CRAFTER_API const std::string& CommentStripped(); // Record a finding at `line` (1-based; pass 0 for a whole-file finding). CRAFTER_API void Report(std::size_t line, std::string message); diff --git a/tests/Lint/main.cpp b/tests/Lint/main.cpp index 1af4d0c..cad9b5e 100644 --- a/tests/Lint/main.cpp +++ b/tests/Lint/main.cpp @@ -504,8 +504,38 @@ int main() { ctx.SetContent("int replaced;\n"); std::span after = ctx.Tokens(); Check(!after.empty() && ctx.TokenText(after.front()) == "int", "tokens: re-lexed after SetContent"); - Check(std::ranges::none_of(after, [&](const LintToken& t) { return ctx.TokenText(t) == "HiddenBranch"; }), - "tokens: stale tokens are dropped after SetContent"); + Check(std::ranges::none_of(after, [&](const LintToken& t) { return ctx.TokenText(t) == "HiddenBranch"; }), "tokens: stale tokens are dropped after SetContent"); + }); + RunLint(cfg, Mode(LintMode::Report)); + } + + // CommentStripped over a raw string holding an ODD number of quotes. The + // character-scanning version treated R"( as an ordinary string open, so the + // quote inside the body closed it early and every following line was + // swallowed as literal text — code after the raw string vanished from the + // stripped view. Lexing gets the extent right. + { + constexpr std::string_view Source = + "auto banner = R\"(he said \"hi)\";\n" // 1: one quote inside the body + "int afterRaw = 2;\n" // 2: must survive as code + "// MARKER comment\n" // 3 + "auto plain = \"MARKER text\";\n"; // 4 + + Scratch s("strip-rawstring"); + s.Write("f", Source); + Configuration cfg = s.Config({"f"}); + cfg.AddLintRule("strip", [](LintContext& ctx) { + const std::string& code = ctx.CommentStripped(); + Check(code.size() == ctx.content.size(), "strip: byte length preserved"); + Check(std::ranges::count(code, '\n') == std::ranges::count(ctx.content, '\n'), "strip: newlines preserved"); + Check(code.contains("afterRaw"), "strip: code after an odd-quoted raw string survives"); + Check(!code.contains("he said"), "strip: raw string body is blanked"); + Check(!code.contains("MARKER"), "strip: comment and literal bodies are blanked"); + Check(code.contains("auto plain ="), "strip: code around a literal survives"); + // The raw string collapses to R"…" — exactly two quotes, so rules + // that bracket a literal by counting quotes still work. + std::string_view line1 = std::string_view(code).substr(0, code.find('\n')); + Check(std::ranges::count(line1, '"') == 2, "strip: raw string leaves exactly two quotes"); }); RunLint(cfg, Mode(LintMode::Report)); }