Catcrafts/Crafter.Build
Watch
2
0
Fork
You've already forked Crafter.Build
 0
Code Issues Pull requests Projects Releases 1 Packages Wiki Activity Actions

Merge branch 'lint-ast': libclang token layer for the linter

Browse source 
Replaces the linter's hand-rolled character scanning with clang's lexer.

StripComments could not see raw string literals, which was documented as a
v1 limitation but was worse than that: an odd number of quotes inside a raw
string desynchronised it for the rest of the file, hiding real code and
leaving the contents of later literals standing as if they were code. There
are 33 raw strings across 6 files here, including the two largest.

The reflow guards had the same class of bug from the other direction. They
were substring probes, so Line(n).contains("//") fired on // inside a string
literal and contains("R\"") fired on the characters R" inside a literal while
missing raw strings opened on an earlier line. Both misfire on this repo:
"MARKER" ends in R". They are now LineHasComment() and
LineHasMultiLineToken(), and two wrapped call sites in tests/Lint that had
been silently unjoinable join as a result.

libclang is dlopen'd rather than linked, so the mingw and MSVC cross-builds
keep linking and the release tarballs stay self-contained. Tokens need no
PCMs, no build flags and no prior build, and they cover preprocessor branches
that are inactive for the host — which is why the layout rules belong on
tokens rather than on an AST that would only ever see one platform's slice.

No rule behaviour was intended to change beyond the two guard fixes; lint
over this repo produced byte-identical output across the CommentStripped
swap.
This commit is contained in:
Jorijn van der Graaf 2026-07-30 22:57:10 +02:00
commit 5fa9c8a816
6 changed files with 553 additions and 88 deletions
Download patch file Download diff file Expand all files Collapse all files

43
tests/HouseRules/main.cpp
View file

@ -252,6 +252,49 @@ int main() {
Check(second.summary.changedFiles.empty(), "wrap-join is idempotent after the fixpoint");
}
// The guards used to be substring probes over the raw line, so a literal
// whose TEXT contained `//` or `R"` looked like a comment or a raw string
// and silently disabled the rule. Both shapes occur in this repo's own
// sources — "MARKER" ends in the characters R", and any string mentioning
// a lint-disable directive contains //.
{
RuleRun r = RunRule("void F() {\n" " auto hit = text.find(\"MARKER\",\n" " start);\n" "}\n", "wrap-join", LintMode::Apply);
Check(r.text.contains("text.find(\"MARKER\", start);"), "R\" inside a literal no longer blocks wrap-join");
}
{
RuleRun r = RunRule("void F() {\n" " Check(read() == \"// lint-disable-next-line trim\\n\",\n" " \"message\");\n" "}\n", "wrap-join", LintMode::Apply);
Check(r.text.contains("\\n\", \"message\");"), "// inside a literal no longer blocks wrap-join");
}
// A real trailing comment still blocks the join: text pulled up past a
// `//` would be swallowed by it.
{
RuleRun r = RunRule("void F() {\n" " auto v = g(alpha, // why\n" " beta);\n" "}\n", "wrap-join", LintMode::Apply);
Check(r.text.contains("g(alpha, // why\n"), "a real comment still blocks wrap-join");
}
// A genuinely multi-line literal is never reflowed — joining its lines
// would change the string's contents.
{
std::string_view source = "void F() {\n"
" auto text = R\"sql(SELECT a,\n"
" b FROM t)sql\";\n"
"}\n";
RuleRun r = RunRule(source, "wrap-join", LintMode::Apply);
Check(r.text == source, "wrap-join leaves a multi-line raw string alone");
RuleRun paren = RunRule(source, "paren-spacing", LintMode::Apply);
Check(paren.text == source, "paren-spacing leaves a multi-line raw string alone");
}
// Type keywords inside a literal are text, not declarations.
{
std::string_view source = "void F() {\n"
" auto sql = R\"q(int x; unsigned long y;)q\";\n"
"}\n";
RuleRun r = RunRule(source, "fixed-width-types", LintMode::Apply);
Check(r.text == source, "fixed-width-types leaves type names inside a raw string alone");
}
if (Failures > 0) {
std::println(std::cerr, "{} assertions failed", Failures);
return 1;

136
tests/Lint/main.cpp
View file

@ -156,8 +156,7 @@ int main() {
Configuration cfg = FixtureConfig();
cfg.AddLintRule("marker", [](LintContext& ctx) {
const std::string& code = ctx.CommentStripped();
for (std::size_t pos = code.find("MARKER"); pos != std::string::npos;
pos = code.find("MARKER", pos + 1)) {
for (std::size_t pos = code.find("MARKER"); pos != std::string::npos; pos = code.find("MARKER", pos + 1)) {
std::size_t line = 1 + std::count(code.begin(), code.begin() + pos, '\n');
ctx.Report(line, "MARKER in code");
}
@ -370,8 +369,7 @@ int main() {
Configuration cfg = s.Config({"a"});
AddTrimRule(cfg);
RunLint(cfg, Mode(LintMode::Apply));
Check(s.Read("a") == "// lint-disable-next-line trim\nkeep \ntrim\n",
"suppressed line keeps its bytes; the unsuppressed one is fixed");
Check(s.Read("a") == "// lint-disable-next-line trim\nkeep \ntrim\n", "suppressed line keeps its bytes; the unsuppressed one is fixed");
}
// Multiple rule names on one directive (space- or comma-separated).
@ -427,6 +425,136 @@ int main() {
Check(s.Read("a") == "// lint-disable-file flag\nbad\n", "other rules still format");
}
// ---------------- token layer ----------------
//
// The source is spelled with escaped literals rather than a raw string so
// that this file stays lintable by the very rules under test; the scratch
// file it writes does contain a genuine multi-line raw string.
{
constexpr std::string_view Source =
"#ifdef CRAFTER_LINT_NEVER_DEFINED\n" // 1
"void HiddenBranch();\n" // 2
"#endif\n" // 3
"// a real comment\n" // 4
"int url = 1; // https://example.com\n" // 5
"auto raw = R\"raw(spans lines\n" // 6
" // not a comment\n" // 7
" int notADecl;\n" // 8
")raw\";\n"; // 9
Scratch s("tokens");
s.Write("f", Source);
Configuration cfg = s.Config({"f"});
cfg.AddLintRule("tokens", [](LintContext& ctx) {
std::span<const LintToken> toks = ctx.Tokens();
Check(!toks.empty(), "tokens: file lexes to a non-empty stream");
// Every token's offset/length must address its own bytes, or a
// transform editing at an offset would corrupt the file.
bool offsetsSound = true;
for (const LintToken& t : toks) {
if (t.offset + t.length > ctx.content.size() || ctx.TokenText(t).empty()) offsetsSound = false;
}
Check(offsetsSound, "tokens: every offset/length addresses real bytes");
// Ordered by offset, so binary search in TokensOnLine is valid.
bool ordered = std::ranges::is_sorted(toks, {}, &LintToken::offset);
Check(ordered, "tokens: stream is in source order");
// Inactive #ifdef branch is still lexed — this is what keeps token
// rules covering every platform, unlike an AST.
bool sawHidden = std::ranges::any_of(toks, [&](const LintToken& t) {
return t.kind == LintTokenKind::Identifier && ctx.TokenText(t) == "HiddenBranch";
});
Check(sawHidden, "tokens: inactive #ifdef branch is lexed");
// A multi-line raw string is exactly one Literal, comment markers
// and declarations inside it included.
auto isRaw = [&](const LintToken& t) { return ctx.TokenText(t).starts_with("R\"raw("); };
Check(std::ranges::count_if(toks, isRaw) == 1, "tokens: raw string is a single token");
auto raw = std::ranges::find_if(toks, isRaw);
if (raw != toks.end()) {
Check(raw->kind == LintTokenKind::Literal, "tokens: raw string is a Literal");
Check(raw->line == 6, "tokens: raw string starts on line 6");
Check(ctx.TokenText(*raw).contains("// not a comment"), "tokens: raw string body kept intact");
Check(ctx.TokenText(*raw).ends_with(")raw\""), "tokens: raw string spans to its own terminator");
}
// The only comments are the two real ones on lines 4 and 5 — the
// `//` on line 7 lives inside the raw string.
std::vector<std::size_t> commentLines;
for (const LintToken& t : toks) {
if (t.kind == LintTokenKind::Comment) commentLines.push_back(t.line);
}
Check(commentLines == std::vector<std::size_t>{4, 5}, "tokens: only real comments are Comment tokens");
Check(ctx.LineHasComment(4), "tokens: LineHasComment finds a whole-line comment");
Check(ctx.LineHasComment(5), "tokens: LineHasComment finds a trailing comment");
Check(!ctx.LineHasComment(7), "tokens: `//` inside a raw string is not a comment");
Check(!ctx.LineHasComment(2), "tokens: code-only line has no comment");
// TokensOnLine brackets by starting line.
std::span<const LintToken> line2 = ctx.TokensOnLine(2);
Check(!line2.empty() && ctx.TokenText(line2.front()) == "void", "tokens: TokensOnLine starts at the line's first token");
Check(std::ranges::all_of(line2, [](const LintToken& t) { return t.line == 2; }), "tokens: TokensOnLine stays on its line");
// SetContent must invalidate the cache, or offsets point into a
// buffer that no longer exists.
ctx.SetContent("int replaced;\n");
std::span<const LintToken> after = ctx.Tokens();
Check(!after.empty() && ctx.TokenText(after.front()) == "int", "tokens: re-lexed after SetContent");
Check(std::ranges::none_of(after, [&](const LintToken& t) { return ctx.TokenText(t) == "HiddenBranch"; }), "tokens: stale tokens are dropped after SetContent");
});
RunLint(cfg, Mode(LintMode::Report));
}
// CommentStripped over a raw string holding an ODD number of quotes. The
// character-scanning version treated R"( as an ordinary string open, so the
// quote inside the body closed it early and every following line was
// swallowed as literal text — code after the raw string vanished from the
// stripped view. Lexing gets the extent right.
{
constexpr std::string_view Source =
"auto banner = R\"(he said \"hi)\";\n" // 1: one quote inside the body
"int afterRaw = 2;\n" // 2: must survive as code
"// MARKER comment\n" // 3
"auto plain = \"MARKER text\";\n"; // 4
Scratch s("strip-rawstring");
s.Write("f", Source);
Configuration cfg = s.Config({"f"});
cfg.AddLintRule("strip", [](LintContext& ctx) {
const std::string& code = ctx.CommentStripped();
Check(code.size() == ctx.content.size(), "strip: byte length preserved");
Check(std::ranges::count(code, '\n') == std::ranges::count(ctx.content, '\n'), "strip: newlines preserved");
Check(code.contains("afterRaw"), "strip: code after an odd-quoted raw string survives");
Check(!code.contains("he said"), "strip: raw string body is blanked");
Check(!code.contains("MARKER"), "strip: comment and literal bodies are blanked");
Check(code.contains("auto plain ="), "strip: code around a literal survives");
// The raw string collapses to R"…" — exactly two quotes, so rules
// that bracket a literal by counting quotes still work.
std::string_view line1 = std::string_view(code).substr(0, code.find('\n'));
Check(std::ranges::count(line1, '"') == 2, "strip: raw string leaves exactly two quotes");
});
RunLint(cfg, Mode(LintMode::Report));
}
// Non-C++ extensions are not lexed: GLSL through a C++ lexer would produce
// plausible-looking nonsense rather than an honest refusal.
{
Scratch s("tokens-foreign");
fs::path shader = s.dir / "f.frag";
{
std::ofstream f(shader, std::ios::binary | std::ios::trunc);
f << "#version 450\nvoid main() { }\n";
}
Configuration cfg = s.Config({});
cfg.shaders.emplace_back(fs::path(shader), "main", ShaderType::Fragment);
cfg.AddLintRule("no-lex", [](LintContext& ctx) {
Check(ctx.Tokens().empty(), "tokens: shaders are not lexed as C++");
});
RunLint(cfg, Mode(LintMode::Report));
}
if (Failures > 0) {
std::println(std::cerr, "{} assertions failed", Failures);
return 1;