Merge branch 'lint-ast': libclang token layer for the linter
Replaces the linter's hand-rolled character scanning with clang's lexer.
StripComments could not see raw string literals, which was documented as a
v1 limitation but was worse than that: an odd number of quotes inside a raw
string desynchronised it for the rest of the file, hiding real code and
leaving the contents of later literals standing as if they were code. There
are 33 raw strings across 6 files here, including the two largest.
The reflow guards had the same class of bug from the other direction. They
were substring probes, so Line(n).contains("//") fired on // inside a string
literal and contains("R\"") fired on the characters R" inside a literal while
missing raw strings opened on an earlier line. Both misfire on this repo:
"MARKER" ends in R". They are now LineHasComment() and
LineHasMultiLineToken(), and two wrapped call sites in tests/Lint that had
been silently unjoinable join as a result.
libclang is dlopen'd rather than linked, so the mingw and MSVC cross-builds
keep linking and the release tarballs stay self-contained. Tokens need no
PCMs, no build flags and no prior build, and they cover preprocessor branches
that are inactive for the host — which is why the layout rules belong on
tokens rather than on an AST that would only ever see one platform's slice.
No rule behaviour was intended to change beyond the two guard fixes; lint
over this repo produced byte-identical output across the CommentStripped
swap.
This commit is contained in:
commit
5fa9c8a816
6 changed files with 553 additions and 88 deletions
|
|
@ -2,6 +2,12 @@
|
|||
// SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts®
|
||||
|
||||
module;
|
||||
#include <clang-c/Index.h>
|
||||
#if defined(CRAFTER_BUILD_CONFIGURATION_TARGET_x86_64_pc_windows_msvc) || defined(CRAFTER_BUILD_CONFIGURATION_TARGET_x86_64_w64_mingw32)
|
||||
#include <windows.h>
|
||||
#else
|
||||
#include <dlfcn.h>
|
||||
#endif
|
||||
export module Crafter.Build:Lint_impl;
|
||||
import std;
|
||||
import :Lint;
|
||||
|
|
@ -12,72 +18,242 @@ namespace fs = std::filesystem;
|
|||
using namespace Crafter;
|
||||
|
||||
namespace {
|
||||
// Blank //-comments, /*...*/ comments and string/char literal bodies to
|
||||
// spaces while copying '\n' through, so byte offsets and line numbers in
|
||||
// the result match the original text. Raw string literals are not
|
||||
// recognized (documented v1 limitation) — their bodies pass through as
|
||||
// ordinary string content until the first '"'.
|
||||
std::string StripComments(std::string_view src) {
|
||||
enum class State { Code, LineComment, BlockComment, String, Char };
|
||||
std::string out;
|
||||
out.reserve(src.size());
|
||||
State state = State::Code;
|
||||
for (std::size_t i = 0; i < src.size(); ++i) {
|
||||
char c = src[i];
|
||||
char next = i + 1 < src.size() ? src[i + 1] : '\0';
|
||||
switch (state) {
|
||||
case State::Code:
|
||||
if (c == '/' && next == '/') {
|
||||
state = State::LineComment;
|
||||
out += " ";
|
||||
++i;
|
||||
} else if (c == '/' && next == '*') {
|
||||
state = State::BlockComment;
|
||||
out += " ";
|
||||
++i;
|
||||
} else if (c == '"') {
|
||||
state = State::String;
|
||||
out += c; // keep the delimiter so quoting stays visible
|
||||
} else if (c == '\'') {
|
||||
state = State::Char;
|
||||
out += c;
|
||||
} else {
|
||||
out += c;
|
||||
}
|
||||
break;
|
||||
case State::LineComment:
|
||||
if (c == '\n') {
|
||||
state = State::Code;
|
||||
out += c;
|
||||
} else {
|
||||
out += ' ';
|
||||
}
|
||||
break;
|
||||
case State::BlockComment:
|
||||
if (c == '*' && next == '/') {
|
||||
state = State::Code;
|
||||
out += " ";
|
||||
++i;
|
||||
} else {
|
||||
out += c == '\n' ? '\n' : ' ';
|
||||
}
|
||||
break;
|
||||
case State::String:
|
||||
case State::Char: {
|
||||
char delim = state == State::String ? '"' : '\'';
|
||||
if (c == '\\' && next != '\0') {
|
||||
out += " ";
|
||||
++i;
|
||||
if (next == '\n') out.back() = '\n';
|
||||
} else if (c == delim) {
|
||||
state = State::Code;
|
||||
out += c;
|
||||
} else {
|
||||
out += c == '\n' ? '\n' : ' ';
|
||||
}
|
||||
break;
|
||||
}
|
||||
// ---------------- libclang ----------------
|
||||
//
|
||||
// libclang is loaded at runtime rather than linked. Linking -lclang would
|
||||
// break the mingw and MSVC cross-builds at link time and would put a
|
||||
// libclang.so.NN runtime dependency into the otherwise self-contained
|
||||
// release tarballs; the clang-c header is used for its declarations only,
|
||||
// and every call goes through a pointer resolved here. Failure to load is
|
||||
// a hard error surfaced once by RunLint — there is deliberately no second,
|
||||
// weaker lexer to fall back to, because two engines disagreeing about what
|
||||
// is a comment is a worse failure than not running.
|
||||
#if defined(CRAFTER_BUILD_CONFIGURATION_TARGET_x86_64_pc_windows_msvc) || defined(CRAFTER_BUILD_CONFIGURATION_TARGET_x86_64_w64_mingw32)
|
||||
using LibHandle = HMODULE;
|
||||
LibHandle OpenLibrary(const std::string& name) { return LoadLibraryA(name.c_str()); }
|
||||
void* LibrarySymbol(LibHandle handle, const std::string& name) { return reinterpret_cast<void*>(GetProcAddress(handle, name.c_str())); }
|
||||
constexpr std::string_view LibClangNames[] = {"libclang.dll", "clang.dll"};
|
||||
#else
|
||||
using LibHandle = void*;
|
||||
LibHandle OpenLibrary(const std::string& name) { return dlopen(name.c_str(), RTLD_NOW | RTLD_LOCAL); }
|
||||
void* LibrarySymbol(LibHandle handle, const std::string& name) { return dlsym(handle, name.c_str()); }
|
||||
constexpr std::string_view LibClangNames[] = {
|
||||
"libclang.so", "libclang.so.22.1", "libclang.so.21.1", "libclang.so.20.1",
|
||||
"libclang.so.1", "libclang.dylib",
|
||||
};
|
||||
#endif
|
||||
|
||||
// Signatures come from decltype on the header's declarations, so they can
|
||||
// never drift from the real API. decltype is unevaluated, so naming the
|
||||
// functions here does not create a link-time reference to them.
|
||||
struct LibClang {
|
||||
LibHandle handle = nullptr;
|
||||
std::string error; // non-empty exactly when handle is null
|
||||
|
||||
decltype(&clang_createIndex) CreateIndex = nullptr;
|
||||
decltype(&clang_disposeIndex) DisposeIndex = nullptr;
|
||||
decltype(&clang_parseTranslationUnit) ParseTranslationUnit = nullptr;
|
||||
decltype(&clang_disposeTranslationUnit) DisposeTranslationUnit = nullptr;
|
||||
decltype(&clang_getFile) GetFile = nullptr;
|
||||
decltype(&clang_getLocationForOffset) GetLocationForOffset = nullptr;
|
||||
decltype(&clang_getRange) GetRange = nullptr;
|
||||
decltype(&clang_getRangeStart) GetRangeStart = nullptr;
|
||||
decltype(&clang_getRangeEnd) GetRangeEnd = nullptr;
|
||||
decltype(&clang_getFileLocation) GetFileLocation = nullptr;
|
||||
decltype(&clang_tokenize) Tokenize = nullptr;
|
||||
decltype(&clang_disposeTokens) DisposeTokens = nullptr;
|
||||
decltype(&clang_getTokenKind) GetTokenKind = nullptr;
|
||||
decltype(&clang_getTokenExtent) GetTokenExtent = nullptr;
|
||||
};
|
||||
|
||||
LibClang LoadLibClang() {
|
||||
LibClang lib;
|
||||
std::vector<std::string> tried;
|
||||
// CRAFTER_BUILD_LIBCLANG pins an exact path, mirroring the LIBCXX_DIR /
|
||||
// CRAFTER_MINGW_DIR overrides used elsewhere. It is exclusive: pointing
|
||||
// it at a broken path must fail loudly rather than quietly succeed with
|
||||
// some other libclang, or the override is useless for diagnosing which
|
||||
// library is actually in play.
|
||||
std::vector<std::string> candidates;
|
||||
if (const char* pinned = std::getenv("CRAFTER_BUILD_LIBCLANG"); pinned && *pinned) {
|
||||
candidates.emplace_back(pinned);
|
||||
} else {
|
||||
for (std::string_view name : LibClangNames) candidates.emplace_back(name);
|
||||
}
|
||||
|
||||
auto join = [](const std::vector<std::string>& parts) {
|
||||
std::string joined;
|
||||
for (const std::string& part : parts) {
|
||||
if (!joined.empty()) joined += ", ";
|
||||
joined += part;
|
||||
}
|
||||
return joined;
|
||||
};
|
||||
|
||||
for (const std::string& name : candidates) {
|
||||
lib.handle = OpenLibrary(name);
|
||||
if (lib.handle) break;
|
||||
tried.push_back(name);
|
||||
}
|
||||
if (!lib.handle) {
|
||||
lib.error = std::format("could not load libclang (tried {}); install clang, or point CRAFTER_BUILD_LIBCLANG at it", join(tried));
|
||||
return lib;
|
||||
}
|
||||
|
||||
std::vector<std::string> missing;
|
||||
auto bind = [&](auto& slot, const std::string& name) {
|
||||
slot = reinterpret_cast<std::remove_reference_t<decltype(slot)>>(LibrarySymbol(lib.handle, name));
|
||||
if (!slot) missing.push_back(name);
|
||||
};
|
||||
bind(lib.CreateIndex, "clang_createIndex");
|
||||
bind(lib.DisposeIndex, "clang_disposeIndex");
|
||||
bind(lib.ParseTranslationUnit, "clang_parseTranslationUnit");
|
||||
bind(lib.DisposeTranslationUnit, "clang_disposeTranslationUnit");
|
||||
bind(lib.GetFile, "clang_getFile");
|
||||
bind(lib.GetLocationForOffset, "clang_getLocationForOffset");
|
||||
bind(lib.GetRange, "clang_getRange");
|
||||
bind(lib.GetRangeStart, "clang_getRangeStart");
|
||||
bind(lib.GetRangeEnd, "clang_getRangeEnd");
|
||||
bind(lib.GetFileLocation, "clang_getFileLocation");
|
||||
bind(lib.Tokenize, "clang_tokenize");
|
||||
bind(lib.DisposeTokens, "clang_disposeTokens");
|
||||
bind(lib.GetTokenKind, "clang_getTokenKind");
|
||||
bind(lib.GetTokenExtent, "clang_getTokenExtent");
|
||||
if (!missing.empty()) {
|
||||
lib.handle = nullptr;
|
||||
lib.error = std::format("loaded {} but it is missing {}", candidates.front(), join(missing));
|
||||
}
|
||||
return lib;
|
||||
}
|
||||
|
||||
const LibClang& Clang() {
|
||||
static const LibClang Lib = LoadLibClang();
|
||||
return Lib;
|
||||
}
|
||||
|
||||
// The -x language for a source file, or empty when we must not lex it.
|
||||
// .cppm needs c++-module explicitly: libclang does not infer a module unit
|
||||
// from the extension and silently treats every flag as a linker input if
|
||||
// left to guess. Shaders and data files return empty — lexing GLSL as C++
|
||||
// yields plausible-looking nonsense.
|
||||
std::string_view LexLanguage(const fs::path& file) {
|
||||
std::string ext = file.extension().string();
|
||||
if (ext == ".cppm" || ext == ".ixx") return "c++-module";
|
||||
if (ext == ".cpp" || ext == ".cc" || ext == ".cxx" || ext == ".h" || ext == ".hpp" || ext == ".cu") return "c++";
|
||||
if (ext == ".c") return "c";
|
||||
return {};
|
||||
}
|
||||
|
||||
LintTokenKind MapTokenKind(CXTokenKind kind) {
|
||||
switch (kind) {
|
||||
case CXToken_Punctuation: return LintTokenKind::Punctuation;
|
||||
case CXToken_Keyword: return LintTokenKind::Keyword;
|
||||
case CXToken_Identifier: return LintTokenKind::Identifier;
|
||||
case CXToken_Literal: return LintTokenKind::Literal;
|
||||
case CXToken_Comment: return LintTokenKind::Comment;
|
||||
}
|
||||
return LintTokenKind::Punctuation;
|
||||
}
|
||||
|
||||
// Lex `content` as if it were `file`, returning tokens in source order.
|
||||
//
|
||||
// The buffer is handed over as an unsaved file, so a transform's in-memory
|
||||
// edits are what get lexed — never the stale bytes on disk. The parse is
|
||||
// expected to fail (a module unit's `import std;` cannot resolve without
|
||||
// PCMs, and we deliberately do not supply the build's flags here); that
|
||||
// does not matter, because clang_tokenize re-lexes the buffer and lexing
|
||||
// has no semantic prerequisites. SingleFileParse keeps it from chasing
|
||||
// #includes it does not need.
|
||||
std::vector<LintToken> LexFile(const fs::path& file, const std::string& content) {
|
||||
std::string_view language = LexLanguage(file);
|
||||
if (language.empty()) return {};
|
||||
const LibClang& lc = Clang();
|
||||
if (!lc.handle) return {};
|
||||
|
||||
std::string path = file.string();
|
||||
std::string languageArg = std::format("-x{}", language);
|
||||
std::string standardArg = language == "c" ? "-std=c23" : "-std=c++26";
|
||||
// clang's argv is char* by contract; keep the raw pointers confined to
|
||||
// this call rather than letting them into any signature of ours.
|
||||
std::array<const char*, 4> args{languageArg.c_str(), standardArg.c_str(), "-ferror-limit=0", "-w"};
|
||||
|
||||
CXUnsavedFile unsaved{};
|
||||
unsaved.Filename = path.c_str();
|
||||
unsaved.Contents = content.data();
|
||||
unsaved.Length = static_cast<std::uint32_t>(content.size());
|
||||
|
||||
CXIndex index = lc.CreateIndex(0, 0);
|
||||
if (!index) return {};
|
||||
CXTranslationUnit tu = lc.ParseTranslationUnit(index, path.c_str(), args.data(), static_cast<std::int32_t>(args.size()), &unsaved, 1, CXTranslationUnit_SingleFileParse | CXTranslationUnit_SkipFunctionBodies | CXTranslationUnit_KeepGoing);
|
||||
if (!tu) {
|
||||
lc.DisposeIndex(index);
|
||||
return {};
|
||||
}
|
||||
|
||||
std::vector<LintToken> tokens;
|
||||
if (CXFile cxFile = lc.GetFile(tu, path.c_str())) {
|
||||
CXSourceRange whole = lc.GetRange(lc.GetLocationForOffset(tu, cxFile, 0), lc.GetLocationForOffset(tu, cxFile, static_cast<std::uint32_t>(content.size())));
|
||||
CXToken* raw = nullptr;
|
||||
std::uint32_t count = 0;
|
||||
lc.Tokenize(tu, whole, &raw, &count);
|
||||
tokens.reserve(count);
|
||||
for (std::uint32_t i = 0; i < count; ++i) {
|
||||
CXSourceRange extent = lc.GetTokenExtent(tu, raw[i]);
|
||||
std::uint32_t line = 0;
|
||||
std::uint32_t column = 0;
|
||||
std::uint32_t begin = 0;
|
||||
std::uint32_t end = 0;
|
||||
lc.GetFileLocation(lc.GetRangeStart(extent), nullptr, &line, &column, &begin);
|
||||
lc.GetFileLocation(lc.GetRangeEnd(extent), nullptr, nullptr, nullptr, &end);
|
||||
if (end < begin || begin > content.size()) continue;
|
||||
tokens.push_back({MapTokenKind(lc.GetTokenKind(raw[i])), begin, std::min<std::size_t>(end - begin, content.size() - begin), line, column});
|
||||
}
|
||||
if (raw) lc.DisposeTokens(tu, raw, count);
|
||||
}
|
||||
lc.DisposeTranslationUnit(tu);
|
||||
lc.DisposeIndex(index);
|
||||
return tokens;
|
||||
}
|
||||
|
||||
// Blank comments and the bodies of string/character literals to spaces,
|
||||
// copying '\n' through so byte offsets and line numbers in the result
|
||||
// match the original text exactly.
|
||||
//
|
||||
// Derived from the token stream rather than scanned character by
|
||||
// character, which is what makes raw strings, escapes, encoding prefixes
|
||||
// and a literal like '"' come out right. Editing a copy of the buffer in
|
||||
// place — rather than appending to a fresh string — makes the
|
||||
// length-preserving property structural instead of something every branch
|
||||
// has to remember.
|
||||
std::string StripLiterals(const std::string& content, std::span<const LintToken> tokens) {
|
||||
std::string out = content;
|
||||
auto blank = [&out](std::size_t from, std::size_t to) {
|
||||
for (std::size_t i = from; i < to && i < out.size(); ++i) {
|
||||
if (out[i] != '\n') out[i] = ' ';
|
||||
}
|
||||
};
|
||||
for (const LintToken& token : tokens) {
|
||||
std::size_t begin = token.offset;
|
||||
std::size_t end = token.offset + token.length;
|
||||
if (token.kind == LintTokenKind::Comment) {
|
||||
blank(begin, end);
|
||||
continue;
|
||||
}
|
||||
if (token.kind != LintTokenKind::Literal || token.length < 2) continue;
|
||||
std::string_view text(content.data() + begin, token.length);
|
||||
// Numeric literals are code and stay; only string and character
|
||||
// literals have a body to hide. A digit separator makes 1'000 look
|
||||
// quote-ish, so require everything before the quote to be an
|
||||
// encoding prefix (L, u, U, u8, R and their combinations).
|
||||
std::size_t quote = text.find_first_of("\"'");
|
||||
if (quote == std::string_view::npos) continue;
|
||||
std::string_view prefix = text.substr(0, quote);
|
||||
if (!std::ranges::all_of(prefix, [](char c) { return c == 'L' || c == 'u' || c == 'U' || c == '8' || c == 'R'; })) continue;
|
||||
// Keep the opening quote and the closing one, blank everything
|
||||
// between. For a raw string that also blanks the R"delim( and
|
||||
// )delim" scaffolding, leaving exactly two quotes — which is what
|
||||
// rules counting quotes to find a literal's extent rely on.
|
||||
blank(begin + quote + 1, end - 1);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
|
@ -159,11 +335,50 @@ std::string_view LintContext::Line(std::size_t n) const {
|
|||
|
||||
const std::string& LintContext::CommentStripped() {
|
||||
if (!commentStrippedCache) {
|
||||
commentStrippedCache = StripComments(content);
|
||||
commentStrippedCache = StripLiterals(content, Tokens());
|
||||
}
|
||||
return *commentStrippedCache;
|
||||
}
|
||||
|
||||
std::span<const LintToken> LintContext::Tokens() {
|
||||
if (!tokenCache) tokenCache = LexFile(file, content);
|
||||
return *tokenCache;
|
||||
}
|
||||
|
||||
std::string_view LintContext::TokenText(const LintToken& token) const {
|
||||
if (token.offset >= content.size()) return {};
|
||||
return std::string_view(content).substr(token.offset, token.length);
|
||||
}
|
||||
|
||||
std::span<const LintToken> LintContext::TokensOnLine(std::size_t line) {
|
||||
// Tokens come back in source order, so one line's tokens are a contiguous
|
||||
// run and can be bracketed by binary search.
|
||||
std::span<const LintToken> all = Tokens();
|
||||
auto begin = std::ranges::lower_bound(all, line, {}, &LintToken::line);
|
||||
auto end = std::ranges::upper_bound(all, line, {}, &LintToken::line);
|
||||
return all.subspan(static_cast<std::size_t>(begin - all.begin()), static_cast<std::size_t>(end - begin));
|
||||
}
|
||||
|
||||
bool LintContext::LineHasComment(std::size_t line) {
|
||||
std::span<const LintToken> onLine = TokensOnLine(line);
|
||||
return std::ranges::any_of(onLine, [](const LintToken& t) { return t.kind == LintTokenKind::Comment; });
|
||||
}
|
||||
|
||||
bool LintContext::LineHasMultiLineToken(std::size_t line) {
|
||||
if (!spannedLineCache) {
|
||||
std::vector<bool> spanned(lines.size(), false);
|
||||
for (const LintToken& token : Tokens()) {
|
||||
std::size_t crossed = static_cast<std::size_t>(std::ranges::count(TokenText(token), '\n'));
|
||||
if (crossed == 0) continue;
|
||||
for (std::size_t n = token.line; n <= token.line + crossed && n <= spanned.size(); ++n) {
|
||||
spanned[n - 1] = true;
|
||||
}
|
||||
}
|
||||
spannedLineCache = std::move(spanned);
|
||||
}
|
||||
return line >= 1 && line <= spannedLineCache->size() && (*spannedLineCache)[line - 1];
|
||||
}
|
||||
|
||||
void LintContext::Report(std::size_t line, std::string message) {
|
||||
sink->push_back({file, line, activeRule, std::move(message)});
|
||||
}
|
||||
|
|
@ -226,6 +441,8 @@ void LintContext::SetContent(std::string newContent) {
|
|||
lines = SplitLines(content);
|
||||
commentStrippedCache.reset();
|
||||
suppressionsCache.reset(); // line numbers may have shifted — re-parse
|
||||
tokenCache.reset(); // offsets refer to the old buffer — re-lex
|
||||
spannedLineCache.reset(); // derived from tokenCache
|
||||
}
|
||||
|
||||
void Configuration::AddLintRule(std::string name, std::function<void(LintContext&)> check) {
|
||||
|
|
@ -235,6 +452,15 @@ void Configuration::AddLintRule(std::string name, std::function<void(LintContext
|
|||
LintSummary Crafter::RunLint(Configuration& projectCfg, const RunLintOptions& opts) {
|
||||
LintSummary summary;
|
||||
|
||||
// libclang backs the lexer every rule reads through, so a failed load is
|
||||
// fatal rather than a downgrade: running the rules without it would report
|
||||
// against a substrate that disagrees with the one they were written for.
|
||||
if (const LibClang& lc = Clang(); !lc.handle) {
|
||||
std::println(std::cerr, "lint: {}", lc.error);
|
||||
++summary.errors;
|
||||
return summary;
|
||||
}
|
||||
|
||||
fs::path projectRoot = opts.projectFile.empty()
|
||||
? fs::absolute(projectCfg.path)
|
||||
: opts.projectFile.parent_path();
|
||||
|
|
|
|||
Loading…
Reference in a new issue