Catcrafts/Crafter.Build
Watch
2
0
Fork
You've already forked Crafter.Build
 0
Code Issues Pull requests Projects Releases 1 Packages Wiki Activity Actions

feat(incremental): rebuild when an #included header changes

Browse source 
The staleness check compared an artifact against its own source and its
module imports. Headers were in neither set: the scanner reads `import`
lines, and a .cppm or .cpp keeps its mtime when a header it includes is
edited — so the build reported nothing to do and left objects compiled
against the previous contents. Same silent mixed-layout binary as issue
27, reached through #include instead of import.

Ask the compiler what it actually opened. Every C++ and C compile now
passes -MD -MF <artifact>.d, and Check reads that depfile back through
NewestPrerequisite, comparing every prerequisite's mtime against the
artifact. A missing depfile (an object from a crafter-build that wrote
none) or a prerequisite that no longer exists reads as "rebuild": neither
is evidence of freshness.

The parser unescapes make syntax rather than splitting on whitespace,
since clang wraps depfiles onto continuation lines and escapes spaces in
filenames.
This commit is contained in:
catbot 2026-07-31 11:11:38 +00:00
commit 1d0d8e1e28
6 changed files with 111 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

11
implementations/Crafter.Build-Clang.cpp
View file

@ -780,12 +780,15 @@ BuildResult Crafter::Build(Configuration& config, std::unordered_map<fs::path, s
files += std::format(" {}_source.o ", (buildDir / cFile.filename()).string());
const std::string objPath = std::format("{}_source.o", (buildDir / cFile.filename()).string());
const std::string srcPath = std::format("{}.c", cFile.string());
if (!fs::exists(objPath) || (fs::exists(srcPath) && fs::last_write_time(srcPath) > fs::last_write_time(objPath))) {
// C sources carry their dependencies entirely in #includes, so the
// depfile the compile below writes is the whole record of what this
// object was built from beyond the .c itself.
if (!fs::exists(objPath) || (fs::exists(srcPath) && fs::last_write_time(srcPath) > fs::last_write_time(objPath)) || NewestPrerequisite(std::format("{}.d", objPath)) > fs::last_write_time(objPath)) {
threads.emplace_back([&cFile, &buildDir, &buildError, &buildCancelled, &config, &includeFlags, &defineFlags, &userFlags, &cArchFlags, &ltoCompileFlags]() {
Progress::Task task(std::format("Compiling {}.c", cFile.filename().string()));
if (buildCancelled.load(std::memory_order_relaxed)) return;
std::string result = RunCommand(std::format("clang {}.c --target={}{} -O3{} -c{}{}{} -o {}_source.o", cFile.string(), config.target, cArchFlags, ltoCompileFlags, includeFlags, defineFlags, userFlags, (buildDir / cFile.filename()).string()));
std::string result = RunCommand(std::format("clang {0}.c --target={1}{2} -O3{3} -c{4}{5}{6} -MD -MF {7}_source.o.d -o {7}_source.o", cFile.string(), config.target, cArchFlags, ltoCompileFlags, includeFlags, defineFlags, userFlags, (buildDir / cFile.filename()).string()));
if (result.empty()) return;
bool expected = false;
@ -800,6 +803,10 @@ BuildResult Crafter::Build(Configuration& config, std::unordered_map<fs::path, s
files += std::format(" {}_source.o ", (buildDir / cFile.filename()).string());
const std::string objPath = std::format("{}_source.o", (buildDir / cFile.filename()).string());
const std::string srcPath = std::format("{}.cu", cFile.string());
// Still .cu-mtime only: the header tracking the C and C++ paths get
// comes from asking the compiler for a depfile, and nvcc's -MD/-MF are
// untested here (no CUDA toolkit in CI), so a .cu that only sees a
// changed header does not rebuild yet.
if (!fs::exists(objPath) || (fs::exists(srcPath) && fs::last_write_time(srcPath) > fs::last_write_time(objPath))) {
threads.emplace_back([&cFile, &buildDir, &buildError, &buildCancelled]() {
Progress::Task task(std::format("Compiling {}.cu", cFile.filename().string()));

8
implementations/Crafter.Build-Implementation.cpp
View file

@ -15,7 +15,10 @@ namespace Crafter {
bool Implementation::Check(const fs::path& buildDir, const fs::path& pcmDir, fs::file_time_type sourceFloor) const {
std::string objPath = std::format("{}_impl.o", (buildDir/path.filename()).string());
std::string cppPath = std::format("{}.cpp", path.string());
if(!fs::exists(objPath) || std::max(fs::last_write_time(cppPath), sourceFloor) >= fs::last_write_time(objPath)) {
// The depfile clang wrote beside the object lists the headers this TU
// included; nothing else in the configuration knows about them, since
// the source scanner only reads `import` lines.
if(!fs::exists(objPath) || std::max({fs::last_write_time(cppPath), sourceFloor, NewestPrerequisite(std::format("{}.d", objPath))}) >= fs::last_write_time(objPath)) {
return true;
}
fs::file_time_type objTime = fs::last_write_time(objPath);
@ -48,7 +51,8 @@ namespace Crafter {
return;
}
std::string result = RunCommand(std::format("{} {}.cpp -c -o {}_impl.o", clang, path.string(), (buildDir/path.filename()).string()));
// -MD leaves <name>_impl.o.d beside the object for Check to read.
std::string result = RunCommand(std::format("{0} {1}.cpp -c -MD -MF {2}_impl.o.d -o {2}_impl.o", clang, path.string(), (buildDir/path.filename()).string()));
bool expected = false;
if(!result.empty() && buildCancelled.compare_exchange_strong(expected, true)) {

20
implementations/Crafter.Build-Interface.cpp
View file

@ -15,7 +15,13 @@ namespace Crafter {
checked = true;
std::string pcmPath = std::format("{}.pcm", (pcmDir/path.filename()).generic_string());
std::string cppmPath = std::format("{}.cppm", path.generic_string());
if(fs::exists(pcmPath) && std::max(fs::last_write_time(cppmPath), sourceFloor) < fs::last_write_time(pcmPath)) {
// NewestPrerequisite covers the headers this partition #includes —
// they are inputs to the BMI just as much as the .cppm is, and the
// .cppm's own mtime says nothing about them. It only ever bounds
// *this* artifact, so it goes in the comparison below and not into
// the `sourceFloor` handed to the recursive Checks: each of those
// reads its own depfile.
if(fs::exists(pcmPath) && std::max({fs::last_write_time(cppmPath), sourceFloor, NewestPrerequisite(std::format("{}.d", pcmPath))}) < fs::last_write_time(pcmPath)) {
fs::file_time_type pcmTime = fs::last_write_time(pcmPath);
for(ModulePartition* dependency : partitionDependencies) {
if(dependency->Check(pcmDir, sourceFloor)) {
@ -71,7 +77,9 @@ namespace Crafter {
return;
}
std::string result = RunCommand(std::format("{} {}.cppm --precompile -o {}.pcm", clang, path.string(), (pcmDir/path.filename()).string()));
// -MD records every header the preamble pulled in, next to the BMI as
// <name>.pcm.d, so the next Check can see an edit to one of them.
std::string result = RunCommand(std::format("{0} {1}.cppm --precompile -MD -MF {2}.pcm.d -o {2}.pcm", clang, path.string(), (pcmDir/path.filename()).string()));
if (!result.empty()) {
bool expected = false;
@ -103,7 +111,9 @@ namespace Crafter {
checked = true;
std::string pcmPath = std::format("{}.pcm", (pcmDir/path.filename()).generic_string());
std::string cppmPath = std::format("{}.cppm", path.generic_string());
if(fs::exists(pcmPath) && std::max(fs::last_write_time(cppmPath), sourceFloor) < fs::last_write_time(pcmPath)) {
// See ModulePartition::Check — the depfile clang wrote beside the BMI
// is the only record of which headers this interface unit included.
if(fs::exists(pcmPath) && std::max({fs::last_write_time(cppmPath), sourceFloor, NewestPrerequisite(std::format("{}.d", pcmPath))}) < fs::last_write_time(pcmPath)) {
fs::file_time_type pcmTime = fs::last_write_time(pcmPath);
// Every partition gets Check()ed even once one is known stale:
// Compile() drives partition rebuilds off their own
@ -191,7 +201,9 @@ namespace Crafter {
return;
}
std::string result = RunCommand(std::format("{} {}.cppm --precompile -o {}.pcm", clang, path.string(), (pcmDir/path.filename()).string()));
// -MD records every header the preamble pulled in, next to the BMI as
// <name>.pcm.d, so the next Check can see an edit to one of them.
std::string result = RunCommand(std::format("{0} {1}.cppm --precompile -MD -MF {2}.pcm.d -o {2}.pcm", clang, path.string(), (pcmDir/path.filename()).string()));
if (!result.empty()) {
bool expected = false;

65
implementations/Crafter.Build-Platform.cpp
View file

@ -164,6 +164,71 @@ fs::path Crafter::GetCrafterBuildHome() {
throw std::runtime_error(msg);
}
fs::file_time_type Crafter::NewestPrerequisite(const fs::path& depFile) {
std::ifstream in(depFile, std::ios::binary);
if (!in) return fs::file_time_type::max();
std::ostringstream buffer;
buffer << in.rdbuf();
const std::string text = std::move(buffer).str();
// Make-rule tokenizing, as clang writes it: `target: prereq prereq`, with a
// trailing '\' continuing the rule onto the next line (clang wraps at ~80
// columns, so any real depfile has several), '\ ' and '\#' escaping a
// character into a filename, and '$$' standing for a literal '$'. A
// backslash that is none of those is a Windows path separator and stays.
std::vector<std::string> tokens;
std::string token;
auto flush = [&tokens, &token]() {
if (!token.empty()) {
tokens.push_back(token);
token.clear();
}
};
for (std::size_t i = 0; i < text.size(); ++i) {
const char c = text[i];
const char next = i + 1 < text.size() ? text[i + 1] : '\0';
if (c == '\\' && (next == '\n' || next == '\r')) {
flush();
++i;
} else if (c == '\\' && (next == ' ' || next == '\t' || next == '#')) {
token += next;
++i;
} else if (c == '$' && next == '$') {
token += '$';
++i;
} else if (c == ' ' || c == '\t' || c == '\n' || c == '\r') {
flush();
} else {
token += c;
}
}
flush();
// A token ending in ':' names a rule target, not an input. The first one
// closes the target list (and a Windows `C:\build\x.o:` still ends in ':',
// where splitting on the first colon would not); a later one opens another
// rule, as `-MP` phony targets do, whose name is not a file to stat.
bool sawTarget = false;
fs::file_time_type newest = fs::file_time_type::min();
for (const std::string& prerequisite : tokens) {
if (prerequisite.ends_with(':')) {
sawTarget = true;
continue;
}
if (!sawTarget) continue;
std::error_code ec;
const fs::file_time_type stamp = fs::last_write_time(prerequisite, ec);
// A prerequisite that has since been deleted makes the recorded set
// unusable as evidence — the source that included it must have changed
// too, and rebuilding is what writes a depfile that no longer names it.
if (ec) return fs::file_time_type::max();
if (stamp > newest) newest = stamp;
}
// No rule at all: a truncated or empty depfile proves nothing.
if (!sawTarget) return fs::file_time_type::max();
return newest;
}
bool Crafter::MatchGlob(std::string_view glob, std::string_view name) {
std::size_t gi = 0;
std::size_t ni = 0;