diff --git a/README.md b/README.md index f5347b2..fd508f9 100644 --- a/README.md +++ b/README.md @@ -121,12 +121,15 @@ Per-import precise tracking for both within-project and cross-project module dep - Touch `lib/Hello.cppm` → only consumers of `Hello` rebuild. - Touch `lib/Other.cppm` → only consumers of `Other` rebuild. +- Touch a header → only the units that `#include` it (directly or transitively) rebuild. - External CMake dep produces fresh `.a` files → whole project rebuilds (deliberately coarse — cmake-dep changes are rare). Diamond deps (`A → {B, C}; B → X; C → X`) build `X` exactly once via a `std::shared_future` cache. Tracking is derived from each translation unit's `import` statements, which are scanned when the sources are declared. Every kind of unit is scanned — primary module interfaces, partitions and implementation units alike — and an interface that imports a sibling module in the same `Configuration` also gets its compile ordered behind it. `cfg.dependencies` is often assigned *afterwards* — `AddTest` works that way — so `Build()` re-resolves any import that matched nothing at scan time before it compares mtimes. Without all of that, a consumer of a dependency's module carried no edge to it at all: adding a data member to that dependency's interface rebuilt the library, relinked the consumer, and left the consumer's object compiled against the old class layout. Nothing fails to link when a member is added, so the result was a working build and a crash later. +`#include`s are tracked separately, because scanning can't see them: a header pulls in more headers, and which ones are open depends on the preprocessor state at the point of inclusion. So every compile is asked to report what it actually opened — `clang -MD -MF .d`, written next to the object or BMI it belongs to — and the next staleness check reads that file back and compares every prerequisite's mtime against the artifact. C sources are covered the same way, and there they are the *only* dependency record. An artifact whose depfile is missing, or that names a prerequisite which no longer exists, is rebuilt once: neither state is evidence of freshness. Before this, a header edit changed no `.cppm` or `.cpp` mtime at all, so a build reported nothing to do and left objects compiled against the old contents — the same mixed-layout failure as above, reached through `#include` instead of `import`. + Everything that changes what gets built belongs in the variant id, since it names the `bin/` and `build/` directory. That includes project args crafter-build itself doesn't interpret: `crafter-build` and `crafter-build -- --no-webgpu` get separate directories rather than interleaving their outputs in one. The cached host PCMs under `/crafter.build/-/` are shared by every crafter-build on the machine, so they are invalidated by a hash of the module sources rather than by mtime — an mtime can't tell a newer PCM from one built by a different install. `crafter-build clean` removes the project's `bin/` and `build/` trees. It doesn't load `project.cpp`, so it still works when the project no longer compiles. diff --git a/implementations/Crafter.Build-Clang.cpp b/implementations/Crafter.Build-Clang.cpp index 4e7a8f9..c59df50 100644 --- a/implementations/Crafter.Build-Clang.cpp +++ b/implementations/Crafter.Build-Clang.cpp @@ -780,12 +780,15 @@ BuildResult Crafter::Build(Configuration& config, std::unordered_map fs::last_write_time(objPath))) { + // C sources carry their dependencies entirely in #includes, so the + // depfile the compile below writes is the whole record of what this + // object was built from beyond the .c itself. + if (!fs::exists(objPath) || (fs::exists(srcPath) && fs::last_write_time(srcPath) > fs::last_write_time(objPath)) || NewestPrerequisite(std::format("{}.d", objPath)) > fs::last_write_time(objPath)) { threads.emplace_back([&cFile, &buildDir, &buildError, &buildCancelled, &config, &includeFlags, &defineFlags, &userFlags, &cArchFlags, <oCompileFlags]() { Progress::Task task(std::format("Compiling {}.c", cFile.filename().string())); if (buildCancelled.load(std::memory_order_relaxed)) return; - std::string result = RunCommand(std::format("clang {}.c --target={}{} -O3{} -c{}{}{} -o {}_source.o", cFile.string(), config.target, cArchFlags, ltoCompileFlags, includeFlags, defineFlags, userFlags, (buildDir / cFile.filename()).string())); + std::string result = RunCommand(std::format("clang {0}.c --target={1}{2} -O3{3} -c{4}{5}{6} -MD -MF {7}_source.o.d -o {7}_source.o", cFile.string(), config.target, cArchFlags, ltoCompileFlags, includeFlags, defineFlags, userFlags, (buildDir / cFile.filename()).string())); if (result.empty()) return; bool expected = false; @@ -800,6 +803,10 @@ BuildResult Crafter::Build(Configuration& config, std::unordered_map fs::last_write_time(objPath))) { threads.emplace_back([&cFile, &buildDir, &buildError, &buildCancelled]() { Progress::Task task(std::format("Compiling {}.cu", cFile.filename().string())); diff --git a/implementations/Crafter.Build-Implementation.cpp b/implementations/Crafter.Build-Implementation.cpp index 579c09d..a0ba2b1 100644 --- a/implementations/Crafter.Build-Implementation.cpp +++ b/implementations/Crafter.Build-Implementation.cpp @@ -15,7 +15,10 @@ namespace Crafter { bool Implementation::Check(const fs::path& buildDir, const fs::path& pcmDir, fs::file_time_type sourceFloor) const { std::string objPath = std::format("{}_impl.o", (buildDir/path.filename()).string()); std::string cppPath = std::format("{}.cpp", path.string()); - if(!fs::exists(objPath) || std::max(fs::last_write_time(cppPath), sourceFloor) >= fs::last_write_time(objPath)) { + // The depfile clang wrote beside the object lists the headers this TU + // included; nothing else in the configuration knows about them, since + // the source scanner only reads `import` lines. + if(!fs::exists(objPath) || std::max({fs::last_write_time(cppPath), sourceFloor, NewestPrerequisite(std::format("{}.d", objPath))}) >= fs::last_write_time(objPath)) { return true; } fs::file_time_type objTime = fs::last_write_time(objPath); @@ -48,7 +51,8 @@ namespace Crafter { return; } - std::string result = RunCommand(std::format("{} {}.cpp -c -o {}_impl.o", clang, path.string(), (buildDir/path.filename()).string())); + // -MD leaves _impl.o.d beside the object for Check to read. + std::string result = RunCommand(std::format("{0} {1}.cpp -c -MD -MF {2}_impl.o.d -o {2}_impl.o", clang, path.string(), (buildDir/path.filename()).string())); bool expected = false; if(!result.empty() && buildCancelled.compare_exchange_strong(expected, true)) { diff --git a/implementations/Crafter.Build-Interface.cpp b/implementations/Crafter.Build-Interface.cpp index b92e9c6..526cf3b 100644 --- a/implementations/Crafter.Build-Interface.cpp +++ b/implementations/Crafter.Build-Interface.cpp @@ -15,7 +15,13 @@ namespace Crafter { checked = true; std::string pcmPath = std::format("{}.pcm", (pcmDir/path.filename()).generic_string()); std::string cppmPath = std::format("{}.cppm", path.generic_string()); - if(fs::exists(pcmPath) && std::max(fs::last_write_time(cppmPath), sourceFloor) < fs::last_write_time(pcmPath)) { + // NewestPrerequisite covers the headers this partition #includes — + // they are inputs to the BMI just as much as the .cppm is, and the + // .cppm's own mtime says nothing about them. It only ever bounds + // *this* artifact, so it goes in the comparison below and not into + // the `sourceFloor` handed to the recursive Checks: each of those + // reads its own depfile. + if(fs::exists(pcmPath) && std::max({fs::last_write_time(cppmPath), sourceFloor, NewestPrerequisite(std::format("{}.d", pcmPath))}) < fs::last_write_time(pcmPath)) { fs::file_time_type pcmTime = fs::last_write_time(pcmPath); for(ModulePartition* dependency : partitionDependencies) { if(dependency->Check(pcmDir, sourceFloor)) { @@ -71,7 +77,9 @@ namespace Crafter { return; } - std::string result = RunCommand(std::format("{} {}.cppm --precompile -o {}.pcm", clang, path.string(), (pcmDir/path.filename()).string())); + // -MD records every header the preamble pulled in, next to the BMI as + // .pcm.d, so the next Check can see an edit to one of them. + std::string result = RunCommand(std::format("{0} {1}.cppm --precompile -MD -MF {2}.pcm.d -o {2}.pcm", clang, path.string(), (pcmDir/path.filename()).string())); if (!result.empty()) { bool expected = false; @@ -103,7 +111,9 @@ namespace Crafter { checked = true; std::string pcmPath = std::format("{}.pcm", (pcmDir/path.filename()).generic_string()); std::string cppmPath = std::format("{}.cppm", path.generic_string()); - if(fs::exists(pcmPath) && std::max(fs::last_write_time(cppmPath), sourceFloor) < fs::last_write_time(pcmPath)) { + // See ModulePartition::Check — the depfile clang wrote beside the BMI + // is the only record of which headers this interface unit included. + if(fs::exists(pcmPath) && std::max({fs::last_write_time(cppmPath), sourceFloor, NewestPrerequisite(std::format("{}.d", pcmPath))}) < fs::last_write_time(pcmPath)) { fs::file_time_type pcmTime = fs::last_write_time(pcmPath); // Every partition gets Check()ed even once one is known stale: // Compile() drives partition rebuilds off their own @@ -191,7 +201,9 @@ namespace Crafter { return; } - std::string result = RunCommand(std::format("{} {}.cppm --precompile -o {}.pcm", clang, path.string(), (pcmDir/path.filename()).string())); + // -MD records every header the preamble pulled in, next to the BMI as + // .pcm.d, so the next Check can see an edit to one of them. + std::string result = RunCommand(std::format("{0} {1}.cppm --precompile -MD -MF {2}.pcm.d -o {2}.pcm", clang, path.string(), (pcmDir/path.filename()).string())); if (!result.empty()) { bool expected = false; diff --git a/implementations/Crafter.Build-Platform.cpp b/implementations/Crafter.Build-Platform.cpp index 62ee0ab..051dcae 100644 --- a/implementations/Crafter.Build-Platform.cpp +++ b/implementations/Crafter.Build-Platform.cpp @@ -164,6 +164,71 @@ fs::path Crafter::GetCrafterBuildHome() { throw std::runtime_error(msg); } +fs::file_time_type Crafter::NewestPrerequisite(const fs::path& depFile) { + std::ifstream in(depFile, std::ios::binary); + if (!in) return fs::file_time_type::max(); + std::ostringstream buffer; + buffer << in.rdbuf(); + const std::string text = std::move(buffer).str(); + + // Make-rule tokenizing, as clang writes it: `target: prereq prereq`, with a + // trailing '\' continuing the rule onto the next line (clang wraps at ~80 + // columns, so any real depfile has several), '\ ' and '\#' escaping a + // character into a filename, and '$$' standing for a literal '$'. A + // backslash that is none of those is a Windows path separator and stays. + std::vector tokens; + std::string token; + auto flush = [&tokens, &token]() { + if (!token.empty()) { + tokens.push_back(token); + token.clear(); + } + }; + for (std::size_t i = 0; i < text.size(); ++i) { + const char c = text[i]; + const char next = i + 1 < text.size() ? text[i + 1] : '\0'; + if (c == '\\' && (next == '\n' || next == '\r')) { + flush(); + ++i; + } else if (c == '\\' && (next == ' ' || next == '\t' || next == '#')) { + token += next; + ++i; + } else if (c == '$' && next == '$') { + token += '$'; + ++i; + } else if (c == ' ' || c == '\t' || c == '\n' || c == '\r') { + flush(); + } else { + token += c; + } + } + flush(); + + // A token ending in ':' names a rule target, not an input. The first one + // closes the target list (and a Windows `C:\build\x.o:` still ends in ':', + // where splitting on the first colon would not); a later one opens another + // rule, as `-MP` phony targets do, whose name is not a file to stat. + bool sawTarget = false; + fs::file_time_type newest = fs::file_time_type::min(); + for (const std::string& prerequisite : tokens) { + if (prerequisite.ends_with(':')) { + sawTarget = true; + continue; + } + if (!sawTarget) continue; + std::error_code ec; + const fs::file_time_type stamp = fs::last_write_time(prerequisite, ec); + // A prerequisite that has since been deleted makes the recorded set + // unusable as evidence — the source that included it must have changed + // too, and rebuilding is what writes a depfile that no longer names it. + if (ec) return fs::file_time_type::max(); + if (stamp > newest) newest = stamp; + } + // No rule at all: a truncated or empty depfile proves nothing. + if (!sawTarget) return fs::file_time_type::max(); + return newest; +} + bool Crafter::MatchGlob(std::string_view glob, std::string_view name) { std::size_t gi = 0; std::size_t ni = 0; diff --git a/interfaces/Crafter.Build-Platform.cppm b/interfaces/Crafter.Build-Platform.cppm index eeb6fa9..b91ea12 100644 --- a/interfaces/Crafter.Build-Platform.cppm +++ b/interfaces/Crafter.Build-Platform.cppm @@ -27,6 +27,18 @@ namespace Crafter { // module sources, wasi-runtime/, etc). Honors CRAFTER_BUILD_HOME; otherwise // derives /share/crafter-build from the running executable's path. export CRAFTER_API fs::path GetCrafterBuildHome(); + // Newest mtime among the prerequisites a compiler recorded in `depFile` + // (clang's `-MD -MF .d`). This is how a build step learns which + // headers its source pulled in: the module scanner only reads `import` + // lines, so an `#include`d file is invisible to it until the compiler + // reports what it actually opened. + // + // Returns file_time_type::max() when freshness cannot be proven — no + // depfile (nothing has compiled this artifact since depfiles were emitted, + // including every object left over from an older crafter-build) or a + // prerequisite that no longer exists. Callers compare the result against + // their artifact's mtime, so max() reads as "rebuild". + fs::file_time_type NewestPrerequisite(const fs::path& depFile); // Wildcard name matching ('*', '?') shared by the test and lint verbs. bool MatchGlob(std::string_view glob, std::string_view name); // Empty `globs` matches everything.