diff --git a/implementations/Crafter.Build-Clang.cpp b/implementations/Crafter.Build-Clang.cpp index 7c7f5c0..da2ddc4 100644 --- a/implementations/Crafter.Build-Clang.cpp +++ b/implementations/Crafter.Build-Clang.cpp @@ -246,11 +246,14 @@ void Configuration::GetInterfacesAndImplementations(std::span interfac } fs::path Crafter::StdPcmDir(const Configuration& config) { - // The std PCM is cached per target+march, but a wasm variant pass compiles - // it with extra codegen flags (e.g. -mrelaxed-simd) — its BMI must not be - // shared with the baseline's, or the consuming TUs see a target-feature - // mismatch. Suffix the cache dir with the variant flags when present. - std::string stdPcmKey = std::format("{}-{}", config.target, config.march); + // The std PCM is cached per target+march+toolchain: a PCM is only valid + // for the exact clang that wrote it, so a compiler bump starts a cold + // directory instead of handing the new clang a PCM it will crash on. A + // wasm variant pass compiles it with extra codegen flags (e.g. + // -mrelaxed-simd) — its BMI must not be shared with the baseline's, or + // the consuming TUs see a target-feature mismatch. Suffix the cache dir + // with the variant flags when present. + std::string stdPcmKey = std::format("{}-{}-{}", config.target, config.march, GetToolchain().Key()); for (const std::string& f : config.wasmVariantFlags) { stdPcmKey += "+"; for (char c : f) stdPcmKey += (c == '/' || c == '\\') ? '_' : c; diff --git a/implementations/Crafter.Build-Platform.cpp b/implementations/Crafter.Build-Platform.cpp index 4181bb6..a6f8793 100644 --- a/implementations/Crafter.Build-Platform.cpp +++ b/implementations/Crafter.Build-Platform.cpp @@ -90,15 +90,35 @@ namespace { return cacheDir / "crafter-build-sources.stamp"; } - std::string ReadCacheStamp(const fs::path& cacheDir) { - std::ifstream in(CacheStampPath(cacheDir), std::ios::binary); + // A stamp is a one-line content hash kept next to the artifact it + // describes. Written last, after the artifact, so an interrupted build + // leaves a stamp that disagrees and the next run starts over. + std::string ReadStamp(const fs::path& file) { + std::ifstream in(file, std::ios::binary); std::string stamp; std::getline(in, stamp); return stamp; } - void WriteCacheStamp(const fs::path& cacheDir, const std::string& stamp) { - std::ofstream(CacheStampPath(cacheDir), std::ios::binary | std::ios::trunc) << stamp << '\n'; + void WriteStamp(const fs::path& file, std::string_view stamp) { + std::ofstream(file, std::ios::binary | std::ios::trunc) << stamp << '\n'; + } + + fs::path StampPath(const fs::path& artifact) { + fs::path stamp = artifact; + stamp += ".stamp"; + return stamp; + } + + std::string HashStamp(std::string_view content) { + return std::format("{:016x}", std::hash{}(content)); + } + + std::string FileBytes(const fs::path& file) { + std::ifstream in(file, std::ios::binary); + std::ostringstream buffer; + buffer << in.rdbuf(); + return buffer.str(); } } @@ -495,12 +515,17 @@ namespace { return out; } - // project.so predates its source, the crafter-build that will load it, or - // a pre-dependency artifact — any of them means recompiling. - bool ProjectFileStale(const fs::path& artifact, const fs::path& project, const fs::path& hostExe, const ProjectFileDependencies& deps) { + // project.so is stale when its stamp (crafter-build version, toolchain, + // module sources and compile command) disagrees, or when it predates its + // source or a pre-dependency artifact. The executable's mtime is not an + // input any more: an OSTree deployment dates every installed file at the + // epoch, so a comparison against it never fired after an update and the + // old project.so was loaded into a binary it did not match. + bool ProjectFileStale(const fs::path& artifact, const fs::path& project, std::string_view stamp, const ProjectFileDependencies& deps) { if (!fs::exists(artifact)) return true; + if (ReadStamp(StampPath(artifact)) != stamp) return true; const fs::file_time_type built = fs::last_write_time(artifact); - return built < fs::last_write_time(project) || built < fs::last_write_time(hostExe) || built < deps.newestArtifact; + return built < fs::last_write_time(project) || built < deps.newestArtifact; } } @@ -611,21 +636,22 @@ std::string Crafter::BuildStdPcm(const Configuration& config, fs::path stdPcm) { fs::path stdCc = GetToolchain().mingwSysroot / "include" / "c++" / mingWversion / "bits" / "std.cc"; - if(!fs::exists(stdPcm) || fs::last_write_time(stdPcm) < fs::last_write_time(stdCc)) { - // -femulated-tls keeps PCM TLS access matching libstdc++'s emutls - // definitions; mismatch surfaces as undefined std::__once_callable. - // The std.cc → std.cppm copy uses a plain filename: held under the - // cache lock, no other builder writes this path concurrently. - fs::path stdCppm = stdPcm.parent_path() / "std.cppm"; - std::error_code copyEc; - fs::copy_file(stdCc, stdCppm, fs::copy_options::overwrite_existing, copyEc); - if (copyEc) { - return std::format("copy {} -> {}: {}", stdCc.string(), stdCppm.string(), copyEc.message()); - } - return RunBuildCommand(std::format("{} --target={} -march={} -mtune={} -femulated-tls -O3 -std=c++26 -Wno-reserved-identifier -Wno-reserved-module-identifier --precompile {} -o {}", GetBaseCommand(config), config.target, config.march, config.mtune, stdCppm.string(), stdPcm.string())); - } else { - return ""; + // -femulated-tls keeps PCM TLS access matching libstdc++'s emutls + // definitions; mismatch surfaces as undefined std::__once_callable. + // The std.cc → std.cppm copy uses a plain filename: held under the + // cache lock, no other builder writes this path concurrently. + fs::path stdCppm = stdPcm.parent_path() / "std.cppm"; + std::string command = std::format("{} --target={} -march={} -mtune={} -femulated-tls -O3 -std=c++26 -Wno-reserved-identifier -Wno-reserved-module-identifier --precompile {} -o {}", GetBaseCommand(config), config.target, config.march, config.mtune, stdCppm.string(), stdPcm.string()); + std::string stamp = HashStamp(command + '\0' + FileBytes(stdCc)); + if (fs::exists(stdPcm) && ReadStamp(StampPath(stdPcm)) == stamp) return ""; + std::error_code copyEc; + fs::copy_file(stdCc, stdCppm, fs::copy_options::overwrite_existing, copyEc); + if (copyEc) { + return std::format("copy {} -> {}: {}", stdCc.string(), stdCppm.string(), copyEc.message()); } + std::string result = RunBuildCommand(command); + if (result.empty()) WriteStamp(StampPath(stdPcm), stamp); + return result; } else { const bool isWasm = config.target.starts_with("wasm32"); // wasi-sdk drops std.cppm at /share/libc++/v1/, the rest of @@ -669,11 +695,16 @@ std::string Crafter::BuildStdPcm(const Configuration& config, fs::path stdPcm) { if (!isWasm && !config.sysroot.empty()) { archFlags += std::format(" -nostdinc++ -isystem {}/usr/include/c++/v1", config.sysroot); } - if(!fs::exists(stdPcm) || fs::last_write_time(stdPcm) < fs::last_write_time(stdCppm)) { - return RunBuildCommand(std::format("{} --target={} -std=c++26{}{} -O3 -Wno-reserved-identifier -Wno-reserved-module-identifier --precompile {} -o {}", GetBaseCommand(config), config.target, sysrootFlag, archFlags, stdCppm, stdPcm.string())); - } else { - return ""; - } + // Keyed on the command and the module source's bytes, never on + // mtimes: an installed std.cppm can be dated at the epoch (OSTree), + // and a libc++ update under the same clang changes the bytes but + // not the path. + std::string command = std::format("{} --target={} -std=c++26{}{} -O3 -Wno-reserved-identifier -Wno-reserved-module-identifier --precompile {} -o {}", GetBaseCommand(config), config.target, sysrootFlag, archFlags, stdCppm, stdPcm.string()); + std::string stamp = HashStamp(command + '\0' + FileBytes(stdCppm)); + if (fs::exists(stdPcm) && ReadStamp(StampPath(stdPcm)) == stamp) return ""; + std::string result = RunBuildCommand(command); + if (result.empty()) WriteStamp(StampPath(stdPcm), stamp); + return result; } } @@ -721,12 +752,17 @@ namespace { "Crafter.Build", }; - void EnsureCrafterBuildPcms(const fs::path& sourceDir, const fs::path& cacheDir, const Configuration& host) { + // Returns the stamp of the module sources the cache was built from; it is + // one of project.so's inputs. + std::string EnsureCrafterBuildPcms(const fs::path& sourceDir, const fs::path& cacheDir, const Configuration& host) { CacheLock lock(cacheDir); // Re-derived under the lock: another builder may have refreshed the // cache from its own sources while we waited. - std::string stamp = CrafterBuildSourceStamp(sourceDir, CrafterBuildModules); - const bool upToDate = ReadCacheStamp(cacheDir) == stamp; + // The std PCM is an input to every module PCM: its stamp is folded in + // so a rebuilt std.pcm (a libc++ update under the same clang) rebuilds + // them too, instead of leaving PCMs that import a std.pcm that is gone. + std::string stamp = HashStamp(CrafterBuildSourceStamp(sourceDir, CrafterBuildModules) + '\0' + ReadStamp(StampPath(cacheDir / "std.pcm"))); + const bool upToDate = ReadStamp(CacheStampPath(cacheDir)) == stamp; for (std::string_view name : CrafterBuildModules) { fs::path cppmPath = sourceDir / std::format("{}.cppm", name); fs::path pcmPath = cacheDir / std::format("{}.pcm", name); @@ -751,7 +787,8 @@ namespace { // Last, so an interrupted rebuild leaves the stamp disagreeing with the // sources and the next run starts over rather than trusting a half-built // set of PCMs. - if (!upToDate) WriteCacheStamp(cacheDir, stamp); + if (!upToDate) WriteStamp(CacheStampPath(cacheDir), stamp); + return stamp; } } @@ -765,8 +802,6 @@ Configuration Crafter::LoadProject(const fs::path& projectFile, std::span a // Same LoadProject wiring as ConcurrentCacheRace above. cfg.AddTest("HostCacheSourceStamp").Dependencies({ CrafterBuildLib.get() }) .LinkFlag("-Wl,--export-dynamic").LinkFlag("-ldl"); + cfg.AddTest("HostCacheToolchainKey").Dependencies({ CrafterBuildLib.get() }) + .LinkFlag("-Wl,--export-dynamic").LinkFlag("-ldl"); cfg.AddTest("ProjectDependency").Dependencies({ CrafterBuildLib.get() }) .LinkFlag("-Wl,--export-dynamic").LinkFlag("-ldl"); cfg.AddTest("ConcurrentDependencyReset").Dependencies({ CrafterBuildLib.get() }); diff --git a/tests/HostCacheToolchainKey/main.cpp b/tests/HostCacheToolchainKey/main.cpp new file mode 100644 index 0000000..58c35f8 --- /dev/null +++ b/tests/HostCacheToolchainKey/main.cpp @@ -0,0 +1,154 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts® + +#include +import std; +import Crafter.Build; +namespace fs = std::filesystem; +using namespace Crafter; + +// Freshness of the std PCM and of project.so used to be an mtime comparison +// against files that come with the install: std.cppm and the crafter-build +// executable. A Flatpak (any OSTree deployment) gives every installed file +// mtime 0, so those inputs always looked older than the cache: a clang bump +// would never rebuild std.pcm, a crafter-build update would never recompile +// project.so, and clang crashes on the stale PCM instead of saying so. +// +// Both are now keyed on content stamps (the compile command, the std module +// source, the crafter-build version, the toolchain key and the module source +// stamp) and the cache directory carries the toolchain key in its name. This +// test drives the shapes an mtime rule gets wrong: a std.pcm dated before its +// source must still be reused, a warm load must leave project.so alone, and a +// stamp that disagrees must rebuild even when the artifact is dated after +// every input. +namespace { + std::int32_t Failures = 0; + + void Check(bool cond, std::string_view msg) { + if (!cond) { + std::println(std::cerr, "FAIL: {}", msg); + ++Failures; + } + } + + fs::path FindModuleCacheDir(const fs::path& cacheRoot) { + for (const fs::directory_entry& entry : fs::directory_iterator(cacheRoot / "crafter.build")) { + if (entry.is_directory()) return entry.path(); + } + return {}; + } + + bool Load(const fs::path& projectFile, std::string_view what) { + std::array noArgs = {}; + try { + (void)LoadProject(projectFile, noArgs); + return true; + } catch (const std::exception& e) { + std::println(std::cerr, "FAIL: LoadProject ({}) threw: {}", what, e.what()); + ++Failures; + return false; + } + } + + fs::path StampOf(const fs::path& artifact) { + fs::path stamp = artifact; + stamp += ".stamp"; + return stamp; + } +} + +int main() { + fs::path installedHome = GetCrafterBuildHome(); + if (!fs::exists(installedHome / "Crafter.Build.cppm")) { + std::println(std::cerr, "SKIP: no module sources at {}", installedHome.string()); + return 77; + } + + fs::path scratch = fs::temp_directory_path() / "crafter-build-host-cache-toolchain-key"; + std::error_code ec; + fs::remove_all(scratch, ec); + fs::create_directories(scratch); + + // A private copy of the module sources and a cold private cache, as in + // HostCacheSourceStamp. + fs::path home = scratch / "share" / "crafter-build"; + fs::create_directories(home.parent_path()); + fs::copy(installedHome, home, fs::copy_options::recursive); + setenv("CRAFTER_BUILD_HOME", home.string().c_str(), 1); + fs::path cacheRoot = scratch / "cache"; + fs::create_directories(cacheRoot); + setenv("XDG_CACHE_HOME", cacheRoot.string().c_str(), 1); + + fs::path projectDir = scratch / "project"; + fs::create_directories(projectDir); + fs::path projectFile = projectDir / "project.cpp"; + std::ofstream(projectFile) << + "import std;\n" + "import Crafter.Build;\n" + "using namespace Crafter;\n" + "extern \"C\" Configuration CrafterBuildProject(std::span) {\n" + " Configuration cfg;\n" + " cfg.path = \"./\";\n" + " cfg.name = \"keyed\";\n" + " cfg.outputName = \"keyed\";\n" + " cfg.type = ConfigurationType::Executable;\n" + " return cfg;\n" + "}\n"; + + if (!Load(projectFile, "cold")) return 1; + + const fs::path cacheDir = FindModuleCacheDir(cacheRoot); + Check(!cacheDir.empty(), "a module cache directory was created"); + if (cacheDir.empty()) return 1; + Check(cacheDir.filename().string().contains(GetToolchain().Key()), std::format("the cache directory is named for the toolchain (got '{}', key '{}')", cacheDir.filename().string(), GetToolchain().Key())); + + const fs::path stdPcm = cacheDir / "std.pcm"; + const fs::path projectSo = projectDir / "build" / "project.so"; + Check(fs::exists(stdPcm) && fs::exists(StampOf(stdPcm)), "std.pcm and its stamp exist after a cold load"); + Check(fs::exists(projectSo) && fs::exists(StampOf(projectSo)), "project.so and its stamp exist after a cold load"); + if (!fs::exists(StampOf(stdPcm)) || !fs::exists(StampOf(projectSo))) return 1; + + // The OSTree shape, inverted: std.pcm dated before its source. A stamp + // rule reuses it; an mtime rule would rebuild it on every run. project.so + // keeps two mtime inputs that live next to it (project.cpp and dependency + // artifacts) but the executable is no longer one of them, so a warm load + // with nothing changed leaves it alone. + const fs::file_time_type epoch{}; + fs::last_write_time(stdPcm, epoch); + const fs::file_time_type afterCold = fs::last_write_time(projectSo); + if (!Load(projectFile, "std.pcm dated at the epoch")) return 1; + Check(fs::last_write_time(stdPcm) == epoch, "std.pcm dated before its source is still reused"); + Check(fs::last_write_time(projectSo) == afterCold, "project.so is left alone when nothing changed"); + + // The other half: artifacts dated after every input, with a stamp that + // disagrees. An mtime rule keeps them; the stamp rule rebuilds. + const fs::file_time_type future = fs::file_time_type::clock::now() + std::chrono::hours(24); + std::ofstream(StampOf(stdPcm)) << "stale"; + fs::last_write_time(stdPcm, future); + if (!Load(projectFile, "stale std.pcm stamp")) return 1; + Check(fs::last_write_time(stdPcm) != future, "a disagreeing stamp rebuilds std.pcm even when it is dated after its source"); + { + std::ifstream in(StampOf(stdPcm)); + std::string stamp; + std::getline(in, stamp); + Check(stamp != "stale" && !stamp.empty(), "the rebuilt std.pcm gets its stamp back"); + } + + std::ofstream(StampOf(projectSo)) << "stale"; + fs::last_write_time(projectSo, future); + if (!Load(projectFile, "stale project.so stamp")) return 1; + Check(fs::last_write_time(projectSo) != future, "a disagreeing stamp recompiles project.so even when it is dated after its inputs"); + + // The one mtime input that legitimately remains: the project file itself. + const fs::file_time_type beforeEdit = fs::last_write_time(projectSo); + { + std::ofstream append(projectFile, std::ios::app); + append << "// edited\n"; + } + fs::last_write_time(projectFile, fs::file_time_type::clock::now() + std::chrono::seconds(2)); + if (!Load(projectFile, "edited project.cpp")) return 1; + Check(fs::last_write_time(projectSo) != beforeEdit, "an edited project.cpp recompiles project.so"); + + fs::remove_all(scratch, ec); + return Failures == 0 ? 0 : 1; +}