diff --git a/README.md b/README.md index 400f9f1..6af758c 100644 --- a/README.md +++ b/README.md @@ -107,7 +107,7 @@ import Crafter.LintRules; /bin/crafter-build # the executable /lib/libcrafter-build.a # static archive for downstream consumers /share/crafter-build/*.cppm # module sources (rebuilt on user machine on first run) -~/.cache/crafter.build/---/ # std.pcm + Crafter.Build*.pcm, built locally per toolchain +~/.cache/crafter.build/-/ # std.pcm + Crafter.Build*.pcm, built locally ``` ## Tests diff --git a/implementations/Crafter.Build-Clang.cpp b/implementations/Crafter.Build-Clang.cpp index da2ddc4..7c7f5c0 100644 --- a/implementations/Crafter.Build-Clang.cpp +++ b/implementations/Crafter.Build-Clang.cpp @@ -246,14 +246,11 @@ void Configuration::GetInterfacesAndImplementations(std::span interfac } fs::path Crafter::StdPcmDir(const Configuration& config) { - // The std PCM is cached per target+march+toolchain: a PCM is only valid - // for the exact clang that wrote it, so a compiler bump starts a cold - // directory instead of handing the new clang a PCM it will crash on. A - // wasm variant pass compiles it with extra codegen flags (e.g. - // -mrelaxed-simd) — its BMI must not be shared with the baseline's, or - // the consuming TUs see a target-feature mismatch. Suffix the cache dir - // with the variant flags when present. - std::string stdPcmKey = std::format("{}-{}-{}", config.target, config.march, GetToolchain().Key()); + // The std PCM is cached per target+march, but a wasm variant pass compiles + // it with extra codegen flags (e.g. -mrelaxed-simd) — its BMI must not be + // shared with the baseline's, or the consuming TUs see a target-feature + // mismatch. Suffix the cache dir with the variant flags when present. + std::string stdPcmKey = std::format("{}-{}", config.target, config.march); for (const std::string& f : config.wasmVariantFlags) { stdPcmKey += "+"; for (char c : f) stdPcmKey += (c == '/' || c == '\\') ? '_' : c; diff --git a/implementations/Crafter.Build-Platform.cpp b/implementations/Crafter.Build-Platform.cpp index a6f8793..4181bb6 100644 --- a/implementations/Crafter.Build-Platform.cpp +++ b/implementations/Crafter.Build-Platform.cpp @@ -90,35 +90,15 @@ namespace { return cacheDir / "crafter-build-sources.stamp"; } - // A stamp is a one-line content hash kept next to the artifact it - // describes. Written last, after the artifact, so an interrupted build - // leaves a stamp that disagrees and the next run starts over. - std::string ReadStamp(const fs::path& file) { - std::ifstream in(file, std::ios::binary); + std::string ReadCacheStamp(const fs::path& cacheDir) { + std::ifstream in(CacheStampPath(cacheDir), std::ios::binary); std::string stamp; std::getline(in, stamp); return stamp; } - void WriteStamp(const fs::path& file, std::string_view stamp) { - std::ofstream(file, std::ios::binary | std::ios::trunc) << stamp << '\n'; - } - - fs::path StampPath(const fs::path& artifact) { - fs::path stamp = artifact; - stamp += ".stamp"; - return stamp; - } - - std::string HashStamp(std::string_view content) { - return std::format("{:016x}", std::hash{}(content)); - } - - std::string FileBytes(const fs::path& file) { - std::ifstream in(file, std::ios::binary); - std::ostringstream buffer; - buffer << in.rdbuf(); - return buffer.str(); + void WriteCacheStamp(const fs::path& cacheDir, const std::string& stamp) { + std::ofstream(CacheStampPath(cacheDir), std::ios::binary | std::ios::trunc) << stamp << '\n'; } } @@ -515,17 +495,12 @@ namespace { return out; } - // project.so is stale when its stamp (crafter-build version, toolchain, - // module sources and compile command) disagrees, or when it predates its - // source or a pre-dependency artifact. The executable's mtime is not an - // input any more: an OSTree deployment dates every installed file at the - // epoch, so a comparison against it never fired after an update and the - // old project.so was loaded into a binary it did not match. - bool ProjectFileStale(const fs::path& artifact, const fs::path& project, std::string_view stamp, const ProjectFileDependencies& deps) { + // project.so predates its source, the crafter-build that will load it, or + // a pre-dependency artifact — any of them means recompiling. + bool ProjectFileStale(const fs::path& artifact, const fs::path& project, const fs::path& hostExe, const ProjectFileDependencies& deps) { if (!fs::exists(artifact)) return true; - if (ReadStamp(StampPath(artifact)) != stamp) return true; const fs::file_time_type built = fs::last_write_time(artifact); - return built < fs::last_write_time(project) || built < deps.newestArtifact; + return built < fs::last_write_time(project) || built < fs::last_write_time(hostExe) || built < deps.newestArtifact; } } @@ -636,22 +611,21 @@ std::string Crafter::BuildStdPcm(const Configuration& config, fs::path stdPcm) { fs::path stdCc = GetToolchain().mingwSysroot / "include" / "c++" / mingWversion / "bits" / "std.cc"; - // -femulated-tls keeps PCM TLS access matching libstdc++'s emutls - // definitions; mismatch surfaces as undefined std::__once_callable. - // The std.cc → std.cppm copy uses a plain filename: held under the - // cache lock, no other builder writes this path concurrently. - fs::path stdCppm = stdPcm.parent_path() / "std.cppm"; - std::string command = std::format("{} --target={} -march={} -mtune={} -femulated-tls -O3 -std=c++26 -Wno-reserved-identifier -Wno-reserved-module-identifier --precompile {} -o {}", GetBaseCommand(config), config.target, config.march, config.mtune, stdCppm.string(), stdPcm.string()); - std::string stamp = HashStamp(command + '\0' + FileBytes(stdCc)); - if (fs::exists(stdPcm) && ReadStamp(StampPath(stdPcm)) == stamp) return ""; - std::error_code copyEc; - fs::copy_file(stdCc, stdCppm, fs::copy_options::overwrite_existing, copyEc); - if (copyEc) { - return std::format("copy {} -> {}: {}", stdCc.string(), stdCppm.string(), copyEc.message()); + if(!fs::exists(stdPcm) || fs::last_write_time(stdPcm) < fs::last_write_time(stdCc)) { + // -femulated-tls keeps PCM TLS access matching libstdc++'s emutls + // definitions; mismatch surfaces as undefined std::__once_callable. + // The std.cc → std.cppm copy uses a plain filename: held under the + // cache lock, no other builder writes this path concurrently. + fs::path stdCppm = stdPcm.parent_path() / "std.cppm"; + std::error_code copyEc; + fs::copy_file(stdCc, stdCppm, fs::copy_options::overwrite_existing, copyEc); + if (copyEc) { + return std::format("copy {} -> {}: {}", stdCc.string(), stdCppm.string(), copyEc.message()); + } + return RunBuildCommand(std::format("{} --target={} -march={} -mtune={} -femulated-tls -O3 -std=c++26 -Wno-reserved-identifier -Wno-reserved-module-identifier --precompile {} -o {}", GetBaseCommand(config), config.target, config.march, config.mtune, stdCppm.string(), stdPcm.string())); + } else { + return ""; } - std::string result = RunBuildCommand(command); - if (result.empty()) WriteStamp(StampPath(stdPcm), stamp); - return result; } else { const bool isWasm = config.target.starts_with("wasm32"); // wasi-sdk drops std.cppm at /share/libc++/v1/, the rest of @@ -695,16 +669,11 @@ std::string Crafter::BuildStdPcm(const Configuration& config, fs::path stdPcm) { if (!isWasm && !config.sysroot.empty()) { archFlags += std::format(" -nostdinc++ -isystem {}/usr/include/c++/v1", config.sysroot); } - // Keyed on the command and the module source's bytes, never on - // mtimes: an installed std.cppm can be dated at the epoch (OSTree), - // and a libc++ update under the same clang changes the bytes but - // not the path. - std::string command = std::format("{} --target={} -std=c++26{}{} -O3 -Wno-reserved-identifier -Wno-reserved-module-identifier --precompile {} -o {}", GetBaseCommand(config), config.target, sysrootFlag, archFlags, stdCppm, stdPcm.string()); - std::string stamp = HashStamp(command + '\0' + FileBytes(stdCppm)); - if (fs::exists(stdPcm) && ReadStamp(StampPath(stdPcm)) == stamp) return ""; - std::string result = RunBuildCommand(command); - if (result.empty()) WriteStamp(StampPath(stdPcm), stamp); - return result; + if(!fs::exists(stdPcm) || fs::last_write_time(stdPcm) < fs::last_write_time(stdCppm)) { + return RunBuildCommand(std::format("{} --target={} -std=c++26{}{} -O3 -Wno-reserved-identifier -Wno-reserved-module-identifier --precompile {} -o {}", GetBaseCommand(config), config.target, sysrootFlag, archFlags, stdCppm, stdPcm.string())); + } else { + return ""; + } } } @@ -752,17 +721,12 @@ namespace { "Crafter.Build", }; - // Returns the stamp of the module sources the cache was built from; it is - // one of project.so's inputs. - std::string EnsureCrafterBuildPcms(const fs::path& sourceDir, const fs::path& cacheDir, const Configuration& host) { + void EnsureCrafterBuildPcms(const fs::path& sourceDir, const fs::path& cacheDir, const Configuration& host) { CacheLock lock(cacheDir); // Re-derived under the lock: another builder may have refreshed the // cache from its own sources while we waited. - // The std PCM is an input to every module PCM: its stamp is folded in - // so a rebuilt std.pcm (a libc++ update under the same clang) rebuilds - // them too, instead of leaving PCMs that import a std.pcm that is gone. - std::string stamp = HashStamp(CrafterBuildSourceStamp(sourceDir, CrafterBuildModules) + '\0' + ReadStamp(StampPath(cacheDir / "std.pcm"))); - const bool upToDate = ReadStamp(CacheStampPath(cacheDir)) == stamp; + std::string stamp = CrafterBuildSourceStamp(sourceDir, CrafterBuildModules); + const bool upToDate = ReadCacheStamp(cacheDir) == stamp; for (std::string_view name : CrafterBuildModules) { fs::path cppmPath = sourceDir / std::format("{}.cppm", name); fs::path pcmPath = cacheDir / std::format("{}.pcm", name); @@ -787,8 +751,7 @@ namespace { // Last, so an interrupted rebuild leaves the stamp disagreeing with the // sources and the next run starts over rather than trusting a half-built // set of PCMs. - if (!upToDate) WriteStamp(CacheStampPath(cacheDir), stamp); - return stamp; + if (!upToDate) WriteCacheStamp(cacheDir, stamp); } } @@ -802,6 +765,8 @@ Configuration Crafter::LoadProject(const fs::path& projectFile, std::span a // Same LoadProject wiring as ConcurrentCacheRace above. cfg.AddTest("HostCacheSourceStamp").Dependencies({ CrafterBuildLib.get() }) .LinkFlag("-Wl,--export-dynamic").LinkFlag("-ldl"); - cfg.AddTest("HostCacheToolchainKey").Dependencies({ CrafterBuildLib.get() }) - .LinkFlag("-Wl,--export-dynamic").LinkFlag("-ldl"); cfg.AddTest("ProjectDependency").Dependencies({ CrafterBuildLib.get() }) .LinkFlag("-Wl,--export-dynamic").LinkFlag("-ldl"); cfg.AddTest("ConcurrentDependencyReset").Dependencies({ CrafterBuildLib.get() }); diff --git a/tests/HostCacheToolchainKey/main.cpp b/tests/HostCacheToolchainKey/main.cpp deleted file mode 100644 index 58c35f8..0000000 --- a/tests/HostCacheToolchainKey/main.cpp +++ /dev/null @@ -1,154 +0,0 @@ -// SPDX-License-Identifier: LGPL-3.0-only -// SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts® - -#include -import std; -import Crafter.Build; -namespace fs = std::filesystem; -using namespace Crafter; - -// Freshness of the std PCM and of project.so used to be an mtime comparison -// against files that come with the install: std.cppm and the crafter-build -// executable. A Flatpak (any OSTree deployment) gives every installed file -// mtime 0, so those inputs always looked older than the cache: a clang bump -// would never rebuild std.pcm, a crafter-build update would never recompile -// project.so, and clang crashes on the stale PCM instead of saying so. -// -// Both are now keyed on content stamps (the compile command, the std module -// source, the crafter-build version, the toolchain key and the module source -// stamp) and the cache directory carries the toolchain key in its name. This -// test drives the shapes an mtime rule gets wrong: a std.pcm dated before its -// source must still be reused, a warm load must leave project.so alone, and a -// stamp that disagrees must rebuild even when the artifact is dated after -// every input. -namespace { - std::int32_t Failures = 0; - - void Check(bool cond, std::string_view msg) { - if (!cond) { - std::println(std::cerr, "FAIL: {}", msg); - ++Failures; - } - } - - fs::path FindModuleCacheDir(const fs::path& cacheRoot) { - for (const fs::directory_entry& entry : fs::directory_iterator(cacheRoot / "crafter.build")) { - if (entry.is_directory()) return entry.path(); - } - return {}; - } - - bool Load(const fs::path& projectFile, std::string_view what) { - std::array noArgs = {}; - try { - (void)LoadProject(projectFile, noArgs); - return true; - } catch (const std::exception& e) { - std::println(std::cerr, "FAIL: LoadProject ({}) threw: {}", what, e.what()); - ++Failures; - return false; - } - } - - fs::path StampOf(const fs::path& artifact) { - fs::path stamp = artifact; - stamp += ".stamp"; - return stamp; - } -} - -int main() { - fs::path installedHome = GetCrafterBuildHome(); - if (!fs::exists(installedHome / "Crafter.Build.cppm")) { - std::println(std::cerr, "SKIP: no module sources at {}", installedHome.string()); - return 77; - } - - fs::path scratch = fs::temp_directory_path() / "crafter-build-host-cache-toolchain-key"; - std::error_code ec; - fs::remove_all(scratch, ec); - fs::create_directories(scratch); - - // A private copy of the module sources and a cold private cache, as in - // HostCacheSourceStamp. - fs::path home = scratch / "share" / "crafter-build"; - fs::create_directories(home.parent_path()); - fs::copy(installedHome, home, fs::copy_options::recursive); - setenv("CRAFTER_BUILD_HOME", home.string().c_str(), 1); - fs::path cacheRoot = scratch / "cache"; - fs::create_directories(cacheRoot); - setenv("XDG_CACHE_HOME", cacheRoot.string().c_str(), 1); - - fs::path projectDir = scratch / "project"; - fs::create_directories(projectDir); - fs::path projectFile = projectDir / "project.cpp"; - std::ofstream(projectFile) << - "import std;\n" - "import Crafter.Build;\n" - "using namespace Crafter;\n" - "extern \"C\" Configuration CrafterBuildProject(std::span) {\n" - " Configuration cfg;\n" - " cfg.path = \"./\";\n" - " cfg.name = \"keyed\";\n" - " cfg.outputName = \"keyed\";\n" - " cfg.type = ConfigurationType::Executable;\n" - " return cfg;\n" - "}\n"; - - if (!Load(projectFile, "cold")) return 1; - - const fs::path cacheDir = FindModuleCacheDir(cacheRoot); - Check(!cacheDir.empty(), "a module cache directory was created"); - if (cacheDir.empty()) return 1; - Check(cacheDir.filename().string().contains(GetToolchain().Key()), std::format("the cache directory is named for the toolchain (got '{}', key '{}')", cacheDir.filename().string(), GetToolchain().Key())); - - const fs::path stdPcm = cacheDir / "std.pcm"; - const fs::path projectSo = projectDir / "build" / "project.so"; - Check(fs::exists(stdPcm) && fs::exists(StampOf(stdPcm)), "std.pcm and its stamp exist after a cold load"); - Check(fs::exists(projectSo) && fs::exists(StampOf(projectSo)), "project.so and its stamp exist after a cold load"); - if (!fs::exists(StampOf(stdPcm)) || !fs::exists(StampOf(projectSo))) return 1; - - // The OSTree shape, inverted: std.pcm dated before its source. A stamp - // rule reuses it; an mtime rule would rebuild it on every run. project.so - // keeps two mtime inputs that live next to it (project.cpp and dependency - // artifacts) but the executable is no longer one of them, so a warm load - // with nothing changed leaves it alone. - const fs::file_time_type epoch{}; - fs::last_write_time(stdPcm, epoch); - const fs::file_time_type afterCold = fs::last_write_time(projectSo); - if (!Load(projectFile, "std.pcm dated at the epoch")) return 1; - Check(fs::last_write_time(stdPcm) == epoch, "std.pcm dated before its source is still reused"); - Check(fs::last_write_time(projectSo) == afterCold, "project.so is left alone when nothing changed"); - - // The other half: artifacts dated after every input, with a stamp that - // disagrees. An mtime rule keeps them; the stamp rule rebuilds. - const fs::file_time_type future = fs::file_time_type::clock::now() + std::chrono::hours(24); - std::ofstream(StampOf(stdPcm)) << "stale"; - fs::last_write_time(stdPcm, future); - if (!Load(projectFile, "stale std.pcm stamp")) return 1; - Check(fs::last_write_time(stdPcm) != future, "a disagreeing stamp rebuilds std.pcm even when it is dated after its source"); - { - std::ifstream in(StampOf(stdPcm)); - std::string stamp; - std::getline(in, stamp); - Check(stamp != "stale" && !stamp.empty(), "the rebuilt std.pcm gets its stamp back"); - } - - std::ofstream(StampOf(projectSo)) << "stale"; - fs::last_write_time(projectSo, future); - if (!Load(projectFile, "stale project.so stamp")) return 1; - Check(fs::last_write_time(projectSo) != future, "a disagreeing stamp recompiles project.so even when it is dated after its inputs"); - - // The one mtime input that legitimately remains: the project file itself. - const fs::file_time_type beforeEdit = fs::last_write_time(projectSo); - { - std::ofstream append(projectFile, std::ios::app); - append << "// edited\n"; - } - fs::last_write_time(projectFile, fs::file_time_type::clock::now() + std::chrono::seconds(2)); - if (!Load(projectFile, "edited project.cpp")) return 1; - Check(fs::last_write_time(projectSo) != beforeEdit, "an edited project.cpp recompiles project.so"); - - fs::remove_all(scratch, ec); - return Failures == 0 ? 0 : 1; -}