From d22a8603dfaaf9ccb34b4ebccfc7ee90df36d265 Mon Sep 17 00:00:00 2001 From: Jorijn van der Graaf Date: Fri, 2 Oct 2026 23:18:07 +0200 Subject: [PATCH 1/7] fix(toolchain): pass -L as well as an rpath for a toolchain outside /usr Inside the Flatpak the bundled toolchain compiled every bootstrap object, then glslang's configure failed its test link with `unable to find library -lc++`. clang searches its per-target runtime directory (lib/) and the system library paths for libc++, not /lib, so with the classic layout a toolchain outside /usr needs the search path spelled out, just as its loader needs the rpath. LibcxxLinkFlags and build.sh now emit both, the same pair an explicit CRAFTER_BUILD_LIBCXX_DIR already got. Co-Authored-By: Claude Fable 5.1 --- build.sh | 17 +++++++++-------- implementations/Crafter.Build-Toolchain.cpp | 13 ++++++++----- interfaces/Crafter.Build-Toolchain.cppm | 10 ++++++---- tests/ToolchainResolve/main.cpp | 4 ++-- 4 files changed, 25 insertions(+), 19 deletions(-) diff --git a/build.sh b/build.sh index eeff75f..2d7df47 100755 --- a/build.sh +++ b/build.sh @@ -14,12 +14,13 @@ if [ -n "${CRAFTER_BUILD_LLVM_DIR:-}" ]; then CC="$CRAFTER_BUILD_LLVM_DIR/bin/clang" AR="$CRAFTER_BUILD_LLVM_DIR/bin/llvm-ar" LLVM_CONFIG="$CRAFTER_BUILD_LLVM_DIR/bin/llvm-config" - # A prefix other than the system one keeps its libc++ off the loader's - # default path; the rpath is what lets the binary start. + # A prefix other than the system one keeps its libc++ off both the + # linker's search path (clang looks in lib/ and the system dirs, + # not /lib) and the loader's; -L and an rpath cover the two. if [ "$CRAFTER_BUILD_LLVM_DIR" != /usr ]; then - RPATH_FLAG="-Wl,-rpath,$CRAFTER_BUILD_LLVM_DIR/lib" + LIBCXX_LINK_FLAGS="-L$CRAFTER_BUILD_LLVM_DIR/lib -Wl,-rpath,$CRAFTER_BUILD_LLVM_DIR/lib" else - RPATH_FLAG="" + LIBCXX_LINK_FLAGS="" fi if [ -f "$CRAFTER_BUILD_LLVM_DIR/share/libc++/v1/std.cppm" ]; then STD_CPPM="$CRAFTER_BUILD_LLVM_DIR/share/libc++/v1/std.cppm" @@ -31,7 +32,7 @@ else CC=clang AR=llvm-ar LLVM_CONFIG=llvm-config - RPATH_FLAG="" + LIBCXX_LINK_FLAGS="" STD_CPPM=/usr/share/libc++/v1/std.cppm fi @@ -65,8 +66,8 @@ cmake -B build \ -DCMAKE_C_COMPILER="$CC" \ -DCMAKE_CXX_COMPILER="$CXX" \ -DCMAKE_CXX_FLAGS="-stdlib=libc++" \ - -DCMAKE_EXE_LINKER_FLAGS="-stdlib=libc++ $RPATH_FLAG" \ - -DCMAKE_SHARED_LINKER_FLAGS="-stdlib=libc++ $RPATH_FLAG" \ + -DCMAKE_EXE_LINKER_FLAGS="-stdlib=libc++ $LIBCXX_LINK_FLAGS" \ + -DCMAKE_SHARED_LINKER_FLAGS="-stdlib=libc++ $LIBCXX_LINK_FLAGS" \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_ARCHIVE_OUTPUT_DIRECTORY="$current_dir/build" \ -DCMAKE_LIBRARY_OUTPUT_DIRECTORY="$current_dir/build" \ @@ -117,7 +118,7 @@ rm -f ./lib/libcrafter-build.a "$AR" rcs ./lib/libcrafter-build.a $OBJECTS "$CXX" --target="$TARGET" -std=c++26 -stdlib=libc++ -O3 -march="$MARCH" -mtune="$MTUNE" -fuse-ld=lld \ - -Wl,--export-dynamic $RPATH_FLAG \ + -Wl,--export-dynamic $LIBCXX_LINK_FLAGS \ -L./build \ $OBJECTS \ ./build/main.o \ diff --git a/implementations/Crafter.Build-Toolchain.cpp b/implementations/Crafter.Build-Toolchain.cpp index 8a236a9..d7e4868 100644 --- a/implementations/Crafter.Build-Toolchain.cpp +++ b/implementations/Crafter.Build-Toolchain.cpp @@ -51,14 +51,17 @@ std::string Toolchain::LibcxxCompileFlags() const { } std::string Toolchain::LibcxxLinkFlags() const { - std::string flags; + // clang searches its per-target runtime directory and the system library + // paths for -lc++, not /lib, so a toolchain outside /usr with the + // classic layout needs the search path as well as the run-time path. + fs::path lib; if (!libcxxDir.empty()) { - fs::path lib = libcxxDir / "lib"; - flags += std::format(" -L{} -Wl,-rpath,{}", lib.string(), lib.string()); + lib = libcxxDir / "lib"; } else if (!prefix.empty() && prefix != "/usr") { - flags += std::format(" -Wl,-rpath,{}", (prefix / "lib").string()); + lib = prefix / "lib"; } - return flags; + if (lib.empty()) return {}; + return std::format(" -L{} -Wl,-rpath,{}", lib.string(), lib.string()); } std::string Toolchain::Key() const { diff --git a/interfaces/Crafter.Build-Toolchain.cppm b/interfaces/Crafter.Build-Toolchain.cppm index b0fd2dc..853b2b6 100644 --- a/interfaces/Crafter.Build-Toolchain.cppm +++ b/interfaces/Crafter.Build-Toolchain.cppm @@ -44,10 +44,12 @@ export namespace Crafter { // Flags for compiling against libcxxDir; empty without one. std::string LibcxxCompileFlags() const; - // Flags for linking against the libc++ the compiler will use: an - // rpath into the prefix when that is not the system one (a bundled - // toolchain's libc++.so.1 is not on the loader's default path), plus - // -L and an rpath for an explicit libcxxDir. + // Flags for linking against the libc++ the compiler will use: a -L + // and an rpath for /lib when the prefix is not the system one + // (clang searches only its per-target runtime directory and the + // system paths, and a bundled toolchain's libc++.so.1 is on neither + // the linker's nor the loader's default path), or for an explicit + // libcxxDir. std::string LibcxxLinkFlags() const; // Names this toolchain for cache directory names: the clang version // followed by a hash of what decides which headers and libraries a diff --git a/tests/ToolchainResolve/main.cpp b/tests/ToolchainResolve/main.cpp index aedb261..51d8af6 100644 --- a/tests/ToolchainResolve/main.cpp +++ b/tests/ToolchainResolve/main.cpp @@ -73,7 +73,7 @@ int main() { Check(tc.stdCppm == full / "share/libc++/v1/std.cppm", "the prefix's std module source is used"); Check(tc.wasiSysroot == full / "share/wasi-sysroot", "the prefix's wasi sysroot is used"); Check(tc.LibcxxCompileFlags().empty(), "the compiler finds its own libc++ headers without flags"); - Check(Contains(tc.LibcxxLinkFlags(), std::format("-Wl,-rpath,{}", (full / "lib").string())), "binaries get an rpath into the prefix's lib dir"); + Check(Contains(tc.LibcxxLinkFlags(), std::format("-L{}", (full / "lib").string())) && Contains(tc.LibcxxLinkFlags(), std::format("-Wl,-rpath,{}", (full / "lib").string())), "binaries get -L and an rpath for the prefix's lib dir"); } { ToolchainInputs in; @@ -95,7 +95,7 @@ int main() { Check(tc.llvmIncludeDir.empty(), "no clang-c headers in the prefix: nothing added"); Check(tc.stdCppm == "/usr/share/libc++/v1/std.cppm", "no std module in the prefix: the system copy"); Check(tc.wasiSysroot == "/usr/share/wasi-sysroot", "no wasi sysroot in the prefix: the system one"); - Check(Contains(tc.LibcxxLinkFlags(), std::format("-Wl,-rpath,{}", (bare / "lib").string())), "the rpath follows the explicit prefix"); + Check(Contains(tc.LibcxxLinkFlags(), std::format("-L{}", (bare / "lib").string())) && Contains(tc.LibcxxLinkFlags(), std::format("-Wl,-rpath,{}", (bare / "lib").string())), "the link flags follow the explicit prefix"); } { ToolchainInputs in; From 59fc9501f147232ae1675b4cc7b16c14eef5624a Mon Sep 17 00:00:00 2001 From: Jorijn van der Graaf Date: Fri, 2 Oct 2026 22:32:35 +0200 Subject: [PATCH 2/7] fix: key PCM and project.so freshness on content, not installed mtimes std.pcm was rebuilt when it was older than std.cppm, and project.so when it was older than the crafter-build executable. Both inputs are installed files, and an OSTree deployment (the Flatpak) gives every installed file mtime 0: std.cppm and the executable always looked older than the cache, so a clang bump would never rebuild std.pcm and a crafter-build update would never recompile project.so. clang crashes on a PCM from another compiler instead of reporting it, so the failure would have been a backtrace on the user's first build after an update. Each artifact now has a one-line stamp beside it. std.pcm's covers the precompile command and the module source's bytes, which also catches a libc++ update under the same clang. project.so's covers the crafter-build version, the toolchain key, the module source stamp and the compile command; project.cpp's and the pre-dependency artifacts' mtimes remain inputs, since those live next to it with real timestamps. The module source stamp stands in for the executable: project.so resolves Crafter:: symbols from the running binary, so only declarations matter, and an implementation-only rebuild of crafter-build no longer recompiles it. The std.pcm stamp folds into the module cache stamp, so a rebuilt std.pcm rebuilds the PCMs that import it. The cache directory carries the toolchain key (---), so a compiler bump starts cold instead of handing the new clang PCMs it will crash on. LoadProject and Build() now derive the directory from the same StdPcmDir. Old - directories are left behind; `clean` never touched the cache and still does not. Co-Authored-By: Claude Fable 5.1 --- implementations/Crafter.Build-Clang.cpp | 13 +- implementations/Crafter.Build-Platform.cpp | 141 ++++++++++++------- project.cpp | 2 + tests/HostCacheToolchainKey/main.cpp | 154 +++++++++++++++++++++ 4 files changed, 255 insertions(+), 55 deletions(-) create mode 100644 tests/HostCacheToolchainKey/main.cpp diff --git a/implementations/Crafter.Build-Clang.cpp b/implementations/Crafter.Build-Clang.cpp index 7c7f5c0..da2ddc4 100644 --- a/implementations/Crafter.Build-Clang.cpp +++ b/implementations/Crafter.Build-Clang.cpp @@ -246,11 +246,14 @@ void Configuration::GetInterfacesAndImplementations(std::span interfac } fs::path Crafter::StdPcmDir(const Configuration& config) { - // The std PCM is cached per target+march, but a wasm variant pass compiles - // it with extra codegen flags (e.g. -mrelaxed-simd) — its BMI must not be - // shared with the baseline's, or the consuming TUs see a target-feature - // mismatch. Suffix the cache dir with the variant flags when present. - std::string stdPcmKey = std::format("{}-{}", config.target, config.march); + // The std PCM is cached per target+march+toolchain: a PCM is only valid + // for the exact clang that wrote it, so a compiler bump starts a cold + // directory instead of handing the new clang a PCM it will crash on. A + // wasm variant pass compiles it with extra codegen flags (e.g. + // -mrelaxed-simd) — its BMI must not be shared with the baseline's, or + // the consuming TUs see a target-feature mismatch. Suffix the cache dir + // with the variant flags when present. + std::string stdPcmKey = std::format("{}-{}-{}", config.target, config.march, GetToolchain().Key()); for (const std::string& f : config.wasmVariantFlags) { stdPcmKey += "+"; for (char c : f) stdPcmKey += (c == '/' || c == '\\') ? '_' : c; diff --git a/implementations/Crafter.Build-Platform.cpp b/implementations/Crafter.Build-Platform.cpp index 4181bb6..a6f8793 100644 --- a/implementations/Crafter.Build-Platform.cpp +++ b/implementations/Crafter.Build-Platform.cpp @@ -90,15 +90,35 @@ namespace { return cacheDir / "crafter-build-sources.stamp"; } - std::string ReadCacheStamp(const fs::path& cacheDir) { - std::ifstream in(CacheStampPath(cacheDir), std::ios::binary); + // A stamp is a one-line content hash kept next to the artifact it + // describes. Written last, after the artifact, so an interrupted build + // leaves a stamp that disagrees and the next run starts over. + std::string ReadStamp(const fs::path& file) { + std::ifstream in(file, std::ios::binary); std::string stamp; std::getline(in, stamp); return stamp; } - void WriteCacheStamp(const fs::path& cacheDir, const std::string& stamp) { - std::ofstream(CacheStampPath(cacheDir), std::ios::binary | std::ios::trunc) << stamp << '\n'; + void WriteStamp(const fs::path& file, std::string_view stamp) { + std::ofstream(file, std::ios::binary | std::ios::trunc) << stamp << '\n'; + } + + fs::path StampPath(const fs::path& artifact) { + fs::path stamp = artifact; + stamp += ".stamp"; + return stamp; + } + + std::string HashStamp(std::string_view content) { + return std::format("{:016x}", std::hash{}(content)); + } + + std::string FileBytes(const fs::path& file) { + std::ifstream in(file, std::ios::binary); + std::ostringstream buffer; + buffer << in.rdbuf(); + return buffer.str(); } } @@ -495,12 +515,17 @@ namespace { return out; } - // project.so predates its source, the crafter-build that will load it, or - // a pre-dependency artifact — any of them means recompiling. - bool ProjectFileStale(const fs::path& artifact, const fs::path& project, const fs::path& hostExe, const ProjectFileDependencies& deps) { + // project.so is stale when its stamp (crafter-build version, toolchain, + // module sources and compile command) disagrees, or when it predates its + // source or a pre-dependency artifact. The executable's mtime is not an + // input any more: an OSTree deployment dates every installed file at the + // epoch, so a comparison against it never fired after an update and the + // old project.so was loaded into a binary it did not match. + bool ProjectFileStale(const fs::path& artifact, const fs::path& project, std::string_view stamp, const ProjectFileDependencies& deps) { if (!fs::exists(artifact)) return true; + if (ReadStamp(StampPath(artifact)) != stamp) return true; const fs::file_time_type built = fs::last_write_time(artifact); - return built < fs::last_write_time(project) || built < fs::last_write_time(hostExe) || built < deps.newestArtifact; + return built < fs::last_write_time(project) || built < deps.newestArtifact; } } @@ -611,21 +636,22 @@ std::string Crafter::BuildStdPcm(const Configuration& config, fs::path stdPcm) { fs::path stdCc = GetToolchain().mingwSysroot / "include" / "c++" / mingWversion / "bits" / "std.cc"; - if(!fs::exists(stdPcm) || fs::last_write_time(stdPcm) < fs::last_write_time(stdCc)) { - // -femulated-tls keeps PCM TLS access matching libstdc++'s emutls - // definitions; mismatch surfaces as undefined std::__once_callable. - // The std.cc → std.cppm copy uses a plain filename: held under the - // cache lock, no other builder writes this path concurrently. - fs::path stdCppm = stdPcm.parent_path() / "std.cppm"; - std::error_code copyEc; - fs::copy_file(stdCc, stdCppm, fs::copy_options::overwrite_existing, copyEc); - if (copyEc) { - return std::format("copy {} -> {}: {}", stdCc.string(), stdCppm.string(), copyEc.message()); - } - return RunBuildCommand(std::format("{} --target={} -march={} -mtune={} -femulated-tls -O3 -std=c++26 -Wno-reserved-identifier -Wno-reserved-module-identifier --precompile {} -o {}", GetBaseCommand(config), config.target, config.march, config.mtune, stdCppm.string(), stdPcm.string())); - } else { - return ""; + // -femulated-tls keeps PCM TLS access matching libstdc++'s emutls + // definitions; mismatch surfaces as undefined std::__once_callable. + // The std.cc → std.cppm copy uses a plain filename: held under the + // cache lock, no other builder writes this path concurrently. + fs::path stdCppm = stdPcm.parent_path() / "std.cppm"; + std::string command = std::format("{} --target={} -march={} -mtune={} -femulated-tls -O3 -std=c++26 -Wno-reserved-identifier -Wno-reserved-module-identifier --precompile {} -o {}", GetBaseCommand(config), config.target, config.march, config.mtune, stdCppm.string(), stdPcm.string()); + std::string stamp = HashStamp(command + '\0' + FileBytes(stdCc)); + if (fs::exists(stdPcm) && ReadStamp(StampPath(stdPcm)) == stamp) return ""; + std::error_code copyEc; + fs::copy_file(stdCc, stdCppm, fs::copy_options::overwrite_existing, copyEc); + if (copyEc) { + return std::format("copy {} -> {}: {}", stdCc.string(), stdCppm.string(), copyEc.message()); } + std::string result = RunBuildCommand(command); + if (result.empty()) WriteStamp(StampPath(stdPcm), stamp); + return result; } else { const bool isWasm = config.target.starts_with("wasm32"); // wasi-sdk drops std.cppm at /share/libc++/v1/, the rest of @@ -669,11 +695,16 @@ std::string Crafter::BuildStdPcm(const Configuration& config, fs::path stdPcm) { if (!isWasm && !config.sysroot.empty()) { archFlags += std::format(" -nostdinc++ -isystem {}/usr/include/c++/v1", config.sysroot); } - if(!fs::exists(stdPcm) || fs::last_write_time(stdPcm) < fs::last_write_time(stdCppm)) { - return RunBuildCommand(std::format("{} --target={} -std=c++26{}{} -O3 -Wno-reserved-identifier -Wno-reserved-module-identifier --precompile {} -o {}", GetBaseCommand(config), config.target, sysrootFlag, archFlags, stdCppm, stdPcm.string())); - } else { - return ""; - } + // Keyed on the command and the module source's bytes, never on + // mtimes: an installed std.cppm can be dated at the epoch (OSTree), + // and a libc++ update under the same clang changes the bytes but + // not the path. + std::string command = std::format("{} --target={} -std=c++26{}{} -O3 -Wno-reserved-identifier -Wno-reserved-module-identifier --precompile {} -o {}", GetBaseCommand(config), config.target, sysrootFlag, archFlags, stdCppm, stdPcm.string()); + std::string stamp = HashStamp(command + '\0' + FileBytes(stdCppm)); + if (fs::exists(stdPcm) && ReadStamp(StampPath(stdPcm)) == stamp) return ""; + std::string result = RunBuildCommand(command); + if (result.empty()) WriteStamp(StampPath(stdPcm), stamp); + return result; } } @@ -721,12 +752,17 @@ namespace { "Crafter.Build", }; - void EnsureCrafterBuildPcms(const fs::path& sourceDir, const fs::path& cacheDir, const Configuration& host) { + // Returns the stamp of the module sources the cache was built from; it is + // one of project.so's inputs. + std::string EnsureCrafterBuildPcms(const fs::path& sourceDir, const fs::path& cacheDir, const Configuration& host) { CacheLock lock(cacheDir); // Re-derived under the lock: another builder may have refreshed the // cache from its own sources while we waited. - std::string stamp = CrafterBuildSourceStamp(sourceDir, CrafterBuildModules); - const bool upToDate = ReadCacheStamp(cacheDir) == stamp; + // The std PCM is an input to every module PCM: its stamp is folded in + // so a rebuilt std.pcm (a libc++ update under the same clang) rebuilds + // them too, instead of leaving PCMs that import a std.pcm that is gone. + std::string stamp = HashStamp(CrafterBuildSourceStamp(sourceDir, CrafterBuildModules) + '\0' + ReadStamp(StampPath(cacheDir / "std.pcm"))); + const bool upToDate = ReadStamp(CacheStampPath(cacheDir)) == stamp; for (std::string_view name : CrafterBuildModules) { fs::path cppmPath = sourceDir / std::format("{}.cppm", name); fs::path pcmPath = cacheDir / std::format("{}.pcm", name); @@ -751,7 +787,8 @@ namespace { // Last, so an interrupted rebuild leaves the stamp disagreeing with the // sources and the next run starts over rather than trusting a half-built // set of PCMs. - if (!upToDate) WriteCacheStamp(cacheDir, stamp); + if (!upToDate) WriteStamp(CacheStampPath(cacheDir), stamp); + return stamp; } } @@ -765,8 +802,6 @@ Configuration Crafter::LoadProject(const fs::path& projectFile, std::span a // Same LoadProject wiring as ConcurrentCacheRace above. cfg.AddTest("HostCacheSourceStamp").Dependencies({ CrafterBuildLib.get() }) .LinkFlag("-Wl,--export-dynamic").LinkFlag("-ldl"); + cfg.AddTest("HostCacheToolchainKey").Dependencies({ CrafterBuildLib.get() }) + .LinkFlag("-Wl,--export-dynamic").LinkFlag("-ldl"); cfg.AddTest("ProjectDependency").Dependencies({ CrafterBuildLib.get() }) .LinkFlag("-Wl,--export-dynamic").LinkFlag("-ldl"); cfg.AddTest("ConcurrentDependencyReset").Dependencies({ CrafterBuildLib.get() }); diff --git a/tests/HostCacheToolchainKey/main.cpp b/tests/HostCacheToolchainKey/main.cpp new file mode 100644 index 0000000..58c35f8 --- /dev/null +++ b/tests/HostCacheToolchainKey/main.cpp @@ -0,0 +1,154 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts® + +#include +import std; +import Crafter.Build; +namespace fs = std::filesystem; +using namespace Crafter; + +// Freshness of the std PCM and of project.so used to be an mtime comparison +// against files that come with the install: std.cppm and the crafter-build +// executable. A Flatpak (any OSTree deployment) gives every installed file +// mtime 0, so those inputs always looked older than the cache: a clang bump +// would never rebuild std.pcm, a crafter-build update would never recompile +// project.so, and clang crashes on the stale PCM instead of saying so. +// +// Both are now keyed on content stamps (the compile command, the std module +// source, the crafter-build version, the toolchain key and the module source +// stamp) and the cache directory carries the toolchain key in its name. This +// test drives the shapes an mtime rule gets wrong: a std.pcm dated before its +// source must still be reused, a warm load must leave project.so alone, and a +// stamp that disagrees must rebuild even when the artifact is dated after +// every input. +namespace { + std::int32_t Failures = 0; + + void Check(bool cond, std::string_view msg) { + if (!cond) { + std::println(std::cerr, "FAIL: {}", msg); + ++Failures; + } + } + + fs::path FindModuleCacheDir(const fs::path& cacheRoot) { + for (const fs::directory_entry& entry : fs::directory_iterator(cacheRoot / "crafter.build")) { + if (entry.is_directory()) return entry.path(); + } + return {}; + } + + bool Load(const fs::path& projectFile, std::string_view what) { + std::array noArgs = {}; + try { + (void)LoadProject(projectFile, noArgs); + return true; + } catch (const std::exception& e) { + std::println(std::cerr, "FAIL: LoadProject ({}) threw: {}", what, e.what()); + ++Failures; + return false; + } + } + + fs::path StampOf(const fs::path& artifact) { + fs::path stamp = artifact; + stamp += ".stamp"; + return stamp; + } +} + +int main() { + fs::path installedHome = GetCrafterBuildHome(); + if (!fs::exists(installedHome / "Crafter.Build.cppm")) { + std::println(std::cerr, "SKIP: no module sources at {}", installedHome.string()); + return 77; + } + + fs::path scratch = fs::temp_directory_path() / "crafter-build-host-cache-toolchain-key"; + std::error_code ec; + fs::remove_all(scratch, ec); + fs::create_directories(scratch); + + // A private copy of the module sources and a cold private cache, as in + // HostCacheSourceStamp. + fs::path home = scratch / "share" / "crafter-build"; + fs::create_directories(home.parent_path()); + fs::copy(installedHome, home, fs::copy_options::recursive); + setenv("CRAFTER_BUILD_HOME", home.string().c_str(), 1); + fs::path cacheRoot = scratch / "cache"; + fs::create_directories(cacheRoot); + setenv("XDG_CACHE_HOME", cacheRoot.string().c_str(), 1); + + fs::path projectDir = scratch / "project"; + fs::create_directories(projectDir); + fs::path projectFile = projectDir / "project.cpp"; + std::ofstream(projectFile) << + "import std;\n" + "import Crafter.Build;\n" + "using namespace Crafter;\n" + "extern \"C\" Configuration CrafterBuildProject(std::span) {\n" + " Configuration cfg;\n" + " cfg.path = \"./\";\n" + " cfg.name = \"keyed\";\n" + " cfg.outputName = \"keyed\";\n" + " cfg.type = ConfigurationType::Executable;\n" + " return cfg;\n" + "}\n"; + + if (!Load(projectFile, "cold")) return 1; + + const fs::path cacheDir = FindModuleCacheDir(cacheRoot); + Check(!cacheDir.empty(), "a module cache directory was created"); + if (cacheDir.empty()) return 1; + Check(cacheDir.filename().string().contains(GetToolchain().Key()), std::format("the cache directory is named for the toolchain (got '{}', key '{}')", cacheDir.filename().string(), GetToolchain().Key())); + + const fs::path stdPcm = cacheDir / "std.pcm"; + const fs::path projectSo = projectDir / "build" / "project.so"; + Check(fs::exists(stdPcm) && fs::exists(StampOf(stdPcm)), "std.pcm and its stamp exist after a cold load"); + Check(fs::exists(projectSo) && fs::exists(StampOf(projectSo)), "project.so and its stamp exist after a cold load"); + if (!fs::exists(StampOf(stdPcm)) || !fs::exists(StampOf(projectSo))) return 1; + + // The OSTree shape, inverted: std.pcm dated before its source. A stamp + // rule reuses it; an mtime rule would rebuild it on every run. project.so + // keeps two mtime inputs that live next to it (project.cpp and dependency + // artifacts) but the executable is no longer one of them, so a warm load + // with nothing changed leaves it alone. + const fs::file_time_type epoch{}; + fs::last_write_time(stdPcm, epoch); + const fs::file_time_type afterCold = fs::last_write_time(projectSo); + if (!Load(projectFile, "std.pcm dated at the epoch")) return 1; + Check(fs::last_write_time(stdPcm) == epoch, "std.pcm dated before its source is still reused"); + Check(fs::last_write_time(projectSo) == afterCold, "project.so is left alone when nothing changed"); + + // The other half: artifacts dated after every input, with a stamp that + // disagrees. An mtime rule keeps them; the stamp rule rebuilds. + const fs::file_time_type future = fs::file_time_type::clock::now() + std::chrono::hours(24); + std::ofstream(StampOf(stdPcm)) << "stale"; + fs::last_write_time(stdPcm, future); + if (!Load(projectFile, "stale std.pcm stamp")) return 1; + Check(fs::last_write_time(stdPcm) != future, "a disagreeing stamp rebuilds std.pcm even when it is dated after its source"); + { + std::ifstream in(StampOf(stdPcm)); + std::string stamp; + std::getline(in, stamp); + Check(stamp != "stale" && !stamp.empty(), "the rebuilt std.pcm gets its stamp back"); + } + + std::ofstream(StampOf(projectSo)) << "stale"; + fs::last_write_time(projectSo, future); + if (!Load(projectFile, "stale project.so stamp")) return 1; + Check(fs::last_write_time(projectSo) != future, "a disagreeing stamp recompiles project.so even when it is dated after its inputs"); + + // The one mtime input that legitimately remains: the project file itself. + const fs::file_time_type beforeEdit = fs::last_write_time(projectSo); + { + std::ofstream append(projectFile, std::ios::app); + append << "// edited\n"; + } + fs::last_write_time(projectFile, fs::file_time_type::clock::now() + std::chrono::seconds(2)); + if (!Load(projectFile, "edited project.cpp")) return 1; + Check(fs::last_write_time(projectSo) != beforeEdit, "an edited project.cpp recompiles project.so"); + + fs::remove_all(scratch, ec); + return Failures == 0 ? 0 : 1; +} From c3485b7ab1f15d240cc7774b12f4a00ba0e30ab6 Mon Sep 17 00:00:00 2001 From: Jorijn van der Graaf Date: Fri, 2 Oct 2026 22:32:35 +0200 Subject: [PATCH 3/7] docs: name the toolchain in the cache directory layout Co-Authored-By: Claude Fable 5.1 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6af758c..400f9f1 100644 --- a/README.md +++ b/README.md @@ -107,7 +107,7 @@ import Crafter.LintRules; /bin/crafter-build # the executable /lib/libcrafter-build.a # static archive for downstream consumers /share/crafter-build/*.cppm # module sources (rebuilt on user machine on first run) -~/.cache/crafter.build/-/ # std.pcm + Crafter.Build*.pcm, built locally +~/.cache/crafter.build/---/ # std.pcm + Crafter.Build*.pcm, built locally per toolchain ``` ## Tests From add30b9a4f9dac4c2f49b5c47570df5be8421a50 Mon Sep 17 00:00:00 2001 From: Jorijn van der Graaf Date: Fri, 2 Oct 2026 22:34:57 +0200 Subject: [PATCH 4/7] feat(packaging): ship crafter-build as a Flatpak with its own toolchain packaging/flatpak/ holds the manifest for net.catcrafts.CrafterBuild and its side files. The runtime is the Freedesktop Sdk, not the Platform: a build tool needs glibc's headers and crt objects, gcc's libgcc_s, cmake, ninja, git and python at run time. Everything compiler-shaped is ours under /app, built from one LLVM 22.1.8 source tree: clang, lld, libclang, libc++ with its std module, compiler-rt for the host and for wasm32, the WASI sysroot (wasi-sdk 34 in sysroot-only mode, driven by the clang just built) and wasmtime for running the result. One tree means the modules crafter-build builds and the compiler that consumes them cannot drift apart, and the toolchain resolver finds all of it beside the executable with no configuration. The crafter-build module runs the bootstrap, the self-host build and the test suite inside the sandbox, so a Flatpak that installs has passed the suite with the toolchain it ships. CRAFTER_BUILD_MARCH is x86-64-v2 for the shipped binary; user projects keep -march=native. WasmHelloRuns compiles a fixture for wasm32-wasip1 and runs it under wasmtime. The existing wasm tests only inspect generated files, so this is the one that proves the sysroot, the std module for wasm and wasm-ld work together; it skips where the sysroot or wasmtime is missing. Co-Authored-By: Claude Fable 5.1 --- .gitignore | 4 +- packaging/flatpak/catcrafts.flatpakrepo | 8 + .../net.catcrafts.CrafterBuild.flatpakref | 13 ++ .../net.catcrafts.CrafterBuild.metainfo.xml | 34 ++++ .../flatpak/net.catcrafts.CrafterBuild.yaml | 182 ++++++++++++++++++ packaging/flatpak/wasi-host-toolchain.cmake | 23 +++ project.cpp | 1 + tests/WasmHelloRuns/fixture/main.cpp | 9 + tests/WasmHelloRuns/main.cpp | 57 ++++++ 9 files changed, 330 insertions(+), 1 deletion(-) create mode 100644 packaging/flatpak/catcrafts.flatpakrepo create mode 100644 packaging/flatpak/net.catcrafts.CrafterBuild.flatpakref create mode 100644 packaging/flatpak/net.catcrafts.CrafterBuild.metainfo.xml create mode 100644 packaging/flatpak/net.catcrafts.CrafterBuild.yaml create mode 100644 packaging/flatpak/wasi-host-toolchain.cmake create mode 100644 tests/WasmHelloRuns/fixture/main.cpp create mode 100644 tests/WasmHelloRuns/main.cpp diff --git a/.gitignore b/.gitignore index a47aaf6..67d2a68 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,6 @@ build/ bin/ lib/*.a share/crafter-build/ -.claude \ No newline at end of file +.claude +# written by CI next to the manifest so relative source paths still resolve +packaging/flatpak/ci-manifest.yaml diff --git a/packaging/flatpak/catcrafts.flatpakrepo b/packaging/flatpak/catcrafts.flatpakrepo new file mode 100644 index 0000000..8792ccd --- /dev/null +++ b/packaging/flatpak/catcrafts.flatpakrepo @@ -0,0 +1,8 @@ +[Flatpak Repo] +Title=Catcrafts +Url=https://flatpak.catcrafts.net/repo/ +Homepage=https://forgejo.catcrafts.net/Catcrafts +Comment=Catcrafts tools and applications +Description=crafter-build and other Catcrafts software, built and signed by Catcrafts CI +DefaultBranch=stable +GPGKey=GPG_KEY_BASE64_PLACEHOLDER diff --git a/packaging/flatpak/net.catcrafts.CrafterBuild.flatpakref b/packaging/flatpak/net.catcrafts.CrafterBuild.flatpakref new file mode 100644 index 0000000..16359ae --- /dev/null +++ b/packaging/flatpak/net.catcrafts.CrafterBuild.flatpakref @@ -0,0 +1,13 @@ +[Flatpak Ref] +Title=Crafter Build +Name=net.catcrafts.CrafterBuild +Branch=stable +Url=https://flatpak.catcrafts.net/repo/ +SuggestRemoteName=catcrafts +Homepage=https://forgejo.catcrafts.net/Catcrafts/Crafter.Build +Comment=C++26 modules build system +Description=crafter-build with its own LLVM toolchain and WASI sysroot +Icon= +RuntimeRepo=https://dl.flathub.org/repo/flathub.flatpakrepo +IsRuntime=false +GPGKey=GPG_KEY_BASE64_PLACEHOLDER diff --git a/packaging/flatpak/net.catcrafts.CrafterBuild.metainfo.xml b/packaging/flatpak/net.catcrafts.CrafterBuild.metainfo.xml new file mode 100644 index 0000000..7108966 --- /dev/null +++ b/packaging/flatpak/net.catcrafts.CrafterBuild.metainfo.xml @@ -0,0 +1,34 @@ + + + + net.catcrafts.CrafterBuild + Crafter Build + C++26 modules build system + CC0-1.0 + LGPL-3.0-only + + Catcrafts + + +

+ A build system for C++26 modules. Project descriptions are C++: a + project.cpp constructs a Configuration and returns it, and crafter-build + compiles, loads and executes it. The Flatpak carries its own LLVM, so + every machine builds with the same clang, lld, libc++ and WASI sysroot. +

+ + https://forgejo.catcrafts.net/Catcrafts/Crafter.Build + https://forgejo.catcrafts.net/Catcrafts/Crafter.Build/issues + https://forgejo.catcrafts.net/Catcrafts/Crafter.Build + + crafter-build + + + + + +

First release distributed as a Flatpak, with a bundled LLVM 22 toolchain and WASI sysroot. Windows host support is removed; crafter-build gains --version and resolves its compiler from one place.

+ + + + diff --git a/packaging/flatpak/net.catcrafts.CrafterBuild.yaml b/packaging/flatpak/net.catcrafts.CrafterBuild.yaml new file mode 100644 index 0000000..01e286e --- /dev/null +++ b/packaging/flatpak/net.catcrafts.CrafterBuild.yaml @@ -0,0 +1,182 @@ +# SPDX-License-Identifier: LGPL-3.0-only +# SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts® +# +# crafter-build as a Flatpak. The runtime is the Freedesktop *Sdk*, not the +# Platform: a build tool needs glibc's headers and crt objects, gcc's +# libgcc_s, cmake, ninja, make, git and python at run time. Everything +# compiler-shaped is ours and lives under /app: clang, lld, libclang, libc++, +# compiler-rt (native and wasm32) and the WASI sysroot, all built from one +# LLVM source tree so the modules crafter-build builds and the compiler that +# consumes them can never drift apart. +app-id: net.catcrafts.CrafterBuild +runtime: org.freedesktop.Sdk +runtime-version: '26.08' +sdk: org.freedesktop.Sdk +command: crafter-build +finish-args: + # git dependencies, and the local web server `-r` starts for browser builds + - --share=network + - --socket=ssh-auth + # projects live anywhere; user sysroots too + - --filesystem=host + # so `-r` can launch what it built, games included + - --share=ipc + - --socket=wayland + - --socket=fallback-x11 + - --socket=pulseaudio + - --device=all + # gdb and friends from the Sdk + - --allow=devel + - --env=PATH=/app/bin:/usr/bin +build-options: + env: + CRAFTER_BUILD_LLVM_DIR: /app + # The shipped binary runs on any x86_64 from the last decade; user + # projects still build with -march=native by default. + CRAFTER_BUILD_MARCH: x86-64-v2 + CRAFTER_BUILD_MTUNE: generic + # CI replaces this with the tag; a local build is a development build. + CRAFTER_BUILD_VERSION: 0.0.0-dev +cleanup: + - /share/man + - /share/doc +modules: + - name: llvm + buildsystem: cmake-ninja + builddir: true + subdir: llvm + build-options: + # Debug info for all of LLVM would be a multi-gigabyte .Debug extension. + no-debuginfo: true + config-opts: + - -DCMAKE_BUILD_TYPE=Release + - -DLLVM_ENABLE_ASSERTIONS=OFF + - -DLLVM_ENABLE_PROJECTS=clang;lld + - -DLLVM_ENABLE_RUNTIMES=compiler-rt;libcxx;libcxxabi;libunwind + - -DLLVM_TARGETS_TO_BUILD=X86;AArch64;WebAssembly + # One shared LLVM and one shared clang library instead of a statically + # linked copy per tool: a fraction of the install size. + - -DLLVM_BUILD_LLVM_DYLIB=ON + - -DLLVM_LINK_LLVM_DYLIB=ON + - -DCLANG_LINK_CLANG_DYLIB=ON + - -DCLANG_DEFAULT_CXX_STDLIB=libc++ + - -DCLANG_DEFAULT_LINKER=lld + - -DLIBCXX_INSTALL_MODULES=ON + - -DLLVM_INCLUDE_TESTS=OFF + - -DLLVM_INCLUDE_EXAMPLES=OFF + - -DLLVM_INCLUDE_DOCS=OFF + - -DLLVM_INCLUDE_BENCHMARKS=OFF + - -DLLVM_INSTALL_UTILS=OFF + - -DCLANG_INCLUDE_TESTS=OFF + - -DCOMPILER_RT_INCLUDE_TESTS=OFF + # The Sdk links with binutils' ld; two libLLVM-sized links at a time + # fit the CX53's memory with room to spare. + - -DLLVM_PARALLEL_LINK_JOBS=2 + cleanup: + - /include/llvm + - /include/llvm-c + - /include/clang + - /include/clang-tidy + - /include/lld + - /lib/cmake + - /lib/*.a + - /share/opt-viewer + - /share/scan-build + - /share/scan-view + - /bin/clang-check + - /bin/clang-extdef-mapping + - /bin/clang-import-test + - /bin/clang-linker-wrapper + - /bin/clang-nvlink-wrapper + - /bin/clang-offload-bundler + - /bin/clang-offload-packager + - /bin/clang-refactor + - /bin/clang-rename + - /bin/clang-repl + - /bin/clang-sycl-linker + - /bin/diagtool + - /bin/git-clang-format + - /bin/hmaptool + - /bin/llvm-exegesis + - /bin/scan-build* + - /bin/scan-view + sources: + - type: archive + url: https://github.com/llvm/llvm-project/releases/download/llvmorg-22.1.8/llvm-project-22.1.8.src.tar.xz + sha256: 922f1817a0df7b1489272d18134ee0087a8b068828f87ac63b9861b1a9965888 + + - name: wasi-sysroot + # wasi-sdk's sysroot-only mode, driven by the clang built above: + # wasi-libc, compiler-rt builtins (installed into our clang's resource + # dir, where the wasm32 driver looks for them) and libc++ for wasip1 and + # wasip1-threads. The llvm-project tree is the same tarball as above so + # the wasm libc++ matches the native one. + buildsystem: cmake-ninja + builddir: true + build-options: + no-debuginfo: true + config-opts: + - -DCMAKE_TOOLCHAIN_FILE=/run/build/wasi-sysroot/wasi-host-toolchain.cmake + - -DWASI_SDK_BUILD_TOOLCHAIN=OFF + - -DWASI_SDK_INSTALL_TO_CLANG_RESOURCE_DIR=ON + - -DWASI_SDK_TARGETS=wasm32-wasip1;wasm32-wasip1-threads + - -DWASI_SDK_INCLUDE_TESTS=OFF + - -DCMAKE_INSTALL_PREFIX=/app + post-install: + # wasi-sdk installs into /share/wasi-sysroot; crafter-build + # looks there (Crafter.Build:Toolchain), so nothing to move. + - test -f /app/share/wasi-sysroot/share/libc++/v1/std.cppm + sources: + - type: git + url: https://github.com/WebAssembly/wasi-sdk.git + tag: wasi-sdk-34 + disable-submodules: true + - type: git + url: https://github.com/WebAssembly/wasi-libc.git + commit: 2e6fb9d8ee0cdf9e431fbcabe8af3115de000a13 + dest: src/wasi-libc + - type: archive + url: https://github.com/llvm/llvm-project/releases/download/llvmorg-22.1.8/llvm-project-22.1.8.src.tar.xz + sha256: 922f1817a0df7b1489272d18134ee0087a8b068828f87ac63b9861b1a9965888 + dest: src/llvm-project + - type: file + path: wasi-host-toolchain.cmake + + - name: wasmtime + # The runner for wasm32-wasi tests and `-r` on a non-browser wasm build. + buildsystem: simple + build-commands: + - install -Dm755 wasmtime /app/bin/wasmtime + sources: + - type: archive + url: https://github.com/bytecodealliance/wasmtime/releases/download/v49.0.2/wasmtime-v49.0.2-x86_64-linux.tar.xz + sha256: a4d6e9e3a5a60f527cf7793d674c48930c80c2e8977995b8a275cad3254b9322 + + - name: crafter-build + buildsystem: simple + build-options: + # The self-host build fetches Crafter.Math, Crafter.Asset, + # Crafter.LintRules and glslang from git; they are unpinned on purpose + # (sibling repos tracking their default branch, see AGENTS.md). + build-args: + - --share=network + build-commands: + # Bootstrap by hand, then let the result build itself: only the + # self-hosted binary has asset support and ThinLTO. + - ./build.sh + - ./bin/crafter-build + - CRAFTER_BUILD_HOME=$PWD/share/crafter-build ./bin/crafter-build test + - install -Dm755 bin/crafter.build-exe-*/crafter-build /app/bin/crafter-build + - install -Dm644 bin/crafter.build-lib-*/libcrafter-build.a /app/lib/libcrafter-build.a + - mkdir -p /app/share && cp -r share/crafter-build /app/share/ + - install -Dm644 LICENSE LICENSE.GPL -t /app/share/licenses/net.catcrafts.CrafterBuild/ + - install -Dm644 packaging/flatpak/net.catcrafts.CrafterBuild.metainfo.xml -t /app/share/metainfo/ + sources: + - type: dir + path: ../.. + skip: + - .git + - bin + - build + - lib/libcrafter-build.a + - share/crafter-build diff --git a/packaging/flatpak/wasi-host-toolchain.cmake b/packaging/flatpak/wasi-host-toolchain.cmake new file mode 100644 index 0000000..662d822 --- /dev/null +++ b/packaging/flatpak/wasi-host-toolchain.cmake @@ -0,0 +1,23 @@ +# SPDX-License-Identifier: LGPL-3.0-only +# SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts® +# +# Points wasi-sdk's sysroot build at the clang the Flatpak built, instead of +# the one wasi-sdk would build itself. Mirrors the wasi-sdk.cmake toolchain +# file an installed wasi-sdk ships, with our prefix. +set(WASI_SDK_PREFIX /app) +set(CMAKE_SYSTEM_NAME WASI) +set(CMAKE_SYSTEM_VERSION 1) +set(CMAKE_SYSTEM_PROCESSOR wasm32) +set(triple wasm32-wasip1) +set(CMAKE_C_COMPILER ${WASI_SDK_PREFIX}/bin/clang) +set(CMAKE_CXX_COMPILER ${WASI_SDK_PREFIX}/bin/clang++) +set(CMAKE_ASM_COMPILER ${WASI_SDK_PREFIX}/bin/clang) +set(CMAKE_AR ${WASI_SDK_PREFIX}/bin/llvm-ar) +set(CMAKE_RANLIB ${WASI_SDK_PREFIX}/bin/llvm-ranlib) +set(CMAKE_NM ${WASI_SDK_PREFIX}/bin/llvm-nm) +set(CMAKE_C_COMPILER_TARGET ${triple}) +set(CMAKE_CXX_COMPILER_TARGET ${triple}) +set(CMAKE_ASM_COMPILER_TARGET ${triple}) +# The sysroot does not exist yet when cmake probes the compiler. +set(CMAKE_C_COMPILER_WORKS ON) +set(CMAKE_CXX_COMPILER_WORKS ON) diff --git a/project.cpp b/project.cpp index f2a982b..d3364d3 100644 --- a/project.cpp +++ b/project.cpp @@ -132,6 +132,7 @@ extern "C" Configuration CrafterBuildProject(std::span a cfg.AddTest("VersionFlag").Dependencies({ CrafterBuildLib.get() }); cfg.AddTest("WasiBrowserRuntime").Dependencies({ CrafterBuildLib.get() }); cfg.AddTest("WasmVariants").Dependencies({ CrafterBuildLib.get() }); + cfg.AddTest("WasmHelloRuns").Dependencies({ CrafterBuildLib.get() }); cfg.AddTest("RunSingleTestExit").Dependencies({ CrafterBuildLib.get() }); // LoadProject dlopens the synthesized project.so, which references // Crafter:: symbols (HostTarget, Configuration ctors) that have to be diff --git a/tests/WasmHelloRuns/fixture/main.cpp b/tests/WasmHelloRuns/fixture/main.cpp new file mode 100644 index 0000000..59fb23d --- /dev/null +++ b/tests/WasmHelloRuns/fixture/main.cpp @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts® + +import std; + +int main() { + std::println("hello from wasm"); + return 0; +} diff --git a/tests/WasmHelloRuns/main.cpp b/tests/WasmHelloRuns/main.cpp new file mode 100644 index 0000000..58a910f --- /dev/null +++ b/tests/WasmHelloRuns/main.cpp @@ -0,0 +1,57 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts® + +import std; +import Crafter.Build; +namespace fs = std::filesystem; +using namespace Crafter; + +// A wasm build end to end: the fixture is compiled for wasm32-wasip1 against +// the resolved toolchain's WASI sysroot and the result is run under +// wasmtime. The other wasm tests only inspect generated files; this is the +// one that proves the sysroot, the std module for wasm and wasm-ld all work +// together. It skips when the sysroot or wasmtime is missing, so a +// development box without them is not a failure. Inside the Flatpak both are +// bundled, and there the skip would be the failure to look for. +int main() { + const Toolchain& tc = GetToolchain(); + if (!fs::exists(tc.wasiSysroot / "include")) { + std::println(std::cerr, "SKIP: no WASI sysroot at {}", tc.wasiSysroot.string()); + return 77; + } + if (std::system("which wasmtime > /dev/null 2>&1") != 0) { + std::println(std::cerr, "SKIP: wasmtime not on PATH"); + return 77; + } + + Configuration cfg; + cfg.path = fs::current_path() / "tests" / "WasmHelloRuns" / "fixture"; + cfg.name = "wasm-hello"; + cfg.outputName = "wasm-hello"; + cfg.target = "wasm32-wasip1"; + cfg.type = ConfigurationType::Executable; + std::array ifaces = {}; + std::array impls = { "main" }; + cfg.GetInterfacesAndImplementations(ifaces, impls); + + std::unordered_map> depResults; + std::mutex depMutex; + BuildResult r = Build(cfg, depResults, depMutex); + if (!r.result.empty()) { + std::println(std::cerr, "build failed: {}", r.result); + return 1; + } + + const fs::path wasm = ExpectedOutput(cfg); + if (!fs::exists(wasm)) { + std::println(std::cerr, "no .wasm produced at {}", wasm.string()); + return 1; + } + + const auto run = RunCommandWithTimeout(std::format("wasmtime {}", wasm.string()), std::chrono::seconds(60)); + if (run.exitCode != 0 || !run.output.contains("hello from wasm")) { + std::println(std::cerr, "wasmtime run failed (exit {}): {}", run.exitCode, run.output); + return 1; + } + return 0; +} From ff2e8c03a9c4f634c2ed829db50f0df5e261bf4a Mon Sep 17 00:00:00 2001 From: Jorijn van der Graaf Date: Fri, 2 Oct 2026 23:02:44 +0200 Subject: [PATCH 5/7] fix(packaging): classic libc++ layout, no debug info, clean wasm flags Three findings from building the manifest locally. LLVM's runtimes build installs libc++ under lib// by default. clang finds it there, but the rpath crafter-build adds for a toolchain outside /usr points at /lib, and so does every distro layout the resolver was modelled on; the bootstrap failed to start with libc++.so.1 missing. LLVM_ENABLE_PER_TARGET_RUNTIME_DIR=OFF puts the runtimes in lib/ and include/c++/v1. flatpak-builder's default CFLAGS carry -g. For all of LLVM that is a 30 GB build tree and a multi-gigabyte .Debug extension; the module now builds without debug info and strips, keeping the hardening flags. The no-debuginfo key also moves under build-options, where flatpak-builder actually reads it. The same default CFLAGS and LDFLAGS (-fcf-protection, -z relro) reach wasi-sdk's nested wasm32 builds through the environment, and clang rejects them for that target. The wasi-sysroot module clears them. Co-Authored-By: Claude Fable 5.1 --- .../flatpak/net.catcrafts.CrafterBuild.yaml | 25 ++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/packaging/flatpak/net.catcrafts.CrafterBuild.yaml b/packaging/flatpak/net.catcrafts.CrafterBuild.yaml index 01e286e..7226ad1 100644 --- a/packaging/flatpak/net.catcrafts.CrafterBuild.yaml +++ b/packaging/flatpak/net.catcrafts.CrafterBuild.yaml @@ -46,14 +46,27 @@ modules: builddir: true subdir: llvm build-options: - # Debug info for all of LLVM would be a multi-gigabyte .Debug extension. + # flatpak-builder's default CFLAGS carry -g. For all of LLVM that is a + # 30 GB build tree and a multi-gigabyte .Debug extension, so build + # without debug info, strip, and skip the split. The hardening flags + # from the defaults stay. no-debuginfo: true + strip: true + cflags: -O2 -pipe -fexceptions -fasynchronous-unwind-tables -fstack-protector-strong -fstack-clash-protection -fcf-protection + cflags-override: true + cxxflags: -O2 -pipe -fexceptions -fasynchronous-unwind-tables -fstack-protector-strong -fstack-clash-protection -fcf-protection + cxxflags-override: true config-opts: - -DCMAKE_BUILD_TYPE=Release - -DLLVM_ENABLE_ASSERTIONS=OFF - -DLLVM_ENABLE_PROJECTS=clang;lld - -DLLVM_ENABLE_RUNTIMES=compiler-rt;libcxx;libcxxabi;libunwind - -DLLVM_TARGETS_TO_BUILD=X86;AArch64;WebAssembly + # Classic layout: libc++ in lib/ and include/c++/v1, not under a + # per-target subdirectory, the same shape as a distro install. + # crafter-build adds -L and an rpath for /lib itself, since + # clang searches only its per-target directory and the system paths. + - -DLLVM_ENABLE_PER_TARGET_RUNTIME_DIR=OFF # One shared LLVM and one shared clang library instead of a statically # linked copy per tool: a fraction of the install size. - -DLLVM_BUILD_LLVM_DYLIB=ON @@ -115,6 +128,16 @@ modules: builddir: true build-options: no-debuginfo: true + # flatpak-builder's default CFLAGS/LDFLAGS are x86-64 hardening flags + # (-fcf-protection, -z relro) that the nested wasm32 builds inherit + # through the environment and clang rejects for that target. + cflags: "" + cflags-override: true + cxxflags: "" + cxxflags-override: true + ldflags: "" + ldflags-override: true + strip: true config-opts: - -DCMAKE_TOOLCHAIN_FILE=/run/build/wasi-sysroot/wasi-host-toolchain.cmake - -DWASI_SDK_BUILD_TOOLCHAIN=OFF From 47d6d2ccd127398c3d9090a0f81f3737571f415f Mon Sep 17 00:00:00 2001 From: Jorijn van der Graaf Date: Fri, 2 Oct 2026 22:36:55 +0200 Subject: [PATCH 6/7] feat(ci): build and publish the Flatpak One job replaces the tarball pipeline. On a pull request it builds the Flatpak with flatpak-builder, which runs the bootstrap, the self-host build and the test suite inside the sandbox, and smoke-tests the result with `crafter-build --version` and the hello-world example. On a push to master it also bundles the build, promotes it into the live repo as the `beta` branch and refreshes the rolling `latest` prerelease. On a tag X.Y.Z it promotes as `stable`, refuses to proceed unless the metainfo's newest release is that version, and publishes a release whose notes are that entry's description. The job runs on the `flatpak` runner label: the privileged forgejo-runner on the Hetzner box, with the Sdk install, the flatpak-builder state and the repo's web root as volumes. The state volume is what keeps a normal run short: flatpak-builder rebuilds the LLVM and WASI modules only when their sources change. The live repo is canonical and never built into; builds are promoted with build-commit-from, then build-update-repo signs and prunes it and generates static deltas. The musl job, the per-march tarballs, the toolchain packages the runner used to install and the PKGBUILD go: the Flatpak is the one artifact. Co-Authored-By: Claude Fable 5.1 --- .forgejo/workflows/ci.yaml | 326 ++++++++++++++++--------------------- AGENTS.md | 16 +- PKGBUILD | 35 ---- 3 files changed, 152 insertions(+), 225 deletions(-) delete mode 100644 PKGBUILD diff --git a/.forgejo/workflows/ci.yaml b/.forgejo/workflows/ci.yaml index 6d9d973..d011ec3 100644 --- a/.forgejo/workflows/ci.yaml +++ b/.forgejo/workflows/ci.yaml @@ -1,229 +1,185 @@ -name: CI +name: Flatpak +# One job, three outcomes: +# pull request -> build the Flatpak and run the suite inside it +# push master -> also publish the result as the `beta` branch of the +# Catcrafts repo and refresh the rolling `latest` release +# tag X.Y.Z -> publish as `stable` and create release X.Y.Z +# +# The `flatpak` runner label is the privileged runner on the Hetzner box +# (bubblewrap needs it) with three volumes allowed: the Sdk install, the +# flatpak-builder state (ccache and the LLVM module, so only crafter-build +# itself rebuilds on a normal run) and the published repo's web root. on: pull_request: branches: [master] push: branches: [master] + tags: ['[0-9]+.[0-9]+.[0-9]+'] workflow_dispatch: +env: + APP_ID: net.catcrafts.CrafterBuild + MANIFEST: packaging/flatpak/net.catcrafts.CrafterBuild.yaml + CI_MANIFEST: packaging/flatpak/ci-manifest.yaml + RUNTIME: org.freedesktop.Sdk//26.08 + REPO_DIR: /srv/flatpak/repo + WEB_DIR: /srv/flatpak + jobs: - build-test-release: - runs-on: arch-latest - env: - # x86-64-v2 = SSE4.2 baseline; runs on the CI SBC (Intel N5105 / Tremont, - # no AVX) and on essentially every x86_64 CPU since ~2011. The bootstrap - # MUST stay v2 — anything higher would emit instructions the SBC can't - # execute and the bootstrap binary itself would crash before it could - # rebuild for higher levels. v3/v4 variants are produced from this v2 - # bootstrap by re-running it with CRAFTER_BUILD_MARCH overridden. - CRAFTER_BUILD_MARCH: x86-64-v2 - CRAFTER_BUILD_MTUNE: generic + flatpak: + runs-on: flatpak + container: + image: archlinux:latest + options: >- + -v crafter-flatpak-system:/var/lib/flatpak + -v crafter-flatpak-builder:/root/.cache/flatpak-builder + -v /var/www/flatpak:/srv/flatpak steps: - - name: Install build dependencies + - name: Install tools run: | - # The slim archlinux:latest image arrives without a populated - # pacman keyring AND without a local pacman master key, so: - # 1. --init generates the local signing key (needed when pacman - # itself rewrites the keyring during package upgrades) - # 2. --populate archlinux imports the upstream master keys so - # currently-shipping signatures verify - # After that we can refresh archlinux-keyring to pick up keys for - # packagers added after the image's snapshot, then -Syu the rest. pacman-key --init pacman-key --populate archlinux pacman -Sy --noconfirm --needed archlinux-keyring - pacman -Syu --noconfirm --needed \ - base-devel git zip tar jq \ - clang llvm lld libc++ cmake \ - mingw-w64-gcc \ - wasi-libc wasi-libc++ wasi-libc++abi wasi-compiler-rt \ - nodejs - # The container runs as root and the workspace may be owned by a - # different uid; tell git not to refuse operations on it. + pacman -Syu --noconfirm --needed flatpak flatpak-builder ccache git nodejs jq gnupg go-yq ostree git config --global --add safe.directory '*' + - name: Sdk + # A no-op once the volume is warm; `update` follows point releases. + run: | + flatpak remote-add --if-not-exists flathub https://dl.flathub.org/repo/flathub.flatpakrepo + flatpak install -y --noninteractive flathub "$RUNTIME" + flatpak update -y --noninteractive + - name: Checkout uses: actions/checkout@v4 with: - # Persist the auth token so the 'Update rolling latest tag' step - # below can push the tag back via the implicit GITHUB_TOKEN. + fetch-depth: 0 persist-credentials: true - - name: Cache glslang clone+build - # Cache covers crafter-build's external dep cache. The first run builds - # glslang; the cache makes subsequent runs reuse that build. - uses: actions/cache@v4 - with: - path: | - ~/.cache/crafter.build/external - key: glslang-${{ runner.os }}-v2 - - - name: Bootstrap (build.sh) - run: ./build.sh - - # Full style gate: Report() findings (naming, enum-class, char*, …) AND - # would-reformat findings from the transform rules. A dirty tree means - # someone skipped `crafter-build format` — fail fast, before the - # (much slower) test suite. - # - name: Lint - # run: CRAFTER_BUILD_HOME=$PWD/share/crafter-build ./bin/crafter-build lint - - - name: Run tests - run: CRAFTER_BUILD_HOME=$PWD/share/crafter-build ./bin/crafter-build test - - # Everything below produces release artifacts and only runs on push to - # master. PRs stop after the test step above — the bootstrap binary is - # enough to validate the change; the per-march variant builds and the - # archive uploads are wasted work on a PR. - - name: Build Linux x86-64-v2 (matches bootstrap) - if: (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.ref == 'refs/heads/master' - run: CRAFTER_BUILD_HOME=$PWD/share/crafter-build ./bin/crafter-build - - - name: Build Linux x86-64-v3 - if: (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.ref == 'refs/heads/master' - run: CRAFTER_BUILD_HOME=$PWD/share/crafter-build CRAFTER_BUILD_MARCH=x86-64-v3 ./bin/crafter-build - - - name: Build Linux x86-64-v4 - if: (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.ref == 'refs/heads/master' - run: CRAFTER_BUILD_HOME=$PWD/share/crafter-build CRAFTER_BUILD_MARCH=x86-64-v4 ./bin/crafter-build - - - name: Package artifacts - if: (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.ref == 'refs/heads/master' + - name: Version and channel + id: meta + run: | + set -eu + if [ "${GITHUB_REF_TYPE}" = tag ]; then + version="$GITHUB_REF_NAME" + branch=stable + # The metainfo's newest release is the release notes and must be + # the version being tagged. + top=$(grep -m1 -oP '&2 + exit 1 + fi + else + version="0.0.0-dev+$(git rev-parse --short HEAD)" + branch=beta + fi + echo "version=$version" >> "$GITHUB_OUTPUT" + echo "branch=$branch" >> "$GITHUB_OUTPUT" + echo "version $version, flatpak branch $branch" + + - name: Build + # The manifest is rewritten beside the original so its relative + # source paths still resolve. The suite runs inside this build. + env: + VERSION: ${{ steps.meta.outputs.version }} + run: | + set -eu + yq '."build-options".env.CRAFTER_BUILD_VERSION = strenv(VERSION)' "$MANIFEST" > "$CI_MANIFEST" + grep -n 'CRAFTER_BUILD_VERSION' "$CI_MANIFEST" + flatpak-builder --force-clean --ccache --disable-rofiles-fuse \ + --state-dir=/root/.cache/flatpak-builder \ + --repo=repo --default-branch="${{ steps.meta.outputs.branch }}" \ + build-dir "$CI_MANIFEST" + + - name: Smoke test + run: | + flatpak-builder --run build-dir "$CI_MANIFEST" crafter-build --version + flatpak-builder --run build-dir "$CI_MANIFEST" sh -c 'cd examples/hello-world && crafter-build && ls bin/*/hello*' + + # Everything below publishes and only runs on master and on tags. + - name: Bundle + if: github.event_name != 'pull_request' run: | - set -eux mkdir -p dist + flatpak build-bundle repo "dist/crafter-build-${{ steps.meta.outputs.version }}.flatpak" "$APP_ID" "${{ steps.meta.outputs.branch }}" + cp packaging/flatpak/net.catcrafts.CrafterBuild.flatpakref dist/ + (cd dist && sha256sum * > SHA256SUMS) + ls -la dist - # 3 archives, one per march level. Each carries the executable, the - # static library and share/; libc++ comes from the host. - for march in x86-64-v2 x86-64-v3 x86-64-v4; do - # Short suffix (v2/v3/v4) for archive filenames; the upload-artifact - # steps below reference these short names. - short=${march##*-} - - # VariantId is "----"; CI pins - # mtune=generic so each (target, march) resolves to a single subdir - # but the trailing hash isn't known to the workflow, hence the glob. - stage_lin=$(mktemp -d) - mkdir -p "$stage_lin/bin" "$stage_lin/lib" - cp bin/crafter.build-exe-x86_64-pc-linux-gnu-$march-*/crafter-build "$stage_lin/bin/" - cp bin/crafter.build-lib-x86_64-pc-linux-gnu-$march-*/libcrafter-build.a "$stage_lin/lib/" - cp -r share "$stage_lin/" - tar czf "dist/crafter-build-linux-x86_64-$short.tar.gz" -C "$stage_lin" . - done - - ls -la dist/ - - # Upload each archive as its own workflow artifact (one upload per file) - # so users browsing the PR / run page get small, individually-named - # downloads instead of one wrapper zip containing the lot. - - name: Upload Linux v2 artifact - if: (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.ref == 'refs/heads/master' + - name: Upload bundle + if: github.event_name != 'pull_request' uses: actions/upload-artifact@v3 - with: { name: crafter-build-linux-x86_64-v2, path: dist/crafter-build-linux-x86_64-v2.tar.gz, if-no-files-found: error } - - name: Upload Linux v3 artifact - if: (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.ref == 'refs/heads/master' - uses: actions/upload-artifact@v3 - with: { name: crafter-build-linux-x86_64-v3, path: dist/crafter-build-linux-x86_64-v3.tar.gz, if-no-files-found: error } - - name: Upload Linux v4 artifact - if: (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.ref == 'refs/heads/master' - uses: actions/upload-artifact@v3 - with: { name: crafter-build-linux-x86_64-v4, path: dist/crafter-build-linux-x86_64-v4.tar.gz, if-no-files-found: error } + with: + name: crafter-build-${{ steps.meta.outputs.version }} + path: dist/*.flatpak + if-no-files-found: error - - name: Update rolling 'latest' tag - if: (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.ref == 'refs/heads/master' + - name: Publish to the Catcrafts repo + if: github.event_name != 'pull_request' + env: + FLATPAK_GPG_PRIVATE_KEY: ${{ secrets.FLATPAK_GPG_PRIVATE_KEY }} + FLATPAK_GPG_KEY_ID: ${{ secrets.FLATPAK_GPG_KEY_ID }} + run: | + set -eu + export GNUPGHOME=$(mktemp -d) + printf '%s' "$FLATPAK_GPG_PRIVATE_KEY" | gpg --batch --quiet --import + # The live repo is the canonical copy; builds are promoted into it, + # never built into it. + [ -d "$REPO_DIR" ] || ostree init --repo="$REPO_DIR" --mode=archive + flatpak build-commit-from --src-repo=repo \ + --gpg-sign="$FLATPAK_GPG_KEY_ID" --gpg-homedir="$GNUPGHOME" \ + "$REPO_DIR" "app/$APP_ID/x86_64/${{ steps.meta.outputs.branch }}" + flatpak build-update-repo --gpg-sign="$FLATPAK_GPG_KEY_ID" --gpg-homedir="$GNUPGHOME" \ + --generate-static-deltas --prune "$REPO_DIR" + # The repo description and the app ref carry the public key. + key=$(gpg --export "$FLATPAK_GPG_KEY_ID" | base64 --wrap=0) + sed "s|GPGKey=.*|GPGKey=$key|" packaging/flatpak/catcrafts.flatpakrepo > "$WEB_DIR/catcrafts.flatpakrepo" + sed "s|GPGKey=.*|GPGKey=$key|" packaging/flatpak/net.catcrafts.CrafterBuild.flatpakref > "$WEB_DIR/net.catcrafts.CrafterBuild.flatpakref" + + - name: Rolling 'latest' release + if: github.event_name != 'pull_request' && github.ref_type != 'tag' run: | git config user.email "ci@catcrafts.net" git config user.name "Crafter Build CI" git tag -f latest git push origin latest --force - - name: Publish rolling 'latest' release - # forgejo-release uploads each file in release-dir as a separate - # release asset, so the Releases page shows individually - # downloadable archives — pick the v-level matching your CPU. - if: (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.ref == 'refs/heads/master' + if: github.event_name != 'pull_request' && github.ref_type != 'tag' uses: https://code.forgejo.org/actions/forgejo-release@v2 with: direction: upload url: ${{ github.server_url }} repo: ${{ github.repository }} tag: latest - title: Latest main build + title: Latest master build prerelease: true override: true release-dir: dist token: ${{ secrets.GITHUB_TOKEN }} - # musl build for Alpine / postmarketOS hosts: the glibc launcher above cannot - # run there (musl has no glibc loader and gcompat lacks libgcc_s + the - # __isoc23_* symbols). Bootstrap natively on alpine:edge and ship the - # bootstrap output — the same thing the Arch PKGBUILD installs. Dynamic - # against musl libc++ (crafter-build dlopens project.so, so it cannot be - # fully static); consumers need `apk add clang lld libc++-dev llvm-runtimes`. - release-musl: - needs: build-test-release - if: (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.ref == 'refs/heads/master' - runs-on: arch-latest - container: - image: alpine:edge - env: - CRAFTER_BUILD_MARCH: x86-64-v2 - CRAFTER_BUILD_MTUNE: generic - steps: - - name: Install build dependencies - # nodejs: actions/checkout is a Node action. build-base: gcc's - # libgcc_s + crt objects, which clang's musl driver links against, plus - # make/binutils for the glslang cmake build (python3: its generators). - # clang-dev + llvm-dev: the libclang header Crafter.Build-Lint includes - # lives under llvm-config --includedir here, not /usr/include. llvm: - # llvm-ar for the LTO static archives the self-host/test builds write. + - name: Release notes + if: github.ref_type == 'tag' run: | - apk add -q nodejs git curl jq tar clang clang-dev lld libc++-dev \ - llvm-libunwind-dev llvm-runtimes llvm llvm-dev cmake build-base python3 - git config --global --add safe.directory '*' - - - name: Checkout - uses: actions/checkout@v4 - - - name: Bootstrap (build.sh) - run: ./build.sh - - - name: Run tests - run: CRAFTER_BUILD_HOME=$PWD/share/crafter-build ./bin/crafter-build test - - - name: Package artifact - run: | - set -eux - file bin/crafter-build || true - mkdir -p dist - stage=$(mktemp -d) - mkdir -p "$stage/bin" "$stage/lib" - cp bin/crafter-build "$stage/bin/" - cp lib/libcrafter-build.a "$stage/lib/" - cp -r share "$stage/" - tar czf dist/crafter-build-linux-x86_64-musl-v2.tar.gz -C "$stage" . - ls -la dist/ - - - name: Upload musl artifact - uses: actions/upload-artifact@v3 - with: { name: crafter-build-linux-x86_64-musl-v2, path: dist/crafter-build-linux-x86_64-musl-v2.tar.gz, if-no-files-found: error } - - # Add the asset to the rolling 'latest' release the first job just - # (re)published, via the API rather than forgejo-release: that action's - # override mode would drop the other job's assets. - - name: Attach to rolling 'latest' release - env: - TOKEN: ${{ secrets.GITHUB_TOKEN }} - API: ${{ github.server_url }}/api/v1/repos/${{ github.repository }} - run: | - set -eu - asset=crafter-build-linux-x86_64-musl-v2.tar.gz - rel=$(curl -fsS -H "Authorization: token $TOKEN" "$API/releases/tags/latest") - id=$(printf '%s' "$rel" | jq -r .id) - for old in $(printf '%s' "$rel" | jq -r --arg n "$asset" '.assets[] | select(.name==$n) | .id'); do - curl -fsS -X DELETE -H "Authorization: token $TOKEN" "$API/releases/$id/assets/$old" - done - curl -fsS -H "Authorization: token $TOKEN" -F "attachment=@dist/$asset" \ - "$API/releases/$id/assets?name=$asset" > /dev/null - echo "attached $asset to release $id" + # The newest block's description, as plain text. + python3 - > release-notes.md <<'EOF' + import re, html + text = open("packaging/flatpak/net.catcrafts.CrafterBuild.metainfo.xml").read() + block = re.search(r"]*>(.*?)", text, re.S).group(1) + paras = re.findall(r"

(.*?)

", block, re.S) + print("\n\n".join(html.unescape(" ".join(p.split())) for p in paras)) + print("\nInstall:\n\n flatpak remote-add --if-not-exists catcrafts https://flatpak.catcrafts.net/catcrafts.flatpakrepo\n flatpak install catcrafts net.catcrafts.CrafterBuild\n") + EOF + - name: Publish release + if: github.ref_type == 'tag' + uses: https://code.forgejo.org/actions/forgejo-release@v2 + with: + direction: upload + url: ${{ github.server_url }} + repo: ${{ github.repository }} + tag: ${{ github.ref_name }} + title: crafter-build ${{ github.ref_name }} + release-notes: release-notes.md + release-dir: dist + token: ${{ secrets.GITHUB_TOKEN }} diff --git a/AGENTS.md b/AGENTS.md index b33dad4..8a1fc1d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -17,7 +17,7 @@ crafter-build is a C++26 modules build system. A project is described by a `proj | `wasi-runtime/` | `runtime.js` and `index.html.in`, shipped for browser targets. | | `lib/` | Vendored third-party headers (nlohmann/json, glslang's DirStackFileIncluder). Do not edit. | | `share/crafter-build/`, `bin/`, `build/`, `lib/*.a` | Generated and gitignored. | -| `PKGBUILD`, `.forgejo/workflows/ci.yaml` | Arch package and the Forgejo Actions CI. | +| `packaging/flatpak/`, `.forgejo/workflows/ci.yaml` | The Flatpak manifest with its metainfo, repo and ref files, and the Forgejo Actions CI that builds and publishes it. | Partitions and what they own: @@ -116,7 +116,7 @@ These were decided deliberately. Do not undo them without the maintainer's say-s 1. **Configuration is C++.** No JSON, YAML, TOML or DSL for project, runner or lint configuration. Lint rules are C++ callbacks; there is no clang-format or clang-tidy integration and no separate "format rule" type. A rule that calls `ctx.SetContent` is a transform, and that is the whole mechanism. 2. **Per-project artifacts go in `/build` and `/bin`. Only cross-project things go in `~/.cache/crafter.build/`:** the std PCM and Crafter.Build PCMs per target and march, and external clones keyed by their spec hash. -3. **Sources ship, PCMs build locally.** Packages install `*.cppm` under `share/crafter-build/`. Nothing target-baked ships except the executable itself, and that defaults to a portable `-march` in `PKGBUILD` and CI. +3. **Sources ship, PCMs build locally.** The Flatpak installs `*.cppm` under `/app/share/crafter-build/`. Nothing target-baked ships except the executable and the toolchain beside it, and the executable is built for a portable `-march`. 4. **crafter-build is a monolithic executable with `--export-dynamic`.** `project.so` resolves `Crafter::` symbols from the running binary at dlopen time. There is no Windows host; `x86_64-w64-mingw32` is a cross-compile target for user projects only. 5. **Convention over configuration.** Lead with the zero-config path. Env var, then CLI flag, then full `project.cpp` are the escalation, not the entry point. 6. **One skip mechanism** (runner probe plus exit 77) and **one `TestRunner` shape** (template fields with `{bin}`, `{bundle}` and similar placeholders; `Cmd(...)` covers any wrapper). Do not add a special-case runner class or a hardcoded "skip on this OS" branch. @@ -131,9 +131,9 @@ These were decided deliberately. Do not undo them without the maintainer's say-s - Subject line: `type(scope): imperative summary`, with `feat`, `fix`, `refactor`, `docs` or `test`; the scope (`lint`, `incremental`, ...) is optional. Body: what was wrong, why, and why the fix has this shape, wrapped at about 75 columns. Read `git log` for the voice. - Separate commits for the fix, its test and the README change when each stands on its own. - Update the README in the same PR when user-visible behaviour changes. -- `format`, `lint` and `test` green before pushing. CI runs the bootstrap and the test suite on PRs and publishes release archives on `master`. +- `format`, `lint` and `test` green before pushing. CI builds the Flatpak and runs the suite inside it on PRs, publishes the `beta` branch of the Catcrafts repo on every push to `master`, and the `stable` branch plus a release on an `X.Y.Z` tag. - Never commit `bin/`, `build/`, `share/crafter-build/`, `lib/*.a` or fixture build outputs (all gitignored), and never edit the vendored files in `lib/`. -- Keep CI's `CRAFTER_BUILD_MARCH=x86-64-v2` for the bootstrap: the runner's CPU cannot execute v3 code, and the v3 and v4 artifacts are produced from the v2 binary. +- Keep `CRAFTER_BUILD_MARCH=x86-64-v2` in the manifest for the shipped binary: it has to run on any x86_64 machine. User projects still default to `native`. ## Security considerations @@ -141,9 +141,15 @@ These were decided deliberately. Do not undo them without the maintainer's say-s - Unpinned dependencies `git pull` on every invocation, so builds and tests need network. A missing or private repo surfaces as a clone failure before anything is built. - `--runner=cmd:` executes the named wrapper around every test binary. Wrappers come from `--runner=` or `CRAFTER_BUILD_RUNNER_*`; do not hardcode hosts, wrappers or credentials in `project.cpp` or tests. - The linter loads `libclang` at runtime by name. `CRAFTER_BUILD_LIBCLANG` pins an exact path and must fail loudly when wrong, never fall back to another library. -- CI force-pushes the rolling `latest` tag and publishes a prerelease on every push to `master` with the workflow's own token. Do not add secrets to the workflow or echo the environment in it. +- CI force-pushes the rolling `latest` tag and publishes a prerelease on every push to `master` with the workflow's own token, and signs the Flatpak repo with the key in the `FLATPAK_GPG_PRIVATE_KEY` secret. Do not add other secrets to the workflow or echo the environment in it. - Tests write only under `fs::temp_directory_path()` and the fixture's own `bin/` and `build/`. Keep it that way. +## CI and publishing + +`.forgejo/workflows/ci.yaml` runs on the `flatpak` runner label: the privileged forgejo-runner on the Hetzner box (bubblewrap inside the job container needs it). Its `config.yml` allows three volumes, which the workflow mounts: `crafter-flatpak-system` (the Sdk install at `/var/lib/flatpak`), `crafter-flatpak-builder` (flatpak-builder state and ccache, so the LLVM and WASI modules are rebuilt only when their sources change) and `/var/www/flatpak`, the web root Caddy serves as `https://flatpak.catcrafts.net/`. Secrets: `FLATPAK_GPG_PRIVATE_KEY` (armored, no passphrase) and `FLATPAK_GPG_KEY_ID`. The live repo at `/var/www/flatpak/repo` is canonical; builds are promoted into it with `flatpak build-commit-from`, never built into it. + +To release: add a `` to `packaging/flatpak/net.catcrafts.CrafterBuild.metainfo.xml` (its description becomes the release notes), merge, and push the tag `X.Y.Z`. CI refuses a tag that does not match the newest entry. Local builds: `flatpak-builder --user --force-clean --ccache --repo=repo build-dir packaging/flatpak/net.catcrafts.CrafterBuild.yaml`, then `flatpak-builder --run build-dir crafter-build --version`. + ## Gotchas - A `#pragma crafter dependency` target is rebuilt on every load on purpose: clang crashes rather than erroring when it imports a BMI whose own imported PCM was rebuilt underneath it, and the depfile check is what catches a regenerated host cache. Do not optimise the rebuild away. diff --git a/PKGBUILD b/PKGBUILD deleted file mode 100644 index 38299d1..0000000 --- a/PKGBUILD +++ /dev/null @@ -1,35 +0,0 @@ -#SPDX-License-Identifier: LGPL-3.0-only -#SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts® - -pkgname=crafter-build -pkgver=0.1.0 -pkgrel=1 -pkgdesc='C++26 modules build system' -arch=('x86_64') -url='https://forgejo.catcrafts.net/Catcrafts/Crafter.Build' -license=('LGPL-3.0-only') -depends=('clang' 'libc++' 'lld') -optdepends=('wasi-libc: WASI/WebAssembly target support' - 'wasi-libc++: C++ standard library for WASI' - 'wasi-libc++abi: C++ ABI support for WASI') -makedepends=('cmake' 'git') -source=() -sha256sums=() -options=('!strip' '!debug' '!lto') - -build() { - cd "$startdir" - rm -rf bin build share/crafter-build - CRAFTER_BUILD_MARCH=x86-64-v3 CRAFTER_BUILD_MTUNE=generic ./build.sh -} - -package() { - cd "$startdir" - install -Dm755 bin/crafter-build "$pkgdir/usr/bin/crafter-build" - install -Dm644 lib/libcrafter-build.a "$pkgdir/usr/lib/libcrafter-build.a" - install -dm755 "$pkgdir/usr/share/crafter-build" - install -m644 share/crafter-build/*.cppm "$pkgdir/usr/share/crafter-build/" - install -dm755 "$pkgdir/usr/share/crafter-build/wasi-runtime" - install -m644 share/crafter-build/wasi-runtime/runtime.js "$pkgdir/usr/share/crafter-build/wasi-runtime/" - install -m644 share/crafter-build/wasi-runtime/index.html.in "$pkgdir/usr/share/crafter-build/wasi-runtime/" -} From 2c3b8637f565c1bbfe8b9987b5de5ad08213c1e7 Mon Sep 17 00:00:00 2001 From: Jorijn van der Graaf Date: Fri, 2 Oct 2026 22:36:56 +0200 Subject: [PATCH 7/7] docs: install from the Catcrafts Flatpak repo The README leads with the Flatpak install, keeps the native loop as the way to build from source, and drops the tarball, musl and makepkg text that no longer describes anything CI produces. The install layout shows the /app tree and the per-app cache directory. Co-Authored-By: Claude Fable 5.1 --- AGENTS.md | 2 ++ README.md | 39 ++++++++++++++++++++++++++------------- 2 files changed, 28 insertions(+), 13 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 8a1fc1d..1ad41da 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -32,6 +32,8 @@ Partitions and what they own: ## Dev environment +The shipped artifact is the Flatpak (see "CI and publishing"); the loop below is the native development loop. + Prerequisites on Linux: `clang` (with `libclang`, which the AST lint rules load at runtime), `lld`, `libc++` with its std module source (`share/libc++/v1/std.cppm`), `cmake`, `git`. `crafter-build --version` prints which compiler was resolved; `CRAFTER_BUILD_LLVM_DIR` points it at another LLVM prefix, and `build.sh` honours the same variable. Network access to `forgejo.catcrafts.net` and `github.com` is needed: loading `project.cpp` fetches Crafter.Math, Crafter.Asset and Crafter.LintRules into `~/.cache/crafter.build/external/`, and the bootstrap clones glslang. Those sources are unpinned, so every invocation runs `git pull` on them. ```bash diff --git a/README.md b/README.md index 400f9f1..5296f74 100644 --- a/README.md +++ b/README.md @@ -2,9 +2,21 @@ A C++26 modules build system. Project descriptions are written in C++ with no JSON, no Lua, no embedded DSL. You write a `project.cpp` that constructs a `Configuration` and returns it; Crafter Build compiles, loads, and executes it. -## Quick start (Linux) +## Install -Bootstrap requires `clang`, `cmake`, `git`, `lld`, and `libc++`. +crafter-build ships as a Flatpak that carries its own clang, lld, libc++ and WASI sysroot, so every machine builds with the same compiler: + +```bash +flatpak remote-add --if-not-exists catcrafts https://flatpak.catcrafts.net/catcrafts.flatpakrepo +flatpak install catcrafts net.catcrafts.CrafterBuild +alias crafter-build='flatpak run net.catcrafts.CrafterBuild' +``` + +Pre-release builds of `master` are the `beta` branch (`flatpak install catcrafts net.catcrafts.CrafterBuild//beta`), and every release also attaches a single-file bundle on the Forgejo releases page (`flatpak install --user crafter-build-.flatpak`). The sandbox sees the whole host filesystem, so projects can live anywhere. Binaries built for the host link the bundled `libc++.so.1` through an rpath into the Flatpak: they run inside it (`-r`, the test runner), and are not meant to be copied out onto the host. + +## Building from source (Linux) + +The development loop is native. It needs `clang`, `cmake`, `git`, `lld` and `libc++` with its std module source: ```bash ./build.sh # produces bin/crafter-build @@ -13,13 +25,7 @@ Bootstrap requires `clang`, `cmake`, `git`, `lld`, and `libc++`. The compiler is resolved once per run: `CRAFTER_BUILD_LLVM_DIR` if set, else a `clang++` installed beside crafter-build (`/bin`), else PATH. `crafter-build --version` prints the version and the compiler in use; `crafter-build --help` lists the other overrides (`CRAFTER_BUILD_LIBCXX_DIR`, `CRAFTER_BUILD_LIBCLANG`, `CRAFTER_BUILD_WASI_SYSROOT`, `CRAFTER_MINGW_DIR`). A `Configuration` targets the host triple unless `cfg.target` says otherwise. -The bootstrap targets the host toolchain's own triple (`clang++ -print-target-triple`): `x86_64-pc-linux-gnu` on glibc distros, `x86_64-alpine-linux-musl` on Alpine/postmarketOS (`apk add clang clang-dev lld libc++-dev llvm-libunwind-dev llvm-runtimes llvm llvm-dev cmake build-base python3 git`). Override with `CRAFTER_BUILD_TARGET=`. CI ships both: `crafter-build-linux-x86_64-v{2,3,4}.tar.gz` (glibc) and `crafter-build-linux-x86_64-musl-v2.tar.gz` (musl, needs `libc++` at runtime — the launcher dlopens `project.so`, so it cannot be static). - -To build the system as a distro package on Arch: - -```bash -makepkg -si # uses CRAFTER_BUILD_MARCH=x86-64-v3 by default for portability -``` +The bootstrap targets the toolchain's own triple (`clang++ -print-target-triple`); override with `CRAFTER_BUILD_TARGET=`. The Flatpak is built from `packaging/flatpak/net.catcrafts.CrafterBuild.yaml`, which runs this same bootstrap, the self-host build and the test suite inside the sandbox against the bundled toolchain (`CRAFTER_BUILD_LLVM_DIR=/app`). ## Writing a project.cpp @@ -103,13 +109,20 @@ import Crafter.LintRules; ## Install layout +Inside the Flatpak: + ``` -/bin/crafter-build # the executable -/lib/libcrafter-build.a # static archive for downstream consumers -/share/crafter-build/*.cppm # module sources (rebuilt on user machine on first run) -~/.cache/crafter.build/---/ # std.pcm + Crafter.Build*.pcm, built locally per toolchain +/app/bin/crafter-build # the executable +/app/bin/clang++ lld llvm-ar wasmtime # the bundled toolchain the executable resolves first +/app/lib/libcrafter-build.a # static archive for downstream consumers +/app/share/crafter-build/*.cppm # module sources (rebuilt on the user's machine on first run) +/app/share/wasi-sysroot/ # WASI sysroot for wasm32 targets +~/.var/app/net.catcrafts.CrafterBuild/cache/crafter.build/---/ + # std.pcm + Crafter.Build*.pcm, built locally per toolchain ``` +A source build uses the same shape under the checkout (`bin/`, `lib/`, `share/crafter-build/`) with the system compiler and `~/.cache/crafter.build/`. + ## Tests Tests live under `tests//`. The simplest case is a single C++ file: