diff --git a/.gitignore b/.gitignore index a47aaf6..67d2a68 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,6 @@ build/ bin/ lib/*.a share/crafter-build/ -.claude \ No newline at end of file +.claude +# written by CI next to the manifest so relative source paths still resolve +packaging/flatpak/ci-manifest.yaml diff --git a/README.md b/README.md index 6af758c..400f9f1 100644 --- a/README.md +++ b/README.md @@ -107,7 +107,7 @@ import Crafter.LintRules; /bin/crafter-build # the executable /lib/libcrafter-build.a # static archive for downstream consumers /share/crafter-build/*.cppm # module sources (rebuilt on user machine on first run) -~/.cache/crafter.build/-/ # std.pcm + Crafter.Build*.pcm, built locally +~/.cache/crafter.build/---/ # std.pcm + Crafter.Build*.pcm, built locally per toolchain ``` ## Tests diff --git a/build.sh b/build.sh index eeff75f..2d7df47 100755 --- a/build.sh +++ b/build.sh @@ -14,12 +14,13 @@ if [ -n "${CRAFTER_BUILD_LLVM_DIR:-}" ]; then CC="$CRAFTER_BUILD_LLVM_DIR/bin/clang" AR="$CRAFTER_BUILD_LLVM_DIR/bin/llvm-ar" LLVM_CONFIG="$CRAFTER_BUILD_LLVM_DIR/bin/llvm-config" - # A prefix other than the system one keeps its libc++ off the loader's - # default path; the rpath is what lets the binary start. + # A prefix other than the system one keeps its libc++ off both the + # linker's search path (clang looks in lib/ and the system dirs, + # not /lib) and the loader's; -L and an rpath cover the two. if [ "$CRAFTER_BUILD_LLVM_DIR" != /usr ]; then - RPATH_FLAG="-Wl,-rpath,$CRAFTER_BUILD_LLVM_DIR/lib" + LIBCXX_LINK_FLAGS="-L$CRAFTER_BUILD_LLVM_DIR/lib -Wl,-rpath,$CRAFTER_BUILD_LLVM_DIR/lib" else - RPATH_FLAG="" + LIBCXX_LINK_FLAGS="" fi if [ -f "$CRAFTER_BUILD_LLVM_DIR/share/libc++/v1/std.cppm" ]; then STD_CPPM="$CRAFTER_BUILD_LLVM_DIR/share/libc++/v1/std.cppm" @@ -31,7 +32,7 @@ else CC=clang AR=llvm-ar LLVM_CONFIG=llvm-config - RPATH_FLAG="" + LIBCXX_LINK_FLAGS="" STD_CPPM=/usr/share/libc++/v1/std.cppm fi @@ -65,8 +66,8 @@ cmake -B build \ -DCMAKE_C_COMPILER="$CC" \ -DCMAKE_CXX_COMPILER="$CXX" \ -DCMAKE_CXX_FLAGS="-stdlib=libc++" \ - -DCMAKE_EXE_LINKER_FLAGS="-stdlib=libc++ $RPATH_FLAG" \ - -DCMAKE_SHARED_LINKER_FLAGS="-stdlib=libc++ $RPATH_FLAG" \ + -DCMAKE_EXE_LINKER_FLAGS="-stdlib=libc++ $LIBCXX_LINK_FLAGS" \ + -DCMAKE_SHARED_LINKER_FLAGS="-stdlib=libc++ $LIBCXX_LINK_FLAGS" \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_ARCHIVE_OUTPUT_DIRECTORY="$current_dir/build" \ -DCMAKE_LIBRARY_OUTPUT_DIRECTORY="$current_dir/build" \ @@ -117,7 +118,7 @@ rm -f ./lib/libcrafter-build.a "$AR" rcs ./lib/libcrafter-build.a $OBJECTS "$CXX" --target="$TARGET" -std=c++26 -stdlib=libc++ -O3 -march="$MARCH" -mtune="$MTUNE" -fuse-ld=lld \ - -Wl,--export-dynamic $RPATH_FLAG \ + -Wl,--export-dynamic $LIBCXX_LINK_FLAGS \ -L./build \ $OBJECTS \ ./build/main.o \ diff --git a/implementations/Crafter.Build-Clang.cpp b/implementations/Crafter.Build-Clang.cpp index 7c7f5c0..da2ddc4 100644 --- a/implementations/Crafter.Build-Clang.cpp +++ b/implementations/Crafter.Build-Clang.cpp @@ -246,11 +246,14 @@ void Configuration::GetInterfacesAndImplementations(std::span interfac } fs::path Crafter::StdPcmDir(const Configuration& config) { - // The std PCM is cached per target+march, but a wasm variant pass compiles - // it with extra codegen flags (e.g. -mrelaxed-simd) — its BMI must not be - // shared with the baseline's, or the consuming TUs see a target-feature - // mismatch. Suffix the cache dir with the variant flags when present. - std::string stdPcmKey = std::format("{}-{}", config.target, config.march); + // The std PCM is cached per target+march+toolchain: a PCM is only valid + // for the exact clang that wrote it, so a compiler bump starts a cold + // directory instead of handing the new clang a PCM it will crash on. A + // wasm variant pass compiles it with extra codegen flags (e.g. + // -mrelaxed-simd) — its BMI must not be shared with the baseline's, or + // the consuming TUs see a target-feature mismatch. Suffix the cache dir + // with the variant flags when present. + std::string stdPcmKey = std::format("{}-{}-{}", config.target, config.march, GetToolchain().Key()); for (const std::string& f : config.wasmVariantFlags) { stdPcmKey += "+"; for (char c : f) stdPcmKey += (c == '/' || c == '\\') ? '_' : c; diff --git a/implementations/Crafter.Build-Platform.cpp b/implementations/Crafter.Build-Platform.cpp index 4181bb6..a6f8793 100644 --- a/implementations/Crafter.Build-Platform.cpp +++ b/implementations/Crafter.Build-Platform.cpp @@ -90,15 +90,35 @@ namespace { return cacheDir / "crafter-build-sources.stamp"; } - std::string ReadCacheStamp(const fs::path& cacheDir) { - std::ifstream in(CacheStampPath(cacheDir), std::ios::binary); + // A stamp is a one-line content hash kept next to the artifact it + // describes. Written last, after the artifact, so an interrupted build + // leaves a stamp that disagrees and the next run starts over. + std::string ReadStamp(const fs::path& file) { + std::ifstream in(file, std::ios::binary); std::string stamp; std::getline(in, stamp); return stamp; } - void WriteCacheStamp(const fs::path& cacheDir, const std::string& stamp) { - std::ofstream(CacheStampPath(cacheDir), std::ios::binary | std::ios::trunc) << stamp << '\n'; + void WriteStamp(const fs::path& file, std::string_view stamp) { + std::ofstream(file, std::ios::binary | std::ios::trunc) << stamp << '\n'; + } + + fs::path StampPath(const fs::path& artifact) { + fs::path stamp = artifact; + stamp += ".stamp"; + return stamp; + } + + std::string HashStamp(std::string_view content) { + return std::format("{:016x}", std::hash{}(content)); + } + + std::string FileBytes(const fs::path& file) { + std::ifstream in(file, std::ios::binary); + std::ostringstream buffer; + buffer << in.rdbuf(); + return buffer.str(); } } @@ -495,12 +515,17 @@ namespace { return out; } - // project.so predates its source, the crafter-build that will load it, or - // a pre-dependency artifact — any of them means recompiling. - bool ProjectFileStale(const fs::path& artifact, const fs::path& project, const fs::path& hostExe, const ProjectFileDependencies& deps) { + // project.so is stale when its stamp (crafter-build version, toolchain, + // module sources and compile command) disagrees, or when it predates its + // source or a pre-dependency artifact. The executable's mtime is not an + // input any more: an OSTree deployment dates every installed file at the + // epoch, so a comparison against it never fired after an update and the + // old project.so was loaded into a binary it did not match. + bool ProjectFileStale(const fs::path& artifact, const fs::path& project, std::string_view stamp, const ProjectFileDependencies& deps) { if (!fs::exists(artifact)) return true; + if (ReadStamp(StampPath(artifact)) != stamp) return true; const fs::file_time_type built = fs::last_write_time(artifact); - return built < fs::last_write_time(project) || built < fs::last_write_time(hostExe) || built < deps.newestArtifact; + return built < fs::last_write_time(project) || built < deps.newestArtifact; } } @@ -611,21 +636,22 @@ std::string Crafter::BuildStdPcm(const Configuration& config, fs::path stdPcm) { fs::path stdCc = GetToolchain().mingwSysroot / "include" / "c++" / mingWversion / "bits" / "std.cc"; - if(!fs::exists(stdPcm) || fs::last_write_time(stdPcm) < fs::last_write_time(stdCc)) { - // -femulated-tls keeps PCM TLS access matching libstdc++'s emutls - // definitions; mismatch surfaces as undefined std::__once_callable. - // The std.cc → std.cppm copy uses a plain filename: held under the - // cache lock, no other builder writes this path concurrently. - fs::path stdCppm = stdPcm.parent_path() / "std.cppm"; - std::error_code copyEc; - fs::copy_file(stdCc, stdCppm, fs::copy_options::overwrite_existing, copyEc); - if (copyEc) { - return std::format("copy {} -> {}: {}", stdCc.string(), stdCppm.string(), copyEc.message()); - } - return RunBuildCommand(std::format("{} --target={} -march={} -mtune={} -femulated-tls -O3 -std=c++26 -Wno-reserved-identifier -Wno-reserved-module-identifier --precompile {} -o {}", GetBaseCommand(config), config.target, config.march, config.mtune, stdCppm.string(), stdPcm.string())); - } else { - return ""; + // -femulated-tls keeps PCM TLS access matching libstdc++'s emutls + // definitions; mismatch surfaces as undefined std::__once_callable. + // The std.cc → std.cppm copy uses a plain filename: held under the + // cache lock, no other builder writes this path concurrently. + fs::path stdCppm = stdPcm.parent_path() / "std.cppm"; + std::string command = std::format("{} --target={} -march={} -mtune={} -femulated-tls -O3 -std=c++26 -Wno-reserved-identifier -Wno-reserved-module-identifier --precompile {} -o {}", GetBaseCommand(config), config.target, config.march, config.mtune, stdCppm.string(), stdPcm.string()); + std::string stamp = HashStamp(command + '\0' + FileBytes(stdCc)); + if (fs::exists(stdPcm) && ReadStamp(StampPath(stdPcm)) == stamp) return ""; + std::error_code copyEc; + fs::copy_file(stdCc, stdCppm, fs::copy_options::overwrite_existing, copyEc); + if (copyEc) { + return std::format("copy {} -> {}: {}", stdCc.string(), stdCppm.string(), copyEc.message()); } + std::string result = RunBuildCommand(command); + if (result.empty()) WriteStamp(StampPath(stdPcm), stamp); + return result; } else { const bool isWasm = config.target.starts_with("wasm32"); // wasi-sdk drops std.cppm at /share/libc++/v1/, the rest of @@ -669,11 +695,16 @@ std::string Crafter::BuildStdPcm(const Configuration& config, fs::path stdPcm) { if (!isWasm && !config.sysroot.empty()) { archFlags += std::format(" -nostdinc++ -isystem {}/usr/include/c++/v1", config.sysroot); } - if(!fs::exists(stdPcm) || fs::last_write_time(stdPcm) < fs::last_write_time(stdCppm)) { - return RunBuildCommand(std::format("{} --target={} -std=c++26{}{} -O3 -Wno-reserved-identifier -Wno-reserved-module-identifier --precompile {} -o {}", GetBaseCommand(config), config.target, sysrootFlag, archFlags, stdCppm, stdPcm.string())); - } else { - return ""; - } + // Keyed on the command and the module source's bytes, never on + // mtimes: an installed std.cppm can be dated at the epoch (OSTree), + // and a libc++ update under the same clang changes the bytes but + // not the path. + std::string command = std::format("{} --target={} -std=c++26{}{} -O3 -Wno-reserved-identifier -Wno-reserved-module-identifier --precompile {} -o {}", GetBaseCommand(config), config.target, sysrootFlag, archFlags, stdCppm, stdPcm.string()); + std::string stamp = HashStamp(command + '\0' + FileBytes(stdCppm)); + if (fs::exists(stdPcm) && ReadStamp(StampPath(stdPcm)) == stamp) return ""; + std::string result = RunBuildCommand(command); + if (result.empty()) WriteStamp(StampPath(stdPcm), stamp); + return result; } } @@ -721,12 +752,17 @@ namespace { "Crafter.Build", }; - void EnsureCrafterBuildPcms(const fs::path& sourceDir, const fs::path& cacheDir, const Configuration& host) { + // Returns the stamp of the module sources the cache was built from; it is + // one of project.so's inputs. + std::string EnsureCrafterBuildPcms(const fs::path& sourceDir, const fs::path& cacheDir, const Configuration& host) { CacheLock lock(cacheDir); // Re-derived under the lock: another builder may have refreshed the // cache from its own sources while we waited. - std::string stamp = CrafterBuildSourceStamp(sourceDir, CrafterBuildModules); - const bool upToDate = ReadCacheStamp(cacheDir) == stamp; + // The std PCM is an input to every module PCM: its stamp is folded in + // so a rebuilt std.pcm (a libc++ update under the same clang) rebuilds + // them too, instead of leaving PCMs that import a std.pcm that is gone. + std::string stamp = HashStamp(CrafterBuildSourceStamp(sourceDir, CrafterBuildModules) + '\0' + ReadStamp(StampPath(cacheDir / "std.pcm"))); + const bool upToDate = ReadStamp(CacheStampPath(cacheDir)) == stamp; for (std::string_view name : CrafterBuildModules) { fs::path cppmPath = sourceDir / std::format("{}.cppm", name); fs::path pcmPath = cacheDir / std::format("{}.pcm", name); @@ -751,7 +787,8 @@ namespace { // Last, so an interrupted rebuild leaves the stamp disagreeing with the // sources and the next run starts over rather than trusting a half-built // set of PCMs. - if (!upToDate) WriteCacheStamp(cacheDir, stamp); + if (!upToDate) WriteStamp(CacheStampPath(cacheDir), stamp); + return stamp; } } @@ -765,8 +802,6 @@ Configuration Crafter::LoadProject(const fs::path& projectFile, std::span/lib, so a toolchain outside /usr with the + // classic layout needs the search path as well as the run-time path. + fs::path lib; if (!libcxxDir.empty()) { - fs::path lib = libcxxDir / "lib"; - flags += std::format(" -L{} -Wl,-rpath,{}", lib.string(), lib.string()); + lib = libcxxDir / "lib"; } else if (!prefix.empty() && prefix != "/usr") { - flags += std::format(" -Wl,-rpath,{}", (prefix / "lib").string()); + lib = prefix / "lib"; } - return flags; + if (lib.empty()) return {}; + return std::format(" -L{} -Wl,-rpath,{}", lib.string(), lib.string()); } std::string Toolchain::Key() const { diff --git a/interfaces/Crafter.Build-Toolchain.cppm b/interfaces/Crafter.Build-Toolchain.cppm index b0fd2dc..853b2b6 100644 --- a/interfaces/Crafter.Build-Toolchain.cppm +++ b/interfaces/Crafter.Build-Toolchain.cppm @@ -44,10 +44,12 @@ export namespace Crafter { // Flags for compiling against libcxxDir; empty without one. std::string LibcxxCompileFlags() const; - // Flags for linking against the libc++ the compiler will use: an - // rpath into the prefix when that is not the system one (a bundled - // toolchain's libc++.so.1 is not on the loader's default path), plus - // -L and an rpath for an explicit libcxxDir. + // Flags for linking against the libc++ the compiler will use: a -L + // and an rpath for /lib when the prefix is not the system one + // (clang searches only its per-target runtime directory and the + // system paths, and a bundled toolchain's libc++.so.1 is on neither + // the linker's nor the loader's default path), or for an explicit + // libcxxDir. std::string LibcxxLinkFlags() const; // Names this toolchain for cache directory names: the clang version // followed by a hash of what decides which headers and libraries a diff --git a/packaging/flatpak/catcrafts.flatpakrepo b/packaging/flatpak/catcrafts.flatpakrepo new file mode 100644 index 0000000..8792ccd --- /dev/null +++ b/packaging/flatpak/catcrafts.flatpakrepo @@ -0,0 +1,8 @@ +[Flatpak Repo] +Title=Catcrafts +Url=https://flatpak.catcrafts.net/repo/ +Homepage=https://forgejo.catcrafts.net/Catcrafts +Comment=Catcrafts tools and applications +Description=crafter-build and other Catcrafts software, built and signed by Catcrafts CI +DefaultBranch=stable +GPGKey=GPG_KEY_BASE64_PLACEHOLDER diff --git a/packaging/flatpak/net.catcrafts.CrafterBuild.flatpakref b/packaging/flatpak/net.catcrafts.CrafterBuild.flatpakref new file mode 100644 index 0000000..16359ae --- /dev/null +++ b/packaging/flatpak/net.catcrafts.CrafterBuild.flatpakref @@ -0,0 +1,13 @@ +[Flatpak Ref] +Title=Crafter Build +Name=net.catcrafts.CrafterBuild +Branch=stable +Url=https://flatpak.catcrafts.net/repo/ +SuggestRemoteName=catcrafts +Homepage=https://forgejo.catcrafts.net/Catcrafts/Crafter.Build +Comment=C++26 modules build system +Description=crafter-build with its own LLVM toolchain and WASI sysroot +Icon= +RuntimeRepo=https://dl.flathub.org/repo/flathub.flatpakrepo +IsRuntime=false +GPGKey=GPG_KEY_BASE64_PLACEHOLDER diff --git a/packaging/flatpak/net.catcrafts.CrafterBuild.metainfo.xml b/packaging/flatpak/net.catcrafts.CrafterBuild.metainfo.xml new file mode 100644 index 0000000..7108966 --- /dev/null +++ b/packaging/flatpak/net.catcrafts.CrafterBuild.metainfo.xml @@ -0,0 +1,34 @@ + + + + net.catcrafts.CrafterBuild + Crafter Build + C++26 modules build system + CC0-1.0 + LGPL-3.0-only + + Catcrafts + + +

+ A build system for C++26 modules. Project descriptions are C++: a + project.cpp constructs a Configuration and returns it, and crafter-build + compiles, loads and executes it. The Flatpak carries its own LLVM, so + every machine builds with the same clang, lld, libc++ and WASI sysroot. +

+ + https://forgejo.catcrafts.net/Catcrafts/Crafter.Build + https://forgejo.catcrafts.net/Catcrafts/Crafter.Build/issues + https://forgejo.catcrafts.net/Catcrafts/Crafter.Build + + crafter-build + + + + + +

First release distributed as a Flatpak, with a bundled LLVM 22 toolchain and WASI sysroot. Windows host support is removed; crafter-build gains --version and resolves its compiler from one place.

+ + + + diff --git a/packaging/flatpak/net.catcrafts.CrafterBuild.yaml b/packaging/flatpak/net.catcrafts.CrafterBuild.yaml new file mode 100644 index 0000000..7226ad1 --- /dev/null +++ b/packaging/flatpak/net.catcrafts.CrafterBuild.yaml @@ -0,0 +1,205 @@ +# SPDX-License-Identifier: LGPL-3.0-only +# SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts® +# +# crafter-build as a Flatpak. The runtime is the Freedesktop *Sdk*, not the +# Platform: a build tool needs glibc's headers and crt objects, gcc's +# libgcc_s, cmake, ninja, make, git and python at run time. Everything +# compiler-shaped is ours and lives under /app: clang, lld, libclang, libc++, +# compiler-rt (native and wasm32) and the WASI sysroot, all built from one +# LLVM source tree so the modules crafter-build builds and the compiler that +# consumes them can never drift apart. +app-id: net.catcrafts.CrafterBuild +runtime: org.freedesktop.Sdk +runtime-version: '26.08' +sdk: org.freedesktop.Sdk +command: crafter-build +finish-args: + # git dependencies, and the local web server `-r` starts for browser builds + - --share=network + - --socket=ssh-auth + # projects live anywhere; user sysroots too + - --filesystem=host + # so `-r` can launch what it built, games included + - --share=ipc + - --socket=wayland + - --socket=fallback-x11 + - --socket=pulseaudio + - --device=all + # gdb and friends from the Sdk + - --allow=devel + - --env=PATH=/app/bin:/usr/bin +build-options: + env: + CRAFTER_BUILD_LLVM_DIR: /app + # The shipped binary runs on any x86_64 from the last decade; user + # projects still build with -march=native by default. + CRAFTER_BUILD_MARCH: x86-64-v2 + CRAFTER_BUILD_MTUNE: generic + # CI replaces this with the tag; a local build is a development build. + CRAFTER_BUILD_VERSION: 0.0.0-dev +cleanup: + - /share/man + - /share/doc +modules: + - name: llvm + buildsystem: cmake-ninja + builddir: true + subdir: llvm + build-options: + # flatpak-builder's default CFLAGS carry -g. For all of LLVM that is a + # 30 GB build tree and a multi-gigabyte .Debug extension, so build + # without debug info, strip, and skip the split. The hardening flags + # from the defaults stay. + no-debuginfo: true + strip: true + cflags: -O2 -pipe -fexceptions -fasynchronous-unwind-tables -fstack-protector-strong -fstack-clash-protection -fcf-protection + cflags-override: true + cxxflags: -O2 -pipe -fexceptions -fasynchronous-unwind-tables -fstack-protector-strong -fstack-clash-protection -fcf-protection + cxxflags-override: true + config-opts: + - -DCMAKE_BUILD_TYPE=Release + - -DLLVM_ENABLE_ASSERTIONS=OFF + - -DLLVM_ENABLE_PROJECTS=clang;lld + - -DLLVM_ENABLE_RUNTIMES=compiler-rt;libcxx;libcxxabi;libunwind + - -DLLVM_TARGETS_TO_BUILD=X86;AArch64;WebAssembly + # Classic layout: libc++ in lib/ and include/c++/v1, not under a + # per-target subdirectory, the same shape as a distro install. + # crafter-build adds -L and an rpath for /lib itself, since + # clang searches only its per-target directory and the system paths. + - -DLLVM_ENABLE_PER_TARGET_RUNTIME_DIR=OFF + # One shared LLVM and one shared clang library instead of a statically + # linked copy per tool: a fraction of the install size. + - -DLLVM_BUILD_LLVM_DYLIB=ON + - -DLLVM_LINK_LLVM_DYLIB=ON + - -DCLANG_LINK_CLANG_DYLIB=ON + - -DCLANG_DEFAULT_CXX_STDLIB=libc++ + - -DCLANG_DEFAULT_LINKER=lld + - -DLIBCXX_INSTALL_MODULES=ON + - -DLLVM_INCLUDE_TESTS=OFF + - -DLLVM_INCLUDE_EXAMPLES=OFF + - -DLLVM_INCLUDE_DOCS=OFF + - -DLLVM_INCLUDE_BENCHMARKS=OFF + - -DLLVM_INSTALL_UTILS=OFF + - -DCLANG_INCLUDE_TESTS=OFF + - -DCOMPILER_RT_INCLUDE_TESTS=OFF + # The Sdk links with binutils' ld; two libLLVM-sized links at a time + # fit the CX53's memory with room to spare. + - -DLLVM_PARALLEL_LINK_JOBS=2 + cleanup: + - /include/llvm + - /include/llvm-c + - /include/clang + - /include/clang-tidy + - /include/lld + - /lib/cmake + - /lib/*.a + - /share/opt-viewer + - /share/scan-build + - /share/scan-view + - /bin/clang-check + - /bin/clang-extdef-mapping + - /bin/clang-import-test + - /bin/clang-linker-wrapper + - /bin/clang-nvlink-wrapper + - /bin/clang-offload-bundler + - /bin/clang-offload-packager + - /bin/clang-refactor + - /bin/clang-rename + - /bin/clang-repl + - /bin/clang-sycl-linker + - /bin/diagtool + - /bin/git-clang-format + - /bin/hmaptool + - /bin/llvm-exegesis + - /bin/scan-build* + - /bin/scan-view + sources: + - type: archive + url: https://github.com/llvm/llvm-project/releases/download/llvmorg-22.1.8/llvm-project-22.1.8.src.tar.xz + sha256: 922f1817a0df7b1489272d18134ee0087a8b068828f87ac63b9861b1a9965888 + + - name: wasi-sysroot + # wasi-sdk's sysroot-only mode, driven by the clang built above: + # wasi-libc, compiler-rt builtins (installed into our clang's resource + # dir, where the wasm32 driver looks for them) and libc++ for wasip1 and + # wasip1-threads. The llvm-project tree is the same tarball as above so + # the wasm libc++ matches the native one. + buildsystem: cmake-ninja + builddir: true + build-options: + no-debuginfo: true + # flatpak-builder's default CFLAGS/LDFLAGS are x86-64 hardening flags + # (-fcf-protection, -z relro) that the nested wasm32 builds inherit + # through the environment and clang rejects for that target. + cflags: "" + cflags-override: true + cxxflags: "" + cxxflags-override: true + ldflags: "" + ldflags-override: true + strip: true + config-opts: + - -DCMAKE_TOOLCHAIN_FILE=/run/build/wasi-sysroot/wasi-host-toolchain.cmake + - -DWASI_SDK_BUILD_TOOLCHAIN=OFF + - -DWASI_SDK_INSTALL_TO_CLANG_RESOURCE_DIR=ON + - -DWASI_SDK_TARGETS=wasm32-wasip1;wasm32-wasip1-threads + - -DWASI_SDK_INCLUDE_TESTS=OFF + - -DCMAKE_INSTALL_PREFIX=/app + post-install: + # wasi-sdk installs into /share/wasi-sysroot; crafter-build + # looks there (Crafter.Build:Toolchain), so nothing to move. + - test -f /app/share/wasi-sysroot/share/libc++/v1/std.cppm + sources: + - type: git + url: https://github.com/WebAssembly/wasi-sdk.git + tag: wasi-sdk-34 + disable-submodules: true + - type: git + url: https://github.com/WebAssembly/wasi-libc.git + commit: 2e6fb9d8ee0cdf9e431fbcabe8af3115de000a13 + dest: src/wasi-libc + - type: archive + url: https://github.com/llvm/llvm-project/releases/download/llvmorg-22.1.8/llvm-project-22.1.8.src.tar.xz + sha256: 922f1817a0df7b1489272d18134ee0087a8b068828f87ac63b9861b1a9965888 + dest: src/llvm-project + - type: file + path: wasi-host-toolchain.cmake + + - name: wasmtime + # The runner for wasm32-wasi tests and `-r` on a non-browser wasm build. + buildsystem: simple + build-commands: + - install -Dm755 wasmtime /app/bin/wasmtime + sources: + - type: archive + url: https://github.com/bytecodealliance/wasmtime/releases/download/v49.0.2/wasmtime-v49.0.2-x86_64-linux.tar.xz + sha256: a4d6e9e3a5a60f527cf7793d674c48930c80c2e8977995b8a275cad3254b9322 + + - name: crafter-build + buildsystem: simple + build-options: + # The self-host build fetches Crafter.Math, Crafter.Asset, + # Crafter.LintRules and glslang from git; they are unpinned on purpose + # (sibling repos tracking their default branch, see AGENTS.md). + build-args: + - --share=network + build-commands: + # Bootstrap by hand, then let the result build itself: only the + # self-hosted binary has asset support and ThinLTO. + - ./build.sh + - ./bin/crafter-build + - CRAFTER_BUILD_HOME=$PWD/share/crafter-build ./bin/crafter-build test + - install -Dm755 bin/crafter.build-exe-*/crafter-build /app/bin/crafter-build + - install -Dm644 bin/crafter.build-lib-*/libcrafter-build.a /app/lib/libcrafter-build.a + - mkdir -p /app/share && cp -r share/crafter-build /app/share/ + - install -Dm644 LICENSE LICENSE.GPL -t /app/share/licenses/net.catcrafts.CrafterBuild/ + - install -Dm644 packaging/flatpak/net.catcrafts.CrafterBuild.metainfo.xml -t /app/share/metainfo/ + sources: + - type: dir + path: ../.. + skip: + - .git + - bin + - build + - lib/libcrafter-build.a + - share/crafter-build diff --git a/packaging/flatpak/wasi-host-toolchain.cmake b/packaging/flatpak/wasi-host-toolchain.cmake new file mode 100644 index 0000000..662d822 --- /dev/null +++ b/packaging/flatpak/wasi-host-toolchain.cmake @@ -0,0 +1,23 @@ +# SPDX-License-Identifier: LGPL-3.0-only +# SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts® +# +# Points wasi-sdk's sysroot build at the clang the Flatpak built, instead of +# the one wasi-sdk would build itself. Mirrors the wasi-sdk.cmake toolchain +# file an installed wasi-sdk ships, with our prefix. +set(WASI_SDK_PREFIX /app) +set(CMAKE_SYSTEM_NAME WASI) +set(CMAKE_SYSTEM_VERSION 1) +set(CMAKE_SYSTEM_PROCESSOR wasm32) +set(triple wasm32-wasip1) +set(CMAKE_C_COMPILER ${WASI_SDK_PREFIX}/bin/clang) +set(CMAKE_CXX_COMPILER ${WASI_SDK_PREFIX}/bin/clang++) +set(CMAKE_ASM_COMPILER ${WASI_SDK_PREFIX}/bin/clang) +set(CMAKE_AR ${WASI_SDK_PREFIX}/bin/llvm-ar) +set(CMAKE_RANLIB ${WASI_SDK_PREFIX}/bin/llvm-ranlib) +set(CMAKE_NM ${WASI_SDK_PREFIX}/bin/llvm-nm) +set(CMAKE_C_COMPILER_TARGET ${triple}) +set(CMAKE_CXX_COMPILER_TARGET ${triple}) +set(CMAKE_ASM_COMPILER_TARGET ${triple}) +# The sysroot does not exist yet when cmake probes the compiler. +set(CMAKE_C_COMPILER_WORKS ON) +set(CMAKE_CXX_COMPILER_WORKS ON) diff --git a/project.cpp b/project.cpp index c414bce..d3364d3 100644 --- a/project.cpp +++ b/project.cpp @@ -132,6 +132,7 @@ extern "C" Configuration CrafterBuildProject(std::span a cfg.AddTest("VersionFlag").Dependencies({ CrafterBuildLib.get() }); cfg.AddTest("WasiBrowserRuntime").Dependencies({ CrafterBuildLib.get() }); cfg.AddTest("WasmVariants").Dependencies({ CrafterBuildLib.get() }); + cfg.AddTest("WasmHelloRuns").Dependencies({ CrafterBuildLib.get() }); cfg.AddTest("RunSingleTestExit").Dependencies({ CrafterBuildLib.get() }); // LoadProject dlopens the synthesized project.so, which references // Crafter:: symbols (HostTarget, Configuration ctors) that have to be @@ -142,6 +143,8 @@ extern "C" Configuration CrafterBuildProject(std::span a // Same LoadProject wiring as ConcurrentCacheRace above. cfg.AddTest("HostCacheSourceStamp").Dependencies({ CrafterBuildLib.get() }) .LinkFlag("-Wl,--export-dynamic").LinkFlag("-ldl"); + cfg.AddTest("HostCacheToolchainKey").Dependencies({ CrafterBuildLib.get() }) + .LinkFlag("-Wl,--export-dynamic").LinkFlag("-ldl"); cfg.AddTest("ProjectDependency").Dependencies({ CrafterBuildLib.get() }) .LinkFlag("-Wl,--export-dynamic").LinkFlag("-ldl"); cfg.AddTest("ConcurrentDependencyReset").Dependencies({ CrafterBuildLib.get() }); diff --git a/tests/HostCacheToolchainKey/main.cpp b/tests/HostCacheToolchainKey/main.cpp new file mode 100644 index 0000000..58c35f8 --- /dev/null +++ b/tests/HostCacheToolchainKey/main.cpp @@ -0,0 +1,154 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts® + +#include +import std; +import Crafter.Build; +namespace fs = std::filesystem; +using namespace Crafter; + +// Freshness of the std PCM and of project.so used to be an mtime comparison +// against files that come with the install: std.cppm and the crafter-build +// executable. A Flatpak (any OSTree deployment) gives every installed file +// mtime 0, so those inputs always looked older than the cache: a clang bump +// would never rebuild std.pcm, a crafter-build update would never recompile +// project.so, and clang crashes on the stale PCM instead of saying so. +// +// Both are now keyed on content stamps (the compile command, the std module +// source, the crafter-build version, the toolchain key and the module source +// stamp) and the cache directory carries the toolchain key in its name. This +// test drives the shapes an mtime rule gets wrong: a std.pcm dated before its +// source must still be reused, a warm load must leave project.so alone, and a +// stamp that disagrees must rebuild even when the artifact is dated after +// every input. +namespace { + std::int32_t Failures = 0; + + void Check(bool cond, std::string_view msg) { + if (!cond) { + std::println(std::cerr, "FAIL: {}", msg); + ++Failures; + } + } + + fs::path FindModuleCacheDir(const fs::path& cacheRoot) { + for (const fs::directory_entry& entry : fs::directory_iterator(cacheRoot / "crafter.build")) { + if (entry.is_directory()) return entry.path(); + } + return {}; + } + + bool Load(const fs::path& projectFile, std::string_view what) { + std::array noArgs = {}; + try { + (void)LoadProject(projectFile, noArgs); + return true; + } catch (const std::exception& e) { + std::println(std::cerr, "FAIL: LoadProject ({}) threw: {}", what, e.what()); + ++Failures; + return false; + } + } + + fs::path StampOf(const fs::path& artifact) { + fs::path stamp = artifact; + stamp += ".stamp"; + return stamp; + } +} + +int main() { + fs::path installedHome = GetCrafterBuildHome(); + if (!fs::exists(installedHome / "Crafter.Build.cppm")) { + std::println(std::cerr, "SKIP: no module sources at {}", installedHome.string()); + return 77; + } + + fs::path scratch = fs::temp_directory_path() / "crafter-build-host-cache-toolchain-key"; + std::error_code ec; + fs::remove_all(scratch, ec); + fs::create_directories(scratch); + + // A private copy of the module sources and a cold private cache, as in + // HostCacheSourceStamp. + fs::path home = scratch / "share" / "crafter-build"; + fs::create_directories(home.parent_path()); + fs::copy(installedHome, home, fs::copy_options::recursive); + setenv("CRAFTER_BUILD_HOME", home.string().c_str(), 1); + fs::path cacheRoot = scratch / "cache"; + fs::create_directories(cacheRoot); + setenv("XDG_CACHE_HOME", cacheRoot.string().c_str(), 1); + + fs::path projectDir = scratch / "project"; + fs::create_directories(projectDir); + fs::path projectFile = projectDir / "project.cpp"; + std::ofstream(projectFile) << + "import std;\n" + "import Crafter.Build;\n" + "using namespace Crafter;\n" + "extern \"C\" Configuration CrafterBuildProject(std::span) {\n" + " Configuration cfg;\n" + " cfg.path = \"./\";\n" + " cfg.name = \"keyed\";\n" + " cfg.outputName = \"keyed\";\n" + " cfg.type = ConfigurationType::Executable;\n" + " return cfg;\n" + "}\n"; + + if (!Load(projectFile, "cold")) return 1; + + const fs::path cacheDir = FindModuleCacheDir(cacheRoot); + Check(!cacheDir.empty(), "a module cache directory was created"); + if (cacheDir.empty()) return 1; + Check(cacheDir.filename().string().contains(GetToolchain().Key()), std::format("the cache directory is named for the toolchain (got '{}', key '{}')", cacheDir.filename().string(), GetToolchain().Key())); + + const fs::path stdPcm = cacheDir / "std.pcm"; + const fs::path projectSo = projectDir / "build" / "project.so"; + Check(fs::exists(stdPcm) && fs::exists(StampOf(stdPcm)), "std.pcm and its stamp exist after a cold load"); + Check(fs::exists(projectSo) && fs::exists(StampOf(projectSo)), "project.so and its stamp exist after a cold load"); + if (!fs::exists(StampOf(stdPcm)) || !fs::exists(StampOf(projectSo))) return 1; + + // The OSTree shape, inverted: std.pcm dated before its source. A stamp + // rule reuses it; an mtime rule would rebuild it on every run. project.so + // keeps two mtime inputs that live next to it (project.cpp and dependency + // artifacts) but the executable is no longer one of them, so a warm load + // with nothing changed leaves it alone. + const fs::file_time_type epoch{}; + fs::last_write_time(stdPcm, epoch); + const fs::file_time_type afterCold = fs::last_write_time(projectSo); + if (!Load(projectFile, "std.pcm dated at the epoch")) return 1; + Check(fs::last_write_time(stdPcm) == epoch, "std.pcm dated before its source is still reused"); + Check(fs::last_write_time(projectSo) == afterCold, "project.so is left alone when nothing changed"); + + // The other half: artifacts dated after every input, with a stamp that + // disagrees. An mtime rule keeps them; the stamp rule rebuilds. + const fs::file_time_type future = fs::file_time_type::clock::now() + std::chrono::hours(24); + std::ofstream(StampOf(stdPcm)) << "stale"; + fs::last_write_time(stdPcm, future); + if (!Load(projectFile, "stale std.pcm stamp")) return 1; + Check(fs::last_write_time(stdPcm) != future, "a disagreeing stamp rebuilds std.pcm even when it is dated after its source"); + { + std::ifstream in(StampOf(stdPcm)); + std::string stamp; + std::getline(in, stamp); + Check(stamp != "stale" && !stamp.empty(), "the rebuilt std.pcm gets its stamp back"); + } + + std::ofstream(StampOf(projectSo)) << "stale"; + fs::last_write_time(projectSo, future); + if (!Load(projectFile, "stale project.so stamp")) return 1; + Check(fs::last_write_time(projectSo) != future, "a disagreeing stamp recompiles project.so even when it is dated after its inputs"); + + // The one mtime input that legitimately remains: the project file itself. + const fs::file_time_type beforeEdit = fs::last_write_time(projectSo); + { + std::ofstream append(projectFile, std::ios::app); + append << "// edited\n"; + } + fs::last_write_time(projectFile, fs::file_time_type::clock::now() + std::chrono::seconds(2)); + if (!Load(projectFile, "edited project.cpp")) return 1; + Check(fs::last_write_time(projectSo) != beforeEdit, "an edited project.cpp recompiles project.so"); + + fs::remove_all(scratch, ec); + return Failures == 0 ? 0 : 1; +} diff --git a/tests/ToolchainResolve/main.cpp b/tests/ToolchainResolve/main.cpp index aedb261..51d8af6 100644 --- a/tests/ToolchainResolve/main.cpp +++ b/tests/ToolchainResolve/main.cpp @@ -73,7 +73,7 @@ int main() { Check(tc.stdCppm == full / "share/libc++/v1/std.cppm", "the prefix's std module source is used"); Check(tc.wasiSysroot == full / "share/wasi-sysroot", "the prefix's wasi sysroot is used"); Check(tc.LibcxxCompileFlags().empty(), "the compiler finds its own libc++ headers without flags"); - Check(Contains(tc.LibcxxLinkFlags(), std::format("-Wl,-rpath,{}", (full / "lib").string())), "binaries get an rpath into the prefix's lib dir"); + Check(Contains(tc.LibcxxLinkFlags(), std::format("-L{}", (full / "lib").string())) && Contains(tc.LibcxxLinkFlags(), std::format("-Wl,-rpath,{}", (full / "lib").string())), "binaries get -L and an rpath for the prefix's lib dir"); } { ToolchainInputs in; @@ -95,7 +95,7 @@ int main() { Check(tc.llvmIncludeDir.empty(), "no clang-c headers in the prefix: nothing added"); Check(tc.stdCppm == "/usr/share/libc++/v1/std.cppm", "no std module in the prefix: the system copy"); Check(tc.wasiSysroot == "/usr/share/wasi-sysroot", "no wasi sysroot in the prefix: the system one"); - Check(Contains(tc.LibcxxLinkFlags(), std::format("-Wl,-rpath,{}", (bare / "lib").string())), "the rpath follows the explicit prefix"); + Check(Contains(tc.LibcxxLinkFlags(), std::format("-L{}", (bare / "lib").string())) && Contains(tc.LibcxxLinkFlags(), std::format("-Wl,-rpath,{}", (bare / "lib").string())), "the link flags follow the explicit prefix"); } { ToolchainInputs in; diff --git a/tests/WasmHelloRuns/fixture/main.cpp b/tests/WasmHelloRuns/fixture/main.cpp new file mode 100644 index 0000000..59fb23d --- /dev/null +++ b/tests/WasmHelloRuns/fixture/main.cpp @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts® + +import std; + +int main() { + std::println("hello from wasm"); + return 0; +} diff --git a/tests/WasmHelloRuns/main.cpp b/tests/WasmHelloRuns/main.cpp new file mode 100644 index 0000000..58a910f --- /dev/null +++ b/tests/WasmHelloRuns/main.cpp @@ -0,0 +1,57 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts® + +import std; +import Crafter.Build; +namespace fs = std::filesystem; +using namespace Crafter; + +// A wasm build end to end: the fixture is compiled for wasm32-wasip1 against +// the resolved toolchain's WASI sysroot and the result is run under +// wasmtime. The other wasm tests only inspect generated files; this is the +// one that proves the sysroot, the std module for wasm and wasm-ld all work +// together. It skips when the sysroot or wasmtime is missing, so a +// development box without them is not a failure. Inside the Flatpak both are +// bundled, and there the skip would be the failure to look for. +int main() { + const Toolchain& tc = GetToolchain(); + if (!fs::exists(tc.wasiSysroot / "include")) { + std::println(std::cerr, "SKIP: no WASI sysroot at {}", tc.wasiSysroot.string()); + return 77; + } + if (std::system("which wasmtime > /dev/null 2>&1") != 0) { + std::println(std::cerr, "SKIP: wasmtime not on PATH"); + return 77; + } + + Configuration cfg; + cfg.path = fs::current_path() / "tests" / "WasmHelloRuns" / "fixture"; + cfg.name = "wasm-hello"; + cfg.outputName = "wasm-hello"; + cfg.target = "wasm32-wasip1"; + cfg.type = ConfigurationType::Executable; + std::array ifaces = {}; + std::array impls = { "main" }; + cfg.GetInterfacesAndImplementations(ifaces, impls); + + std::unordered_map> depResults; + std::mutex depMutex; + BuildResult r = Build(cfg, depResults, depMutex); + if (!r.result.empty()) { + std::println(std::cerr, "build failed: {}", r.result); + return 1; + } + + const fs::path wasm = ExpectedOutput(cfg); + if (!fs::exists(wasm)) { + std::println(std::cerr, "no .wasm produced at {}", wasm.string()); + return 1; + } + + const auto run = RunCommandWithTimeout(std::format("wasmtime {}", wasm.string()), std::chrono::seconds(60)); + if (run.exitCode != 0 || !run.output.contains("hello from wasm")) { + std::println(std::cerr, "wasmtime run failed (exit {}): {}", run.exitCode, run.output); + return 1; + } + return 0; +}