From dc239f42570e6a783b1082192aa5203e355bee41 Mon Sep 17 00:00:00 2001 From: Jorijn van der Graaf Date: Tue, 1 Sep 2026 23:49:00 +0200 Subject: [PATCH] musl host support: bootstrap and run on Alpine (x86_64-alpine-linux-musl) The Linux host code paths were gated on the x86_64_pc_linux_gnu target define alone, and LoadProject hardcoded --target=x86_64-pc-linux-gnu when compiling project.cpp and the module PCMs. Both break on musl hosts: the glibc release launcher cannot run there at all (no glibc loader; gcompat lacks libgcc_s and the __isoc23_* symbols), and a musl-built launcher would still compile project.cpp for the wrong libc. - The host gates test clang's __linux__ instead of one triple's define: they mean "Linux host", whatever the libc (or, later, the CPU). - LoadProject derives the host triple from HostTarget() (clang's -print-target-triple), which is also more correct on glibc distros whose triple isn't x86_64-pc-linux-gnu. - project.cpp / the cmake libc++ flags treat any *-linux-gnu or *-linux-musl target as Linux. - build.sh targets the host toolchain's triple (CRAFTER_BUILD_TARGET overrides) and derives the per-target define from it. On Arch this is byte-for-byte the previous behaviour. - CI: a release-musl job bootstraps on alpine:edge, runs the tests, and attaches crafter-build-linux-x86_64-musl-v2.tar.gz to the rolling latest release (via the API, so the glibc assets survive). Dynamic against musl libc++: the launcher dlopens project.so, so it cannot be fully static. First consumer: imsd's package CI, which cross-compiles for aarch64 from an Alpine container. Co-Authored-By: Claude Fable 5 --- .forgejo/workflows/ci.yaml | 73 ++++++++++++++++++++++ README.md | 2 + build.sh | 19 +++++- implementations/Crafter.Build-Clang.cpp | 10 +-- implementations/Crafter.Build-External.cpp | 4 +- implementations/Crafter.Build-Platform.cpp | 27 +++++--- project.cpp | 18 +++++- 7 files changed, 131 insertions(+), 22 deletions(-) diff --git a/.forgejo/workflows/ci.yaml b/.forgejo/workflows/ci.yaml index c0964e8..2feeb1d 100644 --- a/.forgejo/workflows/ci.yaml +++ b/.forgejo/workflows/ci.yaml @@ -186,3 +186,76 @@ jobs: override: true release-dir: dist token: ${{ secrets.GITHUB_TOKEN }} + + # musl build for Alpine / postmarketOS hosts: the glibc launcher above cannot + # run there (musl has no glibc loader and gcompat lacks libgcc_s + the + # __isoc23_* symbols). Bootstrap natively on alpine:edge and ship the + # bootstrap output — the same thing the Arch PKGBUILD installs. Dynamic + # against musl libc++ (crafter-build dlopens project.so, so it cannot be + # fully static); consumers need `apk add clang lld libc++-dev llvm-runtimes`. + release-musl: + needs: build-test-release + if: (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.ref == 'refs/heads/master' + runs-on: arch-latest + container: + image: alpine:edge + env: + CRAFTER_BUILD_MARCH: x86-64-v2 + CRAFTER_BUILD_MTUNE: generic + steps: + - name: Install build dependencies + # nodejs: actions/checkout is a Node action. build-base: gcc's + # libgcc_s + crt objects, which clang's musl driver links against, plus + # make/binutils for the glslang cmake build (python3: its generators). + # clang-dev + llvm-dev: the libclang header Crafter.Build-Lint includes + # lives under llvm-config --includedir here, not /usr/include. llvm: + # llvm-ar for the LTO static archives the self-host/test builds write. + run: | + apk add -q nodejs git curl jq tar clang clang-dev lld libc++-dev \ + llvm-libunwind-dev llvm-runtimes llvm llvm-dev cmake build-base python3 + git config --global --add safe.directory '*' + + - name: Checkout + uses: actions/checkout@v4 + + - name: Bootstrap (build.sh) + run: ./build.sh + + - name: Run tests + run: CRAFTER_BUILD_HOME=$PWD/share/crafter-build ./bin/crafter-build test + + - name: Package artifact + run: | + set -eux + file bin/crafter-build || true + mkdir -p dist + stage=$(mktemp -d) + mkdir -p "$stage/bin" "$stage/lib" + cp bin/crafter-build "$stage/bin/" + cp lib/libcrafter-build.a "$stage/lib/" + cp -r share "$stage/" + tar czf dist/crafter-build-linux-x86_64-musl-v2.tar.gz -C "$stage" . + ls -la dist/ + + - name: Upload musl artifact + uses: actions/upload-artifact@v3 + with: { name: crafter-build-linux-x86_64-musl-v2, path: dist/crafter-build-linux-x86_64-musl-v2.tar.gz, if-no-files-found: error } + + # Add the asset to the rolling 'latest' release the first job just + # (re)published, via the API rather than forgejo-release: that action's + # override mode would drop the other job's assets. + - name: Attach to rolling 'latest' release + env: + TOKEN: ${{ secrets.GITHUB_TOKEN }} + API: ${{ github.server_url }}/api/v1/repos/${{ github.repository }} + run: | + set -eu + asset=crafter-build-linux-x86_64-musl-v2.tar.gz + rel=$(curl -fsS -H "Authorization: token $TOKEN" "$API/releases/tags/latest") + id=$(printf '%s' "$rel" | jq -r .id) + for old in $(printf '%s' "$rel" | jq -r --arg n "$asset" '.assets[] | select(.name==$n) | .id'); do + curl -fsS -X DELETE -H "Authorization: token $TOKEN" "$API/releases/$id/assets/$old" + done + curl -fsS -H "Authorization: token $TOKEN" -F "attachment=@dist/$asset" \ + "$API/releases/$id/assets?name=$asset" > /dev/null + echo "attached $asset to release $id" diff --git a/README.md b/README.md index 8a777ce..64dfa97 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,8 @@ Bootstrap requires `clang`, `cmake`, `git`, `lld`, and `libc++`. For distro-packaged installs, `crafter-build` finds its modules at `/share/crafter-build/` automatically — no env var required. +The bootstrap targets the host toolchain's own triple (`clang++ -print-target-triple`): `x86_64-pc-linux-gnu` on glibc distros, `x86_64-alpine-linux-musl` on Alpine/postmarketOS (`apk add clang clang-dev lld libc++-dev llvm-libunwind-dev llvm-runtimes llvm llvm-dev cmake build-base python3 git`). Override with `CRAFTER_BUILD_TARGET=`. CI ships both: `crafter-build-linux-x86_64-v{2,3,4}.tar.gz` (glibc) and `crafter-build-linux-x86_64-musl-v2.tar.gz` (musl, needs `libc++` at runtime — the launcher dlopens `project.so`, so it cannot be static). + To build the system as a distro package on Arch: ```bash diff --git a/build.sh b/build.sh index c579760..5ff6fb8 100755 --- a/build.sh +++ b/build.sh @@ -44,10 +44,23 @@ cd ../../ MARCH="${CRAFTER_BUILD_MARCH:-native}" MTUNE="${CRAFTER_BUILD_MTUNE:-native}" +# The bootstrap targets the host toolchain's own triple: x86_64-pc-linux-gnu +# on glibc distros, x86_64-alpine-linux-musl on Alpine. Override with +# CRAFTER_BUILD_TARGET. The per-target define (triple with '-' -> '_') is the +# same one the self-hosted build emits, so the host-gated code paths match. +TARGET="${CRAFTER_BUILD_TARGET:-$(clang++ -print-target-triple)}" +TARGET_DEFINE="CRAFTER_BUILD_CONFIGURATION_TARGET_$(printf '%s' "$TARGET" | tr - _)" -common_options="--target=x86_64-pc-linux-gnu -stdlib=libc++ -I./build/glslang -std=c++26 -O3 -march=$MARCH -mtune=$MTUNE -fprebuilt-module-path=./build -D CRAFTER_BUILD_CONFIGURATION_TARGET_x86_64_pc_linux_gnu -D CRAFTER_BUILD_CONFIGURATION_TYPE_EXECUTABLE -c" +# Crafter.Build-Lint includes libclang's clang-c/Index.h. glibc distros keep +# it in /usr/include; Alpine keeps LLVM under /usr/lib/llvmNN, so ask +# llvm-config and add the dir when it is not already on the default path +# (never add /usr/include itself: that breaks libc++'s #include_next). +LLVM_INCDIR="$(llvm-config --includedir 2>/dev/null || true)" +case "$LLVM_INCDIR" in ""|/usr/include) LLVM_INCFLAG="" ;; *) LLVM_INCFLAG="-I$LLVM_INCDIR" ;; esac -clang++ --target=x86_64-pc-linux-gnu -std=c++26 -stdlib=libc++ -O3 -march=$MARCH -mtune=$MTUNE -Wno-reserved-identifier -Wno-reserved-module-identifier --precompile /usr/share/libc++/v1/std.cppm -o ./build/std.pcm +common_options="--target=$TARGET -stdlib=libc++ -I./build/glslang $LLVM_INCFLAG -std=c++26 -O3 -march=$MARCH -mtune=$MTUNE -fprebuilt-module-path=./build -D $TARGET_DEFINE -D CRAFTER_BUILD_CONFIGURATION_TYPE_EXECUTABLE -c" + +clang++ --target=$TARGET -std=c++26 -stdlib=libc++ -O3 -march=$MARCH -mtune=$MTUNE -Wno-reserved-identifier -Wno-reserved-module-identifier --precompile /usr/share/libc++/v1/std.cppm -o ./build/std.pcm clang++ $common_options -fmodule-output interfaces/Crafter.Build-Shader.cppm -o ./build/Crafter.Build-Shader.o clang++ $common_options -fmodule-output interfaces/Crafter.Build-Platform.cppm -o ./build/Crafter.Build-Platform.o @@ -100,7 +113,7 @@ ar rcs ./lib/libcrafter-build.a \ ./build/Crafter.Build-Progress_impl.o \ ./build/Crafter.Build-Asset_impl.o -clang++ -std=c++26 -stdlib=libc++ -O3 -march=$MARCH -mtune=$MTUNE -fuse-ld=lld \ +clang++ --target=$TARGET -std=c++26 -stdlib=libc++ -O3 -march=$MARCH -mtune=$MTUNE -fuse-ld=lld \ -Wl,--export-dynamic \ -L./build \ ./build/Crafter.Build-Shader.o \ diff --git a/implementations/Crafter.Build-Clang.cpp b/implementations/Crafter.Build-Clang.cpp index cd929bc..840d29f 100644 --- a/implementations/Crafter.Build-Clang.cpp +++ b/implementations/Crafter.Build-Clang.cpp @@ -307,7 +307,7 @@ CompileCommand Crafter::GetCompileCommand(const Configuration& config) { } if(config.type == ConfigurationType::LibraryDynamic) { - #ifdef CRAFTER_BUILD_CONFIGURATION_TARGET_x86_64_pc_linux_gnu + #ifdef __linux__ out.command += " -fPIC -D CRAFTER_BUILD_CONFIGURATION_TYPE_SHARED_LIBRARY"; #endif #if defined(CRAFTER_BUILD_CONFIGURATION_TARGET_x86_64_pc_windows_msvc) || defined(CRAFTER_BUILD_CONFIGURATION_TARGET_x86_64_w64_mingw32) @@ -453,7 +453,7 @@ BuildResult Crafter::Build(Configuration& config, std::unordered_map #include #include -#ifdef CRAFTER_BUILD_CONFIGURATION_TARGET_x86_64_pc_linux_gnu +#ifdef __linux__ #include #include #include @@ -41,7 +41,7 @@ namespace { // descriptor is close-on-exec / non-inheritable so spawned clang processes // don't keep the lock alive past our own release. class CacheLock { -#ifdef CRAFTER_BUILD_CONFIGURATION_TARGET_x86_64_pc_linux_gnu +#ifdef __linux__ std::int32_t fd_ = -1; public: explicit CacheLock(const fs::path& cacheDir) { @@ -765,7 +765,7 @@ Configuration Crafter::LoadProject(const fs::path& projectFile, std::span buffer; @@ -796,7 +796,9 @@ CommandResult Crafter::RunCommandWithTimeout(std::string_view cmd, std::chrono:: std::array buffer; CommandResult result{}; - std::string wrapped = std::format("timeout --kill-after=2 {} {} 2>&1", timeout.count(), cmd); + // -k (not --kill-after=): the short form is what GNU coreutils and busybox + // (Alpine) have in common. + std::string wrapped = std::format("timeout -k 2 {} {} 2>&1", timeout.count(), cmd); FILE* pipe = popen(wrapped.c_str(), "r"); if (!pipe) throw std::runtime_error("popen() failed!"); @@ -968,12 +970,12 @@ namespace { continue; } std::string cmd = std::format( - "clang++ --target=x86_64-pc-linux-gnu -march=native -mtune=native " + "clang++ --target={} -march=native -mtune=native " "-std=c++26 -stdlib=libc++ -O3 " "-Wno-reserved-identifier -Wno-reserved-module-identifier " "-fprebuilt-module-path={} " "--precompile {} -o {}", - cacheDir.string(), cppmPath.string(), pcmPath.string()); + HostTarget(), cacheDir.string(), cppmPath.string(), pcmPath.string()); CommandResult r = Crafter::RunCommandChecked(cmd); if (r.exitCode != 0) { throw std::runtime_error(std::format("Failed to precompile {} (exit {}): {}", name, r.exitCode, r.output)); @@ -998,8 +1000,11 @@ Configuration Crafter::LoadProject(const fs::path& projectFile, std::span a CrafterBuildLib->dependencies = { math, asset }; CrafterBuildLib->defines.push_back({"CRAFTER_BUILD_HAS_ASSET", ""}); + // Crafter.Build-Lint includes libclang's clang-c/Index.h. glibc distros + // keep it in /usr/include; Alpine keeps LLVM under /usr/lib/llvmNN, so ask + // llvm-config and add the dir when it is not already on the default path + // (never /usr/include itself: that breaks libc++'s #include_next). + if (CrafterBuildLib->target.ends_with("-linux-gnu") || CrafterBuildLib->target.ends_with("-linux-musl")) { + auto llvmInc = RunCommandWithTimeout("llvm-config --includedir 2>/dev/null", std::chrono::seconds(10)); + std::string incDir = llvmInc.output; + while (!incDir.empty() && (incDir.back() == '\n' || incDir.back() == '\r')) incDir.pop_back(); + if (llvmInc.exitCode == 0 && !incDir.empty() && incDir != "/usr/include") { + CrafterBuildLib->compileFlags.push_back(std::format("-I{}", incDir)); + } + } { std::array interfaces = { "interfaces/Crafter.Build", @@ -87,7 +99,9 @@ extern "C" Configuration CrafterBuildProject(std::span a std::array implementations = { "implementations/main" }; cfg.GetInterfacesAndImplementations(interfaces, implementations); } - if (cfg.target == "x86_64-pc-linux-gnu") { + // Linux hosts, glibc or musl (Alpine): -ldl is a libc stub on musl, harmless. + const bool linuxTarget = cfg.target.ends_with("-linux-gnu") || cfg.target.ends_with("-linux-musl"); + if (linuxTarget) { cfg.linkFlags.push_back("-Wl,--export-dynamic"); cfg.linkFlags.push_back("-ldl"); } @@ -102,7 +116,7 @@ extern "C" Configuration CrafterBuildProject(std::span a // links each test exe against CrafterBuildLib built from the local // sources — so the code under test is whatever's in this checkout. // Mirrors how downstream consumers link their own libraries into tests. - if (cfg.target == "x86_64-pc-linux-gnu") { + if (linuxTarget) { cfg.AddTest("HelloWorld").Dependencies({ CrafterBuildLib.get() }); cfg.AddTest("StaticLib").Dependencies({ CrafterBuildLib.get() }); cfg.AddTest("ModuleInterface").Dependencies({ CrafterBuildLib.get() });