Crafter.Build/implementations/Crafter.Build-Toolchain.cpp
Jorijn van der Graaf 653dd0bb9b
Some checks failed
Flatpak / flatpak (push) Failing after 1m12s
fix: rebuild and relink on what changed, submodules at their revisions, and mingw found where it is
- Shaders record the files glslang opened (a depfile) and their settings
  (source path, stage, entry point, defines, include directories) in the
  build directory, and rebuild when an included file or a setting changes,
  not only the shader's own source.
- An executable relinks when its link command changed, not only when an
  object is newer: a link flag in project.cpp, or a dependency's library
  that moved (an ExternalDependency built into a new cache directory). The
  command is recorded beside the objects; its flags come from ordered sets,
  so the order the dependency threads finish in can't change it.
- An external's newest artifact counts its libDirs and versioned .so names.
- Submodules are checked out at the revisions the fetched commit records,
  from the URLs its .gitmodules gives, after every clone, switch, checkout
  and pull, so a pinned commit builds against its own submodules, and one
  moved by hand is put back; with no .gitmodules, git isn't asked.
- Each external clone's fetch and build hold a file lock: parallel builds
  pulled into one clone at once ("cannot lock ref").
- mingw:
  - -stdlib=libstdc++ always;
  - the std module's source is found through libstdc++.modules.json in the
    mingw toolchain's own library directories (the generic lib/ holds the
    host's Linux one), asking the compiler once a run;
  - a toolchain that isn't there is a build error saying where it looked,
    not std::terminate;
  - CRAFTER_MINGW_DIR names the prefix, the directory holding
    x86_64-w64-mingw32/ and lib/gcc/;
  - inside the Flatpak the host's /usr mingw is used.
- RunScript's exit-status check reads as the one range it is.
- Tests: ConcurrentExternalFetch, ExternalArtifactTime, GitSubmodules,
  LinkFlagsChange, MingwMissing, ShaderIncludeChange; ShaderCompile and
  ToolchainResolve extended.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-20 20:20:29 +02:00

241 lines
11 KiB
C++

// SPDX-License-Identifier: LGPL-3.0-only
// SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts®
module;
// Supplied on the command line by build.sh and project.cpp (from the
// CRAFTER_BUILD_VERSION environment variable or `git describe`); a build
// that passes nothing is a development build.
#ifndef CRAFTER_BUILD_VERSION
#define CRAFTER_BUILD_VERSION "0.0.0-dev"
#endif
module Crafter.Build:Toolchain_impl;
import std;
import :Toolchain;
import :Platform;
namespace fs = std::filesystem;
using namespace Crafter;
namespace {
constexpr std::string_view EnvVars[] = {
"CRAFTER_BUILD_LLVM_DIR",
"CRAFTER_BUILD_LIBCXX_DIR",
"CRAFTER_BUILD_LIBCLANG",
"CRAFTER_BUILD_WASI_SYSROOT",
"CRAFTER_MINGW_DIR",
};
std::string_view Env(const ToolchainInputs& in, std::string_view name) {
auto it = in.env.find(name);
return it == in.env.end() ? std::string_view{} : std::string_view(it->second);
}
// "Arch Linux clang version 22.1.8" / "clang version 22.1.8" -> "22.1.8".
std::string ShortVersion(std::string_view versionLine) {
constexpr std::string_view Marker = "version ";
std::size_t at = versionLine.find(Marker);
if (at == std::string_view::npos) return "unknown";
std::string_view rest = versionLine.substr(at + Marker.size());
std::size_t end = rest.find_first_of(" \t(");
return std::string(rest.substr(0, end));
}
std::string Trimmed(std::string s) {
while (!s.empty() && (s.back() == '\n' || s.back() == '\r' || s.back() == ' ')) s.pop_back();
return s;
}
}
std::string Toolchain::LibcxxCompileFlags() const {
if (libcxxDir.empty()) return {};
return std::format(" -nostdinc++ -isystem {}", (libcxxDir / "include" / "c++" / "v1").string());
}
std::string Toolchain::LibcxxLinkFlags() const {
// clang searches its per-target runtime directory and the system library
// paths for -lc++, not <prefix>/lib, so a toolchain outside /usr with the
// classic layout needs the search path as well as the run-time path.
fs::path lib;
if (!libcxxDir.empty()) {
lib = libcxxDir / "lib";
} else if (!prefix.empty() && prefix != "/usr") {
lib = prefix / "lib";
}
if (lib.empty()) return {};
return std::format(" -L{} -Wl,-rpath,{}", lib.string(), lib.string());
}
std::string Toolchain::Key() const {
std::string identity;
for (const std::string& part : {clangxx.string(), clangVersion, libcxxDir.string(), stdCppm.string()}) {
identity += part;
identity += '\0';
}
return std::format("{}-{:08x}", ShortVersion(clangVersion), static_cast<std::uint32_t>(std::hash<std::string>{}(identity)));
}
Toolchain Crafter::ResolveToolchain(const ToolchainInputs& in) {
Toolchain tc;
// 1. The compiler: an explicit prefix, else one installed beside
// crafter-build, else PATH. An explicit prefix without a clang++ is an
// error rather than a fallback, for the same reason CRAFTER_BUILD_LIBCLANG
// is exclusive: an override that silently yields to something else
// cannot be used to diagnose which compiler is in play.
if (std::string_view dir = Env(in, "CRAFTER_BUILD_LLVM_DIR"); !dir.empty()) {
tc.prefix = fs::path(dir);
if (!fs::exists(tc.prefix / "bin" / "clang++")) {
tc.error = std::format("clang++ not found at {} (CRAFTER_BUILD_LLVM_DIR)", (tc.prefix / "bin" / "clang++").string());
}
} else if (!in.crafterPrefix.empty() && fs::exists(in.crafterPrefix / "bin" / "clang++")) {
tc.prefix = in.crafterPrefix;
}
if (tc.prefix.empty()) {
tc.clangxx = "clang++";
tc.clang = "clang";
tc.llvmAr = "llvm-ar";
} else {
tc.clangxx = tc.prefix / "bin" / "clang++";
tc.clang = tc.prefix / "bin" / "clang";
tc.llvmAr = tc.prefix / "bin" / "llvm-ar";
if (tc.prefix != "/usr" && fs::exists(tc.prefix / "include" / "clang-c" / "Index.h")) {
tc.llvmIncludeDir = tc.prefix / "include";
}
}
// 2. libclang. The override is exclusive and unchecked on purpose: a
// broken path must fail loudly when dlopened, not be papered over.
if (std::string_view pinned = Env(in, "CRAFTER_BUILD_LIBCLANG"); !pinned.empty()) {
tc.libclang = fs::path(pinned);
} else if (!tc.prefix.empty() && fs::exists(tc.prefix / "lib" / "libclang.so")) {
tc.libclang = tc.prefix / "lib" / "libclang.so";
}
// 3. libc++ and its std module source. The compiler's own copy lives at
// <prefix>/share/libc++/v1 when the prefix carries libc++; the system
// copy is the last resort either way. The headers of the same install
// say which release the source belongs to.
fs::path libcxxRoot = "/usr";
if (std::string_view dir = Env(in, "CRAFTER_BUILD_LIBCXX_DIR"); !dir.empty()) {
tc.libcxxDir = fs::path(dir);
libcxxRoot = tc.libcxxDir;
} else if (!tc.prefix.empty() && fs::exists(tc.prefix / "share" / "libc++" / "v1" / "std.cppm")) {
libcxxRoot = tc.prefix;
}
tc.stdCppm = libcxxRoot / "share" / "libc++" / "v1" / "std.cppm";
tc.libcxxVersion = LibcxxVersion(libcxxRoot / "include" / "c++" / "v1");
if (!in.crafterPrefix.empty()) {
tc.stdModulesDir = in.crafterPrefix / "share" / "crafter-build" / "libc++-modules";
}
// 4. Sysroots for the cross targets that have a conventional location.
if (std::string_view dir = Env(in, "CRAFTER_BUILD_WASI_SYSROOT"); !dir.empty()) {
tc.wasiSysroot = fs::path(dir);
} else if (!tc.prefix.empty() && fs::exists(tc.prefix / "share" / "wasi-sysroot")) {
tc.wasiSysroot = tc.prefix / "share" / "wasi-sysroot";
} else {
tc.wasiSysroot = "/usr/share/wasi-sysroot";
}
if (std::string_view dir = Env(in, "CRAFTER_MINGW_DIR"); !dir.empty()) {
tc.mingwSysroot = fs::path(dir);
} else if (!in.hostRoot.empty() && fs::exists(in.hostRoot / "usr" / "x86_64-w64-mingw32")) {
// The sandbox has no mingw-w64 of its own, and no mingw gcc on PATH
// for clang to find one by.
tc.mingwSysroot = in.hostRoot / "usr";
}
return tc;
}
const Toolchain& Crafter::GetToolchain() {
static const Toolchain Resolved = []() {
ToolchainInputs in;
for (std::string_view name : EnvVars) {
if (const char* value = std::getenv(std::string(name).c_str()); value && *value) {
in.env.emplace(std::string(name), value);
}
}
// GetCrafterBuildHome throws when no share/crafter-build is found
// near the executable; a toolchain can still come from PATH then.
try {
in.crafterPrefix = GetCrafterBuildHome().parent_path().parent_path();
} catch (const std::exception&) {}
if (fs::exists("/.flatpak-info")) in.hostRoot = "/run/host";
Toolchain tc = ResolveToolchain(in);
if (!tc.error.empty()) return tc;
CommandResult triple = RunCommandChecked(std::format("{} -print-target-triple", tc.clangxx.string()));
if (triple.exitCode != 0) {
tc.error = std::format("could not run {} (exit {}): {}", tc.clangxx.string(), triple.exitCode, Trimmed(triple.output));
if (tc.prefix.empty()) tc.error += "; install clang or set CRAFTER_BUILD_LLVM_DIR";
return tc;
}
tc.hostTriple = Trimmed(std::move(triple.output));
CommandResult version = RunCommandChecked(std::format("{} --version", tc.clangxx.string()));
if (version.exitCode == 0) {
std::string first = version.output.substr(0, version.output.find('\n'));
tc.clangVersion = Trimmed(std::move(first));
}
return tc;
}();
return Resolved;
}
const Toolchain& Crafter::RequireToolchain() {
const Toolchain& tc = GetToolchain();
if (!tc.error.empty()) throw std::runtime_error(tc.error);
return tc;
}
std::int32_t Crafter::LibcxxVersion(const fs::path& includeDir) {
// "# define _LIBCPP_VERSION 230102", the one place libc++ states its
// release in a form the preprocessor does not have to be run for.
constexpr std::string_view Marker = "define _LIBCPP_VERSION ";
std::ifstream config(includeDir / "__config");
std::string line;
while (std::getline(config, line)) {
const std::size_t at = line.find(Marker);
if (at == std::string::npos) continue;
const std::string_view digits = std::string_view(line).substr(at + Marker.size());
std::int32_t version = 0;
if (std::from_chars(digits.data(), digits.data() + digits.size(), version).ec == std::errc{}) return version;
}
return 0;
}
std::string Crafter::LibcxxVersionString(std::int32_t version) {
return std::format("{}.{}.{}", version / 10000, version / 100 % 100, version % 100);
}
StdModuleSource Crafter::SysrootStdModule(const Toolchain& tc, const fs::path& sysroot) {
const fs::path own = sysroot / "usr" / "share" / "libc++" / "v1" / "std.cppm";
if (fs::exists(own)) return {own, {}};
const fs::path includeDir = sysroot / "usr" / "include" / "c++" / "v1";
if (!fs::exists(includeDir / "__config")) {
return {{}, std::format("sysroot {} has no libc++ headers at {}; install its libc++ development package", sysroot.string(), includeDir.string())};
}
const std::int32_t wanted = LibcxxVersion(includeDir);
const bool comparable = wanted != 0 && tc.libcxxVersion != 0;
// Releases are matched on the major version: a release branch only takes
// fixes, so 22.1.3 headers and the 22.1.8 module source name the same
// declarations.
if (comparable && wanted / 10000 == tc.libcxxVersion / 10000 && fs::exists(tc.stdCppm)) return {tc.stdCppm, {}};
if (wanted != 0 && !tc.stdModulesDir.empty()) {
const fs::path shipped = tc.stdModulesDir / std::to_string(wanted / 10000) / "std.cppm";
if (fs::exists(shipped)) return {shipped, {}};
}
// One of the two releases is unknown, so there is nothing to compare:
// hand over the toolchain's source and let the compile decide, which is
// what happened for every sysroot before releases were compared.
if (!comparable && fs::exists(tc.stdCppm)) return {tc.stdCppm, {}};
const std::string sysrootRelease = wanted != 0 ? LibcxxVersionString(wanted) : std::string("of an unknown release");
const std::string toolchainSource = !fs::exists(tc.stdCppm)
? std::string("the toolchain has no std module source at all")
: std::format("the toolchain's is for libc++ {}", LibcxxVersionString(tc.libcxxVersion));
return {{}, std::format("sysroot {} carries libc++ {} without its std module source, and crafter-build has none for that release ({}); install the share/libc++/v1 directory of that libc++ release (std.cppm and std/) as {}", sysroot.string(), sysrootRelease, toolchainSource, own.parent_path().string())};
}
std::string_view Crafter::CrafterBuildVersion() {
return CRAFTER_BUILD_VERSION;
}