// SPDX-License-Identifier: LGPL-3.0-only // SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts® module Crafter.Build:External_impl; import std; import :External; import :Toolchain; import :Platform; import :Progress; import :Clang; namespace fs = std::filesystem; using namespace Crafter; namespace { std::string DeriveName(const GitSource& source) { std::string url = source.url; while (!url.empty() && (url.back() == '/' || url.back() == '\\')) url.pop_back(); if (url.ends_with(".git")) url.resize(url.size() - 4); auto slash = url.find_last_of("/\\"); return slash == std::string::npos ? url : url.substr(slash + 1); } std::string JoinOptions(std::span options) { std::string out; for (const std::string& opt : options) { if (!out.empty()) out += ' '; out += ShellQuote(opt); } return out; } std::string FetchGit(const GitSource& source, const fs::path& cloneDir) { auto runGit = [](std::string_view cmd) -> std::optional { CommandResult r = RunCommandChecked(cmd); if (r.exitCode != 0) return std::format("`{}` failed (exit {}): {}", cmd, r.exitCode, r.output); return std::nullopt; }; // Submodules follow the checkout only when asked to: after every clone, // switch, checkout or pull, so they match the revision that was fetched // instead of whichever one was checked out before, from the URLs that // revision's .gitmodules gives (sync). A no-op, and offline, when they // already match, which puts back submodules moved by hand; and no git at // all without any. auto updateSubmodules = [&runGit](const fs::path& dir) -> std::optional { if (!fs::exists(dir / ".gitmodules")) return std::nullopt; const std::string quoted = ShellQuote(dir.string()); if (auto err = runGit(std::format("git -C {} submodule sync --recursive", quoted))) return err; return runGit(std::format("git -C {} submodule update --init --recursive", quoted)); }; if (fs::exists(cloneDir)) { CommandResult remote = RunCommandChecked(std::format("git -C {} remote get-url origin", ShellQuote(cloneDir.string()))); std::string currentUrl = remote.output; while (!currentUrl.empty() && (currentUrl.back() == '\n' || currentUrl.back() == '\r')) currentUrl.pop_back(); if (remote.exitCode != 0 || currentUrl != source.url) { fs::remove_all(cloneDir); } else if (!source.commit.empty()) { CommandResult head = RunCommandChecked(std::format("git -C {} rev-parse HEAD", ShellQuote(cloneDir.string()))); std::string currentHead = head.output; while (!currentHead.empty() && (currentHead.back() == '\n' || currentHead.back() == '\r')) currentHead.pop_back(); if (head.exitCode == 0 && currentHead != source.commit) { if (auto err = runGit(std::format("git -C {} fetch origin", ShellQuote(cloneDir.string())))) return *err; if (auto err = runGit(std::format("git -C {} checkout {}", ShellQuote(cloneDir.string()), ShellQuote(source.commit)))) return *err; } // Also when the commit already matched: a clone made before // submodules were updated has them at the default branch's. if (auto err = updateSubmodules(cloneDir)) return *err; return ""; } else { // No commit pinned — pull latest. The user opted into "track latest" // by not pinning; the cost is one network round-trip per build. // Pin source.commit to a SHA for reproducible, offline-capable builds. if (!source.branch.empty()) { if (auto err = runGit(std::format("git -C {} pull origin {}", ShellQuote(cloneDir.string()), ShellQuote(source.branch)))) return *err; } else { if (auto err = runGit(std::format("git -C {} pull", ShellQuote(cloneDir.string())))) return *err; } if (auto err = updateSubmodules(cloneDir)) return *err; return ""; } } // Random per-invocation tmp suffix so concurrent crafter-build processes // resolving to the same cache key don't share an in-flight `.tmp` // directory — git's pack tempfiles would get yanked when one process // removes or renames the dir out from under the other. std::random_device rd; fs::path tmpDir = cloneDir; tmpDir += std::format(".tmp.{:08x}{:08x}", rd(), rd()); // Not --recursive: that checks out the default branch's submodules, and // the requested revision's come once it is checked out. if (auto err = runGit(std::format("git clone {} {}", ShellQuote(source.url), ShellQuote(tmpDir.string())))) { if (fs::exists(tmpDir)) fs::remove_all(tmpDir); return *err; } if (!source.branch.empty()) { if (auto err = runGit(std::format("git -C {} switch {}", ShellQuote(tmpDir.string()), ShellQuote(source.branch)))) { fs::remove_all(tmpDir); return *err; } } if (!source.commit.empty()) { if (auto err = runGit(std::format("git -C {} checkout {}", ShellQuote(tmpDir.string()), ShellQuote(source.commit)))) { fs::remove_all(tmpDir); return *err; } } if (auto err = updateSubmodules(tmpDir)) { fs::remove_all(tmpDir); return *err; } std::error_code ec; fs::rename(tmpDir, cloneDir, ec); if (ec) { fs::remove_all(tmpDir); // If a concurrent build won the race, cloneDir now exists with the // same source revision (cache key includes url+branch+commit), so // discard our clone and reuse theirs. if (fs::exists(cloneDir)) return ""; return std::format("rename {} -> {} failed: {}", tmpDir.string(), cloneDir.string(), ec.message()); } return ""; } std::string BuildInjectedCMakeFlags(const fs::path& cmakeBuildDir, std::string_view target) { std::string out; // Without an explicit build type, some packages (glslang) append a 'd' // suffix to library names assuming a debug build, breaking -l // linking. Pin to Release. out += " -DCMAKE_BUILD_TYPE=Release"; const Toolchain& tc = GetToolchain(); out += std::format(" -DCMAKE_C_COMPILER={}", ShellQuote(tc.clang.string())); out += std::format(" -DCMAKE_CXX_COMPILER={}", ShellQuote(tc.clangxx.string())); // Stdlib choice: libc++ for Linux targets (matches how crafter-build // itself is built with -stdlib=libc++); the mingw cross-compile uses the // libstdc++ shipped by the mingw-w64 toolchain, so it keeps the default. if (target.ends_with("-linux-gnu") || target.ends_with("-linux-musl")) { out += std::format(" -DCMAKE_CXX_FLAGS={}", ShellQuote(std::format("-stdlib=libc++{}", tc.LibcxxCompileFlags()))); out += std::format(" -DCMAKE_EXE_LINKER_FLAGS={}", ShellQuote(std::format("-stdlib=libc++{}", tc.LibcxxLinkFlags()))); out += std::format(" -DCMAKE_SHARED_LINKER_FLAGS={}", ShellQuote(std::format("-stdlib=libc++{}", tc.LibcxxLinkFlags()))); } if (target == "x86_64-w64-mingw32") { out += " -DCMAKE_SYSTEM_NAME=Windows"; out += std::format(" -DCMAKE_C_COMPILER_TARGET={}", target); out += std::format(" -DCMAKE_CXX_COMPILER_TARGET={}", target); // The libstdc++ and the mingw-w64 install the project's own compiles // use (GetBaseCommand). out += " -DCMAKE_CXX_FLAGS=-stdlib=libstdc++"; if (!tc.mingwSysroot.empty()) out += std::format(" -DCMAKE_SYSROOT={}", ShellQuote(tc.mingwSysroot.string())); } fs::path absBuildDir = fs::absolute(cmakeBuildDir); out += std::format(" -DCMAKE_ARCHIVE_OUTPUT_DIRECTORY={}", ShellQuote(absBuildDir.string())); out += std::format(" -DCMAKE_LIBRARY_OUTPUT_DIRECTORY={}", ShellQuote(absBuildDir.string())); return out; } std::string ConfigureCMake(const fs::path& cloneDir, const fs::path& cmakeBuildDir, std::span options, std::string_view target) { fs::path optionsFile = cmakeBuildDir / ".crafter-options"; std::string optionsKey = JoinOptions(options); // The injected flags are inputs too: a different compiler or libc++ // must reconfigure rather than reuse a CMakeCache made with the old one. const std::string injected = BuildInjectedCMakeFlags(cmakeBuildDir, target); const std::string configureKey = injected + '\n' + optionsKey; bool needsConfigure = !fs::exists(cmakeBuildDir / "CMakeCache.txt"); if (!needsConfigure) { std::ifstream existing(optionsFile); if (!existing) { needsConfigure = true; } else { std::stringstream buf; buf << existing.rdbuf(); if (buf.str() != configureKey) needsConfigure = true; } } if (!needsConfigure) return ""; if (!fs::exists(cmakeBuildDir)) fs::create_directories(cmakeBuildDir); std::string cmd = std::format("cmake -S {} -B {}{}", ShellQuote(fs::absolute(cloneDir).string()), ShellQuote(fs::absolute(cmakeBuildDir).string()), injected); if (!options.empty()) { cmd += " "; cmd += optionsKey; } CommandResult r = RunCommandChecked(cmd); if (r.exitCode != 0) { return std::format("cmake configure failed (exit {}): {}", r.exitCode, r.output); } std::ofstream(optionsFile) << configureKey; return ""; } std::string BuildCMake(const fs::path& cmakeBuildDir) { // Without --parallel, the Unix Makefiles generator builds one translation // unit at a time, leaving every core but one idle. Pass an explicit job // count (a bare --parallel maps to an unbounded `make -j` fork bomb on the // Makefiles generator) so deps like DPP/msquic/glslang compile ~N× faster. std::uint32_t jobs = std::max(1u, std::thread::hardware_concurrency()); std::string cmd = std::format("cmake --build {} --parallel {}", ShellQuote(fs::absolute(cmakeBuildDir).string()), jobs); CommandResult r = RunCommandChecked(cmd); if (r.exitCode != 0) { return std::format("cmake --build failed (exit {}): {}", r.exitCode, r.output); } return ""; } } // namespace fs::path Crafter::ExternalCloneDir(const ExternalDependency& dep, std::string_view target) { std::string name = dep.name.empty() ? DeriveName(dep.source) : dep.name; if (name.empty()) return {}; std::string keyMaterial = std::format("{}|{}|{}|{}|{}", dep.source.url, dep.source.branch, dep.source.commit, JoinOptions(dep.options), target); std::size_t key = std::hash{}(keyMaterial); return GetCacheDir() / "external" / std::format("{}-{:016x}", name, key); } std::vector Crafter::ExternalIncludeFlags(const ExternalDependency& dep, std::string_view target) { std::vector flags; fs::path cloneDir = ExternalCloneDir(dep, target); if (cloneDir.empty()) return flags; for (const fs::path& include : dep.includeDirs) { fs::path full = include.empty() ? cloneDir : cloneDir / include; flags.push_back(std::format("-I{}", fs::absolute(full).string())); } return flags; } ExternalBuildResult Crafter::BuildExternal(const ExternalDependency& dep, std::string_view target, std::atomic& cancelled) { ExternalBuildResult result; if (cancelled.load(std::memory_order_relaxed)) return result; std::string name = dep.name.empty() ? DeriveName(dep.source) : dep.name; if (name.empty()) { result.error = std::format("Could not derive name for external dependency from URL '{}'", dep.source.url); return result; } fs::path externalRoot = GetCacheDir() / "external"; if (!fs::exists(externalRoot)) { std::error_code ec; fs::create_directories(externalRoot, ec); if (ec) { result.error = std::format("Failed to create {}: {}", externalRoot.string(), ec.message()); return result; } } fs::path cloneDir = ExternalCloneDir(dep, target); // Builds running side by side (tests built in parallel, two // crafter-builds) share the clone: one fetches and builds it at a time. FileLock lock(fs::path(std::format("{}.lock", cloneDir.string()))); std::string fetchErr = FetchGit(dep.source, cloneDir); if (!fetchErr.empty()) { result.error = std::format("git fetch for '{}': {}", name, fetchErr); return result; } if (cancelled.load(std::memory_order_relaxed)) return result; fs::path cmakeBuildDir; if (dep.builder == ExternalBuilder::CMake) { cmakeBuildDir = cloneDir / "build"; if (std::string err = ConfigureCMake(cloneDir, cmakeBuildDir, dep.options, target); !err.empty()) { result.error = std::format("cmake configure for '{}': {}", name, err); return result; } if (cancelled.load(std::memory_order_relaxed)) return result; if (std::string err = BuildCMake(cmakeBuildDir); !err.empty()) { result.error = std::format("cmake build for '{}': {}", name, err); return result; } } result.compileFlags = ExternalIncludeFlags(dep, target); if (dep.builder == ExternalBuilder::CMake) { // Each search path gets both a -L (link-time) and a -Wl,-rpath // (runtime-loader). The rpath stays embedded in the produced // binary so it picks up shared deps from the cache without any // LD_LIBRARY_PATH gymnastics. Static deps (.a) ignore the rpath // harmlessly. The PE/COFF target (mingw) resolves DLLs via PATH or // adjacency rather than rpath, so lld warns if we pass it. const bool isPe = target == "x86_64-w64-mingw32"; std::string buildDirAbs = fs::absolute(cmakeBuildDir).string(); result.linkFlags.push_back(std::format("-L{}", buildDirAbs)); if (!isPe) result.linkFlags.push_back(std::format("-Wl,-rpath,{}", buildDirAbs)); for (const fs::path& libDir : dep.libDirs) { fs::path full = libDir.is_absolute() ? libDir : cmakeBuildDir / libDir; std::string fullAbs = fs::absolute(full).string(); result.linkFlags.push_back(std::format("-L{}", fullAbs)); if (!isPe) result.linkFlags.push_back(std::format("-Wl,-rpath,{}", fullAbs)); } for (const std::string& lib : dep.libs) { result.linkFlags.push_back(std::format("-l{}", lib)); } // The newest library in every directory the link searches: the build // directory and the libDirs, where projects like msquic put theirs, // versioned names (libmsquic.so.2) included. auto isLibrary = [](const std::string& name) { return name.ends_with(".a") || name.ends_with(".so") || name.contains(".so.") || name.ends_with(".dll"); }; std::vector searched = { cmakeBuildDir }; for (const fs::path& libDir : dep.libDirs) searched.push_back(libDir.is_absolute() ? libDir : cmakeBuildDir / libDir); for (const fs::path& dir : searched) { std::error_code listEc; for (fs::directory_iterator it(dir, listEc), end; !listEc && it != end; it.increment(listEc)) { std::error_code ec; if (!it->is_regular_file(ec) || !isLibrary(it->path().filename().string())) continue; fs::file_time_type t = it->last_write_time(ec); if (!ec && t > result.latestArtifact) result.latestArtifact = t; } } } return result; } namespace { std::mutex ProjectCacheMutex; std::unordered_map> ProjectCache; // Run the dep's project.cpp from its own directory so its relative paths // (cfg.path = "./", interfaces/, lib/, ...) resolve against the dep // root, then rebase cfg.path and all other relative path collections to // absolute so Build() can run from any cwd without misresolving them. // The mutex is NOT held across LoadProject — the dep may itself call // GitProject/LocalProject, and a non-recursive lock would deadlock. Configuration LoadProjectFromRoot(const fs::path& projectPath, const fs::path& depRoot, std::span args) { fs::path savedCwd = fs::current_path(); Configuration cfg; fs::current_path(depRoot); try { cfg = LoadProject(projectPath, args); } catch (...) { fs::current_path(savedCwd); throw; } fs::current_path(savedCwd); cfg.path = fs::absolute(depRoot / cfg.path).lexically_normal(); auto abs = [&](const fs::path& p) -> fs::path { return p.is_absolute() ? p : fs::absolute(depRoot / p).lexically_normal(); }; for (fs::path& p : cfg.cFiles) p = abs(p); for (fs::path& p : cfg.cuda) p = abs(p); for (fs::path& p : cfg.files) p = abs(p); for (fs::path& p : cfg.buildFiles) p = abs(p); for (Shader& s : cfg.shaders) s.path = abs(s.path); return cfg; } } Configuration* Crafter::GitProject(const GitProjectSpec& spec) { // Two hashes: the clone is shared across specs that point at the same // source revision (forwarded args don't change what gets cloned, just // how the dep's project.cpp interprets them at runtime), while the // in-memory Configuration is keyed by the full spec so each (args) // permutation gets its own Configuration object. std::string srcKey = std::format("git|{}|{}|{}|{}", spec.source.url, spec.source.branch, spec.source.commit, spec.projectFile.string()); std::string srcHash = std::format("{:016x}", std::hash{}(srcKey)); std::string fullKey = srcKey; for (const std::string& a : spec.args) { fullKey += '|'; fullKey += a; } std::string fullHash = std::format("{:016x}", std::hash{}(fullKey)); { std::lock_guard lock(ProjectCacheMutex); if (auto it = ProjectCache.find(fullHash); it != ProjectCache.end()) { return it->second.get(); } } std::string name = DeriveName(spec.source); if (name.empty()) { throw std::runtime_error(std::format("GitProject: could not derive name from URL '{}'", spec.source.url)); } fs::path externalRoot = GetCacheDir() / "external"; fs::create_directories(externalRoot); fs::path cloneDir = externalRoot / std::format("{}-{}", name, srcHash); { // One fetch of a clone at a time, as in BuildExternal. FileLock lock(fs::path(std::format("{}.lock", cloneDir.string()))); const bool exists = fs::exists(cloneDir); Progress::Task task(std::format("{} {}", exists ? "Updating" : "Cloning", name)); if (std::string err = FetchGit(spec.source, cloneDir); !err.empty()) { throw std::runtime_error(std::format("GitProject({}): {}", spec.source.url, err)); } } fs::path projectPath = cloneDir / spec.projectFile; if (!fs::exists(projectPath)) { throw std::runtime_error(std::format("GitProject({}): {} not found in clone", spec.source.url, spec.projectFile.string())); } std::vector argViews(spec.args.begin(), spec.args.end()); Configuration cfg = LoadProjectFromRoot(projectPath, cloneDir, argViews); std::lock_guard lock(ProjectCacheMutex); if (auto it = ProjectCache.find(fullHash); it != ProjectCache.end()) { return it->second.get(); } auto stored = std::make_unique(std::move(cfg)); Configuration* ptr = stored.get(); ProjectCache.emplace(std::move(fullHash), std::move(stored)); return ptr; } fs::path Crafter::GitFetch(const GitSource& source) { std::string name = DeriveName(source); if (name.empty()) { throw std::runtime_error(std::format("GitFetch: could not derive name from URL '{}'", source.url)); } fs::path externalRoot = GetCacheDir() / "external"; std::error_code ec; fs::create_directories(externalRoot, ec); if (ec) { throw std::runtime_error(std::format("GitFetch: failed to create {}: {}", externalRoot.string(), ec.message())); } std::string keyMaterial = std::format("git|{}|{}|{}", source.url, source.branch, source.commit); std::size_t key = std::hash{}(keyMaterial); fs::path cloneDir = externalRoot / std::format("{}-{:016x}", name, key); { // One fetch of a clone at a time, as in BuildExternal. FileLock lock(fs::path(std::format("{}.lock", cloneDir.string()))); const bool exists = fs::exists(cloneDir); Progress::Task task(std::format("{} {}", exists ? "Updating" : "Cloning", name)); if (std::string err = FetchGit(source, cloneDir); !err.empty()) { throw std::runtime_error(std::format("GitFetch({}): {}", source.url, err)); } } return cloneDir; } Configuration* Crafter::LocalProject(const LocalProjectSpec& spec) { fs::path projectPath = fs::absolute(spec.projectFile).lexically_normal(); if (!fs::exists(projectPath)) { throw std::runtime_error(std::format("LocalProject: {} not found", projectPath.string())); } fs::path canonical = fs::canonical(projectPath); std::string fullKey = std::format("local|{}", canonical.string()); for (const std::string& a : spec.args) { fullKey += '|'; fullKey += a; } std::string fullHash = std::format("{:016x}", std::hash{}(fullKey)); { std::lock_guard lock(ProjectCacheMutex); if (auto it = ProjectCache.find(fullHash); it != ProjectCache.end()) { return it->second.get(); } } fs::path depRoot = canonical.parent_path(); std::vector argViews(spec.args.begin(), spec.args.end()); Configuration cfg = LoadProjectFromRoot(canonical, depRoot, argViews); std::lock_guard lock(ProjectCacheMutex); if (auto it = ProjectCache.find(fullHash); it != ProjectCache.end()) { return it->second.get(); } auto stored = std::make_unique(std::move(cfg)); Configuration* ptr = stored.get(); ProjectCache.emplace(std::move(fullHash), std::move(stored)); return ptr; }