A project.cpp could only import std and Crafter.Build, so anything shared
between build descriptions had to be a header every consumer copies. Now
#pragma crafter dependency "<source>" [branch "<b>" | commit "<sha>"] [project "<file>"]
names a sibling crafter-build project that project.cpp itself imports a
module from. LoadProject scans for the directive, resolves it through
GitProject (a URL) or LocalProject (a path relative to the project file),
forces the result into host-tooling mode (the host triple LoadProject's
own cache is keyed on, native march, no debug, -fPIC so the archive links
into a shared object), builds it, and puts its BMIs and archive on the
project.so command line. The dependency's own project.cpp goes through
the same loader, so its directives resolve recursively; a cycle between
project files is an error instead of unbounded recursion.
The dependency is built on every load, fresh or not: clang crashes rather
than erroring when it imports a BMI whose own imported PCM was rebuilt
underneath it, and the module depfile — which Module::Check already reads —
is what notices the host PCM cache was regenerated. Its artifact mtime is
an input to project.so's staleness check, so editing the dependency
recompiles project.cpp without touching it. The Linux link gains
-fuse-ld=lld because release archives hold ThinLTO bitcode.
ProjectDependency(name) hands the same Configuration* back from inside
CrafterBuildProject, by configuration name or exported module name, so a
test that links the module needn't spell the source twice. It reads a
thread_local frame stack that an RAII guard keeps for the whole of
LoadProject, which also provides the cycle check.
ExpectedOutput() is the link gate's artifact-path lambda hoisted into the
:Clang partition, so the staleness check and the -r runner share the one
rule. Windows branches are wired identically but unverified.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
1338 lines
58 KiB
C++
1338 lines
58 KiB
C++
// SPDX-License-Identifier: LGPL-3.0-only
|
|
// SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts®
|
|
|
|
module;
|
|
#if defined(CRAFTER_BUILD_CONFIGURATION_TARGET_x86_64_pc_windows_msvc) || defined(CRAFTER_BUILD_CONFIGURATION_TARGET_x86_64_w64_mingw32)
|
|
#include <windows.h>
|
|
#endif
|
|
#include <stdio.h>
|
|
#include <stdlib.h>
|
|
#include <limits.h>
|
|
#ifdef CRAFTER_BUILD_CONFIGURATION_TARGET_x86_64_pc_linux_gnu
|
|
#include <unistd.h>
|
|
#include <dlfcn.h>
|
|
#include <sys/wait.h>
|
|
#include <sys/file.h>
|
|
#include <fcntl.h>
|
|
#include <errno.h>
|
|
#endif
|
|
module Crafter.Build:Platform_impl;
|
|
import std;
|
|
import :Platform;
|
|
import :Clang;
|
|
import :Interface;
|
|
import :External;
|
|
import :Progress;
|
|
namespace fs = std::filesystem;
|
|
using namespace Crafter;
|
|
|
|
namespace {
|
|
// Exclusive advisory lock over one PCM cache dir (`<cache>/<target>-<march>/`),
|
|
// held for the duration of a (re)build. Concurrent crafter-build invocations —
|
|
// and the worker threads inside a single invocation — that share a cache
|
|
// serialize their builds through it: the first builder to acquire the lock
|
|
// precompiles std.pcm / the Crafter.Build-*.pcm modules while everyone else
|
|
// blocks, then re-checks freshness and reuses the finished file instead of
|
|
// re-running clang and racing to overwrite the same path. Each builder must
|
|
// pass through the lock before it reaches the compile steps that *read* those
|
|
// PCMs, so a reader never observes a half-written file.
|
|
//
|
|
// The lock lives in the OS, keyed on a `.lock` file in the cache dir, and is
|
|
// released automatically when the descriptor/handle closes — including on
|
|
// process death — so an interrupted build can never leave a stale lock. The
|
|
// descriptor is close-on-exec / non-inheritable so spawned clang processes
|
|
// don't keep the lock alive past our own release.
|
|
class CacheLock {
|
|
#ifdef CRAFTER_BUILD_CONFIGURATION_TARGET_x86_64_pc_linux_gnu
|
|
std::int32_t fd_ = -1;
|
|
public:
|
|
explicit CacheLock(const fs::path& cacheDir) {
|
|
fd_ = ::open((cacheDir / ".lock").c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0644);
|
|
if (fd_ >= 0) {
|
|
while (::flock(fd_, LOCK_EX) != 0 && errno == EINTR) {}
|
|
}
|
|
}
|
|
~CacheLock() {
|
|
if (fd_ >= 0) {
|
|
::close(fd_); // closing the descriptor drops the flock
|
|
}
|
|
}
|
|
#else
|
|
HANDLE handle_ = INVALID_HANDLE_VALUE;
|
|
public:
|
|
explicit CacheLock(const fs::path& cacheDir) {
|
|
handle_ = CreateFileW((cacheDir / ".lock").wstring().c_str(), GENERIC_READ | GENERIC_WRITE, FILE_SHARE_READ | FILE_SHARE_WRITE, nullptr, OPEN_ALWAYS, FILE_ATTRIBUTE_NORMAL, nullptr);
|
|
if (handle_ != INVALID_HANDLE_VALUE) {
|
|
OVERLAPPED ov{};
|
|
LockFileEx(handle_, LOCKFILE_EXCLUSIVE_LOCK, 0, MAXDWORD, MAXDWORD, &ov);
|
|
}
|
|
}
|
|
~CacheLock() {
|
|
if (handle_ != INVALID_HANDLE_VALUE) {
|
|
CloseHandle(handle_); // closing the handle drops the lock
|
|
}
|
|
}
|
|
#endif
|
|
CacheLock(const CacheLock&) = delete;
|
|
CacheLock& operator=(const CacheLock&) = delete;
|
|
};
|
|
|
|
// The cached Crafter.Build PCMs are keyed by `<target>-<march>` alone, so
|
|
// every crafter-build on the machine writes to the same files regardless of
|
|
// which share/crafter-build its module sources came from. Freshness used to
|
|
// be a per-file mtime comparison, which cannot distinguish "this PCM is
|
|
// newer than my source" from "this PCM was built from *different* sources
|
|
// that happen to be newer" — so a second install or checkout silently
|
|
// compiled its project against the other's declarations. Same failure shape
|
|
// as issue #27: no error, a binary built against a layout nobody linked.
|
|
//
|
|
// A stamp over the bytes of every module source answers what the mtime
|
|
// can't. It covers the whole set rather than one file at a time because the
|
|
// PCMs import each other: a change to :Interface invalidates :Clang's PCM
|
|
// even though Crafter.Build-Clang.cppm is untouched.
|
|
std::string CrafterBuildSourceStamp(const fs::path& sourceDir, std::span<const std::string_view> moduleNames) {
|
|
std::string all;
|
|
auto append = [&all, &sourceDir](const fs::path& relative) {
|
|
std::ifstream in(sourceDir / relative, std::ios::binary);
|
|
std::ostringstream buffer;
|
|
buffer << in.rdbuf();
|
|
all += relative.string();
|
|
all += '\0';
|
|
all += buffer.str();
|
|
all += '\0';
|
|
};
|
|
for (std::string_view name : moduleNames) {
|
|
append(fs::path(std::format("{}.cppm", name)));
|
|
}
|
|
// Exported through `module;` preambles, so its contents land in the PCMs
|
|
// too.
|
|
append("Crafter.Build-Api.h");
|
|
return std::format("{:016x}", std::hash<std::string>{}(all));
|
|
}
|
|
|
|
fs::path CacheStampPath(const fs::path& cacheDir) {
|
|
return cacheDir / "crafter-build-sources.stamp";
|
|
}
|
|
|
|
std::string ReadCacheStamp(const fs::path& cacheDir) {
|
|
std::ifstream in(CacheStampPath(cacheDir), std::ios::binary);
|
|
std::string stamp;
|
|
std::getline(in, stamp);
|
|
return stamp;
|
|
}
|
|
|
|
void WriteCacheStamp(const fs::path& cacheDir, const std::string& stamp) {
|
|
std::ofstream(CacheStampPath(cacheDir), std::ios::binary | std::ios::trunc) << stamp << '\n';
|
|
}
|
|
}
|
|
|
|
fs::path Crafter::GetCrafterBuildHome() {
|
|
if (const char* envHome = std::getenv("CRAFTER_BUILD_HOME")) {
|
|
return fs::path(envHome);
|
|
}
|
|
#if defined(CRAFTER_BUILD_CONFIGURATION_TARGET_x86_64_pc_windows_msvc) || defined(CRAFTER_BUILD_CONFIGURATION_TARGET_x86_64_w64_mingw32)
|
|
char buf[MAX_PATH];
|
|
DWORD len = GetModuleFileNameA(nullptr, buf, MAX_PATH);
|
|
if (len == 0 || len == MAX_PATH) {
|
|
throw std::runtime_error("GetModuleFileName failed");
|
|
}
|
|
fs::path hostExe(std::string(buf, len));
|
|
#else
|
|
fs::path hostExe = fs::read_symlink("/proc/self/exe");
|
|
#endif
|
|
|
|
// Walk up from the exe's directory looking for share/crafter-build with
|
|
// a known module source present as a sentinel. Handles:
|
|
// <prefix>/bin/crafter-build -> <prefix>/share/crafter-build (FHS install)
|
|
// <repo>/bin/crafter-build -> <repo>/share/crafter-build (build.sh)
|
|
// <repo>/bin/<arch>/crafter-build -> <repo>/share/crafter-build (self-host)
|
|
fs::path dir = hostExe.parent_path();
|
|
std::vector<fs::path> tried;
|
|
for (;;) {
|
|
fs::path candidate = dir / "share" / "crafter-build";
|
|
tried.push_back(candidate);
|
|
std::error_code ec;
|
|
if (fs::exists(candidate / "Crafter.Build.cppm", ec)) {
|
|
return candidate;
|
|
}
|
|
fs::path parent = dir.parent_path();
|
|
if (parent == dir) break;
|
|
dir = parent;
|
|
}
|
|
std::string msg = std::format("could not locate crafter-build runtime assets relative to {} (set CRAFTER_BUILD_HOME). Tried:", hostExe.string());
|
|
for (const auto& p : tried) {
|
|
msg += std::format("\n {}", p.string());
|
|
}
|
|
throw std::runtime_error(msg);
|
|
}
|
|
|
|
fs::file_time_type Crafter::NewestPrerequisite(const fs::path& depFile) {
|
|
std::ifstream in(depFile, std::ios::binary);
|
|
if (!in) return fs::file_time_type::max();
|
|
std::ostringstream buffer;
|
|
buffer << in.rdbuf();
|
|
const std::string text = std::move(buffer).str();
|
|
|
|
// Make-rule tokenizing, as clang writes it: `target: prereq prereq`, with a
|
|
// trailing '\' continuing the rule onto the next line (clang wraps at ~80
|
|
// columns, so any real depfile has several), '\ ' and '\#' escaping a
|
|
// character into a filename, and '$$' standing for a literal '$'. A
|
|
// backslash that is none of those is a Windows path separator and stays.
|
|
std::vector<std::string> tokens;
|
|
std::string token;
|
|
auto flush = [&tokens, &token]() {
|
|
if (!token.empty()) {
|
|
tokens.push_back(token);
|
|
token.clear();
|
|
}
|
|
};
|
|
for (std::size_t i = 0; i < text.size(); ++i) {
|
|
const char c = text[i];
|
|
const char next = i + 1 < text.size() ? text[i + 1] : '\0';
|
|
if (c == '\\' && (next == '\n' || next == '\r')) {
|
|
flush();
|
|
++i;
|
|
} else if (c == '\\' && (next == ' ' || next == '\t' || next == '#')) {
|
|
token += next;
|
|
++i;
|
|
} else if (c == '$' && next == '$') {
|
|
token += '$';
|
|
++i;
|
|
} else if (c == ' ' || c == '\t' || c == '\n' || c == '\r') {
|
|
flush();
|
|
} else {
|
|
token += c;
|
|
}
|
|
}
|
|
flush();
|
|
|
|
// A token ending in ':' names a rule target, not an input. The first one
|
|
// closes the target list (and a Windows `C:\build\x.o:` still ends in ':',
|
|
// where splitting on the first colon would not); a later one opens another
|
|
// rule, as `-MP` phony targets do, whose name is not a file to stat.
|
|
bool sawTarget = false;
|
|
fs::file_time_type newest = fs::file_time_type::min();
|
|
for (const std::string& prerequisite : tokens) {
|
|
if (prerequisite.ends_with(':')) {
|
|
sawTarget = true;
|
|
continue;
|
|
}
|
|
if (!sawTarget) continue;
|
|
std::error_code ec;
|
|
const fs::file_time_type stamp = fs::last_write_time(prerequisite, ec);
|
|
// A prerequisite that has since been deleted makes the recorded set
|
|
// unusable as evidence — the source that included it must have changed
|
|
// too, and rebuilding is what writes a depfile that no longer names it.
|
|
if (ec) return fs::file_time_type::max();
|
|
if (stamp > newest) newest = stamp;
|
|
}
|
|
// No rule at all: a truncated or empty depfile proves nothing.
|
|
if (!sawTarget) return fs::file_time_type::max();
|
|
return newest;
|
|
}
|
|
|
|
bool Crafter::MatchGlob(std::string_view glob, std::string_view name) {
|
|
std::size_t gi = 0;
|
|
std::size_t ni = 0;
|
|
std::size_t star = std::string_view::npos;
|
|
std::size_t mark = 0;
|
|
while (ni < name.size()) {
|
|
if (gi < glob.size() && (glob[gi] == '?' || glob[gi] == name[ni])) {
|
|
++gi; ++ni;
|
|
} else if (gi < glob.size() && glob[gi] == '*') {
|
|
star = gi++;
|
|
mark = ni;
|
|
} else if (star != std::string_view::npos) {
|
|
gi = star + 1;
|
|
ni = ++mark;
|
|
} else {
|
|
return false;
|
|
}
|
|
}
|
|
while (gi < glob.size() && glob[gi] == '*') ++gi;
|
|
return gi == glob.size();
|
|
}
|
|
|
|
bool Crafter::MatchAny(std::span<const std::string> globs, std::string_view name) {
|
|
if (globs.empty()) return true;
|
|
for (const auto& g : globs) {
|
|
if (MatchGlob(g, name)) return true;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
// See the declaration in Crafter.Build-Platform.cppm for why this branches on
|
|
// the exit code rather than on whether the command said anything.
|
|
std::string Crafter::RunBuildCommand(std::string_view cmd) {
|
|
Progress::EchoCommand(cmd);
|
|
CommandResult r = RunCommandChecked(cmd);
|
|
|
|
if (r.exitCode == 0) {
|
|
// Warnings from a step that succeeded. Show them — until now they were
|
|
// invisible on an incremental build and fatal on a cold one.
|
|
if (!r.output.empty()) Progress::Diagnostic(r.output);
|
|
return "";
|
|
}
|
|
|
|
// A compiler killed outright (the OOM killer is the common one) prints
|
|
// nothing, so the old "output means failure" rule read it as success and
|
|
// let the build carry on with a missing object. Say what happened instead.
|
|
if (r.output.empty()) {
|
|
return r.crashed
|
|
? std::format("terminated by signal {}: {}", r.signal, cmd)
|
|
: std::format("exited with code {}: {}", r.exitCode, cmd);
|
|
}
|
|
if (r.crashed) {
|
|
return std::format("{}\nterminated by signal {}", r.output, r.signal);
|
|
}
|
|
return r.output;
|
|
}
|
|
|
|
// --- Dependencies of project.cpp itself -------------------------------------
|
|
//
|
|
// #pragma crafter dependency "<source>" [branch "<b>" | commit "<sha>"] [project "<file>"]
|
|
//
|
|
// names a sibling crafter-build project that project.cpp imports a module
|
|
// from. Anything a file needs before it can compile cannot be C++ inside that
|
|
// file, so this is a directive next to the `import` it enables — where
|
|
// `#include` lives. Each one is fetched, built for the host, and put on
|
|
// project.so's module path and link line before project.cpp is compiled. The
|
|
// dependency's own project.cpp goes through LoadProject, so its pragmas
|
|
// resolve recursively.
|
|
namespace {
|
|
struct PragmaDependency {
|
|
std::string source;
|
|
std::string branch;
|
|
std::string commit;
|
|
std::string project;
|
|
std::size_t line = 0;
|
|
};
|
|
|
|
[[noreturn]] void PragmaError(const fs::path& file, std::size_t line, std::string_view message) {
|
|
throw std::runtime_error(std::format("{}:{}: {}", file.string(), line, message));
|
|
}
|
|
|
|
// The tokens of one directive: bare words and double-quoted strings (no
|
|
// escapes — a path or URL never needs one). Reading stops at `//`.
|
|
class PragmaCursor {
|
|
public:
|
|
PragmaCursor(std::string_view text, const fs::path& file, std::size_t line) : text_(text), file_(file), line_(line) {}
|
|
|
|
bool AtEnd() {
|
|
SkipSpace();
|
|
return pos_ >= text_.size() || text_.substr(pos_).starts_with("//");
|
|
}
|
|
|
|
bool Take(char c) {
|
|
SkipSpace();
|
|
if (pos_ >= text_.size() || text_[pos_] != c) return false;
|
|
++pos_;
|
|
return true;
|
|
}
|
|
|
|
std::string_view Word() {
|
|
SkipSpace();
|
|
const std::size_t start = pos_;
|
|
while (pos_ < text_.size() && IsWordChar(text_[pos_])) ++pos_;
|
|
return text_.substr(start, pos_ - start);
|
|
}
|
|
|
|
std::optional<std::string_view> Quoted() {
|
|
SkipSpace();
|
|
if (pos_ >= text_.size() || text_[pos_] != '"') return std::nullopt;
|
|
const std::size_t close = text_.find('"', pos_ + 1);
|
|
if (close == std::string_view::npos) Fail("unterminated string");
|
|
const std::string_view inner = text_.substr(pos_ + 1, close - pos_ - 1);
|
|
pos_ = close + 1;
|
|
return inner;
|
|
}
|
|
|
|
[[noreturn]] void Fail(std::string_view message) const { PragmaError(file_, line_, message); }
|
|
|
|
private:
|
|
static bool IsWordChar(char c) { return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_'; }
|
|
|
|
void SkipSpace() {
|
|
while (pos_ < text_.size() && (text_[pos_] == ' ' || text_[pos_] == '\t' || text_[pos_] == '\r')) ++pos_;
|
|
}
|
|
|
|
std::string_view text_;
|
|
const fs::path& file_;
|
|
std::size_t line_;
|
|
std::size_t pos_ = 0;
|
|
};
|
|
|
|
// One directive per line, anywhere in the file. Comments aren't parsed:
|
|
// a directive inside a block comment is honoured, and `//` lines fall out
|
|
// because they don't start with `#`.
|
|
std::vector<PragmaDependency> ScanProjectFilePragmas(const fs::path& absProject) {
|
|
std::vector<PragmaDependency> found;
|
|
std::ifstream in(absProject);
|
|
std::string raw;
|
|
for (std::size_t lineNo = 1; std::getline(in, raw); ++lineNo) {
|
|
PragmaCursor cur(raw, absProject, lineNo);
|
|
if (!cur.Take('#') || cur.Word() != "pragma" || cur.Word() != "crafter") continue;
|
|
const std::string_view verb = cur.Word();
|
|
if (verb != "dependency") cur.Fail(std::format("unknown crafter pragma '{}'", verb));
|
|
|
|
PragmaDependency dep;
|
|
dep.line = lineNo;
|
|
const std::optional<std::string_view> source = cur.Quoted();
|
|
if (!source) cur.Fail("expected a quoted source after 'dependency'");
|
|
dep.source = *source;
|
|
while (!cur.AtEnd()) {
|
|
const std::string_view key = cur.Word();
|
|
std::string* field = key == "branch" ? &dep.branch : key == "commit" ? &dep.commit : key == "project" ? &dep.project : nullptr;
|
|
if (!field) cur.Fail(key.empty() ? "unexpected text after the source" : std::format("unknown keyword '{}' (expected branch, commit or project)", key));
|
|
if (!field->empty()) cur.Fail(std::format("'{}' given twice", key));
|
|
const std::optional<std::string_view> value = cur.Quoted();
|
|
if (!value || value->empty()) cur.Fail(std::format("expected a quoted value after '{}'", key));
|
|
*field = *value;
|
|
}
|
|
if (dep.source.empty()) cur.Fail("the source is empty");
|
|
if (!dep.branch.empty() && !dep.commit.empty()) cur.Fail("'branch' and 'commit' are mutually exclusive");
|
|
found.push_back(std::move(dep));
|
|
}
|
|
return found;
|
|
}
|
|
|
|
struct ProjectFileFrame {
|
|
fs::path project;
|
|
std::vector<Configuration*> dependencies;
|
|
};
|
|
|
|
// LoadProject nests: a dependency's project.cpp loads inside its
|
|
// consumer's pre-pass, and GitProject inside CrafterBuildProject nests
|
|
// again. Several threads may also load at once (ConcurrentCacheRace), so
|
|
// the stack is per thread. ProjectDependency() reads the top frame.
|
|
thread_local std::vector<ProjectFileFrame> LoadingFrames;
|
|
|
|
// Held for the whole of one LoadProject call. Pushing a file that is
|
|
// already on the stack is a cycle, which used to recurse forever.
|
|
class ProjectFileGuard {
|
|
public:
|
|
explicit ProjectFileGuard(const fs::path& absProject) {
|
|
for (const ProjectFileFrame& frame : LoadingFrames) {
|
|
if (frame.project != absProject) continue;
|
|
std::string chain;
|
|
for (const ProjectFileFrame& f : LoadingFrames) chain += std::format("{} -> ", f.project.string());
|
|
throw std::runtime_error(std::format("circular project dependency: {}{}", chain, absProject.string()));
|
|
}
|
|
index_ = LoadingFrames.size();
|
|
LoadingFrames.push_back({ absProject, {} });
|
|
}
|
|
~ProjectFileGuard() { LoadingFrames.pop_back(); }
|
|
ProjectFileGuard(const ProjectFileGuard&) = delete;
|
|
ProjectFileGuard& operator=(const ProjectFileGuard&) = delete;
|
|
|
|
ProjectFileFrame& Frame() { return LoadingFrames[index_]; }
|
|
|
|
private:
|
|
std::size_t index_ = 0;
|
|
};
|
|
|
|
// A pre-dependency is host tooling: project.so is loaded into the running
|
|
// crafter-build, so the dependency is built for the host and against the
|
|
// same PCM cache LoadProject populates — the literal host triple, not
|
|
// HostTarget(), which can spell the host differently and would land the
|
|
// dependency's std/Crafter.Build PCMs in another cache dir. Recursive,
|
|
// because the dependency's project.cpp forwards no arguments but still
|
|
// reads CRAFTER_BUILD_MARCH from the environment for its own deps.
|
|
void ForceHostTooling(Configuration& cfg, const Configuration& host, std::unordered_set<Configuration*>& seen) {
|
|
if (!seen.insert(&cfg).second) return;
|
|
cfg.target = host.target;
|
|
cfg.march = host.march;
|
|
cfg.mtune = host.mtune;
|
|
cfg.debug = false;
|
|
// project.so is a shared object, so the archive it links has to be
|
|
// position-independent; static libraries aren't by default. Part of
|
|
// VariantId, so this build never shares a bin dir with a normal one.
|
|
// Checked before pushing: the same Configuration comes back from the
|
|
// project cache for every consumer in this process.
|
|
if (host.target == "x86_64-pc-linux-gnu" && !std::ranges::contains(cfg.compileFlags, "-fPIC")) {
|
|
cfg.compileFlags.emplace_back("-fPIC");
|
|
}
|
|
for (Configuration* dep : cfg.dependencies) ForceHostTooling(*dep, host, seen);
|
|
}
|
|
|
|
struct ProjectFileDependencies {
|
|
std::vector<Configuration*> configs;
|
|
std::string compileFlags; // " -fprebuilt-module-path=…" for each dependency and its transitive deps
|
|
std::string linkFlags; // every dependency's BuildResult.libs, verbatim
|
|
fs::file_time_type newestArtifact = fs::file_time_type::min();
|
|
};
|
|
|
|
bool IsGitSource(std::string_view source) {
|
|
return source.contains("://") || source.starts_with("git@");
|
|
}
|
|
|
|
ProjectFileDependencies ResolveProjectFileDependencies(const fs::path& absProject, const Configuration& host) {
|
|
ProjectFileDependencies out;
|
|
const std::vector<PragmaDependency> pragmas = ScanProjectFilePragmas(absProject);
|
|
if (pragmas.empty()) return out;
|
|
|
|
std::unordered_map<fs::path, std::shared_future<BuildResult>> depResults;
|
|
std::mutex depMutex;
|
|
std::unordered_set<Configuration*> forced;
|
|
std::unordered_set<Configuration*> walked;
|
|
std::unordered_set<std::string> libs;
|
|
|
|
for (const PragmaDependency& pragma : pragmas) {
|
|
Configuration* cfg = nullptr;
|
|
if (IsGitSource(pragma.source)) {
|
|
GitProjectSpec spec;
|
|
spec.source = { pragma.source, pragma.branch, pragma.commit };
|
|
if (!pragma.project.empty()) spec.projectFile = pragma.project;
|
|
cfg = GitProject(spec);
|
|
} else {
|
|
if (!pragma.branch.empty() || !pragma.commit.empty()) PragmaError(absProject, pragma.line, "'branch' and 'commit' only apply to a git source");
|
|
// Relative to the project file, not the cwd: LoadProject runs
|
|
// from wherever the consumer was invoked.
|
|
fs::path local = (absProject.parent_path() / pragma.source).lexically_normal();
|
|
if (fs::is_directory(local)) {
|
|
local /= pragma.project.empty() ? fs::path("project.cpp") : fs::path(pragma.project);
|
|
} else if (!pragma.project.empty()) {
|
|
PragmaError(absProject, pragma.line, "'project' only applies to a directory source");
|
|
}
|
|
if (!fs::exists(local)) PragmaError(absProject, pragma.line, std::format("no such project file: {}", local.string()));
|
|
cfg = LocalProject({ .projectFile = local });
|
|
}
|
|
if (std::ranges::contains(out.configs, cfg)) continue;
|
|
|
|
ForceHostTooling(*cfg, host, forced);
|
|
BuildResult built;
|
|
{
|
|
Progress::Task task(std::format("Building project dependency {}", cfg->name));
|
|
built = Build(*cfg, depResults, depMutex);
|
|
}
|
|
if (!built.result.empty()) PragmaError(absProject, pragma.line, std::format("project dependency '{}' failed to build: {}", cfg->name, built.result));
|
|
out.configs.push_back(cfg);
|
|
libs.insert(built.libs.begin(), built.libs.end());
|
|
|
|
// The module path needs the dependency and everything it depends
|
|
// on, same as GetCompileCommand does for a consumer; the newest
|
|
// artifact over the same set is what project.so is stale against.
|
|
std::function<void(Configuration*)> walk = [&](Configuration* c) {
|
|
if (!walked.insert(c).second) return;
|
|
out.compileFlags += std::format(" -fprebuilt-module-path={}", c->PcmDir().string());
|
|
std::error_code ec;
|
|
const fs::file_time_type t = fs::last_write_time(ExpectedOutput(*c), ec);
|
|
if (!ec && t > out.newestArtifact) out.newestArtifact = t;
|
|
for (Configuration* sub : c->dependencies) walk(sub);
|
|
};
|
|
walk(cfg);
|
|
}
|
|
for (const std::string& lib : libs) out.linkFlags += std::format(" {}", lib);
|
|
return out;
|
|
}
|
|
|
|
// project.so predates its source, the crafter-build that will load it, or
|
|
// a pre-dependency artifact — any of them means recompiling.
|
|
bool ProjectFileStale(const fs::path& artifact, const fs::path& project, const fs::path& hostExe, const ProjectFileDependencies& deps) {
|
|
if (!fs::exists(artifact)) return true;
|
|
const fs::file_time_type built = fs::last_write_time(artifact);
|
|
return built < fs::last_write_time(project) || built < fs::last_write_time(hostExe) || built < deps.newestArtifact;
|
|
}
|
|
}
|
|
|
|
Configuration* Crafter::ProjectDependency(std::string_view name) {
|
|
if (LoadingFrames.empty()) throw std::runtime_error(std::format("ProjectDependency('{}'): called outside a project file", name));
|
|
const ProjectFileFrame& frame = LoadingFrames.back();
|
|
for (Configuration* cfg : frame.dependencies) {
|
|
if (cfg->name == name) return cfg;
|
|
for (const std::unique_ptr<Module>& iface : cfg->interfaces) {
|
|
if (iface->name == name) return cfg;
|
|
}
|
|
}
|
|
std::string declared;
|
|
for (Configuration* cfg : frame.dependencies) {
|
|
if (!declared.empty()) declared += ", ";
|
|
declared += cfg->name;
|
|
}
|
|
throw std::runtime_error(std::format("ProjectDependency('{}'): not declared by {} (declared: {})", name, frame.project.string(), declared.empty() ? "none" : declared));
|
|
}
|
|
|
|
#if defined(CRAFTER_BUILD_CONFIGURATION_TARGET_x86_64_pc_windows_msvc) || defined(CRAFTER_BUILD_CONFIGURATION_TARGET_x86_64_w64_mingw32)
|
|
CommandResult Crafter::RunCommandChecked(std::string_view cmd) {
|
|
std::array<char, 128> buffer;
|
|
CommandResult result{};
|
|
|
|
std::string with = std::format("cmd /C \"{} 2>&1\"", std::string(cmd));
|
|
|
|
FILE* pipe = _popen(with.c_str(), "r");
|
|
if (!pipe) {
|
|
throw std::runtime_error("_popen() failed!");
|
|
}
|
|
|
|
while (fgets(buffer.data(), static_cast<std::int32_t>(buffer.size()), pipe) != nullptr) {
|
|
result.output += buffer.data();
|
|
}
|
|
|
|
result.exitCode = _pclose(pipe);
|
|
return result;
|
|
}
|
|
|
|
CommandResult Crafter::RunCommandWithTimeout(std::string_view cmd, std::chrono::seconds timeout) {
|
|
CommandResult result{};
|
|
|
|
SECURITY_ATTRIBUTES sa{ sizeof(sa), nullptr, TRUE };
|
|
HANDLE readEnd = nullptr;
|
|
HANDLE writeEnd = nullptr;
|
|
if (!CreatePipe(&readEnd, &writeEnd, &sa, 0)) {
|
|
throw std::runtime_error("CreatePipe failed");
|
|
}
|
|
SetHandleInformation(readEnd, HANDLE_FLAG_INHERIT, 0);
|
|
|
|
// KILL_ON_JOB_CLOSE so the cmd.exe wrapper plus whatever it spawned
|
|
// (the test binary, ssh, etc.) all die when the job handle goes away —
|
|
// that's how we enforce the timeout reliably across the whole tree.
|
|
HANDLE job = CreateJobObjectA(nullptr, nullptr);
|
|
if (!job) {
|
|
CloseHandle(readEnd);
|
|
CloseHandle(writeEnd);
|
|
throw std::runtime_error("CreateJobObject failed");
|
|
}
|
|
JOBOBJECT_EXTENDED_LIMIT_INFORMATION jeli{};
|
|
jeli.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE;
|
|
SetInformationJobObject(job, JobObjectExtendedLimitInformation, &jeli, sizeof(jeli));
|
|
|
|
STARTUPINFOA si{};
|
|
si.cb = sizeof(si);
|
|
si.dwFlags = STARTF_USESTDHANDLES;
|
|
si.hStdOutput = writeEnd;
|
|
si.hStdError = writeEnd;
|
|
si.hStdInput = GetStdHandle(STD_INPUT_HANDLE);
|
|
|
|
PROCESS_INFORMATION pi{};
|
|
// cmd /C "...": gives users the same shell-syntax surface (redirects,
|
|
// &&, ||) as popen("/bin/sh -c …") does on Linux. cmd's special-case
|
|
// /C parsing strips the outer quote pair when the inner text contains
|
|
// additional quotes, which is the common case here.
|
|
std::string wrapped = std::format("cmd /C \"{}\"", std::string(cmd));
|
|
std::vector<char> cmdBuf(wrapped.begin(), wrapped.end());
|
|
cmdBuf.push_back('\0');
|
|
|
|
BOOL ok = CreateProcessA(nullptr, cmdBuf.data(), nullptr, nullptr, TRUE, CREATE_NO_WINDOW | CREATE_SUSPENDED, nullptr, nullptr, &si, &pi);
|
|
if (!ok) {
|
|
CloseHandle(readEnd);
|
|
CloseHandle(writeEnd);
|
|
CloseHandle(job);
|
|
throw std::runtime_error("CreateProcess failed");
|
|
}
|
|
|
|
AssignProcessToJobObject(job, pi.hProcess);
|
|
ResumeThread(pi.hThread);
|
|
// Drop our writer ref so the read end sees EOF once the child (its
|
|
// own dup of writeEnd) exits.
|
|
CloseHandle(writeEnd);
|
|
|
|
std::string output;
|
|
std::jthread reader([&output, readEnd]() {
|
|
char buf[4096];
|
|
DWORD n = 0;
|
|
while (ReadFile(readEnd, buf, sizeof(buf), &n, nullptr) && n > 0) {
|
|
output.append(buf, n);
|
|
}
|
|
});
|
|
|
|
DWORD waitMs = static_cast<DWORD>(std::min<std::int64_t>(static_cast<std::int64_t>(timeout.count()) * 1000LL, static_cast<std::int64_t>(INFINITE) - 1));
|
|
DWORD waitResult = WaitForSingleObject(pi.hProcess, waitMs);
|
|
|
|
if (waitResult == WAIT_TIMEOUT) {
|
|
TerminateJobObject(job, 1);
|
|
WaitForSingleObject(pi.hProcess, INFINITE);
|
|
result.timedOut = true;
|
|
result.exitCode = 124;
|
|
} else {
|
|
DWORD exit = 0;
|
|
GetExitCodeProcess(pi.hProcess, &exit);
|
|
// NTSTATUS error severity (top two bits set) means the process
|
|
// terminated by exception — STATUS_ACCESS_VIOLATION 0xC0000005,
|
|
// STATUS_STACK_OVERFLOW 0xC00000FD, etc. Surface those as crashes
|
|
// so the runner shows 💥 instead of a numeric exit code.
|
|
if ((exit & 0xC0000000) == 0xC0000000) {
|
|
result.crashed = true;
|
|
result.signal = static_cast<std::int32_t>(exit);
|
|
}
|
|
result.exitCode = static_cast<std::int32_t>(exit);
|
|
}
|
|
|
|
reader.join();
|
|
CloseHandle(readEnd);
|
|
CloseHandle(pi.hThread);
|
|
CloseHandle(pi.hProcess);
|
|
CloseHandle(job);
|
|
|
|
result.output = std::move(output);
|
|
return result;
|
|
}
|
|
|
|
fs::path Crafter::GetCacheDir() {
|
|
if (const char* local = std::getenv("LOCALAPPDATA")) {
|
|
return fs::path(local) / "crafter.build";
|
|
}
|
|
|
|
throw std::runtime_error("LOCALAPPDATA not set");
|
|
}
|
|
|
|
namespace {
|
|
constexpr std::array<std::string_view, 11> CrafterBuildModules = {
|
|
"Crafter.Build-Shader",
|
|
"Crafter.Build-Platform",
|
|
"Crafter.Build-Interface",
|
|
"Crafter.Build-Implementation",
|
|
"Crafter.Build-External",
|
|
"Crafter.Build-Clang",
|
|
"Crafter.Build-Test",
|
|
"Crafter.Build-Lint",
|
|
"Crafter.Build-Progress",
|
|
"Crafter.Build-Asset",
|
|
"Crafter.Build",
|
|
};
|
|
}
|
|
#endif
|
|
|
|
#ifdef CRAFTER_BUILD_CONFIGURATION_TARGET_x86_64_pc_windows_msvc
|
|
// Native Windows build via build.cmd: MSVC ABI + libc++ from %LIBCXX_DIR%.
|
|
|
|
std::string Crafter::BuildStdPcm(const Configuration& config, fs::path stdPcm) {
|
|
std::string libcxx = std::getenv("LIBCXX_DIR");
|
|
std::string stdcppm = std::format("{}\\modules\\c++\\v1\\std.cppm", libcxx);
|
|
|
|
CacheLock lock(stdPcm.parent_path());
|
|
if(!fs::exists(stdPcm) || fs::last_write_time(stdPcm) < fs::last_write_time(stdcppm)) {
|
|
return RunBuildCommand(std::format("clang++ --target={} -march={} -mtune={} -isystem %LIBCXX_DIR%\\include\\c++\\v1 -nostdinc++ -nostdlib++ -std=c++26 -Wno-reserved-identifier -Wno-reserved-module-identifier --precompile %LIBCXX_DIR%\\modules\\c++\\v1\\std.cppm -o {}", config.target, config.march, config.mtune, stdPcm.string()));
|
|
}
|
|
return "";
|
|
}
|
|
|
|
std::string Crafter::GetBaseCommand(const Configuration& config) {
|
|
return std::format("clang++ -nostdinc++ -nostdlib++ -isystem %LIBCXX_DIR%\\include\\c++\\v1");
|
|
}
|
|
|
|
namespace {
|
|
void EnsureCrafterBuildPcms(const fs::path& sourceDir, const fs::path& cacheDir) {
|
|
CacheLock lock(cacheDir);
|
|
// Re-derived under the lock: another builder may have refreshed the
|
|
// cache from its own sources while we waited.
|
|
std::string stamp = CrafterBuildSourceStamp(sourceDir, CrafterBuildModules);
|
|
bool upToDate = ReadCacheStamp(cacheDir) == stamp;
|
|
for (std::string_view name : CrafterBuildModules) {
|
|
fs::path cppmPath = sourceDir / std::format("{}.cppm", name);
|
|
fs::path pcmPath = cacheDir / std::format("{}.pcm", name);
|
|
if (!fs::exists(cppmPath)) {
|
|
throw std::runtime_error(std::format("module source {} not found in {} (set CRAFTER_BUILD_HOME)", name, sourceDir.string()));
|
|
}
|
|
if (upToDate && fs::exists(pcmPath)) {
|
|
continue;
|
|
}
|
|
std::string cmd = std::format(
|
|
"clang++ --target=x86_64-pc-windows-msvc -march=native -mtune=native "
|
|
"-std=c++26 -O3 -D CRAFTER_BUILD_DLL_IMPORT "
|
|
"-nostdinc++ -nostdlib++ -isystem %LIBCXX_DIR%\\include\\c++\\v1 "
|
|
"-Wno-reserved-identifier -Wno-reserved-module-identifier "
|
|
"-fprebuilt-module-path={} "
|
|
"--precompile {} -o {}",
|
|
cacheDir.string(), cppmPath.string(), pcmPath.string());
|
|
CommandResult r = Crafter::RunCommandChecked(cmd);
|
|
if (r.exitCode != 0) {
|
|
throw std::runtime_error(std::format("Failed to precompile {} (exit {}): {}", name, r.exitCode, r.output));
|
|
}
|
|
}
|
|
// Last, so an interrupted rebuild leaves the stamp disagreeing with the
|
|
// sources and the next run starts over rather than trusting a half-built
|
|
// set of PCMs.
|
|
if (!upToDate) WriteCacheStamp(cacheDir, stamp);
|
|
}
|
|
}
|
|
|
|
Configuration Crafter::LoadProject(const fs::path& projectFile, std::span<const std::string_view> args) {
|
|
fs::path absProject = fs::canonical(projectFile);
|
|
ProjectFileGuard guard(absProject);
|
|
fs::path buildDir = absProject.parent_path() / "build";
|
|
if (!fs::exists(buildDir)) {
|
|
fs::create_directories(buildDir);
|
|
}
|
|
fs::path dllPath = buildDir / std::format("{}.dll", absProject.stem().string());
|
|
|
|
char hostExeBuf[MAX_PATH];
|
|
DWORD hostExeLen = GetModuleFileNameA(nullptr, hostExeBuf, MAX_PATH);
|
|
if (hostExeLen == 0 || hostExeLen == MAX_PATH) {
|
|
throw std::runtime_error("GetModuleFileName failed");
|
|
}
|
|
fs::path hostExe(std::string(hostExeBuf, hostExeLen));
|
|
|
|
fs::path sourceDir = Crafter::GetCrafterBuildHome();
|
|
|
|
Configuration hostConfig;
|
|
hostConfig.target = "x86_64-pc-windows-msvc";
|
|
hostConfig.march = "native";
|
|
hostConfig.mtune = "native";
|
|
fs::path cacheDir = GetCacheDir() / std::format("{}-{}", hostConfig.target, hostConfig.march);
|
|
if (!fs::exists(cacheDir)) fs::create_directories(cacheDir);
|
|
|
|
std::string stdResult = BuildStdPcm(hostConfig, cacheDir / "std.pcm");
|
|
if (!stdResult.empty()) {
|
|
throw std::runtime_error(std::format("Failed to build std.pcm: {}", stdResult));
|
|
}
|
|
|
|
EnsureCrafterBuildPcms(sourceDir, cacheDir);
|
|
|
|
// See the Linux LoadProject for why this sits here and runs every time.
|
|
ProjectFileDependencies preDeps = ResolveProjectFileDependencies(absProject, hostConfig);
|
|
|
|
bool stale = ProjectFileStale(dllPath, absProject, hostExe, preDeps);
|
|
|
|
if (stale) {
|
|
fs::path crafterBuildLib = hostExe.parent_path() / "crafter-build.lib";
|
|
|
|
std::string compileCmd = std::format(
|
|
"clang++ --target=x86_64-pc-windows-msvc -march=native -mtune=native "
|
|
"-std=c++26 -shared -O3 -Wno-return-type-c-linkage -fuse-ld=lld "
|
|
"-D CRAFTER_BUILD_DLL_IMPORT "
|
|
"-nostdinc++ -nostdlib++ -isystem %LIBCXX_DIR%\\include\\c++\\v1 "
|
|
"-fprebuilt-module-path={}{} "
|
|
"-Wl,/EXPORT:CrafterBuildProject "
|
|
"{} {}{} -o {} -L %LIBCXX_DIR%\\lib -lc++",
|
|
cacheDir.string(), preDeps.compileFlags,
|
|
absProject.string(), crafterBuildLib.string(), preDeps.linkFlags, dllPath.string());
|
|
|
|
std::string result = RunBuildCommand(compileCmd);
|
|
if (!result.empty()) {
|
|
throw std::runtime_error(std::format("Failed to compile project {}: {}", absProject.string(), result));
|
|
}
|
|
}
|
|
|
|
// project.dll links libc++ dynamically (-lc++) and the dll typically lives
|
|
// under %LIBCXX_DIR%\lib. Add it to the loader's search path so the user
|
|
// doesn't have to put it on PATH manually.
|
|
if (const char* libcxx = std::getenv("LIBCXX_DIR"); libcxx && *libcxx) {
|
|
fs::path libcxxLib = fs::path(libcxx) / "lib";
|
|
SetDllDirectoryA(libcxxLib.string().c_str());
|
|
}
|
|
|
|
HMODULE handle = LoadLibraryA(dllPath.string().c_str());
|
|
if (!handle) {
|
|
throw std::runtime_error(std::format("Failed to load project {}: error {}", dllPath.string(), GetLastError()));
|
|
}
|
|
|
|
using ProjectFn = Configuration (*)(std::span<const std::string_view>);
|
|
auto fn = reinterpret_cast<ProjectFn>(GetProcAddress(handle, "CrafterBuildProject"));
|
|
if (!fn) {
|
|
throw std::runtime_error(std::format("CrafterBuildProject not found in {}: error {}", dllPath.string(), GetLastError()));
|
|
}
|
|
|
|
guard.Frame().dependencies = std::move(preDeps.configs);
|
|
return fn(args);
|
|
}
|
|
#endif
|
|
|
|
#ifdef CRAFTER_BUILD_CONFIGURATION_TARGET_x86_64_w64_mingw32
|
|
// Linux→mingw cross-compile run on Windows: mingw ABI + libstdc++ from msys2.
|
|
// Resolves the mingw sysroot at C:\msys64\ucrt64 by default (UCRT runtime,
|
|
// matching the Arch cross-compile that produced this binary). User can
|
|
// override with CRAFTER_MINGW_DIR if they have mingw-w64 installed elsewhere
|
|
// or want the legacy MSVCRT mingw at C:\msys64\mingw64. clang itself is the
|
|
// user's existing standalone clang on PATH.
|
|
|
|
namespace {
|
|
fs::path MingwPrefix() {
|
|
if (const char* env = std::getenv("CRAFTER_MINGW_DIR"); env && *env) {
|
|
return fs::path(env);
|
|
}
|
|
return "C:\\msys64\\ucrt64";
|
|
}
|
|
|
|
std::string MingwGccVersion() {
|
|
fs::path includeRoot = MingwPrefix() / "include" / "c++";
|
|
if (!fs::exists(includeRoot)) {
|
|
throw std::runtime_error(std::format("mingw-w64 not found at {} (install msys2 mingw-w64-x86_64-toolchain or set CRAFTER_MINGW_DIR)", MingwPrefix().string()));
|
|
}
|
|
std::vector<std::string> versions;
|
|
for (const auto& entry : fs::directory_iterator(includeRoot)) {
|
|
if (entry.is_directory()) versions.push_back(entry.path().filename().string());
|
|
}
|
|
if (versions.empty()) {
|
|
throw std::runtime_error(std::format("no C++ versions under {}", includeRoot.string()));
|
|
}
|
|
std::sort(versions.begin(), versions.end(), std::greater<>());
|
|
return versions.front();
|
|
}
|
|
}
|
|
|
|
std::string Crafter::BuildStdPcm(const Configuration& config, fs::path stdPcm) {
|
|
CacheLock lock(stdPcm.parent_path());
|
|
if (config.target == "x86_64-pc-windows-msvc") {
|
|
// MSVC target on mingw host: same MSVC libc++ logic as the
|
|
// native-MSVC LoadProject path. User must have LIBCXX_DIR pointing
|
|
// at a Windows libc++ install.
|
|
const char* libcxxEnv = std::getenv("LIBCXX_DIR");
|
|
if (!libcxxEnv || !*libcxxEnv) {
|
|
return "MSVC target requires LIBCXX_DIR pointing at a Windows libc++ install";
|
|
}
|
|
std::string stdcppm = std::format("{}\\modules\\c++\\v1\\std.cppm", libcxxEnv);
|
|
if (fs::exists(stdPcm) && fs::last_write_time(stdPcm) >= fs::last_write_time(stdcppm)) {
|
|
return "";
|
|
}
|
|
return RunBuildCommand(std::format(
|
|
"clang++ --target={} -march={} -mtune={} -isystem %LIBCXX_DIR%\\include\\c++\\v1 "
|
|
"-nostdinc++ -nostdlib++ -std=c++26 -Wno-reserved-identifier -Wno-reserved-module-identifier "
|
|
"--precompile %LIBCXX_DIR%\\modules\\c++\\v1\\std.cppm -o {}",
|
|
config.target, config.march, config.mtune, stdPcm.string()));
|
|
}
|
|
|
|
// Default: mingw target. Look up mingw-w64 libstdc++ via the msys2 prefix.
|
|
fs::path prefix = MingwPrefix();
|
|
fs::path stdCc = prefix / "include" / "c++" / MingwGccVersion() / "bits" / "std.cc";
|
|
if (!fs::exists(stdCc)) {
|
|
return std::format("std.cc not found at {}", stdCc.string());
|
|
}
|
|
if (fs::exists(stdPcm) && fs::last_write_time(stdPcm) >= fs::last_write_time(stdCc)) {
|
|
return "";
|
|
}
|
|
// Copy std.cc → std.cppm in C++ rather than via cmd's `copy /Y`: keeping
|
|
// it in-process means one less shell round-trip and a std::error_code we
|
|
// can report directly. Held under the cache lock, so a plain filename is
|
|
// safe — no other builder writes this path concurrently.
|
|
fs::path stdCppm = stdPcm.parent_path() / "std.cppm";
|
|
std::error_code ec;
|
|
fs::copy_file(stdCc, stdCppm, fs::copy_options::overwrite_existing, ec);
|
|
if (ec) {
|
|
return std::format("copy {} -> {}: {}", stdCc.string(), stdCppm.string(), ec.message());
|
|
}
|
|
return RunBuildCommand(std::format(
|
|
"clang++ --target={} -march={} -mtune={} "
|
|
"--sysroot=\"{}\" -femulated-tls "
|
|
"-O3 -std=c++26 -Wno-reserved-identifier -Wno-reserved-module-identifier "
|
|
"--precompile \"{}\" -o {}",
|
|
config.target, config.march, config.mtune,
|
|
prefix.string(),
|
|
stdCppm.string(),
|
|
stdPcm.string()));
|
|
}
|
|
|
|
std::string Crafter::GetBaseCommand(const Configuration& config) {
|
|
if (config.target == "x86_64-pc-windows-msvc") {
|
|
return std::format("clang++ -nostdinc++ -nostdlib++ -isystem %LIBCXX_DIR%\\include\\c++\\v1");
|
|
}
|
|
return std::format("clang++ --sysroot=\"{}\" -femulated-tls", MingwPrefix().string());
|
|
}
|
|
|
|
namespace {
|
|
void EnsureCrafterBuildPcms(const fs::path& sourceDir, const fs::path& cacheDir) {
|
|
CacheLock lock(cacheDir);
|
|
// Re-derived under the lock: another builder may have refreshed the
|
|
// cache from its own sources while we waited.
|
|
std::string stamp = CrafterBuildSourceStamp(sourceDir, CrafterBuildModules);
|
|
bool upToDate = ReadCacheStamp(cacheDir) == stamp;
|
|
fs::path prefix = MingwPrefix();
|
|
for (std::string_view name : CrafterBuildModules) {
|
|
fs::path cppmPath = sourceDir / std::format("{}.cppm", name);
|
|
fs::path pcmPath = cacheDir / std::format("{}.pcm", name);
|
|
if (!fs::exists(cppmPath)) {
|
|
throw std::runtime_error(std::format("module source {} not found in {} (set CRAFTER_BUILD_HOME)", name, sourceDir.string()));
|
|
}
|
|
if (upToDate && fs::exists(pcmPath)) {
|
|
continue;
|
|
}
|
|
std::string cmd = std::format(
|
|
"clang++ --target=x86_64-w64-mingw32 -march=native -mtune=native "
|
|
"--sysroot=\"{}\" -femulated-tls "
|
|
"-std=c++26 -O3 -D CRAFTER_BUILD_DLL_IMPORT "
|
|
"-Wno-reserved-identifier -Wno-reserved-module-identifier "
|
|
"-fprebuilt-module-path={} "
|
|
"--precompile {} -o {}",
|
|
prefix.string(), cacheDir.string(), cppmPath.string(), pcmPath.string());
|
|
CommandResult r = Crafter::RunCommandChecked(cmd);
|
|
if (r.exitCode != 0) {
|
|
throw std::runtime_error(std::format("Failed to precompile {} (exit {}): {}", name, r.exitCode, r.output));
|
|
}
|
|
}
|
|
// Last, so an interrupted rebuild leaves the stamp disagreeing with the
|
|
// sources and the next run starts over rather than trusting a half-built
|
|
// set of PCMs.
|
|
if (!upToDate) WriteCacheStamp(cacheDir, stamp);
|
|
}
|
|
}
|
|
|
|
Configuration Crafter::LoadProject(const fs::path& projectFile, std::span<const std::string_view> args) {
|
|
fs::path absProject = fs::canonical(projectFile);
|
|
ProjectFileGuard guard(absProject);
|
|
fs::path buildDir = absProject.parent_path() / "build";
|
|
if (!fs::exists(buildDir)) fs::create_directories(buildDir);
|
|
fs::path dllPath = buildDir / std::format("{}.dll", absProject.stem().string());
|
|
|
|
char hostExeBuf[MAX_PATH];
|
|
DWORD hostExeLen = GetModuleFileNameA(nullptr, hostExeBuf, MAX_PATH);
|
|
if (hostExeLen == 0 || hostExeLen == MAX_PATH) {
|
|
throw std::runtime_error("GetModuleFileName failed");
|
|
}
|
|
fs::path hostExe(std::string(hostExeBuf, hostExeLen));
|
|
|
|
fs::path sourceDir = Crafter::GetCrafterBuildHome();
|
|
|
|
Configuration hostConfig;
|
|
hostConfig.target = "x86_64-w64-mingw32";
|
|
hostConfig.march = "native";
|
|
hostConfig.mtune = "native";
|
|
fs::path cacheDir = GetCacheDir() / std::format("{}-{}", hostConfig.target, hostConfig.march);
|
|
if (!fs::exists(cacheDir)) fs::create_directories(cacheDir);
|
|
|
|
std::string stdResult = BuildStdPcm(hostConfig, cacheDir / "std.pcm");
|
|
if (!stdResult.empty()) {
|
|
throw std::runtime_error(std::format("Failed to build std.pcm: {}", stdResult));
|
|
}
|
|
|
|
EnsureCrafterBuildPcms(sourceDir, cacheDir);
|
|
|
|
// See the Linux LoadProject for why this sits here and runs every time.
|
|
ProjectFileDependencies preDeps = ResolveProjectFileDependencies(absProject, hostConfig);
|
|
|
|
bool stale = ProjectFileStale(dllPath, absProject, hostExe, preDeps);
|
|
|
|
if (stale) {
|
|
// The import lib lives next to the launcher exe (Build()'s mingw
|
|
// copy step puts libcrafter-build.dll.a there) so -L<exe-dir>
|
|
// -lcrafter-build resolves it.
|
|
fs::path prefix = MingwPrefix();
|
|
std::string compileCmd = std::format(
|
|
"clang++ --target=x86_64-w64-mingw32 -march=native -mtune=native "
|
|
"--sysroot=\"{}\" -femulated-tls "
|
|
"-std=c++26 -shared -O3 -Wno-return-type-c-linkage -fuse-ld=lld "
|
|
"-D CRAFTER_BUILD_DLL_IMPORT "
|
|
"-fprebuilt-module-path={}{} "
|
|
// mingw-lld doesn't accept /EXPORT:NAME — use the catch-all so
|
|
// CrafterBuildProject is reachable via GetProcAddress without
|
|
// forcing project.cpp templates to add __declspec(dllexport).
|
|
"-Wl,--export-all-symbols "
|
|
// Dependency archives go before -lcrafter-build: they reference
|
|
// Crafter.Build symbols, and ld.lld resolves archives in order.
|
|
"{} -o {}{} -L\"{}\" -lcrafter-build -lstdc++exp -lpthread",
|
|
prefix.string(),
|
|
cacheDir.string(), preDeps.compileFlags,
|
|
absProject.string(), dllPath.string(), preDeps.linkFlags,
|
|
hostExe.parent_path().string());
|
|
|
|
std::string result = RunBuildCommand(compileCmd);
|
|
if (!result.empty()) {
|
|
throw std::runtime_error(std::format("Failed to compile project {}: {}", absProject.string(), result));
|
|
}
|
|
}
|
|
|
|
// project.dll is a mingw DLL linking libstdc++-6.dll / libgcc_s_seh-1.dll /
|
|
// libwinpthread-1.dll dynamically (LoadProject's compile uses -lstdc++exp
|
|
// -lpthread, not -static). We already know where they live from
|
|
// MingwPrefix(), so add bin/ to the loader's search path here instead of
|
|
// requiring the user to prepend it to PATH manually.
|
|
fs::path mingwBin = MingwPrefix() / "bin";
|
|
SetDllDirectoryA(mingwBin.string().c_str());
|
|
|
|
HMODULE handle = LoadLibraryA(dllPath.string().c_str());
|
|
if (!handle) {
|
|
throw std::runtime_error(std::format("Failed to load project {}: error {}", dllPath.string(), GetLastError()));
|
|
}
|
|
|
|
using ProjectFn = Configuration (*)(std::span<const std::string_view>);
|
|
auto fn = reinterpret_cast<ProjectFn>(GetProcAddress(handle, "CrafterBuildProject"));
|
|
if (!fn) {
|
|
throw std::runtime_error(std::format("CrafterBuildProject not found in {}: error {}", dllPath.string(), GetLastError()));
|
|
}
|
|
|
|
guard.Frame().dependencies = std::move(preDeps.configs);
|
|
return fn(args);
|
|
}
|
|
#endif
|
|
|
|
#ifdef CRAFTER_BUILD_CONFIGURATION_TARGET_x86_64_pc_linux_gnu
|
|
|
|
CommandResult Crafter::RunCommandChecked(std::string_view cmd) {
|
|
std::array<char, 128> buffer;
|
|
CommandResult result{};
|
|
|
|
std::string with = std::format("{} 2>&1", cmd);
|
|
FILE* pipe = popen(with.c_str(), "r");
|
|
if (!pipe) throw std::runtime_error("popen() failed!");
|
|
|
|
while (fgets(buffer.data(), buffer.size(), pipe) != nullptr) {
|
|
result.output += buffer.data();
|
|
}
|
|
|
|
std::int32_t status = pclose(pipe);
|
|
if (WIFEXITED(status)) {
|
|
result.exitCode = WEXITSTATUS(status);
|
|
} else if (WIFSIGNALED(status)) {
|
|
result.signal = WTERMSIG(status);
|
|
result.exitCode = 128 + result.signal;
|
|
result.crashed = true;
|
|
} else {
|
|
result.exitCode = -1;
|
|
}
|
|
return result;
|
|
}
|
|
|
|
CommandResult Crafter::RunCommandWithTimeout(std::string_view cmd, std::chrono::seconds timeout) {
|
|
std::array<char, 128> buffer;
|
|
CommandResult result{};
|
|
|
|
std::string wrapped = std::format("timeout --kill-after=2 {} {} 2>&1", timeout.count(), cmd);
|
|
|
|
FILE* pipe = popen(wrapped.c_str(), "r");
|
|
if (!pipe) throw std::runtime_error("popen() failed!");
|
|
|
|
while (fgets(buffer.data(), buffer.size(), pipe) != nullptr) {
|
|
result.output += buffer.data();
|
|
}
|
|
|
|
std::int32_t status = pclose(pipe);
|
|
if (WIFEXITED(status)) {
|
|
std::int32_t code = WEXITSTATUS(status);
|
|
if (code == 124) {
|
|
result.timedOut = true;
|
|
result.exitCode = 124;
|
|
} else if (code >= 128) {
|
|
result.signal = code - 128;
|
|
result.exitCode = code;
|
|
result.crashed = true;
|
|
} else {
|
|
result.exitCode = code;
|
|
}
|
|
} else if (WIFSIGNALED(status)) {
|
|
result.signal = WTERMSIG(status);
|
|
result.exitCode = 128 + result.signal;
|
|
result.crashed = true;
|
|
} else {
|
|
result.exitCode = -1;
|
|
}
|
|
return result;
|
|
}
|
|
|
|
std::string Crafter::BuildStdPcm(const Configuration& config, fs::path stdPcm) {
|
|
CacheLock lock(stdPcm.parent_path());
|
|
if(config.target == "x86_64-w64-mingw32") {
|
|
std::vector<std::string> folders;
|
|
|
|
// Iterate through the directory and collect all subdirectories
|
|
for (const auto& entry : fs::directory_iterator("/usr/x86_64-w64-mingw32/include/c++")) {
|
|
if (entry.is_directory()) {
|
|
folders.push_back(entry.path().filename().string());
|
|
}
|
|
}
|
|
|
|
// Sort the folders by version in descending order
|
|
std::sort(folders.begin(), folders.end(), [](const std::string& a, const std::string& b) {
|
|
return std::lexicographical_compare(b.begin(), b.end(), a.begin(), a.end());
|
|
});
|
|
|
|
std::string mingWversion = folders.front();
|
|
|
|
fs::path stdCc = fs::path(std::format("/usr/x86_64-w64-mingw32/include/c++/{}/bits/std.cc", mingWversion));
|
|
|
|
if(!fs::exists(stdPcm) || fs::last_write_time(stdPcm) < fs::last_write_time(stdCc)) {
|
|
// -femulated-tls keeps PCM TLS access matching libstdc++'s emutls
|
|
// definitions; mismatch surfaces as undefined std::__once_callable.
|
|
// The std.cc → std.cppm copy uses a plain filename: held under the
|
|
// cache lock, no other builder writes this path concurrently.
|
|
fs::path stdCppm = stdPcm.parent_path() / "std.cppm";
|
|
std::error_code copyEc;
|
|
fs::copy_file(stdCc, stdCppm, fs::copy_options::overwrite_existing, copyEc);
|
|
if (copyEc) {
|
|
return std::format("copy {} -> {}: {}", stdCc.string(), stdCppm.string(), copyEc.message());
|
|
}
|
|
return RunBuildCommand(std::format("clang++ --target={} -march={} -mtune={} -femulated-tls -O3 -std=c++26 -Wno-reserved-identifier -Wno-reserved-module-identifier --precompile {} -o {}", config.target, config.march, config.mtune, stdCppm.string(), stdPcm.string()));
|
|
} else {
|
|
return "";
|
|
}
|
|
} else {
|
|
const bool isWasm = config.target.starts_with("wasm32");
|
|
// wasi-sdk drops std.cppm at <sysroot>/share/libc++/v1/, the rest of
|
|
// the libc++ ecosystem (e.g. /opt/aarch64-rootfs) follows FHS at
|
|
// <sysroot>/usr/share/libc++/v1/.
|
|
std::string stdCppm;
|
|
if (isWasm) {
|
|
stdCppm = std::format("{}/share/libc++/v1/std.cppm", config.sysroot);
|
|
} else if (config.sysroot.empty()) {
|
|
stdCppm = "/usr/share/libc++/v1/std.cppm";
|
|
} else {
|
|
stdCppm = std::format("{}/usr/share/libc++/v1/std.cppm", config.sysroot);
|
|
// not every distro ships the std module sources (Alpine's
|
|
// libc++-dev doesn't); the host copy is interchangeable as long
|
|
// as the libc++ versions line up, which the compile of the PCM
|
|
// against the sysroot's headers verifies anyway
|
|
if (!fs::exists(stdCppm) && fs::exists("/usr/share/libc++/v1/std.cppm")) {
|
|
stdCppm = "/usr/share/libc++/v1/std.cppm";
|
|
}
|
|
}
|
|
std::string sysrootFlag = config.sysroot.empty()
|
|
? std::string()
|
|
: std::format(" --sysroot={}", config.sysroot);
|
|
// wasm32 rejects -march. wasi-libc++ headers require these baselines
|
|
// to compile: setjmp.h needs -mllvm -wasm-enable-sjlj (lowered to wasm
|
|
// EH); signal.h requires the emulation define; and EH itself isn't
|
|
// wired up so -fno-exceptions stays.
|
|
std::string archFlags = isWasm
|
|
? std::string(" -fno-exceptions -msimd128 -fno-c++-static-destructors -mllvm -wasm-enable-sjlj -D_WASI_EMULATED_SIGNAL")
|
|
: std::format(" -march={} -mtune={}", config.march, config.mtune);
|
|
// Match the variant codegen flags the consuming TUs are built with
|
|
// (e.g. -mrelaxed-simd) so the std BMI is feature-compatible. wasm only;
|
|
// wasmVariantFlags is empty on every other path.
|
|
if (isWasm) {
|
|
for (const std::string& f : config.wasmVariantFlags) archFlags += std::format(" {}", f);
|
|
}
|
|
// non-wasm cross builds: the driver ranks the host toolchain's own
|
|
// libc++ headers (<install>/../include/c++/v1) above --sysroot, so
|
|
// the PCM would mix host libc++ with the foreign libc — force the
|
|
// sysroot's copy (same trick as the Windows LIBCXX_DIR path)
|
|
if (!isWasm && !config.sysroot.empty()) {
|
|
archFlags += std::format(" -nostdinc++ -isystem {}/usr/include/c++/v1", config.sysroot);
|
|
}
|
|
if(!fs::exists(stdPcm) || fs::last_write_time(stdPcm) < fs::last_write_time(stdCppm)) {
|
|
return RunBuildCommand(std::format("clang++ --target={} -std=c++26 -stdlib=libc++{}{} -O3 -Wno-reserved-identifier -Wno-reserved-module-identifier --precompile {} -o {}", config.target, sysrootFlag, archFlags, stdCppm, stdPcm.string()));
|
|
} else {
|
|
return "";
|
|
}
|
|
}
|
|
}
|
|
|
|
fs::path Crafter::GetCacheDir() {
|
|
if (const char* xdg = std::getenv("XDG_CACHE_HOME"); xdg && *xdg) {
|
|
return fs::path(xdg) / "crafter.build";
|
|
}
|
|
|
|
if (const char* home = std::getenv("HOME")) {
|
|
return fs::path(home) / ".cache" / "crafter.build";
|
|
}
|
|
|
|
throw std::runtime_error("Neither XDG_CACHE_HOME nor HOME set");
|
|
}
|
|
|
|
std::string Crafter::GetBaseCommand(const Configuration& config) {
|
|
std::string stdlib;
|
|
if(config.target == "x86_64-w64-mingw32") {
|
|
stdlib = "";
|
|
} else {
|
|
stdlib = "-stdlib=libc++";
|
|
}
|
|
return std::format("clang++ {}", stdlib);
|
|
}
|
|
|
|
namespace {
|
|
constexpr std::array<std::string_view, 11> CrafterBuildModules = {
|
|
"Crafter.Build-Shader",
|
|
"Crafter.Build-Platform",
|
|
"Crafter.Build-Interface",
|
|
"Crafter.Build-Implementation",
|
|
"Crafter.Build-External",
|
|
"Crafter.Build-Clang",
|
|
"Crafter.Build-Test",
|
|
"Crafter.Build-Lint",
|
|
"Crafter.Build-Progress",
|
|
"Crafter.Build-Asset",
|
|
"Crafter.Build",
|
|
};
|
|
|
|
void EnsureCrafterBuildPcms(const fs::path& sourceDir, const fs::path& cacheDir) {
|
|
CacheLock lock(cacheDir);
|
|
// Re-derived under the lock: another builder may have refreshed the
|
|
// cache from its own sources while we waited.
|
|
std::string stamp = CrafterBuildSourceStamp(sourceDir, CrafterBuildModules);
|
|
const bool upToDate = ReadCacheStamp(cacheDir) == stamp;
|
|
for (std::string_view name : CrafterBuildModules) {
|
|
fs::path cppmPath = sourceDir / std::format("{}.cppm", name);
|
|
fs::path pcmPath = cacheDir / std::format("{}.pcm", name);
|
|
if (!fs::exists(cppmPath)) {
|
|
throw std::runtime_error(std::format("module source {} not found in {} (set CRAFTER_BUILD_HOME)", name, sourceDir.string()));
|
|
}
|
|
if (upToDate && fs::exists(pcmPath)) {
|
|
continue;
|
|
}
|
|
std::string cmd = std::format(
|
|
"clang++ --target=x86_64-pc-linux-gnu -march=native -mtune=native "
|
|
"-std=c++26 -stdlib=libc++ -O3 "
|
|
"-Wno-reserved-identifier -Wno-reserved-module-identifier "
|
|
"-fprebuilt-module-path={} "
|
|
"--precompile {} -o {}",
|
|
cacheDir.string(), cppmPath.string(), pcmPath.string());
|
|
CommandResult r = Crafter::RunCommandChecked(cmd);
|
|
if (r.exitCode != 0) {
|
|
throw std::runtime_error(std::format("Failed to precompile {} (exit {}): {}", name, r.exitCode, r.output));
|
|
}
|
|
}
|
|
// Last, so an interrupted rebuild leaves the stamp disagreeing with the
|
|
// sources and the next run starts over rather than trusting a half-built
|
|
// set of PCMs.
|
|
if (!upToDate) WriteCacheStamp(cacheDir, stamp);
|
|
}
|
|
}
|
|
|
|
Configuration Crafter::LoadProject(const fs::path& projectFile, std::span<const std::string_view> args) {
|
|
fs::path absProject = fs::canonical(projectFile);
|
|
ProjectFileGuard guard(absProject);
|
|
fs::path buildDir = absProject.parent_path() / "build";
|
|
if (!fs::exists(buildDir)) {
|
|
fs::create_directories(buildDir);
|
|
}
|
|
fs::path soPath = buildDir / std::format("{}.so", absProject.stem().string());
|
|
|
|
fs::path hostExe = fs::read_symlink("/proc/self/exe");
|
|
|
|
fs::path sourceDir = Crafter::GetCrafterBuildHome();
|
|
|
|
Configuration hostConfig;
|
|
hostConfig.target = "x86_64-pc-linux-gnu";
|
|
hostConfig.march = "native";
|
|
hostConfig.mtune = "native";
|
|
fs::path cacheDir = GetCacheDir() / std::format("{}-{}", hostConfig.target, hostConfig.march);
|
|
if (!fs::exists(cacheDir)) fs::create_directories(cacheDir);
|
|
|
|
std::string stdResult = BuildStdPcm(hostConfig, cacheDir / "std.pcm");
|
|
if (!stdResult.empty()) {
|
|
throw std::runtime_error(std::format("Failed to build std.pcm: {}", stdResult));
|
|
}
|
|
|
|
EnsureCrafterBuildPcms(sourceDir, cacheDir);
|
|
|
|
// After the host PCMs exist (a dependency's build imports Crafter.Build
|
|
// from that cache) and before the staleness check (its artifacts are
|
|
// inputs to project.so). Built on every load, fresh or not: a dependency
|
|
// BMI whose own imports were rebuilt underneath it doesn't fail cleanly,
|
|
// clang crashes on it.
|
|
ProjectFileDependencies preDeps = ResolveProjectFileDependencies(absProject, hostConfig);
|
|
|
|
const bool stale = ProjectFileStale(soPath, absProject, hostExe, preDeps);
|
|
|
|
if (stale) {
|
|
std::string compileCmd = std::format(
|
|
"clang++ --target=x86_64-pc-linux-gnu -march=native -mtune=native "
|
|
"-std=c++26 -stdlib=libc++ -shared -fPIC -O3 "
|
|
"-Wno-return-type-c-linkage "
|
|
"-fprebuilt-module-path={}{} "
|
|
// lld: a dependency archive built in release holds ThinLTO
|
|
// bitcode, which GNU ld can't read. Every Build() link already
|
|
// requires lld.
|
|
"{} -o {}{} -fuse-ld=lld",
|
|
cacheDir.string(), preDeps.compileFlags,
|
|
absProject.string(), soPath.string(), preDeps.linkFlags);
|
|
|
|
std::string result = RunBuildCommand(compileCmd);
|
|
if (!result.empty()) {
|
|
throw std::runtime_error(std::format("Failed to compile project {}: {}", absProject.string(), result));
|
|
}
|
|
}
|
|
|
|
void* handle = dlopen(soPath.c_str(), RTLD_NOW | RTLD_GLOBAL);
|
|
if (!handle) {
|
|
throw std::runtime_error(std::format("Failed to load project {}: {}", soPath.string(), dlerror()));
|
|
}
|
|
|
|
using ProjectFn = Configuration (*)(std::span<const std::string_view>);
|
|
dlerror();
|
|
auto fn = reinterpret_cast<ProjectFn>(dlsym(handle, "CrafterBuildProject"));
|
|
if (const char* err = dlerror()) {
|
|
throw std::runtime_error(std::format("CrafterBuildProject not found in {}: {}", soPath.string(), err));
|
|
}
|
|
|
|
guard.Frame().dependencies = std::move(preDeps.configs);
|
|
return fn(args);
|
|
}
|
|
|
|
#endif
|