Merge branch 'claude/project-dependencies': project.cpp dependencies, AGENTS.md
Some checks failed
CI / build-test-release (push) Failing after 4m7s
CI / release-musl (push) Has been skipped

Brings in `#pragma crafter dependency`, the switch to Crafter.LintRules
for the house style, and the contributor guide in AGENTS.md.

The branch predates the musl host work on master, and both rewrote the
command that compiles project.so, so the merge reconciles them by hand:

- LoadProject compiles project.so for hostConfig.target (HostTarget(),
  from master) with the pre-dependency module path and archives from
  the branch. -fuse-ld=lld appears once, with both reasons in the
  comment.
- ForceHostTooling added -fPIC only for the literal x86_64-pc-linux-gnu
  triple. hostConfig now comes from HostTarget(), so on Alpine the host
  is x86_64-alpine-linux-musl and the dependency archive would not have
  been position-independent. It now matches any *-linux-gnu or
  *-linux-musl host, as project.cpp and the cmake flags do since the
  musl change, and its comment no longer argues against HostTarget().
- The ProjectDependency test compared the dependency's target against
  the glibc triple, which the musl CI job would fail; it compares
  against HostTarget().
- README keeps master's paragraph on the bootstrap triple. AGENTS.md
  describes the test block as `if (linuxTarget)`, which is what
  project.cpp has now.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Jorijn van der Graaf 2026-10-02 20:47:23 +02:00
commit c3d97be59e
Signed by: jorijnvdgraaf
GPG key ID: 2937E59CDCC1BCFB
18 changed files with 784 additions and 1635 deletions

View file

@ -20,6 +20,8 @@ module Crafter.Build:Platform_impl;
import std;
import :Platform;
import :Clang;
import :Interface;
import :External;
import :Progress;
namespace fs = std::filesystem;
using namespace Crafter;
@ -286,6 +288,272 @@ std::string Crafter::RunBuildCommand(std::string_view cmd) {
return r.output;
}
// --- Dependencies of project.cpp itself -------------------------------------
//
// #pragma crafter dependency "<source>" [branch "<b>" | commit "<sha>"] [project "<file>"]
//
// names a sibling crafter-build project that project.cpp imports a module
// from. Anything a file needs before it can compile cannot be C++ inside that
// file, so this is a directive next to the `import` it enables — where
// `#include` lives. Each one is fetched, built for the host, and put on
// project.so's module path and link line before project.cpp is compiled. The
// dependency's own project.cpp goes through LoadProject, so its pragmas
// resolve recursively.
namespace {
struct PragmaDependency {
std::string source;
std::string branch;
std::string commit;
std::string project;
std::size_t line = 0;
};
[[noreturn]] void PragmaError(const fs::path& file, std::size_t line, std::string_view message) {
throw std::runtime_error(std::format("{}:{}: {}", file.string(), line, message));
}
// The tokens of one directive: bare words and double-quoted strings (no
// escapes — a path or URL never needs one). Reading stops at `//`.
class PragmaCursor {
public:
PragmaCursor(std::string_view text, const fs::path& file, std::size_t line) : text_(text), file_(file), line_(line) {}
bool AtEnd() {
SkipSpace();
return pos_ >= text_.size() || text_.substr(pos_).starts_with("//");
}
bool Take(char c) {
SkipSpace();
if (pos_ >= text_.size() || text_[pos_] != c) return false;
++pos_;
return true;
}
std::string_view Word() {
SkipSpace();
const std::size_t start = pos_;
while (pos_ < text_.size() && IsWordChar(text_[pos_])) ++pos_;
return text_.substr(start, pos_ - start);
}
std::optional<std::string_view> Quoted() {
SkipSpace();
if (pos_ >= text_.size() || text_[pos_] != '"') return std::nullopt;
const std::size_t close = text_.find('"', pos_ + 1);
if (close == std::string_view::npos) Fail("unterminated string");
const std::string_view inner = text_.substr(pos_ + 1, close - pos_ - 1);
pos_ = close + 1;
return inner;
}
[[noreturn]] void Fail(std::string_view message) const { PragmaError(file_, line_, message); }
private:
static bool IsWordChar(char c) { return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_'; }
void SkipSpace() {
while (pos_ < text_.size() && (text_[pos_] == ' ' || text_[pos_] == '\t' || text_[pos_] == '\r')) ++pos_;
}
std::string_view text_;
const fs::path& file_;
std::size_t line_;
std::size_t pos_ = 0;
};
// One directive per line, anywhere in the file. Comments aren't parsed:
// a directive inside a block comment is honoured, and `//` lines fall out
// because they don't start with `#`.
std::vector<PragmaDependency> ScanProjectFilePragmas(const fs::path& absProject) {
std::vector<PragmaDependency> found;
std::ifstream in(absProject);
std::string raw;
for (std::size_t lineNo = 1; std::getline(in, raw); ++lineNo) {
PragmaCursor cur(raw, absProject, lineNo);
if (!cur.Take('#') || cur.Word() != "pragma" || cur.Word() != "crafter") continue;
const std::string_view verb = cur.Word();
if (verb != "dependency") cur.Fail(std::format("unknown crafter pragma '{}'", verb));
PragmaDependency dep;
dep.line = lineNo;
const std::optional<std::string_view> source = cur.Quoted();
if (!source) cur.Fail("expected a quoted source after 'dependency'");
dep.source = *source;
while (!cur.AtEnd()) {
const std::string_view key = cur.Word();
std::string* field = key == "branch" ? &dep.branch : key == "commit" ? &dep.commit : key == "project" ? &dep.project : nullptr;
if (!field) cur.Fail(key.empty() ? "unexpected text after the source" : std::format("unknown keyword '{}' (expected branch, commit or project)", key));
if (!field->empty()) cur.Fail(std::format("'{}' given twice", key));
const std::optional<std::string_view> value = cur.Quoted();
if (!value || value->empty()) cur.Fail(std::format("expected a quoted value after '{}'", key));
*field = *value;
}
if (dep.source.empty()) cur.Fail("the source is empty");
if (!dep.branch.empty() && !dep.commit.empty()) cur.Fail("'branch' and 'commit' are mutually exclusive");
found.push_back(std::move(dep));
}
return found;
}
struct ProjectFileFrame {
fs::path project;
std::vector<Configuration*> dependencies;
};
// LoadProject nests: a dependency's project.cpp loads inside its
// consumer's pre-pass, and GitProject inside CrafterBuildProject nests
// again. Several threads may also load at once (ConcurrentCacheRace), so
// the stack is per thread. ProjectDependency() reads the top frame.
thread_local std::vector<ProjectFileFrame> LoadingFrames;
// Held for the whole of one LoadProject call. Pushing a file that is
// already on the stack is a cycle, which used to recurse forever.
class ProjectFileGuard {
public:
explicit ProjectFileGuard(const fs::path& absProject) {
for (const ProjectFileFrame& frame : LoadingFrames) {
if (frame.project != absProject) continue;
std::string chain;
for (const ProjectFileFrame& f : LoadingFrames) chain += std::format("{} -> ", f.project.string());
throw std::runtime_error(std::format("circular project dependency: {}{}", chain, absProject.string()));
}
index_ = LoadingFrames.size();
LoadingFrames.push_back({ absProject, {} });
}
~ProjectFileGuard() { LoadingFrames.pop_back(); }
ProjectFileGuard(const ProjectFileGuard&) = delete;
ProjectFileGuard& operator=(const ProjectFileGuard&) = delete;
ProjectFileFrame& Frame() { return LoadingFrames[index_]; }
private:
std::size_t index_ = 0;
};
// A pre-dependency is host tooling: project.so is loaded into the running
// crafter-build, so the dependency is built for the host and against the
// same PCM cache LoadProject populates — the hostConfig LoadProject itself
// compiles project.so with, so the dependency's std/Crafter.Build PCMs
// land in the cache dir project.so is built against. Recursive, because
// the dependency's project.cpp forwards no arguments but still reads
// CRAFTER_BUILD_MARCH from the environment for its own deps.
void ForceHostTooling(Configuration& cfg, const Configuration& host, std::unordered_set<Configuration*>& seen) {
if (!seen.insert(&cfg).second) return;
cfg.target = host.target;
cfg.march = host.march;
cfg.mtune = host.mtune;
cfg.debug = false;
// project.so is a shared object, so the archive it links has to be
// position-independent; static libraries aren't by default. Part of
// VariantId, so this build never shares a bin dir with a normal one.
// Checked before pushing: the same Configuration comes back from the
// project cache for every consumer in this process. Linux hosts only,
// glibc or musl: Windows targets have no use for the flag.
const bool linuxHost = host.target.ends_with("-linux-gnu") || host.target.ends_with("-linux-musl");
if (linuxHost && !std::ranges::contains(cfg.compileFlags, "-fPIC")) {
cfg.compileFlags.emplace_back("-fPIC");
}
for (Configuration* dep : cfg.dependencies) ForceHostTooling(*dep, host, seen);
}
struct ProjectFileDependencies {
std::vector<Configuration*> configs;
std::string compileFlags; // " -fprebuilt-module-path=…" for each dependency and its transitive deps
std::string linkFlags; // every dependency's BuildResult.libs, verbatim
fs::file_time_type newestArtifact = fs::file_time_type::min();
};
bool IsGitSource(std::string_view source) {
return source.contains("://") || source.starts_with("git@");
}
ProjectFileDependencies ResolveProjectFileDependencies(const fs::path& absProject, const Configuration& host) {
ProjectFileDependencies out;
const std::vector<PragmaDependency> pragmas = ScanProjectFilePragmas(absProject);
if (pragmas.empty()) return out;
std::unordered_map<fs::path, std::shared_future<BuildResult>> depResults;
std::mutex depMutex;
std::unordered_set<Configuration*> forced;
std::unordered_set<Configuration*> walked;
std::unordered_set<std::string> libs;
for (const PragmaDependency& pragma : pragmas) {
Configuration* cfg = nullptr;
if (IsGitSource(pragma.source)) {
GitProjectSpec spec;
spec.source = { pragma.source, pragma.branch, pragma.commit };
if (!pragma.project.empty()) spec.projectFile = pragma.project;
cfg = GitProject(spec);
} else {
if (!pragma.branch.empty() || !pragma.commit.empty()) PragmaError(absProject, pragma.line, "'branch' and 'commit' only apply to a git source");
// Relative to the project file, not the cwd: LoadProject runs
// from wherever the consumer was invoked.
fs::path local = (absProject.parent_path() / pragma.source).lexically_normal();
if (fs::is_directory(local)) {
local /= pragma.project.empty() ? fs::path("project.cpp") : fs::path(pragma.project);
} else if (!pragma.project.empty()) {
PragmaError(absProject, pragma.line, "'project' only applies to a directory source");
}
if (!fs::exists(local)) PragmaError(absProject, pragma.line, std::format("no such project file: {}", local.string()));
cfg = LocalProject({ .projectFile = local });
}
if (std::ranges::contains(out.configs, cfg)) continue;
ForceHostTooling(*cfg, host, forced);
BuildResult built;
{
Progress::Task task(std::format("Building project dependency {}", cfg->name));
built = Build(*cfg, depResults, depMutex);
}
if (!built.result.empty()) PragmaError(absProject, pragma.line, std::format("project dependency '{}' failed to build: {}", cfg->name, built.result));
out.configs.push_back(cfg);
libs.insert(built.libs.begin(), built.libs.end());
// The module path needs the dependency and everything it depends
// on, same as GetCompileCommand does for a consumer; the newest
// artifact over the same set is what project.so is stale against.
std::function<void(Configuration*)> walk = [&](Configuration* c) {
if (!walked.insert(c).second) return;
out.compileFlags += std::format(" -fprebuilt-module-path={}", c->PcmDir().string());
std::error_code ec;
const fs::file_time_type t = fs::last_write_time(ExpectedOutput(*c), ec);
if (!ec && t > out.newestArtifact) out.newestArtifact = t;
for (Configuration* sub : c->dependencies) walk(sub);
};
walk(cfg);
}
for (const std::string& lib : libs) out.linkFlags += std::format(" {}", lib);
return out;
}
// project.so predates its source, the crafter-build that will load it, or
// a pre-dependency artifact — any of them means recompiling.
bool ProjectFileStale(const fs::path& artifact, const fs::path& project, const fs::path& hostExe, const ProjectFileDependencies& deps) {
if (!fs::exists(artifact)) return true;
const fs::file_time_type built = fs::last_write_time(artifact);
return built < fs::last_write_time(project) || built < fs::last_write_time(hostExe) || built < deps.newestArtifact;
}
}
Configuration* Crafter::ProjectDependency(std::string_view name) {
if (LoadingFrames.empty()) throw std::runtime_error(std::format("ProjectDependency('{}'): called outside a project file", name));
const ProjectFileFrame& frame = LoadingFrames.back();
for (Configuration* cfg : frame.dependencies) {
if (cfg->name == name) return cfg;
for (const std::unique_ptr<Module>& iface : cfg->interfaces) {
if (iface->name == name) return cfg;
}
}
std::string declared;
for (Configuration* cfg : frame.dependencies) {
if (!declared.empty()) declared += ", ";
declared += cfg->name;
}
throw std::runtime_error(std::format("ProjectDependency('{}'): not declared by {} (declared: {})", name, frame.project.string(), declared.empty() ? "none" : declared));
}
#if defined(CRAFTER_BUILD_CONFIGURATION_TARGET_x86_64_pc_windows_msvc) || defined(CRAFTER_BUILD_CONFIGURATION_TARGET_x86_64_w64_mingw32)
CommandResult Crafter::RunCommandChecked(std::string_view cmd) {
std::array<char, 128> buffer;
@ -482,6 +750,7 @@ namespace {
Configuration Crafter::LoadProject(const fs::path& projectFile, std::span<const std::string_view> args) {
fs::path absProject = fs::canonical(projectFile);
ProjectFileGuard guard(absProject);
fs::path buildDir = absProject.parent_path() / "build";
if (!fs::exists(buildDir)) {
fs::create_directories(buildDir);
@ -511,7 +780,10 @@ Configuration Crafter::LoadProject(const fs::path& projectFile, std::span<const
EnsureCrafterBuildPcms(sourceDir, cacheDir);
bool stale = !fs::exists(dllPath) || fs::last_write_time(dllPath) < fs::last_write_time(absProject) || fs::last_write_time(dllPath) < fs::last_write_time(hostExe);
// See the Linux LoadProject for why this sits here and runs every time.
ProjectFileDependencies preDeps = ResolveProjectFileDependencies(absProject, hostConfig);
bool stale = ProjectFileStale(dllPath, absProject, hostExe, preDeps);
if (stale) {
fs::path crafterBuildLib = hostExe.parent_path() / "crafter-build.lib";
@ -521,11 +793,11 @@ Configuration Crafter::LoadProject(const fs::path& projectFile, std::span<const
"-std=c++26 -shared -O3 -Wno-return-type-c-linkage -fuse-ld=lld "
"-D CRAFTER_BUILD_DLL_IMPORT "
"-nostdinc++ -nostdlib++ -isystem %LIBCXX_DIR%\\include\\c++\\v1 "
"-fprebuilt-module-path={} "
"-fprebuilt-module-path={}{} "
"-Wl,/EXPORT:CrafterBuildProject "
"{} {} -o {} -L %LIBCXX_DIR%\\lib -lc++",
cacheDir.string(),
absProject.string(), crafterBuildLib.string(), dllPath.string());
"{} {}{} -o {} -L %LIBCXX_DIR%\\lib -lc++",
cacheDir.string(), preDeps.compileFlags,
absProject.string(), crafterBuildLib.string(), preDeps.linkFlags, dllPath.string());
std::string result = RunBuildCommand(compileCmd);
if (!result.empty()) {
@ -552,6 +824,7 @@ Configuration Crafter::LoadProject(const fs::path& projectFile, std::span<const
throw std::runtime_error(std::format("CrafterBuildProject not found in {}: error {}", dllPath.string(), GetLastError()));
}
guard.Frame().dependencies = std::move(preDeps.configs);
return fn(args);
}
#endif
@ -686,6 +959,7 @@ namespace {
Configuration Crafter::LoadProject(const fs::path& projectFile, std::span<const std::string_view> args) {
fs::path absProject = fs::canonical(projectFile);
ProjectFileGuard guard(absProject);
fs::path buildDir = absProject.parent_path() / "build";
if (!fs::exists(buildDir)) fs::create_directories(buildDir);
fs::path dllPath = buildDir / std::format("{}.dll", absProject.stem().string());
@ -713,7 +987,10 @@ Configuration Crafter::LoadProject(const fs::path& projectFile, std::span<const
EnsureCrafterBuildPcms(sourceDir, cacheDir);
bool stale = !fs::exists(dllPath) || fs::last_write_time(dllPath) < fs::last_write_time(absProject) || fs::last_write_time(dllPath) < fs::last_write_time(hostExe);
// See the Linux LoadProject for why this sits here and runs every time.
ProjectFileDependencies preDeps = ResolveProjectFileDependencies(absProject, hostConfig);
bool stale = ProjectFileStale(dllPath, absProject, hostExe, preDeps);
if (stale) {
// The import lib lives next to the launcher exe (Build()'s mingw
@ -725,15 +1002,17 @@ Configuration Crafter::LoadProject(const fs::path& projectFile, std::span<const
"--sysroot=\"{}\" -femulated-tls "
"-std=c++26 -shared -O3 -Wno-return-type-c-linkage -fuse-ld=lld "
"-D CRAFTER_BUILD_DLL_IMPORT "
"-fprebuilt-module-path={} "
"-fprebuilt-module-path={}{} "
// mingw-lld doesn't accept /EXPORT:NAME — use the catch-all so
// CrafterBuildProject is reachable via GetProcAddress without
// forcing project.cpp templates to add __declspec(dllexport).
"-Wl,--export-all-symbols "
"{} -o {} -L\"{}\" -lcrafter-build -lstdc++exp -lpthread",
// Dependency archives go before -lcrafter-build: they reference
// Crafter.Build symbols, and ld.lld resolves archives in order.
"{} -o {}{} -L\"{}\" -lcrafter-build -lstdc++exp -lpthread",
prefix.string(),
cacheDir.string(),
absProject.string(), dllPath.string(),
cacheDir.string(), preDeps.compileFlags,
absProject.string(), dllPath.string(), preDeps.linkFlags,
hostExe.parent_path().string());
std::string result = RunBuildCommand(compileCmd);
@ -761,6 +1040,7 @@ Configuration Crafter::LoadProject(const fs::path& projectFile, std::span<const
throw std::runtime_error(std::format("CrafterBuildProject not found in {}: error {}", dllPath.string(), GetLastError()));
}
guard.Frame().dependencies = std::move(preDeps.configs);
return fn(args);
}
#endif
@ -990,6 +1270,7 @@ namespace {
Configuration Crafter::LoadProject(const fs::path& projectFile, std::span<const std::string_view> args) {
fs::path absProject = fs::canonical(projectFile);
ProjectFileGuard guard(absProject);
fs::path buildDir = absProject.parent_path() / "build";
if (!fs::exists(buildDir)) {
fs::create_directories(buildDir);
@ -1017,19 +1298,28 @@ Configuration Crafter::LoadProject(const fs::path& projectFile, std::span<const
EnsureCrafterBuildPcms(sourceDir, cacheDir);
const bool stale = !fs::exists(soPath) || fs::last_write_time(soPath) < fs::last_write_time(absProject) || fs::last_write_time(soPath) < fs::last_write_time(hostExe);
// After the host PCMs exist (a dependency's build imports Crafter.Build
// from that cache) and before the staleness check (its artifacts are
// inputs to project.so). Built on every load, fresh or not: a dependency
// BMI whose own imports were rebuilt underneath it doesn't fail cleanly,
// clang crashes on it.
ProjectFileDependencies preDeps = ResolveProjectFileDependencies(absProject, hostConfig);
const bool stale = ProjectFileStale(soPath, absProject, hostExe, preDeps);
if (stale) {
// -fuse-ld=lld like every other link the tool performs: lld is a hard
// dependency, binutils' ld is not (Alpine's clang would default to it).
// dependency, binutils' ld is not (Alpine's clang would default to it),
// and a dependency archive built in release holds ThinLTO bitcode,
// which GNU ld can't read.
std::string compileCmd = std::format(
"clang++ --target={} -march=native -mtune=native "
"-std=c++26 -stdlib=libc++ -shared -fPIC -O3 -fuse-ld=lld "
"-Wno-return-type-c-linkage "
"-fprebuilt-module-path={} "
"{} -o {}",
hostConfig.target, cacheDir.string(),
absProject.string(), soPath.string());
"-fprebuilt-module-path={}{} "
"{} -o {}{}",
hostConfig.target, cacheDir.string(), preDeps.compileFlags,
absProject.string(), soPath.string(), preDeps.linkFlags);
std::string result = RunBuildCommand(compileCmd);
if (!result.empty()) {
@ -1049,6 +1339,7 @@ Configuration Crafter::LoadProject(const fs::path& projectFile, std::span<const
throw std::runtime_error(std::format("CrafterBuildProject not found in {}: {}", soPath.string(), err));
}
guard.Frame().dependencies = std::move(preDeps.configs);
return fn(args);
}