Catcrafts/Crafter.Build
2
0
Fork 0
Code Issues Pull requests Projects Releases 1 Packages Wiki Activity Actions

new tests
All checks were successful
CI / build-test-release (push) Successful in 1h4m52s
Details

Browse source
This commit is contained in:
Jorijn van der Graaf 2026-05-27 19:45:05 +02:00
commit 603840879d
11 changed files with 283 additions and 18235 deletions
Download patch file Download diff file Expand all files Collapse all files

9
implementations/Crafter.Build-Clang.cpp
View file

@ -1318,12 +1318,9 @@ Test options (after the `test` subcommand):
--runner=<spec> Override the test runner for this run. Specs:
local
cmd:<command> (e.g. cmd:wine)
ssh:<host>[:<remoteDir>]
sshwin:<host>[:<remoteDir>]
wsl[:<remoteDir>]
--target=<triple> Filter to tests whose cfg.target matches; this is
also forwarded to project-args so per-target tests
build for that triple. Default: host triple.
--target=<triple> Filter to tests whose cfg.target matches. Default:
sweep across every distinct target declared by the
project's tests plus the host triple.
<glob> One or more name globs to filter tests (e.g. 'Unit*').
Project args:

385
implementations/Crafter.Build-Test.cpp
View file

@ -18,11 +18,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
module;
// toml++ is consumed as a translation-unit-private dependency in the GMF: the
// parser is only needed for test.toml discovery here, so keeping it out of
// `import std`-using module purviews avoids dragging it through the module
// graph (and through every PCM consumer of Crafter.Build).
#include "../lib/toml.hpp"
export module Crafter.Build:Test_impl;
import std;
import :Test;
@ -387,74 +382,6 @@ TestResult Crafter::RunSingleTest(const Test& test, const fs::path& binary, std:
}
namespace {
// Declarative test metadata loaded from tests/<Name>/test.toml. Lets a test
// ship just main.cpp + a few lines of config instead of a whole project.cpp
// when its needs are "pick a target, gate on prerequisites, run with these
// args". project.cpp stays the escape hatch for outer-driver tests that
// call Build() / inspect intermediate state.
struct TestManifest {
std::optional<std::string> target;
std::optional<std::string> march;
std::optional<std::string> mtune;
std::optional<std::string> sysroot;
std::vector<std::string> requires_;
std::optional<int> timeoutSeconds;
std::vector<std::string> args;
std::vector<std::pair<std::string, std::string>> defines;
};
TestManifest ParseTestManifest(const fs::path& path) {
// toml++ builds with exceptions enabled by default; parse_file throws
// toml::parse_error on malformed input. Rethrow with the path attached
// so the discovery loop's catch can surface "where the error came from"
// alongside toml++'s "what was wrong".
toml::table t;
try {
t = toml::parse_file(path.string());
} catch (const toml::parse_error& e) {
throw std::runtime_error(std::format(
"test.toml parse error in {}: {}",
path.string(),
std::string_view(e.description())));
}
TestManifest m;
if (auto v = t["target"].value<std::string>()) m.target = *v;
if (auto v = t["march"].value<std::string>()) m.march = *v;
if (auto v = t["mtune"].value<std::string>()) m.mtune = *v;
if (auto v = t["sysroot"].value<std::string>()) m.sysroot = *v;
if (auto v = t["timeout"].value<int64_t>()) m.timeoutSeconds = static_cast<int>(*v);
if (auto arr = t["requires"].as_array()) {
for (auto& el : *arr) {
if (auto s = el.value<std::string>()) m.requires_.push_back(*s);
}
}
if (auto arr = t["args"].as_array()) {
for (auto& el : *arr) {
if (auto s = el.value<std::string>()) m.args.push_back(*s);
}
}
if (auto tbl = t["defines"].as_table()) {
for (auto&& [k, v] : *tbl) {
if (auto s = v.value<std::string>()) {
m.defines.emplace_back(std::string(k.str()), *s);
}
}
}
return m;
}
// Apply manifest overlay onto a Configuration synthesized from the test
// folder. Target overrides come last so a manifest's `target = "..."`
// wins over the synth default (= run's targetFilter). Defines accumulate;
// they don't replace pre-existing ones.
void ApplyManifest(Configuration& cfg, const TestManifest& m) {
if (m.target) cfg.target = *m.target;
if (m.march) cfg.march = *m.march;
if (m.mtune) cfg.mtune = *m.mtune;
if (m.sysroot) cfg.sysroot = *m.sysroot;
for (auto& [k, v] : m.defines) cfg.defines.push_back({k, v});
}
bool ToolOnPath(std::string_view name) {
#ifdef _WIN32
std::string cmd = std::format("where {} > nul 2>&1", name);
@ -527,99 +454,103 @@ namespace {
}
}
namespace {
// Synthesize a Configuration for tests/<Name>/ folders that don't contain
// a project.cpp. Convention: cfg.path = the folder, cfg.name/outputName =
// folder basename, cfg.target = host (overridable via test.toml `target`),
// cfg.type = exe. Sources: top-level *.cpp (excluding project.cpp) become
// implementations, interfaces/*.cppm become module interfaces. Tests with
// deeper layouts or dependencies still need an explicit project.cpp.
//
// Why host-default instead of targetFilter-default: under the multi-target
// sweep, an arch-agnostic test (no test.toml target) should run at the
// host iteration only — not get rebuilt against every cross-target the
// suite happens to declare. Cross-targeting is an opt-in via test.toml.
Configuration SynthesizeTest(const fs::path& dir) {
Configuration cfg;
cfg.path = dir;
cfg.name = dir.filename().string();
cfg.outputName = cfg.name;
cfg.target = HostTarget();
cfg.type = ConfigurationType::Executable;
TestBuilder Configuration::AddTest(std::string_view name) {
// Empty-interfaces trampoline. The span we hand to the (name, interfaces)
// overload wraps a stack array; it's only consulted inside that call, so
// its lifetime is fine for the duration.
std::array<fs::path, 0> noIfaces = {};
return AddTest(name, noIfaces);
}
std::vector<fs::path> impls;
for (auto& e : fs::directory_iterator(dir)) {
if (!e.is_regular_file()) continue;
auto p = e.path();
if (p.extension() != ".cpp") continue;
if (p.filename() == "project.cpp") continue;
impls.push_back(p.stem());
}
std::ranges::sort(impls);
TestBuilder Configuration::AddTest(std::string_view name, std::span<fs::path> interfaces) {
Test t;
// Default to "./" so tests/<name>/main.cpp resolves at the project root
// (where crafter-build sets cwd from project.cpp's location). Projects
// whose lib lives in a subdir (e.g. cfg.path = "./mylib/") get this
// right by default; if the test layout is unusual, override via .Path().
t.config.path = "./";
t.config.name = std::string(name);
t.config.outputName = std::string(name);
t.config.target = this->target;
t.config.march = this->march;
t.config.mtune = this->mtune;
t.config.debug = this->debug;
t.config.type = ConfigurationType::Executable;
std::vector<fs::path> ifaces;
fs::path interfacesDir = dir / "interfaces";
if (fs::exists(interfacesDir) && fs::is_directory(interfacesDir)) {
for (auto& e : fs::directory_iterator(interfacesDir)) {
if (!e.is_regular_file()) continue;
auto p = e.path();
if (p.extension() != ".cppm") continue;
ifaces.push_back(fs::path("interfaces") / p.stem());
}
std::ranges::sort(ifaces);
}
// Default source layout: tests/<name>/main.cpp resolved against the
// parent Configuration's path. cfg.path is typically "./" (project
// root), which puts the test sources at <repo>/tests/<name>/main.cpp.
fs::path mainSource = fs::path("tests") / std::string(name) / "main";
std::array<fs::path, 1> impls = { mainSource };
t.config.GetInterfacesAndImplementations(interfaces, impls);
if (impls.empty() && ifaces.empty()) {
throw std::runtime_error(std::format(
"no .cpp or interfaces/*.cppm files found in {}", dir.string()));
}
tests.push_back(std::move(t));
return TestBuilder{this, tests.size() - 1};
}
cfg.GetInterfacesAndImplementations(ifaces, impls);
return cfg;
void Configuration::AddMarchVariants(std::string_view name,
std::span<fs::path> interfaces,
std::span<const MarchTier> tiers) {
for (const auto& tier : tiers) {
Test t;
t.config.path = "./";
t.config.name = std::format("{}-{}", name, tier.march);
t.config.outputName = t.config.name;
t.config.target = this->target;
t.config.march = tier.march;
t.config.mtune = tier.mtune;
t.config.debug = this->debug;
t.config.type = ConfigurationType::Executable;
fs::path mainSource = fs::path("tests") / std::string(name) / "main";
std::array<fs::path, 1> impls = { mainSource };
// Interfaces stamped directly into each variant's Configuration —
// NOT via cfg.dependencies. The dependency machinery caches PCMs by
// dep->VariantId(), which would compile the parent's interfaces
// once with the parent's march; for SIMD-sensitive code each
// variant needs its own compile. Listing the interfaces as the
// test's own forces per-variant recompilation.
t.config.GetInterfacesAndImplementations(interfaces, impls);
tests.push_back(std::move(t));
}
}
TestBuilder& TestBuilder::Path(fs::path p) { test().config.path = std::move(p); return *this; }
TestBuilder& TestBuilder::Target(std::string t) { test().config.target = std::move(t); return *this; }
TestBuilder& TestBuilder::March(std::string m) { test().config.march = std::move(m); return *this; }
TestBuilder& TestBuilder::Mtune(std::string m) { test().config.mtune = std::move(m); return *this; }
TestBuilder& TestBuilder::Sysroot(fs::path s) { test().config.sysroot = s.string(); return *this; }
TestBuilder& TestBuilder::Debug(bool d) { test().config.debug = d; return *this; }
TestBuilder& TestBuilder::Define(std::string n, std::string v) {
test().config.defines.push_back({std::move(n), std::move(v)});
return *this;
}
TestBuilder& TestBuilder::Timeout(std::chrono::seconds s) { test().timeout = s; return *this; }
TestBuilder& TestBuilder::Args(std::vector<std::string> a) { test().args = std::move(a); return *this; }
TestBuilder& TestBuilder::Requires(std::string r) { test().requires_.push_back(std::move(r)); return *this; }
TestBuilder& TestBuilder::Dependencies(std::vector<Configuration*> d) {
test().config.dependencies = std::move(d);
return *this;
}
TestBuilder& TestBuilder::LinkFlag(std::string f) { test().config.linkFlags.push_back(std::move(f)); return *this; }
TestBuilder& TestBuilder::CompileFlag(std::string f) { test().config.compileFlags.push_back(std::move(f)); return *this; }
TestSummary Crafter::RunTests(Configuration& projectCfg, const RunTestsOptions& opts, std::span<const std::string_view> projectArgs) {
// Multi-target sweep: when no --target= was given, the run covers every
// distinct target a test.toml declares plus the host target. Lets a bare
// `crafter-build test` exercise cross-arch tests without the user having
// to know which targets exist in this project. An explicit --target=X
// distinct target declared across projectCfg.tests plus the host target.
// Lets a bare `crafter-build test` exercise cross-arch tests without the
// user having to know which targets exist. An explicit --target=X
// bypasses the sweep and runs that target only.
if (opts.targetFilter.empty()) {
std::set<std::string> sweep;
sweep.insert(HostTarget());
fs::path testsDir = fs::current_path() / "tests";
if (fs::exists(testsDir) && fs::is_directory(testsDir)) {
for (auto& e : fs::directory_iterator(testsDir)) {
if (!e.is_directory()) continue;
auto stem = e.path().filename().string();
if (stem.empty() || stem[0] == '_' || stem[0] == '.') continue;
fs::path tomlPath = e.path() / "test.toml";
if (!fs::exists(tomlPath)) continue;
try {
TestManifest m = ParseTestManifest(tomlPath);
if (m.target) sweep.insert(*m.target);
} catch (...) {
// Parse failures surface as discovery failures during the
// actual run; the sweep phase just collects targets.
}
}
for (const Test& t : projectCfg.tests) {
if (!t.config.target.empty()) sweep.insert(t.config.target);
}
TestSummary aggregate;
// Inline tests pushed by the caller (fixture-driven inner RunTests
// calls, e.g. RunnerClassification) must survive each sweep
// iteration. Configuration isn't copyable so we can't snapshot+restore;
// instead, remember the inline count and erase only the entries
// appended by the previous iteration's auto-discovery.
size_t inlineCount = projectCfg.tests.size();
for (const auto& target : sweep) {
RunTestsOptions perTarget = opts;
perTarget.targetFilter = target;
if (projectCfg.tests.size() > inlineCount) {
projectCfg.tests.erase(
projectCfg.tests.begin() + inlineCount,
projectCfg.tests.end());
}
if (sweep.size() > 1) {
Progress::Clear();
std::println("\n=== target: {} ===", target);
@ -636,152 +567,37 @@ TestSummary Crafter::RunTests(Configuration& projectCfg, const RunTestsOptions&
}
TestSummary summary;
std::vector<TestResult> discoveryFailures;
// Auto-discover tests one layer deep: each <project>/tests/<Name>/ folder
// is a test. If it contains project.cpp, that's loaded for full control;
// otherwise the Configuration is synthesized from the folder contents.
// Folders whose name starts with '_' or '.' are skipped (so tests/_shared/
// holds cross-test code without becoming a test). Each project.cpp receives
// the same args the root project did, so --target=... propagates through.
//
// Discovery is keyed off cwd (= the project root, since crafter-build loads
// ./project.cpp), not projectCfg.path: tests live at the project root even
// when projectCfg.path points at a subdirectory like "./src/" or "./lib/".
fs::path testsDir = fs::current_path() / "tests";
if (fs::exists(testsDir) && fs::is_directory(testsDir)) {
// A discovered fixture is one of:
// - project.cpp present → outer-driver test (LoadProject)
// - test.toml present → declarative synth + manifest
// - neither, just *.cpp → bare synth (host-target)
// - both project.cpp and test.toml → XOR violation, discovery Fail
struct TestEntry {
fs::path dir;
fs::path pcpp; // outer-driver path
std::optional<TestManifest> manifest;
};
std::vector<TestEntry> entries;
for (auto& entry : fs::directory_iterator(testsDir)) {
if (!entry.is_directory()) continue;
auto stem = entry.path().filename().string();
if (stem.empty() || stem[0] == '_' || stem[0] == '.') continue;
TestEntry te;
te.dir = entry.path();
auto pcpp = te.dir / "project.cpp";
auto tomlPath = te.dir / "test.toml";
bool hasPcpp = fs::exists(pcpp);
bool hasToml = fs::exists(tomlPath);
if (hasPcpp && hasToml) {
TestResult r;
r.name = stem;
r.outcome = TestOutcome::Fail;
r.exitCode = -1;
r.output = "both project.cpp and test.toml present — they're "
"mutually exclusive (delete one to disambiguate "
"outer-driver vs declarative test)";
discoveryFailures.push_back(std::move(r));
continue;
}
if (hasPcpp) te.pcpp = pcpp;
if (hasToml) {
try {
te.manifest = ParseTestManifest(tomlPath);
} catch (const std::exception& e) {
TestResult r;
r.name = stem;
r.outcome = TestOutcome::Fail;
r.exitCode = -1;
r.output = e.what();
discoveryFailures.push_back(std::move(r));
continue;
}
}
entries.push_back(std::move(te));
}
std::ranges::sort(entries, [](auto& a, auto& b) { return a.dir < b.dir; });
// Inject --target=<filter> into the args we hand each fixture so its
// CrafterBuildProject can default to the run's target. The CLI's own
// --target=... propagates through projectArgs already; this only
// appends when missing so an explicit user choice wins.
std::string targetArg = std::format("--target={}", opts.targetFilter);
std::vector<std::string_view> fixtureArgs(projectArgs.begin(), projectArgs.end());
bool hasTarget = std::ranges::any_of(fixtureArgs, [](std::string_view a) {
return a.starts_with("--target=");
});
if (!hasTarget) fixtureArgs.push_back(targetArg);
for (auto& te : entries) {
Test t;
try {
if (!te.pcpp.empty()) {
t.config = LoadProject(te.pcpp, fixtureArgs);
} else {
t.config = SynthesizeTest(te.dir);
if (te.manifest) {
ApplyManifest(t.config, *te.manifest);
if (te.manifest->timeoutSeconds) {
t.timeout = std::chrono::seconds(*te.manifest->timeoutSeconds);
}
t.args = te.manifest->args;
t.requires_ = te.manifest->requires_;
}
}
} catch (const std::exception& e) {
// A broken fixture shouldn't kill the whole run. Surface as a
// Fail and let other tests proceed.
TestResult r;
r.name = te.dir.filename().string();
r.outcome = TestOutcome::Fail;
r.exitCode = -1;
r.output = !te.pcpp.empty()
? std::format("project.cpp failed to load: {}", e.what())
: std::format("test discovery failed: {}", e.what());
discoveryFailures.push_back(std::move(r));
continue;
}
if (t.config.target != opts.targetFilter) continue;
t.runner = TestRunner::FromEnv(t.config.target, TestRunner::ForTarget(t.config));
if (opts.runnerOverride) {
if (auto r = TestRunner::FromSpec(*opts.runnerOverride)) {
t.runner = std::move(*r);
}
}
projectCfg.tests.push_back(std::move(t));
}
}
// Filter by target + glob, derive each Test's runner just-in-time. The
// runner is recomputed each sweep iteration (rather than once at AddTest
// time) because runnerOverride and FromEnv resolution depend on the
// run's options, which the project.cpp doesn't know about.
std::vector<Test*> filtered;
filtered.reserve(projectCfg.tests.size());
for (auto& test : projectCfg.tests) {
if (MatchAny(opts.globs, test.config.name)) {
filtered.push_back(&test);
if (test.config.target != opts.targetFilter) continue;
if (!MatchAny(opts.globs, test.config.name)) continue;
test.runner = TestRunner::FromEnv(test.config.target, TestRunner::ForTarget(test.config));
if (opts.runnerOverride) {
if (auto r = TestRunner::FromSpec(*opts.runnerOverride)) {
test.runner = std::move(*r);
}
}
filtered.push_back(&test);
}
if (opts.listOnly) {
for (auto& r : discoveryFailures) {
std::println("{} (project.cpp broken)", r.name);
}
for (auto* t : filtered) {
std::println("{}", t->config.name);
}
return summary;
}
if (filtered.empty() && discoveryFailures.empty()) {
if (filtered.empty()) {
std::println("No tests matched.");
return summary;
}
// Render discovery failures upfront so they appear before parallel test
// results. They're already-determined Fails — no need to put them through
// the worker pool.
for (auto& r : discoveryFailures) {
PrintResult(r, "");
WriteLog(projectCfg.path, r.name, r.output);
}
int jobs = opts.jobs > 0
? opts.jobs
: std::max(1u, std::thread::hardware_concurrency());
@ -816,12 +632,11 @@ TestSummary Crafter::RunTests(Configuration& projectCfg, const RunTestsOptions&
TestResult r;
r.name = t.config.name;
// Declarative preconditions (test.toml requires = [...] or Test.requires_
// set in project.cpp). Evaluated before the build so a missing tool/file/env
// turns into a Skip without paying the compile cost. Reports the first
// failure only — once one precondition is unmet the test couldn't run
// anyway, and a wall of "also missing X, also missing Y" buries the
// actionable root cause.
// Declarative preconditions set via TestBuilder::Requires. Evaluated
// before the build so a missing tool/file/env turns into a Skip without
// paying the compile cost. Reports the first failure only — once one
// precondition is unmet the test couldn't run anyway, and a wall of
// "also missing X, also missing Y" buries the actionable root cause.
if (auto req = EvaluateRequires(t.requires_); !req.ok) {
r.outcome = TestOutcome::Skipped;
r.output = req.reason;
@ -846,7 +661,7 @@ TestSummary Crafter::RunTests(Configuration& projectCfg, const RunTestsOptions&
r.exitCode = -1;
r.output = std::format(
"runner '{}' unavailable and not declared in requires "
"(add 'tool:{}' to test.toml requires to permit skipping)",
"(add .Requires(\"tool:{}\") to permit skipping)",
t.runner.name, tool);
} else {
r.outcome = TestOutcome::Skipped;
@ -916,10 +731,6 @@ TestSummary Crafter::RunTests(Configuration& projectCfg, const RunTestsOptions&
}
threads.clear(); // joins all jthreads
for (auto& r : discoveryFailures) {
results.push_back(std::move(r));
}
for (auto& r : results) {
switch (r.outcome) {
case TestOutcome::Pass: summary.passed++; break;