13697cd026
Adding a data member to a class in a module interface did not rebuild every object compiled against the old layout. The build succeeded with no error or warning and the resulting binary mixed both layouts, surfacing later as a SIGSEGV in a destructor. GetInterfacesAndImplementations scans a TU's `import X;` statements when the source is declared. An import that matches neither a module in the Configuration nor one reachable through `dependencies` was dropped on the floor, leaving that TU with no staleness edge to the interface it consumes. `dependencies` is frequently assigned *after* the scan — AddTest does exactly that, resolving tests/<name>/main.cpp and only then returning a builder whose .Dependencies() supplies the library — so consumers of a dependency's modules routinely carried no edge at all. A layout change then rebuilt the library, relinked the consumer, and kept the consumer's object as it was. Unresolved names are now remembered on the partition/implementation as pendingImports, and Configuration::ResolvePendingImports retries them against the dependency DAG as it stands. Build() calls it immediately before comparing mtimes, which closes the window for every caller rather than only the ones that declare in the right order; TestBuilder::Dependencies also calls it so the Configuration is coherent for anyone inspecting it before the build. Resolves #27
719 lines
29 KiB
C++
719 lines
29 KiB
C++
// SPDX-License-Identifier: LGPL-3.0-only
|
|
// SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts®
|
|
|
|
module;
|
|
export module Crafter.Build:Test_impl;
|
|
import std;
|
|
import :Test;
|
|
import :Clang;
|
|
import :Platform;
|
|
import :Progress;
|
|
namespace fs = std::filesystem;
|
|
using namespace Crafter;
|
|
|
|
namespace {
|
|
bool TargetIsWindows(std::string_view target) {
|
|
return target.find("windows") != std::string_view::npos || target.find("mingw") != std::string_view::npos;
|
|
}
|
|
|
|
fs::path TestBinaryPath(const Configuration& cfg) {
|
|
fs::path outputDir = cfg.BinDir();
|
|
return outputDir / (TargetIsWindows(cfg.target) ? std::format("{}.exe", cfg.outputName) : cfg.outputName);
|
|
}
|
|
|
|
// MatchGlob/MatchAny live in :Platform — shared with the lint verb.
|
|
|
|
std::string ShellQuoteSh(std::string_view s) {
|
|
std::string out;
|
|
out.reserve(s.size() + 2);
|
|
out.push_back('\'');
|
|
for (char c : s) {
|
|
if (c == '\'') out += "'\\''";
|
|
else out.push_back(c);
|
|
}
|
|
out.push_back('\'');
|
|
return out;
|
|
}
|
|
|
|
// cmd.exe doesn't recognize '...' as quoting (it would pass the single
|
|
// quotes through to the executable). Wrap in "..." for cmd; embedded "
|
|
// is rare in paths but escape it to be safe. Backslash sequences before
|
|
// " don't need the MS-CRT doubling rules because we go through
|
|
// `cmd /C "..."`, which uses cmd's parser, not the CRT's argv splitter.
|
|
std::string ShellQuoteCmd(std::string_view s) {
|
|
std::string out;
|
|
out.reserve(s.size() + 2);
|
|
out.push_back('"');
|
|
for (char c : s) {
|
|
if (c == '"') out += "\\\"";
|
|
else out.push_back(c);
|
|
}
|
|
out.push_back('"');
|
|
return out;
|
|
}
|
|
|
|
// Host-shell quoting: sh on Linux, cmd on Windows. For args/paths that
|
|
// hit the local shell (Local runner exec, Cmd-prefix runner exec).
|
|
std::string ShellQuoteHost(std::string_view s) {
|
|
#ifdef _WIN32
|
|
return ShellQuoteCmd(s);
|
|
#else
|
|
return ShellQuoteSh(s);
|
|
#endif
|
|
}
|
|
|
|
std::string JoinAndQuoteArgs(std::span<const std::string> args) {
|
|
std::string out;
|
|
for (const auto& a : args) {
|
|
if (!out.empty()) out.push_back(' ');
|
|
out += ShellQuoteHost(a);
|
|
}
|
|
return out;
|
|
}
|
|
|
|
std::string Substitute(std::string_view tmpl, const std::map<std::string, std::string>& ph) {
|
|
std::string out;
|
|
out.reserve(tmpl.size());
|
|
std::size_t i = 0;
|
|
while (i < tmpl.size()) {
|
|
if (tmpl[i] == '{') {
|
|
std::size_t end = tmpl.find('}', i + 1);
|
|
if (end != std::string_view::npos) {
|
|
std::string key(tmpl.substr(i, end - i + 1));
|
|
if (auto it = ph.find(key); it != ph.end()) {
|
|
out += it->second;
|
|
i = end + 1;
|
|
continue;
|
|
}
|
|
}
|
|
}
|
|
out.push_back(tmpl[i++]);
|
|
}
|
|
return out;
|
|
}
|
|
|
|
std::string SignalName(std::int32_t sig) {
|
|
switch (sig) {
|
|
case 1: return "SIGHUP";
|
|
case 2: return "SIGINT";
|
|
case 4: return "SIGILL";
|
|
case 6: return "SIGABRT";
|
|
case 8: return "SIGFPE";
|
|
case 9: return "SIGKILL";
|
|
case 11: return "SIGSEGV";
|
|
case 13: return "SIGPIPE";
|
|
case 14: return "SIGALRM";
|
|
case 15: return "SIGTERM";
|
|
default: return std::format("signal {}", sig);
|
|
}
|
|
}
|
|
|
|
void WriteLog(const fs::path& projectPath, const std::string& name, const std::string& output) {
|
|
fs::path logDir = projectPath / "build" / "test-logs";
|
|
std::error_code ec;
|
|
fs::create_directories(logDir, ec);
|
|
if (ec) return;
|
|
std::ofstream(logDir / (std::format("{}.log", name))) << output;
|
|
}
|
|
|
|
void PrintResult(const TestResult& r, std::string_view runnerName) {
|
|
Progress::Clear();
|
|
auto ms = r.duration.count();
|
|
std::string runnerSuffix = (runnerName.empty() || runnerName == "local")
|
|
? std::string()
|
|
: std::format(" ({})", runnerName);
|
|
switch (r.outcome) {
|
|
case TestOutcome::Pass:
|
|
std::println("✅ {}{} ({}ms)", r.name, runnerSuffix, ms);
|
|
break;
|
|
case TestOutcome::Fail:
|
|
std::println("❌ {}{} ({}ms) exit {}", r.name, runnerSuffix, ms, r.exitCode);
|
|
if (!r.output.empty()) {
|
|
for (auto line : std::views::split(r.output, '\n')) {
|
|
std::string_view sv(line.begin(), line.end());
|
|
if (!sv.empty()) std::println(" {}", sv);
|
|
}
|
|
}
|
|
break;
|
|
case TestOutcome::Crash:
|
|
std::println("\U0001F4A5 {}{} ({}ms) crashed: {}", r.name, runnerSuffix, ms, SignalName(r.signal));
|
|
if (!r.output.empty()) {
|
|
for (auto line : std::views::split(r.output, '\n')) {
|
|
std::string_view sv(line.begin(), line.end());
|
|
if (!sv.empty()) std::println(" {}", sv);
|
|
}
|
|
}
|
|
break;
|
|
case TestOutcome::Timeout:
|
|
std::println("⏱ {}{} ({}ms) timeout", r.name, runnerSuffix, ms);
|
|
break;
|
|
case TestOutcome::Skipped:
|
|
std::println("⏭ {}{} skipped: {}", r.name, runnerSuffix, r.output.empty() ? std::string("(no reason)") : r.output);
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
namespace {
|
|
std::atomic<Configuration*> ParentProject{nullptr};
|
|
|
|
Configuration* FindLibInTree(Configuration* root, std::string_view name, std::unordered_set<Configuration*>& seen) {
|
|
if (!seen.insert(root).second) return nullptr;
|
|
if (root->name == name) return root;
|
|
for (Configuration* dep : root->dependencies) {
|
|
if (auto found = FindLibInTree(dep, name, seen)) return found;
|
|
}
|
|
return nullptr;
|
|
}
|
|
}
|
|
|
|
void Crafter::SetParentProject(Configuration* parent) {
|
|
ParentProject.store(parent);
|
|
}
|
|
|
|
Configuration* Crafter::ParentLib(std::string_view name) {
|
|
Configuration* root = ParentProject.load();
|
|
if (!root) {
|
|
throw std::runtime_error(std::format("Crafter::ParentLib('{}'): no parent project set", name));
|
|
}
|
|
std::unordered_set<Configuration*> seen;
|
|
if (auto found = FindLibInTree(root, name, seen)) return found;
|
|
throw std::runtime_error(std::format("Crafter::ParentLib('{}'): not found in parent project '{}'", name, root->name));
|
|
}
|
|
|
|
TestRunner TestRunner::Local() {
|
|
TestRunner r;
|
|
r.name = "local";
|
|
return r;
|
|
}
|
|
|
|
TestRunner TestRunner::Cmd(std::string command) {
|
|
TestRunner r;
|
|
r.name = std::format("cmd:{}", command);
|
|
r.exec = std::format("{} {{bin}} {{args}}", command);
|
|
#ifdef _WIN32
|
|
r.probe = std::format("where {}", command);
|
|
#else
|
|
r.probe = std::format("which {}", command);
|
|
#endif
|
|
return r;
|
|
}
|
|
|
|
TestRunner TestRunner::Wine() {
|
|
TestRunner r;
|
|
r.name = "wine";
|
|
r.exec = "wine {bin} {args}";
|
|
#ifdef _WIN32
|
|
r.probe = "where wine";
|
|
#else
|
|
r.probe = "which wine";
|
|
#endif
|
|
return r;
|
|
}
|
|
|
|
TestRunner TestRunner::ForTarget(const Configuration& cfg) {
|
|
const std::string& target = cfg.target;
|
|
|
|
// Same triple as the host → run the binary directly. Covers the common
|
|
// case (cfg.target defaulted to HostTarget()) without any wrapper.
|
|
if (target == HostTarget()) return Local();
|
|
|
|
// Windows targets: native on a Windows host, Wine on Linux. We don't
|
|
// distinguish mingw vs msvc here — the produced .exe runs the same way.
|
|
if (TargetIsWindows(target)) {
|
|
return TargetIsWindows(HostTarget()) ? Local() : Wine();
|
|
}
|
|
|
|
// WASI: a .wasm file isn't directly executable; wasmtime is the canonical
|
|
// runtime. wasi-cli also works but the upstream Bytecode Alliance name is
|
|
// wasmtime, so we standardize on that.
|
|
if (target.starts_with("wasm32-wasi") || target.starts_with("wasm64-wasi")) {
|
|
return Cmd("wasmtime");
|
|
}
|
|
|
|
// Non-host Linux triple: extract the architecture and route through
|
|
// qemu-user. Triple is <arch>-<vendor>-<os>-<env> (or sometimes 3 parts);
|
|
// qemu-user's binary names mostly follow the arch field, with two known
|
|
// mismatches handled below. cfg.sysroot, when set, becomes QEMU_LD_PREFIX
|
|
// so the target's dynamic linker / shared libs are reachable — without
|
|
// it qemu-user crashes on dynamic ELFs with "could not open /lib/ld...".
|
|
if (target.find("-linux-") != std::string::npos) {
|
|
auto dash = target.find('-');
|
|
std::string arch = target.substr(0, dash);
|
|
// i686-linux-gnu → qemu-i386; arm-* already matches qemu-arm; aarch64,
|
|
// riscv64, ppc64le, mips, mips64, s390x all match their qemu names.
|
|
if (arch == "i686") arch = "i386";
|
|
TestRunner r = Cmd(std::format("qemu-{}", arch));
|
|
if (!cfg.sysroot.empty()) {
|
|
// Use `env VAR=value cmd` rather than the shell's `VAR=value cmd`
|
|
// prefix syntax: RunCommandWithTimeout pipes through GNU `timeout`,
|
|
// which execvp's its argument list directly without going through a
|
|
// shell. A bare VAR=value would be exec'd as a command path and
|
|
// fail with "No such file or directory".
|
|
// QEMU_LD_PREFIX only redirects the ELF interpreter lookup; the
|
|
// emulated loader then searches ITS default paths against the
|
|
// host filesystem, picking up host-arch libraries from /lib —
|
|
// LD_LIBRARY_PATH steers it back into the sysroot.
|
|
r.exec = std::format("env QEMU_LD_PREFIX={0} LD_LIBRARY_PATH={0}/lib:{0}/usr/lib {1}", cfg.sysroot, r.exec);
|
|
}
|
|
return r;
|
|
}
|
|
|
|
// Unknown / bare-metal / freestanding targets: fall back to Local. The
|
|
// caller's runner-availability probe (or absence of the binary) surfaces
|
|
// the problem rather than us inventing a wrong wrapper here.
|
|
return Local();
|
|
}
|
|
|
|
namespace {
|
|
std::string NormalizeTriple(std::string_view target) {
|
|
std::string out(target);
|
|
for (char& c : out) {
|
|
if (c == '-' || c == '.') c = '_';
|
|
}
|
|
return out;
|
|
}
|
|
|
|
// Spec grammar: "local" | "cmd:<binary>". Used by CRAFTER_BUILD_RUNNER_*
|
|
// env vars and --runner=. The set used to include ssh/sshwin/wsl variants;
|
|
// those were removed when transport-style runners were dropped (issue #8).
|
|
std::optional<TestRunner> ParseRunnerSpec(std::string_view spec) {
|
|
if (spec.empty()) return std::nullopt;
|
|
if (spec == "local") return TestRunner::Local();
|
|
if (spec.starts_with("cmd:") && spec.size() > 4) {
|
|
return TestRunner::Cmd(std::string(spec.substr(4)));
|
|
}
|
|
throw std::runtime_error(std::format("TestRunner::FromSpec: unrecognized runner spec '{}' " "(expected 'local' or 'cmd:<binary>')", spec));
|
|
}
|
|
}
|
|
|
|
std::optional<TestRunner> TestRunner::FromSpec(std::string_view spec) {
|
|
return ParseRunnerSpec(spec);
|
|
}
|
|
|
|
TestRunner TestRunner::FromEnv(std::string_view target, TestRunner fallback) {
|
|
std::string envName = std::format("CRAFTER_BUILD_RUNNER_{}", NormalizeTriple(target));
|
|
const char* v = std::getenv(envName.c_str());
|
|
if (!v || !*v) return fallback;
|
|
if (auto r = ParseRunnerSpec(v)) return std::move(*r);
|
|
return fallback;
|
|
}
|
|
|
|
TestResult Crafter::RunSingleTest(const Test& test, const fs::path& binary, std::chrono::seconds timeout) {
|
|
TestResult result;
|
|
result.name = test.config.name;
|
|
|
|
std::map<std::string, std::string> ph;
|
|
ph["{args}"] = JoinAndQuoteArgs(test.args);
|
|
|
|
auto start = std::chrono::steady_clock::now();
|
|
CommandResult r;
|
|
|
|
if (test.runner.exec.empty()) {
|
|
// Local runner: spawn the binary directly through the host shell.
|
|
std::string cmd = std::format("{} {}", ShellQuoteHost(binary.string()), ph["{args}"]);
|
|
r = RunCommandWithTimeout(cmd, timeout);
|
|
} else {
|
|
// Prefix runner (qemu-user, wasmtime, wine, ...): templated exec
|
|
// wraps the local binary.
|
|
ph["{bin}"] = binary.string();
|
|
r = RunCommandWithTimeout(Substitute(test.runner.exec, ph), timeout);
|
|
}
|
|
|
|
auto end = std::chrono::steady_clock::now();
|
|
result.duration = std::chrono::duration_cast<std::chrono::milliseconds>(end - start);
|
|
result.output = std::move(r.output);
|
|
result.exitCode = r.exitCode;
|
|
result.signal = r.signal;
|
|
|
|
if (r.timedOut) result.outcome = TestOutcome::Timeout;
|
|
else if (r.crashed) result.outcome = TestOutcome::Crash;
|
|
else if (r.exitCode == 77) result.outcome = TestOutcome::Skipped;
|
|
else if (r.exitCode != 0) result.outcome = TestOutcome::Fail;
|
|
else result.outcome = TestOutcome::Pass;
|
|
|
|
return result;
|
|
}
|
|
|
|
namespace {
|
|
bool ToolOnPath(std::string_view name) {
|
|
#ifdef _WIN32
|
|
std::string cmd = std::format("where {} > nul 2>&1", name);
|
|
#else
|
|
std::string cmd = std::format("which {} > /dev/null 2>&1", name);
|
|
#endif
|
|
return std::system(cmd.c_str()) == 0;
|
|
}
|
|
|
|
struct RequireResult {
|
|
bool ok;
|
|
std::string reason; // human-readable when !ok
|
|
};
|
|
|
|
// Evaluate each `<kind>:<arg>` precondition. Returns the first failure
|
|
// (short-circuit; reporting one missing dep at a time is enough to act on
|
|
// and keeps the test log uncluttered).
|
|
RequireResult EvaluateRequires(std::span<const std::string> reqs) {
|
|
for (const auto& r : reqs) {
|
|
auto sep = r.find(':');
|
|
if (sep == std::string::npos || sep == 0 || sep == r.size() - 1) {
|
|
return {false, std::format("malformed require '{}' (expected kind:arg)", r)};
|
|
}
|
|
std::string_view kind(r.data(), sep);
|
|
std::string_view arg(r.data() + sep + 1, r.size() - sep - 1);
|
|
if (kind == "tool") {
|
|
if (!ToolOnPath(arg)) {
|
|
return {false, std::format("tool '{}' not on PATH", arg)};
|
|
}
|
|
} else if (kind == "file") {
|
|
if (!fs::exists(std::string(arg))) {
|
|
return {false, std::format("file '{}' missing", arg)};
|
|
}
|
|
} else if (kind == "env") {
|
|
const char* v = std::getenv(std::string(arg).c_str());
|
|
if (!v || !*v) {
|
|
return {false, std::format("env '{}' unset", arg)};
|
|
}
|
|
} else {
|
|
return {false, std::format("unknown require kind '{}' (expected tool/file/env)", kind)};
|
|
}
|
|
}
|
|
return {true, ""};
|
|
}
|
|
|
|
// Match a runner's tool dependency against the test's declared
|
|
// requirements. Used to decide between Skip (declared, may legitimately
|
|
// be missing) and Fail (runner unavailable but test didn't declare it —
|
|
// a silent skip would mask broken cross-arch CI configuration).
|
|
bool RequiresMentionsTool(std::span<const std::string> reqs, std::string_view tool) {
|
|
std::string needle = std::format("tool:{}", tool);
|
|
return std::ranges::any_of(reqs, [&](const std::string& s) { return s == needle; });
|
|
}
|
|
|
|
// Best-effort extraction of the runner-tool name from a TestRunner so the
|
|
// hard-fail-unless-declared check can match it against `requires`. For
|
|
// Cmd("foo"), the name is "cmd:foo"; for Wine, it's "wine". Anything else
|
|
// (Local, transport runners) returns empty — those don't trigger the
|
|
// declared/undeclared gate.
|
|
std::string RunnerToolName(const TestRunner& runner) {
|
|
if (runner.name == "wine") return "wine";
|
|
if (runner.name.starts_with("cmd:")) {
|
|
std::string tool = runner.name.substr(4);
|
|
// QEMU_LD_PREFIX prefix may be glued onto exec but the runner's
|
|
// `name` field already isolates the command, so no extra parsing.
|
|
return tool;
|
|
}
|
|
return "";
|
|
}
|
|
}
|
|
|
|
TestBuilder Configuration::AddTest(std::string_view name) {
|
|
// Empty-interfaces trampoline. The span we hand to the (name, interfaces)
|
|
// overload wraps a stack array; it's only consulted inside that call, so
|
|
// its lifetime is fine for the duration.
|
|
std::array<fs::path, 0> noIfaces = {};
|
|
return AddTest(name, noIfaces);
|
|
}
|
|
|
|
TestBuilder Configuration::AddTest(std::string_view name, std::span<fs::path> interfaces) {
|
|
Test t;
|
|
// Default to "./" so tests/<name>/main.cpp resolves at the project root
|
|
// (where crafter-build sets cwd from project.cpp's location). Projects
|
|
// whose lib lives in a subdir (e.g. cfg.path = "./mylib/") get this
|
|
// right by default; if the test layout is unusual, override via .Path().
|
|
t.config.path = "./";
|
|
t.config.name = std::string(name);
|
|
t.config.outputName = std::string(name);
|
|
t.config.target = this->target;
|
|
t.config.march = this->march;
|
|
t.config.mtune = this->mtune;
|
|
t.config.sysroot = this->sysroot;
|
|
t.config.debug = this->debug;
|
|
t.config.type = ConfigurationType::Executable;
|
|
|
|
// Default source layout: tests/<name>/main.cpp resolved against the
|
|
// parent Configuration's path. cfg.path is typically "./" (project
|
|
// root), which puts the test sources at <repo>/tests/<name>/main.cpp.
|
|
fs::path mainSource = fs::path("tests") / std::string(name) / "main";
|
|
std::array<fs::path, 1> impls = { mainSource };
|
|
t.config.GetInterfacesAndImplementations(interfaces, impls);
|
|
|
|
tests.push_back(std::move(t));
|
|
return TestBuilder{this, tests.size() - 1};
|
|
}
|
|
|
|
void Configuration::AddMarchVariants(std::string_view name, std::span<fs::path> interfaces, std::span<const MarchTier> tiers) {
|
|
for (const auto& tier : tiers) {
|
|
Test t;
|
|
t.config.path = "./";
|
|
t.config.name = std::format("{}-{}", name, tier.march);
|
|
t.config.outputName = t.config.name;
|
|
t.config.target = this->target;
|
|
t.config.march = tier.march;
|
|
t.config.mtune = tier.mtune;
|
|
t.config.sysroot = this->sysroot;
|
|
t.config.debug = this->debug;
|
|
t.config.type = ConfigurationType::Executable;
|
|
|
|
fs::path mainSource = fs::path("tests") / std::string(name) / "main";
|
|
std::array<fs::path, 1> impls = { mainSource };
|
|
// Interfaces stamped directly into each variant's Configuration —
|
|
// NOT via cfg.dependencies. The dependency machinery caches PCMs by
|
|
// dep->VariantId(), which would compile the parent's interfaces
|
|
// once with the parent's march; for SIMD-sensitive code each
|
|
// variant needs its own compile. Listing the interfaces as the
|
|
// test's own forces per-variant recompilation.
|
|
t.config.GetInterfacesAndImplementations(interfaces, impls);
|
|
tests.push_back(std::move(t));
|
|
}
|
|
}
|
|
|
|
TestBuilder& TestBuilder::Path(fs::path p) { Ref().config.path = std::move(p); return *this; }
|
|
TestBuilder& TestBuilder::Target(std::string t) { Ref().config.target = std::move(t); return *this; }
|
|
TestBuilder& TestBuilder::March(std::string m) { Ref().config.march = std::move(m); return *this; }
|
|
TestBuilder& TestBuilder::Mtune(std::string m) { Ref().config.mtune = std::move(m); return *this; }
|
|
TestBuilder& TestBuilder::Sysroot(fs::path s) { Ref().config.sysroot = s.string(); return *this; }
|
|
TestBuilder& TestBuilder::Debug(bool d) { Ref().config.debug = d; return *this; }
|
|
TestBuilder& TestBuilder::Define(std::string n, std::string v) {
|
|
Ref().config.defines.push_back({std::move(n), std::move(v)});
|
|
return *this;
|
|
}
|
|
TestBuilder& TestBuilder::Timeout(std::chrono::seconds s) { Ref().timeout = s; return *this; }
|
|
TestBuilder& TestBuilder::Args(std::vector<std::string> a) { Ref().args = std::move(a); return *this; }
|
|
TestBuilder& TestBuilder::Requires(std::string r) { Ref().requires_.push_back(std::move(r)); return *this; }
|
|
TestBuilder& TestBuilder::Dependencies(std::vector<Configuration*> d) {
|
|
Ref().config.dependencies = std::move(d);
|
|
// AddTest already scanned tests/<name>/main.cpp, at which point this test
|
|
// had no dependencies, so every `import <DepModule>;` in it came back
|
|
// unresolved. Place them now that the libraries are known — without this
|
|
// the test's object carries no staleness edge to the interfaces it consumes
|
|
// and survives a layout change to them (issue #27). Build() re-runs the
|
|
// same sweep as a backstop; doing it here keeps the Configuration coherent
|
|
// for anyone inspecting it before the build.
|
|
Ref().config.ResolvePendingImports();
|
|
return *this;
|
|
}
|
|
TestBuilder& TestBuilder::LinkFlag(std::string f) { Ref().config.linkFlags.push_back(std::move(f)); return *this; }
|
|
TestBuilder& TestBuilder::CompileFlag(std::string f) { Ref().config.compileFlags.push_back(std::move(f)); return *this; }
|
|
|
|
TestSummary Crafter::RunTests(Configuration& projectCfg, const RunTestsOptions& opts, std::span<const std::string_view> projectArgs) {
|
|
// Multi-target sweep: when no --target= was given, the run covers every
|
|
// distinct target declared across projectCfg.tests plus the host target.
|
|
// Lets a bare `crafter-build test` exercise cross-arch tests without the
|
|
// user having to know which targets exist. An explicit --target=X
|
|
// bypasses the sweep and runs that target only.
|
|
if (opts.targetFilter.empty()) {
|
|
std::set<std::string> sweep;
|
|
sweep.insert(HostTarget());
|
|
for (const Test& t : projectCfg.tests) {
|
|
if (!t.config.target.empty()) sweep.insert(t.config.target);
|
|
}
|
|
TestSummary aggregate;
|
|
for (const auto& target : sweep) {
|
|
RunTestsOptions perTarget = opts;
|
|
perTarget.targetFilter = target;
|
|
if (sweep.size() > 1) {
|
|
Progress::Clear();
|
|
std::println("\n=== target: {} ===", target);
|
|
}
|
|
TestSummary s = RunTests(projectCfg, perTarget, projectArgs);
|
|
aggregate.passed += s.passed;
|
|
aggregate.failed += s.failed;
|
|
aggregate.crashed += s.crashed;
|
|
aggregate.timedOut += s.timedOut;
|
|
aggregate.skipped += s.skipped;
|
|
for (auto& r : s.results) aggregate.results.push_back(std::move(r));
|
|
}
|
|
return aggregate;
|
|
}
|
|
|
|
TestSummary summary;
|
|
|
|
// Filter by target + glob, derive each Test's runner just-in-time. The
|
|
// runner is recomputed each sweep iteration (rather than once at AddTest
|
|
// time) because runnerOverride and FromEnv resolution depend on the
|
|
// run's options, which the project.cpp doesn't know about.
|
|
std::vector<Test*> filtered;
|
|
filtered.reserve(projectCfg.tests.size());
|
|
for (auto& test : projectCfg.tests) {
|
|
if (test.config.target != opts.targetFilter) continue;
|
|
if (!MatchAny(opts.globs, test.config.name)) continue;
|
|
test.runner = TestRunner::FromEnv(test.config.target, TestRunner::ForTarget(test.config));
|
|
if (opts.runnerOverride) {
|
|
if (auto r = TestRunner::FromSpec(*opts.runnerOverride)) {
|
|
test.runner = std::move(*r);
|
|
}
|
|
}
|
|
filtered.push_back(&test);
|
|
}
|
|
|
|
if (opts.listOnly) {
|
|
for (auto* t : filtered) {
|
|
std::println("{}", t->config.name);
|
|
}
|
|
return summary;
|
|
}
|
|
|
|
if (filtered.empty()) {
|
|
std::println("No tests matched.");
|
|
return summary;
|
|
}
|
|
|
|
std::int32_t jobs = opts.jobs > 0
|
|
? opts.jobs
|
|
: std::max(1u, std::thread::hardware_concurrency());
|
|
jobs = std::min(jobs, static_cast<std::int32_t>(filtered.size()));
|
|
|
|
std::unordered_map<fs::path, std::shared_future<BuildResult>> depResults;
|
|
std::mutex depMutex;
|
|
std::mutex printMutex;
|
|
std::mutex probeMutex;
|
|
std::unordered_map<std::string, bool> probeCache;
|
|
std::atomic<std::size_t> next{0};
|
|
std::vector<TestResult> results(filtered.size());
|
|
|
|
auto runnerAvailable = [&](const TestRunner& runner) -> bool {
|
|
if (runner.probe.empty()) return true;
|
|
std::lock_guard lk(probeMutex);
|
|
if (auto it = probeCache.find(runner.name); it != probeCache.end()) {
|
|
return it->second;
|
|
}
|
|
// RunCommandChecked captures and discards output internally, so probe
|
|
// specs don't need `> /dev/null 2>&1` (which doesn't translate to cmd).
|
|
bool ok = (RunCommandChecked(runner.probe).exitCode == 0);
|
|
probeCache[runner.name] = ok;
|
|
return ok;
|
|
};
|
|
|
|
auto worker = [&]() {
|
|
while (true) {
|
|
std::size_t i = next.fetch_add(1);
|
|
if (i >= filtered.size()) break;
|
|
Test& t = *filtered[i];
|
|
TestResult r;
|
|
r.name = t.config.name;
|
|
|
|
// Declarative preconditions set via TestBuilder::Requires. Evaluated
|
|
// before the build so a missing tool/file/env turns into a Skip without
|
|
// paying the compile cost. Reports the first failure only — once one
|
|
// precondition is unmet the test couldn't run anyway, and a wall of
|
|
// "also missing X, also missing Y" buries the actionable root cause.
|
|
if (auto req = EvaluateRequires(t.requires_); !req.ok) {
|
|
r.outcome = TestOutcome::Skipped;
|
|
r.output = req.reason;
|
|
{
|
|
std::lock_guard lk(printMutex);
|
|
PrintResult(r, t.runner.name);
|
|
}
|
|
results[i] = std::move(r);
|
|
continue;
|
|
}
|
|
|
|
if (!runnerAvailable(t.runner)) {
|
|
// Hard-fail-unless-declared: if the runner depends on a tool
|
|
// (qemu-aarch64, wasmtime, wine, ...) and the test didn't say
|
|
// "tool:<that>" in requires, the missing runner is a Fail. The
|
|
// intent is to surface broken cross-arch CI configuration
|
|
// instead of letting it masquerade as a Skip; tests that
|
|
// legitimately may run without their runner have to opt in.
|
|
std::string tool = RunnerToolName(t.runner);
|
|
if (!tool.empty() && !RequiresMentionsTool(t.requires_, tool)) {
|
|
r.outcome = TestOutcome::Fail;
|
|
r.exitCode = -1;
|
|
r.output = std::format("runner '{}' unavailable and not declared in requires " "(add .Requires(\"tool:{}\") to permit skipping)", t.runner.name, tool);
|
|
} else {
|
|
r.outcome = TestOutcome::Skipped;
|
|
r.output = std::format("runner '{}' not available", t.runner.name);
|
|
}
|
|
{
|
|
std::lock_guard lk(printMutex);
|
|
PrintResult(r, t.runner.name);
|
|
}
|
|
results[i] = std::move(r);
|
|
continue;
|
|
}
|
|
|
|
BuildResult br;
|
|
try {
|
|
br = Build(t.config, depResults, depMutex);
|
|
} catch (const std::exception& e) {
|
|
r.outcome = TestOutcome::Fail;
|
|
r.output = std::format("build threw: {}", e.what());
|
|
r.exitCode = -1;
|
|
{
|
|
std::lock_guard lk(printMutex);
|
|
PrintResult(r, t.runner.name);
|
|
}
|
|
results[i] = std::move(r);
|
|
continue;
|
|
}
|
|
|
|
if (!br.result.empty()) {
|
|
r.outcome = TestOutcome::Fail;
|
|
r.output = std::format("build failed: {}", br.result);
|
|
r.exitCode = -1;
|
|
{
|
|
std::lock_guard lk(printMutex);
|
|
PrintResult(r, t.runner.name);
|
|
}
|
|
results[i] = std::move(r);
|
|
continue;
|
|
}
|
|
|
|
std::chrono::seconds timeout = opts.timeoutOverride.value_or(t.timeout);
|
|
fs::path binary = TestBinaryPath(t.config);
|
|
try {
|
|
r = RunSingleTest(t, binary, timeout);
|
|
} catch (const std::exception& e) {
|
|
r.outcome = TestOutcome::Fail;
|
|
r.output = std::format("runner threw: {}", e.what());
|
|
r.exitCode = -1;
|
|
}
|
|
|
|
if (r.outcome != TestOutcome::Pass && r.outcome != TestOutcome::Skipped && !r.output.empty()) {
|
|
WriteLog(projectCfg.path, r.name, r.output);
|
|
}
|
|
|
|
{
|
|
std::lock_guard lk(printMutex);
|
|
PrintResult(r, t.runner.name);
|
|
}
|
|
results[i] = std::move(r);
|
|
}
|
|
};
|
|
|
|
std::vector<std::jthread> threads;
|
|
threads.reserve(jobs);
|
|
for (std::int32_t j = 0; j < jobs; ++j) {
|
|
threads.emplace_back(worker);
|
|
}
|
|
threads.clear(); // joins all jthreads
|
|
|
|
for (auto& r : results) {
|
|
switch (r.outcome) {
|
|
case TestOutcome::Pass: summary.passed++; break;
|
|
case TestOutcome::Fail: summary.failed++; break;
|
|
case TestOutcome::Crash: summary.crashed++; break;
|
|
case TestOutcome::Timeout: summary.timedOut++; break;
|
|
case TestOutcome::Skipped: summary.skipped++; break;
|
|
}
|
|
}
|
|
summary.results = std::move(results);
|
|
|
|
Progress::Clear();
|
|
std::print("\n");
|
|
std::vector<std::string> parts;
|
|
if (summary.passed) parts.push_back(std::format("{} passed", summary.passed));
|
|
if (summary.failed) parts.push_back(std::format("{} failed", summary.failed));
|
|
if (summary.crashed) parts.push_back(std::format("{} crashed", summary.crashed));
|
|
if (summary.timedOut) parts.push_back(std::format("{} timed out", summary.timedOut));
|
|
if (summary.skipped) parts.push_back(std::format("{} skipped", summary.skipped));
|
|
std::string joined;
|
|
for (std::size_t i = 0; i < parts.size(); ++i) {
|
|
if (i) joined += ", ";
|
|
joined += parts[i];
|
|
}
|
|
std::println("{}", joined);
|
|
|
|
return summary;
|
|
}
|