Jorijn van der Graaf
46082a3117
crafter-build is moving to a single Flatpak distribution, and there is no Windows host in that picture: no build.cmd, no MSVC or mingw host build, no DLL plus launcher. Delete every host-Windows path instead of leaving it to rot unverified: the cmd.exe command runner and quoting, the LOCALAPPDATA cache, the MSVC and mingw-host LoadProject variants with their LIBCXX_DIR and CRAFTER_MINGW_DIR plumbing, the winsock port probe, the LoadLibrary libclang loader, the console-width and isatty shims, and the preprocessor guards around the Linux bodies. The x86_64-w64-mingw32 cross-compile target stays. It is target logic (output names, -femulated-tls, the libstdc++ link extras, copying a dependency DLL next to a mingw exe, wine as the runner) and did not depend on any of the removed code. The msvc target goes with the host that was the only thing able to use it. Three target-path warts fixed on the way: ExpectedOutput named a mingw static library <name>.lib while the archive step wrote lib<name>.a, so those libraries relinked on every run; the import lib is no longer copied next to a mingw exe, since the dependency's own bin dir is already on the link line; and llvm-ar now writes every archive, not only the LTO ones, so Debug and Release archives are made the same way. CRAFTER_API and Crafter.Build-Api.h existed only for the DLL boundary. Nothing downstream used them, so they are removed rather than kept as an empty macro nobody can verify. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
682 lines
28 KiB
C++
682 lines
28 KiB
C++
// SPDX-License-Identifier: LGPL-3.0-only
|
|
// SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts®
|
|
|
|
module;
|
|
export module Crafter.Build:Test_impl;
|
|
import std;
|
|
import :Test;
|
|
import :Clang;
|
|
import :Platform;
|
|
import :Progress;
|
|
namespace fs = std::filesystem;
|
|
using namespace Crafter;
|
|
|
|
namespace {
|
|
bool TargetIsWindows(std::string_view target) {
|
|
return target.find("windows") != std::string_view::npos || target.find("mingw") != std::string_view::npos;
|
|
}
|
|
|
|
fs::path TestBinaryPath(const Configuration& cfg) {
|
|
fs::path outputDir = cfg.BinDir();
|
|
return outputDir / (TargetIsWindows(cfg.target) ? std::format("{}.exe", cfg.outputName) : cfg.outputName);
|
|
}
|
|
|
|
// MatchGlob/MatchAny live in :Platform — shared with the lint verb.
|
|
|
|
std::string ShellQuoteSh(std::string_view s) {
|
|
std::string out;
|
|
out.reserve(s.size() + 2);
|
|
out.push_back('\'');
|
|
for (char c : s) {
|
|
if (c == '\'') out += "'\\''";
|
|
else out.push_back(c);
|
|
}
|
|
out.push_back('\'');
|
|
return out;
|
|
}
|
|
|
|
std::string JoinAndQuoteArgs(std::span<const std::string> args) {
|
|
std::string out;
|
|
for (const auto& a : args) {
|
|
if (!out.empty()) out.push_back(' ');
|
|
out += ShellQuoteSh(a);
|
|
}
|
|
return out;
|
|
}
|
|
|
|
std::string Substitute(std::string_view tmpl, const std::map<std::string, std::string>& ph) {
|
|
std::string out;
|
|
out.reserve(tmpl.size());
|
|
std::size_t i = 0;
|
|
while (i < tmpl.size()) {
|
|
if (tmpl[i] == '{') {
|
|
std::size_t end = tmpl.find('}', i + 1);
|
|
if (end != std::string_view::npos) {
|
|
std::string key(tmpl.substr(i, end - i + 1));
|
|
if (auto it = ph.find(key); it != ph.end()) {
|
|
out += it->second;
|
|
i = end + 1;
|
|
continue;
|
|
}
|
|
}
|
|
}
|
|
out.push_back(tmpl[i++]);
|
|
}
|
|
return out;
|
|
}
|
|
|
|
std::string SignalName(std::int32_t sig) {
|
|
switch (sig) {
|
|
case 1: return "SIGHUP";
|
|
case 2: return "SIGINT";
|
|
case 4: return "SIGILL";
|
|
case 6: return "SIGABRT";
|
|
case 8: return "SIGFPE";
|
|
case 9: return "SIGKILL";
|
|
case 11: return "SIGSEGV";
|
|
case 13: return "SIGPIPE";
|
|
case 14: return "SIGALRM";
|
|
case 15: return "SIGTERM";
|
|
default: return std::format("signal {}", sig);
|
|
}
|
|
}
|
|
|
|
void WriteLog(const fs::path& projectPath, const std::string& name, const std::string& output) {
|
|
fs::path logDir = projectPath / "build" / "test-logs";
|
|
std::error_code ec;
|
|
fs::create_directories(logDir, ec);
|
|
if (ec) return;
|
|
std::ofstream(logDir / (std::format("{}.log", name))) << output;
|
|
}
|
|
|
|
void PrintResult(const TestResult& r, std::string_view runnerName) {
|
|
Progress::Clear();
|
|
auto ms = r.duration.count();
|
|
std::string runnerSuffix = (runnerName.empty() || runnerName == "local")
|
|
? std::string()
|
|
: std::format(" ({})", runnerName);
|
|
switch (r.outcome) {
|
|
case TestOutcome::Pass:
|
|
std::println("✅ {}{} ({}ms)", r.name, runnerSuffix, ms);
|
|
break;
|
|
case TestOutcome::Fail:
|
|
std::println("❌ {}{} ({}ms) exit {}", r.name, runnerSuffix, ms, r.exitCode);
|
|
if (!r.output.empty()) {
|
|
for (auto line : std::views::split(r.output, '\n')) {
|
|
std::string_view sv(line.begin(), line.end());
|
|
if (!sv.empty()) std::println(" {}", sv);
|
|
}
|
|
}
|
|
break;
|
|
case TestOutcome::Crash:
|
|
std::println("\U0001F4A5 {}{} ({}ms) crashed: {}", r.name, runnerSuffix, ms, SignalName(r.signal));
|
|
if (!r.output.empty()) {
|
|
for (auto line : std::views::split(r.output, '\n')) {
|
|
std::string_view sv(line.begin(), line.end());
|
|
if (!sv.empty()) std::println(" {}", sv);
|
|
}
|
|
}
|
|
break;
|
|
case TestOutcome::Timeout:
|
|
std::println("⏱ {}{} ({}ms) timeout", r.name, runnerSuffix, ms);
|
|
break;
|
|
case TestOutcome::Skipped:
|
|
std::println("⏭ {}{} skipped: {}", r.name, runnerSuffix, r.output.empty() ? std::string("(no reason)") : r.output);
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
namespace {
|
|
std::atomic<Configuration*> ParentProject{nullptr};
|
|
|
|
Configuration* FindLibInTree(Configuration* root, std::string_view name, std::unordered_set<Configuration*>& seen) {
|
|
if (!seen.insert(root).second) return nullptr;
|
|
if (root->name == name) return root;
|
|
for (Configuration* dep : root->dependencies) {
|
|
if (auto found = FindLibInTree(dep, name, seen)) return found;
|
|
}
|
|
return nullptr;
|
|
}
|
|
}
|
|
|
|
void Crafter::SetParentProject(Configuration* parent) {
|
|
ParentProject.store(parent);
|
|
}
|
|
|
|
Configuration* Crafter::ParentLib(std::string_view name) {
|
|
Configuration* root = ParentProject.load();
|
|
if (!root) {
|
|
throw std::runtime_error(std::format("Crafter::ParentLib('{}'): no parent project set", name));
|
|
}
|
|
std::unordered_set<Configuration*> seen;
|
|
if (auto found = FindLibInTree(root, name, seen)) return found;
|
|
throw std::runtime_error(std::format("Crafter::ParentLib('{}'): not found in parent project '{}'", name, root->name));
|
|
}
|
|
|
|
TestRunner TestRunner::Local() {
|
|
TestRunner r;
|
|
r.name = "local";
|
|
return r;
|
|
}
|
|
|
|
TestRunner TestRunner::Cmd(std::string command) {
|
|
TestRunner r;
|
|
r.name = std::format("cmd:{}", command);
|
|
r.exec = std::format("{} {{bin}} {{args}}", command);
|
|
r.probe = std::format("which {}", command);
|
|
return r;
|
|
}
|
|
|
|
TestRunner TestRunner::Wine() {
|
|
TestRunner r;
|
|
r.name = "wine";
|
|
r.exec = "wine {bin} {args}";
|
|
r.probe = "which wine";
|
|
return r;
|
|
}
|
|
|
|
TestRunner TestRunner::ForTarget(const Configuration& cfg) {
|
|
const std::string& target = cfg.target;
|
|
|
|
// Same triple as the host → run the binary directly. Covers the common
|
|
// case (cfg.target defaulted to HostTarget()) without any wrapper.
|
|
if (target == HostTarget()) return Local();
|
|
|
|
// Windows targets run under Wine; there is no Windows host.
|
|
if (TargetIsWindows(target)) return Wine();
|
|
|
|
// WASI: a .wasm file isn't directly executable; wasmtime is the canonical
|
|
// runtime. wasi-cli also works but the upstream Bytecode Alliance name is
|
|
// wasmtime, so we standardize on that.
|
|
if (target.starts_with("wasm32-wasi") || target.starts_with("wasm64-wasi")) {
|
|
return Cmd("wasmtime");
|
|
}
|
|
|
|
// Non-host Linux triple: extract the architecture and route through
|
|
// qemu-user. Triple is <arch>-<vendor>-<os>-<env> (or sometimes 3 parts);
|
|
// qemu-user's binary names mostly follow the arch field, with two known
|
|
// mismatches handled below. cfg.sysroot, when set, becomes QEMU_LD_PREFIX
|
|
// so the target's dynamic linker / shared libs are reachable — without
|
|
// it qemu-user crashes on dynamic ELFs with "could not open /lib/ld...".
|
|
if (target.find("-linux-") != std::string::npos) {
|
|
auto dash = target.find('-');
|
|
std::string arch = target.substr(0, dash);
|
|
// i686-linux-gnu → qemu-i386; arm-* already matches qemu-arm; aarch64,
|
|
// riscv64, ppc64le, mips, mips64, s390x all match their qemu names.
|
|
if (arch == "i686") arch = "i386";
|
|
TestRunner r = Cmd(std::format("qemu-{}", arch));
|
|
if (!cfg.sysroot.empty()) {
|
|
// Use `env VAR=value cmd` rather than the shell's `VAR=value cmd`
|
|
// prefix syntax: RunCommandWithTimeout pipes through GNU `timeout`,
|
|
// which execvp's its argument list directly without going through a
|
|
// shell. A bare VAR=value would be exec'd as a command path and
|
|
// fail with "No such file or directory".
|
|
// QEMU_LD_PREFIX only redirects the ELF interpreter lookup; the
|
|
// emulated loader then searches ITS default paths against the
|
|
// host filesystem, picking up host-arch libraries from /lib —
|
|
// LD_LIBRARY_PATH steers it back into the sysroot.
|
|
r.exec = std::format("env QEMU_LD_PREFIX={0} LD_LIBRARY_PATH={0}/lib:{0}/usr/lib {1}", cfg.sysroot, r.exec);
|
|
}
|
|
return r;
|
|
}
|
|
|
|
// Unknown / bare-metal / freestanding targets: fall back to Local. The
|
|
// caller's runner-availability probe (or absence of the binary) surfaces
|
|
// the problem rather than us inventing a wrong wrapper here.
|
|
return Local();
|
|
}
|
|
|
|
namespace {
|
|
std::string NormalizeTriple(std::string_view target) {
|
|
std::string out(target);
|
|
for (char& c : out) {
|
|
if (c == '-' || c == '.') c = '_';
|
|
}
|
|
return out;
|
|
}
|
|
|
|
// Spec grammar: "local" | "cmd:<binary>". Used by CRAFTER_BUILD_RUNNER_*
|
|
// env vars and --runner=. The set used to include ssh/sshwin/wsl variants;
|
|
// those were removed when transport-style runners were dropped (issue #8).
|
|
std::optional<TestRunner> ParseRunnerSpec(std::string_view spec) {
|
|
if (spec.empty()) return std::nullopt;
|
|
if (spec == "local") return TestRunner::Local();
|
|
if (spec.starts_with("cmd:") && spec.size() > 4) {
|
|
return TestRunner::Cmd(std::string(spec.substr(4)));
|
|
}
|
|
throw std::runtime_error(std::format("TestRunner::FromSpec: unrecognized runner spec '{}' " "(expected 'local' or 'cmd:<binary>')", spec));
|
|
}
|
|
}
|
|
|
|
std::optional<TestRunner> TestRunner::FromSpec(std::string_view spec) {
|
|
return ParseRunnerSpec(spec);
|
|
}
|
|
|
|
TestRunner TestRunner::FromEnv(std::string_view target, TestRunner fallback) {
|
|
std::string envName = std::format("CRAFTER_BUILD_RUNNER_{}", NormalizeTriple(target));
|
|
const char* v = std::getenv(envName.c_str());
|
|
if (!v || !*v) return fallback;
|
|
if (auto r = ParseRunnerSpec(v)) return std::move(*r);
|
|
return fallback;
|
|
}
|
|
|
|
TestResult Crafter::RunSingleTest(const Test& test, const fs::path& binary, std::chrono::seconds timeout) {
|
|
TestResult result;
|
|
result.name = test.config.name;
|
|
|
|
std::map<std::string, std::string> ph;
|
|
ph["{args}"] = JoinAndQuoteArgs(test.args);
|
|
|
|
auto start = std::chrono::steady_clock::now();
|
|
CommandResult r;
|
|
|
|
if (test.runner.exec.empty()) {
|
|
// Local runner: spawn the binary directly through the host shell.
|
|
std::string cmd = std::format("{} {}", ShellQuoteSh(binary.string()), ph["{args}"]);
|
|
r = RunCommandWithTimeout(cmd, timeout);
|
|
} else {
|
|
// Prefix runner (qemu-user, wasmtime, wine, ...): templated exec
|
|
// wraps the local binary.
|
|
ph["{bin}"] = binary.string();
|
|
r = RunCommandWithTimeout(Substitute(test.runner.exec, ph), timeout);
|
|
}
|
|
|
|
auto end = std::chrono::steady_clock::now();
|
|
result.duration = std::chrono::duration_cast<std::chrono::milliseconds>(end - start);
|
|
result.output = std::move(r.output);
|
|
result.exitCode = r.exitCode;
|
|
result.signal = r.signal;
|
|
|
|
if (r.timedOut) result.outcome = TestOutcome::Timeout;
|
|
else if (r.crashed) result.outcome = TestOutcome::Crash;
|
|
else if (r.exitCode == 77) result.outcome = TestOutcome::Skipped;
|
|
else if (r.exitCode != 0) result.outcome = TestOutcome::Fail;
|
|
else result.outcome = TestOutcome::Pass;
|
|
|
|
return result;
|
|
}
|
|
|
|
namespace {
|
|
bool ToolOnPath(std::string_view name) {
|
|
std::string cmd = std::format("which {} > /dev/null 2>&1", name);
|
|
return std::system(cmd.c_str()) == 0;
|
|
}
|
|
|
|
struct RequireResult {
|
|
bool ok;
|
|
std::string reason; // human-readable when !ok
|
|
};
|
|
|
|
// Evaluate each `<kind>:<arg>` precondition. Returns the first failure
|
|
// (short-circuit; reporting one missing dep at a time is enough to act on
|
|
// and keeps the test log uncluttered).
|
|
RequireResult EvaluateRequires(std::span<const std::string> reqs) {
|
|
for (const auto& r : reqs) {
|
|
auto sep = r.find(':');
|
|
if (sep == std::string::npos || sep == 0 || sep == r.size() - 1) {
|
|
return {false, std::format("malformed require '{}' (expected kind:arg)", r)};
|
|
}
|
|
std::string_view kind(r.data(), sep);
|
|
std::string_view arg(r.data() + sep + 1, r.size() - sep - 1);
|
|
if (kind == "tool") {
|
|
if (!ToolOnPath(arg)) {
|
|
return {false, std::format("tool '{}' not on PATH", arg)};
|
|
}
|
|
} else if (kind == "file") {
|
|
if (!fs::exists(std::string(arg))) {
|
|
return {false, std::format("file '{}' missing", arg)};
|
|
}
|
|
} else if (kind == "env") {
|
|
const char* v = std::getenv(std::string(arg).c_str());
|
|
if (!v || !*v) {
|
|
return {false, std::format("env '{}' unset", arg)};
|
|
}
|
|
} else {
|
|
return {false, std::format("unknown require kind '{}' (expected tool/file/env)", kind)};
|
|
}
|
|
}
|
|
return {true, ""};
|
|
}
|
|
|
|
// Match a runner's tool dependency against the test's declared
|
|
// requirements. Used to decide between Skip (declared, may legitimately
|
|
// be missing) and Fail (runner unavailable but test didn't declare it —
|
|
// a silent skip would mask broken cross-arch CI configuration).
|
|
bool RequiresMentionsTool(std::span<const std::string> reqs, std::string_view tool) {
|
|
std::string needle = std::format("tool:{}", tool);
|
|
return std::ranges::any_of(reqs, [&](const std::string& s) { return s == needle; });
|
|
}
|
|
|
|
// Best-effort extraction of the runner-tool name from a TestRunner so the
|
|
// hard-fail-unless-declared check can match it against `requires`. For
|
|
// Cmd("foo"), the name is "cmd:foo"; for Wine, it's "wine". Anything else
|
|
// (Local, transport runners) returns empty — those don't trigger the
|
|
// declared/undeclared gate.
|
|
std::string RunnerToolName(const TestRunner& runner) {
|
|
if (runner.name == "wine") return "wine";
|
|
if (runner.name.starts_with("cmd:")) {
|
|
std::string tool = runner.name.substr(4);
|
|
// QEMU_LD_PREFIX prefix may be glued onto exec but the runner's
|
|
// `name` field already isolates the command, so no extra parsing.
|
|
return tool;
|
|
}
|
|
return "";
|
|
}
|
|
}
|
|
|
|
TestBuilder Configuration::AddTest(std::string_view name) {
|
|
// Empty-interfaces trampoline. The span we hand to the (name, interfaces)
|
|
// overload wraps a stack array; it's only consulted inside that call, so
|
|
// its lifetime is fine for the duration.
|
|
std::array<fs::path, 0> noIfaces = {};
|
|
return AddTest(name, noIfaces);
|
|
}
|
|
|
|
TestBuilder Configuration::AddTest(std::string_view name, std::span<fs::path> interfaces) {
|
|
Test t;
|
|
// Default to "./" so tests/<name>/main.cpp resolves at the project root
|
|
// (where crafter-build sets cwd from project.cpp's location). Projects
|
|
// whose lib lives in a subdir (e.g. cfg.path = "./mylib/") get this
|
|
// right by default; if the test layout is unusual, override via .Path().
|
|
t.config.path = "./";
|
|
t.config.name = std::string(name);
|
|
t.config.outputName = std::string(name);
|
|
t.config.target = this->target;
|
|
t.config.march = this->march;
|
|
t.config.mtune = this->mtune;
|
|
t.config.sysroot = this->sysroot;
|
|
t.config.debug = this->debug;
|
|
// Inherited so a project flag that changes what the library contains also
|
|
// moves the test's own outputs, rather than letting two flag settings share
|
|
// one test bin dir.
|
|
t.config.projectArgs = this->projectArgs;
|
|
t.config.type = ConfigurationType::Executable;
|
|
|
|
// Default source layout: tests/<name>/main.cpp resolved against the
|
|
// parent Configuration's path. cfg.path is typically "./" (project
|
|
// root), which puts the test sources at <repo>/tests/<name>/main.cpp.
|
|
fs::path mainSource = fs::path("tests") / std::string(name) / "main";
|
|
std::array<fs::path, 1> impls = { mainSource };
|
|
t.config.GetInterfacesAndImplementations(interfaces, impls);
|
|
|
|
tests.push_back(std::move(t));
|
|
return TestBuilder{this, tests.size() - 1};
|
|
}
|
|
|
|
void Configuration::AddMarchVariants(std::string_view name, std::span<fs::path> interfaces, std::span<const MarchTier> tiers) {
|
|
for (const auto& tier : tiers) {
|
|
Test t;
|
|
t.config.path = "./";
|
|
t.config.name = std::format("{}-{}", name, tier.march);
|
|
t.config.outputName = t.config.name;
|
|
t.config.target = this->target;
|
|
t.config.march = tier.march;
|
|
t.config.mtune = tier.mtune;
|
|
t.config.sysroot = this->sysroot;
|
|
t.config.debug = this->debug;
|
|
t.config.projectArgs = this->projectArgs;
|
|
t.config.type = ConfigurationType::Executable;
|
|
|
|
fs::path mainSource = fs::path("tests") / std::string(name) / "main";
|
|
std::array<fs::path, 1> impls = { mainSource };
|
|
// Interfaces stamped directly into each variant's Configuration —
|
|
// NOT via cfg.dependencies. The dependency machinery caches PCMs by
|
|
// dep->VariantId(), which would compile the parent's interfaces
|
|
// once with the parent's march; for SIMD-sensitive code each
|
|
// variant needs its own compile. Listing the interfaces as the
|
|
// test's own forces per-variant recompilation.
|
|
t.config.GetInterfacesAndImplementations(interfaces, impls);
|
|
tests.push_back(std::move(t));
|
|
}
|
|
}
|
|
|
|
TestBuilder& TestBuilder::Path(fs::path p) { Ref().config.path = std::move(p); return *this; }
|
|
TestBuilder& TestBuilder::Target(std::string t) { Ref().config.target = std::move(t); return *this; }
|
|
TestBuilder& TestBuilder::March(std::string m) { Ref().config.march = std::move(m); return *this; }
|
|
TestBuilder& TestBuilder::Mtune(std::string m) { Ref().config.mtune = std::move(m); return *this; }
|
|
TestBuilder& TestBuilder::Sysroot(fs::path s) { Ref().config.sysroot = s.string(); return *this; }
|
|
TestBuilder& TestBuilder::Debug(bool d) { Ref().config.debug = d; return *this; }
|
|
TestBuilder& TestBuilder::Define(std::string n, std::string v) {
|
|
Ref().config.defines.push_back({std::move(n), std::move(v)});
|
|
return *this;
|
|
}
|
|
TestBuilder& TestBuilder::Timeout(std::chrono::seconds s) { Ref().timeout = s; return *this; }
|
|
TestBuilder& TestBuilder::Args(std::vector<std::string> a) { Ref().args = std::move(a); return *this; }
|
|
TestBuilder& TestBuilder::Requires(std::string r) { Ref().requires_.push_back(std::move(r)); return *this; }
|
|
TestBuilder& TestBuilder::Dependencies(std::vector<Configuration*> d) {
|
|
Ref().config.dependencies = std::move(d);
|
|
// AddTest already scanned tests/<name>/main.cpp, at which point this test
|
|
// had no dependencies, so every `import <DepModule>;` in it came back
|
|
// unresolved. Place them now that the libraries are known — without this
|
|
// the test's object carries no staleness edge to the interfaces it consumes
|
|
// and survives a layout change to them (issue #27). Build() re-runs the
|
|
// same sweep as a backstop; doing it here keeps the Configuration coherent
|
|
// for anyone inspecting it before the build.
|
|
Ref().config.ResolvePendingImports();
|
|
return *this;
|
|
}
|
|
TestBuilder& TestBuilder::LinkFlag(std::string f) { Ref().config.linkFlags.push_back(std::move(f)); return *this; }
|
|
TestBuilder& TestBuilder::CompileFlag(std::string f) { Ref().config.compileFlags.push_back(std::move(f)); return *this; }
|
|
|
|
TestSummary Crafter::RunTests(Configuration& projectCfg, const RunTestsOptions& opts, std::span<const std::string_view> projectArgs) {
|
|
// Multi-target sweep: when no --target= was given, the run covers every
|
|
// distinct target declared across projectCfg.tests plus the host target.
|
|
// Lets a bare `crafter-build test` exercise cross-arch tests without the
|
|
// user having to know which targets exist. An explicit --target=X
|
|
// bypasses the sweep and runs that target only.
|
|
if (opts.targetFilter.empty()) {
|
|
std::set<std::string> sweep;
|
|
sweep.insert(HostTarget());
|
|
for (const Test& t : projectCfg.tests) {
|
|
if (!t.config.target.empty()) sweep.insert(t.config.target);
|
|
}
|
|
TestSummary aggregate;
|
|
for (const auto& target : sweep) {
|
|
RunTestsOptions perTarget = opts;
|
|
perTarget.targetFilter = target;
|
|
if (sweep.size() > 1) {
|
|
Progress::Clear();
|
|
std::println("\n=== target: {} ===", target);
|
|
}
|
|
TestSummary s = RunTests(projectCfg, perTarget, projectArgs);
|
|
aggregate.passed += s.passed;
|
|
aggregate.failed += s.failed;
|
|
aggregate.crashed += s.crashed;
|
|
aggregate.timedOut += s.timedOut;
|
|
aggregate.skipped += s.skipped;
|
|
for (auto& r : s.results) aggregate.results.push_back(std::move(r));
|
|
}
|
|
return aggregate;
|
|
}
|
|
|
|
TestSummary summary;
|
|
|
|
// Filter by target + glob, derive each Test's runner just-in-time. The
|
|
// runner is recomputed each sweep iteration (rather than once at AddTest
|
|
// time) because runnerOverride and FromEnv resolution depend on the
|
|
// run's options, which the project.cpp doesn't know about.
|
|
std::vector<Test*> filtered;
|
|
filtered.reserve(projectCfg.tests.size());
|
|
for (auto& test : projectCfg.tests) {
|
|
if (test.config.target != opts.targetFilter) continue;
|
|
if (!MatchAny(opts.globs, test.config.name)) continue;
|
|
test.runner = TestRunner::FromEnv(test.config.target, TestRunner::ForTarget(test.config));
|
|
if (opts.runnerOverride) {
|
|
if (auto r = TestRunner::FromSpec(*opts.runnerOverride)) {
|
|
test.runner = std::move(*r);
|
|
}
|
|
}
|
|
filtered.push_back(&test);
|
|
}
|
|
|
|
if (opts.listOnly) {
|
|
for (auto* t : filtered) {
|
|
std::println("{}", t->config.name);
|
|
}
|
|
return summary;
|
|
}
|
|
|
|
if (filtered.empty()) {
|
|
std::println("No tests matched.");
|
|
return summary;
|
|
}
|
|
|
|
std::int32_t jobs = opts.jobs > 0
|
|
? opts.jobs
|
|
: std::max(1u, std::thread::hardware_concurrency());
|
|
jobs = std::min(jobs, static_cast<std::int32_t>(filtered.size()));
|
|
|
|
std::unordered_map<fs::path, std::shared_future<BuildResult>> depResults;
|
|
std::mutex depMutex;
|
|
std::mutex printMutex;
|
|
std::mutex probeMutex;
|
|
std::unordered_map<std::string, bool> probeCache;
|
|
std::atomic<std::size_t> next{0};
|
|
std::vector<TestResult> results(filtered.size());
|
|
|
|
auto runnerAvailable = [&](const TestRunner& runner) -> bool {
|
|
if (runner.probe.empty()) return true;
|
|
std::lock_guard lk(probeMutex);
|
|
if (auto it = probeCache.find(runner.name); it != probeCache.end()) {
|
|
return it->second;
|
|
}
|
|
// RunCommandChecked captures and discards output internally, so probe
|
|
// specs don't need `> /dev/null 2>&1` (which doesn't translate to cmd).
|
|
bool ok = (RunCommandChecked(runner.probe).exitCode == 0);
|
|
probeCache[runner.name] = ok;
|
|
return ok;
|
|
};
|
|
|
|
auto worker = [&]() {
|
|
while (true) {
|
|
std::size_t i = next.fetch_add(1);
|
|
if (i >= filtered.size()) break;
|
|
Test& t = *filtered[i];
|
|
TestResult r;
|
|
r.name = t.config.name;
|
|
|
|
// Declarative preconditions set via TestBuilder::Requires. Evaluated
|
|
// before the build so a missing tool/file/env turns into a Skip without
|
|
// paying the compile cost. Reports the first failure only — once one
|
|
// precondition is unmet the test couldn't run anyway, and a wall of
|
|
// "also missing X, also missing Y" buries the actionable root cause.
|
|
if (auto req = EvaluateRequires(t.requires_); !req.ok) {
|
|
r.outcome = TestOutcome::Skipped;
|
|
r.output = req.reason;
|
|
{
|
|
std::lock_guard lk(printMutex);
|
|
PrintResult(r, t.runner.name);
|
|
}
|
|
results[i] = std::move(r);
|
|
continue;
|
|
}
|
|
|
|
if (!runnerAvailable(t.runner)) {
|
|
// Hard-fail-unless-declared: if the runner depends on a tool
|
|
// (qemu-aarch64, wasmtime, wine, ...) and the test didn't say
|
|
// "tool:<that>" in requires, the missing runner is a Fail. The
|
|
// intent is to surface broken cross-arch CI configuration
|
|
// instead of letting it masquerade as a Skip; tests that
|
|
// legitimately may run without their runner have to opt in.
|
|
std::string tool = RunnerToolName(t.runner);
|
|
if (!tool.empty() && !RequiresMentionsTool(t.requires_, tool)) {
|
|
r.outcome = TestOutcome::Fail;
|
|
r.exitCode = -1;
|
|
r.output = std::format("runner '{}' unavailable and not declared in requires " "(add .Requires(\"tool:{}\") to permit skipping)", t.runner.name, tool);
|
|
} else {
|
|
r.outcome = TestOutcome::Skipped;
|
|
r.output = std::format("runner '{}' not available", t.runner.name);
|
|
}
|
|
{
|
|
std::lock_guard lk(printMutex);
|
|
PrintResult(r, t.runner.name);
|
|
}
|
|
results[i] = std::move(r);
|
|
continue;
|
|
}
|
|
|
|
BuildResult br;
|
|
try {
|
|
br = Build(t.config, depResults, depMutex);
|
|
} catch (const std::exception& e) {
|
|
r.outcome = TestOutcome::Fail;
|
|
r.output = std::format("build threw: {}", e.what());
|
|
r.exitCode = -1;
|
|
{
|
|
std::lock_guard lk(printMutex);
|
|
PrintResult(r, t.runner.name);
|
|
}
|
|
results[i] = std::move(r);
|
|
continue;
|
|
}
|
|
|
|
if (!br.result.empty()) {
|
|
r.outcome = TestOutcome::Fail;
|
|
r.output = std::format("build failed: {}", br.result);
|
|
r.exitCode = -1;
|
|
{
|
|
std::lock_guard lk(printMutex);
|
|
PrintResult(r, t.runner.name);
|
|
}
|
|
results[i] = std::move(r);
|
|
continue;
|
|
}
|
|
|
|
std::chrono::seconds timeout = opts.timeoutOverride.value_or(t.timeout);
|
|
fs::path binary = TestBinaryPath(t.config);
|
|
try {
|
|
r = RunSingleTest(t, binary, timeout);
|
|
} catch (const std::exception& e) {
|
|
r.outcome = TestOutcome::Fail;
|
|
r.output = std::format("runner threw: {}", e.what());
|
|
r.exitCode = -1;
|
|
}
|
|
|
|
if (r.outcome != TestOutcome::Pass && r.outcome != TestOutcome::Skipped && !r.output.empty()) {
|
|
WriteLog(projectCfg.path, r.name, r.output);
|
|
}
|
|
|
|
{
|
|
std::lock_guard lk(printMutex);
|
|
PrintResult(r, t.runner.name);
|
|
}
|
|
results[i] = std::move(r);
|
|
}
|
|
};
|
|
|
|
std::vector<std::jthread> threads;
|
|
threads.reserve(jobs);
|
|
for (std::int32_t j = 0; j < jobs; ++j) {
|
|
threads.emplace_back(worker);
|
|
}
|
|
threads.clear(); // joins all jthreads
|
|
|
|
for (auto& r : results) {
|
|
switch (r.outcome) {
|
|
case TestOutcome::Pass: summary.passed++; break;
|
|
case TestOutcome::Fail: summary.failed++; break;
|
|
case TestOutcome::Crash: summary.crashed++; break;
|
|
case TestOutcome::Timeout: summary.timedOut++; break;
|
|
case TestOutcome::Skipped: summary.skipped++; break;
|
|
}
|
|
}
|
|
summary.results = std::move(results);
|
|
|
|
Progress::Clear();
|
|
std::print("\n");
|
|
std::vector<std::string> parts;
|
|
if (summary.passed) parts.push_back(std::format("{} passed", summary.passed));
|
|
if (summary.failed) parts.push_back(std::format("{} failed", summary.failed));
|
|
if (summary.crashed) parts.push_back(std::format("{} crashed", summary.crashed));
|
|
if (summary.timedOut) parts.push_back(std::format("{} timed out", summary.timedOut));
|
|
if (summary.skipped) parts.push_back(std::format("{} skipped", summary.skipped));
|
|
std::string joined;
|
|
for (std::size_t i = 0; i < parts.size(); ++i) {
|
|
if (i) joined += ", ";
|
|
joined += parts[i];
|
|
}
|
|
std::println("{}", joined);
|
|
|
|
return summary;
|
|
}
|