/* Crafter® Build Copyright (C) 2026 Catcrafts® Catcrafts.net This library is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License version 3.0 as published by the Free Software Foundation; This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. You should have received a copy of the GNU Lesser General Public License along with this library; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA */ module; // toml++ is consumed as a translation-unit-private dependency in the GMF: the // parser is only needed for test.toml discovery here, so keeping it out of // `import std`-using module purviews avoids dragging it through the module // graph (and through every PCM consumer of Crafter.Build). #include "../lib/toml.hpp" export module Crafter.Build:Test_impl; import std; import :Test; import :Clang; import :Platform; import :Progress; namespace fs = std::filesystem; using namespace Crafter; namespace { bool TargetIsWindows(std::string_view target) { return target.find("windows") != std::string_view::npos || target.find("mingw") != std::string_view::npos; } fs::path TestBinaryPath(const Configuration& cfg) { fs::path outputDir = cfg.BinDir(); return outputDir / (TargetIsWindows(cfg.target) ? cfg.outputName + ".exe" : cfg.outputName); } bool MatchGlob(std::string_view glob, std::string_view name) { std::size_t gi = 0, ni = 0, star = std::string_view::npos, mark = 0; while (ni < name.size()) { if (gi < glob.size() && (glob[gi] == '?' || glob[gi] == name[ni])) { ++gi; ++ni; } else if (gi < glob.size() && glob[gi] == '*') { star = gi++; mark = ni; } else if (star != std::string_view::npos) { gi = star + 1; ni = ++mark; } else { return false; } } while (gi < glob.size() && glob[gi] == '*') ++gi; return gi == glob.size(); } bool MatchAny(std::span globs, std::string_view name) { if (globs.empty()) return true; for (const auto& g : globs) { if (MatchGlob(g, name)) return true; } return false; } std::string ShellQuoteSh(std::string_view s) { std::string out; out.reserve(s.size() + 2); out.push_back('\''); for (char c : s) { if (c == '\'') out += "'\\''"; else out.push_back(c); } out.push_back('\''); return out; } // cmd.exe doesn't recognize '...' as quoting (it would pass the single // quotes through to the executable). Wrap in "..." for cmd; embedded " // is rare in paths but escape it to be safe. Backslash sequences before // " don't need the MS-CRT doubling rules because we go through // `cmd /C "..."`, which uses cmd's parser, not the CRT's argv splitter. std::string ShellQuoteCmd(std::string_view s) { std::string out; out.reserve(s.size() + 2); out.push_back('"'); for (char c : s) { if (c == '"') out += "\\\""; else out.push_back(c); } out.push_back('"'); return out; } // Host-shell quoting: sh on Linux, cmd on Windows. For args/paths that // hit the local shell (Local runner exec, Cmd-prefix runner exec). std::string ShellQuoteHost(std::string_view s) { #ifdef _WIN32 return ShellQuoteCmd(s); #else return ShellQuoteSh(s); #endif } std::string JoinAndQuoteArgs(std::span args) { std::string out; for (const auto& a : args) { if (!out.empty()) out.push_back(' '); out += ShellQuoteHost(a); } return out; } std::string Substitute(std::string_view tmpl, const std::map& ph) { std::string out; out.reserve(tmpl.size()); std::size_t i = 0; while (i < tmpl.size()) { if (tmpl[i] == '{') { std::size_t end = tmpl.find('}', i + 1); if (end != std::string_view::npos) { std::string key(tmpl.substr(i, end - i + 1)); if (auto it = ph.find(key); it != ph.end()) { out += it->second; i = end + 1; continue; } } } out.push_back(tmpl[i++]); } return out; } std::string SignalName(int sig) { switch (sig) { case 1: return "SIGHUP"; case 2: return "SIGINT"; case 4: return "SIGILL"; case 6: return "SIGABRT"; case 8: return "SIGFPE"; case 9: return "SIGKILL"; case 11: return "SIGSEGV"; case 13: return "SIGPIPE"; case 14: return "SIGALRM"; case 15: return "SIGTERM"; default: return std::format("signal {}", sig); } } void WriteLog(const fs::path& projectPath, const std::string& name, const std::string& output) { fs::path logDir = projectPath / "build" / "test-logs"; std::error_code ec; fs::create_directories(logDir, ec); if (ec) return; std::ofstream(logDir / (name + ".log")) << output; } void PrintResult(const TestResult& r, std::string_view runnerName) { Progress::Clear(); auto ms = r.duration.count(); std::string runnerSuffix = (runnerName.empty() || runnerName == "local") ? std::string() : std::format(" ({})", runnerName); switch (r.outcome) { case TestOutcome::Pass: std::println("✅ {}{} ({}ms)", r.name, runnerSuffix, ms); break; case TestOutcome::Fail: std::println("❌ {}{} ({}ms) exit {}", r.name, runnerSuffix, ms, r.exitCode); if (!r.output.empty()) { for (auto line : std::views::split(r.output, '\n')) { std::string_view sv(line.begin(), line.end()); if (!sv.empty()) std::println(" {}", sv); } } break; case TestOutcome::Crash: std::println("\U0001F4A5 {}{} ({}ms) crashed: {}", r.name, runnerSuffix, ms, SignalName(r.signal)); if (!r.output.empty()) { for (auto line : std::views::split(r.output, '\n')) { std::string_view sv(line.begin(), line.end()); if (!sv.empty()) std::println(" {}", sv); } } break; case TestOutcome::Timeout: std::println("⏱ {}{} ({}ms) timeout", r.name, runnerSuffix, ms); break; case TestOutcome::Skipped: std::println("⏭ {}{} skipped: {}", r.name, runnerSuffix, r.output.empty() ? std::string("(no reason)") : r.output); break; } } } namespace { std::atomic g_parentProject{nullptr}; Configuration* FindLibInTree(Configuration* root, std::string_view name, std::unordered_set& seen) { if (!seen.insert(root).second) return nullptr; if (root->name == name) return root; for (Configuration* dep : root->dependencies) { if (auto found = FindLibInTree(dep, name, seen)) return found; } return nullptr; } } void Crafter::SetParentProject(Configuration* parent) { g_parentProject.store(parent); } Configuration* Crafter::ParentLib(std::string_view name) { Configuration* root = g_parentProject.load(); if (!root) { throw std::runtime_error(std::format( "Crafter::ParentLib('{}'): no parent project set", name)); } std::unordered_set seen; if (auto found = FindLibInTree(root, name, seen)) return found; throw std::runtime_error(std::format( "Crafter::ParentLib('{}'): not found in parent project '{}'", name, root->name)); } TestRunner TestRunner::Local() { TestRunner r; r.name = "local"; return r; } TestRunner TestRunner::Cmd(std::string command) { TestRunner r; r.name = std::format("cmd:{}", command); r.exec = std::format("{} {{bin}} {{args}}", command); #ifdef _WIN32 r.probe = std::format("where {}", command); #else r.probe = std::format("which {}", command); #endif return r; } TestRunner TestRunner::Wine() { TestRunner r; r.name = "wine"; r.exec = "wine {bin} {args}"; #ifdef _WIN32 r.probe = "where wine"; #else r.probe = "which wine"; #endif return r; } TestRunner TestRunner::ForTarget(const Configuration& cfg) { const std::string& target = cfg.target; // Same triple as the host → run the binary directly. Covers the common // case (cfg.target defaulted to HostTarget()) without any wrapper. if (target == HostTarget()) return Local(); // Windows targets: native on a Windows host, Wine on Linux. We don't // distinguish mingw vs msvc here — the produced .exe runs the same way. if (TargetIsWindows(target)) { return TargetIsWindows(HostTarget()) ? Local() : Wine(); } // WASI: a .wasm file isn't directly executable; wasmtime is the canonical // runtime. wasi-cli also works but the upstream Bytecode Alliance name is // wasmtime, so we standardize on that. if (target.starts_with("wasm32-wasi") || target.starts_with("wasm64-wasi")) { return Cmd("wasmtime"); } // Non-host Linux triple: extract the architecture and route through // qemu-user. Triple is --- (or sometimes 3 parts); // qemu-user's binary names mostly follow the arch field, with two known // mismatches handled below. cfg.sysroot, when set, becomes QEMU_LD_PREFIX // so the target's dynamic linker / shared libs are reachable — without // it qemu-user crashes on dynamic ELFs with "could not open /lib/ld...". if (target.find("-linux-") != std::string::npos) { auto dash = target.find('-'); std::string arch = target.substr(0, dash); // i686-linux-gnu → qemu-i386; arm-* already matches qemu-arm; aarch64, // riscv64, ppc64le, mips, mips64, s390x all match their qemu names. if (arch == "i686") arch = "i386"; TestRunner r = Cmd(std::format("qemu-{}", arch)); if (!cfg.sysroot.empty()) { // Use `env VAR=value cmd` rather than the shell's `VAR=value cmd` // prefix syntax: RunCommandWithTimeout pipes through GNU `timeout`, // which execvp's its argument list directly without going through a // shell. A bare VAR=value would be exec'd as a command path and // fail with "No such file or directory". r.exec = std::format("env QEMU_LD_PREFIX={} {}", cfg.sysroot, r.exec); } return r; } // Unknown / bare-metal / freestanding targets: fall back to Local. The // caller's runner-availability probe (or absence of the binary) surfaces // the problem rather than us inventing a wrong wrapper here. return Local(); } namespace { std::string NormalizeTriple(std::string_view target) { std::string out(target); for (char& c : out) { if (c == '-' || c == '.') c = '_'; } return out; } // Spec grammar: "local" | "cmd:". Used by CRAFTER_BUILD_RUNNER_* // env vars and --runner=. The set used to include ssh/sshwin/wsl variants; // those were removed when transport-style runners were dropped (issue #8). std::optional ParseRunnerSpec(std::string_view spec) { if (spec.empty()) return std::nullopt; if (spec == "local") return TestRunner::Local(); if (spec.starts_with("cmd:") && spec.size() > 4) { return TestRunner::Cmd(std::string(spec.substr(4))); } throw std::runtime_error(std::format( "TestRunner::FromSpec: unrecognized runner spec '{}' " "(expected 'local' or 'cmd:')", spec)); } } std::optional TestRunner::FromSpec(std::string_view spec) { return ParseRunnerSpec(spec); } TestRunner TestRunner::FromEnv(std::string_view target, TestRunner fallback) { std::string envName = std::format("CRAFTER_BUILD_RUNNER_{}", NormalizeTriple(target)); const char* v = std::getenv(envName.c_str()); if (!v || !*v) return fallback; if (auto r = ParseRunnerSpec(v)) return std::move(*r); return fallback; } TestResult Crafter::RunSingleTest(const Test& test, const fs::path& binary, std::chrono::seconds timeout) { TestResult result; result.name = test.config.name; std::map ph; ph["{args}"] = JoinAndQuoteArgs(test.args); auto start = std::chrono::steady_clock::now(); CommandResult r; if (test.runner.exec.empty()) { // Local runner: spawn the binary directly through the host shell. std::string cmd = std::format("{} {}", ShellQuoteHost(binary.string()), ph["{args}"]); r = RunCommandWithTimeout(cmd, timeout); } else { // Prefix runner (qemu-user, wasmtime, wine, ...): templated exec // wraps the local binary. ph["{bin}"] = binary.string(); r = RunCommandWithTimeout(Substitute(test.runner.exec, ph), timeout); } auto end = std::chrono::steady_clock::now(); result.duration = std::chrono::duration_cast(end - start); result.output = std::move(r.output); result.exitCode = r.exitCode; result.signal = r.signal; if (r.timedOut) result.outcome = TestOutcome::Timeout; else if (r.crashed) result.outcome = TestOutcome::Crash; else if (r.exitCode == 77) result.outcome = TestOutcome::Skipped; else if (r.exitCode != 0) result.outcome = TestOutcome::Fail; else result.outcome = TestOutcome::Pass; return result; } namespace { // Declarative test metadata loaded from tests//test.toml. Lets a test // ship just main.cpp + a few lines of config instead of a whole project.cpp // when its needs are "pick a target, gate on prerequisites, run with these // args". project.cpp stays the escape hatch for outer-driver tests that // call Build() / inspect intermediate state. struct TestManifest { std::optional target; std::optional march; std::optional mtune; std::optional sysroot; std::vector requires_; std::optional timeoutSeconds; std::vector args; std::vector> defines; }; TestManifest ParseTestManifest(const fs::path& path) { // toml++ builds with exceptions enabled by default; parse_file throws // toml::parse_error on malformed input. Rethrow with the path attached // so the discovery loop's catch can surface "where the error came from" // alongside toml++'s "what was wrong". toml::table t; try { t = toml::parse_file(path.string()); } catch (const toml::parse_error& e) { throw std::runtime_error(std::format( "test.toml parse error in {}: {}", path.string(), std::string_view(e.description()))); } TestManifest m; if (auto v = t["target"].value()) m.target = *v; if (auto v = t["march"].value()) m.march = *v; if (auto v = t["mtune"].value()) m.mtune = *v; if (auto v = t["sysroot"].value()) m.sysroot = *v; if (auto v = t["timeout"].value()) m.timeoutSeconds = static_cast(*v); if (auto arr = t["requires"].as_array()) { for (auto& el : *arr) { if (auto s = el.value()) m.requires_.push_back(*s); } } if (auto arr = t["args"].as_array()) { for (auto& el : *arr) { if (auto s = el.value()) m.args.push_back(*s); } } if (auto tbl = t["defines"].as_table()) { for (auto&& [k, v] : *tbl) { if (auto s = v.value()) { m.defines.emplace_back(std::string(k.str()), *s); } } } return m; } // Apply manifest overlay onto a Configuration synthesized from the test // folder. Target overrides come last so a manifest's `target = "..."` // wins over the synth default (= run's targetFilter). Defines accumulate; // they don't replace pre-existing ones. void ApplyManifest(Configuration& cfg, const TestManifest& m) { if (m.target) cfg.target = *m.target; if (m.march) cfg.march = *m.march; if (m.mtune) cfg.mtune = *m.mtune; if (m.sysroot) cfg.sysroot = *m.sysroot; for (auto& [k, v] : m.defines) cfg.defines.push_back({k, v}); } bool ToolOnPath(std::string_view name) { #ifdef _WIN32 std::string cmd = std::format("where {} > nul 2>&1", name); #else std::string cmd = std::format("which {} > /dev/null 2>&1", name); #endif return std::system(cmd.c_str()) == 0; } struct RequireResult { bool ok; std::string reason; // human-readable when !ok }; // Evaluate each `:` precondition. Returns the first failure // (short-circuit; reporting one missing dep at a time is enough to act on // and keeps the test log uncluttered). RequireResult EvaluateRequires(std::span reqs) { for (const auto& r : reqs) { auto sep = r.find(':'); if (sep == std::string::npos || sep == 0 || sep == r.size() - 1) { return {false, std::format("malformed require '{}' (expected kind:arg)", r)}; } std::string_view kind(r.data(), sep); std::string_view arg(r.data() + sep + 1, r.size() - sep - 1); if (kind == "tool") { if (!ToolOnPath(arg)) { return {false, std::format("tool '{}' not on PATH", arg)}; } } else if (kind == "file") { if (!fs::exists(std::string(arg))) { return {false, std::format("file '{}' missing", arg)}; } } else if (kind == "env") { const char* v = std::getenv(std::string(arg).c_str()); if (!v || !*v) { return {false, std::format("env '{}' unset", arg)}; } } else { return {false, std::format( "unknown require kind '{}' (expected tool/file/env)", kind)}; } } return {true, ""}; } // Match a runner's tool dependency against the test's declared // requirements. Used to decide between Skip (declared, may legitimately // be missing) and Fail (runner unavailable but test didn't declare it — // a silent skip would mask broken cross-arch CI configuration). bool RequiresMentionsTool(std::span reqs, std::string_view tool) { std::string needle = std::format("tool:{}", tool); return std::ranges::any_of(reqs, [&](const std::string& s) { return s == needle; }); } // Best-effort extraction of the runner-tool name from a TestRunner so the // hard-fail-unless-declared check can match it against `requires`. For // Cmd("foo"), the name is "cmd:foo"; for Wine, it's "wine". Anything else // (Local, transport runners) returns empty — those don't trigger the // declared/undeclared gate. std::string RunnerToolName(const TestRunner& runner) { if (runner.name == "wine") return "wine"; if (runner.name.starts_with("cmd:")) { std::string tool = runner.name.substr(4); // QEMU_LD_PREFIX prefix may be glued onto exec but the runner's // `name` field already isolates the command, so no extra parsing. return tool; } return ""; } } namespace { // Synthesize a Configuration for tests// folders that don't contain // a project.cpp. Convention: cfg.path = the folder, cfg.name/outputName = // folder basename, cfg.target = host (overridable via test.toml `target`), // cfg.type = exe. Sources: top-level *.cpp (excluding project.cpp) become // implementations, interfaces/*.cppm become module interfaces. Tests with // deeper layouts or dependencies still need an explicit project.cpp. // // Why host-default instead of targetFilter-default: under the multi-target // sweep, an arch-agnostic test (no test.toml target) should run at the // host iteration only — not get rebuilt against every cross-target the // suite happens to declare. Cross-targeting is an opt-in via test.toml. Configuration SynthesizeTest(const fs::path& dir) { Configuration cfg; cfg.path = dir; cfg.name = dir.filename().string(); cfg.outputName = cfg.name; cfg.target = HostTarget(); cfg.type = ConfigurationType::Executable; std::vector impls; for (auto& e : fs::directory_iterator(dir)) { if (!e.is_regular_file()) continue; auto p = e.path(); if (p.extension() != ".cpp") continue; if (p.filename() == "project.cpp") continue; impls.push_back(p.stem()); } std::ranges::sort(impls); std::vector ifaces; fs::path interfacesDir = dir / "interfaces"; if (fs::exists(interfacesDir) && fs::is_directory(interfacesDir)) { for (auto& e : fs::directory_iterator(interfacesDir)) { if (!e.is_regular_file()) continue; auto p = e.path(); if (p.extension() != ".cppm") continue; ifaces.push_back(fs::path("interfaces") / p.stem()); } std::ranges::sort(ifaces); } if (impls.empty() && ifaces.empty()) { throw std::runtime_error(std::format( "no .cpp or interfaces/*.cppm files found in {}", dir.string())); } cfg.GetInterfacesAndImplementations(ifaces, impls); return cfg; } } TestSummary Crafter::RunTests(Configuration& projectCfg, const RunTestsOptions& opts, std::span projectArgs) { // Multi-target sweep: when no --target= was given, the run covers every // distinct target a test.toml declares plus the host target. Lets a bare // `crafter-build test` exercise cross-arch tests without the user having // to know which targets exist in this project. An explicit --target=X // bypasses the sweep and runs that target only. if (opts.targetFilter.empty()) { std::set sweep; sweep.insert(HostTarget()); fs::path testsDir = fs::current_path() / "tests"; if (fs::exists(testsDir) && fs::is_directory(testsDir)) { for (auto& e : fs::directory_iterator(testsDir)) { if (!e.is_directory()) continue; auto stem = e.path().filename().string(); if (stem.empty() || stem[0] == '_' || stem[0] == '.') continue; fs::path tomlPath = e.path() / "test.toml"; if (!fs::exists(tomlPath)) continue; try { TestManifest m = ParseTestManifest(tomlPath); if (m.target) sweep.insert(*m.target); } catch (...) { // Parse failures surface as discovery failures during the // actual run; the sweep phase just collects targets. } } } TestSummary aggregate; // Inline tests pushed by the caller (fixture-driven inner RunTests // calls, e.g. RunnerClassification) must survive each sweep // iteration. Configuration isn't copyable so we can't snapshot+restore; // instead, remember the inline count and erase only the entries // appended by the previous iteration's auto-discovery. size_t inlineCount = projectCfg.tests.size(); for (const auto& target : sweep) { RunTestsOptions perTarget = opts; perTarget.targetFilter = target; if (projectCfg.tests.size() > inlineCount) { projectCfg.tests.erase( projectCfg.tests.begin() + inlineCount, projectCfg.tests.end()); } if (sweep.size() > 1) { Progress::Clear(); std::println("\n=== target: {} ===", target); } TestSummary s = RunTests(projectCfg, perTarget, projectArgs); aggregate.passed += s.passed; aggregate.failed += s.failed; aggregate.crashed += s.crashed; aggregate.timedOut += s.timedOut; aggregate.skipped += s.skipped; for (auto& r : s.results) aggregate.results.push_back(std::move(r)); } return aggregate; } TestSummary summary; std::vector discoveryFailures; // Auto-discover tests one layer deep: each /tests// folder // is a test. If it contains project.cpp, that's loaded for full control; // otherwise the Configuration is synthesized from the folder contents. // Folders whose name starts with '_' or '.' are skipped (so tests/_shared/ // holds cross-test code without becoming a test). Each project.cpp receives // the same args the root project did, so --target=... propagates through. // // Discovery is keyed off cwd (= the project root, since crafter-build loads // ./project.cpp), not projectCfg.path: tests live at the project root even // when projectCfg.path points at a subdirectory like "./src/" or "./lib/". fs::path testsDir = fs::current_path() / "tests"; if (fs::exists(testsDir) && fs::is_directory(testsDir)) { // A discovered fixture is one of: // - project.cpp present → outer-driver test (LoadProject) // - test.toml present → declarative synth + manifest // - neither, just *.cpp → bare synth (host-target) // - both project.cpp and test.toml → XOR violation, discovery Fail struct TestEntry { fs::path dir; fs::path pcpp; // outer-driver path std::optional manifest; }; std::vector entries; for (auto& entry : fs::directory_iterator(testsDir)) { if (!entry.is_directory()) continue; auto stem = entry.path().filename().string(); if (stem.empty() || stem[0] == '_' || stem[0] == '.') continue; TestEntry te; te.dir = entry.path(); auto pcpp = te.dir / "project.cpp"; auto tomlPath = te.dir / "test.toml"; bool hasPcpp = fs::exists(pcpp); bool hasToml = fs::exists(tomlPath); if (hasPcpp && hasToml) { TestResult r; r.name = stem; r.outcome = TestOutcome::Fail; r.exitCode = -1; r.output = "both project.cpp and test.toml present — they're " "mutually exclusive (delete one to disambiguate " "outer-driver vs declarative test)"; discoveryFailures.push_back(std::move(r)); continue; } if (hasPcpp) te.pcpp = pcpp; if (hasToml) { try { te.manifest = ParseTestManifest(tomlPath); } catch (const std::exception& e) { TestResult r; r.name = stem; r.outcome = TestOutcome::Fail; r.exitCode = -1; r.output = e.what(); discoveryFailures.push_back(std::move(r)); continue; } } entries.push_back(std::move(te)); } std::ranges::sort(entries, [](auto& a, auto& b) { return a.dir < b.dir; }); // Inject --target= into the args we hand each fixture so its // CrafterBuildProject can default to the run's target. The CLI's own // --target=... propagates through projectArgs already; this only // appends when missing so an explicit user choice wins. std::string targetArg = std::format("--target={}", opts.targetFilter); std::vector fixtureArgs(projectArgs.begin(), projectArgs.end()); bool hasTarget = std::ranges::any_of(fixtureArgs, [](std::string_view a) { return a.starts_with("--target="); }); if (!hasTarget) fixtureArgs.push_back(targetArg); for (auto& te : entries) { Test t; try { if (!te.pcpp.empty()) { t.config = LoadProject(te.pcpp, fixtureArgs); } else { t.config = SynthesizeTest(te.dir); if (te.manifest) { ApplyManifest(t.config, *te.manifest); if (te.manifest->timeoutSeconds) { t.timeout = std::chrono::seconds(*te.manifest->timeoutSeconds); } t.args = te.manifest->args; t.requires_ = te.manifest->requires_; } } } catch (const std::exception& e) { // A broken fixture shouldn't kill the whole run. Surface as a // Fail and let other tests proceed. TestResult r; r.name = te.dir.filename().string(); r.outcome = TestOutcome::Fail; r.exitCode = -1; r.output = !te.pcpp.empty() ? std::format("project.cpp failed to load: {}", e.what()) : std::format("test discovery failed: {}", e.what()); discoveryFailures.push_back(std::move(r)); continue; } if (t.config.target != opts.targetFilter) continue; t.runner = TestRunner::FromEnv(t.config.target, TestRunner::ForTarget(t.config)); if (opts.runnerOverride) { if (auto r = TestRunner::FromSpec(*opts.runnerOverride)) { t.runner = std::move(*r); } } projectCfg.tests.push_back(std::move(t)); } } std::vector filtered; filtered.reserve(projectCfg.tests.size()); for (auto& test : projectCfg.tests) { if (MatchAny(opts.globs, test.config.name)) { filtered.push_back(&test); } } if (opts.listOnly) { for (auto& r : discoveryFailures) { std::println("{} (project.cpp broken)", r.name); } for (auto* t : filtered) { std::println("{}", t->config.name); } return summary; } if (filtered.empty() && discoveryFailures.empty()) { std::println("No tests matched."); return summary; } // Render discovery failures upfront so they appear before parallel test // results. They're already-determined Fails — no need to put them through // the worker pool. for (auto& r : discoveryFailures) { PrintResult(r, ""); WriteLog(projectCfg.path, r.name, r.output); } int jobs = opts.jobs > 0 ? opts.jobs : std::max(1u, std::thread::hardware_concurrency()); jobs = std::min(jobs, static_cast(filtered.size())); std::unordered_map> depResults; std::mutex depMutex; std::mutex printMutex; std::mutex probeMutex; std::unordered_map probeCache; std::atomic next{0}; std::vector results(filtered.size()); auto runnerAvailable = [&](const TestRunner& runner) -> bool { if (runner.probe.empty()) return true; std::lock_guard lk(probeMutex); if (auto it = probeCache.find(runner.name); it != probeCache.end()) { return it->second; } // RunCommandChecked captures and discards output internally, so probe // specs don't need `> /dev/null 2>&1` (which doesn't translate to cmd). bool ok = (RunCommandChecked(runner.probe).exitCode == 0); probeCache[runner.name] = ok; return ok; }; auto worker = [&]() { while (true) { std::size_t i = next.fetch_add(1); if (i >= filtered.size()) break; Test& t = *filtered[i]; TestResult r; r.name = t.config.name; // Declarative preconditions (test.toml requires = [...] or Test.requires_ // set in project.cpp). Evaluated before the build so a missing tool/file/env // turns into a Skip without paying the compile cost. Reports the first // failure only — once one precondition is unmet the test couldn't run // anyway, and a wall of "also missing X, also missing Y" buries the // actionable root cause. if (auto req = EvaluateRequires(t.requires_); !req.ok) { r.outcome = TestOutcome::Skipped; r.output = req.reason; { std::lock_guard lk(printMutex); PrintResult(r, t.runner.name); } results[i] = std::move(r); continue; } if (!runnerAvailable(t.runner)) { // Hard-fail-unless-declared: if the runner depends on a tool // (qemu-aarch64, wasmtime, wine, ...) and the test didn't say // "tool:" in requires, the missing runner is a Fail. The // intent is to surface broken cross-arch CI configuration // instead of letting it masquerade as a Skip; tests that // legitimately may run without their runner have to opt in. std::string tool = RunnerToolName(t.runner); if (!tool.empty() && !RequiresMentionsTool(t.requires_, tool)) { r.outcome = TestOutcome::Fail; r.exitCode = -1; r.output = std::format( "runner '{}' unavailable and not declared in requires " "(add 'tool:{}' to test.toml requires to permit skipping)", t.runner.name, tool); } else { r.outcome = TestOutcome::Skipped; r.output = std::format("runner '{}' not available", t.runner.name); } { std::lock_guard lk(printMutex); PrintResult(r, t.runner.name); } results[i] = std::move(r); continue; } BuildResult br; try { br = Build(t.config, depResults, depMutex); } catch (const std::exception& e) { r.outcome = TestOutcome::Fail; r.output = std::format("build threw: {}", e.what()); r.exitCode = -1; { std::lock_guard lk(printMutex); PrintResult(r, t.runner.name); } results[i] = std::move(r); continue; } if (!br.result.empty()) { r.outcome = TestOutcome::Fail; r.output = std::format("build failed: {}", br.result); r.exitCode = -1; { std::lock_guard lk(printMutex); PrintResult(r, t.runner.name); } results[i] = std::move(r); continue; } std::chrono::seconds timeout = opts.timeoutOverride.value_or(t.timeout); fs::path binary = TestBinaryPath(t.config); try { r = RunSingleTest(t, binary, timeout); } catch (const std::exception& e) { r.outcome = TestOutcome::Fail; r.output = std::format("runner threw: {}", e.what()); r.exitCode = -1; } if (r.outcome != TestOutcome::Pass && r.outcome != TestOutcome::Skipped && !r.output.empty()) { WriteLog(projectCfg.path, r.name, r.output); } { std::lock_guard lk(printMutex); PrintResult(r, t.runner.name); } results[i] = std::move(r); } }; std::vector threads; threads.reserve(jobs); for (int j = 0; j < jobs; ++j) { threads.emplace_back(worker); } threads.clear(); // joins all jthreads for (auto& r : discoveryFailures) { results.push_back(std::move(r)); } for (auto& r : results) { switch (r.outcome) { case TestOutcome::Pass: summary.passed++; break; case TestOutcome::Fail: summary.failed++; break; case TestOutcome::Crash: summary.crashed++; break; case TestOutcome::Timeout: summary.timedOut++; break; case TestOutcome::Skipped: summary.skipped++; break; } } summary.results = std::move(results); Progress::Clear(); std::print("\n"); std::vector parts; if (summary.passed) parts.push_back(std::format("{} passed", summary.passed)); if (summary.failed) parts.push_back(std::format("{} failed", summary.failed)); if (summary.crashed) parts.push_back(std::format("{} crashed", summary.crashed)); if (summary.timedOut) parts.push_back(std::format("{} timed out", summary.timedOut)); if (summary.skipped) parts.push_back(std::format("{} skipped", summary.skipped)); std::string joined; for (std::size_t i = 0; i < parts.size(); ++i) { if (i) joined += ", "; joined += parts[i]; } std::println("{}", joined); return summary; }