/* Crafter® Build Copyright (C) 2026 Catcrafts® Catcrafts.net This library is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License version 3.0 as published by the Free Software Foundation; This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. You should have received a copy of the GNU Lesser General Public License along with this library; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA */ module; export module Crafter.Build:Test_impl; import std; import :Test; import :Clang; import :Platform; import :Progress; namespace fs = std::filesystem; using namespace Crafter; namespace { bool TargetIsWindows(std::string_view target) { return target.find("windows") != std::string_view::npos || target.find("mingw") != std::string_view::npos; } fs::path TestBinaryPath(const Configuration& cfg) { fs::path outputDir = cfg.BinDir(); return outputDir / (TargetIsWindows(cfg.target) ? cfg.outputName + ".exe" : cfg.outputName); } bool MatchGlob(std::string_view glob, std::string_view name) { std::size_t gi = 0, ni = 0, star = std::string_view::npos, mark = 0; while (ni < name.size()) { if (gi < glob.size() && (glob[gi] == '?' || glob[gi] == name[ni])) { ++gi; ++ni; } else if (gi < glob.size() && glob[gi] == '*') { star = gi++; mark = ni; } else if (star != std::string_view::npos) { gi = star + 1; ni = ++mark; } else { return false; } } while (gi < glob.size() && glob[gi] == '*') ++gi; return gi == glob.size(); } bool MatchAny(std::span globs, std::string_view name) { if (globs.empty()) return true; for (const auto& g : globs) { if (MatchGlob(g, name)) return true; } return false; } std::string ShellQuoteSh(std::string_view s) { std::string out; out.reserve(s.size() + 2); out.push_back('\''); for (char c : s) { if (c == '\'') out += "'\\''"; else out.push_back(c); } out.push_back('\''); return out; } // cmd.exe doesn't recognize '...' as quoting (it would pass the single // quotes through to the executable). Wrap in "..." for cmd; embedded " // is rare in paths but escape it to be safe. Backslash sequences before // " don't need the MS-CRT doubling rules because we go through // `cmd /C "..."`, which uses cmd's parser, not the CRT's argv splitter. std::string ShellQuoteCmd(std::string_view s) { std::string out; out.reserve(s.size() + 2); out.push_back('"'); for (char c : s) { if (c == '"') out += "\\\""; else out.push_back(c); } out.push_back('"'); return out; } // Host-shell quoting: sh on Linux, cmd on Windows. For args/paths that // hit the local shell (Local runner exec, Cmd-prefix runner exec). std::string ShellQuoteHost(std::string_view s) { #ifdef _WIN32 return ShellQuoteCmd(s); #else return ShellQuoteSh(s); #endif } std::string JoinAndQuoteArgs(std::span args) { std::string out; for (const auto& a : args) { if (!out.empty()) out.push_back(' '); out += ShellQuoteHost(a); } return out; } std::string Substitute(std::string_view tmpl, const std::map& ph) { std::string out; out.reserve(tmpl.size()); std::size_t i = 0; while (i < tmpl.size()) { if (tmpl[i] == '{') { std::size_t end = tmpl.find('}', i + 1); if (end != std::string_view::npos) { std::string key(tmpl.substr(i, end - i + 1)); if (auto it = ph.find(key); it != ph.end()) { out += it->second; i = end + 1; continue; } } } out.push_back(tmpl[i++]); } return out; } std::string SignalName(int sig) { switch (sig) { case 1: return "SIGHUP"; case 2: return "SIGINT"; case 4: return "SIGILL"; case 6: return "SIGABRT"; case 8: return "SIGFPE"; case 9: return "SIGKILL"; case 11: return "SIGSEGV"; case 13: return "SIGPIPE"; case 14: return "SIGALRM"; case 15: return "SIGTERM"; default: return std::format("signal {}", sig); } } void WriteLog(const fs::path& projectPath, const std::string& name, const std::string& output) { fs::path logDir = projectPath / "build" / "test-logs"; std::error_code ec; fs::create_directories(logDir, ec); if (ec) return; std::ofstream(logDir / (name + ".log")) << output; } void PrintResult(const TestResult& r, std::string_view runnerName) { Progress::Clear(); auto ms = r.duration.count(); std::string runnerSuffix = (runnerName.empty() || runnerName == "local") ? std::string() : std::format(" ({})", runnerName); switch (r.outcome) { case TestOutcome::Pass: std::println("✅ {}{} ({}ms)", r.name, runnerSuffix, ms); break; case TestOutcome::Fail: std::println("❌ {}{} ({}ms) exit {}", r.name, runnerSuffix, ms, r.exitCode); if (!r.output.empty()) { for (auto line : std::views::split(r.output, '\n')) { std::string_view sv(line.begin(), line.end()); if (!sv.empty()) std::println(" {}", sv); } } break; case TestOutcome::Crash: std::println("\U0001F4A5 {}{} ({}ms) crashed: {}", r.name, runnerSuffix, ms, SignalName(r.signal)); if (!r.output.empty()) { for (auto line : std::views::split(r.output, '\n')) { std::string_view sv(line.begin(), line.end()); if (!sv.empty()) std::println(" {}", sv); } } break; case TestOutcome::Timeout: std::println("⏱ {}{} ({}ms) timeout", r.name, runnerSuffix, ms); break; case TestOutcome::Skipped: std::println("⏭ {}{} skipped: {}", r.name, runnerSuffix, r.output.empty() ? std::string("(no reason)") : r.output); break; } } } namespace { std::atomic g_parentProject{nullptr}; Configuration* FindLibInTree(Configuration* root, std::string_view name, std::unordered_set& seen) { if (!seen.insert(root).second) return nullptr; if (root->name == name) return root; for (Configuration* dep : root->dependencies) { if (auto found = FindLibInTree(dep, name, seen)) return found; } return nullptr; } } void Crafter::SetParentProject(Configuration* parent) { g_parentProject.store(parent); } Configuration* Crafter::ParentLib(std::string_view name) { Configuration* root = g_parentProject.load(); if (!root) { throw std::runtime_error(std::format( "Crafter::ParentLib('{}'): no parent project set", name)); } std::unordered_set seen; if (auto found = FindLibInTree(root, name, seen)) return found; throw std::runtime_error(std::format( "Crafter::ParentLib('{}'): not found in parent project '{}'", name, root->name)); } TestRunner TestRunner::Local() { TestRunner r; r.name = "local"; return r; } TestRunner TestRunner::Cmd(std::string command) { TestRunner r; r.name = std::format("cmd:{}", command); r.exec = std::format("{} {{bin}} {{args}}", command); #ifdef _WIN32 r.probe = std::format("where {}", command); #else r.probe = std::format("which {}", command); #endif return r; } TestRunner TestRunner::Wine() { TestRunner r; r.name = "wine"; r.exec = "wine {bin} {args}"; #ifdef _WIN32 r.probe = "where wine"; #else r.probe = "which wine"; #endif return r; } TestRunner TestRunner::ForTarget(const Configuration& cfg) { const std::string& target = cfg.target; // Same triple as the host → run the binary directly. Covers the common // case (cfg.target defaulted to HostTarget()) without any wrapper. if (target == HostTarget()) return Local(); // Windows targets: native on a Windows host, Wine on Linux. We don't // distinguish mingw vs msvc here — the produced .exe runs the same way. if (TargetIsWindows(target)) { return TargetIsWindows(HostTarget()) ? Local() : Wine(); } // WASI: a .wasm file isn't directly executable; wasmtime is the canonical // runtime. wasi-cli also works but the upstream Bytecode Alliance name is // wasmtime, so we standardize on that. if (target.starts_with("wasm32-wasi") || target.starts_with("wasm64-wasi")) { return Cmd("wasmtime"); } // Non-host Linux triple: extract the architecture and route through // qemu-user. Triple is --- (or sometimes 3 parts); // qemu-user's binary names mostly follow the arch field, with two known // mismatches handled below. cfg.sysroot, when set, becomes QEMU_LD_PREFIX // so the target's dynamic linker / shared libs are reachable — without // it qemu-user crashes on dynamic ELFs with "could not open /lib/ld...". if (target.find("-linux-") != std::string::npos) { auto dash = target.find('-'); std::string arch = target.substr(0, dash); // i686-linux-gnu → qemu-i386; arm-* already matches qemu-arm; aarch64, // riscv64, ppc64le, mips, mips64, s390x all match their qemu names. if (arch == "i686") arch = "i386"; TestRunner r = Cmd(std::format("qemu-{}", arch)); if (!cfg.sysroot.empty()) { // Use `env VAR=value cmd` rather than the shell's `VAR=value cmd` // prefix syntax: RunCommandWithTimeout pipes through GNU `timeout`, // which execvp's its argument list directly without going through a // shell. A bare VAR=value would be exec'd as a command path and // fail with "No such file or directory". r.exec = std::format("env QEMU_LD_PREFIX={} {}", cfg.sysroot, r.exec); } return r; } // Unknown / bare-metal / freestanding targets: fall back to Local. The // caller's runner-availability probe (or absence of the binary) surfaces // the problem rather than us inventing a wrong wrapper here. return Local(); } namespace { std::string NormalizeTriple(std::string_view target) { std::string out(target); for (char& c : out) { if (c == '-' || c == '.') c = '_'; } return out; } // Spec grammar: "local" | "cmd:". Used by CRAFTER_BUILD_RUNNER_* // env vars and --runner=. The set used to include ssh/sshwin/wsl variants; // those were removed when transport-style runners were dropped (issue #8). std::optional ParseRunnerSpec(std::string_view spec) { if (spec.empty()) return std::nullopt; if (spec == "local") return TestRunner::Local(); if (spec.starts_with("cmd:") && spec.size() > 4) { return TestRunner::Cmd(std::string(spec.substr(4))); } throw std::runtime_error(std::format( "TestRunner::FromSpec: unrecognized runner spec '{}' " "(expected 'local' or 'cmd:')", spec)); } } std::optional TestRunner::FromSpec(std::string_view spec) { return ParseRunnerSpec(spec); } TestRunner TestRunner::FromEnv(std::string_view target, TestRunner fallback) { std::string envName = std::format("CRAFTER_BUILD_RUNNER_{}", NormalizeTriple(target)); const char* v = std::getenv(envName.c_str()); if (!v || !*v) return fallback; if (auto r = ParseRunnerSpec(v)) return std::move(*r); return fallback; } TestResult Crafter::RunSingleTest(const Test& test, const fs::path& binary, std::chrono::seconds timeout) { TestResult result; result.name = test.config.name; std::map ph; ph["{args}"] = JoinAndQuoteArgs(test.args); auto start = std::chrono::steady_clock::now(); CommandResult r; if (test.runner.exec.empty()) { // Local runner: spawn the binary directly through the host shell. std::string cmd = std::format("{} {}", ShellQuoteHost(binary.string()), ph["{args}"]); r = RunCommandWithTimeout(cmd, timeout); } else { // Prefix runner (qemu-user, wasmtime, wine, ...): templated exec // wraps the local binary. ph["{bin}"] = binary.string(); r = RunCommandWithTimeout(Substitute(test.runner.exec, ph), timeout); } auto end = std::chrono::steady_clock::now(); result.duration = std::chrono::duration_cast(end - start); result.output = std::move(r.output); result.exitCode = r.exitCode; result.signal = r.signal; if (r.timedOut) result.outcome = TestOutcome::Timeout; else if (r.crashed) result.outcome = TestOutcome::Crash; else if (r.exitCode == 77) result.outcome = TestOutcome::Skipped; else if (r.exitCode != 0) result.outcome = TestOutcome::Fail; else result.outcome = TestOutcome::Pass; return result; } namespace { bool ToolOnPath(std::string_view name) { #ifdef _WIN32 std::string cmd = std::format("where {} > nul 2>&1", name); #else std::string cmd = std::format("which {} > /dev/null 2>&1", name); #endif return std::system(cmd.c_str()) == 0; } struct RequireResult { bool ok; std::string reason; // human-readable when !ok }; // Evaluate each `:` precondition. Returns the first failure // (short-circuit; reporting one missing dep at a time is enough to act on // and keeps the test log uncluttered). RequireResult EvaluateRequires(std::span reqs) { for (const auto& r : reqs) { auto sep = r.find(':'); if (sep == std::string::npos || sep == 0 || sep == r.size() - 1) { return {false, std::format("malformed require '{}' (expected kind:arg)", r)}; } std::string_view kind(r.data(), sep); std::string_view arg(r.data() + sep + 1, r.size() - sep - 1); if (kind == "tool") { if (!ToolOnPath(arg)) { return {false, std::format("tool '{}' not on PATH", arg)}; } } else if (kind == "file") { if (!fs::exists(std::string(arg))) { return {false, std::format("file '{}' missing", arg)}; } } else if (kind == "env") { const char* v = std::getenv(std::string(arg).c_str()); if (!v || !*v) { return {false, std::format("env '{}' unset", arg)}; } } else { return {false, std::format( "unknown require kind '{}' (expected tool/file/env)", kind)}; } } return {true, ""}; } // Match a runner's tool dependency against the test's declared // requirements. Used to decide between Skip (declared, may legitimately // be missing) and Fail (runner unavailable but test didn't declare it — // a silent skip would mask broken cross-arch CI configuration). bool RequiresMentionsTool(std::span reqs, std::string_view tool) { std::string needle = std::format("tool:{}", tool); return std::ranges::any_of(reqs, [&](const std::string& s) { return s == needle; }); } // Best-effort extraction of the runner-tool name from a TestRunner so the // hard-fail-unless-declared check can match it against `requires`. For // Cmd("foo"), the name is "cmd:foo"; for Wine, it's "wine". Anything else // (Local, transport runners) returns empty — those don't trigger the // declared/undeclared gate. std::string RunnerToolName(const TestRunner& runner) { if (runner.name == "wine") return "wine"; if (runner.name.starts_with("cmd:")) { std::string tool = runner.name.substr(4); // QEMU_LD_PREFIX prefix may be glued onto exec but the runner's // `name` field already isolates the command, so no extra parsing. return tool; } return ""; } } TestBuilder Configuration::AddTest(std::string_view name) { // Empty-interfaces trampoline. The span we hand to the (name, interfaces) // overload wraps a stack array; it's only consulted inside that call, so // its lifetime is fine for the duration. std::array noIfaces = {}; return AddTest(name, noIfaces); } TestBuilder Configuration::AddTest(std::string_view name, std::span interfaces) { Test t; // Default to "./" so tests//main.cpp resolves at the project root // (where crafter-build sets cwd from project.cpp's location). Projects // whose lib lives in a subdir (e.g. cfg.path = "./mylib/") get this // right by default; if the test layout is unusual, override via .Path(). t.config.path = "./"; t.config.name = std::string(name); t.config.outputName = std::string(name); t.config.target = this->target; t.config.march = this->march; t.config.mtune = this->mtune; t.config.debug = this->debug; t.config.type = ConfigurationType::Executable; // Default source layout: tests//main.cpp resolved against the // parent Configuration's path. cfg.path is typically "./" (project // root), which puts the test sources at /tests//main.cpp. fs::path mainSource = fs::path("tests") / std::string(name) / "main"; std::array impls = { mainSource }; t.config.GetInterfacesAndImplementations(interfaces, impls); tests.push_back(std::move(t)); return TestBuilder{this, tests.size() - 1}; } void Configuration::AddMarchVariants(std::string_view name, std::span interfaces, std::span tiers) { for (const auto& tier : tiers) { Test t; t.config.path = "./"; t.config.name = std::format("{}-{}", name, tier.march); t.config.outputName = t.config.name; t.config.target = this->target; t.config.march = tier.march; t.config.mtune = tier.mtune; t.config.debug = this->debug; t.config.type = ConfigurationType::Executable; fs::path mainSource = fs::path("tests") / std::string(name) / "main"; std::array impls = { mainSource }; // Interfaces stamped directly into each variant's Configuration — // NOT via cfg.dependencies. The dependency machinery caches PCMs by // dep->VariantId(), which would compile the parent's interfaces // once with the parent's march; for SIMD-sensitive code each // variant needs its own compile. Listing the interfaces as the // test's own forces per-variant recompilation. t.config.GetInterfacesAndImplementations(interfaces, impls); tests.push_back(std::move(t)); } } TestBuilder& TestBuilder::Path(fs::path p) { test().config.path = std::move(p); return *this; } TestBuilder& TestBuilder::Target(std::string t) { test().config.target = std::move(t); return *this; } TestBuilder& TestBuilder::March(std::string m) { test().config.march = std::move(m); return *this; } TestBuilder& TestBuilder::Mtune(std::string m) { test().config.mtune = std::move(m); return *this; } TestBuilder& TestBuilder::Sysroot(fs::path s) { test().config.sysroot = s.string(); return *this; } TestBuilder& TestBuilder::Debug(bool d) { test().config.debug = d; return *this; } TestBuilder& TestBuilder::Define(std::string n, std::string v) { test().config.defines.push_back({std::move(n), std::move(v)}); return *this; } TestBuilder& TestBuilder::Timeout(std::chrono::seconds s) { test().timeout = s; return *this; } TestBuilder& TestBuilder::Args(std::vector a) { test().args = std::move(a); return *this; } TestBuilder& TestBuilder::Requires(std::string r) { test().requires_.push_back(std::move(r)); return *this; } TestBuilder& TestBuilder::Dependencies(std::vector d) { test().config.dependencies = std::move(d); return *this; } TestBuilder& TestBuilder::LinkFlag(std::string f) { test().config.linkFlags.push_back(std::move(f)); return *this; } TestBuilder& TestBuilder::CompileFlag(std::string f) { test().config.compileFlags.push_back(std::move(f)); return *this; } TestSummary Crafter::RunTests(Configuration& projectCfg, const RunTestsOptions& opts, std::span projectArgs) { // Multi-target sweep: when no --target= was given, the run covers every // distinct target declared across projectCfg.tests plus the host target. // Lets a bare `crafter-build test` exercise cross-arch tests without the // user having to know which targets exist. An explicit --target=X // bypasses the sweep and runs that target only. if (opts.targetFilter.empty()) { std::set sweep; sweep.insert(HostTarget()); for (const Test& t : projectCfg.tests) { if (!t.config.target.empty()) sweep.insert(t.config.target); } TestSummary aggregate; for (const auto& target : sweep) { RunTestsOptions perTarget = opts; perTarget.targetFilter = target; if (sweep.size() > 1) { Progress::Clear(); std::println("\n=== target: {} ===", target); } TestSummary s = RunTests(projectCfg, perTarget, projectArgs); aggregate.passed += s.passed; aggregate.failed += s.failed; aggregate.crashed += s.crashed; aggregate.timedOut += s.timedOut; aggregate.skipped += s.skipped; for (auto& r : s.results) aggregate.results.push_back(std::move(r)); } return aggregate; } TestSummary summary; // Filter by target + glob, derive each Test's runner just-in-time. The // runner is recomputed each sweep iteration (rather than once at AddTest // time) because runnerOverride and FromEnv resolution depend on the // run's options, which the project.cpp doesn't know about. std::vector filtered; filtered.reserve(projectCfg.tests.size()); for (auto& test : projectCfg.tests) { if (test.config.target != opts.targetFilter) continue; if (!MatchAny(opts.globs, test.config.name)) continue; test.runner = TestRunner::FromEnv(test.config.target, TestRunner::ForTarget(test.config)); if (opts.runnerOverride) { if (auto r = TestRunner::FromSpec(*opts.runnerOverride)) { test.runner = std::move(*r); } } filtered.push_back(&test); } if (opts.listOnly) { for (auto* t : filtered) { std::println("{}", t->config.name); } return summary; } if (filtered.empty()) { std::println("No tests matched."); return summary; } int jobs = opts.jobs > 0 ? opts.jobs : std::max(1u, std::thread::hardware_concurrency()); jobs = std::min(jobs, static_cast(filtered.size())); std::unordered_map> depResults; std::mutex depMutex; std::mutex printMutex; std::mutex probeMutex; std::unordered_map probeCache; std::atomic next{0}; std::vector results(filtered.size()); auto runnerAvailable = [&](const TestRunner& runner) -> bool { if (runner.probe.empty()) return true; std::lock_guard lk(probeMutex); if (auto it = probeCache.find(runner.name); it != probeCache.end()) { return it->second; } // RunCommandChecked captures and discards output internally, so probe // specs don't need `> /dev/null 2>&1` (which doesn't translate to cmd). bool ok = (RunCommandChecked(runner.probe).exitCode == 0); probeCache[runner.name] = ok; return ok; }; auto worker = [&]() { while (true) { std::size_t i = next.fetch_add(1); if (i >= filtered.size()) break; Test& t = *filtered[i]; TestResult r; r.name = t.config.name; // Declarative preconditions set via TestBuilder::Requires. Evaluated // before the build so a missing tool/file/env turns into a Skip without // paying the compile cost. Reports the first failure only — once one // precondition is unmet the test couldn't run anyway, and a wall of // "also missing X, also missing Y" buries the actionable root cause. if (auto req = EvaluateRequires(t.requires_); !req.ok) { r.outcome = TestOutcome::Skipped; r.output = req.reason; { std::lock_guard lk(printMutex); PrintResult(r, t.runner.name); } results[i] = std::move(r); continue; } if (!runnerAvailable(t.runner)) { // Hard-fail-unless-declared: if the runner depends on a tool // (qemu-aarch64, wasmtime, wine, ...) and the test didn't say // "tool:" in requires, the missing runner is a Fail. The // intent is to surface broken cross-arch CI configuration // instead of letting it masquerade as a Skip; tests that // legitimately may run without their runner have to opt in. std::string tool = RunnerToolName(t.runner); if (!tool.empty() && !RequiresMentionsTool(t.requires_, tool)) { r.outcome = TestOutcome::Fail; r.exitCode = -1; r.output = std::format( "runner '{}' unavailable and not declared in requires " "(add .Requires(\"tool:{}\") to permit skipping)", t.runner.name, tool); } else { r.outcome = TestOutcome::Skipped; r.output = std::format("runner '{}' not available", t.runner.name); } { std::lock_guard lk(printMutex); PrintResult(r, t.runner.name); } results[i] = std::move(r); continue; } BuildResult br; try { br = Build(t.config, depResults, depMutex); } catch (const std::exception& e) { r.outcome = TestOutcome::Fail; r.output = std::format("build threw: {}", e.what()); r.exitCode = -1; { std::lock_guard lk(printMutex); PrintResult(r, t.runner.name); } results[i] = std::move(r); continue; } if (!br.result.empty()) { r.outcome = TestOutcome::Fail; r.output = std::format("build failed: {}", br.result); r.exitCode = -1; { std::lock_guard lk(printMutex); PrintResult(r, t.runner.name); } results[i] = std::move(r); continue; } std::chrono::seconds timeout = opts.timeoutOverride.value_or(t.timeout); fs::path binary = TestBinaryPath(t.config); try { r = RunSingleTest(t, binary, timeout); } catch (const std::exception& e) { r.outcome = TestOutcome::Fail; r.output = std::format("runner threw: {}", e.what()); r.exitCode = -1; } if (r.outcome != TestOutcome::Pass && r.outcome != TestOutcome::Skipped && !r.output.empty()) { WriteLog(projectCfg.path, r.name, r.output); } { std::lock_guard lk(printMutex); PrintResult(r, t.runner.name); } results[i] = std::move(r); } }; std::vector threads; threads.reserve(jobs); for (int j = 0; j < jobs; ++j) { threads.emplace_back(worker); } threads.clear(); // joins all jthreads for (auto& r : results) { switch (r.outcome) { case TestOutcome::Pass: summary.passed++; break; case TestOutcome::Fail: summary.failed++; break; case TestOutcome::Crash: summary.crashed++; break; case TestOutcome::Timeout: summary.timedOut++; break; case TestOutcome::Skipped: summary.skipped++; break; } } summary.results = std::move(results); Progress::Clear(); std::print("\n"); std::vector parts; if (summary.passed) parts.push_back(std::format("{} passed", summary.passed)); if (summary.failed) parts.push_back(std::format("{} failed", summary.failed)); if (summary.crashed) parts.push_back(std::format("{} crashed", summary.crashed)); if (summary.timedOut) parts.push_back(std::format("{} timed out", summary.timedOut)); if (summary.skipped) parts.push_back(std::format("{} skipped", summary.skipped)); std::string joined; for (std::size_t i = 0; i < parts.size(); ++i) { if (i) joined += ", "; joined += parts[i]; } std::println("{}", joined); return summary; }