/* Crafter® Build Copyright (C) 2026 Catcrafts® Catcrafts.net This library is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License version 3.0 as published by the Free Software Foundation; This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. You should have received a copy of the GNU Lesser General Public License along with this library; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA */ export module Crafter.Build:Test_impl; import std; import :Test; import :Clang; import :Platform; namespace fs = std::filesystem; using namespace Crafter; namespace { bool TargetIsWindows(std::string_view target) { return target.find("windows") != std::string_view::npos || target.find("mingw") != std::string_view::npos; } fs::path TestBinaryPath(const Configuration& cfg) { fs::path outputDir = cfg.path/"bin"/std::format("{}-{}-{}", cfg.name, cfg.target, cfg.march); return outputDir / (TargetIsWindows(cfg.target) ? cfg.outputName + ".exe" : cfg.outputName); } bool MatchGlob(std::string_view glob, std::string_view name) { std::size_t gi = 0, ni = 0, star = std::string_view::npos, mark = 0; while (ni < name.size()) { if (gi < glob.size() && (glob[gi] == '?' || glob[gi] == name[ni])) { ++gi; ++ni; } else if (gi < glob.size() && glob[gi] == '*') { star = gi++; mark = ni; } else if (star != std::string_view::npos) { gi = star + 1; ni = ++mark; } else { return false; } } while (gi < glob.size() && glob[gi] == '*') ++gi; return gi == glob.size(); } bool MatchAny(std::span globs, std::string_view name) { if (globs.empty()) return true; for (const auto& g : globs) { if (MatchGlob(g, name)) return true; } return false; } std::string ShellQuote(std::string_view s) { std::string out; out.reserve(s.size() + 2); out.push_back('\''); for (char c : s) { if (c == '\'') out += "'\\''"; else out.push_back(c); } out.push_back('\''); return out; } std::string JoinAndQuoteArgs(std::span args) { std::string out; for (const auto& a : args) { if (!out.empty()) out.push_back(' '); out += ShellQuote(a); } return out; } std::string Substitute(std::string_view tmpl, const std::map& ph) { std::string out; out.reserve(tmpl.size()); std::size_t i = 0; while (i < tmpl.size()) { if (tmpl[i] == '{') { std::size_t end = tmpl.find('}', i + 1); if (end != std::string_view::npos) { std::string key(tmpl.substr(i, end - i + 1)); if (auto it = ph.find(key); it != ph.end()) { out += it->second; i = end + 1; continue; } } } out.push_back(tmpl[i++]); } return out; } std::string SignalName(int sig) { switch (sig) { case 1: return "SIGHUP"; case 2: return "SIGINT"; case 4: return "SIGILL"; case 6: return "SIGABRT"; case 8: return "SIGFPE"; case 9: return "SIGKILL"; case 11: return "SIGSEGV"; case 13: return "SIGPIPE"; case 14: return "SIGALRM"; case 15: return "SIGTERM"; default: return std::format("signal {}", sig); } } void WriteLog(const fs::path& projectPath, const std::string& name, const std::string& output) { fs::path logDir = projectPath / "build" / "test-logs"; std::error_code ec; fs::create_directories(logDir, ec); if (ec) return; std::ofstream(logDir / (name + ".log")) << output; } void PrintResult(const TestResult& r, std::string_view runnerName) { auto ms = r.duration.count(); std::string runnerSuffix = (runnerName.empty() || runnerName == "local") ? std::string() : std::format(" ({})", runnerName); switch (r.outcome) { case TestOutcome::Pass: std::println("✅ {}{} ({}ms)", r.name, runnerSuffix, ms); break; case TestOutcome::Fail: std::println("❌ {}{} ({}ms) exit {}", r.name, runnerSuffix, ms, r.exitCode); if (!r.output.empty()) { for (auto line : std::views::split(r.output, '\n')) { std::string_view sv(line.begin(), line.end()); if (!sv.empty()) std::println(" {}", sv); } } break; case TestOutcome::Crash: std::println("\U0001F4A5 {}{} ({}ms) crashed: {}", r.name, runnerSuffix, ms, SignalName(r.signal)); if (!r.output.empty()) { for (auto line : std::views::split(r.output, '\n')) { std::string_view sv(line.begin(), line.end()); if (!sv.empty()) std::println(" {}", sv); } } break; case TestOutcome::Timeout: std::println("⏱ {}{} ({}ms) timeout", r.name, runnerSuffix, ms); break; case TestOutcome::Skipped: std::println("⏭ {}{} skipped: {}", r.name, runnerSuffix, r.output.empty() ? std::string("(no reason)") : r.output); break; } } } TestRunner TestRunner::Local() { TestRunner r; r.name = "local"; return r; } TestRunner TestRunner::Ssh(std::string host, std::string remoteDir) { TestRunner r; r.name = std::format("ssh:{}", host); r.remoteDir = std::move(remoteDir); r.copy = std::format("ssh -q {0} 'mkdir -p {{remote_bundle}}' && scp -r -q {{bundle}}/. {0}:{{remote_bundle}}/", host); r.exec = std::format("ssh -q {} 'cd {{remote_bundle}} && ./{{bin_name}} {{args}}'", host); r.cleanup = std::format("ssh -q {} 'rm -rf {{remote_bundle}}' || true", host); r.probe = std::format("ssh -q -o BatchMode=yes -o ConnectTimeout=5 {} true > /dev/null 2>&1", host); return r; } TestRunner TestRunner::SshWin(std::string host, std::string remoteDir) { TestRunner r; r.name = std::format("sshwin:{}", host); r.remoteDir = std::move(remoteDir); // cmd.exe-friendly templates. Use backslash variants of remote paths because // cmd's mkdir won't auto-create intermediate directories with forward // slashes. scp tolerates either, so we keep forward slashes for the scp // dest (which `{remote_bundle}` provides). 2>nul + `& rem ok` swallows // mkdir's "already exists" error and forces exit code 0. r.copy = std::format("ssh -q {0} \"mkdir {{remote_bundle_win}} 2>nul & rem ok\" && scp -r -q {{bundle}}/. {0}:{{remote_bundle}}/", host); r.exec = std::format("ssh -q {} \"{{bin_win}} {{args}}\"", host); r.cleanup = std::format("ssh -q {} \"rd /s /q {{remote_bundle_win}} 2>nul & rem ok\"", host); r.probe = std::format("ssh -q -o BatchMode=yes -o ConnectTimeout=5 {} \"ver\" > /dev/null 2>&1", host); return r; } TestRunner TestRunner::QemuUser(std::string qemuBin) { TestRunner r; r.name = std::format("qemu:{}", qemuBin); r.exec = std::format("{} {{bin}} {{args}}", qemuBin); r.probe = std::format("which {} > /dev/null 2>&1", qemuBin); return r; } namespace { std::string NormalizeTriple(std::string_view target) { std::string out(target); for (char& c : out) { if (c == '-' || c == '.') c = '_'; } return out; } std::optional ParseRunnerSpec(std::string_view spec) { if (spec.empty()) return std::nullopt; std::vector parts; for (auto piece : std::views::split(spec, ':')) { parts.emplace_back(std::string_view(piece.begin(), piece.end())); } if (parts.empty()) return std::nullopt; if (parts[0] == "local") return TestRunner::Local(); if (parts[0] == "qemu" && parts.size() == 2) return TestRunner::QemuUser(parts[1]); if (parts[0] == "ssh" && parts.size() == 2) return TestRunner::Ssh(parts[1]); if (parts[0] == "ssh" && parts.size() >= 3) { std::string remote; for (std::size_t i = 2; i < parts.size(); ++i) { if (i > 2) remote.push_back(':'); remote += parts[i]; } return TestRunner::Ssh(parts[1], remote); } if (parts[0] == "sshwin" && parts.size() == 2) return TestRunner::SshWin(parts[1]); if (parts[0] == "sshwin" && parts.size() >= 3) { std::string remote; for (std::size_t i = 2; i < parts.size(); ++i) { if (i > 2) remote.push_back(':'); remote += parts[i]; } return TestRunner::SshWin(parts[1], remote); } throw std::runtime_error(std::format( "TestRunner::FromEnv: unrecognized runner spec '{}'", spec)); } } TestRunner TestRunner::FromEnv(std::string_view target, TestRunner fallback) { std::string envName = std::format("CRAFTER_BUILD_RUNNER_{}", NormalizeTriple(target)); const char* v = std::getenv(envName.c_str()); if (!v || !*v) return fallback; if (auto r = ParseRunnerSpec(v)) return std::move(*r); return fallback; } TestResult Crafter::RunSingleTest(const Test& test, const fs::path& binary, std::chrono::seconds timeout) { using namespace std::chrono_literals; TestResult result; result.name = test.config.name; std::map ph; ph["{args}"] = JoinAndQuoteArgs(test.args); ph["{bin_name}"] = binary.filename().string(); ph["{bundle}"] = binary.parent_path().string(); auto start = std::chrono::steady_clock::now(); CommandResult r; if (test.runner.exec.empty()) { // Pure-local runner: spawn the binary directly. std::string cmd = std::format("{} {}", ShellQuote(binary.string()), ph["{args}"]); r = RunCommandWithTimeout(cmd, timeout); } else if (test.runner.copy.empty()) { // Prefix runner (qemu-user, wsl, ...): templated exec wraps a local binary. ph["{bin}"] = binary.string(); r = RunCommandWithTimeout(Substitute(test.runner.exec, ph), timeout); } else { // Transport runner (ssh, ...): copy bundle → exec remotely → cleanup. std::string unique = std::format("{}-{}-{}", test.config.name, std::hash{}(std::this_thread::get_id()), std::chrono::steady_clock::now().time_since_epoch().count()); ph["{remote_bundle}"] = std::format("{}/{}", test.runner.remoteDir, unique); ph["{bin}"] = std::format("{}/{}", ph["{remote_bundle}"], ph["{bin_name}"]); // Backslash variants for cmd.exe-shell remotes (cmd's mkdir won't // auto-create intermediate directories when path uses forward slashes). ph["{remote_bundle_win}"] = ph["{remote_bundle}"]; std::replace(ph["{remote_bundle_win}"].begin(), ph["{remote_bundle_win}"].end(), '/', '\\'); ph["{bin_win}"] = ph["{bin}"]; std::replace(ph["{bin_win}"].begin(), ph["{bin_win}"].end(), '/', '\\'); CommandResult cp = RunCommandWithTimeout(Substitute(test.runner.copy, ph), 5min); if (cp.exitCode != 0) { auto end = std::chrono::steady_clock::now(); result.duration = std::chrono::duration_cast(end - start); result.outcome = TestOutcome::Fail; result.exitCode = cp.exitCode; result.output = std::format("copy step failed (exit {}):\n{}", cp.exitCode, cp.output); return result; } r = RunCommandWithTimeout(Substitute(test.runner.exec, ph), timeout); if (!test.runner.cleanup.empty()) { std::system(Substitute(test.runner.cleanup, ph).c_str()); } } auto end = std::chrono::steady_clock::now(); result.duration = std::chrono::duration_cast(end - start); result.output = std::move(r.output); result.exitCode = r.exitCode; result.signal = r.signal; if (r.timedOut) result.outcome = TestOutcome::Timeout; else if (r.crashed) result.outcome = TestOutcome::Crash; else if (r.exitCode != 0) result.outcome = TestOutcome::Fail; else result.outcome = TestOutcome::Pass; return result; } TestSummary Crafter::RunTests(Configuration& projectCfg, const RunTestsOptions& opts) { TestSummary summary; std::vector filtered; filtered.reserve(projectCfg.tests.size()); for (auto& test : projectCfg.tests) { if (MatchAny(opts.globs, test.config.name)) { filtered.push_back(&test); } } if (opts.listOnly) { for (auto* t : filtered) { std::println("{}", t->config.name); } return summary; } if (filtered.empty()) { std::println("No tests matched."); return summary; } int jobs = opts.jobs > 0 ? opts.jobs : std::max(1u, std::thread::hardware_concurrency()); jobs = std::min(jobs, static_cast(filtered.size())); std::unordered_map> depResults; std::mutex depMutex; std::mutex printMutex; std::mutex probeMutex; std::unordered_map probeCache; std::atomic next{0}; std::vector results(filtered.size()); auto runnerAvailable = [&](const TestRunner& runner) -> bool { if (runner.probe.empty()) return true; std::lock_guard lk(probeMutex); if (auto it = probeCache.find(runner.name); it != probeCache.end()) { return it->second; } bool ok = (std::system(runner.probe.c_str()) == 0); probeCache[runner.name] = ok; return ok; }; auto worker = [&]() { while (true) { std::size_t i = next.fetch_add(1); if (i >= filtered.size()) break; Test& t = *filtered[i]; TestResult r; r.name = t.config.name; if (!runnerAvailable(t.runner)) { r.outcome = TestOutcome::Skipped; r.output = std::format("runner '{}' not available", t.runner.name); { std::lock_guard lk(printMutex); PrintResult(r, t.runner.name); } results[i] = std::move(r); continue; } BuildResult br; try { br = Build(t.config, depResults, depMutex); } catch (const std::exception& e) { r.outcome = TestOutcome::Fail; r.output = std::format("build threw: {}", e.what()); r.exitCode = -1; { std::lock_guard lk(printMutex); PrintResult(r, t.runner.name); } results[i] = std::move(r); continue; } if (!br.result.empty()) { r.outcome = TestOutcome::Fail; r.output = std::format("build failed: {}", br.result); r.exitCode = -1; { std::lock_guard lk(printMutex); PrintResult(r, t.runner.name); } results[i] = std::move(r); continue; } std::chrono::seconds timeout = opts.timeoutOverride.value_or(t.timeout); fs::path binary = TestBinaryPath(t.config); try { r = RunSingleTest(t, binary, timeout); } catch (const std::exception& e) { r.outcome = TestOutcome::Fail; r.output = std::format("runner threw: {}", e.what()); r.exitCode = -1; } if (r.outcome != TestOutcome::Pass && r.outcome != TestOutcome::Skipped && !r.output.empty()) { WriteLog(projectCfg.path, r.name, r.output); } { std::lock_guard lk(printMutex); PrintResult(r, t.runner.name); } results[i] = std::move(r); } }; std::vector threads; threads.reserve(jobs); for (int j = 0; j < jobs; ++j) { threads.emplace_back(worker); } threads.clear(); // joins all jthreads for (auto& r : results) { switch (r.outcome) { case TestOutcome::Pass: summary.passed++; break; case TestOutcome::Fail: summary.failed++; break; case TestOutcome::Crash: summary.crashed++; break; case TestOutcome::Timeout: summary.timedOut++; break; case TestOutcome::Skipped: summary.skipped++; break; } } summary.results = std::move(results); std::print("\n"); std::vector parts; if (summary.passed) parts.push_back(std::format("{} passed", summary.passed)); if (summary.failed) parts.push_back(std::format("{} failed", summary.failed)); if (summary.crashed) parts.push_back(std::format("{} crashed", summary.crashed)); if (summary.timedOut) parts.push_back(std::format("{} timed out", summary.timedOut)); if (summary.skipped) parts.push_back(std::format("{} skipped", summary.skipped)); std::string joined; for (std::size_t i = 0; i < parts.size(); ++i) { if (i) joined += ", "; joined += parts[i]; } std::println("{}", joined); return summary; }