Jorijn van der Graaf
dc27c5c204
Vendors toml++ v3.4.0 as lib/toml.hpp and wires it into Crafter.Build-Test to parse a declarative test.toml manifest (target/march/mtune/sysroot/ requires/timeout/args/defines). Test discovery now treats project.cpp and test.toml as mutually exclusive: project.cpp stays the escape hatch for outer-driver tests, test.toml gives downstream test authors a no-boilerplate path. Adds: - TestRunner::Wine() and TestRunner::ForTarget(cfg) — runner is now derived from cfg.target (Local for host, Wine for Windows-on-Linux, wasmtime for WASI, qemu-<arch> with QEMU_LD_PREFIX for non-host Linux). The env-var override CRAFTER_BUILD_RUNNER_<target> still wins as a power-user escape hatch via FromEnv. - Declarative preconditions: tool:<name>, file:<path>, env:<VAR> are evaluated before the build; missing preconditions Skip without paying the compile cost. - Hard-fail-unless-declared: when a derived runner's tool is missing AND the test didn't declare 'tool:<that>' in requires, the missing runner is a Fail instead of a silent Skip. Surfaces broken cross-arch CI config that previously hid as "skipped". - Multi-target sweep: bare `crafter-build test` (no --target=) now iterates every distinct test.toml-declared target plus the host, so cross-arch tests run by default without the user needing to know which targets exist. `--target=X` bypasses the sweep. Test struct gains a `requires_` vector so project.cpp users can declare preconditions too (matching what test.toml writes there). Existing tests, factories (Ssh/SshWin/Wsl/Cmd), and CRAFTER_BUILD_RUNNER_* machinery remain intact — this commit only adds; migration and deletion follow in subsequent commits. Refs issue #8. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
1086 lines
45 KiB
C++
1086 lines
45 KiB
C++
/*
|
|
Crafter® Build
|
|
Copyright (C) 2026 Catcrafts®
|
|
Catcrafts.net
|
|
|
|
This library is free software; you can redistribute it and/or
|
|
modify it under the terms of the GNU Lesser General Public
|
|
License version 3.0 as published by the Free Software Foundation;
|
|
|
|
This library is distributed in the hope that it will be useful,
|
|
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
Lesser General Public License for more details.
|
|
|
|
You should have received a copy of the GNU Lesser General Public
|
|
License along with this library; if not, write to the Free Software
|
|
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
|
*/
|
|
|
|
module;
|
|
// toml++ is consumed as a translation-unit-private dependency in the GMF: the
|
|
// parser is only needed for test.toml discovery here, so keeping it out of
|
|
// `import std`-using module purviews avoids dragging it through the module
|
|
// graph (and through every PCM consumer of Crafter.Build).
|
|
#include "../lib/toml.hpp"
|
|
export module Crafter.Build:Test_impl;
|
|
import std;
|
|
import :Test;
|
|
import :Clang;
|
|
import :Platform;
|
|
import :Progress;
|
|
namespace fs = std::filesystem;
|
|
using namespace Crafter;
|
|
|
|
namespace {
|
|
bool TargetIsWindows(std::string_view target) {
|
|
return target.find("windows") != std::string_view::npos
|
|
|| target.find("mingw") != std::string_view::npos;
|
|
}
|
|
|
|
fs::path TestBinaryPath(const Configuration& cfg) {
|
|
fs::path outputDir = cfg.BinDir();
|
|
return outputDir / (TargetIsWindows(cfg.target) ? cfg.outputName + ".exe" : cfg.outputName);
|
|
}
|
|
|
|
bool MatchGlob(std::string_view glob, std::string_view name) {
|
|
std::size_t gi = 0, ni = 0, star = std::string_view::npos, mark = 0;
|
|
while (ni < name.size()) {
|
|
if (gi < glob.size() && (glob[gi] == '?' || glob[gi] == name[ni])) {
|
|
++gi; ++ni;
|
|
} else if (gi < glob.size() && glob[gi] == '*') {
|
|
star = gi++;
|
|
mark = ni;
|
|
} else if (star != std::string_view::npos) {
|
|
gi = star + 1;
|
|
ni = ++mark;
|
|
} else {
|
|
return false;
|
|
}
|
|
}
|
|
while (gi < glob.size() && glob[gi] == '*') ++gi;
|
|
return gi == glob.size();
|
|
}
|
|
|
|
bool MatchAny(std::span<const std::string> globs, std::string_view name) {
|
|
if (globs.empty()) return true;
|
|
for (const auto& g : globs) {
|
|
if (MatchGlob(g, name)) return true;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
std::string ShellQuoteSh(std::string_view s) {
|
|
std::string out;
|
|
out.reserve(s.size() + 2);
|
|
out.push_back('\'');
|
|
for (char c : s) {
|
|
if (c == '\'') out += "'\\''";
|
|
else out.push_back(c);
|
|
}
|
|
out.push_back('\'');
|
|
return out;
|
|
}
|
|
|
|
// cmd.exe doesn't recognize '...' as quoting (it would pass the single
|
|
// quotes through to the executable). Wrap in "..." for cmd; embedded "
|
|
// is rare in paths but escape it to be safe. Backslash sequences before
|
|
// " don't need the MS-CRT doubling rules because we go through
|
|
// `cmd /C "..."`, which uses cmd's parser, not the CRT's argv splitter.
|
|
std::string ShellQuoteCmd(std::string_view s) {
|
|
std::string out;
|
|
out.reserve(s.size() + 2);
|
|
out.push_back('"');
|
|
for (char c : s) {
|
|
if (c == '"') out += "\\\"";
|
|
else out.push_back(c);
|
|
}
|
|
out.push_back('"');
|
|
return out;
|
|
}
|
|
|
|
// Host-shell quoting: sh on Linux, cmd on Windows. For args/paths that
|
|
// hit the local shell (Local runner exec, Cmd-prefix runner exec).
|
|
std::string ShellQuoteHost(std::string_view s) {
|
|
#ifdef _WIN32
|
|
return ShellQuoteCmd(s);
|
|
#else
|
|
return ShellQuoteSh(s);
|
|
#endif
|
|
}
|
|
|
|
std::string JoinAndQuoteArgs(std::span<const std::string> args, TestRunner::Shell shell) {
|
|
std::string out;
|
|
for (const auto& a : args) {
|
|
if (!out.empty()) out.push_back(' ');
|
|
switch (shell) {
|
|
case TestRunner::Shell::Sh: out += ShellQuoteSh(a); break;
|
|
case TestRunner::Shell::Cmd: out += ShellQuoteCmd(a); break;
|
|
case TestRunner::Shell::Host: out += ShellQuoteHost(a); break;
|
|
}
|
|
}
|
|
return out;
|
|
}
|
|
|
|
std::string Substitute(std::string_view tmpl, const std::map<std::string, std::string>& ph) {
|
|
std::string out;
|
|
out.reserve(tmpl.size());
|
|
std::size_t i = 0;
|
|
while (i < tmpl.size()) {
|
|
if (tmpl[i] == '{') {
|
|
std::size_t end = tmpl.find('}', i + 1);
|
|
if (end != std::string_view::npos) {
|
|
std::string key(tmpl.substr(i, end - i + 1));
|
|
if (auto it = ph.find(key); it != ph.end()) {
|
|
out += it->second;
|
|
i = end + 1;
|
|
continue;
|
|
}
|
|
}
|
|
}
|
|
out.push_back(tmpl[i++]);
|
|
}
|
|
return out;
|
|
}
|
|
|
|
std::string SignalName(int sig) {
|
|
switch (sig) {
|
|
case 1: return "SIGHUP";
|
|
case 2: return "SIGINT";
|
|
case 4: return "SIGILL";
|
|
case 6: return "SIGABRT";
|
|
case 8: return "SIGFPE";
|
|
case 9: return "SIGKILL";
|
|
case 11: return "SIGSEGV";
|
|
case 13: return "SIGPIPE";
|
|
case 14: return "SIGALRM";
|
|
case 15: return "SIGTERM";
|
|
default: return std::format("signal {}", sig);
|
|
}
|
|
}
|
|
|
|
void WriteLog(const fs::path& projectPath, const std::string& name, const std::string& output) {
|
|
fs::path logDir = projectPath / "build" / "test-logs";
|
|
std::error_code ec;
|
|
fs::create_directories(logDir, ec);
|
|
if (ec) return;
|
|
std::ofstream(logDir / (name + ".log")) << output;
|
|
}
|
|
|
|
void PrintResult(const TestResult& r, std::string_view runnerName) {
|
|
Progress::Clear();
|
|
auto ms = r.duration.count();
|
|
std::string runnerSuffix = (runnerName.empty() || runnerName == "local")
|
|
? std::string()
|
|
: std::format(" ({})", runnerName);
|
|
switch (r.outcome) {
|
|
case TestOutcome::Pass:
|
|
std::println("✅ {}{} ({}ms)", r.name, runnerSuffix, ms);
|
|
break;
|
|
case TestOutcome::Fail:
|
|
std::println("❌ {}{} ({}ms) exit {}", r.name, runnerSuffix, ms, r.exitCode);
|
|
if (!r.output.empty()) {
|
|
for (auto line : std::views::split(r.output, '\n')) {
|
|
std::string_view sv(line.begin(), line.end());
|
|
if (!sv.empty()) std::println(" {}", sv);
|
|
}
|
|
}
|
|
break;
|
|
case TestOutcome::Crash:
|
|
std::println("\U0001F4A5 {}{} ({}ms) crashed: {}", r.name, runnerSuffix, ms, SignalName(r.signal));
|
|
if (!r.output.empty()) {
|
|
for (auto line : std::views::split(r.output, '\n')) {
|
|
std::string_view sv(line.begin(), line.end());
|
|
if (!sv.empty()) std::println(" {}", sv);
|
|
}
|
|
}
|
|
break;
|
|
case TestOutcome::Timeout:
|
|
std::println("⏱ {}{} ({}ms) timeout", r.name, runnerSuffix, ms);
|
|
break;
|
|
case TestOutcome::Skipped:
|
|
std::println("⏭ {}{} skipped: {}", r.name, runnerSuffix,
|
|
r.output.empty() ? std::string("(no reason)") : r.output);
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
namespace {
|
|
std::atomic<Configuration*> g_parentProject{nullptr};
|
|
|
|
Configuration* FindLibInTree(Configuration* root,
|
|
std::string_view name,
|
|
std::unordered_set<Configuration*>& seen) {
|
|
if (!seen.insert(root).second) return nullptr;
|
|
if (root->name == name) return root;
|
|
for (Configuration* dep : root->dependencies) {
|
|
if (auto found = FindLibInTree(dep, name, seen)) return found;
|
|
}
|
|
return nullptr;
|
|
}
|
|
}
|
|
|
|
void Crafter::SetParentProject(Configuration* parent) {
|
|
g_parentProject.store(parent);
|
|
}
|
|
|
|
Configuration* Crafter::ParentLib(std::string_view name) {
|
|
Configuration* root = g_parentProject.load();
|
|
if (!root) {
|
|
throw std::runtime_error(std::format(
|
|
"Crafter::ParentLib('{}'): no parent project set", name));
|
|
}
|
|
std::unordered_set<Configuration*> seen;
|
|
if (auto found = FindLibInTree(root, name, seen)) return found;
|
|
throw std::runtime_error(std::format(
|
|
"Crafter::ParentLib('{}'): not found in parent project '{}'",
|
|
name, root->name));
|
|
}
|
|
|
|
TestRunner TestRunner::Local() {
|
|
TestRunner r;
|
|
r.name = "local";
|
|
r.argsShell = Shell::Host;
|
|
return r;
|
|
}
|
|
|
|
TestRunner TestRunner::Ssh(std::string host, std::string remoteDir) {
|
|
TestRunner r;
|
|
r.name = std::format("ssh:{}", host);
|
|
r.remoteDir = std::move(remoteDir);
|
|
r.argsShell = Shell::Sh;
|
|
// Outer "..." (not '...') so the wrapper survives both sh (Linux host)
|
|
// and cmd (Windows host). cmd doesn't honor single quotes, which would
|
|
// cause it to split on the inner '&&'. Args inside use POSIX quoting
|
|
// because they reach a remote bash regardless of which host issued them.
|
|
r.copy = std::format("ssh -q {0} \"mkdir -p {{remote_bundle}}\" && scp -r -q {{bundle}}/. {0}:{{remote_bundle}}/", host);
|
|
r.exec = std::format("ssh -q {} \"cd {{remote_bundle}} && ./{{bin_name}} {{args}}\"", host);
|
|
r.cleanup = std::format("ssh -q {} \"rm -rf {{remote_bundle}}\"", host);
|
|
// RunCommandChecked captures stdout+stderr internally — no shell redirect
|
|
// needed in the probe spec, which means it works the same way under cmd
|
|
// (Windows host) and sh (Linux host) since `> /dev/null` doesn't translate.
|
|
r.probe = std::format("ssh -q -o BatchMode=yes -o ConnectTimeout=5 {} true", host);
|
|
return r;
|
|
}
|
|
|
|
TestRunner TestRunner::SshWin(std::string host, std::string remoteDir) {
|
|
TestRunner r;
|
|
r.name = std::format("sshwin:{}", host);
|
|
r.remoteDir = std::move(remoteDir);
|
|
r.argsShell = Shell::Cmd;
|
|
// cmd.exe-friendly templates. Use backslash variants of remote paths because
|
|
// cmd's mkdir won't auto-create intermediate directories with forward
|
|
// slashes. scp tolerates either, so we keep forward slashes for the scp
|
|
// dest (which `{remote_bundle}` provides). 2>nul + `& rem ok` swallows
|
|
// mkdir's "already exists" error and forces exit code 0.
|
|
r.copy = std::format("ssh -q {0} \"mkdir {{remote_bundle_win}} 2>nul & rem ok\" && scp -r -q {{bundle}}/. {0}:{{remote_bundle}}/", host);
|
|
r.exec = std::format("ssh -q {} \"{{bin_win}} {{args}}\"", host);
|
|
r.cleanup = std::format("ssh -q {} \"rd /s /q {{remote_bundle_win}} 2>nul & rem ok\"", host);
|
|
// No shell redirect in the probe — see TestRunner::Ssh for rationale.
|
|
r.probe = std::format("ssh -q -o BatchMode=yes -o ConnectTimeout=5 {} \"ver\"", host);
|
|
return r;
|
|
}
|
|
|
|
TestRunner TestRunner::Wsl(std::string remoteDir) {
|
|
TestRunner r;
|
|
r.name = "wsl";
|
|
r.argsShell = Shell::Sh;
|
|
r.remoteDir = std::move(remoteDir);
|
|
// Transport runner: stage the test bundle into WSL's native filesystem
|
|
// (faster than executing in-place from /mnt/c) then run via bash. {bundle_wsl}
|
|
// is the local Windows path translated to /mnt/<drive>/... form so wsl cp
|
|
// can read it. Args are POSIX-quoted because they reach a Linux shell.
|
|
r.copy = "wsl mkdir -p {remote_bundle} && wsl cp -r {bundle_wsl}/. {remote_bundle}/";
|
|
r.exec = "wsl bash -c \"cd {remote_bundle} && ./{bin_name} {args}\"";
|
|
r.cleanup = "wsl rm -rf {remote_bundle}";
|
|
// `where wsl` matches the host-shell convention; on a non-Windows host
|
|
// it fails (no `where` command), which correctly skips this runner.
|
|
r.probe = "where wsl";
|
|
return r;
|
|
}
|
|
|
|
TestRunner TestRunner::Cmd(std::string command) {
|
|
TestRunner r;
|
|
r.name = std::format("cmd:{}", command);
|
|
r.argsShell = Shell::Host;
|
|
r.exec = std::format("{} {{bin}} {{args}}", command);
|
|
#ifdef _WIN32
|
|
r.probe = std::format("where {}", command);
|
|
#else
|
|
r.probe = std::format("which {}", command);
|
|
#endif
|
|
return r;
|
|
}
|
|
|
|
TestRunner TestRunner::Wine() {
|
|
TestRunner r;
|
|
r.name = "wine";
|
|
r.argsShell = Shell::Host;
|
|
r.exec = "wine {bin} {args}";
|
|
#ifdef _WIN32
|
|
r.probe = "where wine";
|
|
#else
|
|
r.probe = "which wine";
|
|
#endif
|
|
return r;
|
|
}
|
|
|
|
TestRunner TestRunner::ForTarget(const Configuration& cfg) {
|
|
const std::string& target = cfg.target;
|
|
|
|
// Same triple as the host → run the binary directly. Covers the common
|
|
// case (cfg.target defaulted to HostTarget()) without any wrapper.
|
|
if (target == HostTarget()) return Local();
|
|
|
|
// Windows targets: native on a Windows host, Wine on Linux. We don't
|
|
// distinguish mingw vs msvc here — the produced .exe runs the same way.
|
|
if (TargetIsWindows(target)) {
|
|
return TargetIsWindows(HostTarget()) ? Local() : Wine();
|
|
}
|
|
|
|
// WASI: a .wasm file isn't directly executable; wasmtime is the canonical
|
|
// runtime. wasi-cli also works but the upstream Bytecode Alliance name is
|
|
// wasmtime, so we standardize on that.
|
|
if (target.starts_with("wasm32-wasi") || target.starts_with("wasm64-wasi")) {
|
|
return Cmd("wasmtime");
|
|
}
|
|
|
|
// Non-host Linux triple: extract the architecture and route through
|
|
// qemu-user. Triple is <arch>-<vendor>-<os>-<env> (or sometimes 3 parts);
|
|
// qemu-user's binary names mostly follow the arch field, with two known
|
|
// mismatches handled below. cfg.sysroot, when set, becomes QEMU_LD_PREFIX
|
|
// so the target's dynamic linker / shared libs are reachable — without
|
|
// it qemu-user crashes on dynamic ELFs with "could not open /lib/ld...".
|
|
if (target.find("-linux-") != std::string::npos) {
|
|
auto dash = target.find('-');
|
|
std::string arch = target.substr(0, dash);
|
|
// i686-linux-gnu → qemu-i386; arm-* already matches qemu-arm; aarch64,
|
|
// riscv64, ppc64le, mips, mips64, s390x all match their qemu names.
|
|
if (arch == "i686") arch = "i386";
|
|
TestRunner r = Cmd(std::format("qemu-{}", arch));
|
|
if (!cfg.sysroot.empty()) {
|
|
// Use `env VAR=value cmd` rather than the shell's `VAR=value cmd`
|
|
// prefix syntax: RunCommandWithTimeout pipes through GNU `timeout`,
|
|
// which execvp's its argument list directly without going through a
|
|
// shell. A bare VAR=value would be exec'd as a command path and
|
|
// fail with "No such file or directory".
|
|
r.exec = std::format("env QEMU_LD_PREFIX={} {}", cfg.sysroot, r.exec);
|
|
}
|
|
return r;
|
|
}
|
|
|
|
// Unknown / bare-metal / freestanding targets: fall back to Local. The
|
|
// caller's runner-availability probe (or absence of the binary) surfaces
|
|
// the problem rather than us inventing a wrong wrapper here.
|
|
return Local();
|
|
}
|
|
|
|
namespace {
|
|
std::string NormalizeTriple(std::string_view target) {
|
|
std::string out(target);
|
|
for (char& c : out) {
|
|
if (c == '-' || c == '.') c = '_';
|
|
}
|
|
return out;
|
|
}
|
|
|
|
std::optional<TestRunner> ParseRunnerSpec(std::string_view spec) {
|
|
if (spec.empty()) return std::nullopt;
|
|
std::vector<std::string> parts;
|
|
for (auto piece : std::views::split(spec, ':')) {
|
|
parts.emplace_back(std::string_view(piece.begin(), piece.end()));
|
|
}
|
|
if (parts.empty()) return std::nullopt;
|
|
if (parts[0] == "local") return TestRunner::Local();
|
|
if (parts[0] == "cmd" && parts.size() == 2) return TestRunner::Cmd(parts[1]);
|
|
if (parts[0] == "ssh" && parts.size() == 2) return TestRunner::Ssh(parts[1]);
|
|
if (parts[0] == "ssh" && parts.size() >= 3) {
|
|
std::string remote;
|
|
for (std::size_t i = 2; i < parts.size(); ++i) {
|
|
if (i > 2) remote.push_back(':');
|
|
remote += parts[i];
|
|
}
|
|
return TestRunner::Ssh(parts[1], remote);
|
|
}
|
|
if (parts[0] == "sshwin" && parts.size() == 2) return TestRunner::SshWin(parts[1]);
|
|
if (parts[0] == "sshwin" && parts.size() >= 3) {
|
|
std::string remote;
|
|
for (std::size_t i = 2; i < parts.size(); ++i) {
|
|
if (i > 2) remote.push_back(':');
|
|
remote += parts[i];
|
|
}
|
|
return TestRunner::SshWin(parts[1], remote);
|
|
}
|
|
if (parts[0] == "wsl" && parts.size() == 1) return TestRunner::Wsl();
|
|
if (parts[0] == "wsl" && parts.size() >= 2) {
|
|
std::string remote;
|
|
for (std::size_t i = 1; i < parts.size(); ++i) {
|
|
if (i > 1) remote.push_back(':');
|
|
remote += parts[i];
|
|
}
|
|
return TestRunner::Wsl(remote);
|
|
}
|
|
throw std::runtime_error(std::format(
|
|
"TestRunner::FromSpec: unrecognized runner spec '{}'", spec));
|
|
}
|
|
|
|
// C:\Users\jorij -> /mnt/c/Users/jorij. Idempotent on paths that don't
|
|
// start with a drive letter, so callers can compute it unconditionally.
|
|
std::string WindowsPathToWsl(std::string_view p) {
|
|
if (p.size() < 2 || p[1] != ':') return std::string(p);
|
|
char drive = static_cast<char>(std::tolower(static_cast<unsigned char>(p[0])));
|
|
std::string out = std::format("/mnt/{}", drive);
|
|
for (std::size_t i = 2; i < p.size(); ++i) {
|
|
out.push_back(p[i] == '\\' ? '/' : p[i]);
|
|
}
|
|
return out;
|
|
}
|
|
}
|
|
|
|
std::optional<TestRunner> TestRunner::FromSpec(std::string_view spec) {
|
|
return ParseRunnerSpec(spec);
|
|
}
|
|
|
|
TestRunner TestRunner::FromEnv(std::string_view target, TestRunner fallback) {
|
|
std::string envName = std::format("CRAFTER_BUILD_RUNNER_{}", NormalizeTriple(target));
|
|
const char* v = std::getenv(envName.c_str());
|
|
if (!v || !*v) return fallback;
|
|
if (auto r = ParseRunnerSpec(v)) return std::move(*r);
|
|
return fallback;
|
|
}
|
|
|
|
TestResult Crafter::RunSingleTest(const Test& test, const fs::path& binary, std::chrono::seconds timeout) {
|
|
using namespace std::chrono_literals;
|
|
TestResult result;
|
|
result.name = test.config.name;
|
|
|
|
std::map<std::string, std::string> ph;
|
|
ph["{args}"] = JoinAndQuoteArgs(test.args, test.runner.argsShell);
|
|
ph["{bin_name}"] = binary.filename().string();
|
|
ph["{bundle}"] = binary.parent_path().string();
|
|
|
|
auto start = std::chrono::steady_clock::now();
|
|
CommandResult r;
|
|
|
|
if (test.runner.exec.empty()) {
|
|
// Pure-local runner: spawn the binary directly through the host shell.
|
|
std::string cmd = std::format("{} {}", ShellQuoteHost(binary.string()), ph["{args}"]);
|
|
r = RunCommandWithTimeout(cmd, timeout);
|
|
} else if (test.runner.copy.empty()) {
|
|
// Prefix runner (qemu-user, wsl, ...): templated exec wraps a local binary.
|
|
ph["{bin}"] = binary.string();
|
|
r = RunCommandWithTimeout(Substitute(test.runner.exec, ph), timeout);
|
|
} else {
|
|
// Transport runner (ssh, ...): copy bundle → exec remotely → cleanup.
|
|
std::string unique = std::format("{}-{}-{}",
|
|
test.config.name,
|
|
std::hash<std::thread::id>{}(std::this_thread::get_id()),
|
|
std::chrono::steady_clock::now().time_since_epoch().count());
|
|
ph["{remote_bundle}"] = std::format("{}/{}", test.runner.remoteDir, unique);
|
|
ph["{bin}"] = std::format("{}/{}", ph["{remote_bundle}"], ph["{bin_name}"]);
|
|
// Backslash variants for cmd.exe-shell remotes (cmd's mkdir won't
|
|
// auto-create intermediate directories when path uses forward slashes).
|
|
ph["{remote_bundle_win}"] = ph["{remote_bundle}"];
|
|
std::replace(ph["{remote_bundle_win}"].begin(), ph["{remote_bundle_win}"].end(), '/', '\\');
|
|
ph["{bin_win}"] = ph["{bin}"];
|
|
std::replace(ph["{bin_win}"].begin(), ph["{bin_win}"].end(), '/', '\\');
|
|
// WSL form of the local bundle path: C:\foo -> /mnt/c/foo. Used by
|
|
// the Wsl runner's `wsl cp -r` step to read the test bundle out of
|
|
// the Windows host's filesystem.
|
|
ph["{bundle_wsl}"] = WindowsPathToWsl(ph["{bundle}"]);
|
|
|
|
CommandResult cp = RunCommandWithTimeout(Substitute(test.runner.copy, ph), 5min);
|
|
if (cp.exitCode != 0) {
|
|
auto end = std::chrono::steady_clock::now();
|
|
result.duration = std::chrono::duration_cast<std::chrono::milliseconds>(end - start);
|
|
result.outcome = TestOutcome::Fail;
|
|
result.exitCode = cp.exitCode;
|
|
result.output = std::format("copy step failed (exit {}):\n{}", cp.exitCode, cp.output);
|
|
return result;
|
|
}
|
|
|
|
r = RunCommandWithTimeout(Substitute(test.runner.exec, ph), timeout);
|
|
|
|
if (!test.runner.cleanup.empty()) {
|
|
std::system(Substitute(test.runner.cleanup, ph).c_str());
|
|
}
|
|
}
|
|
|
|
auto end = std::chrono::steady_clock::now();
|
|
result.duration = std::chrono::duration_cast<std::chrono::milliseconds>(end - start);
|
|
result.output = std::move(r.output);
|
|
result.exitCode = r.exitCode;
|
|
result.signal = r.signal;
|
|
|
|
if (r.timedOut) result.outcome = TestOutcome::Timeout;
|
|
else if (r.crashed) result.outcome = TestOutcome::Crash;
|
|
else if (r.exitCode == 77) result.outcome = TestOutcome::Skipped;
|
|
else if (r.exitCode != 0) result.outcome = TestOutcome::Fail;
|
|
else result.outcome = TestOutcome::Pass;
|
|
|
|
return result;
|
|
}
|
|
|
|
namespace {
|
|
// Declarative test metadata loaded from tests/<Name>/test.toml. Lets a test
|
|
// ship just main.cpp + a few lines of config instead of a whole project.cpp
|
|
// when its needs are "pick a target, gate on prerequisites, run with these
|
|
// args". project.cpp stays the escape hatch for outer-driver tests that
|
|
// call Build() / inspect intermediate state.
|
|
struct TestManifest {
|
|
std::optional<std::string> target;
|
|
std::optional<std::string> march;
|
|
std::optional<std::string> mtune;
|
|
std::optional<std::string> sysroot;
|
|
std::vector<std::string> requires_;
|
|
std::optional<int> timeoutSeconds;
|
|
std::vector<std::string> args;
|
|
std::vector<std::pair<std::string, std::string>> defines;
|
|
};
|
|
|
|
TestManifest ParseTestManifest(const fs::path& path) {
|
|
// toml++ builds with exceptions enabled by default; parse_file throws
|
|
// toml::parse_error on malformed input. Rethrow with the path attached
|
|
// so the discovery loop's catch can surface "where the error came from"
|
|
// alongside toml++'s "what was wrong".
|
|
toml::table t;
|
|
try {
|
|
t = toml::parse_file(path.string());
|
|
} catch (const toml::parse_error& e) {
|
|
throw std::runtime_error(std::format(
|
|
"test.toml parse error in {}: {}",
|
|
path.string(),
|
|
std::string_view(e.description())));
|
|
}
|
|
TestManifest m;
|
|
if (auto v = t["target"].value<std::string>()) m.target = *v;
|
|
if (auto v = t["march"].value<std::string>()) m.march = *v;
|
|
if (auto v = t["mtune"].value<std::string>()) m.mtune = *v;
|
|
if (auto v = t["sysroot"].value<std::string>()) m.sysroot = *v;
|
|
if (auto v = t["timeout"].value<int64_t>()) m.timeoutSeconds = static_cast<int>(*v);
|
|
if (auto arr = t["requires"].as_array()) {
|
|
for (auto& el : *arr) {
|
|
if (auto s = el.value<std::string>()) m.requires_.push_back(*s);
|
|
}
|
|
}
|
|
if (auto arr = t["args"].as_array()) {
|
|
for (auto& el : *arr) {
|
|
if (auto s = el.value<std::string>()) m.args.push_back(*s);
|
|
}
|
|
}
|
|
if (auto tbl = t["defines"].as_table()) {
|
|
for (auto&& [k, v] : *tbl) {
|
|
if (auto s = v.value<std::string>()) {
|
|
m.defines.emplace_back(std::string(k.str()), *s);
|
|
}
|
|
}
|
|
}
|
|
return m;
|
|
}
|
|
|
|
// Apply manifest overlay onto a Configuration synthesized from the test
|
|
// folder. Target overrides come last so a manifest's `target = "..."`
|
|
// wins over the synth default (= run's targetFilter). Defines accumulate;
|
|
// they don't replace pre-existing ones.
|
|
void ApplyManifest(Configuration& cfg, const TestManifest& m) {
|
|
if (m.target) cfg.target = *m.target;
|
|
if (m.march) cfg.march = *m.march;
|
|
if (m.mtune) cfg.mtune = *m.mtune;
|
|
if (m.sysroot) cfg.sysroot = *m.sysroot;
|
|
for (auto& [k, v] : m.defines) cfg.defines.push_back({k, v});
|
|
}
|
|
|
|
bool ToolOnPath(std::string_view name) {
|
|
#ifdef _WIN32
|
|
std::string cmd = std::format("where {} > nul 2>&1", name);
|
|
#else
|
|
std::string cmd = std::format("which {} > /dev/null 2>&1", name);
|
|
#endif
|
|
return std::system(cmd.c_str()) == 0;
|
|
}
|
|
|
|
struct RequireResult {
|
|
bool ok;
|
|
std::string reason; // human-readable when !ok
|
|
};
|
|
|
|
// Evaluate each `<kind>:<arg>` precondition. Returns the first failure
|
|
// (short-circuit; reporting one missing dep at a time is enough to act on
|
|
// and keeps the test log uncluttered).
|
|
RequireResult EvaluateRequires(std::span<const std::string> reqs) {
|
|
for (const auto& r : reqs) {
|
|
auto sep = r.find(':');
|
|
if (sep == std::string::npos || sep == 0 || sep == r.size() - 1) {
|
|
return {false, std::format("malformed require '{}' (expected kind:arg)", r)};
|
|
}
|
|
std::string_view kind(r.data(), sep);
|
|
std::string_view arg(r.data() + sep + 1, r.size() - sep - 1);
|
|
if (kind == "tool") {
|
|
if (!ToolOnPath(arg)) {
|
|
return {false, std::format("tool '{}' not on PATH", arg)};
|
|
}
|
|
} else if (kind == "file") {
|
|
if (!fs::exists(std::string(arg))) {
|
|
return {false, std::format("file '{}' missing", arg)};
|
|
}
|
|
} else if (kind == "env") {
|
|
const char* v = std::getenv(std::string(arg).c_str());
|
|
if (!v || !*v) {
|
|
return {false, std::format("env '{}' unset", arg)};
|
|
}
|
|
} else {
|
|
return {false, std::format(
|
|
"unknown require kind '{}' (expected tool/file/env)", kind)};
|
|
}
|
|
}
|
|
return {true, ""};
|
|
}
|
|
|
|
// Match a runner's tool dependency against the test's declared
|
|
// requirements. Used to decide between Skip (declared, may legitimately
|
|
// be missing) and Fail (runner unavailable but test didn't declare it —
|
|
// a silent skip would mask broken cross-arch CI configuration).
|
|
bool RequiresMentionsTool(std::span<const std::string> reqs, std::string_view tool) {
|
|
std::string needle = std::format("tool:{}", tool);
|
|
return std::ranges::any_of(reqs, [&](const std::string& s) { return s == needle; });
|
|
}
|
|
|
|
// Best-effort extraction of the runner-tool name from a TestRunner so the
|
|
// hard-fail-unless-declared check can match it against `requires`. For
|
|
// Cmd("foo"), the name is "cmd:foo"; for Wine, it's "wine". Anything else
|
|
// (Local, transport runners) returns empty — those don't trigger the
|
|
// declared/undeclared gate.
|
|
std::string RunnerToolName(const TestRunner& runner) {
|
|
if (runner.name == "wine") return "wine";
|
|
if (runner.name.starts_with("cmd:")) {
|
|
std::string tool = runner.name.substr(4);
|
|
// QEMU_LD_PREFIX prefix may be glued onto exec but the runner's
|
|
// `name` field already isolates the command, so no extra parsing.
|
|
return tool;
|
|
}
|
|
return "";
|
|
}
|
|
}
|
|
|
|
namespace {
|
|
// Synthesize a Configuration for tests/<Name>/ folders that don't contain
|
|
// a project.cpp. Convention: cfg.path = the folder, cfg.name/outputName =
|
|
// folder basename, cfg.target = host (overridable via test.toml `target`),
|
|
// cfg.type = exe. Sources: top-level *.cpp (excluding project.cpp) become
|
|
// implementations, interfaces/*.cppm become module interfaces. Tests with
|
|
// deeper layouts or dependencies still need an explicit project.cpp.
|
|
//
|
|
// Why host-default instead of targetFilter-default: under the multi-target
|
|
// sweep, an arch-agnostic test (no test.toml target) should run at the
|
|
// host iteration only — not get rebuilt against every cross-target the
|
|
// suite happens to declare. Cross-targeting is an opt-in via test.toml.
|
|
Configuration SynthesizeTest(const fs::path& dir) {
|
|
Configuration cfg;
|
|
cfg.path = dir;
|
|
cfg.name = dir.filename().string();
|
|
cfg.outputName = cfg.name;
|
|
cfg.target = HostTarget();
|
|
cfg.type = ConfigurationType::Executable;
|
|
|
|
std::vector<fs::path> impls;
|
|
for (auto& e : fs::directory_iterator(dir)) {
|
|
if (!e.is_regular_file()) continue;
|
|
auto p = e.path();
|
|
if (p.extension() != ".cpp") continue;
|
|
if (p.filename() == "project.cpp") continue;
|
|
impls.push_back(p.stem());
|
|
}
|
|
std::ranges::sort(impls);
|
|
|
|
std::vector<fs::path> ifaces;
|
|
fs::path interfacesDir = dir / "interfaces";
|
|
if (fs::exists(interfacesDir) && fs::is_directory(interfacesDir)) {
|
|
for (auto& e : fs::directory_iterator(interfacesDir)) {
|
|
if (!e.is_regular_file()) continue;
|
|
auto p = e.path();
|
|
if (p.extension() != ".cppm") continue;
|
|
ifaces.push_back(fs::path("interfaces") / p.stem());
|
|
}
|
|
std::ranges::sort(ifaces);
|
|
}
|
|
|
|
if (impls.empty() && ifaces.empty()) {
|
|
throw std::runtime_error(std::format(
|
|
"no .cpp or interfaces/*.cppm files found in {}", dir.string()));
|
|
}
|
|
|
|
cfg.GetInterfacesAndImplementations(ifaces, impls);
|
|
return cfg;
|
|
}
|
|
}
|
|
|
|
TestSummary Crafter::RunTests(Configuration& projectCfg, const RunTestsOptions& opts, std::span<const std::string_view> projectArgs) {
|
|
// Multi-target sweep: when no --target= was given, the run covers every
|
|
// distinct target a test.toml declares plus the host target. Lets a bare
|
|
// `crafter-build test` exercise cross-arch tests without the user having
|
|
// to know which targets exist in this project. An explicit --target=X
|
|
// bypasses the sweep and runs that target only.
|
|
if (opts.targetFilter.empty()) {
|
|
std::set<std::string> sweep;
|
|
sweep.insert(HostTarget());
|
|
fs::path testsDir = fs::current_path() / "tests";
|
|
if (fs::exists(testsDir) && fs::is_directory(testsDir)) {
|
|
for (auto& e : fs::directory_iterator(testsDir)) {
|
|
if (!e.is_directory()) continue;
|
|
auto stem = e.path().filename().string();
|
|
if (stem.empty() || stem[0] == '_' || stem[0] == '.') continue;
|
|
fs::path tomlPath = e.path() / "test.toml";
|
|
if (!fs::exists(tomlPath)) continue;
|
|
try {
|
|
TestManifest m = ParseTestManifest(tomlPath);
|
|
if (m.target) sweep.insert(*m.target);
|
|
} catch (...) {
|
|
// Parse failures surface as discovery failures during the
|
|
// actual run; the sweep phase just collects targets.
|
|
}
|
|
}
|
|
}
|
|
TestSummary aggregate;
|
|
// Inline tests pushed by the caller (fixture-driven inner RunTests
|
|
// calls, e.g. RunnerClassification) must survive each sweep
|
|
// iteration. Configuration isn't copyable so we can't snapshot+restore;
|
|
// instead, remember the inline count and erase only the entries
|
|
// appended by the previous iteration's auto-discovery.
|
|
size_t inlineCount = projectCfg.tests.size();
|
|
for (const auto& target : sweep) {
|
|
RunTestsOptions perTarget = opts;
|
|
perTarget.targetFilter = target;
|
|
if (projectCfg.tests.size() > inlineCount) {
|
|
projectCfg.tests.erase(
|
|
projectCfg.tests.begin() + inlineCount,
|
|
projectCfg.tests.end());
|
|
}
|
|
if (sweep.size() > 1) {
|
|
Progress::Clear();
|
|
std::println("\n=== target: {} ===", target);
|
|
}
|
|
TestSummary s = RunTests(projectCfg, perTarget, projectArgs);
|
|
aggregate.passed += s.passed;
|
|
aggregate.failed += s.failed;
|
|
aggregate.crashed += s.crashed;
|
|
aggregate.timedOut += s.timedOut;
|
|
aggregate.skipped += s.skipped;
|
|
for (auto& r : s.results) aggregate.results.push_back(std::move(r));
|
|
}
|
|
return aggregate;
|
|
}
|
|
|
|
TestSummary summary;
|
|
std::vector<TestResult> discoveryFailures;
|
|
|
|
// Auto-discover tests one layer deep: each <project>/tests/<Name>/ folder
|
|
// is a test. If it contains project.cpp, that's loaded for full control;
|
|
// otherwise the Configuration is synthesized from the folder contents.
|
|
// Folders whose name starts with '_' or '.' are skipped (so tests/_shared/
|
|
// holds cross-test code without becoming a test). Each project.cpp receives
|
|
// the same args the root project did, so --target=... propagates through.
|
|
//
|
|
// Discovery is keyed off cwd (= the project root, since crafter-build loads
|
|
// ./project.cpp), not projectCfg.path: tests live at the project root even
|
|
// when projectCfg.path points at a subdirectory like "./src/" or "./lib/".
|
|
fs::path testsDir = fs::current_path() / "tests";
|
|
if (fs::exists(testsDir) && fs::is_directory(testsDir)) {
|
|
// A discovered fixture is one of:
|
|
// - project.cpp present → outer-driver test (LoadProject)
|
|
// - test.toml present → declarative synth + manifest
|
|
// - neither, just *.cpp → bare synth (host-target)
|
|
// - both project.cpp and test.toml → XOR violation, discovery Fail
|
|
struct TestEntry {
|
|
fs::path dir;
|
|
fs::path pcpp; // outer-driver path
|
|
std::optional<TestManifest> manifest;
|
|
};
|
|
std::vector<TestEntry> entries;
|
|
for (auto& entry : fs::directory_iterator(testsDir)) {
|
|
if (!entry.is_directory()) continue;
|
|
auto stem = entry.path().filename().string();
|
|
if (stem.empty() || stem[0] == '_' || stem[0] == '.') continue;
|
|
TestEntry te;
|
|
te.dir = entry.path();
|
|
auto pcpp = te.dir / "project.cpp";
|
|
auto tomlPath = te.dir / "test.toml";
|
|
bool hasPcpp = fs::exists(pcpp);
|
|
bool hasToml = fs::exists(tomlPath);
|
|
if (hasPcpp && hasToml) {
|
|
TestResult r;
|
|
r.name = stem;
|
|
r.outcome = TestOutcome::Fail;
|
|
r.exitCode = -1;
|
|
r.output = "both project.cpp and test.toml present — they're "
|
|
"mutually exclusive (delete one to disambiguate "
|
|
"outer-driver vs declarative test)";
|
|
discoveryFailures.push_back(std::move(r));
|
|
continue;
|
|
}
|
|
if (hasPcpp) te.pcpp = pcpp;
|
|
if (hasToml) {
|
|
try {
|
|
te.manifest = ParseTestManifest(tomlPath);
|
|
} catch (const std::exception& e) {
|
|
TestResult r;
|
|
r.name = stem;
|
|
r.outcome = TestOutcome::Fail;
|
|
r.exitCode = -1;
|
|
r.output = e.what();
|
|
discoveryFailures.push_back(std::move(r));
|
|
continue;
|
|
}
|
|
}
|
|
entries.push_back(std::move(te));
|
|
}
|
|
std::ranges::sort(entries, [](auto& a, auto& b) { return a.dir < b.dir; });
|
|
|
|
// Inject --target=<filter> into the args we hand each fixture so its
|
|
// CrafterBuildProject can default to the run's target. The CLI's own
|
|
// --target=... propagates through projectArgs already; this only
|
|
// appends when missing so an explicit user choice wins.
|
|
std::string targetArg = std::format("--target={}", opts.targetFilter);
|
|
std::vector<std::string_view> fixtureArgs(projectArgs.begin(), projectArgs.end());
|
|
bool hasTarget = std::ranges::any_of(fixtureArgs, [](std::string_view a) {
|
|
return a.starts_with("--target=");
|
|
});
|
|
if (!hasTarget) fixtureArgs.push_back(targetArg);
|
|
|
|
for (auto& te : entries) {
|
|
Test t;
|
|
try {
|
|
if (!te.pcpp.empty()) {
|
|
t.config = LoadProject(te.pcpp, fixtureArgs);
|
|
} else {
|
|
t.config = SynthesizeTest(te.dir);
|
|
if (te.manifest) {
|
|
ApplyManifest(t.config, *te.manifest);
|
|
if (te.manifest->timeoutSeconds) {
|
|
t.timeout = std::chrono::seconds(*te.manifest->timeoutSeconds);
|
|
}
|
|
t.args = te.manifest->args;
|
|
t.requires_ = te.manifest->requires_;
|
|
}
|
|
}
|
|
} catch (const std::exception& e) {
|
|
// A broken fixture shouldn't kill the whole run. Surface as a
|
|
// Fail and let other tests proceed.
|
|
TestResult r;
|
|
r.name = te.dir.filename().string();
|
|
r.outcome = TestOutcome::Fail;
|
|
r.exitCode = -1;
|
|
r.output = !te.pcpp.empty()
|
|
? std::format("project.cpp failed to load: {}", e.what())
|
|
: std::format("test discovery failed: {}", e.what());
|
|
discoveryFailures.push_back(std::move(r));
|
|
continue;
|
|
}
|
|
if (t.config.target != opts.targetFilter) continue;
|
|
t.runner = TestRunner::FromEnv(t.config.target, TestRunner::ForTarget(t.config));
|
|
if (opts.runnerOverride) {
|
|
if (auto r = TestRunner::FromSpec(*opts.runnerOverride)) {
|
|
t.runner = std::move(*r);
|
|
}
|
|
}
|
|
projectCfg.tests.push_back(std::move(t));
|
|
}
|
|
}
|
|
|
|
std::vector<Test*> filtered;
|
|
filtered.reserve(projectCfg.tests.size());
|
|
for (auto& test : projectCfg.tests) {
|
|
if (MatchAny(opts.globs, test.config.name)) {
|
|
filtered.push_back(&test);
|
|
}
|
|
}
|
|
|
|
if (opts.listOnly) {
|
|
for (auto& r : discoveryFailures) {
|
|
std::println("{} (project.cpp broken)", r.name);
|
|
}
|
|
for (auto* t : filtered) {
|
|
std::println("{}", t->config.name);
|
|
}
|
|
return summary;
|
|
}
|
|
|
|
if (filtered.empty() && discoveryFailures.empty()) {
|
|
std::println("No tests matched.");
|
|
return summary;
|
|
}
|
|
|
|
// Render discovery failures upfront so they appear before parallel test
|
|
// results. They're already-determined Fails — no need to put them through
|
|
// the worker pool.
|
|
for (auto& r : discoveryFailures) {
|
|
PrintResult(r, "");
|
|
WriteLog(projectCfg.path, r.name, r.output);
|
|
}
|
|
|
|
int jobs = opts.jobs > 0
|
|
? opts.jobs
|
|
: std::max(1u, std::thread::hardware_concurrency());
|
|
jobs = std::min(jobs, static_cast<int>(filtered.size()));
|
|
|
|
std::unordered_map<fs::path, std::shared_future<BuildResult>> depResults;
|
|
std::mutex depMutex;
|
|
std::mutex printMutex;
|
|
std::mutex probeMutex;
|
|
std::unordered_map<std::string, bool> probeCache;
|
|
std::atomic<std::size_t> next{0};
|
|
std::vector<TestResult> results(filtered.size());
|
|
|
|
auto runnerAvailable = [&](const TestRunner& runner) -> bool {
|
|
if (runner.probe.empty()) return true;
|
|
std::lock_guard lk(probeMutex);
|
|
if (auto it = probeCache.find(runner.name); it != probeCache.end()) {
|
|
return it->second;
|
|
}
|
|
// RunCommandChecked captures and discards output internally, so probe
|
|
// specs don't need `> /dev/null 2>&1` (which doesn't translate to cmd).
|
|
bool ok = (RunCommandChecked(runner.probe).exitCode == 0);
|
|
probeCache[runner.name] = ok;
|
|
return ok;
|
|
};
|
|
|
|
auto worker = [&]() {
|
|
while (true) {
|
|
std::size_t i = next.fetch_add(1);
|
|
if (i >= filtered.size()) break;
|
|
Test& t = *filtered[i];
|
|
TestResult r;
|
|
r.name = t.config.name;
|
|
|
|
// Declarative preconditions (test.toml requires = [...] or Test.requires_
|
|
// set in project.cpp). Evaluated before the build so a missing tool/file/env
|
|
// turns into a Skip without paying the compile cost. Reports the first
|
|
// failure only — once one precondition is unmet the test couldn't run
|
|
// anyway, and a wall of "also missing X, also missing Y" buries the
|
|
// actionable root cause.
|
|
if (auto req = EvaluateRequires(t.requires_); !req.ok) {
|
|
r.outcome = TestOutcome::Skipped;
|
|
r.output = req.reason;
|
|
{
|
|
std::lock_guard lk(printMutex);
|
|
PrintResult(r, t.runner.name);
|
|
}
|
|
results[i] = std::move(r);
|
|
continue;
|
|
}
|
|
|
|
if (!runnerAvailable(t.runner)) {
|
|
// Hard-fail-unless-declared: if the runner depends on a tool
|
|
// (qemu-aarch64, wasmtime, wine, ...) and the test didn't say
|
|
// "tool:<that>" in requires, the missing runner is a Fail. The
|
|
// intent is to surface broken cross-arch CI configuration
|
|
// instead of letting it masquerade as a Skip; tests that
|
|
// legitimately may run without their runner have to opt in.
|
|
std::string tool = RunnerToolName(t.runner);
|
|
if (!tool.empty() && !RequiresMentionsTool(t.requires_, tool)) {
|
|
r.outcome = TestOutcome::Fail;
|
|
r.exitCode = -1;
|
|
r.output = std::format(
|
|
"runner '{}' unavailable and not declared in requires "
|
|
"(add 'tool:{}' to test.toml requires to permit skipping)",
|
|
t.runner.name, tool);
|
|
} else {
|
|
r.outcome = TestOutcome::Skipped;
|
|
r.output = std::format("runner '{}' not available", t.runner.name);
|
|
}
|
|
{
|
|
std::lock_guard lk(printMutex);
|
|
PrintResult(r, t.runner.name);
|
|
}
|
|
results[i] = std::move(r);
|
|
continue;
|
|
}
|
|
|
|
BuildResult br;
|
|
try {
|
|
br = Build(t.config, depResults, depMutex);
|
|
} catch (const std::exception& e) {
|
|
r.outcome = TestOutcome::Fail;
|
|
r.output = std::format("build threw: {}", e.what());
|
|
r.exitCode = -1;
|
|
{
|
|
std::lock_guard lk(printMutex);
|
|
PrintResult(r, t.runner.name);
|
|
}
|
|
results[i] = std::move(r);
|
|
continue;
|
|
}
|
|
|
|
if (!br.result.empty()) {
|
|
r.outcome = TestOutcome::Fail;
|
|
r.output = std::format("build failed: {}", br.result);
|
|
r.exitCode = -1;
|
|
{
|
|
std::lock_guard lk(printMutex);
|
|
PrintResult(r, t.runner.name);
|
|
}
|
|
results[i] = std::move(r);
|
|
continue;
|
|
}
|
|
|
|
std::chrono::seconds timeout = opts.timeoutOverride.value_or(t.timeout);
|
|
fs::path binary = TestBinaryPath(t.config);
|
|
try {
|
|
r = RunSingleTest(t, binary, timeout);
|
|
} catch (const std::exception& e) {
|
|
r.outcome = TestOutcome::Fail;
|
|
r.output = std::format("runner threw: {}", e.what());
|
|
r.exitCode = -1;
|
|
}
|
|
|
|
if (r.outcome != TestOutcome::Pass && r.outcome != TestOutcome::Skipped && !r.output.empty()) {
|
|
WriteLog(projectCfg.path, r.name, r.output);
|
|
}
|
|
|
|
{
|
|
std::lock_guard lk(printMutex);
|
|
PrintResult(r, t.runner.name);
|
|
}
|
|
results[i] = std::move(r);
|
|
}
|
|
};
|
|
|
|
std::vector<std::jthread> threads;
|
|
threads.reserve(jobs);
|
|
for (int j = 0; j < jobs; ++j) {
|
|
threads.emplace_back(worker);
|
|
}
|
|
threads.clear(); // joins all jthreads
|
|
|
|
for (auto& r : discoveryFailures) {
|
|
results.push_back(std::move(r));
|
|
}
|
|
|
|
for (auto& r : results) {
|
|
switch (r.outcome) {
|
|
case TestOutcome::Pass: summary.passed++; break;
|
|
case TestOutcome::Fail: summary.failed++; break;
|
|
case TestOutcome::Crash: summary.crashed++; break;
|
|
case TestOutcome::Timeout: summary.timedOut++; break;
|
|
case TestOutcome::Skipped: summary.skipped++; break;
|
|
}
|
|
}
|
|
summary.results = std::move(results);
|
|
|
|
Progress::Clear();
|
|
std::print("\n");
|
|
std::vector<std::string> parts;
|
|
if (summary.passed) parts.push_back(std::format("{} passed", summary.passed));
|
|
if (summary.failed) parts.push_back(std::format("{} failed", summary.failed));
|
|
if (summary.crashed) parts.push_back(std::format("{} crashed", summary.crashed));
|
|
if (summary.timedOut) parts.push_back(std::format("{} timed out", summary.timedOut));
|
|
if (summary.skipped) parts.push_back(std::format("{} skipped", summary.skipped));
|
|
std::string joined;
|
|
for (std::size_t i = 0; i < parts.size(); ++i) {
|
|
if (i) joined += ", ";
|
|
joined += parts[i];
|
|
}
|
|
std::println("{}", joined);
|
|
|
|
return summary;
|
|
}
|