Crafter.Build/implementations/Crafter.Build-Test.cpp

// SPDX-License-Identifier: LGPL-3.0-only
// SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts®

module;
export module Crafter.Build:Test_impl;
import std;
import :Test;
import :Clang;
import :Platform;
import :Progress;
namespace fs = std::filesystem;
using namespace Crafter;

namespace {
    bool TargetIsWindows(std::string_view target) {
        return target.find("windows") != std::string_view::npos || target.find("mingw") != std::string_view::npos;
    }

    fs::path TestBinaryPath(const Configuration& cfg) {
        fs::path outputDir = cfg.BinDir();
        return outputDir / (TargetIsWindows(cfg.target) ? std::format("{}.exe", cfg.outputName) : cfg.outputName);
    }

    // MatchGlob/MatchAny live in :Platform — shared with the lint verb.

    std::string JoinAndQuoteArgs(std::span<const std::string> args) {
        std::string out;
        for (const auto& a : args) {
            if (!out.empty()) out.push_back(' ');
            out += ShellQuote(a);
        }
        return out;
    }

    std::string Substitute(std::string_view tmpl, const std::map<std::string, std::string>& ph) {
        std::string out;
        out.reserve(tmpl.size());
        std::size_t i = 0;
        while (i < tmpl.size()) {
            if (tmpl[i] == '{') {
                std::size_t end = tmpl.find('}', i + 1);
                if (end != std::string_view::npos) {
                    std::string key(tmpl.substr(i, end - i + 1));
                    if (auto it = ph.find(key); it != ph.end()) {
                        out += it->second;
                        i = end + 1;
                        continue;
                    }
                }
            }
            out.push_back(tmpl[i++]);
        }
        return out;
    }

    std::string SignalName(std::int32_t sig) {
        switch (sig) {
            case 1: return "SIGHUP";
            case 2: return "SIGINT";
            case 4: return "SIGILL";
            case 6: return "SIGABRT";
            case 8: return "SIGFPE";
            case 9: return "SIGKILL";
            case 11: return "SIGSEGV";
            case 13: return "SIGPIPE";
            case 14: return "SIGALRM";
            case 15: return "SIGTERM";
            default: return std::format("signal {}", sig);
        }
    }

    void WriteLog(const fs::path& projectPath, const std::string& name, const std::string& output) {
        fs::path logDir = projectPath / "build" / "test-logs";
        std::error_code ec;
        fs::create_directories(logDir, ec);
        if (ec) return;
        std::ofstream(logDir / (std::format("{}.log", name))) << output;
    }

    void PrintResult(const TestResult& r, std::string_view runnerName) {
        Progress::Clear();
        auto ms = r.duration.count();
        std::string runnerSuffix = (runnerName.empty() || runnerName == "local")
            ? std::string()
            : std::format(" ({})", runnerName);
        switch (r.outcome) {
            case TestOutcome::Pass:
                std::println("✅ {}{} ({}ms)", r.name, runnerSuffix, ms);
                break;
            case TestOutcome::Fail:
                std::println("❌ {}{} ({}ms) exit {}", r.name, runnerSuffix, ms, r.exitCode);
                if (!r.output.empty()) {
                    for (auto line : std::views::split(r.output, '\n')) {
                        std::string_view sv(line.begin(), line.end());
                        if (!sv.empty()) std::println("    {}", sv);
                    }
                }
                break;
            case TestOutcome::Crash:
                std::println("\U0001F4A5 {}{} ({}ms) crashed: {}", r.name, runnerSuffix, ms, SignalName(r.signal));
                if (!r.output.empty()) {
                    for (auto line : std::views::split(r.output, '\n')) {
                        std::string_view sv(line.begin(), line.end());
                        if (!sv.empty()) std::println("    {}", sv);
                    }
                }
                break;
            case TestOutcome::Timeout:
                std::println("⏱  {}{} ({}ms) timeout", r.name, runnerSuffix, ms);
                break;
            case TestOutcome::Skipped:
                std::println("⏭  {}{} skipped: {}", r.name, runnerSuffix, r.output.empty() ? std::string("(no reason)") : r.output);
                break;
        }
    }
}

namespace {
    std::atomic<Configuration*> ParentProject{nullptr};

    Configuration* FindLibInTree(Configuration* root, std::string_view name, std::unordered_set<Configuration*>& seen) {
        if (!seen.insert(root).second) return nullptr;
        if (root->name == name) return root;
        for (Configuration* dep : root->dependencies) {
            if (auto found = FindLibInTree(dep, name, seen)) return found;
        }
        return nullptr;
    }
}

void Crafter::SetParentProject(Configuration* parent) {
    ParentProject.store(parent);
}

Configuration* Crafter::ParentLib(std::string_view name) {
    Configuration* root = ParentProject.load();
    if (!root) {
        throw std::runtime_error(std::format("Crafter::ParentLib('{}'): no parent project set", name));
    }
    std::unordered_set<Configuration*> seen;
    if (auto found = FindLibInTree(root, name, seen)) return found;
    throw std::runtime_error(std::format("Crafter::ParentLib('{}'): not found in parent project '{}'", name, root->name));
}

TestRunner TestRunner::Local() {
    TestRunner r;
    r.name = "local";
    return r;
}

TestRunner TestRunner::Cmd(std::string command) {
    TestRunner r;
    r.name = std::format("cmd:{}", command);
    r.exec = std::format("{} {{bin}} {{args}}", command);
    r.probe = std::format("which {}", command);
    return r;
}

TestRunner TestRunner::Wine() {
    TestRunner r;
    r.name = "wine";
    r.exec = "wine {bin} {args}";
    r.probe = "which wine";
    return r;
}

TestRunner TestRunner::ForTarget(const Configuration& cfg) {
    const std::string& target = cfg.target;

    // Same triple as the host → run the binary directly. Covers the common
    // case (cfg.target defaulted to HostTarget()) without any wrapper.
    if (SameTarget(target, HostTarget())) return Local();

    // Windows targets run under Wine; there is no Windows host.
    if (TargetIsWindows(target)) return Wine();

    // WASI: a .wasm file isn't directly executable; wasmtime is the canonical
    // runtime. wasi-cli also works but the upstream Bytecode Alliance name is
    // wasmtime, so we standardize on that.
    if (target.starts_with("wasm32-wasi") || target.starts_with("wasm64-wasi")) {
        return Cmd("wasmtime");
    }

    // Non-host Linux triple: extract the architecture and route through
    // qemu-user. Triple is <arch>-<vendor>-<os>-<env> (or sometimes 3 parts);
    // qemu-user's binary names mostly follow the arch field, with two known
    // mismatches handled below. cfg.sysroot, when set, becomes QEMU_LD_PREFIX
    // so the target's dynamic linker / shared libs are reachable — without
    // it qemu-user crashes on dynamic ELFs with "could not open /lib/ld...".
    if (target.find("-linux-") != std::string::npos) {
        auto dash = target.find('-');
        std::string arch = target.substr(0, dash);
        // i686-linux-gnu → qemu-i386; arm-* already matches qemu-arm; aarch64,
        // riscv64, ppc64le, mips, mips64, s390x all match their qemu names.
        if (arch == "i686") arch = "i386";
        TestRunner r = Cmd(std::format("qemu-{}", arch));
        if (!cfg.sysroot.empty()) {
            // Use `env VAR=value cmd` rather than the shell's `VAR=value cmd`
            // prefix syntax: RunCommandWithTimeout pipes through GNU `timeout`,
            // which execvp's its argument list directly without going through a
            // shell. A bare VAR=value would be exec'd as a command path and
            // fail with "No such file or directory".
            // QEMU_LD_PREFIX only redirects the ELF interpreter lookup; the
            // emulated loader then searches ITS default paths against the
            // host filesystem, picking up host-arch libraries from /lib —
            // LD_LIBRARY_PATH steers it back into the sysroot.
            r.exec = std::format("env QEMU_LD_PREFIX={0} LD_LIBRARY_PATH={0}/lib:{0}/usr/lib {1}", cfg.sysroot, r.exec);
        }
        return r;
    }

    // Unknown / bare-metal / freestanding targets: fall back to Local. The
    // caller's runner-availability probe (or absence of the binary) surfaces
    // the problem rather than us inventing a wrong wrapper here.
    return Local();
}

namespace {
    std::string NormalizeTriple(std::string_view target) {
        std::string out(target);
        for (char& c : out) {
            if (c == '-' || c == '.') c = '_';
        }
        return out;
    }

    // Spec grammar: "local" | "cmd:<binary>". Used by CRAFTER_BUILD_RUNNER_*
    // env vars and --runner=. The set used to include ssh/sshwin/wsl variants;
    // those were removed when transport-style runners were dropped (issue #8).
    std::optional<TestRunner> ParseRunnerSpec(std::string_view spec) {
        if (spec.empty()) return std::nullopt;
        if (spec == "local") return TestRunner::Local();
        if (spec.starts_with("cmd:") && spec.size() > 4) {
            return TestRunner::Cmd(std::string(spec.substr(4)));
        }
        throw std::runtime_error(std::format("TestRunner::FromSpec: unrecognized runner spec '{}' " "(expected 'local' or 'cmd:<binary>')", spec));
    }
}

std::optional<TestRunner> TestRunner::FromSpec(std::string_view spec) {
    return ParseRunnerSpec(spec);
}

TestRunner TestRunner::FromEnv(std::string_view target, TestRunner fallback) {
    std::string envName = std::format("CRAFTER_BUILD_RUNNER_{}", NormalizeTriple(target));
    const char* v = std::getenv(envName.c_str());
    if (!v || !*v) return fallback;
    if (auto r = ParseRunnerSpec(v)) return std::move(*r);
    return fallback;
}

TestResult Crafter::RunSingleTest(const Test& test, const fs::path& binary, std::chrono::seconds timeout) {
    TestResult result;
    result.name = test.config.name;

    std::map<std::string, std::string> ph;
    ph["{args}"] = JoinAndQuoteArgs(test.args);

    auto start = std::chrono::steady_clock::now();
    CommandResult r;

    if (test.runner.exec.empty()) {
        // Local runner: spawn the binary directly through the host shell.
        std::string cmd = std::format("{} {}", ShellQuote(binary.string()), ph["{args}"]);
        r = RunCommandWithTimeout(cmd, timeout);
    } else {
        // Prefix runner (qemu-user, wasmtime, wine, ...): templated exec
        // wraps the local binary.
        ph["{bin}"] = binary.string();
        r = RunCommandWithTimeout(Substitute(test.runner.exec, ph), timeout);
    }

    auto end = std::chrono::steady_clock::now();
    result.duration = std::chrono::duration_cast<std::chrono::milliseconds>(end - start);
    result.output = std::move(r.output);
    result.exitCode = r.exitCode;
    result.signal = r.signal;

    if (r.timedOut) result.outcome = TestOutcome::Timeout;
    else if (r.crashed) result.outcome = TestOutcome::Crash;
    else if (r.exitCode == 77) result.outcome = TestOutcome::Skipped;
    else if (r.exitCode != 0) result.outcome = TestOutcome::Fail;
    else result.outcome = TestOutcome::Pass;

    return result;
}

namespace {
    bool ToolOnPath(std::string_view name) {
        std::string cmd = std::format("which {} > /dev/null 2>&1", name);
        return std::system(cmd.c_str()) == 0;
    }

    struct RequireResult {
        bool ok;
        std::string reason;  // human-readable when !ok
    };

    // Evaluate each `<kind>:<arg>` precondition. Returns the first failure
    // (short-circuit; reporting one missing dep at a time is enough to act on
    // and keeps the test log uncluttered).
    RequireResult EvaluateRequires(std::span<const std::string> reqs) {
        for (const auto& r : reqs) {
            auto sep = r.find(':');
            if (sep == std::string::npos || sep == 0 || sep == r.size() - 1) {
                return {false, std::format("malformed require '{}' (expected kind:arg)", r)};
            }
            std::string_view kind(r.data(), sep);
            std::string_view arg(r.data() + sep + 1, r.size() - sep - 1);
            if (kind == "tool") {
                if (!ToolOnPath(arg)) {
                    return {false, std::format("tool '{}' not on PATH", arg)};
                }
            } else if (kind == "file") {
                if (!fs::exists(std::string(arg))) {
                    return {false, std::format("file '{}' missing", arg)};
                }
            } else if (kind == "env") {
                const char* v = std::getenv(std::string(arg).c_str());
                if (!v || !*v) {
                    return {false, std::format("env '{}' unset", arg)};
                }
            } else {
                return {false, std::format("unknown require kind '{}' (expected tool/file/env)", kind)};
            }
        }
        return {true, ""};
    }

    // Match a runner's tool dependency against the test's declared
    // requirements. Used to decide between Skip (declared, may legitimately
    // be missing) and Fail (runner unavailable but test didn't declare it —
    // a silent skip would mask broken cross-arch CI configuration).
    bool RequiresMentionsTool(std::span<const std::string> reqs, std::string_view tool) {
        std::string needle = std::format("tool:{}", tool);
        return std::ranges::any_of(reqs, [&](const std::string& s) { return s == needle; });
    }

    // Best-effort extraction of the runner-tool name from a TestRunner so the
    // hard-fail-unless-declared check can match it against `requires`. For
    // Cmd("foo"), the name is "cmd:foo"; for Wine, it's "wine". Anything else
    // (Local, transport runners) returns empty — those don't trigger the
    // declared/undeclared gate.
    std::string RunnerToolName(const TestRunner& runner) {
        if (runner.name == "wine") return "wine";
        if (runner.name.starts_with("cmd:")) {
            std::string tool = runner.name.substr(4);
            // QEMU_LD_PREFIX prefix may be glued onto exec but the runner's
            // `name` field already isolates the command, so no extra parsing.
            return tool;
        }
        return "";
    }
}

TestBuilder Configuration::AddTest(std::string_view name) {
    // Empty-interfaces trampoline. The span we hand to the (name, interfaces)
    // overload wraps a stack array; it's only consulted inside that call, so
    // its lifetime is fine for the duration.
    std::array<fs::path, 0> noIfaces = {};
    return AddTest(name, noIfaces);
}

TestBuilder Configuration::AddTest(std::string_view name, std::span<fs::path> interfaces) {
    Test t;
    // Default to "./" so tests/<name>/main.cpp resolves at the project root
    // (where crafter-build sets cwd from project.cpp's location). Projects
    // whose lib lives in a subdir (e.g. cfg.path = "./mylib/") get this
    // right by default; if the test layout is unusual, override via .Path().
    t.config.path = "./";
    t.config.name = std::string(name);
    t.config.outputName = std::string(name);
    t.config.target = this->target;
    t.config.march = this->march;
    t.config.mtune = this->mtune;
    t.config.sysroot = this->sysroot;
    t.config.debug = this->debug;
    // Inherited so a project flag that changes what the library contains also
    // moves the test's own outputs, rather than letting two flag settings share
    // one test bin dir.
    t.config.projectArgs = this->projectArgs;
    t.config.type = ConfigurationType::Executable;

    // Default source layout: tests/<name>/main.cpp resolved against the
    // parent Configuration's path. cfg.path is typically "./" (project
    // root), which puts the test sources at <repo>/tests/<name>/main.cpp.
    fs::path mainSource = fs::path("tests") / std::string(name) / "main";
    std::array<fs::path, 1> impls = { mainSource };
    t.config.GetInterfacesAndImplementations(interfaces, impls);

    tests.push_back(std::move(t));
    return TestBuilder{this, tests.size() - 1};
}

void Configuration::AddMarchVariants(std::string_view name, std::span<fs::path> interfaces, std::span<const MarchTier> tiers) {
    for (const auto& tier : tiers) {
        Test t;
        t.config.path = "./";
        t.config.name = std::format("{}-{}", name, tier.march);
        t.config.outputName = t.config.name;
        t.config.target = this->target;
        t.config.march = tier.march;
        t.config.mtune = tier.mtune;
        t.config.sysroot = this->sysroot;
        t.config.debug = this->debug;
        t.config.projectArgs = this->projectArgs;
        t.config.type = ConfigurationType::Executable;

        fs::path mainSource = fs::path("tests") / std::string(name) / "main";
        std::array<fs::path, 1> impls = { mainSource };
        // Interfaces stamped directly into each variant's Configuration —
        // NOT via cfg.dependencies. The dependency machinery caches PCMs by
        // dep->VariantId(), which would compile the parent's interfaces
        // once with the parent's march; for SIMD-sensitive code each
        // variant needs its own compile. Listing the interfaces as the
        // test's own forces per-variant recompilation.
        t.config.GetInterfacesAndImplementations(interfaces, impls);
        tests.push_back(std::move(t));
    }
}

TestBuilder& TestBuilder::Path(fs::path p)                  { Ref().config.path = std::move(p); return *this; }
TestBuilder& TestBuilder::Target(std::string t)             { Ref().config.target = std::move(t); return *this; }
TestBuilder& TestBuilder::March(std::string m)              { Ref().config.march = std::move(m); return *this; }
TestBuilder& TestBuilder::Mtune(std::string m)              { Ref().config.mtune = std::move(m); return *this; }
TestBuilder& TestBuilder::Sysroot(fs::path s)               { Ref().config.sysroot = s.string(); return *this; }
TestBuilder& TestBuilder::Debug(bool d)                     { Ref().config.debug = d; return *this; }
TestBuilder& TestBuilder::Define(std::string n, std::string v) {
    Ref().config.defines.push_back({std::move(n), std::move(v)});
    return *this;
}
TestBuilder& TestBuilder::Timeout(std::chrono::seconds s)   { Ref().timeout = s; return *this; }
TestBuilder& TestBuilder::Args(std::vector<std::string> a)  { Ref().args = std::move(a); return *this; }
TestBuilder& TestBuilder::Requires(std::string r)           { Ref().requires_.push_back(std::move(r)); return *this; }
TestBuilder& TestBuilder::Dependencies(std::vector<Configuration*> d) {
    Ref().config.dependencies = std::move(d);
    // AddTest already scanned tests/<name>/main.cpp, at which point this test
    // had no dependencies, so every `import <DepModule>;` in it came back
    // unresolved. Place them now that the libraries are known — without this
    // the test's object carries no staleness edge to the interfaces it consumes
    // and survives a layout change to them (issue #27). Build() re-runs the
    // same sweep as a backstop; doing it here keeps the Configuration coherent
    // for anyone inspecting it before the build.
    Ref().config.ResolvePendingImports();
    return *this;
}
TestBuilder& TestBuilder::LinkFlag(std::string f)           { Ref().config.linkFlags.push_back(std::move(f)); return *this; }
TestBuilder& TestBuilder::CompileFlag(std::string f)        { Ref().config.compileFlags.push_back(std::move(f)); return *this; }

TestSummary Crafter::RunTests(Configuration& projectCfg, const RunTestsOptions& opts, std::span<const std::string_view> projectArgs) {
    // Multi-target sweep: when no --target= was given, the run covers every
    // distinct target declared across projectCfg.tests plus the host target.
    // Lets a bare `crafter-build test` exercise cross-arch tests without the
    // user having to know which targets exist. An explicit --target=X
    // bypasses the sweep and runs that target only.
    if (opts.targetFilter.empty()) {
        // Keyed by the normalized triple so x86_64-pc-linux-gnu and
        // x86_64-unknown-linux-gnu make one pass; the first spelling seen is
        // the one shown and filtered on.
        std::map<std::string, std::string> sweep;
        sweep.emplace(NormalizeTarget(HostTarget()), HostTarget());
        for (const Test& t : projectCfg.tests) {
            if (!t.config.target.empty()) sweep.emplace(NormalizeTarget(t.config.target), t.config.target);
        }
        TestSummary aggregate;
        for (const auto& [normalized, target] : sweep) {
            RunTestsOptions perTarget = opts;
            perTarget.targetFilter = target;
            if (sweep.size() > 1) {
                Progress::Clear();
                std::println("\n=== target: {} ===", target);
            }
            TestSummary s = RunTests(projectCfg, perTarget, projectArgs);
            aggregate.passed   += s.passed;
            aggregate.failed   += s.failed;
            aggregate.crashed  += s.crashed;
            aggregate.timedOut += s.timedOut;
            aggregate.skipped  += s.skipped;
            for (auto& r : s.results) aggregate.results.push_back(std::move(r));
        }
        return aggregate;
    }

    TestSummary summary;

    // Filter by target + glob, derive each Test's runner just-in-time. The
    // runner is recomputed each sweep iteration (rather than once at AddTest
    // time) because runnerOverride and FromEnv resolution depend on the
    // run's options, which the project.cpp doesn't know about.
    std::vector<Test*> filtered;
    filtered.reserve(projectCfg.tests.size());
    for (auto& test : projectCfg.tests) {
        if (!SameTarget(test.config.target, opts.targetFilter)) continue;
        if (!MatchAny(opts.globs, test.config.name)) continue;
        test.runner = TestRunner::FromEnv(test.config.target, TestRunner::ForTarget(test.config));
        if (opts.runnerOverride) {
            if (auto r = TestRunner::FromSpec(*opts.runnerOverride)) {
                test.runner = std::move(*r);
            }
        }
        filtered.push_back(&test);
    }

    if (opts.listOnly) {
        for (auto* t : filtered) {
            std::println("{}", t->config.name);
        }
        return summary;
    }

    if (filtered.empty()) {
        std::println("No tests matched.");
        return summary;
    }

    std::int32_t jobs = opts.jobs > 0
        ? opts.jobs
        : std::max(1u, std::thread::hardware_concurrency());
    jobs = std::min(jobs, static_cast<std::int32_t>(filtered.size()));

    std::unordered_map<fs::path, std::shared_future<BuildResult>> depResults;
    std::mutex depMutex;
    std::mutex printMutex;
    std::mutex probeMutex;
    std::unordered_map<std::string, bool> probeCache;
    std::atomic<std::size_t> next{0};
    std::vector<TestResult> results(filtered.size());

    auto runnerAvailable = [&](const TestRunner& runner) -> bool {
        if (runner.probe.empty()) return true;
        std::lock_guard lk(probeMutex);
        if (auto it = probeCache.find(runner.name); it != probeCache.end()) {
            return it->second;
        }
        // RunCommandChecked captures and discards output internally, so probe
        // specs don't need `> /dev/null 2>&1` (which doesn't translate to cmd).
        bool ok = (RunCommandChecked(runner.probe).exitCode == 0);
        probeCache[runner.name] = ok;
        return ok;
    };

    auto worker = [&]() {
        while (true) {
            std::size_t i = next.fetch_add(1);
            if (i >= filtered.size()) break;
            Test& t = *filtered[i];
            TestResult r;
            r.name = t.config.name;

            // Declarative preconditions set via TestBuilder::Requires. Evaluated
            // before the build so a missing tool/file/env turns into a Skip without
            // paying the compile cost. Reports the first failure only — once one
            // precondition is unmet the test couldn't run anyway, and a wall of
            // "also missing X, also missing Y" buries the actionable root cause.
            if (auto req = EvaluateRequires(t.requires_); !req.ok) {
                r.outcome = TestOutcome::Skipped;
                r.output = req.reason;
                {
                    std::lock_guard lk(printMutex);
                    PrintResult(r, t.runner.name);
                }
                results[i] = std::move(r);
                continue;
            }

            if (!runnerAvailable(t.runner)) {
                // Hard-fail-unless-declared: if the runner depends on a tool
                // (qemu-aarch64, wasmtime, wine, ...) and the test didn't say
                // "tool:<that>" in requires, the missing runner is a Fail. The
                // intent is to surface broken cross-arch CI configuration
                // instead of letting it masquerade as a Skip; tests that
                // legitimately may run without their runner have to opt in.
                std::string tool = RunnerToolName(t.runner);
                if (!tool.empty() && !RequiresMentionsTool(t.requires_, tool)) {
                    r.outcome = TestOutcome::Fail;
                    r.exitCode = -1;
                    r.output = std::format("runner '{}' unavailable and not declared in requires " "(add .Requires(\"tool:{}\") to permit skipping)", t.runner.name, tool);
                } else {
                    r.outcome = TestOutcome::Skipped;
                    r.output = std::format("runner '{}' not available", t.runner.name);
                }
                {
                    std::lock_guard lk(printMutex);
                    PrintResult(r, t.runner.name);
                }
                results[i] = std::move(r);
                continue;
            }

            BuildResult br;
            try {
                br = Build(t.config, depResults, depMutex);
            } catch (const std::exception& e) {
                r.outcome = TestOutcome::Fail;
                r.output = std::format("build threw: {}", e.what());
                r.exitCode = -1;
                {
                    std::lock_guard lk(printMutex);
                    PrintResult(r, t.runner.name);
                }
                results[i] = std::move(r);
                continue;
            }

            if (!br.result.empty()) {
                r.outcome = TestOutcome::Fail;
                r.output = std::format("build failed: {}", br.result);
                r.exitCode = -1;
                {
                    std::lock_guard lk(printMutex);
                    PrintResult(r, t.runner.name);
                }
                results[i] = std::move(r);
                continue;
            }

            std::chrono::seconds timeout = opts.timeoutOverride.value_or(t.timeout);
            fs::path binary = TestBinaryPath(t.config);
            try {
                r = RunSingleTest(t, binary, timeout);
            } catch (const std::exception& e) {
                r.outcome = TestOutcome::Fail;
                r.output = std::format("runner threw: {}", e.what());
                r.exitCode = -1;
            }

            if (r.outcome != TestOutcome::Pass && r.outcome != TestOutcome::Skipped && !r.output.empty()) {
                WriteLog(projectCfg.path, r.name, r.output);
            }

            {
                std::lock_guard lk(printMutex);
                PrintResult(r, t.runner.name);
            }
            results[i] = std::move(r);
        }
    };

    std::vector<std::jthread> threads;
    threads.reserve(jobs);
    for (std::int32_t j = 0; j < jobs; ++j) {
        threads.emplace_back(worker);
    }
    threads.clear(); // joins all jthreads

    for (auto& r : results) {
        switch (r.outcome) {
            case TestOutcome::Pass:    summary.passed++; break;
            case TestOutcome::Fail:    summary.failed++; break;
            case TestOutcome::Crash:   summary.crashed++; break;
            case TestOutcome::Timeout: summary.timedOut++; break;
            case TestOutcome::Skipped: summary.skipped++; break;
        }
    }
    summary.results = std::move(results);

    Progress::Clear();
    std::print("\n");
    std::vector<std::string> parts;
    if (summary.passed)   parts.push_back(std::format("{} passed",  summary.passed));
    if (summary.failed)   parts.push_back(std::format("{} failed",  summary.failed));
    if (summary.crashed)  parts.push_back(std::format("{} crashed", summary.crashed));
    if (summary.timedOut) parts.push_back(std::format("{} timed out", summary.timedOut));
    if (summary.skipped)  parts.push_back(std::format("{} skipped", summary.skipped));
    std::string joined;
    for (std::size_t i = 0; i < parts.size(); ++i) {
        if (i) joined += ", ";
        joined += parts[i];
    }
    std::println("{}", joined);

    return summary;
}