Crafter.Build/implementations/Crafter.Build-Test.cpp

/*
Crafter® Build
Copyright (C) 2026 Catcrafts®
Catcrafts.net

This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License version 3.0 as published by the Free Software Foundation;

This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.

You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA
*/

export module Crafter.Build:Test_impl;
import std;
import :Test;
import :Clang;
import :Platform;
namespace fs = std::filesystem;
using namespace Crafter;

namespace {
    bool TargetIsWindows(std::string_view target) {
        return target.find("windows") != std::string_view::npos
            || target.find("mingw") != std::string_view::npos;
    }

    fs::path TestBinaryPath(const Configuration& cfg) {
        fs::path outputDir = cfg.path/"bin"/std::format("{}-{}-{}", cfg.name, cfg.target, cfg.march);
        return outputDir / (TargetIsWindows(cfg.target) ? cfg.outputName + ".exe" : cfg.outputName);
    }

    bool MatchGlob(std::string_view glob, std::string_view name) {
        std::size_t gi = 0, ni = 0, star = std::string_view::npos, mark = 0;
        while (ni < name.size()) {
            if (gi < glob.size() && (glob[gi] == '?' || glob[gi] == name[ni])) {
                ++gi; ++ni;
            } else if (gi < glob.size() && glob[gi] == '*') {
                star = gi++;
                mark = ni;
            } else if (star != std::string_view::npos) {
                gi = star + 1;
                ni = ++mark;
            } else {
                return false;
            }
        }
        while (gi < glob.size() && glob[gi] == '*') ++gi;
        return gi == glob.size();
    }

    bool MatchAny(std::span<const std::string> globs, std::string_view name) {
        if (globs.empty()) return true;
        for (const auto& g : globs) {
            if (MatchGlob(g, name)) return true;
        }
        return false;
    }

    std::string ShellQuoteSh(std::string_view s) {
        std::string out;
        out.reserve(s.size() + 2);
        out.push_back('\'');
        for (char c : s) {
            if (c == '\'') out += "'\\''";
            else out.push_back(c);
        }
        out.push_back('\'');
        return out;
    }

    // cmd.exe doesn't recognize '...' as quoting (it would pass the single
    // quotes through to the executable). Wrap in "..." for cmd; embedded "
    // is rare in paths but escape it to be safe. Backslash sequences before
    // " don't need the MS-CRT doubling rules because we go through
    // `cmd /C "..."`, which uses cmd's parser, not the CRT's argv splitter.
    std::string ShellQuoteCmd(std::string_view s) {
        std::string out;
        out.reserve(s.size() + 2);
        out.push_back('"');
        for (char c : s) {
            if (c == '"') out += "\\\"";
            else out.push_back(c);
        }
        out.push_back('"');
        return out;
    }

    // Host-shell quoting: sh on Linux, cmd on Windows. For args/paths that
    // hit the local shell (Local runner exec, Cmd-prefix runner exec).
    std::string ShellQuoteHost(std::string_view s) {
#ifdef _WIN32
        return ShellQuoteCmd(s);
#else
        return ShellQuoteSh(s);
#endif
    }

    std::string JoinAndQuoteArgs(std::span<const std::string> args, TestRunner::Shell shell) {
        std::string out;
        for (const auto& a : args) {
            if (!out.empty()) out.push_back(' ');
            switch (shell) {
                case TestRunner::Shell::Sh:   out += ShellQuoteSh(a);   break;
                case TestRunner::Shell::Cmd:  out += ShellQuoteCmd(a);  break;
                case TestRunner::Shell::Host: out += ShellQuoteHost(a); break;
            }
        }
        return out;
    }

    std::string Substitute(std::string_view tmpl, const std::map<std::string, std::string>& ph) {
        std::string out;
        out.reserve(tmpl.size());
        std::size_t i = 0;
        while (i < tmpl.size()) {
            if (tmpl[i] == '{') {
                std::size_t end = tmpl.find('}', i + 1);
                if (end != std::string_view::npos) {
                    std::string key(tmpl.substr(i, end - i + 1));
                    if (auto it = ph.find(key); it != ph.end()) {
                        out += it->second;
                        i = end + 1;
                        continue;
                    }
                }
            }
            out.push_back(tmpl[i++]);
        }
        return out;
    }

    std::string SignalName(int sig) {
        switch (sig) {
            case 1: return "SIGHUP";
            case 2: return "SIGINT";
            case 4: return "SIGILL";
            case 6: return "SIGABRT";
            case 8: return "SIGFPE";
            case 9: return "SIGKILL";
            case 11: return "SIGSEGV";
            case 13: return "SIGPIPE";
            case 14: return "SIGALRM";
            case 15: return "SIGTERM";
            default: return std::format("signal {}", sig);
        }
    }

    void WriteLog(const fs::path& projectPath, const std::string& name, const std::string& output) {
        fs::path logDir = projectPath / "build" / "test-logs";
        std::error_code ec;
        fs::create_directories(logDir, ec);
        if (ec) return;
        std::ofstream(logDir / (name + ".log")) << output;
    }

    void PrintResult(const TestResult& r, std::string_view runnerName) {
        auto ms = r.duration.count();
        std::string runnerSuffix = (runnerName.empty() || runnerName == "local")
            ? std::string()
            : std::format(" ({})", runnerName);
        switch (r.outcome) {
            case TestOutcome::Pass:
                std::println("✅ {}{} ({}ms)", r.name, runnerSuffix, ms);
                break;
            case TestOutcome::Fail:
                std::println("❌ {}{} ({}ms) exit {}", r.name, runnerSuffix, ms, r.exitCode);
                if (!r.output.empty()) {
                    for (auto line : std::views::split(r.output, '\n')) {
                        std::string_view sv(line.begin(), line.end());
                        if (!sv.empty()) std::println("    {}", sv);
                    }
                }
                break;
            case TestOutcome::Crash:
                std::println("\U0001F4A5 {}{} ({}ms) crashed: {}", r.name, runnerSuffix, ms, SignalName(r.signal));
                if (!r.output.empty()) {
                    for (auto line : std::views::split(r.output, '\n')) {
                        std::string_view sv(line.begin(), line.end());
                        if (!sv.empty()) std::println("    {}", sv);
                    }
                }
                break;
            case TestOutcome::Timeout:
                std::println("⏱  {}{} ({}ms) timeout", r.name, runnerSuffix, ms);
                break;
            case TestOutcome::Skipped:
                std::println("⏭  {}{} skipped: {}", r.name, runnerSuffix,
                    r.output.empty() ? std::string("(no reason)") : r.output);
                break;
        }
    }
}

namespace {
    std::atomic<Configuration*> g_parentProject{nullptr};

    Configuration* FindLibInTree(Configuration* root,
                                  std::string_view name,
                                  std::unordered_set<Configuration*>& seen) {
        if (!seen.insert(root).second) return nullptr;
        if (root->name == name) return root;
        for (Configuration* dep : root->dependencies) {
            if (auto found = FindLibInTree(dep, name, seen)) return found;
        }
        return nullptr;
    }
}

void Crafter::SetParentProject(Configuration* parent) {
    g_parentProject.store(parent);
}

Configuration* Crafter::ParentLib(std::string_view name) {
    Configuration* root = g_parentProject.load();
    if (!root) {
        throw std::runtime_error(std::format(
            "Crafter::ParentLib('{}'): no parent project set", name));
    }
    std::unordered_set<Configuration*> seen;
    if (auto found = FindLibInTree(root, name, seen)) return found;
    throw std::runtime_error(std::format(
        "Crafter::ParentLib('{}'): not found in parent project '{}'",
        name, root->name));
}

TestRunner TestRunner::Local() {
    TestRunner r;
    r.name = "local";
    r.argsShell = Shell::Host;
    return r;
}

TestRunner TestRunner::Ssh(std::string host, std::string remoteDir) {
    TestRunner r;
    r.name = std::format("ssh:{}", host);
    r.remoteDir = std::move(remoteDir);
    r.argsShell = Shell::Sh;
    // Outer "..." (not '...') so the wrapper survives both sh (Linux host)
    // and cmd (Windows host). cmd doesn't honor single quotes, which would
    // cause it to split on the inner '&&'. Args inside use POSIX quoting
    // because they reach a remote bash regardless of which host issued them.
    r.copy    = std::format("ssh -q {0} \"mkdir -p {{remote_bundle}}\" && scp -r -q {{bundle}}/. {0}:{{remote_bundle}}/", host);
    r.exec    = std::format("ssh -q {} \"cd {{remote_bundle}} && ./{{bin_name}} {{args}}\"", host);
    r.cleanup = std::format("ssh -q {} \"rm -rf {{remote_bundle}}\"", host);
    // RunCommandChecked captures stdout+stderr internally — no shell redirect
    // needed in the probe spec, which means it works the same way under cmd
    // (Windows host) and sh (Linux host) since `> /dev/null` doesn't translate.
    r.probe   = std::format("ssh -q -o BatchMode=yes -o ConnectTimeout=5 {} true", host);
    return r;
}

TestRunner TestRunner::SshWin(std::string host, std::string remoteDir) {
    TestRunner r;
    r.name = std::format("sshwin:{}", host);
    r.remoteDir = std::move(remoteDir);
    r.argsShell = Shell::Cmd;
    // cmd.exe-friendly templates. Use backslash variants of remote paths because
    // cmd's mkdir won't auto-create intermediate directories with forward
    // slashes. scp tolerates either, so we keep forward slashes for the scp
    // dest (which `{remote_bundle}` provides). 2>nul + `& rem ok` swallows
    // mkdir's "already exists" error and forces exit code 0.
    r.copy    = std::format("ssh -q {0} \"mkdir {{remote_bundle_win}} 2>nul & rem ok\" && scp -r -q {{bundle}}/. {0}:{{remote_bundle}}/", host);
    r.exec    = std::format("ssh -q {} \"{{bin_win}} {{args}}\"", host);
    r.cleanup = std::format("ssh -q {} \"rd /s /q {{remote_bundle_win}} 2>nul & rem ok\"", host);
    // No shell redirect in the probe — see TestRunner::Ssh for rationale.
    r.probe   = std::format("ssh -q -o BatchMode=yes -o ConnectTimeout=5 {} \"ver\"", host);
    return r;
}

TestRunner TestRunner::Wsl(std::string remoteDir) {
    TestRunner r;
    r.name = "wsl";
    r.argsShell = Shell::Sh;
    r.remoteDir = std::move(remoteDir);
    // Transport runner: stage the test bundle into WSL's native filesystem
    // (faster than executing in-place from /mnt/c) then run via bash. {bundle_wsl}
    // is the local Windows path translated to /mnt/<drive>/... form so wsl cp
    // can read it. Args are POSIX-quoted because they reach a Linux shell.
    r.copy    = "wsl mkdir -p {remote_bundle} && wsl cp -r {bundle_wsl}/. {remote_bundle}/";
    r.exec    = "wsl bash -c \"cd {remote_bundle} && ./{bin_name} {args}\"";
    r.cleanup = "wsl rm -rf {remote_bundle}";
    // `where wsl` matches the host-shell convention; on a non-Windows host
    // it fails (no `where` command), which correctly skips this runner.
    r.probe   = "where wsl";
    return r;
}

TestRunner TestRunner::Cmd(std::string command) {
    TestRunner r;
    r.name = std::format("cmd:{}", command);
    r.argsShell = Shell::Host;
    r.exec = std::format("{} {{bin}} {{args}}", command);
#ifdef _WIN32
    r.probe = std::format("where {}", command);
#else
    r.probe = std::format("which {}", command);
#endif
    return r;
}

namespace {
    std::string NormalizeTriple(std::string_view target) {
        std::string out(target);
        for (char& c : out) {
            if (c == '-' || c == '.') c = '_';
        }
        return out;
    }

    std::optional<TestRunner> ParseRunnerSpec(std::string_view spec) {
        if (spec.empty()) return std::nullopt;
        std::vector<std::string> parts;
        for (auto piece : std::views::split(spec, ':')) {
            parts.emplace_back(std::string_view(piece.begin(), piece.end()));
        }
        if (parts.empty()) return std::nullopt;
        if (parts[0] == "local") return TestRunner::Local();
        if (parts[0] == "cmd" && parts.size() == 2) return TestRunner::Cmd(parts[1]);
        if (parts[0] == "ssh" && parts.size() == 2) return TestRunner::Ssh(parts[1]);
        if (parts[0] == "ssh" && parts.size() >= 3) {
            std::string remote;
            for (std::size_t i = 2; i < parts.size(); ++i) {
                if (i > 2) remote.push_back(':');
                remote += parts[i];
            }
            return TestRunner::Ssh(parts[1], remote);
        }
        if (parts[0] == "sshwin" && parts.size() == 2) return TestRunner::SshWin(parts[1]);
        if (parts[0] == "sshwin" && parts.size() >= 3) {
            std::string remote;
            for (std::size_t i = 2; i < parts.size(); ++i) {
                if (i > 2) remote.push_back(':');
                remote += parts[i];
            }
            return TestRunner::SshWin(parts[1], remote);
        }
        if (parts[0] == "wsl" && parts.size() == 1) return TestRunner::Wsl();
        if (parts[0] == "wsl" && parts.size() >= 2) {
            std::string remote;
            for (std::size_t i = 1; i < parts.size(); ++i) {
                if (i > 1) remote.push_back(':');
                remote += parts[i];
            }
            return TestRunner::Wsl(remote);
        }
        throw std::runtime_error(std::format(
            "TestRunner::FromSpec: unrecognized runner spec '{}'", spec));
    }

    // C:\Users\jorij -> /mnt/c/Users/jorij. Idempotent on paths that don't
    // start with a drive letter, so callers can compute it unconditionally.
    std::string WindowsPathToWsl(std::string_view p) {
        if (p.size() < 2 || p[1] != ':') return std::string(p);
        char drive = static_cast<char>(std::tolower(static_cast<unsigned char>(p[0])));
        std::string out = std::format("/mnt/{}", drive);
        for (std::size_t i = 2; i < p.size(); ++i) {
            out.push_back(p[i] == '\\' ? '/' : p[i]);
        }
        return out;
    }
}

std::optional<TestRunner> TestRunner::FromSpec(std::string_view spec) {
    return ParseRunnerSpec(spec);
}

TestRunner TestRunner::FromEnv(std::string_view target, TestRunner fallback) {
    std::string envName = std::format("CRAFTER_BUILD_RUNNER_{}", NormalizeTriple(target));
    const char* v = std::getenv(envName.c_str());
    if (!v || !*v) return fallback;
    if (auto r = ParseRunnerSpec(v)) return std::move(*r);
    return fallback;
}

TestResult Crafter::RunSingleTest(const Test& test, const fs::path& binary, std::chrono::seconds timeout) {
    using namespace std::chrono_literals;
    TestResult result;
    result.name = test.config.name;

    std::map<std::string, std::string> ph;
    ph["{args}"] = JoinAndQuoteArgs(test.args, test.runner.argsShell);
    ph["{bin_name}"] = binary.filename().string();
    ph["{bundle}"] = binary.parent_path().string();

    auto start = std::chrono::steady_clock::now();
    CommandResult r;

    if (test.runner.exec.empty()) {
        // Pure-local runner: spawn the binary directly through the host shell.
        std::string cmd = std::format("{} {}", ShellQuoteHost(binary.string()), ph["{args}"]);
        r = RunCommandWithTimeout(cmd, timeout);
    } else if (test.runner.copy.empty()) {
        // Prefix runner (qemu-user, wsl, ...): templated exec wraps a local binary.
        ph["{bin}"] = binary.string();
        r = RunCommandWithTimeout(Substitute(test.runner.exec, ph), timeout);
    } else {
        // Transport runner (ssh, ...): copy bundle → exec remotely → cleanup.
        std::string unique = std::format("{}-{}-{}",
            test.config.name,
            std::hash<std::thread::id>{}(std::this_thread::get_id()),
            std::chrono::steady_clock::now().time_since_epoch().count());
        ph["{remote_bundle}"] = std::format("{}/{}", test.runner.remoteDir, unique);
        ph["{bin}"] = std::format("{}/{}", ph["{remote_bundle}"], ph["{bin_name}"]);
        // Backslash variants for cmd.exe-shell remotes (cmd's mkdir won't
        // auto-create intermediate directories when path uses forward slashes).
        ph["{remote_bundle_win}"] = ph["{remote_bundle}"];
        std::replace(ph["{remote_bundle_win}"].begin(), ph["{remote_bundle_win}"].end(), '/', '\\');
        ph["{bin_win}"] = ph["{bin}"];
        std::replace(ph["{bin_win}"].begin(), ph["{bin_win}"].end(), '/', '\\');
        // WSL form of the local bundle path: C:\foo -> /mnt/c/foo. Used by
        // the Wsl runner's `wsl cp -r` step to read the test bundle out of
        // the Windows host's filesystem.
        ph["{bundle_wsl}"] = WindowsPathToWsl(ph["{bundle}"]);

        CommandResult cp = RunCommandWithTimeout(Substitute(test.runner.copy, ph), 5min);
        if (cp.exitCode != 0) {
            auto end = std::chrono::steady_clock::now();
            result.duration = std::chrono::duration_cast<std::chrono::milliseconds>(end - start);
            result.outcome = TestOutcome::Fail;
            result.exitCode = cp.exitCode;
            result.output = std::format("copy step failed (exit {}):\n{}", cp.exitCode, cp.output);
            return result;
        }

        r = RunCommandWithTimeout(Substitute(test.runner.exec, ph), timeout);

        if (!test.runner.cleanup.empty()) {
            std::system(Substitute(test.runner.cleanup, ph).c_str());
        }
    }

    auto end = std::chrono::steady_clock::now();
    result.duration = std::chrono::duration_cast<std::chrono::milliseconds>(end - start);
    result.output = std::move(r.output);
    result.exitCode = r.exitCode;
    result.signal = r.signal;

    if (r.timedOut) result.outcome = TestOutcome::Timeout;
    else if (r.crashed) result.outcome = TestOutcome::Crash;
    else if (r.exitCode == 77) result.outcome = TestOutcome::Skipped;
    else if (r.exitCode != 0) result.outcome = TestOutcome::Fail;
    else result.outcome = TestOutcome::Pass;

    return result;
}

namespace {
    // Synthesize a Configuration for tests/<Name>/ folders that don't contain
    // a project.cpp. Convention: cfg.path = the folder, cfg.name/outputName =
    // folder basename, cfg.target = the run's targetFilter, cfg.type = exe.
    // Sources: top-level *.cpp (excluding project.cpp) become implementations,
    // interfaces/*.cppm become module interfaces (matching the layout used
    // elsewhere in this codebase). Tests with deeper layouts, defines, or
    // dependencies still need an explicit project.cpp.
    Configuration SynthesizeTest(const fs::path& dir, std::string_view target) {
        Configuration cfg;
        cfg.path = dir;
        cfg.name = dir.filename().string();
        cfg.outputName = cfg.name;
        cfg.target = std::string(target);
        cfg.type = ConfigurationType::Executable;

        std::vector<fs::path> impls;
        for (auto& e : fs::directory_iterator(dir)) {
            if (!e.is_regular_file()) continue;
            auto p = e.path();
            if (p.extension() != ".cpp") continue;
            if (p.filename() == "project.cpp") continue;
            impls.push_back(p.stem());
        }
        std::ranges::sort(impls);

        std::vector<fs::path> ifaces;
        fs::path interfacesDir = dir / "interfaces";
        if (fs::exists(interfacesDir) && fs::is_directory(interfacesDir)) {
            for (auto& e : fs::directory_iterator(interfacesDir)) {
                if (!e.is_regular_file()) continue;
                auto p = e.path();
                if (p.extension() != ".cppm") continue;
                ifaces.push_back(fs::path("interfaces") / p.stem());
            }
            std::ranges::sort(ifaces);
        }

        if (impls.empty() && ifaces.empty()) {
            throw std::runtime_error(std::format(
                "no .cpp or interfaces/*.cppm files found in {}", dir.string()));
        }

        cfg.GetInterfacesAndImplementations(ifaces, impls);
        return cfg;
    }
}

TestSummary Crafter::RunTests(Configuration& projectCfg, const RunTestsOptions& opts, std::span<const std::string_view> projectArgs) {
    TestSummary summary;
    std::vector<TestResult> discoveryFailures;

    // Auto-discover tests one layer deep: each <project>/tests/<Name>/ folder
    // is a test. If it contains project.cpp, that's loaded for full control;
    // otherwise the Configuration is synthesized from the folder contents.
    // Folders whose name starts with '_' or '.' are skipped (so tests/_shared/
    // holds cross-test code without becoming a test). Each project.cpp receives
    // the same args the root project did, so --target=... propagates through.
    //
    // Discovery is keyed off cwd (= the project root, since crafter-build loads
    // ./project.cpp), not projectCfg.path: tests live at the project root even
    // when projectCfg.path points at a subdirectory like "./src/" or "./lib/".
    fs::path testsDir = fs::current_path() / "tests";
    if (fs::exists(testsDir) && fs::is_directory(testsDir)) {
        struct TestEntry { fs::path dir; fs::path pcpp; };  // pcpp empty = synth
        std::vector<TestEntry> entries;
        for (auto& entry : fs::directory_iterator(testsDir)) {
            if (!entry.is_directory()) continue;
            auto stem = entry.path().filename().string();
            if (stem.empty() || stem[0] == '_' || stem[0] == '.') continue;
            TestEntry te;
            te.dir = entry.path();
            auto pcpp = te.dir / "project.cpp";
            if (fs::exists(pcpp)) te.pcpp = pcpp;
            entries.push_back(std::move(te));
        }
        std::ranges::sort(entries, [](auto& a, auto& b) { return a.dir < b.dir; });

        // Inject --target=<filter> into the args we hand each fixture so its
        // CrafterBuildProject can default to the run's target. The CLI's own
        // --target=... propagates through projectArgs already; this only
        // appends when missing so an explicit user choice wins.
        std::string targetArg = std::format("--target={}", opts.targetFilter);
        std::vector<std::string_view> fixtureArgs(projectArgs.begin(), projectArgs.end());
        bool hasTarget = std::ranges::any_of(fixtureArgs, [](std::string_view a) {
            return a.starts_with("--target=");
        });
        if (!hasTarget) fixtureArgs.push_back(targetArg);

        for (auto& te : entries) {
            Test t;
            try {
                if (!te.pcpp.empty()) {
                    t.config = LoadProject(te.pcpp, fixtureArgs);
                } else {
                    t.config = SynthesizeTest(te.dir, opts.targetFilter);
                }
            } catch (const std::exception& e) {
                // A broken fixture shouldn't kill the whole run. Surface as a
                // Fail and let other tests proceed.
                TestResult r;
                r.name = te.dir.filename().string();
                r.outcome = TestOutcome::Fail;
                r.exitCode = -1;
                r.output = !te.pcpp.empty()
                    ? std::format("project.cpp failed to load: {}", e.what())
                    : std::format("test discovery failed: {}", e.what());
                discoveryFailures.push_back(std::move(r));
                continue;
            }
            if (t.config.target != opts.targetFilter) continue;
            t.runner = TestRunner::FromEnv(t.config.target, TestRunner::Local());
            if (opts.runnerOverride) {
                if (auto r = TestRunner::FromSpec(*opts.runnerOverride)) {
                    t.runner = std::move(*r);
                }
            }
            projectCfg.tests.push_back(std::move(t));
        }
    }

    std::vector<Test*> filtered;
    filtered.reserve(projectCfg.tests.size());
    for (auto& test : projectCfg.tests) {
        if (MatchAny(opts.globs, test.config.name)) {
            filtered.push_back(&test);
        }
    }

    if (opts.listOnly) {
        for (auto& r : discoveryFailures) {
            std::println("{} (project.cpp broken)", r.name);
        }
        for (auto* t : filtered) {
            std::println("{}", t->config.name);
        }
        return summary;
    }

    if (filtered.empty() && discoveryFailures.empty()) {
        std::println("No tests matched.");
        return summary;
    }

    // Render discovery failures upfront so they appear before parallel test
    // results. They're already-determined Fails — no need to put them through
    // the worker pool.
    for (auto& r : discoveryFailures) {
        PrintResult(r, "");
        WriteLog(projectCfg.path, r.name, r.output);
    }

    int jobs = opts.jobs > 0
        ? opts.jobs
        : std::max(1u, std::thread::hardware_concurrency());
    jobs = std::min(jobs, static_cast<int>(filtered.size()));

    std::unordered_map<fs::path, std::shared_future<BuildResult>> depResults;
    std::mutex depMutex;
    std::mutex printMutex;
    std::mutex probeMutex;
    std::unordered_map<std::string, bool> probeCache;
    std::atomic<std::size_t> next{0};
    std::vector<TestResult> results(filtered.size());

    auto runnerAvailable = [&](const TestRunner& runner) -> bool {
        if (runner.probe.empty()) return true;
        std::lock_guard lk(probeMutex);
        if (auto it = probeCache.find(runner.name); it != probeCache.end()) {
            return it->second;
        }
        // RunCommandChecked captures and discards output internally, so probe
        // specs don't need `> /dev/null 2>&1` (which doesn't translate to cmd).
        bool ok = (RunCommandChecked(runner.probe).exitCode == 0);
        probeCache[runner.name] = ok;
        return ok;
    };

    auto worker = [&]() {
        while (true) {
            std::size_t i = next.fetch_add(1);
            if (i >= filtered.size()) break;
            Test& t = *filtered[i];
            TestResult r;
            r.name = t.config.name;

            if (!runnerAvailable(t.runner)) {
                r.outcome = TestOutcome::Skipped;
                r.output = std::format("runner '{}' not available", t.runner.name);
                {
                    std::lock_guard lk(printMutex);
                    PrintResult(r, t.runner.name);
                }
                results[i] = std::move(r);
                continue;
            }

            BuildResult br;
            try {
                br = Build(t.config, depResults, depMutex);
            } catch (const std::exception& e) {
                r.outcome = TestOutcome::Fail;
                r.output = std::format("build threw: {}", e.what());
                r.exitCode = -1;
                {
                    std::lock_guard lk(printMutex);
                    PrintResult(r, t.runner.name);
                }
                results[i] = std::move(r);
                continue;
            }

            if (!br.result.empty()) {
                r.outcome = TestOutcome::Fail;
                r.output = std::format("build failed: {}", br.result);
                r.exitCode = -1;
                {
                    std::lock_guard lk(printMutex);
                    PrintResult(r, t.runner.name);
                }
                results[i] = std::move(r);
                continue;
            }

            std::chrono::seconds timeout = opts.timeoutOverride.value_or(t.timeout);
            fs::path binary = TestBinaryPath(t.config);
            try {
                r = RunSingleTest(t, binary, timeout);
            } catch (const std::exception& e) {
                r.outcome = TestOutcome::Fail;
                r.output = std::format("runner threw: {}", e.what());
                r.exitCode = -1;
            }

            if (r.outcome != TestOutcome::Pass && r.outcome != TestOutcome::Skipped && !r.output.empty()) {
                WriteLog(projectCfg.path, r.name, r.output);
            }

            {
                std::lock_guard lk(printMutex);
                PrintResult(r, t.runner.name);
            }
            results[i] = std::move(r);
        }
    };

    std::vector<std::jthread> threads;
    threads.reserve(jobs);
    for (int j = 0; j < jobs; ++j) {
        threads.emplace_back(worker);
    }
    threads.clear(); // joins all jthreads

    for (auto& r : discoveryFailures) {
        results.push_back(std::move(r));
    }

    for (auto& r : results) {
        switch (r.outcome) {
            case TestOutcome::Pass:    summary.passed++; break;
            case TestOutcome::Fail:    summary.failed++; break;
            case TestOutcome::Crash:   summary.crashed++; break;
            case TestOutcome::Timeout: summary.timedOut++; break;
            case TestOutcome::Skipped: summary.skipped++; break;
        }
    }
    summary.results = std::move(results);

    std::print("\n");
    std::vector<std::string> parts;
    if (summary.passed)   parts.push_back(std::format("{} passed",  summary.passed));
    if (summary.failed)   parts.push_back(std::format("{} failed",  summary.failed));
    if (summary.crashed)  parts.push_back(std::format("{} crashed", summary.crashed));
    if (summary.timedOut) parts.push_back(std::format("{} timed out", summary.timedOut));
    if (summary.skipped)  parts.push_back(std::format("{} skipped", summary.skipped));
    std::string joined;
    for (std::size_t i = 0; i < parts.size(); ++i) {
        if (i) joined += ", ";
        joined += parts[i];
    }
    std::println("{}", joined);

    return summary;
}