Crafter.Build/implementations/Crafter.Build-Test.cpp

/*
Crafter® Build
Copyright (C) 2026 Catcrafts®
Catcrafts.net

This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License version 3.0 as published by the Free Software Foundation;

This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.

You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA
*/

export module Crafter.Build:Test_impl;
import std;
import :Test;
import :Clang;
import :Platform;
namespace fs = std::filesystem;
using namespace Crafter;

namespace {
    bool TargetIsWindows(std::string_view target) {
        return target.find("windows") != std::string_view::npos
            || target.find("mingw") != std::string_view::npos;
    }

    fs::path TestBinaryPath(const Configuration& cfg) {
        fs::path outputDir = cfg.path/"bin"/std::format("{}-{}-{}", cfg.name, cfg.target, cfg.march);
        return outputDir / (TargetIsWindows(cfg.target) ? cfg.outputName + ".exe" : cfg.outputName);
    }

    bool MatchGlob(std::string_view glob, std::string_view name) {
        std::size_t gi = 0, ni = 0, star = std::string_view::npos, mark = 0;
        while (ni < name.size()) {
            if (gi < glob.size() && (glob[gi] == '?' || glob[gi] == name[ni])) {
                ++gi; ++ni;
            } else if (gi < glob.size() && glob[gi] == '*') {
                star = gi++;
                mark = ni;
            } else if (star != std::string_view::npos) {
                gi = star + 1;
                ni = ++mark;
            } else {
                return false;
            }
        }
        while (gi < glob.size() && glob[gi] == '*') ++gi;
        return gi == glob.size();
    }

    bool MatchAny(std::span<const std::string> globs, std::string_view name) {
        if (globs.empty()) return true;
        for (const auto& g : globs) {
            if (MatchGlob(g, name)) return true;
        }
        return false;
    }

    std::string ShellQuote(std::string_view s) {
        std::string out;
        out.reserve(s.size() + 2);
        out.push_back('\'');
        for (char c : s) {
            if (c == '\'') out += "'\\''";
            else out.push_back(c);
        }
        out.push_back('\'');
        return out;
    }

    std::string JoinAndQuoteArgs(std::span<const std::string> args) {
        std::string out;
        for (const auto& a : args) {
            if (!out.empty()) out.push_back(' ');
            out += ShellQuote(a);
        }
        return out;
    }

    std::string Substitute(std::string_view tmpl, const std::map<std::string, std::string>& ph) {
        std::string out;
        out.reserve(tmpl.size());
        std::size_t i = 0;
        while (i < tmpl.size()) {
            if (tmpl[i] == '{') {
                std::size_t end = tmpl.find('}', i + 1);
                if (end != std::string_view::npos) {
                    std::string key(tmpl.substr(i, end - i + 1));
                    if (auto it = ph.find(key); it != ph.end()) {
                        out += it->second;
                        i = end + 1;
                        continue;
                    }
                }
            }
            out.push_back(tmpl[i++]);
        }
        return out;
    }

    std::string SignalName(int sig) {
        switch (sig) {
            case 1: return "SIGHUP";
            case 2: return "SIGINT";
            case 4: return "SIGILL";
            case 6: return "SIGABRT";
            case 8: return "SIGFPE";
            case 9: return "SIGKILL";
            case 11: return "SIGSEGV";
            case 13: return "SIGPIPE";
            case 14: return "SIGALRM";
            case 15: return "SIGTERM";
            default: return std::format("signal {}", sig);
        }
    }

    void WriteLog(const fs::path& projectPath, const std::string& name, const std::string& output) {
        fs::path logDir = projectPath / "build" / "test-logs";
        std::error_code ec;
        fs::create_directories(logDir, ec);
        if (ec) return;
        std::ofstream(logDir / (name + ".log")) << output;
    }

    void PrintResult(const TestResult& r, std::string_view runnerName) {
        auto ms = r.duration.count();
        std::string runnerSuffix = (runnerName.empty() || runnerName == "local")
            ? std::string()
            : std::format(" ({})", runnerName);
        switch (r.outcome) {
            case TestOutcome::Pass:
                std::println("✅ {}{} ({}ms)", r.name, runnerSuffix, ms);
                break;
            case TestOutcome::Fail:
                std::println("❌ {}{} ({}ms) exit {}", r.name, runnerSuffix, ms, r.exitCode);
                if (!r.output.empty()) {
                    for (auto line : std::views::split(r.output, '\n')) {
                        std::string_view sv(line.begin(), line.end());
                        if (!sv.empty()) std::println("    {}", sv);
                    }
                }
                break;
            case TestOutcome::Crash:
                std::println("\U0001F4A5 {}{} ({}ms) crashed: {}", r.name, runnerSuffix, ms, SignalName(r.signal));
                if (!r.output.empty()) {
                    for (auto line : std::views::split(r.output, '\n')) {
                        std::string_view sv(line.begin(), line.end());
                        if (!sv.empty()) std::println("    {}", sv);
                    }
                }
                break;
            case TestOutcome::Timeout:
                std::println("⏱  {}{} ({}ms) timeout", r.name, runnerSuffix, ms);
                break;
            case TestOutcome::Skipped:
                std::println("⏭  {}{} skipped: {}", r.name, runnerSuffix,
                    r.output.empty() ? std::string("(no reason)") : r.output);
                break;
        }
    }
}

TestRunner TestRunner::Local() {
    TestRunner r;
    r.name = "local";
    return r;
}

TestRunner TestRunner::Ssh(std::string host, std::string remoteDir) {
    TestRunner r;
    r.name = std::format("ssh:{}", host);
    r.remoteDir = std::move(remoteDir);
    r.copy    = std::format("ssh -q {0} 'mkdir -p {{remote_bundle}}' && scp -r -q {{bundle}}/. {0}:{{remote_bundle}}/", host);
    r.exec    = std::format("ssh -q {} 'cd {{remote_bundle}} && ./{{bin_name}} {{args}}'", host);
    r.cleanup = std::format("ssh -q {} 'rm -rf {{remote_bundle}}' || true", host);
    r.probe   = std::format("ssh -q -o BatchMode=yes -o ConnectTimeout=5 {} true > /dev/null 2>&1", host);
    return r;
}

TestRunner TestRunner::SshWin(std::string host, std::string remoteDir) {
    TestRunner r;
    r.name = std::format("sshwin:{}", host);
    r.remoteDir = std::move(remoteDir);
    // cmd.exe-friendly templates. Use backslash variants of remote paths because
    // cmd's mkdir won't auto-create intermediate directories with forward
    // slashes. scp tolerates either, so we keep forward slashes for the scp
    // dest (which `{remote_bundle}` provides). 2>nul + `& rem ok` swallows
    // mkdir's "already exists" error and forces exit code 0.
    r.copy    = std::format("ssh -q {0} \"mkdir {{remote_bundle_win}} 2>nul & rem ok\" && scp -r -q {{bundle}}/. {0}:{{remote_bundle}}/", host);
    r.exec    = std::format("ssh -q {} \"{{bin_win}} {{args}}\"", host);
    r.cleanup = std::format("ssh -q {} \"rd /s /q {{remote_bundle_win}} 2>nul & rem ok\"", host);
    r.probe   = std::format("ssh -q -o BatchMode=yes -o ConnectTimeout=5 {} \"ver\" > /dev/null 2>&1", host);
    return r;
}

TestRunner TestRunner::QemuUser(std::string qemuBin) {
    TestRunner r;
    r.name = std::format("qemu:{}", qemuBin);
    r.exec = std::format("{} {{bin}} {{args}}", qemuBin);
    r.probe = std::format("which {} > /dev/null 2>&1", qemuBin);
    return r;
}

namespace {
    std::string NormalizeTriple(std::string_view target) {
        std::string out(target);
        for (char& c : out) {
            if (c == '-' || c == '.') c = '_';
        }
        return out;
    }

    std::optional<TestRunner> ParseRunnerSpec(std::string_view spec) {
        if (spec.empty()) return std::nullopt;
        std::vector<std::string> parts;
        for (auto piece : std::views::split(spec, ':')) {
            parts.emplace_back(std::string_view(piece.begin(), piece.end()));
        }
        if (parts.empty()) return std::nullopt;
        if (parts[0] == "local") return TestRunner::Local();
        if (parts[0] == "qemu" && parts.size() == 2) return TestRunner::QemuUser(parts[1]);
        if (parts[0] == "ssh" && parts.size() == 2) return TestRunner::Ssh(parts[1]);
        if (parts[0] == "ssh" && parts.size() >= 3) {
            std::string remote;
            for (std::size_t i = 2; i < parts.size(); ++i) {
                if (i > 2) remote.push_back(':');
                remote += parts[i];
            }
            return TestRunner::Ssh(parts[1], remote);
        }
        if (parts[0] == "sshwin" && parts.size() == 2) return TestRunner::SshWin(parts[1]);
        if (parts[0] == "sshwin" && parts.size() >= 3) {
            std::string remote;
            for (std::size_t i = 2; i < parts.size(); ++i) {
                if (i > 2) remote.push_back(':');
                remote += parts[i];
            }
            return TestRunner::SshWin(parts[1], remote);
        }
        throw std::runtime_error(std::format(
            "TestRunner::FromEnv: unrecognized runner spec '{}'", spec));
    }
}

TestRunner TestRunner::FromEnv(std::string_view target, TestRunner fallback) {
    std::string envName = std::format("CRAFTER_BUILD_RUNNER_{}", NormalizeTriple(target));
    const char* v = std::getenv(envName.c_str());
    if (!v || !*v) return fallback;
    if (auto r = ParseRunnerSpec(v)) return std::move(*r);
    return fallback;
}

TestResult Crafter::RunSingleTest(const Test& test, const fs::path& binary, std::chrono::seconds timeout) {
    using namespace std::chrono_literals;
    TestResult result;
    result.name = test.config.name;

    std::map<std::string, std::string> ph;
    ph["{args}"] = JoinAndQuoteArgs(test.args);
    ph["{bin_name}"] = binary.filename().string();
    ph["{bundle}"] = binary.parent_path().string();

    auto start = std::chrono::steady_clock::now();
    CommandResult r;

    if (test.runner.exec.empty()) {
        // Pure-local runner: spawn the binary directly.
        std::string cmd = std::format("{} {}", ShellQuote(binary.string()), ph["{args}"]);
        r = RunCommandWithTimeout(cmd, timeout);
    } else if (test.runner.copy.empty()) {
        // Prefix runner (qemu-user, wsl, ...): templated exec wraps a local binary.
        ph["{bin}"] = binary.string();
        r = RunCommandWithTimeout(Substitute(test.runner.exec, ph), timeout);
    } else {
        // Transport runner (ssh, ...): copy bundle → exec remotely → cleanup.
        std::string unique = std::format("{}-{}-{}",
            test.config.name,
            std::hash<std::thread::id>{}(std::this_thread::get_id()),
            std::chrono::steady_clock::now().time_since_epoch().count());
        ph["{remote_bundle}"] = std::format("{}/{}", test.runner.remoteDir, unique);
        ph["{bin}"] = std::format("{}/{}", ph["{remote_bundle}"], ph["{bin_name}"]);
        // Backslash variants for cmd.exe-shell remotes (cmd's mkdir won't
        // auto-create intermediate directories when path uses forward slashes).
        ph["{remote_bundle_win}"] = ph["{remote_bundle}"];
        std::replace(ph["{remote_bundle_win}"].begin(), ph["{remote_bundle_win}"].end(), '/', '\\');
        ph["{bin_win}"] = ph["{bin}"];
        std::replace(ph["{bin_win}"].begin(), ph["{bin_win}"].end(), '/', '\\');

        CommandResult cp = RunCommandWithTimeout(Substitute(test.runner.copy, ph), 5min);
        if (cp.exitCode != 0) {
            auto end = std::chrono::steady_clock::now();
            result.duration = std::chrono::duration_cast<std::chrono::milliseconds>(end - start);
            result.outcome = TestOutcome::Fail;
            result.exitCode = cp.exitCode;
            result.output = std::format("copy step failed (exit {}):\n{}", cp.exitCode, cp.output);
            return result;
        }

        r = RunCommandWithTimeout(Substitute(test.runner.exec, ph), timeout);

        if (!test.runner.cleanup.empty()) {
            std::system(Substitute(test.runner.cleanup, ph).c_str());
        }
    }

    auto end = std::chrono::steady_clock::now();
    result.duration = std::chrono::duration_cast<std::chrono::milliseconds>(end - start);
    result.output = std::move(r.output);
    result.exitCode = r.exitCode;
    result.signal = r.signal;

    if (r.timedOut) result.outcome = TestOutcome::Timeout;
    else if (r.crashed) result.outcome = TestOutcome::Crash;
    else if (r.exitCode != 0) result.outcome = TestOutcome::Fail;
    else result.outcome = TestOutcome::Pass;

    return result;
}

TestSummary Crafter::RunTests(Configuration& projectCfg, const RunTestsOptions& opts) {
    TestSummary summary;

    std::vector<Test*> filtered;
    filtered.reserve(projectCfg.tests.size());
    for (auto& test : projectCfg.tests) {
        if (MatchAny(opts.globs, test.config.name)) {
            filtered.push_back(&test);
        }
    }

    if (opts.listOnly) {
        for (auto* t : filtered) {
            std::println("{}", t->config.name);
        }
        return summary;
    }

    if (filtered.empty()) {
        std::println("No tests matched.");
        return summary;
    }

    int jobs = opts.jobs > 0
        ? opts.jobs
        : std::max(1u, std::thread::hardware_concurrency());
    jobs = std::min(jobs, static_cast<int>(filtered.size()));

    std::unordered_map<fs::path, std::shared_future<BuildResult>> depResults;
    std::mutex depMutex;
    std::mutex printMutex;
    std::mutex probeMutex;
    std::unordered_map<std::string, bool> probeCache;
    std::atomic<std::size_t> next{0};
    std::vector<TestResult> results(filtered.size());

    auto runnerAvailable = [&](const TestRunner& runner) -> bool {
        if (runner.probe.empty()) return true;
        std::lock_guard lk(probeMutex);
        if (auto it = probeCache.find(runner.name); it != probeCache.end()) {
            return it->second;
        }
        bool ok = (std::system(runner.probe.c_str()) == 0);
        probeCache[runner.name] = ok;
        return ok;
    };

    auto worker = [&]() {
        while (true) {
            std::size_t i = next.fetch_add(1);
            if (i >= filtered.size()) break;
            Test& t = *filtered[i];
            TestResult r;
            r.name = t.config.name;

            if (!runnerAvailable(t.runner)) {
                r.outcome = TestOutcome::Skipped;
                r.output = std::format("runner '{}' not available", t.runner.name);
                {
                    std::lock_guard lk(printMutex);
                    PrintResult(r, t.runner.name);
                }
                results[i] = std::move(r);
                continue;
            }

            BuildResult br;
            try {
                br = Build(t.config, depResults, depMutex);
            } catch (const std::exception& e) {
                r.outcome = TestOutcome::Fail;
                r.output = std::format("build threw: {}", e.what());
                r.exitCode = -1;
                {
                    std::lock_guard lk(printMutex);
                    PrintResult(r, t.runner.name);
                }
                results[i] = std::move(r);
                continue;
            }

            if (!br.result.empty()) {
                r.outcome = TestOutcome::Fail;
                r.output = std::format("build failed: {}", br.result);
                r.exitCode = -1;
                {
                    std::lock_guard lk(printMutex);
                    PrintResult(r, t.runner.name);
                }
                results[i] = std::move(r);
                continue;
            }

            std::chrono::seconds timeout = opts.timeoutOverride.value_or(t.timeout);
            fs::path binary = TestBinaryPath(t.config);
            try {
                r = RunSingleTest(t, binary, timeout);
            } catch (const std::exception& e) {
                r.outcome = TestOutcome::Fail;
                r.output = std::format("runner threw: {}", e.what());
                r.exitCode = -1;
            }

            if (r.outcome != TestOutcome::Pass && r.outcome != TestOutcome::Skipped && !r.output.empty()) {
                WriteLog(projectCfg.path, r.name, r.output);
            }

            {
                std::lock_guard lk(printMutex);
                PrintResult(r, t.runner.name);
            }
            results[i] = std::move(r);
        }
    };

    std::vector<std::jthread> threads;
    threads.reserve(jobs);
    for (int j = 0; j < jobs; ++j) {
        threads.emplace_back(worker);
    }
    threads.clear(); // joins all jthreads

    for (auto& r : results) {
        switch (r.outcome) {
            case TestOutcome::Pass:    summary.passed++; break;
            case TestOutcome::Fail:    summary.failed++; break;
            case TestOutcome::Crash:   summary.crashed++; break;
            case TestOutcome::Timeout: summary.timedOut++; break;
            case TestOutcome::Skipped: summary.skipped++; break;
        }
    }
    summary.results = std::move(results);

    std::print("\n");
    std::vector<std::string> parts;
    if (summary.passed)   parts.push_back(std::format("{} passed",  summary.passed));
    if (summary.failed)   parts.push_back(std::format("{} failed",  summary.failed));
    if (summary.crashed)  parts.push_back(std::format("{} crashed", summary.crashed));
    if (summary.timedOut) parts.push_back(std::format("{} timed out", summary.timedOut));
    if (summary.skipped)  parts.push_back(std::format("{} skipped", summary.skipped));
    std::string joined;
    for (std::size_t i = 0; i < parts.size(); ++i) {
        if (i) joined += ", ";
        joined += parts[i];
    }
    std::println("{}", joined);

    return summary;
}