Jorijn van der Graaf
4d09eaac2a
Some checks failed
CI / build-test-release (push) Failing after 4m41s
740 lines
30 KiB
C++
740 lines
30 KiB
C++
/*
|
|
Crafter® Build
|
|
Copyright (C) 2026 Catcrafts®
|
|
Catcrafts.net
|
|
|
|
This library is free software; you can redistribute it and/or
|
|
modify it under the terms of the GNU Lesser General Public
|
|
License version 3.0 as published by the Free Software Foundation;
|
|
|
|
This library is distributed in the hope that it will be useful,
|
|
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
Lesser General Public License for more details.
|
|
|
|
You should have received a copy of the GNU Lesser General Public
|
|
License along with this library; if not, write to the Free Software
|
|
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
|
*/
|
|
|
|
export module Crafter.Build:Test_impl;
|
|
import std;
|
|
import :Test;
|
|
import :Clang;
|
|
import :Platform;
|
|
import :Progress;
|
|
namespace fs = std::filesystem;
|
|
using namespace Crafter;
|
|
|
|
namespace {
|
|
bool TargetIsWindows(std::string_view target) {
|
|
return target.find("windows") != std::string_view::npos
|
|
|| target.find("mingw") != std::string_view::npos;
|
|
}
|
|
|
|
fs::path TestBinaryPath(const Configuration& cfg) {
|
|
fs::path outputDir = cfg.path/"bin"/std::format("{}-{}-{}", cfg.name, cfg.target, cfg.march);
|
|
return outputDir / (TargetIsWindows(cfg.target) ? cfg.outputName + ".exe" : cfg.outputName);
|
|
}
|
|
|
|
bool MatchGlob(std::string_view glob, std::string_view name) {
|
|
std::size_t gi = 0, ni = 0, star = std::string_view::npos, mark = 0;
|
|
while (ni < name.size()) {
|
|
if (gi < glob.size() && (glob[gi] == '?' || glob[gi] == name[ni])) {
|
|
++gi; ++ni;
|
|
} else if (gi < glob.size() && glob[gi] == '*') {
|
|
star = gi++;
|
|
mark = ni;
|
|
} else if (star != std::string_view::npos) {
|
|
gi = star + 1;
|
|
ni = ++mark;
|
|
} else {
|
|
return false;
|
|
}
|
|
}
|
|
while (gi < glob.size() && glob[gi] == '*') ++gi;
|
|
return gi == glob.size();
|
|
}
|
|
|
|
bool MatchAny(std::span<const std::string> globs, std::string_view name) {
|
|
if (globs.empty()) return true;
|
|
for (const auto& g : globs) {
|
|
if (MatchGlob(g, name)) return true;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
std::string ShellQuoteSh(std::string_view s) {
|
|
std::string out;
|
|
out.reserve(s.size() + 2);
|
|
out.push_back('\'');
|
|
for (char c : s) {
|
|
if (c == '\'') out += "'\\''";
|
|
else out.push_back(c);
|
|
}
|
|
out.push_back('\'');
|
|
return out;
|
|
}
|
|
|
|
// cmd.exe doesn't recognize '...' as quoting (it would pass the single
|
|
// quotes through to the executable). Wrap in "..." for cmd; embedded "
|
|
// is rare in paths but escape it to be safe. Backslash sequences before
|
|
// " don't need the MS-CRT doubling rules because we go through
|
|
// `cmd /C "..."`, which uses cmd's parser, not the CRT's argv splitter.
|
|
std::string ShellQuoteCmd(std::string_view s) {
|
|
std::string out;
|
|
out.reserve(s.size() + 2);
|
|
out.push_back('"');
|
|
for (char c : s) {
|
|
if (c == '"') out += "\\\"";
|
|
else out.push_back(c);
|
|
}
|
|
out.push_back('"');
|
|
return out;
|
|
}
|
|
|
|
// Host-shell quoting: sh on Linux, cmd on Windows. For args/paths that
|
|
// hit the local shell (Local runner exec, Cmd-prefix runner exec).
|
|
std::string ShellQuoteHost(std::string_view s) {
|
|
#ifdef _WIN32
|
|
return ShellQuoteCmd(s);
|
|
#else
|
|
return ShellQuoteSh(s);
|
|
#endif
|
|
}
|
|
|
|
std::string JoinAndQuoteArgs(std::span<const std::string> args, TestRunner::Shell shell) {
|
|
std::string out;
|
|
for (const auto& a : args) {
|
|
if (!out.empty()) out.push_back(' ');
|
|
switch (shell) {
|
|
case TestRunner::Shell::Sh: out += ShellQuoteSh(a); break;
|
|
case TestRunner::Shell::Cmd: out += ShellQuoteCmd(a); break;
|
|
case TestRunner::Shell::Host: out += ShellQuoteHost(a); break;
|
|
}
|
|
}
|
|
return out;
|
|
}
|
|
|
|
std::string Substitute(std::string_view tmpl, const std::map<std::string, std::string>& ph) {
|
|
std::string out;
|
|
out.reserve(tmpl.size());
|
|
std::size_t i = 0;
|
|
while (i < tmpl.size()) {
|
|
if (tmpl[i] == '{') {
|
|
std::size_t end = tmpl.find('}', i + 1);
|
|
if (end != std::string_view::npos) {
|
|
std::string key(tmpl.substr(i, end - i + 1));
|
|
if (auto it = ph.find(key); it != ph.end()) {
|
|
out += it->second;
|
|
i = end + 1;
|
|
continue;
|
|
}
|
|
}
|
|
}
|
|
out.push_back(tmpl[i++]);
|
|
}
|
|
return out;
|
|
}
|
|
|
|
std::string SignalName(int sig) {
|
|
switch (sig) {
|
|
case 1: return "SIGHUP";
|
|
case 2: return "SIGINT";
|
|
case 4: return "SIGILL";
|
|
case 6: return "SIGABRT";
|
|
case 8: return "SIGFPE";
|
|
case 9: return "SIGKILL";
|
|
case 11: return "SIGSEGV";
|
|
case 13: return "SIGPIPE";
|
|
case 14: return "SIGALRM";
|
|
case 15: return "SIGTERM";
|
|
default: return std::format("signal {}", sig);
|
|
}
|
|
}
|
|
|
|
void WriteLog(const fs::path& projectPath, const std::string& name, const std::string& output) {
|
|
fs::path logDir = projectPath / "build" / "test-logs";
|
|
std::error_code ec;
|
|
fs::create_directories(logDir, ec);
|
|
if (ec) return;
|
|
std::ofstream(logDir / (name + ".log")) << output;
|
|
}
|
|
|
|
void PrintResult(const TestResult& r, std::string_view runnerName) {
|
|
Progress::Clear();
|
|
auto ms = r.duration.count();
|
|
std::string runnerSuffix = (runnerName.empty() || runnerName == "local")
|
|
? std::string()
|
|
: std::format(" ({})", runnerName);
|
|
switch (r.outcome) {
|
|
case TestOutcome::Pass:
|
|
std::println("✅ {}{} ({}ms)", r.name, runnerSuffix, ms);
|
|
break;
|
|
case TestOutcome::Fail:
|
|
std::println("❌ {}{} ({}ms) exit {}", r.name, runnerSuffix, ms, r.exitCode);
|
|
if (!r.output.empty()) {
|
|
for (auto line : std::views::split(r.output, '\n')) {
|
|
std::string_view sv(line.begin(), line.end());
|
|
if (!sv.empty()) std::println(" {}", sv);
|
|
}
|
|
}
|
|
break;
|
|
case TestOutcome::Crash:
|
|
std::println("\U0001F4A5 {}{} ({}ms) crashed: {}", r.name, runnerSuffix, ms, SignalName(r.signal));
|
|
if (!r.output.empty()) {
|
|
for (auto line : std::views::split(r.output, '\n')) {
|
|
std::string_view sv(line.begin(), line.end());
|
|
if (!sv.empty()) std::println(" {}", sv);
|
|
}
|
|
}
|
|
break;
|
|
case TestOutcome::Timeout:
|
|
std::println("⏱ {}{} ({}ms) timeout", r.name, runnerSuffix, ms);
|
|
break;
|
|
case TestOutcome::Skipped:
|
|
std::println("⏭ {}{} skipped: {}", r.name, runnerSuffix,
|
|
r.output.empty() ? std::string("(no reason)") : r.output);
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
namespace {
|
|
std::atomic<Configuration*> g_parentProject{nullptr};
|
|
|
|
Configuration* FindLibInTree(Configuration* root,
|
|
std::string_view name,
|
|
std::unordered_set<Configuration*>& seen) {
|
|
if (!seen.insert(root).second) return nullptr;
|
|
if (root->name == name) return root;
|
|
for (Configuration* dep : root->dependencies) {
|
|
if (auto found = FindLibInTree(dep, name, seen)) return found;
|
|
}
|
|
return nullptr;
|
|
}
|
|
}
|
|
|
|
void Crafter::SetParentProject(Configuration* parent) {
|
|
g_parentProject.store(parent);
|
|
}
|
|
|
|
Configuration* Crafter::ParentLib(std::string_view name) {
|
|
Configuration* root = g_parentProject.load();
|
|
if (!root) {
|
|
throw std::runtime_error(std::format(
|
|
"Crafter::ParentLib('{}'): no parent project set", name));
|
|
}
|
|
std::unordered_set<Configuration*> seen;
|
|
if (auto found = FindLibInTree(root, name, seen)) return found;
|
|
throw std::runtime_error(std::format(
|
|
"Crafter::ParentLib('{}'): not found in parent project '{}'",
|
|
name, root->name));
|
|
}
|
|
|
|
TestRunner TestRunner::Local() {
|
|
TestRunner r;
|
|
r.name = "local";
|
|
r.argsShell = Shell::Host;
|
|
return r;
|
|
}
|
|
|
|
TestRunner TestRunner::Ssh(std::string host, std::string remoteDir) {
|
|
TestRunner r;
|
|
r.name = std::format("ssh:{}", host);
|
|
r.remoteDir = std::move(remoteDir);
|
|
r.argsShell = Shell::Sh;
|
|
// Outer "..." (not '...') so the wrapper survives both sh (Linux host)
|
|
// and cmd (Windows host). cmd doesn't honor single quotes, which would
|
|
// cause it to split on the inner '&&'. Args inside use POSIX quoting
|
|
// because they reach a remote bash regardless of which host issued them.
|
|
r.copy = std::format("ssh -q {0} \"mkdir -p {{remote_bundle}}\" && scp -r -q {{bundle}}/. {0}:{{remote_bundle}}/", host);
|
|
r.exec = std::format("ssh -q {} \"cd {{remote_bundle}} && ./{{bin_name}} {{args}}\"", host);
|
|
r.cleanup = std::format("ssh -q {} \"rm -rf {{remote_bundle}}\"", host);
|
|
// RunCommandChecked captures stdout+stderr internally — no shell redirect
|
|
// needed in the probe spec, which means it works the same way under cmd
|
|
// (Windows host) and sh (Linux host) since `> /dev/null` doesn't translate.
|
|
r.probe = std::format("ssh -q -o BatchMode=yes -o ConnectTimeout=5 {} true", host);
|
|
return r;
|
|
}
|
|
|
|
TestRunner TestRunner::SshWin(std::string host, std::string remoteDir) {
|
|
TestRunner r;
|
|
r.name = std::format("sshwin:{}", host);
|
|
r.remoteDir = std::move(remoteDir);
|
|
r.argsShell = Shell::Cmd;
|
|
// cmd.exe-friendly templates. Use backslash variants of remote paths because
|
|
// cmd's mkdir won't auto-create intermediate directories with forward
|
|
// slashes. scp tolerates either, so we keep forward slashes for the scp
|
|
// dest (which `{remote_bundle}` provides). 2>nul + `& rem ok` swallows
|
|
// mkdir's "already exists" error and forces exit code 0.
|
|
r.copy = std::format("ssh -q {0} \"mkdir {{remote_bundle_win}} 2>nul & rem ok\" && scp -r -q {{bundle}}/. {0}:{{remote_bundle}}/", host);
|
|
r.exec = std::format("ssh -q {} \"{{bin_win}} {{args}}\"", host);
|
|
r.cleanup = std::format("ssh -q {} \"rd /s /q {{remote_bundle_win}} 2>nul & rem ok\"", host);
|
|
// No shell redirect in the probe — see TestRunner::Ssh for rationale.
|
|
r.probe = std::format("ssh -q -o BatchMode=yes -o ConnectTimeout=5 {} \"ver\"", host);
|
|
return r;
|
|
}
|
|
|
|
TestRunner TestRunner::Wsl(std::string remoteDir) {
|
|
TestRunner r;
|
|
r.name = "wsl";
|
|
r.argsShell = Shell::Sh;
|
|
r.remoteDir = std::move(remoteDir);
|
|
// Transport runner: stage the test bundle into WSL's native filesystem
|
|
// (faster than executing in-place from /mnt/c) then run via bash. {bundle_wsl}
|
|
// is the local Windows path translated to /mnt/<drive>/... form so wsl cp
|
|
// can read it. Args are POSIX-quoted because they reach a Linux shell.
|
|
r.copy = "wsl mkdir -p {remote_bundle} && wsl cp -r {bundle_wsl}/. {remote_bundle}/";
|
|
r.exec = "wsl bash -c \"cd {remote_bundle} && ./{bin_name} {args}\"";
|
|
r.cleanup = "wsl rm -rf {remote_bundle}";
|
|
// `where wsl` matches the host-shell convention; on a non-Windows host
|
|
// it fails (no `where` command), which correctly skips this runner.
|
|
r.probe = "where wsl";
|
|
return r;
|
|
}
|
|
|
|
TestRunner TestRunner::Cmd(std::string command) {
|
|
TestRunner r;
|
|
r.name = std::format("cmd:{}", command);
|
|
r.argsShell = Shell::Host;
|
|
r.exec = std::format("{} {{bin}} {{args}}", command);
|
|
#ifdef _WIN32
|
|
r.probe = std::format("where {}", command);
|
|
#else
|
|
r.probe = std::format("which {}", command);
|
|
#endif
|
|
return r;
|
|
}
|
|
|
|
namespace {
|
|
std::string NormalizeTriple(std::string_view target) {
|
|
std::string out(target);
|
|
for (char& c : out) {
|
|
if (c == '-' || c == '.') c = '_';
|
|
}
|
|
return out;
|
|
}
|
|
|
|
std::optional<TestRunner> ParseRunnerSpec(std::string_view spec) {
|
|
if (spec.empty()) return std::nullopt;
|
|
std::vector<std::string> parts;
|
|
for (auto piece : std::views::split(spec, ':')) {
|
|
parts.emplace_back(std::string_view(piece.begin(), piece.end()));
|
|
}
|
|
if (parts.empty()) return std::nullopt;
|
|
if (parts[0] == "local") return TestRunner::Local();
|
|
if (parts[0] == "cmd" && parts.size() == 2) return TestRunner::Cmd(parts[1]);
|
|
if (parts[0] == "ssh" && parts.size() == 2) return TestRunner::Ssh(parts[1]);
|
|
if (parts[0] == "ssh" && parts.size() >= 3) {
|
|
std::string remote;
|
|
for (std::size_t i = 2; i < parts.size(); ++i) {
|
|
if (i > 2) remote.push_back(':');
|
|
remote += parts[i];
|
|
}
|
|
return TestRunner::Ssh(parts[1], remote);
|
|
}
|
|
if (parts[0] == "sshwin" && parts.size() == 2) return TestRunner::SshWin(parts[1]);
|
|
if (parts[0] == "sshwin" && parts.size() >= 3) {
|
|
std::string remote;
|
|
for (std::size_t i = 2; i < parts.size(); ++i) {
|
|
if (i > 2) remote.push_back(':');
|
|
remote += parts[i];
|
|
}
|
|
return TestRunner::SshWin(parts[1], remote);
|
|
}
|
|
if (parts[0] == "wsl" && parts.size() == 1) return TestRunner::Wsl();
|
|
if (parts[0] == "wsl" && parts.size() >= 2) {
|
|
std::string remote;
|
|
for (std::size_t i = 1; i < parts.size(); ++i) {
|
|
if (i > 1) remote.push_back(':');
|
|
remote += parts[i];
|
|
}
|
|
return TestRunner::Wsl(remote);
|
|
}
|
|
throw std::runtime_error(std::format(
|
|
"TestRunner::FromSpec: unrecognized runner spec '{}'", spec));
|
|
}
|
|
|
|
// C:\Users\jorij -> /mnt/c/Users/jorij. Idempotent on paths that don't
|
|
// start with a drive letter, so callers can compute it unconditionally.
|
|
std::string WindowsPathToWsl(std::string_view p) {
|
|
if (p.size() < 2 || p[1] != ':') return std::string(p);
|
|
char drive = static_cast<char>(std::tolower(static_cast<unsigned char>(p[0])));
|
|
std::string out = std::format("/mnt/{}", drive);
|
|
for (std::size_t i = 2; i < p.size(); ++i) {
|
|
out.push_back(p[i] == '\\' ? '/' : p[i]);
|
|
}
|
|
return out;
|
|
}
|
|
}
|
|
|
|
std::optional<TestRunner> TestRunner::FromSpec(std::string_view spec) {
|
|
return ParseRunnerSpec(spec);
|
|
}
|
|
|
|
TestRunner TestRunner::FromEnv(std::string_view target, TestRunner fallback) {
|
|
std::string envName = std::format("CRAFTER_BUILD_RUNNER_{}", NormalizeTriple(target));
|
|
const char* v = std::getenv(envName.c_str());
|
|
if (!v || !*v) return fallback;
|
|
if (auto r = ParseRunnerSpec(v)) return std::move(*r);
|
|
return fallback;
|
|
}
|
|
|
|
TestResult Crafter::RunSingleTest(const Test& test, const fs::path& binary, std::chrono::seconds timeout) {
|
|
using namespace std::chrono_literals;
|
|
TestResult result;
|
|
result.name = test.config.name;
|
|
|
|
std::map<std::string, std::string> ph;
|
|
ph["{args}"] = JoinAndQuoteArgs(test.args, test.runner.argsShell);
|
|
ph["{bin_name}"] = binary.filename().string();
|
|
ph["{bundle}"] = binary.parent_path().string();
|
|
|
|
auto start = std::chrono::steady_clock::now();
|
|
CommandResult r;
|
|
|
|
if (test.runner.exec.empty()) {
|
|
// Pure-local runner: spawn the binary directly through the host shell.
|
|
std::string cmd = std::format("{} {}", ShellQuoteHost(binary.string()), ph["{args}"]);
|
|
r = RunCommandWithTimeout(cmd, timeout);
|
|
} else if (test.runner.copy.empty()) {
|
|
// Prefix runner (qemu-user, wsl, ...): templated exec wraps a local binary.
|
|
ph["{bin}"] = binary.string();
|
|
r = RunCommandWithTimeout(Substitute(test.runner.exec, ph), timeout);
|
|
} else {
|
|
// Transport runner (ssh, ...): copy bundle → exec remotely → cleanup.
|
|
std::string unique = std::format("{}-{}-{}",
|
|
test.config.name,
|
|
std::hash<std::thread::id>{}(std::this_thread::get_id()),
|
|
std::chrono::steady_clock::now().time_since_epoch().count());
|
|
ph["{remote_bundle}"] = std::format("{}/{}", test.runner.remoteDir, unique);
|
|
ph["{bin}"] = std::format("{}/{}", ph["{remote_bundle}"], ph["{bin_name}"]);
|
|
// Backslash variants for cmd.exe-shell remotes (cmd's mkdir won't
|
|
// auto-create intermediate directories when path uses forward slashes).
|
|
ph["{remote_bundle_win}"] = ph["{remote_bundle}"];
|
|
std::replace(ph["{remote_bundle_win}"].begin(), ph["{remote_bundle_win}"].end(), '/', '\\');
|
|
ph["{bin_win}"] = ph["{bin}"];
|
|
std::replace(ph["{bin_win}"].begin(), ph["{bin_win}"].end(), '/', '\\');
|
|
// WSL form of the local bundle path: C:\foo -> /mnt/c/foo. Used by
|
|
// the Wsl runner's `wsl cp -r` step to read the test bundle out of
|
|
// the Windows host's filesystem.
|
|
ph["{bundle_wsl}"] = WindowsPathToWsl(ph["{bundle}"]);
|
|
|
|
CommandResult cp = RunCommandWithTimeout(Substitute(test.runner.copy, ph), 5min);
|
|
if (cp.exitCode != 0) {
|
|
auto end = std::chrono::steady_clock::now();
|
|
result.duration = std::chrono::duration_cast<std::chrono::milliseconds>(end - start);
|
|
result.outcome = TestOutcome::Fail;
|
|
result.exitCode = cp.exitCode;
|
|
result.output = std::format("copy step failed (exit {}):\n{}", cp.exitCode, cp.output);
|
|
return result;
|
|
}
|
|
|
|
r = RunCommandWithTimeout(Substitute(test.runner.exec, ph), timeout);
|
|
|
|
if (!test.runner.cleanup.empty()) {
|
|
std::system(Substitute(test.runner.cleanup, ph).c_str());
|
|
}
|
|
}
|
|
|
|
auto end = std::chrono::steady_clock::now();
|
|
result.duration = std::chrono::duration_cast<std::chrono::milliseconds>(end - start);
|
|
result.output = std::move(r.output);
|
|
result.exitCode = r.exitCode;
|
|
result.signal = r.signal;
|
|
|
|
if (r.timedOut) result.outcome = TestOutcome::Timeout;
|
|
else if (r.crashed) result.outcome = TestOutcome::Crash;
|
|
else if (r.exitCode == 77) result.outcome = TestOutcome::Skipped;
|
|
else if (r.exitCode != 0) result.outcome = TestOutcome::Fail;
|
|
else result.outcome = TestOutcome::Pass;
|
|
|
|
return result;
|
|
}
|
|
|
|
namespace {
|
|
// Synthesize a Configuration for tests/<Name>/ folders that don't contain
|
|
// a project.cpp. Convention: cfg.path = the folder, cfg.name/outputName =
|
|
// folder basename, cfg.target = the run's targetFilter, cfg.type = exe.
|
|
// Sources: top-level *.cpp (excluding project.cpp) become implementations,
|
|
// interfaces/*.cppm become module interfaces (matching the layout used
|
|
// elsewhere in this codebase). Tests with deeper layouts, defines, or
|
|
// dependencies still need an explicit project.cpp.
|
|
Configuration SynthesizeTest(const fs::path& dir, std::string_view target) {
|
|
Configuration cfg;
|
|
cfg.path = dir;
|
|
cfg.name = dir.filename().string();
|
|
cfg.outputName = cfg.name;
|
|
cfg.target = std::string(target);
|
|
cfg.type = ConfigurationType::Executable;
|
|
|
|
std::vector<fs::path> impls;
|
|
for (auto& e : fs::directory_iterator(dir)) {
|
|
if (!e.is_regular_file()) continue;
|
|
auto p = e.path();
|
|
if (p.extension() != ".cpp") continue;
|
|
if (p.filename() == "project.cpp") continue;
|
|
impls.push_back(p.stem());
|
|
}
|
|
std::ranges::sort(impls);
|
|
|
|
std::vector<fs::path> ifaces;
|
|
fs::path interfacesDir = dir / "interfaces";
|
|
if (fs::exists(interfacesDir) && fs::is_directory(interfacesDir)) {
|
|
for (auto& e : fs::directory_iterator(interfacesDir)) {
|
|
if (!e.is_regular_file()) continue;
|
|
auto p = e.path();
|
|
if (p.extension() != ".cppm") continue;
|
|
ifaces.push_back(fs::path("interfaces") / p.stem());
|
|
}
|
|
std::ranges::sort(ifaces);
|
|
}
|
|
|
|
if (impls.empty() && ifaces.empty()) {
|
|
throw std::runtime_error(std::format(
|
|
"no .cpp or interfaces/*.cppm files found in {}", dir.string()));
|
|
}
|
|
|
|
cfg.GetInterfacesAndImplementations(ifaces, impls);
|
|
return cfg;
|
|
}
|
|
}
|
|
|
|
TestSummary Crafter::RunTests(Configuration& projectCfg, const RunTestsOptions& opts, std::span<const std::string_view> projectArgs) {
|
|
TestSummary summary;
|
|
std::vector<TestResult> discoveryFailures;
|
|
|
|
// Auto-discover tests one layer deep: each <project>/tests/<Name>/ folder
|
|
// is a test. If it contains project.cpp, that's loaded for full control;
|
|
// otherwise the Configuration is synthesized from the folder contents.
|
|
// Folders whose name starts with '_' or '.' are skipped (so tests/_shared/
|
|
// holds cross-test code without becoming a test). Each project.cpp receives
|
|
// the same args the root project did, so --target=... propagates through.
|
|
//
|
|
// Discovery is keyed off cwd (= the project root, since crafter-build loads
|
|
// ./project.cpp), not projectCfg.path: tests live at the project root even
|
|
// when projectCfg.path points at a subdirectory like "./src/" or "./lib/".
|
|
fs::path testsDir = fs::current_path() / "tests";
|
|
if (fs::exists(testsDir) && fs::is_directory(testsDir)) {
|
|
struct TestEntry { fs::path dir; fs::path pcpp; }; // pcpp empty = synth
|
|
std::vector<TestEntry> entries;
|
|
for (auto& entry : fs::directory_iterator(testsDir)) {
|
|
if (!entry.is_directory()) continue;
|
|
auto stem = entry.path().filename().string();
|
|
if (stem.empty() || stem[0] == '_' || stem[0] == '.') continue;
|
|
TestEntry te;
|
|
te.dir = entry.path();
|
|
auto pcpp = te.dir / "project.cpp";
|
|
if (fs::exists(pcpp)) te.pcpp = pcpp;
|
|
entries.push_back(std::move(te));
|
|
}
|
|
std::ranges::sort(entries, [](auto& a, auto& b) { return a.dir < b.dir; });
|
|
|
|
// Inject --target=<filter> into the args we hand each fixture so its
|
|
// CrafterBuildProject can default to the run's target. The CLI's own
|
|
// --target=... propagates through projectArgs already; this only
|
|
// appends when missing so an explicit user choice wins.
|
|
std::string targetArg = std::format("--target={}", opts.targetFilter);
|
|
std::vector<std::string_view> fixtureArgs(projectArgs.begin(), projectArgs.end());
|
|
bool hasTarget = std::ranges::any_of(fixtureArgs, [](std::string_view a) {
|
|
return a.starts_with("--target=");
|
|
});
|
|
if (!hasTarget) fixtureArgs.push_back(targetArg);
|
|
|
|
for (auto& te : entries) {
|
|
Test t;
|
|
try {
|
|
if (!te.pcpp.empty()) {
|
|
t.config = LoadProject(te.pcpp, fixtureArgs);
|
|
} else {
|
|
t.config = SynthesizeTest(te.dir, opts.targetFilter);
|
|
}
|
|
} catch (const std::exception& e) {
|
|
// A broken fixture shouldn't kill the whole run. Surface as a
|
|
// Fail and let other tests proceed.
|
|
TestResult r;
|
|
r.name = te.dir.filename().string();
|
|
r.outcome = TestOutcome::Fail;
|
|
r.exitCode = -1;
|
|
r.output = !te.pcpp.empty()
|
|
? std::format("project.cpp failed to load: {}", e.what())
|
|
: std::format("test discovery failed: {}", e.what());
|
|
discoveryFailures.push_back(std::move(r));
|
|
continue;
|
|
}
|
|
if (t.config.target != opts.targetFilter) continue;
|
|
t.runner = TestRunner::FromEnv(t.config.target, TestRunner::Local());
|
|
if (opts.runnerOverride) {
|
|
if (auto r = TestRunner::FromSpec(*opts.runnerOverride)) {
|
|
t.runner = std::move(*r);
|
|
}
|
|
}
|
|
projectCfg.tests.push_back(std::move(t));
|
|
}
|
|
}
|
|
|
|
std::vector<Test*> filtered;
|
|
filtered.reserve(projectCfg.tests.size());
|
|
for (auto& test : projectCfg.tests) {
|
|
if (MatchAny(opts.globs, test.config.name)) {
|
|
filtered.push_back(&test);
|
|
}
|
|
}
|
|
|
|
if (opts.listOnly) {
|
|
for (auto& r : discoveryFailures) {
|
|
std::println("{} (project.cpp broken)", r.name);
|
|
}
|
|
for (auto* t : filtered) {
|
|
std::println("{}", t->config.name);
|
|
}
|
|
return summary;
|
|
}
|
|
|
|
if (filtered.empty() && discoveryFailures.empty()) {
|
|
std::println("No tests matched.");
|
|
return summary;
|
|
}
|
|
|
|
// Render discovery failures upfront so they appear before parallel test
|
|
// results. They're already-determined Fails — no need to put them through
|
|
// the worker pool.
|
|
for (auto& r : discoveryFailures) {
|
|
PrintResult(r, "");
|
|
WriteLog(projectCfg.path, r.name, r.output);
|
|
}
|
|
|
|
int jobs = opts.jobs > 0
|
|
? opts.jobs
|
|
: std::max(1u, std::thread::hardware_concurrency());
|
|
jobs = std::min(jobs, static_cast<int>(filtered.size()));
|
|
|
|
std::unordered_map<fs::path, std::shared_future<BuildResult>> depResults;
|
|
std::mutex depMutex;
|
|
std::mutex printMutex;
|
|
std::mutex probeMutex;
|
|
std::unordered_map<std::string, bool> probeCache;
|
|
std::atomic<std::size_t> next{0};
|
|
std::vector<TestResult> results(filtered.size());
|
|
|
|
auto runnerAvailable = [&](const TestRunner& runner) -> bool {
|
|
if (runner.probe.empty()) return true;
|
|
std::lock_guard lk(probeMutex);
|
|
if (auto it = probeCache.find(runner.name); it != probeCache.end()) {
|
|
return it->second;
|
|
}
|
|
// RunCommandChecked captures and discards output internally, so probe
|
|
// specs don't need `> /dev/null 2>&1` (which doesn't translate to cmd).
|
|
bool ok = (RunCommandChecked(runner.probe).exitCode == 0);
|
|
probeCache[runner.name] = ok;
|
|
return ok;
|
|
};
|
|
|
|
auto worker = [&]() {
|
|
while (true) {
|
|
std::size_t i = next.fetch_add(1);
|
|
if (i >= filtered.size()) break;
|
|
Test& t = *filtered[i];
|
|
TestResult r;
|
|
r.name = t.config.name;
|
|
|
|
if (!runnerAvailable(t.runner)) {
|
|
r.outcome = TestOutcome::Skipped;
|
|
r.output = std::format("runner '{}' not available", t.runner.name);
|
|
{
|
|
std::lock_guard lk(printMutex);
|
|
PrintResult(r, t.runner.name);
|
|
}
|
|
results[i] = std::move(r);
|
|
continue;
|
|
}
|
|
|
|
BuildResult br;
|
|
try {
|
|
br = Build(t.config, depResults, depMutex);
|
|
} catch (const std::exception& e) {
|
|
r.outcome = TestOutcome::Fail;
|
|
r.output = std::format("build threw: {}", e.what());
|
|
r.exitCode = -1;
|
|
{
|
|
std::lock_guard lk(printMutex);
|
|
PrintResult(r, t.runner.name);
|
|
}
|
|
results[i] = std::move(r);
|
|
continue;
|
|
}
|
|
|
|
if (!br.result.empty()) {
|
|
r.outcome = TestOutcome::Fail;
|
|
r.output = std::format("build failed: {}", br.result);
|
|
r.exitCode = -1;
|
|
{
|
|
std::lock_guard lk(printMutex);
|
|
PrintResult(r, t.runner.name);
|
|
}
|
|
results[i] = std::move(r);
|
|
continue;
|
|
}
|
|
|
|
std::chrono::seconds timeout = opts.timeoutOverride.value_or(t.timeout);
|
|
fs::path binary = TestBinaryPath(t.config);
|
|
try {
|
|
r = RunSingleTest(t, binary, timeout);
|
|
} catch (const std::exception& e) {
|
|
r.outcome = TestOutcome::Fail;
|
|
r.output = std::format("runner threw: {}", e.what());
|
|
r.exitCode = -1;
|
|
}
|
|
|
|
if (r.outcome != TestOutcome::Pass && r.outcome != TestOutcome::Skipped && !r.output.empty()) {
|
|
WriteLog(projectCfg.path, r.name, r.output);
|
|
}
|
|
|
|
{
|
|
std::lock_guard lk(printMutex);
|
|
PrintResult(r, t.runner.name);
|
|
}
|
|
results[i] = std::move(r);
|
|
}
|
|
};
|
|
|
|
std::vector<std::jthread> threads;
|
|
threads.reserve(jobs);
|
|
for (int j = 0; j < jobs; ++j) {
|
|
threads.emplace_back(worker);
|
|
}
|
|
threads.clear(); // joins all jthreads
|
|
|
|
for (auto& r : discoveryFailures) {
|
|
results.push_back(std::move(r));
|
|
}
|
|
|
|
for (auto& r : results) {
|
|
switch (r.outcome) {
|
|
case TestOutcome::Pass: summary.passed++; break;
|
|
case TestOutcome::Fail: summary.failed++; break;
|
|
case TestOutcome::Crash: summary.crashed++; break;
|
|
case TestOutcome::Timeout: summary.timedOut++; break;
|
|
case TestOutcome::Skipped: summary.skipped++; break;
|
|
}
|
|
}
|
|
summary.results = std::move(results);
|
|
|
|
Progress::Clear();
|
|
std::print("\n");
|
|
std::vector<std::string> parts;
|
|
if (summary.passed) parts.push_back(std::format("{} passed", summary.passed));
|
|
if (summary.failed) parts.push_back(std::format("{} failed", summary.failed));
|
|
if (summary.crashed) parts.push_back(std::format("{} crashed", summary.crashed));
|
|
if (summary.timedOut) parts.push_back(std::format("{} timed out", summary.timedOut));
|
|
if (summary.skipped) parts.push_back(std::format("{} skipped", summary.skipped));
|
|
std::string joined;
|
|
for (std::size_t i = 0; i < parts.size(); ++i) {
|
|
if (i) joined += ", ";
|
|
joined += parts[i];
|
|
}
|
|
std::println("{}", joined);
|
|
|
|
return summary;
|
|
}
|