test runner, cross-target runners, lib/exe split
- subprocess-isolated test runner (replaces V1 dlopen-RunTest); Pass/Fail/Crash/Timeout/Skipped outcomes via :Test partition - TestRunner abstraction with command templates: Local, Ssh, SshWin (cmd.exe-shell), QemuUser, FromEnv; probe-based skip when runner unreachable - transitive PCM-path propagation in Build(); resolveImport walks deps recursively; depResults cache keyed by PcmDir() so per-target builds don't collide - cfg.sysroot threaded through BuildStdPcm + base compile/link command (enables aarch64 cross via Arch Linux ARM rootfs) - lib + exe split: project.cpp defines crafterBuildLib (LibraryStatic) + crafterBuildExe (Executable depending on it); build.sh produces lib/libcrafter-build.a alongside bin/crafter-build for downstream static-link consumers - Windows DLL+launcher: CRAFTER_API macro, /EXPORT flag for project.dll's CrafterBuildProject; Crafter::Run as the real entry point with main.cpp as a thin wrapper - 18 tests: HelloWorld/WithModule/Defines/CrossProjectModule/ Diamond × (Linux + sshwin:winvm), plus Incremental, BuildError, Libraries, RunnerClassification, QemuUser, SshRunner, WindowsViaSsh, CrossArchAarch64 - single ./bin/crafter-build test runs everything; Windows variants skip gracefully if winvm SSH alias unreachable Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
parent
f13671b2be
commit
cdfdb976c8
60 changed files with 2029 additions and 104 deletions
|
|
@ -21,6 +21,7 @@ export module Crafter.Build:Clang_impl;
|
|||
import std;
|
||||
import :Clang;
|
||||
import :Platform;
|
||||
import :Test;
|
||||
namespace fs = std::filesystem;
|
||||
using namespace Crafter;
|
||||
|
||||
|
|
@ -35,7 +36,10 @@ void Configuration::GetInterfacesAndImplementations(std::span<fs::path> interfac
|
|||
return true;
|
||||
}
|
||||
}
|
||||
for(Configuration* depCfg : this->dependencies) {
|
||||
|
||||
std::unordered_set<Configuration*> seen;
|
||||
std::function<bool(Configuration*)> walk = [&](Configuration* depCfg) -> bool {
|
||||
if (!seen.insert(depCfg).second) return false;
|
||||
for(const std::unique_ptr<Module>& depInterface : depCfg->interfaces) {
|
||||
if(depInterface->name == importName) {
|
||||
fs::path depPcmPath = (depCfg->PcmDir() / depInterface->path.filename()).string() + ".pcm";
|
||||
|
|
@ -43,6 +47,14 @@ void Configuration::GetInterfacesAndImplementations(std::span<fs::path> interfac
|
|||
return true;
|
||||
}
|
||||
}
|
||||
for(Configuration* sub : depCfg->dependencies) {
|
||||
if (walk(sub)) return true;
|
||||
}
|
||||
return false;
|
||||
};
|
||||
|
||||
for(Configuration* depCfg : this->dependencies) {
|
||||
if (walk(depCfg)) return true;
|
||||
}
|
||||
return false;
|
||||
};
|
||||
|
|
@ -314,6 +326,10 @@ BuildResult Crafter::Build(Configuration& config, std::unordered_map<fs::path, s
|
|||
|
||||
std::string command = std::format("{} --target={} -march={} -mtune={} -std=c++26 -D CRAFTER_BUILD_CONFIGURATION_TARGET=\\\"{}\\\" -D CRAFTER_BUILD_CONFIGURATION_TARGET_{} -fprebuilt-module-path={} -fprebuilt-module-path={}", GetBaseCommand(config), config.target, config.march, config.mtune, editedTarget, editedTarget, stdPcmDir.string(), pcmDir.string());
|
||||
|
||||
if (!config.sysroot.empty()) {
|
||||
command += std::format(" --sysroot={}", config.sysroot);
|
||||
}
|
||||
|
||||
if(config.type == ConfigurationType::LibraryDynamic) {
|
||||
#ifdef CRAFTER_BUILD_CONFIGURATION_TARGET_x86_64_pc_linux_gnu
|
||||
command += " -fPIC -D CRAFTER_BUILD_CONFIGURATION_TYPE_SHARED_LIBRARY";
|
||||
|
|
@ -334,15 +350,26 @@ BuildResult Crafter::Build(Configuration& config, std::unordered_map<fs::path, s
|
|||
depThreads.reserve(config.dependencies.size());
|
||||
std::atomic<bool> repack(false);
|
||||
|
||||
for(Configuration* dep : config.dependencies) {
|
||||
for (const auto& entry : fs::recursive_directory_iterator(dep->path)) {
|
||||
if (entry.is_directory() && entry.path().filename() == "include") {
|
||||
command += " -I" + entry.path().string();
|
||||
{
|
||||
std::unordered_set<Configuration*> seen;
|
||||
std::function<void(Configuration*)> addFlags = [&](Configuration* dep) {
|
||||
if (!seen.insert(dep).second) return;
|
||||
for (const auto& entry : fs::recursive_directory_iterator(dep->path)) {
|
||||
if (entry.is_directory() && entry.path().filename() == "include") {
|
||||
command += " -I" + entry.path().string();
|
||||
}
|
||||
}
|
||||
command += std::format(" -I{} -fprebuilt-module-path={}", dep->path.string(), dep->PcmDir().string());
|
||||
for (Configuration* sub : dep->dependencies) {
|
||||
addFlags(sub);
|
||||
}
|
||||
};
|
||||
for (Configuration* dep : config.dependencies) {
|
||||
addFlags(dep);
|
||||
}
|
||||
}
|
||||
|
||||
command += std::format(" -I{} -fprebuilt-module-path={}", dep->path.string(), dep->PcmDir().string());
|
||||
|
||||
for(Configuration* dep : config.dependencies) {
|
||||
depThreads.emplace_back([&, dep](){
|
||||
try {
|
||||
if (buildCancelled.load(std::memory_order_relaxed)) return;
|
||||
|
|
@ -352,12 +379,13 @@ BuildResult Crafter::Build(Configuration& config, std::unordered_map<fs::path, s
|
|||
bool isBuilder = false;
|
||||
|
||||
depMutex.lock();
|
||||
auto it = depResults.find(dep->path);
|
||||
fs::path cacheKey = dep->PcmDir();
|
||||
auto it = depResults.find(cacheKey);
|
||||
if (it == depResults.end()) {
|
||||
isBuilder = true;
|
||||
promise = std::make_shared<std::promise<BuildResult>>();
|
||||
resultFuture = promise->get_future().share();
|
||||
depResults.emplace(dep->path, resultFuture);
|
||||
depResults.emplace(cacheKey, resultFuture);
|
||||
} else {
|
||||
resultFuture = it->second;
|
||||
}
|
||||
|
|
@ -594,10 +622,23 @@ int Crafter::Run(int argc, char** argv) {
|
|||
std::vector<std::string_view> projectArgs;
|
||||
projectArgs.reserve(argc);
|
||||
|
||||
bool runTests = false;
|
||||
RunTestsOptions testOpts;
|
||||
|
||||
for (int i = 1; i < argc; ++i) {
|
||||
std::string_view arg = argv[i];
|
||||
if (arg.starts_with("--project=")) {
|
||||
if (arg == "test") {
|
||||
runTests = true;
|
||||
} else if (arg.starts_with("--project=")) {
|
||||
projectFile = arg.substr(std::string_view("--project=").size());
|
||||
} else if (runTests && arg.starts_with("--jobs=")) {
|
||||
testOpts.jobs = std::stoi(std::string(arg.substr(std::string_view("--jobs=").size())));
|
||||
} else if (runTests && arg.starts_with("--timeout=")) {
|
||||
testOpts.timeoutOverride = std::chrono::seconds(std::stoi(std::string(arg.substr(std::string_view("--timeout=").size()))));
|
||||
} else if (runTests && arg == "--list") {
|
||||
testOpts.listOnly = true;
|
||||
} else if (runTests && !arg.starts_with("-")) {
|
||||
testOpts.globs.emplace_back(arg);
|
||||
} else {
|
||||
projectArgs.push_back(arg);
|
||||
}
|
||||
|
|
@ -610,6 +651,11 @@ int Crafter::Run(int argc, char** argv) {
|
|||
|
||||
Configuration config = LoadProject(projectFile, projectArgs);
|
||||
|
||||
if (runTests) {
|
||||
TestSummary summary = RunTests(config, testOpts);
|
||||
return summary.AllPassed() ? 0 : 1;
|
||||
}
|
||||
|
||||
std::unordered_map<fs::path, std::shared_future<BuildResult>> depResults;
|
||||
std::mutex depMutex;
|
||||
BuildResult result = Build(config, depResults, depMutex);
|
||||
|
|
|
|||
|
|
@ -59,7 +59,7 @@ std::string Crafter::RunCommand(const std::string_view cmd) {
|
|||
|
||||
CommandResult Crafter::RunCommandChecked(std::string_view cmd) {
|
||||
std::array<char, 128> buffer;
|
||||
CommandResult result{0, ""};
|
||||
CommandResult result{};
|
||||
|
||||
std::string with = "cmd /C \"" + std::string(cmd) + " 2>&1\"";
|
||||
|
||||
|
|
@ -76,6 +76,10 @@ CommandResult Crafter::RunCommandChecked(std::string_view cmd) {
|
|||
return result;
|
||||
}
|
||||
|
||||
CommandResult Crafter::RunCommandWithTimeout(std::string_view, std::chrono::seconds) {
|
||||
throw std::runtime_error("RunCommandWithTimeout not yet implemented on Windows");
|
||||
}
|
||||
|
||||
std::string Crafter::BuildStdPcm(const Configuration& config, fs::path stdPcm) {
|
||||
std::string libcxx = std::getenv("LIBCXX_DIR");
|
||||
std::string stdcppm = std::format("{}\\modules\\c++\\v1\\std.cppm", libcxx);
|
||||
|
|
@ -99,13 +103,14 @@ std::string Crafter::GetBaseCommand(const Configuration& config) {
|
|||
}
|
||||
|
||||
namespace {
|
||||
constexpr std::array<std::string_view, 7> kCrafterBuildModules = {
|
||||
constexpr std::array<std::string_view, 8> kCrafterBuildModules = {
|
||||
"Crafter.Build-Shader",
|
||||
"Crafter.Build-Platform",
|
||||
"Crafter.Build-Interface",
|
||||
"Crafter.Build-Implementation",
|
||||
"Crafter.Build-External",
|
||||
"Crafter.Build-Clang",
|
||||
"Crafter.Build-Test",
|
||||
"Crafter.Build",
|
||||
};
|
||||
|
||||
|
|
@ -121,7 +126,7 @@ namespace {
|
|||
}
|
||||
std::string cmd = std::format(
|
||||
"clang++ --target=x86_64-pc-windows-msvc -march=native -mtune=native "
|
||||
"-std=c++26 -O3 "
|
||||
"-std=c++26 -O3 -D CRAFTER_BUILD_DLL_IMPORT "
|
||||
"-nostdinc++ -nostdlib++ -isystem %LIBCXX_DIR%\\include\\c++\\v1 "
|
||||
"-Wno-reserved-identifier -Wno-reserved-module-identifier "
|
||||
"-fprebuilt-module-path={} "
|
||||
|
|
@ -178,9 +183,11 @@ Configuration Crafter::LoadProject(const fs::path& projectFile, std::span<const
|
|||
|
||||
std::string compileCmd = std::format(
|
||||
"clang++ --target=x86_64-pc-windows-msvc -march=native -mtune=native "
|
||||
"-std=c++26 -shared -O3 -Wno-return-type-c-linkage "
|
||||
"-std=c++26 -shared -O3 -Wno-return-type-c-linkage -fuse-ld=lld "
|
||||
"-D CRAFTER_BUILD_DLL_IMPORT "
|
||||
"-nostdinc++ -nostdlib++ -isystem %LIBCXX_DIR%\\include\\c++\\v1 "
|
||||
"-fprebuilt-module-path={} "
|
||||
"-Wl,/EXPORT:CrafterBuildProject "
|
||||
"{} {} -o {} -L %LIBCXX_DIR%\\lib -lc++",
|
||||
cacheDir.string(),
|
||||
absProject.string(), crafterBuildLib.string(), dllPath.string());
|
||||
|
|
@ -229,7 +236,7 @@ std::string Crafter::RunCommand(const std::string_view cmd) {
|
|||
|
||||
CommandResult Crafter::RunCommandChecked(std::string_view cmd) {
|
||||
std::array<char, 128> buffer;
|
||||
CommandResult result{0, ""};
|
||||
CommandResult result{};
|
||||
|
||||
std::string with = std::string(cmd) + " 2>&1";
|
||||
FILE* pipe = popen(with.c_str(), "r");
|
||||
|
|
@ -243,7 +250,47 @@ CommandResult Crafter::RunCommandChecked(std::string_view cmd) {
|
|||
if (WIFEXITED(status)) {
|
||||
result.exitCode = WEXITSTATUS(status);
|
||||
} else if (WIFSIGNALED(status)) {
|
||||
result.exitCode = 128 + WTERMSIG(status);
|
||||
result.signal = WTERMSIG(status);
|
||||
result.exitCode = 128 + result.signal;
|
||||
result.crashed = true;
|
||||
} else {
|
||||
result.exitCode = -1;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
CommandResult Crafter::RunCommandWithTimeout(std::string_view cmd, std::chrono::seconds timeout) {
|
||||
std::array<char, 128> buffer;
|
||||
CommandResult result{};
|
||||
|
||||
std::string wrapped = std::format(
|
||||
"timeout --kill-after=2 {} {} 2>&1",
|
||||
timeout.count(), cmd);
|
||||
|
||||
FILE* pipe = popen(wrapped.c_str(), "r");
|
||||
if (!pipe) throw std::runtime_error("popen() failed!");
|
||||
|
||||
while (fgets(buffer.data(), buffer.size(), pipe) != nullptr) {
|
||||
result.output += buffer.data();
|
||||
}
|
||||
|
||||
int status = pclose(pipe);
|
||||
if (WIFEXITED(status)) {
|
||||
int code = WEXITSTATUS(status);
|
||||
if (code == 124) {
|
||||
result.timedOut = true;
|
||||
result.exitCode = 124;
|
||||
} else if (code >= 128) {
|
||||
result.signal = code - 128;
|
||||
result.exitCode = code;
|
||||
result.crashed = true;
|
||||
} else {
|
||||
result.exitCode = code;
|
||||
}
|
||||
} else if (WIFSIGNALED(status)) {
|
||||
result.signal = WTERMSIG(status);
|
||||
result.exitCode = 128 + result.signal;
|
||||
result.crashed = true;
|
||||
} else {
|
||||
result.exitCode = -1;
|
||||
}
|
||||
|
|
@ -276,8 +323,14 @@ std::string Crafter::BuildStdPcm(const Configuration& config, fs::path stdPcm) {
|
|||
return "";
|
||||
}
|
||||
} else {
|
||||
if(!fs::exists(stdPcm) || fs::last_write_time(stdPcm) < fs::last_write_time("/usr/share/libc++/v1/std.cppm")) {
|
||||
return RunCommand(std::format("clang++ --target={} -std=c++26 -stdlib=libc++ -march={} -mtune={} -O3 -Wno-reserved-identifier -Wno-reserved-module-identifier --precompile /usr/share/libc++/v1/std.cppm -o {}", config.target, config.march, config.mtune, stdPcm.string()));
|
||||
std::string stdCppm = config.sysroot.empty()
|
||||
? std::string("/usr/share/libc++/v1/std.cppm")
|
||||
: std::format("{}/usr/share/libc++/v1/std.cppm", config.sysroot);
|
||||
std::string sysrootFlag = config.sysroot.empty()
|
||||
? std::string()
|
||||
: std::format(" --sysroot={}", config.sysroot);
|
||||
if(!fs::exists(stdPcm) || fs::last_write_time(stdPcm) < fs::last_write_time(stdCppm)) {
|
||||
return RunCommand(std::format("clang++ --target={} -std=c++26 -stdlib=libc++{} -march={} -mtune={} -O3 -Wno-reserved-identifier -Wno-reserved-module-identifier --precompile {} -o {}", config.target, sysrootFlag, config.march, config.mtune, stdCppm, stdPcm.string()));
|
||||
} else {
|
||||
return "";
|
||||
}
|
||||
|
|
@ -307,13 +360,14 @@ std::string Crafter::GetBaseCommand(const Configuration& config) {
|
|||
}
|
||||
|
||||
namespace {
|
||||
constexpr std::array<std::string_view, 7> kCrafterBuildModules = {
|
||||
constexpr std::array<std::string_view, 8> kCrafterBuildModules = {
|
||||
"Crafter.Build-Shader",
|
||||
"Crafter.Build-Platform",
|
||||
"Crafter.Build-Interface",
|
||||
"Crafter.Build-Implementation",
|
||||
"Crafter.Build-External",
|
||||
"Crafter.Build-Clang",
|
||||
"Crafter.Build-Test",
|
||||
"Crafter.Build",
|
||||
};
|
||||
|
||||
|
|
|
|||
475
implementations/Crafter.Build-Test.cpp
Normal file
475
implementations/Crafter.Build-Test.cpp
Normal file
|
|
@ -0,0 +1,475 @@
|
|||
/*
|
||||
Crafter® Build
|
||||
Copyright (C) 2026 Catcrafts®
|
||||
Catcrafts.net
|
||||
|
||||
This library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License version 3.0 as published by the Free Software Foundation;
|
||||
|
||||
This library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with this library; if not, write to the Free Software
|
||||
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
*/
|
||||
|
||||
export module Crafter.Build:Test_impl;
|
||||
import std;
|
||||
import :Test;
|
||||
import :Clang;
|
||||
import :Platform;
|
||||
namespace fs = std::filesystem;
|
||||
using namespace Crafter;
|
||||
|
||||
namespace {
|
||||
bool TargetIsWindows(std::string_view target) {
|
||||
return target.find("windows") != std::string_view::npos
|
||||
|| target.find("mingw") != std::string_view::npos;
|
||||
}
|
||||
|
||||
fs::path TestBinaryPath(const Configuration& cfg) {
|
||||
fs::path outputDir = cfg.path/"bin"/std::format("{}-{}-{}", cfg.name, cfg.target, cfg.march);
|
||||
return outputDir / (TargetIsWindows(cfg.target) ? cfg.outputName + ".exe" : cfg.outputName);
|
||||
}
|
||||
|
||||
bool MatchGlob(std::string_view glob, std::string_view name) {
|
||||
std::size_t gi = 0, ni = 0, star = std::string_view::npos, mark = 0;
|
||||
while (ni < name.size()) {
|
||||
if (gi < glob.size() && (glob[gi] == '?' || glob[gi] == name[ni])) {
|
||||
++gi; ++ni;
|
||||
} else if (gi < glob.size() && glob[gi] == '*') {
|
||||
star = gi++;
|
||||
mark = ni;
|
||||
} else if (star != std::string_view::npos) {
|
||||
gi = star + 1;
|
||||
ni = ++mark;
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
while (gi < glob.size() && glob[gi] == '*') ++gi;
|
||||
return gi == glob.size();
|
||||
}
|
||||
|
||||
bool MatchAny(std::span<const std::string> globs, std::string_view name) {
|
||||
if (globs.empty()) return true;
|
||||
for (const auto& g : globs) {
|
||||
if (MatchGlob(g, name)) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
std::string ShellQuote(std::string_view s) {
|
||||
std::string out;
|
||||
out.reserve(s.size() + 2);
|
||||
out.push_back('\'');
|
||||
for (char c : s) {
|
||||
if (c == '\'') out += "'\\''";
|
||||
else out.push_back(c);
|
||||
}
|
||||
out.push_back('\'');
|
||||
return out;
|
||||
}
|
||||
|
||||
std::string JoinAndQuoteArgs(std::span<const std::string> args) {
|
||||
std::string out;
|
||||
for (const auto& a : args) {
|
||||
if (!out.empty()) out.push_back(' ');
|
||||
out += ShellQuote(a);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
std::string Substitute(std::string_view tmpl, const std::map<std::string, std::string>& ph) {
|
||||
std::string out;
|
||||
out.reserve(tmpl.size());
|
||||
std::size_t i = 0;
|
||||
while (i < tmpl.size()) {
|
||||
if (tmpl[i] == '{') {
|
||||
std::size_t end = tmpl.find('}', i + 1);
|
||||
if (end != std::string_view::npos) {
|
||||
std::string key(tmpl.substr(i, end - i + 1));
|
||||
if (auto it = ph.find(key); it != ph.end()) {
|
||||
out += it->second;
|
||||
i = end + 1;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
out.push_back(tmpl[i++]);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
std::string SignalName(int sig) {
|
||||
switch (sig) {
|
||||
case 1: return "SIGHUP";
|
||||
case 2: return "SIGINT";
|
||||
case 4: return "SIGILL";
|
||||
case 6: return "SIGABRT";
|
||||
case 8: return "SIGFPE";
|
||||
case 9: return "SIGKILL";
|
||||
case 11: return "SIGSEGV";
|
||||
case 13: return "SIGPIPE";
|
||||
case 14: return "SIGALRM";
|
||||
case 15: return "SIGTERM";
|
||||
default: return std::format("signal {}", sig);
|
||||
}
|
||||
}
|
||||
|
||||
void WriteLog(const fs::path& projectPath, const std::string& name, const std::string& output) {
|
||||
fs::path logDir = projectPath / "build" / "test-logs";
|
||||
std::error_code ec;
|
||||
fs::create_directories(logDir, ec);
|
||||
if (ec) return;
|
||||
std::ofstream(logDir / (name + ".log")) << output;
|
||||
}
|
||||
|
||||
void PrintResult(const TestResult& r, std::string_view runnerName) {
|
||||
auto ms = r.duration.count();
|
||||
std::string runnerSuffix = (runnerName.empty() || runnerName == "local")
|
||||
? std::string()
|
||||
: std::format(" ({})", runnerName);
|
||||
switch (r.outcome) {
|
||||
case TestOutcome::Pass:
|
||||
std::println("✅ {}{} ({}ms)", r.name, runnerSuffix, ms);
|
||||
break;
|
||||
case TestOutcome::Fail:
|
||||
std::println("❌ {}{} ({}ms) exit {}", r.name, runnerSuffix, ms, r.exitCode);
|
||||
if (!r.output.empty()) {
|
||||
for (auto line : std::views::split(r.output, '\n')) {
|
||||
std::string_view sv(line.begin(), line.end());
|
||||
if (!sv.empty()) std::println(" {}", sv);
|
||||
}
|
||||
}
|
||||
break;
|
||||
case TestOutcome::Crash:
|
||||
std::println("\U0001F4A5 {}{} ({}ms) crashed: {}", r.name, runnerSuffix, ms, SignalName(r.signal));
|
||||
if (!r.output.empty()) {
|
||||
for (auto line : std::views::split(r.output, '\n')) {
|
||||
std::string_view sv(line.begin(), line.end());
|
||||
if (!sv.empty()) std::println(" {}", sv);
|
||||
}
|
||||
}
|
||||
break;
|
||||
case TestOutcome::Timeout:
|
||||
std::println("⏱ {}{} ({}ms) timeout", r.name, runnerSuffix, ms);
|
||||
break;
|
||||
case TestOutcome::Skipped:
|
||||
std::println("⏭ {}{} skipped: {}", r.name, runnerSuffix,
|
||||
r.output.empty() ? std::string("(no reason)") : r.output);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TestRunner TestRunner::Local() {
|
||||
TestRunner r;
|
||||
r.name = "local";
|
||||
return r;
|
||||
}
|
||||
|
||||
TestRunner TestRunner::Ssh(std::string host, std::string remoteDir) {
|
||||
TestRunner r;
|
||||
r.name = std::format("ssh:{}", host);
|
||||
r.remoteDir = std::move(remoteDir);
|
||||
r.copy = std::format("ssh -q {0} 'mkdir -p {{remote_bundle}}' && scp -r -q {{bundle}}/. {0}:{{remote_bundle}}/", host);
|
||||
r.exec = std::format("ssh -q {} 'cd {{remote_bundle}} && ./{{bin_name}} {{args}}'", host);
|
||||
r.cleanup = std::format("ssh -q {} 'rm -rf {{remote_bundle}}' || true", host);
|
||||
r.probe = std::format("ssh -q -o BatchMode=yes -o ConnectTimeout=5 {} true > /dev/null 2>&1", host);
|
||||
return r;
|
||||
}
|
||||
|
||||
TestRunner TestRunner::SshWin(std::string host, std::string remoteDir) {
|
||||
TestRunner r;
|
||||
r.name = std::format("sshwin:{}", host);
|
||||
r.remoteDir = std::move(remoteDir);
|
||||
// cmd.exe-friendly templates. Use backslash variants of remote paths because
|
||||
// cmd's mkdir won't auto-create intermediate directories with forward
|
||||
// slashes. scp tolerates either, so we keep forward slashes for the scp
|
||||
// dest (which `{remote_bundle}` provides). 2>nul + `& rem ok` swallows
|
||||
// mkdir's "already exists" error and forces exit code 0.
|
||||
r.copy = std::format("ssh -q {0} \"mkdir {{remote_bundle_win}} 2>nul & rem ok\" && scp -r -q {{bundle}}/. {0}:{{remote_bundle}}/", host);
|
||||
r.exec = std::format("ssh -q {} \"{{bin_win}} {{args}}\"", host);
|
||||
r.cleanup = std::format("ssh -q {} \"rd /s /q {{remote_bundle_win}} 2>nul & rem ok\"", host);
|
||||
r.probe = std::format("ssh -q -o BatchMode=yes -o ConnectTimeout=5 {} \"ver\" > /dev/null 2>&1", host);
|
||||
return r;
|
||||
}
|
||||
|
||||
TestRunner TestRunner::QemuUser(std::string qemuBin) {
|
||||
TestRunner r;
|
||||
r.name = std::format("qemu:{}", qemuBin);
|
||||
r.exec = std::format("{} {{bin}} {{args}}", qemuBin);
|
||||
r.probe = std::format("which {} > /dev/null 2>&1", qemuBin);
|
||||
return r;
|
||||
}
|
||||
|
||||
namespace {
|
||||
std::string NormalizeTriple(std::string_view target) {
|
||||
std::string out(target);
|
||||
for (char& c : out) {
|
||||
if (c == '-' || c == '.') c = '_';
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
std::optional<TestRunner> ParseRunnerSpec(std::string_view spec) {
|
||||
if (spec.empty()) return std::nullopt;
|
||||
std::vector<std::string> parts;
|
||||
for (auto piece : std::views::split(spec, ':')) {
|
||||
parts.emplace_back(std::string_view(piece.begin(), piece.end()));
|
||||
}
|
||||
if (parts.empty()) return std::nullopt;
|
||||
if (parts[0] == "local") return TestRunner::Local();
|
||||
if (parts[0] == "qemu" && parts.size() == 2) return TestRunner::QemuUser(parts[1]);
|
||||
if (parts[0] == "ssh" && parts.size() == 2) return TestRunner::Ssh(parts[1]);
|
||||
if (parts[0] == "ssh" && parts.size() >= 3) {
|
||||
std::string remote;
|
||||
for (std::size_t i = 2; i < parts.size(); ++i) {
|
||||
if (i > 2) remote.push_back(':');
|
||||
remote += parts[i];
|
||||
}
|
||||
return TestRunner::Ssh(parts[1], remote);
|
||||
}
|
||||
if (parts[0] == "sshwin" && parts.size() == 2) return TestRunner::SshWin(parts[1]);
|
||||
if (parts[0] == "sshwin" && parts.size() >= 3) {
|
||||
std::string remote;
|
||||
for (std::size_t i = 2; i < parts.size(); ++i) {
|
||||
if (i > 2) remote.push_back(':');
|
||||
remote += parts[i];
|
||||
}
|
||||
return TestRunner::SshWin(parts[1], remote);
|
||||
}
|
||||
throw std::runtime_error(std::format(
|
||||
"TestRunner::FromEnv: unrecognized runner spec '{}'", spec));
|
||||
}
|
||||
}
|
||||
|
||||
TestRunner TestRunner::FromEnv(std::string_view target, TestRunner fallback) {
|
||||
std::string envName = std::format("CRAFTER_BUILD_RUNNER_{}", NormalizeTriple(target));
|
||||
const char* v = std::getenv(envName.c_str());
|
||||
if (!v || !*v) return fallback;
|
||||
if (auto r = ParseRunnerSpec(v)) return std::move(*r);
|
||||
return fallback;
|
||||
}
|
||||
|
||||
TestResult Crafter::RunSingleTest(const Test& test, const fs::path& binary, std::chrono::seconds timeout) {
|
||||
using namespace std::chrono_literals;
|
||||
TestResult result;
|
||||
result.name = test.config.name;
|
||||
|
||||
std::map<std::string, std::string> ph;
|
||||
ph["{args}"] = JoinAndQuoteArgs(test.args);
|
||||
ph["{bin_name}"] = binary.filename().string();
|
||||
ph["{bundle}"] = binary.parent_path().string();
|
||||
|
||||
auto start = std::chrono::steady_clock::now();
|
||||
CommandResult r;
|
||||
|
||||
if (test.runner.exec.empty()) {
|
||||
// Pure-local runner: spawn the binary directly.
|
||||
std::string cmd = std::format("{} {}", ShellQuote(binary.string()), ph["{args}"]);
|
||||
r = RunCommandWithTimeout(cmd, timeout);
|
||||
} else if (test.runner.copy.empty()) {
|
||||
// Prefix runner (qemu-user, wsl, ...): templated exec wraps a local binary.
|
||||
ph["{bin}"] = binary.string();
|
||||
r = RunCommandWithTimeout(Substitute(test.runner.exec, ph), timeout);
|
||||
} else {
|
||||
// Transport runner (ssh, ...): copy bundle → exec remotely → cleanup.
|
||||
std::string unique = std::format("{}-{}-{}",
|
||||
test.config.name,
|
||||
std::hash<std::thread::id>{}(std::this_thread::get_id()),
|
||||
std::chrono::steady_clock::now().time_since_epoch().count());
|
||||
ph["{remote_bundle}"] = std::format("{}/{}", test.runner.remoteDir, unique);
|
||||
ph["{bin}"] = std::format("{}/{}", ph["{remote_bundle}"], ph["{bin_name}"]);
|
||||
// Backslash variants for cmd.exe-shell remotes (cmd's mkdir won't
|
||||
// auto-create intermediate directories when path uses forward slashes).
|
||||
ph["{remote_bundle_win}"] = ph["{remote_bundle}"];
|
||||
std::replace(ph["{remote_bundle_win}"].begin(), ph["{remote_bundle_win}"].end(), '/', '\\');
|
||||
ph["{bin_win}"] = ph["{bin}"];
|
||||
std::replace(ph["{bin_win}"].begin(), ph["{bin_win}"].end(), '/', '\\');
|
||||
|
||||
CommandResult cp = RunCommandWithTimeout(Substitute(test.runner.copy, ph), 5min);
|
||||
if (cp.exitCode != 0) {
|
||||
auto end = std::chrono::steady_clock::now();
|
||||
result.duration = std::chrono::duration_cast<std::chrono::milliseconds>(end - start);
|
||||
result.outcome = TestOutcome::Fail;
|
||||
result.exitCode = cp.exitCode;
|
||||
result.output = std::format("copy step failed (exit {}):\n{}", cp.exitCode, cp.output);
|
||||
return result;
|
||||
}
|
||||
|
||||
r = RunCommandWithTimeout(Substitute(test.runner.exec, ph), timeout);
|
||||
|
||||
if (!test.runner.cleanup.empty()) {
|
||||
std::system(Substitute(test.runner.cleanup, ph).c_str());
|
||||
}
|
||||
}
|
||||
|
||||
auto end = std::chrono::steady_clock::now();
|
||||
result.duration = std::chrono::duration_cast<std::chrono::milliseconds>(end - start);
|
||||
result.output = std::move(r.output);
|
||||
result.exitCode = r.exitCode;
|
||||
result.signal = r.signal;
|
||||
|
||||
if (r.timedOut) result.outcome = TestOutcome::Timeout;
|
||||
else if (r.crashed) result.outcome = TestOutcome::Crash;
|
||||
else if (r.exitCode != 0) result.outcome = TestOutcome::Fail;
|
||||
else result.outcome = TestOutcome::Pass;
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
TestSummary Crafter::RunTests(Configuration& projectCfg, const RunTestsOptions& opts) {
|
||||
TestSummary summary;
|
||||
|
||||
std::vector<Test*> filtered;
|
||||
filtered.reserve(projectCfg.tests.size());
|
||||
for (auto& test : projectCfg.tests) {
|
||||
if (MatchAny(opts.globs, test.config.name)) {
|
||||
filtered.push_back(&test);
|
||||
}
|
||||
}
|
||||
|
||||
if (opts.listOnly) {
|
||||
for (auto* t : filtered) {
|
||||
std::println("{}", t->config.name);
|
||||
}
|
||||
return summary;
|
||||
}
|
||||
|
||||
if (filtered.empty()) {
|
||||
std::println("No tests matched.");
|
||||
return summary;
|
||||
}
|
||||
|
||||
int jobs = opts.jobs > 0
|
||||
? opts.jobs
|
||||
: std::max(1u, std::thread::hardware_concurrency());
|
||||
jobs = std::min(jobs, static_cast<int>(filtered.size()));
|
||||
|
||||
std::unordered_map<fs::path, std::shared_future<BuildResult>> depResults;
|
||||
std::mutex depMutex;
|
||||
std::mutex printMutex;
|
||||
std::mutex probeMutex;
|
||||
std::unordered_map<std::string, bool> probeCache;
|
||||
std::atomic<std::size_t> next{0};
|
||||
std::vector<TestResult> results(filtered.size());
|
||||
|
||||
auto runnerAvailable = [&](const TestRunner& runner) -> bool {
|
||||
if (runner.probe.empty()) return true;
|
||||
std::lock_guard lk(probeMutex);
|
||||
if (auto it = probeCache.find(runner.name); it != probeCache.end()) {
|
||||
return it->second;
|
||||
}
|
||||
bool ok = (std::system(runner.probe.c_str()) == 0);
|
||||
probeCache[runner.name] = ok;
|
||||
return ok;
|
||||
};
|
||||
|
||||
auto worker = [&]() {
|
||||
while (true) {
|
||||
std::size_t i = next.fetch_add(1);
|
||||
if (i >= filtered.size()) break;
|
||||
Test& t = *filtered[i];
|
||||
TestResult r;
|
||||
r.name = t.config.name;
|
||||
|
||||
if (!runnerAvailable(t.runner)) {
|
||||
r.outcome = TestOutcome::Skipped;
|
||||
r.output = std::format("runner '{}' not available", t.runner.name);
|
||||
{
|
||||
std::lock_guard lk(printMutex);
|
||||
PrintResult(r, t.runner.name);
|
||||
}
|
||||
results[i] = std::move(r);
|
||||
continue;
|
||||
}
|
||||
|
||||
BuildResult br;
|
||||
try {
|
||||
br = Build(t.config, depResults, depMutex);
|
||||
} catch (const std::exception& e) {
|
||||
r.outcome = TestOutcome::Fail;
|
||||
r.output = std::format("build threw: {}", e.what());
|
||||
r.exitCode = -1;
|
||||
{
|
||||
std::lock_guard lk(printMutex);
|
||||
PrintResult(r, t.runner.name);
|
||||
}
|
||||
results[i] = std::move(r);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!br.result.empty()) {
|
||||
r.outcome = TestOutcome::Fail;
|
||||
r.output = std::format("build failed: {}", br.result);
|
||||
r.exitCode = -1;
|
||||
{
|
||||
std::lock_guard lk(printMutex);
|
||||
PrintResult(r, t.runner.name);
|
||||
}
|
||||
results[i] = std::move(r);
|
||||
continue;
|
||||
}
|
||||
|
||||
std::chrono::seconds timeout = opts.timeoutOverride.value_or(t.timeout);
|
||||
fs::path binary = TestBinaryPath(t.config);
|
||||
try {
|
||||
r = RunSingleTest(t, binary, timeout);
|
||||
} catch (const std::exception& e) {
|
||||
r.outcome = TestOutcome::Fail;
|
||||
r.output = std::format("runner threw: {}", e.what());
|
||||
r.exitCode = -1;
|
||||
}
|
||||
|
||||
if (r.outcome != TestOutcome::Pass && r.outcome != TestOutcome::Skipped && !r.output.empty()) {
|
||||
WriteLog(projectCfg.path, r.name, r.output);
|
||||
}
|
||||
|
||||
{
|
||||
std::lock_guard lk(printMutex);
|
||||
PrintResult(r, t.runner.name);
|
||||
}
|
||||
results[i] = std::move(r);
|
||||
}
|
||||
};
|
||||
|
||||
std::vector<std::jthread> threads;
|
||||
threads.reserve(jobs);
|
||||
for (int j = 0; j < jobs; ++j) {
|
||||
threads.emplace_back(worker);
|
||||
}
|
||||
threads.clear(); // joins all jthreads
|
||||
|
||||
for (auto& r : results) {
|
||||
switch (r.outcome) {
|
||||
case TestOutcome::Pass: summary.passed++; break;
|
||||
case TestOutcome::Fail: summary.failed++; break;
|
||||
case TestOutcome::Crash: summary.crashed++; break;
|
||||
case TestOutcome::Timeout: summary.timedOut++; break;
|
||||
case TestOutcome::Skipped: summary.skipped++; break;
|
||||
}
|
||||
}
|
||||
summary.results = std::move(results);
|
||||
|
||||
std::print("\n");
|
||||
std::vector<std::string> parts;
|
||||
if (summary.passed) parts.push_back(std::format("{} passed", summary.passed));
|
||||
if (summary.failed) parts.push_back(std::format("{} failed", summary.failed));
|
||||
if (summary.crashed) parts.push_back(std::format("{} crashed", summary.crashed));
|
||||
if (summary.timedOut) parts.push_back(std::format("{} timed out", summary.timedOut));
|
||||
if (summary.skipped) parts.push_back(std::format("{} skipped", summary.skipped));
|
||||
std::string joined;
|
||||
for (std::size_t i = 0; i < parts.size(); ++i) {
|
||||
if (i) joined += ", ";
|
||||
joined += parts[i];
|
||||
}
|
||||
std::println("{}", joined);
|
||||
|
||||
return summary;
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue