fix: rebuild and relink on what changed, submodules at their revisions, and mingw found where it is
Some checks failed
Flatpak / flatpak (push) Failing after 1m12s
Some checks failed
Flatpak / flatpak (push) Failing after 1m12s
- Shaders record the files glslang opened (a depfile) and their settings
(source path, stage, entry point, defines, include directories) in the
build directory, and rebuild when an included file or a setting changes,
not only the shader's own source.
- An executable relinks when its link command changed, not only when an
object is newer: a link flag in project.cpp, or a dependency's library
that moved (an ExternalDependency built into a new cache directory). The
command is recorded beside the objects; its flags come from ordered sets,
so the order the dependency threads finish in can't change it.
- An external's newest artifact counts its libDirs and versioned .so names.
- Submodules are checked out at the revisions the fetched commit records,
from the URLs its .gitmodules gives, after every clone, switch, checkout
and pull, so a pinned commit builds against its own submodules, and one
moved by hand is put back; with no .gitmodules, git isn't asked.
- Each external clone's fetch and build hold a file lock: parallel builds
pulled into one clone at once ("cannot lock ref").
- mingw:
- -stdlib=libstdc++ always;
- the std module's source is found through libstdc++.modules.json in the
mingw toolchain's own library directories (the generic lib/ holds the
host's Linux one), asking the compiler once a run;
- a toolchain that isn't there is a build error saying where it looked,
not std::terminate;
- CRAFTER_MINGW_DIR names the prefix, the directory holding
x86_64-w64-mingw32/ and lib/gcc/;
- inside the Flatpak the host's /usr mingw is used.
- RunScript's exit-status check reads as the one range it is.
- Tests: ConcurrentExternalFetch, ExternalArtifactTime, GitSubmodules,
LinkFlagsChange, MingwMissing, ShaderIncludeChange; ShaderCompile and
ToolchainResolve extended.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
959c65dbb2
commit
653dd0bb9b
GPG key ID:
2937E59CDCC1BCFB
26 changed files with 1075 additions and 109 deletions
|
|
@ -535,14 +535,14 @@ BuildResult Crafter::Build(Configuration& config, std::unordered_map<fs::path, s
|
|||
}
|
||||
for (const Shader& shader : config.shaders) {
|
||||
if (buildCancelled.load(std::memory_order_relaxed)) break;
|
||||
if (shader.Check(outputDir)) continue;
|
||||
threads.emplace_back([&shader, &outputDir, &shaderIncludeDirs, &buildError, &buildCancelled]() {
|
||||
if (shader.Check(outputDir, buildDir, shaderIncludeDirs)) continue;
|
||||
threads.emplace_back([&shader, &outputDir, &buildDir, &shaderIncludeDirs, &buildError, &buildCancelled]() {
|
||||
// A source registered more than once shows up once per variant;
|
||||
// name the variant so the lines can be told apart.
|
||||
Progress::Task task(shader.variant.empty() ? std::format("Compiling shader {}", shader.path.filename().string()) : std::format("Compiling shader {} ({})", shader.path.filename().string(), shader.variant));
|
||||
if (buildCancelled.load(std::memory_order_relaxed)) return;
|
||||
|
||||
std::string result = shader.Compile(outputDir, shaderIncludeDirs);
|
||||
std::string result = shader.Compile(outputDir, buildDir, shaderIncludeDirs);
|
||||
if (result.empty()) return;
|
||||
|
||||
bool expected = false;
|
||||
|
|
@ -699,7 +699,13 @@ BuildResult Crafter::Build(Configuration& config, std::unordered_map<fs::path, s
|
|||
std::string stdPcmResult;
|
||||
{
|
||||
Progress::Task task(std::format("Building std PCM ({}-{})", config.target, config.march));
|
||||
stdPcmResult = BuildStdPcm(config, stdPcmDir/"std.pcm");
|
||||
// The shader and external threads are running: an exception leaving
|
||||
// here would destroy them joinable, which terminates the process.
|
||||
try {
|
||||
stdPcmResult = BuildStdPcm(config, stdPcmDir/"std.pcm");
|
||||
} catch (const std::exception& e) {
|
||||
stdPcmResult = std::format("building the std module for {}: {}", config.target, e.what());
|
||||
}
|
||||
}
|
||||
if(!stdPcmResult.empty()) {
|
||||
buildCancelled.store(true);
|
||||
|
|
@ -728,8 +734,8 @@ BuildResult Crafter::Build(Configuration& config, std::unordered_map<fs::path, s
|
|||
const bool isWasm = config.target.starts_with("wasm32");
|
||||
|
||||
std::string files;
|
||||
std::unordered_set<std::string> libSet;
|
||||
std::unordered_set<std::string> publicFlagSet;
|
||||
std::set<std::string> libSet;
|
||||
std::set<std::string> publicFlagSet;
|
||||
std::mutex fileMutex;
|
||||
std::vector<std::thread> depThreads;
|
||||
depThreads.reserve(config.dependencies.size());
|
||||
|
|
@ -804,6 +810,9 @@ BuildResult Crafter::Build(Configuration& config, std::unordered_map<fs::path, s
|
|||
: std::format(" -march={} -mtune={}", config.march, config.mtune);
|
||||
if (!config.sysroot.empty()) {
|
||||
cArchFlags += std::format(" --sysroot={}", config.sysroot);
|
||||
} else if (config.target == "x86_64-w64-mingw32" && !GetToolchain().mingwSysroot.empty()) {
|
||||
// The C++ side gets it through GetBaseCommand.
|
||||
cArchFlags += std::format(" --sysroot={}", GetToolchain().mingwSysroot.string());
|
||||
}
|
||||
if (cIsWasm) {
|
||||
// Matches the C++ path's wasi flag set so any libc shim defines
|
||||
|
|
@ -1149,8 +1158,48 @@ BuildResult Crafter::Build(Configuration& config, std::unordered_map<fs::path, s
|
|||
}
|
||||
}
|
||||
|
||||
// The exact command the link (or archive) step runs, kept beside the
|
||||
// objects once it has succeeded. A different one relinks even with no
|
||||
// object newer than the output: a link flag changed in project.cpp, or a
|
||||
// dependency's library moved, as an ExternalDependency's does when its
|
||||
// options change and it is built into a new cache directory.
|
||||
std::string linkCommand;
|
||||
if (config.type == ConfigurationType::Executable) {
|
||||
if (config.target.starts_with("wasm32")) {
|
||||
linkCommand = std::format("{}{} -o {}.wasm -fuse-ld=lld{}", command, files, (outputDir/config.outputName).string(), linkExtras);
|
||||
} else {
|
||||
linkCommand = std::format("{}{} -o {} -fuse-ld=lld{}", command, files, (outputDir/config.outputName).string(), linkExtras);
|
||||
}
|
||||
} else if (config.type == ConfigurationType::LibraryStatic) {
|
||||
// ThinLTO emits LLVM bitcode objects; plain `ar` writes an archive
|
||||
// index that omits their symbols, so a consumer's lld link can't
|
||||
// pull the needed members. llvm-ar writes a bitcode-aware index
|
||||
// and handles plain ELF and COFF objects just as well, so every
|
||||
// build uses it.
|
||||
linkCommand = std::format("{} rcs {}.a {}", GetToolchain().llvmAr.string(), (outputDir/fs::path(std::string("lib")+config.outputName)).string(), files);
|
||||
} else if (config.target == "x86_64-w64-mingw32") {
|
||||
// LibraryDynamic. Output names follow each target's convention so
|
||||
// consumers can link via the standard linker search:
|
||||
// mingw: <name>.dll + lib<name>.dll.a (lld --out-implib)
|
||||
// unix: lib<name>.so (rpath $ORIGIN)
|
||||
fs::path dll = outputDir / std::format("{}.dll", config.outputName);
|
||||
fs::path implib = outputDir / std::format("lib{}.dll.a", config.outputName);
|
||||
linkCommand = std::format("{}{} -shared -o {} -Wl,--out-implib,{} -fuse-ld=lld{}", command, files, dll.string(), implib.string(), linkExtras);
|
||||
} else {
|
||||
linkCommand = std::format("{}{} -shared -o {}.so -Wl,-rpath,'$ORIGIN' -fuse-ld=lld{}", command, files, (outputDir/(std::string("lib")+config.outputName)).string(), linkExtras);
|
||||
}
|
||||
const fs::path linkRecord = buildDir / std::format("{}.link", config.outputName);
|
||||
if (!buildResult.repack) {
|
||||
std::ifstream recorded(linkRecord, std::ios::binary);
|
||||
const std::string previous((std::istreambuf_iterator<char>(recorded)), std::istreambuf_iterator<char>());
|
||||
if (previous != linkCommand) buildResult.repack = true;
|
||||
}
|
||||
|
||||
if(buildResult.repack) {
|
||||
Progress::Task task(std::format("Linking {}", config.outputName));
|
||||
// The record goes first, so a link that fails leaves it stale.
|
||||
std::error_code removed;
|
||||
fs::remove(linkRecord, removed);
|
||||
if(config.type == ConfigurationType::Executable) {
|
||||
if(config.target == "x86_64-w64-mingw32") {
|
||||
try {
|
||||
|
|
@ -1181,30 +1230,10 @@ BuildResult Crafter::Build(Configuration& config, std::unordered_map<fs::path, s
|
|||
return {e.what(), false, {}};
|
||||
}
|
||||
}
|
||||
if (config.target.starts_with("wasm32")) {
|
||||
buildResult.result = RunBuildCommand(std::format("{}{} -o {}.wasm -fuse-ld=lld{}", command, files, (outputDir/config.outputName).string(), linkExtras));
|
||||
} else {
|
||||
buildResult.result = RunBuildCommand(std::format("{}{} -o {} -fuse-ld=lld{}", command, files, (outputDir/config.outputName).string(), linkExtras));
|
||||
}
|
||||
} else if(config.type == ConfigurationType::LibraryStatic) {
|
||||
// ThinLTO emits LLVM bitcode objects; plain `ar` writes an archive
|
||||
// index that omits their symbols, so a consumer's lld link can't
|
||||
// pull the needed members. llvm-ar writes a bitcode-aware index
|
||||
// and handles plain ELF and COFF objects just as well, so every
|
||||
// build uses it.
|
||||
buildResult.result = RunBuildCommand(std::format("{} rcs {}.a {}", GetToolchain().llvmAr.string(), (outputDir/fs::path(std::string("lib")+config.outputName)).string(), files));
|
||||
} else {
|
||||
// LibraryDynamic. Output names follow each target's convention so
|
||||
// consumers can link via the standard linker search:
|
||||
// mingw: <name>.dll + lib<name>.dll.a (lld --out-implib)
|
||||
// unix: lib<name>.so (rpath $ORIGIN)
|
||||
if (config.target == "x86_64-w64-mingw32") {
|
||||
fs::path dll = outputDir / std::format("{}.dll", config.outputName);
|
||||
fs::path implib = outputDir / std::format("lib{}.dll.a", config.outputName);
|
||||
buildResult.result = RunBuildCommand(std::format("{}{} -shared -o {} -Wl,--out-implib,{} -fuse-ld=lld{}", command, files, dll.string(), implib.string(), linkExtras));
|
||||
} else {
|
||||
buildResult.result = RunBuildCommand(std::format("{}{} -shared -o {}.so -Wl,-rpath,'$ORIGIN' -fuse-ld=lld{}", command, files, (outputDir/(std::string("lib")+config.outputName)).string(), linkExtras));
|
||||
}
|
||||
}
|
||||
buildResult.result = RunBuildCommand(linkCommand);
|
||||
if (buildResult.result.empty()) {
|
||||
std::ofstream(linkRecord, std::ios::binary | std::ios::trunc) << linkCommand;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -1689,7 +1718,10 @@ Environment:
|
|||
instead of the compiler's own.
|
||||
CRAFTER_BUILD_LIBCLANG Exact libclang to load for lint.
|
||||
CRAFTER_BUILD_WASI_SYSROOT WASI sysroot for wasm32 targets.
|
||||
CRAFTER_MINGW_DIR mingw-w64 sysroot, passed as --sysroot.
|
||||
CRAFTER_MINGW_DIR mingw-w64 prefix (holding x86_64-w64-mingw32/
|
||||
and lib/gcc/), passed as --sysroot. Default:
|
||||
the mingw gcc on PATH; in the Flatpak, the
|
||||
host's /usr when it has mingw-w64.
|
||||
CRAFTER_BUILD_HOME Directory holding the module sources.
|
||||
|
||||
Exit status:
|
||||
|
|
|
|||
|
|
@ -36,6 +36,18 @@ std::string FetchGit(const GitSource& source, const fs::path& cloneDir) {
|
|||
if (r.exitCode != 0) return std::format("`{}` failed (exit {}): {}", cmd, r.exitCode, r.output);
|
||||
return std::nullopt;
|
||||
};
|
||||
// Submodules follow the checkout only when asked to: after every clone,
|
||||
// switch, checkout or pull, so they match the revision that was fetched
|
||||
// instead of whichever one was checked out before, from the URLs that
|
||||
// revision's .gitmodules gives (sync). A no-op, and offline, when they
|
||||
// already match, which puts back submodules moved by hand; and no git at
|
||||
// all without any.
|
||||
auto updateSubmodules = [&runGit](const fs::path& dir) -> std::optional<std::string> {
|
||||
if (!fs::exists(dir / ".gitmodules")) return std::nullopt;
|
||||
const std::string quoted = ShellQuote(dir.string());
|
||||
if (auto err = runGit(std::format("git -C {} submodule sync --recursive", quoted))) return err;
|
||||
return runGit(std::format("git -C {} submodule update --init --recursive", quoted));
|
||||
};
|
||||
|
||||
if (fs::exists(cloneDir)) {
|
||||
CommandResult remote = RunCommandChecked(std::format("git -C {} remote get-url origin", ShellQuote(cloneDir.string())));
|
||||
|
|
@ -51,6 +63,9 @@ std::string FetchGit(const GitSource& source, const fs::path& cloneDir) {
|
|||
if (auto err = runGit(std::format("git -C {} fetch origin", ShellQuote(cloneDir.string())))) return *err;
|
||||
if (auto err = runGit(std::format("git -C {} checkout {}", ShellQuote(cloneDir.string()), ShellQuote(source.commit)))) return *err;
|
||||
}
|
||||
// Also when the commit already matched: a clone made before
|
||||
// submodules were updated has them at the default branch's.
|
||||
if (auto err = updateSubmodules(cloneDir)) return *err;
|
||||
return "";
|
||||
} else {
|
||||
// No commit pinned — pull latest. The user opted into "track latest"
|
||||
|
|
@ -61,6 +76,7 @@ std::string FetchGit(const GitSource& source, const fs::path& cloneDir) {
|
|||
} else {
|
||||
if (auto err = runGit(std::format("git -C {} pull", ShellQuote(cloneDir.string())))) return *err;
|
||||
}
|
||||
if (auto err = updateSubmodules(cloneDir)) return *err;
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
|
@ -73,7 +89,9 @@ std::string FetchGit(const GitSource& source, const fs::path& cloneDir) {
|
|||
fs::path tmpDir = cloneDir;
|
||||
tmpDir += std::format(".tmp.{:08x}{:08x}", rd(), rd());
|
||||
|
||||
if (auto err = runGit(std::format("git clone --recursive {} {}", ShellQuote(source.url), ShellQuote(tmpDir.string())))) {
|
||||
// Not --recursive: that checks out the default branch's submodules, and
|
||||
// the requested revision's come once it is checked out.
|
||||
if (auto err = runGit(std::format("git clone {} {}", ShellQuote(source.url), ShellQuote(tmpDir.string())))) {
|
||||
if (fs::exists(tmpDir)) fs::remove_all(tmpDir);
|
||||
return *err;
|
||||
}
|
||||
|
|
@ -89,6 +107,10 @@ std::string FetchGit(const GitSource& source, const fs::path& cloneDir) {
|
|||
return *err;
|
||||
}
|
||||
}
|
||||
if (auto err = updateSubmodules(tmpDir)) {
|
||||
fs::remove_all(tmpDir);
|
||||
return *err;
|
||||
}
|
||||
|
||||
std::error_code ec;
|
||||
fs::rename(tmpDir, cloneDir, ec);
|
||||
|
|
@ -124,6 +146,10 @@ std::string BuildInjectedCMakeFlags(const fs::path& cmakeBuildDir, std::string_v
|
|||
out += " -DCMAKE_SYSTEM_NAME=Windows";
|
||||
out += std::format(" -DCMAKE_C_COMPILER_TARGET={}", target);
|
||||
out += std::format(" -DCMAKE_CXX_COMPILER_TARGET={}", target);
|
||||
// The libstdc++ and the mingw-w64 install the project's own compiles
|
||||
// use (GetBaseCommand).
|
||||
out += " -DCMAKE_CXX_FLAGS=-stdlib=libstdc++";
|
||||
if (!tc.mingwSysroot.empty()) out += std::format(" -DCMAKE_SYSROOT={}", ShellQuote(tc.mingwSysroot.string()));
|
||||
}
|
||||
fs::path absBuildDir = fs::absolute(cmakeBuildDir);
|
||||
out += std::format(" -DCMAKE_ARCHIVE_OUTPUT_DIRECTORY={}", ShellQuote(absBuildDir.string()));
|
||||
|
|
@ -227,6 +253,9 @@ ExternalBuildResult Crafter::BuildExternal(const ExternalDependency& dep, std::s
|
|||
}
|
||||
|
||||
fs::path cloneDir = ExternalCloneDir(dep, target);
|
||||
// Builds running side by side (tests built in parallel, two
|
||||
// crafter-builds) share the clone: one fetches and builds it at a time.
|
||||
FileLock lock(fs::path(std::format("{}.lock", cloneDir.string())));
|
||||
|
||||
std::string fetchErr = FetchGit(dep.source, cloneDir);
|
||||
if (!fetchErr.empty()) {
|
||||
|
|
@ -273,13 +302,20 @@ ExternalBuildResult Crafter::BuildExternal(const ExternalDependency& dep, std::s
|
|||
result.linkFlags.push_back(std::format("-l{}", lib));
|
||||
}
|
||||
|
||||
if (fs::exists(cmakeBuildDir)) {
|
||||
for (const auto& entry : fs::directory_iterator(cmakeBuildDir)) {
|
||||
if (!entry.is_regular_file()) continue;
|
||||
const fs::path& p = entry.path();
|
||||
if (p.extension() != ".a" && p.extension() != ".so") continue;
|
||||
// The newest library in every directory the link searches: the build
|
||||
// directory and the libDirs, where projects like msquic put theirs,
|
||||
// versioned names (libmsquic.so.2) included.
|
||||
auto isLibrary = [](const std::string& name) {
|
||||
return name.ends_with(".a") || name.ends_with(".so") || name.contains(".so.") || name.ends_with(".dll");
|
||||
};
|
||||
std::vector<fs::path> searched = { cmakeBuildDir };
|
||||
for (const fs::path& libDir : dep.libDirs) searched.push_back(libDir.is_absolute() ? libDir : cmakeBuildDir / libDir);
|
||||
for (const fs::path& dir : searched) {
|
||||
std::error_code listEc;
|
||||
for (fs::directory_iterator it(dir, listEc), end; !listEc && it != end; it.increment(listEc)) {
|
||||
std::error_code ec;
|
||||
fs::file_time_type t = entry.last_write_time(ec);
|
||||
if (!it->is_regular_file(ec) || !isLibrary(it->path().filename().string())) continue;
|
||||
fs::file_time_type t = it->last_write_time(ec);
|
||||
if (!ec && t > result.latestArtifact) result.latestArtifact = t;
|
||||
}
|
||||
}
|
||||
|
|
@ -355,6 +391,8 @@ Configuration* Crafter::GitProject(const GitProjectSpec& spec) {
|
|||
fs::path cloneDir = externalRoot / std::format("{}-{}", name, srcHash);
|
||||
|
||||
{
|
||||
// One fetch of a clone at a time, as in BuildExternal.
|
||||
FileLock lock(fs::path(std::format("{}.lock", cloneDir.string())));
|
||||
const bool exists = fs::exists(cloneDir);
|
||||
Progress::Task task(std::format("{} {}", exists ? "Updating" : "Cloning", name));
|
||||
if (std::string err = FetchGit(spec.source, cloneDir); !err.empty()) {
|
||||
|
|
@ -396,6 +434,8 @@ fs::path Crafter::GitFetch(const GitSource& source) {
|
|||
fs::path cloneDir = externalRoot / std::format("{}-{:016x}", name, key);
|
||||
|
||||
{
|
||||
// One fetch of a clone at a time, as in BuildExternal.
|
||||
FileLock lock(fs::path(std::format("{}.lock", cloneDir.string())));
|
||||
const bool exists = fs::exists(cloneDir);
|
||||
Progress::Task task(std::format("{} {}", exists ? "Updating" : "Cloning", name));
|
||||
if (std::string err = FetchGit(source, cloneDir); !err.empty()) {
|
||||
|
|
|
|||
|
|
@ -33,27 +33,10 @@ namespace {
|
|||
// pass through the lock before it reaches the compile steps that *read* those
|
||||
// PCMs, so a reader never observes a half-written file.
|
||||
//
|
||||
// The lock lives in the OS, keyed on a `.lock` file in the cache dir, and is
|
||||
// released automatically when the descriptor/handle closes — including on
|
||||
// process death — so an interrupted build can never leave a stale lock. The
|
||||
// descriptor is close-on-exec / non-inheritable so spawned clang processes
|
||||
// don't keep the lock alive past our own release.
|
||||
class CacheLock {
|
||||
std::int32_t fd_ = -1;
|
||||
public:
|
||||
explicit CacheLock(const fs::path& cacheDir) {
|
||||
fd_ = ::open((cacheDir / ".lock").c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0644);
|
||||
if (fd_ >= 0) {
|
||||
while (::flock(fd_, LOCK_EX) != 0 && errno == EINTR) {}
|
||||
}
|
||||
}
|
||||
~CacheLock() {
|
||||
if (fd_ >= 0) {
|
||||
::close(fd_); // closing the descriptor drops the flock
|
||||
}
|
||||
}
|
||||
CacheLock(const CacheLock&) = delete;
|
||||
CacheLock& operator=(const CacheLock&) = delete;
|
||||
// A FileLock on a `.lock` file in the cache dir.
|
||||
struct CacheLock {
|
||||
FileLock lock;
|
||||
explicit CacheLock(const fs::path& cacheDir) : lock(cacheDir / ".lock") {}
|
||||
};
|
||||
|
||||
// The cached Crafter.Build PCMs are keyed by `<target>-<march>` alone, so
|
||||
|
|
@ -120,6 +103,51 @@ namespace {
|
|||
buffer << in.rdbuf();
|
||||
return buffer.str();
|
||||
}
|
||||
|
||||
// The string value of `"key": "..."` in `object`, if it has one.
|
||||
std::optional<std::string> JsonString(std::string_view object, std::string_view key) {
|
||||
const std::string quoted = std::format("\"{}\"", key);
|
||||
std::size_t at = object.find(quoted);
|
||||
if (at == std::string_view::npos) return std::nullopt;
|
||||
at = object.find_first_not_of(" \t\r\n", at + quoted.size());
|
||||
if (at == std::string_view::npos || object[at] != ':') return std::nullopt;
|
||||
at = object.find_first_not_of(" \t\r\n", at + 1);
|
||||
if (at == std::string_view::npos || object[at] != '"') return std::nullopt;
|
||||
std::string value;
|
||||
for (std::size_t i = at + 1; i < object.size(); ++i) {
|
||||
if (object[i] == '"') return value;
|
||||
if (object[i] == '\\' && i + 1 < object.size()) ++i;
|
||||
value += object[i];
|
||||
}
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
// The std module's "source-path" in a libstdc++.modules.json: the object
|
||||
// in its "modules" array whose "logical-name" is "std". The manifest is
|
||||
// a flat list of such objects, so each one is the text between its
|
||||
// braces.
|
||||
std::optional<std::string> StdModuleSourcePath(std::string_view manifest) {
|
||||
for (std::size_t open = manifest.find('{', manifest.find("\"modules\"")); open != std::string_view::npos; open = manifest.find('{', open + 1)) {
|
||||
const std::size_t close = manifest.find('}', open);
|
||||
if (close == std::string_view::npos) break;
|
||||
const std::string_view object = manifest.substr(open, close - open);
|
||||
if (JsonString(object, "logical-name") == "std") return JsonString(object, "source-path");
|
||||
}
|
||||
return std::nullopt;
|
||||
}
|
||||
}
|
||||
|
||||
Crafter::FileLock::FileLock(const fs::path& path) {
|
||||
fd = ::open(path.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0644);
|
||||
if (fd >= 0) {
|
||||
while (::flock(fd, LOCK_EX) != 0 && errno == EINTR) {}
|
||||
}
|
||||
}
|
||||
|
||||
Crafter::FileLock::~FileLock() {
|
||||
if (fd >= 0) {
|
||||
::close(fd); // closing the descriptor drops the flock
|
||||
}
|
||||
}
|
||||
|
||||
fs::path Crafter::GetCrafterBuildHome() {
|
||||
|
|
@ -464,7 +492,7 @@ namespace {
|
|||
std::mutex depMutex;
|
||||
std::unordered_set<Configuration*> forced;
|
||||
std::unordered_set<Configuration*> walked;
|
||||
std::unordered_set<std::string> libs;
|
||||
std::set<std::string> libs;
|
||||
|
||||
for (const PragmaDependency& pragma : pragmas) {
|
||||
Configuration* cfg = nullptr;
|
||||
|
|
@ -646,23 +674,52 @@ CommandResult Crafter::RunCommandWithTimeout(std::string_view cmd, std::chrono::
|
|||
std::string Crafter::BuildStdPcm(const Configuration& config, fs::path stdPcm) {
|
||||
CacheLock lock(stdPcm.parent_path());
|
||||
if(config.target == "x86_64-w64-mingw32") {
|
||||
std::vector<std::string> folders;
|
||||
|
||||
// Iterate through the directory and collect all subdirectories
|
||||
for (const auto& entry : fs::directory_iterator(GetToolchain().mingwSysroot / "include" / "c++")) {
|
||||
if (entry.is_directory()) {
|
||||
folders.push_back(entry.path().filename().string());
|
||||
// libstdc++ 15 and newer install libstdc++.modules.json beside the
|
||||
// library, naming the std module's source relative to itself. It is
|
||||
// looked for in the library directories the compiler searches for
|
||||
// this target, as it would look for libstdc++ itself, but only in
|
||||
// those that belong to the mingw toolchain: the list also has the
|
||||
// generic <prefix>/lib, where the host's own (Linux) libstdc++ keeps
|
||||
// its manifest. A toolchain that isn't there is a build error, not
|
||||
// an exception: Build() runs this with its worker threads started.
|
||||
const Toolchain& tc = GetToolchain();
|
||||
const std::string where = tc.mingwSysroot.empty()
|
||||
? std::string("where clang finds it by the x86_64-w64-mingw32 gcc on PATH")
|
||||
: std::format("under {}", tc.mingwSysroot.string());
|
||||
const std::string missing = std::format("no mingw-w64 libstdc++ with the std module (libstdc++ 15 or newer) {}. Install mingw-w64, or point CRAFTER_MINGW_DIR at its prefix, the directory holding x86_64-w64-mingw32/ and lib/gcc/", where);
|
||||
// Asked once a run for each compiler and target: every mingw
|
||||
// configuration comes through here, up to date or not, and the answer
|
||||
// costs a clang.
|
||||
static std::mutex probeMutex;
|
||||
static std::unordered_map<std::string, CommandResult> probes;
|
||||
const std::string probe = std::format("{} --target={} -print-search-dirs", GetBaseCommand(config), config.target);
|
||||
CommandResult dirs;
|
||||
{
|
||||
std::lock_guard guard(probeMutex);
|
||||
auto found = probes.find(probe);
|
||||
if (found == probes.end()) found = probes.emplace(probe, RunCommandChecked(probe)).first;
|
||||
dirs = found->second;
|
||||
}
|
||||
if (dirs.exitCode != 0) return std::format("{}: {}", missing, dirs.output);
|
||||
fs::path manifestPath;
|
||||
constexpr std::string_view librariesLine = "libraries: =";
|
||||
if (const std::size_t at = dirs.output.find(librariesLine); at != std::string::npos) {
|
||||
const std::size_t end = dirs.output.find('\n', at);
|
||||
const std::string list = dirs.output.substr(at + librariesLine.size(), end == std::string::npos ? std::string::npos : end - at - librariesLine.size());
|
||||
for (const auto part : std::views::split(list, ':')) {
|
||||
const fs::path dir(std::string_view(part.begin(), part.end()));
|
||||
const bool mingwOwn = std::ranges::contains(dir, fs::path("x86_64-w64-mingw32"));
|
||||
if (mingwOwn && fs::exists(dir / "libstdc++.modules.json")) {
|
||||
manifestPath = dir / "libstdc++.modules.json";
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Sort the folders by version in descending order
|
||||
std::sort(folders.begin(), folders.end(), [](const std::string& a, const std::string& b) {
|
||||
return std::lexicographical_compare(b.begin(), b.end(), a.begin(), a.end());
|
||||
});
|
||||
|
||||
std::string mingWversion = folders.front();
|
||||
|
||||
fs::path stdCc = GetToolchain().mingwSysroot / "include" / "c++" / mingWversion / "bits" / "std.cc";
|
||||
if (manifestPath.empty()) return missing;
|
||||
const std::optional<std::string> source = StdModuleSourcePath(FileBytes(manifestPath));
|
||||
if (!source) return std::format("{} names no std module", manifestPath.string());
|
||||
const fs::path stdCc = (manifestPath.parent_path() / *source).lexically_normal();
|
||||
if (!fs::exists(stdCc)) return std::format("{}, which {} names as the std module, is missing", stdCc.string(), manifestPath.string());
|
||||
|
||||
// -femulated-tls keeps PCM TLS access matching libstdc++'s emutls
|
||||
// definitions; mismatch surfaces as undefined std::__once_callable.
|
||||
|
|
@ -751,9 +808,11 @@ std::string Crafter::GetBaseCommand(const Configuration& config) {
|
|||
const Toolchain& tc = GetToolchain();
|
||||
std::string command = tc.clangxx.string();
|
||||
if (config.target == "x86_64-w64-mingw32") {
|
||||
// libstdc++ from the mingw toolchain, found by clang's own mingw
|
||||
// discovery unless the sysroot was given explicitly.
|
||||
if (tc.mingwSysrootExplicit) command += std::format(" --sysroot={}", tc.mingwSysroot.string());
|
||||
// libstdc++ from the mingw toolchain, said outright: a clang built to
|
||||
// default to libc++ (the Flatpak's) would otherwise reach for that.
|
||||
// Found by clang's own mingw discovery unless there is a sysroot.
|
||||
command += " -stdlib=libstdc++";
|
||||
if (!tc.mingwSysroot.empty()) command += std::format(" --sysroot={}", tc.mingwSysroot.string());
|
||||
return command;
|
||||
}
|
||||
command += " -stdlib=libc++";
|
||||
|
|
|
|||
|
|
@ -87,7 +87,7 @@ returned; ctx.args holds the arguments.)msg");
|
|||
};
|
||||
try {
|
||||
const std::int32_t code = found->run(ctx);
|
||||
if (code == 0 || (code >= 1 && code <= 255)) return code;
|
||||
if (code >= 0 && code <= 255) return code;
|
||||
std::println(std::cerr, "script '{}' returned {}, which is not an exit status (0..255); exiting with 1", found->name, code);
|
||||
return 1;
|
||||
} catch (const std::exception& e) {
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ module;
|
|||
#include "../lib/DirStackFileIncluder.h"
|
||||
module Crafter.Build:Shader_impl;
|
||||
import :Shader;
|
||||
import :Platform;
|
||||
import std;
|
||||
namespace fs = std::filesystem;
|
||||
|
||||
|
|
@ -31,6 +32,39 @@ namespace {
|
|||
}
|
||||
return EShLangVertex;
|
||||
}
|
||||
|
||||
// A path as a Make prerequisite, escaped the way NewestPrerequisite reads it.
|
||||
std::string MakeEscaped(const std::string& path) {
|
||||
std::string escaped;
|
||||
for (const char c : path) {
|
||||
if (c == ' ' || c == '\t' || c == '#') escaped += '\\';
|
||||
if (c == '$') escaped += '$';
|
||||
escaped += c;
|
||||
}
|
||||
return escaped;
|
||||
}
|
||||
|
||||
std::string ReadFile(const fs::path& file) {
|
||||
std::ifstream stream(file, std::ios::binary);
|
||||
return std::string(std::istreambuf_iterator<char>(stream), std::istreambuf_iterator<char>());
|
||||
}
|
||||
|
||||
// What a compile's output depends on besides files' contents, one line
|
||||
// each: two entries of one source differ here, and an #include searched
|
||||
// for from another source directory or in other directories can find
|
||||
// another file.
|
||||
std::string Settings(const Crafter::Shader& shader, std::span<const fs::path> includeDirs) {
|
||||
std::string settings = std::format("source {}\nstage {}\nentry {}\n", fs::absolute(shader.path).string(), static_cast<int>(shader.type), shader.entrypoint);
|
||||
for (const std::string& define : shader.defines) settings += std::format("define {}\n", define);
|
||||
for (const fs::path& dir : includeDirs) settings += std::format("include {}\n", dir.string());
|
||||
return settings;
|
||||
}
|
||||
|
||||
// Where Compile records, for Check, what the output in the bin dir was
|
||||
// built from: beside the configuration's objects, as their depfiles are.
|
||||
fs::path Record(const Crafter::Shader& shader, const fs::path& buildDir, std::string_view kind) {
|
||||
return buildDir / std::format("{}.{}", shader.OutputName().string(), kind);
|
||||
}
|
||||
}
|
||||
|
||||
namespace Crafter {
|
||||
|
|
@ -45,11 +79,25 @@ namespace Crafter {
|
|||
name.replace_extension(variant.empty() ? std::string("spv") : std::format("{}.spv", variant));
|
||||
return name;
|
||||
}
|
||||
bool Shader::Check(const fs::path& outputDir) const {
|
||||
fs::path spv = outputDir / OutputName();
|
||||
return fs::exists(spv) && fs::last_write_time(path) < fs::last_write_time(spv);
|
||||
bool Shader::Check(const fs::path& outputDir, const fs::path& buildDir, std::span<const fs::path> includeDirs) const {
|
||||
std::error_code ec;
|
||||
const fs::file_time_type built = fs::last_write_time(outputDir / OutputName(), ec);
|
||||
if (ec) return false;
|
||||
// The source and every file it #included, as the last compile
|
||||
// recorded them: without a record, or with an included file since
|
||||
// deleted, NewestPrerequisite reads max(), which is stale.
|
||||
if (std::max(fs::last_write_time(path), NewestPrerequisite(Record(*this, buildDir, "d"))) >= built) return false;
|
||||
return ReadFile(Record(*this, buildDir, "settings")) == Settings(*this, includeDirs);
|
||||
}
|
||||
std::string Shader::Compile(const fs::path& outputDir, std::span<const fs::path> includeDirs) const {
|
||||
std::string Shader::Compile(const fs::path& outputDir, const fs::path& buildDir, std::span<const fs::path> includeDirs) const {
|
||||
// The records go first, so a compile that fails leaves the output
|
||||
// stale, whatever it wrote.
|
||||
const fs::path depFile = Record(*this, buildDir, "d");
|
||||
const fs::path settingsFile = Record(*this, buildDir, "settings");
|
||||
std::error_code removed;
|
||||
fs::remove(depFile, removed);
|
||||
fs::remove(settingsFile, removed);
|
||||
|
||||
EShLanguage glslangType = ToEShLanguage(type);
|
||||
glslang::InitializeProcess();
|
||||
|
||||
|
|
@ -157,6 +205,25 @@ namespace Crafter {
|
|||
if (!spvLog.empty()) {
|
||||
return fail("SPIR-V codegen reported issues", std::move(spvLog));
|
||||
}
|
||||
|
||||
// What Check compares the output against next time: the source and
|
||||
// the files it included, as absolute paths so that reading them back
|
||||
// doesn't depend on the directory a build runs from, and the settings.
|
||||
std::ofstream dep(depFile, std::ios::binary);
|
||||
dep << MakeEscaped(fs::absolute(outputDir / filename).string()) << ": " << MakeEscaped(fs::absolute(path).string());
|
||||
for (const std::string& included : includeDir.getIncludedFiles()) {
|
||||
dep << " \\\n " << MakeEscaped(fs::absolute(included).string());
|
||||
}
|
||||
dep << '\n';
|
||||
dep.close();
|
||||
std::ofstream settings(settingsFile, std::ios::binary);
|
||||
settings << Settings(*this, includeDirs);
|
||||
settings.close();
|
||||
if (!dep || !settings) {
|
||||
fs::remove(depFile, removed);
|
||||
fs::remove(settingsFile, removed);
|
||||
return fail("failed to record what the shader was built from", buildDir.string());
|
||||
}
|
||||
glslang::FinalizeProcess();
|
||||
return {};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -137,9 +137,10 @@ Toolchain Crafter::ResolveToolchain(const ToolchainInputs& in) {
|
|||
}
|
||||
if (std::string_view dir = Env(in, "CRAFTER_MINGW_DIR"); !dir.empty()) {
|
||||
tc.mingwSysroot = fs::path(dir);
|
||||
tc.mingwSysrootExplicit = true;
|
||||
} else {
|
||||
tc.mingwSysroot = "/usr/x86_64-w64-mingw32";
|
||||
} else if (!in.hostRoot.empty() && fs::exists(in.hostRoot / "usr" / "x86_64-w64-mingw32")) {
|
||||
// The sandbox has no mingw-w64 of its own, and no mingw gcc on PATH
|
||||
// for clang to find one by.
|
||||
tc.mingwSysroot = in.hostRoot / "usr";
|
||||
}
|
||||
return tc;
|
||||
}
|
||||
|
|
@ -157,6 +158,7 @@ const Toolchain& Crafter::GetToolchain() {
|
|||
try {
|
||||
in.crafterPrefix = GetCrafterBuildHome().parent_path().parent_path();
|
||||
} catch (const std::exception&) {}
|
||||
if (fs::exists("/.flatpak-info")) in.hostRoot = "/run/host";
|
||||
|
||||
Toolchain tc = ResolveToolchain(in);
|
||||
if (!tc.error.empty()) return tc;
|
||||
|
|
|
|||
Loading…
Reference in a new issue