fixes
All checks were successful
CI / build-test-release (push) Successful in 15m14s

This commit is contained in:
Jorijn van der Graaf 2026-05-02 21:08:51 +02:00
commit d7a9c85ea6
11 changed files with 183 additions and 43 deletions

View file

@ -247,6 +247,29 @@ BuildResult Crafter::Build(Configuration& config, std::unordered_map<fs::path, s
BuildResult buildResult; BuildResult buildResult;
// glslang #include search paths for every shader compiled in this
// configuration: each transitive (incl. self) buildFiles entry's parent
// dir, or the entry itself if it points at a directory. Collected once
// up front so the per-shader threads can capture the resulting span by
// reference. No file copy involved — glslang reads the includes in
// place from the dep's source tree.
std::vector<fs::path> shaderIncludeDirs;
{
std::unordered_set<std::string> seenDirs;
std::unordered_set<const Configuration*> seenCfg;
std::function<void(const Configuration*)> collect = [&](const Configuration* c) {
if (!seenCfg.insert(c).second) return;
for (const fs::path& bf : c->buildFiles) {
fs::path dir = fs::is_directory(bf) ? bf : bf.parent_path();
if (seenDirs.insert(dir.string()).second) {
shaderIncludeDirs.push_back(std::move(dir));
}
}
for (const Configuration* sub : c->dependencies) collect(sub);
};
collect(&config);
}
std::vector<std::thread> threads; std::vector<std::thread> threads;
threads.reserve(config.shaders.size() + 1 + config.interfaces.size() + config.implementations.size()); threads.reserve(config.shaders.size() + 1 + config.interfaces.size() + config.implementations.size());
@ -254,11 +277,11 @@ BuildResult Crafter::Build(Configuration& config, std::unordered_map<fs::path, s
std::atomic<bool> buildCancelled{false}; std::atomic<bool> buildCancelled{false};
for (const Shader& shader : config.shaders) { for (const Shader& shader : config.shaders) {
if (shader.Check(outputDir)) continue; if (shader.Check(outputDir)) continue;
threads.emplace_back([&shader, &outputDir, &buildError, &buildCancelled]() { threads.emplace_back([&shader, &outputDir, &shaderIncludeDirs, &buildError, &buildCancelled]() {
Progress::Task task(std::format("Compiling shader {}", shader.path.filename().string())); Progress::Task task(std::format("Compiling shader {}", shader.path.filename().string()));
if (buildCancelled.load(std::memory_order_relaxed)) return; if (buildCancelled.load(std::memory_order_relaxed)) return;
std::string result = shader.Compile(outputDir); std::string result = shader.Compile(outputDir, shaderIncludeDirs);
if (result.empty()) return; if (result.empty()) return;
bool expected = false; bool expected = false;
@ -274,40 +297,26 @@ BuildResult Crafter::Build(Configuration& config, std::unordered_map<fs::path, s
try { try {
for (const fs::path& additionalFile : config.files) { for (const fs::path& additionalFile : config.files) {
fs::path destination = outputDir / additionalFile.filename(); fs::path destination = outputDir / additionalFile.filename();
if (fs::is_directory(additionalFile)) { if (fs::is_directory(additionalFile)) {
for (const auto& entry : fs::recursive_directory_iterator(additionalFile)) { for (const auto& entry : fs::recursive_directory_iterator(additionalFile)) {
const fs::path& sourcePath = entry.path(); const fs::path& sourcePath = entry.path();
// Compute relative path inside the directory
fs::path relativePath = fs::relative(sourcePath, additionalFile); fs::path relativePath = fs::relative(sourcePath, additionalFile);
fs::path destPath = destination / relativePath; fs::path destPath = destination / relativePath;
if (entry.is_directory()) { if (entry.is_directory()) {
// Ensure directory exists in destination if (!fs::exists(destPath)) fs::create_directories(destPath);
if (!fs::exists(destPath)) {
fs::create_directories(destPath);
}
} else if (entry.is_regular_file()) { } else if (entry.is_regular_file()) {
// Ensure parent directory exists
fs::create_directories(destPath.parent_path()); fs::create_directories(destPath.parent_path());
if (!fs::exists(destPath)) { if (!fs::exists(destPath)) {
fs::copy_file(sourcePath, destPath); fs::copy_file(sourcePath, destPath);
} } else if (fs::last_write_time(sourcePath) > fs::last_write_time(destPath)) {
else if (fs::last_write_time(sourcePath) > fs::last_write_time(destPath)) {
fs::copy_file(sourcePath, destPath, fs::copy_options::overwrite_existing); fs::copy_file(sourcePath, destPath, fs::copy_options::overwrite_existing);
} }
} }
} }
} else { } else {
// Handle regular file
if (!fs::exists(destination)) { if (!fs::exists(destination)) {
fs::copy_file(additionalFile, destination); fs::copy_file(additionalFile, destination);
} } else if (fs::last_write_time(additionalFile) > fs::last_write_time(destination)) {
else if (fs::last_write_time(additionalFile) > fs::last_write_time(destination)) {
fs::copy_file(additionalFile, destination, fs::copy_options::overwrite_existing); fs::copy_file(additionalFile, destination, fs::copy_options::overwrite_existing);
} }
} }
@ -572,30 +581,60 @@ BuildResult Crafter::Build(Configuration& config, std::unordered_map<fs::path, s
return {buildError, false, {}}; return {buildError, false, {}};
} }
// Ship .spv files from transitive deps' bin dirs alongside the executable // Ship runtime artifacts from transitive deps' bin dirs alongside the
// so consumers find shaders next to the binary at runtime. Only an exe is // executable: compiled .spv files (cfg.shaders) and asset files/dirs
// a deployment unit; intermediate libs don't need to forward shaders since // (cfg.files). The lib already mirrored these into its own bin dir
// the exe walks all transitive deps. Runs outside the repack gate because // during its build, but a consumer exe loads them from its own dir at
// the relink mtime check above only watches .so/.dll/.a, so a shader-only // runtime. Only an exe is a deployment unit; intermediate libs don't
// change in a dep wouldn't trigger repack but still needs the new .spv // need to forward since the exe walks all transitive deps. Runs outside
// copied across. // the repack gate because the relink mtime check above only watches
// .so/.dll/.a, so a shader/file-only change in a dep wouldn't trigger
// repack but still needs the new artifact copied across.
// (cfg.buildFiles use a different mechanism — they're exposed to shader
// compiles as #include search paths in place, no copy.)
if (config.type == ConfigurationType::Executable) { if (config.type == ConfigurationType::Executable) {
try { try {
std::unordered_set<Configuration*> shaderSeen; auto copyTree = [](const fs::path& src, const fs::path& dest) {
std::function<void(Configuration*)> copyDepShaders = [&](Configuration* dep) { if (fs::is_directory(src)) {
if (!shaderSeen.insert(dep).second) return; for (const auto& entry : fs::recursive_directory_iterator(src)) {
fs::path depDir = dep->BinDir(); fs::path rel = fs::relative(entry.path(), src);
for (const Shader& shader : dep->shaders) { fs::path destPath = dest / rel;
fs::path src = depDir / shader.path.filename().replace_extension("spv"); if (entry.is_directory()) {
if (!fs::exists(src)) continue; if (!fs::exists(destPath)) fs::create_directories(destPath);
fs::path dest = outputDir / src.filename(); } else if (entry.is_regular_file()) {
if (!fs::exists(dest) || fs::last_write_time(src) > fs::last_write_time(dest)) { fs::create_directories(destPath.parent_path());
if (!fs::exists(destPath)) {
fs::copy_file(entry.path(), destPath);
} else if (fs::last_write_time(entry.path()) > fs::last_write_time(destPath)) {
fs::copy_file(entry.path(), destPath, fs::copy_options::overwrite_existing);
}
}
}
} else {
if (!fs::exists(dest)) {
fs::copy_file(src, dest);
} else if (fs::last_write_time(src) > fs::last_write_time(dest)) {
fs::copy_file(src, dest, fs::copy_options::overwrite_existing); fs::copy_file(src, dest, fs::copy_options::overwrite_existing);
} }
} }
for (Configuration* sub : dep->dependencies) copyDepShaders(sub);
}; };
for (Configuration* dep : config.dependencies) copyDepShaders(dep); std::unordered_set<Configuration*> seen;
std::function<void(Configuration*)> forwardDepArtifacts = [&](Configuration* dep) {
if (!seen.insert(dep).second) return;
fs::path depBinDir = dep->BinDir();
for (const Shader& shader : dep->shaders) {
fs::path src = depBinDir / shader.path.filename().replace_extension("spv");
if (!fs::exists(src)) continue;
copyTree(src, outputDir / src.filename());
}
for (const fs::path& additionalFile : dep->files) {
fs::path src = depBinDir / additionalFile.filename();
if (!fs::exists(src)) continue;
copyTree(src, outputDir / additionalFile.filename());
}
for (Configuration* sub : dep->dependencies) forwardDepArtifacts(sub);
};
for (Configuration* dep : config.dependencies) forwardDepArtifacts(dep);
} catch (const fs::filesystem_error& e) { } catch (const fs::filesystem_error& e) {
for(std::thread& thread : threads) thread.join(); for(std::thread& thread : threads) thread.join();
return {e.what(), false, {}}; return {e.what(), false, {}};

View file

@ -317,6 +317,7 @@ namespace {
for (fs::path& p : cfg.cFiles) p = abs(p); for (fs::path& p : cfg.cFiles) p = abs(p);
for (fs::path& p : cfg.cuda) p = abs(p); for (fs::path& p : cfg.cuda) p = abs(p);
for (fs::path& p : cfg.files) p = abs(p); for (fs::path& p : cfg.files) p = abs(p);
for (fs::path& p : cfg.buildFiles) p = abs(p);
for (Shader& s : cfg.shaders) s.path = abs(s.path); for (Shader& s : cfg.shaders) s.path = abs(s.path);
return cfg; return cfg;
} }

View file

@ -57,7 +57,7 @@ namespace Crafter {
fs::path spv = outputDir / path.filename().replace_extension("spv"); fs::path spv = outputDir / path.filename().replace_extension("spv");
return fs::exists(spv) && fs::last_write_time(path) < fs::last_write_time(spv); return fs::exists(spv) && fs::last_write_time(path) < fs::last_write_time(spv);
} }
std::string Shader::Compile(const fs::path& outputDir) const { std::string Shader::Compile(const fs::path& outputDir, std::span<const fs::path> includeDirs) const {
EShLanguage glslangType = ToEShLanguage(type); EShLanguage glslangType = ToEShLanguage(type);
glslang::InitializeProcess(); glslang::InitializeProcess();
@ -98,6 +98,9 @@ namespace Crafter {
shader.setEnvTarget(glslang::EShTargetSpv, glslang::EShTargetSpv_1_4); shader.setEnvTarget(glslang::EShTargetSpv, glslang::EShTargetSpv_1_4);
DirStackFileIncluder includeDir; DirStackFileIncluder includeDir;
includeDir.pushExternalLocalDirectory(path.parent_path().generic_string()); includeDir.pushExternalLocalDirectory(path.parent_path().generic_string());
for (const fs::path& dir : includeDirs) {
includeDir.pushExternalLocalDirectory(dir.generic_string());
}
if (!shader.parse(GetDefaultResources(), 100, false, messages, includeDir)) { if (!shader.parse(GetDefaultResources(), 100, false, messages, includeDir)) {
return fail("GLSL parse failed", std::string(shader.getInfoLog()) + shader.getInfoDebugLog()); return fail("GLSL parse failed", std::string(shader.getInfoLog()) + shader.getInfoDebugLog());

View file

@ -115,6 +115,14 @@ export namespace Crafter {
std::vector<fs::path> cuda; std::vector<fs::path> cuda;
std::vector<Configuration*> dependencies; std::vector<Configuration*> dependencies;
std::vector<fs::path> files; std::vector<fs::path> files;
// Build-time-only source files this configuration exposes to its own
// and its consumers' shader compiles as glslang #include search
// paths. Each entry's parent directory (or the entry itself, if it's
// a directory) is added to the includer for every shader compiled in
// this configuration and in any configuration that transitively
// depends on it. Files are NOT copied — they're read in place from
// the dep's source tree, like C++ -I include dirs.
std::vector<fs::path> buildFiles;
std::vector<Define> defines; std::vector<Define> defines;
std::vector<Shader> shaders; std::vector<Shader> shaders;
std::vector<ExternalDependency> externalDependencies; std::vector<ExternalDependency> externalDependencies;

View file

@ -48,6 +48,6 @@ namespace Crafter {
ShaderType type; ShaderType type;
CRAFTER_API Shader(fs::path&& path, std::string&& entrypoint, ShaderType type); CRAFTER_API Shader(fs::path&& path, std::string&& entrypoint, ShaderType type);
CRAFTER_API bool Check(const fs::path& outputDir) const; CRAFTER_API bool Check(const fs::path& outputDir) const;
CRAFTER_API std::string Compile(const fs::path& outputDir) const; CRAFTER_API std::string Compile(const fs::path& outputDir, std::span<const fs::path> includeDirs) const;
}; };
} }

View file

@ -5,9 +5,16 @@ Catcrafts.net
LGPL-3.0-only. LGPL-3.0-only.
Verifies that when an executable consumes a library that registers a shader, Verifies that when an executable consumes a library that registers shaders
the resulting .spv ships in the executable's bin dir alongside the binary or asset files, those runtime artifacts ship in the executable's bin dir
not just in the lib's own bin dir, where the runtime exe wouldn't find it. alongside the binary not just in the lib's own bin dir, where the runtime
exe wouldn't find them. Covers a .spv (cfg.shaders), a flat asset
(cfg.files file), and a directory tree (cfg.files dir).
Also verifies cfg.buildFiles' include-path semantic: the consumer's own
shader compile resolves `#include "ui-shared.glsl"` against the lib's
buildFiles in-place from the dep's source tree (no copy), and the file is
NOT mirrored into any build/bin dir.
*/ */
import std; import std;
@ -50,6 +57,58 @@ int main() {
return 1; return 1;
} }
fs::path appAsset = cfg.BinDir() / "asset.txt";
if (!fs::exists(appAsset)) {
std::println(std::cerr, "exe-side asset.txt missing at {}", appAsset.string());
return 1;
}
if (ReadFile(appAsset) != "crafter-build asset\n") {
std::println(std::cerr, "asset.txt content mismatch at {}", appAsset.string());
return 1;
}
fs::path appNested = cfg.BinDir() / "data" / "nested.txt";
if (!fs::exists(appNested)) {
std::println(std::cerr, "exe-side data/nested.txt missing at {}", appNested.string());
return 1;
}
if (ReadFile(appNested) != "nested asset\n") {
std::println(std::cerr, "data/nested.txt content mismatch at {}", appNested.string());
return 1;
}
// The consumer shader includes a header from the lib's buildFiles.
// Successful build (above) means glslang resolved the include
// in-place from the dep's source tree — verify the resulting .spv
// and the SPIR-V magic, same as for the lib's shader.
fs::path consumerSpv = cfg.BinDir() / "consumer.spv";
if (!fs::exists(consumerSpv)) {
std::println(std::cerr, "consumer .spv missing at {}", consumerSpv.string());
return 1;
}
std::ifstream cf(consumerSpv, std::ios::binary);
unsigned char cmagic[4] = {};
cf.read(reinterpret_cast<char*>(cmagic), 4);
if (cmagic[0] != 0x03 || cmagic[1] != 0x02 || cmagic[2] != 0x23 || cmagic[3] != 0x07) {
std::println(std::cerr,
"SPIR-V magic mismatch in consumer.spv: got {:#04x} {:#04x} {:#04x} {:#04x}",
cmagic[0], cmagic[1], cmagic[2], cmagic[3]);
return 1;
}
// buildFiles must NOT be copied anywhere — they're read in place.
for (fs::path probe : {
cfg.BinDir() / "ui-shared.glsl",
cfg.BuildDir() / "ui-shared.glsl",
cfg.dependencies[0]->BinDir() / "ui-shared.glsl",
cfg.dependencies[0]->BuildDir() / "ui-shared.glsl",
}) {
if (fs::exists(probe)) {
std::println(std::cerr, "ui-shared.glsl unexpectedly copied to {}", probe.string());
return 1;
}
}
return 0; return 0;
} catch (const std::exception& e) { } catch (const std::exception& e) {
std::println(std::cerr, "test exception: {}", e.what()); std::println(std::cerr, "test exception: {}", e.what());

View file

@ -0,0 +1,16 @@
#version 450
#extension GL_GOOGLE_include_directive : require
#include "ui-shared.glsl"
layout(location = 0) out vec3 color;
void main() {
vec2 positions[3] = vec2[](
vec2( 0.0, -0.5),
vec2( 0.5, 0.5),
vec2(-0.5, 0.5)
);
gl_Position = vec4(positions[gl_VertexIndex], 0.0, 1.0);
color = ui_color();
}

View file

@ -0,0 +1 @@
crafter-build asset

View file

@ -0,0 +1 @@
nested asset

View file

@ -0,0 +1,8 @@
#ifndef UI_SHARED_GLSL
#define UI_SHARED_GLSL
vec3 ui_color() {
return vec3(0.25, 0.5, 1.0);
}
#endif

View file

@ -16,6 +16,9 @@ extern "C" Configuration CrafterBuildProject(std::span<const std::string_view>)
ShaderLib->GetInterfacesAndImplementations(ifaces, impls); ShaderLib->GetInterfacesAndImplementations(ifaces, impls);
} }
ShaderLib->shaders.emplace_back("./lib/triangle.glsl", "main", ShaderType::Vertex); ShaderLib->shaders.emplace_back("./lib/triangle.glsl", "main", ShaderType::Vertex);
ShaderLib->files.emplace_back("./lib/asset.txt");
ShaderLib->files.emplace_back("./lib/data");
ShaderLib->buildFiles.emplace_back("./lib/ui-shared.glsl");
Configuration app; Configuration app;
app.path = "./"; app.path = "./";
@ -29,5 +32,6 @@ extern "C" Configuration CrafterBuildProject(std::span<const std::string_view>)
std::array<fs::path, 1> impls = { "main" }; std::array<fs::path, 1> impls = { "main" };
app.GetInterfacesAndImplementations(ifaces, impls); app.GetInterfacesAndImplementations(ifaces, impls);
} }
app.shaders.emplace_back("./consumer.glsl", "main", ShaderType::Vertex);
return app; return app;
} }