fixes
All checks were successful
CI / build-test-release (push) Successful in 15m14s

This commit is contained in:
Jorijn van der Graaf 2026-05-02 21:08:51 +02:00
commit d7a9c85ea6
11 changed files with 183 additions and 43 deletions

View file

@ -247,6 +247,29 @@ BuildResult Crafter::Build(Configuration& config, std::unordered_map<fs::path, s
BuildResult buildResult;
// glslang #include search paths for every shader compiled in this
// configuration: each transitive (incl. self) buildFiles entry's parent
// dir, or the entry itself if it points at a directory. Collected once
// up front so the per-shader threads can capture the resulting span by
// reference. No file copy involved — glslang reads the includes in
// place from the dep's source tree.
std::vector<fs::path> shaderIncludeDirs;
{
std::unordered_set<std::string> seenDirs;
std::unordered_set<const Configuration*> seenCfg;
std::function<void(const Configuration*)> collect = [&](const Configuration* c) {
if (!seenCfg.insert(c).second) return;
for (const fs::path& bf : c->buildFiles) {
fs::path dir = fs::is_directory(bf) ? bf : bf.parent_path();
if (seenDirs.insert(dir.string()).second) {
shaderIncludeDirs.push_back(std::move(dir));
}
}
for (const Configuration* sub : c->dependencies) collect(sub);
};
collect(&config);
}
std::vector<std::thread> threads;
threads.reserve(config.shaders.size() + 1 + config.interfaces.size() + config.implementations.size());
@ -254,11 +277,11 @@ BuildResult Crafter::Build(Configuration& config, std::unordered_map<fs::path, s
std::atomic<bool> buildCancelled{false};
for (const Shader& shader : config.shaders) {
if (shader.Check(outputDir)) continue;
threads.emplace_back([&shader, &outputDir, &buildError, &buildCancelled]() {
threads.emplace_back([&shader, &outputDir, &shaderIncludeDirs, &buildError, &buildCancelled]() {
Progress::Task task(std::format("Compiling shader {}", shader.path.filename().string()));
if (buildCancelled.load(std::memory_order_relaxed)) return;
std::string result = shader.Compile(outputDir);
std::string result = shader.Compile(outputDir, shaderIncludeDirs);
if (result.empty()) return;
bool expected = false;
@ -272,42 +295,28 @@ BuildResult Crafter::Build(Configuration& config, std::unordered_map<fs::path, s
Progress::Task task(std::format("Copying files for {}", config.name));
if (buildCancelled.load(std::memory_order_relaxed)) return;
try {
for (const fs::path& additionalFile : config.files) {
for (const fs::path& additionalFile : config.files) {
fs::path destination = outputDir / additionalFile.filename();
if (fs::is_directory(additionalFile)) {
for (const auto& entry : fs::recursive_directory_iterator(additionalFile)) {
const fs::path& sourcePath = entry.path();
// Compute relative path inside the directory
fs::path relativePath = fs::relative(sourcePath, additionalFile);
fs::path destPath = destination / relativePath;
if (entry.is_directory()) {
// Ensure directory exists in destination
if (!fs::exists(destPath)) {
fs::create_directories(destPath);
}
if (!fs::exists(destPath)) fs::create_directories(destPath);
} else if (entry.is_regular_file()) {
// Ensure parent directory exists
fs::create_directories(destPath.parent_path());
if (!fs::exists(destPath)) {
fs::copy_file(sourcePath, destPath);
}
else if (fs::last_write_time(sourcePath) > fs::last_write_time(destPath)) {
} else if (fs::last_write_time(sourcePath) > fs::last_write_time(destPath)) {
fs::copy_file(sourcePath, destPath, fs::copy_options::overwrite_existing);
}
}
}
} else {
// Handle regular file
if (!fs::exists(destination)) {
fs::copy_file(additionalFile, destination);
}
else if (fs::last_write_time(additionalFile) > fs::last_write_time(destination)) {
} else if (fs::last_write_time(additionalFile) > fs::last_write_time(destination)) {
fs::copy_file(additionalFile, destination, fs::copy_options::overwrite_existing);
}
}
@ -572,30 +581,60 @@ BuildResult Crafter::Build(Configuration& config, std::unordered_map<fs::path, s
return {buildError, false, {}};
}
// Ship .spv files from transitive deps' bin dirs alongside the executable
// so consumers find shaders next to the binary at runtime. Only an exe is
// a deployment unit; intermediate libs don't need to forward shaders since
// the exe walks all transitive deps. Runs outside the repack gate because
// the relink mtime check above only watches .so/.dll/.a, so a shader-only
// change in a dep wouldn't trigger repack but still needs the new .spv
// copied across.
// Ship runtime artifacts from transitive deps' bin dirs alongside the
// executable: compiled .spv files (cfg.shaders) and asset files/dirs
// (cfg.files). The lib already mirrored these into its own bin dir
// during its build, but a consumer exe loads them from its own dir at
// runtime. Only an exe is a deployment unit; intermediate libs don't
// need to forward since the exe walks all transitive deps. Runs outside
// the repack gate because the relink mtime check above only watches
// .so/.dll/.a, so a shader/file-only change in a dep wouldn't trigger
// repack but still needs the new artifact copied across.
// (cfg.buildFiles use a different mechanism — they're exposed to shader
// compiles as #include search paths in place, no copy.)
if (config.type == ConfigurationType::Executable) {
try {
std::unordered_set<Configuration*> shaderSeen;
std::function<void(Configuration*)> copyDepShaders = [&](Configuration* dep) {
if (!shaderSeen.insert(dep).second) return;
fs::path depDir = dep->BinDir();
for (const Shader& shader : dep->shaders) {
fs::path src = depDir / shader.path.filename().replace_extension("spv");
if (!fs::exists(src)) continue;
fs::path dest = outputDir / src.filename();
if (!fs::exists(dest) || fs::last_write_time(src) > fs::last_write_time(dest)) {
auto copyTree = [](const fs::path& src, const fs::path& dest) {
if (fs::is_directory(src)) {
for (const auto& entry : fs::recursive_directory_iterator(src)) {
fs::path rel = fs::relative(entry.path(), src);
fs::path destPath = dest / rel;
if (entry.is_directory()) {
if (!fs::exists(destPath)) fs::create_directories(destPath);
} else if (entry.is_regular_file()) {
fs::create_directories(destPath.parent_path());
if (!fs::exists(destPath)) {
fs::copy_file(entry.path(), destPath);
} else if (fs::last_write_time(entry.path()) > fs::last_write_time(destPath)) {
fs::copy_file(entry.path(), destPath, fs::copy_options::overwrite_existing);
}
}
}
} else {
if (!fs::exists(dest)) {
fs::copy_file(src, dest);
} else if (fs::last_write_time(src) > fs::last_write_time(dest)) {
fs::copy_file(src, dest, fs::copy_options::overwrite_existing);
}
}
for (Configuration* sub : dep->dependencies) copyDepShaders(sub);
};
for (Configuration* dep : config.dependencies) copyDepShaders(dep);
std::unordered_set<Configuration*> seen;
std::function<void(Configuration*)> forwardDepArtifacts = [&](Configuration* dep) {
if (!seen.insert(dep).second) return;
fs::path depBinDir = dep->BinDir();
for (const Shader& shader : dep->shaders) {
fs::path src = depBinDir / shader.path.filename().replace_extension("spv");
if (!fs::exists(src)) continue;
copyTree(src, outputDir / src.filename());
}
for (const fs::path& additionalFile : dep->files) {
fs::path src = depBinDir / additionalFile.filename();
if (!fs::exists(src)) continue;
copyTree(src, outputDir / additionalFile.filename());
}
for (Configuration* sub : dep->dependencies) forwardDepArtifacts(sub);
};
for (Configuration* dep : config.dependencies) forwardDepArtifacts(dep);
} catch (const fs::filesystem_error& e) {
for(std::thread& thread : threads) thread.join();
return {e.what(), false, {}};

View file

@ -317,6 +317,7 @@ namespace {
for (fs::path& p : cfg.cFiles) p = abs(p);
for (fs::path& p : cfg.cuda) p = abs(p);
for (fs::path& p : cfg.files) p = abs(p);
for (fs::path& p : cfg.buildFiles) p = abs(p);
for (Shader& s : cfg.shaders) s.path = abs(s.path);
return cfg;
}

View file

@ -57,7 +57,7 @@ namespace Crafter {
fs::path spv = outputDir / path.filename().replace_extension("spv");
return fs::exists(spv) && fs::last_write_time(path) < fs::last_write_time(spv);
}
std::string Shader::Compile(const fs::path& outputDir) const {
std::string Shader::Compile(const fs::path& outputDir, std::span<const fs::path> includeDirs) const {
EShLanguage glslangType = ToEShLanguage(type);
glslang::InitializeProcess();
@ -98,6 +98,9 @@ namespace Crafter {
shader.setEnvTarget(glslang::EShTargetSpv, glslang::EShTargetSpv_1_4);
DirStackFileIncluder includeDir;
includeDir.pushExternalLocalDirectory(path.parent_path().generic_string());
for (const fs::path& dir : includeDirs) {
includeDir.pushExternalLocalDirectory(dir.generic_string());
}
if (!shader.parse(GetDefaultResources(), 100, false, messages, includeDir)) {
return fail("GLSL parse failed", std::string(shader.getInfoLog()) + shader.getInfoDebugLog());

View file

@ -115,6 +115,14 @@ export namespace Crafter {
std::vector<fs::path> cuda;
std::vector<Configuration*> dependencies;
std::vector<fs::path> files;
// Build-time-only source files this configuration exposes to its own
// and its consumers' shader compiles as glslang #include search
// paths. Each entry's parent directory (or the entry itself, if it's
// a directory) is added to the includer for every shader compiled in
// this configuration and in any configuration that transitively
// depends on it. Files are NOT copied — they're read in place from
// the dep's source tree, like C++ -I include dirs.
std::vector<fs::path> buildFiles;
std::vector<Define> defines;
std::vector<Shader> shaders;
std::vector<ExternalDependency> externalDependencies;

View file

@ -48,6 +48,6 @@ namespace Crafter {
ShaderType type;
CRAFTER_API Shader(fs::path&& path, std::string&& entrypoint, ShaderType type);
CRAFTER_API bool Check(const fs::path& outputDir) const;
CRAFTER_API std::string Compile(const fs::path& outputDir) const;
CRAFTER_API std::string Compile(const fs::path& outputDir, std::span<const fs::path> includeDirs) const;
};
}

View file

@ -5,9 +5,16 @@ Catcrafts.net
LGPL-3.0-only.
Verifies that when an executable consumes a library that registers a shader,
the resulting .spv ships in the executable's bin dir alongside the binary
not just in the lib's own bin dir, where the runtime exe wouldn't find it.
Verifies that when an executable consumes a library that registers shaders
or asset files, those runtime artifacts ship in the executable's bin dir
alongside the binary not just in the lib's own bin dir, where the runtime
exe wouldn't find them. Covers a .spv (cfg.shaders), a flat asset
(cfg.files file), and a directory tree (cfg.files dir).
Also verifies cfg.buildFiles' include-path semantic: the consumer's own
shader compile resolves `#include "ui-shared.glsl"` against the lib's
buildFiles in-place from the dep's source tree (no copy), and the file is
NOT mirrored into any build/bin dir.
*/
import std;
@ -50,6 +57,58 @@ int main() {
return 1;
}
fs::path appAsset = cfg.BinDir() / "asset.txt";
if (!fs::exists(appAsset)) {
std::println(std::cerr, "exe-side asset.txt missing at {}", appAsset.string());
return 1;
}
if (ReadFile(appAsset) != "crafter-build asset\n") {
std::println(std::cerr, "asset.txt content mismatch at {}", appAsset.string());
return 1;
}
fs::path appNested = cfg.BinDir() / "data" / "nested.txt";
if (!fs::exists(appNested)) {
std::println(std::cerr, "exe-side data/nested.txt missing at {}", appNested.string());
return 1;
}
if (ReadFile(appNested) != "nested asset\n") {
std::println(std::cerr, "data/nested.txt content mismatch at {}", appNested.string());
return 1;
}
// The consumer shader includes a header from the lib's buildFiles.
// Successful build (above) means glslang resolved the include
// in-place from the dep's source tree — verify the resulting .spv
// and the SPIR-V magic, same as for the lib's shader.
fs::path consumerSpv = cfg.BinDir() / "consumer.spv";
if (!fs::exists(consumerSpv)) {
std::println(std::cerr, "consumer .spv missing at {}", consumerSpv.string());
return 1;
}
std::ifstream cf(consumerSpv, std::ios::binary);
unsigned char cmagic[4] = {};
cf.read(reinterpret_cast<char*>(cmagic), 4);
if (cmagic[0] != 0x03 || cmagic[1] != 0x02 || cmagic[2] != 0x23 || cmagic[3] != 0x07) {
std::println(std::cerr,
"SPIR-V magic mismatch in consumer.spv: got {:#04x} {:#04x} {:#04x} {:#04x}",
cmagic[0], cmagic[1], cmagic[2], cmagic[3]);
return 1;
}
// buildFiles must NOT be copied anywhere — they're read in place.
for (fs::path probe : {
cfg.BinDir() / "ui-shared.glsl",
cfg.BuildDir() / "ui-shared.glsl",
cfg.dependencies[0]->BinDir() / "ui-shared.glsl",
cfg.dependencies[0]->BuildDir() / "ui-shared.glsl",
}) {
if (fs::exists(probe)) {
std::println(std::cerr, "ui-shared.glsl unexpectedly copied to {}", probe.string());
return 1;
}
}
return 0;
} catch (const std::exception& e) {
std::println(std::cerr, "test exception: {}", e.what());

View file

@ -0,0 +1,16 @@
#version 450
#extension GL_GOOGLE_include_directive : require
#include "ui-shared.glsl"
layout(location = 0) out vec3 color;
void main() {
vec2 positions[3] = vec2[](
vec2( 0.0, -0.5),
vec2( 0.5, 0.5),
vec2(-0.5, 0.5)
);
gl_Position = vec4(positions[gl_VertexIndex], 0.0, 1.0);
color = ui_color();
}

View file

@ -0,0 +1 @@
crafter-build asset

View file

@ -0,0 +1 @@
nested asset

View file

@ -0,0 +1,8 @@
#ifndef UI_SHARED_GLSL
#define UI_SHARED_GLSL
vec3 ui_color() {
return vec3(0.25, 0.5, 1.0);
}
#endif

View file

@ -16,6 +16,9 @@ extern "C" Configuration CrafterBuildProject(std::span<const std::string_view>)
ShaderLib->GetInterfacesAndImplementations(ifaces, impls);
}
ShaderLib->shaders.emplace_back("./lib/triangle.glsl", "main", ShaderType::Vertex);
ShaderLib->files.emplace_back("./lib/asset.txt");
ShaderLib->files.emplace_back("./lib/data");
ShaderLib->buildFiles.emplace_back("./lib/ui-shared.glsl");
Configuration app;
app.path = "./";
@ -29,5 +32,6 @@ extern "C" Configuration CrafterBuildProject(std::span<const std::string_view>)
std::array<fs::path, 1> impls = { "main" };
app.GetInterfacesAndImplementations(ifaces, impls);
}
app.shaders.emplace_back("./consumer.glsl", "main", ShaderType::Vertex);
return app;
}