fixes
All checks were successful
CI / build-test-release (push) Successful in 15m14s

This commit is contained in:
Jorijn van der Graaf 2026-05-02 21:08:51 +02:00
commit d7a9c85ea6
11 changed files with 183 additions and 43 deletions

View file

@ -247,6 +247,29 @@ BuildResult Crafter::Build(Configuration& config, std::unordered_map<fs::path, s
BuildResult buildResult;
// glslang #include search paths for every shader compiled in this
// configuration: each transitive (incl. self) buildFiles entry's parent
// dir, or the entry itself if it points at a directory. Collected once
// up front so the per-shader threads can capture the resulting span by
// reference. No file copy involved — glslang reads the includes in
// place from the dep's source tree.
std::vector<fs::path> shaderIncludeDirs;
{
std::unordered_set<std::string> seenDirs;
std::unordered_set<const Configuration*> seenCfg;
std::function<void(const Configuration*)> collect = [&](const Configuration* c) {
if (!seenCfg.insert(c).second) return;
for (const fs::path& bf : c->buildFiles) {
fs::path dir = fs::is_directory(bf) ? bf : bf.parent_path();
if (seenDirs.insert(dir.string()).second) {
shaderIncludeDirs.push_back(std::move(dir));
}
}
for (const Configuration* sub : c->dependencies) collect(sub);
};
collect(&config);
}
std::vector<std::thread> threads;
threads.reserve(config.shaders.size() + 1 + config.interfaces.size() + config.implementations.size());
@ -254,11 +277,11 @@ BuildResult Crafter::Build(Configuration& config, std::unordered_map<fs::path, s
std::atomic<bool> buildCancelled{false};
for (const Shader& shader : config.shaders) {
if (shader.Check(outputDir)) continue;
threads.emplace_back([&shader, &outputDir, &buildError, &buildCancelled]() {
threads.emplace_back([&shader, &outputDir, &shaderIncludeDirs, &buildError, &buildCancelled]() {
Progress::Task task(std::format("Compiling shader {}", shader.path.filename().string()));
if (buildCancelled.load(std::memory_order_relaxed)) return;
std::string result = shader.Compile(outputDir);
std::string result = shader.Compile(outputDir, shaderIncludeDirs);
if (result.empty()) return;
bool expected = false;
@ -274,40 +297,26 @@ BuildResult Crafter::Build(Configuration& config, std::unordered_map<fs::path, s
try {
for (const fs::path& additionalFile : config.files) {
fs::path destination = outputDir / additionalFile.filename();
if (fs::is_directory(additionalFile)) {
for (const auto& entry : fs::recursive_directory_iterator(additionalFile)) {
const fs::path& sourcePath = entry.path();
// Compute relative path inside the directory
fs::path relativePath = fs::relative(sourcePath, additionalFile);
fs::path destPath = destination / relativePath;
if (entry.is_directory()) {
// Ensure directory exists in destination
if (!fs::exists(destPath)) {
fs::create_directories(destPath);
}
if (!fs::exists(destPath)) fs::create_directories(destPath);
} else if (entry.is_regular_file()) {
// Ensure parent directory exists
fs::create_directories(destPath.parent_path());
if (!fs::exists(destPath)) {
fs::copy_file(sourcePath, destPath);
}
else if (fs::last_write_time(sourcePath) > fs::last_write_time(destPath)) {
} else if (fs::last_write_time(sourcePath) > fs::last_write_time(destPath)) {
fs::copy_file(sourcePath, destPath, fs::copy_options::overwrite_existing);
}
}
}
} else {
// Handle regular file
if (!fs::exists(destination)) {
fs::copy_file(additionalFile, destination);
}
else if (fs::last_write_time(additionalFile) > fs::last_write_time(destination)) {
} else if (fs::last_write_time(additionalFile) > fs::last_write_time(destination)) {
fs::copy_file(additionalFile, destination, fs::copy_options::overwrite_existing);
}
}
@ -572,30 +581,60 @@ BuildResult Crafter::Build(Configuration& config, std::unordered_map<fs::path, s
return {buildError, false, {}};
}
// Ship .spv files from transitive deps' bin dirs alongside the executable
// so consumers find shaders next to the binary at runtime. Only an exe is
// a deployment unit; intermediate libs don't need to forward shaders since
// the exe walks all transitive deps. Runs outside the repack gate because
// the relink mtime check above only watches .so/.dll/.a, so a shader-only
// change in a dep wouldn't trigger repack but still needs the new .spv
// copied across.
// Ship runtime artifacts from transitive deps' bin dirs alongside the
// executable: compiled .spv files (cfg.shaders) and asset files/dirs
// (cfg.files). The lib already mirrored these into its own bin dir
// during its build, but a consumer exe loads them from its own dir at
// runtime. Only an exe is a deployment unit; intermediate libs don't
// need to forward since the exe walks all transitive deps. Runs outside
// the repack gate because the relink mtime check above only watches
// .so/.dll/.a, so a shader/file-only change in a dep wouldn't trigger
// repack but still needs the new artifact copied across.
// (cfg.buildFiles use a different mechanism — they're exposed to shader
// compiles as #include search paths in place, no copy.)
if (config.type == ConfigurationType::Executable) {
try {
std::unordered_set<Configuration*> shaderSeen;
std::function<void(Configuration*)> copyDepShaders = [&](Configuration* dep) {
if (!shaderSeen.insert(dep).second) return;
fs::path depDir = dep->BinDir();
for (const Shader& shader : dep->shaders) {
fs::path src = depDir / shader.path.filename().replace_extension("spv");
if (!fs::exists(src)) continue;
fs::path dest = outputDir / src.filename();
if (!fs::exists(dest) || fs::last_write_time(src) > fs::last_write_time(dest)) {
auto copyTree = [](const fs::path& src, const fs::path& dest) {
if (fs::is_directory(src)) {
for (const auto& entry : fs::recursive_directory_iterator(src)) {
fs::path rel = fs::relative(entry.path(), src);
fs::path destPath = dest / rel;
if (entry.is_directory()) {
if (!fs::exists(destPath)) fs::create_directories(destPath);
} else if (entry.is_regular_file()) {
fs::create_directories(destPath.parent_path());
if (!fs::exists(destPath)) {
fs::copy_file(entry.path(), destPath);
} else if (fs::last_write_time(entry.path()) > fs::last_write_time(destPath)) {
fs::copy_file(entry.path(), destPath, fs::copy_options::overwrite_existing);
}
}
}
} else {
if (!fs::exists(dest)) {
fs::copy_file(src, dest);
} else if (fs::last_write_time(src) > fs::last_write_time(dest)) {
fs::copy_file(src, dest, fs::copy_options::overwrite_existing);
}
}
for (Configuration* sub : dep->dependencies) copyDepShaders(sub);
};
for (Configuration* dep : config.dependencies) copyDepShaders(dep);
std::unordered_set<Configuration*> seen;
std::function<void(Configuration*)> forwardDepArtifacts = [&](Configuration* dep) {
if (!seen.insert(dep).second) return;
fs::path depBinDir = dep->BinDir();
for (const Shader& shader : dep->shaders) {
fs::path src = depBinDir / shader.path.filename().replace_extension("spv");
if (!fs::exists(src)) continue;
copyTree(src, outputDir / src.filename());
}
for (const fs::path& additionalFile : dep->files) {
fs::path src = depBinDir / additionalFile.filename();
if (!fs::exists(src)) continue;
copyTree(src, outputDir / additionalFile.filename());
}
for (Configuration* sub : dep->dependencies) forwardDepArtifacts(sub);
};
for (Configuration* dep : config.dependencies) forwardDepArtifacts(dep);
} catch (const fs::filesystem_error& e) {
for(std::thread& thread : threads) thread.join();
return {e.what(), false, {}};

View file

@ -317,6 +317,7 @@ namespace {
for (fs::path& p : cfg.cFiles) p = abs(p);
for (fs::path& p : cfg.cuda) p = abs(p);
for (fs::path& p : cfg.files) p = abs(p);
for (fs::path& p : cfg.buildFiles) p = abs(p);
for (Shader& s : cfg.shaders) s.path = abs(s.path);
return cfg;
}

View file

@ -57,7 +57,7 @@ namespace Crafter {
fs::path spv = outputDir / path.filename().replace_extension("spv");
return fs::exists(spv) && fs::last_write_time(path) < fs::last_write_time(spv);
}
std::string Shader::Compile(const fs::path& outputDir) const {
std::string Shader::Compile(const fs::path& outputDir, std::span<const fs::path> includeDirs) const {
EShLanguage glslangType = ToEShLanguage(type);
glslang::InitializeProcess();
@ -98,6 +98,9 @@ namespace Crafter {
shader.setEnvTarget(glslang::EShTargetSpv, glslang::EShTargetSpv_1_4);
DirStackFileIncluder includeDir;
includeDir.pushExternalLocalDirectory(path.parent_path().generic_string());
for (const fs::path& dir : includeDirs) {
includeDir.pushExternalLocalDirectory(dir.generic_string());
}
if (!shader.parse(GetDefaultResources(), 100, false, messages, includeDir)) {
return fail("GLSL parse failed", std::string(shader.getInfoLog()) + shader.getInfoDebugLog());

View file

@ -115,6 +115,14 @@ export namespace Crafter {
std::vector<fs::path> cuda;
std::vector<Configuration*> dependencies;
std::vector<fs::path> files;
// Build-time-only source files this configuration exposes to its own
// and its consumers' shader compiles as glslang #include search
// paths. Each entry's parent directory (or the entry itself, if it's
// a directory) is added to the includer for every shader compiled in
// this configuration and in any configuration that transitively
// depends on it. Files are NOT copied — they're read in place from
// the dep's source tree, like C++ -I include dirs.
std::vector<fs::path> buildFiles;
std::vector<Define> defines;
std::vector<Shader> shaders;
std::vector<ExternalDependency> externalDependencies;

View file

@ -48,6 +48,6 @@ namespace Crafter {
ShaderType type;
CRAFTER_API Shader(fs::path&& path, std::string&& entrypoint, ShaderType type);
CRAFTER_API bool Check(const fs::path& outputDir) const;
CRAFTER_API std::string Compile(const fs::path& outputDir) const;
CRAFTER_API std::string Compile(const fs::path& outputDir, std::span<const fs::path> includeDirs) const;
};
}

View file

@ -5,9 +5,16 @@ Catcrafts.net
LGPL-3.0-only.
Verifies that when an executable consumes a library that registers a shader,
the resulting .spv ships in the executable's bin dir alongside the binary —
not just in the lib's own bin dir, where the runtime exe wouldn't find it.
Verifies that when an executable consumes a library that registers shaders
or asset files, those runtime artifacts ship in the executable's bin dir
alongside the binary — not just in the lib's own bin dir, where the runtime
exe wouldn't find them. Covers a .spv (cfg.shaders), a flat asset
(cfg.files file), and a directory tree (cfg.files dir).
Also verifies cfg.buildFiles' include-path semantic: the consumer's own
shader compile resolves `#include "ui-shared.glsl"` against the lib's
buildFiles in-place from the dep's source tree (no copy), and the file is
NOT mirrored into any build/bin dir.
*/
import std;
@ -50,6 +57,58 @@ int main() {
return 1;
}
fs::path appAsset = cfg.BinDir() / "asset.txt";
if (!fs::exists(appAsset)) {
std::println(std::cerr, "exe-side asset.txt missing at {}", appAsset.string());
return 1;
}
if (ReadFile(appAsset) != "crafter-build asset\n") {
std::println(std::cerr, "asset.txt content mismatch at {}", appAsset.string());
return 1;
}
fs::path appNested = cfg.BinDir() / "data" / "nested.txt";
if (!fs::exists(appNested)) {
std::println(std::cerr, "exe-side data/nested.txt missing at {}", appNested.string());
return 1;
}
if (ReadFile(appNested) != "nested asset\n") {
std::println(std::cerr, "data/nested.txt content mismatch at {}", appNested.string());
return 1;
}
// The consumer shader includes a header from the lib's buildFiles.
// Successful build (above) means glslang resolved the include
// in-place from the dep's source tree — verify the resulting .spv
// and the SPIR-V magic, same as for the lib's shader.
fs::path consumerSpv = cfg.BinDir() / "consumer.spv";
if (!fs::exists(consumerSpv)) {
std::println(std::cerr, "consumer .spv missing at {}", consumerSpv.string());
return 1;
}
std::ifstream cf(consumerSpv, std::ios::binary);
unsigned char cmagic[4] = {};
cf.read(reinterpret_cast<char*>(cmagic), 4);
if (cmagic[0] != 0x03 || cmagic[1] != 0x02 || cmagic[2] != 0x23 || cmagic[3] != 0x07) {
std::println(std::cerr,
"SPIR-V magic mismatch in consumer.spv: got {:#04x} {:#04x} {:#04x} {:#04x}",
cmagic[0], cmagic[1], cmagic[2], cmagic[3]);
return 1;
}
// buildFiles must NOT be copied anywhere — they're read in place.
for (fs::path probe : {
cfg.BinDir() / "ui-shared.glsl",
cfg.BuildDir() / "ui-shared.glsl",
cfg.dependencies[0]->BinDir() / "ui-shared.glsl",
cfg.dependencies[0]->BuildDir() / "ui-shared.glsl",
}) {
if (fs::exists(probe)) {
std::println(std::cerr, "ui-shared.glsl unexpectedly copied to {}", probe.string());
return 1;
}
}
return 0;
} catch (const std::exception& e) {
std::println(std::cerr, "test exception: {}", e.what());

View file

@ -0,0 +1,16 @@
#version 450
#extension GL_GOOGLE_include_directive : require
#include "ui-shared.glsl"
layout(location = 0) out vec3 color;
void main() {
vec2 positions[3] = vec2[](
vec2( 0.0, -0.5),
vec2( 0.5, 0.5),
vec2(-0.5, 0.5)
);
gl_Position = vec4(positions[gl_VertexIndex], 0.0, 1.0);
color = ui_color();
}

View file

@ -0,0 +1 @@
crafter-build asset

View file

@ -0,0 +1 @@
nested asset

View file

@ -0,0 +1,8 @@
#ifndef UI_SHARED_GLSL
#define UI_SHARED_GLSL
vec3 ui_color() {
return vec3(0.25, 0.5, 1.0);
}
#endif

View file

@ -16,6 +16,9 @@ extern "C" Configuration CrafterBuildProject(std::span<const std::string_view>)
ShaderLib->GetInterfacesAndImplementations(ifaces, impls);
}
ShaderLib->shaders.emplace_back("./lib/triangle.glsl", "main", ShaderType::Vertex);
ShaderLib->files.emplace_back("./lib/asset.txt");
ShaderLib->files.emplace_back("./lib/data");
ShaderLib->buildFiles.emplace_back("./lib/ui-shared.glsl");
Configuration app;
app.path = "./";
@ -29,5 +32,6 @@ extern "C" Configuration CrafterBuildProject(std::span<const std::string_view>)
std::array<fs::path, 1> impls = { "main" };
app.GetInterfacesAndImplementations(ifaces, impls);
}
app.shaders.emplace_back("./consumer.glsl", "main", ShaderType::Vertex);
return app;
}