/* Crafter® Build Copyright (C) 2026 Catcrafts® Catcrafts.net This library is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License version 3.0 as published by the Free Software Foundation; This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. You should have received a copy of the GNU Lesser General Public License along with this library; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA */ export module Crafter.Build:Clang_impl; import std; import :Clang; import :Platform; import :Test; namespace fs = std::filesystem; using namespace Crafter; void Configuration::GetInterfacesAndImplementations(std::span interfaces, std::span implementations) { auto resolveImport = [this](const std::string& importName, std::vector& localDeps, std::vector>& externalDeps) -> bool { for(const std::unique_ptr& interface : this->interfaces) { if(interface->name == importName) { localDeps.push_back(interface.get()); return true; } } std::unordered_set seen; std::function walk = [&](Configuration* depCfg) -> bool { if (!seen.insert(depCfg).second) return false; for(const std::unique_ptr& depInterface : depCfg->interfaces) { if(depInterface->name == importName) { fs::path depPcmPath = (depCfg->PcmDir() / depInterface->path.filename()).string() + ".pcm"; externalDeps.emplace_back(depInterface.get(), std::move(depPcmPath)); return true; } } for(Configuration* sub : depCfg->dependencies) { if (walk(sub)) return true; } return false; }; for(Configuration* depCfg : this->dependencies) { if (walk(depCfg)) return true; } return false; }; std::vector> tempModulePaths = std::vector>(interfaces.size()); for(std::uint16_t i = 0; i < interfaces.size(); i++){ fs::path file = path / interfaces[i]; file += ".cppm"; std::ifstream t(file); std::stringstream buffer; buffer << t.rdbuf(); std::string fileContent = buffer.str(); fileContent = std::regex_replace(fileContent, std::regex(R"(//[^\n]*)"), ""); fileContent = std::regex_replace(fileContent, std::regex(R"(/\*.*?\*/)"), ""); tempModulePaths[i] = {file, fileContent, nullptr, nullptr}; } std::erase_if(tempModulePaths, [this](std::tuple& file) { std::smatch match; if (std::regex_search(std::get<1>(file), match, std::regex(R"(export module ([a-zA-Z0-9_\.\-]+);)"))) { std::get<0>(file).replace_extension(""); this->interfaces.push_back(std::make_unique(std::move(match[1].str()), std::move(std::get<0>(file)))); return true; } else { return false; } }); for(std::uint16_t i = 0; i < tempModulePaths.size(); i++) { std::smatch match; if (std::regex_search(std::get<1>(tempModulePaths[i]), match, std::regex(R"(export module ([a-zA-Z_0-9\.\-]+):([a-zA-Z_0-9\.\-]+);)"))) { for(const std::unique_ptr& modulee : this->interfaces) { if(modulee->name == match[1]) { std::string name = match[2].str(); fs::path pthCpy = std::get<0>(tempModulePaths[i]); pthCpy.replace_extension(""); std::unique_ptr partition = std::make_unique(std::move(name), std::move(pthCpy)); std::get<2>(tempModulePaths[i]) = partition.get(); modulee->partitions.push_back(std::move(partition)); std::get<3>(tempModulePaths[i]) = modulee.get(); goto next; } } throw std::runtime_error(std::format("Module {} not found, referenced in {}", match[1].str(), std::get<0>(tempModulePaths[i]).string())); } else { throw std::runtime_error(std::format("No module declaration found in {}", std::get<0>(tempModulePaths[i]).string())); } next:; } for(std::tuple& file : tempModulePaths) { ModulePartition* partition = std::get<2>(file); Module* parentModule = std::get<3>(file); const std::string& fileContent = std::get<1>(file); std::regex partitionPattern(R"(import :([a-zA-Z_\-0-9\.]+);)"); std::sregex_iterator currentMatch(fileContent.begin(), fileContent.end(), partitionPattern); std::sregex_iterator lastMatch; while (currentMatch != lastMatch) { std::smatch match = *currentMatch; for(std::unique_ptr& sibling : parentModule->partitions) { if(sibling->name == match[1]) { partition->partitionDependencies.push_back(sibling.get()); goto next2; } } throw std::runtime_error(std::format("imported partition {}:{} not found, referenced in {}", parentModule->name, match[1].str(), std::get<0>(file).string())); next2: ++currentMatch; } std::regex modulePattern(R"(import ([a-zA-Z_0-9\.\-]+);)"); std::sregex_iterator modCurrent(fileContent.begin(), fileContent.end(), modulePattern); while (modCurrent != lastMatch) { std::smatch match = *modCurrent; resolveImport(match[1].str(), partition->moduleDependencies, partition->externalModuleDependencies); ++modCurrent; } } for(const fs::path& tempFile : implementations) { fs::path file = path / tempFile; file += ".cpp"; std::ifstream t(file); std::stringstream buffer; buffer << t.rdbuf(); std::string fileContent = buffer.str(); fileContent = std::regex_replace(fileContent, std::regex(R"(//[^\n]*)"), ""); fileContent = std::regex_replace(fileContent, std::regex(R"(/\*.*?\*/)"), ""); std::smatch match; fs::path fileCopy = file; fileCopy.replace_extension(""); Implementation& implementation = this->implementations.emplace_back(std::move(fileCopy)); if (std::regex_search(fileContent, match, std::regex(R"(module ([a-zA-Z0-9_\.\-]+)(:[a-zA-Z0-9_\.\-]+)?\s*;)"))) { bool isPartitionImpl = match[2].length() > 0; for(const std::unique_ptr& interface : this->interfaces) { if(interface->name == match[1]) { if (!isPartitionImpl) { implementation.moduleDependencies.push_back(interface.get()); } std::regex partitionPattern(R"(import :([a-zA-Z_\-0-9\.]+);)"); std::sregex_iterator currentMatch(fileContent.begin(), fileContent.end(), partitionPattern); std::sregex_iterator lastMatch; while (currentMatch != lastMatch) { std::smatch match2 = *currentMatch; for(const std::unique_ptr& partition : interface->partitions) { if(partition->name == match2[1]) { implementation.partitionDependencies.push_back(partition.get()); goto next3; } } throw std::runtime_error(std::format("imported partition {}:{} not found, referenced in {}", match[1].str(), match2[1].str(), file.string())); next3: ++currentMatch; } std::regex modulePattern(R"(import ([a-zA-Z_0-9\.\-]+);)"); std::sregex_iterator modCurrent(fileContent.begin(), fileContent.end(), modulePattern); while (modCurrent != lastMatch) { std::smatch match2 = *modCurrent; if (match2[1] != match[1]) { resolveImport(match2[1].str(), implementation.moduleDependencies, implementation.externalModuleDependencies); } ++modCurrent; } goto next4; } } throw std::runtime_error(std::format("Module {} not found not found, referenced in {}", match[1].str(), file.string())); next4:; } else { std::regex pattern(R"(import ([a-zA-Z_\-0-9\.]+);)"); std::sregex_iterator currentMatch(fileContent.begin(), fileContent.end(), pattern); std::sregex_iterator lastMatch; while (currentMatch != lastMatch) { std::smatch match2 = *currentMatch; resolveImport(match2[1].str(), implementation.moduleDependencies, implementation.externalModuleDependencies); ++currentMatch; } } } } BuildResult Crafter::Build(Configuration& config, std::unordered_map>& depResults, std::mutex& depMutex) { fs::path buildDir = config.path/"build"/std::format("{}-{}-{}", config.name, config.target, config.march); fs::path outputDir = config.path/"bin"/std::format("{}-{}-{}", config.name, config.target, config.march); if (!fs::exists(buildDir)) { fs::create_directories(buildDir); } if (!fs::exists(outputDir)) { fs::create_directories(outputDir); } BuildResult buildResult; std::vector threads; threads.reserve(config.shaders.size() + 1 + config.interfaces.size() + config.implementations.size()); std::string buildError; std::atomic buildCancelled{false}; for (const Shader& shader : config.shaders) { if (shader.Check(outputDir)) continue; threads.emplace_back([&shader, &outputDir, &buildError, &buildCancelled]() { if (buildCancelled.load(std::memory_order_relaxed)) return; std::string result = shader.Compile(outputDir); if (result.empty()) return; bool expected = false; if (buildCancelled.compare_exchange_strong(expected, true)) { buildError = std::move(result); } }); } threads.emplace_back([&config, &outputDir, &buildCancelled, &buildError]() { if (buildCancelled.load(std::memory_order_relaxed)) return; try { for (const fs::path& additionalFile : config.files) { fs::path destination = outputDir / additionalFile.filename(); if (fs::is_directory(additionalFile)) { for (const auto& entry : fs::recursive_directory_iterator(additionalFile)) { const fs::path& sourcePath = entry.path(); // Compute relative path inside the directory fs::path relativePath = fs::relative(sourcePath, additionalFile); fs::path destPath = destination / relativePath; if (entry.is_directory()) { // Ensure directory exists in destination if (!fs::exists(destPath)) { fs::create_directories(destPath); } } else if (entry.is_regular_file()) { // Ensure parent directory exists fs::create_directories(destPath.parent_path()); if (!fs::exists(destPath)) { fs::copy_file(sourcePath, destPath); } else if (fs::last_write_time(sourcePath) > fs::last_write_time(destPath)) { fs::copy_file(sourcePath, destPath, fs::copy_options::overwrite_existing); } } } } else { // Handle regular file if (!fs::exists(destination)) { fs::copy_file(additionalFile, destination); } else if (fs::last_write_time(additionalFile) > fs::last_write_time(destination)) { fs::copy_file(additionalFile, destination, fs::copy_options::overwrite_existing); } } } } catch(std::exception& e) { bool expected = false; if (buildCancelled.compare_exchange_strong(expected, true)) { buildError = e.what(); } } }); std::vector externalResults(config.externalDependencies.size()); std::vector externalThreads; externalThreads.reserve(config.externalDependencies.size()); for (std::size_t i = 0; i < config.externalDependencies.size(); ++i) { externalThreads.emplace_back([&, i]() { if (buildCancelled.load(std::memory_order_relaxed)) return; externalResults[i] = BuildExternal(config.externalDependencies[i], buildCancelled); if (!externalResults[i].error.empty()) { bool expected = false; if (buildCancelled.compare_exchange_strong(expected, true)) { buildError = externalResults[i].error; } } }); } fs::path stdPcmDir = GetCacheDir()/(config.target+"-"+config.march); if (!fs::exists(stdPcmDir)) { fs::create_directories(stdPcmDir); } std::string stdPcmResult = BuildStdPcm(config, stdPcmDir/"std.pcm"); if(!stdPcmResult.empty()) { buildCancelled.store(true); for(std::thread& thread : threads) thread.join(); for(std::thread& thread : externalThreads) thread.join(); return {stdPcmResult, false, {}}; } fs::path pcmDir; if(config.type != ConfigurationType::Executable) { pcmDir = outputDir; } else { pcmDir = buildDir; } fs::copy_file(stdPcmDir/"std.pcm", pcmDir/"std.pcm", fs::copy_options::update_existing); std::string editedTarget = config.target; std::replace(editedTarget.begin(), editedTarget.end(), '-', '_'); std::string command = std::format("{} --target={} -march={} -mtune={} -std=c++26 -D CRAFTER_BUILD_CONFIGURATION_TARGET=\\\"{}\\\" -D CRAFTER_BUILD_CONFIGURATION_TARGET_{} -fprebuilt-module-path={} -fprebuilt-module-path={}", GetBaseCommand(config), config.target, config.march, config.mtune, editedTarget, editedTarget, stdPcmDir.string(), pcmDir.string()); if (!config.sysroot.empty()) { command += std::format(" --sysroot={}", config.sysroot); } if(config.type == ConfigurationType::LibraryDynamic) { #ifdef CRAFTER_BUILD_CONFIGURATION_TARGET_x86_64_pc_linux_gnu command += " -fPIC -D CRAFTER_BUILD_CONFIGURATION_TYPE_SHARED_LIBRARY"; #endif #if defined(CRAFTER_BUILD_CONFIGURATION_TARGET_x86_64_pc_windows_msvc) || defined(CRAFTER_BUILD_CONFIGURATION_TARGET_x86_64_w64_mingw32) command += " -D CRAFTER_BUILD_CONFIGURATION_TYPE_SHARED_LIBRARY"; #endif } else if(config.type == ConfigurationType::Executable) { command += " -D CRAFTER_BUILD_CONFIGURATION_TYPE_EXECUTABLE"; } else { command += " -D CRAFTER_BUILD_CONFIGURATION_TYPE_LIBRARY"; } std::string files; std::unordered_set libSet; std::mutex fileMutex; std::vector depThreads; depThreads.reserve(config.dependencies.size()); std::atomic repack(false); { std::unordered_set seen; std::function addFlags = [&](Configuration* dep) { if (!seen.insert(dep).second) return; for (const auto& entry : fs::recursive_directory_iterator(dep->path)) { if (entry.is_directory() && entry.path().filename() == "include") { command += " -I" + entry.path().string(); } } command += std::format(" -I{} -fprebuilt-module-path={}", dep->path.string(), dep->PcmDir().string()); for (Configuration* sub : dep->dependencies) { addFlags(sub); } }; for (Configuration* dep : config.dependencies) { addFlags(dep); } } for(Configuration* dep : config.dependencies) { depThreads.emplace_back([&, dep](){ try { if (buildCancelled.load(std::memory_order_relaxed)) return; std::shared_ptr> promise; std::shared_future resultFuture; bool isBuilder = false; depMutex.lock(); fs::path cacheKey = dep->PcmDir(); auto it = depResults.find(cacheKey); if (it == depResults.end()) { isBuilder = true; promise = std::make_shared>(); resultFuture = promise->get_future().share(); depResults.emplace(cacheKey, resultFuture); } else { resultFuture = it->second; } depMutex.unlock(); if (isBuilder) { BuildResult built; try { built = Build(*dep, depResults, depMutex); } catch (...) { promise->set_exception(std::current_exception()); throw; } promise->set_value(std::move(built)); } const BuildResult& result = resultFuture.get(); fileMutex.lock(); for (const std::string& lib : result.libs) libSet.insert(lib); fileMutex.unlock(); if (!result.result.empty()) { bool expected = false; if (buildCancelled.compare_exchange_strong(expected, true)) { buildError = result.result; } } if (result.repack) { repack = true; } } catch (const std::exception& e) { bool expected = false; if (buildCancelled.compare_exchange_strong(expected, true)) { buildError = std::format("dep build for {} threw: {}", dep->path.string(), e.what()); } } }); } for(const Define& define : config.defines) { if(define.value.empty()) { command += std::format(" -D {}", define.name); } else { command += std::format(" -D {}={}", define.name, define.value); } } for(const std::string& flag : config.compileFlags) { command += " " + flag; } std::string cmakeBuildType; if(config.debug) { cmakeBuildType = "Debug"; command += " -g -D CRAFTER_BUILD_CONFIGURATION_DEBUG"; } else { cmakeBuildType = "Release"; command += " -O3"; } for (const fs::path& cFile : config.cFiles) { files += std::format(" {}_source.o ", (buildDir / cFile.filename()).string()); const std::string objPath = (buildDir / cFile.filename()).string() + "_source.o"; const std::string srcPath = cFile.string() + ".c"; if (!fs::exists(objPath) || fs::last_write_time(srcPath) > fs::last_write_time(objPath)) { threads.emplace_back([&cFile, &buildDir, &buildError, &buildCancelled, &config]() { if (buildCancelled.load(std::memory_order_relaxed)) return; std::string result = RunCommand(std::format("clang {}.c --target={} -march={} -mtune={} -O3 -c -o {}_source.o", cFile.string(), config.target, config.march, config.mtune, (buildDir / cFile.filename()).string())); if (result.empty()) return; bool expected = false; if (buildCancelled.compare_exchange_strong(expected, true)) { buildError = std::move(result); } }); } } for (const fs::path& cFile : config.cuda) { files += std::format(" {}_source.o ", (buildDir / cFile.filename()).string()); const std::string objPath = (buildDir / cFile.filename()).string() + "_source.o"; const std::string srcPath = cFile.string() + ".cu"; if (!fs::exists(objPath) || fs::last_write_time(srcPath) > fs::last_write_time(objPath)) { threads.emplace_back([&cFile, &buildDir, &buildError, &buildCancelled]() { if (buildCancelled.load(std::memory_order_relaxed)) return; std::string result = RunCommand(std::format("nvcc {}.cu -c -o {}_source.o -O3 -arch=sm_89", cFile.string(), (buildDir / cFile.filename()).string())); if (result.empty()) return; bool expected = false; if (buildCancelled.compare_exchange_strong(expected, true)) { buildError = std::move(result); } }); } } for(std::thread& thread : depThreads) { thread.join(); } for(std::thread& thread : externalThreads) { thread.join(); } if(buildCancelled.load()) { for(std::thread& thread : threads) thread.join(); return {buildError, false, {}}; } if(repack.load()) { buildResult.repack = true; } buildResult.libs = std::move(libSet); for(const std::string& flag : config.linkFlags) { buildResult.libs.insert(flag); } fs::file_time_type externalFloor = fs::file_time_type::min(); for(const ExternalBuildResult& ext : externalResults) { for(const std::string& flag : ext.compileFlags) { command += " " + flag; } for(const std::string& flag : ext.linkFlags) { buildResult.libs.insert(flag); } if (ext.latestArtifact > externalFloor) externalFloor = ext.latestArtifact; } for(std::unique_ptr& interface : config.interfaces) { if(interface->Check(pcmDir, externalFloor)) { Module* mod = interface.get(); threads.emplace_back([mod, &command, &pcmDir, &buildDir, &buildCancelled, &buildError]() { try { mod->Compile(command, pcmDir, buildDir, buildCancelled, buildError); } catch (const std::exception& e) { bool expected = false; if (buildCancelled.compare_exchange_strong(expected, true)) { buildError = std::format("Module::Compile threw: {}", e.what()); } } }); buildResult.repack = true; } files += std::format(" {}/{}.o", buildDir.string(), interface->path.filename().string()); for(std::unique_ptr& part : interface->partitions) { files += std::format(" {}/{}.o", buildDir.string(), part->path.filename().string()); } } for(Implementation& implementation : config.implementations) { if(implementation.Check(buildDir, pcmDir, externalFloor)) { buildResult.repack = true; Implementation* impl = &implementation; threads.emplace_back([impl, &command, &buildDir, &buildCancelled, &buildError]() { try { impl->Compile(command, buildDir, buildCancelled, buildError); } catch (const std::exception& e) { bool expected = false; if (buildCancelled.compare_exchange_strong(expected, true)) { buildError = std::format("Implementation::Compile threw: {}", e.what()); } } }); } files += std::format(" {}/{}_impl.o", buildDir.string(), implementation.path.filename().string()); } for(std::thread& thread : threads) { thread.join(); } if(buildCancelled.load()) { return {buildError, false, {}}; } std::string linkExtras; for(const std::string& flag : buildResult.libs) { linkExtras += " " + flag; } if(buildResult.repack) { if(config.type == ConfigurationType::Executable) { #ifdef CRAFTER_BUILD_CONFIGURATION_TARGET_x86_64_pc_linux_gnu if(config.target == "x86_64-w64-mingw32") { try { // Iterate over the source directory for (const auto& entry : fs::directory_iterator("/usr/x86_64-w64-mingw32/bin/")) { // Check if the file is a regular file and ends with ".dll" if (fs::is_regular_file(entry) && entry.path().extension() == ".dll") { // Construct the destination file path fs::path dest_file = outputDir / entry.path().filename(); // Check if the destination file exists and if it is older than the source file if (!fs::exists(dest_file) || fs::last_write_time(entry.path()) > fs::last_write_time(dest_file)) { // Copy the file if it doesn't exist or is older fs::copy(entry.path(), dest_file, fs::copy_options::overwrite_existing); } } } } catch (const fs::filesystem_error& e) { return {e.what(), false, {}}; } } buildResult.result = RunCommand(std::format("{}{} -o {} -fuse-ld=lld{}", command, files, (outputDir/config.outputName).string(), linkExtras)); #endif #if defined(CRAFTER_BUILD_CONFIGURATION_TARGET_x86_64_pc_windows_msvc) || defined(CRAFTER_BUILD_CONFIGURATION_TARGET_x86_64_w64_mingw32) std::system(std::format("copy \"%LIBCXX_DIR%\\lib\\c++.dll\" \"{}\\c++.dll\"", outputDir.string()).c_str()); buildResult.result = RunCommand(std::format("{}{} -o {}.exe -fuse-ld=lld -L %LIBCXX_DIR%\\lib -lc++ -nostdinc++ -nostdlib++{}", command, files, (outputDir/config.outputName).string(), linkExtras)); #endif } else if(config.type == ConfigurationType::LibraryStatic) { #ifdef CRAFTER_BUILD_CONFIGURATION_TARGET_x86_64_pc_linux_gnu buildResult.result = RunCommand(std::format("ar rcs {}.a {}", (outputDir/fs::path(std::string("lib")+config.outputName)).string(), files)); #endif #if defined(CRAFTER_BUILD_CONFIGURATION_TARGET_x86_64_pc_windows_msvc) || defined(CRAFTER_BUILD_CONFIGURATION_TARGET_x86_64_w64_mingw32) buildResult.result = RunCommand(std::format("llvm-lib.exe {} /OUT:{}.lib", files, (outputDir/fs::path(config.outputName)).string())); #endif } else { buildResult.result = RunCommand(std::format("{}{} -shared -o {}.so -Wl,-rpath,'$ORIGIN' -fuse-ld=lld{}", command, files, (outputDir/(std::string("lib")+config.outputName)).string(), linkExtras)); } } if (config.type == ConfigurationType::LibraryStatic || config.type == ConfigurationType::LibraryDynamic) { buildResult.libs.insert(std::format("-L{}", outputDir.string())); buildResult.libs.insert(std::format("-l{}", config.outputName)); } return buildResult; } int Crafter::Run(int argc, char** argv) { try { fs::path projectFile = "./project.cpp"; std::vector projectArgs; projectArgs.reserve(argc); bool runTests = false; RunTestsOptions testOpts; for (int i = 1; i < argc; ++i) { std::string_view arg = argv[i]; if (arg == "test") { runTests = true; } else if (arg.starts_with("--project=")) { projectFile = arg.substr(std::string_view("--project=").size()); } else if (runTests && arg.starts_with("--jobs=")) { testOpts.jobs = std::stoi(std::string(arg.substr(std::string_view("--jobs=").size()))); } else if (runTests && arg.starts_with("--timeout=")) { testOpts.timeoutOverride = std::chrono::seconds(std::stoi(std::string(arg.substr(std::string_view("--timeout=").size())))); } else if (runTests && arg == "--list") { testOpts.listOnly = true; } else if (runTests && !arg.starts_with("-")) { testOpts.globs.emplace_back(arg); } else { projectArgs.push_back(arg); } } if (!fs::exists(projectFile)) { std::println(std::cerr, "No project file at {}", projectFile.string()); return 1; } Configuration config = LoadProject(projectFile, projectArgs); if (runTests) { TestSummary summary = RunTests(config, testOpts); return summary.AllPassed() ? 0 : 1; } std::unordered_map> depResults; std::mutex depMutex; BuildResult result = Build(config, depResults, depMutex); if (!result.result.empty()) { std::println(std::cerr, "{}", result.result); return 1; } return 0; } catch (const std::exception& e) { std::println(std::cerr, "{}", e.what()); return 1; } }