Catcrafts/Crafter.Build
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

multithreaded module compilation

Browse source
This commit is contained in:
Jorijn van der Graaf 2024-12-31 20:32:00 +01:00
commit c0a54c8282
13 changed files with 200 additions and 46 deletions
Download patch file Download diff file Expand all files Collapse all files

6
Crafter.Build-Configuration.cpp
View file

@ -1,11 +1,12 @@
/*
Crafter.Build
Copyright (C) 2024 Catcrafts
Catcrafts.net
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 3 of the License, or (at your option) any later version.
version 3.0 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
@ -14,8 +15,7 @@ Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301
USA
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
module;

6
Crafter.Build-Configuration.cppm
View file

@ -1,11 +1,12 @@
/*
Crafter.Build
Copyright (C) 2024 Catcrafts
Catcrafts.net
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 3 of the License, or (at your option) any later version.
version 3.0 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
@ -14,8 +15,7 @@ Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301
USA
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
module;

6
Crafter.Build-Dependency.cpp
View file

@ -1,11 +1,12 @@
/*
Crafter.Build
Copyright (C) 2024 Catcrafts
Catcrafts.net
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 3 of the License, or (at your option) any later version.
version 3.0 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
@ -14,8 +15,7 @@ Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301
USA
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
module;

5
Crafter.Build-Dependency.cppm
View file

@ -5,7 +5,7 @@ Copyright (C) 2024 Catcrafts
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 3 of the License, or (at your option) any later version.
version 3.0 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
@ -14,8 +14,7 @@ Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301
USA
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
module;

106
Crafter.Build-ModuleFile.cpp Normal file
View file

@ -0,0 +1,106 @@
/*
Crafter.Build
Copyright (C) 2024 Catcrafts
Catcrafts.net
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 3.0 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
module;
#include <string>
#include <filesystem>
#include <fstream>
#include <iostream>
#include <regex>
#include <print>
module Crafter.Build;
using namespace Crafter::Build;
namespace fs = std::filesystem;
ModuleFile* ModuleFile::CompileModuleFile(fs::path path, std::string clangDir, const Configuration& config, fs::path pcmDir, std::string target) {
if(fs::exists(path.generic_string()+".cppm")) {
allFilesMutex.lock();
if(!allFiles.contains(path)) {
return new ModuleFile(path, clangDir, config, pcmDir, target);
} else{
allFilesMutex.unlock();
return allFiles[path];
}
} else{
return nullptr;
}
}
ModuleFile::ModuleFile(fs::path path, std::string clangDir, const Configuration& config, fs::path pcmDir, std::string target): path(path), recompiled(false) {
fileMutex.lock();
allFiles.insert({path,this});
allFilesMutex.unlock();
needsRecompiling = !fs::exists((pcmDir/path.filename()).generic_string()+".pcm") || fs::last_write_time(path.generic_string()+".cppm") > fs::last_write_time((pcmDir/path.filename()).generic_string()+".pcm");
std::ifstream t(path.generic_string()+".cppm");
std::stringstream buffer;
buffer << t.rdbuf();
std::string fileContent = buffer.str();
std::regex rgx("import (.*);");
std::smatch matches;
std::regex_search(fileContent, matches, rgx);
std::string::const_iterator searchStart( fileContent.cbegin() );
while ( regex_search( searchStart, fileContent.cend(), matches, rgx ) )
{
std::string matchString = matches[1].str();
if(matchString[0] == ':'){
std::regex rgx2("export module ([a-zA-Z_\\-0-9\\.]*).*;");
std::smatch matches2;
std::regex_search(fileContent, matches2, rgx2);
matchString.erase(0, 1);
matchString = matches2[1].str()+"-"+matchString;
}
fs::path importPath = path.remove_filename()/fs::path(matchString);
ModuleFile* file = CompileModuleFile(importPath,clangDir, config, pcmDir, target);
if(file){
file->fileMutex.lock();
if(file->needsRecompiling){
needsRecompiling = true;
}
file->fileMutex.unlock();
dependencies.push_back(file);
}
searchStart = matches.suffix().first;
}
fileMutex.unlock();
Compile(clangDir, config, pcmDir, target);
}
void ModuleFile::Compile(std::string clangDir, const Configuration& config, fs::path pcmDir, std::string target) {
if(needsRecompiling) {
fileMutex.lock();
if(!recompiled){
for(ModuleFile* dep : dependencies){
dep->Compile(clangDir, config, pcmDir, target);
}
system(std::format("{} -std={} {}.cppm --precompile -fprebuilt-module-path={} -o {}.pcm {}", clangDir, config.standard, path.generic_string(), pcmDir.generic_string(), (pcmDir/path.filename()).generic_string(), target).c_str());
recompiled = true;
fileMutex.unlock();
system(std::format("{} -std={} {}.pcm -fprebuilt-module-path={} -c -O{} -o {}.o {}", clangDir, config.standard, (pcmDir/path.filename()).generic_string(), pcmDir.generic_string(), config.optimizationLevel, (config.buildDir/path.filename()).generic_string(), target).c_str());
} else {
fileMutex.unlock();
}
}
}

45
Crafter.Build-ModuleFile.cppm Normal file
View file

@ -0,0 +1,45 @@
/*
Crafter.Build
Copyright (C) 2024 Catcrafts
Catcrafts.net
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 3.0 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
module;
#include <string>
#include <filesystem>
#include <mutex>
#include <unordered_map>
export module Crafter.Build:ModuleFile;
import :Configuration;
namespace fs = std::filesystem;
export namespace Crafter::Build {
class ModuleFile {
public:
inline static std::mutex allFilesMutex;
std::mutex fileMutex;
inline static std::unordered_map<fs::path, ModuleFile*> allFiles;
bool needsRecompiling;
bool recompiled = false;
fs::path path;
std::vector<ModuleFile*> dependencies;
static ModuleFile* CompileModuleFile(fs::path path, std::string clangDir, const Configuration& config, fs::path pcmDir, std::string target);
private:
ModuleFile(fs::path path, std::string clangDir, const Configuration& config, fs::path pcmDir, std::string target);
void Compile(std::string clangDir, const Configuration& config, fs::path pcmDir, std::string target);
};
}

42
Crafter.Build-Project.cpp
View file

@ -1,11 +1,12 @@
/*
Crafter.Build
Copyright (C) 2024 Catcrafts
Catcrafts.net
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 3 of the License, or (at your option) any later version.
version 3.0 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
@ -14,8 +15,7 @@ Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301
USA
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
module;
@ -111,34 +111,32 @@ void Project::Build(Configuration config, fs::path outputDir) const {
thread.join();
}
//clangDir+= std::format(" -I {} ", pcmDir);
std::vector<fs::path> updatedModules;
std::string files;
for(const fs::path& moduleFile : config.moduleFiles){
if(!fs::exists((pcmDir/moduleFile.filename()).generic_string()+".pcm") || fs::last_write_time(moduleFile.generic_string()+".cppm") > fs::last_write_time((pcmDir/moduleFile.filename()).generic_string()+".pcm")) {
updatedModules.push_back(pcmDir/moduleFile.filename());
system(std::format("{} -std={} {}.cppm --precompile -fprebuilt-module-path={} -o {}.pcm {}", clangDir, config.standard, moduleFile.generic_string(), pcmDir.generic_string(), (pcmDir/moduleFile.filename()).generic_string(), target).c_str());
}
files+=std::format("{}.o ",(config.buildDir/moduleFile.filename()).generic_string());
}
std::vector<std::thread> threads;
for(const fs::path moduleFile : updatedModules) {
threads.emplace_back([moduleFile, config, pcmDir, target, clangDir](){
system(std::format("{} -std={} {}.pcm -fprebuilt-module-path={} -c -O{} -o {}.o {}", clangDir, config.standard, (pcmDir/moduleFile.filename()).generic_string(), pcmDir.generic_string(), config.optimizationLevel, (config.buildDir/moduleFile.filename()).generic_string(), target).c_str());
std::vector<std::thread> moduleThreads(config.moduleFiles.size());
for(std::uint_fast32_t i = 0; i < config.moduleFiles.size(); i++) {
moduleThreads[i] = std::thread([i, &config, pcmDir, target, clangDir](){
ModuleFile::CompileModuleFile(config.moduleFiles[i], clangDir, config, pcmDir, target);
});
files+=std::format("{}.o ",(config.buildDir/config.moduleFiles[i].filename()).generic_string());
}
for(const fs::path& soureFile : config.sourceFiles) {
files+=std::format("{}_source.o ",(config.buildDir/soureFile.filename()).generic_string());
if(!fs::exists((config.buildDir/soureFile.filename()).generic_string()+"_source.o") || fs::last_write_time(soureFile.generic_string()+".cpp") > fs::last_write_time((config.buildDir/soureFile.filename()).generic_string()+"_source.o")) {
threads.emplace_back([soureFile, config, pcmDir, target, clangDir](){
system(std::format("{} -std={} {}.cpp -fprebuilt-module-path={} -c -O{} -o {}_source.o {}", clangDir, config.standard, soureFile.generic_string(), pcmDir.generic_string(), config.optimizationLevel, (config.buildDir/soureFile.filename()).generic_string(), target).c_str());
for(std::thread& thread : moduleThreads){
thread.join();
}
std::vector<std::thread> threads;
for(std::uint_fast32_t i = 0; i < config.sourceFiles.size(); i++) {
files+=std::format("{}_source.o ",(config.buildDir/config.sourceFiles[i].filename()).generic_string());
if(!fs::exists((config.buildDir/config.sourceFiles[i].filename()).generic_string()+"_source.o") || fs::last_write_time(config.sourceFiles[i].generic_string()+".cpp") > fs::last_write_time((config.buildDir/config.sourceFiles[i].filename()).generic_string()+"_source.o")) {
threads.emplace_back([i, &config, pcmDir, target, clangDir](){
system(std::format("{} -std={} {}.cpp -fprebuilt-module-path={} -c -O{} -o {}_source.o {}", clangDir, config.standard, config.sourceFiles[i].generic_string(), pcmDir.generic_string(), config.optimizationLevel, (config.buildDir/config.sourceFiles[i].filename()).generic_string(), target).c_str());
});
}
}
for(std::thread& thread : threads){
thread.join();
}
if(config.type == "executable"){
system(std::format("{} {}-O{} -o {} {} {}", clangDir, files, config.optimizationLevel, (outputDir/name).generic_string(), target, libs).c_str());
} else if(config.type == "library"){

6
Crafter.Build-Project.cppm
View file

@ -1,11 +1,12 @@
/*
Crafter.Build
Copyright (C) 2024 Catcrafts
Catcrafts.net
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 3 of the License, or (at your option) any later version.
version 3.0 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
@ -14,8 +15,7 @@ Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301
USA
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
module;

7
Crafter.Build.cppm
View file

@ -1,11 +1,12 @@
/*
Crafter.Build
Copyright (C) 2024 Catcrafts
Catcrafts.net
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 3 of the License, or (at your option) any later version.
version 3.0 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
@ -14,11 +15,11 @@ Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301
USA
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
export module Crafter.Build;
export import :Dependency;
export import :Project;
export import :Configuration;
export import :ModuleFile;

1
LICENSE
View file

@ -1,4 +1,3 @@
GNU LESSER GENERAL PUBLIC LICENSE
Version 3, 29 June 2007

8
build.sh
View file

@ -4,16 +4,22 @@ mkdir bin
clang++ -std=c++26 Crafter.Build-Dependency.cppm --precompile -fprebuilt-module-path=./build -o ./build/Crafter.Build-Dependency.pcm
clang++ -std=c++26 Crafter.Build-Configuration.cppm --precompile -fprebuilt-module-path=./build -o ./build/Crafter.Build-Configuration.pcm
clang++ -std=c++26 Crafter.Build-ModuleFile.cppm --precompile -fprebuilt-module-path=./build -o ./build/Crafter.Build-ModuleFile.pcm
clang++ -std=c++26 Crafter.Build-Project.cppm --precompile -fprebuilt-module-path=./build -o ./build/Crafter.Build-Project.pcm
clang++ -std=c++26 Crafter.Build.cppm --precompile -fprebuilt-module-path=./build -o ./build/Crafter.Build.pcm
clang++ -std=c++26 Crafter.Build-Dependency.cpp -fprebuilt-module-path=./build -c -O3 -o ./build/Crafter.Build-Dependency_source.o
clang++ -std=c++26 Crafter.Build-Configuration.cpp -fprebuilt-module-path=./build -c -O3 -o ./build/Crafter.Build-Configuration_source.o
clang++ -std=c++26 Crafter.Build-ModuleFile.cpp -fprebuilt-module-path=./build -c -O3 -o ./build/Crafter.Build-ModuleFile_source.o
clang++ -std=c++26 Crafter.Build-Project.cpp -fprebuilt-module-path=./build -c -O3 -o ./build/Crafter.Build-Project_source.o
clang++ -std=c++26 main.cpp -fprebuilt-module-path=./build -c -o ./build/main.o
clang++ -std=c++26 ./build/Crafter.Build-Project.pcm -fprebuilt-module-path=./build -c -O3 -o ./build/Crafter.Build-Project.o
clang++ -std=c++26 ./build/Crafter.Build-Configuration.pcm -fprebuilt-module-path=./build -c -O3 -o ./build/Crafter.Build-Configuration.o
clang++ -std=c++26 ./build/Crafter.Build-ModuleFile.pcm -fprebuilt-module-path=./build -c -O3 -o ./build/Crafter.Build-ModuleFile.o
clang++ -std=c++26 ./build/Crafter.Build-Dependency.pcm -fprebuilt-module-path=./build -c -O3 -o ./build/Crafter.Build-Dependency.o
clang++ -std=c++26 ./build/Crafter.Build.pcm -fprebuilt-module-path=./build -c -O3 -o ./build/Crafter.Build.o
clang++ ./build/main.o ./build/Crafter.Build.o ./build/Crafter.Build-Configuration.o ./build/Crafter.Build-Configuration_source.o ./build/Crafter.Build-Project.o ./build/Crafter.Build-Project_source.o ./build/Crafter.Build-Dependency.o ./build/Crafter.Build-Dependency_source.o -O3 -o ./bin/crafter-build
clang++ ./build/main.o ./build/Crafter.Build-ModuleFile_source.o ./build/Crafter.Build-ModuleFile.o ./build/Crafter.Build.o ./build/Crafter.Build-Configuration.o ./build/Crafter.Build-Configuration_source.o ./build/Crafter.Build-Project.o ./build/Crafter.Build-Project_source.o ./build/Crafter.Build-Dependency.o ./build/Crafter.Build-Dependency_source.o -O3 -o ./bin/crafter-build
rm -rf build;
# crafter-build -c debug

4
main.cpp
View file

@ -1,6 +1,7 @@
/*
Crafter.Build
Copyright (C) 2024 Catcrafts
Catcrafts.net
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
@ -14,8 +15,7 @@ Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301
USA
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include <cstdint>

4
project.json
View file

@ -4,8 +4,8 @@
{
"name": "base",
"standard": "c++26",
"source_files": ["Crafter.Build-Configuration", "Crafter.Build-Project", "Crafter.Build-Dependency"],
"module_files": ["Crafter.Build-Dependency", "Crafter.Build-Configuration", "Crafter.Build-Project", "Crafter.Build"],
"source_files": ["Crafter.Build-Configuration", "Crafter.Build-Project", "Crafter.Build-Dependency", "Crafter.Build-ModuleFile"],
"module_files": ["Crafter.Build-Dependency", "Crafter.Build-Configuration", "Crafter.Build-Project", "Crafter.Build", "Crafter.Build-ModuleFile"],
"build_dir": "./build",
"output_dir": "./bin"
},