multithreaded module compilation

Crafter.Build-Configuration.cpp

@@ -1,11 +1,12 @@
 /*
 Crafter.Build
 Copyright (C) 2024 Catcrafts
+Catcrafts.net
 
 This library is free software; you can redistribute it and/or
 modify it under the terms of the GNU Lesser General Public
 License as published by the Free Software Foundation; either
-version 3 of the License, or (at your option) any later version.
+version 3.0 of the License, or (at your option) any later version.
 
  This library is distributed in the hope that it will be useful,
 but WITHOUT ANY WARRANTY; without even the implied warranty of
@@ -14,8 +15,7 @@ Lesser General Public License for more details.
 
 You should have received a copy of the GNU Lesser General Public
 License along with this library; if not, write to the Free Software
-Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA
-USA
 */
 
 module;

Crafter.Build-Configuration.cppm

@@ -1,11 +1,12 @@
 /*
 Crafter.Build
 Copyright (C) 2024 Catcrafts
+Catcrafts.net
 
 This library is free software; you can redistribute it and/or
 modify it under the terms of the GNU Lesser General Public
 License as published by the Free Software Foundation; either
-version 3 of the License, or (at your option) any later version.
+version 3.0 of the License, or (at your option) any later version.
 
 This library is distributed in the hope that it will be useful,
 but WITHOUT ANY WARRANTY; without even the implied warranty of
@@ -14,8 +15,7 @@ Lesser General Public License for more details.
 
 You should have received a copy of the GNU Lesser General Public
 License along with this library; if not, write to the Free Software
-Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA
-USA
 */
 
 module;

Crafter.Build-Dependency.cpp

@@ -1,11 +1,12 @@
 /*
 Crafter.Build
 Copyright (C) 2024 Catcrafts
+Catcrafts.net
 
 This library is free software; you can redistribute it and/or
 modify it under the terms of the GNU Lesser General Public
 License as published by the Free Software Foundation; either
-version 3 of the License, or (at your option) any later version.
+version 3.0 of the License, or (at your option) any later version.
 
 This library is distributed in the hope that it will be useful,
 but WITHOUT ANY WARRANTY; without even the implied warranty of
@@ -14,8 +15,7 @@ Lesser General Public License for more details.
 
 You should have received a copy of the GNU Lesser General Public
 License along with this library; if not, write to the Free Software
-Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA
-USA
 */
 
 module;

Crafter.Build-Dependency.cppm

@@ -5,7 +5,7 @@ Copyright (C) 2024 Catcrafts
 This library is free software; you can redistribute it and/or
 modify it under the terms of the GNU Lesser General Public
 License as published by the Free Software Foundation; either
-version 3 of the License, or (at your option) any later version.
+version 3.0 of the License, or (at your option) any later version.
 
 This library is distributed in the hope that it will be useful,
 but WITHOUT ANY WARRANTY; without even the implied warranty of
@@ -14,8 +14,7 @@ Lesser General Public License for more details.
 
 You should have received a copy of the GNU Lesser General Public
 License along with this library; if not, write to the Free Software
-Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA
-USA
 */
 
 module;

Crafter.Build-ModuleFile.cpp

@@ -0,0 +1,106 @@
+/*
+Crafter.Build
+Copyright (C) 2024 Catcrafts
+Catcrafts.net
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Lesser General Public
+License as published by the Free Software Foundation; either
+version 3.0 of the License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public
+License along with this library; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA
+*/
+
+module;
+#include <string>
+#include <filesystem>
+#include <fstream>
+#include <iostream>
+#include <regex>
+#include <print>
+module Crafter.Build;
+using namespace Crafter::Build;
+namespace fs = std::filesystem;
+
+
+ModuleFile* ModuleFile::CompileModuleFile(fs::path path, std::string clangDir, const Configuration& config, fs::path pcmDir, std::string target) {
+    if(fs::exists(path.generic_string()+".cppm")) {
+        allFilesMutex.lock();
+        if(!allFiles.contains(path)) {
+            return new ModuleFile(path, clangDir, config, pcmDir, target);
+        } else{
+            allFilesMutex.unlock();
+            return allFiles[path];
+        }
+    } else{
+        return nullptr;
+    }
+}
+
+ModuleFile::ModuleFile(fs::path path, std::string clangDir, const Configuration& config, fs::path pcmDir, std::string target): path(path), recompiled(false) {
+    fileMutex.lock();
+    allFiles.insert({path,this});
+    allFilesMutex.unlock();
+    needsRecompiling = !fs::exists((pcmDir/path.filename()).generic_string()+".pcm") || fs::last_write_time(path.generic_string()+".cppm") > fs::last_write_time((pcmDir/path.filename()).generic_string()+".pcm");
+    std::ifstream t(path.generic_string()+".cppm");
+    std::stringstream buffer;
+    buffer << t.rdbuf();
+    std::string fileContent = buffer.str();
+
+    std::regex rgx("import (.*);");
+    std::smatch matches;
+    std::regex_search(fileContent, matches, rgx);
+
+    std::string::const_iterator searchStart( fileContent.cbegin() );
+    while ( regex_search( searchStart, fileContent.cend(), matches, rgx ) )
+    {
+        std::string matchString = matches[1].str();
+        if(matchString[0] == ':'){
+            std::regex rgx2("export module ([a-zA-Z_\\-0-9\\.]*).*;");
+            std::smatch matches2;
+            std::regex_search(fileContent, matches2, rgx2);
+            matchString.erase(0, 1);
+            matchString = matches2[1].str()+"-"+matchString;
+        }
+        fs::path importPath = path.remove_filename()/fs::path(matchString);
+        ModuleFile* file = CompileModuleFile(importPath,clangDir, config, pcmDir, target);
+        if(file){
+            file->fileMutex.lock();
+
+            if(file->needsRecompiling){
+                needsRecompiling = true;
+            }
+
+            file->fileMutex.unlock();
+            dependencies.push_back(file);
+        }
+        searchStart = matches.suffix().first;
+    }
+    fileMutex.unlock();
+    Compile(clangDir, config, pcmDir, target);
+}
+
+void ModuleFile::Compile(std::string clangDir, const Configuration& config, fs::path pcmDir, std::string target) {
+    if(needsRecompiling) {
+        fileMutex.lock();
+        if(!recompiled){
+            for(ModuleFile* dep : dependencies){
+                dep->Compile(clangDir, config, pcmDir, target);
+            }
+            system(std::format("{} -std={} {}.cppm --precompile -fprebuilt-module-path={} -o {}.pcm {}", clangDir, config.standard, path.generic_string(), pcmDir.generic_string(), (pcmDir/path.filename()).generic_string(), target).c_str());
+
+            recompiled = true;
+            fileMutex.unlock();
+            system(std::format("{} -std={} {}.pcm -fprebuilt-module-path={} -c -O{} -o {}.o {}", clangDir, config.standard, (pcmDir/path.filename()).generic_string(), pcmDir.generic_string(), config.optimizationLevel, (config.buildDir/path.filename()).generic_string(), target).c_str());
+        } else {
+            fileMutex.unlock();
+        }
+    }
+}

Crafter.Build-ModuleFile.cppm

@@ -0,0 +1,45 @@
+/*
+Crafter.Build
+Copyright (C) 2024 Catcrafts
+Catcrafts.net
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Lesser General Public
+License as published by the Free Software Foundation; either
+version 3.0 of the License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public
+License along with this library; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA
+*/
+
+module;
+#include <string>
+#include <filesystem>
+#include <mutex>
+#include <unordered_map>
+export module Crafter.Build:ModuleFile;
+import :Configuration;
+namespace fs = std::filesystem;
+
+export namespace Crafter::Build {
+    class ModuleFile {
+    public:
+        inline static std::mutex allFilesMutex;
+        std::mutex fileMutex;
+        inline static std::unordered_map<fs::path, ModuleFile*> allFiles;
+        bool needsRecompiling;
+        bool recompiled = false;
+        fs::path path;
+        std::vector<ModuleFile*> dependencies;
+        static ModuleFile* CompileModuleFile(fs::path path, std::string clangDir, const Configuration& config, fs::path pcmDir, std::string target);
+    private:
+        ModuleFile(fs::path path, std::string clangDir, const Configuration& config, fs::path pcmDir, std::string target);
+        void Compile(std::string clangDir, const Configuration& config, fs::path pcmDir, std::string target);
+    };
+}

Crafter.Build-Project.cpp

@@ -1,11 +1,12 @@
 /*
 Crafter.Build
 Copyright (C) 2024 Catcrafts
+Catcrafts.net
 
 This library is free software; you can redistribute it and/or
 modify it under the terms of the GNU Lesser General Public
 License as published by the Free Software Foundation; either
-version 3 of the License, or (at your option) any later version.
+version 3.0 of the License, or (at your option) any later version.
 
 This library is distributed in the hope that it will be useful,
 but WITHOUT ANY WARRANTY; without even the implied warranty of
@@ -14,8 +15,7 @@ Lesser General Public License for more details.
 
 You should have received a copy of the GNU Lesser General Public
 License along with this library; if not, write to the Free Software
-Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA
-USA
 */
 
 module;
@@ -111,34 +111,32 @@ void Project::Build(Configuration config, fs::path outputDir) const {
         thread.join();
     }
 
-    //clangDir+= std::format(" -I {} ", pcmDir);
-
-    std::vector<fs::path> updatedModules;
     std::string files;
-    for(const fs::path& moduleFile : config.moduleFiles){
+    std::vector<std::thread> moduleThreads(config.moduleFiles.size());
-        if(!fs::exists((pcmDir/moduleFile.filename()).generic_string()+".pcm") || fs::last_write_time(moduleFile.generic_string()+".cppm") > fs::last_write_time((pcmDir/moduleFile.filename()).generic_string()+".pcm")) {
+    for(std::uint_fast32_t i = 0; i < config.moduleFiles.size(); i++) {
-            updatedModules.push_back(pcmDir/moduleFile.filename());
+        moduleThreads[i] = std::thread([i, &config, pcmDir, target, clangDir](){
-            system(std::format("{} -std={} {}.cppm --precompile -fprebuilt-module-path={} -o {}.pcm {}", clangDir, config.standard, moduleFile.generic_string(), pcmDir.generic_string(), (pcmDir/moduleFile.filename()).generic_string(), target).c_str());
+            ModuleFile::CompileModuleFile(config.moduleFiles[i], clangDir, config, pcmDir, target);
-        }
-        files+=std::format("{}.o ",(config.buildDir/moduleFile.filename()).generic_string());
-    }
-    std::vector<std::thread> threads;
-    for(const fs::path moduleFile : updatedModules) {
-        threads.emplace_back([moduleFile, config, pcmDir, target, clangDir](){
-            system(std::format("{} -std={} {}.pcm -fprebuilt-module-path={} -c -O{} -o {}.o {}", clangDir, config.standard, (pcmDir/moduleFile.filename()).generic_string(), pcmDir.generic_string(), config.optimizationLevel, (config.buildDir/moduleFile.filename()).generic_string(), target).c_str());
         });
+        files+=std::format("{}.o ",(config.buildDir/config.moduleFiles[i].filename()).generic_string());
     }
-    for(const fs::path& soureFile : config.sourceFiles) {
+    for(std::thread& thread : moduleThreads){
-        files+=std::format("{}_source.o ",(config.buildDir/soureFile.filename()).generic_string());
+        thread.join();
-        if(!fs::exists((config.buildDir/soureFile.filename()).generic_string()+"_source.o") || fs::last_write_time(soureFile.generic_string()+".cpp") > fs::last_write_time((config.buildDir/soureFile.filename()).generic_string()+"_source.o")) {
+    }
-            threads.emplace_back([soureFile, config, pcmDir, target, clangDir](){
+
-                system(std::format("{} -std={} {}.cpp -fprebuilt-module-path={} -c -O{} -o {}_source.o {}", clangDir, config.standard, soureFile.generic_string(), pcmDir.generic_string(), config.optimizationLevel, (config.buildDir/soureFile.filename()).generic_string(), target).c_str());
+    std::vector<std::thread> threads;
+    for(std::uint_fast32_t i = 0; i < config.sourceFiles.size(); i++) {
+        files+=std::format("{}_source.o ",(config.buildDir/config.sourceFiles[i].filename()).generic_string());
+        if(!fs::exists((config.buildDir/config.sourceFiles[i].filename()).generic_string()+"_source.o") || fs::last_write_time(config.sourceFiles[i].generic_string()+".cpp") > fs::last_write_time((config.buildDir/config.sourceFiles[i].filename()).generic_string()+"_source.o")) {
+            threads.emplace_back([i, &config, pcmDir, target, clangDir](){
+                system(std::format("{} -std={} {}.cpp -fprebuilt-module-path={} -c -O{} -o {}_source.o {}", clangDir, config.standard, config.sourceFiles[i].generic_string(), pcmDir.generic_string(), config.optimizationLevel, (config.buildDir/config.sourceFiles[i].filename()).generic_string(), target).c_str());
             });
         }
     }
+
     for(std::thread& thread : threads){
         thread.join();
     }
+
     if(config.type == "executable"){
          system(std::format("{} {}-O{} -o {} {} {}", clangDir, files, config.optimizationLevel, (outputDir/name).generic_string(), target, libs).c_str());
     } else if(config.type == "library"){

Crafter.Build-Project.cppm

@@ -1,11 +1,12 @@
 /*
 Crafter.Build
 Copyright (C) 2024 Catcrafts
+Catcrafts.net
 
 This library is free software; you can redistribute it and/or
 modify it under the terms of the GNU Lesser General Public
 License as published by the Free Software Foundation; either
-version 3 of the License, or (at your option) any later version.
+version 3.0 of the License, or (at your option) any later version.
 
 This library is distributed in the hope that it will be useful,
 but WITHOUT ANY WARRANTY; without even the implied warranty of
@@ -14,8 +15,7 @@ Lesser General Public License for more details.
 
 You should have received a copy of the GNU Lesser General Public
 License along with this library; if not, write to the Free Software
-Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA
-USA
 */
 
 module;

Crafter.Build.cppm

@@ -1,11 +1,12 @@
 /*
 Crafter.Build
 Copyright (C) 2024 Catcrafts
+Catcrafts.net
 
 This library is free software; you can redistribute it and/or
 modify it under the terms of the GNU Lesser General Public
 License as published by the Free Software Foundation; either
-version 3 of the License, or (at your option) any later version.
+version 3.0 of the License, or (at your option) any later version.
 
 This library is distributed in the hope that it will be useful,
 but WITHOUT ANY WARRANTY; without even the implied warranty of
@@ -14,11 +15,11 @@ Lesser General Public License for more details.
 
 You should have received a copy of the GNU Lesser General Public
 License along with this library; if not, write to the Free Software
-Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA
-USA
 */
 
 export module Crafter.Build;
 export import :Dependency;
 export import :Project;
 export import :Configuration;
+export import :ModuleFile;

LICENSE

@@ -1,4 +1,3 @@
-
 GNU LESSER GENERAL PUBLIC LICENSE
 
 Version 3, 29 June 2007

build.sh

@@ -4,16 +4,22 @@ mkdir bin
 
 clang++ -std=c++26 Crafter.Build-Dependency.cppm --precompile -fprebuilt-module-path=./build -o ./build/Crafter.Build-Dependency.pcm
 clang++ -std=c++26 Crafter.Build-Configuration.cppm --precompile -fprebuilt-module-path=./build -o ./build/Crafter.Build-Configuration.pcm
+clang++ -std=c++26 Crafter.Build-ModuleFile.cppm --precompile -fprebuilt-module-path=./build -o ./build/Crafter.Build-ModuleFile.pcm
 clang++ -std=c++26 Crafter.Build-Project.cppm --precompile -fprebuilt-module-path=./build -o ./build/Crafter.Build-Project.pcm
 clang++ -std=c++26 Crafter.Build.cppm --precompile -fprebuilt-module-path=./build -o ./build/Crafter.Build.pcm
 
 clang++ -std=c++26 Crafter.Build-Dependency.cpp -fprebuilt-module-path=./build -c -O3 -o ./build/Crafter.Build-Dependency_source.o
 clang++ -std=c++26 Crafter.Build-Configuration.cpp -fprebuilt-module-path=./build -c -O3 -o ./build/Crafter.Build-Configuration_source.o
+clang++ -std=c++26 Crafter.Build-ModuleFile.cpp -fprebuilt-module-path=./build -c -O3 -o ./build/Crafter.Build-ModuleFile_source.o
 clang++ -std=c++26 Crafter.Build-Project.cpp -fprebuilt-module-path=./build -c -O3 -o ./build/Crafter.Build-Project_source.o
 clang++ -std=c++26 main.cpp -fprebuilt-module-path=./build -c -o ./build/main.o
 
 clang++ -std=c++26 ./build/Crafter.Build-Project.pcm -fprebuilt-module-path=./build -c -O3 -o ./build/Crafter.Build-Project.o
 clang++ -std=c++26 ./build/Crafter.Build-Configuration.pcm -fprebuilt-module-path=./build -c -O3 -o ./build/Crafter.Build-Configuration.o
+clang++ -std=c++26 ./build/Crafter.Build-ModuleFile.pcm -fprebuilt-module-path=./build -c -O3 -o ./build/Crafter.Build-ModuleFile.o
 clang++ -std=c++26 ./build/Crafter.Build-Dependency.pcm -fprebuilt-module-path=./build -c -O3 -o ./build/Crafter.Build-Dependency.o
 clang++ -std=c++26 ./build/Crafter.Build.pcm -fprebuilt-module-path=./build -c -O3 -o ./build/Crafter.Build.o
-clang++ ./build/main.o ./build/Crafter.Build.o ./build/Crafter.Build-Configuration.o ./build/Crafter.Build-Configuration_source.o ./build/Crafter.Build-Project.o ./build/Crafter.Build-Project_source.o ./build/Crafter.Build-Dependency.o ./build/Crafter.Build-Dependency_source.o -O3 -o ./bin/crafter-build
+clang++ ./build/main.o ./build/Crafter.Build-ModuleFile_source.o ./build/Crafter.Build-ModuleFile.o ./build/Crafter.Build.o ./build/Crafter.Build-Configuration.o ./build/Crafter.Build-Configuration_source.o ./build/Crafter.Build-Project.o ./build/Crafter.Build-Project_source.o ./build/Crafter.Build-Dependency.o ./build/Crafter.Build-Dependency_source.o -O3 -o ./bin/crafter-build
+
+rm -rf build;
+# crafter-build -c debug

main.cpp

@@ -1,6 +1,7 @@
 /*
 Crafter.Build
 Copyright (C) 2024 Catcrafts
+Catcrafts.net
 
 This library is free software; you can redistribute it and/or
 modify it under the terms of the GNU Lesser General Public
@@ -14,8 +15,7 @@ Lesser General Public License for more details.
 
 You should have received a copy of the GNU Lesser General Public
 License along with this library; if not, write to the Free Software
-Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA
-USA
 */
 
 #include <cstdint>

project.json

@@ -4,8 +4,8 @@
         {
             "name": "base",
             "standard": "c++26",
-            "source_files": ["Crafter.Build-Configuration", "Crafter.Build-Project", "Crafter.Build-Dependency"],
+            "source_files": ["Crafter.Build-Configuration", "Crafter.Build-Project", "Crafter.Build-Dependency", "Crafter.Build-ModuleFile"],
-            "module_files": ["Crafter.Build-Dependency", "Crafter.Build-Configuration", "Crafter.Build-Project", "Crafter.Build"],
+            "module_files": ["Crafter.Build-Dependency", "Crafter.Build-Configuration", "Crafter.Build-Project", "Crafter.Build", "Crafter.Build-ModuleFile"],
             "build_dir": "./build",
             "output_dir": "./bin"
         },