Crafter.Build/implementations/Crafter.Build-External.cpp

/*
Crafter® Build
Copyright (C) 2026 Catcrafts®
Catcrafts.net

This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License version 3.0 as published by the Free Software Foundation;

This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.

You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA
*/

module Crafter.Build:External_impl;
import std;
import :External;
import :Platform;
import :Progress;
import :Clang;
namespace fs = std::filesystem;
using namespace Crafter;

namespace {

std::string DeriveName(const GitSource& source) {
    std::string url = source.url;
    while (!url.empty() && (url.back() == '/' || url.back() == '\\')) url.pop_back();
    if (url.ends_with(".git")) url.resize(url.size() - 4);
    auto slash = url.find_last_of("/\\");
    return slash == std::string::npos ? url : url.substr(slash + 1);
}

std::string ShellQuote(std::string_view s) {
    // Windows cmd treats single quotes as literal characters — only double
    // quotes work. Paths don't contain " in practice, so a simple wrap
    // suffices; backslashes inside the path stay backslashes.
#ifdef _WIN32
    std::string out = "\"";
    for (char c : s) {
        if (c == '"') out += "\\\"";
        else out += c;
    }
    out += "\"";
    return out;
#else
    std::string out = "'";
    for (char c : s) {
        if (c == '\'') out += "'\\''";
        else out += c;
    }
    out += "'";
    return out;
#endif
}

std::string JoinOptions(std::span<const std::string> options) {
    std::string out;
    for (const std::string& opt : options) {
        if (!out.empty()) out += ' ';
        out += ShellQuote(opt);
    }
    return out;
}

std::string FetchGit(const GitSource& source, const fs::path& cloneDir) {
    auto runGit = [](std::string_view cmd) -> std::optional<std::string> {
        CommandResult r = RunCommandChecked(cmd);
        if (r.exitCode != 0) return std::format("`{}` failed (exit {}): {}", cmd, r.exitCode, r.output);
        return std::nullopt;
    };

    if (fs::exists(cloneDir)) {
        CommandResult remote = RunCommandChecked(std::format("git -C {} remote get-url origin", ShellQuote(cloneDir.string())));
        std::string currentUrl = remote.output;
        while (!currentUrl.empty() && (currentUrl.back() == '\n' || currentUrl.back() == '\r')) currentUrl.pop_back();
        if (remote.exitCode != 0 || currentUrl != source.url) {
            fs::remove_all(cloneDir);
        } else if (!source.commit.empty()) {
            CommandResult head = RunCommandChecked(std::format("git -C {} rev-parse HEAD", ShellQuote(cloneDir.string())));
            std::string currentHead = head.output;
            while (!currentHead.empty() && (currentHead.back() == '\n' || currentHead.back() == '\r')) currentHead.pop_back();
            if (head.exitCode == 0 && currentHead != source.commit) {
                if (auto err = runGit(std::format("git -C {} fetch origin", ShellQuote(cloneDir.string())))) return *err;
                if (auto err = runGit(std::format("git -C {} checkout {}", ShellQuote(cloneDir.string()), ShellQuote(source.commit)))) return *err;
            }
            return "";
        } else {
            // No commit pinned — pull latest. The user opted into "track latest"
            // by not pinning; the cost is one network round-trip per build.
            // Pin source.commit to a SHA for reproducible, offline-capable builds.
            if (!source.branch.empty()) {
                if (auto err = runGit(std::format("git -C {} pull origin {}", ShellQuote(cloneDir.string()), ShellQuote(source.branch)))) return *err;
            } else {
                if (auto err = runGit(std::format("git -C {} pull", ShellQuote(cloneDir.string())))) return *err;
            }
            return "";
        }
    }

    // Random per-invocation tmp suffix so concurrent crafter-build processes
    // resolving to the same cache key don't share an in-flight `.tmp`
    // directory — git's pack tempfiles would get yanked when one process
    // removes or renames the dir out from under the other.
    std::random_device rd;
    fs::path tmpDir = cloneDir;
    tmpDir += std::format(".tmp.{:08x}{:08x}", rd(), rd());

    if (auto err = runGit(std::format("git clone --recursive {} {}", ShellQuote(source.url), ShellQuote(tmpDir.string())))) {
        if (fs::exists(tmpDir)) fs::remove_all(tmpDir);
        return *err;
    }
    if (!source.branch.empty()) {
        if (auto err = runGit(std::format("git -C {} switch {}", ShellQuote(tmpDir.string()), ShellQuote(source.branch)))) {
            fs::remove_all(tmpDir);
            return *err;
        }
    }
    if (!source.commit.empty()) {
        if (auto err = runGit(std::format("git -C {} checkout {}", ShellQuote(tmpDir.string()), ShellQuote(source.commit)))) {
            fs::remove_all(tmpDir);
            return *err;
        }
    }

    std::error_code ec;
    fs::rename(tmpDir, cloneDir, ec);
    if (ec) {
        fs::remove_all(tmpDir);
        // If a concurrent build won the race, cloneDir now exists with the
        // same source revision (cache key includes url+branch+commit), so
        // discard our clone and reuse theirs.
        if (fs::exists(cloneDir)) return "";
        return std::format("rename {} -> {} failed: {}", tmpDir.string(), cloneDir.string(), ec.message());
    }
    return "";
}

std::string BuildInjectedCMakeFlags(const fs::path& cmakeBuildDir, std::string_view target) {
    std::string out;
    // Without an explicit build type, some packages (glslang) append a 'd'
    // suffix to library names assuming a debug build, breaking -l<name>
    // linking. Pin to Release.
    out += " -DCMAKE_BUILD_TYPE=Release";
    out += " -DCMAKE_C_COMPILER=clang";
    out += " -DCMAKE_CXX_COMPILER=clang++";
    // Stdlib choice: libc++ for the Linux host build (matches how crafter-build
    // itself is built with -stdlib=libc++); mingw cross-compile uses libstdc++
    // shipped by the mingw-w64 toolchain so we leave the default; Windows
    // native (msvc target) similarly uses libc++ via LIBCXX_DIR.
    #ifdef CRAFTER_BUILD_CONFIGURATION_TARGET_x86_64_pc_linux_gnu
    if (target == "x86_64-pc-linux-gnu") {
        out += " -DCMAKE_CXX_FLAGS=-stdlib=libc++";
        out += " -DCMAKE_EXE_LINKER_FLAGS=-stdlib=libc++";
        out += " -DCMAKE_SHARED_LINKER_FLAGS=-stdlib=libc++";
    }
    #endif
    if (target == "x86_64-w64-mingw32") {
        out += " -DCMAKE_SYSTEM_NAME=Windows";
        out += std::format(" -DCMAKE_C_COMPILER_TARGET={}", target);
        out += std::format(" -DCMAKE_CXX_COMPILER_TARGET={}", target);
    }
    fs::path absBuildDir = fs::absolute(cmakeBuildDir);
    out += std::format(" -DCMAKE_ARCHIVE_OUTPUT_DIRECTORY={}", ShellQuote(absBuildDir.string()));
    out += std::format(" -DCMAKE_LIBRARY_OUTPUT_DIRECTORY={}", ShellQuote(absBuildDir.string()));
    return out;
}

std::string ConfigureCMake(const fs::path& cloneDir, const fs::path& cmakeBuildDir, std::span<const std::string> options, std::string_view target) {
    fs::path optionsFile = cmakeBuildDir / ".crafter-options";
    std::string optionsKey = JoinOptions(options);

    bool needsConfigure = !fs::exists(cmakeBuildDir / "CMakeCache.txt");
    if (!needsConfigure) {
        std::ifstream existing(optionsFile);
        if (!existing) {
            needsConfigure = true;
        } else {
            std::stringstream buf;
            buf << existing.rdbuf();
            if (buf.str() != optionsKey) needsConfigure = true;
        }
    }

    if (!needsConfigure) return "";

    if (!fs::exists(cmakeBuildDir)) fs::create_directories(cmakeBuildDir);

    std::string cmd = std::format("cmake -S {} -B {}{}",
        ShellQuote(fs::absolute(cloneDir).string()),
        ShellQuote(fs::absolute(cmakeBuildDir).string()),
        BuildInjectedCMakeFlags(cmakeBuildDir, target));
    if (!options.empty()) {
        cmd += " ";
        cmd += optionsKey;
    }

    CommandResult r = RunCommandChecked(cmd);
    if (r.exitCode != 0) {
        return std::format("cmake configure failed (exit {}): {}", r.exitCode, r.output);
    }

    std::ofstream(optionsFile) << optionsKey;
    return "";
}

std::string BuildCMake(const fs::path& cmakeBuildDir) {
    // Without --parallel, the Unix Makefiles generator builds one translation
    // unit at a time, leaving every core but one idle. Pass an explicit job
    // count (a bare --parallel maps to an unbounded `make -j` fork bomb on the
    // Makefiles generator) so deps like DPP/msquic/glslang compile ~N× faster.
    unsigned jobs = std::max(1u, std::thread::hardware_concurrency());
    std::string cmd = std::format("cmake --build {} --parallel {}",
        ShellQuote(fs::absolute(cmakeBuildDir).string()), jobs);
    CommandResult r = RunCommandChecked(cmd);
    if (r.exitCode != 0) {
        return std::format("cmake --build failed (exit {}): {}", r.exitCode, r.output);
    }
    return "";
}

}  // namespace

ExternalBuildResult Crafter::BuildExternal(
        const ExternalDependency& dep,
        std::string_view target,
        std::atomic<bool>& cancelled) {
    ExternalBuildResult result;

    if (cancelled.load(std::memory_order_relaxed)) return result;

    std::string name = dep.name.empty() ? DeriveName(dep.source) : dep.name;
    if (name.empty()) {
        result.error = std::format("Could not derive name for external dependency from URL '{}'", dep.source.url);
        return result;
    }

    fs::path externalRoot = GetCacheDir() / "external";
    if (!fs::exists(externalRoot)) {
        std::error_code ec;
        fs::create_directories(externalRoot, ec);
        if (ec) {
            result.error = std::format("Failed to create {}: {}", externalRoot.string(), ec.message());
            return result;
        }
    }

    std::string keyMaterial = std::format("{}|{}|{}|{}|{}",
        dep.source.url, dep.source.branch, dep.source.commit, JoinOptions(dep.options), target);
    std::size_t key = std::hash<std::string>{}(keyMaterial);
    fs::path cloneDir = externalRoot / std::format("{}-{:016x}", name, key);

    std::string fetchErr = FetchGit(dep.source, cloneDir);
    if (!fetchErr.empty()) {
        result.error = std::format("git fetch for '{}': {}", name, fetchErr);
        return result;
    }

    if (cancelled.load(std::memory_order_relaxed)) return result;

    fs::path cmakeBuildDir;
    if (dep.builder == ExternalBuilder::CMake) {
        cmakeBuildDir = cloneDir / "build";
        if (std::string err = ConfigureCMake(cloneDir, cmakeBuildDir, dep.options, target); !err.empty()) {
            result.error = std::format("cmake configure for '{}': {}", name, err);
            return result;
        }
        if (cancelled.load(std::memory_order_relaxed)) return result;
        if (std::string err = BuildCMake(cmakeBuildDir); !err.empty()) {
            result.error = std::format("cmake build for '{}': {}", name, err);
            return result;
        }
    }

    for (const fs::path& include : dep.includeDirs) {
        fs::path full = include.empty() ? cloneDir : cloneDir / include;
        result.compileFlags.push_back(std::format("-I{}", fs::absolute(full).string()));
    }

    if (dep.builder == ExternalBuilder::CMake) {
        // Each search path gets both a -L (link-time) and a -Wl,-rpath
        // (runtime-loader). The rpath stays embedded in the produced
        // binary so it picks up shared deps from the cache without any
        // LD_LIBRARY_PATH gymnastics. Static deps (.a) ignore the rpath
        // harmlessly. PE/COFF targets (mingw, msvc) resolve DLLs via PATH
        // or adjacency rather than rpath, so lld warns if we pass it.
        bool isPe = target == "x86_64-w64-mingw32" || target == "x86_64-pc-windows-msvc";
        std::string buildDirAbs = fs::absolute(cmakeBuildDir).string();
        result.linkFlags.push_back(std::format("-L{}", buildDirAbs));
        if (!isPe) result.linkFlags.push_back(std::format("-Wl,-rpath,{}", buildDirAbs));
        for (const fs::path& libDir : dep.libDirs) {
            fs::path full = libDir.is_absolute() ? libDir : cmakeBuildDir / libDir;
            std::string fullAbs = fs::absolute(full).string();
            result.linkFlags.push_back(std::format("-L{}", fullAbs));
            if (!isPe) result.linkFlags.push_back(std::format("-Wl,-rpath,{}", fullAbs));
        }
        for (const std::string& lib : dep.libs) {
            result.linkFlags.push_back(std::format("-l{}", lib));
        }

        if (fs::exists(cmakeBuildDir)) {
            for (const auto& entry : fs::directory_iterator(cmakeBuildDir)) {
                if (!entry.is_regular_file()) continue;
                const fs::path& p = entry.path();
                if (p.extension() != ".a" && p.extension() != ".so") continue;
                std::error_code ec;
                fs::file_time_type t = entry.last_write_time(ec);
                if (!ec && t > result.latestArtifact) result.latestArtifact = t;
            }
        }
    }

    return result;
}

namespace {
    std::mutex projectCacheMutex;
    std::unordered_map<std::string, std::unique_ptr<Configuration>> projectCache;

    // Run the dep's project.cpp from its own directory so its relative paths
    // (cfg.path = "./", interfaces/, lib/, ...) resolve against the dep
    // root, then rebase cfg.path and all other relative path collections to
    // absolute so Build() can run from any cwd without misresolving them.
    // The mutex is NOT held across LoadProject — the dep may itself call
    // GitProject/LocalProject, and a non-recursive lock would deadlock.
    Configuration LoadProjectFromRoot(const fs::path& projectPath, const fs::path& depRoot, std::span<const std::string_view> args) {
        fs::path savedCwd = fs::current_path();
        Configuration cfg;
        fs::current_path(depRoot);
        try {
            cfg = LoadProject(projectPath, args);
        } catch (...) {
            fs::current_path(savedCwd);
            throw;
        }
        fs::current_path(savedCwd);
        cfg.path = fs::absolute(depRoot / cfg.path).lexically_normal();
        auto abs = [&](const fs::path& p) -> fs::path {
            return p.is_absolute() ? p : fs::absolute(depRoot / p).lexically_normal();
        };
        for (fs::path& p : cfg.cFiles)  p = abs(p);
        for (fs::path& p : cfg.cuda)    p = abs(p);
        for (fs::path& p : cfg.files)   p = abs(p);
        for (fs::path& p : cfg.buildFiles) p = abs(p);
        for (Shader& s : cfg.shaders)   s.path = abs(s.path);
        return cfg;
    }
}

Configuration* Crafter::GitProject(const GitProjectSpec& spec) {
    // Two hashes: the clone is shared across specs that point at the same
    // source revision (forwarded args don't change what gets cloned, just
    // how the dep's project.cpp interprets them at runtime), while the
    // in-memory Configuration is keyed by the full spec so each (args)
    // permutation gets its own Configuration object.
    std::string srcKey = std::format("git|{}|{}|{}|{}",
        spec.source.url, spec.source.branch, spec.source.commit, spec.projectFile.string());
    std::string srcHash = std::format("{:016x}", std::hash<std::string>{}(srcKey));

    std::string fullKey = srcKey;
    for (const std::string& a : spec.args) {
        fullKey += '|';
        fullKey += a;
    }
    std::string fullHash = std::format("{:016x}", std::hash<std::string>{}(fullKey));

    {
        std::lock_guard lock(projectCacheMutex);
        if (auto it = projectCache.find(fullHash); it != projectCache.end()) {
            return it->second.get();
        }
    }

    std::string name = DeriveName(spec.source);
    if (name.empty()) {
        throw std::runtime_error(std::format("GitProject: could not derive name from URL '{}'", spec.source.url));
    }

    fs::path externalRoot = GetCacheDir() / "external";
    fs::create_directories(externalRoot);
    fs::path cloneDir = externalRoot / std::format("{}-{}", name, srcHash);

    {
        bool exists = fs::exists(cloneDir);
        Progress::Task task(std::format("{} {}", exists ? "Updating" : "Cloning", name));
        if (std::string err = FetchGit(spec.source, cloneDir); !err.empty()) {
            throw std::runtime_error(std::format("GitProject({}): {}", spec.source.url, err));
        }
    }

    fs::path projectPath = cloneDir / spec.projectFile;
    if (!fs::exists(projectPath)) {
        throw std::runtime_error(std::format(
            "GitProject({}): {} not found in clone", spec.source.url, spec.projectFile.string()));
    }

    std::vector<std::string_view> argViews(spec.args.begin(), spec.args.end());
    Configuration cfg = LoadProjectFromRoot(projectPath, cloneDir, argViews);

    std::lock_guard lock(projectCacheMutex);
    if (auto it = projectCache.find(fullHash); it != projectCache.end()) {
        return it->second.get();
    }
    auto stored = std::make_unique<Configuration>(std::move(cfg));
    Configuration* ptr = stored.get();
    projectCache.emplace(std::move(fullHash), std::move(stored));
    return ptr;
}

fs::path Crafter::GitFetch(const GitSource& source) {
    std::string name = DeriveName(source);
    if (name.empty()) {
        throw std::runtime_error(std::format(
            "GitFetch: could not derive name from URL '{}'", source.url));
    }
    fs::path externalRoot = GetCacheDir() / "external";
    std::error_code ec;
    fs::create_directories(externalRoot, ec);
    if (ec) {
        throw std::runtime_error(std::format(
            "GitFetch: failed to create {}: {}", externalRoot.string(), ec.message()));
    }
    std::string keyMaterial = std::format("git|{}|{}|{}",
        source.url, source.branch, source.commit);
    std::size_t key = std::hash<std::string>{}(keyMaterial);
    fs::path cloneDir = externalRoot / std::format("{}-{:016x}", name, key);

    {
        bool exists = fs::exists(cloneDir);
        Progress::Task task(std::format("{} {}", exists ? "Updating" : "Cloning", name));
        if (std::string err = FetchGit(source, cloneDir); !err.empty()) {
            throw std::runtime_error(std::format("GitFetch({}): {}", source.url, err));
        }
    }
    return cloneDir;
}

Configuration* Crafter::LocalProject(const LocalProjectSpec& spec) {
    fs::path projectPath = fs::absolute(spec.projectFile).lexically_normal();
    if (!fs::exists(projectPath)) {
        throw std::runtime_error(std::format("LocalProject: {} not found", projectPath.string()));
    }
    fs::path canonical = fs::canonical(projectPath);

    std::string fullKey = std::format("local|{}", canonical.string());
    for (const std::string& a : spec.args) {
        fullKey += '|';
        fullKey += a;
    }
    std::string fullHash = std::format("{:016x}", std::hash<std::string>{}(fullKey));

    {
        std::lock_guard lock(projectCacheMutex);
        if (auto it = projectCache.find(fullHash); it != projectCache.end()) {
            return it->second.get();
        }
    }

    fs::path depRoot = canonical.parent_path();
    std::vector<std::string_view> argViews(spec.args.begin(), spec.args.end());
    Configuration cfg = LoadProjectFromRoot(canonical, depRoot, argViews);

    std::lock_guard lock(projectCacheMutex);
    if (auto it = projectCache.find(fullHash); it != projectCache.end()) {
        return it->second.get();
    }
    auto stored = std::make_unique<Configuration>(std::move(cfg));
    Configuration* ptr = stored.get();
    projectCache.emplace(std::move(fullHash), std::move(stored));
    return ptr;
}