fix: rebuild and relink on what changed, submodules at their revisions, and mingw found where it is
Some checks failed
Flatpak / flatpak (push) Failing after 1m12s
Some checks failed
Flatpak / flatpak (push) Failing after 1m12s
- Shaders record the files glslang opened (a depfile) and their settings
(source path, stage, entry point, defines, include directories) in the
build directory, and rebuild when an included file or a setting changes,
not only the shader's own source.
- An executable relinks when its link command changed, not only when an
object is newer: a link flag in project.cpp, or a dependency's library
that moved (an ExternalDependency built into a new cache directory). The
command is recorded beside the objects; its flags come from ordered sets,
so the order the dependency threads finish in can't change it.
- An external's newest artifact counts its libDirs and versioned .so names.
- Submodules are checked out at the revisions the fetched commit records,
from the URLs its .gitmodules gives, after every clone, switch, checkout
and pull, so a pinned commit builds against its own submodules, and one
moved by hand is put back; with no .gitmodules, git isn't asked.
- Each external clone's fetch and build hold a file lock: parallel builds
pulled into one clone at once ("cannot lock ref").
- mingw:
- -stdlib=libstdc++ always;
- the std module's source is found through libstdc++.modules.json in the
mingw toolchain's own library directories (the generic lib/ holds the
host's Linux one), asking the compiler once a run;
- a toolchain that isn't there is a build error saying where it looked,
not std::terminate;
- CRAFTER_MINGW_DIR names the prefix, the directory holding
x86_64-w64-mingw32/ and lib/gcc/;
- inside the Flatpak the host's /usr mingw is used.
- RunScript's exit-status check reads as the one range it is.
- Tests: ConcurrentExternalFetch, ExternalArtifactTime, GitSubmodules,
LinkFlagsChange, MingwMissing, ShaderIncludeChange; ShaderCompile and
ToolchainResolve extended.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
959c65dbb2
commit
653dd0bb9b
26 changed files with 1075 additions and 109 deletions
95
tests/ConcurrentExternalFetch/main.cpp
Normal file
95
tests/ConcurrentExternalFetch/main.cpp
Normal file
|
|
@ -0,0 +1,95 @@
|
|||
// SPDX-License-Identifier: LGPL-3.0-only
|
||||
// SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts®
|
||||
|
||||
import std;
|
||||
import Crafter.Build;
|
||||
namespace fs = std::filesystem;
|
||||
using namespace Crafter;
|
||||
|
||||
// Builds that run at the same time share their dependencies' clones in the
|
||||
// cache: the tests `crafter-build test --jobs=N` builds side by side each
|
||||
// declare the same ExternalDependency, and two crafter-builds can run at
|
||||
// once. Fetching a clone that already exists ran `git pull` (and the CMake
|
||||
// build) in it with nothing to keep a second fetch out, and the second git
|
||||
// fails on the first one's index.lock. A clone's fetch and build now hold a
|
||||
// lock of their own.
|
||||
//
|
||||
// Eight threads fetch the same tracked branch of a local repository at
|
||||
// once, a few rounds over, the branch moving between rounds so every pull
|
||||
// has work to do; every fetch has to succeed. Everything is local;
|
||||
// XDG_CACHE_HOME keeps the clone out of the real cache.
|
||||
|
||||
extern "C" int setenv(const char* name, const char* value, int overwrite);
|
||||
|
||||
namespace {
|
||||
std::int32_t failures = 0;
|
||||
|
||||
void Check(bool cond, std::string_view msg) {
|
||||
if (!cond) {
|
||||
std::println(std::cerr, "FAIL: {}", msg);
|
||||
++failures;
|
||||
}
|
||||
}
|
||||
|
||||
void Git(const fs::path& dir, std::string_view args) {
|
||||
const std::string command = std::format("git -C {} {}", ShellQuote(dir.string()), args);
|
||||
auto result = RunCommandWithTimeout(command, std::chrono::seconds(60));
|
||||
if (result.exitCode != 0) throw std::runtime_error(std::format("{} failed: {}", command, result.output));
|
||||
}
|
||||
|
||||
void Commit(const fs::path& repo, std::uint32_t round) {
|
||||
std::ofstream(repo / "round.txt", std::ios::binary | std::ios::trunc) << round;
|
||||
Git(repo, "add round.txt");
|
||||
Git(repo, std::format("commit -q -m round-{}", round));
|
||||
}
|
||||
}
|
||||
|
||||
int main() {
|
||||
const fs::path scratch = fs::temp_directory_path() / "crafter-build-concurrent-external-fetch";
|
||||
fs::remove_all(scratch);
|
||||
fs::create_directories(scratch);
|
||||
setenv("XDG_CACHE_HOME", (scratch / "cache").c_str(), 1);
|
||||
setenv("GIT_CONFIG_NOSYSTEM", "1", 1);
|
||||
setenv("GIT_CONFIG_GLOBAL", "/dev/null", 1);
|
||||
for (const char* name : { "GIT_AUTHOR_NAME", "GIT_COMMITTER_NAME" }) setenv(name, "crafter-build test", 1);
|
||||
for (const char* name : { "GIT_AUTHOR_EMAIL", "GIT_COMMITTER_EMAIL" }) setenv(name, "test@example.invalid", 1);
|
||||
|
||||
const fs::path repo = scratch / "shared";
|
||||
fs::create_directories(repo);
|
||||
Git(repo, "init -q -b main");
|
||||
Commit(repo, 0);
|
||||
const GitSource source { .url = std::format("file://{}", repo.string()), .branch = "main" };
|
||||
const fs::path clone = GitFetch(source);
|
||||
|
||||
constexpr std::uint32_t threads = 8;
|
||||
constexpr std::uint32_t rounds = 4;
|
||||
std::vector<std::string> errors;
|
||||
std::mutex errorsMutex;
|
||||
for (std::uint32_t round = 1; round <= rounds; ++round) {
|
||||
Commit(repo, round);
|
||||
std::vector<std::jthread> fetchers;
|
||||
for (std::uint32_t i = 0; i < threads; ++i) {
|
||||
fetchers.emplace_back([&] {
|
||||
try {
|
||||
(void)GitFetch(source);
|
||||
} catch (const std::exception& e) {
|
||||
std::lock_guard lock(errorsMutex);
|
||||
errors.push_back(e.what());
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
Check(errors.empty(), std::format("{} fetches at once, {} rounds: every one succeeds ({} failed{}{})", threads, rounds, errors.size(), errors.empty() ? "" : ", the first: ", errors.empty() ? "" : errors.front()));
|
||||
std::ifstream round(clone / "round.txt");
|
||||
std::uint32_t last = 0;
|
||||
round >> last;
|
||||
Check(last == rounds, std::format("and the clone ends at the last round ({})", last));
|
||||
|
||||
std::error_code ec;
|
||||
fs::remove_all(scratch, ec);
|
||||
if (failures > 0) {
|
||||
std::println(std::cerr, "{} assertions failed", failures);
|
||||
return 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
Loading…
Reference in a new issue