diff --git a/AGENTS.md b/AGENTS.md index a97508b..6d7b0b1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -29,6 +29,7 @@ Partitions and what they own: - `:Test` holds `AddTest`, `RunTests` and `TestRunner`. - `:Lint` is the lint/format driver plus the libclang token and AST layer. - `:Script` is the driver behind `crafter-build run`: it looks a name up in `cfg.scripts`, calls the callback and turns what it returns into the exit code. +- `:Agents` keeps the dependency notes section of a project's AGENTS.md: it walks the dependencies `cfg.dependencyNotes` selects, reads each one's `AGENTS.consumer.md` and merges them between the section's markers. Which Configurations belong to another project, and where it lives, comes from the loader (`LoadedProjectOf` in `:External`, `PragmaDependencies` in `:Platform`), because only the Configuration `GitProject` and `LocalProject` return has its `path` rebased. - `:Progress` is console output. `:Shader` compiles shaders through glslang. `:Asset` is the hook into Crafter.Asset. ## Dev environment @@ -188,3 +189,4 @@ Local builds are two scripts in `project.cpp`. Run them with the native binary: - A warning is printed only when its unit actually recompiles. A build with nothing to do is silent, which is not evidence that a warning is gone. - A std module source only compiles against the headers of its own libc++ release, and a sysroot does not always ship one (Alpine). The Flatpak therefore carries the sources of the release next to the bundled one under `share/crafter-build/libc++-modules//`, one `libcxx--modules` manifest module each. When the bundled LLVM is bumped, keep the release it leaves behind as such a module: sysroots on the stable distro stay on it for months. The native loop has no such directory; stage one by hand under `share/crafter-build/` to test against a foreign sysroot. - `VariantId()` names the output directory, so CI globs `bin/crafter.build-exe---*/` rather than predicting the hash. The hash includes the compiler, so after a toolchain update the previous variant's directories stay in `bin/` and `build/` until `clean`, and such a glob matches both. +- The self-host build and `test` keep the dependency notes section of this very file. None of Crafter.Math, Crafter.Asset and Crafter.LintRules ships an `AGENTS.consumer.md` yet; once one does, building here appends the section. Commit it with the change that pulled it in. diff --git a/build.sh b/build.sh index 312abde..02b089b 100755 --- a/build.sh +++ b/build.sh @@ -48,7 +48,7 @@ cp wasi-runtime/index.html.in share/crafter-build/wasi-runtime/ # Partition order is the import order: every unit is listed before the # units that import it, and the umbrella last. -PARTITIONS="Toolchain Shader Platform Interface Implementation External Clang Test Lint Script Progress Asset" +PARTITIONS="Toolchain Shader Platform Interface Implementation External Agents Clang Test Lint Script Progress Asset" cp interfaces/Crafter.Build.cppm share/crafter-build/ for p in $PARTITIONS; do diff --git a/implementations/Crafter.Build-Agents.cpp b/implementations/Crafter.Build-Agents.cpp new file mode 100644 index 0000000..f3811ef --- /dev/null +++ b/implementations/Crafter.Build-Agents.cpp @@ -0,0 +1,320 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts® + +module; +#include +module Crafter.Build:Agents_impl; +import std; +import :Agents; +import :Clang; +import :External; +import :Platform; +import :Progress; +namespace fs = std::filesystem; +using namespace Crafter; + +namespace { + constexpr std::string_view BeginMarker = ""; + constexpr std::string_view EndMarker = ""; + // Right under the begin marker, for whoever opens the file: an edit made + // inside the section is gone after the next build. + constexpr std::string_view Preamble = ""; + constexpr std::string_view NotesFileName = "AGENTS.consumer.md"; + + void Warn(std::string_view message) { + Progress::Diagnostic(std::format("warning: {}", message)); + } + + struct Line { + std::string_view text; // without the newline and a '\r' before it + std::size_t begin = 0; // offset of the line's first byte + std::size_t next = 0; // offset of the line after it + }; + + std::vector SplitLines(std::string_view text) { + std::vector lines; + std::size_t pos = 0; + while (pos < text.size()) { + const std::size_t newline = text.find('\n', pos); + const std::size_t end = newline == std::string_view::npos ? text.size() : newline; + std::string_view line = text.substr(pos, end - pos); + if (line.ends_with('\r')) line.remove_suffix(1); + const std::size_t next = newline == std::string_view::npos ? text.size() : newline + 1; + lines.push_back({ .text = line, .begin = pos, .next = next }); + pos = next; + } + return lines; + } + + std::string_view TrimEnd(std::string_view text) { + const std::size_t last = text.find_last_not_of(" \t"); + return last == std::string_view::npos ? std::string_view() : text.substr(0, last + 1); + } + + // Fenced code blocks as CommonMark has them: a run of three or more + // backticks or tildes, after at most three spaces, opens one, and a run + // of the same character at least as long, with nothing after it but + // spaces, closes it. Tracking the character and the length is what lets + // a ~~~ block quote a ``` one, the usual way to show a fenced example. + class CodeFences { + public: + // Feed the lines in order. True for a fence and for every line + // inside a block. + bool InCode(std::string_view line) { + std::size_t pos = 0; + while (pos < line.size() && pos < 3 && line[pos] == ' ') ++pos; + const char ch = pos < line.size() ? line[pos] : '\0'; + std::size_t run = 0; + if (ch == '`' || ch == '~') { + while (pos + run < line.size() && line[pos + run] == ch) ++run; + } + const bool fence = run >= 3; + if (open_ == '\0') { + if (!fence) return false; + open_ = ch; + length_ = run; + return true; + } + if (fence && ch == open_ && run >= length_ && TrimEnd(line.substr(pos + run)).empty()) open_ = '\0'; + return true; + } + bool Open() const { return open_ != '\0'; } + + private: + char open_ = '\0'; + std::size_t length_ = 0; + }; + + // Where a text's marker lines are, outside code blocks, as indices into + // `lines`. + struct MarkerScan { + std::vector lines; + std::vector begins; + std::vector ends; + // The line a code block that never closes opens on. + std::optional unclosedBlock; + }; + + MarkerScan ScanMarkers(std::string_view text) { + MarkerScan scan; + scan.lines = SplitLines(text); + CodeFences fences; + for (std::size_t i = 0; i < scan.lines.size(); ++i) { + const bool wasOpen = fences.Open(); + if (fences.InCode(scan.lines[i].text)) { + if (!wasOpen) scan.unclosedBlock = i; + continue; + } + const std::string_view line = TrimEnd(scan.lines[i].text); + if (line == BeginMarker) scan.begins.push_back(i); + if (line == EndMarker) scan.ends.push_back(i); + } + if (!fences.Open()) scan.unclosedBlock.reset(); + return scan; + } + + // Why `text` cannot go into the section as it is; empty when it can. A + // marker line of its own would end or restart the section once the file + // is read back, and a code block left open would hide the end marker: + // either way the next build could no longer find what this one wrote. + std::string NoteProblem(std::string_view text) { + const MarkerScan scan = ScanMarkers(text); + if (!scan.begins.empty() || !scan.ends.empty()) { + std::size_t line = scan.lines.size(); + if (!scan.begins.empty()) line = scan.begins[0]; + if (!scan.ends.empty()) line = std::min(line, scan.ends[0]); + return std::format("line {} is a dependency notes marker", line + 1); + } + if (scan.unclosedBlock) return std::format("line {} opens a code block that never closes", *scan.unclosedBlock + 1); + return ""; + } + + std::string RenderSection(std::span notes) { + std::string section = std::format("{}\n{}\n", BeginMarker, Preamble); + for (const DependencyNote& note : notes) section += std::format("\n\n{}\n", note.source, note.text); + section += std::format("\n{}\n", EndMarker); + return section; + } + + // Without leading blank lines and trailing whitespace, so the section + // spaces its notes the same whatever the files end with. + std::string TrimBlank(std::string_view text) { + const std::size_t last = text.find_last_not_of(" \t\r\n"); + if (last == std::string_view::npos) return ""; + const std::size_t first = text.find_first_not_of(" \t\r\n"); + const std::size_t newline = text.rfind('\n', first); + const std::size_t begin = newline == std::string_view::npos ? 0 : newline + 1; + return std::string(text.substr(begin, last + 1 - begin)); + } + + std::optional ReadFile(const fs::path& file) { + std::ifstream in(file, std::ios::binary); + if (!in) return std::nullopt; + std::string content((std::istreambuf_iterator(in)), std::istreambuf_iterator()); + if (in.bad()) return std::nullopt; + return content; + } + + // Replace the content of `file` through a temporary beside it: a write + // cut short (a full disk, a crash) leaves the old AGENTS.md rather than + // part of the new one, and a reader never sees a half-written file. The + // rename lands on the symlink's target, so an AGENTS.md that is a link + // stays one, and the target's permissions are carried over. + bool ReplaceFile(const fs::path& file, std::string_view content) { + std::error_code ec; + const fs::path target = fs::canonical(file, ec); + if (ec) return false; + const fs::path temporary = target.parent_path() / std::format(".{}.{}.tmp", target.filename().string(), getpid()); + { + std::ofstream out(temporary, std::ios::binary | std::ios::trunc); + out.write(content.data(), static_cast(content.size())); + out.close(); + if (!out) { + fs::remove(temporary, ec); + return false; + } + } + if (const fs::file_status status = fs::status(target, ec); !ec) fs::permissions(temporary, status.permissions(), ec); + fs::rename(temporary, target, ec); + if (ec) { + std::error_code ignored; + fs::remove(temporary, ignored); + return false; + } + return true; + } + + // The note in `file`, beside the project file of a dependency loaded + // from `url` (empty for a local one); nullopt when there is none or it + // cannot be used. + std::optional ReadNote(const fs::path& file, const std::string& url, const fs::path& projectDir) { + std::error_code ec; + if (!fs::is_regular_file(file, ec)) return std::nullopt; + const std::optional content = ReadFile(file); + if (!content) { + Warn(std::format("{} could not be read, so its notes are left out", file.string())); + return std::nullopt; + } + std::string text = TrimBlank(*content); + if (text.empty()) return std::nullopt; + // On the file as it is, so the line numbers are the author's. + if (const std::string problem = NoteProblem(*content); !problem.empty()) { + Warn(std::format("{}: {}, so its notes are left out", file.string(), problem)); + return std::nullopt; + } + // A URL names the dependency the same on every machine, which a path + // into the cache or a sibling checkout does not, and the section is + // committed along with the rest of AGENTS.md. + std::string source = url.empty() ? file.lexically_relative(projectDir).generic_string() : url; + return DependencyNote{ .source = std::move(source), .text = std::move(text) }; + } + + // The AGENTS.consumer.md of every project `project` depends on, as + // project.dependencyNotes selects them. Breadth-first from the project's + // `#pragma crafter dependency` targets, its dependencies and its tests' + // dependencies, so the projects it names itself come first, in the + // order it names them, and a library several of them use is described + // once, where it is first reached. + // + // A loaded project, one that GitProject or LocalProject returned, + // contributes the file beside its project file. Every other + // Configuration, the project's own library or a dependency's internal + // one, is only walked through. Direct stops at the first loaded project + // on each path. A dependency's own pragma tooling is never reached, + // being no `dependencies` edge, and a dependency's tests are never + // walked: nothing builds them for the consumer, and they may point at a + // Configuration their project file has since moved out of. + std::vector CollectNotes(const Configuration& project, const fs::path& absProject) { + const fs::path projectDir = absProject.parent_path(); + std::deque queue; + // Seen from the start: a test may list the project itself. + std::unordered_set seen = { &project }; + auto enqueue = [&](const Configuration* cfg) { + if (cfg && seen.insert(cfg).second) queue.push_back(cfg); + }; + for (const Configuration* dep : PragmaDependencies(absProject)) enqueue(dep); + for (const Configuration* dep : project.dependencies) enqueue(dep); + for (const Test& test : project.tests) { + for (const Configuration* dep : test.config.dependencies) enqueue(dep); + } + + std::vector notes; + std::unordered_set files; + while (!queue.empty()) { + const Configuration* cfg = queue.front(); + queue.pop_front(); + if (const std::optional loaded = LoadedProjectOf(*cfg)) { + const fs::path file = loaded->projectFile.parent_path() / NotesFileName; + if (files.insert(file).second) { + if (std::optional note = ReadNote(file, loaded->url, projectDir)) notes.push_back(std::move(*note)); + } + if (project.dependencyNotes == DependencyNotes::Direct) continue; + } + for (const Configuration* dep : cfg->dependencies) enqueue(dep); + } + return notes; + } +} + +std::string Crafter::MergeDependencyNotes(std::string_view agentsMd, std::span notes) { + for (const DependencyNote& note : notes) { + if (const std::string problem = NoteProblem(note.text); !problem.empty()) throw std::runtime_error(std::format("the notes from {}: {}", note.source, problem)); + } + + const MarkerScan scan = ScanMarkers(agentsMd); + if (scan.unclosedBlock) throw std::runtime_error(std::format("line {}: a code block opens here and never closes", *scan.unclosedBlock + 1)); + if (scan.begins.size() > 1) throw std::runtime_error(std::format("line {}: a second dependency notes begin marker (the first is on line {})", scan.begins[1] + 1, scan.begins[0] + 1)); + if (scan.ends.size() > 1) throw std::runtime_error(std::format("line {}: a second dependency notes end marker (the first is on line {})", scan.ends[1] + 1, scan.ends[0] + 1)); + if (!scan.ends.empty() && (scan.begins.empty() || scan.ends[0] < scan.begins[0])) throw std::runtime_error(std::format("line {}: a dependency notes end marker with no begin marker before it", scan.ends[0] + 1)); + if (!scan.begins.empty() && scan.ends.empty()) throw std::runtime_error(std::format("line {}: a dependency notes begin marker with no end marker after it", scan.begins[0] + 1)); + + const std::string section = notes.empty() ? std::string() : RenderSection(notes); + if (scan.begins.empty()) { + if (notes.empty()) return std::string(agentsMd); + std::string merged(agentsMd); + if (!merged.empty()) merged += merged.ends_with('\n') ? "\n" : "\n\n"; + return merged + section; + } + std::string_view before = agentsMd.substr(0, scan.lines[scan.begins[0]].begin); + const std::string_view after = agentsMd.substr(scan.lines[scan.ends[0]].next); + // The blank line the section was appended after goes with it. + if (notes.empty() && before.ends_with("\n\n")) before.remove_suffix(1); + return std::format("{}{}{}", before, section, after); +} + +bool Crafter::UpdateAgentsMd(const Configuration& project, const fs::path& projectFile) { + std::error_code ec; + const fs::path absProject = fs::canonical(projectFile, ec); + if (ec) return false; + const fs::path agents = absProject.parent_path() / "AGENTS.md"; + if (!fs::is_regular_file(agents, ec)) return false; + + const std::optional current = ReadFile(agents); + if (!current) { + Warn(std::format("{} could not be read, so its dependency notes were not updated", agents.string())); + return false; + } + const std::vector notes = project.dependencyNotes == DependencyNotes::None ? std::vector() : CollectNotes(project, absProject); + std::string merged; + try { + merged = MergeDependencyNotes(*current, notes); + } catch (const std::exception& e) { + Warn(std::format("{}: {}, so its dependency notes were not updated", agents.string(), e.what())); + return false; + } + if (merged == *current) return false; + if (!ReplaceFile(agents, merged)) { + Warn(std::format("{} could not be written, so its dependency notes were not updated", agents.string())); + return false; + } + // A build that edits a file the user commits says so. + if (Progress::GetVerbosity() != Progress::Verbosity::Quiet) { + Progress::Clear(); + if (notes.empty()) { + std::println("Removed the dependency notes from {}", agents.string()); + } else { + std::println("Updated the dependency notes in {} ({} {})", agents.string(), notes.size(), notes.size() == 1 ? "dependency" : "dependencies"); + } + } + return true; +} diff --git a/implementations/Crafter.Build-Clang.cpp b/implementations/Crafter.Build-Clang.cpp index 416856e..2721c87 100644 --- a/implementations/Crafter.Build-Clang.cpp +++ b/implementations/Crafter.Build-Clang.cpp @@ -13,6 +13,7 @@ import :Toolchain; import :Test; import :Lint; import :Script; +import :Agents; import :Progress; import :Asset; namespace fs = std::filesystem; @@ -1699,6 +1700,13 @@ Clean (after the `clean` subcommand): so it works when the project itself no longer compiles. Every target, variant and dependency artifact under those trees goes with it. +AGENTS.md: + When an AGENTS.md sits beside the project file, a build and `test` keep a + section in it holding the AGENTS.consumer.md each dependency ships beside its + project.cpp, and write the file only when that section changes. An AGENTS.md + is never created. cfg.dependencyNotes picks the dependencies: Transitive (the + default), Direct, or None, which removes the section. + Project args: Any flag not consumed above is forwarded verbatim to CrafterBuildProject as part of its `args` span. Project-specific flags (e.g. --target=, custom @@ -1867,6 +1875,13 @@ std::int32_t Crafter::Run(std::int32_t argc, char** argv) { return lintOpts.mode == LintMode::Check && !lintSummary.changedFiles.empty() ? 1 : 0; } + // Builds and test runs only: lint and `format --check` promise to + // write nothing, `format` rewrites the project's sources and nothing + // else, and `run` is for chores. Here, before compiling, because + // loading has just fetched every dependency, so the notes are + // current, and an agent fixing a build that fails needs them. + UpdateAgentsMd(config, projectFile); + if (runTests) { TestSummary summary = RunTests(config, testOpts, projectArgs); return summary.AllPassed() ? 0 : 1; diff --git a/implementations/Crafter.Build-External.cpp b/implementations/Crafter.Build-External.cpp index a0528ca..390ff73 100644 --- a/implementations/Crafter.Build-External.cpp +++ b/implementations/Crafter.Build-External.cpp @@ -327,6 +327,10 @@ ExternalBuildResult Crafter::BuildExternal(const ExternalDependency& dep, std::s namespace { std::mutex ProjectCacheMutex; std::unordered_map> ProjectCache; + // Keyed by the Configurations ProjectCache owns, which live as long as + // the process does, so the pointers never dangle. Guarded by + // ProjectCacheMutex as well. + std::unordered_map LoadedProjects; // Run the dep's project.cpp from its own directory so its relative paths // (cfg.path = "./", interfaces/, lib/, ...) resolve against the dep @@ -415,6 +419,7 @@ Configuration* Crafter::GitProject(const GitProjectSpec& spec) { auto stored = std::make_unique(std::move(cfg)); Configuration* ptr = stored.get(); ProjectCache.emplace(std::move(fullHash), std::move(stored)); + LoadedProjects.emplace(ptr, LoadedProject{ .projectFile = projectPath.lexically_normal(), .url = spec.source.url }); return ptr; } @@ -477,5 +482,12 @@ Configuration* Crafter::LocalProject(const LocalProjectSpec& spec) { auto stored = std::make_unique(std::move(cfg)); Configuration* ptr = stored.get(); ProjectCache.emplace(std::move(fullHash), std::move(stored)); + LoadedProjects.emplace(ptr, LoadedProject{ .projectFile = canonical, .url = "" }); return ptr; } + +std::optional Crafter::LoadedProjectOf(const Configuration& cfg) { + std::lock_guard lock(ProjectCacheMutex); + if (auto it = LoadedProjects.find(&cfg); it != LoadedProjects.end()) return it->second; + return std::nullopt; +} diff --git a/implementations/Crafter.Build-Platform.cpp b/implementations/Crafter.Build-Platform.cpp index e4ead32..d54831a 100644 --- a/implementations/Crafter.Build-Platform.cpp +++ b/implementations/Crafter.Build-Platform.cpp @@ -422,6 +422,13 @@ namespace { // the stack is per thread. ProjectDependency() reads the top frame. thread_local std::vector LoadingFrames; + // A frame's dependencies, kept past the frame for PragmaDependencies(). + // Process-wide rather than per thread: whoever asks need not be the + // thread that loaded the file. The Configurations live in the project + // cache, which outlives every load. + std::mutex PragmaDependenciesMutex; + std::unordered_map> PragmaDependenciesByProject; + // Held for the whole of one LoadProject call. Pushing a file that is // already on the stack is a cycle, which used to recurse forever. class ProjectFileGuard { @@ -580,6 +587,15 @@ bool Crafter::IsTopLevelProject() { return LoadingFrames.size() <= 1; } +std::vector Crafter::PragmaDependencies(const fs::path& projectFile) { + std::error_code ec; + const fs::path absProject = fs::canonical(projectFile, ec); + if (ec) return {}; + std::lock_guard lock(PragmaDependenciesMutex); + if (auto it = PragmaDependenciesByProject.find(absProject); it != PragmaDependenciesByProject.end()) return it->second; + return {}; +} + namespace { // The one reading of a wait status (pclose, system) for every helper // that runs a command: a signal death is reported the way a shell @@ -823,13 +839,14 @@ std::string Crafter::GetBaseCommand(const Configuration& config) { } namespace { - constexpr std::array CrafterBuildModules = { + constexpr std::array CrafterBuildModules = { "Crafter.Build-Toolchain", "Crafter.Build-Shader", "Crafter.Build-Platform", "Crafter.Build-Interface", "Crafter.Build-Implementation", "Crafter.Build-External", + "Crafter.Build-Agents", "Crafter.Build-Clang", "Crafter.Build-Test", "Crafter.Build-Lint", @@ -966,6 +983,10 @@ Configuration Crafter::LoadProject(const fs::path& projectFile, std::span notes); + + // Refresh the dependency notes section of the AGENTS.md beside + // `projectFile` from the AGENTS.consumer.md of every dependency + // project.dependencyNotes selects. `project` is the Configuration that + // file returned, after LoadProject: loading is what found the + // dependencies. Does nothing when there is no AGENTS.md, which is never + // created. Writes only when the text changes, and returns whether it + // did. An unreadable file, a failed write, malformed markers and a + // dependency's unusable notes are warnings: the notes never fail a build. + bool UpdateAgentsMd(const Configuration& project, const fs::path& projectFile); +} diff --git a/interfaces/Crafter.Build-Clang.cppm b/interfaces/Crafter.Build-Clang.cppm index 36cf1a9..df2705b 100644 --- a/interfaces/Crafter.Build-Clang.cppm +++ b/interfaces/Crafter.Build-Clang.cppm @@ -8,6 +8,7 @@ import :Interface; import :Implementation; import :External; import :Toolchain; +import :Agents; namespace fs = std::filesystem; export namespace Crafter { @@ -494,6 +495,14 @@ export namespace Crafter { // }}); // Not part of VariantId: a script changes nothing that is compiled. std::vector