Crafter.Build/implementations/Crafter.Build-Agents.cpp

// SPDX-License-Identifier: LGPL-3.0-only
// SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts®

module;
#include <errno.h>
#include <fcntl.h>
#include <sys/file.h>
#include <unistd.h>
module Crafter.Build:Agents_impl;
import std;
import :Agents;
import :Clang;
import :External;
import :Platform;
import :Progress;
namespace fs = std::filesystem;
using namespace Crafter;

namespace {
    constexpr std::string_view BeginMarker = "<!-- crafter-build: dependency notes begin -->";
    constexpr std::string_view EndMarker = "<!-- crafter-build: dependency notes end -->";
    // Right under the begin marker, for whoever opens the file: an edit made
    // inside the section is gone after the next build.
    constexpr std::string_view Preamble = "<!-- Written by crafter-build from the AGENTS.consumer.md each dependency ships, and rewritten on every build. Edit those files, not this section; cfg.dependencyNotes in project.cpp picks which dependencies are included. -->";
    constexpr std::string_view NotesFileName = "AGENTS.consumer.md";

    void Warn(std::string_view message) {
        Progress::Diagnostic(std::format("warning: {}", message));
    }

    std::string ErrnoMessage() {
        return std::error_code(errno, std::generic_category()).message();
    }

    // Closes on the way out of scope, which also drops an flock taken
    // through it.
    class Descriptor {
    public:
        explicit Descriptor(std::int32_t fd) : fd_(fd) {}
        ~Descriptor() {
            if (fd_ >= 0) ::close(fd_);
        }
        Descriptor(const Descriptor&) = delete;
        Descriptor& operator=(const Descriptor&) = delete;
        std::int32_t Get() const { return fd_; }
        bool Valid() const { return fd_ >= 0; }

    private:
        std::int32_t fd_;
    };

    // Everything `fd` reads, into `out`; empty on success, otherwise the
    // error. Through read(2) rather than a stream: libc++'s filebuf reports a
    // failed read as the end of the file, so an I/O error came back as a
    // short read that looked successful, and the section was then written
    // over a truncated AGENTS.md.
    std::string ReadAll(std::int32_t fd, std::string& out) {
        out.clear();
        std::array<char, 16384> buffer;
        for (;;) {
            const auto got = ::read(fd, buffer.data(), buffer.size());
            if (got > 0) {
                out.append(buffer.data(), static_cast<std::size_t>(got));
            } else if (got == 0) {
                return "";
            } else if (errno != EINTR) {
                return ErrnoMessage();
            }
        }
    }

    std::string ReadFile(const fs::path& file, std::string& out) {
        const Descriptor in(::open(file.c_str(), O_RDONLY | O_CLOEXEC));
        if (!in.Valid()) return ErrnoMessage();
        return ReadAll(in.Get(), out);
    }

    struct Line {
        std::string_view text;  // without the newline and a '\r' before it
        std::size_t begin = 0;  // offset of the line's first byte
        std::size_t next = 0;   // offset of the line after it
    };

    std::vector<Line> SplitLines(std::string_view text) {
        std::vector<Line> lines;
        std::size_t pos = 0;
        while (pos < text.size()) {
            const std::size_t newline = text.find('\n', pos);
            const std::size_t end = newline == std::string_view::npos ? text.size() : newline;
            std::string_view line = text.substr(pos, end - pos);
            if (line.ends_with('\r')) line.remove_suffix(1);
            const std::size_t next = newline == std::string_view::npos ? text.size() : newline + 1;
            lines.push_back({ .text = line, .begin = pos, .next = next });
            pos = next;
        }
        return lines;
    }

    std::string_view TrimEnd(std::string_view text) {
        const std::size_t last = text.find_last_not_of(" \t");
        return last == std::string_view::npos ? std::string_view() : text.substr(0, last + 1);
    }

    // A line with the block quote markers in front of it taken off.
    struct Shape {
        std::size_t quotes = 0;  // `>` markers
        std::size_t indent = 0;  // columns of indentation after them
        std::string_view rest;   // the line from its first other character
    };

    Shape ShapeOf(std::string_view line) {
        Shape shape;
        std::size_t pos = 0;
        for (;;) {
            std::size_t probe = pos;
            while (probe < line.size() && probe - pos < 3 && line[probe] == ' ') ++probe;
            if (probe >= line.size() || line[probe] != '>') break;
            pos = probe + 1;
            if (pos < line.size() && line[pos] == ' ') ++pos;
            ++shape.quotes;
        }
        for (; pos < line.size() && (line[pos] == ' ' || line[pos] == '\t'); ++pos) {
            shape.indent += line[pos] == '\t' ? 4 - shape.indent % 4 : 1;
        }
        shape.rest = line.substr(pos);
        return shape;
    }

    // The columns a list item marker at the start of `text` takes, with the
    // spaces after it: `-`, `+` or `*`, or up to nine digits and `.` or `)`,
    // then at least one space. Zero when `text` starts with none.
    std::size_t ListMarker(std::string_view text) {
        std::size_t pos = 0;
        if (!text.empty() && (text[0] == '-' || text[0] == '+' || text[0] == '*')) {
            pos = 1;
        } else {
            while (pos < text.size() && pos < 9 && text[pos] >= '0' && text[pos] <= '9') ++pos;
            if (pos == 0 || pos >= text.size() || (text[pos] != '.' && text[pos] != ')')) return 0;
            ++pos;
        }
        const std::size_t marker = pos;
        while (pos < text.size() && text[pos] == ' ') ++pos;
        return pos == marker ? 0 : pos;
    }

    struct Fence {
        char ch = '\0';
        std::size_t length = 0;
        std::string_view after;  // the rest of the line behind the run
    };

    // The run of three or more backticks or tildes `text` starts with.
    std::optional<Fence> FenceRun(std::string_view text) {
        if (text.empty() || (text[0] != '`' && text[0] != '~')) return std::nullopt;
        std::size_t length = 0;
        while (length < text.size() && text[length] == text[0]) ++length;
        if (length < 3) return std::nullopt;
        return Fence{ .ch = text[0], .length = length, .after = text.substr(length) };
    }

    // Fenced code blocks, close to how CommonMark reads them. A run of three
    // or more backticks or tildes opens one after at most three spaces of
    // indentation; a backtick run only when no backtick follows it on the
    // line, since ```make``` is inline code. A run of the same character at
    // least as long, with nothing behind it but spaces, closes it. Keeping
    // the character and the length is what lets a ~~~ block quote a ``` one.
    //
    // Block quotes and list items are followed as far as fences need: the
    // `>` markers come off every line, list markers off the line that opens
    // a block, and a block ends with the container it sits in, at a line with
    // fewer `>` or one indented less than its list item's content. Read
    // without them, `- ```bash` was missed while its indented closer was
    // taken for an opener, and the rest of the file came out inside out. Not
    // followed: indented code blocks, and the continuation lines of a list
    // item before a fence that opens there.
    class CodeFences {
    public:
        // Feed the lines in order. True for a fence and for every line
        // inside a block.
        bool InCode(std::string_view line) {
            const Shape shape = ShapeOf(line);
            if (open_ != '\0') {
                const bool blank = TrimEnd(shape.rest).empty();
                const bool left = !blank && (shape.quotes < quotes_ || shape.indent < content_);
                if (!left) {
                    const std::optional<Fence> fence = FenceRun(shape.rest);
                    if (fence && fence->ch == open_ && fence->length >= length_ && TrimEnd(fence->after).empty() && shape.indent <= content_ + 3) open_ = '\0';
                    return true;
                }
                // The container ended, and the block with it; the line is
                // read again as one outside any block.
                open_ = '\0';
            }
            if (shape.indent > 3) return false;
            std::size_t column = shape.indent;
            std::string_view text = shape.rest;
            std::size_t content = 0;
            for (std::size_t marker = ListMarker(text); marker > 0; marker = ListMarker(text)) {
                column += marker;
                text.remove_prefix(marker);
                content = column;
            }
            const std::optional<Fence> fence = FenceRun(text);
            if (!fence || (fence->ch == '`' && fence->after.contains('`'))) return false;
            open_ = fence->ch;
            length_ = fence->length;
            quotes_ = shape.quotes;
            content_ = content;
            ++opened_;
            return true;
        }
        bool Open() const { return open_ != '\0'; }
        // How many blocks have opened so far.
        std::size_t Opened() const { return opened_; }

    private:
        char open_ = '\0';
        std::size_t length_ = 0;
        std::size_t quotes_ = 0;   // the block quote depth it opened at
        std::size_t content_ = 0;  // its list item's content column; 0 outside a list
        std::size_t opened_ = 0;
    };

    // Where a text's marker lines are, outside code blocks, as indices into
    // `lines`.
    struct MarkerScan {
        std::vector<Line> lines;
        std::vector<std::size_t> begins;
        std::vector<std::size_t> ends;
        // The line a code block that never closes opens on.
        std::optional<std::size_t> unclosedBlock;
    };

    MarkerScan ScanMarkers(std::string_view text) {
        MarkerScan scan;
        scan.lines = SplitLines(text);
        CodeFences fences;
        for (std::size_t i = 0; i < scan.lines.size(); ++i) {
            const std::size_t opened = fences.Opened();
            const bool code = fences.InCode(scan.lines[i].text);
            if (fences.Opened() != opened) scan.unclosedBlock = i;
            if (code) continue;
            const std::string_view line = TrimEnd(scan.lines[i].text);
            if (line == BeginMarker) scan.begins.push_back(i);
            if (line == EndMarker) scan.ends.push_back(i);
        }
        if (!fences.Open()) scan.unclosedBlock.reset();
        return scan;
    }

    // Why `text` cannot go into the section as it is; empty when it can. A
    // marker line of its own would end or restart the section once the file
    // is read back, and a code block left open would hide the end marker:
    // either way the next build could no longer find what this one wrote.
    // Lines are numbered from `firstLine`.
    std::string NoteProblem(std::string_view text, std::size_t firstLine) {
        const MarkerScan scan = ScanMarkers(text);
        std::optional<std::size_t> marker;
        if (!scan.begins.empty()) marker = scan.begins[0];
        if (!scan.ends.empty()) marker = std::min(marker.value_or(scan.ends[0]), scan.ends[0]);
        if (marker) return std::format("line {} is a dependency notes marker", *marker + firstLine);
        if (scan.unclosedBlock) return std::format("line {} opens a code block that never closes", *scan.unclosedBlock + firstLine);
        return "";
    }

    struct NormalizedNote {
        std::string text;
        // The line of the file `text` starts on.
        std::size_t firstLine = 1;
    };

    // A notes file as it goes into the section: LF line ends, no blank lines
    // in front and no whitespace behind, so the section spaces its notes the
    // same whatever the files end with.
    NormalizedNote NormalizeNote(std::string_view raw) {
        std::string text;
        text.reserve(raw.size());
        for (std::size_t i = 0; i < raw.size(); ++i) {
            if (raw[i] == '\r' && i + 1 < raw.size() && raw[i + 1] == '\n') continue;
            text.push_back(raw[i]);
        }
        const std::size_t last = text.find_last_not_of(" \t\r\n");
        if (last == std::string::npos) return {};
        const std::size_t first = text.find_first_not_of(" \t\r\n");
        const std::size_t newline = text.rfind('\n', first);
        const std::size_t begin = newline == std::string::npos ? 0 : newline + 1;
        const auto skipped = std::count(text.begin(), text.begin() + static_cast<std::ptrdiff_t>(begin), '\n');
        return { .text = text.substr(begin, last + 1 - begin), .firstLine = static_cast<std::size_t>(skipped) + 1 };
    }

    // `text` with `eol` ending its lines, whether they ended in LF or CRLF.
    std::string WithLineEnds(std::string_view text, std::string_view eol) {
        std::string out;
        out.reserve(text.size());
        for (std::size_t i = 0; i < text.size(); ++i) {
            if (text[i] == '\r' && i + 1 < text.size() && text[i + 1] == '\n') continue;
            if (text[i] == '\n') {
                out += eol;
            } else {
                out.push_back(text[i]);
            }
        }
        return out;
    }

    std::string RenderSection(std::span<const DependencyNote> notes, std::string_view eol) {
        std::string section = std::format("{}{}{}{}", BeginMarker, eol, Preamble, eol);
        for (const DependencyNote& note : notes) section += std::format("{0}<!-- from {1} -->{0}{2}{0}", eol, note.name, WithLineEnds(note.text, eol));
        section += std::format("{0}{1}{0}", eol, EndMarker);
        return section;
    }

    // The note beside a dependency's project file, or nullopt when there is
    // none or it cannot be used.
    std::optional<DependencyNote> ReadNote(const fs::path& file, const std::string& name) {
        std::error_code ec;
        if (!fs::is_regular_file(file, ec)) return std::nullopt;
        std::string content;
        if (const std::string error = ReadFile(file, content); !error.empty()) {
            Warn(std::format("{} could not be read ({}), so its notes are left out", file.string(), error));
            return std::nullopt;
        }
        NormalizedNote note = NormalizeNote(content);
        if (note.text.empty()) return std::nullopt;
        // On the very bytes the merge checks, so the two can never disagree,
        // and numbered as the author's file is.
        if (const std::string problem = NoteProblem(note.text, note.firstLine); !problem.empty()) {
            Warn(std::format("{}: {}, so its notes are left out", file.string(), problem));
            return std::nullopt;
        }
        return DependencyNote{ .name = name, .text = std::move(note.text) };
    }

    // The loaded projects `configs` lead to without passing through another
    // loaded project, in the order the configurations name them: one ring
    // further out. `walked` spans the whole collection, so each
    // Configuration is looked at once however many paths reach it.
    std::vector<const Configuration*> NextRing(std::span<const Configuration* const> configs, std::unordered_set<const Configuration*>& walked) {
        std::vector<const Configuration*> ring;
        std::vector<const Configuration*> pending(configs.rbegin(), configs.rend());
        while (!pending.empty()) {
            const Configuration* cfg = pending.back();
            pending.pop_back();
            if (!cfg || !walked.insert(cfg).second) continue;
            if (LoadedProjectOf(*cfg)) {
                ring.push_back(cfg);
                continue;
            }
            for (auto dep = cfg->dependencies.rbegin(); dep != cfg->dependencies.rend(); ++dep) pending.push_back(*dep);
        }
        return ring;
    }

    // The AGENTS.consumer.md of every project `project` depends on, as
    // project.dependencyNotes selects them, ring by ring: first the loaded
    // projects (the ones GitProject and LocalProject returned) that its
    // `#pragma crafter dependency` lines and its configurations name, in
    // that order, then the ones those depend on. Its own configurations, and
    // a dependency's internal ones, are only walked through. Direct stops
    // after the first ring.
    //
    // Never walked: the project's tests, since a plain build never touches
    // them and a test's dependency can point at the Configuration the project
    // file returned by value, which has since moved; and a dependency's own
    // pragma tooling, which is about how the dependency is built, not how it
    // is used, and is no `dependencies` edge to begin with.
    std::vector<DependencyNote> CollectNotes(const Configuration& project, const fs::path& projectFile, const fs::path& projectDir) {
        std::unordered_set<const Configuration*> walked = { &project };
        // The project's own notes are for its consumers, so they never go
        // into its own AGENTS.md, whatever dependency they sit beside. Both
        // places count for a symlinked project file.
        std::unordered_set<fs::path> files = { projectDir / NotesFileName };
        std::error_code ec;
        if (const fs::path canonical = fs::canonical(projectFile, ec); !ec) files.insert(canonical.parent_path() / NotesFileName);
        std::unordered_set<std::string> identities;
        std::vector<DependencyNote> notes;

        std::vector<const Configuration*> start;
        for (const Configuration* dep : PragmaDependencies(projectFile)) start.push_back(dep);
        start.insert(start.end(), project.dependencies.begin(), project.dependencies.end());
        for (std::vector<const Configuration*> ring = NextRing(start, walked); !ring.empty();) {
            std::vector<const Configuration*> next;
            for (const Configuration* cfg : ring) {
                const std::optional<LoadedProject> loaded = LoadedProjectOf(*cfg);
                const fs::path file = loaded->projectFile.parent_path() / NotesFileName;
                // One library under two specs (a pin here, another revision
                // further down) is described once, as the ring nearest the
                // project has it.
                if (identities.insert(loaded->identity).second && files.insert(file).second) {
                    if (std::optional<DependencyNote> note = ReadNote(file, loaded->name)) notes.push_back(std::move(*note));
                }
                next.insert(next.end(), cfg->dependencies.begin(), cfg->dependencies.end());
            }
            if (project.dependencyNotes == DependencyNotes::Direct) break;
            ring = NextRing(next, walked);
        }
        return notes;
    }

    // Bring `fd`, which holds `current`, to `wanted`: from the first byte the
    // two differ at, so the text in front of the section is never rewritten,
    // then cut to length and synced. Empty on success, otherwise the error.
    //
    // In place rather than through a temporary renamed over the file: a
    // rename swaps in another file, which drops whatever belongs to the old
    // one (a hard link such as a CLAUDE.md beside it, ACLs, ownership, being
    // read-only) and needs fsyncs to be crash safe at all. The price is that
    // a reader can catch the section half written, and that a write failing
    // partway, on a full disk say, leaves it so. With the section last,
    // where it is appended, only the section can be damaged, and the next
    // build reports its markers instead of guessing.
    std::string WriteChanged(std::int32_t fd, std::string_view current, std::string_view wanted) {
        const auto diverge = std::ranges::mismatch(current, wanted);
        for (auto done = static_cast<std::size_t>(diverge.in2 - wanted.begin()); done < wanted.size();) {
            const auto wrote = ::pwrite(fd, wanted.data() + done, wanted.size() - done, static_cast<off_t>(done));
            if (wrote > 0) {
                done += static_cast<std::size_t>(wrote);
            } else if (wrote == 0) {
                return "the write made no progress";
            } else if (errno != EINTR) {
                return ErrnoMessage();
            }
        }
        if (::ftruncate(fd, static_cast<off_t>(wanted.size())) != 0) return ErrnoMessage();
        if (::fsync(fd) != 0) return ErrnoMessage();
        return "";
    }
}

std::string Crafter::MergeDependencyNotes(std::string_view agentsMd, std::span<const DependencyNote> notes) {
    for (const DependencyNote& note : notes) {
        if (const std::string problem = NoteProblem(note.text, 1); !problem.empty()) throw std::runtime_error(std::format("the notes from {}: {}", note.name, problem));
    }

    const MarkerScan scan = ScanMarkers(agentsMd);
    const bool section = scan.begins.size() == 1 && scan.ends.size() == 1 && scan.begins[0] < scan.ends[0];
    // The file's own line ending, so a CRLF file does not end up mixed.
    const std::string_view eol = agentsMd.contains("\r\n") ? "\r\n" : "\n";

    if (notes.empty()) {
        // With nothing to write, leaving the file alone loses nothing, so
        // anything short of a well-formed section is no concern of this one.
        if (!section) return std::string(agentsMd);
        std::string_view before = agentsMd.substr(0, scan.lines[scan.begins[0]].begin);
        const std::string_view after = agentsMd.substr(scan.lines[scan.ends[0]].next);
        // The blank line a section at the end was appended after goes with
        // it. One the user moved keeps the text around it apart.
        if (after.empty() && before.ends_with(std::format("{0}{0}", eol))) before.remove_suffix(eol.size());
        return std::format("{}{}", before, after);
    }

    if (scan.begins.size() > 1) throw std::runtime_error(std::format("line {}: a second dependency notes begin marker (the first is on line {})", scan.begins[1] + 1, scan.begins[0] + 1));
    if (scan.ends.size() > 1) throw std::runtime_error(std::format("line {}: a second dependency notes end marker (the first is on line {})", scan.ends[1] + 1, scan.ends[0] + 1));
    if (!scan.ends.empty() && (scan.begins.empty() || scan.ends[0] < scan.begins[0])) throw std::runtime_error(std::format("line {}: a dependency notes end marker with no begin marker before it", scan.ends[0] + 1));
    if (!scan.begins.empty() && scan.ends.empty()) throw std::runtime_error(std::format("line {}: a dependency notes begin marker with no end marker after it", scan.begins[0] + 1));

    const std::string rendered = RenderSection(notes, eol);
    if (section) return std::format("{}{}{}", agentsMd.substr(0, scan.lines[scan.begins[0]].begin), rendered, agentsMd.substr(scan.lines[scan.ends[0]].next));
    // Appended, the section would sit inside that block and never be found
    // again, so every build would add another.
    if (scan.unclosedBlock) throw std::runtime_error(std::format("line {}: a code block opens here and never closes", *scan.unclosedBlock + 1));
    if (agentsMd.empty()) return rendered;
    // On a line of its own, after a blank one.
    return std::format("{}{}{}{}", agentsMd, agentsMd.ends_with('\n') ? std::string_view() : eol, eol, rendered);
}

bool Crafter::UpdateAgentsMd(const Configuration& project, const fs::path& projectFile) {
    std::error_code ec;
    // Beside the project file as it was named, so a symlinked project.cpp
    // keeps the AGENTS.md beside the link.
    const fs::path projectDir = fs::absolute(projectFile, ec).lexically_normal().parent_path();
    if (ec) return false;
    const fs::path agents = projectDir / "AGENTS.md";
    if (!fs::is_regular_file(agents, ec)) return false;

    const std::vector<DependencyNote> notes = project.dependencyNotes == DependencyNotes::None ? std::vector<DependencyNote>() : CollectNotes(project, projectFile, projectDir);

    // Locked from the read through the write, so two builds never interleave
    // their writes, and on the file itself, which also covers two projects
    // that link one AGENTS.md. A read-only descriptor takes the lock as
    // well, so a read-only AGENTS.md is still read without complaint.
    const Descriptor reading(::open(agents.c_str(), O_RDONLY | O_CLOEXEC));
    if (!reading.Valid()) {
        Warn(std::format("{} could not be read ({}), so its dependency notes were not updated", agents.string(), ErrnoMessage()));
        return false;
    }
    while (::flock(reading.Get(), LOCK_EX) != 0 && errno == EINTR) {}
    std::string current;
    if (const std::string error = ReadAll(reading.Get(), current); !error.empty()) {
        Warn(std::format("{} could not be read ({}), so its dependency notes were not updated", agents.string(), error));
        return false;
    }

    std::string merged;
    try {
        merged = MergeDependencyNotes(current, notes);
    } catch (const std::exception& e) {
        Warn(std::format("{}: {}, so its dependency notes were not updated", agents.string(), e.what()));
        return false;
    }
    if (merged == current) return false;

    // Only now, with something to write: a read-only AGENTS.md is refused
    // here, and not on every build that leaves it as it is.
    const Descriptor writing(::open(agents.c_str(), O_WRONLY | O_CLOEXEC));
    if (!writing.Valid()) {
        Warn(std::format("{} could not be opened for writing ({}), so its dependency notes were not updated", agents.string(), ErrnoMessage()));
        return false;
    }
    if (const std::string error = WriteChanged(writing.Get(), current, merged); !error.empty()) {
        // Put back what was there. A full disk that refused the new bytes
        // usually still takes the old ones, whose space was just freed.
        if (WriteChanged(writing.Get(), merged, current).empty()) {
            Warn(std::format("{} could not be written ({}), so its dependency notes were not updated", agents.string(), error));
        } else {
            Warn(std::format("{} could not be written ({}) and may now end in part of the section; the next build reports it if so", agents.string(), error));
        }
        return false;
    }
    // A build that edits a file the user commits says so.
    if (Progress::GetVerbosity() != Progress::Verbosity::Quiet) {
        Progress::Clear();
        if (notes.empty()) {
            std::println("Removed the dependency notes from {}", agents.string());
        } else {
            std::println("Updated the dependency notes in {} ({} {})", agents.string(), notes.size(), notes.size() == 1 ? "dependency" : "dependencies");
        }
    }
    return true;
}