diff --git a/.forgejo/workflows/ci.yaml b/.forgejo/workflows/ci.yaml index 0f39550..dcdff57 100644 --- a/.forgejo/workflows/ci.yaml +++ b/.forgejo/workflows/ci.yaml @@ -1,37 +1,52 @@ name: Flatpak -# One job, three outcomes: -# pull request -> build the Flatpak and run the suite inside it -# push master -> also publish the result as the `beta` branch of the -# Catcrafts repo and refresh the rolling `latest` release -# tag X.Y.Z -> publish as `stable` and create release X.Y.Z +# One job, two outcomes: +# pull request, manual run -> build the Flatpak and run the suite inside it +# pushed tag X.Y.Z -> also release the build: publish it as the +# `stable` branch of the Catcrafts repo and +# create release X.Y.Z +# +# A release is a tag that the owner makes and signs locally and pushes with +# the master it releases; a push to master without one runs nothing. The tag +# names the version, and the metainfo's newest entry has to be that +# version: its description is the release notes. # # The `crafter-flatpak` runner label is the privileged runner on the Hetzner # box (bubblewrap needs it) with four volumes allowed: the Sdk install, the # flatpak-builder state (ccache and the LLVM module, so only crafter-build # itself rebuilds on a normal run), the published repo's web root, and the # repo signing key, which lives on the runner host and never in the forge. -# No other runner carries that label. Plain `flatpak` also matches the -# organisation's runner, which mounts none of those volumes; a job it picks -# up fails before building anything. +# No other runner carries that label. Plain `flatpak` is the label of the +# organisation's runner, which serves every Catcrafts repository and also +# mounts the web root and the key, for Crafter.Review's publish job, so any +# job it runs can mount them (AGENTS.template.md, CI and publishing). on: pull_request: branches: [master] push: - branches: [master] # Bare semver tags. Spelled with globs so it means the same whether the - # forge reads it as a workflow pattern or as a plain glob; `latest`, the - # rolling tag this workflow moves itself, does not start with a digit. + # forge reads it as a workflow pattern or as a plain glob. tags: ['[0-9]*.[0-9]*.[0-9]*'] workflow_dispatch: +# Pushes are tags only, so every release run is in one group, and a release +# starts only once the one before it has finished. A pull request's group +# also holds its number, which only a pull request's event carries, so a +# new push to it cancels its own outdated run and no other. +concurrency: + group: flatpak-${{ github.event_name }}-${{ github.event.pull_request.number }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + env: APP_ID: net.catcrafts.CrafterBuild + BRANCH: stable MANIFEST: packaging/flatpak/net.catcrafts.CrafterBuild.yaml CI_MANIFEST: packaging/flatpak/ci-manifest.yaml RUNTIME: org.freedesktop.Sdk//26.08 REPO_DIR: /srv/flatpak/repo WEB_DIR: /srv/flatpak + # A release tag carries a signature Forgejo verifies for this address. + RELEASE_SIGNER: jorijnvdgraaf@catcrafts.net jobs: flatpak: @@ -45,11 +60,12 @@ jobs: -v /etc/crafter-flatpak:/run/crafter-flatpak:ro steps: - name: Install tools + # util-linux: flock, for the Publish step. run: | pacman-key --init pacman-key --populate archlinux pacman -Sy --noconfirm --needed archlinux-keyring - pacman -Syu --noconfirm --needed flatpak flatpak-builder ccache git nodejs jq gnupg go-yq ostree python + pacman -Syu --noconfirm --needed flatpak flatpak-builder ccache git nodejs jq gnupg go-yq ostree python util-linux git config --global --add safe.directory '*' - name: Sdk @@ -60,40 +76,137 @@ jobs: flatpak update -y --noninteractive - name: Checkout + # Every branch and tag, for git describe and the release checks; no + # step pushes, so the token is not kept in the checkout. uses: actions/checkout@v4 with: fetch-depth: 0 - persist-credentials: true + persist-credentials: false - - name: Version and channel + - name: Version id: meta run: | set -eu - if [ "${GITHUB_REF_TYPE}" = tag ]; then - version="$GITHUB_REF_NAME" - branch=stable - # The metainfo's newest release is the release notes and must be - # the version being tagged. - top=$(grep -m1 -oP '&2 + # The metainfo's newest entry is its first element, the + # one parse that gives both its version and the release notes, + # before anything is built: a comment or another attribute order + # would mislead a text search, and an entry whose notes render + # empty stops the run here. The notes go to release-notes.md, for + # the release page, as Markdown. AppStream has no headings, so a + # paragraph that introduces a list is one:

New

    ...
+ # comes out as "## New" over its items, the shape the notes take + # since 3.0.0. A paragraph on its own stays a paragraph, as the + # older entries are written. and get their Markdown + # spelling; a code span also keeps text such as lib//eh + # from being read as an HTML tag by the forge. + newest=$(python3 - <<'EOF' + import sys + import xml.etree.ElementTree as ET + + release = ET.parse("packaging/flatpak/net.catcrafts.CrafterBuild.metainfo.xml").getroot().find("releases/release") + desc = None if release is None else release.find("description") + + def text(el): + parts = [el.text or ""] + for child in el: + inner = text(child) + parts.append({"em": f"*{inner}*", "code": f"`{inner}`"}.get(child.tag, inner)) + parts.append(child.tail or "") + return " ".join("".join(parts).split()) + + blocks = [] if desc is None else list(desc) + out = [] + for i, el in enumerate(blocks): + heads_list = i + 1 < len(blocks) and blocks[i + 1].tag in ("ul", "ol") + if el.tag == "p": + out.append(f"## {text(el)}" if heads_list else text(el)) + elif el.tag == "ul": + out.append("\n".join(f"- {text(li)}" for li in el)) + elif el.tag == "ol": + out.append("\n".join(f"{n}. {text(li)}" for n, li in enumerate(el, 1))) + notes = "\n\n".join(out) + if not notes.strip(): + sys.exit("the metainfo needs a at the top of whose renders release notes") + with open("release-notes.md", "w") as f: + print(notes, file=f) + print(release.get("version", "")) + EOF + ) + if [ "$GITHUB_EVENT_NAME" = push ]; then + # A release: the only pushes this workflow runs for are tags. + # The tag is the version; it has to be annotated, signed by the + # owner as Forgejo verifies it against the keys its users added, + # on master, newer than every published release, and the version + # of the metainfo's newest entry. The tag itself is read from the + # forge: the checkout fetches a pushed tag again as a plain ref + # to its commit, so the local one is never the annotated tag. + version=$GITHUB_REF_NAME + if ! printf '%s\n' "$version" | grep -qxE '[0-9]+\.[0-9]+\.[0-9]+'; then + echo "the tag $version is not a version of the form X.Y.Z" >&2 + exit 1 + fi + api="$GITHUB_SERVER_URL/api/v1/repos/$GITHUB_REPOSITORY" + tag=$(curl -sSf "$api/tags/$version") + object=$(printf '%s' "$tag" | jq -r .id) + commit=$(printf '%s' "$tag" | jq -r .commit.sha) + if [ "$commit" != "$GITHUB_SHA" ]; then + echo "the forge has the tag $version on $commit, not on this run's commit $GITHUB_SHA" >&2 + exit 1 + fi + if [ "$object" = "$commit" ]; then + echo "$version is a lightweight tag; a release tag is annotated and signed: git tag -s $version -m 'crafter-build $version'" >&2 + exit 1 + fi + # Forgejo verifies a signed tag's own signature, and for an + # unsigned one falls back to its commit's, so the verdict also + # has to cover the tag's payload, not the commit's. + signed=$(curl -sSf "$api/git/tags/$object" | + jq -r --arg c "$commit" --arg v "$version" '"\(.verification.verified) \(.verification.signer.email // "") \(.verification.payload // "" | startswith("object \($c)\ntype commit\ntag \($v)\n"))"') + if [ "$signed" != "true $RELEASE_SIGNER true" ]; then + echo "Forgejo does not verify the tag $version itself as signed by $RELEASE_SIGNER (verified, signer, over the tag's payload: $signed)" >&2 + exit 1 + fi + if ! git merge-base --is-ancestor "$GITHUB_SHA" origin/master; then + echo "the tag $version is on $(git rev-parse --short "$GITHUB_SHA"), which is not on master: push the tag with the master it releases" >&2 + exit 1 + fi + # Newer than every release the forge has published, its own + # included, and not every tag: a later tag whose run still + # queues behind this one has no release yet, and a tag that is + # no X.Y.Z version never gets one. + for t in $(git tag --list | awk '/^[0-9]+\.[0-9]+\.[0-9]+$/' | sort -V); do + [ "$(printf '%s\n%s\n' "$t" "$version" | sort -V | tail -n 1)" = "$t" ] || continue + code=$(curl -s -o /dev/null -w '%{http_code}' "$api/releases/tags/$t") + case "$code" in + 404) ;; + 200) + if [ "$t" = "$version" ]; then + echo "$version is released already; to re-run after a release left half made, delete that release first" >&2 + else + echo "$version is not newer than the release $t" >&2 + fi + exit 1 ;; + *) echo "the forge answered $code when asked whether $t is released" >&2; exit 1 ;; + esac + done + if [ "$newest" != "$version" ]; then + echo "the metainfo's newest release is \"$newest\", but the tag is $version: the tagged commit needs the entry for $version at the top of packaging/flatpak/net.catcrafts.CrafterBuild.metainfo.xml" >&2 exit 1 fi else - # A master build names itself after the release it follows, the - # way a local checkout does: 2.0.0-3-g21746b1 is three commits - # past 2.0.0. Before the first tag there is nothing to describe - # from and the bare hash gets a placeholder in front. - described=$(git describe --tags --always --exclude=latest) + # Nothing is published. The build names itself after the + # release it follows, the way a local checkout does: + # 5.0.0-3-g21746b1 is three commits past 5.0.0. Before the first + # tag there is nothing to describe from and the bare hash gets a + # placeholder in front. + described=$(git describe --tags --always) case "$described" in [0-9]*.[0-9]*.[0-9]*) version="$described" ;; *) version="0.0.0-dev+$described" ;; esac - branch=beta fi echo "version=$version" >> "$GITHUB_OUTPUT" - echo "branch=$branch" >> "$GITHUB_OUTPUT" - echo "version $version, flatpak branch $branch" + echo "version $version" - name: Build # The manifest is rewritten beside the original so its relative @@ -106,7 +219,7 @@ jobs: grep -n 'CRAFTER_BUILD_VERSION' "$CI_MANIFEST" flatpak-builder --force-clean --ccache --disable-rofiles-fuse \ --state-dir=/root/.cache/flatpak-builder \ - --repo=repo --default-branch="${{ steps.meta.outputs.branch }}" \ + --repo=repo --default-branch="$BRANCH" \ build-dir "$CI_MANIFEST" - name: Smoke test @@ -114,18 +227,16 @@ jobs: flatpak-builder --run build-dir "$CI_MANIFEST" crafter-build --version flatpak-builder --run build-dir "$CI_MANIFEST" sh -c 'cd examples/hello-world && crafter-build && ls bin/*/hello*' - # Everything below publishes and only runs on master and on tags. + # Everything below releases the build and runs only for a pushed tag. - name: Bundle - if: github.event_name != 'pull_request' + if: github.event_name == 'push' run: | mkdir -p dist - flatpak build-bundle repo "dist/crafter-build-${{ steps.meta.outputs.version }}.flatpak" "$APP_ID" "${{ steps.meta.outputs.branch }}" - cp packaging/flatpak/net.catcrafts.CrafterBuild.flatpakref dist/ - (cd dist && sha256sum * > SHA256SUMS) + flatpak build-bundle repo "dist/crafter-build-${{ steps.meta.outputs.version }}.flatpak" "$APP_ID" "$BRANCH" ls -la dist - name: Upload bundle - if: github.event_name != 'pull_request' + if: github.event_name == 'push' uses: actions/upload-artifact@v3 with: name: crafter-build-${{ steps.meta.outputs.version }} @@ -133,19 +244,24 @@ jobs: if-no-files-found: error - name: Publish to the Catcrafts repo - if: github.event_name != 'pull_request' + if: github.event_name == 'push' run: | set -eu # The signing key is generated once on the runner host # (/etc/crafter-flatpak: signing-key.asc and key-id) and mounted # read-only; it is imported into a throwaway keyring per run. if [ ! -s /run/crafter-flatpak/signing-key.asc ] || [ ! -s /run/crafter-flatpak/key-id ]; then - echo "no signing key under /etc/crafter-flatpak on the runner host; generate it as described in AGENTS.md (CI and publishing)" >&2 + echo "no signing key under /etc/crafter-flatpak on the runner host; generate it as described in AGENTS.template.md (CI and publishing)" >&2 exit 1 fi export GNUPGHOME=$(mktemp -d) gpg --batch --quiet --import /run/crafter-flatpak/signing-key.asc FLATPAK_GPG_KEY_ID=$(cat /run/crafter-flatpak/key-id) + # Crafter.Review's workflow publishes into the same repo and holds + # this lock while it does, so a commit and the summary update of + # one workflow never interleave with the other's. + exec 9> "$WEB_DIR/.publish.lock" + flock 9 # The live repo is the canonical copy; builds are promoted into it, # never built into it. # Keyed on the repo's config file, not the directory: the web root @@ -153,56 +269,49 @@ jobs: [ -f "$REPO_DIR/config" ] || ostree init --repo="$REPO_DIR" --mode=archive flatpak build-commit-from --src-repo=repo \ --gpg-sign="$FLATPAK_GPG_KEY_ID" --gpg-homedir="$GNUPGHOME" \ - "$REPO_DIR" "app/$APP_ID/x86_64/${{ steps.meta.outputs.branch }}" + "$REPO_DIR" "app/$APP_ID/x86_64/$BRANCH" + # Every push to master used to be published as `beta`. Its last + # build is marked end-of-life, once, so that the next + # `flatpak update` of a beta installation says why nothing follows. + # The reason names the branch: with --noninteractive, flatpak's + # message names only the app. + beta="app/$APP_ID/x86_64/beta" + if ostree rev-parse --repo="$REPO_DIR" "$beta" >/dev/null 2>&1 && + ! ostree show --repo="$REPO_DIR" --print-metadata-key=ostree.endoflife "$beta" >/dev/null 2>&1; then + flatpak build-commit-from --src-ref="$beta" \ + --end-of-life="The beta branch gets no more updates: every change now comes as a release on the stable branch. Install stable with flatpak install catcrafts net.catcrafts.CrafterBuild//stable, then remove beta with flatpak uninstall net.catcrafts.CrafterBuild//beta." \ + --gpg-sign="$FLATPAK_GPG_KEY_ID" --gpg-homedir="$GNUPGHOME" \ + "$REPO_DIR" "$beta" + fi flatpak build-update-repo --gpg-sign="$FLATPAK_GPG_KEY_ID" --gpg-homedir="$GNUPGHOME" \ --generate-static-deltas --prune "$REPO_DIR" # The repo description and the app ref carry the public key. key=$(gpg --export "$FLATPAK_GPG_KEY_ID" | base64 --wrap=0) sed "s|GPGKey=.*|GPGKey=$key|" packaging/flatpak/catcrafts.flatpakrepo > "$WEB_DIR/catcrafts.flatpakrepo" sed "s|GPGKey=.*|GPGKey=$key|" packaging/flatpak/net.catcrafts.CrafterBuild.flatpakref > "$WEB_DIR/net.catcrafts.CrafterBuild.flatpakref" + flock -u 9 + # The release attaches this .flatpakref, not the one in the + # checkout: that one has a placeholder where the key goes, and + # flatpak cannot install from it. + cp "$WEB_DIR/net.catcrafts.CrafterBuild.flatpakref" dist/ + (cd dist && sha256sum * > SHA256SUMS) + ls -la dist - - name: Rolling 'latest' release - if: github.event_name != 'pull_request' && github.ref_type != 'tag' - run: | - git config user.email "ci@catcrafts.net" - git config user.name "Crafter Build CI" - git tag -f latest - git push origin latest --force - - name: Publish rolling 'latest' release - if: github.event_name != 'pull_request' && github.ref_type != 'tag' - uses: https://code.forgejo.org/actions/forgejo-release@v2 - with: - direction: upload - url: ${{ github.server_url }} - repo: ${{ github.repository }} - tag: latest - title: Latest master build - prerelease: true - override: true - release-dir: dist - token: ${{ secrets.GITHUB_TOKEN }} - - - name: Release notes - if: github.ref_type == 'tag' - run: | - # The newest block's description, as plain text. - python3 - > release-notes.md <<'EOF' - import re, html - text = open("packaging/flatpak/net.catcrafts.CrafterBuild.metainfo.xml").read() - block = re.search(r"]*>(.*?)", text, re.S).group(1) - paras = re.findall(r"

(.*?)

", block, re.S) - print("\n\n".join(html.unescape(" ".join(p.split())) for p in paras)) - print("\nInstall:\n\n flatpak remote-add --if-not-exists catcrafts https://flatpak.catcrafts.net/catcrafts.flatpakrepo\n flatpak install catcrafts net.catcrafts.CrafterBuild\n") - EOF - name: Publish release - if: github.ref_type == 'tag' + if: github.event_name == 'push' + # The notes are release-notes.md, which the Version step wrote. No + # `override`: it would delete a tag that has moved off this commit + # since the Version step and tag the old commit instead. Without it + # the action refuses such a tag, and a release that an earlier + # attempt left half made, such as a draft whose upload failed, has + # to be deleted before a re-run. uses: https://code.forgejo.org/actions/forgejo-release@v2 with: direction: upload url: ${{ github.server_url }} repo: ${{ github.repository }} - tag: ${{ github.ref_name }} - title: crafter-build ${{ github.ref_name }} + tag: ${{ steps.meta.outputs.version }} + title: crafter-build ${{ steps.meta.outputs.version }} release-notes-file: release-notes.md release-dir: dist token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore index 67d2a68..f92f1f1 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,7 @@ share/crafter-build/ .claude # written by CI next to the manifest so relative source paths still resolve packaging/flatpak/ci-manifest.yaml +# written by every build and test run from AGENTS.template.md +/AGENTS.md +.review-gates/push/ +.review-gates/trees/ diff --git a/.review-gates/config.json b/.review-gates/config.json new file mode 100644 index 0000000..115ee6c --- /dev/null +++ b/.review-gates/config.json @@ -0,0 +1,38 @@ +{ + "base": "origin/master", + "rangeFrom": "upstream", + "authors": [ + "jorijnvdgraaf@catcrafts.net" + ], + "aiTrailers": "forbid", + "requireSigned": true, + "lenses": { + "repo-better-solution": { + "section": ".review-gates/lenses/repo-better-solution.md" + }, + "repo-bug-prover": { + "section": ".review-gates/lenses/repo-bug-prover.md" + }, + "repo-facts": { + "section": ".review-gates/lenses/repo-facts.md" + } + }, + "modes": { + "commits": { + "kind": "commits", + "lenses": [ + "repo-better-solution", + "repo-bug-prover", + "repo-facts" + ], + "effort": "xhigh" + } + }, + "gate0": [ + "./build.sh", + "./bin/crafter-build", + "./bin/crafter-build test", + "./bin/crafter-build format --check", + "./bin/crafter-build lint" + ] +} diff --git a/.review-gates/lenses/repo-better-solution.md b/.review-gates/lenses/repo-better-solution.md new file mode 100644 index 0000000..278b12f --- /dev/null +++ b/.review-gates/lenses/repo-better-solution.md @@ -0,0 +1,42 @@ +## This repository: Crafter.Build + +A C++26 modules build system that builds itself. The stated design is +`README.md` (the manual) and `AGENTS.template.md` (the contributor guide; +`AGENTS.md` is written from it by a crafter-build build or test run, not +by `build.sh`, and is not in the tree). +Its "Design rules" and "Security considerations" are the owner's +decisions: a range that departs from one without saying so is a finding; +whether one should change is the owner's call, not a defect of the range. +Among them, for CI, which holds the workflow's token and the signing key +its runner mounts: no secret is added to the workflow, and the +environment is never echoed in it. + +The layers: the library (`interfaces/`, `implementations/`), which the +executable wraps and every `project.cpp` is compiled against; this +repository's own `project.cpp`; `build.sh`, the bootstrap that builds the +tool without the tool; the Flatpak (`packaging/flatpak/`), which bundles +the LLVM users build with; CI (`.forgejo/workflows/ci.yaml`). A target +difference branches on `cfg.target` at run time, never on the host; an +output path goes through `BinDir()`, `BuildDir()` or `PcmDir()`; whatever +changes what gets compiled or bundled reaches `VariantId()`. + +Search first: the partitions' own helpers (`RunBuildCommand`, +`RunCommandChecked`, `Progress::Diagnostic`, `ApplyStandardArgs`, +`HostTarget`, `GitProject`, `LocalProject`); the Crafter.* libraries the +tree already uses, cloned under `~/.cache/crafter.build/external/`; libc++ +and clang as the host installs them (`/run/host/usr/include/c++/v1`, +`/run/host/usr/lib/clang/`); git, cmake and flatpak-builder. + +Commit shape here: a fix, its test and its README change are separate +commits when each stands on its own, in that order, and a bug fix ships +with a test that fails before it. That split is not a later commit +repairing an earlier one. + +Compatibility: the `project.cpp` of every sibling repository beside this +checkout (`../Crafter.*/project.cpp`) and of every user compiles against +what `interfaces/` exports and is loaded by the binary; the CLI, the +`CRAFTER_BUILD_*` variables, the cache an older version left under +`~/.cache/crafter.build/` and the `bin/` layout CI globs are contracts too. +Users get master through releases on the Flatpak's `stable` branch. A +change an existing `project.cpp` does not survive is a +finding unless the range documents, in the README, what users must change. diff --git a/.review-gates/lenses/repo-bug-prover.md b/.review-gates/lenses/repo-bug-prover.md new file mode 100644 index 0000000..13c9531 --- /dev/null +++ b/.review-gates/lenses/repo-bug-prover.md @@ -0,0 +1,46 @@ +## This repository: Crafter.Build + +The execution contexts to enumerate, beside the charter's: +- `project.cpp` is compiled to a shared object and loaded into the running + crafter-build, resolving `Crafter::` symbols from the executable + (`--export-dynamic`); its static initialisers run at load. Its sibling + projects (`GitProject`, `LocalProject`) load the same way into the same + process, and its `#pragma crafter dependency` targets are built for the + host first, on every load. +- The clang++, clang and llvm-ar the tool spawns, and the git and cmake it + runs, are shell commands (`popen`, `std::system` in `:Platform`), many in + parallel; such a step fails on its exit code, never because it printed. + Running a test or a built artifact is a shell command too (`:Test`, + `:Clang`). A path, file name or flag that reaches one with a space or a + shell metacharacter in it is an input to attack. Other steps run inside + the process, among them glslang's shader compiles, on a thread of their + own per stale shader (`:Shader`), and lint's parse through the libclang + the linter dlopens (`:Lint`). +- Several crafter-build processes share `~/.cache/crafter.build/` at once: + the std and Crafter.Build PCMs per target and march (the + `ConcurrentCacheRace` test), and the external clones with their `.lock` + files. +- Three builds of the tool: the bootstrap from `build.sh` (without + `CRAFTER_BUILD_HAS_ASSET`), the self-hosted binary on the host's LLVM, and + the Flatpak on its own LLVM under `/app`, in a sandbox whose `/usr` is the + Freedesktop SDK's. Their toolchains can be a release apart. +- Targets: the host triple, a sysroot (aarch64, musl), `x86_64-w64-mingw32` + under wine, `wasm32-wasi*` under wasmtime (or wasmer for `-r`), and + browser builds through the runtime in `wasi-runtime/`. +- Incremental builds: a first build, a rebuild after an edit, a rebuild + after a toolchain or flag change and a build after `clean` start from + different states; whatever changes what gets compiled has to reach + `VariantId()`, or a stale artifact is reused. +- Tests: each its own subprocess, in parallel, with a 60 s timeout. A test + that edits or touches files stages a copy under + `fs::temp_directory_path()`; one that only builds its fixture builds it in + place, into the fixture's own `bin/` and `build/` in the checkout. + +Semantics to read on disk rather than recall: libc++ and clang as the host +installs them (`/run/host/usr/include/c++/v1`, +`/run/host/usr/share/libc++/v1/std.cppm`, +`/run/host/usr/lib/clang//include`), the dependencies as cloned +(`~/.cache/crafter.build/external/-/`), the vendored +`lib/DirStackFileIncluder.h`. The Flatpak's own LLVM is not readable from +here: a semantic that may differ between LLVM releases is unverified for +the Flatpak build. diff --git a/.review-gates/lenses/repo-facts.md b/.review-gates/lenses/repo-facts.md new file mode 100644 index 0000000..7129581 --- /dev/null +++ b/.review-gates/lenses/repo-facts.md @@ -0,0 +1,18 @@ +## This repository: Crafter.Build + +- A claim that a tree passes `build.sh`, the self-host build, the test + suite, `format --check` or `lint` traces to the Gate 0 log among the + evidence paths INPUTS names: each command with its exit code and the sha + it ran at. A claim of a run anywhere else (the Flatpak through + `run flatpak`, CI, wine, qemu, node, a browser) traces to a capture of + that run that names the commit, or is unverifiable. +- A test a message names exists as a `cfg.AddTest("")` line in + `project.cpp` and a `tests//main.cpp` whose checks pin what the + message says. +- `AGENTS.md` is generated from `AGENTS.template.md` and the dependencies' + `AGENTS.consumer.md` and is not in the tree: a claim about it is checked + against those sources. +- A `` entry in + `packaging/flatpak/net.catcrafts.CrafterBuild.metainfo.xml` is the + release notes users read: every sentence in it is a claim about the code + since the previous tag. diff --git a/AGENTS.md b/AGENTS.template.md similarity index 63% rename from AGENTS.md rename to AGENTS.template.md index d5227e8..f275a93 100644 --- a/AGENTS.md +++ b/AGENTS.template.md @@ -11,6 +11,7 @@ crafter-build is a C++26 modules build system. A project is described by a `proj | `interfaces/Crafter.Build-.cppm` | Module interface partitions. `Crafter.Build.cppm` is the umbrella that re-exports them. | | `implementations/Crafter.Build-.cpp` | Implementation units, one per partition, named `Crafter.Build:_impl`. `main.cpp` is a thin wrapper around `Crafter::Run`. | | `project.cpp` | The self-host build description: a `crafter.build-lib` library, a `crafter.build-exe` executable, the test declarations, the lint rules and the `flatpak` and `install` scripts. | +| `AGENTS.template.md` | The source of AGENTS.md. Every build and test run writes AGENTS.md from it, with the notes the dependencies ship; AGENTS.md is gitignored, so changes go here. | | `tests//main.cpp` | One self-test per directory; `tests//fixture/` holds the sample project it builds. | | `examples/` | Standalone user-facing projects, MIT licensed. | | `build.sh` | Bootstrap script. It compiles everything by hand, without crafter-build. | @@ -18,10 +19,11 @@ crafter-build is a C++26 modules build system. A project is described by a `proj | `lib/` | Vendored third-party headers (nlohmann/json, glslang's DirStackFileIncluder). Do not edit. | | `share/crafter-build/`, `bin/`, `build/`, `lib/*.a` | Generated and gitignored. | | `packaging/flatpak/`, `.forgejo/workflows/ci.yaml` | The Flatpak manifest with its metainfo, repo and ref files, and the Forgejo Actions CI that builds and publishes it. | +| `.review-gates/` | The pre-push review's `config.json` and this repository's lens sections (`lenses/`), read by Crafter.Review. Its `push/` and `trees/` are generated and gitignored. | Partitions and what they own: -- `:Toolchain` resolves the compiler once per process (`GetToolchain`): clang++, clang, llvm-ar, libclang, libc++'s std module source and the default sysroots, from `CRAFTER_BUILD_LLVM_DIR`, else a toolchain installed beside crafter-build, else PATH. It also holds the version baked in at build time. `SysrootStdModule` picks the std module source for a sysroot build by the sysroot's libc++ release. +- `:Toolchain` resolves the compiler once per process (`GetToolchain`): clang++, clang, llvm-ar, libclang, libc++'s std module source and the default sysroots, from `CRAFTER_BUILD_LLVM_DIR`, else a toolchain installed beside crafter-build, else PATH. It also holds the version baked in at build time. `SysrootStdModule` picks the std module source for a sysroot build by the sysroot's libc++ release. `MingwSysroot` settles the mingw target's `--sysroot`: the mingw-w64 prefix itself, or, for a prefix that also holds a Linux system (the host's `/usr`, anything whose `include/` has `features.h`), a view in the cache of its mingw-w64 directories, since clang would search its `include/` and `lib/` too (issue #43). `MingwViewToPrefix` turns paths into the view back into paths into the prefix for messages. - `:Clang` holds `Configuration`, `Build()`, `VariantId()`, `ApplyStandardArgs`, `HostTarget`, the CLI entry point `Crafter::Run`, and the lint, test and script data types. - `:Platform` runs commands, implements `LoadProject` (compile and load `project.cpp`), the host PCM cache, `#pragma crafter dependency`, and all OS-specific code. - `:Interface` and `:Implementation` model module units, their staleness checks and their compiles. @@ -29,6 +31,7 @@ Partitions and what they own: - `:Test` holds `AddTest`, `RunTests` and `TestRunner`. - `:Lint` is the lint/format driver plus the libclang token and AST layer. - `:Script` is the driver behind `crafter-build run`: it looks a name up in `cfg.scripts`, calls the callback and turns what it returns into the exit code. +- `:Agents` writes a project's AGENTS.md from its AGENTS.template.md: it walks the dependencies `cfg.dependencyNotes` selects, reads each one's `AGENTS.consumer.md` and merges them into the template's dependency notes section. Which Configurations belong to another project, and where it lives, comes from the loader (`LoadedProjectOf` in `:External`, `PragmaDependencies` in `:Platform`), because only the Configuration `GitProject` and `LocalProject` return has its `path` rebased. - `:Progress` is console output. `:Shader` compiles shaders through glslang. `:Asset` is the hook into Crafter.Asset. ## Dev environment @@ -48,7 +51,7 @@ Three things to know about this loop: - Changes under `implementations/` are picked up by the self-host build and by the tests, which link the local library. Changes under `interfaces/` are not: the harness compiles against the copies in `share/crafter-build/` and the PCM cache built from them, so a new or changed declaration is invisible until you run `./build.sh` again. The symptom is `use of undeclared identifier` for the symbol you just added. A changed default member initialiser has no symptom at all: `project.cpp` compiles against the old one and quietly gets the old default, as `project.cpp`'s sibling loads do with a harness that predates `GitProjectSpec::args = ProjectArgs()`. - A new exported symbol that a `project.cpp` reaches on its default path also needs `./build.sh`: the old binary does not export it, and `LoadProject` fails with an undefined symbol. -`./build.sh` clones and builds glslang on first run and takes several minutes; later runs are faster. `./bin/crafter-build clean` deletes `bin/` and `build/` without loading `project.cpp`. That takes `bin/crafter-build` itself with it, and `build/flatpak`, the state of the local Flatpak build (see "CI and publishing"): about 2 GB, and the next `run flatpak` rebuilds LLVM. Set `CRAFTER_BUILD_HOME` only when running a binary from outside this tree; `CRAFTER_BUILD_MARCH` and `CRAFTER_BUILD_MTUNE` default to `native`. +`./build.sh` clones and builds glslang on first run and takes several minutes; later runs are faster. `./bin/crafter-build clean` deletes `bin/` and `build/` without loading `project.cpp`. That takes `bin/crafter-build` itself with it, and `build/flatpak`, the state of the local Flatpak build (see "CI and publishing"): about 4 GB after a first run, and the next `run flatpak` rebuilds LLVM. Set `CRAFTER_BUILD_HOME` only when running a binary from outside this tree; `CRAFTER_BUILD_MARCH` and `CRAFTER_BUILD_MTUNE` default to `native`. ## Build, test and style commands @@ -66,7 +69,7 @@ Three things to know about this loop: ./bin/crafter-build clean ``` -Before you call a change done, all three must exit 0: `format` (then re-read your diff), `lint`, `test`. CI runs `./build.sh` and `crafter-build test` on every PR; its lint step is currently commented out, so lint is enforced only by you. Failed, crashed and timed-out tests leave their output in `build/test-logs/.log`. +Before you call a change done, all three must exit 0: `format` (then re-read your diff), `lint`, `test`. CI runs `./build.sh` and `crafter-build test` for every release tag, every pull request into `master` and every manual run, and has no lint step, so lint is enforced only by you. Failed, crashed and timed-out tests leave their output in `build/test-logs/.log`. ## Testing instructions @@ -116,31 +119,34 @@ Beyond the rules: - There is no host-platform preprocessor split: the host is Linux. Target differences (mingw, wasm) branch on `cfg.target` strings at runtime, in the library as well as in `project.cpp` and tests. - A build step fails when its command exits nonzero, never because it printed. Use `RunBuildCommand` or `RunCommandChecked`; warnings go to `Progress::Diagnostic`. - Every output path goes through `Configuration::BinDir()`, `BuildDir()` and `PcmDir()`. Hand-rolled `path / "bin" / ...` clobbered variants before; do not add new ones. -- Anything that changes what gets compiled or bundled belongs in `VariantId()`. It already hashes type, debug, sysroot, defines, compile flags, wasm variant flags, uninterpreted project args and the toolchain (`Toolchain::Key()`). +- Anything that changes what gets compiled or bundled belongs in `VariantId()`. It already hashes type, debug, sysroot, defines, compile flags, the wasm baseline and variant flags, the std module the units import (`StdPcmKey`), uninterpreted project args and the toolchain (`Toolchain::Key()`). ## Design rules These were decided deliberately. Do not undo them without the maintainer's say-so. 1. **Configuration is C++.** No JSON, YAML, TOML or DSL for project, runner or lint configuration. Lint rules are C++ callbacks; there is no clang-format or clang-tidy integration and no separate "format rule" type. A rule that calls `ctx.SetContent` is a transform, and that is the whole mechanism. -2. **Per-project artifacts go in `/build` and `/bin`. Only cross-project things go in `~/.cache/crafter.build/`:** the std PCM and Crafter.Build PCMs per target and march, and external clones keyed by their spec hash. +2. **Per-project artifacts go in `/build` and `/bin`. Only cross-project things go in `~/.cache/crafter.build/`:** the std PCM and Crafter.Build PCMs per target, march, toolchain and std module flags, external clones keyed by their spec hash, and the mingw sysroot views keyed by prefix. 3. **Sources ship, PCMs build locally.** The Flatpak installs `*.cppm` under `/app/share/crafter-build/`. Nothing target-baked ships except the executable and the toolchain beside it, and the executable is built for a portable `-march`. 4. **crafter-build is a monolithic executable with `--export-dynamic`.** `project.so` resolves `Crafter::` symbols from the running binary at dlopen time. There is no Windows host; `x86_64-w64-mingw32` is a cross-compile target for user projects only. 5. **Convention over configuration.** Lead with the zero-config path. Env var, then CLI flag, then full `project.cpp` are the escalation, not the entry point. 6. **One skip mechanism** (runner probe plus exit 77) and **one `TestRunner` shape** (template fields with `{bin}`, `{bundle}` and similar placeholders; `Cmd(...)` covers any wrapper). Do not add a special-case runner class or a hardcoded "skip on this OS" branch. 7. **The house-style rules stay in their own repo.** They are not library API of crafter-build and are not forced on downstream projects. -8. **Fix warts inline; do not stack workarounds.** The previous version of this tool died of accumulated hacks. If a change needs a shortcut, say so in the PR rather than hiding it. +8. **Fix warts inline; do not stack workarounds.** The previous version of this tool died of accumulated hacks. If a change needs a shortcut, say so in its commit message rather than hiding it. 9. **`project.cpp` keeps the library/executable split.** `crafter.build-lib` owns the sources, dependencies and glslang; `crafter.build-exe` is `main.cpp` plus link flags. 10. **Host-tooling builds** (the target of a `#pragma crafter dependency`) get `-fPIC` and the host triple scoped to that build. Keep it scoped. -## Commit and PR guidelines +## Commit and push guidelines -- Branch from `master` and open the PR against `master` on the project's Forgejo at `forgejo.catcrafts.net/Catcrafts/Crafter.Build`. Issue work uses `claude/issue-` branch names. Pushes go over SSH; HTTPS push is disabled on the server. +- Branch from `master`; issue work uses `claude/issue-` branch names. Once the owner approves its frozen hash, a range goes straight to `master` on the project's Forgejo at `forgejo.catcrafts.net/Catcrafts/Crafter.Build`, as a fast-forward push with no merge commit and no pull request; open a pull request against `master` only when the owner asks for one. If `master` has moved since the freeze, rebase onto it, then freeze and review the rebased range again. Pushes go over SSH; HTTPS push is disabled on the server. - Subject line: `type(scope): imperative summary`, with `feat`, `fix`, `refactor`, `docs` or `test`; the scope (`lint`, `incremental`, ...) is optional. Body: what was wrong, why, and why the fix has this shape, wrapped at about 75 columns. Read `git log` for the voice. - Separate commits for the fix, its test and the README change when each stands on its own. -- Update the README in the same PR when user-visible behaviour changes. -- `format`, `lint` and `test` green before pushing. CI builds the Flatpak and runs the suite inside it on PRs, publishes the `beta` branch of the Catcrafts repo on every push to `master`, and the `stable` branch plus a release on an `X.Y.Z` tag. -- Never commit `bin/`, `build/`, `share/crafter-build/`, `lib/*.a` or fixture build outputs (all gitignored), and never edit the vendored files in `lib/`. +- Update the README in the same push when user-visible behaviour changes. +- `format`, `lint` and `test` green before pushing. CI builds the Flatpak and runs the suite inside it for every release tag, every pull request into `master` and every manual run. Without a pull request, CI does not run on a range before it reaches `master` unless someone starts it by hand (`workflow_dispatch`), so the review's gate0, the commands in `.review-gates/config.json`, is the check before the push. +- Every push to `master` is a release, made by the signed tag pushed with it (see "CI and publishing"). Every pushed range adds a `` entry for its version at the top of `packaging/flatpak/net.catcrafts.CrafterBuild.metainfo.xml`, in a commit of its own (`docs(packaging): add the X.Y.Z release entry`), dated the day of the push in UTC. Its notes have up to three sections, `

New

`, `

Fixes

` and `

Breaking

` in that order, each followed by a `
    ` of its items, the most important first, and nothing after the last section. The version follows from them: a Breaking section makes it a major, New items a minor, and Fixes alone a patch. Once the owner has approved the frozen hash, the commit `master` is to receive gets the signed tag of that version, made locally, and both go out in one push: `git tag -s X.Y.Z -m 'crafter-build X.Y.Z'`, then `git push --atomic origin HEAD:master X.Y.Z`. That commit is the range's last one, or, when the owner asks for a pull request instead, the merge of the reviewed branch into `master`. +- Every push of new commits goes through Crafter.Review's pre-push review first, and goes out only once the owner approves the frozen hash. Run `flatpak run net.catcrafts.CrafterReview init` once in a fresh checkout or worktree: it writes the `/prepush` skill that drives the review, and the skill stays untracked. `freeze` refuses merge commits: a range reaches `master` as a fast-forward, and when the owner asks for a pull request instead, the merge of the reviewed branch into `master` is pushed without a freeze of its own. +- Every commit of a pushed range is GPG-signed, has `jorijnvdgraaf@catcrafts.net` as its author and committer, and carries no AI attribution trailer in its message, neither `Co-Authored-By:` nor `Assisted-by:`. `freeze` refuses a range with a commit that breaks any of these (`requireSigned`, `authors` and `aiTrailers` in `.review-gates/config.json`). +- Never commit `bin/`, `build/`, `share/crafter-build/`, `lib/*.a`, `AGENTS.md` or fixture build outputs (all gitignored), and never edit the vendored files in `lib/`. - Keep `CRAFTER_BUILD_MARCH=x86-64-v2` in the manifest for the shipped binary: it has to run on any x86_64 machine. User projects still default to `native`. ## Security considerations @@ -150,14 +156,14 @@ These were decided deliberately. Do not undo them without the maintainer's say-s - `--runner=cmd:` executes the named wrapper around every test binary. Wrappers come from `--runner=` or `CRAFTER_BUILD_RUNNER_*`; do not hardcode hosts, wrappers or credentials in `project.cpp` or tests. - `crafter-build run ` executes whatever the script does, with your permissions. Only the scripts of the project crafter-build was pointed at can be run, never a dependency's; keep it that way. - The linter loads `libclang` at runtime by name. `CRAFTER_BUILD_LIBCLANG` pins an exact path and must fail loudly when wrong, never fall back to another library. -- CI force-pushes the rolling `latest` tag and publishes a prerelease on every push to `master` with the workflow's own token, and signs the Flatpak repo with the key mounted from the runner host. Do not add secrets to the workflow or echo the environment in it. +- CI creates a release for each release tag with the workflow's own token, pushes nothing to git, and signs the Flatpak repo with the key mounted from the runner host. Do not add secrets to the workflow or echo the environment in it. - Tests write only under `fs::temp_directory_path()` and the fixture's own `bin/` and `build/`. Keep it that way. ## CI and publishing -`.forgejo/workflows/ci.yaml` runs on the `crafter-flatpak` runner label: a dedicated privileged forgejo-runner on the Hetzner box (`forgejo-runner-flatpak.service`, config `/etc/forgejo-runner/config-flatpak.yaml`), registered to this repository only and the only runner with that label. It is privileged because bubblewrap inside the job container needs it, and separate from the other runners so that privilege is not shared with any other repository. Its config allows four volumes, which the workflow mounts: `crafter-flatpak-system` (the Sdk install at `/var/lib/flatpak`), `crafter-flatpak-builder` (flatpak-builder state and ccache, so the LLVM and WASI modules are rebuilt only when their sources change; a cold run takes about 80 minutes on 8 cores, a warm one a fraction of that), `/var/www/flatpak`, the web root Caddy serves as `https://flatpak.catcrafts.net/`, and `/etc/crafter-flatpak`, which holds the repo signing key (`signing-key.asc`, no passphrase, and `key-id`), generated once on the host and mounted read-only into the job; nothing is stored in the forge. The live repo at `/var/www/flatpak/repo` is canonical; builds are promoted into it with `flatpak build-commit-from`, never built into it. +`.forgejo/workflows/ci.yaml` runs on the `crafter-flatpak` runner label: a dedicated privileged forgejo-runner on the Hetzner box (`forgejo-runner-flatpak.service`, config `/etc/forgejo-runner/config-flatpak.yaml`), registered to this repository only and the only runner with that label. It is privileged because bubblewrap inside the job container needs it, and no other repository's job runs on it. Its config allows four volumes, which the workflow mounts: `crafter-flatpak-system` (the Sdk install at `/var/lib/flatpak`), `crafter-flatpak-builder` (flatpak-builder state and ccache, which later runs reuse: flatpak-builder builds a module again when its sources or its `build-commands` change, and every module after it with it, so a new version of `swig` or `libedit`, which come before LLVM, also rebuilds the LLVM and WASI modules), `/var/www/flatpak`, the web root Caddy serves as `https://flatpak.catcrafts.net/`, and `/etc/crafter-flatpak`, which holds the repo signing key (`signing-key.asc`, no passphrase, and `key-id`), generated once on the host and mounted read-only into the job; nothing is stored in the forge. The web root and the key are not this runner's alone: the organisation-wide `flatpak` runner (`forgejo-runner-org-flatpak.service`), which serves every Catcrafts repository, mounts both for Crafter.Review's publish job, and the volumes a runner allows (`valid_volumes` in its config) are open to every job it runs, so a job of any Catcrafts repository that runs there can mount them as well. The live repo at `/var/www/flatpak/repo` is canonical; builds are promoted into it with `flatpak build-commit-from`, never built into it. Crafter.Review's workflow publishes into the same repo, and each workflow holds an flock on `/var/www/flatpak/.publish.lock` while it commits and updates the summary. -Keep the label unique to that runner. Plain `flatpak` also belongs to the organisation-wide runner (`forgejo-runner-org-flatpak.service`), which serves every Catcrafts repository and on purpose allows no volumes. Whenever this job lands there, flatpak-builder stops with `The state dir (/root/.cache/flatpak-builder) is not on the same filesystem as the target dir`; the top of the job log shows which runner took it and lists every volume it ignored. +Keep the label unique to that runner. Plain `flatpak` belongs to the organisation-wide runner. When this job still asked for `flatpak` and that runner took it (runs #368 and #370, in October 2026), the runner allowed none of the job's volumes, and flatpak-builder stopped with `The state dir (/root/.cache/flatpak-builder) is not on the same filesystem as the target dir`; the top of such a job's log shows which runner took it and lists every volume it ignored. The key is made once, as root on the runner host, and the step fails with a message pointing here when it is missing: @@ -169,7 +175,8 @@ gpg --batch --pinentry-mode loopback --passphrase '' --export-secret-keys --armo printf '%s\n' "$KEYID" > /etc/crafter-flatpak/key-id && chmod 600 /etc/crafter-flatpak/signing-key.asc /etc/crafter-flatpak/key-id ``` -To release: add a `` to `packaging/flatpak/net.catcrafts.CrafterBuild.metainfo.xml` (its description becomes the release notes), merge, and push the tag `X.Y.Z`. CI refuses a tag that does not match the newest entry. +A release is a tag `X.Y.Z`, annotated and signed, pushed with the `master` it releases; a push to `master` without one runs nothing. Before building, every run renders the release notes from the newest `` entry in `packaging/flatpak/net.catcrafts.CrafterBuild.metainfo.xml`, and stops when the metainfo does not parse or the notes come out empty. A tag run also checks that Forgejo verifies the tag as signed by the address the workflow's `RELEASE_SIGNER` names, that the tag's commit is on `master`, that its version has the form `X.Y.Z` and is newer than every release the forge has published, its own included, and that the newest entry is that version. Once the build and the suite inside it pass, the tag run publishes `stable` and creates the release, with the notes, the bundle, the `.flatpakref` and `SHA256SUMS` attached. Release runs queue one behind another, and a failed one can be re-run while no newer release exists, once a release it left half made is deleted; when the fix needs a commit of its own, that push is the next version's release. Pull requests and manual runs publish nothing. The `beta` branch, which every push to `master` used to go to, gets no more builds: the publish step marks its last build end-of-life, once, and the next `flatpak update` of a beta installation prints the reason. + Local builds are two scripts in `project.cpp`. Run them with the native binary: the Flatpak build of crafter-build has no `flatpak` to drive. ```bash @@ -177,7 +184,7 @@ Local builds are two scripts in `project.cpp`. Run them with the native binary: ./bin/crafter-build run install # install that build as branch `local` and make it current ``` -`run flatpak` needs `flatpak-builder` or the `org.flatpak.Builder` app, and the Sdk (`flatpak install flathub org.freedesktop.Sdk//26.08`). It builds the working tree, uncommitted changes included, runs the whole suite inside the sandbox and reports version `0.0.0-dev`. Its state is `build/flatpak/` (`state/`, `repo/`, `build-dir/`). The first run builds LLVM and the WASI sysroot, about 25 minutes on a 56-core machine; later runs rebuild only the crafter-build module, a few minutes, because flatpak-builder never caches a `type: dir` source. The checkout must not be under `/tmp`, which `org.flatpak.Builder` cannot see. Use it for anything toolchain-sensitive: the native loop compiles with the distro's clang and the Flatpak with its bundled LLVM, and the two can be a major version apart. +`run flatpak` needs `flatpak-builder` or the `org.flatpak.Builder` app, and the Sdk (`flatpak install flathub org.freedesktop.Sdk//26.08`). It builds the working tree, uncommitted changes included, runs the whole suite inside the sandbox and reports version `0.0.0-dev`. Its state is `build/flatpak/` (`state/`, `repo/`, `build-dir/`). The first run builds every module, LLVM with lldb, the WASI sysroot and the Vulkan validation layer among them, in about 30 minutes with 56 CPU threads; later runs rebuild only the crafter-build module while the manifest is unchanged, a few minutes, because flatpak-builder never caches a `type: dir` source. The checkout must not be under `/tmp`, which `org.flatpak.Builder` cannot see. Use it for anything toolchain-sensitive: the native loop compiles with the distro's clang and the Flatpak with its bundled LLVM, and the two can be a major version apart. `run install` does not build; it installs what `run flatpak` last exported. It points the user remote `crafter-build-local` at `build/flatpak/repo`, installs `net.catcrafts.CrafterBuild//local` from it next to whichever published branch is installed, and makes it the branch `flatpak run net.catcrafts.CrafterBuild` starts. `flatpak make-current --user net.catcrafts.CrafterBuild ` switches back. Before `crafter-build clean`, or before deleting the checkout, run `flatpak uninstall --user net.catcrafts.CrafterBuild//local` and `flatpak remote-delete --user crafter-build-local`; otherwise the remote is left pointing at a repo that is gone. @@ -187,7 +194,13 @@ Local builds are two scripts in `project.cpp`. Run them with the native binary: - A `#pragma crafter dependency` target is rebuilt on every load on purpose: clang crashes rather than erroring when it imports a BMI whose own imported PCM was rebuilt underneath it, and the depfile check is what catches a regenerated host cache. Do not optimise the rebuild away. - A host-tooling library such as Crafter.LintRules may `import Crafter.Build;` without declaring it as a dependency, because `LoadProject` fills the cache it resolves from. Listing Crafter.Build in its `cfg.dependencies` would clone and build all of crafter-build for every consumer. Its test does link crafter-build, so that test is wired behind `IsTopLevelProject()`: without the guard, crafter-build loading the library as a dependency would load its own `project.cpp` again through the test's `GitProject` and hit the cycle guard. - `crafter-build clean` leaves a pragma dependency's build alone; it lives with the clone in the cache. +- Units see a std module rebuilt in place, because their depfiles list the BMIs they import (`-fmodule-file-deps`), but not one that moved to another directory: their old depfiles still name the old one. So anything that changes how the std module is compiled goes into `StdPcmFlags`, which `BuildStdPcm` compiles with and `StdPcmKey` hashes. `VariantId()` hashes `StdPcmKey`, so the build moves with it. A flag added to the command anywhere else leaves both directories where they are. +- A CMake external whose recorded configure inputs (`/.crafter-options`) differ from this build's is built again from an empty build directory, not reconfigured in place: CMake keeps what an old compiler or `--sysroot` compiled, and the check results it cached. - The bootstrap binary is compiled without `CRAFTER_BUILD_HAS_ASSET`, so `cfg.assets` only works in the self-hosted binary. Changing asset extensions means updating two maps in `Crafter.Build-Clang.cpp`: `assetOutput` and the transitive copy loop. - A warning is printed only when its unit actually recompiles. A build with nothing to do is silent, which is not evidence that a warning is gone. - A std module source only compiles against the headers of its own libc++ release, and a sysroot does not always ship one (Alpine). The Flatpak therefore carries the sources of the release next to the bundled one under `share/crafter-build/libc++-modules//`, one `libcxx--modules` manifest module each. When the bundled LLVM is bumped, keep the release it leaves behind as such a module: sysroots on the stable distro stay on it for months. The native loop has no such directory; stage one by hand under `share/crafter-build/` to test against a foreign sysroot. +- The Flatpak carries the debugging tools for what it builds, because those programs run in its sandbox: lldb is a project of the `llvm` module, with `LLDB_ENABLE_PYTHON`, `LIBEDIT`, `CURSES`, `LIBXML2` and `LZMA` set ON so that a missing dependency fails the configure step (SWIG through Python, which needs it), and the `swig` and `libedit` modules come before it. The Khronos validation layer is the `vulkan-validation-layers` module plus four modules before it, build-only except for SPIRV-Tools' programs (`spirv-val` and the rest), all after the LLVM and WASI modules, so that bumping them reuses the cached LLVM and WASI builds. The layer follows the Sdk's Vulkan loader: when `runtime-version` moves to a runtime with a newer loader, move the layer to the Vulkan SDK release of that loader and take the other four revisions from that release's `scripts/known_good.json`. +- The wasm compile flags live in one place, `WasmBaselineFlags` in `Crafter.Build-Clang.cppm`: the std module and every unit have to agree on them, and `VariantId()` and the std PCM directory hash them, so a change rebuilds instead of mixing old and new BMIs. They enable C++ exceptions, so the link takes the exception-enabled libc++ and libunwind from the sysroot's `lib//eh/` (wasi-sdk 33 and later). The Flatpak's sysroot has them and a distro's may not; where they are missing the wasm tests skip, so run those through the Flatpak. A browser page checks the engine with the `WasmBaselineProbes` named in `variants.json`. +- `WasiBrowserRuntimeExit` runs `wasi-runtime/runtime.js` under node and skips where there is none. The Flatpak has no node, so CI never runs it: after touching `runtime.js`, run the suite in the native loop with node on `PATH`. - `VariantId()` names the output directory, so CI globs `bin/crafter.build-exe---*/` rather than predicting the hash. The hash includes the compiler, so after a toolchain update the previous variant's directories stay in `bin/` and `build/` until `clean`, and such a glob matches both. +- AGENTS.md is build output: the self-host build and `test` write it from `AGENTS.template.md`, with the `AGENTS.consumer.md` of each dependency that ships one (Crafter.Math, today) merged in, and it is gitignored. Edit the template; an edit made in AGENTS.md is gone after the next build. A fresh clone or worktree has no AGENTS.md until its first `./bin/crafter-build` or `test`, since `./build.sh` does not write it. A harness from before the template (3.0.0 and older) never creates one and keeps its section in one it finds; the next build with a current harness writes it over. diff --git a/README.md b/README.md index cc1a0d4..cfd2bb1 100644 --- a/README.md +++ b/README.md @@ -4,15 +4,17 @@ A C++26 modules build system. Project descriptions are written in C++ with no JS ## Install -crafter-build ships as a Flatpak that carries its own clang, lld, libc++ and WASI sysroot, so every machine builds with the same compiler: +crafter-build ships as a Flatpak that carries its own clang, lld, libc++ and WASI sysroot, so every machine builds with the same compiler, and lldb and the Vulkan validation layers to debug the result (see [Debugging](#debugging)): ```bash flatpak remote-add --if-not-exists catcrafts https://flatpak.catcrafts.net/catcrafts.flatpakrepo flatpak install catcrafts net.catcrafts.CrafterBuild -alias crafter-build='flatpak run net.catcrafts.CrafterBuild' +alias crafter-build='flatpak run --die-with-parent net.catcrafts.CrafterBuild' ``` -Pre-release builds of `master` are the `beta` branch (`flatpak install catcrafts net.catcrafts.CrafterBuild//beta`), and every release also attaches a single-file bundle on the Forgejo releases page (`flatpak install --user crafter-build-.flatpak`). The sandbox sees the whole host filesystem, so projects can live anywhere. Binaries built for the host link the bundled `libc++.so.1` through an rpath into the Flatpak: they run inside it (`-r`, the test runner), and are not meant to be copied out onto the host. +`--die-with-parent` ends the sandbox with the `flatpak run` that started it. Without the flag, killing only the `flatpak run` process leaves the sandbox running: a build carries on writing into `bin/` and `build/`, and a program started with `-r` keeps running. With it, bubblewrap kills everything in the sandbox with SIGKILL when the `flatpak run` process dies or when that process's parent does. For `crafter-build …` or `crafter-build … &` typed at a bash or zsh prompt, that parent is the shell, so the sandbox ends with the shell, even after a `disown`. Ctrl-C reaches the program either way. + +Releases are published to the `stable` branch that the commands above install, and every release also attaches a single-file bundle on the Forgejo releases page (`flatpak install --user crafter-build-.flatpak`). The sandbox sees the whole host filesystem, so projects can live anywhere. Binaries built for the host link the bundled `libc++.so.1` through an rpath into the Flatpak: they run inside it (`-r`, the test runner), and are not meant to be copied out onto the host. ## Writing a project.cpp @@ -99,6 +101,55 @@ import Crafter.LintRules; Such a dependency is built for the host, whatever the project targets, and its `project.cpp` gets none of the project's args. +### Dependency notes for agents + +An agent working on a project reads its `AGENTS.md`, which says nothing about the libraries the project builds against unless someone copied that in by hand. A library ships it instead: an `AGENTS.consumer.md` beside its `project.cpp`, written for whoever uses the library rather than whoever works on it. + +A consumer that wants its dependencies' notes writes its own instructions in an `AGENTS.template.md` beside its `project.cpp`, and commits that instead of an `AGENTS.md`. `crafter-build` and `crafter-build test` then write `AGENTS.md` from the template with the notes added, so `AGENTS.md` is build output and stays out of version control: a dependency that changes its notes changes nothing a diff shows. A project with a committed `AGENTS.md` moves over from the directory of its `project.cpp`: + +```bash +git mv AGENTS.md AGENTS.template.md +echo /AGENTS.md >> .gitignore +``` + +What a build then writes: + +```markdown + +# The template's text + +... + + + + + +## Crafter.Math + +... + + +``` + +- Each file goes in as written, under a line naming the dependency: its repository's name, or a `LocalProject`'s directory name. That reads the same on every machine, and for a `--local` checkout and its git clone alike. The file lands among the consumer's own headings, so start it with a `##` heading that names the library. +- The section goes after the template's text and a blank line. To have it elsewhere, put the begin and end marker lines in the template where it should go: whatever they enclose is replaced. A template moved over from a committed `AGENTS.md` has them where its section was, around notes that can be deleted. +- `AGENTS.md` is written only when its text changes, in place, so a `CLAUDE.md` linked to it, its ownership and a read-only `AGENTS.md` are respected. An edit made in it is gone after the next build, and a fresh clone has none until its first build. The template is never written. +- An `AGENTS.md` that no build wrote, such as a committed one still next to a new template, is never replaced: each build warns instead, until it is deleted. Without a template, `AGENTS.md` is never touched. One that still holds a section from crafter-build 3.0.0, which kept the notes in a committed `AGENTS.md`, gets a warning: move it over as above, or delete the section. +- The update runs once `project.cpp` has loaded and before the project compiles, so a build that fails still has the notes. A `project.cpp` that fails to load gets no update. `test --list`, `lint`, `format`, `run` and `clean` leave the file alone. +- The section follows what the build loaded: a dependency that only some flags pull in (a `--target=`, a flag of the project's own) comes and goes with them. An unpinned dependency's notes follow its default branch, as its code does, so the section can change on any build. Pin a `commit` to decide when. +- A library reached through two specs, pinned at one revision here and another further down, is described once, as the nearer one has it. +- When there are notes to write, unbalanced markers in the template leave `AGENTS.md` as it is, with a warning. A notes file that would break the section (a code block it never closes, a marker line of its own) is left out, also with a warning. Markers quoted inside a fenced code block don't count. + +`cfg.dependencyNotes` picks the dependencies: + +| Value | Notes from | +|---|---| +| `DependencyNotes::Transitive` (default) | everything the project builds against, all the way down: its `dependencies` and its own `#pragma crafter dependency` tooling | +| `DependencyNotes::Direct` | only the projects the project's own configurations and `#pragma crafter dependency` lines name | +| `DependencyNotes::None` | none: `AGENTS.md` holds the template's text alone | + +Three things never count: the project's own `AGENTS.consumer.md`; its tests' dependencies, since a plain build never looks at the tests; and a dependency's `#pragma crafter dependency` tooling, which is about how the dependency is built, not how it is used. + ## Tests Tests live under `tests//`. The simplest case is a single C++ file: @@ -152,7 +203,17 @@ For elaborate setups (rsync, ssh ControlMaster, env vars on the spawn, `QEMU_LD_ For cross-arch builds, set `cfg.sysroot` on the `Configuration` to point at a sysroot for the target. `BuildStdPcm` and the compile/link commands both consume it: `--sysroot=` is passed to clang, and the std module is precompiled against the sysroot's libc++ headers. Ordinary compiles are pointed at the same headers (`-nostdinc++ -isystem /usr/include/c++/v1`), so a unit that `#include`s a standard header gets the sysroot's libc++ and not the host toolchain's. The module source has to come from the same libc++ release as those headers, so it is taken from `/usr/share/libc++/v1/std.cppm` when the sysroot ships it, else from the toolchain when both are the same major release, else from the sources crafter-build carries for other releases (`share/crafter-build/libc++-modules//`, `crafter-build --version` lists them). Alpine is the common case for the last two: its `libc++-dev` has the headers and no module source. When nothing matches, the build stops with one line naming both releases, installing that release's `share/libc++/v1` directory into the sysroot resolves it. The cheapest aarch64 sysroot is an extracted [Arch Linux ARM rootfs](http://os.archlinuxarm.org/) plus `libc++` + `libc++abi` + `gcc` installed into it via `pacman --root --config --dbpath /var/lib/pacman -Sy …` (point pacman at ALARM's mirrors via explicit config to avoid pulling host-arch packages). -For `--target=x86_64-w64-mingw32` (Windows from Linux), the pipeline cross-compiles as long as `mingw-w64` is installed, with libstdc++ 15 or newer for its std module. clang finds it by the `x86_64-w64-mingw32-gcc` on `PATH`; `CRAFTER_MINGW_DIR` names another prefix (the directory holding `x86_64-w64-mingw32/` and `lib/gcc/`), and inside the Flatpak the host's `/usr` is used when it has mingw-w64. Without one the build stops with a message saying where it looked. Pair it with `--runner=cmd:wine` to run the result. Only the MinGW ABI is supported, there is no MSVC target and no Windows host. `-fuse-ld=lld` is auto-injected on every link, and `-lstdc++exp` is auto-linked for the mingw target so user projects don't need to declare those flags. +For `--target=x86_64-w64-mingw32` (Windows from Linux), the pipeline cross-compiles as long as `mingw-w64` is installed, with libstdc++ 15 or newer for its std module. clang finds it by the `x86_64-w64-mingw32-gcc` on `PATH`; `CRAFTER_MINGW_DIR` names another prefix (the directory holding `x86_64-w64-mingw32/` and `lib/gcc/`), and inside the Flatpak the host's `/usr` is used when it has mingw-w64. Without one the build stops with a message saying where it looked. The prefix is clang's sysroot as it is, so Windows libraries installed into its `include/` and `lib/` are found, unless it also holds a Linux system: the host's `/usr` in the Flatpak, or any prefix whose `include/` has a C library's `features.h`, as `/usr` does. clang searches a sysroot's `include/` and `lib/`, and in such a prefix those hold glibc's headers and Linux libraries, so clang is shown only its `x86_64-w64-mingw32/`, `lib/gcc/` and `lib64/gcc/`, as links in `mingw-sysroot/` in the cache (`~/.cache/crafter.build/`, in the Flatpak `~/.var/app/net.catcrafts.CrafterBuild/cache/crafter.build/`, or `$XDG_RUNTIME_DIR/crafter.build/` when the cache's filesystem takes no symbolic links). Compiler messages still name those headers by their paths in the prefix. Pair it with `--runner=cmd:wine` to run the result. Only the MinGW ABI is supported, there is no MSVC target and no Windows host. `-fuse-ld=lld` is auto-injected on every link, and `-lstdc++exp` is auto-linked for the mingw target so user projects don't need to declare those flags. + +For `--target=wasm32-wasip1`, the WASI sysroot is the one beside the toolchain (the Flatpak bundles one) unless `CRAFTER_BUILD_WASI_SYSROOT` names another, and tests run under `wasmtime`. C++ exceptions work as on the other targets: every unit and the std module are compiled with `-fwasm-exceptions` in the standardized form of WebAssembly exception handling (exnref), against the exception-enabled libc++ in the sysroot's `lib//eh/`, and `-lunwind` is linked automatically. A sysroot without that libc++ (wasi-sdk before 33) stops the build with one line saying so. The result runs on engines with that proposal: Chrome 137, Firefox 131 and Safari 18.4 or newer, and the wasmtime the Flatpak bundles. A page from `EnableWasiBrowserRuntime` checks first and names what an older browser lacks in the console, instead of failing to compile the module. An exception that escapes `main` leaves the program as an uncaught WebAssembly exception (wasmtime reports `thrown Wasm exception` and exits 1, a page gets a `WebAssembly.Exception`) without calling `std::terminate`, so a `std::set_terminate` handler does not run and nothing prints the exception's type or `what()`: wasm exception handling unwinds in a single phase, and nothing finds out that no handler is left until the stack is gone. Catch it in `main` to report it. + +A wasm build refuses three things before it starts, each with a message that says why: `-flto` in `compileFlags` or `linkFlags` (LTO code generation drops the exception handling, [WebAssembly/wasi-sdk#629](https://github.com/WebAssembly/wasi-sdk/issues/629)), `-fno-exceptions` (the std module is built with exceptions, and a unit without them cannot import it) and a shared library (the exception-enabled libc++ is static archives only). `setjmp` and `longjmp` work in C and C++ sources, with `-lsetjmp` linked automatically like `-lunwind`; a `longjmp` across a `noexcept` function ends the program, because it is a wasm exception and the function's terminate handler catches it. Outside the Flatpak, a wasmtime older than 47 has exception handling off and needs `-W exceptions=y`, and a `wasm-opt` on `PATH` from a binaryen that cannot read the standardized encoding breaks release links, since clang runs it on optimized wasm links when it finds one. + +In a page from `EnableWasiBrowserRuntime`, `exit` and `_Exit` end the program as they would anywhere else: none of its code runs on the way out, no destructor and no `catch (...)`, its status is in `window.crafter_wasi.exitCode`, and bridges that call into it on their own are expected to stop. A program that hands the thread to the browser for good and goes on serving calls, as Crafter.Graphics's `Window::StartSync()` does, parks instead of exiting. Parking never returns, runs nothing of the program either, and leaves `main`'s locals valid for the calls that follow: + +```cpp +extern "C" [[noreturn]] __attribute__((import_module("crafter_runtime"), import_name("park"))) void Park(); +``` ### Configuring runners @@ -305,6 +366,32 @@ Things to know: - Inside the Flatpak, commands run in its sandbox: the host's files are visible, but the tools on `PATH` are the ones the Flatpak ships (`git`, `cmake`, `python3`, `tar`, ...), not everything installed on the host. - A return value outside 0 to 255 is not an exit status and becomes 1. So does a script that throws, with the exception's message printed. +## Debugging + +Programs that `-r` and the test runner start run inside the Flatpak's sandbox, and so do the tools to debug them: lldb, with `lldb-server`, `lldb-dap` and `lldb-mcp`, from the same LLVM build as the compiler; `gdb`, `gdbserver`, `valgrind` and `strace` from the Freedesktop Sdk; the Khronos Vulkan validation layer; and the SPIRV-Tools programs (`spirv-val`, `spirv-opt`, `spirv-dis`, ...). A default build is optimised (`-O3`, and ThinLTO except for wasm) and carries no debug information: set `cfg.debug = true` for a program you want to debug, which compiles its C++ units with `-g` and links it without LTO; its C and CUDA sources (`cfg.cFiles`, `cfg.cuda`) stay at `-O3` without debug information. A `project.cpp` that calls `ApplyStandardArgs` sets it from the `--debug` argument, as in `crafter-build --debug -r`. Start the tools with `--command`: + +```bash +flatpak run --die-with-parent --command=lldb net.catcrafts.CrafterBuild ./bin/myapp-/myapp +flatpak run --die-with-parent --command=valgrind net.catcrafts.CrafterBuild ./bin/myapp-/myapp +``` + +Every `flatpak run` is a sandbox of its own, with process IDs of its own, so a debugger in one cannot see a program that another one, such as `crafter-build -r`, started. Either start the program under the debugger, or run the debugger inside that program's sandbox with `flatpak enter`, giving the instance `flatpak ps` lists for net.catcrafts.CrafterBuild and the debugger's full path: + +```bash +flatpak ps --columns=instance,application +flatpak enter /app/bin/lldb -n myapp +``` + +Where `/proc/sys/kernel/yama/ptrace_scope` is 1, a debugger in the sandbox, where processes hold no capabilities, can attach to a program that is not its descendant only when the program allowed it with `prctl(PR_SET_PTRACER, …)`: with the debugger's process ID, or with `PR_SET_PTRACER_ANY` for any process. Otherwise the attach fails. A program started under the debugger is its descendant. Prefer `flatpak enter` to `lldb-server gdbserver` in one sandbox and lldb's `gdb-remote` from another: when a process in lldb's own sandbox has the remote program's process ID, lldb takes that process's executable for the program's: symbols come out wrong, and a breakpoint in the program's source finds no location. + +The validation layer is enabled the usual ways, by the program itself or from the environment: `flatpak run` passes `VK_INSTANCE_LAYERS`, `VK_LOADER_LAYERS_ENABLE` and the layer's `VK_KHRONOS_VALIDATION_*` settings into the sandbox. `VK_KHRONOS_VALIDATION_GPUAV_ENABLE=1` adds GPU-assisted validation (GPU-AV): + +```bash +VK_INSTANCE_LAYERS=VK_LAYER_KHRONOS_validation VK_KHRONOS_VALIDATION_GPUAV_ENABLE=1 crafter-build -r +``` + +The layer is the Vulkan SDK release of the runtime's own Vulkan loader. `VK_LAYER_PATH` and `VK_ADD_LAYER_PATH` are cleared on the way into the sandbox, so a layer installed on the host is out of reach. + ## Examples [examples/](examples/) contains progressively larger self-contained projects: diff --git a/build.sh b/build.sh index 312abde..dcb8696 100755 --- a/build.sh +++ b/build.sh @@ -36,10 +36,9 @@ else STD_CPPM=/usr/share/libc++/v1/std.cppm fi -# The version baked into the binary: the environment (CI sets it from the -# tag), else the nearest tag, else a development marker. `latest` is the -# rolling CI tag and never a version. -VERSION="${CRAFTER_BUILD_VERSION:-$(git describe --tags --always --dirty --exclude=latest 2>/dev/null || echo 0.0.0-dev)}" +# The version baked into the binary: the environment (CI sets it), else the +# nearest tag, else a development marker. +VERSION="${CRAFTER_BUILD_VERSION:-$(git describe --tags --always --dirty 2>/dev/null || echo 0.0.0-dev)}" mkdir -p build bin lib share/crafter-build share/crafter-build/wasi-runtime @@ -48,7 +47,7 @@ cp wasi-runtime/index.html.in share/crafter-build/wasi-runtime/ # Partition order is the import order: every unit is listed before the # units that import it, and the umbrella last. -PARTITIONS="Toolchain Shader Platform Interface Implementation External Clang Test Lint Script Progress Asset" +PARTITIONS="Toolchain Shader Platform Interface Implementation External Agents Clang Test Lint Script Progress Asset" cp interfaces/Crafter.Build.cppm share/crafter-build/ for p in $PARTITIONS; do diff --git a/examples/wasi/README.md b/examples/wasi/README.md index 7ac0f42..8a8db56 100644 --- a/examples/wasi/README.md +++ b/examples/wasi/README.md @@ -4,9 +4,14 @@ Build a C++26 program for the `wasm32-wasi` target and run it in a browser. ## Prerequisites -- `wasi-libc`, `wasi-libc++`, `wasi-libc++abi` (Arch packages — provide - `/usr/share/wasi-sysroot/`, which `crafter-build` finds automatically). - Other distros: install the WASI SDK and set `cfg.sysroot` in `project.cpp`. +- A WASI sysroot with the exception-enabled libc++, `lib/wasm32-wasip1/eh/` + from wasi-sdk 33 or later: wasm builds compile C++ exceptions and link those + libraries. The Flatpak bundles one, which `crafter-build` uses as it is. + Elsewhere, point `CRAFTER_BUILD_WASI_SYSROOT` at a wasi-sdk sysroot, or set + `cfg.sysroot` in `project.cpp`. Arch's `wasi-libc++` packages have no such + libraries, and the build stops with a message saying so. +- For the browser: WebAssembly exception handling, in Chrome 137, Firefox 131, + Safari 18.4 or newer. ## Build diff --git a/implementations/Crafter.Build-Agents.cpp b/implementations/Crafter.Build-Agents.cpp new file mode 100644 index 0000000..bddd41b --- /dev/null +++ b/implementations/Crafter.Build-Agents.cpp @@ -0,0 +1,591 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts® + +module; +#include +#include +#include +#include +module Crafter.Build:Agents_impl; +import std; +import :Agents; +import :Clang; +import :External; +import :Platform; +import :Progress; +namespace fs = std::filesystem; +using namespace Crafter; + +namespace { + constexpr std::string_view BeginMarker = ""; + constexpr std::string_view EndMarker = ""; + // Right under the begin marker, for whoever opens the file: an edit made + // inside the section is gone after the next build. + constexpr std::string_view Preamble = ""; + // The first line of every AGENTS.md a build writes. It tells whoever + // opens the file, an agent asked to change it included, where an edit + // has to be made to outlive the next build, and it is how a build tells + // its own AGENTS.md from one someone wrote, which it never replaces. A + // changed word would make every AGENTS.md written before look like + // someone's own. + constexpr std::string_view GeneratedLine = ""; + constexpr std::string_view NotesFileName = "AGENTS.consumer.md"; + constexpr std::string_view TemplateFileName = "AGENTS.template.md"; + constexpr std::string_view AgentsFileName = "AGENTS.md"; + + void Warn(std::string_view message) { + Progress::Diagnostic(std::format("warning: {}", message)); + } + + // The line ending `text` uses, so a CRLF file does not end up mixed. + std::string_view LineEnd(std::string_view text) { + return text.contains("\r\n") ? "\r\n" : "\n"; + } + + std::string ErrnoMessage() { + return std::error_code(errno, std::generic_category()).message(); + } + + // Closes on the way out of scope, which also drops an flock taken + // through it. + class Descriptor { + public: + explicit Descriptor(std::int32_t fd) : fd_(fd) {} + ~Descriptor() { + if (fd_ >= 0) ::close(fd_); + } + Descriptor(const Descriptor&) = delete; + Descriptor& operator=(const Descriptor&) = delete; + std::int32_t Get() const { return fd_; } + bool Valid() const { return fd_ >= 0; } + + private: + std::int32_t fd_; + }; + + // Everything `fd` reads, into `out`; empty on success, otherwise the + // error. Through read(2) rather than a stream: libc++'s filebuf reports a + // failed read as the end of the file, so an I/O error came back as a + // short read that looked successful, and the section was then written + // over a truncated AGENTS.md. + std::string ReadAll(std::int32_t fd, std::string& out) { + out.clear(); + std::array buffer; + for (;;) { + const auto got = ::read(fd, buffer.data(), buffer.size()); + if (got > 0) { + out.append(buffer.data(), static_cast(got)); + } else if (got == 0) { + return ""; + } else if (errno != EINTR) { + return ErrnoMessage(); + } + } + } + + std::string ReadFile(const fs::path& file, std::string& out) { + const Descriptor in(::open(file.c_str(), O_RDONLY | O_CLOEXEC)); + if (!in.Valid()) return ErrnoMessage(); + return ReadAll(in.Get(), out); + } + + struct Line { + std::string_view text; // without the newline and a '\r' before it + std::size_t begin = 0; // offset of the line's first byte + std::size_t next = 0; // offset of the line after it + }; + + std::vector SplitLines(std::string_view text) { + std::vector lines; + std::size_t pos = 0; + while (pos < text.size()) { + const std::size_t newline = text.find('\n', pos); + const std::size_t end = newline == std::string_view::npos ? text.size() : newline; + std::string_view line = text.substr(pos, end - pos); + if (line.ends_with('\r')) line.remove_suffix(1); + const std::size_t next = newline == std::string_view::npos ? text.size() : newline + 1; + lines.push_back({ .text = line, .begin = pos, .next = next }); + pos = next; + } + return lines; + } + + std::string_view TrimEnd(std::string_view text) { + const std::size_t last = text.find_last_not_of(" \t"); + return last == std::string_view::npos ? std::string_view() : text.substr(0, last + 1); + } + + // A line with the block quote markers in front of it taken off. + struct Shape { + std::size_t quotes = 0; // `>` markers + std::size_t indent = 0; // columns of indentation after them + std::string_view rest; // the line from its first other character + }; + + Shape ShapeOf(std::string_view line) { + Shape shape; + std::size_t pos = 0; + for (;;) { + std::size_t probe = pos; + while (probe < line.size() && probe - pos < 3 && line[probe] == ' ') ++probe; + if (probe >= line.size() || line[probe] != '>') break; + pos = probe + 1; + if (pos < line.size() && line[pos] == ' ') ++pos; + ++shape.quotes; + } + for (; pos < line.size() && (line[pos] == ' ' || line[pos] == '\t'); ++pos) { + shape.indent += line[pos] == '\t' ? 4 - shape.indent % 4 : 1; + } + shape.rest = line.substr(pos); + return shape; + } + + // The columns a list item marker at the start of `text` takes, with the + // spaces after it: `-`, `+` or `*`, or up to nine digits and `.` or `)`, + // then at least one space. Zero when `text` starts with none. + std::size_t ListMarker(std::string_view text) { + std::size_t pos = 0; + if (!text.empty() && (text[0] == '-' || text[0] == '+' || text[0] == '*')) { + pos = 1; + } else { + while (pos < text.size() && pos < 9 && text[pos] >= '0' && text[pos] <= '9') ++pos; + if (pos == 0 || pos >= text.size() || (text[pos] != '.' && text[pos] != ')')) return 0; + ++pos; + } + const std::size_t marker = pos; + while (pos < text.size() && text[pos] == ' ') ++pos; + return pos == marker ? 0 : pos; + } + + struct Fence { + char ch = '\0'; + std::size_t length = 0; + std::string_view after; // the rest of the line behind the run + }; + + // The run of three or more backticks or tildes `text` starts with. + std::optional FenceRun(std::string_view text) { + if (text.empty() || (text[0] != '`' && text[0] != '~')) return std::nullopt; + std::size_t length = 0; + while (length < text.size() && text[length] == text[0]) ++length; + if (length < 3) return std::nullopt; + return Fence{ .ch = text[0], .length = length, .after = text.substr(length) }; + } + + // Fenced code blocks, close to how CommonMark reads them. A run of three + // or more backticks or tildes opens one after at most three spaces of + // indentation; a backtick run only when no backtick follows it on the + // line, since ```make``` is inline code. A run of the same character at + // least as long, with nothing behind it but spaces, closes it. Keeping + // the character and the length is what lets a ~~~ block quote a ``` one. + // + // Block quotes and list items are followed as far as fences need: the + // `>` markers come off every line, list markers off the line that opens + // a block, and a block ends with the container it sits in, at a line with + // fewer `>` or one indented less than its list item's content. Read + // without them, `- ```bash` was missed while its indented closer was + // taken for an opener, and the rest of the file came out inside out. Not + // followed: indented code blocks, and the continuation lines of a list + // item before a fence that opens there. + class CodeFences { + public: + // Feed the lines in order. True for a fence and for every line + // inside a block. + bool InCode(std::string_view line) { + const Shape shape = ShapeOf(line); + if (open_ != '\0') { + const bool blank = TrimEnd(shape.rest).empty(); + const bool left = !blank && (shape.quotes < quotes_ || shape.indent < content_); + if (!left) { + const std::optional fence = FenceRun(shape.rest); + if (fence && fence->ch == open_ && fence->length >= length_ && TrimEnd(fence->after).empty() && shape.indent <= content_ + 3) open_ = '\0'; + return true; + } + // The container ended, and the block with it; the line is + // read again as one outside any block. + open_ = '\0'; + } + if (shape.indent > 3) return false; + std::size_t column = shape.indent; + std::string_view text = shape.rest; + std::size_t content = 0; + for (std::size_t marker = ListMarker(text); marker > 0; marker = ListMarker(text)) { + column += marker; + text.remove_prefix(marker); + content = column; + } + const std::optional fence = FenceRun(text); + if (!fence || (fence->ch == '`' && fence->after.contains('`'))) return false; + open_ = fence->ch; + length_ = fence->length; + quotes_ = shape.quotes; + content_ = content; + ++opened_; + return true; + } + bool Open() const { return open_ != '\0'; } + // How many blocks have opened so far. + std::size_t Opened() const { return opened_; } + + private: + char open_ = '\0'; + std::size_t length_ = 0; + std::size_t quotes_ = 0; // the block quote depth it opened at + std::size_t content_ = 0; // its list item's content column; 0 outside a list + std::size_t opened_ = 0; + }; + + // Where a text's marker lines are, outside code blocks, as indices into + // `lines`. + struct MarkerScan { + std::vector lines; + std::vector begins; + std::vector ends; + // The line a code block that never closes opens on. + std::optional unclosedBlock; + }; + + MarkerScan ScanMarkers(std::string_view text) { + MarkerScan scan; + scan.lines = SplitLines(text); + CodeFences fences; + for (std::size_t i = 0; i < scan.lines.size(); ++i) { + const std::size_t opened = fences.Opened(); + const bool code = fences.InCode(scan.lines[i].text); + if (fences.Opened() != opened) scan.unclosedBlock = i; + if (code) continue; + const std::string_view line = TrimEnd(scan.lines[i].text); + if (line == BeginMarker) scan.begins.push_back(i); + if (line == EndMarker) scan.ends.push_back(i); + } + if (!fences.Open()) scan.unclosedBlock.reset(); + return scan; + } + + // Why `text` cannot go into the section as it is; empty when it can. A + // marker line of its own would end or restart the section for whoever + // reads it, a template copied from the result included, and a code block + // left open would run on over the notes after it and the end marker. + // Lines are numbered from `firstLine`. + std::string NoteProblem(std::string_view text, std::size_t firstLine) { + const MarkerScan scan = ScanMarkers(text); + std::optional marker; + if (!scan.begins.empty()) marker = scan.begins[0]; + if (!scan.ends.empty()) marker = std::min(marker.value_or(scan.ends[0]), scan.ends[0]); + if (marker) return std::format("line {} is a dependency notes marker", *marker + firstLine); + if (scan.unclosedBlock) return std::format("line {} opens a code block that never closes", *scan.unclosedBlock + firstLine); + return ""; + } + + struct NormalizedNote { + std::string text; + // The line of the file `text` starts on. + std::size_t firstLine = 1; + }; + + // A notes file as it goes into the section: LF line ends, no blank lines + // in front and no whitespace behind, so the section spaces its notes the + // same whatever the files end with. + NormalizedNote NormalizeNote(std::string_view raw) { + std::string text; + text.reserve(raw.size()); + for (std::size_t i = 0; i < raw.size(); ++i) { + if (raw[i] == '\r' && i + 1 < raw.size() && raw[i + 1] == '\n') continue; + text.push_back(raw[i]); + } + const std::size_t last = text.find_last_not_of(" \t\r\n"); + if (last == std::string::npos) return {}; + const std::size_t first = text.find_first_not_of(" \t\r\n"); + const std::size_t newline = text.rfind('\n', first); + const std::size_t begin = newline == std::string::npos ? 0 : newline + 1; + const auto skipped = std::count(text.begin(), text.begin() + static_cast(begin), '\n'); + return { .text = text.substr(begin, last + 1 - begin), .firstLine = static_cast(skipped) + 1 }; + } + + // `text` with `eol` ending its lines, whether they ended in LF or CRLF. + std::string WithLineEnds(std::string_view text, std::string_view eol) { + std::string out; + out.reserve(text.size()); + for (std::size_t i = 0; i < text.size(); ++i) { + if (text[i] == '\r' && i + 1 < text.size() && text[i + 1] == '\n') continue; + if (text[i] == '\n') { + out += eol; + } else { + out.push_back(text[i]); + } + } + return out; + } + + std::string RenderSection(std::span notes, std::string_view eol) { + std::string section = std::format("{}{}{}{}", BeginMarker, eol, Preamble, eol); + for (const DependencyNote& note : notes) section += std::format("{0}{0}{2}{0}", eol, note.name, WithLineEnds(note.text, eol)); + section += std::format("{0}{1}{0}", eol, EndMarker); + return section; + } + + // The note beside a dependency's project file, or nullopt when there is + // none or it cannot be used. + std::optional ReadNote(const fs::path& file, const std::string& name) { + std::error_code ec; + if (!fs::is_regular_file(file, ec)) return std::nullopt; + std::string content; + if (const std::string error = ReadFile(file, content); !error.empty()) { + Warn(std::format("{} could not be read ({}), so its notes are left out", file.string(), error)); + return std::nullopt; + } + NormalizedNote note = NormalizeNote(content); + if (note.text.empty()) return std::nullopt; + // On the very bytes the merge checks, so the two can never disagree, + // and numbered as the author's file is. + if (const std::string problem = NoteProblem(note.text, note.firstLine); !problem.empty()) { + Warn(std::format("{}: {}, so its notes are left out", file.string(), problem)); + return std::nullopt; + } + return DependencyNote{ .name = name, .text = std::move(note.text) }; + } + + // The loaded projects `configs` lead to without passing through another + // loaded project, in the order the configurations name them: one ring + // further out. `walked` spans the whole collection, so each + // Configuration is looked at once however many paths reach it. + std::vector NextRing(std::span configs, std::unordered_set& walked) { + std::vector ring; + std::vector pending(configs.rbegin(), configs.rend()); + while (!pending.empty()) { + const Configuration* cfg = pending.back(); + pending.pop_back(); + if (!cfg || !walked.insert(cfg).second) continue; + if (LoadedProjectOf(*cfg)) { + ring.push_back(cfg); + continue; + } + for (auto dep = cfg->dependencies.rbegin(); dep != cfg->dependencies.rend(); ++dep) pending.push_back(*dep); + } + return ring; + } + + // The AGENTS.consumer.md of every project `project` depends on, as + // project.dependencyNotes selects them, ring by ring: first the loaded + // projects (the ones GitProject and LocalProject returned) that its + // `#pragma crafter dependency` lines and its configurations name, in + // that order, then the ones those depend on. Its own configurations, and + // a dependency's internal ones, are only walked through. Direct stops + // after the first ring. + // + // Never walked: the project's tests, since a plain build never touches + // them and a test's dependency can point at the Configuration the project + // file returned by value, which has since moved; and a dependency's own + // pragma tooling, which is about how the dependency is built, not how it + // is used, and is no `dependencies` edge to begin with. + std::vector CollectNotes(const Configuration& project, const fs::path& projectFile, const fs::path& projectDir) { + std::unordered_set walked = { &project }; + // The project's own notes are for its consumers, so they never go + // into its own AGENTS.md, whatever dependency they sit beside. Both + // places count for a symlinked project file. + std::unordered_set files = { projectDir / NotesFileName }; + std::error_code ec; + if (const fs::path canonical = fs::canonical(projectFile, ec); !ec) files.insert(canonical.parent_path() / NotesFileName); + std::unordered_set identities; + std::vector notes; + + std::vector start; + for (const Configuration* dep : PragmaDependencies(projectFile)) start.push_back(dep); + start.insert(start.end(), project.dependencies.begin(), project.dependencies.end()); + for (std::vector ring = NextRing(start, walked); !ring.empty();) { + std::vector next; + for (const Configuration* cfg : ring) { + const std::optional loaded = LoadedProjectOf(*cfg); + const fs::path file = loaded->projectFile.parent_path() / NotesFileName; + // One library under two specs (a pin here, another revision + // further down) is described once, as the ring nearest the + // project has it. + if (identities.insert(loaded->identity).second && files.insert(file).second) { + if (std::optional note = ReadNote(file, loaded->name)) notes.push_back(std::move(*note)); + } + next.insert(next.end(), cfg->dependencies.begin(), cfg->dependencies.end()); + } + if (project.dependencyNotes == DependencyNotes::Direct) break; + ring = NextRing(next, walked); + } + return notes; + } + + // Whether a build may replace `agentsMd`, the AGENTS.md beside a + // template: one a build wrote, or one with no text at all, which is what + // a write cut short leaves of a new file. Anything else someone wrote, + // and beside a template that is easily the case: an AGENTS.md committed + // before the template was, or one an agent kept its own notes in. + bool Replaceable(std::string_view agentsMd) { + if (agentsMd.find_first_not_of(" \t\r\n") == std::string_view::npos) return true; + std::string_view first = agentsMd.substr(0, agentsMd.find('\n')); + if (first.ends_with('\r')) first.remove_suffix(1); + return first == GeneratedLine; + } + + // An AGENTS.md with no template beside it is someone's own, and no build + // writes it. One that still holds a section from the builds that kept the + // notes in a committed AGENTS.md would go stale without a word, under a + // preamble saying every build rewrites it, so each build says so until + // the file has moved to a template or the section is gone. + void WarnStaleSection(const fs::path& agents) { + std::error_code ec; + if (!fs::is_regular_file(agents, ec)) return; + std::string text; + if (!ReadFile(agents, text).empty()) return; + const MarkerScan scan = ScanMarkers(text); + if (scan.begins.empty() && scan.ends.empty()) return; + Warn(std::format("{} has a dependency notes section, which builds no longer update; rename the file to {} and keep {} out of version control to have every build write it with current notes, or delete the section", agents.string(), TemplateFileName, AgentsFileName)); + } + + // Bring `fd`, which holds `current`, to `wanted`: from the first byte the + // two differ at, so the text in front of the section is never rewritten, + // then cut to length and synced. Empty on success, otherwise the error. + // + // In place rather than through a temporary renamed over the file: a + // rename swaps in another file, which drops whatever belongs to the old + // one (a hard link such as a CLAUDE.md beside it, ACLs, ownership, being + // read-only) and needs fsyncs to be crash safe at all. The price is that + // a reader can catch the file half written, and that a write failing + // partway, on a full disk say, leaves it so until the next build writes + // it again. + std::string WriteChanged(std::int32_t fd, std::string_view current, std::string_view wanted) { + const auto diverge = std::ranges::mismatch(current, wanted); + for (auto done = static_cast(diverge.in2 - wanted.begin()); done < wanted.size();) { + const auto wrote = ::pwrite(fd, wanted.data() + done, wanted.size() - done, static_cast(done)); + if (wrote > 0) { + done += static_cast(wrote); + } else if (wrote == 0) { + return "the write made no progress"; + } else if (errno != EINTR) { + return ErrnoMessage(); + } + } + if (::ftruncate(fd, static_cast(wanted.size())) != 0) return ErrnoMessage(); + if (::fsync(fd) != 0) return ErrnoMessage(); + return ""; + } +} + +std::string Crafter::MergeDependencyNotes(std::string_view templateMd, std::span notes) { + for (const DependencyNote& note : notes) { + if (const std::string problem = NoteProblem(note.text, 1); !problem.empty()) throw std::runtime_error(std::format("the notes from {}: {}", note.name, problem)); + } + + const MarkerScan scan = ScanMarkers(templateMd); + const bool section = scan.begins.size() == 1 && scan.ends.size() == 1 && scan.begins[0] < scan.ends[0]; + const std::string_view eol = LineEnd(templateMd); + + if (notes.empty()) { + // With nothing to place, nothing can land in the wrong place, so + // anything short of a well-formed section is left as it was written. + if (!section) return std::string(templateMd); + std::string_view before = templateMd.substr(0, scan.lines[scan.begins[0]].begin); + const std::string_view after = templateMd.substr(scan.lines[scan.ends[0]].next); + // A section that ends the template takes the blank line in front of + // it, which an appended one gets. One placed elsewhere keeps the text + // around it apart. + if (after.empty() && before.ends_with(std::format("{0}{0}", eol))) before.remove_suffix(eol.size()); + return std::format("{}{}", before, after); + } + + if (scan.begins.size() > 1) throw std::runtime_error(std::format("line {}: a second dependency notes begin marker (the first is on line {})", scan.begins[1] + 1, scan.begins[0] + 1)); + if (scan.ends.size() > 1) throw std::runtime_error(std::format("line {}: a second dependency notes end marker (the first is on line {})", scan.ends[1] + 1, scan.ends[0] + 1)); + if (!scan.ends.empty() && (scan.begins.empty() || scan.ends[0] < scan.begins[0])) throw std::runtime_error(std::format("line {}: a dependency notes end marker with no begin marker before it", scan.ends[0] + 1)); + if (!scan.begins.empty() && scan.ends.empty()) throw std::runtime_error(std::format("line {}: a dependency notes begin marker with no end marker after it", scan.begins[0] + 1)); + + const std::string rendered = RenderSection(notes, eol); + if (section) return std::format("{}{}{}", templateMd.substr(0, scan.lines[scan.begins[0]].begin), rendered, templateMd.substr(scan.lines[scan.ends[0]].next)); + // Appended, the section would sit inside that block and read as code. + if (scan.unclosedBlock) throw std::runtime_error(std::format("line {}: a code block opens here and never closes", *scan.unclosedBlock + 1)); + if (templateMd.empty()) return rendered; + // On a line of its own, after a blank one. + return std::format("{}{}{}{}", templateMd, templateMd.ends_with('\n') ? std::string_view() : eol, eol, rendered); +} + +bool Crafter::UpdateAgentsMd(const Configuration& project, const fs::path& projectFile) { + std::error_code ec; + // Beside the project file as it was named, so a symlinked project.cpp + // keeps the AGENTS.md beside the link. + const fs::path projectDir = fs::absolute(projectFile, ec).lexically_normal().parent_path(); + if (ec) return false; + const fs::path agents = projectDir / AgentsFileName; + const fs::path templateFile = projectDir / TemplateFileName; + if (!fs::is_regular_file(templateFile, ec)) { + WarnStaleSection(agents); + return false; + } + // Linked to the template, AGENTS.md would have every build write the + // file that is committed, which is what the template is there to stop. + if (fs::equivalent(agents, templateFile, ec)) { + Warn(std::format("{} and {} are the same file, so {} was not written", agents.string(), templateFile.string(), AgentsFileName)); + return false; + } + std::string templateMd; + if (const std::string error = ReadFile(templateFile, templateMd); !error.empty()) { + Warn(std::format("{} could not be read ({}), so {} was not written", templateFile.string(), error, agents.string())); + return false; + } + + const std::vector notes = project.dependencyNotes == DependencyNotes::None ? std::vector() : CollectNotes(project, projectFile, projectDir); + std::string wanted; + try { + wanted = std::format("{}{}{}", GeneratedLine, LineEnd(templateMd), MergeDependencyNotes(templateMd, notes)); + } catch (const std::exception& e) { + Warn(std::format("{}: {}, so {} was not written", templateFile.string(), e.what(), agents.string())); + return false; + } + + // Created when missing, since a fresh clone has no build output, and only + // now, with the text in hand, so a template the notes cannot be placed + // in leaves no empty AGENTS.md behind. Locked from the read through the + // write, so two builds never interleave their writes, and on the file + // itself, which also covers two projects that link one AGENTS.md. A + // read-only descriptor takes the lock as well, so a read-only AGENTS.md + // is still read without complaint. + const Descriptor reading(::open(agents.c_str(), O_RDONLY | O_CREAT | O_CLOEXEC, 0666)); + if (!reading.Valid()) { + Warn(std::format("{} could not be opened ({}), so it was not written", agents.string(), ErrnoMessage())); + return false; + } + while (::flock(reading.Get(), LOCK_EX) != 0 && errno == EINTR) {} + std::string current; + if (const std::string error = ReadAll(reading.Get(), current); !error.empty()) { + Warn(std::format("{} could not be read ({}), so it was not written", agents.string(), error)); + return false; + } + if (current == wanted) return false; + if (!Replaceable(current)) { + Warn(std::format("{} was not written by a build, so it was not replaced by one written from {}; once the template holds what it says, delete it", agents.string(), TemplateFileName)); + return false; + } + + // Only now, with something to write: a read-only AGENTS.md is refused + // here, and not on every build that leaves it as it is. + const Descriptor writing(::open(agents.c_str(), O_WRONLY | O_CLOEXEC)); + if (!writing.Valid()) { + Warn(std::format("{} could not be opened for writing ({}), so it was not written", agents.string(), ErrnoMessage())); + return false; + } + if (const std::string error = WriteChanged(writing.Get(), current, wanted); !error.empty()) { + // Put back what was there. A full disk that refused the new bytes + // usually still takes the old ones, whose space was just freed. + if (WriteChanged(writing.Get(), wanted, current).empty()) { + Warn(std::format("{} could not be written ({}), so it was left as it was", agents.string(), error)); + } else { + Warn(std::format("{} could not be written ({}) and may now be half written; the next build writes it again", agents.string(), error)); + } + return false; + } + // A file that appears beside project.cpp, or changes there, unasked, + // says where it came from. + if (Progress::GetVerbosity() != Progress::Verbosity::Quiet) { + Progress::Clear(); + if (notes.empty()) { + std::println("Wrote {} from {}", agents.string(), TemplateFileName); + } else { + std::println("Wrote {} from {} and the notes of {} {}", agents.string(), TemplateFileName, notes.size(), notes.size() == 1 ? "dependency" : "dependencies"); + } + } + return true; +} diff --git a/implementations/Crafter.Build-Clang.cpp b/implementations/Crafter.Build-Clang.cpp index 2d1b47c..a302ca9 100644 --- a/implementations/Crafter.Build-Clang.cpp +++ b/implementations/Crafter.Build-Clang.cpp @@ -13,6 +13,7 @@ import :Toolchain; import :Test; import :Lint; import :Script; +import :Agents; import :Progress; import :Asset; namespace fs = std::filesystem; @@ -246,20 +247,57 @@ void Configuration::GetInterfacesAndImplementations(std::span interfac } } -fs::path Crafter::StdPcmDir(const Configuration& config) { - // The std PCM is cached per target+march+toolchain: a PCM is only valid - // for the exact clang that wrote it, so a compiler bump starts a cold - // directory instead of handing the new clang a PCM it will crash on. A - // wasm variant pass compiles it with extra codegen flags (e.g. - // -mrelaxed-simd) — its BMI must not be shared with the baseline's, or - // the consuming TUs see a target-feature mismatch. Suffix the cache dir - // with the variant flags when present. - std::string stdPcmKey = std::format("{}-{}-{}", config.target, config.march, GetToolchain().Key()); - for (const std::string& f : config.wasmVariantFlags) { - stdPcmKey += "+"; - for (char c : f) stdPcmKey += (c == '/' || c == '\\') ? '_' : c; +std::string Crafter::StdPcmFlags(const Configuration& config) { + if (config.target == "x86_64-w64-mingw32") { + // -femulated-tls keeps PCM TLS access matching libstdc++'s emutls + // definitions; a mismatch surfaces as undefined std::__once_callable. + return std::format(" --target={} -march={} -mtune={} -femulated-tls -O3 -std=c++26 -Wno-reserved-identifier -Wno-reserved-module-identifier", config.target, config.march, config.mtune); } - return GetCacheDir()/stdPcmKey; + const bool isWasm = config.target.starts_with("wasm32"); + const std::string sysrootFlag = config.sysroot.empty() ? std::string() : std::format(" --sysroot={}", config.sysroot); + // wasm32 rejects -march, and the std module has to be compiled the way + // the units importing it are, a variant pass's flags (e.g. + // -mrelaxed-simd) included, or the BMI is not feature-compatible. + std::string archFlags = isWasm ? WasmCompileFlags(config) : std::format(" -march={} -mtune={}", config.march, config.mtune); + // non-wasm cross builds: the driver ranks the host toolchain's own + // libc++ headers (/../include/c++/v1) above --sysroot, so the + // PCM would mix host libc++ with the foreign libc. Force the sysroot's + // copy. + if (!isWasm && !config.sysroot.empty()) { + archFlags += std::format(" -nostdinc++ -isystem {}/usr/include/c++/v1", config.sysroot); + } + return std::format(" --target={} -std=c++26{}{} -O3 -Wno-reserved-identifier -Wno-reserved-module-identifier", config.target, sysrootFlag, archFlags); +} + +std::string Crafter::StdPcmKey(const Configuration& config, const Toolchain& toolchain) { + // A PCM is only valid for the clang that wrote it and the flags it was + // written with, so a compiler bump starts a cold directory instead of + // handing the new clang a PCM it will crash on, and so does any flag. + // With only some of them in the name, configurations that differed in + // the rest took turns rebuilding one std PCM in place: -mtune=generic + // against project.cpp's -mtune=native on every run, two sysroots, or two + // releases sharing a cache, as the Flatpak's branches do, each time under + // the PCMs built against the previous one, which clang crashes on rather + // than rejecting. + std::string compiled = StdPcmFlags(config); + // The base command's part: the toolchain's key covers the compiler and + // libc++, and for the mingw target what clang finds mingw-w64 through is + // the prefix, given whole or through a view (MingwSysroot), named here by + // the prefix rather than by the view's own path. + if (config.target == "x86_64-w64-mingw32" && !toolchain.mingwPrefix.empty()) { + compiled += std::format("|mingw:{}|{}", toolchain.mingwPrefix.string(), toolchain.mingwPrefixIsSystem ? "view" : "whole"); + } + return std::format("{}-{}-{}-{:08x}", config.target, config.march, toolchain.Key(), static_cast(std::hash{}(compiled))); +} + +fs::path Crafter::StdPcmDir(const Configuration& config) { + return GetCacheDir() / StdPcmKey(config, GetToolchain()); +} + +std::string Crafter::WasmCompileFlags(const Configuration& config) { + std::string flags = std::format(" {}", WasmBaselineFlags); + for (const std::string& f : config.wasmVariantFlags) flags += std::format(" {}", f); + return flags; } fs::path Crafter::ExpectedOutput(const Configuration& config) { @@ -316,15 +354,11 @@ CompileCommand Crafter::GetCompileCommand(const Configuration& config) { if (isWasm) { // -mllvm is consumed by codegen but not the link driver, which is the // same command line; quiet the unused-flag warning rather than split - // compile and link commands. - out.command += " -fno-exceptions -msimd128 -fno-c++-static-destructors -mllvm -wasm-enable-sjlj -D_WASI_EMULATED_SIGNAL -Wno-unused-command-line-argument"; - // Active variant pass (see the variant driver at the end of Build): - // extra codegen flags (e.g. -mrelaxed-simd) applied to every TU. Empty - // for the baseline build. Part of VariantId, so these objects/PCMs - // land in their own dir. - for (const std::string& f : config.wasmVariantFlags) { - out.command += std::format(" {}", f); - } + // compile and link commands. An active variant pass (see the variant + // driver at the end of Build) adds its codegen flags (e.g. + // -mrelaxed-simd) to every TU; they are part of VariantId, so those + // objects/PCMs land in their own dir. + out.command += std::format("{} -Wno-unused-command-line-argument", WasmCompileFlags(config)); } if (config.target == "x86_64-w64-mingw32") { // mingw libstdc++ defines TLS via __emutls_v.* (emulated TLS); without @@ -404,12 +438,14 @@ CompileCommand Crafter::GetCompileCommand(const Configuration& config) { // (cross-TU inlining) plus dead-section GC and safe identical-code folding. // Default-on in Release, never Debug (keeps builds fast and debuggable). // Excluded for wasm32 — its -mllvm/sjlj codegen flags don't compose with - // LTO here — and for nvcc .cu objects below (they can't emit LLVM bitcode, - // so they link in as ordinary objects alongside the bitcode ones). ThinLTO - // rather than monolithic -flto so link time and memory scale with arbitrary - // user project sizes. Compile-side flags (-flto, -ffunction/-fdata-sections) - // ride on `command`, which is reused as the link base; the link-only -Wl - // flags go on linkExtras so they don't warn on each -c compile. + // LTO here, and wasi-sdk lists exceptions under LTO as broken + // (WebAssembly/wasi-sdk#629) — and for nvcc .cu objects below (they can't + // emit LLVM bitcode, so they link in as ordinary objects alongside the + // bitcode ones). ThinLTO rather than monolithic -flto so link time and + // memory scale with arbitrary user project sizes. Compile-side flags + // (-flto, -ffunction/-fdata-sections) ride on `command`, which is reused + // as the link base; the link-only -Wl flags go on linkExtras so they + // don't warn on each -c compile. out.useLto = !config.debug && !isWasm; out.ltoCompileFlags = out.useLto ? " -flto=thin -ffunction-sections -fdata-sections" : ""; out.ltoLinkFlags = out.useLto ? " -flto=thin -Wl,--gc-sections -Wl,--icf=safe" : ""; @@ -479,6 +515,54 @@ namespace { walk(config, config.name); return error; } + + // What a wasm32 configuration cannot be built with now that every unit + // has C++ exceptions, and what its sysroot has to have for that; empty + // for any other target and when nothing is wrong. Each would otherwise + // fail at the first compile or link with a message that does not say + // why, or not fail at all. + std::string WasmBuildError(const Configuration& config) { + if (!config.target.starts_with("wasm32")) return ""; + for (const std::vector* flags : { &config.compileFlags, &config.linkFlags }) { + for (const std::string& flag : *flags) { + // The compile has the backend options that keep exception + // handling; LTO code generation at the link does not, so it + // drops every landing pad and the build succeeds with throws + // that escape their catch blocks (WebAssembly/wasi-sdk#629). + if (flag.starts_with("-flto")) { + return std::format("{}: {} is not supported on wasm32: LTO code generation drops C++ exception handling (WebAssembly/wasi-sdk#629), so every throw would escape its catch", config.name, flag); + } + // The std module is built with exceptions, and a unit built + // without them cannot import it. + if (flag == "-fno-exceptions" || flag == "-fno-cxx-exceptions") { + return std::format("{}: {} is not supported on wasm32: wasm builds have C++ exceptions, the std module included, and a unit compiled without them cannot import std", config.name, flag); + } + } + } + // wasi-sdk's exception support does not cover shared libraries: the + // libc++ under lib//eh is static archives built without -fPIC. + if (config.type == ConfigurationType::LibraryDynamic) { + return std::format("{}: wasm32 shared libraries are not supported: the exception-enabled libc++ they would link has static archives only", config.name); + } + const fs::path sysroot = config.sysroot; + if (!fs::is_directory(sysroot)) { + return std::format("no WASI sysroot at {}: the Flatpak bundles one, and CRAFTER_BUILD_WASI_SYSROOT selects another", sysroot.string()); + } + const fs::path libDir = sysroot / "lib" / NormalizeTarget(config.target); + if (!fs::is_directory(libDir)) { + return std::format("the WASI sysroot at {} has nothing for {} (no {})", sysroot.string(), config.target, libDir.string()); + } + // Only a link takes the exception-enabled libraries: a static + // library's objects compile against the headers alone. The driver + // looks in eh/ first and then in the triple's own directory, where a + // sysroot built with exceptions only (WASI_SDK_EXCEPTIONS=ON) has them. + // Without them the first link fails with "unable to find library + // -lunwind", which says nothing about why it was asked for. + if (config.type == ConfigurationType::Executable && !fs::exists(libDir / "eh" / "libunwind.a") && !fs::exists(libDir / "libunwind.a")) { + return std::format("the WASI sysroot at {} has no C++ exception support (no {}): wasm builds use WebAssembly exception handling and need the exception-enabled libc++ and libunwind that wasi-sdk 33 and later ship. The Flatpak bundles such a sysroot; CRAFTER_BUILD_WASI_SYSROOT selects another", sysroot.string(), (libDir / "eh" / "libunwind.a").string()); + } + return ""; + } } BuildResult Crafter::Build(Configuration& config, std::unordered_map>& depResults, std::mutex& depMutex) { @@ -539,6 +623,11 @@ BuildResult Crafter::Build(Configuration& config, std::unordered_map.wasm (produced by Build's - // variant driver); the baseline maps to outputName.wasm. Even with no - // declared variants we still emit a one-entry manifest so the runtime has a + // as the fallback. Every entry carries WasmBaselineProbes besides its own: + // every file is built with the baseline flags, so an engine without + // exception handling can run none of them, and runtime.js tells it so + // instead of failing to compile the module. A variant with a non-empty + // label maps to outputName.