Crafter.Build/.forgejo/workflows/ci.yaml

name: Flatpak

# One job, two outcomes:
#   pull request, manual run -> build the Flatpak and run the suite inside it
#   pushed tag X.Y.Z         -> also release the build: publish it as the
#                               `stable` branch of the Catcrafts repo and
#                               create release X.Y.Z
#
# A release is a tag that the owner makes and signs locally and pushes with
# the master it releases; a push to master without one runs nothing. The tag
# names the version, and the metainfo's newest <release> entry has to be that
# version: its description is the release notes.
#
# The `crafter-flatpak` runner label is the privileged runner on the Hetzner
# box (bubblewrap needs it) with four volumes allowed: the Sdk install, the
# flatpak-builder state (ccache and the LLVM module, so only crafter-build
# itself rebuilds on a normal run), the published repo's web root, and the
# repo signing key, which lives on the runner host and never in the forge.
# No other runner carries that label. Plain `flatpak` is the label of the
# organisation's runner, which serves every Catcrafts repository and also
# mounts the web root and the key, for Crafter.Review's publish job, so any
# job it runs can mount them (AGENTS.template.md, CI and publishing).
on:
  pull_request:
    branches: [master]
  push:
    # Bare semver tags. Spelled with globs so it means the same whether the
    # forge reads it as a workflow pattern or as a plain glob.
    tags: ['[0-9]*.[0-9]*.[0-9]*']
  workflow_dispatch:

# Pushes are tags only, so every release run is in one group, and a release
# starts only once the one before it has finished. A pull request's group
# also holds its number, which only a pull request's event carries, so a
# new push to it cancels its own outdated run and no other.
concurrency:
  group: flatpak-${{ github.event_name }}-${{ github.event.pull_request.number }}
  cancel-in-progress: ${{ github.event_name == 'pull_request' }}

env:
  APP_ID: net.catcrafts.CrafterBuild
  BRANCH: stable
  MANIFEST: packaging/flatpak/net.catcrafts.CrafterBuild.yaml
  CI_MANIFEST: packaging/flatpak/ci-manifest.yaml
  RUNTIME: org.freedesktop.Sdk//26.08
  REPO_DIR: /srv/flatpak/repo
  WEB_DIR: /srv/flatpak
  # A release tag carries a signature Forgejo verifies for this address.
  RELEASE_SIGNER: jorijnvdgraaf@catcrafts.net

jobs:
  flatpak:
    runs-on: crafter-flatpak
    container:
      image: archlinux:latest
      options: >-
        -v crafter-flatpak-system:/var/lib/flatpak
        -v crafter-flatpak-builder:/root/.cache/flatpak-builder
        -v /var/www/flatpak:/srv/flatpak
        -v /etc/crafter-flatpak:/run/crafter-flatpak:ro
    steps:
      - name: Install tools
        # util-linux: flock, for the Publish step.
        run: |
          pacman-key --init
          pacman-key --populate archlinux
          pacman -Sy --noconfirm --needed archlinux-keyring
          pacman -Syu --noconfirm --needed flatpak flatpak-builder ccache git nodejs jq gnupg go-yq ostree python util-linux
          git config --global --add safe.directory '*'

      - name: Sdk
        # A no-op once the volume is warm; `update` follows point releases.
        run: |
          flatpak remote-add --if-not-exists flathub https://dl.flathub.org/repo/flathub.flatpakrepo
          flatpak install -y --noninteractive flathub "$RUNTIME"
          flatpak update -y --noninteractive

      - name: Checkout
        # Every branch and tag, for git describe and the release checks; no
        # step pushes, so the token is not kept in the checkout.
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          persist-credentials: false

      - name: Version
        id: meta
        run: |
          set -eu
          # The metainfo's newest entry is its first <release> element, the
          # one parse that gives both its version and the release notes,
          # before anything is built: a comment or another attribute order
          # would mislead a text search, and an entry whose notes render
          # empty stops the run here. The notes go to release-notes.md, for
          # the release page, as Markdown. AppStream has no headings, so a
          # paragraph that introduces a list is one: <p>New</p><ul>...</ul>
          # comes out as "## New" over its items, the shape the notes take
          # since 3.0.0. A paragraph on its own stays a paragraph, as the
          # older entries are written. <em> and <code> get their Markdown
          # spelling; a code span also keeps text such as lib/<triple>/eh
          # from being read as an HTML tag by the forge.
          newest=$(python3 - <<'EOF'
          import sys
          import xml.etree.ElementTree as ET

          release = ET.parse("packaging/flatpak/net.catcrafts.CrafterBuild.metainfo.xml").getroot().find("releases/release")
          desc = None if release is None else release.find("description")

          def text(el):
              parts = [el.text or ""]
              for child in el:
                  inner = text(child)
                  parts.append({"em": f"*{inner}*", "code": f"`{inner}`"}.get(child.tag, inner))
                  parts.append(child.tail or "")
              return " ".join("".join(parts).split())

          blocks = [] if desc is None else list(desc)
          out = []
          for i, el in enumerate(blocks):
              heads_list = i + 1 < len(blocks) and blocks[i + 1].tag in ("ul", "ol")
              if el.tag == "p":
                  out.append(f"## {text(el)}" if heads_list else text(el))
              elif el.tag == "ul":
                  out.append("\n".join(f"- {text(li)}" for li in el))
              elif el.tag == "ol":
                  out.append("\n".join(f"{n}. {text(li)}" for n, li in enumerate(el, 1)))
          notes = "\n\n".join(out)
          if not notes.strip():
              sys.exit("the metainfo needs a <release> at the top of <releases> whose <description> renders release notes")
          with open("release-notes.md", "w") as f:
              print(notes, file=f)
          print(release.get("version", ""))
          EOF
          )
          if [ "$GITHUB_EVENT_NAME" = push ]; then
            # A release: the only pushes this workflow runs for are tags.
            # The tag is the version; it has to be annotated, signed by the
            # owner as Forgejo verifies it against the keys its users added,
            # on master, newer than every published release, and the version
            # of the metainfo's newest entry. The tag itself is read from the
            # forge: the checkout fetches a pushed tag again as a plain ref
            # to its commit, so the local one is never the annotated tag.
            version=$GITHUB_REF_NAME
            if ! printf '%s\n' "$version" | grep -qxE '[0-9]+\.[0-9]+\.[0-9]+'; then
              echo "the tag $version is not a version of the form X.Y.Z" >&2
              exit 1
            fi
            api="$GITHUB_SERVER_URL/api/v1/repos/$GITHUB_REPOSITORY"
            tag=$(curl -sSf "$api/tags/$version")
            object=$(printf '%s' "$tag" | jq -r .id)
            commit=$(printf '%s' "$tag" | jq -r .commit.sha)
            if [ "$commit" != "$GITHUB_SHA" ]; then
              echo "the forge has the tag $version on $commit, not on this run's commit $GITHUB_SHA" >&2
              exit 1
            fi
            if [ "$object" = "$commit" ]; then
              echo "$version is a lightweight tag; a release tag is annotated and signed: git tag -s $version -m 'crafter-build $version'" >&2
              exit 1
            fi
            # Forgejo verifies a signed tag's own signature, and for an
            # unsigned one falls back to its commit's, so the verdict also
            # has to cover the tag's payload, not the commit's.
            signed=$(curl -sSf "$api/git/tags/$object" |
              jq -r --arg c "$commit" --arg v "$version" '"\(.verification.verified) \(.verification.signer.email // "") \(.verification.payload // "" | startswith("object \($c)\ntype commit\ntag \($v)\n"))"')
            if [ "$signed" != "true $RELEASE_SIGNER true" ]; then
              echo "Forgejo does not verify the tag $version itself as signed by $RELEASE_SIGNER (verified, signer, over the tag's payload: $signed)" >&2
              exit 1
            fi
            if ! git merge-base --is-ancestor "$GITHUB_SHA" origin/master; then
              echo "the tag $version is on $(git rev-parse --short "$GITHUB_SHA"), which is not on master: push the tag with the master it releases" >&2
              exit 1
            fi
            # Newer than every release the forge has published, its own
            # included, and not every tag: a later tag whose run still
            # queues behind this one has no release yet, and a tag that is
            # no X.Y.Z version never gets one.
            for t in $(git tag --list | awk '/^[0-9]+\.[0-9]+\.[0-9]+$/' | sort -V); do
              [ "$(printf '%s\n%s\n' "$t" "$version" | sort -V | tail -n 1)" = "$t" ] || continue
              code=$(curl -s -o /dev/null -w '%{http_code}' "$api/releases/tags/$t")
              case "$code" in
                404) ;;
                200)
                  if [ "$t" = "$version" ]; then
                    echo "$version is released already; to re-run after a release left half made, delete that release first" >&2
                  else
                    echo "$version is not newer than the release $t" >&2
                  fi
                  exit 1 ;;
                *) echo "the forge answered $code when asked whether $t is released" >&2; exit 1 ;;
              esac
            done
            if [ "$newest" != "$version" ]; then
              echo "the metainfo's newest release is \"$newest\", but the tag is $version: the tagged commit needs the <release> entry for $version at the top of packaging/flatpak/net.catcrafts.CrafterBuild.metainfo.xml" >&2
              exit 1
            fi
          else
            # Nothing is published. The build names itself after the
            # release it follows, the way a local checkout does:
            # 5.0.0-3-g21746b1 is three commits past 5.0.0. Before the first
            # tag there is nothing to describe from and the bare hash gets a
            # placeholder in front.
            described=$(git describe --tags --always)
            case "$described" in
              [0-9]*.[0-9]*.[0-9]*) version="$described" ;;
              *) version="0.0.0-dev+$described" ;;
            esac
          fi
          echo "version=$version" >> "$GITHUB_OUTPUT"
          echo "version $version"

      - name: Build
        # The manifest is rewritten beside the original so its relative
        # source paths still resolve. The suite runs inside this build.
        env:
          VERSION: ${{ steps.meta.outputs.version }}
        run: |
          set -eu
          yq '(.modules[] | select(.name == "crafter-build") | ."build-options".env.CRAFTER_BUILD_VERSION) = strenv(VERSION)' "$MANIFEST" > "$CI_MANIFEST"
          grep -n 'CRAFTER_BUILD_VERSION' "$CI_MANIFEST"
          flatpak-builder --force-clean --ccache --disable-rofiles-fuse \
            --state-dir=/root/.cache/flatpak-builder \
            --repo=repo --default-branch="$BRANCH" \
            build-dir "$CI_MANIFEST"

      - name: Smoke test
        run: |
          flatpak-builder --run build-dir "$CI_MANIFEST" crafter-build --version
          flatpak-builder --run build-dir "$CI_MANIFEST" sh -c 'cd examples/hello-world && crafter-build && ls bin/*/hello*'

      # Everything below releases the build and runs only for a pushed tag.
      - name: Bundle
        if: github.event_name == 'push'
        run: |
          mkdir -p dist
          flatpak build-bundle repo "dist/crafter-build-${{ steps.meta.outputs.version }}.flatpak" "$APP_ID" "$BRANCH"
          ls -la dist

      - name: Upload bundle
        if: github.event_name == 'push'
        uses: actions/upload-artifact@v3
        with:
          name: crafter-build-${{ steps.meta.outputs.version }}
          path: dist/*.flatpak
          if-no-files-found: error

      - name: Publish to the Catcrafts repo
        if: github.event_name == 'push'
        run: |
          set -eu
          # The signing key is generated once on the runner host
          # (/etc/crafter-flatpak: signing-key.asc and key-id) and mounted
          # read-only; it is imported into a throwaway keyring per run.
          if [ ! -s /run/crafter-flatpak/signing-key.asc ] || [ ! -s /run/crafter-flatpak/key-id ]; then
            echo "no signing key under /etc/crafter-flatpak on the runner host; generate it as described in AGENTS.template.md (CI and publishing)" >&2
            exit 1
          fi
          export GNUPGHOME=$(mktemp -d)
          gpg --batch --quiet --import /run/crafter-flatpak/signing-key.asc
          FLATPAK_GPG_KEY_ID=$(cat /run/crafter-flatpak/key-id)
          # Crafter.Review's workflow publishes into the same repo and holds
          # this lock while it does, so a commit and the summary update of
          # one workflow never interleave with the other's.
          exec 9> "$WEB_DIR/.publish.lock"
          flock 9
          # The live repo is the canonical copy; builds are promoted into it,
          # never built into it.
          # Keyed on the repo's config file, not the directory: the web root
          # may exist as an empty directory before the first publish.
          [ -f "$REPO_DIR/config" ] || ostree init --repo="$REPO_DIR" --mode=archive
          flatpak build-commit-from --src-repo=repo \
            --gpg-sign="$FLATPAK_GPG_KEY_ID" --gpg-homedir="$GNUPGHOME" \
            "$REPO_DIR" "app/$APP_ID/x86_64/$BRANCH"
          # Every push to master used to be published as `beta`. Its last
          # build is marked end-of-life, once, so that the next
          # `flatpak update` of a beta installation says why nothing follows.
          # The reason names the branch: with --noninteractive, flatpak's
          # message names only the app.
          beta="app/$APP_ID/x86_64/beta"
          if ostree rev-parse --repo="$REPO_DIR" "$beta" >/dev/null 2>&1 &&
             ! ostree show --repo="$REPO_DIR" --print-metadata-key=ostree.endoflife "$beta" >/dev/null 2>&1; then
            flatpak build-commit-from --src-ref="$beta" \
              --end-of-life="The beta branch gets no more updates: every change now comes as a release on the stable branch. Install stable with flatpak install catcrafts net.catcrafts.CrafterBuild//stable, then remove beta with flatpak uninstall net.catcrafts.CrafterBuild//beta." \
              --gpg-sign="$FLATPAK_GPG_KEY_ID" --gpg-homedir="$GNUPGHOME" \
              "$REPO_DIR" "$beta"
          fi
          flatpak build-update-repo --gpg-sign="$FLATPAK_GPG_KEY_ID" --gpg-homedir="$GNUPGHOME" \
            --generate-static-deltas --prune "$REPO_DIR"
          # The repo description and the app ref carry the public key.
          key=$(gpg --export "$FLATPAK_GPG_KEY_ID" | base64 --wrap=0)
          sed "s|GPGKey=.*|GPGKey=$key|" packaging/flatpak/catcrafts.flatpakrepo > "$WEB_DIR/catcrafts.flatpakrepo"
          sed "s|GPGKey=.*|GPGKey=$key|" packaging/flatpak/net.catcrafts.CrafterBuild.flatpakref > "$WEB_DIR/net.catcrafts.CrafterBuild.flatpakref"
          flock -u 9
          # The release attaches this .flatpakref, not the one in the
          # checkout: that one has a placeholder where the key goes, and
          # flatpak cannot install from it.
          cp "$WEB_DIR/net.catcrafts.CrafterBuild.flatpakref" dist/
          (cd dist && sha256sum * > SHA256SUMS)
          ls -la dist

      - name: Publish release
        if: github.event_name == 'push'
        # The notes are release-notes.md, which the Version step wrote. No
        # `override`: it would delete a tag that has moved off this commit
        # since the Version step and tag the old commit instead. Without it
        # the action refuses such a tag, and a release that an earlier
        # attempt left half made, such as a draft whose upload failed, has
        # to be deleted before a re-run.
        uses: https://code.forgejo.org/actions/forgejo-release@v2
        with:
          direction: upload
          url: ${{ github.server_url }}
          repo: ${{ github.repository }}
          tag: ${{ steps.meta.outputs.version }}
          title: crafter-build ${{ steps.meta.outputs.version }}
          release-notes-file: release-notes.md
          release-dir: dist
          token: ${{ secrets.GITHUB_TOKEN }}