From 38688a68f5d8c6c10d05cc1d53da86b9f88d5a77 Mon Sep 17 00:00:00 2001 From: Jorijn van der Graaf Date: Mon, 10 Aug 2026 01:51:39 +0200 Subject: [PATCH] CI fix --- .forgejo/workflows/deploy.yaml | 11 ++++-- tools/e2e.sh | 48 +++++++++++++++++++------ tools/fetch-media.sh | 65 +++++++++++++++++++++++++++------- 3 files changed, 99 insertions(+), 25 deletions(-) diff --git a/.forgejo/workflows/deploy.yaml b/.forgejo/workflows/deploy.yaml index d20aa52..a0cc41d 100644 --- a/.forgejo/workflows/deploy.yaml +++ b/.forgejo/workflows/deploy.yaml @@ -109,16 +109,22 @@ jobs: # Writes straight into the mount so the copies persist across deploys — # they are NOT always reproducible, because a source instance deleting a # file leaves ours as the only one. + id: media run: | set -eu if [ -d /deploy-app ]; then mkdir -p /deploy-app/media - tools/fetch-media.sh /deploy-app/media + dir=/deploy-app/media else echo "WARNING: /deploy-app not mounted; mirroring to a throwaway dir." >&2 echo "Media will be re-downloaded on every build until the mount exists." >&2 - tools/fetch-media.sh media + dir=media fi + tools/fetch-media.sh "$dir" + # Handed to the e2e step so it checks the files where they actually + # are. It used to assume ./media and reported every referenced file + # as missing here, which reads as a broken site and means a wrong path. + echo "dir=$dir" >> "$GITHUB_OUTPUT" - name: Build and test the backend id: srv @@ -216,6 +222,7 @@ jobs: # A gate, not a report: a failure here stops the deploy. env: SRV: ${{ steps.srv.outputs.srv }} + E2E_MEDIA_DIR: ${{ steps.media.outputs.dir }} run: tools/e2e.sh "$SRV/catcrafts-server" - name: Pre-compress static assets diff --git a/tools/e2e.sh b/tools/e2e.sh index 8abf8cd..d7fb35e 100755 --- a/tools/e2e.sh +++ b/tools/e2e.sh @@ -28,6 +28,22 @@ fi PORT="${E2E_PORT:-8199}" BASE="http://127.0.0.1:$PORT" + +# Where tools/fetch-media.sh put the mirrored files. +# +# NOT hardcoded to ./media: in the repo that is where they land, but CI points +# the mirror at the persistent mount instead (/deploy-app/media) so the copies +# survive a deploy. A check that assumed the dev layout reported every single +# referenced file as missing on CI while the files were perfectly fine on the +# mount — a failure that said "40 files missing" and meant "wrong directory". +# Override with E2E_MEDIA_DIR; the workflow passes the same path it gave the +# mirror. +MEDIA_DIR="${E2E_MEDIA_DIR:-}" +if [ -z "$MEDIA_DIR" ]; then + for d in media /deploy-app/media; do + if [ -d "$d" ]; then MEDIA_DIR="$d"; break; fi + done +fi WORK="$(mktemp -d)" ORDERS="$WORK/orders.jsonl" @@ -505,18 +521,28 @@ else fi # Every tier has to be a file that exists, or the ladder serves a 404 to # whichever browsers pick that rung — which is precisely the set of - # browsers nobody testing this site is using. - missing=0 - for f in $(curl -s "$BASE/sitemap.xml" | grep -oE '/posts/[a-z0-9-]+' | head -n 20 \ - | while read -r pg; do curl -s "$BASE$pg"; done \ - | grep -oE '(src|srcset)="/media/[^"]+"' \ - | sed 's/.*="//; s/"$//' | sort -u); do - [ -f "media${f#/media}" ] || { missing=$((missing + 1)); echo " missing: $f" >&2; } - done - if [ "$missing" -eq 0 ]; then - ok "every referenced media file is on the mount" + # browsers nobody testing this site is using. The files are served by + # Caddy rather than by this server, so they are checked on disk. + if [ -z "$MEDIA_DIR" ]; then + skip "media files exist" "no media directory found; set E2E_MEDIA_DIR" else - bad "media files" "$missing referenced file(s) not on the mount" + missing=0 + for f in $(curl -s "$BASE/sitemap.xml" | grep -oE '/posts/[a-z0-9-]+' | head -n 20 \ + | while read -r pg; do curl -s "$BASE$pg"; done \ + | grep -oE '(src|srcset)="/media/[^"]+"' \ + | sed 's/.*="//; s/"$//' | sort -u); do + if [ ! -f "$MEDIA_DIR/${f#/media/}" ]; then + missing=$((missing + 1)) + # Bounded: a wrong directory makes EVERY file missing, and a + # hundred identical lines buries the one fact that matters. + [ "$missing" -le 5 ] && echo " missing: $f" >&2 + fi + done + if [ "$missing" -eq 0 ]; then + ok "every referenced media file is in $MEDIA_DIR" + else + bad "media files" "$missing referenced file(s) not in $MEDIA_DIR" + fi fi else skip "image format ladder" "no on this page — ffmpeg absent at mirror time?" diff --git a/tools/fetch-media.sh b/tools/fetch-media.sh index 9f214ae..046e2ca 100755 --- a/tools/fetch-media.sh +++ b/tools/fetch-media.sh @@ -110,12 +110,12 @@ probe_dims() { w=0; h=0 [ "$HAVE_FFPROBE" = 1 ] || return 0 pw=$(ffprobe -v error -select_streams v:0 -show_entries stream=width \ - -of default=nw=1:nk=1 "$1" 2>/dev/null | head -n1 || true) + -of default=nw=1:nk=1 "$1" /dev/null | head -n1 || true) ph=$(ffprobe -v error -select_streams v:0 -show_entries stream=height \ - -of default=nw=1:nk=1 "$1" 2>/dev/null | head -n1 || true) + -of default=nw=1:nk=1 "$1" /dev/null | head -n1 || true) rot=$(ffprobe -v error -select_streams v:0 \ -show_entries stream_side_data=rotation \ - -of default=nw=1:nk=1 "$1" 2>/dev/null | head -n1 || true) + -of default=nw=1:nk=1 "$1" /dev/null | head -n1 || true) case "$pw" in ''|*[!0-9]*) pw=0 ;; esac case "$ph" in ''|*[!0-9]*) ph=0 ;; esac # ffprobe reports this as a signed number that some builds print with a @@ -174,7 +174,7 @@ transcode_image() { # to actually be counted — ~75 ms on a 3 MP image, once per new file. _frames=$(ffprobe -v error -select_streams v:0 -count_frames \ -show_entries stream=nb_read_frames \ - -of default=nw=1:nk=1 "$_file" 2>/dev/null | head -n1) + -of default=nw=1:nk=1 "$_file" /dev/null | head -n1) case "$_frames" in ''|*[!0-9]*|1) ;; # unknown or a single frame: a still *) echo "fetch-media: $_name is animated, serving it as one file" >&2 @@ -184,7 +184,7 @@ transcode_image() { # Alpha has to survive the transcode: an image with a transparent corner # encoded into a format with no alpha plane gains an opaque black one. _pixfmt=$(ffprobe -v error -select_streams v:0 -show_entries stream=pix_fmt \ - -of default=nw=1:nk=1 "$_file" 2>/dev/null | head -n1) + -of default=nw=1:nk=1 "$_file" /dev/null | head -n1) case "$_pixfmt" in yuva*|rgba*|bgra*|argb*|abgr*|gbrap*|ya8|ya16*|pal8) _avif_pix=yuva444p ;; *) _avif_pix=yuv444p ;; @@ -194,7 +194,7 @@ transcode_image() { # wrong-format encode cannot leave a file the next run adopts as finished. if [ -z "$avif_name" ]; then _cand="$_base.avif" - if [ -f "$MEDIA_DIR/$_cand" ]; then + if rendition_ok "$_cand" av1; then avif_name="$_cand" elif encode_rendition "$_file" "$_cand" av1 \ -c:v libaom-av1 -still-picture 1 -crf 26 -cpu-used 6 \ @@ -205,7 +205,7 @@ transcode_image() { if [ -z "$png_name" ]; then _cand="$_base.png" - if [ -f "$MEDIA_DIR/$_cand" ]; then + if rendition_ok "$_cand" png; then png_name="$_cand" elif encode_rendition "$_file" "$_cand" png -c:v png -f image2; then png_name="$_cand" @@ -213,6 +213,31 @@ transcode_image() { fi } +# The codec ffprobe reports for a file, or empty when it cannot say. +codec_of() { + [ "$HAVE_FFPROBE" = 1 ] || return 0 + ffprobe -v error -select_streams v:0 -show_entries stream=codec_name \ + -of default=nw=1:nk=1 "$1" /dev/null | head -n1 +} + +# rendition_ok NAME EXPECTED_CODEC — true when the file is already on the mount +# AND really is that codec. +# +# The second half is what makes the mount self-healing. Renditions are adopted +# by name and never re-derived, so anything wrong that once landed there would +# be trusted forever — which is exactly what a run of MJPEG files under .png +# names would have been. A file that fails re-encodes over the top instead. +rendition_ok() { + [ -f "$MEDIA_DIR/$1" ] || return 1 + _have=$(codec_of "$MEDIA_DIR/$1") + # No ffprobe to ask: trust what is there rather than re-encoding every + # image on every build. + [ -n "$_have" ] || return 0 + [ "$_have" = "$2" ] && return 0 + echo "fetch-media: $1 on the mount is '$_have', not '$2' — re-encoding it" >&2 + return 1 +} + # encode_rendition SRC OUTNAME EXPECTED_CODEC ffmpeg-args... # # Runs the encode into a .part, verifies the result really is the codec asked @@ -224,17 +249,26 @@ transcode_image() { # out.png` silently produced a run of lossy JPEGs sitting under .png names, which # the page then advertised to browsers as image/png. The codec is pinned by the # callers above; this is the check that the pin held. +# +# -nostdin AND /dev/null; then + if ! ffmpeg -nostdin -y -v error -i "$_src" -frames:v 1 "$@" \ + "$MEDIA_DIR/$_out.part" /dev/null; then rm -f "$MEDIA_DIR/$_out.part" echo "fetch-media: could not encode $_out, serving without that tier" >&2 return 1 fi _got=$(ffprobe -v error -select_streams v:0 -show_entries stream=codec_name \ - -of default=nw=1:nk=1 "$MEDIA_DIR/$_out.part" 2>/dev/null | head -n1) + -of default=nw=1:nk=1 "$MEDIA_DIR/$_out.part" /dev/null | head -n1) if [ "$_got" != "$_want" ]; then rm -f "$MEDIA_DIR/$_out.part" echo "fetch-media: $_out came out as '$_got', expected '$_want' — discarding it" >&2 @@ -288,7 +322,14 @@ encoded=0 # # Fed by a here-document rather than a pipe so the counters below survive — in # `jq | while`, the loop runs in a subshell and every increment is discarded. -while IFS= read -r src; do +# +# The list arrives on fd 3, not stdin, and the loop reads it from there. That is +# not decoration: ffmpeg below reads standard input for interactive keystrokes +# and swallowed the front of the next URL straight off the here-document, which +# cost a few posts their mirrored media on every run that had something to +# encode. ffmpeg is told -nostdin as well, but keeping the list off stdin +# entirely is what stops the next tool added to this loop from doing it again. +while IFS= read -r src <&3; do [ -n "$src" ] || continue # Ours already — adopt the file on the mount and do no network at all. @@ -445,7 +486,7 @@ while IFS= read -r src; do --argjson w "${w:-0}" --argjson h "${h:-0}" \ '. + [{src: $src, path: $path, w: $w, h: $h}]' "$MAP" > "$MAP.new" \ && mv "$MAP.new" "$MAP" -done <