CI fix

tools/e2e.sh

@@ -28,6 +28,22 @@ fi
 
 PORT="${E2E_PORT:-8199}"
 BASE="http://127.0.0.1:$PORT"
+
+# Where tools/fetch-media.sh put the mirrored files.
+#
+# NOT hardcoded to ./media: in the repo that is where they land, but CI points
+# the mirror at the persistent mount instead (/deploy-app/media) so the copies
+# survive a deploy. A check that assumed the dev layout reported every single
+# referenced file as missing on CI while the files were perfectly fine on the
+# mount — a failure that said "40 files missing" and meant "wrong directory".
+# Override with E2E_MEDIA_DIR; the workflow passes the same path it gave the
+# mirror.
+MEDIA_DIR="${E2E_MEDIA_DIR:-}"
+if [ -z "$MEDIA_DIR" ]; then
+    for d in media /deploy-app/media; do
+        if [ -d "$d" ]; then MEDIA_DIR="$d"; break; fi
+    done
+fi
 WORK="$(mktemp -d)"
 ORDERS="$WORK/orders.jsonl"
 
@@ -505,18 +521,28 @@ else
         fi
         # Every tier has to be a file that exists, or the ladder serves a 404 to
         # whichever browsers pick that rung — which is precisely the set of
-        # browsers nobody testing this site is using.
-        missing=0
-        for f in $(curl -s "$BASE/sitemap.xml" | grep -oE '/posts/[a-z0-9-]+' | head -n 20 \
-                   | while read -r pg; do curl -s "$BASE$pg"; done \
-                   | grep -oE '(src|srcset)="/media/[^"]+"' \
-                   | sed 's/.*="//; s/"$//' | sort -u); do
-            [ -f "media${f#/media}" ] || { missing=$((missing + 1)); echo "    missing: $f" >&2; }
-        done
-        if [ "$missing" -eq 0 ]; then
-            ok "every referenced media file is on the mount"
+        # browsers nobody testing this site is using. The files are served by
+        # Caddy rather than by this server, so they are checked on disk.
+        if [ -z "$MEDIA_DIR" ]; then
+            skip "media files exist" "no media directory found; set E2E_MEDIA_DIR"
         else
-            bad "media files" "$missing referenced file(s) not on the mount"
+            missing=0
+            for f in $(curl -s "$BASE/sitemap.xml" | grep -oE '/posts/[a-z0-9-]+' | head -n 20 \
+                       | while read -r pg; do curl -s "$BASE$pg"; done \
+                       | grep -oE '(src|srcset)="/media/[^"]+"' \
+                       | sed 's/.*="//; s/"$//' | sort -u); do
+                if [ ! -f "$MEDIA_DIR/${f#/media/}" ]; then
+                    missing=$((missing + 1))
+                    # Bounded: a wrong directory makes EVERY file missing, and a
+                    # hundred identical lines buries the one fact that matters.
+                    [ "$missing" -le 5 ] && echo "      missing: $f" >&2
+                fi
+            done
+            if [ "$missing" -eq 0 ]; then
+                ok "every referenced media file is in $MEDIA_DIR"
+            else
+                bad "media files" "$missing referenced file(s) not in $MEDIA_DIR"
+            fi
         fi
     else
         skip "image format ladder" "no <picture> on this page — ffmpeg absent at mirror time?"

tools/fetch-media.sh

@@ -110,12 +110,12 @@ probe_dims() {
     w=0; h=0
     [ "$HAVE_FFPROBE" = 1 ] || return 0
     pw=$(ffprobe -v error -select_streams v:0 -show_entries stream=width \
-                 -of default=nw=1:nk=1 "$1" 2>/dev/null | head -n1 || true)
+                 -of default=nw=1:nk=1 "$1" </dev/null 2>/dev/null | head -n1 || true)
     ph=$(ffprobe -v error -select_streams v:0 -show_entries stream=height \
-                 -of default=nw=1:nk=1 "$1" 2>/dev/null | head -n1 || true)
+                 -of default=nw=1:nk=1 "$1" </dev/null 2>/dev/null | head -n1 || true)
     rot=$(ffprobe -v error -select_streams v:0 \
                   -show_entries stream_side_data=rotation \
-                  -of default=nw=1:nk=1 "$1" 2>/dev/null | head -n1 || true)
+                  -of default=nw=1:nk=1 "$1" </dev/null 2>/dev/null | head -n1 || true)
     case "$pw" in ''|*[!0-9]*) pw=0 ;; esac
     case "$ph" in ''|*[!0-9]*) ph=0 ;; esac
     # ffprobe reports this as a signed number that some builds print with a
@@ -174,7 +174,7 @@ transcode_image() {
     # to actually be counted — ~75 ms on a 3 MP image, once per new file.
     _frames=$(ffprobe -v error -select_streams v:0 -count_frames \
                       -show_entries stream=nb_read_frames \
-                      -of default=nw=1:nk=1 "$_file" 2>/dev/null | head -n1)
+                      -of default=nw=1:nk=1 "$_file" </dev/null 2>/dev/null | head -n1)
     case "$_frames" in
         ''|*[!0-9]*|1) ;;   # unknown or a single frame: a still
         *) echo "fetch-media: $_name is animated, serving it as one file" >&2
@@ -184,7 +184,7 @@ transcode_image() {
     # Alpha has to survive the transcode: an image with a transparent corner
     # encoded into a format with no alpha plane gains an opaque black one.
     _pixfmt=$(ffprobe -v error -select_streams v:0 -show_entries stream=pix_fmt \
-                      -of default=nw=1:nk=1 "$_file" 2>/dev/null | head -n1)
+                      -of default=nw=1:nk=1 "$_file" </dev/null 2>/dev/null | head -n1)
     case "$_pixfmt" in
         yuva*|rgba*|bgra*|argb*|abgr*|gbrap*|ya8|ya16*|pal8) _avif_pix=yuva444p ;;
         *) _avif_pix=yuv444p ;;
@@ -194,7 +194,7 @@ transcode_image() {
     # wrong-format encode cannot leave a file the next run adopts as finished.
     if [ -z "$avif_name" ]; then
         _cand="$_base.avif"
-        if [ -f "$MEDIA_DIR/$_cand" ]; then
+        if rendition_ok "$_cand" av1; then
             avif_name="$_cand"
         elif encode_rendition "$_file" "$_cand" av1 \
                  -c:v libaom-av1 -still-picture 1 -crf 26 -cpu-used 6 \
@@ -205,7 +205,7 @@ transcode_image() {
 
     if [ -z "$png_name" ]; then
         _cand="$_base.png"
-        if [ -f "$MEDIA_DIR/$_cand" ]; then
+        if rendition_ok "$_cand" png; then
             png_name="$_cand"
         elif encode_rendition "$_file" "$_cand" png -c:v png -f image2; then
             png_name="$_cand"
@@ -213,6 +213,31 @@ transcode_image() {
     fi
 }
 
+# The codec ffprobe reports for a file, or empty when it cannot say.
+codec_of() {
+    [ "$HAVE_FFPROBE" = 1 ] || return 0
+    ffprobe -v error -select_streams v:0 -show_entries stream=codec_name \
+            -of default=nw=1:nk=1 "$1" </dev/null 2>/dev/null | head -n1
+}
+
+# rendition_ok NAME EXPECTED_CODEC — true when the file is already on the mount
+# AND really is that codec.
+#
+# The second half is what makes the mount self-healing. Renditions are adopted
+# by name and never re-derived, so anything wrong that once landed there would
+# be trusted forever — which is exactly what a run of MJPEG files under .png
+# names would have been. A file that fails re-encodes over the top instead.
+rendition_ok() {
+    [ -f "$MEDIA_DIR/$1" ] || return 1
+    _have=$(codec_of "$MEDIA_DIR/$1")
+    # No ffprobe to ask: trust what is there rather than re-encoding every
+    # image on every build.
+    [ -n "$_have" ] || return 0
+    [ "$_have" = "$2" ] && return 0
+    echo "fetch-media: $1 on the mount is '$_have', not '$2' — re-encoding it" >&2
+    return 1
+}
+
 # encode_rendition SRC OUTNAME EXPECTED_CODEC ffmpeg-args...
 #
 # Runs the encode into a .part, verifies the result really is the codec asked
@@ -224,17 +249,26 @@ transcode_image() {
 # out.png` silently produced a run of lossy JPEGs sitting under .png names, which
 # the page then advertised to browsers as image/png. The codec is pinned by the
 # callers above; this is the check that the pin held.
+#
+# -nostdin AND </dev/null, both deliberately. ffmpeg reads standard input for
+# interactive keystrokes, and this runs inside `while read src; do ... done
+# <<EOF` — so it happily ate the front of the NEXT url off the here-document.
+# The symptom was a handful of downloads failing per run with mangled hosts in
+# the log ("ttps://", "s://", "tps://" — a different number of bytes swallowed
+# each time), which then left those posts pointing at third-party media and
+# failed the origin check in CI. It only bit when there was something to encode,
+# so a rerun always "fixed" it.
 encode_rendition() {
     _src="$1"; _out="$2"; _want="$3"
     shift 3
-    if ! ffmpeg -y -v error -i "$_src" -frames:v 1 "$@" \
-                "$MEDIA_DIR/$_out.part" 2>/dev/null; then
+    if ! ffmpeg -nostdin -y -v error -i "$_src" -frames:v 1 "$@" \
+                "$MEDIA_DIR/$_out.part" </dev/null 2>/dev/null; then
         rm -f "$MEDIA_DIR/$_out.part"
         echo "fetch-media: could not encode $_out, serving without that tier" >&2
         return 1
     fi
     _got=$(ffprobe -v error -select_streams v:0 -show_entries stream=codec_name \
-                   -of default=nw=1:nk=1 "$MEDIA_DIR/$_out.part" 2>/dev/null | head -n1)
+                   -of default=nw=1:nk=1 "$MEDIA_DIR/$_out.part" </dev/null 2>/dev/null | head -n1)
     if [ "$_got" != "$_want" ]; then
         rm -f "$MEDIA_DIR/$_out.part"
         echo "fetch-media: $_out came out as '$_got', expected '$_want' — discarding it" >&2
@@ -288,7 +322,14 @@ encoded=0
 #
 # Fed by a here-document rather than a pipe so the counters below survive — in
 # `jq | while`, the loop runs in a subshell and every increment is discarded.
-while IFS= read -r src; do
+#
+# The list arrives on fd 3, not stdin, and the loop reads it from there. That is
+# not decoration: ffmpeg below reads standard input for interactive keystrokes
+# and swallowed the front of the next URL straight off the here-document, which
+# cost a few posts their mirrored media on every run that had something to
+# encode. ffmpeg is told -nostdin as well, but keeping the list off stdin
+# entirely is what stops the next tool added to this loop from doing it again.
+while IFS= read -r src <&3; do
     [ -n "$src" ] || continue
 
     # Ours already — adopt the file on the mount and do no network at all.
@@ -445,7 +486,7 @@ while IFS= read -r src; do
        --argjson w "${w:-0}" --argjson h "${h:-0}" \
        '. + [{src: $src, path: $path, w: $w, h: $h}]' "$MAP" > "$MAP.new" \
        && mv "$MAP.new" "$MAP"
-done <<EOF
+done 3<<EOF
 $(jq -r --arg re "$MEDIA_REF_RE" \
       '[ (.[].media[]? | .src, (.poster // empty)),
          (.[] | .body // "" | scan($re)) ]