CI fix

tools/fetch-media.sh

@@ -110,12 +110,12 @@ probe_dims() {
     w=0; h=0
     [ "$HAVE_FFPROBE" = 1 ] || return 0
     pw=$(ffprobe -v error -select_streams v:0 -show_entries stream=width \
-                 -of default=nw=1:nk=1 "$1" 2>/dev/null | head -n1 || true)
+                 -of default=nw=1:nk=1 "$1" </dev/null 2>/dev/null | head -n1 || true)
     ph=$(ffprobe -v error -select_streams v:0 -show_entries stream=height \
-                 -of default=nw=1:nk=1 "$1" 2>/dev/null | head -n1 || true)
+                 -of default=nw=1:nk=1 "$1" </dev/null 2>/dev/null | head -n1 || true)
     rot=$(ffprobe -v error -select_streams v:0 \
                   -show_entries stream_side_data=rotation \
-                  -of default=nw=1:nk=1 "$1" 2>/dev/null | head -n1 || true)
+                  -of default=nw=1:nk=1 "$1" </dev/null 2>/dev/null | head -n1 || true)
     case "$pw" in ''|*[!0-9]*) pw=0 ;; esac
     case "$ph" in ''|*[!0-9]*) ph=0 ;; esac
     # ffprobe reports this as a signed number that some builds print with a
@@ -174,7 +174,7 @@ transcode_image() {
     # to actually be counted — ~75 ms on a 3 MP image, once per new file.
     _frames=$(ffprobe -v error -select_streams v:0 -count_frames \
                       -show_entries stream=nb_read_frames \
-                      -of default=nw=1:nk=1 "$_file" 2>/dev/null | head -n1)
+                      -of default=nw=1:nk=1 "$_file" </dev/null 2>/dev/null | head -n1)
     case "$_frames" in
         ''|*[!0-9]*|1) ;;   # unknown or a single frame: a still
         *) echo "fetch-media: $_name is animated, serving it as one file" >&2
@@ -184,7 +184,7 @@ transcode_image() {
     # Alpha has to survive the transcode: an image with a transparent corner
     # encoded into a format with no alpha plane gains an opaque black one.
     _pixfmt=$(ffprobe -v error -select_streams v:0 -show_entries stream=pix_fmt \
-                      -of default=nw=1:nk=1 "$_file" 2>/dev/null | head -n1)
+                      -of default=nw=1:nk=1 "$_file" </dev/null 2>/dev/null | head -n1)
     case "$_pixfmt" in
         yuva*|rgba*|bgra*|argb*|abgr*|gbrap*|ya8|ya16*|pal8) _avif_pix=yuva444p ;;
         *) _avif_pix=yuv444p ;;
@@ -194,7 +194,7 @@ transcode_image() {
     # wrong-format encode cannot leave a file the next run adopts as finished.
     if [ -z "$avif_name" ]; then
         _cand="$_base.avif"
-        if [ -f "$MEDIA_DIR/$_cand" ]; then
+        if rendition_ok "$_cand" av1; then
             avif_name="$_cand"
         elif encode_rendition "$_file" "$_cand" av1 \
                  -c:v libaom-av1 -still-picture 1 -crf 26 -cpu-used 6 \
@@ -205,7 +205,7 @@ transcode_image() {
 
     if [ -z "$png_name" ]; then
         _cand="$_base.png"
-        if [ -f "$MEDIA_DIR/$_cand" ]; then
+        if rendition_ok "$_cand" png; then
             png_name="$_cand"
         elif encode_rendition "$_file" "$_cand" png -c:v png -f image2; then
             png_name="$_cand"
@@ -213,6 +213,31 @@ transcode_image() {
     fi
 }
 
+# The codec ffprobe reports for a file, or empty when it cannot say.
+codec_of() {
+    [ "$HAVE_FFPROBE" = 1 ] || return 0
+    ffprobe -v error -select_streams v:0 -show_entries stream=codec_name \
+            -of default=nw=1:nk=1 "$1" </dev/null 2>/dev/null | head -n1
+}
+
+# rendition_ok NAME EXPECTED_CODEC — true when the file is already on the mount
+# AND really is that codec.
+#
+# The second half is what makes the mount self-healing. Renditions are adopted
+# by name and never re-derived, so anything wrong that once landed there would
+# be trusted forever — which is exactly what a run of MJPEG files under .png
+# names would have been. A file that fails re-encodes over the top instead.
+rendition_ok() {
+    [ -f "$MEDIA_DIR/$1" ] || return 1
+    _have=$(codec_of "$MEDIA_DIR/$1")
+    # No ffprobe to ask: trust what is there rather than re-encoding every
+    # image on every build.
+    [ -n "$_have" ] || return 0
+    [ "$_have" = "$2" ] && return 0
+    echo "fetch-media: $1 on the mount is '$_have', not '$2' — re-encoding it" >&2
+    return 1
+}
+
 # encode_rendition SRC OUTNAME EXPECTED_CODEC ffmpeg-args...
 #
 # Runs the encode into a .part, verifies the result really is the codec asked
@@ -224,17 +249,26 @@ transcode_image() {
 # out.png` silently produced a run of lossy JPEGs sitting under .png names, which
 # the page then advertised to browsers as image/png. The codec is pinned by the
 # callers above; this is the check that the pin held.
+#
+# -nostdin AND </dev/null, both deliberately. ffmpeg reads standard input for
+# interactive keystrokes, and this runs inside `while read src; do ... done
+# <<EOF` — so it happily ate the front of the NEXT url off the here-document.
+# The symptom was a handful of downloads failing per run with mangled hosts in
+# the log ("ttps://", "s://", "tps://" — a different number of bytes swallowed
+# each time), which then left those posts pointing at third-party media and
+# failed the origin check in CI. It only bit when there was something to encode,
+# so a rerun always "fixed" it.
 encode_rendition() {
     _src="$1"; _out="$2"; _want="$3"
     shift 3
-    if ! ffmpeg -y -v error -i "$_src" -frames:v 1 "$@" \
-                "$MEDIA_DIR/$_out.part" 2>/dev/null; then
+    if ! ffmpeg -nostdin -y -v error -i "$_src" -frames:v 1 "$@" \
+                "$MEDIA_DIR/$_out.part" </dev/null 2>/dev/null; then
         rm -f "$MEDIA_DIR/$_out.part"
         echo "fetch-media: could not encode $_out, serving without that tier" >&2
         return 1
     fi
     _got=$(ffprobe -v error -select_streams v:0 -show_entries stream=codec_name \
-                   -of default=nw=1:nk=1 "$MEDIA_DIR/$_out.part" 2>/dev/null | head -n1)
+                   -of default=nw=1:nk=1 "$MEDIA_DIR/$_out.part" </dev/null 2>/dev/null | head -n1)
     if [ "$_got" != "$_want" ]; then
         rm -f "$MEDIA_DIR/$_out.part"
         echo "fetch-media: $_out came out as '$_got', expected '$_want' — discarding it" >&2
@@ -288,7 +322,14 @@ encoded=0
 #
 # Fed by a here-document rather than a pipe so the counters below survive — in
 # `jq | while`, the loop runs in a subshell and every increment is discarded.
-while IFS= read -r src; do
+#
+# The list arrives on fd 3, not stdin, and the loop reads it from there. That is
+# not decoration: ffmpeg below reads standard input for interactive keystrokes
+# and swallowed the front of the next URL straight off the here-document, which
+# cost a few posts their mirrored media on every run that had something to
+# encode. ffmpeg is told -nostdin as well, but keeping the list off stdin
+# entirely is what stops the next tool added to this loop from doing it again.
+while IFS= read -r src <&3; do
     [ -n "$src" ] || continue
 
     # Ours already — adopt the file on the mount and do no network at all.
@@ -445,7 +486,7 @@ while IFS= read -r src; do
        --argjson w "${w:-0}" --argjson h "${h:-0}" \
        '. + [{src: $src, path: $path, w: $w, h: $h}]' "$MAP" > "$MAP.new" \
        && mv "$MAP.new" "$MAP"
-done <<EOF
+done 3<<EOF
 $(jq -r --arg re "$MEDIA_REF_RE" \
       '[ (.[].media[]? | .src, (.poster // empty)),
          (.[] | .body // "" | scan($re)) ]