catcrafts.net/tools/fetch-media.sh

#!/bin/sh
# Mirror the media referenced by content/posts.json, and rewrite the entries to
# point at our own copies.
#
# Run AFTER tools/fetch-posts.sh, which records the original URLs.
#
# TWO KINDS OF MEDIA, one pipeline:
#
#   * a post's headline file — `.media`, the recording or screenshot the post is
#     about;
#   * everything embedded inside the body — the screenshots a post argues with,
#     which are just as much content and, until the body was hosted here, were
#     never fetched at all.
#
# Both are content-addressed into the same directory, so a file used as one
# post's headline and quoted inside another's body is stored once. Body files
# are rewritten IN THE MARKDOWN TEXT (the body is still Markdown at this point)
# and additionally recorded in `.body_media`, which is where the renderer reads
# the dimensions, poster and H.264 fallback that Markdown syntax has nowhere to
# carry.
#
# WHY MIRROR rather than embed from the source:
#
#   * Privacy. The privacy notice states that everything the browser loads comes
#     from catcrafts.net, and it should stay true. Embedding directly would send
#     every visitor's IP address to whichever instance hosts the file — an odd
#     thing to do on a site selling a privacy-focused phone.
#   * Durability. These posts ARE their media: the screen recording of VoLTE
#     working is the content. If the source instance deletes it or disappears,
#     a direct embed becomes a broken box and the post loses its point.
#   * Cost. One download per file, ever, instead of one per visitor. Kinder to
#     small instances than hotlinking them.
#
# Files are content-addressed (sha256 of the bytes), so a file already present is
# never downloaded again and a changed file gets a new name — which makes the
# long cache lifetime Caddy sets honest.
#
#   usage: tools/fetch-media.sh [media-dir]      (default: media/)
#
# On any single download failure the entry keeps its original URL and the script
# carries on, so one dead file does not cost the whole page. Exits non-zero only
# if it cannot do its job at all.

set -eu

MEDIA_DIR="${1:-media}"
POSTS="content/posts.json"
MAX_BYTES=$((64 * 1024 * 1024))

# What counts as a media reference inside a post body: an absolute URL or a path
# we have already rewritten, ending in a media extension.
#
# Local paths are in the pattern deliberately. Leaving them out looked right —
# nothing needs downloading twice — but it is what made a second run destructive
# rather than idempotent: the already-rewritten body references were not
# enumerated, so they never re-entered the mirror map, so the body_media sidecar
# came back with only the handful of entries that happened to still be absolute.
# Matching them means they are adopted from the mount and everything is rebuilt
# exactly as it was.
#
# One definition, passed to every jq that needs it, because three copies of a
# regex is three chances for one of them to drift.
MEDIA_REF_RE='(?:https?://|/media/)[^\s)\]"<>]+\.(?:mp4|webm|mov|webp|png|jpe?g|gif|avif)'

# Media we host ourselves, published by tools/publish-media.sh before the post
# that carries it exists. Such a URL is ALREADY the one the page should use, so
# there is nothing to fetch: the bytes are on the media mount, and downloading
# them back from our own web server would only mint a second copy under a second
# name. Skipping the download also skips MAX_BYTES, which is what a 167 MB
# recording on a third-party file host ran into — and it removes the last part of
# a build that could fail because someone else's server was slow, rate-limiting
# or gone.
OWN_ORIGIN="https://catcrafts.net/media/"

command -v jq >/dev/null 2>&1 || { echo "fetch-media: jq not found" >&2; exit 1; }
[ -f "$POSTS" ] || { echo "fetch-media: $POSTS not found — run fetch-posts.sh first" >&2; exit 1; }

mkdir -p "$MEDIA_DIR"

# ffprobe gives real pixel dimensions, which become width/height attributes.
# Without them the browser cannot reserve space and the text below jumps as each
# image arrives; with them the layout is stable on first paint. Optional — the
# markup degrades to no dimensions rather than failing.
HAVE_FFPROBE=0
command -v ffprobe >/dev/null 2>&1 && HAVE_FFPROBE=1

# ffmpeg does the still-image transcodes below. Optional in exactly the same
# way ffprobe is: without it every image is served as the single file the mirror
# downloaded, which is what this site did before the format ladder existed.
HAVE_FFMPEG=0
command -v ffmpeg >/dev/null 2>&1 && HAVE_FFMPEG=1

# Sets $w and $h for the file named in $1, or leaves both 0.
#
# One query per dimension. Asking for both at once and splitting the CSV looked
# simpler but was wrong: for some files ffprobe appends an empty field, so
# `width,height` came back as "854x480x" and splitting on `x` gave a height of
# "480x" — which the digit guard below then threw away, silently costing the
# dimensions of exactly the videos that had the extra field. `nk=1` prints the
# bare value, so there is nothing to split.
#
# ROTATION: a phone records 1920x1080 and attaches a display matrix rather than
# rotating the pixels, so the stream reads landscape while the video plays
# portrait. Believing the stream there reserves a landscape box for a portrait
# video — precisely the layout shift these attributes exist to prevent — so a
# quarter-turn swaps them. Files that went through publish-media.sh have the
# rotation baked into the pixels and report no matrix at all; this is for
# anything mirrored straight from a phone.
probe_dims() {
    w=0; h=0
    [ "$HAVE_FFPROBE" = 1 ] || return 0
    pw=$(ffprobe -v error -select_streams v:0 -show_entries stream=width \
                 -of default=nw=1:nk=1 "$1" 2>/dev/null | head -n1 || true)
    ph=$(ffprobe -v error -select_streams v:0 -show_entries stream=height \
                 -of default=nw=1:nk=1 "$1" 2>/dev/null | head -n1 || true)
    rot=$(ffprobe -v error -select_streams v:0 \
                  -show_entries stream_side_data=rotation \
                  -of default=nw=1:nk=1 "$1" 2>/dev/null | head -n1 || true)
    case "$pw" in ''|*[!0-9]*) pw=0 ;; esac
    case "$ph" in ''|*[!0-9]*) ph=0 ;; esac
    # ffprobe reports this as a signed number that some builds print with a
    # fractional part ("-90.000000"), so compare on the integer portion.
    case "${rot%%.*}" in
        90|-90|270|-270) t=$pw; pw=$ph; ph=$t ;;
    esac
    # Both or neither: a lone dimension is worse than none, because the browser
    # derives the missing one from it and gets the aspect wrong.
    if [ "$pw" -gt 0 ] && [ "$ph" -gt 0 ]; then w=$pw; h=$ph; fi
    if [ "$w" = 0 ]; then
        echo "fetch-media: no dimensions for $1; layout will shift on load" >&2
    fi
}

# Derive the two renditions a mirrored image is served between: AVIF above it
# and PNG below. Sets $avif_name / $png_name to the sibling file names, or
# leaves one empty when that rendition could not be produced — :Media then drops
# the tier rather than pointing at a file that is not on the mount.
#
# Siblings are named after the source file, which is itself the hash of its
# bytes, so a rendition already present is never re-encoded and a changed source
# gets new names. Only genuinely new images cost encoder time; a rebuild costs
# none, which is what keeps this off the critical path of every deploy.
#
# WHY BOTH TIERS. AVIF is smaller than the WebP the instances serve (~15% on
# these screenshots, far more on photographs) and is what almost every visitor
# actually receives. PNG is lossless and universally understood, which is what
# makes it a fallback worth having — but it is also several times the size of
# the WebP beside it, so the <picture> offers the mirrored original in between
# and the PNG is reached only by a browser that understands neither of the
# other two.
#
# The settings, measured against these files rather than guessed:
#   crf 26, cpu-used 6   SSIM 0.997 against the source and still smaller than
#                        it, at roughly half a second per image.
#   yuv444p              these are screenshots of text. Re-subsampling chroma
#                        that pict-rs already subsampled once fringes coloured
#                        text visibly, and full chroma costs about 3% here.
transcode_image() {
    avif_name=""
    png_name=""
    _file="$1"
    _name="$2"
    _base="${_name%.*}"

    # Already that format: serve the mirrored file as the tier rather than
    # re-encoding it into a second copy of itself.
    case "$_name" in *.avif) avif_name="$_name" ;; esac
    case "$_name" in *.png)  png_name="$_name"  ;; esac
    [ "$HAVE_FFMPEG" = 1 ] || return 0

    # An animated source is not a still, and -frames:v 1 would silently freeze
    # it. Leave it entirely alone: one moving GIF is worth more than three
    # copies of its first frame. nb_frames is N/A for WebP, so the frames have
    # to actually be counted — ~75 ms on a 3 MP image, once per new file.
    _frames=$(ffprobe -v error -select_streams v:0 -count_frames \
                      -show_entries stream=nb_read_frames \
                      -of default=nw=1:nk=1 "$_file" 2>/dev/null | head -n1)
    case "$_frames" in
        ''|*[!0-9]*|1) ;;   # unknown or a single frame: a still
        *) echo "fetch-media: $_name is animated, serving it as one file" >&2
           return 0 ;;
    esac

    # Alpha has to survive the transcode: an image with a transparent corner
    # encoded into a format with no alpha plane gains an opaque black one.
    _pixfmt=$(ffprobe -v error -select_streams v:0 -show_entries stream=pix_fmt \
                      -of default=nw=1:nk=1 "$_file" 2>/dev/null | head -n1)
    case "$_pixfmt" in
        yuva*|rgba*|bgra*|argb*|abgr*|gbrap*|ya8|ya16*|pal8) _avif_pix=yuva444p ;;
        *) _avif_pix=yuv444p ;;
    esac

    # Encoded to a .part, checked, and only then renamed — so an interrupted or
    # wrong-format encode cannot leave a file the next run adopts as finished.
    if [ -z "$avif_name" ]; then
        _cand="$_base.avif"
        if [ -f "$MEDIA_DIR/$_cand" ]; then
            avif_name="$_cand"
        elif encode_rendition "$_file" "$_cand" av1 \
                 -c:v libaom-av1 -still-picture 1 -crf 26 -cpu-used 6 \
                 -pix_fmt "$_avif_pix" -f avif; then
            avif_name="$_cand"
        fi
    fi

    if [ -z "$png_name" ]; then
        _cand="$_base.png"
        if [ -f "$MEDIA_DIR/$_cand" ]; then
            png_name="$_cand"
        elif encode_rendition "$_file" "$_cand" png -c:v png -f image2; then
            png_name="$_cand"
        fi
    fi
}

# encode_rendition SRC OUTNAME EXPECTED_CODEC ffmpeg-args...
#
# Runs the encode into a .part, verifies the result really is the codec asked
# for, and only then publishes it. Returns non-zero (leaving nothing behind) if
# either step fails, which drops that tier rather than shipping a broken one.
#
# The verification is not paranoia. ffmpeg picks an encoder from the MUXER when
# one is not named, and the image2 muxer defaults to MJPEG — so `-f image2
# out.png` silently produced a run of lossy JPEGs sitting under .png names, which
# the page then advertised to browsers as image/png. The codec is pinned by the
# callers above; this is the check that the pin held.
encode_rendition() {
    _src="$1"; _out="$2"; _want="$3"
    shift 3
    if ! ffmpeg -y -v error -i "$_src" -frames:v 1 "$@" \
                "$MEDIA_DIR/$_out.part" 2>/dev/null; then
        rm -f "$MEDIA_DIR/$_out.part"
        echo "fetch-media: could not encode $_out, serving without that tier" >&2
        return 1
    fi
    _got=$(ffprobe -v error -select_streams v:0 -show_entries stream=codec_name \
                   -of default=nw=1:nk=1 "$MEDIA_DIR/$_out.part" 2>/dev/null | head -n1)
    if [ "$_got" != "$_want" ]; then
        rm -f "$MEDIA_DIR/$_out.part"
        echo "fetch-media: $_out came out as '$_got', expected '$_want' — discarding it" >&2
        return 1
    fi
    mv "$MEDIA_DIR/$_out.part" "$MEDIA_DIR/$_out"
    chmod 0644 "$MEDIA_DIR/$_out"
    encoded=$((encoded + 1))
    return 0
}

MAP="$(mktemp)"
POSTERMAP="$(mktemp)"
FALLBACKMAP="$(mktemp)"
AVIFMAP="$(mktemp)"
POSTERONLY="$(mktemp)"
trap 'rm -f "$MAP" "$POSTERMAP" "$FALLBACKMAP" "$AVIFMAP" "$POSTERONLY"' EXIT
printf '[]' > "$MAP"
printf '[]' > "$POSTERMAP"
printf '[]' > "$FALLBACKMAP"
printf '[]' > "$AVIFMAP"

# URLs that are ONLY ever a video's poster frame. They are skipped by the
# transcode above, because `poster` takes exactly one URL: a <video> cannot
# negotiate a format the way <picture> can, so the renditions would be files
# nothing is able to ask for. A file that is a poster somewhere and an ordinary
# image somewhere else is not in this list and is transcoded normally.
jq -r --arg re "$MEDIA_REF_RE" \
   '([.[].media[]? | .poster // empty] | map(select(. != "")) | unique) as $posters
    | ([.[].media[]? | .src] + [.[] | .body // "" | scan($re)] | unique) as $srcs
    | ($posters - $srcs) | .[]' "$POSTS" > "$POSTERONLY" 2>/dev/null || true

downloaded=0
reused=0
adopted=0
failed=0
encoded=0

# Every distinct media URL across all posts, so a file shared by two posts is
# fetched once. Video posters are in here too: a poster left pointing at the
# source instance would leak a visitor IP on page load exactly like an embedded
# image would, and it is the frame shown before anyone presses play.
#
# Body URLs are found by pattern rather than by parsing Markdown: anything that
# looks like an absolute URL ending in a media extension is mirrored, whether it
# was written as an embed, as a link, or bare. That is deliberately wider than
# "images the body displays" — the origin rule covers href as well as src, and a
# link whose target is a .webp on someone else's instance is still a third-party
# address on our page. Already-rewritten paths start with /media/ and so do not
# match, which is what makes re-running this a no-op.
#
# Fed by a here-document rather than a pipe so the counters below survive — in
# `jq | while`, the loop runs in a subshell and every increment is discarded.
while IFS= read -r src; do
    [ -n "$src" ] || continue

    # Ours already — adopt the file on the mount and do no network at all.
    # A URL that is on our origin but names a file that is NOT on the mount is
    # left alone rather than invented: that is a post published without its
    # media, and keeping the original URL makes the e2e origin check fail
    # loudly instead of shipping a 404 in a <video> tag.
    case "$src" in
    "$OWN_ORIGIN"*)
        name=${src#"$OWN_ORIGIN"}
        # Refuse anything that is not a bare filename. A path separator or a
        # traversal segment arriving from a post URL must never reach a path we
        # then read or publish.
        case "$name" in
            ''|*/*|*..*)
                echo "fetch-media: refusing suspicious own-origin URL: $src" >&2
                failed=$((failed + 1)); continue ;;
        esac
        # A .h264.mp4 URL is the fediverse-facing form of an AV1 video: the
        # post links the encoding everything can play, because that link is
        # fetched raw by Lemmy apps. Our page can negotiate, so when the AV1
        # sibling is on the mount it becomes the primary <source> and the
        # H.264 drops to the fallback (picked up by name below).
        case "$name" in
            *.h264.mp4)
                av1="${name%.h264.mp4}.mp4"
                [ -f "$MEDIA_DIR/$av1" ] && name="$av1"
                ;;
        esac
        dest="$MEDIA_DIR/$name"
        if [ ! -f "$dest" ]; then
            echo "fetch-media: $name not on the media mount, keeping original URL: $src" >&2
            failed=$((failed + 1)); continue
        fi
        adopted=$((adopted + 1))
        ;;
    /media/*)
        # Already rewritten by an earlier run of this script. Adopt the file on
        # the mount rather than trying to fetch our own path as though it were a
        # URL — which is what makes running this twice a no-op instead of a way
        # to lose every rewrite it made the first time. The script is meant to
        # follow fetch-posts.sh, but "meant to" is not a guarantee, and the
        # failure was silent: the body_media list simply came back empty.
        name=${src#/media/}
        case "$name" in
            ''|*/*|*..*)
                echo "fetch-media: refusing suspicious local path: $src" >&2
                failed=$((failed + 1)); continue ;;
        esac
        dest="$MEDIA_DIR/$name"
        if [ ! -f "$dest" ]; then
            echo "fetch-media: $name not on the media mount, leaving it alone: $src" >&2
            failed=$((failed + 1)); continue
        fi
        adopted=$((adopted + 1))
        ;;
    *)
        ext=$(printf '%s' "$src" | sed -E 's/.*\.([A-Za-z0-9]+)$/\1/' | tr 'A-Z' 'a-z')
        case "$ext" in
            mp4|webm|mov|webp|png|jpg|jpeg|gif|avif) ;;
            *) echo "fetch-media: skipping unexpected extension: $src" >&2; continue ;;
        esac

        tmp="$(mktemp)"
        # --max-filesize refuses an oversized body before writing it; the explicit
        # size check afterwards covers servers that do not send Content-Length.
        if ! curl -fsSL --max-time 120 --max-filesize "$MAX_BYTES" \
                  -A 'catcrafts.net-buildfetch/1.0 (+https://catcrafts.net)' \
                  "$src" -o "$tmp" 2>/dev/null; then
            echo "fetch-media: download failed, keeping original URL: $src" >&2
            rm -f "$tmp"
            failed=$((failed + 1))
            continue
        fi
        if [ "$(wc -c < "$tmp")" -gt "$MAX_BYTES" ]; then
            echo "fetch-media: oversized, keeping original URL: $src" >&2
            rm -f "$tmp"
            failed=$((failed + 1))
            continue
        fi

        hash=$(sha256sum "$tmp" | cut -c1-16)
        name="$hash.$ext"
        dest="$MEDIA_DIR/$name"

        if [ -f "$dest" ]; then
            rm -f "$tmp"
            reused=$((reused + 1))
        else
            mv "$tmp" "$dest"
            chmod 0644 "$dest"
            downloaded=$((downloaded + 1))
        fi
        ;;
    esac

    probe_dims "$dest"

    # A self-hosted video has no Lemmy thumbnail to mirror when the instance
    # cannot decode it — AV1 is the common case, since pict-rs will not generate
    # a still from one. publish-media.sh uploads a poster frame alongside the
    # video under the video's own hash, so look for that sibling and offer it to
    # the rewrite below. Without a poster a preload="metadata" video is a black
    # box until someone presses play.
    case "$name" in
        *.mp4|*.webm|*.mov)
            # Siblings are named after the AV1's hash, so strip the .h264
            # marker too — a video adopted as X.h264.mp4 (its AV1 never
            # published) still finds X.poster.webp.
            base="${name%.*}"
            base="${base%.h264}"
            sibling="$base.poster.webp"
            if [ -f "$MEDIA_DIR/$sibling" ]; then
                jq --arg k "/media/$name" --arg v "/media/$sibling" \
                   '. + [{key: $k, value: $v}]' "$POSTERMAP" > "$POSTERMAP.new" \
                   && mv "$POSTERMAP.new" "$POSTERMAP"
            fi
            # The H.264 sibling publish-media.sh uploaded next to the AV1.
            # Rendered as a second <source> so browsers without AV1 (Safari
            # before 17, Apple hardware without the decoder) get a file they
            # can play instead of an element that will not. The guard against
            # naming itself covers a video adopted as X.h264.mp4 whose AV1 is
            # not on the mount.
            fb="$base.h264.mp4"
            if [ "$fb" != "$name" ] && [ -f "$MEDIA_DIR/$fb" ]; then
                jq --arg k "/media/$name" --arg v "/media/$fb" \
                   '. + [{key: $k, value: $v}]' "$FALLBACKMAP" > "$FALLBACKMAP.new" \
                   && mv "$FALLBACKMAP.new" "$FALLBACKMAP"
            fi
            ;;
        *.webp|*.png|*.jpg|*.jpeg|*.gif|*.avif)
            # The AVIF and PNG tiers this image is served between. Both maps are
            # keyed by the LOCAL path, like the video ones above, so the rewrite
            # below can look them up from the src it has just written.
            if grep -qxF "$src" "$POSTERONLY" 2>/dev/null; then
                : # poster-only; see POSTERONLY above
            else
                transcode_image "$dest" "$name"
                if [ -n "$avif_name" ]; then
                    jq --arg k "/media/$name" --arg v "/media/$avif_name" \
                       '. + [{key: $k, value: $v}]' "$AVIFMAP" > "$AVIFMAP.new" \
                       && mv "$AVIFMAP.new" "$AVIFMAP"
                fi
                if [ -n "$png_name" ]; then
                    jq --arg k "/media/$name" --arg v "/media/$png_name" \
                       '. + [{key: $k, value: $v}]' "$FALLBACKMAP" > "$FALLBACKMAP.new" \
                       && mv "$FALLBACKMAP.new" "$FALLBACKMAP"
                fi
            fi
            ;;
    esac

    jq --arg src "$src" --arg path "/media/$name" \
       --argjson w "${w:-0}" --argjson h "${h:-0}" \
       '. + [{src: $src, path: $path, w: $w, h: $h}]' "$MAP" > "$MAP.new" \
       && mv "$MAP.new" "$MAP"
done <<EOF
$(jq -r --arg re "$MEDIA_REF_RE" \
      '[ (.[].media[]? | .src, (.poster // empty)),
         (.[] | .body // "" | scan($re)) ]
       | map(select(. != "")) | unique[]' "$POSTS")
EOF

echo "fetch-media: $downloaded new, $reused already present, $adopted self-hosted, $failed failed"
echo "fetch-media: $encoded image rendition(s) encoded this run"

# Rewrite each media entry to the local path. An entry with no mapping (download
# failed) keeps its original src, so the page still shows something rather than
# silently dropping the post's whole point.
TMP_POSTS="$(mktemp)"
if jq --slurpfile map "$MAP" --slurpfile posters "$POSTERMAP" \
      --slurpfile fallbacks "$FALLBACKMAP" --slurpfile avifs "$AVIFMAP" '
    ($map[0] | map({key: .src, value: .}) | from_entries) as $m
    | ($posters[0] | from_entries) as $pm
    | ($fallbacks[0] | from_entries) as $fm
    | ($avifs[0] | from_entries) as $am
    | map(.media = ((.media // []) | map(
        . as $item
        | ($m[$item.src] // null) as $hit
        | (if $hit == null then $item
           else $item + { src: $hit.path, w: $hit.w, h: $hit.h }
           end)
        # The poster gets its path rewritten but NOT its dimensions: w/h describe
        # the video, and a poster is a differently-sized still of it. Feeding the
        # poster'\''s size to the <video> element would set the wrong aspect ratio.
        | if (.poster // "") == "" then .
          else . + { poster: (($m[.poster].path) // .poster) }
          end
        # Last resort, and only for media we host: the sibling poster frame
        # publish-media.sh uploaded next to the video. Runs after the rewrite
        # above so it sees the LOCAL src, and only fills a poster that is still
        # empty — a thumbnail the instance did provide always wins.
        | if ((.poster // "") == "") and (($pm[.src] // "") != "")
          then . + { poster: $pm[.src] }
          else . end
        # H.264 fallback, keyed by the LOCAL src like the poster map. Only ever
        # set for media on our own mount — a mirrored third-party file has no
        # sibling to find.
        | if (($fm[.src] // "") != "")
          then . + { fallback: $fm[.src] }
          else . end
        # The AVIF tier, keyed by the LOCAL src like the two maps above. Only
        # ever set for images, and only when the encode actually produced one.
        | if (($am[.src] // "") != "")
          then . + { avif: $am[.src] }
          else . end)))
    # ── the body ──────────────────────────────────────────────────────
    #
    # Substitution is literal (split/join, not gsub), because these URLs are
    # full of regex metacharacters and a mirrored path must land in the text
    # exactly as written. A URL with no mapping — its download failed — is left
    # alone, so the post still shows the image rather than losing it; the
    # accounting at the end of this script reports that as media still pointing
    # at its source.
    | ($m | to_entries) as $subs
    | map(.body = (reduce $subs[] as $s ((.body // "");
                     split($s.key) | join($s.value.path))))
    # Everything the rewritten body now points at, as records the renderer can
    # read: Markdown has nowhere to put a width, a poster frame or a second
    # source, so the sidecar list is how an inline video gets the same treatment
    # as a headline one. Keyed by the LOCAL path, which is what the body says
    # by this point.
    | ($m | map({ key: .path, value: . }) | from_entries) as $byPath
    | map(.body_media = ([ (.body // "")
                           | scan("/media/[A-Za-z0-9._-]+")
                           | . as $path
                           | select($byPath[$path] != null)
                           | { src: $path,
                               kind: (if ($path | test("\\.(?:mp4|webm|mov)$"))
                                      then "video" else "image" end),
                               poster: ($pm[$path] // ""),
                               fallback: ($fm[$path] // ""),
                               avif: ($am[$path] // ""),
                               w: ($byPath[$path].w // 0),
                               h: ($byPath[$path].h // 0) } ]
                         | unique_by(.src)))
    ' "$POSTS" > "$TMP_POSTS" 2>/dev/null; then
    # Same reason as the chmod on each mirrored file: mktemp is 0600 and the
    # mode survives to production, where other users must read this.
    chmod 0644 "$TMP_POSTS"
    mv "$TMP_POSTS" "$POSTS"
else
    rm -f "$TMP_POSTS"
    echo "fetch-media: could not rewrite $POSTS, leaving it unchanged" >&2
    exit 1
fi

# Counted over the bodies too, because a body URL that never mirrored is the
# same privacy leak as a card one and must not be reported as success.
total=$(jq --arg re "$MEDIA_REF_RE" \
        '[ (.[].media[]? | .src, (.poster // empty)),
           (.[] | .body // "" | scan($re)),
           (.[].body_media[]? | .src, (.poster // empty)) ]
         | map(select(. != "")) | unique | length' "$POSTS")
local_count=$(jq '[ (.[].media[]? | .src, (.poster // empty)),
                    (.[].body_media[]? | .src, (.poster // empty)) ]
                  | map(select(startswith("/media/"))) | length' "$POSTS")
inline=$(jq '[.[].body_media[]?] | length' "$POSTS")
echo "fetch-media: $local_count of $total media entries served locally ($inline of them embedded in post bodies; $(du -sh "$MEDIA_DIR" | cut -f1) in $MEDIA_DIR)"
if [ "$local_count" -ne "$total" ]; then
    echo "fetch-media: $((total - local_count)) still point at their source — see the failures above" >&2
fi