catcrafts.net/tools/fetch-media.sh

#!/bin/sh
# Mirror the media referenced by content/posts.json, and rewrite the entries to
# point at our own copies.
#
# Run AFTER tools/fetch-posts.sh, which records the original URLs.
#
# WHY MIRROR rather than embed from the source:
#
#   * Privacy. The privacy notice states that everything the browser loads comes
#     from catcrafts.net, and it should stay true. Embedding directly would send
#     every visitor's IP address to whichever instance hosts the file — an odd
#     thing to do on a site selling a privacy-focused phone.
#   * Durability. These posts ARE their media: the screen recording of VoLTE
#     working is the content. If the source instance deletes it or disappears,
#     a direct embed becomes a broken box and the post loses its point.
#   * Cost. One download per file, ever, instead of one per visitor. Kinder to
#     small instances than hotlinking them.
#
# Files are content-addressed (sha256 of the bytes), so a file already present is
# never downloaded again and a changed file gets a new name — which makes the
# long cache lifetime Caddy sets honest.
#
#   usage: tools/fetch-media.sh [media-dir]      (default: media/)
#
# On any single download failure the entry keeps its original URL and the script
# carries on, so one dead file does not cost the whole page. Exits non-zero only
# if it cannot do its job at all.

set -eu

MEDIA_DIR="${1:-media}"
POSTS="content/posts.json"
MAX_BYTES=$((64 * 1024 * 1024))

# Media we host ourselves, published by tools/publish-media.sh before the post
# that carries it exists. Such a URL is ALREADY the one the page should use, so
# there is nothing to fetch: the bytes are on the media mount, and downloading
# them back from our own web server would only mint a second copy under a second
# name. Skipping the download also skips MAX_BYTES, which is what a 167 MB
# recording on a third-party file host ran into — and it removes the last part of
# a build that could fail because someone else's server was slow, rate-limiting
# or gone.
OWN_ORIGIN="https://catcrafts.net/media/"

command -v jq >/dev/null 2>&1 || { echo "fetch-media: jq not found" >&2; exit 1; }
[ -f "$POSTS" ] || { echo "fetch-media: $POSTS not found — run fetch-posts.sh first" >&2; exit 1; }

mkdir -p "$MEDIA_DIR"

# ffprobe gives real pixel dimensions, which become width/height attributes.
# Without them the browser cannot reserve space and the text below jumps as each
# image arrives; with them the layout is stable on first paint. Optional — the
# markup degrades to no dimensions rather than failing.
HAVE_FFPROBE=0
command -v ffprobe >/dev/null 2>&1 && HAVE_FFPROBE=1

# Sets $w and $h for the file named in $1, or leaves both 0.
#
# One query per dimension. Asking for both at once and splitting the CSV looked
# simpler but was wrong: for some files ffprobe appends an empty field, so
# `width,height` came back as "854x480x" and splitting on `x` gave a height of
# "480x" — which the digit guard below then threw away, silently costing the
# dimensions of exactly the videos that had the extra field. `nk=1` prints the
# bare value, so there is nothing to split.
#
# ROTATION: a phone records 1920x1080 and attaches a display matrix rather than
# rotating the pixels, so the stream reads landscape while the video plays
# portrait. Believing the stream there reserves a landscape box for a portrait
# video — precisely the layout shift these attributes exist to prevent — so a
# quarter-turn swaps them. Files that went through publish-media.sh have the
# rotation baked into the pixels and report no matrix at all; this is for
# anything mirrored straight from a phone.
probe_dims() {
    w=0; h=0
    [ "$HAVE_FFPROBE" = 1 ] || return 0
    pw=$(ffprobe -v error -select_streams v:0 -show_entries stream=width \
                 -of default=nw=1:nk=1 "$1" 2>/dev/null | head -n1 || true)
    ph=$(ffprobe -v error -select_streams v:0 -show_entries stream=height \
                 -of default=nw=1:nk=1 "$1" 2>/dev/null | head -n1 || true)
    rot=$(ffprobe -v error -select_streams v:0 \
                  -show_entries stream_side_data=rotation \
                  -of default=nw=1:nk=1 "$1" 2>/dev/null | head -n1 || true)
    case "$pw" in ''|*[!0-9]*) pw=0 ;; esac
    case "$ph" in ''|*[!0-9]*) ph=0 ;; esac
    # ffprobe reports this as a signed number that some builds print with a
    # fractional part ("-90.000000"), so compare on the integer portion.
    case "${rot%%.*}" in
        90|-90|270|-270) t=$pw; pw=$ph; ph=$t ;;
    esac
    # Both or neither: a lone dimension is worse than none, because the browser
    # derives the missing one from it and gets the aspect wrong.
    if [ "$pw" -gt 0 ] && [ "$ph" -gt 0 ]; then w=$pw; h=$ph; fi
    if [ "$w" = 0 ]; then
        echo "fetch-media: no dimensions for $1; layout will shift on load" >&2
    fi
}

MAP="$(mktemp)"
POSTERMAP="$(mktemp)"
trap 'rm -f "$MAP" "$POSTERMAP"' EXIT
printf '[]' > "$MAP"
printf '[]' > "$POSTERMAP"

downloaded=0
reused=0
adopted=0
failed=0

# Every distinct media URL across all posts, so a file shared by two posts is
# fetched once. Video posters are in here too: a poster left pointing at the
# source instance would leak a visitor IP on page load exactly like an embedded
# image would, and it is the frame shown before anyone presses play.
#
# Fed by a here-document rather than a pipe so the counters below survive — in
# `jq | while`, the loop runs in a subshell and every increment is discarded.
while IFS= read -r src; do
    [ -n "$src" ] || continue

    # Ours already — adopt the file on the mount and do no network at all.
    # A URL that is on our origin but names a file that is NOT on the mount is
    # left alone rather than invented: that is a post published without its
    # media, and keeping the original URL makes the e2e origin check fail
    # loudly instead of shipping a 404 in a <video> tag.
    case "$src" in
    "$OWN_ORIGIN"*)
        name=${src#"$OWN_ORIGIN"}
        # Refuse anything that is not a bare filename. A path separator or a
        # traversal segment arriving from a post URL must never reach a path we
        # then read or publish.
        case "$name" in
            ''|*/*|*..*)
                echo "fetch-media: refusing suspicious own-origin URL: $src" >&2
                failed=$((failed + 1)); continue ;;
        esac
        dest="$MEDIA_DIR/$name"
        if [ ! -f "$dest" ]; then
            echo "fetch-media: $name not on the media mount, keeping original URL: $src" >&2
            failed=$((failed + 1)); continue
        fi
        adopted=$((adopted + 1))
        ;;
    *)
        ext=$(printf '%s' "$src" | sed -E 's/.*\.([A-Za-z0-9]+)$/\1/' | tr 'A-Z' 'a-z')
        case "$ext" in
            mp4|webm|mov|webp|png|jpg|jpeg|gif|avif) ;;
            *) echo "fetch-media: skipping unexpected extension: $src" >&2; continue ;;
        esac

        tmp="$(mktemp)"
        # --max-filesize refuses an oversized body before writing it; the explicit
        # size check afterwards covers servers that do not send Content-Length.
        if ! curl -fsSL --max-time 120 --max-filesize "$MAX_BYTES" \
                  -A 'catcrafts.net-buildfetch/1.0 (+https://catcrafts.net)' \
                  "$src" -o "$tmp" 2>/dev/null; then
            echo "fetch-media: download failed, keeping original URL: $src" >&2
            rm -f "$tmp"
            failed=$((failed + 1))
            continue
        fi
        if [ "$(wc -c < "$tmp")" -gt "$MAX_BYTES" ]; then
            echo "fetch-media: oversized, keeping original URL: $src" >&2
            rm -f "$tmp"
            failed=$((failed + 1))
            continue
        fi

        hash=$(sha256sum "$tmp" | cut -c1-16)
        name="$hash.$ext"
        dest="$MEDIA_DIR/$name"

        if [ -f "$dest" ]; then
            rm -f "$tmp"
            reused=$((reused + 1))
        else
            mv "$tmp" "$dest"
            chmod 0644 "$dest"
            downloaded=$((downloaded + 1))
        fi
        ;;
    esac

    probe_dims "$dest"

    # A self-hosted video has no Lemmy thumbnail to mirror when the instance
    # cannot decode it — AV1 is the common case, since pict-rs will not generate
    # a still from one. publish-media.sh uploads a poster frame alongside the
    # video under the video's own hash, so look for that sibling and offer it to
    # the rewrite below. Without a poster a preload="metadata" video is a black
    # box until someone presses play.
    case "$name" in
        *.mp4|*.webm|*.mov)
            sibling="${name%.*}.poster.webp"
            if [ -f "$MEDIA_DIR/$sibling" ]; then
                jq --arg k "/media/$name" --arg v "/media/$sibling" \
                   '. + [{key: $k, value: $v}]' "$POSTERMAP" > "$POSTERMAP.new" \
                   && mv "$POSTERMAP.new" "$POSTERMAP"
            fi
            ;;
    esac

    jq --arg src "$src" --arg path "/media/$name" \
       --argjson w "${w:-0}" --argjson h "${h:-0}" \
       '. + [{src: $src, path: $path, w: $w, h: $h}]' "$MAP" > "$MAP.new" \
       && mv "$MAP.new" "$MAP"
done <<EOF
$(jq -r '[.[].media[]? | .src, (.poster // empty)] | map(select(. != "")) | unique[]' "$POSTS")
EOF

echo "fetch-media: $downloaded new, $reused already present, $adopted self-hosted, $failed failed"

# Rewrite each media entry to the local path. An entry with no mapping (download
# failed) keeps its original src, so the page still shows something rather than
# silently dropping the post's whole point.
TMP_POSTS="$(mktemp)"
if jq --slurpfile map "$MAP" --slurpfile posters "$POSTERMAP" '
    ($map[0] | map({key: .src, value: .}) | from_entries) as $m
    | ($posters[0] | from_entries) as $pm
    | map(.media = ((.media // []) | map(
        . as $item
        | ($m[$item.src] // null) as $hit
        | (if $hit == null then $item
           else $item + { src: $hit.path, w: $hit.w, h: $hit.h }
           end)
        # The poster gets its path rewritten but NOT its dimensions: w/h describe
        # the video, and a poster is a differently-sized still of it. Feeding the
        # poster'\''s size to the <video> element would set the wrong aspect ratio.
        | if (.poster // "") == "" then .
          else . + { poster: (($m[.poster].path) // .poster) }
          end
        # Last resort, and only for media we host: the sibling poster frame
        # publish-media.sh uploaded next to the video. Runs after the rewrite
        # above so it sees the LOCAL src, and only fills a poster that is still
        # empty — a thumbnail the instance did provide always wins.
        | if ((.poster // "") == "") and (($pm[.src] // "") != "")
          then . + { poster: $pm[.src] }
          else . end)))
    ' "$POSTS" > "$TMP_POSTS" 2>/dev/null; then
    # Same reason as the chmod on each mirrored file: mktemp is 0600 and the
    # mode survives to production, where other users must read this.
    chmod 0644 "$TMP_POSTS"
    mv "$TMP_POSTS" "$POSTS"
else
    rm -f "$TMP_POSTS"
    echo "fetch-media: could not rewrite $POSTS, leaving it unchanged" >&2
    exit 1
fi

total=$(jq '[.[].media[]? | .src, (.poster // empty) | select(. != "")] | length' "$POSTS")
local_count=$(jq '[.[].media[]? | .src, (.poster // empty)
                  | select(startswith("/media/"))] | length' "$POSTS")
echo "fetch-media: $local_count of $total media entries served locally ($(du -sh "$MEDIA_DIR" | cut -f1) in $MEDIA_DIR)"
if [ "$local_count" -ne "$total" ]; then
    echo "fetch-media: $((total - local_count)) still point at their source — see the failures above" >&2
fi