catcrafts.net/tools/fetch-media.sh

#!/bin/sh
# Mirror the media referenced by content/posts.json, and rewrite the entries to
# point at our own copies.
#
# Run AFTER tools/fetch-posts.sh, which records the original URLs.
#
# WHY MIRROR rather than embed from the source:
#
#   * Privacy. The privacy notice states that everything the browser loads comes
#     from catcrafts.net, and it should stay true. Embedding directly would send
#     every visitor's IP address to whichever instance hosts the file — an odd
#     thing to do on a site selling a privacy-focused phone.
#   * Durability. These posts ARE their media: the screen recording of VoLTE
#     working is the content. If the source instance deletes it or disappears,
#     a direct embed becomes a broken box and the post loses its point.
#   * Cost. One download per file, ever, instead of one per visitor. Kinder to
#     small instances than hotlinking them.
#
# Files are content-addressed (sha256 of the bytes), so a file already present is
# never downloaded again and a changed file gets a new name — which makes the
# long cache lifetime Caddy sets honest.
#
#   usage: tools/fetch-media.sh [media-dir]      (default: media/)
#
# On any single download failure the entry keeps its original URL and the script
# carries on, so one dead file does not cost the whole page. Exits non-zero only
# if it cannot do its job at all.

set -eu

MEDIA_DIR="${1:-media}"
POSTS="content/posts.json"
MAX_BYTES=$((64 * 1024 * 1024))

command -v jq >/dev/null 2>&1 || { echo "fetch-media: jq not found" >&2; exit 1; }
[ -f "$POSTS" ] || { echo "fetch-media: $POSTS not found — run fetch-posts.sh first" >&2; exit 1; }

mkdir -p "$MEDIA_DIR"

# ffprobe gives real pixel dimensions, which become width/height attributes.
# Without them the browser cannot reserve space and the text below jumps as each
# image arrives; with them the layout is stable on first paint. Optional — the
# markup degrades to no dimensions rather than failing.
HAVE_FFPROBE=0
command -v ffprobe >/dev/null 2>&1 && HAVE_FFPROBE=1

MAP="$(mktemp)"
trap 'rm -f "$MAP"' EXIT
printf '[]' > "$MAP"

downloaded=0
reused=0
failed=0

# Every distinct media URL across all posts, so a file shared by two posts is
# fetched once. Video posters are in here too: a poster left pointing at the
# source instance would leak a visitor IP on page load exactly like an embedded
# image would, and it is the frame shown before anyone presses play.
#
# Fed by a here-document rather than a pipe so the counters below survive — in
# `jq | while`, the loop runs in a subshell and every increment is discarded.
while IFS= read -r src; do
    [ -n "$src" ] || continue

    ext=$(printf '%s' "$src" | sed -E 's/.*\.([A-Za-z0-9]+)$/\1/' | tr 'A-Z' 'a-z')
    case "$ext" in
        mp4|webm|mov|webp|png|jpg|jpeg|gif|avif) ;;
        *) echo "fetch-media: skipping unexpected extension: $src" >&2; continue ;;
    esac

    tmp="$(mktemp)"
    # --max-filesize refuses an oversized body before writing it; the explicit
    # size check afterwards covers servers that do not send Content-Length.
    if ! curl -fsSL --max-time 120 --max-filesize "$MAX_BYTES" \
              -A 'catcrafts.net-buildfetch/1.0 (+https://catcrafts.net)' \
              "$src" -o "$tmp" 2>/dev/null; then
        echo "fetch-media: download failed, keeping original URL: $src" >&2
        rm -f "$tmp"
        failed=$((failed + 1))
        continue
    fi
    if [ "$(wc -c < "$tmp")" -gt "$MAX_BYTES" ]; then
        echo "fetch-media: oversized, keeping original URL: $src" >&2
        rm -f "$tmp"
        failed=$((failed + 1))
        continue
    fi

    hash=$(sha256sum "$tmp" | cut -c1-16)
    name="$hash.$ext"
    dest="$MEDIA_DIR/$name"

    if [ -f "$dest" ]; then
        rm -f "$tmp"
        reused=$((reused + 1))
    else
        mv "$tmp" "$dest"
        chmod 0644 "$dest"
        downloaded=$((downloaded + 1))
    fi

    # One query per dimension. Asking for both at once and splitting the CSV
    # looked simpler but was wrong: for some files ffprobe appends an empty
    # field, so `width,height` came back as "854x480x" and splitting on `x` gave
    # a height of "480x" — which the digit guard below then threw away, silently
    # costing the dimensions of exactly the videos that had the extra field.
    # `nk=1` prints the bare value, so there is nothing to split.
    w=0; h=0
    if [ "$HAVE_FFPROBE" = 1 ]; then
        pw=$(ffprobe -v error -select_streams v:0 -show_entries stream=width \
                     -of default=nw=1:nk=1 "$dest" 2>/dev/null | head -n1 || true)
        ph=$(ffprobe -v error -select_streams v:0 -show_entries stream=height \
                     -of default=nw=1:nk=1 "$dest" 2>/dev/null | head -n1 || true)
        case "$pw" in ''|*[!0-9]*) pw=0 ;; esac
        case "$ph" in ''|*[!0-9]*) ph=0 ;; esac
        # Both or neither: a lone dimension is worse than none, because the
        # browser derives the missing one from it and gets the aspect wrong.
        if [ "$pw" -gt 0 ] && [ "$ph" -gt 0 ]; then w=$pw; h=$ph; fi
        if [ "$w" = 0 ]; then
            echo "fetch-media: no dimensions for $name; layout will shift on load" >&2
        fi
    fi

    jq --arg src "$src" --arg path "/media/$name" \
       --argjson w "${w:-0}" --argjson h "${h:-0}" \
       '. + [{src: $src, path: $path, w: $w, h: $h}]' "$MAP" > "$MAP.new" \
       && mv "$MAP.new" "$MAP"
done <<EOF
$(jq -r '[.[].media[]? | .src, (.poster // empty)] | map(select(. != "")) | unique[]' "$POSTS")
EOF

echo "fetch-media: $downloaded new, $reused already present, $failed failed"

# Rewrite each media entry to the local path. An entry with no mapping (download
# failed) keeps its original src, so the page still shows something rather than
# silently dropping the post's whole point.
TMP_POSTS="$(mktemp)"
if jq --slurpfile map "$MAP" '
    ($map[0] | map({key: .src, value: .}) | from_entries) as $m
    | map(.media = ((.media // []) | map(
        . as $item
        | ($m[$item.src] // null) as $hit
        | (if $hit == null then $item
           else $item + { src: $hit.path, w: $hit.w, h: $hit.h }
           end)
        # The poster gets its path rewritten but NOT its dimensions: w/h describe
        # the video, and a poster is a differently-sized still of it. Feeding the
        # poster'\''s size to the <video> element would set the wrong aspect ratio.
        | if (.poster // "") == "" then .
          else . + { poster: (($m[.poster].path) // .poster) }
          end)))
    ' "$POSTS" > "$TMP_POSTS" 2>/dev/null; then
    # Same reason as the chmod on each mirrored file: mktemp is 0600 and the
    # mode survives to production, where other users must read this.
    chmod 0644 "$TMP_POSTS"
    mv "$TMP_POSTS" "$POSTS"
else
    rm -f "$TMP_POSTS"
    echo "fetch-media: could not rewrite $POSTS, leaving it unchanged" >&2
    exit 1
fi

total=$(jq '[.[].media[]? | .src, (.poster // empty) | select(. != "")] | length' "$POSTS")
local_count=$(jq '[.[].media[]? | .src, (.poster // empty)
                  | select(startswith("/media/"))] | length' "$POSTS")
echo "fetch-media: $local_count of $total media entries served locally ($(du -sh "$MEDIA_DIR" | cut -f1) in $MEDIA_DIR)"
if [ "$local_count" -ne "$total" ]; then
    echo "fetch-media: $((total - local_count)) still point at their source — see the failures above" >&2
fi