This commit is contained in:
parent
fb2f6079cc
commit
934c94cb5c
50 changed files with 10464 additions and 758 deletions
166
tools/fetch-media.sh
Executable file
166
tools/fetch-media.sh
Executable file
|
|
@ -0,0 +1,166 @@
|
|||
#!/bin/sh
|
||||
# Mirror the media referenced by content/posts.json, and rewrite the entries to
|
||||
# point at our own copies.
|
||||
#
|
||||
# Run AFTER tools/fetch-posts.sh, which records the original URLs.
|
||||
#
|
||||
# WHY MIRROR rather than embed from the source:
|
||||
#
|
||||
# * Privacy. The privacy notice states that everything the browser loads comes
|
||||
# from catcrafts.net, and it should stay true. Embedding directly would send
|
||||
# every visitor's IP address to whichever instance hosts the file — an odd
|
||||
# thing to do on a site selling a privacy-focused phone.
|
||||
# * Durability. These posts ARE their media: the screen recording of VoLTE
|
||||
# working is the content. If the source instance deletes it or disappears,
|
||||
# a direct embed becomes a broken box and the post loses its point.
|
||||
# * Cost. One download per file, ever, instead of one per visitor. Kinder to
|
||||
# small instances than hotlinking them.
|
||||
#
|
||||
# Files are content-addressed (sha256 of the bytes), so a file already present is
|
||||
# never downloaded again and a changed file gets a new name — which makes the
|
||||
# long cache lifetime Caddy sets honest.
|
||||
#
|
||||
# usage: tools/fetch-media.sh [media-dir] (default: media/)
|
||||
#
|
||||
# On any single download failure the entry keeps its original URL and the script
|
||||
# carries on, so one dead file does not cost the whole page. Exits non-zero only
|
||||
# if it cannot do its job at all.
|
||||
|
||||
set -eu
|
||||
|
||||
MEDIA_DIR="${1:-media}"
|
||||
POSTS="content/posts.json"
|
||||
MAX_BYTES=$((64 * 1024 * 1024))
|
||||
|
||||
command -v jq >/dev/null 2>&1 || { echo "fetch-media: jq not found" >&2; exit 1; }
|
||||
[ -f "$POSTS" ] || { echo "fetch-media: $POSTS not found — run fetch-posts.sh first" >&2; exit 1; }
|
||||
|
||||
mkdir -p "$MEDIA_DIR"
|
||||
|
||||
# ffprobe gives real pixel dimensions, which become width/height attributes.
|
||||
# Without them the browser cannot reserve space and the text below jumps as each
|
||||
# image arrives; with them the layout is stable on first paint. Optional — the
|
||||
# markup degrades to no dimensions rather than failing.
|
||||
HAVE_FFPROBE=0
|
||||
command -v ffprobe >/dev/null 2>&1 && HAVE_FFPROBE=1
|
||||
|
||||
MAP="$(mktemp)"
|
||||
trap 'rm -f "$MAP"' EXIT
|
||||
printf '[]' > "$MAP"
|
||||
|
||||
downloaded=0
|
||||
reused=0
|
||||
failed=0
|
||||
|
||||
# Every distinct media URL across all posts, so a file shared by two posts is
|
||||
# fetched once. Video posters are in here too: a poster left pointing at the
|
||||
# source instance would leak a visitor IP on page load exactly like an embedded
|
||||
# image would, and it is the frame shown before anyone presses play.
|
||||
#
|
||||
# Fed by a here-document rather than a pipe so the counters below survive — in
|
||||
# `jq | while`, the loop runs in a subshell and every increment is discarded.
|
||||
while IFS= read -r src; do
|
||||
[ -n "$src" ] || continue
|
||||
|
||||
ext=$(printf '%s' "$src" | sed -E 's/.*\.([A-Za-z0-9]+)$/\1/' | tr 'A-Z' 'a-z')
|
||||
case "$ext" in
|
||||
mp4|webm|mov|webp|png|jpg|jpeg|gif|avif) ;;
|
||||
*) echo "fetch-media: skipping unexpected extension: $src" >&2; continue ;;
|
||||
esac
|
||||
|
||||
tmp="$(mktemp)"
|
||||
# --max-filesize refuses an oversized body before writing it; the explicit
|
||||
# size check afterwards covers servers that do not send Content-Length.
|
||||
if ! curl -fsSL --max-time 120 --max-filesize "$MAX_BYTES" \
|
||||
-A 'catcrafts.net-buildfetch/1.0 (+https://catcrafts.net)' \
|
||||
"$src" -o "$tmp" 2>/dev/null; then
|
||||
echo "fetch-media: download failed, keeping original URL: $src" >&2
|
||||
rm -f "$tmp"
|
||||
failed=$((failed + 1))
|
||||
continue
|
||||
fi
|
||||
if [ "$(wc -c < "$tmp")" -gt "$MAX_BYTES" ]; then
|
||||
echo "fetch-media: oversized, keeping original URL: $src" >&2
|
||||
rm -f "$tmp"
|
||||
failed=$((failed + 1))
|
||||
continue
|
||||
fi
|
||||
|
||||
hash=$(sha256sum "$tmp" | cut -c1-16)
|
||||
name="$hash.$ext"
|
||||
dest="$MEDIA_DIR/$name"
|
||||
|
||||
if [ -f "$dest" ]; then
|
||||
rm -f "$tmp"
|
||||
reused=$((reused + 1))
|
||||
else
|
||||
mv "$tmp" "$dest"
|
||||
chmod 0644 "$dest"
|
||||
downloaded=$((downloaded + 1))
|
||||
fi
|
||||
|
||||
# One query per dimension. Asking for both at once and splitting the CSV
|
||||
# looked simpler but was wrong: for some files ffprobe appends an empty
|
||||
# field, so `width,height` came back as "854x480x" and splitting on `x` gave
|
||||
# a height of "480x" — which the digit guard below then threw away, silently
|
||||
# costing the dimensions of exactly the videos that had the extra field.
|
||||
# `nk=1` prints the bare value, so there is nothing to split.
|
||||
w=0; h=0
|
||||
if [ "$HAVE_FFPROBE" = 1 ]; then
|
||||
pw=$(ffprobe -v error -select_streams v:0 -show_entries stream=width \
|
||||
-of default=nw=1:nk=1 "$dest" 2>/dev/null | head -n1 || true)
|
||||
ph=$(ffprobe -v error -select_streams v:0 -show_entries stream=height \
|
||||
-of default=nw=1:nk=1 "$dest" 2>/dev/null | head -n1 || true)
|
||||
case "$pw" in ''|*[!0-9]*) pw=0 ;; esac
|
||||
case "$ph" in ''|*[!0-9]*) ph=0 ;; esac
|
||||
# Both or neither: a lone dimension is worse than none, because the
|
||||
# browser derives the missing one from it and gets the aspect wrong.
|
||||
if [ "$pw" -gt 0 ] && [ "$ph" -gt 0 ]; then w=$pw; h=$ph; fi
|
||||
if [ "$w" = 0 ]; then
|
||||
echo "fetch-media: no dimensions for $name; layout will shift on load" >&2
|
||||
fi
|
||||
fi
|
||||
|
||||
jq --arg src "$src" --arg path "/media/$name" \
|
||||
--argjson w "${w:-0}" --argjson h "${h:-0}" \
|
||||
'. + [{src: $src, path: $path, w: $w, h: $h}]' "$MAP" > "$MAP.new" \
|
||||
&& mv "$MAP.new" "$MAP"
|
||||
done <<EOF
|
||||
$(jq -r '[.[].media[]? | .src, (.poster // empty)] | map(select(. != "")) | unique[]' "$POSTS")
|
||||
EOF
|
||||
|
||||
echo "fetch-media: $downloaded new, $reused already present, $failed failed"
|
||||
|
||||
# Rewrite each media entry to the local path. An entry with no mapping (download
|
||||
# failed) keeps its original src, so the page still shows something rather than
|
||||
# silently dropping the post's whole point.
|
||||
TMP_POSTS="$(mktemp)"
|
||||
if jq --slurpfile map "$MAP" '
|
||||
($map[0] | map({key: .src, value: .}) | from_entries) as $m
|
||||
| map(.media = ((.media // []) | map(
|
||||
. as $item
|
||||
| ($m[$item.src] // null) as $hit
|
||||
| (if $hit == null then $item
|
||||
else $item + { src: $hit.path, w: $hit.w, h: $hit.h }
|
||||
end)
|
||||
# The poster gets its path rewritten but NOT its dimensions: w/h describe
|
||||
# the video, and a poster is a differently-sized still of it. Feeding the
|
||||
# poster'\''s size to the <video> element would set the wrong aspect ratio.
|
||||
| if (.poster // "") == "" then .
|
||||
else . + { poster: (($m[.poster].path) // .poster) }
|
||||
end)))
|
||||
' "$POSTS" > "$TMP_POSTS" 2>/dev/null; then
|
||||
mv "$TMP_POSTS" "$POSTS"
|
||||
else
|
||||
rm -f "$TMP_POSTS"
|
||||
echo "fetch-media: could not rewrite $POSTS, leaving it unchanged" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
total=$(jq '[.[].media[]? | .src, (.poster // empty) | select(. != "")] | length' "$POSTS")
|
||||
local_count=$(jq '[.[].media[]? | .src, (.poster // empty)
|
||||
| select(startswith("/media/"))] | length' "$POSTS")
|
||||
echo "fetch-media: $local_count of $total media entries served locally ($(du -sh "$MEDIA_DIR" | cut -f1) in $MEDIA_DIR)"
|
||||
if [ "$local_count" -ne "$total" ]; then
|
||||
echo "fetch-media: $((total - local_count)) still point at their source — see the failures above" >&2
|
||||
fi
|
||||
Loading…
Reference in a new issue