Jorijn van der Graaf
38688a68f5
All checks were successful
Deploy / build-deploy (push) Successful in 6m20s
594 lines
29 KiB
Shell
Executable file
594 lines
29 KiB
Shell
Executable file
#!/bin/sh
|
|
# Mirror the media referenced by content/posts.json, and rewrite the entries to
|
|
# point at our own copies.
|
|
#
|
|
# Run AFTER tools/fetch-posts.sh, which records the original URLs.
|
|
#
|
|
# TWO KINDS OF MEDIA, one pipeline:
|
|
#
|
|
# * a post's headline file — `.media`, the recording or screenshot the post is
|
|
# about;
|
|
# * everything embedded inside the body — the screenshots a post argues with,
|
|
# which are just as much content and, until the body was hosted here, were
|
|
# never fetched at all.
|
|
#
|
|
# Both are content-addressed into the same directory, so a file used as one
|
|
# post's headline and quoted inside another's body is stored once. Body files
|
|
# are rewritten IN THE MARKDOWN TEXT (the body is still Markdown at this point)
|
|
# and additionally recorded in `.body_media`, which is where the renderer reads
|
|
# the dimensions, poster and H.264 fallback that Markdown syntax has nowhere to
|
|
# carry.
|
|
#
|
|
# WHY MIRROR rather than embed from the source:
|
|
#
|
|
# * Privacy. The privacy notice states that everything the browser loads comes
|
|
# from catcrafts.net, and it should stay true. Embedding directly would send
|
|
# every visitor's IP address to whichever instance hosts the file — an odd
|
|
# thing to do on a site selling a privacy-focused phone.
|
|
# * Durability. These posts ARE their media: the screen recording of VoLTE
|
|
# working is the content. If the source instance deletes it or disappears,
|
|
# a direct embed becomes a broken box and the post loses its point.
|
|
# * Cost. One download per file, ever, instead of one per visitor. Kinder to
|
|
# small instances than hotlinking them.
|
|
#
|
|
# Files are content-addressed (sha256 of the bytes), so a file already present is
|
|
# never downloaded again and a changed file gets a new name — which makes the
|
|
# long cache lifetime Caddy sets honest.
|
|
#
|
|
# usage: tools/fetch-media.sh [media-dir] (default: media/)
|
|
#
|
|
# On any single download failure the entry keeps its original URL and the script
|
|
# carries on, so one dead file does not cost the whole page. Exits non-zero only
|
|
# if it cannot do its job at all.
|
|
|
|
set -eu
|
|
|
|
MEDIA_DIR="${1:-media}"
|
|
POSTS="content/posts.json"
|
|
MAX_BYTES=$((64 * 1024 * 1024))
|
|
|
|
# What counts as a media reference inside a post body: an absolute URL or a path
|
|
# we have already rewritten, ending in a media extension.
|
|
#
|
|
# Local paths are in the pattern deliberately. Leaving them out looked right —
|
|
# nothing needs downloading twice — but it is what made a second run destructive
|
|
# rather than idempotent: the already-rewritten body references were not
|
|
# enumerated, so they never re-entered the mirror map, so the body_media sidecar
|
|
# came back with only the handful of entries that happened to still be absolute.
|
|
# Matching them means they are adopted from the mount and everything is rebuilt
|
|
# exactly as it was.
|
|
#
|
|
# One definition, passed to every jq that needs it, because three copies of a
|
|
# regex is three chances for one of them to drift.
|
|
MEDIA_REF_RE='(?:https?://|/media/)[^\s)\]"<>]+\.(?:mp4|webm|mov|webp|png|jpe?g|gif|avif)'
|
|
|
|
# Media we host ourselves, published by tools/publish-media.sh before the post
|
|
# that carries it exists. Such a URL is ALREADY the one the page should use, so
|
|
# there is nothing to fetch: the bytes are on the media mount, and downloading
|
|
# them back from our own web server would only mint a second copy under a second
|
|
# name. Skipping the download also skips MAX_BYTES, which is what a 167 MB
|
|
# recording on a third-party file host ran into — and it removes the last part of
|
|
# a build that could fail because someone else's server was slow, rate-limiting
|
|
# or gone.
|
|
OWN_ORIGIN="https://catcrafts.net/media/"
|
|
|
|
command -v jq >/dev/null 2>&1 || { echo "fetch-media: jq not found" >&2; exit 1; }
|
|
[ -f "$POSTS" ] || { echo "fetch-media: $POSTS not found — run fetch-posts.sh first" >&2; exit 1; }
|
|
|
|
mkdir -p "$MEDIA_DIR"
|
|
|
|
# ffprobe gives real pixel dimensions, which become width/height attributes.
|
|
# Without them the browser cannot reserve space and the text below jumps as each
|
|
# image arrives; with them the layout is stable on first paint. Optional — the
|
|
# markup degrades to no dimensions rather than failing.
|
|
HAVE_FFPROBE=0
|
|
command -v ffprobe >/dev/null 2>&1 && HAVE_FFPROBE=1
|
|
|
|
# ffmpeg does the still-image transcodes below. Optional in exactly the same
|
|
# way ffprobe is: without it every image is served as the single file the mirror
|
|
# downloaded, which is what this site did before the format ladder existed.
|
|
HAVE_FFMPEG=0
|
|
command -v ffmpeg >/dev/null 2>&1 && HAVE_FFMPEG=1
|
|
|
|
# Sets $w and $h for the file named in $1, or leaves both 0.
|
|
#
|
|
# One query per dimension. Asking for both at once and splitting the CSV looked
|
|
# simpler but was wrong: for some files ffprobe appends an empty field, so
|
|
# `width,height` came back as "854x480x" and splitting on `x` gave a height of
|
|
# "480x" — which the digit guard below then threw away, silently costing the
|
|
# dimensions of exactly the videos that had the extra field. `nk=1` prints the
|
|
# bare value, so there is nothing to split.
|
|
#
|
|
# ROTATION: a phone records 1920x1080 and attaches a display matrix rather than
|
|
# rotating the pixels, so the stream reads landscape while the video plays
|
|
# portrait. Believing the stream there reserves a landscape box for a portrait
|
|
# video — precisely the layout shift these attributes exist to prevent — so a
|
|
# quarter-turn swaps them. Files that went through publish-media.sh have the
|
|
# rotation baked into the pixels and report no matrix at all; this is for
|
|
# anything mirrored straight from a phone.
|
|
probe_dims() {
|
|
w=0; h=0
|
|
[ "$HAVE_FFPROBE" = 1 ] || return 0
|
|
pw=$(ffprobe -v error -select_streams v:0 -show_entries stream=width \
|
|
-of default=nw=1:nk=1 "$1" </dev/null 2>/dev/null | head -n1 || true)
|
|
ph=$(ffprobe -v error -select_streams v:0 -show_entries stream=height \
|
|
-of default=nw=1:nk=1 "$1" </dev/null 2>/dev/null | head -n1 || true)
|
|
rot=$(ffprobe -v error -select_streams v:0 \
|
|
-show_entries stream_side_data=rotation \
|
|
-of default=nw=1:nk=1 "$1" </dev/null 2>/dev/null | head -n1 || true)
|
|
case "$pw" in ''|*[!0-9]*) pw=0 ;; esac
|
|
case "$ph" in ''|*[!0-9]*) ph=0 ;; esac
|
|
# ffprobe reports this as a signed number that some builds print with a
|
|
# fractional part ("-90.000000"), so compare on the integer portion.
|
|
case "${rot%%.*}" in
|
|
90|-90|270|-270) t=$pw; pw=$ph; ph=$t ;;
|
|
esac
|
|
# Both or neither: a lone dimension is worse than none, because the browser
|
|
# derives the missing one from it and gets the aspect wrong.
|
|
if [ "$pw" -gt 0 ] && [ "$ph" -gt 0 ]; then w=$pw; h=$ph; fi
|
|
if [ "$w" = 0 ]; then
|
|
echo "fetch-media: no dimensions for $1; layout will shift on load" >&2
|
|
fi
|
|
}
|
|
|
|
# Derive the two renditions a mirrored image is served between: AVIF above it
|
|
# and PNG below. Sets $avif_name / $png_name to the sibling file names, or
|
|
# leaves one empty when that rendition could not be produced — :Media then drops
|
|
# the tier rather than pointing at a file that is not on the mount.
|
|
#
|
|
# Siblings are named after the source file, which is itself the hash of its
|
|
# bytes, so a rendition already present is never re-encoded and a changed source
|
|
# gets new names. Only genuinely new images cost encoder time; a rebuild costs
|
|
# none, which is what keeps this off the critical path of every deploy.
|
|
#
|
|
# WHY BOTH TIERS. AVIF is smaller than the WebP the instances serve (~15% on
|
|
# these screenshots, far more on photographs) and is what almost every visitor
|
|
# actually receives. PNG is lossless and universally understood, which is what
|
|
# makes it a fallback worth having — but it is also several times the size of
|
|
# the WebP beside it, so the <picture> offers the mirrored original in between
|
|
# and the PNG is reached only by a browser that understands neither of the
|
|
# other two.
|
|
#
|
|
# The settings, measured against these files rather than guessed:
|
|
# crf 26, cpu-used 6 SSIM 0.997 against the source and still smaller than
|
|
# it, at roughly half a second per image.
|
|
# yuv444p these are screenshots of text. Re-subsampling chroma
|
|
# that pict-rs already subsampled once fringes coloured
|
|
# text visibly, and full chroma costs about 3% here.
|
|
transcode_image() {
|
|
avif_name=""
|
|
png_name=""
|
|
_file="$1"
|
|
_name="$2"
|
|
_base="${_name%.*}"
|
|
|
|
# Already that format: serve the mirrored file as the tier rather than
|
|
# re-encoding it into a second copy of itself.
|
|
case "$_name" in *.avif) avif_name="$_name" ;; esac
|
|
case "$_name" in *.png) png_name="$_name" ;; esac
|
|
[ "$HAVE_FFMPEG" = 1 ] || return 0
|
|
|
|
# An animated source is not a still, and -frames:v 1 would silently freeze
|
|
# it. Leave it entirely alone: one moving GIF is worth more than three
|
|
# copies of its first frame. nb_frames is N/A for WebP, so the frames have
|
|
# to actually be counted — ~75 ms on a 3 MP image, once per new file.
|
|
_frames=$(ffprobe -v error -select_streams v:0 -count_frames \
|
|
-show_entries stream=nb_read_frames \
|
|
-of default=nw=1:nk=1 "$_file" </dev/null 2>/dev/null | head -n1)
|
|
case "$_frames" in
|
|
''|*[!0-9]*|1) ;; # unknown or a single frame: a still
|
|
*) echo "fetch-media: $_name is animated, serving it as one file" >&2
|
|
return 0 ;;
|
|
esac
|
|
|
|
# Alpha has to survive the transcode: an image with a transparent corner
|
|
# encoded into a format with no alpha plane gains an opaque black one.
|
|
_pixfmt=$(ffprobe -v error -select_streams v:0 -show_entries stream=pix_fmt \
|
|
-of default=nw=1:nk=1 "$_file" </dev/null 2>/dev/null | head -n1)
|
|
case "$_pixfmt" in
|
|
yuva*|rgba*|bgra*|argb*|abgr*|gbrap*|ya8|ya16*|pal8) _avif_pix=yuva444p ;;
|
|
*) _avif_pix=yuv444p ;;
|
|
esac
|
|
|
|
# Encoded to a .part, checked, and only then renamed — so an interrupted or
|
|
# wrong-format encode cannot leave a file the next run adopts as finished.
|
|
if [ -z "$avif_name" ]; then
|
|
_cand="$_base.avif"
|
|
if rendition_ok "$_cand" av1; then
|
|
avif_name="$_cand"
|
|
elif encode_rendition "$_file" "$_cand" av1 \
|
|
-c:v libaom-av1 -still-picture 1 -crf 26 -cpu-used 6 \
|
|
-pix_fmt "$_avif_pix" -f avif; then
|
|
avif_name="$_cand"
|
|
fi
|
|
fi
|
|
|
|
if [ -z "$png_name" ]; then
|
|
_cand="$_base.png"
|
|
if rendition_ok "$_cand" png; then
|
|
png_name="$_cand"
|
|
elif encode_rendition "$_file" "$_cand" png -c:v png -f image2; then
|
|
png_name="$_cand"
|
|
fi
|
|
fi
|
|
}
|
|
|
|
# The codec ffprobe reports for a file, or empty when it cannot say.
|
|
codec_of() {
|
|
[ "$HAVE_FFPROBE" = 1 ] || return 0
|
|
ffprobe -v error -select_streams v:0 -show_entries stream=codec_name \
|
|
-of default=nw=1:nk=1 "$1" </dev/null 2>/dev/null | head -n1
|
|
}
|
|
|
|
# rendition_ok NAME EXPECTED_CODEC — true when the file is already on the mount
|
|
# AND really is that codec.
|
|
#
|
|
# The second half is what makes the mount self-healing. Renditions are adopted
|
|
# by name and never re-derived, so anything wrong that once landed there would
|
|
# be trusted forever — which is exactly what a run of MJPEG files under .png
|
|
# names would have been. A file that fails re-encodes over the top instead.
|
|
rendition_ok() {
|
|
[ -f "$MEDIA_DIR/$1" ] || return 1
|
|
_have=$(codec_of "$MEDIA_DIR/$1")
|
|
# No ffprobe to ask: trust what is there rather than re-encoding every
|
|
# image on every build.
|
|
[ -n "$_have" ] || return 0
|
|
[ "$_have" = "$2" ] && return 0
|
|
echo "fetch-media: $1 on the mount is '$_have', not '$2' — re-encoding it" >&2
|
|
return 1
|
|
}
|
|
|
|
# encode_rendition SRC OUTNAME EXPECTED_CODEC ffmpeg-args...
|
|
#
|
|
# Runs the encode into a .part, verifies the result really is the codec asked
|
|
# for, and only then publishes it. Returns non-zero (leaving nothing behind) if
|
|
# either step fails, which drops that tier rather than shipping a broken one.
|
|
#
|
|
# The verification is not paranoia. ffmpeg picks an encoder from the MUXER when
|
|
# one is not named, and the image2 muxer defaults to MJPEG — so `-f image2
|
|
# out.png` silently produced a run of lossy JPEGs sitting under .png names, which
|
|
# the page then advertised to browsers as image/png. The codec is pinned by the
|
|
# callers above; this is the check that the pin held.
|
|
#
|
|
# -nostdin AND </dev/null, both deliberately. ffmpeg reads standard input for
|
|
# interactive keystrokes, and this runs inside `while read src; do ... done
|
|
# <<EOF` — so it happily ate the front of the NEXT url off the here-document.
|
|
# The symptom was a handful of downloads failing per run with mangled hosts in
|
|
# the log ("ttps://", "s://", "tps://" — a different number of bytes swallowed
|
|
# each time), which then left those posts pointing at third-party media and
|
|
# failed the origin check in CI. It only bit when there was something to encode,
|
|
# so a rerun always "fixed" it.
|
|
encode_rendition() {
|
|
_src="$1"; _out="$2"; _want="$3"
|
|
shift 3
|
|
if ! ffmpeg -nostdin -y -v error -i "$_src" -frames:v 1 "$@" \
|
|
"$MEDIA_DIR/$_out.part" </dev/null 2>/dev/null; then
|
|
rm -f "$MEDIA_DIR/$_out.part"
|
|
echo "fetch-media: could not encode $_out, serving without that tier" >&2
|
|
return 1
|
|
fi
|
|
_got=$(ffprobe -v error -select_streams v:0 -show_entries stream=codec_name \
|
|
-of default=nw=1:nk=1 "$MEDIA_DIR/$_out.part" </dev/null 2>/dev/null | head -n1)
|
|
if [ "$_got" != "$_want" ]; then
|
|
rm -f "$MEDIA_DIR/$_out.part"
|
|
echo "fetch-media: $_out came out as '$_got', expected '$_want' — discarding it" >&2
|
|
return 1
|
|
fi
|
|
mv "$MEDIA_DIR/$_out.part" "$MEDIA_DIR/$_out"
|
|
chmod 0644 "$MEDIA_DIR/$_out"
|
|
encoded=$((encoded + 1))
|
|
return 0
|
|
}
|
|
|
|
MAP="$(mktemp)"
|
|
POSTERMAP="$(mktemp)"
|
|
FALLBACKMAP="$(mktemp)"
|
|
AVIFMAP="$(mktemp)"
|
|
POSTERONLY="$(mktemp)"
|
|
trap 'rm -f "$MAP" "$POSTERMAP" "$FALLBACKMAP" "$AVIFMAP" "$POSTERONLY"' EXIT
|
|
printf '[]' > "$MAP"
|
|
printf '[]' > "$POSTERMAP"
|
|
printf '[]' > "$FALLBACKMAP"
|
|
printf '[]' > "$AVIFMAP"
|
|
|
|
# URLs that are ONLY ever a video's poster frame. They are skipped by the
|
|
# transcode above, because `poster` takes exactly one URL: a <video> cannot
|
|
# negotiate a format the way <picture> can, so the renditions would be files
|
|
# nothing is able to ask for. A file that is a poster somewhere and an ordinary
|
|
# image somewhere else is not in this list and is transcoded normally.
|
|
jq -r --arg re "$MEDIA_REF_RE" \
|
|
'([.[].media[]? | .poster // empty] | map(select(. != "")) | unique) as $posters
|
|
| ([.[].media[]? | .src] + [.[] | .body // "" | scan($re)] | unique) as $srcs
|
|
| ($posters - $srcs) | .[]' "$POSTS" > "$POSTERONLY" 2>/dev/null || true
|
|
|
|
downloaded=0
|
|
reused=0
|
|
adopted=0
|
|
failed=0
|
|
encoded=0
|
|
|
|
# Every distinct media URL across all posts, so a file shared by two posts is
|
|
# fetched once. Video posters are in here too: a poster left pointing at the
|
|
# source instance would leak a visitor IP on page load exactly like an embedded
|
|
# image would, and it is the frame shown before anyone presses play.
|
|
#
|
|
# Body URLs are found by pattern rather than by parsing Markdown: anything that
|
|
# looks like an absolute URL ending in a media extension is mirrored, whether it
|
|
# was written as an embed, as a link, or bare. That is deliberately wider than
|
|
# "images the body displays" — the origin rule covers href as well as src, and a
|
|
# link whose target is a .webp on someone else's instance is still a third-party
|
|
# address on our page. Already-rewritten paths start with /media/ and so do not
|
|
# match, which is what makes re-running this a no-op.
|
|
#
|
|
# Fed by a here-document rather than a pipe so the counters below survive — in
|
|
# `jq | while`, the loop runs in a subshell and every increment is discarded.
|
|
#
|
|
# The list arrives on fd 3, not stdin, and the loop reads it from there. That is
|
|
# not decoration: ffmpeg below reads standard input for interactive keystrokes
|
|
# and swallowed the front of the next URL straight off the here-document, which
|
|
# cost a few posts their mirrored media on every run that had something to
|
|
# encode. ffmpeg is told -nostdin as well, but keeping the list off stdin
|
|
# entirely is what stops the next tool added to this loop from doing it again.
|
|
while IFS= read -r src <&3; do
|
|
[ -n "$src" ] || continue
|
|
|
|
# Ours already — adopt the file on the mount and do no network at all.
|
|
# A URL that is on our origin but names a file that is NOT on the mount is
|
|
# left alone rather than invented: that is a post published without its
|
|
# media, and keeping the original URL makes the e2e origin check fail
|
|
# loudly instead of shipping a 404 in a <video> tag.
|
|
case "$src" in
|
|
"$OWN_ORIGIN"*)
|
|
name=${src#"$OWN_ORIGIN"}
|
|
# Refuse anything that is not a bare filename. A path separator or a
|
|
# traversal segment arriving from a post URL must never reach a path we
|
|
# then read or publish.
|
|
case "$name" in
|
|
''|*/*|*..*)
|
|
echo "fetch-media: refusing suspicious own-origin URL: $src" >&2
|
|
failed=$((failed + 1)); continue ;;
|
|
esac
|
|
# A .h264.mp4 URL is the fediverse-facing form of an AV1 video: the
|
|
# post links the encoding everything can play, because that link is
|
|
# fetched raw by Lemmy apps. Our page can negotiate, so when the AV1
|
|
# sibling is on the mount it becomes the primary <source> and the
|
|
# H.264 drops to the fallback (picked up by name below).
|
|
case "$name" in
|
|
*.h264.mp4)
|
|
av1="${name%.h264.mp4}.mp4"
|
|
[ -f "$MEDIA_DIR/$av1" ] && name="$av1"
|
|
;;
|
|
esac
|
|
dest="$MEDIA_DIR/$name"
|
|
if [ ! -f "$dest" ]; then
|
|
echo "fetch-media: $name not on the media mount, keeping original URL: $src" >&2
|
|
failed=$((failed + 1)); continue
|
|
fi
|
|
adopted=$((adopted + 1))
|
|
;;
|
|
/media/*)
|
|
# Already rewritten by an earlier run of this script. Adopt the file on
|
|
# the mount rather than trying to fetch our own path as though it were a
|
|
# URL — which is what makes running this twice a no-op instead of a way
|
|
# to lose every rewrite it made the first time. The script is meant to
|
|
# follow fetch-posts.sh, but "meant to" is not a guarantee, and the
|
|
# failure was silent: the body_media list simply came back empty.
|
|
name=${src#/media/}
|
|
case "$name" in
|
|
''|*/*|*..*)
|
|
echo "fetch-media: refusing suspicious local path: $src" >&2
|
|
failed=$((failed + 1)); continue ;;
|
|
esac
|
|
dest="$MEDIA_DIR/$name"
|
|
if [ ! -f "$dest" ]; then
|
|
echo "fetch-media: $name not on the media mount, leaving it alone: $src" >&2
|
|
failed=$((failed + 1)); continue
|
|
fi
|
|
adopted=$((adopted + 1))
|
|
;;
|
|
*)
|
|
ext=$(printf '%s' "$src" | sed -E 's/.*\.([A-Za-z0-9]+)$/\1/' | tr 'A-Z' 'a-z')
|
|
case "$ext" in
|
|
mp4|webm|mov|webp|png|jpg|jpeg|gif|avif) ;;
|
|
*) echo "fetch-media: skipping unexpected extension: $src" >&2; continue ;;
|
|
esac
|
|
|
|
tmp="$(mktemp)"
|
|
# --max-filesize refuses an oversized body before writing it; the explicit
|
|
# size check afterwards covers servers that do not send Content-Length.
|
|
if ! curl -fsSL --max-time 120 --max-filesize "$MAX_BYTES" \
|
|
-A 'catcrafts.net-buildfetch/1.0 (+https://catcrafts.net)' \
|
|
"$src" -o "$tmp" 2>/dev/null; then
|
|
echo "fetch-media: download failed, keeping original URL: $src" >&2
|
|
rm -f "$tmp"
|
|
failed=$((failed + 1))
|
|
continue
|
|
fi
|
|
if [ "$(wc -c < "$tmp")" -gt "$MAX_BYTES" ]; then
|
|
echo "fetch-media: oversized, keeping original URL: $src" >&2
|
|
rm -f "$tmp"
|
|
failed=$((failed + 1))
|
|
continue
|
|
fi
|
|
|
|
hash=$(sha256sum "$tmp" | cut -c1-16)
|
|
name="$hash.$ext"
|
|
dest="$MEDIA_DIR/$name"
|
|
|
|
if [ -f "$dest" ]; then
|
|
rm -f "$tmp"
|
|
reused=$((reused + 1))
|
|
else
|
|
mv "$tmp" "$dest"
|
|
chmod 0644 "$dest"
|
|
downloaded=$((downloaded + 1))
|
|
fi
|
|
;;
|
|
esac
|
|
|
|
probe_dims "$dest"
|
|
|
|
# A self-hosted video has no Lemmy thumbnail to mirror when the instance
|
|
# cannot decode it — AV1 is the common case, since pict-rs will not generate
|
|
# a still from one. publish-media.sh uploads a poster frame alongside the
|
|
# video under the video's own hash, so look for that sibling and offer it to
|
|
# the rewrite below. Without a poster a preload="metadata" video is a black
|
|
# box until someone presses play.
|
|
case "$name" in
|
|
*.mp4|*.webm|*.mov)
|
|
# Siblings are named after the AV1's hash, so strip the .h264
|
|
# marker too — a video adopted as X.h264.mp4 (its AV1 never
|
|
# published) still finds X.poster.webp.
|
|
base="${name%.*}"
|
|
base="${base%.h264}"
|
|
sibling="$base.poster.webp"
|
|
if [ -f "$MEDIA_DIR/$sibling" ]; then
|
|
jq --arg k "/media/$name" --arg v "/media/$sibling" \
|
|
'. + [{key: $k, value: $v}]' "$POSTERMAP" > "$POSTERMAP.new" \
|
|
&& mv "$POSTERMAP.new" "$POSTERMAP"
|
|
fi
|
|
# The H.264 sibling publish-media.sh uploaded next to the AV1.
|
|
# Rendered as a second <source> so browsers without AV1 (Safari
|
|
# before 17, Apple hardware without the decoder) get a file they
|
|
# can play instead of an element that will not. The guard against
|
|
# naming itself covers a video adopted as X.h264.mp4 whose AV1 is
|
|
# not on the mount.
|
|
fb="$base.h264.mp4"
|
|
if [ "$fb" != "$name" ] && [ -f "$MEDIA_DIR/$fb" ]; then
|
|
jq --arg k "/media/$name" --arg v "/media/$fb" \
|
|
'. + [{key: $k, value: $v}]' "$FALLBACKMAP" > "$FALLBACKMAP.new" \
|
|
&& mv "$FALLBACKMAP.new" "$FALLBACKMAP"
|
|
fi
|
|
;;
|
|
*.webp|*.png|*.jpg|*.jpeg|*.gif|*.avif)
|
|
# The AVIF and PNG tiers this image is served between. Both maps are
|
|
# keyed by the LOCAL path, like the video ones above, so the rewrite
|
|
# below can look them up from the src it has just written.
|
|
if grep -qxF "$src" "$POSTERONLY" 2>/dev/null; then
|
|
: # poster-only; see POSTERONLY above
|
|
else
|
|
transcode_image "$dest" "$name"
|
|
if [ -n "$avif_name" ]; then
|
|
jq --arg k "/media/$name" --arg v "/media/$avif_name" \
|
|
'. + [{key: $k, value: $v}]' "$AVIFMAP" > "$AVIFMAP.new" \
|
|
&& mv "$AVIFMAP.new" "$AVIFMAP"
|
|
fi
|
|
if [ -n "$png_name" ]; then
|
|
jq --arg k "/media/$name" --arg v "/media/$png_name" \
|
|
'. + [{key: $k, value: $v}]' "$FALLBACKMAP" > "$FALLBACKMAP.new" \
|
|
&& mv "$FALLBACKMAP.new" "$FALLBACKMAP"
|
|
fi
|
|
fi
|
|
;;
|
|
esac
|
|
|
|
jq --arg src "$src" --arg path "/media/$name" \
|
|
--argjson w "${w:-0}" --argjson h "${h:-0}" \
|
|
'. + [{src: $src, path: $path, w: $w, h: $h}]' "$MAP" > "$MAP.new" \
|
|
&& mv "$MAP.new" "$MAP"
|
|
done 3<<EOF
|
|
$(jq -r --arg re "$MEDIA_REF_RE" \
|
|
'[ (.[].media[]? | .src, (.poster // empty)),
|
|
(.[] | .body // "" | scan($re)) ]
|
|
| map(select(. != "")) | unique[]' "$POSTS")
|
|
EOF
|
|
|
|
echo "fetch-media: $downloaded new, $reused already present, $adopted self-hosted, $failed failed"
|
|
echo "fetch-media: $encoded image rendition(s) encoded this run"
|
|
|
|
# Rewrite each media entry to the local path. An entry with no mapping (download
|
|
# failed) keeps its original src, so the page still shows something rather than
|
|
# silently dropping the post's whole point.
|
|
TMP_POSTS="$(mktemp)"
|
|
if jq --slurpfile map "$MAP" --slurpfile posters "$POSTERMAP" \
|
|
--slurpfile fallbacks "$FALLBACKMAP" --slurpfile avifs "$AVIFMAP" '
|
|
($map[0] | map({key: .src, value: .}) | from_entries) as $m
|
|
| ($posters[0] | from_entries) as $pm
|
|
| ($fallbacks[0] | from_entries) as $fm
|
|
| ($avifs[0] | from_entries) as $am
|
|
| map(.media = ((.media // []) | map(
|
|
. as $item
|
|
| ($m[$item.src] // null) as $hit
|
|
| (if $hit == null then $item
|
|
else $item + { src: $hit.path, w: $hit.w, h: $hit.h }
|
|
end)
|
|
# The poster gets its path rewritten but NOT its dimensions: w/h describe
|
|
# the video, and a poster is a differently-sized still of it. Feeding the
|
|
# poster'\''s size to the <video> element would set the wrong aspect ratio.
|
|
| if (.poster // "") == "" then .
|
|
else . + { poster: (($m[.poster].path) // .poster) }
|
|
end
|
|
# Last resort, and only for media we host: the sibling poster frame
|
|
# publish-media.sh uploaded next to the video. Runs after the rewrite
|
|
# above so it sees the LOCAL src, and only fills a poster that is still
|
|
# empty — a thumbnail the instance did provide always wins.
|
|
| if ((.poster // "") == "") and (($pm[.src] // "") != "")
|
|
then . + { poster: $pm[.src] }
|
|
else . end
|
|
# H.264 fallback, keyed by the LOCAL src like the poster map. Only ever
|
|
# set for media on our own mount — a mirrored third-party file has no
|
|
# sibling to find.
|
|
| if (($fm[.src] // "") != "")
|
|
then . + { fallback: $fm[.src] }
|
|
else . end
|
|
# The AVIF tier, keyed by the LOCAL src like the two maps above. Only
|
|
# ever set for images, and only when the encode actually produced one.
|
|
| if (($am[.src] // "") != "")
|
|
then . + { avif: $am[.src] }
|
|
else . end)))
|
|
# ── the body ──────────────────────────────────────────────────────
|
|
#
|
|
# Substitution is literal (split/join, not gsub), because these URLs are
|
|
# full of regex metacharacters and a mirrored path must land in the text
|
|
# exactly as written. A URL with no mapping — its download failed — is left
|
|
# alone, so the post still shows the image rather than losing it; the
|
|
# accounting at the end of this script reports that as media still pointing
|
|
# at its source.
|
|
| ($m | to_entries) as $subs
|
|
| map(.body = (reduce $subs[] as $s ((.body // "");
|
|
split($s.key) | join($s.value.path))))
|
|
# Everything the rewritten body now points at, as records the renderer can
|
|
# read: Markdown has nowhere to put a width, a poster frame or a second
|
|
# source, so the sidecar list is how an inline video gets the same treatment
|
|
# as a headline one. Keyed by the LOCAL path, which is what the body says
|
|
# by this point.
|
|
| ($m | map({ key: .path, value: . }) | from_entries) as $byPath
|
|
| map(.body_media = ([ (.body // "")
|
|
| scan("/media/[A-Za-z0-9._-]+")
|
|
| . as $path
|
|
| select($byPath[$path] != null)
|
|
| { src: $path,
|
|
kind: (if ($path | test("\\.(?:mp4|webm|mov)$"))
|
|
then "video" else "image" end),
|
|
poster: ($pm[$path] // ""),
|
|
fallback: ($fm[$path] // ""),
|
|
avif: ($am[$path] // ""),
|
|
w: ($byPath[$path].w // 0),
|
|
h: ($byPath[$path].h // 0) } ]
|
|
| unique_by(.src)))
|
|
' "$POSTS" > "$TMP_POSTS" 2>/dev/null; then
|
|
# Same reason as the chmod on each mirrored file: mktemp is 0600 and the
|
|
# mode survives to production, where other users must read this.
|
|
chmod 0644 "$TMP_POSTS"
|
|
mv "$TMP_POSTS" "$POSTS"
|
|
else
|
|
rm -f "$TMP_POSTS"
|
|
echo "fetch-media: could not rewrite $POSTS, leaving it unchanged" >&2
|
|
exit 1
|
|
fi
|
|
|
|
# Counted over the bodies too, because a body URL that never mirrored is the
|
|
# same privacy leak as a card one and must not be reported as success.
|
|
total=$(jq --arg re "$MEDIA_REF_RE" \
|
|
'[ (.[].media[]? | .src, (.poster // empty)),
|
|
(.[] | .body // "" | scan($re)),
|
|
(.[].body_media[]? | .src, (.poster // empty)) ]
|
|
| map(select(. != "")) | unique | length' "$POSTS")
|
|
local_count=$(jq '[ (.[].media[]? | .src, (.poster // empty)),
|
|
(.[].body_media[]? | .src, (.poster // empty)) ]
|
|
| map(select(startswith("/media/"))) | length' "$POSTS")
|
|
inline=$(jq '[.[].body_media[]?] | length' "$POSTS")
|
|
echo "fetch-media: $local_count of $total media entries served locally ($inline of them embedded in post bodies; $(du -sh "$MEDIA_DIR" | cut -f1) in $MEDIA_DIR)"
|
|
if [ "$local_count" -ne "$total" ]; then
|
|
echo "fetch-media: $((total - local_count)) still point at their source — see the failures above" >&2
|
|
fi
|