This commit is contained in:
parent
2fa6e70af1
commit
6841623e23
17 changed files with 2306 additions and 148 deletions
|
|
@ -4,6 +4,21 @@
|
|||
#
|
||||
# Run AFTER tools/fetch-posts.sh, which records the original URLs.
|
||||
#
|
||||
# TWO KINDS OF MEDIA, one pipeline:
|
||||
#
|
||||
# * a post's headline file — `.media`, the recording or screenshot the post is
|
||||
# about;
|
||||
# * everything embedded inside the body — the screenshots a post argues with,
|
||||
# which are just as much content and, until the body was hosted here, were
|
||||
# never fetched at all.
|
||||
#
|
||||
# Both are content-addressed into the same directory, so a file used as one
|
||||
# post's headline and quoted inside another's body is stored once. Body files
|
||||
# are rewritten IN THE MARKDOWN TEXT (the body is still Markdown at this point)
|
||||
# and additionally recorded in `.body_media`, which is where the renderer reads
|
||||
# the dimensions, poster and H.264 fallback that Markdown syntax has nowhere to
|
||||
# carry.
|
||||
#
|
||||
# WHY MIRROR rather than embed from the source:
|
||||
#
|
||||
# * Privacy. The privacy notice states that everything the browser loads comes
|
||||
|
|
@ -32,6 +47,21 @@ MEDIA_DIR="${1:-media}"
|
|||
POSTS="content/posts.json"
|
||||
MAX_BYTES=$((64 * 1024 * 1024))
|
||||
|
||||
# What counts as a media reference inside a post body: an absolute URL or a path
|
||||
# we have already rewritten, ending in a media extension.
|
||||
#
|
||||
# Local paths are in the pattern deliberately. Leaving them out looked right —
|
||||
# nothing needs downloading twice — but it is what made a second run destructive
|
||||
# rather than idempotent: the already-rewritten body references were not
|
||||
# enumerated, so they never re-entered the mirror map, so the body_media sidecar
|
||||
# came back with only the handful of entries that happened to still be absolute.
|
||||
# Matching them means they are adopted from the mount and everything is rebuilt
|
||||
# exactly as it was.
|
||||
#
|
||||
# One definition, passed to every jq that needs it, because three copies of a
|
||||
# regex is three chances for one of them to drift.
|
||||
MEDIA_REF_RE='(?:https?://|/media/)[^\s)\]"<>]+\.(?:mp4|webm|mov|webp|png|jpe?g|gif|avif)'
|
||||
|
||||
# Media we host ourselves, published by tools/publish-media.sh before the post
|
||||
# that carries it exists. Such a URL is ALREADY the one the page should use, so
|
||||
# there is nothing to fetch: the bytes are on the media mount, and downloading
|
||||
|
|
@ -54,6 +84,12 @@ mkdir -p "$MEDIA_DIR"
|
|||
HAVE_FFPROBE=0
|
||||
command -v ffprobe >/dev/null 2>&1 && HAVE_FFPROBE=1
|
||||
|
||||
# ffmpeg does the still-image transcodes below. Optional in exactly the same
|
||||
# way ffprobe is: without it every image is served as the single file the mirror
|
||||
# downloaded, which is what this site did before the format ladder existed.
|
||||
HAVE_FFMPEG=0
|
||||
command -v ffmpeg >/dev/null 2>&1 && HAVE_FFMPEG=1
|
||||
|
||||
# Sets $w and $h for the file named in $1, or leaves both 0.
|
||||
#
|
||||
# One query per dimension. Asking for both at once and splitting the CSV looked
|
||||
|
|
@ -95,24 +131,161 @@ probe_dims() {
|
|||
fi
|
||||
}
|
||||
|
||||
# Derive the two renditions a mirrored image is served between: AVIF above it
|
||||
# and PNG below. Sets $avif_name / $png_name to the sibling file names, or
|
||||
# leaves one empty when that rendition could not be produced — :Media then drops
|
||||
# the tier rather than pointing at a file that is not on the mount.
|
||||
#
|
||||
# Siblings are named after the source file, which is itself the hash of its
|
||||
# bytes, so a rendition already present is never re-encoded and a changed source
|
||||
# gets new names. Only genuinely new images cost encoder time; a rebuild costs
|
||||
# none, which is what keeps this off the critical path of every deploy.
|
||||
#
|
||||
# WHY BOTH TIERS. AVIF is smaller than the WebP the instances serve (~15% on
|
||||
# these screenshots, far more on photographs) and is what almost every visitor
|
||||
# actually receives. PNG is lossless and universally understood, which is what
|
||||
# makes it a fallback worth having — but it is also several times the size of
|
||||
# the WebP beside it, so the <picture> offers the mirrored original in between
|
||||
# and the PNG is reached only by a browser that understands neither of the
|
||||
# other two.
|
||||
#
|
||||
# The settings, measured against these files rather than guessed:
|
||||
# crf 26, cpu-used 6 SSIM 0.997 against the source and still smaller than
|
||||
# it, at roughly half a second per image.
|
||||
# yuv444p these are screenshots of text. Re-subsampling chroma
|
||||
# that pict-rs already subsampled once fringes coloured
|
||||
# text visibly, and full chroma costs about 3% here.
|
||||
transcode_image() {
|
||||
avif_name=""
|
||||
png_name=""
|
||||
_file="$1"
|
||||
_name="$2"
|
||||
_base="${_name%.*}"
|
||||
|
||||
# Already that format: serve the mirrored file as the tier rather than
|
||||
# re-encoding it into a second copy of itself.
|
||||
case "$_name" in *.avif) avif_name="$_name" ;; esac
|
||||
case "$_name" in *.png) png_name="$_name" ;; esac
|
||||
[ "$HAVE_FFMPEG" = 1 ] || return 0
|
||||
|
||||
# An animated source is not a still, and -frames:v 1 would silently freeze
|
||||
# it. Leave it entirely alone: one moving GIF is worth more than three
|
||||
# copies of its first frame. nb_frames is N/A for WebP, so the frames have
|
||||
# to actually be counted — ~75 ms on a 3 MP image, once per new file.
|
||||
_frames=$(ffprobe -v error -select_streams v:0 -count_frames \
|
||||
-show_entries stream=nb_read_frames \
|
||||
-of default=nw=1:nk=1 "$_file" 2>/dev/null | head -n1)
|
||||
case "$_frames" in
|
||||
''|*[!0-9]*|1) ;; # unknown or a single frame: a still
|
||||
*) echo "fetch-media: $_name is animated, serving it as one file" >&2
|
||||
return 0 ;;
|
||||
esac
|
||||
|
||||
# Alpha has to survive the transcode: an image with a transparent corner
|
||||
# encoded into a format with no alpha plane gains an opaque black one.
|
||||
_pixfmt=$(ffprobe -v error -select_streams v:0 -show_entries stream=pix_fmt \
|
||||
-of default=nw=1:nk=1 "$_file" 2>/dev/null | head -n1)
|
||||
case "$_pixfmt" in
|
||||
yuva*|rgba*|bgra*|argb*|abgr*|gbrap*|ya8|ya16*|pal8) _avif_pix=yuva444p ;;
|
||||
*) _avif_pix=yuv444p ;;
|
||||
esac
|
||||
|
||||
# Encoded to a .part, checked, and only then renamed — so an interrupted or
|
||||
# wrong-format encode cannot leave a file the next run adopts as finished.
|
||||
if [ -z "$avif_name" ]; then
|
||||
_cand="$_base.avif"
|
||||
if [ -f "$MEDIA_DIR/$_cand" ]; then
|
||||
avif_name="$_cand"
|
||||
elif encode_rendition "$_file" "$_cand" av1 \
|
||||
-c:v libaom-av1 -still-picture 1 -crf 26 -cpu-used 6 \
|
||||
-pix_fmt "$_avif_pix" -f avif; then
|
||||
avif_name="$_cand"
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ -z "$png_name" ]; then
|
||||
_cand="$_base.png"
|
||||
if [ -f "$MEDIA_DIR/$_cand" ]; then
|
||||
png_name="$_cand"
|
||||
elif encode_rendition "$_file" "$_cand" png -c:v png -f image2; then
|
||||
png_name="$_cand"
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
# encode_rendition SRC OUTNAME EXPECTED_CODEC ffmpeg-args...
|
||||
#
|
||||
# Runs the encode into a .part, verifies the result really is the codec asked
|
||||
# for, and only then publishes it. Returns non-zero (leaving nothing behind) if
|
||||
# either step fails, which drops that tier rather than shipping a broken one.
|
||||
#
|
||||
# The verification is not paranoia. ffmpeg picks an encoder from the MUXER when
|
||||
# one is not named, and the image2 muxer defaults to MJPEG — so `-f image2
|
||||
# out.png` silently produced a run of lossy JPEGs sitting under .png names, which
|
||||
# the page then advertised to browsers as image/png. The codec is pinned by the
|
||||
# callers above; this is the check that the pin held.
|
||||
encode_rendition() {
|
||||
_src="$1"; _out="$2"; _want="$3"
|
||||
shift 3
|
||||
if ! ffmpeg -y -v error -i "$_src" -frames:v 1 "$@" \
|
||||
"$MEDIA_DIR/$_out.part" 2>/dev/null; then
|
||||
rm -f "$MEDIA_DIR/$_out.part"
|
||||
echo "fetch-media: could not encode $_out, serving without that tier" >&2
|
||||
return 1
|
||||
fi
|
||||
_got=$(ffprobe -v error -select_streams v:0 -show_entries stream=codec_name \
|
||||
-of default=nw=1:nk=1 "$MEDIA_DIR/$_out.part" 2>/dev/null | head -n1)
|
||||
if [ "$_got" != "$_want" ]; then
|
||||
rm -f "$MEDIA_DIR/$_out.part"
|
||||
echo "fetch-media: $_out came out as '$_got', expected '$_want' — discarding it" >&2
|
||||
return 1
|
||||
fi
|
||||
mv "$MEDIA_DIR/$_out.part" "$MEDIA_DIR/$_out"
|
||||
chmod 0644 "$MEDIA_DIR/$_out"
|
||||
encoded=$((encoded + 1))
|
||||
return 0
|
||||
}
|
||||
|
||||
MAP="$(mktemp)"
|
||||
POSTERMAP="$(mktemp)"
|
||||
FALLBACKMAP="$(mktemp)"
|
||||
trap 'rm -f "$MAP" "$POSTERMAP" "$FALLBACKMAP"' EXIT
|
||||
AVIFMAP="$(mktemp)"
|
||||
POSTERONLY="$(mktemp)"
|
||||
trap 'rm -f "$MAP" "$POSTERMAP" "$FALLBACKMAP" "$AVIFMAP" "$POSTERONLY"' EXIT
|
||||
printf '[]' > "$MAP"
|
||||
printf '[]' > "$POSTERMAP"
|
||||
printf '[]' > "$FALLBACKMAP"
|
||||
printf '[]' > "$AVIFMAP"
|
||||
|
||||
# URLs that are ONLY ever a video's poster frame. They are skipped by the
|
||||
# transcode above, because `poster` takes exactly one URL: a <video> cannot
|
||||
# negotiate a format the way <picture> can, so the renditions would be files
|
||||
# nothing is able to ask for. A file that is a poster somewhere and an ordinary
|
||||
# image somewhere else is not in this list and is transcoded normally.
|
||||
jq -r --arg re "$MEDIA_REF_RE" \
|
||||
'([.[].media[]? | .poster // empty] | map(select(. != "")) | unique) as $posters
|
||||
| ([.[].media[]? | .src] + [.[] | .body // "" | scan($re)] | unique) as $srcs
|
||||
| ($posters - $srcs) | .[]' "$POSTS" > "$POSTERONLY" 2>/dev/null || true
|
||||
|
||||
downloaded=0
|
||||
reused=0
|
||||
adopted=0
|
||||
failed=0
|
||||
encoded=0
|
||||
|
||||
# Every distinct media URL across all posts, so a file shared by two posts is
|
||||
# fetched once. Video posters are in here too: a poster left pointing at the
|
||||
# source instance would leak a visitor IP on page load exactly like an embedded
|
||||
# image would, and it is the frame shown before anyone presses play.
|
||||
#
|
||||
# Body URLs are found by pattern rather than by parsing Markdown: anything that
|
||||
# looks like an absolute URL ending in a media extension is mirrored, whether it
|
||||
# was written as an embed, as a link, or bare. That is deliberately wider than
|
||||
# "images the body displays" — the origin rule covers href as well as src, and a
|
||||
# link whose target is a .webp on someone else's instance is still a third-party
|
||||
# address on our page. Already-rewritten paths start with /media/ and so do not
|
||||
# match, which is what makes re-running this a no-op.
|
||||
#
|
||||
# Fed by a here-document rather than a pipe so the counters below survive — in
|
||||
# `jq | while`, the loop runs in a subshell and every increment is discarded.
|
||||
while IFS= read -r src; do
|
||||
|
|
@ -152,6 +325,26 @@ while IFS= read -r src; do
|
|||
fi
|
||||
adopted=$((adopted + 1))
|
||||
;;
|
||||
/media/*)
|
||||
# Already rewritten by an earlier run of this script. Adopt the file on
|
||||
# the mount rather than trying to fetch our own path as though it were a
|
||||
# URL — which is what makes running this twice a no-op instead of a way
|
||||
# to lose every rewrite it made the first time. The script is meant to
|
||||
# follow fetch-posts.sh, but "meant to" is not a guarantee, and the
|
||||
# failure was silent: the body_media list simply came back empty.
|
||||
name=${src#/media/}
|
||||
case "$name" in
|
||||
''|*/*|*..*)
|
||||
echo "fetch-media: refusing suspicious local path: $src" >&2
|
||||
failed=$((failed + 1)); continue ;;
|
||||
esac
|
||||
dest="$MEDIA_DIR/$name"
|
||||
if [ ! -f "$dest" ]; then
|
||||
echo "fetch-media: $name not on the media mount, leaving it alone: $src" >&2
|
||||
failed=$((failed + 1)); continue
|
||||
fi
|
||||
adopted=$((adopted + 1))
|
||||
;;
|
||||
*)
|
||||
ext=$(printf '%s' "$src" | sed -E 's/.*\.([A-Za-z0-9]+)$/\1/' | tr 'A-Z' 'a-z')
|
||||
case "$ext" in
|
||||
|
|
@ -226,6 +419,26 @@ while IFS= read -r src; do
|
|||
&& mv "$FALLBACKMAP.new" "$FALLBACKMAP"
|
||||
fi
|
||||
;;
|
||||
*.webp|*.png|*.jpg|*.jpeg|*.gif|*.avif)
|
||||
# The AVIF and PNG tiers this image is served between. Both maps are
|
||||
# keyed by the LOCAL path, like the video ones above, so the rewrite
|
||||
# below can look them up from the src it has just written.
|
||||
if grep -qxF "$src" "$POSTERONLY" 2>/dev/null; then
|
||||
: # poster-only; see POSTERONLY above
|
||||
else
|
||||
transcode_image "$dest" "$name"
|
||||
if [ -n "$avif_name" ]; then
|
||||
jq --arg k "/media/$name" --arg v "/media/$avif_name" \
|
||||
'. + [{key: $k, value: $v}]' "$AVIFMAP" > "$AVIFMAP.new" \
|
||||
&& mv "$AVIFMAP.new" "$AVIFMAP"
|
||||
fi
|
||||
if [ -n "$png_name" ]; then
|
||||
jq --arg k "/media/$name" --arg v "/media/$png_name" \
|
||||
'. + [{key: $k, value: $v}]' "$FALLBACKMAP" > "$FALLBACKMAP.new" \
|
||||
&& mv "$FALLBACKMAP.new" "$FALLBACKMAP"
|
||||
fi
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
|
||||
jq --arg src "$src" --arg path "/media/$name" \
|
||||
|
|
@ -233,20 +446,25 @@ while IFS= read -r src; do
|
|||
'. + [{src: $src, path: $path, w: $w, h: $h}]' "$MAP" > "$MAP.new" \
|
||||
&& mv "$MAP.new" "$MAP"
|
||||
done <<EOF
|
||||
$(jq -r '[.[].media[]? | .src, (.poster // empty)] | map(select(. != "")) | unique[]' "$POSTS")
|
||||
$(jq -r --arg re "$MEDIA_REF_RE" \
|
||||
'[ (.[].media[]? | .src, (.poster // empty)),
|
||||
(.[] | .body // "" | scan($re)) ]
|
||||
| map(select(. != "")) | unique[]' "$POSTS")
|
||||
EOF
|
||||
|
||||
echo "fetch-media: $downloaded new, $reused already present, $adopted self-hosted, $failed failed"
|
||||
echo "fetch-media: $encoded image rendition(s) encoded this run"
|
||||
|
||||
# Rewrite each media entry to the local path. An entry with no mapping (download
|
||||
# failed) keeps its original src, so the page still shows something rather than
|
||||
# silently dropping the post's whole point.
|
||||
TMP_POSTS="$(mktemp)"
|
||||
if jq --slurpfile map "$MAP" --slurpfile posters "$POSTERMAP" \
|
||||
--slurpfile fallbacks "$FALLBACKMAP" '
|
||||
--slurpfile fallbacks "$FALLBACKMAP" --slurpfile avifs "$AVIFMAP" '
|
||||
($map[0] | map({key: .src, value: .}) | from_entries) as $m
|
||||
| ($posters[0] | from_entries) as $pm
|
||||
| ($fallbacks[0] | from_entries) as $fm
|
||||
| ($avifs[0] | from_entries) as $am
|
||||
| map(.media = ((.media // []) | map(
|
||||
. as $item
|
||||
| ($m[$item.src] // null) as $hit
|
||||
|
|
@ -271,7 +489,42 @@ if jq --slurpfile map "$MAP" --slurpfile posters "$POSTERMAP" \
|
|||
# sibling to find.
|
||||
| if (($fm[.src] // "") != "")
|
||||
then . + { fallback: $fm[.src] }
|
||||
else . end
|
||||
# The AVIF tier, keyed by the LOCAL src like the two maps above. Only
|
||||
# ever set for images, and only when the encode actually produced one.
|
||||
| if (($am[.src] // "") != "")
|
||||
then . + { avif: $am[.src] }
|
||||
else . end)))
|
||||
# ── the body ──────────────────────────────────────────────────────
|
||||
#
|
||||
# Substitution is literal (split/join, not gsub), because these URLs are
|
||||
# full of regex metacharacters and a mirrored path must land in the text
|
||||
# exactly as written. A URL with no mapping — its download failed — is left
|
||||
# alone, so the post still shows the image rather than losing it; the
|
||||
# accounting at the end of this script reports that as media still pointing
|
||||
# at its source.
|
||||
| ($m | to_entries) as $subs
|
||||
| map(.body = (reduce $subs[] as $s ((.body // "");
|
||||
split($s.key) | join($s.value.path))))
|
||||
# Everything the rewritten body now points at, as records the renderer can
|
||||
# read: Markdown has nowhere to put a width, a poster frame or a second
|
||||
# source, so the sidecar list is how an inline video gets the same treatment
|
||||
# as a headline one. Keyed by the LOCAL path, which is what the body says
|
||||
# by this point.
|
||||
| ($m | map({ key: .path, value: . }) | from_entries) as $byPath
|
||||
| map(.body_media = ([ (.body // "")
|
||||
| scan("/media/[A-Za-z0-9._-]+")
|
||||
| . as $path
|
||||
| select($byPath[$path] != null)
|
||||
| { src: $path,
|
||||
kind: (if ($path | test("\\.(?:mp4|webm|mov)$"))
|
||||
then "video" else "image" end),
|
||||
poster: ($pm[$path] // ""),
|
||||
fallback: ($fm[$path] // ""),
|
||||
avif: ($am[$path] // ""),
|
||||
w: ($byPath[$path].w // 0),
|
||||
h: ($byPath[$path].h // 0) } ]
|
||||
| unique_by(.src)))
|
||||
' "$POSTS" > "$TMP_POSTS" 2>/dev/null; then
|
||||
# Same reason as the chmod on each mirrored file: mktemp is 0600 and the
|
||||
# mode survives to production, where other users must read this.
|
||||
|
|
@ -283,10 +536,18 @@ else
|
|||
exit 1
|
||||
fi
|
||||
|
||||
total=$(jq '[.[].media[]? | .src, (.poster // empty) | select(. != "")] | length' "$POSTS")
|
||||
local_count=$(jq '[.[].media[]? | .src, (.poster // empty)
|
||||
| select(startswith("/media/"))] | length' "$POSTS")
|
||||
echo "fetch-media: $local_count of $total media entries served locally ($(du -sh "$MEDIA_DIR" | cut -f1) in $MEDIA_DIR)"
|
||||
# Counted over the bodies too, because a body URL that never mirrored is the
|
||||
# same privacy leak as a card one and must not be reported as success.
|
||||
total=$(jq --arg re "$MEDIA_REF_RE" \
|
||||
'[ (.[].media[]? | .src, (.poster // empty)),
|
||||
(.[] | .body // "" | scan($re)),
|
||||
(.[].body_media[]? | .src, (.poster // empty)) ]
|
||||
| map(select(. != "")) | unique | length' "$POSTS")
|
||||
local_count=$(jq '[ (.[].media[]? | .src, (.poster // empty)),
|
||||
(.[].body_media[]? | .src, (.poster // empty)) ]
|
||||
| map(select(startswith("/media/"))) | length' "$POSTS")
|
||||
inline=$(jq '[.[].body_media[]?] | length' "$POSTS")
|
||||
echo "fetch-media: $local_count of $total media entries served locally ($inline of them embedded in post bodies; $(du -sh "$MEDIA_DIR" | cut -f1) in $MEDIA_DIR)"
|
||||
if [ "$local_count" -ne "$total" ]; then
|
||||
echo "fetch-media: $((total - local_count)) still point at their source — see the failures above" >&2
|
||||
fi
|
||||
|
|
|
|||
Loading…
Reference in a new issue