posts and avif

tools/e2e.sh

@@ -140,6 +140,10 @@ status /legal/nope 404
 # A slug that cannot be one of ours is rejected before any lookup.
 status /shop/BAD--slug 404
 status /demos/nope 404
+# A post slug that parsed but names nothing must be a real 404, or every typo
+# and every retired post becomes an indexable empty page.
+status /posts/nope 404
+status /posts/BAD--slug 404
 # The retired blog URLs are still in the wild; they must redirect, not 404.
 status /blog 301
 status /blog/hello-world 301
@@ -367,7 +371,7 @@ else
 fi
 # `poster` is in the list because a video poster is fetched on page load exactly
 # like an <img> src is, so a third-party poster leaks the same visitor IP.
-if curl -s "$BASE/posts" | grep -qE '(src|href|poster)="https?://[^"]*\.(mp4|webm|webp|png|jpe?g|gif)'; then
+if curl -s "$BASE/posts" | grep -qE '(src|srcset|href|poster)="https?://[^"]*\.(mp4|webm|webp|avif|png|jpe?g|gif)'; then
     bad "/posts media origin" "media loaded from a third party"
 else
     ok "/posts loads no media from a third party"
@@ -417,6 +421,126 @@ else
     bad "video preload" "expected preload=\"metadata\""
 fi
 
+echo "== post pages =="
+# The post page is where the body lives, and the body is the reason the site
+# has anything for a search engine to index beyond a list of links off it. Its
+# slug is data, so take one from the page rather than hardcoding a title that
+# will be wrong the week after it is written.
+POST_PATH=$(curl -s "$BASE/posts" | grep -oE 'href="/posts/[a-z0-9-]+"' \
+            | head -n1 | sed 's/href="//; s/"$//')
+if [ -z "$POST_PATH" ]; then
+    bad "post pages" "/posts links no post page; nothing carries a body"
+else
+    ok "/posts links a post page ($POST_PATH)"
+    status "$POST_PATH" 200
+    body_has /posts 'Read the full post'  "/posts offers the full post"
+    # And it trails the excerpt, immediately after the ellipsis the truncation
+    # left, rather than sitting as its own row below the media. The excerpt is
+    # escaped text, so nothing but the link can put a '<' between the two.
+    if curl -s "$BASE/posts" \
+       | grep -qE '<p class="post-card__excerpt">[^<]*<a class="link-more" href="/posts/'; then
+        ok "read-more trails the excerpt"
+    else
+        bad "read-more placement" "not inside the excerpt paragraph"
+    fi
+    body_has "$POST_PATH" '<div class="post-body">' "post page carries the rendered body"
+    # Rendered, not dumped: a body that reached the page as literal Markdown
+    # would show its own asterisks and hashes to the reader and to a crawler.
+    if curl -s "$BASE$POST_PATH" | grep -qE '<(p|h2|h3|h4|ul|ol|blockquote|pre)>'; then
+        ok "post body is real markup, not literal Markdown"
+    else
+        bad "post body markup" "no block elements found in the body"
+    fi
+    # The canonical points here, not at the instance. That is the entire SEO
+    # argument for hosting the body: two copies of the text exist, and this
+    # says which one is the original as far as this site is concerned.
+    body_has "$POST_PATH" 'rel="canonical" href="https://catcrafts.net/posts/' \
+             "post page is its own canonical"
+    body_has "$POST_PATH" '"@type":"BlogPosting"' "post page carries BlogPosting JSON-LD"
+    body_has "$POST_PATH" '"@id":"https://catcrafts.net/#organization"' \
+             "post JSON-LD joins the organization node"
+    body_has "$POST_PATH" '"@id":"https://catcrafts.net/about#person"' \
+             "post JSON-LD joins the founder node"
+    body_has "$POST_PATH" 'property="og:type" content="article"' "post page is an article to og:"
+    # Hosting the body does not mirror the discussion; the thread is still one
+    # click away and is still where the comments are.
+    if curl -s "$BASE$POST_PATH" | grep -qE 'href="https://[a-z0-9.-]+/post/[0-9]+"'; then
+        ok "post page still links its thread"
+    else
+        bad "post thread link" "no https://<instance>/post/<id> link on the post page"
+    fi
+    # The body is prose, not an application. Same rule as /projects.
+    body_lacks "$POST_PATH" '<script>' "post page ships no executable script"
+    body_lacks "$POST_PATH" '<base'    "post page has no base tag"
+    # Every inline image and video the body embeds is mirrored, exactly like a
+    # card's media — the privacy notice's "everything comes from catcrafts.net"
+    # covers href as well as src, so a body linking a .webp on someone else's
+    # instance is the same leak as embedding one.
+    for pg in $(curl -s "$BASE/sitemap.xml" \
+                | grep -oE '/posts/[a-z0-9-]+' | head -n 20); do
+        if curl -s "$BASE$pg" \
+           | grep -qE '(src|srcset|href|poster)="https?://[^"]*\.(mp4|webm|webp|avif|png|jpe?g|gif)'; then
+            bad "$pg media origin" "body media loaded from a third party"
+        else
+            ok "$pg loads no media from a third party"
+        fi
+    done
+    # The image format ladder: AVIF first, the mirrored original next, and a
+    # PNG on the <img> underneath, so exactly one file is fetched and every
+    # browser can read one of them. Order is the whole point — a browser takes
+    # the first source it understands — so assert the sequence, not just that
+    # the pieces are present.
+    if curl -s "$BASE$POST_PATH" | grep -q '<picture>'; then
+        if curl -s "$BASE$POST_PATH" \
+           | grep -qE '<picture><source srcset="/media/[^"]+\.avif" type="image/avif">'; then
+            ok "inline images lead with an AVIF source"
+        else
+            bad "image ladder" "the first source is not the AVIF"
+        fi
+        if curl -s "$BASE$POST_PATH" \
+           | grep -qE '<img class="post-media__item"[^>]*src="/media/[^"]+\.png"'; then
+            ok "inline images fall back to a PNG the img itself points at"
+        else
+            bad "image fallback" "the <img> base is not a PNG"
+        fi
+        # Every tier has to be a file that exists, or the ladder serves a 404 to
+        # whichever browsers pick that rung — which is precisely the set of
+        # browsers nobody testing this site is using.
+        missing=0
+        for f in $(curl -s "$BASE/sitemap.xml" | grep -oE '/posts/[a-z0-9-]+' | head -n 20 \
+                   | while read -r pg; do curl -s "$BASE$pg"; done \
+                   | grep -oE '(src|srcset)="/media/[^"]+"' \
+                   | sed 's/.*="//; s/"$//' | sort -u); do
+            [ -f "media${f#/media}" ] || { missing=$((missing + 1)); echo "    missing: $f" >&2; }
+        done
+        if [ "$missing" -eq 0 ]; then
+            ok "every referenced media file is on the mount"
+        else
+            bad "media files" "$missing referenced file(s) not on the mount"
+        fi
+    else
+        skip "image format ladder" "no <picture> on this page — ffmpeg absent at mirror time?"
+    fi
+    # Inline screenshots get dimensions from the sidecar list fetch-media.sh
+    # writes, because Markdown syntax has nowhere to carry them — without it
+    # the prose below every image jumps as the file arrives. Conditional: a
+    # post whose body embeds nothing has nothing to check.
+    if curl -s "$BASE$POST_PATH" | grep -q '<img class="post-media__item"'; then
+        if curl -s "$BASE$POST_PATH" \
+           | grep -qE '<img class="post-media__item"[^>]*width="[0-9]+" height="[0-9]+"'; then
+            ok "inline body images carry width/height"
+        else
+            bad "inline image dimensions" "an embedded body image has no dimensions"
+        fi
+    fi
+fi
+# The sitemap has to advertise the pages, or hosting the bodies buys nothing.
+if curl -s "$BASE/sitemap.xml" | grep -qE '<loc>https://catcrafts.net/posts/[a-z0-9-]+</loc>'; then
+    ok "sitemap lists the post pages"
+else
+    bad "sitemap post pages" "no /posts/<slug> entry"
+fi
+
 echo "== headers =="
 header_has /            'x-content-type-options: *nosniff'   "nosniff on pages"
 header_has /            'cache-control: *public'             "pages are cacheable"

tools/fetch-media.sh

@@ -4,6 +4,21 @@
 #
 # Run AFTER tools/fetch-posts.sh, which records the original URLs.
 #
+# TWO KINDS OF MEDIA, one pipeline:
+#
+#   * a post's headline file — `.media`, the recording or screenshot the post is
+#     about;
+#   * everything embedded inside the body — the screenshots a post argues with,
+#     which are just as much content and, until the body was hosted here, were
+#     never fetched at all.
+#
+# Both are content-addressed into the same directory, so a file used as one
+# post's headline and quoted inside another's body is stored once. Body files
+# are rewritten IN THE MARKDOWN TEXT (the body is still Markdown at this point)
+# and additionally recorded in `.body_media`, which is where the renderer reads
+# the dimensions, poster and H.264 fallback that Markdown syntax has nowhere to
+# carry.
+#
 # WHY MIRROR rather than embed from the source:
 #
 #   * Privacy. The privacy notice states that everything the browser loads comes
@@ -32,6 +47,21 @@ MEDIA_DIR="${1:-media}"
 POSTS="content/posts.json"
 MAX_BYTES=$((64 * 1024 * 1024))
 
+# What counts as a media reference inside a post body: an absolute URL or a path
+# we have already rewritten, ending in a media extension.
+#
+# Local paths are in the pattern deliberately. Leaving them out looked right —
+# nothing needs downloading twice — but it is what made a second run destructive
+# rather than idempotent: the already-rewritten body references were not
+# enumerated, so they never re-entered the mirror map, so the body_media sidecar
+# came back with only the handful of entries that happened to still be absolute.
+# Matching them means they are adopted from the mount and everything is rebuilt
+# exactly as it was.
+#
+# One definition, passed to every jq that needs it, because three copies of a
+# regex is three chances for one of them to drift.
+MEDIA_REF_RE='(?:https?://|/media/)[^\s)\]"<>]+\.(?:mp4|webm|mov|webp|png|jpe?g|gif|avif)'
+
 # Media we host ourselves, published by tools/publish-media.sh before the post
 # that carries it exists. Such a URL is ALREADY the one the page should use, so
 # there is nothing to fetch: the bytes are on the media mount, and downloading
@@ -54,6 +84,12 @@ mkdir -p "$MEDIA_DIR"
 HAVE_FFPROBE=0
 command -v ffprobe >/dev/null 2>&1 && HAVE_FFPROBE=1
 
+# ffmpeg does the still-image transcodes below. Optional in exactly the same
+# way ffprobe is: without it every image is served as the single file the mirror
+# downloaded, which is what this site did before the format ladder existed.
+HAVE_FFMPEG=0
+command -v ffmpeg >/dev/null 2>&1 && HAVE_FFMPEG=1
+
 # Sets $w and $h for the file named in $1, or leaves both 0.
 #
 # One query per dimension. Asking for both at once and splitting the CSV looked
@@ -95,24 +131,161 @@ probe_dims() {
     fi
 }
 
+# Derive the two renditions a mirrored image is served between: AVIF above it
+# and PNG below. Sets $avif_name / $png_name to the sibling file names, or
+# leaves one empty when that rendition could not be produced — :Media then drops
+# the tier rather than pointing at a file that is not on the mount.
+#
+# Siblings are named after the source file, which is itself the hash of its
+# bytes, so a rendition already present is never re-encoded and a changed source
+# gets new names. Only genuinely new images cost encoder time; a rebuild costs
+# none, which is what keeps this off the critical path of every deploy.
+#
+# WHY BOTH TIERS. AVIF is smaller than the WebP the instances serve (~15% on
+# these screenshots, far more on photographs) and is what almost every visitor
+# actually receives. PNG is lossless and universally understood, which is what
+# makes it a fallback worth having — but it is also several times the size of
+# the WebP beside it, so the <picture> offers the mirrored original in between
+# and the PNG is reached only by a browser that understands neither of the
+# other two.
+#
+# The settings, measured against these files rather than guessed:
+#   crf 26, cpu-used 6   SSIM 0.997 against the source and still smaller than
+#                        it, at roughly half a second per image.
+#   yuv444p              these are screenshots of text. Re-subsampling chroma
+#                        that pict-rs already subsampled once fringes coloured
+#                        text visibly, and full chroma costs about 3% here.
+transcode_image() {
+    avif_name=""
+    png_name=""
+    _file="$1"
+    _name="$2"
+    _base="${_name%.*}"
+
+    # Already that format: serve the mirrored file as the tier rather than
+    # re-encoding it into a second copy of itself.
+    case "$_name" in *.avif) avif_name="$_name" ;; esac
+    case "$_name" in *.png)  png_name="$_name"  ;; esac
+    [ "$HAVE_FFMPEG" = 1 ] || return 0
+
+    # An animated source is not a still, and -frames:v 1 would silently freeze
+    # it. Leave it entirely alone: one moving GIF is worth more than three
+    # copies of its first frame. nb_frames is N/A for WebP, so the frames have
+    # to actually be counted — ~75 ms on a 3 MP image, once per new file.
+    _frames=$(ffprobe -v error -select_streams v:0 -count_frames \
+                      -show_entries stream=nb_read_frames \
+                      -of default=nw=1:nk=1 "$_file" 2>/dev/null | head -n1)
+    case "$_frames" in
+        ''|*[!0-9]*|1) ;;   # unknown or a single frame: a still
+        *) echo "fetch-media: $_name is animated, serving it as one file" >&2
+           return 0 ;;
+    esac
+
+    # Alpha has to survive the transcode: an image with a transparent corner
+    # encoded into a format with no alpha plane gains an opaque black one.
+    _pixfmt=$(ffprobe -v error -select_streams v:0 -show_entries stream=pix_fmt \
+                      -of default=nw=1:nk=1 "$_file" 2>/dev/null | head -n1)
+    case "$_pixfmt" in
+        yuva*|rgba*|bgra*|argb*|abgr*|gbrap*|ya8|ya16*|pal8) _avif_pix=yuva444p ;;
+        *) _avif_pix=yuv444p ;;
+    esac
+
+    # Encoded to a .part, checked, and only then renamed — so an interrupted or
+    # wrong-format encode cannot leave a file the next run adopts as finished.
+    if [ -z "$avif_name" ]; then
+        _cand="$_base.avif"
+        if [ -f "$MEDIA_DIR/$_cand" ]; then
+            avif_name="$_cand"
+        elif encode_rendition "$_file" "$_cand" av1 \
+                 -c:v libaom-av1 -still-picture 1 -crf 26 -cpu-used 6 \
+                 -pix_fmt "$_avif_pix" -f avif; then
+            avif_name="$_cand"
+        fi
+    fi
+
+    if [ -z "$png_name" ]; then
+        _cand="$_base.png"
+        if [ -f "$MEDIA_DIR/$_cand" ]; then
+            png_name="$_cand"
+        elif encode_rendition "$_file" "$_cand" png -c:v png -f image2; then
+            png_name="$_cand"
+        fi
+    fi
+}
+
+# encode_rendition SRC OUTNAME EXPECTED_CODEC ffmpeg-args...
+#
+# Runs the encode into a .part, verifies the result really is the codec asked
+# for, and only then publishes it. Returns non-zero (leaving nothing behind) if
+# either step fails, which drops that tier rather than shipping a broken one.
+#
+# The verification is not paranoia. ffmpeg picks an encoder from the MUXER when
+# one is not named, and the image2 muxer defaults to MJPEG — so `-f image2
+# out.png` silently produced a run of lossy JPEGs sitting under .png names, which
+# the page then advertised to browsers as image/png. The codec is pinned by the
+# callers above; this is the check that the pin held.
+encode_rendition() {
+    _src="$1"; _out="$2"; _want="$3"
+    shift 3
+    if ! ffmpeg -y -v error -i "$_src" -frames:v 1 "$@" \
+                "$MEDIA_DIR/$_out.part" 2>/dev/null; then
+        rm -f "$MEDIA_DIR/$_out.part"
+        echo "fetch-media: could not encode $_out, serving without that tier" >&2
+        return 1
+    fi
+    _got=$(ffprobe -v error -select_streams v:0 -show_entries stream=codec_name \
+                   -of default=nw=1:nk=1 "$MEDIA_DIR/$_out.part" 2>/dev/null | head -n1)
+    if [ "$_got" != "$_want" ]; then
+        rm -f "$MEDIA_DIR/$_out.part"
+        echo "fetch-media: $_out came out as '$_got', expected '$_want' — discarding it" >&2
+        return 1
+    fi
+    mv "$MEDIA_DIR/$_out.part" "$MEDIA_DIR/$_out"
+    chmod 0644 "$MEDIA_DIR/$_out"
+    encoded=$((encoded + 1))
+    return 0
+}
+
 MAP="$(mktemp)"
 POSTERMAP="$(mktemp)"
 FALLBACKMAP="$(mktemp)"
-trap 'rm -f "$MAP" "$POSTERMAP" "$FALLBACKMAP"' EXIT
+AVIFMAP="$(mktemp)"
+POSTERONLY="$(mktemp)"
+trap 'rm -f "$MAP" "$POSTERMAP" "$FALLBACKMAP" "$AVIFMAP" "$POSTERONLY"' EXIT
 printf '[]' > "$MAP"
 printf '[]' > "$POSTERMAP"
 printf '[]' > "$FALLBACKMAP"
+printf '[]' > "$AVIFMAP"
+
+# URLs that are ONLY ever a video's poster frame. They are skipped by the
+# transcode above, because `poster` takes exactly one URL: a <video> cannot
+# negotiate a format the way <picture> can, so the renditions would be files
+# nothing is able to ask for. A file that is a poster somewhere and an ordinary
+# image somewhere else is not in this list and is transcoded normally.
+jq -r --arg re "$MEDIA_REF_RE" \
+   '([.[].media[]? | .poster // empty] | map(select(. != "")) | unique) as $posters
+    | ([.[].media[]? | .src] + [.[] | .body // "" | scan($re)] | unique) as $srcs
+    | ($posters - $srcs) | .[]' "$POSTS" > "$POSTERONLY" 2>/dev/null || true
 
 downloaded=0
 reused=0
 adopted=0
 failed=0
+encoded=0
 
 # Every distinct media URL across all posts, so a file shared by two posts is
 # fetched once. Video posters are in here too: a poster left pointing at the
 # source instance would leak a visitor IP on page load exactly like an embedded
 # image would, and it is the frame shown before anyone presses play.
 #
+# Body URLs are found by pattern rather than by parsing Markdown: anything that
+# looks like an absolute URL ending in a media extension is mirrored, whether it
+# was written as an embed, as a link, or bare. That is deliberately wider than
+# "images the body displays" — the origin rule covers href as well as src, and a
+# link whose target is a .webp on someone else's instance is still a third-party
+# address on our page. Already-rewritten paths start with /media/ and so do not
+# match, which is what makes re-running this a no-op.
+#
 # Fed by a here-document rather than a pipe so the counters below survive — in
 # `jq | while`, the loop runs in a subshell and every increment is discarded.
 while IFS= read -r src; do
@@ -152,6 +325,26 @@ while IFS= read -r src; do
         fi
         adopted=$((adopted + 1))
         ;;
+    /media/*)
+        # Already rewritten by an earlier run of this script. Adopt the file on
+        # the mount rather than trying to fetch our own path as though it were a
+        # URL — which is what makes running this twice a no-op instead of a way
+        # to lose every rewrite it made the first time. The script is meant to
+        # follow fetch-posts.sh, but "meant to" is not a guarantee, and the
+        # failure was silent: the body_media list simply came back empty.
+        name=${src#/media/}
+        case "$name" in
+            ''|*/*|*..*)
+                echo "fetch-media: refusing suspicious local path: $src" >&2
+                failed=$((failed + 1)); continue ;;
+        esac
+        dest="$MEDIA_DIR/$name"
+        if [ ! -f "$dest" ]; then
+            echo "fetch-media: $name not on the media mount, leaving it alone: $src" >&2
+            failed=$((failed + 1)); continue
+        fi
+        adopted=$((adopted + 1))
+        ;;
     *)
         ext=$(printf '%s' "$src" | sed -E 's/.*\.([A-Za-z0-9]+)$/\1/' | tr 'A-Z' 'a-z')
         case "$ext" in
@@ -226,6 +419,26 @@ while IFS= read -r src; do
                    && mv "$FALLBACKMAP.new" "$FALLBACKMAP"
             fi
             ;;
+        *.webp|*.png|*.jpg|*.jpeg|*.gif|*.avif)
+            # The AVIF and PNG tiers this image is served between. Both maps are
+            # keyed by the LOCAL path, like the video ones above, so the rewrite
+            # below can look them up from the src it has just written.
+            if grep -qxF "$src" "$POSTERONLY" 2>/dev/null; then
+                : # poster-only; see POSTERONLY above
+            else
+                transcode_image "$dest" "$name"
+                if [ -n "$avif_name" ]; then
+                    jq --arg k "/media/$name" --arg v "/media/$avif_name" \
+                       '. + [{key: $k, value: $v}]' "$AVIFMAP" > "$AVIFMAP.new" \
+                       && mv "$AVIFMAP.new" "$AVIFMAP"
+                fi
+                if [ -n "$png_name" ]; then
+                    jq --arg k "/media/$name" --arg v "/media/$png_name" \
+                       '. + [{key: $k, value: $v}]' "$FALLBACKMAP" > "$FALLBACKMAP.new" \
+                       && mv "$FALLBACKMAP.new" "$FALLBACKMAP"
+                fi
+            fi
+            ;;
     esac
 
     jq --arg src "$src" --arg path "/media/$name" \
@@ -233,20 +446,25 @@ while IFS= read -r src; do
        '. + [{src: $src, path: $path, w: $w, h: $h}]' "$MAP" > "$MAP.new" \
        && mv "$MAP.new" "$MAP"
 done <<EOF
-$(jq -r '[.[].media[]? | .src, (.poster // empty)] | map(select(. != "")) | unique[]' "$POSTS")
+$(jq -r --arg re "$MEDIA_REF_RE" \
+      '[ (.[].media[]? | .src, (.poster // empty)),
+         (.[] | .body // "" | scan($re)) ]
+       | map(select(. != "")) | unique[]' "$POSTS")
 EOF
 
 echo "fetch-media: $downloaded new, $reused already present, $adopted self-hosted, $failed failed"
+echo "fetch-media: $encoded image rendition(s) encoded this run"
 
 # Rewrite each media entry to the local path. An entry with no mapping (download
 # failed) keeps its original src, so the page still shows something rather than
 # silently dropping the post's whole point.
 TMP_POSTS="$(mktemp)"
 if jq --slurpfile map "$MAP" --slurpfile posters "$POSTERMAP" \
-      --slurpfile fallbacks "$FALLBACKMAP" '
+      --slurpfile fallbacks "$FALLBACKMAP" --slurpfile avifs "$AVIFMAP" '
     ($map[0] | map({key: .src, value: .}) | from_entries) as $m
     | ($posters[0] | from_entries) as $pm
     | ($fallbacks[0] | from_entries) as $fm
+    | ($avifs[0] | from_entries) as $am
     | map(.media = ((.media // []) | map(
         . as $item
         | ($m[$item.src] // null) as $hit
@@ -271,7 +489,42 @@ if jq --slurpfile map "$MAP" --slurpfile posters "$POSTERMAP" \
         # sibling to find.
         | if (($fm[.src] // "") != "")
           then . + { fallback: $fm[.src] }
+          else . end
+        # The AVIF tier, keyed by the LOCAL src like the two maps above. Only
+        # ever set for images, and only when the encode actually produced one.
+        | if (($am[.src] // "") != "")
+          then . + { avif: $am[.src] }
           else . end)))
+    # ── the body ──────────────────────────────────────────────────────
+    #
+    # Substitution is literal (split/join, not gsub), because these URLs are
+    # full of regex metacharacters and a mirrored path must land in the text
+    # exactly as written. A URL with no mapping — its download failed — is left
+    # alone, so the post still shows the image rather than losing it; the
+    # accounting at the end of this script reports that as media still pointing
+    # at its source.
+    | ($m | to_entries) as $subs
+    | map(.body = (reduce $subs[] as $s ((.body // "");
+                     split($s.key) | join($s.value.path))))
+    # Everything the rewritten body now points at, as records the renderer can
+    # read: Markdown has nowhere to put a width, a poster frame or a second
+    # source, so the sidecar list is how an inline video gets the same treatment
+    # as a headline one. Keyed by the LOCAL path, which is what the body says
+    # by this point.
+    | ($m | map({ key: .path, value: . }) | from_entries) as $byPath
+    | map(.body_media = ([ (.body // "")
+                           | scan("/media/[A-Za-z0-9._-]+")
+                           | . as $path
+                           | select($byPath[$path] != null)
+                           | { src: $path,
+                               kind: (if ($path | test("\\.(?:mp4|webm|mov)$"))
+                                      then "video" else "image" end),
+                               poster: ($pm[$path] // ""),
+                               fallback: ($fm[$path] // ""),
+                               avif: ($am[$path] // ""),
+                               w: ($byPath[$path].w // 0),
+                               h: ($byPath[$path].h // 0) } ]
+                         | unique_by(.src)))
     ' "$POSTS" > "$TMP_POSTS" 2>/dev/null; then
     # Same reason as the chmod on each mirrored file: mktemp is 0600 and the
     # mode survives to production, where other users must read this.
@@ -283,10 +536,18 @@ else
     exit 1
 fi
 
-total=$(jq '[.[].media[]? | .src, (.poster // empty) | select(. != "")] | length' "$POSTS")
-local_count=$(jq '[.[].media[]? | .src, (.poster // empty)
-                  | select(startswith("/media/"))] | length' "$POSTS")
-echo "fetch-media: $local_count of $total media entries served locally ($(du -sh "$MEDIA_DIR" | cut -f1) in $MEDIA_DIR)"
+# Counted over the bodies too, because a body URL that never mirrored is the
+# same privacy leak as a card one and must not be reported as success.
+total=$(jq --arg re "$MEDIA_REF_RE" \
+        '[ (.[].media[]? | .src, (.poster // empty)),
+           (.[] | .body // "" | scan($re)),
+           (.[].body_media[]? | .src, (.poster // empty)) ]
+         | map(select(. != "")) | unique | length' "$POSTS")
+local_count=$(jq '[ (.[].media[]? | .src, (.poster // empty)),
+                    (.[].body_media[]? | .src, (.poster // empty)) ]
+                  | map(select(startswith("/media/"))) | length' "$POSTS")
+inline=$(jq '[.[].body_media[]?] | length' "$POSTS")
+echo "fetch-media: $local_count of $total media entries served locally ($inline of them embedded in post bodies; $(du -sh "$MEDIA_DIR" | cut -f1) in $MEDIA_DIR)"
 if [ "$local_count" -ne "$total" ]; then
     echo "fetch-media: $((total - local_count)) still point at their source — see the failures above" >&2
 fi

tools/fetch-posts.sh

@@ -28,6 +28,12 @@ CONFIG="${1:-content/posts-sources.json}"
 OUT="content/posts.json"
 LIMIT=50
 EXCERPT_CHARS=280
+# A ceiling, not a target: the longest of these bodies is around 6 KB and this
+# file is fetched into memory before the wasm module starts, so an outlier must
+# not be able to grow the bundle without bound. A truncated body renders as
+# truncated prose (the Markdown renderer tolerates an unclosed fence), which is
+# a better failure than a build that silently ships a megabyte.
+MAX_BODY_CHARS=32768
 
 command -v jq >/dev/null 2>&1 || { echo "fetch-posts: jq not found, keeping existing $OUT" >&2; exit 0; }
 [ -f "$CONFIG" ] || { echo "fetch-posts: $CONFIG not found, keeping existing $OUT" >&2; exit 0; }
@@ -84,13 +90,39 @@ fi
 #               Both are ORIGINAL urls here; tools/fetch-media.sh mirrors them
 #               and rewrites to local paths, so nothing the browser loads is
 #               third-party.
-#   excerpt   : body flattened to one line and truncated. Markdown is NOT
-#               rendered — the site has no markdown pipeline by design, so any
-#               surviving syntax would show as literal characters. Strip the
-#               common inline markers and let the rest be plain text.
+#   excerpt   : body flattened to one line and truncated, for the card and for
+#               the meta description. Markdown markers are stripped rather than
+#               rendered — this string lands in places that are plain text by
+#               definition (<meta name="description">, og:description), so any
+#               surviving syntax would show as literal characters.
+#   body      : the post, whole, still as Markdown. Rendered by
+#               Catcrafts.Shared:Markdown at page-render time rather than
+#               converted here, because that module is inside the escaping
+#               guarantee and a shell script writing HTML into a content file
+#               would not be — text from someone else's server must not be able
+#               to become markup anywhere but there.
+#   slug      : the post's own URL at /posts/<slug>, from the title. Collisions
+#               take the post's numeric id as a suffix rather than a positional
+#               one: a later post sharing a title would otherwise renumber an
+#               earlier post's URL out from under everyone who linked it.
 #   deleted / removed posts are dropped rather than rendered as empty cards.
 if ! jq --argjson n "$EXCERPT_CHARS" \
+        --argjson maxbody "$MAX_BODY_CHARS" \
         --slurpfile cfg "$CONFIG" '
+    # Catcrafts.Shared:Route::IsValidSlug is the contract: lowercase ASCII,
+    # digits and single hyphens, no leading or trailing hyphen, at most 64
+    # characters. A slug that fails it is dropped by the loader, which costs
+    # the post its page — so the shape is produced correctly here rather than
+    # sanitised on the way out.
+    def slugify:
+        ascii_downcase
+        | gsub("[^a-z0-9]+"; "-")
+        | sub("^-+"; "") | sub("-+$"; "");
+    # Truncate on a word boundary where there is one: a slug cut mid-word reads
+    # like a typo, and these titles are long enough to hit the limit.
+    def clamp($n):
+        (if (length > $n) then (.[0:$n] | sub("-[^-]*$"; "")) else . end)
+        | sub("-+$"; "");
     ($cfg[0].communities | map(ascii_downcase)) as $allow
     | [ .posts[]
       | select((.post.deleted // false) == false)
@@ -101,6 +133,11 @@ if ! jq --argjson n "$EXCERPT_CHARS" \
       | select(($comm | ascii_downcase) as $c | $allow | index($c))
       | {
           title:     ($p.post.name // ""),
+          slug:      (($p.post.name // "") | slugify | clamp(64)),
+          # Carried only as far as the de-duplication pass below, which strips
+          # it: it is a tie-breaker, not content.
+          uid:       (($p.post.ap_id // "") | sub(".*/"; "") | ascii_downcase
+                        | gsub("[^a-z0-9]"; "") | .[0:12]),
           permalink: ($p.post.ap_id // ""),
           # A link post whose target IS an image or video is a media post, not a
           # link post: the file is captured in `media` and embedded, so keeping
@@ -123,6 +160,10 @@ if ! jq --argjson n "$EXCERPT_CHARS" \
                         | gsub("  +"; " ")
                         | ltrimstr(" ") | rtrimstr(" ")
                         | if (. | length) > $n then (.[0:$n] | sub(" [^ ]*$"; "")) + "…" else . end),
+          # Verbatim apart from CR removal (a stray \r would render as a stray
+          # character inside a code block, where nothing is interpreted) and the
+          # size ceiling.
+          body:      (($p.post.body // "") | gsub("\r"; "") | .[0:$maxbody]),
           media:     ([ ($p.post.url // "")
                         | select(test("\\.(?:mp4|webm|mov|webp|png|jpe?g|gif|avif)$"))
                         | (if test("\\.(mp4|webm|mov)$") then "video" else "image" end) as $kind
@@ -139,7 +180,26 @@ if ! jq --argjson n "$EXCERPT_CHARS" \
           score:     ($p.counts.score // 0),
           comments:  ($p.counts.comments // 0)
         }
-    ]' "$RAW" > "$TMP" 2>/dev/null; then
+    ]
+    # Make every slug unique, and keep it that way across builds.
+    #
+    # Only a duplicate is suffixed, so the ordinary post keeps the clean URL the
+    # title earned. The suffix is the numeric id of the post itself rather than
+    # a counter, because a counter is positional: a new post repeating an older
+    # title arrives at the front of this newest-first list, takes the bare slug,
+    # and silently renumbers the older one — breaking a URL that is already in
+    # search results and in whatever links people have shared.
+    # (Note for editors: this jq program is inside a single-quoted shell string,
+    # so no apostrophes anywhere in it.)
+    | reduce .[] as $e ({ seen: {}, out: [] };
+        (if ($e.slug | length) == 0 then "post" else $e.slug end) as $base
+        | (if (.seen[$base] // false)
+           then (($base | clamp(50)) + "-"
+                 + (if ($e.uid | length) == 0 then "x" else $e.uid end))
+           else $base end) as $slug
+        | { seen: (.seen + { ($base): true, ($slug): true }),
+            out:  (.out + [ ($e | del(.uid)) + { slug: $slug } ]) })
+    | .out' "$RAW" > "$TMP" 2>/dev/null; then
     echo "fetch-posts: response did not match the expected shape, keeping existing $OUT" >&2
     exit 0
 fi