posts and avif

tools/fetch-posts.sh

@@ -28,6 +28,12 @@ CONFIG="${1:-content/posts-sources.json}"
 OUT="content/posts.json"
 LIMIT=50
 EXCERPT_CHARS=280
+# A ceiling, not a target: the longest of these bodies is around 6 KB and this
+# file is fetched into memory before the wasm module starts, so an outlier must
+# not be able to grow the bundle without bound. A truncated body renders as
+# truncated prose (the Markdown renderer tolerates an unclosed fence), which is
+# a better failure than a build that silently ships a megabyte.
+MAX_BODY_CHARS=32768
 
 command -v jq >/dev/null 2>&1 || { echo "fetch-posts: jq not found, keeping existing $OUT" >&2; exit 0; }
 [ -f "$CONFIG" ] || { echo "fetch-posts: $CONFIG not found, keeping existing $OUT" >&2; exit 0; }
@@ -84,13 +90,39 @@ fi
 #               Both are ORIGINAL urls here; tools/fetch-media.sh mirrors them
 #               and rewrites to local paths, so nothing the browser loads is
 #               third-party.
-#   excerpt   : body flattened to one line and truncated. Markdown is NOT
-#               rendered — the site has no markdown pipeline by design, so any
-#               surviving syntax would show as literal characters. Strip the
-#               common inline markers and let the rest be plain text.
+#   excerpt   : body flattened to one line and truncated, for the card and for
+#               the meta description. Markdown markers are stripped rather than
+#               rendered — this string lands in places that are plain text by
+#               definition (<meta name="description">, og:description), so any
+#               surviving syntax would show as literal characters.
+#   body      : the post, whole, still as Markdown. Rendered by
+#               Catcrafts.Shared:Markdown at page-render time rather than
+#               converted here, because that module is inside the escaping
+#               guarantee and a shell script writing HTML into a content file
+#               would not be — text from someone else's server must not be able
+#               to become markup anywhere but there.
+#   slug      : the post's own URL at /posts/<slug>, from the title. Collisions
+#               take the post's numeric id as a suffix rather than a positional
+#               one: a later post sharing a title would otherwise renumber an
+#               earlier post's URL out from under everyone who linked it.
 #   deleted / removed posts are dropped rather than rendered as empty cards.
 if ! jq --argjson n "$EXCERPT_CHARS" \
+        --argjson maxbody "$MAX_BODY_CHARS" \
         --slurpfile cfg "$CONFIG" '
+    # Catcrafts.Shared:Route::IsValidSlug is the contract: lowercase ASCII,
+    # digits and single hyphens, no leading or trailing hyphen, at most 64
+    # characters. A slug that fails it is dropped by the loader, which costs
+    # the post its page — so the shape is produced correctly here rather than
+    # sanitised on the way out.
+    def slugify:
+        ascii_downcase
+        | gsub("[^a-z0-9]+"; "-")
+        | sub("^-+"; "") | sub("-+$"; "");
+    # Truncate on a word boundary where there is one: a slug cut mid-word reads
+    # like a typo, and these titles are long enough to hit the limit.
+    def clamp($n):
+        (if (length > $n) then (.[0:$n] | sub("-[^-]*$"; "")) else . end)
+        | sub("-+$"; "");
     ($cfg[0].communities | map(ascii_downcase)) as $allow
     | [ .posts[]
       | select((.post.deleted // false) == false)
@@ -101,6 +133,11 @@ if ! jq --argjson n "$EXCERPT_CHARS" \
       | select(($comm | ascii_downcase) as $c | $allow | index($c))
       | {
           title:     ($p.post.name // ""),
+          slug:      (($p.post.name // "") | slugify | clamp(64)),
+          # Carried only as far as the de-duplication pass below, which strips
+          # it: it is a tie-breaker, not content.
+          uid:       (($p.post.ap_id // "") | sub(".*/"; "") | ascii_downcase
+                        | gsub("[^a-z0-9]"; "") | .[0:12]),
           permalink: ($p.post.ap_id // ""),
           # A link post whose target IS an image or video is a media post, not a
           # link post: the file is captured in `media` and embedded, so keeping
@@ -123,6 +160,10 @@ if ! jq --argjson n "$EXCERPT_CHARS" \
                         | gsub("  +"; " ")
                         | ltrimstr(" ") | rtrimstr(" ")
                         | if (. | length) > $n then (.[0:$n] | sub(" [^ ]*$"; "")) + "…" else . end),
+          # Verbatim apart from CR removal (a stray \r would render as a stray
+          # character inside a code block, where nothing is interpreted) and the
+          # size ceiling.
+          body:      (($p.post.body // "") | gsub("\r"; "") | .[0:$maxbody]),
           media:     ([ ($p.post.url // "")
                         | select(test("\\.(?:mp4|webm|mov|webp|png|jpe?g|gif|avif)$"))
                         | (if test("\\.(mp4|webm|mov)$") then "video" else "image" end) as $kind
@@ -139,7 +180,26 @@ if ! jq --argjson n "$EXCERPT_CHARS" \
           score:     ($p.counts.score // 0),
           comments:  ($p.counts.comments // 0)
         }
-    ]' "$RAW" > "$TMP" 2>/dev/null; then
+    ]
+    # Make every slug unique, and keep it that way across builds.
+    #
+    # Only a duplicate is suffixed, so the ordinary post keeps the clean URL the
+    # title earned. The suffix is the numeric id of the post itself rather than
+    # a counter, because a counter is positional: a new post repeating an older
+    # title arrives at the front of this newest-first list, takes the bare slug,
+    # and silently renumbers the older one — breaking a URL that is already in
+    # search results and in whatever links people have shared.
+    # (Note for editors: this jq program is inside a single-quoted shell string,
+    # so no apostrophes anywhere in it.)
+    | reduce .[] as $e ({ seen: {}, out: [] };
+        (if ($e.slug | length) == 0 then "post" else $e.slug end) as $base
+        | (if (.seen[$base] // false)
+           then (($base | clamp(50)) + "-"
+                 + (if ($e.uid | length) == 0 then "x" else $e.uid end))
+           else $base end) as $slug
+        | { seen: (.seen + { ($base): true, ($slug): true }),
+            out:  (.out + [ ($e | del(.uid)) + { slug: $slug } ]) })
+    | .out' "$RAW" > "$TMP" 2>/dev/null; then
     echo "fetch-posts: response did not match the expected shape, keeping existing $OUT" >&2
     exit 0
 fi