catcrafts.net/tools/fetch-posts.sh

#!/bin/sh
# Fetch selected fediverse posts and write content/posts.json.
#
# Runs at BUILD time, not run time. The site does not crawl anything, does not
# proxy, and does not mirror comments — each card links out to the thread on
# whatever instance it lives on, which is where the discussion belongs. That
# means no sync service and no runtime dependency on any instance being up.
#
# WHICH POSTS: the Lemmy user API returns everything the account has posted,
# across every community. That is not what belongs on a site about this work, so
# the result is filtered against the community allowlist in
# content/posts-sources.json. Joining a new community does not silently publish
# to the site — it has to be added there first.
#
# The output is a flat array of exactly the fields Catcrafts.Shared:Model reads.
# Doing the transformation here rather than in C++ keeps the parser small and
# makes an upstream API change a one-file fix in shell.
#
#   usage: tools/fetch-posts.sh [config-file]
#
# On any failure the existing content/posts.json is left untouched and the script
# exits 0. A build must not fail because an instance was down, and stale posts
# are strictly better than an empty page.

set -eu

CONFIG="${1:-content/posts-sources.json}"
OUT="content/posts.json"
LIMIT=50
EXCERPT_CHARS=280

command -v jq >/dev/null 2>&1 || { echo "fetch-posts: jq not found, keeping existing $OUT" >&2; exit 0; }
[ -f "$CONFIG" ] || { echo "fetch-posts: $CONFIG not found, keeping existing $OUT" >&2; exit 0; }

USER_NAME=$(jq -r '.username // empty' "$CONFIG")
INSTANCE=$(jq -r '.instance // empty' "$CONFIG")
[ -n "$USER_NAME" ] && [ -n "$INSTANCE" ] || {
    echo "fetch-posts: $CONFIG needs .username and .instance, keeping existing $OUT" >&2; exit 0; }

# An empty allowlist would silently publish everything, which is the opposite of
# what this file is for — treat it as a configuration error, not as "allow all".
COMMUNITY_COUNT=$(jq '.communities | length' "$CONFIG")
[ "$COMMUNITY_COUNT" -gt 0 ] || {
    echo "fetch-posts: .communities is empty — refusing to publish every community" >&2
    echo "fetch-posts: keeping existing $OUT" >&2; exit 0; }

TMP="$(mktemp)"
RAW="$(mktemp)"
trap 'rm -f "$TMP" "$RAW"' EXIT

URL="$INSTANCE/api/v3/user?username=$USER_NAME&sort=New&limit=$LIMIT"

# Identify ourselves: an unattributed scraper on a small instance is rude and
# more likely to get blocked.
if ! curl -fsS --max-time 25 \
     -H 'Accept: application/json' \
     -A 'catcrafts.net-buildfetch/1.0 (+https://catcrafts.net)' \
     "$URL" -o "$RAW"; then
    echo "fetch-posts: request failed, keeping existing $OUT" >&2
    exit 0
fi

# Map PostView -> the flat shape the C++ loader reads.
#
#   community : assembled as name@instance from the community's actor_id, which
#               is what the allowlist matches on. A post made INTO
#               linuxphones@lemmy.ca has that as its community even though the
#               account lives elsewhere — which is exactly the distinction that
#               matters here.
#   permalink : post.ap_id, the canonical federated URL. Correct even when the
#               post lives on another instance, which post.id is not. This is
#               also what the "discuss on the fediverse" link uses, so the reader
#               lands on the real thread rather than a local mirror of it.
#   media     : the post's MAIN media only — `post.url` — not images embedded in
#               the body. The API distinguishes them and so should we: `url` is
#               what the post is about (the screen recording of the work), while
#               body images are illustrations inside the prose, usually
#               screenshots of comments. Pulling both in meant a card showing
#               four files where one was the point.
#   poster    : post.thumbnail_url, the instance-generated still of that media.
#               Used as a video poster so the player shows a frame instead of a
#               black box before playing.
#
#               Both are ORIGINAL urls here; tools/fetch-media.sh mirrors them
#               and rewrites to local paths, so nothing the browser loads is
#               third-party.
#   excerpt   : body flattened to one line and truncated. Markdown is NOT
#               rendered — the site has no markdown pipeline by design, so any
#               surviving syntax would show as literal characters. Strip the
#               common inline markers and let the rest be plain text.
#   deleted / removed posts are dropped rather than rendered as empty cards.
if ! jq --argjson n "$EXCERPT_CHARS" \
        --slurpfile cfg "$CONFIG" '
    ($cfg[0].communities | map(ascii_downcase)) as $allow
    | [ .posts[]
      | select((.post.deleted // false) == false)
      | select((.post.removed // false) == false)
      | . as $p
      | ((.community.name // "") + "@" +
         ((.community.actor_id // "") | sub("^https?://"; "") | sub("/c/.*$"; ""))) as $comm
      | select(($comm | ascii_downcase) as $c | $allow | index($c))
      | {
          title:     ($p.post.name // ""),
          permalink: ($p.post.ap_id // ""),
          # A link post whose target IS an image or video is a media post, not a
          # link post: the file is captured in `media` and embedded, so keeping
          # it here too would render the raw URL as text right above the thing
          # it points at.
          url:       (($p.post.url // "")
                        | if test("\\.(?:mp4|webm|mov|webp|png|jpe?g|gif|avif)$") then "" else . end),
          community: $comm,
          published: ($p.post.published // ""),
          excerpt:   (($p.post.body // "")
                        | gsub("\r"; "")
                        | gsub("\n+"; " ")
                        | gsub("!?\\[(?<t>[^\\]]*)\\]\\([^)]*\\)"; "\(.t)")
                        | gsub("[*_`>#]"; "")
                        # Strip every bare URL, not just media ones. A raw link
                        # in a 280-character preview is noise the reader cannot
                        # use, and the card already links to the thread — where
                        # the link is clickable in its original context.
                        | gsub("https?://[^ )\\]]+"; "")
                        | gsub("  +"; " ")
                        | ltrimstr(" ") | rtrimstr(" ")
                        | if (. | length) > $n then (.[0:$n] | sub(" [^ ]*$"; "")) + "…" else . end),
          media:     ([ ($p.post.url // "")
                        | select(test("\\.(?:mp4|webm|mov|webp|png|jpe?g|gif|avif)$"))
                        | (if test("\\.(mp4|webm|mov)$") then "video" else "image" end) as $kind
                        | { src: .,
                            kind: $kind,
                            # Videos only. For an image post thumbnail_url is a
                            # scaled copy of the image itself, and <img> has no
                            # poster attribute — carrying it would mirror a
                            # second file to render nothing.
                            poster: (if $kind == "video"
                                     then (($p.post.thumbnail_url // "")
                                            | select(test("\\.(?:webp|png|jpe?g|gif|avif)$")) // "")
                                     else "" end) } ]),
          score:     ($p.counts.score // 0),
          comments:  ($p.counts.comments // 0)
        }
    ]' "$RAW" > "$TMP" 2>/dev/null; then
    echo "fetch-posts: response did not match the expected shape, keeping existing $OUT" >&2
    exit 0
fi

# Refuse to replace good content with an empty list. Zero matches usually means
# the allowlist and the account have drifted apart, or the API shape changed —
# either way, silently emptying the posts page on the next deploy is the wrong
# response.
COUNT="$(jq 'length' "$TMP")"
if [ "$COUNT" -eq 0 ]; then
    echo "fetch-posts: no posts matched the community allowlist, keeping existing $OUT" >&2
    echo "fetch-posts: allowlist is $(jq -c '.communities' "$CONFIG")" >&2
    exit 0
fi

# ── point each link at the community's instance ─────────────────────────
#
# `post.ap_id` is the ActivityPub canonical id, and for a post created from this
# account it is on the account's own instance — so linking it sends readers
# there. That is the wrong destination twice over: the community lives somewhere
# else, and the account is not what the site should be advertising.
#
# The community's instance has its own federated copy of the thread at a
# different local id. `resolve_object` is how to find it: hand the instance the
# ap_id and it answers with its local view.
#
# Per post, one request, at build time. Failure is not fatal — the entry keeps
# its ap_id, which still reaches a readable copy of the thread.
resolved=0
kept=0
LINKED="$(mktemp)"
printf '[]' > "$LINKED"

# shellcheck disable=SC2016
while IFS="$(printf '\t')" read -r ap comm; do
    [ -n "$ap" ] || continue
    host=${comm#*@}
    if [ -z "$host" ] || [ "$host" = "$comm" ]; then
        kept=$((kept + 1)); continue
    fi
    local_id=$(curl -fsS --max-time 15 \
                    -A 'catcrafts.net-buildfetch/1.0 (+https://catcrafts.net)' \
                    "https://$host/api/v3/resolve_object?q=$ap" 2>/dev/null \
               | jq -r '.post.post.id // empty' 2>/dev/null || true)
    case "$local_id" in
        ''|*[!0-9]*)
            echo "fetch-posts: could not resolve $ap on $host, keeping the ap_id" >&2
            kept=$((kept + 1))
            ;;
        *)
            jq --arg ap "$ap" --arg url "https://$host/post/$local_id" \
               '. + [{ap: $ap, url: $url}]' "$LINKED" > "$LINKED.new" \
               && mv "$LINKED.new" "$LINKED"
            resolved=$((resolved + 1))
            ;;
    esac
done <<EOF
$(jq -r '.[] | [.permalink, .community] | @tsv' "$TMP")
EOF

REWRITTEN="$(mktemp)"
if jq --slurpfile linked "$LINKED" '
    ($linked[0] | map({key: .ap, value: .url}) | from_entries) as $m
    | map(. + { permalink: ($m[.permalink] // .permalink) })' "$TMP" > "$REWRITTEN" 2>/dev/null; then
    mv "$REWRITTEN" "$TMP"
else
    rm -f "$REWRITTEN"
    echo "fetch-posts: link rewrite failed, keeping ap_ids" >&2
fi
rm -f "$LINKED"

mkdir -p content
# mktemp files are born 0600, and that mode survives rsync -a all the way to
# production — where the server runs as another user and Caddy 403s the copy
# in the web root. The media loop already learned this (chmod there too).
chmod 0644 "$TMP"
mv "$TMP" "$OUT"
trap - EXIT
rm -f "$RAW"
echo "fetch-posts: wrote $COUNT posts to $OUT (from $COMMUNITY_COUNT allowed communities)"
echo "fetch-posts: $resolved links point at the community instance, $kept fell back to the ap_id"