219 lines
10 KiB
Shell
Executable file
219 lines
10 KiB
Shell
Executable file
#!/bin/sh
|
|
# Fetch selected fediverse posts and write content/posts.json.
|
|
#
|
|
# Runs at BUILD time, not run time. The site does not crawl anything, does not
|
|
# proxy, and does not mirror comments — each card links out to the thread on
|
|
# whatever instance it lives on, which is where the discussion belongs. That
|
|
# means no sync service and no runtime dependency on any instance being up.
|
|
#
|
|
# WHICH POSTS: the Lemmy user API returns everything the account has posted,
|
|
# across every community. That is not what belongs on a site about this work, so
|
|
# the result is filtered against the community allowlist in
|
|
# content/posts-sources.json. Joining a new community does not silently publish
|
|
# to the site — it has to be added there first.
|
|
#
|
|
# The output is a flat array of exactly the fields Catcrafts.Shared:Model reads.
|
|
# Doing the transformation here rather than in C++ keeps the parser small and
|
|
# makes an upstream API change a one-file fix in shell.
|
|
#
|
|
# usage: tools/fetch-posts.sh [config-file]
|
|
#
|
|
# On any failure the existing content/posts.json is left untouched and the script
|
|
# exits 0. A build must not fail because an instance was down, and stale posts
|
|
# are strictly better than an empty page.
|
|
|
|
set -eu
|
|
|
|
CONFIG="${1:-content/posts-sources.json}"
|
|
OUT="content/posts.json"
|
|
LIMIT=50
|
|
EXCERPT_CHARS=280
|
|
|
|
command -v jq >/dev/null 2>&1 || { echo "fetch-posts: jq not found, keeping existing $OUT" >&2; exit 0; }
|
|
[ -f "$CONFIG" ] || { echo "fetch-posts: $CONFIG not found, keeping existing $OUT" >&2; exit 0; }
|
|
|
|
USER_NAME=$(jq -r '.username // empty' "$CONFIG")
|
|
INSTANCE=$(jq -r '.instance // empty' "$CONFIG")
|
|
[ -n "$USER_NAME" ] && [ -n "$INSTANCE" ] || {
|
|
echo "fetch-posts: $CONFIG needs .username and .instance, keeping existing $OUT" >&2; exit 0; }
|
|
|
|
# An empty allowlist would silently publish everything, which is the opposite of
|
|
# what this file is for — treat it as a configuration error, not as "allow all".
|
|
COMMUNITY_COUNT=$(jq '.communities | length' "$CONFIG")
|
|
[ "$COMMUNITY_COUNT" -gt 0 ] || {
|
|
echo "fetch-posts: .communities is empty — refusing to publish every community" >&2
|
|
echo "fetch-posts: keeping existing $OUT" >&2; exit 0; }
|
|
|
|
TMP="$(mktemp)"
|
|
RAW="$(mktemp)"
|
|
trap 'rm -f "$TMP" "$RAW"' EXIT
|
|
|
|
URL="$INSTANCE/api/v3/user?username=$USER_NAME&sort=New&limit=$LIMIT"
|
|
|
|
# Identify ourselves: an unattributed scraper on a small instance is rude and
|
|
# more likely to get blocked.
|
|
if ! curl -fsS --max-time 25 \
|
|
-H 'Accept: application/json' \
|
|
-A 'catcrafts.net-buildfetch/1.0 (+https://catcrafts.net)' \
|
|
"$URL" -o "$RAW"; then
|
|
echo "fetch-posts: request failed, keeping existing $OUT" >&2
|
|
exit 0
|
|
fi
|
|
|
|
# Map PostView -> the flat shape the C++ loader reads.
|
|
#
|
|
# community : assembled as name@instance from the community's actor_id, which
|
|
# is what the allowlist matches on. A post made INTO
|
|
# linuxphones@lemmy.ca has that as its community even though the
|
|
# account lives elsewhere — which is exactly the distinction that
|
|
# matters here.
|
|
# permalink : post.ap_id, the canonical federated URL. Correct even when the
|
|
# post lives on another instance, which post.id is not. This is
|
|
# also what the "discuss on the fediverse" link uses, so the reader
|
|
# lands on the real thread rather than a local mirror of it.
|
|
# media : the post's MAIN media only — `post.url` — not images embedded in
|
|
# the body. The API distinguishes them and so should we: `url` is
|
|
# what the post is about (the screen recording of the work), while
|
|
# body images are illustrations inside the prose, usually
|
|
# screenshots of comments. Pulling both in meant a card showing
|
|
# four files where one was the point.
|
|
# poster : post.thumbnail_url, the instance-generated still of that media.
|
|
# Used as a video poster so the player shows a frame instead of a
|
|
# black box before playing.
|
|
#
|
|
# Both are ORIGINAL urls here; tools/fetch-media.sh mirrors them
|
|
# and rewrites to local paths, so nothing the browser loads is
|
|
# third-party.
|
|
# excerpt : body flattened to one line and truncated. Markdown is NOT
|
|
# rendered — the site has no markdown pipeline by design, so any
|
|
# surviving syntax would show as literal characters. Strip the
|
|
# common inline markers and let the rest be plain text.
|
|
# deleted / removed posts are dropped rather than rendered as empty cards.
|
|
if ! jq --argjson n "$EXCERPT_CHARS" \
|
|
--slurpfile cfg "$CONFIG" '
|
|
($cfg[0].communities | map(ascii_downcase)) as $allow
|
|
| [ .posts[]
|
|
| select((.post.deleted // false) == false)
|
|
| select((.post.removed // false) == false)
|
|
| . as $p
|
|
| ((.community.name // "") + "@" +
|
|
((.community.actor_id // "") | sub("^https?://"; "") | sub("/c/.*$"; ""))) as $comm
|
|
| select(($comm | ascii_downcase) as $c | $allow | index($c))
|
|
| {
|
|
title: ($p.post.name // ""),
|
|
permalink: ($p.post.ap_id // ""),
|
|
# A link post whose target IS an image or video is a media post, not a
|
|
# link post: the file is captured in `media` and embedded, so keeping
|
|
# it here too would render the raw URL as text right above the thing
|
|
# it points at.
|
|
url: (($p.post.url // "")
|
|
| if test("\\.(?:mp4|webm|mov|webp|png|jpe?g|gif|avif)$") then "" else . end),
|
|
community: $comm,
|
|
published: ($p.post.published // ""),
|
|
excerpt: (($p.post.body // "")
|
|
| gsub("\r"; "")
|
|
| gsub("\n+"; " ")
|
|
| gsub("!?\\[(?<t>[^\\]]*)\\]\\([^)]*\\)"; "\(.t)")
|
|
| gsub("[*_`>#]"; "")
|
|
# Strip every bare URL, not just media ones. A raw link
|
|
# in a 280-character preview is noise the reader cannot
|
|
# use, and the card already links to the thread — where
|
|
# the link is clickable in its original context.
|
|
| gsub("https?://[^ )\\]]+"; "")
|
|
| gsub(" +"; " ")
|
|
| ltrimstr(" ") | rtrimstr(" ")
|
|
| if (. | length) > $n then (.[0:$n] | sub(" [^ ]*$"; "")) + "…" else . end),
|
|
media: ([ ($p.post.url // "")
|
|
| select(test("\\.(?:mp4|webm|mov|webp|png|jpe?g|gif|avif)$"))
|
|
| (if test("\\.(mp4|webm|mov)$") then "video" else "image" end) as $kind
|
|
| { src: .,
|
|
kind: $kind,
|
|
# Videos only. For an image post thumbnail_url is a
|
|
# scaled copy of the image itself, and <img> has no
|
|
# poster attribute — carrying it would mirror a
|
|
# second file to render nothing.
|
|
poster: (if $kind == "video"
|
|
then (($p.post.thumbnail_url // "")
|
|
| select(test("\\.(?:webp|png|jpe?g|gif|avif)$")) // "")
|
|
else "" end) } ]),
|
|
score: ($p.counts.score // 0),
|
|
comments: ($p.counts.comments // 0)
|
|
}
|
|
]' "$RAW" > "$TMP" 2>/dev/null; then
|
|
echo "fetch-posts: response did not match the expected shape, keeping existing $OUT" >&2
|
|
exit 0
|
|
fi
|
|
|
|
# Refuse to replace good content with an empty list. Zero matches usually means
|
|
# the allowlist and the account have drifted apart, or the API shape changed —
|
|
# either way, silently emptying the posts page on the next deploy is the wrong
|
|
# response.
|
|
COUNT="$(jq 'length' "$TMP")"
|
|
if [ "$COUNT" -eq 0 ]; then
|
|
echo "fetch-posts: no posts matched the community allowlist, keeping existing $OUT" >&2
|
|
echo "fetch-posts: allowlist is $(jq -c '.communities' "$CONFIG")" >&2
|
|
exit 0
|
|
fi
|
|
|
|
# ── point each link at the community's instance ─────────────────────────
|
|
#
|
|
# `post.ap_id` is the ActivityPub canonical id, and for a post created from this
|
|
# account it is on the account's own instance — so linking it sends readers
|
|
# there. That is the wrong destination twice over: the community lives somewhere
|
|
# else, and the account is not what the site should be advertising.
|
|
#
|
|
# The community's instance has its own federated copy of the thread at a
|
|
# different local id. `resolve_object` is how to find it: hand the instance the
|
|
# ap_id and it answers with its local view.
|
|
#
|
|
# Per post, one request, at build time. Failure is not fatal — the entry keeps
|
|
# its ap_id, which still reaches a readable copy of the thread.
|
|
resolved=0
|
|
kept=0
|
|
LINKED="$(mktemp)"
|
|
printf '[]' > "$LINKED"
|
|
|
|
# shellcheck disable=SC2016
|
|
while IFS="$(printf '\t')" read -r ap comm; do
|
|
[ -n "$ap" ] || continue
|
|
host=${comm#*@}
|
|
if [ -z "$host" ] || [ "$host" = "$comm" ]; then
|
|
kept=$((kept + 1)); continue
|
|
fi
|
|
local_id=$(curl -fsS --max-time 15 \
|
|
-A 'catcrafts.net-buildfetch/1.0 (+https://catcrafts.net)' \
|
|
"https://$host/api/v3/resolve_object?q=$ap" 2>/dev/null \
|
|
| jq -r '.post.post.id // empty' 2>/dev/null || true)
|
|
case "$local_id" in
|
|
''|*[!0-9]*)
|
|
echo "fetch-posts: could not resolve $ap on $host, keeping the ap_id" >&2
|
|
kept=$((kept + 1))
|
|
;;
|
|
*)
|
|
jq --arg ap "$ap" --arg url "https://$host/post/$local_id" \
|
|
'. + [{ap: $ap, url: $url}]' "$LINKED" > "$LINKED.new" \
|
|
&& mv "$LINKED.new" "$LINKED"
|
|
resolved=$((resolved + 1))
|
|
;;
|
|
esac
|
|
done <<EOF
|
|
$(jq -r '.[] | [.permalink, .community] | @tsv' "$TMP")
|
|
EOF
|
|
|
|
REWRITTEN="$(mktemp)"
|
|
if jq --slurpfile linked "$LINKED" '
|
|
($linked[0] | map({key: .ap, value: .url}) | from_entries) as $m
|
|
| map(. + { permalink: ($m[.permalink] // .permalink) })' "$TMP" > "$REWRITTEN" 2>/dev/null; then
|
|
mv "$REWRITTEN" "$TMP"
|
|
else
|
|
rm -f "$REWRITTEN"
|
|
echo "fetch-posts: link rewrite failed, keeping ap_ids" >&2
|
|
fi
|
|
rm -f "$LINKED"
|
|
|
|
mkdir -p content
|
|
mv "$TMP" "$OUT"
|
|
trap - EXIT
|
|
rm -f "$RAW"
|
|
echo "fetch-posts: wrote $COUNT posts to $OUT (from $COMMUNITY_COUNT allowed communities)"
|
|
echo "fetch-posts: $resolved links point at the community instance, $kept fell back to the ap_id"
|