This commit is contained in:
parent
fb2f6079cc
commit
934c94cb5c
50 changed files with 10464 additions and 758 deletions
219
tools/fetch-posts.sh
Executable file
219
tools/fetch-posts.sh
Executable file
|
|
@ -0,0 +1,219 @@
|
|||
#!/bin/sh
|
||||
# Fetch selected fediverse posts and write content/posts.json.
|
||||
#
|
||||
# Runs at BUILD time, not run time. The site does not crawl anything, does not
|
||||
# proxy, and does not mirror comments — each card links out to the thread on
|
||||
# whatever instance it lives on, which is where the discussion belongs. That
|
||||
# means no sync service and no runtime dependency on any instance being up.
|
||||
#
|
||||
# WHICH POSTS: the Lemmy user API returns everything the account has posted,
|
||||
# across every community. That is not what belongs on a site about this work, so
|
||||
# the result is filtered against the community allowlist in
|
||||
# content/posts-sources.json. Joining a new community does not silently publish
|
||||
# to the site — it has to be added there first.
|
||||
#
|
||||
# The output is a flat array of exactly the fields Catcrafts.Shared:Model reads.
|
||||
# Doing the transformation here rather than in C++ keeps the parser small and
|
||||
# makes an upstream API change a one-file fix in shell.
|
||||
#
|
||||
# usage: tools/fetch-posts.sh [config-file]
|
||||
#
|
||||
# On any failure the existing content/posts.json is left untouched and the script
|
||||
# exits 0. A build must not fail because an instance was down, and stale posts
|
||||
# are strictly better than an empty page.
|
||||
|
||||
set -eu
|
||||
|
||||
CONFIG="${1:-content/posts-sources.json}"
|
||||
OUT="content/posts.json"
|
||||
LIMIT=50
|
||||
EXCERPT_CHARS=280
|
||||
|
||||
command -v jq >/dev/null 2>&1 || { echo "fetch-posts: jq not found, keeping existing $OUT" >&2; exit 0; }
|
||||
[ -f "$CONFIG" ] || { echo "fetch-posts: $CONFIG not found, keeping existing $OUT" >&2; exit 0; }
|
||||
|
||||
USER_NAME=$(jq -r '.username // empty' "$CONFIG")
|
||||
INSTANCE=$(jq -r '.instance // empty' "$CONFIG")
|
||||
[ -n "$USER_NAME" ] && [ -n "$INSTANCE" ] || {
|
||||
echo "fetch-posts: $CONFIG needs .username and .instance, keeping existing $OUT" >&2; exit 0; }
|
||||
|
||||
# An empty allowlist would silently publish everything, which is the opposite of
|
||||
# what this file is for — treat it as a configuration error, not as "allow all".
|
||||
COMMUNITY_COUNT=$(jq '.communities | length' "$CONFIG")
|
||||
[ "$COMMUNITY_COUNT" -gt 0 ] || {
|
||||
echo "fetch-posts: .communities is empty — refusing to publish every community" >&2
|
||||
echo "fetch-posts: keeping existing $OUT" >&2; exit 0; }
|
||||
|
||||
TMP="$(mktemp)"
|
||||
RAW="$(mktemp)"
|
||||
trap 'rm -f "$TMP" "$RAW"' EXIT
|
||||
|
||||
URL="$INSTANCE/api/v3/user?username=$USER_NAME&sort=New&limit=$LIMIT"
|
||||
|
||||
# Identify ourselves: an unattributed scraper on a small instance is rude and
|
||||
# more likely to get blocked.
|
||||
if ! curl -fsS --max-time 25 \
|
||||
-H 'Accept: application/json' \
|
||||
-A 'catcrafts.net-buildfetch/1.0 (+https://catcrafts.net)' \
|
||||
"$URL" -o "$RAW"; then
|
||||
echo "fetch-posts: request failed, keeping existing $OUT" >&2
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Map PostView -> the flat shape the C++ loader reads.
|
||||
#
|
||||
# community : assembled as name@instance from the community's actor_id, which
|
||||
# is what the allowlist matches on. A post made INTO
|
||||
# linuxphones@lemmy.ca has that as its community even though the
|
||||
# account lives elsewhere — which is exactly the distinction that
|
||||
# matters here.
|
||||
# permalink : post.ap_id, the canonical federated URL. Correct even when the
|
||||
# post lives on another instance, which post.id is not. This is
|
||||
# also what the "discuss on the fediverse" link uses, so the reader
|
||||
# lands on the real thread rather than a local mirror of it.
|
||||
# media : the post's MAIN media only — `post.url` — not images embedded in
|
||||
# the body. The API distinguishes them and so should we: `url` is
|
||||
# what the post is about (the screen recording of the work), while
|
||||
# body images are illustrations inside the prose, usually
|
||||
# screenshots of comments. Pulling both in meant a card showing
|
||||
# four files where one was the point.
|
||||
# poster : post.thumbnail_url, the instance-generated still of that media.
|
||||
# Used as a video poster so the player shows a frame instead of a
|
||||
# black box before playing.
|
||||
#
|
||||
# Both are ORIGINAL urls here; tools/fetch-media.sh mirrors them
|
||||
# and rewrites to local paths, so nothing the browser loads is
|
||||
# third-party.
|
||||
# excerpt : body flattened to one line and truncated. Markdown is NOT
|
||||
# rendered — the site has no markdown pipeline by design, so any
|
||||
# surviving syntax would show as literal characters. Strip the
|
||||
# common inline markers and let the rest be plain text.
|
||||
# deleted / removed posts are dropped rather than rendered as empty cards.
|
||||
if ! jq --argjson n "$EXCERPT_CHARS" \
|
||||
--slurpfile cfg "$CONFIG" '
|
||||
($cfg[0].communities | map(ascii_downcase)) as $allow
|
||||
| [ .posts[]
|
||||
| select((.post.deleted // false) == false)
|
||||
| select((.post.removed // false) == false)
|
||||
| . as $p
|
||||
| ((.community.name // "") + "@" +
|
||||
((.community.actor_id // "") | sub("^https?://"; "") | sub("/c/.*$"; ""))) as $comm
|
||||
| select(($comm | ascii_downcase) as $c | $allow | index($c))
|
||||
| {
|
||||
title: ($p.post.name // ""),
|
||||
permalink: ($p.post.ap_id // ""),
|
||||
# A link post whose target IS an image or video is a media post, not a
|
||||
# link post: the file is captured in `media` and embedded, so keeping
|
||||
# it here too would render the raw URL as text right above the thing
|
||||
# it points at.
|
||||
url: (($p.post.url // "")
|
||||
| if test("\\.(?:mp4|webm|mov|webp|png|jpe?g|gif|avif)$") then "" else . end),
|
||||
community: $comm,
|
||||
published: ($p.post.published // ""),
|
||||
excerpt: (($p.post.body // "")
|
||||
| gsub("\r"; "")
|
||||
| gsub("\n+"; " ")
|
||||
| gsub("!?\\[(?<t>[^\\]]*)\\]\\([^)]*\\)"; "\(.t)")
|
||||
| gsub("[*_`>#]"; "")
|
||||
# Strip every bare URL, not just media ones. A raw link
|
||||
# in a 280-character preview is noise the reader cannot
|
||||
# use, and the card already links to the thread — where
|
||||
# the link is clickable in its original context.
|
||||
| gsub("https?://[^ )\\]]+"; "")
|
||||
| gsub(" +"; " ")
|
||||
| ltrimstr(" ") | rtrimstr(" ")
|
||||
| if (. | length) > $n then (.[0:$n] | sub(" [^ ]*$"; "")) + "…" else . end),
|
||||
media: ([ ($p.post.url // "")
|
||||
| select(test("\\.(?:mp4|webm|mov|webp|png|jpe?g|gif|avif)$"))
|
||||
| (if test("\\.(mp4|webm|mov)$") then "video" else "image" end) as $kind
|
||||
| { src: .,
|
||||
kind: $kind,
|
||||
# Videos only. For an image post thumbnail_url is a
|
||||
# scaled copy of the image itself, and <img> has no
|
||||
# poster attribute — carrying it would mirror a
|
||||
# second file to render nothing.
|
||||
poster: (if $kind == "video"
|
||||
then (($p.post.thumbnail_url // "")
|
||||
| select(test("\\.(?:webp|png|jpe?g|gif|avif)$")) // "")
|
||||
else "" end) } ]),
|
||||
score: ($p.counts.score // 0),
|
||||
comments: ($p.counts.comments // 0)
|
||||
}
|
||||
]' "$RAW" > "$TMP" 2>/dev/null; then
|
||||
echo "fetch-posts: response did not match the expected shape, keeping existing $OUT" >&2
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Refuse to replace good content with an empty list. Zero matches usually means
|
||||
# the allowlist and the account have drifted apart, or the API shape changed —
|
||||
# either way, silently emptying the posts page on the next deploy is the wrong
|
||||
# response.
|
||||
COUNT="$(jq 'length' "$TMP")"
|
||||
if [ "$COUNT" -eq 0 ]; then
|
||||
echo "fetch-posts: no posts matched the community allowlist, keeping existing $OUT" >&2
|
||||
echo "fetch-posts: allowlist is $(jq -c '.communities' "$CONFIG")" >&2
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ── point each link at the community's instance ─────────────────────────
|
||||
#
|
||||
# `post.ap_id` is the ActivityPub canonical id, and for a post created from this
|
||||
# account it is on the account's own instance — so linking it sends readers
|
||||
# there. That is the wrong destination twice over: the community lives somewhere
|
||||
# else, and the account is not what the site should be advertising.
|
||||
#
|
||||
# The community's instance has its own federated copy of the thread at a
|
||||
# different local id. `resolve_object` is how to find it: hand the instance the
|
||||
# ap_id and it answers with its local view.
|
||||
#
|
||||
# Per post, one request, at build time. Failure is not fatal — the entry keeps
|
||||
# its ap_id, which still reaches a readable copy of the thread.
|
||||
resolved=0
|
||||
kept=0
|
||||
LINKED="$(mktemp)"
|
||||
printf '[]' > "$LINKED"
|
||||
|
||||
# shellcheck disable=SC2016
|
||||
while IFS="$(printf '\t')" read -r ap comm; do
|
||||
[ -n "$ap" ] || continue
|
||||
host=${comm#*@}
|
||||
if [ -z "$host" ] || [ "$host" = "$comm" ]; then
|
||||
kept=$((kept + 1)); continue
|
||||
fi
|
||||
local_id=$(curl -fsS --max-time 15 \
|
||||
-A 'catcrafts.net-buildfetch/1.0 (+https://catcrafts.net)' \
|
||||
"https://$host/api/v3/resolve_object?q=$ap" 2>/dev/null \
|
||||
| jq -r '.post.post.id // empty' 2>/dev/null || true)
|
||||
case "$local_id" in
|
||||
''|*[!0-9]*)
|
||||
echo "fetch-posts: could not resolve $ap on $host, keeping the ap_id" >&2
|
||||
kept=$((kept + 1))
|
||||
;;
|
||||
*)
|
||||
jq --arg ap "$ap" --arg url "https://$host/post/$local_id" \
|
||||
'. + [{ap: $ap, url: $url}]' "$LINKED" > "$LINKED.new" \
|
||||
&& mv "$LINKED.new" "$LINKED"
|
||||
resolved=$((resolved + 1))
|
||||
;;
|
||||
esac
|
||||
done <<EOF
|
||||
$(jq -r '.[] | [.permalink, .community] | @tsv' "$TMP")
|
||||
EOF
|
||||
|
||||
REWRITTEN="$(mktemp)"
|
||||
if jq --slurpfile linked "$LINKED" '
|
||||
($linked[0] | map({key: .ap, value: .url}) | from_entries) as $m
|
||||
| map(. + { permalink: ($m[.permalink] // .permalink) })' "$TMP" > "$REWRITTEN" 2>/dev/null; then
|
||||
mv "$REWRITTEN" "$TMP"
|
||||
else
|
||||
rm -f "$REWRITTEN"
|
||||
echo "fetch-posts: link rewrite failed, keeping ap_ids" >&2
|
||||
fi
|
||||
rm -f "$LINKED"
|
||||
|
||||
mkdir -p content
|
||||
mv "$TMP" "$OUT"
|
||||
trap - EXIT
|
||||
rm -f "$RAW"
|
||||
echo "fetch-posts: wrote $COUNT posts to $OUT (from $COMMUNITY_COUNT allowed communities)"
|
||||
echo "fetch-posts: $resolved links point at the community instance, $kept fell back to the ap_id"
|
||||
Loading…
Reference in a new issue