Jorijn van der Graaf
faf881fa0c
All checks were successful
Deploy / build-deploy (push) Successful in 3m0s
257 lines
12 KiB
Shell
Executable file
257 lines
12 KiB
Shell
Executable file
#!/bin/sh
|
|
# Publish a recording, screenshot or audio file to catcrafts.net and print the
|
|
# URL to put in the fediverse post.
|
|
#
|
|
# usage: tools/publish-media.sh [-r] [-c CRF] <file>...
|
|
#
|
|
# -r, --raw upload the file unchanged (no transcode, no poster)
|
|
# -c, --crf N AV1 quality, lower is better (default 32)
|
|
#
|
|
# WHY THIS EXISTS: the media used to be uploaded to a third-party file host and
|
|
# then mirrored back at build time by tools/fetch-media.sh. That worked until a
|
|
# 167 MB recording hit the mirror's size cap, and the deploy failed on a file we
|
|
# had sitting on disk the whole time. Publishing to our own origin FIRST inverts
|
|
# it — the URL in the post is already the URL the page wants, so there is nothing
|
|
# to download, no size cap to clear, and no third party who can be slow, rate-
|
|
# limit us, or delete the file that IS the post.
|
|
#
|
|
# The name is the content hash, matching what fetch-media.sh produces, which is
|
|
# what makes Caddy's one-year immutable cache on /media honest: different bytes
|
|
# can never appear under a name someone already has cached.
|
|
#
|
|
# TRANSCODE: phone recordings are enormous for what they show — the one that
|
|
# started this was 78 s of a dark room at 17 Mbps, 167 MB, where denoising and
|
|
# AV1 give the same picture in 15 MB. Dark handheld footage is mostly sensor
|
|
# noise, which is expensive to encode and worth nothing, so hqdn3d earns its
|
|
# place before the encoder rather than after it.
|
|
#
|
|
# ROTATION: phones record landscape and attach a display matrix instead of
|
|
# rotating pixels. Transcoding bakes the rotation into the frames, so what every
|
|
# downstream consumer measures is what the viewer actually sees — see the
|
|
# rotation note in fetch-media.sh for what goes wrong when it does not.
|
|
#
|
|
# FALLBACK: AV1 alone locks out Safari before 17 and Apple hardware older than
|
|
# A17/M3 — which surfaced the first time an AV1 link was posted and an iPhone
|
|
# viewer got "bad media error" from the raw file. So every transcode also emits
|
|
# an H.264 sibling, uploaded as <hash>.h264.mp4 next to the AV1 (same trick as
|
|
# the poster: named after the AV1's hash so fetch-media.sh finds it with nothing
|
|
# to look up). The URL this script prints for the post is the H.264 one, because
|
|
# a post's link is fetched raw — Lemmy apps and browsers play that exact file,
|
|
# with no <source> negotiation in front of it — and it must be the encoding
|
|
# everything can play. Our own pages recognise the .h264.mp4 name, serve the AV1
|
|
# to browsers that can take it, and keep the H.264 as the <source> fallback.
|
|
#
|
|
# AUDIO: the same two rails, for the same reason. Opus in WebM is the efficient
|
|
# one — the first file through here, 93 s of speech, went from a 1.5 MB 128k MP3
|
|
# to 553 KB — and an MP3 sibling is the one that plays everywhere, so the MP3 is
|
|
# the URL printed for the post. The primary does not have to be universal
|
|
# precisely because the sibling is.
|
|
#
|
|
# Two ways the audio path deliberately differs from the video path above. It does
|
|
# not fold everything to mono: here the audio IS the content rather than a
|
|
# rounding error next to a picture, so the source's channel layout survives and
|
|
# sets the bitrate. And a source that is ALREADY an MP3 is copied to the
|
|
# compatibility rail untouched rather than re-encoded — MP3 to MP3 spends a
|
|
# second generation of lossy artefacts to arrive at the same rail, and speech,
|
|
# which is most of what gets published here, is where that is most audible.
|
|
#
|
|
# Nothing downstream builds an <audio> element yet: fetch-media.sh does not match
|
|
# these names and the post renderer has no audio case. So the printed URL is for
|
|
# posting and for linking by hand, not yet for embedding in a post body.
|
|
|
|
set -eu
|
|
|
|
MEDIA_HOST="${MEDIA_HOST:-root@catcrafts.net}"
|
|
MEDIA_PATH="${MEDIA_PATH:-/srv/catcrafts-app/media}"
|
|
SITE_ORIGIN="${SITE_ORIGIN:-https://catcrafts.net}"
|
|
CRF="${CRF:-32}"
|
|
RAW=0
|
|
|
|
for c in ffmpeg ffprobe sha256sum ssh scp; do
|
|
command -v "$c" >/dev/null 2>&1 || { echo "publish-media: $c not found" >&2; exit 1; }
|
|
done
|
|
|
|
FILES=""
|
|
while [ $# -gt 0 ]; do
|
|
case "$1" in
|
|
-r|--raw) RAW=1; shift ;;
|
|
-c|--crf) CRF="$2"; shift 2 ;;
|
|
-h|--help) sed -n '2,10p' "$0"; exit 0 ;;
|
|
-*) echo "publish-media: unknown option: $1" >&2; exit 1 ;;
|
|
*) FILES="$FILES $1"; shift ;;
|
|
esac
|
|
done
|
|
[ -n "$FILES" ] || { echo "publish-media: no input file. See --help." >&2; exit 1; }
|
|
|
|
WORK="$(mktemp -d)"
|
|
|
|
# One SSH connection for the whole run, not one per operation. The origin runs
|
|
# `ufw limit 22/tcp`, which rejects the sixth new connection from a source inside
|
|
# 30 s — and a video publishes three artifacts at three connections each (probe,
|
|
# copy, rename), so a run used to trip the limit on the poster's rename and die
|
|
# with the H.264 fallback still unsent. The failure does not even look like rate
|
|
# limiting: ufw REJECTs with ICMP port-unreachable, ssh moves on to the AAAA
|
|
# record, and reports the errno of the last address it tried — so a refused IPv4
|
|
# connection surfaces as "Network is unreachable" from a host with no IPv6 route.
|
|
# Multiplexing over one master socket keeps a run at a single connection however
|
|
# many artifacts it produces, and leaves the limit in place doing its real job.
|
|
SSH_CTL="$WORK/cm"
|
|
SSH_OPTS="-o BatchMode=yes -o ControlMaster=auto -o ControlPath=$SSH_CTL -o ControlPersist=60"
|
|
cleanup() {
|
|
if [ -S "$SSH_CTL" ]; then
|
|
ssh -O exit $SSH_OPTS "$MEDIA_HOST" >/dev/null 2>&1 || true
|
|
fi
|
|
rm -rf "$WORK"
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
# Upload only what is not already there. The name is the content hash, so a name
|
|
# that exists on the mount holds these exact bytes and re-sending them would
|
|
# change nothing — which makes re-running this script on the same file free, and
|
|
# makes a half-finished batch safe to just run again.
|
|
publish_file() {
|
|
local_file=$1
|
|
remote_name=$2
|
|
if ssh $SSH_OPTS "$MEDIA_HOST" "test -f '$MEDIA_PATH/$remote_name'"; then
|
|
echo " already published: $remote_name"
|
|
return 0
|
|
fi
|
|
# Land it under a temporary name and move it into place, so a reader can
|
|
# never see a partial file under the name its hash promises.
|
|
scp -q $SSH_OPTS "$local_file" "$MEDIA_HOST:$MEDIA_PATH/.$remote_name.part"
|
|
ssh $SSH_OPTS "$MEDIA_HOST" \
|
|
"chmod 0644 '$MEDIA_PATH/.$remote_name.part' && \
|
|
mv -f '$MEDIA_PATH/.$remote_name.part' '$MEDIA_PATH/$remote_name'"
|
|
echo " uploaded: $remote_name"
|
|
}
|
|
|
|
for src in $FILES; do
|
|
[ -f "$src" ] || { echo "publish-media: no such file: $src" >&2; exit 1; }
|
|
echo "$src"
|
|
|
|
ext=$(printf '%s' "$src" | sed -E 's/.*\.([A-Za-z0-9]+)$/\1/' | tr 'A-Z' 'a-z')
|
|
kind=other
|
|
case "$ext" in
|
|
mp4|webm|mov|mkv|avi) kind=video ;;
|
|
png|jpg|jpeg|webp|gif|avif) kind=image ;;
|
|
mp3|m4a|aac|opus|oga|ogg|flac|wav|wma|aiff) kind=audio ;;
|
|
esac
|
|
|
|
poster=""
|
|
fallback=""
|
|
fallback_suffix=""
|
|
if [ "$RAW" = 1 ] || [ "$kind" = other ]; then
|
|
payload="$src"
|
|
out_ext="$ext"
|
|
elif [ "$kind" = video ]; then
|
|
payload="$WORK/out.mp4"
|
|
out_ext=mp4
|
|
echo " transcoding to AV1 (crf $CRF)…"
|
|
ffmpeg -nostdin -v error -i "$src" \
|
|
-vf "hqdn3d=4:3:6:4.5" \
|
|
-c:v libsvtav1 -preset 4 -crf "$CRF" -pix_fmt yuv420p -g 240 \
|
|
-movflags +faststart \
|
|
-c:a aac -b:a 96k -ac 1 \
|
|
"$payload" -y
|
|
# AAC rather than Opus on purpose: Opus-in-MP4 is still uneven in exactly
|
|
# the players most likely to be shaky about AV1 anyway, and the audio is
|
|
# a rounding error next to the video either way.
|
|
fallback="$WORK/h264.mp4"
|
|
fallback_suffix=h264.mp4
|
|
echo " transcoding H.264 fallback (crf 23)…"
|
|
# Same source and same denoise as the AV1, so the two encodings show the
|
|
# same frames. CRF is x264's own scale (not comparable to SVT-AV1's);
|
|
# 23 is its "looks the same" default, and this file is the compatibility
|
|
# rail, not the one most visitors download — bigger is acceptable here.
|
|
ffmpeg -nostdin -v error -i "$src" \
|
|
-vf "hqdn3d=4:3:6:4.5" \
|
|
-c:v libx264 -preset slow -crf 23 -pix_fmt yuv420p \
|
|
-movflags +faststart \
|
|
-c:a aac -b:a 96k -ac 1 \
|
|
"$fallback" -y
|
|
poster="$WORK/poster.webp"
|
|
# A frame a third of the way in, which beats frame 0 — recordings tend to
|
|
# open on a lock screen, a hand moving into place, or a fade.
|
|
dur=$(ffprobe -v error -show_entries format=duration \
|
|
-of default=nw=1:nk=1 "$payload" 2>/dev/null || echo 0)
|
|
at=$(awk -v d="$dur" 'BEGIN { printf "%.2f", (d > 3 ? d / 3 : 0) }')
|
|
ffmpeg -nostdin -v error -ss "$at" -i "$payload" -frames:v 1 \
|
|
-vf "scale=720:-2" -c:v libwebp -quality 80 "$poster" -y
|
|
elif [ "$kind" = audio ]; then
|
|
payload="$WORK/out.webm"
|
|
out_ext=webm
|
|
fallback_suffix=mp3
|
|
# Channel count sets the bitrate instead of being flattened by it: 48k of
|
|
# mono Opus is comfortable for speech, and stereo music at that figure
|
|
# would not be. These are VBR targets, not ceilings, and libopus does not
|
|
# track them evenly — on the first file through here the 48k target came
|
|
# out at 45 kbps while 64k ran away to 76, which is why the mono rail sits
|
|
# at the lower figure rather than splitting the difference.
|
|
ch=$(ffprobe -v error -select_streams a:0 -show_entries stream=channels \
|
|
-of default=nw=1:nk=1 "$src" </dev/null 2>/dev/null \
|
|
| head -n1 || true)
|
|
case "$ch" in ''|*[!0-9]*) ch=1 ;; esac
|
|
if [ "$ch" -gt 1 ]; then ab=96k; else ab=48k; fi
|
|
echo " transcoding to Opus ($ab, ${ch}ch)…"
|
|
# -vn drops cover art, which arrives as a still video stream and belongs
|
|
# in neither output — without it an MP3 carrying album art fails the
|
|
# encode outright. -map_metadata -1 drops the ID3 tags, which on a file
|
|
# from someone else carry their tooling and their file naming. 48 kHz is
|
|
# the rate Opus works at internally, so saying it makes the resample of
|
|
# the usual 44.1 kHz source explicit rather than implied.
|
|
ffmpeg -nostdin -v error -i "$src" -vn -map_metadata -1 \
|
|
-c:a libopus -b:a "$ab" -ar 48000 \
|
|
"$payload" -y
|
|
fallback="$WORK/fallback.mp3"
|
|
if [ "$ext" = mp3 ]; then
|
|
echo " MP3 fallback: source is already MP3, copied unchanged"
|
|
cp "$src" "$fallback"
|
|
else
|
|
echo " transcoding MP3 fallback (128k)…"
|
|
# Fixed 128k, decodable by anything, and the same "bigger is
|
|
# acceptable on the compatibility rail" trade the H.264 above makes.
|
|
ffmpeg -nostdin -v error -i "$src" -vn -map_metadata -1 \
|
|
-c:a libmp3lame -b:a 128k "$fallback" -y
|
|
fi
|
|
else
|
|
payload="$WORK/out.webp"
|
|
out_ext=webp
|
|
echo " converting to webp…"
|
|
ffmpeg -nostdin -v error -i "$src" -c:v libwebp -quality 82 "$payload" -y
|
|
fi
|
|
|
|
hash=$(sha256sum "$payload" | cut -c1-16)
|
|
name="$hash.$out_ext"
|
|
publish_file "$payload" "$name"
|
|
|
|
# The poster and the H.264 fallback are named after the AV1's hash, not
|
|
# their own. That is the whole point: fetch-media.sh finds both by name,
|
|
# with nothing to look up — the poster when the instance made no thumbnail
|
|
# (pict-rs will not read AV1, so for these uploads that is the normal
|
|
# case), the fallback for every browser that cannot decode AV1. Audio's MP3
|
|
# sibling is named the same way, after the Opus file's hash.
|
|
if [ -n "$poster" ] && [ -f "$poster" ]; then
|
|
publish_file "$poster" "$hash.poster.webp"
|
|
fi
|
|
post_name="$name"
|
|
if [ -n "$fallback" ] && [ -f "$fallback" ]; then
|
|
post_name="$hash.$fallback_suffix"
|
|
publish_file "$fallback" "$post_name"
|
|
fi
|
|
|
|
# The postable URL is the compatibility rail when there is one: the H.264
|
|
# video or the MP3 audio — see FALLBACK and AUDIO above.
|
|
url="$SITE_ORIGIN/media/$post_name"
|
|
# Confirm it is actually being served before handing over a URL that is about
|
|
# to be pasted into a post, where a 404 is public and permanent.
|
|
code=$(curl -s -o /dev/null -w '%{http_code}' --max-time 30 "$url" || echo 000)
|
|
if [ "$code" = 200 ]; then
|
|
printf ' post this: %s\n' "$url"
|
|
else
|
|
echo " WARNING: $url answered HTTP $code — do not post this link yet" >&2
|
|
fi
|
|
printf ' %s -> %s (%s)\n' \
|
|
"$(du -h "$src" | cut -f1)" "$(du -h "$payload" | cut -f1)" "$name"
|
|
[ -z "$fallback" ] || printf ' fallback: %s (%s)\n' \
|
|
"$(du -h "$fallback" | cut -f1)" "$post_name"
|
|
done
|