Jorijn van der Graaf
8cd4d55d00
Server-side only, per the privacy policy: no client-side analytics. Rotated logs ingest exactly once into per-tier DBs; the public tier is anonymized at ingest and never receives censored paths. Log retention raised to a year. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
119 lines
4.5 KiB
Shell
119 lines
4.5 KiB
Shell
#!/bin/sh
|
|
# Rebuilds the analytics reports from Caddy's access logs, in two tiers:
|
|
#
|
|
# /var/www/analytics/index.html PUBLIC - censored. IPs are
|
|
# anonymized at ingest, there is no HOSTS or full-URL REFERRERS panel,
|
|
# and log lines matching $CENSOR_RE never reach its DB at all: the
|
|
# public tier cannot leak what it never ingested.
|
|
# /var/www/analytics-private/index.html PRIVATE (basic auth in Caddy) -
|
|
# uncensored: full IPs, all panels.
|
|
#
|
|
# Each tier has its own persistent DB and ingest ledger: rotated logs are
|
|
# ingested exactly once per tier; the live file is layered on at report time
|
|
# WITHOUT --persist, so its lines never double-count when Caddy rolls it.
|
|
#
|
|
# Filters live at INGEST: the DBs store aggregated, filtered data, so a
|
|
# filter change only applies to new lines. To re-filter history:
|
|
# rm -rf /var/lib/goaccess/db-* /var/lib/goaccess/ingested-*
|
|
# systemctl start catcrafts-analytics # rebuilds from retained raw logs
|
|
#
|
|
# Runs hourly as the caddy user (owner of the 0600 logs) via
|
|
# catcrafts-analytics.timer.
|
|
set -eu
|
|
|
|
LOG_DIR=/var/log/caddy
|
|
LIVE=$LOG_DIR/catcrafts.net.log
|
|
STATE_DIR=/var/lib/goaccess
|
|
OUT_PUBLIC=/var/www/analytics/index.html
|
|
OUT_PRIVATE=/var/www/analytics-private/index.html
|
|
|
|
# Log lines whose URI matches this never enter the public tier. Extend it
|
|
# when the shop launches so order/payment URLs can never surface publicly.
|
|
CENSOR_RE='"uri":"/api'
|
|
|
|
# Serialize runs: a manual run racing the hourly timer once ingested the same
|
|
# rotated log twice (both processes passed the ledger check before either
|
|
# appended). Skip quietly if another run holds the lock - the timer comes
|
|
# around hourly anyway.
|
|
exec 9>$STATE_DIR/.lock
|
|
flock -n 9 || exit 0
|
|
|
|
# goaccess refuses a missing --db-path; recreate after a re-filter wipe.
|
|
mkdir -p "$STATE_DIR/db-private" "$STATE_DIR/db-public"
|
|
|
|
# Own IPs to keep out of the numbers (data quality, both tiers). The IPs
|
|
# live in /etc/goaccess/exclude-ips (one per line, # comments allowed) - ON
|
|
# THE HOST ONLY, never in this public repo.
|
|
exclude_args() {
|
|
[ -r /etc/goaccess/exclude-ips ] || return 0
|
|
while IFS= read -r ip; do
|
|
case "$ip" in ''|'#'*) continue ;; esac
|
|
printf -- '--exclude-ip=%s\n' "$ip"
|
|
done < /etc/goaccess/exclude-ips
|
|
}
|
|
|
|
# Shared quality filters, both tiers:
|
|
# --ignore-crawlers drops self-declared bots (ClaudeBot alone was 18% of
|
|
# traffic); --unknowns-as-crawlers additionally drops the headerless
|
|
# vulnerability scanners (57% of traffic, no User-Agent at all); the
|
|
# browsers-file catches known fake-browser scrapers.
|
|
# --ignore-referrer drops referrer values that can never carry real source
|
|
# information: '*catcrafts.*' covers self-referrals AND the parked redirect
|
|
# TLDs (a 301 never sets the redirecting host as referrer, so those are
|
|
# always crawlers stamping their pre-redirect URL); '*localhost*' is Android
|
|
# fediverse apps whose WebView UI lives on an embedded localhost page.
|
|
# $(exclude_args) is unquoted on purpose: one word per --exclude-ip flag.
|
|
common() {
|
|
goaccess "$@" \
|
|
--log-format=CADDY \
|
|
--ignore-crawlers \
|
|
--unknowns-as-crawlers \
|
|
--browsers-file=/etc/goaccess/browsers.list \
|
|
$(exclude_args) \
|
|
--ignore-referrer='*catcrafts.*' \
|
|
--ignore-referrer='*localhost*' \
|
|
--no-query-string \
|
|
--tz=Europe/Amsterdam
|
|
}
|
|
|
|
run_private() {
|
|
common "$@" --db-path="$STATE_DIR/db-private"
|
|
}
|
|
|
|
# Public extras: --anonymize-ip zeroes the last octet before anything is
|
|
# stored; visitor IPs are personal data and stay out of the public page.
|
|
run_public() {
|
|
common "$@" --db-path="$STATE_DIR/db-public" \
|
|
--anonymize-ip \
|
|
--ignore-panel=HOSTS \
|
|
--ignore-panel=REFERRERS
|
|
}
|
|
|
|
# grep -v exits 1 when it outputs nothing - not an error here.
|
|
censor() {
|
|
grep -vE "$CENSOR_RE" || true
|
|
}
|
|
|
|
ingest() { # $1 = tier
|
|
ledger=$STATE_DIR/ingested-$1
|
|
touch "$ledger"
|
|
for f in "$LOG_DIR"/catcrafts.net-*.log.gz; do
|
|
[ -e "$f" ] || continue
|
|
grep -qxF "$f" "$ledger" && continue
|
|
case "$1" in
|
|
private) zcat "$f" | run_private - --persist --restore --process-and-exit ;;
|
|
public) zcat "$f" | censor | run_public - --persist --restore --process-and-exit ;;
|
|
esac
|
|
printf '%s\n' "$f" >>"$ledger"
|
|
done
|
|
}
|
|
|
|
ingest private
|
|
ingest public
|
|
|
|
run_private "$LIVE" --restore \
|
|
--html-report-title="catcrafts.net (private)" \
|
|
-o "$OUT_PRIVATE"
|
|
censor <"$LIVE" | run_public - --restore \
|
|
--html-report-title="catcrafts.net" \
|
|
-o "$OUT_PUBLIC"
|