catcrafts.net/deploy/catcrafts-analytics

#!/bin/sh
# Rebuilds the analytics reports from Caddy's access logs, in two tiers:
#
#   /var/www/analytics/index.html          PUBLIC - censored. IPs are
#       anonymized at ingest, there is no HOSTS or full-URL REFERRERS panel,
#       and log lines matching $CENSOR_RE never reach its DB at all: the
#       public tier cannot leak what it never ingested.
#   /var/www/analytics-private/index.html  PRIVATE (basic auth in Caddy) -
#       uncensored: full IPs, all panels.
#
# Each tier has its own persistent DB and ingest ledger: rotated logs are
# ingested exactly once per tier; the live file is layered on at report time
# WITHOUT --persist, so its lines never double-count when Caddy rolls it.
#
# Filters live at INGEST: the DBs store aggregated, filtered data, so a
# filter change only applies to new lines. To re-filter history:
#   rm -rf /var/lib/goaccess/db-* /var/lib/goaccess/ingested-*
#   systemctl start catcrafts-analytics   # rebuilds from retained raw logs
#
# Runs hourly as the caddy user (owner of the 0600 logs) via
# catcrafts-analytics.timer.
set -eu

LOG_DIR=/var/log/caddy
LIVE=$LOG_DIR/catcrafts.net.log
STATE_DIR=/var/lib/goaccess
OUT_PUBLIC=/var/www/analytics/index.html
OUT_PRIVATE=/var/www/analytics-private/index.html

# Log lines whose URI matches this never enter the public tier. Extend it
# when the shop launches so order/payment URLs can never surface publicly.
CENSOR_RE='"uri":"/api'

# Serialize runs: a manual run racing the hourly timer once ingested the same
# rotated log twice (both processes passed the ledger check before either
# appended). Skip quietly if another run holds the lock - the timer comes
# around hourly anyway.
exec 9>$STATE_DIR/.lock
flock -n 9 || exit 0

# goaccess refuses a missing --db-path; recreate after a re-filter wipe.
mkdir -p "$STATE_DIR/db-private" "$STATE_DIR/db-public"

# Own IPs to keep out of the numbers (data quality, both tiers). The IPs
# live in /etc/goaccess/exclude-ips (one per line, # comments allowed) - ON
# THE HOST ONLY, never in this public repo.
exclude_args() {
    [ -r /etc/goaccess/exclude-ips ] || return 0
    while IFS= read -r ip; do
        case "$ip" in ''|'#'*) continue ;; esac
        printf -- '--exclude-ip=%s\n' "$ip"
    done < /etc/goaccess/exclude-ips
}

# Shared quality filters, both tiers:
# --ignore-crawlers drops self-declared bots (ClaudeBot alone was 18% of
# traffic); --unknowns-as-crawlers additionally drops the headerless
# vulnerability scanners (57% of traffic, no User-Agent at all); the
# browsers-file catches known fake-browser scrapers.
# --ignore-referrer drops referrer values that can never carry real source
# information: '*catcrafts.*' covers self-referrals AND the parked redirect
# TLDs (a 301 never sets the redirecting host as referrer, so those are
# always crawlers stamping their pre-redirect URL); '*localhost*' is Android
# fediverse apps whose WebView UI lives on an embedded localhost page.
# $(exclude_args) is unquoted on purpose: one word per --exclude-ip flag.
common() {
    goaccess "$@" \
        --log-format=CADDY \
        --ignore-crawlers \
        --unknowns-as-crawlers \
        --browsers-file=/etc/goaccess/browsers.list \
        $(exclude_args) \
        --ignore-referrer='*catcrafts.*' \
        --ignore-referrer='*localhost*' \
        --no-query-string \
        --tz=Europe/Amsterdam
}

run_private() {
    common "$@" --db-path="$STATE_DIR/db-private"
}

# Public extras: --anonymize-ip zeroes the last octet before anything is
# stored; visitor IPs are personal data and stay out of the public page.
run_public() {
    common "$@" --db-path="$STATE_DIR/db-public" \
        --anonymize-ip \
        --ignore-panel=HOSTS \
        --ignore-panel=REFERRERS
}

# grep -v exits 1 when it outputs nothing - not an error here.
censor() {
    grep -vE "$CENSOR_RE" || true
}

ingest() { # $1 = tier
    ledger=$STATE_DIR/ingested-$1
    touch "$ledger"
    for f in "$LOG_DIR"/catcrafts.net-*.log.gz; do
        [ -e "$f" ] || continue
        grep -qxF "$f" "$ledger" && continue
        case "$1" in
        private) zcat "$f" | run_private - --persist --restore --process-and-exit ;;
        public) zcat "$f" | censor | run_public - --persist --restore --process-and-exit ;;
        esac
        printf '%s\n' "$f" >>"$ledger"
    done
}

ingest private
ingest public

run_private "$LIVE" --restore \
    --html-report-title="catcrafts.net (private)" \
    -o "$OUT_PRIVATE"
censor <"$LIVE" | run_public - --restore \
    --html-report-title="catcrafts.net" \
    -o "$OUT_PUBLIC"