diff --git a/deploy/Caddyfile.example b/deploy/Caddyfile.example index ba78869..8e6c7b7 100644 --- a/deploy/Caddyfile.example +++ b/deploy/Caddyfile.example @@ -43,6 +43,37 @@ catcrafts.net { header /styles.css Cross-Origin-Resource-Policy "same-origin" header /favicon.svg Cross-Origin-Resource-Policy "same-origin" + # ── analytics, two tiers ────────────────────────────────────────────── + # + # GoAccess reports rebuilt hourly by catcrafts-analytics.timer from the + # access logs below. Server-side only: no client-side analytics anywhere + # on the site, per the privacy policy (request logging only). + # + # /analytics/ is public and censored - visitor IPs are anonymized at + # ingest, there is no host panel, and sensitive paths never enter its DB + # (see CENSOR_RE in deploy/catcrafts-analytics). /analytics/private/ is + # the uncensored report. Generate the hash with `caddy hash-password`. + redir /analytics /analytics/ 308 + redir /analytics/private /analytics/private/ 308 + + # The privacy policy's "view previous versions" promise. A redirect so + # the policy text stays short and survives repo or path moves - update + # the target here, never the policy wording. + redir /legal/privacy/history https://forgejo.catcrafts.net/Catcrafts/catcrafts.net/commits/branch/master/shared/interfaces/Catcrafts.Shared-Content.cppm 302 + handle_path /analytics/private/* { + basic_auth { + jorijn + } + root * /var/www/analytics-private + header Cache-Control "private, no-store" + file_server + } + handle_path /analytics/* { + root * /var/www/analytics + header Cache-Control "public, max-age=600" + file_server + } + # ── build artifacts: served from disk ───────────────────────────────── # # file_server does sendfile, precompressed variants and range requests far @@ -98,4 +129,18 @@ catcrafts.net { header Cache-Control "no-store" file_server } + + # ── access log ──────────────────────────────────────────────────────── + # + # Feeds the GoAccess analytics (see deploy/catcrafts-analytics). Retention + # sized for that: ~15 MB/day means a roll roughly weekly; 52 compressed + # rolls ~ a year of raw logs (~6 MB each gzipped). roll_keep_for must be + # explicit — lumberjack's default silently deletes after 90 days. + log { + output file /var/log/caddy/catcrafts.net.log { + roll_size 100MiB + roll_keep 52 + roll_keep_for 8760h + } + } } diff --git a/deploy/README.md b/deploy/README.md index c5117ee..878ec8e 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -343,6 +343,60 @@ curl -s https://catcrafts.net/projects | grep -c '[^<]*' ``` +## Analytics + +Server-side only — the privacy policy promises request logging and nothing +else, so there is no client-side analytics anywhere on the site. GoAccess +(Debian package) turns Caddy's JSON access logs into two static HTML +reports, each with its own persistent DB and ingest ledger: + +- `https://catcrafts.net/analytics/` — **public, censored.** Visitor IPs are + anonymized at ingest (last octet zeroed before anything reaches its DB), + no HOSTS or full-URL REFERRERS panels, and log lines matching `CENSOR_RE` + in the script never enter its DB at all — the public tier cannot leak + what it never ingested. Extend `CENSOR_RE` when the shop launches so + order/payment URLs can never surface; keep secrets out of URL *paths* + regardless (query strings are already stripped). +- `https://catcrafts.net/analytics/private/` — **uncensored** (basic auth, + hash in the Caddyfile): full IPs, all panels. + +Raw logs keep full IPs either way — that is the request logging the privacy +policy declares; per-IP forensics work from the logs and the private tier, +never from the public page. + +Three pieces, all in `deploy/`: + +- `catcrafts-analytics` → `/usr/local/bin/` — ingests each rotated + `catcrafts.net-*.log.gz` exactly once into a persistent GoAccess DB + (`/var/lib/goaccess/db`, tracked in `/var/lib/goaccess/ingested`), then + renders the report from DB + live log. The live file is never persisted, + so its lines don't double-count when Caddy rotates it. History therefore + survives log deletion: the DB keeps aggregates forever. +- `catcrafts-analytics.service` — oneshot, runs as `caddy` (owner of the + 0600 logs). +- `catcrafts-analytics.timer` — hourly at :07. + +Bot filtering is the load-bearing part: measured on real traffic, 57% of +requests were headerless vulnerability scanners and another 19% self-declared +bots (mostly ClaudeBot) — only ~24% human. `--ignore-crawlers +--unknowns-as-crawlers` drops both groups. The flags in the script apply at +ingest time and the DB stores aggregated data, so changing filters later only +affects new lines — re-ingesting history means deleting +`/var/lib/goaccess/{db,ingested}` and letting the next run rebuild from +whatever raw logs retention still holds (a year, per the Caddyfile). + +```sh +apt install goaccess +install -m 755 deploy/catcrafts-analytics /usr/local/bin/ +install -m 644 deploy/catcrafts-analytics.{service,timer} /etc/systemd/system/ +mkdir -p /etc/goaccess /var/lib/goaccess /var/www/analytics /var/www/analytics-private +install -m 644 deploy/goaccess-browsers.list /etc/goaccess/browsers.list +# own IPs to keep out of the numbers - host-only file, NOT in this repo +echo "203.0.113.7" > /etc/goaccess/exclude-ips +chown -R caddy:caddy /var/lib/goaccess /var/www/analytics /var/www/analytics-private +systemctl daemon-reload && systemctl enable --now catcrafts-analytics.timer +``` + ## Running it locally ```sh diff --git a/deploy/catcrafts-analytics b/deploy/catcrafts-analytics new file mode 100644 index 0000000..ba97214 --- /dev/null +++ b/deploy/catcrafts-analytics @@ -0,0 +1,119 @@ +#!/bin/sh +# Rebuilds the analytics reports from Caddy's access logs, in two tiers: +# +# /var/www/analytics/index.html PUBLIC - censored. IPs are +# anonymized at ingest, there is no HOSTS or full-URL REFERRERS panel, +# and log lines matching $CENSOR_RE never reach its DB at all: the +# public tier cannot leak what it never ingested. +# /var/www/analytics-private/index.html PRIVATE (basic auth in Caddy) - +# uncensored: full IPs, all panels. +# +# Each tier has its own persistent DB and ingest ledger: rotated logs are +# ingested exactly once per tier; the live file is layered on at report time +# WITHOUT --persist, so its lines never double-count when Caddy rolls it. +# +# Filters live at INGEST: the DBs store aggregated, filtered data, so a +# filter change only applies to new lines. To re-filter history: +# rm -rf /var/lib/goaccess/db-* /var/lib/goaccess/ingested-* +# systemctl start catcrafts-analytics # rebuilds from retained raw logs +# +# Runs hourly as the caddy user (owner of the 0600 logs) via +# catcrafts-analytics.timer. +set -eu + +LOG_DIR=/var/log/caddy +LIVE=$LOG_DIR/catcrafts.net.log +STATE_DIR=/var/lib/goaccess +OUT_PUBLIC=/var/www/analytics/index.html +OUT_PRIVATE=/var/www/analytics-private/index.html + +# Log lines whose URI matches this never enter the public tier. Extend it +# when the shop launches so order/payment URLs can never surface publicly. +CENSOR_RE='"uri":"/api' + +# Serialize runs: a manual run racing the hourly timer once ingested the same +# rotated log twice (both processes passed the ledger check before either +# appended). Skip quietly if another run holds the lock - the timer comes +# around hourly anyway. +exec 9>$STATE_DIR/.lock +flock -n 9 || exit 0 + +# goaccess refuses a missing --db-path; recreate after a re-filter wipe. +mkdir -p "$STATE_DIR/db-private" "$STATE_DIR/db-public" + +# Own IPs to keep out of the numbers (data quality, both tiers). The IPs +# live in /etc/goaccess/exclude-ips (one per line, # comments allowed) - ON +# THE HOST ONLY, never in this public repo. +exclude_args() { + [ -r /etc/goaccess/exclude-ips ] || return 0 + while IFS= read -r ip; do + case "$ip" in ''|'#'*) continue ;; esac + printf -- '--exclude-ip=%s\n' "$ip" + done < /etc/goaccess/exclude-ips +} + +# Shared quality filters, both tiers: +# --ignore-crawlers drops self-declared bots (ClaudeBot alone was 18% of +# traffic); --unknowns-as-crawlers additionally drops the headerless +# vulnerability scanners (57% of traffic, no User-Agent at all); the +# browsers-file catches known fake-browser scrapers. +# --ignore-referrer drops referrer values that can never carry real source +# information: '*catcrafts.*' covers self-referrals AND the parked redirect +# TLDs (a 301 never sets the redirecting host as referrer, so those are +# always crawlers stamping their pre-redirect URL); '*localhost*' is Android +# fediverse apps whose WebView UI lives on an embedded localhost page. +# $(exclude_args) is unquoted on purpose: one word per --exclude-ip flag. +common() { + goaccess "$@" \ + --log-format=CADDY \ + --ignore-crawlers \ + --unknowns-as-crawlers \ + --browsers-file=/etc/goaccess/browsers.list \ + $(exclude_args) \ + --ignore-referrer='*catcrafts.*' \ + --ignore-referrer='*localhost*' \ + --no-query-string \ + --tz=Europe/Amsterdam +} + +run_private() { + common "$@" --db-path="$STATE_DIR/db-private" +} + +# Public extras: --anonymize-ip zeroes the last octet before anything is +# stored; visitor IPs are personal data and stay out of the public page. +run_public() { + common "$@" --db-path="$STATE_DIR/db-public" \ + --anonymize-ip \ + --ignore-panel=HOSTS \ + --ignore-panel=REFERRERS +} + +# grep -v exits 1 when it outputs nothing - not an error here. +censor() { + grep -vE "$CENSOR_RE" || true +} + +ingest() { # $1 = tier + ledger=$STATE_DIR/ingested-$1 + touch "$ledger" + for f in "$LOG_DIR"/catcrafts.net-*.log.gz; do + [ -e "$f" ] || continue + grep -qxF "$f" "$ledger" && continue + case "$1" in + private) zcat "$f" | run_private - --persist --restore --process-and-exit ;; + public) zcat "$f" | censor | run_public - --persist --restore --process-and-exit ;; + esac + printf '%s\n' "$f" >>"$ledger" + done +} + +ingest private +ingest public + +run_private "$LIVE" --restore \ + --html-report-title="catcrafts.net (private)" \ + -o "$OUT_PRIVATE" +censor <"$LIVE" | run_public - --restore \ + --html-report-title="catcrafts.net" \ + -o "$OUT_PUBLIC" diff --git a/deploy/catcrafts-analytics.service b/deploy/catcrafts-analytics.service new file mode 100644 index 0000000..911710e --- /dev/null +++ b/deploy/catcrafts-analytics.service @@ -0,0 +1,11 @@ +[Unit] +Description=Rebuild GoAccess analytics report for catcrafts.net +Documentation=https://goaccess.io/man + +[Service] +Type=oneshot +# caddy owns the 0600 access logs; DB and output dirs are chowned to match. +User=caddy +Group=caddy +ExecStart=/usr/local/bin/catcrafts-analytics +Nice=10 diff --git a/deploy/catcrafts-analytics.timer b/deploy/catcrafts-analytics.timer new file mode 100644 index 0000000..04e658c --- /dev/null +++ b/deploy/catcrafts-analytics.timer @@ -0,0 +1,10 @@ +[Unit] +Description=Hourly GoAccess analytics report rebuild + +[Timer] +OnCalendar=*-*-* *:07:00 +RandomizedDelaySec=3m +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/deploy/goaccess-browsers.list b/deploy/goaccess-browsers.list new file mode 100644 index 0000000..3a210de --- /dev/null +++ b/deploy/goaccess-browsers.list @@ -0,0 +1,12 @@ +# Extra crawler classifications for GoAccess (--browsers-file), TAB-separated. +# Installed at /etc/goaccess/browsers.list; extends the builtin list. +# +# Frozen-UA scraper rotating through ~60 IPs: fakes Referer as reddit.com / +# localhost / none, never sends Sec-Fetch-* or Range, re-downloads media +# whole, probes /test.php. The UA itself is the proof of fakery: real Chrome +# mobile has sent a reduced UA ("Linux; Android 10; K", no device model) +# since ~2023, so bare Chrome claiming "Android 14; Pixel 8" cannot be real. +# Keep patterns EXACT: Voyager/Summit Lemmy apps on genuine Pixel 8s appear +# as "Pixel 8 Build/...; wv)" WebView UAs and must keep counting as humans. +Android 14; Pixel 8) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Crawlers +Android 14; Pixel 8) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Crawlers