From a3f3bf88725129bd0eed6d11100bd08fea769789 Mon Sep 17 00:00:00 2001 From: Marc Froehlich Date: Mon, 14 Sep 2026 20:33:50 +0200 Subject: [PATCH] Changes at the stats analytics --- docs/PROJECT_CONTEXT.md | 27 +- website/ops/analytics/README.md | 320 ++++++-- website/ops/analytics/generate-reports.js | 151 +++- .../hamradioonline-analytics.example | 2 +- .../nginx/analytics-filters.conf.example | 7 + .../nginx/analytics-log.conf.example | 5 +- website/ops/analytics/private-metrics.js | 769 ++++++++++++++++++ website/ops/analytics/sites.example.json | 12 + .../hamradioonline-analytics.service.example | 2 +- website/src/privacy/index.njk | 33 +- website/test/analytics-generator.test.js | 259 +++++- website/test/private-metrics.test.js | 260 ++++++ 12 files changed, 1740 insertions(+), 107 deletions(-) create mode 100644 website/ops/analytics/private-metrics.js create mode 100644 website/test/private-metrics.test.js diff --git a/docs/PROJECT_CONTEXT.md b/docs/PROJECT_CONTEXT.md index ae45fc6d..63fe1cce 100644 --- a/docs/PROJECT_CONTEXT.md +++ b/docs/PROJECT_CONTEXT.md @@ -1,6 +1,6 @@ # KST4Contest Project Context -Last reviewed: 2026-09-13 +Last reviewed: 2026-09-14 This file is the durable technical project context for KST4Contest. It is not a user manual and not a replacement for the changelog. Current code, tests and authoritative external specifications remain the source of truth when this document is stale or ambiguous. @@ -188,23 +188,28 @@ section records only the durable architecture and operational boundaries. #### Architecture and data flow - Production runs on Ubuntu Server 24.04 with Nginx 1.24, Node.js 18.19.1 and GoAccess 1.8.1. GoAccess has GeoIP2/MMDB and OpenSSL support but no Zlib support; missing Zlib is intentional for this operating model. -- Nginx writes eligible `GET` page requests to a reduced, tab-separated analytics log in addition to the normal operational log. It records server name, client IP, time, method, normalized path without query string, protocol, status, bytes and user agent; referrer and remote user are omitted. -- Nginx removes assets, downloads, status/monitoring paths, update feed, counter requests and known crawler clients before logging. GoAccess supplies the second crawler-classification layer, anonymises IP addresses at the configured level and performs Country lookup locally through GeoLite2-Country. -- The hourly systemd timer starts the Node.js generator. Each registered site is processed from its optional uncompressed `.1` rotation followed by the current log; older `.gz` files are not part of regular processing. Logrotate retains raw analytics logs for 14 days and requires `delaycompress` for this handover. -- GoAccess maintains one private report/database per registered site and one combined report/database. The generator then updates its separate daily counter state and publishes `visitor-count.json` for enabled sites. Private reports are served with Basic Auth from `stats.hamradioonline.de`; the public home page requests only its same-origin counter file. +- Nginx writes eligible `GET` page requests to a reduced, tab-separated website analytics log in addition to the normal operational log. It records server name, client IP, time, method, normalized path without query string, protocol, status, bytes and user agent; referrer and remote user are omitted. +- Nginx removes assets, downloads, status/monitoring paths, update feed, counter requests and known crawler clients from the website log. GoAccess supplies the second crawler-classification layer, anonymises IP addresses at the configured level and performs Country lookup locally through GeoLite2-Country. +- A second reduced Nginx log contains only case-sensitive `GET` requests with final status `200` for the exact path `/kst4ContestVersionInfo.xml`. It deliberately retains browser and bot requests and stays excluded from website page views, visits and the public visitor counter. +- The hourly systemd timer starts the Node.js generator. Each registered site and both reduced streams are processed from the optional uncompressed `.1` rotation followed by the current log; older `.gz` files are not part of regular processing. Logrotate retains raw logs for 14 days and requires `delaycompress` for this handover. +- GoAccess maintains one private report/database per registered site and one combined report/database. The generator then updates its separate daily visit-counter state, publishes `visitor-count.json` for enabled sites, and produces durable private website/update-information day and year views. Private reports are served with Basic Auth from `stats.hamradioonline.de`; the public home page requests only its same-origin counter file. #### Durable invariants and persistence - GoAccess defines a visit by IP address, date and user agent. The public value is therefore an approximate visit total, not a count of unique people, and remains separate from page views. - The public contract contains only `schemaVersion`, `visits`, `since` and `updatedAt`. The browser uses no analytics cookie, local storage or third-party request; failed or invalid responses leave the counter hidden. -- The site registry at `/etc/hamradioonline-analytics/sites.json` owns stable site IDs, hostnames, current uncompressed log paths, activation dates and output targets. `stats.hamradioonline.de` is never registered as an analytics site. Existing counter state cannot be continued with a changed hostname or `activatedOn` without an explicit migration or reset decision. -- GoAccess databases below `/var/lib/hamradioonline-analytics/db` retain rolling 395-day detail through `--persist` and `--restore`. `/var/lib/hamradioonline-analytics/public-counter-state.json` retains daily values from activation onward and is the durable business source for the lifetime public total. Reports and public JSON files are derived outputs. +- The site registry at `/etc/hamradioonline-analytics/sites.json` owns stable site IDs, hostnames, current uncompressed log paths, activation/coverage dates and output targets. `stats.hamradioonline.de` is never registered as an analytics site. Existing state cannot be continued with a changed hostname or activation/coverage identity without an explicit migration or reset decision. +- GoAccess databases below `/var/lib/hamradioonline-analytics/db` retain rolling 395-day detail through `--persist` and `--restore`. `/var/lib/hamradioonline-analytics/public-counter-state.json` retains daily visit values from activation onward and remains the durable business source for the lifetime public total. Its schema and public meaning are unchanged. +- `/var/lib/hamradioonline-analytics/private-metrics-state.json` permanently retains website page-view totals, absolute Country values and normalized path totals per day, plus update-information request totals and absolute Country values per day. It contains no per-request IP address, raw user agent or timestamp. Update hours and conservative client groups are retained only for the latest 14 `Europe/Berlin` calendar days. There is no permanent path-by-Country request matrix. +- Update-information requests are not program starts or user counts. Browsers and bots can fetch the XML, and existing application versions do not send a reliable KST4Contest-specific user agent; a Java user agent identifies only an unknown Java application. - GoAccess 1.8.1 exposes Country data as `geolocation`; combined jobs explicitly enable `VIRTUAL_HOSTS` and require `vhosts`. Individual site jobs do not enable that panel. `geo_location` and `virtual_hosts` are invalid interface names. -- Generator inputs and staged GoAccess outputs are validated before publication. Files are replaced atomically, dry-runs use temporary state without the production lock, and productive runs use `/run/hamradioonline-analytics/generator.lock`. Exit codes are 1 for runtime/publication errors, 2 for configuration errors and 3 when the production lock cannot be acquired. +- Private daily values are upserted rather than added. Regular hourly reprocessing and historical imports are repeatable; overlapping aggregates must be equal or monotonically more complete. The generator fails on inconsistent overlaps or when path sums differ from the existing GoAccess page-request definition. +- The explicit historical-import mode accepts verified Nginx combined or reduced TSV logs and inclusive coverage dates. Node.js reads `.gz` files without relying on Zlib support in GoAccess. Missing earlier history remains unknown rather than becoming zero, and the first covered date is stored and displayed. +- Generator inputs and staged GoAccess outputs are validated before publication. Files are replaced atomically, temporary normalized per-day inputs use private run directories and are removed after the run, dry-runs use temporary state without the production lock, and productive runs use `/run/hamradioonline-analytics/generator.lock`. Exit codes are 1 for runtime/publication errors, 2 for configuration errors and 3 when the production lock cannot be acquired. #### Components, roles and external services -- `hamradio-analytics` is the locked, non-login service account which reads configuration, Country MMDB and analytics logs and writes service state. Nginx runs as `www-data`, writes the analytics log and can read only reports and the public counter, not private databases or counter state. `stats-reader` is a local Nginx Basic Auth login whose password file remains outside Git. +- `hamradio-analytics` is the locked, non-login service account which reads configuration, Country MMDB and reduced logs and writes service state. Nginx runs as `www-data`, writes both reduced logs and can read only reports and the public counter, not private databases or either state file. `stats-reader` is a local Nginx Basic Auth login whose password file remains outside Git. - Root-managed program/configuration lives below `/opt/hamradioonline-analytics` and `/etc/hamradioonline-analytics`; service state lives below `/var/lib/hamradioonline-analytics`. Report/public directories are shared read-only with Nginx through group ownership; no ACL dependency exists. - MaxMind is used only by `geoipupdate` to download GeoLite2-Country. Visitor lookups are local; Account ID and License Key remain in the protected server configuration. Let's Encrypt supplies TLS for the statistics vhost through the permanent ACME webroot and `/snap/bin/certbot`. GitHub supplies website source and deployment, not analytics processing. - The website deploy checks out `origin/main`, builds the static site and publishes it to the Nginx document root. It does not install or overwrite analytics programs/configuration, systemd, Nginx, Logrotate, GeoIP, Basic Auth or Certbot state; those remain separate manual server operations. @@ -212,8 +217,8 @@ section records only the durable architecture and operational boundaries. #### Open operational work -- The server has no comprehensive automated backup plan yet. A future server-wide plan must include the non-regenerable counter state, GoAccess databases and protected operational configuration without extending the published 14-day raw-log retention. -- The first real rotation of the dedicated analytics log still requires an explicit operational check of `.1`, ownership/readability, subsequent generator success and absence of duplicate counting. This is not a current service blocker. +- The server has no comprehensive automated backup plan yet. A future server-wide plan must include both non-regenerable state files, GoAccess databases and protected operational configuration without extending the published 14-day raw-log retention. +- Repository implementation and tests do not install server configuration or import production data. The first real rotation after installing the XML log still requires an explicit operational check of both `.1` handovers, ownership/readability, subsequent generator success and absence of duplicate counting. The actual regular Nginx log format and complete historical coverage must be verified on the server before import. ## Important Decisions and Workarounds diff --git a/website/ops/analytics/README.md b/website/ops/analytics/README.md index 7555d8fa..a0099a74 100644 --- a/website/ops/analytics/README.md +++ b/website/ops/analytics/README.md @@ -5,10 +5,12 @@ runbook for the production analytics service on Ubuntu Server 24.04. Nothing in the repository installs, updates or activates the server-side components automatically. -The design has two separate outputs: +The design has three separate outputs: - private static GoAccess HTML and JSON reports for each registered project subdomain and for all registered project subdomains combined; +- private durable daily and annual views of website page views and successful + requests for the update-information XML file; - a small public `visitor-count.json` file for sites which explicitly enable the counter. @@ -37,6 +39,17 @@ durable daily counter state --> public visitor-count.json same-origin home-page request private HTML reports --> Nginx Basic Auth --> stats.hamradioonline.de + +exact GET + HTTP 200 for /kst4ContestVersionInfo.xml + | + v +separate Nginx update-information log (14 days) + | + v +durable daily totals and countries + +--> hourly and client-group detail (14 days) + +eligible page log --> durable daily page views, countries and paths ``` Nginx writes a dedicated, reduced log. For each site, the generator gives the @@ -54,6 +67,13 @@ being added again. This makes repeated runs idempotent. Values older than 395 days remain in the counter state and continue to contribute to the public total. +The private metric state is separate again. It retains website page-view +totals, absolute country values and individual normalized paths per day, plus +the equivalent totals and countries for update-information requests. Hourly +and recognisable client-group detail for update requests is removed after 14 +days. The generator derives annual totals from the durable daily values. It +does not build a permanent path-by-country table. + ## Production platform The confirmed production baseline is: @@ -64,7 +84,7 @@ The confirmed production baseline is: - GoAccess 1.8.1 with GeoIP2/MMDB and OpenSSL support, but without Zlib; - a local GeoLite2-Country database; - systemd for the oneshot generator and its hourly timer; -- Logrotate for the dedicated analytics log. +- Logrotate for the dedicated website and update-information logs. Missing Zlib support is intentional for this operating model. The generator processes the current uncompressed analytics log and the optional uncompressed @@ -74,18 +94,21 @@ processes the current uncompressed analytics log and the optional uncompressed - `generate-reports.js` validates configuration and state, runs GoAccess and publishes outputs atomically. +- `private-metrics.js` parses the reduced logs, maintains durable daily + aggregates and renders the protected daily and annual views. - `sites.example.json` is the registry template. - `goaccess.conf.template` is rendered per report with a private database path and the configured GeoIP2 Country database. - `nginx/` contains the reduced log format, request filters, public endpoint and protected report-vhost examples. - `systemd/` contains a hardened oneshot service and hourly timer. -- `logrotate/` retains 14 daily analytics-log rotations. +- `logrotate/` retains 14 daily rotations for both reduced logs. These repository files are templates and source files. Their productive counterparts are installed separately: - generator: `/opt/hamradioonline-analytics/generate-reports.js`; +- private metric module: `/opt/hamradioonline-analytics/private-metrics.js`; - registry: `/etc/hamradioonline-analytics/sites.json`; - GoAccess template: `/etc/hamradioonline-analytics/goaccess.conf.template`; @@ -104,7 +127,8 @@ counterparts are installed separately: examples. No npm package is required by the generator. GoAccess is the only external -program it starts. +program it starts. Historical `.gz` input is decompressed by Node.js and does +not require Zlib support in the installed GoAccess 1.8.1 binary. The production compatibility baseline is GoAccess 1.8.1 built with `--enable-geoip=mmdb` and `--with-openssl`, but without `--with-zlib`. Zlib is @@ -128,10 +152,10 @@ home directory, `/usr/sbin/nologin` as its shell and a locked password. It runs the generator and GoAccess, reads the reduced logs and configuration, and writes only below the configured service-state directories. -Nginx runs as `www-data`. It writes the dedicated analytics log and reads the -private reports and public counter. It must not be able to read the private -GoAccess databases or `public-counter-state.json`, and it has no write access -to generated output. +Nginx runs as `www-data`. It writes the dedicated website and update-information +logs and reads the private reports and public counter. It must not be able to +read the private GoAccess databases, `public-counter-state.json` or +`private-metrics-state.json`, and it has no write access to generated output. `stats-reader` is the current local Nginx Basic Auth username. It is not a Linux service account and not an account with an external analytics provider. @@ -161,6 +185,9 @@ sudo install -d -o root -g hamradio-analytics -m 0750 \ sudo install -o root -g hamradio-analytics -m 0750 \ website/ops/analytics/generate-reports.js \ /opt/hamradioonline-analytics/generate-reports.js +sudo install -o root -g hamradio-analytics -m 0640 \ + website/ops/analytics/private-metrics.js \ + /opt/hamradioonline-analytics/private-metrics.js sudo install -o root -g hamradio-analytics -m 0640 \ website/ops/analytics/goaccess.conf.template \ /etc/hamradioonline-analytics/goaccess.conf.template @@ -191,6 +218,7 @@ sudo install -d -o hamradio-analytics -g www-data -m 2750 \ /var/lib/hamradioonline-analytics/reports \ /var/lib/hamradioonline-analytics/reports/combined \ /var/lib/hamradioonline-analytics/reports/kst4contest \ + /var/lib/hamradioonline-analytics/reports/metrics \ /var/lib/hamradioonline-analytics/public \ /var/lib/hamradioonline-analytics/public/kst4contest ``` @@ -204,13 +232,16 @@ boundary after systemd has prepared the state directory. The productive ownership and mode boundaries are: - generator: `0750 root:hamradio-analytics`; +- private metric module: `0640 root:hamradio-analytics`; - registry and GoAccess configuration: `0640 root:hamradio-analytics`; -- report directories, including `reports/combined` and - `reports/kst4contest`: `2750 hamradio-analytics:www-data`; +- report directories, including `reports/combined`, `reports/kst4contest` and + `reports/metrics`: `2750 hamradio-analytics:www-data`; - report files: `0640 hamradio-analytics:www-data`; - private database files: `0640 hamradio-analytics:hamradio-analytics`; - `public-counter-state.json`: mode `0640`, owner and group `hamradio-analytics:hamradio-analytics`; +- `private-metrics-state.json`: mode `0640`, owner and group + `hamradio-analytics:hamradio-analytics`; - public output directories, including `public/kst4contest`: mode `2750`, owner and group `hamradio-analytics:www-data`; - public `visitor-count.json`: `0644 hamradio-analytics:www-data`; @@ -234,11 +265,17 @@ if [ ! -e /var/log/nginx/kst4contest-analytics.log ]; then sudo install -o www-data -g hamradio-analytics -m 0640 /dev/null \ /var/log/nginx/kst4contest-analytics.log fi +if [ ! -e /var/log/nginx/kst4contest-update-information.log ]; then + sudo install -o www-data -g hamradio-analytics -m 0640 /dev/null \ + /var/log/nginx/kst4contest-update-information.log +fi sudo stat -c '%U:%G %a %n' /var/log/nginx/kst4contest-analytics.log +sudo stat -c '%U:%G %a %n' \ + /var/log/nginx/kst4contest-update-information.log ``` The resulting log owner and mode must be -`www-data:hamradio-analytics 640`. Logrotate preserves that ownership. The +`www-data:hamradio-analytics 640` for both files. Logrotate preserves that ownership. The generator's configuration check fails clearly if required output directories are missing or if the executing user cannot read an analytics log or the Country database. @@ -252,33 +289,48 @@ server layout. Each site entry contains: - its exact `hostname`; - the current, uncompressed `analyticsLog` path; - the `activatedOn` date used by the public counter; +- `websiteMetricsSince`, the first day covered by the new live website + page-view aggregation; +- an `updateInfo` object containing the exact XML path, its separate current + log and the first day covered by live update-information aggregation; - a `publicCounter` switch; - the private `reportOutputDirectory`; - a `publicJsonPath` when the public counter is enabled. -The top-level `combined.reportOutputDirectory` receives the combined report. +The top-level `privateMetrics` object defines its private state and report +paths, fixes the time zone to `Europe/Berlin` and fixes detailed retention to +14 days. The top-level `combined.reportOutputDirectory` receives the combined report. Only registered sites are included. The generator rejects `stats.hamradioonline.de`, so the report host cannot accidentally become part of the project statistics. +The dates in `sites.example.json` document the repository snapshot; they are +not installation defaults. Before enabling the new logs, replace +`websiteMetricsSince` and `updateInfo.metricsSince` with the actual first live +coverage day. If collection started earlier than the current/`.1` handover, +import the covered rotations before the first regular run. Do not backdate a +field merely to obtain an earlier-looking report. + To add another project subdomain later, add one registry entry and one matching dedicated `access_log` line to its Nginx server block. Do not enable a public counter unless that site should publish one. -Treat `activatedOn` as persistent data. Once counting has started, changing it +Treat `activatedOn`, `websiteMetricsSince` and `updateInfo.metricsSince` as +persistent data. Once counting has started, changing one of these values would change the meaning of the total. The generator refuses to combine a new -activation date with existing counter state. +date with the corresponding existing state. The hostname is persistent identity as well. If an existing site state has a different `activatedOn` or hostname, do not delete the state to make the next run pass. Changing either value requires a deliberate migration or a specifically approved reset of the public count. -The generator derives the optional `.1` path from `analyticsLog`. It is valid -for `.1` not to exist before the first rotation. Do not enter a rotation or a -compressed `.gz` file in the registry. +The generator derives the optional `.1` paths from `analyticsLog` and +`updateInfo.analyticsLog`. It is valid for `.1` not to exist before the first +rotation. Do not enter a rotation or a compressed `.gz` file in the registry. -Adding another project subdomain also requires its own Nginx analytics log, +Adding another project subdomain also requires its own Nginx website and +update-information logs, the corresponding Logrotate ownership, a prepared report directory and, when enabled, a public-output directory and counter location. The combined report uses the logs of every registered project site. The statistics vhost remains @@ -295,9 +347,9 @@ The relevant production configuration files are: - `/etc/nginx/sites-available/stats.hamradioonline.de`. Install the log-format and filter maps from `nginx/` in the `http` context. -Then add a dedicated analytics `access_log` to every registered project server -block. Keep the existing operational access log unless its replacement has -been reviewed separately. If the operational log is inherited from the +Then add the dedicated website and update-information `access_log` directives +to every registered project server block. Keep the existing operational access +log unless its replacement has been reviewed separately. If the operational log is inherited from the `http` context, repeat its directive in the server block before adding the analytics log; an `access_log` at a lower level changes inheritance. @@ -333,9 +385,10 @@ The analytics format contains only: - transferred body size; - user agent. -The fields are tab-separated. The production log is -`/var/log/nginx/kst4contest-analytics.log`. Nginx writes it as `www-data`; the -`hamradio-analytics` group can read it. The confirmed owner and mode are +The fields are tab-separated. The production website log is +`/var/log/nginx/kst4contest-analytics.log`; the separate XML log is +`/var/log/nginx/kst4contest-update-information.log`. Nginx writes both as +`www-data`; the `hamradio-analytics` group can read them. The confirmed owner and mode are `www-data:hamradio-analytics 0640`. The analytics service receives read-only access and must never truncate or otherwise modify this log. @@ -344,7 +397,7 @@ analytics log. It also omits referrer and authenticated remote-user data. The user agent is retained because GoAccess needs it for crawler classification and its visit definition. -Only eligible `GET` requests can be logged. Assets, downloads, status and +Only eligible `GET` requests enter the website analytics log. Assets, downloads, status and monitoring paths, sitemap, robots file, favicons, the update feed and the public counter endpoint are excluded. Known bots, crawlers, monitoring clients, `wget` and `curl` are rejected before logging. GoAccess applies its @@ -352,6 +405,14 @@ own crawler list as a second layer and treats unknown browser or operating system combinations as crawlers. The public counter request therefore cannot count itself, and the statistics vhost has no analytics logging of its own. +The update-information map is deliberately independent of the website and bot +filters. It logs only an exact case-sensitive `GET` request for +`/kst4ContestVersionInfo.xml` when the final response status is `200`. `HEAD`, +`304`, redirects and error responses do not enter this log. Browsers and bots +are not excluded, because the metric describes successful file requests, not +program starts or people. The XML remains excluded from the website log, so it +does not change website page views, visits or `visitor-count.json`. + Review the monitoring-path list against the real server before activation. When a new health endpoint or asset family is added, update the filter first. @@ -363,8 +424,8 @@ sudo nginx -t ### Log rotation -`/etc/logrotate.d/hamradioonline-analytics` rotates the dedicated analytics -logs daily, retains 14 rotations and compresses older files. `delaycompress` +`/etc/logrotate.d/hamradioonline-analytics` rotates both dedicated logs daily, +retains 14 rotations and compresses older files. `delaycompress` is an operational requirement: it keeps the immediately preceding rotation as an uncompressed `.1` file for the next generator run. The `create 0640 www-data hamradio-analytics` directive preserves the write/read boundary. @@ -372,8 +433,8 @@ After rotation, `invoke-rc.d nginx rotate` makes Nginx reopen its logs. The generator processes, in this order: -1. the optional, uncompressed `.1` rotation; -2. the current analytics log. +1. each optional, uncompressed `.1` rotation; +2. each corresponding current log. A missing `.1` is normal, including before the first rotation. Older `.gz` files are retained according to Logrotate but are not imported by the regular @@ -419,20 +480,45 @@ succeeded. Logrotate must use `delaycompress`, as shown in the example. This leaves `.1` uncompressed for one rotation cycle. Older `.gz` files are not part of the -regular hourly run, and importing them is a separate maintenance task outside -this repository workflow. Do not add an unstable decompression pipeline to -the timer service. +regular hourly run. The explicit historical-import mode can read them through +Node.js; it never asks the GoAccess binary to decompress them. Do not add an +incremental decompression pipeline to the timer service. If the generator is unavailable for longer than the uncompressed rotation window, the regular run cannot recover entries found only in older `.gz` files. Preserve those files under the raw-log retention policy and plan any necessary historical import separately before resuming normal processing. +### Durable private aggregates + +For each covered day, the generator rebuilds the relevant slice from the +available reduced logs and replaces or monotonically extends the stored value. +It does not add a whole hourly result to the previous result. Website page +views use the same Nginx page/bot filters and the same GoAccess crawler +classification as the existing reports. The generator verifies that the sum +of all path values equals GoAccess's request total; a discrepancy stops the run +instead of silently establishing a second page-view definition. + +The website series stores page-view totals, absolute Country-panel values and +every normalized path per day. The update-information series stores successful +request totals and absolute Country-panel values per day. A missing Country +assignment is stored as `Unknown`; no city or exact location is inferred. +Hourly update-request totals and conservative client groups are kept only for +the latest 14 calendar days in `Europe/Berlin`. Raw user agents never enter the +durable metric state. A Java user agent is labelled as an unknown Java +application, not as KST4Contest. + +Annual totals, annual Country totals and annual page totals are calculated from +the daily state when the static reports are generated. The first covered day +is shown for every series. Earlier dates are unknown and are not emitted as +zero. There is deliberately no permanent path-by-Country-by-request table. + ### Visit and privacy boundary GoAccess treats requests with the same IP address, date and user agent as one visit. The public number is therefore an approximate visit total, not a count -of uniquely identified people. Page views remain a separate statistic. +of uniquely identified people. Page views remain a separate statistic and are +displayed separately from the durable daily visit values. IP addresses are processed with the configured GoAccess anonymisation level. Country resolution happens locally against GeoLite2-Country; City and host @@ -447,14 +533,15 @@ no visitor-level or daily detail. ## Persistence and publication -The installation has four distinct persistence layers. +The installation has five distinct persistence layers. ### Raw logs -The current analytics log and its rotations are short-lived input. The current -file and `.1` bridge requests across the most recent rotation. Logrotate limits -raw-log retention to the published 14-day policy; backups must not silently -extend that period. +The current website and update-information logs and their rotations are +short-lived input. Each current file and `.1` bridge requests across the most +recent rotation. These files contain individual IP addresses, timestamps and +raw user agents. Logrotate limits their retention to the published 14-day +policy; backups must not silently extend that period. ### GoAccess databases @@ -480,6 +567,22 @@ The stored hostname and `activatedOn` must continue to match the registry. Changing either value requires a planned migration or an approved reset, not an ad-hoc edit or deletion of the state file. +### Private metric state + +`/var/lib/hamradioonline-analytics/private-metrics-state.json` is the durable +source for daily website page views and update-information requests. It stores +daily totals and absolute Country values; website days also store normalized +path totals. Only update-information days within the latest 14-day window may +contain hourly and client-group aggregates. The file contains no individual +IP address, raw user agent or individual timestamp and remains unreadable by +Nginx. + +Daily values are upserted, not added. Re-reading the current log, `.1` or an +overlapping historical import therefore does not multiply a day. A conflicting +or decreasing overlap stops the run for investigation. Keep this file with +the public counter state and GoAccess databases in the later separate +backup/recovery plan. + ### Reports and public files The derived outputs are: @@ -488,6 +591,8 @@ The derived outputs are: - `/var/lib/hamradioonline-analytics/reports/kst4contest/report.json`; - `/var/lib/hamradioonline-analytics/reports/combined/report.html`; - `/var/lib/hamradioonline-analytics/reports/combined/report.json`; +- `/var/lib/hamradioonline-analytics/reports/metrics/report.html` and its + per-site daily/yearly pages; - `/var/lib/hamradioonline-analytics/public/kst4contest/visitor-count.json`. The generator prepares every GoAccess job in a run directory, validates the @@ -500,7 +605,7 @@ transaction across every report, database and counter file; after a storage or permission failure during publication, inspect the complete set and rerun the service after correcting the cause. -Counter state and GoAccess databases are the important persistent sources. +Counter state, private metric state and GoAccess databases are the important persistent sources. HTML/JSON reports and `visitor-count.json` are derived and can be rebuilt when their corresponding source state is available. @@ -512,10 +617,14 @@ the reviewed repository file: ```sh sudo stat -c '%U:%G %a %s %n' \ - /opt/hamradioonline-analytics/generate-reports.js + /opt/hamradioonline-analytics/generate-reports.js \ + /opt/hamradioonline-analytics/private-metrics.js sha256sum /srv/git/kst4contest/website/ops/analytics/generate-reports.js \ - /opt/hamradioonline-analytics/generate-reports.js + /opt/hamradioonline-analytics/generate-reports.js \ + /srv/git/kst4contest/website/ops/analytics/private-metrics.js \ + /opt/hamradioonline-analytics/private-metrics.js /usr/bin/node --check /opt/hamradioonline-analytics/generate-reports.js +/usr/bin/node --check /opt/hamradioonline-analytics/private-metrics.js ``` A zero-byte JavaScript file is syntactically valid and exits successfully @@ -563,6 +672,56 @@ history which is no longer present in the raw logs. Back up this state file: it is the durable source for public totals older than the detailed retention window. +### Historical import + +Historical import is a manual maintenance operation. Do not add import options +to the systemd unit. First make a protected working copy of the still available +regular Nginx access logs, including `.1` and `.gz`, and inventory their actual +first and last records. Use only a continuous date range which is genuinely +covered by the selected files. A missing earlier file is missing history, not a +zero day. + +The importer accepts either the standard Nginx combined format or the reduced +tab-separated analytics format. It rejects malformed lines, unreadable files, +unknown sites and invalid coverage ranges. Do not guess a production log +format: compare a redacted sample with the selected parser before the import. +For a standard combined-log import: + +```sh +sudo -u hamradio-analytics /usr/bin/node \ + /opt/hamradioonline-analytics/generate-reports.js \ + --registry /etc/hamradioonline-analytics/sites.json \ + --config-template /etc/hamradioonline-analytics/goaccess.conf.template \ + --import-site kst4contest \ + --import-format nginx-combined \ + --coverage-from YYYY-MM-DD \ + --coverage-through YYYY-MM-DD \ + --import-log /protected/import/access.log.3.gz \ + --import-log /protected/import/access.log.2.gz \ + --import-log /protected/import/access.log.1 +``` + +Use `analytics-tsv` only when the source is genuinely in the maintained +reduced format. Query strings are removed and paths normalized by the importer. +Website requests pass the same maintained page and known-bot filters and then +GoAccess's crawler classification. Update-information requests require exact +`GET`/`200` semantics and do not exclude bots. + +The selected input set is aggregated by complete day. Duplicate copies of the +same records across selected files are collapsed while repeated identical +records within one source remain counted. Each imported day replaces or +monotonically extends its stored aggregate; rerunning the same command is +idempotent. An overlap which changes distributions without a consistent higher +total fails instead of adding uncertain data. Run once against a protected copy +of the private state or with `--dry-run`, inspect the first covered dates and +totals, then run productively. Keep a pre-import backup until a second identical +run confirms stable totals. + +Node.js decompresses `.gz` inputs itself. The production GoAccess 1.8.1 binary +does not need Zlib support. Remove the protected import copies according to the +14-day raw-data limit after the verified import; do not put them in a durable +backup. + ### Safe verification sequence Use this order for a new installation, a recovered service or a material @@ -585,8 +744,9 @@ generator/configuration update: run ends with `Analytics generation completed`. 9. Check every generated file's path, owner, group, mode and timestamp. 10. Request the public JSON through HTTPS and validate its four fields. -11. Request both private report URLs with Basic Auth. Let the client prompt for - the password; never put it directly on a command line. +11. Request the existing GoAccess URLs and `/metrics/` with Basic Auth. Follow + its website and update-information daily/yearly links. Let the client prompt + for the password; never put it directly on a command line. 12. Run the service a second time and confirm that the total and report values develop plausibly rather than multiplying the existing history. 13. Enable or re-enable the timer only after these checks pass. @@ -655,6 +815,11 @@ The current private endpoints are: - `https://stats.hamradioonline.de/combined/`, which serves the combined report; - `https://stats.hamradioonline.de/kst4contest/`, which serves the site report. +- `https://stats.hamradioonline.de/metrics/`, which links to the separate + website and update-information daily/yearly views. + +The generator adds a small `/metrics/` link to each generated GoAccess HTML +report. The authenticated `/` redirect to `/combined/` remains unchanged. All HTTPS paths, including the redirect target, remain behind Basic Auth. Reports use `Cache-Control: private, no-store`, @@ -767,12 +932,13 @@ analytics logging is active. ## Regular operation -Nginx continuously writes only eligible requests to the dedicated analytics -log. The systemd timer starts the generator once per hour. Every successful -run refreshes the per-site and combined reports, persists the corresponding -GoAccess databases, updates daily counter values and finally publishes enabled -public counters. Logrotate handles the raw log once per day and preserves the -uncompressed `.1` handover file required by the generator. +Nginx continuously writes eligible website requests and exact successful +update-information requests to separate reduced logs. The systemd timer starts +the generator once per hour. Every successful run refreshes the per-site and +combined GoAccess reports, private daily/yearly views, daily visit counter +values and enabled public counters. Logrotate handles both raw logs once per +day and preserves each uncompressed `.1` handover file required by the +generator. The normal operator signal is the service result and journal, not a permanently running process: the generator is a short-lived oneshot service. There is no @@ -880,8 +1046,9 @@ present. The current analytics log remains required. ### `.gz` rotations on a GoAccess build without Zlib This is normal. Regular operation does not read `.gz` files. Do not configure -a compressed or rotated file as `analyticsLog`; any exceptional historical -import must be planned separately. +a compressed or rotated file as `analyticsLog` or `updateInfo.analyticsLog`. +Use `.gz` only through the explicit historical-import options; Node.js, not +GoAccess, decompresses that input. ### MMDB missing or unreadable @@ -900,11 +1067,32 @@ replace or repurpose the normal operational access log. sudo nginx -t sudo nginx -T sudo stat -c '%U:%G %a %s %y %n' \ - /var/log/nginx/kst4contest-analytics.log + /var/log/nginx/kst4contest-analytics.log \ + /var/log/nginx/kst4contest-update-information.log sudo -u hamradio-analytics test -r \ /var/log/nginx/kst4contest-analytics.log +sudo -u hamradio-analytics test -r \ + /var/log/nginx/kst4contest-update-information.log ``` +### Private metrics gap + +If the journal says that a private metric gap starts before the regular +current/`.1` handover window, stop the timer. Do not turn the missing dates +into zeros and do not delete `private-metrics-state.json`. Inventory the +remaining regular and reduced logs, make a protected state backup and use the +documented historical import only for a genuinely covered range. If no +reliable source remains, the first covered date must move forward through a +reviewed state migration; that is not an automatic repair. + +### Historical import rejects a log + +An invalid line normally means that the selected `nginx-combined` or +`analytics-tsv` parser does not match the real file, or that a file is damaged. +Inspect a redacted sample and the file boundaries. Do not delete failing lines +or switch formats until the actual Nginx log format is confirmed. A failed +import leaves the existing private state and reports unchanged. + ### Public counter returns `404` This is expected before the first successful generation. Afterwards inspect @@ -983,6 +1171,7 @@ part of this repository change. At minimum, the later plan must cover: - `/var/lib/hamradioonline-analytics/public-counter-state.json`; +- `/var/lib/hamradioonline-analytics/private-metrics-state.json`; - `/var/lib/hamradioonline-analytics/db`; - `/etc/hamradioonline-analytics`; - the installed systemd units; @@ -997,20 +1186,23 @@ Handle the Basic Auth hash, MaxMind Account ID and License Key, GitHub deploy token, ACME account data and private TLS keys as secrets. Never copy them into Git, public documentation, logs or ordinary support bundles. -The generator is recoverable from GitHub, and GeoLite2-Country can be fetched -again with `geoipupdate`. HTML/JSON reports can be rebuilt when the GoAccess -databases or sufficient raw logs remain. The public JSON can be rebuilt from -the counter state. +The generator modules are recoverable from GitHub, and GeoLite2-Country can be +fetched again with `geoipupdate`. HTML/JSON reports can be rebuilt when their +GoAccess databases or durable state remain. The public JSON can be rebuilt +from the counter state; the private daily/yearly pages can be rebuilt from +`private-metrics-state.json`. The accumulated public total is not fully recoverable without `public-counter-state.json`. Older detailed aggregates are not recoverable without the GoAccess databases, and historical raw requests disappear after the 14-day rotation window. -Do not let backups extend the published raw-log retention by accident. Either -exclude analytics raw logs from durable backups or enforce the same confirmed -retention limit in backup storage. Counter state and anonymised/aggregated -GoAccess state can be governed separately. +The durable website page-view and update-information history is not fully +recoverable without `private-metrics-state.json`. This state is aggregated and +belongs in the protected backup plan. Do not let backups extend the published +raw-log retention by accident: exclude raw logs or enforce the same 14-day +maximum in backup storage. Counter state and anonymised or aggregated GoAccess +state can be governed separately. ### Recovery order @@ -1020,7 +1212,8 @@ GoAccess state can be governed separately. 4. Install the generator and non-secret configuration. 5. Restore secrets and certificate state from protected backup storage. 6. Restore GeoLite2-Country or download it again. -7. Restore the GoAccess databases and public counter state. +7. Restore the GoAccess databases, public counter state and private metric + state. 8. Validate Nginx, systemd and Logrotate configuration. 9. Run the generator with `--check`. 10. Run `--dry-run` and verify that production state remains unchanged. @@ -1030,8 +1223,9 @@ GoAccess state can be governed separately. ## Outstanding operational checks -The first real rotation of the dedicated analytics log still needs explicit -observation. This is not a current service blocker. After rotation, confirm: +The first real rotation after installing the separate update-information log +still needs explicit observation. This is not a current service blocker. For +both reduced logs, confirm: - a new current log exists; - `.1` exists and remains uncompressed; diff --git a/website/ops/analytics/generate-reports.js b/website/ops/analytics/generate-reports.js index cd86a6a6..a041998f 100644 --- a/website/ops/analytics/generate-reports.js +++ b/website/ops/analytics/generate-reports.js @@ -5,6 +5,11 @@ const fs = require("node:fs"); const os = require("node:os"); const path = require("node:path"); const { spawnSync } = require("node:child_process"); +const { + UPDATE_INFO_PATH, + generatePrivateMetrics, + validateImportOptions +} = require("./private-metrics"); const STATE_SCHEMA_VERSION = 1; const PUBLIC_SCHEMA_VERSION = 1; @@ -156,6 +161,34 @@ function validateRegistry(registry) { } analyticsLogPaths.add(analyticsLog); + const websiteMetricsSince = parseIsoDate(site.websiteMetricsSince); + if (!websiteMetricsSince) { + throw new ConfigurationError(`${label}.websiteMetricsSince must be a valid ISO date`); + } + if (!site.updateInfo || typeof site.updateInfo !== "object" || Array.isArray(site.updateInfo)) { + throw new ConfigurationError(`${label}.updateInfo must be an object`); + } + if (site.updateInfo.path !== UPDATE_INFO_PATH) { + throw new ConfigurationError(`${label}.updateInfo.path must be ${UPDATE_INFO_PATH}`); + } + const updateInfoLog = requireAbsolutePath( + site.updateInfo.analyticsLog, + `${label}.updateInfo.analyticsLog` + ); + if (/\.(?:\d+|gz)$/i.test(path.basename(updateInfoLog))) { + throw new ConfigurationError( + `${label}.updateInfo.analyticsLog must identify the current uncompressed log` + ); + } + if (analyticsLogPaths.has(updateInfoLog)) { + throw new ConfigurationError(`analytics log is registered more than once: ${updateInfoLog}`); + } + analyticsLogPaths.add(updateInfoLog); + const updateMetricsSince = parseIsoDate(site.updateInfo.metricsSince); + if (!updateMetricsSince) { + throw new ConfigurationError(`${label}.updateInfo.metricsSince must be a valid ISO date`); + } + const reportOutputDirectory = requireAbsolutePath( site.reportOutputDirectory, `${label}.reportOutputDirectory` @@ -190,6 +223,12 @@ function validateRegistry(registry) { activatedOn, publicCounter: site.publicCounter, analyticsLog, + websiteMetricsSince, + updateInfo: { + path: UPDATE_INFO_PATH, + analyticsLog: updateInfoLog, + metricsSince: updateMetricsSince + }, reportOutputDirectory, publicJsonPath }; @@ -214,6 +253,34 @@ function validateRegistry(registry) { ); } + if (!registry.privateMetrics || typeof registry.privateMetrics !== "object" + || Array.isArray(registry.privateMetrics)) { + throw new ConfigurationError("registry.privateMetrics must be an object"); + } + const privateMetricsStatePath = requireAbsolutePath( + registry.privateMetrics.statePath, + "registry.privateMetrics.statePath" + ); + if (!isWithin(stateDirectory, privateMetricsStatePath) + || privateMetricsStatePath === counterStatePath) { + throw new ConfigurationError("private metrics statePath must be a distinct path below stateDirectory"); + } + const privateMetricsReportOutputDirectory = requireAbsolutePath( + registry.privateMetrics.reportOutputDirectory, + "registry.privateMetrics.reportOutputDirectory" + ); + if (!isWithin(stateDirectory, privateMetricsReportOutputDirectory) + || outputDirectories.has(privateMetricsReportOutputDirectory) + || privateMetricsReportOutputDirectory === combinedReportOutputDirectory) { + throw new ConfigurationError("private metrics reportOutputDirectory must be distinct below stateDirectory"); + } + if (registry.privateMetrics.timeZone !== "Europe/Berlin") { + throw new ConfigurationError("registry.privateMetrics.timeZone must be Europe/Berlin"); + } + if (registry.privateMetrics.detailRetentionDays !== 14) { + throw new ConfigurationError("registry.privateMetrics.detailRetentionDays must be 14"); + } + return { schemaVersion: 1, stateDirectory, @@ -226,6 +293,12 @@ function validateRegistry(registry) { combined: { reportOutputDirectory: combinedReportOutputDirectory }, + privateMetrics: { + statePath: privateMetricsStatePath, + reportOutputDirectory: privateMetricsReportOutputDirectory, + timeZone: "Europe/Berlin", + detailRetentionDays: 14 + }, sites }; } @@ -304,6 +377,10 @@ function validateInputs(registry, configTemplate) { site.id, resolveAnalyticsLogs(site) ])); + const updateLogsBySite = new Map(registry.sites.map(site => [ + site.id, + resolveAnalyticsLogs({ analyticsLog: site.updateInfo.analyticsLog }) + ])); let geoStats; try { @@ -330,6 +407,10 @@ function validateInputs(registry, configTemplate) { registry.combined.reportOutputDirectory, "combined report output directory" ); + requireWritableDirectory( + registry.privateMetrics.reportOutputDirectory, + "private metrics report output directory" + ); for (const site of registry.sites) { requireWritableDirectory( site.reportOutputDirectory, @@ -343,7 +424,7 @@ function validateInputs(registry, configTemplate) { } } - return analyticsLogsBySite; + return { analyticsLogsBySite, updateLogsBySite }; } function renderGoAccessConfig(template, dbPath, restore, geoIpCountryDatabase) { @@ -604,6 +685,11 @@ function atomicCopyFile(source, destination) { atomicWriteFile(destination, fs.readFileSync(source)); } +function addPrivateMetricsLink(html) { + const link = '

Private daily and annual metrics

'; + return /<\/body>/i.test(html) ? html.replace(/<\/body>/i, `${link}`) : `${html}\n${link}\n`; +} + function replaceDirectory(source, destination) { fs.mkdirSync(path.dirname(destination), { recursive: true }); const backup = `${destination}.previous-${process.pid}`; @@ -720,7 +806,12 @@ function generateReports(options, dependencies = {}) { const checkGoAccess = dependencies.checkGoAccess || defaultCheckGoAccess; const now = dependencies.now ? dependencies.now() : new Date(); - const analyticsLogsBySite = validateInputs(registry, configTemplate); + const { analyticsLogsBySite, updateLogsBySite } = validateInputs(registry, configTemplate); + try { + validateImportOptions(options, registry); + } catch (error) { + throw new ConfigurationError(error.message); + } const goAccess = checkGoAccess(options.goaccessBinary || "goaccess"); if (!goAccess || !goAccess.geoIpMmdb) { throw new ConfigurationError( @@ -777,15 +868,24 @@ function generateReports(options, dependencies = {}) { content: `${JSON.stringify(publicPayload(state, site, now), null, 2)}\n` })); + const privateMetrics = generatePrivateMetrics({ + registry, + analyticsLogsBySite, + updateLogsBySite, + options, + visitState: state, + context: { + ...context, + now + } + }); + if (!options.dryRun) { for (const report of prepared) { - atomicCopyFile( - report.outputJson, - path.join(report.outputDirectory, "report.json") - ); - atomicCopyFile( - report.outputHtml, - path.join(report.outputDirectory, "report.html") + atomicCopyFile(report.outputJson, path.join(report.outputDirectory, "report.json")); + atomicWriteFile( + path.join(report.outputDirectory, "report.html"), + addPrivateMetricsLink(fs.readFileSync(report.outputHtml, "utf8")) ); } for (const report of prepared) { @@ -798,6 +898,14 @@ function generateReports(options, dependencies = {}) { for (const publicFile of publicFiles) { atomicWriteFile(publicFile.destination, publicFile.content, 0o644); } + atomicWriteFile( + registry.privateMetrics.statePath, + `${JSON.stringify(privateMetrics.state, null, 2)}\n` + ); + for (const report of privateMetrics.files) { + fs.mkdirSync(path.dirname(report.destination), { recursive: true }); + atomicWriteFile(report.destination, report.content); + } } return { @@ -805,7 +913,9 @@ function generateReports(options, dependencies = {}) { dryRun: Boolean(options.dryRun), sites: registry.sites.length, reports: prepared.length, - publicCounters: publicFiles.length + publicCounters: publicFiles.length, + privateMetricReports: privateMetrics.files.length, + historicalImport: privateMetrics.imported }; } finally { if (runDirectory && fs.existsSync(runDirectory)) { @@ -816,7 +926,12 @@ function generateReports(options, dependencies = {}) { } function parseArguments(argv) { - const options = { check: false, dryRun: false, goaccessBinary: "goaccess" }; + const options = { + check: false, + dryRun: false, + goaccessBinary: "goaccess", + importLogs: [] + }; for (let index = 0; index < argv.length; index += 1) { const argument = argv[index]; @@ -824,7 +939,10 @@ function parseArguments(argv) { options.check = true; } else if (argument === "--dry-run") { options.dryRun = true; - } else if (["--registry", "--config-template", "--goaccess"].includes(argument)) { + } else if ([ + "--registry", "--config-template", "--goaccess", "--import-log", "--import-format", + "--import-site", "--coverage-from", "--coverage-through" + ].includes(argument)) { const value = argv[index + 1]; if (!value) { throw new ConfigurationError(`${argument} requires a value`); @@ -833,6 +951,11 @@ function parseArguments(argv) { if (argument === "--registry") options.registryPath = value; if (argument === "--config-template") options.configTemplatePath = value; if (argument === "--goaccess") options.goaccessBinary = value; + if (argument === "--import-log") options.importLogs.push(value); + if (argument === "--import-format") options.importFormat = value; + if (argument === "--import-site") options.importSite = value; + if (argument === "--coverage-from") options.coverageFrom = value; + if (argument === "--coverage-through") options.coverageThrough = value; } else { throw new ConfigurationError(`unknown argument: ${argument}`); } @@ -841,7 +964,9 @@ function parseArguments(argv) { if (!options.registryPath || !options.configTemplatePath) { throw new ConfigurationError( "usage: generate-reports.js --registry FILE --config-template FILE " - + "[--goaccess FILE] [--check|--dry-run]" + + "[--goaccess FILE] [--check|--dry-run] " + + "[--import-log FILE ... --import-format nginx-combined|analytics-tsv " + + "--import-site ID --coverage-from DATE --coverage-through DATE]" ); } if (options.check && options.dryRun) { diff --git a/website/ops/analytics/logrotate/hamradioonline-analytics.example b/website/ops/analytics/logrotate/hamradioonline-analytics.example index 28f52a55..bd024cea 100644 --- a/website/ops/analytics/logrotate/hamradioonline-analytics.example +++ b/website/ops/analytics/logrotate/hamradioonline-analytics.example @@ -1,4 +1,4 @@ -/var/log/nginx/*-analytics.log { +/var/log/nginx/*-analytics.log /var/log/nginx/*-update-information.log { daily rotate 14 missingok diff --git a/website/ops/analytics/nginx/analytics-filters.conf.example b/website/ops/analytics/nginx/analytics-filters.conf.example index cf087213..c37622a9 100644 --- a/website/ops/analytics/nginx/analytics-filters.conf.example +++ b/website/ops/analytics/nginx/analytics-filters.conf.example @@ -34,3 +34,10 @@ map "$hamradioonline_analytics_method:$hamradioonline_analytics_path:$hamradioon default 0; "1:1:0" 1; } + +# Count the update-information file separately. This deliberately does not +# filter user agents: browsers and bots can fetch the file too. +map "$request_method:$uri:$status" $hamradioonline_update_information_loggable { + default 0; + "GET:/kst4ContestVersionInfo.xml:200" 1; +} diff --git a/website/ops/analytics/nginx/analytics-log.conf.example b/website/ops/analytics/nginx/analytics-log.conf.example index da64d5db..b3813708 100644 --- a/website/ops/analytics/nginx/analytics-log.conf.example +++ b/website/ops/analytics/nginx/analytics-log.conf.example @@ -7,9 +7,10 @@ log_format hamradioonline_analytics # Include the maps below in the Nginx http context as well. include /etc/nginx/snippets/hamradioonline-analytics-filters.conf; -# Add this extra log to each registered project server block. Keep the +# Add these extra logs to each registered project server block. Keep the # existing operational access_log directive; do not replace it implicitly. # # access_log /var/log/nginx/kst4contest-analytics.log # hamradioonline_analytics if=$hamradioonline_analytics_loggable; - +# access_log /var/log/nginx/kst4contest-update-information.log +# hamradioonline_analytics if=$hamradioonline_update_information_loggable; diff --git a/website/ops/analytics/private-metrics.js b/website/ops/analytics/private-metrics.js new file mode 100644 index 00000000..35ade4e5 --- /dev/null +++ b/website/ops/analytics/private-metrics.js @@ -0,0 +1,769 @@ +"use strict"; + +const fs = require("node:fs"); +const path = require("node:path"); +const zlib = require("node:zlib"); + +const PRIVATE_STATE_SCHEMA_VERSION = 1; +const UPDATE_INFO_PATH = "/kst4ContestVersionInfo.xml"; +const UNKNOWN_COUNTRY = "Unknown"; +const MAX_IMPORT_BYTES = 1024 * 1024 * 1024; + +function parseIsoDate(value) { + const match = /^(\d{4})-(\d{2})-(\d{2})$/.exec(value || ""); + if (!match) return null; + const date = new Date(Date.UTC(Number(match[1]), Number(match[2]) - 1, Number(match[3]))); + return date.getUTCFullYear() === Number(match[1]) + && date.getUTCMonth() === Number(match[2]) - 1 + && date.getUTCDate() === Number(match[3]) + ? value + : null; +} + +function dateRange(from, through) { + const result = []; + const current = new Date(`${from}T12:00:00Z`); + const end = new Date(`${through}T12:00:00Z`); + while (current <= end) { + result.push(current.toISOString().slice(0, 10)); + current.setUTCDate(current.getUTCDate() + 1); + } + return result; +} + +function shiftDate(date, days) { + const value = new Date(`${date}T12:00:00Z`); + value.setUTCDate(value.getUTCDate() + days); + return value.toISOString().slice(0, 10); +} + +function localParts(timestamp, timeZone) { + const instant = timestamp instanceof Date ? timestamp : new Date(timestamp); + if (Number.isNaN(instant.getTime())) return null; + const parts = Object.fromEntries(new Intl.DateTimeFormat("en-CA", { + timeZone, + year: "numeric", + month: "2-digit", + day: "2-digit", + hour: "2-digit", + hourCycle: "h23" + }).formatToParts(instant).filter(part => part.type !== "literal") + .map(part => [part.type, part.value])); + return { date: `${parts.year}-${parts.month}-${parts.day}`, hour: parts.hour }; +} + +function normalizePath(value) { + if (typeof value !== "string" || /[\r\n\0]/.test(value)) return null; + const withoutQuery = value.split(/[?#]/, 1)[0]; + if (!withoutQuery.startsWith("/")) return null; + const collapsed = withoutQuery.replace(/\/{2,}/g, "/"); + const trailingSlash = collapsed.length > 1 && collapsed.endsWith("/"); + const normalized = path.posix.normalize(collapsed); + return trailingSlash && normalized !== "/" && !normalized.endsWith("/") + ? `${normalized}/` + : normalized; +} + +function unescapeNginx(value) { + return value.replace(/\\x([0-9A-Fa-f]{2})/g, (_match, hex) => String.fromCharCode(Number.parseInt(hex, 16))) + .replace(/\\"/g, "\"") + .replace(/\\\\/g, "\\"); +} + +function parseAnalyticsLine(line, timeZone = "Europe/Berlin") { + const fields = line.split("\t"); + if (fields.length !== 9) return null; + const status = Number(fields[6]); + const bytes = fields[7] === "-" ? 0 : Number(fields[7]); + const local = localParts(fields[2], timeZone); + const requestPath = normalizePath(fields[4]); + if (!local || !requestPath || !/^\d{3}$/.test(fields[6]) + || !Number.isSafeInteger(bytes) || bytes < 0) return null; + let userAgent = fields[8]; + if (userAgent.startsWith("\"") && userAgent.endsWith("\"")) { + userAgent = userAgent.slice(1, -1); + } + const record = { + host: fields[0], + ip: fields[1], + timestamp: fields[2], + date: local.date, + hour: local.hour, + method: fields[3], + path: requestPath, + protocol: fields[5], + status, + bytes, + userAgent: unescapeNginx(userAgent) + }; + record.line = toAnalyticsLine(record); + return record; +} + +const NGINX_MONTHS = { + Jan: "01", Feb: "02", Mar: "03", Apr: "04", May: "05", Jun: "06", + Jul: "07", Aug: "08", Sep: "09", Oct: "10", Nov: "11", Dec: "12" +}; + +function parseCombinedLine(line, hostname, timeZone = "Europe/Berlin") { + const match = /^(\S+) \S+ \S+ \[(\d{2})\/([A-Za-z]{3})\/(\d{4}):(\d{2}):(\d{2}):(\d{2}) ([+-]\d{4})\] "([A-Z]+) ([^ ]+) ([^"]+)" (\d{3}) (\d+|-) "(?:[^"\\]|\\.)*" "((?:[^"\\]|\\.)*)"(?: .*)?$/.exec(line); + if (!match || !NGINX_MONTHS[match[3]]) return null; + const timestamp = `${match[4]}-${NGINX_MONTHS[match[3]]}-${match[2]}T${match[5]}:${match[6]}:${match[7]}${match[8]}`; + const local = localParts(timestamp, timeZone); + const requestPath = normalizePath(match[10]); + const status = Number(match[12]); + const bytes = match[13] === "-" ? 0 : Number(match[13]); + if (!local || !requestPath || !Number.isSafeInteger(bytes) || bytes < 0) return null; + const userAgent = unescapeNginx(match[14]); + const canonicalTimestamp = timestamp.replace(/([+-]\d{2})(\d{2})$/, "$1:$2"); + return { + host: hostname, + ip: match[1], + timestamp: canonicalTimestamp, + date: local.date, + hour: local.hour, + method: match[9], + path: requestPath, + protocol: match[11], + status, + bytes, + userAgent, + line: toAnalyticsLine({ + host: hostname, + ip: match[1], + timestamp: canonicalTimestamp, + method: match[9], + path: requestPath, + protocol: match[11], + status, + bytes, + userAgent + }) + }; +} + +function escapeLogField(value) { + return value.replace(/\\/g, "\\\\").replace(/"/g, "\\\"") + .replace(/[\t\r\n]/g, character => `\\x${character.charCodeAt(0).toString(16).padStart(2, "0")}`); +} + +function toAnalyticsLine(record) { + return [record.host, record.ip, record.timestamp, record.method, record.path, + record.protocol, record.status, record.bytes, `"${escapeLogField(record.userAgent)}"`].join("\t"); +} + +function isKnownBot(userAgent) { + return /(?:bot|crawler|spider|slurp|headless|monitor|healthcheck|uptime|wget|curl)/i.test(userAgent); +} + +function isEligibleWebsiteRequest(record) { + if (record.method !== "GET" || isKnownBot(record.userAgent)) return false; + if (["/visitor-count.json", UPDATE_INFO_PATH, "/sitemap.xml", "/robots.txt", "/favicon.ico", + "/assets/favicon.svg", "/health", "/healthz", "/ping", "/status"].includes(record.path)) { + return false; + } + return !/^\/(?:assets|manual\/assets)\//i.test(record.path) + && !/\.(?:css|js|mjs|map|json|png|jpe?g|gif|svg|webp|avif|ico|woff2?|ttf|otf|eot|xml|txt|pdf|zip|gz|wasm|mp4|webm)$/i.test(record.path); +} + +function isUpdateRequest(record) { + return record.method === "GET" && record.status === 200 && record.path === UPDATE_INFO_PATH; +} + +function clientGroup(userAgent) { + if (!userAgent || userAgent === "-") return "Missing user agent"; + if (/KST4Contest/i.test(userAgent)) return "KST4Contest (explicit)"; + if (/(?:Edg|Edge)\//i.test(userAgent)) return "Microsoft Edge"; + if (/Firefox\//i.test(userAgent)) return "Firefox"; + if (/(?:Chrome|Chromium)\//i.test(userAgent)) return "Chrome/Chromium"; + if (/Safari\//i.test(userAgent)) return "Safari"; + if (/^Java\//i.test(userAgent)) return "Java runtime (application unknown)"; + if (/curl\//i.test(userAgent)) return "curl"; + if (/Wget\//i.test(userAgent)) return "Wget"; + if (/(?:bot|crawler|spider|slurp)/i.test(userAgent)) return "Bot/crawler"; + return "Other or unrecognised"; +} + +function readLogFile(filePath) { + const stats = fs.statSync(filePath); + if (!stats.isFile()) throw new Error(`log input is not a file: ${filePath}`); + if (stats.size > MAX_IMPORT_BYTES) throw new Error(`log input exceeds the 1 GiB safety limit: ${filePath}`); + const content = fs.readFileSync(filePath); + try { + return filePath.toLowerCase().endsWith(".gz") + ? zlib.gunzipSync(content, { maxOutputLength: MAX_IMPORT_BYTES }).toString("utf8") + : content.toString("utf8"); + } catch (error) { + throw new Error(`could not read compressed log ${filePath}: ${error.message}`); + } +} + +function parseLogFiles(filePaths, parser, { deduplicateAcrossFiles = false, label = "log" } = {}) { + const parsedFiles = []; + for (const filePath of filePaths) { + const records = []; + const lines = readLogFile(filePath).split(/\r?\n/); + for (let index = 0; index < lines.length; index += 1) { + if (lines[index] === "" && index === lines.length - 1) continue; + if (lines[index].trim() === "") continue; + const record = parser(lines[index]); + if (!record) throw new Error(`${label} has an invalid line at ${filePath}:${index + 1}`); + records.push(record); + } + parsedFiles.push(records); + } + if (!deduplicateAcrossFiles) return parsedFiles.flat(); + + const maxima = new Map(); + for (const records of parsedFiles) { + const counts = new Map(); + for (const record of records) counts.set(record.line, (counts.get(record.line) || 0) + 1); + for (const [line, count] of counts) maxima.set(line, Math.max(maxima.get(line) || 0, count)); + } + const byLine = new Map(parsedFiles.flat().map(record => [record.line, record])); + const result = []; + for (const [line, count] of maxima) { + for (let index = 0; index < count; index += 1) result.push({ ...byLine.get(line) }); + } + return result; +} + +function addCount(values, key, count) { + const value = (Object.hasOwn(values, key) ? values[key] : 0) + count; + Object.defineProperty(values, key, { + value, + writable: true, + enumerable: true, + configurable: true + }); +} + +function countPanel(report, panelName) { + const panel = report[panelName]; + if (!panel || !Array.isArray(panel.data)) throw new Error(`GoAccess JSON report has no ${panelName} panel`); + const values = {}; + for (const row of panel.data) { + const rawName = typeof row.data === "string" ? row.data.trim() : ""; + const name = !rawName || /^(?:unknown|n\/a|-|\(not set\))$/i.test(rawName) + ? UNKNOWN_COUNTRY + : rawName; + const count = row && row.hits && row.hits.count; + if (!Number.isSafeInteger(count) || count < 0) throw new Error(`GoAccess ${panelName} panel contains invalid data`); + addCount(values, name, count); + } + return values; +} + +function countCountries(report) { + const panel = report.geolocation; + if (!panel || !Array.isArray(panel.data)) { + throw new Error("GoAccess JSON report has no geolocation panel"); + } + const rows = panel.data.flatMap(row => Array.isArray(row.items) && row.items.length + ? row.items + : [row]); + return countPanel({ geolocation: { data: rows } }, "geolocation"); +} + +function sumValues(values) { + return Object.values(values).reduce((sum, value) => sum + value, 0); +} + +function aggregateGoAccessReport(report, kind) { + const total = report && report.general && report.general.total_requests; + if (!Number.isSafeInteger(total) || total < 0) throw new Error("GoAccess report contains an invalid total request count"); + const countries = countCountries(report); + const countryTotal = sumValues(countries); + if (countryTotal > total) throw new Error("GoAccess country total exceeds the request total"); + if (countryTotal < total) addCount(countries, UNKNOWN_COUNTRY, total - countryTotal); + + if (kind === "updateInfo") return { requests: total, countries }; + const paths = countPanel(report, "requests"); + if (sumValues(paths) !== total) { + throw new Error("website page total differs from the GoAccess request-panel definition"); + } + return { pageViews: total, countries, paths }; +} + +function renderMetricsConfig(template, dbPath, geoIpCountryDatabase, includeCrawlers) { + let rendered = template + .replaceAll("{{DB_PATH}}", dbPath) + .replaceAll("{{RESTORE_DIRECTIVE}}", "# private daily aggregation uses a fresh database") + .replaceAll("{{GEOIP_COUNTRY_DATABASE}}", geoIpCountryDatabase) + .replace(/^max-items\s+\d+$/m, "max-items 1000000") + .replace(/^persist true$/m, "# persistence is disabled for private daily aggregation"); + if (includeCrawlers) { + rendered = rendered.replace(/^ignore-crawlers true$/m, "ignore-crawlers false") + .replace(/^unknowns-as-crawlers true$/m, "unknowns-as-crawlers false"); + } + return rendered; +} + +function aggregateDays({ records, dates, kind, site, context }) { + const byDate = new Map(dates.map(date => [date, []])); + for (const record of records) { + if (byDate.has(record.date)) byDate.get(record.date).push(record); + } + const daily = {}; + for (const date of dates) { + const dayRecords = byDate.get(date); + if (dayRecords.length === 0) { + daily[date] = kind === "website" + ? { pageViews: 0, countries: {}, paths: {} } + : { requests: 0, countries: {}, hours: {}, clients: {} }; + continue; + } + const jobId = `metrics-${site.id}-${kind}-${date}`; + const jobDirectory = path.join(context.runDirectory, jobId); + const dbPath = path.join(jobDirectory, "db"); + const inputPath = path.join(jobDirectory, "input.log"); + const outputJson = path.join(jobDirectory, "report.json"); + const runConfig = path.join(jobDirectory, "goaccess.conf"); + fs.mkdirSync(dbPath, { recursive: true }); + fs.writeFileSync(inputPath, `${dayRecords.map(record => record.line).join("\n")}\n`, { mode: 0o600 }); + fs.writeFileSync(runConfig, renderMetricsConfig( + context.configTemplate, + dbPath, + context.registry.geoIpCountryDatabase, + kind === "updateInfo" + ), { mode: 0o600 }); + context.runGoAccess({ + binary: context.goaccessBinary, + args: [inputPath, "--no-global-config", "--config-file", runConfig, "--output", outputJson], + id: jobId, + metricKind: kind, + date, + outputJson, + dbPath + }); + let report; + try { + report = JSON.parse(fs.readFileSync(outputJson, "utf8")); + } catch (error) { + throw new Error(`private metric GoAccess JSON is invalid for ${site.id}/${date}: ${error.message}`); + } + daily[date] = aggregateGoAccessReport(report, kind); + if (kind === "updateInfo") { + const hours = {}; + const clients = {}; + for (const record of dayRecords) { + addCount(hours, record.hour, 1); + const group = clientGroup(record.userAgent); + addCount(clients, group, 1); + } + if (sumValues(hours) !== daily[date].requests || sumValues(clients) !== daily[date].requests) { + throw new Error(`update detail total differs from GoAccess for ${site.id}/${date}`); + } + daily[date].hours = hours; + daily[date].clients = clients; + } + } + return daily; +} + +function loadState(statePath) { + if (!fs.existsSync(statePath)) return { schemaVersion: PRIVATE_STATE_SCHEMA_VERSION, sites: {} }; + let state; + try { + state = JSON.parse(fs.readFileSync(statePath, "utf8")); + } catch (error) { + throw new Error(`private metrics state is not valid JSON: ${error.message}`); + } + if (!state || state.schemaVersion !== PRIVATE_STATE_SCHEMA_VERSION || !state.sites + || typeof state.sites !== "object" || Array.isArray(state.sites)) { + throw new Error("private metrics state has an unsupported structure"); + } + return state; +} + +function containsAtLeast(candidate, current) { + const keys = new Set([...Object.keys(candidate), ...Object.keys(current)]); + return [...keys].every(key => (candidate[key] || 0) >= (current[key] || 0)); +} + +function sameCounts(left, right) { + const keys = new Set([...Object.keys(left), ...Object.keys(right)]); + return [...keys].every(key => (left[key] || 0) === (right[key] || 0)); +} + +function mergeDay(current, candidate, totalKey, label) { + if (!current) return candidate; + const currentTotal = current[totalKey]; + const candidateTotal = candidate[totalKey]; + if (candidateTotal === currentTotal) { + if (!sameCounts(current.countries, candidate.countries) + || (totalKey === "pageViews" && !sameCounts(current.paths, candidate.paths))) { + throw new Error(`overlapping ${label} aggregates disagree at equal totals`); + } + return { ...candidate, ...(current.hours ? { hours: current.hours, clients: current.clients } : {}) }; + } + const candidateLarger = candidateTotal > currentTotal; + const larger = candidateLarger ? candidate : current; + const smaller = candidateLarger ? current : candidate; + if (!containsAtLeast(larger.countries, smaller.countries) + || (totalKey === "pageViews" && !containsAtLeast(larger.paths, smaller.paths))) { + throw new Error(`overlapping ${label} aggregates are not monotonic`); + } + return larger; +} + +function mergeSeries(target, incoming, totalKey, label) { + for (const [date, candidate] of Object.entries(incoming)) { + target[date] = mergeDay(target[date], candidate, totalKey, `${label}/${date}`); + } +} + +function purgeDetails(siteState, today, retentionDays) { + const firstRetained = shiftDate(today, -(retentionDays - 1)); + for (const [date, value] of Object.entries(siteState.updateInfo.daily)) { + if (date < firstRetained) { + delete value.hours; + delete value.clients; + } + } +} + +function escapeHtml(value) { + return String(value).replace(/[&<>"']/g, character => ({ + "&": "&", "<": "<", ">": ">", "\"": """, "'": "'" + })[character]); +} + +function page(title, body) { + return `\n\n` + + `\n` + + `${escapeHtml(title)}\n` + + `` + + `` + + body + `\n`; +} + +function table(headers, rows) { + return `
${headers.map(value => ``).join("")}` + + `${rows.length ? rows.map(row => `${row.map((value, index) => `${escapeHtml(value)}`).join("")}`).join("") : ``}
${escapeHtml(value)}
No data
`; +} + +function sortedEntries(values) { + return Object.entries(values).sort(([left], [right]) => left.localeCompare(right, "en")); +} + +function annualTotals(daily, totalKey) { + const years = {}; + for (const [date, value] of Object.entries(daily)) { + const year = date.slice(0, 4); + if (!years[year]) years[year] = { total: 0, countries: {}, paths: {} }; + years[year].total += value[totalKey]; + for (const [country, count] of Object.entries(value.countries)) { + addCount(years[year].countries, country, count); + } + for (const [requestPath, count] of Object.entries(value.paths || {})) { + addCount(years[year].paths, requestPath, count); + } + } + return years; +} + +function coverageText(series) { + return series.firstCoveredOn + ? `The first covered day is ${escapeHtml(series.firstCoveredOn)}. Earlier dates are unknown, not zero.` + : "No covered day has been recorded yet."; +} + +function renderDaily(site, kind, visitSite = null) { + const series = site[kind]; + const dailyVisits = visitSite ? visitSite.dailyVisits : {}; + const totalKey = kind === "website" ? "pageViews" : "requests"; + const label = kind === "website" ? "Website page views" : "Update-information requests"; + const dates = [...new Set([ + ...Object.keys(series.daily), + ...(kind === "website" ? Object.keys(dailyVisits) : []) + ])].sort(); + const totals = dates.map(date => kind === "website" + ? [date, series.daily[date] ? series.daily[date][totalKey] : "unknown", dailyVisits[date] ?? "unknown"] + : [date, series.daily[date][totalKey]]); + const metricDates = Object.keys(series.daily).sort(); + const countries = metricDates.flatMap(date => sortedEntries(series.daily[date].countries) + .map(([country, count]) => [date, country, count])); + let details = table(kind === "website" + ? ["Date", label, "Approximate visits"] + : ["Date", label], totals) + `

Countries by day

` + + table(["Date", "Country", kind === "website" ? "Page views" : "Requests"], countries); + if (kind === "website") { + const paths = metricDates.flatMap(date => sortedEntries(series.daily[date].paths) + .map(([requestPath, count]) => [date, requestPath, count])); + details += `

Pages by day

${table(["Date", "Path", "Page views"], paths)}`; + } else { + const hours = dates.flatMap(date => sortedEntries(series.daily[date].hours || {}) + .map(([hour, count]) => [date, `${hour}:00–${hour}:59`, count])); + const clients = dates.flatMap(date => sortedEntries(series.daily[date].clients || {}) + .map(([client, count]) => [date, client, count])); + details += `

Hourly detail (last 14 days)

${table(["Date", "Hour (Europe/Berlin)", "Requests"], hours)}` + + `

Recognisable client groups (last 14 days)

` + + `

Existing KST4Contest versions do not send a reliable application-specific user agent. A Java user agent therefore identifies only a Java runtime, not a KST4Contest start or user.

` + + table(["Date", "Client group", "Requests"], clients); + } + const visitCoverage = kind === "website" && visitSite + ? ` Approximate visits have their own coverage beginning ${escapeHtml(visitSite.since)}.` + : ""; + return page(`${label} by day`, `

${label}: daily view

${coverageText(series)}${visitCoverage}

${details}`); +} + +function renderAnnual(site, kind, visitSite = null) { + const series = site[kind]; + const dailyVisits = visitSite ? visitSite.dailyVisits : {}; + const totalKey = kind === "website" ? "pageViews" : "requests"; + const label = kind === "website" ? "Website page views" : "Update-information requests"; + const years = annualTotals(series.daily, totalKey); + const visitYears = {}; + for (const [date, count] of Object.entries(dailyVisits)) { + visitYears[date.slice(0, 4)] = (visitYears[date.slice(0, 4)] || 0) + count; + } + const allYears = [...new Set([ + ...Object.keys(years), + ...(kind === "website" ? Object.keys(visitYears) : []) + ])].sort(); + const summary = allYears.map(year => kind === "website" + ? [year, years[year] ? years[year].total : "unknown", visitYears[year] ?? "unknown"] + : [year, years[year].total]); + const allDates = [...new Set([ + ...Object.keys(series.daily), + ...(kind === "website" ? Object.keys(dailyVisits) : []) + ])].sort(); + const trend = allDates.map(date => kind === "website" + ? [date, series.daily[date] ? series.daily[date][totalKey] : "unknown", dailyVisits[date] ?? "unknown"] + : [date, series.daily[date][totalKey]]); + const countries = sortedEntries(years).flatMap(([year, value]) => sortedEntries(value.countries) + .map(([country, count]) => [year, country, count])); + let details = table(kind === "website" + ? ["Year", label, "Approximate visits"] + : ["Year", label], summary) + `

Daily values by year

` + + table(kind === "website" + ? ["Date", label, "Approximate visits"] + : ["Date", label], trend) + `

Country totals by year

` + + table(["Year", "Country", kind === "website" ? "Page views" : "Requests"], countries); + if (kind === "website") { + const paths = sortedEntries(years).flatMap(([year, value]) => sortedEntries(value.paths) + .map(([requestPath, count]) => [year, requestPath, count])); + details += `

Page totals by year

${table(["Year", "Path", "Page views"], paths)}`; + } + const visitCoverage = kind === "website" && visitSite + ? ` Approximate visits have their own coverage beginning ${escapeHtml(visitSite.since)}.` + : ""; + return page(`${label} by year`, `

${label}: annual view

${coverageText(series)}${visitCoverage}

${details}`); +} + +function renderReports(state, registry, now, visitState = { sites: {} }) { + const files = []; + const siteLinks = []; + for (const siteConfig of registry.sites) { + const site = state.sites[siteConfig.id]; + const visitSite = Object.hasOwn(visitState.sites, siteConfig.id) + ? visitState.sites[siteConfig.id] + : null; + const base = path.join(registry.privateMetrics.reportOutputDirectory, siteConfig.id); + files.push({ destination: path.join(base, "website", "daily", "report.html"), content: renderDaily(site, "website", visitSite) }); + files.push({ destination: path.join(base, "website", "yearly", "report.html"), content: renderAnnual(site, "website", visitSite) }); + files.push({ destination: path.join(base, "updates", "daily", "report.html"), content: renderDaily(site, "updateInfo") }); + files.push({ destination: path.join(base, "updates", "yearly", "report.html"), content: renderAnnual(site, "updateInfo") }); + siteLinks.push(`

${escapeHtml(siteConfig.hostname)}

`); + } + files.push({ + destination: path.join(registry.privateMetrics.reportOutputDirectory, "report.html"), + content: page("Private website metrics", `

Private website metrics

` + + `

Generated ${escapeHtml(now.toISOString())}. Visits, page views and update-information requests are separate measures.

` + + `

An update-information request is a successful GET request for the exact XML path. Browsers and bots may request it too; it is neither a program-start count nor a user count.

${siteLinks.join("")}`) + }); + return files; +} + +function validateImportOptions(options, registry) { + const hasImport = Array.isArray(options.importLogs) && options.importLogs.length > 0; + const companions = [options.importFormat, options.importSite, options.coverageFrom, options.coverageThrough]; + if (!hasImport && companions.some(Boolean)) throw new Error("historical import options require at least one --import-log"); + if (!hasImport) return null; + if (!options.importFormat || !["nginx-combined", "analytics-tsv"].includes(options.importFormat)) { + throw new Error("--import-format must be nginx-combined or analytics-tsv"); + } + const site = registry.sites.find(entry => entry.id === options.importSite); + if (!site) throw new Error("--import-site must identify a registered site"); + if (!parseIsoDate(options.coverageFrom) || !parseIsoDate(options.coverageThrough) + || options.coverageFrom > options.coverageThrough) { + throw new Error("--coverage-from and --coverage-through must define a valid inclusive range"); + } + const importLogs = options.importLogs.map(filePath => path.resolve(filePath)); + for (const filePath of importLogs) fs.accessSync(filePath, fs.constants.R_OK); + return { ...options, site, importLogs }; +} + +function validCountMap(values) { + return values && typeof values === "object" && !Array.isArray(values) + && Object.entries(values).every(([key, value]) => key !== "" + && Number.isSafeInteger(value) && value >= 0); +} + +function validateStoredSeries(series, totalKey, label) { + if (!series || typeof series !== "object" || !parseIsoDate(series.liveSince) + || (series.firstCoveredOn !== null && !parseIsoDate(series.firstCoveredOn)) + || (series.lastSuccessfulOn !== null && !parseIsoDate(series.lastSuccessfulOn)) + || !series.daily || typeof series.daily !== "object" || Array.isArray(series.daily)) { + throw new Error(`private metrics state for ${label} is invalid`); + } + for (const [date, value] of Object.entries(series.daily)) { + if (!parseIsoDate(date) || !value || !Number.isSafeInteger(value[totalKey]) + || value[totalKey] < 0 || !validCountMap(value.countries) + || sumValues(value.countries) !== value[totalKey]) { + throw new Error(`private metrics state value for ${label}/${date} is invalid`); + } + if (totalKey === "pageViews") { + if (!validCountMap(value.paths) || sumValues(value.paths) !== value[totalKey]) { + throw new Error(`private metrics path value for ${label}/${date} is invalid`); + } + } else if ((value.hours !== undefined || value.clients !== undefined) + && (!validCountMap(value.hours) || !validCountMap(value.clients) + || sumValues(value.hours) !== value[totalKey] + || sumValues(value.clients) !== value[totalKey])) { + throw new Error(`private metrics detail value for ${label}/${date} is invalid`); + } + } +} + +function ensureSiteState(state, site) { + const current = Object.hasOwn(state.sites, site.id) ? state.sites[site.id] : null; + if (current) { + validateStoredSeries(current.website, "pageViews", `${site.id}/website`); + validateStoredSeries(current.updateInfo, "requests", `${site.id}/update information`); + if (current.hostname !== site.hostname + || current.website.liveSince !== site.websiteMetricsSince + || current.updateInfo.liveSince !== site.updateInfo.metricsSince) { + throw new Error(`private metrics identity for ${site.id} differs from existing state`); + } + } + if (!current) { + state.sites[site.id] = { + hostname: site.hostname, + website: { liveSince: site.websiteMetricsSince, firstCoveredOn: null, lastSuccessfulOn: null, daily: {} }, + updateInfo: { liveSince: site.updateInfo.metricsSince, firstCoveredOn: null, lastSuccessfulOn: null, daily: {} } + }; + } + return state.sites[site.id]; +} + +function liveDates(series, today, historicalBridge = null) { + // Re-read the last successful day as well. The first run after midnight must + // still pick up requests which arrived after the final run of the previous day. + const start = series.lastSuccessfulOn || series.liveSince; + const oldestAvailable = shiftDate(today, -1); + if (start < oldestAvailable) { + const dayBeforeHandover = shiftDate(oldestAvailable, -1); + const bridgeIsComplete = historicalBridge + && historicalBridge.coverageFrom <= start + && historicalBridge.coverageThrough >= dayBeforeHandover; + if (!bridgeIsComplete) { + throw new Error( + `private metrics gap begins ${start}; regular current/.1 processing starts no earlier than ${oldestAvailable}` + ); + } + return dateRange(oldestAvailable, today); + } + return dateRange(start > today ? today : start, today); +} + +function generatePrivateMetrics({ registry, analyticsLogsBySite, updateLogsBySite, options, context, visitState }) { + const state = loadState(registry.privateMetrics.statePath); + const today = localParts(context.now, registry.privateMetrics.timeZone).date; + const imported = validateImportOptions(options, registry); + if (imported && imported.coverageThrough > today) { + throw new Error("historical import coverage must not extend into the future"); + } + + for (const site of registry.sites) { + const siteState = ensureSiteState(state, site); + if (site.websiteMetricsSince > today || site.updateInfo.metricsSince > today) { + throw new Error(`private metrics live coverage for ${site.id} must not begin in the future`); + } + const historicalBridge = imported && imported.site.id === site.id ? imported : null; + const websiteDates = liveDates(siteState.website, today, historicalBridge); + const updateDates = liveDates(siteState.updateInfo, today, historicalBridge); + const websiteRecords = parseLogFiles( + analyticsLogsBySite.get(site.id), + line => parseAnalyticsLine(line, registry.privateMetrics.timeZone), + { label: "website analytics log" } + ).filter(isEligibleWebsiteRequest); + const updateRecords = parseLogFiles( + updateLogsBySite.get(site.id), + line => parseAnalyticsLine(line, registry.privateMetrics.timeZone), + { label: "update-information log" } + ).filter(isUpdateRequest); + mergeSeries(siteState.website.daily, aggregateDays({ + records: websiteRecords, + dates: websiteDates, + kind: "website", + site, + context + }), "pageViews", "website"); + mergeSeries(siteState.updateInfo.daily, aggregateDays({ + records: updateRecords, + dates: updateDates, + kind: "updateInfo", + site, + context + }), "requests", "update information"); + for (const series of [siteState.website, siteState.updateInfo]) { + series.firstCoveredOn = !series.firstCoveredOn || series.liveSince < series.firstCoveredOn ? series.liveSince : series.firstCoveredOn; + series.lastSuccessfulOn = today; + } + } + + if (imported) { + const parser = imported.importFormat === "nginx-combined" + ? line => parseCombinedLine(line, imported.site.hostname, registry.privateMetrics.timeZone) + : line => parseAnalyticsLine(line, registry.privateMetrics.timeZone); + const allRecords = parseLogFiles(imported.importLogs, parser, { deduplicateAcrossFiles: true, label: "historical import log" }); + const dates = dateRange(imported.coverageFrom, imported.coverageThrough); + const siteState = state.sites[imported.site.id]; + const website = aggregateDays({ records: allRecords.filter(isEligibleWebsiteRequest), dates, kind: "website", site: imported.site, context }); + const updates = aggregateDays({ records: allRecords.filter(isUpdateRequest), dates, kind: "updateInfo", site: imported.site, context }); + mergeSeries(siteState.website.daily, website, "pageViews", "website import"); + mergeSeries(siteState.updateInfo.daily, updates, "requests", "update-information import"); + for (const series of [siteState.website, siteState.updateInfo]) { + series.firstCoveredOn = !series.firstCoveredOn || imported.coverageFrom < series.firstCoveredOn + ? imported.coverageFrom : series.firstCoveredOn; + } + } + + for (const site of Object.values(state.sites)) { + purgeDetails(site, today, registry.privateMetrics.detailRetentionDays); + } + return { + state, + files: renderReports(state, registry, context.now, visitState), + imported: Boolean(imported) + }; +} + +module.exports = { + UPDATE_INFO_PATH, + aggregateGoAccessReport, + clientGroup, + escapeHtml, + generatePrivateMetrics, + isEligibleWebsiteRequest, + isUpdateRequest, + localParts, + mergeDay, + normalizePath, + parseAnalyticsLine, + parseCombinedLine, + parseLogFiles, + purgeDetails, + renderReports, + validateImportOptions +}; diff --git a/website/ops/analytics/sites.example.json b/website/ops/analytics/sites.example.json index eb977011..2065e8b0 100644 --- a/website/ops/analytics/sites.example.json +++ b/website/ops/analytics/sites.example.json @@ -4,6 +4,12 @@ "counterStatePath": "/var/lib/hamradioonline-analytics/public-counter-state.json", "lockFile": "/run/hamradioonline-analytics/generator.lock", "geoIpCountryDatabase": "/var/lib/GeoIP/GeoLite2-Country.mmdb", + "privateMetrics": { + "statePath": "/var/lib/hamradioonline-analytics/private-metrics-state.json", + "reportOutputDirectory": "/var/lib/hamradioonline-analytics/reports/metrics", + "timeZone": "Europe/Berlin", + "detailRetentionDays": 14 + }, "combined": { "reportOutputDirectory": "/var/lib/hamradioonline-analytics/reports/combined" }, @@ -13,6 +19,12 @@ "hostname": "kst4contest.hamradioonline.de", "analyticsLog": "/var/log/nginx/kst4contest-analytics.log", "activatedOn": "2026-09-11", + "websiteMetricsSince": "2026-09-14", + "updateInfo": { + "path": "/kst4ContestVersionInfo.xml", + "analyticsLog": "/var/log/nginx/kst4contest-update-information.log", + "metricsSince": "2026-09-14" + }, "publicCounter": true, "reportOutputDirectory": "/var/lib/hamradioonline-analytics/reports/kst4contest", "publicJsonPath": "/var/lib/hamradioonline-analytics/public/kst4contest/visitor-count.json" diff --git a/website/ops/analytics/systemd/hamradioonline-analytics.service.example b/website/ops/analytics/systemd/hamradioonline-analytics.service.example index 91bb834e..e6b7c7fd 100644 --- a/website/ops/analytics/systemd/hamradioonline-analytics.service.example +++ b/website/ops/analytics/systemd/hamradioonline-analytics.service.example @@ -1,5 +1,5 @@ [Unit] -Description=Generate private GoAccess reports and public project counters +Description=Generate private analytics reports and public project counters After=nginx.service [Service] diff --git a/website/src/privacy/index.njk b/website/src/privacy/index.njk index 6b45e20a..52a1f0dd 100644 --- a/website/src/privacy/index.njk +++ b/website/src/privacy/index.njk @@ -47,16 +47,37 @@ description: Privacy policy for the KST4Contest website. browsers or operating systems are treated as crawlers.

- IP addresses are anonymised before data is stored in the statistics aggregates. The dedicated - analytics raw logs are retained for 14 days. Anonymised detailed aggregates are retained on a - rolling basis for 395 days. They include countries, but no city or host statistics. Countries - are derived with a Country database only. + IP addresses are anonymised before data is stored in the GoAccess statistics aggregates. The + dedicated analytics raw logs are retained for 14 days. Anonymised GoAccess detail is retained + on a rolling basis for 395 days. Daily totals for visits and page views, absolute country + totals per day and page views for each normalised path are retained permanently. No permanent + table combines an individual page view with both its path and country.

A visit is an approximate daily value based on the combination of IP address, date and user agent used by GoAccess. It must not be read as a number of unique people. Page views and visits - remain separate measures. Bots, monitoring requests, software update checks and static assets - are excluded as far as they can be identified. + remain separate measures. Known bots, monitoring requests, software update checks and static + assets are excluded from the website page and visit statistics as far as they can be identified. + Query strings are not included in the recorded page paths. +

+ +

Update-information statistics

+

+ Successful GET requests with status 200 for the exact path + /kst4ContestVersionInfo.xml are recorded separately from the website statistics. + They do not increase its page views or the public visitor count. Daily totals and absolute + country totals per day are retained permanently. Hourly totals and recognisable client groups + derived from the user agent are retained for no more than 14 days. +

+

+ Browsers and bots can request the update-information file too. Existing KST4Contest versions + do not send a reliable application-specific user agent. The resulting number therefore does + not establish a program start, a single user or a particular person. +

+

+ Country information for both statistics is derived locally from the GeoLite2-Country database. + No visitor address is sent to an external location service. Country assignment is approximate; + it is not an exact location determination.

Public visitor count

diff --git a/website/test/analytics-generator.test.js b/website/test/analytics-generator.test.js index f9114610..9552c8ca 100644 --- a/website/test/analytics-generator.test.js +++ b/website/test/analytics-generator.test.js @@ -7,6 +7,7 @@ const test = require("node:test"); const { formatGoAccessCheck, generateReports, + parseArguments, parseGoAccessVersion, validateReport } = require("../ops/analytics/generate-reports"); @@ -37,6 +38,18 @@ function goAccessReport(dailyVisits, combined = false) { return report; } +function analyticsLine({ + host = "alpha.example.test", + ip = "192.0.2.1", + timestamp = "2026-09-11T09:00:00+02:00", + method = "GET", + requestPath = "/", + status = 200, + userAgent = "Mozilla/5.0 Firefox/130.0" +} = {}) { + return `${host}\t${ip}\t${timestamp}\t${method}\t${requestPath}\tHTTP/1.1\t${status}\t123\t"${userAgent}"\n`; +} + function fixture(siteDefinitions) { const root = fs.mkdtempSync(path.join(os.tmpdir(), "kst4-analytics-test-")); const stateDirectory = path.join(root, "state"); @@ -52,9 +65,25 @@ function fixture(siteDefinitions) { const sites = siteDefinitions.map((definition, index) => { const id = definition.id || `site-${index}`; const log = path.join(root, `${id}.log`); - fs.writeFileSync(log, "example log line\n"); + const updateLog = path.join(root, `${id}-update-information.log`); + fs.writeFileSync(log, analyticsLine({ host: definition.hostname || `${id}.example.test` })); + fs.writeFileSync(updateLog, analyticsLine({ + host: definition.hostname || `${id}.example.test`, + requestPath: "/kst4ContestVersionInfo.xml", + userAgent: "Java/21.0.1" + })); if (definition.rotated !== false) { - fs.writeFileSync(`${log}.1`, "rotated example log line\n"); + fs.writeFileSync(`${log}.1`, analyticsLine({ + host: definition.hostname || `${id}.example.test`, + timestamp: "2026-09-11T08:00:00+02:00", + requestPath: "/privacy/" + })); + fs.writeFileSync(`${updateLog}.1`, analyticsLine({ + host: definition.hostname || `${id}.example.test`, + timestamp: "2026-09-11T08:00:00+02:00", + requestPath: "/kst4ContestVersionInfo.xml", + userAgent: "Mozilla/5.0 Firefox/130.0" + })); } if (definition.compressed) { fs.writeFileSync(`${log}.2.gz`, "compressed placeholder\n"); @@ -64,6 +93,12 @@ function fixture(siteDefinitions) { hostname: definition.hostname || `${id}.example.test`, analyticsLog: log, activatedOn: definition.activatedOn || "2026-01-01", + websiteMetricsSince: "2026-09-11", + updateInfo: { + path: "/kst4ContestVersionInfo.xml", + analyticsLog: updateLog, + metricsSince: "2026-09-11" + }, publicCounter: definition.publicCounter, reportOutputDirectory: path.join(stateDirectory, "reports", id), ...(definition.publicCounter @@ -77,6 +112,12 @@ function fixture(siteDefinitions) { counterStatePath: path.join(stateDirectory, "counter-state.json"), lockFile: path.join(root, "run", "generator.lock"), geoIpCountryDatabase, + privateMetrics: { + statePath: path.join(stateDirectory, "private-metrics-state.json"), + reportOutputDirectory: path.join(stateDirectory, "reports", "metrics"), + timeZone: "Europe/Berlin", + detailRetentionDays: 14 + }, combined: { reportOutputDirectory: path.join(stateDirectory, "reports", "combined") }, @@ -84,6 +125,7 @@ function fixture(siteDefinitions) { }; fs.mkdirSync(stateDirectory, { recursive: true }); fs.mkdirSync(registry.combined.reportOutputDirectory, { recursive: true }); + fs.mkdirSync(registry.privateMetrics.reportOutputDirectory, { recursive: true }); for (const site of sites) { fs.mkdirSync(site.reportOutputDirectory, { recursive: true }); if (site.publicCounter) { @@ -107,6 +149,26 @@ function fakeGoAccess(reports, calls, failureId) { if (invocation.id === failureId) { throw new Error("simulated GoAccess failure"); } + if (invocation.metricKind) { + const lines = fs.readFileSync(invocation.args[0], "utf8").trim().split(/\r?\n/); + const paths = {}; + for (const line of lines) { + const requestPath = line.split("\t")[4]; + paths[requestPath] = (paths[requestPath] || 0) + 1; + } + const report = { + general: { total_requests: lines.length }, + requests: { + data: Object.entries(paths).map(([requestPath, count]) => ({ + data: requestPath, + hits: { count } + })) + }, + geolocation: { data: [{ data: "Test Country", hits: { count: lines.length } }] } + }; + fs.writeFileSync(invocation.outputJson, JSON.stringify(report)); + return; + } fs.writeFileSync( invocation.outputJson, JSON.stringify(reports[invocation.id]) @@ -205,6 +267,42 @@ test("keeps valid outputs unchanged when a GoAccess job fails", () => { } }); +test("keeps all published outputs unchanged when private metric generation fails", () => { + const testFixture = fixture([{ id: "alpha", publicCounter: true }]); + const site = testFixture.registry.sites[0]; + const metricReport = path.join(testFixture.registry.privateMetrics.reportOutputDirectory, "report.html"); + const normalRunner = fakeGoAccess({ + alpha: goAccessReport({ "2026-09-11": 3 }), + combined: goAccessReport({ "2026-09-11": 3 }, true) + }, []); + try { + fs.writeFileSync(path.join(site.reportOutputDirectory, "report.html"), "old-html"); + fs.writeFileSync(site.publicJsonPath, "old-public"); + fs.writeFileSync(metricReport, "old-metrics"); + + assert.throws(() => generateReports({ + registryPath: testFixture.registryPath, + configTemplatePath: testFixture.configTemplatePath, + goaccessBinary: "fake-goaccess" + }, { + checkGoAccess: () => GOACCESS_WITHOUT_ZLIB, + runGoAccess: invocation => { + if (invocation.metricKind) throw new Error("simulated private metric failure"); + normalRunner(invocation); + }, + now: () => new Date("2026-09-11T07:00:00Z"), + skipLock: true + }), /simulated private metric failure/); + + assert.equal(fs.readFileSync(path.join(site.reportOutputDirectory, "report.html"), "utf8"), "old-html"); + assert.equal(fs.readFileSync(site.publicJsonPath, "utf8"), "old-public"); + assert.equal(fs.readFileSync(metricReport, "utf8"), "old-metrics"); + assert.equal(fs.existsSync(testFixture.registry.privateMetrics.statePath), false); + } finally { + testFixture.cleanup(); + } +}); + test("processes subdomains separately and together without publishing disabled counters", () => { const testFixture = fixture([ { id: "alpha", publicCounter: true, compressed: true }, @@ -219,22 +317,23 @@ test("processes subdomains separately and together without publishing disabled c }, calls); assert.equal(result.reports, 3); - assert.deepEqual(calls.map(call => call.id), ["alpha", "bravo", "combined"]); + const reportCalls = calls.filter(call => !call.metricKind); + assert.deepEqual(reportCalls.map(call => call.id), ["alpha", "bravo", "combined"]); const alphaLogs = [ `${testFixture.registry.sites[0].analyticsLog}.1`, testFixture.registry.sites[0].analyticsLog ]; const bravoLogs = [testFixture.registry.sites[1].analyticsLog]; - assert.deepEqual(calls[0].args.slice(0, 2), alphaLogs); - assert.deepEqual(calls[1].args.slice(0, 1), bravoLogs); + assert.deepEqual(reportCalls[0].args.slice(0, 2), alphaLogs); + assert.deepEqual(reportCalls[1].args.slice(0, 1), bravoLogs); assert.deepEqual( - calls[2].args.slice(0, 3), + reportCalls[2].args.slice(0, 3), [...alphaLogs, ...bravoLogs] ); - assert.equal(calls[0].args.includes("--enable-panel=VIRTUAL_HOSTS"), false); - assert.equal(calls[1].args.includes("--enable-panel=VIRTUAL_HOSTS"), false); - assert.equal(calls[2].args.includes("--enable-panel=VIRTUAL_HOSTS"), true); - assert.equal(calls.some(call => call.args.some(argument => argument.endsWith(".gz"))), false); + assert.equal(reportCalls[0].args.includes("--enable-panel=VIRTUAL_HOSTS"), false); + assert.equal(reportCalls[1].args.includes("--enable-panel=VIRTUAL_HOSTS"), false); + assert.equal(reportCalls[2].args.includes("--enable-panel=VIRTUAL_HOSTS"), true); + assert.equal(reportCalls.some(call => call.args.some(argument => argument.endsWith(".gz"))), false); assert.equal(fs.existsSync(path.join( testFixture.registry.stateDirectory, "public", @@ -244,6 +343,16 @@ test("processes subdomains separately and together without publishing disabled c testFixture.registry.combined.reportOutputDirectory, "report.html" )), true); + assert.match(fs.readFileSync(path.join( + testFixture.registry.combined.reportOutputDirectory, + "report.html" + ), "utf8"), /href="\/metrics\/"/); + const privateState = JSON.parse(fs.readFileSync( + testFixture.registry.privateMetrics.statePath, + "utf8" + )); + assert.equal(privateState.sites.alpha.website.daily["2026-09-11"].pageViews, 2); + assert.equal(privateState.sites.alpha.updateInfo.daily["2026-09-11"].requests, 2); } finally { testFixture.cleanup(); } @@ -302,6 +411,7 @@ test("dry-run validates generated data without changing production paths", () => ).length, 0); assert.equal(fs.existsSync(testFixture.registry.sites[0].publicJsonPath), false); assert.equal(fs.existsSync(testFixture.registry.counterStatePath), false); + assert.equal(fs.existsSync(testFixture.registry.privateMetrics.statePath), false); } finally { testFixture.cleanup(); } @@ -402,6 +512,22 @@ test("configuration check verifies analytics-log readability", () => { } }); +test("configuration check rejects a missing update-information log", () => { + const testFixture = fixture([{ id: "alpha", publicCounter: true }]); + try { + fs.rmSync(testFixture.registry.sites[0].updateInfo.analyticsLog); + assert.throws(() => generateReports({ + registryPath: testFixture.registryPath, + configTemplatePath: testFixture.configTemplatePath, + check: true + }, { + checkGoAccess: () => GOACCESS_WITHOUT_ZLIB + }), /analytics log is not readable/); + } finally { + testFixture.cleanup(); + } +}); + test("configuration check reports missing output directories", () => { const testFixture = fixture([{ id: "alpha", publicCounter: true }]); try { @@ -462,6 +588,118 @@ test("Nginx filters exclude non-page traffic before analytics logging", () => { assert.match(source, /\|map\|/); assert.match(source, /\$uri/); assert.match(source, /known_bot/); + assert.match(source, /map "\$request_method:\$uri:\$status" \$hamradioonline_update_information_loggable/); + assert.match(source, /"GET:\/kst4ContestVersionInfo\.xml:200" 1;/); +}); + +test("historical import is repeatable and overlaps live aggregation without addition", () => { + const testFixture = fixture([{ id: "alpha", publicCounter: true }]); + const importPath = path.join(testFixture.root, "access.log.2.gz"); + const historical = [ + '192.0.2.10 - - [10/Sep/2026:12:00:00 +0200] "GET /privacy/?x=1 HTTP/1.1" 200 42 "-" "Firefox/130"', + '192.0.2.11 - - [10/Sep/2026:12:01:00 +0200] "GET /kst4ContestVersionInfo.xml HTTP/1.1" 200 43 "-" "Java/21.0.1"' + ].join("\n"); + const reports = { + alpha: goAccessReport({ "2026-09-11": 3 }), + combined: goAccessReport({ "2026-09-11": 3 }, true) + }; + try { + testFixture.registry.sites[0].websiteMetricsSince = "2026-09-08"; + testFixture.registry.sites[0].updateInfo.metricsSince = "2026-09-08"; + fs.writeFileSync(testFixture.registryPath, JSON.stringify(testFixture.registry)); + fs.writeFileSync(importPath, require("node:zlib").gzipSync(`${historical}\n`)); + const options = { + registryPath: testFixture.registryPath, + configTemplatePath: testFixture.configTemplatePath, + goaccessBinary: "fake-goaccess", + importLogs: [importPath], + importFormat: "nginx-combined", + importSite: "alpha", + coverageFrom: "2026-09-08", + coverageThrough: "2026-09-10" + }; + const dependencies = { + checkGoAccess: () => GOACCESS_WITHOUT_ZLIB, + runGoAccess: fakeGoAccess(reports, []), + now: () => new Date("2026-09-11T07:00:00Z"), + skipLock: true + }; + + generateReports(options, dependencies); + generateReports(options, dependencies); + + const state = JSON.parse(fs.readFileSync(testFixture.registry.privateMetrics.statePath, "utf8")); + assert.equal(state.sites.alpha.website.daily["2026-09-10"].pageViews, 1); + assert.equal(state.sites.alpha.updateInfo.daily["2026-09-10"].requests, 1); + assert.equal(state.sites.alpha.website.daily["2026-09-11"].pageViews, 2); + assert.equal(state.sites.alpha.updateInfo.daily["2026-09-11"].requests, 2); + assert.equal(state.sites.alpha.website.firstCoveredOn, "2026-09-08"); + assert.equal(state.sites.alpha.updateInfo.firstCoveredOn, "2026-09-08"); + assert.equal(fs.existsSync(path.join( + testFixture.registry.privateMetrics.reportOutputDirectory, + "alpha", "updates", "yearly", "report.html" + )), true); + } finally { + testFixture.cleanup(); + } +}); + +test("first run after midnight refreshes the previous day without double counting", () => { + const testFixture = fixture([{ id: "alpha", publicCounter: true }]); + const reports = { + alpha: goAccessReport({ "2026-09-11": 3 }), + combined: goAccessReport({ "2026-09-11": 3 }, true) + }; + const baseOptions = { + registryPath: testFixture.registryPath, + configTemplatePath: testFixture.configTemplatePath, + goaccessBinary: "fake-goaccess" + }; + try { + generateReports(baseOptions, { + checkGoAccess: () => GOACCESS_WITHOUT_ZLIB, + runGoAccess: fakeGoAccess(reports, []), + now: () => new Date("2026-09-11T20:00:00Z"), + skipLock: true + }); + fs.appendFileSync(testFixture.registry.sites[0].analyticsLog, analyticsLine({ + timestamp: "2026-09-11T23:59:00+02:00", + requestPath: "/news/" + })); + generateReports(baseOptions, { + checkGoAccess: () => GOACCESS_WITHOUT_ZLIB, + runGoAccess: fakeGoAccess(reports, []), + now: () => new Date("2026-09-12T00:30:00Z"), + skipLock: true + }); + + const state = JSON.parse(fs.readFileSync(testFixture.registry.privateMetrics.statePath, "utf8")); + assert.equal(state.sites.alpha.website.daily["2026-09-11"].pageViews, 3); + assert.equal(state.sites.alpha.website.daily["2026-09-12"].pageViews, 0); + } finally { + testFixture.cleanup(); + } +}); + +test("historical import CLI requires explicit format, site and coverage", () => { + const options = parseArguments([ + "--registry", "/etc/hamradioonline-analytics/sites.json", + "--config-template", "/etc/hamradioonline-analytics/goaccess.conf.template", + "--import-site", "kst4contest", + "--import-format", "nginx-combined", + "--coverage-from", "2026-09-01", + "--coverage-through", "2026-09-10", + "--import-log", "/protected/access.log.2.gz", + "--import-log", "/protected/access.log.1" + ]); + assert.deepEqual(options.importLogs, [ + "/protected/access.log.2.gz", + "/protected/access.log.1" + ]); + assert.equal(options.importFormat, "nginx-combined"); + assert.equal(options.importSite, "kst4contest"); + assert.equal(options.coverageFrom, "2026-09-01"); + assert.equal(options.coverageThrough, "2026-09-10"); }); test("server templates use the production GeoIP path and permission model", () => { @@ -536,6 +774,7 @@ test("logrotate uses the Ubuntu Nginx rotation action", () => { assert.match(source, /^\s*delaycompress$/m); assert.match(source, /^\s*rotate 14$/m); assert.match(source, /^\s*create 0640 www-data hamradio-analytics$/m); + assert.match(source, /\*-update-information\.log/); }); test("website package records the Node.js 18.19.1 baseline", () => { diff --git a/website/test/private-metrics.test.js b/website/test/private-metrics.test.js new file mode 100644 index 00000000..3103b420 --- /dev/null +++ b/website/test/private-metrics.test.js @@ -0,0 +1,260 @@ +const assert = require("node:assert/strict"); +const fs = require("node:fs"); +const os = require("node:os"); +const path = require("node:path"); +const test = require("node:test"); +const zlib = require("node:zlib"); + +const { + aggregateGoAccessReport, + clientGroup, + isEligibleWebsiteRequest, + isUpdateRequest, + localParts, + mergeDay, + normalizePath, + parseAnalyticsLine, + parseCombinedLine, + parseLogFiles, + purgeDetails, + renderReports +} = require("../ops/analytics/private-metrics"); + +function record(overrides = {}) { + return { + method: "GET", + status: 200, + path: "/kst4ContestVersionInfo.xml", + userAgent: "Java/21.0.1", + ...overrides + }; +} + +test("counts only exact successful GET requests for the update-information path", () => { + assert.equal(isUpdateRequest(record()), true); + assert.equal(isUpdateRequest(record({ method: "HEAD" })), false); + assert.equal(isUpdateRequest(record({ status: 304 })), false); + assert.equal(isUpdateRequest(record({ status: 404 })), false); + assert.equal(isUpdateRequest(record({ path: "/kst4ContestVersionInfo.xml/" })), false); + assert.equal(isUpdateRequest(record({ path: "/Kst4ContestVersionInfo.xml" })), false); +}); + +test("keeps website page and bot exclusions separate from update requests", () => { + assert.equal(isEligibleWebsiteRequest(record({ path: "/privacy/", userAgent: "Firefox/130" })), true); + assert.equal(isEligibleWebsiteRequest(record({ path: "/privacy/?source=test", userAgent: "Firefox/130" })), true); + assert.equal(isEligibleWebsiteRequest(record({ path: "/assets/site.css", userAgent: "Firefox/130" })), false); + assert.equal(isEligibleWebsiteRequest(record({ path: "/manual/assets/page.png", userAgent: "Firefox/130" })), false); + assert.equal(isEligibleWebsiteRequest(record({ path: "/privacy/", userAgent: "ExampleBot/1" })), false); + assert.equal(isEligibleWebsiteRequest(record()), false); + assert.equal(normalizePath("//docs/../privacy/?source=test"), "/privacy/"); +}); + +test("parses analytics and regular Nginx combined records without query strings", () => { + const analytics = parseAnalyticsLine( + 'kst4contest.hamradioonline.de\t192.0.2.1\t2026-09-14T23:30:00+02:00\tGET\t/privacy/?x=1\tHTTP/1.1\t200\t42\t"Firefox/130"' + ); + assert.equal(analytics.date, "2026-09-14"); + assert.equal(analytics.hour, "23"); + assert.equal(analytics.path, "/privacy/"); + + const combined = parseCombinedLine( + '192.0.2.2 - - [14/Sep/2026:23:31:00 +0200] "GET /news/?x=1 HTTP/1.1" 200 43 "-" "Mozilla/5.0 Firefox/130"', + "kst4contest.hamradioonline.de" + ); + assert.equal(combined.date, "2026-09-14"); + assert.equal(combined.path, "/news/"); + assert.match(combined.line, /\t\/news\/\t/); + assert.equal(parseCombinedLine("broken", "example.test"), null); +}); + +test("uses Europe/Berlin across both daylight-saving transitions", () => { + assert.deepEqual(localParts("2026-03-29T00:30:00Z", "Europe/Berlin"), { + date: "2026-03-29", hour: "01" + }); + assert.deepEqual(localParts("2026-03-29T01:30:00Z", "Europe/Berlin"), { + date: "2026-03-29", hour: "03" + }); + assert.deepEqual(localParts("2026-10-25T00:30:00Z", "Europe/Berlin"), { + date: "2026-10-25", hour: "02" + }); + assert.deepEqual(localParts("2026-10-25T01:30:00Z", "Europe/Berlin"), { + date: "2026-10-25", hour: "02" + }); +}); + +test("keeps absolute countries including Switzerland, United Kingdom and unknown", () => { + const website = aggregateGoAccessReport({ + general: { total_requests: 5 }, + geolocation: { data: [ + { data: "Europe", hits: { count: 3 }, items: [ + { data: "Germany", hits: { count: 1 } }, + { data: "Switzerland", hits: { count: 1 } }, + { data: "United Kingdom", hits: { count: 1 } } + ] }, + { data: "Unknown", hits: { count: 1 } } + ] }, + requests: { data: [ + { data: "/", hits: { count: 2 } }, + { data: "/privacy/", hits: { count: 2 } }, + { data: "/news/", hits: { count: 1 } } + ] } + }, "website"); + assert.deepEqual(website.countries, { + Germany: 1, + Switzerland: 1, + "United Kingdom": 1, + Unknown: 2 + }); + assert.deepEqual(website.paths, { "/": 2, "/privacy/": 2, "/news/": 1 }); + assert.throws(() => aggregateGoAccessReport({ + general: { total_requests: 2 }, + geolocation: { data: [{ data: "Germany", hits: { count: 2 } }] }, + requests: { data: [{ data: "/", hits: { count: 1 } }] } + }, "website"), /differs from the GoAccess request-panel definition/); +}); + +test("deduplicates overlapping import files and reads gzip without GoAccess Zlib", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "kst4-private-import-")); + const line = '192.0.2.2 - - [14/Sep/2026:23:31:00 +0200] "GET /news/ HTTP/1.1" 200 43 "-" "Firefox/130"'; + const plain = path.join(root, "access.log.1"); + const compressed = path.join(root, "access.log.2.gz"); + try { + fs.writeFileSync(plain, `${line}\n${line}\n`); + fs.writeFileSync(compressed, zlib.gzipSync(`${line}\n`)); + const parsed = parseLogFiles( + [compressed, plain], + value => parseCombinedLine(value, "kst4contest.hamradioonline.de"), + { deduplicateAcrossFiles: true } + ); + assert.equal(parsed.length, 2); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test("rejects malformed and unreadable historical input", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "kst4-private-invalid-")); + const invalid = path.join(root, "access.log"); + const invalidGzip = path.join(root, "access.log.gz"); + try { + fs.writeFileSync(invalid, "not an access-log record\n"); + fs.writeFileSync(invalidGzip, "not gzip"); + assert.throws(() => parseLogFiles( + [invalid], + value => parseCombinedLine(value, "kst4contest.hamradioonline.de"), + { label: "historical import log" } + ), /invalid line/); + assert.throws(() => parseLogFiles( + [invalidGzip], + value => parseCombinedLine(value, "kst4contest.hamradioonline.de") + ), /could not read compressed log/); + assert.throws(() => parseLogFiles( + [path.join(root, "missing.log")], + value => parseCombinedLine(value, "kst4contest.hamradioonline.de") + )); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test("merges repeated and overlapping daily aggregates without addition", () => { + const current = { pageViews: 2, countries: { Germany: 2 }, paths: { "/": 2 } }; + assert.deepEqual(mergeDay(current, { ...current }, "pageViews", "test"), current); + assert.deepEqual(mergeDay({ + pageViews: 2, + countries: { Germany: 1, Switzerland: 1 }, + paths: { "/": 1, "/privacy/": 1 } + }, { + pageViews: 2, + countries: { Switzerland: 1, Germany: 1 }, + paths: { "/privacy/": 1, "/": 1 } + }, "pageViews", "test").pageViews, 2); + assert.deepEqual(mergeDay(current, { + pageViews: 3, + countries: { Germany: 2, Switzerland: 1 }, + paths: { "/": 2, "/privacy/": 1 } + }, "pageViews", "test"), { + pageViews: 3, + countries: { Germany: 2, Switzerland: 1 }, + paths: { "/": 2, "/privacy/": 1 } + }); + assert.throws(() => mergeDay(current, { + pageViews: 3, + countries: { Germany: 1, Switzerland: 2 }, + paths: { "/": 1, "/privacy/": 2 } + }, "pageViews", "test"), /not monotonic/); +}); + +test("removes hourly and client detail after 14 days but preserves daily totals", () => { + const site = { + updateInfo: { + daily: { + "2026-08-31": { requests: 2, countries: { Germany: 2 }, hours: { "10": 2 }, clients: { Java: 2 } }, + "2026-09-01": { requests: 3, countries: { Germany: 3 }, hours: { "11": 3 }, clients: { Java: 3 } }, + "2026-09-14": { requests: 1, countries: { Unknown: 1 }, hours: { "12": 1 }, clients: { Other: 1 } } + } + } + }; + purgeDetails(site, "2026-09-14", 14); + assert.equal(site.updateInfo.daily["2026-08-31"].requests, 2); + assert.equal(site.updateInfo.daily["2026-08-31"].hours, undefined); + assert.deepEqual(site.updateInfo.daily["2026-09-01"].hours, { "11": 3 }); + assert.deepEqual(site.updateInfo.daily["2026-09-14"].clients, { Other: 1 }); +}); + +test("renders annual sums and escapes all dynamic report labels", () => { + const root = path.join(os.tmpdir(), "metrics-report-output"); + const state = { sites: { alpha: { + hostname: "alpha.example.test", + website: { + firstCoveredOn: "2025-12-31", + daily: { + "2025-12-31": { pageViews: 2, countries: { Germany: 2 }, paths: { "/": 2 } }, + "2026-01-01": { pageViews: 3, countries: { Switzerland: 3 }, paths: { "/": 1 } + } + } + } + } } }; + const registry = { + privateMetrics: { reportOutputDirectory: root }, + sites: [{ id: "alpha", hostname: "alpha.example.test" }] + }; + const files = renderReports(state, registry, new Date("2026-01-02T00:00:00Z")); + const html = files.map(file => file.content).join("\n"); + const websiteAnnual = files.find(file => file.destination.endsWith( + path.join("website", "yearly", "report.html") + )).content; + assert.match(html, /2025/); + assert.match(html, /2026/); + assert.match(html, /Switzerland/); + assert.doesNotMatch(html, /"), "Other or unrecognised"); +}); + +test("privacy notice distinguishes page statistics from private update aggregation", () => { + const privacy = fs.readFileSync(path.join(__dirname, "../src/privacy/index.njk"), "utf8"); + assert.match(privacy, /exact path\s*\/kst4ContestVersionInfo\.xml<\/code>/); + assert.match(privacy, /do not increase its page views or the public visitor count/); + assert.match(privacy, /retained for no more than 14 days/); + assert.match(privacy, /does\s+not establish a program start, a single user/); + assert.match(privacy, /No visitor address is sent to an external location service/); +});