From 914a06cb7fd19a80c4688fa99b825ce6ad4e080e Mon Sep 17 00:00:00 2001 From: Marc Froehlich Date: Sat, 12 Sep 2026 03:55:27 +0200 Subject: [PATCH] Corrections on user counter and analytics to the kst4contest page --- docs/PROJECT_CONTEXT.md | 1 + website/ops/analytics/README.md | 6 ++++ website/ops/analytics/generate-reports.js | 13 ++++++--- website/test/analytics-generator.test.js | 34 +++++++++++++++++++++-- 4 files changed, 47 insertions(+), 7 deletions(-) diff --git a/docs/PROJECT_CONTEXT.md b/docs/PROJECT_CONTEXT.md index 37921610..3eb06475 100644 --- a/docs/PROJECT_CONTEXT.md +++ b/docs/PROJECT_CONTEXT.md @@ -185,6 +185,7 @@ Current website/deployment scripts and update-feed behaviour must be inspected b - The counter endpoint disables its own access log and serves the public JSON with a one-hour public cache policy and `X-Content-Type-Options: nosniff`. - GoAccess is the server-side source. A registry keeps stable site IDs, hostnames, current analytics-log paths, activation dates, public-counter switches and output targets separate for each project subdomain. The Country database is `/var/lib/GeoIP/GeoLite2-Country.mmdb`. A combined report uses only the registered project sites; `stats.hamradioonline.de` is excluded. - The regular generator passes each current analytics log and its optional uncompressed `.1` rotation directly to GoAccess and relies on the persistent GoAccess database for incremental processing. Logrotate therefore uses `delaycompress`. Older `.gz` rotations are not imported during regular runs, and missing Zlib support is an accepted, explicitly reported capability state for the GoAccess 1.8.1 production baseline. +- GoAccess 1.8.1 exposes the Country panel as `geolocation` in JSON. Combined report jobs explicitly enable `VIRTUAL_HOSTS` and require the resulting `vhosts` panel; individual site jobs do not enable it. Missing required panels invalidate the complete staged run. - Node.js 18.19.1 is the production runtime baseline. `--check` requires readable input files, prepared writable output directories and GoAccess built with GeoIP2/MMDB support. OpenSSL and absent Zlib support remain informational. Dry-runs use temporary state and never acquire the production lock. - The generator runs as `hamradio-analytics`. The state root is mode `0711`; only explicitly prepared report and public-output directories are shared read-only with Nginx through the `www-data` group. GoAccess databases and public counter state remain private. No ACL support is assumed. - The protected statistics vhost is enabled in two stages: an IPv4-only HTTP bootstrap obtains the certificate through `/snap/bin/certbot`, then the final configuration retains an IPv4 HTTP block for the webroot ACME challenge and permanently redirects all other HTTP requests to HTTPS. The HTTPS block uses the existing Certbot TLS options and redirects authenticated requests from `/` to `/combined/`. IPv6 remains disabled until the DNS AAAA record has been confirmed. diff --git a/website/ops/analytics/README.md b/website/ops/analytics/README.md index 389ba2d9..fea06f52 100644 --- a/website/ops/analytics/README.md +++ b/website/ops/analytics/README.md @@ -262,6 +262,12 @@ day, requested pages, countries, HTTP status codes and virtual hosts. Host, remote-user, referrer, keyphrase, operating-system, browser and other detailed panels are disabled. +GoAccess 1.8.1 writes the Country panel under the JSON key `geolocation`. +Combined jobs explicitly pass `--enable-panel=VIRTUAL_HOSTS` and require the +resulting `vhosts` key. Site jobs do not enable that panel. The generator treats +either missing key as an invalid report rather than publishing incomplete +statistics. + The Country database is provided through the registry at `/var/lib/GeoIP/GeoLite2-Country.mmdb`. A file whose name contains `City` is rejected. Do not replace it with a City database merely because one happens to diff --git a/website/ops/analytics/generate-reports.js b/website/ops/analytics/generate-reports.js index eb16c6ab..cd86a6a6 100644 --- a/website/ops/analytics/generate-reports.js +++ b/website/ops/analytics/generate-reports.js @@ -450,9 +450,9 @@ function validateReport(report, combined) { if (!report || typeof report !== "object" || Array.isArray(report)) { throw new Error("GoAccess JSON report must be an object"); } - const requiredPanels = ["visitors", "requests", "status_codes", "geo_location"]; + const requiredPanels = ["visitors", "requests", "status_codes", "geolocation"]; if (combined) { - requiredPanels.push("virtual_hosts"); + requiredPanels.push("vhosts"); } if (!report.general || typeof report.general !== "object") { throw new Error("GoAccess JSON report has no general summary"); @@ -682,10 +682,15 @@ function prepareReport(job, context) { const args = [ ...job.logs, "--no-global-config", - "--config-file", runConfig, + "--config-file", runConfig + ]; + if (job.combined) { + args.push("--enable-panel=VIRTUAL_HOSTS"); + } + args.push( "--output", outputJson, "--output", outputHtml - ]; + ); context.runGoAccess({ binary: context.goaccessBinary, args, diff --git a/website/test/analytics-generator.test.js b/website/test/analytics-generator.test.js index 111f2df1..f9114610 100644 --- a/website/test/analytics-generator.test.js +++ b/website/test/analytics-generator.test.js @@ -7,7 +7,8 @@ const test = require("node:test"); const { formatGoAccessCheck, generateReports, - parseGoAccessVersion + parseGoAccessVersion, + validateReport } = require("../ops/analytics/generate-reports"); const GOACCESS_WITHOUT_ZLIB = { @@ -28,10 +29,10 @@ function goAccessReport(dailyVisits, combined = false) { }, requests: { data: [] }, status_codes: { data: [] }, - geo_location: { data: [] } + geolocation: { data: [] } }; if (combined) { - report.virtual_hosts = { data: [] }; + report.vhosts = { data: [] }; } return report; } @@ -230,6 +231,9 @@ test("processes subdomains separately and together without publishing disabled c calls[2].args.slice(0, 3), [...alphaLogs, ...bravoLogs] ); + assert.equal(calls[0].args.includes("--enable-panel=VIRTUAL_HOSTS"), false); + assert.equal(calls[1].args.includes("--enable-panel=VIRTUAL_HOSTS"), false); + assert.equal(calls[2].args.includes("--enable-panel=VIRTUAL_HOSTS"), true); assert.equal(calls.some(call => call.args.some(argument => argument.endsWith(".gz"))), false); assert.equal(fs.existsSync(path.join( testFixture.registry.stateDirectory, @@ -245,6 +249,30 @@ test("processes subdomains separately and together without publishing disabled c } }); +test("validates GoAccess 1.8.1 panel names strictly", () => { + const siteReport = goAccessReport({ "2026-09-11": 3 }); + const combinedReport = goAccessReport({ "2026-09-11": 3 }, true); + + assert.doesNotThrow(() => validateReport(siteReport, false)); + assert.doesNotThrow(() => validateReport(combinedReport, true)); + + const missingGeolocation = goAccessReport({ "2026-09-11": 3 }); + delete missingGeolocation.geolocation; + missingGeolocation.geo_location = { data: [] }; + assert.throws( + () => validateReport(missingGeolocation, false), + /GoAccess JSON report has no geolocation panel/ + ); + + const missingVhosts = goAccessReport({ "2026-09-11": 3 }, true); + delete missingVhosts.vhosts; + missingVhosts.virtual_hosts = { data: [] }; + assert.throws( + () => validateReport(missingVhosts, true), + /GoAccess JSON report has no vhosts panel/ + ); +}); + test("dry-run validates generated data without changing production paths", () => { const testFixture = fixture([{ id: "alpha", publicCounter: true }]); try {