Changes at the stats analytics

2026-09-16 06:05:29 +02:00 · 2026-09-14 20:33:50 +02:00
parent 393c9c49ee
commit a3f3bf8872
12 changed files with 1740 additions and 107 deletions
@@ -7,6 +7,7 @@ const test = require("node:test");
 const {
    formatGoAccessCheck,
    generateReports,
+    parseArguments,
    parseGoAccessVersion,
    validateReport
 } = require("../ops/analytics/generate-reports");
@@ -37,6 +38,18 @@ function goAccessReport(dailyVisits, combined = false) {
    return report;
 }

+function analyticsLine({
+    host = "alpha.example.test",
+    ip = "192.0.2.1",
+    timestamp = "2026-09-11T09:00:00+02:00",
+    method = "GET",
+    requestPath = "/",
+    status = 200,
+    userAgent = "Mozilla/5.0 Firefox/130.0"
+} = {}) {
+    return `${host}\t${ip}\t${timestamp}\t${method}\t${requestPath}\tHTTP/1.1\t${status}\t123\t"${userAgent}"\n`;
+}
+
 function fixture(siteDefinitions) {
    const root = fs.mkdtempSync(path.join(os.tmpdir(), "kst4-analytics-test-"));
    const stateDirectory = path.join(root, "state");
@@ -52,9 +65,25 @@ function fixture(siteDefinitions) {
    const sites = siteDefinitions.map((definition, index) => {
        const id = definition.id || `site-${index}`;
        const log = path.join(root, `${id}.log`);
-        fs.writeFileSync(log, "example log line\n");
+        const updateLog = path.join(root, `${id}-update-information.log`);
+        fs.writeFileSync(log, analyticsLine({ host: definition.hostname || `${id}.example.test` }));
+        fs.writeFileSync(updateLog, analyticsLine({
+            host: definition.hostname || `${id}.example.test`,
+            requestPath: "/kst4ContestVersionInfo.xml",
+            userAgent: "Java/21.0.1"
+        }));
        if (definition.rotated !== false) {
-            fs.writeFileSync(`${log}.1`, "rotated example log line\n");
+            fs.writeFileSync(`${log}.1`, analyticsLine({
+                host: definition.hostname || `${id}.example.test`,
+                timestamp: "2026-09-11T08:00:00+02:00",
+                requestPath: "/privacy/"
+            }));
+            fs.writeFileSync(`${updateLog}.1`, analyticsLine({
+                host: definition.hostname || `${id}.example.test`,
+                timestamp: "2026-09-11T08:00:00+02:00",
+                requestPath: "/kst4ContestVersionInfo.xml",
+                userAgent: "Mozilla/5.0 Firefox/130.0"
+            }));
        }
        if (definition.compressed) {
            fs.writeFileSync(`${log}.2.gz`, "compressed placeholder\n");
@@ -64,6 +93,12 @@ function fixture(siteDefinitions) {
            hostname: definition.hostname || `${id}.example.test`,
            analyticsLog: log,
            activatedOn: definition.activatedOn || "2026-01-01",
+            websiteMetricsSince: "2026-09-11",
+            updateInfo: {
+                path: "/kst4ContestVersionInfo.xml",
+                analyticsLog: updateLog,
+                metricsSince: "2026-09-11"
+            },
            publicCounter: definition.publicCounter,
            reportOutputDirectory: path.join(stateDirectory, "reports", id),
            ...(definition.publicCounter
@@ -77,6 +112,12 @@ function fixture(siteDefinitions) {
        counterStatePath: path.join(stateDirectory, "counter-state.json"),
        lockFile: path.join(root, "run", "generator.lock"),
        geoIpCountryDatabase,
+        privateMetrics: {
+            statePath: path.join(stateDirectory, "private-metrics-state.json"),
+            reportOutputDirectory: path.join(stateDirectory, "reports", "metrics"),
+            timeZone: "Europe/Berlin",
+            detailRetentionDays: 14
+        },
        combined: {
            reportOutputDirectory: path.join(stateDirectory, "reports", "combined")
        },
@@ -84,6 +125,7 @@ function fixture(siteDefinitions) {
    };
    fs.mkdirSync(stateDirectory, { recursive: true });
    fs.mkdirSync(registry.combined.reportOutputDirectory, { recursive: true });
+    fs.mkdirSync(registry.privateMetrics.reportOutputDirectory, { recursive: true });
    for (const site of sites) {
        fs.mkdirSync(site.reportOutputDirectory, { recursive: true });
        if (site.publicCounter) {
@@ -107,6 +149,26 @@ function fakeGoAccess(reports, calls, failureId) {
        if (invocation.id === failureId) {
            throw new Error("simulated GoAccess failure");
        }
+        if (invocation.metricKind) {
+            const lines = fs.readFileSync(invocation.args[0], "utf8").trim().split(/\r?\n/);
+            const paths = {};
+            for (const line of lines) {
+                const requestPath = line.split("\t")[4];
+                paths[requestPath] = (paths[requestPath] || 0) + 1;
+            }
+            const report = {
+                general: { total_requests: lines.length },
+                requests: {
+                    data: Object.entries(paths).map(([requestPath, count]) => ({
+                        data: requestPath,
+                        hits: { count }
+                    }))
+                },
+                geolocation: { data: [{ data: "Test Country", hits: { count: lines.length } }] }
+            };
+            fs.writeFileSync(invocation.outputJson, JSON.stringify(report));
+            return;
+        }
        fs.writeFileSync(
            invocation.outputJson,
            JSON.stringify(reports[invocation.id])
@@ -205,6 +267,42 @@ test("keeps valid outputs unchanged when a GoAccess job fails", () => {
    }
 });

+test("keeps all published outputs unchanged when private metric generation fails", () => {
+    const testFixture = fixture([{ id: "alpha", publicCounter: true }]);
+    const site = testFixture.registry.sites[0];
+    const metricReport = path.join(testFixture.registry.privateMetrics.reportOutputDirectory, "report.html");
+    const normalRunner = fakeGoAccess({
+        alpha: goAccessReport({ "2026-09-11": 3 }),
+        combined: goAccessReport({ "2026-09-11": 3 }, true)
+    }, []);
+    try {
+        fs.writeFileSync(path.join(site.reportOutputDirectory, "report.html"), "old-html");
+        fs.writeFileSync(site.publicJsonPath, "old-public");
+        fs.writeFileSync(metricReport, "old-metrics");
+
+        assert.throws(() => generateReports({
+            registryPath: testFixture.registryPath,
+            configTemplatePath: testFixture.configTemplatePath,
+            goaccessBinary: "fake-goaccess"
+        }, {
+            checkGoAccess: () => GOACCESS_WITHOUT_ZLIB,
+            runGoAccess: invocation => {
+                if (invocation.metricKind) throw new Error("simulated private metric failure");
+                normalRunner(invocation);
+            },
+            now: () => new Date("2026-09-11T07:00:00Z"),
+            skipLock: true
+        }), /simulated private metric failure/);
+
+        assert.equal(fs.readFileSync(path.join(site.reportOutputDirectory, "report.html"), "utf8"), "old-html");
+        assert.equal(fs.readFileSync(site.publicJsonPath, "utf8"), "old-public");
+        assert.equal(fs.readFileSync(metricReport, "utf8"), "old-metrics");
+        assert.equal(fs.existsSync(testFixture.registry.privateMetrics.statePath), false);
+    } finally {
+        testFixture.cleanup();
+    }
+});
+
 test("processes subdomains separately and together without publishing disabled counters", () => {
    const testFixture = fixture([
        { id: "alpha", publicCounter: true, compressed: true },
@@ -219,22 +317,23 @@ test("processes subdomains separately and together without publishing disabled c
        }, calls);

        assert.equal(result.reports, 3);
-        assert.deepEqual(calls.map(call => call.id), ["alpha", "bravo", "combined"]);
+        const reportCalls = calls.filter(call => !call.metricKind);
+        assert.deepEqual(reportCalls.map(call => call.id), ["alpha", "bravo", "combined"]);
        const alphaLogs = [
            `${testFixture.registry.sites[0].analyticsLog}.1`,
            testFixture.registry.sites[0].analyticsLog
        ];
        const bravoLogs = [testFixture.registry.sites[1].analyticsLog];
-        assert.deepEqual(calls[0].args.slice(0, 2), alphaLogs);
-        assert.deepEqual(calls[1].args.slice(0, 1), bravoLogs);
+        assert.deepEqual(reportCalls[0].args.slice(0, 2), alphaLogs);
+        assert.deepEqual(reportCalls[1].args.slice(0, 1), bravoLogs);
        assert.deepEqual(
-            calls[2].args.slice(0, 3),
+            reportCalls[2].args.slice(0, 3),
            [...alphaLogs, ...bravoLogs]
        );
-        assert.equal(calls[0].args.includes("--enable-panel=VIRTUAL_HOSTS"), false);
-        assert.equal(calls[1].args.includes("--enable-panel=VIRTUAL_HOSTS"), false);
-        assert.equal(calls[2].args.includes("--enable-panel=VIRTUAL_HOSTS"), true);
-        assert.equal(calls.some(call => call.args.some(argument => argument.endsWith(".gz"))), false);
+        assert.equal(reportCalls[0].args.includes("--enable-panel=VIRTUAL_HOSTS"), false);
+        assert.equal(reportCalls[1].args.includes("--enable-panel=VIRTUAL_HOSTS"), false);
+        assert.equal(reportCalls[2].args.includes("--enable-panel=VIRTUAL_HOSTS"), true);
+        assert.equal(reportCalls.some(call => call.args.some(argument => argument.endsWith(".gz"))), false);
        assert.equal(fs.existsSync(path.join(
            testFixture.registry.stateDirectory,
            "public",
@@ -244,6 +343,16 @@ test("processes subdomains separately and together without publishing disabled c
            testFixture.registry.combined.reportOutputDirectory,
            "report.html"
        )), true);
+        assert.match(fs.readFileSync(path.join(
+            testFixture.registry.combined.reportOutputDirectory,
+            "report.html"
+        ), "utf8"), /href="\/metrics\/"/);
+        const privateState = JSON.parse(fs.readFileSync(
+            testFixture.registry.privateMetrics.statePath,
+            "utf8"
+        ));
+        assert.equal(privateState.sites.alpha.website.daily["2026-09-11"].pageViews, 2);
+        assert.equal(privateState.sites.alpha.updateInfo.daily["2026-09-11"].requests, 2);
    } finally {
        testFixture.cleanup();
    }
@@ -302,6 +411,7 @@ test("dry-run validates generated data without changing production paths", () =>
        ).length, 0);
        assert.equal(fs.existsSync(testFixture.registry.sites[0].publicJsonPath), false);
        assert.equal(fs.existsSync(testFixture.registry.counterStatePath), false);
+        assert.equal(fs.existsSync(testFixture.registry.privateMetrics.statePath), false);
    } finally {
        testFixture.cleanup();
    }
@@ -402,6 +512,22 @@ test("configuration check verifies analytics-log readability", () => {
    }
 });

+test("configuration check rejects a missing update-information log", () => {
+    const testFixture = fixture([{ id: "alpha", publicCounter: true }]);
+    try {
+        fs.rmSync(testFixture.registry.sites[0].updateInfo.analyticsLog);
+        assert.throws(() => generateReports({
+            registryPath: testFixture.registryPath,
+            configTemplatePath: testFixture.configTemplatePath,
+            check: true
+        }, {
+            checkGoAccess: () => GOACCESS_WITHOUT_ZLIB
+        }), /analytics log is not readable/);
+    } finally {
+        testFixture.cleanup();
+    }
+});
+
 test("configuration check reports missing output directories", () => {
    const testFixture = fixture([{ id: "alpha", publicCounter: true }]);
    try {
@@ -462,6 +588,118 @@ test("Nginx filters exclude non-page traffic before analytics logging", () => {
    assert.match(source, /\|map\|/);
    assert.match(source, /\$uri/);
    assert.match(source, /known_bot/);
+    assert.match(source, /map "\$request_method:\$uri:\$status" \$hamradioonline_update_information_loggable/);
+    assert.match(source, /"GET:\/kst4ContestVersionInfo\.xml:200" 1;/);
+});
+
+test("historical import is repeatable and overlaps live aggregation without addition", () => {
+    const testFixture = fixture([{ id: "alpha", publicCounter: true }]);
+    const importPath = path.join(testFixture.root, "access.log.2.gz");
+    const historical = [
+        '192.0.2.10 - - [10/Sep/2026:12:00:00 +0200] "GET /privacy/?x=1 HTTP/1.1" 200 42 "-" "Firefox/130"',
+        '192.0.2.11 - - [10/Sep/2026:12:01:00 +0200] "GET /kst4ContestVersionInfo.xml HTTP/1.1" 200 43 "-" "Java/21.0.1"'
+    ].join("\n");
+    const reports = {
+        alpha: goAccessReport({ "2026-09-11": 3 }),
+        combined: goAccessReport({ "2026-09-11": 3 }, true)
+    };
+    try {
+        testFixture.registry.sites[0].websiteMetricsSince = "2026-09-08";
+        testFixture.registry.sites[0].updateInfo.metricsSince = "2026-09-08";
+        fs.writeFileSync(testFixture.registryPath, JSON.stringify(testFixture.registry));
+        fs.writeFileSync(importPath, require("node:zlib").gzipSync(`${historical}\n`));
+        const options = {
+            registryPath: testFixture.registryPath,
+            configTemplatePath: testFixture.configTemplatePath,
+            goaccessBinary: "fake-goaccess",
+            importLogs: [importPath],
+            importFormat: "nginx-combined",
+            importSite: "alpha",
+            coverageFrom: "2026-09-08",
+            coverageThrough: "2026-09-10"
+        };
+        const dependencies = {
+            checkGoAccess: () => GOACCESS_WITHOUT_ZLIB,
+            runGoAccess: fakeGoAccess(reports, []),
+            now: () => new Date("2026-09-11T07:00:00Z"),
+            skipLock: true
+        };
+
+        generateReports(options, dependencies);
+        generateReports(options, dependencies);
+
+        const state = JSON.parse(fs.readFileSync(testFixture.registry.privateMetrics.statePath, "utf8"));
+        assert.equal(state.sites.alpha.website.daily["2026-09-10"].pageViews, 1);
+        assert.equal(state.sites.alpha.updateInfo.daily["2026-09-10"].requests, 1);
+        assert.equal(state.sites.alpha.website.daily["2026-09-11"].pageViews, 2);
+        assert.equal(state.sites.alpha.updateInfo.daily["2026-09-11"].requests, 2);
+        assert.equal(state.sites.alpha.website.firstCoveredOn, "2026-09-08");
+        assert.equal(state.sites.alpha.updateInfo.firstCoveredOn, "2026-09-08");
+        assert.equal(fs.existsSync(path.join(
+            testFixture.registry.privateMetrics.reportOutputDirectory,
+            "alpha", "updates", "yearly", "report.html"
+        )), true);
+    } finally {
+        testFixture.cleanup();
+    }
+});
+
+test("first run after midnight refreshes the previous day without double counting", () => {
+    const testFixture = fixture([{ id: "alpha", publicCounter: true }]);
+    const reports = {
+        alpha: goAccessReport({ "2026-09-11": 3 }),
+        combined: goAccessReport({ "2026-09-11": 3 }, true)
+    };
+    const baseOptions = {
+        registryPath: testFixture.registryPath,
+        configTemplatePath: testFixture.configTemplatePath,
+        goaccessBinary: "fake-goaccess"
+    };
+    try {
+        generateReports(baseOptions, {
+            checkGoAccess: () => GOACCESS_WITHOUT_ZLIB,
+            runGoAccess: fakeGoAccess(reports, []),
+            now: () => new Date("2026-09-11T20:00:00Z"),
+            skipLock: true
+        });
+        fs.appendFileSync(testFixture.registry.sites[0].analyticsLog, analyticsLine({
+            timestamp: "2026-09-11T23:59:00+02:00",
+            requestPath: "/news/"
+        }));
+        generateReports(baseOptions, {
+            checkGoAccess: () => GOACCESS_WITHOUT_ZLIB,
+            runGoAccess: fakeGoAccess(reports, []),
+            now: () => new Date("2026-09-12T00:30:00Z"),
+            skipLock: true
+        });
+
+        const state = JSON.parse(fs.readFileSync(testFixture.registry.privateMetrics.statePath, "utf8"));
+        assert.equal(state.sites.alpha.website.daily["2026-09-11"].pageViews, 3);
+        assert.equal(state.sites.alpha.website.daily["2026-09-12"].pageViews, 0);
+    } finally {
+        testFixture.cleanup();
+    }
+});
+
+test("historical import CLI requires explicit format, site and coverage", () => {
+    const options = parseArguments([
+        "--registry", "/etc/hamradioonline-analytics/sites.json",
+        "--config-template", "/etc/hamradioonline-analytics/goaccess.conf.template",
+        "--import-site", "kst4contest",
+        "--import-format", "nginx-combined",
+        "--coverage-from", "2026-09-01",
+        "--coverage-through", "2026-09-10",
+        "--import-log", "/protected/access.log.2.gz",
+        "--import-log", "/protected/access.log.1"
+    ]);
+    assert.deepEqual(options.importLogs, [
+        "/protected/access.log.2.gz",
+        "/protected/access.log.1"
+    ]);
+    assert.equal(options.importFormat, "nginx-combined");
+    assert.equal(options.importSite, "kst4contest");
+    assert.equal(options.coverageFrom, "2026-09-01");
+    assert.equal(options.coverageThrough, "2026-09-10");
 });

 test("server templates use the production GeoIP path and permission model", () => {
@@ -536,6 +774,7 @@ test("logrotate uses the Ubuntu Nginx rotation action", () => {
    assert.match(source, /^\s*delaycompress$/m);
    assert.match(source, /^\s*rotate 14$/m);
    assert.match(source, /^\s*create 0640 www-data hamradio-analytics$/m);
+    assert.match(source, /\*-update-information\.log/);
 });

 test("website package records the Node.js 18.19.1 baseline", () => {