Fix private analytics aggregation

website/test/analytics-generator.test.js

@@ -150,14 +150,18 @@ function fakeGoAccess(reports, calls, failureId) {
             throw new Error("simulated GoAccess failure");
         }
         if (invocation.metricKind) {
-            const lines = fs.readFileSync(invocation.args[0], "utf8").trim().split(/\r?\n/);
+            const sourceLines = fs.readFileSync(invocation.args[0], "utf8").trim().split(/\r?\n/);
+            invocation.inputLines = sourceLines;
+            const lines = invocation.metricKind === "websiteClassifier"
+                ? sourceLines.filter(line => !line.includes("UnknownScanner/1.0"))
+                : sourceLines;
             const paths = {};
             for (const line of lines) {
                 const requestPath = line.split("\t")[4];
                 paths[requestPath] = (paths[requestPath] || 0) + 1;
             }
             const report = {
-                general: { total_requests: lines.length },
+                general: { total_requests: lines.length, valid_requests: lines.length },
                 requests: {
                     data: Object.entries(paths).map(([requestPath, count]) => ({
                         data: requestPath,
@@ -181,6 +185,42 @@ function fakeGoAccess(reports, calls, failureId) {
     };
 }
 
+test("uses GoAccess client classification before aggregating website paths and countries", () => {
+    const testFixture = fixture([{ id: "alpha", publicCounter: true }]);
+    const calls = [];
+    const reports = {
+        alpha: goAccessReport({ "2026-09-11": 2 }),
+        combined: goAccessReport({ "2026-09-11": 2 }, true)
+    };
+    fs.appendFileSync(testFixture.registry.sites[0].analyticsLog, analyticsLine({
+        requestPath: "/scanner/",
+        userAgent: "UnknownScanner/1.0"
+    }));
+    try {
+        generateReports({
+            registryPath: testFixture.registryPath,
+            configTemplatePath: testFixture.configTemplatePath,
+            goaccessBinary: "fake-goaccess"
+        }, {
+            checkGoAccess: () => GOACCESS_WITHOUT_ZLIB,
+            runGoAccess: fakeGoAccess(reports, calls),
+            now: () => new Date("2026-09-11T12:00:00Z"),
+            skipLock: true
+        });
+
+        const state = JSON.parse(fs.readFileSync(testFixture.registry.privateMetrics.statePath, "utf8"));
+        assert.equal(state.sites.alpha.website.daily["2026-09-11"].pageViews, 2);
+        assert.equal(state.sites.alpha.website.daily["2026-09-11"].paths["/scanner/"], undefined);
+        const classifierCall = calls.find(call => call.metricKind === "websiteClassifier");
+        const aggregationCall = calls.find(call => call.metricKind === "website");
+        assert.equal(classifierCall.inputLines.some(line => line.includes("UnknownScanner/1.0")), true);
+        assert.equal(classifierCall.inputLines.every(line => line.split("\t")[4].startsWith("/__client/")), true);
+        assert.equal(aggregationCall.inputLines.some(line => line.includes("UnknownScanner/1.0")), false);
+    } finally {
+        testFixture.cleanup();
+    }
+});
+
 function run(testFixture, reports, calls = [], failureId) {
     return generateReports({
         registryPath: testFixture.registryPath,

website/test/private-metrics.test.js

@@ -57,6 +57,12 @@ test("parses analytics and regular Nginx combined records without query strings"
     assert.equal(analytics.hour, "23");
     assert.equal(analytics.path, "/privacy/");
 
+    const emptyPath = parseAnalyticsLine(
+        'kst4contest.hamradioonline.de\t192.0.2.1\t2026-09-14T23:30:00+02:00\tGET\t\tHTTP/1.1\t400\t166\t"-"'
+    );
+    assert.equal(emptyPath.path, null);
+    assert.equal(isEligibleWebsiteRequest(emptyPath), false);
+
     const combined = parseCombinedLine(
         '192.0.2.2 - - [14/Sep/2026:23:31:00 +0200] "GET /news/?x=1 HTTP/1.1" 200 43 "-" "Mozilla/5.0 Firefox/130"',
         "kst4contest.hamradioonline.de"
@@ -84,7 +90,7 @@ test("uses Europe/Berlin across both daylight-saving transitions", () => {
 
 test("keeps absolute countries including Switzerland, United Kingdom and unknown", () => {
     const website = aggregateGoAccessReport({
-        general: { total_requests: 5 },
+        general: { total_requests: 2245, valid_requests: 5 },
         geolocation: { data: [
             { data: "Europe", hits: { count: 3 }, items: [
                 { data: "Germany", hits: { count: 1 } },
@@ -107,10 +113,27 @@ test("keeps absolute countries including Switzerland, United Kingdom and unknown
     });
     assert.deepEqual(website.paths, { "/": 2, "/privacy/": 2, "/news/": 1 });
     assert.throws(() => aggregateGoAccessReport({
-        general: { total_requests: 2 },
+        general: { total_requests: 10, valid_requests: 2 },
         geolocation: { data: [{ data: "Germany", hits: { count: 2 } }] },
         requests: { data: [{ data: "/", hits: { count: 1 } }] }
-    }, "website"), /differs from the GoAccess request-panel definition/);
+    }, "website"), /valid-request count differs from the request-panel definition/);
+
+    const updates = aggregateGoAccessReport({
+        general: { total_requests: 9, valid_requests: 2 },
+        geolocation: { data: [{ data: "Germany", hits: { count: 2 } }] },
+        requests: { data: [{ data: "/kst4ContestVersionInfo.xml", hits: { count: 2 } }] }
+    }, "updateInfo");
+    assert.equal(updates.requests, 2);
+    assert.throws(() => aggregateGoAccessReport({
+        general: { total_requests: 2, valid_requests: 1 },
+        geolocation: { data: [{ data: "Germany", hits: { count: 2 } }] },
+        requests: { data: [{ data: "/", hits: { count: 1 } }] }
+    }, "website"), /country total exceeds/);
+    assert.throws(() => aggregateGoAccessReport({
+        general: { total_requests: 1, valid_requests: 1 },
+        geolocation: { data: [{ data: "Germany", hits: { count: 1 } }] },
+        requests: { data: [{ data: "/unexpected", hits: { count: 1 } }] }
+    }, "updateInfo"), /unexpected request path/);
 });
 
 test("deduplicates overlapping import files and reads gzip without GoAccess Zlib", () => {