Align private metrics with GoAccess request panel

website/ops/analytics/private-metrics.js

@@ -277,7 +277,9 @@ function aggregateGoAccessReport(report, kind) {
     const paths = countPanel(report, "requests");
     const total = sumValues(paths);
     if (validRequests !== total) {
-        throw new Error("GoAccess valid-request count differs from the request-panel definition");
+        throw new Error(
+            `GoAccess valid-request count ${validRequests} differs from request-panel total ${total}`
+        );
     }
     const countries = countCountries(report);
     const countryTotal = sumValues(countries);
@@ -338,24 +340,19 @@ function runMetricGoAccess({ records, jobId, metricKind, site, context, includeC
 }
 
 function filterWebsiteRecordsWithGoAccess(records, date, site, context) {
-    const representatives = [];
-    const pathByUserAgent = new Map();
-    for (const record of records) {
-        if (pathByUserAgent.has(record.userAgent)) continue;
-        const classifierPath = `/__client/${pathByUserAgent.size}`;
-        pathByUserAgent.set(record.userAgent, classifierPath);
+    const recordIndexByPath = new Map();
+    const representatives = records.map((record, index) => {
+        const extension = path.posix.extname(record.path);
+        const classifierPath = `/__request/${index}${extension}`;
+        recordIndexByPath.set(classifierPath, index);
         const representative = {
             ...record,
             ip: "192.0.2.1",
-            method: "GET",
-            path: classifierPath,
-            protocol: "HTTP/1.1",
-            status: 200,
-            bytes: 0
+            path: classifierPath
         };
         representative.line = toAnalyticsLine(representative);
-        representatives.push(representative);
-    }
+        return representative;
+    });
     const report = runMetricGoAccess({
         records: representatives,
         jobId: `metrics-${site.id}-website-classifier-${date}`,
@@ -365,16 +362,14 @@ function filterWebsiteRecordsWithGoAccess(records, date, site, context) {
         includeCrawlers: false
     });
     const acceptedPaths = countPanel(report, "requests");
-    const knownPaths = new Set(pathByUserAgent.values());
+    const acceptedIndexes = new Set();
     for (const [classifierPath, count] of Object.entries(acceptedPaths)) {
-        if (!knownPaths.has(classifierPath) || count !== 1) {
+        if (!recordIndexByPath.has(classifierPath) || count !== 1) {
             throw new Error(`website client classification is invalid for ${site.id}/${date}`);
         }
+        acceptedIndexes.add(recordIndexByPath.get(classifierPath));
     }
-    const acceptedUserAgents = new Set([...pathByUserAgent]
-        .filter(([_userAgent, classifierPath]) => Object.hasOwn(acceptedPaths, classifierPath))
-        .map(([userAgent]) => userAgent));
-    return records.filter(record => acceptedUserAgents.has(record.userAgent));
+    return records.filter((_record, index) => acceptedIndexes.has(index));
 }
 
 function aggregateDays({ records, dates, kind, site, context }) {
@@ -409,7 +404,11 @@ function aggregateDays({ records, dates, kind, site, context }) {
             context,
             includeCrawlers: true
         });
-        daily[date] = aggregateGoAccessReport(report, kind);
+        try {
+            daily[date] = aggregateGoAccessReport(report, kind);
+        } catch (error) {
+            throw new Error(`private metric aggregation failed for ${site.id}/${kind}/${date}: ${error.message}`);
+        }
         if (kind === "updateInfo") {
             const hours = {};
             const clients = {};

website/test/analytics-generator.test.js

@@ -153,7 +153,10 @@ function fakeGoAccess(reports, calls, failureId) {
             const sourceLines = fs.readFileSync(invocation.args[0], "utf8").trim().split(/\r?\n/);
             invocation.inputLines = sourceLines;
             const lines = invocation.metricKind === "websiteClassifier"
-                ? sourceLines.filter(line => !line.includes("UnknownScanner/1.0"))
+                ? sourceLines.filter(line => {
+                    const fields = line.split("\t");
+                    return fields[6] !== "404" && !line.includes("UnknownScanner/1.0");
+                })
                 : sourceLines;
             const paths = {};
             for (const line of lines) {
@@ -196,6 +199,18 @@ test("uses GoAccess client classification before aggregating website paths and c
         requestPath: "/scanner/",
         userAgent: "UnknownScanner/1.0"
     }));
+    fs.appendFileSync(testFixture.registry.sites[0].analyticsLog, analyticsLine({
+        requestPath: "/missing/",
+        status: 404
+    }));
+    fs.appendFileSync(testFixture.registry.sites[0].analyticsLog, analyticsLine({
+        requestPath: "/cached/",
+        status: 304
+    }));
+    fs.appendFileSync(testFixture.registry.sites[0].analyticsLog, analyticsLine({
+        requestPath: "/page.html",
+        userAgent: "Mozilla/5.0 Chrome/140.0"
+    }));
     try {
         generateReports({
             registryPath: testFixture.registryPath,
@@ -209,13 +224,18 @@ test("uses GoAccess client classification before aggregating website paths and c
         });
 
         const state = JSON.parse(fs.readFileSync(testFixture.registry.privateMetrics.statePath, "utf8"));
-        assert.equal(state.sites.alpha.website.daily["2026-09-11"].pageViews, 2);
+        assert.equal(state.sites.alpha.website.daily["2026-09-11"].pageViews, 4);
         assert.equal(state.sites.alpha.website.daily["2026-09-11"].paths["/scanner/"], undefined);
+        assert.equal(state.sites.alpha.website.daily["2026-09-11"].paths["/missing/"], undefined);
+        assert.equal(state.sites.alpha.website.daily["2026-09-11"].paths["/cached/"], 1);
+        assert.equal(state.sites.alpha.website.daily["2026-09-11"].paths["/page.html"], 1);
         const classifierCall = calls.find(call => call.metricKind === "websiteClassifier");
         const aggregationCall = calls.find(call => call.metricKind === "website");
         assert.equal(classifierCall.inputLines.some(line => line.includes("UnknownScanner/1.0")), true);
-        assert.equal(classifierCall.inputLines.every(line => line.split("\t")[4].startsWith("/__client/")), true);
+        assert.equal(classifierCall.inputLines.every(line => line.split("\t")[4].startsWith("/__request/")), true);
+        assert.equal(classifierCall.inputLines.some(line => line.split("\t")[4].endsWith(".html")), true);
         assert.equal(aggregationCall.inputLines.some(line => line.includes("UnknownScanner/1.0")), false);
+        assert.equal(aggregationCall.inputLines.some(line => line.split("\t")[6] === "404"), false);
     } finally {
         testFixture.cleanup();
     }

website/test/private-metrics.test.js

@@ -116,7 +116,7 @@ test("keeps absolute countries including Switzerland, United Kingdom and unknown
         general: { total_requests: 10, valid_requests: 2 },
         geolocation: { data: [{ data: "Germany", hits: { count: 2 } }] },
         requests: { data: [{ data: "/", hits: { count: 1 } }] }
-    }, "website"), /valid-request count differs from the request-panel definition/);
+    }, "website"), /valid-request count 2 differs from request-panel total 1/);
 
     const updates = aggregateGoAccessReport({
         general: { total_requests: 9, valid_requests: 2 },