From a8f12876ecc376094486e0125a21484773916433 Mon Sep 17 00:00:00 2001 From: Marc Froehlich Date: Mon, 14 Sep 2026 23:01:06 +0200 Subject: [PATCH] Align private metrics with GoAccess request panel --- website/ops/analytics/private-metrics.js | 41 ++++++++++++------------ website/test/analytics-generator.test.js | 26 +++++++++++++-- website/test/private-metrics.test.js | 2 +- 3 files changed, 44 insertions(+), 25 deletions(-) diff --git a/website/ops/analytics/private-metrics.js b/website/ops/analytics/private-metrics.js index 3c8cab5f..b2b256f7 100644 --- a/website/ops/analytics/private-metrics.js +++ b/website/ops/analytics/private-metrics.js @@ -277,7 +277,9 @@ function aggregateGoAccessReport(report, kind) { const paths = countPanel(report, "requests"); const total = sumValues(paths); if (validRequests !== total) { - throw new Error("GoAccess valid-request count differs from the request-panel definition"); + throw new Error( + `GoAccess valid-request count ${validRequests} differs from request-panel total ${total}` + ); } const countries = countCountries(report); const countryTotal = sumValues(countries); @@ -338,24 +340,19 @@ function runMetricGoAccess({ records, jobId, metricKind, site, context, includeC } function filterWebsiteRecordsWithGoAccess(records, date, site, context) { - const representatives = []; - const pathByUserAgent = new Map(); - for (const record of records) { - if (pathByUserAgent.has(record.userAgent)) continue; - const classifierPath = `/__client/${pathByUserAgent.size}`; - pathByUserAgent.set(record.userAgent, classifierPath); + const recordIndexByPath = new Map(); + const representatives = records.map((record, index) => { + const extension = path.posix.extname(record.path); + const classifierPath = `/__request/${index}${extension}`; + recordIndexByPath.set(classifierPath, index); const representative = { ...record, ip: "192.0.2.1", - method: "GET", - path: classifierPath, - protocol: "HTTP/1.1", - status: 200, - bytes: 0 + path: classifierPath }; representative.line = toAnalyticsLine(representative); - representatives.push(representative); - } + return representative; + }); const report = runMetricGoAccess({ records: representatives, jobId: `metrics-${site.id}-website-classifier-${date}`, @@ -365,16 +362,14 @@ function filterWebsiteRecordsWithGoAccess(records, date, site, context) { includeCrawlers: false }); const acceptedPaths = countPanel(report, "requests"); - const knownPaths = new Set(pathByUserAgent.values()); + const acceptedIndexes = new Set(); for (const [classifierPath, count] of Object.entries(acceptedPaths)) { - if (!knownPaths.has(classifierPath) || count !== 1) { + if (!recordIndexByPath.has(classifierPath) || count !== 1) { throw new Error(`website client classification is invalid for ${site.id}/${date}`); } + acceptedIndexes.add(recordIndexByPath.get(classifierPath)); } - const acceptedUserAgents = new Set([...pathByUserAgent] - .filter(([_userAgent, classifierPath]) => Object.hasOwn(acceptedPaths, classifierPath)) - .map(([userAgent]) => userAgent)); - return records.filter(record => acceptedUserAgents.has(record.userAgent)); + return records.filter((_record, index) => acceptedIndexes.has(index)); } function aggregateDays({ records, dates, kind, site, context }) { @@ -409,7 +404,11 @@ function aggregateDays({ records, dates, kind, site, context }) { context, includeCrawlers: true }); - daily[date] = aggregateGoAccessReport(report, kind); + try { + daily[date] = aggregateGoAccessReport(report, kind); + } catch (error) { + throw new Error(`private metric aggregation failed for ${site.id}/${kind}/${date}: ${error.message}`); + } if (kind === "updateInfo") { const hours = {}; const clients = {}; diff --git a/website/test/analytics-generator.test.js b/website/test/analytics-generator.test.js index cfa366fd..748afe60 100644 --- a/website/test/analytics-generator.test.js +++ b/website/test/analytics-generator.test.js @@ -153,7 +153,10 @@ function fakeGoAccess(reports, calls, failureId) { const sourceLines = fs.readFileSync(invocation.args[0], "utf8").trim().split(/\r?\n/); invocation.inputLines = sourceLines; const lines = invocation.metricKind === "websiteClassifier" - ? sourceLines.filter(line => !line.includes("UnknownScanner/1.0")) + ? sourceLines.filter(line => { + const fields = line.split("\t"); + return fields[6] !== "404" && !line.includes("UnknownScanner/1.0"); + }) : sourceLines; const paths = {}; for (const line of lines) { @@ -196,6 +199,18 @@ test("uses GoAccess client classification before aggregating website paths and c requestPath: "/scanner/", userAgent: "UnknownScanner/1.0" })); + fs.appendFileSync(testFixture.registry.sites[0].analyticsLog, analyticsLine({ + requestPath: "/missing/", + status: 404 + })); + fs.appendFileSync(testFixture.registry.sites[0].analyticsLog, analyticsLine({ + requestPath: "/cached/", + status: 304 + })); + fs.appendFileSync(testFixture.registry.sites[0].analyticsLog, analyticsLine({ + requestPath: "/page.html", + userAgent: "Mozilla/5.0 Chrome/140.0" + })); try { generateReports({ registryPath: testFixture.registryPath, @@ -209,13 +224,18 @@ test("uses GoAccess client classification before aggregating website paths and c }); const state = JSON.parse(fs.readFileSync(testFixture.registry.privateMetrics.statePath, "utf8")); - assert.equal(state.sites.alpha.website.daily["2026-09-11"].pageViews, 2); + assert.equal(state.sites.alpha.website.daily["2026-09-11"].pageViews, 4); assert.equal(state.sites.alpha.website.daily["2026-09-11"].paths["/scanner/"], undefined); + assert.equal(state.sites.alpha.website.daily["2026-09-11"].paths["/missing/"], undefined); + assert.equal(state.sites.alpha.website.daily["2026-09-11"].paths["/cached/"], 1); + assert.equal(state.sites.alpha.website.daily["2026-09-11"].paths["/page.html"], 1); const classifierCall = calls.find(call => call.metricKind === "websiteClassifier"); const aggregationCall = calls.find(call => call.metricKind === "website"); assert.equal(classifierCall.inputLines.some(line => line.includes("UnknownScanner/1.0")), true); - assert.equal(classifierCall.inputLines.every(line => line.split("\t")[4].startsWith("/__client/")), true); + assert.equal(classifierCall.inputLines.every(line => line.split("\t")[4].startsWith("/__request/")), true); + assert.equal(classifierCall.inputLines.some(line => line.split("\t")[4].endsWith(".html")), true); assert.equal(aggregationCall.inputLines.some(line => line.includes("UnknownScanner/1.0")), false); + assert.equal(aggregationCall.inputLines.some(line => line.split("\t")[6] === "404"), false); } finally { testFixture.cleanup(); } diff --git a/website/test/private-metrics.test.js b/website/test/private-metrics.test.js index 04f7b930..e12dbb56 100644 --- a/website/test/private-metrics.test.js +++ b/website/test/private-metrics.test.js @@ -116,7 +116,7 @@ test("keeps absolute countries including Switzerland, United Kingdom and unknown general: { total_requests: 10, valid_requests: 2 }, geolocation: { data: [{ data: "Germany", hits: { count: 2 } }] }, requests: { data: [{ data: "/", hits: { count: 1 } }] } - }, "website"), /valid-request count differs from the request-panel definition/); + }, "website"), /valid-request count 2 differs from request-panel total 1/); const updates = aggregateGoAccessReport({ general: { total_requests: 9, valid_requests: 2 },