mirror of
https://github.com/praktimarc/kst4contest.git
synced 2026-09-12 04:05:34 +02:00
880 lines
28 KiB
JavaScript
880 lines
28 KiB
JavaScript
#!/usr/bin/env node
|
|
"use strict";
|
|
|
|
const fs = require("node:fs");
|
|
const os = require("node:os");
|
|
const path = require("node:path");
|
|
const { spawnSync } = require("node:child_process");
|
|
|
|
const STATE_SCHEMA_VERSION = 1;
|
|
const PUBLIC_SCHEMA_VERSION = 1;
|
|
const MAX_SAFE_INTEGER = Number.MAX_SAFE_INTEGER;
|
|
const RESERVED_STATS_HOST = "stats.hamradioonline.de";
|
|
|
|
class ConfigurationError extends Error {
|
|
constructor(message) {
|
|
super(message);
|
|
this.name = "ConfigurationError";
|
|
this.exitCode = 2;
|
|
}
|
|
}
|
|
|
|
class LockError extends Error {
|
|
constructor(message) {
|
|
super(message);
|
|
this.name = "LockError";
|
|
this.exitCode = 3;
|
|
}
|
|
}
|
|
|
|
function readJson(filePath, label) {
|
|
let parsed;
|
|
|
|
try {
|
|
parsed = JSON.parse(fs.readFileSync(filePath, "utf8"));
|
|
} catch (error) {
|
|
throw new ConfigurationError(`${label} is not valid JSON: ${error.message}`);
|
|
}
|
|
|
|
return parsed;
|
|
}
|
|
|
|
function parseIsoDate(value) {
|
|
const match = /^(\d{4})-(\d{2})-(\d{2})$/.exec(value);
|
|
|
|
if (!match) {
|
|
return null;
|
|
}
|
|
|
|
const year = Number(match[1]);
|
|
const month = Number(match[2]);
|
|
const day = Number(match[3]);
|
|
const date = new Date(Date.UTC(year, month - 1, day));
|
|
|
|
if (date.getUTCFullYear() !== year
|
|
|| date.getUTCMonth() !== month - 1
|
|
|| date.getUTCDate() !== day) {
|
|
return null;
|
|
}
|
|
|
|
return `${match[1]}-${match[2]}-${match[3]}`;
|
|
}
|
|
|
|
function requireAbsolutePath(value, label) {
|
|
if (typeof value !== "string" || /[\r\n\0]/.test(value) || !path.isAbsolute(value)) {
|
|
throw new ConfigurationError(`${label} must be an absolute path`);
|
|
}
|
|
|
|
return path.normalize(value);
|
|
}
|
|
|
|
function isWithin(parent, candidate) {
|
|
const relative = path.relative(parent, candidate);
|
|
return relative !== ""
|
|
&& relative !== ".."
|
|
&& !relative.startsWith(`..${path.sep}`)
|
|
&& !path.isAbsolute(relative);
|
|
}
|
|
|
|
function validateRegistry(registry) {
|
|
if (!registry || typeof registry !== "object" || Array.isArray(registry)) {
|
|
throw new ConfigurationError("registry must be an object");
|
|
}
|
|
if (registry.schemaVersion !== 1) {
|
|
throw new ConfigurationError("registry schemaVersion must be 1");
|
|
}
|
|
if (!Array.isArray(registry.sites) || registry.sites.length === 0) {
|
|
throw new ConfigurationError("registry.sites must contain at least one site");
|
|
}
|
|
|
|
const stateDirectory = requireAbsolutePath(
|
|
registry.stateDirectory,
|
|
"registry.stateDirectory"
|
|
);
|
|
const counterStatePath = requireAbsolutePath(
|
|
registry.counterStatePath,
|
|
"registry.counterStatePath"
|
|
);
|
|
if (!isWithin(stateDirectory, counterStatePath)) {
|
|
throw new ConfigurationError("counterStatePath must be below stateDirectory");
|
|
}
|
|
|
|
const ids = new Set();
|
|
const hostnames = new Set();
|
|
const analyticsLogPaths = new Set();
|
|
const outputDirectories = new Set();
|
|
const publicJsonPaths = new Set();
|
|
const sites = registry.sites.map((site, index) => {
|
|
const label = `registry.sites[${index}]`;
|
|
|
|
if (!site || typeof site !== "object" || Array.isArray(site)) {
|
|
throw new ConfigurationError(`${label} must be an object`);
|
|
}
|
|
if (typeof site.id !== "string" || !/^[a-z0-9][a-z0-9-]{0,62}$/.test(site.id)) {
|
|
throw new ConfigurationError(`${label}.id is invalid`);
|
|
}
|
|
if (site.id === "combined") {
|
|
throw new ConfigurationError(`${label}.id is reserved`);
|
|
}
|
|
if (ids.has(site.id)) {
|
|
throw new ConfigurationError(`duplicate site id: ${site.id}`);
|
|
}
|
|
ids.add(site.id);
|
|
|
|
if (typeof site.hostname !== "string"
|
|
|| !/^[a-z0-9.-]+$/.test(site.hostname)
|
|
|| site.hostname.includes("..")) {
|
|
throw new ConfigurationError(`${label}.hostname is invalid`);
|
|
}
|
|
const hostname = site.hostname.toLowerCase();
|
|
if (hostname === RESERVED_STATS_HOST) {
|
|
throw new ConfigurationError(`${RESERVED_STATS_HOST} must not be registered`);
|
|
}
|
|
if (hostnames.has(hostname)) {
|
|
throw new ConfigurationError(`duplicate hostname: ${hostname}`);
|
|
}
|
|
hostnames.add(hostname);
|
|
|
|
const activatedOn = parseIsoDate(site.activatedOn);
|
|
if (!activatedOn) {
|
|
throw new ConfigurationError(`${label}.activatedOn must be a valid ISO date`);
|
|
}
|
|
if (typeof site.publicCounter !== "boolean") {
|
|
throw new ConfigurationError(`${label}.publicCounter must be boolean`);
|
|
}
|
|
const analyticsLog = requireAbsolutePath(
|
|
site.analyticsLog,
|
|
`${label}.analyticsLog`
|
|
);
|
|
if (/\.(?:\d+|gz)$/i.test(path.basename(analyticsLog))) {
|
|
throw new ConfigurationError(
|
|
`${label}.analyticsLog must identify the current uncompressed log`
|
|
);
|
|
}
|
|
if (analyticsLogPaths.has(analyticsLog)) {
|
|
throw new ConfigurationError(`analytics log is registered more than once: ${analyticsLog}`);
|
|
}
|
|
analyticsLogPaths.add(analyticsLog);
|
|
|
|
const reportOutputDirectory = requireAbsolutePath(
|
|
site.reportOutputDirectory,
|
|
`${label}.reportOutputDirectory`
|
|
);
|
|
if (!isWithin(stateDirectory, reportOutputDirectory)) {
|
|
throw new ConfigurationError(`${label}.reportOutputDirectory must be below stateDirectory`);
|
|
}
|
|
if (outputDirectories.has(reportOutputDirectory)) {
|
|
throw new ConfigurationError(`duplicate report output directory: ${reportOutputDirectory}`);
|
|
}
|
|
outputDirectories.add(reportOutputDirectory);
|
|
|
|
const publicJsonPath = site.publicCounter
|
|
? requireAbsolutePath(site.publicJsonPath, `${label}.publicJsonPath`)
|
|
: null;
|
|
if (publicJsonPath && !isWithin(stateDirectory, publicJsonPath)) {
|
|
throw new ConfigurationError(`${label}.publicJsonPath must be below stateDirectory`);
|
|
}
|
|
if (publicJsonPath === counterStatePath) {
|
|
throw new ConfigurationError(`${label}.publicJsonPath conflicts with counterStatePath`);
|
|
}
|
|
if (publicJsonPath && publicJsonPaths.has(publicJsonPath)) {
|
|
throw new ConfigurationError(`duplicate public JSON path: ${publicJsonPath}`);
|
|
}
|
|
if (publicJsonPath) {
|
|
publicJsonPaths.add(publicJsonPath);
|
|
}
|
|
|
|
return {
|
|
id: site.id,
|
|
hostname,
|
|
activatedOn,
|
|
publicCounter: site.publicCounter,
|
|
analyticsLog,
|
|
reportOutputDirectory,
|
|
publicJsonPath
|
|
};
|
|
});
|
|
|
|
if (!registry.combined || typeof registry.combined !== "object") {
|
|
throw new ConfigurationError("registry.combined must be an object");
|
|
}
|
|
|
|
const combinedReportOutputDirectory = requireAbsolutePath(
|
|
registry.combined.reportOutputDirectory,
|
|
"registry.combined.reportOutputDirectory"
|
|
);
|
|
if (!isWithin(stateDirectory, combinedReportOutputDirectory)) {
|
|
throw new ConfigurationError(
|
|
"registry.combined.reportOutputDirectory must be below stateDirectory"
|
|
);
|
|
}
|
|
if (outputDirectories.has(combinedReportOutputDirectory)) {
|
|
throw new ConfigurationError(
|
|
`duplicate report output directory: ${combinedReportOutputDirectory}`
|
|
);
|
|
}
|
|
|
|
return {
|
|
schemaVersion: 1,
|
|
stateDirectory,
|
|
counterStatePath,
|
|
lockFile: requireAbsolutePath(registry.lockFile, "registry.lockFile"),
|
|
geoIpCountryDatabase: requireAbsolutePath(
|
|
registry.geoIpCountryDatabase,
|
|
"registry.geoIpCountryDatabase"
|
|
),
|
|
combined: {
|
|
reportOutputDirectory: combinedReportOutputDirectory
|
|
},
|
|
sites
|
|
};
|
|
}
|
|
|
|
function resolveAnalyticsLogs(site) {
|
|
const logs = [];
|
|
const candidates = [
|
|
{ path: `${site.analyticsLog}.1`, required: false },
|
|
{ path: site.analyticsLog, required: true }
|
|
];
|
|
|
|
for (const candidate of candidates) {
|
|
let stats;
|
|
try {
|
|
stats = fs.statSync(candidate.path);
|
|
} catch (error) {
|
|
if (!candidate.required && error.code === "ENOENT") {
|
|
continue;
|
|
}
|
|
throw new ConfigurationError(`analytics log is not readable: ${candidate.path}`);
|
|
}
|
|
if (!stats.isFile()) {
|
|
throw new ConfigurationError(`analytics log is not a file: ${candidate.path}`);
|
|
}
|
|
try {
|
|
fs.accessSync(candidate.path, fs.constants.R_OK);
|
|
} catch (error) {
|
|
throw new ConfigurationError(
|
|
`analytics log is not readable by the current user: ${candidate.path}`
|
|
);
|
|
}
|
|
logs.push(candidate.path);
|
|
}
|
|
|
|
return logs;
|
|
}
|
|
|
|
function requireWritableDirectory(directory, label) {
|
|
let stats;
|
|
try {
|
|
stats = fs.statSync(directory);
|
|
} catch (error) {
|
|
throw new ConfigurationError(`${label} does not exist: ${directory}`);
|
|
}
|
|
if (!stats.isDirectory()) {
|
|
throw new ConfigurationError(`${label} is not a directory: ${directory}`);
|
|
}
|
|
try {
|
|
fs.accessSync(directory, fs.constants.W_OK | fs.constants.X_OK);
|
|
} catch (error) {
|
|
throw new ConfigurationError(
|
|
`${label} is not writable by the current user: ${directory}`
|
|
);
|
|
}
|
|
}
|
|
|
|
function validateInputs(registry, configTemplate) {
|
|
const requiredConfigParts = [
|
|
"{{DB_PATH}}",
|
|
"{{RESTORE_DIRECTIVE}}",
|
|
"{{GEOIP_COUNTRY_DATABASE}}",
|
|
"persist true",
|
|
"anonymize-ip true",
|
|
"ignore-crawlers true",
|
|
"unknowns-as-crawlers true",
|
|
"keep-last 395"
|
|
];
|
|
|
|
for (const required of requiredConfigParts) {
|
|
if (!configTemplate.includes(required)) {
|
|
throw new ConfigurationError(`GoAccess template is missing: ${required}`);
|
|
}
|
|
}
|
|
|
|
const analyticsLogsBySite = new Map(registry.sites.map(site => [
|
|
site.id,
|
|
resolveAnalyticsLogs(site)
|
|
]));
|
|
|
|
let geoStats;
|
|
try {
|
|
geoStats = fs.statSync(registry.geoIpCountryDatabase);
|
|
} catch (error) {
|
|
throw new ConfigurationError(
|
|
`GeoIP Country database is not readable: ${registry.geoIpCountryDatabase}`
|
|
);
|
|
}
|
|
if (!geoStats.isFile() || /city/i.test(path.basename(registry.geoIpCountryDatabase))) {
|
|
throw new ConfigurationError("geoIpCountryDatabase must be a Country database file");
|
|
}
|
|
try {
|
|
fs.accessSync(registry.geoIpCountryDatabase, fs.constants.R_OK);
|
|
} catch (error) {
|
|
throw new ConfigurationError(
|
|
`GeoIP Country database is not readable by the current user: `
|
|
+ registry.geoIpCountryDatabase
|
|
);
|
|
}
|
|
|
|
requireWritableDirectory(registry.stateDirectory, "stateDirectory");
|
|
requireWritableDirectory(
|
|
registry.combined.reportOutputDirectory,
|
|
"combined report output directory"
|
|
);
|
|
for (const site of registry.sites) {
|
|
requireWritableDirectory(
|
|
site.reportOutputDirectory,
|
|
`report output directory for ${site.id}`
|
|
);
|
|
if (site.publicCounter) {
|
|
requireWritableDirectory(
|
|
path.dirname(site.publicJsonPath),
|
|
`public output directory for ${site.id}`
|
|
);
|
|
}
|
|
}
|
|
|
|
return analyticsLogsBySite;
|
|
}
|
|
|
|
function renderGoAccessConfig(template, dbPath, restore, geoIpCountryDatabase) {
|
|
const rendered = template
|
|
.replaceAll("{{DB_PATH}}", dbPath)
|
|
.replaceAll(
|
|
"{{RESTORE_DIRECTIVE}}",
|
|
restore ? "restore true" : "# restore is disabled until a database exists"
|
|
)
|
|
.replaceAll("{{GEOIP_COUNTRY_DATABASE}}", geoIpCountryDatabase);
|
|
|
|
if (/{{[A-Z_]+}}/.test(rendered)) {
|
|
throw new ConfigurationError("GoAccess template contains an unknown placeholder");
|
|
}
|
|
|
|
return rendered;
|
|
}
|
|
|
|
function defaultRunGoAccess({ binary, args }) {
|
|
const result = spawnSync(binary, args, {
|
|
encoding: "utf8",
|
|
maxBuffer: 1024 * 1024,
|
|
timeout: 30 * 60 * 1000,
|
|
windowsHide: true
|
|
});
|
|
|
|
if (result.error) {
|
|
throw new Error(`could not start GoAccess: ${result.error.message}`);
|
|
}
|
|
if (result.status !== 0) {
|
|
const detail = (result.stderr || result.stdout || "no diagnostic output").trim();
|
|
throw new Error(`GoAccess failed with exit code ${result.status}: ${detail}`);
|
|
}
|
|
}
|
|
|
|
function defaultCheckGoAccess(binary) {
|
|
const result = spawnSync(binary, ["--version"], {
|
|
encoding: "utf8",
|
|
timeout: 10000,
|
|
windowsHide: true
|
|
});
|
|
|
|
if (result.error || result.status !== 0) {
|
|
const detail = result.error
|
|
? result.error.message
|
|
: (result.stderr || result.stdout || "no diagnostic output").trim();
|
|
throw new ConfigurationError(`GoAccess is not available: ${detail}`);
|
|
}
|
|
|
|
return parseGoAccessVersion(`${result.stdout || ""}\n${result.stderr || ""}`);
|
|
}
|
|
|
|
function parseGoAccessVersion(output) {
|
|
const versionMatch = /GoAccess\s+-\s+([0-9]+(?:\.[0-9]+)+)/i.exec(output);
|
|
|
|
return {
|
|
version: versionMatch ? versionMatch[1] : "unknown",
|
|
geoIpMmdb: /--enable-geoip=mmdb\b/i.test(output),
|
|
openSsl: /--with-openssl\b/i.test(output),
|
|
zlib: /--with-zlib\b/i.test(output)
|
|
};
|
|
}
|
|
|
|
function formatGoAccessCheck(capabilities) {
|
|
const detected = [
|
|
`GeoIP2/MMDB ${capabilities.geoIpMmdb ? "enabled" : "not detected"}`,
|
|
`OpenSSL ${capabilities.openSsl ? "enabled" : "not detected"}`,
|
|
`Zlib ${capabilities.zlib ? "enabled" : "not detected"}`
|
|
].join(", ");
|
|
const summary = `GoAccess ${capabilities.version}: ${detected}.`;
|
|
|
|
if (capabilities.zlib) {
|
|
return summary;
|
|
}
|
|
|
|
return `${summary} This build has no Zlib support. That is valid for regular mode: `
|
|
+ "the current analytics log and its optional uncompressed .1 rotation are read "
|
|
+ "directly; older .gz logs are not imported.";
|
|
}
|
|
|
|
function parseGoAccessDate(value) {
|
|
if (typeof value !== "string") {
|
|
return null;
|
|
}
|
|
if (parseIsoDate(value)) {
|
|
return value;
|
|
}
|
|
if (/^\d{8}$/.test(value)) {
|
|
return parseIsoDate(`${value.slice(0, 4)}-${value.slice(4, 6)}-${value.slice(6, 8)}`);
|
|
}
|
|
|
|
const match = /^(\d{2})\/([A-Za-z]{3})\/(\d{4})$/.exec(value);
|
|
const months = {
|
|
Jan: "01", Feb: "02", Mar: "03", Apr: "04", May: "05", Jun: "06",
|
|
Jul: "07", Aug: "08", Sep: "09", Oct: "10", Nov: "11", Dec: "12"
|
|
};
|
|
|
|
return match && months[match[2]]
|
|
? parseIsoDate(`${match[3]}-${months[match[2]]}-${match[1]}`)
|
|
: null;
|
|
}
|
|
|
|
function validateReport(report, combined) {
|
|
if (!report || typeof report !== "object" || Array.isArray(report)) {
|
|
throw new Error("GoAccess JSON report must be an object");
|
|
}
|
|
const requiredPanels = ["visitors", "requests", "status_codes", "geolocation"];
|
|
if (combined) {
|
|
requiredPanels.push("vhosts");
|
|
}
|
|
if (!report.general || typeof report.general !== "object") {
|
|
throw new Error("GoAccess JSON report has no general summary");
|
|
}
|
|
for (const panel of requiredPanels) {
|
|
if (!report[panel] || !Array.isArray(report[panel].data)) {
|
|
throw new Error(`GoAccess JSON report has no ${panel} panel`);
|
|
}
|
|
}
|
|
|
|
extractDailyVisits(report);
|
|
return report;
|
|
}
|
|
|
|
function readAndValidateReport(jsonPath, htmlPath, combined) {
|
|
let report;
|
|
try {
|
|
report = JSON.parse(fs.readFileSync(jsonPath, "utf8"));
|
|
} catch (error) {
|
|
throw new Error(`GoAccess JSON output is invalid: ${error.message}`);
|
|
}
|
|
validateReport(report, combined);
|
|
|
|
const html = fs.readFileSync(htmlPath, "utf8");
|
|
if (html.length < 100 || !/<html(?:\s|>)/i.test(html)) {
|
|
throw new Error("GoAccess HTML output is missing or implausibly small");
|
|
}
|
|
|
|
return report;
|
|
}
|
|
|
|
function extractDailyVisits(report) {
|
|
const daily = {};
|
|
|
|
for (const row of report.visitors.data) {
|
|
const date = parseGoAccessDate(row && row.data);
|
|
const visits = row && row.visitors && row.visitors.count;
|
|
|
|
if (!date || !Number.isSafeInteger(visits) || visits < 0) {
|
|
throw new Error("GoAccess visitors panel contains invalid daily data");
|
|
}
|
|
if (Object.hasOwn(daily, date)) {
|
|
throw new Error(`GoAccess visitors panel contains duplicate date ${date}`);
|
|
}
|
|
daily[date] = visits;
|
|
}
|
|
|
|
return daily;
|
|
}
|
|
|
|
function loadCounterState(statePath) {
|
|
if (!fs.existsSync(statePath)) {
|
|
return { schemaVersion: STATE_SCHEMA_VERSION, sites: {} };
|
|
}
|
|
|
|
const state = readJson(statePath, "counter state");
|
|
if (state.schemaVersion !== STATE_SCHEMA_VERSION
|
|
|| !state.sites || typeof state.sites !== "object" || Array.isArray(state.sites)) {
|
|
throw new ConfigurationError("counter state has an unsupported structure");
|
|
}
|
|
|
|
for (const [siteId, site] of Object.entries(state.sites)) {
|
|
if (!site || typeof site !== "object" || !parseIsoDate(site.since)
|
|
|| !site.dailyVisits || typeof site.dailyVisits !== "object"
|
|
|| Array.isArray(site.dailyVisits)) {
|
|
throw new ConfigurationError(`counter state for ${siteId} is invalid`);
|
|
}
|
|
for (const [date, visits] of Object.entries(site.dailyVisits)) {
|
|
if (!parseIsoDate(date) || !Number.isSafeInteger(visits) || visits < 0) {
|
|
throw new ConfigurationError(`counter state value for ${siteId}/${date} is invalid`);
|
|
}
|
|
}
|
|
}
|
|
|
|
return state;
|
|
}
|
|
|
|
function updateCounterState(state, site, report) {
|
|
const current = state.sites[site.id];
|
|
if (current && current.since !== site.activatedOn) {
|
|
throw new ConfigurationError(
|
|
`activation date for ${site.id} differs from the existing counter state`
|
|
);
|
|
}
|
|
if (current && current.hostname !== site.hostname) {
|
|
throw new ConfigurationError(
|
|
`hostname for ${site.id} differs from the existing counter state`
|
|
);
|
|
}
|
|
|
|
const dailyVisits = current ? { ...current.dailyVisits } : {};
|
|
for (const [date, visits] of Object.entries(extractDailyVisits(report))) {
|
|
if (date >= site.activatedOn) {
|
|
dailyVisits[date] = visits;
|
|
}
|
|
}
|
|
state.sites[site.id] = {
|
|
hostname: site.hostname,
|
|
since: site.activatedOn,
|
|
dailyVisits
|
|
};
|
|
}
|
|
|
|
function publicPayload(state, site, now) {
|
|
const siteState = state.sites[site.id];
|
|
let visits = 0;
|
|
|
|
for (const [date, value] of Object.entries(siteState.dailyVisits)) {
|
|
if (date >= site.activatedOn) {
|
|
visits += value;
|
|
if (!Number.isSafeInteger(visits) || visits > MAX_SAFE_INTEGER) {
|
|
throw new Error(`public visit total for ${site.id} exceeds the safe integer range`);
|
|
}
|
|
}
|
|
}
|
|
|
|
return {
|
|
schemaVersion: PUBLIC_SCHEMA_VERSION,
|
|
visits,
|
|
since: site.activatedOn,
|
|
updatedAt: now.toISOString().replace(/\.\d{3}Z$/, "Z")
|
|
};
|
|
}
|
|
|
|
function atomicWriteFile(destination, content, mode = 0o640) {
|
|
const directory = path.dirname(destination);
|
|
const temporary = path.join(
|
|
directory,
|
|
`.${path.basename(destination)}.${process.pid}.${Date.now()}.tmp`
|
|
);
|
|
const handle = fs.openSync(temporary, "wx", mode);
|
|
|
|
try {
|
|
fs.writeFileSync(handle, content);
|
|
fs.fchmodSync(handle, mode);
|
|
fs.fsyncSync(handle);
|
|
} finally {
|
|
fs.closeSync(handle);
|
|
}
|
|
try {
|
|
fs.renameSync(temporary, destination);
|
|
} catch (error) {
|
|
fs.rmSync(temporary, { force: true });
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
function atomicCopyFile(source, destination) {
|
|
atomicWriteFile(destination, fs.readFileSync(source));
|
|
}
|
|
|
|
function replaceDirectory(source, destination) {
|
|
fs.mkdirSync(path.dirname(destination), { recursive: true });
|
|
const backup = `${destination}.previous-${process.pid}`;
|
|
const hadDestination = fs.existsSync(destination);
|
|
|
|
if (fs.existsSync(backup)) {
|
|
throw new Error(`stale database backup blocks replacement: ${backup}`);
|
|
}
|
|
|
|
try {
|
|
if (hadDestination) {
|
|
fs.renameSync(destination, backup);
|
|
}
|
|
fs.renameSync(source, destination);
|
|
if (hadDestination) {
|
|
fs.rmSync(backup, { recursive: true, force: true });
|
|
}
|
|
} catch (error) {
|
|
if (!fs.existsSync(destination) && fs.existsSync(backup)) {
|
|
fs.renameSync(backup, destination);
|
|
}
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
function acquireLock(lockPath) {
|
|
fs.mkdirSync(path.dirname(lockPath), { recursive: true });
|
|
let descriptor;
|
|
|
|
try {
|
|
descriptor = fs.openSync(lockPath, "wx", 0o640);
|
|
fs.writeFileSync(descriptor, `${process.pid}\n`);
|
|
} catch (error) {
|
|
throw new LockError(`another analytics run is active (${lockPath})`);
|
|
}
|
|
|
|
return () => {
|
|
fs.closeSync(descriptor);
|
|
fs.rmSync(lockPath, { force: true });
|
|
};
|
|
}
|
|
|
|
function createReportJob(id, logs, outputDirectory, combined) {
|
|
return { id, logs, outputDirectory, combined };
|
|
}
|
|
|
|
function prepareReport(job, context) {
|
|
const jobDirectory = path.join(context.runDirectory, job.id);
|
|
const stagedDb = path.join(jobDirectory, "db");
|
|
const currentDb = path.join(context.registry.stateDirectory, "db", job.id);
|
|
const outputJson = path.join(jobDirectory, "report.json");
|
|
const outputHtml = path.join(jobDirectory, "report.html");
|
|
const runConfig = path.join(jobDirectory, "goaccess.conf");
|
|
let hasDatabase = false;
|
|
|
|
if (fs.existsSync(currentDb)) {
|
|
if (!fs.statSync(currentDb).isDirectory()) {
|
|
throw new Error(`GoAccess database path is not a directory: ${currentDb}`);
|
|
}
|
|
hasDatabase = fs.readdirSync(currentDb).length > 0;
|
|
}
|
|
|
|
fs.mkdirSync(jobDirectory, { recursive: true });
|
|
if (hasDatabase) {
|
|
fs.cpSync(currentDb, stagedDb, { recursive: true, errorOnExist: true });
|
|
} else {
|
|
fs.mkdirSync(stagedDb);
|
|
}
|
|
fs.writeFileSync(runConfig, renderGoAccessConfig(
|
|
context.configTemplate,
|
|
stagedDb,
|
|
hasDatabase,
|
|
context.registry.geoIpCountryDatabase
|
|
));
|
|
|
|
const args = [
|
|
...job.logs,
|
|
"--no-global-config",
|
|
"--config-file", runConfig
|
|
];
|
|
if (job.combined) {
|
|
args.push("--enable-panel=VIRTUAL_HOSTS");
|
|
}
|
|
args.push(
|
|
"--output", outputJson,
|
|
"--output", outputHtml
|
|
);
|
|
context.runGoAccess({
|
|
binary: context.goaccessBinary,
|
|
args,
|
|
id: job.id,
|
|
combined: job.combined,
|
|
outputJson,
|
|
outputHtml,
|
|
dbPath: stagedDb
|
|
});
|
|
|
|
return {
|
|
...job,
|
|
report: readAndValidateReport(outputJson, outputHtml, job.combined),
|
|
outputJson,
|
|
outputHtml,
|
|
stagedDb,
|
|
currentDb
|
|
};
|
|
}
|
|
|
|
function generateReports(options, dependencies = {}) {
|
|
const registryPath = path.resolve(options.registryPath);
|
|
const configTemplatePath = path.resolve(options.configTemplatePath);
|
|
const registry = validateRegistry(readJson(registryPath, "site registry"));
|
|
const configTemplate = fs.readFileSync(configTemplatePath, "utf8");
|
|
const runGoAccess = dependencies.runGoAccess || defaultRunGoAccess;
|
|
const checkGoAccess = dependencies.checkGoAccess || defaultCheckGoAccess;
|
|
const now = dependencies.now ? dependencies.now() : new Date();
|
|
|
|
const analyticsLogsBySite = validateInputs(registry, configTemplate);
|
|
const goAccess = checkGoAccess(options.goaccessBinary || "goaccess");
|
|
if (!goAccess || !goAccess.geoIpMmdb) {
|
|
throw new ConfigurationError(
|
|
"GoAccess must be built with GeoIP2/MMDB support (--enable-geoip=mmdb)"
|
|
);
|
|
}
|
|
if (options.check) {
|
|
return { checked: true, sites: registry.sites.length, goAccess };
|
|
}
|
|
|
|
const releaseLock = options.dryRun || dependencies.skipLock
|
|
? () => {}
|
|
: acquireLock(registry.lockFile);
|
|
let runDirectory;
|
|
|
|
try {
|
|
runDirectory = fs.mkdtempSync(path.join(
|
|
options.dryRun ? os.tmpdir() : registry.stateDirectory,
|
|
".analytics-run-"
|
|
));
|
|
|
|
const jobs = registry.sites.map(site => createReportJob(
|
|
site.id,
|
|
analyticsLogsBySite.get(site.id),
|
|
site.reportOutputDirectory,
|
|
false
|
|
));
|
|
jobs.push(createReportJob(
|
|
"combined",
|
|
registry.sites.flatMap(site => analyticsLogsBySite.get(site.id)),
|
|
registry.combined.reportOutputDirectory,
|
|
true
|
|
));
|
|
|
|
const context = {
|
|
registry,
|
|
configTemplate,
|
|
goaccessBinary: options.goaccessBinary || "goaccess",
|
|
runGoAccess,
|
|
runDirectory
|
|
};
|
|
const prepared = jobs.map(job => prepareReport(job, context));
|
|
const state = loadCounterState(registry.counterStatePath);
|
|
|
|
for (const site of registry.sites.filter(entry => entry.publicCounter)) {
|
|
const generated = prepared.find(entry => entry.id === site.id);
|
|
updateCounterState(state, site, generated.report);
|
|
}
|
|
|
|
const publicFiles = registry.sites
|
|
.filter(site => site.publicCounter)
|
|
.map(site => ({
|
|
destination: site.publicJsonPath,
|
|
content: `${JSON.stringify(publicPayload(state, site, now), null, 2)}\n`
|
|
}));
|
|
|
|
if (!options.dryRun) {
|
|
for (const report of prepared) {
|
|
atomicCopyFile(
|
|
report.outputJson,
|
|
path.join(report.outputDirectory, "report.json")
|
|
);
|
|
atomicCopyFile(
|
|
report.outputHtml,
|
|
path.join(report.outputDirectory, "report.html")
|
|
);
|
|
}
|
|
for (const report of prepared) {
|
|
replaceDirectory(report.stagedDb, report.currentDb);
|
|
}
|
|
atomicWriteFile(
|
|
registry.counterStatePath,
|
|
`${JSON.stringify(state, null, 2)}\n`
|
|
);
|
|
for (const publicFile of publicFiles) {
|
|
atomicWriteFile(publicFile.destination, publicFile.content, 0o644);
|
|
}
|
|
}
|
|
|
|
return {
|
|
checked: false,
|
|
dryRun: Boolean(options.dryRun),
|
|
sites: registry.sites.length,
|
|
reports: prepared.length,
|
|
publicCounters: publicFiles.length
|
|
};
|
|
} finally {
|
|
if (runDirectory && fs.existsSync(runDirectory)) {
|
|
fs.rmSync(runDirectory, { recursive: true, force: true });
|
|
}
|
|
releaseLock();
|
|
}
|
|
}
|
|
|
|
function parseArguments(argv) {
|
|
const options = { check: false, dryRun: false, goaccessBinary: "goaccess" };
|
|
|
|
for (let index = 0; index < argv.length; index += 1) {
|
|
const argument = argv[index];
|
|
if (argument === "--check") {
|
|
options.check = true;
|
|
} else if (argument === "--dry-run") {
|
|
options.dryRun = true;
|
|
} else if (["--registry", "--config-template", "--goaccess"].includes(argument)) {
|
|
const value = argv[index + 1];
|
|
if (!value) {
|
|
throw new ConfigurationError(`${argument} requires a value`);
|
|
}
|
|
index += 1;
|
|
if (argument === "--registry") options.registryPath = value;
|
|
if (argument === "--config-template") options.configTemplatePath = value;
|
|
if (argument === "--goaccess") options.goaccessBinary = value;
|
|
} else {
|
|
throw new ConfigurationError(`unknown argument: ${argument}`);
|
|
}
|
|
}
|
|
|
|
if (!options.registryPath || !options.configTemplatePath) {
|
|
throw new ConfigurationError(
|
|
"usage: generate-reports.js --registry FILE --config-template FILE "
|
|
+ "[--goaccess FILE] [--check|--dry-run]"
|
|
);
|
|
}
|
|
if (options.check && options.dryRun) {
|
|
throw new ConfigurationError("--check and --dry-run are mutually exclusive");
|
|
}
|
|
|
|
return options;
|
|
}
|
|
|
|
if (require.main === module) {
|
|
try {
|
|
const result = generateReports(parseArguments(process.argv.slice(2)));
|
|
const action = result.checked ? "Configuration check" : "Analytics generation";
|
|
if (result.checked) {
|
|
process.stdout.write(`${formatGoAccessCheck(result.goAccess)}\n`);
|
|
}
|
|
process.stdout.write(`${action} completed: ${JSON.stringify(result)}\n`);
|
|
} catch (error) {
|
|
process.stderr.write(`analytics: ${error.message}\n`);
|
|
process.exitCode = error.exitCode || 1;
|
|
}
|
|
}
|
|
|
|
module.exports = {
|
|
ConfigurationError,
|
|
extractDailyVisits,
|
|
formatGoAccessCheck,
|
|
generateReports,
|
|
parseGoAccessVersion,
|
|
parseArguments,
|
|
publicPayload,
|
|
updateCounterState,
|
|
validateRegistry,
|
|
validateReport
|
|
};
|