/** * The 30-day uptime ribbons and the dashboard's "since you last looked" summary — both built from what the health checks wrote * down in the audit log. * * Run with: npm test --workspace server */ import { test, describe, before, after } from "node:test"; import assert from "node:assert/strict"; import { mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import type { AddressInfo } from "node:net"; import type { Server } from "node:http"; const dir = mkdtempSync(join(tmpdir(), "hm-uptime-")); process.env.DATABASE_PATH = join(dir, "test.sqlite"); process.env.SESSION_DIR = join(dir, "sessions"); process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32); const express = (await import("express")).default; const { db } = await import("../src/db/client.js"); const { users, servers, integrations, auditLog, maintenanceWindows } = await import("../src/db/schema.js"); const { runMigrations } = await import("../src/db/migrate.js"); const { meRouter } = await import("../src/routes/me.js"); const { uptimeRouter } = await import("../src/routes/uptime.js"); const { computeBuckets, uptimePercent, getUptime } = await import("../src/services/uptimeHistory.js"); const { foldMarks, getChangesSince } = await import("../src/services/changesSince.js"); const { trackIntegrationHealth } = await import("../src/services/integrationHealthMonitor.js"); await runMigrations(); const HOUR = 3600_000; const DAY = 24 * HOUR; const NOW = Math.floor(Date.now() / 1000) * 1000; const dbTime = (ms: number) => new Date(ms).toISOString().slice(0, 19).replace("T", " "); describe("computeBuckets", () => { test("a thing with no problems recorded has been up for as long as it has been known", () => { const buckets = computeBuckets([], NOW - 10 * DAY, NOW, 30); assert.equal(buckets.length, 30); assert.deepEqual(buckets.slice(0, 20).map((b) => b.k), new Array(20).fill(0), "nothing is known before it existed"); assert.ok(buckets.slice(20).every((b) => b.k === 1440 && b.d === 0)); assert.equal(uptimePercent(buckets), 100); }); test("a stretch offline shows as down minutes in the day it happened", () => { const buckets = computeBuckets( [ { at: NOW - 2 * HOUR, down: true }, { at: NOW - HOUR, down: false }, ], NOW - 30 * DAY, NOW, 30, ); assert.equal(buckets[29].d, 60); assert.equal(buckets[29].k, 1440); assert.equal(buckets.slice(0, 29).reduce((n, b) => n + b.d, 0), 0); assert.equal(uptimePercent(buckets), Math.round((1 - 60 / (30 * 1440)) * 1000) / 10); }); test("an outage that spans midnight is split between the days, and one still going counts up to now", () => { const buckets = computeBuckets([{ at: NOW - 36 * HOUR, down: true }], NOW - 30 * DAY, NOW, 30); assert.equal(buckets[29].d, 1440); assert.equal(buckets[28].d, 12 * 60); }); test("a first record of recovery means it was down before that", () => { const buckets = computeBuckets([{ at: NOW - 12 * HOUR, down: false }], NOW - 3 * DAY, NOW, 30); assert.equal(buckets[28].d, 1440); assert.equal(buckets[27].d, 1440); assert.equal(buckets[29].d, 12 * 60, "down until the recovery 12 hours ago"); assert.equal(buckets[26].k, 0, "nothing known before it existed"); }); test("repeated 'down' marks don't double anything", () => { const once = computeBuckets([{ at: NOW - 5 * HOUR, down: true }, { at: NOW - 3 * HOUR, down: false }], NOW - 5 * DAY, NOW, 30); const twice = computeBuckets([{ at: NOW - 5 * HOUR, down: true }, { at: NOW - 4 * HOUR, down: true }, { at: NOW - 3 * HOUR, down: false }], NOW - 5 * DAY, NOW, 30); assert.deepEqual(once, twice); }); }); describe("foldMarks", () => { test("pairs, lone recoveries and open problems", () => { const folded = foldMarks([ { at: 1, problem: false, message: null }, { at: 5, problem: true, message: "a" }, { at: 6, problem: true, message: "dup" }, { at: 9, problem: false, message: null }, { at: 20, problem: true, message: "b" }, ]); assert.deepEqual(folded, [ { start: null, end: 1, message: null }, { start: 5, end: 9, message: "a" }, { start: 20, end: null, message: "b" }, ]); }); }); describe("recording integration transitions", () => { test("the third failure is written down once, and the next success ends it", async () => { await trackIntegrationHealth("ribbon-a", false); await trackIntegrationHealth("ribbon-a", false); assert.equal((await db.select().from(auditLog)).filter((e) => e.targetId === "ribbon-a").length, 0, "two failures are not an outage yet"); await trackIntegrationHealth("ribbon-a", false); await trackIntegrationHealth("ribbon-a", false); await trackIntegrationHealth("ribbon-a", true); await trackIntegrationHealth("ribbon-a", true); const rows = (await db.select().from(auditLog)).filter((e) => e.targetId === "ribbon-a"); assert.deepEqual(rows.map((r) => r.action), ["went_failing", "recovered"]); assert.ok(rows.every((r) => r.category === "integration" && r.targetType === "integration_type")); }); test("a service the log last saw failing is closed off by its first success after a restart", async () => { await db.insert(auditLog).values({ category: "integration", action: "went_failing", targetType: "integration_type", targetId: "ribbon-b", actorLabel: "system" }); await trackIntegrationHealth("ribbon-b", true); // nothing in memory about ribbon-b: this is the first call since "start" const rows = (await db.select().from(auditLog)).filter((e) => e.targetId === "ribbon-b"); assert.deepEqual(rows.map((r) => r.action), ["went_failing", "recovered"]); await trackIntegrationHealth("ribbon-b", true); assert.equal((await db.select().from(auditLog)).filter((e) => e.targetId === "ribbon-b").length, 2, "and only once"); }); test("a healthy service writes nothing", async () => { await trackIntegrationHealth("ribbon-c", true); await trackIntegrationHealth("ribbon-c", true); assert.equal((await db.select().from(auditLog)).filter((e) => e.targetId === "ribbon-c").length, 0); }); }); describe("uptime and changes from the audit log", () => { let app: Server; let base = ""; let who = "admin"; let elsaId = 0; let annaId = 0; before(async () => { await db.insert(users).values([ { oidcSub: "admin", name: "Ada", role: "admin" }, { oidcSub: "viewer", name: "Vic", role: "viewer" }, ]); const made = await db .insert(servers) .values([ { name: "elsa", apiTokenHash: "a", apiTokenPrefix: "a", createdAt: dbTime(NOW - 40 * DAY), lastSeenAt: new Date(NOW).toISOString() }, { name: "anna", apiTokenHash: "b", apiTokenPrefix: "b", createdAt: dbTime(NOW - 40 * DAY), lastSeenAt: new Date(NOW).toISOString() }, { name: "never", apiTokenHash: "c", apiTokenPrefix: "c", createdAt: dbTime(NOW - 40 * DAY) }, ]) .returning({ id: servers.id }); elsaId = made[0].id; annaId = made[1].id; await db.insert(integrations).values({ type: "proxmox", name: "Felicia", baseUrl: "https://pve.lab", config: "{}", createdAt: dbTime(NOW - 40 * DAY) }); const entry = (offsetMs: number, category: string, action: string, targetType: string, targetId: string | number, extra: Record = {}) => ({ category, action, targetType, targetId: String(targetId), actorLabel: "system", createdAt: dbTime(NOW + offsetMs), ...extra, }); await db.insert(auditLog).values([ // elsa: offline for an hour, a while ago — finished entry(-5 * HOUR, "server", "went_offline", "server", elsaId, { detail: JSON.stringify({ message: "No report for 5 minutes" }) }), entry(-4 * HOUR, "server", "back_online", "server", elsaId), // anna: went offline and has not come back entry(-3 * HOUR, "server", "went_offline", "server", annaId), // a service that stopped answering and came back entry(-6 * HOUR, "integration", "went_failing", "integration_type", "proxmox"), entry(-5.5 * HOUR, "integration", "recovered", "integration_type", "proxmox"), // somebody else's edits, and one of Ada's own { category: "secret", action: "update", targetType: "secret", targetId: "1", actorLabel: "Olga", createdAt: dbTime(NOW - 2 * HOUR) }, { category: "secret", action: "update", targetType: "secret", targetId: "2", actorLabel: "Olga", createdAt: dbTime(NOW - 2 * HOUR) }, // before the window we ask about entry(-3 * DAY, "server", "went_offline", "server", elsaId), ]); const a = express(); a.use(express.json()); a.use((req, _res, next) => { (req as unknown as { session: unknown }).session = { user: { sub: who } }; next(); }); a.use("/api/me", meRouter); a.use("/api/uptime", uptimeRouter); app = await new Promise((resolve) => { const s = a.listen(0, "127.0.0.1", () => resolve(s)); }); base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api`; }); after(() => { app.close(); try { rmSync(dir, { recursive: true, force: true }); } catch { // Windows keeps the database file locked until the process exits. } }); test("uptime covers servers that have reported, gives them a percentage, and skips one that never has", async () => { const report = await getUptime(new Date(NOW), 30); assert.ok(report.servers[elsaId]); assert.ok(report.servers[annaId]); assert.equal(Object.keys(report.servers).length, 2); assert.equal(report.servers[annaId].buckets.length, 30); // anna stopped reporting 3 hours ago and the log has not seen her come back assert.equal(report.servers[annaId].buckets[29].d, 180); // elsa went offline 3 days ago, was recorded offline again and then back 4 hours ago: down from then until then assert.equal(report.servers[elsaId].buckets[29].d, 20 * 60); assert.equal(report.servers[elsaId].buckets[28].d, 1440); assert.equal(report.servers[elsaId].buckets[27].d, 1440); assert.equal(report.servers[elsaId].buckets[26].d, 0); assert.ok(report.servers[elsaId].percent < 100 && report.servers[elsaId].percent > 90); }); test("the uptime route answers anyone signed in", async () => { who = "viewer"; const res = await fetch(`${base}/uptime`); assert.equal(res.status, 200); const body = await res.json(); assert.equal(body.days, 30); who = "admin"; }); test("changes since: pairs, an open problem, a service blip, and what others did", async () => { const adaId = (await db.select().from(users)).find((u) => u.oidcSub === "admin")!.id; const report = await getChangesSince({ since: new Date(NOW - 12 * HOUR), now: new Date(NOW), actorId: adaId, canSeeAudit: true }); const titles = report.events.map((e) => e.title); assert.ok(titles.some((t) => /^elsa was offline for 1 h — reporting again/.test(t)), titles.join(" | ")); assert.ok(titles.some((t) => /^anna went offline 3 h ago — still offline/.test(t)), titles.join(" | ")); assert.ok(titles.some((t) => /^Felicia stopped answering for 30 min — working again/.test(t)), titles.join(" | ")); const anna = report.events.find((e) => e.title.startsWith("anna")); assert.equal(anna?.tone, "bad"); assert.equal(anna?.link, `/servers/${annaId}`); assert.equal(report.events.find((e) => e.title.startsWith("elsa"))?.detail, "No report for 5 minutes"); // Olga's two edits count; the app's own entries don't. assert.equal(report.changesByOthers, 2); }); test("a viewer isn't told about the audit log", async () => { const viewerId = (await db.select().from(users)).find((u) => u.oidcSub === "viewer")!.id; const report = await getChangesSince({ since: new Date(NOW - 12 * HOUR), now: new Date(NOW), actorId: viewerId, canSeeAudit: false }); assert.equal(report.changesByOthers, null); }); test("nothing before the start of the summary is in it", async () => { const report = await getChangesSince({ since: new Date(NOW - 2 * HOUR), now: new Date(NOW), actorId: 0, canSeeAudit: true }); assert.equal(report.events.filter((e) => e.kind !== "maintenance").length, 0, "the problems were all earlier than that"); }); test("maintenance that began, ended or was started in bulk is told in few lines", async () => { const mk = (hoursAgoStart: number, hoursAgoEnd: number, target: number, reason: string) => ({ targetType: "server" as const, targetId: target, reason, startedAt: new Date(NOW - hoursAgoStart * HOUR).toISOString(), endsAt: new Date(NOW - hoursAgoEnd * HOUR).toISOString(), createdBy: "Ada", }); const extra = await db .insert(servers) .values([1, 2, 3, 4].map((n) => ({ name: `bulk${n}`, apiTokenHash: `h${n}`, apiTokenPrefix: `p${n}` }))) .returning({ id: servers.id }); await db.insert(maintenanceWindows).values([ mk(10, 8, elsaId, "Kernel update"), // finished ...extra.map((s) => mk(7, -2, s.id, "Power work")), // four at once, still running mk(1, 1, annaId, "withdrawn"), // cancelled before it began ]); const report = await getChangesSince({ since: new Date(NOW - 12 * HOUR), now: new Date(NOW), actorId: 0, canSeeAudit: false }); const maintenance = report.events.filter((e) => e.kind === "maintenance"); assert.equal(maintenance.length, 2, maintenance.map((m) => m.title).join(" | ")); assert.ok(maintenance.some((m) => m.title === "elsa was in maintenance for 2 h" && m.detail === "Kernel update")); assert.ok(maintenance.some((m) => m.title === "4 items are in maintenance — alerts silenced" && m.detail === "Power work")); }); test("the route starts from a day ago the first time, then from where 'Got it' was pressed", async () => { let res = await fetch(`${base}/me/changes`); let body = await res.json(); assert.equal(body.firstTime, true); assert.ok(Array.isArray(body.events)); assert.equal((await fetch(`${base}/me/changes/seen`, { method: "POST" })).status, 200); res = await fetch(`${base}/me/changes`); body = await res.json(); assert.equal(body.firstTime, false); assert.equal(body.events.length, 0, "nothing has happened since just now"); assert.ok(Math.abs(Date.parse(body.since) - Date.now()) < 60_000); }); test("'Got it' only moves the signed-in person's own mark", async () => { who = "viewer"; const res = await fetch(`${base}/me/changes`); assert.equal((await res.json()).firstTime, true); who = "admin"; }); });