Files
Homelab-manager/server/test/uptimeAndChanges.test.ts
T
bobbanandClaude Sonnet 5.5 906220040e Add a since-you-last-looked summary, 30-day uptime ribbons, a maintenance timeline and hover timestamps
- Dashboard card on what changed since the user pressed Got it (users.last_looked_at,
  migration 0021), folding each problem and its recovery into one line.
- Uptime ribbons on servers and integrations, built from the audit log; integrations'
  failing/working-again moments are now recorded.
- Timeline of running and scheduled maintenance windows on the Maintenance page.
- TimeAgo component: relative times with the exact time in the chosen format on hover.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-16 21:55:04 +02:00

300 lines
14 KiB
TypeScript

/**
* The 30-day uptime ribbons and the dashboard's "since you last looked" summary — both built from what the health checks wrote
* down in the audit log.
*
* Run with: npm test --workspace server
*/
import { test, describe, before, after } from "node:test";
import assert from "node:assert/strict";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { AddressInfo } from "node:net";
import type { Server } from "node:http";
const dir = mkdtempSync(join(tmpdir(), "hm-uptime-"));
process.env.DATABASE_PATH = join(dir, "test.sqlite");
process.env.SESSION_DIR = join(dir, "sessions");
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
const express = (await import("express")).default;
const { db } = await import("../src/db/client.js");
const { users, servers, integrations, auditLog, maintenanceWindows } = await import("../src/db/schema.js");
const { runMigrations } = await import("../src/db/migrate.js");
const { meRouter } = await import("../src/routes/me.js");
const { uptimeRouter } = await import("../src/routes/uptime.js");
const { computeBuckets, uptimePercent, getUptime } = await import("../src/services/uptimeHistory.js");
const { foldMarks, getChangesSince } = await import("../src/services/changesSince.js");
const { trackIntegrationHealth } = await import("../src/services/integrationHealthMonitor.js");
await runMigrations();
const HOUR = 3600_000;
const DAY = 24 * HOUR;
const NOW = Math.floor(Date.now() / 1000) * 1000;
const dbTime = (ms: number) => new Date(ms).toISOString().slice(0, 19).replace("T", " ");
describe("computeBuckets", () => {
test("a thing with no problems recorded has been up for as long as it has been known", () => {
const buckets = computeBuckets([], NOW - 10 * DAY, NOW, 30);
assert.equal(buckets.length, 30);
assert.deepEqual(buckets.slice(0, 20).map((b) => b.k), new Array(20).fill(0), "nothing is known before it existed");
assert.ok(buckets.slice(20).every((b) => b.k === 1440 && b.d === 0));
assert.equal(uptimePercent(buckets), 100);
});
test("a stretch offline shows as down minutes in the day it happened", () => {
const buckets = computeBuckets(
[
{ at: NOW - 2 * HOUR, down: true },
{ at: NOW - HOUR, down: false },
],
NOW - 30 * DAY,
NOW,
30,
);
assert.equal(buckets[29].d, 60);
assert.equal(buckets[29].k, 1440);
assert.equal(buckets.slice(0, 29).reduce((n, b) => n + b.d, 0), 0);
assert.equal(uptimePercent(buckets), Math.round((1 - 60 / (30 * 1440)) * 1000) / 10);
});
test("an outage that spans midnight is split between the days, and one still going counts up to now", () => {
const buckets = computeBuckets([{ at: NOW - 36 * HOUR, down: true }], NOW - 30 * DAY, NOW, 30);
assert.equal(buckets[29].d, 1440);
assert.equal(buckets[28].d, 12 * 60);
});
test("a first record of recovery means it was down before that", () => {
const buckets = computeBuckets([{ at: NOW - 12 * HOUR, down: false }], NOW - 3 * DAY, NOW, 30);
assert.equal(buckets[28].d, 1440);
assert.equal(buckets[27].d, 1440);
assert.equal(buckets[29].d, 12 * 60, "down until the recovery 12 hours ago");
assert.equal(buckets[26].k, 0, "nothing known before it existed");
});
test("repeated 'down' marks don't double anything", () => {
const once = computeBuckets([{ at: NOW - 5 * HOUR, down: true }, { at: NOW - 3 * HOUR, down: false }], NOW - 5 * DAY, NOW, 30);
const twice = computeBuckets([{ at: NOW - 5 * HOUR, down: true }, { at: NOW - 4 * HOUR, down: true }, { at: NOW - 3 * HOUR, down: false }], NOW - 5 * DAY, NOW, 30);
assert.deepEqual(once, twice);
});
});
describe("foldMarks", () => {
test("pairs, lone recoveries and open problems", () => {
const folded = foldMarks([
{ at: 1, problem: false, message: null },
{ at: 5, problem: true, message: "a" },
{ at: 6, problem: true, message: "dup" },
{ at: 9, problem: false, message: null },
{ at: 20, problem: true, message: "b" },
]);
assert.deepEqual(folded, [
{ start: null, end: 1, message: null },
{ start: 5, end: 9, message: "a" },
{ start: 20, end: null, message: "b" },
]);
});
});
describe("recording integration transitions", () => {
test("the third failure is written down once, and the next success ends it", async () => {
await trackIntegrationHealth("ribbon-a", false);
await trackIntegrationHealth("ribbon-a", false);
assert.equal((await db.select().from(auditLog)).filter((e) => e.targetId === "ribbon-a").length, 0, "two failures are not an outage yet");
await trackIntegrationHealth("ribbon-a", false);
await trackIntegrationHealth("ribbon-a", false);
await trackIntegrationHealth("ribbon-a", true);
await trackIntegrationHealth("ribbon-a", true);
const rows = (await db.select().from(auditLog)).filter((e) => e.targetId === "ribbon-a");
assert.deepEqual(rows.map((r) => r.action), ["went_failing", "recovered"]);
assert.ok(rows.every((r) => r.category === "integration" && r.targetType === "integration_type"));
});
test("a service the log last saw failing is closed off by its first success after a restart", async () => {
await db.insert(auditLog).values({ category: "integration", action: "went_failing", targetType: "integration_type", targetId: "ribbon-b", actorLabel: "system" });
await trackIntegrationHealth("ribbon-b", true); // nothing in memory about ribbon-b: this is the first call since "start"
const rows = (await db.select().from(auditLog)).filter((e) => e.targetId === "ribbon-b");
assert.deepEqual(rows.map((r) => r.action), ["went_failing", "recovered"]);
await trackIntegrationHealth("ribbon-b", true);
assert.equal((await db.select().from(auditLog)).filter((e) => e.targetId === "ribbon-b").length, 2, "and only once");
});
test("a healthy service writes nothing", async () => {
await trackIntegrationHealth("ribbon-c", true);
await trackIntegrationHealth("ribbon-c", true);
assert.equal((await db.select().from(auditLog)).filter((e) => e.targetId === "ribbon-c").length, 0);
});
});
describe("uptime and changes from the audit log", () => {
let app: Server;
let base = "";
let who = "admin";
let elsaId = 0;
let annaId = 0;
before(async () => {
await db.insert(users).values([
{ oidcSub: "admin", name: "Ada", role: "admin" },
{ oidcSub: "viewer", name: "Vic", role: "viewer" },
]);
const made = await db
.insert(servers)
.values([
{ name: "elsa", apiTokenHash: "a", apiTokenPrefix: "a", createdAt: dbTime(NOW - 40 * DAY), lastSeenAt: new Date(NOW).toISOString() },
{ name: "anna", apiTokenHash: "b", apiTokenPrefix: "b", createdAt: dbTime(NOW - 40 * DAY), lastSeenAt: new Date(NOW).toISOString() },
{ name: "never", apiTokenHash: "c", apiTokenPrefix: "c", createdAt: dbTime(NOW - 40 * DAY) },
])
.returning({ id: servers.id });
elsaId = made[0].id;
annaId = made[1].id;
await db.insert(integrations).values({ type: "proxmox", name: "Felicia", baseUrl: "https://pve.lab", config: "{}", createdAt: dbTime(NOW - 40 * DAY) });
const entry = (offsetMs: number, category: string, action: string, targetType: string, targetId: string | number, extra: Record<string, unknown> = {}) => ({
category,
action,
targetType,
targetId: String(targetId),
actorLabel: "system",
createdAt: dbTime(NOW + offsetMs),
...extra,
});
await db.insert(auditLog).values([
// elsa: offline for an hour, a while ago — finished
entry(-5 * HOUR, "server", "went_offline", "server", elsaId, { detail: JSON.stringify({ message: "No report for 5 minutes" }) }),
entry(-4 * HOUR, "server", "back_online", "server", elsaId),
// anna: went offline and has not come back
entry(-3 * HOUR, "server", "went_offline", "server", annaId),
// a service that stopped answering and came back
entry(-6 * HOUR, "integration", "went_failing", "integration_type", "proxmox"),
entry(-5.5 * HOUR, "integration", "recovered", "integration_type", "proxmox"),
// somebody else's edits, and one of Ada's own
{ category: "secret", action: "update", targetType: "secret", targetId: "1", actorLabel: "Olga", createdAt: dbTime(NOW - 2 * HOUR) },
{ category: "secret", action: "update", targetType: "secret", targetId: "2", actorLabel: "Olga", createdAt: dbTime(NOW - 2 * HOUR) },
// before the window we ask about
entry(-3 * DAY, "server", "went_offline", "server", elsaId),
]);
const a = express();
a.use(express.json());
a.use((req, _res, next) => {
(req as unknown as { session: unknown }).session = { user: { sub: who } };
next();
});
a.use("/api/me", meRouter);
a.use("/api/uptime", uptimeRouter);
app = await new Promise<Server>((resolve) => {
const s = a.listen(0, "127.0.0.1", () => resolve(s));
});
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api`;
});
after(() => {
app.close();
try {
rmSync(dir, { recursive: true, force: true });
} catch {
// Windows keeps the database file locked until the process exits.
}
});
test("uptime covers servers that have reported, gives them a percentage, and skips one that never has", async () => {
const report = await getUptime(new Date(NOW), 30);
assert.ok(report.servers[elsaId]);
assert.ok(report.servers[annaId]);
assert.equal(Object.keys(report.servers).length, 2);
assert.equal(report.servers[annaId].buckets.length, 30);
// anna stopped reporting 3 hours ago and the log has not seen her come back
assert.equal(report.servers[annaId].buckets[29].d, 180);
// elsa went offline 3 days ago, was recorded offline again and then back 4 hours ago: down from then until then
assert.equal(report.servers[elsaId].buckets[29].d, 20 * 60);
assert.equal(report.servers[elsaId].buckets[28].d, 1440);
assert.equal(report.servers[elsaId].buckets[27].d, 1440);
assert.equal(report.servers[elsaId].buckets[26].d, 0);
assert.ok(report.servers[elsaId].percent < 100 && report.servers[elsaId].percent > 90);
});
test("the uptime route answers anyone signed in", async () => {
who = "viewer";
const res = await fetch(`${base}/uptime`);
assert.equal(res.status, 200);
const body = await res.json();
assert.equal(body.days, 30);
who = "admin";
});
test("changes since: pairs, an open problem, a service blip, and what others did", async () => {
const adaId = (await db.select().from(users)).find((u) => u.oidcSub === "admin")!.id;
const report = await getChangesSince({ since: new Date(NOW - 12 * HOUR), now: new Date(NOW), actorId: adaId, canSeeAudit: true });
const titles = report.events.map((e) => e.title);
assert.ok(titles.some((t) => /^elsa was offline for 1 h — reporting again/.test(t)), titles.join(" | "));
assert.ok(titles.some((t) => /^anna went offline 3 h ago — still offline/.test(t)), titles.join(" | "));
assert.ok(titles.some((t) => /^Felicia stopped answering for 30 min — working again/.test(t)), titles.join(" | "));
const anna = report.events.find((e) => e.title.startsWith("anna"));
assert.equal(anna?.tone, "bad");
assert.equal(anna?.link, `/servers/${annaId}`);
assert.equal(report.events.find((e) => e.title.startsWith("elsa"))?.detail, "No report for 5 minutes");
// Olga's two edits count; the app's own entries don't.
assert.equal(report.changesByOthers, 2);
});
test("a viewer isn't told about the audit log", async () => {
const viewerId = (await db.select().from(users)).find((u) => u.oidcSub === "viewer")!.id;
const report = await getChangesSince({ since: new Date(NOW - 12 * HOUR), now: new Date(NOW), actorId: viewerId, canSeeAudit: false });
assert.equal(report.changesByOthers, null);
});
test("nothing before the start of the summary is in it", async () => {
const report = await getChangesSince({ since: new Date(NOW - 2 * HOUR), now: new Date(NOW), actorId: 0, canSeeAudit: true });
assert.equal(report.events.filter((e) => e.kind !== "maintenance").length, 0, "the problems were all earlier than that");
});
test("maintenance that began, ended or was started in bulk is told in few lines", async () => {
const mk = (hoursAgoStart: number, hoursAgoEnd: number, target: number, reason: string) => ({
targetType: "server" as const,
targetId: target,
reason,
startedAt: new Date(NOW - hoursAgoStart * HOUR).toISOString(),
endsAt: new Date(NOW - hoursAgoEnd * HOUR).toISOString(),
createdBy: "Ada",
});
const extra = await db
.insert(servers)
.values([1, 2, 3, 4].map((n) => ({ name: `bulk${n}`, apiTokenHash: `h${n}`, apiTokenPrefix: `p${n}` })))
.returning({ id: servers.id });
await db.insert(maintenanceWindows).values([
mk(10, 8, elsaId, "Kernel update"), // finished
...extra.map((s) => mk(7, -2, s.id, "Power work")), // four at once, still running
mk(1, 1, annaId, "withdrawn"), // cancelled before it began
]);
const report = await getChangesSince({ since: new Date(NOW - 12 * HOUR), now: new Date(NOW), actorId: 0, canSeeAudit: false });
const maintenance = report.events.filter((e) => e.kind === "maintenance");
assert.equal(maintenance.length, 2, maintenance.map((m) => m.title).join(" | "));
assert.ok(maintenance.some((m) => m.title === "elsa was in maintenance for 2 h" && m.detail === "Kernel update"));
assert.ok(maintenance.some((m) => m.title === "4 items are in maintenance — alerts silenced" && m.detail === "Power work"));
});
test("the route starts from a day ago the first time, then from where 'Got it' was pressed", async () => {
let res = await fetch(`${base}/me/changes`);
let body = await res.json();
assert.equal(body.firstTime, true);
assert.ok(Array.isArray(body.events));
assert.equal((await fetch(`${base}/me/changes/seen`, { method: "POST" })).status, 200);
res = await fetch(`${base}/me/changes`);
body = await res.json();
assert.equal(body.firstTime, false);
assert.equal(body.events.length, 0, "nothing has happened since just now");
assert.ok(Math.abs(Date.parse(body.since) - Date.now()) < 60_000);
});
test("'Got it' only moves the signed-in person's own mark", async () => {
who = "viewer";
const res = await fetch(`${base}/me/changes`);
assert.equal((await res.json()).firstTime, true);
who = "admin";
});
});