Acknowledge and hand over alerts, incident timelines, and built-in database backups
- Alerts can be acknowledged (note, assignee, duration); acknowledged alerts are left out of what needs attention (sidebar, dashboard strip, cards) but stay raised and counted by /metrics - Incidents: each time an alert is raised and goes away is recorded (alert_acks, alert_incidents, incident_notes, migration 0032), with a timeline of acks, maintenance, changes, update runs, related alerts and notes; Incidents page and a digest section - Backups of Homelab Manager's own database: daily checked VACUUM INTO copies, retention, download, check, upload, delete, and restore staged and applied at start-up before the database opens (pre-restore copy kept); alerts and /metrics for stale or failing backups - Docs: README, BACKUP, DATABASE, NOTIFICATIONS, ROLES, API; tests Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
7a5b623444
commit
2744a84719
41 files changed
+6573
-26
No files matched your search
@@ -0,0 +1,453 @@
|
||||
/**
|
||||
* Acknowledging and handing over alerts, and the incident record: when an alert began and ended, and what happened around it.
|
||||
*
|
||||
* Run with: npm test --workspace server
|
||||
*/
|
||||
import { test, describe, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { AddressInfo } from "node:net";
|
||||
import type { Server } from "node:http";
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), "hm-incidents-"));
|
||||
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
||||
process.env.SESSION_DIR = join(dir, "sessions");
|
||||
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
||||
process.env.HEALTH_STARTUP_GRACE_MS = "0";
|
||||
|
||||
const express = (await import("express")).default;
|
||||
const { db } = await import("../src/db/client.js");
|
||||
const { users, servers, auditLog, maintenanceWindows, updateRuns, alertAcks, alertIncidents, incidentNotes } = await import("../src/db/schema.js");
|
||||
const { runMigrations } = await import("../src/db/migrate.js");
|
||||
const acks = await import("../src/services/alertAcks.js");
|
||||
const inc = await import("../src/services/incidents.js");
|
||||
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
|
||||
const { getServerActivity } = await import("../src/services/serverActivity.js");
|
||||
const { buildMetrics } = await import("../src/services/metrics.js");
|
||||
const { buildDigest } = await import("../src/services/weeklyDigest.js");
|
||||
const { alertsRouter } = await import("../src/routes/alerts.js");
|
||||
const { incidentsRouter } = await import("../src/routes/incidents.js");
|
||||
const { hashToken } = await import("../src/services/tokens.js");
|
||||
const { eq } = await import("drizzle-orm");
|
||||
|
||||
await runMigrations();
|
||||
|
||||
const T0 = new Date("2026-10-17T12:00:00.000Z");
|
||||
const at = (minutes: number) => new Date(T0.getTime() + minutes * 60_000);
|
||||
const alert = (id: string, extra: Record<string, unknown> = {}) =>
|
||||
({ id, severity: "warning" as const, category: "disk" as const, source: "Server", message: `${id} is a problem`, link: null, silenced: false, target: null, ...extra }) as never;
|
||||
|
||||
let adminId = 0;
|
||||
let operatorId = 0;
|
||||
let viewerId = 0;
|
||||
before(async () => {
|
||||
const rows = await db
|
||||
.insert(users)
|
||||
.values([
|
||||
{ oidcSub: "ada", name: "Ada", role: "admin" },
|
||||
{ oidcSub: "olle", name: "Olle", role: "operator" },
|
||||
{ oidcSub: "vera", name: "Vera", role: "viewer" },
|
||||
])
|
||||
.returning();
|
||||
[adminId, operatorId, viewerId] = rows.map((r) => r.id);
|
||||
});
|
||||
after(() => {
|
||||
try {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Windows keeps the database file locked until the process exits.
|
||||
}
|
||||
});
|
||||
|
||||
describe("acknowledging an alert", () => {
|
||||
test("a note, who has it and for how long are kept; a second one replaces the first; taking it back removes it", async () => {
|
||||
const first = await acks.setAck({ alertId: "disk:server:1:/", note: " looking at it ", assigneeId: operatorId, hours: 4, actor: "Ada" }, T0);
|
||||
assert.equal(first.before, null);
|
||||
assert.equal(first.ack.note, "looking at it");
|
||||
assert.deepEqual(first.ack.assignee, { id: operatorId, name: "Olle" });
|
||||
assert.equal(first.ack.ackedBy, "Ada");
|
||||
assert.equal(first.ack.until, at(240).toISOString());
|
||||
assert.equal(first.ack.active, true);
|
||||
|
||||
const second = await acks.setAck({ alertId: "disk:server:1:/", note: "", assigneeId: null, hours: null, actor: "Olle" }, at(10));
|
||||
assert.equal(second.before!.note, "looking at it");
|
||||
assert.equal(second.ack.assignee, null);
|
||||
assert.equal(second.ack.until, null, "until the alert is gone");
|
||||
assert.equal((await db.select().from(alertAcks)).length, 1, "one per alert");
|
||||
|
||||
assert.equal((await acks.clearAck("disk:server:1:/"))!.ackedBy, "Olle");
|
||||
assert.equal(await acks.clearAck("disk:server:1:/"), null);
|
||||
assert.equal((await acks.loadAcks()).size, 0);
|
||||
});
|
||||
|
||||
test("only people who can act can be handed an alert, and the note and the time are limited", async () => {
|
||||
const refused = async (input: Partial<Parameters<typeof acks.setAck>[0]>, code: string) =>
|
||||
assert.rejects(acks.setAck({ alertId: "x", note: "", assigneeId: null, hours: null, actor: "Ada", ...input }, T0), (e: unknown) => e instanceof acks.AckError && e.code === code, code);
|
||||
await refused({ assigneeId: viewerId }, "invalid_assignee");
|
||||
await refused({ assigneeId: 99999 }, "invalid_assignee");
|
||||
await refused({ note: "x".repeat(acks.MAX_NOTE + 1) }, "note_too_long");
|
||||
await refused({ hours: 0 }, "invalid_duration");
|
||||
await refused({ hours: acks.MAX_HOURS + 1 }, "invalid_duration");
|
||||
await refused({ hours: 1.5 }, "invalid_duration");
|
||||
assert.deepEqual((await acks.assignableUsers()).map((u) => u.name), ["Ada", "Olle"], "no viewers in the list");
|
||||
});
|
||||
|
||||
test("it runs out when it was meant to, and only then does the alert count as unattended again", async () => {
|
||||
await acks.setAck({ alertId: "a:1", note: "", assigneeId: null, hours: 2, actor: "Ada" }, T0);
|
||||
await acks.setAck({ alertId: "a:2", note: "", assigneeId: null, hours: null, actor: "Ada" }, T0);
|
||||
const alerts = [
|
||||
{ id: "a:1", severity: "critical", silenced: false },
|
||||
{ id: "a:2", severity: "warning", silenced: false },
|
||||
{ id: "a:3", severity: "critical", silenced: false },
|
||||
{ id: "a:4", severity: "warning", silenced: true },
|
||||
] as { id: string; severity: "critical" | "warning" | "info"; silenced: boolean }[];
|
||||
assert.deepEqual(acks.attentionCounts(alerts, await acks.loadAcks(at(60))), { critical: 1, warning: 0, acknowledged: 2 }, "a:3 is the only one nobody has looked at; a silenced one is neither");
|
||||
const later = await acks.loadAcks(at(150));
|
||||
assert.equal(later.get("a:1")!.active, false);
|
||||
assert.deepEqual(acks.attentionCounts(alerts, later), { critical: 2, warning: 0, acknowledged: 1 }, "a:1 is unattended again; a:2 has no end");
|
||||
await db.delete(alertAcks);
|
||||
});
|
||||
|
||||
test("it goes when the alert has been gone for half an hour — not for a blip, and not if the alert is back", async () => {
|
||||
await acks.setAck({ alertId: "gone", note: "", assigneeId: null, hours: null, actor: "Ada" }, T0);
|
||||
await acks.setAck({ alertId: "stays", note: "", assigneeId: null, hours: null, actor: "Ada" }, T0);
|
||||
assert.equal(await acks.reconcileAcks(new Set(["stays"]), at(10)), 0, "ten minutes missing is a blip");
|
||||
assert.equal(await acks.reconcileAcks(new Set(["stays"]), at(29)), 0);
|
||||
assert.equal(await acks.reconcileAcks(new Set(["stays"]), at(31)), 1);
|
||||
assert.deepEqual([...(await acks.loadAcks()).keys()], ["stays"]);
|
||||
// a problem that returns later is a new one, nobody having looked at it
|
||||
assert.equal((await acks.loadAcks()).has("gone"), false);
|
||||
await db.delete(alertAcks);
|
||||
});
|
||||
});
|
||||
|
||||
describe("the incident record", () => {
|
||||
const look = (alerts: unknown[], minutes: number, options: { blind?: number; inGrace?: boolean } = {}) => inc.recordAlertSnapshot({ alerts: alerts as never, blind: options.blind ?? 0 }, at(minutes), { inGrace: options.inGrace });
|
||||
const rows = () => db.select().from(alertIncidents).orderBy(alertIncidents.id);
|
||||
|
||||
test("an alert that appears begins an incident; the same look again changes nothing; information-only alerts aren't incidents", async () => {
|
||||
await db.delete(alertIncidents);
|
||||
const a = alert("disk:server:1:/", { target: { type: "server", id: 1, name: "elsa" } });
|
||||
const r1 = await look([a, alert("info:thing", { severity: "info" })], 0);
|
||||
assert.equal(r1.opened.length, 1);
|
||||
assert.deepEqual((await look([a], 1)).opened, []);
|
||||
const [row] = await rows();
|
||||
assert.equal(row.alertKey, "disk:server:1:/");
|
||||
assert.equal(row.openedAt, at(0).toISOString());
|
||||
assert.equal(row.resolvedAt, null);
|
||||
assert.equal(row.targetName, "elsa");
|
||||
assert.equal(row.lastSeenAt, at(1).toISOString());
|
||||
assert.equal((await rows()).length, 1);
|
||||
});
|
||||
|
||||
test("it keeps the worst it got to and the latest wording", async () => {
|
||||
await look([alert("disk:server:1:/", { severity: "critical", message: "now 97% full", target: { type: "server", id: 1, name: "elsa" } })], 5);
|
||||
await look([alert("disk:server:1:/", { severity: "warning", message: "back to 91% full", target: { type: "server", id: 1, name: "elsa" } })], 6);
|
||||
const [row] = await rows();
|
||||
assert.equal(row.severity, "warning");
|
||||
assert.equal(row.peakSeverity, "critical");
|
||||
assert.equal(row.message, "back to 91% full");
|
||||
});
|
||||
|
||||
test("it is only called over after being missing for a couple of minutes in a row, and then it ended when it was first missing", async () => {
|
||||
const a = alert("disk:server:1:/", { target: { type: "server", id: 1, name: "elsa" } });
|
||||
await look([], 10);
|
||||
let [row] = await rows();
|
||||
assert.equal(row.resolvedAt, null);
|
||||
assert.equal(row.missingSince, at(10).toISOString());
|
||||
await look([a], 11); // it was only a blip
|
||||
[row] = await rows();
|
||||
assert.equal(row.missingSince, null, "back before it was called over: the count starts again");
|
||||
assert.equal(row.resolvedAt, null);
|
||||
|
||||
await look([], 20);
|
||||
await look([], 21);
|
||||
[row] = await rows();
|
||||
assert.equal(row.resolvedAt, null, "one minute isn't enough");
|
||||
const done = await look([], 23);
|
||||
assert.equal(done.resolved.length, 1);
|
||||
[row] = await rows();
|
||||
assert.equal(row.resolvedAt, at(20).toISOString(), "it ended when it first wasn't there, not when that was confirmed");
|
||||
assert.equal(row.missingSince, null);
|
||||
});
|
||||
|
||||
test("when it comes back it is a new incident, and the old one stays as it was", async () => {
|
||||
const a = alert("disk:server:1:/", { target: { type: "server", id: 1, name: "elsa" } });
|
||||
const r = await look([a], 60);
|
||||
assert.equal(r.opened.length, 1);
|
||||
const all = await rows();
|
||||
assert.equal(all.length, 2);
|
||||
assert.equal(all[0].resolvedAt, at(20).toISOString());
|
||||
assert.equal(all[1].resolvedAt, null);
|
||||
assert.equal(all[1].peakSeverity, "warning");
|
||||
});
|
||||
|
||||
test("after a check couldn't run it takes much longer to call anything over, and never in the first minutes after the app starts", async () => {
|
||||
await db.delete(alertIncidents);
|
||||
const a = alert("proxmox-backup:1:pve1", { category: "backup" });
|
||||
await look([a], 0);
|
||||
await look([], 5, { blind: 1 });
|
||||
await look([], 20, { blind: 1 });
|
||||
assert.equal((await rows())[0].resolvedAt, null, "fifteen minutes missing isn't enough while a check is failing");
|
||||
assert.equal((await look([], 36, { blind: 1 })).resolved.length, 1);
|
||||
|
||||
await db.delete(alertIncidents);
|
||||
await look([a], 100);
|
||||
await look([], 101, { inGrace: true });
|
||||
await look([], 200, { inGrace: true });
|
||||
const [row] = await rows();
|
||||
assert.equal(row.resolvedAt, null, "server alerts are held back on purpose for a while after the app starts, so missing then means nothing");
|
||||
assert.equal(row.missingSince, null);
|
||||
});
|
||||
|
||||
test("it is recorded from the alerts as they are collected, without anyone asking, and the Alerts page links to it", async () => {
|
||||
await db.delete(alertIncidents);
|
||||
const [s] = await db.insert(servers).values({ name: "quiet", apiTokenHash: hashToken("q"), apiTokenPrefix: "q", lastSeenAt: new Date(Date.now() - 5 * 3600_000).toISOString() }).returning();
|
||||
invalidateAlertsCache();
|
||||
await getAlerts(true);
|
||||
let found: typeof alertIncidents.$inferSelect | undefined;
|
||||
for (let i = 0; i < 40 && !found; i++) {
|
||||
found = (await rows()).find((r) => r.alertKey === `offline:offline:server:${s.id}`);
|
||||
if (!found) await new Promise((r) => setTimeout(r, 50));
|
||||
}
|
||||
assert.ok(found, "an incident for the offline server");
|
||||
assert.equal(found.severity, "critical");
|
||||
assert.deepEqual([found.targetType, found.targetId, found.targetName], ["server", s.id, "quiet"]);
|
||||
assert.equal((await inc.openIncidentIds([found.alertKey])).get(found.alertKey), found.id);
|
||||
await db.delete(servers).where(eq(servers.id, s.id));
|
||||
});
|
||||
|
||||
test("resolved ones are kept for a long while and then removed with their notes; open ones never", async () => {
|
||||
await db.delete(alertIncidents);
|
||||
const old = new Date(Date.now() - (inc.KEEP_DAYS + 5) * 86_400_000).toISOString();
|
||||
const [oldRow] = await db.insert(alertIncidents).values({ alertKey: "old", severity: "warning", peakSeverity: "warning", category: "disk", source: "Server", message: "old", openedAt: old, resolvedAt: old, lastSeenAt: old }).returning();
|
||||
await db.insert(incidentNotes).values({ incidentId: oldRow.id, author: "Ada", body: "n", createdAt: old });
|
||||
const [openRow] = await db.insert(alertIncidents).values({ alertKey: "ancient-open", severity: "warning", peakSeverity: "warning", category: "disk", source: "Server", message: "still open", openedAt: old, lastSeenAt: old }).returning();
|
||||
assert.equal(await inc.pruneIncidents(), 1);
|
||||
assert.deepEqual((await rows()).map((r) => r.id), [openRow.id]);
|
||||
assert.equal((await db.select().from(incidentNotes)).length, 0);
|
||||
});
|
||||
});
|
||||
|
||||
describe("the timeline around an incident", () => {
|
||||
let serverId = 0;
|
||||
let incidentId = 0;
|
||||
|
||||
before(async () => {
|
||||
await db.delete(alertIncidents);
|
||||
const [s] = await db.insert(servers).values({ name: "elsa", apiTokenHash: hashToken("e"), apiTokenPrefix: "e" }).returning();
|
||||
serverId = s.id;
|
||||
const [row] = await db
|
||||
.insert(alertIncidents)
|
||||
.values({ alertKey: `offline:offline:server:${serverId}`, severity: "critical", peakSeverity: "critical", category: "offline", source: "Server", message: "elsa hasn't reported for 2 h", targetType: "server", targetId: serverId, targetName: "elsa", openedAt: at(0).toISOString(), resolvedAt: at(95).toISOString(), lastSeenAt: at(95).toISOString() })
|
||||
.returning();
|
||||
incidentId = row.id;
|
||||
const stamp = (m: number) => at(m).toISOString().slice(0, 19).replace("T", " ");
|
||||
const audit = (m: number, category: string, action: string, detail: object, actor = "system", target: object = { targetType: "server", targetId: String(serverId) }) =>
|
||||
db.insert(auditLog).values({ actorLabel: actor, category, action, detail: JSON.stringify(detail), createdAt: stamp(m), ...target });
|
||||
await audit(1, "server", "went_offline", { message: "elsa hasn't reported for 2 h" });
|
||||
await audit(5, "alert", "ack", { alertId: row.alertKey, message: "x", note: "power cut in the cellar", assignee: "Olle", hours: 4 }, "Ada");
|
||||
await audit(6, "alert", "ack", { alertId: "some:other:alert", note: "not this one" }, "Ada");
|
||||
await audit(30, "server", "update", { name: "elsa" }, "Ada");
|
||||
await audit(90, "server", "back_online", {});
|
||||
await audit(40, "server", "update", {}, "Ada", { targetType: "server", targetId: "9999" }); // another server
|
||||
await db.insert(maintenanceWindows).values({ targetType: "server", targetId: serverId, reason: "UPS swap", startedAt: at(20).toISOString(), endsAt: at(60).toISOString(), createdBy: "Ada" });
|
||||
await db.insert(maintenanceWindows).values({ targetType: "server", targetId: serverId, reason: "withdrawn", startedAt: at(25).toISOString(), endsAt: at(25).toISOString() });
|
||||
await db.insert(updateRuns).values({ serverId, integrationId: 1, projectId: 1, templateId: 1, templateName: "Apply updates", hostLimit: "elsa", status: "done", message: "No updates are waiting now; no restart is needed.", startedBy: "Ada", startedAt: at(61).toISOString(), finishedAt: at(80).toISOString() });
|
||||
await db.insert(alertIncidents).values({ alertKey: `disk:server:${serverId}:/`, severity: "warning", peakSeverity: "warning", category: "disk", source: "Server", message: "/ is 93% full", targetType: "server", targetId: serverId, targetName: "elsa", openedAt: at(8).toISOString(), lastSeenAt: at(8).toISOString() });
|
||||
await db.insert(alertIncidents).values({ alertKey: `disk:server:${serverId}:/old`, severity: "warning", peakSeverity: "warning", category: "disk", source: "Server", message: "long ago", targetType: "server", targetId: serverId, targetName: "elsa", openedAt: at(-600).toISOString(), lastSeenAt: at(-600).toISOString() });
|
||||
await inc.addNote(incidentId, "Olle", "Confirmed: the building lost power.", at(15));
|
||||
});
|
||||
|
||||
test("it puts everything in the order it happened, and only what belongs to this incident", async () => {
|
||||
const d = (await inc.getIncident(incidentId, at(100)))!;
|
||||
assert.equal(d.incident.open, false);
|
||||
assert.equal(d.incident.durationMs, 95 * 60_000);
|
||||
const titles = d.timeline.map((e) => `${e.at.slice(11, 16)} ${e.kind}: ${e.title}`);
|
||||
assert.deepEqual(titles, [
|
||||
"12:00 alert: Alert raised (critical)",
|
||||
"12:01 health: Stopped reporting — marked offline",
|
||||
"12:05 ack: Acknowledged, handed to Olle",
|
||||
"12:08 related: Another alert was raised: disk (warning)",
|
||||
"12:15 note: Olle wrote a note",
|
||||
"12:20 maintenance: Maintenance began — alerts silenced",
|
||||
"12:30 change: Details changed",
|
||||
"12:55 maintenance: Maintenance ended".replace("12:55", "13:00"),
|
||||
"13:01 run: Update run started: Apply updates",
|
||||
"13:20 run: Update run finished",
|
||||
"13:30 health: Reporting again",
|
||||
"13:35 alert: Resolved",
|
||||
]);
|
||||
});
|
||||
|
||||
test("the details say what was written and by whom", async () => {
|
||||
const d = (await inc.getIncident(incidentId, at(100)))!;
|
||||
const ack = d.timeline.find((e) => e.kind === "ack")!;
|
||||
assert.equal(ack.actor, "Ada");
|
||||
assert.match(ack.detail!, /power cut in the cellar · for 4 h/);
|
||||
const note = d.timeline.find((e) => e.kind === "note")!;
|
||||
assert.equal(note.detail, "Confirmed: the building lost power.");
|
||||
assert.ok(note.noteId);
|
||||
const related = d.timeline.find((e) => e.kind === "related")!;
|
||||
assert.match(related.detail!, /93% full/);
|
||||
assert.ok(related.incidentId);
|
||||
assert.equal(d.timeline.find((e) => e.title === "Resolved")!.detail, "It lasted 1.6 h.");
|
||||
assert.equal(d.ack, null, "no acknowledgement to show on one that is over");
|
||||
assert.equal(d.notes.length, 1);
|
||||
assert.equal((await inc.getIncident(999999)), null);
|
||||
});
|
||||
|
||||
test("an open one runs to now and shows the acknowledgement in force", async () => {
|
||||
const [row] = await db.select().from(alertIncidents).where(eq(alertIncidents.alertKey, `disk:server:${serverId}:/`));
|
||||
await acks.setAck({ alertId: row.alertKey, note: "freeing space", assigneeId: operatorId, hours: null, actor: "Ada" }, at(9));
|
||||
const d = (await inc.getIncident(row.id, at(120)))!;
|
||||
assert.equal(d.incident.open, true);
|
||||
assert.equal(d.incident.durationMs, 112 * 60_000);
|
||||
assert.equal(d.ack!.note, "freeing space");
|
||||
assert.equal(d.ack!.assignee!.name, "Olle");
|
||||
assert.ok(!d.timeline.some((e) => e.title === "Resolved"));
|
||||
await db.delete(alertAcks);
|
||||
});
|
||||
|
||||
test("the list has the open ones first, then the rest newest first, with how many notes each has", async () => {
|
||||
const list = await inc.listIncidents({ now: at(120) });
|
||||
assert.equal(list[0].open, true);
|
||||
const firstClosed = list.findIndex((i) => !i.open);
|
||||
assert.ok(list.slice(0, firstClosed).every((i) => i.open) && list.slice(firstClosed).every((i) => !i.open));
|
||||
assert.equal(list.find((i) => i.id === incidentId)!.notes, 1);
|
||||
assert.deepEqual(list.filter((i) => !i.open).map((i) => i.openedAt), list.filter((i) => !i.open).map((i) => i.openedAt).sort().reverse());
|
||||
});
|
||||
});
|
||||
|
||||
describe("in the weekly digest", () => {
|
||||
test("the week's incidents are counted, the longest are named, and acknowledged alerts are said to be", async () => {
|
||||
const digest = await buildDigest(at(120));
|
||||
assert.match(digest.message, /INCIDENTS/);
|
||||
assert.match(digest.message, /\d+ incidents? began, 1 ended, \d+ still open/);
|
||||
assert.match(digest.message, /elsa hasn't reported for 2 h — lasted 1 h 35 min/);
|
||||
});
|
||||
});
|
||||
|
||||
describe("through the routes", () => {
|
||||
let app: Server;
|
||||
let base = "";
|
||||
let who = "ada";
|
||||
let serverId = 0;
|
||||
let alertId = "";
|
||||
let incidentId = 0;
|
||||
const call = (method: string, path: string, body?: unknown) => fetch(`${base}${path}`, { method, headers: { "content-type": "application/json" }, body: body === undefined ? undefined : JSON.stringify(body) });
|
||||
|
||||
before(async () => {
|
||||
await db.delete(alertIncidents);
|
||||
const [s] = await db.insert(servers).values({ name: "zoe", apiTokenHash: hashToken("z"), apiTokenPrefix: "z", lastSeenAt: new Date(Date.now() - 5 * 3600_000).toISOString() }).returning();
|
||||
serverId = s.id;
|
||||
alertId = `offline:offline:server:${serverId}`.replace("offline:offline", "offline:offline");
|
||||
const a = express();
|
||||
a.use(express.json());
|
||||
a.use((req, _res, next) => {
|
||||
(req as unknown as { session: unknown }).session = { user: { sub: who } };
|
||||
next();
|
||||
});
|
||||
a.use("/api/alerts", alertsRouter);
|
||||
a.use("/api/incidents", incidentsRouter);
|
||||
app = await new Promise<Server>((resolve) => {
|
||||
const x = a.listen(0, "127.0.0.1", () => resolve(x));
|
||||
});
|
||||
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api`;
|
||||
invalidateAlertsCache();
|
||||
const report = await (await call("GET", "/alerts?refresh=1")).json();
|
||||
alertId = report.alerts.find((x: { id: string }) => x.id.includes(`:server:${serverId}`)).id;
|
||||
for (let i = 0; i < 40 && !incidentId; i++) {
|
||||
incidentId = (await db.select().from(alertIncidents)).find((r) => r.alertKey === alertId)?.id ?? 0;
|
||||
if (!incidentId) await new Promise((r) => setTimeout(r, 50));
|
||||
}
|
||||
});
|
||||
after(() => app.close());
|
||||
|
||||
test("operators acknowledge and hand over; viewers can look but not; an alert that isn't there can't be", async () => {
|
||||
who = "vera";
|
||||
assert.equal((await call("PUT", "/alerts/ack", { alertId })).status, 403);
|
||||
assert.equal((await call("GET", "/alerts/assignees")).status, 403);
|
||||
assert.equal((await call("DELETE", `/alerts/ack?alertId=${encodeURIComponent(alertId)}`)).status, 403);
|
||||
assert.equal((await call("GET", "/alerts")).status, 200);
|
||||
|
||||
who = "olle";
|
||||
const names = (await (await call("GET", "/alerts/assignees")).json()).users.map((u: { name: string }) => u.name);
|
||||
assert.deepEqual(names, ["Ada", "Olle"]);
|
||||
assert.equal((await call("PUT", "/alerts/ack", { alertId: "no:such:alert" })).status, 404);
|
||||
assert.equal((await call("PUT", "/alerts/ack", { alertId: "" })).status, 400);
|
||||
assert.equal((await call("PUT", "/alerts/ack", { alertId, hours: 0 })).status, 400);
|
||||
const bad = await call("PUT", "/alerts/ack", { alertId, assigneeId: viewerId });
|
||||
assert.equal(bad.status, 400);
|
||||
assert.equal((await bad.json()).error, "invalid_assignee");
|
||||
|
||||
const res = await call("PUT", "/alerts/ack", { alertId, note: "power cut", assigneeId: adminId, hours: 6 });
|
||||
assert.equal(res.status, 200);
|
||||
const { ack } = await res.json();
|
||||
assert.equal(ack.assignee.name, "Ada");
|
||||
assert.equal(ack.ackedBy, "Olle");
|
||||
assert.equal(ack.active, true);
|
||||
});
|
||||
|
||||
test("the Alerts page shows it on the alert, leaves it out of what needs attention, and links to the incident", async () => {
|
||||
invalidateAlertsCache();
|
||||
const report = await (await call("GET", "/alerts")).json();
|
||||
const a = report.alerts.find((x: { id: string }) => x.id === alertId);
|
||||
assert.equal(a.ack.note, "power cut");
|
||||
assert.equal(a.incidentId, incidentId);
|
||||
assert.ok(report.counts.acknowledged >= 1);
|
||||
assert.ok(report.counts.attention.critical < report.counts.critical, "the whole count is still the whole count; attention leaves out the acknowledged");
|
||||
const metrics = await buildMetrics();
|
||||
assert.match(metrics, /homelab_alerts_acknowledged [1-9]/);
|
||||
assert.match(metrics, /homelab_incidents_open [1-9]/);
|
||||
});
|
||||
|
||||
test("it is in the audit log and in the server's history, and taking it back is too", async () => {
|
||||
const entry = (await db.select().from(auditLog)).filter((e) => e.category === "alert" && e.action === "ack").pop()!;
|
||||
assert.equal(entry.targetType, "server");
|
||||
assert.equal(entry.targetId, String(serverId));
|
||||
assert.equal(JSON.parse(entry.detail!).assignee, "Ada");
|
||||
const activity = await getServerActivity(serverId);
|
||||
assert.ok(activity.some((e) => e.title === "Acknowledged, handed to Ada" && /power cut/.test(e.detail ?? "")));
|
||||
|
||||
assert.equal((await call("DELETE", `/alerts/ack?alertId=${encodeURIComponent(alertId)}`)).status, 204);
|
||||
assert.equal((await call("DELETE", `/alerts/ack?alertId=${encodeURIComponent(alertId)}`)).status, 404);
|
||||
invalidateAlertsCache();
|
||||
assert.equal((await (await call("GET", "/alerts")).json()).alerts.find((x: { id: string }) => x.id === alertId).ack, null);
|
||||
assert.ok((await getServerActivity(serverId)).some((e) => e.title === "Acknowledgement taken back"));
|
||||
});
|
||||
|
||||
test("everyone signed in can read the incidents; operators write notes; a note can be removed by its author or an administrator", async () => {
|
||||
who = "vera";
|
||||
const list = await (await call("GET", "/incidents")).json();
|
||||
assert.ok(list.incidents.some((i: { id: number }) => i.id === incidentId));
|
||||
assert.equal((await call("GET", `/incidents/${incidentId}`)).status, 200);
|
||||
assert.equal((await call("GET", "/incidents/999999")).status, 404);
|
||||
assert.equal((await call("GET", "/incidents/x")).status, 400);
|
||||
assert.equal((await call("POST", `/incidents/${incidentId}/notes`, { body: "hello" })).status, 403);
|
||||
|
||||
who = "olle";
|
||||
assert.equal((await call("POST", `/incidents/${incidentId}/notes`, { body: " " })).status, 400);
|
||||
assert.equal((await call("POST", "/incidents/999999/notes", { body: "x" })).status, 404);
|
||||
const made = await call("POST", `/incidents/${incidentId}/notes`, { body: "It was the UPS." });
|
||||
assert.equal(made.status, 201);
|
||||
const noteId = (await made.json()).id;
|
||||
const detail = await (await call("GET", `/incidents/${incidentId}`)).json();
|
||||
assert.ok(detail.timeline.some((e: { kind: string; detail: string }) => e.kind === "note" && e.detail === "It was the UPS."));
|
||||
|
||||
// another operator can't remove it, its author and an administrator can
|
||||
await db.insert(users).values({ oidcSub: "olga", name: "Olga", role: "operator" });
|
||||
who = "olga";
|
||||
assert.equal((await call("DELETE", `/incidents/${incidentId}/notes/${noteId}`)).status, 403);
|
||||
who = "olle";
|
||||
assert.equal((await call("DELETE", `/incidents/${incidentId}/notes/${noteId}`)).status, 204);
|
||||
assert.equal((await call("DELETE", `/incidents/${incidentId}/notes/${noteId}`)).status, 404);
|
||||
const again = await (await call("POST", `/incidents/${incidentId}/notes`, { body: "second" })).json();
|
||||
who = "ada";
|
||||
assert.equal((await call("DELETE", `/incidents/${incidentId}/notes/${again.id}`)).status, 204);
|
||||
const actions = (await db.select().from(auditLog)).filter((e) => e.category === "incident").map((e) => e.action);
|
||||
assert.ok(actions.includes("note") && actions.includes("note_removed"));
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,453 @@
|
||||
/**
|
||||
* Backups of Homelab Manager itself: taking them, checking them, keeping the newest, staging and applying a restore, the routes around
|
||||
* them, and what the alerts and /metrics say about how current they are.
|
||||
*
|
||||
* Run with: npm test --workspace server
|
||||
*/
|
||||
import { test, describe, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { copyFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, statSync, utimesSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { AddressInfo } from "node:net";
|
||||
import type { Server } from "node:http";
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), "hm-selfbackup-"));
|
||||
process.env.DATABASE_PATH = join(dir, "data", "test.sqlite");
|
||||
process.env.SESSION_DIR = join(dir, "sessions");
|
||||
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
||||
process.env.HEALTH_STARTUP_GRACE_MS = "0";
|
||||
|
||||
const express = (await import("express")).default;
|
||||
const { createClient } = await import("@libsql/client");
|
||||
const { db, client } = await import("../src/db/client.js");
|
||||
const { users, auditLog } = await import("../src/db/schema.js");
|
||||
const { runMigrations } = await import("../src/db/migrate.js");
|
||||
const sb = await import("../src/services/selfBackup.js");
|
||||
const { runScheduledBackup, cronForBackup } = await import("../src/services/selfBackupScheduler.js");
|
||||
const { applyPendingRestore } = await import("../src/db/applyRestore.js");
|
||||
const { PENDING_FILE, PENDING_NOTE, RESULT_NOTE } = await import("../src/db/restoreFiles.js");
|
||||
const { dbBackupsRouter } = await import("../src/routes/dbBackups.js");
|
||||
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
|
||||
const { buildMetrics } = await import("../src/services/metrics.js");
|
||||
const { getSettings, updateSettings, setInternalFlag } = await import("../src/services/settingsStore.js");
|
||||
const schedule = (await import("node-schedule")).default;
|
||||
|
||||
await runMigrations();
|
||||
after(async () => {
|
||||
await schedule.gracefulShutdown();
|
||||
try {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Windows keeps the database file locked until the process exits.
|
||||
}
|
||||
});
|
||||
|
||||
const at = (iso: string) => new Date(iso);
|
||||
/** Opens a copy of a backup, never the backup itself: an opened database can't be deleted on some systems until the process ends. */
|
||||
const open = (file: string) => {
|
||||
const copy = join(mkdtempSync(join(dir, "copy-")), "db.sqlite");
|
||||
copyFileSync(file, copy);
|
||||
return createClient({ url: `file:${copy}` });
|
||||
};
|
||||
const wipeBackups = () => rmSync(sb.backupDir(), { recursive: true, force: true });
|
||||
|
||||
describe("taking and checking a backup", () => {
|
||||
test("a backup is a consistent copy that opens, passes SQLite's check, and holds what was there", async () => {
|
||||
await db.insert(users).values({ oidcSub: "ada", name: "Ada", role: "admin" });
|
||||
const { info, check } = await sb.createSnapshot("manual", at("2026-10-17T12:00:00Z"));
|
||||
assert.equal(info.name, "homelab-manager-2026-10-17-12-00-00-manual.sqlite");
|
||||
assert.equal(info.kind, "manual");
|
||||
assert.ok(info.sizeBytes > 10_000);
|
||||
assert.equal(check.ok, true);
|
||||
assert.equal(check.integrity, "ok");
|
||||
assert.equal(check.migrations, sb.appMigrationCount());
|
||||
assert.equal(check.counts!.users, 1);
|
||||
|
||||
// and it really holds it: open the file on its own and read the row back
|
||||
const copy = open(join(sb.backupDir(), info.name));
|
||||
const rows = (await copy.execute("select name from users")).rows;
|
||||
copy.close();
|
||||
assert.deepEqual(rows.map((r) => r.name), ["Ada"]);
|
||||
});
|
||||
|
||||
test("changes made after it was taken are not in it", async () => {
|
||||
const { info } = await sb.createSnapshot("manual", at("2026-10-17T12:01:00Z"));
|
||||
await db.insert(users).values({ oidcSub: "bo", name: "Bo", role: "viewer" });
|
||||
const copy = open(join(sb.backupDir(), info.name));
|
||||
assert.equal(Number((await copy.execute("select count(*) as n from users")).rows[0].n), 1);
|
||||
copy.close();
|
||||
await db.delete(users).where((await import("drizzle-orm")).eq(users.oidcSub, "bo"));
|
||||
});
|
||||
|
||||
test("two taken in the same second don't collide", async () => {
|
||||
const a = await sb.createSnapshot("manual", at("2026-10-17T13:00:00Z"));
|
||||
const b = await sb.createSnapshot("manual", at("2026-10-17T13:00:00Z"));
|
||||
assert.notEqual(a.info.name, b.info.name);
|
||||
assert.match(b.info.name, /-manual-2\.sqlite$/);
|
||||
});
|
||||
|
||||
test("files that aren't a usable backup are said to be, and why", async () => {
|
||||
const junk = join(dir, "junk.sqlite");
|
||||
writeFileSync(junk, "this is not a database at all");
|
||||
assert.match((await sb.checkSnapshotFile(junk)).problem!, /isn't a database file/);
|
||||
assert.match((await sb.checkSnapshotFile(join(dir, "missing.sqlite"))).problem!, /isn't there/);
|
||||
|
||||
// a real SQLite file that isn't ours
|
||||
const other = join(dir, "other.sqlite");
|
||||
const c = createClient({ url: `file:${other}` });
|
||||
await c.execute("create table cats (name text)");
|
||||
c.close();
|
||||
assert.match((await sb.checkSnapshotFile(other)).problem!, /isn't a Homelab Manager database/);
|
||||
|
||||
// one of ours, made by a newer version (more schema changes than this one has)
|
||||
const newer = join(dir, "newer.sqlite");
|
||||
await client.execute(`VACUUM INTO '${newer.replace(/'/g, "''")}'`);
|
||||
const n = createClient({ url: `file:${newer}` });
|
||||
for (let i = 0; i < 3; i++) await n.execute({ sql: "insert into __drizzle_migrations (hash, created_at) values (?, ?)", args: [`future-${i}`, 9_999_999_999_999 + i] });
|
||||
n.close();
|
||||
const check = await sb.checkSnapshotFile(newer);
|
||||
assert.equal(check.ok, false);
|
||||
assert.equal(check.newerThanApp, true);
|
||||
assert.match(check.problem!, /newer version/);
|
||||
assert.ok(check.migrations! > check.appMigrations);
|
||||
});
|
||||
|
||||
test("an older one is fine: this version brings it up to date when it starts", async () => {
|
||||
const older = join(dir, "older.sqlite");
|
||||
await client.execute(`VACUUM INTO '${older.replace(/'/g, "''")}'`);
|
||||
const o = createClient({ url: `file:${older}` });
|
||||
await o.execute("delete from __drizzle_migrations where rowid = (select max(rowid) from __drizzle_migrations)");
|
||||
o.close();
|
||||
const check = await sb.checkSnapshotFile(older);
|
||||
assert.equal(check.ok, true);
|
||||
assert.equal(check.migrations, sb.appMigrationCount() - 1);
|
||||
});
|
||||
|
||||
test("only names of our own files are ever acted on, whatever is asked for", () => {
|
||||
for (const bad of ["../test.sqlite", "..\\test.sqlite", "homelab-manager-2026-10-17-12-00-00.sqlite/../../x", "homelab-manager-x.sqlite", "test.sqlite", "homelab-manager-2026-10-17-12-00-00.sqlite.bak", "/etc/passwd", ""]) {
|
||||
assert.equal(sb.isSnapshotName(bad), false, bad);
|
||||
assert.equal(sb.snapshotPath(bad), null, bad);
|
||||
assert.equal(sb.deleteSnapshot(bad), false, bad);
|
||||
}
|
||||
for (const good of ["homelab-manager-2026-10-17-12-00-00.sqlite", "homelab-manager-2026-10-17-12-00-00-manual.sqlite", "homelab-manager-2026-10-17-12-00-00-pre-restore-2.sqlite", "homelab-manager-2026-10-17-12-00-00-uploaded.sqlite"]) {
|
||||
assert.equal(sb.isSnapshotName(good), true, good);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("keeping the newest", () => {
|
||||
const fake = (name: string, mtime: string) => {
|
||||
mkdirSync(sb.backupDir(), { recursive: true });
|
||||
const f = join(sb.backupDir(), name);
|
||||
writeFileSync(f, "x");
|
||||
utimesSync(f, at(mtime), at(mtime));
|
||||
};
|
||||
const names = () => sb.listSnapshots().map((s) => s.name);
|
||||
|
||||
test("the newest N of the daily ones stay, and each other kind has its own, smaller allowance", () => {
|
||||
wipeBackups();
|
||||
for (let d = 1; d <= 6; d++) fake(`homelab-manager-2026-10-0${d}-03-30-00.sqlite`, `2026-10-0${d}T03:30:00Z`);
|
||||
for (let d = 1; d <= 12; d++) fake(`homelab-manager-2026-10-${String(d).padStart(2, "0")}-09-00-00-manual.sqlite`, `2026-10-${String(d).padStart(2, "0")}T09:00:00Z`);
|
||||
for (let d = 1; d <= 5; d++) fake(`homelab-manager-2026-10-0${d}-10-00-00-pre-restore.sqlite`, `2026-10-0${d}T10:00:00Z`);
|
||||
const removed = sb.pruneSnapshots(3);
|
||||
const left = names();
|
||||
assert.equal(left.filter((n) => !/-(manual|pre-restore|uploaded)/.test(n)).length, 3);
|
||||
assert.equal(left.filter((n) => n.includes("-manual")).length, 10);
|
||||
assert.equal(left.filter((n) => n.includes("-pre-restore")).length, 3);
|
||||
assert.equal(removed.length, 3 + 2 + 2);
|
||||
assert.ok(left.includes("homelab-manager-2026-10-06-03-30-00.sqlite") && !left.includes("homelab-manager-2026-10-01-03-30-00.sqlite"), "the oldest went, not the newest");
|
||||
// other files in the folder are left alone
|
||||
writeFileSync(join(sb.backupDir(), "notes.txt"), "mine");
|
||||
sb.pruneSnapshots(1);
|
||||
assert.ok(existsSync(join(sb.backupDir(), "notes.txt")));
|
||||
wipeBackups();
|
||||
});
|
||||
|
||||
test("a scheduled run takes one and trims to what is kept, and records that it worked", async () => {
|
||||
wipeBackups();
|
||||
await updateSettings({ selfBackup: { keep: 2 } });
|
||||
for (const d of ["2026-10-14T03:30:00Z", "2026-10-15T03:30:00Z", "2026-10-16T03:30:00Z"]) assert.ok(await runScheduledBackup(at(d)));
|
||||
assert.deepEqual(names(), ["homelab-manager-2026-10-16-03-30-00.sqlite", "homelab-manager-2026-10-15-03-30-00.sqlite"]);
|
||||
const last = await sb.lastAttempt();
|
||||
assert.equal(last!.ok, true);
|
||||
assert.equal(last!.kind, "scheduled");
|
||||
assert.equal(cronForBackup("03:30"), "30 3 * * *");
|
||||
assert.equal(cronForBackup("23:05"), "5 23 * * *");
|
||||
await updateSettings({ selfBackup: { keep: 14 } });
|
||||
});
|
||||
|
||||
test("a backup that can't be made is recorded as having failed, says why, and leaves no half file", async () => {
|
||||
wipeBackups();
|
||||
const realExecute = client.execute.bind(client);
|
||||
(client as unknown as { execute: unknown }).execute = async (stmt: unknown) => {
|
||||
if (typeof stmt === "string" && stmt.startsWith("VACUUM INTO")) throw new Error("SQLITE_FULL: database or disk is full");
|
||||
return realExecute(stmt as never);
|
||||
};
|
||||
try {
|
||||
await assert.rejects(sb.createSnapshot("scheduled", at("2026-10-17T03:30:00Z")), /SQLITE_FULL/);
|
||||
const last = await sb.lastAttempt();
|
||||
assert.equal(last!.ok, false);
|
||||
assert.match(last!.error!, /disk is full/);
|
||||
assert.deepEqual(sb.listSnapshots(), [], "nothing half-written is left in the backups folder");
|
||||
assert.equal(await runScheduledBackup(at("2026-10-17T03:31:00Z")), null, "the scheduled run swallows it (and says so through the channels)");
|
||||
} finally {
|
||||
(client as unknown as { execute: unknown }).execute = realExecute;
|
||||
}
|
||||
assert.ok(await runScheduledBackup(at("2026-10-17T03:32:00Z")), "and the next one works");
|
||||
assert.equal((await sb.lastAttempt())!.ok, true);
|
||||
wipeBackups();
|
||||
});
|
||||
});
|
||||
|
||||
describe("restoring", () => {
|
||||
test("a staged restore is checked first, kept beside the database, can be cancelled, and refuses what isn't usable", async () => {
|
||||
wipeBackups();
|
||||
const { info } = await sb.createSnapshot("manual", at("2026-10-17T15:00:00Z"));
|
||||
assert.equal(sb.stagedRestore(), null);
|
||||
|
||||
await assert.rejects(sb.stageRestore("../../etc/passwd", "Ada"), /no backup with that name/);
|
||||
await assert.rejects(sb.stageRestore("homelab-manager-2026-01-01-00-00-00.sqlite", "Ada"), /no backup with that name/);
|
||||
writeFileSync(join(sb.backupDir(), "homelab-manager-2026-10-17-16-00-00-uploaded.sqlite"), "garbage");
|
||||
await assert.rejects(sb.stageRestore("homelab-manager-2026-10-17-16-00-00-uploaded.sqlite", "Ada"), /isn't a database file/);
|
||||
assert.equal(sb.stagedRestore(), null, "a refused one stages nothing");
|
||||
|
||||
const check = await sb.stageRestore(info.name, "Ada", at("2026-10-17T15:05:00Z"));
|
||||
assert.equal(check.ok, true);
|
||||
const staged = sb.stagedRestore()!;
|
||||
assert.deepEqual(staged, { from: info.name, stagedAt: "2026-10-17T15:05:00.000Z", by: "Ada" });
|
||||
const dataDir = join(dir, "data");
|
||||
assert.ok(existsSync(join(dataDir, PENDING_FILE)));
|
||||
assert.equal(readdirSync(dataDir).some((f) => f.endsWith(".copying")), false, "no half-copied file is left");
|
||||
assert.equal(statSync(join(dataDir, PENDING_FILE)).size, info.sizeBytes);
|
||||
|
||||
assert.equal(sb.cancelStagedRestore(), true);
|
||||
assert.equal(sb.stagedRestore(), null);
|
||||
assert.equal(existsSync(join(dataDir, PENDING_FILE)), false);
|
||||
assert.equal(sb.cancelStagedRestore(), false);
|
||||
wipeBackups();
|
||||
});
|
||||
|
||||
test("at start-up the staged file replaces the database, the old one is kept, and nothing happens twice", () => {
|
||||
const folder = mkdtempSync(join(dir, "apply-"));
|
||||
const dbFile = join(folder, "app.sqlite");
|
||||
writeFileSync(dbFile, "OLD DATABASE");
|
||||
writeFileSync(`${dbFile}-wal`, "stale journal");
|
||||
writeFileSync(join(folder, PENDING_FILE), "NEW DATABASE");
|
||||
writeFileSync(join(folder, PENDING_NOTE), JSON.stringify({ from: "homelab-manager-2026-10-01-03-30-00.sqlite", by: "Ada" }));
|
||||
|
||||
const result = applyPendingRestore(dbFile, at("2026-10-17T16:00:00Z"))!;
|
||||
assert.equal(readFileSync(dbFile, "utf8"), "NEW DATABASE");
|
||||
assert.equal(existsSync(`${dbFile}-wal`), false, "a journal from the old file would be taken for the new one's");
|
||||
assert.equal(result.keptAs, "homelab-manager-2026-10-17-16-00-00-pre-restore.sqlite");
|
||||
assert.equal(readFileSync(join(folder, "backups", result.keptAs!), "utf8"), "OLD DATABASE", "what was in use is kept, not deleted");
|
||||
assert.equal(result.from, "homelab-manager-2026-10-01-03-30-00.sqlite");
|
||||
assert.equal(result.by, "Ada");
|
||||
assert.equal(existsSync(join(folder, PENDING_FILE)), false);
|
||||
assert.equal(existsSync(join(folder, PENDING_NOTE)), false);
|
||||
assert.deepEqual(JSON.parse(readFileSync(join(folder, RESULT_NOTE), "utf8")), result);
|
||||
|
||||
assert.equal(applyPendingRestore(dbFile, at("2026-10-17T16:01:00Z")), null, "nothing staged any more");
|
||||
assert.equal(readFileSync(dbFile, "utf8"), "NEW DATABASE");
|
||||
});
|
||||
|
||||
test("with no database there yet (a fresh machine), the staged one simply becomes it", () => {
|
||||
const folder = mkdtempSync(join(dir, "fresh-"));
|
||||
const dbFile = join(folder, "app.sqlite");
|
||||
writeFileSync(join(folder, PENDING_FILE), "NEW DATABASE");
|
||||
const result = applyPendingRestore(dbFile, at("2026-10-17T16:00:00Z"))!;
|
||||
assert.equal(readFileSync(dbFile, "utf8"), "NEW DATABASE");
|
||||
assert.equal(result.keptAs, null);
|
||||
assert.equal(result.from, "an uploaded backup", "no note, no name — said plainly");
|
||||
});
|
||||
|
||||
test("a restore that can't be completed leaves the database exactly as it was", () => {
|
||||
// the folder for keeping the old one can't be made (a file is in its place)
|
||||
const folder = mkdtempSync(join(dir, "fail-"));
|
||||
const dbFile = join(folder, "app.sqlite");
|
||||
writeFileSync(dbFile, "OLD DATABASE");
|
||||
writeFileSync(join(folder, PENDING_FILE), "NEW DATABASE");
|
||||
writeFileSync(join(folder, "backups"), "I am a file, not a folder");
|
||||
assert.equal(applyPendingRestore(dbFile, at("2026-10-17T16:00:00Z")), null);
|
||||
assert.equal(readFileSync(dbFile, "utf8"), "OLD DATABASE", "the database is as it was");
|
||||
assert.equal(readFileSync(join(folder, PENDING_FILE), "utf8"), "NEW DATABASE", "and the staged one is still there to try again");
|
||||
assert.equal(existsSync(join(folder, RESULT_NOTE)), false);
|
||||
|
||||
// something staged that isn't a file is never put in the database's place
|
||||
const odd = mkdtempSync(join(dir, "odd-"));
|
||||
writeFileSync(join(odd, "app.sqlite"), "OLD DATABASE");
|
||||
mkdirSync(join(odd, PENDING_FILE));
|
||||
assert.equal(applyPendingRestore(join(odd, "app.sqlite"), at("2026-10-17T16:00:00Z")), null);
|
||||
assert.equal(readFileSync(join(odd, "app.sqlite"), "utf8"), "OLD DATABASE");
|
||||
});
|
||||
});
|
||||
|
||||
describe("through the routes", () => {
|
||||
let app: Server;
|
||||
let base = "";
|
||||
let who = "ada";
|
||||
const call = (method: string, path: string, body?: unknown, raw?: Buffer) =>
|
||||
fetch(`${base}${path}`, raw ? { method, headers: { "content-type": "application/octet-stream" }, body: new Uint8Array(raw) } : { method, headers: { "content-type": "application/json" }, body: body === undefined ? undefined : JSON.stringify(body) });
|
||||
|
||||
before(async () => {
|
||||
await db.insert(users).values({ oidcSub: "operator", name: "Olle", role: "operator" });
|
||||
const a = express();
|
||||
a.use(express.json());
|
||||
a.use((req, _res, next) => {
|
||||
(req as unknown as { session: unknown }).session = { user: { sub: who } };
|
||||
next();
|
||||
});
|
||||
a.use("/api/db-backups", dbBackupsRouter);
|
||||
app = await new Promise<Server>((resolve) => {
|
||||
const x = a.listen(0, "127.0.0.1", () => resolve(x));
|
||||
});
|
||||
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/db-backups`;
|
||||
wipeBackups();
|
||||
});
|
||||
after(() => app.close());
|
||||
|
||||
test("everything about backups is for administrators only", async () => {
|
||||
who = "operator";
|
||||
for (const [method, path] of [["GET", "/"], ["POST", "/"], ["PUT", "/settings"], ["POST", "/upload"], ["GET", "/x/download"], ["POST", "/x/restore"], ["POST", "/x/check"], ["DELETE", "/x"], ["POST", "/restore/cancel"]] as const) {
|
||||
assert.equal((await call(method, path, method === "GET" || method === "DELETE" ? undefined : {})).status, 403, `${method} ${path}`);
|
||||
}
|
||||
who = "ada";
|
||||
assert.equal((await call("GET", "/")).status, 200);
|
||||
});
|
||||
|
||||
test("settings are checked and kept, and change what is kept at once", async () => {
|
||||
assert.equal((await call("PUT", "/settings", { enabled: true, time: "25:00", keep: 7 })).status, 400);
|
||||
assert.equal((await call("PUT", "/settings", { enabled: true, time: "03:30", keep: 0 })).status, 400);
|
||||
assert.equal((await call("PUT", "/settings", { enabled: true, time: "03:30", keep: 500 })).status, 400);
|
||||
const res = await call("PUT", "/settings", { enabled: true, time: "04:15", keep: 7 });
|
||||
assert.equal(res.status, 200);
|
||||
const body = await res.json();
|
||||
assert.deepEqual(body.settings, { enabled: true, time: "04:15", keep: 7 });
|
||||
assert.ok(body.nextRun, "it says when the next one is");
|
||||
assert.equal((await getSettings()).selfBackup.time, "04:15");
|
||||
const off = await (await call("PUT", "/settings", { enabled: false, time: "04:15", keep: 7 })).json();
|
||||
assert.equal(off.nextRun, null, "off means nothing is scheduled");
|
||||
await call("PUT", "/settings", { enabled: true, time: "03:30", keep: 14 });
|
||||
const entry = (await db.select().from(auditLog)).filter((e) => e.category === "db_backup" && e.action === "settings").pop()!;
|
||||
assert.deepEqual(JSON.parse(entry.detail!).to, { enabled: true, time: "03:30", keep: 14 });
|
||||
});
|
||||
|
||||
test("take one, see it listed, download it byte for byte, check it, and delete it", async () => {
|
||||
const made = await call("POST", "/");
|
||||
assert.equal(made.status, 201);
|
||||
const { snapshot, check } = await made.json();
|
||||
assert.equal(check.ok, true);
|
||||
assert.match(snapshot.name, /-manual\.sqlite$/);
|
||||
|
||||
const overview = await (await call("GET", "/")).json();
|
||||
assert.equal(overview.snapshots.length, 1);
|
||||
assert.equal(overview.snapshots[0].name, snapshot.name);
|
||||
assert.equal(overview.lastAttempt.ok, true);
|
||||
assert.equal(overview.staged, null);
|
||||
|
||||
const download = await call("GET", `/${snapshot.name}/download`);
|
||||
assert.equal(download.status, 200);
|
||||
assert.equal(download.headers.get("cache-control"), "no-store");
|
||||
assert.match(download.headers.get("content-disposition") ?? "", new RegExp(snapshot.name.replace(/\./g, "\\.")));
|
||||
const bytes = Buffer.from(await download.arrayBuffer());
|
||||
assert.deepEqual(bytes, readFileSync(join(sb.backupDir(), snapshot.name)));
|
||||
|
||||
assert.equal((await (await call("POST", `/${snapshot.name}/check`)).json()).check.integrity, "ok");
|
||||
assert.equal((await call("GET", "/homelab-manager-nonsense.sqlite/download")).status, 404);
|
||||
assert.equal((await call("GET", `/..%2Fapp.sqlite/download`)).status, 404);
|
||||
assert.equal((await call("DELETE", `/${snapshot.name}`)).status, 204);
|
||||
assert.equal((await call("DELETE", `/${snapshot.name}`)).status, 404);
|
||||
|
||||
const actions = (await db.select().from(auditLog)).filter((e) => e.category === "db_backup").map((e) => e.action);
|
||||
for (const wanted of ["create", "download", "delete"]) assert.ok(actions.includes(wanted), wanted);
|
||||
});
|
||||
|
||||
test("a backup from elsewhere can be uploaded and then restored like any other; a bad file is refused and leaves nothing", async () => {
|
||||
const { info } = await sb.createSnapshot("manual", at("2026-10-17T18:00:00Z"));
|
||||
const bytes = readFileSync(join(sb.backupDir(), info.name));
|
||||
const before = sb.listSnapshots().length;
|
||||
|
||||
const up = await call("POST", "/upload", undefined, bytes);
|
||||
assert.equal(up.status, 201);
|
||||
const uploaded = (await up.json()).snapshot;
|
||||
assert.match(uploaded.name, /-uploaded\.sqlite$/);
|
||||
assert.equal(sb.listSnapshots().length, before + 1);
|
||||
|
||||
const bad = await call("POST", "/upload", undefined, Buffer.from("not a database"));
|
||||
assert.equal(bad.status, 400);
|
||||
assert.match((await bad.json()).message, /isn't a database file/);
|
||||
assert.equal((await call("POST", "/upload", undefined, Buffer.alloc(0))).status, 400);
|
||||
assert.equal(sb.listSnapshots().length, before + 1, "nothing was kept from the bad ones");
|
||||
|
||||
const restore = await call("POST", `/${uploaded.name}/restore`);
|
||||
assert.equal(restore.status, 200);
|
||||
assert.equal((await restore.json()).staged.from, uploaded.name);
|
||||
assert.equal((await (await call("GET", "/")).json()).staged.from, uploaded.name);
|
||||
assert.equal((await call("POST", "/homelab-manager-2026-01-01-00-00-00.sqlite/restore")).status, 400);
|
||||
assert.equal((await (await call("POST", "/restore/cancel")).json()).cancelled, true);
|
||||
assert.equal((await (await call("GET", "/")).json()).staged, null);
|
||||
assert.ok((await db.select().from(auditLog)).some((e) => e.action === "restore_staged"));
|
||||
assert.ok((await db.select().from(auditLog)).some((e) => e.action === "restore_cancelled"));
|
||||
});
|
||||
|
||||
test("what the last restore did is shown until it is dismissed", async () => {
|
||||
writeFileSync(join(dir, "data", RESULT_NOTE), JSON.stringify({ restoredAt: "2026-10-17T16:00:00.000Z", from: "x.sqlite", keptAs: "y.sqlite", by: "Ada" }));
|
||||
assert.equal((await (await call("GET", "/")).json()).lastRestore.keptAs, "y.sqlite");
|
||||
assert.equal((await call("POST", "/restore-result/dismiss")).status, 200);
|
||||
assert.equal((await (await call("GET", "/")).json()).lastRestore, null);
|
||||
});
|
||||
});
|
||||
|
||||
describe("how current the backups are, in the alerts and /metrics", () => {
|
||||
const selfAlert = async () => {
|
||||
invalidateAlertsCache();
|
||||
return (await getAlerts(true)).alerts.find((a) => a.id === "backup:self-backup");
|
||||
};
|
||||
const oneAged = async (hoursAgo: number) => {
|
||||
wipeBackups();
|
||||
const { info } = await sb.createSnapshot("scheduled", new Date());
|
||||
const when = new Date(Date.now() - hoursAgo * 3600_000);
|
||||
utimesSync(join(sb.backupDir(), info.name), when, when);
|
||||
};
|
||||
|
||||
test("no alert while they are recent, or when none was ever tried; a warning when stale, critical after a week", async () => {
|
||||
wipeBackups();
|
||||
assert.equal(await selfAlert(), undefined, "nothing yet, and nothing failed: a new install isn't scolded");
|
||||
await oneAged(5);
|
||||
assert.equal(await selfAlert(), undefined);
|
||||
await oneAged(60);
|
||||
const stale = await selfAlert();
|
||||
assert.equal(stale!.severity, "warning");
|
||||
assert.match(stale!.message, /60 h old/);
|
||||
assert.equal(stale!.source, "Homelab Manager");
|
||||
assert.equal(stale!.link, "/settings/backup");
|
||||
await oneAged(24 * 8);
|
||||
assert.equal((await selfAlert())!.severity, "critical");
|
||||
assert.match((await selfAlert())!.message, /8 days old/);
|
||||
});
|
||||
|
||||
test("when the last attempt failed, the alert says why; and nothing is said at all while backups are switched off", async () => {
|
||||
await oneAged(60);
|
||||
await setInternalFlag("selfBackupLastAttempt", JSON.stringify({ at: new Date().toISOString(), ok: false, kind: "scheduled", error: "ENOSPC: no space left on device", name: null }));
|
||||
assert.match((await selfAlert())!.message, /The last attempt failed: ENOSPC: no space left on device/);
|
||||
|
||||
wipeBackups();
|
||||
const first = await selfAlert();
|
||||
assert.match(first!.message, /no backup of its own yet\. The last attempt failed|no backup of its own yet/, "a failure with nothing to show for it is said");
|
||||
|
||||
await updateSettings({ selfBackup: { enabled: false } });
|
||||
assert.equal(await selfAlert(), undefined);
|
||||
await updateSettings({ selfBackup: { enabled: true } });
|
||||
});
|
||||
|
||||
test("/metrics says how many are kept, when the newest was made and whether the last try worked", async () => {
|
||||
await oneAged(2);
|
||||
await sb.createSnapshot("manual", new Date());
|
||||
const text = await buildMetrics();
|
||||
assert.match(text, /homelab_selfbackup_snapshots 2/);
|
||||
const ts = /homelab_selfbackup_last_success_timestamp_seconds (\d+)/.exec(text);
|
||||
assert.ok(ts && Math.abs(Number(ts[1]) - Date.now() / 1000) < 120, "the manual one just made is the newest");
|
||||
assert.match(text, /homelab_selfbackup_last_attempt_ok 1/);
|
||||
wipeBackups();
|
||||
});
|
||||
});
|
||||
@@ -282,9 +282,9 @@ describe("following a run to the end", () => {
|
||||
const titles = events.filter((e) => /Update run/.test(e.title)).map((e) => e.title);
|
||||
// (the tests before this one wrote their own lines too, so this looks for the kinds rather than the number)
|
||||
for (const wanted of ["Update run started", "Update run didn't finish", "Update run finished", "Update run stopped being followed"]) assert.ok(titles.includes(wanted), `${wanted} in ${JSON.stringify([...new Set(titles)])}`);
|
||||
const failedLine = events.find((e) => e.title === "Update run didn't finish")!;
|
||||
assert.equal(failedLine.tone, "bad");
|
||||
assert.match(failedLine.detail!, /The playbook failed in Semaphore/);
|
||||
const failedLines = events.filter((e) => e.title === "Update run didn't finish");
|
||||
assert.ok(failedLines.every((e) => e.tone === "bad"), "a run that didn't finish reads as bad");
|
||||
assert.ok(failedLines.some((e) => /The playbook failed in Semaphore/.test(e.detail ?? "")), "and says why");
|
||||
assert.ok((await db.select().from(auditLog)).some((a) => a.category === "update_run" && a.action === "failed"));
|
||||
|
||||
// nothing is held in memory: a fresh pass over the table is all it takes
|
||||
|
||||
Reference in new issue
Block a user