Acknowledge and hand over alerts, incident timelines, and built-in database backups

- Alerts can be acknowledged (note, assignee, duration); acknowledged alerts are left out of what
  needs attention (sidebar, dashboard strip, cards) but stay raised and counted by /metrics
- Incidents: each time an alert is raised and goes away is recorded (alert_acks, alert_incidents,
  incident_notes, migration 0032), with a timeline of acks, maintenance, changes, update runs,
  related alerts and notes; Incidents page and a digest section
- Backups of Homelab Manager's own database: daily checked VACUUM INTO copies, retention, download,
  check, upload, delete, and restore staged and applied at start-up before the database opens
  (pre-restore copy kept); alerts and /metrics for stale or failing backups
- Docs: README, BACKUP, DATABASE, NOTIFICATIONS, ROLES, API; tests

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-18 02:21:08 +02:00
1 parent 7a5b623444
commit 2744a84719
41 files changed
+6573 -26

No files matched your search

+453
View File
@@ -0,0 +1,453 @@
/**
* Acknowledging and handing over alerts, and the incident record: when an alert began and ended, and what happened around it.
*
* Run with: npm test --workspace server
*/
import { test, describe, before, after } from "node:test";
import assert from "node:assert/strict";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { AddressInfo } from "node:net";
import type { Server } from "node:http";
const dir = mkdtempSync(join(tmpdir(), "hm-incidents-"));
process.env.DATABASE_PATH = join(dir, "test.sqlite");
process.env.SESSION_DIR = join(dir, "sessions");
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
process.env.HEALTH_STARTUP_GRACE_MS = "0";
const express = (await import("express")).default;
const { db } = await import("../src/db/client.js");
const { users, servers, auditLog, maintenanceWindows, updateRuns, alertAcks, alertIncidents, incidentNotes } = await import("../src/db/schema.js");
const { runMigrations } = await import("../src/db/migrate.js");
const acks = await import("../src/services/alertAcks.js");
const inc = await import("../src/services/incidents.js");
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
const { getServerActivity } = await import("../src/services/serverActivity.js");
const { buildMetrics } = await import("../src/services/metrics.js");
const { buildDigest } = await import("../src/services/weeklyDigest.js");
const { alertsRouter } = await import("../src/routes/alerts.js");
const { incidentsRouter } = await import("../src/routes/incidents.js");
const { hashToken } = await import("../src/services/tokens.js");
const { eq } = await import("drizzle-orm");
await runMigrations();
const T0 = new Date("2026-10-17T12:00:00.000Z");
const at = (minutes: number) => new Date(T0.getTime() + minutes * 60_000);
const alert = (id: string, extra: Record<string, unknown> = {}) =>
({ id, severity: "warning" as const, category: "disk" as const, source: "Server", message: `${id} is a problem`, link: null, silenced: false, target: null, ...extra }) as never;
let adminId = 0;
let operatorId = 0;
let viewerId = 0;
before(async () => {
const rows = await db
.insert(users)
.values([
{ oidcSub: "ada", name: "Ada", role: "admin" },
{ oidcSub: "olle", name: "Olle", role: "operator" },
{ oidcSub: "vera", name: "Vera", role: "viewer" },
])
.returning();
[adminId, operatorId, viewerId] = rows.map((r) => r.id);
});
after(() => {
try {
rmSync(dir, { recursive: true, force: true });
} catch {
// Windows keeps the database file locked until the process exits.
}
});
describe("acknowledging an alert", () => {
test("a note, who has it and for how long are kept; a second one replaces the first; taking it back removes it", async () => {
const first = await acks.setAck({ alertId: "disk:server:1:/", note: " looking at it ", assigneeId: operatorId, hours: 4, actor: "Ada" }, T0);
assert.equal(first.before, null);
assert.equal(first.ack.note, "looking at it");
assert.deepEqual(first.ack.assignee, { id: operatorId, name: "Olle" });
assert.equal(first.ack.ackedBy, "Ada");
assert.equal(first.ack.until, at(240).toISOString());
assert.equal(first.ack.active, true);
const second = await acks.setAck({ alertId: "disk:server:1:/", note: "", assigneeId: null, hours: null, actor: "Olle" }, at(10));
assert.equal(second.before!.note, "looking at it");
assert.equal(second.ack.assignee, null);
assert.equal(second.ack.until, null, "until the alert is gone");
assert.equal((await db.select().from(alertAcks)).length, 1, "one per alert");
assert.equal((await acks.clearAck("disk:server:1:/"))!.ackedBy, "Olle");
assert.equal(await acks.clearAck("disk:server:1:/"), null);
assert.equal((await acks.loadAcks()).size, 0);
});
test("only people who can act can be handed an alert, and the note and the time are limited", async () => {
const refused = async (input: Partial<Parameters<typeof acks.setAck>[0]>, code: string) =>
assert.rejects(acks.setAck({ alertId: "x", note: "", assigneeId: null, hours: null, actor: "Ada", ...input }, T0), (e: unknown) => e instanceof acks.AckError && e.code === code, code);
await refused({ assigneeId: viewerId }, "invalid_assignee");
await refused({ assigneeId: 99999 }, "invalid_assignee");
await refused({ note: "x".repeat(acks.MAX_NOTE + 1) }, "note_too_long");
await refused({ hours: 0 }, "invalid_duration");
await refused({ hours: acks.MAX_HOURS + 1 }, "invalid_duration");
await refused({ hours: 1.5 }, "invalid_duration");
assert.deepEqual((await acks.assignableUsers()).map((u) => u.name), ["Ada", "Olle"], "no viewers in the list");
});
test("it runs out when it was meant to, and only then does the alert count as unattended again", async () => {
await acks.setAck({ alertId: "a:1", note: "", assigneeId: null, hours: 2, actor: "Ada" }, T0);
await acks.setAck({ alertId: "a:2", note: "", assigneeId: null, hours: null, actor: "Ada" }, T0);
const alerts = [
{ id: "a:1", severity: "critical", silenced: false },
{ id: "a:2", severity: "warning", silenced: false },
{ id: "a:3", severity: "critical", silenced: false },
{ id: "a:4", severity: "warning", silenced: true },
] as { id: string; severity: "critical" | "warning" | "info"; silenced: boolean }[];
assert.deepEqual(acks.attentionCounts(alerts, await acks.loadAcks(at(60))), { critical: 1, warning: 0, acknowledged: 2 }, "a:3 is the only one nobody has looked at; a silenced one is neither");
const later = await acks.loadAcks(at(150));
assert.equal(later.get("a:1")!.active, false);
assert.deepEqual(acks.attentionCounts(alerts, later), { critical: 2, warning: 0, acknowledged: 1 }, "a:1 is unattended again; a:2 has no end");
await db.delete(alertAcks);
});
test("it goes when the alert has been gone for half an hour — not for a blip, and not if the alert is back", async () => {
await acks.setAck({ alertId: "gone", note: "", assigneeId: null, hours: null, actor: "Ada" }, T0);
await acks.setAck({ alertId: "stays", note: "", assigneeId: null, hours: null, actor: "Ada" }, T0);
assert.equal(await acks.reconcileAcks(new Set(["stays"]), at(10)), 0, "ten minutes missing is a blip");
assert.equal(await acks.reconcileAcks(new Set(["stays"]), at(29)), 0);
assert.equal(await acks.reconcileAcks(new Set(["stays"]), at(31)), 1);
assert.deepEqual([...(await acks.loadAcks()).keys()], ["stays"]);
// a problem that returns later is a new one, nobody having looked at it
assert.equal((await acks.loadAcks()).has("gone"), false);
await db.delete(alertAcks);
});
});
describe("the incident record", () => {
const look = (alerts: unknown[], minutes: number, options: { blind?: number; inGrace?: boolean } = {}) => inc.recordAlertSnapshot({ alerts: alerts as never, blind: options.blind ?? 0 }, at(minutes), { inGrace: options.inGrace });
const rows = () => db.select().from(alertIncidents).orderBy(alertIncidents.id);
test("an alert that appears begins an incident; the same look again changes nothing; information-only alerts aren't incidents", async () => {
await db.delete(alertIncidents);
const a = alert("disk:server:1:/", { target: { type: "server", id: 1, name: "elsa" } });
const r1 = await look([a, alert("info:thing", { severity: "info" })], 0);
assert.equal(r1.opened.length, 1);
assert.deepEqual((await look([a], 1)).opened, []);
const [row] = await rows();
assert.equal(row.alertKey, "disk:server:1:/");
assert.equal(row.openedAt, at(0).toISOString());
assert.equal(row.resolvedAt, null);
assert.equal(row.targetName, "elsa");
assert.equal(row.lastSeenAt, at(1).toISOString());
assert.equal((await rows()).length, 1);
});
test("it keeps the worst it got to and the latest wording", async () => {
await look([alert("disk:server:1:/", { severity: "critical", message: "now 97% full", target: { type: "server", id: 1, name: "elsa" } })], 5);
await look([alert("disk:server:1:/", { severity: "warning", message: "back to 91% full", target: { type: "server", id: 1, name: "elsa" } })], 6);
const [row] = await rows();
assert.equal(row.severity, "warning");
assert.equal(row.peakSeverity, "critical");
assert.equal(row.message, "back to 91% full");
});
test("it is only called over after being missing for a couple of minutes in a row, and then it ended when it was first missing", async () => {
const a = alert("disk:server:1:/", { target: { type: "server", id: 1, name: "elsa" } });
await look([], 10);
let [row] = await rows();
assert.equal(row.resolvedAt, null);
assert.equal(row.missingSince, at(10).toISOString());
await look([a], 11); // it was only a blip
[row] = await rows();
assert.equal(row.missingSince, null, "back before it was called over: the count starts again");
assert.equal(row.resolvedAt, null);
await look([], 20);
await look([], 21);
[row] = await rows();
assert.equal(row.resolvedAt, null, "one minute isn't enough");
const done = await look([], 23);
assert.equal(done.resolved.length, 1);
[row] = await rows();
assert.equal(row.resolvedAt, at(20).toISOString(), "it ended when it first wasn't there, not when that was confirmed");
assert.equal(row.missingSince, null);
});
test("when it comes back it is a new incident, and the old one stays as it was", async () => {
const a = alert("disk:server:1:/", { target: { type: "server", id: 1, name: "elsa" } });
const r = await look([a], 60);
assert.equal(r.opened.length, 1);
const all = await rows();
assert.equal(all.length, 2);
assert.equal(all[0].resolvedAt, at(20).toISOString());
assert.equal(all[1].resolvedAt, null);
assert.equal(all[1].peakSeverity, "warning");
});
test("after a check couldn't run it takes much longer to call anything over, and never in the first minutes after the app starts", async () => {
await db.delete(alertIncidents);
const a = alert("proxmox-backup:1:pve1", { category: "backup" });
await look([a], 0);
await look([], 5, { blind: 1 });
await look([], 20, { blind: 1 });
assert.equal((await rows())[0].resolvedAt, null, "fifteen minutes missing isn't enough while a check is failing");
assert.equal((await look([], 36, { blind: 1 })).resolved.length, 1);
await db.delete(alertIncidents);
await look([a], 100);
await look([], 101, { inGrace: true });
await look([], 200, { inGrace: true });
const [row] = await rows();
assert.equal(row.resolvedAt, null, "server alerts are held back on purpose for a while after the app starts, so missing then means nothing");
assert.equal(row.missingSince, null);
});
test("it is recorded from the alerts as they are collected, without anyone asking, and the Alerts page links to it", async () => {
await db.delete(alertIncidents);
const [s] = await db.insert(servers).values({ name: "quiet", apiTokenHash: hashToken("q"), apiTokenPrefix: "q", lastSeenAt: new Date(Date.now() - 5 * 3600_000).toISOString() }).returning();
invalidateAlertsCache();
await getAlerts(true);
let found: typeof alertIncidents.$inferSelect | undefined;
for (let i = 0; i < 40 && !found; i++) {
found = (await rows()).find((r) => r.alertKey === `offline:offline:server:${s.id}`);
if (!found) await new Promise((r) => setTimeout(r, 50));
}
assert.ok(found, "an incident for the offline server");
assert.equal(found.severity, "critical");
assert.deepEqual([found.targetType, found.targetId, found.targetName], ["server", s.id, "quiet"]);
assert.equal((await inc.openIncidentIds([found.alertKey])).get(found.alertKey), found.id);
await db.delete(servers).where(eq(servers.id, s.id));
});
test("resolved ones are kept for a long while and then removed with their notes; open ones never", async () => {
await db.delete(alertIncidents);
const old = new Date(Date.now() - (inc.KEEP_DAYS + 5) * 86_400_000).toISOString();
const [oldRow] = await db.insert(alertIncidents).values({ alertKey: "old", severity: "warning", peakSeverity: "warning", category: "disk", source: "Server", message: "old", openedAt: old, resolvedAt: old, lastSeenAt: old }).returning();
await db.insert(incidentNotes).values({ incidentId: oldRow.id, author: "Ada", body: "n", createdAt: old });
const [openRow] = await db.insert(alertIncidents).values({ alertKey: "ancient-open", severity: "warning", peakSeverity: "warning", category: "disk", source: "Server", message: "still open", openedAt: old, lastSeenAt: old }).returning();
assert.equal(await inc.pruneIncidents(), 1);
assert.deepEqual((await rows()).map((r) => r.id), [openRow.id]);
assert.equal((await db.select().from(incidentNotes)).length, 0);
});
});
describe("the timeline around an incident", () => {
let serverId = 0;
let incidentId = 0;
before(async () => {
await db.delete(alertIncidents);
const [s] = await db.insert(servers).values({ name: "elsa", apiTokenHash: hashToken("e"), apiTokenPrefix: "e" }).returning();
serverId = s.id;
const [row] = await db
.insert(alertIncidents)
.values({ alertKey: `offline:offline:server:${serverId}`, severity: "critical", peakSeverity: "critical", category: "offline", source: "Server", message: "elsa hasn't reported for 2 h", targetType: "server", targetId: serverId, targetName: "elsa", openedAt: at(0).toISOString(), resolvedAt: at(95).toISOString(), lastSeenAt: at(95).toISOString() })
.returning();
incidentId = row.id;
const stamp = (m: number) => at(m).toISOString().slice(0, 19).replace("T", " ");
const audit = (m: number, category: string, action: string, detail: object, actor = "system", target: object = { targetType: "server", targetId: String(serverId) }) =>
db.insert(auditLog).values({ actorLabel: actor, category, action, detail: JSON.stringify(detail), createdAt: stamp(m), ...target });
await audit(1, "server", "went_offline", { message: "elsa hasn't reported for 2 h" });
await audit(5, "alert", "ack", { alertId: row.alertKey, message: "x", note: "power cut in the cellar", assignee: "Olle", hours: 4 }, "Ada");
await audit(6, "alert", "ack", { alertId: "some:other:alert", note: "not this one" }, "Ada");
await audit(30, "server", "update", { name: "elsa" }, "Ada");
await audit(90, "server", "back_online", {});
await audit(40, "server", "update", {}, "Ada", { targetType: "server", targetId: "9999" }); // another server
await db.insert(maintenanceWindows).values({ targetType: "server", targetId: serverId, reason: "UPS swap", startedAt: at(20).toISOString(), endsAt: at(60).toISOString(), createdBy: "Ada" });
await db.insert(maintenanceWindows).values({ targetType: "server", targetId: serverId, reason: "withdrawn", startedAt: at(25).toISOString(), endsAt: at(25).toISOString() });
await db.insert(updateRuns).values({ serverId, integrationId: 1, projectId: 1, templateId: 1, templateName: "Apply updates", hostLimit: "elsa", status: "done", message: "No updates are waiting now; no restart is needed.", startedBy: "Ada", startedAt: at(61).toISOString(), finishedAt: at(80).toISOString() });
await db.insert(alertIncidents).values({ alertKey: `disk:server:${serverId}:/`, severity: "warning", peakSeverity: "warning", category: "disk", source: "Server", message: "/ is 93% full", targetType: "server", targetId: serverId, targetName: "elsa", openedAt: at(8).toISOString(), lastSeenAt: at(8).toISOString() });
await db.insert(alertIncidents).values({ alertKey: `disk:server:${serverId}:/old`, severity: "warning", peakSeverity: "warning", category: "disk", source: "Server", message: "long ago", targetType: "server", targetId: serverId, targetName: "elsa", openedAt: at(-600).toISOString(), lastSeenAt: at(-600).toISOString() });
await inc.addNote(incidentId, "Olle", "Confirmed: the building lost power.", at(15));
});
test("it puts everything in the order it happened, and only what belongs to this incident", async () => {
const d = (await inc.getIncident(incidentId, at(100)))!;
assert.equal(d.incident.open, false);
assert.equal(d.incident.durationMs, 95 * 60_000);
const titles = d.timeline.map((e) => `${e.at.slice(11, 16)} ${e.kind}: ${e.title}`);
assert.deepEqual(titles, [
"12:00 alert: Alert raised (critical)",
"12:01 health: Stopped reporting — marked offline",
"12:05 ack: Acknowledged, handed to Olle",
"12:08 related: Another alert was raised: disk (warning)",
"12:15 note: Olle wrote a note",
"12:20 maintenance: Maintenance began — alerts silenced",
"12:30 change: Details changed",
"12:55 maintenance: Maintenance ended".replace("12:55", "13:00"),
"13:01 run: Update run started: Apply updates",
"13:20 run: Update run finished",
"13:30 health: Reporting again",
"13:35 alert: Resolved",
]);
});
test("the details say what was written and by whom", async () => {
const d = (await inc.getIncident(incidentId, at(100)))!;
const ack = d.timeline.find((e) => e.kind === "ack")!;
assert.equal(ack.actor, "Ada");
assert.match(ack.detail!, /power cut in the cellar · for 4 h/);
const note = d.timeline.find((e) => e.kind === "note")!;
assert.equal(note.detail, "Confirmed: the building lost power.");
assert.ok(note.noteId);
const related = d.timeline.find((e) => e.kind === "related")!;
assert.match(related.detail!, /93% full/);
assert.ok(related.incidentId);
assert.equal(d.timeline.find((e) => e.title === "Resolved")!.detail, "It lasted 1.6 h.");
assert.equal(d.ack, null, "no acknowledgement to show on one that is over");
assert.equal(d.notes.length, 1);
assert.equal((await inc.getIncident(999999)), null);
});
test("an open one runs to now and shows the acknowledgement in force", async () => {
const [row] = await db.select().from(alertIncidents).where(eq(alertIncidents.alertKey, `disk:server:${serverId}:/`));
await acks.setAck({ alertId: row.alertKey, note: "freeing space", assigneeId: operatorId, hours: null, actor: "Ada" }, at(9));
const d = (await inc.getIncident(row.id, at(120)))!;
assert.equal(d.incident.open, true);
assert.equal(d.incident.durationMs, 112 * 60_000);
assert.equal(d.ack!.note, "freeing space");
assert.equal(d.ack!.assignee!.name, "Olle");
assert.ok(!d.timeline.some((e) => e.title === "Resolved"));
await db.delete(alertAcks);
});
test("the list has the open ones first, then the rest newest first, with how many notes each has", async () => {
const list = await inc.listIncidents({ now: at(120) });
assert.equal(list[0].open, true);
const firstClosed = list.findIndex((i) => !i.open);
assert.ok(list.slice(0, firstClosed).every((i) => i.open) && list.slice(firstClosed).every((i) => !i.open));
assert.equal(list.find((i) => i.id === incidentId)!.notes, 1);
assert.deepEqual(list.filter((i) => !i.open).map((i) => i.openedAt), list.filter((i) => !i.open).map((i) => i.openedAt).sort().reverse());
});
});
describe("in the weekly digest", () => {
test("the week's incidents are counted, the longest are named, and acknowledged alerts are said to be", async () => {
const digest = await buildDigest(at(120));
assert.match(digest.message, /INCIDENTS/);
assert.match(digest.message, /\d+ incidents? began, 1 ended, \d+ still open/);
assert.match(digest.message, /elsa hasn't reported for 2 h — lasted 1 h 35 min/);
});
});
describe("through the routes", () => {
let app: Server;
let base = "";
let who = "ada";
let serverId = 0;
let alertId = "";
let incidentId = 0;
const call = (method: string, path: string, body?: unknown) => fetch(`${base}${path}`, { method, headers: { "content-type": "application/json" }, body: body === undefined ? undefined : JSON.stringify(body) });
before(async () => {
await db.delete(alertIncidents);
const [s] = await db.insert(servers).values({ name: "zoe", apiTokenHash: hashToken("z"), apiTokenPrefix: "z", lastSeenAt: new Date(Date.now() - 5 * 3600_000).toISOString() }).returning();
serverId = s.id;
alertId = `offline:offline:server:${serverId}`.replace("offline:offline", "offline:offline");
const a = express();
a.use(express.json());
a.use((req, _res, next) => {
(req as unknown as { session: unknown }).session = { user: { sub: who } };
next();
});
a.use("/api/alerts", alertsRouter);
a.use("/api/incidents", incidentsRouter);
app = await new Promise<Server>((resolve) => {
const x = a.listen(0, "127.0.0.1", () => resolve(x));
});
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api`;
invalidateAlertsCache();
const report = await (await call("GET", "/alerts?refresh=1")).json();
alertId = report.alerts.find((x: { id: string }) => x.id.includes(`:server:${serverId}`)).id;
for (let i = 0; i < 40 && !incidentId; i++) {
incidentId = (await db.select().from(alertIncidents)).find((r) => r.alertKey === alertId)?.id ?? 0;
if (!incidentId) await new Promise((r) => setTimeout(r, 50));
}
});
after(() => app.close());
test("operators acknowledge and hand over; viewers can look but not; an alert that isn't there can't be", async () => {
who = "vera";
assert.equal((await call("PUT", "/alerts/ack", { alertId })).status, 403);
assert.equal((await call("GET", "/alerts/assignees")).status, 403);
assert.equal((await call("DELETE", `/alerts/ack?alertId=${encodeURIComponent(alertId)}`)).status, 403);
assert.equal((await call("GET", "/alerts")).status, 200);
who = "olle";
const names = (await (await call("GET", "/alerts/assignees")).json()).users.map((u: { name: string }) => u.name);
assert.deepEqual(names, ["Ada", "Olle"]);
assert.equal((await call("PUT", "/alerts/ack", { alertId: "no:such:alert" })).status, 404);
assert.equal((await call("PUT", "/alerts/ack", { alertId: "" })).status, 400);
assert.equal((await call("PUT", "/alerts/ack", { alertId, hours: 0 })).status, 400);
const bad = await call("PUT", "/alerts/ack", { alertId, assigneeId: viewerId });
assert.equal(bad.status, 400);
assert.equal((await bad.json()).error, "invalid_assignee");
const res = await call("PUT", "/alerts/ack", { alertId, note: "power cut", assigneeId: adminId, hours: 6 });
assert.equal(res.status, 200);
const { ack } = await res.json();
assert.equal(ack.assignee.name, "Ada");
assert.equal(ack.ackedBy, "Olle");
assert.equal(ack.active, true);
});
test("the Alerts page shows it on the alert, leaves it out of what needs attention, and links to the incident", async () => {
invalidateAlertsCache();
const report = await (await call("GET", "/alerts")).json();
const a = report.alerts.find((x: { id: string }) => x.id === alertId);
assert.equal(a.ack.note, "power cut");
assert.equal(a.incidentId, incidentId);
assert.ok(report.counts.acknowledged >= 1);
assert.ok(report.counts.attention.critical < report.counts.critical, "the whole count is still the whole count; attention leaves out the acknowledged");
const metrics = await buildMetrics();
assert.match(metrics, /homelab_alerts_acknowledged [1-9]/);
assert.match(metrics, /homelab_incidents_open [1-9]/);
});
test("it is in the audit log and in the server's history, and taking it back is too", async () => {
const entry = (await db.select().from(auditLog)).filter((e) => e.category === "alert" && e.action === "ack").pop()!;
assert.equal(entry.targetType, "server");
assert.equal(entry.targetId, String(serverId));
assert.equal(JSON.parse(entry.detail!).assignee, "Ada");
const activity = await getServerActivity(serverId);
assert.ok(activity.some((e) => e.title === "Acknowledged, handed to Ada" && /power cut/.test(e.detail ?? "")));
assert.equal((await call("DELETE", `/alerts/ack?alertId=${encodeURIComponent(alertId)}`)).status, 204);
assert.equal((await call("DELETE", `/alerts/ack?alertId=${encodeURIComponent(alertId)}`)).status, 404);
invalidateAlertsCache();
assert.equal((await (await call("GET", "/alerts")).json()).alerts.find((x: { id: string }) => x.id === alertId).ack, null);
assert.ok((await getServerActivity(serverId)).some((e) => e.title === "Acknowledgement taken back"));
});
test("everyone signed in can read the incidents; operators write notes; a note can be removed by its author or an administrator", async () => {
who = "vera";
const list = await (await call("GET", "/incidents")).json();
assert.ok(list.incidents.some((i: { id: number }) => i.id === incidentId));
assert.equal((await call("GET", `/incidents/${incidentId}`)).status, 200);
assert.equal((await call("GET", "/incidents/999999")).status, 404);
assert.equal((await call("GET", "/incidents/x")).status, 400);
assert.equal((await call("POST", `/incidents/${incidentId}/notes`, { body: "hello" })).status, 403);
who = "olle";
assert.equal((await call("POST", `/incidents/${incidentId}/notes`, { body: " " })).status, 400);
assert.equal((await call("POST", "/incidents/999999/notes", { body: "x" })).status, 404);
const made = await call("POST", `/incidents/${incidentId}/notes`, { body: "It was the UPS." });
assert.equal(made.status, 201);
const noteId = (await made.json()).id;
const detail = await (await call("GET", `/incidents/${incidentId}`)).json();
assert.ok(detail.timeline.some((e: { kind: string; detail: string }) => e.kind === "note" && e.detail === "It was the UPS."));
// another operator can't remove it, its author and an administrator can
await db.insert(users).values({ oidcSub: "olga", name: "Olga", role: "operator" });
who = "olga";
assert.equal((await call("DELETE", `/incidents/${incidentId}/notes/${noteId}`)).status, 403);
who = "olle";
assert.equal((await call("DELETE", `/incidents/${incidentId}/notes/${noteId}`)).status, 204);
assert.equal((await call("DELETE", `/incidents/${incidentId}/notes/${noteId}`)).status, 404);
const again = await (await call("POST", `/incidents/${incidentId}/notes`, { body: "second" })).json();
who = "ada";
assert.equal((await call("DELETE", `/incidents/${incidentId}/notes/${again.id}`)).status, 204);
const actions = (await db.select().from(auditLog)).filter((e) => e.category === "incident").map((e) => e.action);
assert.ok(actions.includes("note") && actions.includes("note_removed"));
});
});
+453
View File
@@ -0,0 +1,453 @@
/**
* Backups of Homelab Manager itself: taking them, checking them, keeping the newest, staging and applying a restore, the routes around
* them, and what the alerts and /metrics say about how current they are.
*
* Run with: npm test --workspace server
*/
import { test, describe, before, after } from "node:test";
import assert from "node:assert/strict";
import { copyFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, statSync, utimesSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { AddressInfo } from "node:net";
import type { Server } from "node:http";
const dir = mkdtempSync(join(tmpdir(), "hm-selfbackup-"));
process.env.DATABASE_PATH = join(dir, "data", "test.sqlite");
process.env.SESSION_DIR = join(dir, "sessions");
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
process.env.HEALTH_STARTUP_GRACE_MS = "0";
const express = (await import("express")).default;
const { createClient } = await import("@libsql/client");
const { db, client } = await import("../src/db/client.js");
const { users, auditLog } = await import("../src/db/schema.js");
const { runMigrations } = await import("../src/db/migrate.js");
const sb = await import("../src/services/selfBackup.js");
const { runScheduledBackup, cronForBackup } = await import("../src/services/selfBackupScheduler.js");
const { applyPendingRestore } = await import("../src/db/applyRestore.js");
const { PENDING_FILE, PENDING_NOTE, RESULT_NOTE } = await import("../src/db/restoreFiles.js");
const { dbBackupsRouter } = await import("../src/routes/dbBackups.js");
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
const { buildMetrics } = await import("../src/services/metrics.js");
const { getSettings, updateSettings, setInternalFlag } = await import("../src/services/settingsStore.js");
const schedule = (await import("node-schedule")).default;
await runMigrations();
after(async () => {
await schedule.gracefulShutdown();
try {
rmSync(dir, { recursive: true, force: true });
} catch {
// Windows keeps the database file locked until the process exits.
}
});
const at = (iso: string) => new Date(iso);
/** Opens a copy of a backup, never the backup itself: an opened database can't be deleted on some systems until the process ends. */
const open = (file: string) => {
const copy = join(mkdtempSync(join(dir, "copy-")), "db.sqlite");
copyFileSync(file, copy);
return createClient({ url: `file:${copy}` });
};
const wipeBackups = () => rmSync(sb.backupDir(), { recursive: true, force: true });
describe("taking and checking a backup", () => {
test("a backup is a consistent copy that opens, passes SQLite's check, and holds what was there", async () => {
await db.insert(users).values({ oidcSub: "ada", name: "Ada", role: "admin" });
const { info, check } = await sb.createSnapshot("manual", at("2026-10-17T12:00:00Z"));
assert.equal(info.name, "homelab-manager-2026-10-17-12-00-00-manual.sqlite");
assert.equal(info.kind, "manual");
assert.ok(info.sizeBytes > 10_000);
assert.equal(check.ok, true);
assert.equal(check.integrity, "ok");
assert.equal(check.migrations, sb.appMigrationCount());
assert.equal(check.counts!.users, 1);
// and it really holds it: open the file on its own and read the row back
const copy = open(join(sb.backupDir(), info.name));
const rows = (await copy.execute("select name from users")).rows;
copy.close();
assert.deepEqual(rows.map((r) => r.name), ["Ada"]);
});
test("changes made after it was taken are not in it", async () => {
const { info } = await sb.createSnapshot("manual", at("2026-10-17T12:01:00Z"));
await db.insert(users).values({ oidcSub: "bo", name: "Bo", role: "viewer" });
const copy = open(join(sb.backupDir(), info.name));
assert.equal(Number((await copy.execute("select count(*) as n from users")).rows[0].n), 1);
copy.close();
await db.delete(users).where((await import("drizzle-orm")).eq(users.oidcSub, "bo"));
});
test("two taken in the same second don't collide", async () => {
const a = await sb.createSnapshot("manual", at("2026-10-17T13:00:00Z"));
const b = await sb.createSnapshot("manual", at("2026-10-17T13:00:00Z"));
assert.notEqual(a.info.name, b.info.name);
assert.match(b.info.name, /-manual-2\.sqlite$/);
});
test("files that aren't a usable backup are said to be, and why", async () => {
const junk = join(dir, "junk.sqlite");
writeFileSync(junk, "this is not a database at all");
assert.match((await sb.checkSnapshotFile(junk)).problem!, /isn't a database file/);
assert.match((await sb.checkSnapshotFile(join(dir, "missing.sqlite"))).problem!, /isn't there/);
// a real SQLite file that isn't ours
const other = join(dir, "other.sqlite");
const c = createClient({ url: `file:${other}` });
await c.execute("create table cats (name text)");
c.close();
assert.match((await sb.checkSnapshotFile(other)).problem!, /isn't a Homelab Manager database/);
// one of ours, made by a newer version (more schema changes than this one has)
const newer = join(dir, "newer.sqlite");
await client.execute(`VACUUM INTO '${newer.replace(/'/g, "''")}'`);
const n = createClient({ url: `file:${newer}` });
for (let i = 0; i < 3; i++) await n.execute({ sql: "insert into __drizzle_migrations (hash, created_at) values (?, ?)", args: [`future-${i}`, 9_999_999_999_999 + i] });
n.close();
const check = await sb.checkSnapshotFile(newer);
assert.equal(check.ok, false);
assert.equal(check.newerThanApp, true);
assert.match(check.problem!, /newer version/);
assert.ok(check.migrations! > check.appMigrations);
});
test("an older one is fine: this version brings it up to date when it starts", async () => {
const older = join(dir, "older.sqlite");
await client.execute(`VACUUM INTO '${older.replace(/'/g, "''")}'`);
const o = createClient({ url: `file:${older}` });
await o.execute("delete from __drizzle_migrations where rowid = (select max(rowid) from __drizzle_migrations)");
o.close();
const check = await sb.checkSnapshotFile(older);
assert.equal(check.ok, true);
assert.equal(check.migrations, sb.appMigrationCount() - 1);
});
test("only names of our own files are ever acted on, whatever is asked for", () => {
for (const bad of ["../test.sqlite", "..\\test.sqlite", "homelab-manager-2026-10-17-12-00-00.sqlite/../../x", "homelab-manager-x.sqlite", "test.sqlite", "homelab-manager-2026-10-17-12-00-00.sqlite.bak", "/etc/passwd", ""]) {
assert.equal(sb.isSnapshotName(bad), false, bad);
assert.equal(sb.snapshotPath(bad), null, bad);
assert.equal(sb.deleteSnapshot(bad), false, bad);
}
for (const good of ["homelab-manager-2026-10-17-12-00-00.sqlite", "homelab-manager-2026-10-17-12-00-00-manual.sqlite", "homelab-manager-2026-10-17-12-00-00-pre-restore-2.sqlite", "homelab-manager-2026-10-17-12-00-00-uploaded.sqlite"]) {
assert.equal(sb.isSnapshotName(good), true, good);
}
});
});
describe("keeping the newest", () => {
const fake = (name: string, mtime: string) => {
mkdirSync(sb.backupDir(), { recursive: true });
const f = join(sb.backupDir(), name);
writeFileSync(f, "x");
utimesSync(f, at(mtime), at(mtime));
};
const names = () => sb.listSnapshots().map((s) => s.name);
test("the newest N of the daily ones stay, and each other kind has its own, smaller allowance", () => {
wipeBackups();
for (let d = 1; d <= 6; d++) fake(`homelab-manager-2026-10-0${d}-03-30-00.sqlite`, `2026-10-0${d}T03:30:00Z`);
for (let d = 1; d <= 12; d++) fake(`homelab-manager-2026-10-${String(d).padStart(2, "0")}-09-00-00-manual.sqlite`, `2026-10-${String(d).padStart(2, "0")}T09:00:00Z`);
for (let d = 1; d <= 5; d++) fake(`homelab-manager-2026-10-0${d}-10-00-00-pre-restore.sqlite`, `2026-10-0${d}T10:00:00Z`);
const removed = sb.pruneSnapshots(3);
const left = names();
assert.equal(left.filter((n) => !/-(manual|pre-restore|uploaded)/.test(n)).length, 3);
assert.equal(left.filter((n) => n.includes("-manual")).length, 10);
assert.equal(left.filter((n) => n.includes("-pre-restore")).length, 3);
assert.equal(removed.length, 3 + 2 + 2);
assert.ok(left.includes("homelab-manager-2026-10-06-03-30-00.sqlite") && !left.includes("homelab-manager-2026-10-01-03-30-00.sqlite"), "the oldest went, not the newest");
// other files in the folder are left alone
writeFileSync(join(sb.backupDir(), "notes.txt"), "mine");
sb.pruneSnapshots(1);
assert.ok(existsSync(join(sb.backupDir(), "notes.txt")));
wipeBackups();
});
test("a scheduled run takes one and trims to what is kept, and records that it worked", async () => {
wipeBackups();
await updateSettings({ selfBackup: { keep: 2 } });
for (const d of ["2026-10-14T03:30:00Z", "2026-10-15T03:30:00Z", "2026-10-16T03:30:00Z"]) assert.ok(await runScheduledBackup(at(d)));
assert.deepEqual(names(), ["homelab-manager-2026-10-16-03-30-00.sqlite", "homelab-manager-2026-10-15-03-30-00.sqlite"]);
const last = await sb.lastAttempt();
assert.equal(last!.ok, true);
assert.equal(last!.kind, "scheduled");
assert.equal(cronForBackup("03:30"), "30 3 * * *");
assert.equal(cronForBackup("23:05"), "5 23 * * *");
await updateSettings({ selfBackup: { keep: 14 } });
});
test("a backup that can't be made is recorded as having failed, says why, and leaves no half file", async () => {
wipeBackups();
const realExecute = client.execute.bind(client);
(client as unknown as { execute: unknown }).execute = async (stmt: unknown) => {
if (typeof stmt === "string" && stmt.startsWith("VACUUM INTO")) throw new Error("SQLITE_FULL: database or disk is full");
return realExecute(stmt as never);
};
try {
await assert.rejects(sb.createSnapshot("scheduled", at("2026-10-17T03:30:00Z")), /SQLITE_FULL/);
const last = await sb.lastAttempt();
assert.equal(last!.ok, false);
assert.match(last!.error!, /disk is full/);
assert.deepEqual(sb.listSnapshots(), [], "nothing half-written is left in the backups folder");
assert.equal(await runScheduledBackup(at("2026-10-17T03:31:00Z")), null, "the scheduled run swallows it (and says so through the channels)");
} finally {
(client as unknown as { execute: unknown }).execute = realExecute;
}
assert.ok(await runScheduledBackup(at("2026-10-17T03:32:00Z")), "and the next one works");
assert.equal((await sb.lastAttempt())!.ok, true);
wipeBackups();
});
});
describe("restoring", () => {
test("a staged restore is checked first, kept beside the database, can be cancelled, and refuses what isn't usable", async () => {
wipeBackups();
const { info } = await sb.createSnapshot("manual", at("2026-10-17T15:00:00Z"));
assert.equal(sb.stagedRestore(), null);
await assert.rejects(sb.stageRestore("../../etc/passwd", "Ada"), /no backup with that name/);
await assert.rejects(sb.stageRestore("homelab-manager-2026-01-01-00-00-00.sqlite", "Ada"), /no backup with that name/);
writeFileSync(join(sb.backupDir(), "homelab-manager-2026-10-17-16-00-00-uploaded.sqlite"), "garbage");
await assert.rejects(sb.stageRestore("homelab-manager-2026-10-17-16-00-00-uploaded.sqlite", "Ada"), /isn't a database file/);
assert.equal(sb.stagedRestore(), null, "a refused one stages nothing");
const check = await sb.stageRestore(info.name, "Ada", at("2026-10-17T15:05:00Z"));
assert.equal(check.ok, true);
const staged = sb.stagedRestore()!;
assert.deepEqual(staged, { from: info.name, stagedAt: "2026-10-17T15:05:00.000Z", by: "Ada" });
const dataDir = join(dir, "data");
assert.ok(existsSync(join(dataDir, PENDING_FILE)));
assert.equal(readdirSync(dataDir).some((f) => f.endsWith(".copying")), false, "no half-copied file is left");
assert.equal(statSync(join(dataDir, PENDING_FILE)).size, info.sizeBytes);
assert.equal(sb.cancelStagedRestore(), true);
assert.equal(sb.stagedRestore(), null);
assert.equal(existsSync(join(dataDir, PENDING_FILE)), false);
assert.equal(sb.cancelStagedRestore(), false);
wipeBackups();
});
test("at start-up the staged file replaces the database, the old one is kept, and nothing happens twice", () => {
const folder = mkdtempSync(join(dir, "apply-"));
const dbFile = join(folder, "app.sqlite");
writeFileSync(dbFile, "OLD DATABASE");
writeFileSync(`${dbFile}-wal`, "stale journal");
writeFileSync(join(folder, PENDING_FILE), "NEW DATABASE");
writeFileSync(join(folder, PENDING_NOTE), JSON.stringify({ from: "homelab-manager-2026-10-01-03-30-00.sqlite", by: "Ada" }));
const result = applyPendingRestore(dbFile, at("2026-10-17T16:00:00Z"))!;
assert.equal(readFileSync(dbFile, "utf8"), "NEW DATABASE");
assert.equal(existsSync(`${dbFile}-wal`), false, "a journal from the old file would be taken for the new one's");
assert.equal(result.keptAs, "homelab-manager-2026-10-17-16-00-00-pre-restore.sqlite");
assert.equal(readFileSync(join(folder, "backups", result.keptAs!), "utf8"), "OLD DATABASE", "what was in use is kept, not deleted");
assert.equal(result.from, "homelab-manager-2026-10-01-03-30-00.sqlite");
assert.equal(result.by, "Ada");
assert.equal(existsSync(join(folder, PENDING_FILE)), false);
assert.equal(existsSync(join(folder, PENDING_NOTE)), false);
assert.deepEqual(JSON.parse(readFileSync(join(folder, RESULT_NOTE), "utf8")), result);
assert.equal(applyPendingRestore(dbFile, at("2026-10-17T16:01:00Z")), null, "nothing staged any more");
assert.equal(readFileSync(dbFile, "utf8"), "NEW DATABASE");
});
test("with no database there yet (a fresh machine), the staged one simply becomes it", () => {
const folder = mkdtempSync(join(dir, "fresh-"));
const dbFile = join(folder, "app.sqlite");
writeFileSync(join(folder, PENDING_FILE), "NEW DATABASE");
const result = applyPendingRestore(dbFile, at("2026-10-17T16:00:00Z"))!;
assert.equal(readFileSync(dbFile, "utf8"), "NEW DATABASE");
assert.equal(result.keptAs, null);
assert.equal(result.from, "an uploaded backup", "no note, no name — said plainly");
});
test("a restore that can't be completed leaves the database exactly as it was", () => {
// the folder for keeping the old one can't be made (a file is in its place)
const folder = mkdtempSync(join(dir, "fail-"));
const dbFile = join(folder, "app.sqlite");
writeFileSync(dbFile, "OLD DATABASE");
writeFileSync(join(folder, PENDING_FILE), "NEW DATABASE");
writeFileSync(join(folder, "backups"), "I am a file, not a folder");
assert.equal(applyPendingRestore(dbFile, at("2026-10-17T16:00:00Z")), null);
assert.equal(readFileSync(dbFile, "utf8"), "OLD DATABASE", "the database is as it was");
assert.equal(readFileSync(join(folder, PENDING_FILE), "utf8"), "NEW DATABASE", "and the staged one is still there to try again");
assert.equal(existsSync(join(folder, RESULT_NOTE)), false);
// something staged that isn't a file is never put in the database's place
const odd = mkdtempSync(join(dir, "odd-"));
writeFileSync(join(odd, "app.sqlite"), "OLD DATABASE");
mkdirSync(join(odd, PENDING_FILE));
assert.equal(applyPendingRestore(join(odd, "app.sqlite"), at("2026-10-17T16:00:00Z")), null);
assert.equal(readFileSync(join(odd, "app.sqlite"), "utf8"), "OLD DATABASE");
});
});
describe("through the routes", () => {
let app: Server;
let base = "";
let who = "ada";
const call = (method: string, path: string, body?: unknown, raw?: Buffer) =>
fetch(`${base}${path}`, raw ? { method, headers: { "content-type": "application/octet-stream" }, body: new Uint8Array(raw) } : { method, headers: { "content-type": "application/json" }, body: body === undefined ? undefined : JSON.stringify(body) });
before(async () => {
await db.insert(users).values({ oidcSub: "operator", name: "Olle", role: "operator" });
const a = express();
a.use(express.json());
a.use((req, _res, next) => {
(req as unknown as { session: unknown }).session = { user: { sub: who } };
next();
});
a.use("/api/db-backups", dbBackupsRouter);
app = await new Promise<Server>((resolve) => {
const x = a.listen(0, "127.0.0.1", () => resolve(x));
});
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/db-backups`;
wipeBackups();
});
after(() => app.close());
test("everything about backups is for administrators only", async () => {
who = "operator";
for (const [method, path] of [["GET", "/"], ["POST", "/"], ["PUT", "/settings"], ["POST", "/upload"], ["GET", "/x/download"], ["POST", "/x/restore"], ["POST", "/x/check"], ["DELETE", "/x"], ["POST", "/restore/cancel"]] as const) {
assert.equal((await call(method, path, method === "GET" || method === "DELETE" ? undefined : {})).status, 403, `${method} ${path}`);
}
who = "ada";
assert.equal((await call("GET", "/")).status, 200);
});
test("settings are checked and kept, and change what is kept at once", async () => {
assert.equal((await call("PUT", "/settings", { enabled: true, time: "25:00", keep: 7 })).status, 400);
assert.equal((await call("PUT", "/settings", { enabled: true, time: "03:30", keep: 0 })).status, 400);
assert.equal((await call("PUT", "/settings", { enabled: true, time: "03:30", keep: 500 })).status, 400);
const res = await call("PUT", "/settings", { enabled: true, time: "04:15", keep: 7 });
assert.equal(res.status, 200);
const body = await res.json();
assert.deepEqual(body.settings, { enabled: true, time: "04:15", keep: 7 });
assert.ok(body.nextRun, "it says when the next one is");
assert.equal((await getSettings()).selfBackup.time, "04:15");
const off = await (await call("PUT", "/settings", { enabled: false, time: "04:15", keep: 7 })).json();
assert.equal(off.nextRun, null, "off means nothing is scheduled");
await call("PUT", "/settings", { enabled: true, time: "03:30", keep: 14 });
const entry = (await db.select().from(auditLog)).filter((e) => e.category === "db_backup" && e.action === "settings").pop()!;
assert.deepEqual(JSON.parse(entry.detail!).to, { enabled: true, time: "03:30", keep: 14 });
});
test("take one, see it listed, download it byte for byte, check it, and delete it", async () => {
const made = await call("POST", "/");
assert.equal(made.status, 201);
const { snapshot, check } = await made.json();
assert.equal(check.ok, true);
assert.match(snapshot.name, /-manual\.sqlite$/);
const overview = await (await call("GET", "/")).json();
assert.equal(overview.snapshots.length, 1);
assert.equal(overview.snapshots[0].name, snapshot.name);
assert.equal(overview.lastAttempt.ok, true);
assert.equal(overview.staged, null);
const download = await call("GET", `/${snapshot.name}/download`);
assert.equal(download.status, 200);
assert.equal(download.headers.get("cache-control"), "no-store");
assert.match(download.headers.get("content-disposition") ?? "", new RegExp(snapshot.name.replace(/\./g, "\\.")));
const bytes = Buffer.from(await download.arrayBuffer());
assert.deepEqual(bytes, readFileSync(join(sb.backupDir(), snapshot.name)));
assert.equal((await (await call("POST", `/${snapshot.name}/check`)).json()).check.integrity, "ok");
assert.equal((await call("GET", "/homelab-manager-nonsense.sqlite/download")).status, 404);
assert.equal((await call("GET", `/..%2Fapp.sqlite/download`)).status, 404);
assert.equal((await call("DELETE", `/${snapshot.name}`)).status, 204);
assert.equal((await call("DELETE", `/${snapshot.name}`)).status, 404);
const actions = (await db.select().from(auditLog)).filter((e) => e.category === "db_backup").map((e) => e.action);
for (const wanted of ["create", "download", "delete"]) assert.ok(actions.includes(wanted), wanted);
});
test("a backup from elsewhere can be uploaded and then restored like any other; a bad file is refused and leaves nothing", async () => {
const { info } = await sb.createSnapshot("manual", at("2026-10-17T18:00:00Z"));
const bytes = readFileSync(join(sb.backupDir(), info.name));
const before = sb.listSnapshots().length;
const up = await call("POST", "/upload", undefined, bytes);
assert.equal(up.status, 201);
const uploaded = (await up.json()).snapshot;
assert.match(uploaded.name, /-uploaded\.sqlite$/);
assert.equal(sb.listSnapshots().length, before + 1);
const bad = await call("POST", "/upload", undefined, Buffer.from("not a database"));
assert.equal(bad.status, 400);
assert.match((await bad.json()).message, /isn't a database file/);
assert.equal((await call("POST", "/upload", undefined, Buffer.alloc(0))).status, 400);
assert.equal(sb.listSnapshots().length, before + 1, "nothing was kept from the bad ones");
const restore = await call("POST", `/${uploaded.name}/restore`);
assert.equal(restore.status, 200);
assert.equal((await restore.json()).staged.from, uploaded.name);
assert.equal((await (await call("GET", "/")).json()).staged.from, uploaded.name);
assert.equal((await call("POST", "/homelab-manager-2026-01-01-00-00-00.sqlite/restore")).status, 400);
assert.equal((await (await call("POST", "/restore/cancel")).json()).cancelled, true);
assert.equal((await (await call("GET", "/")).json()).staged, null);
assert.ok((await db.select().from(auditLog)).some((e) => e.action === "restore_staged"));
assert.ok((await db.select().from(auditLog)).some((e) => e.action === "restore_cancelled"));
});
test("what the last restore did is shown until it is dismissed", async () => {
writeFileSync(join(dir, "data", RESULT_NOTE), JSON.stringify({ restoredAt: "2026-10-17T16:00:00.000Z", from: "x.sqlite", keptAs: "y.sqlite", by: "Ada" }));
assert.equal((await (await call("GET", "/")).json()).lastRestore.keptAs, "y.sqlite");
assert.equal((await call("POST", "/restore-result/dismiss")).status, 200);
assert.equal((await (await call("GET", "/")).json()).lastRestore, null);
});
});
describe("how current the backups are, in the alerts and /metrics", () => {
const selfAlert = async () => {
invalidateAlertsCache();
return (await getAlerts(true)).alerts.find((a) => a.id === "backup:self-backup");
};
const oneAged = async (hoursAgo: number) => {
wipeBackups();
const { info } = await sb.createSnapshot("scheduled", new Date());
const when = new Date(Date.now() - hoursAgo * 3600_000);
utimesSync(join(sb.backupDir(), info.name), when, when);
};
test("no alert while they are recent, or when none was ever tried; a warning when stale, critical after a week", async () => {
wipeBackups();
assert.equal(await selfAlert(), undefined, "nothing yet, and nothing failed: a new install isn't scolded");
await oneAged(5);
assert.equal(await selfAlert(), undefined);
await oneAged(60);
const stale = await selfAlert();
assert.equal(stale!.severity, "warning");
assert.match(stale!.message, /60 h old/);
assert.equal(stale!.source, "Homelab Manager");
assert.equal(stale!.link, "/settings/backup");
await oneAged(24 * 8);
assert.equal((await selfAlert())!.severity, "critical");
assert.match((await selfAlert())!.message, /8 days old/);
});
test("when the last attempt failed, the alert says why; and nothing is said at all while backups are switched off", async () => {
await oneAged(60);
await setInternalFlag("selfBackupLastAttempt", JSON.stringify({ at: new Date().toISOString(), ok: false, kind: "scheduled", error: "ENOSPC: no space left on device", name: null }));
assert.match((await selfAlert())!.message, /The last attempt failed: ENOSPC: no space left on device/);
wipeBackups();
const first = await selfAlert();
assert.match(first!.message, /no backup of its own yet\. The last attempt failed|no backup of its own yet/, "a failure with nothing to show for it is said");
await updateSettings({ selfBackup: { enabled: false } });
assert.equal(await selfAlert(), undefined);
await updateSettings({ selfBackup: { enabled: true } });
});
test("/metrics says how many are kept, when the newest was made and whether the last try worked", async () => {
await oneAged(2);
await sb.createSnapshot("manual", new Date());
const text = await buildMetrics();
assert.match(text, /homelab_selfbackup_snapshots 2/);
const ts = /homelab_selfbackup_last_success_timestamp_seconds (\d+)/.exec(text);
assert.ok(ts && Math.abs(Number(ts[1]) - Date.now() / 1000) < 120, "the manual one just made is the newest");
assert.match(text, /homelab_selfbackup_last_attempt_ok 1/);
wipeBackups();
});
});
+3 -3
View File
@@ -282,9 +282,9 @@ describe("following a run to the end", () => {
const titles = events.filter((e) => /Update run/.test(e.title)).map((e) => e.title);
// (the tests before this one wrote their own lines too, so this looks for the kinds rather than the number)
for (const wanted of ["Update run started", "Update run didn't finish", "Update run finished", "Update run stopped being followed"]) assert.ok(titles.includes(wanted), `${wanted} in ${JSON.stringify([...new Set(titles)])}`);
const failedLine = events.find((e) => e.title === "Update run didn't finish")!;
assert.equal(failedLine.tone, "bad");
assert.match(failedLine.detail!, /The playbook failed in Semaphore/);
const failedLines = events.filter((e) => e.title === "Update run didn't finish");
assert.ok(failedLines.every((e) => e.tone === "bad"), "a run that didn't finish reads as bad");
assert.ok(failedLines.some((e) => /The playbook failed in Semaphore/.test(e.detail ?? "")), "and says why");
assert.ok((await db.select().from(auditLog)).some((a) => a.category === "update_run" && a.action === "failed"));
// nothing is held in memory: a fresh pass over the table is all it takes