Stale-backup alerts, per-server alert limits, and update runs from the Updates page
- Backups: a guest whose newest backup is older than its limit is an alert (backup kind), in the daily backup message, the digest and /metrics (homelab_backup_guests, ..._last_age_seconds) - Per-server thresholds (servers.thresholds, migration 0030): offline minutes, disk percent, forecast window, backup age, and per-disk limits or "leave alone"; applied everywhere the global ones were; saving clears the alerts cache - Update runs (update_runs, migration 0031): Update... on the Updates page puts the server under maintenance, runs a Semaphore playbook limited to it, waits for the host to report, and reports what is left and whether a restart is due; poller survives restarts - Notification toggle for update-run results; tests and docs Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
5dc6cea69c
commit
7a5b623444
42 files changed
+7561
-57
No files matched your search
@@ -0,0 +1,236 @@
|
||||
/**
|
||||
* Alert thresholds that belong to one server: how they are read and checked, what they change in the alerts, the forecast and /metrics,
|
||||
* and who may set them.
|
||||
*
|
||||
* Run with: npm test --workspace server
|
||||
*/
|
||||
import { test, describe, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { AddressInfo } from "node:net";
|
||||
import type { Server } from "node:http";
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), "hm-thresholds-"));
|
||||
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
||||
process.env.SESSION_DIR = join(dir, "sessions");
|
||||
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
||||
process.env.HEALTH_STARTUP_GRACE_MS = "0";
|
||||
|
||||
const express = (await import("express")).default;
|
||||
const { db } = await import("../src/db/client.js");
|
||||
const { users, servers, serverMetrics, auditLog } = await import("../src/db/schema.js");
|
||||
const { runMigrations } = await import("../src/db/migrate.js");
|
||||
const { parseThresholds, checkThresholds, effectiveThresholds, diskLimit } = await import("../src/services/serverThresholds.js");
|
||||
const { evaluateHealth } = await import("../src/services/healthMonitor.js");
|
||||
const { disksFillingSoon } = await import("../src/services/serverMetrics.js");
|
||||
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
|
||||
const { buildMetrics } = await import("../src/services/metrics.js");
|
||||
const { serversRouter } = await import("../src/routes/servers.js");
|
||||
const { hashToken } = await import("../src/services/tokens.js");
|
||||
const { eq } = await import("drizzle-orm");
|
||||
|
||||
await runMigrations();
|
||||
|
||||
const GLOBAL = { serverOfflineMinutes: 60, diskUsagePercent: 90, diskForecastDays: 14, backupStaleHours: 48 };
|
||||
|
||||
describe("reading and checking thresholds", () => {
|
||||
test("what is stored is read back leniently: nonsense is dropped, never an error", () => {
|
||||
assert.equal(parseThresholds(null), null);
|
||||
assert.equal(parseThresholds("not json"), null);
|
||||
assert.equal(parseThresholds("{}"), null);
|
||||
assert.deepEqual(parseThresholds(JSON.stringify({ diskUsagePercent: 95, serverOfflineMinutes: 5, diskForecastDays: "soon", mounts: { "/data": { percent: 97 }, "/x": { percent: 3 }, "/y": "no", "/z": { ignore: true } } })), {
|
||||
diskUsagePercent: 95,
|
||||
mounts: { "/data": { percent: 97 }, "/z": { ignore: true } },
|
||||
});
|
||||
});
|
||||
|
||||
test("a request is refused with a message that says which number is wrong", () => {
|
||||
assert.deepEqual(checkThresholds(null), { ok: true, value: null });
|
||||
assert.deepEqual(checkThresholds({}), { ok: true, value: null });
|
||||
assert.deepEqual(checkThresholds({ diskUsagePercent: 95, serverOfflineMinutes: null }), { ok: true, value: { diskUsagePercent: 95 } });
|
||||
const bad = (input: unknown) => {
|
||||
const r = checkThresholds(input);
|
||||
assert.equal(r.ok, false, JSON.stringify(input));
|
||||
return (r as { message: string }).message;
|
||||
};
|
||||
assert.match(bad({ diskUsagePercent: 20 }), /Disk usage alert at \(%\) must be a whole number from 50 to 99/);
|
||||
assert.match(bad({ serverOfflineMinutes: 5 }), /Offline after \(minutes\)/);
|
||||
assert.match(bad({ diskForecastDays: 1.5 }), /Disk forecast/);
|
||||
assert.match(bad({ backupStaleHours: 99999 }), /Backup older than/);
|
||||
assert.match(bad({ mounts: { "/data": { percent: 100 } } }), /limit for \/data/);
|
||||
assert.match(bad({ mounts: { "": { ignore: true } } }), /missing or too long/);
|
||||
assert.match(bad("high"), /should be an object/);
|
||||
assert.match(bad({ mounts: Object.fromEntries(Array.from({ length: 51 }, (_, i) => [`/m${i}`, { ignore: true }])) }), /At most 50/);
|
||||
});
|
||||
|
||||
test("a value of its own wins, and the rest follows Settings — including 0, which means off", () => {
|
||||
assert.deepEqual(effectiveThresholds(GLOBAL, null), GLOBAL);
|
||||
assert.deepEqual(effectiveThresholds(GLOBAL, { diskUsagePercent: 95, diskForecastDays: 0 }), { ...GLOBAL, diskUsagePercent: 95, diskForecastDays: 0 });
|
||||
});
|
||||
|
||||
test("a disk's limit: its own rule, then the server's, then Settings; or none at all", () => {
|
||||
const own = { diskUsagePercent: 95, mounts: { "/data": { percent: 80 }, "/boot": { ignore: true } } };
|
||||
assert.equal(diskLimit(own, "/data", 90), 80);
|
||||
assert.equal(diskLimit(own, "/", 90), 95);
|
||||
assert.equal(diskLimit(own, "/boot", 90), null);
|
||||
assert.equal(diskLimit(null, "/", 90), 90);
|
||||
assert.equal(diskLimit({}, "/", 90), 90);
|
||||
});
|
||||
});
|
||||
|
||||
describe("what it changes in the health rules", () => {
|
||||
const NOW = Date.parse("2026-10-17T12:00:00Z");
|
||||
const disk = (mount: string, used: number) => ({ mount, sizeBytes: 100, usedBytes: used });
|
||||
const snap = (extra: object) => ({ servers: [{ id: 1, name: "nas", lastSeenAt: new Date(NOW - 5 * 60_000).toISOString(), disks: [disk("/", 92), disk("/data", 85), disk("/boot", 99)], ...extra }], proxmox: [], synology: [] });
|
||||
const keys = (extra: object, now = NOW) => evaluateHealth(snap(extra), { serverOfflineMinutes: 60, diskUsagePercent: 90 }, now).map((c) => c.key).sort();
|
||||
|
||||
test("with nothing of its own, the global limit applies to every disk", () => {
|
||||
assert.deepEqual(keys({}), ["disk:server:1:/", "disk:server:1:/boot"]);
|
||||
});
|
||||
|
||||
test("a server's own percentage replaces the global one; a disk's own rule replaces both; an ignored disk says nothing", () => {
|
||||
assert.deepEqual(keys({ thresholds: { diskUsagePercent: 95 } }), ["disk:server:1:/boot"], "92% is fine for a server that alerts at 95");
|
||||
assert.deepEqual(keys({ thresholds: { diskUsagePercent: 80 } }), ["disk:server:1:/", "disk:server:1:/boot", "disk:server:1:/data"]);
|
||||
assert.deepEqual(keys({ thresholds: { mounts: { "/boot": { ignore: true }, "/data": { percent: 80 } } } }), ["disk:server:1:/", "disk:server:1:/data"]);
|
||||
assert.deepEqual(keys({ thresholds: { diskUsagePercent: 99, mounts: { "/": { percent: 91 }, "/boot": { ignore: true } } } }), ["disk:server:1:/"]);
|
||||
assert.deepEqual(keys({ thresholds: { diskUsagePercent: 99, mounts: { "/": { percent: 91 } } } }), ["disk:server:1:/", "disk:server:1:/boot"], "99% is at the limit");
|
||||
});
|
||||
|
||||
test("a server's own 'offline after' replaces the global one in either direction", () => {
|
||||
const quietFor = (minutes: number) => new Date(NOW - minutes * 60_000).toISOString();
|
||||
assert.deepEqual(keys({ lastSeenAt: quietFor(120), disks: [] }), ["offline:server:1"]);
|
||||
assert.deepEqual(keys({ lastSeenAt: quietFor(120), disks: [], thresholds: { serverOfflineMinutes: 300 } }), [], "a laptop that is off at night");
|
||||
assert.deepEqual(keys({ lastSeenAt: quietFor(30), disks: [], thresholds: { serverOfflineMinutes: 15 } }), ["offline:server:1"], "a host that should speak up sooner");
|
||||
});
|
||||
|
||||
test("the forecast leaves out a disk it is told to ignore, and uses the disk's own limit", () => {
|
||||
const series = [
|
||||
{ metric: "disk:/data", kind: "disk" as const, label: "Disk /data", points: [], latest: 85, trend: { perDay: 2, fit: 0.99, spanDays: 7, daysToFull: 7 } },
|
||||
{ metric: "disk:/scratch", kind: "disk" as const, label: "Disk /scratch", points: [], latest: 60, trend: { perDay: 2, fit: 0.99, spanDays: 7, daysToFull: 20 } },
|
||||
];
|
||||
assert.deepEqual(disksFillingSoon(series, 30, 90).map((f) => f.mount), ["/data", "/scratch"]);
|
||||
assert.deepEqual(disksFillingSoon(series, 30, (m) => (m === "/scratch" ? null : 90)).map((f) => f.mount), ["/data"], "told to leave /scratch alone");
|
||||
assert.deepEqual(disksFillingSoon(series, 30, (m) => (m === "/data" ? 80 : 90)).map((f) => f.mount), ["/scratch"], "/data is past its own 80%, so the full-disk alert has it");
|
||||
});
|
||||
});
|
||||
|
||||
describe("through the server routes, the alerts and /metrics", () => {
|
||||
let app: Server;
|
||||
let base = "";
|
||||
let who = "admin";
|
||||
let id = 0;
|
||||
const call = (method: string, path: string, body?: unknown) => fetch(`${base}${path}`, { method, headers: { "content-type": "application/json" }, body: body === undefined ? undefined : JSON.stringify(body) });
|
||||
const patch = (body: unknown) => call("PATCH", `/servers/${id}`, body);
|
||||
|
||||
before(async () => {
|
||||
await db.insert(users).values([
|
||||
{ oidcSub: "admin", name: "Ada", role: "admin" },
|
||||
{ oidcSub: "operator", name: "Olle", role: "operator" },
|
||||
]);
|
||||
const twoHoursAgo = new Date(Date.now() - 2 * 3600_000).toISOString();
|
||||
const [s] = await db
|
||||
.insert(servers)
|
||||
.values({ name: "laptop", apiTokenHash: hashToken("tok"), apiTokenPrefix: "t", lastSeenAt: twoHoursAgo, disks: JSON.stringify([{ mount: "/", sizeBytes: 100, usedBytes: 93 }, { mount: "/boot", sizeBytes: 100, usedBytes: 99 }]) })
|
||||
.returning();
|
||||
id = s.id;
|
||||
const a = express();
|
||||
a.use(express.json());
|
||||
a.use((req, _res, next) => {
|
||||
(req as unknown as { session: unknown }).session = { user: { sub: who } };
|
||||
next();
|
||||
});
|
||||
a.use("/servers", serversRouter);
|
||||
app = await new Promise<Server>((resolve) => {
|
||||
const x = a.listen(0, "127.0.0.1", () => resolve(x));
|
||||
});
|
||||
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}`;
|
||||
});
|
||||
after(() => {
|
||||
app.close();
|
||||
try {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Windows keeps the database file locked until the process exits.
|
||||
}
|
||||
});
|
||||
|
||||
const alertIds = async () => {
|
||||
invalidateAlertsCache();
|
||||
return (await getAlerts(true)).alerts.filter((a) => a.id.includes(`:server:${id}`)).map((a) => a.id).sort();
|
||||
};
|
||||
|
||||
test("only an administrator can set them, a wrong number is refused, and nothing changes until it is right", async () => {
|
||||
who = "operator";
|
||||
assert.equal((await patch({ thresholds: { diskUsagePercent: 95 } })).status, 403);
|
||||
who = "admin";
|
||||
const bad = await patch({ thresholds: { diskUsagePercent: 20 } });
|
||||
assert.equal(bad.status, 400);
|
||||
const body = await bad.json();
|
||||
assert.equal(body.error, "invalid_thresholds");
|
||||
assert.match(body.message, /Disk usage alert at/);
|
||||
const detail = await (await call("GET", `/servers/${id}/detail`)).json();
|
||||
assert.equal(detail.server.thresholds, null);
|
||||
assert.deepEqual(detail.thresholdDefaults.diskUsagePercent, 90, "the page can say what it would follow");
|
||||
assert.equal(detail.thresholdDefaults.backupStaleHours, 48);
|
||||
});
|
||||
|
||||
test("with none set, the laptop has been quiet for two hours, so it is offline (and an offline server's disks aren't judged)", async () => {
|
||||
assert.deepEqual(await alertIds(), [`offline:offline:server:${id}`]);
|
||||
});
|
||||
|
||||
test("the Alerts page doesn't go on showing the old answer for a minute after a server's limits change", async () => {
|
||||
await patch({ thresholds: null });
|
||||
invalidateAlertsCache();
|
||||
assert.ok((await getAlerts(false)).alerts.some((a) => a.id === `offline:offline:server:${id}`), "offline, as the cache now holds it");
|
||||
await patch({ thresholds: { serverOfflineMinutes: 600 } });
|
||||
assert.equal((await getAlerts(false)).alerts.some((a) => a.id === `offline:offline:server:${id}`), false, "no longer, straight away — without anyone clearing the cache");
|
||||
await patch({ thresholds: null });
|
||||
});
|
||||
|
||||
test("its own rules quiet exactly what they say, show on the detail, and are in the audit log", async () => {
|
||||
const res = await patch({ thresholds: { serverOfflineMinutes: 600, diskUsagePercent: 95, mounts: { "/boot": { ignore: true } } } });
|
||||
assert.equal(res.status, 200);
|
||||
assert.deepEqual((await res.json()).server.thresholds, { serverOfflineMinutes: 600, diskUsagePercent: 95, mounts: { "/boot": { ignore: true } } });
|
||||
assert.deepEqual(await alertIds(), [], "not offline for ten hours, 93% is under 95, and /boot is left alone");
|
||||
const detail = await (await call("GET", `/servers/${id}/detail`)).json();
|
||||
assert.equal(detail.server.thresholds.diskUsagePercent, 95);
|
||||
const entry = (await db.select().from(auditLog)).filter((e) => e.action === "update" && e.targetId === String(id)).pop()!;
|
||||
assert.deepEqual(JSON.parse(entry.detail!).thresholds, { serverOfflineMinutes: 600, diskUsagePercent: 95, mounts: { "/boot": { ignore: true } } });
|
||||
// the list shows them too, as an object
|
||||
const row = (await (await call("GET", "/servers")).json()).servers.find((x: { id: number }) => x.id === id);
|
||||
assert.equal(row.thresholds.serverOfflineMinutes, 600);
|
||||
});
|
||||
|
||||
test("a disk's own limit brings its alert back, and the server's own 'offline after' is what /metrics uses", async () => {
|
||||
await patch({ thresholds: { serverOfflineMinutes: 600, diskUsagePercent: 95, mounts: { "/": { percent: 90 }, "/boot": { ignore: true } } } });
|
||||
assert.deepEqual(await alertIds(), [`disk:disk:server:${id}:/`]);
|
||||
assert.match(await buildMetrics(), /homelab_server_up\{server="laptop"\} 1/, "two hours quiet is fine when it may be quiet for ten");
|
||||
await patch({ thresholds: null });
|
||||
assert.match(await buildMetrics(), /homelab_server_up\{server="laptop"\} 0/, "and offline when it follows the global hour");
|
||||
assert.equal((await (await call("GET", `/servers/${id}/detail`)).json()).server.thresholds, null, "null goes back to following Settings");
|
||||
});
|
||||
|
||||
test("the forecast follows the server's own days (0 turns it off for it) and its ignored disks", async () => {
|
||||
const day = 86_400_000;
|
||||
const rows = [];
|
||||
for (let i = 0; i < 10; i++) rows.push({ serverId: id, metric: "disk:/data", value: 30 + i * 4, recordedAt: new Date(Date.now() - (10 - i) * day).toISOString() });
|
||||
await db.insert(serverMetrics).values(rows);
|
||||
await db.update(servers).set({ lastSeenAt: new Date().toISOString() }).where(eq(servers.id, id)); // online, so its figures count
|
||||
const forecast = async () => {
|
||||
invalidateAlertsCache();
|
||||
return (await getAlerts(true)).alerts.filter((a) => a.id.includes(`disk-forecast:${id}:`)).map((a) => a.id);
|
||||
};
|
||||
assert.equal((await forecast()).length, 1, "66% and +4 a day: full in about nine days");
|
||||
await patch({ thresholds: { diskForecastDays: 0 } });
|
||||
assert.deepEqual(await forecast(), [], "0 turns it off for this server");
|
||||
await patch({ thresholds: { diskForecastDays: 60, mounts: { "/data": { ignore: true } } } });
|
||||
assert.deepEqual(await forecast(), [], "an ignored disk has no forecast either");
|
||||
await patch({ thresholds: { diskForecastDays: 60 } });
|
||||
assert.equal((await forecast()).length, 1, "a longer window still sees it");
|
||||
// and the stale-backup limit rides along, checked like the others
|
||||
assert.equal((await patch({ thresholds: { backupStaleHours: 240 } })).status, 200);
|
||||
assert.equal((await patch({ thresholds: { backupStaleHours: 5000 } })).status, 400);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,185 @@
|
||||
/**
|
||||
* Guests whose newest backup has gone too old: the limit (Settings, or a server's own), the alert, what silences it, the daily list and /metrics.
|
||||
*
|
||||
* Run with: npm test --workspace server
|
||||
*/
|
||||
import { test, describe, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), "hm-stale-"));
|
||||
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
||||
process.env.SESSION_DIR = join(dir, "sessions");
|
||||
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
||||
process.env.HEALTH_STARTUP_GRACE_MS = "0";
|
||||
|
||||
const { db } = await import("../src/db/client.js");
|
||||
const { servers, maintenanceWindows, integrations } = await import("../src/db/schema.js");
|
||||
const { runMigrations } = await import("../src/db/migrate.js");
|
||||
const { buildCoverage, staleBackups, collectStaleBackups, getBackupCoverage, setCoverageLoaderForTests } = await import("../src/services/backupCoverage.js");
|
||||
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
|
||||
const { buildMetrics } = await import("../src/services/metrics.js");
|
||||
const { updateSettings } = await import("../src/services/settingsStore.js");
|
||||
const { notifyStaleBackups } = await import("../src/services/notify.js");
|
||||
const { hashToken } = await import("../src/services/tokens.js");
|
||||
|
||||
await runMigrations();
|
||||
after(() => {
|
||||
setCoverageLoaderForTests(null);
|
||||
try {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Windows keeps the database file locked until the process exits.
|
||||
}
|
||||
});
|
||||
|
||||
const HOUR = 3600_000;
|
||||
const guest = (vmid: number, name: string) => ({ vmid, name, node: "pve1", type: "qemu" as const, status: "running", cpu: null, maxmem: null, mem: null, uptime: null });
|
||||
const allJob = { id: "j1", enabled: true, schedule: "02:00", storage: "pbs", node: null, allGuests: true, vmids: null, exclude: null };
|
||||
const px = (guests: ReturnType<typeof guest>[]) => [{ integrationId: 1, integrationName: "Felicia", guests, jobs: [allJob], tasks: [] }];
|
||||
const snapAgo = (now: number, vmid: number, hours: number) => ({ backupType: "vm", backupId: String(vmid), backupTime: Math.floor((now - hours * HOUR) / 1000), verification: "ok" as const });
|
||||
|
||||
describe("when a guest is stale", () => {
|
||||
const NOW = new Date("2026-10-17T12:00:00Z");
|
||||
const now = NOW.getTime();
|
||||
const input = (hoursAgo: number, extra: { servers?: Parameters<typeof buildCoverage>[0]["servers"]; staleHours?: number } = {}) => ({
|
||||
proxmox: px([guest(100, "web")]),
|
||||
pbs: [{ integrationName: "PBS", groups: [snapAgo(now, 100, hoursAgo)] }],
|
||||
servers: extra.servers ?? [],
|
||||
now: NOW,
|
||||
staleHours: extra.staleHours,
|
||||
});
|
||||
|
||||
test("a guest older than the limit is stale, and the limit it was judged by is on the row", () => {
|
||||
const [g] = buildCoverage(input(60));
|
||||
assert.equal(g.status, "stale");
|
||||
assert.equal(g.staleAfterHours, 48);
|
||||
assert.equal(buildCoverage(input(60, { staleHours: 72 }))[0].status, "ok");
|
||||
assert.equal(buildCoverage(input(60, { staleHours: 72 }))[0].staleAfterHours, 72);
|
||||
});
|
||||
|
||||
test("a server that is this guest can have its own limit — longer for a weekly backup, or 0 for never", () => {
|
||||
const linked = (staleHours: number | null) => [{ id: 7, name: "web", proxmoxIntegrationId: 1, proxmoxVmid: 100, staleHours }];
|
||||
assert.equal(buildCoverage(input(200, { servers: linked(240) }))[0].status, "ok", "eight days is fine for a limit of ten");
|
||||
assert.equal(buildCoverage(input(300, { servers: linked(240) }))[0].status, "stale");
|
||||
assert.equal(buildCoverage(input(5000, { servers: linked(0) }))[0].status, "ok", "0 means this one is never called stale");
|
||||
assert.equal(buildCoverage(input(200, { servers: linked(null) }))[0].status, "stale", "no limit of its own: the global one");
|
||||
// a failure is a failure whatever the limit says
|
||||
const failed = buildCoverage({ ...input(1, { servers: linked(0) }), pbs: [{ integrationName: "PBS", groups: [{ ...snapAgo(now, 100, 1), verification: "failed" as const }] }] });
|
||||
assert.equal(failed[0].status, "failed");
|
||||
});
|
||||
|
||||
test("only stale guests are listed, oldest first, critical at three times the limit", () => {
|
||||
const report = {
|
||||
guests: buildCoverage({
|
||||
proxmox: px([guest(100, "web"), guest(101, "db"), guest(102, "fresh"), guest(103, "never")]),
|
||||
pbs: [{ integrationName: "PBS", groups: [snapAgo(now, 100, 60), snapAgo(now, 101, 160), snapAgo(now, 102, 2)] }],
|
||||
servers: [],
|
||||
now: NOW,
|
||||
}),
|
||||
counts: { ok: 1, stale: 2, failed: 0, none: 0, unknown: 1, total: 4 },
|
||||
sources: [],
|
||||
staleHours: 48,
|
||||
generatedAt: NOW.toISOString(),
|
||||
};
|
||||
const stale = staleBackups(report, NOW);
|
||||
assert.deepEqual(stale.map((s) => [s.guest.name, s.severity]), [["db", "critical"], ["web", "warning"]]);
|
||||
assert.equal(stale[0].key, "backup-stale:1:101");
|
||||
assert.match(stale[0].message, /db \(#101\) \[Felicia\]: its newest backup is 6 days old \(the limit is 48 h\)/);
|
||||
assert.match(stale[1].message, /60 h old \(the limit is 48 h\)/);
|
||||
});
|
||||
|
||||
test("it is silenced by maintenance on the Proxmox integration or on the server the guest is", () => {
|
||||
const base = { guests: buildCoverage({ ...input(60), servers: [{ id: 7, name: "web", proxmoxIntegrationId: 1, proxmoxVmid: 100 }] }), counts: { ok: 0, stale: 1, failed: 0, none: 0, unknown: 0, total: 1 }, sources: [], staleHours: 48, generatedAt: NOW.toISOString() };
|
||||
assert.equal(staleBackups(base, NOW)[0].silenced, false);
|
||||
assert.equal(staleBackups(base, NOW, new Set(["integration:1"]))[0].silenced, true);
|
||||
assert.equal(staleBackups(base, NOW, new Set(["server:7"]))[0].silenced, true);
|
||||
assert.equal(staleBackups(base, NOW, new Set(["server:8", "integration:2"]))[0].silenced, false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("through the alerts, the daily list and /metrics", () => {
|
||||
before(async () => {
|
||||
// The guests are Proxmox's, read from a stand-in; the integration they belong to still has to exist for a server to be linked to one.
|
||||
await db.insert(integrations).values({ id: 1, type: "proxmox", name: "Felicia", baseUrl: "https://pve.example", enabled: false });
|
||||
});
|
||||
|
||||
// The alert pass judges by the real clock, so these backups are dated from it.
|
||||
const fixture = () => {
|
||||
const now = Date.now();
|
||||
return {
|
||||
proxmox: px([guest(100, "web"), guest(101, "db"), guest(102, "weekly"), guest(103, "fine")]).map((p) => ({ ...p })),
|
||||
pbs: [{ integrationName: "PBS", groups: [snapAgo(now, 100, 60), snapAgo(now, 101, 200), snapAgo(now, 102, 150), snapAgo(now, 103, 3)] }],
|
||||
sources: [{ kind: "proxmox" as const, name: "Felicia", ok: true }, { kind: "pbs" as const, name: "PBS", ok: true }],
|
||||
};
|
||||
};
|
||||
const staleAlerts = async () => {
|
||||
invalidateAlertsCache();
|
||||
return (await getAlerts(true)).alerts.filter((a) => a.id.startsWith("backup:backup-stale:"));
|
||||
};
|
||||
|
||||
test("each stale guest is an alert of the backup kind, with a link, a target and a severity", async () => {
|
||||
setCoverageLoaderForTests(async () => fixture());
|
||||
const alerts = await staleAlerts();
|
||||
assert.deepEqual(alerts.map((a) => a.id).sort(), ["backup:backup-stale:1:100", "backup:backup-stale:1:101", "backup:backup-stale:1:102"]);
|
||||
const db101 = alerts.find((a) => a.id.endsWith(":101"))!;
|
||||
assert.equal(db101.category, "backup");
|
||||
assert.equal(db101.severity, "critical", "200 hours is more than three times 48");
|
||||
assert.equal(alerts.find((a) => a.id.endsWith(":100"))!.severity, "warning");
|
||||
assert.equal(db101.link, "/backups");
|
||||
assert.deepEqual(db101.target, { type: "integration", id: 1, name: "Felicia" });
|
||||
assert.equal(db101.silenced, false);
|
||||
});
|
||||
|
||||
test("a server's own limit changes its guest's alert, and that server's maintenance silences it", async () => {
|
||||
const [s] = await db.insert(servers).values({ name: "weekly", apiTokenHash: hashToken("t"), apiTokenPrefix: "t", proxmoxIntegrationId: 1, proxmoxVmid: 102, thresholds: JSON.stringify({ backupStaleHours: 240 }) }).returning();
|
||||
setCoverageLoaderForTests(async () => fixture());
|
||||
let alerts = await staleAlerts();
|
||||
assert.equal(alerts.some((a) => a.id.endsWith(":102")), false, "150 hours is fine for a limit of 240");
|
||||
await db.update(servers).set({ thresholds: JSON.stringify({ backupStaleHours: 100 }) });
|
||||
alerts = await staleAlerts();
|
||||
const weekly = alerts.find((a) => a.id.endsWith(":102"))!;
|
||||
assert.ok(weekly, "150 hours is over a limit of 100");
|
||||
assert.deepEqual(weekly.target && { type: weekly.target.type, id: weekly.target.id }, { type: "server", id: s.id }, "it is the server that can be silenced");
|
||||
|
||||
await db.insert(maintenanceWindows).values({ targetType: "server", targetId: s.id, reason: "x", startedAt: new Date(Date.now() - 60_000).toISOString(), endsAt: new Date(Date.now() + 3600_000).toISOString() });
|
||||
assert.equal((await staleAlerts()).find((a) => a.id.endsWith(":102"))!.silenced, true);
|
||||
await db.delete(maintenanceWindows);
|
||||
await db.update(servers).set({ thresholds: null });
|
||||
});
|
||||
|
||||
test("the limit in Settings is what the others follow, and 0 turns the alerts off", async () => {
|
||||
setCoverageLoaderForTests(async () => fixture());
|
||||
await updateSettings({ healthChecks: { backupStaleHours: 100 } });
|
||||
assert.deepEqual((await staleAlerts()).map((a) => a.id.split(":").pop()).sort(), ["101", "102"], "60 hours is fine for a limit of 100");
|
||||
await updateSettings({ healthChecks: { backupStaleHours: 0 } });
|
||||
assert.deepEqual(await staleAlerts(), []);
|
||||
await updateSettings({ healthChecks: { backupStaleHours: 48 } });
|
||||
});
|
||||
|
||||
test("the daily list is the stale guests that aren't under maintenance", async () => {
|
||||
setCoverageLoaderForTests(async () => fixture());
|
||||
const list = await collectStaleBackups();
|
||||
assert.equal(list.length, 3);
|
||||
await db.insert(maintenanceWindows).values({ targetType: "integration", targetId: 1, reason: "x", startedAt: new Date(Date.now() - 60_000).toISOString(), endsAt: new Date(Date.now() + 3600_000).toISOString() });
|
||||
assert.deepEqual(await collectStaleBackups(), [], "the whole Proxmox integration is being worked on");
|
||||
await db.delete(maintenanceWindows);
|
||||
await notifyStaleBackups([]); // nothing to say, nothing sent, and no error with no channel set up
|
||||
await notifyStaleBackups(list);
|
||||
});
|
||||
|
||||
test("/metrics says how many guests are in each state and how old each newest backup is, from what was last read", async () => {
|
||||
setCoverageLoaderForTests(async () => fixture());
|
||||
invalidateAlertsCache();
|
||||
const text = await buildMetrics();
|
||||
assert.match(text, /homelab_backup_guests\{status="ok"\} 1/);
|
||||
assert.match(text, /homelab_backup_guests\{status="stale"\} 3/);
|
||||
const age = /homelab_backup_last_age_seconds\{guest="web",vmid="100",integration="Felicia"\} (\d+)/.exec(text);
|
||||
assert.ok(age, "an age for each guest with a backup");
|
||||
assert.ok(Math.abs(Number(age[1]) - 60 * 3600) < 120, `about 60 hours, got ${age[1]}`);
|
||||
// the page's own selector still asks for its own limit for every guest
|
||||
assert.equal((await getBackupCoverage({ staleHours: 24 * 14 })).counts.stale, 0);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,390 @@
|
||||
/**
|
||||
* An update run: the maintenance window, the playbook, following it, waiting for the server to report again, and every way it can end.
|
||||
* Semaphore is a stand-in that answers as told.
|
||||
*
|
||||
* Run with: npm test --workspace server
|
||||
*/
|
||||
import { test, describe, before, after, beforeEach } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { AddressInfo } from "node:net";
|
||||
import type { Server } from "node:http";
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), "hm-updaterun-"));
|
||||
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
||||
process.env.SESSION_DIR = join(dir, "sessions");
|
||||
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
||||
process.env.HEALTH_STARTUP_GRACE_MS = "0";
|
||||
|
||||
const express = (await import("express")).default;
|
||||
const { db } = await import("../src/db/client.js");
|
||||
const { users, servers, updateRuns, auditLog, maintenanceWindows } = await import("../src/db/schema.js");
|
||||
const { runMigrations } = await import("../src/db/migrate.js");
|
||||
const { startUpdateRun, advanceUpdateRuns, cancelUpdateRun, listUpdateRuns, setSemaphoreForTests, summarizeOutcome, UpdateRunError, HOST_WAIT_MS, MAX_POLL_ERRORS, RUN_LIMIT_MS } = await import("../src/services/updateRuns.js");
|
||||
const { listActiveWindows } = await import("../src/services/maintenance.js");
|
||||
const { getActivity } = await import("../src/services/serverActivity.js").then((m) => ({ getActivity: m.getServerActivity }));
|
||||
const { updateRunsRouter } = await import("../src/routes/updateRuns.js");
|
||||
const { hashToken } = await import("../src/services/tokens.js");
|
||||
const { eq } = await import("drizzle-orm");
|
||||
|
||||
await runMigrations();
|
||||
|
||||
// ── a Semaphore that answers as told ──
|
||||
interface Fake {
|
||||
templates: { id: number; projectId: number; projectName: string; name: string; playbook: string; lastTask: null }[];
|
||||
runs: { projectId: number; templateId: number; options?: { limit?: string; message?: string } }[];
|
||||
task: { id: number; status: string; created: string | null; start: string | null; end: string | null };
|
||||
refuseToStart: boolean;
|
||||
getTaskFails: boolean;
|
||||
}
|
||||
const fake: Fake = { templates: [], runs: [], task: { id: 41, status: "waiting", created: null, start: null, end: null }, refuseToStart: false, getTaskFails: false };
|
||||
const resetFake = () => {
|
||||
fake.templates = [{ id: 7, projectId: 2, projectName: "Homelab", name: "Apply updates", playbook: "updates.yml", lastTask: null }];
|
||||
fake.runs = [];
|
||||
fake.task = { id: 41, status: "waiting", created: null, start: null, end: null };
|
||||
fake.refuseToStart = false;
|
||||
fake.getTaskFails = false;
|
||||
};
|
||||
setSemaphoreForTests(async (integrationId) =>
|
||||
integrationId === 9
|
||||
? {
|
||||
ping: async () => ({ ok: true }),
|
||||
listTemplatesWithStatus: async () => fake.templates,
|
||||
checkTemplates: async () => ({ templates: fake.templates, failedProjectIds: [] }),
|
||||
runTemplate: async (projectId, templateId, options) => {
|
||||
if (fake.refuseToStart) throw new Error("Semaphore is having a moment");
|
||||
fake.runs.push({ projectId, templateId, options });
|
||||
return { ...fake.task } as never;
|
||||
},
|
||||
getTask: async () => {
|
||||
if (fake.getTaskFails) throw new Error("connection refused");
|
||||
return { ...fake.task } as never;
|
||||
},
|
||||
}
|
||||
: null,
|
||||
);
|
||||
|
||||
const T0 = new Date("2026-10-17T12:00:00.000Z");
|
||||
const at = (minutes: number) => new Date(T0.getTime() + minutes * 60_000);
|
||||
const input = (serverId: number, extra: object = {}) => ({ serverId, integrationId: 9, projectId: 2, templateId: 7, limit: "elsa", windowMinutes: 90, actor: "Olle", ...extra });
|
||||
const runRow = async (id: number) => (await db.select().from(updateRuns).where(eq(updateRuns.id, id)))[0];
|
||||
|
||||
describe("starting a run", () => {
|
||||
let serverId = 0;
|
||||
before(async () => {
|
||||
await db.insert(users).values([{ oidcSub: "admin", name: "Ada", role: "admin" }]);
|
||||
const [s] = await db
|
||||
.insert(servers)
|
||||
.values({ name: "elsa", apiTokenHash: hashToken("tok"), apiTokenPrefix: "t", lastSeenAt: T0.toISOString(), pendingUpdates: JSON.stringify({ manager: "apt", count: 12, security: 3, rebootRequired: false, reportedAt: T0.toISOString() }) })
|
||||
.returning();
|
||||
serverId = s.id;
|
||||
});
|
||||
beforeEach(async () => {
|
||||
resetFake();
|
||||
await db.delete(updateRuns);
|
||||
await db.delete(maintenanceWindows);
|
||||
});
|
||||
after(() => {
|
||||
try {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Windows keeps the database file locked until the process exits.
|
||||
}
|
||||
});
|
||||
|
||||
test("it puts the server under maintenance, starts the playbook limited to that server, and writes the run down", async () => {
|
||||
const run = await startUpdateRun(input(serverId), T0);
|
||||
assert.equal(run.status, "running");
|
||||
assert.equal(run.templateName, "Apply updates");
|
||||
assert.equal(run.updatesBefore, 12);
|
||||
assert.equal(run.taskId, 41);
|
||||
assert.equal(run.windowCreated, true);
|
||||
assert.deepEqual(fake.runs.map((r) => [r.projectId, r.templateId, r.options?.limit]), [[2, 7, "elsa"]]);
|
||||
assert.match(fake.runs[0].options!.message!, /Update run for elsa, started in Homelab Manager by Olle \(limited to elsa\)/);
|
||||
|
||||
const windows = await listActiveWindows(T0);
|
||||
assert.equal(windows.length, 1);
|
||||
assert.equal(windows[0].targetId, serverId);
|
||||
assert.equal(windows[0].reason, "Update run: Apply updates");
|
||||
assert.equal(Date.parse(windows[0].endsAt) - T0.getTime(), 90 * 60_000);
|
||||
assert.equal(run.windowId, windows[0].id);
|
||||
});
|
||||
|
||||
test("it won't start without something to wait for, with another run going, or on a Semaphore or template that isn't there", async () => {
|
||||
const refused = async (promise: Promise<unknown>, status: number, code: string) => {
|
||||
await assert.rejects(promise, (e: unknown) => e instanceof UpdateRunError && e.status === status && e.code === code, code);
|
||||
};
|
||||
await refused(startUpdateRun(input(99999), T0), 404, "server_not_found");
|
||||
const [never] = await db.insert(servers).values({ name: "ghost", apiTokenHash: hashToken("g"), apiTokenPrefix: "g" }).returning();
|
||||
await refused(startUpdateRun(input(never.id), T0), 400, "never_reported");
|
||||
await refused(startUpdateRun(input(serverId, { integrationId: 5 }), T0), 400, "no_semaphore");
|
||||
await refused(startUpdateRun(input(serverId, { templateId: 999 }), T0), 404, "template_not_found");
|
||||
assert.equal(fake.runs.length, 0, "nothing was started by any of those");
|
||||
assert.equal((await listActiveWindows(T0)).length, 0, "and no quiet was left behind");
|
||||
|
||||
await startUpdateRun(input(serverId), T0);
|
||||
await refused(startUpdateRun(input(serverId), at(1)), 409, "already_running");
|
||||
assert.equal(fake.runs.length, 1);
|
||||
});
|
||||
|
||||
test("if Semaphore won't start it the quiet is taken back, and no run is recorded", async () => {
|
||||
fake.refuseToStart = true;
|
||||
await assert.rejects(startUpdateRun(input(serverId), T0), (e: unknown) => e instanceof UpdateRunError && e.status === 502 && /Semaphore wouldn't start it/.test(e.message));
|
||||
assert.equal((await db.select().from(updateRuns)).length, 0);
|
||||
assert.equal((await listActiveWindows(at(1))).length, 0, "the window it made is over");
|
||||
});
|
||||
|
||||
test("a window that was already there is extended and left to its owner; it is not ended when the run is", async () => {
|
||||
await db.insert(maintenanceWindows).values({ targetType: "server", targetId: serverId, reason: "patch night", startedAt: T0.toISOString(), endsAt: at(30).toISOString(), createdBy: "Ada" });
|
||||
const run = await startUpdateRun(input(serverId), at(1));
|
||||
assert.equal(run.windowCreated, false);
|
||||
fake.task = { ...fake.task, status: "error" };
|
||||
await advanceUpdateRuns(at(5));
|
||||
assert.equal((await runRow(run.id)).status, "failed");
|
||||
assert.equal((await listActiveWindows(at(6))).length, 1, "someone else's window still stands");
|
||||
});
|
||||
});
|
||||
|
||||
describe("following a run to the end", () => {
|
||||
let serverId = 0;
|
||||
before(async () => {
|
||||
const [s] = await db.insert(servers).values({ name: "anna", apiTokenHash: hashToken("tok2"), apiTokenPrefix: "t2", lastSeenAt: T0.toISOString(), pendingUpdates: JSON.stringify({ manager: "apt", count: 12, security: 3, rebootRequired: false, reportedAt: T0.toISOString() }) }).returning();
|
||||
serverId = s.id;
|
||||
});
|
||||
beforeEach(async () => {
|
||||
resetFake();
|
||||
await db.delete(updateRuns);
|
||||
await db.delete(maintenanceWindows);
|
||||
await db.update(servers).set({ lastSeenAt: T0.toISOString(), pendingUpdates: JSON.stringify({ manager: "apt", count: 12, security: 3, rebootRequired: false, reportedAt: T0.toISOString() }) }).where(eq(servers.id, serverId));
|
||||
});
|
||||
|
||||
const reportBack = async (minutes: number, count: number, reboot: boolean) =>
|
||||
db.update(servers).set({ lastSeenAt: at(minutes).toISOString(), pendingUpdates: JSON.stringify({ manager: "apt", count, security: 0, rebootRequired: reboot, reportedAt: at(minutes).toISOString() }) }).where(eq(servers.id, serverId));
|
||||
|
||||
test("a good run: running, then waiting for the host, then done with what is left — and the window ends", async () => {
|
||||
const run = await startUpdateRun(input(serverId), T0);
|
||||
fake.task = { ...fake.task, status: "running" };
|
||||
await advanceUpdateRuns(at(1));
|
||||
let r = await runRow(run.id);
|
||||
assert.equal(r.status, "running");
|
||||
assert.equal(r.taskStatus, "running");
|
||||
|
||||
fake.task = { ...fake.task, status: "success", end: at(8).toISOString() };
|
||||
await advanceUpdateRuns(at(9));
|
||||
r = await runRow(run.id);
|
||||
assert.equal(r.status, "waiting_for_host");
|
||||
assert.equal(r.taskEndedAt, at(8).toISOString());
|
||||
assert.equal((await listActiveWindows(at(9))).length, 1, "still quiet while it comes back");
|
||||
|
||||
await advanceUpdateRuns(at(12));
|
||||
assert.equal((await runRow(run.id)).status, "waiting_for_host", "no report since the playbook ended");
|
||||
await reportBack(7, 12, false); // a report from before the playbook finished doesn't count
|
||||
await advanceUpdateRuns(at(13));
|
||||
assert.equal((await runRow(run.id)).status, "waiting_for_host");
|
||||
|
||||
await reportBack(14, 2, true);
|
||||
await advanceUpdateRuns(at(15));
|
||||
r = await runRow(run.id);
|
||||
assert.equal(r.status, "done");
|
||||
assert.equal(r.updatesBefore, 12);
|
||||
assert.equal(r.updatesAfter, 2);
|
||||
assert.equal(r.rebootRequired, true);
|
||||
assert.equal(r.message, "2 updates still waiting (10 updates installed); a restart is still needed.");
|
||||
assert.equal(r.finishedAt, at(15).toISOString());
|
||||
assert.equal((await listActiveWindows(at(16))).length, 0, "the window it made is over");
|
||||
assert.equal((await advanceUpdateRuns(at(17))), 0, "a finished run is left alone");
|
||||
});
|
||||
|
||||
test("a playbook that fails ends the run and the window", async () => {
|
||||
const run = await startUpdateRun(input(serverId), T0);
|
||||
fake.task = { ...fake.task, status: "error" };
|
||||
await advanceUpdateRuns(at(3));
|
||||
const r = await runRow(run.id);
|
||||
assert.equal(r.status, "failed");
|
||||
assert.match(r.message!, /The playbook failed in Semaphore \(run #41\)/);
|
||||
assert.equal((await listActiveWindows(at(4))).length, 0);
|
||||
for (const [status, words] of [["rejected", /rejected/], ["stopped", /stopped/]] as const) {
|
||||
await db.delete(updateRuns);
|
||||
const again = await startUpdateRun(input(serverId), at(10));
|
||||
fake.task = { ...fake.task, status };
|
||||
await advanceUpdateRuns(at(11));
|
||||
assert.match((await runRow(again.id)).message!, words);
|
||||
}
|
||||
});
|
||||
|
||||
test("a Semaphore that stops answering ends it after ten tries; one answer in between starts the count again", async () => {
|
||||
const run = await startUpdateRun(input(serverId), T0);
|
||||
fake.getTaskFails = true;
|
||||
for (let i = 0; i < MAX_POLL_ERRORS - 1; i++) await advanceUpdateRuns(at(1 + i));
|
||||
assert.equal((await runRow(run.id)).status, "running");
|
||||
assert.equal((await runRow(run.id)).pollErrors, MAX_POLL_ERRORS - 1);
|
||||
fake.getTaskFails = false;
|
||||
fake.task = { ...fake.task, status: "running" };
|
||||
await advanceUpdateRuns(at(20));
|
||||
assert.equal((await runRow(run.id)).pollErrors, 0);
|
||||
fake.getTaskFails = true;
|
||||
for (let i = 0; i < MAX_POLL_ERRORS; i++) await advanceUpdateRuns(at(21 + i));
|
||||
const r = await runRow(run.id);
|
||||
assert.equal(r.status, "failed");
|
||||
assert.match(r.message!, /Lost contact with Semaphore/);
|
||||
});
|
||||
|
||||
test("a host that doesn't come back, or a run that never ends, is given up on, and the quiet ends so the alerts can speak", async () => {
|
||||
const run = await startUpdateRun(input(serverId), T0);
|
||||
fake.task = { ...fake.task, status: "success", end: at(5).toISOString() };
|
||||
await advanceUpdateRuns(at(6));
|
||||
await advanceUpdateRuns(at(5 + HOST_WAIT_MS / 60_000 - 1));
|
||||
assert.equal((await runRow(run.id)).status, "waiting_for_host");
|
||||
await advanceUpdateRuns(at(5 + HOST_WAIT_MS / 60_000 + 1));
|
||||
const r = await runRow(run.id);
|
||||
assert.equal(r.status, "failed");
|
||||
assert.match(r.message!, /hasn't reported again in 30 minutes/);
|
||||
assert.equal((await listActiveWindows(at(40))).length, 0);
|
||||
|
||||
await db.delete(updateRuns);
|
||||
const slow = await startUpdateRun(input(serverId), at(100));
|
||||
fake.task = { ...fake.task, status: "running" };
|
||||
await advanceUpdateRuns(new Date(at(100).getTime() + RUN_LIMIT_MS + 60_000));
|
||||
assert.match((await runRow(slow.id)).message!, /longer than three hours/);
|
||||
});
|
||||
|
||||
test("a server whose agent doesn't report updates is done as soon as it reports at all", async () => {
|
||||
await db.update(servers).set({ pendingUpdates: null }).where(eq(servers.id, serverId));
|
||||
const run = await startUpdateRun(input(serverId), T0);
|
||||
fake.task = { ...fake.task, status: "success", end: at(5).toISOString() };
|
||||
await advanceUpdateRuns(at(6));
|
||||
await db.update(servers).set({ lastSeenAt: at(10).toISOString() }).where(eq(servers.id, serverId));
|
||||
await advanceUpdateRuns(at(11));
|
||||
const r = await runRow(run.id);
|
||||
assert.equal(r.status, "done");
|
||||
assert.equal(r.updatesAfter, null);
|
||||
assert.match(r.message!, /doesn't say what is waiting; no restart is needed/);
|
||||
});
|
||||
|
||||
test("it can be stopped: the window ends, the Semaphore run is left alone, and a finished run can't be stopped again", async () => {
|
||||
const run = await startUpdateRun(input(serverId), T0);
|
||||
const stopped = await cancelUpdateRun(run.id, "Ada", at(2));
|
||||
assert.equal(stopped!.status, "cancelled");
|
||||
assert.match(stopped!.message!, /Stopped being followed by Ada/);
|
||||
assert.equal((await listActiveWindows(at(3))).length, 0);
|
||||
assert.equal(fake.runs.length, 1, "nothing was sent to Semaphore to stop it");
|
||||
await assert.rejects(cancelUpdateRun(run.id, "Ada", at(3)), (e: unknown) => e instanceof UpdateRunError && e.status === 409);
|
||||
assert.equal(await cancelUpdateRun(99999, "Ada", at(3)), null);
|
||||
});
|
||||
|
||||
test("the story is on the server's history, and a run that was under way when the app stopped is picked up again", async () => {
|
||||
const run = await startUpdateRun(input(serverId), T0);
|
||||
fake.task = { ...fake.task, status: "error" };
|
||||
await advanceUpdateRuns(at(3));
|
||||
const events = await getActivity(serverId, at(5));
|
||||
const titles = events.filter((e) => /Update run/.test(e.title)).map((e) => e.title);
|
||||
// (the tests before this one wrote their own lines too, so this looks for the kinds rather than the number)
|
||||
for (const wanted of ["Update run started", "Update run didn't finish", "Update run finished", "Update run stopped being followed"]) assert.ok(titles.includes(wanted), `${wanted} in ${JSON.stringify([...new Set(titles)])}`);
|
||||
const failedLine = events.find((e) => e.title === "Update run didn't finish")!;
|
||||
assert.equal(failedLine.tone, "bad");
|
||||
assert.match(failedLine.detail!, /The playbook failed in Semaphore/);
|
||||
assert.ok((await db.select().from(auditLog)).some((a) => a.category === "update_run" && a.action === "failed"));
|
||||
|
||||
// nothing is held in memory: a fresh pass over the table is all it takes
|
||||
await db.delete(updateRuns);
|
||||
const survivor = await startUpdateRun(input(serverId), at(20));
|
||||
fake.task = { ...fake.task, status: "success", end: at(22).toISOString() };
|
||||
assert.equal(await advanceUpdateRuns(at(23)), 1);
|
||||
assert.equal((await runRow(survivor.id)).status, "waiting_for_host");
|
||||
void run;
|
||||
});
|
||||
|
||||
test("the outcome is worded plainly for each case", () => {
|
||||
assert.equal(summarizeOutcome(5, 0, false), "No updates are waiting now (5 updates installed); no restart is needed.");
|
||||
assert.equal(summarizeOutcome(1, 0, true), "No updates are waiting now (1 update installed); a restart is still needed.");
|
||||
assert.equal(summarizeOutcome(null, 3, false), "3 updates still waiting; no restart is needed.");
|
||||
assert.equal(summarizeOutcome(3, 3, false), "3 updates still waiting; no restart is needed.");
|
||||
assert.equal(summarizeOutcome(3, null, true), "The server reported again, but doesn't say what is waiting; a restart is still needed.");
|
||||
});
|
||||
});
|
||||
|
||||
describe("the routes", () => {
|
||||
let app: Server;
|
||||
let base = "";
|
||||
let who = "operator";
|
||||
let serverId = 0;
|
||||
const call = (method: string, path: string, body?: unknown) => fetch(`${base}${path}`, { method, headers: { "content-type": "application/json" }, body: body === undefined ? undefined : JSON.stringify(body) });
|
||||
|
||||
before(async () => {
|
||||
await db.insert(users).values([
|
||||
{ oidcSub: "operator", name: "Olle", role: "operator" },
|
||||
{ oidcSub: "viewer", name: "Vera", role: "viewer" },
|
||||
]);
|
||||
const [s] = await db.insert(servers).values({ name: "zoe", apiTokenHash: hashToken("tok3"), apiTokenPrefix: "t3", lastSeenAt: T0.toISOString() }).returning();
|
||||
serverId = s.id;
|
||||
const a = express();
|
||||
a.use(express.json());
|
||||
a.use((req, _res, next) => {
|
||||
(req as unknown as { session: unknown }).session = { user: { sub: who } };
|
||||
next();
|
||||
});
|
||||
a.use("/api/update-runs", updateRunsRouter);
|
||||
app = await new Promise<Server>((resolve) => {
|
||||
const x = a.listen(0, "127.0.0.1", () => resolve(x));
|
||||
});
|
||||
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/update-runs`;
|
||||
});
|
||||
beforeEach(async () => {
|
||||
resetFake();
|
||||
await db.delete(updateRuns);
|
||||
await db.delete(maintenanceWindows);
|
||||
});
|
||||
after(() => app.close());
|
||||
|
||||
const body = (extra: object = {}) => ({ serverId, integrationId: 9, projectId: 2, templateId: 7, limit: "zoe", windowMinutes: 60, ...extra });
|
||||
|
||||
test("an operator starts one and everyone signed in can see it; a viewer can't start or stop one", async () => {
|
||||
who = "viewer";
|
||||
assert.equal((await call("POST", "/", body())).status, 403);
|
||||
who = "operator";
|
||||
const res = await call("POST", "/", body());
|
||||
assert.equal(res.status, 201);
|
||||
const { run } = await res.json();
|
||||
assert.equal(run.status, "running");
|
||||
assert.equal(run.active, true);
|
||||
assert.equal(run.serverName, "zoe");
|
||||
assert.equal(run.template, "Apply updates");
|
||||
assert.equal(run.startedBy, "Olle");
|
||||
|
||||
who = "viewer";
|
||||
const list = await (await call("GET", "/")).json();
|
||||
assert.equal(list.runs.length, 1);
|
||||
assert.equal(list.runs[0].id, run.id);
|
||||
assert.equal((await (await call("GET", `/?serverId=${serverId}`)).json()).runs.length, 1);
|
||||
assert.equal((await (await call("GET", "/?serverId=999")).json()).runs.length, 0);
|
||||
assert.equal((await call("POST", `/${run.id}/cancel`)).status, 403);
|
||||
|
||||
who = "operator";
|
||||
assert.equal((await call("POST", `/${run.id}/cancel`)).status, 200);
|
||||
assert.equal((await call("POST", `/${run.id}/cancel`)).status, 409, "already over");
|
||||
assert.equal((await call("POST", "/99999/cancel")).status, 404);
|
||||
});
|
||||
|
||||
test("what can't be done says why: a second run, a bad limit, a missing field, Semaphore refusing", async () => {
|
||||
assert.equal((await call("POST", "/", body())).status, 201);
|
||||
const again = await call("POST", "/", body());
|
||||
assert.equal(again.status, 409);
|
||||
assert.equal((await again.json()).error, "already_running");
|
||||
await db.delete(updateRuns);
|
||||
|
||||
for (const bad of [body({ limit: "" }), body({ limit: "--extra-vars x=1" }), body({ limit: "a b" }), body({ windowMinutes: 5 }), body({ windowMinutes: 9999 }), { serverId }]) {
|
||||
assert.equal((await call("POST", "/", bad)).status, 400, JSON.stringify(bad));
|
||||
}
|
||||
const limitReply = await (await call("POST", "/", body({ limit: "--extra-vars x=1" }))).json();
|
||||
assert.match(limitReply.message, /host name as it is in the Semaphore inventory/);
|
||||
assert.equal(fake.runs.length, 1, "only the one that was fine reached Semaphore");
|
||||
|
||||
fake.refuseToStart = true;
|
||||
const refused = await call("POST", "/", body());
|
||||
assert.equal(refused.status, 502);
|
||||
assert.match((await refused.json()).message, /Semaphore wouldn't start it/);
|
||||
assert.equal((await listUpdateRuns()).length, 0);
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user