Files
Homelab-manager/server/test/serverThresholds.test.ts
T
bobbanandClaude Sonnet 5.5 7a5b623444 Stale-backup alerts, per-server alert limits, and update runs from the Updates page
- Backups: a guest whose newest backup is older than its limit is an alert (backup kind),
  in the daily backup message, the digest and /metrics (homelab_backup_guests, ..._last_age_seconds)
- Per-server thresholds (servers.thresholds, migration 0030): offline minutes, disk percent,
  forecast window, backup age, and per-disk limits or "leave alone"; applied everywhere the
  global ones were; saving clears the alerts cache
- Update runs (update_runs, migration 0031): Update... on the Updates page puts the server under
  maintenance, runs a Semaphore playbook limited to it, waits for the host to report, and
  reports what is left and whether a restart is due; poller survives restarts
- Notification toggle for update-run results; tests and docs

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-18 01:25:09 +02:00

237 lines
14 KiB
TypeScript

/**
* Alert thresholds that belong to one server: how they are read and checked, what they change in the alerts, the forecast and /metrics,
* and who may set them.
*
* Run with: npm test --workspace server
*/
import { test, describe, before, after } from "node:test";
import assert from "node:assert/strict";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { AddressInfo } from "node:net";
import type { Server } from "node:http";
const dir = mkdtempSync(join(tmpdir(), "hm-thresholds-"));
process.env.DATABASE_PATH = join(dir, "test.sqlite");
process.env.SESSION_DIR = join(dir, "sessions");
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
process.env.HEALTH_STARTUP_GRACE_MS = "0";
const express = (await import("express")).default;
const { db } = await import("../src/db/client.js");
const { users, servers, serverMetrics, auditLog } = await import("../src/db/schema.js");
const { runMigrations } = await import("../src/db/migrate.js");
const { parseThresholds, checkThresholds, effectiveThresholds, diskLimit } = await import("../src/services/serverThresholds.js");
const { evaluateHealth } = await import("../src/services/healthMonitor.js");
const { disksFillingSoon } = await import("../src/services/serverMetrics.js");
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
const { buildMetrics } = await import("../src/services/metrics.js");
const { serversRouter } = await import("../src/routes/servers.js");
const { hashToken } = await import("../src/services/tokens.js");
const { eq } = await import("drizzle-orm");
await runMigrations();
const GLOBAL = { serverOfflineMinutes: 60, diskUsagePercent: 90, diskForecastDays: 14, backupStaleHours: 48 };
describe("reading and checking thresholds", () => {
test("what is stored is read back leniently: nonsense is dropped, never an error", () => {
assert.equal(parseThresholds(null), null);
assert.equal(parseThresholds("not json"), null);
assert.equal(parseThresholds("{}"), null);
assert.deepEqual(parseThresholds(JSON.stringify({ diskUsagePercent: 95, serverOfflineMinutes: 5, diskForecastDays: "soon", mounts: { "/data": { percent: 97 }, "/x": { percent: 3 }, "/y": "no", "/z": { ignore: true } } })), {
diskUsagePercent: 95,
mounts: { "/data": { percent: 97 }, "/z": { ignore: true } },
});
});
test("a request is refused with a message that says which number is wrong", () => {
assert.deepEqual(checkThresholds(null), { ok: true, value: null });
assert.deepEqual(checkThresholds({}), { ok: true, value: null });
assert.deepEqual(checkThresholds({ diskUsagePercent: 95, serverOfflineMinutes: null }), { ok: true, value: { diskUsagePercent: 95 } });
const bad = (input: unknown) => {
const r = checkThresholds(input);
assert.equal(r.ok, false, JSON.stringify(input));
return (r as { message: string }).message;
};
assert.match(bad({ diskUsagePercent: 20 }), /Disk usage alert at \(%\) must be a whole number from 50 to 99/);
assert.match(bad({ serverOfflineMinutes: 5 }), /Offline after \(minutes\)/);
assert.match(bad({ diskForecastDays: 1.5 }), /Disk forecast/);
assert.match(bad({ backupStaleHours: 99999 }), /Backup older than/);
assert.match(bad({ mounts: { "/data": { percent: 100 } } }), /limit for \/data/);
assert.match(bad({ mounts: { "": { ignore: true } } }), /missing or too long/);
assert.match(bad("high"), /should be an object/);
assert.match(bad({ mounts: Object.fromEntries(Array.from({ length: 51 }, (_, i) => [`/m${i}`, { ignore: true }])) }), /At most 50/);
});
test("a value of its own wins, and the rest follows Settings — including 0, which means off", () => {
assert.deepEqual(effectiveThresholds(GLOBAL, null), GLOBAL);
assert.deepEqual(effectiveThresholds(GLOBAL, { diskUsagePercent: 95, diskForecastDays: 0 }), { ...GLOBAL, diskUsagePercent: 95, diskForecastDays: 0 });
});
test("a disk's limit: its own rule, then the server's, then Settings; or none at all", () => {
const own = { diskUsagePercent: 95, mounts: { "/data": { percent: 80 }, "/boot": { ignore: true } } };
assert.equal(diskLimit(own, "/data", 90), 80);
assert.equal(diskLimit(own, "/", 90), 95);
assert.equal(diskLimit(own, "/boot", 90), null);
assert.equal(diskLimit(null, "/", 90), 90);
assert.equal(diskLimit({}, "/", 90), 90);
});
});
describe("what it changes in the health rules", () => {
const NOW = Date.parse("2026-10-17T12:00:00Z");
const disk = (mount: string, used: number) => ({ mount, sizeBytes: 100, usedBytes: used });
const snap = (extra: object) => ({ servers: [{ id: 1, name: "nas", lastSeenAt: new Date(NOW - 5 * 60_000).toISOString(), disks: [disk("/", 92), disk("/data", 85), disk("/boot", 99)], ...extra }], proxmox: [], synology: [] });
const keys = (extra: object, now = NOW) => evaluateHealth(snap(extra), { serverOfflineMinutes: 60, diskUsagePercent: 90 }, now).map((c) => c.key).sort();
test("with nothing of its own, the global limit applies to every disk", () => {
assert.deepEqual(keys({}), ["disk:server:1:/", "disk:server:1:/boot"]);
});
test("a server's own percentage replaces the global one; a disk's own rule replaces both; an ignored disk says nothing", () => {
assert.deepEqual(keys({ thresholds: { diskUsagePercent: 95 } }), ["disk:server:1:/boot"], "92% is fine for a server that alerts at 95");
assert.deepEqual(keys({ thresholds: { diskUsagePercent: 80 } }), ["disk:server:1:/", "disk:server:1:/boot", "disk:server:1:/data"]);
assert.deepEqual(keys({ thresholds: { mounts: { "/boot": { ignore: true }, "/data": { percent: 80 } } } }), ["disk:server:1:/", "disk:server:1:/data"]);
assert.deepEqual(keys({ thresholds: { diskUsagePercent: 99, mounts: { "/": { percent: 91 }, "/boot": { ignore: true } } } }), ["disk:server:1:/"]);
assert.deepEqual(keys({ thresholds: { diskUsagePercent: 99, mounts: { "/": { percent: 91 } } } }), ["disk:server:1:/", "disk:server:1:/boot"], "99% is at the limit");
});
test("a server's own 'offline after' replaces the global one in either direction", () => {
const quietFor = (minutes: number) => new Date(NOW - minutes * 60_000).toISOString();
assert.deepEqual(keys({ lastSeenAt: quietFor(120), disks: [] }), ["offline:server:1"]);
assert.deepEqual(keys({ lastSeenAt: quietFor(120), disks: [], thresholds: { serverOfflineMinutes: 300 } }), [], "a laptop that is off at night");
assert.deepEqual(keys({ lastSeenAt: quietFor(30), disks: [], thresholds: { serverOfflineMinutes: 15 } }), ["offline:server:1"], "a host that should speak up sooner");
});
test("the forecast leaves out a disk it is told to ignore, and uses the disk's own limit", () => {
const series = [
{ metric: "disk:/data", kind: "disk" as const, label: "Disk /data", points: [], latest: 85, trend: { perDay: 2, fit: 0.99, spanDays: 7, daysToFull: 7 } },
{ metric: "disk:/scratch", kind: "disk" as const, label: "Disk /scratch", points: [], latest: 60, trend: { perDay: 2, fit: 0.99, spanDays: 7, daysToFull: 20 } },
];
assert.deepEqual(disksFillingSoon(series, 30, 90).map((f) => f.mount), ["/data", "/scratch"]);
assert.deepEqual(disksFillingSoon(series, 30, (m) => (m === "/scratch" ? null : 90)).map((f) => f.mount), ["/data"], "told to leave /scratch alone");
assert.deepEqual(disksFillingSoon(series, 30, (m) => (m === "/data" ? 80 : 90)).map((f) => f.mount), ["/scratch"], "/data is past its own 80%, so the full-disk alert has it");
});
});
describe("through the server routes, the alerts and /metrics", () => {
let app: Server;
let base = "";
let who = "admin";
let id = 0;
const call = (method: string, path: string, body?: unknown) => fetch(`${base}${path}`, { method, headers: { "content-type": "application/json" }, body: body === undefined ? undefined : JSON.stringify(body) });
const patch = (body: unknown) => call("PATCH", `/servers/${id}`, body);
before(async () => {
await db.insert(users).values([
{ oidcSub: "admin", name: "Ada", role: "admin" },
{ oidcSub: "operator", name: "Olle", role: "operator" },
]);
const twoHoursAgo = new Date(Date.now() - 2 * 3600_000).toISOString();
const [s] = await db
.insert(servers)
.values({ name: "laptop", apiTokenHash: hashToken("tok"), apiTokenPrefix: "t", lastSeenAt: twoHoursAgo, disks: JSON.stringify([{ mount: "/", sizeBytes: 100, usedBytes: 93 }, { mount: "/boot", sizeBytes: 100, usedBytes: 99 }]) })
.returning();
id = s.id;
const a = express();
a.use(express.json());
a.use((req, _res, next) => {
(req as unknown as { session: unknown }).session = { user: { sub: who } };
next();
});
a.use("/servers", serversRouter);
app = await new Promise<Server>((resolve) => {
const x = a.listen(0, "127.0.0.1", () => resolve(x));
});
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}`;
});
after(() => {
app.close();
try {
rmSync(dir, { recursive: true, force: true });
} catch {
// Windows keeps the database file locked until the process exits.
}
});
const alertIds = async () => {
invalidateAlertsCache();
return (await getAlerts(true)).alerts.filter((a) => a.id.includes(`:server:${id}`)).map((a) => a.id).sort();
};
test("only an administrator can set them, a wrong number is refused, and nothing changes until it is right", async () => {
who = "operator";
assert.equal((await patch({ thresholds: { diskUsagePercent: 95 } })).status, 403);
who = "admin";
const bad = await patch({ thresholds: { diskUsagePercent: 20 } });
assert.equal(bad.status, 400);
const body = await bad.json();
assert.equal(body.error, "invalid_thresholds");
assert.match(body.message, /Disk usage alert at/);
const detail = await (await call("GET", `/servers/${id}/detail`)).json();
assert.equal(detail.server.thresholds, null);
assert.deepEqual(detail.thresholdDefaults.diskUsagePercent, 90, "the page can say what it would follow");
assert.equal(detail.thresholdDefaults.backupStaleHours, 48);
});
test("with none set, the laptop has been quiet for two hours, so it is offline (and an offline server's disks aren't judged)", async () => {
assert.deepEqual(await alertIds(), [`offline:offline:server:${id}`]);
});
test("the Alerts page doesn't go on showing the old answer for a minute after a server's limits change", async () => {
await patch({ thresholds: null });
invalidateAlertsCache();
assert.ok((await getAlerts(false)).alerts.some((a) => a.id === `offline:offline:server:${id}`), "offline, as the cache now holds it");
await patch({ thresholds: { serverOfflineMinutes: 600 } });
assert.equal((await getAlerts(false)).alerts.some((a) => a.id === `offline:offline:server:${id}`), false, "no longer, straight away — without anyone clearing the cache");
await patch({ thresholds: null });
});
test("its own rules quiet exactly what they say, show on the detail, and are in the audit log", async () => {
const res = await patch({ thresholds: { serverOfflineMinutes: 600, diskUsagePercent: 95, mounts: { "/boot": { ignore: true } } } });
assert.equal(res.status, 200);
assert.deepEqual((await res.json()).server.thresholds, { serverOfflineMinutes: 600, diskUsagePercent: 95, mounts: { "/boot": { ignore: true } } });
assert.deepEqual(await alertIds(), [], "not offline for ten hours, 93% is under 95, and /boot is left alone");
const detail = await (await call("GET", `/servers/${id}/detail`)).json();
assert.equal(detail.server.thresholds.diskUsagePercent, 95);
const entry = (await db.select().from(auditLog)).filter((e) => e.action === "update" && e.targetId === String(id)).pop()!;
assert.deepEqual(JSON.parse(entry.detail!).thresholds, { serverOfflineMinutes: 600, diskUsagePercent: 95, mounts: { "/boot": { ignore: true } } });
// the list shows them too, as an object
const row = (await (await call("GET", "/servers")).json()).servers.find((x: { id: number }) => x.id === id);
assert.equal(row.thresholds.serverOfflineMinutes, 600);
});
test("a disk's own limit brings its alert back, and the server's own 'offline after' is what /metrics uses", async () => {
await patch({ thresholds: { serverOfflineMinutes: 600, diskUsagePercent: 95, mounts: { "/": { percent: 90 }, "/boot": { ignore: true } } } });
assert.deepEqual(await alertIds(), [`disk:disk:server:${id}:/`]);
assert.match(await buildMetrics(), /homelab_server_up\{server="laptop"\} 1/, "two hours quiet is fine when it may be quiet for ten");
await patch({ thresholds: null });
assert.match(await buildMetrics(), /homelab_server_up\{server="laptop"\} 0/, "and offline when it follows the global hour");
assert.equal((await (await call("GET", `/servers/${id}/detail`)).json()).server.thresholds, null, "null goes back to following Settings");
});
test("the forecast follows the server's own days (0 turns it off for it) and its ignored disks", async () => {
const day = 86_400_000;
const rows = [];
for (let i = 0; i < 10; i++) rows.push({ serverId: id, metric: "disk:/data", value: 30 + i * 4, recordedAt: new Date(Date.now() - (10 - i) * day).toISOString() });
await db.insert(serverMetrics).values(rows);
await db.update(servers).set({ lastSeenAt: new Date().toISOString() }).where(eq(servers.id, id)); // online, so its figures count
const forecast = async () => {
invalidateAlertsCache();
return (await getAlerts(true)).alerts.filter((a) => a.id.includes(`disk-forecast:${id}:`)).map((a) => a.id);
};
assert.equal((await forecast()).length, 1, "66% and +4 a day: full in about nine days");
await patch({ thresholds: { diskForecastDays: 0 } });
assert.deepEqual(await forecast(), [], "0 turns it off for this server");
await patch({ thresholds: { diskForecastDays: 60, mounts: { "/data": { ignore: true } } } });
assert.deepEqual(await forecast(), [], "an ignored disk has no forecast either");
await patch({ thresholds: { diskForecastDays: 60 } });
assert.equal((await forecast()).length, 1, "a longer window still sees it");
// and the stale-backup limit rides along, checked like the others
assert.equal((await patch({ thresholds: { backupStaleHours: 240 } })).status, 200);
assert.equal((await patch({ thresholds: { backupStaleHours: 5000 } })).status, 400);
});
});