- Map: a Dependencies view showing what would go down with a picked box and what it relies on - Maintenance: lists what depends on a target and can silence those too (same time, audited); the recurring form and update drawer show the same note - Alerts: a headline on where things stand, held-back alerts nested under their cause, severity edge on rows; caused-by now carries the explaining alert's id Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
343 lines
18 KiB
TypeScript
343 lines
18 KiB
TypeScript
/**
|
|
* Folding a guest's "offline" alert under its host's, and what servers say is waiting to be installed.
|
|
*
|
|
* Run with: npm test --workspace server
|
|
*/
|
|
import { test, describe, before, after } from "node:test";
|
|
import assert from "node:assert/strict";
|
|
import { mkdtempSync, rmSync } from "node:fs";
|
|
import { tmpdir } from "node:os";
|
|
import { join } from "node:path";
|
|
import type { AddressInfo } from "node:net";
|
|
import type { Server } from "node:http";
|
|
|
|
const dir = mkdtempSync(join(tmpdir(), "hm-parents-"));
|
|
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
|
process.env.SESSION_DIR = join(dir, "sessions");
|
|
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
|
// No waiting after a restart before servers are judged, or nothing here could be "offline".
|
|
process.env.HEALTH_STARTUP_GRACE_MS = "0";
|
|
|
|
const express = (await import("express")).default;
|
|
const { db } = await import("../src/db/client.js");
|
|
const { users, servers } = await import("../src/db/schema.js");
|
|
const { runMigrations } = await import("../src/db/migrate.js");
|
|
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
|
|
const { diffConditions, alsoList } = await import("../src/services/healthMonitor.js");
|
|
const { resolveParents, coveredByParent, wouldLoop } = await import("../src/services/serverParents.js");
|
|
const { nextUpdateState, parsePendingUpdates } = await import("../src/services/pendingUpdates.js");
|
|
const { serversRouter } = await import("../src/routes/servers.js");
|
|
const { agentReportRouter } = await import("../src/routes/agentReport.js");
|
|
const { hashToken } = await import("../src/services/tokens.js");
|
|
const { eq } = await import("drizzle-orm");
|
|
|
|
await runMigrations();
|
|
|
|
const row = (id: number, name: string, extra: Partial<{ hostname: string | null; parentServerId: number | null; proxmoxNode: string | null }> = {}) => ({
|
|
id,
|
|
name,
|
|
hostname: null,
|
|
parentServerId: null,
|
|
proxmoxNode: null,
|
|
...extra,
|
|
});
|
|
|
|
describe("resolveParents", () => {
|
|
test("a link set by hand wins, and a Proxmox node that is exactly one server is found", () => {
|
|
const rows = [row(1, "pve1", { hostname: "pve1.lab.example" }), row(2, "web", { proxmoxNode: "PVE1" }), row(3, "db", { parentServerId: 2, proxmoxNode: "pve1" }), row(4, "lone")];
|
|
const parents = resolveParents(rows);
|
|
assert.deepEqual(parents.get(2), { parentId: 1, auto: true });
|
|
assert.deepEqual(parents.get(3), { parentId: 2, auto: false }, "by hand beats the node");
|
|
assert.equal(parents.get(4), undefined);
|
|
assert.equal(parents.get(1), undefined);
|
|
});
|
|
|
|
test("two servers that could be the node, or none, give no parent; a dead link is ignored", () => {
|
|
const rows = [row(1, "pve1"), row(2, "pve1", { hostname: "other" }), row(3, "web", { proxmoxNode: "pve1" }), row(4, "x", { proxmoxNode: "nothing" }), row(5, "y", { parentServerId: 99 }), row(6, "z", { parentServerId: 6 })];
|
|
const parents = resolveParents(rows);
|
|
assert.equal(parents.get(3), undefined, "ambiguous");
|
|
assert.equal(parents.get(4), undefined);
|
|
assert.equal(parents.get(5), undefined);
|
|
assert.equal(parents.get(6), undefined, "not its own parent");
|
|
});
|
|
|
|
test("one said to run on nothing is not guessed at, and a link set by hand clears that", () => {
|
|
const rows = [row(1, "pve1"), { ...row(2, "web", { proxmoxNode: "pve1" }), noParent: true }, { ...row(3, "db", { proxmoxNode: "pve1", parentServerId: 1 }), noParent: true }];
|
|
const parents = resolveParents(rows);
|
|
assert.equal(parents.get(2), undefined, "no guess");
|
|
assert.equal(parents.get(3), undefined, "no link either while it is set");
|
|
assert.deepEqual(resolveParents([row(1, "pve1"), row(2, "web", { proxmoxNode: "pve1" })]).get(2), { parentId: 1, auto: true }, "unset, it is found again");
|
|
});
|
|
|
|
test("a server is not its own node", () => {
|
|
assert.equal(resolveParents([row(1, "pve1", { proxmoxNode: "pve1" })]).size, 0);
|
|
});
|
|
});
|
|
|
|
describe("coveredByParent", () => {
|
|
const parents = resolveParents([row(1, "host"), row(2, "vm", { parentServerId: 1 }), row(3, "ct", { parentServerId: 2 }), row(4, "other", { parentServerId: 1 })]);
|
|
|
|
test("a guest is covered only while something above it is down, and by the topmost one", () => {
|
|
assert.deepEqual([...coveredByParent(new Set([1, 2, 3]), parents)].sort(), [[2, 1], [3, 1]]);
|
|
assert.deepEqual([...coveredByParent(new Set([2, 3]), parents)], [[3, 2]], "the host is up: the VM is the cause");
|
|
assert.deepEqual([...coveredByParent(new Set([3]), parents)], [], "nothing above it is down");
|
|
assert.deepEqual([...coveredByParent(new Set([1]), parents)], []);
|
|
assert.deepEqual([...coveredByParent(new Set([1, 3]), parents)], [[3, 1]], "an up VM in between doesn't hide the cause");
|
|
});
|
|
|
|
test("a loop covers nothing, so nothing is swallowed", () => {
|
|
const loop = resolveParents([row(1, "a", { parentServerId: 2 }), row(2, "b", { parentServerId: 1 }), row(3, "c", { parentServerId: 1 })]);
|
|
assert.equal(coveredByParent(new Set([1, 2, 3]), loop).size, 0);
|
|
});
|
|
|
|
test("wouldLoop spots a link that would close a loop", () => {
|
|
const rows = [row(1, "host"), row(2, "vm", { parentServerId: 1 })];
|
|
assert.equal(wouldLoop(rows, 1, 2), true);
|
|
assert.equal(wouldLoop(rows, 1, 1), true);
|
|
assert.equal(wouldLoop(rows, 2, 1), false);
|
|
});
|
|
});
|
|
|
|
describe("diffConditions with covered problems", () => {
|
|
const c = (key: string) => ({ key, source: "server", message: key });
|
|
const names = (list: { key: string }[]) => list.map((x) => x.key).sort();
|
|
|
|
test("a problem explained by another is recorded but not announced, and neither is its end", () => {
|
|
const first = diffConditions({}, [c("offline:server:1"), c("offline:server:2")], new Set(), new Set(), new Set(["offline:server:2"]));
|
|
assert.deepEqual(names(first.added), ["offline:server:1"]);
|
|
assert.deepEqual(names(first.newlyDown), ["offline:server:1", "offline:server:2"], "both are down, for the record");
|
|
assert.equal(first.next["offline:server:2"].covered, true);
|
|
|
|
const later = diffConditions(first.next, [c("offline:server:1")], new Set(), new Set(), new Set());
|
|
assert.deepEqual(names(later.resolved), [], "the guest's end isn't announced either");
|
|
assert.deepEqual(names(later.ended), ["offline:server:2"], "but it did end");
|
|
});
|
|
|
|
test("when the explanation goes away and the problem is still there, it is announced as new", () => {
|
|
const first = diffConditions({}, [c("offline:server:1"), c("offline:server:2")], new Set(), new Set(), new Set(["offline:server:2"]));
|
|
const hostBack = diffConditions(first.next, [c("offline:server:2")], new Set(), new Set(), new Set());
|
|
assert.deepEqual(names(hostBack.added), ["offline:server:2"]);
|
|
assert.deepEqual(names(hostBack.resolved), ["offline:server:1"]);
|
|
assert.deepEqual(names(hostBack.newlyDown), [], "it was already down; nothing began");
|
|
// and only once
|
|
const again = diffConditions(hostBack.next, [c("offline:server:2")], new Set(), new Set(), new Set());
|
|
assert.deepEqual(again.added, []);
|
|
});
|
|
|
|
test("with nothing covered it works as before", () => {
|
|
const run = diffConditions({}, [c("a"), c("b")]);
|
|
assert.deepEqual(names(run.added), ["a", "b"]);
|
|
assert.deepEqual(names(run.newlyDown), ["a", "b"]);
|
|
const gone = diffConditions(run.next, [c("a")]);
|
|
assert.deepEqual(names(gone.resolved), ["b"]);
|
|
assert.deepEqual(names(gone.ended), ["b"]);
|
|
});
|
|
|
|
test("a list of names reads as words", () => {
|
|
assert.equal(alsoList(["a"]), "a");
|
|
assert.equal(alsoList(["a", "b", "c"]), "a, b and c");
|
|
assert.equal(alsoList(["a", "b", "c", "d", "e", "f", "g"]), "a, b, c, d, e and 2 more");
|
|
});
|
|
});
|
|
|
|
describe("the alerts, folded", () => {
|
|
const hoursAgo = (n: number) => new Date(Date.now() - n * 3600_000).toISOString();
|
|
let hostId = 0;
|
|
|
|
before(async () => {
|
|
const made = await db
|
|
.insert(servers)
|
|
.values([
|
|
{ name: "pve1", apiTokenHash: "h", apiTokenPrefix: "h", lastSeenAt: hoursAgo(2) },
|
|
{ name: "web", apiTokenHash: "w", apiTokenPrefix: "w", lastSeenAt: hoursAgo(2), proxmoxNode: "pve1" },
|
|
{ name: "db", apiTokenHash: "d", apiTokenPrefix: "d", lastSeenAt: hoursAgo(2), proxmoxNode: "pve1" },
|
|
{ name: "elsewhere", apiTokenHash: "e", apiTokenPrefix: "e", lastSeenAt: hoursAgo(2) },
|
|
])
|
|
.returning({ id: servers.id });
|
|
hostId = made[0].id;
|
|
});
|
|
|
|
test("the host's alert says what it takes with it, and the guests' are held back, not counted", async () => {
|
|
const report = await getAlerts(true);
|
|
const offline = report.alerts.filter((a) => a.category === "offline");
|
|
assert.equal(offline.length, 4);
|
|
const host = offline.find((a) => a.message.startsWith("pve1"))!;
|
|
assert.deepEqual(host.affects, ["db", "web"]);
|
|
assert.equal(host.silenced, false);
|
|
const web = offline.find((a) => a.message.startsWith("web"))!;
|
|
assert.deepEqual(web.causedBy, { name: "pve1", alertId: host.id });
|
|
assert.equal(web.silenced, true, "held back like a silenced alert");
|
|
assert.equal(offline.find((a) => a.message.startsWith("elsewhere"))!.causedBy, undefined);
|
|
assert.equal(report.counts.critical, 2, "pve1 and the unrelated one");
|
|
assert.equal(report.counts.grouped, 2);
|
|
assert.equal(report.counts.silenced, 0, "none of this is maintenance");
|
|
});
|
|
|
|
test("when the host reports again the guests that are still down stand alone", async () => {
|
|
await db.update(servers).set({ lastSeenAt: new Date().toISOString() }).where(eq(servers.id, hostId));
|
|
invalidateAlertsCache(); // a look under ten seconds old is reused otherwise
|
|
const report = await getAlerts(true);
|
|
assert.equal(report.counts.critical, 3);
|
|
assert.equal(report.counts.grouped, 0);
|
|
assert.ok(report.alerts.every((a) => !a.causedBy));
|
|
});
|
|
});
|
|
|
|
describe("what servers say is waiting to be installed", () => {
|
|
test("how long updates have been waiting carries from report to report", () => {
|
|
const t0 = "2026-10-01T00:00:00.000Z";
|
|
const t1 = "2026-10-05T00:00:00.000Z";
|
|
const t2 = "2026-10-09T00:00:00.000Z";
|
|
const t3 = "2026-10-12T00:00:00.000Z";
|
|
const first = nextUpdateState({ pendingUpdates: null, updatesPendingSince: null }, { manager: "apt", count: 4, security: 1 }, t0);
|
|
assert.equal(first.updatesPendingSince, t0);
|
|
const second = nextUpdateState(first, { manager: "apt", count: 6, security: 2, reboot_required: true }, t1);
|
|
assert.equal(second.updatesPendingSince, t0, "still waiting: the clock keeps running");
|
|
assert.deepEqual(parsePendingUpdates(second.pendingUpdates), { manager: "apt", count: 6, security: 2, rebootRequired: true, reportedAt: t1 });
|
|
const cleared = nextUpdateState(second, { manager: "apt", count: 0, security: 0 }, t2);
|
|
assert.equal(cleared.updatesPendingSince, null);
|
|
assert.equal(nextUpdateState(cleared, { manager: "apt", count: 2 }, t3).updatesPendingSince, t3, "and starts again when some appear");
|
|
assert.equal(parsePendingUpdates(nextUpdateState(cleared, { count: 2 }, t3).pendingUpdates)?.security, null, "unknown security is unknown, not zero");
|
|
});
|
|
|
|
test("garbage is no updates, not an error", () => {
|
|
assert.equal(parsePendingUpdates(null), null);
|
|
assert.equal(parsePendingUpdates("{"), null);
|
|
assert.equal(parsePendingUpdates(JSON.stringify({ count: "many" })), null);
|
|
});
|
|
});
|
|
|
|
describe("the routes", () => {
|
|
let app: Server;
|
|
let base = "";
|
|
let who = "admin";
|
|
let hostId = 0;
|
|
let guestId = 0;
|
|
|
|
before(async () => {
|
|
await db.insert(users).values([
|
|
{ oidcSub: "admin", name: "Ada", role: "admin" },
|
|
{ oidcSub: "operator", name: "Olga", role: "operator" },
|
|
]);
|
|
const [host] = await db.insert(servers).values({ name: "rack", apiTokenHash: hashToken("host-token"), apiTokenPrefix: "ht", lastSeenAt: new Date().toISOString() }).returning();
|
|
const [guest] = await db.insert(servers).values({ name: "vm1", apiTokenHash: hashToken("guest-token"), apiTokenPrefix: "gt", lastSeenAt: new Date().toISOString() }).returning();
|
|
hostId = host.id;
|
|
guestId = guest.id;
|
|
|
|
const a = express();
|
|
a.use("/api/agent/report", agentReportRouter);
|
|
a.use(express.json());
|
|
a.use((req, _res, next) => {
|
|
(req as unknown as { session: unknown }).session = { user: { sub: who } };
|
|
next();
|
|
});
|
|
a.use("/api/servers", serversRouter);
|
|
app = await new Promise<Server>((resolve) => {
|
|
const s = a.listen(0, "127.0.0.1", () => resolve(s));
|
|
});
|
|
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api`;
|
|
});
|
|
|
|
after(() => {
|
|
app.close();
|
|
try {
|
|
rmSync(dir, { recursive: true, force: true });
|
|
} catch {
|
|
// Windows keeps the database file locked until the process exits.
|
|
}
|
|
});
|
|
|
|
const call = (method: string, path: string, body?: unknown) =>
|
|
fetch(`${base}${path}`, { method, headers: { "content-type": "application/json" }, body: body === undefined ? undefined : JSON.stringify(body) });
|
|
const report = (token: string, system: unknown) =>
|
|
fetch(`${base}/agent/report`, { method: "POST", headers: { "content-type": "application/json", authorization: `Bearer ${token}` }, body: JSON.stringify({ hostname: "h", os_type: "linux", system, tasks: [] }) });
|
|
|
|
test("an agent's update counts are stored, a report without them keeps what was said, and a garbled answer costs nothing", async () => {
|
|
assert.equal((await report("guest-token", { updates: { manager: "apt", count: 7, security: 2, reboot_required: true } })).status, 202);
|
|
let [s] = await db.select().from(servers).where(eq(servers.id, guestId));
|
|
assert.deepEqual(parsePendingUpdates(s.pendingUpdates), { manager: "apt", count: 7, security: 2, rebootRequired: true, reportedAt: parsePendingUpdates(s.pendingUpdates)!.reportedAt });
|
|
const since = s.updatesPendingSince;
|
|
assert.ok(since);
|
|
|
|
assert.equal((await report("guest-token", { cpu: { cores: 2 } })).status, 202);
|
|
[s] = await db.select().from(servers).where(eq(servers.id, guestId));
|
|
assert.equal(parsePendingUpdates(s.pendingUpdates)?.count, 7, "an agent that said nothing doesn't wipe it");
|
|
|
|
assert.equal((await report("guest-token", { updates: { count: "lots" }, cpu: { cores: 4 } })).status, 202, "the report itself still goes through");
|
|
[s] = await db.select().from(servers).where(eq(servers.id, guestId));
|
|
assert.equal(s.cpuCores, 4);
|
|
assert.equal(parsePendingUpdates(s.pendingUpdates)?.count, 7);
|
|
assert.equal(s.updatesPendingSince, since);
|
|
|
|
assert.equal((await report("host-token", { updates: { manager: "dnf", count: 0 } })).status, 202);
|
|
});
|
|
|
|
test("the updates list shows every server that reports, with or without figures", async () => {
|
|
await db.insert(servers).values({ name: "silent", apiTokenHash: "s", apiTokenPrefix: "s" }); // never reported
|
|
const body = await (await call("GET", "/servers/updates")).json();
|
|
const byName = new Map(body.servers.map((x: { name: string }) => [x.name, x]));
|
|
assert.ok(!byName.has("silent"));
|
|
const vm = byName.get("vm1") as { updates: { count: number; security: number; rebootRequired: boolean }; pendingSince: string };
|
|
assert.equal(vm.updates.count, 7);
|
|
assert.equal(vm.updates.rebootRequired, true);
|
|
assert.ok(vm.pendingSince);
|
|
assert.equal((byName.get("rack") as { pendingSince: string | null }).pendingSince, null, "nothing waiting");
|
|
});
|
|
|
|
test("an admin says what a server runs on; a loop or a stranger is refused; it shows on both pages", async () => {
|
|
assert.equal((await call("PATCH", `/servers/${guestId}`, { parentServerId: hostId })).status, 200);
|
|
const detail = await (await call("GET", `/servers/${guestId}/detail`)).json();
|
|
assert.deepEqual(detail.relations.parent, { id: hostId, name: "rack", auto: false });
|
|
const hostDetail = await (await call("GET", `/servers/${hostId}/detail`)).json();
|
|
assert.deepEqual(hostDetail.relations.children, [{ id: guestId, name: "vm1", auto: false }]);
|
|
assert.equal(detail.server.pendingUpdates.count, 7, "the detail shows what is waiting, as a figure rather than stored text");
|
|
|
|
const loop = await call("PATCH", `/servers/${hostId}`, { parentServerId: guestId });
|
|
assert.equal(loop.status, 400);
|
|
assert.equal((await loop.json()).error, "parent_loop");
|
|
assert.equal((await call("PATCH", `/servers/${guestId}`, { parentServerId: guestId })).status, 400);
|
|
assert.equal((await call("PATCH", `/servers/${guestId}`, { parentServerId: 99999 })).status, 400);
|
|
|
|
who = "operator";
|
|
assert.equal((await call("PATCH", `/servers/${guestId}`, { parentServerId: null })).status, 403);
|
|
who = "admin";
|
|
assert.equal((await call("PATCH", `/servers/${guestId}`, { parentServerId: null })).status, 200);
|
|
assert.equal((await (await call("GET", `/servers/${guestId}/detail`)).json()).relations.parent, null, "back to working it out");
|
|
});
|
|
|
|
test("an admin can say a server runs on nothing, even when its Proxmox node would be found, and take that back", async () => {
|
|
const detailOf = async () => (await call("GET", `/servers/${guestId}/detail`)).json();
|
|
await db.update(servers).set({ proxmoxNode: "rack" }).where(eq(servers.id, guestId));
|
|
assert.deepEqual((await detailOf()).relations.parent, { id: hostId, name: "rack", auto: true }, "found from the node");
|
|
|
|
assert.equal((await call("PATCH", `/servers/${guestId}`, { noParent: true })).status, 200);
|
|
let d = await detailOf();
|
|
assert.equal(d.relations.parent, null);
|
|
assert.equal(d.server.noParent, true);
|
|
assert.deepEqual((await (await call("GET", `/servers/${hostId}/detail`)).json()).relations.children, [], "and the host no longer lists it");
|
|
|
|
// choosing a server by hand ends it; so does going back to automatic
|
|
assert.equal((await call("PATCH", `/servers/${guestId}`, { parentServerId: hostId })).status, 200);
|
|
d = await detailOf();
|
|
assert.equal(d.server.noParent, false);
|
|
assert.deepEqual(d.relations.parent, { id: hostId, name: "rack", auto: false });
|
|
await call("PATCH", `/servers/${guestId}`, { noParent: true });
|
|
assert.equal((await call("PATCH", `/servers/${guestId}`, { parentServerId: null })).status, 200);
|
|
d = await detailOf();
|
|
assert.equal(d.server.noParent, false);
|
|
assert.deepEqual(d.relations.parent, { id: hostId, name: "rack", auto: true });
|
|
|
|
who = "operator";
|
|
assert.equal((await call("PATCH", `/servers/${guestId}`, { noParent: true })).status, 403);
|
|
who = "admin";
|
|
await db.update(servers).set({ proxmoxNode: null }).where(eq(servers.id, guestId));
|
|
});
|
|
|
|
test("removing the host sets the servers on it free", async () => {
|
|
await call("PATCH", `/servers/${guestId}`, { parentServerId: hostId });
|
|
assert.equal((await call("DELETE", `/servers/${hostId}`)).status, 204);
|
|
const [s] = await db.select().from(servers).where(eq(servers.id, guestId));
|
|
assert.equal(s.parentServerId, null);
|
|
});
|
|
});
|