/** * Heartbeats: what a job reporting in (or not) amounts to, the address it calls, what the pass announces, and who may set them up. * * Run with: npm test --workspace server */ import { test, describe, before, after } from "node:test"; import assert from "node:assert/strict"; import { createServer as createHttpServer, type Server } from "node:http"; import { mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import type { AddressInfo } from "node:net"; const dir = mkdtempSync(join(tmpdir(), "hm-heartbeats-")); process.env.DATABASE_PATH = join(dir, "test.sqlite"); process.env.SESSION_DIR = join(dir, "sessions"); process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32); process.env.HEALTH_STARTUP_GRACE_MS = "0"; const express = (await import("express")).default; const { db } = await import("../src/db/client.js"); const { users, servers, heartbeats, heartbeatPings } = await import("../src/db/schema.js"); const { runMigrations } = await import("../src/db/migrate.js"); const { dueBy, heartbeatState, heartbeatConditions, cleanMessage, parseStamp, recordPing, newToken, TOKEN_PATTERN } = await import("../src/services/heartbeats.js"); const { heartbeatsRouter } = await import("../src/routes/heartbeats.js"); const { heartbeatPingRouter } = await import("../src/routes/heartbeatPing.js"); const { runEndpointPass } = await import("../src/services/endpointMonitor.js"); const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js"); const { startOrExtendWindow } = await import("../src/services/maintenance.js"); const { metaOfConditionKey } = await import("../src/services/notifyRouting.js"); const { subjectOfConditionKey } = await import("../src/services/maintenance.js"); const { updateSettings } = await import("../src/services/settingsStore.js"); const { buildMetrics } = await import("../src/services/metrics.js"); const { eq } = await import("drizzle-orm"); await runMigrations(); type Row = typeof heartbeats.$inferSelect; const NOW = new Date("2026-10-19T03:00:00Z"); const row = (extra: Partial = {}): Row => ({ id: 1, name: "Nightly backup", token: "hb_x", periodMinutes: 1440, graceMinutes: 60, serverId: null, enabled: true, notes: null, lastPingAt: null, lastStatus: null, lastMessage: null, lastDurationMs: null, lastStartAt: null, pingCount: 0, createdAt: "2026-10-18 03:00:00", ...extra, }); const minutes = (n: number) => new Date(NOW.getTime() + n * 60_000); describe("when a job is due, and how it is doing", () => { test("it is due a period and the grace after its last call — or after it was made, if it never called", () => { assert.equal(dueBy(row({ lastPingAt: "2026-10-19T02:00:00.000Z" })).toISOString(), "2026-10-20T03:00:00.000Z"); assert.equal(dueBy(row()).toISOString(), "2026-10-19T04:00:00.000Z", "SQLite's own timestamp format is read as UTC"); assert.equal(parseStamp("2026-10-18 03:00:00"), Date.parse("2026-10-18T03:00:00Z")); }); test("new, up, running, late, failed and switched off", () => { const called = { lastPingAt: NOW.toISOString(), lastStatus: "ok" }; assert.equal(heartbeatState(row({ createdAt: "2026-10-19 02:00:00" }), NOW), "waiting", "made recently, has not called, not yet due"); assert.equal(heartbeatState(row(), NOW), "waiting", "due by 04:00 today"); assert.equal(heartbeatState(row(), minutes(61)), "late", "never called, and now it is overdue"); assert.equal(heartbeatState(row(called), minutes(60 * 24)), "up"); assert.equal(heartbeatState(row(called), minutes(60 * 25)), "up", "inside the grace"); assert.equal(heartbeatState(row(called), minutes(60 * 25 + 1)), "late"); assert.equal(heartbeatState(row({ ...called, lastStartAt: minutes(5).toISOString() }), minutes(10)), "running", "it began after it last finished"); assert.equal(heartbeatState(row({ ...called, lastStartAt: minutes(-5).toISOString() }), minutes(10)), "up", "a start older than the last finish is not a run in progress"); assert.equal(heartbeatState(row({ ...called, lastStatus: "fail", lastMessage: "disk full" }), minutes(10)), "failed", "said so, and does not wait for the time to pass"); assert.equal(heartbeatState(row({ enabled: false }), minutes(9999)), "off"); }); test("the problems: late is a warning with the time it was due, a reported failure is critical; each keyed by the server it runs on", () => { const list = heartbeatConditions( [ row({ id: 1, serverId: 7 }), row({ id: 2, name: "Photos sync", lastPingAt: NOW.toISOString(), lastStatus: "fail", lastMessage: "exit 3" }), row({ id: 3, name: "Fine", lastPingAt: NOW.toISOString(), lastStatus: "ok" }), row({ id: 4, name: "Paused", enabled: false }), ], minutes(120), ); assert.deepEqual(list.map((c) => [c.key, c.severity, c.category, c.serverId]), [["heartbeat:1:server:7", "warning", "automation", 7], ["heartbeat-fail:2", "critical", "automation", null]]); assert.match(list[0].message, /Nightly backup hasn't reported — it was due by 2026-10-19 04:00 UTC \(it has never reported\)/); assert.match(list[1].message, /Photos sync reported a failure — exit 3/); assert.deepEqual(metaOfConditionKey("heartbeat:1:server:7", "warning"), { category: "automation", severity: "warning", serverId: 7 }, "tags and categories route it"); assert.equal(subjectOfConditionKey("heartbeat-fail:2:server:9"), "server:9", "and maintenance on the server holds it back"); }); test("a note from a job is one short line", () => { assert.equal(cleanMessage("3.2 GB\n\tcopied in 41 s\u0000"), "3.2 GB copied in 41 s"); assert.equal(cleanMessage(" "), null); assert.equal(cleanMessage(42), null); assert.equal(cleanMessage("x".repeat(500))?.length, 200); }); test("an address carries a long random token, and only that shape is looked up", () => { const t = newToken(); assert.match(t, TOKEN_PATTERN); assert.notEqual(t, newToken()); for (const bad of ["", "hb_", "hb_short", "nope_" + "a".repeat(24), "hb_" + "a/b".repeat(10), "hb_" + "a".repeat(100)]) assert.equal(TOKEN_PATTERN.test(bad), false, bad); }); }); describe("the address, the routes and the pass", () => { let app: Server; let hook: Server; let base = ""; let who = "admin"; let serverId = 0; let id = 0; let path = ""; const messages: string[] = []; before(async () => { await db.insert(users).values([ { oidcSub: "admin", name: "Ada", role: "admin" }, { oidcSub: "operator", name: "Olga", role: "operator" }, { oidcSub: "viewer", name: "Vic", role: "viewer" }, ]); serverId = (await db.insert(servers).values({ name: "nas", apiTokenHash: "n", apiTokenPrefix: "n", lastSeenAt: new Date().toISOString() }).returning({ id: servers.id }))[0].id; hook = createHttpServer((req, res) => { const chunks: Buffer[] = []; req.on("data", (c) => chunks.push(c)); req.on("end", () => { messages.push(String(JSON.parse(Buffer.concat(chunks).toString()).content)); res.end("{}"); }); }); await new Promise((r) => hook.listen(0, "127.0.0.1", r)); await updateSettings({ webhook: { enabled: true, url: `http://127.0.0.1:${(hook.address() as AddressInfo).port}/` } }); const a = express(); // The ping address comes first, as in the real app: no session, no JSON parser, no sign-in. a.use("/api/heartbeat", heartbeatPingRouter); a.use(express.json()); a.use((req, _res, next) => { (req as unknown as { session: unknown }).session = { user: { sub: who } }; next(); }); a.use("/api/heartbeats", heartbeatsRouter); app = await new Promise((resolve) => { const s = a.listen(0, "127.0.0.1", () => resolve(s)); }); base = `http://127.0.0.1:${(app.address() as AddressInfo).port}`; }); after(() => { app.close(); hook.close(); try { rmSync(dir, { recursive: true, force: true }); } catch { // Windows keeps the database file locked until the process exits. } }); const admin = (method: string, p: string, body?: unknown) => fetch(`${base}/api/heartbeats${p}`, { method, headers: { "content-type": "application/json" }, body: body === undefined ? undefined : JSON.stringify(body) }); const ping = (p: string, init?: RequestInit) => fetch(`${base}${p}`, init); test("an administrator makes one; a wrong one is refused with a reason; nobody else can; everyone can look", async () => { const res = await admin("POST", "/", { name: "Nightly backup", periodMinutes: 1440, graceMinutes: 60, serverId }); assert.equal(res.status, 201); const made = (await res.json()).heartbeat; id = made.id; path = made.path; assert.match(path, /^\/api\/heartbeat\/hb_[A-Za-z0-9_-]{20,}$/); for (const bad of [{ periodMinutes: 5 }, { name: "x" }, { name: "x", periodMinutes: 0 }, { name: "x", periodMinutes: 60 * 24 * 31 }, { name: "x", periodMinutes: 5, graceMinutes: 99999 }, { name: "x", periodMinutes: 5, serverId: 99999 }, { name: " ", periodMinutes: 5 }]) { assert.equal((await admin("POST", "/", bad)).status, 400, JSON.stringify(bad)); } who = "operator"; assert.equal((await admin("POST", "/", { name: "y", periodMinutes: 5 })).status, 403); assert.equal((await admin("DELETE", `/${id}`)).status, 403); const list = await (await admin("GET", "/")).json(); assert.equal(list.heartbeats[0].path, null, "the address is for administrators: it is what lets anyone say \"I'm alive\""); assert.equal(list.baseUrl, null); assert.equal(list.heartbeats[0].state, "waiting"); assert.equal(list.heartbeats[0].server.name, "nas"); who = "admin"; const adminList = await (await admin("GET", "/")).json(); assert.equal(adminList.heartbeats[0].path, path); assert.ok(adminList.baseUrl); assert.equal("token" in adminList.heartbeats[0], false, "the token is not a field of its own"); }); test("a call is noted: success, a note in the body or the query, and how long a run took when it said it started", async () => { assert.equal((await ping(path)).status, 200); assert.equal(await (await ping(path, { method: "HEAD" })).text(), ""); let [h] = await db.select().from(heartbeats).where(eq(heartbeats.id, id)); assert.deepEqual([h.lastStatus, h.pingCount], ["ok", 2]); assert.equal(heartbeatState(h, new Date()), "up"); assert.equal((await ping(`${path}/start`, { method: "POST" })).status, 200); await new Promise((r) => setTimeout(r, 30)); const done = await ping(path, { method: "POST", body: "3.2 GB\ncopied", headers: { "content-type": "text/plain" } }); assert.equal(await done.text(), "OK"); [h] = await db.select().from(heartbeats).where(eq(heartbeats.id, id)); assert.equal(h.lastMessage, "3.2 GB copied"); assert.ok(h.lastDurationMs !== null && h.lastDurationMs >= 20, "timed from the start call"); await ping(`${path}?msg=from%20the%20query`); [h] = await db.select().from(heartbeats).where(eq(heartbeats.id, id)); assert.equal(h.lastMessage, "from the query"); assert.equal(h.lastDurationMs, null, "no start before this one, so no duration"); const rows = await db.select().from(heartbeatPings).where(eq(heartbeatPings.heartbeatId, id)); assert.deepEqual(rows.map((r) => r.kind), ["ok", "ok", "start", "ok", "ok"]); }); test("a token that does not exist, a path that is not one of the three and a method that is not a call all get no information", async () => { const a = await ping("/api/heartbeat/hb_" + "z".repeat(24)); const b = await ping(`${path}/bogus`); const c = await ping("/api/heartbeat/nonsense"); assert.deepEqual([a.status, b.status, c.status], [404, 404, 404]); assert.equal(await a.text(), await b.text(), "never made, and a wrong path, read the same"); assert.equal((await ping(path, { method: "DELETE" })).status, 405); assert.equal((await ping(path)).headers.get("cache-control"), "no-store"); }); test("a job that says it failed is an alert at once, until it next succeeds; the pass says so once, and once when it is over", async () => { messages.length = 0; await db.update(heartbeats).set({ serverId: null }).where(eq(heartbeats.id, id)); assert.equal((await ping(`${path}/fail`, { method: "POST", body: "exit 3: disk full" })).status, 200); invalidateAlertsCache(); const alert = (await getAlerts(true)).alerts.find((x) => x.source === "Heartbeat")!; assert.match(alert.message, /Nightly backup reported a failure — exit 3: disk full/); assert.deepEqual([alert.severity, alert.category, alert.link], ["critical", "automation", "/heartbeats"]); let r = await runEndpointPass(new Date()); assert.equal(r.announced, 1); assert.match(messages.join("\n"), /Nightly backup reported a failure/); const sent = messages.length; r = await runEndpointPass(new Date()); assert.equal(r.announced + r.recovered, 0); assert.equal(messages.length, sent, "not said again"); await ping(path); r = await runEndpointPass(new Date()); assert.equal(r.recovered, 1); assert.match(messages[messages.length - 1], /Nightly backup reported a failure/, "the recovery names what cleared"); invalidateAlertsCache(); assert.equal((await getAlerts(true)).alerts.filter((x) => x.source === "Heartbeat").length, 0); }); test("a call that does not come in time is a warning once it is overdue, silenced by maintenance on the server, and cleared by the call", async () => { messages.length = 0; await db.update(heartbeats).set({ serverId, periodMinutes: 60, graceMinutes: 10 }).where(eq(heartbeats.id, id)); const t = (min: number) => new Date(Date.now() + min * 60_000); let r = await runEndpointPass(t(30)); assert.equal(r.announced, 0, "not due yet"); r = await runEndpointPass(t(75)); assert.equal(r.announced, 1, "an hour and ten minutes with no call"); assert.match(messages.join("\n"), /Nightly backup hasn't reported — it was due by/); // The Alerts page reads the real clock, so the last call is moved back instead of the clock forward. await db.update(heartbeats).set({ lastPingAt: new Date(Date.now() - 120 * 60_000).toISOString() }).where(eq(heartbeats.id, id)); invalidateAlertsCache(); const late = (await getAlerts(true)).alerts.find((x) => x.source === "Heartbeat")!; assert.equal(late.severity, "warning"); assert.deepEqual(late.target && [late.target.type, late.target.id], ["server", serverId]); assert.equal(late.silenced, false); await startOrExtendWindow("server", serverId, 60, "work", "Ada"); invalidateAlertsCache(); assert.equal((await getAlerts(true)).alerts.find((x) => x.source === "Heartbeat")!.silenced, true, "maintenance on the server it runs on silences it"); assert.match(await buildMetrics(t(75)), /homelab_heartbeat_up\{name="Nightly backup"\} 0/); await recordPing((await db.select().from(heartbeats).where(eq(heartbeats.id, id)))[0].token, "ok", null, t(76)); r = await runEndpointPass(t(77)); assert.equal(r.recovered, 1); assert.match(await buildMetrics(t(77)), /homelab_heartbeat_up\{name="Nightly backup"\} 1/); assert.match(await buildMetrics(t(77)), /homelab_heartbeat_last_ping_timestamp_seconds\{name="Nightly backup"\} \d+/); }); test("a switched-off heartbeat still takes calls, says nothing, and starts a fresh clock when it is switched on again", async () => { await admin("PUT", `/${id}`, { name: "Nightly backup", periodMinutes: 60, graceMinutes: 10, serverId: null, enabled: false }); const [h] = await db.select().from(heartbeats).where(eq(heartbeats.id, id)); assert.equal((await ping(`/api/heartbeat/${h.token}`)).status, 200, "pausing a job's heartbeat does not break its script"); assert.equal(heartbeatConditions([{ ...h, lastPingAt: "2020-01-01T00:00:00.000Z", lastStatus: "ok" }], new Date()).length, 0); await admin("PUT", `/${id}`, { name: "Nightly backup", periodMinutes: 60, graceMinutes: 10, serverId: null, enabled: true }); const [after_] = await db.select().from(heartbeats).where(eq(heartbeats.id, id)); assert.equal(heartbeatState(after_, new Date()), "up"); }); test("a new address replaces the old one at once; removing one takes its history", async () => { const res = await admin("POST", `/${id}/new-address`); assert.equal(res.status, 200); const fresh = (await res.json()).path as string; assert.notEqual(fresh, path); assert.equal((await ping(path)).status, 404, "the old one is dead"); assert.equal((await ping(fresh)).status, 200); who = "viewer"; assert.equal((await admin("POST", `/${id}/new-address`)).status, 403); who = "admin"; assert.equal((await admin("DELETE", `/${id}`)).status, 204); assert.equal((await db.select().from(heartbeatPings).where(eq(heartbeatPings.heartbeatId, id))).length, 0); assert.equal((await ping(fresh)).status, 404); assert.equal((await admin("DELETE", `/${id}`)).status, 404); }); });