Heartbeats, Coming up and suggested checks; Operations menu tidied
Heartbeats: a cron job or script calls /api/heartbeat/<token> (and /start, /fail); the app alerts when the call is late or the job says it failed. Coming up: one agenda of everything dated (expiries, warranties, end of support, planned and recurring maintenance, restore tests, waiting updates), with a dashboard card and an iCalendar feed behind a share link. Suggested checks: web ports and certificates the app already knows about, added with a click. Menu: Incidents become Alerts -> History (old /incidents links redirect); new Monitoring group (Endpoints, Heartbeats, Uptime Kuma); osTicket under Automation, Admin Links under Infrastructure, Generator under Administration. Also makes the restore-test metrics test follow the real clock. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
c05c847d17
commit
4e0c6927b2
44 files changed
+7092
-75
No files matched your search
@@ -0,0 +1,292 @@
|
||||
/**
|
||||
* Heartbeats: what a job reporting in (or not) amounts to, the address it calls, what the pass announces, and who may set them up.
|
||||
*
|
||||
* Run with: npm test --workspace server
|
||||
*/
|
||||
import { test, describe, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { createServer as createHttpServer, type Server } from "node:http";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { AddressInfo } from "node:net";
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), "hm-heartbeats-"));
|
||||
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
||||
process.env.SESSION_DIR = join(dir, "sessions");
|
||||
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
||||
process.env.HEALTH_STARTUP_GRACE_MS = "0";
|
||||
|
||||
const express = (await import("express")).default;
|
||||
const { db } = await import("../src/db/client.js");
|
||||
const { users, servers, heartbeats, heartbeatPings } = await import("../src/db/schema.js");
|
||||
const { runMigrations } = await import("../src/db/migrate.js");
|
||||
const { dueBy, heartbeatState, heartbeatConditions, cleanMessage, parseStamp, recordPing, newToken, TOKEN_PATTERN } = await import("../src/services/heartbeats.js");
|
||||
const { heartbeatsRouter } = await import("../src/routes/heartbeats.js");
|
||||
const { heartbeatPingRouter } = await import("../src/routes/heartbeatPing.js");
|
||||
const { runEndpointPass } = await import("../src/services/endpointMonitor.js");
|
||||
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
|
||||
const { startOrExtendWindow } = await import("../src/services/maintenance.js");
|
||||
const { metaOfConditionKey } = await import("../src/services/notifyRouting.js");
|
||||
const { subjectOfConditionKey } = await import("../src/services/maintenance.js");
|
||||
const { updateSettings } = await import("../src/services/settingsStore.js");
|
||||
const { buildMetrics } = await import("../src/services/metrics.js");
|
||||
const { eq } = await import("drizzle-orm");
|
||||
|
||||
await runMigrations();
|
||||
|
||||
type Row = typeof heartbeats.$inferSelect;
|
||||
const NOW = new Date("2026-10-19T03:00:00Z");
|
||||
const row = (extra: Partial<Row> = {}): Row => ({
|
||||
id: 1, name: "Nightly backup", token: "hb_x", periodMinutes: 1440, graceMinutes: 60, serverId: null, enabled: true, notes: null, lastPingAt: null, lastStatus: null, lastMessage: null,
|
||||
lastDurationMs: null, lastStartAt: null, pingCount: 0, createdAt: "2026-10-18 03:00:00", ...extra,
|
||||
});
|
||||
const minutes = (n: number) => new Date(NOW.getTime() + n * 60_000);
|
||||
|
||||
describe("when a job is due, and how it is doing", () => {
|
||||
test("it is due a period and the grace after its last call — or after it was made, if it never called", () => {
|
||||
assert.equal(dueBy(row({ lastPingAt: "2026-10-19T02:00:00.000Z" })).toISOString(), "2026-10-20T03:00:00.000Z");
|
||||
assert.equal(dueBy(row()).toISOString(), "2026-10-19T04:00:00.000Z", "SQLite's own timestamp format is read as UTC");
|
||||
assert.equal(parseStamp("2026-10-18 03:00:00"), Date.parse("2026-10-18T03:00:00Z"));
|
||||
});
|
||||
|
||||
test("new, up, running, late, failed and switched off", () => {
|
||||
const called = { lastPingAt: NOW.toISOString(), lastStatus: "ok" };
|
||||
assert.equal(heartbeatState(row({ createdAt: "2026-10-19 02:00:00" }), NOW), "waiting", "made recently, has not called, not yet due");
|
||||
assert.equal(heartbeatState(row(), NOW), "waiting", "due by 04:00 today");
|
||||
assert.equal(heartbeatState(row(), minutes(61)), "late", "never called, and now it is overdue");
|
||||
assert.equal(heartbeatState(row(called), minutes(60 * 24)), "up");
|
||||
assert.equal(heartbeatState(row(called), minutes(60 * 25)), "up", "inside the grace");
|
||||
assert.equal(heartbeatState(row(called), minutes(60 * 25 + 1)), "late");
|
||||
assert.equal(heartbeatState(row({ ...called, lastStartAt: minutes(5).toISOString() }), minutes(10)), "running", "it began after it last finished");
|
||||
assert.equal(heartbeatState(row({ ...called, lastStartAt: minutes(-5).toISOString() }), minutes(10)), "up", "a start older than the last finish is not a run in progress");
|
||||
assert.equal(heartbeatState(row({ ...called, lastStatus: "fail", lastMessage: "disk full" }), minutes(10)), "failed", "said so, and does not wait for the time to pass");
|
||||
assert.equal(heartbeatState(row({ enabled: false }), minutes(9999)), "off");
|
||||
});
|
||||
|
||||
test("the problems: late is a warning with the time it was due, a reported failure is critical; each keyed by the server it runs on", () => {
|
||||
const list = heartbeatConditions(
|
||||
[
|
||||
row({ id: 1, serverId: 7 }),
|
||||
row({ id: 2, name: "Photos sync", lastPingAt: NOW.toISOString(), lastStatus: "fail", lastMessage: "exit 3" }),
|
||||
row({ id: 3, name: "Fine", lastPingAt: NOW.toISOString(), lastStatus: "ok" }),
|
||||
row({ id: 4, name: "Paused", enabled: false }),
|
||||
],
|
||||
minutes(120),
|
||||
);
|
||||
assert.deepEqual(list.map((c) => [c.key, c.severity, c.category, c.serverId]), [["heartbeat:1:server:7", "warning", "automation", 7], ["heartbeat-fail:2", "critical", "automation", null]]);
|
||||
assert.match(list[0].message, /Nightly backup hasn't reported — it was due by 2026-10-19 04:00 UTC \(it has never reported\)/);
|
||||
assert.match(list[1].message, /Photos sync reported a failure — exit 3/);
|
||||
assert.deepEqual(metaOfConditionKey("heartbeat:1:server:7", "warning"), { category: "automation", severity: "warning", serverId: 7 }, "tags and categories route it");
|
||||
assert.equal(subjectOfConditionKey("heartbeat-fail:2:server:9"), "server:9", "and maintenance on the server holds it back");
|
||||
});
|
||||
|
||||
test("a note from a job is one short line", () => {
|
||||
assert.equal(cleanMessage("3.2 GB\n\tcopied in 41 s\u0000"), "3.2 GB copied in 41 s");
|
||||
assert.equal(cleanMessage(" "), null);
|
||||
assert.equal(cleanMessage(42), null);
|
||||
assert.equal(cleanMessage("x".repeat(500))?.length, 200);
|
||||
});
|
||||
|
||||
test("an address carries a long random token, and only that shape is looked up", () => {
|
||||
const t = newToken();
|
||||
assert.match(t, TOKEN_PATTERN);
|
||||
assert.notEqual(t, newToken());
|
||||
for (const bad of ["", "hb_", "hb_short", "nope_" + "a".repeat(24), "hb_" + "a/b".repeat(10), "hb_" + "a".repeat(100)]) assert.equal(TOKEN_PATTERN.test(bad), false, bad);
|
||||
});
|
||||
});
|
||||
|
||||
describe("the address, the routes and the pass", () => {
|
||||
let app: Server;
|
||||
let hook: Server;
|
||||
let base = "";
|
||||
let who = "admin";
|
||||
let serverId = 0;
|
||||
let id = 0;
|
||||
let path = "";
|
||||
const messages: string[] = [];
|
||||
|
||||
before(async () => {
|
||||
await db.insert(users).values([
|
||||
{ oidcSub: "admin", name: "Ada", role: "admin" },
|
||||
{ oidcSub: "operator", name: "Olga", role: "operator" },
|
||||
{ oidcSub: "viewer", name: "Vic", role: "viewer" },
|
||||
]);
|
||||
serverId = (await db.insert(servers).values({ name: "nas", apiTokenHash: "n", apiTokenPrefix: "n", lastSeenAt: new Date().toISOString() }).returning({ id: servers.id }))[0].id;
|
||||
hook = createHttpServer((req, res) => {
|
||||
const chunks: Buffer[] = [];
|
||||
req.on("data", (c) => chunks.push(c));
|
||||
req.on("end", () => {
|
||||
messages.push(String(JSON.parse(Buffer.concat(chunks).toString()).content));
|
||||
res.end("{}");
|
||||
});
|
||||
});
|
||||
await new Promise<void>((r) => hook.listen(0, "127.0.0.1", r));
|
||||
await updateSettings({ webhook: { enabled: true, url: `http://127.0.0.1:${(hook.address() as AddressInfo).port}/` } });
|
||||
|
||||
const a = express();
|
||||
// The ping address comes first, as in the real app: no session, no JSON parser, no sign-in.
|
||||
a.use("/api/heartbeat", heartbeatPingRouter);
|
||||
a.use(express.json());
|
||||
a.use((req, _res, next) => {
|
||||
(req as unknown as { session: unknown }).session = { user: { sub: who } };
|
||||
next();
|
||||
});
|
||||
a.use("/api/heartbeats", heartbeatsRouter);
|
||||
app = await new Promise<Server>((resolve) => {
|
||||
const s = a.listen(0, "127.0.0.1", () => resolve(s));
|
||||
});
|
||||
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}`;
|
||||
});
|
||||
|
||||
after(() => {
|
||||
app.close();
|
||||
hook.close();
|
||||
try {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Windows keeps the database file locked until the process exits.
|
||||
}
|
||||
});
|
||||
|
||||
const admin = (method: string, p: string, body?: unknown) => fetch(`${base}/api/heartbeats${p}`, { method, headers: { "content-type": "application/json" }, body: body === undefined ? undefined : JSON.stringify(body) });
|
||||
const ping = (p: string, init?: RequestInit) => fetch(`${base}${p}`, init);
|
||||
|
||||
test("an administrator makes one; a wrong one is refused with a reason; nobody else can; everyone can look", async () => {
|
||||
const res = await admin("POST", "/", { name: "Nightly backup", periodMinutes: 1440, graceMinutes: 60, serverId });
|
||||
assert.equal(res.status, 201);
|
||||
const made = (await res.json()).heartbeat;
|
||||
id = made.id;
|
||||
path = made.path;
|
||||
assert.match(path, /^\/api\/heartbeat\/hb_[A-Za-z0-9_-]{20,}$/);
|
||||
for (const bad of [{ periodMinutes: 5 }, { name: "x" }, { name: "x", periodMinutes: 0 }, { name: "x", periodMinutes: 60 * 24 * 31 }, { name: "x", periodMinutes: 5, graceMinutes: 99999 }, { name: "x", periodMinutes: 5, serverId: 99999 }, { name: " ", periodMinutes: 5 }]) {
|
||||
assert.equal((await admin("POST", "/", bad)).status, 400, JSON.stringify(bad));
|
||||
}
|
||||
who = "operator";
|
||||
assert.equal((await admin("POST", "/", { name: "y", periodMinutes: 5 })).status, 403);
|
||||
assert.equal((await admin("DELETE", `/${id}`)).status, 403);
|
||||
const list = await (await admin("GET", "/")).json();
|
||||
assert.equal(list.heartbeats[0].path, null, "the address is for administrators: it is what lets anyone say \"I'm alive\"");
|
||||
assert.equal(list.baseUrl, null);
|
||||
assert.equal(list.heartbeats[0].state, "waiting");
|
||||
assert.equal(list.heartbeats[0].server.name, "nas");
|
||||
who = "admin";
|
||||
const adminList = await (await admin("GET", "/")).json();
|
||||
assert.equal(adminList.heartbeats[0].path, path);
|
||||
assert.ok(adminList.baseUrl);
|
||||
assert.equal("token" in adminList.heartbeats[0], false, "the token is not a field of its own");
|
||||
});
|
||||
|
||||
test("a call is noted: success, a note in the body or the query, and how long a run took when it said it started", async () => {
|
||||
assert.equal((await ping(path)).status, 200);
|
||||
assert.equal(await (await ping(path, { method: "HEAD" })).text(), "");
|
||||
let [h] = await db.select().from(heartbeats).where(eq(heartbeats.id, id));
|
||||
assert.deepEqual([h.lastStatus, h.pingCount], ["ok", 2]);
|
||||
assert.equal(heartbeatState(h, new Date()), "up");
|
||||
|
||||
assert.equal((await ping(`${path}/start`, { method: "POST" })).status, 200);
|
||||
await new Promise((r) => setTimeout(r, 30));
|
||||
const done = await ping(path, { method: "POST", body: "3.2 GB\ncopied", headers: { "content-type": "text/plain" } });
|
||||
assert.equal(await done.text(), "OK");
|
||||
[h] = await db.select().from(heartbeats).where(eq(heartbeats.id, id));
|
||||
assert.equal(h.lastMessage, "3.2 GB copied");
|
||||
assert.ok(h.lastDurationMs !== null && h.lastDurationMs >= 20, "timed from the start call");
|
||||
|
||||
await ping(`${path}?msg=from%20the%20query`);
|
||||
[h] = await db.select().from(heartbeats).where(eq(heartbeats.id, id));
|
||||
assert.equal(h.lastMessage, "from the query");
|
||||
assert.equal(h.lastDurationMs, null, "no start before this one, so no duration");
|
||||
const rows = await db.select().from(heartbeatPings).where(eq(heartbeatPings.heartbeatId, id));
|
||||
assert.deepEqual(rows.map((r) => r.kind), ["ok", "ok", "start", "ok", "ok"]);
|
||||
});
|
||||
|
||||
test("a token that does not exist, a path that is not one of the three and a method that is not a call all get no information", async () => {
|
||||
const a = await ping("/api/heartbeat/hb_" + "z".repeat(24));
|
||||
const b = await ping(`${path}/bogus`);
|
||||
const c = await ping("/api/heartbeat/nonsense");
|
||||
assert.deepEqual([a.status, b.status, c.status], [404, 404, 404]);
|
||||
assert.equal(await a.text(), await b.text(), "never made, and a wrong path, read the same");
|
||||
assert.equal((await ping(path, { method: "DELETE" })).status, 405);
|
||||
assert.equal((await ping(path)).headers.get("cache-control"), "no-store");
|
||||
});
|
||||
|
||||
test("a job that says it failed is an alert at once, until it next succeeds; the pass says so once, and once when it is over", async () => {
|
||||
messages.length = 0;
|
||||
await db.update(heartbeats).set({ serverId: null }).where(eq(heartbeats.id, id));
|
||||
assert.equal((await ping(`${path}/fail`, { method: "POST", body: "exit 3: disk full" })).status, 200);
|
||||
invalidateAlertsCache();
|
||||
const alert = (await getAlerts(true)).alerts.find((x) => x.source === "Heartbeat")!;
|
||||
assert.match(alert.message, /Nightly backup reported a failure — exit 3: disk full/);
|
||||
assert.deepEqual([alert.severity, alert.category, alert.link], ["critical", "automation", "/heartbeats"]);
|
||||
|
||||
let r = await runEndpointPass(new Date());
|
||||
assert.equal(r.announced, 1);
|
||||
assert.match(messages.join("\n"), /Nightly backup reported a failure/);
|
||||
const sent = messages.length;
|
||||
r = await runEndpointPass(new Date());
|
||||
assert.equal(r.announced + r.recovered, 0);
|
||||
assert.equal(messages.length, sent, "not said again");
|
||||
|
||||
await ping(path);
|
||||
r = await runEndpointPass(new Date());
|
||||
assert.equal(r.recovered, 1);
|
||||
assert.match(messages[messages.length - 1], /Nightly backup reported a failure/, "the recovery names what cleared");
|
||||
invalidateAlertsCache();
|
||||
assert.equal((await getAlerts(true)).alerts.filter((x) => x.source === "Heartbeat").length, 0);
|
||||
});
|
||||
|
||||
test("a call that does not come in time is a warning once it is overdue, silenced by maintenance on the server, and cleared by the call", async () => {
|
||||
messages.length = 0;
|
||||
await db.update(heartbeats).set({ serverId, periodMinutes: 60, graceMinutes: 10 }).where(eq(heartbeats.id, id));
|
||||
const t = (min: number) => new Date(Date.now() + min * 60_000);
|
||||
let r = await runEndpointPass(t(30));
|
||||
assert.equal(r.announced, 0, "not due yet");
|
||||
r = await runEndpointPass(t(75));
|
||||
assert.equal(r.announced, 1, "an hour and ten minutes with no call");
|
||||
assert.match(messages.join("\n"), /Nightly backup hasn't reported — it was due by/);
|
||||
|
||||
// The Alerts page reads the real clock, so the last call is moved back instead of the clock forward.
|
||||
await db.update(heartbeats).set({ lastPingAt: new Date(Date.now() - 120 * 60_000).toISOString() }).where(eq(heartbeats.id, id));
|
||||
invalidateAlertsCache();
|
||||
const late = (await getAlerts(true)).alerts.find((x) => x.source === "Heartbeat")!;
|
||||
assert.equal(late.severity, "warning");
|
||||
assert.deepEqual(late.target && [late.target.type, late.target.id], ["server", serverId]);
|
||||
assert.equal(late.silenced, false);
|
||||
await startOrExtendWindow("server", serverId, 60, "work", "Ada");
|
||||
invalidateAlertsCache();
|
||||
assert.equal((await getAlerts(true)).alerts.find((x) => x.source === "Heartbeat")!.silenced, true, "maintenance on the server it runs on silences it");
|
||||
|
||||
assert.match(await buildMetrics(t(75)), /homelab_heartbeat_up\{name="Nightly backup"\} 0/);
|
||||
await recordPing((await db.select().from(heartbeats).where(eq(heartbeats.id, id)))[0].token, "ok", null, t(76));
|
||||
r = await runEndpointPass(t(77));
|
||||
assert.equal(r.recovered, 1);
|
||||
assert.match(await buildMetrics(t(77)), /homelab_heartbeat_up\{name="Nightly backup"\} 1/);
|
||||
assert.match(await buildMetrics(t(77)), /homelab_heartbeat_last_ping_timestamp_seconds\{name="Nightly backup"\} \d+/);
|
||||
});
|
||||
|
||||
test("a switched-off heartbeat still takes calls, says nothing, and starts a fresh clock when it is switched on again", async () => {
|
||||
await admin("PUT", `/${id}`, { name: "Nightly backup", periodMinutes: 60, graceMinutes: 10, serverId: null, enabled: false });
|
||||
const [h] = await db.select().from(heartbeats).where(eq(heartbeats.id, id));
|
||||
assert.equal((await ping(`/api/heartbeat/${h.token}`)).status, 200, "pausing a job's heartbeat does not break its script");
|
||||
assert.equal(heartbeatConditions([{ ...h, lastPingAt: "2020-01-01T00:00:00.000Z", lastStatus: "ok" }], new Date()).length, 0);
|
||||
await admin("PUT", `/${id}`, { name: "Nightly backup", periodMinutes: 60, graceMinutes: 10, serverId: null, enabled: true });
|
||||
const [after_] = await db.select().from(heartbeats).where(eq(heartbeats.id, id));
|
||||
assert.equal(heartbeatState(after_, new Date()), "up");
|
||||
});
|
||||
|
||||
test("a new address replaces the old one at once; removing one takes its history", async () => {
|
||||
const res = await admin("POST", `/${id}/new-address`);
|
||||
assert.equal(res.status, 200);
|
||||
const fresh = (await res.json()).path as string;
|
||||
assert.notEqual(fresh, path);
|
||||
assert.equal((await ping(path)).status, 404, "the old one is dead");
|
||||
assert.equal((await ping(fresh)).status, 200);
|
||||
who = "viewer";
|
||||
assert.equal((await admin("POST", `/${id}/new-address`)).status, 403);
|
||||
who = "admin";
|
||||
assert.equal((await admin("DELETE", `/${id}`)).status, 204);
|
||||
assert.equal((await db.select().from(heartbeatPings).where(eq(heartbeatPings.heartbeatId, id))).length, 0);
|
||||
assert.equal((await ping(fresh)).status, 404);
|
||||
assert.equal((await admin("DELETE", `/${id}`)).status, 404);
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user