Endpoint checks, restore tests, and a change log
- Endpoints: HTTP, TCP and TLS checks with answer time, uptime and certificate expiry; failures and expiring certificates become alerts and notifications; linked to a server for maintenance and tags - Restore tests: record when a backup was restored and how it went; failed tests always alert, overdue ones with a limit set; shown on the Backup coverage page - Change log: one filterable timeline made from the audit log and the incident record Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
c3bc6bbf72
commit
92726568b7
37 files changed
+6918
-21
No files matched your search
@@ -0,0 +1,48 @@
|
||||
CREATE TABLE `endpoint_checks` (
|
||||
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||
`name` text NOT NULL,
|
||||
`kind` text NOT NULL,
|
||||
`target` text NOT NULL,
|
||||
`port` integer,
|
||||
`expect_status` text,
|
||||
`keyword` text,
|
||||
`insecure` integer DEFAULT false NOT NULL,
|
||||
`interval_minutes` integer DEFAULT 5 NOT NULL,
|
||||
`timeout_seconds` integer DEFAULT 10 NOT NULL,
|
||||
`failures_before_alert` integer DEFAULT 2 NOT NULL,
|
||||
`warn_days` integer DEFAULT 14 NOT NULL,
|
||||
`server_id` integer,
|
||||
`enabled` integer DEFAULT true NOT NULL,
|
||||
`last_checked_at` text,
|
||||
`last_ok` integer,
|
||||
`last_message` text,
|
||||
`last_response_ms` integer,
|
||||
`cert_expires_at` text,
|
||||
`consecutive_failures` integer DEFAULT 0 NOT NULL,
|
||||
`last_change_at` text,
|
||||
`created_at` text DEFAULT (current_timestamp) NOT NULL
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE `endpoint_results` (
|
||||
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||
`check_id` integer NOT NULL,
|
||||
`at` text NOT NULL,
|
||||
`ok` integer NOT NULL,
|
||||
`ms` integer,
|
||||
`message` text,
|
||||
FOREIGN KEY (`check_id`) REFERENCES `endpoint_checks`(`id`) ON UPDATE no action ON DELETE cascade
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE INDEX `endpoint_results_check_idx` ON `endpoint_results` (`check_id`,`at`);--> statement-breakpoint
|
||||
CREATE TABLE `restore_tests` (
|
||||
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||
`subject_key` text NOT NULL,
|
||||
`label` text NOT NULL,
|
||||
`tested_at` text NOT NULL,
|
||||
`result` text NOT NULL,
|
||||
`note` text,
|
||||
`tested_by` text,
|
||||
`created_at` text DEFAULT (current_timestamp) NOT NULL
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE INDEX `restore_tests_subject_idx` ON `restore_tests` (`subject_key`,`tested_at`);
|
||||
File diff suppressed because it is too large.
Load diff
@@ -274,6 +274,13 @@
|
||||
"when": 1792354148200,
|
||||
"tag": "0038_wise_malcolm_colcord",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 39,
|
||||
"version": "6",
|
||||
"when": 1792356659440,
|
||||
"tag": "0039_old_blur",
|
||||
"breakpoints": true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -17,6 +17,8 @@ import { incidentsRouter } from "./routes/incidents.js";
|
||||
import { inventoryRouter } from "./routes/inventory.js";
|
||||
import { dependenciesRouter } from "./routes/dependencies.js";
|
||||
import { pushRouter } from "./routes/push.js";
|
||||
import { endpointsRouter } from "./routes/endpoints.js";
|
||||
import { changesRouter } from "./routes/changes.js";
|
||||
import { backupsRouter } from "./routes/backups.js";
|
||||
import { metricsRouter, metricsTokensRouter } from "./routes/metrics.js";
|
||||
import { eventWebhooksRouter } from "./routes/eventWebhooks.js";
|
||||
@@ -110,6 +112,8 @@ export function createApp() {
|
||||
app.use("/api/inventory", inventoryRouter);
|
||||
app.use("/api/dependencies", dependenciesRouter);
|
||||
app.use("/api/push", pushRouter);
|
||||
app.use("/api/endpoints", endpointsRouter);
|
||||
app.use("/api/changes", changesRouter);
|
||||
app.use("/api/backups", backupsRouter);
|
||||
app.use("/api/metrics-tokens", metricsTokensRouter);
|
||||
app.use("/api/event-webhooks", eventWebhooksRouter);
|
||||
|
||||
@@ -127,6 +127,68 @@ export const pushSubscriptions = sqliteTable("push_subscriptions", {
|
||||
categories: text("categories"),
|
||||
});
|
||||
|
||||
// A web address, TCP port or TLS certificate the app asks about every few minutes — see services/endpointChecks.ts. The last answer is kept on the row;
|
||||
// every answer is also one row in endpoint_results (kept two weeks), for uptime and answer times.
|
||||
export const endpointChecks = sqliteTable("endpoint_checks", {
|
||||
id: integer("id").primaryKey({ autoIncrement: true }),
|
||||
name: text("name").notNull(),
|
||||
kind: text("kind").notNull(), // http | tcp | tls
|
||||
target: text("target").notNull(), // http: the web address; tcp and tls: the host
|
||||
port: integer("port"), // tcp and tls
|
||||
expectStatus: text("expect_status"), // http: "200-299, 301"; null = 200-399
|
||||
keyword: text("keyword"), // http: text the page must contain
|
||||
insecure: integer("insecure", { mode: "boolean" }).notNull().default(false), // accept a certificate that isn't trusted (self-signed, a private CA)
|
||||
intervalMinutes: integer("interval_minutes").notNull().default(5),
|
||||
timeoutSeconds: integer("timeout_seconds").notNull().default(10),
|
||||
failuresBeforeAlert: integer("failures_before_alert").notNull().default(2),
|
||||
warnDays: integer("warn_days").notNull().default(14), // warn this long before a certificate expires
|
||||
serverId: integer("server_id"), // soft reference: the server this is about, so its maintenance silences the alerts
|
||||
enabled: integer("enabled", { mode: "boolean" }).notNull().default(true),
|
||||
lastCheckedAt: text("last_checked_at"),
|
||||
lastOk: integer("last_ok", { mode: "boolean" }),
|
||||
lastMessage: text("last_message"),
|
||||
lastResponseMs: integer("last_response_ms"),
|
||||
certExpiresAt: text("cert_expires_at"), // ISO
|
||||
consecutiveFailures: integer("consecutive_failures").notNull().default(0),
|
||||
lastChangeAt: text("last_change_at"), // when it last went from working to failing or back
|
||||
createdAt: text("created_at")
|
||||
.notNull()
|
||||
.default(sql`(current_timestamp)`),
|
||||
});
|
||||
|
||||
export const endpointResults = sqliteTable(
|
||||
"endpoint_results",
|
||||
{
|
||||
id: integer("id").primaryKey({ autoIncrement: true }),
|
||||
checkId: integer("check_id")
|
||||
.notNull()
|
||||
.references(() => endpointChecks.id, { onDelete: "cascade" }),
|
||||
at: text("at").notNull(), // ISO
|
||||
ok: integer("ok", { mode: "boolean" }).notNull(),
|
||||
ms: integer("ms"),
|
||||
message: text("message"),
|
||||
},
|
||||
(t) => [index("endpoint_results_check_idx").on(t.checkId, t.at)],
|
||||
);
|
||||
|
||||
// One time somebody tried to restore a backup, and how it went — see services/restoreTests.ts. History is kept; the newest per subject is what counts.
|
||||
export const restoreTests = sqliteTable(
|
||||
"restore_tests",
|
||||
{
|
||||
id: integer("id").primaryKey({ autoIncrement: true }),
|
||||
subjectKey: text("subject_key").notNull(), // guest:<integration id>:<vmid> for a Proxmox guest, custom:<slug> for anything else
|
||||
label: text("label").notNull(),
|
||||
testedAt: text("tested_at").notNull(), // YYYY-MM-DD
|
||||
result: text("result").notNull(), // ok | partial | failed | not_needed
|
||||
note: text("note"),
|
||||
testedBy: text("tested_by"),
|
||||
createdAt: text("created_at")
|
||||
.notNull()
|
||||
.default(sql`(current_timestamp)`),
|
||||
},
|
||||
(t) => [index("restore_tests_subject_idx").on(t.subjectKey, t.testedAt)],
|
||||
);
|
||||
|
||||
// "A needs B", for things that can be down: a server or an integration each. Polymorphic like maintenance_windows (no foreign key); an edge whose
|
||||
// end was deleted is ignored. When B is down, A's alert is folded under B's — see services/dependencies.ts.
|
||||
export const dependencies = sqliteTable(
|
||||
|
||||
@@ -20,6 +20,7 @@ import { initKumaMaintenanceScheduler } from "./services/kumaMaintenanceSchedule
|
||||
import { initMaintenanceScheduleScheduler } from "./services/maintenanceScheduleScheduler.js";
|
||||
import { initUpdateRuns } from "./services/updateRuns.js";
|
||||
import { initEscalation } from "./services/escalation.js";
|
||||
import { initEndpointMonitor } from "./services/endpointMonitor.js";
|
||||
import { initSelfBackupScheduler } from "./services/selfBackupScheduler.js";
|
||||
import { initIncidentRecorder } from "./services/incidents.js";
|
||||
import { getAlerts } from "./services/alerts.js";
|
||||
@@ -59,6 +60,7 @@ initUpdateRuns();
|
||||
await initSelfBackupScheduler();
|
||||
initIncidentRecorder(() => getAlerts(false));
|
||||
initEscalation();
|
||||
initEndpointMonitor();
|
||||
|
||||
createApp().listen(env.port, () => {
|
||||
console.log(`homelab-manager listening on port ${env.port}`);
|
||||
|
||||
@@ -1,6 +1,14 @@
|
||||
import { Router } from "express";
|
||||
import { requireAuth } from "../auth/middleware.js";
|
||||
import { desc, eq } from "drizzle-orm";
|
||||
import { z } from "zod";
|
||||
import { db } from "../db/client.js";
|
||||
import { restoreTests } from "../db/schema.js";
|
||||
import { requireAuth, requireRole } from "../auth/middleware.js";
|
||||
import { recordAudit } from "../services/audit.js";
|
||||
import { invalidateAlertsCache } from "../services/alerts.js";
|
||||
import { DEFAULT_STALE_HOURS, getBackupCoverage } from "../services/backupCoverage.js";
|
||||
import { customKey, guestKey, KEY_PATTERN, latestTests, restoreInfo, RESULTS } from "../services/restoreTests.js";
|
||||
import { getSettings } from "../services/settingsStore.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
|
||||
export const backupsRouter = Router();
|
||||
@@ -9,8 +17,68 @@ export const backupsRouter = Router();
|
||||
backupsRouter.use(requireAuth);
|
||||
|
||||
// `staleDays`: how old the newest backup may be before a guest counts as stale (default 2). `refresh=1` asks Proxmox again now.
|
||||
// Each guest also says how its newest restore test went (or that there was none).
|
||||
backupsRouter.get("/coverage", asyncHandler(async (req, res) => {
|
||||
const days = Number(req.query.staleDays);
|
||||
const staleHours = Number.isFinite(days) && days >= 1 && days <= 60 ? Math.round(days * 24) : DEFAULT_STALE_HOURS;
|
||||
res.json(await getBackupCoverage({ staleHours, refresh: req.query.refresh === "1" }));
|
||||
const report = await getBackupCoverage({ staleHours, refresh: req.query.refresh === "1" });
|
||||
const now = new Date();
|
||||
const limit = (await getSettings()).healthChecks.restoreTestDays;
|
||||
const latest = await latestTests();
|
||||
res.json({
|
||||
...report,
|
||||
restoreTestDays: limit,
|
||||
guests: report.guests.map((g) => ({ ...g, restoreKey: guestKey(g.integrationId, g.vmid), restoreTest: restoreInfo(latest.get(guestKey(g.integrationId, g.vmid)), now, limit) })),
|
||||
});
|
||||
}));
|
||||
|
||||
// The history of restore tests, and how each thing that isn't a Proxmox guest (a share, a database) stands.
|
||||
backupsRouter.get("/restore-tests", asyncHandler(async (_req, res) => {
|
||||
const now = new Date();
|
||||
const limit = (await getSettings()).healthChecks.restoreTestDays;
|
||||
const latest = await latestTests();
|
||||
const recent = await db.select().from(restoreTests).orderBy(desc(restoreTests.testedAt), desc(restoreTests.id)).limit(100);
|
||||
res.json({
|
||||
limitDays: limit,
|
||||
tests: recent,
|
||||
custom: [...latest.values()].filter((r) => r.subjectKey.startsWith("custom:")).map((r) => ({ key: r.subjectKey, label: r.label, ...restoreInfo(r, now, limit) })).sort((a, b) => a.label.localeCompare(b.label)),
|
||||
});
|
||||
}));
|
||||
|
||||
const testSchema = z.object({
|
||||
/** guest:<integration>:<vmid>, or left out for something that isn't a guest (then it is told apart by its label). */
|
||||
key: z.string().optional(),
|
||||
label: z.string().trim().min(1).max(120),
|
||||
testedAt: z.string().regex(/^\d{4}-\d{2}-\d{2}$/).optional(),
|
||||
result: z.enum(RESULTS),
|
||||
note: z.string().trim().max(500).optional(),
|
||||
});
|
||||
|
||||
backupsRouter.post("/restore-tests", requireRole("operator"), asyncHandler(async (req, res) => {
|
||||
const parsed = testSchema.safeParse(req.body);
|
||||
if (!parsed.success) return res.status(400).json({ error: "invalid_body", message: "Say what was tested, how it went, and on what day (YYYY-MM-DD).", details: parsed.error.flatten() });
|
||||
const d = parsed.data;
|
||||
const key = d.key ?? customKey(d.label);
|
||||
if (!key || !KEY_PATTERN.test(key)) return res.status(400).json({ error: "invalid_key", message: "That can't be told apart from other things; give it a name with some letters or digits." });
|
||||
const today = new Date().toISOString().slice(0, 10);
|
||||
const testedAt = d.testedAt ?? today;
|
||||
if (Number.isNaN(Date.parse(`${testedAt}T00:00:00Z`)) || new Date(`${testedAt}T00:00:00Z`).toISOString().slice(0, 10) !== testedAt) return res.status(400).json({ error: "invalid_date", message: "That isn't a real date." });
|
||||
if (testedAt > today) return res.status(400).json({ error: "future", message: "A restore test can't be in the future." });
|
||||
const actor = req.currentUser!;
|
||||
const [row] = await db.insert(restoreTests).values({ subjectKey: key, label: d.label, testedAt, result: d.result, note: d.note || null, testedBy: actor.name ?? actor.email ?? actor.oidcSub }).returning();
|
||||
await recordAudit({ actor, category: "restore_test", action: "record", targetType: "restore_test", targetId: row.id, detail: { name: row.label, result: row.result, testedAt, note: row.note } });
|
||||
invalidateAlertsCache();
|
||||
res.status(201).json({ test: row });
|
||||
}));
|
||||
|
||||
// A mistaken entry can be taken out; that is an administrator's to do, since it is what the alerts are judged on.
|
||||
backupsRouter.delete("/restore-tests/:id", requireRole("admin"), asyncHandler(async (req, res) => {
|
||||
const id = /^\d+$/.test(req.params.id) ? Number(req.params.id) : NaN;
|
||||
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
|
||||
const [before] = await db.select().from(restoreTests).where(eq(restoreTests.id, id)).limit(1);
|
||||
if (!before) return res.status(404).json({ error: "not_found" });
|
||||
await db.delete(restoreTests).where(eq(restoreTests.id, id));
|
||||
await recordAudit({ actor: req.currentUser!, category: "restore_test", action: "delete", targetType: "restore_test", targetId: id, detail: { name: before.label, testedAt: before.testedAt } });
|
||||
invalidateAlertsCache();
|
||||
res.status(204).end();
|
||||
}));
|
||||
@@ -0,0 +1,27 @@
|
||||
import { Router } from "express";
|
||||
import { requireAuth, requireRole } from "../auth/middleware.js";
|
||||
import { CHANGE_KINDS, KIND_LABELS, getChangeLog, type ChangeKind } from "../services/changeLog.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
|
||||
export const changesRouter = Router();
|
||||
|
||||
// Like the audit log it is made from: operators and administrators.
|
||||
changesRouter.use(requireAuth, requireRole("operator"));
|
||||
|
||||
// ?days=7 &kind=settings &serverId=3 &q=word &before=ISO (the next page) &limit=200
|
||||
changesRouter.get("/", asyncHandler(async (req, res) => {
|
||||
const days = Number(req.query.days);
|
||||
const serverId = typeof req.query.serverId === "string" && /^\d+$/.test(req.query.serverId) ? Number(req.query.serverId) : undefined;
|
||||
const kind = CHANGE_KINDS.find((k) => k === req.query.kind) as ChangeKind | undefined;
|
||||
const before = typeof req.query.before === "string" && !Number.isNaN(Date.parse(req.query.before)) ? req.query.before : undefined;
|
||||
const limit = Number(req.query.limit);
|
||||
const result = await getChangeLog({
|
||||
days: Number.isFinite(days) && days >= 1 ? days : 7,
|
||||
kind,
|
||||
serverId,
|
||||
q: typeof req.query.q === "string" ? req.query.q.slice(0, 100) : undefined,
|
||||
before,
|
||||
limit: Number.isFinite(limit) && limit >= 1 ? limit : 200,
|
||||
});
|
||||
res.json({ ...result, kinds: CHANGE_KINDS.map((k) => ({ id: k, label: KIND_LABELS[k] })) });
|
||||
}));
|
||||
@@ -108,7 +108,7 @@ dependenciesRouter.post("/", requireRole("operator"), asyncHandler(async (req, r
|
||||
return res.status(409).json({ error: "loop", message: `${graph.names.get(to)} already depends on ${graph.names.get(from)}, so this would make them wait for each other.` });
|
||||
}
|
||||
const [row] = await db.insert(dependencies).values(d).returning();
|
||||
await recordAudit({ actor: req.currentUser!, category: "dependency", action: "create", targetType: "dependency", targetId: row.id, detail: { from: graph.names.get(from), to: graph.names.get(to) } });
|
||||
await recordAudit({ actor: req.currentUser!, category: "dependency", action: "create", targetType: "dependency", targetId: row.id, detail: { from: graph.names.get(from), to: graph.names.get(to), fromKey: from, toKey: to } });
|
||||
invalidateAlertsCache();
|
||||
res.status(201).json({ id: row.id });
|
||||
}));
|
||||
@@ -126,7 +126,7 @@ dependenciesRouter.delete("/:id", requireRole("operator"), asyncHandler(async (r
|
||||
action: "delete",
|
||||
targetType: "dependency",
|
||||
targetId: id,
|
||||
detail: { from: graph.names.get(nodeKey(before.fromType as "server", before.fromId)) ?? `${before.fromType} #${before.fromId}`, to: graph.names.get(nodeKey(before.toType as "server", before.toId)) ?? `${before.toType} #${before.toId}` },
|
||||
detail: { from: graph.names.get(nodeKey(before.fromType as "server", before.fromId)) ?? `${before.fromType} #${before.fromId}`, to: graph.names.get(nodeKey(before.toType as "server", before.toId)) ?? `${before.toType} #${before.toId}`, fromKey: `${before.fromType}:${before.fromId}`, toKey: `${before.toType}:${before.toId}` },
|
||||
});
|
||||
invalidateAlertsCache();
|
||||
res.status(204).end();
|
||||
|
||||
@@ -0,0 +1,134 @@
|
||||
import { Router } from "express";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { z } from "zod";
|
||||
import { db } from "../db/client.js";
|
||||
import { endpointChecks, servers } from "../db/schema.js";
|
||||
import { requireAuth, requireRole } from "../auth/middleware.js";
|
||||
import { recordAudit } from "../services/audit.js";
|
||||
import { invalidateAlertsCache } from "../services/alerts.js";
|
||||
import { CHECK_KINDS, isStatusSpec, loadStats, runCheck } from "../services/endpointChecks.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
|
||||
export const endpointsRouter = Router();
|
||||
|
||||
endpointsRouter.use(requireAuth);
|
||||
|
||||
const MAX_CHECKS = 100;
|
||||
|
||||
const checkSchema = z
|
||||
.object({
|
||||
name: z.string().trim().min(1).max(100),
|
||||
kind: z.enum(CHECK_KINDS),
|
||||
target: z.string().trim().min(1).max(500),
|
||||
port: z.number().int().min(1).max(65535).nullable().optional(),
|
||||
expectStatus: z.string().trim().max(60).nullable().optional(),
|
||||
keyword: z.string().max(200).nullable().optional(),
|
||||
insecure: z.boolean().optional(),
|
||||
intervalMinutes: z.number().int().min(1).max(1440).optional(),
|
||||
timeoutSeconds: z.number().int().min(1).max(60).optional(),
|
||||
failuresBeforeAlert: z.number().int().min(1).max(20).optional(),
|
||||
warnDays: z.number().int().min(1).max(90).optional(),
|
||||
serverId: z.number().int().positive().nullable().optional(),
|
||||
enabled: z.boolean().optional(),
|
||||
})
|
||||
.superRefine((v, ctx) => {
|
||||
if (v.kind === "http") {
|
||||
let ok = false;
|
||||
try {
|
||||
const u = new URL(v.target);
|
||||
ok = u.protocol === "http:" || u.protocol === "https:";
|
||||
} catch {
|
||||
ok = false;
|
||||
}
|
||||
if (!ok) ctx.addIssue({ code: "custom", path: ["target"], message: "A web check needs an address that starts with http:// or https://." });
|
||||
if (v.expectStatus && !isStatusSpec(v.expectStatus)) ctx.addIssue({ code: "custom", path: ["expectStatus"], message: "Use status codes like 200, or ranges like 200-299, separated by commas." });
|
||||
} else {
|
||||
if (v.port == null) ctx.addIssue({ code: "custom", path: ["port"], message: v.kind === "tcp" ? "A port check needs a port." : "A certificate check needs a port (usually 443)." });
|
||||
if (/[/\s]/.test(v.target)) ctx.addIssue({ code: "custom", path: ["target"], message: "Give just the host name or address, without http:// or a path." });
|
||||
}
|
||||
});
|
||||
|
||||
/** A blank text field is no value. */
|
||||
const blank = (v: string | null | undefined) => (v === undefined || v === null || v.trim() === "" ? null : v.trim());
|
||||
|
||||
// Everyone signed in can see how things are doing, like the pages the same information appears on.
|
||||
endpointsRouter.get("/", asyncHandler(async (_req, res) => {
|
||||
const rows = await db.select().from(endpointChecks).orderBy(endpointChecks.name);
|
||||
const stats = await loadStats(rows.map((r) => r.id));
|
||||
const serverNames = new Map((await db.select({ id: servers.id, name: servers.name }).from(servers)).map((s) => [s.id, s.name]));
|
||||
res.json({
|
||||
checks: rows.map((r) => ({ ...r, server: r.serverId !== null && serverNames.has(r.serverId) ? { id: r.serverId, name: serverNames.get(r.serverId)! } : null, stats: stats.get(r.id) })),
|
||||
servers: [...serverNames].map(([id, name]) => ({ id, name })).sort((a, b) => a.name.localeCompare(b.name)),
|
||||
});
|
||||
}));
|
||||
|
||||
function valuesOf(d: z.infer<typeof checkSchema>) {
|
||||
return {
|
||||
name: d.name,
|
||||
kind: d.kind,
|
||||
target: d.target,
|
||||
port: d.kind === "http" ? null : (d.port ?? null),
|
||||
expectStatus: d.kind === "http" ? blank(d.expectStatus) : null,
|
||||
keyword: d.kind === "http" ? blank(d.keyword) : null,
|
||||
insecure: d.insecure ?? false,
|
||||
intervalMinutes: d.intervalMinutes ?? 5,
|
||||
timeoutSeconds: d.timeoutSeconds ?? 10,
|
||||
failuresBeforeAlert: d.failuresBeforeAlert ?? 2,
|
||||
warnDays: d.warnDays ?? 14,
|
||||
serverId: d.serverId ?? null,
|
||||
enabled: d.enabled ?? true,
|
||||
};
|
||||
}
|
||||
|
||||
async function serverExists(id: number | null | undefined) {
|
||||
return id == null || (await db.select({ id: servers.id }).from(servers).where(eq(servers.id, id))).length > 0;
|
||||
}
|
||||
|
||||
// Asking things on the network is an administrator's to set up.
|
||||
endpointsRouter.post("/", requireRole("admin"), asyncHandler(async (req, res) => {
|
||||
const parsed = checkSchema.safeParse(req.body);
|
||||
if (!parsed.success) return res.status(400).json({ error: "invalid_body", message: parsed.error.issues[0]?.message ?? "That isn't a check.", details: parsed.error.flatten() });
|
||||
if ((await db.select({ id: endpointChecks.id }).from(endpointChecks)).length >= MAX_CHECKS) return res.status(409).json({ error: "too_many", message: `At most ${MAX_CHECKS} checks.` });
|
||||
if (!(await serverExists(parsed.data.serverId))) return res.status(400).json({ error: "unknown_server", message: "That server doesn't exist." });
|
||||
const [row] = await db.insert(endpointChecks).values(valuesOf(parsed.data)).returning();
|
||||
await recordAudit({ actor: req.currentUser!, category: "endpoint", action: "create", targetType: "endpoint", targetId: row.id, detail: { name: row.name, kind: row.kind, target: row.target } });
|
||||
res.status(201).json({ check: row });
|
||||
}));
|
||||
|
||||
endpointsRouter.put("/:id", requireRole("admin"), asyncHandler(async (req, res) => {
|
||||
const id = /^\d+$/.test(req.params.id) ? Number(req.params.id) : NaN;
|
||||
const parsed = checkSchema.safeParse(req.body);
|
||||
if (!Number.isInteger(id) || !parsed.success) return res.status(400).json({ error: "invalid_body", message: parsed.success ? "That isn't a check." : (parsed.error.issues[0]?.message ?? "That isn't a check."), details: parsed.success ? undefined : parsed.error.flatten() });
|
||||
const [before] = await db.select().from(endpointChecks).where(eq(endpointChecks.id, id)).limit(1);
|
||||
if (!before) return res.status(404).json({ error: "not_found" });
|
||||
if (!(await serverExists(parsed.data.serverId))) return res.status(400).json({ error: "unknown_server", message: "That server doesn't exist." });
|
||||
const v = valuesOf(parsed.data);
|
||||
// What is asked changed, so what was last found out about it no longer stands.
|
||||
const reset = v.kind !== before.kind || v.target !== before.target || v.port !== before.port ? { lastCheckedAt: null, lastOk: null, lastMessage: null, lastResponseMs: null, certExpiresAt: null, consecutiveFailures: 0 } : {};
|
||||
const [row] = await db.update(endpointChecks).set({ ...v, ...reset }).where(eq(endpointChecks.id, id)).returning();
|
||||
await recordAudit({ actor: req.currentUser!, category: "endpoint", action: "update", targetType: "endpoint", targetId: id, detail: { name: row.name, kind: row.kind, target: row.target } });
|
||||
invalidateAlertsCache();
|
||||
res.json({ check: row });
|
||||
}));
|
||||
|
||||
endpointsRouter.delete("/:id", requireRole("admin"), asyncHandler(async (req, res) => {
|
||||
const id = /^\d+$/.test(req.params.id) ? Number(req.params.id) : NaN;
|
||||
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
|
||||
const [before] = await db.select().from(endpointChecks).where(eq(endpointChecks.id, id)).limit(1);
|
||||
if (!before) return res.status(404).json({ error: "not_found" });
|
||||
await db.delete(endpointChecks).where(eq(endpointChecks.id, id));
|
||||
await recordAudit({ actor: req.currentUser!, category: "endpoint", action: "delete", targetType: "endpoint", targetId: id, detail: { name: before.name } });
|
||||
invalidateAlertsCache();
|
||||
res.status(204).end();
|
||||
}));
|
||||
|
||||
// Ask now, instead of at the next time it is due.
|
||||
endpointsRouter.post("/:id/check", requireRole("operator"), asyncHandler(async (req, res) => {
|
||||
const id = /^\d+$/.test(req.params.id) ? Number(req.params.id) : NaN;
|
||||
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
|
||||
const [row] = await db.select().from(endpointChecks).where(eq(endpointChecks.id, id)).limit(1);
|
||||
if (!row) return res.status(404).json({ error: "not_found" });
|
||||
const result = await runCheck(row);
|
||||
invalidateAlertsCache();
|
||||
res.json({ result });
|
||||
}));
|
||||
@@ -143,7 +143,7 @@ const updateSchema = z.object({
|
||||
.partial()
|
||||
.optional(),
|
||||
healthChecks: z
|
||||
.object({ serverOfflineMinutes: z.number().int().min(15).max(10080), diskUsagePercent: z.number().int().min(50).max(99), domainWarnDays: z.number().int().min(1).max(365), diskForecastDays: z.number().int().min(0).max(90), backupStaleHours: z.number().int().min(0).max(2160) })
|
||||
.object({ serverOfflineMinutes: z.number().int().min(15).max(10080), diskUsagePercent: z.number().int().min(50).max(99), domainWarnDays: z.number().int().min(1).max(365), diskForecastDays: z.number().int().min(0).max(90), backupStaleHours: z.number().int().min(0).max(2160), restoreTestDays: z.number().int().min(0).max(730) })
|
||||
.partial()
|
||||
.optional(),
|
||||
quietHours: z
|
||||
|
||||
@@ -11,7 +11,7 @@
|
||||
*/
|
||||
import { eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { integrations, secrets, servers } from "../db/schema.js";
|
||||
import { endpointChecks, integrations, secrets, servers } from "../db/schema.js";
|
||||
import { disksFillingSoon, getServerMetrics } from "./serverMetrics.js";
|
||||
import { diskLimit, effectiveThresholds, parseThresholds } from "./serverThresholds.js";
|
||||
import { getBackupCoverage, staleBackups } from "./backupCoverage.js";
|
||||
@@ -34,6 +34,8 @@ import { getFailingSources } from "./integrationHealthMonitor.js";
|
||||
import { getSettings } from "./settingsStore.js";
|
||||
import { sourceLabel } from "./notify.js";
|
||||
import { alertIdOfNode, dependencyCover, integrationTypeCause, takenByType } from "./dependencies.js";
|
||||
import { endpointConditions } from "./endpointChecks.js";
|
||||
import { latestTests, restoreAlerts } from "./restoreTests.js";
|
||||
import type { Alert, AlertCategory, AlertSeverity, SourceFailure } from "./alertTypes.js";
|
||||
|
||||
export interface AlertsReport {
|
||||
@@ -62,6 +64,7 @@ const SOURCES: Record<string, { label: string; link: string }> = {
|
||||
osticket: { label: "osTicket", link: "/osticket" },
|
||||
secrets: { label: "Secrets", link: "/secrets" },
|
||||
domains: { label: "Domains", link: "/domains" },
|
||||
endpoint: { label: "Endpoint", link: "/endpoints" },
|
||||
};
|
||||
|
||||
const SOURCE_TIMEOUT_MS = 20_000;
|
||||
@@ -268,6 +271,37 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
target: s.guest.server ? targetOf("server", s.guest.server.id) : integrationTarget(s.guest.integrationId),
|
||||
});
|
||||
}
|
||||
// A restore that failed, or hasn't been tried in too long (the limit is in Settings; a failure is always raised).
|
||||
const { healthChecks } = await getSettings();
|
||||
for (const r of restoreAlerts(report, await latestTests(), healthChecks.restoreTestDays, now)) {
|
||||
const g = r.guest;
|
||||
push({
|
||||
category: "backup",
|
||||
severity: r.severity,
|
||||
type: g ? "proxmox" : "homelab",
|
||||
message: r.message,
|
||||
key: r.key,
|
||||
link: "/backups",
|
||||
silenced: g ? subjects.has(`integration:${g.integrationId}`) || (g.server !== null && subjects.has(`server:${g.server.id}`)) : false,
|
||||
target: g ? (g.server ? targetOf("server", g.server.id) : integrationTarget(g.integrationId)) : null,
|
||||
});
|
||||
}
|
||||
}),
|
||||
|
||||
// What the endpoint checks have found: an address, port or certificate that has stopped being fine.
|
||||
check("Endpoint checks", async () => {
|
||||
for (const c of endpointConditions(await db.select().from(endpointChecks), new Date())) {
|
||||
push({
|
||||
category: c.category,
|
||||
severity: c.severity,
|
||||
type: "endpoint",
|
||||
message: c.message,
|
||||
key: c.key,
|
||||
link: "/endpoints",
|
||||
silenced: c.serverId !== null && subjects.has(`server:${c.serverId}`),
|
||||
target: c.serverId !== null ? targetOf("server", c.serverId) : null,
|
||||
});
|
||||
}
|
||||
}),
|
||||
|
||||
// Homelab Manager's own backups: if they are switched on, there should be a recent one. This is what turns a backup that has quietly
|
||||
|
||||
@@ -0,0 +1,307 @@
|
||||
/**
|
||||
* The change log: one timeline of what has changed in the homelab and in this app — agents updated, settings changed, maintenance started and
|
||||
* ended, dependencies edited, hardware added, endpoints set up, alerts raised and cleared — each in a sentence, newest first, and filterable
|
||||
* by what kind of thing it was, by server, by time and by a word. It is made from the audit log and the incident record, so it needs nothing
|
||||
* of its own; the Audit Log page remains the raw table behind it.
|
||||
*/
|
||||
import { and, desc, gte, or, lte } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { alertIncidents, auditLog, endpointChecks, integrations, servers } from "../db/schema.js";
|
||||
import { auditTimeToIso, describeAudit, type ActivityTone } from "./serverActivity.js";
|
||||
|
||||
export const CHANGE_KINDS = ["servers", "maintenance", "alerts", "settings", "integrations", "dependencies", "inventory", "backups", "endpoints", "other"] as const;
|
||||
export type ChangeKind = (typeof CHANGE_KINDS)[number];
|
||||
|
||||
export const KIND_LABELS: Record<ChangeKind, string> = {
|
||||
servers: "Servers and agents",
|
||||
maintenance: "Maintenance",
|
||||
alerts: "Alerts and incidents",
|
||||
settings: "Settings and access",
|
||||
integrations: "Integrations, DNS and secrets",
|
||||
dependencies: "Dependencies",
|
||||
inventory: "Inventory",
|
||||
backups: "Backups and restore tests",
|
||||
endpoints: "Endpoint checks",
|
||||
other: "Other",
|
||||
};
|
||||
|
||||
const KIND_OF_CATEGORY: Record<string, ChangeKind> = {
|
||||
server: "servers",
|
||||
task: "servers",
|
||||
update_run: "servers",
|
||||
tag: "servers",
|
||||
maintenance: "maintenance",
|
||||
alert: "alerts",
|
||||
incident: "alerts",
|
||||
alert_ticket: "alerts",
|
||||
settings: "settings",
|
||||
user: "settings",
|
||||
push: "settings",
|
||||
share_link: "settings",
|
||||
metrics_token: "settings",
|
||||
webhook: "settings",
|
||||
integration: "integrations",
|
||||
dns: "integrations",
|
||||
secret: "integrations",
|
||||
domain: "integrations",
|
||||
ipam: "integrations",
|
||||
network: "integrations",
|
||||
consistency: "integrations",
|
||||
dependency: "dependencies",
|
||||
inventory: "inventory",
|
||||
runbook: "other",
|
||||
db_backup: "backups",
|
||||
restore_test: "backups",
|
||||
endpoint: "endpoints",
|
||||
};
|
||||
|
||||
/** Sign-ins and a person's own data export are not changes to the homelab; the diagnostic log has its own page. */
|
||||
const LEFT_OUT = new Set(["session", "privacy", "diag_log"]);
|
||||
|
||||
export interface ChangeEntry {
|
||||
/** Stable, so a row can be keyed on it. */
|
||||
id: string;
|
||||
/** ISO. */
|
||||
at: string;
|
||||
kind: ChangeKind;
|
||||
tone: ActivityTone;
|
||||
title: string;
|
||||
detail: string | null;
|
||||
/** A person's name, or "system" for the app's own checks; null when nobody did it. */
|
||||
actor: string | null;
|
||||
/** What it is about, and where to read more. */
|
||||
target: { type: string; id: string | null; name: string | null } | null;
|
||||
link: string | null;
|
||||
/** The server it concerns, when it concerns one (so the log can be narrowed to a server). */
|
||||
serverId: number | null;
|
||||
}
|
||||
|
||||
const humanize = (s: string) => {
|
||||
const t = s.replace(/[_-]+/g, " ");
|
||||
return t.charAt(0).toUpperCase() + t.slice(1);
|
||||
};
|
||||
|
||||
function parseDetail(detail: string | null): Record<string, unknown> {
|
||||
if (!detail) return {};
|
||||
try {
|
||||
const v = JSON.parse(detail);
|
||||
return v && typeof v === "object" && !Array.isArray(v) ? (v as Record<string, unknown>) : {};
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
const str = (v: unknown) => (typeof v === "string" && v ? v : null);
|
||||
const MAX_SUMMARY = 220;
|
||||
|
||||
/** "notifications: healthAlerts true → false" for a settings update; nothing for what is only a name. */
|
||||
function settingsSummary(detail: Record<string, unknown>): string | null {
|
||||
const changes = detail.changes;
|
||||
if (!changes || typeof changes !== "object") return null;
|
||||
const parts: string[] = [];
|
||||
for (const [section, fields] of Object.entries(changes as Record<string, Record<string, unknown>>)) {
|
||||
for (const [field, change] of Object.entries(fields ?? {})) {
|
||||
if (change && typeof change === "object" && "to" in (change as object)) {
|
||||
const c = change as { from: unknown; to: unknown };
|
||||
parts.push(`${section}.${field}: ${JSON.stringify(c.from)} → ${JSON.stringify(c.to)}`);
|
||||
} else parts.push(`${section}.${field} changed`);
|
||||
}
|
||||
}
|
||||
const text = parts.join(" · ");
|
||||
return text.length > MAX_SUMMARY ? `${text.slice(0, MAX_SUMMARY)}…` : text || null;
|
||||
}
|
||||
|
||||
/** What is left of the detail once the name and what is already in the sentence are taken out. */
|
||||
function genericSummary(detail: Record<string, unknown>, skip: string[] = []): string | null {
|
||||
const parts = Object.entries(detail)
|
||||
.filter(([k, v]) => !["name", "message", ...skip].includes(k) && v !== null && v !== undefined && v !== "" && !(Array.isArray(v) && v.length === 0))
|
||||
.map(([k, v]) => `${k}: ${typeof v === "object" ? JSON.stringify(v) : String(v)}`);
|
||||
const text = parts.join(" · ");
|
||||
return text.length > MAX_SUMMARY ? `${text.slice(0, MAX_SUMMARY)}…` : text || null;
|
||||
}
|
||||
|
||||
/** One sentence for an audit entry. Servers, alerts and update runs have their own wording (the same as a server's Activity card); the rest is made here. */
|
||||
export function describeChange(category: string, action: string, detail: Record<string, unknown>): { title: string; tone: ActivityTone; detail: string | null } {
|
||||
const name = str(detail.name);
|
||||
const minutes = typeof detail.minutes === "number" ? detail.minutes : null;
|
||||
const forHow = minutes !== null ? (minutes >= 120 ? `${Math.round(minutes / 60)} h` : `${minutes} min`) : null;
|
||||
switch (`${category}/${action}`) {
|
||||
case "maintenance/start":
|
||||
return { title: `Maintenance started${name ? ` on ${name}` : ""}`, tone: "info", detail: [forHow && `for ${forHow}`, str(detail.reason)].filter(Boolean).join(" · ") || null };
|
||||
case "maintenance/extend":
|
||||
return { title: `Maintenance extended${name ? ` on ${name}` : ""}`, tone: "info", detail: [forHow && `for ${forHow} from now`, str(detail.reason)].filter(Boolean).join(" · ") || null };
|
||||
case "maintenance/end":
|
||||
return { title: `Maintenance ended${name ? ` on ${name}` : ""}`, tone: "info", detail: null };
|
||||
case "maintenance/start_bulk":
|
||||
return { title: "Maintenance started on many at once", tone: "info", detail: genericSummary(detail) };
|
||||
case "dependency/create":
|
||||
return { title: `${str(detail.from) ?? "Something"} now needs ${str(detail.to) ?? "something"}`, tone: "neutral", detail: null };
|
||||
case "dependency/delete":
|
||||
return { title: `${str(detail.from) ?? "Something"} no longer needs ${str(detail.to) ?? "something"}`, tone: "neutral", detail: null };
|
||||
case "inventory/create":
|
||||
return { title: `Hardware added${name ? `: ${name}` : ""}`, tone: "neutral", detail: genericSummary(detail, ["kind"]) };
|
||||
case "inventory/update":
|
||||
return { title: `Hardware changed${name ? `: ${name}` : ""}`, tone: "neutral", detail: genericSummary(detail) };
|
||||
case "inventory/delete":
|
||||
return { title: `Hardware removed${name ? `: ${name}` : ""}`, tone: "neutral", detail: null };
|
||||
case "inventory/refresh_from_agent":
|
||||
return { title: `Hardware details taken from the agent${name ? `: ${name}` : ""}`, tone: "neutral", detail: null };
|
||||
case "inventory/update_config":
|
||||
return { title: "Price of power changed", tone: "neutral", detail: null };
|
||||
case "endpoint/create":
|
||||
return { title: `Endpoint check added${name ? `: ${name}` : ""}`, tone: "neutral", detail: [str(detail.kind), str(detail.target)].filter(Boolean).join(" · ") || null };
|
||||
case "endpoint/update":
|
||||
return { title: `Endpoint check changed${name ? `: ${name}` : ""}`, tone: "neutral", detail: [str(detail.kind), str(detail.target)].filter(Boolean).join(" · ") || null };
|
||||
case "endpoint/delete":
|
||||
return { title: `Endpoint check removed${name ? `: ${name}` : ""}`, tone: "neutral", detail: null };
|
||||
case "restore_test/record": {
|
||||
const result = str(detail.result);
|
||||
const tone: ActivityTone = result === "failed" ? "bad" : result === "not_needed" ? "neutral" : "good";
|
||||
const said = result === "ok" ? "passed" : result === "partial" ? "partly worked" : result === "failed" ? "failed" : "marked as not needed";
|
||||
return { title: `Restore test of ${name ?? "something"} ${said}`, tone, detail: [str(detail.testedAt), str(detail.note)].filter(Boolean).join(" · ") || null };
|
||||
}
|
||||
case "restore_test/delete":
|
||||
return { title: `A restore test of ${name ?? "something"} was removed`, tone: "neutral", detail: str(detail.testedAt) };
|
||||
case "settings/update": {
|
||||
const sections = Array.isArray(detail.sections) ? (detail.sections as string[]).join(", ") : null;
|
||||
return { title: `Settings changed${sections ? `: ${sections}` : ""}`, tone: "neutral", detail: settingsSummary(detail) };
|
||||
}
|
||||
case "push/subscribe":
|
||||
return { title: `A browser was added for push notifications${str(detail.label) ? `: ${str(detail.label)}` : ""}`, tone: "neutral", detail: null };
|
||||
case "push/unsubscribe":
|
||||
return { title: `A browser was removed from push notifications${str(detail.label) ? `: ${str(detail.label)}` : ""}`, tone: "neutral", detail: null };
|
||||
case "alert/escalate":
|
||||
return { title: "An alert nobody had acknowledged was escalated", tone: "bad", detail: [str(detail.message), typeof detail.afterMinutes === "number" ? `step after ${detail.afterMinutes} min` : null].filter(Boolean).join(" · ") || null };
|
||||
}
|
||||
if (category === "server" || category === "alert" || category === "update_run") {
|
||||
const d = describeAudit(category, action, detail);
|
||||
return { title: d.title, tone: d.tone, detail: d.detail };
|
||||
}
|
||||
return { title: `${humanize(category)}: ${action.replace(/_/g, " ")}${name ? ` — ${name}` : ""}`, tone: "neutral", detail: genericSummary(detail) };
|
||||
}
|
||||
|
||||
export interface ChangeQuery {
|
||||
/** How far back, in days (default 7). */
|
||||
days?: number;
|
||||
kind?: ChangeKind;
|
||||
serverId?: number;
|
||||
/** A word or two that must all appear somewhere in the entry. */
|
||||
q?: string;
|
||||
/** Only entries older than this (ISO), for the next page. */
|
||||
before?: string;
|
||||
limit?: number;
|
||||
now?: Date;
|
||||
}
|
||||
|
||||
const sqlTime = (d: Date) => d.toISOString().slice(0, 19).replace("T", " ");
|
||||
|
||||
function serverIdOf(targetType: string | null, targetId: string | null, detail: Record<string, unknown>): number | null {
|
||||
if (targetType === "server" && targetId && /^\d+$/.test(targetId)) return Number(targetId);
|
||||
// Some entries about something else name the servers they touch by key ("server:3").
|
||||
for (const k of ["fromKey", "toKey", "key"]) {
|
||||
const m = /^server:(\d+)$/.exec(typeof detail[k] === "string" ? (detail[k] as string) : "");
|
||||
if (m) return Number(m[1]);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
const touchesServer = (e: ChangeEntry, id: number, detail: Record<string, unknown>) =>
|
||||
e.serverId === id || ["fromKey", "toKey"].some((k) => detail[k] === `server:${id}`);
|
||||
|
||||
/** The log, newest first, narrowed as asked. `more` says whether older entries are left beyond the ones returned. */
|
||||
export async function getChangeLog(query: ChangeQuery = {}): Promise<{ entries: ChangeEntry[]; more: boolean }> {
|
||||
const now = query.now ?? new Date();
|
||||
const days = Math.min(Math.max(query.days ?? 7, 1), 400);
|
||||
const limit = Math.min(Math.max(query.limit ?? 200, 1), 500);
|
||||
const since = new Date(now.getTime() - days * 86_400_000);
|
||||
const until = query.before ? new Date(query.before) : null;
|
||||
|
||||
const [serverRows, integrationRows, endpointRows] = await Promise.all([
|
||||
db.select({ id: servers.id, name: servers.name }).from(servers),
|
||||
db.select({ id: integrations.id, name: integrations.name }).from(integrations),
|
||||
db.select({ id: endpointChecks.id, name: endpointChecks.name }).from(endpointChecks),
|
||||
]);
|
||||
const serverName = new Map(serverRows.map((s) => [s.id, s.name]));
|
||||
const integrationName = new Map(integrationRows.map((i) => [i.id, i.name]));
|
||||
const endpointName = new Map(endpointRows.map((e) => [e.id, e.name]));
|
||||
|
||||
const entries: { entry: ChangeEntry; detail: Record<string, unknown> }[] = [];
|
||||
|
||||
const audits = await db
|
||||
.select()
|
||||
.from(auditLog)
|
||||
.where(until ? and(gte(auditLog.createdAt, sqlTime(since)), lte(auditLog.createdAt, sqlTime(until))) : gte(auditLog.createdAt, sqlTime(since)))
|
||||
.orderBy(desc(auditLog.createdAt), desc(auditLog.id))
|
||||
.limit(5000);
|
||||
for (const a of audits) {
|
||||
if (LEFT_OUT.has(a.category)) continue;
|
||||
const detail = parseDetail(a.detail);
|
||||
const d = describeChange(a.category, a.action, detail);
|
||||
const serverId = serverIdOf(a.targetType, a.targetId, detail);
|
||||
const idNum = a.targetId && /^\d+$/.test(a.targetId) ? Number(a.targetId) : null;
|
||||
let targetName = str(detail.name);
|
||||
let link: string | null = null;
|
||||
if (a.targetType === "server" && idNum !== null) {
|
||||
targetName = serverName.get(idNum) ?? targetName;
|
||||
link = serverName.has(idNum) ? `/servers/${idNum}` : null;
|
||||
} else if (a.targetType === "integration" && idNum !== null) {
|
||||
targetName = integrationName.get(idNum) ?? targetName;
|
||||
link = "/integrations";
|
||||
} else if (a.targetType === "endpoint" && idNum !== null) {
|
||||
targetName = endpointName.get(idNum) ?? targetName;
|
||||
link = "/endpoints";
|
||||
} else if (a.targetType === "asset" && idNum !== null) link = `/inventory?asset=${idNum}`;
|
||||
else if (a.targetType === "restore_test") link = "/backups";
|
||||
else if (a.targetType === "dependency") link = "/dependencies";
|
||||
else if (a.targetType === "settings") link = "/settings/notifications";
|
||||
entries.push({
|
||||
detail,
|
||||
entry: {
|
||||
id: `audit-${a.id}`,
|
||||
at: auditTimeToIso(a.createdAt),
|
||||
kind: KIND_OF_CATEGORY[a.category] ?? "other",
|
||||
tone: d.tone,
|
||||
title: d.title,
|
||||
detail: d.detail,
|
||||
actor: a.actorLabel,
|
||||
target: a.targetType ? { type: a.targetType, id: a.targetId, name: targetName } : null,
|
||||
link,
|
||||
serverId,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
// Alerts raised and cleared, from the incident record.
|
||||
const incidents = await db
|
||||
.select()
|
||||
.from(alertIncidents)
|
||||
.where(or(gte(alertIncidents.openedAt, since.toISOString()), gte(alertIncidents.resolvedAt, since.toISOString())))
|
||||
.orderBy(desc(alertIncidents.openedAt))
|
||||
.limit(1000);
|
||||
for (const i of incidents) {
|
||||
const serverId = i.targetType === "server" ? i.targetId : null;
|
||||
const base = { kind: "alerts" as const, actor: null, target: i.targetType ? { type: i.targetType, id: i.targetId === null ? null : String(i.targetId), name: i.targetName } : null, link: `/incidents/${i.id}`, serverId };
|
||||
if (i.openedAt >= since.toISOString() && (!until || i.openedAt <= until.toISOString())) {
|
||||
entries.push({ detail: {}, entry: { ...base, id: `incident-open-${i.id}`, at: i.openedAt, tone: i.peakSeverity === "critical" ? "bad" : "info", title: `Alert raised: ${i.message}`, detail: i.causedBy ? `explained by ${i.causedBy}` : null } });
|
||||
}
|
||||
if (i.resolvedAt && i.resolvedAt >= since.toISOString() && (!until || i.resolvedAt <= until.toISOString())) {
|
||||
entries.push({ detail: {}, entry: { ...base, id: `incident-close-${i.id}`, at: i.resolvedAt, tone: "good", title: `Alert cleared: ${i.message}`, detail: null } });
|
||||
}
|
||||
}
|
||||
|
||||
const terms = (query.q ?? "").toLowerCase().split(/\s+/).filter(Boolean);
|
||||
const kept = entries
|
||||
.filter(({ entry, detail }) => {
|
||||
if (query.kind && entry.kind !== query.kind) return false;
|
||||
if (query.serverId !== undefined && !touchesServer(entry, query.serverId, detail)) return false;
|
||||
if (until && entry.at >= until.toISOString()) return false; // strictly older than the cursor
|
||||
if (terms.length > 0) {
|
||||
const hay = `${entry.title} ${entry.detail ?? ""} ${entry.actor ?? ""} ${entry.target?.name ?? ""} ${KIND_LABELS[entry.kind]}`.toLowerCase();
|
||||
if (!terms.every((t) => hay.includes(t))) return false;
|
||||
}
|
||||
return true;
|
||||
})
|
||||
.map((e) => e.entry)
|
||||
.sort((a, b) => b.at.localeCompare(a.at) || b.id.localeCompare(a.id));
|
||||
return { entries: kept.slice(0, limit), more: kept.length > limit };
|
||||
}
|
||||
@@ -0,0 +1,350 @@
|
||||
/**
|
||||
* Endpoint checks: the app asks a web address, a TCP port or a TLS certificate whether it is fine, every few minutes, and remembers how it
|
||||
* went — answer time, uptime, when the certificate runs out. A check that keeps failing becomes an alert (and a notification), and so does a
|
||||
* certificate that is about to expire. This is for the simple "is it up?" that doesn't need a monitoring server of its own.
|
||||
*
|
||||
* What is asked is typed in by an administrator, so it can't be just anything: loopback, link-local (cloud metadata) and similar addresses
|
||||
* are refused, and the connection goes to the one address that passed the check rather than to the name again (utils/networkGuard.ts).
|
||||
* LAN and public hosts are fine; that is what these are for.
|
||||
*/
|
||||
import http from "node:http";
|
||||
import https from "node:https";
|
||||
import net from "node:net";
|
||||
import tls from "node:tls";
|
||||
import type { TLSSocket } from "node:tls";
|
||||
import { and, asc, eq, gte, inArray, lt } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { endpointChecks, endpointResults } from "../db/schema.js";
|
||||
import { resolveTarget } from "../utils/networkGuard.js";
|
||||
|
||||
export const CHECK_KINDS = ["http", "tcp", "tls"] as const;
|
||||
export type CheckKind = (typeof CHECK_KINDS)[number];
|
||||
export type CheckRow = typeof endpointChecks.$inferSelect;
|
||||
|
||||
export interface ProbeResult {
|
||||
ok: boolean;
|
||||
/** How long it took to answer, when it did. */
|
||||
ms: number | null;
|
||||
message: string;
|
||||
/** ISO — when the certificate it showed runs out (https and tls checks). */
|
||||
certExpiresAt?: string | null;
|
||||
}
|
||||
|
||||
export const KEEP_RESULT_DAYS = 14;
|
||||
const MAX_BODY_BYTES = 64 * 1024;
|
||||
const MAX_REDIRECTS = 5;
|
||||
|
||||
/** "200-299, 301" → a test for a status code. Anything unreadable falls back to the usual 200–399. */
|
||||
export function statusAccepter(spec: string | null | undefined): (code: number) => boolean {
|
||||
const ranges: [number, number][] = [];
|
||||
for (const part of (spec ?? "").split(",")) {
|
||||
const m = /^\s*(\d{3})(?:\s*-\s*(\d{3}))?\s*$/.exec(part);
|
||||
if (m) ranges.push([Number(m[1]), Number(m[2] ?? m[1])]);
|
||||
}
|
||||
if (ranges.length === 0) ranges.push([200, 399]);
|
||||
return (code) => ranges.some(([lo, hi]) => code >= lo && code <= hi);
|
||||
}
|
||||
|
||||
/** Whether a status spec is one statusAccepter would read (for refusing a typo when it is saved). */
|
||||
export const isStatusSpec = (spec: string) => spec.split(",").every((p) => /^\s*\d{3}(\s*-\s*\d{3})?\s*$/.test(p));
|
||||
|
||||
const certExpiry = (socket: unknown): string | null => {
|
||||
try {
|
||||
const cert = (socket as TLSSocket).getPeerCertificate?.();
|
||||
if (!cert || !cert.valid_to) return null;
|
||||
const t = new Date(cert.valid_to);
|
||||
return Number.isNaN(t.getTime()) ? null : t.toISOString();
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
};
|
||||
|
||||
/** What to tell a person when the certificate can't be trusted: the usual reasons, said plainly. */
|
||||
function trustProblem(code: string | undefined, text: string): string {
|
||||
const known: Record<string, string> = {
|
||||
DEPTH_ZERO_SELF_SIGNED_CERT: "it is self-signed",
|
||||
SELF_SIGNED_CERT_IN_CHAIN: "it is signed by a certificate authority this server doesn't know",
|
||||
UNABLE_TO_VERIFY_LEAF_SIGNATURE: "it is signed by a certificate authority this server doesn't know",
|
||||
UNABLE_TO_GET_ISSUER_CERT_LOCALLY: "it is signed by a certificate authority this server doesn't know",
|
||||
CERT_HAS_EXPIRED: "it has expired",
|
||||
ERR_TLS_CERT_ALTNAME_INVALID: "it isn't for this name",
|
||||
};
|
||||
return `The certificate can't be trusted: ${(code && known[code]) || text}. If that is expected for this service, tick "Accept a certificate that isn't trusted".`;
|
||||
}
|
||||
|
||||
interface OnceResult {
|
||||
status: number;
|
||||
location: string | null;
|
||||
body: string;
|
||||
cert: string | null;
|
||||
}
|
||||
|
||||
function requestOnce(url: URL, address: string, insecure: boolean, timeoutMs: number, wantBody: boolean): Promise<OnceResult> {
|
||||
const family = net.isIP(address) === 6 ? 6 : 4;
|
||||
return new Promise((resolve, reject) => {
|
||||
const isHttps = url.protocol === "https:";
|
||||
const bare = url.hostname.replace(/^\[|\]$/g, "");
|
||||
const req = (isHttps ? https : http).request(
|
||||
{
|
||||
host: bare,
|
||||
port: url.port ? Number(url.port) : isHttps ? 443 : 80,
|
||||
method: "GET",
|
||||
path: `${url.pathname}${url.search}`,
|
||||
headers: { "user-agent": "HomelabManager-endpoint-check", accept: "*/*", host: url.host },
|
||||
// Connect to the address that passed the check, not to the name again.
|
||||
lookup: ((_host: string, options: { all?: boolean }, cb: (...args: unknown[]) => void) => (options?.all ? cb(null, [{ address, family }]) : cb(null, address, family))) as never,
|
||||
timeout: timeoutMs,
|
||||
...(isHttps ? { rejectUnauthorized: !insecure, servername: net.isIP(bare) === 0 ? bare : undefined } : {}),
|
||||
},
|
||||
(res) => {
|
||||
const cert = isHttps ? certExpiry(res.socket) : null;
|
||||
const chunks: Buffer[] = [];
|
||||
let size = 0;
|
||||
const finish = () => resolve({ status: res.statusCode ?? 0, location: typeof res.headers.location === "string" ? res.headers.location : null, body: Buffer.concat(chunks).toString("utf8"), cert });
|
||||
if (!wantBody) {
|
||||
res.resume();
|
||||
res.on("end", finish);
|
||||
res.on("close", finish);
|
||||
return;
|
||||
}
|
||||
res.on("data", (c: Buffer) => {
|
||||
size += c.length;
|
||||
chunks.push(c);
|
||||
if (size >= MAX_BODY_BYTES) {
|
||||
res.destroy();
|
||||
finish();
|
||||
}
|
||||
});
|
||||
res.on("end", finish);
|
||||
res.on("close", finish);
|
||||
},
|
||||
);
|
||||
req.on("timeout", () => req.destroy(new Error(`no answer within ${Math.round(timeoutMs / 1000)} s`)));
|
||||
req.on("error", reject);
|
||||
req.end();
|
||||
});
|
||||
}
|
||||
|
||||
async function probeHttp(c: CheckRow): Promise<ProbeResult> {
|
||||
const started = performance.now();
|
||||
const timeoutMs = c.timeoutSeconds * 1000;
|
||||
const accepts = statusAccepter(c.expectStatus);
|
||||
let url: URL;
|
||||
try {
|
||||
url = new URL(c.target);
|
||||
if (url.protocol !== "http:" && url.protocol !== "https:") throw new Error("not http");
|
||||
} catch {
|
||||
return { ok: false, ms: null, message: "The address isn't a web address (it should start with http:// or https://)." };
|
||||
}
|
||||
let cert: string | null = null;
|
||||
for (let hops = 0; ; hops++) {
|
||||
const left = timeoutMs - (performance.now() - started);
|
||||
if (left <= 0) return { ok: false, ms: null, message: `no answer within ${c.timeoutSeconds} s` };
|
||||
let once: OnceResult;
|
||||
try {
|
||||
const address = await resolveTarget(url.hostname.replace(/^\[|\]$/g, ""), { allowPrivate: true });
|
||||
once = await requestOnce(url, address, c.insecure, left, !!c.keyword);
|
||||
} catch (err) {
|
||||
const e = err as NodeJS.ErrnoException;
|
||||
const text = err instanceof Error ? err.message : String(err);
|
||||
const trust = url.protocol === "https:" && /^(DEPTH_ZERO|SELF_SIGNED|UNABLE_TO|CERT_|ERR_TLS_CERT)/.test(e.code ?? "");
|
||||
return { ok: false, ms: null, message: trust ? trustProblem(e.code, text) : text, certExpiresAt: cert };
|
||||
}
|
||||
cert = once.cert ?? cert;
|
||||
// Redirects are only followed when there is a keyword to look for in the page they lead to; otherwise a redirect is an answer.
|
||||
if (c.keyword && [301, 302, 303, 307, 308].includes(once.status) && once.location && hops < MAX_REDIRECTS) {
|
||||
url = new URL(once.location, url);
|
||||
continue;
|
||||
}
|
||||
const ms = Math.round(performance.now() - started);
|
||||
if (!accepts(once.status)) return { ok: false, ms, message: `answered ${once.status}`, certExpiresAt: cert };
|
||||
if (c.keyword && !once.body.includes(c.keyword)) return { ok: false, ms, message: `answered ${once.status}, but the page doesn't contain “${c.keyword}”`, certExpiresAt: cert };
|
||||
return { ok: true, ms, message: `answered ${once.status} in ${ms} ms`, certExpiresAt: cert };
|
||||
}
|
||||
}
|
||||
|
||||
async function probeTcp(c: CheckRow): Promise<ProbeResult> {
|
||||
const started = performance.now();
|
||||
let address: string;
|
||||
try {
|
||||
address = await resolveTarget(c.target, { allowPrivate: true });
|
||||
} catch (err) {
|
||||
return { ok: false, ms: null, message: err instanceof Error ? err.message : String(err) };
|
||||
}
|
||||
return new Promise((resolve) => {
|
||||
const socket = net.connect({ host: address, port: c.port ?? 0 });
|
||||
const done = (r: ProbeResult) => {
|
||||
socket.destroy();
|
||||
resolve(r);
|
||||
};
|
||||
socket.setTimeout(c.timeoutSeconds * 1000, () => done({ ok: false, ms: null, message: `no answer within ${c.timeoutSeconds} s` }));
|
||||
socket.on("connect", () => {
|
||||
const ms = Math.round(performance.now() - started);
|
||||
done({ ok: true, ms, message: `port ${c.port} answered in ${ms} ms` });
|
||||
});
|
||||
socket.on("error", (err: NodeJS.ErrnoException) => done({ ok: false, ms: null, message: err.code === "ECONNREFUSED" ? `port ${c.port} refused the connection` : err.message }));
|
||||
});
|
||||
}
|
||||
|
||||
async function probeTls(c: CheckRow): Promise<ProbeResult> {
|
||||
const started = performance.now();
|
||||
let address: string;
|
||||
try {
|
||||
address = await resolveTarget(c.target, { allowPrivate: true });
|
||||
} catch (err) {
|
||||
return { ok: false, ms: null, message: err instanceof Error ? err.message : String(err) };
|
||||
}
|
||||
return new Promise((resolve) => {
|
||||
const socket = tls.connect({ host: address, port: c.port ?? 443, servername: net.isIP(c.target) === 0 ? c.target : undefined, rejectUnauthorized: false }, () => {
|
||||
const ms = Math.round(performance.now() - started);
|
||||
const expires = certExpiry(socket);
|
||||
const authorized = socket.authorized;
|
||||
const reason = socket.authorizationError ? String(socket.authorizationError) : "";
|
||||
socket.end();
|
||||
if (!expires) return resolve({ ok: false, ms, message: "The server showed no certificate." });
|
||||
const days = Math.floor((Date.parse(expires) - Date.now()) / 86_400_000);
|
||||
if (days < 0) return resolve({ ok: false, ms, message: `The certificate expired on ${expires.slice(0, 10)}.`, certExpiresAt: expires });
|
||||
if (!authorized && !c.insecure) return resolve({ ok: false, ms, message: trustProblem(reason, reason || "it didn't verify"), certExpiresAt: expires });
|
||||
resolve({ ok: true, ms, message: `The certificate is good until ${expires.slice(0, 10)} (${days} days).`, certExpiresAt: expires });
|
||||
});
|
||||
socket.setTimeout(c.timeoutSeconds * 1000, () => {
|
||||
socket.destroy();
|
||||
resolve({ ok: false, ms: null, message: `no answer within ${c.timeoutSeconds} s` });
|
||||
});
|
||||
socket.on("error", (err) => {
|
||||
resolve({ ok: false, ms: null, message: /wrong version number|packet length too long/i.test(err.message) ? `${c.target}:${c.port ?? 443} isn't speaking TLS — is that the right port?` : err.message });
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
export function probe(c: CheckRow): Promise<ProbeResult> {
|
||||
return c.kind === "tcp" ? probeTcp(c) : c.kind === "tls" ? probeTls(c) : probeHttp(c);
|
||||
}
|
||||
|
||||
/** Asks once, and writes down how it went: the check's current state, and one more point in its history. */
|
||||
export async function runCheck(c: CheckRow, now: Date = new Date(), probeFn: (c: CheckRow) => Promise<ProbeResult> = probe): Promise<ProbeResult> {
|
||||
let result: ProbeResult;
|
||||
try {
|
||||
result = await probeFn(c);
|
||||
} catch (err) {
|
||||
result = { ok: false, ms: null, message: err instanceof Error ? err.message : String(err) };
|
||||
}
|
||||
const iso = now.toISOString();
|
||||
const flipped = c.lastOk === null || c.lastOk !== result.ok;
|
||||
await db
|
||||
.update(endpointChecks)
|
||||
.set({
|
||||
lastCheckedAt: iso,
|
||||
lastOk: result.ok,
|
||||
lastMessage: result.message.slice(0, 300),
|
||||
lastResponseMs: result.ms,
|
||||
certExpiresAt: result.certExpiresAt === undefined ? c.certExpiresAt : result.certExpiresAt,
|
||||
consecutiveFailures: result.ok ? 0 : c.consecutiveFailures + 1,
|
||||
lastChangeAt: flipped ? iso : c.lastChangeAt,
|
||||
})
|
||||
.where(eq(endpointChecks.id, c.id));
|
||||
await db.insert(endpointResults).values({ checkId: c.id, at: iso, ok: result.ok, ms: result.ms, message: result.message.slice(0, 300) });
|
||||
return result;
|
||||
}
|
||||
|
||||
/** Runs every enabled check that is due, a few at a time. Returns how many it asked. */
|
||||
export async function runDueChecks(now: Date = new Date(), probeFn?: (c: CheckRow) => Promise<ProbeResult>): Promise<number> {
|
||||
const rows = await db.select().from(endpointChecks).where(eq(endpointChecks.enabled, true)).orderBy(asc(endpointChecks.id));
|
||||
const due = rows.filter((c) => !c.lastCheckedAt || now.getTime() - Date.parse(c.lastCheckedAt) >= c.intervalMinutes * 60_000 - 5_000);
|
||||
let next = 0;
|
||||
const worker = async () => {
|
||||
while (next < due.length) {
|
||||
const c = due[next++];
|
||||
await runCheck(c, now, probeFn);
|
||||
}
|
||||
};
|
||||
await Promise.all(Array.from({ length: Math.min(5, due.length) }, worker));
|
||||
return due.length;
|
||||
}
|
||||
|
||||
export async function pruneResults(now: Date = new Date()): Promise<void> {
|
||||
await db.delete(endpointResults).where(lt(endpointResults.at, new Date(now.getTime() - KEEP_RESULT_DAYS * 86_400_000).toISOString()));
|
||||
}
|
||||
|
||||
// ─── What the history says ─────────────────────────────────────────────────
|
||||
|
||||
export interface CheckStats {
|
||||
uptime24h: number | null;
|
||||
uptime7d: number | null;
|
||||
avgMs24h: number | null;
|
||||
/** The last results, oldest first: how the recent checks went. */
|
||||
recent: { at: string; ok: boolean; ms: number | null }[];
|
||||
}
|
||||
|
||||
/** Pure: uptime, average answer time and the recent run of results, from a check's stored results. */
|
||||
export function summariseResults(results: { at: string; ok: boolean; ms: number | null }[], now: Date): CheckStats {
|
||||
const sorted = [...results].sort((a, b) => a.at.localeCompare(b.at));
|
||||
const since = (ms: number) => sorted.filter((r) => now.getTime() - Date.parse(r.at) <= ms);
|
||||
const pct = (list: typeof sorted) => (list.length === 0 ? null : Math.round((list.filter((r) => r.ok).length / list.length) * 1000) / 10);
|
||||
const day = since(86_400_000);
|
||||
const timed = day.filter((r) => r.ok && r.ms !== null);
|
||||
return {
|
||||
uptime24h: pct(day),
|
||||
uptime7d: pct(since(7 * 86_400_000)),
|
||||
avgMs24h: timed.length === 0 ? null : Math.round(timed.reduce((n, r) => n + (r.ms ?? 0), 0) / timed.length),
|
||||
recent: sorted.slice(-30),
|
||||
};
|
||||
}
|
||||
|
||||
export async function loadStats(ids: number[], now: Date = new Date()): Promise<Map<number, CheckStats>> {
|
||||
const out = new Map<number, CheckStats>();
|
||||
if (ids.length === 0) return out;
|
||||
const rows = await db
|
||||
.select({ checkId: endpointResults.checkId, at: endpointResults.at, ok: endpointResults.ok, ms: endpointResults.ms })
|
||||
.from(endpointResults)
|
||||
.where(and(inArray(endpointResults.checkId, ids), gte(endpointResults.at, new Date(now.getTime() - 7 * 86_400_000).toISOString())));
|
||||
const by = new Map<number, { at: string; ok: boolean; ms: number | null }[]>();
|
||||
for (const r of rows) by.set(r.checkId, [...(by.get(r.checkId) ?? []), r]);
|
||||
for (const id of ids) out.set(id, summariseResults(by.get(id) ?? [], now));
|
||||
return out;
|
||||
}
|
||||
|
||||
// ─── What is wrong ─────────────────────────────────────────────────────────
|
||||
|
||||
export interface EndpointCondition {
|
||||
key: string;
|
||||
source: "endpoint";
|
||||
message: string;
|
||||
severity: "critical" | "warning";
|
||||
/** "monitoring" for a check that fails, "expiry" for a certificate running out. */
|
||||
category: "monitoring" | "expiry";
|
||||
checkId: number;
|
||||
serverId: number | null;
|
||||
}
|
||||
|
||||
const daysLeft = (iso: string, now: Date) => Math.floor((Date.parse(iso) - now.getTime()) / 86_400_000);
|
||||
|
||||
/**
|
||||
* The problems the checks stand for. A check must have failed several times in a row before it is one (so a blip isn't), and a certificate is
|
||||
* one from its warning time on — critical in the last three days. The key carries the server a check is linked to, so maintenance on it silences them.
|
||||
*/
|
||||
export function endpointConditions(rows: CheckRow[], now: Date): EndpointCondition[] {
|
||||
const out: EndpointCondition[] = [];
|
||||
for (const c of rows) {
|
||||
if (!c.enabled) continue;
|
||||
const suffix = c.serverId !== null ? `:server:${c.serverId}` : "";
|
||||
const left = c.certExpiresAt ? daysLeft(c.certExpiresAt, now) : null;
|
||||
const certGone = left !== null && left < 0;
|
||||
if (c.lastOk === false && c.consecutiveFailures >= c.failuresBeforeAlert && !(c.kind === "tls" && certGone)) {
|
||||
out.push({ key: `endpoint:${c.id}${suffix}`, source: "endpoint", category: "monitoring", severity: "critical", checkId: c.id, serverId: c.serverId, message: `${c.name} isn't answering — ${c.lastMessage ?? "no answer"}` });
|
||||
}
|
||||
if (c.certExpiresAt && left !== null && left <= c.warnDays) {
|
||||
out.push({
|
||||
key: `endpoint-cert:${c.id}${suffix}`,
|
||||
source: "endpoint",
|
||||
category: "expiry",
|
||||
severity: left <= 3 ? "critical" : "warning",
|
||||
checkId: c.id,
|
||||
serverId: c.serverId,
|
||||
message: `The certificate for ${c.name} ${left < 0 ? `expired ${-left} day${-left === 1 ? "" : "s"} ago` : left === 0 ? "expires today" : `expires in ${left} day${left === 1 ? "" : "s"}`} (${c.certExpiresAt.slice(0, 10)})`,
|
||||
});
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
/**
|
||||
* The checks' own rhythm: every minute, ask the ones that are due, then see what has started or stopped being a problem and say so — a
|
||||
* notification when a check has failed enough times in a row, another when it works again, the same for a certificate running out. It
|
||||
* uses the same bookkeeping as the server and automation checks (each problem announced once, remembered across a restart), so
|
||||
* a maintenance window on the server a check is linked to holds its problems back, and they alert after it if they are still there.
|
||||
*/
|
||||
import { db } from "../db/client.js";
|
||||
import { endpointChecks } from "../db/schema.js";
|
||||
import { diffConditions, type ActiveState, type HealthCondition } from "./healthMonitor.js";
|
||||
import { activeSubjects } from "./maintenance.js";
|
||||
import { endpointConditions, pruneResults, runDueChecks, type CheckRow, type ProbeResult } from "./endpointChecks.js";
|
||||
import { notifyHealthIssues, notifyHealthRecovered } from "./notify.js";
|
||||
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
|
||||
|
||||
const STATE_FLAG = "endpointActiveConditions";
|
||||
|
||||
async function loadState(): Promise<ActiveState> {
|
||||
const raw = await getInternalFlag(STATE_FLAG);
|
||||
if (!raw) return {};
|
||||
try {
|
||||
return JSON.parse(raw);
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
/** One pass: ask what is due, then announce what changed. Returns what it did. */
|
||||
export async function runEndpointPass(now: Date = new Date(), probeFn?: (c: CheckRow) => Promise<ProbeResult>): Promise<{ asked: number; announced: number; recovered: number }> {
|
||||
const asked = await runDueChecks(now, probeFn);
|
||||
const rows = await db.select().from(endpointChecks);
|
||||
const current: HealthCondition[] = endpointConditions(rows, now).map((c) => ({ key: c.key, source: c.source, message: c.message, severity: c.severity }));
|
||||
const { added, resolved, next } = diffConditions(await loadState(), current, new Set(), await activeSubjects(now));
|
||||
await setInternalFlag(STATE_FLAG, JSON.stringify(next));
|
||||
if (added.length > 0) await notifyHealthIssues(added);
|
||||
if (resolved.length > 0) await notifyHealthRecovered(resolved);
|
||||
return { asked, announced: added.length, recovered: resolved.length };
|
||||
}
|
||||
|
||||
let timer: ReturnType<typeof setInterval> | null = null;
|
||||
|
||||
/** Looks every minute (each check has its own interval, so most passes ask nobody). Once a day it clears out old results. */
|
||||
export function initEndpointMonitor(): void {
|
||||
if (timer) return;
|
||||
let lastPrune = 0;
|
||||
const tick = async () => {
|
||||
try {
|
||||
await runEndpointPass();
|
||||
if (Date.now() - lastPrune > 24 * 3600_000) {
|
||||
lastPrune = Date.now();
|
||||
await pruneResults();
|
||||
}
|
||||
} catch (err) {
|
||||
console.error("[endpoints] the pass failed:", err);
|
||||
}
|
||||
};
|
||||
setTimeout(() => void tick(), 20_000).unref();
|
||||
timer = setInterval(() => void tick(), 60_000);
|
||||
timer.unref();
|
||||
}
|
||||
@@ -25,6 +25,8 @@ export async function listUpcomingWindows(now: Date = new Date()): Promise<Activ
|
||||
export function subjectOfConditionKey(key: string): string | null {
|
||||
let m = /^(?:offline|disk|failed-units|smart|zfs):server:(\d+)(?::|$)/.exec(key);
|
||||
if (m) return `server:${m[1]}`;
|
||||
m = /^endpoint(?:-cert)?:\d+:server:(\d+)$/.exec(key);
|
||||
if (m) return `server:${m[1]}`;
|
||||
m = /^disk:proxmox:(\d+):/.exec(key);
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^(?:synology-volume|synology-disk|disk:synology):(\d+):/.exec(key);
|
||||
|
||||
@@ -4,7 +4,8 @@
|
||||
* integration on its own, so a scrape every fifteen seconds costs a few database reads (the alerts are the app's own minute-old result).
|
||||
*/
|
||||
import { db } from "../db/client.js";
|
||||
import { assets, integrations, maintenanceWindows, secrets, servers } from "../db/schema.js";
|
||||
import { assets, endpointChecks, integrations, maintenanceWindows, secrets, servers } from "../db/schema.js";
|
||||
import { latestTests } from "./restoreTests.js";
|
||||
import { env } from "../env.js";
|
||||
import { getAlerts } from "./alerts.js";
|
||||
import { isAgentOutdated, latestAgentVersions } from "./agentVersions.js";
|
||||
@@ -53,6 +54,22 @@ class Exposition {
|
||||
}
|
||||
}
|
||||
|
||||
/** Endpoint checks and restore tests, for /metrics. */
|
||||
async function addEndpointMetrics(out: Exposition, now: Date) {
|
||||
for (const c of await db.select().from(endpointChecks)) {
|
||||
if (!c.enabled || c.lastOk === null) continue;
|
||||
const labels = { name: c.name, kind: c.kind };
|
||||
out.add("homelab_endpoint_up", "gauge", "1 if the endpoint check's last answer was fine, else 0.", c.lastOk ? 1 : 0, labels);
|
||||
if (c.lastResponseMs !== null) out.add("homelab_endpoint_response_milliseconds", "gauge", "How long the endpoint took to answer the last time.", c.lastResponseMs, labels);
|
||||
if (c.certExpiresAt) out.add("homelab_endpoint_certificate_days_left", "gauge", "Days until the certificate the endpoint shows expires (negative once it has).", Math.floor((Date.parse(c.certExpiresAt) - now.getTime()) / 86_400_000), labels);
|
||||
}
|
||||
for (const r of (await latestTests()).values()) {
|
||||
if (r.result === "not_needed") continue;
|
||||
out.add("homelab_restore_test_age_days", "gauge", "Days since a restore of this was last tested.", Math.max(0, Math.floor((now.getTime() - Date.parse(`${r.testedAt}T12:00:00Z`)) / 86_400_000)), { subject: r.label });
|
||||
out.add("homelab_restore_test_passed", "gauge", "1 if the last restore test passed (fully or partly), 0 if it failed.", r.result === "failed" ? 0 : 1, { subject: r.label });
|
||||
}
|
||||
}
|
||||
|
||||
export async function buildMetrics(now: Date = new Date()): Promise<string> {
|
||||
const started = Date.now();
|
||||
const out = new Exposition();
|
||||
@@ -177,5 +194,6 @@ export async function buildMetrics(now: Date = new Date()): Promise<string> {
|
||||
|
||||
out.add("homelab_scrape_duration_seconds", "gauge", "How long building this answer took.", (Date.now() - started) / 1000);
|
||||
out.add("homelab_scrape_timestamp_seconds", "gauge", "When this answer was built, as a Unix time.", nowSec);
|
||||
await addEndpointMetrics(out, now);
|
||||
return out.toString();
|
||||
}
|
||||
@@ -109,6 +109,8 @@ export function metaOfConditionKey(key: string, severity?: string): NotifyMeta {
|
||||
else if (key.startsWith("failed-units:")) category = "services";
|
||||
else if (/^(disk|smart|zfs|synology-volume|synology-disk):/.test(key)) category = "disk";
|
||||
else if (key.startsWith("automation:")) category = "automation";
|
||||
const server = /^(?:offline|disk|failed-units|smart|zfs):server:(\d+)/.exec(key);
|
||||
else if (key.startsWith("endpoint-cert:")) category = "expiry";
|
||||
else if (key.startsWith("endpoint:")) category = "monitoring";
|
||||
const server = /^(?:(?:offline|disk|failed-units|smart|zfs):server:|endpoint(?:-cert)?:\d+:server:)(\d+)/.exec(key);
|
||||
return { category, severity: severity === "critical" ? "critical" : "warning", serverId: server ? Number(server[1]) : null };
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
/**
|
||||
* Restore tests: a backup nobody has ever restored is a hope, not a backup. Somebody records when they tried to restore a guest (or anything else
|
||||
* — a share, a database) and how it went, and the app keeps the history, shows each guest's newest test beside its backups, and speaks up
|
||||
* when a test failed (always) or when none has been done in too long (if a time limit is set under Settings → Notifications).
|
||||
*/
|
||||
import { desc } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { restoreTests } from "../db/schema.js";
|
||||
import type { CoverageGuest, CoverageReport } from "./backupCoverage.js";
|
||||
|
||||
export const RESULTS = ["ok", "partial", "failed", "not_needed"] as const;
|
||||
export type RestoreResult = (typeof RESULTS)[number];
|
||||
export type TestRow = typeof restoreTests.$inferSelect;
|
||||
|
||||
/** What a subject's newest test amounts to: how it went, or that none was done (never) or that it is too old (stale). */
|
||||
export type RestoreStatus = RestoreResult | "never" | "stale";
|
||||
|
||||
export interface RestoreInfo {
|
||||
status: RestoreStatus;
|
||||
/** YYYY-MM-DD of the newest test; null when there is none. */
|
||||
testedAt: string | null;
|
||||
ageDays: number | null;
|
||||
note: string | null;
|
||||
testedBy: string | null;
|
||||
}
|
||||
|
||||
export const guestKey = (integrationId: number, vmid: number) => `guest:${integrationId}:${vmid}`;
|
||||
export const KEY_PATTERN = /^(guest:\d+:\d+|custom:[a-z0-9][a-z0-9-]{0,60})$/;
|
||||
|
||||
/** "Photos share" → "custom:photos-share". */
|
||||
export function customKey(label: string): string | null {
|
||||
const slug = label
|
||||
.toLowerCase()
|
||||
.normalize("NFKD")
|
||||
.replace(/[̀-ͯ]/g, "")
|
||||
.replace(/[^a-z0-9]+/g, "-")
|
||||
.replace(/^-+|-+$/g, "")
|
||||
.slice(0, 60);
|
||||
return slug ? `custom:${slug}` : null;
|
||||
}
|
||||
|
||||
const ageInDays = (testedAt: string, now: Date) => Math.floor((now.getTime() - Date.parse(`${testedAt}T12:00:00Z`)) / 86_400_000);
|
||||
|
||||
/** Pure: what the newest test (if any) says, against the time limit (0 = no limit). */
|
||||
export function restoreInfo(latest: Pick<TestRow, "result" | "testedAt" | "note" | "testedBy"> | undefined, now: Date, limitDays: number): RestoreInfo {
|
||||
if (!latest) return { status: "never", testedAt: null, ageDays: null, note: null, testedBy: null };
|
||||
const ageDays = ageInDays(latest.testedAt, now);
|
||||
let status: RestoreStatus = latest.result as RestoreResult;
|
||||
if ((status === "ok" || status === "partial") && limitDays > 0 && ageDays > limitDays) status = "stale";
|
||||
return { status, testedAt: latest.testedAt, ageDays, note: latest.note, testedBy: latest.testedBy };
|
||||
}
|
||||
|
||||
/** The newest test of each subject (the one with the latest date, and the later one if two share a date). */
|
||||
export async function latestTests(): Promise<Map<string, TestRow>> {
|
||||
const rows = await db.select().from(restoreTests).orderBy(desc(restoreTests.testedAt), desc(restoreTests.id));
|
||||
const out = new Map<string, TestRow>();
|
||||
for (const r of rows) if (!out.has(r.subjectKey)) out.set(r.subjectKey, r);
|
||||
return out;
|
||||
}
|
||||
|
||||
export interface RestoreAlert {
|
||||
key: string;
|
||||
severity: "critical" | "warning";
|
||||
message: string;
|
||||
guest: CoverageGuest | null;
|
||||
}
|
||||
|
||||
const daysWords = (d: number) => `${d} day${d === 1 ? "" : "s"}`;
|
||||
|
||||
/**
|
||||
* What to speak up about. A test that failed is always a problem (the backups may not be usable) until a newer one passes. With a limit set,
|
||||
* a guest that has backups but whose newest passing test is older than the limit — or that has never had one — is too, and so is anything
|
||||
* else somebody chose to track. A guest marked "no need to test" never is.
|
||||
*/
|
||||
export function restoreAlerts(report: CoverageReport, latest: Map<string, TestRow>, limitDays: number, now: Date): RestoreAlert[] {
|
||||
const out: RestoreAlert[] = [];
|
||||
const seen = new Set<string>();
|
||||
for (const g of report.guests) {
|
||||
const key = guestKey(g.integrationId, g.vmid);
|
||||
seen.add(key);
|
||||
const info = restoreInfo(latest.get(key), now, limitDays);
|
||||
const who = `${g.name} (#${g.vmid}) [${g.integrationName}]`;
|
||||
if (info.status === "failed") out.push({ key: `restore-test:${key}`, severity: "critical", guest: g, message: `${who}: the last restore test (${info.testedAt}) failed${info.note ? ` — ${info.note}` : ""}` });
|
||||
else if (limitDays > 0 && g.lastBackupAt !== null && info.status === "never") out.push({ key: `restore-test:${key}`, severity: "warning", guest: g, message: `${who}: a restore has never been tested` });
|
||||
else if (limitDays > 0 && info.status === "stale") out.push({ key: `restore-test:${key}`, severity: "warning", guest: g, message: `${who}: its last restore test was ${daysWords(info.ageDays ?? 0)} ago (the limit is ${daysWords(limitDays)})` });
|
||||
}
|
||||
for (const [key, row] of latest) {
|
||||
if (seen.has(key) || key.startsWith("guest:")) continue;
|
||||
const info = restoreInfo(row, now, limitDays);
|
||||
if (info.status === "failed") out.push({ key: `restore-test:${key}`, severity: "critical", guest: null, message: `${row.label}: the last restore test (${row.testedAt}) failed${row.note ? ` — ${row.note}` : ""}` });
|
||||
else if (info.status === "stale") out.push({ key: `restore-test:${key}`, severity: "warning", guest: null, message: `${row.label}: its last restore test was ${daysWords(info.ageDays ?? 0)} ago (the limit is ${daysWords(limitDays)})` });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
@@ -111,6 +111,8 @@ export interface HealthCheckSettings {
|
||||
diskForecastDays: number;
|
||||
/** A Proxmox guest whose newest backup is older than this many hours is raised as stale. 0 turns it off. */
|
||||
backupStaleHours: number;
|
||||
/** A restore test older than this many days (or none at all) is a problem for a guest that has backups; 0 = never (a failed test always is). */
|
||||
restoreTestDays: number;
|
||||
}
|
||||
|
||||
/** The weekly digest: one message a week through the notification channels. Off until an admin switches it on under Settings → Notifications. */
|
||||
@@ -257,7 +259,7 @@ const DEFAULTS: AppSettings = {
|
||||
agents: { autoUpdate: false },
|
||||
selfBackup: { enabled: true, time: "03:30", keep: 14 },
|
||||
signIn: { roleFromGroups: false, groupsClaim: "groups", adminGroups: [], operatorGroups: [], otherwise: "viewer" },
|
||||
healthChecks: { serverOfflineMinutes: 60, diskUsagePercent: 90, domainWarnDays: 30, diskForecastDays: 14, backupStaleHours: 48 },
|
||||
healthChecks: { serverOfflineMinutes: 60, diskUsagePercent: 90, domainWarnDays: 30, diskForecastDays: 14, backupStaleHours: 48, restoreTestDays: 0 },
|
||||
// Docker's default bridge (172.17.0.0/16) and the pool it hands custom networks from (172.18–31) live here, and every
|
||||
// Docker host repeats them. Shown on the Consistency page, where it can be removed if 172.16/12 is used as a real LAN.
|
||||
consistency: { excludedRanges: ["172.16.0.0/12"] },
|
||||
|
||||
@@ -0,0 +1,162 @@
|
||||
/**
|
||||
* The change log: the sentences, what is left out, the filters (kind, server, time, a word), paging, and who may read it.
|
||||
*
|
||||
* Run with: npm test --workspace server
|
||||
*/
|
||||
import { test, describe, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { AddressInfo } from "node:net";
|
||||
import type { Server } from "node:http";
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), "hm-changes-"));
|
||||
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
||||
process.env.SESSION_DIR = join(dir, "sessions");
|
||||
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
||||
process.env.HEALTH_STARTUP_GRACE_MS = "0";
|
||||
|
||||
const express = (await import("express")).default;
|
||||
const { db } = await import("../src/db/client.js");
|
||||
const { users, servers, auditLog, alertIncidents } = await import("../src/db/schema.js");
|
||||
const { runMigrations } = await import("../src/db/migrate.js");
|
||||
const { describeChange, getChangeLog } = await import("../src/services/changeLog.js");
|
||||
const { changesRouter } = await import("../src/routes/changes.js");
|
||||
|
||||
await runMigrations();
|
||||
|
||||
describe("the sentences", () => {
|
||||
test("maintenance, dependencies, hardware, restore tests and settings read as sentences", () => {
|
||||
assert.deepEqual(describeChange("maintenance", "start", { name: "nas", minutes: 60, reason: "reboot" }), { title: "Maintenance started on nas", tone: "info", detail: "for 60 min · reboot" });
|
||||
assert.equal(describeChange("maintenance", "extend", { name: "nas", minutes: 240 }).detail, "for 4 h from now");
|
||||
assert.equal(describeChange("dependency", "create", { from: "Gitea", to: "nas" }).title, "Gitea now needs nas");
|
||||
assert.equal(describeChange("dependency", "delete", { from: "Gitea", to: "nas" }).title, "Gitea no longer needs nas");
|
||||
assert.equal(describeChange("inventory", "create", { name: "NAS", kind: "storage" }).title, "Hardware added: NAS");
|
||||
assert.deepEqual(describeChange("restore_test", "record", { name: "elsa", result: "failed", testedAt: "2026-10-01", note: "no boot" }), { title: "Restore test of elsa failed", tone: "bad", detail: "2026-10-01 · no boot" });
|
||||
assert.equal(describeChange("restore_test", "record", { name: "elsa", result: "ok" }).tone, "good");
|
||||
const s = describeChange("settings", "update", { sections: ["notifications", "webhook"], changes: { notifications: { healthAlerts: { from: true, to: false } }, webhook: { url: "(changed)" } } });
|
||||
assert.equal(s.title, "Settings changed: notifications, webhook");
|
||||
assert.equal(s.detail, 'notifications.healthAlerts: true → false · webhook.url changed', "a secret's value is never in it");
|
||||
});
|
||||
|
||||
test("a server's own entries use the wording of its Activity card; anything else is made readable", () => {
|
||||
assert.equal(describeChange("server", "agent_updated", { from: 4, to: 6 }).title, "Agent updated to version 6");
|
||||
assert.equal(describeChange("alert", "ack", { assignee: "Olle", hours: 4 }).title, "Acknowledged, handed to Olle");
|
||||
assert.deepEqual(describeChange("dns", "add_record", { name: "a.example.com", type: "A" }), { title: "Dns: add record — a.example.com", tone: "neutral", detail: "type: A" });
|
||||
});
|
||||
});
|
||||
|
||||
describe("through the routes", () => {
|
||||
let app: Server;
|
||||
let base = "";
|
||||
let who = "operator";
|
||||
let nas = 0;
|
||||
let elsa = 0;
|
||||
|
||||
const audit = (category: string, action: string, extra: { targetType?: string; targetId?: string; detail?: unknown; actor?: string; at: string }) =>
|
||||
db.insert(auditLog).values({ category, action, targetType: extra.targetType ?? null, targetId: extra.targetId ?? null, detail: extra.detail === undefined ? null : JSON.stringify(extra.detail), actorLabel: extra.actor ?? "Ada", createdAt: extra.at });
|
||||
const ago = (min: number) => new Date(Date.now() - min * 60_000).toISOString().slice(0, 19).replace("T", " ");
|
||||
|
||||
before(async () => {
|
||||
await db.insert(users).values([
|
||||
{ oidcSub: "operator", name: "Olga", role: "operator" },
|
||||
{ oidcSub: "viewer", name: "Vic", role: "viewer" },
|
||||
]);
|
||||
const made = await db.insert(servers).values([{ name: "nas", apiTokenHash: "n", apiTokenPrefix: "n" }, { name: "elsa", apiTokenHash: "e", apiTokenPrefix: "e" }]).returning({ id: servers.id });
|
||||
nas = made[0].id;
|
||||
elsa = made[1].id;
|
||||
await audit("maintenance", "start", { targetType: "server", targetId: String(nas), detail: { name: "nas", minutes: 60, reason: "reboot" }, at: ago(10) });
|
||||
await audit("server", "agent_updated", { targetType: "server", targetId: String(elsa), detail: { from: 4, to: 6 }, actor: "system", at: ago(20) });
|
||||
await audit("dependency", "create", { targetType: "dependency", targetId: "1", detail: { from: "Gitea", to: "nas", fromKey: "integration:1", toKey: `server:${nas}` }, at: ago(30) });
|
||||
await audit("settings", "update", { targetType: "settings", detail: { sections: ["display"], changes: { display: { pageSize: { from: 20, to: 50 } } } }, at: ago(40) });
|
||||
await audit("session", "login", { at: ago(5) });
|
||||
await audit("privacy", "export_own_data", { at: ago(6) });
|
||||
await audit("inventory", "create", { targetType: "asset", targetId: "7", detail: { name: "Core switch", kind: "network" }, at: ago(60 * 24 * 3) });
|
||||
await audit("server", "update", { targetType: "server", targetId: String(nas), detail: { name: "nas" }, at: ago(60 * 24 * 20) });
|
||||
await db.insert(alertIncidents).values({ alertKey: "offline:offline:server:1", severity: "critical", peakSeverity: "critical", category: "offline", source: "Server", message: "nas hasn't reported for 3 h", targetType: "server", targetId: nas, targetName: "nas", openedAt: new Date(Date.now() - 50 * 60_000).toISOString(), resolvedAt: new Date(Date.now() - 15 * 60_000).toISOString(), lastSeenAt: new Date().toISOString() });
|
||||
|
||||
const a = express();
|
||||
a.use(express.json());
|
||||
a.use((req, _res, next) => {
|
||||
(req as unknown as { session: unknown }).session = { user: { sub: who } };
|
||||
next();
|
||||
});
|
||||
a.use("/api/changes", changesRouter);
|
||||
app = await new Promise<Server>((resolve) => {
|
||||
const s = a.listen(0, "127.0.0.1", () => resolve(s));
|
||||
});
|
||||
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/changes`;
|
||||
});
|
||||
|
||||
after(() => {
|
||||
app.close();
|
||||
try {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Windows keeps the database file locked until the process exits.
|
||||
}
|
||||
});
|
||||
|
||||
const get = async (query = "") => (await fetch(`${base}${query}`)).json();
|
||||
const titles = (r: { entries: { title: string }[] }) => r.entries.map((e) => e.title);
|
||||
|
||||
test("everything of the last week, newest first, with the alerts raised and cleared in between; sign-ins and data exports are not changes", async () => {
|
||||
const r = await get();
|
||||
assert.deepEqual(titles(r), [
|
||||
"Maintenance started on nas",
|
||||
"Alert cleared: nas hasn't reported for 3 h",
|
||||
"Agent updated to version 6",
|
||||
"Gitea now needs nas",
|
||||
"Settings changed: display",
|
||||
"Alert raised: nas hasn't reported for 3 h",
|
||||
"Hardware added: Core switch",
|
||||
]);
|
||||
assert.ok(!r.entries.some((e: { title: string }) => /login|export/i.test(e.title)));
|
||||
const maint = r.entries[0];
|
||||
assert.deepEqual([maint.kind, maint.actor, maint.link, maint.serverId], ["maintenance", "Ada", `/servers/${nas}`, nas]);
|
||||
assert.equal(r.entries.find((e: { title: string }) => e.title.startsWith("Alert raised")).link.startsWith("/incidents/"), true);
|
||||
assert.equal(r.kinds.length, 10);
|
||||
});
|
||||
|
||||
test("by kind, by how far back, and by a word", async () => {
|
||||
assert.deepEqual(titles(await get("?kind=maintenance")), ["Maintenance started on nas"]);
|
||||
assert.deepEqual(titles(await get("?kind=alerts")).length, 2);
|
||||
assert.equal((await get("?days=1")).entries.length, 6, "the hardware added three days ago is out");
|
||||
assert.equal((await get("?days=30")).entries.length, 8, "and a month covers the old edit too");
|
||||
assert.deepEqual(titles(await get("?q=gitea")), ["Gitea now needs nas"]);
|
||||
assert.deepEqual(titles(await get("?q=version%20elsa")), ["Agent updated to version 6"], "every word must appear somewhere, the server's name included");
|
||||
assert.deepEqual((await get("?kind=nonsense")).entries.length, 7, "an unknown kind is no filter");
|
||||
});
|
||||
|
||||
test("by server: its own entries, its alerts, and what mentions it by key — not the other server's", async () => {
|
||||
assert.deepEqual(titles(await get(`?serverId=${nas}`)), ["Maintenance started on nas", "Alert cleared: nas hasn't reported for 3 h", "Gitea now needs nas", "Alert raised: nas hasn't reported for 3 h"]);
|
||||
assert.deepEqual(titles(await get(`?serverId=${elsa}`)), ["Agent updated to version 6"]);
|
||||
assert.deepEqual((await get("?serverId=99999")).entries, []);
|
||||
});
|
||||
|
||||
test("paged: a limit says there is more, and 'before' goes on from where it stopped", async () => {
|
||||
const first = await get("?limit=3");
|
||||
assert.equal(first.entries.length, 3);
|
||||
assert.equal(first.more, true);
|
||||
const next = await get(`?limit=3&before=${encodeURIComponent(first.entries[2].at)}`);
|
||||
assert.deepEqual(titles(next), ["Gitea now needs nas", "Settings changed: display", "Alert raised: nas hasn't reported for 3 h"]);
|
||||
assert.ok(next.entries.every((e: { at: string }) => e.at < first.entries[2].at));
|
||||
assert.equal(next.more, true, "the hardware added three days ago is still to come");
|
||||
const last = await get(`?limit=3&before=${encodeURIComponent(next.entries[2].at)}`);
|
||||
assert.deepEqual(titles(last), ["Hardware added: Core switch"]);
|
||||
assert.equal(last.more, false);
|
||||
});
|
||||
|
||||
test("operators and administrators can read it; a viewer can't", async () => {
|
||||
who = "viewer";
|
||||
assert.equal((await fetch(base)).status, 403);
|
||||
who = "operator";
|
||||
assert.equal((await fetch(base)).status, 200);
|
||||
});
|
||||
|
||||
test("the service on its own gives the same answer for any moment it is asked about", async () => {
|
||||
const future = await getChangeLog({ now: new Date(Date.now() + 40 * 86_400_000), days: 1 });
|
||||
assert.deepEqual(future.entries, [], "nothing in the day before a month from now");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,407 @@
|
||||
/**
|
||||
* Endpoint checks: what counts as an answer, the probes themselves (against servers this machine runs), what a run records, the problems they
|
||||
* stand for, and what the routes, the alerts and the notifications do with them.
|
||||
*
|
||||
* Run with: npm test --workspace server
|
||||
*/
|
||||
import { test, describe, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { createServer as createHttpServer, type Server } from "node:http";
|
||||
import { createServer as createHttpsServer } from "node:https";
|
||||
import { mkdtempSync, readFileSync, rmSync } from "node:fs";
|
||||
import { networkInterfaces, tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { AddressInfo } from "node:net";
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), "hm-endpoints-"));
|
||||
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
||||
process.env.SESSION_DIR = join(dir, "sessions");
|
||||
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
||||
process.env.HEALTH_STARTUP_GRACE_MS = "0";
|
||||
|
||||
const express = (await import("express")).default;
|
||||
const { db } = await import("../src/db/client.js");
|
||||
const { users, servers, endpointChecks, endpointResults } = await import("../src/db/schema.js");
|
||||
const { runMigrations } = await import("../src/db/migrate.js");
|
||||
const { statusAccepter, isStatusSpec, summariseResults, endpointConditions, probe, runCheck, runDueChecks } = await import("../src/services/endpointChecks.js");
|
||||
const { runEndpointPass } = await import("../src/services/endpointMonitor.js");
|
||||
const { endpointsRouter } = await import("../src/routes/endpoints.js");
|
||||
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
|
||||
const { startOrExtendWindow } = await import("../src/services/maintenance.js");
|
||||
const { updateSettings } = await import("../src/services/settingsStore.js");
|
||||
const { buildMetrics } = await import("../src/services/metrics.js");
|
||||
const { eq } = await import("drizzle-orm");
|
||||
|
||||
await runMigrations();
|
||||
|
||||
type Row = typeof endpointChecks.$inferSelect;
|
||||
const NOW = new Date("2026-10-18T12:00:00Z");
|
||||
const row = (extra: Partial<Row> = {}): Row => ({
|
||||
id: 1, name: "Gitea", kind: "http", target: "https://git.lab/", port: null, expectStatus: null, keyword: null, insecure: false, intervalMinutes: 5, timeoutSeconds: 10, failuresBeforeAlert: 2, warnDays: 14,
|
||||
serverId: null, enabled: true, lastCheckedAt: null, lastOk: null, lastMessage: null, lastResponseMs: null, certExpiresAt: null, consecutiveFailures: 0, lastChangeAt: null, createdAt: "2026-10-01 00:00:00", ...extra,
|
||||
});
|
||||
|
||||
describe("what counts as an answer", () => {
|
||||
test("status codes, ranges and the default of 200 to 399", () => {
|
||||
const usual = statusAccepter(null);
|
||||
assert.deepEqual([199, 200, 301, 399, 400, 500].map(usual), [false, true, true, true, false, false]);
|
||||
const custom = statusAccepter("200-204, 401");
|
||||
assert.deepEqual([200, 204, 205, 401, 403].map(custom), [true, true, false, true, false]);
|
||||
assert.equal(statusAccepter("rubbish")(200), true, "unreadable falls back to the usual");
|
||||
assert.equal(isStatusSpec("200, 301-302"), true);
|
||||
assert.equal(isStatusSpec("2xx"), false);
|
||||
});
|
||||
|
||||
test("uptime, the average answer time and the recent run come from the stored results", () => {
|
||||
const at = (min: number) => new Date(NOW.getTime() - min * 60_000).toISOString();
|
||||
const stats = summariseResults(
|
||||
[
|
||||
{ at: at(10), ok: true, ms: 100 },
|
||||
{ at: at(20), ok: true, ms: 300 },
|
||||
{ at: at(30), ok: false, ms: null },
|
||||
{ at: at(60 * 30), ok: false, ms: null }, // yesterday and a bit
|
||||
{ at: at(60 * 24 * 6), ok: true, ms: 50 },
|
||||
],
|
||||
NOW,
|
||||
);
|
||||
assert.equal(stats.uptime24h, 66.7, "2 of 3 in the last day");
|
||||
assert.equal(stats.uptime7d, 60, "3 of 5 in the last week");
|
||||
assert.equal(stats.avgMs24h, 200, "only answers that came count");
|
||||
assert.equal(stats.recent[0].at, at(60 * 24 * 6), "oldest first");
|
||||
assert.deepEqual(summariseResults([], NOW), { uptime24h: null, uptime7d: null, avgMs24h: null, recent: [] });
|
||||
});
|
||||
});
|
||||
|
||||
describe("what is a problem", () => {
|
||||
test("a check has to fail several times in a row; a blip isn't", () => {
|
||||
assert.deepEqual(endpointConditions([row({ lastOk: false, consecutiveFailures: 1, lastMessage: "answered 500" })], NOW), []);
|
||||
const [c] = endpointConditions([row({ lastOk: false, consecutiveFailures: 2, lastMessage: "answered 500" })], NOW);
|
||||
assert.equal(c.key, "endpoint:1");
|
||||
assert.equal(c.category, "monitoring");
|
||||
assert.equal(c.severity, "critical");
|
||||
assert.match(c.message, /Gitea isn't answering — answered 500/);
|
||||
assert.deepEqual(endpointConditions([row({ lastOk: false, consecutiveFailures: 9, enabled: false })], NOW), [], "a check that is switched off says nothing");
|
||||
assert.deepEqual(endpointConditions([row({ lastOk: true, consecutiveFailures: 0 })], NOW), []);
|
||||
});
|
||||
|
||||
test("a certificate is a problem from its warning time on, critical in the last three days, and the key carries the server it is linked to", () => {
|
||||
const days = (n: number) => new Date(NOW.getTime() + n * 86_400_000).toISOString();
|
||||
assert.deepEqual(endpointConditions([row({ certExpiresAt: days(20) })], NOW), []);
|
||||
const [warn] = endpointConditions([row({ certExpiresAt: days(10), serverId: 4 })], NOW);
|
||||
assert.equal(warn.key, "endpoint-cert:1:server:4");
|
||||
assert.deepEqual([warn.category, warn.severity], ["expiry", "warning"]);
|
||||
assert.match(warn.message, /expires in 10 days/);
|
||||
assert.equal(endpointConditions([row({ certExpiresAt: days(2) })], NOW)[0].severity, "critical");
|
||||
assert.match(endpointConditions([row({ certExpiresAt: days(-3) })], NOW)[0].message, /expired 3 days ago/);
|
||||
});
|
||||
|
||||
test("a certificate check whose certificate has expired says so once, not also that it isn't answering", () => {
|
||||
const expired = new Date(NOW.getTime() - 86_400_000).toISOString();
|
||||
const list = endpointConditions([row({ kind: "tls", target: "git.lab", port: 443, lastOk: false, consecutiveFailures: 5, lastMessage: "The certificate expired", certExpiresAt: expired })], NOW);
|
||||
assert.deepEqual(list.map((c) => c.category), ["expiry"]);
|
||||
});
|
||||
});
|
||||
|
||||
/** An address of this machine that isn't loopback (the checks refuse that), if it has one. */
|
||||
function lanAddress(): string | null {
|
||||
for (const list of Object.values(networkInterfaces())) {
|
||||
for (const i of list ?? []) if (i.family === "IPv4" && !i.internal && !i.address.startsWith("169.254.")) return i.address;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
const LAN = lanAddress();
|
||||
|
||||
describe("asking", { skip: LAN === null && "this machine has no address other than loopback" }, () => {
|
||||
let web: Server;
|
||||
let port = 0;
|
||||
let sleeping: Server;
|
||||
let sleepingPort = 0;
|
||||
const hits: string[] = [];
|
||||
|
||||
before(async () => {
|
||||
web = createHttpServer((req, res) => {
|
||||
hits.push(req.url ?? "");
|
||||
if (req.url === "/ok") return void res.end("<h1>Welcome to the lab</h1>");
|
||||
if (req.url === "/broken") {
|
||||
res.statusCode = 500;
|
||||
return void res.end("oops");
|
||||
}
|
||||
if (req.url === "/moved") {
|
||||
res.statusCode = 301;
|
||||
res.setHeader("location", "/ok");
|
||||
return void res.end();
|
||||
}
|
||||
res.statusCode = 404;
|
||||
res.end("no");
|
||||
});
|
||||
await new Promise<void>((r) => web.listen(0, "0.0.0.0", r));
|
||||
port = (web.address() as AddressInfo).port;
|
||||
// Accepts the connection and never answers.
|
||||
sleeping = createHttpServer(() => {});
|
||||
await new Promise<void>((r) => sleeping.listen(0, "0.0.0.0", r));
|
||||
sleepingPort = (sleeping.address() as AddressInfo).port;
|
||||
});
|
||||
|
||||
after(() => {
|
||||
web.close();
|
||||
sleeping.close();
|
||||
});
|
||||
|
||||
const http = (path: string, extra: Partial<Row> = {}) => row({ kind: "http", target: `http://${LAN}:${port}${path}`, timeoutSeconds: 3, ...extra });
|
||||
|
||||
test("a web address that answers is fine, with how long it took", async () => {
|
||||
const r = await probe(http("/ok"));
|
||||
assert.equal(r.ok, true);
|
||||
assert.match(r.message, /answered 200 in \d+ ms/);
|
||||
assert.ok(r.ms !== null && r.ms >= 0);
|
||||
});
|
||||
|
||||
test("a wrong status fails, unless that is what was expected; a redirect is an answer", async () => {
|
||||
assert.match((await probe(http("/broken"))).message, /answered 500/);
|
||||
assert.equal((await probe(http("/broken", { expectStatus: "500" }))).ok, true);
|
||||
assert.equal((await probe(http("/missing"))).ok, false);
|
||||
assert.equal((await probe(http("/moved"))).ok, true, "301 is within 200 to 399");
|
||||
});
|
||||
|
||||
test("a word the page must contain: found after following a redirect, or missing", async () => {
|
||||
assert.equal((await probe(http("/ok", { keyword: "Welcome" }))).ok, true);
|
||||
const missing = await probe(http("/ok", { keyword: "Goodbye" }));
|
||||
assert.equal(missing.ok, false);
|
||||
assert.match(missing.message, /doesn't contain “Goodbye”/);
|
||||
assert.equal((await probe(http("/moved", { keyword: "Welcome" }))).ok, true, "with a word to find, the redirect is followed to the page");
|
||||
});
|
||||
|
||||
test("no answer in time, and a refused connection, say so", async () => {
|
||||
const slow = await probe(row({ kind: "http", target: `http://${LAN}:${sleepingPort}/`, timeoutSeconds: 1 }));
|
||||
assert.equal(slow.ok, false);
|
||||
assert.match(slow.message, /no answer within 1 s/);
|
||||
const refused = await probe(row({ kind: "tcp", target: LAN!, port: 1, timeoutSeconds: 2 }));
|
||||
assert.equal(refused.ok, false);
|
||||
const open = await probe(row({ kind: "tcp", target: LAN!, port, timeoutSeconds: 2 }));
|
||||
assert.equal(open.ok, true);
|
||||
assert.match(open.message, new RegExp(`port ${port} answered in \\d+ ms`));
|
||||
});
|
||||
|
||||
test("it will not be pointed at this machine itself or at the cloud's metadata address", async () => {
|
||||
for (const target of ["http://127.0.0.1:1/", "http://169.254.169.254/latest/meta-data/"]) {
|
||||
const r = await probe(row({ kind: "http", target }));
|
||||
assert.equal(r.ok, false);
|
||||
assert.match(r.message, /can't be used|isn't allowed/);
|
||||
}
|
||||
assert.equal((await probe(row({ kind: "tcp", target: "127.0.0.1", port: 22 }))).ok, false);
|
||||
assert.equal((await probe(row({ kind: "http", target: "ftp://example.com/" }))).ok, false);
|
||||
});
|
||||
|
||||
test("a certificate that can't be trusted fails unless it is accepted, and either way its expiry is read", async () => {
|
||||
const work = mkdtempSync(join(tmpdir(), "hm-cert-"));
|
||||
const made = spawnSync("openssl", ["req", "-x509", "-newkey", "rsa:2048", "-nodes", "-days", "30", "-subj", "/CN=lab.example", "-keyout", join(work, "k.pem"), "-out", join(work, "c.pem")], { encoding: "utf8" });
|
||||
if (made.status !== 0) {
|
||||
rmSync(work, { recursive: true, force: true });
|
||||
return; // no openssl here: the rest is covered by what the pure parts say
|
||||
}
|
||||
const secure = createHttpsServer({ key: readFileSync(join(work, "k.pem")), cert: readFileSync(join(work, "c.pem")) }, (_req, res) => res.end("secure"));
|
||||
await new Promise<void>((r) => secure.listen(0, "0.0.0.0", r));
|
||||
const sp = (secure.address() as AddressInfo).port;
|
||||
try {
|
||||
const strict = await probe(row({ kind: "http", target: `https://${LAN}:${sp}/`, timeoutSeconds: 5 }));
|
||||
assert.equal(strict.ok, false);
|
||||
assert.match(strict.message, /certificate can't be trusted: it is self-signed/);
|
||||
const lenient = await probe(row({ kind: "http", target: `https://${LAN}:${sp}/`, insecure: true, timeoutSeconds: 5 }));
|
||||
assert.equal(lenient.ok, true);
|
||||
assert.ok(lenient.certExpiresAt && Date.parse(lenient.certExpiresAt) > Date.now() + 25 * 86_400_000, "about 30 days left");
|
||||
|
||||
const tlsStrict = await probe(row({ kind: "tls", target: LAN!, port: sp, timeoutSeconds: 5 }));
|
||||
assert.equal(tlsStrict.ok, false);
|
||||
assert.match(tlsStrict.message, /can't be trusted/);
|
||||
const tlsLenient = await probe(row({ kind: "tls", target: LAN!, port: sp, insecure: true, timeoutSeconds: 5 }));
|
||||
assert.equal(tlsLenient.ok, true);
|
||||
assert.match(tlsLenient.message, /good until \d{4}-\d{2}-\d{2}/);
|
||||
const plain = await probe(row({ kind: "tls", target: LAN!, port, timeoutSeconds: 5 }));
|
||||
assert.equal(plain.ok, false);
|
||||
assert.match(plain.message, /isn't speaking TLS/);
|
||||
} finally {
|
||||
secure.close();
|
||||
rmSync(work, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("a run, a pass, the routes, the alerts and the notifications", () => {
|
||||
let app: Server;
|
||||
let base = "";
|
||||
let who = "admin";
|
||||
let serverId = 0;
|
||||
const messages: string[] = [];
|
||||
let hook: Server;
|
||||
let answer: { ok: boolean; ms: number | null; message: string; certExpiresAt?: string | null } = { ok: true, ms: 12, message: "answered 200 in 12 ms" };
|
||||
|
||||
before(async () => {
|
||||
await db.insert(users).values([
|
||||
{ oidcSub: "admin", name: "Ada", role: "admin" },
|
||||
{ oidcSub: "operator", name: "Olga", role: "operator" },
|
||||
{ oidcSub: "viewer", name: "Vic", role: "viewer" },
|
||||
]);
|
||||
serverId = (await db.insert(servers).values({ name: "nas", apiTokenHash: "n", apiTokenPrefix: "n", lastSeenAt: new Date().toISOString() }).returning({ id: servers.id }))[0].id;
|
||||
hook = createHttpServer((req, res) => {
|
||||
const chunks: Buffer[] = [];
|
||||
req.on("data", (c) => chunks.push(c));
|
||||
req.on("end", () => {
|
||||
messages.push(String(JSON.parse(Buffer.concat(chunks).toString()).content));
|
||||
res.end("{}");
|
||||
});
|
||||
});
|
||||
await new Promise<void>((r) => hook.listen(0, "127.0.0.1", r));
|
||||
await updateSettings({ webhook: { enabled: true, url: `http://127.0.0.1:${(hook.address() as AddressInfo).port}/` } });
|
||||
|
||||
const a = express();
|
||||
a.use(express.json());
|
||||
a.use((req, _res, next) => {
|
||||
(req as unknown as { session: unknown }).session = { user: { sub: who } };
|
||||
next();
|
||||
});
|
||||
a.use("/api/endpoints", endpointsRouter);
|
||||
app = await new Promise<Server>((resolve) => {
|
||||
const s = a.listen(0, "127.0.0.1", () => resolve(s));
|
||||
});
|
||||
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/endpoints`;
|
||||
});
|
||||
|
||||
after(() => {
|
||||
app.close();
|
||||
hook.close();
|
||||
try {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Windows keeps the database file locked until the process exits.
|
||||
}
|
||||
});
|
||||
|
||||
const send = (method: string, path: string, body?: unknown) => fetch(`${base}${path}`, { method, headers: { "content-type": "application/json" }, body: body === undefined ? undefined : JSON.stringify(body) });
|
||||
const fake = async () => answer;
|
||||
let id = 0;
|
||||
|
||||
test("an administrator adds a check; a wrong one is refused with a reason; nobody else can", async () => {
|
||||
const res = await send("POST", "/", { name: "Gitea", kind: "http", target: "https://git.lab/", serverId });
|
||||
assert.equal(res.status, 201);
|
||||
id = (await res.json()).check.id;
|
||||
for (const bad of [
|
||||
{ name: "x", kind: "http", target: "git.lab" },
|
||||
{ name: "x", kind: "http", target: "https://a/", expectStatus: "2xx" },
|
||||
{ name: "x", kind: "tcp", target: "host" },
|
||||
{ name: "x", kind: "tcp", target: "http://host", port: 80 },
|
||||
{ name: "x", kind: "tls", target: "host", port: 70000 },
|
||||
{ name: "x", kind: "ping", target: "host" },
|
||||
{ name: "x", kind: "http", target: "https://a/", serverId: 99999 },
|
||||
]) assert.equal((await send("POST", "/", bad)).status, 400, JSON.stringify(bad));
|
||||
who = "operator";
|
||||
assert.equal((await send("POST", "/", { name: "y", kind: "tcp", target: "h", port: 22 })).status, 403);
|
||||
who = "viewer";
|
||||
assert.equal((await send("GET", "/")).status, 200, "everyone can look");
|
||||
who = "admin";
|
||||
});
|
||||
|
||||
test("a run records the answer on the check and as a point in its history", async () => {
|
||||
const [c] = await db.select().from(endpointChecks).where(eq(endpointChecks.id, id));
|
||||
answer = { ok: true, ms: 40, message: "answered 200 in 40 ms", certExpiresAt: new Date(NOW.getTime() + 3 * 86_400_000).toISOString() };
|
||||
await runCheck(c, new Date("2026-10-18T12:00:00Z"), fake);
|
||||
const [after1] = await db.select().from(endpointChecks).where(eq(endpointChecks.id, id));
|
||||
assert.deepEqual([after1.lastOk, after1.lastResponseMs, after1.consecutiveFailures, after1.lastChangeAt], [true, 40, 0, "2026-10-18T12:00:00.000Z"]);
|
||||
assert.ok(after1.certExpiresAt);
|
||||
assert.equal((await db.select().from(endpointResults).where(eq(endpointResults.checkId, id))).length, 1);
|
||||
|
||||
answer = { ok: false, ms: null, message: "answered 500" };
|
||||
await runCheck(after1, new Date("2026-10-18T12:05:00Z"), fake);
|
||||
const [failed] = await db.select().from(endpointChecks).where(eq(endpointChecks.id, id));
|
||||
assert.deepEqual([failed.lastOk, failed.consecutiveFailures, failed.lastChangeAt, failed.lastMessage], [false, 1, "2026-10-18T12:05:00.000Z", "answered 500"]);
|
||||
await runCheck(failed, new Date("2026-10-18T12:10:00Z"), fake);
|
||||
const [again] = await db.select().from(endpointChecks).where(eq(endpointChecks.id, id));
|
||||
assert.deepEqual([again.consecutiveFailures, again.lastChangeAt], [2, "2026-10-18T12:05:00.000Z"], "the time of the change is when it began failing, not each failure");
|
||||
});
|
||||
|
||||
test("only the checks that are due are asked", async () => {
|
||||
await db.update(endpointChecks).set({ lastCheckedAt: "2026-10-18T12:10:00.000Z" }).where(eq(endpointChecks.id, id));
|
||||
let asked = 0;
|
||||
const counting = async () => {
|
||||
asked++;
|
||||
return answer;
|
||||
};
|
||||
assert.equal(await runDueChecks(new Date("2026-10-18T12:12:00Z"), counting), 0, "two minutes after a five-minute check");
|
||||
assert.equal(await runDueChecks(new Date("2026-10-18T12:15:30Z"), counting), 1);
|
||||
assert.equal(asked, 1);
|
||||
await db.update(endpointChecks).set({ enabled: false }).where(eq(endpointChecks.id, id));
|
||||
assert.equal(await runDueChecks(new Date("2026-10-18T13:00:00Z"), counting), 0, "a check that is switched off is never asked");
|
||||
await db.update(endpointChecks).set({ enabled: true }).where(eq(endpointChecks.id, id));
|
||||
});
|
||||
|
||||
test("a check that keeps failing is an alert, silenced by maintenance on its server, and a certificate running out is another", async () => {
|
||||
await db.update(endpointChecks).set({ lastOk: false, consecutiveFailures: 3, lastMessage: "answered 500", certExpiresAt: new Date(Date.now() + 2 * 86_400_000).toISOString() }).where(eq(endpointChecks.id, id));
|
||||
invalidateAlertsCache();
|
||||
let alerts = (await getAlerts(true)).alerts.filter((a) => a.source === "Endpoint");
|
||||
const down = alerts.find((a) => a.category === "monitoring")!;
|
||||
const cert = alerts.find((a) => a.category === "expiry")!;
|
||||
assert.match(down.message, /Gitea isn't answering — answered 500/);
|
||||
assert.equal(down.severity, "critical");
|
||||
assert.equal(down.link, "/endpoints");
|
||||
assert.deepEqual(down.target && [down.target.type, down.target.id], ["server", serverId]);
|
||||
assert.match(cert.message, /expires in/);
|
||||
assert.equal(cert.severity, "critical", "two days left");
|
||||
assert.equal(down.silenced, false);
|
||||
|
||||
await startOrExtendWindow("server", serverId, 60, "work", "Ada");
|
||||
invalidateAlertsCache();
|
||||
alerts = (await getAlerts(true)).alerts.filter((a) => a.source === "Endpoint");
|
||||
assert.ok(alerts.length === 2 && alerts.every((a) => a.silenced), "both are under the server's maintenance");
|
||||
});
|
||||
|
||||
test("the pass tells you once when a check starts failing, and once when it is fine again", async () => {
|
||||
const { listActiveWindows } = await import("../src/services/maintenance.js");
|
||||
const { maintenanceWindows } = await import("../src/db/schema.js");
|
||||
for (const w of await listActiveWindows()) await db.delete(maintenanceWindows).where(eq(maintenanceWindows.id, w.id));
|
||||
await db.update(endpointChecks).set({ lastOk: true, consecutiveFailures: 0, lastCheckedAt: null, certExpiresAt: null }).where(eq(endpointChecks.id, id));
|
||||
messages.length = 0;
|
||||
answer = { ok: false, ms: null, message: "answered 503" };
|
||||
const t = (min: number) => new Date(Date.now() + min * 60_000);
|
||||
let r = await runEndpointPass(t(0), fake);
|
||||
assert.equal(r.announced, 0, "the first failure is a blip");
|
||||
r = await runEndpointPass(t(6), fake);
|
||||
assert.equal(r.announced, 1, "the second is a problem");
|
||||
assert.match(messages.join("\n"), /Gitea isn't answering — answered 503/);
|
||||
const sent = messages.length;
|
||||
await runEndpointPass(t(12), fake);
|
||||
assert.equal(messages.length, sent, "and it isn't said again");
|
||||
answer = { ok: true, ms: 9, message: "answered 200 in 9 ms" };
|
||||
r = await runEndpointPass(t(18), fake);
|
||||
assert.equal(r.recovered, 1);
|
||||
assert.match(messages[messages.length - 1], /Gitea isn't answering/, "the recovery names what cleared");
|
||||
});
|
||||
|
||||
test("check now is for operators and answers at once; the list carries the figures; metrics have them", async () => {
|
||||
who = "viewer";
|
||||
assert.equal((await send("POST", `/${id}/check`)).status, 403);
|
||||
who = "operator";
|
||||
// the real probe, aimed at a name nothing answers to (and refused if it were loopback): it says what happened
|
||||
await db.update(endpointChecks).set({ target: "http://127.0.0.1:1/" }).where(eq(endpointChecks.id, id));
|
||||
const res = await send("POST", `/${id}/check`);
|
||||
assert.equal(res.status, 200);
|
||||
assert.equal((await res.json()).result.ok, false);
|
||||
who = "viewer";
|
||||
const list = await (await send("GET", "/")).json();
|
||||
const c = list.checks.find((x: { id: number }) => x.id === id);
|
||||
assert.equal(c.server.name, "nas");
|
||||
assert.ok(c.stats.recent.length >= 1);
|
||||
assert.equal(typeof c.stats.uptime24h, "number");
|
||||
assert.match(await buildMetrics(), /homelab_endpoint_up\{name="Gitea",kind="http"\} 0/);
|
||||
});
|
||||
|
||||
test("changing what is asked forgets what was found; removing it takes its history", async () => {
|
||||
who = "admin";
|
||||
const res = await send("PUT", `/${id}`, { name: "Gitea", kind: "http", target: "https://git2.lab/", serverId });
|
||||
assert.equal(res.status, 200);
|
||||
const [c] = await db.select().from(endpointChecks).where(eq(endpointChecks.id, id));
|
||||
assert.deepEqual([c.lastOk, c.consecutiveFailures, c.lastMessage], [null, 0, null]);
|
||||
assert.equal((await send("DELETE", `/${id}`)).status, 204);
|
||||
assert.equal((await db.select().from(endpointResults).where(eq(endpointResults.checkId, id))).length, 0);
|
||||
assert.equal((await send("DELETE", `/${id}`)).status, 404);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,210 @@
|
||||
/**
|
||||
* Restore tests: what the newest test of something amounts to, when to speak up, and the routes that record them.
|
||||
*
|
||||
* Run with: npm test --workspace server
|
||||
*/
|
||||
import { test, describe, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { AddressInfo } from "node:net";
|
||||
import type { Server } from "node:http";
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), "hm-restore-"));
|
||||
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
||||
process.env.SESSION_DIR = join(dir, "sessions");
|
||||
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
||||
process.env.HEALTH_STARTUP_GRACE_MS = "0";
|
||||
|
||||
const express = (await import("express")).default;
|
||||
const { db } = await import("../src/db/client.js");
|
||||
const { users, restoreTests, auditLog } = await import("../src/db/schema.js");
|
||||
const { runMigrations } = await import("../src/db/migrate.js");
|
||||
const { restoreInfo, restoreAlerts, customKey, guestKey } = await import("../src/services/restoreTests.js");
|
||||
const { setCoverageLoaderForTests, buildCoverage } = await import("../src/services/backupCoverage.js");
|
||||
const { backupsRouter } = await import("../src/routes/backups.js");
|
||||
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
|
||||
const { updateSettings } = await import("../src/services/settingsStore.js");
|
||||
const { buildMetrics } = await import("../src/services/metrics.js");
|
||||
|
||||
await runMigrations();
|
||||
|
||||
const NOW = new Date("2026-10-18T12:00:00Z");
|
||||
const daysAgo = (n: number) => new Date(NOW.getTime() - n * 86_400_000).toISOString().slice(0, 10);
|
||||
const test1 = (result: string, ago: number, extra: Record<string, unknown> = {}) => ({ result, testedAt: daysAgo(ago), note: null as string | null, testedBy: "Ada", ...extra });
|
||||
|
||||
describe("what the newest test amounts to", () => {
|
||||
test("how it went; none at all is never; an old pass is stale once there is a limit; a failure or 'not needed' doesn't age", () => {
|
||||
assert.equal(restoreInfo(undefined, NOW, 90).status, "never");
|
||||
assert.deepEqual(restoreInfo(test1("ok", 10), NOW, 90), { status: "ok", testedAt: daysAgo(10), ageDays: 10, note: null, testedBy: "Ada" });
|
||||
assert.equal(restoreInfo(test1("ok", 100), NOW, 90).status, "stale");
|
||||
assert.equal(restoreInfo(test1("ok", 100), NOW, 0).status, "ok", "no limit, no staleness");
|
||||
assert.equal(restoreInfo(test1("partial", 100), NOW, 90).status, "stale");
|
||||
assert.equal(restoreInfo(test1("failed", 400), NOW, 90).status, "failed");
|
||||
assert.equal(restoreInfo(test1("not_needed", 400), NOW, 90).status, "not_needed");
|
||||
});
|
||||
|
||||
test("something that isn't a guest is told apart by its name", () => {
|
||||
assert.equal(customKey("Photos share"), "custom:photos-share");
|
||||
assert.equal(customKey(" Gitea — database! "), "custom:gitea-database");
|
||||
assert.equal(customKey("Åkerblom's NAS"), "custom:akerblom-s-nas");
|
||||
assert.equal(customKey("!!!"), null);
|
||||
assert.equal(guestKey(2, 105), "guest:2:105");
|
||||
});
|
||||
});
|
||||
|
||||
const guest = (vmid: number, name: string) => ({ vmid, name, node: "pve1", type: "qemu" as const, status: "running", cpu: null, maxmem: null, mem: null, uptime: null });
|
||||
const fixture = () => {
|
||||
const loaded = {
|
||||
proxmox: [{ integrationId: 1, integrationName: "Felicia", guests: [guest(100, "elsa"), guest(101, "anna"), guest(102, "zoe")], jobs: [{ id: "j", enabled: true, schedule: "02:00", storage: "pbs", node: null, allGuests: true, vmids: null, exclude: null }], tasks: [] }],
|
||||
pbs: [{ integrationName: "Backup server", groups: [{ backupType: "vm", backupId: "100", backupTime: Math.floor(Date.now() / 1000) - 3600, verification: "ok" as const }, { backupType: "vm", backupId: "101", backupTime: Math.floor(Date.now() / 1000) - 7200, verification: "ok" as const }] }],
|
||||
sources: [{ kind: "proxmox" as const, name: "Felicia", ok: true }],
|
||||
};
|
||||
return loaded;
|
||||
};
|
||||
|
||||
describe("when to speak up", () => {
|
||||
const report = () => ({ guests: buildCoverage({ ...fixture(), servers: [], now: NOW }), counts: {} as never, sources: [], staleHours: 48, generatedAt: NOW.toISOString() });
|
||||
const row = (key: string, label: string, result: string, ago: number) => [key, { id: 1, subjectKey: key, label, testedAt: daysAgo(ago), result, note: null, testedBy: null, createdAt: "" }] as const;
|
||||
|
||||
test("with no limit only a failure speaks; with one, a guest that has backups and no recent test does too", () => {
|
||||
const g = report();
|
||||
assert.deepEqual(restoreAlerts(g, new Map(), 0, NOW), [], "no limit, nothing tested: quiet");
|
||||
const limited = restoreAlerts(g, new Map(), 90, NOW);
|
||||
assert.deepEqual(limited.map((a) => [a.guest?.name, a.severity]), [["elsa", "warning"], ["anna", "warning"]], "zoe has no backups to restore from, so nothing to say");
|
||||
assert.match(limited[0].message, /elsa \(#100\) \[Felicia\]: a restore has never been tested/);
|
||||
const failed = restoreAlerts(g, new Map([row("guest:1:100", "elsa", "failed", 3)]), 0, NOW);
|
||||
assert.equal(failed.length, 1);
|
||||
assert.equal(failed[0].severity, "critical");
|
||||
assert.match(failed[0].message, /the last restore test \(.*\) failed/);
|
||||
});
|
||||
|
||||
test("a recent pass, 'not needed' and an old pass under the limit are all fine; an old one isn't", () => {
|
||||
const g = report();
|
||||
const tests = new Map([row("guest:1:100", "elsa", "ok", 10), row("guest:1:101", "anna", "not_needed", 500)]);
|
||||
assert.deepEqual(restoreAlerts(g, tests, 90, NOW), []);
|
||||
const old = restoreAlerts(g, new Map([row("guest:1:100", "elsa", "ok", 120), row("guest:1:101", "anna", "ok", 1)]), 90, NOW);
|
||||
assert.equal(old.length, 1);
|
||||
assert.match(old[0].message, /last restore test was 120 days ago \(the limit is 90 days\)/);
|
||||
});
|
||||
|
||||
test("something that isn't a guest is told of when it failed, or when it is stale", () => {
|
||||
const g = report();
|
||||
const tests = new Map([row("custom:photos", "Photos share", "ok", 200), row("custom:db", "Gitea database", "failed", 5), row("custom:fine", "Fine", "ok", 3), row("guest:9:9", "gone guest", "ok", 999)]);
|
||||
const out = restoreAlerts(g, tests, 90, NOW);
|
||||
assert.deepEqual(out.map((a) => a.message.split(":")[0]).sort(), ["Gitea database", "Photos share", "anna (#101) [Felicia]", "elsa (#100) [Felicia]"].sort());
|
||||
assert.equal(restoreAlerts(g, tests, 0, NOW).filter((a) => !a.guest).length, 1, "with no limit only the failure");
|
||||
});
|
||||
});
|
||||
|
||||
describe("through the routes, the page, the alerts and the figures", () => {
|
||||
let app: Server;
|
||||
let base = "";
|
||||
let who = "operator";
|
||||
|
||||
before(async () => {
|
||||
await db.insert(users).values([
|
||||
{ oidcSub: "admin", name: "Ada", role: "admin" },
|
||||
{ oidcSub: "operator", name: "Olga", role: "operator" },
|
||||
{ oidcSub: "viewer", name: "Vic", role: "viewer" },
|
||||
]);
|
||||
setCoverageLoaderForTests(async () => fixture());
|
||||
const a = express();
|
||||
a.use(express.json());
|
||||
a.use((req, _res, next) => {
|
||||
(req as unknown as { session: unknown }).session = { user: { sub: who } };
|
||||
next();
|
||||
});
|
||||
a.use("/api/backups", backupsRouter);
|
||||
app = await new Promise<Server>((resolve) => {
|
||||
const s = a.listen(0, "127.0.0.1", () => resolve(s));
|
||||
});
|
||||
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/backups`;
|
||||
});
|
||||
|
||||
after(() => {
|
||||
setCoverageLoaderForTests(null);
|
||||
app.close();
|
||||
try {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Windows keeps the database file locked until the process exits.
|
||||
}
|
||||
});
|
||||
|
||||
const send = (method: string, path: string, body?: unknown) => fetch(`${base}${path}`, { method, headers: { "content-type": "application/json" }, body: body === undefined ? undefined : JSON.stringify(body) });
|
||||
const today = new Date().toISOString().slice(0, 10);
|
||||
|
||||
test("an operator records a test; a viewer can't; a wrong one is refused", async () => {
|
||||
assert.equal((await send("POST", "/restore-tests", { key: "guest:1:100", label: "elsa", result: "ok", note: "booted and the data was there" })).status, 201);
|
||||
who = "viewer";
|
||||
assert.equal((await send("POST", "/restore-tests", { key: "guest:1:101", label: "anna", result: "ok" })).status, 403);
|
||||
who = "operator";
|
||||
for (const bad of [
|
||||
{ key: "guest:1:101", label: "anna", result: "excellent" },
|
||||
{ key: "guest:one:two", label: "anna", result: "ok" },
|
||||
{ key: "../etc", label: "anna", result: "ok" },
|
||||
{ key: "guest:1:101", label: "anna", result: "ok", testedAt: "2999-01-01" },
|
||||
{ key: "guest:1:101", label: "anna", result: "ok", testedAt: "2026-02-30" },
|
||||
{ label: "!!!", result: "ok" },
|
||||
{ key: "guest:1:101", label: "", result: "ok" },
|
||||
]) assert.equal((await send("POST", "/restore-tests", bad)).status, 400, JSON.stringify(bad));
|
||||
assert.equal((await db.select().from(auditLog)).filter((e) => e.category === "restore_test" && e.action === "record").length, 1);
|
||||
});
|
||||
|
||||
test("the Backups page's data says how each guest's restore went, or that nobody has tried", async () => {
|
||||
const cov = await (await send("GET", "/coverage")).json();
|
||||
const by = Object.fromEntries(cov.guests.map((g: { name: string }) => [g.name, g]));
|
||||
assert.equal(by.elsa.restoreKey, "guest:1:100");
|
||||
assert.deepEqual([by.elsa.restoreTest.status, by.elsa.restoreTest.testedAt, by.elsa.restoreTest.note], ["ok", today, "booted and the data was there"]);
|
||||
assert.equal(by.anna.restoreTest.status, "never");
|
||||
assert.equal(cov.restoreTestDays, 0);
|
||||
});
|
||||
|
||||
test("something that isn't a guest gets a key from its name, and is listed with how it stands", async () => {
|
||||
const res = await send("POST", "/restore-tests", { label: "Photos share", result: "ok", testedAt: daysAgo(40) });
|
||||
assert.equal(res.status, 201);
|
||||
assert.equal((await res.json()).test.subjectKey, "custom:photos-share");
|
||||
const list = await (await send("GET", "/restore-tests")).json();
|
||||
assert.deepEqual(list.custom.map((c: { key: string; status: string }) => [c.key, c.status]), [["custom:photos-share", "ok"]]);
|
||||
assert.equal(list.tests.length, 2);
|
||||
});
|
||||
|
||||
test("a failed test is an alert straight away; with a limit, never-tested and old ones are too; marking 'not needed' ends it", async () => {
|
||||
await send("POST", "/restore-tests", { key: "guest:1:101", label: "anna", result: "failed", note: "the disk wouldn't attach" });
|
||||
invalidateAlertsCache();
|
||||
let alerts = (await getAlerts(true)).alerts.filter((a) => a.id.startsWith("backup:restore-test:"));
|
||||
assert.deepEqual(alerts.map((a) => [a.id, a.severity]), [["backup:restore-test:guest:1:101", "critical"]]);
|
||||
assert.match(alerts[0].message, /anna \(#101\).*failed — the disk wouldn't attach/);
|
||||
assert.equal(alerts[0].link, "/backups");
|
||||
|
||||
await updateSettings({ healthChecks: { restoreTestDays: 30 } });
|
||||
invalidateAlertsCache();
|
||||
alerts = (await getAlerts(true)).alerts.filter((a) => a.id.startsWith("backup:restore-test:"));
|
||||
assert.deepEqual(alerts.map((a) => a.id).sort(), ["backup:restore-test:custom:photos-share", "backup:restore-test:guest:1:101"], "elsa is fine; the photos share was last tested 40 days ago");
|
||||
|
||||
await send("POST", "/restore-tests", { key: "guest:1:101", label: "anna", result: "not_needed", note: "a scratch machine" });
|
||||
invalidateAlertsCache();
|
||||
alerts = (await getAlerts(true)).alerts.filter((a) => a.id.startsWith("backup:restore-test:"));
|
||||
assert.deepEqual(alerts.map((a) => a.id), ["backup:restore-test:custom:photos-share"]);
|
||||
await updateSettings({ healthChecks: { restoreTestDays: 0 } });
|
||||
});
|
||||
|
||||
test("metrics say how old each test is and whether it passed", async () => {
|
||||
const text = await buildMetrics();
|
||||
assert.match(text, /homelab_restore_test_age_days\{subject="elsa"\} 0/);
|
||||
assert.match(text, /homelab_restore_test_age_days\{subject="Photos share"\} 40/);
|
||||
assert.match(text, /homelab_restore_test_passed\{subject="elsa"\} 1/);
|
||||
assert.doesNotMatch(text, /subject="anna"/, "'not needed' isn't measured");
|
||||
});
|
||||
|
||||
test("only an administrator takes an entry out, and that is recorded", async () => {
|
||||
const [first] = await db.select().from(restoreTests);
|
||||
assert.equal((await send("DELETE", `/restore-tests/${first.id}`)).status, 403);
|
||||
who = "admin";
|
||||
assert.equal((await send("DELETE", `/restore-tests/${first.id}`)).status, 204);
|
||||
assert.equal((await send("DELETE", `/restore-tests/${first.id}`)).status, 404);
|
||||
assert.equal((await db.select().from(auditLog)).filter((e) => e.category === "restore_test" && e.action === "delete").length, 1);
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user