Dependencies beyond servers, web push, inventory auto-fill with QR labels, Windows agent health

- Dependencies: servers and integrations can depend on each other; alerts are folded under the root cause, and a failure explained by something down isn't announced
- Web push: hand-written VAPID and RFC 8291 encryption, per-device subscriptions, a fifth notification channel, service worker handlers
- Inventory: maker/model/serial from the agents fill blank fields, "Refresh from agent", QR label printing and ?asset= deep links
- Windows agent (v4) reports failed services, disk health, Storage Spaces pools and hardware identity; Linux agent (v5) reports hardware identity

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-18 18:53:30 +02:00
1 parent 5cc51c57de
commit ec9c2f73f0
55 files changed
+12553 -65

No files matched your search

+3
View File
@@ -0,0 +1,3 @@
ALTER TABLE `servers` ADD `hw_manufacturer` text;--> statement-breakpoint
ALTER TABLE `servers` ADD `hw_model` text;--> statement-breakpoint
ALTER TABLE `servers` ADD `hw_serial` text;
@@ -0,0 +1,10 @@
CREATE TABLE `dependencies` (
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
`from_type` text NOT NULL,
`from_id` integer NOT NULL,
`to_type` text NOT NULL,
`to_id` integer NOT NULL,
`created_at` text DEFAULT (current_timestamp) NOT NULL
);
--> statement-breakpoint
CREATE UNIQUE INDEX `dependencies_edge_idx` ON `dependencies` (`from_type`,`from_id`,`to_type`,`to_id`);
@@ -0,0 +1,13 @@
CREATE TABLE `push_subscriptions` (
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
`user_id` integer NOT NULL,
`endpoint` text NOT NULL,
`p256dh` text NOT NULL,
`auth` text NOT NULL,
`label` text,
`created_at` text DEFAULT (current_timestamp) NOT NULL,
`last_success_at` text,
FOREIGN KEY (`user_id`) REFERENCES `users`(`id`) ON UPDATE no action ON DELETE cascade
);
--> statement-breakpoint
CREATE UNIQUE INDEX `push_subscriptions_endpoint_unique` ON `push_subscriptions` (`endpoint`);
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
+21
View File
@@ -239,6 +239,27 @@
"when": 1792329463659,
"tag": "0033_bright_princess_powerful",
"breakpoints": true
},
{
"idx": 34,
"version": "6",
"when": 1792340718369,
"tag": "0034_lethal_mandarin",
"breakpoints": true
},
{
"idx": 35,
"version": "6",
"when": 1792341079438,
"tag": "0035_adorable_silhouette",
"breakpoints": true
},
{
"idx": 36,
"version": "6",
"when": 1792341456680,
"tag": "0036_cheerful_slipstream",
"breakpoints": true
}
]
}
+4
View File
@@ -15,6 +15,8 @@ import { updateRunsRouter } from "./routes/updateRuns.js";
import { dbBackupsRouter } from "./routes/dbBackups.js";
import { incidentsRouter } from "./routes/incidents.js";
import { inventoryRouter } from "./routes/inventory.js";
import { dependenciesRouter } from "./routes/dependencies.js";
import { pushRouter } from "./routes/push.js";
import { backupsRouter } from "./routes/backups.js";
import { metricsRouter, metricsTokensRouter } from "./routes/metrics.js";
import { eventWebhooksRouter } from "./routes/eventWebhooks.js";
@@ -106,6 +108,8 @@ export function createApp() {
app.use("/api/db-backups", dbBackupsRouter);
app.use("/api/incidents", incidentsRouter);
app.use("/api/inventory", inventoryRouter);
app.use("/api/dependencies", dependenciesRouter);
app.use("/api/push", pushRouter);
app.use("/api/backups", backupsRouter);
app.use("/api/metrics-tokens", metricsTokensRouter);
app.use("/api/event-webhooks", eventWebhooksRouter);
+40
View File
@@ -103,6 +103,41 @@ export const maintenanceAliases = sqliteTable(
(table) => [uniqueIndex("maintenance_aliases_unique_idx").on(table.address, table.targetType, table.targetId)],
);
// A browser or phone that asked for push notifications (one row per device). The endpoint is the browser's push service address, which is
// itself a secret: whoever has it with the keys can write to that device. Removed when the push service says it is gone, or the user signs out of it.
export const pushSubscriptions = sqliteTable("push_subscriptions", {
id: integer("id").primaryKey({ autoIncrement: true }),
userId: integer("user_id")
.notNull()
.references(() => users.id, { onDelete: "cascade" }),
endpoint: text("endpoint").notNull().unique(),
p256dh: text("p256dh").notNull(),
auth: text("auth").notNull(),
/** What the person calls it, or the browser it is. */
label: text("label"),
createdAt: text("created_at")
.notNull()
.default(sql`(current_timestamp)`),
lastSuccessAt: text("last_success_at"),
});
// "A needs B", for things that can be down: a server or an integration each. Polymorphic like maintenance_windows (no foreign key); an edge whose
// end was deleted is ignored. When B is down, A's alert is folded under B's — see services/dependencies.ts.
export const dependencies = sqliteTable(
"dependencies",
{
id: integer("id").primaryKey({ autoIncrement: true }),
fromType: text("from_type").notNull(), // server | integration
fromId: integer("from_id").notNull(),
toType: text("to_type").notNull(),
toId: integer("to_id").notNull(),
createdAt: text("created_at")
.notNull()
.default(sql`(current_timestamp)`),
},
(table) => [uniqueIndex("dependencies_edge_idx").on(table.fromType, table.fromId, table.toType, table.toId)],
);
// A short history of each agent-reported server's load, memory and disk use, in percent — see services/serverMetrics.ts.
// `metric` is "cpu", "mem" or "disk:<mount point>".
export const serverMetrics = sqliteTable(
@@ -300,6 +335,11 @@ export const servers = sqliteTable("servers", {
// What the agent says about the machine's own health, as JSON: {failedUnits, smart, zfs, reportedAt} — see services/hostHealth.ts. Null until it says.
hostHealth: text("host_health"),
// What the machine is, as its firmware says (agents report it): maker, model and serial number. Used to fill in the inventory.
hwManufacturer: text("hw_manufacturer"),
hwModel: text("hw_model"),
hwSerial: text("hw_serial"),
// This server's own alert thresholds, as JSON: {serverOfflineMinutes, diskUsagePercent, diskForecastDays, backupStaleHours, mounts: {"/data": {percent, ignore}}} — see
// services/serverThresholds.ts. Whatever isn't there follows Settings. Null when it follows Settings entirely.
thresholds: text("thresholds"),
+14
View File
@@ -52,7 +52,21 @@ const healthSchema = z.object({
.catch(undefined),
});
// What the machine is. Each part is optional and a malformed one is dropped, like the rest.
const hardwareSchema = z.object({
manufacturer: text(100).optional().catch(undefined),
model: text(150).optional().catch(undefined),
serial: text(100).optional().catch(undefined),
});
const systemSchema = z.object({
hardware: z
.unknown()
.optional()
.transform((h) => {
const parsed = hardwareSchema.safeParse(h);
return parsed.success && Object.values(parsed.data).some((v) => typeof v === "string" && v.trim()) ? parsed.data : undefined;
}),
health: z
.unknown()
.optional()
+80
View File
@@ -0,0 +1,80 @@
import { Router } from "express";
import { eq } from "drizzle-orm";
import { z } from "zod";
import { db } from "../db/client.js";
import { dependencies } from "../db/schema.js";
import { requireAuth, requireRole } from "../auth/middleware.js";
import { recordAudit } from "../services/audit.js";
import { invalidateAlertsCache } from "../services/alerts.js";
import { loadGraph, nodeKey, nodeTypes, wouldLoop } from "../services/dependencies.js";
import { asyncHandler } from "../utils/asyncHandler.js";
export const dependenciesRouter = Router();
dependenciesRouter.use(requireAuth);
const edgeSchema = z.object({
fromType: z.enum(nodeTypes),
fromId: z.number().int().positive(),
toType: z.enum(nodeTypes),
toId: z.number().int().positive(),
});
// Everything signed in people may see: what needs what, with the "runs on" links of servers alongside (those are set on the server itself).
dependenciesRouter.get("/", asyncHandler(async (_req, res) => {
const graph = await loadGraph();
const written = new Set(graph.rows.map((r) => `${nodeKey(r.fromType, r.fromId)}>${nodeKey(r.toType, r.toId)}`));
const runsOn: { from: string; to: string }[] = [];
for (const [from, tos] of graph.edges) for (const to of tos) if (!written.has(`${from}>${to}`)) runsOn.push({ from, to });
const nodes = [...graph.names.entries()]
.map(([key, name]) => {
const [type, id] = key.split(":");
return { key, type, id: Number(id), name };
})
.sort((a, b) => a.type.localeCompare(b.type) || a.name.localeCompare(b.name));
res.json({
dependencies: graph.rows
.filter((r) => graph.names.has(nodeKey(r.fromType, r.fromId)) && graph.names.has(nodeKey(r.toType, r.toId)))
.map((r) => ({ id: r.id, from: { type: r.fromType, id: r.fromId, name: graph.names.get(nodeKey(r.fromType, r.fromId))! }, to: { type: r.toType, id: r.toId, name: graph.names.get(nodeKey(r.toType, r.toId))! }, createdAt: r.createdAt })),
runsOn: runsOn.map((r) => ({ from: { key: r.from, name: graph.names.get(r.from) ?? r.from }, to: { key: r.to, name: graph.names.get(r.to) ?? r.to } })),
nodes,
});
}));
dependenciesRouter.post("/", requireRole("operator"), asyncHandler(async (req, res) => {
const parsed = edgeSchema.safeParse(req.body);
if (!parsed.success) return res.status(400).json({ error: "invalid_body", message: "Say what needs what: a server or an integration each.", details: parsed.error.flatten() });
const d = parsed.data;
const graph = await loadGraph();
const from = nodeKey(d.fromType, d.fromId);
const to = nodeKey(d.toType, d.toId);
if (!graph.names.has(from) || !graph.names.has(to)) return res.status(400).json({ error: "unknown_node", message: "One of those doesn't exist." });
if (from === to) return res.status(400).json({ error: "self", message: "Something can't depend on itself." });
if (graph.edges.get(from)?.includes(to)) return res.status(409).json({ error: "duplicate", message: "That is already so." });
if (wouldLoop(graph.edges, from, to)) {
return res.status(409).json({ error: "loop", message: `${graph.names.get(to)} already depends on ${graph.names.get(from)}, so this would make them wait for each other.` });
}
const [row] = await db.insert(dependencies).values(d).returning();
await recordAudit({ actor: req.currentUser!, category: "dependency", action: "create", targetType: "dependency", targetId: row.id, detail: { from: graph.names.get(from), to: graph.names.get(to) } });
invalidateAlertsCache();
res.status(201).json({ id: row.id });
}));
dependenciesRouter.delete("/:id", requireRole("operator"), asyncHandler(async (req, res) => {
const id = /^\d+$/.test(req.params.id) ? Number(req.params.id) : NaN;
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
const [before] = await db.select().from(dependencies).where(eq(dependencies.id, id)).limit(1);
if (!before) return res.status(404).json({ error: "not_found" });
const graph = await loadGraph();
await db.delete(dependencies).where(eq(dependencies.id, id));
await recordAudit({
actor: req.currentUser!,
category: "dependency",
action: "delete",
targetType: "dependency",
targetId: id,
detail: { from: graph.names.get(nodeKey(before.fromType as "server", before.fromId)) ?? `${before.fromType} #${before.fromId}`, to: graph.names.get(nodeKey(before.toType as "server", before.toId)) ?? `${before.toType} #${before.toId}` },
});
invalidateAlertsCache();
res.status(204).end();
}));
+42 -8
View File
@@ -6,6 +6,7 @@ import { assetKinds, assets, servers } from "../db/schema.js";
import { requireAuth, requireRole } from "../auth/middleware.js";
import { recordAudit } from "../services/audit.js";
import { isRealDate } from "../services/inventory.js";
import { blanksToFill, differences, hardwareOf } from "../services/inventoryAutofill.js";
import { getSettings, updateSettings } from "../services/settingsStore.js";
import { asyncHandler } from "../utils/asyncHandler.js";
@@ -53,16 +54,32 @@ function values(d: z.infer<typeof assetSchema>) {
};
}
/** A linked entry's blank maker, model and serial, from what that server's agent says. */
async function fillFromAgent(v: ReturnType<typeof values>) {
if (v.serverId === null) return {};
const [server] = await db.select({ hwManufacturer: servers.hwManufacturer, hwModel: servers.hwModel, hwSerial: servers.hwSerial }).from(servers).where(eq(servers.id, v.serverId)).limit(1);
return blanksToFill(v, hardwareOf(server));
}
const idOf = (v: string) => (/^\d+$/.test(v) ? Number(v) : NaN);
// Everything signed-in people may see: the hardware, and the price of power the running cost is worked out with.
inventoryRouter.get("/", asyncHandler(async (_req, res) => {
const [rows, serverRows, settings] = await Promise.all([db.select().from(assets).orderBy(assets.name), db.select({ id: servers.id, name: servers.name }).from(servers), getSettings()]);
const serverName = new Map(serverRows.map((s) => [s.id, s.name]));
const [rows, serverRows, settings] = await Promise.all([db.select().from(assets).orderBy(assets.name), db.select().from(servers), getSettings()]);
const byId = new Map(serverRows.map((s) => [s.id, s]));
res.json({
assets: rows.map((a) => ({ ...a, server: a.serverId !== null && serverName.has(a.serverId) ? { id: a.serverId, name: serverName.get(a.serverId)! } : null })),
assets: rows.map((a) => {
const s = a.serverId !== null ? byId.get(a.serverId) : undefined;
const hw = hardwareOf(s);
return {
...a,
server: s ? { id: s.id, name: s.name } : null,
// What the machine's agent says it is, and what that would change here (for the "refresh from agent" button).
agent: hw ? { ...hw, differs: differences(a, hw) } : null,
};
}),
config: settings.inventory,
servers: serverRows.sort((a, b) => a.name.localeCompare(b.name)),
servers: serverRows.map((s) => ({ id: s.id, name: s.name })).sort((a, b) => a.name.localeCompare(b.name)),
});
}));
@@ -89,8 +106,9 @@ inventoryRouter.post("/", requireRole("operator"), asyncHandler(async (req, res)
return res.status(400).json({ error: "unknown_server", message: "That server doesn't exist." });
}
const now = new Date().toISOString();
const [row] = await db.insert(assets).values({ ...v, createdAt: now, updatedAt: now }).returning();
await recordAudit({ actor: req.currentUser!, category: "inventory", action: "create", targetType: "asset", targetId: row.id, detail: { name: row.name, kind: row.kind } });
const filled = await fillFromAgent(v);
const [row] = await db.insert(assets).values({ ...v, ...filled, createdAt: now, updatedAt: now }).returning();
await recordAudit({ actor: req.currentUser!, category: "inventory", action: "create", targetType: "asset", targetId: row.id, detail: { name: row.name, kind: row.kind, ...(Object.keys(filled).length ? { filledFromAgent: Object.keys(filled) } : {}) } });
res.status(201).json({ asset: row });
}));
@@ -104,11 +122,27 @@ inventoryRouter.put("/:id", requireRole("operator"), asyncHandler(async (req, re
if (v.serverId !== null && (await db.select({ id: servers.id }).from(servers).where(eq(servers.id, v.serverId))).length === 0) {
return res.status(400).json({ error: "unknown_server", message: "That server doesn't exist." });
}
const [row] = await db.update(assets).set({ ...v, updatedAt: new Date().toISOString() }).where(eq(assets.id, id)).returning();
await recordAudit({ actor: req.currentUser!, category: "inventory", action: "update", targetType: "asset", targetId: id, detail: { name: row.name } });
const filled = await fillFromAgent(v);
const [row] = await db.update(assets).set({ ...v, ...filled, updatedAt: new Date().toISOString() }).where(eq(assets.id, id)).returning();
await recordAudit({ actor: req.currentUser!, category: "inventory", action: "update", targetType: "asset", targetId: id, detail: { name: row.name, ...(Object.keys(filled).length ? { filledFromAgent: Object.keys(filled) } : {}) } });
res.json({ asset: row });
}));
// Takes what the agent says over what is written here: the one deliberate overwrite (the entry's own fields are otherwise only ever filled when blank).
inventoryRouter.post("/:id/refresh-from-agent", requireRole("operator"), asyncHandler(async (req, res) => {
const id = idOf(req.params.id);
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
const [before] = await db.select().from(assets).where(eq(assets.id, id)).limit(1);
if (!before) return res.status(404).json({ error: "not_found" });
if (before.serverId === null) return res.status(400).json({ error: "not_linked", message: "Link the entry to a server first." });
const [server] = await db.select({ hwManufacturer: servers.hwManufacturer, hwModel: servers.hwModel, hwSerial: servers.hwSerial }).from(servers).where(eq(servers.id, before.serverId)).limit(1);
const changes = differences(before, hardwareOf(server));
if (Object.keys(changes).length === 0) return res.json({ asset: before, changed: [] });
const [row] = await db.update(assets).set({ ...changes, updatedAt: new Date().toISOString() }).where(eq(assets.id, id)).returning();
await recordAudit({ actor: req.currentUser!, category: "inventory", action: "refresh_from_agent", targetType: "asset", targetId: id, detail: { name: row.name, from: Object.fromEntries(Object.keys(changes).map((k) => [k, (before as any)[k]])), to: changes } });
res.json({ asset: row, changed: Object.keys(changes) });
}));
inventoryRouter.delete("/:id", requireRole("operator"), asyncHandler(async (req, res) => {
const id = idOf(req.params.id);
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
+1
View File
@@ -63,6 +63,7 @@ privacyRouter.get("/", asyncHandler(async (req, res) => {
ntfy: channel(settings.ntfy.enabled, hostOf(settings.ntfy.url)),
smtp: channel(settings.smtp.enabled, settings.smtp.host || null),
webhook: channel(settings.webhook.enabled, hostOf(settings.webhook.url)),
webPush: channel(settings.webPush.enabled, "the push services of your browsers (Google, Mozilla, Apple …)"),
},
},
});
+97
View File
@@ -0,0 +1,97 @@
import { Router } from "express";
import { and, eq } from "drizzle-orm";
import { z } from "zod";
import { db } from "../db/client.js";
import { pushSubscriptions } from "../db/schema.js";
import { requireAuth } from "../auth/middleware.js";
import { recordAudit } from "../services/audit.js";
import { getSettings } from "../services/settingsStore.js";
import { checkEndpoint, getVapidKeys, sendPush } from "../services/webPush.js";
import { asyncHandler } from "../utils/asyncHandler.js";
export const pushRouter = Router();
pushRouter.use(requireAuth);
/** At most this many browsers per person: a phone, a laptop, a tablet and the odd one. */
const MAX_PER_USER = 10;
const subscribeSchema = z.object({
endpoint: z.string().url().max(2000),
keys: z.object({ p256dh: z.string().min(80).max(100), auth: z.string().min(20).max(30) }),
label: z.string().trim().max(80).optional(),
});
const hostOf = (endpoint: string) => {
try {
return new URL(endpoint).host;
} catch {
return "";
}
};
const off = { error: "push_disabled", message: "Push notifications are switched off. An administrator turns them on under Settings, Notifications." };
// The key a browser subscribes with. Anyone signed in may ask (their own device is theirs to enrol), once push is on.
pushRouter.get("/key", asyncHandler(async (_req, res) => {
const { webPush } = await getSettings();
if (!webPush.enabled) return res.status(409).json(off);
res.json({ publicKey: (await getVapidKeys()).publicKey });
}));
// This person's own browsers.
pushRouter.get("/mine", asyncHandler(async (req, res) => {
const rows = await db.select().from(pushSubscriptions).where(eq(pushSubscriptions.userId, req.currentUser!.id)).orderBy(pushSubscriptions.id);
const { webPush } = await getSettings();
res.json({
enabled: webPush.enabled,
subscriptions: rows.map((r) => ({ id: r.id, label: r.label, host: hostOf(r.endpoint), endpoint: r.endpoint, createdAt: r.createdAt, lastSuccessAt: r.lastSuccessAt })),
});
}));
pushRouter.post("/subscribe", asyncHandler(async (req, res) => {
const { webPush } = await getSettings();
if (!webPush.enabled) return res.status(409).json(off);
const parsed = subscribeSchema.safeParse(req.body);
if (!parsed.success) return res.status(400).json({ error: "invalid_body", message: "That isn't a push subscription.", details: parsed.error.flatten() });
const { endpoint, keys, label } = parsed.data;
try {
await checkEndpoint(endpoint);
} catch (err) {
return res.status(400).json({ error: "bad_endpoint", message: err instanceof Error ? err.message : "That push address can't be used." });
}
const me = req.currentUser!;
const mine = await db.select({ id: pushSubscriptions.id, endpoint: pushSubscriptions.endpoint }).from(pushSubscriptions).where(eq(pushSubscriptions.userId, me.id));
const existing = mine.find((r) => r.endpoint === endpoint);
if (!existing && mine.length >= MAX_PER_USER) return res.status(409).json({ error: "too_many", message: `You already have ${MAX_PER_USER} browsers subscribed. Remove one first.` });
// The same browser subscribing again (or after someone else used it) is the same row, now this person's.
const [row] = await db
.insert(pushSubscriptions)
.values({ userId: me.id, endpoint, p256dh: keys.p256dh, auth: keys.auth, label: label || null })
.onConflictDoUpdate({ target: pushSubscriptions.endpoint, set: { userId: me.id, p256dh: keys.p256dh, auth: keys.auth, label: label || null } })
.returning({ id: pushSubscriptions.id });
if (!existing) await recordAudit({ actor: me, category: "push", action: "subscribe", targetType: "push_subscription", targetId: row.id, detail: { host: hostOf(endpoint), label: label || null } });
res.status(201).json({ id: row.id });
}));
// Stops a browser being written to. A person removes their own; an administrator may remove anyone's.
pushRouter.delete("/:id", asyncHandler(async (req, res) => {
const id = /^\d+$/.test(req.params.id) ? Number(req.params.id) : NaN;
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
const me = req.currentUser!;
const [row] = await db.select().from(pushSubscriptions).where(eq(pushSubscriptions.id, id)).limit(1);
if (!row || (row.userId !== me.id && me.role !== "admin")) return res.status(404).json({ error: "not_found" });
await db.delete(pushSubscriptions).where(and(eq(pushSubscriptions.id, id)));
await recordAudit({ actor: me, category: "push", action: "unsubscribe", targetType: "push_subscription", targetId: id, detail: { host: hostOf(row.endpoint), label: row.label } });
res.status(204).end();
}));
// A message to this person's own browsers, to see that it works.
pushRouter.post("/test", asyncHandler(async (req, res) => {
const { webPush } = await getSettings();
if (!webPush.enabled) return res.status(409).json(off);
const mine = await db.select().from(pushSubscriptions).where(eq(pushSubscriptions.userId, req.currentUser!.id));
if (mine.length === 0) return res.status(400).json({ error: "none", message: "This browser isn't subscribed yet." });
const outcome = await sendPush({ title: "Homelab Manager — Test", body: "Push notifications are working.", url: "/settings/notifications", tag: "test" }, mine);
res.json(outcome);
}));
+1
View File
@@ -64,6 +64,7 @@ const updateSchema = z.object({
.partial()
.optional(),
webhook: z.object({ enabled: z.boolean(), url: z.string(), secret: z.string() }).partial().optional(),
webPush: z.object({ enabled: z.boolean(), contact: z.string().trim().max(200) }).partial().optional(),
notifications: z
.object({
dnsAdd: z.boolean(),
+3 -1
View File
@@ -8,7 +8,9 @@
* 2 — reports pending updates, and its own version
* 3 — Linux: the package manager is asked outside the systemd sandbox, so dnf and zypper can answer too.
* Windows: can replace itself when an administrator has switched automatic updates on.
* 4 — Linux: also reports failed services, disk health (SMART) and ZFS pools, reads pacman, and can replace itself too
* 4 — Linux: also reports failed services, disk health (SMART) and ZFS pools, reads pacman, and can replace itself too.
* Windows: also reports failed services, disk health and Storage Spaces pools, and what the machine is (maker, model, serial)
* 5 — Linux: also says what the machine is (maker, model, serial), for the inventory
*/
import { createHash } from "node:crypto";
import { readFileSync } from "node:fs";
+20 -9
View File
@@ -33,7 +33,7 @@ import { computeSecretStatus } from "./secretStatus.js";
import { getFailingSources } from "./integrationHealthMonitor.js";
import { getSettings } from "./settingsStore.js";
import { sourceLabel } from "./notify.js";
import { coveredByParent, loadParentRows, resolveParents } from "./serverParents.js";
import { dependencyCover, integrationTypeCause, takenByType } from "./dependencies.js";
import type { Alert, AlertCategory, AlertSeverity, SourceFailure } from "./alertTypes.js";
export interface AlertsReport {
@@ -158,6 +158,10 @@ export async function collectAlerts(): Promise<AlertsReport> {
}
}
// The servers found to be offline, for the integration failures that one may explain: the server check fills it in (and it is empty if that check fails).
let offlineKnown!: (ids: Set<number>) => void;
const offlineServers = new Promise<Set<number>>((resolve) => (offlineKnown = resolve));
await Promise.all([
check("Server and storage health", async () => {
const { healthChecks } = await getSettings();
@@ -176,17 +180,17 @@ export async function collectAlerts(): Promise<AlertsReport> {
const m = /^offline:server:(\d+)$/.exec(c.key);
if (m) offlineIds.add(Number(m[1]));
}
const cause = coveredByParent(offlineIds, resolveParents(await loadParentRows()));
const taken = new Map<number, string[]>();
for (const [child, parent] of cause) taken.set(parent, [...(taken.get(parent) ?? []), serverNames.get(child) ?? `#${child}`]);
// (Or because of a service it needs that is failing, or a server it needs that needs one.)
offlineKnown(offlineIds);
const cover = await dependencyCover(offlineIds, getFailingSources().map((f) => f.source));
for (const c of conditions) {
const origin = originOfConditionKey(c.key);
const subject = subjectOfConditionKey(c.key);
const offlineId = /^offline:server:(\d+)$/.exec(c.key)?.[1];
const parentOf = offlineId ? cause.get(Number(offlineId)) : undefined;
const parentOf = offlineId ? cover.cause.get(`server:${offlineId}`) : undefined;
push({
causedBy: parentOf !== undefined ? { name: serverNames.get(parentOf) ?? `#${parentOf}` } : undefined,
affects: offlineId ? taken.get(Number(offlineId))?.sort((a, b) => a.localeCompare(b)) : undefined,
causedBy: parentOf !== undefined ? { name: cover.graph.names.get(parentOf) ?? parentOf } : undefined,
affects: offlineId ? cover.taken.get(`server:${offlineId}`)?.sort((a, b) => a.localeCompare(b)) : undefined,
category: origin.category,
severity: c.severity ?? "warning",
type: origin.type,
@@ -203,7 +207,7 @@ export async function collectAlerts(): Promise<AlertsReport> {
const row = intById.get(Number(m[2]));
cannotCheck(`${SOURCES[m[1]].label} “${row?.name ?? `#${m[2]}`}”${m[3] ? ` (node ${m[3]})` : ""}`, "couldn't be read — see the Diagnostic Log", "Server and storage health");
}
}),
}).finally(() => offlineKnown(new Set())),
check("Automation runs", async () => {
const { items, held } = await collectAutomation();
@@ -447,8 +451,15 @@ export async function collectAlerts(): Promise<AlertsReport> {
check("Integration failures", async () => {
const { notifications } = await getSettings();
for (const f of getFailingSources()) {
const failing = getFailingSources();
// A failure explained by something it needs that is down is folded under that, like a guest under its host.
const cover = failing.length > 0 ? await dependencyCover(await offlineServers, failing.map((f) => f.source)).catch(() => null) : null;
for (const f of failing) {
const cause = cover ? integrationTypeCause(f.source, cover) : null;
const affects = cover ? takenByType(f.source, cover) : [];
push({
causedBy: cause ? { name: cause.name } : undefined,
affects: affects.length > 0 ? affects.sort((a, b) => a.localeCompare(b)) : undefined,
category: "integration",
severity: f.alerted ? "critical" : "warning",
type: f.source,
+192
View File
@@ -0,0 +1,192 @@
/**
* What depends on what, beyond "this server runs on that server": a service that needs a server (Gitea on the box it is installed on), a
* server that needs a service (a guest that only exists while Proxmox answers). When the thing depended on is down, the things that need it
* are down too — and only the one that matters is worth an alert, so the rest are folded under it: shown, not announced.
*
* The graph is the "runs on" links of servers (set by hand or worked out from Proxmox, see serverParents.ts) plus the dependencies written
* here. Servers and integrations are the things that can be down and be needed. (DNS providers raise no alerts of their own, so there is
* nothing to fold.) An integration is judged per kind of service, as its health is: it is folded only if every enabled instance of that
* kind is explained.
*/
import { eq } from "drizzle-orm";
import { db } from "../db/client.js";
import { dependencies, integrations, servers } from "../db/schema.js";
import { getInternalFlag } from "./settingsStore.js";
import { loadParentRows, resolveParents, type ParentLink } from "./serverParents.js";
/** Where the health monitor keeps the problems it is announcing; read here so a service's alert can tell a server it needs is down. */
export const HEALTH_STATE_FLAG = "healthActiveConditions";
export const nodeTypes = ["server", "integration"] as const;
export type NodeType = (typeof nodeTypes)[number];
export const nodeKey = (type: NodeType, id: number) => `${type}:${id}`;
export interface DependencyEdge {
fromType: NodeType;
fromId: number;
toType: NodeType;
toId: number;
}
/** node → the nodes it needs. */
export type Edges = Map<string, string[]>;
/** The graph from the "runs on" links and the written dependencies; an edge to or from something that no longer exists is left out. */
export function buildEdges(parentLinks: Map<number, ParentLink>, rows: DependencyEdge[], exists: (key: string) => boolean): Edges {
const edges: Edges = new Map();
const add = (from: string, to: string) => {
if (from === to || !exists(from) || !exists(to)) return;
const list = edges.get(from) ?? [];
if (!list.includes(to)) edges.set(from, [...list, to]);
};
for (const [child, link] of parentLinks) add(nodeKey("server", child), nodeKey("server", link.parentId));
for (const r of rows) add(nodeKey(r.fromType, r.fromId), nodeKey(r.toType, r.toId));
return edges;
}
/** Everything `node` needs, directly or not, nearest first; `looped` if it ends up needing itself. */
function ancestorsOf(node: string, edges: Edges): { order: string[]; looped: boolean } {
const order: string[] = [];
const seen = new Set<string>();
const queue = [...(edges.get(node) ?? [])];
let looped = false;
while (queue.length > 0) {
const cur = queue.shift()!;
if (cur === node) {
looped = true;
continue;
}
if (seen.has(cur)) continue;
seen.add(cur);
order.push(cur);
queue.push(...(edges.get(cur) ?? []));
}
return { order, looped };
}
/**
* Of the nodes that are down, those whose cause is something they need that is down too: each maps to the root cause — a down node that
* needs nothing that is down. A loop explains nothing (each would be waiting for the other), so its members stand alone and are announced.
*/
export function coveredNodes(down: Set<string>, edges: Edges): Map<string, string> {
const covered = new Map<string, string>();
for (const node of [...down].sort()) {
const { order, looped } = ancestorsOf(node, edges);
if (looped) continue;
const downAbove = order.filter((a) => down.has(a));
const root = downAbove.find((a) => {
const above = ancestorsOf(a, edges);
return !above.looped && !above.order.some((x) => down.has(x));
});
if (root !== undefined) covered.set(node, root);
}
return covered;
}
/** Would `from` needing `to` close a loop? True if `to` is `from` or already needs it, however indirectly. */
export function wouldLoop(edges: Edges, from: string, to: string): boolean {
return from === to || ancestorsOf(to, edges).order.includes(from) || ancestorsOf(to, edges).looped;
}
export interface Graph {
edges: Edges;
/** node → its name, for every node that exists. */
names: Map<string, string>;
rows: (DependencyEdge & { id: number; createdAt: string })[];
/** Enabled instances of each kind of integration. */
instances: Map<string, number[]>;
intRows: { id: number; type: string; enabled: boolean }[];
}
export async function loadGraph(): Promise<Graph> {
const [serverRows, intRows, depRows, parentRows] = await Promise.all([
db.select({ id: servers.id, name: servers.name }).from(servers),
db.select({ id: integrations.id, name: integrations.name, type: integrations.type, enabled: integrations.enabled }).from(integrations),
db.select().from(dependencies),
loadParentRows(),
]);
const names = new Map<string, string>();
for (const s of serverRows) names.set(nodeKey("server", s.id), s.name);
for (const i of intRows) names.set(nodeKey("integration", i.id), i.name);
const rows = depRows.map((r) => ({ id: r.id, fromType: r.fromType as NodeType, fromId: r.fromId, toType: r.toType as NodeType, toId: r.toId, createdAt: r.createdAt }));
const instances = new Map<string, number[]>();
for (const i of intRows) if (i.enabled) instances.set(i.type, [...(instances.get(i.type) ?? []), i.id]);
return { edges: buildEdges(resolveParents(parentRows), rows, (k) => names.has(k)), names, rows, instances, intRows };
}
export interface Cover {
/** Each node that is down because of another → the root cause. */
cause: Map<string, string>;
/** Each root cause → the names of what it takes down with it. */
taken: Map<string, string[]>;
graph: Graph;
}
/** Works out what is down because of what, given the servers that are offline and the kinds of integration that are failing. */
export async function dependencyCover(offlineServerIds: Iterable<number>, failingTypes: Iterable<string>): Promise<Cover> {
const graph = await loadGraph();
const down = new Set<string>();
for (const id of offlineServerIds) down.add(nodeKey("server", id));
const failing = new Set(failingTypes);
for (const [type, ids] of graph.instances) if (failing.has(type)) for (const id of ids) down.add(nodeKey("integration", id));
const cause = coveredNodes(down, graph.edges);
const taken = new Map<string, string[]>();
for (const [child, parent] of cause) taken.set(parent, [...(taken.get(parent) ?? []), graph.names.get(child) ?? child]);
return { cause, taken, graph };
}
/** The root cause behind a failing kind of integration, when every enabled instance of it is explained; null when it stands alone. */
export function integrationTypeCause(type: string, cover: Cover): { key: string; name: string } | null {
const ids = cover.graph.instances.get(type);
if (!ids || ids.length === 0) return null;
let first: string | null = null;
for (const id of ids) {
const root = cover.cause.get(nodeKey("integration", id));
if (root === undefined) return null;
first ??= root;
}
return first === null ? null : { key: first, name: cover.graph.names.get(first) ?? first };
}
/** What a root cause takes down, for a kind of integration: the folded names across all its instances. */
export function takenByType(type: string, cover: Cover): string[] {
const out: string[] = [];
for (const id of cover.graph.instances.get(type) ?? []) out.push(...(cover.taken.get(nodeKey("integration", id)) ?? []));
return out;
}
/** The servers the health monitor is announcing as offline (or holding back as explained), from what it last stored. */
export async function offlineServerIdsFromState(): Promise<number[]> {
try {
const raw = await getInternalFlag(HEALTH_STATE_FLAG);
if (!raw) return [];
const ids: number[] = [];
for (const key of Object.keys(JSON.parse(raw) as Record<string, unknown>)) {
const m = /^offline:server:(\d+)$/.exec(key);
if (m) ids.push(Number(m[1]));
}
return ids;
} catch {
return [];
}
}
/**
* Whether a kind of integration that has started failing is failing because of something it needs that is down — a server the health
* monitor says is offline, or another service that is failing. Never throws: a problem here must not swallow a real outage.
*/
export async function integrationCausedBy(type: string, failingTypes: string[]): Promise<{ name: string } | null> {
try {
const cover = await dependencyCover(await offlineServerIdsFromState(), failingTypes);
return integrationTypeCause(type, cover);
} catch (err) {
console.error("[dependencies] couldn't work out whether a failure is explained:", err);
return null;
}
}
export async function nodeExists(type: NodeType, id: number): Promise<boolean> {
if (type === "server") return (await db.select({ id: servers.id }).from(servers).where(eq(servers.id, id)).limit(1)).length > 0;
return (await db.select({ id: integrations.id }).from(integrations).where(eq(integrations.id, id)).limit(1)).length > 0;
}
+11 -8
View File
@@ -8,11 +8,12 @@ import { notifyHealthIssues, notifyHealthRecovered } from "./notify.js";
import { recordAudit } from "./audit.js";
import { getInternalFlag, getSettings, setInternalFlag } from "./settingsStore.js";
import { activeSubjects, subjectOfConditionKey } from "./maintenance.js";
import { coveredByParent, loadParentRows, resolveParents } from "./serverParents.js";
import { dependencyCover, HEALTH_STATE_FLAG } from "./dependencies.js";
import { getFailingSources } from "./integrationHealthMonitor.js";
import { hostHealthConditions, parseHostHealth, type HostHealth } from "./hostHealth.js";
import { diskLimit, parseThresholds, type ServerThresholds } from "./serverThresholds.js";
const STATE_FLAG = "healthActiveConditions";
const STATE_FLAG = HEALTH_STATE_FLAG;
/** After a restart the agents haven't had a chance to report yet (they run every 15 min), so server-derived conditions are held rather than judged. */
const STARTUP_GRACE_MS = Number(process.env.HEALTH_STARTUP_GRACE_MS ?? 20 * 60 * 1000); // the environment variable is for tests
const PROCESS_START = Date.now();
@@ -396,18 +397,20 @@ export async function runHealthCheck(now: number = Date.now()): Promise<{ added:
const m = /^offline:server:(\d+)$/.exec(c.key);
if (m) offlineIds.add(Number(m[1]));
}
const cause = coveredByParent(offlineIds, resolveParents(await loadParentRows()));
const coveredKeys = new Set([...cause.keys()].map((id) => `offline:server:${id}`));
// (Also when it needs a service that is failing, or runs on a server that needs one.)
const cover = await dependencyCover(offlineIds, getFailingSources().map((f) => f.source));
const coveredKeys = new Set<string>();
for (const child of cover.cause.keys()) {
const m = /^server:(\d+)$/.exec(child);
if (m) coveredKeys.add(`offline:server:${m[1]}`);
}
const { added: announced, resolved, next, newlyDown, ended } = diffConditions(await loadState(), current, held, await activeSubjects(new Date(now)), coveredKeys);
await setInternalFlag(STATE_FLAG, JSON.stringify(next));
// A host's message says what else it takes with it.
const names = new Map(snapshot.servers.map((s) => [s.id, s.name]));
const taken = new Map<number, string[]>();
for (const [child, parent] of cause) taken.set(parent, [...(taken.get(parent) ?? []), names.get(child) ?? `#${child}`]);
const added = announced.map((c) => {
const m = /^offline:server:(\d+)$/.exec(c.key);
const children = m ? taken.get(Number(m[1])) : undefined;
const children = m ? cover.taken.get(`server:${m[1]}`) : undefined;
return children?.length ? { ...c, message: `${c.message} — also takes down ${alsoList(children)}` } : c;
});
+2 -1
View File
@@ -18,6 +18,7 @@ export interface SmartDisk {
/** NVMe and some SSDs say how much of their rated life is used, 0–100 (and past it). */
wearPercent: number | null;
}
/** A storage pool: ZFS on Linux, Storage Spaces on Windows (which reports Healthy, Warning and Unhealthy as ONLINE, DEGRADED and FAULTED). */
export interface ZfsPool {
name: string;
/** ONLINE, DEGRADED, FAULTED, UNAVAIL, SUSPENDED … as zpool says it. */
@@ -117,7 +118,7 @@ export function hostHealthConditions(serverId: number, serverName: string, healt
}
for (const p of health.zfs ?? []) {
if (!poolIsFine(p)) {
out.push({ key: `zfs:server:${serverId}:${p.name}`, severity: "critical", message: `${serverName}: ZFS pool ${p.name} is ${p.health}` });
out.push({ key: `zfs:server:${serverId}:${p.name}`, severity: "critical", message: `${serverName}: storage pool ${p.name} is ${p.health}` });
}
}
return out;
@@ -6,6 +6,7 @@ import { ensureHistoryStart } from "./uptimeHistory.js";
import { getSettings } from "./settingsStore.js";
import { isSourceInMaintenance } from "./maintenance.js";
import { notifyIntegrationDown, notifyIntegrationRecovered } from "./notify.js";
import { integrationCausedBy } from "./dependencies.js";
interface SourceHealth {
consecutiveFailures: number;
@@ -96,7 +97,9 @@ export async function trackIntegrationHealth(source: string, ok: boolean): Promi
if (notifications.integrationFailureAlerts && !state.alerted && state.consecutiveFailures >= notifications.integrationFailureThreshold) {
// During maintenance the failures keep being counted but `alerted` stays false, so if the service is
// still failing once the window ends, the very next failed call alerts — a real outage isn't swallowed.
if (!(await isSourceInMaintenance(source))) {
// Likewise when it needs something that is down (the server it runs on, another service): that one is what gets announced, and if this stays
// down once that one is back, the next failed call alerts.
if (!(await isSourceInMaintenance(source)) && !(await integrationCausedBy(source, [...getFailingSources().map((f) => f.source), source]))) {
state.alerted = true;
await notifyIntegrationDown(source, state.consecutiveFailures);
}
+60
View File
@@ -0,0 +1,60 @@
/**
* Filling in the inventory from what an agent has found out about the machine it runs on: maker, model and serial number. Typing a serial
* number off a label is the part of an inventory nobody keeps up, so a linked entry gets it from the agent — but only into fields that are
* blank, because what a person wrote down is theirs; "Refresh from agent" is the way to overwrite on purpose.
*/
import { eq, isNull, or, and } from "drizzle-orm";
import { db } from "../db/client.js";
import { assets, servers } from "../db/schema.js";
export interface AgentHardware {
manufacturer: string | null;
model: string | null;
serial: string | null;
}
type Fields = { manufacturer: string | null; model: string | null; serial: string | null };
const KEYS = ["manufacturer", "model", "serial"] as const;
/** What the server's agent said the machine is, or null if it has said nothing. */
export function hardwareOf(server: { hwManufacturer: string | null; hwModel: string | null; hwSerial: string | null } | null | undefined): AgentHardware | null {
if (!server) return null;
const hw = { manufacturer: server.hwManufacturer ?? null, model: server.hwModel ?? null, serial: server.hwSerial ?? null };
return hw.manufacturer || hw.model || hw.serial ? hw : null;
}
/** The fields of an entry that are blank and that the agent can fill. Empty when nothing would change. */
export function blanksToFill(current: Fields, hw: AgentHardware | null): Partial<Fields> {
const out: Partial<Fields> = {};
if (!hw) return out;
for (const k of KEYS) if (!current[k]?.trim() && hw[k]) out[k] = hw[k];
return out;
}
/** The fields where the agent says something different from the entry: what a refresh would change. */
export function differences(current: Fields, hw: AgentHardware | null): Partial<Fields> {
const out: Partial<Fields> = {};
if (!hw) return out;
for (const k of KEYS) if (hw[k] && (current[k] ?? "").trim() !== hw[k]) out[k] = hw[k];
return out;
}
/** After an agent has reported, fills the blanks of the inventory entries that are this server. */
export async function fillInventoryFromServer(serverId: number): Promise<number> {
const [server] = await db.select({ hwManufacturer: servers.hwManufacturer, hwModel: servers.hwModel, hwSerial: servers.hwSerial }).from(servers).where(eq(servers.id, serverId)).limit(1);
const hw = hardwareOf(server);
if (!hw) return 0;
// Only entries with something blank are worth reading.
const linked = await db
.select()
.from(assets)
.where(and(eq(assets.serverId, serverId), or(isNull(assets.manufacturer), isNull(assets.model), isNull(assets.serial))));
let changed = 0;
for (const a of linked) {
const fill = blanksToFill(a, hw);
if (Object.keys(fill).length === 0) continue;
await db.update(assets).set({ ...fill, updatedAt: new Date().toISOString() }).where(eq(assets.id, a.id));
changed++;
}
return changed;
}
+1 -1
View File
@@ -95,7 +95,7 @@ export async function buildMetrics(now: Date = new Date()): Promise<string> {
if (d.wearPercent !== null) out.add("homelab_server_disk_wear_percent", "gauge", "How much of an SSD's rated life is used.", d.wearPercent, { server, device: d.device });
if (d.temperatureC !== null) out.add("homelab_server_disk_temperature_celsius", "gauge", "A disk's temperature.", d.temperatureC, { server, device: d.device });
}
for (const p of health?.zfs ?? []) out.add("homelab_server_zfs_pool_online", "gauge", "1 if a ZFS pool is ONLINE, 0 in any other state.", poolIsFine(p) ? 1 : 0, { server, pool: p.name });
for (const p of health?.zfs ?? []) out.add("homelab_server_zfs_pool_online", "gauge", "1 if a storage pool (ZFS, or Storage Spaces on Windows) is ONLINE, 0 in any other state.", poolIsFine(p) ? 1 : 0, { server, pool: p.name });
out.add("homelab_server_agent_outdated", "gauge", "1 if the server's agent is older than the one this installation would install now.", isAgentOutdated(s, latest) ? 1 : 0, { server });
}
// Disks on a steady climb, from each server's own history: days until full at the present rate (left out where there is no steady trend).
+14 -2
View File
@@ -2,6 +2,7 @@ import nodemailer from "nodemailer";
import { getSettings, type NotificationEvents } from "./settingsStore.js";
import { enqueueNotification, listQueuedNotifications, clearQueuedNotifications } from "./notificationQueue.js";
import { emitEvent } from "./eventWebhooks.js";
import { sendPush, subscriptionCount } from "./webPush.js";
// ─── Channel senders (best-effort — failures are logged, never thrown) ──────
@@ -72,8 +73,18 @@ async function sendWebhook(title: string, message: string) {
}
}
async function sendWebPush(title: string, message: string) {
const { webPush } = await getSettings();
if (!webPush.enabled) return;
try {
await sendPush({ title, body: message });
} catch (err) {
console.error("[notify] Web push failed:", err instanceof Error ? err.message : err);
}
}
async function sendToChannels(title: string, message: string): Promise<void> {
await Promise.all([sendGotify(title, message), sendNtfy(title, message), sendSmtp(title, message), sendWebhook(title, message)]);
await Promise.all([sendGotify(title, message), sendNtfy(title, message), sendSmtp(title, message), sendWebhook(title, message), sendWebPush(title, message)]);
}
function currentTimeInZone(timezone: string): { hh: number; mm: number } {
@@ -112,8 +123,9 @@ export async function notifyNow(title: string, message: string): Promise<void> {
/** How many channels are switched on and filled in enough to send, so a "send it now" can say when there is nowhere to send to. */
export async function enabledChannelCount(): Promise<number> {
const { gotify, ntfy, smtp, webhook } = await getSettings();
const { gotify, ntfy, smtp, webhook, webPush } = await getSettings();
return [
webPush.enabled && (await subscriptionCount()) > 0,
gotify.enabled && gotify.url && gotify.token,
ntfy.enabled && ntfy.url && ntfy.topic,
smtp.enabled && smtp.host && smtp.to && smtp.from,
+9
View File
@@ -37,6 +37,13 @@ export interface WebhookSettings {
secret: string;
}
/** Push notifications to phones and browsers that have subscribed (see services/webPush.ts). Who is subscribed is kept apart, per device. */
export interface WebPushSettings {
enabled: boolean;
/** Where a push service can reach whoever runs this: an email address or an https address (VAPID asks for one). Blank is fine. */
contact: string;
}
export interface NotificationEvents {
dnsAdd: boolean;
dnsUpdate: boolean;
@@ -189,6 +196,7 @@ export interface AppSettings {
ntfy: NtfySettings;
smtp: SmtpSettings;
webhook: WebhookSettings;
webPush: WebPushSettings;
notifications: NotificationEvents;
providerColors: ProviderColors;
integrationColors: IntegrationColors;
@@ -211,6 +219,7 @@ const DEFAULTS: AppSettings = {
ntfy: { enabled: false, url: "https://ntfy.sh", topic: "", token: "", priority: 3 },
smtp: { enabled: false, host: "", port: 587, secure: false, username: "", password: "", from: "", to: "" },
webhook: { enabled: false, url: "", secret: "" },
webPush: { enabled: false, contact: "" },
notifications: {
dnsAdd: true,
dnsUpdate: true,
+15
View File
@@ -5,6 +5,7 @@ import { recordServerMetrics } from "./serverMetrics.js";
import { nextUpdateState, type IncomingUpdates } from "./pendingUpdates.js";
import { nextHostHealth, type IncomingHostHealth } from "./hostHealth.js";
import { recordAudit } from "./audit.js";
import { fillInventoryFromServer } from "./inventoryAutofill.js";
export interface IncomingTask {
scheduleType: "cron" | "systemd_timer" | "windows_task";
@@ -27,6 +28,8 @@ export interface IncomingSystemInfo {
updates?: IncomingUpdates;
/** Failed services, disk health, ZFS pools — each part present only if the agent could find it out. */
health?: IncomingHostHealth;
/** What the machine is, from its firmware: each part present only if it says anything. */
hardware?: { manufacturer?: string; model?: string; serial?: string };
}
export interface AgentReport {
@@ -115,6 +118,9 @@ export async function syncServerTasks(serverId: number, report: AgentReport) {
// Only stored when the agent reported it, so an agent that predates this (or couldn't tell this time) doesn't wipe what it said before.
...(system?.updates && before ? nextUpdateState(before, system.updates, now) : {}),
...(system?.health && before ? { hostHealth: nextHostHealth(before.hostHealth, system.health, now) } : {}),
...(system?.hardware?.manufacturer ? { hwManufacturer: system.hardware.manufacturer } : {}),
...(system?.hardware?.model ? { hwModel: system.hardware.model } : {}),
...(system?.hardware?.serial ? { hwSerial: system.hardware.serial } : {}),
...(report.hostname ? { hostname: report.hostname } : {}),
...(report.agentVersion !== undefined ? { agentVersion: report.agentVersion } : {}),
// An agent knows what it runs on better than whoever registered the server did.
@@ -146,6 +152,15 @@ export async function syncServerTasks(serverId: number, report: AgentReport) {
}
}
// A machine whose agent has said what it is fills in the blanks of its inventory entry. Also best-effort.
if (system?.hardware) {
try {
await fillInventoryFromServer(serverId);
} catch (err) {
console.error("[inventory] couldn't fill in from the agent:", err);
}
}
// Keep a little history of the figures too. A failure here must not lose the report itself, which is already saved.
try {
await recordServerMetrics(serverId, system, new Date(now));
+245
View File
@@ -0,0 +1,245 @@
/**
* Web Push: notifications to a phone or a browser that has asked for them, with no app and no account — the browser's own push service
* (Google's, Mozilla's, Apple's) carries the message. Written against the standards directly rather than through a library:
*
* - RFC 8292 (VAPID): the server proves who it is with a short-lived signed token (ES256), so a push service knows the sender is the
* one the subscription was made for.
* - RFC 8291 + 8188 (aes128gcm): the message is encrypted for the one browser it is for; the push service only ever carries it.
*
* The server's key pair is made the first time it is needed and kept (the private half encrypted, like the other credentials).
*/
import { createCipheriv, createECDH, createPrivateKey, hkdfSync, randomBytes, sign, type KeyObject } from "node:crypto";
import { eq } from "drizzle-orm";
import { db } from "../db/client.js";
import { pushSubscriptions } from "../db/schema.js";
import { env } from "../env.js";
import { decryptSecret, encryptSecret } from "../crypto.js";
import { resolveTarget } from "../utils/networkGuard.js";
import { getInternalFlag, getSettings, setInternalFlag } from "./settingsStore.js";
const b64u = (buf: Buffer | Uint8Array) => Buffer.from(buf).toString("base64url");
const fromB64u = (s: string) => Buffer.from(s, "base64url");
// ─── The server's own key pair ─────────────────────────────────────────────
const VAPID_FLAG = "webPushVapid";
const ENCRYPTED = "enc:v1:";
export interface VapidKeys {
/** The uncompressed P-256 point, base64url — what a browser is given to subscribe with. */
publicKey: string;
/** The private scalar, base64url. */
privateKey: string;
}
let cachedKeys: Promise<VapidKeys> | null = null;
/** The key pair, made on first use and kept. */
export function getVapidKeys(): Promise<VapidKeys> {
cachedKeys ??= loadOrMakeKeys().catch((err) => {
cachedKeys = null;
throw err;
});
return cachedKeys;
}
async function loadOrMakeKeys(): Promise<VapidKeys> {
const stored = await getInternalFlag(VAPID_FLAG);
if (stored) {
try {
const v = JSON.parse(stored) as { publicKey: string; privateKey: string };
const privateKey = v.privateKey.startsWith(ENCRYPTED) ? decryptSecret(v.privateKey.slice(ENCRYPTED.length)) : v.privateKey;
return { publicKey: v.publicKey, privateKey };
} catch {
// Unreadable (the encryption key changed): made again below — every browser must then subscribe again, which the page says.
}
}
const ecdh = createECDH("prime256v1");
ecdh.generateKeys();
const keys: VapidKeys = { publicKey: b64u(ecdh.getPublicKey()), privateKey: b64u(ecdh.getPrivateKey().length === 32 ? ecdh.getPrivateKey() : Buffer.concat([Buffer.alloc(32 - ecdh.getPrivateKey().length), ecdh.getPrivateKey()])) };
await setInternalFlag(VAPID_FLAG, JSON.stringify({ publicKey: keys.publicKey, privateKey: env.credentialsEncryptionEnabled ? ENCRYPTED + encryptSecret(keys.privateKey) : keys.privateKey }));
// The subscriptions made for the old key can't be used with the new one.
await db.delete(pushSubscriptions);
return keys;
}
/** For the tests, so one test's keys don't leak into the next. */
export function forgetVapidKeysForTests(): void {
cachedKeys = null;
}
/** A P-256 private key as a key object, from its raw scalar and public point. */
function privateKeyObject(keys: VapidKeys): KeyObject {
const pub = fromB64u(keys.publicKey);
return createPrivateKey({
key: { kty: "EC", crv: "P-256", d: keys.privateKey, x: b64u(pub.subarray(1, 33)), y: b64u(pub.subarray(33, 65)) },
format: "jwk",
});
}
/** The Authorization header's token: who we are, who it is for, and for how long (RFC 8292). */
export function vapidAuthorization(audience: string, subject: string, keys: VapidKeys, now: number = Date.now()): string {
const header = b64u(Buffer.from(JSON.stringify({ typ: "JWT", alg: "ES256" })));
const claims = b64u(Buffer.from(JSON.stringify({ aud: audience, exp: Math.floor(now / 1000) + 12 * 3600, sub: subject })));
const signature = sign("sha256", Buffer.from(`${header}.${claims}`), { key: privateKeyObject(keys), dsaEncoding: "ieee-p1363" });
return `vapid t=${header}.${claims}.${b64u(signature)}, k=${keys.publicKey}`;
}
// ─── Encrypting a message for one browser (RFC 8291) ───────────────────────
export interface PushKeys {
/** The browser's public key, base64url. */
p256dh: string;
/** The browser's 16-byte authentication secret, base64url. */
auth: string;
}
const RECORD_SIZE = 4096;
/**
* The body to send: the message encrypted so that only the browser it is for can read it. `ephemeral` and `salt` are chosen fresh each
* time; they are parameters only so the standard's own worked example can be checked.
*/
export function encryptPayload(payload: Buffer, subscription: PushKeys, fixed?: { ephemeral: ECDHLike; salt: Buffer }): Buffer {
const uaPublic = fromB64u(subscription.p256dh);
const authSecret = fromB64u(subscription.auth);
if (uaPublic.length !== 65 || uaPublic[0] !== 4) throw new Error("The browser's public key isn't a P-256 point.");
if (authSecret.length !== 16) throw new Error("The browser's authentication secret isn't 16 bytes.");
const fresh = fixed ? null : createECDH("prime256v1");
fresh?.generateKeys();
const ecdh: ECDHLike = fixed?.ephemeral ?? fresh!;
const asPublic = ecdh.getPublicKey();
const salt = fixed?.salt ?? randomBytes(16);
const secret = ecdh.computeSecret(uaPublic);
const keyInfo = Buffer.concat([Buffer.from("WebPush: info\0"), uaPublic, asPublic]);
const ikm = Buffer.from(hkdfSync("sha256", secret, authSecret, keyInfo, 32));
const cek = Buffer.from(hkdfSync("sha256", ikm, salt, Buffer.from("Content-Encoding: aes128gcm\0"), 16));
const nonce = Buffer.from(hkdfSync("sha256", ikm, salt, Buffer.from("Content-Encoding: nonce\0"), 12));
// One record: the message, then a 0x02 marking it as the last.
if (payload.length > RECORD_SIZE - 17 - 1) throw new Error("The message is too long for one push.");
const cipher = createCipheriv("aes-128-gcm", cek, nonce);
const encrypted = Buffer.concat([cipher.update(Buffer.concat([payload, Buffer.from([2])])), cipher.final(), cipher.getAuthTag()]);
const header = Buffer.alloc(16 + 4 + 1);
salt.copy(header, 0);
header.writeUInt32BE(RECORD_SIZE, 16);
header.writeUInt8(asPublic.length, 20);
return Buffer.concat([header, asPublic, encrypted]);
}
type ECDHLike = Pick<ReturnType<typeof createECDH>, "getPublicKey" | "computeSecret">;
// ─── Subscriptions ─────────────────────────────────────────────────────────
let allowLocalEndpoints = false;
/** Lets the tests push to a service on this machine, which is otherwise refused. */
export function allowLocalPushEndpointsForTests(allow: boolean): void {
allowLocalEndpoints = allow;
}
/** A push service is a public internet service with an https address; anything else is refused before we ever connect to it. */
export async function checkEndpoint(endpoint: string): Promise<void> {
let url: URL;
try {
url = new URL(endpoint);
} catch {
throw new Error("The push address isn't a web address.");
}
if (url.protocol !== "https:" && !(allowLocalEndpoints && url.protocol === "http:")) throw new Error("The push address must be https.");
if (url.username || url.password) throw new Error("The push address can't carry a login.");
if (allowLocalEndpoints) return;
await resolveTarget(url.hostname, { allowPrivate: false });
}
export interface PushOutcome {
sent: number;
/** Subscriptions the push service said are gone, now removed. */
removed: number;
failed: number;
}
/** The contact the push services may reach us on (a `mailto:` or an https address), as VAPID asks. */
export async function vapidSubject(): Promise<string> {
const { webPush } = await getSettings();
const contact = webPush.contact.trim();
if (/^(mailto:|https:\/\/)/i.test(contact)) return contact;
if (contact.includes("@")) return `mailto:${contact}`;
return /^https:\/\//.test(env.appBaseUrl) ? env.appBaseUrl : "mailto:homelab@example.invalid";
}
export interface PushMessage {
title: string;
body: string;
/** Where in the app a tap should go. */
url?: string;
/** A later message with the same tag replaces an earlier one that is still showing. */
tag?: string;
}
/** Sends one message to one subscription. Resolves to what the push service said; never throws. */
async function deliver(sub: { endpoint: string; p256dh: string; auth: string }, body: Buffer, keys: VapidKeys, subject: string, urgent: boolean): Promise<"sent" | "gone" | "failed"> {
try {
await checkEndpoint(sub.endpoint);
const res = await fetch(sub.endpoint, {
method: "POST",
headers: {
Authorization: vapidAuthorization(new URL(sub.endpoint).origin, subject, keys),
"Content-Encoding": "aes128gcm",
"Content-Type": "application/octet-stream",
TTL: "86400",
Urgency: urgent ? "high" : "normal",
},
body: new Uint8Array(body),
signal: AbortSignal.timeout(15_000),
});
if (res.ok) return "sent";
if (res.status === 404 || res.status === 410) return "gone";
console.error(`[push] ${new URL(sub.endpoint).host} answered ${res.status}: ${(await res.text().catch(() => "")).slice(0, 200)}`);
return "failed";
} catch (err) {
console.error("[push] couldn't deliver:", err instanceof Error ? err.message : err);
return "failed";
}
}
/** Sends to the given subscriptions (all of them, if none are given). Subscriptions the push service says are gone are removed. */
export async function sendPush(message: PushMessage, only?: { id: number; endpoint: string; p256dh: string; auth: string }[]): Promise<PushOutcome> {
const subs = only ?? (await db.select().from(pushSubscriptions));
const outcome: PushOutcome = { sent: 0, removed: 0, failed: 0 };
if (subs.length === 0) return outcome;
const keys = await getVapidKeys();
const subject = await vapidSubject();
const payload = Buffer.from(JSON.stringify({ title: message.title.slice(0, 120), body: message.body.slice(0, 600), url: message.url ?? "/alerts", tag: message.tag }));
await Promise.all(
subs.map(async (s) => {
let body: Buffer;
try {
body = encryptPayload(payload, { p256dh: s.p256dh, auth: s.auth });
} catch (err) {
console.error("[push] a subscription can't be written to:", err instanceof Error ? err.message : err);
outcome.failed++;
return;
}
const result = await deliver(s, body, keys, subject, true);
if (result === "sent") {
outcome.sent++;
await db.update(pushSubscriptions).set({ lastSuccessAt: new Date().toISOString() }).where(eq(pushSubscriptions.id, s.id));
} else if (result === "gone") {
outcome.removed++;
await db.delete(pushSubscriptions).where(eq(pushSubscriptions.id, s.id));
} else {
outcome.failed++;
}
}),
);
return outcome;
}
/** How many browsers are subscribed. */
export async function subscriptionCount(): Promise<number> {
return (await db.select({ id: pushSubscriptions.id }).from(pushSubscriptions)).length;
}
+212
View File
@@ -0,0 +1,212 @@
/**
* Dependencies between servers and services: what is folded under what when something is down, and the routes that say what needs what.
*
* Run with: npm test --workspace server
*/
import { test, describe, before, after } from "node:test";
import assert from "node:assert/strict";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { AddressInfo } from "node:net";
import type { Server } from "node:http";
const dir = mkdtempSync(join(tmpdir(), "hm-deps-"));
process.env.DATABASE_PATH = join(dir, "test.sqlite");
process.env.SESSION_DIR = join(dir, "sessions");
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
process.env.HEALTH_STARTUP_GRACE_MS = "0";
const express = (await import("express")).default;
const { db } = await import("../src/db/client.js");
const { users, servers, integrations, auditLog } = await import("../src/db/schema.js");
const { runMigrations } = await import("../src/db/migrate.js");
const { buildEdges, coveredNodes, wouldLoop, integrationCausedBy, HEALTH_STATE_FLAG } = await import("../src/services/dependencies.js");
const { setInternalFlag } = await import("../src/services/settingsStore.js");
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
const { trackIntegrationHealth } = await import("../src/services/integrationHealthMonitor.js");
const { dependenciesRouter } = await import("../src/routes/dependencies.js");
const { eq } = await import("drizzle-orm");
await runMigrations();
const everything = () => true;
const edge = (fromType: "server" | "integration", fromId: number, toType: "server" | "integration", toId: number) => ({ fromType, fromId, toType, toId });
describe("who is down because of whom", () => {
test("a node is explained by the root of what it needs, however far up; things that are up in between don't matter", () => {
const edges = buildEdges(new Map(), [edge("integration", 1, "server", 1), edge("integration", 2, "integration", 1), edge("server", 2, "integration", 2)], everything);
const covered = coveredNodes(new Set(["server:1", "integration:1", "integration:2", "server:2"]), edges);
assert.deepEqual([...covered.entries()].sort(), [["integration:1", "server:1"], ["integration:2", "server:1"], ["server:2", "server:1"]]);
// the middle one is up: the one above it still explains the one below
const skip = coveredNodes(new Set(["server:1", "server:2"]), edges);
assert.deepEqual([...skip.entries()], [["server:2", "server:1"]]);
});
test("the 'runs on' links of servers are part of the graph", () => {
const edges = buildEdges(new Map([[2, { parentId: 1, auto: false }]]), [edge("integration", 1, "server", 2)], everything);
const covered = coveredNodes(new Set(["server:1", "integration:1"]), edges);
assert.equal(covered.get("integration:1"), "server:1");
});
test("something that needs two things is explained by whichever is down; with none down it stands alone", () => {
const edges = buildEdges(new Map(), [edge("server", 1, "server", 2), edge("server", 1, "integration", 1)], everything);
assert.equal(coveredNodes(new Set(["server:1", "integration:1"]), edges).get("server:1"), "integration:1");
assert.equal(coveredNodes(new Set(["server:1"]), edges).size, 0);
});
test("a loop explains nothing, and edges to things that are gone are ignored", () => {
const loop = buildEdges(new Map(), [edge("server", 1, "server", 2), edge("server", 2, "server", 1)], everything);
assert.equal(coveredNodes(new Set(["server:1", "server:2"]), loop).size, 0);
const gone = buildEdges(new Map(), [edge("server", 1, "server", 99)], (k) => k !== "server:99");
assert.equal(gone.size, 0);
});
test("a new dependency that would close a loop is seen, including through 'runs on'", () => {
const edges = buildEdges(new Map([[2, { parentId: 1, auto: true }]]), [edge("integration", 1, "server", 2)], everything);
assert.equal(wouldLoop(edges, "server:1", "integration:1"), true, "the integration needs server 2, which runs on server 1");
assert.equal(wouldLoop(edges, "server:1", "server:1"), true);
assert.equal(wouldLoop(edges, "integration:1", "server:1"), false, "already needs it, indirectly: not a loop");
});
});
describe("through the routes, the alerts and the notifications", () => {
let app: Server;
let base = "";
let who = "operator";
let box = 0;
let other = 0;
let gitea = 0;
let semaphore = 0;
before(async () => {
await db.insert(users).values([
{ oidcSub: "operator", name: "Olga", role: "operator" },
{ oidcSub: "viewer", name: "Vic", role: "viewer" },
]);
const hoursAgo = (n: number) => new Date(Date.now() - n * 3600_000).toISOString();
const made = await db
.insert(servers)
.values([
{ name: "box", apiTokenHash: "b", apiTokenPrefix: "b", lastSeenAt: hoursAgo(3) },
{ name: "other", apiTokenHash: "o", apiTokenPrefix: "o", lastSeenAt: new Date().toISOString() },
])
.returning({ id: servers.id });
box = made[0].id;
other = made[1].id;
const ints = await db
.insert(integrations)
.values([
{ type: "gitea", name: "Gitea", baseUrl: "https://git.lab", config: "{}" },
{ type: "semaphore", name: "Semaphore", baseUrl: "https://sem.lab", config: "{}" },
])
.returning({ id: integrations.id });
gitea = ints[0].id;
semaphore = ints[1].id;
const a = express();
a.use(express.json());
a.use((req, _res, next) => {
(req as unknown as { session: unknown }).session = { user: { sub: who } };
next();
});
a.use("/api/dependencies", dependenciesRouter);
app = await new Promise<Server>((resolve) => {
const s = a.listen(0, "127.0.0.1", () => resolve(s));
});
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/dependencies`;
});
after(() => {
app.close();
try {
rmSync(dir, { recursive: true, force: true });
} catch {
// Windows keeps the database file locked until the process exits.
}
});
const send = (method: string, path: string, body?: unknown) => fetch(`${base}${path}`, { method, headers: { "content-type": "application/json" }, body: body === undefined ? undefined : JSON.stringify(body) });
let firstId = 0;
test("an operator says what needs what; it is listed with names", async () => {
const res = await send("POST", "/", edge("integration", gitea, "server", box));
assert.equal(res.status, 201);
firstId = (await res.json()).id;
assert.equal((await send("POST", "/", edge("integration", semaphore, "integration", gitea))).status, 201);
const list = await (await send("GET", "/")).json();
assert.deepEqual(list.dependencies.map((d: { from: { name: string }; to: { name: string } }) => `${d.from.name} > ${d.to.name}`).sort(), ["Gitea > box", "Semaphore > Gitea"]);
assert.ok(list.nodes.some((n: { key: string; name: string }) => n.key === `server:${box}` && n.name === "box"));
});
test("it can't be nonsense: not itself, not twice, not a loop, not something that isn't there, not by a viewer", async () => {
assert.equal((await send("POST", "/", edge("server", box, "server", box))).status, 400);
assert.equal((await send("POST", "/", edge("integration", gitea, "server", box))).status, 409, "twice");
const loop = await send("POST", "/", edge("server", box, "integration", semaphore));
assert.equal(loop.status, 409, "the server would wait for the thing that waits for it");
assert.match((await loop.json()).message, /wait for each other/);
assert.equal((await send("POST", "/", edge("server", 99999, "server", box))).status, 400);
assert.equal((await send("POST", "/", { fromType: "dns_provider", fromId: 1, toType: "server", toId: box })).status, 400);
who = "viewer";
assert.equal((await send("POST", "/", edge("server", other, "server", box))).status, 403);
assert.equal((await send("GET", "/")).status, 200, "everyone signed in can look");
who = "operator";
});
test("when the server is down and both services are failing, only the server is announced; the services are held back under it", async () => {
await trackIntegrationHealth("gitea", false);
await trackIntegrationHealth("semaphore", false);
invalidateAlertsCache();
const report = await getAlerts(true);
const offline = report.alerts.find((a) => a.category === "offline" && a.message.startsWith("box"))!;
assert.equal(offline.silenced, false);
assert.deepEqual(offline.affects, ["Gitea", "Semaphore"]);
const failing = report.alerts.filter((a) => a.category === "integration");
assert.equal(failing.length, 2);
for (const f of failing) {
assert.deepEqual(f.causedBy, { name: "box" });
assert.equal(f.silenced, true);
}
assert.equal(failing.find((f) => f.id.includes("gitea"))!.affects, undefined, "only the root says what it takes with it");
assert.equal(report.counts.critical + report.counts.warning, 1);
assert.equal(report.counts.grouped, 2);
});
test("when the server is back, Gitea stands alone and Semaphore is held back under Gitea", async () => {
await db.update(servers).set({ lastSeenAt: new Date().toISOString() }).where(eq(servers.id, box));
invalidateAlertsCache();
const report = await getAlerts(true);
const failing = report.alerts.filter((a) => a.category === "integration");
assert.equal(failing.find((f) => f.id.includes("gitea"))!.causedBy, undefined);
assert.deepEqual(failing.find((f) => f.id.includes("semaphore"))!.causedBy, { name: "Gitea" });
});
test("a kind of service is folded only if every enabled instance of it is explained", async () => {
const [second] = await db.insert(integrations).values({ type: "gitea", name: "Gitea 2", baseUrl: "https://git2.lab", config: "{}" }).returning({ id: integrations.id });
await db.update(servers).set({ lastSeenAt: new Date(Date.now() - 3 * 3600_000).toISOString() }).where(eq(servers.id, box));
invalidateAlertsCache();
const report = await getAlerts(true);
assert.equal(report.alerts.find((a) => a.category === "integration" && a.id.includes("gitea"))!.causedBy, undefined, "the second one doesn't need the box");
await db.delete(integrations).where(eq(integrations.id, second.id));
});
test("a failure that crosses its threshold while the server it needs is down isn't announced; one that doesn't need it is", async () => {
assert.equal(await integrationCausedBy("gitea", ["gitea"]), null, "nothing is known to be down yet");
await setInternalFlag(HEALTH_STATE_FLAG, JSON.stringify({ [`offline:server:${box}`]: { message: "box is offline", source: "server" } }));
assert.deepEqual(await integrationCausedBy("gitea", ["gitea"]), { key: `server:${box}`, name: "box" });
assert.deepEqual(await integrationCausedBy("semaphore", ["gitea", "semaphore"]), { key: `server:${box}`, name: "box" }, "through Gitea");
assert.equal(await integrationCausedBy("tailscale", ["tailscale"]), null, "an unrelated kind");
await setInternalFlag(HEALTH_STATE_FLAG, JSON.stringify({ [`offline:server:${other}`]: { message: "other is offline", source: "server" } }));
assert.equal(await integrationCausedBy("gitea", ["gitea"]), null, "a different server being down explains nothing");
});
test("removing one works, is audited, and the services stand alone again", async () => {
assert.equal((await send("DELETE", `/${firstId}`)).status, 204);
assert.equal((await send("DELETE", `/${firstId}`)).status, 404);
const actions = (await db.select().from(auditLog)).filter((e) => e.category === "dependency").map((e) => e.action);
assert.deepEqual(actions.sort(), ["create", "create", "delete"]);
invalidateAlertsCache();
const report = await getAlerts(true);
assert.equal(report.alerts.find((a) => a.category === "integration" && a.id.includes("gitea"))!.causedBy, undefined);
});
});
+149
View File
@@ -0,0 +1,149 @@
/**
* Filling in the inventory from what an agent says the machine is: only blanks are filled, a refresh overwrites on purpose, and an agent's
* report fills the entries of the machine it came from.
*
* Run with: npm test --workspace server
*/
import { test, describe, before, after } from "node:test";
import assert from "node:assert/strict";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { AddressInfo } from "node:net";
import type { Server } from "node:http";
const dir = mkdtempSync(join(tmpdir(), "hm-invfill-"));
process.env.DATABASE_PATH = join(dir, "test.sqlite");
process.env.SESSION_DIR = join(dir, "sessions");
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
process.env.HEALTH_STARTUP_GRACE_MS = "0";
const express = (await import("express")).default;
const { db } = await import("../src/db/client.js");
const { users, servers, assets, auditLog } = await import("../src/db/schema.js");
const { runMigrations } = await import("../src/db/migrate.js");
const { inventoryRouter } = await import("../src/routes/inventory.js");
const { agentReportRouter } = await import("../src/routes/agentReport.js");
const { hashToken } = await import("../src/services/tokens.js");
const { blanksToFill, differences, hardwareOf } = await import("../src/services/inventoryAutofill.js");
const { eq } = await import("drizzle-orm");
await runMigrations();
describe("what would be filled, and what would change", () => {
const hw = { manufacturer: "Dell Inc.", model: "PowerEdge R730", serial: "ABC123" };
test("only blank fields are filled; what a person wrote stays", () => {
assert.deepEqual(blanksToFill({ manufacturer: null, model: "R730 (mine)", serial: " " }, hw), { manufacturer: "Dell Inc.", serial: "ABC123" });
assert.deepEqual(blanksToFill({ manufacturer: "x", model: "y", serial: "z" }, hw), {});
assert.deepEqual(blanksToFill({ manufacturer: null, model: null, serial: null }, null), {});
assert.deepEqual(blanksToFill({ manufacturer: null, model: null, serial: null }, { manufacturer: "Dell", model: null, serial: null }), { manufacturer: "Dell" });
});
test("a refresh would change the fields where the agent says something else, and nothing that already matches", () => {
assert.deepEqual(differences({ manufacturer: "Dell Inc.", model: "R730", serial: null }, hw), { model: "PowerEdge R730", serial: "ABC123" });
assert.deepEqual(differences(hw, hw), {});
assert.deepEqual(differences({ manufacturer: "Dell Inc. ", model: "PowerEdge R730", serial: "ABC123" }, hw), {}, "stray spaces aren't a difference");
assert.deepEqual(differences({ manufacturer: "x", model: "y", serial: "z" }, { manufacturer: null, model: null, serial: null }), {}, "an agent that says nothing changes nothing");
});
test("a server that has said nothing has no hardware", () => {
assert.equal(hardwareOf(null), null);
assert.equal(hardwareOf({ hwManufacturer: null, hwModel: null, hwSerial: null }), null);
assert.deepEqual(hardwareOf({ hwManufacturer: "A", hwModel: null, hwSerial: null }), { manufacturer: "A", model: null, serial: null });
});
});
describe("through the routes and the agent's report", () => {
let app: Server;
let base = "";
let id = 0;
before(async () => {
await db.insert(users).values([{ oidcSub: "operator", name: "Olle", role: "operator" }]);
const [s] = await db.insert(servers).values({ name: "elsa", apiTokenHash: hashToken("tok"), apiTokenPrefix: "t" }).returning();
id = s.id;
const a = express();
a.use("/api/agent/report", agentReportRouter);
a.use(express.json());
a.use((req, _res, next) => {
(req as unknown as { session: unknown }).session = { user: { sub: "operator" } };
next();
});
a.use("/api/inventory", inventoryRouter);
app = await new Promise<Server>((resolve) => {
const x = a.listen(0, "127.0.0.1", () => resolve(x));
});
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api`;
});
after(() => {
app.close();
try {
rmSync(dir, { recursive: true, force: true });
} catch {
// Windows keeps the database file locked until the process exits.
}
});
const report = (hardware: unknown) =>
fetch(`${base}/agent/report`, { method: "POST", headers: { "content-type": "application/json", authorization: "Bearer tok" }, body: JSON.stringify({ hostname: "h", os_type: "linux", tasks: [], system: { hardware } }) });
const send = (method: string, path: string, body?: unknown) =>
fetch(`${base}/inventory${path}`, { method, headers: { "content-type": "application/json" }, body: body === undefined ? undefined : JSON.stringify(body) });
const entry = async (assetId: number) => (await db.select().from(assets).where(eq(assets.id, assetId)))[0];
let linked = 0;
let unlinked = 0;
test("an entry made before the agent has said anything stays blank, then an agent's report fills it", async () => {
linked = (await (await send("POST", "/", { name: "elsa's box", kind: "server", serverId: id, model: "my own words" })).json()).asset.id;
unlinked = (await (await send("POST", "/", { name: "a switch", kind: "network" })).json()).asset.id;
assert.deepEqual([(await entry(linked)).manufacturer, (await entry(linked)).serial], [null, null]);
assert.equal((await report({ manufacturer: "Dell Inc.", model: "PowerEdge R730", serial: "ABC123" })).status, 202);
const row = await entry(linked);
assert.deepEqual([row.manufacturer, row.model, row.serial], ["Dell Inc.", "my own words", "ABC123"], "blanks filled, the model a person wrote is kept");
const [s] = await db.select().from(servers).where(eq(servers.id, id));
assert.deepEqual([s.hwManufacturer, s.hwModel, s.hwSerial], ["Dell Inc.", "PowerEdge R730", "ABC123"]);
assert.equal((await entry(unlinked)).manufacturer, null, "an entry that isn't this machine is left alone");
});
test("the list shows what the agent says and what a refresh would change", async () => {
const list = (await (await send("GET", "/")).json()).assets;
const a = list.find((x: { id: number }) => x.id === linked);
assert.deepEqual(a.agent, { manufacturer: "Dell Inc.", model: "PowerEdge R730", serial: "ABC123", differs: { model: "PowerEdge R730" } });
assert.equal(list.find((x: { id: number }) => x.id === unlinked).agent, null);
});
test("a new entry linked to a machine that has already reported starts out filled in", async () => {
const made = (await (await send("POST", "/", { name: "elsa again", kind: "server", serverId: id })).json()).asset;
assert.deepEqual([made.manufacturer, made.model, made.serial], ["Dell Inc.", "PowerEdge R730", "ABC123"]);
const edited = (await (await send("PUT", `/${unlinked}`, { name: "a switch", kind: "network", serverId: id })).json()).asset;
assert.equal(edited.serial, "ABC123", "linking an existing entry fills its blanks too");
});
test("a report with nothing in it, or garbage, changes nothing", async () => {
assert.equal((await report({ manufacturer: " " })).status, 202);
assert.equal((await report("nonsense")).status, 202);
const [s] = await db.select().from(servers).where(eq(servers.id, id));
assert.equal(s.hwSerial, "ABC123");
});
test("refreshing from the agent overwrites on purpose, says what changed, and is in the audit log", async () => {
const res = await send("POST", `/${linked}/refresh-from-agent`);
assert.equal(res.status, 200);
const body = await res.json();
assert.deepEqual(body.changed, ["model"]);
assert.equal(body.asset.model, "PowerEdge R730");
const again = await (await send("POST", `/${linked}/refresh-from-agent`)).json();
assert.deepEqual(again.changed, [], "nothing left to change");
const log = (await db.select().from(auditLog)).filter((e) => e.action === "refresh_from_agent");
assert.equal(log.length, 1, "only a refresh that changed something is recorded");
});
test("an entry without a server, or one that isn't there, can't be refreshed", async () => {
const lone = (await (await send("POST", "/", { name: "a shelf", kind: "other" })).json()).asset.id;
assert.equal((await send("POST", `/${lone}/refresh-from-agent`)).status, 400);
assert.equal((await send("POST", "/99999/refresh-from-agent")).status, 404);
});
});
+292
View File
@@ -0,0 +1,292 @@
/**
* Web Push: the encryption against the standard's own worked example, the signed token, the routes, and a message all the way to a stand-in push
* service and back out through a stand-in browser.
*
* Run with: npm test --workspace server
*/
import { test, describe, before, after } from "node:test";
import assert from "node:assert/strict";
import { createDecipheriv, createECDH, createPublicKey, hkdfSync, verify } from "node:crypto";
import { createServer, type IncomingMessage, type Server } from "node:http";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { AddressInfo } from "node:net";
const dir = mkdtempSync(join(tmpdir(), "hm-push-"));
process.env.DATABASE_PATH = join(dir, "test.sqlite");
process.env.SESSION_DIR = join(dir, "sessions");
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
process.env.HEALTH_STARTUP_GRACE_MS = "0";
const express = (await import("express")).default;
const { db } = await import("../src/db/client.js");
const { users, pushSubscriptions, auditLog } = await import("../src/db/schema.js");
const { runMigrations } = await import("../src/db/migrate.js");
const { encryptPayload, vapidAuthorization, getVapidKeys, checkEndpoint, allowLocalPushEndpointsForTests, sendPush } = await import("../src/services/webPush.js");
const { updateSettings } = await import("../src/services/settingsStore.js");
const { notifyNow, enabledChannelCount } = await import("../src/services/notify.js");
const { pushRouter } = await import("../src/routes/push.js");
const { eq } = await import("drizzle-orm");
await runMigrations();
const b64u = (s: string) => Buffer.from(s, "base64url");
/** What a browser does with a push it receives (RFC 8291 §3.4), to read what we wrote. */
function decrypt(body: Buffer, uaPrivate: Buffer, uaPublic: Buffer, authSecret: Buffer): Buffer {
const salt = body.subarray(0, 16);
const idlen = body[20];
const asPublic = body.subarray(21, 21 + idlen);
const ciphertext = body.subarray(21 + idlen);
const ecdh = createECDH("prime256v1");
ecdh.setPrivateKey(uaPrivate);
const secret = ecdh.computeSecret(asPublic);
const ikm = Buffer.from(hkdfSync("sha256", secret, authSecret, Buffer.concat([Buffer.from("WebPush: info\0"), uaPublic, asPublic]), 32));
const cek = Buffer.from(hkdfSync("sha256", ikm, salt, Buffer.from("Content-Encoding: aes128gcm\0"), 16));
const nonce = Buffer.from(hkdfSync("sha256", ikm, salt, Buffer.from("Content-Encoding: nonce\0"), 12));
const d = createDecipheriv("aes-128-gcm", cek, nonce);
d.setAuthTag(ciphertext.subarray(ciphertext.length - 16));
const plain = Buffer.concat([d.update(ciphertext.subarray(0, ciphertext.length - 16)), d.final()]);
assert.equal(plain[plain.length - 1], 2, "ends with the last-record marker");
return plain.subarray(0, plain.length - 1);
}
/** A stand-in browser: its keys, in the shape a subscription carries them. */
function browser() {
const ecdh = createECDH("prime256v1");
ecdh.generateKeys();
const auth = Buffer.from(Array.from({ length: 16 }, (_, i) => i * 7 + 3));
return { ecdh, priv: ecdh.getPrivateKey(), pub: ecdh.getPublicKey(), auth, keys: { p256dh: ecdh.getPublicKey().toString("base64url"), auth: auth.toString("base64url") } };
}
describe("the encryption", () => {
test("it reproduces the worked example in RFC 8291, byte for byte", () => {
const asPrivate = b64u("yfWPiYE-n46HLnH0KqZOF1fJJU3MYrct3AELtAQ-oRw");
const ephemeral = createECDH("prime256v1");
ephemeral.setPrivateKey(asPrivate);
assert.equal(ephemeral.getPublicKey().toString("base64url"), "BP4z9KsN6nGRTbVYI_c7VJSPQTBtkgcy27mlmlMoZIIgDll6e3vCYLocInmYWAmS6TlzAC8wEqKK6PBru3jl7A8");
const body = encryptPayload(
Buffer.from("When I grow up, I want to be a watermelon"),
{ p256dh: "BCVxsr7N_eNgVRqvHtD0zTZsEc6-VV-JvLexhqUzORcxaOzi6-AYWXvTBHm4bjyPjs7Vd8pZGH6SRpkNtoIAiw4", auth: "BTBZMqHH6r4Tts7J_aSIgg" },
{ ephemeral, salt: b64u("DGv6ra1nlYgDCS1FRnbzlw") },
);
assert.equal(
body.toString("base64url"),
"DGv6ra1nlYgDCS1FRnbzlwAAEABBBP4z9KsN6nGRTbVYI_c7VJSPQTBtkgcy27mlmlMoZIIgDll6e3vCYLocInmYWAmS6TlzAC8wEqKK6PBru3jl7A_yl95bQpu6cVPTpK4Mqgkf1CXztLVBSt2Ks3oZwbuwXPXLWyouBWLVWGNWQexSgSxsj_Qulcy4a-fN",
);
});
test("the browser it is for can read it, and it is different every time", () => {
const b = browser();
const one = encryptPayload(Buffer.from("hello"), b.keys);
const two = encryptPayload(Buffer.from("hello"), b.keys);
assert.notDeepEqual(one, two, "a fresh key and salt each time");
assert.equal(decrypt(one, b.priv, b.pub, b.auth).toString(), "hello");
const other = browser();
assert.throws(() => decrypt(one, other.priv, other.pub, other.auth), "someone else's browser can't");
});
test("keys that aren't a browser's, or a message too long for one push, are refused", () => {
const b = browser();
assert.throws(() => encryptPayload(Buffer.from("x"), { p256dh: "AAAA", auth: b.keys.auth }), /P-256/);
assert.throws(() => encryptPayload(Buffer.from("x"), { p256dh: b.keys.p256dh, auth: "AAAA" }), /16 bytes/);
assert.throws(() => encryptPayload(Buffer.alloc(4100), b.keys), /too long/);
});
});
describe("the signed token", () => {
test("it names who it is for, who sent it and when it ends, and the signature checks out against the public key", async () => {
const keys = await getVapidKeys();
const header = vapidAuthorization("https://push.example", "mailto:me@example.com", keys, 1_700_000_000_000);
const m = /^vapid t=([\w-]+)\.([\w-]+)\.([\w-]+), k=([\w-]+)$/.exec(header)!;
assert.ok(m, header);
assert.equal(m[4], keys.publicKey);
assert.deepEqual(JSON.parse(Buffer.from(m[1], "base64url").toString()), { typ: "JWT", alg: "ES256" });
assert.deepEqual(JSON.parse(Buffer.from(m[2], "base64url").toString()), { aud: "https://push.example", exp: 1_700_000_000 + 12 * 3600, sub: "mailto:me@example.com" });
const pub = b64u(keys.publicKey);
const key = createPublicKey({ key: { kty: "EC", crv: "P-256", x: pub.subarray(1, 33).toString("base64url"), y: pub.subarray(33, 65).toString("base64url") }, format: "jwk" });
assert.equal(verify("sha256", Buffer.from(`${m[1]}.${m[2]}`), { key, dsaEncoding: "ieee-p1363" }, b64u(m[3])), true);
assert.equal(b64u(m[3]).length, 64, "the signature is the plain r and s, as JWT wants");
});
test("the server's key pair is kept: the same one comes back", async () => {
const a = await getVapidKeys();
const b = await getVapidKeys();
assert.deepEqual(a, b);
assert.equal(b64u(a.publicKey).length, 65);
assert.equal(b64u(a.privateKey).length, 32);
});
});
describe("which addresses are push services", () => {
test("https only, no login, and never an address on a private network", async () => {
await assert.rejects(checkEndpoint("http://fcm.googleapis.com/x"), /https/);
await assert.rejects(checkEndpoint("https://user:pw@fcm.googleapis.com/x"), /login/);
await assert.rejects(checkEndpoint("not a url"), /web address/);
await assert.rejects(checkEndpoint("https://127.0.0.1/x"));
await assert.rejects(checkEndpoint("https://192.168.1.10/x"));
await assert.rejects(checkEndpoint("https://169.254.169.254/latest"));
await assert.rejects(checkEndpoint("https://localhost/x"));
});
});
describe("through the routes, the channel and a push service", () => {
let app: Server;
let base = "";
let who = "ada";
let pushService: Server;
let pushBase = "";
const received: { path: string; headers: IncomingMessage["headers"]; body: Buffer }[] = [];
let respondWith = 201;
let ada = 0;
before(async () => {
allowLocalPushEndpointsForTests(true);
const made = await db
.insert(users)
.values([
{ oidcSub: "ada", name: "Ada", role: "admin" },
{ oidcSub: "vic", name: "Vic", role: "viewer" },
])
.returning({ id: users.id });
ada = made[0].id;
pushService = createServer((req, res) => {
const chunks: Buffer[] = [];
req.on("data", (c) => chunks.push(c));
req.on("end", () => {
received.push({ path: req.url ?? "", headers: req.headers, body: Buffer.concat(chunks) });
res.statusCode = respondWith;
res.end();
});
});
await new Promise<void>((r) => pushService.listen(0, "127.0.0.1", r));
pushBase = `http://127.0.0.1:${(pushService.address() as AddressInfo).port}`;
const a = express();
a.use(express.json());
a.use((req, _res, next) => {
(req as unknown as { session: unknown }).session = { user: { sub: who } };
next();
});
a.use("/api/push", pushRouter);
app = await new Promise<Server>((resolve) => {
const s = a.listen(0, "127.0.0.1", () => resolve(s));
});
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/push`;
});
after(() => {
allowLocalPushEndpointsForTests(false);
app.close();
pushService.close();
try {
rmSync(dir, { recursive: true, force: true });
} catch {
// Windows keeps the database file locked until the process exits.
}
});
const send = (method: string, path: string, body?: unknown) => fetch(`${base}${path}`, { method, headers: { "content-type": "application/json" }, body: body === undefined ? undefined : JSON.stringify(body) });
const mine = browser();
const vics = browser();
let adaSubId = 0;
test("while push is off nobody can subscribe, and there is no key to subscribe with", async () => {
assert.equal((await send("GET", "/key")).status, 409);
assert.equal((await send("POST", "/subscribe", { endpoint: `${pushBase}/a`, keys: mine.keys })).status, 409);
assert.equal(await enabledChannelCount(), 0);
});
test("with it on, a browser gets the key and subscribes; the same browser again is the same subscription", async () => {
await updateSettings({ webPush: { enabled: true, contact: "me@example.com" } });
const key = await (await send("GET", "/key")).json();
assert.equal(key.publicKey, (await getVapidKeys()).publicKey);
const res = await send("POST", "/subscribe", { endpoint: `${pushBase}/ada-phone`, keys: mine.keys, label: "Ada's phone" });
assert.equal(res.status, 201);
adaSubId = (await res.json()).id;
assert.equal((await send("POST", "/subscribe", { endpoint: `${pushBase}/ada-phone`, keys: mine.keys, label: "Ada's phone" })).status, 201);
assert.equal((await db.select().from(pushSubscriptions)).length, 1);
const list = await (await send("GET", "/mine")).json();
assert.equal(list.subscriptions.length, 1);
assert.equal(list.subscriptions[0].label, "Ada's phone");
assert.equal((await db.select().from(auditLog)).filter((e) => e.category === "push" && e.action === "subscribe").length, 1, "only the first time");
});
test("a subscription that isn't one is refused", async () => {
assert.equal((await send("POST", "/subscribe", { endpoint: "nonsense", keys: mine.keys })).status, 400);
assert.equal((await send("POST", "/subscribe", { endpoint: `${pushBase}/x`, keys: { p256dh: "short", auth: "short" } })).status, 400);
allowLocalPushEndpointsForTests(false);
const refused = await send("POST", "/subscribe", { endpoint: "https://10.0.0.5/push", keys: mine.keys });
allowLocalPushEndpointsForTests(true);
assert.equal(refused.status, 400, "a private address is never a push service");
});
test("a message through the channels is encrypted for the browser, signed, and addressed to the push service", async () => {
received.length = 0;
assert.equal(await enabledChannelCount(), 1, "push counts as a channel once someone is subscribed");
await notifyNow("Server down", "elsa has not reported for 3 hours");
assert.equal(received.length, 1);
const r = received[0];
assert.equal(r.path, "/ada-phone");
assert.equal(r.headers["content-encoding"], "aes128gcm");
assert.equal(r.headers.ttl, "86400");
assert.match(String(r.headers.authorization), /^vapid t=[\w-]+\.[\w-]+\.[\w-]+, k=[\w-]+$/);
const claims = JSON.parse(Buffer.from(String(r.headers.authorization).split(".")[1], "base64url").toString());
assert.equal(claims.aud, pushBase);
assert.equal(claims.sub, "mailto:me@example.com");
const message = JSON.parse(decrypt(r.body, mine.priv, mine.pub, mine.auth).toString());
assert.deepEqual(message, { title: "Server down", body: "elsa has not reported for 3 hours", url: "/alerts" });
assert.notEqual((await db.select().from(pushSubscriptions))[0].lastSuccessAt, null);
});
test("with push switched off nothing is sent", async () => {
await updateSettings({ webPush: { enabled: false } });
received.length = 0;
await notifyNow("Server down", "x");
assert.equal(received.length, 0);
await updateSettings({ webPush: { enabled: true } });
});
test("a push service that says the browser is gone has that subscription removed; one that is merely failing keeps it", async () => {
const rows = await db.select().from(pushSubscriptions);
respondWith = 500;
assert.deepEqual(await sendPush({ title: "t", body: "b" }, rows), { sent: 0, removed: 0, failed: 1 });
assert.equal((await db.select().from(pushSubscriptions)).length, 1);
respondWith = 410;
assert.deepEqual(await sendPush({ title: "t", body: "b" }, rows), { sent: 0, removed: 1, failed: 0 });
assert.equal((await db.select().from(pushSubscriptions)).length, 0);
respondWith = 201;
});
test("the test button writes to this person's own browsers only", async () => {
assert.equal((await send("POST", "/test")).status, 400, "nothing subscribed now");
await send("POST", "/subscribe", { endpoint: `${pushBase}/ada-phone`, keys: mine.keys });
who = "vic";
await send("POST", "/subscribe", { endpoint: `${pushBase}/vic-laptop`, keys: vics.keys });
received.length = 0;
const res = await send("POST", "/test");
assert.deepEqual(await res.json(), { sent: 1, removed: 0, failed: 0 });
assert.deepEqual(received.map((r) => r.path), ["/vic-laptop"]);
assert.equal(JSON.parse(decrypt(received[0].body, vics.priv, vics.pub, vics.auth).toString()).title, "Homelab Manager — Test");
});
test("a person removes their own browsers, not someone else's; an administrator can remove anyone's", async () => {
const all = await db.select().from(pushSubscriptions);
const adas = all.find((s) => s.userId === ada)!;
const vicsRow = all.find((s) => s.userId !== ada)!;
who = "vic";
assert.equal((await send("DELETE", `/${adas.id}`)).status, 404, "not found, rather than 'not yours'");
assert.equal((await send("DELETE", `/${vicsRow.id}`)).status, 204);
who = "ada";
await send("POST", "/subscribe", { endpoint: `${pushBase}/vic-laptop`, keys: vics.keys });
who = "vic";
assert.equal((await send("POST", "/subscribe", { endpoint: `${pushBase}/vic-laptop`, keys: vics.keys })).status, 201, "taking a browser over makes it theirs");
who = "ada";
const taken = (await db.select().from(pushSubscriptions)).find((s) => s.endpoint.endsWith("/vic-laptop"))!;
assert.equal((await send("DELETE", `/${taken.id}`)).status, 204, "an administrator may");
assert.ok(adaSubId > 0);
assert.equal((await db.select().from(auditLog)).filter((e) => e.category === "push" && e.action === "unsubscribe").length, 2);
assert.equal((await db.select().from(pushSubscriptions).where(eq(pushSubscriptions.userId, ada))).length, 1);
});
});
+126
View File
@@ -0,0 +1,126 @@
/**
* What the Windows agent says about the machine: services that have failed, each disk's health, Storage Spaces pools and what the machine
* is. The mapping from what Windows returns is tested with stand-in objects, and the real collectors are run once as a smoke test.
*
* Run with: npm test --workspace server
*/
import { test, describe } from "node:test";
import assert from "node:assert/strict";
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { dirname, join } from "node:path";
import { fileURLToPath } from "node:url";
import { spawnSync } from "node:child_process";
const repoRoot = join(dirname(fileURLToPath(import.meta.url)), "..", "..");
const script = join(repoRoot, "agent", "windows", "report-tasks.ps1");
const probe = process.platform === "win32" ? spawnSync("pwsh", ["-NoProfile", "-Command", "$PSVersionTable.PSVersion.Major"], { encoding: "utf8" }) : null;
const havePwsh = probe?.status === 0;
/** Runs PowerShell with the agent script loaded (without running it) and returns what the snippet printed as JSON. */
function run(snippet: string): unknown {
const work = mkdtempSync(join(tmpdir(), "hm-winhealth-"));
try {
const file = join(work, "t.ps1");
writeFileSync(file, `$ErrorActionPreference = 'Stop'\n. '${script.replace(/'/g, "''")}'\n${snippet}\n`);
const r = spawnSync("pwsh", ["-NoProfile", "-NonInteractive", "-File", file], { encoding: "utf8" });
assert.equal(r.stderr.trim(), "", `PowerShell said:\n${r.stderr}\n${r.stdout}`);
return JSON.parse(r.stdout.trim());
} finally {
rmSync(work, { recursive: true, force: true });
}
}
describe("the Windows agent's health parts", { skip: !havePwsh }, () => {
test("it says it is version 4, and stays plain ASCII", () => {
const text = readFileSync(script, "utf8");
assert.match(text, /^\$script:AgentVersion = 4\s*$/m);
assert.ok(!/[^\x00-\x7f]/.test(text));
});
test("a service has failed when it starts by itself, isn't running, and stopped with an error — not for exit 0, never-started, or on request", () => {
const out = run(`
function svc($n, $d, $mode, $state, $code) { [pscustomobject]@{ Name = $n; DisplayName = $d; StartMode = $mode; State = $state; ExitCode = $code } }
$list = @(
(svc 'Crashed' 'A crashed service' 'Auto' 'Stopped' 1066),
(svc 'Crashed2' 'Another' 'Auto' 'Stopped' 2),
(svc 'Finished' 'Did its job' 'Auto' 'Stopped' 0),
(svc 'Triggered' 'Starts on a trigger' 'Auto' 'Stopped' 1077),
(svc 'OnRequest' 'Manual one' 'Manual' 'Stopped' 1066),
(svc 'Disabled' 'Disabled one' 'Disabled' 'Stopped' 1066),
(svc 'Fine' 'Running' 'Auto' 'Running' 0),
(svc 'Starting' 'On its way' 'Auto' 'Start Pending' 1066),
(svc 'gupdate' 'Google Update' 'Auto' 'Stopped' 1),
(svc 'NoCode' 'No code at all' 'Auto' 'Stopped' $null)
)
ConvertTo-Json -InputObject @{ all = @(ConvertTo-FailedUnits $list); ignoring = @(ConvertTo-FailedUnits $list @('gupdate*', 'Crashed2')); none = @(ConvertTo-FailedUnits @()) } -Compress
`) as { all: { unit: string; description: string }[]; ignoring: { unit: string }[]; none: unknown[] };
assert.deepEqual(out.all.map((u) => u.unit), ["Crashed", "Crashed2", "gupdate"]);
assert.equal(out.all[0].description, "A crashed service");
assert.deepEqual(out.ignoring.map((u) => u.unit), ["Crashed"], "names and patterns in the ignore list are left out");
assert.deepEqual(out.none, []);
});
test("a disk passes when Windows says Healthy; Warning and Unhealthy don't; Unknown says nothing; temperature and wear come from the counter where sane", () => {
const out = run(`
function disk($id, $name, $health, $bus = 'SATA') { [pscustomobject]@{ DeviceId = $id; FriendlyName = $name; HealthStatus = $health; BusType = $bus } }
$disks = @(
(disk '0' ' Samsung SSD 850 ' 'Healthy'),
(disk '1' 'Old HDD' 'Warning'),
(disk '2' 'Dead HDD' 'Unhealthy'),
(disk '3' 'Mystery' 'Unknown'),
(disk '4' 'NVMe' 'Healthy' 'NVMe'),
(disk '5' 'Virtual' 'Healthy' 'File Backed Virtual'),
(disk '6' 'Odd readings' 'Healthy')
)
$counters = @{
'0' = [pscustomobject]@{ Temperature = 36; Wear = 7 }
'4' = [pscustomobject]@{ Temperature = 0; Wear = 0 }
'6' = [pscustomobject]@{ Temperature = 4000; Wear = 900 }
}
ConvertTo-Json -InputObject @(ConvertTo-DiskHealth $disks $counters) -Compress
`) as { device: string; model: string; healthy: boolean | null; temperature_c: number | null; wear_percent: number | null }[];
const byDevice = Object.fromEntries(out.map((d) => [d.device, d]));
assert.deepEqual(Object.keys(byDevice), ["PhysicalDisk0", "PhysicalDisk1", "PhysicalDisk2", "PhysicalDisk3", "PhysicalDisk4", "PhysicalDisk6"], "the virtual disk is left out");
assert.deepEqual([byDevice.PhysicalDisk0.healthy, byDevice.PhysicalDisk1.healthy, byDevice.PhysicalDisk2.healthy, byDevice.PhysicalDisk3.healthy], [true, false, false, null]);
assert.equal(byDevice.PhysicalDisk0.model, "Samsung SSD 850", "names are trimmed");
assert.deepEqual([byDevice.PhysicalDisk0.temperature_c, byDevice.PhysicalDisk0.wear_percent], [36, 7]);
assert.deepEqual([byDevice.PhysicalDisk4.temperature_c, byDevice.PhysicalDisk4.wear_percent], [null, 0], "0 degrees isn't a reading, 0% wear is");
assert.deepEqual([byDevice.PhysicalDisk6.temperature_c, byDevice.PhysicalDisk6.wear_percent], [null, null], "nonsense is no reading");
assert.deepEqual([byDevice.PhysicalDisk1.temperature_c, byDevice.PhysicalDisk1.wear_percent], [null, null], "a disk with no counter has none");
});
test("Storage Spaces pools are reported in ZFS's words, with how full they are; one Windows can't judge is left out", () => {
const out = run(`
function pool($n, $h, $size, $alloc) { [pscustomobject]@{ FriendlyName = $n; HealthStatus = $h; Size = $size; AllocatedSize = $alloc } }
ConvertTo-Json -InputObject @(ConvertTo-PoolHealth @(
(pool 'Media' 'Healthy' 1000 250),
(pool 'Backups' 'Warning' 1000 900),
(pool 'Broken' 'Unhealthy' 0 0),
(pool 'Strange' 'Unknown' 1000 1)
)) -Compress
`) as { name: string; health: string; capacity_percent: number | null }[];
assert.deepEqual(out.map((p) => [p.name, p.health, p.capacity_percent]), [["Media", "ONLINE", 25], ["Backups", "DEGRADED", 90], ["Broken", "FAULTED", null]]);
});
test("a maker's placeholders aren't a model or a serial number", () => {
const out = run(`
$t = @('To Be Filled By O.E.M.', 'Default string', 'System Serial Number', 'System Product Name', '0000000', ' ', 'None', 'Unknown', 'ASUS', 'ROG STRIX B560-F', 'PF2ABCDE', '0A1B')
ConvertTo-Json -InputObject @($t | ForEach-Object { Test-PlaceholderText $_ }) -Compress
`) as boolean[];
assert.deepEqual(out, [true, true, true, true, true, true, true, true, false, false, false, false]);
});
test("on this machine the real collectors answer, in a shape the server reads, and one that can't be read is left out rather than reported well", () => {
const out = run(`
$health = Get-HostHealth @{ IgnoreServices = @() }
$hardware = Get-HardwareInfo
ConvertTo-Json -InputObject @{ health = $health; hardware = $hardware } -Depth 6 -Compress
`) as { health: { failed_units?: unknown[]; smart?: { device: string; healthy: boolean | null }[]; zfs?: unknown[] } | null; hardware: { manufacturer?: string; model?: string; serial?: string } | null };
if (out.health) {
for (const part of ["failed_units", "smart", "zfs"] as const) if (out.health[part] !== undefined) assert.ok(Array.isArray(out.health[part]), `${part} is a list`);
for (const d of out.health.smart ?? []) assert.match(d.device, /^PhysicalDisk\d+$/);
}
if (out.hardware) for (const v of Object.values(out.hardware)) assert.ok(typeof v === "string" && v.length > 0 && v.length <= 150);
});
});
+3 -2
View File
@@ -100,11 +100,12 @@ describe("the Windows agent updating itself", { skip: !havePwsh }, () => {
test("a newer script that matches the checksum replaces the old one, and the token is sent only to the question", async () => {
served = real();
reply = { update: true, version: 3, sha256: sha(served) };
const current = Number(/AgentVersion = (\d+)/.exec(served.toString())![1]);
reply = { update: true, version: current, sha256: sha(served) };
const out = await attempt();
assert.deepEqual(out.content, served, "the installed file is now the served script, byte for byte");
assert.deepEqual(out.leftovers, []);
assert.match(out.stderr, /Updated the agent from version 1 to version 3/);
assert.match(out.stderr, new RegExp(`Updated the agent from version 1 to version ${current}`));
const question = out.asked.find((l) => l.includes("/api/agent/report/update"))!;
assert.match(question, /version=1/);
assert.match(question, /auth=Bearer secret-token/);