Dependencies beyond servers, web push, inventory auto-fill with QR labels, Windows agent health
- Dependencies: servers and integrations can depend on each other; alerts are folded under the root cause, and a failure explained by something down isn't announced - Web push: hand-written VAPID and RFC 8291 encryption, per-device subscriptions, a fifth notification channel, service worker handlers - Inventory: maker/model/serial from the agents fill blank fields, "Refresh from agent", QR label printing and ?asset= deep links - Windows agent (v4) reports failed services, disk health, Storage Spaces pools and hardware identity; Linux agent (v5) reports hardware identity Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
5cc51c57de
commit
ec9c2f73f0
55 files changed
+12553
-65
No files matched your search
@@ -0,0 +1,3 @@
|
||||
ALTER TABLE `servers` ADD `hw_manufacturer` text;--> statement-breakpoint
|
||||
ALTER TABLE `servers` ADD `hw_model` text;--> statement-breakpoint
|
||||
ALTER TABLE `servers` ADD `hw_serial` text;
|
||||
@@ -0,0 +1,10 @@
|
||||
CREATE TABLE `dependencies` (
|
||||
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||
`from_type` text NOT NULL,
|
||||
`from_id` integer NOT NULL,
|
||||
`to_type` text NOT NULL,
|
||||
`to_id` integer NOT NULL,
|
||||
`created_at` text DEFAULT (current_timestamp) NOT NULL
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX `dependencies_edge_idx` ON `dependencies` (`from_type`,`from_id`,`to_type`,`to_id`);
|
||||
@@ -0,0 +1,13 @@
|
||||
CREATE TABLE `push_subscriptions` (
|
||||
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||
`user_id` integer NOT NULL,
|
||||
`endpoint` text NOT NULL,
|
||||
`p256dh` text NOT NULL,
|
||||
`auth` text NOT NULL,
|
||||
`label` text,
|
||||
`created_at` text DEFAULT (current_timestamp) NOT NULL,
|
||||
`last_success_at` text,
|
||||
FOREIGN KEY (`user_id`) REFERENCES `users`(`id`) ON UPDATE no action ON DELETE cascade
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX `push_subscriptions_endpoint_unique` ON `push_subscriptions` (`endpoint`);
|
||||
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
@@ -239,6 +239,27 @@
|
||||
"when": 1792329463659,
|
||||
"tag": "0033_bright_princess_powerful",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 34,
|
||||
"version": "6",
|
||||
"when": 1792340718369,
|
||||
"tag": "0034_lethal_mandarin",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 35,
|
||||
"version": "6",
|
||||
"when": 1792341079438,
|
||||
"tag": "0035_adorable_silhouette",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 36,
|
||||
"version": "6",
|
||||
"when": 1792341456680,
|
||||
"tag": "0036_cheerful_slipstream",
|
||||
"breakpoints": true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -15,6 +15,8 @@ import { updateRunsRouter } from "./routes/updateRuns.js";
|
||||
import { dbBackupsRouter } from "./routes/dbBackups.js";
|
||||
import { incidentsRouter } from "./routes/incidents.js";
|
||||
import { inventoryRouter } from "./routes/inventory.js";
|
||||
import { dependenciesRouter } from "./routes/dependencies.js";
|
||||
import { pushRouter } from "./routes/push.js";
|
||||
import { backupsRouter } from "./routes/backups.js";
|
||||
import { metricsRouter, metricsTokensRouter } from "./routes/metrics.js";
|
||||
import { eventWebhooksRouter } from "./routes/eventWebhooks.js";
|
||||
@@ -106,6 +108,8 @@ export function createApp() {
|
||||
app.use("/api/db-backups", dbBackupsRouter);
|
||||
app.use("/api/incidents", incidentsRouter);
|
||||
app.use("/api/inventory", inventoryRouter);
|
||||
app.use("/api/dependencies", dependenciesRouter);
|
||||
app.use("/api/push", pushRouter);
|
||||
app.use("/api/backups", backupsRouter);
|
||||
app.use("/api/metrics-tokens", metricsTokensRouter);
|
||||
app.use("/api/event-webhooks", eventWebhooksRouter);
|
||||
|
||||
@@ -103,6 +103,41 @@ export const maintenanceAliases = sqliteTable(
|
||||
(table) => [uniqueIndex("maintenance_aliases_unique_idx").on(table.address, table.targetType, table.targetId)],
|
||||
);
|
||||
|
||||
// A browser or phone that asked for push notifications (one row per device). The endpoint is the browser's push service address, which is
|
||||
// itself a secret: whoever has it with the keys can write to that device. Removed when the push service says it is gone, or the user signs out of it.
|
||||
export const pushSubscriptions = sqliteTable("push_subscriptions", {
|
||||
id: integer("id").primaryKey({ autoIncrement: true }),
|
||||
userId: integer("user_id")
|
||||
.notNull()
|
||||
.references(() => users.id, { onDelete: "cascade" }),
|
||||
endpoint: text("endpoint").notNull().unique(),
|
||||
p256dh: text("p256dh").notNull(),
|
||||
auth: text("auth").notNull(),
|
||||
/** What the person calls it, or the browser it is. */
|
||||
label: text("label"),
|
||||
createdAt: text("created_at")
|
||||
.notNull()
|
||||
.default(sql`(current_timestamp)`),
|
||||
lastSuccessAt: text("last_success_at"),
|
||||
});
|
||||
|
||||
// "A needs B", for things that can be down: a server or an integration each. Polymorphic like maintenance_windows (no foreign key); an edge whose
|
||||
// end was deleted is ignored. When B is down, A's alert is folded under B's — see services/dependencies.ts.
|
||||
export const dependencies = sqliteTable(
|
||||
"dependencies",
|
||||
{
|
||||
id: integer("id").primaryKey({ autoIncrement: true }),
|
||||
fromType: text("from_type").notNull(), // server | integration
|
||||
fromId: integer("from_id").notNull(),
|
||||
toType: text("to_type").notNull(),
|
||||
toId: integer("to_id").notNull(),
|
||||
createdAt: text("created_at")
|
||||
.notNull()
|
||||
.default(sql`(current_timestamp)`),
|
||||
},
|
||||
(table) => [uniqueIndex("dependencies_edge_idx").on(table.fromType, table.fromId, table.toType, table.toId)],
|
||||
);
|
||||
|
||||
// A short history of each agent-reported server's load, memory and disk use, in percent — see services/serverMetrics.ts.
|
||||
// `metric` is "cpu", "mem" or "disk:<mount point>".
|
||||
export const serverMetrics = sqliteTable(
|
||||
@@ -300,6 +335,11 @@ export const servers = sqliteTable("servers", {
|
||||
// What the agent says about the machine's own health, as JSON: {failedUnits, smart, zfs, reportedAt} — see services/hostHealth.ts. Null until it says.
|
||||
hostHealth: text("host_health"),
|
||||
|
||||
// What the machine is, as its firmware says (agents report it): maker, model and serial number. Used to fill in the inventory.
|
||||
hwManufacturer: text("hw_manufacturer"),
|
||||
hwModel: text("hw_model"),
|
||||
hwSerial: text("hw_serial"),
|
||||
|
||||
// This server's own alert thresholds, as JSON: {serverOfflineMinutes, diskUsagePercent, diskForecastDays, backupStaleHours, mounts: {"/data": {percent, ignore}}} — see
|
||||
// services/serverThresholds.ts. Whatever isn't there follows Settings. Null when it follows Settings entirely.
|
||||
thresholds: text("thresholds"),
|
||||
|
||||
@@ -52,7 +52,21 @@ const healthSchema = z.object({
|
||||
.catch(undefined),
|
||||
});
|
||||
|
||||
// What the machine is. Each part is optional and a malformed one is dropped, like the rest.
|
||||
const hardwareSchema = z.object({
|
||||
manufacturer: text(100).optional().catch(undefined),
|
||||
model: text(150).optional().catch(undefined),
|
||||
serial: text(100).optional().catch(undefined),
|
||||
});
|
||||
|
||||
const systemSchema = z.object({
|
||||
hardware: z
|
||||
.unknown()
|
||||
.optional()
|
||||
.transform((h) => {
|
||||
const parsed = hardwareSchema.safeParse(h);
|
||||
return parsed.success && Object.values(parsed.data).some((v) => typeof v === "string" && v.trim()) ? parsed.data : undefined;
|
||||
}),
|
||||
health: z
|
||||
.unknown()
|
||||
.optional()
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
import { Router } from "express";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { z } from "zod";
|
||||
import { db } from "../db/client.js";
|
||||
import { dependencies } from "../db/schema.js";
|
||||
import { requireAuth, requireRole } from "../auth/middleware.js";
|
||||
import { recordAudit } from "../services/audit.js";
|
||||
import { invalidateAlertsCache } from "../services/alerts.js";
|
||||
import { loadGraph, nodeKey, nodeTypes, wouldLoop } from "../services/dependencies.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
|
||||
export const dependenciesRouter = Router();
|
||||
|
||||
dependenciesRouter.use(requireAuth);
|
||||
|
||||
const edgeSchema = z.object({
|
||||
fromType: z.enum(nodeTypes),
|
||||
fromId: z.number().int().positive(),
|
||||
toType: z.enum(nodeTypes),
|
||||
toId: z.number().int().positive(),
|
||||
});
|
||||
|
||||
// Everything signed in people may see: what needs what, with the "runs on" links of servers alongside (those are set on the server itself).
|
||||
dependenciesRouter.get("/", asyncHandler(async (_req, res) => {
|
||||
const graph = await loadGraph();
|
||||
const written = new Set(graph.rows.map((r) => `${nodeKey(r.fromType, r.fromId)}>${nodeKey(r.toType, r.toId)}`));
|
||||
const runsOn: { from: string; to: string }[] = [];
|
||||
for (const [from, tos] of graph.edges) for (const to of tos) if (!written.has(`${from}>${to}`)) runsOn.push({ from, to });
|
||||
const nodes = [...graph.names.entries()]
|
||||
.map(([key, name]) => {
|
||||
const [type, id] = key.split(":");
|
||||
return { key, type, id: Number(id), name };
|
||||
})
|
||||
.sort((a, b) => a.type.localeCompare(b.type) || a.name.localeCompare(b.name));
|
||||
res.json({
|
||||
dependencies: graph.rows
|
||||
.filter((r) => graph.names.has(nodeKey(r.fromType, r.fromId)) && graph.names.has(nodeKey(r.toType, r.toId)))
|
||||
.map((r) => ({ id: r.id, from: { type: r.fromType, id: r.fromId, name: graph.names.get(nodeKey(r.fromType, r.fromId))! }, to: { type: r.toType, id: r.toId, name: graph.names.get(nodeKey(r.toType, r.toId))! }, createdAt: r.createdAt })),
|
||||
runsOn: runsOn.map((r) => ({ from: { key: r.from, name: graph.names.get(r.from) ?? r.from }, to: { key: r.to, name: graph.names.get(r.to) ?? r.to } })),
|
||||
nodes,
|
||||
});
|
||||
}));
|
||||
|
||||
dependenciesRouter.post("/", requireRole("operator"), asyncHandler(async (req, res) => {
|
||||
const parsed = edgeSchema.safeParse(req.body);
|
||||
if (!parsed.success) return res.status(400).json({ error: "invalid_body", message: "Say what needs what: a server or an integration each.", details: parsed.error.flatten() });
|
||||
const d = parsed.data;
|
||||
const graph = await loadGraph();
|
||||
const from = nodeKey(d.fromType, d.fromId);
|
||||
const to = nodeKey(d.toType, d.toId);
|
||||
if (!graph.names.has(from) || !graph.names.has(to)) return res.status(400).json({ error: "unknown_node", message: "One of those doesn't exist." });
|
||||
if (from === to) return res.status(400).json({ error: "self", message: "Something can't depend on itself." });
|
||||
if (graph.edges.get(from)?.includes(to)) return res.status(409).json({ error: "duplicate", message: "That is already so." });
|
||||
if (wouldLoop(graph.edges, from, to)) {
|
||||
return res.status(409).json({ error: "loop", message: `${graph.names.get(to)} already depends on ${graph.names.get(from)}, so this would make them wait for each other.` });
|
||||
}
|
||||
const [row] = await db.insert(dependencies).values(d).returning();
|
||||
await recordAudit({ actor: req.currentUser!, category: "dependency", action: "create", targetType: "dependency", targetId: row.id, detail: { from: graph.names.get(from), to: graph.names.get(to) } });
|
||||
invalidateAlertsCache();
|
||||
res.status(201).json({ id: row.id });
|
||||
}));
|
||||
|
||||
dependenciesRouter.delete("/:id", requireRole("operator"), asyncHandler(async (req, res) => {
|
||||
const id = /^\d+$/.test(req.params.id) ? Number(req.params.id) : NaN;
|
||||
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
|
||||
const [before] = await db.select().from(dependencies).where(eq(dependencies.id, id)).limit(1);
|
||||
if (!before) return res.status(404).json({ error: "not_found" });
|
||||
const graph = await loadGraph();
|
||||
await db.delete(dependencies).where(eq(dependencies.id, id));
|
||||
await recordAudit({
|
||||
actor: req.currentUser!,
|
||||
category: "dependency",
|
||||
action: "delete",
|
||||
targetType: "dependency",
|
||||
targetId: id,
|
||||
detail: { from: graph.names.get(nodeKey(before.fromType as "server", before.fromId)) ?? `${before.fromType} #${before.fromId}`, to: graph.names.get(nodeKey(before.toType as "server", before.toId)) ?? `${before.toType} #${before.toId}` },
|
||||
});
|
||||
invalidateAlertsCache();
|
||||
res.status(204).end();
|
||||
}));
|
||||
@@ -6,6 +6,7 @@ import { assetKinds, assets, servers } from "../db/schema.js";
|
||||
import { requireAuth, requireRole } from "../auth/middleware.js";
|
||||
import { recordAudit } from "../services/audit.js";
|
||||
import { isRealDate } from "../services/inventory.js";
|
||||
import { blanksToFill, differences, hardwareOf } from "../services/inventoryAutofill.js";
|
||||
import { getSettings, updateSettings } from "../services/settingsStore.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
|
||||
@@ -53,16 +54,32 @@ function values(d: z.infer<typeof assetSchema>) {
|
||||
};
|
||||
}
|
||||
|
||||
/** A linked entry's blank maker, model and serial, from what that server's agent says. */
|
||||
async function fillFromAgent(v: ReturnType<typeof values>) {
|
||||
if (v.serverId === null) return {};
|
||||
const [server] = await db.select({ hwManufacturer: servers.hwManufacturer, hwModel: servers.hwModel, hwSerial: servers.hwSerial }).from(servers).where(eq(servers.id, v.serverId)).limit(1);
|
||||
return blanksToFill(v, hardwareOf(server));
|
||||
}
|
||||
|
||||
const idOf = (v: string) => (/^\d+$/.test(v) ? Number(v) : NaN);
|
||||
|
||||
// Everything signed-in people may see: the hardware, and the price of power the running cost is worked out with.
|
||||
inventoryRouter.get("/", asyncHandler(async (_req, res) => {
|
||||
const [rows, serverRows, settings] = await Promise.all([db.select().from(assets).orderBy(assets.name), db.select({ id: servers.id, name: servers.name }).from(servers), getSettings()]);
|
||||
const serverName = new Map(serverRows.map((s) => [s.id, s.name]));
|
||||
const [rows, serverRows, settings] = await Promise.all([db.select().from(assets).orderBy(assets.name), db.select().from(servers), getSettings()]);
|
||||
const byId = new Map(serverRows.map((s) => [s.id, s]));
|
||||
res.json({
|
||||
assets: rows.map((a) => ({ ...a, server: a.serverId !== null && serverName.has(a.serverId) ? { id: a.serverId, name: serverName.get(a.serverId)! } : null })),
|
||||
assets: rows.map((a) => {
|
||||
const s = a.serverId !== null ? byId.get(a.serverId) : undefined;
|
||||
const hw = hardwareOf(s);
|
||||
return {
|
||||
...a,
|
||||
server: s ? { id: s.id, name: s.name } : null,
|
||||
// What the machine's agent says it is, and what that would change here (for the "refresh from agent" button).
|
||||
agent: hw ? { ...hw, differs: differences(a, hw) } : null,
|
||||
};
|
||||
}),
|
||||
config: settings.inventory,
|
||||
servers: serverRows.sort((a, b) => a.name.localeCompare(b.name)),
|
||||
servers: serverRows.map((s) => ({ id: s.id, name: s.name })).sort((a, b) => a.name.localeCompare(b.name)),
|
||||
});
|
||||
}));
|
||||
|
||||
@@ -89,8 +106,9 @@ inventoryRouter.post("/", requireRole("operator"), asyncHandler(async (req, res)
|
||||
return res.status(400).json({ error: "unknown_server", message: "That server doesn't exist." });
|
||||
}
|
||||
const now = new Date().toISOString();
|
||||
const [row] = await db.insert(assets).values({ ...v, createdAt: now, updatedAt: now }).returning();
|
||||
await recordAudit({ actor: req.currentUser!, category: "inventory", action: "create", targetType: "asset", targetId: row.id, detail: { name: row.name, kind: row.kind } });
|
||||
const filled = await fillFromAgent(v);
|
||||
const [row] = await db.insert(assets).values({ ...v, ...filled, createdAt: now, updatedAt: now }).returning();
|
||||
await recordAudit({ actor: req.currentUser!, category: "inventory", action: "create", targetType: "asset", targetId: row.id, detail: { name: row.name, kind: row.kind, ...(Object.keys(filled).length ? { filledFromAgent: Object.keys(filled) } : {}) } });
|
||||
res.status(201).json({ asset: row });
|
||||
}));
|
||||
|
||||
@@ -104,11 +122,27 @@ inventoryRouter.put("/:id", requireRole("operator"), asyncHandler(async (req, re
|
||||
if (v.serverId !== null && (await db.select({ id: servers.id }).from(servers).where(eq(servers.id, v.serverId))).length === 0) {
|
||||
return res.status(400).json({ error: "unknown_server", message: "That server doesn't exist." });
|
||||
}
|
||||
const [row] = await db.update(assets).set({ ...v, updatedAt: new Date().toISOString() }).where(eq(assets.id, id)).returning();
|
||||
await recordAudit({ actor: req.currentUser!, category: "inventory", action: "update", targetType: "asset", targetId: id, detail: { name: row.name } });
|
||||
const filled = await fillFromAgent(v);
|
||||
const [row] = await db.update(assets).set({ ...v, ...filled, updatedAt: new Date().toISOString() }).where(eq(assets.id, id)).returning();
|
||||
await recordAudit({ actor: req.currentUser!, category: "inventory", action: "update", targetType: "asset", targetId: id, detail: { name: row.name, ...(Object.keys(filled).length ? { filledFromAgent: Object.keys(filled) } : {}) } });
|
||||
res.json({ asset: row });
|
||||
}));
|
||||
|
||||
// Takes what the agent says over what is written here: the one deliberate overwrite (the entry's own fields are otherwise only ever filled when blank).
|
||||
inventoryRouter.post("/:id/refresh-from-agent", requireRole("operator"), asyncHandler(async (req, res) => {
|
||||
const id = idOf(req.params.id);
|
||||
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
|
||||
const [before] = await db.select().from(assets).where(eq(assets.id, id)).limit(1);
|
||||
if (!before) return res.status(404).json({ error: "not_found" });
|
||||
if (before.serverId === null) return res.status(400).json({ error: "not_linked", message: "Link the entry to a server first." });
|
||||
const [server] = await db.select({ hwManufacturer: servers.hwManufacturer, hwModel: servers.hwModel, hwSerial: servers.hwSerial }).from(servers).where(eq(servers.id, before.serverId)).limit(1);
|
||||
const changes = differences(before, hardwareOf(server));
|
||||
if (Object.keys(changes).length === 0) return res.json({ asset: before, changed: [] });
|
||||
const [row] = await db.update(assets).set({ ...changes, updatedAt: new Date().toISOString() }).where(eq(assets.id, id)).returning();
|
||||
await recordAudit({ actor: req.currentUser!, category: "inventory", action: "refresh_from_agent", targetType: "asset", targetId: id, detail: { name: row.name, from: Object.fromEntries(Object.keys(changes).map((k) => [k, (before as any)[k]])), to: changes } });
|
||||
res.json({ asset: row, changed: Object.keys(changes) });
|
||||
}));
|
||||
|
||||
inventoryRouter.delete("/:id", requireRole("operator"), asyncHandler(async (req, res) => {
|
||||
const id = idOf(req.params.id);
|
||||
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
|
||||
|
||||
@@ -63,6 +63,7 @@ privacyRouter.get("/", asyncHandler(async (req, res) => {
|
||||
ntfy: channel(settings.ntfy.enabled, hostOf(settings.ntfy.url)),
|
||||
smtp: channel(settings.smtp.enabled, settings.smtp.host || null),
|
||||
webhook: channel(settings.webhook.enabled, hostOf(settings.webhook.url)),
|
||||
webPush: channel(settings.webPush.enabled, "the push services of your browsers (Google, Mozilla, Apple …)"),
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
import { Router } from "express";
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { z } from "zod";
|
||||
import { db } from "../db/client.js";
|
||||
import { pushSubscriptions } from "../db/schema.js";
|
||||
import { requireAuth } from "../auth/middleware.js";
|
||||
import { recordAudit } from "../services/audit.js";
|
||||
import { getSettings } from "../services/settingsStore.js";
|
||||
import { checkEndpoint, getVapidKeys, sendPush } from "../services/webPush.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
|
||||
export const pushRouter = Router();
|
||||
|
||||
pushRouter.use(requireAuth);
|
||||
|
||||
/** At most this many browsers per person: a phone, a laptop, a tablet and the odd one. */
|
||||
const MAX_PER_USER = 10;
|
||||
|
||||
const subscribeSchema = z.object({
|
||||
endpoint: z.string().url().max(2000),
|
||||
keys: z.object({ p256dh: z.string().min(80).max(100), auth: z.string().min(20).max(30) }),
|
||||
label: z.string().trim().max(80).optional(),
|
||||
});
|
||||
|
||||
const hostOf = (endpoint: string) => {
|
||||
try {
|
||||
return new URL(endpoint).host;
|
||||
} catch {
|
||||
return "";
|
||||
}
|
||||
};
|
||||
|
||||
const off = { error: "push_disabled", message: "Push notifications are switched off. An administrator turns them on under Settings, Notifications." };
|
||||
|
||||
// The key a browser subscribes with. Anyone signed in may ask (their own device is theirs to enrol), once push is on.
|
||||
pushRouter.get("/key", asyncHandler(async (_req, res) => {
|
||||
const { webPush } = await getSettings();
|
||||
if (!webPush.enabled) return res.status(409).json(off);
|
||||
res.json({ publicKey: (await getVapidKeys()).publicKey });
|
||||
}));
|
||||
|
||||
// This person's own browsers.
|
||||
pushRouter.get("/mine", asyncHandler(async (req, res) => {
|
||||
const rows = await db.select().from(pushSubscriptions).where(eq(pushSubscriptions.userId, req.currentUser!.id)).orderBy(pushSubscriptions.id);
|
||||
const { webPush } = await getSettings();
|
||||
res.json({
|
||||
enabled: webPush.enabled,
|
||||
subscriptions: rows.map((r) => ({ id: r.id, label: r.label, host: hostOf(r.endpoint), endpoint: r.endpoint, createdAt: r.createdAt, lastSuccessAt: r.lastSuccessAt })),
|
||||
});
|
||||
}));
|
||||
|
||||
pushRouter.post("/subscribe", asyncHandler(async (req, res) => {
|
||||
const { webPush } = await getSettings();
|
||||
if (!webPush.enabled) return res.status(409).json(off);
|
||||
const parsed = subscribeSchema.safeParse(req.body);
|
||||
if (!parsed.success) return res.status(400).json({ error: "invalid_body", message: "That isn't a push subscription.", details: parsed.error.flatten() });
|
||||
const { endpoint, keys, label } = parsed.data;
|
||||
try {
|
||||
await checkEndpoint(endpoint);
|
||||
} catch (err) {
|
||||
return res.status(400).json({ error: "bad_endpoint", message: err instanceof Error ? err.message : "That push address can't be used." });
|
||||
}
|
||||
const me = req.currentUser!;
|
||||
const mine = await db.select({ id: pushSubscriptions.id, endpoint: pushSubscriptions.endpoint }).from(pushSubscriptions).where(eq(pushSubscriptions.userId, me.id));
|
||||
const existing = mine.find((r) => r.endpoint === endpoint);
|
||||
if (!existing && mine.length >= MAX_PER_USER) return res.status(409).json({ error: "too_many", message: `You already have ${MAX_PER_USER} browsers subscribed. Remove one first.` });
|
||||
// The same browser subscribing again (or after someone else used it) is the same row, now this person's.
|
||||
const [row] = await db
|
||||
.insert(pushSubscriptions)
|
||||
.values({ userId: me.id, endpoint, p256dh: keys.p256dh, auth: keys.auth, label: label || null })
|
||||
.onConflictDoUpdate({ target: pushSubscriptions.endpoint, set: { userId: me.id, p256dh: keys.p256dh, auth: keys.auth, label: label || null } })
|
||||
.returning({ id: pushSubscriptions.id });
|
||||
if (!existing) await recordAudit({ actor: me, category: "push", action: "subscribe", targetType: "push_subscription", targetId: row.id, detail: { host: hostOf(endpoint), label: label || null } });
|
||||
res.status(201).json({ id: row.id });
|
||||
}));
|
||||
|
||||
// Stops a browser being written to. A person removes their own; an administrator may remove anyone's.
|
||||
pushRouter.delete("/:id", asyncHandler(async (req, res) => {
|
||||
const id = /^\d+$/.test(req.params.id) ? Number(req.params.id) : NaN;
|
||||
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
|
||||
const me = req.currentUser!;
|
||||
const [row] = await db.select().from(pushSubscriptions).where(eq(pushSubscriptions.id, id)).limit(1);
|
||||
if (!row || (row.userId !== me.id && me.role !== "admin")) return res.status(404).json({ error: "not_found" });
|
||||
await db.delete(pushSubscriptions).where(and(eq(pushSubscriptions.id, id)));
|
||||
await recordAudit({ actor: me, category: "push", action: "unsubscribe", targetType: "push_subscription", targetId: id, detail: { host: hostOf(row.endpoint), label: row.label } });
|
||||
res.status(204).end();
|
||||
}));
|
||||
|
||||
// A message to this person's own browsers, to see that it works.
|
||||
pushRouter.post("/test", asyncHandler(async (req, res) => {
|
||||
const { webPush } = await getSettings();
|
||||
if (!webPush.enabled) return res.status(409).json(off);
|
||||
const mine = await db.select().from(pushSubscriptions).where(eq(pushSubscriptions.userId, req.currentUser!.id));
|
||||
if (mine.length === 0) return res.status(400).json({ error: "none", message: "This browser isn't subscribed yet." });
|
||||
const outcome = await sendPush({ title: "Homelab Manager — Test", body: "Push notifications are working.", url: "/settings/notifications", tag: "test" }, mine);
|
||||
res.json(outcome);
|
||||
}));
|
||||
@@ -64,6 +64,7 @@ const updateSchema = z.object({
|
||||
.partial()
|
||||
.optional(),
|
||||
webhook: z.object({ enabled: z.boolean(), url: z.string(), secret: z.string() }).partial().optional(),
|
||||
webPush: z.object({ enabled: z.boolean(), contact: z.string().trim().max(200) }).partial().optional(),
|
||||
notifications: z
|
||||
.object({
|
||||
dnsAdd: z.boolean(),
|
||||
|
||||
@@ -8,7 +8,9 @@
|
||||
* 2 — reports pending updates, and its own version
|
||||
* 3 — Linux: the package manager is asked outside the systemd sandbox, so dnf and zypper can answer too.
|
||||
* Windows: can replace itself when an administrator has switched automatic updates on.
|
||||
* 4 — Linux: also reports failed services, disk health (SMART) and ZFS pools, reads pacman, and can replace itself too
|
||||
* 4 — Linux: also reports failed services, disk health (SMART) and ZFS pools, reads pacman, and can replace itself too.
|
||||
* Windows: also reports failed services, disk health and Storage Spaces pools, and what the machine is (maker, model, serial)
|
||||
* 5 — Linux: also says what the machine is (maker, model, serial), for the inventory
|
||||
*/
|
||||
import { createHash } from "node:crypto";
|
||||
import { readFileSync } from "node:fs";
|
||||
|
||||
@@ -33,7 +33,7 @@ import { computeSecretStatus } from "./secretStatus.js";
|
||||
import { getFailingSources } from "./integrationHealthMonitor.js";
|
||||
import { getSettings } from "./settingsStore.js";
|
||||
import { sourceLabel } from "./notify.js";
|
||||
import { coveredByParent, loadParentRows, resolveParents } from "./serverParents.js";
|
||||
import { dependencyCover, integrationTypeCause, takenByType } from "./dependencies.js";
|
||||
import type { Alert, AlertCategory, AlertSeverity, SourceFailure } from "./alertTypes.js";
|
||||
|
||||
export interface AlertsReport {
|
||||
@@ -158,6 +158,10 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
}
|
||||
}
|
||||
|
||||
// The servers found to be offline, for the integration failures that one may explain: the server check fills it in (and it is empty if that check fails).
|
||||
let offlineKnown!: (ids: Set<number>) => void;
|
||||
const offlineServers = new Promise<Set<number>>((resolve) => (offlineKnown = resolve));
|
||||
|
||||
await Promise.all([
|
||||
check("Server and storage health", async () => {
|
||||
const { healthChecks } = await getSettings();
|
||||
@@ -176,17 +180,17 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
const m = /^offline:server:(\d+)$/.exec(c.key);
|
||||
if (m) offlineIds.add(Number(m[1]));
|
||||
}
|
||||
const cause = coveredByParent(offlineIds, resolveParents(await loadParentRows()));
|
||||
const taken = new Map<number, string[]>();
|
||||
for (const [child, parent] of cause) taken.set(parent, [...(taken.get(parent) ?? []), serverNames.get(child) ?? `#${child}`]);
|
||||
// (Or because of a service it needs that is failing, or a server it needs that needs one.)
|
||||
offlineKnown(offlineIds);
|
||||
const cover = await dependencyCover(offlineIds, getFailingSources().map((f) => f.source));
|
||||
for (const c of conditions) {
|
||||
const origin = originOfConditionKey(c.key);
|
||||
const subject = subjectOfConditionKey(c.key);
|
||||
const offlineId = /^offline:server:(\d+)$/.exec(c.key)?.[1];
|
||||
const parentOf = offlineId ? cause.get(Number(offlineId)) : undefined;
|
||||
const parentOf = offlineId ? cover.cause.get(`server:${offlineId}`) : undefined;
|
||||
push({
|
||||
causedBy: parentOf !== undefined ? { name: serverNames.get(parentOf) ?? `#${parentOf}` } : undefined,
|
||||
affects: offlineId ? taken.get(Number(offlineId))?.sort((a, b) => a.localeCompare(b)) : undefined,
|
||||
causedBy: parentOf !== undefined ? { name: cover.graph.names.get(parentOf) ?? parentOf } : undefined,
|
||||
affects: offlineId ? cover.taken.get(`server:${offlineId}`)?.sort((a, b) => a.localeCompare(b)) : undefined,
|
||||
category: origin.category,
|
||||
severity: c.severity ?? "warning",
|
||||
type: origin.type,
|
||||
@@ -203,7 +207,7 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
const row = intById.get(Number(m[2]));
|
||||
cannotCheck(`${SOURCES[m[1]].label} “${row?.name ?? `#${m[2]}`}”${m[3] ? ` (node ${m[3]})` : ""}`, "couldn't be read — see the Diagnostic Log", "Server and storage health");
|
||||
}
|
||||
}),
|
||||
}).finally(() => offlineKnown(new Set())),
|
||||
|
||||
check("Automation runs", async () => {
|
||||
const { items, held } = await collectAutomation();
|
||||
@@ -447,8 +451,15 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
|
||||
check("Integration failures", async () => {
|
||||
const { notifications } = await getSettings();
|
||||
for (const f of getFailingSources()) {
|
||||
const failing = getFailingSources();
|
||||
// A failure explained by something it needs that is down is folded under that, like a guest under its host.
|
||||
const cover = failing.length > 0 ? await dependencyCover(await offlineServers, failing.map((f) => f.source)).catch(() => null) : null;
|
||||
for (const f of failing) {
|
||||
const cause = cover ? integrationTypeCause(f.source, cover) : null;
|
||||
const affects = cover ? takenByType(f.source, cover) : [];
|
||||
push({
|
||||
causedBy: cause ? { name: cause.name } : undefined,
|
||||
affects: affects.length > 0 ? affects.sort((a, b) => a.localeCompare(b)) : undefined,
|
||||
category: "integration",
|
||||
severity: f.alerted ? "critical" : "warning",
|
||||
type: f.source,
|
||||
|
||||
@@ -0,0 +1,192 @@
|
||||
/**
|
||||
* What depends on what, beyond "this server runs on that server": a service that needs a server (Gitea on the box it is installed on), a
|
||||
* server that needs a service (a guest that only exists while Proxmox answers). When the thing depended on is down, the things that need it
|
||||
* are down too — and only the one that matters is worth an alert, so the rest are folded under it: shown, not announced.
|
||||
*
|
||||
* The graph is the "runs on" links of servers (set by hand or worked out from Proxmox, see serverParents.ts) plus the dependencies written
|
||||
* here. Servers and integrations are the things that can be down and be needed. (DNS providers raise no alerts of their own, so there is
|
||||
* nothing to fold.) An integration is judged per kind of service, as its health is: it is folded only if every enabled instance of that
|
||||
* kind is explained.
|
||||
*/
|
||||
import { eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { dependencies, integrations, servers } from "../db/schema.js";
|
||||
import { getInternalFlag } from "./settingsStore.js";
|
||||
import { loadParentRows, resolveParents, type ParentLink } from "./serverParents.js";
|
||||
|
||||
/** Where the health monitor keeps the problems it is announcing; read here so a service's alert can tell a server it needs is down. */
|
||||
export const HEALTH_STATE_FLAG = "healthActiveConditions";
|
||||
|
||||
export const nodeTypes = ["server", "integration"] as const;
|
||||
export type NodeType = (typeof nodeTypes)[number];
|
||||
|
||||
export const nodeKey = (type: NodeType, id: number) => `${type}:${id}`;
|
||||
|
||||
export interface DependencyEdge {
|
||||
fromType: NodeType;
|
||||
fromId: number;
|
||||
toType: NodeType;
|
||||
toId: number;
|
||||
}
|
||||
|
||||
/** node → the nodes it needs. */
|
||||
export type Edges = Map<string, string[]>;
|
||||
|
||||
/** The graph from the "runs on" links and the written dependencies; an edge to or from something that no longer exists is left out. */
|
||||
export function buildEdges(parentLinks: Map<number, ParentLink>, rows: DependencyEdge[], exists: (key: string) => boolean): Edges {
|
||||
const edges: Edges = new Map();
|
||||
const add = (from: string, to: string) => {
|
||||
if (from === to || !exists(from) || !exists(to)) return;
|
||||
const list = edges.get(from) ?? [];
|
||||
if (!list.includes(to)) edges.set(from, [...list, to]);
|
||||
};
|
||||
for (const [child, link] of parentLinks) add(nodeKey("server", child), nodeKey("server", link.parentId));
|
||||
for (const r of rows) add(nodeKey(r.fromType, r.fromId), nodeKey(r.toType, r.toId));
|
||||
return edges;
|
||||
}
|
||||
|
||||
/** Everything `node` needs, directly or not, nearest first; `looped` if it ends up needing itself. */
|
||||
function ancestorsOf(node: string, edges: Edges): { order: string[]; looped: boolean } {
|
||||
const order: string[] = [];
|
||||
const seen = new Set<string>();
|
||||
const queue = [...(edges.get(node) ?? [])];
|
||||
let looped = false;
|
||||
while (queue.length > 0) {
|
||||
const cur = queue.shift()!;
|
||||
if (cur === node) {
|
||||
looped = true;
|
||||
continue;
|
||||
}
|
||||
if (seen.has(cur)) continue;
|
||||
seen.add(cur);
|
||||
order.push(cur);
|
||||
queue.push(...(edges.get(cur) ?? []));
|
||||
}
|
||||
return { order, looped };
|
||||
}
|
||||
|
||||
/**
|
||||
* Of the nodes that are down, those whose cause is something they need that is down too: each maps to the root cause — a down node that
|
||||
* needs nothing that is down. A loop explains nothing (each would be waiting for the other), so its members stand alone and are announced.
|
||||
*/
|
||||
export function coveredNodes(down: Set<string>, edges: Edges): Map<string, string> {
|
||||
const covered = new Map<string, string>();
|
||||
for (const node of [...down].sort()) {
|
||||
const { order, looped } = ancestorsOf(node, edges);
|
||||
if (looped) continue;
|
||||
const downAbove = order.filter((a) => down.has(a));
|
||||
const root = downAbove.find((a) => {
|
||||
const above = ancestorsOf(a, edges);
|
||||
return !above.looped && !above.order.some((x) => down.has(x));
|
||||
});
|
||||
if (root !== undefined) covered.set(node, root);
|
||||
}
|
||||
return covered;
|
||||
}
|
||||
|
||||
/** Would `from` needing `to` close a loop? True if `to` is `from` or already needs it, however indirectly. */
|
||||
export function wouldLoop(edges: Edges, from: string, to: string): boolean {
|
||||
return from === to || ancestorsOf(to, edges).order.includes(from) || ancestorsOf(to, edges).looped;
|
||||
}
|
||||
|
||||
export interface Graph {
|
||||
edges: Edges;
|
||||
/** node → its name, for every node that exists. */
|
||||
names: Map<string, string>;
|
||||
rows: (DependencyEdge & { id: number; createdAt: string })[];
|
||||
/** Enabled instances of each kind of integration. */
|
||||
instances: Map<string, number[]>;
|
||||
intRows: { id: number; type: string; enabled: boolean }[];
|
||||
}
|
||||
|
||||
export async function loadGraph(): Promise<Graph> {
|
||||
const [serverRows, intRows, depRows, parentRows] = await Promise.all([
|
||||
db.select({ id: servers.id, name: servers.name }).from(servers),
|
||||
db.select({ id: integrations.id, name: integrations.name, type: integrations.type, enabled: integrations.enabled }).from(integrations),
|
||||
db.select().from(dependencies),
|
||||
loadParentRows(),
|
||||
]);
|
||||
const names = new Map<string, string>();
|
||||
for (const s of serverRows) names.set(nodeKey("server", s.id), s.name);
|
||||
for (const i of intRows) names.set(nodeKey("integration", i.id), i.name);
|
||||
const rows = depRows.map((r) => ({ id: r.id, fromType: r.fromType as NodeType, fromId: r.fromId, toType: r.toType as NodeType, toId: r.toId, createdAt: r.createdAt }));
|
||||
const instances = new Map<string, number[]>();
|
||||
for (const i of intRows) if (i.enabled) instances.set(i.type, [...(instances.get(i.type) ?? []), i.id]);
|
||||
return { edges: buildEdges(resolveParents(parentRows), rows, (k) => names.has(k)), names, rows, instances, intRows };
|
||||
}
|
||||
|
||||
export interface Cover {
|
||||
/** Each node that is down because of another → the root cause. */
|
||||
cause: Map<string, string>;
|
||||
/** Each root cause → the names of what it takes down with it. */
|
||||
taken: Map<string, string[]>;
|
||||
graph: Graph;
|
||||
}
|
||||
|
||||
/** Works out what is down because of what, given the servers that are offline and the kinds of integration that are failing. */
|
||||
export async function dependencyCover(offlineServerIds: Iterable<number>, failingTypes: Iterable<string>): Promise<Cover> {
|
||||
const graph = await loadGraph();
|
||||
const down = new Set<string>();
|
||||
for (const id of offlineServerIds) down.add(nodeKey("server", id));
|
||||
const failing = new Set(failingTypes);
|
||||
for (const [type, ids] of graph.instances) if (failing.has(type)) for (const id of ids) down.add(nodeKey("integration", id));
|
||||
const cause = coveredNodes(down, graph.edges);
|
||||
const taken = new Map<string, string[]>();
|
||||
for (const [child, parent] of cause) taken.set(parent, [...(taken.get(parent) ?? []), graph.names.get(child) ?? child]);
|
||||
return { cause, taken, graph };
|
||||
}
|
||||
|
||||
/** The root cause behind a failing kind of integration, when every enabled instance of it is explained; null when it stands alone. */
|
||||
export function integrationTypeCause(type: string, cover: Cover): { key: string; name: string } | null {
|
||||
const ids = cover.graph.instances.get(type);
|
||||
if (!ids || ids.length === 0) return null;
|
||||
let first: string | null = null;
|
||||
for (const id of ids) {
|
||||
const root = cover.cause.get(nodeKey("integration", id));
|
||||
if (root === undefined) return null;
|
||||
first ??= root;
|
||||
}
|
||||
return first === null ? null : { key: first, name: cover.graph.names.get(first) ?? first };
|
||||
}
|
||||
|
||||
/** What a root cause takes down, for a kind of integration: the folded names across all its instances. */
|
||||
export function takenByType(type: string, cover: Cover): string[] {
|
||||
const out: string[] = [];
|
||||
for (const id of cover.graph.instances.get(type) ?? []) out.push(...(cover.taken.get(nodeKey("integration", id)) ?? []));
|
||||
return out;
|
||||
}
|
||||
|
||||
/** The servers the health monitor is announcing as offline (or holding back as explained), from what it last stored. */
|
||||
export async function offlineServerIdsFromState(): Promise<number[]> {
|
||||
try {
|
||||
const raw = await getInternalFlag(HEALTH_STATE_FLAG);
|
||||
if (!raw) return [];
|
||||
const ids: number[] = [];
|
||||
for (const key of Object.keys(JSON.parse(raw) as Record<string, unknown>)) {
|
||||
const m = /^offline:server:(\d+)$/.exec(key);
|
||||
if (m) ids.push(Number(m[1]));
|
||||
}
|
||||
return ids;
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether a kind of integration that has started failing is failing because of something it needs that is down — a server the health
|
||||
* monitor says is offline, or another service that is failing. Never throws: a problem here must not swallow a real outage.
|
||||
*/
|
||||
export async function integrationCausedBy(type: string, failingTypes: string[]): Promise<{ name: string } | null> {
|
||||
try {
|
||||
const cover = await dependencyCover(await offlineServerIdsFromState(), failingTypes);
|
||||
return integrationTypeCause(type, cover);
|
||||
} catch (err) {
|
||||
console.error("[dependencies] couldn't work out whether a failure is explained:", err);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export async function nodeExists(type: NodeType, id: number): Promise<boolean> {
|
||||
if (type === "server") return (await db.select({ id: servers.id }).from(servers).where(eq(servers.id, id)).limit(1)).length > 0;
|
||||
return (await db.select({ id: integrations.id }).from(integrations).where(eq(integrations.id, id)).limit(1)).length > 0;
|
||||
}
|
||||
@@ -8,11 +8,12 @@ import { notifyHealthIssues, notifyHealthRecovered } from "./notify.js";
|
||||
import { recordAudit } from "./audit.js";
|
||||
import { getInternalFlag, getSettings, setInternalFlag } from "./settingsStore.js";
|
||||
import { activeSubjects, subjectOfConditionKey } from "./maintenance.js";
|
||||
import { coveredByParent, loadParentRows, resolveParents } from "./serverParents.js";
|
||||
import { dependencyCover, HEALTH_STATE_FLAG } from "./dependencies.js";
|
||||
import { getFailingSources } from "./integrationHealthMonitor.js";
|
||||
import { hostHealthConditions, parseHostHealth, type HostHealth } from "./hostHealth.js";
|
||||
import { diskLimit, parseThresholds, type ServerThresholds } from "./serverThresholds.js";
|
||||
|
||||
const STATE_FLAG = "healthActiveConditions";
|
||||
const STATE_FLAG = HEALTH_STATE_FLAG;
|
||||
/** After a restart the agents haven't had a chance to report yet (they run every 15 min), so server-derived conditions are held rather than judged. */
|
||||
const STARTUP_GRACE_MS = Number(process.env.HEALTH_STARTUP_GRACE_MS ?? 20 * 60 * 1000); // the environment variable is for tests
|
||||
const PROCESS_START = Date.now();
|
||||
@@ -396,18 +397,20 @@ export async function runHealthCheck(now: number = Date.now()): Promise<{ added:
|
||||
const m = /^offline:server:(\d+)$/.exec(c.key);
|
||||
if (m) offlineIds.add(Number(m[1]));
|
||||
}
|
||||
const cause = coveredByParent(offlineIds, resolveParents(await loadParentRows()));
|
||||
const coveredKeys = new Set([...cause.keys()].map((id) => `offline:server:${id}`));
|
||||
// (Also when it needs a service that is failing, or runs on a server that needs one.)
|
||||
const cover = await dependencyCover(offlineIds, getFailingSources().map((f) => f.source));
|
||||
const coveredKeys = new Set<string>();
|
||||
for (const child of cover.cause.keys()) {
|
||||
const m = /^server:(\d+)$/.exec(child);
|
||||
if (m) coveredKeys.add(`offline:server:${m[1]}`);
|
||||
}
|
||||
const { added: announced, resolved, next, newlyDown, ended } = diffConditions(await loadState(), current, held, await activeSubjects(new Date(now)), coveredKeys);
|
||||
await setInternalFlag(STATE_FLAG, JSON.stringify(next));
|
||||
|
||||
// A host's message says what else it takes with it.
|
||||
const names = new Map(snapshot.servers.map((s) => [s.id, s.name]));
|
||||
const taken = new Map<number, string[]>();
|
||||
for (const [child, parent] of cause) taken.set(parent, [...(taken.get(parent) ?? []), names.get(child) ?? `#${child}`]);
|
||||
const added = announced.map((c) => {
|
||||
const m = /^offline:server:(\d+)$/.exec(c.key);
|
||||
const children = m ? taken.get(Number(m[1])) : undefined;
|
||||
const children = m ? cover.taken.get(`server:${m[1]}`) : undefined;
|
||||
return children?.length ? { ...c, message: `${c.message} — also takes down ${alsoList(children)}` } : c;
|
||||
});
|
||||
|
||||
|
||||
@@ -18,6 +18,7 @@ export interface SmartDisk {
|
||||
/** NVMe and some SSDs say how much of their rated life is used, 0–100 (and past it). */
|
||||
wearPercent: number | null;
|
||||
}
|
||||
/** A storage pool: ZFS on Linux, Storage Spaces on Windows (which reports Healthy, Warning and Unhealthy as ONLINE, DEGRADED and FAULTED). */
|
||||
export interface ZfsPool {
|
||||
name: string;
|
||||
/** ONLINE, DEGRADED, FAULTED, UNAVAIL, SUSPENDED … as zpool says it. */
|
||||
@@ -117,7 +118,7 @@ export function hostHealthConditions(serverId: number, serverName: string, healt
|
||||
}
|
||||
for (const p of health.zfs ?? []) {
|
||||
if (!poolIsFine(p)) {
|
||||
out.push({ key: `zfs:server:${serverId}:${p.name}`, severity: "critical", message: `${serverName}: ZFS pool ${p.name} is ${p.health}` });
|
||||
out.push({ key: `zfs:server:${serverId}:${p.name}`, severity: "critical", message: `${serverName}: storage pool ${p.name} is ${p.health}` });
|
||||
}
|
||||
}
|
||||
return out;
|
||||
|
||||
@@ -6,6 +6,7 @@ import { ensureHistoryStart } from "./uptimeHistory.js";
|
||||
import { getSettings } from "./settingsStore.js";
|
||||
import { isSourceInMaintenance } from "./maintenance.js";
|
||||
import { notifyIntegrationDown, notifyIntegrationRecovered } from "./notify.js";
|
||||
import { integrationCausedBy } from "./dependencies.js";
|
||||
|
||||
interface SourceHealth {
|
||||
consecutiveFailures: number;
|
||||
@@ -96,7 +97,9 @@ export async function trackIntegrationHealth(source: string, ok: boolean): Promi
|
||||
if (notifications.integrationFailureAlerts && !state.alerted && state.consecutiveFailures >= notifications.integrationFailureThreshold) {
|
||||
// During maintenance the failures keep being counted but `alerted` stays false, so if the service is
|
||||
// still failing once the window ends, the very next failed call alerts — a real outage isn't swallowed.
|
||||
if (!(await isSourceInMaintenance(source))) {
|
||||
// Likewise when it needs something that is down (the server it runs on, another service): that one is what gets announced, and if this stays
|
||||
// down once that one is back, the next failed call alerts.
|
||||
if (!(await isSourceInMaintenance(source)) && !(await integrationCausedBy(source, [...getFailingSources().map((f) => f.source), source]))) {
|
||||
state.alerted = true;
|
||||
await notifyIntegrationDown(source, state.consecutiveFailures);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
/**
|
||||
* Filling in the inventory from what an agent has found out about the machine it runs on: maker, model and serial number. Typing a serial
|
||||
* number off a label is the part of an inventory nobody keeps up, so a linked entry gets it from the agent — but only into fields that are
|
||||
* blank, because what a person wrote down is theirs; "Refresh from agent" is the way to overwrite on purpose.
|
||||
*/
|
||||
import { eq, isNull, or, and } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { assets, servers } from "../db/schema.js";
|
||||
|
||||
export interface AgentHardware {
|
||||
manufacturer: string | null;
|
||||
model: string | null;
|
||||
serial: string | null;
|
||||
}
|
||||
|
||||
type Fields = { manufacturer: string | null; model: string | null; serial: string | null };
|
||||
const KEYS = ["manufacturer", "model", "serial"] as const;
|
||||
|
||||
/** What the server's agent said the machine is, or null if it has said nothing. */
|
||||
export function hardwareOf(server: { hwManufacturer: string | null; hwModel: string | null; hwSerial: string | null } | null | undefined): AgentHardware | null {
|
||||
if (!server) return null;
|
||||
const hw = { manufacturer: server.hwManufacturer ?? null, model: server.hwModel ?? null, serial: server.hwSerial ?? null };
|
||||
return hw.manufacturer || hw.model || hw.serial ? hw : null;
|
||||
}
|
||||
|
||||
/** The fields of an entry that are blank and that the agent can fill. Empty when nothing would change. */
|
||||
export function blanksToFill(current: Fields, hw: AgentHardware | null): Partial<Fields> {
|
||||
const out: Partial<Fields> = {};
|
||||
if (!hw) return out;
|
||||
for (const k of KEYS) if (!current[k]?.trim() && hw[k]) out[k] = hw[k];
|
||||
return out;
|
||||
}
|
||||
|
||||
/** The fields where the agent says something different from the entry: what a refresh would change. */
|
||||
export function differences(current: Fields, hw: AgentHardware | null): Partial<Fields> {
|
||||
const out: Partial<Fields> = {};
|
||||
if (!hw) return out;
|
||||
for (const k of KEYS) if (hw[k] && (current[k] ?? "").trim() !== hw[k]) out[k] = hw[k];
|
||||
return out;
|
||||
}
|
||||
|
||||
/** After an agent has reported, fills the blanks of the inventory entries that are this server. */
|
||||
export async function fillInventoryFromServer(serverId: number): Promise<number> {
|
||||
const [server] = await db.select({ hwManufacturer: servers.hwManufacturer, hwModel: servers.hwModel, hwSerial: servers.hwSerial }).from(servers).where(eq(servers.id, serverId)).limit(1);
|
||||
const hw = hardwareOf(server);
|
||||
if (!hw) return 0;
|
||||
// Only entries with something blank are worth reading.
|
||||
const linked = await db
|
||||
.select()
|
||||
.from(assets)
|
||||
.where(and(eq(assets.serverId, serverId), or(isNull(assets.manufacturer), isNull(assets.model), isNull(assets.serial))));
|
||||
let changed = 0;
|
||||
for (const a of linked) {
|
||||
const fill = blanksToFill(a, hw);
|
||||
if (Object.keys(fill).length === 0) continue;
|
||||
await db.update(assets).set({ ...fill, updatedAt: new Date().toISOString() }).where(eq(assets.id, a.id));
|
||||
changed++;
|
||||
}
|
||||
return changed;
|
||||
}
|
||||
@@ -95,7 +95,7 @@ export async function buildMetrics(now: Date = new Date()): Promise<string> {
|
||||
if (d.wearPercent !== null) out.add("homelab_server_disk_wear_percent", "gauge", "How much of an SSD's rated life is used.", d.wearPercent, { server, device: d.device });
|
||||
if (d.temperatureC !== null) out.add("homelab_server_disk_temperature_celsius", "gauge", "A disk's temperature.", d.temperatureC, { server, device: d.device });
|
||||
}
|
||||
for (const p of health?.zfs ?? []) out.add("homelab_server_zfs_pool_online", "gauge", "1 if a ZFS pool is ONLINE, 0 in any other state.", poolIsFine(p) ? 1 : 0, { server, pool: p.name });
|
||||
for (const p of health?.zfs ?? []) out.add("homelab_server_zfs_pool_online", "gauge", "1 if a storage pool (ZFS, or Storage Spaces on Windows) is ONLINE, 0 in any other state.", poolIsFine(p) ? 1 : 0, { server, pool: p.name });
|
||||
out.add("homelab_server_agent_outdated", "gauge", "1 if the server's agent is older than the one this installation would install now.", isAgentOutdated(s, latest) ? 1 : 0, { server });
|
||||
}
|
||||
// Disks on a steady climb, from each server's own history: days until full at the present rate (left out where there is no steady trend).
|
||||
|
||||
@@ -2,6 +2,7 @@ import nodemailer from "nodemailer";
|
||||
import { getSettings, type NotificationEvents } from "./settingsStore.js";
|
||||
import { enqueueNotification, listQueuedNotifications, clearQueuedNotifications } from "./notificationQueue.js";
|
||||
import { emitEvent } from "./eventWebhooks.js";
|
||||
import { sendPush, subscriptionCount } from "./webPush.js";
|
||||
|
||||
// ─── Channel senders (best-effort — failures are logged, never thrown) ──────
|
||||
|
||||
@@ -72,8 +73,18 @@ async function sendWebhook(title: string, message: string) {
|
||||
}
|
||||
}
|
||||
|
||||
async function sendWebPush(title: string, message: string) {
|
||||
const { webPush } = await getSettings();
|
||||
if (!webPush.enabled) return;
|
||||
try {
|
||||
await sendPush({ title, body: message });
|
||||
} catch (err) {
|
||||
console.error("[notify] Web push failed:", err instanceof Error ? err.message : err);
|
||||
}
|
||||
}
|
||||
|
||||
async function sendToChannels(title: string, message: string): Promise<void> {
|
||||
await Promise.all([sendGotify(title, message), sendNtfy(title, message), sendSmtp(title, message), sendWebhook(title, message)]);
|
||||
await Promise.all([sendGotify(title, message), sendNtfy(title, message), sendSmtp(title, message), sendWebhook(title, message), sendWebPush(title, message)]);
|
||||
}
|
||||
|
||||
function currentTimeInZone(timezone: string): { hh: number; mm: number } {
|
||||
@@ -112,8 +123,9 @@ export async function notifyNow(title: string, message: string): Promise<void> {
|
||||
|
||||
/** How many channels are switched on and filled in enough to send, so a "send it now" can say when there is nowhere to send to. */
|
||||
export async function enabledChannelCount(): Promise<number> {
|
||||
const { gotify, ntfy, smtp, webhook } = await getSettings();
|
||||
const { gotify, ntfy, smtp, webhook, webPush } = await getSettings();
|
||||
return [
|
||||
webPush.enabled && (await subscriptionCount()) > 0,
|
||||
gotify.enabled && gotify.url && gotify.token,
|
||||
ntfy.enabled && ntfy.url && ntfy.topic,
|
||||
smtp.enabled && smtp.host && smtp.to && smtp.from,
|
||||
|
||||
@@ -37,6 +37,13 @@ export interface WebhookSettings {
|
||||
secret: string;
|
||||
}
|
||||
|
||||
/** Push notifications to phones and browsers that have subscribed (see services/webPush.ts). Who is subscribed is kept apart, per device. */
|
||||
export interface WebPushSettings {
|
||||
enabled: boolean;
|
||||
/** Where a push service can reach whoever runs this: an email address or an https address (VAPID asks for one). Blank is fine. */
|
||||
contact: string;
|
||||
}
|
||||
|
||||
export interface NotificationEvents {
|
||||
dnsAdd: boolean;
|
||||
dnsUpdate: boolean;
|
||||
@@ -189,6 +196,7 @@ export interface AppSettings {
|
||||
ntfy: NtfySettings;
|
||||
smtp: SmtpSettings;
|
||||
webhook: WebhookSettings;
|
||||
webPush: WebPushSettings;
|
||||
notifications: NotificationEvents;
|
||||
providerColors: ProviderColors;
|
||||
integrationColors: IntegrationColors;
|
||||
@@ -211,6 +219,7 @@ const DEFAULTS: AppSettings = {
|
||||
ntfy: { enabled: false, url: "https://ntfy.sh", topic: "", token: "", priority: 3 },
|
||||
smtp: { enabled: false, host: "", port: 587, secure: false, username: "", password: "", from: "", to: "" },
|
||||
webhook: { enabled: false, url: "", secret: "" },
|
||||
webPush: { enabled: false, contact: "" },
|
||||
notifications: {
|
||||
dnsAdd: true,
|
||||
dnsUpdate: true,
|
||||
|
||||
@@ -5,6 +5,7 @@ import { recordServerMetrics } from "./serverMetrics.js";
|
||||
import { nextUpdateState, type IncomingUpdates } from "./pendingUpdates.js";
|
||||
import { nextHostHealth, type IncomingHostHealth } from "./hostHealth.js";
|
||||
import { recordAudit } from "./audit.js";
|
||||
import { fillInventoryFromServer } from "./inventoryAutofill.js";
|
||||
|
||||
export interface IncomingTask {
|
||||
scheduleType: "cron" | "systemd_timer" | "windows_task";
|
||||
@@ -27,6 +28,8 @@ export interface IncomingSystemInfo {
|
||||
updates?: IncomingUpdates;
|
||||
/** Failed services, disk health, ZFS pools — each part present only if the agent could find it out. */
|
||||
health?: IncomingHostHealth;
|
||||
/** What the machine is, from its firmware: each part present only if it says anything. */
|
||||
hardware?: { manufacturer?: string; model?: string; serial?: string };
|
||||
}
|
||||
|
||||
export interface AgentReport {
|
||||
@@ -115,6 +118,9 @@ export async function syncServerTasks(serverId: number, report: AgentReport) {
|
||||
// Only stored when the agent reported it, so an agent that predates this (or couldn't tell this time) doesn't wipe what it said before.
|
||||
...(system?.updates && before ? nextUpdateState(before, system.updates, now) : {}),
|
||||
...(system?.health && before ? { hostHealth: nextHostHealth(before.hostHealth, system.health, now) } : {}),
|
||||
...(system?.hardware?.manufacturer ? { hwManufacturer: system.hardware.manufacturer } : {}),
|
||||
...(system?.hardware?.model ? { hwModel: system.hardware.model } : {}),
|
||||
...(system?.hardware?.serial ? { hwSerial: system.hardware.serial } : {}),
|
||||
...(report.hostname ? { hostname: report.hostname } : {}),
|
||||
...(report.agentVersion !== undefined ? { agentVersion: report.agentVersion } : {}),
|
||||
// An agent knows what it runs on better than whoever registered the server did.
|
||||
@@ -146,6 +152,15 @@ export async function syncServerTasks(serverId: number, report: AgentReport) {
|
||||
}
|
||||
}
|
||||
|
||||
// A machine whose agent has said what it is fills in the blanks of its inventory entry. Also best-effort.
|
||||
if (system?.hardware) {
|
||||
try {
|
||||
await fillInventoryFromServer(serverId);
|
||||
} catch (err) {
|
||||
console.error("[inventory] couldn't fill in from the agent:", err);
|
||||
}
|
||||
}
|
||||
|
||||
// Keep a little history of the figures too. A failure here must not lose the report itself, which is already saved.
|
||||
try {
|
||||
await recordServerMetrics(serverId, system, new Date(now));
|
||||
|
||||
@@ -0,0 +1,245 @@
|
||||
/**
|
||||
* Web Push: notifications to a phone or a browser that has asked for them, with no app and no account — the browser's own push service
|
||||
* (Google's, Mozilla's, Apple's) carries the message. Written against the standards directly rather than through a library:
|
||||
*
|
||||
* - RFC 8292 (VAPID): the server proves who it is with a short-lived signed token (ES256), so a push service knows the sender is the
|
||||
* one the subscription was made for.
|
||||
* - RFC 8291 + 8188 (aes128gcm): the message is encrypted for the one browser it is for; the push service only ever carries it.
|
||||
*
|
||||
* The server's key pair is made the first time it is needed and kept (the private half encrypted, like the other credentials).
|
||||
*/
|
||||
import { createCipheriv, createECDH, createPrivateKey, hkdfSync, randomBytes, sign, type KeyObject } from "node:crypto";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { pushSubscriptions } from "../db/schema.js";
|
||||
import { env } from "../env.js";
|
||||
import { decryptSecret, encryptSecret } from "../crypto.js";
|
||||
import { resolveTarget } from "../utils/networkGuard.js";
|
||||
import { getInternalFlag, getSettings, setInternalFlag } from "./settingsStore.js";
|
||||
|
||||
const b64u = (buf: Buffer | Uint8Array) => Buffer.from(buf).toString("base64url");
|
||||
const fromB64u = (s: string) => Buffer.from(s, "base64url");
|
||||
|
||||
// ─── The server's own key pair ─────────────────────────────────────────────
|
||||
|
||||
const VAPID_FLAG = "webPushVapid";
|
||||
const ENCRYPTED = "enc:v1:";
|
||||
|
||||
export interface VapidKeys {
|
||||
/** The uncompressed P-256 point, base64url — what a browser is given to subscribe with. */
|
||||
publicKey: string;
|
||||
/** The private scalar, base64url. */
|
||||
privateKey: string;
|
||||
}
|
||||
|
||||
let cachedKeys: Promise<VapidKeys> | null = null;
|
||||
|
||||
/** The key pair, made on first use and kept. */
|
||||
export function getVapidKeys(): Promise<VapidKeys> {
|
||||
cachedKeys ??= loadOrMakeKeys().catch((err) => {
|
||||
cachedKeys = null;
|
||||
throw err;
|
||||
});
|
||||
return cachedKeys;
|
||||
}
|
||||
|
||||
async function loadOrMakeKeys(): Promise<VapidKeys> {
|
||||
const stored = await getInternalFlag(VAPID_FLAG);
|
||||
if (stored) {
|
||||
try {
|
||||
const v = JSON.parse(stored) as { publicKey: string; privateKey: string };
|
||||
const privateKey = v.privateKey.startsWith(ENCRYPTED) ? decryptSecret(v.privateKey.slice(ENCRYPTED.length)) : v.privateKey;
|
||||
return { publicKey: v.publicKey, privateKey };
|
||||
} catch {
|
||||
// Unreadable (the encryption key changed): made again below — every browser must then subscribe again, which the page says.
|
||||
}
|
||||
}
|
||||
const ecdh = createECDH("prime256v1");
|
||||
ecdh.generateKeys();
|
||||
const keys: VapidKeys = { publicKey: b64u(ecdh.getPublicKey()), privateKey: b64u(ecdh.getPrivateKey().length === 32 ? ecdh.getPrivateKey() : Buffer.concat([Buffer.alloc(32 - ecdh.getPrivateKey().length), ecdh.getPrivateKey()])) };
|
||||
await setInternalFlag(VAPID_FLAG, JSON.stringify({ publicKey: keys.publicKey, privateKey: env.credentialsEncryptionEnabled ? ENCRYPTED + encryptSecret(keys.privateKey) : keys.privateKey }));
|
||||
// The subscriptions made for the old key can't be used with the new one.
|
||||
await db.delete(pushSubscriptions);
|
||||
return keys;
|
||||
}
|
||||
|
||||
/** For the tests, so one test's keys don't leak into the next. */
|
||||
export function forgetVapidKeysForTests(): void {
|
||||
cachedKeys = null;
|
||||
}
|
||||
|
||||
/** A P-256 private key as a key object, from its raw scalar and public point. */
|
||||
function privateKeyObject(keys: VapidKeys): KeyObject {
|
||||
const pub = fromB64u(keys.publicKey);
|
||||
return createPrivateKey({
|
||||
key: { kty: "EC", crv: "P-256", d: keys.privateKey, x: b64u(pub.subarray(1, 33)), y: b64u(pub.subarray(33, 65)) },
|
||||
format: "jwk",
|
||||
});
|
||||
}
|
||||
|
||||
/** The Authorization header's token: who we are, who it is for, and for how long (RFC 8292). */
|
||||
export function vapidAuthorization(audience: string, subject: string, keys: VapidKeys, now: number = Date.now()): string {
|
||||
const header = b64u(Buffer.from(JSON.stringify({ typ: "JWT", alg: "ES256" })));
|
||||
const claims = b64u(Buffer.from(JSON.stringify({ aud: audience, exp: Math.floor(now / 1000) + 12 * 3600, sub: subject })));
|
||||
const signature = sign("sha256", Buffer.from(`${header}.${claims}`), { key: privateKeyObject(keys), dsaEncoding: "ieee-p1363" });
|
||||
return `vapid t=${header}.${claims}.${b64u(signature)}, k=${keys.publicKey}`;
|
||||
}
|
||||
|
||||
// ─── Encrypting a message for one browser (RFC 8291) ───────────────────────
|
||||
|
||||
export interface PushKeys {
|
||||
/** The browser's public key, base64url. */
|
||||
p256dh: string;
|
||||
/** The browser's 16-byte authentication secret, base64url. */
|
||||
auth: string;
|
||||
}
|
||||
|
||||
const RECORD_SIZE = 4096;
|
||||
|
||||
/**
|
||||
* The body to send: the message encrypted so that only the browser it is for can read it. `ephemeral` and `salt` are chosen fresh each
|
||||
* time; they are parameters only so the standard's own worked example can be checked.
|
||||
*/
|
||||
export function encryptPayload(payload: Buffer, subscription: PushKeys, fixed?: { ephemeral: ECDHLike; salt: Buffer }): Buffer {
|
||||
const uaPublic = fromB64u(subscription.p256dh);
|
||||
const authSecret = fromB64u(subscription.auth);
|
||||
if (uaPublic.length !== 65 || uaPublic[0] !== 4) throw new Error("The browser's public key isn't a P-256 point.");
|
||||
if (authSecret.length !== 16) throw new Error("The browser's authentication secret isn't 16 bytes.");
|
||||
|
||||
const fresh = fixed ? null : createECDH("prime256v1");
|
||||
fresh?.generateKeys();
|
||||
const ecdh: ECDHLike = fixed?.ephemeral ?? fresh!;
|
||||
const asPublic = ecdh.getPublicKey();
|
||||
const salt = fixed?.salt ?? randomBytes(16);
|
||||
const secret = ecdh.computeSecret(uaPublic);
|
||||
|
||||
const keyInfo = Buffer.concat([Buffer.from("WebPush: info\0"), uaPublic, asPublic]);
|
||||
const ikm = Buffer.from(hkdfSync("sha256", secret, authSecret, keyInfo, 32));
|
||||
const cek = Buffer.from(hkdfSync("sha256", ikm, salt, Buffer.from("Content-Encoding: aes128gcm\0"), 16));
|
||||
const nonce = Buffer.from(hkdfSync("sha256", ikm, salt, Buffer.from("Content-Encoding: nonce\0"), 12));
|
||||
|
||||
// One record: the message, then a 0x02 marking it as the last.
|
||||
if (payload.length > RECORD_SIZE - 17 - 1) throw new Error("The message is too long for one push.");
|
||||
const cipher = createCipheriv("aes-128-gcm", cek, nonce);
|
||||
const encrypted = Buffer.concat([cipher.update(Buffer.concat([payload, Buffer.from([2])])), cipher.final(), cipher.getAuthTag()]);
|
||||
|
||||
const header = Buffer.alloc(16 + 4 + 1);
|
||||
salt.copy(header, 0);
|
||||
header.writeUInt32BE(RECORD_SIZE, 16);
|
||||
header.writeUInt8(asPublic.length, 20);
|
||||
return Buffer.concat([header, asPublic, encrypted]);
|
||||
}
|
||||
|
||||
type ECDHLike = Pick<ReturnType<typeof createECDH>, "getPublicKey" | "computeSecret">;
|
||||
|
||||
// ─── Subscriptions ─────────────────────────────────────────────────────────
|
||||
|
||||
let allowLocalEndpoints = false;
|
||||
/** Lets the tests push to a service on this machine, which is otherwise refused. */
|
||||
export function allowLocalPushEndpointsForTests(allow: boolean): void {
|
||||
allowLocalEndpoints = allow;
|
||||
}
|
||||
|
||||
/** A push service is a public internet service with an https address; anything else is refused before we ever connect to it. */
|
||||
export async function checkEndpoint(endpoint: string): Promise<void> {
|
||||
let url: URL;
|
||||
try {
|
||||
url = new URL(endpoint);
|
||||
} catch {
|
||||
throw new Error("The push address isn't a web address.");
|
||||
}
|
||||
if (url.protocol !== "https:" && !(allowLocalEndpoints && url.protocol === "http:")) throw new Error("The push address must be https.");
|
||||
if (url.username || url.password) throw new Error("The push address can't carry a login.");
|
||||
if (allowLocalEndpoints) return;
|
||||
await resolveTarget(url.hostname, { allowPrivate: false });
|
||||
}
|
||||
|
||||
export interface PushOutcome {
|
||||
sent: number;
|
||||
/** Subscriptions the push service said are gone, now removed. */
|
||||
removed: number;
|
||||
failed: number;
|
||||
}
|
||||
|
||||
/** The contact the push services may reach us on (a `mailto:` or an https address), as VAPID asks. */
|
||||
export async function vapidSubject(): Promise<string> {
|
||||
const { webPush } = await getSettings();
|
||||
const contact = webPush.contact.trim();
|
||||
if (/^(mailto:|https:\/\/)/i.test(contact)) return contact;
|
||||
if (contact.includes("@")) return `mailto:${contact}`;
|
||||
return /^https:\/\//.test(env.appBaseUrl) ? env.appBaseUrl : "mailto:homelab@example.invalid";
|
||||
}
|
||||
|
||||
export interface PushMessage {
|
||||
title: string;
|
||||
body: string;
|
||||
/** Where in the app a tap should go. */
|
||||
url?: string;
|
||||
/** A later message with the same tag replaces an earlier one that is still showing. */
|
||||
tag?: string;
|
||||
}
|
||||
|
||||
/** Sends one message to one subscription. Resolves to what the push service said; never throws. */
|
||||
async function deliver(sub: { endpoint: string; p256dh: string; auth: string }, body: Buffer, keys: VapidKeys, subject: string, urgent: boolean): Promise<"sent" | "gone" | "failed"> {
|
||||
try {
|
||||
await checkEndpoint(sub.endpoint);
|
||||
const res = await fetch(sub.endpoint, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
Authorization: vapidAuthorization(new URL(sub.endpoint).origin, subject, keys),
|
||||
"Content-Encoding": "aes128gcm",
|
||||
"Content-Type": "application/octet-stream",
|
||||
TTL: "86400",
|
||||
Urgency: urgent ? "high" : "normal",
|
||||
},
|
||||
body: new Uint8Array(body),
|
||||
signal: AbortSignal.timeout(15_000),
|
||||
});
|
||||
if (res.ok) return "sent";
|
||||
if (res.status === 404 || res.status === 410) return "gone";
|
||||
console.error(`[push] ${new URL(sub.endpoint).host} answered ${res.status}: ${(await res.text().catch(() => "")).slice(0, 200)}`);
|
||||
return "failed";
|
||||
} catch (err) {
|
||||
console.error("[push] couldn't deliver:", err instanceof Error ? err.message : err);
|
||||
return "failed";
|
||||
}
|
||||
}
|
||||
|
||||
/** Sends to the given subscriptions (all of them, if none are given). Subscriptions the push service says are gone are removed. */
|
||||
export async function sendPush(message: PushMessage, only?: { id: number; endpoint: string; p256dh: string; auth: string }[]): Promise<PushOutcome> {
|
||||
const subs = only ?? (await db.select().from(pushSubscriptions));
|
||||
const outcome: PushOutcome = { sent: 0, removed: 0, failed: 0 };
|
||||
if (subs.length === 0) return outcome;
|
||||
const keys = await getVapidKeys();
|
||||
const subject = await vapidSubject();
|
||||
const payload = Buffer.from(JSON.stringify({ title: message.title.slice(0, 120), body: message.body.slice(0, 600), url: message.url ?? "/alerts", tag: message.tag }));
|
||||
await Promise.all(
|
||||
subs.map(async (s) => {
|
||||
let body: Buffer;
|
||||
try {
|
||||
body = encryptPayload(payload, { p256dh: s.p256dh, auth: s.auth });
|
||||
} catch (err) {
|
||||
console.error("[push] a subscription can't be written to:", err instanceof Error ? err.message : err);
|
||||
outcome.failed++;
|
||||
return;
|
||||
}
|
||||
const result = await deliver(s, body, keys, subject, true);
|
||||
if (result === "sent") {
|
||||
outcome.sent++;
|
||||
await db.update(pushSubscriptions).set({ lastSuccessAt: new Date().toISOString() }).where(eq(pushSubscriptions.id, s.id));
|
||||
} else if (result === "gone") {
|
||||
outcome.removed++;
|
||||
await db.delete(pushSubscriptions).where(eq(pushSubscriptions.id, s.id));
|
||||
} else {
|
||||
outcome.failed++;
|
||||
}
|
||||
}),
|
||||
);
|
||||
return outcome;
|
||||
}
|
||||
|
||||
/** How many browsers are subscribed. */
|
||||
export async function subscriptionCount(): Promise<number> {
|
||||
return (await db.select({ id: pushSubscriptions.id }).from(pushSubscriptions)).length;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,212 @@
|
||||
/**
|
||||
* Dependencies between servers and services: what is folded under what when something is down, and the routes that say what needs what.
|
||||
*
|
||||
* Run with: npm test --workspace server
|
||||
*/
|
||||
import { test, describe, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { AddressInfo } from "node:net";
|
||||
import type { Server } from "node:http";
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), "hm-deps-"));
|
||||
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
||||
process.env.SESSION_DIR = join(dir, "sessions");
|
||||
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
||||
process.env.HEALTH_STARTUP_GRACE_MS = "0";
|
||||
|
||||
const express = (await import("express")).default;
|
||||
const { db } = await import("../src/db/client.js");
|
||||
const { users, servers, integrations, auditLog } = await import("../src/db/schema.js");
|
||||
const { runMigrations } = await import("../src/db/migrate.js");
|
||||
const { buildEdges, coveredNodes, wouldLoop, integrationCausedBy, HEALTH_STATE_FLAG } = await import("../src/services/dependencies.js");
|
||||
const { setInternalFlag } = await import("../src/services/settingsStore.js");
|
||||
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
|
||||
const { trackIntegrationHealth } = await import("../src/services/integrationHealthMonitor.js");
|
||||
const { dependenciesRouter } = await import("../src/routes/dependencies.js");
|
||||
const { eq } = await import("drizzle-orm");
|
||||
|
||||
await runMigrations();
|
||||
|
||||
const everything = () => true;
|
||||
const edge = (fromType: "server" | "integration", fromId: number, toType: "server" | "integration", toId: number) => ({ fromType, fromId, toType, toId });
|
||||
|
||||
describe("who is down because of whom", () => {
|
||||
test("a node is explained by the root of what it needs, however far up; things that are up in between don't matter", () => {
|
||||
const edges = buildEdges(new Map(), [edge("integration", 1, "server", 1), edge("integration", 2, "integration", 1), edge("server", 2, "integration", 2)], everything);
|
||||
const covered = coveredNodes(new Set(["server:1", "integration:1", "integration:2", "server:2"]), edges);
|
||||
assert.deepEqual([...covered.entries()].sort(), [["integration:1", "server:1"], ["integration:2", "server:1"], ["server:2", "server:1"]]);
|
||||
// the middle one is up: the one above it still explains the one below
|
||||
const skip = coveredNodes(new Set(["server:1", "server:2"]), edges);
|
||||
assert.deepEqual([...skip.entries()], [["server:2", "server:1"]]);
|
||||
});
|
||||
|
||||
test("the 'runs on' links of servers are part of the graph", () => {
|
||||
const edges = buildEdges(new Map([[2, { parentId: 1, auto: false }]]), [edge("integration", 1, "server", 2)], everything);
|
||||
const covered = coveredNodes(new Set(["server:1", "integration:1"]), edges);
|
||||
assert.equal(covered.get("integration:1"), "server:1");
|
||||
});
|
||||
|
||||
test("something that needs two things is explained by whichever is down; with none down it stands alone", () => {
|
||||
const edges = buildEdges(new Map(), [edge("server", 1, "server", 2), edge("server", 1, "integration", 1)], everything);
|
||||
assert.equal(coveredNodes(new Set(["server:1", "integration:1"]), edges).get("server:1"), "integration:1");
|
||||
assert.equal(coveredNodes(new Set(["server:1"]), edges).size, 0);
|
||||
});
|
||||
|
||||
test("a loop explains nothing, and edges to things that are gone are ignored", () => {
|
||||
const loop = buildEdges(new Map(), [edge("server", 1, "server", 2), edge("server", 2, "server", 1)], everything);
|
||||
assert.equal(coveredNodes(new Set(["server:1", "server:2"]), loop).size, 0);
|
||||
const gone = buildEdges(new Map(), [edge("server", 1, "server", 99)], (k) => k !== "server:99");
|
||||
assert.equal(gone.size, 0);
|
||||
});
|
||||
|
||||
test("a new dependency that would close a loop is seen, including through 'runs on'", () => {
|
||||
const edges = buildEdges(new Map([[2, { parentId: 1, auto: true }]]), [edge("integration", 1, "server", 2)], everything);
|
||||
assert.equal(wouldLoop(edges, "server:1", "integration:1"), true, "the integration needs server 2, which runs on server 1");
|
||||
assert.equal(wouldLoop(edges, "server:1", "server:1"), true);
|
||||
assert.equal(wouldLoop(edges, "integration:1", "server:1"), false, "already needs it, indirectly: not a loop");
|
||||
});
|
||||
});
|
||||
|
||||
describe("through the routes, the alerts and the notifications", () => {
|
||||
let app: Server;
|
||||
let base = "";
|
||||
let who = "operator";
|
||||
let box = 0;
|
||||
let other = 0;
|
||||
let gitea = 0;
|
||||
let semaphore = 0;
|
||||
|
||||
before(async () => {
|
||||
await db.insert(users).values([
|
||||
{ oidcSub: "operator", name: "Olga", role: "operator" },
|
||||
{ oidcSub: "viewer", name: "Vic", role: "viewer" },
|
||||
]);
|
||||
const hoursAgo = (n: number) => new Date(Date.now() - n * 3600_000).toISOString();
|
||||
const made = await db
|
||||
.insert(servers)
|
||||
.values([
|
||||
{ name: "box", apiTokenHash: "b", apiTokenPrefix: "b", lastSeenAt: hoursAgo(3) },
|
||||
{ name: "other", apiTokenHash: "o", apiTokenPrefix: "o", lastSeenAt: new Date().toISOString() },
|
||||
])
|
||||
.returning({ id: servers.id });
|
||||
box = made[0].id;
|
||||
other = made[1].id;
|
||||
const ints = await db
|
||||
.insert(integrations)
|
||||
.values([
|
||||
{ type: "gitea", name: "Gitea", baseUrl: "https://git.lab", config: "{}" },
|
||||
{ type: "semaphore", name: "Semaphore", baseUrl: "https://sem.lab", config: "{}" },
|
||||
])
|
||||
.returning({ id: integrations.id });
|
||||
gitea = ints[0].id;
|
||||
semaphore = ints[1].id;
|
||||
|
||||
const a = express();
|
||||
a.use(express.json());
|
||||
a.use((req, _res, next) => {
|
||||
(req as unknown as { session: unknown }).session = { user: { sub: who } };
|
||||
next();
|
||||
});
|
||||
a.use("/api/dependencies", dependenciesRouter);
|
||||
app = await new Promise<Server>((resolve) => {
|
||||
const s = a.listen(0, "127.0.0.1", () => resolve(s));
|
||||
});
|
||||
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/dependencies`;
|
||||
});
|
||||
|
||||
after(() => {
|
||||
app.close();
|
||||
try {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Windows keeps the database file locked until the process exits.
|
||||
}
|
||||
});
|
||||
|
||||
const send = (method: string, path: string, body?: unknown) => fetch(`${base}${path}`, { method, headers: { "content-type": "application/json" }, body: body === undefined ? undefined : JSON.stringify(body) });
|
||||
let firstId = 0;
|
||||
|
||||
test("an operator says what needs what; it is listed with names", async () => {
|
||||
const res = await send("POST", "/", edge("integration", gitea, "server", box));
|
||||
assert.equal(res.status, 201);
|
||||
firstId = (await res.json()).id;
|
||||
assert.equal((await send("POST", "/", edge("integration", semaphore, "integration", gitea))).status, 201);
|
||||
const list = await (await send("GET", "/")).json();
|
||||
assert.deepEqual(list.dependencies.map((d: { from: { name: string }; to: { name: string } }) => `${d.from.name} > ${d.to.name}`).sort(), ["Gitea > box", "Semaphore > Gitea"]);
|
||||
assert.ok(list.nodes.some((n: { key: string; name: string }) => n.key === `server:${box}` && n.name === "box"));
|
||||
});
|
||||
|
||||
test("it can't be nonsense: not itself, not twice, not a loop, not something that isn't there, not by a viewer", async () => {
|
||||
assert.equal((await send("POST", "/", edge("server", box, "server", box))).status, 400);
|
||||
assert.equal((await send("POST", "/", edge("integration", gitea, "server", box))).status, 409, "twice");
|
||||
const loop = await send("POST", "/", edge("server", box, "integration", semaphore));
|
||||
assert.equal(loop.status, 409, "the server would wait for the thing that waits for it");
|
||||
assert.match((await loop.json()).message, /wait for each other/);
|
||||
assert.equal((await send("POST", "/", edge("server", 99999, "server", box))).status, 400);
|
||||
assert.equal((await send("POST", "/", { fromType: "dns_provider", fromId: 1, toType: "server", toId: box })).status, 400);
|
||||
who = "viewer";
|
||||
assert.equal((await send("POST", "/", edge("server", other, "server", box))).status, 403);
|
||||
assert.equal((await send("GET", "/")).status, 200, "everyone signed in can look");
|
||||
who = "operator";
|
||||
});
|
||||
|
||||
test("when the server is down and both services are failing, only the server is announced; the services are held back under it", async () => {
|
||||
await trackIntegrationHealth("gitea", false);
|
||||
await trackIntegrationHealth("semaphore", false);
|
||||
invalidateAlertsCache();
|
||||
const report = await getAlerts(true);
|
||||
const offline = report.alerts.find((a) => a.category === "offline" && a.message.startsWith("box"))!;
|
||||
assert.equal(offline.silenced, false);
|
||||
assert.deepEqual(offline.affects, ["Gitea", "Semaphore"]);
|
||||
const failing = report.alerts.filter((a) => a.category === "integration");
|
||||
assert.equal(failing.length, 2);
|
||||
for (const f of failing) {
|
||||
assert.deepEqual(f.causedBy, { name: "box" });
|
||||
assert.equal(f.silenced, true);
|
||||
}
|
||||
assert.equal(failing.find((f) => f.id.includes("gitea"))!.affects, undefined, "only the root says what it takes with it");
|
||||
assert.equal(report.counts.critical + report.counts.warning, 1);
|
||||
assert.equal(report.counts.grouped, 2);
|
||||
});
|
||||
|
||||
test("when the server is back, Gitea stands alone and Semaphore is held back under Gitea", async () => {
|
||||
await db.update(servers).set({ lastSeenAt: new Date().toISOString() }).where(eq(servers.id, box));
|
||||
invalidateAlertsCache();
|
||||
const report = await getAlerts(true);
|
||||
const failing = report.alerts.filter((a) => a.category === "integration");
|
||||
assert.equal(failing.find((f) => f.id.includes("gitea"))!.causedBy, undefined);
|
||||
assert.deepEqual(failing.find((f) => f.id.includes("semaphore"))!.causedBy, { name: "Gitea" });
|
||||
});
|
||||
|
||||
test("a kind of service is folded only if every enabled instance of it is explained", async () => {
|
||||
const [second] = await db.insert(integrations).values({ type: "gitea", name: "Gitea 2", baseUrl: "https://git2.lab", config: "{}" }).returning({ id: integrations.id });
|
||||
await db.update(servers).set({ lastSeenAt: new Date(Date.now() - 3 * 3600_000).toISOString() }).where(eq(servers.id, box));
|
||||
invalidateAlertsCache();
|
||||
const report = await getAlerts(true);
|
||||
assert.equal(report.alerts.find((a) => a.category === "integration" && a.id.includes("gitea"))!.causedBy, undefined, "the second one doesn't need the box");
|
||||
await db.delete(integrations).where(eq(integrations.id, second.id));
|
||||
});
|
||||
|
||||
test("a failure that crosses its threshold while the server it needs is down isn't announced; one that doesn't need it is", async () => {
|
||||
assert.equal(await integrationCausedBy("gitea", ["gitea"]), null, "nothing is known to be down yet");
|
||||
await setInternalFlag(HEALTH_STATE_FLAG, JSON.stringify({ [`offline:server:${box}`]: { message: "box is offline", source: "server" } }));
|
||||
assert.deepEqual(await integrationCausedBy("gitea", ["gitea"]), { key: `server:${box}`, name: "box" });
|
||||
assert.deepEqual(await integrationCausedBy("semaphore", ["gitea", "semaphore"]), { key: `server:${box}`, name: "box" }, "through Gitea");
|
||||
assert.equal(await integrationCausedBy("tailscale", ["tailscale"]), null, "an unrelated kind");
|
||||
await setInternalFlag(HEALTH_STATE_FLAG, JSON.stringify({ [`offline:server:${other}`]: { message: "other is offline", source: "server" } }));
|
||||
assert.equal(await integrationCausedBy("gitea", ["gitea"]), null, "a different server being down explains nothing");
|
||||
});
|
||||
|
||||
test("removing one works, is audited, and the services stand alone again", async () => {
|
||||
assert.equal((await send("DELETE", `/${firstId}`)).status, 204);
|
||||
assert.equal((await send("DELETE", `/${firstId}`)).status, 404);
|
||||
const actions = (await db.select().from(auditLog)).filter((e) => e.category === "dependency").map((e) => e.action);
|
||||
assert.deepEqual(actions.sort(), ["create", "create", "delete"]);
|
||||
invalidateAlertsCache();
|
||||
const report = await getAlerts(true);
|
||||
assert.equal(report.alerts.find((a) => a.category === "integration" && a.id.includes("gitea"))!.causedBy, undefined);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,149 @@
|
||||
/**
|
||||
* Filling in the inventory from what an agent says the machine is: only blanks are filled, a refresh overwrites on purpose, and an agent's
|
||||
* report fills the entries of the machine it came from.
|
||||
*
|
||||
* Run with: npm test --workspace server
|
||||
*/
|
||||
import { test, describe, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { AddressInfo } from "node:net";
|
||||
import type { Server } from "node:http";
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), "hm-invfill-"));
|
||||
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
||||
process.env.SESSION_DIR = join(dir, "sessions");
|
||||
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
||||
process.env.HEALTH_STARTUP_GRACE_MS = "0";
|
||||
|
||||
const express = (await import("express")).default;
|
||||
const { db } = await import("../src/db/client.js");
|
||||
const { users, servers, assets, auditLog } = await import("../src/db/schema.js");
|
||||
const { runMigrations } = await import("../src/db/migrate.js");
|
||||
const { inventoryRouter } = await import("../src/routes/inventory.js");
|
||||
const { agentReportRouter } = await import("../src/routes/agentReport.js");
|
||||
const { hashToken } = await import("../src/services/tokens.js");
|
||||
const { blanksToFill, differences, hardwareOf } = await import("../src/services/inventoryAutofill.js");
|
||||
const { eq } = await import("drizzle-orm");
|
||||
|
||||
await runMigrations();
|
||||
|
||||
describe("what would be filled, and what would change", () => {
|
||||
const hw = { manufacturer: "Dell Inc.", model: "PowerEdge R730", serial: "ABC123" };
|
||||
|
||||
test("only blank fields are filled; what a person wrote stays", () => {
|
||||
assert.deepEqual(blanksToFill({ manufacturer: null, model: "R730 (mine)", serial: " " }, hw), { manufacturer: "Dell Inc.", serial: "ABC123" });
|
||||
assert.deepEqual(blanksToFill({ manufacturer: "x", model: "y", serial: "z" }, hw), {});
|
||||
assert.deepEqual(blanksToFill({ manufacturer: null, model: null, serial: null }, null), {});
|
||||
assert.deepEqual(blanksToFill({ manufacturer: null, model: null, serial: null }, { manufacturer: "Dell", model: null, serial: null }), { manufacturer: "Dell" });
|
||||
});
|
||||
|
||||
test("a refresh would change the fields where the agent says something else, and nothing that already matches", () => {
|
||||
assert.deepEqual(differences({ manufacturer: "Dell Inc.", model: "R730", serial: null }, hw), { model: "PowerEdge R730", serial: "ABC123" });
|
||||
assert.deepEqual(differences(hw, hw), {});
|
||||
assert.deepEqual(differences({ manufacturer: "Dell Inc. ", model: "PowerEdge R730", serial: "ABC123" }, hw), {}, "stray spaces aren't a difference");
|
||||
assert.deepEqual(differences({ manufacturer: "x", model: "y", serial: "z" }, { manufacturer: null, model: null, serial: null }), {}, "an agent that says nothing changes nothing");
|
||||
});
|
||||
|
||||
test("a server that has said nothing has no hardware", () => {
|
||||
assert.equal(hardwareOf(null), null);
|
||||
assert.equal(hardwareOf({ hwManufacturer: null, hwModel: null, hwSerial: null }), null);
|
||||
assert.deepEqual(hardwareOf({ hwManufacturer: "A", hwModel: null, hwSerial: null }), { manufacturer: "A", model: null, serial: null });
|
||||
});
|
||||
});
|
||||
|
||||
describe("through the routes and the agent's report", () => {
|
||||
let app: Server;
|
||||
let base = "";
|
||||
let id = 0;
|
||||
|
||||
before(async () => {
|
||||
await db.insert(users).values([{ oidcSub: "operator", name: "Olle", role: "operator" }]);
|
||||
const [s] = await db.insert(servers).values({ name: "elsa", apiTokenHash: hashToken("tok"), apiTokenPrefix: "t" }).returning();
|
||||
id = s.id;
|
||||
const a = express();
|
||||
a.use("/api/agent/report", agentReportRouter);
|
||||
a.use(express.json());
|
||||
a.use((req, _res, next) => {
|
||||
(req as unknown as { session: unknown }).session = { user: { sub: "operator" } };
|
||||
next();
|
||||
});
|
||||
a.use("/api/inventory", inventoryRouter);
|
||||
app = await new Promise<Server>((resolve) => {
|
||||
const x = a.listen(0, "127.0.0.1", () => resolve(x));
|
||||
});
|
||||
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api`;
|
||||
});
|
||||
|
||||
after(() => {
|
||||
app.close();
|
||||
try {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Windows keeps the database file locked until the process exits.
|
||||
}
|
||||
});
|
||||
|
||||
const report = (hardware: unknown) =>
|
||||
fetch(`${base}/agent/report`, { method: "POST", headers: { "content-type": "application/json", authorization: "Bearer tok" }, body: JSON.stringify({ hostname: "h", os_type: "linux", tasks: [], system: { hardware } }) });
|
||||
const send = (method: string, path: string, body?: unknown) =>
|
||||
fetch(`${base}/inventory${path}`, { method, headers: { "content-type": "application/json" }, body: body === undefined ? undefined : JSON.stringify(body) });
|
||||
const entry = async (assetId: number) => (await db.select().from(assets).where(eq(assets.id, assetId)))[0];
|
||||
|
||||
let linked = 0;
|
||||
let unlinked = 0;
|
||||
|
||||
test("an entry made before the agent has said anything stays blank, then an agent's report fills it", async () => {
|
||||
linked = (await (await send("POST", "/", { name: "elsa's box", kind: "server", serverId: id, model: "my own words" })).json()).asset.id;
|
||||
unlinked = (await (await send("POST", "/", { name: "a switch", kind: "network" })).json()).asset.id;
|
||||
assert.deepEqual([(await entry(linked)).manufacturer, (await entry(linked)).serial], [null, null]);
|
||||
|
||||
assert.equal((await report({ manufacturer: "Dell Inc.", model: "PowerEdge R730", serial: "ABC123" })).status, 202);
|
||||
const row = await entry(linked);
|
||||
assert.deepEqual([row.manufacturer, row.model, row.serial], ["Dell Inc.", "my own words", "ABC123"], "blanks filled, the model a person wrote is kept");
|
||||
const [s] = await db.select().from(servers).where(eq(servers.id, id));
|
||||
assert.deepEqual([s.hwManufacturer, s.hwModel, s.hwSerial], ["Dell Inc.", "PowerEdge R730", "ABC123"]);
|
||||
assert.equal((await entry(unlinked)).manufacturer, null, "an entry that isn't this machine is left alone");
|
||||
});
|
||||
|
||||
test("the list shows what the agent says and what a refresh would change", async () => {
|
||||
const list = (await (await send("GET", "/")).json()).assets;
|
||||
const a = list.find((x: { id: number }) => x.id === linked);
|
||||
assert.deepEqual(a.agent, { manufacturer: "Dell Inc.", model: "PowerEdge R730", serial: "ABC123", differs: { model: "PowerEdge R730" } });
|
||||
assert.equal(list.find((x: { id: number }) => x.id === unlinked).agent, null);
|
||||
});
|
||||
|
||||
test("a new entry linked to a machine that has already reported starts out filled in", async () => {
|
||||
const made = (await (await send("POST", "/", { name: "elsa again", kind: "server", serverId: id })).json()).asset;
|
||||
assert.deepEqual([made.manufacturer, made.model, made.serial], ["Dell Inc.", "PowerEdge R730", "ABC123"]);
|
||||
const edited = (await (await send("PUT", `/${unlinked}`, { name: "a switch", kind: "network", serverId: id })).json()).asset;
|
||||
assert.equal(edited.serial, "ABC123", "linking an existing entry fills its blanks too");
|
||||
});
|
||||
|
||||
test("a report with nothing in it, or garbage, changes nothing", async () => {
|
||||
assert.equal((await report({ manufacturer: " " })).status, 202);
|
||||
assert.equal((await report("nonsense")).status, 202);
|
||||
const [s] = await db.select().from(servers).where(eq(servers.id, id));
|
||||
assert.equal(s.hwSerial, "ABC123");
|
||||
});
|
||||
|
||||
test("refreshing from the agent overwrites on purpose, says what changed, and is in the audit log", async () => {
|
||||
const res = await send("POST", `/${linked}/refresh-from-agent`);
|
||||
assert.equal(res.status, 200);
|
||||
const body = await res.json();
|
||||
assert.deepEqual(body.changed, ["model"]);
|
||||
assert.equal(body.asset.model, "PowerEdge R730");
|
||||
const again = await (await send("POST", `/${linked}/refresh-from-agent`)).json();
|
||||
assert.deepEqual(again.changed, [], "nothing left to change");
|
||||
const log = (await db.select().from(auditLog)).filter((e) => e.action === "refresh_from_agent");
|
||||
assert.equal(log.length, 1, "only a refresh that changed something is recorded");
|
||||
});
|
||||
|
||||
test("an entry without a server, or one that isn't there, can't be refreshed", async () => {
|
||||
const lone = (await (await send("POST", "/", { name: "a shelf", kind: "other" })).json()).asset.id;
|
||||
assert.equal((await send("POST", `/${lone}/refresh-from-agent`)).status, 400);
|
||||
assert.equal((await send("POST", "/99999/refresh-from-agent")).status, 404);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,292 @@
|
||||
/**
|
||||
* Web Push: the encryption against the standard's own worked example, the signed token, the routes, and a message all the way to a stand-in push
|
||||
* service and back out through a stand-in browser.
|
||||
*
|
||||
* Run with: npm test --workspace server
|
||||
*/
|
||||
import { test, describe, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { createDecipheriv, createECDH, createPublicKey, hkdfSync, verify } from "node:crypto";
|
||||
import { createServer, type IncomingMessage, type Server } from "node:http";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { AddressInfo } from "node:net";
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), "hm-push-"));
|
||||
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
||||
process.env.SESSION_DIR = join(dir, "sessions");
|
||||
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
||||
process.env.HEALTH_STARTUP_GRACE_MS = "0";
|
||||
|
||||
const express = (await import("express")).default;
|
||||
const { db } = await import("../src/db/client.js");
|
||||
const { users, pushSubscriptions, auditLog } = await import("../src/db/schema.js");
|
||||
const { runMigrations } = await import("../src/db/migrate.js");
|
||||
const { encryptPayload, vapidAuthorization, getVapidKeys, checkEndpoint, allowLocalPushEndpointsForTests, sendPush } = await import("../src/services/webPush.js");
|
||||
const { updateSettings } = await import("../src/services/settingsStore.js");
|
||||
const { notifyNow, enabledChannelCount } = await import("../src/services/notify.js");
|
||||
const { pushRouter } = await import("../src/routes/push.js");
|
||||
const { eq } = await import("drizzle-orm");
|
||||
|
||||
await runMigrations();
|
||||
|
||||
const b64u = (s: string) => Buffer.from(s, "base64url");
|
||||
|
||||
/** What a browser does with a push it receives (RFC 8291 §3.4), to read what we wrote. */
|
||||
function decrypt(body: Buffer, uaPrivate: Buffer, uaPublic: Buffer, authSecret: Buffer): Buffer {
|
||||
const salt = body.subarray(0, 16);
|
||||
const idlen = body[20];
|
||||
const asPublic = body.subarray(21, 21 + idlen);
|
||||
const ciphertext = body.subarray(21 + idlen);
|
||||
const ecdh = createECDH("prime256v1");
|
||||
ecdh.setPrivateKey(uaPrivate);
|
||||
const secret = ecdh.computeSecret(asPublic);
|
||||
const ikm = Buffer.from(hkdfSync("sha256", secret, authSecret, Buffer.concat([Buffer.from("WebPush: info\0"), uaPublic, asPublic]), 32));
|
||||
const cek = Buffer.from(hkdfSync("sha256", ikm, salt, Buffer.from("Content-Encoding: aes128gcm\0"), 16));
|
||||
const nonce = Buffer.from(hkdfSync("sha256", ikm, salt, Buffer.from("Content-Encoding: nonce\0"), 12));
|
||||
const d = createDecipheriv("aes-128-gcm", cek, nonce);
|
||||
d.setAuthTag(ciphertext.subarray(ciphertext.length - 16));
|
||||
const plain = Buffer.concat([d.update(ciphertext.subarray(0, ciphertext.length - 16)), d.final()]);
|
||||
assert.equal(plain[plain.length - 1], 2, "ends with the last-record marker");
|
||||
return plain.subarray(0, plain.length - 1);
|
||||
}
|
||||
|
||||
/** A stand-in browser: its keys, in the shape a subscription carries them. */
|
||||
function browser() {
|
||||
const ecdh = createECDH("prime256v1");
|
||||
ecdh.generateKeys();
|
||||
const auth = Buffer.from(Array.from({ length: 16 }, (_, i) => i * 7 + 3));
|
||||
return { ecdh, priv: ecdh.getPrivateKey(), pub: ecdh.getPublicKey(), auth, keys: { p256dh: ecdh.getPublicKey().toString("base64url"), auth: auth.toString("base64url") } };
|
||||
}
|
||||
|
||||
describe("the encryption", () => {
|
||||
test("it reproduces the worked example in RFC 8291, byte for byte", () => {
|
||||
const asPrivate = b64u("yfWPiYE-n46HLnH0KqZOF1fJJU3MYrct3AELtAQ-oRw");
|
||||
const ephemeral = createECDH("prime256v1");
|
||||
ephemeral.setPrivateKey(asPrivate);
|
||||
assert.equal(ephemeral.getPublicKey().toString("base64url"), "BP4z9KsN6nGRTbVYI_c7VJSPQTBtkgcy27mlmlMoZIIgDll6e3vCYLocInmYWAmS6TlzAC8wEqKK6PBru3jl7A8");
|
||||
const body = encryptPayload(
|
||||
Buffer.from("When I grow up, I want to be a watermelon"),
|
||||
{ p256dh: "BCVxsr7N_eNgVRqvHtD0zTZsEc6-VV-JvLexhqUzORcxaOzi6-AYWXvTBHm4bjyPjs7Vd8pZGH6SRpkNtoIAiw4", auth: "BTBZMqHH6r4Tts7J_aSIgg" },
|
||||
{ ephemeral, salt: b64u("DGv6ra1nlYgDCS1FRnbzlw") },
|
||||
);
|
||||
assert.equal(
|
||||
body.toString("base64url"),
|
||||
"DGv6ra1nlYgDCS1FRnbzlwAAEABBBP4z9KsN6nGRTbVYI_c7VJSPQTBtkgcy27mlmlMoZIIgDll6e3vCYLocInmYWAmS6TlzAC8wEqKK6PBru3jl7A_yl95bQpu6cVPTpK4Mqgkf1CXztLVBSt2Ks3oZwbuwXPXLWyouBWLVWGNWQexSgSxsj_Qulcy4a-fN",
|
||||
);
|
||||
});
|
||||
|
||||
test("the browser it is for can read it, and it is different every time", () => {
|
||||
const b = browser();
|
||||
const one = encryptPayload(Buffer.from("hello"), b.keys);
|
||||
const two = encryptPayload(Buffer.from("hello"), b.keys);
|
||||
assert.notDeepEqual(one, two, "a fresh key and salt each time");
|
||||
assert.equal(decrypt(one, b.priv, b.pub, b.auth).toString(), "hello");
|
||||
const other = browser();
|
||||
assert.throws(() => decrypt(one, other.priv, other.pub, other.auth), "someone else's browser can't");
|
||||
});
|
||||
|
||||
test("keys that aren't a browser's, or a message too long for one push, are refused", () => {
|
||||
const b = browser();
|
||||
assert.throws(() => encryptPayload(Buffer.from("x"), { p256dh: "AAAA", auth: b.keys.auth }), /P-256/);
|
||||
assert.throws(() => encryptPayload(Buffer.from("x"), { p256dh: b.keys.p256dh, auth: "AAAA" }), /16 bytes/);
|
||||
assert.throws(() => encryptPayload(Buffer.alloc(4100), b.keys), /too long/);
|
||||
});
|
||||
});
|
||||
|
||||
describe("the signed token", () => {
|
||||
test("it names who it is for, who sent it and when it ends, and the signature checks out against the public key", async () => {
|
||||
const keys = await getVapidKeys();
|
||||
const header = vapidAuthorization("https://push.example", "mailto:me@example.com", keys, 1_700_000_000_000);
|
||||
const m = /^vapid t=([\w-]+)\.([\w-]+)\.([\w-]+), k=([\w-]+)$/.exec(header)!;
|
||||
assert.ok(m, header);
|
||||
assert.equal(m[4], keys.publicKey);
|
||||
assert.deepEqual(JSON.parse(Buffer.from(m[1], "base64url").toString()), { typ: "JWT", alg: "ES256" });
|
||||
assert.deepEqual(JSON.parse(Buffer.from(m[2], "base64url").toString()), { aud: "https://push.example", exp: 1_700_000_000 + 12 * 3600, sub: "mailto:me@example.com" });
|
||||
const pub = b64u(keys.publicKey);
|
||||
const key = createPublicKey({ key: { kty: "EC", crv: "P-256", x: pub.subarray(1, 33).toString("base64url"), y: pub.subarray(33, 65).toString("base64url") }, format: "jwk" });
|
||||
assert.equal(verify("sha256", Buffer.from(`${m[1]}.${m[2]}`), { key, dsaEncoding: "ieee-p1363" }, b64u(m[3])), true);
|
||||
assert.equal(b64u(m[3]).length, 64, "the signature is the plain r and s, as JWT wants");
|
||||
});
|
||||
|
||||
test("the server's key pair is kept: the same one comes back", async () => {
|
||||
const a = await getVapidKeys();
|
||||
const b = await getVapidKeys();
|
||||
assert.deepEqual(a, b);
|
||||
assert.equal(b64u(a.publicKey).length, 65);
|
||||
assert.equal(b64u(a.privateKey).length, 32);
|
||||
});
|
||||
});
|
||||
|
||||
describe("which addresses are push services", () => {
|
||||
test("https only, no login, and never an address on a private network", async () => {
|
||||
await assert.rejects(checkEndpoint("http://fcm.googleapis.com/x"), /https/);
|
||||
await assert.rejects(checkEndpoint("https://user:pw@fcm.googleapis.com/x"), /login/);
|
||||
await assert.rejects(checkEndpoint("not a url"), /web address/);
|
||||
await assert.rejects(checkEndpoint("https://127.0.0.1/x"));
|
||||
await assert.rejects(checkEndpoint("https://192.168.1.10/x"));
|
||||
await assert.rejects(checkEndpoint("https://169.254.169.254/latest"));
|
||||
await assert.rejects(checkEndpoint("https://localhost/x"));
|
||||
});
|
||||
});
|
||||
|
||||
describe("through the routes, the channel and a push service", () => {
|
||||
let app: Server;
|
||||
let base = "";
|
||||
let who = "ada";
|
||||
let pushService: Server;
|
||||
let pushBase = "";
|
||||
const received: { path: string; headers: IncomingMessage["headers"]; body: Buffer }[] = [];
|
||||
let respondWith = 201;
|
||||
let ada = 0;
|
||||
|
||||
before(async () => {
|
||||
allowLocalPushEndpointsForTests(true);
|
||||
const made = await db
|
||||
.insert(users)
|
||||
.values([
|
||||
{ oidcSub: "ada", name: "Ada", role: "admin" },
|
||||
{ oidcSub: "vic", name: "Vic", role: "viewer" },
|
||||
])
|
||||
.returning({ id: users.id });
|
||||
ada = made[0].id;
|
||||
pushService = createServer((req, res) => {
|
||||
const chunks: Buffer[] = [];
|
||||
req.on("data", (c) => chunks.push(c));
|
||||
req.on("end", () => {
|
||||
received.push({ path: req.url ?? "", headers: req.headers, body: Buffer.concat(chunks) });
|
||||
res.statusCode = respondWith;
|
||||
res.end();
|
||||
});
|
||||
});
|
||||
await new Promise<void>((r) => pushService.listen(0, "127.0.0.1", r));
|
||||
pushBase = `http://127.0.0.1:${(pushService.address() as AddressInfo).port}`;
|
||||
|
||||
const a = express();
|
||||
a.use(express.json());
|
||||
a.use((req, _res, next) => {
|
||||
(req as unknown as { session: unknown }).session = { user: { sub: who } };
|
||||
next();
|
||||
});
|
||||
a.use("/api/push", pushRouter);
|
||||
app = await new Promise<Server>((resolve) => {
|
||||
const s = a.listen(0, "127.0.0.1", () => resolve(s));
|
||||
});
|
||||
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/push`;
|
||||
});
|
||||
|
||||
after(() => {
|
||||
allowLocalPushEndpointsForTests(false);
|
||||
app.close();
|
||||
pushService.close();
|
||||
try {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Windows keeps the database file locked until the process exits.
|
||||
}
|
||||
});
|
||||
|
||||
const send = (method: string, path: string, body?: unknown) => fetch(`${base}${path}`, { method, headers: { "content-type": "application/json" }, body: body === undefined ? undefined : JSON.stringify(body) });
|
||||
const mine = browser();
|
||||
const vics = browser();
|
||||
let adaSubId = 0;
|
||||
|
||||
test("while push is off nobody can subscribe, and there is no key to subscribe with", async () => {
|
||||
assert.equal((await send("GET", "/key")).status, 409);
|
||||
assert.equal((await send("POST", "/subscribe", { endpoint: `${pushBase}/a`, keys: mine.keys })).status, 409);
|
||||
assert.equal(await enabledChannelCount(), 0);
|
||||
});
|
||||
|
||||
test("with it on, a browser gets the key and subscribes; the same browser again is the same subscription", async () => {
|
||||
await updateSettings({ webPush: { enabled: true, contact: "me@example.com" } });
|
||||
const key = await (await send("GET", "/key")).json();
|
||||
assert.equal(key.publicKey, (await getVapidKeys()).publicKey);
|
||||
const res = await send("POST", "/subscribe", { endpoint: `${pushBase}/ada-phone`, keys: mine.keys, label: "Ada's phone" });
|
||||
assert.equal(res.status, 201);
|
||||
adaSubId = (await res.json()).id;
|
||||
assert.equal((await send("POST", "/subscribe", { endpoint: `${pushBase}/ada-phone`, keys: mine.keys, label: "Ada's phone" })).status, 201);
|
||||
assert.equal((await db.select().from(pushSubscriptions)).length, 1);
|
||||
const list = await (await send("GET", "/mine")).json();
|
||||
assert.equal(list.subscriptions.length, 1);
|
||||
assert.equal(list.subscriptions[0].label, "Ada's phone");
|
||||
assert.equal((await db.select().from(auditLog)).filter((e) => e.category === "push" && e.action === "subscribe").length, 1, "only the first time");
|
||||
});
|
||||
|
||||
test("a subscription that isn't one is refused", async () => {
|
||||
assert.equal((await send("POST", "/subscribe", { endpoint: "nonsense", keys: mine.keys })).status, 400);
|
||||
assert.equal((await send("POST", "/subscribe", { endpoint: `${pushBase}/x`, keys: { p256dh: "short", auth: "short" } })).status, 400);
|
||||
allowLocalPushEndpointsForTests(false);
|
||||
const refused = await send("POST", "/subscribe", { endpoint: "https://10.0.0.5/push", keys: mine.keys });
|
||||
allowLocalPushEndpointsForTests(true);
|
||||
assert.equal(refused.status, 400, "a private address is never a push service");
|
||||
});
|
||||
|
||||
test("a message through the channels is encrypted for the browser, signed, and addressed to the push service", async () => {
|
||||
received.length = 0;
|
||||
assert.equal(await enabledChannelCount(), 1, "push counts as a channel once someone is subscribed");
|
||||
await notifyNow("Server down", "elsa has not reported for 3 hours");
|
||||
assert.equal(received.length, 1);
|
||||
const r = received[0];
|
||||
assert.equal(r.path, "/ada-phone");
|
||||
assert.equal(r.headers["content-encoding"], "aes128gcm");
|
||||
assert.equal(r.headers.ttl, "86400");
|
||||
assert.match(String(r.headers.authorization), /^vapid t=[\w-]+\.[\w-]+\.[\w-]+, k=[\w-]+$/);
|
||||
const claims = JSON.parse(Buffer.from(String(r.headers.authorization).split(".")[1], "base64url").toString());
|
||||
assert.equal(claims.aud, pushBase);
|
||||
assert.equal(claims.sub, "mailto:me@example.com");
|
||||
const message = JSON.parse(decrypt(r.body, mine.priv, mine.pub, mine.auth).toString());
|
||||
assert.deepEqual(message, { title: "Server down", body: "elsa has not reported for 3 hours", url: "/alerts" });
|
||||
assert.notEqual((await db.select().from(pushSubscriptions))[0].lastSuccessAt, null);
|
||||
});
|
||||
|
||||
test("with push switched off nothing is sent", async () => {
|
||||
await updateSettings({ webPush: { enabled: false } });
|
||||
received.length = 0;
|
||||
await notifyNow("Server down", "x");
|
||||
assert.equal(received.length, 0);
|
||||
await updateSettings({ webPush: { enabled: true } });
|
||||
});
|
||||
|
||||
test("a push service that says the browser is gone has that subscription removed; one that is merely failing keeps it", async () => {
|
||||
const rows = await db.select().from(pushSubscriptions);
|
||||
respondWith = 500;
|
||||
assert.deepEqual(await sendPush({ title: "t", body: "b" }, rows), { sent: 0, removed: 0, failed: 1 });
|
||||
assert.equal((await db.select().from(pushSubscriptions)).length, 1);
|
||||
respondWith = 410;
|
||||
assert.deepEqual(await sendPush({ title: "t", body: "b" }, rows), { sent: 0, removed: 1, failed: 0 });
|
||||
assert.equal((await db.select().from(pushSubscriptions)).length, 0);
|
||||
respondWith = 201;
|
||||
});
|
||||
|
||||
test("the test button writes to this person's own browsers only", async () => {
|
||||
assert.equal((await send("POST", "/test")).status, 400, "nothing subscribed now");
|
||||
await send("POST", "/subscribe", { endpoint: `${pushBase}/ada-phone`, keys: mine.keys });
|
||||
who = "vic";
|
||||
await send("POST", "/subscribe", { endpoint: `${pushBase}/vic-laptop`, keys: vics.keys });
|
||||
received.length = 0;
|
||||
const res = await send("POST", "/test");
|
||||
assert.deepEqual(await res.json(), { sent: 1, removed: 0, failed: 0 });
|
||||
assert.deepEqual(received.map((r) => r.path), ["/vic-laptop"]);
|
||||
assert.equal(JSON.parse(decrypt(received[0].body, vics.priv, vics.pub, vics.auth).toString()).title, "Homelab Manager — Test");
|
||||
});
|
||||
|
||||
test("a person removes their own browsers, not someone else's; an administrator can remove anyone's", async () => {
|
||||
const all = await db.select().from(pushSubscriptions);
|
||||
const adas = all.find((s) => s.userId === ada)!;
|
||||
const vicsRow = all.find((s) => s.userId !== ada)!;
|
||||
who = "vic";
|
||||
assert.equal((await send("DELETE", `/${adas.id}`)).status, 404, "not found, rather than 'not yours'");
|
||||
assert.equal((await send("DELETE", `/${vicsRow.id}`)).status, 204);
|
||||
who = "ada";
|
||||
await send("POST", "/subscribe", { endpoint: `${pushBase}/vic-laptop`, keys: vics.keys });
|
||||
who = "vic";
|
||||
assert.equal((await send("POST", "/subscribe", { endpoint: `${pushBase}/vic-laptop`, keys: vics.keys })).status, 201, "taking a browser over makes it theirs");
|
||||
who = "ada";
|
||||
const taken = (await db.select().from(pushSubscriptions)).find((s) => s.endpoint.endsWith("/vic-laptop"))!;
|
||||
assert.equal((await send("DELETE", `/${taken.id}`)).status, 204, "an administrator may");
|
||||
assert.ok(adaSubId > 0);
|
||||
assert.equal((await db.select().from(auditLog)).filter((e) => e.category === "push" && e.action === "unsubscribe").length, 2);
|
||||
assert.equal((await db.select().from(pushSubscriptions).where(eq(pushSubscriptions.userId, ada))).length, 1);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,126 @@
|
||||
/**
|
||||
* What the Windows agent says about the machine: services that have failed, each disk's health, Storage Spaces pools and what the machine
|
||||
* is. The mapping from what Windows returns is tested with stand-in objects, and the real collectors are run once as a smoke test.
|
||||
*
|
||||
* Run with: npm test --workspace server
|
||||
*/
|
||||
import { test, describe } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { dirname, join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { spawnSync } from "node:child_process";
|
||||
|
||||
const repoRoot = join(dirname(fileURLToPath(import.meta.url)), "..", "..");
|
||||
const script = join(repoRoot, "agent", "windows", "report-tasks.ps1");
|
||||
const probe = process.platform === "win32" ? spawnSync("pwsh", ["-NoProfile", "-Command", "$PSVersionTable.PSVersion.Major"], { encoding: "utf8" }) : null;
|
||||
const havePwsh = probe?.status === 0;
|
||||
|
||||
/** Runs PowerShell with the agent script loaded (without running it) and returns what the snippet printed as JSON. */
|
||||
function run(snippet: string): unknown {
|
||||
const work = mkdtempSync(join(tmpdir(), "hm-winhealth-"));
|
||||
try {
|
||||
const file = join(work, "t.ps1");
|
||||
writeFileSync(file, `$ErrorActionPreference = 'Stop'\n. '${script.replace(/'/g, "''")}'\n${snippet}\n`);
|
||||
const r = spawnSync("pwsh", ["-NoProfile", "-NonInteractive", "-File", file], { encoding: "utf8" });
|
||||
assert.equal(r.stderr.trim(), "", `PowerShell said:\n${r.stderr}\n${r.stdout}`);
|
||||
return JSON.parse(r.stdout.trim());
|
||||
} finally {
|
||||
rmSync(work, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
describe("the Windows agent's health parts", { skip: !havePwsh }, () => {
|
||||
test("it says it is version 4, and stays plain ASCII", () => {
|
||||
const text = readFileSync(script, "utf8");
|
||||
assert.match(text, /^\$script:AgentVersion = 4\s*$/m);
|
||||
assert.ok(!/[^\x00-\x7f]/.test(text));
|
||||
});
|
||||
|
||||
test("a service has failed when it starts by itself, isn't running, and stopped with an error — not for exit 0, never-started, or on request", () => {
|
||||
const out = run(`
|
||||
function svc($n, $d, $mode, $state, $code) { [pscustomobject]@{ Name = $n; DisplayName = $d; StartMode = $mode; State = $state; ExitCode = $code } }
|
||||
$list = @(
|
||||
(svc 'Crashed' 'A crashed service' 'Auto' 'Stopped' 1066),
|
||||
(svc 'Crashed2' 'Another' 'Auto' 'Stopped' 2),
|
||||
(svc 'Finished' 'Did its job' 'Auto' 'Stopped' 0),
|
||||
(svc 'Triggered' 'Starts on a trigger' 'Auto' 'Stopped' 1077),
|
||||
(svc 'OnRequest' 'Manual one' 'Manual' 'Stopped' 1066),
|
||||
(svc 'Disabled' 'Disabled one' 'Disabled' 'Stopped' 1066),
|
||||
(svc 'Fine' 'Running' 'Auto' 'Running' 0),
|
||||
(svc 'Starting' 'On its way' 'Auto' 'Start Pending' 1066),
|
||||
(svc 'gupdate' 'Google Update' 'Auto' 'Stopped' 1),
|
||||
(svc 'NoCode' 'No code at all' 'Auto' 'Stopped' $null)
|
||||
)
|
||||
ConvertTo-Json -InputObject @{ all = @(ConvertTo-FailedUnits $list); ignoring = @(ConvertTo-FailedUnits $list @('gupdate*', 'Crashed2')); none = @(ConvertTo-FailedUnits @()) } -Compress
|
||||
`) as { all: { unit: string; description: string }[]; ignoring: { unit: string }[]; none: unknown[] };
|
||||
assert.deepEqual(out.all.map((u) => u.unit), ["Crashed", "Crashed2", "gupdate"]);
|
||||
assert.equal(out.all[0].description, "A crashed service");
|
||||
assert.deepEqual(out.ignoring.map((u) => u.unit), ["Crashed"], "names and patterns in the ignore list are left out");
|
||||
assert.deepEqual(out.none, []);
|
||||
});
|
||||
|
||||
test("a disk passes when Windows says Healthy; Warning and Unhealthy don't; Unknown says nothing; temperature and wear come from the counter where sane", () => {
|
||||
const out = run(`
|
||||
function disk($id, $name, $health, $bus = 'SATA') { [pscustomobject]@{ DeviceId = $id; FriendlyName = $name; HealthStatus = $health; BusType = $bus } }
|
||||
$disks = @(
|
||||
(disk '0' ' Samsung SSD 850 ' 'Healthy'),
|
||||
(disk '1' 'Old HDD' 'Warning'),
|
||||
(disk '2' 'Dead HDD' 'Unhealthy'),
|
||||
(disk '3' 'Mystery' 'Unknown'),
|
||||
(disk '4' 'NVMe' 'Healthy' 'NVMe'),
|
||||
(disk '5' 'Virtual' 'Healthy' 'File Backed Virtual'),
|
||||
(disk '6' 'Odd readings' 'Healthy')
|
||||
)
|
||||
$counters = @{
|
||||
'0' = [pscustomobject]@{ Temperature = 36; Wear = 7 }
|
||||
'4' = [pscustomobject]@{ Temperature = 0; Wear = 0 }
|
||||
'6' = [pscustomobject]@{ Temperature = 4000; Wear = 900 }
|
||||
}
|
||||
ConvertTo-Json -InputObject @(ConvertTo-DiskHealth $disks $counters) -Compress
|
||||
`) as { device: string; model: string; healthy: boolean | null; temperature_c: number | null; wear_percent: number | null }[];
|
||||
const byDevice = Object.fromEntries(out.map((d) => [d.device, d]));
|
||||
assert.deepEqual(Object.keys(byDevice), ["PhysicalDisk0", "PhysicalDisk1", "PhysicalDisk2", "PhysicalDisk3", "PhysicalDisk4", "PhysicalDisk6"], "the virtual disk is left out");
|
||||
assert.deepEqual([byDevice.PhysicalDisk0.healthy, byDevice.PhysicalDisk1.healthy, byDevice.PhysicalDisk2.healthy, byDevice.PhysicalDisk3.healthy], [true, false, false, null]);
|
||||
assert.equal(byDevice.PhysicalDisk0.model, "Samsung SSD 850", "names are trimmed");
|
||||
assert.deepEqual([byDevice.PhysicalDisk0.temperature_c, byDevice.PhysicalDisk0.wear_percent], [36, 7]);
|
||||
assert.deepEqual([byDevice.PhysicalDisk4.temperature_c, byDevice.PhysicalDisk4.wear_percent], [null, 0], "0 degrees isn't a reading, 0% wear is");
|
||||
assert.deepEqual([byDevice.PhysicalDisk6.temperature_c, byDevice.PhysicalDisk6.wear_percent], [null, null], "nonsense is no reading");
|
||||
assert.deepEqual([byDevice.PhysicalDisk1.temperature_c, byDevice.PhysicalDisk1.wear_percent], [null, null], "a disk with no counter has none");
|
||||
});
|
||||
|
||||
test("Storage Spaces pools are reported in ZFS's words, with how full they are; one Windows can't judge is left out", () => {
|
||||
const out = run(`
|
||||
function pool($n, $h, $size, $alloc) { [pscustomobject]@{ FriendlyName = $n; HealthStatus = $h; Size = $size; AllocatedSize = $alloc } }
|
||||
ConvertTo-Json -InputObject @(ConvertTo-PoolHealth @(
|
||||
(pool 'Media' 'Healthy' 1000 250),
|
||||
(pool 'Backups' 'Warning' 1000 900),
|
||||
(pool 'Broken' 'Unhealthy' 0 0),
|
||||
(pool 'Strange' 'Unknown' 1000 1)
|
||||
)) -Compress
|
||||
`) as { name: string; health: string; capacity_percent: number | null }[];
|
||||
assert.deepEqual(out.map((p) => [p.name, p.health, p.capacity_percent]), [["Media", "ONLINE", 25], ["Backups", "DEGRADED", 90], ["Broken", "FAULTED", null]]);
|
||||
});
|
||||
|
||||
test("a maker's placeholders aren't a model or a serial number", () => {
|
||||
const out = run(`
|
||||
$t = @('To Be Filled By O.E.M.', 'Default string', 'System Serial Number', 'System Product Name', '0000000', ' ', 'None', 'Unknown', 'ASUS', 'ROG STRIX B560-F', 'PF2ABCDE', '0A1B')
|
||||
ConvertTo-Json -InputObject @($t | ForEach-Object { Test-PlaceholderText $_ }) -Compress
|
||||
`) as boolean[];
|
||||
assert.deepEqual(out, [true, true, true, true, true, true, true, true, false, false, false, false]);
|
||||
});
|
||||
|
||||
test("on this machine the real collectors answer, in a shape the server reads, and one that can't be read is left out rather than reported well", () => {
|
||||
const out = run(`
|
||||
$health = Get-HostHealth @{ IgnoreServices = @() }
|
||||
$hardware = Get-HardwareInfo
|
||||
ConvertTo-Json -InputObject @{ health = $health; hardware = $hardware } -Depth 6 -Compress
|
||||
`) as { health: { failed_units?: unknown[]; smart?: { device: string; healthy: boolean | null }[]; zfs?: unknown[] } | null; hardware: { manufacturer?: string; model?: string; serial?: string } | null };
|
||||
if (out.health) {
|
||||
for (const part of ["failed_units", "smart", "zfs"] as const) if (out.health[part] !== undefined) assert.ok(Array.isArray(out.health[part]), `${part} is a list`);
|
||||
for (const d of out.health.smart ?? []) assert.match(d.device, /^PhysicalDisk\d+$/);
|
||||
}
|
||||
if (out.hardware) for (const v of Object.values(out.hardware)) assert.ok(typeof v === "string" && v.length > 0 && v.length <= 150);
|
||||
});
|
||||
});
|
||||
@@ -100,11 +100,12 @@ describe("the Windows agent updating itself", { skip: !havePwsh }, () => {
|
||||
|
||||
test("a newer script that matches the checksum replaces the old one, and the token is sent only to the question", async () => {
|
||||
served = real();
|
||||
reply = { update: true, version: 3, sha256: sha(served) };
|
||||
const current = Number(/AgentVersion = (\d+)/.exec(served.toString())![1]);
|
||||
reply = { update: true, version: current, sha256: sha(served) };
|
||||
const out = await attempt();
|
||||
assert.deepEqual(out.content, served, "the installed file is now the served script, byte for byte");
|
||||
assert.deepEqual(out.leftovers, []);
|
||||
assert.match(out.stderr, /Updated the agent from version 1 to version 3/);
|
||||
assert.match(out.stderr, new RegExp(`Updated the agent from version 1 to version ${current}`));
|
||||
const question = out.asked.find((l) => l.includes("/api/agent/report/update"))!;
|
||||
assert.match(question, /version=1/);
|
||||
assert.match(question, /auth=Bearer secret-token/);
|
||||
|
||||
Reference in new issue
Block a user