Dependencies beyond servers, web push, inventory auto-fill with QR labels, Windows agent health

- Dependencies: servers and integrations can depend on each other; alerts are folded under the root cause, and a failure explained by something down isn't announced
- Web push: hand-written VAPID and RFC 8291 encryption, per-device subscriptions, a fifth notification channel, service worker handlers
- Inventory: maker/model/serial from the agents fill blank fields, "Refresh from agent", QR label printing and ?asset= deep links
- Windows agent (v4) reports failed services, disk health, Storage Spaces pools and hardware identity; Linux agent (v5) reports hardware identity

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-18 18:53:30 +02:00
1 parent 5cc51c57de
commit ec9c2f73f0
55 files changed
+12553 -65

No files matched your search

+4
View File
@@ -15,6 +15,8 @@ import { updateRunsRouter } from "./routes/updateRuns.js";
import { dbBackupsRouter } from "./routes/dbBackups.js";
import { incidentsRouter } from "./routes/incidents.js";
import { inventoryRouter } from "./routes/inventory.js";
import { dependenciesRouter } from "./routes/dependencies.js";
import { pushRouter } from "./routes/push.js";
import { backupsRouter } from "./routes/backups.js";
import { metricsRouter, metricsTokensRouter } from "./routes/metrics.js";
import { eventWebhooksRouter } from "./routes/eventWebhooks.js";
@@ -106,6 +108,8 @@ export function createApp() {
app.use("/api/db-backups", dbBackupsRouter);
app.use("/api/incidents", incidentsRouter);
app.use("/api/inventory", inventoryRouter);
app.use("/api/dependencies", dependenciesRouter);
app.use("/api/push", pushRouter);
app.use("/api/backups", backupsRouter);
app.use("/api/metrics-tokens", metricsTokensRouter);
app.use("/api/event-webhooks", eventWebhooksRouter);
+40
View File
@@ -103,6 +103,41 @@ export const maintenanceAliases = sqliteTable(
(table) => [uniqueIndex("maintenance_aliases_unique_idx").on(table.address, table.targetType, table.targetId)],
);
// A browser or phone that asked for push notifications (one row per device). The endpoint is the browser's push service address, which is
// itself a secret: whoever has it with the keys can write to that device. Removed when the push service says it is gone, or the user signs out of it.
export const pushSubscriptions = sqliteTable("push_subscriptions", {
id: integer("id").primaryKey({ autoIncrement: true }),
userId: integer("user_id")
.notNull()
.references(() => users.id, { onDelete: "cascade" }),
endpoint: text("endpoint").notNull().unique(),
p256dh: text("p256dh").notNull(),
auth: text("auth").notNull(),
/** What the person calls it, or the browser it is. */
label: text("label"),
createdAt: text("created_at")
.notNull()
.default(sql`(current_timestamp)`),
lastSuccessAt: text("last_success_at"),
});
// "A needs B", for things that can be down: a server or an integration each. Polymorphic like maintenance_windows (no foreign key); an edge whose
// end was deleted is ignored. When B is down, A's alert is folded under B's — see services/dependencies.ts.
export const dependencies = sqliteTable(
"dependencies",
{
id: integer("id").primaryKey({ autoIncrement: true }),
fromType: text("from_type").notNull(), // server | integration
fromId: integer("from_id").notNull(),
toType: text("to_type").notNull(),
toId: integer("to_id").notNull(),
createdAt: text("created_at")
.notNull()
.default(sql`(current_timestamp)`),
},
(table) => [uniqueIndex("dependencies_edge_idx").on(table.fromType, table.fromId, table.toType, table.toId)],
);
// A short history of each agent-reported server's load, memory and disk use, in percent — see services/serverMetrics.ts.
// `metric` is "cpu", "mem" or "disk:<mount point>".
export const serverMetrics = sqliteTable(
@@ -300,6 +335,11 @@ export const servers = sqliteTable("servers", {
// What the agent says about the machine's own health, as JSON: {failedUnits, smart, zfs, reportedAt} — see services/hostHealth.ts. Null until it says.
hostHealth: text("host_health"),
// What the machine is, as its firmware says (agents report it): maker, model and serial number. Used to fill in the inventory.
hwManufacturer: text("hw_manufacturer"),
hwModel: text("hw_model"),
hwSerial: text("hw_serial"),
// This server's own alert thresholds, as JSON: {serverOfflineMinutes, diskUsagePercent, diskForecastDays, backupStaleHours, mounts: {"/data": {percent, ignore}}} — see
// services/serverThresholds.ts. Whatever isn't there follows Settings. Null when it follows Settings entirely.
thresholds: text("thresholds"),
+14
View File
@@ -52,7 +52,21 @@ const healthSchema = z.object({
.catch(undefined),
});
// What the machine is. Each part is optional and a malformed one is dropped, like the rest.
const hardwareSchema = z.object({
manufacturer: text(100).optional().catch(undefined),
model: text(150).optional().catch(undefined),
serial: text(100).optional().catch(undefined),
});
const systemSchema = z.object({
hardware: z
.unknown()
.optional()
.transform((h) => {
const parsed = hardwareSchema.safeParse(h);
return parsed.success && Object.values(parsed.data).some((v) => typeof v === "string" && v.trim()) ? parsed.data : undefined;
}),
health: z
.unknown()
.optional()
+80
View File
@@ -0,0 +1,80 @@
import { Router } from "express";
import { eq } from "drizzle-orm";
import { z } from "zod";
import { db } from "../db/client.js";
import { dependencies } from "../db/schema.js";
import { requireAuth, requireRole } from "../auth/middleware.js";
import { recordAudit } from "../services/audit.js";
import { invalidateAlertsCache } from "../services/alerts.js";
import { loadGraph, nodeKey, nodeTypes, wouldLoop } from "../services/dependencies.js";
import { asyncHandler } from "../utils/asyncHandler.js";
export const dependenciesRouter = Router();
dependenciesRouter.use(requireAuth);
const edgeSchema = z.object({
fromType: z.enum(nodeTypes),
fromId: z.number().int().positive(),
toType: z.enum(nodeTypes),
toId: z.number().int().positive(),
});
// Everything signed in people may see: what needs what, with the "runs on" links of servers alongside (those are set on the server itself).
dependenciesRouter.get("/", asyncHandler(async (_req, res) => {
const graph = await loadGraph();
const written = new Set(graph.rows.map((r) => `${nodeKey(r.fromType, r.fromId)}>${nodeKey(r.toType, r.toId)}`));
const runsOn: { from: string; to: string }[] = [];
for (const [from, tos] of graph.edges) for (const to of tos) if (!written.has(`${from}>${to}`)) runsOn.push({ from, to });
const nodes = [...graph.names.entries()]
.map(([key, name]) => {
const [type, id] = key.split(":");
return { key, type, id: Number(id), name };
})
.sort((a, b) => a.type.localeCompare(b.type) || a.name.localeCompare(b.name));
res.json({
dependencies: graph.rows
.filter((r) => graph.names.has(nodeKey(r.fromType, r.fromId)) && graph.names.has(nodeKey(r.toType, r.toId)))
.map((r) => ({ id: r.id, from: { type: r.fromType, id: r.fromId, name: graph.names.get(nodeKey(r.fromType, r.fromId))! }, to: { type: r.toType, id: r.toId, name: graph.names.get(nodeKey(r.toType, r.toId))! }, createdAt: r.createdAt })),
runsOn: runsOn.map((r) => ({ from: { key: r.from, name: graph.names.get(r.from) ?? r.from }, to: { key: r.to, name: graph.names.get(r.to) ?? r.to } })),
nodes,
});
}));
dependenciesRouter.post("/", requireRole("operator"), asyncHandler(async (req, res) => {
const parsed = edgeSchema.safeParse(req.body);
if (!parsed.success) return res.status(400).json({ error: "invalid_body", message: "Say what needs what: a server or an integration each.", details: parsed.error.flatten() });
const d = parsed.data;
const graph = await loadGraph();
const from = nodeKey(d.fromType, d.fromId);
const to = nodeKey(d.toType, d.toId);
if (!graph.names.has(from) || !graph.names.has(to)) return res.status(400).json({ error: "unknown_node", message: "One of those doesn't exist." });
if (from === to) return res.status(400).json({ error: "self", message: "Something can't depend on itself." });
if (graph.edges.get(from)?.includes(to)) return res.status(409).json({ error: "duplicate", message: "That is already so." });
if (wouldLoop(graph.edges, from, to)) {
return res.status(409).json({ error: "loop", message: `${graph.names.get(to)} already depends on ${graph.names.get(from)}, so this would make them wait for each other.` });
}
const [row] = await db.insert(dependencies).values(d).returning();
await recordAudit({ actor: req.currentUser!, category: "dependency", action: "create", targetType: "dependency", targetId: row.id, detail: { from: graph.names.get(from), to: graph.names.get(to) } });
invalidateAlertsCache();
res.status(201).json({ id: row.id });
}));
dependenciesRouter.delete("/:id", requireRole("operator"), asyncHandler(async (req, res) => {
const id = /^\d+$/.test(req.params.id) ? Number(req.params.id) : NaN;
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
const [before] = await db.select().from(dependencies).where(eq(dependencies.id, id)).limit(1);
if (!before) return res.status(404).json({ error: "not_found" });
const graph = await loadGraph();
await db.delete(dependencies).where(eq(dependencies.id, id));
await recordAudit({
actor: req.currentUser!,
category: "dependency",
action: "delete",
targetType: "dependency",
targetId: id,
detail: { from: graph.names.get(nodeKey(before.fromType as "server", before.fromId)) ?? `${before.fromType} #${before.fromId}`, to: graph.names.get(nodeKey(before.toType as "server", before.toId)) ?? `${before.toType} #${before.toId}` },
});
invalidateAlertsCache();
res.status(204).end();
}));
+42 -8
View File
@@ -6,6 +6,7 @@ import { assetKinds, assets, servers } from "../db/schema.js";
import { requireAuth, requireRole } from "../auth/middleware.js";
import { recordAudit } from "../services/audit.js";
import { isRealDate } from "../services/inventory.js";
import { blanksToFill, differences, hardwareOf } from "../services/inventoryAutofill.js";
import { getSettings, updateSettings } from "../services/settingsStore.js";
import { asyncHandler } from "../utils/asyncHandler.js";
@@ -53,16 +54,32 @@ function values(d: z.infer<typeof assetSchema>) {
};
}
/** A linked entry's blank maker, model and serial, from what that server's agent says. */
async function fillFromAgent(v: ReturnType<typeof values>) {
if (v.serverId === null) return {};
const [server] = await db.select({ hwManufacturer: servers.hwManufacturer, hwModel: servers.hwModel, hwSerial: servers.hwSerial }).from(servers).where(eq(servers.id, v.serverId)).limit(1);
return blanksToFill(v, hardwareOf(server));
}
const idOf = (v: string) => (/^\d+$/.test(v) ? Number(v) : NaN);
// Everything signed-in people may see: the hardware, and the price of power the running cost is worked out with.
inventoryRouter.get("/", asyncHandler(async (_req, res) => {
const [rows, serverRows, settings] = await Promise.all([db.select().from(assets).orderBy(assets.name), db.select({ id: servers.id, name: servers.name }).from(servers), getSettings()]);
const serverName = new Map(serverRows.map((s) => [s.id, s.name]));
const [rows, serverRows, settings] = await Promise.all([db.select().from(assets).orderBy(assets.name), db.select().from(servers), getSettings()]);
const byId = new Map(serverRows.map((s) => [s.id, s]));
res.json({
assets: rows.map((a) => ({ ...a, server: a.serverId !== null && serverName.has(a.serverId) ? { id: a.serverId, name: serverName.get(a.serverId)! } : null })),
assets: rows.map((a) => {
const s = a.serverId !== null ? byId.get(a.serverId) : undefined;
const hw = hardwareOf(s);
return {
...a,
server: s ? { id: s.id, name: s.name } : null,
// What the machine's agent says it is, and what that would change here (for the "refresh from agent" button).
agent: hw ? { ...hw, differs: differences(a, hw) } : null,
};
}),
config: settings.inventory,
servers: serverRows.sort((a, b) => a.name.localeCompare(b.name)),
servers: serverRows.map((s) => ({ id: s.id, name: s.name })).sort((a, b) => a.name.localeCompare(b.name)),
});
}));
@@ -89,8 +106,9 @@ inventoryRouter.post("/", requireRole("operator"), asyncHandler(async (req, res)
return res.status(400).json({ error: "unknown_server", message: "That server doesn't exist." });
}
const now = new Date().toISOString();
const [row] = await db.insert(assets).values({ ...v, createdAt: now, updatedAt: now }).returning();
await recordAudit({ actor: req.currentUser!, category: "inventory", action: "create", targetType: "asset", targetId: row.id, detail: { name: row.name, kind: row.kind } });
const filled = await fillFromAgent(v);
const [row] = await db.insert(assets).values({ ...v, ...filled, createdAt: now, updatedAt: now }).returning();
await recordAudit({ actor: req.currentUser!, category: "inventory", action: "create", targetType: "asset", targetId: row.id, detail: { name: row.name, kind: row.kind, ...(Object.keys(filled).length ? { filledFromAgent: Object.keys(filled) } : {}) } });
res.status(201).json({ asset: row });
}));
@@ -104,11 +122,27 @@ inventoryRouter.put("/:id", requireRole("operator"), asyncHandler(async (req, re
if (v.serverId !== null && (await db.select({ id: servers.id }).from(servers).where(eq(servers.id, v.serverId))).length === 0) {
return res.status(400).json({ error: "unknown_server", message: "That server doesn't exist." });
}
const [row] = await db.update(assets).set({ ...v, updatedAt: new Date().toISOString() }).where(eq(assets.id, id)).returning();
await recordAudit({ actor: req.currentUser!, category: "inventory", action: "update", targetType: "asset", targetId: id, detail: { name: row.name } });
const filled = await fillFromAgent(v);
const [row] = await db.update(assets).set({ ...v, ...filled, updatedAt: new Date().toISOString() }).where(eq(assets.id, id)).returning();
await recordAudit({ actor: req.currentUser!, category: "inventory", action: "update", targetType: "asset", targetId: id, detail: { name: row.name, ...(Object.keys(filled).length ? { filledFromAgent: Object.keys(filled) } : {}) } });
res.json({ asset: row });
}));
// Takes what the agent says over what is written here: the one deliberate overwrite (the entry's own fields are otherwise only ever filled when blank).
inventoryRouter.post("/:id/refresh-from-agent", requireRole("operator"), asyncHandler(async (req, res) => {
const id = idOf(req.params.id);
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
const [before] = await db.select().from(assets).where(eq(assets.id, id)).limit(1);
if (!before) return res.status(404).json({ error: "not_found" });
if (before.serverId === null) return res.status(400).json({ error: "not_linked", message: "Link the entry to a server first." });
const [server] = await db.select({ hwManufacturer: servers.hwManufacturer, hwModel: servers.hwModel, hwSerial: servers.hwSerial }).from(servers).where(eq(servers.id, before.serverId)).limit(1);
const changes = differences(before, hardwareOf(server));
if (Object.keys(changes).length === 0) return res.json({ asset: before, changed: [] });
const [row] = await db.update(assets).set({ ...changes, updatedAt: new Date().toISOString() }).where(eq(assets.id, id)).returning();
await recordAudit({ actor: req.currentUser!, category: "inventory", action: "refresh_from_agent", targetType: "asset", targetId: id, detail: { name: row.name, from: Object.fromEntries(Object.keys(changes).map((k) => [k, (before as any)[k]])), to: changes } });
res.json({ asset: row, changed: Object.keys(changes) });
}));
inventoryRouter.delete("/:id", requireRole("operator"), asyncHandler(async (req, res) => {
const id = idOf(req.params.id);
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
+1
View File
@@ -63,6 +63,7 @@ privacyRouter.get("/", asyncHandler(async (req, res) => {
ntfy: channel(settings.ntfy.enabled, hostOf(settings.ntfy.url)),
smtp: channel(settings.smtp.enabled, settings.smtp.host || null),
webhook: channel(settings.webhook.enabled, hostOf(settings.webhook.url)),
webPush: channel(settings.webPush.enabled, "the push services of your browsers (Google, Mozilla, Apple …)"),
},
},
});
+97
View File
@@ -0,0 +1,97 @@
import { Router } from "express";
import { and, eq } from "drizzle-orm";
import { z } from "zod";
import { db } from "../db/client.js";
import { pushSubscriptions } from "../db/schema.js";
import { requireAuth } from "../auth/middleware.js";
import { recordAudit } from "../services/audit.js";
import { getSettings } from "../services/settingsStore.js";
import { checkEndpoint, getVapidKeys, sendPush } from "../services/webPush.js";
import { asyncHandler } from "../utils/asyncHandler.js";
export const pushRouter = Router();
pushRouter.use(requireAuth);
/** At most this many browsers per person: a phone, a laptop, a tablet and the odd one. */
const MAX_PER_USER = 10;
const subscribeSchema = z.object({
endpoint: z.string().url().max(2000),
keys: z.object({ p256dh: z.string().min(80).max(100), auth: z.string().min(20).max(30) }),
label: z.string().trim().max(80).optional(),
});
const hostOf = (endpoint: string) => {
try {
return new URL(endpoint).host;
} catch {
return "";
}
};
const off = { error: "push_disabled", message: "Push notifications are switched off. An administrator turns them on under Settings, Notifications." };
// The key a browser subscribes with. Anyone signed in may ask (their own device is theirs to enrol), once push is on.
pushRouter.get("/key", asyncHandler(async (_req, res) => {
const { webPush } = await getSettings();
if (!webPush.enabled) return res.status(409).json(off);
res.json({ publicKey: (await getVapidKeys()).publicKey });
}));
// This person's own browsers.
pushRouter.get("/mine", asyncHandler(async (req, res) => {
const rows = await db.select().from(pushSubscriptions).where(eq(pushSubscriptions.userId, req.currentUser!.id)).orderBy(pushSubscriptions.id);
const { webPush } = await getSettings();
res.json({
enabled: webPush.enabled,
subscriptions: rows.map((r) => ({ id: r.id, label: r.label, host: hostOf(r.endpoint), endpoint: r.endpoint, createdAt: r.createdAt, lastSuccessAt: r.lastSuccessAt })),
});
}));
pushRouter.post("/subscribe", asyncHandler(async (req, res) => {
const { webPush } = await getSettings();
if (!webPush.enabled) return res.status(409).json(off);
const parsed = subscribeSchema.safeParse(req.body);
if (!parsed.success) return res.status(400).json({ error: "invalid_body", message: "That isn't a push subscription.", details: parsed.error.flatten() });
const { endpoint, keys, label } = parsed.data;
try {
await checkEndpoint(endpoint);
} catch (err) {
return res.status(400).json({ error: "bad_endpoint", message: err instanceof Error ? err.message : "That push address can't be used." });
}
const me = req.currentUser!;
const mine = await db.select({ id: pushSubscriptions.id, endpoint: pushSubscriptions.endpoint }).from(pushSubscriptions).where(eq(pushSubscriptions.userId, me.id));
const existing = mine.find((r) => r.endpoint === endpoint);
if (!existing && mine.length >= MAX_PER_USER) return res.status(409).json({ error: "too_many", message: `You already have ${MAX_PER_USER} browsers subscribed. Remove one first.` });
// The same browser subscribing again (or after someone else used it) is the same row, now this person's.
const [row] = await db
.insert(pushSubscriptions)
.values({ userId: me.id, endpoint, p256dh: keys.p256dh, auth: keys.auth, label: label || null })
.onConflictDoUpdate({ target: pushSubscriptions.endpoint, set: { userId: me.id, p256dh: keys.p256dh, auth: keys.auth, label: label || null } })
.returning({ id: pushSubscriptions.id });
if (!existing) await recordAudit({ actor: me, category: "push", action: "subscribe", targetType: "push_subscription", targetId: row.id, detail: { host: hostOf(endpoint), label: label || null } });
res.status(201).json({ id: row.id });
}));
// Stops a browser being written to. A person removes their own; an administrator may remove anyone's.
pushRouter.delete("/:id", asyncHandler(async (req, res) => {
const id = /^\d+$/.test(req.params.id) ? Number(req.params.id) : NaN;
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
const me = req.currentUser!;
const [row] = await db.select().from(pushSubscriptions).where(eq(pushSubscriptions.id, id)).limit(1);
if (!row || (row.userId !== me.id && me.role !== "admin")) return res.status(404).json({ error: "not_found" });
await db.delete(pushSubscriptions).where(and(eq(pushSubscriptions.id, id)));
await recordAudit({ actor: me, category: "push", action: "unsubscribe", targetType: "push_subscription", targetId: id, detail: { host: hostOf(row.endpoint), label: row.label } });
res.status(204).end();
}));
// A message to this person's own browsers, to see that it works.
pushRouter.post("/test", asyncHandler(async (req, res) => {
const { webPush } = await getSettings();
if (!webPush.enabled) return res.status(409).json(off);
const mine = await db.select().from(pushSubscriptions).where(eq(pushSubscriptions.userId, req.currentUser!.id));
if (mine.length === 0) return res.status(400).json({ error: "none", message: "This browser isn't subscribed yet." });
const outcome = await sendPush({ title: "Homelab Manager — Test", body: "Push notifications are working.", url: "/settings/notifications", tag: "test" }, mine);
res.json(outcome);
}));
+1
View File
@@ -64,6 +64,7 @@ const updateSchema = z.object({
.partial()
.optional(),
webhook: z.object({ enabled: z.boolean(), url: z.string(), secret: z.string() }).partial().optional(),
webPush: z.object({ enabled: z.boolean(), contact: z.string().trim().max(200) }).partial().optional(),
notifications: z
.object({
dnsAdd: z.boolean(),
+3 -1
View File
@@ -8,7 +8,9 @@
* 2 — reports pending updates, and its own version
* 3 — Linux: the package manager is asked outside the systemd sandbox, so dnf and zypper can answer too.
* Windows: can replace itself when an administrator has switched automatic updates on.
* 4 — Linux: also reports failed services, disk health (SMART) and ZFS pools, reads pacman, and can replace itself too
* 4 — Linux: also reports failed services, disk health (SMART) and ZFS pools, reads pacman, and can replace itself too.
* Windows: also reports failed services, disk health and Storage Spaces pools, and what the machine is (maker, model, serial)
* 5 — Linux: also says what the machine is (maker, model, serial), for the inventory
*/
import { createHash } from "node:crypto";
import { readFileSync } from "node:fs";
+20 -9
View File
@@ -33,7 +33,7 @@ import { computeSecretStatus } from "./secretStatus.js";
import { getFailingSources } from "./integrationHealthMonitor.js";
import { getSettings } from "./settingsStore.js";
import { sourceLabel } from "./notify.js";
import { coveredByParent, loadParentRows, resolveParents } from "./serverParents.js";
import { dependencyCover, integrationTypeCause, takenByType } from "./dependencies.js";
import type { Alert, AlertCategory, AlertSeverity, SourceFailure } from "./alertTypes.js";
export interface AlertsReport {
@@ -158,6 +158,10 @@ export async function collectAlerts(): Promise<AlertsReport> {
}
}
// The servers found to be offline, for the integration failures that one may explain: the server check fills it in (and it is empty if that check fails).
let offlineKnown!: (ids: Set<number>) => void;
const offlineServers = new Promise<Set<number>>((resolve) => (offlineKnown = resolve));
await Promise.all([
check("Server and storage health", async () => {
const { healthChecks } = await getSettings();
@@ -176,17 +180,17 @@ export async function collectAlerts(): Promise<AlertsReport> {
const m = /^offline:server:(\d+)$/.exec(c.key);
if (m) offlineIds.add(Number(m[1]));
}
const cause = coveredByParent(offlineIds, resolveParents(await loadParentRows()));
const taken = new Map<number, string[]>();
for (const [child, parent] of cause) taken.set(parent, [...(taken.get(parent) ?? []), serverNames.get(child) ?? `#${child}`]);
// (Or because of a service it needs that is failing, or a server it needs that needs one.)
offlineKnown(offlineIds);
const cover = await dependencyCover(offlineIds, getFailingSources().map((f) => f.source));
for (const c of conditions) {
const origin = originOfConditionKey(c.key);
const subject = subjectOfConditionKey(c.key);
const offlineId = /^offline:server:(\d+)$/.exec(c.key)?.[1];
const parentOf = offlineId ? cause.get(Number(offlineId)) : undefined;
const parentOf = offlineId ? cover.cause.get(`server:${offlineId}`) : undefined;
push({
causedBy: parentOf !== undefined ? { name: serverNames.get(parentOf) ?? `#${parentOf}` } : undefined,
affects: offlineId ? taken.get(Number(offlineId))?.sort((a, b) => a.localeCompare(b)) : undefined,
causedBy: parentOf !== undefined ? { name: cover.graph.names.get(parentOf) ?? parentOf } : undefined,
affects: offlineId ? cover.taken.get(`server:${offlineId}`)?.sort((a, b) => a.localeCompare(b)) : undefined,
category: origin.category,
severity: c.severity ?? "warning",
type: origin.type,
@@ -203,7 +207,7 @@ export async function collectAlerts(): Promise<AlertsReport> {
const row = intById.get(Number(m[2]));
cannotCheck(`${SOURCES[m[1]].label} “${row?.name ?? `#${m[2]}`}”${m[3] ? ` (node ${m[3]})` : ""}`, "couldn't be read — see the Diagnostic Log", "Server and storage health");
}
}),
}).finally(() => offlineKnown(new Set())),
check("Automation runs", async () => {
const { items, held } = await collectAutomation();
@@ -447,8 +451,15 @@ export async function collectAlerts(): Promise<AlertsReport> {
check("Integration failures", async () => {
const { notifications } = await getSettings();
for (const f of getFailingSources()) {
const failing = getFailingSources();
// A failure explained by something it needs that is down is folded under that, like a guest under its host.
const cover = failing.length > 0 ? await dependencyCover(await offlineServers, failing.map((f) => f.source)).catch(() => null) : null;
for (const f of failing) {
const cause = cover ? integrationTypeCause(f.source, cover) : null;
const affects = cover ? takenByType(f.source, cover) : [];
push({
causedBy: cause ? { name: cause.name } : undefined,
affects: affects.length > 0 ? affects.sort((a, b) => a.localeCompare(b)) : undefined,
category: "integration",
severity: f.alerted ? "critical" : "warning",
type: f.source,
+192
View File
@@ -0,0 +1,192 @@
/**
* What depends on what, beyond "this server runs on that server": a service that needs a server (Gitea on the box it is installed on), a
* server that needs a service (a guest that only exists while Proxmox answers). When the thing depended on is down, the things that need it
* are down too — and only the one that matters is worth an alert, so the rest are folded under it: shown, not announced.
*
* The graph is the "runs on" links of servers (set by hand or worked out from Proxmox, see serverParents.ts) plus the dependencies written
* here. Servers and integrations are the things that can be down and be needed. (DNS providers raise no alerts of their own, so there is
* nothing to fold.) An integration is judged per kind of service, as its health is: it is folded only if every enabled instance of that
* kind is explained.
*/
import { eq } from "drizzle-orm";
import { db } from "../db/client.js";
import { dependencies, integrations, servers } from "../db/schema.js";
import { getInternalFlag } from "./settingsStore.js";
import { loadParentRows, resolveParents, type ParentLink } from "./serverParents.js";
/** Where the health monitor keeps the problems it is announcing; read here so a service's alert can tell a server it needs is down. */
export const HEALTH_STATE_FLAG = "healthActiveConditions";
export const nodeTypes = ["server", "integration"] as const;
export type NodeType = (typeof nodeTypes)[number];
export const nodeKey = (type: NodeType, id: number) => `${type}:${id}`;
export interface DependencyEdge {
fromType: NodeType;
fromId: number;
toType: NodeType;
toId: number;
}
/** node → the nodes it needs. */
export type Edges = Map<string, string[]>;
/** The graph from the "runs on" links and the written dependencies; an edge to or from something that no longer exists is left out. */
export function buildEdges(parentLinks: Map<number, ParentLink>, rows: DependencyEdge[], exists: (key: string) => boolean): Edges {
const edges: Edges = new Map();
const add = (from: string, to: string) => {
if (from === to || !exists(from) || !exists(to)) return;
const list = edges.get(from) ?? [];
if (!list.includes(to)) edges.set(from, [...list, to]);
};
for (const [child, link] of parentLinks) add(nodeKey("server", child), nodeKey("server", link.parentId));
for (const r of rows) add(nodeKey(r.fromType, r.fromId), nodeKey(r.toType, r.toId));
return edges;
}
/** Everything `node` needs, directly or not, nearest first; `looped` if it ends up needing itself. */
function ancestorsOf(node: string, edges: Edges): { order: string[]; looped: boolean } {
const order: string[] = [];
const seen = new Set<string>();
const queue = [...(edges.get(node) ?? [])];
let looped = false;
while (queue.length > 0) {
const cur = queue.shift()!;
if (cur === node) {
looped = true;
continue;
}
if (seen.has(cur)) continue;
seen.add(cur);
order.push(cur);
queue.push(...(edges.get(cur) ?? []));
}
return { order, looped };
}
/**
* Of the nodes that are down, those whose cause is something they need that is down too: each maps to the root cause — a down node that
* needs nothing that is down. A loop explains nothing (each would be waiting for the other), so its members stand alone and are announced.
*/
export function coveredNodes(down: Set<string>, edges: Edges): Map<string, string> {
const covered = new Map<string, string>();
for (const node of [...down].sort()) {
const { order, looped } = ancestorsOf(node, edges);
if (looped) continue;
const downAbove = order.filter((a) => down.has(a));
const root = downAbove.find((a) => {
const above = ancestorsOf(a, edges);
return !above.looped && !above.order.some((x) => down.has(x));
});
if (root !== undefined) covered.set(node, root);
}
return covered;
}
/** Would `from` needing `to` close a loop? True if `to` is `from` or already needs it, however indirectly. */
export function wouldLoop(edges: Edges, from: string, to: string): boolean {
return from === to || ancestorsOf(to, edges).order.includes(from) || ancestorsOf(to, edges).looped;
}
export interface Graph {
edges: Edges;
/** node → its name, for every node that exists. */
names: Map<string, string>;
rows: (DependencyEdge & { id: number; createdAt: string })[];
/** Enabled instances of each kind of integration. */
instances: Map<string, number[]>;
intRows: { id: number; type: string; enabled: boolean }[];
}
export async function loadGraph(): Promise<Graph> {
const [serverRows, intRows, depRows, parentRows] = await Promise.all([
db.select({ id: servers.id, name: servers.name }).from(servers),
db.select({ id: integrations.id, name: integrations.name, type: integrations.type, enabled: integrations.enabled }).from(integrations),
db.select().from(dependencies),
loadParentRows(),
]);
const names = new Map<string, string>();
for (const s of serverRows) names.set(nodeKey("server", s.id), s.name);
for (const i of intRows) names.set(nodeKey("integration", i.id), i.name);
const rows = depRows.map((r) => ({ id: r.id, fromType: r.fromType as NodeType, fromId: r.fromId, toType: r.toType as NodeType, toId: r.toId, createdAt: r.createdAt }));
const instances = new Map<string, number[]>();
for (const i of intRows) if (i.enabled) instances.set(i.type, [...(instances.get(i.type) ?? []), i.id]);
return { edges: buildEdges(resolveParents(parentRows), rows, (k) => names.has(k)), names, rows, instances, intRows };
}
export interface Cover {
/** Each node that is down because of another → the root cause. */
cause: Map<string, string>;
/** Each root cause → the names of what it takes down with it. */
taken: Map<string, string[]>;
graph: Graph;
}
/** Works out what is down because of what, given the servers that are offline and the kinds of integration that are failing. */
export async function dependencyCover(offlineServerIds: Iterable<number>, failingTypes: Iterable<string>): Promise<Cover> {
const graph = await loadGraph();
const down = new Set<string>();
for (const id of offlineServerIds) down.add(nodeKey("server", id));
const failing = new Set(failingTypes);
for (const [type, ids] of graph.instances) if (failing.has(type)) for (const id of ids) down.add(nodeKey("integration", id));
const cause = coveredNodes(down, graph.edges);
const taken = new Map<string, string[]>();
for (const [child, parent] of cause) taken.set(parent, [...(taken.get(parent) ?? []), graph.names.get(child) ?? child]);
return { cause, taken, graph };
}
/** The root cause behind a failing kind of integration, when every enabled instance of it is explained; null when it stands alone. */
export function integrationTypeCause(type: string, cover: Cover): { key: string; name: string } | null {
const ids = cover.graph.instances.get(type);
if (!ids || ids.length === 0) return null;
let first: string | null = null;
for (const id of ids) {
const root = cover.cause.get(nodeKey("integration", id));
if (root === undefined) return null;
first ??= root;
}
return first === null ? null : { key: first, name: cover.graph.names.get(first) ?? first };
}
/** What a root cause takes down, for a kind of integration: the folded names across all its instances. */
export function takenByType(type: string, cover: Cover): string[] {
const out: string[] = [];
for (const id of cover.graph.instances.get(type) ?? []) out.push(...(cover.taken.get(nodeKey("integration", id)) ?? []));
return out;
}
/** The servers the health monitor is announcing as offline (or holding back as explained), from what it last stored. */
export async function offlineServerIdsFromState(): Promise<number[]> {
try {
const raw = await getInternalFlag(HEALTH_STATE_FLAG);
if (!raw) return [];
const ids: number[] = [];
for (const key of Object.keys(JSON.parse(raw) as Record<string, unknown>)) {
const m = /^offline:server:(\d+)$/.exec(key);
if (m) ids.push(Number(m[1]));
}
return ids;
} catch {
return [];
}
}
/**
* Whether a kind of integration that has started failing is failing because of something it needs that is down — a server the health
* monitor says is offline, or another service that is failing. Never throws: a problem here must not swallow a real outage.
*/
export async function integrationCausedBy(type: string, failingTypes: string[]): Promise<{ name: string } | null> {
try {
const cover = await dependencyCover(await offlineServerIdsFromState(), failingTypes);
return integrationTypeCause(type, cover);
} catch (err) {
console.error("[dependencies] couldn't work out whether a failure is explained:", err);
return null;
}
}
export async function nodeExists(type: NodeType, id: number): Promise<boolean> {
if (type === "server") return (await db.select({ id: servers.id }).from(servers).where(eq(servers.id, id)).limit(1)).length > 0;
return (await db.select({ id: integrations.id }).from(integrations).where(eq(integrations.id, id)).limit(1)).length > 0;
}
+11 -8
View File
@@ -8,11 +8,12 @@ import { notifyHealthIssues, notifyHealthRecovered } from "./notify.js";
import { recordAudit } from "./audit.js";
import { getInternalFlag, getSettings, setInternalFlag } from "./settingsStore.js";
import { activeSubjects, subjectOfConditionKey } from "./maintenance.js";
import { coveredByParent, loadParentRows, resolveParents } from "./serverParents.js";
import { dependencyCover, HEALTH_STATE_FLAG } from "./dependencies.js";
import { getFailingSources } from "./integrationHealthMonitor.js";
import { hostHealthConditions, parseHostHealth, type HostHealth } from "./hostHealth.js";
import { diskLimit, parseThresholds, type ServerThresholds } from "./serverThresholds.js";
const STATE_FLAG = "healthActiveConditions";
const STATE_FLAG = HEALTH_STATE_FLAG;
/** After a restart the agents haven't had a chance to report yet (they run every 15 min), so server-derived conditions are held rather than judged. */
const STARTUP_GRACE_MS = Number(process.env.HEALTH_STARTUP_GRACE_MS ?? 20 * 60 * 1000); // the environment variable is for tests
const PROCESS_START = Date.now();
@@ -396,18 +397,20 @@ export async function runHealthCheck(now: number = Date.now()): Promise<{ added:
const m = /^offline:server:(\d+)$/.exec(c.key);
if (m) offlineIds.add(Number(m[1]));
}
const cause = coveredByParent(offlineIds, resolveParents(await loadParentRows()));
const coveredKeys = new Set([...cause.keys()].map((id) => `offline:server:${id}`));
// (Also when it needs a service that is failing, or runs on a server that needs one.)
const cover = await dependencyCover(offlineIds, getFailingSources().map((f) => f.source));
const coveredKeys = new Set<string>();
for (const child of cover.cause.keys()) {
const m = /^server:(\d+)$/.exec(child);
if (m) coveredKeys.add(`offline:server:${m[1]}`);
}
const { added: announced, resolved, next, newlyDown, ended } = diffConditions(await loadState(), current, held, await activeSubjects(new Date(now)), coveredKeys);
await setInternalFlag(STATE_FLAG, JSON.stringify(next));
// A host's message says what else it takes with it.
const names = new Map(snapshot.servers.map((s) => [s.id, s.name]));
const taken = new Map<number, string[]>();
for (const [child, parent] of cause) taken.set(parent, [...(taken.get(parent) ?? []), names.get(child) ?? `#${child}`]);
const added = announced.map((c) => {
const m = /^offline:server:(\d+)$/.exec(c.key);
const children = m ? taken.get(Number(m[1])) : undefined;
const children = m ? cover.taken.get(`server:${m[1]}`) : undefined;
return children?.length ? { ...c, message: `${c.message} — also takes down ${alsoList(children)}` } : c;
});
+2 -1
View File
@@ -18,6 +18,7 @@ export interface SmartDisk {
/** NVMe and some SSDs say how much of their rated life is used, 0–100 (and past it). */
wearPercent: number | null;
}
/** A storage pool: ZFS on Linux, Storage Spaces on Windows (which reports Healthy, Warning and Unhealthy as ONLINE, DEGRADED and FAULTED). */
export interface ZfsPool {
name: string;
/** ONLINE, DEGRADED, FAULTED, UNAVAIL, SUSPENDED … as zpool says it. */
@@ -117,7 +118,7 @@ export function hostHealthConditions(serverId: number, serverName: string, healt
}
for (const p of health.zfs ?? []) {
if (!poolIsFine(p)) {
out.push({ key: `zfs:server:${serverId}:${p.name}`, severity: "critical", message: `${serverName}: ZFS pool ${p.name} is ${p.health}` });
out.push({ key: `zfs:server:${serverId}:${p.name}`, severity: "critical", message: `${serverName}: storage pool ${p.name} is ${p.health}` });
}
}
return out;
@@ -6,6 +6,7 @@ import { ensureHistoryStart } from "./uptimeHistory.js";
import { getSettings } from "./settingsStore.js";
import { isSourceInMaintenance } from "./maintenance.js";
import { notifyIntegrationDown, notifyIntegrationRecovered } from "./notify.js";
import { integrationCausedBy } from "./dependencies.js";
interface SourceHealth {
consecutiveFailures: number;
@@ -96,7 +97,9 @@ export async function trackIntegrationHealth(source: string, ok: boolean): Promi
if (notifications.integrationFailureAlerts && !state.alerted && state.consecutiveFailures >= notifications.integrationFailureThreshold) {
// During maintenance the failures keep being counted but `alerted` stays false, so if the service is
// still failing once the window ends, the very next failed call alerts — a real outage isn't swallowed.
if (!(await isSourceInMaintenance(source))) {
// Likewise when it needs something that is down (the server it runs on, another service): that one is what gets announced, and if this stays
// down once that one is back, the next failed call alerts.
if (!(await isSourceInMaintenance(source)) && !(await integrationCausedBy(source, [...getFailingSources().map((f) => f.source), source]))) {
state.alerted = true;
await notifyIntegrationDown(source, state.consecutiveFailures);
}
+60
View File
@@ -0,0 +1,60 @@
/**
* Filling in the inventory from what an agent has found out about the machine it runs on: maker, model and serial number. Typing a serial
* number off a label is the part of an inventory nobody keeps up, so a linked entry gets it from the agent — but only into fields that are
* blank, because what a person wrote down is theirs; "Refresh from agent" is the way to overwrite on purpose.
*/
import { eq, isNull, or, and } from "drizzle-orm";
import { db } from "../db/client.js";
import { assets, servers } from "../db/schema.js";
export interface AgentHardware {
manufacturer: string | null;
model: string | null;
serial: string | null;
}
type Fields = { manufacturer: string | null; model: string | null; serial: string | null };
const KEYS = ["manufacturer", "model", "serial"] as const;
/** What the server's agent said the machine is, or null if it has said nothing. */
export function hardwareOf(server: { hwManufacturer: string | null; hwModel: string | null; hwSerial: string | null } | null | undefined): AgentHardware | null {
if (!server) return null;
const hw = { manufacturer: server.hwManufacturer ?? null, model: server.hwModel ?? null, serial: server.hwSerial ?? null };
return hw.manufacturer || hw.model || hw.serial ? hw : null;
}
/** The fields of an entry that are blank and that the agent can fill. Empty when nothing would change. */
export function blanksToFill(current: Fields, hw: AgentHardware | null): Partial<Fields> {
const out: Partial<Fields> = {};
if (!hw) return out;
for (const k of KEYS) if (!current[k]?.trim() && hw[k]) out[k] = hw[k];
return out;
}
/** The fields where the agent says something different from the entry: what a refresh would change. */
export function differences(current: Fields, hw: AgentHardware | null): Partial<Fields> {
const out: Partial<Fields> = {};
if (!hw) return out;
for (const k of KEYS) if (hw[k] && (current[k] ?? "").trim() !== hw[k]) out[k] = hw[k];
return out;
}
/** After an agent has reported, fills the blanks of the inventory entries that are this server. */
export async function fillInventoryFromServer(serverId: number): Promise<number> {
const [server] = await db.select({ hwManufacturer: servers.hwManufacturer, hwModel: servers.hwModel, hwSerial: servers.hwSerial }).from(servers).where(eq(servers.id, serverId)).limit(1);
const hw = hardwareOf(server);
if (!hw) return 0;
// Only entries with something blank are worth reading.
const linked = await db
.select()
.from(assets)
.where(and(eq(assets.serverId, serverId), or(isNull(assets.manufacturer), isNull(assets.model), isNull(assets.serial))));
let changed = 0;
for (const a of linked) {
const fill = blanksToFill(a, hw);
if (Object.keys(fill).length === 0) continue;
await db.update(assets).set({ ...fill, updatedAt: new Date().toISOString() }).where(eq(assets.id, a.id));
changed++;
}
return changed;
}
+1 -1
View File
@@ -95,7 +95,7 @@ export async function buildMetrics(now: Date = new Date()): Promise<string> {
if (d.wearPercent !== null) out.add("homelab_server_disk_wear_percent", "gauge", "How much of an SSD's rated life is used.", d.wearPercent, { server, device: d.device });
if (d.temperatureC !== null) out.add("homelab_server_disk_temperature_celsius", "gauge", "A disk's temperature.", d.temperatureC, { server, device: d.device });
}
for (const p of health?.zfs ?? []) out.add("homelab_server_zfs_pool_online", "gauge", "1 if a ZFS pool is ONLINE, 0 in any other state.", poolIsFine(p) ? 1 : 0, { server, pool: p.name });
for (const p of health?.zfs ?? []) out.add("homelab_server_zfs_pool_online", "gauge", "1 if a storage pool (ZFS, or Storage Spaces on Windows) is ONLINE, 0 in any other state.", poolIsFine(p) ? 1 : 0, { server, pool: p.name });
out.add("homelab_server_agent_outdated", "gauge", "1 if the server's agent is older than the one this installation would install now.", isAgentOutdated(s, latest) ? 1 : 0, { server });
}
// Disks on a steady climb, from each server's own history: days until full at the present rate (left out where there is no steady trend).
+14 -2
View File
@@ -2,6 +2,7 @@ import nodemailer from "nodemailer";
import { getSettings, type NotificationEvents } from "./settingsStore.js";
import { enqueueNotification, listQueuedNotifications, clearQueuedNotifications } from "./notificationQueue.js";
import { emitEvent } from "./eventWebhooks.js";
import { sendPush, subscriptionCount } from "./webPush.js";
// ─── Channel senders (best-effort — failures are logged, never thrown) ──────
@@ -72,8 +73,18 @@ async function sendWebhook(title: string, message: string) {
}
}
async function sendWebPush(title: string, message: string) {
const { webPush } = await getSettings();
if (!webPush.enabled) return;
try {
await sendPush({ title, body: message });
} catch (err) {
console.error("[notify] Web push failed:", err instanceof Error ? err.message : err);
}
}
async function sendToChannels(title: string, message: string): Promise<void> {
await Promise.all([sendGotify(title, message), sendNtfy(title, message), sendSmtp(title, message), sendWebhook(title, message)]);
await Promise.all([sendGotify(title, message), sendNtfy(title, message), sendSmtp(title, message), sendWebhook(title, message), sendWebPush(title, message)]);
}
function currentTimeInZone(timezone: string): { hh: number; mm: number } {
@@ -112,8 +123,9 @@ export async function notifyNow(title: string, message: string): Promise<void> {
/** How many channels are switched on and filled in enough to send, so a "send it now" can say when there is nowhere to send to. */
export async function enabledChannelCount(): Promise<number> {
const { gotify, ntfy, smtp, webhook } = await getSettings();
const { gotify, ntfy, smtp, webhook, webPush } = await getSettings();
return [
webPush.enabled && (await subscriptionCount()) > 0,
gotify.enabled && gotify.url && gotify.token,
ntfy.enabled && ntfy.url && ntfy.topic,
smtp.enabled && smtp.host && smtp.to && smtp.from,
+9
View File
@@ -37,6 +37,13 @@ export interface WebhookSettings {
secret: string;
}
/** Push notifications to phones and browsers that have subscribed (see services/webPush.ts). Who is subscribed is kept apart, per device. */
export interface WebPushSettings {
enabled: boolean;
/** Where a push service can reach whoever runs this: an email address or an https address (VAPID asks for one). Blank is fine. */
contact: string;
}
export interface NotificationEvents {
dnsAdd: boolean;
dnsUpdate: boolean;
@@ -189,6 +196,7 @@ export interface AppSettings {
ntfy: NtfySettings;
smtp: SmtpSettings;
webhook: WebhookSettings;
webPush: WebPushSettings;
notifications: NotificationEvents;
providerColors: ProviderColors;
integrationColors: IntegrationColors;
@@ -211,6 +219,7 @@ const DEFAULTS: AppSettings = {
ntfy: { enabled: false, url: "https://ntfy.sh", topic: "", token: "", priority: 3 },
smtp: { enabled: false, host: "", port: 587, secure: false, username: "", password: "", from: "", to: "" },
webhook: { enabled: false, url: "", secret: "" },
webPush: { enabled: false, contact: "" },
notifications: {
dnsAdd: true,
dnsUpdate: true,
+15
View File
@@ -5,6 +5,7 @@ import { recordServerMetrics } from "./serverMetrics.js";
import { nextUpdateState, type IncomingUpdates } from "./pendingUpdates.js";
import { nextHostHealth, type IncomingHostHealth } from "./hostHealth.js";
import { recordAudit } from "./audit.js";
import { fillInventoryFromServer } from "./inventoryAutofill.js";
export interface IncomingTask {
scheduleType: "cron" | "systemd_timer" | "windows_task";
@@ -27,6 +28,8 @@ export interface IncomingSystemInfo {
updates?: IncomingUpdates;
/** Failed services, disk health, ZFS pools — each part present only if the agent could find it out. */
health?: IncomingHostHealth;
/** What the machine is, from its firmware: each part present only if it says anything. */
hardware?: { manufacturer?: string; model?: string; serial?: string };
}
export interface AgentReport {
@@ -115,6 +118,9 @@ export async function syncServerTasks(serverId: number, report: AgentReport) {
// Only stored when the agent reported it, so an agent that predates this (or couldn't tell this time) doesn't wipe what it said before.
...(system?.updates && before ? nextUpdateState(before, system.updates, now) : {}),
...(system?.health && before ? { hostHealth: nextHostHealth(before.hostHealth, system.health, now) } : {}),
...(system?.hardware?.manufacturer ? { hwManufacturer: system.hardware.manufacturer } : {}),
...(system?.hardware?.model ? { hwModel: system.hardware.model } : {}),
...(system?.hardware?.serial ? { hwSerial: system.hardware.serial } : {}),
...(report.hostname ? { hostname: report.hostname } : {}),
...(report.agentVersion !== undefined ? { agentVersion: report.agentVersion } : {}),
// An agent knows what it runs on better than whoever registered the server did.
@@ -146,6 +152,15 @@ export async function syncServerTasks(serverId: number, report: AgentReport) {
}
}
// A machine whose agent has said what it is fills in the blanks of its inventory entry. Also best-effort.
if (system?.hardware) {
try {
await fillInventoryFromServer(serverId);
} catch (err) {
console.error("[inventory] couldn't fill in from the agent:", err);
}
}
// Keep a little history of the figures too. A failure here must not lose the report itself, which is already saved.
try {
await recordServerMetrics(serverId, system, new Date(now));
+245
View File
@@ -0,0 +1,245 @@
/**
* Web Push: notifications to a phone or a browser that has asked for them, with no app and no account — the browser's own push service
* (Google's, Mozilla's, Apple's) carries the message. Written against the standards directly rather than through a library:
*
* - RFC 8292 (VAPID): the server proves who it is with a short-lived signed token (ES256), so a push service knows the sender is the
* one the subscription was made for.
* - RFC 8291 + 8188 (aes128gcm): the message is encrypted for the one browser it is for; the push service only ever carries it.
*
* The server's key pair is made the first time it is needed and kept (the private half encrypted, like the other credentials).
*/
import { createCipheriv, createECDH, createPrivateKey, hkdfSync, randomBytes, sign, type KeyObject } from "node:crypto";
import { eq } from "drizzle-orm";
import { db } from "../db/client.js";
import { pushSubscriptions } from "../db/schema.js";
import { env } from "../env.js";
import { decryptSecret, encryptSecret } from "../crypto.js";
import { resolveTarget } from "../utils/networkGuard.js";
import { getInternalFlag, getSettings, setInternalFlag } from "./settingsStore.js";
const b64u = (buf: Buffer | Uint8Array) => Buffer.from(buf).toString("base64url");
const fromB64u = (s: string) => Buffer.from(s, "base64url");
// ─── The server's own key pair ─────────────────────────────────────────────
const VAPID_FLAG = "webPushVapid";
const ENCRYPTED = "enc:v1:";
export interface VapidKeys {
/** The uncompressed P-256 point, base64url — what a browser is given to subscribe with. */
publicKey: string;
/** The private scalar, base64url. */
privateKey: string;
}
let cachedKeys: Promise<VapidKeys> | null = null;
/** The key pair, made on first use and kept. */
export function getVapidKeys(): Promise<VapidKeys> {
cachedKeys ??= loadOrMakeKeys().catch((err) => {
cachedKeys = null;
throw err;
});
return cachedKeys;
}
async function loadOrMakeKeys(): Promise<VapidKeys> {
const stored = await getInternalFlag(VAPID_FLAG);
if (stored) {
try {
const v = JSON.parse(stored) as { publicKey: string; privateKey: string };
const privateKey = v.privateKey.startsWith(ENCRYPTED) ? decryptSecret(v.privateKey.slice(ENCRYPTED.length)) : v.privateKey;
return { publicKey: v.publicKey, privateKey };
} catch {
// Unreadable (the encryption key changed): made again below — every browser must then subscribe again, which the page says.
}
}
const ecdh = createECDH("prime256v1");
ecdh.generateKeys();
const keys: VapidKeys = { publicKey: b64u(ecdh.getPublicKey()), privateKey: b64u(ecdh.getPrivateKey().length === 32 ? ecdh.getPrivateKey() : Buffer.concat([Buffer.alloc(32 - ecdh.getPrivateKey().length), ecdh.getPrivateKey()])) };
await setInternalFlag(VAPID_FLAG, JSON.stringify({ publicKey: keys.publicKey, privateKey: env.credentialsEncryptionEnabled ? ENCRYPTED + encryptSecret(keys.privateKey) : keys.privateKey }));
// The subscriptions made for the old key can't be used with the new one.
await db.delete(pushSubscriptions);
return keys;
}
/** For the tests, so one test's keys don't leak into the next. */
export function forgetVapidKeysForTests(): void {
cachedKeys = null;
}
/** A P-256 private key as a key object, from its raw scalar and public point. */
function privateKeyObject(keys: VapidKeys): KeyObject {
const pub = fromB64u(keys.publicKey);
return createPrivateKey({
key: { kty: "EC", crv: "P-256", d: keys.privateKey, x: b64u(pub.subarray(1, 33)), y: b64u(pub.subarray(33, 65)) },
format: "jwk",
});
}
/** The Authorization header's token: who we are, who it is for, and for how long (RFC 8292). */
export function vapidAuthorization(audience: string, subject: string, keys: VapidKeys, now: number = Date.now()): string {
const header = b64u(Buffer.from(JSON.stringify({ typ: "JWT", alg: "ES256" })));
const claims = b64u(Buffer.from(JSON.stringify({ aud: audience, exp: Math.floor(now / 1000) + 12 * 3600, sub: subject })));
const signature = sign("sha256", Buffer.from(`${header}.${claims}`), { key: privateKeyObject(keys), dsaEncoding: "ieee-p1363" });
return `vapid t=${header}.${claims}.${b64u(signature)}, k=${keys.publicKey}`;
}
// ─── Encrypting a message for one browser (RFC 8291) ───────────────────────
export interface PushKeys {
/** The browser's public key, base64url. */
p256dh: string;
/** The browser's 16-byte authentication secret, base64url. */
auth: string;
}
const RECORD_SIZE = 4096;
/**
* The body to send: the message encrypted so that only the browser it is for can read it. `ephemeral` and `salt` are chosen fresh each
* time; they are parameters only so the standard's own worked example can be checked.
*/
export function encryptPayload(payload: Buffer, subscription: PushKeys, fixed?: { ephemeral: ECDHLike; salt: Buffer }): Buffer {
const uaPublic = fromB64u(subscription.p256dh);
const authSecret = fromB64u(subscription.auth);
if (uaPublic.length !== 65 || uaPublic[0] !== 4) throw new Error("The browser's public key isn't a P-256 point.");
if (authSecret.length !== 16) throw new Error("The browser's authentication secret isn't 16 bytes.");
const fresh = fixed ? null : createECDH("prime256v1");
fresh?.generateKeys();
const ecdh: ECDHLike = fixed?.ephemeral ?? fresh!;
const asPublic = ecdh.getPublicKey();
const salt = fixed?.salt ?? randomBytes(16);
const secret = ecdh.computeSecret(uaPublic);
const keyInfo = Buffer.concat([Buffer.from("WebPush: info\0"), uaPublic, asPublic]);
const ikm = Buffer.from(hkdfSync("sha256", secret, authSecret, keyInfo, 32));
const cek = Buffer.from(hkdfSync("sha256", ikm, salt, Buffer.from("Content-Encoding: aes128gcm\0"), 16));
const nonce = Buffer.from(hkdfSync("sha256", ikm, salt, Buffer.from("Content-Encoding: nonce\0"), 12));
// One record: the message, then a 0x02 marking it as the last.
if (payload.length > RECORD_SIZE - 17 - 1) throw new Error("The message is too long for one push.");
const cipher = createCipheriv("aes-128-gcm", cek, nonce);
const encrypted = Buffer.concat([cipher.update(Buffer.concat([payload, Buffer.from([2])])), cipher.final(), cipher.getAuthTag()]);
const header = Buffer.alloc(16 + 4 + 1);
salt.copy(header, 0);
header.writeUInt32BE(RECORD_SIZE, 16);
header.writeUInt8(asPublic.length, 20);
return Buffer.concat([header, asPublic, encrypted]);
}
type ECDHLike = Pick<ReturnType<typeof createECDH>, "getPublicKey" | "computeSecret">;
// ─── Subscriptions ─────────────────────────────────────────────────────────
let allowLocalEndpoints = false;
/** Lets the tests push to a service on this machine, which is otherwise refused. */
export function allowLocalPushEndpointsForTests(allow: boolean): void {
allowLocalEndpoints = allow;
}
/** A push service is a public internet service with an https address; anything else is refused before we ever connect to it. */
export async function checkEndpoint(endpoint: string): Promise<void> {
let url: URL;
try {
url = new URL(endpoint);
} catch {
throw new Error("The push address isn't a web address.");
}
if (url.protocol !== "https:" && !(allowLocalEndpoints && url.protocol === "http:")) throw new Error("The push address must be https.");
if (url.username || url.password) throw new Error("The push address can't carry a login.");
if (allowLocalEndpoints) return;
await resolveTarget(url.hostname, { allowPrivate: false });
}
export interface PushOutcome {
sent: number;
/** Subscriptions the push service said are gone, now removed. */
removed: number;
failed: number;
}
/** The contact the push services may reach us on (a `mailto:` or an https address), as VAPID asks. */
export async function vapidSubject(): Promise<string> {
const { webPush } = await getSettings();
const contact = webPush.contact.trim();
if (/^(mailto:|https:\/\/)/i.test(contact)) return contact;
if (contact.includes("@")) return `mailto:${contact}`;
return /^https:\/\//.test(env.appBaseUrl) ? env.appBaseUrl : "mailto:homelab@example.invalid";
}
export interface PushMessage {
title: string;
body: string;
/** Where in the app a tap should go. */
url?: string;
/** A later message with the same tag replaces an earlier one that is still showing. */
tag?: string;
}
/** Sends one message to one subscription. Resolves to what the push service said; never throws. */
async function deliver(sub: { endpoint: string; p256dh: string; auth: string }, body: Buffer, keys: VapidKeys, subject: string, urgent: boolean): Promise<"sent" | "gone" | "failed"> {
try {
await checkEndpoint(sub.endpoint);
const res = await fetch(sub.endpoint, {
method: "POST",
headers: {
Authorization: vapidAuthorization(new URL(sub.endpoint).origin, subject, keys),
"Content-Encoding": "aes128gcm",
"Content-Type": "application/octet-stream",
TTL: "86400",
Urgency: urgent ? "high" : "normal",
},
body: new Uint8Array(body),
signal: AbortSignal.timeout(15_000),
});
if (res.ok) return "sent";
if (res.status === 404 || res.status === 410) return "gone";
console.error(`[push] ${new URL(sub.endpoint).host} answered ${res.status}: ${(await res.text().catch(() => "")).slice(0, 200)}`);
return "failed";
} catch (err) {
console.error("[push] couldn't deliver:", err instanceof Error ? err.message : err);
return "failed";
}
}
/** Sends to the given subscriptions (all of them, if none are given). Subscriptions the push service says are gone are removed. */
export async function sendPush(message: PushMessage, only?: { id: number; endpoint: string; p256dh: string; auth: string }[]): Promise<PushOutcome> {
const subs = only ?? (await db.select().from(pushSubscriptions));
const outcome: PushOutcome = { sent: 0, removed: 0, failed: 0 };
if (subs.length === 0) return outcome;
const keys = await getVapidKeys();
const subject = await vapidSubject();
const payload = Buffer.from(JSON.stringify({ title: message.title.slice(0, 120), body: message.body.slice(0, 600), url: message.url ?? "/alerts", tag: message.tag }));
await Promise.all(
subs.map(async (s) => {
let body: Buffer;
try {
body = encryptPayload(payload, { p256dh: s.p256dh, auth: s.auth });
} catch (err) {
console.error("[push] a subscription can't be written to:", err instanceof Error ? err.message : err);
outcome.failed++;
return;
}
const result = await deliver(s, body, keys, subject, true);
if (result === "sent") {
outcome.sent++;
await db.update(pushSubscriptions).set({ lastSuccessAt: new Date().toISOString() }).where(eq(pushSubscriptions.id, s.id));
} else if (result === "gone") {
outcome.removed++;
await db.delete(pushSubscriptions).where(eq(pushSubscriptions.id, s.id));
} else {
outcome.failed++;
}
}),
);
return outcome;
}
/** How many browsers are subscribed. */
export async function subscriptionCount(): Promise<number> {
return (await db.select({ id: pushSubscriptions.id }).from(pushSubscriptions)).length;
}