Dependencies beyond servers, web push, inventory auto-fill with QR labels, Windows agent health
- Dependencies: servers and integrations can depend on each other; alerts are folded under the root cause, and a failure explained by something down isn't announced - Web push: hand-written VAPID and RFC 8291 encryption, per-device subscriptions, a fifth notification channel, service worker handlers - Inventory: maker/model/serial from the agents fill blank fields, "Refresh from agent", QR label printing and ?asset= deep links - Windows agent (v4) reports failed services, disk health, Storage Spaces pools and hardware identity; Linux agent (v5) reports hardware identity Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
5cc51c57de
commit
ec9c2f73f0
55 files changed
+12553
-65
No files matched your search
@@ -15,6 +15,8 @@ import { updateRunsRouter } from "./routes/updateRuns.js";
|
||||
import { dbBackupsRouter } from "./routes/dbBackups.js";
|
||||
import { incidentsRouter } from "./routes/incidents.js";
|
||||
import { inventoryRouter } from "./routes/inventory.js";
|
||||
import { dependenciesRouter } from "./routes/dependencies.js";
|
||||
import { pushRouter } from "./routes/push.js";
|
||||
import { backupsRouter } from "./routes/backups.js";
|
||||
import { metricsRouter, metricsTokensRouter } from "./routes/metrics.js";
|
||||
import { eventWebhooksRouter } from "./routes/eventWebhooks.js";
|
||||
@@ -106,6 +108,8 @@ export function createApp() {
|
||||
app.use("/api/db-backups", dbBackupsRouter);
|
||||
app.use("/api/incidents", incidentsRouter);
|
||||
app.use("/api/inventory", inventoryRouter);
|
||||
app.use("/api/dependencies", dependenciesRouter);
|
||||
app.use("/api/push", pushRouter);
|
||||
app.use("/api/backups", backupsRouter);
|
||||
app.use("/api/metrics-tokens", metricsTokensRouter);
|
||||
app.use("/api/event-webhooks", eventWebhooksRouter);
|
||||
|
||||
@@ -103,6 +103,41 @@ export const maintenanceAliases = sqliteTable(
|
||||
(table) => [uniqueIndex("maintenance_aliases_unique_idx").on(table.address, table.targetType, table.targetId)],
|
||||
);
|
||||
|
||||
// A browser or phone that asked for push notifications (one row per device). The endpoint is the browser's push service address, which is
|
||||
// itself a secret: whoever has it with the keys can write to that device. Removed when the push service says it is gone, or the user signs out of it.
|
||||
export const pushSubscriptions = sqliteTable("push_subscriptions", {
|
||||
id: integer("id").primaryKey({ autoIncrement: true }),
|
||||
userId: integer("user_id")
|
||||
.notNull()
|
||||
.references(() => users.id, { onDelete: "cascade" }),
|
||||
endpoint: text("endpoint").notNull().unique(),
|
||||
p256dh: text("p256dh").notNull(),
|
||||
auth: text("auth").notNull(),
|
||||
/** What the person calls it, or the browser it is. */
|
||||
label: text("label"),
|
||||
createdAt: text("created_at")
|
||||
.notNull()
|
||||
.default(sql`(current_timestamp)`),
|
||||
lastSuccessAt: text("last_success_at"),
|
||||
});
|
||||
|
||||
// "A needs B", for things that can be down: a server or an integration each. Polymorphic like maintenance_windows (no foreign key); an edge whose
|
||||
// end was deleted is ignored. When B is down, A's alert is folded under B's — see services/dependencies.ts.
|
||||
export const dependencies = sqliteTable(
|
||||
"dependencies",
|
||||
{
|
||||
id: integer("id").primaryKey({ autoIncrement: true }),
|
||||
fromType: text("from_type").notNull(), // server | integration
|
||||
fromId: integer("from_id").notNull(),
|
||||
toType: text("to_type").notNull(),
|
||||
toId: integer("to_id").notNull(),
|
||||
createdAt: text("created_at")
|
||||
.notNull()
|
||||
.default(sql`(current_timestamp)`),
|
||||
},
|
||||
(table) => [uniqueIndex("dependencies_edge_idx").on(table.fromType, table.fromId, table.toType, table.toId)],
|
||||
);
|
||||
|
||||
// A short history of each agent-reported server's load, memory and disk use, in percent — see services/serverMetrics.ts.
|
||||
// `metric` is "cpu", "mem" or "disk:<mount point>".
|
||||
export const serverMetrics = sqliteTable(
|
||||
@@ -300,6 +335,11 @@ export const servers = sqliteTable("servers", {
|
||||
// What the agent says about the machine's own health, as JSON: {failedUnits, smart, zfs, reportedAt} — see services/hostHealth.ts. Null until it says.
|
||||
hostHealth: text("host_health"),
|
||||
|
||||
// What the machine is, as its firmware says (agents report it): maker, model and serial number. Used to fill in the inventory.
|
||||
hwManufacturer: text("hw_manufacturer"),
|
||||
hwModel: text("hw_model"),
|
||||
hwSerial: text("hw_serial"),
|
||||
|
||||
// This server's own alert thresholds, as JSON: {serverOfflineMinutes, diskUsagePercent, diskForecastDays, backupStaleHours, mounts: {"/data": {percent, ignore}}} — see
|
||||
// services/serverThresholds.ts. Whatever isn't there follows Settings. Null when it follows Settings entirely.
|
||||
thresholds: text("thresholds"),
|
||||
|
||||
@@ -52,7 +52,21 @@ const healthSchema = z.object({
|
||||
.catch(undefined),
|
||||
});
|
||||
|
||||
// What the machine is. Each part is optional and a malformed one is dropped, like the rest.
|
||||
const hardwareSchema = z.object({
|
||||
manufacturer: text(100).optional().catch(undefined),
|
||||
model: text(150).optional().catch(undefined),
|
||||
serial: text(100).optional().catch(undefined),
|
||||
});
|
||||
|
||||
const systemSchema = z.object({
|
||||
hardware: z
|
||||
.unknown()
|
||||
.optional()
|
||||
.transform((h) => {
|
||||
const parsed = hardwareSchema.safeParse(h);
|
||||
return parsed.success && Object.values(parsed.data).some((v) => typeof v === "string" && v.trim()) ? parsed.data : undefined;
|
||||
}),
|
||||
health: z
|
||||
.unknown()
|
||||
.optional()
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
import { Router } from "express";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { z } from "zod";
|
||||
import { db } from "../db/client.js";
|
||||
import { dependencies } from "../db/schema.js";
|
||||
import { requireAuth, requireRole } from "../auth/middleware.js";
|
||||
import { recordAudit } from "../services/audit.js";
|
||||
import { invalidateAlertsCache } from "../services/alerts.js";
|
||||
import { loadGraph, nodeKey, nodeTypes, wouldLoop } from "../services/dependencies.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
|
||||
export const dependenciesRouter = Router();
|
||||
|
||||
dependenciesRouter.use(requireAuth);
|
||||
|
||||
const edgeSchema = z.object({
|
||||
fromType: z.enum(nodeTypes),
|
||||
fromId: z.number().int().positive(),
|
||||
toType: z.enum(nodeTypes),
|
||||
toId: z.number().int().positive(),
|
||||
});
|
||||
|
||||
// Everything signed in people may see: what needs what, with the "runs on" links of servers alongside (those are set on the server itself).
|
||||
dependenciesRouter.get("/", asyncHandler(async (_req, res) => {
|
||||
const graph = await loadGraph();
|
||||
const written = new Set(graph.rows.map((r) => `${nodeKey(r.fromType, r.fromId)}>${nodeKey(r.toType, r.toId)}`));
|
||||
const runsOn: { from: string; to: string }[] = [];
|
||||
for (const [from, tos] of graph.edges) for (const to of tos) if (!written.has(`${from}>${to}`)) runsOn.push({ from, to });
|
||||
const nodes = [...graph.names.entries()]
|
||||
.map(([key, name]) => {
|
||||
const [type, id] = key.split(":");
|
||||
return { key, type, id: Number(id), name };
|
||||
})
|
||||
.sort((a, b) => a.type.localeCompare(b.type) || a.name.localeCompare(b.name));
|
||||
res.json({
|
||||
dependencies: graph.rows
|
||||
.filter((r) => graph.names.has(nodeKey(r.fromType, r.fromId)) && graph.names.has(nodeKey(r.toType, r.toId)))
|
||||
.map((r) => ({ id: r.id, from: { type: r.fromType, id: r.fromId, name: graph.names.get(nodeKey(r.fromType, r.fromId))! }, to: { type: r.toType, id: r.toId, name: graph.names.get(nodeKey(r.toType, r.toId))! }, createdAt: r.createdAt })),
|
||||
runsOn: runsOn.map((r) => ({ from: { key: r.from, name: graph.names.get(r.from) ?? r.from }, to: { key: r.to, name: graph.names.get(r.to) ?? r.to } })),
|
||||
nodes,
|
||||
});
|
||||
}));
|
||||
|
||||
dependenciesRouter.post("/", requireRole("operator"), asyncHandler(async (req, res) => {
|
||||
const parsed = edgeSchema.safeParse(req.body);
|
||||
if (!parsed.success) return res.status(400).json({ error: "invalid_body", message: "Say what needs what: a server or an integration each.", details: parsed.error.flatten() });
|
||||
const d = parsed.data;
|
||||
const graph = await loadGraph();
|
||||
const from = nodeKey(d.fromType, d.fromId);
|
||||
const to = nodeKey(d.toType, d.toId);
|
||||
if (!graph.names.has(from) || !graph.names.has(to)) return res.status(400).json({ error: "unknown_node", message: "One of those doesn't exist." });
|
||||
if (from === to) return res.status(400).json({ error: "self", message: "Something can't depend on itself." });
|
||||
if (graph.edges.get(from)?.includes(to)) return res.status(409).json({ error: "duplicate", message: "That is already so." });
|
||||
if (wouldLoop(graph.edges, from, to)) {
|
||||
return res.status(409).json({ error: "loop", message: `${graph.names.get(to)} already depends on ${graph.names.get(from)}, so this would make them wait for each other.` });
|
||||
}
|
||||
const [row] = await db.insert(dependencies).values(d).returning();
|
||||
await recordAudit({ actor: req.currentUser!, category: "dependency", action: "create", targetType: "dependency", targetId: row.id, detail: { from: graph.names.get(from), to: graph.names.get(to) } });
|
||||
invalidateAlertsCache();
|
||||
res.status(201).json({ id: row.id });
|
||||
}));
|
||||
|
||||
dependenciesRouter.delete("/:id", requireRole("operator"), asyncHandler(async (req, res) => {
|
||||
const id = /^\d+$/.test(req.params.id) ? Number(req.params.id) : NaN;
|
||||
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
|
||||
const [before] = await db.select().from(dependencies).where(eq(dependencies.id, id)).limit(1);
|
||||
if (!before) return res.status(404).json({ error: "not_found" });
|
||||
const graph = await loadGraph();
|
||||
await db.delete(dependencies).where(eq(dependencies.id, id));
|
||||
await recordAudit({
|
||||
actor: req.currentUser!,
|
||||
category: "dependency",
|
||||
action: "delete",
|
||||
targetType: "dependency",
|
||||
targetId: id,
|
||||
detail: { from: graph.names.get(nodeKey(before.fromType as "server", before.fromId)) ?? `${before.fromType} #${before.fromId}`, to: graph.names.get(nodeKey(before.toType as "server", before.toId)) ?? `${before.toType} #${before.toId}` },
|
||||
});
|
||||
invalidateAlertsCache();
|
||||
res.status(204).end();
|
||||
}));
|
||||
@@ -6,6 +6,7 @@ import { assetKinds, assets, servers } from "../db/schema.js";
|
||||
import { requireAuth, requireRole } from "../auth/middleware.js";
|
||||
import { recordAudit } from "../services/audit.js";
|
||||
import { isRealDate } from "../services/inventory.js";
|
||||
import { blanksToFill, differences, hardwareOf } from "../services/inventoryAutofill.js";
|
||||
import { getSettings, updateSettings } from "../services/settingsStore.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
|
||||
@@ -53,16 +54,32 @@ function values(d: z.infer<typeof assetSchema>) {
|
||||
};
|
||||
}
|
||||
|
||||
/** A linked entry's blank maker, model and serial, from what that server's agent says. */
|
||||
async function fillFromAgent(v: ReturnType<typeof values>) {
|
||||
if (v.serverId === null) return {};
|
||||
const [server] = await db.select({ hwManufacturer: servers.hwManufacturer, hwModel: servers.hwModel, hwSerial: servers.hwSerial }).from(servers).where(eq(servers.id, v.serverId)).limit(1);
|
||||
return blanksToFill(v, hardwareOf(server));
|
||||
}
|
||||
|
||||
const idOf = (v: string) => (/^\d+$/.test(v) ? Number(v) : NaN);
|
||||
|
||||
// Everything signed-in people may see: the hardware, and the price of power the running cost is worked out with.
|
||||
inventoryRouter.get("/", asyncHandler(async (_req, res) => {
|
||||
const [rows, serverRows, settings] = await Promise.all([db.select().from(assets).orderBy(assets.name), db.select({ id: servers.id, name: servers.name }).from(servers), getSettings()]);
|
||||
const serverName = new Map(serverRows.map((s) => [s.id, s.name]));
|
||||
const [rows, serverRows, settings] = await Promise.all([db.select().from(assets).orderBy(assets.name), db.select().from(servers), getSettings()]);
|
||||
const byId = new Map(serverRows.map((s) => [s.id, s]));
|
||||
res.json({
|
||||
assets: rows.map((a) => ({ ...a, server: a.serverId !== null && serverName.has(a.serverId) ? { id: a.serverId, name: serverName.get(a.serverId)! } : null })),
|
||||
assets: rows.map((a) => {
|
||||
const s = a.serverId !== null ? byId.get(a.serverId) : undefined;
|
||||
const hw = hardwareOf(s);
|
||||
return {
|
||||
...a,
|
||||
server: s ? { id: s.id, name: s.name } : null,
|
||||
// What the machine's agent says it is, and what that would change here (for the "refresh from agent" button).
|
||||
agent: hw ? { ...hw, differs: differences(a, hw) } : null,
|
||||
};
|
||||
}),
|
||||
config: settings.inventory,
|
||||
servers: serverRows.sort((a, b) => a.name.localeCompare(b.name)),
|
||||
servers: serverRows.map((s) => ({ id: s.id, name: s.name })).sort((a, b) => a.name.localeCompare(b.name)),
|
||||
});
|
||||
}));
|
||||
|
||||
@@ -89,8 +106,9 @@ inventoryRouter.post("/", requireRole("operator"), asyncHandler(async (req, res)
|
||||
return res.status(400).json({ error: "unknown_server", message: "That server doesn't exist." });
|
||||
}
|
||||
const now = new Date().toISOString();
|
||||
const [row] = await db.insert(assets).values({ ...v, createdAt: now, updatedAt: now }).returning();
|
||||
await recordAudit({ actor: req.currentUser!, category: "inventory", action: "create", targetType: "asset", targetId: row.id, detail: { name: row.name, kind: row.kind } });
|
||||
const filled = await fillFromAgent(v);
|
||||
const [row] = await db.insert(assets).values({ ...v, ...filled, createdAt: now, updatedAt: now }).returning();
|
||||
await recordAudit({ actor: req.currentUser!, category: "inventory", action: "create", targetType: "asset", targetId: row.id, detail: { name: row.name, kind: row.kind, ...(Object.keys(filled).length ? { filledFromAgent: Object.keys(filled) } : {}) } });
|
||||
res.status(201).json({ asset: row });
|
||||
}));
|
||||
|
||||
@@ -104,11 +122,27 @@ inventoryRouter.put("/:id", requireRole("operator"), asyncHandler(async (req, re
|
||||
if (v.serverId !== null && (await db.select({ id: servers.id }).from(servers).where(eq(servers.id, v.serverId))).length === 0) {
|
||||
return res.status(400).json({ error: "unknown_server", message: "That server doesn't exist." });
|
||||
}
|
||||
const [row] = await db.update(assets).set({ ...v, updatedAt: new Date().toISOString() }).where(eq(assets.id, id)).returning();
|
||||
await recordAudit({ actor: req.currentUser!, category: "inventory", action: "update", targetType: "asset", targetId: id, detail: { name: row.name } });
|
||||
const filled = await fillFromAgent(v);
|
||||
const [row] = await db.update(assets).set({ ...v, ...filled, updatedAt: new Date().toISOString() }).where(eq(assets.id, id)).returning();
|
||||
await recordAudit({ actor: req.currentUser!, category: "inventory", action: "update", targetType: "asset", targetId: id, detail: { name: row.name, ...(Object.keys(filled).length ? { filledFromAgent: Object.keys(filled) } : {}) } });
|
||||
res.json({ asset: row });
|
||||
}));
|
||||
|
||||
// Takes what the agent says over what is written here: the one deliberate overwrite (the entry's own fields are otherwise only ever filled when blank).
|
||||
inventoryRouter.post("/:id/refresh-from-agent", requireRole("operator"), asyncHandler(async (req, res) => {
|
||||
const id = idOf(req.params.id);
|
||||
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
|
||||
const [before] = await db.select().from(assets).where(eq(assets.id, id)).limit(1);
|
||||
if (!before) return res.status(404).json({ error: "not_found" });
|
||||
if (before.serverId === null) return res.status(400).json({ error: "not_linked", message: "Link the entry to a server first." });
|
||||
const [server] = await db.select({ hwManufacturer: servers.hwManufacturer, hwModel: servers.hwModel, hwSerial: servers.hwSerial }).from(servers).where(eq(servers.id, before.serverId)).limit(1);
|
||||
const changes = differences(before, hardwareOf(server));
|
||||
if (Object.keys(changes).length === 0) return res.json({ asset: before, changed: [] });
|
||||
const [row] = await db.update(assets).set({ ...changes, updatedAt: new Date().toISOString() }).where(eq(assets.id, id)).returning();
|
||||
await recordAudit({ actor: req.currentUser!, category: "inventory", action: "refresh_from_agent", targetType: "asset", targetId: id, detail: { name: row.name, from: Object.fromEntries(Object.keys(changes).map((k) => [k, (before as any)[k]])), to: changes } });
|
||||
res.json({ asset: row, changed: Object.keys(changes) });
|
||||
}));
|
||||
|
||||
inventoryRouter.delete("/:id", requireRole("operator"), asyncHandler(async (req, res) => {
|
||||
const id = idOf(req.params.id);
|
||||
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
|
||||
|
||||
@@ -63,6 +63,7 @@ privacyRouter.get("/", asyncHandler(async (req, res) => {
|
||||
ntfy: channel(settings.ntfy.enabled, hostOf(settings.ntfy.url)),
|
||||
smtp: channel(settings.smtp.enabled, settings.smtp.host || null),
|
||||
webhook: channel(settings.webhook.enabled, hostOf(settings.webhook.url)),
|
||||
webPush: channel(settings.webPush.enabled, "the push services of your browsers (Google, Mozilla, Apple …)"),
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
import { Router } from "express";
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { z } from "zod";
|
||||
import { db } from "../db/client.js";
|
||||
import { pushSubscriptions } from "../db/schema.js";
|
||||
import { requireAuth } from "../auth/middleware.js";
|
||||
import { recordAudit } from "../services/audit.js";
|
||||
import { getSettings } from "../services/settingsStore.js";
|
||||
import { checkEndpoint, getVapidKeys, sendPush } from "../services/webPush.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
|
||||
export const pushRouter = Router();
|
||||
|
||||
pushRouter.use(requireAuth);
|
||||
|
||||
/** At most this many browsers per person: a phone, a laptop, a tablet and the odd one. */
|
||||
const MAX_PER_USER = 10;
|
||||
|
||||
const subscribeSchema = z.object({
|
||||
endpoint: z.string().url().max(2000),
|
||||
keys: z.object({ p256dh: z.string().min(80).max(100), auth: z.string().min(20).max(30) }),
|
||||
label: z.string().trim().max(80).optional(),
|
||||
});
|
||||
|
||||
const hostOf = (endpoint: string) => {
|
||||
try {
|
||||
return new URL(endpoint).host;
|
||||
} catch {
|
||||
return "";
|
||||
}
|
||||
};
|
||||
|
||||
const off = { error: "push_disabled", message: "Push notifications are switched off. An administrator turns them on under Settings, Notifications." };
|
||||
|
||||
// The key a browser subscribes with. Anyone signed in may ask (their own device is theirs to enrol), once push is on.
|
||||
pushRouter.get("/key", asyncHandler(async (_req, res) => {
|
||||
const { webPush } = await getSettings();
|
||||
if (!webPush.enabled) return res.status(409).json(off);
|
||||
res.json({ publicKey: (await getVapidKeys()).publicKey });
|
||||
}));
|
||||
|
||||
// This person's own browsers.
|
||||
pushRouter.get("/mine", asyncHandler(async (req, res) => {
|
||||
const rows = await db.select().from(pushSubscriptions).where(eq(pushSubscriptions.userId, req.currentUser!.id)).orderBy(pushSubscriptions.id);
|
||||
const { webPush } = await getSettings();
|
||||
res.json({
|
||||
enabled: webPush.enabled,
|
||||
subscriptions: rows.map((r) => ({ id: r.id, label: r.label, host: hostOf(r.endpoint), endpoint: r.endpoint, createdAt: r.createdAt, lastSuccessAt: r.lastSuccessAt })),
|
||||
});
|
||||
}));
|
||||
|
||||
pushRouter.post("/subscribe", asyncHandler(async (req, res) => {
|
||||
const { webPush } = await getSettings();
|
||||
if (!webPush.enabled) return res.status(409).json(off);
|
||||
const parsed = subscribeSchema.safeParse(req.body);
|
||||
if (!parsed.success) return res.status(400).json({ error: "invalid_body", message: "That isn't a push subscription.", details: parsed.error.flatten() });
|
||||
const { endpoint, keys, label } = parsed.data;
|
||||
try {
|
||||
await checkEndpoint(endpoint);
|
||||
} catch (err) {
|
||||
return res.status(400).json({ error: "bad_endpoint", message: err instanceof Error ? err.message : "That push address can't be used." });
|
||||
}
|
||||
const me = req.currentUser!;
|
||||
const mine = await db.select({ id: pushSubscriptions.id, endpoint: pushSubscriptions.endpoint }).from(pushSubscriptions).where(eq(pushSubscriptions.userId, me.id));
|
||||
const existing = mine.find((r) => r.endpoint === endpoint);
|
||||
if (!existing && mine.length >= MAX_PER_USER) return res.status(409).json({ error: "too_many", message: `You already have ${MAX_PER_USER} browsers subscribed. Remove one first.` });
|
||||
// The same browser subscribing again (or after someone else used it) is the same row, now this person's.
|
||||
const [row] = await db
|
||||
.insert(pushSubscriptions)
|
||||
.values({ userId: me.id, endpoint, p256dh: keys.p256dh, auth: keys.auth, label: label || null })
|
||||
.onConflictDoUpdate({ target: pushSubscriptions.endpoint, set: { userId: me.id, p256dh: keys.p256dh, auth: keys.auth, label: label || null } })
|
||||
.returning({ id: pushSubscriptions.id });
|
||||
if (!existing) await recordAudit({ actor: me, category: "push", action: "subscribe", targetType: "push_subscription", targetId: row.id, detail: { host: hostOf(endpoint), label: label || null } });
|
||||
res.status(201).json({ id: row.id });
|
||||
}));
|
||||
|
||||
// Stops a browser being written to. A person removes their own; an administrator may remove anyone's.
|
||||
pushRouter.delete("/:id", asyncHandler(async (req, res) => {
|
||||
const id = /^\d+$/.test(req.params.id) ? Number(req.params.id) : NaN;
|
||||
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
|
||||
const me = req.currentUser!;
|
||||
const [row] = await db.select().from(pushSubscriptions).where(eq(pushSubscriptions.id, id)).limit(1);
|
||||
if (!row || (row.userId !== me.id && me.role !== "admin")) return res.status(404).json({ error: "not_found" });
|
||||
await db.delete(pushSubscriptions).where(and(eq(pushSubscriptions.id, id)));
|
||||
await recordAudit({ actor: me, category: "push", action: "unsubscribe", targetType: "push_subscription", targetId: id, detail: { host: hostOf(row.endpoint), label: row.label } });
|
||||
res.status(204).end();
|
||||
}));
|
||||
|
||||
// A message to this person's own browsers, to see that it works.
|
||||
pushRouter.post("/test", asyncHandler(async (req, res) => {
|
||||
const { webPush } = await getSettings();
|
||||
if (!webPush.enabled) return res.status(409).json(off);
|
||||
const mine = await db.select().from(pushSubscriptions).where(eq(pushSubscriptions.userId, req.currentUser!.id));
|
||||
if (mine.length === 0) return res.status(400).json({ error: "none", message: "This browser isn't subscribed yet." });
|
||||
const outcome = await sendPush({ title: "Homelab Manager — Test", body: "Push notifications are working.", url: "/settings/notifications", tag: "test" }, mine);
|
||||
res.json(outcome);
|
||||
}));
|
||||
@@ -64,6 +64,7 @@ const updateSchema = z.object({
|
||||
.partial()
|
||||
.optional(),
|
||||
webhook: z.object({ enabled: z.boolean(), url: z.string(), secret: z.string() }).partial().optional(),
|
||||
webPush: z.object({ enabled: z.boolean(), contact: z.string().trim().max(200) }).partial().optional(),
|
||||
notifications: z
|
||||
.object({
|
||||
dnsAdd: z.boolean(),
|
||||
|
||||
@@ -8,7 +8,9 @@
|
||||
* 2 — reports pending updates, and its own version
|
||||
* 3 — Linux: the package manager is asked outside the systemd sandbox, so dnf and zypper can answer too.
|
||||
* Windows: can replace itself when an administrator has switched automatic updates on.
|
||||
* 4 — Linux: also reports failed services, disk health (SMART) and ZFS pools, reads pacman, and can replace itself too
|
||||
* 4 — Linux: also reports failed services, disk health (SMART) and ZFS pools, reads pacman, and can replace itself too.
|
||||
* Windows: also reports failed services, disk health and Storage Spaces pools, and what the machine is (maker, model, serial)
|
||||
* 5 — Linux: also says what the machine is (maker, model, serial), for the inventory
|
||||
*/
|
||||
import { createHash } from "node:crypto";
|
||||
import { readFileSync } from "node:fs";
|
||||
|
||||
@@ -33,7 +33,7 @@ import { computeSecretStatus } from "./secretStatus.js";
|
||||
import { getFailingSources } from "./integrationHealthMonitor.js";
|
||||
import { getSettings } from "./settingsStore.js";
|
||||
import { sourceLabel } from "./notify.js";
|
||||
import { coveredByParent, loadParentRows, resolveParents } from "./serverParents.js";
|
||||
import { dependencyCover, integrationTypeCause, takenByType } from "./dependencies.js";
|
||||
import type { Alert, AlertCategory, AlertSeverity, SourceFailure } from "./alertTypes.js";
|
||||
|
||||
export interface AlertsReport {
|
||||
@@ -158,6 +158,10 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
}
|
||||
}
|
||||
|
||||
// The servers found to be offline, for the integration failures that one may explain: the server check fills it in (and it is empty if that check fails).
|
||||
let offlineKnown!: (ids: Set<number>) => void;
|
||||
const offlineServers = new Promise<Set<number>>((resolve) => (offlineKnown = resolve));
|
||||
|
||||
await Promise.all([
|
||||
check("Server and storage health", async () => {
|
||||
const { healthChecks } = await getSettings();
|
||||
@@ -176,17 +180,17 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
const m = /^offline:server:(\d+)$/.exec(c.key);
|
||||
if (m) offlineIds.add(Number(m[1]));
|
||||
}
|
||||
const cause = coveredByParent(offlineIds, resolveParents(await loadParentRows()));
|
||||
const taken = new Map<number, string[]>();
|
||||
for (const [child, parent] of cause) taken.set(parent, [...(taken.get(parent) ?? []), serverNames.get(child) ?? `#${child}`]);
|
||||
// (Or because of a service it needs that is failing, or a server it needs that needs one.)
|
||||
offlineKnown(offlineIds);
|
||||
const cover = await dependencyCover(offlineIds, getFailingSources().map((f) => f.source));
|
||||
for (const c of conditions) {
|
||||
const origin = originOfConditionKey(c.key);
|
||||
const subject = subjectOfConditionKey(c.key);
|
||||
const offlineId = /^offline:server:(\d+)$/.exec(c.key)?.[1];
|
||||
const parentOf = offlineId ? cause.get(Number(offlineId)) : undefined;
|
||||
const parentOf = offlineId ? cover.cause.get(`server:${offlineId}`) : undefined;
|
||||
push({
|
||||
causedBy: parentOf !== undefined ? { name: serverNames.get(parentOf) ?? `#${parentOf}` } : undefined,
|
||||
affects: offlineId ? taken.get(Number(offlineId))?.sort((a, b) => a.localeCompare(b)) : undefined,
|
||||
causedBy: parentOf !== undefined ? { name: cover.graph.names.get(parentOf) ?? parentOf } : undefined,
|
||||
affects: offlineId ? cover.taken.get(`server:${offlineId}`)?.sort((a, b) => a.localeCompare(b)) : undefined,
|
||||
category: origin.category,
|
||||
severity: c.severity ?? "warning",
|
||||
type: origin.type,
|
||||
@@ -203,7 +207,7 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
const row = intById.get(Number(m[2]));
|
||||
cannotCheck(`${SOURCES[m[1]].label} “${row?.name ?? `#${m[2]}`}”${m[3] ? ` (node ${m[3]})` : ""}`, "couldn't be read — see the Diagnostic Log", "Server and storage health");
|
||||
}
|
||||
}),
|
||||
}).finally(() => offlineKnown(new Set())),
|
||||
|
||||
check("Automation runs", async () => {
|
||||
const { items, held } = await collectAutomation();
|
||||
@@ -447,8 +451,15 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
|
||||
check("Integration failures", async () => {
|
||||
const { notifications } = await getSettings();
|
||||
for (const f of getFailingSources()) {
|
||||
const failing = getFailingSources();
|
||||
// A failure explained by something it needs that is down is folded under that, like a guest under its host.
|
||||
const cover = failing.length > 0 ? await dependencyCover(await offlineServers, failing.map((f) => f.source)).catch(() => null) : null;
|
||||
for (const f of failing) {
|
||||
const cause = cover ? integrationTypeCause(f.source, cover) : null;
|
||||
const affects = cover ? takenByType(f.source, cover) : [];
|
||||
push({
|
||||
causedBy: cause ? { name: cause.name } : undefined,
|
||||
affects: affects.length > 0 ? affects.sort((a, b) => a.localeCompare(b)) : undefined,
|
||||
category: "integration",
|
||||
severity: f.alerted ? "critical" : "warning",
|
||||
type: f.source,
|
||||
|
||||
@@ -0,0 +1,192 @@
|
||||
/**
|
||||
* What depends on what, beyond "this server runs on that server": a service that needs a server (Gitea on the box it is installed on), a
|
||||
* server that needs a service (a guest that only exists while Proxmox answers). When the thing depended on is down, the things that need it
|
||||
* are down too — and only the one that matters is worth an alert, so the rest are folded under it: shown, not announced.
|
||||
*
|
||||
* The graph is the "runs on" links of servers (set by hand or worked out from Proxmox, see serverParents.ts) plus the dependencies written
|
||||
* here. Servers and integrations are the things that can be down and be needed. (DNS providers raise no alerts of their own, so there is
|
||||
* nothing to fold.) An integration is judged per kind of service, as its health is: it is folded only if every enabled instance of that
|
||||
* kind is explained.
|
||||
*/
|
||||
import { eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { dependencies, integrations, servers } from "../db/schema.js";
|
||||
import { getInternalFlag } from "./settingsStore.js";
|
||||
import { loadParentRows, resolveParents, type ParentLink } from "./serverParents.js";
|
||||
|
||||
/** Where the health monitor keeps the problems it is announcing; read here so a service's alert can tell a server it needs is down. */
|
||||
export const HEALTH_STATE_FLAG = "healthActiveConditions";
|
||||
|
||||
export const nodeTypes = ["server", "integration"] as const;
|
||||
export type NodeType = (typeof nodeTypes)[number];
|
||||
|
||||
export const nodeKey = (type: NodeType, id: number) => `${type}:${id}`;
|
||||
|
||||
export interface DependencyEdge {
|
||||
fromType: NodeType;
|
||||
fromId: number;
|
||||
toType: NodeType;
|
||||
toId: number;
|
||||
}
|
||||
|
||||
/** node → the nodes it needs. */
|
||||
export type Edges = Map<string, string[]>;
|
||||
|
||||
/** The graph from the "runs on" links and the written dependencies; an edge to or from something that no longer exists is left out. */
|
||||
export function buildEdges(parentLinks: Map<number, ParentLink>, rows: DependencyEdge[], exists: (key: string) => boolean): Edges {
|
||||
const edges: Edges = new Map();
|
||||
const add = (from: string, to: string) => {
|
||||
if (from === to || !exists(from) || !exists(to)) return;
|
||||
const list = edges.get(from) ?? [];
|
||||
if (!list.includes(to)) edges.set(from, [...list, to]);
|
||||
};
|
||||
for (const [child, link] of parentLinks) add(nodeKey("server", child), nodeKey("server", link.parentId));
|
||||
for (const r of rows) add(nodeKey(r.fromType, r.fromId), nodeKey(r.toType, r.toId));
|
||||
return edges;
|
||||
}
|
||||
|
||||
/** Everything `node` needs, directly or not, nearest first; `looped` if it ends up needing itself. */
|
||||
function ancestorsOf(node: string, edges: Edges): { order: string[]; looped: boolean } {
|
||||
const order: string[] = [];
|
||||
const seen = new Set<string>();
|
||||
const queue = [...(edges.get(node) ?? [])];
|
||||
let looped = false;
|
||||
while (queue.length > 0) {
|
||||
const cur = queue.shift()!;
|
||||
if (cur === node) {
|
||||
looped = true;
|
||||
continue;
|
||||
}
|
||||
if (seen.has(cur)) continue;
|
||||
seen.add(cur);
|
||||
order.push(cur);
|
||||
queue.push(...(edges.get(cur) ?? []));
|
||||
}
|
||||
return { order, looped };
|
||||
}
|
||||
|
||||
/**
|
||||
* Of the nodes that are down, those whose cause is something they need that is down too: each maps to the root cause — a down node that
|
||||
* needs nothing that is down. A loop explains nothing (each would be waiting for the other), so its members stand alone and are announced.
|
||||
*/
|
||||
export function coveredNodes(down: Set<string>, edges: Edges): Map<string, string> {
|
||||
const covered = new Map<string, string>();
|
||||
for (const node of [...down].sort()) {
|
||||
const { order, looped } = ancestorsOf(node, edges);
|
||||
if (looped) continue;
|
||||
const downAbove = order.filter((a) => down.has(a));
|
||||
const root = downAbove.find((a) => {
|
||||
const above = ancestorsOf(a, edges);
|
||||
return !above.looped && !above.order.some((x) => down.has(x));
|
||||
});
|
||||
if (root !== undefined) covered.set(node, root);
|
||||
}
|
||||
return covered;
|
||||
}
|
||||
|
||||
/** Would `from` needing `to` close a loop? True if `to` is `from` or already needs it, however indirectly. */
|
||||
export function wouldLoop(edges: Edges, from: string, to: string): boolean {
|
||||
return from === to || ancestorsOf(to, edges).order.includes(from) || ancestorsOf(to, edges).looped;
|
||||
}
|
||||
|
||||
export interface Graph {
|
||||
edges: Edges;
|
||||
/** node → its name, for every node that exists. */
|
||||
names: Map<string, string>;
|
||||
rows: (DependencyEdge & { id: number; createdAt: string })[];
|
||||
/** Enabled instances of each kind of integration. */
|
||||
instances: Map<string, number[]>;
|
||||
intRows: { id: number; type: string; enabled: boolean }[];
|
||||
}
|
||||
|
||||
export async function loadGraph(): Promise<Graph> {
|
||||
const [serverRows, intRows, depRows, parentRows] = await Promise.all([
|
||||
db.select({ id: servers.id, name: servers.name }).from(servers),
|
||||
db.select({ id: integrations.id, name: integrations.name, type: integrations.type, enabled: integrations.enabled }).from(integrations),
|
||||
db.select().from(dependencies),
|
||||
loadParentRows(),
|
||||
]);
|
||||
const names = new Map<string, string>();
|
||||
for (const s of serverRows) names.set(nodeKey("server", s.id), s.name);
|
||||
for (const i of intRows) names.set(nodeKey("integration", i.id), i.name);
|
||||
const rows = depRows.map((r) => ({ id: r.id, fromType: r.fromType as NodeType, fromId: r.fromId, toType: r.toType as NodeType, toId: r.toId, createdAt: r.createdAt }));
|
||||
const instances = new Map<string, number[]>();
|
||||
for (const i of intRows) if (i.enabled) instances.set(i.type, [...(instances.get(i.type) ?? []), i.id]);
|
||||
return { edges: buildEdges(resolveParents(parentRows), rows, (k) => names.has(k)), names, rows, instances, intRows };
|
||||
}
|
||||
|
||||
export interface Cover {
|
||||
/** Each node that is down because of another → the root cause. */
|
||||
cause: Map<string, string>;
|
||||
/** Each root cause → the names of what it takes down with it. */
|
||||
taken: Map<string, string[]>;
|
||||
graph: Graph;
|
||||
}
|
||||
|
||||
/** Works out what is down because of what, given the servers that are offline and the kinds of integration that are failing. */
|
||||
export async function dependencyCover(offlineServerIds: Iterable<number>, failingTypes: Iterable<string>): Promise<Cover> {
|
||||
const graph = await loadGraph();
|
||||
const down = new Set<string>();
|
||||
for (const id of offlineServerIds) down.add(nodeKey("server", id));
|
||||
const failing = new Set(failingTypes);
|
||||
for (const [type, ids] of graph.instances) if (failing.has(type)) for (const id of ids) down.add(nodeKey("integration", id));
|
||||
const cause = coveredNodes(down, graph.edges);
|
||||
const taken = new Map<string, string[]>();
|
||||
for (const [child, parent] of cause) taken.set(parent, [...(taken.get(parent) ?? []), graph.names.get(child) ?? child]);
|
||||
return { cause, taken, graph };
|
||||
}
|
||||
|
||||
/** The root cause behind a failing kind of integration, when every enabled instance of it is explained; null when it stands alone. */
|
||||
export function integrationTypeCause(type: string, cover: Cover): { key: string; name: string } | null {
|
||||
const ids = cover.graph.instances.get(type);
|
||||
if (!ids || ids.length === 0) return null;
|
||||
let first: string | null = null;
|
||||
for (const id of ids) {
|
||||
const root = cover.cause.get(nodeKey("integration", id));
|
||||
if (root === undefined) return null;
|
||||
first ??= root;
|
||||
}
|
||||
return first === null ? null : { key: first, name: cover.graph.names.get(first) ?? first };
|
||||
}
|
||||
|
||||
/** What a root cause takes down, for a kind of integration: the folded names across all its instances. */
|
||||
export function takenByType(type: string, cover: Cover): string[] {
|
||||
const out: string[] = [];
|
||||
for (const id of cover.graph.instances.get(type) ?? []) out.push(...(cover.taken.get(nodeKey("integration", id)) ?? []));
|
||||
return out;
|
||||
}
|
||||
|
||||
/** The servers the health monitor is announcing as offline (or holding back as explained), from what it last stored. */
|
||||
export async function offlineServerIdsFromState(): Promise<number[]> {
|
||||
try {
|
||||
const raw = await getInternalFlag(HEALTH_STATE_FLAG);
|
||||
if (!raw) return [];
|
||||
const ids: number[] = [];
|
||||
for (const key of Object.keys(JSON.parse(raw) as Record<string, unknown>)) {
|
||||
const m = /^offline:server:(\d+)$/.exec(key);
|
||||
if (m) ids.push(Number(m[1]));
|
||||
}
|
||||
return ids;
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether a kind of integration that has started failing is failing because of something it needs that is down — a server the health
|
||||
* monitor says is offline, or another service that is failing. Never throws: a problem here must not swallow a real outage.
|
||||
*/
|
||||
export async function integrationCausedBy(type: string, failingTypes: string[]): Promise<{ name: string } | null> {
|
||||
try {
|
||||
const cover = await dependencyCover(await offlineServerIdsFromState(), failingTypes);
|
||||
return integrationTypeCause(type, cover);
|
||||
} catch (err) {
|
||||
console.error("[dependencies] couldn't work out whether a failure is explained:", err);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export async function nodeExists(type: NodeType, id: number): Promise<boolean> {
|
||||
if (type === "server") return (await db.select({ id: servers.id }).from(servers).where(eq(servers.id, id)).limit(1)).length > 0;
|
||||
return (await db.select({ id: integrations.id }).from(integrations).where(eq(integrations.id, id)).limit(1)).length > 0;
|
||||
}
|
||||
@@ -8,11 +8,12 @@ import { notifyHealthIssues, notifyHealthRecovered } from "./notify.js";
|
||||
import { recordAudit } from "./audit.js";
|
||||
import { getInternalFlag, getSettings, setInternalFlag } from "./settingsStore.js";
|
||||
import { activeSubjects, subjectOfConditionKey } from "./maintenance.js";
|
||||
import { coveredByParent, loadParentRows, resolveParents } from "./serverParents.js";
|
||||
import { dependencyCover, HEALTH_STATE_FLAG } from "./dependencies.js";
|
||||
import { getFailingSources } from "./integrationHealthMonitor.js";
|
||||
import { hostHealthConditions, parseHostHealth, type HostHealth } from "./hostHealth.js";
|
||||
import { diskLimit, parseThresholds, type ServerThresholds } from "./serverThresholds.js";
|
||||
|
||||
const STATE_FLAG = "healthActiveConditions";
|
||||
const STATE_FLAG = HEALTH_STATE_FLAG;
|
||||
/** After a restart the agents haven't had a chance to report yet (they run every 15 min), so server-derived conditions are held rather than judged. */
|
||||
const STARTUP_GRACE_MS = Number(process.env.HEALTH_STARTUP_GRACE_MS ?? 20 * 60 * 1000); // the environment variable is for tests
|
||||
const PROCESS_START = Date.now();
|
||||
@@ -396,18 +397,20 @@ export async function runHealthCheck(now: number = Date.now()): Promise<{ added:
|
||||
const m = /^offline:server:(\d+)$/.exec(c.key);
|
||||
if (m) offlineIds.add(Number(m[1]));
|
||||
}
|
||||
const cause = coveredByParent(offlineIds, resolveParents(await loadParentRows()));
|
||||
const coveredKeys = new Set([...cause.keys()].map((id) => `offline:server:${id}`));
|
||||
// (Also when it needs a service that is failing, or runs on a server that needs one.)
|
||||
const cover = await dependencyCover(offlineIds, getFailingSources().map((f) => f.source));
|
||||
const coveredKeys = new Set<string>();
|
||||
for (const child of cover.cause.keys()) {
|
||||
const m = /^server:(\d+)$/.exec(child);
|
||||
if (m) coveredKeys.add(`offline:server:${m[1]}`);
|
||||
}
|
||||
const { added: announced, resolved, next, newlyDown, ended } = diffConditions(await loadState(), current, held, await activeSubjects(new Date(now)), coveredKeys);
|
||||
await setInternalFlag(STATE_FLAG, JSON.stringify(next));
|
||||
|
||||
// A host's message says what else it takes with it.
|
||||
const names = new Map(snapshot.servers.map((s) => [s.id, s.name]));
|
||||
const taken = new Map<number, string[]>();
|
||||
for (const [child, parent] of cause) taken.set(parent, [...(taken.get(parent) ?? []), names.get(child) ?? `#${child}`]);
|
||||
const added = announced.map((c) => {
|
||||
const m = /^offline:server:(\d+)$/.exec(c.key);
|
||||
const children = m ? taken.get(Number(m[1])) : undefined;
|
||||
const children = m ? cover.taken.get(`server:${m[1]}`) : undefined;
|
||||
return children?.length ? { ...c, message: `${c.message} — also takes down ${alsoList(children)}` } : c;
|
||||
});
|
||||
|
||||
|
||||
@@ -18,6 +18,7 @@ export interface SmartDisk {
|
||||
/** NVMe and some SSDs say how much of their rated life is used, 0–100 (and past it). */
|
||||
wearPercent: number | null;
|
||||
}
|
||||
/** A storage pool: ZFS on Linux, Storage Spaces on Windows (which reports Healthy, Warning and Unhealthy as ONLINE, DEGRADED and FAULTED). */
|
||||
export interface ZfsPool {
|
||||
name: string;
|
||||
/** ONLINE, DEGRADED, FAULTED, UNAVAIL, SUSPENDED … as zpool says it. */
|
||||
@@ -117,7 +118,7 @@ export function hostHealthConditions(serverId: number, serverName: string, healt
|
||||
}
|
||||
for (const p of health.zfs ?? []) {
|
||||
if (!poolIsFine(p)) {
|
||||
out.push({ key: `zfs:server:${serverId}:${p.name}`, severity: "critical", message: `${serverName}: ZFS pool ${p.name} is ${p.health}` });
|
||||
out.push({ key: `zfs:server:${serverId}:${p.name}`, severity: "critical", message: `${serverName}: storage pool ${p.name} is ${p.health}` });
|
||||
}
|
||||
}
|
||||
return out;
|
||||
|
||||
@@ -6,6 +6,7 @@ import { ensureHistoryStart } from "./uptimeHistory.js";
|
||||
import { getSettings } from "./settingsStore.js";
|
||||
import { isSourceInMaintenance } from "./maintenance.js";
|
||||
import { notifyIntegrationDown, notifyIntegrationRecovered } from "./notify.js";
|
||||
import { integrationCausedBy } from "./dependencies.js";
|
||||
|
||||
interface SourceHealth {
|
||||
consecutiveFailures: number;
|
||||
@@ -96,7 +97,9 @@ export async function trackIntegrationHealth(source: string, ok: boolean): Promi
|
||||
if (notifications.integrationFailureAlerts && !state.alerted && state.consecutiveFailures >= notifications.integrationFailureThreshold) {
|
||||
// During maintenance the failures keep being counted but `alerted` stays false, so if the service is
|
||||
// still failing once the window ends, the very next failed call alerts — a real outage isn't swallowed.
|
||||
if (!(await isSourceInMaintenance(source))) {
|
||||
// Likewise when it needs something that is down (the server it runs on, another service): that one is what gets announced, and if this stays
|
||||
// down once that one is back, the next failed call alerts.
|
||||
if (!(await isSourceInMaintenance(source)) && !(await integrationCausedBy(source, [...getFailingSources().map((f) => f.source), source]))) {
|
||||
state.alerted = true;
|
||||
await notifyIntegrationDown(source, state.consecutiveFailures);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
/**
|
||||
* Filling in the inventory from what an agent has found out about the machine it runs on: maker, model and serial number. Typing a serial
|
||||
* number off a label is the part of an inventory nobody keeps up, so a linked entry gets it from the agent — but only into fields that are
|
||||
* blank, because what a person wrote down is theirs; "Refresh from agent" is the way to overwrite on purpose.
|
||||
*/
|
||||
import { eq, isNull, or, and } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { assets, servers } from "../db/schema.js";
|
||||
|
||||
export interface AgentHardware {
|
||||
manufacturer: string | null;
|
||||
model: string | null;
|
||||
serial: string | null;
|
||||
}
|
||||
|
||||
type Fields = { manufacturer: string | null; model: string | null; serial: string | null };
|
||||
const KEYS = ["manufacturer", "model", "serial"] as const;
|
||||
|
||||
/** What the server's agent said the machine is, or null if it has said nothing. */
|
||||
export function hardwareOf(server: { hwManufacturer: string | null; hwModel: string | null; hwSerial: string | null } | null | undefined): AgentHardware | null {
|
||||
if (!server) return null;
|
||||
const hw = { manufacturer: server.hwManufacturer ?? null, model: server.hwModel ?? null, serial: server.hwSerial ?? null };
|
||||
return hw.manufacturer || hw.model || hw.serial ? hw : null;
|
||||
}
|
||||
|
||||
/** The fields of an entry that are blank and that the agent can fill. Empty when nothing would change. */
|
||||
export function blanksToFill(current: Fields, hw: AgentHardware | null): Partial<Fields> {
|
||||
const out: Partial<Fields> = {};
|
||||
if (!hw) return out;
|
||||
for (const k of KEYS) if (!current[k]?.trim() && hw[k]) out[k] = hw[k];
|
||||
return out;
|
||||
}
|
||||
|
||||
/** The fields where the agent says something different from the entry: what a refresh would change. */
|
||||
export function differences(current: Fields, hw: AgentHardware | null): Partial<Fields> {
|
||||
const out: Partial<Fields> = {};
|
||||
if (!hw) return out;
|
||||
for (const k of KEYS) if (hw[k] && (current[k] ?? "").trim() !== hw[k]) out[k] = hw[k];
|
||||
return out;
|
||||
}
|
||||
|
||||
/** After an agent has reported, fills the blanks of the inventory entries that are this server. */
|
||||
export async function fillInventoryFromServer(serverId: number): Promise<number> {
|
||||
const [server] = await db.select({ hwManufacturer: servers.hwManufacturer, hwModel: servers.hwModel, hwSerial: servers.hwSerial }).from(servers).where(eq(servers.id, serverId)).limit(1);
|
||||
const hw = hardwareOf(server);
|
||||
if (!hw) return 0;
|
||||
// Only entries with something blank are worth reading.
|
||||
const linked = await db
|
||||
.select()
|
||||
.from(assets)
|
||||
.where(and(eq(assets.serverId, serverId), or(isNull(assets.manufacturer), isNull(assets.model), isNull(assets.serial))));
|
||||
let changed = 0;
|
||||
for (const a of linked) {
|
||||
const fill = blanksToFill(a, hw);
|
||||
if (Object.keys(fill).length === 0) continue;
|
||||
await db.update(assets).set({ ...fill, updatedAt: new Date().toISOString() }).where(eq(assets.id, a.id));
|
||||
changed++;
|
||||
}
|
||||
return changed;
|
||||
}
|
||||
@@ -95,7 +95,7 @@ export async function buildMetrics(now: Date = new Date()): Promise<string> {
|
||||
if (d.wearPercent !== null) out.add("homelab_server_disk_wear_percent", "gauge", "How much of an SSD's rated life is used.", d.wearPercent, { server, device: d.device });
|
||||
if (d.temperatureC !== null) out.add("homelab_server_disk_temperature_celsius", "gauge", "A disk's temperature.", d.temperatureC, { server, device: d.device });
|
||||
}
|
||||
for (const p of health?.zfs ?? []) out.add("homelab_server_zfs_pool_online", "gauge", "1 if a ZFS pool is ONLINE, 0 in any other state.", poolIsFine(p) ? 1 : 0, { server, pool: p.name });
|
||||
for (const p of health?.zfs ?? []) out.add("homelab_server_zfs_pool_online", "gauge", "1 if a storage pool (ZFS, or Storage Spaces on Windows) is ONLINE, 0 in any other state.", poolIsFine(p) ? 1 : 0, { server, pool: p.name });
|
||||
out.add("homelab_server_agent_outdated", "gauge", "1 if the server's agent is older than the one this installation would install now.", isAgentOutdated(s, latest) ? 1 : 0, { server });
|
||||
}
|
||||
// Disks on a steady climb, from each server's own history: days until full at the present rate (left out where there is no steady trend).
|
||||
|
||||
@@ -2,6 +2,7 @@ import nodemailer from "nodemailer";
|
||||
import { getSettings, type NotificationEvents } from "./settingsStore.js";
|
||||
import { enqueueNotification, listQueuedNotifications, clearQueuedNotifications } from "./notificationQueue.js";
|
||||
import { emitEvent } from "./eventWebhooks.js";
|
||||
import { sendPush, subscriptionCount } from "./webPush.js";
|
||||
|
||||
// ─── Channel senders (best-effort — failures are logged, never thrown) ──────
|
||||
|
||||
@@ -72,8 +73,18 @@ async function sendWebhook(title: string, message: string) {
|
||||
}
|
||||
}
|
||||
|
||||
async function sendWebPush(title: string, message: string) {
|
||||
const { webPush } = await getSettings();
|
||||
if (!webPush.enabled) return;
|
||||
try {
|
||||
await sendPush({ title, body: message });
|
||||
} catch (err) {
|
||||
console.error("[notify] Web push failed:", err instanceof Error ? err.message : err);
|
||||
}
|
||||
}
|
||||
|
||||
async function sendToChannels(title: string, message: string): Promise<void> {
|
||||
await Promise.all([sendGotify(title, message), sendNtfy(title, message), sendSmtp(title, message), sendWebhook(title, message)]);
|
||||
await Promise.all([sendGotify(title, message), sendNtfy(title, message), sendSmtp(title, message), sendWebhook(title, message), sendWebPush(title, message)]);
|
||||
}
|
||||
|
||||
function currentTimeInZone(timezone: string): { hh: number; mm: number } {
|
||||
@@ -112,8 +123,9 @@ export async function notifyNow(title: string, message: string): Promise<void> {
|
||||
|
||||
/** How many channels are switched on and filled in enough to send, so a "send it now" can say when there is nowhere to send to. */
|
||||
export async function enabledChannelCount(): Promise<number> {
|
||||
const { gotify, ntfy, smtp, webhook } = await getSettings();
|
||||
const { gotify, ntfy, smtp, webhook, webPush } = await getSettings();
|
||||
return [
|
||||
webPush.enabled && (await subscriptionCount()) > 0,
|
||||
gotify.enabled && gotify.url && gotify.token,
|
||||
ntfy.enabled && ntfy.url && ntfy.topic,
|
||||
smtp.enabled && smtp.host && smtp.to && smtp.from,
|
||||
|
||||
@@ -37,6 +37,13 @@ export interface WebhookSettings {
|
||||
secret: string;
|
||||
}
|
||||
|
||||
/** Push notifications to phones and browsers that have subscribed (see services/webPush.ts). Who is subscribed is kept apart, per device. */
|
||||
export interface WebPushSettings {
|
||||
enabled: boolean;
|
||||
/** Where a push service can reach whoever runs this: an email address or an https address (VAPID asks for one). Blank is fine. */
|
||||
contact: string;
|
||||
}
|
||||
|
||||
export interface NotificationEvents {
|
||||
dnsAdd: boolean;
|
||||
dnsUpdate: boolean;
|
||||
@@ -189,6 +196,7 @@ export interface AppSettings {
|
||||
ntfy: NtfySettings;
|
||||
smtp: SmtpSettings;
|
||||
webhook: WebhookSettings;
|
||||
webPush: WebPushSettings;
|
||||
notifications: NotificationEvents;
|
||||
providerColors: ProviderColors;
|
||||
integrationColors: IntegrationColors;
|
||||
@@ -211,6 +219,7 @@ const DEFAULTS: AppSettings = {
|
||||
ntfy: { enabled: false, url: "https://ntfy.sh", topic: "", token: "", priority: 3 },
|
||||
smtp: { enabled: false, host: "", port: 587, secure: false, username: "", password: "", from: "", to: "" },
|
||||
webhook: { enabled: false, url: "", secret: "" },
|
||||
webPush: { enabled: false, contact: "" },
|
||||
notifications: {
|
||||
dnsAdd: true,
|
||||
dnsUpdate: true,
|
||||
|
||||
@@ -5,6 +5,7 @@ import { recordServerMetrics } from "./serverMetrics.js";
|
||||
import { nextUpdateState, type IncomingUpdates } from "./pendingUpdates.js";
|
||||
import { nextHostHealth, type IncomingHostHealth } from "./hostHealth.js";
|
||||
import { recordAudit } from "./audit.js";
|
||||
import { fillInventoryFromServer } from "./inventoryAutofill.js";
|
||||
|
||||
export interface IncomingTask {
|
||||
scheduleType: "cron" | "systemd_timer" | "windows_task";
|
||||
@@ -27,6 +28,8 @@ export interface IncomingSystemInfo {
|
||||
updates?: IncomingUpdates;
|
||||
/** Failed services, disk health, ZFS pools — each part present only if the agent could find it out. */
|
||||
health?: IncomingHostHealth;
|
||||
/** What the machine is, from its firmware: each part present only if it says anything. */
|
||||
hardware?: { manufacturer?: string; model?: string; serial?: string };
|
||||
}
|
||||
|
||||
export interface AgentReport {
|
||||
@@ -115,6 +118,9 @@ export async function syncServerTasks(serverId: number, report: AgentReport) {
|
||||
// Only stored when the agent reported it, so an agent that predates this (or couldn't tell this time) doesn't wipe what it said before.
|
||||
...(system?.updates && before ? nextUpdateState(before, system.updates, now) : {}),
|
||||
...(system?.health && before ? { hostHealth: nextHostHealth(before.hostHealth, system.health, now) } : {}),
|
||||
...(system?.hardware?.manufacturer ? { hwManufacturer: system.hardware.manufacturer } : {}),
|
||||
...(system?.hardware?.model ? { hwModel: system.hardware.model } : {}),
|
||||
...(system?.hardware?.serial ? { hwSerial: system.hardware.serial } : {}),
|
||||
...(report.hostname ? { hostname: report.hostname } : {}),
|
||||
...(report.agentVersion !== undefined ? { agentVersion: report.agentVersion } : {}),
|
||||
// An agent knows what it runs on better than whoever registered the server did.
|
||||
@@ -146,6 +152,15 @@ export async function syncServerTasks(serverId: number, report: AgentReport) {
|
||||
}
|
||||
}
|
||||
|
||||
// A machine whose agent has said what it is fills in the blanks of its inventory entry. Also best-effort.
|
||||
if (system?.hardware) {
|
||||
try {
|
||||
await fillInventoryFromServer(serverId);
|
||||
} catch (err) {
|
||||
console.error("[inventory] couldn't fill in from the agent:", err);
|
||||
}
|
||||
}
|
||||
|
||||
// Keep a little history of the figures too. A failure here must not lose the report itself, which is already saved.
|
||||
try {
|
||||
await recordServerMetrics(serverId, system, new Date(now));
|
||||
|
||||
@@ -0,0 +1,245 @@
|
||||
/**
|
||||
* Web Push: notifications to a phone or a browser that has asked for them, with no app and no account — the browser's own push service
|
||||
* (Google's, Mozilla's, Apple's) carries the message. Written against the standards directly rather than through a library:
|
||||
*
|
||||
* - RFC 8292 (VAPID): the server proves who it is with a short-lived signed token (ES256), so a push service knows the sender is the
|
||||
* one the subscription was made for.
|
||||
* - RFC 8291 + 8188 (aes128gcm): the message is encrypted for the one browser it is for; the push service only ever carries it.
|
||||
*
|
||||
* The server's key pair is made the first time it is needed and kept (the private half encrypted, like the other credentials).
|
||||
*/
|
||||
import { createCipheriv, createECDH, createPrivateKey, hkdfSync, randomBytes, sign, type KeyObject } from "node:crypto";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { pushSubscriptions } from "../db/schema.js";
|
||||
import { env } from "../env.js";
|
||||
import { decryptSecret, encryptSecret } from "../crypto.js";
|
||||
import { resolveTarget } from "../utils/networkGuard.js";
|
||||
import { getInternalFlag, getSettings, setInternalFlag } from "./settingsStore.js";
|
||||
|
||||
const b64u = (buf: Buffer | Uint8Array) => Buffer.from(buf).toString("base64url");
|
||||
const fromB64u = (s: string) => Buffer.from(s, "base64url");
|
||||
|
||||
// ─── The server's own key pair ─────────────────────────────────────────────
|
||||
|
||||
const VAPID_FLAG = "webPushVapid";
|
||||
const ENCRYPTED = "enc:v1:";
|
||||
|
||||
export interface VapidKeys {
|
||||
/** The uncompressed P-256 point, base64url — what a browser is given to subscribe with. */
|
||||
publicKey: string;
|
||||
/** The private scalar, base64url. */
|
||||
privateKey: string;
|
||||
}
|
||||
|
||||
let cachedKeys: Promise<VapidKeys> | null = null;
|
||||
|
||||
/** The key pair, made on first use and kept. */
|
||||
export function getVapidKeys(): Promise<VapidKeys> {
|
||||
cachedKeys ??= loadOrMakeKeys().catch((err) => {
|
||||
cachedKeys = null;
|
||||
throw err;
|
||||
});
|
||||
return cachedKeys;
|
||||
}
|
||||
|
||||
async function loadOrMakeKeys(): Promise<VapidKeys> {
|
||||
const stored = await getInternalFlag(VAPID_FLAG);
|
||||
if (stored) {
|
||||
try {
|
||||
const v = JSON.parse(stored) as { publicKey: string; privateKey: string };
|
||||
const privateKey = v.privateKey.startsWith(ENCRYPTED) ? decryptSecret(v.privateKey.slice(ENCRYPTED.length)) : v.privateKey;
|
||||
return { publicKey: v.publicKey, privateKey };
|
||||
} catch {
|
||||
// Unreadable (the encryption key changed): made again below — every browser must then subscribe again, which the page says.
|
||||
}
|
||||
}
|
||||
const ecdh = createECDH("prime256v1");
|
||||
ecdh.generateKeys();
|
||||
const keys: VapidKeys = { publicKey: b64u(ecdh.getPublicKey()), privateKey: b64u(ecdh.getPrivateKey().length === 32 ? ecdh.getPrivateKey() : Buffer.concat([Buffer.alloc(32 - ecdh.getPrivateKey().length), ecdh.getPrivateKey()])) };
|
||||
await setInternalFlag(VAPID_FLAG, JSON.stringify({ publicKey: keys.publicKey, privateKey: env.credentialsEncryptionEnabled ? ENCRYPTED + encryptSecret(keys.privateKey) : keys.privateKey }));
|
||||
// The subscriptions made for the old key can't be used with the new one.
|
||||
await db.delete(pushSubscriptions);
|
||||
return keys;
|
||||
}
|
||||
|
||||
/** For the tests, so one test's keys don't leak into the next. */
|
||||
export function forgetVapidKeysForTests(): void {
|
||||
cachedKeys = null;
|
||||
}
|
||||
|
||||
/** A P-256 private key as a key object, from its raw scalar and public point. */
|
||||
function privateKeyObject(keys: VapidKeys): KeyObject {
|
||||
const pub = fromB64u(keys.publicKey);
|
||||
return createPrivateKey({
|
||||
key: { kty: "EC", crv: "P-256", d: keys.privateKey, x: b64u(pub.subarray(1, 33)), y: b64u(pub.subarray(33, 65)) },
|
||||
format: "jwk",
|
||||
});
|
||||
}
|
||||
|
||||
/** The Authorization header's token: who we are, who it is for, and for how long (RFC 8292). */
|
||||
export function vapidAuthorization(audience: string, subject: string, keys: VapidKeys, now: number = Date.now()): string {
|
||||
const header = b64u(Buffer.from(JSON.stringify({ typ: "JWT", alg: "ES256" })));
|
||||
const claims = b64u(Buffer.from(JSON.stringify({ aud: audience, exp: Math.floor(now / 1000) + 12 * 3600, sub: subject })));
|
||||
const signature = sign("sha256", Buffer.from(`${header}.${claims}`), { key: privateKeyObject(keys), dsaEncoding: "ieee-p1363" });
|
||||
return `vapid t=${header}.${claims}.${b64u(signature)}, k=${keys.publicKey}`;
|
||||
}
|
||||
|
||||
// ─── Encrypting a message for one browser (RFC 8291) ───────────────────────
|
||||
|
||||
export interface PushKeys {
|
||||
/** The browser's public key, base64url. */
|
||||
p256dh: string;
|
||||
/** The browser's 16-byte authentication secret, base64url. */
|
||||
auth: string;
|
||||
}
|
||||
|
||||
const RECORD_SIZE = 4096;
|
||||
|
||||
/**
|
||||
* The body to send: the message encrypted so that only the browser it is for can read it. `ephemeral` and `salt` are chosen fresh each
|
||||
* time; they are parameters only so the standard's own worked example can be checked.
|
||||
*/
|
||||
export function encryptPayload(payload: Buffer, subscription: PushKeys, fixed?: { ephemeral: ECDHLike; salt: Buffer }): Buffer {
|
||||
const uaPublic = fromB64u(subscription.p256dh);
|
||||
const authSecret = fromB64u(subscription.auth);
|
||||
if (uaPublic.length !== 65 || uaPublic[0] !== 4) throw new Error("The browser's public key isn't a P-256 point.");
|
||||
if (authSecret.length !== 16) throw new Error("The browser's authentication secret isn't 16 bytes.");
|
||||
|
||||
const fresh = fixed ? null : createECDH("prime256v1");
|
||||
fresh?.generateKeys();
|
||||
const ecdh: ECDHLike = fixed?.ephemeral ?? fresh!;
|
||||
const asPublic = ecdh.getPublicKey();
|
||||
const salt = fixed?.salt ?? randomBytes(16);
|
||||
const secret = ecdh.computeSecret(uaPublic);
|
||||
|
||||
const keyInfo = Buffer.concat([Buffer.from("WebPush: info\0"), uaPublic, asPublic]);
|
||||
const ikm = Buffer.from(hkdfSync("sha256", secret, authSecret, keyInfo, 32));
|
||||
const cek = Buffer.from(hkdfSync("sha256", ikm, salt, Buffer.from("Content-Encoding: aes128gcm\0"), 16));
|
||||
const nonce = Buffer.from(hkdfSync("sha256", ikm, salt, Buffer.from("Content-Encoding: nonce\0"), 12));
|
||||
|
||||
// One record: the message, then a 0x02 marking it as the last.
|
||||
if (payload.length > RECORD_SIZE - 17 - 1) throw new Error("The message is too long for one push.");
|
||||
const cipher = createCipheriv("aes-128-gcm", cek, nonce);
|
||||
const encrypted = Buffer.concat([cipher.update(Buffer.concat([payload, Buffer.from([2])])), cipher.final(), cipher.getAuthTag()]);
|
||||
|
||||
const header = Buffer.alloc(16 + 4 + 1);
|
||||
salt.copy(header, 0);
|
||||
header.writeUInt32BE(RECORD_SIZE, 16);
|
||||
header.writeUInt8(asPublic.length, 20);
|
||||
return Buffer.concat([header, asPublic, encrypted]);
|
||||
}
|
||||
|
||||
type ECDHLike = Pick<ReturnType<typeof createECDH>, "getPublicKey" | "computeSecret">;
|
||||
|
||||
// ─── Subscriptions ─────────────────────────────────────────────────────────
|
||||
|
||||
let allowLocalEndpoints = false;
|
||||
/** Lets the tests push to a service on this machine, which is otherwise refused. */
|
||||
export function allowLocalPushEndpointsForTests(allow: boolean): void {
|
||||
allowLocalEndpoints = allow;
|
||||
}
|
||||
|
||||
/** A push service is a public internet service with an https address; anything else is refused before we ever connect to it. */
|
||||
export async function checkEndpoint(endpoint: string): Promise<void> {
|
||||
let url: URL;
|
||||
try {
|
||||
url = new URL(endpoint);
|
||||
} catch {
|
||||
throw new Error("The push address isn't a web address.");
|
||||
}
|
||||
if (url.protocol !== "https:" && !(allowLocalEndpoints && url.protocol === "http:")) throw new Error("The push address must be https.");
|
||||
if (url.username || url.password) throw new Error("The push address can't carry a login.");
|
||||
if (allowLocalEndpoints) return;
|
||||
await resolveTarget(url.hostname, { allowPrivate: false });
|
||||
}
|
||||
|
||||
export interface PushOutcome {
|
||||
sent: number;
|
||||
/** Subscriptions the push service said are gone, now removed. */
|
||||
removed: number;
|
||||
failed: number;
|
||||
}
|
||||
|
||||
/** The contact the push services may reach us on (a `mailto:` or an https address), as VAPID asks. */
|
||||
export async function vapidSubject(): Promise<string> {
|
||||
const { webPush } = await getSettings();
|
||||
const contact = webPush.contact.trim();
|
||||
if (/^(mailto:|https:\/\/)/i.test(contact)) return contact;
|
||||
if (contact.includes("@")) return `mailto:${contact}`;
|
||||
return /^https:\/\//.test(env.appBaseUrl) ? env.appBaseUrl : "mailto:homelab@example.invalid";
|
||||
}
|
||||
|
||||
export interface PushMessage {
|
||||
title: string;
|
||||
body: string;
|
||||
/** Where in the app a tap should go. */
|
||||
url?: string;
|
||||
/** A later message with the same tag replaces an earlier one that is still showing. */
|
||||
tag?: string;
|
||||
}
|
||||
|
||||
/** Sends one message to one subscription. Resolves to what the push service said; never throws. */
|
||||
async function deliver(sub: { endpoint: string; p256dh: string; auth: string }, body: Buffer, keys: VapidKeys, subject: string, urgent: boolean): Promise<"sent" | "gone" | "failed"> {
|
||||
try {
|
||||
await checkEndpoint(sub.endpoint);
|
||||
const res = await fetch(sub.endpoint, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
Authorization: vapidAuthorization(new URL(sub.endpoint).origin, subject, keys),
|
||||
"Content-Encoding": "aes128gcm",
|
||||
"Content-Type": "application/octet-stream",
|
||||
TTL: "86400",
|
||||
Urgency: urgent ? "high" : "normal",
|
||||
},
|
||||
body: new Uint8Array(body),
|
||||
signal: AbortSignal.timeout(15_000),
|
||||
});
|
||||
if (res.ok) return "sent";
|
||||
if (res.status === 404 || res.status === 410) return "gone";
|
||||
console.error(`[push] ${new URL(sub.endpoint).host} answered ${res.status}: ${(await res.text().catch(() => "")).slice(0, 200)}`);
|
||||
return "failed";
|
||||
} catch (err) {
|
||||
console.error("[push] couldn't deliver:", err instanceof Error ? err.message : err);
|
||||
return "failed";
|
||||
}
|
||||
}
|
||||
|
||||
/** Sends to the given subscriptions (all of them, if none are given). Subscriptions the push service says are gone are removed. */
|
||||
export async function sendPush(message: PushMessage, only?: { id: number; endpoint: string; p256dh: string; auth: string }[]): Promise<PushOutcome> {
|
||||
const subs = only ?? (await db.select().from(pushSubscriptions));
|
||||
const outcome: PushOutcome = { sent: 0, removed: 0, failed: 0 };
|
||||
if (subs.length === 0) return outcome;
|
||||
const keys = await getVapidKeys();
|
||||
const subject = await vapidSubject();
|
||||
const payload = Buffer.from(JSON.stringify({ title: message.title.slice(0, 120), body: message.body.slice(0, 600), url: message.url ?? "/alerts", tag: message.tag }));
|
||||
await Promise.all(
|
||||
subs.map(async (s) => {
|
||||
let body: Buffer;
|
||||
try {
|
||||
body = encryptPayload(payload, { p256dh: s.p256dh, auth: s.auth });
|
||||
} catch (err) {
|
||||
console.error("[push] a subscription can't be written to:", err instanceof Error ? err.message : err);
|
||||
outcome.failed++;
|
||||
return;
|
||||
}
|
||||
const result = await deliver(s, body, keys, subject, true);
|
||||
if (result === "sent") {
|
||||
outcome.sent++;
|
||||
await db.update(pushSubscriptions).set({ lastSuccessAt: new Date().toISOString() }).where(eq(pushSubscriptions.id, s.id));
|
||||
} else if (result === "gone") {
|
||||
outcome.removed++;
|
||||
await db.delete(pushSubscriptions).where(eq(pushSubscriptions.id, s.id));
|
||||
} else {
|
||||
outcome.failed++;
|
||||
}
|
||||
}),
|
||||
);
|
||||
return outcome;
|
||||
}
|
||||
|
||||
/** How many browsers are subscribed. */
|
||||
export async function subscriptionCount(): Promise<number> {
|
||||
return (await db.select({ id: pushSubscriptions.id }).from(pushSubscriptions)).length;
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user