Dependencies beyond servers, web push, inventory auto-fill with QR labels, Windows agent health
- Dependencies: servers and integrations can depend on each other; alerts are folded under the root cause, and a failure explained by something down isn't announced - Web push: hand-written VAPID and RFC 8291 encryption, per-device subscriptions, a fifth notification channel, service worker handlers - Inventory: maker/model/serial from the agents fill blank fields, "Refresh from agent", QR label printing and ?asset= deep links - Windows agent (v4) reports failed services, disk health, Storage Spaces pools and hardware identity; Linux agent (v5) reports hardware identity Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
5cc51c57de
commit
ec9c2f73f0
55 files changed
+12553
-65
No files matched your search
@@ -8,7 +8,9 @@
|
||||
* 2 — reports pending updates, and its own version
|
||||
* 3 — Linux: the package manager is asked outside the systemd sandbox, so dnf and zypper can answer too.
|
||||
* Windows: can replace itself when an administrator has switched automatic updates on.
|
||||
* 4 — Linux: also reports failed services, disk health (SMART) and ZFS pools, reads pacman, and can replace itself too
|
||||
* 4 — Linux: also reports failed services, disk health (SMART) and ZFS pools, reads pacman, and can replace itself too.
|
||||
* Windows: also reports failed services, disk health and Storage Spaces pools, and what the machine is (maker, model, serial)
|
||||
* 5 — Linux: also says what the machine is (maker, model, serial), for the inventory
|
||||
*/
|
||||
import { createHash } from "node:crypto";
|
||||
import { readFileSync } from "node:fs";
|
||||
|
||||
@@ -33,7 +33,7 @@ import { computeSecretStatus } from "./secretStatus.js";
|
||||
import { getFailingSources } from "./integrationHealthMonitor.js";
|
||||
import { getSettings } from "./settingsStore.js";
|
||||
import { sourceLabel } from "./notify.js";
|
||||
import { coveredByParent, loadParentRows, resolveParents } from "./serverParents.js";
|
||||
import { dependencyCover, integrationTypeCause, takenByType } from "./dependencies.js";
|
||||
import type { Alert, AlertCategory, AlertSeverity, SourceFailure } from "./alertTypes.js";
|
||||
|
||||
export interface AlertsReport {
|
||||
@@ -158,6 +158,10 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
}
|
||||
}
|
||||
|
||||
// The servers found to be offline, for the integration failures that one may explain: the server check fills it in (and it is empty if that check fails).
|
||||
let offlineKnown!: (ids: Set<number>) => void;
|
||||
const offlineServers = new Promise<Set<number>>((resolve) => (offlineKnown = resolve));
|
||||
|
||||
await Promise.all([
|
||||
check("Server and storage health", async () => {
|
||||
const { healthChecks } = await getSettings();
|
||||
@@ -176,17 +180,17 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
const m = /^offline:server:(\d+)$/.exec(c.key);
|
||||
if (m) offlineIds.add(Number(m[1]));
|
||||
}
|
||||
const cause = coveredByParent(offlineIds, resolveParents(await loadParentRows()));
|
||||
const taken = new Map<number, string[]>();
|
||||
for (const [child, parent] of cause) taken.set(parent, [...(taken.get(parent) ?? []), serverNames.get(child) ?? `#${child}`]);
|
||||
// (Or because of a service it needs that is failing, or a server it needs that needs one.)
|
||||
offlineKnown(offlineIds);
|
||||
const cover = await dependencyCover(offlineIds, getFailingSources().map((f) => f.source));
|
||||
for (const c of conditions) {
|
||||
const origin = originOfConditionKey(c.key);
|
||||
const subject = subjectOfConditionKey(c.key);
|
||||
const offlineId = /^offline:server:(\d+)$/.exec(c.key)?.[1];
|
||||
const parentOf = offlineId ? cause.get(Number(offlineId)) : undefined;
|
||||
const parentOf = offlineId ? cover.cause.get(`server:${offlineId}`) : undefined;
|
||||
push({
|
||||
causedBy: parentOf !== undefined ? { name: serverNames.get(parentOf) ?? `#${parentOf}` } : undefined,
|
||||
affects: offlineId ? taken.get(Number(offlineId))?.sort((a, b) => a.localeCompare(b)) : undefined,
|
||||
causedBy: parentOf !== undefined ? { name: cover.graph.names.get(parentOf) ?? parentOf } : undefined,
|
||||
affects: offlineId ? cover.taken.get(`server:${offlineId}`)?.sort((a, b) => a.localeCompare(b)) : undefined,
|
||||
category: origin.category,
|
||||
severity: c.severity ?? "warning",
|
||||
type: origin.type,
|
||||
@@ -203,7 +207,7 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
const row = intById.get(Number(m[2]));
|
||||
cannotCheck(`${SOURCES[m[1]].label} “${row?.name ?? `#${m[2]}`}”${m[3] ? ` (node ${m[3]})` : ""}`, "couldn't be read — see the Diagnostic Log", "Server and storage health");
|
||||
}
|
||||
}),
|
||||
}).finally(() => offlineKnown(new Set())),
|
||||
|
||||
check("Automation runs", async () => {
|
||||
const { items, held } = await collectAutomation();
|
||||
@@ -447,8 +451,15 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
|
||||
check("Integration failures", async () => {
|
||||
const { notifications } = await getSettings();
|
||||
for (const f of getFailingSources()) {
|
||||
const failing = getFailingSources();
|
||||
// A failure explained by something it needs that is down is folded under that, like a guest under its host.
|
||||
const cover = failing.length > 0 ? await dependencyCover(await offlineServers, failing.map((f) => f.source)).catch(() => null) : null;
|
||||
for (const f of failing) {
|
||||
const cause = cover ? integrationTypeCause(f.source, cover) : null;
|
||||
const affects = cover ? takenByType(f.source, cover) : [];
|
||||
push({
|
||||
causedBy: cause ? { name: cause.name } : undefined,
|
||||
affects: affects.length > 0 ? affects.sort((a, b) => a.localeCompare(b)) : undefined,
|
||||
category: "integration",
|
||||
severity: f.alerted ? "critical" : "warning",
|
||||
type: f.source,
|
||||
|
||||
@@ -0,0 +1,192 @@
|
||||
/**
|
||||
* What depends on what, beyond "this server runs on that server": a service that needs a server (Gitea on the box it is installed on), a
|
||||
* server that needs a service (a guest that only exists while Proxmox answers). When the thing depended on is down, the things that need it
|
||||
* are down too — and only the one that matters is worth an alert, so the rest are folded under it: shown, not announced.
|
||||
*
|
||||
* The graph is the "runs on" links of servers (set by hand or worked out from Proxmox, see serverParents.ts) plus the dependencies written
|
||||
* here. Servers and integrations are the things that can be down and be needed. (DNS providers raise no alerts of their own, so there is
|
||||
* nothing to fold.) An integration is judged per kind of service, as its health is: it is folded only if every enabled instance of that
|
||||
* kind is explained.
|
||||
*/
|
||||
import { eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { dependencies, integrations, servers } from "../db/schema.js";
|
||||
import { getInternalFlag } from "./settingsStore.js";
|
||||
import { loadParentRows, resolveParents, type ParentLink } from "./serverParents.js";
|
||||
|
||||
/** Where the health monitor keeps the problems it is announcing; read here so a service's alert can tell a server it needs is down. */
|
||||
export const HEALTH_STATE_FLAG = "healthActiveConditions";
|
||||
|
||||
export const nodeTypes = ["server", "integration"] as const;
|
||||
export type NodeType = (typeof nodeTypes)[number];
|
||||
|
||||
export const nodeKey = (type: NodeType, id: number) => `${type}:${id}`;
|
||||
|
||||
export interface DependencyEdge {
|
||||
fromType: NodeType;
|
||||
fromId: number;
|
||||
toType: NodeType;
|
||||
toId: number;
|
||||
}
|
||||
|
||||
/** node → the nodes it needs. */
|
||||
export type Edges = Map<string, string[]>;
|
||||
|
||||
/** The graph from the "runs on" links and the written dependencies; an edge to or from something that no longer exists is left out. */
|
||||
export function buildEdges(parentLinks: Map<number, ParentLink>, rows: DependencyEdge[], exists: (key: string) => boolean): Edges {
|
||||
const edges: Edges = new Map();
|
||||
const add = (from: string, to: string) => {
|
||||
if (from === to || !exists(from) || !exists(to)) return;
|
||||
const list = edges.get(from) ?? [];
|
||||
if (!list.includes(to)) edges.set(from, [...list, to]);
|
||||
};
|
||||
for (const [child, link] of parentLinks) add(nodeKey("server", child), nodeKey("server", link.parentId));
|
||||
for (const r of rows) add(nodeKey(r.fromType, r.fromId), nodeKey(r.toType, r.toId));
|
||||
return edges;
|
||||
}
|
||||
|
||||
/** Everything `node` needs, directly or not, nearest first; `looped` if it ends up needing itself. */
|
||||
function ancestorsOf(node: string, edges: Edges): { order: string[]; looped: boolean } {
|
||||
const order: string[] = [];
|
||||
const seen = new Set<string>();
|
||||
const queue = [...(edges.get(node) ?? [])];
|
||||
let looped = false;
|
||||
while (queue.length > 0) {
|
||||
const cur = queue.shift()!;
|
||||
if (cur === node) {
|
||||
looped = true;
|
||||
continue;
|
||||
}
|
||||
if (seen.has(cur)) continue;
|
||||
seen.add(cur);
|
||||
order.push(cur);
|
||||
queue.push(...(edges.get(cur) ?? []));
|
||||
}
|
||||
return { order, looped };
|
||||
}
|
||||
|
||||
/**
|
||||
* Of the nodes that are down, those whose cause is something they need that is down too: each maps to the root cause — a down node that
|
||||
* needs nothing that is down. A loop explains nothing (each would be waiting for the other), so its members stand alone and are announced.
|
||||
*/
|
||||
export function coveredNodes(down: Set<string>, edges: Edges): Map<string, string> {
|
||||
const covered = new Map<string, string>();
|
||||
for (const node of [...down].sort()) {
|
||||
const { order, looped } = ancestorsOf(node, edges);
|
||||
if (looped) continue;
|
||||
const downAbove = order.filter((a) => down.has(a));
|
||||
const root = downAbove.find((a) => {
|
||||
const above = ancestorsOf(a, edges);
|
||||
return !above.looped && !above.order.some((x) => down.has(x));
|
||||
});
|
||||
if (root !== undefined) covered.set(node, root);
|
||||
}
|
||||
return covered;
|
||||
}
|
||||
|
||||
/** Would `from` needing `to` close a loop? True if `to` is `from` or already needs it, however indirectly. */
|
||||
export function wouldLoop(edges: Edges, from: string, to: string): boolean {
|
||||
return from === to || ancestorsOf(to, edges).order.includes(from) || ancestorsOf(to, edges).looped;
|
||||
}
|
||||
|
||||
export interface Graph {
|
||||
edges: Edges;
|
||||
/** node → its name, for every node that exists. */
|
||||
names: Map<string, string>;
|
||||
rows: (DependencyEdge & { id: number; createdAt: string })[];
|
||||
/** Enabled instances of each kind of integration. */
|
||||
instances: Map<string, number[]>;
|
||||
intRows: { id: number; type: string; enabled: boolean }[];
|
||||
}
|
||||
|
||||
export async function loadGraph(): Promise<Graph> {
|
||||
const [serverRows, intRows, depRows, parentRows] = await Promise.all([
|
||||
db.select({ id: servers.id, name: servers.name }).from(servers),
|
||||
db.select({ id: integrations.id, name: integrations.name, type: integrations.type, enabled: integrations.enabled }).from(integrations),
|
||||
db.select().from(dependencies),
|
||||
loadParentRows(),
|
||||
]);
|
||||
const names = new Map<string, string>();
|
||||
for (const s of serverRows) names.set(nodeKey("server", s.id), s.name);
|
||||
for (const i of intRows) names.set(nodeKey("integration", i.id), i.name);
|
||||
const rows = depRows.map((r) => ({ id: r.id, fromType: r.fromType as NodeType, fromId: r.fromId, toType: r.toType as NodeType, toId: r.toId, createdAt: r.createdAt }));
|
||||
const instances = new Map<string, number[]>();
|
||||
for (const i of intRows) if (i.enabled) instances.set(i.type, [...(instances.get(i.type) ?? []), i.id]);
|
||||
return { edges: buildEdges(resolveParents(parentRows), rows, (k) => names.has(k)), names, rows, instances, intRows };
|
||||
}
|
||||
|
||||
export interface Cover {
|
||||
/** Each node that is down because of another → the root cause. */
|
||||
cause: Map<string, string>;
|
||||
/** Each root cause → the names of what it takes down with it. */
|
||||
taken: Map<string, string[]>;
|
||||
graph: Graph;
|
||||
}
|
||||
|
||||
/** Works out what is down because of what, given the servers that are offline and the kinds of integration that are failing. */
|
||||
export async function dependencyCover(offlineServerIds: Iterable<number>, failingTypes: Iterable<string>): Promise<Cover> {
|
||||
const graph = await loadGraph();
|
||||
const down = new Set<string>();
|
||||
for (const id of offlineServerIds) down.add(nodeKey("server", id));
|
||||
const failing = new Set(failingTypes);
|
||||
for (const [type, ids] of graph.instances) if (failing.has(type)) for (const id of ids) down.add(nodeKey("integration", id));
|
||||
const cause = coveredNodes(down, graph.edges);
|
||||
const taken = new Map<string, string[]>();
|
||||
for (const [child, parent] of cause) taken.set(parent, [...(taken.get(parent) ?? []), graph.names.get(child) ?? child]);
|
||||
return { cause, taken, graph };
|
||||
}
|
||||
|
||||
/** The root cause behind a failing kind of integration, when every enabled instance of it is explained; null when it stands alone. */
|
||||
export function integrationTypeCause(type: string, cover: Cover): { key: string; name: string } | null {
|
||||
const ids = cover.graph.instances.get(type);
|
||||
if (!ids || ids.length === 0) return null;
|
||||
let first: string | null = null;
|
||||
for (const id of ids) {
|
||||
const root = cover.cause.get(nodeKey("integration", id));
|
||||
if (root === undefined) return null;
|
||||
first ??= root;
|
||||
}
|
||||
return first === null ? null : { key: first, name: cover.graph.names.get(first) ?? first };
|
||||
}
|
||||
|
||||
/** What a root cause takes down, for a kind of integration: the folded names across all its instances. */
|
||||
export function takenByType(type: string, cover: Cover): string[] {
|
||||
const out: string[] = [];
|
||||
for (const id of cover.graph.instances.get(type) ?? []) out.push(...(cover.taken.get(nodeKey("integration", id)) ?? []));
|
||||
return out;
|
||||
}
|
||||
|
||||
/** The servers the health monitor is announcing as offline (or holding back as explained), from what it last stored. */
|
||||
export async function offlineServerIdsFromState(): Promise<number[]> {
|
||||
try {
|
||||
const raw = await getInternalFlag(HEALTH_STATE_FLAG);
|
||||
if (!raw) return [];
|
||||
const ids: number[] = [];
|
||||
for (const key of Object.keys(JSON.parse(raw) as Record<string, unknown>)) {
|
||||
const m = /^offline:server:(\d+)$/.exec(key);
|
||||
if (m) ids.push(Number(m[1]));
|
||||
}
|
||||
return ids;
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether a kind of integration that has started failing is failing because of something it needs that is down — a server the health
|
||||
* monitor says is offline, or another service that is failing. Never throws: a problem here must not swallow a real outage.
|
||||
*/
|
||||
export async function integrationCausedBy(type: string, failingTypes: string[]): Promise<{ name: string } | null> {
|
||||
try {
|
||||
const cover = await dependencyCover(await offlineServerIdsFromState(), failingTypes);
|
||||
return integrationTypeCause(type, cover);
|
||||
} catch (err) {
|
||||
console.error("[dependencies] couldn't work out whether a failure is explained:", err);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export async function nodeExists(type: NodeType, id: number): Promise<boolean> {
|
||||
if (type === "server") return (await db.select({ id: servers.id }).from(servers).where(eq(servers.id, id)).limit(1)).length > 0;
|
||||
return (await db.select({ id: integrations.id }).from(integrations).where(eq(integrations.id, id)).limit(1)).length > 0;
|
||||
}
|
||||
@@ -8,11 +8,12 @@ import { notifyHealthIssues, notifyHealthRecovered } from "./notify.js";
|
||||
import { recordAudit } from "./audit.js";
|
||||
import { getInternalFlag, getSettings, setInternalFlag } from "./settingsStore.js";
|
||||
import { activeSubjects, subjectOfConditionKey } from "./maintenance.js";
|
||||
import { coveredByParent, loadParentRows, resolveParents } from "./serverParents.js";
|
||||
import { dependencyCover, HEALTH_STATE_FLAG } from "./dependencies.js";
|
||||
import { getFailingSources } from "./integrationHealthMonitor.js";
|
||||
import { hostHealthConditions, parseHostHealth, type HostHealth } from "./hostHealth.js";
|
||||
import { diskLimit, parseThresholds, type ServerThresholds } from "./serverThresholds.js";
|
||||
|
||||
const STATE_FLAG = "healthActiveConditions";
|
||||
const STATE_FLAG = HEALTH_STATE_FLAG;
|
||||
/** After a restart the agents haven't had a chance to report yet (they run every 15 min), so server-derived conditions are held rather than judged. */
|
||||
const STARTUP_GRACE_MS = Number(process.env.HEALTH_STARTUP_GRACE_MS ?? 20 * 60 * 1000); // the environment variable is for tests
|
||||
const PROCESS_START = Date.now();
|
||||
@@ -396,18 +397,20 @@ export async function runHealthCheck(now: number = Date.now()): Promise<{ added:
|
||||
const m = /^offline:server:(\d+)$/.exec(c.key);
|
||||
if (m) offlineIds.add(Number(m[1]));
|
||||
}
|
||||
const cause = coveredByParent(offlineIds, resolveParents(await loadParentRows()));
|
||||
const coveredKeys = new Set([...cause.keys()].map((id) => `offline:server:${id}`));
|
||||
// (Also when it needs a service that is failing, or runs on a server that needs one.)
|
||||
const cover = await dependencyCover(offlineIds, getFailingSources().map((f) => f.source));
|
||||
const coveredKeys = new Set<string>();
|
||||
for (const child of cover.cause.keys()) {
|
||||
const m = /^server:(\d+)$/.exec(child);
|
||||
if (m) coveredKeys.add(`offline:server:${m[1]}`);
|
||||
}
|
||||
const { added: announced, resolved, next, newlyDown, ended } = diffConditions(await loadState(), current, held, await activeSubjects(new Date(now)), coveredKeys);
|
||||
await setInternalFlag(STATE_FLAG, JSON.stringify(next));
|
||||
|
||||
// A host's message says what else it takes with it.
|
||||
const names = new Map(snapshot.servers.map((s) => [s.id, s.name]));
|
||||
const taken = new Map<number, string[]>();
|
||||
for (const [child, parent] of cause) taken.set(parent, [...(taken.get(parent) ?? []), names.get(child) ?? `#${child}`]);
|
||||
const added = announced.map((c) => {
|
||||
const m = /^offline:server:(\d+)$/.exec(c.key);
|
||||
const children = m ? taken.get(Number(m[1])) : undefined;
|
||||
const children = m ? cover.taken.get(`server:${m[1]}`) : undefined;
|
||||
return children?.length ? { ...c, message: `${c.message} — also takes down ${alsoList(children)}` } : c;
|
||||
});
|
||||
|
||||
|
||||
@@ -18,6 +18,7 @@ export interface SmartDisk {
|
||||
/** NVMe and some SSDs say how much of their rated life is used, 0–100 (and past it). */
|
||||
wearPercent: number | null;
|
||||
}
|
||||
/** A storage pool: ZFS on Linux, Storage Spaces on Windows (which reports Healthy, Warning and Unhealthy as ONLINE, DEGRADED and FAULTED). */
|
||||
export interface ZfsPool {
|
||||
name: string;
|
||||
/** ONLINE, DEGRADED, FAULTED, UNAVAIL, SUSPENDED … as zpool says it. */
|
||||
@@ -117,7 +118,7 @@ export function hostHealthConditions(serverId: number, serverName: string, healt
|
||||
}
|
||||
for (const p of health.zfs ?? []) {
|
||||
if (!poolIsFine(p)) {
|
||||
out.push({ key: `zfs:server:${serverId}:${p.name}`, severity: "critical", message: `${serverName}: ZFS pool ${p.name} is ${p.health}` });
|
||||
out.push({ key: `zfs:server:${serverId}:${p.name}`, severity: "critical", message: `${serverName}: storage pool ${p.name} is ${p.health}` });
|
||||
}
|
||||
}
|
||||
return out;
|
||||
|
||||
@@ -6,6 +6,7 @@ import { ensureHistoryStart } from "./uptimeHistory.js";
|
||||
import { getSettings } from "./settingsStore.js";
|
||||
import { isSourceInMaintenance } from "./maintenance.js";
|
||||
import { notifyIntegrationDown, notifyIntegrationRecovered } from "./notify.js";
|
||||
import { integrationCausedBy } from "./dependencies.js";
|
||||
|
||||
interface SourceHealth {
|
||||
consecutiveFailures: number;
|
||||
@@ -96,7 +97,9 @@ export async function trackIntegrationHealth(source: string, ok: boolean): Promi
|
||||
if (notifications.integrationFailureAlerts && !state.alerted && state.consecutiveFailures >= notifications.integrationFailureThreshold) {
|
||||
// During maintenance the failures keep being counted but `alerted` stays false, so if the service is
|
||||
// still failing once the window ends, the very next failed call alerts — a real outage isn't swallowed.
|
||||
if (!(await isSourceInMaintenance(source))) {
|
||||
// Likewise when it needs something that is down (the server it runs on, another service): that one is what gets announced, and if this stays
|
||||
// down once that one is back, the next failed call alerts.
|
||||
if (!(await isSourceInMaintenance(source)) && !(await integrationCausedBy(source, [...getFailingSources().map((f) => f.source), source]))) {
|
||||
state.alerted = true;
|
||||
await notifyIntegrationDown(source, state.consecutiveFailures);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
/**
|
||||
* Filling in the inventory from what an agent has found out about the machine it runs on: maker, model and serial number. Typing a serial
|
||||
* number off a label is the part of an inventory nobody keeps up, so a linked entry gets it from the agent — but only into fields that are
|
||||
* blank, because what a person wrote down is theirs; "Refresh from agent" is the way to overwrite on purpose.
|
||||
*/
|
||||
import { eq, isNull, or, and } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { assets, servers } from "../db/schema.js";
|
||||
|
||||
export interface AgentHardware {
|
||||
manufacturer: string | null;
|
||||
model: string | null;
|
||||
serial: string | null;
|
||||
}
|
||||
|
||||
type Fields = { manufacturer: string | null; model: string | null; serial: string | null };
|
||||
const KEYS = ["manufacturer", "model", "serial"] as const;
|
||||
|
||||
/** What the server's agent said the machine is, or null if it has said nothing. */
|
||||
export function hardwareOf(server: { hwManufacturer: string | null; hwModel: string | null; hwSerial: string | null } | null | undefined): AgentHardware | null {
|
||||
if (!server) return null;
|
||||
const hw = { manufacturer: server.hwManufacturer ?? null, model: server.hwModel ?? null, serial: server.hwSerial ?? null };
|
||||
return hw.manufacturer || hw.model || hw.serial ? hw : null;
|
||||
}
|
||||
|
||||
/** The fields of an entry that are blank and that the agent can fill. Empty when nothing would change. */
|
||||
export function blanksToFill(current: Fields, hw: AgentHardware | null): Partial<Fields> {
|
||||
const out: Partial<Fields> = {};
|
||||
if (!hw) return out;
|
||||
for (const k of KEYS) if (!current[k]?.trim() && hw[k]) out[k] = hw[k];
|
||||
return out;
|
||||
}
|
||||
|
||||
/** The fields where the agent says something different from the entry: what a refresh would change. */
|
||||
export function differences(current: Fields, hw: AgentHardware | null): Partial<Fields> {
|
||||
const out: Partial<Fields> = {};
|
||||
if (!hw) return out;
|
||||
for (const k of KEYS) if (hw[k] && (current[k] ?? "").trim() !== hw[k]) out[k] = hw[k];
|
||||
return out;
|
||||
}
|
||||
|
||||
/** After an agent has reported, fills the blanks of the inventory entries that are this server. */
|
||||
export async function fillInventoryFromServer(serverId: number): Promise<number> {
|
||||
const [server] = await db.select({ hwManufacturer: servers.hwManufacturer, hwModel: servers.hwModel, hwSerial: servers.hwSerial }).from(servers).where(eq(servers.id, serverId)).limit(1);
|
||||
const hw = hardwareOf(server);
|
||||
if (!hw) return 0;
|
||||
// Only entries with something blank are worth reading.
|
||||
const linked = await db
|
||||
.select()
|
||||
.from(assets)
|
||||
.where(and(eq(assets.serverId, serverId), or(isNull(assets.manufacturer), isNull(assets.model), isNull(assets.serial))));
|
||||
let changed = 0;
|
||||
for (const a of linked) {
|
||||
const fill = blanksToFill(a, hw);
|
||||
if (Object.keys(fill).length === 0) continue;
|
||||
await db.update(assets).set({ ...fill, updatedAt: new Date().toISOString() }).where(eq(assets.id, a.id));
|
||||
changed++;
|
||||
}
|
||||
return changed;
|
||||
}
|
||||
@@ -95,7 +95,7 @@ export async function buildMetrics(now: Date = new Date()): Promise<string> {
|
||||
if (d.wearPercent !== null) out.add("homelab_server_disk_wear_percent", "gauge", "How much of an SSD's rated life is used.", d.wearPercent, { server, device: d.device });
|
||||
if (d.temperatureC !== null) out.add("homelab_server_disk_temperature_celsius", "gauge", "A disk's temperature.", d.temperatureC, { server, device: d.device });
|
||||
}
|
||||
for (const p of health?.zfs ?? []) out.add("homelab_server_zfs_pool_online", "gauge", "1 if a ZFS pool is ONLINE, 0 in any other state.", poolIsFine(p) ? 1 : 0, { server, pool: p.name });
|
||||
for (const p of health?.zfs ?? []) out.add("homelab_server_zfs_pool_online", "gauge", "1 if a storage pool (ZFS, or Storage Spaces on Windows) is ONLINE, 0 in any other state.", poolIsFine(p) ? 1 : 0, { server, pool: p.name });
|
||||
out.add("homelab_server_agent_outdated", "gauge", "1 if the server's agent is older than the one this installation would install now.", isAgentOutdated(s, latest) ? 1 : 0, { server });
|
||||
}
|
||||
// Disks on a steady climb, from each server's own history: days until full at the present rate (left out where there is no steady trend).
|
||||
|
||||
@@ -2,6 +2,7 @@ import nodemailer from "nodemailer";
|
||||
import { getSettings, type NotificationEvents } from "./settingsStore.js";
|
||||
import { enqueueNotification, listQueuedNotifications, clearQueuedNotifications } from "./notificationQueue.js";
|
||||
import { emitEvent } from "./eventWebhooks.js";
|
||||
import { sendPush, subscriptionCount } from "./webPush.js";
|
||||
|
||||
// ─── Channel senders (best-effort — failures are logged, never thrown) ──────
|
||||
|
||||
@@ -72,8 +73,18 @@ async function sendWebhook(title: string, message: string) {
|
||||
}
|
||||
}
|
||||
|
||||
async function sendWebPush(title: string, message: string) {
|
||||
const { webPush } = await getSettings();
|
||||
if (!webPush.enabled) return;
|
||||
try {
|
||||
await sendPush({ title, body: message });
|
||||
} catch (err) {
|
||||
console.error("[notify] Web push failed:", err instanceof Error ? err.message : err);
|
||||
}
|
||||
}
|
||||
|
||||
async function sendToChannels(title: string, message: string): Promise<void> {
|
||||
await Promise.all([sendGotify(title, message), sendNtfy(title, message), sendSmtp(title, message), sendWebhook(title, message)]);
|
||||
await Promise.all([sendGotify(title, message), sendNtfy(title, message), sendSmtp(title, message), sendWebhook(title, message), sendWebPush(title, message)]);
|
||||
}
|
||||
|
||||
function currentTimeInZone(timezone: string): { hh: number; mm: number } {
|
||||
@@ -112,8 +123,9 @@ export async function notifyNow(title: string, message: string): Promise<void> {
|
||||
|
||||
/** How many channels are switched on and filled in enough to send, so a "send it now" can say when there is nowhere to send to. */
|
||||
export async function enabledChannelCount(): Promise<number> {
|
||||
const { gotify, ntfy, smtp, webhook } = await getSettings();
|
||||
const { gotify, ntfy, smtp, webhook, webPush } = await getSettings();
|
||||
return [
|
||||
webPush.enabled && (await subscriptionCount()) > 0,
|
||||
gotify.enabled && gotify.url && gotify.token,
|
||||
ntfy.enabled && ntfy.url && ntfy.topic,
|
||||
smtp.enabled && smtp.host && smtp.to && smtp.from,
|
||||
|
||||
@@ -37,6 +37,13 @@ export interface WebhookSettings {
|
||||
secret: string;
|
||||
}
|
||||
|
||||
/** Push notifications to phones and browsers that have subscribed (see services/webPush.ts). Who is subscribed is kept apart, per device. */
|
||||
export interface WebPushSettings {
|
||||
enabled: boolean;
|
||||
/** Where a push service can reach whoever runs this: an email address or an https address (VAPID asks for one). Blank is fine. */
|
||||
contact: string;
|
||||
}
|
||||
|
||||
export interface NotificationEvents {
|
||||
dnsAdd: boolean;
|
||||
dnsUpdate: boolean;
|
||||
@@ -189,6 +196,7 @@ export interface AppSettings {
|
||||
ntfy: NtfySettings;
|
||||
smtp: SmtpSettings;
|
||||
webhook: WebhookSettings;
|
||||
webPush: WebPushSettings;
|
||||
notifications: NotificationEvents;
|
||||
providerColors: ProviderColors;
|
||||
integrationColors: IntegrationColors;
|
||||
@@ -211,6 +219,7 @@ const DEFAULTS: AppSettings = {
|
||||
ntfy: { enabled: false, url: "https://ntfy.sh", topic: "", token: "", priority: 3 },
|
||||
smtp: { enabled: false, host: "", port: 587, secure: false, username: "", password: "", from: "", to: "" },
|
||||
webhook: { enabled: false, url: "", secret: "" },
|
||||
webPush: { enabled: false, contact: "" },
|
||||
notifications: {
|
||||
dnsAdd: true,
|
||||
dnsUpdate: true,
|
||||
|
||||
@@ -5,6 +5,7 @@ import { recordServerMetrics } from "./serverMetrics.js";
|
||||
import { nextUpdateState, type IncomingUpdates } from "./pendingUpdates.js";
|
||||
import { nextHostHealth, type IncomingHostHealth } from "./hostHealth.js";
|
||||
import { recordAudit } from "./audit.js";
|
||||
import { fillInventoryFromServer } from "./inventoryAutofill.js";
|
||||
|
||||
export interface IncomingTask {
|
||||
scheduleType: "cron" | "systemd_timer" | "windows_task";
|
||||
@@ -27,6 +28,8 @@ export interface IncomingSystemInfo {
|
||||
updates?: IncomingUpdates;
|
||||
/** Failed services, disk health, ZFS pools — each part present only if the agent could find it out. */
|
||||
health?: IncomingHostHealth;
|
||||
/** What the machine is, from its firmware: each part present only if it says anything. */
|
||||
hardware?: { manufacturer?: string; model?: string; serial?: string };
|
||||
}
|
||||
|
||||
export interface AgentReport {
|
||||
@@ -115,6 +118,9 @@ export async function syncServerTasks(serverId: number, report: AgentReport) {
|
||||
// Only stored when the agent reported it, so an agent that predates this (or couldn't tell this time) doesn't wipe what it said before.
|
||||
...(system?.updates && before ? nextUpdateState(before, system.updates, now) : {}),
|
||||
...(system?.health && before ? { hostHealth: nextHostHealth(before.hostHealth, system.health, now) } : {}),
|
||||
...(system?.hardware?.manufacturer ? { hwManufacturer: system.hardware.manufacturer } : {}),
|
||||
...(system?.hardware?.model ? { hwModel: system.hardware.model } : {}),
|
||||
...(system?.hardware?.serial ? { hwSerial: system.hardware.serial } : {}),
|
||||
...(report.hostname ? { hostname: report.hostname } : {}),
|
||||
...(report.agentVersion !== undefined ? { agentVersion: report.agentVersion } : {}),
|
||||
// An agent knows what it runs on better than whoever registered the server did.
|
||||
@@ -146,6 +152,15 @@ export async function syncServerTasks(serverId: number, report: AgentReport) {
|
||||
}
|
||||
}
|
||||
|
||||
// A machine whose agent has said what it is fills in the blanks of its inventory entry. Also best-effort.
|
||||
if (system?.hardware) {
|
||||
try {
|
||||
await fillInventoryFromServer(serverId);
|
||||
} catch (err) {
|
||||
console.error("[inventory] couldn't fill in from the agent:", err);
|
||||
}
|
||||
}
|
||||
|
||||
// Keep a little history of the figures too. A failure here must not lose the report itself, which is already saved.
|
||||
try {
|
||||
await recordServerMetrics(serverId, system, new Date(now));
|
||||
|
||||
@@ -0,0 +1,245 @@
|
||||
/**
|
||||
* Web Push: notifications to a phone or a browser that has asked for them, with no app and no account — the browser's own push service
|
||||
* (Google's, Mozilla's, Apple's) carries the message. Written against the standards directly rather than through a library:
|
||||
*
|
||||
* - RFC 8292 (VAPID): the server proves who it is with a short-lived signed token (ES256), so a push service knows the sender is the
|
||||
* one the subscription was made for.
|
||||
* - RFC 8291 + 8188 (aes128gcm): the message is encrypted for the one browser it is for; the push service only ever carries it.
|
||||
*
|
||||
* The server's key pair is made the first time it is needed and kept (the private half encrypted, like the other credentials).
|
||||
*/
|
||||
import { createCipheriv, createECDH, createPrivateKey, hkdfSync, randomBytes, sign, type KeyObject } from "node:crypto";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { pushSubscriptions } from "../db/schema.js";
|
||||
import { env } from "../env.js";
|
||||
import { decryptSecret, encryptSecret } from "../crypto.js";
|
||||
import { resolveTarget } from "../utils/networkGuard.js";
|
||||
import { getInternalFlag, getSettings, setInternalFlag } from "./settingsStore.js";
|
||||
|
||||
const b64u = (buf: Buffer | Uint8Array) => Buffer.from(buf).toString("base64url");
|
||||
const fromB64u = (s: string) => Buffer.from(s, "base64url");
|
||||
|
||||
// ─── The server's own key pair ─────────────────────────────────────────────
|
||||
|
||||
const VAPID_FLAG = "webPushVapid";
|
||||
const ENCRYPTED = "enc:v1:";
|
||||
|
||||
export interface VapidKeys {
|
||||
/** The uncompressed P-256 point, base64url — what a browser is given to subscribe with. */
|
||||
publicKey: string;
|
||||
/** The private scalar, base64url. */
|
||||
privateKey: string;
|
||||
}
|
||||
|
||||
let cachedKeys: Promise<VapidKeys> | null = null;
|
||||
|
||||
/** The key pair, made on first use and kept. */
|
||||
export function getVapidKeys(): Promise<VapidKeys> {
|
||||
cachedKeys ??= loadOrMakeKeys().catch((err) => {
|
||||
cachedKeys = null;
|
||||
throw err;
|
||||
});
|
||||
return cachedKeys;
|
||||
}
|
||||
|
||||
async function loadOrMakeKeys(): Promise<VapidKeys> {
|
||||
const stored = await getInternalFlag(VAPID_FLAG);
|
||||
if (stored) {
|
||||
try {
|
||||
const v = JSON.parse(stored) as { publicKey: string; privateKey: string };
|
||||
const privateKey = v.privateKey.startsWith(ENCRYPTED) ? decryptSecret(v.privateKey.slice(ENCRYPTED.length)) : v.privateKey;
|
||||
return { publicKey: v.publicKey, privateKey };
|
||||
} catch {
|
||||
// Unreadable (the encryption key changed): made again below — every browser must then subscribe again, which the page says.
|
||||
}
|
||||
}
|
||||
const ecdh = createECDH("prime256v1");
|
||||
ecdh.generateKeys();
|
||||
const keys: VapidKeys = { publicKey: b64u(ecdh.getPublicKey()), privateKey: b64u(ecdh.getPrivateKey().length === 32 ? ecdh.getPrivateKey() : Buffer.concat([Buffer.alloc(32 - ecdh.getPrivateKey().length), ecdh.getPrivateKey()])) };
|
||||
await setInternalFlag(VAPID_FLAG, JSON.stringify({ publicKey: keys.publicKey, privateKey: env.credentialsEncryptionEnabled ? ENCRYPTED + encryptSecret(keys.privateKey) : keys.privateKey }));
|
||||
// The subscriptions made for the old key can't be used with the new one.
|
||||
await db.delete(pushSubscriptions);
|
||||
return keys;
|
||||
}
|
||||
|
||||
/** For the tests, so one test's keys don't leak into the next. */
|
||||
export function forgetVapidKeysForTests(): void {
|
||||
cachedKeys = null;
|
||||
}
|
||||
|
||||
/** A P-256 private key as a key object, from its raw scalar and public point. */
|
||||
function privateKeyObject(keys: VapidKeys): KeyObject {
|
||||
const pub = fromB64u(keys.publicKey);
|
||||
return createPrivateKey({
|
||||
key: { kty: "EC", crv: "P-256", d: keys.privateKey, x: b64u(pub.subarray(1, 33)), y: b64u(pub.subarray(33, 65)) },
|
||||
format: "jwk",
|
||||
});
|
||||
}
|
||||
|
||||
/** The Authorization header's token: who we are, who it is for, and for how long (RFC 8292). */
|
||||
export function vapidAuthorization(audience: string, subject: string, keys: VapidKeys, now: number = Date.now()): string {
|
||||
const header = b64u(Buffer.from(JSON.stringify({ typ: "JWT", alg: "ES256" })));
|
||||
const claims = b64u(Buffer.from(JSON.stringify({ aud: audience, exp: Math.floor(now / 1000) + 12 * 3600, sub: subject })));
|
||||
const signature = sign("sha256", Buffer.from(`${header}.${claims}`), { key: privateKeyObject(keys), dsaEncoding: "ieee-p1363" });
|
||||
return `vapid t=${header}.${claims}.${b64u(signature)}, k=${keys.publicKey}`;
|
||||
}
|
||||
|
||||
// ─── Encrypting a message for one browser (RFC 8291) ───────────────────────
|
||||
|
||||
export interface PushKeys {
|
||||
/** The browser's public key, base64url. */
|
||||
p256dh: string;
|
||||
/** The browser's 16-byte authentication secret, base64url. */
|
||||
auth: string;
|
||||
}
|
||||
|
||||
const RECORD_SIZE = 4096;
|
||||
|
||||
/**
|
||||
* The body to send: the message encrypted so that only the browser it is for can read it. `ephemeral` and `salt` are chosen fresh each
|
||||
* time; they are parameters only so the standard's own worked example can be checked.
|
||||
*/
|
||||
export function encryptPayload(payload: Buffer, subscription: PushKeys, fixed?: { ephemeral: ECDHLike; salt: Buffer }): Buffer {
|
||||
const uaPublic = fromB64u(subscription.p256dh);
|
||||
const authSecret = fromB64u(subscription.auth);
|
||||
if (uaPublic.length !== 65 || uaPublic[0] !== 4) throw new Error("The browser's public key isn't a P-256 point.");
|
||||
if (authSecret.length !== 16) throw new Error("The browser's authentication secret isn't 16 bytes.");
|
||||
|
||||
const fresh = fixed ? null : createECDH("prime256v1");
|
||||
fresh?.generateKeys();
|
||||
const ecdh: ECDHLike = fixed?.ephemeral ?? fresh!;
|
||||
const asPublic = ecdh.getPublicKey();
|
||||
const salt = fixed?.salt ?? randomBytes(16);
|
||||
const secret = ecdh.computeSecret(uaPublic);
|
||||
|
||||
const keyInfo = Buffer.concat([Buffer.from("WebPush: info\0"), uaPublic, asPublic]);
|
||||
const ikm = Buffer.from(hkdfSync("sha256", secret, authSecret, keyInfo, 32));
|
||||
const cek = Buffer.from(hkdfSync("sha256", ikm, salt, Buffer.from("Content-Encoding: aes128gcm\0"), 16));
|
||||
const nonce = Buffer.from(hkdfSync("sha256", ikm, salt, Buffer.from("Content-Encoding: nonce\0"), 12));
|
||||
|
||||
// One record: the message, then a 0x02 marking it as the last.
|
||||
if (payload.length > RECORD_SIZE - 17 - 1) throw new Error("The message is too long for one push.");
|
||||
const cipher = createCipheriv("aes-128-gcm", cek, nonce);
|
||||
const encrypted = Buffer.concat([cipher.update(Buffer.concat([payload, Buffer.from([2])])), cipher.final(), cipher.getAuthTag()]);
|
||||
|
||||
const header = Buffer.alloc(16 + 4 + 1);
|
||||
salt.copy(header, 0);
|
||||
header.writeUInt32BE(RECORD_SIZE, 16);
|
||||
header.writeUInt8(asPublic.length, 20);
|
||||
return Buffer.concat([header, asPublic, encrypted]);
|
||||
}
|
||||
|
||||
type ECDHLike = Pick<ReturnType<typeof createECDH>, "getPublicKey" | "computeSecret">;
|
||||
|
||||
// ─── Subscriptions ─────────────────────────────────────────────────────────
|
||||
|
||||
let allowLocalEndpoints = false;
|
||||
/** Lets the tests push to a service on this machine, which is otherwise refused. */
|
||||
export function allowLocalPushEndpointsForTests(allow: boolean): void {
|
||||
allowLocalEndpoints = allow;
|
||||
}
|
||||
|
||||
/** A push service is a public internet service with an https address; anything else is refused before we ever connect to it. */
|
||||
export async function checkEndpoint(endpoint: string): Promise<void> {
|
||||
let url: URL;
|
||||
try {
|
||||
url = new URL(endpoint);
|
||||
} catch {
|
||||
throw new Error("The push address isn't a web address.");
|
||||
}
|
||||
if (url.protocol !== "https:" && !(allowLocalEndpoints && url.protocol === "http:")) throw new Error("The push address must be https.");
|
||||
if (url.username || url.password) throw new Error("The push address can't carry a login.");
|
||||
if (allowLocalEndpoints) return;
|
||||
await resolveTarget(url.hostname, { allowPrivate: false });
|
||||
}
|
||||
|
||||
export interface PushOutcome {
|
||||
sent: number;
|
||||
/** Subscriptions the push service said are gone, now removed. */
|
||||
removed: number;
|
||||
failed: number;
|
||||
}
|
||||
|
||||
/** The contact the push services may reach us on (a `mailto:` or an https address), as VAPID asks. */
|
||||
export async function vapidSubject(): Promise<string> {
|
||||
const { webPush } = await getSettings();
|
||||
const contact = webPush.contact.trim();
|
||||
if (/^(mailto:|https:\/\/)/i.test(contact)) return contact;
|
||||
if (contact.includes("@")) return `mailto:${contact}`;
|
||||
return /^https:\/\//.test(env.appBaseUrl) ? env.appBaseUrl : "mailto:homelab@example.invalid";
|
||||
}
|
||||
|
||||
export interface PushMessage {
|
||||
title: string;
|
||||
body: string;
|
||||
/** Where in the app a tap should go. */
|
||||
url?: string;
|
||||
/** A later message with the same tag replaces an earlier one that is still showing. */
|
||||
tag?: string;
|
||||
}
|
||||
|
||||
/** Sends one message to one subscription. Resolves to what the push service said; never throws. */
|
||||
async function deliver(sub: { endpoint: string; p256dh: string; auth: string }, body: Buffer, keys: VapidKeys, subject: string, urgent: boolean): Promise<"sent" | "gone" | "failed"> {
|
||||
try {
|
||||
await checkEndpoint(sub.endpoint);
|
||||
const res = await fetch(sub.endpoint, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
Authorization: vapidAuthorization(new URL(sub.endpoint).origin, subject, keys),
|
||||
"Content-Encoding": "aes128gcm",
|
||||
"Content-Type": "application/octet-stream",
|
||||
TTL: "86400",
|
||||
Urgency: urgent ? "high" : "normal",
|
||||
},
|
||||
body: new Uint8Array(body),
|
||||
signal: AbortSignal.timeout(15_000),
|
||||
});
|
||||
if (res.ok) return "sent";
|
||||
if (res.status === 404 || res.status === 410) return "gone";
|
||||
console.error(`[push] ${new URL(sub.endpoint).host} answered ${res.status}: ${(await res.text().catch(() => "")).slice(0, 200)}`);
|
||||
return "failed";
|
||||
} catch (err) {
|
||||
console.error("[push] couldn't deliver:", err instanceof Error ? err.message : err);
|
||||
return "failed";
|
||||
}
|
||||
}
|
||||
|
||||
/** Sends to the given subscriptions (all of them, if none are given). Subscriptions the push service says are gone are removed. */
|
||||
export async function sendPush(message: PushMessage, only?: { id: number; endpoint: string; p256dh: string; auth: string }[]): Promise<PushOutcome> {
|
||||
const subs = only ?? (await db.select().from(pushSubscriptions));
|
||||
const outcome: PushOutcome = { sent: 0, removed: 0, failed: 0 };
|
||||
if (subs.length === 0) return outcome;
|
||||
const keys = await getVapidKeys();
|
||||
const subject = await vapidSubject();
|
||||
const payload = Buffer.from(JSON.stringify({ title: message.title.slice(0, 120), body: message.body.slice(0, 600), url: message.url ?? "/alerts", tag: message.tag }));
|
||||
await Promise.all(
|
||||
subs.map(async (s) => {
|
||||
let body: Buffer;
|
||||
try {
|
||||
body = encryptPayload(payload, { p256dh: s.p256dh, auth: s.auth });
|
||||
} catch (err) {
|
||||
console.error("[push] a subscription can't be written to:", err instanceof Error ? err.message : err);
|
||||
outcome.failed++;
|
||||
return;
|
||||
}
|
||||
const result = await deliver(s, body, keys, subject, true);
|
||||
if (result === "sent") {
|
||||
outcome.sent++;
|
||||
await db.update(pushSubscriptions).set({ lastSuccessAt: new Date().toISOString() }).where(eq(pushSubscriptions.id, s.id));
|
||||
} else if (result === "gone") {
|
||||
outcome.removed++;
|
||||
await db.delete(pushSubscriptions).where(eq(pushSubscriptions.id, s.id));
|
||||
} else {
|
||||
outcome.failed++;
|
||||
}
|
||||
}),
|
||||
);
|
||||
return outcome;
|
||||
}
|
||||
|
||||
/** How many browsers are subscribed. */
|
||||
export async function subscriptionCount(): Promise<number> {
|
||||
return (await db.select({ id: pushSubscriptions.id }).from(pushSubscriptions)).length;
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user