Stale-backup alerts, per-server alert limits, and update runs from the Updates page

- Backups: a guest whose newest backup is older than its limit is an alert (backup kind),
  in the daily backup message, the digest and /metrics (homelab_backup_guests, ..._last_age_seconds)
- Per-server thresholds (servers.thresholds, migration 0030): offline minutes, disk percent,
  forecast window, backup age, and per-disk limits or "leave alone"; applied everywhere the
  global ones were; saving clears the alerts cache
- Update runs (update_runs, migration 0031): Update... on the Updates page puts the server under
  maintenance, runs a Semaphore playbook limited to it, waits for the host to report, and
  reports what is left and whether a restart is due; poller survives restarts
- Notification toggle for update-run results; tests and docs

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-18 01:25:09 +02:00
1 parent 5dc6cea69c
commit 7a5b623444
42 files changed
+7561 -57

No files matched your search

+2
View File
@@ -11,6 +11,7 @@ import { meRouter } from "./routes/me.js";
import { uptimeRouter } from "./routes/uptime.js";
import { shareLinksRouter } from "./routes/shareLinks.js";
import { runbooksRouter } from "./routes/runbooks.js";
import { updateRunsRouter } from "./routes/updateRuns.js";
import { inventoryRouter } from "./routes/inventory.js";
import { backupsRouter } from "./routes/backups.js";
import { metricsRouter, metricsTokensRouter } from "./routes/metrics.js";
@@ -98,6 +99,7 @@ export function createApp() {
app.use("/api/uptime", uptimeRouter);
app.use("/api/share-links", shareLinksRouter);
app.use("/api/runbooks", runbooksRouter);
app.use("/api/update-runs", updateRunsRouter);
app.use("/api/inventory", inventoryRouter);
app.use("/api/backups", backupsRouter);
app.use("/api/metrics-tokens", metricsTokensRouter);
+39
View File
@@ -300,6 +300,10 @@ export const servers = sqliteTable("servers", {
// What the agent says about the machine's own health, as JSON: {failedUnits, smart, zfs, reportedAt} — see services/hostHealth.ts. Null until it says.
hostHealth: text("host_health"),
// This server's own alert thresholds, as JSON: {serverOfflineMinutes, diskUsagePercent, diskForecastDays, backupStaleHours, mounts: {"/data": {percent, ignore}}} — see
// services/serverThresholds.ts. Whatever isn't there follows Settings. Null when it follows Settings entirely.
thresholds: text("thresholds"),
// Which version of the agent script is running on it, as the agent says in each report (see agent/*/report-tasks.*). Null until it says:
// an agent that predates the number being reported is version 1.
agentVersion: integer("agent_version"),
@@ -568,6 +572,41 @@ export const runbooks = sqliteTable(
(t) => [index("runbooks_target_idx").on(t.targetType, t.targetId)],
);
// ─── Update runs: apply a server's updates and see it through ──────────────
// A playbook run in Semaphore for one server, wrapped in a maintenance window and followed until the server reports again — see
// services/updateRuns.ts. status: running | waiting_for_host | done | failed | cancelled.
export const updateRuns = sqliteTable(
"update_runs",
{
id: integer("id").primaryKey({ autoIncrement: true }),
serverId: integer("server_id")
.notNull()
.references(() => servers.id, { onDelete: "cascade" }),
integrationId: integer("integration_id").notNull(), // the Semaphore integration; no FK — one removed later leaves the history readable
projectId: integer("project_id").notNull(),
templateId: integer("template_id").notNull(),
templateName: text("template_name").notNull(),
hostLimit: text("host_limit").notNull(), // ansible's --limit
taskId: integer("task_id"),
taskStatus: text("task_status"), // Semaphore's own word for the run
windowId: integer("window_id"), // the maintenance window put round it
windowCreated: integer("window_created", { mode: "boolean" }).notNull().default(false), // false if it only extended one that was already there (which is then left alone)
status: text("status").notNull(),
message: text("message"),
startedBy: text("started_by"),
startedAt: text("started_at").notNull(), // ISO
taskEndedAt: text("task_ended_at"), // ISO — when the playbook finished
hostReportedAt: text("host_reported_at"), // ISO — the report that came after it
finishedAt: text("finished_at"), // ISO
updatesBefore: integer("updates_before"),
updatesAfter: integer("updates_after"),
rebootRequired: integer("reboot_required", { mode: "boolean" }).notNull().default(false),
pollErrors: integer("poll_errors").notNull().default(0),
},
(t) => [index("update_runs_server_idx").on(t.serverId, t.startedAt), index("update_runs_status_idx").on(t.status)],
);
// ─── Inventory: the hardware behind it all ──────────────────────────────────
export const assetKinds = ["server", "storage", "network", "power", "client", "other"] as const;
+2
View File
@@ -16,6 +16,7 @@ import { initHealthScheduler } from "./services/healthScheduler.js";
import { initAlertTicketScheduler } from "./services/alertTicketScheduler.js";
import { initKumaMaintenanceScheduler } from "./services/kumaMaintenanceScheduler.js";
import { initMaintenanceScheduleScheduler } from "./services/maintenanceScheduleScheduler.js";
import { initUpdateRuns } from "./services/updateRuns.js";
{
// Sessions are signed with this secret. Without one, the app would sign them with a value that's in the source code.
@@ -48,6 +49,7 @@ await initHealthScheduler();
await initAlertTicketScheduler();
await initKumaMaintenanceScheduler();
await initMaintenanceScheduleScheduler();
initUpdateRuns();
createApp().listen(env.port, () => {
console.log(`homelab-manager listening on port ${env.port}`);
+22 -7
View File
@@ -9,6 +9,7 @@ import { recordAudit } from "../services/audit.js";
import { loadParentRows, resolveParents, wouldLoop } from "../services/serverParents.js";
import { parsePendingUpdates } from "../services/pendingUpdates.js";
import { parseHostHealth } from "../services/hostHealth.js";
import { checkThresholds, parseThresholds } from "../services/serverThresholds.js";
import { latestAgentVersions } from "../services/agentVersions.js";
import { getServerMetrics, RETENTION_DAYS } from "../services/serverMetrics.js";
import { getServerActivity } from "../services/serverActivity.js";
@@ -20,14 +21,15 @@ import { serverPortsRouter } from "./serverPorts.js";
import { InvalidTagError, normalizeTags, parseTags } from "../services/serverTags.js";
import { buildServerSummary } from "../services/serverSummary.js";
import { activeSubjects } from "../services/maintenance.js";
import { invalidateAlertsCache } from "../services/alerts.js";
import { getSettings, updateSettings } from "../services/settingsStore.js";
export const serversRouter = Router();
/** A server row as the API returns it: no token hash, and tags as an array rather than the stored JSON. */
function publicServer<T extends { apiTokenHash: string; tags: string | null; pendingUpdates: string | null; hostHealth: string | null }>(row: T) {
const { apiTokenHash: _hash, tags, pendingUpdates, hostHealth, ...rest } = row;
return { ...rest, tags: parseTags(tags), pendingUpdates: parsePendingUpdates(pendingUpdates), hostHealth: parseHostHealth(hostHealth) };
function publicServer<T extends { apiTokenHash: string; tags: string | null; pendingUpdates: string | null; hostHealth: string | null; thresholds: string | null }>(row: T) {
const { apiTokenHash: _hash, tags, pendingUpdates, hostHealth, thresholds, ...rest } = row;
return { ...rest, tags: parseTags(tags), pendingUpdates: parsePendingUpdates(pendingUpdates), hostHealth: parseHostHealth(hostHealth), thresholds: parseThresholds(thresholds) };
}
serversRouter.use(requireAuth);
@@ -135,6 +137,8 @@ const updateServerSchema = z
parentServerId: z.number().int().nullable().optional(),
// True says it runs on nothing, which also stops the Proxmox-node guess. Choosing a server, or null, turns it off again.
noParent: z.boolean().optional(),
// This server's own alert thresholds (null goes back to following Settings). Checked on its own below, so a wrong number gets a message that says which.
thresholds: z.unknown().optional(),
})
.refine(
(data) => {
@@ -181,10 +185,20 @@ serversRouter.patch("/:id", requireRole("admin"), asyncHandler(async (req, res)
}
}
const changes: typeof parsed.data = { ...parsed.data };
const { thresholds: requestedThresholds, ...fields } = parsed.data;
const changes: typeof fields & { thresholds?: string | null } = { ...fields };
if (changes.noParent === true) changes.parentServerId = null;
else if (changes.parentServerId !== undefined) changes.noParent = false;
let thresholdsChange: unknown;
if (requestedThresholds !== undefined) {
const checked = checkThresholds(requestedThresholds);
if (!checked.ok) return res.status(400).json({ error: "invalid_thresholds", message: checked.message });
changes.thresholds = checked.value ? JSON.stringify(checked.value) : null;
thresholdsChange = checked.value;
}
const [updated] = await db.update(servers).set(changes).where(eq(servers.id, id)).returning();
// A server's own limits (or the server it runs on) change what is an alert, so the Alerts page shouldn't go on showing the old answer for up to a minute.
if (requestedThresholds !== undefined || fields.parentServerId !== undefined || fields.noParent !== undefined) invalidateAlertsCache();
await recordAudit({
actor: req.currentUser!,
@@ -192,7 +206,7 @@ serversRouter.patch("/:id", requireRole("admin"), asyncHandler(async (req, res)
action: "update",
targetType: "server",
targetId: id,
detail: { name: updated.name, ...(parsed.data.parentServerId !== undefined ? { parentServerId: parsed.data.parentServerId } : {}), ...(parsed.data.noParent !== undefined ? { noParent: parsed.data.noParent } : {}) },
detail: { name: updated.name, ...(parsed.data.parentServerId !== undefined ? { parentServerId: parsed.data.parentServerId } : {}), ...(parsed.data.noParent !== undefined ? { noParent: parsed.data.noParent } : {}), ...(requestedThresholds !== undefined ? { thresholds: thresholdsChange } : {}) },
});
const serverOut = publicServer(updated);
@@ -203,7 +217,7 @@ serversRouter.patch("/:id", requireRole("admin"), asyncHandler(async (req, res)
// (which viewers can't read), so the widget and the alerts always agree about what "offline" and "full" mean.
serversRouter.get("/summary", asyncHandler(async (_req, res) => {
const rows = await db
.select({ id: servers.id, name: servers.name, osType: servers.osType, lastSeenAt: servers.lastSeenAt, disks: servers.disks })
.select({ id: servers.id, name: servers.name, osType: servers.osType, lastSeenAt: servers.lastSeenAt, disks: servers.disks, thresholds: servers.thresholds })
.from(servers);
const { healthChecks } = await getSettings();
res.json(buildServerSummary(rows, healthChecks, Date.now(), await activeSubjects()));
@@ -367,6 +381,7 @@ serversRouter.get("/:id/detail", asyncHandler(async (req, res) => {
lastPortScan: _lastPortScan,
pendingUpdates: rawUpdates,
hostHealth: rawHealth,
thresholds: rawThresholds,
...serverOut
} = server;
@@ -380,7 +395,7 @@ serversRouter.get("/:id/detail", asyncHandler(async (req, res) => {
children: [...links2.entries()].filter(([, l]) => l.parentId === id).map(([childId, l]) => ({ id: childId, name: nameOf.get(childId) ?? "", auto: l.auto })).sort((a, b) => a.name.localeCompare(b.name)),
};
res.json({ server: { ...serverOut, tags: parseTags(rawTags), pendingUpdates: parsePendingUpdates(rawUpdates), hostHealth: parseHostHealth(rawHealth) }, ipAddresses, hardware, dnsMatches, links, relations, agentLatest: latestAgentVersions() });
res.json({ server: { ...serverOut, tags: parseTags(rawTags), pendingUpdates: parsePendingUpdates(rawUpdates), hostHealth: parseHostHealth(rawHealth), thresholds: parseThresholds(rawThresholds) }, thresholdDefaults: (await getSettings()).healthChecks, ipAddresses, hardware, dnsMatches, links, relations, agentLatest: latestAgentVersions() });
}));
const tagsSchema = z.object({ tags: z.array(z.string().max(100)).max(50) });
+2 -1
View File
@@ -76,6 +76,7 @@ const updateSchema = z.object({
pbsVerificationCheck: z.boolean(),
healthAlerts: z.boolean(),
automationAlerts: z.boolean(),
updateRunResults: z.boolean(),
domainExpiryCheck: z.boolean(),
secretCheckTime: z.string().regex(/^\d{2}:\d{2}$/),
timezone: z.string(),
@@ -95,7 +96,7 @@ const updateSchema = z.object({
.partial()
.optional(),
healthChecks: z
.object({ serverOfflineMinutes: z.number().int().min(15).max(10080), diskUsagePercent: z.number().int().min(50).max(99), domainWarnDays: z.number().int().min(1).max(365), diskForecastDays: z.number().int().min(0).max(90) })
.object({ serverOfflineMinutes: z.number().int().min(15).max(10080), diskUsagePercent: z.number().int().min(50).max(99), domainWarnDays: z.number().int().min(1).max(365), diskForecastDays: z.number().int().min(0).max(90), backupStaleHours: z.number().int().min(0).max(2160) })
.partial()
.optional(),
quietHours: z
+78
View File
@@ -0,0 +1,78 @@
import { Router } from "express";
import { z } from "zod";
import { requireAuth, requireRole } from "../auth/middleware.js";
import { asyncHandler } from "../utils/asyncHandler.js";
import { ACTIVE_STATUSES, cancelUpdateRun, listUpdateRuns, startUpdateRun, UpdateRunError } from "../services/updateRuns.js";
export const updateRunsRouter = Router();
// Anyone signed in can see how updates are going, like the Updates page itself. Starting or stopping one is for operators.
updateRunsRouter.use(requireAuth);
// Which hosts the playbook may touch: names, addresses and ansible patterns — and nothing that could be taken for an option. The same rule the
// Semaphore page uses, except that it may not be left out: an update run is for one server.
const startSchema = z.object({
serverId: z.number().int().positive(),
integrationId: z.number().int().positive(),
projectId: z.number().int().positive(),
templateId: z.number().int().positive(),
limit: z.string().trim().regex(/^[A-Za-z0-9][A-Za-z0-9._,:*\-]{0,199}$/, "Use the host name as it is in the Semaphore inventory, like web01"),
windowMinutes: z.number().int().min(15).max(480).default(90),
});
const shape = (r: Awaited<ReturnType<typeof listUpdateRuns>>[number]) => ({
id: r.id,
serverId: r.serverId,
serverName: r.serverName,
integrationId: r.integrationId,
template: r.templateName,
limit: r.hostLimit,
taskId: r.taskId,
taskStatus: r.taskStatus,
status: r.status,
active: ACTIVE_STATUSES.includes(r.status as (typeof ACTIVE_STATUSES)[number]),
message: r.message,
startedBy: r.startedBy,
startedAt: r.startedAt,
taskEndedAt: r.taskEndedAt,
finishedAt: r.finishedAt,
updatesBefore: r.updatesBefore,
updatesAfter: r.updatesAfter,
rebootRequired: r.rebootRequired,
});
updateRunsRouter.get("/", asyncHandler(async (req, res) => {
const serverId = Number(req.query.serverId);
const runs = await listUpdateRuns({ serverId: Number.isInteger(serverId) && serverId > 0 ? serverId : undefined, limit: 30 });
res.json({ runs: runs.map(shape) });
}));
updateRunsRouter.post("/", requireRole("operator"), asyncHandler(async (req, res) => {
const parsed = startSchema.safeParse(req.body);
if (!parsed.success) {
return res.status(400).json({ error: "invalid_body", message: parsed.error.issues[0]?.message ?? "Choose a server, a template and the host name it runs on.", details: parsed.error.flatten() });
}
const actor = req.currentUser!;
try {
const run = await startUpdateRun({ ...parsed.data, actor: actor.name ?? actor.email ?? actor.oidcSub });
const [shaped] = (await listUpdateRuns({ serverId: run.serverId, limit: 1 })).map(shape);
res.status(201).json({ run: shaped });
} catch (err) {
if (err instanceof UpdateRunError) return res.status(err.status).json({ error: err.code, message: err.message });
throw err;
}
}));
updateRunsRouter.post("/:id/cancel", requireRole("operator"), asyncHandler(async (req, res) => {
const id = Number(req.params.id);
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
const actor = req.currentUser!;
try {
const run = await cancelUpdateRun(id, actor.name ?? actor.email ?? actor.oidcSub);
if (!run) return res.status(404).json({ error: "not_found" });
res.json({ status: run.status });
} catch (err) {
if (err instanceof UpdateRunError) return res.status(err.status).json({ error: err.code, message: err.message });
throw err;
}
}));
+27 -3
View File
@@ -13,6 +13,8 @@ import { eq } from "drizzle-orm";
import { db } from "../db/client.js";
import { integrations, secrets, servers } from "../db/schema.js";
import { disksFillingSoon, getServerMetrics } from "./serverMetrics.js";
import { diskLimit, effectiveThresholds, parseThresholds } from "./serverThresholds.js";
import { getBackupCoverage, staleBackups } from "./backupCoverage.js";
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
import { createUptimeKumaAdapter } from "../integrations/uptimekuma/adapter.js";
import { createOsTicketAdapter } from "../integrations/osticket/adapter.js";
@@ -241,6 +243,25 @@ export async function collectAlerts(): Promise<AlertsReport> {
sourceFailures.forEach((f) => cannotCheckIntegration(f, "Proxmox backups"));
}),
// A guest whose newest backup is older than its limit — which Proxmox's own "latest run failed" alert can't see, because that only
// looks at the most recent run per node.
check("Backup coverage", async () => {
const now = new Date();
const report = await getBackupCoverage({ now, rules: true });
for (const s of staleBackups(report, now, subjects)) {
push({
category: "backup",
severity: s.severity,
type: "proxmox",
message: s.message,
key: s.key,
link: "/backups",
silenced: s.silenced,
target: s.guest.server ? targetOf("server", s.guest.server.id) : integrationTarget(s.guest.integrationId),
});
}
}),
check("Backup verification", async () => {
const { problems, sourceFailures } = await collectPbsProblems({ skipSilenced: false });
for (const p of problems) {
@@ -376,10 +397,13 @@ export async function collectAlerts(): Promise<AlertsReport> {
// Not a notification: a disk that is on course to fill is something to see coming, so it is shown here, from the server's own history.
check("Disk forecasts", async () => {
const { healthChecks } = await getSettings();
if (!(healthChecks.diskForecastDays > 0)) return;
for (const s of await db.select({ id: servers.id, name: servers.name }).from(servers)) {
for (const s of await db.select({ id: servers.id, name: servers.name, thresholds: servers.thresholds }).from(servers)) {
// A server may have its own forecast window (0 turns it off for that server), its own limit, and disks it is told to leave alone.
const own = parseThresholds(s.thresholds);
const forecastDays = effectiveThresholds(healthChecks, own).diskForecastDays;
if (!(forecastDays > 0)) continue;
const series = await getServerMetrics(s.id, 14);
for (const f of disksFillingSoon(series, healthChecks.diskForecastDays, healthChecks.diskUsagePercent)) {
for (const f of disksFillingSoon(series, forecastDays, (mount) => diskLimit(own, mount, healthChecks.diskUsagePercent))) {
push({
category: "disk",
severity: f.daysToFull <= 3 ? "critical" : "warning",
+82 -11
View File
@@ -14,6 +14,9 @@ import { db } from "../db/client.js";
import { integrations, servers } from "../db/schema.js";
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
import { createPbsAdapter, type PbsBackupGroup } from "../integrations/pbs/adapter.js";
import { parseThresholds } from "./serverThresholds.js";
import { activeSubjects } from "./maintenance.js";
import { getSettings } from "./settingsStore.js";
import { createProxmoxAdapter, guestsWithoutBackupCoverage, type ProxmoxBackupJob, type ProxmoxBackupTask, type ProxmoxGuest } from "../integrations/proxmox/adapter.js";
export type CoverageStatus = "ok" | "stale" | "failed" | "none" | "unknown";
@@ -36,6 +39,8 @@ export interface CoverageGuest {
status: CoverageStatus;
/** Why it is "failed", when it is. */
failure: string | null;
/** How old its newest backup may be before it counts as stale, in hours — the limit that applied to this guest (0 = never stale). */
staleAfterHours: number;
/** The server in this app that is this guest, if one is linked to it. */
server: { id: number; name: string } | null;
}
@@ -65,6 +70,8 @@ export interface LinkedServer {
name: string;
proxmoxIntegrationId: number | null;
proxmoxVmid: number | null;
/** This server's own "backup older than" limit, when it has one and the caller wants it used. */
staleHours?: number | null;
}
export interface CoverageReport {
@@ -81,7 +88,7 @@ export interface CoverageReport {
* with nothing to say when it last ran is "unknown".
*/
export function buildCoverage(input: { proxmox: ProxmoxSource[]; pbs: PbsSource[]; servers: LinkedServer[]; now: Date; staleHours?: number }): CoverageGuest[] {
const staleMs = (input.staleHours ?? DEFAULT_STALE_HOURS) * 3600_000;
const defaultHours = input.staleHours ?? DEFAULT_STALE_HOURS;
const nowMs = input.now.getTime();
// The newest snapshot of each guest, across every Backup Server (guests are told apart there by "vm/100" or "ct/101").
@@ -113,12 +120,14 @@ export function buildCoverage(input: { proxmox: ProxmoxSource[]; pbs: PbsSource[
else if (latestTask && !latestTask.ok && latestTask.status !== "running" && (!newest || Date.parse(latestTask.startTime) > newest.at)) failure = `the latest backup didn't succeed (${latestTask.status})`;
const inJob = !uncovered.has(g.vmid);
const linked = input.servers.find((s) => s.proxmoxIntegrationId === src.integrationId && s.proxmoxVmid === g.vmid);
// The server this guest is may have its own limit (a weekly backup is fine at 8 days); 0 means this one is never called stale.
const staleAfterHours = linked?.staleHours ?? defaultHours;
let status: CoverageStatus;
if (failure) status = "failed";
else if (newest) status = nowMs - newest.at > staleMs ? "stale" : "ok";
else if (newest) status = staleAfterHours > 0 && nowMs - newest.at > staleAfterHours * 3600_000 ? "stale" : "ok";
else status = inJob ? "unknown" : "none";
const linked = input.servers.find((s) => s.proxmoxIntegrationId === src.integrationId && s.proxmoxVmid === g.vmid);
out.push({
integrationId: src.integrationId,
integrationName: src.integrationName,
@@ -132,6 +141,7 @@ export function buildCoverage(input: { proxmox: ProxmoxSource[]; pbs: PbsSource[
lastSource: newest ? newest.source : null,
status,
failure,
staleAfterHours,
server: linked ? { id: linked.id, name: linked.name } : null,
});
}
@@ -205,15 +215,76 @@ async function loadSources(): Promise<Loaded> {
return loaded;
}
export async function getBackupCoverage(options: { staleHours?: number; refresh?: boolean; now?: Date } = {}): Promise<CoverageReport> {
/**
* The coverage report. `staleHours` is how old a backup may be before it counts as stale, for every guest — what the Backups page's own
* selector asks for. With `rules: true` the limit comes from Settings instead (`healthChecks.backupStaleHours`), and a server that has its
* own limit uses that for the guest it is — which is what the alerts, the daily message, the digest and /metrics use.
* `cachedOnly` never goes to Proxmox or Backup Server: it answers from what was last read, or null if nothing has been read yet.
*/
export async function getBackupCoverage(options: { staleHours?: number; refresh?: boolean; now?: Date; rules?: boolean; cachedOnly?: boolean } = {}): Promise<CoverageReport> {
const report = await readCoverage(options);
if (!report) throw new Error("no backup coverage has been read yet");
return report;
}
export async function getCachedBackupCoverage(now: Date = new Date()): Promise<CoverageReport | null> {
return readCoverage({ now, rules: true, cachedOnly: true });
}
async function readCoverage(options: { staleHours?: number; refresh?: boolean; now?: Date; rules?: boolean; cachedOnly?: boolean }): Promise<CoverageReport | null> {
const now = options.now ?? new Date();
const staleHours = options.staleHours ?? DEFAULT_STALE_HOURS;
if (!cache || options.refresh || now.getTime() - cache.at > CACHE_MS || loaderOverride) {
if (options.cachedOnly) {
if (!cache) return null;
} else if (!cache || options.refresh || now.getTime() - cache.at > CACHE_MS || loaderOverride) {
cache = { at: now.getTime(), loaded: await loadSources() };
}
const serverRows = await db.select({ id: servers.id, name: servers.name, proxmoxIntegrationId: servers.proxmoxIntegrationId, proxmoxVmid: servers.proxmoxVmid }).from(servers);
const guests = buildCoverage({ proxmox: cache.loaded.proxmox, pbs: cache.loaded.pbs, servers: serverRows, now, staleHours }).sort(
(a, b) => a.name.localeCompare(b.name) || a.vmid - b.vmid,
);
return { guests, counts: summarize(guests), sources: cache.loaded.sources, staleHours, generatedAt: now.toISOString() };
const loaded = cache!.loaded;
const staleHours = options.rules ? (await getSettings()).healthChecks.backupStaleHours : (options.staleHours ?? DEFAULT_STALE_HOURS);
const rows = await db.select({ id: servers.id, name: servers.name, proxmoxIntegrationId: servers.proxmoxIntegrationId, proxmoxVmid: servers.proxmoxVmid, thresholds: servers.thresholds }).from(servers);
const serverRows: LinkedServer[] = rows.map((r) => ({ id: r.id, name: r.name, proxmoxIntegrationId: r.proxmoxIntegrationId, proxmoxVmid: r.proxmoxVmid, staleHours: options.rules ? (parseThresholds(r.thresholds)?.backupStaleHours ?? null) : null }));
const guests = buildCoverage({ proxmox: loaded.proxmox, pbs: loaded.pbs, servers: serverRows, now, staleHours }).sort((a, b) => a.name.localeCompare(b.name) || a.vmid - b.vmid);
return { guests, counts: summarize(guests), sources: loaded.sources, staleHours, generatedAt: now.toISOString() };
}
export interface StaleBackup {
guest: CoverageGuest;
/** How long ago its newest backup was, in hours. */
ageHours: number;
severity: "critical" | "warning";
/** Stable for the same guest, so it alerts once. */
key: string;
/** In maintenance: on the server it is, or on the Proxmox integration it lives in. */
silenced: boolean;
message: string;
}
// Hours up to three days, days after that — so a 48-hour limit and a 60-hour-old backup read as "48 h" and "60 h", not both as "2 days".
const hoursWords = (h: number) => (h < 72 ? `${Math.round(h)} h` : `${Math.floor(h / 24)} days`);
/**
* Guests whose newest backup is older than their limit. A failed backup is not here: the latest failing run on a node, and snapshots that
* failed verification, already raise their own alerts, and a guest that goes on failing becomes stale once its limit passes. Critical at
* three times the limit.
*/
export function staleBackups(report: CoverageReport, now: Date, silencedSubjects: Set<string> = new Set()): StaleBackup[] {
const out: StaleBackup[] = [];
for (const g of report.guests) {
if (g.status !== "stale" || !g.lastBackupAt) continue;
const ageHours = (now.getTime() - Date.parse(g.lastBackupAt)) / 3600_000;
out.push({
guest: g,
ageHours,
severity: ageHours >= g.staleAfterHours * 3 ? "critical" : "warning",
key: `backup-stale:${g.integrationId}:${g.vmid}`,
silenced: silencedSubjects.has(`integration:${g.integrationId}`) || (g.server !== null && silencedSubjects.has(`server:${g.server.id}`)),
message: `${g.name} (#${g.vmid}) [${g.integrationName}]: its newest backup is ${hoursWords(ageHours)} old (the limit is ${hoursWords(g.staleAfterHours)})`,
});
}
return out.sort((a, b) => b.ageHours - a.ageHours);
}
/** What the daily check looks at: the stale guests that aren't under a maintenance window, read fresh from Proxmox and Backup Server. */
export async function collectStaleBackups(now: Date = new Date()): Promise<StaleBackup[]> {
const report = await getBackupCoverage({ now, rules: true, refresh: true });
return staleBackups(report, now, await activeSubjects(now)).filter((s) => !s.silenced);
}
+8 -3
View File
@@ -10,6 +10,7 @@ import { getInternalFlag, getSettings, setInternalFlag } from "./settingsStore.j
import { activeSubjects, subjectOfConditionKey } from "./maintenance.js";
import { coveredByParent, loadParentRows, resolveParents } from "./serverParents.js";
import { hostHealthConditions, parseHostHealth, type HostHealth } from "./hostHealth.js";
import { diskLimit, parseThresholds, type ServerThresholds } from "./serverThresholds.js";
const STATE_FLAG = "healthActiveConditions";
/** After a restart the agents haven't had a chance to report yet (they run every 15 min), so server-derived conditions are held rather than judged. */
@@ -35,6 +36,8 @@ export interface ServerSnapshot {
disks: { mount: string; sizeBytes: number; usedBytes: number }[];
/** What the agent says about failed services, disk health and ZFS pools, if it says anything. */
health?: HostHealth | null;
/** This server's own thresholds, where it has any: they replace the global ones for it. */
thresholds?: ServerThresholds | null;
}
export interface ProxmoxSnapshot {
integrationId: number;
@@ -106,7 +109,7 @@ export function evaluateHealth(snapshot: HealthSnapshot, thresholds: Thresholds,
let offline = false;
if (s.lastSeenAt) {
const age = now - parseTimestamp(s.lastSeenAt);
if (Number.isFinite(age) && age > thresholds.serverOfflineMinutes * 60_000) {
if (Number.isFinite(age) && age > (s.thresholds?.serverOfflineMinutes ?? thresholds.serverOfflineMinutes) * 60_000) {
offline = true;
out.push({
key: `offline:server:${s.id}`,
@@ -121,7 +124,9 @@ export function evaluateHealth(snapshot: HealthSnapshot, thresholds: Thresholds,
for (const c of hostHealthConditions(s.id, s.name, s.health ?? null)) out.push({ key: c.key, source: "server", message: c.message, severity: c.severity });
for (const d of s.disks) {
const p = usage(d.usedBytes, d.sizeBytes);
if (p !== null && p >= limit) {
// A disk can have its own limit, or be left alone altogether; otherwise the server's, then the global one.
const diskAlertAt = diskLimit(s.thresholds, d.mount, limit);
if (diskAlertAt !== null && p !== null && p >= diskAlertAt) {
out.push({
key: `disk:server:${s.id}:${d.mount}`,
source: "server",
@@ -300,7 +305,7 @@ export async function collectSnapshot(): Promise<{ snapshot: HealthSnapshot; hel
} catch {
// a malformed report shouldn't stop every other server being checked
}
snapshot.servers.push({ id: s.id, name: s.name, lastSeenAt: s.lastSeenAt, disks, health: parseHostHealth(s.hostHealth) });
snapshot.servers.push({ id: s.id, name: s.name, lastSeenAt: s.lastSeenAt, disks, health: parseHostHealth(s.hostHealth), thresholds: parseThresholds(s.thresholds) });
}
const rows = await db
+12 -1
View File
@@ -11,6 +11,8 @@ import { isAgentOutdated, latestAgentVersions } from "./agentVersions.js";
import { getSourceStatus } from "./integrationHealthMonitor.js";
import { parsePendingUpdates } from "./pendingUpdates.js";
import { parseHostHealth, poolIsFine } from "./hostHealth.js";
import { parseThresholds } from "./serverThresholds.js";
import { getCachedBackupCoverage } from "./backupCoverage.js";
import { disksFillingSoon, getServerMetrics } from "./serverMetrics.js";
import { computeSecretStatus } from "./secretStatus.js";
import { getSettings } from "./settingsStore.js";
@@ -58,8 +60,9 @@ export async function buildMetrics(now: Date = new Date()): Promise<string> {
// ── Servers ──
const latest = latestAgentVersions();
const serverRows = await db.select().from(servers);
const offlineAfterMs = settings.healthChecks.serverOfflineMinutes * 60_000;
for (const s of serverRows) {
// A server may have its own "offline after"; otherwise the global one.
const offlineAfterMs = (parseThresholds(s.thresholds)?.serverOfflineMinutes ?? settings.healthChecks.serverOfflineMinutes) * 60_000;
if (!s.lastSeenAt) continue; // never reported: nothing to say about it
const server = s.name;
const seen = Date.parse(s.lastSeenAt);
@@ -115,6 +118,14 @@ export async function buildMetrics(now: Date = new Date()): Promise<string> {
// ── Alerts ──
try {
const report = await getAlerts(false);
// What the backup check last found (the alert pass above refreshes it); never reads Proxmox or Backup Server itself.
const coverage = await getCachedBackupCoverage(now);
if (coverage) {
for (const status of ["ok", "stale", "failed", "none", "unknown"] as const) out.add("homelab_backup_guests", "gauge", "Proxmox guests by how well they are backed up (ok, stale, failed, none, unknown).", coverage.counts[status], { status });
for (const g of coverage.guests) {
if (g.lastBackupAt) out.add("homelab_backup_last_age_seconds", "gauge", "How long ago a guest's newest backup was.", Math.max(0, Math.floor((now.getTime() - Date.parse(g.lastBackupAt)) / 1000)), { guest: g.name, vmid: String(g.vmid), integration: g.integrationName });
}
}
for (const severity of ["critical", "warning", "info"] as const) out.add("homelab_alerts", "gauge", "Active alerts by severity (not counting silenced ones).", report.counts[severity], { severity });
out.add("homelab_alerts_silenced", "gauge", "Alerts held back by a maintenance window.", report.counts.silenced);
out.add("homelab_alerts_grouped", "gauge", "Alerts folded under a server that is down.", report.counts.grouped);
+17
View File
@@ -294,6 +294,23 @@ export async function notifyProxmoxUncoveredGuests(
);
}
export async function notifyStaleBackups(stale: { message: string }[]): Promise<void> {
if (stale.length === 0) return;
if (!(await eventEnabled("proxmoxBackupCheck"))) return;
await notify(
"Homelab Manager — Backups Gone Stale",
`${stale.length} guest${stale.length !== 1 ? "s have" : " has"} no recent backup:\n\n${stale.map((s) => s.message).join("\n")}`,
);
}
export async function notifyUpdateRunFinished(run: { serverName: string; templateName: string; ok: boolean; message: string }): Promise<void> {
if (!(await eventEnabled("updateRunResults"))) return;
await notify(
run.ok ? `Homelab Manager — ${run.serverName} updated` : `Homelab Manager — Update of ${run.serverName} didn't finish`,
`"${run.templateName}" on ${run.serverName}: ${run.message}`,
);
}
export async function notifyPbsVerificationFailed(
failures: { integrationName: string; datastore: string; failedCount: number; error: string | null }[],
): Promise<void> {
@@ -4,7 +4,8 @@ import { db } from "../db/client.js";
import { integrations } from "../db/schema.js";
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
import { createProxmoxAdapter, guestsWithoutBackupCoverage, type ProxmoxBackupTask } from "../integrations/proxmox/adapter.js";
import { notifyProxmoxBackupFailure, notifyProxmoxUncoveredGuests } from "./notify.js";
import { notifyProxmoxBackupFailure, notifyProxmoxUncoveredGuests, notifyStaleBackups } from "./notify.js";
import { collectStaleBackups } from "./backupCoverage.js";
import { getSettings, getInternalFlag, setInternalFlag } from "./settingsStore.js";
import { activeSubjects } from "./maintenance.js";
import type { SourceFailure } from "./alertTypes.js";
@@ -92,6 +93,13 @@ async function checkProxmoxBackups(): Promise<void> {
const { failures, uncovered } = await collectProxmoxBackupProblems({ skipSilenced: true });
await notifyProxmoxBackupFailure(failures);
await notifyProxmoxUncoveredGuests(uncovered);
// Guests whose newest backup has gone too old (the limit in Settings, or a server's own). A problem reading Proxmox or Backup Server
// here must not stop the other two messages, which have already gone.
try {
await notifyStaleBackups(await collectStaleBackups());
} catch (err) {
console.error("[proxmoxBackup] couldn't check for stale backups:", err);
}
}
async function checkProxmoxBackupsOnce(): Promise<void> {
+1 -1
View File
@@ -40,7 +40,7 @@ export async function buildPublicWall(detail: ShareLinkDetail, now: Date = new D
getAlerts(false),
getSettings(),
listActiveWindows(now),
db.select({ id: servers.id, name: servers.name, osType: servers.osType, lastSeenAt: servers.lastSeenAt, disks: servers.disks }).from(servers),
db.select({ id: servers.id, name: servers.name, osType: servers.osType, lastSeenAt: servers.lastSeenAt, disks: servers.disks, thresholds: servers.thresholds }).from(servers),
db.select({ id: integrations.id, name: integrations.name, type: integrations.type, enabled: integrations.enabled }).from(integrations),
activeSubjects(now),
]);
+13
View File
@@ -72,6 +72,19 @@ export function describeAudit(category: string, action: string, detail: Record<s
return { title: "Admin link removed", tone: "neutral", kind: "change", detail: label };
}
}
if (category === "update_run") {
const template = typeof detail.template === "string" ? detail.template : null;
switch (action) {
case "start":
return { title: "Update run started", tone: "info", kind: "change", detail: template ? `${template}, limited to ${detail.limit ?? "this server"}` : null };
case "finished":
return { title: "Update run finished", tone: "good", kind: "change", detail: message };
case "failed":
return { title: "Update run didn't finish", tone: "bad", kind: "change", detail: message };
case "cancelled":
return { title: "Update run stopped being followed", tone: "neutral", kind: "change", detail: message };
}
}
return { title: `${humanize(category)}: ${action.replace(/_/g, " ")}`, tone: "neutral", kind: "change", detail: message ?? label };
}
+7 -4
View File
@@ -140,17 +140,20 @@ export const FORECAST_MIN_FIT = 0.9;
/**
* The disks that are heading for full within `withinDays`. A disk already at or above `alreadyAlertedAt` percent is left out — the
* ordinary "disk nearly full" alert covers it — and so is one whose growth is erratic (a big copy, a log that rotates), where a
* straight-line forecast would just be a guess.
* straight-line forecast would just be a guess. `alreadyAlertedAt` may be a function of the disk, for servers with a limit of their own
* for one of them; returning null leaves that disk out altogether (it has been told not to alert).
*/
export function disksFillingSoon(series: MetricSeries[], withinDays: number, alreadyAlertedAt: number): DiskForecast[] {
export function disksFillingSoon(series: MetricSeries[], withinDays: number, alreadyAlertedAt: number | ((mount: string) => number | null)): DiskForecast[] {
const out: DiskForecast[] = [];
for (const s of series) {
if (s.kind !== "disk" || !s.trend || s.latest === null) continue;
const { perDay, fit, spanDays, daysToFull } = s.trend;
if (daysToFull === null || daysToFull > withinDays) continue;
if (spanDays < FORECAST_MIN_SPAN_DAYS || fit < FORECAST_MIN_FIT) continue;
if (s.latest >= alreadyAlertedAt) continue;
out.push({ mount: s.metric.slice("disk:".length), latest: s.latest, perDay, daysToFull });
const mount = s.metric.slice("disk:".length);
const limit = typeof alreadyAlertedAt === "function" ? alreadyAlertedAt(mount) : alreadyAlertedAt;
if (limit === null || s.latest >= limit) continue;
out.push({ mount, latest: s.latest, perDay, daysToFull });
}
return out.sort((a, b) => a.daysToFull - b.daysToFull);
}
+4 -1
View File
@@ -1,4 +1,5 @@
import { evaluateHealth, type Thresholds } from "./healthMonitor.js";
import { parseThresholds } from "./serverThresholds.js";
/** The columns of a server row the summary reads. */
export interface SummaryRow {
@@ -7,6 +8,8 @@ export interface SummaryRow {
osType: string;
lastSeenAt: string | null;
disks: string | null; // JSON, as the agent's report is stored
/** This server's own thresholds, as stored. */
thresholds?: string | null;
}
export interface ServerSummary {
@@ -40,7 +43,7 @@ function parseDisks(text: string | null): { mount: string; sizeBytes: number; us
*/
export function buildServerSummary(rows: SummaryRow[], thresholds: Thresholds, now: number, silencedSubjects: Set<string> = new Set()): ServerSummary {
const snapshot = {
servers: rows.map((r) => ({ id: r.id, name: r.name, lastSeenAt: r.lastSeenAt, disks: parseDisks(r.disks) })),
servers: rows.map((r) => ({ id: r.id, name: r.name, lastSeenAt: r.lastSeenAt, disks: parseDisks(r.disks), thresholds: parseThresholds(r.thresholds) })),
proxmox: [],
synology: [],
};
+124
View File
@@ -0,0 +1,124 @@
/**
* Per-server alert thresholds. The ones in Settings → Notifications apply to every server; some machines are different — a NAS that sits at
* 92% by design, a database host that should speak up at 80%, a laptop that is meant to be off at night. Each server may override the
* server-offline time, the disk-usage percentage, the disk forecast window and (for the Proxmox guest it is linked to) how old a backup may
* be, and may set a limit for one disk or say not to alert about that disk at all. Anything not overridden follows the global setting, so
* changing the global one still reaches every server that hasn't said otherwise.
*/
export interface MountThreshold {
/** Alert when this disk is this full (instead of the server's or the global percentage). */
percent?: number;
/** Never alert about this disk — neither when it is full nor when it is filling. */
ignore?: boolean;
}
export interface ServerThresholds {
serverOfflineMinutes?: number;
diskUsagePercent?: number;
/** 0 turns the "filling up" forecast off for this server. */
diskForecastDays?: number;
/** 0 turns the stale-backup alert off for this server's guest. */
backupStaleHours?: number;
mounts?: Record<string, MountThreshold>;
}
/** The global values these override. */
export interface GlobalThresholds {
serverOfflineMinutes: number;
diskUsagePercent: number;
diskForecastDays: number;
backupStaleHours: number;
}
export const LIMITS = {
serverOfflineMinutes: { min: 15, max: 10080 },
diskUsagePercent: { min: 50, max: 99 },
diskForecastDays: { min: 0, max: 90 },
backupStaleHours: { min: 0, max: 2160 },
} as const;
export const MAX_MOUNT_RULES = 50;
const inRange = (v: unknown, key: keyof typeof LIMITS): v is number => typeof v === "number" && Number.isInteger(v) && v >= LIMITS[key].min && v <= LIMITS[key].max;
/** What is stored, read back leniently: anything that doesn't make sense is dropped, never an error. Null for no overrides at all. */
export function parseThresholds(stored: string | null | undefined): ServerThresholds | null {
if (!stored) return null;
try {
const v = JSON.parse(stored);
if (!v || typeof v !== "object") return null;
const out: ServerThresholds = {};
for (const key of Object.keys(LIMITS) as (keyof typeof LIMITS)[]) if (inRange(v[key], key)) out[key] = v[key];
if (v.mounts && typeof v.mounts === "object" && !Array.isArray(v.mounts)) {
const mounts: Record<string, MountThreshold> = {};
for (const [mount, rule] of Object.entries(v.mounts as Record<string, unknown>).slice(0, MAX_MOUNT_RULES)) {
if (!rule || typeof rule !== "object") continue;
const r = rule as { percent?: unknown; ignore?: unknown };
const clean: MountThreshold = {};
if (inRange(r.percent, "diskUsagePercent")) clean.percent = r.percent;
if (r.ignore === true) clean.ignore = true;
if (clean.percent !== undefined || clean.ignore) mounts[mount] = clean;
}
if (Object.keys(mounts).length > 0) out.mounts = mounts;
}
return Object.keys(out).length > 0 ? out : null;
} catch {
return null;
}
}
/** What a request may say. Strict where parseThresholds is lenient: a wrong number is refused, with a message that says which. */
export function checkThresholds(input: unknown): { ok: true; value: ServerThresholds | null } | { ok: false; message: string } {
if (input === null) return { ok: true, value: null };
if (typeof input !== "object" || Array.isArray(input)) return { ok: false, message: "Thresholds should be an object." };
const v = input as Record<string, unknown>;
const out: ServerThresholds = {};
const names: Record<keyof typeof LIMITS, string> = {
serverOfflineMinutes: "Offline after (minutes)",
diskUsagePercent: "Disk usage alert at (%)",
diskForecastDays: "Disk forecast (days)",
backupStaleHours: "Backup older than (hours)",
};
for (const key of Object.keys(LIMITS) as (keyof typeof LIMITS)[]) {
if (v[key] === undefined || v[key] === null) continue;
if (!inRange(v[key], key)) return { ok: false, message: `${names[key]} must be a whole number from ${LIMITS[key].min} to ${LIMITS[key].max}.` };
out[key] = v[key] as number;
}
if (v.mounts !== undefined && v.mounts !== null) {
if (typeof v.mounts !== "object" || Array.isArray(v.mounts)) return { ok: false, message: "Disk rules should be an object." };
const entries = Object.entries(v.mounts as Record<string, unknown>);
if (entries.length > MAX_MOUNT_RULES) return { ok: false, message: `At most ${MAX_MOUNT_RULES} disks can have their own rule.` };
const mounts: Record<string, MountThreshold> = {};
for (const [mount, rule] of entries) {
if (mount.length === 0 || mount.length > 500) return { ok: false, message: "A disk's name is missing or too long." };
if (!rule || typeof rule !== "object" || Array.isArray(rule)) return { ok: false, message: `The rule for ${mount} should be an object.` };
const r = rule as { percent?: unknown; ignore?: unknown };
const clean: MountThreshold = {};
if (r.percent !== undefined && r.percent !== null) {
if (!inRange(r.percent, "diskUsagePercent")) return { ok: false, message: `The limit for ${mount} must be a whole number from ${LIMITS.diskUsagePercent.min} to ${LIMITS.diskUsagePercent.max}.` };
clean.percent = r.percent;
}
if (r.ignore === true) clean.ignore = true;
if (clean.percent !== undefined || clean.ignore) mounts[mount] = clean;
}
if (Object.keys(mounts).length > 0) out.mounts = mounts;
}
return { ok: true, value: Object.keys(out).length > 0 ? out : null };
}
/** What applies to one server: its own value where it gave one, the global one otherwise. */
export function effectiveThresholds(global: GlobalThresholds, own: ServerThresholds | null | undefined): GlobalThresholds {
return {
serverOfflineMinutes: own?.serverOfflineMinutes ?? global.serverOfflineMinutes,
diskUsagePercent: own?.diskUsagePercent ?? global.diskUsagePercent,
diskForecastDays: own?.diskForecastDays ?? global.diskForecastDays,
backupStaleHours: own?.backupStaleHours ?? global.backupStaleHours,
};
}
/** The usage percentage at which a disk alerts — its own rule first, then the server's, then the global one — or null if it is to be left alone. */
export function diskLimit(own: ServerThresholds | null | undefined, mount: string, globalPercent: number): number | null {
const rule = own?.mounts?.[mount];
if (rule?.ignore) return null;
return rule?.percent ?? own?.diskUsagePercent ?? globalPercent;
}
+6 -1
View File
@@ -50,6 +50,8 @@ export interface NotificationEvents {
healthAlerts: boolean;
/** A Semaphore template or Gitea repo whose latest run failed, and when it succeeds again. */
automationAlerts: boolean;
/** How an update run started from the Updates page ended: what is still waiting, whether a restart is due, or why it failed. */
updateRunResults: boolean;
/** Daily reminder while a domain registration is close to expiring or has expired. */
domainExpiryCheck: boolean;
secretCheckTime: string; // "HH:MM" — shared by the secret-expiry, Tailscale key-expiry, Docker update, and Proxmox backup checks
@@ -97,6 +99,8 @@ export interface HealthCheckSettings {
domainWarnDays: number;
/** An agent-reported disk that is on course to be full within this many days is flagged on the Alerts page. 0 turns it off. */
diskForecastDays: number;
/** A Proxmox guest whose newest backup is older than this many hours is raised as stale. 0 turns it off. */
backupStaleHours: number;
}
/** The weekly digest: one message a week through the notification channels. Off until an admin switches it on under Settings → Notifications. */
@@ -208,6 +212,7 @@ const DEFAULTS: AppSettings = {
pbsVerificationCheck: true,
healthAlerts: true,
automationAlerts: true,
updateRunResults: true,
domainExpiryCheck: true,
secretCheckTime: "08:00",
timezone: "UTC",
@@ -223,7 +228,7 @@ const DEFAULTS: AppSettings = {
inventory: { currency: "", pricePerKwh: null },
agents: { autoUpdate: false },
signIn: { roleFromGroups: false, groupsClaim: "groups", adminGroups: [], operatorGroups: [], otherwise: "viewer" },
healthChecks: { serverOfflineMinutes: 60, diskUsagePercent: 90, domainWarnDays: 30, diskForecastDays: 14 },
healthChecks: { serverOfflineMinutes: 60, diskUsagePercent: 90, domainWarnDays: 30, diskForecastDays: 14, backupStaleHours: 48 },
// Docker's default bridge (172.17.0.0/16) and the pool it hands custom networks from (172.18–31) live here, and every
// Docker host repeats them. Shown on the Consistency page, where it can be removed if 172.16/12 is used as a real LAN.
consistency: { excludedRanges: ["172.16.0.0/12"] },
+2 -1
View File
@@ -10,6 +10,7 @@ import { dnsRecordsCache, dnsZonesCache, integrations, servers } from "../db/sch
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
import { createUptimeKumaAdapter } from "../integrations/uptimekuma/adapter.js";
import { getSettings } from "./settingsStore.js";
import { parseThresholds } from "./serverThresholds.js";
import { attachMatchedServers, toMatchableServers } from "./uptimeKumaMatch.js";
export type MapStatus = "ok" | "bad" | "warn" | "unknown";
@@ -64,7 +65,6 @@ export async function buildTopology(now: Date = new Date()): Promise<MapData> {
const notes: string[] = [];
const { healthChecks } = await getSettings();
const offlineAfterMs = healthChecks.serverOfflineMinutes * 60_000;
const serverRows = await db.select().from(servers);
const integrationRows = await db.select({ id: integrations.id, name: integrations.name, type: integrations.type, enabled: integrations.enabled }).from(integrations);
const integrationName = new Map(integrationRows.map((i) => [i.id, i.name]));
@@ -73,6 +73,7 @@ export async function buildTopology(now: Date = new Date()): Promise<MapData> {
const hostIds = new Set<string>();
for (const s of serverRows) {
const lastSeen = s.lastSeenAt ? Date.parse(s.lastSeenAt) : null;
const offlineAfterMs = (parseThresholds(s.thresholds)?.serverOfflineMinutes ?? healthChecks.serverOfflineMinutes) * 60_000;
const status: MapStatus = lastSeen === null ? "unknown" : now.getTime() - lastSeen > offlineAfterMs ? "bad" : "ok";
nodes.push({ id: `server:${s.id}`, kind: "server", label: s.name, sub: s.hostname, status, href: `/servers/${s.id}` });
if (s.proxmoxIntegrationId && s.proxmoxNode) {
+271
View File
@@ -0,0 +1,271 @@
/**
* An update run: apply a server's waiting updates in one go, and see it through. It puts the server under maintenance (so nothing alerts
* while it is being worked on and restarted), starts a Semaphore playbook limited to that server, follows the run until it is over, waits
* for the server's agent to report again, then says how many updates are still waiting and whether a restart is due, and ends the
* maintenance window it made. Every step is written down on the run, which survives a restart of this app — a poller picks up where it was.
*
* What it does not do: it does not restart the server (the playbook does, if it is written to), and it cannot stop a run in Semaphore —
* cancelling here only stops watching and ends the window.
*/
import { and, desc, eq, inArray } from "drizzle-orm";
import { db } from "../db/client.js";
import { maintenanceWindows, servers, updateRuns } from "../db/schema.js";
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
import { createSemaphoreAdapter, type SemaphoreAdapter, type SemaphoreTask } from "../integrations/semaphore/adapter.js";
import { invalidateAlertsCache } from "./alerts.js";
import { recordAudit } from "./audit.js";
import { startOrExtendWindow } from "./maintenance.js";
import { notifyUpdateRunFinished } from "./notify.js";
import { parsePendingUpdates } from "./pendingUpdates.js";
export type UpdateRunRow = typeof updateRuns.$inferSelect;
export type UpdateRunStatus = "running" | "waiting_for_host" | "done" | "failed" | "cancelled";
export const ACTIVE_STATUSES: UpdateRunStatus[] = ["running", "waiting_for_host"];
/** How long a run may take from start to finish before it is given up on. */
export const RUN_LIMIT_MS = 3 * 3600_000;
/** After the playbook is over, how long the server has to report again (it may be restarting). */
export const HOST_WAIT_MS = 30 * 60_000;
/** Consecutive polls Semaphore may fail to answer before the run is given up on (at the poller's pace, about five minutes). */
export const MAX_POLL_ERRORS = 10;
export const POLL_INTERVAL_MS = 20_000;
const FINISHED_TASK = new Set<SemaphoreTask["status"]>(["success", "error", "stopped", "rejected"]);
// ─── Semaphore, replaceable for the tests ───────────────────────────────────
let adapterOverride: ((integrationId: number) => Promise<SemaphoreAdapter | null>) | null = null;
export function setSemaphoreForTests(factory: ((integrationId: number) => Promise<SemaphoreAdapter | null>) | null): void {
adapterOverride = factory;
}
async function semaphoreFor(integrationId: number): Promise<SemaphoreAdapter | null> {
if (adapterOverride) return adapterOverride(integrationId);
const loaded = await loadIntegrationConfig(integrationId);
if (!loaded || loaded.integration.type !== "semaphore" || !loaded.integration.enabled) return null;
return createSemaphoreAdapter(loaded.config as any);
}
// ─── Starting ───────────────────────────────────────────────────────────────
export class UpdateRunError extends Error {
constructor(
readonly status: number,
readonly code: string,
message: string,
) {
super(message);
}
}
export interface StartInput {
serverId: number;
integrationId: number;
projectId: number;
templateId: number;
/** Which hosts of the inventory the playbook may touch (ansible's --limit). Never empty: an update run is always for this one server. */
limit: string;
/** How long the maintenance window lasts at most; it ends sooner when the run does. */
windowMinutes: number;
actor: string;
}
export async function startUpdateRun(input: StartInput, now: Date = new Date()): Promise<UpdateRunRow> {
const [server] = await db.select().from(servers).where(eq(servers.id, input.serverId)).limit(1);
if (!server) throw new UpdateRunError(404, "server_not_found", "That server doesn't exist.");
if (!server.lastSeenAt) throw new UpdateRunError(400, "never_reported", "This server has never reported, so there is nothing to wait for after the run.");
const [busy] = await db
.select({ id: updateRuns.id })
.from(updateRuns)
.where(and(eq(updateRuns.serverId, server.id), inArray(updateRuns.status, ACTIVE_STATUSES)))
.limit(1);
if (busy) throw new UpdateRunError(409, "already_running", `An update run for ${server.name} is already under way.`);
const adapter = await semaphoreFor(input.integrationId);
if (!adapter) throw new UpdateRunError(400, "no_semaphore", "That Semaphore integration doesn't exist or isn't switched on.");
let templateName: string;
try {
const { templates } = await adapter.checkTemplates();
const template = templates.find((t) => t.projectId === input.projectId && t.id === input.templateId);
if (!template) throw new UpdateRunError(404, "template_not_found", "That template isn't in Semaphore (any more).");
templateName = template.name;
} catch (err) {
if (err instanceof UpdateRunError) throw err;
throw new UpdateRunError(502, "semaphore_unreachable", `Couldn't reach Semaphore: ${err instanceof Error ? err.message : String(err)}`);
}
// Quiet first, so nothing alerts about the server being worked on — and if the playbook can't be started the quiet is taken back.
const started = await startOrExtendWindow("server", server.id, input.windowMinutes, `Update run: ${templateName}`, input.actor, now);
let task: SemaphoreTask;
try {
task = await adapter.runTemplate(input.projectId, input.templateId, { limit: input.limit, message: `Update run for ${server.name}, started in Homelab Manager by ${input.actor} (limited to ${input.limit})` });
} catch (err) {
if (started && !started.extended) await db.update(maintenanceWindows).set({ endsAt: now.toISOString() }).where(eq(maintenanceWindows.id, started.window.id));
invalidateAlertsCache();
throw new UpdateRunError(502, "semaphore_failed", `Semaphore wouldn't start it: ${err instanceof Error ? err.message : String(err)}`);
}
invalidateAlertsCache();
const updates = parsePendingUpdates(server.pendingUpdates);
const [run] = await db
.insert(updateRuns)
.values({
serverId: server.id,
integrationId: input.integrationId,
projectId: input.projectId,
templateId: input.templateId,
templateName,
hostLimit: input.limit,
taskId: task.id,
taskStatus: task.status,
windowId: started?.window.id ?? null,
windowCreated: started ? !started.extended : false,
status: "running",
message: "The playbook has been started.",
startedBy: input.actor,
startedAt: now.toISOString(),
updatesBefore: updates?.count ?? null,
})
.returning();
await recordAudit({ category: "update_run", action: "start", targetType: "server", targetId: server.id, detail: { run: run.id, template: templateName, limit: input.limit, task: task.id, by: input.actor } });
return run;
}
// ─── Following ──────────────────────────────────────────────────────────────
/** Ends the maintenance window an update run made (one it only extended belongs to whoever made it, and is left to run out). */
async function endWindow(run: UpdateRunRow, now: Date): Promise<void> {
if (!run.windowCreated || run.windowId === null) return;
await db
.update(maintenanceWindows)
.set({ endsAt: now.toISOString() })
.where(and(eq(maintenanceWindows.id, run.windowId)));
invalidateAlertsCache();
}
async function finish(run: UpdateRunRow, status: "done" | "failed" | "cancelled", message: string, now: Date, extra: Partial<UpdateRunRow> = {}): Promise<void> {
await endWindow(run, now);
await db.update(updateRuns).set({ status, message, finishedAt: now.toISOString(), ...extra }).where(eq(updateRuns.id, run.id));
const [server] = await db.select({ name: servers.name }).from(servers).where(eq(servers.id, run.serverId)).limit(1);
await recordAudit({ category: "update_run", action: status === "done" ? "finished" : status, targetType: "server", targetId: run.serverId, detail: { run: run.id, template: run.templateName, message } });
if (status !== "cancelled") {
await notifyUpdateRunFinished({ serverName: server?.name ?? `#${run.serverId}`, templateName: run.templateName, ok: status === "done", message }).catch((err) => console.error("[updateRuns] couldn't send the result:", err));
}
}
const updatesWords = (n: number) => (n === 1 ? "1 update" : `${n} updates`);
/** What a finished run says about the server, from the report that came after it. */
export function summarizeOutcome(before: number | null, after: number | null, reboot: boolean): string {
const parts: string[] = [];
if (after !== null) {
parts.push(after === 0 ? "No updates are waiting now" : `${updatesWords(after)} still waiting`);
if (before !== null && before > after) parts[0] += ` (${updatesWords(before - after)} installed)`;
} else {
parts.push("The server reported again, but doesn't say what is waiting");
}
parts.push(reboot ? "a restart is still needed" : "no restart is needed");
return `${parts.join("; ")}.`;
}
async function advance(run: UpdateRunRow, now: Date): Promise<void> {
if (now.getTime() - Date.parse(run.startedAt) > RUN_LIMIT_MS) {
await finish(run, "failed", "It took longer than three hours, so it is no longer being followed. Check Semaphore.", now);
return;
}
if (run.status === "running") {
const adapter = await semaphoreFor(run.integrationId);
let task: SemaphoreTask | null = null;
try {
task = adapter && run.taskId !== null ? await adapter.getTask(run.projectId, run.taskId) : null;
} catch {
task = null;
}
if (!task) {
const errors = run.pollErrors + 1;
if (errors >= MAX_POLL_ERRORS) await finish(run, "failed", "Lost contact with Semaphore, so it is no longer being followed. Check Semaphore.", now, { pollErrors: errors });
else await db.update(updateRuns).set({ pollErrors: errors }).where(eq(updateRuns.id, run.id));
return;
}
if (!FINISHED_TASK.has(task.status)) {
if (task.status !== run.taskStatus || run.pollErrors > 0) await db.update(updateRuns).set({ taskStatus: task.status, pollErrors: 0 }).where(eq(updateRuns.id, run.id));
return;
}
if (task.status === "success") {
const ended = task.end ? new Date(task.end).toISOString() : now.toISOString();
await db.update(updateRuns).set({ status: "waiting_for_host", taskStatus: task.status, taskEndedAt: ended, pollErrors: 0, message: "The playbook finished. Waiting for the server to report again." }).where(eq(updateRuns.id, run.id));
return;
}
const why = task.status === "error" ? "The playbook failed in Semaphore" : task.status === "rejected" ? "The run was rejected in Semaphore" : "The run was stopped in Semaphore";
await finish(run, "failed", `${why} (run #${run.taskId}). The server is no longer under maintenance.`, now, { taskStatus: task.status });
return;
}
// waiting_for_host: the agent has reported since the playbook ended (and, where it reports updates, those are newer than the run too).
const ended = Date.parse(run.taskEndedAt ?? run.startedAt);
const [server] = await db.select().from(servers).where(eq(servers.id, run.serverId)).limit(1);
if (!server) {
await finish(run, "failed", "The server was removed.", now);
return;
}
const reported = server.lastSeenAt ? Date.parse(server.lastSeenAt) > ended : false;
const updates = parsePendingUpdates(server.pendingUpdates);
const updatesFresh = updates ? Date.parse(updates.reportedAt) > ended : true;
if (reported && updatesFresh) {
await finish(run, "done", summarizeOutcome(run.updatesBefore, updates?.count ?? null, updates?.rebootRequired ?? false), now, {
hostReportedAt: server.lastSeenAt,
updatesAfter: updates?.count ?? null,
rebootRequired: updates?.rebootRequired ?? false,
});
return;
}
if (now.getTime() - ended > HOST_WAIT_MS) {
await finish(run, "failed", `The playbook finished, but the server hasn't reported again in ${Math.round(HOST_WAIT_MS / 60_000)} minutes. Check that it came back.`, now);
}
}
/** One pass over every run that isn't over. Returns how many it looked at (for the tests). */
export async function advanceUpdateRuns(now: Date = new Date()): Promise<number> {
const active = await db.select().from(updateRuns).where(inArray(updateRuns.status, ACTIVE_STATUSES));
for (const run of active) {
try {
await advance(run, now);
} catch (err) {
console.error(`[updateRuns] run ${run.id}:`, err);
}
}
return active.length;
}
export async function cancelUpdateRun(id: number, actor: string, now: Date = new Date()): Promise<UpdateRunRow | null> {
const [run] = await db.select().from(updateRuns).where(eq(updateRuns.id, id)).limit(1);
if (!run) return null;
if (!ACTIVE_STATUSES.includes(run.status as UpdateRunStatus)) throw new UpdateRunError(409, "not_active", "That run is already over.");
await finish(run, "cancelled", `Stopped being followed by ${actor}. A playbook already started keeps running in Semaphore.`, now);
const [after] = await db.select().from(updateRuns).where(eq(updateRuns.id, id)).limit(1);
return after;
}
/** Recent runs, those still going first; for one server or for all. */
export async function listUpdateRuns(options: { serverId?: number; limit?: number } = {}): Promise<(UpdateRunRow & { serverName: string | null })[]> {
const rows = await db
.select({ run: updateRuns, serverName: servers.name })
.from(updateRuns)
.leftJoin(servers, eq(servers.id, updateRuns.serverId))
.where(options.serverId !== undefined ? eq(updateRuns.serverId, options.serverId) : undefined)
.orderBy(desc(updateRuns.startedAt))
.limit(options.limit ?? 30);
const out = rows.map((r) => ({ ...r.run, serverName: r.serverName }));
return [...out.filter((r) => ACTIVE_STATUSES.includes(r.status as UpdateRunStatus)), ...out.filter((r) => !ACTIVE_STATUSES.includes(r.status as UpdateRunStatus))];
}
let timer: ReturnType<typeof setInterval> | null = null;
/** Follows runs that are under way, every twenty seconds — including ones that were under way when this app last stopped. */
export function initUpdateRuns(): void {
if (timer) return;
timer = setInterval(() => void advanceUpdateRuns().catch((err) => console.error("[updateRuns]", err)), POLL_INTERVAL_MS);
}
+9 -5
View File
@@ -17,6 +17,7 @@ import { notify, notifyNow } from "./notify.js";
import { parsePendingUpdates } from "./pendingUpdates.js";
import { computeSecretStatus } from "./secretStatus.js";
import { disksFillingSoon, getServerMetrics } from "./serverMetrics.js";
import { diskLimit, effectiveThresholds, parseThresholds } from "./serverThresholds.js";
import { getSettings } from "./settingsStore.js";
import { getUptime } from "./uptimeHistory.js";
@@ -68,7 +69,7 @@ export async function buildDigest(now: Date = new Date()): Promise<Digest> {
const sections: Section[] = [];
const serverRows = await db
.select({ id: servers.id, name: servers.name, lastSeenAt: servers.lastSeenAt, pendingUpdates: servers.pendingUpdates, updatesPendingSince: servers.updatesPendingSince })
.select({ id: servers.id, name: servers.name, lastSeenAt: servers.lastSeenAt, pendingUpdates: servers.pendingUpdates, updatesPendingSince: servers.updatesPendingSince, thresholds: servers.thresholds })
.from(servers);
const serverName = new Map(serverRows.map((s) => [s.id, s.name]));
@@ -134,11 +135,14 @@ export async function buildDigest(now: Date = new Date()): Promise<Digest> {
// ── Disks heading for full ──
try {
const { diskForecastDays, diskUsagePercent } = settings.healthChecks;
if (diskForecastDays > 0) {
const { diskUsagePercent } = settings.healthChecks;
{
const lines: string[] = [];
for (const s of serverRows) {
for (const f of disksFillingSoon(await getServerMetrics(s.id, 14, now), diskForecastDays, diskUsagePercent)) {
const own = parseThresholds(s.thresholds);
const forecastDays = effectiveThresholds(settings.healthChecks, own).diskForecastDays;
if (!(forecastDays > 0)) continue;
for (const f of disksFillingSoon(await getServerMetrics(s.id, 14, now), forecastDays, (mount) => diskLimit(own, mount, diskUsagePercent))) {
lines.push(`- ${s.name}: ${f.mount} is ${Math.round(f.latest)}% full and filling by about ${f.perDay.toFixed(1)} points a day — full in about ${plural(f.daysToFull, "day")}`);
}
}
@@ -196,7 +200,7 @@ export async function buildDigest(now: Date = new Date()): Promise<Digest> {
// ── Backups ──
try {
const coverage = await getBackupCoverage({ now });
const coverage = await getBackupCoverage({ now, rules: true });
const order = { failed: 0, none: 1, stale: 2 } as const;
const bad = coverage.guests.filter((g): g is typeof g & { status: keyof typeof order } => g.status === "failed" || g.status === "none" || g.status === "stale").sort((a, b) => order[a.status] - order[b.status] || a.name.localeCompare(b.name));
if (bad.length > 0) {