Stale-backup alerts, per-server alert limits, and update runs from the Updates page
- Backups: a guest whose newest backup is older than its limit is an alert (backup kind), in the daily backup message, the digest and /metrics (homelab_backup_guests, ..._last_age_seconds) - Per-server thresholds (servers.thresholds, migration 0030): offline minutes, disk percent, forecast window, backup age, and per-disk limits or "leave alone"; applied everywhere the global ones were; saving clears the alerts cache - Update runs (update_runs, migration 0031): Update... on the Updates page puts the server under maintenance, runs a Semaphore playbook limited to it, waits for the host to report, and reports what is left and whether a restart is due; poller survives restarts - Notification toggle for update-run results; tests and docs Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
5dc6cea69c
commit
7a5b623444
42 files changed
+7561
-57
No files matched your search
@@ -0,0 +1 @@
|
||||
ALTER TABLE `servers` ADD `thresholds` text;
|
||||
@@ -0,0 +1,28 @@
|
||||
CREATE TABLE `update_runs` (
|
||||
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||
`server_id` integer NOT NULL,
|
||||
`integration_id` integer NOT NULL,
|
||||
`project_id` integer NOT NULL,
|
||||
`template_id` integer NOT NULL,
|
||||
`template_name` text NOT NULL,
|
||||
`host_limit` text NOT NULL,
|
||||
`task_id` integer,
|
||||
`task_status` text,
|
||||
`window_id` integer,
|
||||
`window_created` integer DEFAULT false NOT NULL,
|
||||
`status` text NOT NULL,
|
||||
`message` text,
|
||||
`started_by` text,
|
||||
`started_at` text NOT NULL,
|
||||
`task_ended_at` text,
|
||||
`host_reported_at` text,
|
||||
`finished_at` text,
|
||||
`updates_before` integer,
|
||||
`updates_after` integer,
|
||||
`reboot_required` integer DEFAULT false NOT NULL,
|
||||
`poll_errors` integer DEFAULT 0 NOT NULL,
|
||||
FOREIGN KEY (`server_id`) REFERENCES `servers`(`id`) ON UPDATE no action ON DELETE cascade
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE INDEX `update_runs_server_idx` ON `update_runs` (`server_id`,`started_at`);--> statement-breakpoint
|
||||
CREATE INDEX `update_runs_status_idx` ON `update_runs` (`status`);
|
||||
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
@@ -211,6 +211,20 @@
|
||||
"when": 1792273577264,
|
||||
"tag": "0029_past_monster_badoon",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 30,
|
||||
"version": "6",
|
||||
"when": 1792278520773,
|
||||
"tag": "0030_icy_snowbird",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 31,
|
||||
"version": "6",
|
||||
"when": 1792278718505,
|
||||
"tag": "0031_sad_lizard",
|
||||
"breakpoints": true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -11,6 +11,7 @@ import { meRouter } from "./routes/me.js";
|
||||
import { uptimeRouter } from "./routes/uptime.js";
|
||||
import { shareLinksRouter } from "./routes/shareLinks.js";
|
||||
import { runbooksRouter } from "./routes/runbooks.js";
|
||||
import { updateRunsRouter } from "./routes/updateRuns.js";
|
||||
import { inventoryRouter } from "./routes/inventory.js";
|
||||
import { backupsRouter } from "./routes/backups.js";
|
||||
import { metricsRouter, metricsTokensRouter } from "./routes/metrics.js";
|
||||
@@ -98,6 +99,7 @@ export function createApp() {
|
||||
app.use("/api/uptime", uptimeRouter);
|
||||
app.use("/api/share-links", shareLinksRouter);
|
||||
app.use("/api/runbooks", runbooksRouter);
|
||||
app.use("/api/update-runs", updateRunsRouter);
|
||||
app.use("/api/inventory", inventoryRouter);
|
||||
app.use("/api/backups", backupsRouter);
|
||||
app.use("/api/metrics-tokens", metricsTokensRouter);
|
||||
|
||||
@@ -300,6 +300,10 @@ export const servers = sqliteTable("servers", {
|
||||
// What the agent says about the machine's own health, as JSON: {failedUnits, smart, zfs, reportedAt} — see services/hostHealth.ts. Null until it says.
|
||||
hostHealth: text("host_health"),
|
||||
|
||||
// This server's own alert thresholds, as JSON: {serverOfflineMinutes, diskUsagePercent, diskForecastDays, backupStaleHours, mounts: {"/data": {percent, ignore}}} — see
|
||||
// services/serverThresholds.ts. Whatever isn't there follows Settings. Null when it follows Settings entirely.
|
||||
thresholds: text("thresholds"),
|
||||
|
||||
// Which version of the agent script is running on it, as the agent says in each report (see agent/*/report-tasks.*). Null until it says:
|
||||
// an agent that predates the number being reported is version 1.
|
||||
agentVersion: integer("agent_version"),
|
||||
@@ -568,6 +572,41 @@ export const runbooks = sqliteTable(
|
||||
(t) => [index("runbooks_target_idx").on(t.targetType, t.targetId)],
|
||||
);
|
||||
|
||||
// ─── Update runs: apply a server's updates and see it through ──────────────
|
||||
|
||||
// A playbook run in Semaphore for one server, wrapped in a maintenance window and followed until the server reports again — see
|
||||
// services/updateRuns.ts. status: running | waiting_for_host | done | failed | cancelled.
|
||||
export const updateRuns = sqliteTable(
|
||||
"update_runs",
|
||||
{
|
||||
id: integer("id").primaryKey({ autoIncrement: true }),
|
||||
serverId: integer("server_id")
|
||||
.notNull()
|
||||
.references(() => servers.id, { onDelete: "cascade" }),
|
||||
integrationId: integer("integration_id").notNull(), // the Semaphore integration; no FK — one removed later leaves the history readable
|
||||
projectId: integer("project_id").notNull(),
|
||||
templateId: integer("template_id").notNull(),
|
||||
templateName: text("template_name").notNull(),
|
||||
hostLimit: text("host_limit").notNull(), // ansible's --limit
|
||||
taskId: integer("task_id"),
|
||||
taskStatus: text("task_status"), // Semaphore's own word for the run
|
||||
windowId: integer("window_id"), // the maintenance window put round it
|
||||
windowCreated: integer("window_created", { mode: "boolean" }).notNull().default(false), // false if it only extended one that was already there (which is then left alone)
|
||||
status: text("status").notNull(),
|
||||
message: text("message"),
|
||||
startedBy: text("started_by"),
|
||||
startedAt: text("started_at").notNull(), // ISO
|
||||
taskEndedAt: text("task_ended_at"), // ISO — when the playbook finished
|
||||
hostReportedAt: text("host_reported_at"), // ISO — the report that came after it
|
||||
finishedAt: text("finished_at"), // ISO
|
||||
updatesBefore: integer("updates_before"),
|
||||
updatesAfter: integer("updates_after"),
|
||||
rebootRequired: integer("reboot_required", { mode: "boolean" }).notNull().default(false),
|
||||
pollErrors: integer("poll_errors").notNull().default(0),
|
||||
},
|
||||
(t) => [index("update_runs_server_idx").on(t.serverId, t.startedAt), index("update_runs_status_idx").on(t.status)],
|
||||
);
|
||||
|
||||
// ─── Inventory: the hardware behind it all ──────────────────────────────────
|
||||
|
||||
export const assetKinds = ["server", "storage", "network", "power", "client", "other"] as const;
|
||||
|
||||
@@ -16,6 +16,7 @@ import { initHealthScheduler } from "./services/healthScheduler.js";
|
||||
import { initAlertTicketScheduler } from "./services/alertTicketScheduler.js";
|
||||
import { initKumaMaintenanceScheduler } from "./services/kumaMaintenanceScheduler.js";
|
||||
import { initMaintenanceScheduleScheduler } from "./services/maintenanceScheduleScheduler.js";
|
||||
import { initUpdateRuns } from "./services/updateRuns.js";
|
||||
|
||||
{
|
||||
// Sessions are signed with this secret. Without one, the app would sign them with a value that's in the source code.
|
||||
@@ -48,6 +49,7 @@ await initHealthScheduler();
|
||||
await initAlertTicketScheduler();
|
||||
await initKumaMaintenanceScheduler();
|
||||
await initMaintenanceScheduleScheduler();
|
||||
initUpdateRuns();
|
||||
|
||||
createApp().listen(env.port, () => {
|
||||
console.log(`homelab-manager listening on port ${env.port}`);
|
||||
|
||||
@@ -9,6 +9,7 @@ import { recordAudit } from "../services/audit.js";
|
||||
import { loadParentRows, resolveParents, wouldLoop } from "../services/serverParents.js";
|
||||
import { parsePendingUpdates } from "../services/pendingUpdates.js";
|
||||
import { parseHostHealth } from "../services/hostHealth.js";
|
||||
import { checkThresholds, parseThresholds } from "../services/serverThresholds.js";
|
||||
import { latestAgentVersions } from "../services/agentVersions.js";
|
||||
import { getServerMetrics, RETENTION_DAYS } from "../services/serverMetrics.js";
|
||||
import { getServerActivity } from "../services/serverActivity.js";
|
||||
@@ -20,14 +21,15 @@ import { serverPortsRouter } from "./serverPorts.js";
|
||||
import { InvalidTagError, normalizeTags, parseTags } from "../services/serverTags.js";
|
||||
import { buildServerSummary } from "../services/serverSummary.js";
|
||||
import { activeSubjects } from "../services/maintenance.js";
|
||||
import { invalidateAlertsCache } from "../services/alerts.js";
|
||||
import { getSettings, updateSettings } from "../services/settingsStore.js";
|
||||
|
||||
export const serversRouter = Router();
|
||||
|
||||
/** A server row as the API returns it: no token hash, and tags as an array rather than the stored JSON. */
|
||||
function publicServer<T extends { apiTokenHash: string; tags: string | null; pendingUpdates: string | null; hostHealth: string | null }>(row: T) {
|
||||
const { apiTokenHash: _hash, tags, pendingUpdates, hostHealth, ...rest } = row;
|
||||
return { ...rest, tags: parseTags(tags), pendingUpdates: parsePendingUpdates(pendingUpdates), hostHealth: parseHostHealth(hostHealth) };
|
||||
function publicServer<T extends { apiTokenHash: string; tags: string | null; pendingUpdates: string | null; hostHealth: string | null; thresholds: string | null }>(row: T) {
|
||||
const { apiTokenHash: _hash, tags, pendingUpdates, hostHealth, thresholds, ...rest } = row;
|
||||
return { ...rest, tags: parseTags(tags), pendingUpdates: parsePendingUpdates(pendingUpdates), hostHealth: parseHostHealth(hostHealth), thresholds: parseThresholds(thresholds) };
|
||||
}
|
||||
|
||||
serversRouter.use(requireAuth);
|
||||
@@ -135,6 +137,8 @@ const updateServerSchema = z
|
||||
parentServerId: z.number().int().nullable().optional(),
|
||||
// True says it runs on nothing, which also stops the Proxmox-node guess. Choosing a server, or null, turns it off again.
|
||||
noParent: z.boolean().optional(),
|
||||
// This server's own alert thresholds (null goes back to following Settings). Checked on its own below, so a wrong number gets a message that says which.
|
||||
thresholds: z.unknown().optional(),
|
||||
})
|
||||
.refine(
|
||||
(data) => {
|
||||
@@ -181,10 +185,20 @@ serversRouter.patch("/:id", requireRole("admin"), asyncHandler(async (req, res)
|
||||
}
|
||||
}
|
||||
|
||||
const changes: typeof parsed.data = { ...parsed.data };
|
||||
const { thresholds: requestedThresholds, ...fields } = parsed.data;
|
||||
const changes: typeof fields & { thresholds?: string | null } = { ...fields };
|
||||
if (changes.noParent === true) changes.parentServerId = null;
|
||||
else if (changes.parentServerId !== undefined) changes.noParent = false;
|
||||
let thresholdsChange: unknown;
|
||||
if (requestedThresholds !== undefined) {
|
||||
const checked = checkThresholds(requestedThresholds);
|
||||
if (!checked.ok) return res.status(400).json({ error: "invalid_thresholds", message: checked.message });
|
||||
changes.thresholds = checked.value ? JSON.stringify(checked.value) : null;
|
||||
thresholdsChange = checked.value;
|
||||
}
|
||||
const [updated] = await db.update(servers).set(changes).where(eq(servers.id, id)).returning();
|
||||
// A server's own limits (or the server it runs on) change what is an alert, so the Alerts page shouldn't go on showing the old answer for up to a minute.
|
||||
if (requestedThresholds !== undefined || fields.parentServerId !== undefined || fields.noParent !== undefined) invalidateAlertsCache();
|
||||
|
||||
await recordAudit({
|
||||
actor: req.currentUser!,
|
||||
@@ -192,7 +206,7 @@ serversRouter.patch("/:id", requireRole("admin"), asyncHandler(async (req, res)
|
||||
action: "update",
|
||||
targetType: "server",
|
||||
targetId: id,
|
||||
detail: { name: updated.name, ...(parsed.data.parentServerId !== undefined ? { parentServerId: parsed.data.parentServerId } : {}), ...(parsed.data.noParent !== undefined ? { noParent: parsed.data.noParent } : {}) },
|
||||
detail: { name: updated.name, ...(parsed.data.parentServerId !== undefined ? { parentServerId: parsed.data.parentServerId } : {}), ...(parsed.data.noParent !== undefined ? { noParent: parsed.data.noParent } : {}), ...(requestedThresholds !== undefined ? { thresholds: thresholdsChange } : {}) },
|
||||
});
|
||||
|
||||
const serverOut = publicServer(updated);
|
||||
@@ -203,7 +217,7 @@ serversRouter.patch("/:id", requireRole("admin"), asyncHandler(async (req, res)
|
||||
// (which viewers can't read), so the widget and the alerts always agree about what "offline" and "full" mean.
|
||||
serversRouter.get("/summary", asyncHandler(async (_req, res) => {
|
||||
const rows = await db
|
||||
.select({ id: servers.id, name: servers.name, osType: servers.osType, lastSeenAt: servers.lastSeenAt, disks: servers.disks })
|
||||
.select({ id: servers.id, name: servers.name, osType: servers.osType, lastSeenAt: servers.lastSeenAt, disks: servers.disks, thresholds: servers.thresholds })
|
||||
.from(servers);
|
||||
const { healthChecks } = await getSettings();
|
||||
res.json(buildServerSummary(rows, healthChecks, Date.now(), await activeSubjects()));
|
||||
@@ -367,6 +381,7 @@ serversRouter.get("/:id/detail", asyncHandler(async (req, res) => {
|
||||
lastPortScan: _lastPortScan,
|
||||
pendingUpdates: rawUpdates,
|
||||
hostHealth: rawHealth,
|
||||
thresholds: rawThresholds,
|
||||
...serverOut
|
||||
} = server;
|
||||
|
||||
@@ -380,7 +395,7 @@ serversRouter.get("/:id/detail", asyncHandler(async (req, res) => {
|
||||
children: [...links2.entries()].filter(([, l]) => l.parentId === id).map(([childId, l]) => ({ id: childId, name: nameOf.get(childId) ?? "", auto: l.auto })).sort((a, b) => a.name.localeCompare(b.name)),
|
||||
};
|
||||
|
||||
res.json({ server: { ...serverOut, tags: parseTags(rawTags), pendingUpdates: parsePendingUpdates(rawUpdates), hostHealth: parseHostHealth(rawHealth) }, ipAddresses, hardware, dnsMatches, links, relations, agentLatest: latestAgentVersions() });
|
||||
res.json({ server: { ...serverOut, tags: parseTags(rawTags), pendingUpdates: parsePendingUpdates(rawUpdates), hostHealth: parseHostHealth(rawHealth), thresholds: parseThresholds(rawThresholds) }, thresholdDefaults: (await getSettings()).healthChecks, ipAddresses, hardware, dnsMatches, links, relations, agentLatest: latestAgentVersions() });
|
||||
}));
|
||||
|
||||
const tagsSchema = z.object({ tags: z.array(z.string().max(100)).max(50) });
|
||||
|
||||
@@ -76,6 +76,7 @@ const updateSchema = z.object({
|
||||
pbsVerificationCheck: z.boolean(),
|
||||
healthAlerts: z.boolean(),
|
||||
automationAlerts: z.boolean(),
|
||||
updateRunResults: z.boolean(),
|
||||
domainExpiryCheck: z.boolean(),
|
||||
secretCheckTime: z.string().regex(/^\d{2}:\d{2}$/),
|
||||
timezone: z.string(),
|
||||
@@ -95,7 +96,7 @@ const updateSchema = z.object({
|
||||
.partial()
|
||||
.optional(),
|
||||
healthChecks: z
|
||||
.object({ serverOfflineMinutes: z.number().int().min(15).max(10080), diskUsagePercent: z.number().int().min(50).max(99), domainWarnDays: z.number().int().min(1).max(365), diskForecastDays: z.number().int().min(0).max(90) })
|
||||
.object({ serverOfflineMinutes: z.number().int().min(15).max(10080), diskUsagePercent: z.number().int().min(50).max(99), domainWarnDays: z.number().int().min(1).max(365), diskForecastDays: z.number().int().min(0).max(90), backupStaleHours: z.number().int().min(0).max(2160) })
|
||||
.partial()
|
||||
.optional(),
|
||||
quietHours: z
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
import { Router } from "express";
|
||||
import { z } from "zod";
|
||||
import { requireAuth, requireRole } from "../auth/middleware.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
import { ACTIVE_STATUSES, cancelUpdateRun, listUpdateRuns, startUpdateRun, UpdateRunError } from "../services/updateRuns.js";
|
||||
|
||||
export const updateRunsRouter = Router();
|
||||
|
||||
// Anyone signed in can see how updates are going, like the Updates page itself. Starting or stopping one is for operators.
|
||||
updateRunsRouter.use(requireAuth);
|
||||
|
||||
// Which hosts the playbook may touch: names, addresses and ansible patterns — and nothing that could be taken for an option. The same rule the
|
||||
// Semaphore page uses, except that it may not be left out: an update run is for one server.
|
||||
const startSchema = z.object({
|
||||
serverId: z.number().int().positive(),
|
||||
integrationId: z.number().int().positive(),
|
||||
projectId: z.number().int().positive(),
|
||||
templateId: z.number().int().positive(),
|
||||
limit: z.string().trim().regex(/^[A-Za-z0-9][A-Za-z0-9._,:*\-]{0,199}$/, "Use the host name as it is in the Semaphore inventory, like web01"),
|
||||
windowMinutes: z.number().int().min(15).max(480).default(90),
|
||||
});
|
||||
|
||||
const shape = (r: Awaited<ReturnType<typeof listUpdateRuns>>[number]) => ({
|
||||
id: r.id,
|
||||
serverId: r.serverId,
|
||||
serverName: r.serverName,
|
||||
integrationId: r.integrationId,
|
||||
template: r.templateName,
|
||||
limit: r.hostLimit,
|
||||
taskId: r.taskId,
|
||||
taskStatus: r.taskStatus,
|
||||
status: r.status,
|
||||
active: ACTIVE_STATUSES.includes(r.status as (typeof ACTIVE_STATUSES)[number]),
|
||||
message: r.message,
|
||||
startedBy: r.startedBy,
|
||||
startedAt: r.startedAt,
|
||||
taskEndedAt: r.taskEndedAt,
|
||||
finishedAt: r.finishedAt,
|
||||
updatesBefore: r.updatesBefore,
|
||||
updatesAfter: r.updatesAfter,
|
||||
rebootRequired: r.rebootRequired,
|
||||
});
|
||||
|
||||
updateRunsRouter.get("/", asyncHandler(async (req, res) => {
|
||||
const serverId = Number(req.query.serverId);
|
||||
const runs = await listUpdateRuns({ serverId: Number.isInteger(serverId) && serverId > 0 ? serverId : undefined, limit: 30 });
|
||||
res.json({ runs: runs.map(shape) });
|
||||
}));
|
||||
|
||||
updateRunsRouter.post("/", requireRole("operator"), asyncHandler(async (req, res) => {
|
||||
const parsed = startSchema.safeParse(req.body);
|
||||
if (!parsed.success) {
|
||||
return res.status(400).json({ error: "invalid_body", message: parsed.error.issues[0]?.message ?? "Choose a server, a template and the host name it runs on.", details: parsed.error.flatten() });
|
||||
}
|
||||
const actor = req.currentUser!;
|
||||
try {
|
||||
const run = await startUpdateRun({ ...parsed.data, actor: actor.name ?? actor.email ?? actor.oidcSub });
|
||||
const [shaped] = (await listUpdateRuns({ serverId: run.serverId, limit: 1 })).map(shape);
|
||||
res.status(201).json({ run: shaped });
|
||||
} catch (err) {
|
||||
if (err instanceof UpdateRunError) return res.status(err.status).json({ error: err.code, message: err.message });
|
||||
throw err;
|
||||
}
|
||||
}));
|
||||
|
||||
updateRunsRouter.post("/:id/cancel", requireRole("operator"), asyncHandler(async (req, res) => {
|
||||
const id = Number(req.params.id);
|
||||
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
|
||||
const actor = req.currentUser!;
|
||||
try {
|
||||
const run = await cancelUpdateRun(id, actor.name ?? actor.email ?? actor.oidcSub);
|
||||
if (!run) return res.status(404).json({ error: "not_found" });
|
||||
res.json({ status: run.status });
|
||||
} catch (err) {
|
||||
if (err instanceof UpdateRunError) return res.status(err.status).json({ error: err.code, message: err.message });
|
||||
throw err;
|
||||
}
|
||||
}));
|
||||
@@ -13,6 +13,8 @@ import { eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { integrations, secrets, servers } from "../db/schema.js";
|
||||
import { disksFillingSoon, getServerMetrics } from "./serverMetrics.js";
|
||||
import { diskLimit, effectiveThresholds, parseThresholds } from "./serverThresholds.js";
|
||||
import { getBackupCoverage, staleBackups } from "./backupCoverage.js";
|
||||
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
|
||||
import { createUptimeKumaAdapter } from "../integrations/uptimekuma/adapter.js";
|
||||
import { createOsTicketAdapter } from "../integrations/osticket/adapter.js";
|
||||
@@ -241,6 +243,25 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
sourceFailures.forEach((f) => cannotCheckIntegration(f, "Proxmox backups"));
|
||||
}),
|
||||
|
||||
// A guest whose newest backup is older than its limit — which Proxmox's own "latest run failed" alert can't see, because that only
|
||||
// looks at the most recent run per node.
|
||||
check("Backup coverage", async () => {
|
||||
const now = new Date();
|
||||
const report = await getBackupCoverage({ now, rules: true });
|
||||
for (const s of staleBackups(report, now, subjects)) {
|
||||
push({
|
||||
category: "backup",
|
||||
severity: s.severity,
|
||||
type: "proxmox",
|
||||
message: s.message,
|
||||
key: s.key,
|
||||
link: "/backups",
|
||||
silenced: s.silenced,
|
||||
target: s.guest.server ? targetOf("server", s.guest.server.id) : integrationTarget(s.guest.integrationId),
|
||||
});
|
||||
}
|
||||
}),
|
||||
|
||||
check("Backup verification", async () => {
|
||||
const { problems, sourceFailures } = await collectPbsProblems({ skipSilenced: false });
|
||||
for (const p of problems) {
|
||||
@@ -376,10 +397,13 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
// Not a notification: a disk that is on course to fill is something to see coming, so it is shown here, from the server's own history.
|
||||
check("Disk forecasts", async () => {
|
||||
const { healthChecks } = await getSettings();
|
||||
if (!(healthChecks.diskForecastDays > 0)) return;
|
||||
for (const s of await db.select({ id: servers.id, name: servers.name }).from(servers)) {
|
||||
for (const s of await db.select({ id: servers.id, name: servers.name, thresholds: servers.thresholds }).from(servers)) {
|
||||
// A server may have its own forecast window (0 turns it off for that server), its own limit, and disks it is told to leave alone.
|
||||
const own = parseThresholds(s.thresholds);
|
||||
const forecastDays = effectiveThresholds(healthChecks, own).diskForecastDays;
|
||||
if (!(forecastDays > 0)) continue;
|
||||
const series = await getServerMetrics(s.id, 14);
|
||||
for (const f of disksFillingSoon(series, healthChecks.diskForecastDays, healthChecks.diskUsagePercent)) {
|
||||
for (const f of disksFillingSoon(series, forecastDays, (mount) => diskLimit(own, mount, healthChecks.diskUsagePercent))) {
|
||||
push({
|
||||
category: "disk",
|
||||
severity: f.daysToFull <= 3 ? "critical" : "warning",
|
||||
|
||||
@@ -14,6 +14,9 @@ import { db } from "../db/client.js";
|
||||
import { integrations, servers } from "../db/schema.js";
|
||||
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
|
||||
import { createPbsAdapter, type PbsBackupGroup } from "../integrations/pbs/adapter.js";
|
||||
import { parseThresholds } from "./serverThresholds.js";
|
||||
import { activeSubjects } from "./maintenance.js";
|
||||
import { getSettings } from "./settingsStore.js";
|
||||
import { createProxmoxAdapter, guestsWithoutBackupCoverage, type ProxmoxBackupJob, type ProxmoxBackupTask, type ProxmoxGuest } from "../integrations/proxmox/adapter.js";
|
||||
|
||||
export type CoverageStatus = "ok" | "stale" | "failed" | "none" | "unknown";
|
||||
@@ -36,6 +39,8 @@ export interface CoverageGuest {
|
||||
status: CoverageStatus;
|
||||
/** Why it is "failed", when it is. */
|
||||
failure: string | null;
|
||||
/** How old its newest backup may be before it counts as stale, in hours — the limit that applied to this guest (0 = never stale). */
|
||||
staleAfterHours: number;
|
||||
/** The server in this app that is this guest, if one is linked to it. */
|
||||
server: { id: number; name: string } | null;
|
||||
}
|
||||
@@ -65,6 +70,8 @@ export interface LinkedServer {
|
||||
name: string;
|
||||
proxmoxIntegrationId: number | null;
|
||||
proxmoxVmid: number | null;
|
||||
/** This server's own "backup older than" limit, when it has one and the caller wants it used. */
|
||||
staleHours?: number | null;
|
||||
}
|
||||
|
||||
export interface CoverageReport {
|
||||
@@ -81,7 +88,7 @@ export interface CoverageReport {
|
||||
* with nothing to say when it last ran is "unknown".
|
||||
*/
|
||||
export function buildCoverage(input: { proxmox: ProxmoxSource[]; pbs: PbsSource[]; servers: LinkedServer[]; now: Date; staleHours?: number }): CoverageGuest[] {
|
||||
const staleMs = (input.staleHours ?? DEFAULT_STALE_HOURS) * 3600_000;
|
||||
const defaultHours = input.staleHours ?? DEFAULT_STALE_HOURS;
|
||||
const nowMs = input.now.getTime();
|
||||
|
||||
// The newest snapshot of each guest, across every Backup Server (guests are told apart there by "vm/100" or "ct/101").
|
||||
@@ -113,12 +120,14 @@ export function buildCoverage(input: { proxmox: ProxmoxSource[]; pbs: PbsSource[
|
||||
else if (latestTask && !latestTask.ok && latestTask.status !== "running" && (!newest || Date.parse(latestTask.startTime) > newest.at)) failure = `the latest backup didn't succeed (${latestTask.status})`;
|
||||
|
||||
const inJob = !uncovered.has(g.vmid);
|
||||
const linked = input.servers.find((s) => s.proxmoxIntegrationId === src.integrationId && s.proxmoxVmid === g.vmid);
|
||||
// The server this guest is may have its own limit (a weekly backup is fine at 8 days); 0 means this one is never called stale.
|
||||
const staleAfterHours = linked?.staleHours ?? defaultHours;
|
||||
let status: CoverageStatus;
|
||||
if (failure) status = "failed";
|
||||
else if (newest) status = nowMs - newest.at > staleMs ? "stale" : "ok";
|
||||
else if (newest) status = staleAfterHours > 0 && nowMs - newest.at > staleAfterHours * 3600_000 ? "stale" : "ok";
|
||||
else status = inJob ? "unknown" : "none";
|
||||
|
||||
const linked = input.servers.find((s) => s.proxmoxIntegrationId === src.integrationId && s.proxmoxVmid === g.vmid);
|
||||
out.push({
|
||||
integrationId: src.integrationId,
|
||||
integrationName: src.integrationName,
|
||||
@@ -132,6 +141,7 @@ export function buildCoverage(input: { proxmox: ProxmoxSource[]; pbs: PbsSource[
|
||||
lastSource: newest ? newest.source : null,
|
||||
status,
|
||||
failure,
|
||||
staleAfterHours,
|
||||
server: linked ? { id: linked.id, name: linked.name } : null,
|
||||
});
|
||||
}
|
||||
@@ -205,15 +215,76 @@ async function loadSources(): Promise<Loaded> {
|
||||
return loaded;
|
||||
}
|
||||
|
||||
export async function getBackupCoverage(options: { staleHours?: number; refresh?: boolean; now?: Date } = {}): Promise<CoverageReport> {
|
||||
/**
|
||||
* The coverage report. `staleHours` is how old a backup may be before it counts as stale, for every guest — what the Backups page's own
|
||||
* selector asks for. With `rules: true` the limit comes from Settings instead (`healthChecks.backupStaleHours`), and a server that has its
|
||||
* own limit uses that for the guest it is — which is what the alerts, the daily message, the digest and /metrics use.
|
||||
* `cachedOnly` never goes to Proxmox or Backup Server: it answers from what was last read, or null if nothing has been read yet.
|
||||
*/
|
||||
export async function getBackupCoverage(options: { staleHours?: number; refresh?: boolean; now?: Date; rules?: boolean; cachedOnly?: boolean } = {}): Promise<CoverageReport> {
|
||||
const report = await readCoverage(options);
|
||||
if (!report) throw new Error("no backup coverage has been read yet");
|
||||
return report;
|
||||
}
|
||||
|
||||
export async function getCachedBackupCoverage(now: Date = new Date()): Promise<CoverageReport | null> {
|
||||
return readCoverage({ now, rules: true, cachedOnly: true });
|
||||
}
|
||||
|
||||
async function readCoverage(options: { staleHours?: number; refresh?: boolean; now?: Date; rules?: boolean; cachedOnly?: boolean }): Promise<CoverageReport | null> {
|
||||
const now = options.now ?? new Date();
|
||||
const staleHours = options.staleHours ?? DEFAULT_STALE_HOURS;
|
||||
if (!cache || options.refresh || now.getTime() - cache.at > CACHE_MS || loaderOverride) {
|
||||
if (options.cachedOnly) {
|
||||
if (!cache) return null;
|
||||
} else if (!cache || options.refresh || now.getTime() - cache.at > CACHE_MS || loaderOverride) {
|
||||
cache = { at: now.getTime(), loaded: await loadSources() };
|
||||
}
|
||||
const serverRows = await db.select({ id: servers.id, name: servers.name, proxmoxIntegrationId: servers.proxmoxIntegrationId, proxmoxVmid: servers.proxmoxVmid }).from(servers);
|
||||
const guests = buildCoverage({ proxmox: cache.loaded.proxmox, pbs: cache.loaded.pbs, servers: serverRows, now, staleHours }).sort(
|
||||
(a, b) => a.name.localeCompare(b.name) || a.vmid - b.vmid,
|
||||
);
|
||||
return { guests, counts: summarize(guests), sources: cache.loaded.sources, staleHours, generatedAt: now.toISOString() };
|
||||
const loaded = cache!.loaded;
|
||||
const staleHours = options.rules ? (await getSettings()).healthChecks.backupStaleHours : (options.staleHours ?? DEFAULT_STALE_HOURS);
|
||||
const rows = await db.select({ id: servers.id, name: servers.name, proxmoxIntegrationId: servers.proxmoxIntegrationId, proxmoxVmid: servers.proxmoxVmid, thresholds: servers.thresholds }).from(servers);
|
||||
const serverRows: LinkedServer[] = rows.map((r) => ({ id: r.id, name: r.name, proxmoxIntegrationId: r.proxmoxIntegrationId, proxmoxVmid: r.proxmoxVmid, staleHours: options.rules ? (parseThresholds(r.thresholds)?.backupStaleHours ?? null) : null }));
|
||||
const guests = buildCoverage({ proxmox: loaded.proxmox, pbs: loaded.pbs, servers: serverRows, now, staleHours }).sort((a, b) => a.name.localeCompare(b.name) || a.vmid - b.vmid);
|
||||
return { guests, counts: summarize(guests), sources: loaded.sources, staleHours, generatedAt: now.toISOString() };
|
||||
}
|
||||
|
||||
export interface StaleBackup {
|
||||
guest: CoverageGuest;
|
||||
/** How long ago its newest backup was, in hours. */
|
||||
ageHours: number;
|
||||
severity: "critical" | "warning";
|
||||
/** Stable for the same guest, so it alerts once. */
|
||||
key: string;
|
||||
/** In maintenance: on the server it is, or on the Proxmox integration it lives in. */
|
||||
silenced: boolean;
|
||||
message: string;
|
||||
}
|
||||
|
||||
// Hours up to three days, days after that — so a 48-hour limit and a 60-hour-old backup read as "48 h" and "60 h", not both as "2 days".
|
||||
const hoursWords = (h: number) => (h < 72 ? `${Math.round(h)} h` : `${Math.floor(h / 24)} days`);
|
||||
|
||||
/**
|
||||
* Guests whose newest backup is older than their limit. A failed backup is not here: the latest failing run on a node, and snapshots that
|
||||
* failed verification, already raise their own alerts, and a guest that goes on failing becomes stale once its limit passes. Critical at
|
||||
* three times the limit.
|
||||
*/
|
||||
export function staleBackups(report: CoverageReport, now: Date, silencedSubjects: Set<string> = new Set()): StaleBackup[] {
|
||||
const out: StaleBackup[] = [];
|
||||
for (const g of report.guests) {
|
||||
if (g.status !== "stale" || !g.lastBackupAt) continue;
|
||||
const ageHours = (now.getTime() - Date.parse(g.lastBackupAt)) / 3600_000;
|
||||
out.push({
|
||||
guest: g,
|
||||
ageHours,
|
||||
severity: ageHours >= g.staleAfterHours * 3 ? "critical" : "warning",
|
||||
key: `backup-stale:${g.integrationId}:${g.vmid}`,
|
||||
silenced: silencedSubjects.has(`integration:${g.integrationId}`) || (g.server !== null && silencedSubjects.has(`server:${g.server.id}`)),
|
||||
message: `${g.name} (#${g.vmid}) [${g.integrationName}]: its newest backup is ${hoursWords(ageHours)} old (the limit is ${hoursWords(g.staleAfterHours)})`,
|
||||
});
|
||||
}
|
||||
return out.sort((a, b) => b.ageHours - a.ageHours);
|
||||
}
|
||||
|
||||
/** What the daily check looks at: the stale guests that aren't under a maintenance window, read fresh from Proxmox and Backup Server. */
|
||||
export async function collectStaleBackups(now: Date = new Date()): Promise<StaleBackup[]> {
|
||||
const report = await getBackupCoverage({ now, rules: true, refresh: true });
|
||||
return staleBackups(report, now, await activeSubjects(now)).filter((s) => !s.silenced);
|
||||
}
|
||||
@@ -10,6 +10,7 @@ import { getInternalFlag, getSettings, setInternalFlag } from "./settingsStore.j
|
||||
import { activeSubjects, subjectOfConditionKey } from "./maintenance.js";
|
||||
import { coveredByParent, loadParentRows, resolveParents } from "./serverParents.js";
|
||||
import { hostHealthConditions, parseHostHealth, type HostHealth } from "./hostHealth.js";
|
||||
import { diskLimit, parseThresholds, type ServerThresholds } from "./serverThresholds.js";
|
||||
|
||||
const STATE_FLAG = "healthActiveConditions";
|
||||
/** After a restart the agents haven't had a chance to report yet (they run every 15 min), so server-derived conditions are held rather than judged. */
|
||||
@@ -35,6 +36,8 @@ export interface ServerSnapshot {
|
||||
disks: { mount: string; sizeBytes: number; usedBytes: number }[];
|
||||
/** What the agent says about failed services, disk health and ZFS pools, if it says anything. */
|
||||
health?: HostHealth | null;
|
||||
/** This server's own thresholds, where it has any: they replace the global ones for it. */
|
||||
thresholds?: ServerThresholds | null;
|
||||
}
|
||||
export interface ProxmoxSnapshot {
|
||||
integrationId: number;
|
||||
@@ -106,7 +109,7 @@ export function evaluateHealth(snapshot: HealthSnapshot, thresholds: Thresholds,
|
||||
let offline = false;
|
||||
if (s.lastSeenAt) {
|
||||
const age = now - parseTimestamp(s.lastSeenAt);
|
||||
if (Number.isFinite(age) && age > thresholds.serverOfflineMinutes * 60_000) {
|
||||
if (Number.isFinite(age) && age > (s.thresholds?.serverOfflineMinutes ?? thresholds.serverOfflineMinutes) * 60_000) {
|
||||
offline = true;
|
||||
out.push({
|
||||
key: `offline:server:${s.id}`,
|
||||
@@ -121,7 +124,9 @@ export function evaluateHealth(snapshot: HealthSnapshot, thresholds: Thresholds,
|
||||
for (const c of hostHealthConditions(s.id, s.name, s.health ?? null)) out.push({ key: c.key, source: "server", message: c.message, severity: c.severity });
|
||||
for (const d of s.disks) {
|
||||
const p = usage(d.usedBytes, d.sizeBytes);
|
||||
if (p !== null && p >= limit) {
|
||||
// A disk can have its own limit, or be left alone altogether; otherwise the server's, then the global one.
|
||||
const diskAlertAt = diskLimit(s.thresholds, d.mount, limit);
|
||||
if (diskAlertAt !== null && p !== null && p >= diskAlertAt) {
|
||||
out.push({
|
||||
key: `disk:server:${s.id}:${d.mount}`,
|
||||
source: "server",
|
||||
@@ -300,7 +305,7 @@ export async function collectSnapshot(): Promise<{ snapshot: HealthSnapshot; hel
|
||||
} catch {
|
||||
// a malformed report shouldn't stop every other server being checked
|
||||
}
|
||||
snapshot.servers.push({ id: s.id, name: s.name, lastSeenAt: s.lastSeenAt, disks, health: parseHostHealth(s.hostHealth) });
|
||||
snapshot.servers.push({ id: s.id, name: s.name, lastSeenAt: s.lastSeenAt, disks, health: parseHostHealth(s.hostHealth), thresholds: parseThresholds(s.thresholds) });
|
||||
}
|
||||
|
||||
const rows = await db
|
||||
|
||||
@@ -11,6 +11,8 @@ import { isAgentOutdated, latestAgentVersions } from "./agentVersions.js";
|
||||
import { getSourceStatus } from "./integrationHealthMonitor.js";
|
||||
import { parsePendingUpdates } from "./pendingUpdates.js";
|
||||
import { parseHostHealth, poolIsFine } from "./hostHealth.js";
|
||||
import { parseThresholds } from "./serverThresholds.js";
|
||||
import { getCachedBackupCoverage } from "./backupCoverage.js";
|
||||
import { disksFillingSoon, getServerMetrics } from "./serverMetrics.js";
|
||||
import { computeSecretStatus } from "./secretStatus.js";
|
||||
import { getSettings } from "./settingsStore.js";
|
||||
@@ -58,8 +60,9 @@ export async function buildMetrics(now: Date = new Date()): Promise<string> {
|
||||
// ── Servers ──
|
||||
const latest = latestAgentVersions();
|
||||
const serverRows = await db.select().from(servers);
|
||||
const offlineAfterMs = settings.healthChecks.serverOfflineMinutes * 60_000;
|
||||
for (const s of serverRows) {
|
||||
// A server may have its own "offline after"; otherwise the global one.
|
||||
const offlineAfterMs = (parseThresholds(s.thresholds)?.serverOfflineMinutes ?? settings.healthChecks.serverOfflineMinutes) * 60_000;
|
||||
if (!s.lastSeenAt) continue; // never reported: nothing to say about it
|
||||
const server = s.name;
|
||||
const seen = Date.parse(s.lastSeenAt);
|
||||
@@ -115,6 +118,14 @@ export async function buildMetrics(now: Date = new Date()): Promise<string> {
|
||||
// ── Alerts ──
|
||||
try {
|
||||
const report = await getAlerts(false);
|
||||
// What the backup check last found (the alert pass above refreshes it); never reads Proxmox or Backup Server itself.
|
||||
const coverage = await getCachedBackupCoverage(now);
|
||||
if (coverage) {
|
||||
for (const status of ["ok", "stale", "failed", "none", "unknown"] as const) out.add("homelab_backup_guests", "gauge", "Proxmox guests by how well they are backed up (ok, stale, failed, none, unknown).", coverage.counts[status], { status });
|
||||
for (const g of coverage.guests) {
|
||||
if (g.lastBackupAt) out.add("homelab_backup_last_age_seconds", "gauge", "How long ago a guest's newest backup was.", Math.max(0, Math.floor((now.getTime() - Date.parse(g.lastBackupAt)) / 1000)), { guest: g.name, vmid: String(g.vmid), integration: g.integrationName });
|
||||
}
|
||||
}
|
||||
for (const severity of ["critical", "warning", "info"] as const) out.add("homelab_alerts", "gauge", "Active alerts by severity (not counting silenced ones).", report.counts[severity], { severity });
|
||||
out.add("homelab_alerts_silenced", "gauge", "Alerts held back by a maintenance window.", report.counts.silenced);
|
||||
out.add("homelab_alerts_grouped", "gauge", "Alerts folded under a server that is down.", report.counts.grouped);
|
||||
|
||||
@@ -294,6 +294,23 @@ export async function notifyProxmoxUncoveredGuests(
|
||||
);
|
||||
}
|
||||
|
||||
export async function notifyStaleBackups(stale: { message: string }[]): Promise<void> {
|
||||
if (stale.length === 0) return;
|
||||
if (!(await eventEnabled("proxmoxBackupCheck"))) return;
|
||||
await notify(
|
||||
"Homelab Manager — Backups Gone Stale",
|
||||
`${stale.length} guest${stale.length !== 1 ? "s have" : " has"} no recent backup:\n\n${stale.map((s) => s.message).join("\n")}`,
|
||||
);
|
||||
}
|
||||
|
||||
export async function notifyUpdateRunFinished(run: { serverName: string; templateName: string; ok: boolean; message: string }): Promise<void> {
|
||||
if (!(await eventEnabled("updateRunResults"))) return;
|
||||
await notify(
|
||||
run.ok ? `Homelab Manager — ${run.serverName} updated` : `Homelab Manager — Update of ${run.serverName} didn't finish`,
|
||||
`"${run.templateName}" on ${run.serverName}: ${run.message}`,
|
||||
);
|
||||
}
|
||||
|
||||
export async function notifyPbsVerificationFailed(
|
||||
failures: { integrationName: string; datastore: string; failedCount: number; error: string | null }[],
|
||||
): Promise<void> {
|
||||
|
||||
@@ -4,7 +4,8 @@ import { db } from "../db/client.js";
|
||||
import { integrations } from "../db/schema.js";
|
||||
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
|
||||
import { createProxmoxAdapter, guestsWithoutBackupCoverage, type ProxmoxBackupTask } from "../integrations/proxmox/adapter.js";
|
||||
import { notifyProxmoxBackupFailure, notifyProxmoxUncoveredGuests } from "./notify.js";
|
||||
import { notifyProxmoxBackupFailure, notifyProxmoxUncoveredGuests, notifyStaleBackups } from "./notify.js";
|
||||
import { collectStaleBackups } from "./backupCoverage.js";
|
||||
import { getSettings, getInternalFlag, setInternalFlag } from "./settingsStore.js";
|
||||
import { activeSubjects } from "./maintenance.js";
|
||||
import type { SourceFailure } from "./alertTypes.js";
|
||||
@@ -92,6 +93,13 @@ async function checkProxmoxBackups(): Promise<void> {
|
||||
const { failures, uncovered } = await collectProxmoxBackupProblems({ skipSilenced: true });
|
||||
await notifyProxmoxBackupFailure(failures);
|
||||
await notifyProxmoxUncoveredGuests(uncovered);
|
||||
// Guests whose newest backup has gone too old (the limit in Settings, or a server's own). A problem reading Proxmox or Backup Server
|
||||
// here must not stop the other two messages, which have already gone.
|
||||
try {
|
||||
await notifyStaleBackups(await collectStaleBackups());
|
||||
} catch (err) {
|
||||
console.error("[proxmoxBackup] couldn't check for stale backups:", err);
|
||||
}
|
||||
}
|
||||
|
||||
async function checkProxmoxBackupsOnce(): Promise<void> {
|
||||
|
||||
@@ -40,7 +40,7 @@ export async function buildPublicWall(detail: ShareLinkDetail, now: Date = new D
|
||||
getAlerts(false),
|
||||
getSettings(),
|
||||
listActiveWindows(now),
|
||||
db.select({ id: servers.id, name: servers.name, osType: servers.osType, lastSeenAt: servers.lastSeenAt, disks: servers.disks }).from(servers),
|
||||
db.select({ id: servers.id, name: servers.name, osType: servers.osType, lastSeenAt: servers.lastSeenAt, disks: servers.disks, thresholds: servers.thresholds }).from(servers),
|
||||
db.select({ id: integrations.id, name: integrations.name, type: integrations.type, enabled: integrations.enabled }).from(integrations),
|
||||
activeSubjects(now),
|
||||
]);
|
||||
|
||||
@@ -72,6 +72,19 @@ export function describeAudit(category: string, action: string, detail: Record<s
|
||||
return { title: "Admin link removed", tone: "neutral", kind: "change", detail: label };
|
||||
}
|
||||
}
|
||||
if (category === "update_run") {
|
||||
const template = typeof detail.template === "string" ? detail.template : null;
|
||||
switch (action) {
|
||||
case "start":
|
||||
return { title: "Update run started", tone: "info", kind: "change", detail: template ? `${template}, limited to ${detail.limit ?? "this server"}` : null };
|
||||
case "finished":
|
||||
return { title: "Update run finished", tone: "good", kind: "change", detail: message };
|
||||
case "failed":
|
||||
return { title: "Update run didn't finish", tone: "bad", kind: "change", detail: message };
|
||||
case "cancelled":
|
||||
return { title: "Update run stopped being followed", tone: "neutral", kind: "change", detail: message };
|
||||
}
|
||||
}
|
||||
return { title: `${humanize(category)}: ${action.replace(/_/g, " ")}`, tone: "neutral", kind: "change", detail: message ?? label };
|
||||
}
|
||||
|
||||
|
||||
@@ -140,17 +140,20 @@ export const FORECAST_MIN_FIT = 0.9;
|
||||
/**
|
||||
* The disks that are heading for full within `withinDays`. A disk already at or above `alreadyAlertedAt` percent is left out — the
|
||||
* ordinary "disk nearly full" alert covers it — and so is one whose growth is erratic (a big copy, a log that rotates), where a
|
||||
* straight-line forecast would just be a guess.
|
||||
* straight-line forecast would just be a guess. `alreadyAlertedAt` may be a function of the disk, for servers with a limit of their own
|
||||
* for one of them; returning null leaves that disk out altogether (it has been told not to alert).
|
||||
*/
|
||||
export function disksFillingSoon(series: MetricSeries[], withinDays: number, alreadyAlertedAt: number): DiskForecast[] {
|
||||
export function disksFillingSoon(series: MetricSeries[], withinDays: number, alreadyAlertedAt: number | ((mount: string) => number | null)): DiskForecast[] {
|
||||
const out: DiskForecast[] = [];
|
||||
for (const s of series) {
|
||||
if (s.kind !== "disk" || !s.trend || s.latest === null) continue;
|
||||
const { perDay, fit, spanDays, daysToFull } = s.trend;
|
||||
if (daysToFull === null || daysToFull > withinDays) continue;
|
||||
if (spanDays < FORECAST_MIN_SPAN_DAYS || fit < FORECAST_MIN_FIT) continue;
|
||||
if (s.latest >= alreadyAlertedAt) continue;
|
||||
out.push({ mount: s.metric.slice("disk:".length), latest: s.latest, perDay, daysToFull });
|
||||
const mount = s.metric.slice("disk:".length);
|
||||
const limit = typeof alreadyAlertedAt === "function" ? alreadyAlertedAt(mount) : alreadyAlertedAt;
|
||||
if (limit === null || s.latest >= limit) continue;
|
||||
out.push({ mount, latest: s.latest, perDay, daysToFull });
|
||||
}
|
||||
return out.sort((a, b) => a.daysToFull - b.daysToFull);
|
||||
}
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { evaluateHealth, type Thresholds } from "./healthMonitor.js";
|
||||
import { parseThresholds } from "./serverThresholds.js";
|
||||
|
||||
/** The columns of a server row the summary reads. */
|
||||
export interface SummaryRow {
|
||||
@@ -7,6 +8,8 @@ export interface SummaryRow {
|
||||
osType: string;
|
||||
lastSeenAt: string | null;
|
||||
disks: string | null; // JSON, as the agent's report is stored
|
||||
/** This server's own thresholds, as stored. */
|
||||
thresholds?: string | null;
|
||||
}
|
||||
|
||||
export interface ServerSummary {
|
||||
@@ -40,7 +43,7 @@ function parseDisks(text: string | null): { mount: string; sizeBytes: number; us
|
||||
*/
|
||||
export function buildServerSummary(rows: SummaryRow[], thresholds: Thresholds, now: number, silencedSubjects: Set<string> = new Set()): ServerSummary {
|
||||
const snapshot = {
|
||||
servers: rows.map((r) => ({ id: r.id, name: r.name, lastSeenAt: r.lastSeenAt, disks: parseDisks(r.disks) })),
|
||||
servers: rows.map((r) => ({ id: r.id, name: r.name, lastSeenAt: r.lastSeenAt, disks: parseDisks(r.disks), thresholds: parseThresholds(r.thresholds) })),
|
||||
proxmox: [],
|
||||
synology: [],
|
||||
};
|
||||
|
||||
@@ -0,0 +1,124 @@
|
||||
/**
|
||||
* Per-server alert thresholds. The ones in Settings → Notifications apply to every server; some machines are different — a NAS that sits at
|
||||
* 92% by design, a database host that should speak up at 80%, a laptop that is meant to be off at night. Each server may override the
|
||||
* server-offline time, the disk-usage percentage, the disk forecast window and (for the Proxmox guest it is linked to) how old a backup may
|
||||
* be, and may set a limit for one disk or say not to alert about that disk at all. Anything not overridden follows the global setting, so
|
||||
* changing the global one still reaches every server that hasn't said otherwise.
|
||||
*/
|
||||
|
||||
export interface MountThreshold {
|
||||
/** Alert when this disk is this full (instead of the server's or the global percentage). */
|
||||
percent?: number;
|
||||
/** Never alert about this disk — neither when it is full nor when it is filling. */
|
||||
ignore?: boolean;
|
||||
}
|
||||
|
||||
export interface ServerThresholds {
|
||||
serverOfflineMinutes?: number;
|
||||
diskUsagePercent?: number;
|
||||
/** 0 turns the "filling up" forecast off for this server. */
|
||||
diskForecastDays?: number;
|
||||
/** 0 turns the stale-backup alert off for this server's guest. */
|
||||
backupStaleHours?: number;
|
||||
mounts?: Record<string, MountThreshold>;
|
||||
}
|
||||
|
||||
/** The global values these override. */
|
||||
export interface GlobalThresholds {
|
||||
serverOfflineMinutes: number;
|
||||
diskUsagePercent: number;
|
||||
diskForecastDays: number;
|
||||
backupStaleHours: number;
|
||||
}
|
||||
|
||||
export const LIMITS = {
|
||||
serverOfflineMinutes: { min: 15, max: 10080 },
|
||||
diskUsagePercent: { min: 50, max: 99 },
|
||||
diskForecastDays: { min: 0, max: 90 },
|
||||
backupStaleHours: { min: 0, max: 2160 },
|
||||
} as const;
|
||||
export const MAX_MOUNT_RULES = 50;
|
||||
|
||||
const inRange = (v: unknown, key: keyof typeof LIMITS): v is number => typeof v === "number" && Number.isInteger(v) && v >= LIMITS[key].min && v <= LIMITS[key].max;
|
||||
|
||||
/** What is stored, read back leniently: anything that doesn't make sense is dropped, never an error. Null for no overrides at all. */
|
||||
export function parseThresholds(stored: string | null | undefined): ServerThresholds | null {
|
||||
if (!stored) return null;
|
||||
try {
|
||||
const v = JSON.parse(stored);
|
||||
if (!v || typeof v !== "object") return null;
|
||||
const out: ServerThresholds = {};
|
||||
for (const key of Object.keys(LIMITS) as (keyof typeof LIMITS)[]) if (inRange(v[key], key)) out[key] = v[key];
|
||||
if (v.mounts && typeof v.mounts === "object" && !Array.isArray(v.mounts)) {
|
||||
const mounts: Record<string, MountThreshold> = {};
|
||||
for (const [mount, rule] of Object.entries(v.mounts as Record<string, unknown>).slice(0, MAX_MOUNT_RULES)) {
|
||||
if (!rule || typeof rule !== "object") continue;
|
||||
const r = rule as { percent?: unknown; ignore?: unknown };
|
||||
const clean: MountThreshold = {};
|
||||
if (inRange(r.percent, "diskUsagePercent")) clean.percent = r.percent;
|
||||
if (r.ignore === true) clean.ignore = true;
|
||||
if (clean.percent !== undefined || clean.ignore) mounts[mount] = clean;
|
||||
}
|
||||
if (Object.keys(mounts).length > 0) out.mounts = mounts;
|
||||
}
|
||||
return Object.keys(out).length > 0 ? out : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** What a request may say. Strict where parseThresholds is lenient: a wrong number is refused, with a message that says which. */
|
||||
export function checkThresholds(input: unknown): { ok: true; value: ServerThresholds | null } | { ok: false; message: string } {
|
||||
if (input === null) return { ok: true, value: null };
|
||||
if (typeof input !== "object" || Array.isArray(input)) return { ok: false, message: "Thresholds should be an object." };
|
||||
const v = input as Record<string, unknown>;
|
||||
const out: ServerThresholds = {};
|
||||
const names: Record<keyof typeof LIMITS, string> = {
|
||||
serverOfflineMinutes: "Offline after (minutes)",
|
||||
diskUsagePercent: "Disk usage alert at (%)",
|
||||
diskForecastDays: "Disk forecast (days)",
|
||||
backupStaleHours: "Backup older than (hours)",
|
||||
};
|
||||
for (const key of Object.keys(LIMITS) as (keyof typeof LIMITS)[]) {
|
||||
if (v[key] === undefined || v[key] === null) continue;
|
||||
if (!inRange(v[key], key)) return { ok: false, message: `${names[key]} must be a whole number from ${LIMITS[key].min} to ${LIMITS[key].max}.` };
|
||||
out[key] = v[key] as number;
|
||||
}
|
||||
if (v.mounts !== undefined && v.mounts !== null) {
|
||||
if (typeof v.mounts !== "object" || Array.isArray(v.mounts)) return { ok: false, message: "Disk rules should be an object." };
|
||||
const entries = Object.entries(v.mounts as Record<string, unknown>);
|
||||
if (entries.length > MAX_MOUNT_RULES) return { ok: false, message: `At most ${MAX_MOUNT_RULES} disks can have their own rule.` };
|
||||
const mounts: Record<string, MountThreshold> = {};
|
||||
for (const [mount, rule] of entries) {
|
||||
if (mount.length === 0 || mount.length > 500) return { ok: false, message: "A disk's name is missing or too long." };
|
||||
if (!rule || typeof rule !== "object" || Array.isArray(rule)) return { ok: false, message: `The rule for ${mount} should be an object.` };
|
||||
const r = rule as { percent?: unknown; ignore?: unknown };
|
||||
const clean: MountThreshold = {};
|
||||
if (r.percent !== undefined && r.percent !== null) {
|
||||
if (!inRange(r.percent, "diskUsagePercent")) return { ok: false, message: `The limit for ${mount} must be a whole number from ${LIMITS.diskUsagePercent.min} to ${LIMITS.diskUsagePercent.max}.` };
|
||||
clean.percent = r.percent;
|
||||
}
|
||||
if (r.ignore === true) clean.ignore = true;
|
||||
if (clean.percent !== undefined || clean.ignore) mounts[mount] = clean;
|
||||
}
|
||||
if (Object.keys(mounts).length > 0) out.mounts = mounts;
|
||||
}
|
||||
return { ok: true, value: Object.keys(out).length > 0 ? out : null };
|
||||
}
|
||||
|
||||
/** What applies to one server: its own value where it gave one, the global one otherwise. */
|
||||
export function effectiveThresholds(global: GlobalThresholds, own: ServerThresholds | null | undefined): GlobalThresholds {
|
||||
return {
|
||||
serverOfflineMinutes: own?.serverOfflineMinutes ?? global.serverOfflineMinutes,
|
||||
diskUsagePercent: own?.diskUsagePercent ?? global.diskUsagePercent,
|
||||
diskForecastDays: own?.diskForecastDays ?? global.diskForecastDays,
|
||||
backupStaleHours: own?.backupStaleHours ?? global.backupStaleHours,
|
||||
};
|
||||
}
|
||||
|
||||
/** The usage percentage at which a disk alerts — its own rule first, then the server's, then the global one — or null if it is to be left alone. */
|
||||
export function diskLimit(own: ServerThresholds | null | undefined, mount: string, globalPercent: number): number | null {
|
||||
const rule = own?.mounts?.[mount];
|
||||
if (rule?.ignore) return null;
|
||||
return rule?.percent ?? own?.diskUsagePercent ?? globalPercent;
|
||||
}
|
||||
@@ -50,6 +50,8 @@ export interface NotificationEvents {
|
||||
healthAlerts: boolean;
|
||||
/** A Semaphore template or Gitea repo whose latest run failed, and when it succeeds again. */
|
||||
automationAlerts: boolean;
|
||||
/** How an update run started from the Updates page ended: what is still waiting, whether a restart is due, or why it failed. */
|
||||
updateRunResults: boolean;
|
||||
/** Daily reminder while a domain registration is close to expiring or has expired. */
|
||||
domainExpiryCheck: boolean;
|
||||
secretCheckTime: string; // "HH:MM" — shared by the secret-expiry, Tailscale key-expiry, Docker update, and Proxmox backup checks
|
||||
@@ -97,6 +99,8 @@ export interface HealthCheckSettings {
|
||||
domainWarnDays: number;
|
||||
/** An agent-reported disk that is on course to be full within this many days is flagged on the Alerts page. 0 turns it off. */
|
||||
diskForecastDays: number;
|
||||
/** A Proxmox guest whose newest backup is older than this many hours is raised as stale. 0 turns it off. */
|
||||
backupStaleHours: number;
|
||||
}
|
||||
|
||||
/** The weekly digest: one message a week through the notification channels. Off until an admin switches it on under Settings → Notifications. */
|
||||
@@ -208,6 +212,7 @@ const DEFAULTS: AppSettings = {
|
||||
pbsVerificationCheck: true,
|
||||
healthAlerts: true,
|
||||
automationAlerts: true,
|
||||
updateRunResults: true,
|
||||
domainExpiryCheck: true,
|
||||
secretCheckTime: "08:00",
|
||||
timezone: "UTC",
|
||||
@@ -223,7 +228,7 @@ const DEFAULTS: AppSettings = {
|
||||
inventory: { currency: "", pricePerKwh: null },
|
||||
agents: { autoUpdate: false },
|
||||
signIn: { roleFromGroups: false, groupsClaim: "groups", adminGroups: [], operatorGroups: [], otherwise: "viewer" },
|
||||
healthChecks: { serverOfflineMinutes: 60, diskUsagePercent: 90, domainWarnDays: 30, diskForecastDays: 14 },
|
||||
healthChecks: { serverOfflineMinutes: 60, diskUsagePercent: 90, domainWarnDays: 30, diskForecastDays: 14, backupStaleHours: 48 },
|
||||
// Docker's default bridge (172.17.0.0/16) and the pool it hands custom networks from (172.18–31) live here, and every
|
||||
// Docker host repeats them. Shown on the Consistency page, where it can be removed if 172.16/12 is used as a real LAN.
|
||||
consistency: { excludedRanges: ["172.16.0.0/12"] },
|
||||
|
||||
@@ -10,6 +10,7 @@ import { dnsRecordsCache, dnsZonesCache, integrations, servers } from "../db/sch
|
||||
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
|
||||
import { createUptimeKumaAdapter } from "../integrations/uptimekuma/adapter.js";
|
||||
import { getSettings } from "./settingsStore.js";
|
||||
import { parseThresholds } from "./serverThresholds.js";
|
||||
import { attachMatchedServers, toMatchableServers } from "./uptimeKumaMatch.js";
|
||||
|
||||
export type MapStatus = "ok" | "bad" | "warn" | "unknown";
|
||||
@@ -64,7 +65,6 @@ export async function buildTopology(now: Date = new Date()): Promise<MapData> {
|
||||
const notes: string[] = [];
|
||||
|
||||
const { healthChecks } = await getSettings();
|
||||
const offlineAfterMs = healthChecks.serverOfflineMinutes * 60_000;
|
||||
const serverRows = await db.select().from(servers);
|
||||
const integrationRows = await db.select({ id: integrations.id, name: integrations.name, type: integrations.type, enabled: integrations.enabled }).from(integrations);
|
||||
const integrationName = new Map(integrationRows.map((i) => [i.id, i.name]));
|
||||
@@ -73,6 +73,7 @@ export async function buildTopology(now: Date = new Date()): Promise<MapData> {
|
||||
const hostIds = new Set<string>();
|
||||
for (const s of serverRows) {
|
||||
const lastSeen = s.lastSeenAt ? Date.parse(s.lastSeenAt) : null;
|
||||
const offlineAfterMs = (parseThresholds(s.thresholds)?.serverOfflineMinutes ?? healthChecks.serverOfflineMinutes) * 60_000;
|
||||
const status: MapStatus = lastSeen === null ? "unknown" : now.getTime() - lastSeen > offlineAfterMs ? "bad" : "ok";
|
||||
nodes.push({ id: `server:${s.id}`, kind: "server", label: s.name, sub: s.hostname, status, href: `/servers/${s.id}` });
|
||||
if (s.proxmoxIntegrationId && s.proxmoxNode) {
|
||||
|
||||
@@ -0,0 +1,271 @@
|
||||
/**
|
||||
* An update run: apply a server's waiting updates in one go, and see it through. It puts the server under maintenance (so nothing alerts
|
||||
* while it is being worked on and restarted), starts a Semaphore playbook limited to that server, follows the run until it is over, waits
|
||||
* for the server's agent to report again, then says how many updates are still waiting and whether a restart is due, and ends the
|
||||
* maintenance window it made. Every step is written down on the run, which survives a restart of this app — a poller picks up where it was.
|
||||
*
|
||||
* What it does not do: it does not restart the server (the playbook does, if it is written to), and it cannot stop a run in Semaphore —
|
||||
* cancelling here only stops watching and ends the window.
|
||||
*/
|
||||
import { and, desc, eq, inArray } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { maintenanceWindows, servers, updateRuns } from "../db/schema.js";
|
||||
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
|
||||
import { createSemaphoreAdapter, type SemaphoreAdapter, type SemaphoreTask } from "../integrations/semaphore/adapter.js";
|
||||
import { invalidateAlertsCache } from "./alerts.js";
|
||||
import { recordAudit } from "./audit.js";
|
||||
import { startOrExtendWindow } from "./maintenance.js";
|
||||
import { notifyUpdateRunFinished } from "./notify.js";
|
||||
import { parsePendingUpdates } from "./pendingUpdates.js";
|
||||
|
||||
export type UpdateRunRow = typeof updateRuns.$inferSelect;
|
||||
export type UpdateRunStatus = "running" | "waiting_for_host" | "done" | "failed" | "cancelled";
|
||||
export const ACTIVE_STATUSES: UpdateRunStatus[] = ["running", "waiting_for_host"];
|
||||
|
||||
/** How long a run may take from start to finish before it is given up on. */
|
||||
export const RUN_LIMIT_MS = 3 * 3600_000;
|
||||
/** After the playbook is over, how long the server has to report again (it may be restarting). */
|
||||
export const HOST_WAIT_MS = 30 * 60_000;
|
||||
/** Consecutive polls Semaphore may fail to answer before the run is given up on (at the poller's pace, about five minutes). */
|
||||
export const MAX_POLL_ERRORS = 10;
|
||||
export const POLL_INTERVAL_MS = 20_000;
|
||||
|
||||
const FINISHED_TASK = new Set<SemaphoreTask["status"]>(["success", "error", "stopped", "rejected"]);
|
||||
|
||||
// ─── Semaphore, replaceable for the tests ───────────────────────────────────
|
||||
|
||||
let adapterOverride: ((integrationId: number) => Promise<SemaphoreAdapter | null>) | null = null;
|
||||
export function setSemaphoreForTests(factory: ((integrationId: number) => Promise<SemaphoreAdapter | null>) | null): void {
|
||||
adapterOverride = factory;
|
||||
}
|
||||
|
||||
async function semaphoreFor(integrationId: number): Promise<SemaphoreAdapter | null> {
|
||||
if (adapterOverride) return adapterOverride(integrationId);
|
||||
const loaded = await loadIntegrationConfig(integrationId);
|
||||
if (!loaded || loaded.integration.type !== "semaphore" || !loaded.integration.enabled) return null;
|
||||
return createSemaphoreAdapter(loaded.config as any);
|
||||
}
|
||||
|
||||
// ─── Starting ───────────────────────────────────────────────────────────────
|
||||
|
||||
export class UpdateRunError extends Error {
|
||||
constructor(
|
||||
readonly status: number,
|
||||
readonly code: string,
|
||||
message: string,
|
||||
) {
|
||||
super(message);
|
||||
}
|
||||
}
|
||||
|
||||
export interface StartInput {
|
||||
serverId: number;
|
||||
integrationId: number;
|
||||
projectId: number;
|
||||
templateId: number;
|
||||
/** Which hosts of the inventory the playbook may touch (ansible's --limit). Never empty: an update run is always for this one server. */
|
||||
limit: string;
|
||||
/** How long the maintenance window lasts at most; it ends sooner when the run does. */
|
||||
windowMinutes: number;
|
||||
actor: string;
|
||||
}
|
||||
|
||||
export async function startUpdateRun(input: StartInput, now: Date = new Date()): Promise<UpdateRunRow> {
|
||||
const [server] = await db.select().from(servers).where(eq(servers.id, input.serverId)).limit(1);
|
||||
if (!server) throw new UpdateRunError(404, "server_not_found", "That server doesn't exist.");
|
||||
if (!server.lastSeenAt) throw new UpdateRunError(400, "never_reported", "This server has never reported, so there is nothing to wait for after the run.");
|
||||
|
||||
const [busy] = await db
|
||||
.select({ id: updateRuns.id })
|
||||
.from(updateRuns)
|
||||
.where(and(eq(updateRuns.serverId, server.id), inArray(updateRuns.status, ACTIVE_STATUSES)))
|
||||
.limit(1);
|
||||
if (busy) throw new UpdateRunError(409, "already_running", `An update run for ${server.name} is already under way.`);
|
||||
|
||||
const adapter = await semaphoreFor(input.integrationId);
|
||||
if (!adapter) throw new UpdateRunError(400, "no_semaphore", "That Semaphore integration doesn't exist or isn't switched on.");
|
||||
|
||||
let templateName: string;
|
||||
try {
|
||||
const { templates } = await adapter.checkTemplates();
|
||||
const template = templates.find((t) => t.projectId === input.projectId && t.id === input.templateId);
|
||||
if (!template) throw new UpdateRunError(404, "template_not_found", "That template isn't in Semaphore (any more).");
|
||||
templateName = template.name;
|
||||
} catch (err) {
|
||||
if (err instanceof UpdateRunError) throw err;
|
||||
throw new UpdateRunError(502, "semaphore_unreachable", `Couldn't reach Semaphore: ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
|
||||
// Quiet first, so nothing alerts about the server being worked on — and if the playbook can't be started the quiet is taken back.
|
||||
const started = await startOrExtendWindow("server", server.id, input.windowMinutes, `Update run: ${templateName}`, input.actor, now);
|
||||
let task: SemaphoreTask;
|
||||
try {
|
||||
task = await adapter.runTemplate(input.projectId, input.templateId, { limit: input.limit, message: `Update run for ${server.name}, started in Homelab Manager by ${input.actor} (limited to ${input.limit})` });
|
||||
} catch (err) {
|
||||
if (started && !started.extended) await db.update(maintenanceWindows).set({ endsAt: now.toISOString() }).where(eq(maintenanceWindows.id, started.window.id));
|
||||
invalidateAlertsCache();
|
||||
throw new UpdateRunError(502, "semaphore_failed", `Semaphore wouldn't start it: ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
invalidateAlertsCache();
|
||||
|
||||
const updates = parsePendingUpdates(server.pendingUpdates);
|
||||
const [run] = await db
|
||||
.insert(updateRuns)
|
||||
.values({
|
||||
serverId: server.id,
|
||||
integrationId: input.integrationId,
|
||||
projectId: input.projectId,
|
||||
templateId: input.templateId,
|
||||
templateName,
|
||||
hostLimit: input.limit,
|
||||
taskId: task.id,
|
||||
taskStatus: task.status,
|
||||
windowId: started?.window.id ?? null,
|
||||
windowCreated: started ? !started.extended : false,
|
||||
status: "running",
|
||||
message: "The playbook has been started.",
|
||||
startedBy: input.actor,
|
||||
startedAt: now.toISOString(),
|
||||
updatesBefore: updates?.count ?? null,
|
||||
})
|
||||
.returning();
|
||||
await recordAudit({ category: "update_run", action: "start", targetType: "server", targetId: server.id, detail: { run: run.id, template: templateName, limit: input.limit, task: task.id, by: input.actor } });
|
||||
return run;
|
||||
}
|
||||
|
||||
// ─── Following ──────────────────────────────────────────────────────────────
|
||||
|
||||
/** Ends the maintenance window an update run made (one it only extended belongs to whoever made it, and is left to run out). */
|
||||
async function endWindow(run: UpdateRunRow, now: Date): Promise<void> {
|
||||
if (!run.windowCreated || run.windowId === null) return;
|
||||
await db
|
||||
.update(maintenanceWindows)
|
||||
.set({ endsAt: now.toISOString() })
|
||||
.where(and(eq(maintenanceWindows.id, run.windowId)));
|
||||
invalidateAlertsCache();
|
||||
}
|
||||
|
||||
async function finish(run: UpdateRunRow, status: "done" | "failed" | "cancelled", message: string, now: Date, extra: Partial<UpdateRunRow> = {}): Promise<void> {
|
||||
await endWindow(run, now);
|
||||
await db.update(updateRuns).set({ status, message, finishedAt: now.toISOString(), ...extra }).where(eq(updateRuns.id, run.id));
|
||||
const [server] = await db.select({ name: servers.name }).from(servers).where(eq(servers.id, run.serverId)).limit(1);
|
||||
await recordAudit({ category: "update_run", action: status === "done" ? "finished" : status, targetType: "server", targetId: run.serverId, detail: { run: run.id, template: run.templateName, message } });
|
||||
if (status !== "cancelled") {
|
||||
await notifyUpdateRunFinished({ serverName: server?.name ?? `#${run.serverId}`, templateName: run.templateName, ok: status === "done", message }).catch((err) => console.error("[updateRuns] couldn't send the result:", err));
|
||||
}
|
||||
}
|
||||
|
||||
const updatesWords = (n: number) => (n === 1 ? "1 update" : `${n} updates`);
|
||||
|
||||
/** What a finished run says about the server, from the report that came after it. */
|
||||
export function summarizeOutcome(before: number | null, after: number | null, reboot: boolean): string {
|
||||
const parts: string[] = [];
|
||||
if (after !== null) {
|
||||
parts.push(after === 0 ? "No updates are waiting now" : `${updatesWords(after)} still waiting`);
|
||||
if (before !== null && before > after) parts[0] += ` (${updatesWords(before - after)} installed)`;
|
||||
} else {
|
||||
parts.push("The server reported again, but doesn't say what is waiting");
|
||||
}
|
||||
parts.push(reboot ? "a restart is still needed" : "no restart is needed");
|
||||
return `${parts.join("; ")}.`;
|
||||
}
|
||||
|
||||
async function advance(run: UpdateRunRow, now: Date): Promise<void> {
|
||||
if (now.getTime() - Date.parse(run.startedAt) > RUN_LIMIT_MS) {
|
||||
await finish(run, "failed", "It took longer than three hours, so it is no longer being followed. Check Semaphore.", now);
|
||||
return;
|
||||
}
|
||||
|
||||
if (run.status === "running") {
|
||||
const adapter = await semaphoreFor(run.integrationId);
|
||||
let task: SemaphoreTask | null = null;
|
||||
try {
|
||||
task = adapter && run.taskId !== null ? await adapter.getTask(run.projectId, run.taskId) : null;
|
||||
} catch {
|
||||
task = null;
|
||||
}
|
||||
if (!task) {
|
||||
const errors = run.pollErrors + 1;
|
||||
if (errors >= MAX_POLL_ERRORS) await finish(run, "failed", "Lost contact with Semaphore, so it is no longer being followed. Check Semaphore.", now, { pollErrors: errors });
|
||||
else await db.update(updateRuns).set({ pollErrors: errors }).where(eq(updateRuns.id, run.id));
|
||||
return;
|
||||
}
|
||||
if (!FINISHED_TASK.has(task.status)) {
|
||||
if (task.status !== run.taskStatus || run.pollErrors > 0) await db.update(updateRuns).set({ taskStatus: task.status, pollErrors: 0 }).where(eq(updateRuns.id, run.id));
|
||||
return;
|
||||
}
|
||||
if (task.status === "success") {
|
||||
const ended = task.end ? new Date(task.end).toISOString() : now.toISOString();
|
||||
await db.update(updateRuns).set({ status: "waiting_for_host", taskStatus: task.status, taskEndedAt: ended, pollErrors: 0, message: "The playbook finished. Waiting for the server to report again." }).where(eq(updateRuns.id, run.id));
|
||||
return;
|
||||
}
|
||||
const why = task.status === "error" ? "The playbook failed in Semaphore" : task.status === "rejected" ? "The run was rejected in Semaphore" : "The run was stopped in Semaphore";
|
||||
await finish(run, "failed", `${why} (run #${run.taskId}). The server is no longer under maintenance.`, now, { taskStatus: task.status });
|
||||
return;
|
||||
}
|
||||
|
||||
// waiting_for_host: the agent has reported since the playbook ended (and, where it reports updates, those are newer than the run too).
|
||||
const ended = Date.parse(run.taskEndedAt ?? run.startedAt);
|
||||
const [server] = await db.select().from(servers).where(eq(servers.id, run.serverId)).limit(1);
|
||||
if (!server) {
|
||||
await finish(run, "failed", "The server was removed.", now);
|
||||
return;
|
||||
}
|
||||
const reported = server.lastSeenAt ? Date.parse(server.lastSeenAt) > ended : false;
|
||||
const updates = parsePendingUpdates(server.pendingUpdates);
|
||||
const updatesFresh = updates ? Date.parse(updates.reportedAt) > ended : true;
|
||||
if (reported && updatesFresh) {
|
||||
await finish(run, "done", summarizeOutcome(run.updatesBefore, updates?.count ?? null, updates?.rebootRequired ?? false), now, {
|
||||
hostReportedAt: server.lastSeenAt,
|
||||
updatesAfter: updates?.count ?? null,
|
||||
rebootRequired: updates?.rebootRequired ?? false,
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (now.getTime() - ended > HOST_WAIT_MS) {
|
||||
await finish(run, "failed", `The playbook finished, but the server hasn't reported again in ${Math.round(HOST_WAIT_MS / 60_000)} minutes. Check that it came back.`, now);
|
||||
}
|
||||
}
|
||||
|
||||
/** One pass over every run that isn't over. Returns how many it looked at (for the tests). */
|
||||
export async function advanceUpdateRuns(now: Date = new Date()): Promise<number> {
|
||||
const active = await db.select().from(updateRuns).where(inArray(updateRuns.status, ACTIVE_STATUSES));
|
||||
for (const run of active) {
|
||||
try {
|
||||
await advance(run, now);
|
||||
} catch (err) {
|
||||
console.error(`[updateRuns] run ${run.id}:`, err);
|
||||
}
|
||||
}
|
||||
return active.length;
|
||||
}
|
||||
|
||||
export async function cancelUpdateRun(id: number, actor: string, now: Date = new Date()): Promise<UpdateRunRow | null> {
|
||||
const [run] = await db.select().from(updateRuns).where(eq(updateRuns.id, id)).limit(1);
|
||||
if (!run) return null;
|
||||
if (!ACTIVE_STATUSES.includes(run.status as UpdateRunStatus)) throw new UpdateRunError(409, "not_active", "That run is already over.");
|
||||
await finish(run, "cancelled", `Stopped being followed by ${actor}. A playbook already started keeps running in Semaphore.`, now);
|
||||
const [after] = await db.select().from(updateRuns).where(eq(updateRuns.id, id)).limit(1);
|
||||
return after;
|
||||
}
|
||||
|
||||
/** Recent runs, those still going first; for one server or for all. */
|
||||
export async function listUpdateRuns(options: { serverId?: number; limit?: number } = {}): Promise<(UpdateRunRow & { serverName: string | null })[]> {
|
||||
const rows = await db
|
||||
.select({ run: updateRuns, serverName: servers.name })
|
||||
.from(updateRuns)
|
||||
.leftJoin(servers, eq(servers.id, updateRuns.serverId))
|
||||
.where(options.serverId !== undefined ? eq(updateRuns.serverId, options.serverId) : undefined)
|
||||
.orderBy(desc(updateRuns.startedAt))
|
||||
.limit(options.limit ?? 30);
|
||||
const out = rows.map((r) => ({ ...r.run, serverName: r.serverName }));
|
||||
return [...out.filter((r) => ACTIVE_STATUSES.includes(r.status as UpdateRunStatus)), ...out.filter((r) => !ACTIVE_STATUSES.includes(r.status as UpdateRunStatus))];
|
||||
}
|
||||
|
||||
let timer: ReturnType<typeof setInterval> | null = null;
|
||||
|
||||
/** Follows runs that are under way, every twenty seconds — including ones that were under way when this app last stopped. */
|
||||
export function initUpdateRuns(): void {
|
||||
if (timer) return;
|
||||
timer = setInterval(() => void advanceUpdateRuns().catch((err) => console.error("[updateRuns]", err)), POLL_INTERVAL_MS);
|
||||
}
|
||||
@@ -17,6 +17,7 @@ import { notify, notifyNow } from "./notify.js";
|
||||
import { parsePendingUpdates } from "./pendingUpdates.js";
|
||||
import { computeSecretStatus } from "./secretStatus.js";
|
||||
import { disksFillingSoon, getServerMetrics } from "./serverMetrics.js";
|
||||
import { diskLimit, effectiveThresholds, parseThresholds } from "./serverThresholds.js";
|
||||
import { getSettings } from "./settingsStore.js";
|
||||
import { getUptime } from "./uptimeHistory.js";
|
||||
|
||||
@@ -68,7 +69,7 @@ export async function buildDigest(now: Date = new Date()): Promise<Digest> {
|
||||
const sections: Section[] = [];
|
||||
|
||||
const serverRows = await db
|
||||
.select({ id: servers.id, name: servers.name, lastSeenAt: servers.lastSeenAt, pendingUpdates: servers.pendingUpdates, updatesPendingSince: servers.updatesPendingSince })
|
||||
.select({ id: servers.id, name: servers.name, lastSeenAt: servers.lastSeenAt, pendingUpdates: servers.pendingUpdates, updatesPendingSince: servers.updatesPendingSince, thresholds: servers.thresholds })
|
||||
.from(servers);
|
||||
const serverName = new Map(serverRows.map((s) => [s.id, s.name]));
|
||||
|
||||
@@ -134,11 +135,14 @@ export async function buildDigest(now: Date = new Date()): Promise<Digest> {
|
||||
|
||||
// ── Disks heading for full ──
|
||||
try {
|
||||
const { diskForecastDays, diskUsagePercent } = settings.healthChecks;
|
||||
if (diskForecastDays > 0) {
|
||||
const { diskUsagePercent } = settings.healthChecks;
|
||||
{
|
||||
const lines: string[] = [];
|
||||
for (const s of serverRows) {
|
||||
for (const f of disksFillingSoon(await getServerMetrics(s.id, 14, now), diskForecastDays, diskUsagePercent)) {
|
||||
const own = parseThresholds(s.thresholds);
|
||||
const forecastDays = effectiveThresholds(settings.healthChecks, own).diskForecastDays;
|
||||
if (!(forecastDays > 0)) continue;
|
||||
for (const f of disksFillingSoon(await getServerMetrics(s.id, 14, now), forecastDays, (mount) => diskLimit(own, mount, diskUsagePercent))) {
|
||||
lines.push(`- ${s.name}: ${f.mount} is ${Math.round(f.latest)}% full and filling by about ${f.perDay.toFixed(1)} points a day — full in about ${plural(f.daysToFull, "day")}`);
|
||||
}
|
||||
}
|
||||
@@ -196,7 +200,7 @@ export async function buildDigest(now: Date = new Date()): Promise<Digest> {
|
||||
|
||||
// ── Backups ──
|
||||
try {
|
||||
const coverage = await getBackupCoverage({ now });
|
||||
const coverage = await getBackupCoverage({ now, rules: true });
|
||||
const order = { failed: 0, none: 1, stale: 2 } as const;
|
||||
const bad = coverage.guests.filter((g): g is typeof g & { status: keyof typeof order } => g.status === "failed" || g.status === "none" || g.status === "stale").sort((a, b) => order[a.status] - order[b.status] || a.name.localeCompare(b.name));
|
||||
if (bad.length > 0) {
|
||||
|
||||
@@ -0,0 +1,236 @@
|
||||
/**
|
||||
* Alert thresholds that belong to one server: how they are read and checked, what they change in the alerts, the forecast and /metrics,
|
||||
* and who may set them.
|
||||
*
|
||||
* Run with: npm test --workspace server
|
||||
*/
|
||||
import { test, describe, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { AddressInfo } from "node:net";
|
||||
import type { Server } from "node:http";
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), "hm-thresholds-"));
|
||||
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
||||
process.env.SESSION_DIR = join(dir, "sessions");
|
||||
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
||||
process.env.HEALTH_STARTUP_GRACE_MS = "0";
|
||||
|
||||
const express = (await import("express")).default;
|
||||
const { db } = await import("../src/db/client.js");
|
||||
const { users, servers, serverMetrics, auditLog } = await import("../src/db/schema.js");
|
||||
const { runMigrations } = await import("../src/db/migrate.js");
|
||||
const { parseThresholds, checkThresholds, effectiveThresholds, diskLimit } = await import("../src/services/serverThresholds.js");
|
||||
const { evaluateHealth } = await import("../src/services/healthMonitor.js");
|
||||
const { disksFillingSoon } = await import("../src/services/serverMetrics.js");
|
||||
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
|
||||
const { buildMetrics } = await import("../src/services/metrics.js");
|
||||
const { serversRouter } = await import("../src/routes/servers.js");
|
||||
const { hashToken } = await import("../src/services/tokens.js");
|
||||
const { eq } = await import("drizzle-orm");
|
||||
|
||||
await runMigrations();
|
||||
|
||||
const GLOBAL = { serverOfflineMinutes: 60, diskUsagePercent: 90, diskForecastDays: 14, backupStaleHours: 48 };
|
||||
|
||||
describe("reading and checking thresholds", () => {
|
||||
test("what is stored is read back leniently: nonsense is dropped, never an error", () => {
|
||||
assert.equal(parseThresholds(null), null);
|
||||
assert.equal(parseThresholds("not json"), null);
|
||||
assert.equal(parseThresholds("{}"), null);
|
||||
assert.deepEqual(parseThresholds(JSON.stringify({ diskUsagePercent: 95, serverOfflineMinutes: 5, diskForecastDays: "soon", mounts: { "/data": { percent: 97 }, "/x": { percent: 3 }, "/y": "no", "/z": { ignore: true } } })), {
|
||||
diskUsagePercent: 95,
|
||||
mounts: { "/data": { percent: 97 }, "/z": { ignore: true } },
|
||||
});
|
||||
});
|
||||
|
||||
test("a request is refused with a message that says which number is wrong", () => {
|
||||
assert.deepEqual(checkThresholds(null), { ok: true, value: null });
|
||||
assert.deepEqual(checkThresholds({}), { ok: true, value: null });
|
||||
assert.deepEqual(checkThresholds({ diskUsagePercent: 95, serverOfflineMinutes: null }), { ok: true, value: { diskUsagePercent: 95 } });
|
||||
const bad = (input: unknown) => {
|
||||
const r = checkThresholds(input);
|
||||
assert.equal(r.ok, false, JSON.stringify(input));
|
||||
return (r as { message: string }).message;
|
||||
};
|
||||
assert.match(bad({ diskUsagePercent: 20 }), /Disk usage alert at \(%\) must be a whole number from 50 to 99/);
|
||||
assert.match(bad({ serverOfflineMinutes: 5 }), /Offline after \(minutes\)/);
|
||||
assert.match(bad({ diskForecastDays: 1.5 }), /Disk forecast/);
|
||||
assert.match(bad({ backupStaleHours: 99999 }), /Backup older than/);
|
||||
assert.match(bad({ mounts: { "/data": { percent: 100 } } }), /limit for \/data/);
|
||||
assert.match(bad({ mounts: { "": { ignore: true } } }), /missing or too long/);
|
||||
assert.match(bad("high"), /should be an object/);
|
||||
assert.match(bad({ mounts: Object.fromEntries(Array.from({ length: 51 }, (_, i) => [`/m${i}`, { ignore: true }])) }), /At most 50/);
|
||||
});
|
||||
|
||||
test("a value of its own wins, and the rest follows Settings — including 0, which means off", () => {
|
||||
assert.deepEqual(effectiveThresholds(GLOBAL, null), GLOBAL);
|
||||
assert.deepEqual(effectiveThresholds(GLOBAL, { diskUsagePercent: 95, diskForecastDays: 0 }), { ...GLOBAL, diskUsagePercent: 95, diskForecastDays: 0 });
|
||||
});
|
||||
|
||||
test("a disk's limit: its own rule, then the server's, then Settings; or none at all", () => {
|
||||
const own = { diskUsagePercent: 95, mounts: { "/data": { percent: 80 }, "/boot": { ignore: true } } };
|
||||
assert.equal(diskLimit(own, "/data", 90), 80);
|
||||
assert.equal(diskLimit(own, "/", 90), 95);
|
||||
assert.equal(diskLimit(own, "/boot", 90), null);
|
||||
assert.equal(diskLimit(null, "/", 90), 90);
|
||||
assert.equal(diskLimit({}, "/", 90), 90);
|
||||
});
|
||||
});
|
||||
|
||||
describe("what it changes in the health rules", () => {
|
||||
const NOW = Date.parse("2026-10-17T12:00:00Z");
|
||||
const disk = (mount: string, used: number) => ({ mount, sizeBytes: 100, usedBytes: used });
|
||||
const snap = (extra: object) => ({ servers: [{ id: 1, name: "nas", lastSeenAt: new Date(NOW - 5 * 60_000).toISOString(), disks: [disk("/", 92), disk("/data", 85), disk("/boot", 99)], ...extra }], proxmox: [], synology: [] });
|
||||
const keys = (extra: object, now = NOW) => evaluateHealth(snap(extra), { serverOfflineMinutes: 60, diskUsagePercent: 90 }, now).map((c) => c.key).sort();
|
||||
|
||||
test("with nothing of its own, the global limit applies to every disk", () => {
|
||||
assert.deepEqual(keys({}), ["disk:server:1:/", "disk:server:1:/boot"]);
|
||||
});
|
||||
|
||||
test("a server's own percentage replaces the global one; a disk's own rule replaces both; an ignored disk says nothing", () => {
|
||||
assert.deepEqual(keys({ thresholds: { diskUsagePercent: 95 } }), ["disk:server:1:/boot"], "92% is fine for a server that alerts at 95");
|
||||
assert.deepEqual(keys({ thresholds: { diskUsagePercent: 80 } }), ["disk:server:1:/", "disk:server:1:/boot", "disk:server:1:/data"]);
|
||||
assert.deepEqual(keys({ thresholds: { mounts: { "/boot": { ignore: true }, "/data": { percent: 80 } } } }), ["disk:server:1:/", "disk:server:1:/data"]);
|
||||
assert.deepEqual(keys({ thresholds: { diskUsagePercent: 99, mounts: { "/": { percent: 91 }, "/boot": { ignore: true } } } }), ["disk:server:1:/"]);
|
||||
assert.deepEqual(keys({ thresholds: { diskUsagePercent: 99, mounts: { "/": { percent: 91 } } } }), ["disk:server:1:/", "disk:server:1:/boot"], "99% is at the limit");
|
||||
});
|
||||
|
||||
test("a server's own 'offline after' replaces the global one in either direction", () => {
|
||||
const quietFor = (minutes: number) => new Date(NOW - minutes * 60_000).toISOString();
|
||||
assert.deepEqual(keys({ lastSeenAt: quietFor(120), disks: [] }), ["offline:server:1"]);
|
||||
assert.deepEqual(keys({ lastSeenAt: quietFor(120), disks: [], thresholds: { serverOfflineMinutes: 300 } }), [], "a laptop that is off at night");
|
||||
assert.deepEqual(keys({ lastSeenAt: quietFor(30), disks: [], thresholds: { serverOfflineMinutes: 15 } }), ["offline:server:1"], "a host that should speak up sooner");
|
||||
});
|
||||
|
||||
test("the forecast leaves out a disk it is told to ignore, and uses the disk's own limit", () => {
|
||||
const series = [
|
||||
{ metric: "disk:/data", kind: "disk" as const, label: "Disk /data", points: [], latest: 85, trend: { perDay: 2, fit: 0.99, spanDays: 7, daysToFull: 7 } },
|
||||
{ metric: "disk:/scratch", kind: "disk" as const, label: "Disk /scratch", points: [], latest: 60, trend: { perDay: 2, fit: 0.99, spanDays: 7, daysToFull: 20 } },
|
||||
];
|
||||
assert.deepEqual(disksFillingSoon(series, 30, 90).map((f) => f.mount), ["/data", "/scratch"]);
|
||||
assert.deepEqual(disksFillingSoon(series, 30, (m) => (m === "/scratch" ? null : 90)).map((f) => f.mount), ["/data"], "told to leave /scratch alone");
|
||||
assert.deepEqual(disksFillingSoon(series, 30, (m) => (m === "/data" ? 80 : 90)).map((f) => f.mount), ["/scratch"], "/data is past its own 80%, so the full-disk alert has it");
|
||||
});
|
||||
});
|
||||
|
||||
describe("through the server routes, the alerts and /metrics", () => {
|
||||
let app: Server;
|
||||
let base = "";
|
||||
let who = "admin";
|
||||
let id = 0;
|
||||
const call = (method: string, path: string, body?: unknown) => fetch(`${base}${path}`, { method, headers: { "content-type": "application/json" }, body: body === undefined ? undefined : JSON.stringify(body) });
|
||||
const patch = (body: unknown) => call("PATCH", `/servers/${id}`, body);
|
||||
|
||||
before(async () => {
|
||||
await db.insert(users).values([
|
||||
{ oidcSub: "admin", name: "Ada", role: "admin" },
|
||||
{ oidcSub: "operator", name: "Olle", role: "operator" },
|
||||
]);
|
||||
const twoHoursAgo = new Date(Date.now() - 2 * 3600_000).toISOString();
|
||||
const [s] = await db
|
||||
.insert(servers)
|
||||
.values({ name: "laptop", apiTokenHash: hashToken("tok"), apiTokenPrefix: "t", lastSeenAt: twoHoursAgo, disks: JSON.stringify([{ mount: "/", sizeBytes: 100, usedBytes: 93 }, { mount: "/boot", sizeBytes: 100, usedBytes: 99 }]) })
|
||||
.returning();
|
||||
id = s.id;
|
||||
const a = express();
|
||||
a.use(express.json());
|
||||
a.use((req, _res, next) => {
|
||||
(req as unknown as { session: unknown }).session = { user: { sub: who } };
|
||||
next();
|
||||
});
|
||||
a.use("/servers", serversRouter);
|
||||
app = await new Promise<Server>((resolve) => {
|
||||
const x = a.listen(0, "127.0.0.1", () => resolve(x));
|
||||
});
|
||||
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}`;
|
||||
});
|
||||
after(() => {
|
||||
app.close();
|
||||
try {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Windows keeps the database file locked until the process exits.
|
||||
}
|
||||
});
|
||||
|
||||
const alertIds = async () => {
|
||||
invalidateAlertsCache();
|
||||
return (await getAlerts(true)).alerts.filter((a) => a.id.includes(`:server:${id}`)).map((a) => a.id).sort();
|
||||
};
|
||||
|
||||
test("only an administrator can set them, a wrong number is refused, and nothing changes until it is right", async () => {
|
||||
who = "operator";
|
||||
assert.equal((await patch({ thresholds: { diskUsagePercent: 95 } })).status, 403);
|
||||
who = "admin";
|
||||
const bad = await patch({ thresholds: { diskUsagePercent: 20 } });
|
||||
assert.equal(bad.status, 400);
|
||||
const body = await bad.json();
|
||||
assert.equal(body.error, "invalid_thresholds");
|
||||
assert.match(body.message, /Disk usage alert at/);
|
||||
const detail = await (await call("GET", `/servers/${id}/detail`)).json();
|
||||
assert.equal(detail.server.thresholds, null);
|
||||
assert.deepEqual(detail.thresholdDefaults.diskUsagePercent, 90, "the page can say what it would follow");
|
||||
assert.equal(detail.thresholdDefaults.backupStaleHours, 48);
|
||||
});
|
||||
|
||||
test("with none set, the laptop has been quiet for two hours, so it is offline (and an offline server's disks aren't judged)", async () => {
|
||||
assert.deepEqual(await alertIds(), [`offline:offline:server:${id}`]);
|
||||
});
|
||||
|
||||
test("the Alerts page doesn't go on showing the old answer for a minute after a server's limits change", async () => {
|
||||
await patch({ thresholds: null });
|
||||
invalidateAlertsCache();
|
||||
assert.ok((await getAlerts(false)).alerts.some((a) => a.id === `offline:offline:server:${id}`), "offline, as the cache now holds it");
|
||||
await patch({ thresholds: { serverOfflineMinutes: 600 } });
|
||||
assert.equal((await getAlerts(false)).alerts.some((a) => a.id === `offline:offline:server:${id}`), false, "no longer, straight away — without anyone clearing the cache");
|
||||
await patch({ thresholds: null });
|
||||
});
|
||||
|
||||
test("its own rules quiet exactly what they say, show on the detail, and are in the audit log", async () => {
|
||||
const res = await patch({ thresholds: { serverOfflineMinutes: 600, diskUsagePercent: 95, mounts: { "/boot": { ignore: true } } } });
|
||||
assert.equal(res.status, 200);
|
||||
assert.deepEqual((await res.json()).server.thresholds, { serverOfflineMinutes: 600, diskUsagePercent: 95, mounts: { "/boot": { ignore: true } } });
|
||||
assert.deepEqual(await alertIds(), [], "not offline for ten hours, 93% is under 95, and /boot is left alone");
|
||||
const detail = await (await call("GET", `/servers/${id}/detail`)).json();
|
||||
assert.equal(detail.server.thresholds.diskUsagePercent, 95);
|
||||
const entry = (await db.select().from(auditLog)).filter((e) => e.action === "update" && e.targetId === String(id)).pop()!;
|
||||
assert.deepEqual(JSON.parse(entry.detail!).thresholds, { serverOfflineMinutes: 600, diskUsagePercent: 95, mounts: { "/boot": { ignore: true } } });
|
||||
// the list shows them too, as an object
|
||||
const row = (await (await call("GET", "/servers")).json()).servers.find((x: { id: number }) => x.id === id);
|
||||
assert.equal(row.thresholds.serverOfflineMinutes, 600);
|
||||
});
|
||||
|
||||
test("a disk's own limit brings its alert back, and the server's own 'offline after' is what /metrics uses", async () => {
|
||||
await patch({ thresholds: { serverOfflineMinutes: 600, diskUsagePercent: 95, mounts: { "/": { percent: 90 }, "/boot": { ignore: true } } } });
|
||||
assert.deepEqual(await alertIds(), [`disk:disk:server:${id}:/`]);
|
||||
assert.match(await buildMetrics(), /homelab_server_up\{server="laptop"\} 1/, "two hours quiet is fine when it may be quiet for ten");
|
||||
await patch({ thresholds: null });
|
||||
assert.match(await buildMetrics(), /homelab_server_up\{server="laptop"\} 0/, "and offline when it follows the global hour");
|
||||
assert.equal((await (await call("GET", `/servers/${id}/detail`)).json()).server.thresholds, null, "null goes back to following Settings");
|
||||
});
|
||||
|
||||
test("the forecast follows the server's own days (0 turns it off for it) and its ignored disks", async () => {
|
||||
const day = 86_400_000;
|
||||
const rows = [];
|
||||
for (let i = 0; i < 10; i++) rows.push({ serverId: id, metric: "disk:/data", value: 30 + i * 4, recordedAt: new Date(Date.now() - (10 - i) * day).toISOString() });
|
||||
await db.insert(serverMetrics).values(rows);
|
||||
await db.update(servers).set({ lastSeenAt: new Date().toISOString() }).where(eq(servers.id, id)); // online, so its figures count
|
||||
const forecast = async () => {
|
||||
invalidateAlertsCache();
|
||||
return (await getAlerts(true)).alerts.filter((a) => a.id.includes(`disk-forecast:${id}:`)).map((a) => a.id);
|
||||
};
|
||||
assert.equal((await forecast()).length, 1, "66% and +4 a day: full in about nine days");
|
||||
await patch({ thresholds: { diskForecastDays: 0 } });
|
||||
assert.deepEqual(await forecast(), [], "0 turns it off for this server");
|
||||
await patch({ thresholds: { diskForecastDays: 60, mounts: { "/data": { ignore: true } } } });
|
||||
assert.deepEqual(await forecast(), [], "an ignored disk has no forecast either");
|
||||
await patch({ thresholds: { diskForecastDays: 60 } });
|
||||
assert.equal((await forecast()).length, 1, "a longer window still sees it");
|
||||
// and the stale-backup limit rides along, checked like the others
|
||||
assert.equal((await patch({ thresholds: { backupStaleHours: 240 } })).status, 200);
|
||||
assert.equal((await patch({ thresholds: { backupStaleHours: 5000 } })).status, 400);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,185 @@
|
||||
/**
|
||||
* Guests whose newest backup has gone too old: the limit (Settings, or a server's own), the alert, what silences it, the daily list and /metrics.
|
||||
*
|
||||
* Run with: npm test --workspace server
|
||||
*/
|
||||
import { test, describe, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), "hm-stale-"));
|
||||
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
||||
process.env.SESSION_DIR = join(dir, "sessions");
|
||||
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
||||
process.env.HEALTH_STARTUP_GRACE_MS = "0";
|
||||
|
||||
const { db } = await import("../src/db/client.js");
|
||||
const { servers, maintenanceWindows, integrations } = await import("../src/db/schema.js");
|
||||
const { runMigrations } = await import("../src/db/migrate.js");
|
||||
const { buildCoverage, staleBackups, collectStaleBackups, getBackupCoverage, setCoverageLoaderForTests } = await import("../src/services/backupCoverage.js");
|
||||
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
|
||||
const { buildMetrics } = await import("../src/services/metrics.js");
|
||||
const { updateSettings } = await import("../src/services/settingsStore.js");
|
||||
const { notifyStaleBackups } = await import("../src/services/notify.js");
|
||||
const { hashToken } = await import("../src/services/tokens.js");
|
||||
|
||||
await runMigrations();
|
||||
after(() => {
|
||||
setCoverageLoaderForTests(null);
|
||||
try {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Windows keeps the database file locked until the process exits.
|
||||
}
|
||||
});
|
||||
|
||||
const HOUR = 3600_000;
|
||||
const guest = (vmid: number, name: string) => ({ vmid, name, node: "pve1", type: "qemu" as const, status: "running", cpu: null, maxmem: null, mem: null, uptime: null });
|
||||
const allJob = { id: "j1", enabled: true, schedule: "02:00", storage: "pbs", node: null, allGuests: true, vmids: null, exclude: null };
|
||||
const px = (guests: ReturnType<typeof guest>[]) => [{ integrationId: 1, integrationName: "Felicia", guests, jobs: [allJob], tasks: [] }];
|
||||
const snapAgo = (now: number, vmid: number, hours: number) => ({ backupType: "vm", backupId: String(vmid), backupTime: Math.floor((now - hours * HOUR) / 1000), verification: "ok" as const });
|
||||
|
||||
describe("when a guest is stale", () => {
|
||||
const NOW = new Date("2026-10-17T12:00:00Z");
|
||||
const now = NOW.getTime();
|
||||
const input = (hoursAgo: number, extra: { servers?: Parameters<typeof buildCoverage>[0]["servers"]; staleHours?: number } = {}) => ({
|
||||
proxmox: px([guest(100, "web")]),
|
||||
pbs: [{ integrationName: "PBS", groups: [snapAgo(now, 100, hoursAgo)] }],
|
||||
servers: extra.servers ?? [],
|
||||
now: NOW,
|
||||
staleHours: extra.staleHours,
|
||||
});
|
||||
|
||||
test("a guest older than the limit is stale, and the limit it was judged by is on the row", () => {
|
||||
const [g] = buildCoverage(input(60));
|
||||
assert.equal(g.status, "stale");
|
||||
assert.equal(g.staleAfterHours, 48);
|
||||
assert.equal(buildCoverage(input(60, { staleHours: 72 }))[0].status, "ok");
|
||||
assert.equal(buildCoverage(input(60, { staleHours: 72 }))[0].staleAfterHours, 72);
|
||||
});
|
||||
|
||||
test("a server that is this guest can have its own limit — longer for a weekly backup, or 0 for never", () => {
|
||||
const linked = (staleHours: number | null) => [{ id: 7, name: "web", proxmoxIntegrationId: 1, proxmoxVmid: 100, staleHours }];
|
||||
assert.equal(buildCoverage(input(200, { servers: linked(240) }))[0].status, "ok", "eight days is fine for a limit of ten");
|
||||
assert.equal(buildCoverage(input(300, { servers: linked(240) }))[0].status, "stale");
|
||||
assert.equal(buildCoverage(input(5000, { servers: linked(0) }))[0].status, "ok", "0 means this one is never called stale");
|
||||
assert.equal(buildCoverage(input(200, { servers: linked(null) }))[0].status, "stale", "no limit of its own: the global one");
|
||||
// a failure is a failure whatever the limit says
|
||||
const failed = buildCoverage({ ...input(1, { servers: linked(0) }), pbs: [{ integrationName: "PBS", groups: [{ ...snapAgo(now, 100, 1), verification: "failed" as const }] }] });
|
||||
assert.equal(failed[0].status, "failed");
|
||||
});
|
||||
|
||||
test("only stale guests are listed, oldest first, critical at three times the limit", () => {
|
||||
const report = {
|
||||
guests: buildCoverage({
|
||||
proxmox: px([guest(100, "web"), guest(101, "db"), guest(102, "fresh"), guest(103, "never")]),
|
||||
pbs: [{ integrationName: "PBS", groups: [snapAgo(now, 100, 60), snapAgo(now, 101, 160), snapAgo(now, 102, 2)] }],
|
||||
servers: [],
|
||||
now: NOW,
|
||||
}),
|
||||
counts: { ok: 1, stale: 2, failed: 0, none: 0, unknown: 1, total: 4 },
|
||||
sources: [],
|
||||
staleHours: 48,
|
||||
generatedAt: NOW.toISOString(),
|
||||
};
|
||||
const stale = staleBackups(report, NOW);
|
||||
assert.deepEqual(stale.map((s) => [s.guest.name, s.severity]), [["db", "critical"], ["web", "warning"]]);
|
||||
assert.equal(stale[0].key, "backup-stale:1:101");
|
||||
assert.match(stale[0].message, /db \(#101\) \[Felicia\]: its newest backup is 6 days old \(the limit is 48 h\)/);
|
||||
assert.match(stale[1].message, /60 h old \(the limit is 48 h\)/);
|
||||
});
|
||||
|
||||
test("it is silenced by maintenance on the Proxmox integration or on the server the guest is", () => {
|
||||
const base = { guests: buildCoverage({ ...input(60), servers: [{ id: 7, name: "web", proxmoxIntegrationId: 1, proxmoxVmid: 100 }] }), counts: { ok: 0, stale: 1, failed: 0, none: 0, unknown: 0, total: 1 }, sources: [], staleHours: 48, generatedAt: NOW.toISOString() };
|
||||
assert.equal(staleBackups(base, NOW)[0].silenced, false);
|
||||
assert.equal(staleBackups(base, NOW, new Set(["integration:1"]))[0].silenced, true);
|
||||
assert.equal(staleBackups(base, NOW, new Set(["server:7"]))[0].silenced, true);
|
||||
assert.equal(staleBackups(base, NOW, new Set(["server:8", "integration:2"]))[0].silenced, false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("through the alerts, the daily list and /metrics", () => {
|
||||
before(async () => {
|
||||
// The guests are Proxmox's, read from a stand-in; the integration they belong to still has to exist for a server to be linked to one.
|
||||
await db.insert(integrations).values({ id: 1, type: "proxmox", name: "Felicia", baseUrl: "https://pve.example", enabled: false });
|
||||
});
|
||||
|
||||
// The alert pass judges by the real clock, so these backups are dated from it.
|
||||
const fixture = () => {
|
||||
const now = Date.now();
|
||||
return {
|
||||
proxmox: px([guest(100, "web"), guest(101, "db"), guest(102, "weekly"), guest(103, "fine")]).map((p) => ({ ...p })),
|
||||
pbs: [{ integrationName: "PBS", groups: [snapAgo(now, 100, 60), snapAgo(now, 101, 200), snapAgo(now, 102, 150), snapAgo(now, 103, 3)] }],
|
||||
sources: [{ kind: "proxmox" as const, name: "Felicia", ok: true }, { kind: "pbs" as const, name: "PBS", ok: true }],
|
||||
};
|
||||
};
|
||||
const staleAlerts = async () => {
|
||||
invalidateAlertsCache();
|
||||
return (await getAlerts(true)).alerts.filter((a) => a.id.startsWith("backup:backup-stale:"));
|
||||
};
|
||||
|
||||
test("each stale guest is an alert of the backup kind, with a link, a target and a severity", async () => {
|
||||
setCoverageLoaderForTests(async () => fixture());
|
||||
const alerts = await staleAlerts();
|
||||
assert.deepEqual(alerts.map((a) => a.id).sort(), ["backup:backup-stale:1:100", "backup:backup-stale:1:101", "backup:backup-stale:1:102"]);
|
||||
const db101 = alerts.find((a) => a.id.endsWith(":101"))!;
|
||||
assert.equal(db101.category, "backup");
|
||||
assert.equal(db101.severity, "critical", "200 hours is more than three times 48");
|
||||
assert.equal(alerts.find((a) => a.id.endsWith(":100"))!.severity, "warning");
|
||||
assert.equal(db101.link, "/backups");
|
||||
assert.deepEqual(db101.target, { type: "integration", id: 1, name: "Felicia" });
|
||||
assert.equal(db101.silenced, false);
|
||||
});
|
||||
|
||||
test("a server's own limit changes its guest's alert, and that server's maintenance silences it", async () => {
|
||||
const [s] = await db.insert(servers).values({ name: "weekly", apiTokenHash: hashToken("t"), apiTokenPrefix: "t", proxmoxIntegrationId: 1, proxmoxVmid: 102, thresholds: JSON.stringify({ backupStaleHours: 240 }) }).returning();
|
||||
setCoverageLoaderForTests(async () => fixture());
|
||||
let alerts = await staleAlerts();
|
||||
assert.equal(alerts.some((a) => a.id.endsWith(":102")), false, "150 hours is fine for a limit of 240");
|
||||
await db.update(servers).set({ thresholds: JSON.stringify({ backupStaleHours: 100 }) });
|
||||
alerts = await staleAlerts();
|
||||
const weekly = alerts.find((a) => a.id.endsWith(":102"))!;
|
||||
assert.ok(weekly, "150 hours is over a limit of 100");
|
||||
assert.deepEqual(weekly.target && { type: weekly.target.type, id: weekly.target.id }, { type: "server", id: s.id }, "it is the server that can be silenced");
|
||||
|
||||
await db.insert(maintenanceWindows).values({ targetType: "server", targetId: s.id, reason: "x", startedAt: new Date(Date.now() - 60_000).toISOString(), endsAt: new Date(Date.now() + 3600_000).toISOString() });
|
||||
assert.equal((await staleAlerts()).find((a) => a.id.endsWith(":102"))!.silenced, true);
|
||||
await db.delete(maintenanceWindows);
|
||||
await db.update(servers).set({ thresholds: null });
|
||||
});
|
||||
|
||||
test("the limit in Settings is what the others follow, and 0 turns the alerts off", async () => {
|
||||
setCoverageLoaderForTests(async () => fixture());
|
||||
await updateSettings({ healthChecks: { backupStaleHours: 100 } });
|
||||
assert.deepEqual((await staleAlerts()).map((a) => a.id.split(":").pop()).sort(), ["101", "102"], "60 hours is fine for a limit of 100");
|
||||
await updateSettings({ healthChecks: { backupStaleHours: 0 } });
|
||||
assert.deepEqual(await staleAlerts(), []);
|
||||
await updateSettings({ healthChecks: { backupStaleHours: 48 } });
|
||||
});
|
||||
|
||||
test("the daily list is the stale guests that aren't under maintenance", async () => {
|
||||
setCoverageLoaderForTests(async () => fixture());
|
||||
const list = await collectStaleBackups();
|
||||
assert.equal(list.length, 3);
|
||||
await db.insert(maintenanceWindows).values({ targetType: "integration", targetId: 1, reason: "x", startedAt: new Date(Date.now() - 60_000).toISOString(), endsAt: new Date(Date.now() + 3600_000).toISOString() });
|
||||
assert.deepEqual(await collectStaleBackups(), [], "the whole Proxmox integration is being worked on");
|
||||
await db.delete(maintenanceWindows);
|
||||
await notifyStaleBackups([]); // nothing to say, nothing sent, and no error with no channel set up
|
||||
await notifyStaleBackups(list);
|
||||
});
|
||||
|
||||
test("/metrics says how many guests are in each state and how old each newest backup is, from what was last read", async () => {
|
||||
setCoverageLoaderForTests(async () => fixture());
|
||||
invalidateAlertsCache();
|
||||
const text = await buildMetrics();
|
||||
assert.match(text, /homelab_backup_guests\{status="ok"\} 1/);
|
||||
assert.match(text, /homelab_backup_guests\{status="stale"\} 3/);
|
||||
const age = /homelab_backup_last_age_seconds\{guest="web",vmid="100",integration="Felicia"\} (\d+)/.exec(text);
|
||||
assert.ok(age, "an age for each guest with a backup");
|
||||
assert.ok(Math.abs(Number(age[1]) - 60 * 3600) < 120, `about 60 hours, got ${age[1]}`);
|
||||
// the page's own selector still asks for its own limit for every guest
|
||||
assert.equal((await getBackupCoverage({ staleHours: 24 * 14 })).counts.stale, 0);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,390 @@
|
||||
/**
|
||||
* An update run: the maintenance window, the playbook, following it, waiting for the server to report again, and every way it can end.
|
||||
* Semaphore is a stand-in that answers as told.
|
||||
*
|
||||
* Run with: npm test --workspace server
|
||||
*/
|
||||
import { test, describe, before, after, beforeEach } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { AddressInfo } from "node:net";
|
||||
import type { Server } from "node:http";
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), "hm-updaterun-"));
|
||||
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
||||
process.env.SESSION_DIR = join(dir, "sessions");
|
||||
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
||||
process.env.HEALTH_STARTUP_GRACE_MS = "0";
|
||||
|
||||
const express = (await import("express")).default;
|
||||
const { db } = await import("../src/db/client.js");
|
||||
const { users, servers, updateRuns, auditLog, maintenanceWindows } = await import("../src/db/schema.js");
|
||||
const { runMigrations } = await import("../src/db/migrate.js");
|
||||
const { startUpdateRun, advanceUpdateRuns, cancelUpdateRun, listUpdateRuns, setSemaphoreForTests, summarizeOutcome, UpdateRunError, HOST_WAIT_MS, MAX_POLL_ERRORS, RUN_LIMIT_MS } = await import("../src/services/updateRuns.js");
|
||||
const { listActiveWindows } = await import("../src/services/maintenance.js");
|
||||
const { getActivity } = await import("../src/services/serverActivity.js").then((m) => ({ getActivity: m.getServerActivity }));
|
||||
const { updateRunsRouter } = await import("../src/routes/updateRuns.js");
|
||||
const { hashToken } = await import("../src/services/tokens.js");
|
||||
const { eq } = await import("drizzle-orm");
|
||||
|
||||
await runMigrations();
|
||||
|
||||
// ── a Semaphore that answers as told ──
|
||||
interface Fake {
|
||||
templates: { id: number; projectId: number; projectName: string; name: string; playbook: string; lastTask: null }[];
|
||||
runs: { projectId: number; templateId: number; options?: { limit?: string; message?: string } }[];
|
||||
task: { id: number; status: string; created: string | null; start: string | null; end: string | null };
|
||||
refuseToStart: boolean;
|
||||
getTaskFails: boolean;
|
||||
}
|
||||
const fake: Fake = { templates: [], runs: [], task: { id: 41, status: "waiting", created: null, start: null, end: null }, refuseToStart: false, getTaskFails: false };
|
||||
const resetFake = () => {
|
||||
fake.templates = [{ id: 7, projectId: 2, projectName: "Homelab", name: "Apply updates", playbook: "updates.yml", lastTask: null }];
|
||||
fake.runs = [];
|
||||
fake.task = { id: 41, status: "waiting", created: null, start: null, end: null };
|
||||
fake.refuseToStart = false;
|
||||
fake.getTaskFails = false;
|
||||
};
|
||||
setSemaphoreForTests(async (integrationId) =>
|
||||
integrationId === 9
|
||||
? {
|
||||
ping: async () => ({ ok: true }),
|
||||
listTemplatesWithStatus: async () => fake.templates,
|
||||
checkTemplates: async () => ({ templates: fake.templates, failedProjectIds: [] }),
|
||||
runTemplate: async (projectId, templateId, options) => {
|
||||
if (fake.refuseToStart) throw new Error("Semaphore is having a moment");
|
||||
fake.runs.push({ projectId, templateId, options });
|
||||
return { ...fake.task } as never;
|
||||
},
|
||||
getTask: async () => {
|
||||
if (fake.getTaskFails) throw new Error("connection refused");
|
||||
return { ...fake.task } as never;
|
||||
},
|
||||
}
|
||||
: null,
|
||||
);
|
||||
|
||||
const T0 = new Date("2026-10-17T12:00:00.000Z");
|
||||
const at = (minutes: number) => new Date(T0.getTime() + minutes * 60_000);
|
||||
const input = (serverId: number, extra: object = {}) => ({ serverId, integrationId: 9, projectId: 2, templateId: 7, limit: "elsa", windowMinutes: 90, actor: "Olle", ...extra });
|
||||
const runRow = async (id: number) => (await db.select().from(updateRuns).where(eq(updateRuns.id, id)))[0];
|
||||
|
||||
describe("starting a run", () => {
|
||||
let serverId = 0;
|
||||
before(async () => {
|
||||
await db.insert(users).values([{ oidcSub: "admin", name: "Ada", role: "admin" }]);
|
||||
const [s] = await db
|
||||
.insert(servers)
|
||||
.values({ name: "elsa", apiTokenHash: hashToken("tok"), apiTokenPrefix: "t", lastSeenAt: T0.toISOString(), pendingUpdates: JSON.stringify({ manager: "apt", count: 12, security: 3, rebootRequired: false, reportedAt: T0.toISOString() }) })
|
||||
.returning();
|
||||
serverId = s.id;
|
||||
});
|
||||
beforeEach(async () => {
|
||||
resetFake();
|
||||
await db.delete(updateRuns);
|
||||
await db.delete(maintenanceWindows);
|
||||
});
|
||||
after(() => {
|
||||
try {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Windows keeps the database file locked until the process exits.
|
||||
}
|
||||
});
|
||||
|
||||
test("it puts the server under maintenance, starts the playbook limited to that server, and writes the run down", async () => {
|
||||
const run = await startUpdateRun(input(serverId), T0);
|
||||
assert.equal(run.status, "running");
|
||||
assert.equal(run.templateName, "Apply updates");
|
||||
assert.equal(run.updatesBefore, 12);
|
||||
assert.equal(run.taskId, 41);
|
||||
assert.equal(run.windowCreated, true);
|
||||
assert.deepEqual(fake.runs.map((r) => [r.projectId, r.templateId, r.options?.limit]), [[2, 7, "elsa"]]);
|
||||
assert.match(fake.runs[0].options!.message!, /Update run for elsa, started in Homelab Manager by Olle \(limited to elsa\)/);
|
||||
|
||||
const windows = await listActiveWindows(T0);
|
||||
assert.equal(windows.length, 1);
|
||||
assert.equal(windows[0].targetId, serverId);
|
||||
assert.equal(windows[0].reason, "Update run: Apply updates");
|
||||
assert.equal(Date.parse(windows[0].endsAt) - T0.getTime(), 90 * 60_000);
|
||||
assert.equal(run.windowId, windows[0].id);
|
||||
});
|
||||
|
||||
test("it won't start without something to wait for, with another run going, or on a Semaphore or template that isn't there", async () => {
|
||||
const refused = async (promise: Promise<unknown>, status: number, code: string) => {
|
||||
await assert.rejects(promise, (e: unknown) => e instanceof UpdateRunError && e.status === status && e.code === code, code);
|
||||
};
|
||||
await refused(startUpdateRun(input(99999), T0), 404, "server_not_found");
|
||||
const [never] = await db.insert(servers).values({ name: "ghost", apiTokenHash: hashToken("g"), apiTokenPrefix: "g" }).returning();
|
||||
await refused(startUpdateRun(input(never.id), T0), 400, "never_reported");
|
||||
await refused(startUpdateRun(input(serverId, { integrationId: 5 }), T0), 400, "no_semaphore");
|
||||
await refused(startUpdateRun(input(serverId, { templateId: 999 }), T0), 404, "template_not_found");
|
||||
assert.equal(fake.runs.length, 0, "nothing was started by any of those");
|
||||
assert.equal((await listActiveWindows(T0)).length, 0, "and no quiet was left behind");
|
||||
|
||||
await startUpdateRun(input(serverId), T0);
|
||||
await refused(startUpdateRun(input(serverId), at(1)), 409, "already_running");
|
||||
assert.equal(fake.runs.length, 1);
|
||||
});
|
||||
|
||||
test("if Semaphore won't start it the quiet is taken back, and no run is recorded", async () => {
|
||||
fake.refuseToStart = true;
|
||||
await assert.rejects(startUpdateRun(input(serverId), T0), (e: unknown) => e instanceof UpdateRunError && e.status === 502 && /Semaphore wouldn't start it/.test(e.message));
|
||||
assert.equal((await db.select().from(updateRuns)).length, 0);
|
||||
assert.equal((await listActiveWindows(at(1))).length, 0, "the window it made is over");
|
||||
});
|
||||
|
||||
test("a window that was already there is extended and left to its owner; it is not ended when the run is", async () => {
|
||||
await db.insert(maintenanceWindows).values({ targetType: "server", targetId: serverId, reason: "patch night", startedAt: T0.toISOString(), endsAt: at(30).toISOString(), createdBy: "Ada" });
|
||||
const run = await startUpdateRun(input(serverId), at(1));
|
||||
assert.equal(run.windowCreated, false);
|
||||
fake.task = { ...fake.task, status: "error" };
|
||||
await advanceUpdateRuns(at(5));
|
||||
assert.equal((await runRow(run.id)).status, "failed");
|
||||
assert.equal((await listActiveWindows(at(6))).length, 1, "someone else's window still stands");
|
||||
});
|
||||
});
|
||||
|
||||
describe("following a run to the end", () => {
|
||||
let serverId = 0;
|
||||
before(async () => {
|
||||
const [s] = await db.insert(servers).values({ name: "anna", apiTokenHash: hashToken("tok2"), apiTokenPrefix: "t2", lastSeenAt: T0.toISOString(), pendingUpdates: JSON.stringify({ manager: "apt", count: 12, security: 3, rebootRequired: false, reportedAt: T0.toISOString() }) }).returning();
|
||||
serverId = s.id;
|
||||
});
|
||||
beforeEach(async () => {
|
||||
resetFake();
|
||||
await db.delete(updateRuns);
|
||||
await db.delete(maintenanceWindows);
|
||||
await db.update(servers).set({ lastSeenAt: T0.toISOString(), pendingUpdates: JSON.stringify({ manager: "apt", count: 12, security: 3, rebootRequired: false, reportedAt: T0.toISOString() }) }).where(eq(servers.id, serverId));
|
||||
});
|
||||
|
||||
const reportBack = async (minutes: number, count: number, reboot: boolean) =>
|
||||
db.update(servers).set({ lastSeenAt: at(minutes).toISOString(), pendingUpdates: JSON.stringify({ manager: "apt", count, security: 0, rebootRequired: reboot, reportedAt: at(minutes).toISOString() }) }).where(eq(servers.id, serverId));
|
||||
|
||||
test("a good run: running, then waiting for the host, then done with what is left — and the window ends", async () => {
|
||||
const run = await startUpdateRun(input(serverId), T0);
|
||||
fake.task = { ...fake.task, status: "running" };
|
||||
await advanceUpdateRuns(at(1));
|
||||
let r = await runRow(run.id);
|
||||
assert.equal(r.status, "running");
|
||||
assert.equal(r.taskStatus, "running");
|
||||
|
||||
fake.task = { ...fake.task, status: "success", end: at(8).toISOString() };
|
||||
await advanceUpdateRuns(at(9));
|
||||
r = await runRow(run.id);
|
||||
assert.equal(r.status, "waiting_for_host");
|
||||
assert.equal(r.taskEndedAt, at(8).toISOString());
|
||||
assert.equal((await listActiveWindows(at(9))).length, 1, "still quiet while it comes back");
|
||||
|
||||
await advanceUpdateRuns(at(12));
|
||||
assert.equal((await runRow(run.id)).status, "waiting_for_host", "no report since the playbook ended");
|
||||
await reportBack(7, 12, false); // a report from before the playbook finished doesn't count
|
||||
await advanceUpdateRuns(at(13));
|
||||
assert.equal((await runRow(run.id)).status, "waiting_for_host");
|
||||
|
||||
await reportBack(14, 2, true);
|
||||
await advanceUpdateRuns(at(15));
|
||||
r = await runRow(run.id);
|
||||
assert.equal(r.status, "done");
|
||||
assert.equal(r.updatesBefore, 12);
|
||||
assert.equal(r.updatesAfter, 2);
|
||||
assert.equal(r.rebootRequired, true);
|
||||
assert.equal(r.message, "2 updates still waiting (10 updates installed); a restart is still needed.");
|
||||
assert.equal(r.finishedAt, at(15).toISOString());
|
||||
assert.equal((await listActiveWindows(at(16))).length, 0, "the window it made is over");
|
||||
assert.equal((await advanceUpdateRuns(at(17))), 0, "a finished run is left alone");
|
||||
});
|
||||
|
||||
test("a playbook that fails ends the run and the window", async () => {
|
||||
const run = await startUpdateRun(input(serverId), T0);
|
||||
fake.task = { ...fake.task, status: "error" };
|
||||
await advanceUpdateRuns(at(3));
|
||||
const r = await runRow(run.id);
|
||||
assert.equal(r.status, "failed");
|
||||
assert.match(r.message!, /The playbook failed in Semaphore \(run #41\)/);
|
||||
assert.equal((await listActiveWindows(at(4))).length, 0);
|
||||
for (const [status, words] of [["rejected", /rejected/], ["stopped", /stopped/]] as const) {
|
||||
await db.delete(updateRuns);
|
||||
const again = await startUpdateRun(input(serverId), at(10));
|
||||
fake.task = { ...fake.task, status };
|
||||
await advanceUpdateRuns(at(11));
|
||||
assert.match((await runRow(again.id)).message!, words);
|
||||
}
|
||||
});
|
||||
|
||||
test("a Semaphore that stops answering ends it after ten tries; one answer in between starts the count again", async () => {
|
||||
const run = await startUpdateRun(input(serverId), T0);
|
||||
fake.getTaskFails = true;
|
||||
for (let i = 0; i < MAX_POLL_ERRORS - 1; i++) await advanceUpdateRuns(at(1 + i));
|
||||
assert.equal((await runRow(run.id)).status, "running");
|
||||
assert.equal((await runRow(run.id)).pollErrors, MAX_POLL_ERRORS - 1);
|
||||
fake.getTaskFails = false;
|
||||
fake.task = { ...fake.task, status: "running" };
|
||||
await advanceUpdateRuns(at(20));
|
||||
assert.equal((await runRow(run.id)).pollErrors, 0);
|
||||
fake.getTaskFails = true;
|
||||
for (let i = 0; i < MAX_POLL_ERRORS; i++) await advanceUpdateRuns(at(21 + i));
|
||||
const r = await runRow(run.id);
|
||||
assert.equal(r.status, "failed");
|
||||
assert.match(r.message!, /Lost contact with Semaphore/);
|
||||
});
|
||||
|
||||
test("a host that doesn't come back, or a run that never ends, is given up on, and the quiet ends so the alerts can speak", async () => {
|
||||
const run = await startUpdateRun(input(serverId), T0);
|
||||
fake.task = { ...fake.task, status: "success", end: at(5).toISOString() };
|
||||
await advanceUpdateRuns(at(6));
|
||||
await advanceUpdateRuns(at(5 + HOST_WAIT_MS / 60_000 - 1));
|
||||
assert.equal((await runRow(run.id)).status, "waiting_for_host");
|
||||
await advanceUpdateRuns(at(5 + HOST_WAIT_MS / 60_000 + 1));
|
||||
const r = await runRow(run.id);
|
||||
assert.equal(r.status, "failed");
|
||||
assert.match(r.message!, /hasn't reported again in 30 minutes/);
|
||||
assert.equal((await listActiveWindows(at(40))).length, 0);
|
||||
|
||||
await db.delete(updateRuns);
|
||||
const slow = await startUpdateRun(input(serverId), at(100));
|
||||
fake.task = { ...fake.task, status: "running" };
|
||||
await advanceUpdateRuns(new Date(at(100).getTime() + RUN_LIMIT_MS + 60_000));
|
||||
assert.match((await runRow(slow.id)).message!, /longer than three hours/);
|
||||
});
|
||||
|
||||
test("a server whose agent doesn't report updates is done as soon as it reports at all", async () => {
|
||||
await db.update(servers).set({ pendingUpdates: null }).where(eq(servers.id, serverId));
|
||||
const run = await startUpdateRun(input(serverId), T0);
|
||||
fake.task = { ...fake.task, status: "success", end: at(5).toISOString() };
|
||||
await advanceUpdateRuns(at(6));
|
||||
await db.update(servers).set({ lastSeenAt: at(10).toISOString() }).where(eq(servers.id, serverId));
|
||||
await advanceUpdateRuns(at(11));
|
||||
const r = await runRow(run.id);
|
||||
assert.equal(r.status, "done");
|
||||
assert.equal(r.updatesAfter, null);
|
||||
assert.match(r.message!, /doesn't say what is waiting; no restart is needed/);
|
||||
});
|
||||
|
||||
test("it can be stopped: the window ends, the Semaphore run is left alone, and a finished run can't be stopped again", async () => {
|
||||
const run = await startUpdateRun(input(serverId), T0);
|
||||
const stopped = await cancelUpdateRun(run.id, "Ada", at(2));
|
||||
assert.equal(stopped!.status, "cancelled");
|
||||
assert.match(stopped!.message!, /Stopped being followed by Ada/);
|
||||
assert.equal((await listActiveWindows(at(3))).length, 0);
|
||||
assert.equal(fake.runs.length, 1, "nothing was sent to Semaphore to stop it");
|
||||
await assert.rejects(cancelUpdateRun(run.id, "Ada", at(3)), (e: unknown) => e instanceof UpdateRunError && e.status === 409);
|
||||
assert.equal(await cancelUpdateRun(99999, "Ada", at(3)), null);
|
||||
});
|
||||
|
||||
test("the story is on the server's history, and a run that was under way when the app stopped is picked up again", async () => {
|
||||
const run = await startUpdateRun(input(serverId), T0);
|
||||
fake.task = { ...fake.task, status: "error" };
|
||||
await advanceUpdateRuns(at(3));
|
||||
const events = await getActivity(serverId, at(5));
|
||||
const titles = events.filter((e) => /Update run/.test(e.title)).map((e) => e.title);
|
||||
// (the tests before this one wrote their own lines too, so this looks for the kinds rather than the number)
|
||||
for (const wanted of ["Update run started", "Update run didn't finish", "Update run finished", "Update run stopped being followed"]) assert.ok(titles.includes(wanted), `${wanted} in ${JSON.stringify([...new Set(titles)])}`);
|
||||
const failedLine = events.find((e) => e.title === "Update run didn't finish")!;
|
||||
assert.equal(failedLine.tone, "bad");
|
||||
assert.match(failedLine.detail!, /The playbook failed in Semaphore/);
|
||||
assert.ok((await db.select().from(auditLog)).some((a) => a.category === "update_run" && a.action === "failed"));
|
||||
|
||||
// nothing is held in memory: a fresh pass over the table is all it takes
|
||||
await db.delete(updateRuns);
|
||||
const survivor = await startUpdateRun(input(serverId), at(20));
|
||||
fake.task = { ...fake.task, status: "success", end: at(22).toISOString() };
|
||||
assert.equal(await advanceUpdateRuns(at(23)), 1);
|
||||
assert.equal((await runRow(survivor.id)).status, "waiting_for_host");
|
||||
void run;
|
||||
});
|
||||
|
||||
test("the outcome is worded plainly for each case", () => {
|
||||
assert.equal(summarizeOutcome(5, 0, false), "No updates are waiting now (5 updates installed); no restart is needed.");
|
||||
assert.equal(summarizeOutcome(1, 0, true), "No updates are waiting now (1 update installed); a restart is still needed.");
|
||||
assert.equal(summarizeOutcome(null, 3, false), "3 updates still waiting; no restart is needed.");
|
||||
assert.equal(summarizeOutcome(3, 3, false), "3 updates still waiting; no restart is needed.");
|
||||
assert.equal(summarizeOutcome(3, null, true), "The server reported again, but doesn't say what is waiting; a restart is still needed.");
|
||||
});
|
||||
});
|
||||
|
||||
describe("the routes", () => {
|
||||
let app: Server;
|
||||
let base = "";
|
||||
let who = "operator";
|
||||
let serverId = 0;
|
||||
const call = (method: string, path: string, body?: unknown) => fetch(`${base}${path}`, { method, headers: { "content-type": "application/json" }, body: body === undefined ? undefined : JSON.stringify(body) });
|
||||
|
||||
before(async () => {
|
||||
await db.insert(users).values([
|
||||
{ oidcSub: "operator", name: "Olle", role: "operator" },
|
||||
{ oidcSub: "viewer", name: "Vera", role: "viewer" },
|
||||
]);
|
||||
const [s] = await db.insert(servers).values({ name: "zoe", apiTokenHash: hashToken("tok3"), apiTokenPrefix: "t3", lastSeenAt: T0.toISOString() }).returning();
|
||||
serverId = s.id;
|
||||
const a = express();
|
||||
a.use(express.json());
|
||||
a.use((req, _res, next) => {
|
||||
(req as unknown as { session: unknown }).session = { user: { sub: who } };
|
||||
next();
|
||||
});
|
||||
a.use("/api/update-runs", updateRunsRouter);
|
||||
app = await new Promise<Server>((resolve) => {
|
||||
const x = a.listen(0, "127.0.0.1", () => resolve(x));
|
||||
});
|
||||
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/update-runs`;
|
||||
});
|
||||
beforeEach(async () => {
|
||||
resetFake();
|
||||
await db.delete(updateRuns);
|
||||
await db.delete(maintenanceWindows);
|
||||
});
|
||||
after(() => app.close());
|
||||
|
||||
const body = (extra: object = {}) => ({ serverId, integrationId: 9, projectId: 2, templateId: 7, limit: "zoe", windowMinutes: 60, ...extra });
|
||||
|
||||
test("an operator starts one and everyone signed in can see it; a viewer can't start or stop one", async () => {
|
||||
who = "viewer";
|
||||
assert.equal((await call("POST", "/", body())).status, 403);
|
||||
who = "operator";
|
||||
const res = await call("POST", "/", body());
|
||||
assert.equal(res.status, 201);
|
||||
const { run } = await res.json();
|
||||
assert.equal(run.status, "running");
|
||||
assert.equal(run.active, true);
|
||||
assert.equal(run.serverName, "zoe");
|
||||
assert.equal(run.template, "Apply updates");
|
||||
assert.equal(run.startedBy, "Olle");
|
||||
|
||||
who = "viewer";
|
||||
const list = await (await call("GET", "/")).json();
|
||||
assert.equal(list.runs.length, 1);
|
||||
assert.equal(list.runs[0].id, run.id);
|
||||
assert.equal((await (await call("GET", `/?serverId=${serverId}`)).json()).runs.length, 1);
|
||||
assert.equal((await (await call("GET", "/?serverId=999")).json()).runs.length, 0);
|
||||
assert.equal((await call("POST", `/${run.id}/cancel`)).status, 403);
|
||||
|
||||
who = "operator";
|
||||
assert.equal((await call("POST", `/${run.id}/cancel`)).status, 200);
|
||||
assert.equal((await call("POST", `/${run.id}/cancel`)).status, 409, "already over");
|
||||
assert.equal((await call("POST", "/99999/cancel")).status, 404);
|
||||
});
|
||||
|
||||
test("what can't be done says why: a second run, a bad limit, a missing field, Semaphore refusing", async () => {
|
||||
assert.equal((await call("POST", "/", body())).status, 201);
|
||||
const again = await call("POST", "/", body());
|
||||
assert.equal(again.status, 409);
|
||||
assert.equal((await again.json()).error, "already_running");
|
||||
await db.delete(updateRuns);
|
||||
|
||||
for (const bad of [body({ limit: "" }), body({ limit: "--extra-vars x=1" }), body({ limit: "a b" }), body({ windowMinutes: 5 }), body({ windowMinutes: 9999 }), { serverId }]) {
|
||||
assert.equal((await call("POST", "/", bad)).status, 400, JSON.stringify(bad));
|
||||
}
|
||||
const limitReply = await (await call("POST", "/", body({ limit: "--extra-vars x=1" }))).json();
|
||||
assert.match(limitReply.message, /host name as it is in the Semaphore inventory/);
|
||||
assert.equal(fake.runs.length, 1, "only the one that was fine reached Semaphore");
|
||||
|
||||
fake.refuseToStart = true;
|
||||
const refused = await call("POST", "/", body());
|
||||
assert.equal(refused.status, 502);
|
||||
assert.match((await refused.json()).message, /Semaphore wouldn't start it/);
|
||||
assert.equal((await listUpdateRuns()).length, 0);
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user