Stale-backup alerts, per-server alert limits, and update runs from the Updates page
- Backups: a guest whose newest backup is older than its limit is an alert (backup kind), in the daily backup message, the digest and /metrics (homelab_backup_guests, ..._last_age_seconds) - Per-server thresholds (servers.thresholds, migration 0030): offline minutes, disk percent, forecast window, backup age, and per-disk limits or "leave alone"; applied everywhere the global ones were; saving clears the alerts cache - Update runs (update_runs, migration 0031): Update... on the Updates page puts the server under maintenance, runs a Semaphore playbook limited to it, waits for the host to report, and reports what is left and whether a restart is due; poller survives restarts - Notification toggle for update-run results; tests and docs Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
5dc6cea69c
commit
7a5b623444
42 files changed
+7561
-57
No files matched your search
@@ -9,6 +9,7 @@ import { recordAudit } from "../services/audit.js";
|
||||
import { loadParentRows, resolveParents, wouldLoop } from "../services/serverParents.js";
|
||||
import { parsePendingUpdates } from "../services/pendingUpdates.js";
|
||||
import { parseHostHealth } from "../services/hostHealth.js";
|
||||
import { checkThresholds, parseThresholds } from "../services/serverThresholds.js";
|
||||
import { latestAgentVersions } from "../services/agentVersions.js";
|
||||
import { getServerMetrics, RETENTION_DAYS } from "../services/serverMetrics.js";
|
||||
import { getServerActivity } from "../services/serverActivity.js";
|
||||
@@ -20,14 +21,15 @@ import { serverPortsRouter } from "./serverPorts.js";
|
||||
import { InvalidTagError, normalizeTags, parseTags } from "../services/serverTags.js";
|
||||
import { buildServerSummary } from "../services/serverSummary.js";
|
||||
import { activeSubjects } from "../services/maintenance.js";
|
||||
import { invalidateAlertsCache } from "../services/alerts.js";
|
||||
import { getSettings, updateSettings } from "../services/settingsStore.js";
|
||||
|
||||
export const serversRouter = Router();
|
||||
|
||||
/** A server row as the API returns it: no token hash, and tags as an array rather than the stored JSON. */
|
||||
function publicServer<T extends { apiTokenHash: string; tags: string | null; pendingUpdates: string | null; hostHealth: string | null }>(row: T) {
|
||||
const { apiTokenHash: _hash, tags, pendingUpdates, hostHealth, ...rest } = row;
|
||||
return { ...rest, tags: parseTags(tags), pendingUpdates: parsePendingUpdates(pendingUpdates), hostHealth: parseHostHealth(hostHealth) };
|
||||
function publicServer<T extends { apiTokenHash: string; tags: string | null; pendingUpdates: string | null; hostHealth: string | null; thresholds: string | null }>(row: T) {
|
||||
const { apiTokenHash: _hash, tags, pendingUpdates, hostHealth, thresholds, ...rest } = row;
|
||||
return { ...rest, tags: parseTags(tags), pendingUpdates: parsePendingUpdates(pendingUpdates), hostHealth: parseHostHealth(hostHealth), thresholds: parseThresholds(thresholds) };
|
||||
}
|
||||
|
||||
serversRouter.use(requireAuth);
|
||||
@@ -135,6 +137,8 @@ const updateServerSchema = z
|
||||
parentServerId: z.number().int().nullable().optional(),
|
||||
// True says it runs on nothing, which also stops the Proxmox-node guess. Choosing a server, or null, turns it off again.
|
||||
noParent: z.boolean().optional(),
|
||||
// This server's own alert thresholds (null goes back to following Settings). Checked on its own below, so a wrong number gets a message that says which.
|
||||
thresholds: z.unknown().optional(),
|
||||
})
|
||||
.refine(
|
||||
(data) => {
|
||||
@@ -181,10 +185,20 @@ serversRouter.patch("/:id", requireRole("admin"), asyncHandler(async (req, res)
|
||||
}
|
||||
}
|
||||
|
||||
const changes: typeof parsed.data = { ...parsed.data };
|
||||
const { thresholds: requestedThresholds, ...fields } = parsed.data;
|
||||
const changes: typeof fields & { thresholds?: string | null } = { ...fields };
|
||||
if (changes.noParent === true) changes.parentServerId = null;
|
||||
else if (changes.parentServerId !== undefined) changes.noParent = false;
|
||||
let thresholdsChange: unknown;
|
||||
if (requestedThresholds !== undefined) {
|
||||
const checked = checkThresholds(requestedThresholds);
|
||||
if (!checked.ok) return res.status(400).json({ error: "invalid_thresholds", message: checked.message });
|
||||
changes.thresholds = checked.value ? JSON.stringify(checked.value) : null;
|
||||
thresholdsChange = checked.value;
|
||||
}
|
||||
const [updated] = await db.update(servers).set(changes).where(eq(servers.id, id)).returning();
|
||||
// A server's own limits (or the server it runs on) change what is an alert, so the Alerts page shouldn't go on showing the old answer for up to a minute.
|
||||
if (requestedThresholds !== undefined || fields.parentServerId !== undefined || fields.noParent !== undefined) invalidateAlertsCache();
|
||||
|
||||
await recordAudit({
|
||||
actor: req.currentUser!,
|
||||
@@ -192,7 +206,7 @@ serversRouter.patch("/:id", requireRole("admin"), asyncHandler(async (req, res)
|
||||
action: "update",
|
||||
targetType: "server",
|
||||
targetId: id,
|
||||
detail: { name: updated.name, ...(parsed.data.parentServerId !== undefined ? { parentServerId: parsed.data.parentServerId } : {}), ...(parsed.data.noParent !== undefined ? { noParent: parsed.data.noParent } : {}) },
|
||||
detail: { name: updated.name, ...(parsed.data.parentServerId !== undefined ? { parentServerId: parsed.data.parentServerId } : {}), ...(parsed.data.noParent !== undefined ? { noParent: parsed.data.noParent } : {}), ...(requestedThresholds !== undefined ? { thresholds: thresholdsChange } : {}) },
|
||||
});
|
||||
|
||||
const serverOut = publicServer(updated);
|
||||
@@ -203,7 +217,7 @@ serversRouter.patch("/:id", requireRole("admin"), asyncHandler(async (req, res)
|
||||
// (which viewers can't read), so the widget and the alerts always agree about what "offline" and "full" mean.
|
||||
serversRouter.get("/summary", asyncHandler(async (_req, res) => {
|
||||
const rows = await db
|
||||
.select({ id: servers.id, name: servers.name, osType: servers.osType, lastSeenAt: servers.lastSeenAt, disks: servers.disks })
|
||||
.select({ id: servers.id, name: servers.name, osType: servers.osType, lastSeenAt: servers.lastSeenAt, disks: servers.disks, thresholds: servers.thresholds })
|
||||
.from(servers);
|
||||
const { healthChecks } = await getSettings();
|
||||
res.json(buildServerSummary(rows, healthChecks, Date.now(), await activeSubjects()));
|
||||
@@ -367,6 +381,7 @@ serversRouter.get("/:id/detail", asyncHandler(async (req, res) => {
|
||||
lastPortScan: _lastPortScan,
|
||||
pendingUpdates: rawUpdates,
|
||||
hostHealth: rawHealth,
|
||||
thresholds: rawThresholds,
|
||||
...serverOut
|
||||
} = server;
|
||||
|
||||
@@ -380,7 +395,7 @@ serversRouter.get("/:id/detail", asyncHandler(async (req, res) => {
|
||||
children: [...links2.entries()].filter(([, l]) => l.parentId === id).map(([childId, l]) => ({ id: childId, name: nameOf.get(childId) ?? "", auto: l.auto })).sort((a, b) => a.name.localeCompare(b.name)),
|
||||
};
|
||||
|
||||
res.json({ server: { ...serverOut, tags: parseTags(rawTags), pendingUpdates: parsePendingUpdates(rawUpdates), hostHealth: parseHostHealth(rawHealth) }, ipAddresses, hardware, dnsMatches, links, relations, agentLatest: latestAgentVersions() });
|
||||
res.json({ server: { ...serverOut, tags: parseTags(rawTags), pendingUpdates: parsePendingUpdates(rawUpdates), hostHealth: parseHostHealth(rawHealth), thresholds: parseThresholds(rawThresholds) }, thresholdDefaults: (await getSettings()).healthChecks, ipAddresses, hardware, dnsMatches, links, relations, agentLatest: latestAgentVersions() });
|
||||
}));
|
||||
|
||||
const tagsSchema = z.object({ tags: z.array(z.string().max(100)).max(50) });
|
||||
|
||||
@@ -76,6 +76,7 @@ const updateSchema = z.object({
|
||||
pbsVerificationCheck: z.boolean(),
|
||||
healthAlerts: z.boolean(),
|
||||
automationAlerts: z.boolean(),
|
||||
updateRunResults: z.boolean(),
|
||||
domainExpiryCheck: z.boolean(),
|
||||
secretCheckTime: z.string().regex(/^\d{2}:\d{2}$/),
|
||||
timezone: z.string(),
|
||||
@@ -95,7 +96,7 @@ const updateSchema = z.object({
|
||||
.partial()
|
||||
.optional(),
|
||||
healthChecks: z
|
||||
.object({ serverOfflineMinutes: z.number().int().min(15).max(10080), diskUsagePercent: z.number().int().min(50).max(99), domainWarnDays: z.number().int().min(1).max(365), diskForecastDays: z.number().int().min(0).max(90) })
|
||||
.object({ serverOfflineMinutes: z.number().int().min(15).max(10080), diskUsagePercent: z.number().int().min(50).max(99), domainWarnDays: z.number().int().min(1).max(365), diskForecastDays: z.number().int().min(0).max(90), backupStaleHours: z.number().int().min(0).max(2160) })
|
||||
.partial()
|
||||
.optional(),
|
||||
quietHours: z
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
import { Router } from "express";
|
||||
import { z } from "zod";
|
||||
import { requireAuth, requireRole } from "../auth/middleware.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
import { ACTIVE_STATUSES, cancelUpdateRun, listUpdateRuns, startUpdateRun, UpdateRunError } from "../services/updateRuns.js";
|
||||
|
||||
export const updateRunsRouter = Router();
|
||||
|
||||
// Anyone signed in can see how updates are going, like the Updates page itself. Starting or stopping one is for operators.
|
||||
updateRunsRouter.use(requireAuth);
|
||||
|
||||
// Which hosts the playbook may touch: names, addresses and ansible patterns — and nothing that could be taken for an option. The same rule the
|
||||
// Semaphore page uses, except that it may not be left out: an update run is for one server.
|
||||
const startSchema = z.object({
|
||||
serverId: z.number().int().positive(),
|
||||
integrationId: z.number().int().positive(),
|
||||
projectId: z.number().int().positive(),
|
||||
templateId: z.number().int().positive(),
|
||||
limit: z.string().trim().regex(/^[A-Za-z0-9][A-Za-z0-9._,:*\-]{0,199}$/, "Use the host name as it is in the Semaphore inventory, like web01"),
|
||||
windowMinutes: z.number().int().min(15).max(480).default(90),
|
||||
});
|
||||
|
||||
const shape = (r: Awaited<ReturnType<typeof listUpdateRuns>>[number]) => ({
|
||||
id: r.id,
|
||||
serverId: r.serverId,
|
||||
serverName: r.serverName,
|
||||
integrationId: r.integrationId,
|
||||
template: r.templateName,
|
||||
limit: r.hostLimit,
|
||||
taskId: r.taskId,
|
||||
taskStatus: r.taskStatus,
|
||||
status: r.status,
|
||||
active: ACTIVE_STATUSES.includes(r.status as (typeof ACTIVE_STATUSES)[number]),
|
||||
message: r.message,
|
||||
startedBy: r.startedBy,
|
||||
startedAt: r.startedAt,
|
||||
taskEndedAt: r.taskEndedAt,
|
||||
finishedAt: r.finishedAt,
|
||||
updatesBefore: r.updatesBefore,
|
||||
updatesAfter: r.updatesAfter,
|
||||
rebootRequired: r.rebootRequired,
|
||||
});
|
||||
|
||||
updateRunsRouter.get("/", asyncHandler(async (req, res) => {
|
||||
const serverId = Number(req.query.serverId);
|
||||
const runs = await listUpdateRuns({ serverId: Number.isInteger(serverId) && serverId > 0 ? serverId : undefined, limit: 30 });
|
||||
res.json({ runs: runs.map(shape) });
|
||||
}));
|
||||
|
||||
updateRunsRouter.post("/", requireRole("operator"), asyncHandler(async (req, res) => {
|
||||
const parsed = startSchema.safeParse(req.body);
|
||||
if (!parsed.success) {
|
||||
return res.status(400).json({ error: "invalid_body", message: parsed.error.issues[0]?.message ?? "Choose a server, a template and the host name it runs on.", details: parsed.error.flatten() });
|
||||
}
|
||||
const actor = req.currentUser!;
|
||||
try {
|
||||
const run = await startUpdateRun({ ...parsed.data, actor: actor.name ?? actor.email ?? actor.oidcSub });
|
||||
const [shaped] = (await listUpdateRuns({ serverId: run.serverId, limit: 1 })).map(shape);
|
||||
res.status(201).json({ run: shaped });
|
||||
} catch (err) {
|
||||
if (err instanceof UpdateRunError) return res.status(err.status).json({ error: err.code, message: err.message });
|
||||
throw err;
|
||||
}
|
||||
}));
|
||||
|
||||
updateRunsRouter.post("/:id/cancel", requireRole("operator"), asyncHandler(async (req, res) => {
|
||||
const id = Number(req.params.id);
|
||||
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
|
||||
const actor = req.currentUser!;
|
||||
try {
|
||||
const run = await cancelUpdateRun(id, actor.name ?? actor.email ?? actor.oidcSub);
|
||||
if (!run) return res.status(404).json({ error: "not_found" });
|
||||
res.json({ status: run.status });
|
||||
} catch (err) {
|
||||
if (err instanceof UpdateRunError) return res.status(err.status).json({ error: err.code, message: err.message });
|
||||
throw err;
|
||||
}
|
||||
}));
|
||||
Reference in new issue
Block a user